From 3b354016e9e23edc28bd4ca78f8714fdb006760e Mon Sep 17 00:00:00 2001
From: Igor Saprykin <isaprykin@google.com>
Date: Wed, 4 Oct 2017 12:47:05 -0700
Subject: [PATCH 001/909] Rename SavedModelExporter to LatestExporter.

PiperOrigin-RevId: 171048345
---
 tensorflow/python/estimator/exporter.py      |  2 +-
 tensorflow/python/estimator/exporter_test.py | 15 ++++++++-------
 2 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/tensorflow/python/estimator/exporter.py b/tensorflow/python/estimator/exporter.py
index 621dece119..505820dd93 100644
--- a/tensorflow/python/estimator/exporter.py
+++ b/tensorflow/python/estimator/exporter.py
@@ -55,7 +55,7 @@ class Exporter(object):
     pass
 
 
-class SavedModelExporter(Exporter):
+class LatestExporter(Exporter):
   """This class exports the serving graph and checkpoints.
 
   In addition, the class also garbage collects stale exports.
diff --git a/tensorflow/python/estimator/exporter_test.py b/tensorflow/python/estimator/exporter_test.py
index 106202c9c2..2ceff1bfd6 100644
--- a/tensorflow/python/estimator/exporter_test.py
+++ b/tensorflow/python/estimator/exporter_test.py
@@ -30,14 +30,15 @@ from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.util import compat
 
 
-class SavedModelExporterTest(test.TestCase):
+class LatestExporterTest(test.TestCase):
 
   def test_error_out_if_exports_to_keep_is_zero(self):
     def _serving_input_fn():
       pass
+
     with self.assertRaisesRegexp(ValueError, "positive number"):
-      exporter_lib.SavedModelExporter(
-          name="saved_model_exporter",
+      exporter_lib.LatestExporter(
+          name="latest_exporter",
           serving_input_fn=_serving_input_fn,
           exports_to_keep=0)
 
@@ -49,8 +50,8 @@ class SavedModelExporterTest(test.TestCase):
     export_dir_base = tempfile.mkdtemp() + "export/"
     gfile.MkDir(export_dir_base)
 
-    exporter = exporter_lib.SavedModelExporter(
-        name="saved_model_exporter",
+    exporter = exporter_lib.LatestExporter(
+        name="latest_exporter",
         serving_input_fn=_serving_input_fn,
         assets_extra={"from/path": "to/path"},
         as_text=False,
@@ -85,8 +86,8 @@ class SavedModelExporterTest(test.TestCase):
     def _serving_input_fn():
       return array_ops.constant([1]), None
 
-    exporter = exporter_lib.SavedModelExporter(
-        name="saved_model_exporter",
+    exporter = exporter_lib.LatestExporter(
+        name="latest_exporter",
         serving_input_fn=_serving_input_fn,
         exports_to_keep=2)
     estimator = test.mock.Mock(spec=estimator_lib.Estimator)
-- 
GitLab


From 491584ff4dce4888227fc4227f81ffca12942534 Mon Sep 17 00:00:00 2001
From: Asim Shankar <ashankar@google.com>
Date: Wed, 4 Oct 2017 12:48:27 -0700
Subject: [PATCH 002/909] eager: Always run dataset iterator operations on CPU.

It has no kernels for other devices.
With an explicit "tf.device()" before invoking the kernel we ensure
that Iterator.next() functions even when placed inside a:

with tf.device("/device:GPU:0")

PiperOrigin-RevId: 171048558
---
 tensorflow/contrib/eager/python/datasets.py | 39 ++++++++++++---------
 1 file changed, 23 insertions(+), 16 deletions(-)

diff --git a/tensorflow/contrib/eager/python/datasets.py b/tensorflow/contrib/eager/python/datasets.py
index 9973f4eee2..fb9fabd6c1 100644
--- a/tensorflow/contrib/eager/python/datasets.py
+++ b/tensorflow/contrib/eager/python/datasets.py
@@ -23,6 +23,7 @@ import threading
 from tensorflow.python.data.util import nest
 from tensorflow.python.eager import context
 from tensorflow.python.framework import errors
+from tensorflow.python.framework import ops
 from tensorflow.python.ops import gen_dataset_ops
 from tensorflow.python.ops import resource_variable_ops
 
@@ -62,20 +63,22 @@ class Iterator(object):
       raise RuntimeError(
           "{} objects only make sense when eager execution is enabled".format(
               type(self)))
-    ds_variant = dataset._as_variant_tensor()  # pylint: disable=protected-access
-    self._output_types = dataset.output_types
-    self._flat_output_types = nest.flatten(dataset.output_types)
-    self._flat_output_shapes = nest.flatten(dataset.output_shapes)
-    self._resource = gen_dataset_ops.iterator(
-        container="",
-        shared_name=_iterator_shared_name(),
-        output_types=self._flat_output_types,
-        output_shapes=self._flat_output_shapes)
-    gen_dataset_ops.make_iterator(ds_variant, self._resource)
+    with ops.device("/device:CPU:0"):
+      ds_variant = dataset._as_variant_tensor()  # pylint: disable=protected-access
+      self._output_types = dataset.output_types
+      self._flat_output_types = nest.flatten(dataset.output_types)
+      self._flat_output_shapes = nest.flatten(dataset.output_shapes)
+      self._resource = gen_dataset_ops.iterator(
+          container="",
+          shared_name=_iterator_shared_name(),
+          output_types=self._flat_output_types,
+          output_shapes=self._flat_output_shapes)
+      gen_dataset_ops.make_iterator(ds_variant, self._resource)
 
   def __del__(self):
     if self._resource is not None:
-      resource_variable_ops.destroy_resource_op(self._resource)
+      with ops.device("/device:CPU:0"):
+        resource_variable_ops.destroy_resource_op(self._resource)
     self._resource = None
 
   def __iter__(self):
@@ -87,10 +90,14 @@ class Iterator(object):
   def next(self):
     """Return the next tf.Tensor from the dataset."""
     try:
-      ret = gen_dataset_ops.iterator_get_next(
-          self._resource,
-          output_types=self._flat_output_types,
-          output_shapes=self._flat_output_shapes)
-      return nest.pack_sequence_as(self._output_types, ret)
+      # TODO(ashankar): Consider removing this ops.device() contextmanager
+      # and instead mimic ops placement in graphs: Operations on resource
+      # handles execute on the same device as where the resource is placed.
+      with ops.device("/device:CPU:0"):
+        ret = gen_dataset_ops.iterator_get_next(
+            self._resource,
+            output_types=self._flat_output_types,
+            output_shapes=self._flat_output_shapes)
+        return nest.pack_sequence_as(self._output_types, ret)
     except errors.OutOfRangeError:
       raise StopIteration
-- 
GitLab


From cf17ec96ed987386d73c645cd8b44aa32b7568b1 Mon Sep 17 00:00:00 2001
From: Yangzihao Wang <yangzihao@google.com>
Date: Wed, 4 Oct 2017 12:50:36 -0700
Subject: [PATCH 003/909] Add V2 versions of output window size computation
 functions for convolution. These V2 versions take arbitrary dilation rates.
 In preparation for the support of native cudnn dilated convolution.

PiperOrigin-RevId: 171048878
---
 tensorflow/core/framework/common_shape_fns.cc | 100 +++++++++++++++---
 tensorflow/core/framework/common_shape_fns.h  |  56 +++++++++-
 tensorflow/core/kernels/conv_grad_ops.cc      |  79 ++++++++++----
 tensorflow/core/kernels/conv_grad_ops.h       |   8 ++
 4 files changed, 204 insertions(+), 39 deletions(-)

diff --git a/tensorflow/core/framework/common_shape_fns.cc b/tensorflow/core/framework/common_shape_fns.cc
index 92f9fd451b..4796c3c00a 100644
--- a/tensorflow/core/framework/common_shape_fns.cc
+++ b/tensorflow/core/framework/common_shape_fns.cc
@@ -17,24 +17,31 @@ limitations under the License.
 
 namespace tensorflow {
 
-Status GetWindowedOutputSizeVerbose(int64 input_size, int64 filter_size,
-                                    int64 stride, Padding padding_type,
-                                    int64* output_size, int64* padding_before,
-                                    int64* padding_after) {
+Status GetWindowedOutputSizeVerboseV2(int64 input_size, int64 filter_size,
+                                      int64 dilation_rate, int64 stride,
+                                      Padding padding_type, int64* output_size,
+                                      int64* padding_before,
+                                      int64* padding_after) {
   if (stride <= 0) {
     return errors::InvalidArgument("Stride must be > 0, but got ", stride);
   }
+  if (dilation_rate < 1) {
+    return errors::InvalidArgument("Dilation rate must be >= 1, but got ",
+                                   dilation_rate);
+  }
 
-  // See also the parallel implementation in GetWindowedOutputSizeFromDims.
+  // See also the parallel implementation in GetWindowedOutputSizeFromDimsV2.
+  int64 effective_filter_size = (filter_size - 1) * dilation_rate + 1;
   switch (padding_type) {
     case Padding::VALID:
-      *output_size = (input_size - filter_size + stride) / stride;
+      *output_size = (input_size - effective_filter_size + stride) / stride;
       *padding_before = *padding_after = 0;
       break;
     case Padding::SAME:
       *output_size = (input_size + stride - 1) / stride;
       const int64 padding_needed =
-          std::max(0LL, (*output_size - 1) * stride + filter_size - input_size);
+          std::max(0LL, (*output_size - 1) * stride + effective_filter_size -
+                            input_size);
       // For odd values of total padding, add more padding at the 'right'
       // side of the given dimension.
       *padding_before = padding_needed / 2;
@@ -47,15 +54,35 @@ Status GetWindowedOutputSizeVerbose(int64 input_size, int64 filter_size,
   return Status::OK();
 }
 
+Status GetWindowedOutputSizeVerbose(int64 input_size, int64 filter_size,
+                                    int64 stride, Padding padding_type,
+                                    int64* output_size, int64* padding_before,
+                                    int64* padding_after) {
+  return GetWindowedOutputSizeVerboseV2(input_size, filter_size,
+                                        /*dilation_rate=*/1, stride,
+                                        padding_type, output_size,
+                                        padding_before, padding_after);
+}
+
 Status GetWindowedOutputSize(int64 input_size, int64 filter_size, int64 stride,
                              Padding padding_type, int64* output_size,
-                             int64* padding) {
+                             int64* padding_size) {
   int64 padding_after_unused;
   return GetWindowedOutputSizeVerbose(input_size, filter_size, stride,
-                                      padding_type, output_size, padding,
+                                      padding_type, output_size, padding_size,
                                       &padding_after_unused);
 }
 
+Status GetWindowedOutputSizeV2(int64 input_size, int64 filter_size,
+                               int64 dilation_rate, int64 stride,
+                               Padding padding_type, int64* output_size,
+                               int64* padding_size) {
+  int64 padding_after_unused;
+  return GetWindowedOutputSizeVerboseV2(input_size, filter_size, dilation_rate,
+                                        stride, padding_type, output_size,
+                                        padding_size, &padding_after_unused);
+}
+
 Status Get3dOutputSize(const std::array<int64, 3>& input,
                        const std::array<int64, 3>& window,
                        const std::array<int64, 3>& strides,
@@ -69,34 +96,77 @@ Status Get3dOutputSize(const std::array<int64, 3>& input,
   return Status::OK();
 }
 
+Status Get3dOutputSizeV2(const std::array<int64, 3>& input,
+                         const std::array<int64, 3>& window,
+                         const std::array<int64, 3>& dilations,
+                         const std::array<int64, 3>& strides,
+                         Padding padding_type, std::array<int64, 3>* output_ptr,
+                         std::array<int64, 3>* padding_ptr) {
+  for (size_t i = 0; i < input.size(); ++i) {
+    TF_RETURN_IF_ERROR(GetWindowedOutputSizeV2(
+        input[i], window[i], dilations[i], strides[i], padding_type,
+        &(*output_ptr)[i], &(*padding_ptr)[i]));
+  }
+  return Status::OK();
+}
+
 namespace shape_inference {
 
-Status GetWindowedOutputSizeFromDims(
+// The V2 version computes windowed output size with arbitrary dilation_rate,
+// while the original version only handles the cases where dilation_rates equal
+// to 1.
+Status GetWindowedOutputSizeFromDimsV2(
     shape_inference::InferenceContext* c,
     shape_inference::DimensionHandle input_size,
-    shape_inference::DimensionOrConstant filter_size, int64 stride,
-    Padding padding_type, shape_inference::DimensionHandle* output_size) {
+    shape_inference::DimensionOrConstant filter_size, int64 dilation_rate,
+    int64 stride, Padding padding_type,
+    shape_inference::DimensionHandle* output_size) {
   if (stride <= 0) {
     return errors::InvalidArgument("Stride must be > 0, but got ", stride);
   }
 
+  if (dilation_rate < 1) {
+    return errors::InvalidArgument("Dilation rate must be >= 1, but got ",
+                                   dilation_rate);
+  }
+
   // See also the parallel implementation in GetWindowedOutputSizeVerbose.
   switch (padding_type) {
     case Padding::VALID:
-      TF_RETURN_IF_ERROR(c->Subtract(input_size, filter_size, output_size));
+      if (dilation_rate > 1) {
+        DimensionHandle window_size;
+        TF_RETURN_IF_ERROR(
+            c->Subtract(c->MakeDim(filter_size), 1, &window_size));
+        TF_RETURN_IF_ERROR(
+            c->Multiply(window_size, dilation_rate, &window_size));
+        TF_RETURN_IF_ERROR(c->Add(window_size, 1, &window_size));
+        TF_RETURN_IF_ERROR(c->Subtract(input_size, window_size, output_size));
+      } else {
+        TF_RETURN_IF_ERROR(c->Subtract(input_size, filter_size, output_size));
+      }
       TF_RETURN_IF_ERROR(c->Add(*output_size, stride, output_size));
       TF_RETURN_IF_ERROR(c->Divide(*output_size, stride,
-                                   false /* evenly_divisible */, output_size));
+                                   /*evenly_divisible=*/false, output_size));
       break;
     case Padding::SAME:
       TF_RETURN_IF_ERROR(c->Add(input_size, stride - 1, output_size));
       TF_RETURN_IF_ERROR(c->Divide(*output_size, stride,
-                                   false /* evenly_divisible */, output_size));
+                                   /*evenly_divisible=*/false, output_size));
       break;
   }
   return Status::OK();
 }
 
+Status GetWindowedOutputSizeFromDims(
+    shape_inference::InferenceContext* c,
+    shape_inference::DimensionHandle input_size,
+    shape_inference::DimensionOrConstant filter_size, int64 stride,
+    Padding padding_type, shape_inference::DimensionHandle* output_size) {
+  return GetWindowedOutputSizeFromDimsV2(c, input_size, filter_size,
+                                         /*dilation_rate=*/1, stride,
+                                         padding_type, output_size);
+}
+
 Status UnchangedShape(shape_inference::InferenceContext* c) {
   c->set_output(0, c->input(0));
   return Status::OK();
diff --git a/tensorflow/core/framework/common_shape_fns.h b/tensorflow/core/framework/common_shape_fns.h
index 88fea550a6..c0deb473a2 100644
--- a/tensorflow/core/framework/common_shape_fns.h
+++ b/tensorflow/core/framework/common_shape_fns.h
@@ -75,6 +75,32 @@ Status GetWindowedOutputSize(int64 input_size, int64 filter_size, int64 stride,
                              Padding padding_type, int64* output_size,
                              int64* padding_size);
 
+// The V2 version computes the same outputs with arbitrary dilation_rate.
+// The output dimensions are computed as follows:
+// - When adding dilation_rate (D), we compute an effective filter size (K'):
+//     K' = (K - 1) * D + 1
+// - When Padding = SAME: the output size is (H'), where
+//     H' = ceil(float(H) / float(S))
+//   where ceil is the ceiling function. The number of padded cells
+//   is computed as:
+//     Pc = ((H' - 1) * S + K' - H) / 2
+//   When the stride is 1, the expression simplifies to
+//     H' = H, Pc = (K'-1)/2.
+//   This is where SAME comes from - the output has the same size as the input
+//   has.
+//
+// - When Padding = VALID: the output size is computed as
+//     H' = ceil(float(H - K' + 1) / float(S))
+//   and the number of padded cells is always zero.
+//   When the stride is 1, the expression simplifies to
+//     H' = H-K'+1.
+//
+// TODO(b/67112639): Merge V2 versions and the original versions eventually.
+Status GetWindowedOutputSizeV2(int64 input_size, int64 filter_size,
+                               int64 dilation_rate, int64 stride,
+                               Padding padding_type, int64* output_size,
+                               int64* padding_size);
+
 // Returns the same output dimensions as in GetWindowedOutputSize, but returns
 // verbose padding dimensions (before/after). Any excess padding
 // (caused by an odd padding size value) is added to the 'padding_after'
@@ -84,6 +110,14 @@ Status GetWindowedOutputSizeVerbose(int64 input_size, int64 filter_size,
                                     int64* output_size, int64* padding_before,
                                     int64* padding_after);
 
+// The V2 version computes the same outputs with arbitrary dilation_rate. For
+// detailed equations, refer to the comments for GetWindowedOutputSizeV2().
+Status GetWindowedOutputSizeVerboseV2(int64 input_size, int64 filter_size,
+                                      int64 dilation_rate, int64 stride,
+                                      Padding padding_type, int64* output_size,
+                                      int64* padding_before,
+                                      int64* padding_after);
+
 // Given an input tensor, kernel, stride and padding type, populates the 3D size
 // of the output tensor and padding to be applied to the input tensor at the
 // lower end of every dimension. Use for 3D convolutions, where the input data
@@ -92,8 +126,17 @@ Status GetWindowedOutputSizeVerbose(int64 input_size, int64 filter_size,
 Status Get3dOutputSize(const std::array<int64, 3>& input,
                        const std::array<int64, 3>& window,
                        const std::array<int64, 3>& strides,
-                       Padding padding_type, std::array<int64, 3>* output,
-                       std::array<int64, 3>* padding);
+                       Padding padding_type, std::array<int64, 3>* output_ptr,
+                       std::array<int64, 3>* padding_ptr);
+
+// The V2 version computes the same outputs with arbitrary dilation_rate. For
+// detailed equations, refer to the comments for GetWindowedOutputSizeV2().
+Status Get3dOutputSizeV2(const std::array<int64, 3>& input,
+                         const std::array<int64, 3>& window,
+                         const std::array<int64, 3>& dilations,
+                         const std::array<int64, 3>& strides,
+                         Padding padding_type, std::array<int64, 3>* output_ptr,
+                         std::array<int64, 3>* padding_ptr);
 
 namespace shape_inference {
 
@@ -104,6 +147,15 @@ Status GetWindowedOutputSizeFromDims(InferenceContext* c,
                                      int64 stride, Padding padding_type,
                                      DimensionHandle* output_size);
 
+// The V2 version computes the same outputs with arbitrary dilation_rate. For
+// detailed equations, refer to the comments for GetWindowedOutputSizeV2().
+Status GetWindowedOutputSizeFromDimsV2(InferenceContext* c,
+                                       DimensionHandle input_size,
+                                       DimensionOrConstant filter_size,
+                                       int64 dilation_rate, int64 stride,
+                                       Padding padding_type,
+                                       DimensionHandle* output_size);
+
 // Transfers shape of input(0) to output(0).
 Status UnchangedShape(shape_inference::InferenceContext* c);
 
diff --git a/tensorflow/core/kernels/conv_grad_ops.cc b/tensorflow/core/kernels/conv_grad_ops.cc
index 4c864c08a5..170ce31d17 100644
--- a/tensorflow/core/kernels/conv_grad_ops.cc
+++ b/tensorflow/core/kernels/conv_grad_ops.cc
@@ -23,6 +23,7 @@ limitations under the License.
 #include <algorithm>
 #include <vector>
 
+#include "tensorflow/core/framework/common_shape_fns.h"
 #include "tensorflow/core/framework/numeric_op.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
@@ -40,46 +41,64 @@ limitations under the License.
 
 namespace tensorflow {
 
-Status ConvBackpropExtractAndVerifyDimension(
+// The V2 version computes windowed output size with arbitrary dilation_rate,
+// while the original version only handles the cases where dilation_rates equal
+// to 1.
+Status ConvBackpropExtractAndVerifyDimensionV2(
     StringPiece label, const TensorShape& input_shape,
     const TensorShape& filter_shape, const TensorShape& output_shape,
-    const std::vector<int32>& strides, Padding padding, int spatial_dim,
-    int filter_spatial_dim, ConvBackpropSpatialDimension* dim) {
+    const gtl::ArraySlice<int32>& dilations, const std::vector<int32>& strides,
+    Padding padding, int spatial_dim, int filter_spatial_dim,
+    ConvBackpropSpatialDimension* dim) {
   dim->input_size = input_shape.dim_size(spatial_dim);
   dim->filter_size = filter_shape.dim_size(filter_spatial_dim);
   dim->output_size = output_shape.dim_size(spatial_dim);
   dim->stride = strides[spatial_dim];
+  dim->dilation = dilations[spatial_dim];
   int64 out_size = 0, pad_size = 0;
-  TF_RETURN_IF_ERROR(GetWindowedOutputSize(dim->input_size, dim->filter_size,
-                                           dim->stride, padding, &out_size,
-                                           &pad_size));
+  TF_RETURN_IF_ERROR(GetWindowedOutputSizeV2(dim->input_size, dim->filter_size,
+                                             dim->dilation, dim->stride,
+                                             padding, &out_size, &pad_size));
   if (dim->output_size != out_size) {
     return errors::InvalidArgument(
         label, ": Size of out_backprop doesn't match computed: ", "actual = ",
-        dim->output_size, ", computed = ", out_size);
+        dim->output_size, ", computed = ", out_size,
+        "spatial_dim: ", spatial_dim, " input: ", dim->input_size,
+        " filter: ", dim->filter_size, " output: ", dim->output_size,
+        " stride: ", dim->stride, " dilation: ", dim->dilation);
   }
 
+  int64 effective_filter_size = (dim->filter_size - 1) * dim->dilation + 1;
   dim->expanded_output_size = (dim->output_size - 1) * dim->stride + 1;
-  const auto padded_out_size = dim->input_size + dim->filter_size - 1;
-  dim->pad_before = dim->filter_size - 1 - pad_size;
+  const auto padded_out_size = dim->input_size + effective_filter_size - 1;
+  dim->pad_before = effective_filter_size - 1 - pad_size;
   dim->pad_after =
       padded_out_size - dim->expanded_output_size - dim->pad_before;
   VLOG(2) << label << ": expanded_out = " << dim->expanded_output_size
-          << ", filter = " << dim->filter_size
+          << ", effective_filter_size = " << effective_filter_size
           << ", padded_out = " << padded_out_size
           << ", pad_before = " << dim->pad_before
           << ", pad_after = " << dim->pad_after
-          << ", strides = " << dim->stride;
+          << ", dilation = " << dim->dilation << ", strides = " << dim->stride;
   return Status::OK();
 }
 
-Status ConvBackpropComputeDimensions(StringPiece label, int num_spatial_dims,
-                                     const TensorShape& input_shape,
-                                     const TensorShape& filter_shape,
-                                     const TensorShape& out_backprop_shape,
-                                     const std::vector<int32>& strides,
-                                     Padding padding, TensorFormat data_format,
-                                     ConvBackpropDimensions* dims) {
+Status ConvBackpropExtractAndVerifyDimension(
+    StringPiece label, const TensorShape& input_shape,
+    const TensorShape& filter_shape, const TensorShape& output_shape,
+    const std::vector<int32>& strides, Padding padding, int spatial_dim,
+    int filter_spatial_dim, ConvBackpropSpatialDimension* dim) {
+  static constexpr std::array<int32, 5> one_dilations = {{1, 1, 1, 1, 1}};
+  return ConvBackpropExtractAndVerifyDimensionV2(
+      label, input_shape, filter_shape, output_shape, one_dilations, strides,
+      padding, spatial_dim, filter_spatial_dim, dim);
+}
+
+Status ConvBackpropComputeDimensionsV2(
+    StringPiece label, int num_spatial_dims, const TensorShape& input_shape,
+    const TensorShape& filter_shape, const TensorShape& out_backprop_shape,
+    const gtl::ArraySlice<int32>& dilations, const std::vector<int32>& strides,
+    Padding padding, TensorFormat data_format, ConvBackpropDimensions* dims) {
   // The + 2 in the following line is for the batch and feature dimensions.
   const int num_dims = num_spatial_dims + 2;
   if (input_shape.dims() != num_dims) {
@@ -98,7 +117,10 @@ Status ConvBackpropComputeDimensions(StringPiece label, int num_spatial_dims,
   dims->batch_size = input_shape.dim_size(batch_dim);
   if (dims->batch_size != out_backprop_shape.dim_size(batch_dim)) {
     return errors::InvalidArgument(
-        label, ": input and out_backprop must have the same batch size");
+        label, ": input and out_backprop must have the same batch size",
+        "input batch: ", dims->batch_size,
+        "outbackprop batch: ", out_backprop_shape.dim_size(batch_dim),
+        " batch_dim: ", batch_dim);
   }
 
   int feature_dim = GetTensorFeatureDimIndex(num_dims, data_format);
@@ -118,11 +140,24 @@ Status ConvBackpropComputeDimensions(StringPiece label, int num_spatial_dims,
   dims->spatial_dims.resize(num_spatial_dims);
   for (int i = 0; i < num_spatial_dims; ++i) {
     int image_dim = GetTensorSpatialDimIndex(num_dims, data_format, i);
-    TF_RETURN_IF_ERROR(ConvBackpropExtractAndVerifyDimension(
-        label, input_shape, filter_shape, out_backprop_shape, strides, padding,
-        image_dim, i, &dims->spatial_dims[i]));
+    TF_RETURN_IF_ERROR(ConvBackpropExtractAndVerifyDimensionV2(
+        label, input_shape, filter_shape, out_backprop_shape, dilations,
+        strides, padding, image_dim, i, &dims->spatial_dims[i]));
   }
   return Status::OK();
 }
 
+Status ConvBackpropComputeDimensions(StringPiece label, int num_spatial_dims,
+                                     const TensorShape& input_shape,
+                                     const TensorShape& filter_shape,
+                                     const TensorShape& out_backprop_shape,
+                                     const std::vector<int32>& strides,
+                                     Padding padding, TensorFormat data_format,
+                                     ConvBackpropDimensions* dims) {
+  static constexpr std::array<int32, 5> one_dilations = {{1, 1, 1, 1, 1}};
+  return ConvBackpropComputeDimensionsV2(
+      label, num_spatial_dims, input_shape, filter_shape, out_backprop_shape,
+      one_dilations, strides, padding, data_format, dims);
+}
+
 }  // namespace tensorflow
diff --git a/tensorflow/core/kernels/conv_grad_ops.h b/tensorflow/core/kernels/conv_grad_ops.h
index 2926bb3a86..3a3492304b 100644
--- a/tensorflow/core/kernels/conv_grad_ops.h
+++ b/tensorflow/core/kernels/conv_grad_ops.h
@@ -212,6 +212,7 @@ struct ConvBackpropSpatialDimension {
   int64 filter_size;
   int64 output_size;
   int64 stride;
+  int64 dilation;
   int64 expanded_output_size;
 
   // Number of padding elements to be added before/after this dimension of
@@ -242,6 +243,13 @@ Status ConvBackpropComputeDimensions(StringPiece label, int num_spatial_dims,
                                      Padding padding, TensorFormat data_format,
                                      ConvBackpropDimensions* dims);
 
+// The V2 version computes the same outputs with arbitrary dilation rate.
+// TODO(b/67112639): Merge V2 versions and the original versions eventually.
+Status ConvBackpropComputeDimensionsV2(
+    StringPiece label, int num_spatial_dims, const TensorShape& input_shape,
+    const TensorShape& filter_shape, const TensorShape& out_backprop_shape,
+    const std::vector<int32>& dilations, const std::vector<int32>& strides,
+    Padding padding, TensorFormat data_format, ConvBackpropDimensions* dims);
 }  // namespace tensorflow
 
 #endif  // TENSORFLOW_CORE_KERNELS_CONV_GRAD_OPS_H_
-- 
GitLab


From 3cf41b2edd4384a9df385430868dbdd887ecab86 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 4 Oct 2017 13:07:44 -0700
Subject: [PATCH 004/909] Test save/restore variable from graph_callable.

PiperOrigin-RevId: 171051237
---
 tensorflow/contrib/eager/python/BUILD         |  1 +
 tensorflow/contrib/eager/python/saver_test.py | 51 +++++++++++++++++++
 2 files changed, 52 insertions(+)

diff --git a/tensorflow/contrib/eager/python/BUILD b/tensorflow/contrib/eager/python/BUILD
index dd305a78dc..9185c963f7 100644
--- a/tensorflow/contrib/eager/python/BUILD
+++ b/tensorflow/contrib/eager/python/BUILD
@@ -81,6 +81,7 @@ cuda_py_test(
         "//tensorflow/python:array_ops",
         "//tensorflow/python:client",
         "//tensorflow/python:client_testlib",
+        "//tensorflow/python/eager:graph_callable",
         "//tensorflow/python:platform_test",
         "//tensorflow/python:variables",
     ],
diff --git a/tensorflow/contrib/eager/python/saver_test.py b/tensorflow/contrib/eager/python/saver_test.py
index cdec50ebd7..29af2b531f 100644
--- a/tensorflow/contrib/eager/python/saver_test.py
+++ b/tensorflow/contrib/eager/python/saver_test.py
@@ -21,10 +21,14 @@ import os
 
 from tensorflow.contrib.eager.python import saver as _saver
 from tensorflow.python.eager import context
+from tensorflow.python.eager import graph_callable
+from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import init_ops
 from tensorflow.python.ops import resource_variable_ops
+from tensorflow.python.ops import variable_scope
 from tensorflow.python.platform import test
 
 
@@ -87,6 +91,53 @@ class SaverTest(test.TestCase):
         with _saver.restore_variables_on_create(ckpt_prefix):
           _ = model(resource_variable_ops.ResourceVariable(1.0, name='v2'))
 
+  def testSaveRestoreGraphCallable(self):
+    with context.eager_mode(), ops.device(self._dev()):
+      @graph_callable.graph_callable(
+          [graph_callable.ShapeAndDtype(shape=(), dtype=dtypes.float32)])
+      def model(x):
+        v = variable_scope.get_variable(
+            'v', initializer=init_ops.zeros_initializer(), shape=())
+        return v + x
+
+      # Default 2 + 0 = 2
+      self.assertEqual(
+          2, model(array_ops.constant(2, dtype=dtypes.float32)).numpy())
+
+      # Save the variable value 0.
+      ckpt_prefix = os.path.join(test.get_temp_dir(), 'ckpt')
+      _saver.Saver(model.variables).save(ckpt_prefix)
+
+      # update variable to 1, so that 2 + 1 = 3
+      model.variables[0].assign(1.)
+      self.assertEqual(
+          3, model(array_ops.constant(2, dtype=dtypes.float32)).numpy())
+
+      # load the variable value 0, so that 2 + 0 = 2
+      _saver.Saver(model.variables).restore(ckpt_prefix)
+      self.assertEqual(
+          2, model(array_ops.constant(2, dtype=dtypes.float32)).numpy())
+
+      # update checkpoint variable to 1 and memory value to 2.
+      model.variables[0].assign(1.)
+      _saver.Saver(model.variables).save(ckpt_prefix)
+      model.variables[0].assign(2.)
+      self.assertEqual(
+          4, model(array_ops.constant(2, dtype=dtypes.float32)).numpy())
+
+      # reset the graph and reload on create, so that 1 + 2 = 3
+      with ops.Graph().as_default():
+        with _saver.restore_variables_on_create(ckpt_prefix):
+          @graph_callable.graph_callable(
+              [graph_callable.ShapeAndDtype(shape=(), dtype=dtypes.float32)])
+          def model2(x):
+            v = variable_scope.get_variable(
+                'v', initializer=init_ops.zeros_initializer(), shape=())
+            return v + x
+
+          self.assertEqual(
+              3, model2(array_ops.constant(2, dtype=dtypes.float32)).numpy())
+
 
 if __name__ == '__main__':
   test.main()
-- 
GitLab


From ad69076ebd4c40226d0cd0f61ec1d4138d6bc46f Mon Sep 17 00:00:00 2001
From: Mustafa Ispir <ispir@google.com>
Date: Wed, 4 Oct 2017 13:14:04 -0700
Subject: [PATCH 005/909] Added get variable utils to tf.estimator.Estimator.

PiperOrigin-RevId: 171052121
---
 tensorflow/python/estimator/estimator.py      | 35 ++++++++++++++++++
 tensorflow/python/estimator/estimator_test.py | 37 +++++++++++++++++++
 ...nsorflow.estimator.-d-n-n-classifier.pbtxt |  8 ++++
 ...or.-d-n-n-linear-combined-classifier.pbtxt |  8 ++++
 ...tor.-d-n-n-linear-combined-regressor.pbtxt |  8 ++++
 ...ensorflow.estimator.-d-n-n-regressor.pbtxt |  8 ++++
 .../tensorflow.estimator.-estimator.pbtxt     |  8 ++++
 ...sorflow.estimator.-linear-classifier.pbtxt |  8 ++++
 ...nsorflow.estimator.-linear-regressor.pbtxt |  8 ++++
 9 files changed, 128 insertions(+)

diff --git a/tensorflow/python/estimator/estimator.py b/tensorflow/python/estimator/estimator.py
index eee48419b0..1197366256 100644
--- a/tensorflow/python/estimator/estimator.py
+++ b/tensorflow/python/estimator/estimator.py
@@ -204,6 +204,34 @@ class Estimator(object):
 
     return public_model_fn
 
+  # TODO(ispir): support a list of names
+  def get_variable_value(self, name):
+    """Returns value of the variable given by name.
+
+    Args:
+      name: string or a list of string, name of the tensor.
+
+    Returns:
+      Numpy array - value of the tensor.
+
+    Raises:
+      ValueError: If the Estimator has not produced a checkpoint yet.
+    """
+    _check_checkpoint_available(self.model_dir)
+    return training.load_variable(self.model_dir, name)
+
+  def get_variable_names(self):
+    """Returns list of all variable names in this model.
+
+    Returns:
+      List of names.
+
+    Raises:
+      ValueError: If the Estimator has not produced a checkpoint yet.
+    """
+    _check_checkpoint_available(self.model_dir)
+    return [name for name, _ in training.list_variables(self.model_dir)]
+
   def latest_checkpoint(self):
     """Finds the filename of latest saved checkpoint file in `model_dir`.
 
@@ -818,6 +846,13 @@ class Estimator(object):
     return eval_results
 
 
+def _check_checkpoint_available(model_dir):
+  latest_path = saver.latest_checkpoint(model_dir)
+  if not latest_path:
+    raise ValueError(
+        'Could not find trained model in model_dir: {}.'.format(model_dir))
+
+
 def _check_hooks_type(hooks):
   """Returns hooks if all are SessionRunHook, raises TypeError otherwise."""
   hooks = list(hooks or [])
diff --git a/tensorflow/python/estimator/estimator_test.py b/tensorflow/python/estimator/estimator_test.py
index e532d3bd2b..cdffe3378f 100644
--- a/tensorflow/python/estimator/estimator_test.py
+++ b/tensorflow/python/estimator/estimator_test.py
@@ -862,6 +862,43 @@ class _StepCounterHook(session_run_hook.SessionRunHook):
     return self._steps
 
 
+class EstimatorGetVariablesTest(test.TestCase):
+
+  def test_model_should_be_trained(self):
+
+    def _model_fn(features, labels, mode):
+      _, _ = features, labels
+      variables.Variable(1., name='one')
+      return model_fn_lib.EstimatorSpec(
+          mode=mode,
+          loss=constant_op.constant(0.),
+          train_op=state_ops.assign_add(training.get_global_step(), 1))
+
+    est = estimator.Estimator(model_fn=_model_fn)
+    with self.assertRaisesRegexp(ValueError, 'not find trained model'):
+      est.get_variable_names()
+    with self.assertRaisesRegexp(ValueError, 'not find trained model'):
+      est.get_variable_value('one')
+
+  def test_get_variable_utils(self):
+
+    def _model_fn(features, labels, mode):
+      _, _ = features, labels
+      variables.Variable(1., name='one')
+      variables.Variable(3., name='three')
+      return model_fn_lib.EstimatorSpec(
+          mode=mode,
+          loss=constant_op.constant(0.),
+          train_op=state_ops.assign_add(training.get_global_step(), 1))
+
+    est = estimator.Estimator(model_fn=_model_fn)
+    est.train(input_fn=dummy_input_fn, steps=1)
+    self.assertEqual(
+        set(['one', 'three', 'global_step']), set(est.get_variable_names()))
+    self.assertEqual(1., est.get_variable_value('one'))
+    self.assertEqual(3., est.get_variable_value('three'))
+
+
 class EstimatorEvaluateTest(test.TestCase):
 
   def test_input_fn_args(self):
diff --git a/tensorflow/tools/api/golden/tensorflow.estimator.-d-n-n-classifier.pbtxt b/tensorflow/tools/api/golden/tensorflow.estimator.-d-n-n-classifier.pbtxt
index b54e8517c7..16e3b24615 100644
--- a/tensorflow/tools/api/golden/tensorflow.estimator.-d-n-n-classifier.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.estimator.-d-n-n-classifier.pbtxt
@@ -31,6 +31,14 @@ tf_class {
     name: "export_savedmodel"
     argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
   }
+  member_method {
+    name: "get_variable_names"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_variable_value"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "latest_checkpoint"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/tensorflow.estimator.-d-n-n-linear-combined-classifier.pbtxt b/tensorflow/tools/api/golden/tensorflow.estimator.-d-n-n-linear-combined-classifier.pbtxt
index eb3a8eedbe..c6765ae277 100644
--- a/tensorflow/tools/api/golden/tensorflow.estimator.-d-n-n-linear-combined-classifier.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.estimator.-d-n-n-linear-combined-classifier.pbtxt
@@ -31,6 +31,14 @@ tf_class {
     name: "export_savedmodel"
     argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
   }
+  member_method {
+    name: "get_variable_names"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_variable_value"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "latest_checkpoint"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/tensorflow.estimator.-d-n-n-linear-combined-regressor.pbtxt b/tensorflow/tools/api/golden/tensorflow.estimator.-d-n-n-linear-combined-regressor.pbtxt
index 42003052f5..e3a820db46 100644
--- a/tensorflow/tools/api/golden/tensorflow.estimator.-d-n-n-linear-combined-regressor.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.estimator.-d-n-n-linear-combined-regressor.pbtxt
@@ -31,6 +31,14 @@ tf_class {
     name: "export_savedmodel"
     argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
   }
+  member_method {
+    name: "get_variable_names"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_variable_value"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "latest_checkpoint"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/tensorflow.estimator.-d-n-n-regressor.pbtxt b/tensorflow/tools/api/golden/tensorflow.estimator.-d-n-n-regressor.pbtxt
index 32f5e8810a..a4c8cf6671 100644
--- a/tensorflow/tools/api/golden/tensorflow.estimator.-d-n-n-regressor.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.estimator.-d-n-n-regressor.pbtxt
@@ -31,6 +31,14 @@ tf_class {
     name: "export_savedmodel"
     argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
   }
+  member_method {
+    name: "get_variable_names"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_variable_value"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "latest_checkpoint"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/tensorflow.estimator.-estimator.pbtxt b/tensorflow/tools/api/golden/tensorflow.estimator.-estimator.pbtxt
index 78e1c75b13..787952eced 100644
--- a/tensorflow/tools/api/golden/tensorflow.estimator.-estimator.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.estimator.-estimator.pbtxt
@@ -30,6 +30,14 @@ tf_class {
     name: "export_savedmodel"
     argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
   }
+  member_method {
+    name: "get_variable_names"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_variable_value"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "latest_checkpoint"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/tensorflow.estimator.-linear-classifier.pbtxt b/tensorflow/tools/api/golden/tensorflow.estimator.-linear-classifier.pbtxt
index cb3b5d01ff..99c03aa629 100644
--- a/tensorflow/tools/api/golden/tensorflow.estimator.-linear-classifier.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.estimator.-linear-classifier.pbtxt
@@ -31,6 +31,14 @@ tf_class {
     name: "export_savedmodel"
     argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
   }
+  member_method {
+    name: "get_variable_names"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_variable_value"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "latest_checkpoint"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/tensorflow.estimator.-linear-regressor.pbtxt b/tensorflow/tools/api/golden/tensorflow.estimator.-linear-regressor.pbtxt
index e5d596887e..e2ab96d5b4 100644
--- a/tensorflow/tools/api/golden/tensorflow.estimator.-linear-regressor.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.estimator.-linear-regressor.pbtxt
@@ -31,6 +31,14 @@ tf_class {
     name: "export_savedmodel"
     argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\'], "
   }
+  member_method {
+    name: "get_variable_names"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_variable_value"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "latest_checkpoint"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-- 
GitLab


From 6c954d0b3f02ea586a5fd3f9c2ea13bf8473d17f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 4 Oct 2017 13:16:05 -0700
Subject: [PATCH 006/909] Adding TF Boosted trees regression example on boston
 dataset, minor fix for mnist example.

PiperOrigin-RevId: 171052367
---
 .../contrib/boosted_trees/examples/boston.py  | 155 ++++++++++++++++++
 .../contrib/boosted_trees/examples/mnist.py   |   4 +-
 2 files changed, 157 insertions(+), 2 deletions(-)
 create mode 100644 tensorflow/contrib/boosted_trees/examples/boston.py

diff --git a/tensorflow/contrib/boosted_trees/examples/boston.py b/tensorflow/contrib/boosted_trees/examples/boston.py
new file mode 100644
index 0000000000..0cb9e956ef
--- /dev/null
+++ b/tensorflow/contrib/boosted_trees/examples/boston.py
@@ -0,0 +1,155 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+r"""Demonstrates a regression on Boston housing data.
+
+  This example demonstrates how to run experiments with TF Boosted Trees on
+  a regression dataset. We split all the data into 20% test and 80% train,
+  and are using l2 loss and l2 regularization.
+
+  Example Usage:
+
+  python tensorflow/contrib/boosted_trees/examples/boston.py \
+  --batch_size=404 --output_dir="/tmp/boston" --depth=4 --learning_rate=0.1 \
+  --num_eval_steps=1 --num_trees=500 --l2=4 \
+  --vmodule=training_ops=1
+
+  When training is done, mean squared error on eval data is reported.
+  Point tensorboard to the directory for the run to see how the training
+  progresses:
+
+  tensorboard --logdir=/tmp/boston
+
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import argparse
+import sys
+import tensorflow as tf
+from tensorflow.contrib.boosted_trees.estimator_batch.estimator import GradientBoostedDecisionTreeRegressor
+from tensorflow.contrib.boosted_trees.proto import learner_pb2
+from tensorflow.contrib.layers.python.layers import feature_column
+from tensorflow.contrib.learn import learn_runner
+
+_TEST_SPLIT_RATIO = 0.2
+_TEST_SPLIT_SEED = 42
+_BOSTON_NUM_FEATURES = 13
+
+
+# Main config - creates a TF Boosted Trees Estimator based on flags.
+def _get_tfbt(output_dir, feature_cols):
+  """Configures TF Boosted Trees estimator based on flags."""
+  learner_config = learner_pb2.LearnerConfig()
+
+  learner_config.learning_rate_tuner.fixed.learning_rate = FLAGS.learning_rate
+  learner_config.regularization.l1 = 0.0
+  # Set the regularization per instance in such a way that
+  # regularization for the full training data is equal to l2 flag.
+  learner_config.regularization.l2 = FLAGS.l2 / FLAGS.batch_size
+  learner_config.constraints.max_tree_depth = FLAGS.depth
+  learner_config.growing_mode = learner_pb2.LearnerConfig.WHOLE_TREE
+
+  run_config = tf.contrib.learn.RunConfig(save_checkpoints_secs=300)
+
+  # Create a TF Boosted trees regression estimator.
+  estimator = GradientBoostedDecisionTreeRegressor(
+      learner_config=learner_config,
+      # For the WHOLE_TREE strategy, set the examples_per_layer to be equal to
+      # batch size.
+      examples_per_layer=FLAGS.batch_size,
+      feature_columns=feature_cols,
+      label_dimension=1,
+      model_dir=output_dir,
+      num_trees=FLAGS.num_trees,
+      center_bias=False,
+      config=run_config)
+  return estimator
+
+
+def _make_experiment_fn(output_dir):
+  """Creates experiment for gradient boosted decision trees."""
+  (x_train, y_train), (x_test,
+                       y_test) = tf.keras.datasets.boston_housing.load_data()
+
+  train_input_fn = tf.estimator.inputs.numpy_input_fn(
+      x={"x": x_train},
+      y=y_train,
+      batch_size=FLAGS.batch_size,
+      num_epochs=None,
+      shuffle=True)
+
+  eval_input_fn = tf.estimator.inputs.numpy_input_fn(
+      x={"x": x_test}, y=y_test, num_epochs=1, shuffle=False)
+
+  feature_columns = [
+      feature_column.real_valued_column("x", dimension=_BOSTON_NUM_FEATURES)
+  ]
+
+  return tf.contrib.learn.Experiment(
+      estimator=_get_tfbt(output_dir, feature_columns),
+      train_input_fn=train_input_fn,
+      eval_input_fn=eval_input_fn,
+      train_steps=None,
+      eval_steps=FLAGS.num_eval_steps,
+      eval_metrics=None)
+
+
+def main(unused_argv):
+  learn_runner.run(
+      experiment_fn=_make_experiment_fn,
+      output_dir=FLAGS.output_dir,
+      schedule="train_and_evaluate")
+
+
+if __name__ == "__main__":
+  tf.logging.set_verbosity(tf.logging.INFO)
+  parser = argparse.ArgumentParser()
+  # Define the list of flags that users can change.
+  parser.add_argument(
+      "--batch_size",
+      type=int,
+      default=1000,
+      help="The batch size for reading data.")
+  parser.add_argument(
+      "--output_dir",
+      type=str,
+      required=True,
+      help="Choose the dir for the output.")
+  parser.add_argument(
+      "--num_eval_steps",
+      type=int,
+      default=1,
+      help="The number of steps to run evaluation for.")
+  # Flags for gradient boosted trees config.
+  parser.add_argument(
+      "--depth", type=int, default=4, help="Maximum depth of weak learners.")
+  parser.add_argument(
+      "--l2", type=float, default=1.0, help="l2 regularization per batch.")
+  parser.add_argument(
+      "--learning_rate",
+      type=float,
+      default=0.1,
+      help="Learning rate (shrinkage weight) with which each new tree is added."
+  )
+  parser.add_argument(
+      "--num_trees",
+      type=int,
+      default=None,
+      required=True,
+      help="Number of trees to grow before stopping.")
+
+  FLAGS, unparsed = parser.parse_known_args()
+  tf.app.run(main=main, argv=[sys.argv[0]] + unparsed)
diff --git a/tensorflow/contrib/boosted_trees/examples/mnist.py b/tensorflow/contrib/boosted_trees/examples/mnist.py
index 7e34d2f2d3..a3b1cb5154 100644
--- a/tensorflow/contrib/boosted_trees/examples/mnist.py
+++ b/tensorflow/contrib/boosted_trees/examples/mnist.py
@@ -129,8 +129,8 @@ def _get_tfbt(output_dir):
 def _make_experiment_fn(output_dir):
   """Creates experiment for gradient boosted decision trees."""
   data = tf.contrib.learn.datasets.mnist.load_mnist()
-  train_input_fn = get_input_fn(data.train, batch_size=256)
-  eval_input_fn = get_input_fn(data.validation, batch_size=5000)
+  train_input_fn = get_input_fn(data.train, FLAGS.batch_size)
+  eval_input_fn = get_input_fn(data.validation, FLAGS.eval_batch_size)
 
   return tf.contrib.learn.Experiment(
       estimator=_get_tfbt(output_dir),
-- 
GitLab


From 15155493b941a28d2d9c1e1cb1ed5873612b360a Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Wed, 4 Oct 2017 13:26:11 -0700
Subject: [PATCH 007/909] Fast path for tf.conj when it should be pass-through.

PiperOrigin-RevId: 171053662
---
 tensorflow/python/ops/math_ops.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tensorflow/python/ops/math_ops.py b/tensorflow/python/ops/math_ops.py
index 131f3724eb..9383d72f14 100644
--- a/tensorflow/python/ops/math_ops.py
+++ b/tensorflow/python/ops/math_ops.py
@@ -2317,6 +2317,10 @@ def conj(x, name=None):
   Raises:
     TypeError: If `x` is not a numeric tensor.
   """
+  if isinstance(x, ops.Tensor):
+    dt = x.dtype
+    if dt.is_floating or dt.is_integer:
+      return x
   with ops.name_scope(name, "Conj", [x]) as name:
     x = ops.convert_to_tensor(x, name="x")
     if x.dtype.is_complex or x.dtype == dtypes.variant:
-- 
GitLab


From 2fe6cf285d2bf4222ea09f9e929e538b64bc376b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 4 Oct 2017 13:26:47 -0700
Subject: [PATCH 008/909] Internal cleanup

PiperOrigin-RevId: 171053770
---
 tensorflow/python/eager/execute.py            | 10 ++++++---
 tensorflow/python/layers/base.py              | 22 ++++++++++++++-----
 tensorflow/python/layers/normalization.py     |  2 +-
 .../python/ops/resource_variable_ops.py       | 12 ++--------
 4 files changed, 27 insertions(+), 19 deletions(-)

diff --git a/tensorflow/python/eager/execute.py b/tensorflow/python/eager/execute.py
index 8bb4c0687d..04634daba4 100644
--- a/tensorflow/python/eager/execute.py
+++ b/tensorflow/python/eager/execute.py
@@ -168,27 +168,31 @@ def make_tensor(v, arg_name):
 
 def args_to_matching_eager(l, ctx, default_dtype=None):
   """Convert sequence `l` to eager same-type Tensors."""
+  EagerTensor = ops.EagerTensor  # pylint: disable=invalid-name
+  if all(isinstance(x, EagerTensor) for x in l):
+    return l[0].dtype, l
   # TODO(josh11b): Could we do a better job if we also passed in the
   # allowed dtypes when that was known?
 
   # Is some input already a Tensor with a dtype?
   dtype = None
   for t in l:
-    if isinstance(t, ops.EagerTensor):
+    if isinstance(t, EagerTensor):
       dtype = t.dtype
       break
 
+  internal_convert_to_tensor = ops.internal_convert_to_tensor
   if dtype is None:
     # Infer a dtype based on the first value, and use that dtype for the
     # remaining values.
     ret = []
     for t in l:
-      ret.append(ops.internal_convert_to_tensor(
+      ret.append(internal_convert_to_tensor(
           t, dtype, preferred_dtype=default_dtype, ctx=ctx))
       if dtype is None:
         dtype = ret[-1].dtype
   else:
-    ret = [ops.internal_convert_to_tensor(t, dtype, ctx=ctx) for t in l]
+    ret = [internal_convert_to_tensor(t, dtype, ctx=ctx) for t in l]
 
   return dtype, ret
 
diff --git a/tensorflow/python/layers/base.py b/tensorflow/python/layers/base.py
index 9e7cdd493f..1e11d1ae8d 100644
--- a/tensorflow/python/layers/base.py
+++ b/tensorflow/python/layers/base.py
@@ -112,8 +112,10 @@ class Layer(object):
     self._per_input_losses = {}
     self._per_input_updates = {}
     self._dtype = None if dtype is None else dtypes.as_dtype(dtype).name
-    self._compute_previous_mask = ('mask' in estimator_util.fn_args(self.call)
-                                   or hasattr(self, 'compute_mask'))
+    call_fn_args = estimator_util.fn_args(self.call)
+    self._compute_previous_mask = ('mask' in call_fn_args or
+                                   hasattr(self, 'compute_mask'))
+    self._call_has_scope_arg = 'scope' in call_fn_args
 
     # These lists will be filled via successive calls
     # to self._add_inbound_node().
@@ -555,7 +557,15 @@ class Layer(object):
             self.build(input_shapes[0])
           else:
             self.build(input_shapes)
-        if 'scope' in estimator_util.fn_args(self.call):
+        try:
+          # Note: not all sub-classes of Layer call Layer.__init__ (especially
+          # the ones under tensorflow/python/keras). Hence we recompute this
+          # attribute here if it is not set.
+          # TODO(agarwal): Fix the sub-classes and avoid this complexity.
+          call_has_scope_arg = self._call_has_scope_arg
+        except AttributeError:
+          call_has_scope_arg = 'scope' in estimator_util.fn_args(self.call)
+        if call_has_scope_arg:
           kwargs['scope'] = scope
         # Check input assumptions set after layer building, e.g. input shape.
         if in_graph_mode:
@@ -1433,8 +1443,10 @@ class Network(Layer):
     self._activity_regularizer = None
     self._scope = next(vs.variable_scope(None, default_name=base_name).gen)
     self._base_name = base_name
-    self._compute_previous_mask = ('mask' in estimator_util.fn_args(self.call)
-                                   or hasattr(self, 'compute_mask'))
+    call_fn_args = estimator_util.fn_args(self.call)
+    self._compute_previous_mask = ('mask' in call_fn_args or
+                                   hasattr(self, 'compute_mask'))
+    self._call_has_scope_arg = 'scope' in call_fn_args
 
     # This acts just like the `trainable` attribute of any layer instance.
     # It does not affect users of the underlying layers, only users of the
diff --git a/tensorflow/python/layers/normalization.py b/tensorflow/python/layers/normalization.py
index 0521129b27..ebcf397625 100644
--- a/tensorflow/python/layers/normalization.py
+++ b/tensorflow/python/layers/normalization.py
@@ -330,7 +330,7 @@ class BatchNormalization(base.Layer):
                                       lambda: self._one_minus_decay,
                                       lambda: 0.)
     else:
-      one_minus_decay = self._one_minus_decay
+      one_minus_decay = ops.convert_to_tensor(self._one_minus_decay)
     if training_value or training_value is None:
       mean_update = self._assign_moving_average(self.moving_mean, mean,
                                                 one_minus_decay)
diff --git a/tensorflow/python/ops/resource_variable_ops.py b/tensorflow/python/ops/resource_variable_ops.py
index bf4759e9ee..4ef9b05d51 100644
--- a/tensorflow/python/ops/resource_variable_ops.py
+++ b/tensorflow/python/ops/resource_variable_ops.py
@@ -540,16 +540,8 @@ class ResourceVariable(variables.Variable):
      the read operation.
     """
     with ops.name_scope("Read"):
-      # In graph mode, ensure we read the variable in the same device as the
-      # handle. In eager mode, however, this sometimes tries to read a GPU
-      # variable in the CPU because the handle is host memory. For now, then, we
-      # need to skip the device block in eager. TODO(apassos): eager should have
-      # separate notions of device and memory, so handle.device can be GPU while
-      # handle.memory_space is always CPU.
-      if context.in_graph_mode():
-        with ops.device(self._handle_device):
-          value = self._read_variable_op()
-      else:
+      # Ensure we read the variable in the same device as the handle.
+      with ops.device(self._handle_device):
         value = self._read_variable_op()
     # Return an identity so it can get placed on whatever device the context
     # specifies instead of the device where the variable is.
-- 
GitLab


From 083bd5dde5e6845a6f5e3b83ea2e074d7b28d61f Mon Sep 17 00:00:00 2001
From: Asim Shankar <ashankar@google.com>
Date: Wed, 4 Oct 2017 13:33:07 -0700
Subject: [PATCH 009/909] Java: Add support for loading op libraries
 dynamically.

This change adds the equivalent of tf.load_op_library in Python to Java.
(https://github.com/tensorflow/tensorflow/commit/5c7f9e316d8c7735308a217310350d416d7498cc
 was required to make this possible)

Though, TensorFlow.loadLibrary() is likely to fail on Windows as symbols
required by custom op libraries (those exported by the tensorflow_framework library)
are not exported by the monolithic JNI library yet.

This should help with #10454 and #13476

PiperOrigin-RevId: 171054707
---
 tensorflow/java/BUILD                         |  9 ++++-
 .../main/java/org/tensorflow/TensorFlow.java  | 30 ++++++++++++++++
 .../java/src/main/native/tensorflow_jni.cc    | 35 +++++++++++++++++++
 .../java/src/main/native/tensorflow_jni.h     | 30 ++++++++++++++--
 .../java/org/tensorflow/TensorFlowTest.java   | 23 ++++++++++++
 tensorflow/java/src/test/native/my_test_op.cc | 21 +++++++++++
 6 files changed, 145 insertions(+), 3 deletions(-)
 create mode 100644 tensorflow/java/src/test/native/my_test_op.cc

diff --git a/tensorflow/java/BUILD b/tensorflow/java/BUILD
index 9de79af7d2..a380bc2c71 100644
--- a/tensorflow/java/BUILD
+++ b/tensorflow/java/BUILD
@@ -10,8 +10,9 @@ load(":src/gen/gen_ops.bzl", "tf_java_op_gen_srcjar")
 load(
     "//tensorflow:tensorflow.bzl",
     "tf_binary_additional_srcs",
-    "tf_copts",
     "tf_cc_binary",
+    "tf_copts",
+    "tf_custom_op_library",
     "tf_java_test",
 )
 
@@ -180,10 +181,16 @@ tf_java_test(
     ],
 )
 
+tf_custom_op_library(
+    name = "my_test_op.so",
+    srcs = ["src/test/native/my_test_op.cc"],
+)
+
 tf_java_test(
     name = "TensorFlowTest",
     size = "small",
     srcs = ["src/test/java/org/tensorflow/TensorFlowTest.java"],
+    data = [":my_test_op.so"],
     javacopts = JAVACOPTS,
     test_class = "org.tensorflow.TensorFlowTest",
     deps = [
diff --git a/tensorflow/java/src/main/java/org/tensorflow/TensorFlow.java b/tensorflow/java/src/main/java/org/tensorflow/TensorFlow.java
index c21214b763..c90655f25d 100644
--- a/tensorflow/java/src/main/java/org/tensorflow/TensorFlow.java
+++ b/tensorflow/java/src/main/java/org/tensorflow/TensorFlow.java
@@ -29,6 +29,36 @@ public final class TensorFlow {
    */
   public static native byte[] registeredOpList();
 
+  /**
+   * Load the dynamic library in filename and register the operations and kernels present in that
+   * library.
+   *
+   * @param filename Path of the dynamic library containing operations and kernels to load.
+   * @return Serialized bytes of the <a
+   *     href="https://www.tensorflow.org/code/tensorflow/core/framework/op_def.proto">OpList</a>
+   *     protocol buffer message defining the operations defined in the library.
+   * @throws UnsatisfiedLinkError if filename cannot be loaded.
+   */
+  public static byte[] loadLibrary(String filename) {
+    long h = 0;
+    try {
+      h = libraryLoad(filename);
+    } catch (RuntimeException e) {
+      throw new UnsatisfiedLinkError(e.getMessage());
+    }
+    try {
+      return libraryOpList(h);
+    } finally {
+      libraryDelete(h);
+    }
+  }
+
+  private static native long libraryLoad(String filename);
+
+  private static native void libraryDelete(long handle);
+
+  private static native byte[] libraryOpList(long handle);
+
   private TensorFlow() {}
 
   /** Load the TensorFlow runtime C library. */
diff --git a/tensorflow/java/src/main/native/tensorflow_jni.cc b/tensorflow/java/src/main/native/tensorflow_jni.cc
index c553582e38..946ab502d1 100644
--- a/tensorflow/java/src/main/native/tensorflow_jni.cc
+++ b/tensorflow/java/src/main/native/tensorflow_jni.cc
@@ -14,7 +14,10 @@ limitations under the License.
 ==============================================================================*/
 
 #include "tensorflow/java/src/main/native/tensorflow_jni.h"
+
+#include <limits>
 #include "tensorflow/c/c_api.h"
+#include "tensorflow/java/src/main/native/exception_jni.h"
 
 JNIEXPORT jstring JNICALL Java_org_tensorflow_TensorFlow_version(JNIEnv* env,
                                                                  jclass clazz) {
@@ -30,3 +33,35 @@ Java_org_tensorflow_TensorFlow_registeredOpList(JNIEnv* env, jclass clazz) {
   TF_DeleteBuffer(buf);
   return ret;
 }
+
+JNIEXPORT jlong JNICALL Java_org_tensorflow_TensorFlow_libraryLoad(
+    JNIEnv* env, jclass clazz, jstring filename) {
+  TF_Status* status = TF_NewStatus();
+  const char* cname = env->GetStringUTFChars(filename, nullptr);
+  TF_Library* h = TF_LoadLibrary(cname, status);
+  throwExceptionIfNotOK(env, status);
+  env->ReleaseStringUTFChars(filename, cname);
+  TF_DeleteStatus(status);
+  return reinterpret_cast<jlong>(h);
+}
+
+JNIEXPORT void JNICALL Java_org_tensorflow_TensorFlow_libraryDelete(
+    JNIEnv* env, jclass clazz, jlong handle) {
+  if (handle != 0) {
+    TF_DeleteLibraryHandle(reinterpret_cast<TF_Library*>(handle));
+  }
+}
+
+JNIEXPORT jbyteArray JNICALL Java_org_tensorflow_TensorFlow_libraryOpList(
+    JNIEnv* env, jclass clazz, jlong handle) {
+  TF_Buffer buf = TF_GetOpList(reinterpret_cast<TF_Library*>(handle));
+  if (buf.length > std::numeric_limits<jint>::max()) {
+    throwException(env, kIndexOutOfBoundsException,
+                   "Serialized OpList is too large for a byte[] array");
+    return nullptr;
+  }
+  auto ret_len = static_cast<jint>(buf.length);
+  jbyteArray ret = env->NewByteArray(ret_len);
+  env->SetByteArrayRegion(ret, 0, ret_len, static_cast<const jbyte*>(buf.data));
+  return ret;
+}
diff --git a/tensorflow/java/src/main/native/tensorflow_jni.h b/tensorflow/java/src/main/native/tensorflow_jni.h
index ecd9b15828..c0c9322020 100644
--- a/tensorflow/java/src/main/native/tensorflow_jni.h
+++ b/tensorflow/java/src/main/native/tensorflow_jni.h
@@ -27,7 +27,7 @@ extern "C" {
  *  Method:    version
  *  Signature: ()Ljava/lang/String;
  */
-JNIEXPORT jstring JNICALL Java_org_tensorflow_TensorFlow_version(JNIEnv*,
+JNIEXPORT jstring JNICALL Java_org_tensorflow_TensorFlow_version(JNIEnv *,
                                                                  jclass);
 
 /*
@@ -36,7 +36,33 @@ JNIEXPORT jstring JNICALL Java_org_tensorflow_TensorFlow_version(JNIEnv*,
  * Signature: ()[B
  */
 JNIEXPORT jbyteArray JNICALL
-Java_org_tensorflow_TensorFlow_registeredOpList(JNIEnv*, jclass);
+Java_org_tensorflow_TensorFlow_registeredOpList(JNIEnv *, jclass);
+
+/*
+ * Class:     org_tensorflow_TensorFlow
+ * Method:    libraryLoad
+ * Signature: (Ljava/lang/String;)J
+ */
+JNIEXPORT jlong JNICALL Java_org_tensorflow_TensorFlow_libraryLoad(JNIEnv *,
+                                                                   jclass,
+                                                                   jstring);
+
+/*
+ * Class:     org_tensorflow_TensorFlow
+ * Method:    libraryDelete
+ * Signature: (J)V
+ */
+JNIEXPORT void JNICALL Java_org_tensorflow_TensorFlow_libraryDelete(JNIEnv *,
+                                                                    jclass,
+                                                                    jlong);
+
+/*
+ * Class:     org_tensorflow_TensorFlow
+ * Method:    libraryOpList
+ * Signature: (J)[B
+ */
+JNIEXPORT jbyteArray JNICALL
+Java_org_tensorflow_TensorFlow_libraryOpList(JNIEnv *, jclass, jlong);
 
 #ifdef __cplusplus
 }  // extern "C"
diff --git a/tensorflow/java/src/test/java/org/tensorflow/TensorFlowTest.java b/tensorflow/java/src/test/java/org/tensorflow/TensorFlowTest.java
index a31ea900d1..b1fa3f0d7e 100644
--- a/tensorflow/java/src/test/java/org/tensorflow/TensorFlowTest.java
+++ b/tensorflow/java/src/test/java/org/tensorflow/TensorFlowTest.java
@@ -16,6 +16,7 @@ limitations under the License.
 package org.tensorflow;
 
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
 
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -36,4 +37,26 @@ public class TensorFlowTest {
     // was not sorted out. Revisit? Till then, at least exercise the code.
     assertTrue(TensorFlow.registeredOpList().length > 0);
   }
+
+  @Test
+  public void loadLibrary() {
+    // TODO(ashankar): This tell will fail when built with --config=monolithic.
+    // Figure out how we can ignore the test in that case.
+    try (Graph g = new Graph()) {
+      // Build a graph with an unrecognized operation.
+      try {
+        g.opBuilder("MyTest", "MyTest").build();
+        fail("should not be able to construct graphs with unregistered ops");
+      } catch (IllegalArgumentException e) {
+        // expected exception
+      }
+
+      // Load the library containing the operation.
+      byte[] opList = TensorFlow.loadLibrary("tensorflow/java/my_test_op.so");
+      assertTrue(opList.length > 0);
+
+      // Now graph building should succeed.
+      g.opBuilder("MyTest", "MyTest").build();
+    }
+  }
 }
diff --git a/tensorflow/java/src/test/native/my_test_op.cc b/tensorflow/java/src/test/native/my_test_op.cc
new file mode 100644
index 0000000000..eb755901ed
--- /dev/null
+++ b/tensorflow/java/src/test/native/my_test_op.cc
@@ -0,0 +1,21 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/framework/common_shape_fns.h"
+#include "tensorflow/core/framework/op.h"
+
+REGISTER_OP("MyTest")
+    .Doc("Custom operation for testing.")
+    .SetShapeFn(tensorflow::shape_inference::UnknownShape);
-- 
GitLab


From d66e77f7c3ad4e5880af5ed3f287e472b6873f93 Mon Sep 17 00:00:00 2001
From: Mustafa Ispir <ispir@google.com>
Date: Wed, 4 Oct 2017 13:14:04 -0700
Subject: [PATCH 010/909] Added get variable utils to tf.estimator.Estimator.

PiperOrigin-RevId: 171052121
---
 .../contrib/boosted_trees/examples/boston.py  | 155 ------------------
 .../contrib/boosted_trees/examples/mnist.py   |   4 +-
 tensorflow/java/BUILD                         |   9 +-
 .../main/java/org/tensorflow/TensorFlow.java  |  30 ----
 .../java/src/main/native/tensorflow_jni.cc    |  35 ----
 .../java/src/main/native/tensorflow_jni.h     |  30 +---
 .../java/org/tensorflow/TensorFlowTest.java   |  23 ---
 tensorflow/java/src/test/native/my_test_op.cc |  21 ---
 tensorflow/python/eager/execute.py            |  10 +-
 tensorflow/python/layers/base.py              |  22 +--
 tensorflow/python/layers/normalization.py     |   2 +-
 tensorflow/python/ops/math_ops.py             |   4 -
 .../python/ops/resource_variable_ops.py       |  12 +-
 13 files changed, 24 insertions(+), 333 deletions(-)
 delete mode 100644 tensorflow/contrib/boosted_trees/examples/boston.py
 delete mode 100644 tensorflow/java/src/test/native/my_test_op.cc

diff --git a/tensorflow/contrib/boosted_trees/examples/boston.py b/tensorflow/contrib/boosted_trees/examples/boston.py
deleted file mode 100644
index 0cb9e956ef..0000000000
--- a/tensorflow/contrib/boosted_trees/examples/boston.py
+++ /dev/null
@@ -1,155 +0,0 @@
-# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-r"""Demonstrates a regression on Boston housing data.
-
-  This example demonstrates how to run experiments with TF Boosted Trees on
-  a regression dataset. We split all the data into 20% test and 80% train,
-  and are using l2 loss and l2 regularization.
-
-  Example Usage:
-
-  python tensorflow/contrib/boosted_trees/examples/boston.py \
-  --batch_size=404 --output_dir="/tmp/boston" --depth=4 --learning_rate=0.1 \
-  --num_eval_steps=1 --num_trees=500 --l2=4 \
-  --vmodule=training_ops=1
-
-  When training is done, mean squared error on eval data is reported.
-  Point tensorboard to the directory for the run to see how the training
-  progresses:
-
-  tensorboard --logdir=/tmp/boston
-
-"""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import argparse
-import sys
-import tensorflow as tf
-from tensorflow.contrib.boosted_trees.estimator_batch.estimator import GradientBoostedDecisionTreeRegressor
-from tensorflow.contrib.boosted_trees.proto import learner_pb2
-from tensorflow.contrib.layers.python.layers import feature_column
-from tensorflow.contrib.learn import learn_runner
-
-_TEST_SPLIT_RATIO = 0.2
-_TEST_SPLIT_SEED = 42
-_BOSTON_NUM_FEATURES = 13
-
-
-# Main config - creates a TF Boosted Trees Estimator based on flags.
-def _get_tfbt(output_dir, feature_cols):
-  """Configures TF Boosted Trees estimator based on flags."""
-  learner_config = learner_pb2.LearnerConfig()
-
-  learner_config.learning_rate_tuner.fixed.learning_rate = FLAGS.learning_rate
-  learner_config.regularization.l1 = 0.0
-  # Set the regularization per instance in such a way that
-  # regularization for the full training data is equal to l2 flag.
-  learner_config.regularization.l2 = FLAGS.l2 / FLAGS.batch_size
-  learner_config.constraints.max_tree_depth = FLAGS.depth
-  learner_config.growing_mode = learner_pb2.LearnerConfig.WHOLE_TREE
-
-  run_config = tf.contrib.learn.RunConfig(save_checkpoints_secs=300)
-
-  # Create a TF Boosted trees regression estimator.
-  estimator = GradientBoostedDecisionTreeRegressor(
-      learner_config=learner_config,
-      # For the WHOLE_TREE strategy, set the examples_per_layer to be equal to
-      # batch size.
-      examples_per_layer=FLAGS.batch_size,
-      feature_columns=feature_cols,
-      label_dimension=1,
-      model_dir=output_dir,
-      num_trees=FLAGS.num_trees,
-      center_bias=False,
-      config=run_config)
-  return estimator
-
-
-def _make_experiment_fn(output_dir):
-  """Creates experiment for gradient boosted decision trees."""
-  (x_train, y_train), (x_test,
-                       y_test) = tf.keras.datasets.boston_housing.load_data()
-
-  train_input_fn = tf.estimator.inputs.numpy_input_fn(
-      x={"x": x_train},
-      y=y_train,
-      batch_size=FLAGS.batch_size,
-      num_epochs=None,
-      shuffle=True)
-
-  eval_input_fn = tf.estimator.inputs.numpy_input_fn(
-      x={"x": x_test}, y=y_test, num_epochs=1, shuffle=False)
-
-  feature_columns = [
-      feature_column.real_valued_column("x", dimension=_BOSTON_NUM_FEATURES)
-  ]
-
-  return tf.contrib.learn.Experiment(
-      estimator=_get_tfbt(output_dir, feature_columns),
-      train_input_fn=train_input_fn,
-      eval_input_fn=eval_input_fn,
-      train_steps=None,
-      eval_steps=FLAGS.num_eval_steps,
-      eval_metrics=None)
-
-
-def main(unused_argv):
-  learn_runner.run(
-      experiment_fn=_make_experiment_fn,
-      output_dir=FLAGS.output_dir,
-      schedule="train_and_evaluate")
-
-
-if __name__ == "__main__":
-  tf.logging.set_verbosity(tf.logging.INFO)
-  parser = argparse.ArgumentParser()
-  # Define the list of flags that users can change.
-  parser.add_argument(
-      "--batch_size",
-      type=int,
-      default=1000,
-      help="The batch size for reading data.")
-  parser.add_argument(
-      "--output_dir",
-      type=str,
-      required=True,
-      help="Choose the dir for the output.")
-  parser.add_argument(
-      "--num_eval_steps",
-      type=int,
-      default=1,
-      help="The number of steps to run evaluation for.")
-  # Flags for gradient boosted trees config.
-  parser.add_argument(
-      "--depth", type=int, default=4, help="Maximum depth of weak learners.")
-  parser.add_argument(
-      "--l2", type=float, default=1.0, help="l2 regularization per batch.")
-  parser.add_argument(
-      "--learning_rate",
-      type=float,
-      default=0.1,
-      help="Learning rate (shrinkage weight) with which each new tree is added."
-  )
-  parser.add_argument(
-      "--num_trees",
-      type=int,
-      default=None,
-      required=True,
-      help="Number of trees to grow before stopping.")
-
-  FLAGS, unparsed = parser.parse_known_args()
-  tf.app.run(main=main, argv=[sys.argv[0]] + unparsed)
diff --git a/tensorflow/contrib/boosted_trees/examples/mnist.py b/tensorflow/contrib/boosted_trees/examples/mnist.py
index a3b1cb5154..7e34d2f2d3 100644
--- a/tensorflow/contrib/boosted_trees/examples/mnist.py
+++ b/tensorflow/contrib/boosted_trees/examples/mnist.py
@@ -129,8 +129,8 @@ def _get_tfbt(output_dir):
 def _make_experiment_fn(output_dir):
   """Creates experiment for gradient boosted decision trees."""
   data = tf.contrib.learn.datasets.mnist.load_mnist()
-  train_input_fn = get_input_fn(data.train, FLAGS.batch_size)
-  eval_input_fn = get_input_fn(data.validation, FLAGS.eval_batch_size)
+  train_input_fn = get_input_fn(data.train, batch_size=256)
+  eval_input_fn = get_input_fn(data.validation, batch_size=5000)
 
   return tf.contrib.learn.Experiment(
       estimator=_get_tfbt(output_dir),
diff --git a/tensorflow/java/BUILD b/tensorflow/java/BUILD
index a380bc2c71..9de79af7d2 100644
--- a/tensorflow/java/BUILD
+++ b/tensorflow/java/BUILD
@@ -10,9 +10,8 @@ load(":src/gen/gen_ops.bzl", "tf_java_op_gen_srcjar")
 load(
     "//tensorflow:tensorflow.bzl",
     "tf_binary_additional_srcs",
-    "tf_cc_binary",
     "tf_copts",
-    "tf_custom_op_library",
+    "tf_cc_binary",
     "tf_java_test",
 )
 
@@ -181,16 +180,10 @@ tf_java_test(
     ],
 )
 
-tf_custom_op_library(
-    name = "my_test_op.so",
-    srcs = ["src/test/native/my_test_op.cc"],
-)
-
 tf_java_test(
     name = "TensorFlowTest",
     size = "small",
     srcs = ["src/test/java/org/tensorflow/TensorFlowTest.java"],
-    data = [":my_test_op.so"],
     javacopts = JAVACOPTS,
     test_class = "org.tensorflow.TensorFlowTest",
     deps = [
diff --git a/tensorflow/java/src/main/java/org/tensorflow/TensorFlow.java b/tensorflow/java/src/main/java/org/tensorflow/TensorFlow.java
index c90655f25d..c21214b763 100644
--- a/tensorflow/java/src/main/java/org/tensorflow/TensorFlow.java
+++ b/tensorflow/java/src/main/java/org/tensorflow/TensorFlow.java
@@ -29,36 +29,6 @@ public final class TensorFlow {
    */
   public static native byte[] registeredOpList();
 
-  /**
-   * Load the dynamic library in filename and register the operations and kernels present in that
-   * library.
-   *
-   * @param filename Path of the dynamic library containing operations and kernels to load.
-   * @return Serialized bytes of the <a
-   *     href="https://www.tensorflow.org/code/tensorflow/core/framework/op_def.proto">OpList</a>
-   *     protocol buffer message defining the operations defined in the library.
-   * @throws UnsatisfiedLinkError if filename cannot be loaded.
-   */
-  public static byte[] loadLibrary(String filename) {
-    long h = 0;
-    try {
-      h = libraryLoad(filename);
-    } catch (RuntimeException e) {
-      throw new UnsatisfiedLinkError(e.getMessage());
-    }
-    try {
-      return libraryOpList(h);
-    } finally {
-      libraryDelete(h);
-    }
-  }
-
-  private static native long libraryLoad(String filename);
-
-  private static native void libraryDelete(long handle);
-
-  private static native byte[] libraryOpList(long handle);
-
   private TensorFlow() {}
 
   /** Load the TensorFlow runtime C library. */
diff --git a/tensorflow/java/src/main/native/tensorflow_jni.cc b/tensorflow/java/src/main/native/tensorflow_jni.cc
index 946ab502d1..c553582e38 100644
--- a/tensorflow/java/src/main/native/tensorflow_jni.cc
+++ b/tensorflow/java/src/main/native/tensorflow_jni.cc
@@ -14,10 +14,7 @@ limitations under the License.
 ==============================================================================*/
 
 #include "tensorflow/java/src/main/native/tensorflow_jni.h"
-
-#include <limits>
 #include "tensorflow/c/c_api.h"
-#include "tensorflow/java/src/main/native/exception_jni.h"
 
 JNIEXPORT jstring JNICALL Java_org_tensorflow_TensorFlow_version(JNIEnv* env,
                                                                  jclass clazz) {
@@ -33,35 +30,3 @@ Java_org_tensorflow_TensorFlow_registeredOpList(JNIEnv* env, jclass clazz) {
   TF_DeleteBuffer(buf);
   return ret;
 }
-
-JNIEXPORT jlong JNICALL Java_org_tensorflow_TensorFlow_libraryLoad(
-    JNIEnv* env, jclass clazz, jstring filename) {
-  TF_Status* status = TF_NewStatus();
-  const char* cname = env->GetStringUTFChars(filename, nullptr);
-  TF_Library* h = TF_LoadLibrary(cname, status);
-  throwExceptionIfNotOK(env, status);
-  env->ReleaseStringUTFChars(filename, cname);
-  TF_DeleteStatus(status);
-  return reinterpret_cast<jlong>(h);
-}
-
-JNIEXPORT void JNICALL Java_org_tensorflow_TensorFlow_libraryDelete(
-    JNIEnv* env, jclass clazz, jlong handle) {
-  if (handle != 0) {
-    TF_DeleteLibraryHandle(reinterpret_cast<TF_Library*>(handle));
-  }
-}
-
-JNIEXPORT jbyteArray JNICALL Java_org_tensorflow_TensorFlow_libraryOpList(
-    JNIEnv* env, jclass clazz, jlong handle) {
-  TF_Buffer buf = TF_GetOpList(reinterpret_cast<TF_Library*>(handle));
-  if (buf.length > std::numeric_limits<jint>::max()) {
-    throwException(env, kIndexOutOfBoundsException,
-                   "Serialized OpList is too large for a byte[] array");
-    return nullptr;
-  }
-  auto ret_len = static_cast<jint>(buf.length);
-  jbyteArray ret = env->NewByteArray(ret_len);
-  env->SetByteArrayRegion(ret, 0, ret_len, static_cast<const jbyte*>(buf.data));
-  return ret;
-}
diff --git a/tensorflow/java/src/main/native/tensorflow_jni.h b/tensorflow/java/src/main/native/tensorflow_jni.h
index c0c9322020..ecd9b15828 100644
--- a/tensorflow/java/src/main/native/tensorflow_jni.h
+++ b/tensorflow/java/src/main/native/tensorflow_jni.h
@@ -27,7 +27,7 @@ extern "C" {
  *  Method:    version
  *  Signature: ()Ljava/lang/String;
  */
-JNIEXPORT jstring JNICALL Java_org_tensorflow_TensorFlow_version(JNIEnv *,
+JNIEXPORT jstring JNICALL Java_org_tensorflow_TensorFlow_version(JNIEnv*,
                                                                  jclass);
 
 /*
@@ -36,33 +36,7 @@ JNIEXPORT jstring JNICALL Java_org_tensorflow_TensorFlow_version(JNIEnv *,
  * Signature: ()[B
  */
 JNIEXPORT jbyteArray JNICALL
-Java_org_tensorflow_TensorFlow_registeredOpList(JNIEnv *, jclass);
-
-/*
- * Class:     org_tensorflow_TensorFlow
- * Method:    libraryLoad
- * Signature: (Ljava/lang/String;)J
- */
-JNIEXPORT jlong JNICALL Java_org_tensorflow_TensorFlow_libraryLoad(JNIEnv *,
-                                                                   jclass,
-                                                                   jstring);
-
-/*
- * Class:     org_tensorflow_TensorFlow
- * Method:    libraryDelete
- * Signature: (J)V
- */
-JNIEXPORT void JNICALL Java_org_tensorflow_TensorFlow_libraryDelete(JNIEnv *,
-                                                                    jclass,
-                                                                    jlong);
-
-/*
- * Class:     org_tensorflow_TensorFlow
- * Method:    libraryOpList
- * Signature: (J)[B
- */
-JNIEXPORT jbyteArray JNICALL
-Java_org_tensorflow_TensorFlow_libraryOpList(JNIEnv *, jclass, jlong);
+Java_org_tensorflow_TensorFlow_registeredOpList(JNIEnv*, jclass);
 
 #ifdef __cplusplus
 }  // extern "C"
diff --git a/tensorflow/java/src/test/java/org/tensorflow/TensorFlowTest.java b/tensorflow/java/src/test/java/org/tensorflow/TensorFlowTest.java
index b1fa3f0d7e..a31ea900d1 100644
--- a/tensorflow/java/src/test/java/org/tensorflow/TensorFlowTest.java
+++ b/tensorflow/java/src/test/java/org/tensorflow/TensorFlowTest.java
@@ -16,7 +16,6 @@ limitations under the License.
 package org.tensorflow;
 
 import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
 
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -37,26 +36,4 @@ public class TensorFlowTest {
     // was not sorted out. Revisit? Till then, at least exercise the code.
     assertTrue(TensorFlow.registeredOpList().length > 0);
   }
-
-  @Test
-  public void loadLibrary() {
-    // TODO(ashankar): This tell will fail when built with --config=monolithic.
-    // Figure out how we can ignore the test in that case.
-    try (Graph g = new Graph()) {
-      // Build a graph with an unrecognized operation.
-      try {
-        g.opBuilder("MyTest", "MyTest").build();
-        fail("should not be able to construct graphs with unregistered ops");
-      } catch (IllegalArgumentException e) {
-        // expected exception
-      }
-
-      // Load the library containing the operation.
-      byte[] opList = TensorFlow.loadLibrary("tensorflow/java/my_test_op.so");
-      assertTrue(opList.length > 0);
-
-      // Now graph building should succeed.
-      g.opBuilder("MyTest", "MyTest").build();
-    }
-  }
 }
diff --git a/tensorflow/java/src/test/native/my_test_op.cc b/tensorflow/java/src/test/native/my_test_op.cc
deleted file mode 100644
index eb755901ed..0000000000
--- a/tensorflow/java/src/test/native/my_test_op.cc
+++ /dev/null
@@ -1,21 +0,0 @@
-/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#include "tensorflow/core/framework/common_shape_fns.h"
-#include "tensorflow/core/framework/op.h"
-
-REGISTER_OP("MyTest")
-    .Doc("Custom operation for testing.")
-    .SetShapeFn(tensorflow::shape_inference::UnknownShape);
diff --git a/tensorflow/python/eager/execute.py b/tensorflow/python/eager/execute.py
index 04634daba4..8bb4c0687d 100644
--- a/tensorflow/python/eager/execute.py
+++ b/tensorflow/python/eager/execute.py
@@ -168,31 +168,27 @@ def make_tensor(v, arg_name):
 
 def args_to_matching_eager(l, ctx, default_dtype=None):
   """Convert sequence `l` to eager same-type Tensors."""
-  EagerTensor = ops.EagerTensor  # pylint: disable=invalid-name
-  if all(isinstance(x, EagerTensor) for x in l):
-    return l[0].dtype, l
   # TODO(josh11b): Could we do a better job if we also passed in the
   # allowed dtypes when that was known?
 
   # Is some input already a Tensor with a dtype?
   dtype = None
   for t in l:
-    if isinstance(t, EagerTensor):
+    if isinstance(t, ops.EagerTensor):
       dtype = t.dtype
       break
 
-  internal_convert_to_tensor = ops.internal_convert_to_tensor
   if dtype is None:
     # Infer a dtype based on the first value, and use that dtype for the
     # remaining values.
     ret = []
     for t in l:
-      ret.append(internal_convert_to_tensor(
+      ret.append(ops.internal_convert_to_tensor(
           t, dtype, preferred_dtype=default_dtype, ctx=ctx))
       if dtype is None:
         dtype = ret[-1].dtype
   else:
-    ret = [internal_convert_to_tensor(t, dtype, ctx=ctx) for t in l]
+    ret = [ops.internal_convert_to_tensor(t, dtype, ctx=ctx) for t in l]
 
   return dtype, ret
 
diff --git a/tensorflow/python/layers/base.py b/tensorflow/python/layers/base.py
index 1e11d1ae8d..9e7cdd493f 100644
--- a/tensorflow/python/layers/base.py
+++ b/tensorflow/python/layers/base.py
@@ -112,10 +112,8 @@ class Layer(object):
     self._per_input_losses = {}
     self._per_input_updates = {}
     self._dtype = None if dtype is None else dtypes.as_dtype(dtype).name
-    call_fn_args = estimator_util.fn_args(self.call)
-    self._compute_previous_mask = ('mask' in call_fn_args or
-                                   hasattr(self, 'compute_mask'))
-    self._call_has_scope_arg = 'scope' in call_fn_args
+    self._compute_previous_mask = ('mask' in estimator_util.fn_args(self.call)
+                                   or hasattr(self, 'compute_mask'))
 
     # These lists will be filled via successive calls
     # to self._add_inbound_node().
@@ -557,15 +555,7 @@ class Layer(object):
             self.build(input_shapes[0])
           else:
             self.build(input_shapes)
-        try:
-          # Note: not all sub-classes of Layer call Layer.__init__ (especially
-          # the ones under tensorflow/python/keras). Hence we recompute this
-          # attribute here if it is not set.
-          # TODO(agarwal): Fix the sub-classes and avoid this complexity.
-          call_has_scope_arg = self._call_has_scope_arg
-        except AttributeError:
-          call_has_scope_arg = 'scope' in estimator_util.fn_args(self.call)
-        if call_has_scope_arg:
+        if 'scope' in estimator_util.fn_args(self.call):
           kwargs['scope'] = scope
         # Check input assumptions set after layer building, e.g. input shape.
         if in_graph_mode:
@@ -1443,10 +1433,8 @@ class Network(Layer):
     self._activity_regularizer = None
     self._scope = next(vs.variable_scope(None, default_name=base_name).gen)
     self._base_name = base_name
-    call_fn_args = estimator_util.fn_args(self.call)
-    self._compute_previous_mask = ('mask' in call_fn_args or
-                                   hasattr(self, 'compute_mask'))
-    self._call_has_scope_arg = 'scope' in call_fn_args
+    self._compute_previous_mask = ('mask' in estimator_util.fn_args(self.call)
+                                   or hasattr(self, 'compute_mask'))
 
     # This acts just like the `trainable` attribute of any layer instance.
     # It does not affect users of the underlying layers, only users of the
diff --git a/tensorflow/python/layers/normalization.py b/tensorflow/python/layers/normalization.py
index ebcf397625..0521129b27 100644
--- a/tensorflow/python/layers/normalization.py
+++ b/tensorflow/python/layers/normalization.py
@@ -330,7 +330,7 @@ class BatchNormalization(base.Layer):
                                       lambda: self._one_minus_decay,
                                       lambda: 0.)
     else:
-      one_minus_decay = ops.convert_to_tensor(self._one_minus_decay)
+      one_minus_decay = self._one_minus_decay
     if training_value or training_value is None:
       mean_update = self._assign_moving_average(self.moving_mean, mean,
                                                 one_minus_decay)
diff --git a/tensorflow/python/ops/math_ops.py b/tensorflow/python/ops/math_ops.py
index 9383d72f14..131f3724eb 100644
--- a/tensorflow/python/ops/math_ops.py
+++ b/tensorflow/python/ops/math_ops.py
@@ -2317,10 +2317,6 @@ def conj(x, name=None):
   Raises:
     TypeError: If `x` is not a numeric tensor.
   """
-  if isinstance(x, ops.Tensor):
-    dt = x.dtype
-    if dt.is_floating or dt.is_integer:
-      return x
   with ops.name_scope(name, "Conj", [x]) as name:
     x = ops.convert_to_tensor(x, name="x")
     if x.dtype.is_complex or x.dtype == dtypes.variant:
diff --git a/tensorflow/python/ops/resource_variable_ops.py b/tensorflow/python/ops/resource_variable_ops.py
index 4ef9b05d51..bf4759e9ee 100644
--- a/tensorflow/python/ops/resource_variable_ops.py
+++ b/tensorflow/python/ops/resource_variable_ops.py
@@ -540,8 +540,16 @@ class ResourceVariable(variables.Variable):
      the read operation.
     """
     with ops.name_scope("Read"):
-      # Ensure we read the variable in the same device as the handle.
-      with ops.device(self._handle_device):
+      # In graph mode, ensure we read the variable in the same device as the
+      # handle. In eager mode, however, this sometimes tries to read a GPU
+      # variable in the CPU because the handle is host memory. For now, then, we
+      # need to skip the device block in eager. TODO(apassos): eager should have
+      # separate notions of device and memory, so handle.device can be GPU while
+      # handle.memory_space is always CPU.
+      if context.in_graph_mode():
+        with ops.device(self._handle_device):
+          value = self._read_variable_op()
+      else:
         value = self._read_variable_op()
     # Return an identity so it can get placed on whatever device the context
     # specifies instead of the device where the variable is.
-- 
GitLab


From c41dbc3c1832bc6c3662d4d942d095baa1fb49c9 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 4 Oct 2017 13:16:05 -0700
Subject: [PATCH 011/909] Adding TF Boosted trees regression example on boston
 dataset, minor fix for mnist example.

PiperOrigin-RevId: 171052367
---
 .../contrib/boosted_trees/examples/boston.py  | 155 ++++++++++++++++++
 .../contrib/boosted_trees/examples/mnist.py   |   4 +-
 2 files changed, 157 insertions(+), 2 deletions(-)
 create mode 100644 tensorflow/contrib/boosted_trees/examples/boston.py

diff --git a/tensorflow/contrib/boosted_trees/examples/boston.py b/tensorflow/contrib/boosted_trees/examples/boston.py
new file mode 100644
index 0000000000..0cb9e956ef
--- /dev/null
+++ b/tensorflow/contrib/boosted_trees/examples/boston.py
@@ -0,0 +1,155 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+r"""Demonstrates a regression on Boston housing data.
+
+  This example demonstrates how to run experiments with TF Boosted Trees on
+  a regression dataset. We split all the data into 20% test and 80% train,
+  and are using l2 loss and l2 regularization.
+
+  Example Usage:
+
+  python tensorflow/contrib/boosted_trees/examples/boston.py \
+  --batch_size=404 --output_dir="/tmp/boston" --depth=4 --learning_rate=0.1 \
+  --num_eval_steps=1 --num_trees=500 --l2=4 \
+  --vmodule=training_ops=1
+
+  When training is done, mean squared error on eval data is reported.
+  Point tensorboard to the directory for the run to see how the training
+  progresses:
+
+  tensorboard --logdir=/tmp/boston
+
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import argparse
+import sys
+import tensorflow as tf
+from tensorflow.contrib.boosted_trees.estimator_batch.estimator import GradientBoostedDecisionTreeRegressor
+from tensorflow.contrib.boosted_trees.proto import learner_pb2
+from tensorflow.contrib.layers.python.layers import feature_column
+from tensorflow.contrib.learn import learn_runner
+
+_TEST_SPLIT_RATIO = 0.2
+_TEST_SPLIT_SEED = 42
+_BOSTON_NUM_FEATURES = 13
+
+
+# Main config - creates a TF Boosted Trees Estimator based on flags.
+def _get_tfbt(output_dir, feature_cols):
+  """Configures TF Boosted Trees estimator based on flags."""
+  learner_config = learner_pb2.LearnerConfig()
+
+  learner_config.learning_rate_tuner.fixed.learning_rate = FLAGS.learning_rate
+  learner_config.regularization.l1 = 0.0
+  # Set the regularization per instance in such a way that
+  # regularization for the full training data is equal to l2 flag.
+  learner_config.regularization.l2 = FLAGS.l2 / FLAGS.batch_size
+  learner_config.constraints.max_tree_depth = FLAGS.depth
+  learner_config.growing_mode = learner_pb2.LearnerConfig.WHOLE_TREE
+
+  run_config = tf.contrib.learn.RunConfig(save_checkpoints_secs=300)
+
+  # Create a TF Boosted trees regression estimator.
+  estimator = GradientBoostedDecisionTreeRegressor(
+      learner_config=learner_config,
+      # For the WHOLE_TREE strategy, set the examples_per_layer to be equal to
+      # batch size.
+      examples_per_layer=FLAGS.batch_size,
+      feature_columns=feature_cols,
+      label_dimension=1,
+      model_dir=output_dir,
+      num_trees=FLAGS.num_trees,
+      center_bias=False,
+      config=run_config)
+  return estimator
+
+
+def _make_experiment_fn(output_dir):
+  """Creates experiment for gradient boosted decision trees."""
+  (x_train, y_train), (x_test,
+                       y_test) = tf.keras.datasets.boston_housing.load_data()
+
+  train_input_fn = tf.estimator.inputs.numpy_input_fn(
+      x={"x": x_train},
+      y=y_train,
+      batch_size=FLAGS.batch_size,
+      num_epochs=None,
+      shuffle=True)
+
+  eval_input_fn = tf.estimator.inputs.numpy_input_fn(
+      x={"x": x_test}, y=y_test, num_epochs=1, shuffle=False)
+
+  feature_columns = [
+      feature_column.real_valued_column("x", dimension=_BOSTON_NUM_FEATURES)
+  ]
+
+  return tf.contrib.learn.Experiment(
+      estimator=_get_tfbt(output_dir, feature_columns),
+      train_input_fn=train_input_fn,
+      eval_input_fn=eval_input_fn,
+      train_steps=None,
+      eval_steps=FLAGS.num_eval_steps,
+      eval_metrics=None)
+
+
+def main(unused_argv):
+  learn_runner.run(
+      experiment_fn=_make_experiment_fn,
+      output_dir=FLAGS.output_dir,
+      schedule="train_and_evaluate")
+
+
+if __name__ == "__main__":
+  tf.logging.set_verbosity(tf.logging.INFO)
+  parser = argparse.ArgumentParser()
+  # Define the list of flags that users can change.
+  parser.add_argument(
+      "--batch_size",
+      type=int,
+      default=1000,
+      help="The batch size for reading data.")
+  parser.add_argument(
+      "--output_dir",
+      type=str,
+      required=True,
+      help="Choose the dir for the output.")
+  parser.add_argument(
+      "--num_eval_steps",
+      type=int,
+      default=1,
+      help="The number of steps to run evaluation for.")
+  # Flags for gradient boosted trees config.
+  parser.add_argument(
+      "--depth", type=int, default=4, help="Maximum depth of weak learners.")
+  parser.add_argument(
+      "--l2", type=float, default=1.0, help="l2 regularization per batch.")
+  parser.add_argument(
+      "--learning_rate",
+      type=float,
+      default=0.1,
+      help="Learning rate (shrinkage weight) with which each new tree is added."
+  )
+  parser.add_argument(
+      "--num_trees",
+      type=int,
+      default=None,
+      required=True,
+      help="Number of trees to grow before stopping.")
+
+  FLAGS, unparsed = parser.parse_known_args()
+  tf.app.run(main=main, argv=[sys.argv[0]] + unparsed)
diff --git a/tensorflow/contrib/boosted_trees/examples/mnist.py b/tensorflow/contrib/boosted_trees/examples/mnist.py
index 7e34d2f2d3..a3b1cb5154 100644
--- a/tensorflow/contrib/boosted_trees/examples/mnist.py
+++ b/tensorflow/contrib/boosted_trees/examples/mnist.py
@@ -129,8 +129,8 @@ def _get_tfbt(output_dir):
 def _make_experiment_fn(output_dir):
   """Creates experiment for gradient boosted decision trees."""
   data = tf.contrib.learn.datasets.mnist.load_mnist()
-  train_input_fn = get_input_fn(data.train, batch_size=256)
-  eval_input_fn = get_input_fn(data.validation, batch_size=5000)
+  train_input_fn = get_input_fn(data.train, FLAGS.batch_size)
+  eval_input_fn = get_input_fn(data.validation, FLAGS.eval_batch_size)
 
   return tf.contrib.learn.Experiment(
       estimator=_get_tfbt(output_dir),
-- 
GitLab


From cc8ee6c0f5270de5ef2baa0b21c44b0319813548 Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Wed, 4 Oct 2017 13:26:11 -0700
Subject: [PATCH 012/909] Fast path for tf.conj when it should be pass-through.

PiperOrigin-RevId: 171053662
---
 tensorflow/python/ops/math_ops.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tensorflow/python/ops/math_ops.py b/tensorflow/python/ops/math_ops.py
index 131f3724eb..9383d72f14 100644
--- a/tensorflow/python/ops/math_ops.py
+++ b/tensorflow/python/ops/math_ops.py
@@ -2317,6 +2317,10 @@ def conj(x, name=None):
   Raises:
     TypeError: If `x` is not a numeric tensor.
   """
+  if isinstance(x, ops.Tensor):
+    dt = x.dtype
+    if dt.is_floating or dt.is_integer:
+      return x
   with ops.name_scope(name, "Conj", [x]) as name:
     x = ops.convert_to_tensor(x, name="x")
     if x.dtype.is_complex or x.dtype == dtypes.variant:
-- 
GitLab


From e7c53698e09f63e6268888d0b9ebe779ce28a1e7 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 4 Oct 2017 13:26:47 -0700
Subject: [PATCH 013/909] Internal cleanup

PiperOrigin-RevId: 171053770
---
 tensorflow/python/eager/execute.py            | 10 ++++++---
 tensorflow/python/layers/base.py              | 22 ++++++++++++++-----
 tensorflow/python/layers/normalization.py     |  2 +-
 .../python/ops/resource_variable_ops.py       | 12 ++--------
 4 files changed, 27 insertions(+), 19 deletions(-)

diff --git a/tensorflow/python/eager/execute.py b/tensorflow/python/eager/execute.py
index 8bb4c0687d..04634daba4 100644
--- a/tensorflow/python/eager/execute.py
+++ b/tensorflow/python/eager/execute.py
@@ -168,27 +168,31 @@ def make_tensor(v, arg_name):
 
 def args_to_matching_eager(l, ctx, default_dtype=None):
   """Convert sequence `l` to eager same-type Tensors."""
+  EagerTensor = ops.EagerTensor  # pylint: disable=invalid-name
+  if all(isinstance(x, EagerTensor) for x in l):
+    return l[0].dtype, l
   # TODO(josh11b): Could we do a better job if we also passed in the
   # allowed dtypes when that was known?
 
   # Is some input already a Tensor with a dtype?
   dtype = None
   for t in l:
-    if isinstance(t, ops.EagerTensor):
+    if isinstance(t, EagerTensor):
       dtype = t.dtype
       break
 
+  internal_convert_to_tensor = ops.internal_convert_to_tensor
   if dtype is None:
     # Infer a dtype based on the first value, and use that dtype for the
     # remaining values.
     ret = []
     for t in l:
-      ret.append(ops.internal_convert_to_tensor(
+      ret.append(internal_convert_to_tensor(
           t, dtype, preferred_dtype=default_dtype, ctx=ctx))
       if dtype is None:
         dtype = ret[-1].dtype
   else:
-    ret = [ops.internal_convert_to_tensor(t, dtype, ctx=ctx) for t in l]
+    ret = [internal_convert_to_tensor(t, dtype, ctx=ctx) for t in l]
 
   return dtype, ret
 
diff --git a/tensorflow/python/layers/base.py b/tensorflow/python/layers/base.py
index 9e7cdd493f..1e11d1ae8d 100644
--- a/tensorflow/python/layers/base.py
+++ b/tensorflow/python/layers/base.py
@@ -112,8 +112,10 @@ class Layer(object):
     self._per_input_losses = {}
     self._per_input_updates = {}
     self._dtype = None if dtype is None else dtypes.as_dtype(dtype).name
-    self._compute_previous_mask = ('mask' in estimator_util.fn_args(self.call)
-                                   or hasattr(self, 'compute_mask'))
+    call_fn_args = estimator_util.fn_args(self.call)
+    self._compute_previous_mask = ('mask' in call_fn_args or
+                                   hasattr(self, 'compute_mask'))
+    self._call_has_scope_arg = 'scope' in call_fn_args
 
     # These lists will be filled via successive calls
     # to self._add_inbound_node().
@@ -555,7 +557,15 @@ class Layer(object):
             self.build(input_shapes[0])
           else:
             self.build(input_shapes)
-        if 'scope' in estimator_util.fn_args(self.call):
+        try:
+          # Note: not all sub-classes of Layer call Layer.__init__ (especially
+          # the ones under tensorflow/python/keras). Hence we recompute this
+          # attribute here if it is not set.
+          # TODO(agarwal): Fix the sub-classes and avoid this complexity.
+          call_has_scope_arg = self._call_has_scope_arg
+        except AttributeError:
+          call_has_scope_arg = 'scope' in estimator_util.fn_args(self.call)
+        if call_has_scope_arg:
           kwargs['scope'] = scope
         # Check input assumptions set after layer building, e.g. input shape.
         if in_graph_mode:
@@ -1433,8 +1443,10 @@ class Network(Layer):
     self._activity_regularizer = None
     self._scope = next(vs.variable_scope(None, default_name=base_name).gen)
     self._base_name = base_name
-    self._compute_previous_mask = ('mask' in estimator_util.fn_args(self.call)
-                                   or hasattr(self, 'compute_mask'))
+    call_fn_args = estimator_util.fn_args(self.call)
+    self._compute_previous_mask = ('mask' in call_fn_args or
+                                   hasattr(self, 'compute_mask'))
+    self._call_has_scope_arg = 'scope' in call_fn_args
 
     # This acts just like the `trainable` attribute of any layer instance.
     # It does not affect users of the underlying layers, only users of the
diff --git a/tensorflow/python/layers/normalization.py b/tensorflow/python/layers/normalization.py
index 0521129b27..ebcf397625 100644
--- a/tensorflow/python/layers/normalization.py
+++ b/tensorflow/python/layers/normalization.py
@@ -330,7 +330,7 @@ class BatchNormalization(base.Layer):
                                       lambda: self._one_minus_decay,
                                       lambda: 0.)
     else:
-      one_minus_decay = self._one_minus_decay
+      one_minus_decay = ops.convert_to_tensor(self._one_minus_decay)
     if training_value or training_value is None:
       mean_update = self._assign_moving_average(self.moving_mean, mean,
                                                 one_minus_decay)
diff --git a/tensorflow/python/ops/resource_variable_ops.py b/tensorflow/python/ops/resource_variable_ops.py
index bf4759e9ee..4ef9b05d51 100644
--- a/tensorflow/python/ops/resource_variable_ops.py
+++ b/tensorflow/python/ops/resource_variable_ops.py
@@ -540,16 +540,8 @@ class ResourceVariable(variables.Variable):
      the read operation.
     """
     with ops.name_scope("Read"):
-      # In graph mode, ensure we read the variable in the same device as the
-      # handle. In eager mode, however, this sometimes tries to read a GPU
-      # variable in the CPU because the handle is host memory. For now, then, we
-      # need to skip the device block in eager. TODO(apassos): eager should have
-      # separate notions of device and memory, so handle.device can be GPU while
-      # handle.memory_space is always CPU.
-      if context.in_graph_mode():
-        with ops.device(self._handle_device):
-          value = self._read_variable_op()
-      else:
+      # Ensure we read the variable in the same device as the handle.
+      with ops.device(self._handle_device):
         value = self._read_variable_op()
     # Return an identity so it can get placed on whatever device the context
     # specifies instead of the device where the variable is.
-- 
GitLab


From 70fc9bf9b668adebe20ef6d1f7a0e182d7d02cc4 Mon Sep 17 00:00:00 2001
From: Asim Shankar <ashankar@google.com>
Date: Wed, 4 Oct 2017 13:33:07 -0700
Subject: [PATCH 014/909] Java: Add support for loading op libraries
 dynamically.

This change adds the equivalent of tf.load_op_library in Python to Java.
(https://github.com/tensorflow/tensorflow/commit/5c7f9e316d8c7735308a217310350d416d7498cc
 was required to make this possible)

Though, TensorFlow.loadLibrary() is likely to fail on Windows as symbols
required by custom op libraries (those exported by the tensorflow_framework library)
are not exported by the monolithic JNI library yet.

This should help with #10454 and #13476

PiperOrigin-RevId: 171054707
---
 tensorflow/java/BUILD                         |  9 ++++-
 .../main/java/org/tensorflow/TensorFlow.java  | 30 ++++++++++++++++
 .../java/src/main/native/tensorflow_jni.cc    | 35 +++++++++++++++++++
 .../java/src/main/native/tensorflow_jni.h     | 30 ++++++++++++++--
 .../java/org/tensorflow/TensorFlowTest.java   | 23 ++++++++++++
 tensorflow/java/src/test/native/my_test_op.cc | 21 +++++++++++
 6 files changed, 145 insertions(+), 3 deletions(-)
 create mode 100644 tensorflow/java/src/test/native/my_test_op.cc

diff --git a/tensorflow/java/BUILD b/tensorflow/java/BUILD
index 9de79af7d2..a380bc2c71 100644
--- a/tensorflow/java/BUILD
+++ b/tensorflow/java/BUILD
@@ -10,8 +10,9 @@ load(":src/gen/gen_ops.bzl", "tf_java_op_gen_srcjar")
 load(
     "//tensorflow:tensorflow.bzl",
     "tf_binary_additional_srcs",
-    "tf_copts",
     "tf_cc_binary",
+    "tf_copts",
+    "tf_custom_op_library",
     "tf_java_test",
 )
 
@@ -180,10 +181,16 @@ tf_java_test(
     ],
 )
 
+tf_custom_op_library(
+    name = "my_test_op.so",
+    srcs = ["src/test/native/my_test_op.cc"],
+)
+
 tf_java_test(
     name = "TensorFlowTest",
     size = "small",
     srcs = ["src/test/java/org/tensorflow/TensorFlowTest.java"],
+    data = [":my_test_op.so"],
     javacopts = JAVACOPTS,
     test_class = "org.tensorflow.TensorFlowTest",
     deps = [
diff --git a/tensorflow/java/src/main/java/org/tensorflow/TensorFlow.java b/tensorflow/java/src/main/java/org/tensorflow/TensorFlow.java
index c21214b763..c90655f25d 100644
--- a/tensorflow/java/src/main/java/org/tensorflow/TensorFlow.java
+++ b/tensorflow/java/src/main/java/org/tensorflow/TensorFlow.java
@@ -29,6 +29,36 @@ public final class TensorFlow {
    */
   public static native byte[] registeredOpList();
 
+  /**
+   * Load the dynamic library in filename and register the operations and kernels present in that
+   * library.
+   *
+   * @param filename Path of the dynamic library containing operations and kernels to load.
+   * @return Serialized bytes of the <a
+   *     href="https://www.tensorflow.org/code/tensorflow/core/framework/op_def.proto">OpList</a>
+   *     protocol buffer message defining the operations defined in the library.
+   * @throws UnsatisfiedLinkError if filename cannot be loaded.
+   */
+  public static byte[] loadLibrary(String filename) {
+    long h = 0;
+    try {
+      h = libraryLoad(filename);
+    } catch (RuntimeException e) {
+      throw new UnsatisfiedLinkError(e.getMessage());
+    }
+    try {
+      return libraryOpList(h);
+    } finally {
+      libraryDelete(h);
+    }
+  }
+
+  private static native long libraryLoad(String filename);
+
+  private static native void libraryDelete(long handle);
+
+  private static native byte[] libraryOpList(long handle);
+
   private TensorFlow() {}
 
   /** Load the TensorFlow runtime C library. */
diff --git a/tensorflow/java/src/main/native/tensorflow_jni.cc b/tensorflow/java/src/main/native/tensorflow_jni.cc
index c553582e38..946ab502d1 100644
--- a/tensorflow/java/src/main/native/tensorflow_jni.cc
+++ b/tensorflow/java/src/main/native/tensorflow_jni.cc
@@ -14,7 +14,10 @@ limitations under the License.
 ==============================================================================*/
 
 #include "tensorflow/java/src/main/native/tensorflow_jni.h"
+
+#include <limits>
 #include "tensorflow/c/c_api.h"
+#include "tensorflow/java/src/main/native/exception_jni.h"
 
 JNIEXPORT jstring JNICALL Java_org_tensorflow_TensorFlow_version(JNIEnv* env,
                                                                  jclass clazz) {
@@ -30,3 +33,35 @@ Java_org_tensorflow_TensorFlow_registeredOpList(JNIEnv* env, jclass clazz) {
   TF_DeleteBuffer(buf);
   return ret;
 }
+
+JNIEXPORT jlong JNICALL Java_org_tensorflow_TensorFlow_libraryLoad(
+    JNIEnv* env, jclass clazz, jstring filename) {
+  TF_Status* status = TF_NewStatus();
+  const char* cname = env->GetStringUTFChars(filename, nullptr);
+  TF_Library* h = TF_LoadLibrary(cname, status);
+  throwExceptionIfNotOK(env, status);
+  env->ReleaseStringUTFChars(filename, cname);
+  TF_DeleteStatus(status);
+  return reinterpret_cast<jlong>(h);
+}
+
+JNIEXPORT void JNICALL Java_org_tensorflow_TensorFlow_libraryDelete(
+    JNIEnv* env, jclass clazz, jlong handle) {
+  if (handle != 0) {
+    TF_DeleteLibraryHandle(reinterpret_cast<TF_Library*>(handle));
+  }
+}
+
+JNIEXPORT jbyteArray JNICALL Java_org_tensorflow_TensorFlow_libraryOpList(
+    JNIEnv* env, jclass clazz, jlong handle) {
+  TF_Buffer buf = TF_GetOpList(reinterpret_cast<TF_Library*>(handle));
+  if (buf.length > std::numeric_limits<jint>::max()) {
+    throwException(env, kIndexOutOfBoundsException,
+                   "Serialized OpList is too large for a byte[] array");
+    return nullptr;
+  }
+  auto ret_len = static_cast<jint>(buf.length);
+  jbyteArray ret = env->NewByteArray(ret_len);
+  env->SetByteArrayRegion(ret, 0, ret_len, static_cast<const jbyte*>(buf.data));
+  return ret;
+}
diff --git a/tensorflow/java/src/main/native/tensorflow_jni.h b/tensorflow/java/src/main/native/tensorflow_jni.h
index ecd9b15828..c0c9322020 100644
--- a/tensorflow/java/src/main/native/tensorflow_jni.h
+++ b/tensorflow/java/src/main/native/tensorflow_jni.h
@@ -27,7 +27,7 @@ extern "C" {
  *  Method:    version
  *  Signature: ()Ljava/lang/String;
  */
-JNIEXPORT jstring JNICALL Java_org_tensorflow_TensorFlow_version(JNIEnv*,
+JNIEXPORT jstring JNICALL Java_org_tensorflow_TensorFlow_version(JNIEnv *,
                                                                  jclass);
 
 /*
@@ -36,7 +36,33 @@ JNIEXPORT jstring JNICALL Java_org_tensorflow_TensorFlow_version(JNIEnv*,
  * Signature: ()[B
  */
 JNIEXPORT jbyteArray JNICALL
-Java_org_tensorflow_TensorFlow_registeredOpList(JNIEnv*, jclass);
+Java_org_tensorflow_TensorFlow_registeredOpList(JNIEnv *, jclass);
+
+/*
+ * Class:     org_tensorflow_TensorFlow
+ * Method:    libraryLoad
+ * Signature: (Ljava/lang/String;)J
+ */
+JNIEXPORT jlong JNICALL Java_org_tensorflow_TensorFlow_libraryLoad(JNIEnv *,
+                                                                   jclass,
+                                                                   jstring);
+
+/*
+ * Class:     org_tensorflow_TensorFlow
+ * Method:    libraryDelete
+ * Signature: (J)V
+ */
+JNIEXPORT void JNICALL Java_org_tensorflow_TensorFlow_libraryDelete(JNIEnv *,
+                                                                    jclass,
+                                                                    jlong);
+
+/*
+ * Class:     org_tensorflow_TensorFlow
+ * Method:    libraryOpList
+ * Signature: (J)[B
+ */
+JNIEXPORT jbyteArray JNICALL
+Java_org_tensorflow_TensorFlow_libraryOpList(JNIEnv *, jclass, jlong);
 
 #ifdef __cplusplus
 }  // extern "C"
diff --git a/tensorflow/java/src/test/java/org/tensorflow/TensorFlowTest.java b/tensorflow/java/src/test/java/org/tensorflow/TensorFlowTest.java
index a31ea900d1..b1fa3f0d7e 100644
--- a/tensorflow/java/src/test/java/org/tensorflow/TensorFlowTest.java
+++ b/tensorflow/java/src/test/java/org/tensorflow/TensorFlowTest.java
@@ -16,6 +16,7 @@ limitations under the License.
 package org.tensorflow;
 
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
 
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -36,4 +37,26 @@ public class TensorFlowTest {
     // was not sorted out. Revisit? Till then, at least exercise the code.
     assertTrue(TensorFlow.registeredOpList().length > 0);
   }
+
+  @Test
+  public void loadLibrary() {
+    // TODO(ashankar): This tell will fail when built with --config=monolithic.
+    // Figure out how we can ignore the test in that case.
+    try (Graph g = new Graph()) {
+      // Build a graph with an unrecognized operation.
+      try {
+        g.opBuilder("MyTest", "MyTest").build();
+        fail("should not be able to construct graphs with unregistered ops");
+      } catch (IllegalArgumentException e) {
+        // expected exception
+      }
+
+      // Load the library containing the operation.
+      byte[] opList = TensorFlow.loadLibrary("tensorflow/java/my_test_op.so");
+      assertTrue(opList.length > 0);
+
+      // Now graph building should succeed.
+      g.opBuilder("MyTest", "MyTest").build();
+    }
+  }
 }
diff --git a/tensorflow/java/src/test/native/my_test_op.cc b/tensorflow/java/src/test/native/my_test_op.cc
new file mode 100644
index 0000000000..eb755901ed
--- /dev/null
+++ b/tensorflow/java/src/test/native/my_test_op.cc
@@ -0,0 +1,21 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/framework/common_shape_fns.h"
+#include "tensorflow/core/framework/op.h"
+
+REGISTER_OP("MyTest")
+    .Doc("Custom operation for testing.")
+    .SetShapeFn(tensorflow::shape_inference::UnknownShape);
-- 
GitLab


From 53cc63a2d96522ea182a7f6619e25664b1ae6b0d Mon Sep 17 00:00:00 2001
From: Dhananjay Nakrani <dhananjayn@google.com>
Date: Wed, 4 Oct 2017 13:57:18 -0700
Subject: [PATCH 015/909] [part 1] Add support for int32 & int64 in
 RandomPoissonOp.

This computes int32/int64-precision poisson samples with double precision intermediate calculations (same as it's done for `half`) respectively.

part 2 will switch over python calls to new op once forward compatibility period has passed.

PiperOrigin-RevId: 171058336
---
 tensorflow/core/kernels/random_poisson_op.cc  | 75 ++++++++++++++-----
 tensorflow/core/kernels/random_poisson_op.h   |  2 +-
 tensorflow/core/ops/random_ops.cc             | 46 ++++++++++++
 .../kernel_tests/random_poisson_test.py       | 19 +++++
 4 files changed, 122 insertions(+), 20 deletions(-)

diff --git a/tensorflow/core/kernels/random_poisson_op.cc b/tensorflow/core/kernels/random_poisson_op.cc
index b3957cbed6..3f635dbbaf 100644
--- a/tensorflow/core/kernels/random_poisson_op.cc
+++ b/tensorflow/core/kernels/random_poisson_op.cc
@@ -21,6 +21,7 @@ limitations under the License.
 
 #include <algorithm>
 #include <cmath>
+#include <limits>
 #include <memory>
 
 #include "tensorflow/core/framework/op_kernel.h"
@@ -69,34 +70,42 @@ struct PoissonComputeType<Eigen::half> {
   typedef float ComputeType;
 };
 
+template <>
+struct PoissonComputeType<int32> {
+  typedef double ComputeType;
+};
+
+template <>
+struct PoissonComputeType<int64> {
+  typedef double ComputeType;
+};
+
 }  // namespace
 
 namespace functor {
 
-template <typename Device, typename T>
+template <typename Device, typename T, typename U>
 struct PoissonFunctor {
   void operator()(OpKernelContext* ctx, const Device& d, const T* rate_flat,
                   int num_rate, int num_samples,
-                  const random::PhiloxRandom& rng, T* samples_flat);
+                  const random::PhiloxRandom& rng, U* samples_flat);
 };
 
-template <typename T>
-struct PoissonFunctor<CPUDevice, T> {
+template <typename T, typename U>
+struct PoissonFunctor<CPUDevice, T, U> {
   void operator()(OpKernelContext* ctx, const CPUDevice& d, const T* rate_flat,
                   int num_rate, int num_samples,
-                  const random::PhiloxRandom& rng, T* samples_flat) {
+                  const random::PhiloxRandom& rng, U* samples_flat) {
     // Two different algorithms are employed, depending on the size of
     // rate.
     // If rate < 10, we use an algorithm attributed to Knuth:
     // Seminumerical Algorithms. Art of Computer Programming, Volume 2.
     //
     // This algorithm runs in O(rate) time, and will require O(rate)
-    // uniform
-    // variates.
+    // uniform variates.
     //
     // If rate >= 10 we use a transformation-rejection algorithm from
-    // pairs
-    // of uniform random variables due to Hormann.
+    // pairs of uniform random variables due to Hormann.
     // http://www.sciencedirect.com/science/article/pii/0167668793909974
     //
     // The algorithm has an acceptance rate of ~89% for the smallest rate
@@ -154,8 +163,9 @@ struct PoissonFunctor<CPUDevice, T> {
             while (true) {
               UNIFORM(u);
               prod = prod * u;
-              if (prod <= exp_neg_rate) {
-                samples_rate_output[sample_idx * num_rate] = T(x);
+              if (prod <= exp_neg_rate &&
+                  x <= CT(Eigen::NumTraits<U>::highest())) {
+                samples_rate_output[sample_idx * num_rate] = U(x);
                 break;
               }
               x += 1;
@@ -216,13 +226,18 @@ struct PoissonFunctor<CPUDevice, T> {
             CT k = Eigen::numext::floor((CT(2) * a / u_shifted + b) * u + rate +
                                         CT(0.43));
 
+            if (k > CT(Eigen::NumTraits<U>::highest())) {
+              // retry in case of overflow.
+              continue;
+            }
+
             // When alpha * f(G(U)) * G'(U) is close to 1, it is possible to
             // find a rectangle (-u_r, u_r) x (0, v_r) under the curve, such
             // that if v <= v_r and |u| <= u_r, then we can accept.
             // Here v_r = 0.9227 - 3.6224 / (b - 2) and u_r = 0.43.
             if (u_shifted >= CT(0.07) &&
                 v <= CT(0.9277) - CT(3.6224) / (b - CT(2))) {
-              samples_rate_output[sample_idx * num_rate] = T(k);
+              samples_rate_output[sample_idx * num_rate] = U(k);
               break;
             }
 
@@ -235,7 +250,7 @@ struct PoissonFunctor<CPUDevice, T> {
             CT s = log(v * inv_alpha / (a / (u_shifted * u_shifted) + b));
             CT t = -rate + k * log_rate - Eigen::numext::lgamma(k + 1);
             if (s <= t) {
-              samples_rate_output[sample_idx * num_rate] = T(k);
+              samples_rate_output[sample_idx * num_rate] = U(k);
               break;
             }
           }
@@ -280,7 +295,7 @@ struct PoissonFunctor<CPUDevice, T> {
 namespace {
 
 // Samples from one or more Poisson distributions.
-template <typename T>
+template <typename T, typename U>
 class RandomPoissonOp : public OpKernel {
  public:
   explicit RandomPoissonOp(OpKernelConstruction* context) : OpKernel(context) {
@@ -303,13 +318,13 @@ class RandomPoissonOp : public OpKernel {
 
     const auto rate_flat = rate_t.flat<T>().data();
     const int64 num_rate = rate_t.NumElements();
-    auto samples_flat = samples_t->flat<T>().data();
+    auto samples_flat = samples_t->flat<U>().data();
     random::PhiloxRandom rng = generator_.ReserveRandomOutputs(
         num_samples * num_rate, kReservedSamplesPerOutput);
 
-    functor::PoissonFunctor<CPUDevice, T>()(ctx, ctx->eigen_device<CPUDevice>(),
-                                            rate_flat, num_rate, num_samples,
-                                            rng, samples_flat);
+    functor::PoissonFunctor<CPUDevice, T, U>()(
+        ctx, ctx->eigen_device<CPUDevice>(), rate_flat, num_rate, num_samples,
+        rng, samples_flat);
   }
 
  private:
@@ -324,12 +339,34 @@ class RandomPoissonOp : public OpKernel {
 #define REGISTER(TYPE)                                                        \
   REGISTER_KERNEL_BUILDER(                                                    \
       Name("RandomPoisson").Device(DEVICE_CPU).TypeConstraint<TYPE>("dtype"), \
-      RandomPoissonOp<TYPE>);
+      RandomPoissonOp<TYPE, TYPE>);
 
 TF_CALL_half(REGISTER);
 TF_CALL_float(REGISTER);
 TF_CALL_double(REGISTER);
 
+#define REGISTER_V2(RTYPE, OTYPE)                              \
+  REGISTER_KERNEL_BUILDER(Name("RandomPoissonV2")              \
+                              .Device(DEVICE_CPU)              \
+                              .TypeConstraint<RTYPE>("R")      \
+                              .TypeConstraint<OTYPE>("dtype"), \
+                          RandomPoissonOp<RTYPE, OTYPE>);
+
+#define REGISTER_ALL(RTYPE)        \
+  REGISTER_V2(RTYPE, Eigen::half); \
+  REGISTER_V2(RTYPE, float);       \
+  REGISTER_V2(RTYPE, double);      \
+  REGISTER_V2(RTYPE, int32);       \
+  REGISTER_V2(RTYPE, int64);
+
+REGISTER_ALL(Eigen::half);
+REGISTER_ALL(float);
+REGISTER_ALL(double);
+REGISTER_ALL(int32);
+REGISTER_ALL(int64);
+
+#undef REGISTER_ALL
+#undef REGISTER_V2
 #undef REGISTER
 
 }  // end namespace tensorflow
diff --git a/tensorflow/core/kernels/random_poisson_op.h b/tensorflow/core/kernels/random_poisson_op.h
index 6c49acc800..4e9fd62520 100644
--- a/tensorflow/core/kernels/random_poisson_op.h
+++ b/tensorflow/core/kernels/random_poisson_op.h
@@ -21,7 +21,7 @@ namespace tensorflow {
 namespace functor {
 
 // Generic helper functor for the Random Poisson Op.
-template <typename Device, typename T>
+template <typename Device, typename T /* rate */, typename U /* output */>
 struct PoissonFunctor;
 
 }  // namespace functor
diff --git a/tensorflow/core/ops/random_ops.cc b/tensorflow/core/ops/random_ops.cc
index 2e3fdc7c57..eee1ed1d2a 100644
--- a/tensorflow/core/ops/random_ops.cc
+++ b/tensorflow/core/ops/random_ops.cc
@@ -265,6 +265,8 @@ output: A tensor with shape `shape + shape(alpha)`. Each slice
   `alpha[i0, i1, ...iN]`. The dtype of the output matches the dtype of alpha.
 )doc");
 
+// TODO(dhananayn): Deprecate RandomPoisson and switch over to RandomPoissonV2
+// after forward compatibility period has passed.
 REGISTER_OP("RandomPoisson")
     .SetIsStateful()
     .Input("shape: S")
@@ -309,4 +311,48 @@ output: A tensor with shape `shape + shape(rate)`. Each slice
   rate.
 )doc");
 
+REGISTER_OP("RandomPoissonV2")
+    .SetIsStateful()
+    .Input("shape: S")
+    .Input("rate: R")
+    .Output("output: dtype")
+    .Attr("seed: int = 0")
+    .Attr("seed2: int = 0")
+    .Attr("S: {int32, int64}")
+    .Attr("R: {half, float, double, int32, int64} = DT_DOUBLE")
+    .Attr("dtype: {half, float, double, int32, int64} = DT_INT64")
+    .SetShapeFn([](InferenceContext* c) {
+      ShapeHandle out;
+      TF_RETURN_IF_ERROR(c->MakeShapeFromShapeTensor(0, &out));
+      TF_RETURN_IF_ERROR(c->Concatenate(out, c->input(1), &out));
+      c->set_output(0, out);
+      return Status::OK();
+    })
+    .Doc(R"doc(
+Outputs random values from the Poisson distribution(s) described by rate.
+
+This op uses two algorithms, depending on rate. If rate >= 10, then
+the algorithm by Hormann is used to acquire samples via
+transformation-rejection.
+See http://www.sciencedirect.com/science/article/pii/0167668793909974.
+
+Otherwise, Knuth's algorithm is used to acquire samples via multiplying uniform
+random variables.
+See Donald E. Knuth (1969). Seminumerical Algorithms. The Art of Computer
+Programming, Volume 2. Addison Wesley
+
+shape: 1-D integer tensor. Shape of independent samples to draw from each
+  distribution described by the shape parameters given in rate.
+rate: A tensor in which each scalar is a "rate" parameter describing the
+  associated poisson distribution.
+seed: If either `seed` or `seed2` are set to be non-zero, the random number
+  generator is seeded by the given seed.  Otherwise, it is seeded by a
+  random seed.
+seed2: A second seed to avoid seed collision.
+
+output: A tensor with shape `shape + shape(rate)`. Each slice
+  `[:, ..., :, i0, i1, ...iN]` contains the samples drawn for
+  `rate[i0, i1, ...iN]`.
+)doc");
+
 }  // namespace tensorflow
diff --git a/tensorflow/python/kernel_tests/random_poisson_test.py b/tensorflow/python/kernel_tests/random_poisson_test.py
index 107c9bbe14..ca57e380e8 100644
--- a/tensorflow/python/kernel_tests/random_poisson_test.py
+++ b/tensorflow/python/kernel_tests/random_poisson_test.py
@@ -20,9 +20,11 @@ from __future__ import print_function
 import numpy as np
 from six.moves import xrange  # pylint: disable=redefined-builtin
 
+from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import gen_random_ops
 from tensorflow.python.ops import random_ops
 from tensorflow.python.platform import test
 from tensorflow.python.platform import tf_logging
@@ -179,6 +181,23 @@ class RandomPoissonTest(test.TestCase):
         seed=12345)
     self.assertIs(None, rnd.get_shape().ndims)
 
+  def testDTypeCombinationsV2(self):
+    """Tests random_poisson_v2() for all supported dtype combinations."""
+    # All supported dtypes by random_poisson_v2().
+    supported_dtypes = [
+        dtypes.float16, dtypes.float32, dtypes.float64, dtypes.int32,
+        dtypes.int64
+    ]
+
+    with self.test_session():
+      for lam_dt in supported_dtypes:
+        for out_dt in supported_dtypes:
+          # TODO(dhananjayn): Change this to use random_poisson() after
+          # switching it to RandomPoissonV2.
+          gen_random_ops.random_poisson_v2(
+              [10], constant_op.constant([1], dtype=lam_dt),
+              dtype=out_dt).eval()
+
 
 if __name__ == "__main__":
   test.main()
-- 
GitLab


From 3b4477000da27f4039ce275ad66f03e770c72a78 Mon Sep 17 00:00:00 2001
From: Saurabh Saxena <srbs@google.com>
Date: Wed, 4 Oct 2017 14:29:09 -0700
Subject: [PATCH 016/909] Make VariantTensorData::tensors_size() const.

PiperOrigin-RevId: 171063397
---
 tensorflow/core/framework/variant_tensor_data.cc | 2 +-
 tensorflow/core/framework/variant_tensor_data.h  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/framework/variant_tensor_data.cc b/tensorflow/core/framework/variant_tensor_data.cc
index 93fac46e8e..82479193d2 100644
--- a/tensorflow/core/framework/variant_tensor_data.cc
+++ b/tensorflow/core/framework/variant_tensor_data.cc
@@ -28,7 +28,7 @@ VariantTensorData::VariantTensorData(const VariantTensorDataProto& proto) {
 
 VariantTensorData::~VariantTensorData() {}
 
-int VariantTensorData::tensors_size() { return tensors_.size(); }
+int VariantTensorData::tensors_size() const { return tensors_.size(); }
 
 const Tensor& VariantTensorData::tensors(int index) const {
   return tensors_[index];
diff --git a/tensorflow/core/framework/variant_tensor_data.h b/tensorflow/core/framework/variant_tensor_data.h
index 4ee3df89fb..6e04879494 100644
--- a/tensorflow/core/framework/variant_tensor_data.h
+++ b/tensorflow/core/framework/variant_tensor_data.h
@@ -61,7 +61,7 @@ class VariantTensorData {
   }
 
   // Tensors contained within objects being serialized.
-  int tensors_size();
+  int tensors_size() const;
   const Tensor& tensors(int index) const;
   std::vector<Tensor> tensors();
   Tensor* add_tensors();
-- 
GitLab


From 39565c0cbcd89a96a678e3453d3ab608d1293db1 Mon Sep 17 00:00:00 2001
From: Martin Wicke <wicke@google.com>
Date: Wed, 4 Oct 2017 14:47:53 -0700
Subject: [PATCH 017/909] Publish train_and_evaluate and associated classes.

PiperOrigin-RevId: 171066379
---
 tensorflow/python/estimator/estimator_lib.py  | 18 ++++++++
 tensorflow/python/estimator/training.py       |  4 --
 .../tensorflow.estimator.-eval-spec.pbtxt     | 43 +++++++++++++++++++
 .../tensorflow.estimator.-exporter.pbtxt      | 16 +++++++
 ...ensorflow.estimator.-latest-exporter.pbtxt | 18 ++++++++
 .../tensorflow.estimator.-train-spec.pbtxt    | 27 ++++++++++++
 .../api/golden/tensorflow.estimator.pbtxt     | 20 +++++++++
 7 files changed, 142 insertions(+), 4 deletions(-)
 create mode 100644 tensorflow/tools/api/golden/tensorflow.estimator.-eval-spec.pbtxt
 create mode 100644 tensorflow/tools/api/golden/tensorflow.estimator.-exporter.pbtxt
 create mode 100644 tensorflow/tools/api/golden/tensorflow.estimator.-latest-exporter.pbtxt
 create mode 100644 tensorflow/tools/api/golden/tensorflow.estimator.-train-spec.pbtxt

diff --git a/tensorflow/python/estimator/estimator_lib.py b/tensorflow/python/estimator/estimator_lib.py
index 8e7d966564..a5b3faeffb 100644
--- a/tensorflow/python/estimator/estimator_lib.py
+++ b/tensorflow/python/estimator/estimator_lib.py
@@ -29,29 +29,47 @@ from tensorflow.python.estimator.canned.parsing_utils import classifier_parse_ex
 from tensorflow.python.estimator.canned.parsing_utils import regressor_parse_example_spec
 from tensorflow.python.estimator.estimator import Estimator
 from tensorflow.python.estimator.export import export_lib as export
+from tensorflow.python.estimator.exporter import Exporter
+from tensorflow.python.estimator.exporter import LatestExporter
 from tensorflow.python.estimator.inputs import inputs
 from tensorflow.python.estimator.model_fn import EstimatorSpec
 from tensorflow.python.estimator.model_fn import ModeKeys
 from tensorflow.python.estimator.run_config import RunConfig
+from tensorflow.python.estimator.training import EvalSpec
+from tensorflow.python.estimator.training import train_and_evaluate
+from tensorflow.python.estimator.training import TrainSpec
+
 
 from tensorflow.python.util.all_util import remove_undocumented
 # pylint: enable=unused-import,line-too-long,wildcard-import
 
 _allowed_symbols = [
+    # Canned Estimators
     'DNNClassifier',
     'DNNRegressor',
     'DNNLinearCombinedClassifier',
     'DNNLinearCombinedRegressor',
     'LinearClassifier',
     'LinearRegressor',
+
+    # I/O
     'classifier_parse_example_spec',
     'regressor_parse_example_spec',
     'inputs',
     'export',
+
+    # Estimator
     'Estimator',
     'EstimatorSpec',
     'ModeKeys',
     'RunConfig',
+
+    # Training utilities
+    'train_and_evaluate',
+    'EvalSpec',
+    'TrainSpec',
+    'Exporter',
+    'LatestExporter',
 ]
 
 remove_undocumented(__name__, allowed_exception_list=_allowed_symbols)
diff --git a/tensorflow/python/estimator/training.py b/tensorflow/python/estimator/training.py
index 953e970eea..1bed19760b 100644
--- a/tensorflow/python/estimator/training.py
+++ b/tensorflow/python/estimator/training.py
@@ -479,10 +479,6 @@ class _StopAtSecsHook(session_run_hook.SessionRunHook):
       run_context.request_stop()
 
 
-class UnimplementedError(Exception):
-  pass
-
-
 class _TrainingExecutor(object):
   """The executor to run `Estimator` training and evaluation.
 
diff --git a/tensorflow/tools/api/golden/tensorflow.estimator.-eval-spec.pbtxt b/tensorflow/tools/api/golden/tensorflow.estimator.-eval-spec.pbtxt
new file mode 100644
index 0000000000..db83ba1bd8
--- /dev/null
+++ b/tensorflow/tools/api/golden/tensorflow.estimator.-eval-spec.pbtxt
@@ -0,0 +1,43 @@
+path: "tensorflow.estimator.EvalSpec"
+tf_class {
+  is_instance: "<class \'tensorflow.python.estimator.training.EvalSpec\'>"
+  is_instance: "<class \'tensorflow.python.estimator.training.EvalSpec\'>"
+  is_instance: "<type \'tuple\'>"
+  member {
+    name: "exporters"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "hooks"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_fn"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "start_delay_secs"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "steps"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "throttle_secs"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+  }
+  member_method {
+    name: "count"
+  }
+  member_method {
+    name: "index"
+  }
+}
diff --git a/tensorflow/tools/api/golden/tensorflow.estimator.-exporter.pbtxt b/tensorflow/tools/api/golden/tensorflow.estimator.-exporter.pbtxt
new file mode 100644
index 0000000000..c69e4c7a30
--- /dev/null
+++ b/tensorflow/tools/api/golden/tensorflow.estimator.-exporter.pbtxt
@@ -0,0 +1,16 @@
+path: "tensorflow.estimator.Exporter"
+tf_class {
+  is_instance: "<class \'tensorflow.python.estimator.exporter.Exporter\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "name"
+    mtype: "<class \'abc.abstractproperty\'>"
+  }
+  member_method {
+    name: "__init__"
+  }
+  member_method {
+    name: "export"
+    argspec: "args=[\'self\', \'estimator\', \'export_path\', \'checkpoint_path\', \'eval_result\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/tensorflow.estimator.-latest-exporter.pbtxt b/tensorflow/tools/api/golden/tensorflow.estimator.-latest-exporter.pbtxt
new file mode 100644
index 0000000000..c3f98f84b8
--- /dev/null
+++ b/tensorflow/tools/api/golden/tensorflow.estimator.-latest-exporter.pbtxt
@@ -0,0 +1,18 @@
+path: "tensorflow.estimator.LatestExporter"
+tf_class {
+  is_instance: "<class \'tensorflow.python.estimator.exporter.LatestExporter\'>"
+  is_instance: "<class \'tensorflow.python.estimator.exporter.Exporter\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name\', \'serving_input_fn\', \'assets_extra\', \'as_text\', \'exports_to_keep\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'5\'], "
+  }
+  member_method {
+    name: "export"
+    argspec: "args=[\'self\', \'estimator\', \'export_path\', \'checkpoint_path\', \'eval_result\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/tensorflow.estimator.-train-spec.pbtxt b/tensorflow/tools/api/golden/tensorflow.estimator.-train-spec.pbtxt
new file mode 100644
index 0000000000..7d2f77438a
--- /dev/null
+++ b/tensorflow/tools/api/golden/tensorflow.estimator.-train-spec.pbtxt
@@ -0,0 +1,27 @@
+path: "tensorflow.estimator.TrainSpec"
+tf_class {
+  is_instance: "<class \'tensorflow.python.estimator.training.TrainSpec\'>"
+  is_instance: "<class \'tensorflow.python.estimator.training.TrainSpec\'>"
+  is_instance: "<type \'tuple\'>"
+  member {
+    name: "hooks"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "input_fn"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "max_steps"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+  }
+  member_method {
+    name: "count"
+  }
+  member_method {
+    name: "index"
+  }
+}
diff --git a/tensorflow/tools/api/golden/tensorflow.estimator.pbtxt b/tensorflow/tools/api/golden/tensorflow.estimator.pbtxt
index 07b04810b5..25e94a14a6 100644
--- a/tensorflow/tools/api/golden/tensorflow.estimator.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.estimator.pbtxt
@@ -24,6 +24,18 @@ tf_module {
     name: "EstimatorSpec"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "EvalSpec"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "Exporter"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "LatestExporter"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "LinearClassifier"
     mtype: "<type \'type\'>"
@@ -40,6 +52,10 @@ tf_module {
     name: "RunConfig"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "TrainSpec"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "export"
     mtype: "<type \'module\'>"
@@ -56,4 +72,8 @@ tf_module {
     name: "regressor_parse_example_spec"
     argspec: "args=[\'feature_columns\', \'label_key\', \'label_dtype\', \'label_default\', \'label_dimension\', \'weight_column\'], varargs=None, keywords=None, defaults=[\"<dtype: \'float32\'>\", \'None\', \'1\', \'None\'], "
   }
+  member_method {
+    name: "train_and_evaluate"
+    argspec: "args=[\'estimator\', \'train_spec\', \'eval_spec\'], varargs=None, keywords=None, defaults=None"
+  }
 }
-- 
GitLab


From 4486b4f69b55633274f7903158d680bf2e9eabff Mon Sep 17 00:00:00 2001
From: Akshay Agrawal <akshayka@google.com>
Date: Wed, 4 Oct 2017 14:52:13 -0700
Subject: [PATCH 018/909] Make graph_callable compatible with functions that do
 not return anything

PiperOrigin-RevId: 171067061
---
 tensorflow/python/eager/graph_callable.py     |  7 +++++-
 .../python/eager/graph_callable_test.py       | 23 +++++++++++++++++++
 2 files changed, 29 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/eager/graph_callable.py b/tensorflow/python/eager/graph_callable.py
index a6131bea08..5933da7865 100644
--- a/tensorflow/python/eager/graph_callable.py
+++ b/tensorflow/python/eager/graph_callable.py
@@ -324,7 +324,9 @@ def _graph_callable_internal(func, shape_and_dtypes):
           captures):
         func_outputs = func(*func_inputs)
       outputs_list = nest.flatten(func_outputs)
-      output_shapes = [x.shape for x in outputs_list if x is not None]
+      if len(outputs_list) == 1 and outputs_list[0] is None:
+        outputs_list = []
+      output_shapes = [x.shape for x in outputs_list]
       if not all(isinstance(x, tf_ops.Tensor) for x in outputs_list):
         raise ValueError("Found non-tensor output in %s" % str(outputs_list))
       initializing_operations = tmp_graph.get_operations()
@@ -420,6 +422,9 @@ def graph_callable(shape_and_dtypes):
   Note that the wrapped function is not allowed to change the values of the
   variables, just use them.
 
+  The return value of the wrapped function must be one of the following:
+  (1) None,  (2) a Tensor, or (3) a possibly nested sequence of Tensors.
+
   Example:
 
   ```python
diff --git a/tensorflow/python/eager/graph_callable_test.py b/tensorflow/python/eager/graph_callable_test.py
index 54a1c73dfd..cee6adec04 100644
--- a/tensorflow/python/eager/graph_callable_test.py
+++ b/tensorflow/python/eager/graph_callable_test.py
@@ -45,6 +45,29 @@ class GraphCallableTest(test.TestCase):
     self.assertEqual(
         3, my_function(constant_op.constant(2, dtype=dtypes.float32)).numpy())
 
+  def testFunctionWithoutReturnValue(self):
+
+    @graph_callable.graph_callable(
+        [graph_callable.ShapeAndDtype(shape=(), dtype=dtypes.float32)])
+    def my_function(x):
+      v = variable_scope.get_variable(
+          "v", initializer=init_ops.zeros_initializer(), shape=())
+      v.assign(x)
+
+    my_function(constant_op.constant(4, dtype=dtypes.float32))
+    self.assertEqual(4, my_function.variables[0].read_value().numpy())
+
+  def testFunctionWithoutReturnValueAndArgs(self):
+
+    @graph_callable.graph_callable([])
+    def my_function():
+      v = variable_scope.get_variable(
+          "v", initializer=init_ops.zeros_initializer(), shape=())
+      v.assign(4)
+
+    my_function()
+    self.assertEqual(4, my_function.variables[0].read_value().numpy())
+
   def testVariableAPI(self):
 
     @graph_callable.graph_callable(
-- 
GitLab


From 89df2e336218f7f3ecf2c70f8478c64985345ded Mon Sep 17 00:00:00 2001
From: Igor Saprykin <isaprykin@google.com>
Date: Wed, 4 Oct 2017 15:13:33 -0700
Subject: [PATCH 019/909] Add the 'is_the_final_export' signal to Exporters.
 Use them in training.

When the training ends, the final export is performed via `Exporter.export()` call.  That final export is going to have is_the_final_export parameter being set to true.

If `TrainSpec.max_steps` is `None`, then "when training ends" is undefined.  We are going to train forever.  In that case, `is_the_final_export` is going to be always False.  I added a note about it.

PiperOrigin-RevId: 171070760
---
 tensorflow/python/estimator/exporter.py      | 26 ++++++-
 tensorflow/python/estimator/exporter_test.py | 41 +++++++++-
 tensorflow/python/estimator/training.py      | 37 ++++++---
 tensorflow/python/estimator/training_test.py | 81 ++++++++++++++++++++
 4 files changed, 169 insertions(+), 16 deletions(-)

diff --git a/tensorflow/python/estimator/exporter.py b/tensorflow/python/estimator/exporter.py
index 505820dd93..2faca11f6e 100644
--- a/tensorflow/python/estimator/exporter.py
+++ b/tensorflow/python/estimator/exporter.py
@@ -40,7 +40,8 @@ class Exporter(object):
     pass
 
   @abc.abstractmethod
-  def export(self, estimator, export_path, checkpoint_path, eval_result):
+  def export(self, estimator, export_path, checkpoint_path, eval_result,
+             is_the_final_export):
     """Exports the given `Estimator` to a specific format.
 
     Args:
@@ -48,6 +49,13 @@ class Exporter(object):
       export_path: A string containing a directory where to write the export.
       checkpoint_path: The checkpoint path to export.
       eval_result: The output of `Estimator.evaluate` on this checkpoint.
+      is_the_final_export: This boolean is True when this is an export in the
+        end of training.  It is False for the intermediate exports during
+        the training.
+
+        When passing `Exporter` to `tf.estimator.train_and_evaluate`
+        `is_the_final_export` is always False if `TrainSpec.max_steps` is
+        `None`.
 
     Returns:
       The string path to the exported directory or `None` if export is skipped.
@@ -66,7 +74,8 @@ class LatestExporter(Exporter):
                serving_input_fn,
                assets_extra=None,
                as_text=False,
-               exports_to_keep=5):
+               exports_to_keep=5,
+               only_the_final_export=False):
     """Create an `Exporter` to use with `tf.estimator.EvalSpec`.
 
     Args:
@@ -86,6 +95,8 @@ class LatestExporter(Exporter):
       exports_to_keep: Number of exports to keep.  Older exports will be
        garbage-collected.  Defaults to 5.  Set to `None` to disable garbage
        collection.
+     only_the_final_export:  Only the final export in the end of training will
+        happen if this is set to True.
 
     Raises:
       ValueError: if any arguments is invalid.
@@ -95,6 +106,8 @@ class LatestExporter(Exporter):
     self._assets_extra = assets_extra
     self._as_text = as_text
     self._exports_to_keep = exports_to_keep
+    self._only_the_final_export = only_the_final_export
+
     if exports_to_keep is not None and exports_to_keep <= 0:
       raise ValueError(
           '`exports_to_keep`, if provided, must be positive number')
@@ -103,7 +116,14 @@ class LatestExporter(Exporter):
   def name(self):
     return self._name
 
-  def export(self, estimator, export_path, checkpoint_path, eval_result):
+  def export(self, estimator, export_path, checkpoint_path, eval_result,
+             is_the_final_export):
+    if not is_the_final_export and self._only_the_final_export:
+      return None
+
+    if is_the_final_export:
+      tf_logging.info('Performing the final export in the end of training.')
+
     export_result = estimator.export_savedmodel(
         export_path,
         self._serving_input_fn,
diff --git a/tensorflow/python/estimator/exporter_test.py b/tensorflow/python/estimator/exporter_test.py
index 2ceff1bfd6..01582ac595 100644
--- a/tensorflow/python/estimator/exporter_test.py
+++ b/tensorflow/python/estimator/exporter_test.py
@@ -42,7 +42,7 @@ class LatestExporterTest(test.TestCase):
           serving_input_fn=_serving_input_fn,
           exports_to_keep=0)
 
-  def test_saved_model_exporter(self):
+  def test_latest_exporter(self):
 
     def _serving_input_fn():
       pass
@@ -60,7 +60,42 @@ class LatestExporterTest(test.TestCase):
     estimator.export_savedmodel.return_value = "export_result_path"
 
     export_result = exporter.export(estimator, export_dir_base,
-                                    "checkpoint_path", {})
+                                    "checkpoint_path", {}, False)
+
+    self.assertEqual("export_result_path", export_result)
+    estimator.export_savedmodel.assert_called_with(
+        export_dir_base,
+        _serving_input_fn,
+        assets_extra={"from/path": "to/path"},
+        as_text=False,
+        checkpoint_path="checkpoint_path")
+
+  def test_only_the_last_export_is_saved(self):
+
+    def _serving_input_fn():
+      pass
+
+    export_dir_base = tempfile.mkdtemp() + "export/"
+    gfile.MkDir(export_dir_base)
+
+    exporter = exporter_lib.LatestExporter(
+        name="latest_exporter",
+        serving_input_fn=_serving_input_fn,
+        assets_extra={"from/path": "to/path"},
+        as_text=False,
+        exports_to_keep=5,
+        only_the_final_export=True)
+    estimator = test.mock.Mock(spec=estimator_lib.Estimator)
+    estimator.export_savedmodel.return_value = "export_result_path"
+
+    export_result = exporter.export(estimator, export_dir_base,
+                                    "checkpoint_path", {}, False)
+
+    self.assertFalse(estimator.export_savedmodel.called)
+    self.assertEqual(None, export_result)
+
+    export_result = exporter.export(estimator, export_dir_base,
+                                    "checkpoint_path", {}, True)
 
     self.assertEqual("export_result_path", export_result)
     estimator.export_savedmodel.assert_called_with(
@@ -93,7 +128,7 @@ class LatestExporterTest(test.TestCase):
     estimator = test.mock.Mock(spec=estimator_lib.Estimator)
     # Garbage collect all but the most recent 2 exports,
     # where recency is determined based on the timestamp directory names.
-    exporter.export(estimator, export_dir_base, None, None)
+    exporter.export(estimator, export_dir_base, None, None, False)
 
     self.assertFalse(gfile.Exists(export_dir_1))
     self.assertFalse(gfile.Exists(export_dir_2))
diff --git a/tensorflow/python/estimator/training.py b/tensorflow/python/estimator/training.py
index 1bed19760b..0a558a67b9 100644
--- a/tensorflow/python/estimator/training.py
+++ b/tensorflow/python/estimator/training.py
@@ -519,8 +519,11 @@ class _TrainingExecutor(object):
     class NewCheckpointListener(
         basic_session_run_hooks.CheckpointSaverListener):
 
-      def __init__(self, estimator, eval_spec):
-        self._evaluator = _TrainingExecutor._Evaluator(estimator, eval_spec)  # pylint: disable=protected-access
+      def __init__(self, estimator, eval_spec, max_training_steps):
+        # pylint: disable=protected-access
+        self._evaluator = _TrainingExecutor._Evaluator(estimator, eval_spec,
+                                                       max_training_steps)
+        # pylint: enable=protected-access
 
       def after_save(self, session, global_step_value):
         del session, global_step_value
@@ -528,8 +531,10 @@ class _TrainingExecutor(object):
 
     # When the underlying `Estimator` object saves a new checkpoint, we would
     # like this callback to be called so that evaluation and export can trigger.
-    saving_listeners = [NewCheckpointListener(self._estimator, self._eval_spec)]
-
+    saving_listeners = [
+        NewCheckpointListener(self._estimator, self._eval_spec,
+                              self._train_spec.max_steps)
+    ]
     return self._start_distributed_training(saving_listeners=saving_listeners)
 
   def run_evaluator(self):
@@ -566,7 +571,8 @@ class _TrainingExecutor(object):
                  'after {} secs (eval_spec.throttle_secs) or training is '
                  'finished.'.format(self._eval_spec.throttle_secs))
 
-    evaluator = _TrainingExecutor._Evaluator(self._estimator, self._eval_spec)
+    evaluator = _TrainingExecutor._Evaluator(self._estimator, self._eval_spec,
+                                             self._train_spec.max_steps)
 
     while True:
       self._estimator.train(
@@ -636,7 +642,8 @@ class _TrainingExecutor(object):
       time.sleep(start_delay_secs)
 
     latest_eval_result = None
-    evaluator = _TrainingExecutor._Evaluator(self._estimator, self._eval_spec)
+    evaluator = _TrainingExecutor._Evaluator(self._estimator, self._eval_spec,
+                                             self._train_spec.max_steps)
 
     while True:
       if latest_eval_result:
@@ -663,11 +670,12 @@ class _TrainingExecutor(object):
   class _Evaluator(object):
     """A helper class to call `Estimator.evaluate` and export model."""
 
-    def __init__(self, estimator, eval_spec):
+    def __init__(self, estimator, eval_spec, max_training_steps):
       self._estimator = estimator
       self._eval_spec = eval_spec
       self._previous_ckpt_path = None
       self._last_warning_time = 0
+      self._max_training_steps = max_training_steps
 
     def evaluate_and_export(self):
       """Evaluate and (maybe) export the current model.
@@ -712,7 +720,14 @@ class _TrainingExecutor(object):
             'Internal error: `Estimator.evaluate` result should have '
             '`global_step` in result. Given {}'.format(eval_result))
 
-      self._export_eval_result(eval_result, latest_ckpt_path)
+      # TODO(isaprykin):  There is a potential race condition here in the
+      #  distributed setting.  The worker job that performs training
+      #  might stop at a later global step value than the evalutor job.
+      is_the_final_export = (eval_result[ops.GraphKeys.GLOBAL_STEP] >=
+                             self._max_training_steps
+                             if self._max_training_steps else False)
+      self._export_eval_result(eval_result, latest_ckpt_path,
+                               is_the_final_export)
 
       self._last_warning_time = 0
       self._previous_ckpt_path = latest_ckpt_path
@@ -725,7 +740,8 @@ class _TrainingExecutor(object):
         logging.warning(message)
         self._last_warning_time = current_time
 
-    def _export_eval_result(self, eval_result, checkpoint_path):
+    def _export_eval_result(self, eval_result, checkpoint_path,
+                            is_the_final_export):
       """Export `eval_result` according to exporters in `EvalSpec`."""
       export_dir_base = os.path.join(
           compat.as_str_any(self._estimator.model_dir),
@@ -738,4 +754,5 @@ class _TrainingExecutor(object):
                 compat.as_str_any(export_dir_base),
                 compat.as_str_any(exporter.name)),
             checkpoint_path=checkpoint_path,
-            eval_result=eval_result)
+            eval_result=eval_result,
+            is_the_final_export=is_the_final_export)
diff --git a/tensorflow/python/estimator/training_test.py b/tensorflow/python/estimator/training_test.py
index e4c400ca7f..08d11d7d25 100644
--- a/tensorflow/python/estimator/training_test.py
+++ b/tensorflow/python/estimator/training_test.py
@@ -802,6 +802,46 @@ class TrainingExecutorRunEvaluatorTest(test.TestCase):
     self.assertEqual(2, mock_est.evaluate.call_count)
     self.assertEqual(2, exporter.export.call_count)
 
+  def test_final_export_is_true_in_the_end(self):
+    training_max_step = 200
+
+    mock_est = test.mock.Mock(spec=estimator_lib.Estimator)
+    mock_est.model_dir = compat.as_bytes(test.get_temp_dir())
+    mock_est.evaluate.side_effect = [
+        {_GLOBAL_STEP_KEY: training_max_step // 2},
+        {_GLOBAL_STEP_KEY: training_max_step}
+    ]
+    mock_est.latest_checkpoint.side_effect = ['path_1', 'path_2']
+
+    mock_train_spec = test.mock.Mock(spec=training.TrainSpec)
+    mock_train_spec.max_steps = training_max_step
+
+    mock_est.times_export_fn_was_called = 0
+    mock_est.times_the_final_export_was_true = 0
+    def export(estimator, export_path, checkpoint_path, eval_result,
+               is_the_final_export):
+      del export_path, checkpoint_path, eval_result
+      estimator.times_export_fn_was_called += 1
+      if is_the_final_export:
+        estimator.times_the_final_export_was_true += 1
+
+    exporter = test.mock.PropertyMock(spec=exporter_lib.Exporter)
+    exporter.name = 'see_how_many_times_export_is_called'
+    exporter.export = export
+
+    eval_spec = training.EvalSpec(
+        input_fn=lambda: 1,
+        start_delay_secs=0,
+        throttle_secs=0,
+        exporters=exporter)
+
+    executor = training._TrainingExecutor(mock_est, mock_train_spec, eval_spec)
+    executor.run_evaluator()
+
+    self.assertEqual(2, mock_est.evaluate.call_count)
+    self.assertEqual(2, mock_est.times_export_fn_was_called)
+    self.assertEqual(1, mock_est.times_the_final_export_was_true)
+
   def test_skip_evaluation_due_to_ckpt(self):
     training_max_step = 200
     mock_est = test.mock.Mock(spec=estimator_lib.Estimator)
@@ -1134,6 +1174,47 @@ class TrainingExecutorRunLocalTest(test.TestCase):
     with self.assertRaisesRegexp(RuntimeError, _STALE_CHECKPOINT_MSG):
       executor.run_local()
 
+  def test_final_export_is_true_in_the_end(self):
+    mock_est = test.mock.Mock(spec=estimator_lib.Estimator, model_dir='path/')
+    mock_est.latest_checkpoint = self.unique_checkpoint_every_time_fn
+
+    mock_est.times_export_fn_was_called = 0
+    mock_est.times_the_final_export_was_true = 0
+    def export(estimator, export_path, checkpoint_path, eval_result,
+               is_the_final_export):
+      del export_path, checkpoint_path, eval_result
+      estimator.times_export_fn_was_called += 1
+      if is_the_final_export:
+        estimator.times_the_final_export_was_true += 1
+
+    exporter = test.mock.PropertyMock(spec=exporter_lib.Exporter)
+    exporter.name = 'see_how_many_times_export_is_called'
+    exporter.export = export
+
+    train_spec = training.TrainSpec(
+        input_fn=lambda: 1, max_steps=300, hooks=[_FakeHook()])
+    eval_spec = training.EvalSpec(
+        input_fn=lambda: 1,
+        hooks=[_FakeHook()],
+        throttle_secs=100,
+        exporters=exporter)
+    # should be called 3 times.
+    mock_est.evaluate.side_effect = [{
+        _GLOBAL_STEP_KEY: train_spec.max_steps - 100
+    }, {
+        _GLOBAL_STEP_KEY: train_spec.max_steps - 50
+    }, {
+        _GLOBAL_STEP_KEY: train_spec.max_steps
+    }]
+
+    executor = training._TrainingExecutor(mock_est, train_spec, eval_spec)
+    executor.run_local()
+
+    self.assertEqual(3, mock_est.train.call_count)
+    self.assertEqual(3, mock_est.evaluate.call_count)
+    self.assertEqual(3, mock_est.times_export_fn_was_called)
+    self.assertEqual(1, mock_est.times_the_final_export_was_true)
+
   def test_train_and_evaluate_args(self):
     mock_est = test.mock.Mock(spec=estimator_lib.Estimator, model_dir='path/')
     mock_est.latest_checkpoint.return_value = 'checkpoint_path/'
-- 
GitLab


From 89aaac4bc3ab5a6c65dfa143e42a8fad02e0223f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 4 Oct 2017 15:14:06 -0700
Subject: [PATCH 020/909] Allow Layer.add_update() in Eager mode.

PiperOrigin-RevId: 171070861
---
 tensorflow/python/layers/base.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/tensorflow/python/layers/base.py b/tensorflow/python/layers/base.py
index 1e11d1ae8d..4cf566bc8b 100644
--- a/tensorflow/python/layers/base.py
+++ b/tensorflow/python/layers/base.py
@@ -225,18 +225,17 @@ class Layer(object):
     The `get_updates_for` method allows to retrieve the updates relevant to a
     specific set of inputs.
 
+    This call is ignored in Eager mode.
+
     Arguments:
       updates: Update op, or list/tuple of update ops.
       inputs: Optional input tensor(s) that the update(s) depend on. Must
         match the `inputs` argument passed to the `__call__` method at the time
         the updates are created. If `None` is passed, the updates are assumed
         to be unconditional, and will apply across all dataflows of the layer.
-
-    Raises:
-      RuntimeError: If called in Eager mode.
     """
     if context.in_eager_mode():
-      raise RuntimeError('Layer.add_update not supported in Eager mode.')
+      return  # Updates already applied when in eager mode.
     updates = _to_list(updates)
     if not updates:
       return
-- 
GitLab


From a02116882de2cfee41afac8e5b85df3cee565aee Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Wed, 4 Oct 2017 15:44:14 -0700
Subject: [PATCH 021/909] [XLA:CPU] Put the HLO name in IR values that hold the
 HLO's value.

PiperOrigin-RevId: 171075449
---
 tensorflow/compiler/xla/service/cpu/ir_emitter.cc | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
index 2a952328a7..1e81a815d8 100644
--- a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
@@ -2833,6 +2833,15 @@ Status IrEmitter::Preprocess(HloInstruction* hlo) {
 }
 
 Status IrEmitter::Postprocess(HloInstruction* hlo) {
+  // Set the name of the emitted llvm::Value to IrName(hlo).  Outfeed and send
+  // the only ops that don't emit a value.
+  if (hlo->opcode() != HloOpcode::kOutfeed &&
+      hlo->opcode() != HloOpcode::kSend) {
+    auto it = emitted_value_.find(hlo);
+    CHECK(it != emitted_value_.end());
+    it->second->setName(AsStringRef(IrName(hlo)));
+  }
+
   if (auto* prof_counter = GetProfileCounterFor(hlo)) {
     profiling_state_.RecordCycleDelta(&ir_builder_, hlo, prof_counter);
   }
-- 
GitLab


From ee0fdc296ca00a3dde3def7dbe18252fa9c736dc Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Wed, 4 Oct 2017 15:44:34 -0700
Subject: [PATCH 022/909] Add noasan tag to estimator_test

PiperOrigin-RevId: 171075499
---
 tensorflow/python/keras/BUILD | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index f29d40f729..f1266cdf9e 100644
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -667,7 +667,10 @@ py_test(
     size = "medium",
     srcs = ["_impl/keras/estimator_test.py"],
     srcs_version = "PY2AND3",
-    tags = ["notsan"],
+    tags = [
+        "noasan",
+        "notsan",
+    ],
     deps = [
         ":keras",
         "//tensorflow/python:client_testlib",
-- 
GitLab


From eba759f74e98342bec09d6d7ddaf9ca638ec6056 Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Wed, 4 Oct 2017 15:44:48 -0700
Subject: [PATCH 023/909] Switch some contextlib.contextmanagers to regular
 objects

Converts just the frequently-accessed scopes in eager mode. @contextlib.contextmanagers create a few extra Python objects via generators and a wrapper class.

PiperOrigin-RevId: 171075529
---
 tensorflow/python/framework/errors_impl.py    |  38 +-
 tensorflow/python/framework/ops.py            | 113 ++--
 tensorflow/python/layers/base.py              |  16 +-
 tensorflow/python/ops/variable_scope.py       | 561 ++++++++++--------
 .../tools/api/golden/tensorflow.errors.pbtxt  |   8 +-
 ...ors.raise_exception_on_not_ok_status.pbtxt |   8 +
 .../tensorflow.keras.backend.name_scope.pbtxt |   9 +
 .../api/golden/tensorflow.keras.backend.pbtxt |   8 +-
 .../api/golden/tensorflow.name_scope.pbtxt    |   9 +
 tensorflow/tools/api/golden/tensorflow.pbtxt  |  16 +-
 .../golden/tensorflow.variable_scope.pbtxt    |   9 +
 11 files changed, 463 insertions(+), 332 deletions(-)
 create mode 100644 tensorflow/tools/api/golden/tensorflow.errors.raise_exception_on_not_ok_status.pbtxt
 create mode 100644 tensorflow/tools/api/golden/tensorflow.keras.backend.name_scope.pbtxt
 create mode 100644 tensorflow/tools/api/golden/tensorflow.name_scope.pbtxt
 create mode 100644 tensorflow/tools/api/golden/tensorflow.variable_scope.pbtxt

diff --git a/tensorflow/python/framework/errors_impl.py b/tensorflow/python/framework/errors_impl.py
index fa956c3d29..c3b2c498c3 100644
--- a/tensorflow/python/framework/errors_impl.py
+++ b/tensorflow/python/framework/errors_impl.py
@@ -18,7 +18,6 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import contextlib
 import traceback
 import warnings
 
@@ -455,17 +454,26 @@ def _make_specific_exception(node_def, op, message, error_code):
     return UnknownError(node_def, op, message, error_code)
 
 
-@contextlib.contextmanager
-def raise_exception_on_not_ok_status():
-  status = c_api_util.ScopedTFStatus()
-  yield status.status
-  try:
-    if c_api.TF_GetCode(status.status) != 0:
-      raise _make_specific_exception(
-          None, None,
-          compat.as_text(c_api.TF_Message(status.status)),
-          c_api.TF_GetCode(status.status))
-  # Delete the underlying status object from memory otherwise it stays alive
-  # as there is a reference to status from this from the traceback due to raise.
-  finally:
-    del status
+# Named like a function for backwards compatibility with the
+# @tf_contextlib.contextmanager version, which was switched to a class to avoid
+# some object creation overhead.
+class raise_exception_on_not_ok_status(object):  # pylint: disable=invalid-name
+  """Context manager to check for C API status."""
+
+  def __enter__(self):
+    self.status = c_api_util.ScopedTFStatus()
+    return self.status.status
+
+  def __exit__(self, type_arg, value_arg, traceback_arg):
+    try:
+      if c_api.TF_GetCode(self.status.status) != 0:
+        raise _make_specific_exception(
+            None, None,
+            compat.as_text(c_api.TF_Message(self.status.status)),
+            c_api.TF_GetCode(self.status.status))
+    # Delete the underlying status object from memory otherwise it stays alive
+    # as there is a reference to status from this from the traceback due to
+    # raise.
+    finally:
+      del self.status
+    return False  # False values do not suppress exceptions
diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index d1744f451e..50aa070985 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -73,9 +73,13 @@ def tensor_id(tensor):
   return tensor._id  # pylint: disable=protected-access
 
 
-@tf_contextlib.contextmanager
-def _null_contextmanager():
-  yield
+class _NullContextmanager(object):
+
+  def __enter__(self):
+    pass
+
+  def __exit__(self, type_arg, value_arg, traceback_arg):
+    return False  # False values do not suppress exceptions
 
 
 def _override_helper(clazz_object, operator, func):
@@ -4263,7 +4267,7 @@ def colocate_with(op, ignore_existing=False):
     if op is not None:
       return device(op.device)
     else:
-      return _null_contextmanager()
+      return _NullContextmanager()
 
 
 def control_dependencies(control_inputs):
@@ -4285,7 +4289,7 @@ def control_dependencies(control_inputs):
   if context.in_graph_mode():
     return get_default_graph().control_dependencies(control_inputs)
   else:
-    return _null_contextmanager()
+    return _NullContextmanager()
 
 
 class _DefaultStack(threading.local):
@@ -4839,10 +4843,11 @@ def get_all_collection_keys():
   return get_default_graph().get_all_collection_keys()
 
 
-# pylint: disable=g-doc-return-or-yield
-@tf_contextlib.contextmanager
-def name_scope(name, default_name=None, values=None):
-  """Returns a context manager for use when defining a Python op.
+# Named like a function for backwards compatibility with the
+# @tf_contextlib.contextmanager version, which was switched to a class to avoid
+# some object creation overhead.
+class name_scope(object):  # pylint: disable=invalid-name
+  """A context manager for use when defining a Python op.
 
   This context manager validates that the given `values` are from the
   same graph, makes that graph the default graph, and pushes a
@@ -4861,48 +4866,64 @@ def name_scope(name, default_name=None, values=None):
       # Define some computation that uses `a`, `b`, and `c`.
       return foo_op(..., name=scope)
   ```
+  """
 
-  Args:
-    name: The name argument that is passed to the op function.
-    default_name: The default name to use if the `name` argument is `None`.
-    values: The list of `Tensor` arguments that are passed to the op function.
+  def __init__(self, name, default_name=None, values=None):
+    """Initialize the context manager.
 
-  Returns:
-    A context manager for use in defining Python ops. Yields the name scope.
+    Args:
+      name: The name argument that is passed to the op function.
+      default_name: The default name to use if the `name` argument is `None`.
+      values: The list of `Tensor` arguments that are passed to the op function.
+    """
+    self._name = default_name if name is None else name
+    self._default_name = default_name
+    self._values = values
+    self._ctx = context.context()
+    self._in_eager_mode = self._ctx.in_eager_mode()
 
-  Raises:
-    ValueError: if neither `name` nor `default_name` is provided
-      but `values` are.
-  """
-  name = default_name if name is None else name
-  ctx = context.context()
-  if ctx.in_eager_mode():
-    old_name = ctx.scope_name
-    if name:
-      scope_name = "%s%s/" % (old_name, name) if old_name else "%s/" % name
-    else:
-      scope_name = ""
-    ctx.scope_name = scope_name
-    try:
-      yield scope_name
-    finally:
-      ctx.scope_name = old_name
-  else:
-    if name is None and values is not None:
-      # We only raise an error if values is not None (provided) because
-      # currently tf.name_scope(None) (values=None then) is sometimes used as an
-      # idiom to reset to top scope.
-      raise ValueError(
-          "At least one of name (%s) and default_name (%s) must be provided." %
-          (name, default_name))
-    if values is None:
-      values = []
-    g = _get_graph_from_inputs(values)
-    with g.as_default(), g.name_scope(name) as scope:
-      yield scope
+  def __enter__(self):
+    """Start the scope block.
 
+    Returns:
+      The scope name.
 
-# pylint: enable=g-doc-return-or-yield
+    Raises:
+      ValueError: if neither `name` nor `default_name` is provided
+        but `values` are.
+    """
+    if self._in_eager_mode:
+      self._old_name = self._ctx.scope_name
+      if self._name:
+        scope_name = (self._old_name + self._name + "/"
+                      if self._old_name else self._name + "/")
+      else:
+        scope_name = ""
+      self._ctx.scope_name = scope_name
+      return scope_name
+    else:
+      if self._name is None and self._values is not None:
+        # We only raise an error if values is not None (provided) because
+        # currently tf.name_scope(None) (values=None then) is sometimes used as
+        # an idiom to reset to top scope.
+        raise ValueError(
+            "At least one of name (%s) and default_name (%s) must be provided."
+            % (self._name, self._default_name))
+      if self._values is None:
+        self._values = []
+      g = _get_graph_from_inputs(self._values)
+      self._g_manager = g.as_default()
+      self._g_manager.__enter__()
+      self._name_scope = g.name_scope(self._name)
+      return self._name_scope.__enter__()
+
+  def __exit__(self, type_arg, value_arg, traceback_arg):
+    if self._in_eager_mode:
+      self._ctx.scope_name = self._old_name
+    else:
+      self._name_scope.__exit__(type_arg, value_arg, traceback_arg)
+      self._g_manager.__exit__(type_arg, value_arg, traceback_arg)
+    return False  # False values do not suppress exceptions
 
 
 def strip_name_scope(name, export_scope):
diff --git a/tensorflow/python/layers/base.py b/tensorflow/python/layers/base.py
index 4cf566bc8b..711ffdfa9c 100644
--- a/tensorflow/python/layers/base.py
+++ b/tensorflow/python/layers/base.py
@@ -136,7 +136,8 @@ class Layer(object):
     # Determine variable scope.
     scope = kwargs.get('_scope')
     if scope:
-      self._scope = next(vs.variable_scope(scope).gen)
+      with vs.variable_scope(scope) as captured_scope:
+        self._scope = captured_scope
     else:
       self._scope = None
 
@@ -402,11 +403,13 @@ class Layer(object):
     if self._scope is None:
       # If constructed with _scope=None, lazy setting of scope.
       if self._reuse:
-        self._scope = next(vs.variable_scope(
-            scope if scope is not None else self._base_name).gen)
+        with vs.variable_scope(
+            scope if scope is not None else self._base_name) as captured_scope:
+          self._scope = captured_scope
       else:
-        self._scope = next(vs.variable_scope(
-            scope, default_name=self._base_name).gen)
+        with vs.variable_scope(
+            scope, default_name=self._base_name) as captured_scope:
+          self._scope = captured_scope
 
   def add_variable(self, name, shape, dtype=None,
                    initializer=None, regularizer=None,
@@ -1440,7 +1443,8 @@ class Network(Layer):
       base_name = _to_snake_case(self.__class__.__name__)
       self._name = _unique_layer_name(base_name)
     self._activity_regularizer = None
-    self._scope = next(vs.variable_scope(None, default_name=base_name).gen)
+    with vs.variable_scope(None, default_name=base_name) as captured_scope:
+      self._scope = captured_scope
     self._base_name = base_name
     call_fn_args = estimator_util.fn_args(self.call)
     self._compute_previous_mask = ('mask' in call_fn_args or
diff --git a/tensorflow/python/ops/variable_scope.py b/tensorflow/python/ops/variable_scope.py
index 33790c5d0a..d0ebfdb85e 100644
--- a/tensorflow/python/ops/variable_scope.py
+++ b/tensorflow/python/ops/variable_scope.py
@@ -1406,139 +1406,162 @@ def _get_partitioned_variable(name,
   # pylint: enable=protected-access
 
 
-@tf_contextlib.contextmanager
-def _pure_variable_scope(name_or_scope,
-                         reuse=None,
-                         initializer=None,
-                         regularizer=None,
-                         caching_device=None,
-                         partitioner=None,
-                         custom_getter=None,
-                         old_name_scope=None,
-                         dtype=dtypes.float32,
-                         use_resource=None,
-                         constraint=None):
-  """Creates a context for the variable_scope, see `variable_scope` for docs.
-
-  Note: this does not create a name scope.
+# Named like a function for compatibility with the previous
+# @tf_contextlib.contextmanager definition.
+class _pure_variable_scope(object):  # pylint: disable=invalid-name
+  """A context for the variable_scope, see `variable_scope` for docs."""
 
-  Args:
-    name_or_scope: `string` or `VariableScope`: the scope to open.
-    reuse: `True` or None, or tf.AUTO_REUSE; if `None`, we inherit the parent
-      scope's reuse flag.
-    initializer: default initializer for variables within this scope.
-    regularizer: default regularizer for variables within this scope.
-    caching_device: default caching device for variables within this scope.
-    partitioner: default partitioner for variables within this scope.
-    custom_getter: default custom getter for variables within this scope.
-    old_name_scope: the original name scope when re-entering a variable scope.
-    dtype: type of the variables within this scope (defaults to `DT_FLOAT`).
-    use_resource: If False, variables in this scope will be regular Variables.
-      If True, experimental ResourceVariables will be creates instead, with
-      well-defined semantics. Defaults to False (will later change to True).
-    constraint: An optional projection function to be applied to the variable
-      after being updated by an `Optimizer` (e.g. used to implement norm
-      constraints or value constraints for layer weights). The function must
-      take as input the unprojected Tensor representing the value of the
-      variable and return the Tensor for the projected value
-      (which must have the same shape). Constraints are not safe to
-      use when doing asynchronous distributed training.
+  def __init__(self,
+               name_or_scope,
+               reuse=None,
+               initializer=None,
+               regularizer=None,
+               caching_device=None,
+               partitioner=None,
+               custom_getter=None,
+               old_name_scope=None,
+               dtype=dtypes.float32,
+               use_resource=None,
+               constraint=None):
+    """Creates a context for the variable_scope, see `variable_scope` for docs.
 
-  Yields:
-    A scope that can be captured and reused.
+    Note: this does not create a name scope.
 
-  Raises:
-    ValueError: when trying to reuse within a create scope, or create within
-      a reuse scope, or if reuse is not `None` or `True`.
-    TypeError: when the types of some arguments are not appropriate.
+    Args:
+      name_or_scope: `string` or `VariableScope`: the scope to open.
+      reuse: `True` or None, or tf.AUTO_REUSE; if `None`, we inherit the parent
+        scope's reuse flag.
+      initializer: default initializer for variables within this scope.
+      regularizer: default regularizer for variables within this scope.
+      caching_device: default caching device for variables within this scope.
+      partitioner: default partitioner for variables within this scope.
+      custom_getter: default custom getter for variables within this scope.
+      old_name_scope: the original name scope when re-entering a variable scope.
+      dtype: type of the variables within this scope (defaults to `DT_FLOAT`).
+      use_resource: If False, variables in this scope will be regular Variables.
+        If True, experimental ResourceVariables will be creates instead, with
+        well-defined semantics. Defaults to False (will later change to True).
+      constraint: An optional projection function to be applied to the variable
+        after being updated by an `Optimizer` (e.g. used to implement norm
+        constraints or value constraints for layer weights). The function must
+        take as input the unprojected Tensor representing the value of the
+        variable and return the Tensor for the projected value
+        (which must have the same shape). Constraints are not safe to
+        use when doing asynchronous distributed training.
+    """
+    self._name_or_scope = name_or_scope
+    self._reuse = reuse
+    self._initializer = initializer
+    self._regularizer = regularizer
+    self._caching_device = caching_device
+    self._partitioner = partitioner
+    self._custom_getter = custom_getter
+    self._old_name_scope = old_name_scope
+    self._dtype = dtype
+    self._use_resource = use_resource
+    self._constraint = constraint
 
-  """
-  get_variable_scope()  # Ensure that a default exists, then get a pointer.
-  # Get the reference to the collection as we want to modify it in place.
-  default_varscope = ops.get_collection_ref(_VARSCOPE_KEY)
-  old = default_varscope[0]
-  var_store = _get_default_variable_store()
-  if isinstance(name_or_scope, VariableScope):
-    new_name = name_or_scope.name
-  else:
-    new_name = old.name + "/" + name_or_scope if old.name else name_or_scope
-  try:
-    var_store.open_variable_scope(new_name)
-    if isinstance(name_or_scope, VariableScope):
-      old_subscopes = copy.copy(var_store.variable_scopes_count)
-      name_scope = name_or_scope._name_scope  # pylint: disable=protected-access
-      # Handler for the case when we jump to a shared scope.
-      #   We create a new VariableScope (default_varscope[0]) that contains
-      #   a copy of the provided shared scope, possibly with changed reuse
-      #   and initializer, if the user requested this.
-      default_varscope[0] = VariableScope(
-          name_or_scope.reuse if not reuse else reuse,
-          name=new_name,
-          initializer=name_or_scope.initializer,
-          regularizer=name_or_scope.regularizer,
-          caching_device=name_or_scope.caching_device,
-          partitioner=name_or_scope.partitioner,
-          dtype=name_or_scope.dtype,
-          custom_getter=name_or_scope.custom_getter,
+  def __enter__(self):
+    """Begins the scope block.
+
+    Returns:
+      A VariableScope.
+    Raises:
+      ValueError: when trying to reuse within a create scope, or create within
+        a reuse scope, or if reuse is not `None` or `True`.
+      TypeError: when the types of some arguments are not appropriate.
+    """
+    get_variable_scope()  # Ensure that a default exists, then get a pointer.
+    # Get the reference to the collection as we want to modify it in place.
+    self._default_varscope = ops.get_collection_ref(_VARSCOPE_KEY)
+    self._old = self._default_varscope[0]
+    self._var_store = _get_default_variable_store()
+    if isinstance(self._name_or_scope, VariableScope):
+      self._new_name = self._name_or_scope.name
+    else:
+      self._new_name = (
+          self._old.name + "/" + self._name_or_scope if self._old.name
+          else self._name_or_scope)
+    self._var_store.open_variable_scope(self._new_name)
+    if isinstance(self._name_or_scope, VariableScope):
+      self._old_subscopes = copy.copy(self._var_store.variable_scopes_count)
+      name_scope = self._name_or_scope._name_scope  # pylint: disable=protected-access
+      # Handler for the case when we jump to a shared scope.  We create a new
+      #   VariableScope (self._default_varscope[0]) that contains a copy of the
+      #   provided shared scope, possibly with changed reuse and initializer, if
+      #   the user requested this.
+      self._default_varscope[0] = VariableScope(
+          self._name_or_scope.reuse if not self._reuse else self._reuse,
+          name=self._new_name,
+          initializer=self._name_or_scope.initializer,
+          regularizer=self._name_or_scope.regularizer,
+          caching_device=self._name_or_scope.caching_device,
+          partitioner=self._name_or_scope.partitioner,
+          dtype=self._name_or_scope.dtype,
+          custom_getter=self._name_or_scope.custom_getter,
           name_scope=name_scope,
-          use_resource=name_or_scope.use_resource,
-          constraint=constraint)
-      if initializer is not None:
-        default_varscope[0].set_initializer(initializer)
-      if regularizer is not None:
-        default_varscope[0].set_regularizer(regularizer)
-      if caching_device is not None:
-        default_varscope[0].set_caching_device(caching_device)
-      if partitioner is not None:
-        default_varscope[0].set_partitioner(partitioner)
-      if custom_getter is not None:
-        default_varscope[0].set_custom_getter(
+          use_resource=self._name_or_scope.use_resource,
+          constraint=self._constraint)
+      if self._initializer is not None:
+        self._default_varscope[0].set_initializer(self._initializer)
+      if self._regularizer is not None:
+        self._default_varscope[0].set_regularizer(self._regularizer)
+      if self._caching_device is not None:
+        self._default_varscope[0].set_caching_device(self._caching_device)
+      if self._partitioner is not None:
+        self._default_varscope[0].set_partitioner(self._partitioner)
+      if self._custom_getter is not None:
+        self._default_varscope[0].set_custom_getter(
             _maybe_wrap_custom_getter(
-                custom_getter, name_or_scope.custom_getter))
-      if dtype is not None:
-        default_varscope[0].set_dtype(dtype)
-      if use_resource is not None:
-        default_varscope[0].set_use_resource(use_resource)
-      yield default_varscope[0]
+                self._custom_getter, self._name_or_scope.custom_getter))
+      if self._dtype is not None:
+        self._default_varscope[0].set_dtype(self._dtype)
+      if self._use_resource is not None:
+        self._default_varscope[0].set_use_resource(self._use_resource)
+      return self._default_varscope[0]
     else:
       # Handler for the case when we just prolong current variable scope.
       #   VariableScope with name extended by the provided one, and inherited
       #   reuse and initializer (except if the user provided values to set).
-      reuse = reuse or old.reuse  # Re-using is inherited by sub-scopes.
-      default_varscope[0] = VariableScope(
-          reuse,
-          name=new_name,
-          initializer=old.initializer,
-          regularizer=old.regularizer,
-          caching_device=old.caching_device,
-          partitioner=old.partitioner,
-          dtype=old.dtype,
-          use_resource=old.use_resource,
-          custom_getter=old.custom_getter,
-          name_scope=old_name_scope or name_or_scope,
-          constraint=constraint)
-      if initializer is not None:
-        default_varscope[0].set_initializer(initializer)
-      if regularizer is not None:
-        default_varscope[0].set_regularizer(regularizer)
-      if caching_device is not None:
-        default_varscope[0].set_caching_device(caching_device)
-      if partitioner is not None:
-        default_varscope[0].set_partitioner(partitioner)
-      if custom_getter is not None:
-        default_varscope[0].set_custom_getter(
-            _maybe_wrap_custom_getter(custom_getter, old.custom_getter))
-      if dtype is not None:
-        default_varscope[0].set_dtype(dtype)
-      if use_resource is not None:
-        default_varscope[0].set_use_resource(use_resource)
-      yield default_varscope[0]
-  finally:
-    var_store.close_variable_subscopes(new_name)
+      self._reuse = (self._reuse
+                     or self._old.reuse)  # Re-using is inherited by sub-scopes.
+      self._default_varscope[0] = VariableScope(
+          self._reuse,
+          name=self._new_name,
+          initializer=self._old.initializer,
+          regularizer=self._old.regularizer,
+          caching_device=self._old.caching_device,
+          partitioner=self._old.partitioner,
+          dtype=self._old.dtype,
+          use_resource=self._old.use_resource,
+          custom_getter=self._old.custom_getter,
+          name_scope=self._old_name_scope or self._name_or_scope,
+          constraint=self._constraint)
+      if self._initializer is not None:
+        self._default_varscope[0].set_initializer(self._initializer)
+      if self._regularizer is not None:
+        self._default_varscope[0].set_regularizer(self._regularizer)
+      if self._caching_device is not None:
+        self._default_varscope[0].set_caching_device(self._caching_device)
+      if self._partitioner is not None:
+        self._default_varscope[0].set_partitioner(self._partitioner)
+      if self._custom_getter is not None:
+        self._default_varscope[0].set_custom_getter(
+            _maybe_wrap_custom_getter(self._custom_getter,
+                                      self._old.custom_getter))
+      if self._dtype is not None:
+        self._default_varscope[0].set_dtype(self._dtype)
+      if self._use_resource is not None:
+        self._default_varscope[0].set_use_resource(self._use_resource)
+      return self._default_varscope[0]
+
+  def __exit__(self, type_arg, value_arg, traceback_arg):
     # If jumping out from a non-prolonged scope, restore counts.
-    if isinstance(name_or_scope, VariableScope):
-      var_store.variable_scopes_count = old_subscopes
-    default_varscope[0] = old
+    if isinstance(self._name_or_scope, VariableScope):
+      self._var_store.variable_scopes_count = self._old_subscopes
+    else:
+      self._var_store.close_variable_subscopes(self._new_name)
+    self._default_varscope[0] = self._old
 
 
 def _maybe_wrap_custom_getter(custom_getter, old_getter):
@@ -1574,25 +1597,15 @@ def _get_unique_variable_scope(prefix):
   return prefix + ("_%d" % idx)
 
 
-# pylint: disable=g-doc-return-or-yield
-@tf_contextlib.contextmanager
-def variable_scope(name_or_scope,
-                   default_name=None,
-                   values=None,
-                   initializer=None,
-                   regularizer=None,
-                   caching_device=None,
-                   partitioner=None,
-                   custom_getter=None,
-                   reuse=None,
-                   dtype=None,
-                   use_resource=None,
-                   constraint=None):
-  """Returns a context manager for defining ops that creates variables (layers).
+# Named like a function for backwards compatibility with the
+# @tf_contextlib.contextmanager version, which was switched to a class to avoid
+# some object creation overhead.
+class variable_scope(object):  # pylint: disable=invalid-name
+  """A context manager for defining ops that creates variables (layers).
 
-  This context manager validates that the (optional) `values` are from
-  the same graph, ensures that graph is the default graph, and pushes a
-  name scope and a variable scope.
+  This context manager validates that the (optional) `values` are from the same
+  graph, ensures that graph is the default graph, and pushes a name scope and a
+  variable scope.
 
   If `name_or_scope` is not None, it is used as is. If `scope` is None, then
   `default_name` is used.  In that case, if the same name has been previously
@@ -1600,8 +1613,8 @@ def variable_scope(name_or_scope,
 
   Variable scope allows you to create new variables and to share already created
   ones while providing checks to not create or share by accident. For details,
-  see the @{$variables$Variable Scope How To},
-  here we present only a few basic examples.
+  see the @{$variables$Variable Scope How To}, here we present only a few basic
+  examples.
 
   Simple example of how to create a new variable:
 
@@ -1645,8 +1658,8 @@ def variable_scope(name_or_scope,
   assert v1 == v
   ```
 
-  To prevent accidental sharing of variables, we raise an exception when
-  getting an existing variable in a non-reusing scope.
+  To prevent accidental sharing of variables, we raise an exception when getting
+  an existing variable in a non-reusing scope.
 
   ```python
   with tf.variable_scope("foo"):
@@ -1655,8 +1668,8 @@ def variable_scope(name_or_scope,
       #  Raises ValueError("... v already exists ...").
   ```
 
-  Similarly, we raise an exception when trying to get a variable that
-  does not exist in reuse mode.
+  Similarly, we raise an exception when trying to get a variable that does not
+  exist in reuse mode.
 
   ```python
   with tf.variable_scope("foo", reuse=True):
@@ -1664,123 +1677,173 @@ def variable_scope(name_or_scope,
       #  Raises ValueError("... v does not exists ...").
   ```
 
-  Note that the `reuse` flag is inherited: if we open a reusing scope,
-  then all its sub-scopes become reusing as well.
+  Note that the `reuse` flag is inherited: if we open a reusing scope, then all
+  its sub-scopes become reusing as well.
 
   A note about name scoping: Setting `reuse` does not impact the naming of other
-  ops such as mult. See related discussion on [github#6189](https://github.com/tensorflow/tensorflow/issues/6189)
+  ops such as mult. See related discussion on
+  [github#6189](https://github.com/tensorflow/tensorflow/issues/6189)
 
-  Note that up to and including version 1.0, it was allowed (though
-  explicitly discouraged) to pass False to the reuse argument, yielding
-  undocumented behaviour slightly different from None. Starting at 1.1.0
-  passing None and False as reuse has exactly the same effect.
+  Note that up to and including version 1.0, it was allowed (though explicitly
+  discouraged) to pass False to the reuse argument, yielding undocumented
+  behaviour slightly different from None. Starting at 1.1.0 passing None and
+  False as reuse has exactly the same effect.
+  """
 
-  Args:
-    name_or_scope: `string` or `VariableScope`: the scope to open.
-    default_name: The default name to use if the `name_or_scope` argument is
-      `None`, this name will be uniquified. If name_or_scope is provided it
-      won't be used and therefore it is not required and can be None.
-    values: The list of `Tensor` arguments that are passed to the op function.
-    initializer: default initializer for variables within this scope.
-    regularizer: default regularizer for variables within this scope.
-    caching_device: default caching device for variables within this scope.
-    partitioner: default partitioner for variables within this scope.
-    custom_getter: default custom getter for variables within this scope.
-    reuse: `True`, None, or tf.AUTO_REUSE; if `True`, we go into reuse mode
-      for this scope as well as all sub-scopes; if tf.AUTO_REUSE, we create
-      variables if they do not exist, and return them otherwise; if None, we
-      inherit the parent scope's reuse flag. In Eager mode, this argument is
-      always forced to be tf.AUTO_REUSE.
-    dtype: type of variables created in this scope (defaults to the type
-      in the passed scope, or inherited from parent scope).
-    use_resource: If False, all variables will be regular Variables. If True,
-      experimental ResourceVariables with well-defined semantics will be used
-      instead. Defaults to False (will later change to True). In Eager mode,
-      this argument is always forced to be True.
-    constraint: An optional projection function to be applied to the variable
-      after being updated by an `Optimizer` (e.g. used to implement norm
-      constraints or value constraints for layer weights). The function must
-      take as input the unprojected Tensor representing the value of the
-      variable and return the Tensor for the projected value
-      (which must have the same shape). Constraints are not safe to
-      use when doing asynchronous distributed training.
+  def __init__(self,
+               name_or_scope,
+               default_name=None,
+               values=None,
+               initializer=None,
+               regularizer=None,
+               caching_device=None,
+               partitioner=None,
+               custom_getter=None,
+               reuse=None,
+               dtype=None,
+               use_resource=None,
+               constraint=None):
+    """Initialize the context manager.
 
-  Returns:
-    A scope that can be captured and reused.
+    Args:
+      name_or_scope: `string` or `VariableScope`: the scope to open.
+      default_name: The default name to use if the `name_or_scope` argument is
+        `None`, this name will be uniquified. If name_or_scope is provided it
+        won't be used and therefore it is not required and can be None.
+      values: The list of `Tensor` arguments that are passed to the op function.
+      initializer: default initializer for variables within this scope.
+      regularizer: default regularizer for variables within this scope.
+      caching_device: default caching device for variables within this scope.
+      partitioner: default partitioner for variables within this scope.
+      custom_getter: default custom getter for variables within this scope.
+      reuse: `True`, None, or tf.AUTO_REUSE; if `True`, we go into reuse mode
+        for this scope as well as all sub-scopes; if tf.AUTO_REUSE, we create
+        variables if they do not exist, and return them otherwise; if None, we
+        inherit the parent scope's reuse flag. In Eager mode, this argument is
+        always forced to be tf.AUTO_REUSE.
+      dtype: type of variables created in this scope (defaults to the type
+        in the passed scope, or inherited from parent scope).
+      use_resource: If False, all variables will be regular Variables. If True,
+        experimental ResourceVariables with well-defined semantics will be used
+        instead. Defaults to False (will later change to True). In Eager mode,
+        this argument is always forced to be True.
+      constraint: An optional projection function to be applied to the variable
+        after being updated by an `Optimizer` (e.g. used to implement norm
+        constraints or value constraints for layer weights). The function must
+        take as input the unprojected Tensor representing the value of the
+        variable and return the Tensor for the projected value
+        (which must have the same shape). Constraints are not safe to
+        use when doing asynchronous distributed training.
 
-  Raises:
-    ValueError: when trying to reuse within a create scope, or create within
-      a reuse scope.
-    TypeError: when the types of some arguments are not appropriate.
-  """
-  if default_name is None and name_or_scope is None:
-    raise TypeError("If default_name is None then name_or_scope is required")
-  if reuse is False:  # We don't allow non-inheriting scopes, False = None here.
-    reuse = None
-  if not (reuse is True or reuse is None or reuse is AUTO_REUSE):
-    raise ValueError("The reuse parameter must be True or False or None.")
-  if values is None:
-    values = []
-  g = ops._get_graph_from_inputs(values)  # pylint: disable=protected-access
-  with g.as_default():
-    if name_or_scope is not None:
-      if not isinstance(name_or_scope, (VariableScope,) + six.string_types):
+    Returns:
+      A scope that can be captured and reused.
+
+    Raises:
+      ValueError: when trying to reuse within a create scope, or create within
+        a reuse scope.
+      TypeError: when the types of some arguments are not appropriate.
+    """
+    self._name_or_scope = name_or_scope
+    self._default_name = default_name
+    self._values = values
+    self._initializer = initializer
+    self._regularizer = regularizer
+    self._caching_device = caching_device
+    self._partitioner = partitioner
+    self._custom_getter = custom_getter
+    self._reuse = reuse
+    self._dtype = dtype
+    self._use_resource = use_resource
+    self._constraint = constraint
+    if self._default_name is None and self._name_or_scope is None:
+      raise TypeError("If default_name is None then name_or_scope is required")
+    if self._reuse is False:
+      # We don't allow non-inheriting scopes, False = None here.
+      self._reuse = None
+    if not (self._reuse is True
+            or self._reuse is None
+            or self._reuse is AUTO_REUSE):
+      raise ValueError("The reuse parameter must be True or False or None.")
+    if self._values is None:
+      self._values = []
+    self._in_graph_mode = not context.in_eager_mode()
+    if self._in_graph_mode:
+      self._graph = ops._get_graph_from_inputs(self._values)  # pylint: disable=protected-access
+
+  def __enter__(self):
+    if self._in_graph_mode:
+      self._graph_context_manager = self._graph.as_default()
+      self._graph_context_manager.__enter__()
+    if self._name_or_scope is not None:
+      if not isinstance(self._name_or_scope,
+                        (VariableScope,) + six.string_types):
         raise TypeError("VariableScope: name_or_scope must be a string or "
                         "VariableScope.")
-      if isinstance(name_or_scope, six.string_types):
-        name_scope = name_or_scope
+      if isinstance(self._name_or_scope, six.string_types):
+        name_scope = self._name_or_scope
       else:
-        name_scope = name_or_scope.name.split("/")[-1]
+        name_scope = self._name_or_scope.name.split("/")[-1]
       if name_scope:
-        with ops.name_scope(name_scope) as cur_name_scope:
-          if isinstance(name_or_scope, six.string_types):
-            old_name_scope = cur_name_scope
-          else:
-            old_name_scope = name_or_scope.original_name_scope
-          with _pure_variable_scope(
-              name_or_scope,
-              reuse=reuse,
-              initializer=initializer,
-              regularizer=regularizer,
-              caching_device=caching_device,
-              partitioner=partitioner,
-              custom_getter=custom_getter,
-              old_name_scope=old_name_scope,
-              dtype=dtype,
-              use_resource=use_resource,
-              constraint=constraint) as vs:
-            yield vs
+        self._current_name_scope = ops.name_scope(name_scope)
+        current_name_scope_name = self._current_name_scope.__enter__()
+        if isinstance(self._name_or_scope, six.string_types):
+          old_name_scope = current_name_scope_name
+        else:
+          old_name_scope = self._name_or_scope.original_name_scope
+        self._pure_variable_scope = _pure_variable_scope(
+            self._name_or_scope,
+            reuse=self._reuse,
+            initializer=self._initializer,
+            regularizer=self._regularizer,
+            caching_device=self._caching_device,
+            partitioner=self._partitioner,
+            custom_getter=self._custom_getter,
+            old_name_scope=old_name_scope,
+            dtype=self._dtype,
+            use_resource=self._use_resource,
+            constraint=self._constraint)
+        return self._pure_variable_scope.__enter__()
       else:
+        self._current_name_scope = None
         # This can only happen if someone is entering the root variable scope.
-        with _pure_variable_scope(
-            name_or_scope,
-            reuse=reuse,
-            initializer=initializer,
-            regularizer=regularizer,
-            caching_device=caching_device,
-            partitioner=partitioner,
-            custom_getter=custom_getter,
-            dtype=dtype,
-            use_resource=use_resource,
-            constraint=constraint) as vs:
-          yield vs
+        self._pure_variable_scope = _pure_variable_scope(
+            self._name_or_scope,
+            reuse=self._reuse,
+            initializer=self._initializer,
+            regularizer=self._regularizer,
+            caching_device=self._caching_device,
+            partitioner=self._partitioner,
+            custom_getter=self._custom_getter,
+            dtype=self._dtype,
+            use_resource=self._use_resource,
+            constraint=self._constraint)
+        return self._pure_variable_scope.__enter__()
+
     else:  # Here name_or_scope is None. Using default name, but made unique.
-      if reuse:
+      if self._reuse:
         raise ValueError("reuse=True cannot be used without a name_or_scope")
-      with ops.name_scope(default_name) as scope:
-        unique_default_name = _get_unique_variable_scope(default_name)
-        with _pure_variable_scope(
-            unique_default_name,
-            initializer=initializer,
-            regularizer=regularizer,
-            caching_device=caching_device,
-            partitioner=partitioner,
-            custom_getter=custom_getter,
-            old_name_scope=scope,
-            dtype=dtype,
-            use_resource=use_resource,
-            constraint=constraint) as vs:
-          yield vs
+      self._current_name_scope = ops.name_scope(self._default_name)
+      current_name_scope_name = self._current_name_scope.__enter__()
+      unique_default_name = _get_unique_variable_scope(self._default_name)
+      self._pure_variable_scope = _pure_variable_scope(
+          unique_default_name,
+          initializer=self._initializer,
+          regularizer=self._regularizer,
+          caching_device=self._caching_device,
+          partitioner=self._partitioner,
+          custom_getter=self._custom_getter,
+          old_name_scope=current_name_scope_name,
+          dtype=self._dtype,
+          use_resource=self._use_resource,
+          constraint=self._constraint)
+      return self._pure_variable_scope.__enter__()
+
+  def __exit__(self, type_arg, value_arg, traceback_arg):
+    self._pure_variable_scope.__exit__(type_arg, value_arg, traceback_arg)
+    if self._current_name_scope:
+      self._current_name_scope.__exit__(type_arg, value_arg, traceback_arg)
+    if self._in_graph_mode:
+      self._graph_context_manager.__exit__(type_arg, value_arg, traceback_arg)
 
 
 # pylint: disable=g-doc-return-or-yield
diff --git a/tensorflow/tools/api/golden/tensorflow.errors.pbtxt b/tensorflow/tools/api/golden/tensorflow.errors.pbtxt
index 0ad1c19603..c5fe49baab 100644
--- a/tensorflow/tools/api/golden/tensorflow.errors.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.errors.pbtxt
@@ -136,6 +136,10 @@ tf_module {
     name: "UnknownError"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "raise_exception_on_not_ok_status"
+    mtype: "<type \'type\'>"
+  }
   member_method {
     name: "error_code_from_exception_type"
     argspec: "args=[\'cls\'], varargs=None, keywords=None, defaults=None"
@@ -144,8 +148,4 @@ tf_module {
     name: "exception_type_from_error_code"
     argspec: "args=[\'error_code\'], varargs=None, keywords=None, defaults=None"
   }
-  member_method {
-    name: "raise_exception_on_not_ok_status"
-    argspec: "args=[], varargs=args, keywords=kwds, defaults=None"
-  }
 }
diff --git a/tensorflow/tools/api/golden/tensorflow.errors.raise_exception_on_not_ok_status.pbtxt b/tensorflow/tools/api/golden/tensorflow.errors.raise_exception_on_not_ok_status.pbtxt
new file mode 100644
index 0000000000..5d25ec769a
--- /dev/null
+++ b/tensorflow/tools/api/golden/tensorflow.errors.raise_exception_on_not_ok_status.pbtxt
@@ -0,0 +1,8 @@
+path: "tensorflow.errors.raise_exception_on_not_ok_status"
+tf_class {
+  is_instance: "<class \'tensorflow.python.framework.errors_impl.raise_exception_on_not_ok_status\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+  }
+}
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.backend.name_scope.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.backend.name_scope.pbtxt
new file mode 100644
index 0000000000..43692a6c73
--- /dev/null
+++ b/tensorflow/tools/api/golden/tensorflow.keras.backend.name_scope.pbtxt
@@ -0,0 +1,9 @@
+path: "tensorflow.keras.backend.name_scope"
+tf_class {
+  is_instance: "<class \'tensorflow.python.framework.ops.name_scope\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name\', \'default_name\', \'values\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.backend.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.backend.pbtxt
index 6204ffa814..44fbe0f7a0 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.backend.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.backend.pbtxt
@@ -1,5 +1,9 @@
 path: "tensorflow.keras.backend"
 tf_module {
+  member {
+    name: "name_scope"
+    mtype: "<type \'type\'>"
+  }
   member_method {
     name: "abs"
     argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
@@ -288,10 +292,6 @@ tf_module {
     name: "moving_average_update"
     argspec: "args=[\'x\', \'value\', \'momentum\'], varargs=None, keywords=None, defaults=None"
   }
-  member_method {
-    name: "name_scope"
-    argspec: "args=[\'name\', \'default_name\', \'values\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
-  }
   member_method {
     name: "ndim"
     argspec: "args=[\'x\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/tensorflow.name_scope.pbtxt b/tensorflow/tools/api/golden/tensorflow.name_scope.pbtxt
new file mode 100644
index 0000000000..107f066c29
--- /dev/null
+++ b/tensorflow/tools/api/golden/tensorflow.name_scope.pbtxt
@@ -0,0 +1,9 @@
+path: "tensorflow.name_scope"
+tf_class {
+  is_instance: "<class \'tensorflow.python.framework.ops.name_scope\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name\', \'default_name\', \'values\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/tensorflow.pbtxt b/tensorflow/tools/api/golden/tensorflow.pbtxt
index 5ecf34d2ed..32a86e420a 100644
--- a/tensorflow/tools/api/golden/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.pbtxt
@@ -392,6 +392,10 @@ tf_module {
     name: "metrics"
     mtype: "<type \'module\'>"
   }
+  member {
+    name: "name_scope"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "newaxis"
     mtype: "<type \'NoneType\'>"
@@ -508,6 +512,10 @@ tf_module {
     name: "user_ops"
     mtype: "<type \'module\'>"
   }
+  member {
+    name: "variable_scope"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "variance_scaling_initializer"
     mtype: "<type \'type\'>"
@@ -1380,10 +1388,6 @@ tf_module {
     name: "multiply"
     argspec: "args=[\'x\', \'y\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
-  member_method {
-    name: "name_scope"
-    argspec: "args=[\'name\', \'default_name\', \'values\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
-  }
   member_method {
     name: "negative"
     argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
@@ -2028,10 +2032,6 @@ tf_module {
     name: "variable_op_scope"
     argspec: "args=[\'values\', \'name_or_scope\', \'default_name\', \'initializer\', \'regularizer\', \'caching_device\', \'partitioner\', \'custom_getter\', \'reuse\', \'dtype\', \'use_resource\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\'], "
   }
-  member_method {
-    name: "variable_scope"
-    argspec: "args=[\'name_or_scope\', \'default_name\', \'values\', \'initializer\', \'regularizer\', \'caching_device\', \'partitioner\', \'custom_getter\', \'reuse\', \'dtype\', \'use_resource\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\'], "
-  }
   member_method {
     name: "variables_initializer"
     argspec: "args=[\'var_list\', \'name\'], varargs=None, keywords=None, defaults=[\'init\'], "
diff --git a/tensorflow/tools/api/golden/tensorflow.variable_scope.pbtxt b/tensorflow/tools/api/golden/tensorflow.variable_scope.pbtxt
new file mode 100644
index 0000000000..de1ad7e860
--- /dev/null
+++ b/tensorflow/tools/api/golden/tensorflow.variable_scope.pbtxt
@@ -0,0 +1,9 @@
+path: "tensorflow.variable_scope"
+tf_class {
+  is_instance: "<class \'tensorflow.python.ops.variable_scope.variable_scope\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name_or_scope\', \'default_name\', \'values\', \'initializer\', \'regularizer\', \'caching_device\', \'partitioner\', \'custom_getter\', \'reuse\', \'dtype\', \'use_resource\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\'], "
+  }
+}
-- 
GitLab


From 32dc203f55a7462ddf780c68d619af574daedd46 Mon Sep 17 00:00:00 2001
From: Eugene Brevdo <ebrevdo@google.com>
Date: Wed, 4 Oct 2017 15:59:02 -0700
Subject: [PATCH 024/909] Improve gradient shape validation errors.

PiperOrigin-RevId: 171077826
---
 tensorflow/python/ops/gradients_impl.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/ops/gradients_impl.py b/tensorflow/python/ops/gradients_impl.py
index cb7d409f3b..d9b14de984 100644
--- a/tensorflow/python/ops/gradients_impl.py
+++ b/tensorflow/python/ops/gradients_impl.py
@@ -594,11 +594,19 @@ def gradients(ys,
           # If no grad_fn is defined or none of out_grads is available,
           # just propagate a list of None backwards.
           in_grads = [None] * len(op.inputs)
-        for t_in, in_grad in zip(op.inputs, in_grads):
+        for i, (t_in, in_grad) in enumerate(zip(op.inputs, in_grads)):
           if in_grad is not None:
             if (isinstance(in_grad, ops.Tensor) and
                 t_in.dtype != dtypes.resource):
-              in_grad.set_shape(t_in.get_shape())
+              try:
+                in_grad.set_shape(t_in.get_shape())
+              except ValueError:
+                raise ValueError(
+                    "Incompatible shapes between op input and calculated "
+                    "input gradient.  Forward operation: %s.  Input index: %d. "
+                    "Original input shape: %s.  "
+                    "Calculated input gradient shape: %s"
+                    % (op.name, i, t_in.shape, in_grad.shape))
             _SetGrad(grads, t_in, in_grad)
         if loop_state:
           loop_state.ExitGradWhileContext(op, before=False)
-- 
GitLab


From c57a4ace4a9a9a5cf871e6a090a4252f0c9ef2ad Mon Sep 17 00:00:00 2001
From: Michael Case <mikecase@google.com>
Date: Wed, 4 Oct 2017 16:10:19 -0700
Subject: [PATCH 025/909] Fix error when loading s3 file system library.

If attempting to call tf.load_file_system_library on the S3 library you
would previously get an error similiar to...
s3_file_system.so: undefined symbol: _ZN5nsync13nsync_mu_lockEPNS_11nsync_mu_s_E

Changing the build rule to be tf_cc_binary instead of cc_binary fixes this issue.

PiperOrigin-RevId: 171079804
---
 tensorflow/contrib/s3/BUILD | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/s3/BUILD b/tensorflow/contrib/s3/BUILD
index a4daed01e7..b7bc1a11d6 100644
--- a/tensorflow/contrib/s3/BUILD
+++ b/tensorflow/contrib/s3/BUILD
@@ -9,6 +9,7 @@ exports_files(["LICENSE"])
 
 load(
     "//tensorflow:tensorflow.bzl",
+    "tf_cc_binary",
     "tf_cc_test",
 )
 
@@ -24,7 +25,7 @@ filegroup(
     visibility = ["//tensorflow:__subpackages__"],
 )
 
-cc_binary(
+tf_cc_binary(
     name = "s3_file_system.so",
     srcs = [
         "s3_crypto.cc",
-- 
GitLab


From cd12a89b4cbc05b16667695fa483d9c375821b99 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 4 Oct 2017 16:14:38 -0700
Subject: [PATCH 026/909] Add shape inference function for _XlaRecv.

PiperOrigin-RevId: 171080445
---
 tensorflow/compiler/tf2xla/ops/sendrecv_ops.cc | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/tensorflow/compiler/tf2xla/ops/sendrecv_ops.cc b/tensorflow/compiler/tf2xla/ops/sendrecv_ops.cc
index b6947bfe57..4b41c16a8b 100644
--- a/tensorflow/compiler/tf2xla/ops/sendrecv_ops.cc
+++ b/tensorflow/compiler/tf2xla/ops/sendrecv_ops.cc
@@ -37,7 +37,14 @@ REGISTER_OP("_XLARecv")
     .Attr("tensor_name: string")
     .Attr("shape: shape")
     .SetIsStateful()
-    .SetShapeFn(shape_inference::UnknownShape)
+    .SetShapeFn([](shape_inference::InferenceContext* c) {
+      TensorShape shape_attr;
+      TF_RETURN_IF_ERROR(c->GetAttr("shape", &shape_attr));
+      shape_inference::ShapeHandle s;
+      TF_RETURN_IF_ERROR(c->MakeShapeFromTensorShape(shape_attr, &s));
+      c->set_output(0, s);
+      return Status::OK();
+    })
     .Doc(R"doc(
 Receives the named tensor from another XLA computation.
 
-- 
GitLab


From cfad8bfa77a8adfa093599c277b459708f0a95ff Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Wed, 4 Oct 2017 16:50:02 -0700
Subject: [PATCH 027/909] Don't use dlsym to resolve symbols in the CPU JIT

Instead of resolving symbols via dlsym when JITting for the CPU backend, use a
registry based mechanism.  This lets us kill off the --export_dynamic hack that
we used to need for CustomCall on the CPU backend.

PiperOrigin-RevId: 171084886
---
 tensorflow/compiler/tf2xla/kernels/BUILD      |   5 +-
 .../kernels/gather_op_kernel_float_int32.cc   |   3 +
 .../kernels/gather_op_kernel_float_int64.cc   |   3 +
 .../index_ops_kernel_argmax_float_1d.cc       |   3 +
 .../index_ops_kernel_argmax_float_2d.cc       |   3 +
 tensorflow/compiler/xla/service/cpu/BUILD     |  12 ++
 .../cpu/custom_call_target_registry.cc        |  39 ++++
 .../service/cpu/custom_call_target_registry.h |  74 +++++++
 .../xla/service/cpu/simple_orc_jit.cc         | 195 ++++++++++--------
 tensorflow/compiler/xla/tests/BUILD           |   3 +-
 .../compiler/xla/tests/custom_call_test.cc    |  14 +-
 tensorflow/compiler/xla/xla.bzl               |   8 -
 12 files changed, 266 insertions(+), 96 deletions(-)
 create mode 100644 tensorflow/compiler/xla/service/cpu/custom_call_target_registry.cc
 create mode 100644 tensorflow/compiler/xla/service/cpu/custom_call_target_registry.h

diff --git a/tensorflow/compiler/tf2xla/kernels/BUILD b/tensorflow/compiler/tf2xla/kernels/BUILD
index 6a0c4fef75..915c95e945 100644
--- a/tensorflow/compiler/tf2xla/kernels/BUILD
+++ b/tensorflow/compiler/tf2xla/kernels/BUILD
@@ -5,7 +5,6 @@ package(
 )
 
 load("//tensorflow:tensorflow.bzl", "tf_kernel_library")
-load("//tensorflow/compiler/xla:xla.bzl", "export_dynamic_linkopts")
 
 tf_kernel_library(
     name = "xla_ops",
@@ -155,6 +154,7 @@ cc_library(
     visibility = ["//visibility:public"],
     deps = [
         "//tensorflow/compiler/tf2xla:xla_local_runtime_context",
+        "//tensorflow/compiler/xla/service/cpu:custom_call_target_registry",
         "//tensorflow/core:framework_lite",
         "//tensorflow/core/kernels:bounds_check",
         "//tensorflow/core/kernels:gather_functor_hdr",
@@ -169,6 +169,7 @@ cc_library(
     visibility = ["//visibility:public"],
     deps = [
         "//tensorflow/compiler/tf2xla:xla_local_runtime_context",
+        "//tensorflow/compiler/xla/service/cpu:custom_call_target_registry",
         "//tensorflow/core:framework_lite",
         "//tensorflow/core/kernels:bounds_check",
         "//tensorflow/core/kernels:gather_functor_hdr",
@@ -182,6 +183,7 @@ cc_library(
     srcs = ["index_ops_kernel_argmax_float_1d.cc"],
     visibility = ["//visibility:public"],
     deps = [
+        "//tensorflow/compiler/xla/service/cpu:custom_call_target_registry",
         "//tensorflow/core:framework_lite",
         "//third_party/eigen3",
     ],
@@ -193,6 +195,7 @@ cc_library(
     srcs = ["index_ops_kernel_argmax_float_2d.cc"],
     visibility = ["//visibility:public"],
     deps = [
+        "//tensorflow/compiler/xla/service/cpu:custom_call_target_registry",
         "//tensorflow/core:framework_lite",
         "//third_party/eigen3",
     ],
diff --git a/tensorflow/compiler/tf2xla/kernels/gather_op_kernel_float_int32.cc b/tensorflow/compiler/tf2xla/kernels/gather_op_kernel_float_int32.cc
index 33b1b087d0..0b44e0c6f8 100644
--- a/tensorflow/compiler/tf2xla/kernels/gather_op_kernel_float_int32.cc
+++ b/tensorflow/compiler/tf2xla/kernels/gather_op_kernel_float_int32.cc
@@ -17,6 +17,7 @@ limitations under the License.
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 #include "tensorflow/compiler/tf2xla/xla_local_runtime_context.h"
+#include "tensorflow/compiler/xla/service/cpu/custom_call_target_registry.h"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/kernels/gather_functor.h"
 #include "tensorflow/core/platform/dynamic_annotations.h"
@@ -70,3 +71,5 @@ EIGEN_STRONG_INLINE void gather_float_int32_xla_impl(float* out, void** data) {
 extern "C" void TF_EXPORT gather_float_int32_xla_impl(float* out, void** data) {
   tensorflow::gather_float_int32_xla_impl(out, data);
 }
+
+REGISTER_CUSTOM_CALL_TARGET(gather_float_int32_xla_impl);
diff --git a/tensorflow/compiler/tf2xla/kernels/gather_op_kernel_float_int64.cc b/tensorflow/compiler/tf2xla/kernels/gather_op_kernel_float_int64.cc
index 5e2d872ce0..d7c7a7bf2c 100644
--- a/tensorflow/compiler/tf2xla/kernels/gather_op_kernel_float_int64.cc
+++ b/tensorflow/compiler/tf2xla/kernels/gather_op_kernel_float_int64.cc
@@ -17,6 +17,7 @@ limitations under the License.
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 #include "tensorflow/compiler/tf2xla/xla_local_runtime_context.h"
+#include "tensorflow/compiler/xla/service/cpu/custom_call_target_registry.h"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/kernels/gather_functor.h"
 #include "tensorflow/core/platform/dynamic_annotations.h"
@@ -70,3 +71,5 @@ EIGEN_STRONG_INLINE void gather_float_int64_xla_impl(float* out, void** data) {
 extern "C" void TF_EXPORT gather_float_int64_xla_impl(float* out, void** data) {
   tensorflow::gather_float_int64_xla_impl(out, data);
 }
+
+REGISTER_CUSTOM_CALL_TARGET(gather_float_int64_xla_impl);
diff --git a/tensorflow/compiler/tf2xla/kernels/index_ops_kernel_argmax_float_1d.cc b/tensorflow/compiler/tf2xla/kernels/index_ops_kernel_argmax_float_1d.cc
index afbd64ca50..47cf8c6675 100644
--- a/tensorflow/compiler/tf2xla/kernels/index_ops_kernel_argmax_float_1d.cc
+++ b/tensorflow/compiler/tf2xla/kernels/index_ops_kernel_argmax_float_1d.cc
@@ -16,6 +16,7 @@ limitations under the License.
 #define EIGEN_USE_THREADS
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "tensorflow/compiler/xla/service/cpu/custom_call_target_registry.h"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/platform/dynamic_annotations.h"
 #include "tensorflow/core/platform/macros.h"
@@ -47,3 +48,5 @@ EIGEN_STRONG_INLINE void argmax_float_1d_xla_impl(void* out, void** data) {
 extern "C" void TF_EXPORT argmax_float_1d_xla_impl(void* out, void** data) {
   tensorflow::argmax_float_1d_xla_impl(out, data);
 }
+
+REGISTER_CUSTOM_CALL_TARGET(argmax_float_1d_xla_impl);
diff --git a/tensorflow/compiler/tf2xla/kernels/index_ops_kernel_argmax_float_2d.cc b/tensorflow/compiler/tf2xla/kernels/index_ops_kernel_argmax_float_2d.cc
index 841ff2f4df..9b83392d8f 100644
--- a/tensorflow/compiler/tf2xla/kernels/index_ops_kernel_argmax_float_2d.cc
+++ b/tensorflow/compiler/tf2xla/kernels/index_ops_kernel_argmax_float_2d.cc
@@ -16,6 +16,7 @@ limitations under the License.
 #define EIGEN_USE_THREADS
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "tensorflow/compiler/xla/service/cpu/custom_call_target_registry.h"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/platform/dynamic_annotations.h"
 #include "tensorflow/core/platform/macros.h"
@@ -49,3 +50,5 @@ EIGEN_STRONG_INLINE void argmax_float_2d_xla_impl(void* out, void** data) {
 extern "C" void TF_EXPORT argmax_float_2d_xla_impl(void* out, void** data) {
   tensorflow::argmax_float_2d_xla_impl(out, data);
 }
+
+REGISTER_CUSTOM_CALL_TARGET(argmax_float_2d_xla_impl);
diff --git a/tensorflow/compiler/xla/service/cpu/BUILD b/tensorflow/compiler/xla/service/cpu/BUILD
index fa6e5b2313..5d13b82427 100644
--- a/tensorflow/compiler/xla/service/cpu/BUILD
+++ b/tensorflow/compiler/xla/service/cpu/BUILD
@@ -129,6 +129,7 @@ cc_library(
         ":cpu_runtime_avx",
         ":cpu_runtime_neon",
         ":cpu_runtime_sse4_1",
+        ":custom_call_target_registry",
         ":disassembler",
         ":runtime_conv2d",
         ":runtime_matmul",
@@ -674,6 +675,17 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "custom_call_target_registry",
+    srcs = [
+        "custom_call_target_registry.cc",
+    ],
+    hdrs = [
+        "custom_call_target_registry.h",
+    ],
+    visibility = ["//visibility:public"],
+)
+
 # -----------------------------------------------------------------------------
 
 filegroup(
diff --git a/tensorflow/compiler/xla/service/cpu/custom_call_target_registry.cc b/tensorflow/compiler/xla/service/cpu/custom_call_target_registry.cc
new file mode 100644
index 0000000000..5f5803874b
--- /dev/null
+++ b/tensorflow/compiler/xla/service/cpu/custom_call_target_registry.cc
@@ -0,0 +1,39 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/service/cpu/custom_call_target_registry.h"
+
+namespace xla {
+namespace cpu {
+
+CustomCallTargetRegistry* CustomCallTargetRegistry::Global() {
+  static auto* registry = new CustomCallTargetRegistry;
+  return registry;
+}
+
+void CustomCallTargetRegistry::Register(const std::string& symbol,
+                                        void* address) {
+  std::lock_guard<std::mutex> lock(mu_);
+  registered_symbols_[symbol] = address;
+}
+
+void* CustomCallTargetRegistry::Lookup(const std::string& symbol) const {
+  std::lock_guard<std::mutex> lock(mu_);
+  auto it = registered_symbols_.find(symbol);
+  return it == registered_symbols_.end() ? nullptr : it->second;
+}
+
+}  // namespace cpu
+}  // namespace xla
diff --git a/tensorflow/compiler/xla/service/cpu/custom_call_target_registry.h b/tensorflow/compiler/xla/service/cpu/custom_call_target_registry.h
new file mode 100644
index 0000000000..2994642356
--- /dev/null
+++ b/tensorflow/compiler/xla/service/cpu/custom_call_target_registry.h
@@ -0,0 +1,74 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef THIRD_PARTY_TENSORFLOW_COMPILER_XLA_SERVICE_CPU_CUSTOM_CALL_TARGET_REGISTRY_H_
+#define THIRD_PARTY_TENSORFLOW_COMPILER_XLA_SERVICE_CPU_CUSTOM_CALL_TARGET_REGISTRY_H_
+
+// This file is depended on by kernels that have to build for mobile devices.
+// For this reason, we avoid relying on TensorFlow and instead only use the
+// standard C++ library.
+
+#include <mutex>  // NOLINT
+#include <string>
+#include <unordered_map>
+
+namespace xla {
+namespace cpu {
+
+// The CPU JIT compiler uses this registry to resolve symbolic CustomCall
+// targets; so when using the CPU JIT, CustomCall targets need to be registered
+// here with the symbol name used in the CustomCall.
+//
+// The XLA AOT compiler links using a standard offline linker; so when compiling
+// in AOT mode, you *also* need to make sure the name of the callee (presumably
+// implemented in C++) matches up with the symbolic name used in the CustomCall.
+//
+// We maintain the registry in both the JIT and the AOT cases for simplicity,
+// but we only use it when running in JIT mode.
+class CustomCallTargetRegistry {
+ public:
+  static CustomCallTargetRegistry* Global();
+
+  void Register(const std::string& symbol, void* address);
+  void* Lookup(const std::string& symbol) const;
+
+ private:
+  std::unordered_map<std::string, void*> registered_symbols_;
+  mutable std::mutex mu_;
+};
+
+class RegisterCustomCallTarget {
+ public:
+  explicit RegisterCustomCallTarget(const std::string& name, void* address) {
+    CustomCallTargetRegistry::Global()->Register(name, address);
+  }
+};
+
+#define REGISTER_CUSTOM_CALL_CONCAT(a, b) a##b
+
+#define REGISTER_CUSTOM_CALL_TARGET_WITH_SYM_HELPER(symbol, address, counter) \
+  static ::xla::cpu::RegisterCustomCallTarget REGISTER_CUSTOM_CALL_CONCAT(    \
+      custom_call_target_register, counter)(symbol,                           \
+                                            reinterpret_cast<void*>(address))
+
+#define REGISTER_CUSTOM_CALL_TARGET_WITH_SYM(symbol, address) \
+  REGISTER_CUSTOM_CALL_TARGET_WITH_SYM_HELPER(symbol, address, __COUNTER__)
+
+#define REGISTER_CUSTOM_CALL_TARGET(function) \
+  REGISTER_CUSTOM_CALL_TARGET_WITH_SYM(#function, function)
+
+}  // namespace cpu
+}  // namespace xla
+
+#endif  // THIRD_PARTY_TENSORFLOW_COMPILER_XLA_SERVICE_CPU_CUSTOM_CALL_TARGET_REGISTRY_H_
diff --git a/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc b/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc
index c3c11df090..0711c9de27 100644
--- a/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc
+++ b/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc
@@ -31,6 +31,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/cpu/cpu_runtime_avx.h"
 #include "tensorflow/compiler/xla/service/cpu/cpu_runtime_neon.h"
 #include "tensorflow/compiler/xla/service/cpu/cpu_runtime_sse4_1.h"
+#include "tensorflow/compiler/xla/service/cpu/custom_call_target_registry.h"
 #include "tensorflow/compiler/xla/service/cpu/runtime_conv2d.h"
 #include "tensorflow/compiler/xla/service/cpu/runtime_matmul.h"
 #include "tensorflow/compiler/xla/service/cpu/runtime_single_threaded_conv2d.h"
@@ -42,90 +43,10 @@ namespace xla {
 namespace cpu {
 namespace {
 
-// Converts a symbol 'name' into the form expected by dlsym().
-std::string CanonicalizeSymbol(const std::string& name) {
-#if defined(__APPLE__)
-  // On Mac OS X, dlsym() expects names not to be prefixed with a leading
-  // underscore.
-  if (!name.empty() && name.front() == '_') {
-    return name.substr(1);
-  }
-#endif
-  return name;
-}
-
-class JITSymbolTable {
- public:
-  JITSymbolTable() { Populate(); }
-
-  void* Lookup(llvm::StringRef jit_symbol_name) const {
-    auto it = jit_symbol_table_.find(jit_symbol_name);
-    return it == jit_symbol_table_.end() ? nullptr : it->getValue();
-  }
-
-  static bool MustBeInTable(llvm::StringRef name) {
-    // In particular, names starting with
-    // runtime::kXlaCpuRuntimeSymbolNamePrefix should not be dlsym'ed.
-    return name.startswith(runtime::kXlaCpuRuntimeSymbolNamePrefix);
-  }
-
- private:
-  void AddJITSymbolToTable(llvm::StringRef jit_symbol_name,
-                           llvm::StringRef cpp_symbol_name,
-                           void* jit_symbol_value) {
-    // The JIT symbol name and the C++ symbol name (with an extern "C" linkage)
-    // need to match, otherwise AOT links will fail.
-    CHECK(jit_symbol_name == cpp_symbol_name);
-    CHECK(jit_symbol_table_.insert({jit_symbol_name, jit_symbol_value}).second);
-  }
-
-  void Populate() {
-#define ADD_JIT_SYMBOL_TO_TABLE(base_name)                       \
-  do {                                                           \
-    AddJITSymbolToTable(                                         \
-        xla::cpu::runtime::k##base_name##SymbolName,             \
-        "__xla_cpu_runtime_" #base_name,                         \
-        reinterpret_cast<void*>(__xla_cpu_runtime_##base_name)); \
-  } while (false)
-
-    ADD_JIT_SYMBOL_TO_TABLE(AcquireInfeedBufferForDequeue);
-    ADD_JIT_SYMBOL_TO_TABLE(ReleaseInfeedBufferAfterDequeue);
-    ADD_JIT_SYMBOL_TO_TABLE(AcquireOutfeedBufferForPopulation);
-    ADD_JIT_SYMBOL_TO_TABLE(ReleaseOutfeedBufferAfterPopulation);
-    ADD_JIT_SYMBOL_TO_TABLE(ExpV8F32AVX);
-    ADD_JIT_SYMBOL_TO_TABLE(LogV8F32AVX);
-    ADD_JIT_SYMBOL_TO_TABLE(ExpV4F32SSE);
-    ADD_JIT_SYMBOL_TO_TABLE(LogV4F32SSE);
-    ADD_JIT_SYMBOL_TO_TABLE(ExpV4F32NEON);
-    ADD_JIT_SYMBOL_TO_TABLE(LogV4F32NEON);
-    ADD_JIT_SYMBOL_TO_TABLE(EigenConvF32);
-    ADD_JIT_SYMBOL_TO_TABLE(EigenMatMulF32);
-    ADD_JIT_SYMBOL_TO_TABLE(EigenMatMulF64);
-    ADD_JIT_SYMBOL_TO_TABLE(EigenSingleThreadedConvF32);
-    ADD_JIT_SYMBOL_TO_TABLE(EigenSingleThreadedMatMulF32);
-    ADD_JIT_SYMBOL_TO_TABLE(EigenSingleThreadedMatMulF64);
-
-#undef ADD_JIT_SYMBOL_TO_TABLE
-  }
-
-  llvm::StringMap<void*> jit_symbol_table_;
-};
-
-const JITSymbolTable& GetJITSymbolTable() {
-  static JITSymbolTable* symbol_table = new JITSymbolTable;
-  return *symbol_table;
-}
-
 // A simple SymbolResolver that delegates to the host dynamic linker.
 struct SimpleResolver : public llvm::JITSymbolResolver {
   llvm::JITSymbol findSymbol(const std::string& name) override {
-    std::string canonical_name = CanonicalizeSymbol(name);
-    const JITSymbolTable& jit_symbol_table = GetJITSymbolTable();
-
-    void* func_addr = JITSymbolTable::MustBeInTable(canonical_name)
-                          ? jit_symbol_table.Lookup(canonical_name)
-                          : dlsym(RTLD_DEFAULT, canonical_name.c_str());
-
+    void* func_addr = CustomCallTargetRegistry::Global()->Lookup(name);
     if (func_addr == nullptr) {
       return nullptr;
     }
@@ -238,5 +159,117 @@ llvm::JITSymbol SimpleOrcJIT::FindSymbol(const std::string& name) {
   return nullptr;
 }
 
+namespace {
+// Register some known symbols with the CustomCallTargetRegistry.
+bool RegisterKnownJITSymbols() {
+  CustomCallTargetRegistry* registry = CustomCallTargetRegistry::Global();
+
+#define REGISTER_CPU_RUNTIME_SYMBOL(base_name)                                \
+  do {                                                                        \
+    auto* function_address =                                                  \
+        reinterpret_cast<void*>(__xla_cpu_runtime_##base_name);               \
+    registry->Register(xla::cpu::runtime::k##base_name##SymbolName,           \
+                       function_address);                                     \
+    CHECK_EQ(                                                                 \
+        tensorflow::StringPiece(xla::cpu::runtime::k##base_name##SymbolName), \
+        "__xla_cpu_runtime_" #base_name);                                     \
+  } while (false)
+
+  REGISTER_CPU_RUNTIME_SYMBOL(AcquireInfeedBufferForDequeue);
+  REGISTER_CPU_RUNTIME_SYMBOL(ReleaseInfeedBufferAfterDequeue);
+  REGISTER_CPU_RUNTIME_SYMBOL(AcquireOutfeedBufferForPopulation);
+  REGISTER_CPU_RUNTIME_SYMBOL(ReleaseOutfeedBufferAfterPopulation);
+  REGISTER_CPU_RUNTIME_SYMBOL(ExpV8F32AVX);
+  REGISTER_CPU_RUNTIME_SYMBOL(LogV8F32AVX);
+  REGISTER_CPU_RUNTIME_SYMBOL(ExpV4F32SSE);
+  REGISTER_CPU_RUNTIME_SYMBOL(LogV4F32SSE);
+  REGISTER_CPU_RUNTIME_SYMBOL(ExpV4F32NEON);
+  REGISTER_CPU_RUNTIME_SYMBOL(LogV4F32NEON);
+  REGISTER_CPU_RUNTIME_SYMBOL(EigenConvF32);
+  REGISTER_CPU_RUNTIME_SYMBOL(EigenMatMulF32);
+  REGISTER_CPU_RUNTIME_SYMBOL(EigenMatMulF64);
+  REGISTER_CPU_RUNTIME_SYMBOL(EigenSingleThreadedConvF32);
+  REGISTER_CPU_RUNTIME_SYMBOL(EigenSingleThreadedMatMulF32);
+  REGISTER_CPU_RUNTIME_SYMBOL(EigenSingleThreadedMatMulF64);
+
+#undef REGISTER_CPU_RUNTIME_SYMBOL
+
+#define REGISTER_LIBM_SYMBOL(name)                                    \
+  do {                                                                \
+    /* Register both the F32 and F64 variants of the libm symbol.  */ \
+    registry->Register(#name "f", reinterpret_cast<void*>(name##f));  \
+    registry->Register(#name, reinterpret_cast<void*>(name));         \
+  } while (false)
+
+  REGISTER_LIBM_SYMBOL(acos);
+  REGISTER_LIBM_SYMBOL(acosh);
+  REGISTER_LIBM_SYMBOL(asin);
+  REGISTER_LIBM_SYMBOL(asinh);
+  REGISTER_LIBM_SYMBOL(atan);
+  REGISTER_LIBM_SYMBOL(atan2);
+  REGISTER_LIBM_SYMBOL(atanh);
+  REGISTER_LIBM_SYMBOL(cbrt);
+  REGISTER_LIBM_SYMBOL(ceil);
+  REGISTER_LIBM_SYMBOL(copysign);
+  REGISTER_LIBM_SYMBOL(cos);
+  REGISTER_LIBM_SYMBOL(cosh);
+  REGISTER_LIBM_SYMBOL(erf);
+  REGISTER_LIBM_SYMBOL(erfc);
+  REGISTER_LIBM_SYMBOL(exp);
+  REGISTER_LIBM_SYMBOL(exp2);
+  REGISTER_LIBM_SYMBOL(expm1);
+  REGISTER_LIBM_SYMBOL(fabs);
+  REGISTER_LIBM_SYMBOL(fdim);
+  REGISTER_LIBM_SYMBOL(floor);
+  REGISTER_LIBM_SYMBOL(fma);
+  REGISTER_LIBM_SYMBOL(fmax);
+  REGISTER_LIBM_SYMBOL(fmin);
+  REGISTER_LIBM_SYMBOL(fmod);
+  REGISTER_LIBM_SYMBOL(frexp);
+  REGISTER_LIBM_SYMBOL(hypot);
+  REGISTER_LIBM_SYMBOL(ilogb);
+  REGISTER_LIBM_SYMBOL(ldexp);
+  REGISTER_LIBM_SYMBOL(lgamma);
+  REGISTER_LIBM_SYMBOL(llrint);
+  REGISTER_LIBM_SYMBOL(llround);
+  REGISTER_LIBM_SYMBOL(log);
+  REGISTER_LIBM_SYMBOL(log10);
+  REGISTER_LIBM_SYMBOL(log1p);
+  REGISTER_LIBM_SYMBOL(log2);
+  REGISTER_LIBM_SYMBOL(logb);
+  REGISTER_LIBM_SYMBOL(lrint);
+  REGISTER_LIBM_SYMBOL(lround);
+  REGISTER_LIBM_SYMBOL(modf);
+  REGISTER_LIBM_SYMBOL(nan);
+  REGISTER_LIBM_SYMBOL(nearbyint);
+  REGISTER_LIBM_SYMBOL(nextafter);
+  REGISTER_LIBM_SYMBOL(nexttoward);
+  REGISTER_LIBM_SYMBOL(pow);
+  REGISTER_LIBM_SYMBOL(remainder);
+  REGISTER_LIBM_SYMBOL(remquo);
+  REGISTER_LIBM_SYMBOL(rint);
+  REGISTER_LIBM_SYMBOL(round);
+  REGISTER_LIBM_SYMBOL(scalbln);
+  REGISTER_LIBM_SYMBOL(scalbn);
+  REGISTER_LIBM_SYMBOL(sin);
+  REGISTER_LIBM_SYMBOL(sincos);
+  REGISTER_LIBM_SYMBOL(sinh);
+  REGISTER_LIBM_SYMBOL(sqrt);
+  REGISTER_LIBM_SYMBOL(tan);
+  REGISTER_LIBM_SYMBOL(tanh);
+  REGISTER_LIBM_SYMBOL(tgamma);
+  REGISTER_LIBM_SYMBOL(trunc);
+
+#undef REGISTER_LIBM_SYMBOL
+
+  registry->Register("memcpy", reinterpret_cast<void*>(memcpy));
+  registry->Register("memmove", reinterpret_cast<void*>(memmove));
+  registry->Register("memset", reinterpret_cast<void*>(memset));
+  return true;
+}
+
+bool unused = RegisterKnownJITSymbols();
+}  // namespace
+
 }  // namespace cpu
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/tests/BUILD b/tensorflow/compiler/xla/tests/BUILD
index e45b839afd..84bebd4708 100644
--- a/tensorflow/compiler/xla/tests/BUILD
+++ b/tensorflow/compiler/xla/tests/BUILD
@@ -23,7 +23,6 @@ filegroup(
     ]),
 )
 
-load("//tensorflow/compiler/xla:xla.bzl", "export_dynamic_linkopts")
 load("//tensorflow/compiler/xla/tests:build_defs.bzl", "xla_test")
 load("//tensorflow/compiler/xla/tests:build_defs.bzl", "xla_test_library")
 load("//tensorflow/compiler/xla/tests:build_defs.bzl", "generate_backend_suites")
@@ -981,13 +980,13 @@ xla_test(
 xla_test(
     name = "custom_call_test",
     srcs = ["custom_call_test.cc"],
-    linkopts = export_dynamic_linkopts,
     deps = [
         "//tensorflow/compiler/xla:literal_util",
         "//tensorflow/compiler/xla:shape_util",
         "//tensorflow/compiler/xla:util",
         "//tensorflow/compiler/xla:xla_data_proto",
         "//tensorflow/compiler/xla/service:hlo",
+        "//tensorflow/compiler/xla/service/cpu:custom_call_target_registry",
         "//tensorflow/compiler/xla/tests:hlo_test_base",
         "//tensorflow/compiler/xla/tests:literal_test_util",
         "//tensorflow/compiler/xla/tests:xla_internal_test_main",
diff --git a/tensorflow/compiler/xla/tests/custom_call_test.cc b/tensorflow/compiler/xla/tests/custom_call_test.cc
index 342478bc74..74f73a1ddc 100644
--- a/tensorflow/compiler/xla/tests/custom_call_test.cc
+++ b/tensorflow/compiler/xla/tests/custom_call_test.cc
@@ -18,6 +18,7 @@ limitations under the License.
 
 #include "tensorflow/compiler/xla/literal_util.h"
 #include "tensorflow/compiler/xla/ptr_util.h"
+#include "tensorflow/compiler/xla/service/cpu/custom_call_target_registry.h"
 #include "tensorflow/compiler/xla/service/hlo_computation.h"
 #include "tensorflow/compiler/xla/service/hlo_instruction.h"
 #include "tensorflow/compiler/xla/service/hlo_module.h"
@@ -31,19 +32,19 @@ limitations under the License.
 #include "tensorflow/core/platform/macros.h"
 #include "tensorflow/core/platform/test.h"
 
-
-extern "C" void TF_EXPORT R0F32Add2(float* out, float** in) {
+namespace {
+void R0F32Add2(float* out, float** in) {
   TF_ANNOTATE_MEMORY_IS_INITIALIZED(in, sizeof(float*));
   *out = **in + 2.0f;
 }
 
-extern "C" void TF_EXPORT R2F32ReduceSum(float* out, float** in) {
+void R2F32ReduceSum(float* out, float** in) {
   TF_ANNOTATE_MEMORY_IS_INITIALIZED(in, sizeof(float) * 4);
   float* array = in[0];
   *out = array[0] + array[1] + array[2] + array[3];
 }
 
-extern "C" void TF_EXPORT Add1ToValues(float* out, float** in) {
+void Add1ToValues(float* out, float** in) {
   TF_ANNOTATE_MEMORY_IS_INITIALIZED(in, sizeof(float) * 4);
   float* array = in[0];
   out[0] = array[0] + 1;
@@ -51,6 +52,11 @@ extern "C" void TF_EXPORT Add1ToValues(float* out, float** in) {
   out[2] = array[2] + 1;
   out[3] = array[3] + 1;
 }
+}  // namespace
+
+REGISTER_CUSTOM_CALL_TARGET(R0F32Add2);
+REGISTER_CUSTOM_CALL_TARGET(R2F32ReduceSum);
+REGISTER_CUSTOM_CALL_TARGET(Add1ToValues);
 
 namespace xla {
 namespace {
diff --git a/tensorflow/compiler/xla/xla.bzl b/tensorflow/compiler/xla/xla.bzl
index 22e70ec97a..3fa5bcc1df 100644
--- a/tensorflow/compiler/xla/xla.bzl
+++ b/tensorflow/compiler/xla/xla.bzl
@@ -17,11 +17,3 @@ def xla_proto_library(name, srcs=[], deps=[], visibility=None, testonly=0):
                    protoc="@protobuf_archive//:protoc",
                    testonly=testonly,
                    visibility=visibility,)
-
-# Flags required for modules that export symbols that are to be called by the
-# XLA CustomCall operator. CustomCall must be able to find symbols with dlsym(),
-# which on Linux requires we link with --export-dynamic.
-export_dynamic_linkopts = select({
-    "//tensorflow:darwin": [],
-    "//conditions:default": ["-Wl,--export-dynamic"],
-})
-- 
GitLab


From 875df6262977eebd73d558600c5a216882b88164 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Wed, 4 Oct 2017 17:25:04 -0700
Subject: [PATCH 028/909] [XLA:CPU] Mark loads of parameter addresses as
 invariant.

Also delete a dead member in the IrEmitter, make param names match
between the header and the cc file, and make a cosmetic comment fix.

PiperOrigin-RevId: 171088993
---
 tensorflow/compiler/xla/service/cpu/ir_emitter.cc | 12 ++++++++++--
 tensorflow/compiler/xla/service/cpu/ir_emitter.h  | 12 ++++--------
 2 files changed, 14 insertions(+), 10 deletions(-)

diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
index 1e81a815d8..8b777bcf84 100644
--- a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
@@ -1457,6 +1457,14 @@ Status IrEmitter::HandleParameter(HloInstruction* parameter) {
       llvm_ir::EmitBufferIndexingGEP(params, param_number, &ir_builder_);
   llvm::LoadInst* param_address_untyped =
       ir_builder_.CreateLoad(param_address_offset);
+  if (hlo_module_config_.debug_options()
+          .xla_llvm_enable_invariant_load_metadata()) {
+    // We never reassign parameters, so this load is invariant.
+    param_address_untyped->setMetadata(
+        llvm::LLVMContext::MD_invariant_load,
+        llvm::MDNode::get(param_address_untyped->getContext(), /*MDs=*/{}));
+  }
+
   llvm::Value* param_address_typed = ir_builder_.CreateBitCast(
       param_address_untyped, IrShapeType(param_shape)->getPointerTo());
   emitted_value_[parameter] = param_address_typed;
@@ -2924,8 +2932,8 @@ llvm::Value* IrEmitter::EmitTempBufferPointer(
       ir_builder_.CreateLoad(tempbuf_address_ptr);
   if (hlo_module_config_.debug_options()
           .xla_llvm_enable_invariant_load_metadata()) {
-    //  Loading the address of a buffer is invariant of the point at which the
-    //  load is executed in the program because we never reassign buffers.
+    // Loading the address of a buffer is invariant of the point at which the
+    // load is executed in the program because we never reassign buffers.
     tempbuf_address_base->setMetadata(
         llvm::LLVMContext::MD_invariant_load,
         llvm::MDNode::get(tempbuf_address_base->getContext(), /*MDs=*/{}));
diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.h b/tensorflow/compiler/xla/service/cpu/ir_emitter.h
index 8042e03e69..05663b6038 100644
--- a/tensorflow/compiler/xla/service/cpu/ir_emitter.h
+++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.h
@@ -146,7 +146,7 @@ class IrEmitter : public DfsHloVisitorWithDefault {
   //
   // Default action which emits code for most operations. Operations which are
   // special in some way are handled explicitly in HandleFoo methods.
-  Status DefaultAction(HloInstruction* hlo_instruction) override;
+  Status DefaultAction(HloInstruction* hlo) override;
 
   Status HandleBitcast(HloInstruction* bitcast) override;
   Status HandleConstant(HloInstruction* constant,
@@ -175,7 +175,7 @@ class IrEmitter : public DfsHloVisitorWithDefault {
   Status HandleReduceWindow(HloInstruction* reduce_window,
                             HloInstruction* operand, const Window& window,
                             HloComputation* function) override;
-  Status HandleSelectAndScatter(HloInstruction* instruction) override;
+  Status HandleSelectAndScatter(HloInstruction* select_and_scatter) override;
   Status HandleSend(HloInstruction* send) override;
   Status HandleSlice(HloInstruction* slice,
                      HloInstruction* /*operand*/) override;
@@ -208,7 +208,7 @@ class IrEmitter : public DfsHloVisitorWithDefault {
   Status FinishVisit(HloInstruction* root) override;
 
   Status Preprocess(HloInstruction* hlo) override;
-  Status Postprocess(HloInstruction* visited) override;
+  Status Postprocess(HloInstruction* hlo) override;
 
  private:
   // Private helper to initialize an IR function for the computation.
@@ -304,7 +304,7 @@ class IrEmitter : public DfsHloVisitorWithDefault {
   void EmitArrayFunctionCallInto(
       llvm::Function* function,
       tensorflow::gtl::ArraySlice<llvm::Value*> parameter_addresses,
-      llvm::Value* return_value, tensorflow::StringPiece name);
+      llvm::Value* return_value_buffer, tensorflow::StringPiece name);
 
   // Array function call emitter.  Returns a Value for the function's return
   // value buffer address. The return value buffer is alloca'ed by this
@@ -447,10 +447,6 @@ class IrEmitter : public DfsHloVisitorWithDefault {
                             const llvm_ir::IrArray& target_array,
                             const llvm_ir::IrArray& source_array);
 
-  // Name of the computation entry function. This function serves as the
-  // top-level "main" of the computation and will be invoked by the JIT.
-  string entry_function_name_;
-
   // Assignment of the temporary buffers needed by the computation and their
   // shape information.
   const BufferAssignment& assignment_;
-- 
GitLab


From fa86731b3dd081cf437fbeecbfcae30596c2873b Mon Sep 17 00:00:00 2001
From: Igor Saprykin <isaprykin@google.com>
Date: Wed, 4 Oct 2017 17:26:34 -0700
Subject: [PATCH 029/909] Automated g4 rollback of changelist 171070760

PiperOrigin-RevId: 171089134
---
 tensorflow/python/estimator/exporter.py      | 26 +------
 tensorflow/python/estimator/exporter_test.py | 41 +---------
 tensorflow/python/estimator/training.py      | 37 +++------
 tensorflow/python/estimator/training_test.py | 81 --------------------
 4 files changed, 16 insertions(+), 169 deletions(-)

diff --git a/tensorflow/python/estimator/exporter.py b/tensorflow/python/estimator/exporter.py
index 2faca11f6e..505820dd93 100644
--- a/tensorflow/python/estimator/exporter.py
+++ b/tensorflow/python/estimator/exporter.py
@@ -40,8 +40,7 @@ class Exporter(object):
     pass
 
   @abc.abstractmethod
-  def export(self, estimator, export_path, checkpoint_path, eval_result,
-             is_the_final_export):
+  def export(self, estimator, export_path, checkpoint_path, eval_result):
     """Exports the given `Estimator` to a specific format.
 
     Args:
@@ -49,13 +48,6 @@ class Exporter(object):
       export_path: A string containing a directory where to write the export.
       checkpoint_path: The checkpoint path to export.
       eval_result: The output of `Estimator.evaluate` on this checkpoint.
-      is_the_final_export: This boolean is True when this is an export in the
-        end of training.  It is False for the intermediate exports during
-        the training.
-
-        When passing `Exporter` to `tf.estimator.train_and_evaluate`
-        `is_the_final_export` is always False if `TrainSpec.max_steps` is
-        `None`.
 
     Returns:
       The string path to the exported directory or `None` if export is skipped.
@@ -74,8 +66,7 @@ class LatestExporter(Exporter):
                serving_input_fn,
                assets_extra=None,
                as_text=False,
-               exports_to_keep=5,
-               only_the_final_export=False):
+               exports_to_keep=5):
     """Create an `Exporter` to use with `tf.estimator.EvalSpec`.
 
     Args:
@@ -95,8 +86,6 @@ class LatestExporter(Exporter):
       exports_to_keep: Number of exports to keep.  Older exports will be
        garbage-collected.  Defaults to 5.  Set to `None` to disable garbage
        collection.
-     only_the_final_export:  Only the final export in the end of training will
-        happen if this is set to True.
 
     Raises:
       ValueError: if any arguments is invalid.
@@ -106,8 +95,6 @@ class LatestExporter(Exporter):
     self._assets_extra = assets_extra
     self._as_text = as_text
     self._exports_to_keep = exports_to_keep
-    self._only_the_final_export = only_the_final_export
-
     if exports_to_keep is not None and exports_to_keep <= 0:
       raise ValueError(
           '`exports_to_keep`, if provided, must be positive number')
@@ -116,14 +103,7 @@ class LatestExporter(Exporter):
   def name(self):
     return self._name
 
-  def export(self, estimator, export_path, checkpoint_path, eval_result,
-             is_the_final_export):
-    if not is_the_final_export and self._only_the_final_export:
-      return None
-
-    if is_the_final_export:
-      tf_logging.info('Performing the final export in the end of training.')
-
+  def export(self, estimator, export_path, checkpoint_path, eval_result):
     export_result = estimator.export_savedmodel(
         export_path,
         self._serving_input_fn,
diff --git a/tensorflow/python/estimator/exporter_test.py b/tensorflow/python/estimator/exporter_test.py
index 01582ac595..2ceff1bfd6 100644
--- a/tensorflow/python/estimator/exporter_test.py
+++ b/tensorflow/python/estimator/exporter_test.py
@@ -42,7 +42,7 @@ class LatestExporterTest(test.TestCase):
           serving_input_fn=_serving_input_fn,
           exports_to_keep=0)
 
-  def test_latest_exporter(self):
+  def test_saved_model_exporter(self):
 
     def _serving_input_fn():
       pass
@@ -60,42 +60,7 @@ class LatestExporterTest(test.TestCase):
     estimator.export_savedmodel.return_value = "export_result_path"
 
     export_result = exporter.export(estimator, export_dir_base,
-                                    "checkpoint_path", {}, False)
-
-    self.assertEqual("export_result_path", export_result)
-    estimator.export_savedmodel.assert_called_with(
-        export_dir_base,
-        _serving_input_fn,
-        assets_extra={"from/path": "to/path"},
-        as_text=False,
-        checkpoint_path="checkpoint_path")
-
-  def test_only_the_last_export_is_saved(self):
-
-    def _serving_input_fn():
-      pass
-
-    export_dir_base = tempfile.mkdtemp() + "export/"
-    gfile.MkDir(export_dir_base)
-
-    exporter = exporter_lib.LatestExporter(
-        name="latest_exporter",
-        serving_input_fn=_serving_input_fn,
-        assets_extra={"from/path": "to/path"},
-        as_text=False,
-        exports_to_keep=5,
-        only_the_final_export=True)
-    estimator = test.mock.Mock(spec=estimator_lib.Estimator)
-    estimator.export_savedmodel.return_value = "export_result_path"
-
-    export_result = exporter.export(estimator, export_dir_base,
-                                    "checkpoint_path", {}, False)
-
-    self.assertFalse(estimator.export_savedmodel.called)
-    self.assertEqual(None, export_result)
-
-    export_result = exporter.export(estimator, export_dir_base,
-                                    "checkpoint_path", {}, True)
+                                    "checkpoint_path", {})
 
     self.assertEqual("export_result_path", export_result)
     estimator.export_savedmodel.assert_called_with(
@@ -128,7 +93,7 @@ class LatestExporterTest(test.TestCase):
     estimator = test.mock.Mock(spec=estimator_lib.Estimator)
     # Garbage collect all but the most recent 2 exports,
     # where recency is determined based on the timestamp directory names.
-    exporter.export(estimator, export_dir_base, None, None, False)
+    exporter.export(estimator, export_dir_base, None, None)
 
     self.assertFalse(gfile.Exists(export_dir_1))
     self.assertFalse(gfile.Exists(export_dir_2))
diff --git a/tensorflow/python/estimator/training.py b/tensorflow/python/estimator/training.py
index 0a558a67b9..1bed19760b 100644
--- a/tensorflow/python/estimator/training.py
+++ b/tensorflow/python/estimator/training.py
@@ -519,11 +519,8 @@ class _TrainingExecutor(object):
     class NewCheckpointListener(
         basic_session_run_hooks.CheckpointSaverListener):
 
-      def __init__(self, estimator, eval_spec, max_training_steps):
-        # pylint: disable=protected-access
-        self._evaluator = _TrainingExecutor._Evaluator(estimator, eval_spec,
-                                                       max_training_steps)
-        # pylint: enable=protected-access
+      def __init__(self, estimator, eval_spec):
+        self._evaluator = _TrainingExecutor._Evaluator(estimator, eval_spec)  # pylint: disable=protected-access
 
       def after_save(self, session, global_step_value):
         del session, global_step_value
@@ -531,10 +528,8 @@ class _TrainingExecutor(object):
 
     # When the underlying `Estimator` object saves a new checkpoint, we would
     # like this callback to be called so that evaluation and export can trigger.
-    saving_listeners = [
-        NewCheckpointListener(self._estimator, self._eval_spec,
-                              self._train_spec.max_steps)
-    ]
+    saving_listeners = [NewCheckpointListener(self._estimator, self._eval_spec)]
+
     return self._start_distributed_training(saving_listeners=saving_listeners)
 
   def run_evaluator(self):
@@ -571,8 +566,7 @@ class _TrainingExecutor(object):
                  'after {} secs (eval_spec.throttle_secs) or training is '
                  'finished.'.format(self._eval_spec.throttle_secs))
 
-    evaluator = _TrainingExecutor._Evaluator(self._estimator, self._eval_spec,
-                                             self._train_spec.max_steps)
+    evaluator = _TrainingExecutor._Evaluator(self._estimator, self._eval_spec)
 
     while True:
       self._estimator.train(
@@ -642,8 +636,7 @@ class _TrainingExecutor(object):
       time.sleep(start_delay_secs)
 
     latest_eval_result = None
-    evaluator = _TrainingExecutor._Evaluator(self._estimator, self._eval_spec,
-                                             self._train_spec.max_steps)
+    evaluator = _TrainingExecutor._Evaluator(self._estimator, self._eval_spec)
 
     while True:
       if latest_eval_result:
@@ -670,12 +663,11 @@ class _TrainingExecutor(object):
   class _Evaluator(object):
     """A helper class to call `Estimator.evaluate` and export model."""
 
-    def __init__(self, estimator, eval_spec, max_training_steps):
+    def __init__(self, estimator, eval_spec):
       self._estimator = estimator
       self._eval_spec = eval_spec
       self._previous_ckpt_path = None
       self._last_warning_time = 0
-      self._max_training_steps = max_training_steps
 
     def evaluate_and_export(self):
       """Evaluate and (maybe) export the current model.
@@ -720,14 +712,7 @@ class _TrainingExecutor(object):
             'Internal error: `Estimator.evaluate` result should have '
             '`global_step` in result. Given {}'.format(eval_result))
 
-      # TODO(isaprykin):  There is a potential race condition here in the
-      #  distributed setting.  The worker job that performs training
-      #  might stop at a later global step value than the evalutor job.
-      is_the_final_export = (eval_result[ops.GraphKeys.GLOBAL_STEP] >=
-                             self._max_training_steps
-                             if self._max_training_steps else False)
-      self._export_eval_result(eval_result, latest_ckpt_path,
-                               is_the_final_export)
+      self._export_eval_result(eval_result, latest_ckpt_path)
 
       self._last_warning_time = 0
       self._previous_ckpt_path = latest_ckpt_path
@@ -740,8 +725,7 @@ class _TrainingExecutor(object):
         logging.warning(message)
         self._last_warning_time = current_time
 
-    def _export_eval_result(self, eval_result, checkpoint_path,
-                            is_the_final_export):
+    def _export_eval_result(self, eval_result, checkpoint_path):
       """Export `eval_result` according to exporters in `EvalSpec`."""
       export_dir_base = os.path.join(
           compat.as_str_any(self._estimator.model_dir),
@@ -754,5 +738,4 @@ class _TrainingExecutor(object):
                 compat.as_str_any(export_dir_base),
                 compat.as_str_any(exporter.name)),
             checkpoint_path=checkpoint_path,
-            eval_result=eval_result,
-            is_the_final_export=is_the_final_export)
+            eval_result=eval_result)
diff --git a/tensorflow/python/estimator/training_test.py b/tensorflow/python/estimator/training_test.py
index 08d11d7d25..e4c400ca7f 100644
--- a/tensorflow/python/estimator/training_test.py
+++ b/tensorflow/python/estimator/training_test.py
@@ -802,46 +802,6 @@ class TrainingExecutorRunEvaluatorTest(test.TestCase):
     self.assertEqual(2, mock_est.evaluate.call_count)
     self.assertEqual(2, exporter.export.call_count)
 
-  def test_final_export_is_true_in_the_end(self):
-    training_max_step = 200
-
-    mock_est = test.mock.Mock(spec=estimator_lib.Estimator)
-    mock_est.model_dir = compat.as_bytes(test.get_temp_dir())
-    mock_est.evaluate.side_effect = [
-        {_GLOBAL_STEP_KEY: training_max_step // 2},
-        {_GLOBAL_STEP_KEY: training_max_step}
-    ]
-    mock_est.latest_checkpoint.side_effect = ['path_1', 'path_2']
-
-    mock_train_spec = test.mock.Mock(spec=training.TrainSpec)
-    mock_train_spec.max_steps = training_max_step
-
-    mock_est.times_export_fn_was_called = 0
-    mock_est.times_the_final_export_was_true = 0
-    def export(estimator, export_path, checkpoint_path, eval_result,
-               is_the_final_export):
-      del export_path, checkpoint_path, eval_result
-      estimator.times_export_fn_was_called += 1
-      if is_the_final_export:
-        estimator.times_the_final_export_was_true += 1
-
-    exporter = test.mock.PropertyMock(spec=exporter_lib.Exporter)
-    exporter.name = 'see_how_many_times_export_is_called'
-    exporter.export = export
-
-    eval_spec = training.EvalSpec(
-        input_fn=lambda: 1,
-        start_delay_secs=0,
-        throttle_secs=0,
-        exporters=exporter)
-
-    executor = training._TrainingExecutor(mock_est, mock_train_spec, eval_spec)
-    executor.run_evaluator()
-
-    self.assertEqual(2, mock_est.evaluate.call_count)
-    self.assertEqual(2, mock_est.times_export_fn_was_called)
-    self.assertEqual(1, mock_est.times_the_final_export_was_true)
-
   def test_skip_evaluation_due_to_ckpt(self):
     training_max_step = 200
     mock_est = test.mock.Mock(spec=estimator_lib.Estimator)
@@ -1174,47 +1134,6 @@ class TrainingExecutorRunLocalTest(test.TestCase):
     with self.assertRaisesRegexp(RuntimeError, _STALE_CHECKPOINT_MSG):
       executor.run_local()
 
-  def test_final_export_is_true_in_the_end(self):
-    mock_est = test.mock.Mock(spec=estimator_lib.Estimator, model_dir='path/')
-    mock_est.latest_checkpoint = self.unique_checkpoint_every_time_fn
-
-    mock_est.times_export_fn_was_called = 0
-    mock_est.times_the_final_export_was_true = 0
-    def export(estimator, export_path, checkpoint_path, eval_result,
-               is_the_final_export):
-      del export_path, checkpoint_path, eval_result
-      estimator.times_export_fn_was_called += 1
-      if is_the_final_export:
-        estimator.times_the_final_export_was_true += 1
-
-    exporter = test.mock.PropertyMock(spec=exporter_lib.Exporter)
-    exporter.name = 'see_how_many_times_export_is_called'
-    exporter.export = export
-
-    train_spec = training.TrainSpec(
-        input_fn=lambda: 1, max_steps=300, hooks=[_FakeHook()])
-    eval_spec = training.EvalSpec(
-        input_fn=lambda: 1,
-        hooks=[_FakeHook()],
-        throttle_secs=100,
-        exporters=exporter)
-    # should be called 3 times.
-    mock_est.evaluate.side_effect = [{
-        _GLOBAL_STEP_KEY: train_spec.max_steps - 100
-    }, {
-        _GLOBAL_STEP_KEY: train_spec.max_steps - 50
-    }, {
-        _GLOBAL_STEP_KEY: train_spec.max_steps
-    }]
-
-    executor = training._TrainingExecutor(mock_est, train_spec, eval_spec)
-    executor.run_local()
-
-    self.assertEqual(3, mock_est.train.call_count)
-    self.assertEqual(3, mock_est.evaluate.call_count)
-    self.assertEqual(3, mock_est.times_export_fn_was_called)
-    self.assertEqual(1, mock_est.times_the_final_export_was_true)
-
   def test_train_and_evaluate_args(self):
     mock_est = test.mock.Mock(spec=estimator_lib.Estimator, model_dir='path/')
     mock_est.latest_checkpoint.return_value = 'checkpoint_path/'
-- 
GitLab


From 466d84d2896336390e8dc1efeaaf5e385697b386 Mon Sep 17 00:00:00 2001
From: Chris Leary <leary@google.com>
Date: Wed, 4 Oct 2017 17:39:52 -0700
Subject: [PATCH 030/909] [XLA] Avoid check-failure when passing bad reduce
 window arguments.

PiperOrigin-RevId: 171090558
---
 tensorflow/compiler/xla/client/BUILD          |  2 ++
 .../xla/client/computation_builder.cc         | 16 +++++++++++---
 tensorflow/compiler/xla/client/padding.cc     | 21 +++++++++++++++++--
 tensorflow/compiler/xla/client/padding.h      | 11 +++++++++-
 .../compiler/xla/tests/reduce_window_test.cc  | 14 +++++++++++++
 5 files changed, 58 insertions(+), 6 deletions(-)

diff --git a/tensorflow/compiler/xla/client/BUILD b/tensorflow/compiler/xla/client/BUILD
index 2b142d933d..b612698143 100644
--- a/tensorflow/compiler/xla/client/BUILD
+++ b/tensorflow/compiler/xla/client/BUILD
@@ -41,7 +41,9 @@ cc_library(
     srcs = ["padding.cc"],
     hdrs = ["padding.h"],
     deps = [
+        "//tensorflow/compiler/xla:statusor",
         "//tensorflow/compiler/xla:types",
+        "//tensorflow/compiler/xla:util",
         "//tensorflow/core:lib",
     ],
 )
diff --git a/tensorflow/compiler/xla/client/computation_builder.cc b/tensorflow/compiler/xla/client/computation_builder.cc
index a80412e951..15a713513f 100644
--- a/tensorflow/compiler/xla/client/computation_builder.cc
+++ b/tensorflow/compiler/xla/client/computation_builder.cc
@@ -1433,10 +1433,20 @@ ComputationDataHandle ComputationBuilder::ReduceWindow(
     return ComputationDataHandle();
   }
 
-  return ReduceWindowWithGeneralPadding(
-      operand, init_value, computation, window_dimensions, window_strides,
+  Status padding_valid =
+      ValidatePaddingValues(AsInt64Slice(shape.ValueOrDie()->dimensions()),
+                            window_dimensions, window_strides);
+  if (!padding_valid.ok()) {
+    first_error_ = padding_valid;
+    return ComputationDataHandle();
+  }
+
+  std::vector<std::pair<int64, int64>> padding_values =
       MakePadding(AsInt64Slice(shape.ValueOrDie()->dimensions()),
-                  window_dimensions, window_strides, padding));
+                  window_dimensions, window_strides, padding);
+  return ReduceWindowWithGeneralPadding(operand, init_value, computation,
+                                        window_dimensions, window_strides,
+                                        padding_values);
 }
 
 ComputationDataHandle ComputationBuilder::ReduceWindowWithGeneralPadding(
diff --git a/tensorflow/compiler/xla/client/padding.cc b/tensorflow/compiler/xla/client/padding.cc
index 0b18d8946a..6a9cf466ac 100644
--- a/tensorflow/compiler/xla/client/padding.cc
+++ b/tensorflow/compiler/xla/client/padding.cc
@@ -17,17 +17,34 @@ limitations under the License.
 
 #include <algorithm>
 
+#include "tensorflow/compiler/xla/util.h"
 #include "tensorflow/core/lib/math/math_util.h"
 #include "tensorflow/core/platform/logging.h"
 
 namespace xla {
 
+Status ValidatePaddingValues(
+    tensorflow::gtl::ArraySlice<int64> input_dimensions,
+    tensorflow::gtl::ArraySlice<int64> window_dimensions,
+    tensorflow::gtl::ArraySlice<int64> window_strides) {
+  bool ok = input_dimensions.size() == window_dimensions.size() &&
+            input_dimensions.size() == window_strides.size();
+  if (!ok) {
+    return InvalidArgument(
+        "Want input dimensions size %zu = window dimensions size %zu = window "
+        "strides size %zu",
+        input_dimensions.size(), window_dimensions.size(),
+        window_strides.size());
+  }
+  return Status::OK();
+}
+
 std::vector<std::pair<int64, int64>> MakePadding(
     tensorflow::gtl::ArraySlice<int64> input_dimensions,
     tensorflow::gtl::ArraySlice<int64> window_dimensions,
     tensorflow::gtl::ArraySlice<int64> window_strides, Padding padding) {
-  CHECK_EQ(input_dimensions.size(), window_dimensions.size());
-  CHECK_EQ(input_dimensions.size(), window_strides.size());
+  TF_CHECK_OK(ValidatePaddingValues(input_dimensions, window_dimensions,
+                                    window_strides));
   std::vector<std::pair<int64, int64>> low_high_padding;
   switch (padding) {
     case Padding::kValid:
diff --git a/tensorflow/compiler/xla/client/padding.h b/tensorflow/compiler/xla/client/padding.h
index dce2d87e8d..e23b0b3a90 100644
--- a/tensorflow/compiler/xla/client/padding.h
+++ b/tensorflow/compiler/xla/client/padding.h
@@ -19,6 +19,7 @@ limitations under the License.
 #include <utility>
 #include <vector>
 
+#include "tensorflow/compiler/xla/statusor.h"
 #include "tensorflow/compiler/xla/types.h"
 #include "tensorflow/core/lib/gtl/array_slice.h"
 
@@ -37,6 +38,14 @@ enum class Padding {
   kValid,
 };
 
+// Validates that the slices are acceptable for determining padding -- this can
+// be used to check the preconditions of MakePadding below to produce an error
+// message that can be returned to the user.
+Status ValidatePaddingValues(
+    tensorflow::gtl::ArraySlice<int64> input_dimensions,
+    tensorflow::gtl::ArraySlice<int64> window_dimensions,
+    tensorflow::gtl::ArraySlice<int64> window_strides);
+
 // Returns the padding needed for the base area, given the base area dimensions,
 // window dimensions, strides, and the type of padding.
 //
@@ -51,7 +60,7 @@ enum class Padding {
 std::vector<std::pair<int64, int64>> MakePadding(
     tensorflow::gtl::ArraySlice<int64> input_dimensions,
     tensorflow::gtl::ArraySlice<int64> window_dimensions,
-    tensorflow::gtl::ArraySlice<int64> strides, Padding padding);
+    tensorflow::gtl::ArraySlice<int64> window_strides, Padding padding);
 
 }  // namespace xla
 
diff --git a/tensorflow/compiler/xla/tests/reduce_window_test.cc b/tensorflow/compiler/xla/tests/reduce_window_test.cc
index 7b7f268728..6c9b62b48d 100644
--- a/tensorflow/compiler/xla/tests/reduce_window_test.cc
+++ b/tensorflow/compiler/xla/tests/reduce_window_test.cc
@@ -76,6 +76,20 @@ class ReduceWindowTest : public ClientLibraryTestBase {
   ComputationBuilder builder_;
 };
 
+TEST_F(ReduceWindowTest, MismatchedRanksGivesErrorStatus) {
+  const auto input = builder_.ConstantR1<float>({1, 1, 1, 1});
+  const auto init_value = builder_.ConstantR0<float>(0);
+  TF_ASSERT_OK(builder_.first_error());
+  builder_.ReduceWindow(input, init_value,
+                        CreateScalarAddComputation(F32, &builder_),
+                        /*window_dimensions=*/{1, 2},
+                        /*window_strides=*/{1}, Padding::kValid);
+  ASSERT_EQ(builder_.first_error().code(), tensorflow::error::INVALID_ARGUMENT)
+      << builder_.first_error();
+  ASSERT_THAT(builder_.first_error().error_message(),
+              ::testing::HasSubstr("Want input dimensions size"));
+}
+
 TEST_F(ReduceWindowTest, Min3In5Stride2) {
   const auto input = builder_.ConstantR1<float>({10000, 1000, 100, 10, 1});
   ReduceWindowMin(input, {3}, {2}, Padding::kValid);
-- 
GitLab


From 578b9a29b252b4cbd57c2f6bdd9eaef4aae3e207 Mon Sep 17 00:00:00 2001
From: Jianwei Xie <xiejw@google.com>
Date: Wed, 4 Oct 2017 18:24:54 -0700
Subject: [PATCH 031/909] Adds integration test for
 tf.estimator.train_and_evaluate.

PiperOrigin-RevId: 171094690
---
 tensorflow/python/estimator/BUILD            |   6 +-
 tensorflow/python/estimator/training_test.py | 145 ++++++++++++++++++-
 2 files changed, 149 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/estimator/BUILD b/tensorflow/python/estimator/BUILD
index 9085ef419b..3507d9fedc 100644
--- a/tensorflow/python/estimator/BUILD
+++ b/tensorflow/python/estimator/BUILD
@@ -141,12 +141,15 @@ py_library(
 
 py_test(
     name = "training_test",
-    size = "small",
+    size = "medium",
     srcs = ["training_test.py"],
+    shard_count = 4,
     srcs_version = "PY2AND3",
     deps = [
+        ":dnn",
         ":estimator",
         ":exporter",
+        ":inputs",
         ":run_config",
         ":training",
         "//tensorflow/python:client_testlib",
@@ -155,6 +158,7 @@ py_test(
         "//tensorflow/python:platform",
         "//tensorflow/python:training",
         "//tensorflow/python:util",
+        "//tensorflow/python/feature_column",
     ],
 )
 
diff --git a/tensorflow/python/estimator/training_test.py b/tensorflow/python/estimator/training_test.py
index e4c400ca7f..51aed757a2 100644
--- a/tensorflow/python/estimator/training_test.py
+++ b/tensorflow/python/estimator/training_test.py
@@ -19,19 +19,32 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-
+import glob
 import json
+import os
 import random
+import shutil
+import tempfile
 import time
 
+import numpy as np
+
 from tensorflow.python.estimator import estimator as estimator_lib
 from tensorflow.python.estimator import exporter as exporter_lib
 from tensorflow.python.estimator import run_config as run_config_lib
 from tensorflow.python.estimator import training
+from tensorflow.python.estimator.canned import dnn
+from tensorflow.python.estimator.canned import prediction_keys
+from tensorflow.python.estimator.export import export as export_lib
+from tensorflow.python.estimator.inputs import numpy_io
+from tensorflow.python.feature_column import feature_column
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.platform import gfile
 from tensorflow.python.platform import test
 from tensorflow.python.platform import tf_logging as logging
+from tensorflow.python.summary import summary_iterator
+from tensorflow.python.summary.writer import writer_cache
 from tensorflow.python.training import monitored_session
 from tensorflow.python.training import server_lib
 from tensorflow.python.training import session_run_hook
@@ -1230,5 +1243,135 @@ class TrainingExecutorRunLocalTest(test.TestCase):
       executor.run_local()
 
 
+class TrainAndEvaluateIntegrationTest(test.TestCase):
+
+  def setUp(self):
+    self._model_dir = tempfile.mkdtemp()
+
+  def tearDown(self):
+    if self._model_dir:
+      shutil.rmtree(self._model_dir)
+
+  def _as_label(self, data_in_float):
+    return np.rint(data_in_float).astype(np.int64)
+
+  def _get_exporter(self, name, fc):
+    feature_spec = feature_column.make_parse_example_spec(fc)
+    serving_input_receiver_fn = (
+        export_lib.build_parsing_serving_input_receiver_fn(feature_spec))
+    return exporter_lib.LatestExporter(
+        name, serving_input_fn=serving_input_receiver_fn)
+
+  def _extract_loss_and_global_step(self, event_folder):
+    """Returns the loss and global step in last event."""
+    event_paths = glob.glob(os.path.join(event_folder, 'events*'))
+
+    loss = None
+    global_step_count = None
+
+    for e in summary_iterator.summary_iterator(event_paths[-1]):
+      current_loss = None
+      for v in e.summary.value:
+        if v.tag == 'loss':
+          current_loss = v.simple_value
+
+      # If loss is not found, global step is meaningless.
+      if current_loss is None:
+        continue
+
+      current_global_step = e.step
+      if global_step_count is None or current_global_step > global_step_count:
+        global_step_count = current_global_step
+        loss = current_loss
+
+    return (loss, global_step_count)
+
+  def test_complete_flow_with_non_distributed_configuration(self):
+    n_classes = 3
+    input_dimension = 2
+    batch_size = 10
+
+    eval_name = 'foo'
+    exporter_name = 'saved_model_exporter'
+
+    # max_steps should be larger than save_summary_steps
+    max_steps = 10
+    save_summary_steps = 2
+
+    data = np.linspace(
+        0., n_classes - 1., batch_size * input_dimension, dtype=np.float32)
+    x_data = data.reshape(batch_size, input_dimension)
+    y_data = np.reshape(self._as_label(data[:batch_size]), (batch_size, 1))
+
+    # learn y = x
+    train_input_fn = numpy_io.numpy_input_fn(
+        x={'x': x_data},
+        y=y_data,
+        batch_size=batch_size,
+        num_epochs=None,
+        shuffle=True)
+
+    eval_input_fn = numpy_io.numpy_input_fn(
+        x={'x': x_data},
+        y=y_data,
+        batch_size=batch_size,
+        num_epochs=1,
+        shuffle=False)
+
+    predict_input_fn = numpy_io.numpy_input_fn(
+        x={'x': x_data},
+        batch_size=batch_size,
+        shuffle=False)
+
+    feature_columns = [
+        feature_column.numeric_column('x', shape=(input_dimension,))]
+
+    est = dnn.DNNClassifier(
+        hidden_units=(2, 2),
+        feature_columns=feature_columns,
+        n_classes=n_classes,
+        config=run_config_lib.RunConfig(save_summary_steps=save_summary_steps),
+        model_dir=self._model_dir)
+
+    train_spec = training.TrainSpec(input_fn=train_input_fn,
+                                    max_steps=max_steps)
+
+    eval_spec = training.EvalSpec(
+        name=eval_name, input_fn=eval_input_fn, steps=None,
+        exporters=self._get_exporter(exporter_name, feature_columns),
+        throttle_secs=2)
+
+    training.train_and_evaluate(est, train_spec, eval_spec)
+
+    # Make sure nothing is stuck in limbo.
+    writer_cache.FileWriterCache.clear()
+
+    # Examine the training events. Use a range to check global step to avoid
+    # flakyness due to global step race condition.
+    training_loss, training_global_step = self._extract_loss_and_global_step(
+        est.model_dir)
+    self.assertIsNotNone(training_loss)
+    self.assertTrue(
+        max_steps - save_summary_steps < training_global_step <= max_steps)
+
+    # Examine the eval events. The global step should be accurate.
+    eval_loss, eval_global_step = self._extract_loss_and_global_step(
+        event_folder=os.path.join(est.model_dir, 'eval_' + eval_name))
+    self.assertIsNotNone(eval_loss)
+    self.assertEqual(max_steps, eval_global_step)
+
+    # Examine the export folder.
+    export_dir = os.path.join(os.path.join(est.model_dir, 'export'),
+                              exporter_name)
+    self.assertTrue(gfile.Exists(export_dir))
+
+    # Examine the ckpt for predict.
+    predicted_proba = np.array([
+        x[prediction_keys.PredictionKeys.PROBABILITIES]
+        for x in est.predict(predict_input_fn)
+    ])
+    self.assertAllEqual((batch_size, n_classes), predicted_proba.shape)
+
+
 if __name__ == '__main__':
   test.main()
-- 
GitLab


From 2ae5bfce5519fc40019378280a6f26d36d924cf0 Mon Sep 17 00:00:00 2001
From: James Qin <jamesqin@google.com>
Date: Wed, 4 Oct 2017 18:31:16 -0700
Subject: [PATCH 032/909] Introduce CudnnRNN layers

* Layerize CudnnRNN APIs
  * Support build(), call() APIs
  * Support building custom saveable() as a member method
  * Custom saveable built as part of build()
* Support forward-compatible opaque param initialization w/ weight & bias initializer.
* Add more documentation.

Unittest revamp
* Introduce CudnnTestModel class to build graph used by all unittests, avoid repeatedly building similar graphs.
* Split tests by RNN types, for more explicit error localization.
* Use custom gradient check routine which is cleaner.
* Deleted golden-based inference tests since we use regular rnn as reference impl now.

PiperOrigin-RevId: 171095161
---
 tensorflow/contrib/cudnn_rnn/BUILD            |   61 +-
 .../python/kernel_tests/cudnn_rnn_test.py     | 1050 +++++++++++++++++
 .../cudnn_rnn/python/layers/cudnn_rnn.py      |  552 +++++++++
 .../cudnn_rnn/python/ops/cudnn_rnn_ops.py     |  111 +-
 4 files changed, 1724 insertions(+), 50 deletions(-)
 create mode 100644 tensorflow/contrib/cudnn_rnn/python/kernel_tests/cudnn_rnn_test.py
 create mode 100644 tensorflow/contrib/cudnn_rnn/python/layers/cudnn_rnn.py

diff --git a/tensorflow/contrib/cudnn_rnn/BUILD b/tensorflow/contrib/cudnn_rnn/BUILD
index d4214587cd..ae9413fdd6 100644
--- a/tensorflow/contrib/cudnn_rnn/BUILD
+++ b/tensorflow/contrib/cudnn_rnn/BUILD
@@ -54,7 +54,7 @@ tf_gen_op_wrapper_py(
 )
 
 tf_custom_op_py_library(
-    name = "cudnn_rnn_py",
+    name = "cudnn_rnn_ops_py",
     srcs = [
         "__init__.py",
         "python/ops/cudnn_rnn_ops.py",
@@ -81,10 +81,67 @@ tf_custom_op_py_library(
     ],
 )
 
+tf_custom_op_py_library(
+    name = "cudnn_rnn_py",
+    srcs = [
+        "__init__.py",
+        "python/layers/cudnn_rnn.py",
+    ],
+    dso = [
+        ":python/ops/_cudnn_rnn_ops.so",
+    ],
+    kernels = [
+        ":cudnn_rnn_kernels",
+        ":cudnn_rnn_ops_op_lib",
+    ],
+    srcs_version = "PY2AND3",
+    visibility = ["//visibility:public"],
+    deps = [
+        ":cudnn_rnn_ops",
+        ":cudnn_rnn_ops_py",
+        "//tensorflow/contrib/util:util_py",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:control_flow_ops",
+        "//tensorflow/python:framework",
+        "//tensorflow/python:framework_for_generated_wrappers",
+        "//tensorflow/python:platform",
+        "//tensorflow/python:state_ops",
+        "//tensorflow/python:training",
+    ],
+)
+
 cuda_py_test(
     name = "cudnn_rnn_ops_test",
     size = "large",
     srcs = ["python/kernel_tests/cudnn_rnn_ops_test.py"],
+    additional_deps = [
+        ":cudnn_rnn_ops_py",
+        "//tensorflow/core:protos_all_py",
+        "//tensorflow/contrib/rnn:rnn_py",
+        "//tensorflow/python/ops/losses:losses",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework",
+        "//tensorflow/python:framework_for_generated_wrappers",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:platform_test",
+        "//tensorflow/python:random_ops",
+        "//tensorflow/python:state_ops",
+        "//tensorflow/python:training",
+        "//tensorflow/python:variables",
+    ],
+    shard_count = 6,
+    tags = [
+        "manual",
+        "requires_cudnn5",
+    ],
+)
+
+cuda_py_test(
+    name = "cudnn_rnn_test",
+    size = "large",
+    srcs = ["python/kernel_tests/cudnn_rnn_test.py"],
     additional_deps = [
         ":cudnn_rnn_py",
         "//tensorflow/core:protos_all_py",
@@ -114,7 +171,7 @@ cuda_py_test(
     size = "large",
     srcs = ["python/kernel_tests/cudnn_rnn_ops_benchmark.py"],
     additional_deps = [
-        ":cudnn_rnn_py",
+        ":cudnn_rnn_ops_py",
         "//tensorflow/contrib/rnn:rnn_py",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:client",
diff --git a/tensorflow/contrib/cudnn_rnn/python/kernel_tests/cudnn_rnn_test.py b/tensorflow/contrib/cudnn_rnn/python/kernel_tests/cudnn_rnn_test.py
new file mode 100644
index 0000000000..9e627bcaf4
--- /dev/null
+++ b/tensorflow/contrib/cudnn_rnn/python/kernel_tests/cudnn_rnn_test.py
@@ -0,0 +1,1050 @@
+# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for Cudnn RNN models."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import itertools
+import os
+import unittest
+
+import numpy as np
+
+from tensorflow.contrib.cudnn_rnn.python.layers import cudnn_rnn
+from tensorflow.contrib.cudnn_rnn.python.ops import cudnn_rnn_ops
+from tensorflow.contrib.rnn.python.ops import rnn as contrib_rnn_lib
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import random_seed
+from tensorflow.python.framework.test_util import TensorFlowTestCase
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import gen_nn_ops
+from tensorflow.python.ops import gradients_impl as gradients
+from tensorflow.python.ops import init_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import random_ops
+from tensorflow.python.ops import rnn as rnn_lib
+from tensorflow.python.ops import rnn_cell_impl
+from tensorflow.python.ops import state_ops
+from tensorflow.python.ops import variable_scope as vs
+from tensorflow.python.ops import variables
+from tensorflow.python.ops.losses import losses
+from tensorflow.python.platform import googletest
+from tensorflow.python.platform import test
+from tensorflow.python.platform import tf_logging as logging
+from tensorflow.python.training import gradient_descent
+from tensorflow.python.training import saver as saver_lib
+
+CUDNN_LSTM = cudnn_rnn_ops.CUDNN_LSTM
+CUDNN_GRU = cudnn_rnn_ops.CUDNN_GRU
+CUDNN_RNN_RELU = cudnn_rnn_ops.CUDNN_RNN_RELU
+CUDNN_RNN_TANH = cudnn_rnn_ops.CUDNN_RNN_TANH
+CUDNN_RNN_UNIDIRECTION = cudnn_rnn_ops.CUDNN_RNN_UNIDIRECTION
+CUDNN_RNN_BIDIRECTION = cudnn_rnn_ops.CUDNN_RNN_BIDIRECTION
+
+CUDNN_LSTM_PARAMS_PER_LAYER = cudnn_rnn_ops.CUDNN_LSTM_PARAMS_PER_LAYER
+CUDNN_GRU_PARAMS_PER_LAYER = cudnn_rnn_ops.CUDNN_GRU_PARAMS_PER_LAYER
+CUDNN_RNN_TANH_PARAMS_PER_LAYER = cudnn_rnn_ops.CUDNN_RNN_TANH_PARAMS_PER_LAYER
+CUDNN_RNN_RELU_PARAMS_PER_LAYER = cudnn_rnn_ops.CUDNN_RNN_RELU_PARAMS_PER_LAYER
+
+
+class CudnnTestModel(object):
+  """Model with convenient APIs for easier building and running test graph.
+
+  The graph built is used by all tests below to avoid repeatedly building
+  similar test graphs.
+  """
+
+  def __init__(self,
+               rnn_mode,
+               num_layers,
+               num_units,
+               input_size,
+               direction=CUDNN_RNN_UNIDIRECTION,
+               dropout=0.,
+               dtype=dtypes.float32,
+               training=False,
+               kernel_initializer=None,
+               bias_initializer=None):
+    if dtype not in (dtypes.float32, dtypes.float64):
+      raise ValueError("Invalid dtype: %s" % dtype)
+    self._dtype = dtype
+
+    self._inputs = array_ops.placeholder(
+        dtype=dtype, shape=[None, None, input_size], name="inputs")
+    h = array_ops.placeholder(
+        dtype=dtype, shape=[None, None, num_units], name="h")
+    c = array_ops.placeholder(
+        dtype=dtype, shape=[None, None, num_units], name="c")
+    if rnn_mode == CUDNN_LSTM:
+      model_fn = cudnn_rnn.CudnnLSTM
+      self._initial_state = (h, c)
+    elif rnn_mode == CUDNN_GRU:
+      model_fn = cudnn_rnn.CudnnGRU
+      self._initial_state = (h,)
+    elif rnn_mode == CUDNN_RNN_TANH:
+      model_fn = cudnn_rnn.CudnnRNNTanh
+      self._initial_state = (h,)
+    elif rnn_mode == CUDNN_RNN_RELU:
+      model_fn = cudnn_rnn.CudnnRNNRelu
+      self._initial_state = (h,)
+    else:
+      raise ValueError("Invalid rnn_mode: %s" % rnn_mode)
+    self._rnn = model_fn(
+        num_layers,
+        num_units,
+        direction=direction,
+        dropout=dropout,
+        dtype=dtype,
+        kernel_initializer=kernel_initializer,
+        bias_initializer=bias_initializer)
+    self._rnn.build([None, None, input_size])
+
+    self._outputs, self._output_state = self._rnn(
+        self._inputs, initial_state=self._initial_state, training=training)
+
+  def _AddUp(self, outputs, output_state):
+    total = math_ops.reduce_sum(outputs)
+    for s in output_state:
+      total += math_ops.reduce_sum(s)
+    return total
+
+  @property
+  def inputs(self):
+    return self._inputs
+
+  @property
+  def initial_state(self):
+    return self._initial_state
+
+  @property
+  def outputs(self):
+    return self._outputs
+
+  @property
+  def output_state(self):
+    return self._output_state
+
+  @property
+  def rnn(self):
+    return self._rnn
+
+  @property
+  def total_sum(self):
+    return self._AddUp(self.outputs, self.output_state)
+
+  def SynthesizeInput(self, seq_length, batch_size, seed=1234):
+    """Synthesizes input and initial state values for testing."""
+    np.random.seed(seed)
+    num_layers = self._rnn.num_layers
+    dir_count = self._rnn.num_dirs
+    num_units = self._rnn.num_units
+    input_size = self._rnn.input_size
+
+    np_dtype = np.float32 if self._dtype == dtypes.float32 else np.float64
+    inputs = np.random.randn(seq_length, batch_size,
+                             input_size).astype(np_dtype)
+    input_h = np.random.randn(num_layers * dir_count, batch_size,
+                              num_units).astype(np_dtype)
+    if self._rnn.rnn_mode == CUDNN_LSTM:
+      input_c = np.random.randn(num_layers * dir_count, batch_size,
+                                num_units).astype(np_dtype)
+      initial_state = (input_h, input_c)
+    else:
+      initial_state = (input_h,)
+    return inputs, initial_state
+
+  def ZeroState(self, batch_size):
+    num_layers = self._rnn.num_layers
+    dir_count = self._rnn.num_dirs
+    num_units = self._rnn.num_units
+
+    np_dtype = np.float32 if self._dtype == dtypes.float32 else np.float64
+    input_h = np.zeros((num_layers * dir_count, batch_size,
+                        num_units)).astype(np_dtype)
+    if self._rnn.rnn_mode == CUDNN_LSTM:
+      input_c = np.zeros((num_layers * dir_count, batch_size,
+                          num_units)).astype(np_dtype)
+      initial_state = (input_h, input_c)
+    else:
+      initial_state = (input_h,)
+    return initial_state
+
+  def FProp(self, inputs_t, initial_state_t, training):
+    """Builds additional subgraph with given inputs and state.
+
+    Args:
+      inputs_t: a tensor.
+      initial_state_t: a tensor.
+      training: boolean, true if training mode.
+    Returns:
+      A tensor of the forward pass output of the model.
+    """
+    outputs, output_state = self._rnn(
+        inputs_t, initial_state=initial_state_t, training=training)
+    return self._AddUp(outputs, output_state)
+
+  def Feed(self, sess, inputs, initial_state=None, return_sum=True):
+    """Runs graph with given inputs and initial state."""
+    batch_size = inputs.shape[1]
+    if initial_state is None:
+      initial_state = self.ZeroState(batch_size)
+    if return_sum:
+      return sess.run(
+          self.total_sum,
+          feed_dict={self.inputs: inputs,
+                     self.initial_state: initial_state})
+    else:
+      return sess.run(
+          [self.outputs, self.output_state],
+          feed_dict={self.inputs: inputs,
+                     self.initial_state: initial_state})
+
+
+def _CreateCudnnCompatibleCanonicalRNN(rnn, inputs, is_bidi=False, scope=None):
+  mode = rnn.rnn_mode
+  num_units = rnn.num_units
+  num_layers = rnn.num_layers
+
+  # To reuse cuDNN-trained models, must use cudnn compatible rnn cells.
+  if mode == CUDNN_LSTM:
+    single_cell = lambda: cudnn_rnn_ops.CudnnCompatibleLSTMCell(num_units)
+  elif mode == CUDNN_GRU:
+    single_cell = lambda: cudnn_rnn_ops.CudnnCompatibleGRUCell(num_units)
+  elif mode == CUDNN_RNN_TANH:
+    single_cell = (lambda: rnn_cell_impl.BasicRNNCell(num_units, math_ops.tanh))
+  elif mode == CUDNN_RNN_RELU:
+    single_cell = (
+        lambda: rnn_cell_impl.BasicRNNCell(num_units, gen_nn_ops.relu))
+  else:
+    raise ValueError("%s is not supported!" % mode)
+
+  if not is_bidi:
+    cell = rnn_cell_impl.MultiRNNCell(
+        [single_cell() for _ in range(num_layers)])
+    return rnn_lib.dynamic_rnn(
+        cell, inputs, dtype=dtypes.float32, time_major=True, scope=scope)
+  else:
+    cells_fw = [single_cell() for _ in range(num_layers)]
+    cells_bw = [single_cell() for _ in range(num_layers)]
+
+    (outputs, output_state_fw,
+     output_state_bw) = contrib_rnn_lib.stack_bidirectional_dynamic_rnn(
+         cells_fw,
+         cells_bw,
+         inputs,
+         dtype=dtypes.float32,
+         time_major=True,
+         scope=scope)
+    return outputs, (output_state_fw, output_state_bw)
+
+
+class CudnnRNNTestBasic(TensorFlowTestCase):
+
+  @unittest.skipUnless(test.is_built_with_cuda(),
+                       "Test only applicable when running on GPUs")
+  def testLayerBasic(self):
+    num_layers = 4
+    num_units = 2
+    batch_size = 8
+    direction = CUDNN_RNN_UNIDIRECTION
+    dir_count = 1
+
+    with vs.variable_scope("main"):
+      kernel_initializer = init_ops.constant_initializer(0.)
+      bias_initializer = init_ops.constant_initializer(0.)
+      inputs = random_ops.random_uniform([
+          num_layers * dir_count, batch_size, num_units], dtype=dtypes.float32)
+
+      lstm = cudnn_rnn.CudnnLSTM(num_layers, num_units,
+                                 direction=direction,
+                                 kernel_initializer=kernel_initializer,
+                                 bias_initializer=bias_initializer,
+                                 name="awesome_lstm")
+
+      # Build the layer
+      outputs1, _ = lstm(inputs)
+      # Reuse the layer
+      outputs2, _ = lstm(inputs)
+
+      total_sum1 = math_ops.reduce_sum(outputs1)
+      total_sum2 = math_ops.reduce_sum(outputs2)
+
+    with vs.variable_scope("main", reuse=True):
+      lstm = cudnn_rnn.CudnnLSTM(num_layers, num_units,
+                                 direction=direction,
+                                 kernel_initializer=kernel_initializer,
+                                 bias_initializer=bias_initializer,
+                                 name="awesome_lstm")
+
+      # Reuse the layer
+      outputs3, _ = lstm(inputs)
+      total_sum3 = math_ops.reduce_sum(outputs3)
+
+    self.assertEqual(1, len(variables.trainable_variables()))
+    self.assertEqual(1, len(ops.get_collection(ops.GraphKeys.SAVEABLE_OBJECTS)))
+    self.assertEqual("main/awesome_lstm/opaque_kernel",
+                     variables.trainable_variables()[0].op.name)
+
+    with self.test_session(use_gpu=True) as sess:
+      sess.run(variables.global_variables_initializer())
+      (total_sum1_v, total_sum2_v, total_sum3_v) = sess.run(
+          [total_sum1, total_sum2, total_sum3])
+      self.assertEqual(0, total_sum1_v)
+      self.assertEqual(0, total_sum2_v)
+      self.assertEqual(0, total_sum3_v)
+
+
+# TODO(jamesqin): Transform to parameterized test after it is included in the
+# TF open source codebase.
+class CudnnRNNTestSaveRestore(TensorFlowTestCase):
+
+  def _CompareWeights(self, lhs, rhs):
+    self.assertEqual(len(lhs), len(rhs))
+    for lw, rw in zip(lhs, rhs):
+      self.assertAllEqual(lw, rw)
+
+  def _CompareBiases(self, lhs, rhs, rnn_mode, num_layers, direction):
+    self.assertEqual(len(lhs), len(rhs))
+    if rnn_mode == CUDNN_LSTM:
+      num_params_per_layer = CUDNN_LSTM_PARAMS_PER_LAYER
+    elif rnn_mode == CUDNN_GRU:
+      num_params_per_layer = CUDNN_GRU_PARAMS_PER_LAYER
+    elif rnn_mode == CUDNN_RNN_TANH:
+      num_params_per_layer = CUDNN_RNN_TANH_PARAMS_PER_LAYER
+    else:
+      num_params_per_layer = CUDNN_RNN_RELU_PARAMS_PER_LAYER
+    num_dirs = 1 if direction == CUDNN_RNN_UNIDIRECTION else 2
+    num_params_per_layer *= num_dirs
+    self.assertEqual(num_params_per_layer * num_layers, len(lhs))
+
+    for i in range(num_layers):
+      layer_lhs = lhs[i * num_params_per_layer: (i+1) * num_params_per_layer]
+      layer_rhs = rhs[i * num_params_per_layer: (i+1) * num_params_per_layer]
+      if direction == CUDNN_RNN_UNIDIRECTION:
+        self._CompareSingleLayerBiases(layer_lhs, layer_rhs)
+      else:
+        size = len(layer_lhs)
+        fw_lhs, bw_lhs = layer_lhs[:size//2], layer_lhs[size//2:]
+        fw_rhs, bw_rhs = layer_rhs[:size//2], layer_rhs[size//2:]
+        self._CompareSingleLayerBiases(fw_lhs, fw_rhs)
+        self._CompareSingleLayerBiases(bw_lhs, bw_rhs)
+
+  def _CompareSingleLayerBiases(self, lhs, rhs):
+    self.assertEqual(len(lhs), len(rhs))
+
+    lf_lhs, rt_lhs = lhs[:len(lhs)//2], lhs[len(lhs)//2:]
+    lf_rhs, rt_rhs = rhs[:len(rhs)//2], rhs[len(rhs)//2:]
+    self.assertEqual(len(lf_lhs), len(rt_lhs))
+    self.assertEqual(len(lf_rhs), len(rt_rhs))
+
+    sum_lhs, sum_rhs = [], []
+    for lf, rt in zip(lf_lhs, rt_lhs):
+      sum_lhs.append(lf + rt)
+    for lf, rt in zip(lf_rhs, rt_rhs):
+      sum_rhs.append(lf + rt)
+    self.assertEqual(len(sum_lhs), len(sum_rhs))
+    for lf, rt in zip(sum_lhs, sum_rhs):
+      self.assertAllEqual(lf, rt)
+
+  def _TestSaveRestoreVariable(self, rnn_mode, direction, dtype):
+    input_size = 3
+    num_layers = 2
+    num_units = 7
+    with ops.Graph().as_default() as g:
+      random_seed.set_random_seed(1234)
+      model = CudnnTestModel(
+          rnn_mode,
+          num_layers,
+          num_units,
+          input_size,
+          direction=direction,
+          dtype=dtype)
+      rnn = model.rnn
+      save_path = os.path.join(self.get_temp_dir(),
+                               "save-restore-variable-test")
+      saver = saver_lib.Saver()
+      weights, biases = model.rnn.saveable._OpaqueParamsToCanonical()
+      opaque_params = rnn.trainable_variables[0]
+      # CudnnTestModel() creates CudnnOpaqueParamsSaveable that helps saver save
+      # Cudnn vars in canonical format.
+      reset_op = state_ops.assign(
+          opaque_params,
+          array_ops.zeros(array_ops.shape(opaque_params), dtype=dtype))
+      # Passing graph explictly, otherwise an old sess would be reused.
+      with self.test_session(use_gpu=True, graph=g) as sess:
+        sess.run(variables.global_variables_initializer())
+        val = saver.save(sess, save_path)
+        self.assertEqual(save_path, val)
+        weights_v, biases_v = sess.run([weights, biases])
+
+        # Reset opaque param
+        sess.run(reset_op)
+        saver.restore(sess, save_path)
+        weights_v_restored, biases_v_restored = sess.run([weights, biases])
+
+        self._CompareWeights(weights_v, weights_v_restored)
+        self._CompareBiases(biases_v, biases_v_restored, rnn_mode, num_layers,
+                            direction)
+
+  def _TestSaveRestoreTwoVariables(self, rnn_mode, direction, dtype):
+    input_size = 3
+    num_layers = 2
+    num_units = 7
+    with ops.Graph().as_default() as g:
+      random_seed.set_random_seed(1234)
+      with vs.variable_scope("m1"):
+        model1 = CudnnTestModel(
+            rnn_mode,
+            num_layers,
+            num_units,
+            input_size,
+            direction=direction,
+            dtype=dtype)
+      with vs.variable_scope("m2"):
+        model2 = CudnnTestModel(
+            rnn_mode,
+            num_layers,
+            num_units,
+            input_size,
+            direction=direction,
+            dtype=dtype)
+      opaque_params = (model1.rnn.trainable_variables[0],
+                       model2.rnn.trainable_variables[0])
+      weights1, biases1 = model1.rnn.saveable._OpaqueParamsToCanonical()
+      weights2, biases2 = model2.rnn.saveable._OpaqueParamsToCanonical()
+      reset_params = [
+          state_ops.assign(params,
+                           array_ops.zeros_like(params, dtype=dtype))
+          for params in opaque_params
+      ]
+      reset_op = control_flow_ops.group(*reset_params)
+      save_path = os.path.join(self.get_temp_dir(),
+                               "save-restore-variable-test2")
+      saver = saver_lib.Saver()
+      # Passing graph explictly, otherwise an old sess would be reused.
+      with self.test_session(use_gpu=True, graph=g) as sess:
+        sess.run(variables.global_variables_initializer())
+        val = saver.save(sess, save_path)
+        self.assertEqual(save_path, val)
+
+        weights1_v, biases1_v = sess.run([weights1, biases1])
+        weights2_v, biases2_v = sess.run([weights2, biases2])
+
+        sess.run(reset_op)
+        saver.restore(sess, save_path)
+        weights1_v_restored, biases1_v_restored = sess.run([weights1, biases1])
+        weights2_v_restored, biases2_v_restored = sess.run([weights2, biases2])
+
+        self._CompareWeights(weights1_v, weights1_v_restored)
+        self._CompareWeights(weights2_v, weights2_v_restored)
+        self._CompareBiases(biases1_v, biases1_v_restored, rnn_mode, num_layers,
+                            direction)
+        self._CompareBiases(biases2_v, biases2_v_restored, rnn_mode, num_layers,
+                            direction)
+
+  def _TestSaveRestoreOutput(self, rnn_mode, direction, dtype):
+    with ops.Graph().as_default() as g:
+      num_layers = 2
+      num_units = 7
+      input_size = 7
+      seq_length = 8
+      batch_size = 4
+      model = CudnnTestModel(
+          rnn_mode,
+          num_layers,
+          num_units,
+          input_size,
+          direction=direction,
+          dtype=dtype,
+          training=False)
+      rnn = model.rnn
+
+      save_path = os.path.join(self.get_temp_dir(), "save-restore-output-test")
+      saver = saver_lib.Saver()
+
+      # Only one opaque var in a cudnn layer.
+      assert len(rnn.trainable_variables) == 1
+      reset_params = state_ops.assign(
+          rnn.trainable_variables[0],
+          array_ops.zeros(
+              array_ops.shape(rnn.trainable_variables[0]), dtype=dtype))
+
+      # Passing graph explictly, otherwise an old sess would be reused.
+      with self.test_session(use_gpu=True, graph=g) as sess:
+        sess.run(variables.global_variables_initializer())
+        inputs, initial_state = model.SynthesizeInput(seq_length, batch_size)
+        total_sum_v = model.Feed(sess, inputs, initial_state)
+        val = saver.save(sess, save_path)
+        self.assertEqual(save_path, val)
+
+        sess.run(reset_params)
+        saver.restore(sess, save_path)
+        total_sum_v_restored = model.Feed(sess, inputs, initial_state)
+        self.assertAllClose(total_sum_v, total_sum_v_restored, atol=1e-5)
+
+  def _TestSaveRestoreHelper(self, rnn_mode):
+    directions = [CUDNN_RNN_UNIDIRECTION, CUDNN_RNN_BIDIRECTION]
+    dtype_list = [dtypes.float32, dtypes.float64]
+    for direction, dtype in itertools.product(directions, dtype_list):
+      self._TestSaveRestoreVariable(rnn_mode, direction, dtype)
+      self._TestSaveRestoreTwoVariables(rnn_mode, direction, dtype)
+      self._TestSaveRestoreOutput(rnn_mode, direction, dtype)
+
+  @unittest.skipUnless(test.is_built_with_cuda(),
+                       "Test only applicable when running on GPUs")
+  def testSaveRestoreRepeatedlyCreateCustomSaveable(self):
+    input_size = 3
+    num_layers = 2
+    num_units = 7
+    with ops.Graph().as_default():
+      random_seed.set_random_seed(1234)
+      model = CudnnTestModel(
+          CUDNN_LSTM,
+          num_layers,
+          num_units,
+          input_size,
+          direction=CUDNN_RNN_UNIDIRECTION,
+          dtype=dtypes.float32)
+      with self.assertRaisesRegexp(RuntimeError,
+                                   "Cudnn saveable already created"):
+        model.rnn._create_saveable()
+
+  @unittest.skipUnless(test.is_built_with_cuda(),
+                       "Test only applicable when running on GPUs")
+  def testSaveRestoreLSTM(self):
+    self._TestSaveRestoreHelper(CUDNN_LSTM)
+
+  @unittest.skipUnless(test.is_built_with_cuda(),
+                       "Test only applicable when running on GPUs")
+  def testSaveRestoreGRU(self):
+    self._TestSaveRestoreHelper(CUDNN_GRU)
+
+  @unittest.skipUnless(test.is_built_with_cuda(),
+                       "Test only applicable when running on GPUs")
+  def testSaveRestoreRNNTanh(self):
+    self._TestSaveRestoreHelper(CUDNN_RNN_TANH)
+
+  @unittest.skipUnless(test.is_built_with_cuda(),
+                       "Test only applicable when running on GPUs")
+  def testSaveRestoreRNNRelu(self):
+    self._TestSaveRestoreHelper(CUDNN_RNN_RELU)
+
+
+# TODO(jamesqin): Transform to parameterized test after it is included in the
+# TF open source codebase.
+class CudnnRNNTestCompatibleRNNCells(TensorFlowTestCase):
+
+  @unittest.skipUnless(test.is_built_with_cuda(),
+                       "Test only applicable when running on GPUs")
+  def testCudnnCompatibleLSTM(self):
+    self._TestCudnnCompatibleRnnCellsHelper(CUDNN_LSTM)
+
+  @unittest.skipUnless(test.is_built_with_cuda(),
+                       "Test only applicable when running on GPUs")
+  def testCudnnCompatibleGRU(self):
+    self._TestCudnnCompatibleRnnCellsHelper(CUDNN_GRU)
+
+  @unittest.skipUnless(test.is_built_with_cuda(),
+                       "Test only applicable when running on GPUs")
+  def testCudnnCompatibleRNNTanh(self):
+    self._TestCudnnCompatibleRnnCellsHelper(CUDNN_RNN_TANH)
+
+  @unittest.skipUnless(test.is_built_with_cuda(),
+                       "Test only applicable when running on GPUs")
+  def testCudnnCompatibleRNNRelu(self):
+    self._TestCudnnCompatibleRnnCellsHelper(CUDNN_RNN_RELU)
+
+  def _TestCudnnCompatibleRnnCellsHelper(self, rnn_mode):
+    configs = [
+        {
+            "num_layers": 1,
+            "seq_length": 3,
+            "num_units": 4,
+            "input_size": 5,
+            "batch_size": 6,
+        },
+        {
+            "num_layers": 2,
+            "seq_length": 8,
+            "num_units": 4,
+            "input_size": 8,
+            "batch_size": 16,
+        },
+        {
+            "num_layers": 2,
+            "seq_length": 3,
+            "num_units": 4,
+            "input_size": 5,
+            "batch_size": 6,
+        },
+        {
+            "num_layers": 1,
+            "seq_length": 2,
+            "num_units": 2,
+            "input_size": 4,
+            "batch_size": 1,
+        },
+    ]
+    directions = [CUDNN_RNN_UNIDIRECTION, CUDNN_RNN_BIDIRECTION]
+    for cfg, direction in zip(configs, directions):
+      self._TestCudnnCompatibleRnnCells(cfg["num_layers"], cfg["seq_length"],
+                                        cfg["num_units"], cfg["input_size"],
+                                        cfg["batch_size"], rnn_mode, direction)
+
+  def _TestCudnnCompatibleRnnCells(self, num_layers, seq_length, num_units,
+                                   input_size, batch_size, rnn_mode, direction):
+    dtype = dtypes.float32
+    # Train graph
+    with ops.Graph().as_default() as g:
+      model = CudnnTestModel(
+          rnn_mode,
+          num_layers,
+          num_units,
+          input_size,
+          direction=direction,
+          dtype=dtype,
+          training=True)
+      target_output = array_ops.placeholder(dtype=dtype)
+      loss_op = losses.log_loss(
+          labels=target_output, predictions=model.total_sum)
+      optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=1e-2)
+      train_op = optimizer.minimize(loss_op)
+
+      saver = saver_lib.Saver()
+
+      # Train Cudnn model
+      seed = 0
+      with self.test_session(use_gpu=True, graph=g) as sess:
+        sess.run(variables.global_variables_initializer())
+        # Train 128 steps
+        num_steps = 128
+        for _ in range(num_steps):
+          inputs, _ = model.SynthesizeInput(seq_length, batch_size, seed)
+          targets = np.random.rand()
+          sess.run(
+              train_op,
+              feed_dict={
+                  model.inputs: inputs,
+                  model.initial_state: model.ZeroState(batch_size),
+                  target_output: targets
+              })
+          seed += 1
+
+        save_path = os.path.join(self.get_temp_dir(),
+                                 ("cudnn-rnn-%s-test" % rnn_mode))
+        save_v = saver.save(sess, save_path)
+        self.assertEqual(save_path, save_v)
+
+    # Cudnn inference graph
+    with ops.Graph().as_default() as g:
+      model = CudnnTestModel(
+          rnn_mode,
+          num_layers,
+          num_units,
+          input_size,
+          direction=direction,
+          dtype=dtype,
+          training=False)
+      rnn = model.rnn
+      saver = saver_lib.Saver()
+
+      inference_input = np.random.rand(seq_length, batch_size,
+                                       input_size).astype(np.float32)
+      with self.test_session(use_gpu=True, graph=g) as sess:
+        sess.run(variables.global_variables_initializer())
+        saver.restore(sess, save_path)
+
+        # Cudnn inference
+        cudnn_outputs_v, cudnn_output_states_v = model.Feed(
+            sess, inference_input, return_sum=False)
+
+    # Canonical RNN inference graph
+    with ops.Graph().as_default() as g:
+      cell_inputs = array_ops.placeholder(
+          dtype, shape=[seq_length, batch_size, input_size])
+      if direction == CUDNN_RNN_UNIDIRECTION:
+        # outputs is one tensor, states are num_layer tuples, each 2 tensors
+        (outputs, states) = _CreateCudnnCompatibleCanonicalRNN(rnn, cell_inputs)
+        if rnn_mode == CUDNN_LSTM:
+          output_h = array_ops.stack([s.h for s in states])
+          output_c = array_ops.stack([s.c for s in states])
+        else:
+          output_state = array_ops.stack([s for s in states])
+      else:
+        # outputs is one tensor.
+        # states is a tuple of 2 tuples:
+        # each sub tuple is num_layer tuples, each with 2 tensors.
+        (outputs, states) = _CreateCudnnCompatibleCanonicalRNN(
+            rnn, cell_inputs, is_bidi=True)
+        output_state_fw, output_state_bw = states
+        if rnn_mode == CUDNN_LSTM:
+          output_h, output_c = [], []
+          for s_fw, s_bw in zip(output_state_fw, output_state_bw):
+            output_h.append(array_ops.stack([s_fw.h, s_bw.h]))
+            output_c.append(array_ops.stack([s_fw.c, s_bw.c]))
+          output_h = array_ops.concat(output_h, axis=0)
+          output_c = array_ops.concat(output_c, axis=0)
+        else:
+          output_state = []
+          for s_fw, s_bw in zip(output_state_fw, output_state_bw):
+            output_state.append(array_ops.stack([s_fw, s_bw]))
+          output_state = array_ops.concat(output_state, axis=0)
+      saver = saver_lib.Saver()
+
+      with self.test_session(use_gpu=True, graph=g) as sess:
+        saver.restore(sess, save_path)
+
+        # BlockCell inference
+        if rnn_mode == CUDNN_LSTM:
+          outputs_v, output_h_v, output_c_v = sess.run(
+              [outputs, output_h, output_c],
+              feed_dict={cell_inputs: inference_input})
+          self.assertAllClose(cudnn_outputs_v, outputs_v)
+          cudnn_output_h_v, cudnn_output_c_v = cudnn_output_states_v
+          self.assertAllClose(cudnn_output_h_v, output_h_v)
+          self.assertAllClose(cudnn_output_c_v, output_c_v)
+        else:
+          outputs_v, output_state_v = sess.run(
+              [outputs, output_state],
+              feed_dict={cell_inputs: inference_input})
+          self.assertAllClose(cudnn_outputs_v, outputs_v, atol=1e-5, rtol=1e-5)
+          (cudnn_output_h_v,) = cudnn_output_states_v
+          self.assertAllClose(cudnn_output_h_v, output_state_v, atol=1e-5,
+                              rtol=1e-5)
+
+
+class CudnnRNNTestParamsSize(TensorFlowTestCase):
+
+  def _TestOpaqueParamsSize(self, rnn_mode, num_layers, num_units, input_size,
+                            direction):
+    logging.info("Testing one lstm param size with config: %s", locals())
+    dtype = dtypes.float32
+
+    model = CudnnTestModel(
+        rnn_mode,
+        num_layers,
+        num_units,
+        input_size,
+        dtype=dtype,
+        direction=direction)
+    rnn = model.rnn
+
+    # Min param size estimate = sum(weights.size) + sum(biases.size)
+    min_params_size = (
+        np.sum(map(np.prod, rnn.canonical_weight_shapes)) +
+        np.sum([sp[0] for sp in rnn.canonical_bias_shapes]))
+
+    opaque_params = rnn.trainable_variables[0]
+    with self.test_session(use_gpu=True, graph=ops.get_default_graph()):
+      variables.global_variables_initializer().run()
+      opaque_params_size_v = opaque_params.eval().size
+      self.assertLessEqual(min_params_size, opaque_params_size_v)
+
+  @unittest.skipUnless(test.is_built_with_cuda(),
+                       "Test only applicable when running on GPUs")
+  def testOpaqueParamsSize(self):
+    test_configs = [
+        [4, 200, 200],
+        [4, 200, 300],
+        [4, 200, 100],
+        [1, 100, 200],
+        [2, 200, 100],
+        [3, 200, 400],
+    ]
+    directions = [CUDNN_RNN_UNIDIRECTION, CUDNN_RNN_BIDIRECTION]
+    rnns = [CUDNN_LSTM, CUDNN_GRU, CUDNN_RNN_RELU, CUDNN_RNN_TANH]
+    for (rnn, config, direction) in itertools.product(rnns, test_configs,
+                                                      directions):
+      num_layers, num_units, input_size = config
+      with ops.Graph().as_default():
+        self._TestOpaqueParamsSize(rnn, num_layers, num_units, input_size,
+                                   direction)
+
+
+class CudnnRNNTestTraining(TensorFlowTestCase):
+
+  def _ComputeNumericGrad(self, sess, y, x, delta=1e-4, step=1):
+    """Compute the numeric gradient of y wrt to x.
+
+    Args:
+      sess: The TF session constructed with a graph containing x and y.
+      y: A scalar TF Tensor in the graph constructed in sess.
+      x: A TF Tensor in the graph constructed in sess.
+      delta: Gradient checker's small perturbation of x[i].
+      step: Only compute numerical gradients for a subset of x values.
+        I.e. dy/dx[i] is computed if i % step == 0.
+    Returns:
+      A Tensor of the same shape and dtype as x. If x[i] is not chosen
+      to compute the numerical gradient dy/x[i], the corresponding
+      value is set to 0.
+    """
+
+    x_data = sess.run(x)
+    x_size = x_data.size
+    x_shape = x_data.shape
+
+    numeric_grad = np.zeros(x_size, dtype=x_data.dtype)
+
+    for i in range(0, x_size, step):
+      x_pos = x_data.copy()
+      if x_size == 1:
+        x_pos += delta
+      else:
+        x_pos.flat[i] += delta
+      y_pos_feed_dict = dict([(x.name, x_pos)])
+      y_pos = sess.run(y, feed_dict=y_pos_feed_dict)
+
+      x_neg = x_data.copy()
+      if x_size == 1:
+        x_neg -= delta
+      else:
+        x_neg.flat[i] -= delta
+      y_neg_feed_dict = dict([(x.name, x_neg)])
+      y_neg = sess.run(y, feed_dict=y_neg_feed_dict)
+      numeric_grad[i] = (y_pos - y_neg) / (2 * delta)
+    return numeric_grad.reshape(x_shape)
+
+  def _GradientCheck(self, sess, y, xs, tolerance=1e-6, delta=1e-4):
+    sym_grads_t = gradients.gradients(y, xs)
+    sym_grads = sess.run(sym_grads_t)
+
+    num_grads = [self._ComputeNumericGrad(sess, y, x, delta) for x in xs]
+    self.assertEqual(len(sym_grads), len(num_grads))
+    for sym, num in zip(sym_grads, num_grads):
+      self.assertFalse(np.any(np.isnan(sym)))
+      self.assertFalse(np.any(np.isnan(num)))
+      self.assertAllClose(sym, num, atol=tolerance, rtol=tolerance)
+
+  def _TestOneSimpleTraining(self, rnn_mode, num_layers, num_units, input_size,
+                             batch_size, seq_length, dir_count, dropout, dtype,
+                             delta, tolerance):
+    # Gradient checking runs two forward ops with almost the same input. Need to
+    # make sure the drop patterns across the two runs are the same.
+    logging.info("Training test with config: %s", locals())
+    old_env_state = os.environ.get("TF_CUDNN_RESET_RND_GEN_STATE", str(False))
+    os.environ["TF_CUDNN_RESET_RND_GEN_STATE"] = str(True)
+    random_seed.set_random_seed(5678)
+    has_input_c = (rnn_mode == CUDNN_LSTM)
+    direction = (CUDNN_RNN_UNIDIRECTION
+                 if dir_count == 1 else CUDNN_RNN_BIDIRECTION)
+    model = CudnnTestModel(
+        rnn_mode,
+        num_layers,
+        num_units,
+        input_size,
+        direction=direction,
+        dropout=dropout,
+        dtype=dtype,
+        training=True,
+        bias_initializer=init_ops.random_normal_initializer(
+            mean=1., dtype=dtype))
+    rnn = model.rnn
+    params = rnn.trainable_variables[0]
+
+    inputs = variables.Variable(
+        random_ops.random_uniform(
+            [seq_length, batch_size, input_size], dtype=dtype),
+        dtype=dtype)
+    input_h = variables.Variable(
+        random_ops.random_uniform(
+            [num_layers * dir_count, batch_size, num_units], dtype=dtype),
+        dtype=dtype)
+    if has_input_c:
+      input_c = variables.Variable(
+          random_ops.random_uniform(
+              [num_layers * dir_count, batch_size, num_units], dtype=dtype),
+          dtype=dtype)
+      initial_state = (input_h, input_c)
+    else:
+      initial_state = (input_h,)
+    total_sum = model.FProp(inputs, initial_state, training=True)
+
+    with self.test_session(use_gpu=True, graph=ops.get_default_graph()) as sess:
+      sess.run(variables.global_variables_initializer())
+      all_inputs = [inputs, params]
+      for s in initial_state:
+        all_inputs.append(s)
+      self._GradientCheck(
+          sess, total_sum, all_inputs, tolerance=tolerance, delta=delta)
+      os.environ["TF_CUDNN_RESET_RND_GEN_STATE"] = old_env_state
+
+  def _TestSimpleTrainingHelper(self, rnn_mode, test_configs):
+    dropouts = [0., 0.5, 1.]
+    for config, dropout in itertools.product(test_configs, dropouts):
+      dtype = config.get("dtype", dtypes.float32)
+      delta = config.get("delta", 1e-4)
+      tolerance = config.get("tolerance", 1e-6)
+      dir_count = config.get("dir_count", 1)
+      shape = config["shape"]
+      with ops.Graph().as_default():
+        self._TestOneSimpleTraining(rnn_mode, shape["num_layers"],
+                                    shape["num_units"], shape["input_size"],
+                                    shape["batch_size"], shape["seq_length"],
+                                    dir_count, dropout, dtype, delta, tolerance)
+
+  @unittest.skipUnless(test.is_built_with_cuda(),
+                       "Test only applicable when running on GPUs")
+  def testSimpleTrainingLSTM64(self):
+    test_configs = [
+        {
+            "dtype": dtypes.float64,
+            "tolerance": 5e-6,
+            "shape": {
+                "num_layers": 2,
+                "num_units": 3,
+                "input_size": 4,
+                "batch_size": 3,
+                "seq_length": 4,
+            },
+        },
+    ]
+    self._TestSimpleTrainingHelper(CUDNN_LSTM, test_configs)
+
+  @unittest.skipUnless(test.is_built_with_cuda(),
+                       "Test only applicable when running on GPUs")
+  def testSimpleTrainingLSTM32(self):
+    test_configs = [
+        {
+            "dtype": dtypes.float32,
+            "delta": 1e-4,
+            "tolerance": 9e-2,
+            "shape": {
+                "num_layers": 2,
+                "num_units": 3,
+                "input_size": 4,
+                "batch_size": 3,
+                "seq_length": 4,
+            },
+        },
+    ]
+    self._TestSimpleTrainingHelper(CUDNN_LSTM, test_configs)
+
+  @unittest.skipUnless(test.is_built_with_cuda(),
+                       "Test only applicable when running on GPUs")
+  def testSimpleTrainingGRU64(self):
+    test_configs = [
+        {
+            "dtype": dtypes.float64,
+            "tolerance": 5e-6,
+            "shape": {
+                "num_layers": 2,
+                "num_units": 3,
+                "input_size": 4,
+                "batch_size": 3,
+                "seq_length": 4,
+            }
+        },
+    ]
+    self._TestSimpleTrainingHelper(CUDNN_GRU, test_configs)
+
+  @unittest.skipUnless(test.is_built_with_cuda(),
+                       "Test only applicable when running on GPUs")
+  def testSimpleTrainingGRU32(self):
+    test_configs = [
+        {
+            "dtype": dtypes.float32,
+            "delta": 1e-3,
+            "tolerance": 4e-3,
+            "shape": {
+                "num_layers": 2,
+                "num_units": 3,
+                "input_size": 4,
+                "batch_size": 3,
+                "seq_length": 4,
+            },
+        },
+    ]
+    self._TestSimpleTrainingHelper(CUDNN_GRU, test_configs)
+
+  @unittest.skipUnless(test.is_built_with_cuda(),
+                       "Test only applicable when running on GPUs")
+  def testSimpleTrainingRNNTanh64(self):
+    test_configs = [
+        {
+            "dtype": dtypes.float64,
+            "tolerance": 5e-6,
+            "shape": {
+                "num_layers": 2,
+                "num_units": 3,
+                "input_size": 4,
+                "batch_size": 3,
+                "seq_length": 4,
+            },
+        },
+    ]
+    self._TestSimpleTrainingHelper(CUDNN_RNN_TANH, test_configs)
+
+  @unittest.skipUnless(test.is_built_with_cuda(),
+                       "Test only applicable when running on GPUs")
+  def testSimpleTrainingRNNTanh32(self):
+    test_configs = [
+        {
+            "dtype": dtypes.float32,
+            "delta": 1e-3,
+            "tolerance": 5e-3,
+            "shape": {
+                "num_layers": 2,
+                "num_units": 3,
+                "input_size": 4,
+                "batch_size": 3,
+                "seq_length": 4,
+            },
+        },
+    ]
+    self._TestSimpleTrainingHelper(CUDNN_RNN_TANH, test_configs)
+
+  @unittest.skipUnless(test.is_built_with_cuda(),
+                       "Test only applicable when running on GPUs")
+  def testSimpleTrainingRNNRelu64(self):
+    test_configs = [
+        {
+            "dtype": dtypes.float64,
+            "tolerance": 5e-6,
+            "shape": {
+                "num_layers": 2,
+                "num_units": 3,
+                "input_size": 4,
+                "batch_size": 3,
+                "seq_length": 4,
+            },
+        },
+    ]
+    self._TestSimpleTrainingHelper(CUDNN_RNN_RELU, test_configs)
+
+  @unittest.skipUnless(test.is_built_with_cuda(),
+                       "Test only applicable when running on GPUs")
+  def testSimpleTrainingRNNRelu32(self):
+    test_configs = [
+        {
+            "dtype": dtypes.float32,
+            "delta": 1e-3,
+            "tolerance": 7e-2,
+            "shape": {
+                "num_layers": 2,
+                "num_units": 3,
+                "input_size": 4,
+                "batch_size": 3,
+                "seq_length": 4,
+            },
+        },
+    ]
+    self._TestSimpleTrainingHelper(CUDNN_RNN_RELU, test_configs)
+
+
+if __name__ == "__main__":
+  googletest.main()
diff --git a/tensorflow/contrib/cudnn_rnn/python/layers/cudnn_rnn.py b/tensorflow/contrib/cudnn_rnn/python/layers/cudnn_rnn.py
new file mode 100644
index 0000000000..810fb6450c
--- /dev/null
+++ b/tensorflow/contrib/cudnn_rnn/python/layers/cudnn_rnn.py
@@ -0,0 +1,552 @@
+# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Cudnn RNN operators."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.contrib.cudnn_rnn.python.ops import cudnn_rnn_ops
+from tensorflow.contrib.util import loader
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import tensor_shape
+from tensorflow.python.layers import base as base_layer
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import init_ops
+from tensorflow.python.ops import variable_scope as vs
+from tensorflow.python.platform import resource_loader
+from tensorflow.python.platform import tf_logging as logging
+
+_cudnn_rnn_ops_so = loader.load_op_library(
+    resource_loader.get_path_to_datafile("_cudnn_rnn_ops.so"))
+
+CUDNN_RNN_UNIDIRECTION = cudnn_rnn_ops.CUDNN_RNN_UNIDIRECTION
+CUDNN_RNN_BIDIRECTION = cudnn_rnn_ops.CUDNN_RNN_BIDIRECTION
+CUDNN_LSTM = cudnn_rnn_ops.CUDNN_LSTM
+CUDNN_GRU = cudnn_rnn_ops.CUDNN_GRU
+CUDNN_RNN_RELU = cudnn_rnn_ops.CUDNN_RNN_RELU
+CUDNN_RNN_TANH = cudnn_rnn_ops.CUDNN_RNN_TANH
+
+# Half for cell input, half for hidden states.
+CUDNN_LSTM_PARAMS_PER_LAYER = cudnn_rnn_ops.CUDNN_LSTM_PARAMS_PER_LAYER
+CUDNN_GRU_PARAMS_PER_LAYER = cudnn_rnn_ops.CUDNN_GRU_PARAMS_PER_LAYER
+CUDNN_RNN_TANH_PARAMS_PER_LAYER = cudnn_rnn_ops.CUDNN_RNN_TANH_PARAMS_PER_LAYER
+CUDNN_RNN_RELU_PARAMS_PER_LAYER = cudnn_rnn_ops.CUDNN_RNN_RELU_PARAMS_PER_LAYER
+
+CUDNN_INPUT_LINEAR_MODE = cudnn_rnn_ops.CUDNN_INPUT_LINEAR_MODE
+CUDNN_INPUT_SKIP_MODE = cudnn_rnn_ops.CUDNN_INPUT_SKIP_MODE
+CUDNN_INPUT_AUTO_MODE = cudnn_rnn_ops.CUDNN_INPUT_AUTO_MODE
+
+
+class _CudnnRNN(base_layer.Layer):
+  # pylint:disable=line-too-long
+  """Abstract class for RNN layers with Cudnn implementation.
+
+  Cudnn RNNs have two major differences from other platform-independent RNNs tf
+  provides:
+  * Cudnn LSTM and GRU are mathematically different from their tf counterparts.
+    (e.g. @{tf.contrib.rnn.LSTMBlockCell} and @{tf.nn.rnn_cell.GRUCell}.
+  * Cudnn-trained checkpoints are not directly compatible with tf RNNs:
+    * They use a single opaque parameter buffer for the entire (possibly)
+      multi-layer multi-directional RNN; Whereas tf RNN weights are per-cell and
+      layer.
+    * The size and layout of the parameter buffers may change between
+      CUDA/CuDNN/GPU generations. Because of that, the opaque parameter variable
+      does not have a static shape and is not partitionable. Instead of using
+      partitioning to alleviate the PS's traffic load, try building a
+      multi-tower model and do gradient aggregation locally within the host
+      before updating the PS. See https://www.tensorflow.org/performance/performance_models#parameter_server_variables
+      for a detailed performance guide.
+
+  Consequently, if one plans to use Cudnn trained models on both GPU and CPU
+  for inference and training, one needs to:
+  * Create a CudnnOpaqueParamsSaveable subclass object to save RNN params in
+    canonical format. (This is done for you automatically during layer building
+    process.)
+  * When not using a Cudnn RNN class, use CudnnCompatibleRNN classes to load the
+    checkpoints. These classes are platform-independent and perform the same
+    computation as Cudnn for training and inference.
+  Similarly, CudnnCompatibleRNN-trained checkpoints can be loaded by CudnnRNN
+  classes seamlessly.
+
+  Below is a typical workflow(using LSTM as an example):
+  for detailed performance guide.
+
+  # Use Cudnn-trained checkpoints with CudnnCompatibleRNNs
+  ```python
+  with tf.Graph().as_default():
+    lstm = CudnnLSTM(num_layers, num_units, direction, ...)
+
+    outputs, output_states = lstm(inputs, initial_states, training=True)
+
+    # If user plans to delay calling the cell with inputs, one can do
+    # lstm.build(input_shape)
+
+    saver = Saver()
+
+    # training subgraph
+    ...
+
+    # Once in a while save the model.
+    saver.save(save_path)
+
+  # Inference subgraph for unidirectional RNN on, e.g., CPU or mobile.
+  with tf.Graph().as_default():
+    single_cell = lambda: tf.contrib.cudnn_rnn.CudnnCompatibleLSTM(num_units)
+
+    # NOTE: Even if there's only one layer, the cell needs to be wrapped in
+    # MultiRNNCell.
+    cell = tf.nn.rnn_cell.MultiRNNCell(
+      [single_cell() for _ in range(num_layers)])
+
+    # Leave the scope arg unset.
+    outputs, final_state = tf.nn.dynamic_rnn(cell, inputs, initial_state, ...)
+
+    saver = Saver()
+
+    # Create session
+    sess = ...
+
+    # Restores
+    saver.restore(sess, save_path)
+
+  # Inference subgraph for bidirectional RNN
+  with tf.Graph().as_default():
+    single_cell = lambda: tf.contrib.cudnn_rnn.CudnnCompatibleLSTM(num_units)
+    cells_fw = [single_cell() for _ in range(num_layers)]
+    cells_bw = [single_cell() for _ in range(num_layers)]
+
+    # Leave the scope arg unset.
+    (outputs, output_state_fw,
+     output_state_bw) = tf.contrib.rnn.stack_bidirectional_dynamic_rnn(
+         cells_fw, cells_bw, inputs, ...)
+    saver = Saver()
+
+    # Create session
+    sess = ...
+
+    # Restores
+    saver.restore(sess, save_path)
+  ```
+  """
+  # pylint:enable=line-too-long
+
+  # The following are constants defined by subclasses.
+  # Type of RNN cell.
+  _rnn_mode = None
+  # Number of cell weights(or biases) per layer.
+  _num_params_per_layer = None
+  # Custom SaveableObject class for the CudnnRNN class.
+  _saveable_cls = None
+
+  # TODO(jamesqin): support float16 CuDNN RNN
+  def __init__(self,
+               num_layers,
+               num_units,
+               input_mode=CUDNN_INPUT_LINEAR_MODE,
+               direction=CUDNN_RNN_UNIDIRECTION,
+               dropout=0.,
+               seed=None,
+               dtype=dtypes.float32,
+               kernel_initializer=None,
+               bias_initializer=None,
+               name=None):
+    """Creates a CudnnRNN model from model spec.
+
+    Args:
+      num_layers: the number of layers for the RNN model.
+      num_units: the number of units within the RNN model.
+      input_mode: indicate whether there is a linear projection between the
+          input and the actual computation before the first layer. It can be
+          'linear_input', 'skip_input' or 'auto_select'.
+          'linear_input' (default) always applies a linear projection of input
+          onto RNN hidden state. (standard RNN behavior).
+          'skip_input' is only allowed when input_size == num_units;
+          'auto_select' implies 'skip_input' when input_size == num_units;
+          otherwise, it implies 'linear_input'.
+      direction: the direction model that the model operates. Can be either
+          'unidirectional' or 'bidirectional'
+      dropout: dropout rate, a number between [0, 1]. Dropout is applied on
+          inputs of each layer. When set to 0, dropout is disabled.
+      seed: the op seed used for initializing dropout. See @{tf.set_random_seed}
+          for behavior.
+      dtype: tf.float32 or tf.float64
+      kernel_initializer: starting value to initialize the weight.
+      bias_initializer: starting value to initialize the bias
+        (default is all zeros).
+      name: VariableScope for the created subgraph; defaults to class name.
+        This only serves the default scope if later no scope is specified when
+        invoking __call__().
+
+    Raises:
+      ValueError: if direction is invalid.
+    """
+    super(_CudnnRNN, self).__init__(dtype=dtype, name=name)
+    cudnn_rnn_ops.check_direction(direction)
+    cudnn_rnn_ops.check_input_mode(input_mode)
+
+    self._num_layers = num_layers
+    self._num_units = num_units
+    self._input_mode = input_mode
+    self._direction = direction
+    self._dropout = dropout
+    self._seed = seed
+    self._kernel_initializer = kernel_initializer
+    self._bias_initializer = bias_initializer
+    # Init input_size to None, which will be set after build().
+    self._input_size = None
+    self._saveable = None
+
+  @property
+  def num_layers(self):
+    return self._num_layers
+
+  @property
+  def num_units(self):
+    return self._num_units
+
+  @property
+  def input_mode(self):
+    """Input mode of first layer.
+
+    Indicates whether there is a linear projection between the input and the
+    actual computation before the first layer. It can be
+    * 'linear_input': (default) always applies a linear projection of input
+      onto RNN hidden state. (standard RNN behavior)
+    * 'skip_input': 'skip_input' is only allowed when input_size == num_units.
+    * 'auto_select'. implies 'skip_input' when input_size == num_units;
+      otherwise, it implies 'linear_input'.
+
+    Returns:
+      'linear_input', 'skip_input' or 'auto_select'.
+    """
+    return self._input_mode
+
+  @property
+  def input_size(self):
+    if not self._input_size:
+      raise ValueError(
+          "\'input_size\' is unknown since layer has not been built.")
+    return self._input_size
+
+  @property
+  def rnn_mode(self):
+    """Type of RNN cell used.
+
+    Returns:
+      `lstm`, `gru`, `rnn_relu` or `rnn_tanh`.
+    """
+    return self._rnn_mode
+
+  @property
+  def direction(self):
+    """Returns `unidirectional` or `bidirectional`."""
+    return self._direction
+
+  @property
+  def num_dirs(self):
+    return 1 if self._direction == CUDNN_RNN_UNIDIRECTION else 2
+
+  @property
+  def saveable(self):
+    return self._saveable
+
+  @property
+  def canonical_weight_shapes(self):
+    """Shapes of Cudnn canonical weight tensors."""
+    if not self._input_size:
+      raise RuntimeError(
+          "%s.canonical_weight_shapes invoked before input shape is known" %
+          type(self).__name__)
+
+    shapes = []
+    for i in range(self._num_layers):
+      shapes.extend(self._canonical_weight_shape(i))
+    return shapes
+
+  @property
+  def canonical_bias_shapes(self):
+    """Shapes of Cudnn canonical bias tensors."""
+    return self._canonical_bias_shape(0) * self._num_layers
+
+  def _update_trainable_weights(self, getter, *args, **kwargs):
+    """Custom getter for layer variables."""
+    # Add variables to layer's `(non_)trainable_weights` list(s).
+    variable = getter(*args, **kwargs)
+    trainable = kwargs.get("trainable", True)
+    if trainable and variable not in self._trainable_weights:
+      self._trainable_weights.append(variable)
+    elif not trainable and variable not in self._non_trainable_weights:
+      self._non_trainable_weights.append(variable)
+    return variable
+
+  def build(self, input_shape):
+    """Create variables of the Cudnn RNN.
+
+    It can be called manually before `__call__()` or automatically through
+    `__call__()`. In the former case, subsequent `__call__()`s will skip
+    creating variables.
+    Args:
+      input_shape: network input tensor shape, a python list or a TensorShape
+        object with 3 dimensions.
+    Raises:
+      ValueError: if input_shape has wrong dimension or unknown 3rd dimension.
+    """
+    if self.built:
+      return
+
+    input_shape = tensor_shape.TensorShape(input_shape)
+    if input_shape.ndims != 3:
+      raise ValueError("Expecting input_shape with 3 dims, got %d" %
+                       input_shape.ndims)
+    if input_shape[-1].value is None:
+      raise ValueError("The last dimension of the inputs to `CudnnRNN` "
+                       "should be defined. Found `None`.")
+    self._input_size = input_shape[-1].value
+    self.input_spec = base_layer.InputSpec(ndim=3, axes={-1: self._input_size})
+
+    self._set_scope(None)
+
+    # Not using base class `add_variable()` since the it calls
+    # `tf.get_variable()` with a callable initializer whereas here with a
+    # tensor. The difference is mandated to support forward-compatibility with
+    # Cudnn.
+    with vs.variable_scope(
+        self._scope,
+        reuse=self.built,
+        custom_getter=self._update_trainable_weights):
+      if self._kernel_initializer is None:
+        self._kernel_initializer = init_ops.glorot_uniform_initializer(
+            seed=self._seed, dtype=self.dtype)
+      if self._bias_initializer is None:
+        self._bias_initializer = init_ops.constant_initializer(
+            0.0, dtype=self.dtype)
+
+      weights = [
+          self._kernel_initializer(sp, dtype=self.dtype)
+          for sp in self.canonical_weight_shapes
+      ]
+      biases = [
+          self._bias_initializer(sp, dtype=self.dtype)
+          for sp in self.canonical_bias_shapes
+      ]
+      opaque_params_t = self._canonical_to_opaque(weights, biases)
+
+      if vs.get_variable_scope().partitioner is not None:
+        logging.warn(
+            "Partitioner is not supported for Cudnn RNN layer variables, using "
+            "it will create forward-compatibility issues with future "
+            "CUDA/CuDNN generations.")
+      # Initialize opaque params with a tensor.
+      self.kernel = vs.get_variable(
+          "opaque_kernel", initializer=opaque_params_t, validate_shape=False)
+    # Create saveable in the outer scope of the cudnn subgraph, such that
+    # alternative subgraph with platform-independent rnn cells can load the
+    # checkpoints directly.
+    if not (self.built or vs.get_variable_scope().reuse):
+      self._create_saveable()
+    self.built = True
+
+  def call(self, inputs, initial_state=None, training=True):
+    """Runs the forward step for the RNN model.
+
+    Args:
+      inputs: `3-D` tensor with shape `[time_len, batch_size, input_size]`.
+      initial_state: a tuple of tensor(s) of shape
+        `[num_layers * num_dirs, batch_size, num_units]`. If not provided, use
+        zero initial states. The tuple size is 2 for LSTM and 1 for other RNNs.
+      training: whether this operation will be used in training or inference.
+    Returns:
+      output: a tensor of shape `[time_len, batch_size, num_dirs * num_units]`.
+        It is a `concat([fwd_output, bak_output], axis=2)`.
+      output_states: a tuple of tensor(s) of the same shape and structure as
+        `initial_state`.
+    Raises:
+      ValueError: initial_state is not a tuple.
+    """
+    if initial_state is not None and not isinstance(initial_state, tuple):
+      raise ValueError("Invalid initial_state type: %s, expecting tuple.",
+                       type(initial_state))
+    dtype = self.dtype
+    inputs = ops.convert_to_tensor(inputs, dtype=dtype)
+
+    batch_size = array_ops.shape(inputs)[1]
+    if initial_state is None:
+      initial_state = self._zero_state(batch_size)
+    if self._rnn_mode == CUDNN_LSTM:
+      h, c = initial_state  # pylint:disable=unbalanced-tuple-unpacking,unpacking-non-sequence
+    else:
+      h, = initial_state  # pylint:disable=unbalanced-tuple-unpacking,unpacking-non-sequence
+    h = ops.convert_to_tensor(h, dtype=dtype)
+    if self._rnn_mode == CUDNN_LSTM:
+      c = ops.convert_to_tensor(c, dtype=dtype)
+    else:
+      # For model that doesn't take input_c, replace with a dummy tensor.
+      c = array_ops.constant([], dtype=dtype)
+    outputs, (output_h, output_c) = self._forward(inputs, h, c, self.kernel,
+                                                  training)
+    if self._rnn_mode == CUDNN_LSTM:
+      return outputs, (output_h, output_c)
+    else:
+      return outputs, (output_h,)
+
+  def state_shape(self, batch_size):
+    raise NotImplementedError
+
+  def _zero_state(self, batch_size):
+    res = []
+    for sp in self.state_shape(batch_size):
+      res.append(array_ops.zeros(sp, dtype=self.dtype))
+    return tuple(res)
+
+  def _canonical_weight_shape(self, layer):
+    """Shapes of Cudnn canonical weight tensors for given layer."""
+    if layer < 0 or layer >= self._num_layers:
+      raise ValueError("\'layer\' is not valid, got %s, expecting [%d, %d]" %
+                       (layer, 0, self._num_layers-1))
+    if not self._input_size:
+      raise RuntimeError(
+          "%s._canonical_weight_shape invoked before input shape is known" %
+          type(self).__name__)
+
+    input_size = self._input_size
+    num_units = self._num_units
+    num_gates = self._num_params_per_layer // 2
+    is_bidi = self._direction == CUDNN_RNN_BIDIRECTION
+
+    if layer == 0:
+      wts_applied_on_inputs = [(num_units, input_size)] * num_gates
+    else:
+      if is_bidi:
+        wts_applied_on_inputs = [(num_units, 2 * num_units)] * num_gates
+      else:
+        wts_applied_on_inputs = [(num_units, num_units)] * num_gates
+    wts_applied_on_hidden_states = [(num_units, num_units)] * num_gates
+    tf_wts = wts_applied_on_inputs + wts_applied_on_hidden_states
+    return tf_wts if not is_bidi else tf_wts * 2
+
+  def _canonical_bias_shape(self, unused_layer):
+    """Shapes of Cudnn canonical bias tensors for given layer."""
+    num_dirs = 1 if self._direction == CUDNN_RNN_UNIDIRECTION else 2
+    return [[self._num_units]] * num_dirs * self._num_params_per_layer
+
+  def _canonical_to_opaque(self, cu_weights, cu_biases):
+    if not self._input_size:
+      raise RuntimeError(
+          "%s._canonical_to_opaque invoked before input shape is known" %
+          type(self).__name__)
+    return cudnn_rnn_ops.cudnn_rnn_canonical_to_opaque_params(
+        rnn_mode=self._rnn_mode,
+        num_layers=self._num_layers,
+        num_units=self._num_units,
+        input_size=self._input_size,
+        weights=cu_weights,
+        biases=cu_biases,
+        input_mode=self._input_mode,
+        direction=self._direction)
+
+  def _forward(self, inputs, h, c, opaque_params, training):
+    output, output_h, output_c = cudnn_rnn_ops._cudnn_rnn(  # pylint:disable=protected-access
+        inputs,
+        h,
+        c,
+        opaque_params,
+        training,
+        self._rnn_mode,
+        input_mode=self._input_mode,
+        direction=self._direction,
+        dropout=self._dropout,
+        seed=self._seed)
+    return output, (output_h, output_c)
+
+  def _create_saveable(self):
+    """Create custom saveable for the Cudnn layer.
+
+    Called during layer building process to make sharing checkpoints between
+    Cudnn and Cudnn-compatible RNNs easy.
+    Returns:
+      a `CudnnOpaqueParamsSaveable` object.
+    Raises:
+      RuntimeError: if any custom saveable is already created for this layer.
+    """
+    if self._saveable is not None:
+      raise RuntimeError("Cudnn saveable already created.")
+    self._saveable = self._saveable_cls(  # pylint:disable=not-callable
+        self.trainable_variables[0],
+        self.num_layers,
+        self.num_units,
+        self.input_size,
+        self.input_mode,
+        self.direction,
+        scope=vs.get_variable_scope(),
+        name="%s_saveable" % self.trainable_variables[0].op.name)
+    ops.add_to_collection(ops.GraphKeys.SAVEABLE_OBJECTS, self._saveable)
+
+
+class CudnnLSTM(_CudnnRNN):
+  """Cudnn implementation of LSTM layer."""
+  _rnn_mode = CUDNN_LSTM
+  _num_params_per_layer = CUDNN_LSTM_PARAMS_PER_LAYER
+  _saveable_cls = cudnn_rnn_ops.CudnnLSTMSaveable
+
+  def state_shape(self, batch_size):
+    """Shape of Cudnn LSTM states.
+
+    Shape is a 2-element tuple. Each is
+    [num_layers * num_dirs, batch_size, num_units]
+    Args:
+      batch_size: an int
+    Returns:
+      a tuple of python arrays.
+    """
+    return ([self.num_layers * self.num_dirs, batch_size, self.num_units],
+            [self.num_layers * self.num_dirs, batch_size, self.num_units])
+
+
+class _CudnnRNNNoInputC(_CudnnRNN):
+  """Abstract simple CudnnRNN layer without input_c."""
+
+  def state_shape(self, batch_size):
+    """Shape of the state of Cudnn RNN cells w/o. input_c.
+
+    Shape is a 1-element tuple,
+    [num_layers * num_dirs, batch_size, num_units]
+    Args:
+      batch_size: an int
+    Returns:
+      a tuple of python arrays.
+    """
+    return [self.num_layers * self.num_dirs, batch_size, self.num_units],
+
+
+class CudnnGRU(_CudnnRNNNoInputC):
+  """Cudnn implementation of the GRU layer."""
+  _rnn_mode = CUDNN_GRU
+  _num_params_per_layer = CUDNN_GRU_PARAMS_PER_LAYER
+  _saveable_cls = cudnn_rnn_ops.CudnnGRUSaveable
+
+
+class CudnnRNNTanh(_CudnnRNNNoInputC):
+  """Cudnn implementation of the RNN-tanh layer."""
+  _rnn_mode = CUDNN_RNN_TANH
+  _num_params_per_layer = CUDNN_RNN_TANH_PARAMS_PER_LAYER
+  _saveable_cls = cudnn_rnn_ops.CudnnRNNTanhSaveable
+
+
+class CudnnRNNRelu(_CudnnRNNNoInputC):
+  """Cudnn implementation of the RNN-relu layer."""
+  _rnn_mode = CUDNN_RNN_RELU
+  _num_params_per_layer = CUDNN_RNN_RELU_PARAMS_PER_LAYER
+  _saveable_cls = cudnn_rnn_ops.CudnnRNNReluSaveable
diff --git a/tensorflow/contrib/cudnn_rnn/python/ops/cudnn_rnn_ops.py b/tensorflow/contrib/cudnn_rnn/python/ops/cudnn_rnn_ops.py
index bbf1bd9bca..7d658c746e 100644
--- a/tensorflow/contrib/cudnn_rnn/python/ops/cudnn_rnn_ops.py
+++ b/tensorflow/contrib/cudnn_rnn/python/ops/cudnn_rnn_ops.py
@@ -717,12 +717,6 @@ _cudnn_rnn_common_doc_string = """
 """
 
 
-def _check_direction(direction):
-  if direction not in (CUDNN_RNN_UNIDIRECTION, CUDNN_RNN_BIDIRECTION):
-    raise ValueError("Invalid direction: %s, expect %s or %s" %
-                     (direction, CUDNN_RNN_UNIDIRECTION, CUDNN_RNN_BIDIRECTION))
-
-
 def _check_rnn_mode(rnn_mode):
   if rnn_mode not in (CUDNN_LSTM, CUDNN_GRU, CUDNN_RNN_TANH, CUDNN_RNN_RELU):
     raise ValueError("Invalid rnn_mode: %s, expect one of (%s, %s, %s, %s)" %
@@ -737,14 +731,31 @@ def _get_seed(seed):
   return seed, seed2
 
 
+def check_direction(direction):
+  """Check validity of direction."""
+  if direction not in (CUDNN_RNN_UNIDIRECTION, CUDNN_RNN_BIDIRECTION):
+    raise ValueError("Invalid direction: %s, expecting %s or %s" %
+                     (direction, CUDNN_RNN_UNIDIRECTION, CUDNN_RNN_BIDIRECTION))
+
+
+def check_input_mode(input_mode):
+  if input_mode not in (CUDNN_INPUT_LINEAR_MODE, CUDNN_INPUT_SKIP_MODE,
+                        CUDNN_INPUT_AUTO_MODE):
+    raise ValueError("Invalid input_mode: %s, expect one of (%s, %s, %s)" %
+                     (input_mode, CUDNN_INPUT_LINEAR_MODE,
+                      CUDNN_INPUT_SKIP_MODE, CUDNN_INPUT_AUTO_MODE))
+
+
 def _get_num_params(rnn_mode, num_layers, direction):
   """Return num params for given Cudnn config."""
   if rnn_mode == CUDNN_LSTM:
-    num_params_per_layer = 8
+    num_params_per_layer = CUDNN_LSTM_PARAMS_PER_LAYER
   elif rnn_mode == CUDNN_GRU:
-    num_params_per_layer = 6
-  elif rnn_mode in (CUDNN_RNN_RELU, CUDNN_RNN_TANH):
-    num_params_per_layer = 2
+    num_params_per_layer = CUDNN_GRU_PARAMS_PER_LAYER
+  elif rnn_mode == CUDNN_RNN_RELU:
+    num_params_per_layer = CUDNN_RNN_RELU_PARAMS_PER_LAYER
+  elif rnn_mode == CUDNN_RNN_TANH:
+    num_params_per_layer = CUDNN_RNN_TANH_PARAMS_PER_LAYER
   else:
     raise ValueError("Invalid \'rnn_mode\': %s", rnn_mode)
   num_params = num_layers * num_params_per_layer
@@ -794,7 +805,8 @@ def _cudnn_rnn(inputs,
     outputs, output_h, output_c
   """
   _check_rnn_mode(rnn_mode)
-  _check_direction(direction)
+  check_direction(direction)
+  check_input_mode(input_mode)
   seed, seed2 = random_seed.get_seed(seed)
   outputs, output_h, output_c, _ = gen_cudnn_rnn_ops.cudnn_rnn(
       input=inputs,
@@ -1017,16 +1029,16 @@ def cudnn_rnn_tanh(inputs,
                                seed, name)
 
 
-def cudnn_rnn_params_to_canonical(rnn_mode,
-                                  num_layers,
-                                  num_units,
-                                  input_size,
-                                  params,
-                                  input_mode=CUDNN_INPUT_LINEAR_MODE,
-                                  direction=CUDNN_RNN_UNIDIRECTION,
-                                  dropout=0,
-                                  seed=0,
-                                  name=None):
+def cudnn_rnn_opaque_params_to_canonical(rnn_mode,
+                                         num_layers,
+                                         num_units,
+                                         input_size,
+                                         params,
+                                         input_mode=CUDNN_INPUT_LINEAR_MODE,
+                                         direction=CUDNN_RNN_UNIDIRECTION,
+                                         dropout=0,
+                                         seed=0,
+                                         name=None):
   """Convert cudnn opaque params to canonical.
 
   Args:
@@ -1058,7 +1070,8 @@ def cudnn_rnn_params_to_canonical(rnn_mode,
   """
 
   _check_rnn_mode(rnn_mode)
-  _check_direction(direction)
+  check_direction(direction)
+  check_input_mode(input_mode)
   num_params = _get_num_params(rnn_mode, num_layers, direction)
   seed, seed2 = random_seed.get_seed(seed)
   weights, biases = gen_cudnn_rnn_ops.cudnn_rnn_params_to_canonical(
@@ -1077,17 +1090,17 @@ def cudnn_rnn_params_to_canonical(rnn_mode,
   return weights, biases
 
 
-def cudnn_rnn_canonical_to_params(rnn_mode,
-                                  num_layers,
-                                  num_units,
-                                  input_size,
-                                  weights,
-                                  biases,
-                                  input_mode=CUDNN_INPUT_LINEAR_MODE,
-                                  direction=CUDNN_RNN_UNIDIRECTION,
-                                  dropout=0,
-                                  seed=0,
-                                  name=None):
+def cudnn_rnn_canonical_to_opaque_params(rnn_mode,
+                                         num_layers,
+                                         num_units,
+                                         input_size,
+                                         weights,
+                                         biases,
+                                         input_mode=CUDNN_INPUT_LINEAR_MODE,
+                                         direction=CUDNN_RNN_UNIDIRECTION,
+                                         dropout=0,
+                                         seed=0,
+                                         name=None):
   """Converts params from the canonical format to a specific format of cuDNN.
 
   Args:
@@ -1119,7 +1132,8 @@ def cudnn_rnn_canonical_to_params(rnn_mode,
     ValueError: if rnn_mode or direction is invalid.
   """
   _check_rnn_mode(rnn_mode)
-  _check_direction(direction)
+  check_direction(direction)
+  check_input_mode(input_mode)
   seed, seed2 = random_seed.get_seed(seed)
   return gen_cudnn_rnn_ops.cudnn_rnn_canonical_to_params(
       rnn_mode=rnn_mode,
@@ -1136,16 +1150,16 @@ def cudnn_rnn_canonical_to_params(rnn_mode,
       name=name)
 
 
-def cudnn_opaque_params_size(rnn_mode,
-                             num_layers,
-                             num_units,
-                             input_size,
-                             input_mode=CUDNN_INPUT_LINEAR_MODE,
-                             direction=CUDNN_RNN_UNIDIRECTION,
-                             dtype=dtypes.float32,
-                             dropout=0,
-                             seed=0,
-                             name=None):
+def cudnn_rnn_opaque_params_size(rnn_mode,
+                                 num_layers,
+                                 num_units,
+                                 input_size,
+                                 input_mode=CUDNN_INPUT_LINEAR_MODE,
+                                 direction=CUDNN_RNN_UNIDIRECTION,
+                                 dtype=dtypes.float32,
+                                 dropout=0,
+                                 seed=0,
+                                 name=None):
   """Returns opaque params size for specific Cudnn config.
 
   Args:
@@ -1176,7 +1190,8 @@ def cudnn_opaque_params_size(rnn_mode,
     ValueError: if rnn_mode or direction is invalid.
   """
   _check_rnn_mode(rnn_mode)
-  _check_direction(direction)
+  check_direction(direction)
+  check_input_mode(input_mode)
   seed, seed2 = random_seed.get_seed(seed)
   return gen_cudnn_rnn_ops.cudnn_rnn_params_size(
       rnn_mode=rnn_mode,
@@ -1278,7 +1293,7 @@ class _CudnnRNN(object):
     Returns:
       The calculated parameter buffer size.
     """
-    return cudnn_opaque_params_size(
+    return cudnn_rnn_opaque_params_size(
         rnn_mode=self._rnn_mode,
         num_layers=self._num_layers,
         num_units=self._num_units,
@@ -1327,7 +1342,7 @@ class _CudnnRNN(object):
     Returns:
       A function for the specific-to-canonical conversion.
     """
-    return cudnn_rnn_params_to_canonical(
+    return cudnn_rnn_opaque_params_to_canonical(
         rnn_mode=self._rnn_mode,
         num_layers=self._num_layers,
         num_units=self._num_units,
@@ -1348,7 +1363,7 @@ class _CudnnRNN(object):
     Returns:
       A function for the canonical-to-params-to-specific conversion..
     """
-    return cudnn_rnn_canonical_to_params(
+    return cudnn_rnn_canonical_to_opaque_params(
         rnn_mode=self._rnn_mode,
         num_layers=self._num_layers,
         num_units=self._num_units,
-- 
GitLab


From 76eb8726160a192ebe6ac5e61d0a0a539cc0dc1a Mon Sep 17 00:00:00 2001
From: Colin Raffel <craffel@gmail.com>
Date: Wed, 4 Oct 2017 18:51:57 -0700
Subject: [PATCH 033/909] Fix documentation error in tf.reverse docstring (#1)

The first example in the tf.reverse docstring causes a ValueError:

```Python
In [1]: import tensorflow as tf
In [2]: t = tf.constant([[[[ 0,  1,  2,  3], [ 4,  5,  6,  7], [ 8,  9, 10, 11]], [[12, 13, 14, 15], [16, 17, 18, 19], [20, 21, 22, 23]]]])
In [3]: dims = -1
In [4]: sess = tf.InteractiveSession()
In [5]: tf.reverse(t, dims).eval()
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-6-7752813cc8a9> in <module>()
----> 1 tf.reverse(t, dims).eval()

/Users/craffel/.pyenv/versions/2.7.13/lib/python2.7/site-packages/tensorflow/python/ops/array_ops.pyc in reverse(tensor, axis, name)
   2332
   2333 def reverse(tensor, axis, name=None):
-> 2334   return gen_array_ops.reverse_v2(tensor, axis, name)
   2335 reverse.__doc__ = gen_array_ops.reverse_v2.__doc__
   2336

/Users/craffel/.pyenv/versions/2.7.13/lib/python2.7/site-packages/tensorflow/python/ops/gen_array_ops.pyc in reverse_v2(tensor, axis, name)
   2697   """
   2698   result = _op_def_lib.apply_op("ReverseV2", tensor=tensor, axis=axis,
-> 2699                                 name=name)
   2700   return result
   2701

/Users/craffel/.pyenv/versions/2.7.13/lib/python2.7/site-packages/tensorflow/python/framework/op_def_library.pyc in apply_op(self, op_type_name, name, **keywords)
    765         op = g.create_op(op_type_name, inputs, output_types, name=scope,
    766                          input_types=input_types, attrs=attr_protos,
--> 767                          op_def=op_def)
    768         if output_structure:
    769           outputs = op.outputs

/Users/craffel/.pyenv/versions/2.7.13/lib/python2.7/site-packages/tensorflow/python/framework/ops.pyc in create_op(self, op_type, inputs, dtypes, input_types, name, attrs, op_def, compute_shapes, compute_device)
   2506                     original_op=self._default_original_op, op_def=op_def)
   2507     if compute_shapes:
-> 2508       set_shapes_for_outputs(ret)
   2509     self._add_op(ret)
   2510     self._record_op_seen_by_control_dependencies(ret)

/Users/craffel/.pyenv/versions/2.7.13/lib/python2.7/site-packages/tensorflow/python/framework/ops.pyc in set_shapes_for_outputs(op)
   1871       shape_func = _call_cpp_shape_fn_and_require_op
   1872
-> 1873   shapes = shape_func(op)
   1874   if shapes is None:
   1875     raise RuntimeError(

/Users/craffel/.pyenv/versions/2.7.13/lib/python2.7/site-packages/tensorflow/python/framework/ops.pyc in call_with_requiring(op)
   1821
   1822   def call_with_requiring(op):
-> 1823     return call_cpp_shape_fn(op, require_shape_fn=True)
   1824
   1825   _call_cpp_shape_fn_and_require_op = call_with_requiring

/Users/craffel/.pyenv/versions/2.7.13/lib/python2.7/site-packages/tensorflow/python/framework/common_shapes.pyc in call_cpp_shape_fn(op, input_tensors_needed, input_tensors_as_shapes_needed, debug_python_shape_fn, require_shape_fn)
    608     res = _call_cpp_shape_fn_impl(op, input_tensors_needed,
    609                                   input_tensors_as_shapes_needed,
--> 610                                   debug_python_shape_fn, require_shape_fn)
    611     if not isinstance(res, dict):
    612       # Handles the case where _call_cpp_shape_fn_impl calls unknown_shape(op).

/Users/craffel/.pyenv/versions/2.7.13/lib/python2.7/site-packages/tensorflow/python/framework/common_shapes.pyc in _call_cpp_shape_fn_impl(op, input_tensors_needed, input_tensors_as_shapes_needed, debug_python_shape_fn, require_shape_fn)
    674       missing_shape_fn = True
    675     else:
--> 676       raise ValueError(err.message)
    677
    678   if missing_shape_fn:

ValueError: Shape must be rank 1 but is rank 0 for 'ReverseV2' (op: 'ReverseV2') with input shapes: [1,2,3,4], [].
```
---
 tensorflow/core/ops/array_ops.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/ops/array_ops.cc b/tensorflow/core/ops/array_ops.cc
index ad111fc6b8..8397ff52aa 100644
--- a/tensorflow/core/ops/array_ops.cc
+++ b/tensorflow/core/ops/array_ops.cc
@@ -1117,7 +1117,7 @@ For example:
 #                  [20, 21, 22, 23]]]]
 # tensor 't' shape is [1, 2, 3, 4]
 
-# 'dims' is [3] or 'dims' is -1
+# 'dims' is [3] or 'dims' is [-1]
 reverse(t, dims) ==> [[[[ 3,  2,  1,  0],
                         [ 7,  6,  5,  4],
                         [ 11, 10, 9, 8]],
-- 
GitLab


From f6e187acdd9bd1d3ac2d1d08809fffb25f4bd105 Mon Sep 17 00:00:00 2001
From: Derek Murray <mrry@google.com>
Date: Wed, 4 Oct 2017 19:07:31 -0700
Subject: [PATCH 034/909] Update the release notes with information about
 tf.data.

Also adds a short porting guide to the tf.contrib.data README.

PiperOrigin-RevId: 171097798
---
 RELEASE.md                        | 15 +++++++++++++
 tensorflow/contrib/data/README.md | 37 +++++++++++++++++++++++++++----
 2 files changed, 48 insertions(+), 4 deletions(-)

diff --git a/RELEASE.md b/RELEASE.md
index 634b31b82b..c5f1e8b309 100644
--- a/RELEASE.md
+++ b/RELEASE.md
@@ -1,6 +1,16 @@
 # Release 1.4.0
 
 ## Major Features And Improvements
+* `tf.data` is now part of the core TensorFlow API.
+  * The API is now subject to backwards compatibility guarantees.
+  * For a guide to migrating from the `tf.contrib.data` API, see the
+    [README] (https://github.com/tensorflow/tensorflow/blob/r1.4/tensorflow/contrib/data/README.md).
+  * Major new features include `Dataset.from_generator()` (for building an input
+    pipeline from a Python generator), and the `Dataset.apply()` method for
+    applying custom transformation functions.
+  * Several custom transformation functions have been added, including
+    `tf.contrib.data.batch_and_drop_remainder()` and
+    `tf.contrib.data.sloppy_interleave()`.
 * Java:
   * Generics (e.g., `Tensor<Integer>`) for improved type-safety (courtesy @andrewcmyers).
   * Support for multi-dimensional string tensors.
@@ -16,6 +26,11 @@
   flexible and reproducible package, is available via the new
   `tf.contrib.data.Dataset.from_generator` method!
 
+## Breaking Changes to the API
+* The signature of the `tf.contrib.data.rejection_resample()` function has been
+  changed. It now returns a function that can be used as an argument to
+  `Dataset.apply()`.
+
 # Release 1.3.0
 
 See also [TensorBoard 0.1.4](https://github.com/tensorflow/tensorboard/releases/tag/0.1.4) release notes.
diff --git a/tensorflow/contrib/data/README.md b/tensorflow/contrib/data/README.md
index 04f0560b09..30e909111f 100644
--- a/tensorflow/contrib/data/README.md
+++ b/tensorflow/contrib/data/README.md
@@ -2,9 +2,38 @@
 =====================
 
 NOTE: The `tf.contrib.data` module has been deprecated. Use `tf.data` instead.
+We are continuing to support existing code using the `tf.contrib.data` APIs in
+the current version of TensorFlow, but will eventually remove support. The
+`tf.data` APIs are subject to backwards compatibility guarantees.
 
-This directory contains the Python API for the `tf.contrib.data.Dataset` and
-`tf.contrib.data.Iterator` classes, which can be used to build input pipelines.
+Porting your code to `tf.data`
+------------------------------
 
-The documentation for `tf.data` API has moved to the programmers'
-guide, [here](../../docs_src/programmers_guide/datasets.md).
+The `tf.contrib.data.Dataset` class has been renamed to `tf.data.Dataset`, and
+the `tf.contrib.data.Iterator` class has been renamed to `tf.data.Iterator`.
+Most code can be ported by removing `.contrib` from the names of the classes.
+However, there are some small differences, which are outlined below.
+
+The arguments accepted by the `Dataset.map()` transformation have changed:
+
+* `dataset.map(..., num_threads=T)` is now `dataset.map(num_parallel_calls=T)`.
+* `dataset.map(..., output_buffer_size=B)` is now
+  `dataset.map(...).prefetch(B).
+
+Some transformations have been removed from `tf.data.Dataset`, and you must
+instead apply them using `Dataset.apply()` transformation. The full list of
+changes is as follows:
+
+* `dataset.dense_to_sparse_batch(...)` is now
+  `dataset.apply(tf.contrib.data.dense_to_sparse_batch(...)`.
+* `dataset.enumerate(...)` is now
+  `dataset.apply(tf.contrib.data.enumerate_dataset(...))`.
+* `dataset.group_by_window(...)` is now
+  `dataset.apply(tf.contrib.data.group_by_window(...))`.
+* `dataset.ignore_errors()` is now
+  `dataset.apply(tf.contrib.data.ignore_errors())`.
+* `dataset.unbatch()` is now `dataset.apply(tf.contrib.data.unbatch())`.
+
+The `Dataset.make_dataset_resource()` and `Iterator.dispose_op()` methods have
+been removed from the API. Please open a GitHub issue if you have a need for
+either of these.
-- 
GitLab


From 73b1adc5085ee8f4a8a190287e3e4d33fe1409f8 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 4 Oct 2017 19:07:51 -0700
Subject: [PATCH 035/909] Renames variable for consistency with flag.

PiperOrigin-RevId: 171097818
---
 .../examples/speech_commands/test_streaming_accuracy.cc       | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/examples/speech_commands/test_streaming_accuracy.cc b/tensorflow/examples/speech_commands/test_streaming_accuracy.cc
index 5a98264401..2972ab778b 100644
--- a/tensorflow/examples/speech_commands/test_streaming_accuracy.cc
+++ b/tensorflow/examples/speech_commands/test_streaming_accuracy.cc
@@ -231,7 +231,7 @@ int main(int argc, char* argv[]) {
   }
 
   const int64 clip_duration_samples = (clip_duration_ms * sample_rate) / 1000;
-  const int64 sample_stride_samples = (clip_stride_ms * sample_rate) / 1000;
+  const int64 clip_stride_samples = (clip_stride_ms * sample_rate) / 1000;
   Tensor audio_data_tensor(tensorflow::DT_FLOAT,
                            tensorflow::TensorShape({clip_duration_samples, 1}));
 
@@ -246,7 +246,7 @@ int main(int argc, char* argv[]) {
 
   const int64 audio_data_end = (sample_count - clip_duration_ms);
   for (int64 audio_data_offset = 0; audio_data_offset < audio_data_end;
-       audio_data_offset += sample_stride_samples) {
+       audio_data_offset += clip_stride_samples) {
     const float* input_start = &(audio_data[audio_data_offset]);
     const float* input_end = input_start + clip_duration_samples;
     std::copy(input_start, input_end, audio_data_tensor.flat<float>().data());
-- 
GitLab


From c38773f18bfdce1de16ab5110e0cbbd50f0d6a79 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Wed, 4 Oct 2017 19:11:41 -0700
Subject: [PATCH 036/909] [XLA] Fix build of dumped_computation_to_text after
 change that removed an arg from CompileExecutable.

PiperOrigin-RevId: 171098077
---
 tensorflow/compiler/xla/tools/dumped_computation_to_text.cc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/compiler/xla/tools/dumped_computation_to_text.cc b/tensorflow/compiler/xla/tools/dumped_computation_to_text.cc
index 2a3a880328..78d8fb1f43 100644
--- a/tensorflow/compiler/xla/tools/dumped_computation_to_text.cc
+++ b/tensorflow/compiler/xla/tools/dumped_computation_to_text.cc
@@ -61,9 +61,9 @@ void RealMain(tensorflow::gtl::ArraySlice<char*> args, bool compile) {
         layouts.push_back(&program_shape->parameters(i));
       }
       StatusOr<std::unique_ptr<Executable>> executable =
-          local_service->CompileExecutable(
-              computation.handle(), layouts, &program_shape->result(),
-              /*device_ordinal=*/0, /*has_hybrid_result=*/true);
+          local_service->CompileExecutable(computation.handle(), layouts,
+                                           &program_shape->result(),
+                                           /*device_ordinal=*/0);
 
       const HloModule& module = executable.ValueOrDie()->module();
 
-- 
GitLab


From 0b863e0fef15f470265e0a87e660e421c6bc5ea1 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 4 Oct 2017 19:07:51 -0700
Subject: [PATCH 037/909] Renames variable for consistency with flag.

PiperOrigin-RevId: 171097818
---
 tensorflow/compiler/xla/tools/dumped_computation_to_text.cc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/compiler/xla/tools/dumped_computation_to_text.cc b/tensorflow/compiler/xla/tools/dumped_computation_to_text.cc
index 78d8fb1f43..2a3a880328 100644
--- a/tensorflow/compiler/xla/tools/dumped_computation_to_text.cc
+++ b/tensorflow/compiler/xla/tools/dumped_computation_to_text.cc
@@ -61,9 +61,9 @@ void RealMain(tensorflow::gtl::ArraySlice<char*> args, bool compile) {
         layouts.push_back(&program_shape->parameters(i));
       }
       StatusOr<std::unique_ptr<Executable>> executable =
-          local_service->CompileExecutable(computation.handle(), layouts,
-                                           &program_shape->result(),
-                                           /*device_ordinal=*/0);
+          local_service->CompileExecutable(
+              computation.handle(), layouts, &program_shape->result(),
+              /*device_ordinal=*/0, /*has_hybrid_result=*/true);
 
       const HloModule& module = executable.ValueOrDie()->module();
 
-- 
GitLab


From f2114a01130ded172ea4afb8f3ca20294ae62961 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Wed, 4 Oct 2017 19:11:41 -0700
Subject: [PATCH 038/909] [XLA] Fix build of dumped_computation_to_text after
 change that removed an arg from CompileExecutable.

PiperOrigin-RevId: 171098077
---
 tensorflow/compiler/xla/tools/dumped_computation_to_text.cc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/compiler/xla/tools/dumped_computation_to_text.cc b/tensorflow/compiler/xla/tools/dumped_computation_to_text.cc
index 2a3a880328..78d8fb1f43 100644
--- a/tensorflow/compiler/xla/tools/dumped_computation_to_text.cc
+++ b/tensorflow/compiler/xla/tools/dumped_computation_to_text.cc
@@ -61,9 +61,9 @@ void RealMain(tensorflow::gtl::ArraySlice<char*> args, bool compile) {
         layouts.push_back(&program_shape->parameters(i));
       }
       StatusOr<std::unique_ptr<Executable>> executable =
-          local_service->CompileExecutable(
-              computation.handle(), layouts, &program_shape->result(),
-              /*device_ordinal=*/0, /*has_hybrid_result=*/true);
+          local_service->CompileExecutable(computation.handle(), layouts,
+                                           &program_shape->result(),
+                                           /*device_ordinal=*/0);
 
       const HloModule& module = executable.ValueOrDie()->module();
 
-- 
GitLab


From ef2ee630e8fe290b06363f13ff440b4efcec9c81 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 4 Oct 2017 19:12:55 -0700
Subject: [PATCH 039/909] Fixes docs.

PiperOrigin-RevId: 171098172
---
 tensorflow/docs_src/tutorials/audio_recognition.md              | 2 +-
 tensorflow/examples/speech_commands/freeze.py                   | 2 +-
 .../examples/speech_commands/generate_streaming_test_wav.py     | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/docs_src/tutorials/audio_recognition.md b/tensorflow/docs_src/tutorials/audio_recognition.md
index 1ede915c01..670e480b12 100644
--- a/tensorflow/docs_src/tutorials/audio_recognition.md
+++ b/tensorflow/docs_src/tutorials/audio_recognition.md
@@ -361,7 +361,7 @@ This will output information about the number of words correctly matched, how
 many were given the wrong labels, and how many times the model triggered when
 there was no real word spoken. There are various parameters that control how the
 signal averaging works, including `--average_window_ms` which sets the length of
-time to average results over, `--sample_stride_ms` which is the time between
+time to average results over, `--clip_stride_ms` which is the time between
 applications of the model, `--suppression_ms` which stops subsequent word
 detections from triggering for a certain time after an initial one is found, and
 `--detection_threshold`, which controls how high the average score must be
diff --git a/tensorflow/examples/speech_commands/freeze.py b/tensorflow/examples/speech_commands/freeze.py
index cc2df9660a..c8671d9c41 100644
--- a/tensorflow/examples/speech_commands/freeze.py
+++ b/tensorflow/examples/speech_commands/freeze.py
@@ -153,7 +153,7 @@ if __name__ == '__main__':
       '--window_stride_ms',
       type=float,
       default=10.0,
-      help='How long each spectrogram timeslice is',)
+      help='How long the stride is between spectrogram timeslices',)
   parser.add_argument(
       '--dct_coefficient_count',
       type=int,
diff --git a/tensorflow/examples/speech_commands/generate_streaming_test_wav.py b/tensorflow/examples/speech_commands/generate_streaming_test_wav.py
index ac7c11856e..053206ae2f 100644
--- a/tensorflow/examples/speech_commands/generate_streaming_test_wav.py
+++ b/tensorflow/examples/speech_commands/generate_streaming_test_wav.py
@@ -240,7 +240,7 @@ if __name__ == '__main__':
       '--window_stride_ms',
       type=float,
       default=10.0,
-      help='How long each spectrogram timeslice is',)
+      help='How long the stride is between spectrogram timeslices',)
   parser.add_argument(
       '--dct_coefficient_count',
       type=int,
-- 
GitLab


From 2c3bf9eff79156e32512e8d6da2179cd044167b8 Mon Sep 17 00:00:00 2001
From: Derek Murray <mrry@google.com>
Date: Wed, 4 Oct 2017 19:14:02 -0700
Subject: [PATCH 040/909] [Windows] Include tf.contrib.image ops as part of the
 Windows build.

Fixes #9672.

PiperOrigin-RevId: 171098255
---
 tensorflow/contrib/cmake/tf_core_kernels.cmake            | 8 ++++++++
 tensorflow/contrib/cmake/tf_core_ops.cmake                | 2 ++
 tensorflow/contrib/cmake/tf_python.cmake                  | 4 ++++
 tensorflow/contrib/cmake/tf_tests.cmake                   | 1 +
 tensorflow/contrib/image/BUILD                            | 1 +
 tensorflow/contrib/image/python/ops/distort_image_ops.py  | 3 ++-
 .../python/ops/single_image_random_dot_stereograms.py     | 3 ++-
 7 files changed, 20 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/cmake/tf_core_kernels.cmake b/tensorflow/contrib/cmake/tf_core_kernels.cmake
index 61c6686ee0..46c680aad5 100644
--- a/tensorflow/contrib/cmake/tf_core_kernels.cmake
+++ b/tensorflow/contrib/cmake/tf_core_kernels.cmake
@@ -74,6 +74,13 @@ if(tensorflow_BUILD_CONTRIB_KERNELS)
       #"${tensorflow_source_dir}/tensorflow/contrib/ffmpeg/encode_audio_op.cc"
       "${tensorflow_source_dir}/tensorflow/contrib/framework/kernels/zero_initializer_op.cc"
       "${tensorflow_source_dir}/tensorflow/contrib/framework/ops/variable_ops.cc"
+      "${tensorflow_source_dir}/tensorflow/contrib/image/kernels/adjust_hsv_in_yiq_op.cc"
+      "${tensorflow_source_dir}/tensorflow/contrib/image/kernels/bipartite_match_op.cc"
+      "${tensorflow_source_dir}/tensorflow/contrib/image/kernels/image_ops.cc"
+      "${tensorflow_source_dir}/tensorflow/contrib/image/kernels/single_image_random_dot_stereograms_ops.cc"
+      "${tensorflow_source_dir}/tensorflow/contrib/image/ops/distort_image_ops.cc"
+      "${tensorflow_source_dir}/tensorflow/contrib/image/ops/image_ops.cc"
+      "${tensorflow_source_dir}/tensorflow/contrib/image/ops/single_image_random_dot_stereograms_ops.cc"
       "${tensorflow_source_dir}/tensorflow/contrib/layers/kernels/sparse_feature_cross_kernel.cc"
       "${tensorflow_source_dir}/tensorflow/contrib/layers/ops/sparse_feature_cross_op.cc"
       "${tensorflow_source_dir}/tensorflow/contrib/nccl/kernels/nccl_manager.cc"
@@ -167,6 +174,7 @@ endif(WIN32)
 file(GLOB_RECURSE tf_core_gpu_kernels_srcs
     "${tensorflow_source_dir}/tensorflow/core/kernels/*.cu.cc"
     "${tensorflow_source_dir}/tensorflow/contrib/framework/kernels/zero_initializer_op_gpu.cu.cc"
+    "${tensorflow_source_dir}/tensorflow/contrib/image/kernels/*.cu.cc"
     "${tensorflow_source_dir}/tensorflow/contrib/rnn/kernels/*.cu.cc"
     "${tensorflow_source_dir}/tensorflow/contrib/seq2seq/kernels/*.cu.cc"
 )
diff --git a/tensorflow/contrib/cmake/tf_core_ops.cmake b/tensorflow/contrib/cmake/tf_core_ops.cmake
index 78bccc08a3..dc9973917e 100644
--- a/tensorflow/contrib/cmake/tf_core_ops.cmake
+++ b/tensorflow/contrib/cmake/tf_core_ops.cmake
@@ -84,6 +84,8 @@ GENERATE_CONTRIB_OP_LIBRARY(factorization_factorization "${tensorflow_source_dir
 GENERATE_CONTRIB_OP_LIBRARY(framework_variable "${tensorflow_source_dir}/tensorflow/contrib/framework/ops/variable_ops.cc")
 GENERATE_CONTRIB_OP_LIBRARY(input_pipeline "${tensorflow_source_dir}/tensorflow/contrib/input_pipeline/ops/input_pipeline_ops.cc")
 GENERATE_CONTRIB_OP_LIBRARY(image "${tensorflow_source_dir}/tensorflow/contrib/image/ops/image_ops.cc")
+GENERATE_CONTRIB_OP_LIBRARY(image_distort_image "${tensorflow_source_dir}/tensorflow/contrib/image/ops/distort_image_ops.cc")
+GENERATE_CONTRIB_OP_LIBRARY(image_sirds "${tensorflow_source_dir}/tensorflow/contrib/image/ops/single_image_random_dot_stereograms_ops.cc")
 GENERATE_CONTRIB_OP_LIBRARY(layers_sparse_feature_cross "${tensorflow_source_dir}/tensorflow/contrib/layers/ops/sparse_feature_cross_op.cc")
 GENERATE_CONTRIB_OP_LIBRARY(memory_stats "${tensorflow_source_dir}/tensorflow/contrib/memory_stats/ops/memory_stats_ops.cc")
 GENERATE_CONTRIB_OP_LIBRARY(nccl "${tensorflow_source_dir}/tensorflow/contrib/nccl/ops/nccl_ops.cc")
diff --git a/tensorflow/contrib/cmake/tf_python.cmake b/tensorflow/contrib/cmake/tf_python.cmake
index 1e78f1e983..bb3e69d53c 100755
--- a/tensorflow/contrib/cmake/tf_python.cmake
+++ b/tensorflow/contrib/cmake/tf_python.cmake
@@ -773,6 +773,10 @@ GENERATE_PYTHON_OP_LIB("contrib_input_pipeline_ops"
   DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/input_pipeline/ops/gen_input_pipeline_ops.py)
 GENERATE_PYTHON_OP_LIB("contrib_image_ops"
   DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/image/ops/gen_image_ops.py)
+GENERATE_PYTHON_OP_LIB("contrib_image_distort_image_ops"
+  DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/image/ops/gen_distort_image_ops.py)
+GENERATE_PYTHON_OP_LIB("contrib_image_sirds_ops"
+  DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/image/ops/gen_single_image_random_dot_stereograms_ops.py)
 GENERATE_PYTHON_OP_LIB("contrib_layers_sparse_feature_cross_ops"
   DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/layers/ops/gen_sparse_feature_cross_op.py)
 GENERATE_PYTHON_OP_LIB("contrib_memory_stats_ops"
diff --git a/tensorflow/contrib/cmake/tf_tests.cmake b/tensorflow/contrib/cmake/tf_tests.cmake
index ba78e87ac0..658d19e493 100644
--- a/tensorflow/contrib/cmake/tf_tests.cmake
+++ b/tensorflow/contrib/cmake/tf_tests.cmake
@@ -152,6 +152,7 @@ if (tensorflow_BUILD_PYTHON_TESTS)
     "${tensorflow_source_dir}/tensorflow/python/training/*_test.py"
     "${tensorflow_source_dir}/tensorflow/contrib/data/*_test.py"
     "${tensorflow_source_dir}/tensorflow/contrib/factorization/*_test.py"
+    "${tensorflow_source_dir}/tensorflow/contrib/image/*_test.py"
     "${tensorflow_source_dir}/tensorflow/contrib/keras/python/keras/integration_test.py"
     "${tensorflow_source_dir}/tensorflow/contrib/nearest_neighbor/python/kernel_tests/*_test.py"
     "${tensorflow_source_dir}/tensorflow/contrib/seq2seq/python/kernel_tests/*_test.py"
diff --git a/tensorflow/contrib/image/BUILD b/tensorflow/contrib/image/BUILD
index a18f14112e..d0600d4668 100755
--- a/tensorflow/contrib/image/BUILD
+++ b/tensorflow/contrib/image/BUILD
@@ -211,6 +211,7 @@ py_library(
     srcs_version = "PY2AND3",
     deps = [
         ":image_py",
+        ":single_image_random_dot_stereograms_ops",
         "//tensorflow/contrib/util:util_py",
         "//tensorflow/python:framework_ops",
         "//tensorflow/python:platform",
diff --git a/tensorflow/contrib/image/python/ops/distort_image_ops.py b/tensorflow/contrib/image/python/ops/distort_image_ops.py
index 39f023a2b4..06e8e4ee72 100644
--- a/tensorflow/contrib/image/python/ops/distort_image_ops.py
+++ b/tensorflow/contrib/image/python/ops/distort_image_ops.py
@@ -17,6 +17,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from tensorflow.contrib.image.ops import gen_distort_image_ops
 from tensorflow.contrib.util import loader
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
@@ -132,7 +133,7 @@ def adjust_hsv_in_yiq(image,
     orig_dtype = image.dtype
     flt_image = image_ops.convert_image_dtype(image, dtypes.float32)
 
-    rgb_altered = _distort_image_ops.adjust_hsv_in_yiq(
+    rgb_altered = gen_distort_image_ops.adjust_hsv_in_yiq(
         flt_image, delta_hue, scale_saturation, scale_value)
 
     return image_ops.convert_image_dtype(rgb_altered, orig_dtype)
diff --git a/tensorflow/contrib/image/python/ops/single_image_random_dot_stereograms.py b/tensorflow/contrib/image/python/ops/single_image_random_dot_stereograms.py
index 79261c5e75..5cccf26028 100755
--- a/tensorflow/contrib/image/python/ops/single_image_random_dot_stereograms.py
+++ b/tensorflow/contrib/image/python/ops/single_image_random_dot_stereograms.py
@@ -17,6 +17,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from tensorflow.contrib.image.ops import gen_single_image_random_dot_stereograms_ops
 from tensorflow.contrib.util import loader
 from tensorflow.python.framework import ops
 from tensorflow.python.platform import resource_loader
@@ -107,7 +108,7 @@ def single_image_random_dot_stereograms(
     'depth_values'
   """
 
-  result = _sirds_ops.single_image_random_dot_stereograms(
+  result = gen_single_image_random_dot_stereograms_ops.single_image_random_dot_stereograms(  # pylint: disable=line-too-long
       depth_values=depth_values,
       hidden_surface_removal=hidden_surface_removal,
       convergence_dots_size=convergence_dots_size,
-- 
GitLab


From a3e5b1628322102914a46a5fbfca2db5cb8b9e11 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 4 Oct 2017 19:28:04 -0700
Subject: [PATCH 041/909] Avoids adding duplicate legacy_init_op to the
 saved_model's exported meta graph.

Previously, when the user restores graph from one meta graph generated from
saved_model and then re-generates another saved model, the re-generated model
will be invalid because it will contain duplicate legacy_init_ops.

PiperOrigin-RevId: 171099152
---
 tensorflow/python/saved_model/builder_impl.py |  7 ++++-
 .../python/saved_model/saved_model_test.py    | 30 ++++++++++++++++++-
 2 files changed, 35 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/saved_model/builder_impl.py b/tensorflow/python/saved_model/builder_impl.py
index 73a3f9075d..16651ffebc 100644
--- a/tensorflow/python/saved_model/builder_impl.py
+++ b/tensorflow/python/saved_model/builder_impl.py
@@ -140,11 +140,16 @@ class SavedModelBuilder(object):
 
     Raises:
       TypeError if legacy init op is not of type `Operation`.
+      AssertionError if the graph already contains one or more legacy init ops.
     """
     if legacy_init_op is not None:
       if not isinstance(legacy_init_op, ops.Operation):
         raise TypeError("legacy_init_op needs to be an Operation: %r" %
                         legacy_init_op)
+      if ops.get_collection(constants.LEGACY_INIT_OP_KEY):
+        raise AssertionError(
+            "graph already contains one or more legacy init ops under the "
+            "collection {}.".format(constants.LEGACY_INIT_OP_KEY))
       ops.add_to_collection(constants.LEGACY_INIT_OP_KEY, legacy_init_op)
 
   def _add_main_op(self, main_op):
@@ -258,7 +263,7 @@ class SavedModelBuilder(object):
 
     Raises:
       AssertionError: If the variables for the SavedModel have not been saved
-          yet.
+          yet, or if the graph already contains one or more legacy init ops.
     """
     if not self._has_saved_variables:
       raise AssertionError(
diff --git a/tensorflow/python/saved_model/saved_model_test.py b/tensorflow/python/saved_model/saved_model_test.py
index 5639e6855d..c6d2c32293 100644
--- a/tensorflow/python/saved_model/saved_model_test.py
+++ b/tensorflow/python/saved_model/saved_model_test.py
@@ -1,4 +1,4 @@
-## Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -637,6 +637,34 @@ class SavedModelTest(test.TestCase):
       # the legacy_init_op, following a restore.
       self.assertEqual(3, ops.get_collection("v")[2].eval())
 
+  def testLegacyInitOpWithNonEmptyCollection(self):
+    export_dir = os.path.join(test.get_temp_dir(),
+                              "test_legacy_init_op_with_non_empty_collection")
+    builder = saved_model_builder.SavedModelBuilder(export_dir)
+
+    with self.test_session(graph=ops.Graph()) as sess:
+      # Initialize variable `v1` to 1.
+      v1 = variables.Variable(1, name="v1")
+      ops.add_to_collection("v", v1)
+
+      # Initialize another variable `v2` to 42.
+      v2 = variables.Variable(42, name="v2", trainable=False, collections=[])
+      ops.add_to_collection("v", v2)
+
+      # Set up an assignment op to be run as part of the legacy_init_op.
+      assign_v2 = state_ops.assign(v2, v1)
+      legacy_init_op = control_flow_ops.group(assign_v2, name="legacy_init_op")
+
+      sess.run(variables.global_variables_initializer())
+
+      ops.add_to_collection(constants.LEGACY_INIT_OP_KEY,
+                            control_flow_ops.no_op())
+      # AssertionError should be raised since the LEGACY_INIT_OP_KEY collection
+      # is not empty and we don't support multiple init ops.
+      with self.assertRaises(AssertionError):
+        builder.add_meta_graph_and_variables(
+            sess, ["foo"], legacy_init_op=legacy_init_op)
+
   def testMultipleAssets(self):
     export_dir = os.path.join(test.get_temp_dir(), "test_multiple_assets")
     builder = saved_model_builder.SavedModelBuilder(export_dir)
-- 
GitLab


From 2f0787e1c8a7090fd231dac217e26824d8bc09c3 Mon Sep 17 00:00:00 2001
From: Jianwei Xie <xiejw@google.com>
Date: Wed, 4 Oct 2017 19:31:06 -0700
Subject: [PATCH 042/909] Change all quotes for TF_CONFIG from ' to " as JSON
 requires that.

PiperOrigin-RevId: 171099341
---
 tensorflow/python/estimator/training.py | 64 ++++++++++++-------------
 1 file changed, 32 insertions(+), 32 deletions(-)

diff --git a/tensorflow/python/estimator/training.py b/tensorflow/python/estimator/training.py
index 1bed19760b..17c072566a 100644
--- a/tensorflow/python/estimator/training.py
+++ b/tensorflow/python/estimator/training.py
@@ -328,29 +328,29 @@ def train_and_evaluate(estimator, train_spec, eval_spec):
   Setting environment variable depends on the platform. For example, on Linux,
   it can be done as follows (`$` is the shell prompt):
   ```
-  $ TF_CONFIG="<replace_with_real_content>" python train_model.py
+  $ TF_CONFIG='<replace_with_real_content>' python train_model.py
   ```
 
   For the content in `TF_CONFIG`, assume that the training cluster spec looks
   like:
   ```
-  cluster = {'chief': ['host0:2222'],
-             'worker': ['host1:2222', 'host2:2222', 'host3:2222'],
-             'ps': ['host4:2222', 'host5:2222']}
+  cluster = {"chief": ["host0:2222"],
+             "worker": ["host1:2222", "host2:2222", "host3:2222"],
+             "ps": ["host4:2222", "host5:2222"]}
   ```
 
   Example of `TF_CONFIG` for chief training worker (must have one and only one):
   ```
   # This should be a JSON string, which is set as environment variable. Usually
   # the cluster manager handles that.
-  TF_CONFIG="{
-      'cluster': {
-          'chief': ['host0:2222'],
-          'worker': ['host1:2222', 'host2:2222', 'host3:2222'],
-          'ps': ['host4:2222', 'host5:2222']
+  TF_CONFIG='{
+      "cluster": {
+          "chief": ["host0:2222"],
+          "worker": ["host1:2222", "host2:2222", "host3:2222"],
+          "ps": ["host4:2222", "host5:2222"]
       },
-      'task': {'type': 'chief', 'index': 0}
-  }"
+      "task": {"type": "chief", "index": 0}
+  }'
   ```
   Note that the chief worker also does the model training job, similar to other
   non-chief training workers (see next paragraph). In addition to the model
@@ -362,14 +362,14 @@ def train_and_evaluate(estimator, train_spec, eval_spec):
   ```
   # This should be a JSON string, which is set as environment variable. Usually
   # the cluster manager handles that.
-  TF_CONFIG="{
-      'cluster': {
-          'chief': ['host0:2222'],
-          'worker': ['host1:2222', 'host2:2222', 'host3:2222'],
-          'ps': ['host4:2222', 'host5:2222']
+  TF_CONFIG='{
+      "cluster": {
+          "chief": ["host0:2222"],
+          "worker": ["host1:2222", "host2:2222", "host3:2222"],
+          "ps": ["host4:2222", "host5:2222"]
       },
-      'task': {'type': 'worker', 'index': 0}
-  }"
+      "task": {"type": "worker", "index": 0}
+  }'
   ```
   where the `task.index` should be set as 0, 1, 2, in this example, respectively
   for non-chief training workers.
@@ -378,14 +378,14 @@ def train_and_evaluate(estimator, train_spec, eval_spec):
   ```
   # This should be a JSON string, which is set as environment variable. Usually
   # the cluster manager handles that.
-  TF_CONFIG="{
-      'cluster': {
-          'chief': ['host0:2222'],
-          'worker': ['host1:2222', 'host2:2222', 'host3:2222'],
-          'ps': ['host4:2222', 'host5:2222']
+  TF_CONFIG='{
+      "cluster": {
+          "chief": ["host0:2222"],
+          "worker": ["host1:2222", "host2:2222", "host3:2222"],
+          "ps": ["host4:2222", "host5:2222"]
       },
-      'task': {'type': 'ps', 'index': 0}
-  }"
+      "task": {"type": "ps", "index": 0}
+  }'
   ```
   where the `task.index` should be set as 0 and 1, in this example, respectively
   for parameter servers.
@@ -396,14 +396,14 @@ def train_and_evaluate(estimator, train_spec, eval_spec):
   ```
   # This should be a JSON string, which is set as environment variable. Usually
   # the cluster manager handles that.
-  TF_CONFIG="{
-      'cluster': {
-          'chief': ['host0:2222'],
-          'worker': ['host1:2222', 'host2:2222', 'host3:2222'],
-          'ps': ['host4:2222', 'host5:2222']
+  TF_CONFIG='{
+      "cluster": {
+          "chief": ["host0:2222"],
+          "worker": ["host1:2222", "host2:2222", "host3:2222"],
+          "ps": ["host4:2222", "host5:2222"]
       },
-      'task': {'type': 'evaluator', 'index': 0}
-  }"
+      "task": {"type": "evaluator", "index": 0}
+  }'
   ```
 
   Args:
-- 
GitLab


From 5267759301eeda724c788c6eb9fdaf624c644a7e Mon Sep 17 00:00:00 2001
From: Chris Leary <leary@google.com>
Date: Wed, 4 Oct 2017 19:42:46 -0700
Subject: [PATCH 043/909] [XLA] Add shape print-out to message for rank-test
 failure.

PiperOrigin-RevId: 171100052
---
 tensorflow/compiler/xla/tests/literal_test_util.cc | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xla/tests/literal_test_util.cc b/tensorflow/compiler/xla/tests/literal_test_util.cc
index 4d8b50fbbf..061a4e190f 100644
--- a/tensorflow/compiler/xla/tests/literal_test_util.cc
+++ b/tensorflow/compiler/xla/tests/literal_test_util.cc
@@ -49,7 +49,9 @@ namespace xla {
       AssertEqualShapes(expected.tuple_shapes(i), actual.tuple_shapes(i));
     }
   } else {
-    ASSERT_EQ(ShapeUtil::Rank(expected), ShapeUtil::Rank(actual));
+    ASSERT_EQ(ShapeUtil::Rank(expected), ShapeUtil::Rank(actual))
+        << "want rank of: " << ShapeUtil::HumanString(expected)
+        << " got rank of: " << ShapeUtil::HumanString(actual);
     ASSERT_EQ(expected.element_type(), actual.element_type())
         << PrimitiveType_Name(expected.element_type()) << " vs "
         << PrimitiveType_Name(actual.element_type());
-- 
GitLab


From df2768c93b60fd60e353cebddc27de8390bebd4b Mon Sep 17 00:00:00 2001
From: Suharsh Sivakumar <suharshs@google.com>
Date: Wed, 4 Oct 2017 20:17:39 -0700
Subject: [PATCH 044/909] Fix silly typo

PiperOrigin-RevId: 171102230
---
 tensorflow/contrib/quantize/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/quantize/__init__.py b/tensorflow/contrib/quantize/__init__.py
index f137723cb6..5d4e4575c9 100644
--- a/tensorflow/contrib/quantize/__init__.py
+++ b/tensorflow/contrib/quantize/__init__.py
@@ -25,7 +25,7 @@ from tensorflow.contrib.quantize.python.quantize_graph import *
 from tensorflow.python.util.all_util import remove_undocumented
 
 _allowed_symbols = [
-    "create_eval_graph,"
+    "create_eval_graph",
     "create_training_graph",
 ]
 
-- 
GitLab


From 929e9c5578c3d38df28da57ca22d1e4ce2600987 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 4 Oct 2017 21:21:50 -0700
Subject: [PATCH 045/909] Fix docstring.

PiperOrigin-RevId: 171105949
---
 tensorflow/contrib/gan/python/namedtuples.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/gan/python/namedtuples.py b/tensorflow/contrib/gan/python/namedtuples.py
index 27512526c4..48f5e8e47d 100644
--- a/tensorflow/contrib/gan/python/namedtuples.py
+++ b/tensorflow/contrib/gan/python/namedtuples.py
@@ -120,7 +120,7 @@ class GANLoss(
   """GANLoss contains the generator and discriminator losses.
 
   Args:
-    generator_loss: A tensor for the generator loss..
+    generator_loss: A tensor for the generator loss.
     discriminator_loss: A tensor for the discriminator loss.
   """
 
-- 
GitLab


From 165dd023351359171b0fe4f19c63a42aac4c2e47 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 4 Oct 2017 21:33:15 -0700
Subject: [PATCH 046/909] Update ops-related pbtxt files.

PiperOrigin-RevId: 171106509
---
 .../core/ops/compat/ops_history.v1.pbtxt      |  99 ++++++++++++
 tensorflow/core/ops/ops.pbtxt                 | 143 ++++++++++++++----
 2 files changed, 213 insertions(+), 29 deletions(-)

diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index e28b43c916..950422305e 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -12835,6 +12835,33 @@ op {
     }
   }
 }
+op {
+  name: "LogMatrixDeterminant"
+  input_arg {
+    name: "input"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "sign"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "log_abs_determinant"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+      }
+    }
+  }
+}
 op {
   name: "LogSoftmax"
   input_arg {
@@ -20216,6 +20243,78 @@ op {
   }
   is_stateful: true
 }
+op {
+  name: "RandomPoissonV2"
+  input_arg {
+    name: "shape"
+    type_attr: "S"
+  }
+  input_arg {
+    name: "rate"
+    type_attr: "R"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "dtype"
+  }
+  attr {
+    name: "seed"
+    type: "int"
+    default_value {
+      i: 0
+    }
+  }
+  attr {
+    name: "seed2"
+    type: "int"
+    default_value {
+      i: 0
+    }
+  }
+  attr {
+    name: "S"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+  attr {
+    name: "R"
+    type: "type"
+    default_value {
+      type: DT_DOUBLE
+    }
+    allowed_values {
+      list {
+        type: DT_HALF
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+  attr {
+    name: "dtype"
+    type: "type"
+    default_value {
+      type: DT_INT64
+    }
+    allowed_values {
+      list {
+        type: DT_HALF
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+  is_stateful: true
+}
 op {
   name: "RandomShuffle"
   input_arg {
diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index b8f827f1f7..cbde462325 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -11632,6 +11632,38 @@ op {
   summary: "Computes natural logarithm of (1 + x) element-wise."
   description: "I.e., \\\\(y = \\log_e (1 + x)\\\\)."
 }
+op {
+  name: "LogMatrixDeterminant"
+  input_arg {
+    name: "input"
+    description: "Shape is `[N, M, M]`."
+    type_attr: "T"
+  }
+  output_arg {
+    name: "sign"
+    description: "The signs of the log determinants of the inputs. Shape is `[N]`."
+    type_attr: "T"
+  }
+  output_arg {
+    name: "log_abs_determinant"
+    description: "The logs of the absolute values of the determinants\nof the N input matrices.  Shape is `[N]`."
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+      }
+    }
+  }
+  summary: "Computes the sign and the log of the absolute value of the determinant of"
+  description: "one or more square matrices.\n\nThe input is a tensor of shape `[N, M, M]` whose inner-most 2 dimensions\nform square matrices. The outputs are two tensors containing the signs and\nabsolute values of the log determinants for all N input submatrices\n`[..., :, :]` such that the determinant = sign*exp(log_abs_determinant).\nThe log_abs_determinant is computed as det(P)*sum(log(diag(LU))) where LU\nis the LU decomposition of the input and P is the corresponding\npermutation matrix."
+}
 op {
   name: "LogSoftmax"
   input_arg {
@@ -18778,6 +18810,85 @@ op {
   description: "This op uses two algorithms, depending on rate. If rate >= 10, then\nthe algorithm by Hormann is used to acquire samples via\ntransformation-rejection.\nSee http://www.sciencedirect.com/science/article/pii/0167668793909974.\n\nOtherwise, Knuth\'s algorithm is used to acquire samples via multiplying uniform\nrandom variables.\nSee Donald E. Knuth (1969). Seminumerical Algorithms. The Art of Computer\nProgramming, Volume 2. Addison Wesley"
   is_stateful: true
 }
+op {
+  name: "RandomPoissonV2"
+  input_arg {
+    name: "shape"
+    description: "1-D integer tensor. Shape of independent samples to draw from each\ndistribution described by the shape parameters given in rate."
+    type_attr: "S"
+  }
+  input_arg {
+    name: "rate"
+    description: "A tensor in which each scalar is a \"rate\" parameter describing the\nassociated poisson distribution."
+    type_attr: "R"
+  }
+  output_arg {
+    name: "output"
+    description: "A tensor with shape `shape + shape(rate)`. Each slice\n`[:, ..., :, i0, i1, ...iN]` contains the samples drawn for\n`rate[i0, i1, ...iN]`."
+    type_attr: "dtype"
+  }
+  attr {
+    name: "seed"
+    type: "int"
+    default_value {
+      i: 0
+    }
+    description: "If either `seed` or `seed2` are set to be non-zero, the random number\ngenerator is seeded by the given seed.  Otherwise, it is seeded by a\nrandom seed."
+  }
+  attr {
+    name: "seed2"
+    type: "int"
+    default_value {
+      i: 0
+    }
+    description: "A second seed to avoid seed collision."
+  }
+  attr {
+    name: "S"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+  attr {
+    name: "R"
+    type: "type"
+    default_value {
+      type: DT_DOUBLE
+    }
+    allowed_values {
+      list {
+        type: DT_HALF
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+  attr {
+    name: "dtype"
+    type: "type"
+    default_value {
+      type: DT_INT64
+    }
+    allowed_values {
+      list {
+        type: DT_HALF
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+  summary: "Outputs random values from the Poisson distribution(s) described by rate."
+  description: "This op uses two algorithms, depending on rate. If rate >= 10, then\nthe algorithm by Hormann is used to acquire samples via\ntransformation-rejection.\nSee http://www.sciencedirect.com/science/article/pii/0167668793909974.\n\nOtherwise, Knuth\'s algorithm is used to acquire samples via multiplying uniform\nrandom variables.\nSee Donald E. Knuth (1969). Seminumerical Algorithms. The Art of Computer\nProgramming, Volume 2. Addison Wesley"
+  is_stateful: true
+}
 op {
   name: "RandomShuffle"
   input_arg {
@@ -31758,40 +31869,14 @@ op {
   name: "Where"
   input_arg {
     name: "input"
-    type_attr: "T"
+    type: DT_BOOL
   }
   output_arg {
     name: "index"
     type: DT_INT64
   }
-  attr {
-    name: "T"
-    type: "type"
-    default_value {
-      type: DT_BOOL
-    }
-    allowed_values {
-      list {
-        type: DT_FLOAT
-        type: DT_DOUBLE
-        type: DT_INT64
-        type: DT_INT32
-        type: DT_UINT8
-        type: DT_UINT16
-        type: DT_INT16
-        type: DT_INT8
-        type: DT_COMPLEX64
-        type: DT_COMPLEX128
-        type: DT_QINT8
-        type: DT_QUINT8
-        type: DT_QINT32
-        type: DT_HALF
-        type: DT_BOOL
-      }
-    }
-  }
-  summary: "Returns locations of nonzero / true values in a tensor."
-  description: "This operation returns the coordinates of true elements in `input`. The\ncoordinates are returned in a 2-D tensor where the first dimension (rows)\nrepresents the number of true elements, and the second dimension (columns)\nrepresents the coordinates of the true elements. Keep in mind, the shape of\nthe output tensor can vary depending on how many true values there are in\n`input`. Indices are output in row-major order.\n\nFor example:\n\n```\n# \'input\' tensor is [[True, False]\n#                    [True, False]]\n# \'input\' has two true values, so output has two coordinates.\n# \'input\' has rank of 2, so coordinates have two indices.\nwhere(input) ==> [[0, 0],\n                  [1, 0]]\n\n# `input` tensor is [[[True, False]\n#                     [True, False]]\n#                    [[False, True]\n#                     [False, True]]\n#                    [[False, False]\n#                     [False, True]]]\n# \'input\' has 5 true values, so output has 5 coordinates.\n# \'input\' has rank of 3, so coordinates have three indices.\nwhere(input) ==> [[0, 0, 0],\n                  [0, 1, 0],\n                  [1, 0, 1],\n                  [1, 1, 1],\n                  [2, 1, 1]]\n\n# `input` tensor is [[[1.5,  0.0]\n#                     [-0.5, 0.0]]\n#                    [[0.0,  0.25]\n#                     [0.0,  0.75]]\n#                    [[0.0,  0.0]\n#                     [0.0,  0.01]]]\n# \'input\' has 5 nonzero values, so output has 5 coordinates.\n# \'input\' has rank of 3, so coordinates have three indices.\nwhere(input) ==> [[0, 0, 0],\n                  [0, 1, 0],\n                  [1, 0, 1],\n                  [1, 1, 1],\n                  [2, 1, 1]]\n\n# `input` tensor is [[[1.5 + 0.0j, 0.0  + 0.0j]\n#                     [0.0 + 0.5j, 0.0  + 0.0j]]\n#                    [[0.0 + 0.0j, 0.25 + 1.5j]\n#                     [0.0 + 0.0j, 0.75 + 0.0j]]\n#                    [[0.0 + 0.0j, 0.0  + 0.0j]\n#                     [0.0 + 0.0j, 0.01 + 0.0j]]]\n# \'input\' has 5 nonzero magnitude values, so output has 5 coordinates.\n# \'input\' has rank of 3, so coordinates have three indices.\nwhere(input) ==> [[0, 0, 0],\n                  [0, 1, 0],\n                  [1, 0, 1],\n                  [1, 1, 1],\n                  [2, 1, 1]]\n```"
+  summary: "Returns locations of true values in a boolean tensor."
+  description: "This operation returns the coordinates of true elements in `input`. The\ncoordinates are returned in a 2-D tensor where the first dimension (rows)\nrepresents the number of true elements, and the second dimension (columns)\nrepresents the coordinates of the true elements. Keep in mind, the shape of\nthe output tensor can vary depending on how many true values there are in\n`input`. Indices are output in row-major order.\n\nFor example:\n\n```\n# \'input\' tensor is [[True, False]\n#                    [True, False]]\n# \'input\' has two true values, so output has two coordinates.\n# \'input\' has rank of 2, so coordinates have two indices.\nwhere(input) ==> [[0, 0],\n                  [1, 0]]\n\n# `input` tensor is [[[True, False]\n#                     [True, False]]\n#                    [[False, True]\n#                     [False, True]]\n#                    [[False, False]\n#                     [False, True]]]\n# \'input\' has 5 true values, so output has 5 coordinates.\n# \'input\' has rank of 3, so coordinates have three indices.\nwhere(input) ==> [[0, 0, 0],\n                  [0, 1, 0],\n                  [1, 0, 1],\n                  [1, 1, 1],\n                  [2, 1, 1]]\n```"
 }
 op {
   name: "WholeFileReader"
-- 
GitLab


From 55e765b578529364522b92d732d1240243412197 Mon Sep 17 00:00:00 2001
From: Ian Langmore <langmore@google.com>
Date: Wed, 4 Oct 2017 21:35:16 -0700
Subject: [PATCH 047/909] BUGFIX:  AbsoluteValue.invert(y) raises if y < 0 and
 validate_args

PiperOrigin-RevId: 171106639
---
 .../bijectors/absolute_value_test.py          | 12 ++++++++++
 .../ops/bijectors/absolute_value_impl.py      | 23 +++++++++++++++++--
 2 files changed, 33 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/absolute_value_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/absolute_value_test.py
index da50037d6e..e0d65c79b2 100644
--- a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/absolute_value_test.py
+++ b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/absolute_value_test.py
@@ -68,6 +68,18 @@ class AbsoluteValueTest(test.TestCase):
         sess.run(abs_bijector.inverse_log_det_jacobian([1.]),
                  feed_dict={event_ndims: 1})
 
+  def testNegativeYRaisesForInverseIfValidateArgs(self):
+    with self.test_session() as sess:
+      bijector = AbsoluteValue(event_ndims=0, validate_args=True)
+      with self.assertRaisesOpError("y was negative"):
+        sess.run(bijector.inverse(-1.))
+
+  def testNegativeYRaisesForILDJIfValidateArgs(self):
+    with self.test_session() as sess:
+      bijector = AbsoluteValue(event_ndims=0, validate_args=True)
+      with self.assertRaisesOpError("y was negative"):
+        sess.run(bijector.inverse_log_det_jacobian(-1.))
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/absolute_value_impl.py b/tensorflow/contrib/distributions/python/ops/bijectors/absolute_value_impl.py
index 065a049cf7..b84502003a 100644
--- a/tensorflow/contrib/distributions/python/ops/bijectors/absolute_value_impl.py
+++ b/tensorflow/contrib/distributions/python/ops/bijectors/absolute_value_impl.py
@@ -35,7 +35,17 @@ class AbsoluteValue(bijector.Bijector):
   """Computes `Y = g(X) = Abs(X)`, element-wise.
 
   This non-injective bijector allows for transformations of scalar distributions
-  with the absolute value function.
+  with the absolute value function, which maps `(-inf, inf)` to `[0, inf)`.
+
+  * For `y in (0, inf)`, `AbsoluteValue.inverse(y)` returns the set inverse
+    `{x in (-inf, inf) : |x| = y}` as a tuple, `-y, y`.
+  * `AbsoluteValue.inverse(0)` returns `0, 0`, which is not the set inverse
+    (the set inverse is the singleton `{0}`), but "works" in conjunction with
+    `TransformedDistribution` to produce a left semi-continuous pdf.
+  * For `y < 0`, `AbsoluteValue.inverse(y)` happily returns the
+    wrong thing, `-y, y`.  This is done for efficiency.  If
+    `validate_args == True`, `y < 0` will raise an exception.
+
 
   ```python
   abs = ds.bijectors.AbsoluteValue()
@@ -68,7 +78,8 @@ class AbsoluteValue(bijector.Bijector):
         with a particular draw from the distribution.  Currently only zero is
         supported.
       validate_args: Python `bool` indicating whether arguments should be
-        checked for correctness.
+        checked for correctness, in particular whether inputs to `inverse` and
+        `inverse_log_det_jacobian` are non-negative.
       name: Python `str` name given to ops managed by this object.
 
     Raises:
@@ -98,6 +109,10 @@ class AbsoluteValue(bijector.Bijector):
     return math_ops.abs(x)
 
   def _inverse(self, y):
+    if self.validate_args:
+      y = control_flow_ops.with_dependencies(
+          [check_ops.assert_non_negative(y, message="Argument y was negative")],
+          y)
     return -y, y
 
   def _inverse_log_det_jacobian(self, y):
@@ -106,6 +121,10 @@ class AbsoluteValue(bijector.Bijector):
     # so Log|DF^{-1}(y)| = Log[1, 1] = [0, 0].
     batch_shape = array_ops.shape(y)[:array_ops.rank(y) - self.event_ndims]
     zeros = array_ops.zeros(batch_shape, dtype=y.dtype)
+    if self.validate_args:
+      zeros = control_flow_ops.with_dependencies(
+          [check_ops.assert_non_negative(y, message="Argument y was negative")],
+          zeros)
     return zeros, zeros
 
   @property
-- 
GitLab


From 07124fac0ec20e584d018035300d44ee55e451f0 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Wed, 4 Oct 2017 22:24:29 -0700
Subject: [PATCH 048/909] Fix build of dumped_computation_to_operation_list.

CompileExecutable had its last arg removed.

PiperOrigin-RevId: 171109500
---
 .../xla/tools/dumped_computation_to_operation_list.cc       | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/compiler/xla/tools/dumped_computation_to_operation_list.cc b/tensorflow/compiler/xla/tools/dumped_computation_to_operation_list.cc
index aa297ac171..5ede37b873 100644
--- a/tensorflow/compiler/xla/tools/dumped_computation_to_operation_list.cc
+++ b/tensorflow/compiler/xla/tools/dumped_computation_to_operation_list.cc
@@ -86,9 +86,9 @@ void RealMain(tensorflow::gtl::ArraySlice<char*> args) {
       layouts.push_back(&program_shape->parameters(i));
     }
     StatusOr<std::unique_ptr<Executable>> executable =
-        local_service->CompileExecutable(
-            computation.handle(), layouts, &program_shape->result(),
-            /*device_ordinal=*/0, /*has_hybrid_result=*/true);
+        local_service->CompileExecutable(computation.handle(), layouts,
+                                         &program_shape->result(),
+                                         /*device_ordinal=*/0);
 
     const HloModule& module = executable.ValueOrDie()->module();
 
-- 
GitLab


From cde6636b0130e639fcc3e157dc09aeb816a35e05 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 4 Oct 2017 22:33:17 -0700
Subject: [PATCH 049/909] Go: Update generated wrapper functions for TensorFlow
 ops.

PiperOrigin-RevId: 171110005
---
 tensorflow/go/op/wrappers.go | 1026 ++++++++++++++++++----------------
 1 file changed, 551 insertions(+), 475 deletions(-)

diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go
index 09a509f21b..ef1f8a9df6 100644
--- a/tensorflow/go/op/wrappers.go
+++ b/tensorflow/go/op/wrappers.go
@@ -1412,7 +1412,7 @@ func BroadcastArgs(scope *Scope, s0 tf.Output, s1 tf.Output) (r0 tf.Output) {
 	return op.Output(0)
 }
 
-// Returns locations of nonzero / true values in a tensor.
+// Returns locations of true values in a boolean tensor.
 //
 // This operation returns the coordinates of true elements in `input`. The
 // coordinates are returned in a 2-D tensor where the first dimension (rows)
@@ -1444,34 +1444,6 @@ func BroadcastArgs(scope *Scope, s0 tf.Output, s1 tf.Output) (r0 tf.Output) {
 //                   [1, 0, 1],
 //                   [1, 1, 1],
 //                   [2, 1, 1]]
-//
-// # `input` tensor is [[[1.5,  0.0]
-// #                     [-0.5, 0.0]]
-// #                    [[0.0,  0.25]
-// #                     [0.0,  0.75]]
-// #                    [[0.0,  0.0]
-// #                     [0.0,  0.01]]]
-// # 'input' has 5 nonzero values, so output has 5 coordinates.
-// # 'input' has rank of 3, so coordinates have three indices.
-// where(input) ==> [[0, 0, 0],
-//                   [0, 1, 0],
-//                   [1, 0, 1],
-//                   [1, 1, 1],
-//                   [2, 1, 1]]
-//
-// # `input` tensor is [[[1.5 + 0.0j, 0.0  + 0.0j]
-// #                     [0.0 + 0.5j, 0.0  + 0.0j]]
-// #                    [[0.0 + 0.0j, 0.25 + 1.5j]
-// #                     [0.0 + 0.0j, 0.75 + 0.0j]]
-// #                    [[0.0 + 0.0j, 0.0  + 0.0j]
-// #                     [0.0 + 0.0j, 0.01 + 0.0j]]]
-// # 'input' has 5 nonzero magnitude values, so output has 5 coordinates.
-// # 'input' has rank of 3, so coordinates have three indices.
-// where(input) ==> [[0, 0, 0],
-//                   [0, 1, 0],
-//                   [1, 0, 1],
-//                   [1, 1, 1],
-//                   [2, 1, 1]]
 // ```
 func Where(scope *Scope, input tf.Output) (index tf.Output) {
 	if scope.Err() != nil {
@@ -6994,194 +6966,6 @@ func ExtractJpegShape(scope *Scope, contents tf.Output, optional ...ExtractJpegS
 	return op.Output(0)
 }
 
-// AllCandidateSamplerAttr is an optional argument to AllCandidateSampler.
-type AllCandidateSamplerAttr func(optionalAttr)
-
-// AllCandidateSamplerSeed sets the optional seed attribute to value.
-//
-// value: If either seed or seed2 are set to be non-zero, the random number
-// generator is seeded by the given seed.  Otherwise, it is seeded by a
-// random seed.
-// If not specified, defaults to 0
-func AllCandidateSamplerSeed(value int64) AllCandidateSamplerAttr {
-	return func(m optionalAttr) {
-		m["seed"] = value
-	}
-}
-
-// AllCandidateSamplerSeed2 sets the optional seed2 attribute to value.
-//
-// value: An second seed to avoid seed collision.
-// If not specified, defaults to 0
-func AllCandidateSamplerSeed2(value int64) AllCandidateSamplerAttr {
-	return func(m optionalAttr) {
-		m["seed2"] = value
-	}
-}
-
-// Generates labels for candidate sampling with a learned unigram distribution.
-//
-// See explanations of candidate sampling and the data formats at
-// go/candidate-sampling.
-//
-// For each batch, this op picks a single set of sampled candidate labels.
-//
-// The advantages of sampling candidates per-batch are simplicity and the
-// possibility of efficient dense matrix multiplication. The disadvantage is that
-// the sampled candidates must be chosen independently of the context and of the
-// true labels.
-//
-// Arguments:
-//	true_classes: A batch_size * num_true matrix, in which each row contains the
-// IDs of the num_true target_classes in the corresponding original label.
-//	num_true: Number of true labels per context.
-//	num_sampled: Number of candidates to produce.
-//	unique: If unique is true, we sample with rejection, so that all sampled
-// candidates in a batch are unique. This requires some approximation to
-// estimate the post-rejection sampling probabilities.
-//
-// Returns A vector of length num_sampled, in which each element is
-// the ID of a sampled candidate.A batch_size * num_true matrix, representing
-// the number of times each candidate is expected to occur in a batch
-// of sampled candidates. If unique=true, then this is a probability.A vector of length num_sampled, for each sampled
-// candidate representing the number of times the candidate is expected
-// to occur in a batch of sampled candidates.  If unique=true, then this is a
-// probability.
-func AllCandidateSampler(scope *Scope, true_classes tf.Output, num_true int64, num_sampled int64, unique bool, optional ...AllCandidateSamplerAttr) (sampled_candidates tf.Output, true_expected_count tf.Output, sampled_expected_count tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"num_true": num_true, "num_sampled": num_sampled, "unique": unique}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "AllCandidateSampler",
-		Input: []tf.Input{
-			true_classes,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
-}
-
-// DecodeAndCropJpegAttr is an optional argument to DecodeAndCropJpeg.
-type DecodeAndCropJpegAttr func(optionalAttr)
-
-// DecodeAndCropJpegChannels sets the optional channels attribute to value.
-//
-// value: Number of color channels for the decoded image.
-// If not specified, defaults to 0
-func DecodeAndCropJpegChannels(value int64) DecodeAndCropJpegAttr {
-	return func(m optionalAttr) {
-		m["channels"] = value
-	}
-}
-
-// DecodeAndCropJpegRatio sets the optional ratio attribute to value.
-//
-// value: Downscaling ratio.
-// If not specified, defaults to 1
-func DecodeAndCropJpegRatio(value int64) DecodeAndCropJpegAttr {
-	return func(m optionalAttr) {
-		m["ratio"] = value
-	}
-}
-
-// DecodeAndCropJpegFancyUpscaling sets the optional fancy_upscaling attribute to value.
-//
-// value: If true use a slower but nicer upscaling of the
-// chroma planes (yuv420/422 only).
-// If not specified, defaults to true
-func DecodeAndCropJpegFancyUpscaling(value bool) DecodeAndCropJpegAttr {
-	return func(m optionalAttr) {
-		m["fancy_upscaling"] = value
-	}
-}
-
-// DecodeAndCropJpegTryRecoverTruncated sets the optional try_recover_truncated attribute to value.
-//
-// value: If true try to recover an image from truncated input.
-// If not specified, defaults to false
-func DecodeAndCropJpegTryRecoverTruncated(value bool) DecodeAndCropJpegAttr {
-	return func(m optionalAttr) {
-		m["try_recover_truncated"] = value
-	}
-}
-
-// DecodeAndCropJpegAcceptableFraction sets the optional acceptable_fraction attribute to value.
-//
-// value: The minimum required fraction of lines before a truncated
-// input is accepted.
-// If not specified, defaults to 1
-func DecodeAndCropJpegAcceptableFraction(value float32) DecodeAndCropJpegAttr {
-	return func(m optionalAttr) {
-		m["acceptable_fraction"] = value
-	}
-}
-
-// DecodeAndCropJpegDctMethod sets the optional dct_method attribute to value.
-//
-// value: string specifying a hint about the algorithm used for
-// decompression.  Defaults to "" which maps to a system-specific
-// default.  Currently valid values are ["INTEGER_FAST",
-// "INTEGER_ACCURATE"].  The hint may be ignored (e.g., the internal
-// jpeg library changes to a version that does not have that specific
-// option.)
-// If not specified, defaults to ""
-func DecodeAndCropJpegDctMethod(value string) DecodeAndCropJpegAttr {
-	return func(m optionalAttr) {
-		m["dct_method"] = value
-	}
-}
-
-// Decode and Crop a JPEG-encoded image to a uint8 tensor.
-//
-// The attr `channels` indicates the desired number of color channels for the
-// decoded image.
-//
-// Accepted values are:
-//
-// *   0: Use the number of channels in the JPEG-encoded image.
-// *   1: output a grayscale image.
-// *   3: output an RGB image.
-//
-// If needed, the JPEG-encoded image is transformed to match the requested number
-// of color channels.
-//
-// The attr `ratio` allows downscaling the image by an integer factor during
-// decoding.  Allowed values are: 1, 2, 4, and 8.  This is much faster than
-// downscaling the image later.
-//
-//
-// It is equivalent to a combination of decode and crop, but much faster by only
-// decoding partial jpeg image.
-//
-// Arguments:
-//	contents: 0-D.  The JPEG-encoded image.
-//	crop_window: 1-D.  The crop window: [crop_y, crop_x, crop_height, crop_width].
-//
-// Returns 3-D with shape `[height, width, channels]`..
-func DecodeAndCropJpeg(scope *Scope, contents tf.Output, crop_window tf.Output, optional ...DecodeAndCropJpegAttr) (image tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "DecodeAndCropJpeg",
-		Input: []tf.Input{
-			contents, crop_window,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
 // DecodeJpegAttr is an optional argument to DecodeJpeg.
 type DecodeJpegAttr func(optionalAttr)
 
@@ -11179,6 +10963,37 @@ func DenseToDenseSetOperation(scope *Scope, set1 tf.Output, set2 tf.Output, set_
 	return op.Output(0), op.Output(1), op.Output(2)
 }
 
+// Computes the sign and the log of the absolute value of the determinant of
+//
+// one or more square matrices.
+//
+// The input is a tensor of shape `[N, M, M]` whose inner-most 2 dimensions
+// form square matrices. The outputs are two tensors containing the signs and
+// absolute values of the log determinants for all N input submatrices
+// `[..., :, :]` such that the determinant = sign*exp(log_abs_determinant).
+// The log_abs_determinant is computed as det(P)*sum(log(diag(LU))) where LU
+// is the LU decomposition of the input and P is the corresponding
+// permutation matrix.
+//
+// Arguments:
+//	input: Shape is `[N, M, M]`.
+//
+// Returns The signs of the log determinants of the inputs. Shape is `[N]`.The logs of the absolute values of the determinants
+// of the N input matrices.  Shape is `[N]`.
+func LogMatrixDeterminant(scope *Scope, input tf.Output) (sign tf.Output, log_abs_determinant tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "LogMatrixDeterminant",
+		Input: []tf.Input{
+			input,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1)
+}
+
 // SetSizeAttr is an optional argument to SetSize.
 type SetSizeAttr func(optionalAttr)
 
@@ -11590,28 +11405,400 @@ func SparseSparseMinimum(scope *Scope, a_indices tf.Output, a_values tf.Output,
 	return op.Output(0), op.Output(1)
 }
 
-// Computes the gradient of the sigmoid of `x` wrt its input.
+// AllCandidateSamplerAttr is an optional argument to AllCandidateSampler.
+type AllCandidateSamplerAttr func(optionalAttr)
+
+// AllCandidateSamplerSeed sets the optional seed attribute to value.
 //
-// Specifically, `grad = dy * y * (1 - y)`, where `y = sigmoid(x)`, and
-// `dy` is the corresponding input gradient.
-func SigmoidGrad(scope *Scope, y tf.Output, dy tf.Output) (z tf.Output) {
-	if scope.Err() != nil {
-		return
+// value: If either seed or seed2 are set to be non-zero, the random number
+// generator is seeded by the given seed.  Otherwise, it is seeded by a
+// random seed.
+// If not specified, defaults to 0
+func AllCandidateSamplerSeed(value int64) AllCandidateSamplerAttr {
+	return func(m optionalAttr) {
+		m["seed"] = value
 	}
-	opspec := tf.OpSpec{
-		Type: "SigmoidGrad",
-		Input: []tf.Input{
-			y, dy,
-		},
+}
+
+// AllCandidateSamplerSeed2 sets the optional seed2 attribute to value.
+//
+// value: An second seed to avoid seed collision.
+// If not specified, defaults to 0
+func AllCandidateSamplerSeed2(value int64) AllCandidateSamplerAttr {
+	return func(m optionalAttr) {
+		m["seed2"] = value
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
 }
 
-// Subtracts a value from the current value of a variable.
+// Generates labels for candidate sampling with a learned unigram distribution.
 //
-// Any ReadVariableOp which depends directly or indirectly on this assign is
-// guaranteed to see the incremented value or a subsequent newer one.
+// See explanations of candidate sampling and the data formats at
+// go/candidate-sampling.
+//
+// For each batch, this op picks a single set of sampled candidate labels.
+//
+// The advantages of sampling candidates per-batch are simplicity and the
+// possibility of efficient dense matrix multiplication. The disadvantage is that
+// the sampled candidates must be chosen independently of the context and of the
+// true labels.
+//
+// Arguments:
+//	true_classes: A batch_size * num_true matrix, in which each row contains the
+// IDs of the num_true target_classes in the corresponding original label.
+//	num_true: Number of true labels per context.
+//	num_sampled: Number of candidates to produce.
+//	unique: If unique is true, we sample with rejection, so that all sampled
+// candidates in a batch are unique. This requires some approximation to
+// estimate the post-rejection sampling probabilities.
+//
+// Returns A vector of length num_sampled, in which each element is
+// the ID of a sampled candidate.A batch_size * num_true matrix, representing
+// the number of times each candidate is expected to occur in a batch
+// of sampled candidates. If unique=true, then this is a probability.A vector of length num_sampled, for each sampled
+// candidate representing the number of times the candidate is expected
+// to occur in a batch of sampled candidates.  If unique=true, then this is a
+// probability.
+func AllCandidateSampler(scope *Scope, true_classes tf.Output, num_true int64, num_sampled int64, unique bool, optional ...AllCandidateSamplerAttr) (sampled_candidates tf.Output, true_expected_count tf.Output, sampled_expected_count tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"num_true": num_true, "num_sampled": num_sampled, "unique": unique}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "AllCandidateSampler",
+		Input: []tf.Input{
+			true_classes,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1), op.Output(2)
+}
+
+// DecodeAndCropJpegAttr is an optional argument to DecodeAndCropJpeg.
+type DecodeAndCropJpegAttr func(optionalAttr)
+
+// DecodeAndCropJpegChannels sets the optional channels attribute to value.
+//
+// value: Number of color channels for the decoded image.
+// If not specified, defaults to 0
+func DecodeAndCropJpegChannels(value int64) DecodeAndCropJpegAttr {
+	return func(m optionalAttr) {
+		m["channels"] = value
+	}
+}
+
+// DecodeAndCropJpegRatio sets the optional ratio attribute to value.
+//
+// value: Downscaling ratio.
+// If not specified, defaults to 1
+func DecodeAndCropJpegRatio(value int64) DecodeAndCropJpegAttr {
+	return func(m optionalAttr) {
+		m["ratio"] = value
+	}
+}
+
+// DecodeAndCropJpegFancyUpscaling sets the optional fancy_upscaling attribute to value.
+//
+// value: If true use a slower but nicer upscaling of the
+// chroma planes (yuv420/422 only).
+// If not specified, defaults to true
+func DecodeAndCropJpegFancyUpscaling(value bool) DecodeAndCropJpegAttr {
+	return func(m optionalAttr) {
+		m["fancy_upscaling"] = value
+	}
+}
+
+// DecodeAndCropJpegTryRecoverTruncated sets the optional try_recover_truncated attribute to value.
+//
+// value: If true try to recover an image from truncated input.
+// If not specified, defaults to false
+func DecodeAndCropJpegTryRecoverTruncated(value bool) DecodeAndCropJpegAttr {
+	return func(m optionalAttr) {
+		m["try_recover_truncated"] = value
+	}
+}
+
+// DecodeAndCropJpegAcceptableFraction sets the optional acceptable_fraction attribute to value.
+//
+// value: The minimum required fraction of lines before a truncated
+// input is accepted.
+// If not specified, defaults to 1
+func DecodeAndCropJpegAcceptableFraction(value float32) DecodeAndCropJpegAttr {
+	return func(m optionalAttr) {
+		m["acceptable_fraction"] = value
+	}
+}
+
+// DecodeAndCropJpegDctMethod sets the optional dct_method attribute to value.
+//
+// value: string specifying a hint about the algorithm used for
+// decompression.  Defaults to "" which maps to a system-specific
+// default.  Currently valid values are ["INTEGER_FAST",
+// "INTEGER_ACCURATE"].  The hint may be ignored (e.g., the internal
+// jpeg library changes to a version that does not have that specific
+// option.)
+// If not specified, defaults to ""
+func DecodeAndCropJpegDctMethod(value string) DecodeAndCropJpegAttr {
+	return func(m optionalAttr) {
+		m["dct_method"] = value
+	}
+}
+
+// Decode and Crop a JPEG-encoded image to a uint8 tensor.
+//
+// The attr `channels` indicates the desired number of color channels for the
+// decoded image.
+//
+// Accepted values are:
+//
+// *   0: Use the number of channels in the JPEG-encoded image.
+// *   1: output a grayscale image.
+// *   3: output an RGB image.
+//
+// If needed, the JPEG-encoded image is transformed to match the requested number
+// of color channels.
+//
+// The attr `ratio` allows downscaling the image by an integer factor during
+// decoding.  Allowed values are: 1, 2, 4, and 8.  This is much faster than
+// downscaling the image later.
+//
+//
+// It is equivalent to a combination of decode and crop, but much faster by only
+// decoding partial jpeg image.
+//
+// Arguments:
+//	contents: 0-D.  The JPEG-encoded image.
+//	crop_window: 1-D.  The crop window: [crop_y, crop_x, crop_height, crop_width].
+//
+// Returns 3-D with shape `[height, width, channels]`..
+func DecodeAndCropJpeg(scope *Scope, contents tf.Output, crop_window tf.Output, optional ...DecodeAndCropJpegAttr) (image tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "DecodeAndCropJpeg",
+		Input: []tf.Input{
+			contents, crop_window,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// RandomPoissonV2Attr is an optional argument to RandomPoissonV2.
+type RandomPoissonV2Attr func(optionalAttr)
+
+// RandomPoissonV2Seed sets the optional seed attribute to value.
+//
+// value: If either `seed` or `seed2` are set to be non-zero, the random number
+// generator is seeded by the given seed.  Otherwise, it is seeded by a
+// random seed.
+// If not specified, defaults to 0
+func RandomPoissonV2Seed(value int64) RandomPoissonV2Attr {
+	return func(m optionalAttr) {
+		m["seed"] = value
+	}
+}
+
+// RandomPoissonV2Seed2 sets the optional seed2 attribute to value.
+//
+// value: A second seed to avoid seed collision.
+// If not specified, defaults to 0
+func RandomPoissonV2Seed2(value int64) RandomPoissonV2Attr {
+	return func(m optionalAttr) {
+		m["seed2"] = value
+	}
+}
+
+// RandomPoissonV2Dtype sets the optional dtype attribute to value.
+// If not specified, defaults to DT_INT64
+func RandomPoissonV2Dtype(value tf.DataType) RandomPoissonV2Attr {
+	return func(m optionalAttr) {
+		m["dtype"] = value
+	}
+}
+
+// Outputs random values from the Poisson distribution(s) described by rate.
+//
+// This op uses two algorithms, depending on rate. If rate >= 10, then
+// the algorithm by Hormann is used to acquire samples via
+// transformation-rejection.
+// See http://www.sciencedirect.com/science/article/pii/0167668793909974.
+//
+// Otherwise, Knuth's algorithm is used to acquire samples via multiplying uniform
+// random variables.
+// See Donald E. Knuth (1969). Seminumerical Algorithms. The Art of Computer
+// Programming, Volume 2. Addison Wesley
+//
+// Arguments:
+//	shape: 1-D integer tensor. Shape of independent samples to draw from each
+// distribution described by the shape parameters given in rate.
+//	rate: A tensor in which each scalar is a "rate" parameter describing the
+// associated poisson distribution.
+//
+// Returns A tensor with shape `shape + shape(rate)`. Each slice
+// `[:, ..., :, i0, i1, ...iN]` contains the samples drawn for
+// `rate[i0, i1, ...iN]`.
+func RandomPoissonV2(scope *Scope, shape tf.Output, rate tf.Output, optional ...RandomPoissonV2Attr) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "RandomPoissonV2",
+		Input: []tf.Input{
+			shape, rate,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// OrderedMapPeekAttr is an optional argument to OrderedMapPeek.
+type OrderedMapPeekAttr func(optionalAttr)
+
+// OrderedMapPeekCapacity sets the optional capacity attribute to value.
+// If not specified, defaults to 0
+//
+// REQUIRES: value >= 0
+func OrderedMapPeekCapacity(value int64) OrderedMapPeekAttr {
+	return func(m optionalAttr) {
+		m["capacity"] = value
+	}
+}
+
+// OrderedMapPeekMemoryLimit sets the optional memory_limit attribute to value.
+// If not specified, defaults to 0
+//
+// REQUIRES: value >= 0
+func OrderedMapPeekMemoryLimit(value int64) OrderedMapPeekAttr {
+	return func(m optionalAttr) {
+		m["memory_limit"] = value
+	}
+}
+
+// OrderedMapPeekContainer sets the optional container attribute to value.
+// If not specified, defaults to ""
+func OrderedMapPeekContainer(value string) OrderedMapPeekAttr {
+	return func(m optionalAttr) {
+		m["container"] = value
+	}
+}
+
+// OrderedMapPeekSharedName sets the optional shared_name attribute to value.
+// If not specified, defaults to ""
+func OrderedMapPeekSharedName(value string) OrderedMapPeekAttr {
+	return func(m optionalAttr) {
+		m["shared_name"] = value
+	}
+}
+
+// Op peeks at the values at the specified key.  If the
+//
+// underlying container does not contain this key
+// this op will block until it does.   This Op is optimized for
+// performance.
+func OrderedMapPeek(scope *Scope, key tf.Output, indices tf.Output, dtypes []tf.DataType, optional ...OrderedMapPeekAttr) (values []tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"dtypes": dtypes}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "OrderedMapPeek",
+		Input: []tf.Input{
+			key, indices,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	if scope.Err() != nil {
+		return
+	}
+	var idx int
+	var err error
+	if values, idx, err = makeOutputList(op, idx, "values"); err != nil {
+		scope.UpdateErr("OrderedMapPeek", err)
+		return
+	}
+	return values
+}
+
+// Adds two `SparseTensor` objects to produce another `SparseTensor`.
+//
+// The input `SparseTensor` objects' indices are assumed ordered in standard
+// lexicographic order.  If this is not the case, before this step run
+// `SparseReorder` to restore index ordering.
+//
+// By default, if two values sum to zero at some index, the output `SparseTensor`
+// would still include that particular location in its index, storing a zero in the
+// corresponding value slot.  To override this, callers can specify `thresh`,
+// indicating that if the sum has a magnitude strictly smaller than `thresh`, its
+// corresponding value and index would then not be included.  In particular,
+// `thresh == 0` (default) means everything is kept and actual thresholding happens
+// only for a positive value.
+//
+// In the following shapes, `nnz` is the count after taking `thresh` into account.
+//
+// Arguments:
+//	a_indices: 2-D.  The `indices` of the first `SparseTensor`, size `[nnz, ndims]` Matrix.
+//	a_values: 1-D.  The `values` of the first `SparseTensor`, size `[nnz]` Vector.
+//	a_shape: 1-D.  The `shape` of the first `SparseTensor`, size `[ndims]` Vector.
+//	b_indices: 2-D.  The `indices` of the second `SparseTensor`, size `[nnz, ndims]` Matrix.
+//	b_values: 1-D.  The `values` of the second `SparseTensor`, size `[nnz]` Vector.
+//	b_shape: 1-D.  The `shape` of the second `SparseTensor`, size `[ndims]` Vector.
+//	thresh: 0-D.  The magnitude threshold that determines if an output value/index
+// pair takes space.
+func SparseAdd(scope *Scope, a_indices tf.Output, a_values tf.Output, a_shape tf.Output, b_indices tf.Output, b_values tf.Output, b_shape tf.Output, thresh tf.Output) (sum_indices tf.Output, sum_values tf.Output, sum_shape tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "SparseAdd",
+		Input: []tf.Input{
+			a_indices, a_values, a_shape, b_indices, b_values, b_shape, thresh,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1), op.Output(2)
+}
+
+// Computes the gradient of the sigmoid of `x` wrt its input.
+//
+// Specifically, `grad = dy * y * (1 - y)`, where `y = sigmoid(x)`, and
+// `dy` is the corresponding input gradient.
+func SigmoidGrad(scope *Scope, y tf.Output, dy tf.Output) (z tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "SigmoidGrad",
+		Input: []tf.Input{
+			y, dy,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Subtracts a value from the current value of a variable.
+//
+// Any ReadVariableOp which depends directly or indirectly on this assign is
+// guaranteed to see the incremented value or a subsequent newer one.
 //
 // Outputs the incremented value, which can be used to totally order the
 // increments to this variable.
@@ -16263,80 +16450,6 @@ func BitwiseXor(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
 	return op.Output(0)
 }
 
-// Compute the polygamma function \\(\psi^{(n)}(x)\\).
-//
-// The polygamma function is defined as:
-//
-//
-// \\(\psi^{(n)}(x) = \frac{d^n}{dx^n} \psi(x)\\)
-//
-// where \\(\psi(x)\\) is the digamma function.
-func Polygamma(scope *Scope, a tf.Output, x tf.Output) (z tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "Polygamma",
-		Input: []tf.Input{
-			a, x,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Computes the grayscale dilation of 4-D `input` and 3-D `filter` tensors.
-//
-// The `input` tensor has shape `[batch, in_height, in_width, depth]` and the
-// `filter` tensor has shape `[filter_height, filter_width, depth]`, i.e., each
-// input channel is processed independently of the others with its own structuring
-// function. The `output` tensor has shape
-// `[batch, out_height, out_width, depth]`. The spatial dimensions of the output
-// tensor depend on the `padding` algorithm. We currently only support the default
-// "NHWC" `data_format`.
-//
-// In detail, the grayscale morphological 2-D dilation is the max-sum correlation
-// (for consistency with `conv2d`, we use unmirrored filters):
-//
-//     output[b, y, x, c] =
-//        max_{dy, dx} input[b,
-//                           strides[1] * y + rates[1] * dy,
-//                           strides[2] * x + rates[2] * dx,
-//                           c] +
-//                     filter[dy, dx, c]
-//
-// Max-pooling is a special case when the filter has size equal to the pooling
-// kernel size and contains all zeros.
-//
-// Note on duality: The dilation of `input` by the `filter` is equal to the
-// negation of the erosion of `-input` by the reflected `filter`.
-//
-// Arguments:
-//	input: 4-D with shape `[batch, in_height, in_width, depth]`.
-//	filter: 3-D with shape `[filter_height, filter_width, depth]`.
-//	strides: The stride of the sliding window for each dimension of the input
-// tensor. Must be: `[1, stride_height, stride_width, 1]`.
-//	rates: The input stride for atrous morphological dilation. Must be:
-// `[1, rate_height, rate_width, 1]`.
-//	padding: The type of padding algorithm to use.
-//
-// Returns 4-D with shape `[batch, out_height, out_width, depth]`.
-func Dilation2D(scope *Scope, input tf.Output, filter tf.Output, strides []int64, rates []int64, padding string) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"strides": strides, "rates": rates, "padding": padding}
-	opspec := tf.OpSpec{
-		Type: "Dilation2D",
-		Input: []tf.Input{
-			input, filter,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
 // Creates a summary file writer accessible by the given resource handle.
 //
 // Arguments:
@@ -16697,31 +16810,71 @@ func RealTout(value tf.DataType) RealAttr {
 
 // Returns the real part of a complex number.
 //
-// Given a tensor `input` of complex numbers, this operation returns a tensor of
-// type `float` that is the real part of each element in `input`. All elements in
-// `input` must be complex numbers of the form \\(a + bj\\), where *a* is the real
-//  part returned by this operation and *b* is the imaginary part.
+// Given a tensor `input` of complex numbers, this operation returns a tensor of
+// type `float` that is the real part of each element in `input`. All elements in
+// `input` must be complex numbers of the form \\(a + bj\\), where *a* is the real
+//  part returned by this operation and *b* is the imaginary part.
+//
+// For example:
+//
+// ```
+// # tensor 'input' is [-2.25 + 4.75j, 3.25 + 5.75j]
+// tf.real(input) ==> [-2.25, 3.25]
+// ```
+func Real(scope *Scope, input tf.Output, optional ...RealAttr) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "Real",
+		Input: []tf.Input{
+			input,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// 2D real-valued fast Fourier transform.
+//
+// Computes the 2-dimensional discrete Fourier transform of a real-valued signal
+// over the inner-most 2 dimensions of `input`.
+//
+// Since the DFT of a real signal is Hermitian-symmetric, `RFFT2D` only returns the
+// `fft_length / 2 + 1` unique components of the FFT for the inner-most dimension
+// of `output`: the zero-frequency term, followed by the `fft_length / 2`
+// positive-frequency terms.
+//
+// Along each axis `RFFT2D` is computed on, if `fft_length` is smaller than the
+// corresponding dimension of `input`, the dimension is cropped. If it is larger,
+// the dimension is padded with zeros.
+//
+// Arguments:
+//	input: A float32 tensor.
+//	fft_length: An int32 tensor of shape [2]. The FFT length for each dimension.
 //
-// For example:
+// Returns A complex64 tensor of the same rank as `input`. The inner-most 2
+//   dimensions of `input` are replaced with their 2D Fourier transform. The
+//   inner-most dimension contains `fft_length / 2 + 1` unique frequency
+//   components.
 //
-// ```
-// # tensor 'input' is [-2.25 + 4.75j, 3.25 + 5.75j]
-// tf.real(input) ==> [-2.25, 3.25]
-// ```
-func Real(scope *Scope, input tf.Output, optional ...RealAttr) (output tf.Output) {
+// @compatibility(numpy)
+// Equivalent to np.fft.rfft2
+// @end_compatibility
+func RFFT2D(scope *Scope, input tf.Output, fft_length tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "Real",
+		Type: "RFFT2D",
 		Input: []tf.Input{
-			input,
+			input, fft_length,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
@@ -17139,117 +17292,6 @@ func AvgPool3DGrad(scope *Scope, orig_input_shape tf.Output, grad tf.Output, ksi
 	return op.Output(0)
 }
 
-// OrderedMapPeekAttr is an optional argument to OrderedMapPeek.
-type OrderedMapPeekAttr func(optionalAttr)
-
-// OrderedMapPeekCapacity sets the optional capacity attribute to value.
-// If not specified, defaults to 0
-//
-// REQUIRES: value >= 0
-func OrderedMapPeekCapacity(value int64) OrderedMapPeekAttr {
-	return func(m optionalAttr) {
-		m["capacity"] = value
-	}
-}
-
-// OrderedMapPeekMemoryLimit sets the optional memory_limit attribute to value.
-// If not specified, defaults to 0
-//
-// REQUIRES: value >= 0
-func OrderedMapPeekMemoryLimit(value int64) OrderedMapPeekAttr {
-	return func(m optionalAttr) {
-		m["memory_limit"] = value
-	}
-}
-
-// OrderedMapPeekContainer sets the optional container attribute to value.
-// If not specified, defaults to ""
-func OrderedMapPeekContainer(value string) OrderedMapPeekAttr {
-	return func(m optionalAttr) {
-		m["container"] = value
-	}
-}
-
-// OrderedMapPeekSharedName sets the optional shared_name attribute to value.
-// If not specified, defaults to ""
-func OrderedMapPeekSharedName(value string) OrderedMapPeekAttr {
-	return func(m optionalAttr) {
-		m["shared_name"] = value
-	}
-}
-
-// Op peeks at the values at the specified key.  If the
-//
-// underlying container does not contain this key
-// this op will block until it does.   This Op is optimized for
-// performance.
-func OrderedMapPeek(scope *Scope, key tf.Output, indices tf.Output, dtypes []tf.DataType, optional ...OrderedMapPeekAttr) (values []tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"dtypes": dtypes}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "OrderedMapPeek",
-		Input: []tf.Input{
-			key, indices,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	if scope.Err() != nil {
-		return
-	}
-	var idx int
-	var err error
-	if values, idx, err = makeOutputList(op, idx, "values"); err != nil {
-		scope.UpdateErr("OrderedMapPeek", err)
-		return
-	}
-	return values
-}
-
-// Adds two `SparseTensor` objects to produce another `SparseTensor`.
-//
-// The input `SparseTensor` objects' indices are assumed ordered in standard
-// lexicographic order.  If this is not the case, before this step run
-// `SparseReorder` to restore index ordering.
-//
-// By default, if two values sum to zero at some index, the output `SparseTensor`
-// would still include that particular location in its index, storing a zero in the
-// corresponding value slot.  To override this, callers can specify `thresh`,
-// indicating that if the sum has a magnitude strictly smaller than `thresh`, its
-// corresponding value and index would then not be included.  In particular,
-// `thresh == 0` (default) means everything is kept and actual thresholding happens
-// only for a positive value.
-//
-// In the following shapes, `nnz` is the count after taking `thresh` into account.
-//
-// Arguments:
-//	a_indices: 2-D.  The `indices` of the first `SparseTensor`, size `[nnz, ndims]` Matrix.
-//	a_values: 1-D.  The `values` of the first `SparseTensor`, size `[nnz]` Vector.
-//	a_shape: 1-D.  The `shape` of the first `SparseTensor`, size `[ndims]` Vector.
-//	b_indices: 2-D.  The `indices` of the second `SparseTensor`, size `[nnz, ndims]` Matrix.
-//	b_values: 1-D.  The `values` of the second `SparseTensor`, size `[nnz]` Vector.
-//	b_shape: 1-D.  The `shape` of the second `SparseTensor`, size `[ndims]` Vector.
-//	thresh: 0-D.  The magnitude threshold that determines if an output value/index
-// pair takes space.
-func SparseAdd(scope *Scope, a_indices tf.Output, a_values tf.Output, a_shape tf.Output, b_indices tf.Output, b_values tf.Output, b_shape tf.Output, thresh tf.Output) (sum_indices tf.Output, sum_values tf.Output, sum_shape tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "SparseAdd",
-		Input: []tf.Input{
-			a_indices, a_values, a_shape, b_indices, b_values, b_shape, thresh,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
-}
-
 // Creates a dataset that batches `batch_size` elements from `input_dataset`.
 //
 // Arguments:
@@ -20337,6 +20379,80 @@ func Sub(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
 	return op.Output(0)
 }
 
+// Compute the polygamma function \\(\psi^{(n)}(x)\\).
+//
+// The polygamma function is defined as:
+//
+//
+// \\(\psi^{(n)}(x) = \frac{d^n}{dx^n} \psi(x)\\)
+//
+// where \\(\psi(x)\\) is the digamma function.
+func Polygamma(scope *Scope, a tf.Output, x tf.Output) (z tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "Polygamma",
+		Input: []tf.Input{
+			a, x,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Computes the grayscale dilation of 4-D `input` and 3-D `filter` tensors.
+//
+// The `input` tensor has shape `[batch, in_height, in_width, depth]` and the
+// `filter` tensor has shape `[filter_height, filter_width, depth]`, i.e., each
+// input channel is processed independently of the others with its own structuring
+// function. The `output` tensor has shape
+// `[batch, out_height, out_width, depth]`. The spatial dimensions of the output
+// tensor depend on the `padding` algorithm. We currently only support the default
+// "NHWC" `data_format`.
+//
+// In detail, the grayscale morphological 2-D dilation is the max-sum correlation
+// (for consistency with `conv2d`, we use unmirrored filters):
+//
+//     output[b, y, x, c] =
+//        max_{dy, dx} input[b,
+//                           strides[1] * y + rates[1] * dy,
+//                           strides[2] * x + rates[2] * dx,
+//                           c] +
+//                     filter[dy, dx, c]
+//
+// Max-pooling is a special case when the filter has size equal to the pooling
+// kernel size and contains all zeros.
+//
+// Note on duality: The dilation of `input` by the `filter` is equal to the
+// negation of the erosion of `-input` by the reflected `filter`.
+//
+// Arguments:
+//	input: 4-D with shape `[batch, in_height, in_width, depth]`.
+//	filter: 3-D with shape `[filter_height, filter_width, depth]`.
+//	strides: The stride of the sliding window for each dimension of the input
+// tensor. Must be: `[1, stride_height, stride_width, 1]`.
+//	rates: The input stride for atrous morphological dilation. Must be:
+// `[1, rate_height, rate_width, 1]`.
+//	padding: The type of padding algorithm to use.
+//
+// Returns 4-D with shape `[batch, out_height, out_width, depth]`.
+func Dilation2D(scope *Scope, input tf.Output, filter tf.Output, strides []int64, rates []int64, padding string) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"strides": strides, "rates": rates, "padding": padding}
+	opspec := tf.OpSpec{
+		Type: "Dilation2D",
+		Input: []tf.Input{
+			input, filter,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // AudioSpectrogramAttr is an optional argument to AudioSpectrogram.
 type AudioSpectrogramAttr func(optionalAttr)
 
@@ -23117,46 +23233,6 @@ func Erfc(scope *Scope, x tf.Output) (y tf.Output) {
 	return op.Output(0)
 }
 
-// 2D real-valued fast Fourier transform.
-//
-// Computes the 2-dimensional discrete Fourier transform of a real-valued signal
-// over the inner-most 2 dimensions of `input`.
-//
-// Since the DFT of a real signal is Hermitian-symmetric, `RFFT2D` only returns the
-// `fft_length / 2 + 1` unique components of the FFT for the inner-most dimension
-// of `output`: the zero-frequency term, followed by the `fft_length / 2`
-// positive-frequency terms.
-//
-// Along each axis `RFFT2D` is computed on, if `fft_length` is smaller than the
-// corresponding dimension of `input`, the dimension is cropped. If it is larger,
-// the dimension is padded with zeros.
-//
-// Arguments:
-//	input: A float32 tensor.
-//	fft_length: An int32 tensor of shape [2]. The FFT length for each dimension.
-//
-// Returns A complex64 tensor of the same rank as `input`. The inner-most 2
-//   dimensions of `input` are replaced with their 2D Fourier transform. The
-//   inner-most dimension contains `fft_length / 2 + 1` unique frequency
-//   components.
-//
-// @compatibility(numpy)
-// Equivalent to np.fft.rfft2
-// @end_compatibility
-func RFFT2D(scope *Scope, input tf.Output, fft_length tf.Output) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "RFFT2D",
-		Input: []tf.Input{
-			input, fft_length,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
 // Computes sin of x element-wise.
 func Sin(scope *Scope, x tf.Output) (y tf.Output) {
 	if scope.Err() != nil {
-- 
GitLab


From f6b15b08bbedc500549b0793b236bc90289d07dc Mon Sep 17 00:00:00 2001
From: RJ Ryan <rjryan@google.com>
Date: Wed, 4 Oct 2017 23:33:04 -0700
Subject: [PATCH 050/909] Update the tf.contrib.signal guide to include
 guidance on computing Mel spectrograms and MFCCs.

PiperOrigin-RevId: 171113759
---
 .../api_guides/python/contrib.signal.md       | 127 +++++++++++++-----
 1 file changed, 93 insertions(+), 34 deletions(-)

diff --git a/tensorflow/docs_src/api_guides/python/contrib.signal.md b/tensorflow/docs_src/api_guides/python/contrib.signal.md
index c16c5cb649..85ef3ad134 100644
--- a/tensorflow/docs_src/api_guides/python/contrib.signal.md
+++ b/tensorflow/docs_src/api_guides/python/contrib.signal.md
@@ -1,16 +1,17 @@
 # Signal Processing (contrib)
 [TOC]
 
-@{tf.contrib.signal} is a module for signal processing primitives. All 
-operations have GPU support and are differentiable.
+@{tf.contrib.signal} is a module for signal processing primitives. All
+operations have GPU support and are differentiable. This module is especially
+helpful for building TensorFlow models that process or generate audio, though
+the techniques are useful in many domains.
 
-# Common Tasks
+## Framing variable length sequences
 
-## Framing variable length sequences:
-
-When dealing with variable length signals (e.g. audio) it is common to
-"frame" them into multiple fixed length, potentially overlapping windows.
-@{tf.contrib.signal.frame} does exactly this. For example:
+When dealing with variable length signals (e.g. audio) it is common to "frame"
+them into multiple fixed length windows. These windows can overlap if the 'step'
+of the frame is less than the frame length. @{tf.contrib.signal.frame} does
+exactly this. For example:
 
 ```python
 # A batch of float32 time-domain signals in the range [-1, 1] with shape
@@ -18,8 +19,9 @@ When dealing with variable length signals (e.g. audio) it is common to
 signals = tf.placeholder(tf.float32, [None, None])
 
 # Compute a [batch_size, ?, 128] tensor of fixed length, overlapping windows
-# where each window overlaps the previous by 50%.
-frames = tf.contrib.signal.frame(signals, frame_length=128, frame_step=64)
+# where each window overlaps the previous by 75% (frame_length - frame_step
+# samples of overlap).
+frames = tf.contrib.signal.frame(signals, frame_length=128, frame_step=32)
 ```
 
 The `axis` parameter to @{tf.contrib.signal.frame} allows you to frame tensors
@@ -27,54 +29,52 @@ with inner structure (e.g. a spectrogram):
 
 ```python
 # `magnitude_spectrograms` is a [batch_size, ?, 127] tensor of spectrograms. We
-# would like to produce overlapping fixed-size spectrogram patches e.g. for use
-# in a situation where a fixed size input is needed.
+# would like to produce overlapping fixed-size spectrogram patches; for example,
+# for use in a situation where a fixed size input is needed.
 magnitude_spectrograms = tf.abs(tf.contrib.signal.stft(
-    signals, frame_length=256, frame_step=128, fft_length=256))
+    signals, frame_length=256, frame_step=64, fft_length=256))
 
-# `spectrogram_patches` is a [batch_size, ?, 64, 127] tensor containing a 
+# `spectrogram_patches` is a [batch_size, ?, 64, 127] tensor containing a
 # variable number of [64, 127] spectrogram patches per batch item.
 spectrogram_patches = tf.contrib.signal.frame(
-    magnitude_spectrograms, frame_length=64, frame_step=32, axis=1)
+    magnitude_spectrograms, frame_length=64, frame_step=16, axis=1)
 ```
 
-## Reconstructing framed sequences and applying a tapering window:
+## Reconstructing framed sequences and applying a tapering window
 
 @{tf.contrib.signal.overlap_and_add} can be used to reconstruct a signal from a
-framed representation produced in the above example.
+framed representation. For example, the following code reconstructs the signal
+produced in the preceding example:
 
 ```python
 # Reconstructs `signals` from `frames` produced in the above example. However,
 # the magnitude of `reconstructed_signals` will be greater than `signals`.
-reconstructed_signals = tf.contrib.signal.overlap_and_add(frames, frame_step=64)
+reconstructed_signals = tf.contrib.signal.overlap_and_add(frames, frame_step=32)
 ```
 
-Note that because `frame_step` is 50% of `frame_length` in the above example,
+Note that because `frame_step` is 25% of `frame_length` in the above example,
 the resulting reconstruction will have a greater magnitude than the original
-`signals`.
-
-To compensate for this, we can use a tapering window function. If the
+`signals`. To compensate for this, we can use a tapering window function. If the
 window function satisfies the Constant Overlap-Add (COLA) property for the given
 frame step, then it will recover the original `signals`.
 
 @{tf.contrib.signal.hamming_window} and @{tf.contrib.signal.hann_window} both
-satisfy the COLA property for a 50% overlap.
+satisfy the COLA property for a 75% overlap.
 
 ```python
 frame_length = 128
-frame_step = 64
+frame_step = 32
 windowed_frames = frames * tf.contrib.signal.hann_window(frame_length)
 reconstructed_signals = tf.contrib.signal.overlap_and_add(
     windowed_frames, frame_step)
 ```
 
-## Computing spectrograms:
+## Computing spectrograms
 
 A spectrogram is a time-frequency decomposition of a signal that indicates its
-frequency content over time. There are many variants on how to compute a
-spectrogram, but the most common approach is by taking the magnitude of the
-[Short-time Fourier Transform][stft] (STFT), which can be computed with
-@{tf.contrib.signal.stft}.
+frequency content over time. The most common approach to computing spectrograms
+is to take the magnitude of the [Short-time Fourier Transform][stft] (STFT),
+which @{tf.contrib.signal.stft} can compute as follows:
 
 ```python
 # A batch of float32 time-domain signals in the range [-1, 1] with shape
@@ -82,7 +82,7 @@ spectrogram, but the most common approach is by taking the magnitude of the
 signals = tf.placeholder(tf.float32, [None, None])
 
 # `stfts` is a complex64 Tensor representing the Short-time Fourier Transform of
-# each signal in `signals`. Its shape is [batch_size, ?, fft_unique_bins] 
+# each signal in `signals`. Its shape is [batch_size, ?, fft_unique_bins]
 # where fft_unique_bins = fft_length // 2 + 1 = 513.
 stfts = tf.contrib.signal.stft(signals, frame_length=1024, frame_step=512,
                                fft_length=1024)
@@ -96,18 +96,77 @@ power_spectrograms = tf.real(stfts * tf.conj(stfts))
 magnitude_spectrograms = tf.abs(stfts)
 ```
 
-## Logarithmic compression:
+You may use a power spectrogram or a magnitude spectrogram; each has its
+advantages. Note that if you apply logarithmic compression, the power
+spectrogram and magnitude spectrogram will differ by a factor of 2.
+
+## Logarithmic compression
 
 It is common practice to apply a compressive nonlinearity such as a logarithm or
-power-law compression to spectrograms.
+power-law compression to spectrograms. This helps to balance the importance of
+detail in low and high energy regions of the spectrum, which more closely
+matches human auditory sensitivity.
 
-When compressing with a logarithm, it's a good idea to use a stabilizing offset 
+When compressing with a logarithm, it's a good idea to use a stabilizing offset
 to avoid high dynamic ranges caused by the singularity at zero.
 
 ```python
 log_offset = 1e-6
 log_magnitude_spectrograms = tf.log(magnitude_spectrograms + log_offset)
-log_power_spectrograms = tf.log(power_spectrograms + log_offset)
+```
+
+## Computing log-mel spectrograms
+
+When working with spectral representations of audio, the [mel scale][mel] is a
+common reweighting of the frequency dimension, which results in a
+lower-dimensional and more perceptually-relevant representation of the audio.
+
+@{tf.contrib.signal.linear_to_mel_weight_matrix} produces a matrix you can use
+to convert a spectrogram to the mel scale.
+
+```python
+# Warp the linear-scale, magnitude spectrograms into the mel-scale.
+num_spectrogram_bins = magnitude_spectrograms.shape[-1].value
+lower_edge_hertz, upper_edge_hertz, num_mel_bins = 80.0, 7600.0, 64
+linear_to_mel_weight_matrix = tf.contrib.signal.linear_to_mel_weight_matrix(
+  num_mel_bins, num_spectrogram_bins, sample_rate, lower_edge_hertz,
+  upper_edge_hertz)
+mel_spectrograms = tf.tensordot(
+  magnitude_spectrograms, linear_to_mel_weight_matrix, 1)
+# Note: Shape inference for `tf.tensordot` does not currently handle this case.
+mel_spectrograms.set_shape(magnitude_spectrograms.shape[:-1].concatenate(
+  linear_to_mel_weight_matrix.shape[-1:]))
+```
+
+If desired, compress the mel spectrogram magnitudes. For example, you may use
+logarithmic compression (as discussed in the previous section).
+
+Order matters! Compressing the spectrogram magnitudes after
+reweighting the frequencies is different from reweighting the compressed
+spectrogram magnitudes. According to the perceptual justification of the mel
+scale, conversion from linear scale entails summing intensity or energy among
+adjacent bands, i.e. it should be applied before logarithmic compression. Taking
+the weighted sum of log-compressed values amounts to multiplying the
+pre-logarithm values, which rarely, if ever, makes sense.
+
+```python
+log_offset = 1e-6
+log_mel_spectrograms = tf.log(mel_spectrograms + log_offset)
+```
+
+## Computing Mel-Frequency Cepstral Coefficients (MFCCs)
+
+Call @{tf.contrib.signal.mfccs_from_log_mel_spectrograms} to compute
+[MFCCs][mfcc] from log-magnitude, mel-scale spectrograms (as computed in the
+preceding example):
+
+```python
+num_mfccs = 13
+# Keep the first `num_mfccs` MFCCs.
+mfccs = tf.contrib.signal.mfccs_from_log_mel_spectrograms(
+    log_mel_spectrograms)[..., :num_mfccs]
 ```
 
 [stft]: https://en.wikipedia.org/wiki/Short-time_Fourier_transform
+[mel]: https://en.wikipedia.org/wiki/Mel_scale
+[mfcc]: https://en.wikipedia.org/wiki/Mel-frequency_cepstrum
-- 
GitLab


From 220515bffdf1df5379a7f8921f5a12deb2e0dee7 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 5 Oct 2017 03:46:13 -0700
Subject: [PATCH 051/909] Replace owning raw pointers with unique pointers

PiperOrigin-RevId: 171132628
---
 tensorflow/c/checkpoint_reader.cc | 26 ++++++++++----------------
 tensorflow/c/checkpoint_reader.h  | 15 ++++++++-------
 2 files changed, 18 insertions(+), 23 deletions(-)

diff --git a/tensorflow/c/checkpoint_reader.cc b/tensorflow/c/checkpoint_reader.cc
index e7b9bca5b5..fc86e92f3b 100644
--- a/tensorflow/c/checkpoint_reader.cc
+++ b/tensorflow/c/checkpoint_reader.cc
@@ -24,7 +24,6 @@ limitations under the License.
 #include "tensorflow/core/util/saved_tensor_slice_util.h"
 
 namespace tensorflow {
-
 namespace checkpoint {
 
 class TensorSliceReader;
@@ -37,30 +36,24 @@ CheckpointReader::CheckpointReader(const string& filename,
   std::vector<string> v2_path;
   if (Env::Default()->GetMatchingPaths(MetaFilename(filename), &v2_path).ok() &&
       !v2_path.empty()) {
-    v2_reader_ =
-        new BundleReader(Env::Default(), filename /* prefix to a V2 ckpt */);
+    v2_reader_.reset(
+        new BundleReader(Env::Default(), filename /* prefix to a V2 ckpt */));
     if (!v2_reader_->status().ok()) {
       Set_TF_Status_from_Status(out_status, v2_reader_->status());
       return;
     }
     var_to_shape_map_ptr_ = BuildV2VarToShapeMap();
   } else {
-    reader_ = new TensorSliceReader(filename);
+    reader_.reset(new TensorSliceReader(filename));
     if (!reader_->status().ok()) {
       Set_TF_Status_from_Status(out_status, reader_->status());
       return;
     }
-    var_to_shape_map_ptr_ =
-        new TensorSliceReader::VarToShapeMap(reader_->GetVariableToShapeMap());
+    var_to_shape_map_ptr_.reset(
+        new TensorSliceReader::VarToShapeMap(reader_->GetVariableToShapeMap()));
   }
 }
 
-CheckpointReader::~CheckpointReader() {
-  delete var_to_shape_map_ptr_;
-  delete reader_;
-  delete v2_reader_;
-}
-
 bool CheckpointReader::HasTensor(const string& name) const {
   if (reader_ != nullptr) {
     return reader_->HasTensor(name, nullptr, nullptr);
@@ -100,7 +93,8 @@ void CheckpointReader::GetTensor(
   }
 }
 
-TensorSliceReader::VarToShapeMap* CheckpointReader::BuildV2VarToShapeMap() {
+std::unique_ptr<TensorSliceReader::VarToShapeMap>
+CheckpointReader::BuildV2VarToShapeMap() {
   CHECK(v2_reader_ != nullptr);
   CHECK(v2_reader_->status().ok());
 
@@ -123,8 +117,8 @@ TensorSliceReader::VarToShapeMap* CheckpointReader::BuildV2VarToShapeMap() {
   }
 
   // Second pass: adds the entries, ignoring the filtered keys.
-  TensorSliceReader::VarToShapeMap* var_to_shape_map =
-      new TensorSliceReader::VarToShapeMap;
+  std::unique_ptr<TensorSliceReader::VarToShapeMap> var_to_shape_map(
+      new TensorSliceReader::VarToShapeMap);
   v2_reader_->Seek(kHeaderEntryKey);
   for (v2_reader_->Next(); v2_reader_->Valid(); v2_reader_->Next()) {
     if (filtered_keys.count(v2_reader_->key().ToString()) > 0) continue;
@@ -134,7 +128,7 @@ TensorSliceReader::VarToShapeMap* CheckpointReader::BuildV2VarToShapeMap() {
     (*var_to_shape_map)[v2_reader_->key().ToString()] =
         TensorShape(entry.shape());
   }
-  return var_to_shape_map;  // Owned by caller.
+  return var_to_shape_map;
 }
 
 }  // namespace checkpoint
diff --git a/tensorflow/c/checkpoint_reader.h b/tensorflow/c/checkpoint_reader.h
index 1124416380..470c8d1e10 100644
--- a/tensorflow/c/checkpoint_reader.h
+++ b/tensorflow/c/checkpoint_reader.h
@@ -16,6 +16,9 @@ limitations under the License.
 #ifndef TENSORFLOW_C_CHECKPOINT_READER_H
 #define TENSORFLOW_C_CHECKPOINT_READER_H
 
+#include <memory>
+#include <string>
+
 #include "tensorflow/c/tf_status_helper.h"
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/lib/core/status.h"
@@ -24,7 +27,6 @@ limitations under the License.
 #include "tensorflow/core/util/tensor_slice_reader.h"
 
 namespace tensorflow {
-
 namespace checkpoint {
 
 class TensorSliceReader;
@@ -38,7 +40,6 @@ class TensorSliceReader;
 class CheckpointReader {
  public:
   CheckpointReader(const string& filepattern, TF_Status* out_status);
-  ~CheckpointReader();
 
   bool HasTensor(const string& name) const;
   const string DebugString() const;
@@ -56,12 +57,12 @@ class CheckpointReader {
  private:
   // Uses "v2_reader_" to build a "var name -> shape" map; owned by caller.
   // REQUIRES: "v2_reader_ != nullptr && v2_reader_.status().ok()".
-  TensorSliceReader::VarToShapeMap* BuildV2VarToShapeMap();
+  std::unique_ptr<TensorSliceReader::VarToShapeMap> BuildV2VarToShapeMap();
 
-  // Invariant: exactly one of "reader_" and "v2_reader_" is non-nullptr.
-  TensorSliceReader* reader_;                               // Owned.
-  BundleReader* v2_reader_;                                 // Owned.
-  TensorSliceReader::VarToShapeMap* var_to_shape_map_ptr_;  // Owned.
+  // Invariant: exactly one of "reader_" and "v2_reader_" is non-null.
+  std::unique_ptr<TensorSliceReader> reader_;
+  std::unique_ptr<BundleReader> v2_reader_;
+  std::unique_ptr<TensorSliceReader::VarToShapeMap> var_to_shape_map_ptr_;
 
   TF_DISALLOW_COPY_AND_ASSIGN(CheckpointReader);
 };
-- 
GitLab


From a8c5d5fe011e796593d20c74d8b927c014a27c89 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 5 Oct 2017 06:57:46 -0700
Subject: [PATCH 052/909] Expose data type information in checkpoint reader.

PiperOrigin-RevId: 171147196
---
 tensorflow/c/checkpoint_reader.cc             | 40 ++++++++++++++-----
 tensorflow/c/checkpoint_reader.h              | 17 ++++++--
 tensorflow/core/util/tensor_slice_reader.cc   | 13 +++++-
 tensorflow/core/util/tensor_slice_reader.h    |  5 +++
 tensorflow/python/util/py_checkpoint_reader.i | 38 ++++++++++++++++++
 5 files changed, 98 insertions(+), 15 deletions(-)

diff --git a/tensorflow/c/checkpoint_reader.cc b/tensorflow/c/checkpoint_reader.cc
index fc86e92f3b..b1f7bdaa54 100644
--- a/tensorflow/c/checkpoint_reader.cc
+++ b/tensorflow/c/checkpoint_reader.cc
@@ -16,6 +16,7 @@ limitations under the License.
 #include "tensorflow/c/checkpoint_reader.h"
 
 #include <unordered_set>
+#include <utility>
 
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/core/stringpiece.h"
@@ -30,7 +31,10 @@ class TensorSliceReader;
 
 CheckpointReader::CheckpointReader(const string& filename,
                                    TF_Status* out_status)
-    : reader_(nullptr), v2_reader_(nullptr), var_to_shape_map_ptr_(nullptr) {
+    : reader_(nullptr),
+      v2_reader_(nullptr),
+      var_to_shape_map_(nullptr),
+      var_to_data_type_map_(nullptr) {
   // Depending on whether this is a V2 ckpt, initializes "reader_" or
   // "v2_reader_".
   std::vector<string> v2_path;
@@ -42,15 +46,19 @@ CheckpointReader::CheckpointReader(const string& filename,
       Set_TF_Status_from_Status(out_status, v2_reader_->status());
       return;
     }
-    var_to_shape_map_ptr_ = BuildV2VarToShapeMap();
+    auto result = BuildV2VarMaps();
+    var_to_shape_map_.swap(result.first);
+    var_to_data_type_map_.swap(result.second);
   } else {
     reader_.reset(new TensorSliceReader(filename));
     if (!reader_->status().ok()) {
       Set_TF_Status_from_Status(out_status, reader_->status());
       return;
     }
-    var_to_shape_map_ptr_.reset(
+    var_to_shape_map_.reset(
         new TensorSliceReader::VarToShapeMap(reader_->GetVariableToShapeMap()));
+    var_to_data_type_map_.reset(new TensorSliceReader::VarToDataTypeMap(
+        reader_->GetVariableToDataTypeMap()));
   }
 }
 
@@ -63,8 +71,14 @@ bool CheckpointReader::HasTensor(const string& name) const {
 
 const TensorSliceReader::VarToShapeMap&
 CheckpointReader::GetVariableToShapeMap() const {
-  CHECK(var_to_shape_map_ptr_);
-  return *var_to_shape_map_ptr_;
+  CHECK(var_to_shape_map_);
+  return *var_to_shape_map_;
+}
+
+const TensorSliceReader::VarToDataTypeMap&
+CheckpointReader::GetVariableToDataTypeMap() const {
+  CHECK(var_to_data_type_map_);
+  return *var_to_data_type_map_;
 }
 
 const string CheckpointReader::DebugString() const {
@@ -93,8 +107,9 @@ void CheckpointReader::GetTensor(
   }
 }
 
-std::unique_ptr<TensorSliceReader::VarToShapeMap>
-CheckpointReader::BuildV2VarToShapeMap() {
+std::pair<std::unique_ptr<TensorSliceReader::VarToShapeMap>,
+          std::unique_ptr<TensorSliceReader::VarToDataTypeMap>>
+CheckpointReader::BuildV2VarMaps() {
   CHECK(v2_reader_ != nullptr);
   CHECK(v2_reader_->status().ok());
 
@@ -119,16 +134,21 @@ CheckpointReader::BuildV2VarToShapeMap() {
   // Second pass: adds the entries, ignoring the filtered keys.
   std::unique_ptr<TensorSliceReader::VarToShapeMap> var_to_shape_map(
       new TensorSliceReader::VarToShapeMap);
+  std::unique_ptr<TensorSliceReader::VarToDataTypeMap> var_to_data_type_map(
+      new TensorSliceReader::VarToDataTypeMap);
   v2_reader_->Seek(kHeaderEntryKey);
   for (v2_reader_->Next(); v2_reader_->Valid(); v2_reader_->Next()) {
     if (filtered_keys.count(v2_reader_->key().ToString()) > 0) continue;
     CHECK(entry.ParseFromArray(v2_reader_->value().data(),
                                v2_reader_->value().size()))
         << entry.InitializationErrorString();
-    (*var_to_shape_map)[v2_reader_->key().ToString()] =
-        TensorShape(entry.shape());
+    string key = v2_reader_->key().ToString();
+    (*var_to_shape_map)[key] = TensorShape(entry.shape());
+    (*var_to_data_type_map)[key] = DataType(entry.dtype());
   }
-  return var_to_shape_map;
+  // The returned pointers are owned by the caller.
+  return std::make_pair(std::move(var_to_shape_map),
+                        std::move(var_to_data_type_map));
 }
 
 }  // namespace checkpoint
diff --git a/tensorflow/c/checkpoint_reader.h b/tensorflow/c/checkpoint_reader.h
index 470c8d1e10..4de1300a7f 100644
--- a/tensorflow/c/checkpoint_reader.h
+++ b/tensorflow/c/checkpoint_reader.h
@@ -44,10 +44,14 @@ class CheckpointReader {
   bool HasTensor(const string& name) const;
   const string DebugString() const;
 
-  // Returns a map from variable names to its shape.  Slices of a partitioned
+  // Returns a map from variable names to their shapes.  Slices of a partitioned
   // tensor are combined into a single entry.
   const TensorSliceReader::VarToShapeMap& GetVariableToShapeMap() const;
 
+  // Returns a map from variable names to their data types.  Slices of a
+  // partitioned tensor are combined into a single entry.
+  const TensorSliceReader::VarToDataTypeMap& GetVariableToDataTypeMap() const;
+
   // Attempts to look up the tensor named "name" and stores the found result in
   // "out_tensor".
   void GetTensor(const string& name,
@@ -55,14 +59,19 @@ class CheckpointReader {
                  TF_Status* out_status) const;
 
  private:
-  // Uses "v2_reader_" to build a "var name -> shape" map; owned by caller.
+  // Uses "v2_reader_" to build "var name -> shape" and "var name -> data type"
+  // maps; both owned by caller.
   // REQUIRES: "v2_reader_ != nullptr && v2_reader_.status().ok()".
-  std::unique_ptr<TensorSliceReader::VarToShapeMap> BuildV2VarToShapeMap();
+  std::pair<std::unique_ptr<TensorSliceReader::VarToShapeMap>,
+            std::unique_ptr<TensorSliceReader::VarToDataTypeMap> >
+  BuildV2VarMaps();
 
   // Invariant: exactly one of "reader_" and "v2_reader_" is non-null.
   std::unique_ptr<TensorSliceReader> reader_;
   std::unique_ptr<BundleReader> v2_reader_;
-  std::unique_ptr<TensorSliceReader::VarToShapeMap> var_to_shape_map_ptr_;
+
+  std::unique_ptr<TensorSliceReader::VarToShapeMap> var_to_shape_map_;
+  std::unique_ptr<TensorSliceReader::VarToDataTypeMap> var_to_data_type_map_;
 
   TF_DISALLOW_COPY_AND_ASSIGN(CheckpointReader);
 };
diff --git a/tensorflow/core/util/tensor_slice_reader.cc b/tensorflow/core/util/tensor_slice_reader.cc
index cd49034719..c6dda2ec29 100644
--- a/tensorflow/core/util/tensor_slice_reader.cc
+++ b/tensorflow/core/util/tensor_slice_reader.cc
@@ -278,13 +278,24 @@ TensorSliceReader::VarToShapeMap TensorSliceReader::GetVariableToShapeMap()
     const {
   VarToShapeMap name_to_shape;
   if (status().ok()) {
-    for (auto e : Tensors()) {
+    for (auto& e : Tensors()) {
       name_to_shape[e.first] = e.second->shape();
     }
   }
   return name_to_shape;
 }
 
+TensorSliceReader::VarToDataTypeMap
+TensorSliceReader::GetVariableToDataTypeMap() const {
+  VarToDataTypeMap name_to_dtype;
+  if (status().ok()) {
+    for (auto& e : Tensors()) {
+      name_to_dtype[e.first] = e.second->type();
+    }
+  }
+  return name_to_dtype;
+}
+
 const string TensorSliceReader::DebugString() const {
   string shape_str;
   if (status().ok()) {
diff --git a/tensorflow/core/util/tensor_slice_reader.h b/tensorflow/core/util/tensor_slice_reader.h
index 5932d59a15..4bb2b24615 100644
--- a/tensorflow/core/util/tensor_slice_reader.h
+++ b/tensorflow/core/util/tensor_slice_reader.h
@@ -103,9 +103,14 @@ class TensorSliceReader {
                    std::unique_ptr<tensorflow::Tensor>* out_tensor) const;
 
   typedef std::unordered_map<string, TensorShape> VarToShapeMap;
+  typedef std::unordered_map<string, DataType> VarToDataTypeMap;
+
   // Returns a map from tensor name to shape.
   VarToShapeMap GetVariableToShapeMap() const;
 
+  // Returns a map from tensor name to data type.
+  VarToDataTypeMap GetVariableToDataTypeMap() const;
+
   // Returns a string containing names and shapes of all the tensors.
   const string DebugString() const;
 
diff --git a/tensorflow/python/util/py_checkpoint_reader.i b/tensorflow/python/util/py_checkpoint_reader.i
index 1d20f9756f..0cd095d9d9 100644
--- a/tensorflow/python/util/py_checkpoint_reader.i
+++ b/tensorflow/python/util/py_checkpoint_reader.i
@@ -68,6 +68,38 @@ limitations under the License.
   $result = output_map.release();
 }
 
+%typemap(out) const tensorflow::checkpoint::TensorSliceReader::VarToDataTypeMap& {
+  tensorflow::Safe_PyObjectPtr output_map(tensorflow::make_safe(PyDict_New()));
+  for (auto v : *$1) {
+%#if PY_MAJOR_VERSION >= 3
+    tensorflow::Safe_PyObjectPtr key(
+        tensorflow::make_safe(PyUnicode_FromStringAndSize(v.first.c_str(), v.first.size())));
+%#else
+    tensorflow::Safe_PyObjectPtr key(
+        tensorflow::make_safe(PyString_FromStringAndSize(v.first.c_str(), v.first.size())));
+%#endif
+    if (!key) {
+      SWIG_fail;
+    }
+%#if PY_MAJOR_VERSION >= 3
+    tensorflow::Safe_PyObjectPtr value(tensorflow::make_safe(PyLong_FromLong(v.second)));
+%#else
+    tensorflow::Safe_PyObjectPtr value(tensorflow::make_safe(PyInt_FromLong(v.second)));
+%#endif
+    if (!value) {
+      SWIG_fail;
+    }
+    if (PyDict_SetItem(output_map.get(), key.get(), value.get()) == -1) {
+      SWIG_fail;
+    } else {
+      key.release();
+      value.release();
+    }
+  }
+
+  $result = output_map.release();
+}
+
 %{
 static PyObject* CheckpointReader_GetTensor(
       tensorflow::checkpoint::CheckpointReader* reader,
@@ -102,11 +134,17 @@ PyObject* CheckpointReader_GetTensor(
 %unignore tensorflow::checkpoint::CheckpointReader::~CheckpointReader;
 %rename("debug_string") tensorflow::checkpoint::CheckpointReader::DebugString;
 %rename("get_variable_to_shape_map") tensorflow::checkpoint::CheckpointReader::GetVariableToShapeMap;
+%rename("_GetVariableToDataTypeMap") tensorflow::checkpoint::CheckpointReader::GetVariableToDataTypeMap;
 %rename("_HasTensor") tensorflow::checkpoint::CheckpointReader::HasTensor;
 %unignore CheckpointReader_GetTensor;
 
 %extend tensorflow::checkpoint::CheckpointReader {
 %insert("python") %{
+  def get_variable_to_dtype_map(self):
+    from tensorflow.python.framework import dtypes
+    return {name: dtypes.DType(type_enum)
+            for name, type_enum in self._GetVariableToDataTypeMap().items()}
+
   def has_tensor(self, tensor_str):
     from tensorflow.python.util import compat
     return self._HasTensor(compat.as_bytes(tensor_str))
-- 
GitLab


From 6cf9ffeab4da4ad38bdf2afd803bf44cdc58d15d Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Thu, 5 Oct 2017 07:50:09 -0700
Subject: [PATCH 053/909] Removes use of _grad_fn_accepts_none_for_indices in
 magic_gradient_function.

Leaves the one in imperative_grad, which seems to matter.

PiperOrigin-RevId: 171152474
---
 tensorflow/python/eager/backprop.py | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/tensorflow/python/eager/backprop.py b/tensorflow/python/eager/backprop.py
index 55df6496ed..5e3af16fb2 100644
--- a/tensorflow/python/eager/backprop.py
+++ b/tensorflow/python/eager/backprop.py
@@ -388,12 +388,6 @@ def _magic_gradient_function(op_name, attr_tuple, num_inputs,
   if grad_fn is None:
     return [None] * num_inputs
 
-  none_indices = _grad_fn_accepts_none_for_indices.get(op_name, [])
-  out_grads = [
-      o if (o is not None or i in none_indices)
-      else array_ops.zeros_like(outputs[i])
-      for i, o in enumerate(out_grads)
-  ]
   return grad_fn(mock_op, *out_grads)
 
 
-- 
GitLab


From 7d9f8ffdcaf48968b137f7e785d04a689436449f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 5 Oct 2017 07:52:36 -0700
Subject: [PATCH 054/909] Make a branch of the KMeans estimator that is ported
 to the core Estimator API.

PiperOrigin-RevId: 171152686
---
 tensorflow/contrib/cmake/tf_tests.cmake       |   1 +
 tensorflow/contrib/factorization/BUILD        |  24 +
 tensorflow/contrib/factorization/__init__.py  |  12 +-
 .../factorization/python/ops/kmeans.py        | 417 +++++++++++++
 .../factorization/python/ops/kmeans_test.py   | 575 ++++++++++++++++++
 5 files changed, 1024 insertions(+), 5 deletions(-)
 create mode 100644 tensorflow/contrib/factorization/python/ops/kmeans.py
 create mode 100644 tensorflow/contrib/factorization/python/ops/kmeans_test.py

diff --git a/tensorflow/contrib/cmake/tf_tests.cmake b/tensorflow/contrib/cmake/tf_tests.cmake
index 658d19e493..55d57b7574 100644
--- a/tensorflow/contrib/cmake/tf_tests.cmake
+++ b/tensorflow/contrib/cmake/tf_tests.cmake
@@ -296,6 +296,7 @@ if (tensorflow_BUILD_PYTHON_TESTS)
       "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/transformed_distribution_test.py"
       "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/vector_student_t_test.py"
       "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/wishart_test.py"
+      "${tensorflow_source_dir}/tensorflow/contrib/factorization/python/ops/kmeans_test.py"
       "${tensorflow_source_dir}/tensorflow/contrib/learn/python/learn/estimators/kmeans_test.py"
       # Failing with TF 1.3 (TODO)
       "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/estimator_test.py"
diff --git a/tensorflow/contrib/factorization/BUILD b/tensorflow/contrib/factorization/BUILD
index c468c544d3..8a7825c614 100644
--- a/tensorflow/contrib/factorization/BUILD
+++ b/tensorflow/contrib/factorization/BUILD
@@ -8,6 +8,7 @@ exports_files(["LICENSE"])
 
 package(default_visibility = ["//tensorflow:__subpackages__"])
 
+load("//tensorflow:tensorflow.bzl", "py_test")
 load("//tensorflow:tensorflow.bzl", "tf_custom_op_library")
 load("//tensorflow:tensorflow.bzl", "tf_gen_op_wrapper_py")
 load("//tensorflow:tensorflow.bzl", "tf_gen_op_libs")
@@ -23,6 +24,7 @@ tf_custom_op_py_library(
         "python/ops/factorization_ops.py",
         "python/ops/gmm.py",
         "python/ops/gmm_ops.py",
+        "python/ops/kmeans.py",
         "python/ops/wals.py",
     ],
     dso = [
@@ -199,6 +201,28 @@ tf_py_test(
 )
 
 # Estimators tests
+py_test(
+    name = "kmeans_test",
+    size = "medium",
+    srcs = ["python/ops/kmeans_test.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":factorization_py",
+        ":factorization_py_CYCLIC_DEPENDENCIES_THAT_NEED_TO_GO",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:control_flow_ops",
+        "//tensorflow/python:data_flow_ops",
+        "//tensorflow/python:framework_for_generated_wrappers",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:platform",
+        "//tensorflow/python:platform_benchmark",
+        "//tensorflow/python:random_ops",
+        "//tensorflow/python:training",
+        "//third_party/py/numpy",
+    ],
+)
+
 tf_py_test(
     name = "wals_test",
     size = "large",
diff --git a/tensorflow/contrib/factorization/__init__.py b/tensorflow/contrib/factorization/__init__.py
index 486c2ea933..6112c9d830 100644
--- a/tensorflow/contrib/factorization/__init__.py
+++ b/tensorflow/contrib/factorization/__init__.py
@@ -23,22 +23,24 @@ from tensorflow.contrib.factorization.python.ops.clustering_ops import *
 from tensorflow.contrib.factorization.python.ops.factorization_ops import *
 from tensorflow.contrib.factorization.python.ops.gmm import *
 from tensorflow.contrib.factorization.python.ops.gmm_ops import *
+from tensorflow.contrib.factorization.python.ops.kmeans import *
 from tensorflow.contrib.factorization.python.ops.wals import *
 # pylint: enable=wildcard-import
 
 from tensorflow.python.util.all_util import remove_undocumented
 
 _allowed_symbols = [
-    'KMeans',
     'COSINE_DISTANCE',
-    'KMEANS_PLUS_PLUS_INIT',
-    'RANDOM_INIT',
-    'SQUARED_EUCLIDEAN_DISTANCE',
-    'WALSModel',
     'GMM',
     'gmm',
     'GmmAlgorithm',
+    'KMeans',
+    'KMEANS_PLUS_PLUS_INIT',
+    'KMeansClustering',
+    'RANDOM_INIT',
+    'SQUARED_EUCLIDEAN_DISTANCE',
     'WALSMatrixFactorization',
+    'WALSModel',
 ]
 
 remove_undocumented(__name__, _allowed_symbols)
diff --git a/tensorflow/contrib/factorization/python/ops/kmeans.py b/tensorflow/contrib/factorization/python/ops/kmeans.py
new file mode 100644
index 0000000000..6284768bdd
--- /dev/null
+++ b/tensorflow/contrib/factorization/python/ops/kmeans.py
@@ -0,0 +1,417 @@
+# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""A canned Estimator for k-means clustering."""
+
+# TODO(ccolby): Move clustering_ops.py into this file and streamline the code.
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import time
+import numpy as np
+
+from tensorflow.contrib.factorization.python.ops import clustering_ops
+from tensorflow.python.estimator import estimator
+from tensorflow.python.estimator import model_fn as model_fn_lib
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import metrics
+from tensorflow.python.ops import state_ops
+from tensorflow.python.platform import tf_logging as logging
+from tensorflow.python.summary import summary
+from tensorflow.python.training import session_run_hook
+from tensorflow.python.training import training_util
+
+
+class _LossRelativeChangeHook(session_run_hook.SessionRunHook):
+  """Stops when the change in loss goes below a tolerance."""
+
+  def __init__(self, loss_tensor, tolerance):
+    """Creates a _LossRelativeChangeHook.
+
+    Args:
+      loss_tensor: A scalar tensor of the loss value.
+      tolerance: A relative tolerance of loss change between iterations.
+    """
+    self._loss_tensor = loss_tensor
+    self._tolerance = tolerance
+    self._prev_loss = None
+
+  def before_run(self, run_context):
+    del run_context  # unused
+    return session_run_hook.SessionRunArgs(self._loss_tensor)
+
+  def after_run(self, run_context, run_values):
+    loss = run_values.results
+    assert loss is not None
+    if self._prev_loss:
+      relative_change = (abs(loss - self._prev_loss) /
+                         (1 + abs(self._prev_loss)))
+      if relative_change < self._tolerance:
+        run_context.request_stop()
+    self._prev_loss = loss
+
+
+class _InitializeClustersHook(session_run_hook.SessionRunHook):
+  """Initializes the cluster centers.
+
+  The chief repeatedly invokes an initialization op until all cluster centers
+  are initialized. The workers wait for the initialization phase to complete.
+  """
+
+  def __init__(self, init_op, is_initialized_var, is_chief):
+    """Creates an _InitializeClustersHook.
+
+    Args:
+      init_op: An op that, when run, will choose some initial cluster centers.
+          This op may need to be run multiple times to choose all the centers.
+      is_initialized_var: A boolean variable reporting whether all initial
+          centers have been chosen.
+      is_chief: A boolean specifying whether this task is the chief.
+    """
+    self._init_op = init_op
+    self._is_initialized_var = is_initialized_var
+    self._is_chief = is_chief
+
+  def after_create_session(self, session, coord):
+    del coord  # unused
+    assert self._init_op.graph is ops.get_default_graph()
+    assert self._is_initialized_var.graph is self._init_op.graph
+    while True:
+      try:
+        if session.run(self._is_initialized_var):
+          break
+        elif self._is_chief:
+          session.run(self._init_op)
+        else:
+          time.sleep(1)
+      except RuntimeError as e:
+        logging.info(e)
+
+
+def _parse_tensor_or_dict(features):
+  """Helper function to convert the input points into a usable format.
+
+  Args:
+    features: The input points.
+
+  Returns:
+    If `features` is a dict of `k` features, each of which is a vector of `n`
+    scalars, the return value is a Tensor of shape `(n, k)` representing `n`
+    input points, where the items in the `k` dimension are sorted
+    lexicographically by `features` key. If `features` is not a dict, it is
+    returned unmodified.
+  """
+  if isinstance(features, dict):
+    keys = sorted(features.keys())
+    with ops.colocate_with(features[keys[0]]):
+      features = array_ops.concat([features[k] for k in keys], axis=1)
+  return features
+
+
+class _ModelFn(object):
+  """Model function for the estimator."""
+
+  def __init__(self, num_clusters, initial_clusters, distance_metric,
+               random_seed, use_mini_batch, mini_batch_steps_per_iteration,
+               kmeans_plus_plus_num_retries, relative_tolerance):
+    self._num_clusters = num_clusters
+    self._initial_clusters = initial_clusters
+    self._distance_metric = distance_metric
+    self._random_seed = random_seed
+    self._use_mini_batch = use_mini_batch
+    self._mini_batch_steps_per_iteration = mini_batch_steps_per_iteration
+    self._kmeans_plus_plus_num_retries = kmeans_plus_plus_num_retries
+    self._relative_tolerance = relative_tolerance
+
+  def model_fn(self, features, mode, config):
+    """Model function for the estimator.
+
+    Note that this does not take a `1abels` arg. This works, but `input_fn` must
+    return either `features` or, equivalently, `(features, None)`.
+
+    Args:
+      features: The input points. See @{tf.estimator.Estimator}.
+      mode: See @{tf.estimator.Estimator}.
+      config: See @{tf.estimator.Estimator}.
+
+    Returns:
+      A @{tf.estimator.EstimatorSpec} (see @{tf.estimator.Estimator}) specifying
+      this behavior:
+        * `train_op`: Execute one mini-batch or full-batch run of Lloyd's
+             algorithm.
+        * `loss`: The sum of the squared distances from each input point to its
+             closest center.
+        * `eval_metric_ops`: Maps `SCORE` to `loss`.
+        * `predictions`: Maps `ALL_DISTANCES` to the distance from each input
+             point to each cluster center; maps `CLUSTER_INDEX` to the index of
+             the closest cluster center for each input point; maps `CLUSTERS` to
+             the cluster centers (which ignores the input points).
+    """
+    # input_points is a single Tensor. Therefore, the sharding functionality
+    # in clustering_ops is unused, and some of the values below are lists of a
+    # single item.
+    input_points = _parse_tensor_or_dict(features)
+
+    # Let N = the number of input_points.
+    # all_distances: A list of one matrix of shape (N, num_clusters). Each value
+    #   is the distance from an input point to a cluster center.
+    # model_predictions: A list of one vector of shape (N). Each value is the
+    #   cluster id of an input point.
+    # losses: Similar to cluster_idx but provides the distance to the cluster
+    #   center.
+    # is_initialized: scalar indicating whether the initial cluster centers
+    #   have been chosen; see init_op.
+    # cluster_centers_var: a Variable containing the cluster centers.
+    # init_op: an op to choose the initial cluster centers. A single worker
+    #   repeatedly executes init_op until is_initialized becomes True.
+    # training_op: an op that runs an iteration of training, either an entire
+    #   Lloyd iteration or a mini-batch of a Lloyd iteration. Multiple workers
+    #   may execute this op, but only after is_initialized becomes True.
+    (all_distances, model_predictions, losses, is_initialized,
+     cluster_centers_var, init_op, training_op) = clustering_ops.KMeans(
+         inputs=input_points,
+         num_clusters=self._num_clusters,
+         initial_clusters=self._initial_clusters,
+         distance_metric=self._distance_metric,
+         use_mini_batch=self._use_mini_batch,
+         mini_batch_steps_per_iteration=self._mini_batch_steps_per_iteration,
+         random_seed=self._random_seed,
+         kmeans_plus_plus_num_retries=self._kmeans_plus_plus_num_retries
+     ).training_graph()
+
+    loss = math_ops.reduce_sum(losses)
+    summary.scalar('loss/raw', loss)
+
+    incr_step = state_ops.assign_add(training_util.get_global_step(), 1)
+    training_op = control_flow_ops.with_dependencies([training_op, incr_step],
+                                                     loss)
+
+    training_hooks = [
+        _InitializeClustersHook(init_op, is_initialized, config.is_chief)
+    ]
+    if self._relative_tolerance is not None:
+      training_hooks.append(
+          _LossRelativeChangeHook(loss, self._relative_tolerance))
+
+    return model_fn_lib.EstimatorSpec(
+        mode=mode,
+        predictions={
+            KMeansClustering.ALL_DISTANCES: all_distances[0],
+            KMeansClustering.CLUSTER_INDEX: model_predictions[0],
+            KMeansClustering.CLUSTERS: cluster_centers_var.value(),
+        },
+        loss=loss,
+        train_op=training_op,
+        eval_metric_ops={KMeansClustering.SCORE: metrics.mean(loss)},
+        training_hooks=training_hooks)
+
+
+# TODO(agarwal,ands): support sharded input.
+class KMeansClustering(estimator.Estimator):
+  """An Estimator for K-Means clustering."""
+
+  # Valid values for the distance_metric constructor argument.
+  SQUARED_EUCLIDEAN_DISTANCE = clustering_ops.SQUARED_EUCLIDEAN_DISTANCE
+  COSINE_DISTANCE = clustering_ops.COSINE_DISTANCE
+
+  # Values for initial_clusters constructor argument.
+  RANDOM_INIT = clustering_ops.RANDOM_INIT
+  KMEANS_PLUS_PLUS_INIT = clustering_ops.KMEANS_PLUS_PLUS_INIT
+
+  # Metric returned by evaluate(): The sum of the squared distances from each
+  # input point to its closest center.
+  SCORE = 'score'
+
+  # Keys returned by predict().
+  # ALL_DISTANCES: The distance from each input  point to each cluster center.
+  # CLUSTER_INDEX: The index of the closest cluster center for each input point.
+  # CLUSTERS: The cluster centers (which ignores the input points).
+  CLUSTER_INDEX = 'cluster_index'
+  CLUSTERS = 'clusters'
+  ALL_DISTANCES = 'all_distances'
+
+  def __init__(self,
+               num_clusters,
+               model_dir=None,
+               initial_clusters=RANDOM_INIT,
+               distance_metric=SQUARED_EUCLIDEAN_DISTANCE,
+               random_seed=0,
+               use_mini_batch=True,
+               mini_batch_steps_per_iteration=1,
+               kmeans_plus_plus_num_retries=2,
+               relative_tolerance=None,
+               config=None):
+    """Creates an Estimator for running KMeans training and inference.
+
+    This Estimator implements the following variants of the K-means algorithm:
+
+    If `use_mini_batch` is False, it runs standard full batch K-means. Each
+    training step runs a single iteration of K-Means and must process the full
+    input at once. To run in this mode, the `input_fn` passed to `train` must
+    return the entire input dataset.
+
+    If `use_mini_batch` is True, it runs a generalization of the mini-batch
+    K-means algorithm. It runs multiple iterations, where each iteration is
+    composed of `mini_batch_steps_per_iteration` steps. Each training step
+    accumulates the contribution from one mini-batch into temporary storage.
+    Every `mini_batch_steps_per_iteration` steps, the cluster centers are
+    updated and the temporary storage cleared for the next iteration. Note
+    that:
+      * If `mini_batch_steps_per_iteration=1`, the algorithm reduces to the
+        standard K-means mini-batch algorithm.
+      * If `mini_batch_steps_per_iteration = num_inputs / batch_size`, the
+        algorithm becomes an asynchronous version of the full-batch algorithm.
+        However, there is no guarantee by this implementation that each input
+        is seen exactly once per iteration. Also, different updates are applied
+        asynchronously without locking. So this asynchronous version may not
+        behave exactly like a full-batch version.
+
+    Args:
+      num_clusters: An integer tensor specifying the number of clusters. This
+        argument is ignored if `initial_clusters` is a tensor or numpy array.
+      model_dir: The directory to save the model results and log files.
+      initial_clusters: Specifies how the initial cluster centers are chosen.
+        One of the following:
+        * a tensor or numpy array with the initial cluster centers.
+        * a callable `f(inputs, k)` that selects and returns up to `k` centers
+              from an input batch. `f` is free to return any number of centers
+              from `0` to `k`. It will be invoked on successive input batches
+              as necessary until all `num_clusters` centers are chosen.
+        * `KMeansClustering.RANDOM_INIT`: Choose centers randomly from an input
+              batch. If the batch size is less than `num_clusters` then the
+              entire batch is chosen to be initial cluster centers and the
+              remaining centers are chosen from successive input batches.
+        * `KMeansClustering.KMEANS_PLUS_PLUS_INIT`: Use kmeans++ to choose
+              centers from the first input batch. If the batch size is less
+              than `num_clusters`, a TensorFlow runtime error occurs.
+      distance_metric: The distance metric used for clustering. One of:
+        * `KMeansClustering.SQUARED_EUCLIDEAN_DISTANCE`: Euclidean distance
+             between vectors `u` and `v` is defined as `||u - v||_2` which is
+             the square root of the sum of the absolute squares of the elements'
+             difference.
+        * `KMeansClustering.COSINE_DISTANCE`: Cosine distance between vectors
+             `u` and `v` is defined as `1 - (u . v) / (||u||_2 ||v||_2)`.
+      random_seed: Python integer. Seed for PRNG used to initialize centers.
+      use_mini_batch: A boolean specifying whether to use the mini-batch k-means
+        algorithm. See explanation above.
+      mini_batch_steps_per_iteration: The number of steps after which the
+        updated cluster centers are synced back to a master copy. Used only if
+        `use_mini_batch=True`. See explanation above.
+      kmeans_plus_plus_num_retries: For each point that is sampled during
+        kmeans++ initialization, this parameter specifies the number of
+        additional points to draw from the current distribution before selecting
+        the best. If a negative value is specified, a heuristic is used to
+        sample `O(log(num_to_sample))` additional points. Used only if
+        `initial_clusters=KMeansClustering.KMEANS_PLUS_PLUS_INIT`.
+      relative_tolerance: A relative tolerance of change in the loss between
+        iterations. Stops learning if the loss changes less than this amount.
+        This may not work correctly if `use_mini_batch=True`.
+      config: See @{tf.estimator.Estimator}.
+
+    Raises:
+      ValueError: An invalid argument was passed to `initial_clusters` or
+        `distance_metric`.
+    """
+    if isinstance(initial_clusters, str) and initial_clusters not in [
+        KMeansClustering.RANDOM_INIT, KMeansClustering.KMEANS_PLUS_PLUS_INIT
+    ]:
+      raise ValueError(
+          "Unsupported initialization algorithm '%s'" % initial_clusters)
+    if distance_metric not in [
+        KMeansClustering.SQUARED_EUCLIDEAN_DISTANCE,
+        KMeansClustering.COSINE_DISTANCE
+    ]:
+      raise ValueError("Unsupported distance metric '%s'" % distance_metric)
+    super(KMeansClustering, self).__init__(
+        model_fn=_ModelFn(
+            num_clusters, initial_clusters, distance_metric, random_seed,
+            use_mini_batch, mini_batch_steps_per_iteration,
+            kmeans_plus_plus_num_retries, relative_tolerance).model_fn,
+        model_dir=model_dir,
+        config=config)
+
+  def _predict_one_key(self, input_fn, predict_key):
+    for result in self.predict(input_fn=input_fn, predict_keys=[predict_key]):
+      yield result[predict_key]
+
+  def predict_cluster_index(self, input_fn):
+    """Finds the index of the closest cluster center to each input point.
+
+    Args:
+      input_fn: Input points. See @{tf.estimator.Estimator.predict}.
+
+    Yields:
+      The index of the closest cluster center for each input point.
+    """
+    for index in self._predict_one_key(input_fn,
+                                       KMeansClustering.CLUSTER_INDEX):
+      yield index
+
+  def score(self, input_fn):
+    """Returns the sum of squared distances to nearest clusters.
+
+    Note that this function is different from the corresponding one in sklearn
+    which returns the negative sum.
+
+    Args:
+      input_fn: Input points. See @{tf.estimator.Estimator.evaluate}. Only one
+          batch is retrieved.
+
+    Returns:
+      The sum of the squared distance from each point in the first batch of
+      inputs to its nearest cluster center.
+    """
+    return self.evaluate(input_fn=input_fn, steps=1)[KMeansClustering.SCORE]
+
+  def transform(self, input_fn):
+    """Transforms each input point to its distances to all cluster centers.
+
+    Note that if `distance_metric=KMeansClustering.SQUARED_EUCLIDEAN_DISTANCE`,
+    this
+    function returns the squared Euclidean distance while the corresponding
+    sklearn function returns the Euclidean distance.
+
+    Args:
+      input_fn: Input points. See @{tf.estimator.Estimator.predict}.
+
+    Yields:
+      The distances from each input point to each cluster center.
+    """
+    for distances in self._predict_one_key(input_fn,
+                                           KMeansClustering.ALL_DISTANCES):
+      yield distances
+
+  def cluster_centers(self):
+    """Returns the cluster centers."""
+
+    # TODO(ccolby): Fix this clunky code once cl/168262087 is submitted.
+    # Discussion: go/estimator-get-variable-value
+    class RunOnceHook(session_run_hook.SessionRunHook):
+      """Stops after a single run."""
+
+      def after_run(self, run_context, run_values):
+        del run_values  # unused
+        run_context.request_stop()
+
+    result = self.predict(
+        input_fn=lambda: (constant_op.constant([], shape=[0, 1]), None),
+        predict_keys=[KMeansClustering.CLUSTERS],
+        hooks=[RunOnceHook()])
+    return np.array([r[KMeansClustering.CLUSTERS] for r in result])
diff --git a/tensorflow/contrib/factorization/python/ops/kmeans_test.py b/tensorflow/contrib/factorization/python/ops/kmeans_test.py
new file mode 100644
index 0000000000..4709d79425
--- /dev/null
+++ b/tensorflow/contrib/factorization/python/ops/kmeans_test.py
@@ -0,0 +1,575 @@
+# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for KMeans."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import math
+import time
+
+import numpy as np
+from sklearn.cluster import KMeans as SklearnKMeans
+
+# pylint: disable=g-import-not-at-top
+from tensorflow.contrib.factorization.python.ops import kmeans as kmeans_lib
+from tensorflow.python.estimator import run_config
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import data_flow_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import random_ops
+from tensorflow.python.platform import benchmark
+from tensorflow.python.platform import flags
+from tensorflow.python.platform import test
+from tensorflow.python.training import input as input_lib
+from tensorflow.python.training import queue_runner
+
+FLAGS = flags.FLAGS
+
+
+def normalize(x):
+  return x / np.sqrt(np.sum(x * x, axis=-1, keepdims=True))
+
+
+def cosine_similarity(x, y):
+  return np.dot(normalize(x), np.transpose(normalize(y)))
+
+
+def make_random_centers(num_centers, num_dims, center_norm=500):
+  return np.round(
+      np.random.rand(num_centers, num_dims).astype(np.float32) * center_norm)
+
+
+def make_random_points(centers, num_points, max_offset=20):
+  num_centers, num_dims = centers.shape
+  assignments = np.random.choice(num_centers, num_points)
+  offsets = np.round(
+      np.random.randn(num_points, num_dims).astype(np.float32) * max_offset)
+  return (centers[assignments] + offsets, assignments, np.add.reduce(
+      offsets * offsets, 1))
+
+
+class KMeansTestBase(test.TestCase):
+
+  def input_fn(self,
+               batch_size=None,
+               points=None,
+               randomize=None,
+               num_epochs=None):
+    """Returns an input_fn that randomly selects batches from given points."""
+    batch_size = batch_size or self.batch_size
+    points = points if points is not None else self.points
+    num_points = points.shape[0]
+    if randomize is None:
+      randomize = (self.use_mini_batch and
+                   self.mini_batch_steps_per_iteration <= 1)
+
+    def _fn():
+      x = constant_op.constant(points)
+      if batch_size == num_points:
+        return input_lib.limit_epochs(x, num_epochs=num_epochs), None
+      if randomize:
+        indices = random_ops.random_uniform(
+            constant_op.constant([batch_size]),
+            minval=0,
+            maxval=num_points - 1,
+            dtype=dtypes.int32,
+            seed=10)
+      else:
+        # We need to cycle through the indices sequentially. We create a queue
+        # to maintain the list of indices.
+        q = data_flow_ops.FIFOQueue(num_points, dtypes.int32, ())
+
+        # Conditionally initialize the Queue.
+        def _init_q():
+          with ops.control_dependencies(
+              [q.enqueue_many(math_ops.range(num_points))]):
+            return control_flow_ops.no_op()
+
+        init_q = control_flow_ops.cond(q.size() <= 0, _init_q,
+                                       control_flow_ops.no_op)
+        with ops.control_dependencies([init_q]):
+          offsets = q.dequeue_many(batch_size)
+          with ops.control_dependencies([q.enqueue_many(offsets)]):
+            indices = array_ops.identity(offsets)
+      batch = array_ops.gather(x, indices)
+      return (input_lib.limit_epochs(batch, num_epochs=num_epochs), None)
+
+    return _fn
+
+  @staticmethod
+  def config(tf_random_seed):
+    return run_config.RunConfig().replace(tf_random_seed=tf_random_seed)
+
+  @property
+  def initial_clusters(self):
+    return kmeans_lib.KMeansClustering.KMEANS_PLUS_PLUS_INIT
+
+  @property
+  def batch_size(self):
+    return self.num_points
+
+  @property
+  def use_mini_batch(self):
+    return False
+
+  @property
+  def mini_batch_steps_per_iteration(self):
+    return 1
+
+
+class KMeansTest(KMeansTestBase):
+
+  def setUp(self):
+    np.random.seed(3)
+    self.num_centers = 5
+    self.num_dims = 2
+    self.num_points = 1000
+    self.true_centers = make_random_centers(self.num_centers, self.num_dims)
+    self.points, _, self.scores = make_random_points(self.true_centers,
+                                                     self.num_points)
+    self.true_score = np.add.reduce(self.scores)
+
+  def _kmeans(self, relative_tolerance=None):
+    return kmeans_lib.KMeansClustering(
+        self.num_centers,
+        initial_clusters=self.initial_clusters,
+        distance_metric=kmeans_lib.KMeansClustering.SQUARED_EUCLIDEAN_DISTANCE,
+        use_mini_batch=self.use_mini_batch,
+        mini_batch_steps_per_iteration=self.mini_batch_steps_per_iteration,
+        random_seed=24,
+        relative_tolerance=relative_tolerance)
+
+  def test_clusters(self):
+    kmeans = self._kmeans()
+    kmeans.train(input_fn=self.input_fn(), steps=1)
+    clusters = kmeans.cluster_centers()
+    self.assertAllEqual(list(clusters.shape), [self.num_centers, self.num_dims])
+
+  def test_fit(self):
+    kmeans = self._kmeans()
+    kmeans.train(input_fn=self.input_fn(), steps=1)
+    score1 = kmeans.score(input_fn=self.input_fn(batch_size=self.num_points))
+    steps = 10 * self.num_points // self.batch_size
+    kmeans.train(input_fn=self.input_fn(), steps=steps)
+    score2 = kmeans.score(input_fn=self.input_fn(batch_size=self.num_points))
+    self.assertTrue(score1 > score2)
+    self.assertNear(self.true_score, score2, self.true_score * 0.05)
+
+  def test_monitor(self):
+    if self.use_mini_batch:
+      # We don't test for use_mini_batch case since the loss value can be noisy.
+      return
+    kmeans = kmeans_lib.KMeansClustering(
+        self.num_centers,
+        initial_clusters=self.initial_clusters,
+        distance_metric=kmeans_lib.KMeansClustering.SQUARED_EUCLIDEAN_DISTANCE,
+        use_mini_batch=self.use_mini_batch,
+        mini_batch_steps_per_iteration=self.mini_batch_steps_per_iteration,
+        config=self.config(14),
+        random_seed=12,
+        relative_tolerance=1e-4)
+
+    kmeans.train(
+        input_fn=self.input_fn(),
+        # Force it to train until the relative tolerance monitor stops it.
+        steps=None)
+    score = kmeans.score(input_fn=self.input_fn(batch_size=self.num_points))
+    self.assertNear(self.true_score, score, self.true_score * 0.01)
+
+  def test_infer(self):
+    kmeans = self._kmeans()
+    # Make a call to fit to initialize the cluster centers.
+    max_steps = 1
+    kmeans.train(input_fn=self.input_fn(), max_steps=max_steps)
+    clusters = kmeans.cluster_centers()
+
+    # Make a small test set
+    num_points = 10
+    points, true_assignments, true_offsets = make_random_points(
+        clusters, num_points)
+    input_fn = self.input_fn(batch_size=num_points, points=points, num_epochs=1)
+    # Test predict
+    assignments = list(kmeans.predict_cluster_index(input_fn))
+    self.assertAllEqual(assignments, true_assignments)
+
+    # Test score
+    score = kmeans.score(input_fn=lambda: (constant_op.constant(points), None))
+    self.assertNear(score, np.sum(true_offsets), 0.01 * score)
+
+    # Test transform
+    transform = list(kmeans.transform(input_fn))
+    true_transform = np.maximum(
+        0,
+        np.sum(np.square(points), axis=1, keepdims=True) -
+        2 * np.dot(points, np.transpose(clusters)) + np.transpose(
+            np.sum(np.square(clusters), axis=1, keepdims=True)))
+    self.assertAllClose(transform, true_transform, rtol=0.05, atol=10)
+
+
+class KMeansTestMultiStageInit(KMeansTestBase):
+
+  def test_random(self):
+    points = np.array(
+        [[1, 2], [3, 4], [5, 6], [7, 8], [9, 0]], dtype=np.float32)
+    kmeans = kmeans_lib.KMeansClustering(
+        num_clusters=points.shape[0],
+        initial_clusters=kmeans_lib.KMeansClustering.RANDOM_INIT,
+        distance_metric=kmeans_lib.KMeansClustering.SQUARED_EUCLIDEAN_DISTANCE,
+        use_mini_batch=True,
+        mini_batch_steps_per_iteration=100,
+        random_seed=24,
+        relative_tolerance=None)
+    kmeans.train(
+        input_fn=self.input_fn(batch_size=1, points=points, randomize=False),
+        steps=1)
+    clusters = kmeans.cluster_centers()
+    self.assertAllEqual(points, clusters)
+
+  def test_kmeans_plus_plus_batch_just_right(self):
+    points = np.array([[1, 2]], dtype=np.float32)
+    kmeans = kmeans_lib.KMeansClustering(
+        num_clusters=points.shape[0],
+        initial_clusters=kmeans_lib.KMeansClustering.KMEANS_PLUS_PLUS_INIT,
+        distance_metric=kmeans_lib.KMeansClustering.SQUARED_EUCLIDEAN_DISTANCE,
+        use_mini_batch=True,
+        mini_batch_steps_per_iteration=100,
+        random_seed=24,
+        relative_tolerance=None)
+    kmeans.train(
+        input_fn=self.input_fn(batch_size=1, points=points, randomize=False),
+        steps=1)
+    clusters = kmeans.cluster_centers()
+    self.assertAllEqual(points, clusters)
+
+  def test_kmeans_plus_plus_batch_too_small(self):
+    points = np.array(
+        [[1, 2], [3, 4], [5, 6], [7, 8], [9, 0]], dtype=np.float32)
+    kmeans = kmeans_lib.KMeansClustering(
+        num_clusters=points.shape[0],
+        initial_clusters=kmeans_lib.KMeansClustering.KMEANS_PLUS_PLUS_INIT,
+        distance_metric=kmeans_lib.KMeansClustering.SQUARED_EUCLIDEAN_DISTANCE,
+        use_mini_batch=True,
+        mini_batch_steps_per_iteration=100,
+        random_seed=24,
+        relative_tolerance=None)
+    with self.assertRaisesOpError(AssertionError):
+      kmeans.train(
+          input_fn=self.input_fn(batch_size=4, points=points, randomize=False),
+          steps=1)
+
+
+class MiniBatchKMeansTest(KMeansTest):
+
+  @property
+  def batch_size(self):
+    return 50
+
+  @property
+  def use_mini_batch(self):
+    return True
+
+
+class FullBatchAsyncKMeansTest(KMeansTest):
+
+  @property
+  def batch_size(self):
+    return 50
+
+  @property
+  def use_mini_batch(self):
+    return True
+
+  @property
+  def mini_batch_steps_per_iteration(self):
+    return self.num_points // self.batch_size
+
+
+class KMeansCosineDistanceTest(KMeansTestBase):
+
+  def setUp(self):
+    self.points = np.array(
+        [[2.5, 0.1], [2, 0.2], [3, 0.1], [4, 0.2], [0.1, 2.5], [0.2, 2],
+         [0.1, 3], [0.2, 4]],
+        dtype=np.float32)
+    self.num_points = self.points.shape[0]
+    self.true_centers = np.array(
+        [
+            normalize(
+                np.mean(normalize(self.points)[0:4, :], axis=0,
+                        keepdims=True))[0],
+            normalize(
+                np.mean(normalize(self.points)[4:, :], axis=0,
+                        keepdims=True))[0]
+        ],
+        dtype=np.float32)
+    self.true_assignments = np.array([0] * 4 + [1] * 4)
+    self.true_score = len(self.points) - np.tensordot(
+        normalize(self.points), self.true_centers[self.true_assignments])
+
+    self.num_centers = 2
+    self.kmeans = kmeans_lib.KMeansClustering(
+        self.num_centers,
+        initial_clusters=kmeans_lib.KMeansClustering.RANDOM_INIT,
+        distance_metric=kmeans_lib.KMeansClustering.COSINE_DISTANCE,
+        use_mini_batch=self.use_mini_batch,
+        mini_batch_steps_per_iteration=self.mini_batch_steps_per_iteration,
+        config=self.config(3))
+
+  def test_fit(self):
+    max_steps = 10 * self.num_points // self.batch_size
+    self.kmeans.train(input_fn=self.input_fn(), max_steps=max_steps)
+    centers = normalize(self.kmeans.cluster_centers())
+    centers = centers[centers[:, 0].argsort()]
+    true_centers = self.true_centers[self.true_centers[:, 0].argsort()]
+    self.assertAllClose(centers, true_centers, atol=0.04)
+
+  def test_transform(self):
+    self.kmeans.train(input_fn=self.input_fn(), steps=10)
+    centers = normalize(self.kmeans.cluster_centers())
+    true_transform = 1 - cosine_similarity(self.points, centers)
+    transform = list(
+        self.kmeans.transform(
+            input_fn=self.input_fn(batch_size=self.num_points, num_epochs=1)))
+    self.assertAllClose(transform, true_transform, atol=1e-3)
+
+  def test_predict(self):
+    max_steps = 10 * self.num_points // self.batch_size
+    self.kmeans.train(input_fn=self.input_fn(), max_steps=max_steps)
+    centers = normalize(self.kmeans.cluster_centers())
+
+    assignments = list(
+        self.kmeans.predict_cluster_index(
+            input_fn=self.input_fn(num_epochs=1, batch_size=self.num_points)))
+    self.assertAllClose(
+        centers[assignments],
+        self.true_centers[self.true_assignments],
+        atol=1e-2)
+
+    centers = centers[centers[:, 0].argsort()]
+    true_centers = self.true_centers[self.true_centers[:, 0].argsort()]
+    self.assertAllClose(centers, true_centers, atol=0.04)
+    score = self.kmeans.score(
+        input_fn=self.input_fn(batch_size=self.num_points))
+    self.assertAllClose(score, self.true_score, atol=1e-2)
+
+  def test_predict_kmeans_plus_plus(self):
+    # Most points are concetrated near one center. KMeans++ is likely to find
+    # the less populated centers.
+    points = np.array(
+        [[2.5, 3.5], [2.5, 3.5], [-2, 3], [-2, 3], [-3, -3], [-3.1, -3.2],
+         [-2.8, -3.], [-2.9, -3.1], [-3., -3.1], [-3., -3.1], [-3.2, -3.],
+         [-3., -3.]],
+        dtype=np.float32)
+    true_centers = np.array(
+        [
+            normalize(
+                np.mean(normalize(points)[0:2, :], axis=0, keepdims=True))[0],
+            normalize(
+                np.mean(normalize(points)[2:4, :], axis=0, keepdims=True))[0],
+            normalize(np.mean(normalize(points)[4:, :], axis=0,
+                              keepdims=True))[0]
+        ],
+        dtype=np.float32)
+    true_assignments = [0] * 2 + [1] * 2 + [2] * 8
+    true_score = len(points) - np.tensordot(
+        normalize(points), true_centers[true_assignments])
+
+    kmeans = kmeans_lib.KMeansClustering(
+        3,
+        initial_clusters=self.initial_clusters,
+        distance_metric=kmeans_lib.KMeansClustering.COSINE_DISTANCE,
+        use_mini_batch=self.use_mini_batch,
+        mini_batch_steps_per_iteration=self.mini_batch_steps_per_iteration,
+        config=self.config(3))
+    kmeans.train(
+        input_fn=lambda: (constant_op.constant(points), None), steps=30)
+
+    centers = normalize(kmeans.cluster_centers())
+    self.assertAllClose(
+        sorted(centers.tolist()), sorted(true_centers.tolist()), atol=1e-2)
+
+    def _input_fn():
+      return (input_lib.limit_epochs(
+          constant_op.constant(points), num_epochs=1), None)
+
+    assignments = list(kmeans.predict_cluster_index(input_fn=_input_fn))
+    self.assertAllClose(
+        centers[assignments], true_centers[true_assignments], atol=1e-2)
+
+    score = kmeans.score(input_fn=lambda: (constant_op.constant(points), None))
+    self.assertAllClose(score, true_score, atol=1e-2)
+
+
+class MiniBatchKMeansCosineTest(KMeansCosineDistanceTest):
+
+  @property
+  def batch_size(self):
+    return 2
+
+  @property
+  def use_mini_batch(self):
+    return True
+
+
+class FullBatchAsyncKMeansCosineTest(KMeansCosineDistanceTest):
+
+  @property
+  def batch_size(self):
+    return 2
+
+  @property
+  def use_mini_batch(self):
+    return True
+
+  @property
+  def mini_batch_steps_per_iteration(self):
+    return self.num_points // self.batch_size
+
+
+class KMeansBenchmark(benchmark.Benchmark):
+  """Base class for benchmarks."""
+
+  def SetUp(self,
+            dimension=50,
+            num_clusters=50,
+            points_per_cluster=10000,
+            center_norm=500,
+            cluster_width=20):
+    np.random.seed(123456)
+    self.num_clusters = num_clusters
+    self.num_points = num_clusters * points_per_cluster
+    self.centers = make_random_centers(
+        self.num_clusters, dimension, center_norm=center_norm)
+    self.points, _, scores = make_random_points(
+        self.centers, self.num_points, max_offset=cluster_width)
+    self.score = float(np.sum(scores))
+
+  def _report(self, num_iters, start, end, scores):
+    print(scores)
+    self.report_benchmark(
+        iters=num_iters,
+        wall_time=(end - start) / num_iters,
+        extras={'true_sum_squared_distances': self.score,
+                'fit_scores': scores})
+
+  def _fit(self, num_iters=10):
+    pass
+
+  def benchmark_01_2dim_5center_500point(self):
+    self.SetUp(dimension=2, num_clusters=5, points_per_cluster=100)
+    self._fit()
+
+  def benchmark_02_20dim_20center_10kpoint(self):
+    self.SetUp(dimension=20, num_clusters=20, points_per_cluster=500)
+    self._fit()
+
+  def benchmark_03_100dim_50center_50kpoint(self):
+    self.SetUp(dimension=100, num_clusters=50, points_per_cluster=1000)
+    self._fit()
+
+  def benchmark_03_100dim_50center_50kpoint_unseparated(self):
+    self.SetUp(
+        dimension=100,
+        num_clusters=50,
+        points_per_cluster=1000,
+        cluster_width=250)
+    self._fit()
+
+  def benchmark_04_100dim_500center_500kpoint(self):
+    self.SetUp(dimension=100, num_clusters=500, points_per_cluster=1000)
+    self._fit(num_iters=4)
+
+  def benchmark_05_100dim_500center_500kpoint_unseparated(self):
+    self.SetUp(
+        dimension=100,
+        num_clusters=500,
+        points_per_cluster=1000,
+        cluster_width=250)
+    self._fit(num_iters=4)
+
+
+class TensorflowKMeansBenchmark(KMeansBenchmark):
+
+  def _fit(self, num_iters=10):
+    scores = []
+    start = time.time()
+    for i in range(num_iters):
+      print('Starting tensorflow KMeans: %d' % i)
+      tf_kmeans = kmeans_lib.KMeansClustering(
+          self.num_clusters,
+          initial_clusters=kmeans_lib.KMeansClustering.KMEANS_PLUS_PLUS_INIT,
+          kmeans_plus_plus_num_retries=int(math.log(self.num_clusters) + 2),
+          random_seed=i * 42,
+          relative_tolerance=1e-6,
+          config=self.config(3))
+      tf_kmeans.train(
+          input_fn=lambda: (constant_op.constant(self.points), None), steps=50)
+      _ = tf_kmeans.cluster_centers()
+      scores.append(
+          tf_kmeans.score(
+              input_fn=lambda: (constant_op.constant(self.points), None)))
+    self._report(num_iters, start, time.time(), scores)
+
+
+class SklearnKMeansBenchmark(KMeansBenchmark):
+
+  def _fit(self, num_iters=10):
+    scores = []
+    start = time.time()
+    for i in range(num_iters):
+      print('Starting sklearn KMeans: %d' % i)
+      sklearn_kmeans = SklearnKMeans(
+          n_clusters=self.num_clusters,
+          init='k-means++',
+          max_iter=50,
+          n_init=1,
+          tol=1e-4,
+          random_state=i * 42)
+      sklearn_kmeans.train(self.points)
+      scores.append(sklearn_kmeans.inertia_)
+    self._report(num_iters, start, time.time(), scores)
+
+
+class KMeansTestQueues(test.TestCase):
+
+  def input_fn(self):
+
+    def _fn():
+      queue = data_flow_ops.FIFOQueue(
+          capacity=10, dtypes=dtypes.float32, shapes=[10, 3])
+      enqueue_op = queue.enqueue(array_ops.zeros([10, 3], dtype=dtypes.float32))
+      queue_runner.add_queue_runner(
+          queue_runner.QueueRunner(queue, [enqueue_op]))
+      return queue.dequeue(), None
+
+    return _fn
+
+  # This test makes sure that there are no deadlocks when using a QueueRunner.
+  # Note that since cluster initialization is dependendent on inputs, if input
+  # is generated using a QueueRunner, one has to make sure that these runners
+  # are started before the initialization.
+  def test_queues(self):
+    kmeans = kmeans_lib.KMeansClustering(5)
+    kmeans.train(input_fn=self.input_fn(), steps=1)
+
+
+if __name__ == '__main__':
+  test.main()
-- 
GitLab


From 37d297d00a0639c53bf7366afd7d4836c2e09fcf Mon Sep 17 00:00:00 2001
From: David Norman <DavidNorman@users.noreply.github.com>
Date: Thu, 5 Oct 2017 16:46:27 +0100
Subject: [PATCH 055/909] Re-instate the plugin BUILD (#13291)

* Re-instate the plugin BUILD

* Adding a README to describe the purpose of this directory
---
 tensorflow/compiler/jit/BUILD        |  1 +
 tensorflow/compiler/plugin/BUILD     | 42 ++++++++++++++++++++++++++++
 tensorflow/compiler/plugin/README.md | 16 +++++++++++
 3 files changed, 59 insertions(+)
 create mode 100644 tensorflow/compiler/plugin/BUILD
 create mode 100644 tensorflow/compiler/plugin/README.md

diff --git a/tensorflow/compiler/jit/BUILD b/tensorflow/compiler/jit/BUILD
index bf63b7e501..bf7d9cf14d 100644
--- a/tensorflow/compiler/jit/BUILD
+++ b/tensorflow/compiler/jit/BUILD
@@ -33,6 +33,7 @@ cc_library(
     deps = [
         ":xla_cpu_device",
         ":xla_cpu_jit",
+        "//tensorflow/compiler/plugin",
     ] + if_cuda_is_configured([
         ":xla_gpu_device",
         ":xla_gpu_jit",
diff --git a/tensorflow/compiler/plugin/BUILD b/tensorflow/compiler/plugin/BUILD
new file mode 100644
index 0000000000..f088672154
--- /dev/null
+++ b/tensorflow/compiler/plugin/BUILD
@@ -0,0 +1,42 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""Configuration file for an XLA plugin.
+
+  please don't check in changes to this file. to prevent changes appearing
+  in git status, use:
+
+  git update-index --assume-unchanged tensorflow/compiler/plugin/BUILD
+
+  To add additional devices to the XLA subsystem, add targets to the
+  dependency list in the 'plugin' target. For instance:
+
+    deps = ["//tensorflow/compiler/plugin/example:plugin_lib"],
+
+  ** Please don't remove this file - it is supporting some 3rd party plugins **
+"""
+
+licenses(["notice"])
+
+package(
+    default_visibility = ["//visibility:public"],
+)
+
+cc_library(
+    name = "plugin",
+    deps = [
+        #"//tensorflow/compiler/plugin/example:example_lib",
+    ],
+)
diff --git a/tensorflow/compiler/plugin/README.md b/tensorflow/compiler/plugin/README.md
new file mode 100644
index 0000000000..9dd0d2bdab
--- /dev/null
+++ b/tensorflow/compiler/plugin/README.md
@@ -0,0 +1,16 @@
+3rd party XLA devices
+---------------------
+
+This directory is intended as a place for 3rd party XLA devices which are _not_
+integrated into the public repository.
+
+By adding entries to the BUILD target in this directory, a third party device
+can be included as a dependency of the JIT subsystem.
+
+For integration into the unit test system, see the files:
+
+- tensorflow/compiler/tests/plugin.bzl
+- tensorflow/compiler/xla/tests/plugin.bzl
+
+
+- 
-- 
GitLab


From ae98ba9ac2e9889ea38c45539296ab8efe432933 Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Thu, 5 Oct 2017 08:51:37 -0700
Subject: [PATCH 056/909] imperative_gradient doesn't fail if some variables
 are not connected to the output

PiperOrigin-RevId: 171158798
---
 tensorflow/python/eager/backprop.py      | 14 ++++----------
 tensorflow/python/eager/backprop_test.py | 10 ++++++++++
 2 files changed, 14 insertions(+), 10 deletions(-)

diff --git a/tensorflow/python/eager/backprop.py b/tensorflow/python/eager/backprop.py
index 5e3af16fb2..1d729cc2e1 100644
--- a/tensorflow/python/eager/backprop.py
+++ b/tensorflow/python/eager/backprop.py
@@ -99,7 +99,7 @@ def _prepare_backprop(target, tensor_to_op, op_to_entry, id_sources):
   o_to_e = {}  # Copy of just the bits we need from op_to_entry
   while tensor_stack:
     t = tensor_stack.pop()
-    op = tensor_to_op[t]
+    op = tensor_to_op.get(t, None)
     # op is None if the tensor is a source (i.e. was watched directly)
     if op is None or op in o_to_e:
       continue
@@ -313,15 +313,9 @@ def imperative_grad(
   for i, s in enumerate(sources):
     g = gradients.get(ops.tensor_id(s), None)
     if g is None:
-      # TODO(apassos): figure out a way to summarize why sources and targets are
-      # not connected.
-      raise ValueError("There is no sequence of operations connecting source "
-                       "tensor %s (%s) to any of the target Tensors. This is "
-                       "commonly caused by the tape not recording all "
-                       "operations in the forward pass or if by mistake a "
-                       "source was only used in non-differentiable operations."
-                       % (i, s))
-    result.append(_aggregate_grads(g))
+      result.append(None)
+    else:
+      result.append(_aggregate_grads(g))
   return result
 
 _op_attr_type_cache = {}
diff --git a/tensorflow/python/eager/backprop_test.py b/tensorflow/python/eager/backprop_test.py
index 07d2d2a148..3b72974fc7 100644
--- a/tensorflow/python/eager/backprop_test.py
+++ b/tensorflow/python/eager/backprop_test.py
@@ -255,6 +255,16 @@ class BackpropTest(test.TestCase):
     self.assertAllEqual(dx.numpy(), y.numpy())
     self.assertAllEqual(dy.numpy(), x.numpy())
 
+  def testUnconnectedNone(self):
+    v = resource_variable_ops.ResourceVariable(
+        1.0, name='testUnconnectedNone')
+
+    def f():
+      v.read_value()
+      return constant_op.constant(1.0)
+
+    self.assertEqual(backprop.implicit_grad(f)()[0][0], None)
+
   def testEmptyParamsForValueAndGradFunction(self):
     def fn(a, b):
       return a * b
-- 
GitLab


From 8dc5e3718b85b72a8bc6e5a2ea8270eecfdf99a1 Mon Sep 17 00:00:00 2001
From: Jacques Pienaar <jpienaar@google.com>
Date: Thu, 5 Oct 2017 09:41:13 -0700
Subject: [PATCH 057/909] [TFXLA] Functionalize tf.cond.

Convert tf.cond to functional form
  output = cond ? then_branch(inputs) : else_branch(inputs)
where then_branch and else_branch are functions.

PiperOrigin-RevId: 171164597
---
 tensorflow/compiler/tf2xla/BUILD              |   4 +
 .../tf2xla/functionalize_control_flow.cc      | 813 +++++++++++++++++-
 .../tf2xla/functionalize_control_flow.h       |   1 -
 .../tf2xla/functionalize_control_flow_test.cc | 129 +++
 .../compiler/tf2xla/ops/functional_ops.cc     |  39 +-
 tensorflow/python/ops/control_flow_ops.py     |   6 +
 6 files changed, 949 insertions(+), 43 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/BUILD b/tensorflow/compiler/tf2xla/BUILD
index 08f2249e0d..4da2ed722e 100644
--- a/tensorflow/compiler/tf2xla/BUILD
+++ b/tensorflow/compiler/tf2xla/BUILD
@@ -253,6 +253,7 @@ tf_cc_test(
         "//tensorflow/core:tensor_testutil",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
+        "//tensorflow/core:testlib",
     ],
 )
 
@@ -347,6 +348,7 @@ cc_library(
     hdrs = ["functionalize_control_flow.h"],
     deps = [
         "//tensorflow/compiler/jit:graph_to_functiondef",
+        "//tensorflow/compiler/jit:union_find",
         "//tensorflow/compiler/tf2xla:dump_graph",
         "//tensorflow/compiler/tf2xla/ops:functional_ops",
         "//tensorflow/compiler/xla:status_macros",
@@ -354,6 +356,7 @@ cc_library(
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:core_cpu_internal",
         "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
     ],
 )
 
@@ -371,6 +374,7 @@ tf_cc_test(
         "//tensorflow/compiler/tf2xla/cc:functional_ops",
         "//tensorflow/compiler/xla:status_macros",
         "//tensorflow/core:core_cpu",
+        "//tensorflow/core:core_cpu_internal",
         "//tensorflow/core:framework",
         "//tensorflow/core:framework_internal",
         "//tensorflow/core:ops",
diff --git a/tensorflow/compiler/tf2xla/functionalize_control_flow.cc b/tensorflow/compiler/tf2xla/functionalize_control_flow.cc
index 1c7a2046aa..56d8bb4f2c 100644
--- a/tensorflow/compiler/tf2xla/functionalize_control_flow.cc
+++ b/tensorflow/compiler/tf2xla/functionalize_control_flow.cc
@@ -17,15 +17,19 @@ limitations under the License.
 
 #include <algorithm>
 #include <deque>
+#include <stack>
 #include <unordered_set>
 #include <vector>
 
 #include "tensorflow/compiler/jit/graph_to_functiondef.h"
+#include "tensorflow/compiler/jit/union_find.h"
 #include "tensorflow/compiler/tf2xla/dump_graph.h"
 #include "tensorflow/compiler/xla/ptr_util.h"
 #include "tensorflow/compiler/xla/status_macros.h"
+#include "tensorflow/core/common_runtime/function.h"
 #include "tensorflow/core/framework/node_def_builder.h"
 #include "tensorflow/core/graph/control_flow.h"
+#include "tensorflow/core/lib/gtl/optional.h"
 
 namespace tensorflow {
 
@@ -74,7 +78,8 @@ struct Frame {
 // starting at nodes in vector `stack`.
 // `node_map` is a vector indexed by source node ID to dest nodes.
 // Does not traverse into nodes in `node_map`, so by adding nodes to `node_map`
-// before the traversal clients can cut the graph. Returns an error if the
+// before the traversal clients can cut the graph. If a frame is provided (frame
+// != nullptr), then this functions will return an error if the
 // traversal leaves 'frame'; the client must add enough nodes to `node_map` to
 // cut the graph and prevent the traversal from escaping.
 //
@@ -84,7 +89,7 @@ struct Frame {
 // taking from the Switch node was not necessarily the first output, but _Arg
 // nodes only have one output. By adding the Switch node to `squash_src_outputs`
 // we rewrite the src_output of the corresponding edge to be 0.
-Status CopySubgraph(const Graph& graph, const Frame& frame,
+Status CopySubgraph(const Graph& graph, const Frame* frame,
                     std::vector<Node*> stack,
                     const std::vector<bool>& squash_src_outputs,
                     std::vector<Node*>* node_map, Graph* output) {
@@ -100,9 +105,9 @@ Status CopySubgraph(const Graph& graph, const Frame& frame,
 
     for (const Edge* e : n->in_edges()) {
       Node* src = e->src();
-      if (frame.nodes.find(src) == frame.nodes.end()) {
+      if (frame != nullptr && frame->nodes.find(src) == frame->nodes.end()) {
         // We traversed out of the loop frame, without encountering a cut node.
-        return errors::Internal("Graph traversal of loop frame ", frame.name,
+        return errors::Internal("Graph traversal of loop frame ", frame->name,
                                 " escaped frame at ", src->name(),
                                 " without encountering an argument node.");
       }
@@ -119,27 +124,31 @@ Status CopySubgraph(const Graph& graph, const Frame& frame,
   return Status::OK();
 }
 
-Status BuildArgNode(Graph* graph, DataType type, int index, Node** arg_node) {
+xla::StatusOr<Node*> AddNode(const NodeDef& node_def, Graph* graph) {
+  Status status;
+  Node* inserted_node = graph->AddNode(node_def, &status);
+  if (!status.ok()) {
+    return status;
+  }
+  return inserted_node;
+}
+
+xla::StatusOr<Node*> BuildArgNode(Graph* graph, DataType type, int index) {
   NodeDef arg_def;
-  NodeDefBuilder builder(strings::StrCat("_Arg", index), kArgOp);
+  NodeDefBuilder builder(strings::StrCat(kArgOp, index), kArgOp);
   builder.Attr("T", type);
   builder.Attr("index", index);
   TF_RETURN_IF_ERROR(builder.Finalize(&arg_def));
-  Status status;
-  *arg_node = graph->AddNode(arg_def, &status);
-  return status;
+  return AddNode(arg_def, graph);
 }
 
-Status BuildRetvalNode(Graph* graph, DataType type, int index,
-                       Node** retval_node) {
+xla::StatusOr<Node*> BuildRetvalNode(Graph* graph, DataType type, int index) {
   NodeDef ret_def;
   ret_def.set_op(kRetValOp);
-  ret_def.set_name(strings::StrCat("_Retval", index));
+  ret_def.set_name(strings::StrCat(kRetValOp, index));
   AddNodeAttr("T", type, &ret_def);
   AddNodeAttr("index", index, &ret_def);
-  Status status;
-  *retval_node = graph->AddNode(ret_def, &status);
-  return status;
+  return AddNode(ret_def, graph);
 }
 
 // Builds a graph for the loop condition.
@@ -157,9 +166,8 @@ Status BuildLoopCondition(const Graph& graph, Frame* frame,
   for (int i = 0; i < frame->args.size(); ++i) {
     const Arg& arg = frame->args[i];
 
-    Node* arg_node;
-    TF_RETURN_IF_ERROR(
-        BuildArgNode(output, arg.enter->input_type(0), i, &arg_node));
+    TF_ASSIGN_OR_RETURN(Node * arg_node,
+                        BuildArgNode(output, arg.enter->input_type(0), i));
     if (arg.is_loop_invariant) {
       node_map[arg.enter->id()] = arg_node;
     } else {
@@ -169,16 +177,14 @@ Status BuildLoopCondition(const Graph& graph, Frame* frame,
 
   // Build a Retval node for the loop condition. The LoopCond nodes are always
   // boolean because of the type constraints on the LoopCond op.
-  TF_RETURN_IF_ERROR(
-      BuildRetvalNode(output, DT_BOOL, 0, &node_map[frame->loop_cond->id()]));
+  TF_ASSIGN_OR_RETURN(node_map[frame->loop_cond->id()],
+                      BuildRetvalNode(output, DT_BOOL, 0));
 
   // Performs a reverse DFS, copying nodes and edges to the output graph.
   // The _Arg and _Retval nodes were added unconditionally above, so we are
   // guaranteed to get the correct function signature.
-  TF_RETURN_IF_ERROR(CopySubgraph(graph, *frame, {frame->loop_cond},
-                                  squash_src_outputs, &node_map, output));
-
-  return Status::OK();
+  return CopySubgraph(graph, frame, {frame->loop_cond}, squash_src_outputs,
+                      &node_map, output);
 }
 
 // Builds a graph for the loop body.
@@ -202,8 +208,8 @@ Status BuildLoopBody(const Graph& graph, Frame* frame,
 
     DataType dtype = arg.enter->input_type(0);
     arg_types->push_back(dtype);
-    Node* arg_node;
-    TF_RETURN_IF_ERROR(BuildArgNode(output, dtype, i, &arg_node));
+
+    TF_ASSIGN_OR_RETURN(Node * arg_node, BuildArgNode(output, dtype, i));
 
     if (dtype == DT_RESOURCE) {
       // The convention of the XLA bridge is that resource variable arguments
@@ -213,8 +219,8 @@ Status BuildLoopBody(const Graph& graph, Frame* frame,
       TF_RET_CHECK(arg.is_loop_invariant);
       node_map[arg.enter->id()] = arg_node;
     } else {
-      Node* retval_node;
-      TF_RETURN_IF_ERROR(BuildRetvalNode(output, dtype, i, &retval_node));
+      TF_ASSIGN_OR_RETURN(Node * retval_node,
+                          BuildRetvalNode(output, dtype, i));
 
       if (arg.is_loop_invariant) {
         // Argument is loop-invariant. Forward it from the Arg to the Retval.
@@ -237,7 +243,7 @@ Status BuildLoopBody(const Graph& graph, Frame* frame,
   // Performs a reverse DFS, copying nodes and edges to the output graph.
   // The _Arg and _Retval nodes were added unconditionally above, so we are
   // guaranteed to get the correct function signature.
-  TF_RETURN_IF_ERROR(CopySubgraph(graph, *frame, std::move(next_iterations),
+  TF_RETURN_IF_ERROR(CopySubgraph(graph, frame, std::move(next_iterations),
                                   squash_src_outputs, &node_map, output));
 
   return Status::OK();
@@ -450,12 +456,7 @@ Status FunctionalizeLoop(Graph* graph, Frame* frame,
   }
   builder.Input(inputs);
   TF_RETURN_IF_ERROR(builder.Finalize(&while_def));
-
-  Status status;
-  Node* while_node = graph->AddNode(while_def, &status);
-  if (!status.ok()) {
-    return status;
-  }
+  TF_ASSIGN_OR_RETURN(Node * while_node, AddNode(while_def, graph));
 
   // Copies edges to the Enter nodes and from the Exit nodes onto the While.
   for (int i = 0; i < frame->args.size(); ++i) {
@@ -488,6 +489,7 @@ Status FunctionalizeLoop(Graph* graph, Frame* frame,
   for (Node* node : frame->nodes) {
     graph->RemoveNode(node);
   }
+  frame->nodes.clear();
   frame->parent->nodes.insert(while_node);
 
   VLOG(2) << "Frame " << frame->name << " after: "
@@ -496,6 +498,742 @@ Status FunctionalizeLoop(Graph* graph, Frame* frame,
   return Status::OK();
 }
 
+class FunctionalizeCond {
+ public:
+  // Identifies the connected parts of the tf.Cond.
+  struct ClusterHandle {
+    explicit ClusterHandle(int representative = -1)
+        : representative(representative) {}
+
+    bool operator==(const ClusterHandle& other) const {
+      return representative == other.representative;
+    }
+
+    bool operator!=(const ClusterHandle& other) const {
+      return !(*this == other);
+    }
+
+    bool operator<(const ClusterHandle& other) const {
+      return representative < other.representative;
+    }
+
+    bool operator>(const ClusterHandle& other) const {
+      return representative > other.representative;
+    }
+
+    string ToString() const {
+      return strings::StrCat("Cluster_", representative);
+    }
+
+    // Vector of UnionFind<ClusterHandle> indexable by ClusterHandle and Node*.
+    struct Vector {
+      explicit Vector(size_t size) : clusters(size) {}
+
+      UnionFind<ClusterHandle>& at(const ClusterHandle& cluster) {
+        return clusters.at(cluster.representative);
+      }
+
+      UnionFind<ClusterHandle>& at(const Node* node) {
+        return clusters.at(node->id());
+      }
+
+      UnionFind<ClusterHandle>& operator[](const Node* node) {
+        return clusters.at(node->id());
+      }
+
+      size_t size() const { return clusters.size(); }
+
+      void resize(size_t count) { return clusters.resize(count); }
+
+     private:
+      std::vector<UnionFind<ClusterHandle>> clusters;
+    };
+
+   private:
+    int representative;
+  };
+
+  // Represents a node in the clustered graph consisting of switch_nodes,
+  // merge_nodes as well as the edges into and out of this node to other
+  // Clusters. Each Cluster corresponds to a ClusterHandle and has a
+  // corresponding representative.
+  struct Cluster {
+    std::unordered_set<Node*> switch_nodes;
+    std::unordered_set<Node*> merge_nodes;
+    std::unordered_set<Cluster*> in_nodes;
+    std::unordered_set<Cluster*> out_nodes;
+
+    // A member of the ClusterHandle corresponding to this Cluster.
+    ClusterHandle representative;
+    bool visited = false;
+  };
+
+  // Represent the clustered graph as map from cluster representative to
+  // Cluster.
+  using ClusteredGraph = std::map<ClusterHandle, Cluster>;
+
+  // The arguments and condition of a XlaIf. The arguments are ordered by node
+  // id in the original graph.
+  struct CondArgs {
+    struct CondCmp {
+      bool operator()(const Node* a, const Node* b) {
+        return a->id() < b->id();
+      }
+    };
+    Node* conditional = nullptr;
+    std::set<Node*, CondCmp> args;
+  };
+
+  static Status Functionalize(Graph* graph, FunctionLibraryDefinition* library);
+
+ private:
+  FunctionalizeCond(Graph* graph, FunctionLibraryDefinition* library)
+      : clusters_(graph->num_node_ids()), library_(library), graph_(graph) {}
+
+  // Returns a vector of Merge nodes from the clustered graph where the nodes
+  // are sorted by the number of switch nodes minus number of merge nodes
+  // from a root of the clustered graph to the given Merge node, with ties
+  // broken by the representative of the Cluster.
+  std::vector<std::pair<int, Cluster*>> SortedMergeNodes();
+
+  // Returns whether the graph has no conditionals.
+  bool NoConditionals() const { return merge_nodes_.empty(); }
+
+  // Construct the clustered graph by creating nodes for each cluster and the
+  // connections between the clusters. Switch and Merge nodes partition
+  // clusters, so iterate over those. Note: a Cluster may have neither a
+  // Merge or Switch but will have an in/out edge from a Cluster that has.
+  void CreateClusters();
+
+  // Creates the clustered graph by identifying all the edges between different
+  // clusters and collecting all switch and merge nodes that correspond to a
+  // cluster.
+  void CreateClusteredGraph();
+
+  // If `from` and `to` correspond to different clusters, then merge the nodes
+  // in the clustered graph corresponding to `from` and `to`.
+  void ContractEdge(Cluster* from, Cluster* to);
+
+  // Converts a Merge node to a XlaIf. This encapsulates the process of
+  // extracting the bodies needed for the then and else branch, creates a XlaIf
+  // node, removing the nodes of the branches from the graph and replacing the
+  // merge node with a XlaIf.
+  Status ConvertMergeToXlaIf(Cluster* merge_cluster);
+
+  // Returns the switch cluster corresponding to the merge node. This function
+  // only returns the switch cluster in the simple case where we have a switch
+  // node is the entry of a diamond corresponding to a conditional:
+  //
+  //           Switch
+  //          /      \
+  //     Branch      Branch
+  //          \      /
+  //           merge_cluster
+  gtl::optional<Cluster*> GetSwitchCluster(const Cluster& merge_cluster);
+
+  // Determines the arguments needed as input to the Merge cluster originating
+  // from the Switch cluster.
+  xla::StatusOr<CondArgs> DetermineCondArgs(const Cluster& merge_cluster,
+                                            const Cluster& switch_cluster);
+
+  // Builds a XlaIfOp to replace the Merge node with.
+  xla::StatusOr<Node*> BuildAndAddXlaIfOp(const CondArgs& cond_args,
+                                          const Cluster& merge_cluster,
+                                          const std::vector<Node*>& outputs);
+
+  // Extracts a function body corresponding to the given input edge of the merge
+  // node.
+  Status ExtractBody(const CondArgs& cond_args, const Cluster& merge_cluster,
+                     const std::vector<Node*>& outputs, int input_edge,
+                     Graph* body);
+
+  // Adds all the input edges to `if_node` corresponding to the arguments.
+  Status AddInputEdges(const CondArgs& cond_args, Node* if_node);
+
+  // Adds all output edges from the `if_node`.
+  Status AddOutputEdges(const std::vector<Node*>& outputs, Node* if_node);
+
+  // Removes all nodes from the graph that are part of cluster.
+  void RemoveClusterNodes(Cluster* cluster);
+
+  // Removes all argument nodes that are unused.
+  template <class T>
+  void RemoveUnusedArgs(const T& args);
+
+  // Removes all Merge nodes that are unused.
+  void RemoveUnusedMergeNodes(Cluster* merge_cluster);
+
+  // Returns the representative member of the corresponding cluster.
+  ClusterHandle Representative(const Node* node) {
+    return clusters_.at(node).Get();
+  }
+
+  ClusteredGraph clustered_graph_;
+  ClusterHandle::Vector clusters_;
+  std::unordered_set<Node*> merge_nodes_;
+  std::unordered_set<Node*> switch_nodes_;
+  FunctionLibraryDefinition* library_;
+  Graph* graph_;
+};
+
+std::ostream& operator<<(std::ostream& os,
+                         const FunctionalizeCond::ClusterHandle& c) {
+  os << c.ToString();
+  return os;
+}
+
+// Returns a dot representation of the clustered graph showing the connections
+// between the nodes and the nodes in each cluster.
+string DebugString(const Graph& graph,
+                   FunctionalizeCond::ClusterHandle::Vector* clusters) {
+  string ret = "digraph {\ncompound=true;labeljust=\"r\";\n";
+  std::map<FunctionalizeCond::ClusterHandle, string> subgraphs;
+  for (Node* n : graph.nodes()) {
+    if (n->IsOp()) {
+      strings::StrAppend(&subgraphs[clusters->at(n).Get()], n->id(),
+                         " [label=\"", n->name(), "\"];\n");
+    }
+  }
+  for (auto kv : subgraphs) {
+    strings::StrAppend(&ret, "subgraph cluster_", kv.first.ToString(), " {\n",
+                       "label = \"", kv.first.ToString(), "\";\n", kv.second,
+                       "}\n");
+  }
+  for (Node* n : graph.nodes()) {
+    if (!n->IsOp()) {
+      continue;
+    }
+    for (Node* in : n->in_nodes()) {
+      if (in->IsOp()) {
+        strings::StrAppend(&ret, in->id(), " -> ", n->id(), ";\n");
+      }
+    }
+  }
+  return strings::StrCat(ret, "}");
+}
+
+bool IsDeadSwitch(const Node* node) {
+  for (const Edge* e : node->out_edges()) {
+    const Node* dst = e->dst();
+    if (!dst->IsIdentity()) {
+      return false;
+    }
+    for (const Edge* ee : dst->out_edges()) {
+      if (!ee->IsControlEdge() || !ee->dst()->IsSink()) {
+        return false;
+      }
+    }
+  }
+  return true;
+}
+
+void FunctionalizeCond::CreateClusters() {
+  for (Node* node : graph_->nodes()) {
+    if (!node->IsOp()) {
+      continue;
+    }
+    if (IsSwitch(node)) {
+      switch_nodes_.insert(node);
+    } else if (IsMerge(node)) {
+      merge_nodes_.insert(node);
+    }
+    ClusterHandle& cluster = clusters_.at(node).Get();
+    cluster = ClusterHandle(node->id());
+  }
+
+  // If there are no Merge nodes, then terminate.
+  if (merge_nodes_.empty()) {
+    return;
+  }
+
+  // Remove all dead Switch nodes.
+  RemoveUnusedArgs(switch_nodes_);
+
+  // All parent_'s are still nullptr so clusters_ may still be resized. Resize
+  // conservatively assuming all merge nodes become XlaIf nodes.
+  clusters_.resize(clusters_.size() + merge_nodes_.size());
+
+  // Merge a cluster with its input, unless the input is a Switch node or the
+  // node is a Merge node.
+  for (const Node* node : graph_->nodes()) {
+    if (IsMerge(node) || !node->IsOp()) {
+      continue;
+    }
+    for (const Node* in : node->in_nodes()) {
+      if (!IsSwitch(in) && in->IsOp()) {
+        clusters_.at(node).Merge(&clusters_.at(in));
+      }
+    }
+  }
+}
+
+void FunctionalizeCond::ContractEdge(Cluster* from, Cluster* to) {
+  VLOG(3) << "ContractEdge from = " << from->representative
+          << " to = " << to->representative;
+  if (from->representative == to->representative) {
+    return;
+  }
+  to->merge_nodes.insert(from->merge_nodes.begin(), from->merge_nodes.end());
+  from->merge_nodes.clear();
+  to->switch_nodes.insert(from->switch_nodes.begin(), from->switch_nodes.end());
+  from->switch_nodes.clear();
+
+  for (Cluster* from_out : from->out_nodes) {
+    from_out->in_nodes.erase(from);
+    if (from_out->representative != to->representative) {
+      from_out->in_nodes.insert(to);
+      to->out_nodes.insert(from_out);
+    }
+  }
+  from->out_nodes.clear();
+
+  for (Cluster* from_in : from->in_nodes) {
+    from_in->out_nodes.erase(from);
+    if (from_in->representative != to->representative) {
+      from_in->out_nodes.insert(to);
+      to->in_nodes.insert(from_in);
+    }
+  }
+  from->in_nodes.clear();
+
+  to->in_nodes.erase(from);
+  to->out_nodes.erase(from);
+  clusters_.at(to->representative).Merge(&clusters_.at(from->representative));
+  from->visited = true;
+}
+
+void FunctionalizeCond::CreateClusteredGraph() {
+  auto update_cluster_for_node = [this](Node* node) -> Cluster& {
+    ClusterHandle repr = Representative(node);
+    Cluster& cluster_node = clustered_graph_[repr];
+    cluster_node.representative = repr;
+    for (const Node* in : node->in_nodes()) {
+      ClusterHandle other_repr = Representative(in);
+      // Skip source, sink and internal edges.
+      if (!in->IsOp() || other_repr == repr) {
+        continue;
+      }
+      Cluster& cluster_node_in = clustered_graph_[other_repr];
+      cluster_node.in_nodes.insert(&cluster_node_in);
+      cluster_node_in.out_nodes.insert(&cluster_node);
+      cluster_node_in.representative = other_repr;
+    }
+    for (const Node* out : node->out_nodes()) {
+      ClusterHandle other_repr = Representative(out);
+      // Skip source, sink and internal edges.
+      if (!out->IsOp() || other_repr == repr) {
+        continue;
+      }
+      Cluster& cluster_node_out = clustered_graph_[other_repr];
+      cluster_node.out_nodes.insert(&cluster_node_out);
+      cluster_node_out.in_nodes.insert(&cluster_node);
+      cluster_node_out.representative = other_repr;
+    }
+    return cluster_node;
+  };
+  for (Node* node : switch_nodes_) {
+    update_cluster_for_node(node).switch_nodes.insert(node);
+  }
+  for (Node* node : merge_nodes_) {
+    update_cluster_for_node(node).merge_nodes.insert(node);
+  }
+
+  // Merge Merge nodes with common input together.
+  for (Node* node : merge_nodes_) {
+    Cluster& cluster = clustered_graph_.at(Representative(node));
+    for (const Node* in : node->in_nodes()) {
+      if (!in->IsOp()) {
+        continue;
+      }
+      Cluster& cluster_node_in = clustered_graph_.at(Representative(in));
+      for (auto it = cluster_node_in.out_nodes.begin();
+           it != cluster_node_in.out_nodes.end();) {
+        ContractEdge(*it++, &cluster);
+      }
+    }
+  }
+
+  VLOG(3) << "ClusteredGraph: " << DebugString(*graph_, &clusters_);
+}
+
+gtl::optional<FunctionalizeCond::Cluster*> FunctionalizeCond::GetSwitchCluster(
+    const Cluster& merge_cluster) {
+  VLOG(3) << "GetSwitchCluster for " << merge_cluster.representative;
+  gtl::optional<Cluster*> switch_cluster;
+  if (merge_cluster.in_nodes.size() != 2) {
+    return gtl::nullopt;
+  }
+  for (const Cluster* in : merge_cluster.in_nodes) {
+    if (in->in_nodes.size() != 1) {
+      return gtl::nullopt;
+    }
+    for (auto inin : in->in_nodes) {
+      if (switch_cluster.has_value()) {
+        if (*switch_cluster != inin) {
+          return gtl::nullopt;
+        }
+      } else {
+        switch_cluster = inin;
+      }
+    }
+  }
+  return switch_cluster;
+}
+
+xla::StatusOr<FunctionalizeCond::CondArgs> FunctionalizeCond::DetermineCondArgs(
+    const Cluster& merge_cluster, const Cluster& switch_cluster) {
+  VLOG(2) << "DetermineCondArgs for " << merge_cluster.representative
+          << " with switch cluster " << switch_cluster.representative;
+  CondArgs ret;
+  auto feeds_into_branch_cluster = [&](Node* switch_cluster) {
+    for (Node* out : switch_cluster->out_nodes()) {
+      ClusterHandle repr = Representative(out);
+      for (Cluster* in : merge_cluster.in_nodes) {
+        if (repr == in->representative) {
+          return true;
+        }
+      }
+    }
+    return false;
+  };
+  for (Node* switch_cluster_node : switch_cluster.switch_nodes) {
+    if (!feeds_into_branch_cluster(switch_cluster_node)) {
+      continue;
+    }
+
+    Node* tmp;
+    TF_RETURN_IF_ERROR(switch_cluster_node->input_node(1, &tmp));
+    if (ret.conditional == nullptr) {
+      ret.conditional = tmp;
+    } else if (ret.conditional != tmp) {
+      return errors::Unimplemented(
+          "Switch statements with different conditionals cannot be "
+          "converted into functional conditional.");
+    }
+    ret.args.insert(switch_cluster_node);
+  }
+  return ret;
+}
+
+xla::StatusOr<Node*> FunctionalizeCond::BuildAndAddXlaIfOp(
+    const CondArgs& cond_args, const Cluster& merge_cluster,
+    const std::vector<Node*>& outputs) {
+  VLOG(2) << "Build if op for {"
+          << str_util::Join(merge_cluster.merge_nodes, ", ",
+                            [](string* out, const Node* node) {
+                              strings::StrAppend(out, node->name());
+                            })
+          << "}";
+  NodeDef if_def;
+  // Create a new If node using the name of the merge node.
+  NodeDefBuilder builder(
+      strings::StrCat((*merge_cluster.merge_nodes.begin())->name(), "_If"),
+      "XlaIf");
+  string branch[] = {"else_branch", "then_branch"};
+  for (int i = 0; i < 2; ++i) {
+    static std::atomic<int64> sequence_num(0LL);
+    int64 id = ++sequence_num;
+
+    NameAttrList body_name;
+    body_name.set_name(
+        strings::StrCat("_functionalize_if_", branch[i], "_", id));
+    auto body = xla::MakeUnique<Graph>(graph_->op_registry());
+    TF_RETURN_IF_ERROR(
+        ExtractBody(cond_args, merge_cluster, outputs, i, body.get()));
+    FunctionDef body_fdef;
+    TF_RETURN_IF_ERROR(GraphToFunctionDef(*body, body_name.name(), &body_fdef));
+    TF_RETURN_IF_ERROR(library_->AddFunctionDef(body_fdef));
+    builder.Attr(branch[i], body_name);
+  }
+
+  // Build input type.
+  std::vector<NodeDefBuilder::NodeOut> inputs;
+  DataTypeVector in_arg_types;
+  for (const Node* arg : cond_args.args) {
+    const Edge* in_edge;
+    TF_RETURN_IF_ERROR(arg->input_edge(0, &in_edge));
+    if (in_edge->IsControlEdge()) {
+      builder.ControlInput(in_edge->src()->name());
+    } else {
+      DataType dtype = arg->input_type(0);
+      inputs.emplace_back(NodeDefBuilder::NodeOut(
+          in_edge->src()->name(), in_edge->src_output(), dtype));
+      in_arg_types.push_back(dtype);
+    }
+  }
+  builder.Attr("Tin", in_arg_types);
+
+  // Build output type.
+  DataTypeVector out_type;
+  for (const Node* merge : merge_cluster.merge_nodes) {
+    DataType dtype = merge->output_type(0);
+    out_type.push_back(dtype);
+  }
+  builder.Attr("Tout", out_type);
+
+  builder.Attr("Tcond", DT_BOOL);
+  builder.Device(cond_args.conditional->assigned_device_name());
+  // Conditional should be the first input ...
+  builder.Input(NodeDefBuilder::NodeOut(cond_args.conditional->name(), 0,
+                                        cond_args.conditional->output_type(0)));
+  // ... followed by the other inputs.
+  builder.Input(inputs);
+
+  TF_RETURN_IF_ERROR(builder.Finalize(&if_def));
+  TF_ASSIGN_OR_RETURN(Node * if_node, AddNode(if_def, graph_));
+  return if_node;
+}
+
+void FunctionalizeCond::RemoveClusterNodes(Cluster* cluster) {
+  VLOG(3) << "RemoveClusterNodes for " << cluster->representative;
+  ClusterHandle repr = cluster->representative;
+  std::deque<Node*> to_delete;
+  for (Node* node : graph_->nodes()) {
+    if (Representative(node) == repr) {
+      to_delete.push_back(node);
+    }
+  }
+  for (Node* n : to_delete) {
+    graph_->RemoveNode(n);
+  }
+}
+
+template <class T>
+void FunctionalizeCond::RemoveUnusedArgs(const T& args) {
+  VLOG(2) << "RemoveUnusedArgs among: "
+          << str_util::Join(args, ", ", [](string* output, const Node* node) {
+               strings::StrAppend(output, node->name());
+             });
+
+  std::deque<Node*> to_delete;
+  for (Node* arg : args) {
+    if (IsDeadSwitch(arg)) {
+      to_delete.push_back(arg);
+      for (Node* n : arg->out_nodes()) {
+        to_delete.push_back(n);
+      }
+    }
+  }
+  for (Node* n : to_delete) {
+    switch_nodes_.erase(n);
+    auto it = clustered_graph_.find(Representative(n));
+    if (it != clustered_graph_.end()) {
+      it->second.switch_nodes.erase(n);
+    }
+    graph_->RemoveNode(n);
+  }
+}
+
+Status FunctionalizeCond::ExtractBody(const CondArgs& cond_args,
+                                      const Cluster& merge_cluster,
+                                      const std::vector<Node*>& outputs,
+                                      int input_edge, Graph* body) {
+  VLOG(2) << "ExtractBody for " << merge_cluster.representative;
+  std::vector<bool> squash_src_outputs(graph_->num_node_ids(), false);
+  std::vector<Node*> node_map(graph_->num_node_ids(), nullptr);
+  int arg_count = 0;
+  for (const auto* arg : cond_args.args) {
+    DataType dtype = arg->input_type(0);
+    TF_ASSIGN_OR_RETURN(Node * arg_node,
+                        BuildArgNode(body, dtype, arg_count++));
+    if (dtype == DT_RESOURCE) {
+      bool constant;
+      TF_RETURN_IF_ERROR(GetNodeAttr(arg->attrs(), "is_constant", &constant));
+      TF_RET_CHECK(constant);
+    }
+    node_map.at(arg->id()) = arg_node;
+    squash_src_outputs.at(arg->id()) = true;
+  }
+
+  std::vector<Node*> stack;
+  stack.reserve(outputs.size());
+  for (int j = 0; j < outputs.size(); ++j) {
+    Node* node = outputs[j];
+    TF_ASSIGN_OR_RETURN(node_map.at(node->id()),
+                        BuildRetvalNode(body, node->output_type(0),
+                                        /*index=*/j));
+    Node* in;
+    TF_RETURN_IF_ERROR(node->input_node(input_edge, &in));
+    if (node_map.at(in->id()) == nullptr) {
+      node_map.at(in->id()) = body->CopyNode(in);
+    }
+    body->AddEdge(node_map.at(in->id()), j, node_map.at(node->id()), 0);
+    stack.push_back(in);
+  }
+
+  return CopySubgraph(*graph_, nullptr, stack, squash_src_outputs, &node_map,
+                      body);
+}
+
+Status FunctionalizeCond::AddInputEdges(const CondArgs& cond_args,
+                                        Node* if_node) {
+  VLOG(3) << "AddInputEdges for " << if_node->name();
+  int i = 0;
+  graph_->AddEdge(cond_args.conditional, 0, if_node, i++);
+  for (const Node* arg : cond_args.args) {
+    const Edge* in_edge;
+    TF_RETURN_IF_ERROR(arg->input_edge(0, &in_edge));
+    if (in_edge->IsControlEdge()) {
+      graph_->AddControlEdge(in_edge->src(), if_node);
+    } else {
+      graph_->AddEdge(in_edge->src(), in_edge->src_output(), if_node, i++);
+    }
+  }
+  return Status::OK();
+}
+
+Status FunctionalizeCond::AddOutputEdges(const std::vector<Node*>& outputs,
+                                         Node* if_node) {
+  VLOG(3) << "AddOutputEdges for " << if_node->name();
+  for (int i = 0; i < outputs.size(); ++i) {
+    Node* node = outputs[i];
+    std::vector<const Edge*> edges(node->out_edges().begin(),
+                                   node->out_edges().end());
+    for (const Edge* edge : edges) {
+      Node* dst = edge->dst();
+      int dst_input = edge->dst_input();
+
+      if (edge->src_output() > 0) {
+        return errors::Unimplemented("Output of index (", edge->src_output(),
+                                     ") of merge node ", node->name());
+      }
+      graph_->RemoveEdge(edge);
+
+      int src_output =
+          dst_input == Graph::kControlSlot ? Graph::kControlSlot : i;
+      graph_->AddEdge(if_node, src_output, dst, dst_input);
+    }
+  }
+  return Status::OK();
+}
+
+void FunctionalizeCond::RemoveUnusedMergeNodes(Cluster* merge_cluster) {
+  VLOG(3) << "RemoveUnusedMergeNodes for " << merge_cluster->representative;
+  // Remove all merge nodes now dead post extraction of If.
+  for (auto it = merge_cluster->merge_nodes.begin();
+       it != merge_cluster->merge_nodes.end();) {
+    Node* node = *it;
+    if (node->out_edges().empty()) {
+      graph_->RemoveNode(node);
+      merge_cluster->merge_nodes.erase(*it++);
+    }
+  }
+}
+
+Status FunctionalizeCond::ConvertMergeToXlaIf(Cluster* merge_cluster) {
+  VLOG(1) << "ConvertMergeToXlaIf for " << merge_cluster->representative;
+  gtl::optional<Cluster*> switch_cluster = GetSwitchCluster(*merge_cluster);
+  if (!switch_cluster.has_value()) {
+    return errors::FailedPrecondition(
+        "Merge cluster was not part of a simple conditional in the clustered "
+        "graph. Graph nodes in merge cluster {",
+        str_util::Join(merge_cluster->merge_nodes, ", ",
+                       [](string* output, Node* node) {
+                         strings::StrAppend(output, node->name());
+                       }),
+        "}");
+  }
+  TF_ASSIGN_OR_RETURN(auto cond_args,
+                      DetermineCondArgs(*merge_cluster, **switch_cluster));
+
+  // Sort the outputs by ID to produce more stable output.
+  std::vector<Node*> outputs(merge_cluster->merge_nodes.begin(),
+                             merge_cluster->merge_nodes.end());
+  std::sort(
+      outputs.begin(), outputs.end(),
+      [](const Node* lhs, const Node* rhs) { return lhs->id() < rhs->id(); });
+
+  // Extract bodies and builds a If operator.
+  TF_ASSIGN_OR_RETURN(Node * if_node,
+                      BuildAndAddXlaIfOp(cond_args, *merge_cluster, outputs));
+  TF_RETURN_IF_ERROR(AddInputEdges(cond_args, if_node));
+  TF_RETURN_IF_ERROR(AddOutputEdges(outputs, if_node));
+
+  // Remove the old nodes from the graph_ and contract the edges of the
+  // clustered graph.
+  for (auto in : merge_cluster->in_nodes) {
+    RemoveClusterNodes(in);
+  }
+  RemoveUnusedArgs(cond_args.args);
+  auto in_nodes = merge_cluster->in_nodes;
+  for (auto it = in_nodes.begin(); it != in_nodes.end();) {
+    ContractEdge(*it++, merge_cluster);
+  }
+  ContractEdge(*switch_cluster, merge_cluster);
+  RemoveUnusedMergeNodes(merge_cluster);
+  clusters_[if_node].Get() = ClusterHandle(merge_cluster->representative);
+
+  return Status::OK();
+}
+
+std::vector<std::pair<int, FunctionalizeCond::Cluster*>>
+FunctionalizeCond::SortedMergeNodes() {
+  VLOG(2) << "ProcessClusteredGraph";
+  std::stack<std::pair<int, Cluster*>> stack;
+  for (auto& c : clustered_graph_) {
+    if (c.second.in_nodes.empty()) {
+      stack.push({0, &c.second});
+    }
+  }
+
+  // Perform a depth-first traversal of the clustered graph computing the
+  // switch-merge depth.
+  std::vector<std::pair<int, Cluster*>> queue;
+  std::unordered_set<Cluster*> visited;
+  while (!stack.empty()) {
+    Cluster* n = stack.top().second;
+    size_t depth = stack.top().first;
+    stack.pop();
+
+    auto inserted = visited.insert(n);
+    if (!inserted.second) {
+      continue;
+    }
+
+    size_t new_depth = depth;
+    if (!n->merge_nodes.empty()) {
+      queue.emplace_back(depth, n);
+      --new_depth;
+    }
+    if (!n->switch_nodes.empty()) {
+      ++new_depth;
+    }
+    for (Cluster* e : n->out_nodes) {
+      stack.emplace(new_depth, e);
+    }
+  }
+
+  // Sort in reverse order of switch-merge depth with ties broken by the
+  // ClusterHandle.
+  std::sort(queue.begin(), queue.end(),
+            [](const std::pair<int, Cluster*>& lhs,
+               const std::pair<int, Cluster*>& rhs) {
+              return std::tie(lhs.first, lhs.second->representative) >
+                     std::tie(rhs.first, rhs.second->representative);
+            });
+
+  return queue;
+}
+
+Status FunctionalizeCond::Functionalize(Graph* graph,
+                                        FunctionLibraryDefinition* library) {
+  VLOG(1) << "FunctionalizeCond::Functionalize";
+  FunctionalizeCond fc(graph, library);
+  fc.CreateClusters();
+  if (fc.NoConditionals()) {
+    return Status::OK();
+  }
+  fc.CreateClusteredGraph();
+
+  auto queue = fc.SortedMergeNodes();
+  for (auto it = queue.begin(); it != queue.end();) {
+    Cluster* merge_cluster = (*it).second;
+    ++it;
+    TF_RETURN_IF_ERROR(fc.ConvertMergeToXlaIf(merge_cluster));
+  }
+  return Status::OK();
+}
+
 }  // namespace
 
 // Transformation that converts Tensorflow's graph control flow constructs into
@@ -577,7 +1315,10 @@ Status FunctionalizeControlFlow(Graph* graph,
     }
   }
 
-  return Status::OK();
+  // FunctionalizeControlFlow is invoked for every function, so the loops's
+  // bodies and conditionals that were extracted into functions will be handled
+  // in successive invocations.
+  return FunctionalizeCond::Functionalize(graph, library);
 }
 
 }  // namespace tensorflow
diff --git a/tensorflow/compiler/tf2xla/functionalize_control_flow.h b/tensorflow/compiler/tf2xla/functionalize_control_flow.h
index 1535dc80b0..4d4ee3054c 100644
--- a/tensorflow/compiler/tf2xla/functionalize_control_flow.h
+++ b/tensorflow/compiler/tf2xla/functionalize_control_flow.h
@@ -23,7 +23,6 @@ namespace tensorflow {
 
 // Transformation that converts tf.while_loop() loops into functional While
 // operators, suitable for XLA compilation.
-// TODO(b/36470387): add support for conditionals.
 Status FunctionalizeControlFlow(Graph* graph,
                                 FunctionLibraryDefinition* library);
 
diff --git a/tensorflow/compiler/tf2xla/functionalize_control_flow_test.cc b/tensorflow/compiler/tf2xla/functionalize_control_flow_test.cc
index 914c8999a6..8f155ca85e 100644
--- a/tensorflow/compiler/tf2xla/functionalize_control_flow_test.cc
+++ b/tensorflow/compiler/tf2xla/functionalize_control_flow_test.cc
@@ -23,6 +23,7 @@ limitations under the License.
 #include "tensorflow/compiler/tf2xla/cc/ops/functional_ops.h"
 #include "tensorflow/compiler/tf2xla/test_util.h"
 #include "tensorflow/compiler/xla/status_macros.h"
+#include "tensorflow/core/common_runtime/function.h"
 #include "tensorflow/core/framework/function.h"
 #include "tensorflow/core/framework/node_def_util.h"
 #include "tensorflow/core/framework/op.h"
@@ -35,6 +36,134 @@ limitations under the License.
 namespace tensorflow {
 namespace {
 
+// Returns the names of the "then" and "else" functions for the XlaIf node in a
+// graph.
+Status FindIfThenAndElse(const GraphDef& graph, NameAttrList* then_fn,
+                         NameAttrList* else_fn) {
+  for (const NodeDef& node : graph.node()) {
+    if (node.op() == "XlaIf") {
+      const NameAttrList* result;
+      TF_RETURN_IF_ERROR(GetNodeAttr(node, "then_branch", &result));
+      *then_fn = *result;
+      TF_RETURN_IF_ERROR(GetNodeAttr(node, "else_branch", &result));
+      *else_fn = *result;
+      return Status::OK();
+    }
+  }
+  return errors::NotFound("No XlaIf node found in graph");
+}
+
+// Graph:
+// x = array_ops.placeholder(dtypes.int32)
+// y = array_ops.placeholder(dtypes.int32)
+// z = control_flow_ops.cond(
+//     math_ops.less(y, x), lambda: math_ops.multiply(y, 17),
+//     lambda: math_ops.add(x, 23))
+TEST(FunctionalizeControlFlow, Conditional) {
+  Graph graph(OpRegistry::Global());
+  {
+    Scope scope = Scope::NewRootScope().ExitOnError();
+
+    auto x = ops::Placeholder(scope.WithOpName("x"), DT_INT32);
+    auto y = ops::Placeholder(scope.WithOpName("y"), DT_INT32);
+    auto less = ops::Less(scope.WithOpName("cond/Less"), y, x);
+    auto switch_1 = ops::Switch(scope.WithOpName("cond/Switch"), less, less);
+
+    auto identity_t =
+        ops::Identity(scope.WithOpName("cond/Identity"), switch_1.output_true);
+    auto seventeen = ops::Const<int32>(
+        scope.WithOpName("cond").WithControlDependencies(identity_t), 17);
+    auto switch_2 = ops::Switch(scope.WithOpName("cond/Switch"), y, less);
+    auto mul = ops::Multiply(scope.WithOpName("cond/Mul"), switch_2.output_true,
+                             seventeen);
+
+    auto identity_f =
+        ops::Identity(scope.WithOpName("cond/Identity"), switch_1.output_false);
+    auto twenty_three = ops::Const<int32>(
+        scope.WithOpName("cond").WithControlDependencies(identity_f), 23);
+    auto switch_3 = ops::Switch(scope.WithOpName("cond/Switch"), x, less);
+    auto add = ops::Add(scope.WithOpName("cond/false/add"),
+                        switch_3.output_false, twenty_three);
+
+    auto merge = ops::Merge(scope.WithOpName("cond/Merge"),
+                            std::initializer_list<Input>{add, mul});
+
+    TF_EXPECT_OK(scope.ToGraph(&graph));
+  }
+
+  FunctionLibraryDefinition library(OpRegistry::Global(), {});
+  TF_ASSERT_OK(FunctionalizeControlFlow(&graph, &library));
+
+  GraphDef graph_def;
+  graph.ToGraphDef(&graph_def);
+  NameAttrList then_fn;
+  NameAttrList else_fn;
+  TF_EXPECT_OK(FindIfThenAndElse(graph_def, &then_fn, &else_fn));
+  InstantiationResultForTest else_result;
+  TF_EXPECT_OK(
+      InstantiateFunctionForTest(else_fn.name(), library, &else_result));
+
+  // Outer graph
+  {
+    Scope scope = Scope::NewRootScope().ExitOnError();
+    auto y = ops::Placeholder(scope.WithOpName("y"), DT_INT32);
+    auto x = ops::Placeholder(scope.WithOpName("x"), DT_INT32);
+    auto less = ops::Less(scope.WithOpName("cond/Less"), y, x);
+    auto if_op = ops::XlaIf(scope.WithOpName("cond/Merge_If"), less,
+                            std::initializer_list<Input>{x, y, less}, then_fn,
+                            else_fn, {DT_INT32});
+    GraphDef expected;
+    TF_EXPECT_OK(scope.ToGraphDef(&expected));
+    TF_EXPECT_GRAPH_EQ(expected, graph_def);
+  }
+
+  // then body.
+  {
+    Scope scope = Scope::NewRootScope().ExitOnError();
+    auto arg_0 = ops::_Arg(scope.WithOpName("_arg0"), DT_INT32, 0);
+    auto arg_1 = ops::_Arg(scope.WithOpName("_arg1"), DT_INT32, 1);
+    auto arg_2 = ops::_Arg(scope.WithOpName("_arg2"), DT_BOOL, 2);
+    auto identity = ops::Identity(scope.WithOpName("cond/Identity"), arg_2);
+    auto cond = ops::Const(
+        scope.WithOpName("cond").WithControlDependencies(identity), 17);
+    auto mul = ops::Mul(scope.WithOpName("cond/Mul"), arg_1, cond);
+    auto retval0 = ops::_Retval(scope.WithOpName("_retval0_RetVal"), mul, 0);
+
+    GraphDef expected;
+    TF_EXPECT_OK(scope.ToGraphDef(&expected));
+
+    InstantiationResultForTest result;
+    TF_EXPECT_OK(InstantiateFunctionForTest(then_fn.name(), library, &result));
+
+    EXPECT_EQ(DataTypeVector{DT_INT32}, result.ret_types);
+    EXPECT_EQ((DataTypeVector{DT_INT32, DT_INT32, DT_BOOL}), result.arg_types);
+    TF_EXPECT_GRAPH_EQ(expected, result.gdef);
+  }
+
+  // else body.
+  {
+    Scope scope = Scope::NewRootScope().ExitOnError();
+    auto arg_0 = ops::_Arg(scope.WithOpName("_arg0"), DT_INT32, 0);
+    auto arg_1 = ops::_Arg(scope.WithOpName("_arg1"), DT_INT32, 1);
+    auto arg_2 = ops::_Arg(scope.WithOpName("_arg2"), DT_BOOL, 2);
+    auto identity = ops::Identity(scope.WithOpName("cond/Identity_1"), arg_2);
+    auto cond_1 = ops::Const(
+        scope.WithOpName("cond_1").WithControlDependencies(identity), 23);
+    auto add = ops::Add(scope.WithOpName("cond/false/add"), arg_0, cond_1);
+    auto retval0 = ops::_Retval(scope.WithOpName("_retval0_RetVal"), add, 0);
+
+    GraphDef expected;
+    TF_EXPECT_OK(scope.ToGraphDef(&expected));
+
+    InstantiationResultForTest result;
+    TF_EXPECT_OK(InstantiateFunctionForTest(else_fn.name(), library, &result));
+
+    EXPECT_EQ(DataTypeVector{DT_INT32}, result.ret_types);
+    EXPECT_EQ((DataTypeVector{DT_INT32, DT_INT32, DT_BOOL}), result.arg_types);
+    TF_EXPECT_GRAPH_EQ(expected, result.gdef);
+  }
+}
+
 // Returns the names of the "cond" and "body" functions for the While node
 // in a graph.
 Status FindWhileCondAndBody(const GraphDef& graph, NameAttrList* cond,
diff --git a/tensorflow/compiler/tf2xla/ops/functional_ops.cc b/tensorflow/compiler/tf2xla/ops/functional_ops.cc
index c1005405f9..4a669f8e6e 100644
--- a/tensorflow/compiler/tf2xla/ops/functional_ops.cc
+++ b/tensorflow/compiler/tf2xla/ops/functional_ops.cc
@@ -34,14 +34,41 @@ output = input; While (Cond(output)) { output = Body(output) }
 input: A list of input tensors whose types are T.
 output: A list of output tensors whose types are T.
 cond: A function takes 'input' and returns a tensor.  If the tensor is
-    a scalar of non-boolean, the scalar is converted to a boolean
-    according to the following rule: if the scalar is a numerical
-    value, non-zero means True and zero means False; if the scalar is
-    a string, non-empty means True and empty means False. If the
-    tensor is not a scalar, non-emptiness means True and False
-    otherwise.
+      a scalar of non-boolean, the scalar is converted to a boolean
+      according to the following rule: if the scalar is a numerical
+      value, non-zero means True and zero means False; if the scalar is
+      a string, non-empty means True and empty means False. If the
+      tensor is not a scalar, non-emptiness means True and False
+      otherwise.
 body: A function that takes a list of tensors and returns another
       list of tensors. Both lists have the same types as specified by T.
 )doc");
 
+// TODO(b/37549631) setting the If Op to always be stateful is too
+// conservative.
+REGISTER_OP("XlaIf")
+    .Input("cond: Tcond")
+    .Input("inputs: Tin")
+    .Output("output: Tout")
+    .Attr("Tcond: type")
+    .Attr("then_branch: func")
+    .Attr("else_branch: func")
+    .Attr("Tin: list(type) >= 0")
+    .Attr("Tout: list(type) >= 0")
+    .SetIsStateful()
+    .SetShapeFn(shape_inference::UnknownShape)
+    .Doc(R"doc(
+output = cond ? then_branch(inputs) : else_branch(inputs).
+
+cond: A boolean scalar.
+inputs: A list of input tensors.
+output: A list of tensors returned by either then_branch(inputs) or
+        else_branch(inputs). The input shapes of the then_branch and
+        else_branch must match.
+then_branch: A function takes 'inputs' and returns a list of tensors,
+             whose types are the same as what else_branch returns.
+else_branch: A function takes 'inputs' and returns a list of tensors.
+             whose types are the same as what then_branch returns.
+)doc");
+
 }  // namespace tensorflow
diff --git a/tensorflow/python/ops/control_flow_ops.py b/tensorflow/python/ops/control_flow_ops.py
index b341eab7ce..29aac913f0 100644
--- a/tensorflow/python/ops/control_flow_ops.py
+++ b/tensorflow/python/ops/control_flow_ops.py
@@ -1636,6 +1636,9 @@ class CondContext(ControlFlowContext):
         self._values.add(result.name)
       with ops.control_dependencies(None):
         result = _SwitchRefOrTensor(result, self._pred)[self._branch]
+        if self._outer_context:
+          self._outer_context.AddInnerOp(result.op)
+
       result.op.graph.prevent_fetching(result.op)
       # pylint: disable=protected-access
       result.op._set_control_flow_context(self)
@@ -1678,6 +1681,9 @@ class CondContext(ControlFlowContext):
     if self._outer_context or not IsLoopExit(op):
       op.graph.prevent_fetching(op)
 
+    if self._outer_context:
+      self._outer_context.AddInnerOp(op)
+
   def _ProcessOutputTensor(self, val):
     """Process an output tensor of a conditional branch."""
     real_val = val
-- 
GitLab


From b0e751a73d211872f8d937e5778b9e0e0a7b950b Mon Sep 17 00:00:00 2001
From: Yangzihao Wang <yangzihao@google.com>
Date: Thu, 5 Oct 2017 09:45:14 -0700
Subject: [PATCH 058/909] Add dilation rates support for
 ConvolutionDescriptor... ...in stream executor. In preparation for the
 support of native cudnn dilated convolution.

PiperOrigin-RevId: 171165137
---
 tensorflow/stream_executor/cuda/cuda_dnn.cc | 12 ++++++----
 tensorflow/stream_executor/dnn.cc           | 15 +++++++++----
 tensorflow/stream_executor/dnn.h            | 25 +++++++++++++++++++++
 3 files changed, 44 insertions(+), 8 deletions(-)

diff --git a/tensorflow/stream_executor/cuda/cuda_dnn.cc b/tensorflow/stream_executor/cuda/cuda_dnn.cc
index fc205f61fa..bf8380ebbd 100644
--- a/tensorflow/stream_executor/cuda/cuda_dnn.cc
+++ b/tensorflow/stream_executor/cuda/cuda_dnn.cc
@@ -583,6 +583,7 @@ class ScopedConvolutionDescriptor {
     }
     const auto& strides64 = convolution_descriptor.strides();
     const auto& padding64 = convolution_descriptor.padding();
+    const auto& dilations64 = convolution_descriptor.dilations();
     if (convolution_descriptor.pad_alignment() ==
         dnn::PadAlignment::kTensorFlowPadding) {
       LOG(ERROR) << "TensorFlow padding alignment is not supported.";
@@ -591,15 +592,19 @@ class ScopedConvolutionDescriptor {
     // cuDNN requires arrays of ints.
     std::vector<int> strides(convolution_descriptor.ndims());
     std::vector<int> padding(convolution_descriptor.ndims());
+    std::vector<int> dilations(convolution_descriptor.ndims());
     std::transform(strides64.cbegin(), strides64.cend(), strides.begin(),
                    &CheckedNarrowing<int64, int>);
     std::transform(padding64.cbegin(), padding64.cend(), padding.begin(),
                    &CheckedNarrowing<int64, int>);
-    std::vector<int> upscale(convolution_descriptor.ndims(), 1);
+    // TODO(yangzihao): Test with negative dilation to make sure that cudnn
+    // doesn't crash.
+    std::transform(dilations64.cbegin(), dilations64.cend(), dilations.begin(),
+                   &CheckedNarrowing<int64, int>);
 
     status = wrap::cudnnSetConvolutionNdDescriptor(
         parent_, handle_, convolution_descriptor.ndims(), padding.data(),
-        strides.data(), upscale.data(),
+        strides.data(), dilations.data(),
         // NOTE(keveman): cuDNN supports convolution and cross correlation.
         // However, almost all the use cases do cross correlation, so just
         // hard coding it here.
@@ -2982,7 +2987,6 @@ bool CudnnSupport::DoConvolveBackwardDataImpl(
       if (memory_limit_bytes < 0) {
         memory_limit_bytes = 0;
       }
-
       cudnnConvolutionBwdDataAlgo_t algo_to_use;
       cudnnStatus_t status = wrap::cudnnGetConvolutionBackwardDataAlgorithm(
           parent_, ToHandle(dnn_handle_),
@@ -2995,7 +2999,7 @@ bool CudnnSupport::DoConvolveBackwardDataImpl(
           /*algo=*/&algo_to_use);
       CHECK_EQ(status, CUDNN_STATUS_SUCCESS) << "Unable to find a suitable "
                                                 "algorithm for doing backward "
-                                                "filter convolution";
+                                                "data convolution";
       return algo_to_use;
     };
 
diff --git a/tensorflow/stream_executor/dnn.cc b/tensorflow/stream_executor/dnn.cc
index ed9bdf2bc2..2c40e18f5c 100644
--- a/tensorflow/stream_executor/dnn.cc
+++ b/tensorflow/stream_executor/dnn.cc
@@ -424,6 +424,7 @@ int64 FilterDescriptor::ComputeWeightCount() const {
 ConvolutionDescriptor::ConvolutionDescriptor(int ndims)
     : zero_padding_(ndims, 0),
       filter_strides_(ndims, 1),
+      dilation_rates_(ndims, 1),
       pad_alignment_(PadAlignment::kDefault),
       ndims_(ndims) {}
 
@@ -435,15 +436,18 @@ ConvolutionDescriptor::~ConvolutionDescriptor() {}
 string ConvolutionDescriptor::ToString() const {
   string padding;
   string strides;
+  string dilations;
   for (int i = 0; i < ndims_; i++) {
     port::Appendf(&padding, "%lld ", zero_padding_[i]);
     port::Appendf(&strides, "%lld ", filter_strides_[i]);
+    port::Appendf(&dilations, "%lld ", dilation_rates_[i]);
   }
 
-  return port::Printf("{zero_padding: %s pad_alignment: %s filter_strides: %s}",
-                      padding.c_str(),
-                      PadAlignmentString(pad_alignment_).c_str(),
-                      strides.c_str());
+  return port::Printf(
+      "{zero_padding: %s pad_alignment: %s filter_strides: %s dilation_rates: "
+      "%s}",
+      padding.c_str(), PadAlignmentString(pad_alignment_).c_str(),
+      strides.c_str(), dilations.c_str());
 }
 
 string ConvolutionDescriptor::ToShortString() const {
@@ -455,6 +459,9 @@ string ConvolutionDescriptor::ToShortString() const {
   for (int i = 0; i < ndims_; i++) {
     port::Appendf(&desc, "_s%d:%lld", i, filter_strides_[i]);
   }
+  for (int i = 0; i < ndims_; i++) {
+    port::Appendf(&desc, "_d%d:%lld", i, dilation_rates_[i]);
+  }
   return desc;
 }
 
diff --git a/tensorflow/stream_executor/dnn.h b/tensorflow/stream_executor/dnn.h
index 4beb46090c..5fe523602a 100644
--- a/tensorflow/stream_executor/dnn.h
+++ b/tensorflow/stream_executor/dnn.h
@@ -487,6 +487,10 @@ string PadAlignmentString(PadAlignment alignment);
 //    window is moved in the "y dimension" according to this stride value.
 // - horizontal_filter_stride: analogous to the vertical stride above, but in
 //    the "x dimension".
+// - vertical_dilation_rate: there will be (vertical_dilation_rate - 1) skipped
+//   cells between each filter element in the "y dimension".
+// - horizontal_dilation_rate: there will be (horizontal_dilation_rate - 1)
+//   skipped cells between each filter element in the "x dimension".
 class ConvolutionDescriptor {
  public:
   // By default construction, there is no zero-padding and the filter stride is
@@ -523,6 +527,18 @@ class ConvolutionDescriptor {
     SetDim(&filter_strides_, dim, value);
     return *this;
   }
+  ConvolutionDescriptor& set_vertical_dilation_rate(int64 value) {
+    SetDim(&dilation_rates_, DimIndex::Y, value);
+    return *this;
+  }
+  ConvolutionDescriptor& set_horizontal_dilation_rate(int64 value) {
+    SetDim(&dilation_rates_, DimIndex::X, value);
+    return *this;
+  }
+  ConvolutionDescriptor& set_dilation_rate(DimIndex dim, int64 value) {
+    SetDim(&dilation_rates_, dim, value);
+    return *this;
+  }
   ConvolutionDescriptor& set_pad_alignment(PadAlignment pad_alignment) {
     pad_alignment_ = pad_alignment;
     return *this;
@@ -539,19 +555,28 @@ class ConvolutionDescriptor {
   int64 horizontal_filter_stride() const {
     return GetDim(filter_strides_, DimIndex::X);
   }
+  int64 vertical_dilation_rate() const {
+    return GetDim(dilation_rates_, DimIndex::Y);
+  }
+  int64 horizontal_dilation_rate() const {
+    return GetDim(dilation_rates_, DimIndex::X);
+  }
 
   int zero_padding(DimIndex dim) const { return GetDim(zero_padding_, dim); }
   int filter_stride(DimIndex dim) const { return GetDim(filter_strides_, dim); }
+  int dilation_rate(DimIndex dim) const { return GetDim(dilation_rates_, dim); }
   PadAlignment pad_alignment() const { return pad_alignment_; }
   int ndims() const { return ndims_; }
 
   std::vector<int64> strides() const { return filter_strides_; }
+  std::vector<int64> dilations() const { return dilation_rates_; }
   std::vector<int64> padding() const { return zero_padding_; }
 
  private:
   // Stored as: .. y, x.
   std::vector<int64> zero_padding_;
   std::vector<int64> filter_strides_;
+  std::vector<int64> dilation_rates_;
   PadAlignment pad_alignment_;
   int ndims_;
   // TODO(leary) cudnn provides these fields, but need to characterize what
-- 
GitLab


From 09fa4a4e355171fa30f5793ff9eb1b61a4e34ed0 Mon Sep 17 00:00:00 2001
From: Yangzihao Wang <yangzihao@google.com>
Date: Thu, 5 Oct 2017 09:45:58 -0700
Subject: [PATCH 059/909] Fix ConvBackpropComputeDimensionsV2() interface.

PiperOrigin-RevId: 171165222
---
 tensorflow/core/kernels/conv_grad_ops.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/kernels/conv_grad_ops.h b/tensorflow/core/kernels/conv_grad_ops.h
index 3a3492304b..e068fb8684 100644
--- a/tensorflow/core/kernels/conv_grad_ops.h
+++ b/tensorflow/core/kernels/conv_grad_ops.h
@@ -248,7 +248,7 @@ Status ConvBackpropComputeDimensions(StringPiece label, int num_spatial_dims,
 Status ConvBackpropComputeDimensionsV2(
     StringPiece label, int num_spatial_dims, const TensorShape& input_shape,
     const TensorShape& filter_shape, const TensorShape& out_backprop_shape,
-    const std::vector<int32>& dilations, const std::vector<int32>& strides,
+    const gtl::ArraySlice<int32>& dilations, const std::vector<int32>& strides,
     Padding padding, TensorFormat data_format, ConvBackpropDimensions* dims);
 }  // namespace tensorflow
 
-- 
GitLab


From f97195c6f936ee3edd9ad2620c091b742bb45476 Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Thu, 5 Oct 2017 09:58:20 -0700
Subject: [PATCH 060/909] Use --config=monolithic for the Android CI build

---
 tensorflow/tools/ci_build/builds/android_full.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/tools/ci_build/builds/android_full.sh b/tensorflow/tools/ci_build/builds/android_full.sh
index 63250e0a4d..9d449241e8 100755
--- a/tensorflow/tools/ci_build/builds/android_full.sh
+++ b/tensorflow/tools/ci_build/builds/android_full.sh
@@ -40,7 +40,7 @@ rm -rf ${AAR_LIB_TMP}
 for CPU in ${CPUS//,/ }
 do
     echo "========== Building native libs for Android ${CPU} =========="
-    bazel build -c opt --cpu=${CPU} \
+    bazel build -c opt --config=monolithic --cpu=${CPU} \
         --crosstool_top=//external:android/crosstool \
         --host_crosstool_top=@bazel_tools//tools/cpp:toolchain \
         //tensorflow/core:android_tensorflow_lib \
@@ -62,7 +62,7 @@ done
 # in assets/ dir (see https://github.com/bazelbuild/bazel/issues/2334)
 # TODO(gunan): remove extra flags once sandboxing is enabled for all builds.
 echo "========== Building TensorFlow Android Jar and Demo =========="
-bazel --bazelrc=/dev/null build -c opt --fat_apk_cpu=${CPUS} \
+bazel --bazelrc=/dev/null build -c opt --config=monolithic --fat_apk_cpu=${CPUS} \
     --spawn_strategy=sandboxed --genrule_strategy=sandboxed \
     //tensorflow/contrib/android:android_tensorflow_inference_java \
     //tensorflow/contrib/android:android_tensorflow_inference_java.aar \
-- 
GitLab


From 7e7d55c0f5bae2380a76d39fbc51131f843c0320 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 5 Oct 2017 09:50:17 -0700
Subject: [PATCH 061/909] [tf.data] Iterator and data/nest documentation fixes

PiperOrigin-RevId: 171165796
---
 tensorflow/python/data/ops/iterator_ops.py | 8 ++++----
 tensorflow/python/data/util/nest.py        | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/tensorflow/python/data/ops/iterator_ops.py b/tensorflow/python/data/ops/iterator_ops.py
index d11112d004..d4f05a055a 100644
--- a/tensorflow/python/data/ops/iterator_ops.py
+++ b/tensorflow/python/data/ops/iterator_ops.py
@@ -138,21 +138,21 @@ class Iterator(object):
     This method allows you to define a "feedable" iterator where you can choose
     between concrete iterators by feeding a value in a @{tf.Session.run} call.
     In that case, `string_handle` would a @{tf.placeholder}, and you would feed
-    it with the value of @{tf.contrib.data.Iterator.string_handle} in each step.
+    it with the value of @{tf.data.Iterator.string_handle} in each step.
 
     For example, if you had two iterators that marked the current position in
     a training dataset and a test dataset, you could choose which to use in
     each step as follows:
 
     ```python
-    train_iterator = tf.contrib.data.Dataset(...).make_one_shot_iterator()
+    train_iterator = tf.data.Dataset(...).make_one_shot_iterator()
     train_iterator_handle = sess.run(train_iterator.string_handle())
 
-    test_iterator = tf.contrib.data.Dataset(...).make_one_shot_iterator()
+    test_iterator = tf.data.Dataset(...).make_one_shot_iterator()
     test_iterator_handle = sess.run(test_iterator.string_handle())
 
     handle = tf.placeholder(tf.string, shape=[])
-    iterator = tf.contrib.data.Iterator.from_string_handle(
+    iterator = tf.data.Iterator.from_string_handle(
         handle, train_iterator.output_types)
 
     next_element = iterator.get_next()
diff --git a/tensorflow/python/data/util/nest.py b/tensorflow/python/data/util/nest.py
index 83908d8a0e..421513cafc 100644
--- a/tensorflow/python/data/util/nest.py
+++ b/tensorflow/python/data/util/nest.py
@@ -106,7 +106,7 @@ def is_sequence(seq):
 
   NOTE(mrry): This differs from `tensorflow.python.util.nest.is_sequence()`,
   which *does* treat a Python list as a sequence. For ergonomic
-  reasons, `tf.contrib.data` users would prefer to treat lists as
+  reasons, `tf.data` users would prefer to treat lists as
   implict `tf.Tensor` objects, and dicts as (nested) sequences.
 
   Args:
-- 
GitLab


From 5f97262ae6f36000e141b01b33c55f8eb1ee94a1 Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Thu, 5 Oct 2017 09:50:49 -0700
Subject: [PATCH 062/909] Splits backprop.py in two files, one of which can be
 converted to C

PiperOrigin-RevId: 171165855
---
 tensorflow/python/eager/BUILD              |   8 +
 tensorflow/python/eager/backprop.py        | 380 ++++++---------------
 tensorflow/python/eager/custom_gradient.py |   2 +-
 tensorflow/python/eager/function.py        |   2 +-
 tensorflow/python/eager/imperative_grad.py | 227 ++++++++++++
 tensorflow/python/framework/ops.py         |   2 +-
 6 files changed, 335 insertions(+), 286 deletions(-)
 create mode 100644 tensorflow/python/eager/imperative_grad.py

diff --git a/tensorflow/python/eager/BUILD b/tensorflow/python/eager/BUILD
index 09ec4ee12b..4069ef1c70 100644
--- a/tensorflow/python/eager/BUILD
+++ b/tensorflow/python/eager/BUILD
@@ -339,7 +339,9 @@ py_library(
     srcs_version = "PY2AND3",
     visibility = ["//tensorflow:internal"],
     deps = [
+        ":imperative_grad",
         "//tensorflow/python:array_ops",
+        "//tensorflow/python:constant_op",
         "//tensorflow/python:dtypes",
         "//tensorflow/python:errors",
         "//tensorflow/python:framework_ops",
@@ -425,3 +427,9 @@ filegroup(
     ),
     visibility = ["//tensorflow:__subpackages__"],
 )
+
+py_library(
+    name = "imperative_grad",
+    srcs = ["imperative_grad.py"],
+    deps = [":tape"],
+)
diff --git a/tensorflow/python/eager/backprop.py b/tensorflow/python/eager/backprop.py
index 1d729cc2e1..3c84cbbd6f 100644
--- a/tensorflow/python/eager/backprop.py
+++ b/tensorflow/python/eager/backprop.py
@@ -18,7 +18,6 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import collections
 import functools
 import operator
 import threading
@@ -28,6 +27,7 @@ import six
 from tensorflow.python import pywrap_tensorflow
 from tensorflow.python.eager import context
 from tensorflow.python.eager import execute
+from tensorflow.python.eager import imperative_grad
 from tensorflow.python.eager import tape
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
@@ -36,288 +36,10 @@ from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
-from tensorflow.python.util import tf_contextlib
+from tensorflow.python.util import nest
 from tensorflow.python.util import tf_inspect
 
 
-# If over MIN_AGGREGATE_COUNT gradients are accumulated and the total
-# memory consumption is over MIN_AGGREGATE_BYTES, do an early aggregation
-# so as to release the gradient tensor to save memory.
-_MIN_AGGREGATE_COUNT = 4
-_MIN_AGGREGATE_BYTES = 128 * 1024 * 1024
-
-# Terminology:
-#
-#  - op: a possibly composite operation, which has an entry in the tape
-#  - target: dy in dx/dy
-#  - source: dx in dx/dy
-#  - tensor: one of the many inputs or outputs of an operation
-#
-# Below here we do the gradient algorithm. It works as follows:
-#
-# First we filter the tape to just the subset of operations we want to
-# differentiate. In the process of doing so we count how many times each Tensor
-# is used as an input to an op (so we know when we're done computing gradients
-# for that Tensor). We also count, for each tape entry, how many of its output
-# Tensors need gradients to be computed (Tensors which are not used do not need
-# any gradients to be computed).
-#
-# Finally, we start a backprop stack with a set of tape entries for which we
-# have all gradients available. This set usually is a subset of the set of
-# targets (not all since targets which have outputs in the tape will not have
-# gradients available initially).
-#
-# Then we repeatedly pop an entry from the stack, run its backprop, and update
-# the gradients of its inputs. Once we have computed all gradients for a single
-# input we can mark this input as done, and this can trigger adding an entry to
-# the stack if all outputs of that entry are now done.
-#
-# When the stack is empty we have gradients for all tensors we're interested in.
-
-
-def _prepare_backprop(target, tensor_to_op, op_to_entry, id_sources):
-  """Filters the tape to only include relevant entries and counts tensor usages.
-
-  Args:
-    target: the target to optimize.
-    tensor_to_op: Map from tensor id to key in op_to_entry that produced it.
-    op_to_entry: Map from op id to a tape.TapeEntry object
-    id_sources: the ids of the sources wrt the gradient is being taken.
-
-  Returns:
-    usage counts (how many entries downstream from a tensor use it)
-    op_to_entry_map: entry map (a filtered tape, with only the relevant
-     entries),
-    missing: map from tensor id to how many downstream gradients still need
-     to be computed before this tensor's gradient can be computed.
-  """
-  if isinstance(target, (ops.Tensor)):
-    tensor_stack = [ops.tensor_id(target)]
-  else:
-    tensor_stack = list([ops.tensor_id(x) for x in target])
-  tensor_usage_counts = {}
-  o_to_e = {}  # Copy of just the bits we need from op_to_entry
-  while tensor_stack:
-    t = tensor_stack.pop()
-    op = tensor_to_op.get(t, None)
-    # op is None if the tensor is a source (i.e. was watched directly)
-    if op is None or op in o_to_e:
-      continue
-    op_trace = op_to_entry[op]
-    o_to_e[op] = op_trace
-    for it in op_trace.input_ids:
-      if it in tensor_usage_counts:
-        tensor_usage_counts[it] += 1
-      else:
-        tensor_usage_counts[it] = 1
-        if it not in id_sources and it in tensor_to_op:
-          tensor_stack.append(it)
-  op_missing_tensor_counts = collections.defaultdict(int)
-  for t in tensor_usage_counts:
-    if t in tensor_to_op and tensor_to_op[t] is not None:
-      op_missing_tensor_counts[tensor_to_op[t]] += 1
-  return tensor_usage_counts, o_to_e, op_missing_tensor_counts
-
-
-def _initialize_backprop_stack(op_to_entry, op_missing_tensor):
-  """Returns the set of tape entries which are available for backprop."""
-  ready_ops = []
-  for op in op_to_entry:
-    if op not in op_missing_tensor:
-      ready_ops.append(op)
-  return ready_ops
-
-
-def _initial_gradients(target, output_gradients, tensor_usage_counts):
-  """Computes the initial gradients for each Tensor."""
-  # Initialize the backprop stack
-  gradients = collections.defaultdict(list)
-  if isinstance(target, ops.Tensor):
-    if output_gradients is not None:
-      output_gradient = output_gradients
-    else:
-      output_gradient = array_ops.ones_like(target)
-    gradients[ops.tensor_id(target)].append(output_gradient)
-  else:
-    for i, t in enumerate(target):
-      if ops.tensor_id(t) in tensor_usage_counts:
-        # Can't provide a gradient of something we're trying to differentiate
-        assert output_gradients is None or output_gradients[i] is None
-      else:
-        if output_gradients is None or output_gradients[i] is None:
-          out_grad = array_ops.ones_like(t)
-        else:
-          out_grad = output_gradients[i]
-        gradients[ops.tensor_id(t)].append(out_grad)
-  return gradients
-
-
-@tf_contextlib.contextmanager
-def _no_op():
-  yield
-
-
-def _aggregate_grads(gradients):
-  """Aggregate gradients from multiple sources.
-
-  Args:
-    gradients: A list of 'Tensor' or 'IndexedSlices' gradients.
-
-  Returns:
-    If 'gradients' only has 'Tensor', returns an aggregated 'Tensor'.
-    Otherwise returns an aggregated 'IndexedSlices'.
-  """
-  assert gradients, "No gradients to aggregate"
-
-  if len(gradients) == 1:
-    return gradients[0]
-  if all([isinstance(g, ops.Tensor) for g in gradients]):
-    return math_ops.add_n(gradients)
-  else:
-    assert all([isinstance(g, (ops.Tensor, ops.IndexedSlices))
-                for g in gradients])
-    indexed_slices_list = []
-    for grad in gradients:
-      # TODO(xpan): Support nested IndexedSlices and core IndexedSlices
-      if isinstance(grad, ops.Tensor):
-        indexed_slices = ops.IndexedSlices(
-            grad,
-            constant_op.constant(range(grad.shape[0])),
-            constant_op.constant(grad.shape.as_list()))
-        indexed_slices_list.append(indexed_slices)
-      else:
-        indexed_slices_list.append(grad)
-
-    # Dense shapes from all gradients should be the same.
-    dense_shape = indexed_slices_list[0].dense_shape
-    # For simplicity now, always cast to int64.
-    indices = array_ops.concat([math_ops.cast(x.indices, dtypes.int64)
-                                for x in indexed_slices_list], 0)
-    values = array_ops.concat([x.values for x in indexed_slices_list], 0)
-    return ops.IndexedSlices(values, indices, dense_shape)
-
-
-def _add_new_grads(gradients, gradients_size, tid, grad):
-  """Adds a new gradient and maybe aggregate the gradients.
-
-  Args:
-    gradients: A dict map from tensor id to list of gradients.
-    gradients_size: A dict map from tensor id to its total units. Might
-       not be initialized.
-    tid: Tensor id.
-    grad: New gradient for the `tid`, either a Tensor or IndexedSlices.
-
-  Raises:
-    ValueError: if `grad` is neight Tensor nor IndexedSlices.
-  """
-  tensor_grads = gradients[tid]
-  tensor_grads.append(grad)
-  if len(tensor_grads) < _MIN_AGGREGATE_COUNT:
-    return
-  elif tid not in gradients_size:
-    if isinstance(grad, ops.Tensor):
-      size = functools.reduce(operator.mul, grad._shape_tuple(), 1)  # pylint: disable=protected-access
-    elif isinstance(grad, ops.IndexedSlices):
-      size = functools.reduce(operator.mul, grad.values._shape_tuple(), 1)  # pylint: disable=protected-access
-    else:
-      raise ValueError("Unexpected gradient type: %s" % type(grad))
-    gradients_size[tid] = size
-  else:
-    size = gradients_size[tid]
-
-  # For simplicity, assume each element to be 4 bytes now.
-  if len(tensor_grads) * size * 4 > _MIN_AGGREGATE_BYTES:
-    gradients[tid] = [_aggregate_grads(tensor_grads)]
-
-
-def imperative_grad(
-    target,
-    sources,
-    output_gradients=None):
-  """Computes gradients from the imperatively defined tape on top of the stack.
-
-  Works by filtering the tape, computing how many downstream usages are of each
-  tensor and entry, and repeatedly applying backward functions until we have
-  gradients for all sources.
-
-  Args:
-   target: either a Tensor or list of Tensors to be differentiated.
-   sources: list of Tensors for which we want gradients
-   output_gradients: if not None, a list of gradient provided for each Target,
-    or None if we are to use the target's computed downstream gradient.
-
-  Returns:
-   the gradient wrt each of the sources.
-
-  Raises:
-    RuntimeError: if something goes wrong.
-    ValueError: if there is no sequence of differentiable operations connecting
-     a source and any target Tensor. This can happen either if the target is
-     not computed based on the source, if the tracing was set up incorrectly,
-     or if only non-differentiable functions of the source were used in the
-     computation of target.
-  """
-  if not tape._tape_stack.stack:  # pylint: disable=protected-access
-    raise RuntimeError("Computing a gradient with no tape present")
-  bp_tape = tape.pop_tape()
-  tensor_to_op, op_to_entry = bp_tape.export()
-  # This overwrites the op_to_entry variable, which will release all memory used
-  # to keep traces that are irrelevant to the gradient computation we're doing
-  # here.
-  id_sources = [ops.tensor_id(t) for t in sources]
-  tensor_usage_counts, op_to_entry, op_missing_tensor = _prepare_backprop(
-      target, tensor_to_op, op_to_entry, id_sources)
-  ready_ops = _initialize_backprop_stack(op_to_entry, op_missing_tensor)
-  gradients = _initial_gradients(target, output_gradients,
-                                 tensor_usage_counts)
-  gradients_size = dict()
-  # Now exhaust the backprop stack
-  while ready_ops:
-    op = ready_ops.pop()
-    op_trace = op_to_entry.pop(op)
-    out_gradients = [gradients.pop(t, None) for t in op_trace.output_ids]
-    for i in range(len(out_gradients)):
-      if out_gradients[i] is None:
-        # TODO(apassos) this should be in the right device
-        none_indices = _grad_fn_accepts_none_for_indices.get(
-            op_trace.op_type, None)
-        if none_indices is None or i not in none_indices:
-          out_gradients[i] = array_ops.zeros(
-              *op_trace.output_shape_and_dtype[i])
-      else:
-        out_gradients[i] = _aggregate_grads(out_gradients[i])
-
-    in_gradients = op_trace.backward_function(
-        *(out_gradients + op_trace.side_outputs))
-    in_gradients = ([in_gradients]
-                    if isinstance(in_gradients, (ops.Tensor,
-                                                 ops.IndexedSlices,
-                                                 type(None)))
-                    else in_gradients)
-    for i, t in enumerate(op_trace.input_ids):
-      if in_gradients[i] is not None:
-        _add_new_grads(gradients, gradients_size, t, in_gradients[i])
-      if tensor_usage_counts.get(t, 0) > 0:
-        tensor_usage_counts[t] -= 1
-        if (t in tensor_to_op
-            and tensor_usage_counts[t] == 0
-            and t not in id_sources):
-          in_op = tensor_to_op[t]
-          if in_op is None:
-            continue
-          if op_missing_tensor.get(in_op, 0) > 0:
-            op_missing_tensor[in_op] -= 1
-            if op_missing_tensor.get(in_op, 0) == 0:
-              ready_ops.append(in_op)
-  result = []
-  for i, s in enumerate(sources):
-    g = gradients.get(ops.tensor_id(s), None)
-    if g is None:
-      result.append(None)
-    else:
-      result.append(_aggregate_grads(g))
-  return result
-
 _op_attr_type_cache = {}
 
 
@@ -557,7 +279,7 @@ def _record_gradient(op_name, inputs, attrs, results, name):
     if _tracing:
       print("Gradient for", (name if name else op_name), "inputs", op_inputs,
             "output_grads", orig_outputs, "gradients", result)
-    return result
+    return nest.flatten(result)
 
   tape.record_operation(op_name, results, inputs, [], grad_fn)
   if _tracing:
@@ -615,7 +337,9 @@ def implicit_val_and_grad(f):
     end_node = f(*args)
     variables = tape.top_tape_watched_variables()
     sources = [x.handle for x in variables]
-    grad = imperative_grad(end_node, sources)
+    grad = imperative_grad.imperative_grad(_default_vspace,
+                                           nest.flatten(end_node),
+                                           sources)
     return end_node, list(zip(grad, variables))
 
   return grad_fn
@@ -849,6 +573,96 @@ def val_and_grad_function(f, params=None):
       sources.append(args[i])
       tape.watch(args[i])
     result = f(*args)
-    return result, imperative_grad(result, sources, output_gradients=dy)
+    return result, imperative_grad.imperative_grad(
+        _default_vspace, nest.flatten(result), sources,
+        output_gradients=nest.flatten(dy) if dy is not None else None)
 
   return decorated
+
+
+def _aggregate_grads(gradients):
+  """Aggregate gradients from multiple sources.
+
+  Args:
+    gradients: A list of 'Tensor' or 'IndexedSlices' gradients.
+
+  Returns:
+    If 'gradients' only has 'Tensor', returns an aggregated 'Tensor'.
+    Otherwise returns an aggregated 'IndexedSlices'.
+  """
+  assert gradients, "No gradients to aggregate"
+
+  if len(gradients) == 1:
+    return gradients[0]
+  if all([isinstance(g, ops.Tensor) for g in gradients]):
+    return math_ops.add_n(gradients)
+  else:
+    assert all([isinstance(g, (ops.Tensor, ops.IndexedSlices))
+                for g in gradients])
+    indexed_slices_list = []
+    for grad in gradients:
+      # TODO(xpan): Support nested IndexedSlices and core IndexedSlices
+      if isinstance(grad, ops.Tensor):
+        indexed_slices = ops.IndexedSlices(
+            grad,
+            constant_op.constant(range(grad.shape[0])),
+            constant_op.constant(grad.shape.as_list()))
+        indexed_slices_list.append(indexed_slices)
+      else:
+        indexed_slices_list.append(grad)
+
+    # Dense shapes from all gradients should be the same.
+    dense_shape = indexed_slices_list[0].dense_shape
+    # For simplicity now, always cast to int64.
+    indices = array_ops.concat([math_ops.cast(x.indices, dtypes.int64)
+                                for x in indexed_slices_list], 0)
+    values = array_ops.concat([x.values for x in indexed_slices_list], 0)
+    return ops.IndexedSlices(values, indices, dense_shape)
+
+
+# If over MIN_AGGREGATE_COUNT gradients are accumulated and the total
+# memory consumption is over MIN_AGGREGATE_BYTES, do an early aggregation
+# so as to release the gradient tensor to save memory.
+_MIN_AGGREGATE_COUNT = 4
+_MIN_AGGREGATE_BYTES = 128 * 1024 * 1024
+
+
+def _add_new_grads(gradients, gradients_size, tid, grad):
+  """Adds a new gradient and maybe aggregate the gradients.
+
+  Args:
+    gradients: A dict map from tensor id to list of gradients.
+    gradients_size: A dict map from tensor id to its total units. Might
+       not be initialized.
+    tid: Tensor id.
+    grad: New gradient for the `tid`, either a Tensor or IndexedSlices.
+
+  Raises:
+    ValueError: if `grad` is neight Tensor nor IndexedSlices.
+  """
+  tensor_grads = gradients[tid]
+  tensor_grads.append(grad)
+  if len(tensor_grads) < _MIN_AGGREGATE_COUNT:
+    return
+  elif tid not in gradients_size:
+    if isinstance(grad, ops.Tensor):
+      size = functools.reduce(operator.mul, grad._shape_tuple(), 1)  # pylint: disable=protected-access
+    elif isinstance(grad, ops.IndexedSlices):
+      size = functools.reduce(operator.mul, grad.values._shape_tuple(), 1)  # pylint: disable=protected-access
+    else:
+      raise ValueError("Unexpected gradient type: %s" % type(grad))
+    gradients_size[tid] = size
+  else:
+    size = gradients_size[tid]
+
+  # For simplicity, assume each element to be 4 bytes now.
+  if len(tensor_grads) * size * 4 > _MIN_AGGREGATE_BYTES:
+    gradients[tid] = [_aggregate_grads(tensor_grads)]
+
+
+_default_vspace = imperative_grad.VSpace(
+    add_new_grads_fn=_add_new_grads,
+    aggregate_fn=_aggregate_grads,
+    tensor_id=ops.tensor_id,
+    zeros=array_ops.zeros,
+    ones_like=array_ops.ones_like)
diff --git a/tensorflow/python/eager/custom_gradient.py b/tensorflow/python/eager/custom_gradient.py
index 67c9015bf0..4360e53225 100644
--- a/tensorflow/python/eager/custom_gradient.py
+++ b/tensorflow/python/eager/custom_gradient.py
@@ -78,7 +78,7 @@ def custom_gradient(f):
     # second derivative this way if they capture any output tensors. Change the
     # signature of custom_gradient.
     def actual_grad_fn(*outputs):
-      return grad_fn(*outputs)
+      return nest.flatten(grad_fn(*outputs))
 
     flat_result = nest.flatten(result)
     tape.record_operation(
diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py
index cb70d23f06..6ffc914f73 100644
--- a/tensorflow/python/eager/function.py
+++ b/tensorflow/python/eager/function.py
@@ -88,7 +88,7 @@ def _convert_to_graph_tensor(value, dtype=None, name=None, as_ref=False):
   else:
     captured_value = captured_value[1]
   tape.record_operation("captured_value", [captured_value], [value], [],
-                        lambda x: x)
+                        lambda x: [x])
   return captured_value
 
 
diff --git a/tensorflow/python/eager/imperative_grad.py b/tensorflow/python/eager/imperative_grad.py
new file mode 100644
index 0000000000..b81f5bba14
--- /dev/null
+++ b/tensorflow/python/eager/imperative_grad.py
@@ -0,0 +1,227 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Code for backpropagation using the tape utilities."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import collections
+
+from tensorflow.python.eager import tape
+
+
+# Terminology:
+#
+#  - op: a possibly composite operation, which has an entry in the tape
+#  - target: dy in dx/dy
+#  - source: dx in dx/dy
+#  - tensor: one of the many inputs or outputs of an operation
+#
+# Below here we do the gradient algorithm. It works as follows:
+#
+# First we filter the tape to just the subset of operations we want to
+# differentiate. In the process of doing so we count how many times each Tensor
+# is used as an input to an op (so we know when we're done computing gradients
+# for that Tensor). We also count, for each tape entry, how many of its output
+# Tensors need gradients to be computed (Tensors which are not used do not need
+# any gradients to be computed).
+#
+# Finally, we start a backprop stack with a set of tape entries for which we
+# have all gradients available. This set usually is a subset of the set of
+# targets (not all since targets which have outputs in the tape will not have
+# gradients available initially).
+#
+# Then we repeatedly pop an entry from the stack, run its backprop, and update
+# the gradients of its inputs. Once we have computed all gradients for a single
+# input we can mark this input as done, and this can trigger adding an entry to
+# the stack if all outputs of that entry are now done.
+#
+# When the stack is empty we have gradients for all tensors we're interested in.
+def _prepare_backprop(vspace, target, tensor_to_op, op_to_entry, id_sources):
+  """Filters the tape to only include relevant entries and counts tensor usages.
+
+  Args:
+    vspace: information about the space we're differentiating in.
+    target: the target to optimize.
+    tensor_to_op: Map from tensor id to key in op_to_entry that produced it.
+    op_to_entry: Map from op id to a tape.TapeEntry object
+    id_sources: the ids of the sources wrt the gradient is being taken.
+
+  Returns:
+    usage counts (how many entries downstream from a tensor use it)
+    op_to_entry_map: entry map (a filtered tape, with only the relevant
+     entries),
+    missing: map from tensor id to how many downstream gradients still need
+     to be computed before this tensor's gradient can be computed.
+  """
+  tensor_stack = [vspace.tensor_id(x) for x in target]
+  tensor_usage_counts = {}
+  o_to_e = {}  # Copy of just the bits we need from op_to_entry
+  while tensor_stack:
+    t = tensor_stack.pop()
+    op = tensor_to_op.get(t, None)
+    # op is None if the tensor is a source (i.e. was watched directly)
+    if op is None or op in o_to_e:
+      continue
+    op_trace = op_to_entry[op]
+    o_to_e[op] = op_trace
+    for it in op_trace.input_ids:
+      if it in tensor_usage_counts:
+        tensor_usage_counts[it] += 1
+      else:
+        tensor_usage_counts[it] = 1
+        if it not in id_sources and it in tensor_to_op:
+          tensor_stack.append(it)
+  op_missing_tensor_counts = collections.defaultdict(int)
+  for t in tensor_usage_counts:
+    if t in tensor_to_op and tensor_to_op[t] is not None:
+      op_missing_tensor_counts[tensor_to_op[t]] += 1
+  return tensor_usage_counts, o_to_e, op_missing_tensor_counts
+
+
+def _initialize_backprop_stack(op_to_entry, op_missing_tensor):
+  """Returns the set of tape entries which are available for backprop."""
+  ready_ops = []
+  for op in op_to_entry:
+    if op not in op_missing_tensor:
+      ready_ops.append(op)
+  return ready_ops
+
+
+def _initial_gradients(vspace, target, output_gradients, tensor_usage_counts):
+  """Computes the initial gradients for each Tensor."""
+  # Initialize the backprop stack
+  gradients = collections.defaultdict(list)
+  for i, t in enumerate(target):
+    if vspace.tensor_id(t) in tensor_usage_counts:
+      # Can't provide a gradient of something we're trying to differentiate
+      assert output_gradients is None or output_gradients[i] is None
+    else:
+      if output_gradients is None or output_gradients[i] is None:
+        out_grad = vspace.ones_like(t)
+      else:
+        out_grad = output_gradients[i]
+      gradients[vspace.tensor_id(t)].append(out_grad)
+  return gradients
+
+
+VSpace = collections.namedtuple(
+    "VSpace",
+    ["add_new_grads_fn", "aggregate_fn", "tensor_id", "zeros", "ones_like"])
+
+
+def imperative_grad(
+    vspace,
+    target,
+    sources,
+    output_gradients=None):
+  """Computes gradients from the imperatively defined tape on top of the stack.
+
+  Works by filtering the tape, computing how many downstream usages are of each
+  tensor and entry, and repeatedly applying backward functions until we have
+  gradients for all sources.
+
+  Args:
+   vspace: the vector space in which to differentiate.
+   target: either a Tensor or list of Tensors to be differentiated.
+   sources: list of Tensors for which we want gradients
+   output_gradients: if not None, a list of gradient provided for each Target,
+    or None if we are to use the target's computed downstream gradient.
+
+  Returns:
+   the gradient wrt each of the sources.
+
+  Raises:
+    RuntimeError: if something goes wrong.
+    ValueError: if there is no sequence of differentiable operations connecting
+     a source and any target Tensor. This can happen either if the target is
+     not computed based on the source, if the tracing was set up incorrectly,
+     or if only non-differentiable functions of the source were used in the
+     computation of target.
+  """
+  if not tape._tape_stack.stack:  # pylint: disable=protected-access
+    raise RuntimeError("Computing a gradient with no tape present")
+  bp_tape = tape.pop_tape()
+  tensor_to_op, op_to_entry = bp_tape.export()
+  # This overwrites the op_to_entry variable, which will release all memory used
+  # to keep traces that are irrelevant to the gradient computation we're doing
+  # here.
+  id_sources = [vspace.tensor_id(t) for t in sources]
+  tensor_usage_counts, op_to_entry, op_missing_tensor = _prepare_backprop(
+      vspace, target, tensor_to_op, op_to_entry, id_sources)
+  ready_ops = _initialize_backprop_stack(op_to_entry, op_missing_tensor)
+  gradients = _initial_gradients(vspace, target, output_gradients,
+                                 tensor_usage_counts)
+  gradients_size = dict()
+  # Now exhaust the backprop stack
+  while ready_ops:
+    op = ready_ops.pop()
+    op_trace = op_to_entry.pop(op)
+    out_gradients = [gradients.pop(t, None) for t in op_trace.output_ids]
+    for i in range(len(out_gradients)):
+      if out_gradients[i] is None:
+        # TODO(apassos) this should be in the right device
+        none_indices = _grad_fn_accepts_none_for_indices.get(
+            op_trace.op_type, None)
+        if none_indices is None or i not in none_indices:
+          out_gradients[i] = vspace.zeros(
+              *op_trace.output_shape_and_dtype[i])
+      else:
+        out_gradients[i] = vspace.aggregate_fn(out_gradients[i])
+
+    in_gradients = op_trace.backward_function(
+        *(out_gradients + op_trace.side_outputs))
+    for i, t in enumerate(op_trace.input_ids):
+      if in_gradients[i] is not None:
+        vspace.add_new_grads_fn(gradients, gradients_size, t, in_gradients[i])
+      if tensor_usage_counts.get(t, 0) > 0:
+        tensor_usage_counts[t] -= 1
+        if (t in tensor_to_op
+            and tensor_usage_counts[t] == 0
+            and t not in id_sources):
+          in_op = tensor_to_op[t]
+          if in_op is None:
+            continue
+          if op_missing_tensor.get(in_op, 0) > 0:
+            op_missing_tensor[in_op] -= 1
+            if op_missing_tensor.get(in_op, 0) == 0:
+              ready_ops.append(in_op)
+  result = []
+  for i, s in enumerate(sources):
+    g = gradients.get(vspace.tensor_id(s), None)
+    if g is None:
+      result.append(None)
+    else:
+      result.append(vspace.aggregate_fn(g))
+  return result
+
+
+# TODO(agarwal): use an automatic mechanism for handling None arguments to
+# gradient functions.
+# Some gradient functions can accept None arguments for gradients. The following
+# maps the operation name to the indices at which the corresponding gradient
+# function can accept None values.
+# e.g. FusedBatchNorm outputs 5 values and hence receives 5 gradient values
+# during backprop. However the gradient function uses only the first of those
+# values and ignores the rest. The entry, "FusedBatchNorm": [1, 2, 3, 4],
+# indicates that only the gradient corresponding to index 0 is used, and the
+# gradient values at indices 1-4 are ignored (and hence can be None). The
+# backprop algorithm can then leverage this by not constructing zeros to
+# pass for those indices.
+_grad_fn_accepts_none_for_indices = {
+    "SoftmaxCrossEntropyWithLogits": [1],
+    "FusedBatchNorm": [1, 2, 3, 4]
+}
diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index 50aa070985..ae84297690 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -675,7 +675,7 @@ class _EagerTensorBase(Tensor):
     if not context.in_graph_mode():
       self_device = self.device
       def grad_fun(dresult):
-        return dresult._copy(device_name=self_device)
+        return [dresult._copy(device_name=self_device)]
       tape.record_operation("_copy", [new_tensor], [self], [], grad_fun)
     return new_tensor
     # pylint: enable=protected-access
-- 
GitLab


From c49eeeee5463aff02b4bafbd1596288ba4b27739 Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Thu, 5 Oct 2017 09:54:37 -0700
Subject: [PATCH 063/909] Add a Cython build dependency, start using some
 Cython tensor utilities

PiperOrigin-RevId: 171166294
---
 .../core/platform/default/build_config.bzl    |  71 +++++++++++-
 tensorflow/python/BUILD                       |  11 +-
 .../python/framework/fast_tensor_util.pyx     | 103 ++++++++++++++++++
 tensorflow/python/framework/tensor_util.py    |   3 +-
 tensorflow/workspace.bzl                      |  11 ++
 third_party/cython.BUILD                      |  28 +++++
 6 files changed, 222 insertions(+), 5 deletions(-)
 create mode 100644 tensorflow/python/framework/fast_tensor_util.pyx
 create mode 100644 third_party/cython.BUILD

diff --git a/tensorflow/core/platform/default/build_config.bzl b/tensorflow/core/platform/default/build_config.bzl
index 8a67951b24..51d37291ee 100644
--- a/tensorflow/core/platform/default/build_config.bzl
+++ b/tensorflow/core/platform/default/build_config.bzl
@@ -28,6 +28,76 @@ def tf_deps(deps, suffix):
 
   return tf_deps
 
+# Modified from @cython//:Tools/rules.bzl
+def pyx_library(
+    name,
+    deps=[],
+    py_deps=[],
+    srcs=[],
+    **kwargs):
+  """Compiles a group of .pyx / .pxd / .py files.
+
+  First runs Cython to create .cpp files for each input .pyx or .py + .pxd
+  pair. Then builds a shared object for each, passing "deps" to each cc_binary
+  rule (includes Python headers by default). Finally, creates a py_library rule
+  with the shared objects and any pure Python "srcs", with py_deps as its
+  dependencies; the shared objects can be imported like normal Python files.
+
+  Args:
+    name: Name for the rule.
+    deps: C/C++ dependencies of the Cython (e.g. Numpy headers).
+    py_deps: Pure Python dependencies of the final library.
+    srcs: .py, .pyx, or .pxd files to either compile or pass through.
+    **kwargs: Extra keyword arguments passed to the py_library.
+  """
+  # First filter out files that should be run compiled vs. passed through.
+  py_srcs = []
+  pyx_srcs = []
+  pxd_srcs = []
+  for src in srcs:
+    if src.endswith(".pyx") or (src.endswith(".py")
+                                and src[:-3] + ".pxd" in srcs):
+      pyx_srcs.append(src)
+    elif src.endswith(".py"):
+      py_srcs.append(src)
+    else:
+      pxd_srcs.append(src)
+    if src.endswith("__init__.py"):
+      pxd_srcs.append(src)
+
+  # Invoke cython to produce the shared object libraries.
+  cpp_outs = [src.split(".")[0] + ".cpp" for src in pyx_srcs]
+  native.genrule(
+      name = name + "_cython_translation",
+      srcs = pyx_srcs,
+      outs = cpp_outs,
+      cmd = ("PYTHONHASHSEED=0 $(location @cython//:cython_binary) --cplus $(SRCS)"
+             # Rename outputs to expected location.
+             + """ && python -c 'import shutil, sys; n = len(sys.argv); [shutil.copyfile(src.split(".")[0] + ".cpp", dst) for src, dst in zip(sys.argv[1:], sys.argv[1+n//2:])]' $(SRCS) $(OUTS)"""),
+      tools = ["@cython//:cython_binary"] + pxd_srcs,
+  )
+
+  shared_objects = []
+  for src in pyx_srcs:
+    stem = src.split(".")[0]
+    shared_object_name = stem + ".so"
+    native.cc_binary(
+        name=shared_object_name,
+        srcs=[stem + ".cpp"],
+        deps=deps + ["//util/python:python_headers"],
+        linkshared = 1,
+    )
+    shared_objects.append(shared_object_name)
+
+  # Now create a py_library with these shared objects as data.
+  native.py_library(
+      name=name,
+      srcs=py_srcs,
+      deps=py_deps,
+      srcs_version = "PY2AND3",
+      data=shared_objects,
+      **kwargs
+  )
 
 def _proto_cc_hdrs(srcs, use_grpc_plugin=False):
   ret = [s[:-len(".proto")] + ".pb.h" for s in srcs]
@@ -299,7 +369,6 @@ def tf_additional_proto_srcs():
 def tf_additional_all_protos():
   return ["//tensorflow/core:protos_all"]
 
-
 def tf_protos_all_impl():
   return ["//tensorflow/core:protos_all_cc_impl"]
 
diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 3e846cd18a..407ff079c1 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -24,6 +24,7 @@ load("//tensorflow:tensorflow.bzl", "tf_py_wrap_cc")
 load("//tensorflow:tensorflow.bzl", "tf_cc_shared_object")
 load("//tensorflow:tensorflow.bzl", "cuda_py_test")
 load("//tensorflow:tensorflow.bzl", "cuda_py_tests")
+load("//tensorflow/core:platform/default/build_config.bzl", "pyx_library")
 load("//tensorflow/core:platform/default/build_config.bzl", "tf_proto_library")
 load("//tensorflow/core:platform/default/build_config.bzl", "tf_proto_library_py")
 load("//tensorflow/core:platform/default/build_config.bzl", "tf_additional_lib_deps")
@@ -503,6 +504,7 @@ py_library(
         ":common_shapes",
         ":cpp_shape_inference_proto_py",
         ":errors",
+        ":framework_fast_tensor_util",
         ":framework_for_generated_wrappers",
         ":function",
         ":graph_util",
@@ -733,8 +735,6 @@ py_library(
     ],
 )
 
-# load("//third_party/py/cython:build_defs.bzl", "pyx_library")
-
 py_library(
     name = "extra_py_tests_deps",
     srcs_version = "PY2AND3",
@@ -4358,3 +4358,10 @@ py_test(
         "//third_party/py/numpy",
     ],
 )
+
+pyx_library(
+    name = "framework_fast_tensor_util",
+    srcs = ["framework/fast_tensor_util.pyx"],
+    py_deps = ["//tensorflow/python:util"],
+    deps = ["//third_party/py/numpy:headers"],
+)
diff --git a/tensorflow/python/framework/fast_tensor_util.pyx b/tensorflow/python/framework/fast_tensor_util.pyx
new file mode 100644
index 0000000000..b43ddb4ad3
--- /dev/null
+++ b/tensorflow/python/framework/fast_tensor_util.pyx
@@ -0,0 +1,103 @@
+#cython: boundscheck=False
+#cython: wraparound=False
+#cython: infer_types=True
+import numpy as np
+cimport numpy as np
+
+from tensorflow.python.util import compat
+
+
+def AppendFloat32ArrayToTensorProto(
+    tensor_proto, np.ndarray[np.float32_t, ndim=1] nparray):
+  cdef long i, n
+  n = nparray.size
+  for i in range(n):
+    tensor_proto.float_val.append(nparray[i])
+
+
+def AppendFloat64ArrayToTensorProto(
+    tensor_proto, np.ndarray[np.float64_t, ndim=1] nparray):
+  cdef long i, n
+  n = nparray.size
+  for i in range(n):
+    tensor_proto.double_val.append(nparray[i])
+
+
+def AppendInt32ArrayToTensorProto(
+    tensor_proto, np.ndarray[np.int32_t, ndim=1] nparray):
+  cdef long i, n
+  n = nparray.size
+  for i in range(n):
+    tensor_proto.int_val.append(nparray[i])
+
+
+def AppendInt64ArrayToTensorProto(
+    tensor_proto, np.ndarray[np.int64_t, ndim=1] nparray):
+  cdef long i, n
+  n = nparray.size
+  for i in range(n):
+    tensor_proto.int64_val.append(nparray[i])
+
+
+def AppendUInt8ArrayToTensorProto(
+    tensor_proto, np.ndarray[np.uint8_t, ndim=1] nparray):
+  cdef long i, n
+  n = nparray.size
+  for i in range(n):
+    tensor_proto.int_val.append(nparray[i])
+
+
+def AppendUInt16ArrayToTensorProto(
+    tensor_proto, np.ndarray[np.uint16_t, ndim=1] nparray):
+  cdef long i, n
+  n = nparray.size
+  for i in range(n):
+    tensor_proto.int_val.append(nparray[i])
+
+
+def AppendInt16ArrayToTensorProto(
+    tensor_proto, np.ndarray[np.int16_t, ndim=1] nparray):
+  cdef long i, n
+  n = nparray.size
+  for i in range(n):
+    tensor_proto.int_val.append(nparray[i])
+
+
+def AppendInt8ArrayToTensorProto(
+    tensor_proto, np.ndarray[np.int8_t, ndim=1] nparray):
+  cdef long i, n
+  n = nparray.size
+  for i in range(n):
+    tensor_proto.int_val.append(nparray[i])
+
+
+def AppendComplex64ArrayToTensorProto(
+    tensor_proto, np.ndarray[np.complex64_t, ndim=1] nparray):
+  cdef long i, n
+  n = nparray.size
+  for i in range(n):
+    tensor_proto.scomplex_val.append(nparray[i].real)
+    tensor_proto.scomplex_val.append(nparray[i].imag)
+
+
+def AppendComplex128ArrayToTensorProto(
+    tensor_proto, np.ndarray[np.complex128_t, ndim=1] nparray):
+  cdef long i, n
+  n = nparray.size
+  for i in range(n):
+    tensor_proto.dcomplex_val.append(nparray[i].real)
+    tensor_proto.dcomplex_val.append(nparray[i].imag)
+
+
+def AppendObjectArrayToTensorProto(tensor_proto, np.ndarray nparray):
+  cdef long i, n
+  n = nparray.size
+  for i in range(n):
+    tensor_proto.string_val.append(compat.as_bytes(nparray[i]))
+
+
+def AppendBoolArrayToTensorProto(tensor_proto, nparray):
+  cdef long i, n
+  n = nparray.size
+  for i in range(n):
+    tensor_proto.bool_val.append(np.asscalar(nparray[i]))
diff --git a/tensorflow/python/framework/tensor_util.py b/tensorflow/python/framework/tensor_util.py
index 335db92a73..414c61e930 100644
--- a/tensorflow/python/framework/tensor_util.py
+++ b/tensorflow/python/framework/tensor_util.py
@@ -27,8 +27,7 @@ from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
 from tensorflow.python.util import compat
 
-# TODO(opensource): Add support for pyx_library in the open-source build.
-# For now, we use the slow versions that fast_tensor_util replaces.
+# Fallback in case fast_tensor_util is not properly compiled.
 # pylint: disable=g-import-not-at-top
 try:
   from tensorflow.python.framework import fast_tensor_util
diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index f33a942dc9..b226184261 100644
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -713,6 +713,17 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       actual = "@cub_archive//:cub",
   )
 
+  native.new_http_archive(
+      name = "cython",
+      sha256 = "6dcd30b5ceb887b2b965ee7ceb82ea3acb5f0642fe2206c7636b45acea4798e5",
+      urls = [
+          "http://mirror.bazel.build/github.com/cython/cython/archive/3732784c45cfb040a5b0936951d196f83a12ea17.tar.gz",
+          "https://github.com/cython/cython/archive/3732784c45cfb040a5b0936951d196f83a12ea17.tar.gz",
+      ],
+      strip_prefix = "cython-3732784c45cfb040a5b0936951d196f83a12ea17",
+      build_file = str(Label("//third_party:cython.BUILD")),
+  )
+
   native.http_archive(
       name = "bazel_toolchains",
       urls = [
diff --git a/third_party/cython.BUILD b/third_party/cython.BUILD
new file mode 100644
index 0000000000..a8e72a1e36
--- /dev/null
+++ b/third_party/cython.BUILD
@@ -0,0 +1,28 @@
+# Modified version of @cython//:BUILD.bazel
+
+py_library(
+    name = "cython_lib",
+    srcs = glob(
+        ["Cython/**/*.py"],
+        exclude = [
+            "**/Tests/*.py",
+        ],
+    ) + ["cython.py"],
+    data = glob([
+        "Cython/**/*.pyx",
+        "Cython/Utility/*.*",
+        "Cython/Includes/**/*.pxd",
+    ]),
+    srcs_version = "PY2AND3",
+    visibility = ["//visibility:public"],
+)
+
+# May not be named "cython", since that conflicts with Cython/ on OSX
+py_binary(
+    name = "cython_binary",
+    srcs = ["cython.py"],
+    main = "cython.py",
+    srcs_version = "PY2AND3",
+    visibility = ["//visibility:public"],
+    deps = ["cython_lib"],
+)
-- 
GitLab


From 376147cd71d1a240dad428c3ff82ca4ea5f4e88e Mon Sep 17 00:00:00 2001
From: RJ Ryan <rjryan@google.com>
Date: Thu, 5 Oct 2017 10:03:12 -0700
Subject: [PATCH 064/909] Save an unnecessary logical_not in the
 maximum/minimum gradient.

PiperOrigin-RevId: 171167415
---
 tensorflow/cc/gradients/math_grad.cc | 2 +-
 tensorflow/python/ops/math_grad.py   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/cc/gradients/math_grad.cc b/tensorflow/cc/gradients/math_grad.cc
index ac288b1d83..2417bf18a9 100644
--- a/tensorflow/cc/gradients/math_grad.cc
+++ b/tensorflow/cc/gradients/math_grad.cc
@@ -484,7 +484,7 @@ Status MaximumMinimumGradCommon(const Scope& scope, const Operation& op,
   auto grad = grad_inputs[0];
   auto zeros = ZerosLike(scope, grad);
   auto gx_1 = Where3(scope, comparator, grad, zeros);
-  auto gx_2 = Where3(scope, LogicalNot(scope, comparator), grad, zeros);
+  auto gx_2 = Where3(scope, comparator, zeros, grad);
   return BinaryGradCommon(scope, op, grad_outputs, gx_1, gx_2);
 }
 
diff --git a/tensorflow/python/ops/math_grad.py b/tensorflow/python/ops/math_grad.py
index ee9cbda0c0..d36d66f899 100644
--- a/tensorflow/python/ops/math_grad.py
+++ b/tensorflow/python/ops/math_grad.py
@@ -851,7 +851,7 @@ def _MaximumMinimumGrad(op, grad, selector_op):
   xmask = selector_op(x, y)
   rx, ry = gen_array_ops._broadcast_gradient_args(sx, sy)
   xgrad = array_ops.where(xmask, grad, zeros)
-  ygrad = array_ops.where(math_ops.logical_not(xmask), grad, zeros)
+  ygrad = array_ops.where(xmask, zeros, grad)
   gx = array_ops.reshape(math_ops.reduce_sum(xgrad, rx), sx)
   gy = array_ops.reshape(math_ops.reduce_sum(ygrad, ry), sy)
   return (gx, gy)
-- 
GitLab


From 23227f038d909d4f415683d4cf2a62a68d774b2c Mon Sep 17 00:00:00 2001
From: "Joshua V. Dillon" <jvdillon@google.com>
Date: Thu, 5 Oct 2017 10:16:17 -0700
Subject: [PATCH 065/909] Add
 tf.contrib.distributions.MixtureSameFamily.log_cdf.

PiperOrigin-RevId: 171169340
---
 tensorflow/contrib/distributions/BUILD        |  2 +
 .../kernel_tests/mixture_same_family_test.py  | 88 ++++++++++++-------
 .../python/ops/mixture_same_family.py         |  8 ++
 3 files changed, 65 insertions(+), 33 deletions(-)

diff --git a/tensorflow/contrib/distributions/BUILD b/tensorflow/contrib/distributions/BUILD
index aef73f0598..dcdfbbeba2 100644
--- a/tensorflow/contrib/distributions/BUILD
+++ b/tensorflow/contrib/distributions/BUILD
@@ -305,6 +305,8 @@ cuda_py_test(
     additional_deps = [
         ":distributions_py",
         "//third_party/py/numpy",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:math_ops",
         "//tensorflow/python:client_testlib",
     ],
 )
diff --git a/tensorflow/contrib/distributions/python/kernel_tests/mixture_same_family_test.py b/tensorflow/contrib/distributions/python/kernel_tests/mixture_same_family_test.py
index 47ac412500..ee4f989dac 100644
--- a/tensorflow/contrib/distributions/python/kernel_tests/mixture_same_family_test.py
+++ b/tensorflow/contrib/distributions/python/kernel_tests/mixture_same_family_test.py
@@ -23,67 +23,75 @@ import numpy as np
 from tensorflow.contrib.distributions.python.ops import mixture_same_family as mixture_same_family_lib
 from tensorflow.contrib.distributions.python.ops import mvn_diag as mvn_diag_lib
 from tensorflow.contrib.distributions.python.ops import test_util
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import math_ops
 from tensorflow.python.ops.distributions import bernoulli as bernoulli_lib
 from tensorflow.python.ops.distributions import categorical as categorical_lib
 from tensorflow.python.ops.distributions import normal as normal_lib
 from tensorflow.python.platform import test
 
 
-class MixtureSameFamilyTest(
-    test_util.VectorDistributionTestHelpers, test.TestCase):
+class MixtureSameFamilyTest(test_util.VectorDistributionTestHelpers,
+                            test.TestCase):
 
   def testSampleAndLogProbUnivariateShapes(self):
     with self.test_session():
       gm = mixture_same_family_lib.MixtureSameFamily(
-          mixture_distribution=categorical_lib.Categorical(
-              probs=[0.3, 0.7]),
+          mixture_distribution=categorical_lib.Categorical(probs=[0.3, 0.7]),
           components_distribution=normal_lib.Normal(
-              loc=[-1., 1],
-              scale=[0.1, 0.5]))
-      x = gm.sample([4, 5])
+              loc=[-1., 1], scale=[0.1, 0.5]))
+      x = gm.sample([4, 5], seed=42)
       log_prob_x = gm.log_prob(x)
       self.assertEqual([4, 5], x.shape)
       self.assertEqual([4, 5], log_prob_x.shape)
 
   def testSampleAndLogProbShapesBroadcastMix(self):
     mix_probs = np.float32([.3, .7])
-    bern_probs = np.float32([[.4, .6],
-                             [.25, .75]])
+    bern_probs = np.float32([[.4, .6], [.25, .75]])
     with self.test_session():
       bm = mixture_same_family_lib.MixtureSameFamily(
-          mixture_distribution=categorical_lib.Categorical(
-              probs=mix_probs),
-          components_distribution=bernoulli_lib.Bernoulli(
-              probs=bern_probs))
-      x = bm.sample([4, 5])
+          mixture_distribution=categorical_lib.Categorical(probs=mix_probs),
+          components_distribution=bernoulli_lib.Bernoulli(probs=bern_probs))
+      x = bm.sample([4, 5], seed=42)
       log_prob_x = bm.log_prob(x)
       x_ = x.eval()
       self.assertEqual([4, 5, 2], x.shape)
       self.assertEqual([4, 5, 2], log_prob_x.shape)
-      self.assertAllEqual(np.ones_like(x_, dtype=np.bool),
-                          np.logical_or(x_ == 0., x_ == 1.))
+      self.assertAllEqual(
+          np.ones_like(x_, dtype=np.bool), np.logical_or(x_ == 0., x_ == 1.))
 
   def testSampleAndLogProbMultivariateShapes(self):
     with self.test_session():
       gm = mixture_same_family_lib.MixtureSameFamily(
-          mixture_distribution=categorical_lib.Categorical(
-              probs=[0.3, 0.7]),
+          mixture_distribution=categorical_lib.Categorical(probs=[0.3, 0.7]),
           components_distribution=mvn_diag_lib.MultivariateNormalDiag(
-              loc=[[-1., 1], [1, -1]],
-              scale_identity_multiplier=[1., 0.5]))
-      x = gm.sample([4, 5])
+              loc=[[-1., 1], [1, -1]], scale_identity_multiplier=[1., 0.5]))
+      x = gm.sample([4, 5], seed=42)
       log_prob_x = gm.log_prob(x)
       self.assertEqual([4, 5, 2], x.shape)
       self.assertEqual([4, 5], log_prob_x.shape)
 
+  def testSampleAndLogProbBatchMultivariateShapes(self):
+    with self.test_session():
+      gm = mixture_same_family_lib.MixtureSameFamily(
+          mixture_distribution=categorical_lib.Categorical(probs=[0.3, 0.7]),
+          components_distribution=mvn_diag_lib.MultivariateNormalDiag(
+              loc=[[[-1., 1],
+                    [1, -1]],
+                   [[0., 1],
+                    [1, 0]]],
+              scale_identity_multiplier=[1., 0.5]))
+      x = gm.sample([4, 5], seed=42)
+      log_prob_x = gm.log_prob(x)
+      self.assertEqual([4, 5, 2, 2], x.shape)
+      self.assertEqual([4, 5, 2], log_prob_x.shape)
+
   def testSampleConsistentLogProb(self):
     with self.test_session() as sess:
       gm = mixture_same_family_lib.MixtureSameFamily(
-          mixture_distribution=categorical_lib.Categorical(
-              probs=[0.3, 0.7]),
+          mixture_distribution=categorical_lib.Categorical(probs=[0.3, 0.7]),
           components_distribution=mvn_diag_lib.MultivariateNormalDiag(
-              loc=[[-1., 1], [1, -1]],
-              scale_identity_multiplier=[1., 0.5]))
+              loc=[[-1., 1], [1, -1]], scale_identity_multiplier=[1., 0.5]))
       # Ball centered at component0's mean.
       self.run_test_sample_consistent_log_prob(
           sess, gm, radius=1., center=[-1., 1], rtol=0.02)
@@ -91,26 +99,40 @@ class MixtureSameFamilyTest(
       self.run_test_sample_consistent_log_prob(
           sess, gm, radius=1., center=[1., -1], rtol=0.02)
 
+  def testLogCdf(self):
+    with self.test_session() as sess:
+      gm = mixture_same_family_lib.MixtureSameFamily(
+          mixture_distribution=categorical_lib.Categorical(probs=[0.3, 0.7]),
+          components_distribution=normal_lib.Normal(
+              loc=[-1., 1], scale=[0.1, 0.5]))
+      x = gm.sample(10, seed=42)
+      actual_log_cdf = gm.log_cdf(x)
+      expected_log_cdf = math_ops.reduce_logsumexp(
+          (gm.mixture_distribution.logits +
+           gm.components_distribution.log_cdf(x[..., array_ops.newaxis])),
+          axis=1)
+      actual_log_cdf_, expected_log_cdf_ = sess.run([
+          actual_log_cdf, expected_log_cdf])
+      self.assertAllClose(actual_log_cdf_, expected_log_cdf_,
+                          rtol=1e-6, atol=0.0)
+
   def testSampleConsistentMeanCovariance(self):
     with self.test_session() as sess:
       gm = mixture_same_family_lib.MixtureSameFamily(
-          mixture_distribution=categorical_lib.Categorical(
-              probs=[0.3, 0.7]),
+          mixture_distribution=categorical_lib.Categorical(probs=[0.3, 0.7]),
           components_distribution=mvn_diag_lib.MultivariateNormalDiag(
-              loc=[[-1., 1], [1, -1]],
-              scale_identity_multiplier=[1., 0.5]))
+              loc=[[-1., 1], [1, -1]], scale_identity_multiplier=[1., 0.5]))
       self.run_test_sample_consistent_mean_covariance(sess, gm)
 
   def testVarianceConsistentCovariance(self):
     with self.test_session() as sess:
       gm = mixture_same_family_lib.MixtureSameFamily(
-          mixture_distribution=categorical_lib.Categorical(
-              probs=[0.3, 0.7]),
+          mixture_distribution=categorical_lib.Categorical(probs=[0.3, 0.7]),
           components_distribution=mvn_diag_lib.MultivariateNormalDiag(
-              loc=[[-1., 1], [1, -1]],
-              scale_identity_multiplier=[1., 0.5]))
+              loc=[[-1., 1], [1, -1]], scale_identity_multiplier=[1., 0.5]))
       cov_, var_ = sess.run([gm.covariance(), gm.variance()])
       self.assertAllClose(cov_.diagonal(), var_, atol=0.)
 
+
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/contrib/distributions/python/ops/mixture_same_family.py b/tensorflow/contrib/distributions/python/ops/mixture_same_family.py
index e92bcf8c1f..5558ef0f25 100644
--- a/tensorflow/contrib/distributions/python/ops/mixture_same_family.py
+++ b/tensorflow/contrib/distributions/python/ops/mixture_same_family.py
@@ -260,6 +260,14 @@ class MixtureSameFamily(distribution.Distribution):
           probs * self.components_distribution.mean(),
           axis=-1 - self._event_ndims)                       # [B, E]
 
+  def _log_cdf(self, x):
+    x = self._pad_sample_dims(x)
+    log_cdf_x = self.components_distribution.log_cdf(x)      # [S, B, k]
+    log_mix_prob = nn_ops.log_softmax(
+        self.mixture_distribution.logits, dim=-1)            # [B, k]
+    return math_ops.reduce_logsumexp(
+        log_cdf_x + log_mix_prob, axis=-1)                   # [S, B]
+
   def _variance(self):
     with ops.control_dependencies(self._runtime_assertions):
       # Law of total variance: Var(Y) = E[Var(Y|X)] + Var(E[Y|X])
-- 
GitLab


From 3b679ec63be33ccfaa99dce3d2c65bad9c36961f Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Thu, 5 Oct 2017 10:46:21 -0700
Subject: [PATCH 066/909] Add srcs_version="PY2AND3"

PiperOrigin-RevId: 171173975
---
 tensorflow/python/eager/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/python/eager/BUILD b/tensorflow/python/eager/BUILD
index 4069ef1c70..76d4f37e9a 100644
--- a/tensorflow/python/eager/BUILD
+++ b/tensorflow/python/eager/BUILD
@@ -431,5 +431,6 @@ filegroup(
 py_library(
     name = "imperative_grad",
     srcs = ["imperative_grad.py"],
+    srcs_version = "PY2AND3",
     deps = [":tape"],
 )
-- 
GitLab


From fd5326666ac5297e2bec09b29728d8731951be23 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 5 Oct 2017 10:52:58 -0700
Subject: [PATCH 067/909] Fixes markdown formatting of EstimatorSpec
 constructor. Before, it was rendering as italics because of the missing
 newline.

PiperOrigin-RevId: 171175131
---
 tensorflow/python/estimator/model_fn.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/python/estimator/model_fn.py b/tensorflow/python/estimator/model_fn.py
index d58e03f6ef..da202408c3 100644
--- a/tensorflow/python/estimator/model_fn.py
+++ b/tensorflow/python/estimator/model_fn.py
@@ -77,6 +77,7 @@ class EstimatorSpec(
     """Creates a validated `EstimatorSpec` instance.
 
     Depending on the value of `mode`, different arguments are required. Namely
+
     * For `mode == ModeKeys.TRAIN`: required fields are `loss` and `train_op`.
     * For `mode == ModeKeys.EVAL`: required field is `loss`.
     * For `mode == ModeKeys.PREDICT`: required fields are `predictions`.
-- 
GitLab


From 8818469ff81e8877eb7f042df19241b5eaa31637 Mon Sep 17 00:00:00 2001
From: Derek Murray <mrry@google.com>
Date: Thu, 5 Oct 2017 11:35:23 -0700
Subject: [PATCH 068/909] [tf.data] Update more `tf.contrib.data` references to
 `tf.data`.

PiperOrigin-RevId: 171182644
---
 .../contrib/data/python/ops/batching.py       | 10 ++++----
 .../contrib/data/python/ops/enumerate_ops.py  |  2 +-
 .../contrib/data/python/ops/error_ops.py      |  4 ++--
 .../contrib/data/python/ops/grouping.py       |  2 +-
 .../contrib/data/python/ops/resampling.py     |  2 +-
 .../contrib/data/python/ops/sloppy_ops.py     |  2 +-
 .../api_guides/python/threading_and_queues.md | 23 ++++++++++---------
 tensorflow/docs_src/programmers_guide/faq.md  |  6 ++---
 8 files changed, 26 insertions(+), 25 deletions(-)

diff --git a/tensorflow/contrib/data/python/ops/batching.py b/tensorflow/contrib/data/python/ops/batching.py
index 16f01557a2..ccfa8747da 100644
--- a/tensorflow/contrib/data/python/ops/batching.py
+++ b/tensorflow/contrib/data/python/ops/batching.py
@@ -68,7 +68,7 @@ def dense_to_sparse_batch(batch_size, row_shape):
 
   Returns:
     A `Dataset` transformation function, which can be passed to
-    @{tf.contrib.data.Dataset.apply}.
+    @{tf.data.Dataset.apply}.
   """
 
   def _apply_fn(dataset):
@@ -87,7 +87,7 @@ def unbatch():
 
   Returns:
     A `Dataset` transformation function, which can be passed to
-    @{tf.contrib.data.Dataset.apply}.
+    @{tf.data.Dataset.apply}.
   """
 
   def _apply_fn(dataset):
@@ -106,7 +106,7 @@ def unbatch():
 def batch_and_drop_remainder(batch_size):
   """A batching transformation that omits the final small batch (if present).
 
-  Like @{tf.contrib.data.Dataset.batch}, this transformation combines
+  Like @{tf.data.Dataset.batch}, this transformation combines
   consecutive elements of this dataset into batches. However, if the batch
   size does not evenly divide the input dataset size, this transformation will
   drop the final smaller element.
@@ -115,7 +115,7 @@ def batch_and_drop_remainder(batch_size):
   transformation and `Dataset.batch()`:
 
   ```python
-  dataset = tf.contrib.data.Dataset.range(200)
+  dataset = tf.data.Dataset.range(200)
   batched = dataset.apply(tf.contrib.data.batch_and_drop_remainder(128))
   print(batched.output_shapes)  # ==> "(128,)" (the batch dimension is known)
   ```
@@ -130,7 +130,7 @@ def batch_and_drop_remainder(batch_size):
 
   Returns:
     A `Dataset` transformation function, which can be passed to
-    @{tf.contrib.data.Dataset.apply}
+    @{tf.data.Dataset.apply}
   """
 
   def _apply_fn(dataset):
diff --git a/tensorflow/contrib/data/python/ops/enumerate_ops.py b/tensorflow/contrib/data/python/ops/enumerate_ops.py
index 40e7315f1f..ac2b386b81 100644
--- a/tensorflow/contrib/data/python/ops/enumerate_ops.py
+++ b/tensorflow/contrib/data/python/ops/enumerate_ops.py
@@ -47,7 +47,7 @@ def enumerate_dataset(start=0):
 
   Returns:
     A `Dataset` transformation function, which can be passed to
-    @{tf.contrib.data.Dataset.apply}.
+    @{tf.data.Dataset.apply}.
   """
 
   def _apply_fn(dataset):
diff --git a/tensorflow/contrib/data/python/ops/error_ops.py b/tensorflow/contrib/data/python/ops/error_ops.py
index dffa8b7f7d..238bb52b02 100644
--- a/tensorflow/contrib/data/python/ops/error_ops.py
+++ b/tensorflow/contrib/data/python/ops/error_ops.py
@@ -30,7 +30,7 @@ def ignore_errors():
   example:
 
   ```python
-  dataset = tf.contrib.data.Dataset.from_tensor_slices([1., 2., 0., 4.])
+  dataset = tf.data.Dataset.from_tensor_slices([1., 2., 0., 4.])
 
   # Computing `tf.check_numerics(1. / 0.)` will raise an InvalidArgumentError.
   dataset = dataset.map(lambda x: tf.check_numerics(1. / x, "error"))
@@ -42,7 +42,7 @@ def ignore_errors():
 
   Returns:
     A `Dataset` transformation function, which can be passed to
-    @{tf.contrib.data.Dataset.apply}.
+    @{tf.data.Dataset.apply}.
   """
 
   def _apply_fn(dataset):
diff --git a/tensorflow/contrib/data/python/ops/grouping.py b/tensorflow/contrib/data/python/ops/grouping.py
index 2cf7e8f4ee..6df7b22fb6 100644
--- a/tensorflow/contrib/data/python/ops/grouping.py
+++ b/tensorflow/contrib/data/python/ops/grouping.py
@@ -57,7 +57,7 @@ def group_by_window(key_func,
 
   Returns:
     A `Dataset` transformation function, which can be passed to
-    @{tf.contrib.data.Dataset.apply}.
+    @{tf.data.Dataset.apply}.
 
   Raises:
     ValueError: if neither or both of {`window_size`, `window_size_func`} are
diff --git a/tensorflow/contrib/data/python/ops/resampling.py b/tensorflow/contrib/data/python/ops/resampling.py
index f4f2d42854..ee46f3e852 100644
--- a/tensorflow/contrib/data/python/ops/resampling.py
+++ b/tensorflow/contrib/data/python/ops/resampling.py
@@ -48,7 +48,7 @@ def rejection_resample(class_func, target_dist, initial_dist=None, seed=None):
 
   Returns:
     A `Dataset` transformation function, which can be passed to
-    @{tf.contrib.data.Dataset.apply}.
+    @{tf.data.Dataset.apply}.
   """
 
   def _apply_fn(dataset):
diff --git a/tensorflow/contrib/data/python/ops/sloppy_ops.py b/tensorflow/contrib/data/python/ops/sloppy_ops.py
index 01e234f1d0..058c497320 100644
--- a/tensorflow/contrib/data/python/ops/sloppy_ops.py
+++ b/tensorflow/contrib/data/python/ops/sloppy_ops.py
@@ -118,7 +118,7 @@ def sloppy_interleave(map_func, cycle_length, block_length=1):
 
   Returns:
     A `Dataset` transformation function, which can be passed to
-    @{tf.contrib.data.Dataset.apply}.
+    @{tf.data.Dataset.apply}.
   """
   def _apply_fn(dataset):
     return SloppyInterleaveDataset(
diff --git a/tensorflow/docs_src/api_guides/python/threading_and_queues.md b/tensorflow/docs_src/api_guides/python/threading_and_queues.md
index 9d8a05c7dc..ab95ce0af9 100644
--- a/tensorflow/docs_src/api_guides/python/threading_and_queues.md
+++ b/tensorflow/docs_src/api_guides/python/threading_and_queues.md
@@ -1,13 +1,14 @@
 # Threading and Queues
 
 Note: In versions of TensorFlow before 1.2, we recommended using multi-threaded,
-queue-based input pipelines for performance. Beginning with TensorFlow 1.2,
-however, we recommend using the `tf.contrib.data` module instead. (See
-[Datasets](datasets) for details.) The `tf.contrib.data` module offers an
-easier-to-use interface for constructing efficient input pipelines. Furthermore,
-we've stopped developing the old multi-threaded, queue-based input pipelines.
-We've retained the documentation in this file to help developers who are still
-maintaining older code.
+queue-based input pipelines for performance. Beginning with TensorFlow 1.4,
+however, we recommend using the `tf.data` module instead. (See
+[Datasets](datasets) for details. In TensorFlow 1.2 and 1.3, the module was
+called `tf.contrib.data`.) The `tf.data` module offers an easier-to-use
+interface for constructing efficient input pipelines. Furthermore, we've stopped
+developing the old multi-threaded, queue-based input pipelines.  We've retained
+the documentation in this file to help developers who are still maintaining
+older code.
 
 Multithreaded queues are a powerful and widely used mechanism supporting
 asynchronous computation.
@@ -58,9 +59,9 @@ prepare inputs for training a model as follows:
 * A training thread executes a training op that dequeues mini-batches from the
   queue
 
-We recommend using the @{tf.contrib.data.Dataset.shuffle$`shuffle`}
-and @{tf.contrib.data.Dataset.batch$`batch`} methods of a
-@{tf.contrib.data.Dataset$`Dataset`} to accomplish this. However, if you'd prefer
+We recommend using the @{tf.data.Dataset.shuffle$`shuffle`}
+and @{tf.data.Dataset.batch$`batch`} methods of a
+@{tf.data.Dataset$`Dataset`} to accomplish this. However, if you'd prefer
 to use a queue-based version instead, you can find a full implementation in the
 @{tf.train.shuffle_batch} function.
 
@@ -103,7 +104,7 @@ The simplest possible use of this function might be something like this:
 ``` python
 # create a dataset that counts from 0 to 99
 input = tf.constant(list(range(100)))
-input = tf.contrib.data.Dataset.from_tensor_slices(input)
+input = tf.data.Dataset.from_tensor_slices(input)
 input = input.make_one_shot_iterator().get_next()
 
 # Create a slightly shuffled batch from the sorted elements
diff --git a/tensorflow/docs_src/programmers_guide/faq.md b/tensorflow/docs_src/programmers_guide/faq.md
index 865016dc02..67ed0a9a60 100644
--- a/tensorflow/docs_src/programmers_guide/faq.md
+++ b/tensorflow/docs_src/programmers_guide/faq.md
@@ -269,13 +269,13 @@ See the how-to documentation for
 There are three main options for dealing with data in a custom format.
 
 The easiest option is to write parsing code in Python that transforms the data
-into a numpy array. Then use @{tf.contrib.data.Dataset.from_tensor_slices} to
+into a numpy array. Then use @{tf.data.Dataset.from_tensor_slices} to
 create an input pipeline from the in-memory data.
 
 If your data doesn't fit in memory, try doing the parsing in the Dataset
 pipeline. Start with an appropriate file reader, like
-@{tf.contrib.data.TextLineDataset}. Then convert the dataset by mapping
-@{tf.contrib.data.Dataset.map$mapping} appropriate operations over it.
+@{tf.data.TextLineDataset}. Then convert the dataset by mapping
+@{tf.data.Dataset.map$mapping} appropriate operations over it.
 Prefer predefined TensorFlow operations such as @{tf.decode_raw},
 @{tf.decode_csv}, @{tf.parse_example}, or @{tf.image.decode_png}.
 
-- 
GitLab


From 6c875f0da3c61610063f705111b9bfa2e26ca52f Mon Sep 17 00:00:00 2001
From: Igor Saprykin <isaprykin@google.com>
Date: Thu, 5 Oct 2017 11:56:28 -0700
Subject: [PATCH 069/909] Add the 'is_the_final_export' signal to Exporters.

Instead of adding the option to respect `is_the_final_export` into the `Exporter` that also does garbage collection, such exporter is split into two:  `LatestExporter` and `FinalExporter`.  There is a concern that options `exports_to_keep` and `only_the_final_export` overlap significantly and are somewhat in conflict.  What does it mean to keep last 5 exports but only export the final one?

After splitting in two classes there is a lot of code duplication.  The common implementation is gathered in a private base class.

When the training ends, the final export is performed via `Exporter.export()` call.  That final export is going to have is_the_final_export parameter being set to true.

If `TrainSpec.max_steps` is `None`, then "when training ends" is undefined.  We are going to train forever.  In that case, `is_the_final_export` is going to be always False.  I added a note about it.

PiperOrigin-RevId: 171185881
---
 tensorflow/python/estimator/estimator_lib.py  |   2 +
 tensorflow/python/estimator/exporter.py       | 134 ++++++++++++++++--
 tensorflow/python/estimator/exporter_test.py  |  42 +++++-
 tensorflow/python/estimator/training.py       |  37 +++--
 tensorflow/python/estimator/training_test.py  |  81 +++++++++++
 .../tensorflow.estimator.-exporter.pbtxt      |   2 +-
 ...tensorflow.estimator.-final-exporter.pbtxt |  18 +++
 ...ensorflow.estimator.-latest-exporter.pbtxt |   2 +-
 .../api/golden/tensorflow.estimator.pbtxt     |   4 +
 9 files changed, 293 insertions(+), 29 deletions(-)
 create mode 100644 tensorflow/tools/api/golden/tensorflow.estimator.-final-exporter.pbtxt

diff --git a/tensorflow/python/estimator/estimator_lib.py b/tensorflow/python/estimator/estimator_lib.py
index a5b3faeffb..5b82fd75ff 100644
--- a/tensorflow/python/estimator/estimator_lib.py
+++ b/tensorflow/python/estimator/estimator_lib.py
@@ -30,6 +30,7 @@ from tensorflow.python.estimator.canned.parsing_utils import regressor_parse_exa
 from tensorflow.python.estimator.estimator import Estimator
 from tensorflow.python.estimator.export import export_lib as export
 from tensorflow.python.estimator.exporter import Exporter
+from tensorflow.python.estimator.exporter import FinalExporter
 from tensorflow.python.estimator.exporter import LatestExporter
 from tensorflow.python.estimator.inputs import inputs
 from tensorflow.python.estimator.model_fn import EstimatorSpec
@@ -70,6 +71,7 @@ _allowed_symbols = [
     'TrainSpec',
     'Exporter',
     'LatestExporter',
+    'FinalExporter',
 ]
 
 remove_undocumented(__name__, allowed_exception_list=_allowed_symbols)
diff --git a/tensorflow/python/estimator/exporter.py b/tensorflow/python/estimator/exporter.py
index 505820dd93..56400ab935 100644
--- a/tensorflow/python/estimator/exporter.py
+++ b/tensorflow/python/estimator/exporter.py
@@ -40,7 +40,8 @@ class Exporter(object):
     pass
 
   @abc.abstractmethod
-  def export(self, estimator, export_path, checkpoint_path, eval_result):
+  def export(self, estimator, export_path, checkpoint_path, eval_result,
+             is_the_final_export):
     """Exports the given `Estimator` to a specific format.
 
     Args:
@@ -48,6 +49,12 @@ class Exporter(object):
       export_path: A string containing a directory where to write the export.
       checkpoint_path: The checkpoint path to export.
       eval_result: The output of `Estimator.evaluate` on this checkpoint.
+      is_the_final_export: This boolean is True when this is an export in the
+        end of training.  It is False for the intermediate exports during
+        the training.
+        When passing `Exporter` to `tf.estimator.train_and_evaluate`
+        `is_the_final_export` is always False if `TrainSpec.max_steps` is
+        `None`.
 
     Returns:
       The string path to the exported directory or `None` if export is skipped.
@@ -55,18 +62,18 @@ class Exporter(object):
     pass
 
 
-class LatestExporter(Exporter):
+class _SavedModelExporter(Exporter):
   """This class exports the serving graph and checkpoints.
 
-  In addition, the class also garbage collects stale exports.
+     This class provides a basic exporting functionality and serves as a
+     foundation for specialized `Exporter`s.
   """
 
   def __init__(self,
                name,
                serving_input_fn,
                assets_extra=None,
-               as_text=False,
-               exports_to_keep=5):
+               as_text=False):
     """Create an `Exporter` to use with `tf.estimator.EvalSpec`.
 
     Args:
@@ -83,9 +90,6 @@ class LatestExporter(Exporter):
         `{'my_asset_file.txt': '/path/to/my_asset_file.txt'}`.
       as_text: whether to write the SavedModel proto in text format. Defaults to
         `False`.
-      exports_to_keep: Number of exports to keep.  Older exports will be
-       garbage-collected.  Defaults to 5.  Set to `None` to disable garbage
-       collection.
 
     Raises:
       ValueError: if any arguments is invalid.
@@ -94,16 +98,15 @@ class LatestExporter(Exporter):
     self._serving_input_fn = serving_input_fn
     self._assets_extra = assets_extra
     self._as_text = as_text
-    self._exports_to_keep = exports_to_keep
-    if exports_to_keep is not None and exports_to_keep <= 0:
-      raise ValueError(
-          '`exports_to_keep`, if provided, must be positive number')
 
   @property
   def name(self):
     return self._name
 
-  def export(self, estimator, export_path, checkpoint_path, eval_result):
+  def export(self, estimator, export_path, checkpoint_path, eval_result,
+             is_the_final_export):
+    del is_the_final_export
+
     export_result = estimator.export_savedmodel(
         export_path,
         self._serving_input_fn,
@@ -111,6 +114,111 @@ class LatestExporter(Exporter):
         as_text=self._as_text,
         checkpoint_path=checkpoint_path)
 
+    return export_result
+
+
+class FinalExporter(Exporter):
+  """This class exports the serving graph and checkpoints in the end.
+
+  This class performs a single export in the end of training.
+  """
+
+  def __init__(self,
+               name,
+               serving_input_fn,
+               assets_extra=None,
+               as_text=False):
+    """Create an `Exporter` to use with `tf.estimator.EvalSpec`.
+
+    Args:
+      name: unique name of this `Exporter` that is going to be used in the
+        export path.
+      serving_input_fn: a function that takes no arguments and returns an
+        `ServingInputReceiver`.
+      assets_extra: An optional dict specifying how to populate the assets.extra
+        directory within the exported SavedModel.  Each key should give the
+        destination path (including the filename) relative to the assets.extra
+        directory.  The corresponding value gives the full path of the source
+        file to be copied.  For example, the simple case of copying a single
+        file without renaming it is specified as
+        `{'my_asset_file.txt': '/path/to/my_asset_file.txt'}`.
+      as_text: whether to write the SavedModel proto in text format. Defaults to
+        `False`.
+
+    Raises:
+      ValueError: if any arguments is invalid.
+    """
+    self._saved_model_exporter = _SavedModelExporter(name, serving_input_fn,
+                                                     assets_extra, as_text)
+
+  @property
+  def name(self):
+    return self._saved_model_exporter.name
+
+  def export(self, estimator, export_path, checkpoint_path, eval_result,
+             is_the_final_export):
+    if not is_the_final_export:
+      return None
+
+    tf_logging.info('Performing the final export in the end of training.')
+
+    return self._saved_model_exporter.export(estimator, export_path,
+                                             checkpoint_path, eval_result,
+                                             is_the_final_export)
+
+
+class LatestExporter(Exporter):
+  """This class regularly exports the serving graph and checkpoints.
+
+  In addition to exporting, this class also garbage collects stale exports.
+  """
+
+  def __init__(self,
+               name,
+               serving_input_fn,
+               assets_extra=None,
+               as_text=False,
+               exports_to_keep=5):
+    """Create an `Exporter` to use with `tf.estimator.EvalSpec`.
+
+    Args:
+      name: unique name of this `Exporter` that is going to be used in the
+        export path.
+      serving_input_fn: a function that takes no arguments and returns an
+        `ServingInputReceiver`.
+      assets_extra: An optional dict specifying how to populate the assets.extra
+        directory within the exported SavedModel.  Each key should give the
+        destination path (including the filename) relative to the assets.extra
+        directory.  The corresponding value gives the full path of the source
+        file to be copied.  For example, the simple case of copying a single
+        file without renaming it is specified as
+        `{'my_asset_file.txt': '/path/to/my_asset_file.txt'}`.
+      as_text: whether to write the SavedModel proto in text format. Defaults to
+        `False`.
+      exports_to_keep: Number of exports to keep.  Older exports will be
+       garbage-collected.  Defaults to 5.  Set to `None` to disable garbage
+       collection.
+
+    Raises:
+      ValueError: if any arguments is invalid.
+    """
+    self._saved_model_exporter = _SavedModelExporter(name, serving_input_fn,
+                                                     assets_extra, as_text)
+    self._exports_to_keep = exports_to_keep
+    if exports_to_keep is not None and exports_to_keep <= 0:
+      raise ValueError(
+          '`exports_to_keep`, if provided, must be positive number')
+
+  @property
+  def name(self):
+    return self._saved_model_exporter.name
+
+  def export(self, estimator, export_path, checkpoint_path, eval_result,
+             is_the_final_export):
+    export_result = self._saved_model_exporter.export(
+        estimator, export_path, checkpoint_path, eval_result,
+        is_the_final_export)
+
     self._garbage_collect_exports(export_path)
     return export_result
 
diff --git a/tensorflow/python/estimator/exporter_test.py b/tensorflow/python/estimator/exporter_test.py
index 2ceff1bfd6..f90c35dce7 100644
--- a/tensorflow/python/estimator/exporter_test.py
+++ b/tensorflow/python/estimator/exporter_test.py
@@ -37,12 +37,13 @@ class LatestExporterTest(test.TestCase):
       pass
 
     with self.assertRaisesRegexp(ValueError, "positive number"):
-      exporter_lib.LatestExporter(
+      exporter = exporter_lib.LatestExporter(
           name="latest_exporter",
           serving_input_fn=_serving_input_fn,
           exports_to_keep=0)
+      self.assertEqual("latest_exporter", exporter.name)
 
-  def test_saved_model_exporter(self):
+  def test_latest_exporter(self):
 
     def _serving_input_fn():
       pass
@@ -60,7 +61,40 @@ class LatestExporterTest(test.TestCase):
     estimator.export_savedmodel.return_value = "export_result_path"
 
     export_result = exporter.export(estimator, export_dir_base,
-                                    "checkpoint_path", {})
+                                    "checkpoint_path", {}, False)
+
+    self.assertEqual("export_result_path", export_result)
+    estimator.export_savedmodel.assert_called_with(
+        export_dir_base,
+        _serving_input_fn,
+        assets_extra={"from/path": "to/path"},
+        as_text=False,
+        checkpoint_path="checkpoint_path")
+
+  def test_only_the_last_export_is_saved(self):
+
+    def _serving_input_fn():
+      pass
+
+    export_dir_base = tempfile.mkdtemp() + "export/"
+    gfile.MkDir(export_dir_base)
+
+    exporter = exporter_lib.FinalExporter(
+        name="latest_exporter",
+        serving_input_fn=_serving_input_fn,
+        assets_extra={"from/path": "to/path"},
+        as_text=False)
+    estimator = test.mock.Mock(spec=estimator_lib.Estimator)
+    estimator.export_savedmodel.return_value = "export_result_path"
+
+    export_result = exporter.export(estimator, export_dir_base,
+                                    "checkpoint_path", {}, False)
+
+    self.assertFalse(estimator.export_savedmodel.called)
+    self.assertEqual(None, export_result)
+
+    export_result = exporter.export(estimator, export_dir_base,
+                                    "checkpoint_path", {}, True)
 
     self.assertEqual("export_result_path", export_result)
     estimator.export_savedmodel.assert_called_with(
@@ -93,7 +127,7 @@ class LatestExporterTest(test.TestCase):
     estimator = test.mock.Mock(spec=estimator_lib.Estimator)
     # Garbage collect all but the most recent 2 exports,
     # where recency is determined based on the timestamp directory names.
-    exporter.export(estimator, export_dir_base, None, None)
+    exporter.export(estimator, export_dir_base, None, None, False)
 
     self.assertFalse(gfile.Exists(export_dir_1))
     self.assertFalse(gfile.Exists(export_dir_2))
diff --git a/tensorflow/python/estimator/training.py b/tensorflow/python/estimator/training.py
index 17c072566a..5c0ebbea35 100644
--- a/tensorflow/python/estimator/training.py
+++ b/tensorflow/python/estimator/training.py
@@ -519,8 +519,11 @@ class _TrainingExecutor(object):
     class NewCheckpointListener(
         basic_session_run_hooks.CheckpointSaverListener):
 
-      def __init__(self, estimator, eval_spec):
-        self._evaluator = _TrainingExecutor._Evaluator(estimator, eval_spec)  # pylint: disable=protected-access
+      def __init__(self, estimator, eval_spec, max_training_steps):
+        # pylint: disable=protected-access
+        self._evaluator = _TrainingExecutor._Evaluator(estimator, eval_spec,
+                                                       max_training_steps)
+        # pylint: enable=protected-access
 
       def after_save(self, session, global_step_value):
         del session, global_step_value
@@ -528,8 +531,10 @@ class _TrainingExecutor(object):
 
     # When the underlying `Estimator` object saves a new checkpoint, we would
     # like this callback to be called so that evaluation and export can trigger.
-    saving_listeners = [NewCheckpointListener(self._estimator, self._eval_spec)]
-
+    saving_listeners = [
+        NewCheckpointListener(self._estimator, self._eval_spec,
+                              self._train_spec.max_steps)
+    ]
     return self._start_distributed_training(saving_listeners=saving_listeners)
 
   def run_evaluator(self):
@@ -566,7 +571,8 @@ class _TrainingExecutor(object):
                  'after {} secs (eval_spec.throttle_secs) or training is '
                  'finished.'.format(self._eval_spec.throttle_secs))
 
-    evaluator = _TrainingExecutor._Evaluator(self._estimator, self._eval_spec)
+    evaluator = _TrainingExecutor._Evaluator(self._estimator, self._eval_spec,
+                                             self._train_spec.max_steps)
 
     while True:
       self._estimator.train(
@@ -636,7 +642,8 @@ class _TrainingExecutor(object):
       time.sleep(start_delay_secs)
 
     latest_eval_result = None
-    evaluator = _TrainingExecutor._Evaluator(self._estimator, self._eval_spec)
+    evaluator = _TrainingExecutor._Evaluator(self._estimator, self._eval_spec,
+                                             self._train_spec.max_steps)
 
     while True:
       if latest_eval_result:
@@ -663,11 +670,12 @@ class _TrainingExecutor(object):
   class _Evaluator(object):
     """A helper class to call `Estimator.evaluate` and export model."""
 
-    def __init__(self, estimator, eval_spec):
+    def __init__(self, estimator, eval_spec, max_training_steps):
       self._estimator = estimator
       self._eval_spec = eval_spec
       self._previous_ckpt_path = None
       self._last_warning_time = 0
+      self._max_training_steps = max_training_steps
 
     def evaluate_and_export(self):
       """Evaluate and (maybe) export the current model.
@@ -712,7 +720,14 @@ class _TrainingExecutor(object):
             'Internal error: `Estimator.evaluate` result should have '
             '`global_step` in result. Given {}'.format(eval_result))
 
-      self._export_eval_result(eval_result, latest_ckpt_path)
+      # TODO(isaprykin):  There is a potential race condition here in the
+      #  distributed setting.  The worker job that performs training
+      #  might stop at a later global step value than the evalutor job.
+      is_the_final_export = (eval_result[ops.GraphKeys.GLOBAL_STEP] >=
+                             self._max_training_steps
+                             if self._max_training_steps else False)
+      self._export_eval_result(eval_result, latest_ckpt_path,
+                               is_the_final_export)
 
       self._last_warning_time = 0
       self._previous_ckpt_path = latest_ckpt_path
@@ -725,7 +740,8 @@ class _TrainingExecutor(object):
         logging.warning(message)
         self._last_warning_time = current_time
 
-    def _export_eval_result(self, eval_result, checkpoint_path):
+    def _export_eval_result(self, eval_result, checkpoint_path,
+                            is_the_final_export):
       """Export `eval_result` according to exporters in `EvalSpec`."""
       export_dir_base = os.path.join(
           compat.as_str_any(self._estimator.model_dir),
@@ -738,4 +754,5 @@ class _TrainingExecutor(object):
                 compat.as_str_any(export_dir_base),
                 compat.as_str_any(exporter.name)),
             checkpoint_path=checkpoint_path,
-            eval_result=eval_result)
+            eval_result=eval_result,
+            is_the_final_export=is_the_final_export)
diff --git a/tensorflow/python/estimator/training_test.py b/tensorflow/python/estimator/training_test.py
index 51aed757a2..40972ab5a0 100644
--- a/tensorflow/python/estimator/training_test.py
+++ b/tensorflow/python/estimator/training_test.py
@@ -815,6 +815,46 @@ class TrainingExecutorRunEvaluatorTest(test.TestCase):
     self.assertEqual(2, mock_est.evaluate.call_count)
     self.assertEqual(2, exporter.export.call_count)
 
+  def test_final_export_is_true_in_the_end(self):
+    training_max_step = 200
+
+    mock_est = test.mock.Mock(spec=estimator_lib.Estimator)
+    mock_est.model_dir = compat.as_bytes(test.get_temp_dir())
+    mock_est.evaluate.side_effect = [
+        {_GLOBAL_STEP_KEY: training_max_step // 2},
+        {_GLOBAL_STEP_KEY: training_max_step}
+    ]
+    mock_est.latest_checkpoint.side_effect = ['path_1', 'path_2']
+
+    mock_train_spec = test.mock.Mock(spec=training.TrainSpec)
+    mock_train_spec.max_steps = training_max_step
+
+    mock_est.times_export_fn_was_called = 0
+    mock_est.times_the_final_export_was_true = 0
+    def export(estimator, export_path, checkpoint_path, eval_result,
+               is_the_final_export):
+      del export_path, checkpoint_path, eval_result
+      estimator.times_export_fn_was_called += 1
+      if is_the_final_export:
+        estimator.times_the_final_export_was_true += 1
+
+    exporter = test.mock.PropertyMock(spec=exporter_lib.Exporter)
+    exporter.name = 'see_how_many_times_export_is_called'
+    exporter.export = export
+
+    eval_spec = training.EvalSpec(
+        input_fn=lambda: 1,
+        start_delay_secs=0,
+        throttle_secs=0,
+        exporters=exporter)
+
+    executor = training._TrainingExecutor(mock_est, mock_train_spec, eval_spec)
+    executor.run_evaluator()
+
+    self.assertEqual(2, mock_est.evaluate.call_count)
+    self.assertEqual(2, mock_est.times_export_fn_was_called)
+    self.assertEqual(1, mock_est.times_the_final_export_was_true)
+
   def test_skip_evaluation_due_to_ckpt(self):
     training_max_step = 200
     mock_est = test.mock.Mock(spec=estimator_lib.Estimator)
@@ -1147,6 +1187,47 @@ class TrainingExecutorRunLocalTest(test.TestCase):
     with self.assertRaisesRegexp(RuntimeError, _STALE_CHECKPOINT_MSG):
       executor.run_local()
 
+  def test_final_export_is_true_in_the_end(self):
+    mock_est = test.mock.Mock(spec=estimator_lib.Estimator, model_dir='path/')
+    mock_est.latest_checkpoint = self.unique_checkpoint_every_time_fn
+
+    mock_est.times_export_fn_was_called = 0
+    mock_est.times_the_final_export_was_true = 0
+    def export(estimator, export_path, checkpoint_path, eval_result,
+               is_the_final_export):
+      del export_path, checkpoint_path, eval_result
+      estimator.times_export_fn_was_called += 1
+      if is_the_final_export:
+        estimator.times_the_final_export_was_true += 1
+
+    exporter = test.mock.PropertyMock(spec=exporter_lib.Exporter)
+    exporter.name = 'see_how_many_times_export_is_called'
+    exporter.export = export
+
+    train_spec = training.TrainSpec(
+        input_fn=lambda: 1, max_steps=300, hooks=[_FakeHook()])
+    eval_spec = training.EvalSpec(
+        input_fn=lambda: 1,
+        hooks=[_FakeHook()],
+        throttle_secs=100,
+        exporters=exporter)
+    # should be called 3 times.
+    mock_est.evaluate.side_effect = [{
+        _GLOBAL_STEP_KEY: train_spec.max_steps - 100
+    }, {
+        _GLOBAL_STEP_KEY: train_spec.max_steps - 50
+    }, {
+        _GLOBAL_STEP_KEY: train_spec.max_steps
+    }]
+
+    executor = training._TrainingExecutor(mock_est, train_spec, eval_spec)
+    executor.run_local()
+
+    self.assertEqual(3, mock_est.train.call_count)
+    self.assertEqual(3, mock_est.evaluate.call_count)
+    self.assertEqual(3, mock_est.times_export_fn_was_called)
+    self.assertEqual(1, mock_est.times_the_final_export_was_true)
+
   def test_train_and_evaluate_args(self):
     mock_est = test.mock.Mock(spec=estimator_lib.Estimator, model_dir='path/')
     mock_est.latest_checkpoint.return_value = 'checkpoint_path/'
diff --git a/tensorflow/tools/api/golden/tensorflow.estimator.-exporter.pbtxt b/tensorflow/tools/api/golden/tensorflow.estimator.-exporter.pbtxt
index c69e4c7a30..035af70e52 100644
--- a/tensorflow/tools/api/golden/tensorflow.estimator.-exporter.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.estimator.-exporter.pbtxt
@@ -11,6 +11,6 @@ tf_class {
   }
   member_method {
     name: "export"
-    argspec: "args=[\'self\', \'estimator\', \'export_path\', \'checkpoint_path\', \'eval_result\'], varargs=None, keywords=None, defaults=None"
+    argspec: "args=[\'self\', \'estimator\', \'export_path\', \'checkpoint_path\', \'eval_result\', \'is_the_final_export\'], varargs=None, keywords=None, defaults=None"
   }
 }
diff --git a/tensorflow/tools/api/golden/tensorflow.estimator.-final-exporter.pbtxt b/tensorflow/tools/api/golden/tensorflow.estimator.-final-exporter.pbtxt
new file mode 100644
index 0000000000..4c2dbc4d37
--- /dev/null
+++ b/tensorflow/tools/api/golden/tensorflow.estimator.-final-exporter.pbtxt
@@ -0,0 +1,18 @@
+path: "tensorflow.estimator.FinalExporter"
+tf_class {
+  is_instance: "<class \'tensorflow.python.estimator.exporter.FinalExporter\'>"
+  is_instance: "<class \'tensorflow.python.estimator.exporter.Exporter\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'name\', \'serving_input_fn\', \'assets_extra\', \'as_text\'], varargs=None, keywords=None, defaults=[\'None\', \'False\'], "
+  }
+  member_method {
+    name: "export"
+    argspec: "args=[\'self\', \'estimator\', \'export_path\', \'checkpoint_path\', \'eval_result\', \'is_the_final_export\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/tensorflow.estimator.-latest-exporter.pbtxt b/tensorflow/tools/api/golden/tensorflow.estimator.-latest-exporter.pbtxt
index c3f98f84b8..ae1483bf3f 100644
--- a/tensorflow/tools/api/golden/tensorflow.estimator.-latest-exporter.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.estimator.-latest-exporter.pbtxt
@@ -13,6 +13,6 @@ tf_class {
   }
   member_method {
     name: "export"
-    argspec: "args=[\'self\', \'estimator\', \'export_path\', \'checkpoint_path\', \'eval_result\'], varargs=None, keywords=None, defaults=None"
+    argspec: "args=[\'self\', \'estimator\', \'export_path\', \'checkpoint_path\', \'eval_result\', \'is_the_final_export\'], varargs=None, keywords=None, defaults=None"
   }
 }
diff --git a/tensorflow/tools/api/golden/tensorflow.estimator.pbtxt b/tensorflow/tools/api/golden/tensorflow.estimator.pbtxt
index 25e94a14a6..ef93a61bd8 100644
--- a/tensorflow/tools/api/golden/tensorflow.estimator.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.estimator.pbtxt
@@ -32,6 +32,10 @@ tf_module {
     name: "Exporter"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "FinalExporter"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "LatestExporter"
     mtype: "<type \'type\'>"
-- 
GitLab


From 9f00851a636e77223d4445a5ffa1fe1bf506f54e Mon Sep 17 00:00:00 2001
From: Jonathan Shen <jonathanasdf@google.com>
Date: Thu, 5 Oct 2017 12:09:44 -0700
Subject: [PATCH 070/909] Register GPU bool Fill op.

PiperOrigin-RevId: 171187907
---
 tensorflow/core/kernels/constant_op.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/core/kernels/constant_op.cc b/tensorflow/core/kernels/constant_op.cc
index 0cc2ea0109..618d4f580b 100644
--- a/tensorflow/core/kernels/constant_op.cc
+++ b/tensorflow/core/kernels/constant_op.cc
@@ -247,6 +247,7 @@ REGISTER_KERNEL(GPU, int8);
 REGISTER_KERNEL(GPU, uint16);
 REGISTER_KERNEL(GPU, int16);
 REGISTER_KERNEL(GPU, int64);
+REGISTER_KERNEL(GPU, bool);
 // Currently we do not support filling strings and complex64 on GPU
 
 // A special GPU kernel for int32.
-- 
GitLab


From 4bf27f8d4acee2cb8df27427668bddc92137e2ef Mon Sep 17 00:00:00 2001
From: Asim Shankar <ashankar@google.com>
Date: Thu, 5 Oct 2017 12:22:32 -0700
Subject: [PATCH 071/909] eager: Release Python GIL when executing kernels.

As a side effect, this enables use of py_func.

PiperOrigin-RevId: 171189922
---
 tensorflow/contrib/eager/python/datasets_test.py | 12 ++++++++++++
 tensorflow/python/eager/pywrap_tfe_src.cc        |  2 ++
 2 files changed, 14 insertions(+)

diff --git a/tensorflow/contrib/eager/python/datasets_test.py b/tensorflow/contrib/eager/python/datasets_test.py
index a2da6b28c6..076c92e73f 100644
--- a/tensorflow/contrib/eager/python/datasets_test.py
+++ b/tensorflow/contrib/eager/python/datasets_test.py
@@ -19,7 +19,9 @@ from __future__ import print_function
 from tensorflow.contrib.data import Dataset
 from tensorflow.contrib.eager.python import datasets
 from tensorflow.python.eager import test
+from tensorflow.python.framework import dtypes
 from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import script_ops
 
 
 class IteratorTest(test.TestCase):
@@ -69,6 +71,16 @@ class IteratorTest(test.TestCase):
     got2 = [x.numpy() for x in datasets.Iterator(ds)]
     self.assertAllEqual(got1, got2)
 
+  def testPyFunc(self):
+
+    def my_map(inp):
+      return [[x + 1 for x in inp]]
+
+    ds = Dataset.range(4).map(
+        lambda x: script_ops.py_func(my_map, [[x]], dtypes.int64))
+    got = [x.numpy() for x in datasets.Iterator(ds)]
+    self.assertAllEqual([[1], [2], [3], [4]], got)
+
 
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/python/eager/pywrap_tfe_src.cc b/tensorflow/python/eager/pywrap_tfe_src.cc
index a2079d009f..3d64c875ec 100644
--- a/tensorflow/python/eager/pywrap_tfe_src.cc
+++ b/tensorflow/python/eager/pywrap_tfe_src.cc
@@ -342,6 +342,7 @@ void TFE_Py_Execute(TFE_Context* ctx, const char* device_name,
   if (TF_GetCode(out_status) == TF_OK) {
     SetOpAttrs(ctx, op, attrs, out_status);
   }
+  Py_BEGIN_ALLOW_THREADS;
   if (TF_GetCode(out_status) == TF_OK) {
     int num_outputs = outputs->size();
     TFE_Execute(op, outputs->data(), &num_outputs, out_status);
@@ -354,6 +355,7 @@ void TFE_Py_Execute(TFE_Context* ctx, const char* device_name,
                      .c_str());
   }
   TFE_DeleteOp(op);
+  Py_END_ALLOW_THREADS;
 }
 
 PyObject* TFE_Py_RegisterExceptionClass(PyObject* e) {
-- 
GitLab


From b31c03565e18fef7ab4539032dd5c69a94487a05 Mon Sep 17 00:00:00 2001
From: Mustafa Ispir <ispir@google.com>
Date: Thu, 5 Oct 2017 12:55:19 -0700
Subject: [PATCH 072/909] Move profiler hook from contrib to core.

PiperOrigin-RevId: 171194291
---
 tensorflow/contrib/hooks/BUILD                |  20 ---
 .../hooks/python/training/profiler_hook.py    |  87 +------------
 .../python/training/profiler_hook_test.py     | 122 ------------------
 tensorflow/python/BUILD                       |   1 +
 .../training/basic_session_run_hooks.py       |  82 +++++++++++-
 .../training/basic_session_run_hooks_test.py  |  93 +++++++++++++
 tensorflow/python/training/training.py        |   2 +
 .../tensorflow.train.-profiler-hook.pbtxt     |  30 +++++
 .../tools/api/golden/tensorflow.train.pbtxt   |   4 +
 9 files changed, 214 insertions(+), 227 deletions(-)
 delete mode 100644 tensorflow/contrib/hooks/python/training/profiler_hook_test.py
 create mode 100644 tensorflow/tools/api/golden/tensorflow.train.-profiler-hook.pbtxt

diff --git a/tensorflow/contrib/hooks/BUILD b/tensorflow/contrib/hooks/BUILD
index d81e868d4a..1576c9ec9b 100644
--- a/tensorflow/contrib/hooks/BUILD
+++ b/tensorflow/contrib/hooks/BUILD
@@ -19,26 +19,6 @@ py_library(
     ],
     srcs_version = "PY2AND3",
     deps = [
-        "//tensorflow/core:protos_all_py",
-        "//tensorflow/python:client",
-        "//tensorflow/python:platform",
-        "//tensorflow/python:training",
-        "//tensorflow/python:util",
-    ],
-)
-
-py_test(
-    name = "profiler_hook_test",
-    size = "small",
-    srcs = ["python/training/profiler_hook_test.py"],
-    srcs_version = "PY2AND3",
-    deps = [
-        ":hooks",
-        "//tensorflow/contrib/framework:framework_py",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:framework_for_generated_wrappers",
-        "//tensorflow/python:platform",
-        "//tensorflow/python:state_ops",
         "//tensorflow/python:training",
     ],
 )
diff --git a/tensorflow/contrib/hooks/python/training/profiler_hook.py b/tensorflow/contrib/hooks/python/training/profiler_hook.py
index 35aa25edfd..6173aa0797 100644
--- a/tensorflow/contrib/hooks/python/training/profiler_hook.py
+++ b/tensorflow/contrib/hooks/python/training/profiler_hook.py
@@ -12,93 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Additional `SessionRunHook` implementations to complement those in
-tensorflow/python/training.
-
-"""
+"""Placeholder of ProfilerHook for backward compatibility."""
 
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import os.path
-
-from tensorflow.core.protobuf import config_pb2
-from tensorflow.python.client import timeline
-from tensorflow.python.platform import gfile
-from tensorflow.python.platform import tf_logging as logging
-from tensorflow.python.training.basic_session_run_hooks import SecondOrStepTimer
-from tensorflow.python.training.session_run_hook import SessionRunArgs
-from tensorflow.python.training import session_run_hook
-from tensorflow.python.training import training_util
-
-
-class ProfilerHook(session_run_hook.SessionRunHook):
-  """Captures CPU/GPU profiling information every N steps or seconds.
-
-  This produces files called "timeline-<step>.json", which are in Chrome
-  Trace format.
-
-  For more information see:
-  https://github.com/catapult-project/catapult/blob/master/tracing/README.md"""
-
-  def __init__(self,
-               save_steps=None,
-               save_secs=None,
-               output_dir="",
-               show_dataflow=True,
-               show_memory=False):
-    """Initializes a hook that takes periodic profiling snapshots.
-
-    Args:
-      save_steps: `int`, save profile traces every N steps. Exactly one of
-          `save_secs` and `save_steps` should be set.
-      save_secs: `int`, save profile traces every N seconds.
-      output_dir: `string`, the directory to save the profile traces to.
-          Defaults to the current directory.
-      show_dataflow: `bool`, if True, add flow events to the trace connecting
-          producers and consumers of tensors.
-      show_memory: `bool`, if True, add object snapshot events to the trace
-          showing the sizes and lifetimes of tensors.
-    """
-    self._output_file = os.path.join(output_dir, "timeline-{}.json")
-    self._show_dataflow = show_dataflow
-    self._show_memory = show_memory
-    self._timer = SecondOrStepTimer(every_secs=save_secs,
-                                    every_steps=save_steps)
-
-  def begin(self):
-    self._next_step = None
-    self._global_step_tensor = training_util.get_global_step()
-    if self._global_step_tensor is None:
-      raise RuntimeError(
-          "Global step should be created to use ProfilerHook.")
-
-  def before_run(self, run_context):
-    self._request_summary = (
-        self._next_step is None or
-        self._timer.should_trigger_for_step(self._next_step))
-    requests = {"global_step": self._global_step_tensor}
-    opts = (config_pb2.RunOptions(trace_level=config_pb2.RunOptions.FULL_TRACE)
-            if self._request_summary else None)
-
-    return SessionRunArgs(requests, options=opts)
-
-  def after_run(self, run_context, run_values):
-    global_step = run_values.results["global_step"]
-
-    if self._request_summary:
-      self._timer.update_last_triggered_step(global_step)
-      self._save(global_step,
-                 self._output_file.format(global_step),
-                 run_values.run_metadata.step_stats)
-
-    self._next_step = global_step + 1
+from tensorflow.python.training import basic_session_run_hooks
 
-  def _save(self, step, save_path, step_stats):
-    logging.info("Saving timeline for %d into '%s'.", step, save_path)
-    with gfile.Open(save_path, "w") as f:
-      trace = timeline.Timeline(step_stats)
-      f.write(trace.generate_chrome_trace_format(
-          show_dataflow=self._show_dataflow,
-          show_memory=self._show_memory))
+ProfilerHook = basic_session_run_hooks.ProfilerHook  # pylint: disable=invalid-name
diff --git a/tensorflow/contrib/hooks/python/training/profiler_hook_test.py b/tensorflow/contrib/hooks/python/training/profiler_hook_test.py
deleted file mode 100644
index e7ecb5eb2f..0000000000
--- a/tensorflow/contrib/hooks/python/training/profiler_hook_test.py
+++ /dev/null
@@ -1,122 +0,0 @@
-# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Tests for profiler_hook."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import os.path
-import shutil
-import tempfile
-
-from tensorflow.contrib.framework.python.ops import variables
-from tensorflow.contrib.hooks.python.training import ProfilerHook
-from tensorflow.python.framework import ops
-from tensorflow.python.ops import state_ops
-from tensorflow.python.platform import gfile
-from tensorflow.python.platform import test
-from tensorflow.python.training import monitored_session
-
-
-class ProfilerHookTest(test.TestCase):
-
-  def setUp(self):
-    super(ProfilerHookTest, self).setUp()
-    self.output_dir = tempfile.mkdtemp()
-    self.graph = ops.Graph()
-    self.filepattern = os.path.join(self.output_dir, "timeline-*.json")
-    with self.graph.as_default():
-      self.global_step = variables.get_or_create_global_step()
-      self.train_op = state_ops.assign_add(self.global_step, 1)
-
-  def tearDown(self):
-    super(ProfilerHookTest, self).tearDown()
-    shutil.rmtree(self.output_dir, ignore_errors=True)
-
-  def _count_timeline_files(self):
-    return len(gfile.Glob(self.filepattern))
-
-  def test_raise_in_both_secs_and_steps(self):
-    with self.assertRaises(ValueError):
-      ProfilerHook(save_secs=10, save_steps=20)
-
-  def test_raise_in_none_secs_and_steps(self):
-    with self.assertRaises(ValueError):
-      ProfilerHook(save_secs=None, save_steps=None)
-
-  def test_save_secs_saves_in_first_step(self):
-    with self.graph.as_default():
-      hook = ProfilerHook(save_secs=2, output_dir=self.output_dir)
-      with monitored_session.SingularMonitoredSession(hooks=[hook]) as sess:
-        sess.run(self.train_op)
-        self.assertEqual(1, self._count_timeline_files())
-
-  @test.mock.patch('time.time')
-  def test_save_secs_saves_periodically(self, mock_time):
-    # Pick a fixed start time.
-    current_time = 1484863632.320497
-
-    with self.graph.as_default():
-      mock_time.return_value = current_time
-      hook = ProfilerHook(save_secs=2, output_dir=self.output_dir)
-      with monitored_session.SingularMonitoredSession(hooks=[hook]) as sess:
-        sess.run(self.train_op)  # Saved.
-        self.assertEqual(1, self._count_timeline_files())
-        sess.run(self.train_op)  # Not saved.
-        self.assertEqual(1, self._count_timeline_files())
-        # Simulate 2.5 seconds of sleep.
-        mock_time.return_value = current_time + 2.5
-        sess.run(self.train_op)  # Saved.
-
-        # Pretend some small amount of time has passed.
-        mock_time.return_value = current_time + 0.1
-        sess.run(self.train_op)  # Not saved.
-        # Edge test just before we should save the timeline.
-        mock_time.return_value = current_time + 1.9
-        sess.run(self.train_op)  # Not saved.
-        self.assertEqual(2, self._count_timeline_files())
-
-        mock_time.return_value = current_time + 4.5
-        sess.run(self.train_op)  # Saved.
-        self.assertEqual(3, self._count_timeline_files())
-
-  def test_save_steps_saves_in_first_step(self):
-    with self.graph.as_default():
-      hook = ProfilerHook(save_secs=2, output_dir=self.output_dir)
-      with monitored_session.SingularMonitoredSession(hooks=[hook]) as sess:
-        sess.run(self.train_op)  # Saved.
-        sess.run(self.train_op)  # Not saved.
-        self.assertEqual(1, self._count_timeline_files())
-
-  def test_save_steps_saves_periodically(self):
-    with self.graph.as_default():
-      hook = ProfilerHook(save_steps=2, output_dir=self.output_dir)
-      with monitored_session.SingularMonitoredSession(hooks=[hook]) as sess:
-        self.assertEqual(0, self._count_timeline_files())
-        sess.run(self.train_op)  # Saved.
-        self.assertEqual(1, self._count_timeline_files())
-        sess.run(self.train_op)  # Not saved.
-        self.assertEqual(1, self._count_timeline_files())
-        sess.run(self.train_op)  # Saved.
-        self.assertEqual(2, self._count_timeline_files())
-        sess.run(self.train_op)  # Not saved.
-        self.assertEqual(2, self._count_timeline_files())
-        sess.run(self.train_op)  # Saved.
-        self.assertEqual(3, self._count_timeline_files())
-
-
-if __name__ == '__main__':
-  test.main()
diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 407ff079c1..ab3b851ef8 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -3626,6 +3626,7 @@ py_test(
         ":variables",
         "//tensorflow/contrib/framework:framework_py",
         "//tensorflow/contrib/testing:testing_py",
+        "//tensorflow/core:protos_all_py",
     ],
 )
 
diff --git a/tensorflow/python/training/basic_session_run_hooks.py b/tensorflow/python/training/basic_session_run_hooks.py
index 99f057e837..1fb00343ef 100644
--- a/tensorflow/python/training/basic_session_run_hooks.py
+++ b/tensorflow/python/training/basic_session_run_hooks.py
@@ -22,7 +22,7 @@
 @@NanTensorHook
 @@SummarySaverHook
 @@GlobalStepWaiterHook
-
+@@ProfilerHook
 """
 
 from __future__ import absolute_import
@@ -36,9 +36,12 @@ import numpy as np
 import six
 
 from tensorflow.core.framework.summary_pb2 import Summary
+from tensorflow.core.protobuf import config_pb2
 from tensorflow.core.util.event_pb2 import SessionLog
+from tensorflow.python.client import timeline
 from tensorflow.python.framework import meta_graph
 from tensorflow.python.framework import ops
+from tensorflow.python.platform import gfile
 from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.training import session_run_hook
 from tensorflow.python.training import training_util
@@ -778,6 +781,83 @@ class FeedFnHook(session_run_hook.SessionRunHook):
         fetches=None, feed_dict=self.feed_fn())
 
 
+class ProfilerHook(session_run_hook.SessionRunHook):
+  """Captures CPU/GPU profiling information every N steps or seconds.
+
+  This produces files called "timeline-<step>.json", which are in Chrome
+  Trace format.
+
+  For more information see:
+  https://github.com/catapult-project/catapult/blob/master/tracing/README.md
+  """
+
+  def __init__(self,
+               save_steps=None,
+               save_secs=None,
+               output_dir="",
+               show_dataflow=True,
+               show_memory=False):
+    """Initializes a hook that takes periodic profiling snapshots.
+
+    `options.run_metadata` argument of `tf.Session.Run` is used to collect
+    metadata about execution. This hook sets the metadata and dumps it in Chrome
+    Trace format.
+
+
+    Args:
+      save_steps: `int`, save profile traces every N steps. Exactly one of
+          `save_secs` and `save_steps` should be set.
+      save_secs: `int` or `float`, save profile traces every N seconds.
+      output_dir: `string`, the directory to save the profile traces to.
+          Defaults to the current directory.
+      show_dataflow: `bool`, if True, add flow events to the trace connecting
+          producers and consumers of tensors.
+      show_memory: `bool`, if True, add object snapshot events to the trace
+          showing the sizes and lifetimes of tensors.
+    """
+    self._output_file = os.path.join(output_dir, "timeline-{}.json")
+    self._show_dataflow = show_dataflow
+    self._show_memory = show_memory
+    self._timer = SecondOrStepTimer(
+        every_secs=save_secs, every_steps=save_steps)
+
+  def begin(self):
+    self._next_step = None
+    self._global_step_tensor = training_util._get_or_create_global_step_read()  # pylint: disable=protected-access
+    if self._global_step_tensor is None:
+      raise RuntimeError("Global step should be created to use ProfilerHook.")
+
+  def before_run(self, run_context):
+    self._request_summary = (
+        self._next_step is None or
+        self._timer.should_trigger_for_step(self._next_step))
+    requests = {"global_step": self._global_step_tensor}
+    opts = (config_pb2.RunOptions(trace_level=config_pb2.RunOptions.FULL_TRACE)
+            if self._request_summary else None)
+
+    return SessionRunArgs(requests, options=opts)
+
+  def after_run(self, run_context, run_values):
+    stale_global_step = run_values.results["global_step"]
+    global_step = stale_global_step + 1
+    if self._request_summary:
+      global_step = run_context.session.run(self._global_step_tensor)
+      self._timer.update_last_triggered_step(global_step)
+      self._save(global_step,
+                 self._output_file.format(global_step),
+                 run_values.run_metadata.step_stats)
+
+    self._next_step = global_step + 1
+
+  def _save(self, step, save_path, step_stats):
+    logging.info("Saving timeline for %d into '%s'.", step, save_path)
+    with gfile.Open(save_path, "w") as f:
+      trace = timeline.Timeline(step_stats)
+      f.write(
+          trace.generate_chrome_trace_format(
+              show_dataflow=self._show_dataflow, show_memory=self._show_memory))
+
+
 def _as_graph_element(obj):
   """Retrieves Graph element."""
   graph = ops.get_default_graph()
diff --git a/tensorflow/python/training/basic_session_run_hooks_test.py b/tensorflow/python/training/basic_session_run_hooks_test.py
index 96c13edd4c..e7ff7e1221 100644
--- a/tensorflow/python/training/basic_session_run_hooks_test.py
+++ b/tensorflow/python/training/basic_session_run_hooks_test.py
@@ -19,6 +19,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import os.path
 import shutil
 import tempfile
 import threading
@@ -38,6 +39,7 @@ from tensorflow.python.ops import state_ops
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops import variables as variables_lib
 import tensorflow.python.ops.nn_grad  # pylint: disable=unused-import
+from tensorflow.python.platform import gfile
 from tensorflow.python.platform import test
 from tensorflow.python.platform import tf_logging
 from tensorflow.python.summary import summary as summary_lib
@@ -1161,5 +1163,96 @@ class FeedFnHookTest(test.TestCase):
       self.assertEqual(mon_sess.run(y), 2)
 
 
+class ProfilerHookTest(test.TestCase):
+
+  def setUp(self):
+    super(ProfilerHookTest, self).setUp()
+    self.output_dir = tempfile.mkdtemp()
+    self.graph = ops.Graph()
+    self.filepattern = os.path.join(self.output_dir, 'timeline-*.json')
+    with self.graph.as_default():
+      self.global_step = variables.get_or_create_global_step()
+      self.train_op = state_ops.assign_add(self.global_step, 1)
+
+  def tearDown(self):
+    super(ProfilerHookTest, self).tearDown()
+    shutil.rmtree(self.output_dir, ignore_errors=True)
+
+  def _count_timeline_files(self):
+    return len(gfile.Glob(self.filepattern))
+
+  def test_raise_in_both_secs_and_steps(self):
+    with self.assertRaises(ValueError):
+      basic_session_run_hooks.ProfilerHook(save_secs=10, save_steps=20)
+
+  def test_raise_in_none_secs_and_steps(self):
+    with self.assertRaises(ValueError):
+      basic_session_run_hooks.ProfilerHook(save_secs=None, save_steps=None)
+
+  def test_save_secs_saves_in_first_step(self):
+    with self.graph.as_default():
+      hook = basic_session_run_hooks.ProfilerHook(
+          save_secs=2, output_dir=self.output_dir)
+      with monitored_session.SingularMonitoredSession(hooks=[hook]) as sess:
+        sess.run(self.train_op)
+        self.assertEqual(1, self._count_timeline_files())
+
+  @test.mock.patch.object(time, 'time')
+  def test_save_secs_saves_periodically(self, mock_time):
+    # Pick a fixed start time.
+    current_time = 1484863632.320497
+
+    with self.graph.as_default():
+      mock_time.return_value = current_time
+      hook = basic_session_run_hooks.ProfilerHook(
+          save_secs=2, output_dir=self.output_dir)
+      with monitored_session.SingularMonitoredSession(hooks=[hook]) as sess:
+        sess.run(self.train_op)  # Saved.
+        self.assertEqual(1, self._count_timeline_files())
+        sess.run(self.train_op)  # Not saved.
+        self.assertEqual(1, self._count_timeline_files())
+        # Simulate 2.5 seconds of sleep.
+        mock_time.return_value = current_time + 2.5
+        sess.run(self.train_op)  # Saved.
+
+        # Pretend some small amount of time has passed.
+        mock_time.return_value = current_time + 0.1
+        sess.run(self.train_op)  # Not saved.
+        # Edge test just before we should save the timeline.
+        mock_time.return_value = current_time + 1.9
+        sess.run(self.train_op)  # Not saved.
+        self.assertEqual(2, self._count_timeline_files())
+
+        mock_time.return_value = current_time + 4.5
+        sess.run(self.train_op)  # Saved.
+        self.assertEqual(3, self._count_timeline_files())
+
+  def test_save_steps_saves_in_first_step(self):
+    with self.graph.as_default():
+      hook = basic_session_run_hooks.ProfilerHook(
+          save_secs=2, output_dir=self.output_dir)
+      with monitored_session.SingularMonitoredSession(hooks=[hook]) as sess:
+        sess.run(self.train_op)  # Saved.
+        sess.run(self.train_op)  # Not saved.
+        self.assertEqual(1, self._count_timeline_files())
+
+  def test_save_steps_saves_periodically(self):
+    with self.graph.as_default():
+      hook = basic_session_run_hooks.ProfilerHook(
+          save_steps=2, output_dir=self.output_dir)
+      with monitored_session.SingularMonitoredSession(hooks=[hook]) as sess:
+        self.assertEqual(0, self._count_timeline_files())
+        sess.run(self.train_op)  # Saved.
+        self.assertEqual(1, self._count_timeline_files())
+        sess.run(self.train_op)  # Not saved.
+        self.assertEqual(1, self._count_timeline_files())
+        sess.run(self.train_op)  # Saved.
+        self.assertEqual(2, self._count_timeline_files())
+        sess.run(self.train_op)  # Not saved.
+        self.assertEqual(2, self._count_timeline_files())
+        sess.run(self.train_op)  # Saved.
+        self.assertEqual(3, self._count_timeline_files())
+
+
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/python/training/training.py b/tensorflow/python/training/training.py
index e2a7b28e2b..741dddc991 100644
--- a/tensorflow/python/training/training.py
+++ b/tensorflow/python/training/training.py
@@ -77,6 +77,7 @@ See the @{$python/train} guide.
 @@GlobalStepWaiterHook
 @@FinalOpsHook
 @@FeedFnHook
+@@ProfilerHook
 @@SecondOrStepTimer
 @@global_step
 @@basic_train_loop
@@ -145,6 +146,7 @@ from tensorflow.python.training.basic_session_run_hooks import SummarySaverHook
 from tensorflow.python.training.basic_session_run_hooks import GlobalStepWaiterHook
 from tensorflow.python.training.basic_session_run_hooks import FinalOpsHook
 from tensorflow.python.training.basic_session_run_hooks import FeedFnHook
+from tensorflow.python.training.basic_session_run_hooks import ProfilerHook
 from tensorflow.python.training.basic_loops import basic_train_loop
 from tensorflow.python.training.checkpoint_utils import init_from_checkpoint
 from tensorflow.python.training.checkpoint_utils import list_variables
diff --git a/tensorflow/tools/api/golden/tensorflow.train.-profiler-hook.pbtxt b/tensorflow/tools/api/golden/tensorflow.train.-profiler-hook.pbtxt
new file mode 100644
index 0000000000..4df6c4156a
--- /dev/null
+++ b/tensorflow/tools/api/golden/tensorflow.train.-profiler-hook.pbtxt
@@ -0,0 +1,30 @@
+path: "tensorflow.train.ProfilerHook"
+tf_class {
+  is_instance: "<class \'tensorflow.python.training.basic_session_run_hooks.ProfilerHook\'>"
+  is_instance: "<class \'tensorflow.python.training.session_run_hook.SessionRunHook\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'save_steps\', \'save_secs\', \'output_dir\', \'show_dataflow\', \'show_memory\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'\', \'True\', \'False\'], "
+  }
+  member_method {
+    name: "after_create_session"
+    argspec: "args=[\'self\', \'session\', \'coord\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "after_run"
+    argspec: "args=[\'self\', \'run_context\', \'run_values\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "before_run"
+    argspec: "args=[\'self\', \'run_context\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "begin"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "end"
+    argspec: "args=[\'self\', \'session\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/tensorflow.train.pbtxt b/tensorflow/tools/api/golden/tensorflow.train.pbtxt
index 835d3f835d..edc29e62dd 100644
--- a/tensorflow/tools/api/golden/tensorflow.train.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.train.pbtxt
@@ -128,6 +128,10 @@ tf_module {
     name: "Optimizer"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "ProfilerHook"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "ProximalAdagradOptimizer"
     mtype: "<type \'type\'>"
-- 
GitLab


From a429d07bf545b5fd25c44f95fd50e012440bf99b Mon Sep 17 00:00:00 2001
From: Martin Wicke <wicke@google.com>
Date: Thu, 5 Oct 2017 12:58:48 -0700
Subject: [PATCH 073/909] Move Head to the new summary API. This may change the
 names of summaries produced, but will avoid tag collisions.

PiperOrigin-RevId: 171194758
---
 .../learn/python/learn/estimators/head.py      | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/tensorflow/contrib/learn/python/learn/estimators/head.py b/tensorflow/contrib/learn/python/learn/estimators/head.py
index a67694d1c9..468d792a0d 100644
--- a/tensorflow/contrib/learn/python/learn/estimators/head.py
+++ b/tensorflow/contrib/learn/python/learn/estimators/head.py
@@ -33,7 +33,6 @@ from tensorflow.python.framework import ops
 from tensorflow.python.framework import sparse_tensor
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
-from tensorflow.python.ops import logging_ops
 from tensorflow.python.ops import lookup_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import metrics as metrics_lib
@@ -635,10 +634,11 @@ def _create_model_fn_ops(features,
   if (mode != model_fn.ModeKeys.INFER) and (labels is not None):
     weight_tensor = _weight_tensor(features, weight_column_name)
     loss, weighted_average_loss = loss_fn(labels, logits, weight_tensor)
-    # Uses the deprecated API to set the tag explicitly.
-    # Without it, training and eval losses will show up in different graphs.
-    logging_ops.scalar_summary(
-        _summary_key(head_name, mkey.LOSS), weighted_average_loss)
+    # The name_scope escapism is needed to maintain the same summary tag
+    # after switching away from the now unsupported API.
+    with ops.name_scope(""):
+      summary_loss = array_ops.identity(weighted_average_loss)
+      summary.scalar(_summary_key(head_name, mkey.LOSS), summary_loss)
 
     if mode == model_fn.ModeKeys.TRAIN:
       if train_op_fn is None:
@@ -1484,8 +1484,12 @@ class _LossOnlyHead(Head):
         loss = self._loss_fn()
         if isinstance(loss, list):
           loss = math_ops.add_n(loss)
-        logging_ops.scalar_summary(
-            _summary_key(self.head_name, mkey.LOSS), loss)
+        # The name_scope escapism is needed to maintain the same summary tag
+        # after switching away from the now unsupported API.
+        with ops.name_scope(""):
+          summary_loss = array_ops.identity(loss)
+          summary.scalar(_summary_key(self.head_name, mkey.LOSS),
+                         summary_loss)
         if mode == model_fn.ModeKeys.TRAIN:
           if train_op_fn is None:
             raise ValueError("train_op_fn can not be None in TRAIN mode")
-- 
GitLab


From 631d3434ff33debfd0bf46d9d8602172f549c82d Mon Sep 17 00:00:00 2001
From: Jianwei Xie <xiejw@google.com>
Date: Thu, 5 Oct 2017 12:58:51 -0700
Subject: [PATCH 074/909] Adds throlle_secs into run_master

PiperOrigin-RevId: 171194766
---
 tensorflow/python/estimator/training.py      |  74 +++--
 tensorflow/python/estimator/training_test.py | 268 +++++++++++++++++--
 2 files changed, 307 insertions(+), 35 deletions(-)

diff --git a/tensorflow/python/estimator/training.py b/tensorflow/python/estimator/training.py
index 5c0ebbea35..64b014a6b5 100644
--- a/tensorflow/python/estimator/training.py
+++ b/tensorflow/python/estimator/training.py
@@ -519,23 +519,51 @@ class _TrainingExecutor(object):
     class NewCheckpointListener(
         basic_session_run_hooks.CheckpointSaverListener):
 
-      def __init__(self, estimator, eval_spec, max_training_steps):
-        # pylint: disable=protected-access
-        self._evaluator = _TrainingExecutor._Evaluator(estimator, eval_spec,
-                                                       max_training_steps)
-        # pylint: enable=protected-access
+      def __init__(self, evaluator, eval_throttle_secs):
+        self._evaluator = evaluator
+        self._eval_throttle_secs = eval_throttle_secs
+
+      def begin(self):
+        self._timer = basic_session_run_hooks.SecondOrStepTimer(
+            every_secs=self._eval_throttle_secs)
 
       def after_save(self, session, global_step_value):
-        del session, global_step_value
-        self._evaluator.evaluate_and_export()
+        del session  # unused; required by signature.
+
+        if self._timer.should_trigger_for_step(global_step_value):
+          self._timer.update_last_triggered_step(global_step_value)
+          self._evaluator.evaluate_and_export()
+        else:
+          logging.info(
+              'Skip the current checkpoint eval due to throttle secs '
+              '({} secs).'.format(self._eval_throttle_secs))
+
+    # Final export signal: For any eval result with global_step >= train
+    # max_steps, the evaluator will send the final export signal. There is a
+    # small chance that the Estimator.train stopping logic sees a different
+    # global_step value (due to global step race condition and the fact the
+    # saver sees a larger value for checkpoing saving), which does not end
+    # the training. When the training ends, a new checkpoint is generated, which
+    # triggers the listener again. So, it could be the case the final export is
+    # triggered twice.
+    #
+    # But here, throttle_secs will skip the next intermediate checkpoint and,
+    # so, the double final export chance is very small.
+    evaluator = _TrainingExecutor._Evaluator(
+        self._estimator, self._eval_spec, self._train_spec.max_steps)
 
     # When the underlying `Estimator` object saves a new checkpoint, we would
     # like this callback to be called so that evaluation and export can trigger.
     saving_listeners = [
-        NewCheckpointListener(self._estimator, self._eval_spec,
-                              self._train_spec.max_steps)
+        NewCheckpointListener(evaluator, self._eval_spec.throttle_secs)
     ]
-    return self._start_distributed_training(saving_listeners=saving_listeners)
+    self._start_distributed_training(saving_listeners=saving_listeners)
+
+    if not evaluator.is_final_export_triggered:
+      logging.info('Training has already ended. But the last eval is skipped '
+                   'due to eval throttle_secs. Now evaluating the final '
+                   'checkpoint.')
+      evaluator.evaluate_and_export()
 
   def run_evaluator(self):
     """Runs task evaluator."""
@@ -580,6 +608,11 @@ class _TrainingExecutor(object):
           max_steps=self._train_spec.max_steps,
           hooks=train_hooks)
 
+      # Final export signal: For any eval result with global_step >= train
+      # max_steps, the evaluator will send the final export signal. The
+      # _should_stop_local_train will then end the while True as the stopping
+      # condition is satisfied (both checks use the same global_step value,
+      # i.e., no race condition)
       metrics = evaluator.evaluate_and_export()
 
       if not metrics:
@@ -656,6 +689,11 @@ class _TrainingExecutor(object):
               self._train_spec.max_steps)
           return
 
+      # Final export signal: For any eval result with global_step >= train
+      # max_steps, the evaluator will send the final export signal. The next
+      # iteration of while loop will end the continuous eval as the stopping
+      # condition is satisfied (both checks use the same global_step value,
+      # i.e., no race condition)
       start = time.time()
       latest_eval_result = evaluator.evaluate_and_export()
 
@@ -673,10 +711,15 @@ class _TrainingExecutor(object):
     def __init__(self, estimator, eval_spec, max_training_steps):
       self._estimator = estimator
       self._eval_spec = eval_spec
+      self._is_final_export_triggered = False
       self._previous_ckpt_path = None
       self._last_warning_time = 0
       self._max_training_steps = max_training_steps
 
+    @property
+    def is_final_export_triggered(self):
+      return self._is_final_export_triggered
+
     def evaluate_and_export(self):
       """Evaluate and (maybe) export the current model.
 
@@ -720,15 +763,16 @@ class _TrainingExecutor(object):
             'Internal error: `Estimator.evaluate` result should have '
             '`global_step` in result. Given {}'.format(eval_result))
 
-      # TODO(isaprykin):  There is a potential race condition here in the
-      #  distributed setting.  The worker job that performs training
-      #  might stop at a later global step value than the evalutor job.
       is_the_final_export = (eval_result[ops.GraphKeys.GLOBAL_STEP] >=
                              self._max_training_steps
                              if self._max_training_steps else False)
       self._export_eval_result(eval_result, latest_ckpt_path,
                                is_the_final_export)
 
+      if is_the_final_export:
+        logging.debug('Calling exporter with the `is_the_final_export=True`.')
+        self._is_final_export_triggered = True
+
       self._last_warning_time = 0
       self._previous_ckpt_path = latest_ckpt_path
       return eval_result
@@ -749,8 +793,8 @@ class _TrainingExecutor(object):
 
       for exporter in self._eval_spec.exporters:
         exporter.export(
-            self._estimator,
-            os.path.join(
+            estimator=self._estimator,
+            export_path=os.path.join(
                 compat.as_str_any(export_dir_base),
                 compat.as_str_any(exporter.name)),
             checkpoint_path=checkpoint_path,
diff --git a/tensorflow/python/estimator/training_test.py b/tensorflow/python/estimator/training_test.py
index 40972ab5a0..8c00ebddf3 100644
--- a/tensorflow/python/estimator/training_test.py
+++ b/tensorflow/python/estimator/training_test.py
@@ -45,6 +45,7 @@ from tensorflow.python.platform import test
 from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.summary import summary_iterator
 from tensorflow.python.summary.writer import writer_cache
+from tensorflow.python.training import basic_session_run_hooks
 from tensorflow.python.training import monitored_session
 from tensorflow.python.training import server_lib
 from tensorflow.python.training import session_run_hook
@@ -692,37 +693,145 @@ class TrainingExecutorRunChiefTest(_TrainingExecutorTrainingTest,
       mock_sleep.assert_not_called()
 
 
-class TrainingExecutorRunMasterTest(_TrainingExecutorTrainingTest,
-                                    test.TestCase):
+class TrainingExecutorRunMasterTest(test.TestCase):
   """Tests run_chief of _TrainingExecutor."""
 
-  def __init__(self, methodName='runTest'):  # pylint: disable=invalid-name
-    test.TestCase.__init__(self, methodName)
-    _TrainingExecutorTrainingTest.__init__(
-        self,
-        run_config=_create_run_config_with_cluster_spec(_TF_CONFIG_FOR_MASTER))
+  def setUp(self):
+    self._run_config = _create_run_config_with_cluster_spec(
+        _TF_CONFIG_FOR_MASTER)
 
   @test.mock.patch.object(server_lib, 'Server')
   def test_no_delay_for_master(self, _):
     mock_est = test.mock.Mock(spec=estimator_lib.Estimator)
+    mock_est.evaluate = lambda *args, **kw: {ops.GraphKeys.GLOBAL_STEP: 123}
     mock_est.config = self._run_config
     mock_train_spec = test.mock.Mock(spec=training.TrainSpec)
-    mock_eval_spec = test.mock.Mock(spec=training.EvalSpec)
+    mock_eval_spec = test.mock.Mock(spec=training.EvalSpec, exporters=[])
 
     executor = training._TrainingExecutor(mock_est, mock_train_spec,
                                           mock_eval_spec)
 
     with test.mock.patch.object(time, 'sleep') as mock_sleep:
-      self._run_task(executor)
+      executor.run_master()
       mock_sleep.assert_not_called()
 
+  @test.mock.patch.object(time, 'sleep')
+  @test.mock.patch.object(server_lib, 'Server')
+  def test_train_with_train_spec(self, mock_server, unused_mock_sleep):
+    mock_est = test.mock.Mock(spec=estimator_lib.Estimator)
+    mock_est.evaluate = lambda *args, **kw: {ops.GraphKeys.GLOBAL_STEP: 123}
+    mock_est.config = self._run_config
+    train_spec = training.TrainSpec(
+        input_fn=lambda: 1, max_steps=2, hooks=[_FakeHook()])
+    mock_eval_spec = test.mock.Mock(spec=training.EvalSpec, exporters=[])
+    mock_server_instance = mock_server.return_value
+
+    executor = training._TrainingExecutor(mock_est, train_spec, mock_eval_spec)
+    executor.run_master()
+
+    mock_server.assert_called_with(
+        mock_est.config.cluster_spec,
+        job_name=mock_est.config.task_type,
+        task_index=mock_est.config.task_id,
+        config=test.mock.ANY,
+        start=False)
+
+    self.assertTrue(mock_server_instance.start.called)
+
+    mock_est.train.assert_called_with(input_fn=train_spec.input_fn,
+                                      max_steps=train_spec.max_steps,
+                                      hooks=train_spec.hooks,
+                                      saving_listeners=test.mock.ANY)
+    mock_est.export_savedmodel.assert_not_called()
+
+  @test.mock.patch.object(time, 'sleep')
+  @test.mock.patch.object(server_lib, 'Server')
+  def test_no_server_startup_in_google(self, mock_server, unused_mock_sleep):
+    mock_est = test.mock.Mock(spec=estimator_lib.Estimator)
+    mock_est.evaluate = lambda *args, **kw: {ops.GraphKeys.GLOBAL_STEP: 123}
+    mock_est.config = self._run_config
+    mock_train_spec = test.mock.Mock(spec=training.TrainSpec)
+    mock_eval_spec = test.mock.Mock(spec=training.EvalSpec, exporters=[])
+
+    executor = training._TrainingExecutor(mock_est, mock_train_spec,
+                                          mock_eval_spec)
+    tf_config = {'TF_CONFIG': json.dumps(_TF_CONFIG_FOR_GOOGLE)}
+    with test.mock.patch.dict('os.environ', tf_config):
+      executor.run_master()
+      mock_server.assert_not_called()
+
+  def test_fail_with_empty_cluster_spec(self):
+    mock_est = test.mock.Mock(spec=estimator_lib.Estimator)
+    mock_train_spec = test.mock.Mock(spec=training.TrainSpec)
+    mock_eval_spec = test.mock.Mock(spec=training.EvalSpec)
+
+    mock_est.config = test.mock.PropertyMock(spec=run_config_lib.RunConfig)
+    mock_est.config.cluster_spec = None
+    mock_est.config.master = 'grpc://...'
+    mock_est.config.task_type = 'worker'
+    mock_est.config.task_id = 2
+
+    with self.assertRaisesRegexp(RuntimeError,
+                                 _INVALID_CONFIG_FOR_STD_SERVER_MSG):
+      training._TrainingExecutor(
+          mock_est, mock_train_spec, mock_eval_spec).run_master()
+
+  def test_fail_with_empty_master(self):
+    mock_est = test.mock.Mock(spec=estimator_lib.Estimator)
+    mock_train_spec = test.mock.Mock(spec=training.TrainSpec)
+    mock_eval_spec = test.mock.Mock(spec=training.EvalSpec)
+
+    mock_est.config = test.mock.PropertyMock(spec=run_config_lib.RunConfig)
+    mock_est.config.cluster_spec = {'worker': 'dummy'}
+    mock_est.config.master = ''
+    mock_est.config.task_type = 'worker'
+    mock_est.config.task_id = 2
+
+    with self.assertRaisesRegexp(RuntimeError,
+                                 _INVALID_CONFIG_FOR_STD_SERVER_MSG):
+      training._TrainingExecutor(
+          mock_est, mock_train_spec, mock_eval_spec).run_master()
+
+  def test_fail_with_empty_task_type(self):
+    mock_est = test.mock.Mock(spec=estimator_lib.Estimator)
+    mock_train_spec = test.mock.Mock(spec=training.TrainSpec)
+    mock_eval_spec = test.mock.Mock(spec=training.EvalSpec)
+
+    mock_est.config = test.mock.PropertyMock(spec=run_config_lib.RunConfig)
+    mock_est.config.cluster_spec = {'worker': 'dummy'}
+    mock_est.config.master = 'grpc://...'
+    mock_est.config.task_type = ''
+    mock_est.config.task_id = 2
+
+    with self.assertRaisesRegexp(RuntimeError,
+                                 _INVALID_CONFIG_FOR_STD_SERVER_MSG):
+      training._TrainingExecutor(
+          mock_est, mock_train_spec, mock_eval_spec).run_master()
+
+  def test_fail_with_none_task_id(self):
+    mock_est = test.mock.Mock(spec=estimator_lib.Estimator)
+    mock_train_spec = test.mock.Mock(spec=training.TrainSpec)
+    mock_eval_spec = test.mock.Mock(spec=training.EvalSpec)
+
+    mock_est.config = test.mock.PropertyMock(spec=run_config_lib.RunConfig)
+    mock_est.config.cluster_spec = {'worker': 'dummy'}
+    mock_est.config.master = 'grpc://...'
+    mock_est.config.task_type = 'worker'
+    mock_est.config.task_id = None
+
+    with self.assertRaisesRegexp(RuntimeError,
+                                 _INVALID_CONFIG_FOR_STD_SERVER_MSG):
+      training._TrainingExecutor(
+          mock_est, mock_train_spec, mock_eval_spec).run_master()
+
   @test.mock.patch.object(server_lib, 'Server')
-  def test_run_master_triggers_evaluate(self, _):
+  def test_run_master_triggers_evaluate_and_export(self, _):
 
     def estimator_train(saving_listeners, *args, **kwargs):
       #  There shalt be a saving_listener.  Estimator is going to call
       # `after_save`.
       del args, kwargs
+      saving_listeners[0].begin()
       saving_listeners[0].after_save(session=None, global_step_value=None)
 
     mock_est = test.mock.Mock(
@@ -730,18 +839,14 @@ class TrainingExecutorRunMasterTest(_TrainingExecutorTrainingTest,
     mock_est.latest_checkpoint.return_value = 'checkpoint_path/'
     mock_est.config = self._run_config
 
-    def export(estimator, *args, **kwargs):
-      del args, kwargs
-      estimator.export_was_called = True
-
     exporter = test.mock.PropertyMock(spec=exporter_lib.Exporter)
     exporter.name = 'see_whether_export_is_called'
-    exporter.export = export
 
     train_spec = training.TrainSpec(input_fn=lambda: 1, max_steps=300)
     eval_spec = training.EvalSpec(
         input_fn=lambda: 1, steps=2, exporters=exporter)
-    mock_est.evaluate.return_value = {_GLOBAL_STEP_KEY: train_spec.max_steps}
+    eval_result = {_GLOBAL_STEP_KEY: train_spec.max_steps}
+    mock_est.evaluate.return_value = eval_result
 
     executor = training._TrainingExecutor(mock_est, train_spec, eval_spec)
     executor.run_master()
@@ -752,7 +857,109 @@ class TrainingExecutorRunMasterTest(_TrainingExecutorTrainingTest,
         steps=eval_spec.steps,
         checkpoint_path='checkpoint_path/',
         hooks=eval_spec.hooks)
-    self.assertTrue(mock_est.export_was_called)
+    self.assertEqual(1, exporter.export.call_count)
+    exporter.export.assert_called_with(
+        estimator=mock_est,
+        export_path=os.path.join('path/', 'export', exporter.name),
+        checkpoint_path='checkpoint_path/',
+        eval_result=eval_result,
+        is_the_final_export=True)
+
+  @test.mock.patch.object(basic_session_run_hooks, 'SecondOrStepTimer')
+  @test.mock.patch.object(server_lib, 'Server')
+  def test_run_master_throttle_eval(self, _, mock_timer_class):
+    mock_est = test.mock.Mock(spec=estimator_lib.Estimator, model_dir='path/')
+
+    mock_timer = test.mock.Mock()
+    mock_timer_class.return_value = mock_timer
+
+    def estimator_train(saving_listeners, *args, **kwargs):
+      del args, kwargs
+      saving_listeners[0].begin()
+
+      # Call three times.
+      mock_timer.should_trigger_for_step.return_value = True
+      saving_listeners[0].after_save(session=None, global_step_value=None)
+
+      mock_timer.should_trigger_for_step.return_value = False
+      saving_listeners[0].after_save(session=None, global_step_value=None)
+
+      mock_timer.should_trigger_for_step.return_value = True
+      saving_listeners[0].after_save(session=None, global_step_value=None)
+
+    mock_est.train = estimator_train
+    mock_est.latest_checkpoint.side_effect = ['ckpt1', 'ckpt2']
+    mock_est.config = self._run_config
+
+    exporter = test.mock.PropertyMock(spec=exporter_lib.Exporter)
+    exporter.name = 'see_whether_export_is_called'
+
+    train_spec = training.TrainSpec(input_fn=lambda: 1, max_steps=300)
+    eval_spec = training.EvalSpec(
+        input_fn=lambda: 1, steps=2, exporters=exporter, throttle_secs=10)
+
+    mock_est.evaluate.side_effect = [
+        {_GLOBAL_STEP_KEY: train_spec.max_steps //2},
+        {_GLOBAL_STEP_KEY: train_spec.max_steps}
+    ]
+
+    executor = training._TrainingExecutor(mock_est, train_spec, eval_spec)
+    executor.run_master()
+
+    self.assertEqual(2, mock_est.evaluate.call_count)
+    self.assertEqual(2, exporter.export.call_count)
+
+    is_final_export_list = [call[1]['is_the_final_export']
+                            for call in exporter.export.call_args_list]
+    self.assertEqual([False, True], is_final_export_list)
+
+  @test.mock.patch.object(basic_session_run_hooks, 'SecondOrStepTimer')
+  @test.mock.patch.object(server_lib, 'Server')
+  def test_run_master_throttle_eval_which_skips_final_ckpt(
+      self, _, mock_timer_class):
+    mock_est = test.mock.Mock(spec=estimator_lib.Estimator, model_dir='path/')
+
+    mock_timer = test.mock.Mock()
+    mock_timer_class.return_value = mock_timer
+
+    def estimator_train(saving_listeners, *args, **kwargs):
+      del args, kwargs
+      saving_listeners[0].begin()
+
+      # Call two times.
+      mock_timer.should_trigger_for_step.return_value = True
+      saving_listeners[0].after_save(session=None, global_step_value=None)
+
+      # The final ckpt is skipped by the timer. It will be picked up the final
+      # export check in the code.
+      mock_timer.should_trigger_for_step.return_value = False
+      saving_listeners[0].after_save(session=None, global_step_value=None)
+
+    mock_est.train = estimator_train
+    mock_est.latest_checkpoint.side_effect = ['ckpt1', 'ckpt2']
+    mock_est.config = self._run_config
+
+    exporter = test.mock.PropertyMock(spec=exporter_lib.Exporter)
+    exporter.name = 'see_whether_export_is_called'
+
+    train_spec = training.TrainSpec(input_fn=lambda: 1, max_steps=300)
+    eval_spec = training.EvalSpec(
+        input_fn=lambda: 1, steps=2, exporters=exporter, throttle_secs=10)
+
+    mock_est.evaluate.side_effect = [
+        {_GLOBAL_STEP_KEY: train_spec.max_steps //2},
+        {_GLOBAL_STEP_KEY: train_spec.max_steps}
+    ]
+
+    executor = training._TrainingExecutor(mock_est, train_spec, eval_spec)
+    executor.run_master()
+
+    self.assertEqual(2, mock_est.evaluate.call_count)
+    self.assertEqual(2, exporter.export.call_count)
+
+    is_final_export_list = [call[1]['is_the_final_export']
+                            for call in exporter.export.call_args_list]
+    self.assertEqual([False, True], is_final_export_list)
 
 
 class TrainingExecutorRunEvaluatorTest(test.TestCase):
@@ -803,6 +1010,19 @@ class TrainingExecutorRunEvaluatorTest(test.TestCase):
     exporter = test.mock.PropertyMock(spec=exporter_lib.Exporter)
     exporter.name = 'see_how_many_times_export_is_called'
 
+    mock_est.times_export_was_called = 0
+    mock_est.times_final_export_was_called = 0
+    def export(estimator, export_path, checkpoint_path, eval_result,
+               is_the_final_export):
+      del export_path, checkpoint_path, eval_result
+      estimator.times_export_was_called += 1
+      # final_export is happend at the end.
+      self.assertEqual(0, estimator.times_final_export_was_called)
+      if is_the_final_export:
+        estimator.times_final_export_was_called += 1
+
+    exporter.export = export
+
     eval_spec = training.EvalSpec(
         input_fn=lambda: 1,
         start_delay_secs=0,
@@ -813,7 +1033,8 @@ class TrainingExecutorRunEvaluatorTest(test.TestCase):
     executor.run_evaluator()
 
     self.assertEqual(2, mock_est.evaluate.call_count)
-    self.assertEqual(2, exporter.export.call_count)
+    self.assertEqual(2, mock_est.times_export_was_called)
+    self.assertEqual(1, mock_est.times_final_export_was_called)
 
   def test_final_export_is_true_in_the_end(self):
     training_max_step = 200
@@ -1135,9 +1356,15 @@ class TrainingExecutorRunLocalTest(test.TestCase):
     mock_est.latest_checkpoint = self.unique_checkpoint_every_time_fn
 
     mock_est.times_export_was_called = 0
-    def export(estimator, *args, **kwargs):
-      del args, kwargs
+    mock_est.times_final_export_was_called = 0
+    def export(estimator, export_path, checkpoint_path, eval_result,
+               is_the_final_export):
+      del export_path, checkpoint_path, eval_result
       estimator.times_export_was_called += 1
+      # final_export is happend at the end.
+      self.assertEqual(0, estimator.times_final_export_was_called)
+      if is_the_final_export:
+        estimator.times_final_export_was_called += 1
 
     exporter = test.mock.PropertyMock(spec=exporter_lib.Exporter)
     exporter.name = 'see_how_many_times_export_is_called'
@@ -1165,6 +1392,7 @@ class TrainingExecutorRunLocalTest(test.TestCase):
     self.assertEqual(3, mock_est.train.call_count)
     self.assertEqual(3, mock_est.evaluate.call_count)
     self.assertEqual(3, mock_est.times_export_was_called)
+    self.assertEqual(1, mock_est.times_final_export_was_called)
 
   def test_handles_no_new_checkpoint_found(self):
     mock_est = test.mock.Mock(spec=estimator_lib.Estimator, model_dir='path/')
-- 
GitLab


From c8b3f67ba3f8895ebaf0cc78f1859a604ac68c16 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 5 Oct 2017 13:01:02 -0700
Subject: [PATCH 075/909] Fix checkpoint_path is None handling in export_fn of
 make_best_model_export_strategy.

PiperOrigin-RevId: 171195079
---
 .../learn/python/learn/utils/saved_model_export_utils.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/learn/python/learn/utils/saved_model_export_utils.py b/tensorflow/contrib/learn/python/learn/utils/saved_model_export_utils.py
index ee8856ac34..5975103f4f 100644
--- a/tensorflow/contrib/learn/python/learn/utils/saved_model_export_utils.py
+++ b/tensorflow/contrib/learn/python/learn/utils/saved_model_export_utils.py
@@ -50,6 +50,7 @@ from tensorflow.python.platform import gfile
 from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.saved_model import signature_constants
 from tensorflow.python.saved_model import signature_def_utils
+from tensorflow.python.training import saver
 
 from tensorflow.python.util import compat
 
@@ -616,7 +617,13 @@ def make_best_model_export_strategy(serving_input_fn,
     Returns:
       The string path to the exported directory.
     """
-
+    if not checkpoint_path:
+      # TODO(b/67425018): switch to
+      #    checkpoint_path = estimator.latest_checkpoint()
+      #  as soon as contrib is cleaned up and we can thus be sure that
+      #  estimator is a tf.estimator.Estimator and not a
+      #  tf.contrib.learn.Estimator
+      checkpoint_path = saver.latest_checkpoint(estimator.model_dir)
     export_checkpoint_path, export_eval_result = best_model_selector.update(
         checkpoint_path, eval_result)
 
-- 
GitLab


From b56568b8db2b5cfedf53d92ddcff13e3603fbc29 Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Thu, 5 Oct 2017 13:31:18 -0700
Subject: [PATCH 076/909] Disable six tests. One is too big, three are broken
 due to knowon matrix_set_diag issues on windows, one is failing due to
 numerical discrepancies between OSs, and one is broken when multiple GPUs are
 present.

PiperOrigin-RevId: 171199546
---
 tensorflow/contrib/cmake/tf_tests.cmake | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/tensorflow/contrib/cmake/tf_tests.cmake b/tensorflow/contrib/cmake/tf_tests.cmake
index 55d57b7574..4cf22a9c47 100644
--- a/tensorflow/contrib/cmake/tf_tests.cmake
+++ b/tensorflow/contrib/cmake/tf_tests.cmake
@@ -229,6 +229,8 @@ if (tensorflow_BUILD_PYTHON_TESTS)
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/cholesky_op_test.py"
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/diag_op_test.py"
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/linalg_ops_test.py"
+      "${tensorflow_source_dir}/tensorflow/python/ops/init_ops.py"
+      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/self_adjoint_eig_op_test.py"
       # misc
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/variable_scope_test.py"
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/reshape_op_test.py"
@@ -244,6 +246,9 @@ if (tensorflow_BUILD_PYTHON_TESTS)
       "${tensorflow_source_dir}/tensorflow/python/training/supervisor_test.py"  # Flaky I/O error on rename.
       "${tensorflow_source_dir}/tensorflow/python/training/sync_replicas_optimizer_test.py"  # Needs portpicker.
       "${tensorflow_source_dir}/tensorflow/python/training/server_lib_test.py"  # Test occasionally deadlocks.
+      "${tensorflow_source_dir}/tensorflow/python/debug/lib/session_debug_multi_gpu_test.py"
+      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/concat_op_test.py"  # numerical issues
+      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/linalg_grad_test.py"  # cudaSolver handle creation fails.
 
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/array_ops_test.py"  # depends on python/framework/test_ops     
       # Dataset tests
@@ -303,6 +308,7 @@ if (tensorflow_BUILD_PYTHON_TESTS)
       "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/bijectors/sinh_arcsinh_test.py"
       # Test should only be run manually
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/reduction_ops_test_big.py"
+      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/svd_op_test.py"
   )
   endif()
   list(REMOVE_ITEM tf_test_src_py ${tf_test_src_py_exclude})
-- 
GitLab


From 94b81fabaedc85a143fca37304b5b143f936f541 Mon Sep 17 00:00:00 2001
From: Mike Case <mikecase@google.com>
Date: Thu, 5 Oct 2017 13:38:39 -0700
Subject: [PATCH 077/909] Make GCS and HDFS default build options.

---
 configure.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/configure.py b/configure.py
index 9ca614f8f9..6d22d33b99 100644
--- a/configure.py
+++ b/configure.py
@@ -988,9 +988,9 @@ def main():
   set_build_var(environ_cp, 'TF_NEED_JEMALLOC', 'jemalloc as malloc',
                 'with_jemalloc', True)
   set_build_var(environ_cp, 'TF_NEED_GCP', 'Google Cloud Platform',
-                'with_gcp_support', False, 'gcp')
+                'with_gcp_support', True, 'gcp')
   set_build_var(environ_cp, 'TF_NEED_HDFS', 'Hadoop File System',
-                'with_hdfs_support', False, 'hdfs')
+                'with_hdfs_support', True, 'hdfs')
   set_build_var(environ_cp, 'TF_ENABLE_XLA', 'XLA JIT', 'with_xla_support',
                 False, 'xla')
   set_build_var(environ_cp, 'TF_NEED_GDR', 'GDR', 'with_gdr_support',
-- 
GitLab


From 2198b8cfe8acb5af7bb5a1dac54c18ff72c98002 Mon Sep 17 00:00:00 2001
From: Igor Ganichev <iga@google.com>
Date: Thu, 5 Oct 2017 13:41:54 -0700
Subject: [PATCH 078/909] Minimize python code in supporting TF_Function.

After this change when C API is enabled, function support in Python is
done with minimal use of Python code. In particular, we don't create
or store FunctionDef in Python. Small changes include:
- We don't use _hash_str for function comparisons in Python. Instead,
  we delegate this logic to TF_GraphCopyFunction in C API.
- We checking for duplication function additions from
  _DefinedFunction.add_to_graph(graph) to Graph._add_function in all
  cases. This is more logical and make it easier to support both modes.
- We change some error messages to be same in both modes.
- Since we don't store FunctionDef in C API mode in Python but get it
  on demand, access to common attributes like name or signature can
  become expensive. To mitigate this, we cache the signature (OpDef)
  of the function in Python. Signatures are generally much smaller
  than whole definitions.
- Add context manager for creating and destroying TF_Buffers.
- Allow zero output tensorflow functions in Python
  The C API and C++ runtime support functions without outputs, but Python
  APIs explicitly disallowed them before this change. This change allows
  zero output functions in Python and cleans some hacks that were added
  to side-step regular Python function APIs before.

PiperOrigin-RevId: 171201162
---
 tensorflow/compiler/tests/jit_test.py         |  27 +--
 tensorflow/python/framework/c_api_util.py     |  23 +++
 tensorflow/python/framework/function.py       | 160 ++++++++++++------
 tensorflow/python/framework/function_test.py  |  40 ++---
 .../python/framework/graph_to_function_def.py |   9 +-
 tensorflow/python/framework/ops.py            |  32 +++-
 6 files changed, 180 insertions(+), 111 deletions(-)

diff --git a/tensorflow/compiler/tests/jit_test.py b/tensorflow/compiler/tests/jit_test.py
index 11914080ec..2d8236e2cb 100644
--- a/tensorflow/compiler/tests/jit_test.py
+++ b/tensorflow/compiler/tests/jit_test.py
@@ -21,15 +21,12 @@ from __future__ import print_function
 import numpy as np
 
 from tensorflow.contrib.compiler import jit
-from tensorflow.core.framework import function_pb2
-from tensorflow.core.framework import node_def_pb2
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.python.client import session as session_lib
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import function
 from tensorflow.python.framework import ops
-from tensorflow.python.framework import tensor_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import gradients_impl
@@ -118,31 +115,13 @@ class JitLaunchTest(test.TestCase):
 
   def testNoOutputs(self):
     with session_lib.Session() as sess:
-      # Build a function with a single Const node, whose output is ignored.
-      fdef = function_pb2.FunctionDef()
-      fdef.signature.name = "KernelWithNoOutputs"
-      node = node_def_pb2.NodeDef()
-      node.op = "Const"
-      node.name = "ignored"
-      node.attr["dtype"].type = dtypes.int32.as_datatype_enum
-      tensor = tensor_util.make_tensor_proto([0], dtype=dtypes.int32, shape=[])
-      node.attr["value"].tensor.CopyFrom(tensor)
-      fdef.node_def.extend([node])
 
       # Check that calling the result as a compiled kernel doesn't crash.
       @function.Defun(compiled=True)
       def KernelWithNoOutputs():
-        return constant_op.constant(100)
-
-      # Hack to override the definition.  By accessing .definition, we
-      # force the _DefinedFunction initialized internally. Then, we
-      # replace it's internal FunctionDef proto. We do this hack here
-      # because one typically can't construct KernelWithNoOutputs
-      # function via Defun decorator directly.
-      _ = KernelWithNoOutputs.definition
-      foo = KernelWithNoOutputs
-      foo._definition = fdef
-      call = KernelWithNoOutputs()
+        a = constant_op.constant(100)  # pylint: disable=unused-variable
+
+      call = KernelWithNoOutputs()  # pylint: disable=assignment-from-no-return
       sess.run(call, {})
 
   def testAliasing(self):
diff --git a/tensorflow/python/framework/c_api_util.py b/tensorflow/python/framework/c_api_util.py
index 379ba19def..ddababd5b8 100644
--- a/tensorflow/python/framework/c_api_util.py
+++ b/tensorflow/python/framework/c_api_util.py
@@ -20,6 +20,7 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.python import pywrap_tensorflow as c_api
+from tensorflow.python.util import tf_contextlib
 
 
 class ScopedTFStatus(object):
@@ -46,3 +47,25 @@ class ScopedTFGraph(object):
     # terminating) we can have already deleted other modules.
     if c_api.TF_DeleteGraph is not None:
       c_api.TF_DeleteGraph(self.graph)
+
+
+@tf_contextlib.contextmanager
+def tf_buffer():
+  """Context manager that creates and deletes TF_Buffer.
+
+  Example usage:
+    wtih tf_buffer() as buf:
+      # get serialized graph def into buf
+      ...
+      proto_data = c_api.TF_GetBuffer(buf)
+      graph_def.ParseFromString(compat.as_bytes(proto_data))
+    # buf has been deleted
+
+  Yields:
+    Created TF_Buffer
+  """
+  buf = c_api.TF_NewBuffer()
+  try:
+    yield buf
+  finally:
+    c_api.TF_DeleteBuffer(buf)
diff --git a/tensorflow/python/framework/function.py b/tensorflow/python/framework/function.py
index 068e3125aa..7068e72009 100644
--- a/tensorflow/python/framework/function.py
+++ b/tensorflow/python/framework/function.py
@@ -25,8 +25,10 @@ import collections
 import hashlib
 
 from tensorflow.core.framework import attr_value_pb2
+from tensorflow.core.framework import function_pb2
 from tensorflow.python import pywrap_tensorflow as c_api
 from tensorflow.python.eager import context
+from tensorflow.python.framework import c_api_util
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import graph_to_function_def
@@ -242,9 +244,17 @@ class _DefinedFunction(object):
     self._shape_func = shape_func
     self._capture_by_value = capture_by_value
     self._extra_kwargs = kwargs
-    self._definition = None  # Constructed lazily.
-    self._c_func = None  # Constructed with definition.
-    self._sub_functions = dict()  # Constructed with definition.
+    # Constructed only when C API is disabled, lazily
+    self._definition = None
+    # Constructed only when C API is enabled, lazily
+    self._c_func = None
+    self._sub_functions = dict()  # Constructed with _definition or _c_func
+
+    # Cached OpDef for this function. When C API is enabled, this is
+    # the only part of FunctionDef that we cache in Python. When C API
+    # is disabled the whole _definition is available and this is simply
+    # another reference to _definition.signature
+    self._op_def = None
 
     self._args = []
     assert isinstance(input_types, (list, tuple))
@@ -263,8 +273,21 @@ class _DefinedFunction(object):
   def definition(self):
     """Function definition proto."""
     self._create_definition_if_needed()
+    if self._c_func:
+      with c_api_util.tf_buffer() as buf:
+        with errors.raise_exception_on_not_ok_status() as status:
+          c_api.TF_FunctionToFunctionDef(self._c_func, buf, status)
+        fdef = function_pb2.FunctionDef()
+        proto_data = c_api.TF_GetBuffer(buf)
+        fdef.ParseFromString(compat.as_bytes(proto_data))
+      return fdef
     return self._definition
 
+  @property
+  def _signature(self):
+    self._create_definition_if_needed()
+    return self._op_def
+
   def set_grad_func(self, grad_func):
     """Specifies the gradient function of this function."""
     assert not self._grad_func
@@ -299,7 +322,7 @@ class _DefinedFunction(object):
 
   def _create_definition_if_needed_impl(self):
     """This is not what you want, see _create_definition_if_needed."""
-    if self._definition is not None:
+    if self._definition is not None or self._c_func is not None:
       return
 
     # Create the func_def object.
@@ -313,11 +336,23 @@ class _DefinedFunction(object):
       # Call func and gather the output tensors.
       with vs.variable_scope("", custom_getter=temp_graph.getvar):
         outputs = self._func(*inputs)
-      # If func only returned one value, make it a tuple.
-      if not isinstance(outputs, (list, tuple)):
-        outputs = (outputs,)
-      if any([_ is None for _ in outputs]):
-        raise ValueError("Function can not return None.")
+
+      # There is no way of distinguishing between a function not returning
+      # anything and a function returning None in Python.
+      # We need to allow the former and ideally want to forbid the latter as
+      # it is most likely user error.
+      # TODO(iga): Consider adding a @NoOutput decorator on top of @Defun to
+      # allow users to explicitly mark the function as not returning anything.
+      # For now, we allow a single None return and interpret it as a function
+      # with no output.
+      if outputs is None:
+        outputs = []
+      else:
+        # If func only returned one value, make it a tuple.
+        if not isinstance(outputs, (list, tuple)):
+          outputs = (outputs,)
+        if any([_ is None for _ in outputs]):
+          raise ValueError("Function can not return None.")
       # Ensures each output is a Tensor.
       outputs = [ops.convert_to_tensor(_) for _ in outputs]
     self._extra_inputs = temp_graph.extra_inputs
@@ -326,44 +361,47 @@ class _DefinedFunction(object):
     self._sub_functions = temp_graph._functions
     # pylint: enable=protected-access
 
-    # Build the FunctionDef
-    self._definition = graph_to_function_def.graph_to_function_def(
-        temp_graph,
-        temp_graph.get_operations(),
-        inputs,
-        outputs,
-        out_names=self._out_names)
-
     # Extra kwargs are treated as attrs on the function def.
-    sig_pre_func_name = self._func_name or _get_func_name(self._func)
-    kwargs_attr = _parse_kwargs_as_attrs(sig_pre_func_name,
+    base_func_name = self._func_name or _get_func_name(self._func)
+    kwargs_attr = _parse_kwargs_as_attrs(base_func_name,
                                          **self._extra_kwargs)
-    for k in kwargs_attr:
-      self._definition.attr[k].CopyFrom(kwargs_attr[k])
-
-    # Hash the definition and its dependencies.
-    self._hash_str = self._create_hash_str(
-        self._definition.signature.input_arg,
-        self._definition.signature.output_arg, self._definition.node_def)
-
-    # Finally, we decide the function name to use.  If not specified,
-    # make up something which is almost certainly unique (but deterministic).
-    if not self._func_name:
-      self._func_name = "_".join([_get_func_name(self._func), self._hash_str])
-    self._definition.signature.name = self._func_name
-    if self._func.__doc__:
-      self._definition.signature.description = self._func.__doc__
 
-    # pylint: disable=protected-access
-    if temp_graph._c_graph:
+    if not temp_graph._c_graph:  # pylint: disable=protected-access
+      # Build the FunctionDef
+      self._definition = graph_to_function_def.graph_to_function_def(
+          temp_graph,
+          temp_graph.get_operations(),
+          inputs,
+          outputs,
+          out_names=self._out_names)
+
+      for k in kwargs_attr:
+        self._definition.attr[k].CopyFrom(kwargs_attr[k])
+
+      # Hash the definition and its dependencies.
+      self._hash_str = self._create_hash_str(
+          self._definition.signature.input_arg,
+          self._definition.signature.output_arg, self._definition.node_def)
+
+      # Finally, we decide the function name to use.  If not specified,
+      # make up something which is almost certainly unique (but deterministic).
+      if not self._func_name:
+        self._func_name = "_".join([base_func_name, self._hash_str])
+      self._definition.signature.name = self._func_name
+      if self._func.__doc__:
+        self._definition.signature.description = self._func.__doc__
+
+      self._op_def = self._definition.signature
+    else:  # C API is enabled
       output_names = ([compat.as_bytes(x) for x in self._out_names]
                       if self._out_names else [])
       description = self._func.__doc__ or None
+      # pylint: disable=protected-access
       with errors.raise_exception_on_not_ok_status() as status:
         self._c_func = c_api.TF_GraphToFunction_wrapper(
             temp_graph._c_graph,
-            self._func_name,
-            False,  # append_hash_to_fn_name
+            base_func_name,
+            self._func_name is None,  # append_hash_to_fn_name
             None,  # opers
             [t._as_tf_output() for t in inputs],
             [t._as_tf_output() for t in outputs],
@@ -371,8 +409,15 @@ class _DefinedFunction(object):
             None,  # opts
             description,
             status)
+      # pylint: enable=protected-access
       self._set_c_attrs(kwargs_attr)
-    # pylint: enable=protected-access
+
+      # Set cached fields: _op_def and _func_name (if not already set)
+      self._op_def = self.definition.signature
+      if self._func_name:
+        assert self._func_name == self._op_def.name
+      else:
+        self._func_name = self._op_def.name
 
   def _set_c_attrs(self, attrs):
     """Sets `attrs` as attributes of self._c_func.
@@ -440,13 +485,8 @@ class _DefinedFunction(object):
     """Adds this function into the graph g."""
     self._create_definition_if_needed()
 
-    # pylint: disable=protected-access
-    # If 'g' has an identical function already, do nothing.
-    prev = g._get_function(self.name)
-    if prev and (prev._hash_str == self._hash_str):
-      return
-
     # Adds this function into 'g'.
+    # pylint: disable=protected-access
     if context.in_graph_mode():
       g._add_function(self)
     else:
@@ -464,7 +504,7 @@ class _DefinedFunction(object):
   def __call__(self, *args, **kwargs):
     self.add_to_graph(ops.get_default_graph())
     args = [ops.convert_to_tensor(_) for _ in args] + self._extra_inputs
-    ret, op = _call(self._definition.signature, *args, **kwargs)
+    ret, op = _call(self._signature, *args, **kwargs)
     if self._shape_func is not None:
       shapes = self._shape_func(op)
       if len(shapes) != len(op.outputs):
@@ -554,7 +594,7 @@ class _OverloadedFunction(object):
         # right input types.
         output_types = [
             dtypes.DType(_.type)
-            for _ in defined.definition.signature.output_arg
+            for _ in defined._signature.output_arg  # pylint: disable=protected-access
         ]
         # pylint: disable=protected-access
         defined._grad_func = self._grad_func.instantiate(
@@ -759,6 +799,9 @@ def _from_definition(fdef, grad_func=None):
   Returns:
     A _DefinedFunction representing fdef
   """
+  # TODO(iga): This method does major surgery on _DefinedFunction.
+  # Make it a named constructor using @classmethod of _DefinedFunction.
+
   # The Python callable is only needed to create a FunctionDef. Since we have
   # the FunctionDef here, we don't need to set _DefinedFunction._func (nor do we
   # have access to such a callable here).
@@ -774,15 +817,22 @@ def _from_definition(fdef, grad_func=None):
   result = _DefinedFunction(func, argnames, input_types, func_name, grad_func,
                             python_grad_func, out_names)
   # pylint: disable=protected-access
-  result._definition = fdef
-  # Captured inputs are added as regular inputs to a function when it's
-  # serialized, i.e. any extra inputs from the original function are now
-  # included in `result`._args
-  result._extra_inputs = []
-  result._hash_str = result._create_hash_str(
-      result._definition.signature.input_arg,
-      result._definition.signature.output_arg, result._definition.node_def)
+  if ops._USE_C_API:
+    serialized = fdef.SerializeToString()
+    with errors.raise_exception_on_not_ok_status() as status:
+      result._c_func = c_api.TF_FunctionImportFunctionDef(serialized, status)
+    result._extra_inputs = []
+  else:
+    result._definition = fdef
+    # Captured inputs are added as regular inputs to a function when it's
+    # serialized, i.e. any extra inputs from the original function are now
+    # included in `result`._args
+    result._extra_inputs = []
+    result._hash_str = result._create_hash_str(
+        result._definition.signature.input_arg,
+        result._definition.signature.output_arg, result._definition.node_def)
   # pylint: enable=protected-access
+
   return result
 
 
diff --git a/tensorflow/python/framework/function_test.py b/tensorflow/python/framework/function_test.py
index 3c359b8700..fea2129922 100644
--- a/tensorflow/python/framework/function_test.py
+++ b/tensorflow/python/framework/function_test.py
@@ -107,8 +107,9 @@ class FunctionTest(test.TestCase):
 
     with ops.Graph().as_default():
       with self.assertRaisesRegexp(
-          ValueError, (r"Length of out_names \(2\) does not match number of "
-                       r"outputs \(1\): my_result1, my_result2")):
+          errors_impl.InvalidArgumentError,
+          (r"output names must be either empty or equal in size to outputs. "
+           "output names size = 2 outputs size = 1")):
         MyIdentityFunc([18.0])
 
   def testDefineFunction2Args(self):
@@ -123,18 +124,16 @@ class FunctionTest(test.TestCase):
       with session.Session() as sess:
         self.assertAllEqual([5.0], sess.run(call))
 
-  def testValueErrorOnFunctionWithNoOutput(self):
-    # TODO(iga): Remove this restriction and this test
+  def testFunctionWithNoOutput(self):
 
     @function.Defun(dtypes.float32, dtypes.float32)
     def APlus2B(a, b):
-      print(a + b * 2)  # Create some ops to have nodes in the body
-      # Using 'print' to make lint happy
+      c = a + b * 2  # Create some ops to have nodes in the body
+      print(c)  # Using 'print' to make lint happy
 
     with ops.Graph().as_default():
-      with self.assertRaisesRegexp(ValueError,
-                                   "Function can not return None"):
-        APlus2B([1.0], [2.0])
+      # Call function. There should be no exceptions.
+      APlus2B([1.0], [2.0])
 
   def testDefineFunction2ArgsOutputName(self):
 
@@ -499,14 +498,6 @@ class FunctionTest(test.TestCase):
 
   def testDefineErrors(self):
     with ops.Graph().as_default():
-      with self.assertRaisesRegexp(ValueError, "can not return None"):
-
-        @function.Defun()
-        def NoResult():
-          pass
-
-        _ = NoResult.definition
-
       with self.assertRaisesRegexp(ValueError, "can not return None"):
 
         @function.Defun()
@@ -730,7 +721,14 @@ class FunctionTest(test.TestCase):
     def Foo(x, y, z):
       return math_ops.tanh(math_ops.matmul(x, y) + z)
 
-    self.assertEqual("Foo_d643acf7", Foo.instantiate([dtypes.float32] * 3).name)
+    # We added more randomness to function names in C API.
+    # TODO(iga): Remove this if statement when we switch to C API.
+    if ops._USE_C_API:  # pylint: disable=protected-access
+      self.assertEqual("Foo_aCYSbwBkR5A",
+                       Foo.instantiate([dtypes.float32] * 3).name)
+    else:
+      self.assertEqual("Foo_d643acf7",
+                       Foo.instantiate([dtypes.float32] * 3).name)
 
   def testSignatureHash(self):
     # Foo.Inner and Bar.Inner have identical function body but have
@@ -1007,7 +1005,8 @@ class FunctionsFromProtos(test.TestCase):
     library.function.extend([F1.definition])
 
     with self.assertRaisesRegexp(
-        ValueError, "FunctionDefLibrary missing 'G1_........' FunctionDef"):
+        ValueError,
+        "FunctionDefLibrary missing 'G1_[0-9a-zA-Z]{8,11}' FunctionDef"):
       function._from_library(library)
 
     # Create invalid function def that is missing F1 function def
@@ -1016,7 +1015,8 @@ class FunctionsFromProtos(test.TestCase):
     library.function.extend([G1.definition])
 
     with self.assertRaisesRegexp(
-        ValueError, "FunctionDefLibrary missing 'F1_........' FunctionDef"):
+        ValueError,
+        "FunctionDefLibrary missing 'F1_[0-9a-zA-Z]{8,11}' FunctionDef"):
       function._from_library(library)
 
   def testFromLibraryCyclicGradFuncs(self):
diff --git a/tensorflow/python/framework/graph_to_function_def.py b/tensorflow/python/framework/graph_to_function_def.py
index 33a417a1da..448f87aa6e 100644
--- a/tensorflow/python/framework/graph_to_function_def.py
+++ b/tensorflow/python/framework/graph_to_function_def.py
@@ -22,6 +22,7 @@ import re
 
 from tensorflow.core.framework import function_pb2
 from tensorflow.core.framework import op_def_pb2
+from tensorflow.python.framework import errors_impl
 from tensorflow.python.framework import op_def_registry
 
 
@@ -151,9 +152,11 @@ def graph_to_function_def(graph, operations, inputs, outputs, out_names=None):
     func.signature.output_arg.extend(
         [_tensor_to_argdef(o, used_names=used_names) for o in outputs])
   elif len(outputs) != len(out_names):
-    raise ValueError(
-        "Length of out_names (%d) does not match number of outputs (%d): %s" %
-        (len(out_names), len(outputs), ", ".join(out_names)))
+    raise errors_impl.InvalidArgumentError(
+        None, None,
+        "output names must be either empty or equal in size to outputs. "
+        "output names size = %d outputs size = %d" %
+        (len(out_names), len(outputs)))
   elif len(out_names) != len(set(out_names)):
     raise ValueError(
         "Must not have duplicates in out_names: %s" % ", ".join(out_names))
diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index ae84297690..e6e6b9c6ca 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -2808,19 +2808,14 @@ class Graph(object):
       ValueError: if another function is defined with the same name.
     """
     name = function.name
-    previous = self._functions.get(name, None)
-    if previous:
-      raise ValueError("Another function is already defined with that name")
     # Sanity checks on gradient definition.
     if (function.grad_func_name is not None) and (function.python_grad_func is
                                                   not None):
       raise ValueError("Gradient defined twice for function %s" % name)
-    # Need a new-enough consumer to support the functions we add to the graph.
-    if self._graph_def_versions.min_consumer < 12:
-      self._graph_def_versions.min_consumer = 12
-    self._functions[name] = function
+
+    # Add function to graph
+    # pylint: disable=protected-access
     if self._c_graph:
-      # pylint: disable=protected-access
       assert function._c_func, (
           "Cannot add function created without C API support to graph "
           "created with C API support")
@@ -2828,7 +2823,26 @@ class Graph(object):
         gradient = function._grad_func._c_func if function._grad_func else None
         c_api.TF_GraphCopyFunction(self._c_graph, function._c_func, gradient,
                                    status)
-      # pylint: enable=protected-access
+    else:
+      # If there is already a function with the same name, raise an error
+      # if bodies are different. Else, do nothing. The C API version above
+      # has the same behavior.
+      previous = self._functions.get(name, None)
+      if previous:
+        # This check is not ideal as we can have a hash collision with only
+        # 32 bits in the hash, but the non C API mode is being deprecated.
+        # Don't bother changing it now.
+        if previous._hash_str == function._hash_str:
+          return
+        else:
+          raise ValueError("Another function is already defined with that name")
+    # pylint: enable=protected-access
+
+    self._functions[name] = function
+
+    # Need a new-enough consumer to support the functions we add to the graph.
+    if self._graph_def_versions.min_consumer < 12:
+      self._graph_def_versions.min_consumer = 12
 
   @property
   def building_function(self):
-- 
GitLab


From 83b5768431bb06d749cf67ab64d9cd3fd36ec943 Mon Sep 17 00:00:00 2001
From: Fan Xia <FrankXia0404@users.noreply.github.com>
Date: Thu, 5 Oct 2017 14:22:01 -0700
Subject: [PATCH 079/909] Make code Python 2 and 3 compatible (#13489)

Update the Python implementation so that both Python 2 and Python 3 environment can execute
---
 tensorflow/docs_src/get_started/estimator.md | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/tensorflow/docs_src/get_started/estimator.md b/tensorflow/docs_src/get_started/estimator.md
index 4f3a438d17..11c3dc6e53 100644
--- a/tensorflow/docs_src/get_started/estimator.md
+++ b/tensorflow/docs_src/get_started/estimator.md
@@ -28,7 +28,7 @@ from __future__ import division
 from __future__ import print_function
 
 import os
-import urllib
+from six.moves.urllib.request import urlopen
 
 import numpy as np
 import tensorflow as tf
@@ -44,13 +44,13 @@ IRIS_TEST_URL = "http://download.tensorflow.org/data/iris_test.csv"
 def main():
   # If the training and test sets aren't stored locally, download them.
   if not os.path.exists(IRIS_TRAINING):
-    raw = urllib.urlopen(IRIS_TRAINING_URL).read()
-    with open(IRIS_TRAINING, "w") as f:
+    raw = urlopen(IRIS_TRAINING_URL).read()
+    with open(IRIS_TRAINING, "wb") as f:
       f.write(raw)
 
   if not os.path.exists(IRIS_TEST):
-    raw = urllib.urlopen(IRIS_TEST_URL).read()
-    with open(IRIS_TEST, "w") as f:
+    raw = urlopen(IRIS_TEST_URL).read()
+    with open(IRIS_TEST, "wb") as f:
       f.write(raw)
 
   # Load datasets.
@@ -167,7 +167,7 @@ from __future__ import division
 from __future__ import print_function
 
 import os
-import urllib
+from six.moves.urllib.request import urlopen
 
 import tensorflow as tf
 import numpy as np
@@ -184,13 +184,13 @@ them.
 
 ```python
 if not os.path.exists(IRIS_TRAINING):
-  raw = urllib.urlopen(IRIS_TRAINING_URL).read()
-  with open(IRIS_TRAINING,'w') as f:
+  raw = urlopen(IRIS_TRAINING_URL).read()
+  with open(IRIS_TRAINING,'wb') as f:
     f.write(raw)
 
 if not os.path.exists(IRIS_TEST):
-  raw = urllib.urlopen(IRIS_TEST_URL).read()
-  with open(IRIS_TEST,'w') as f:
+  raw = urlopen(IRIS_TEST_URL).read()
+  with open(IRIS_TEST,'wb') as f:
     f.write(raw)
 ```
 
-- 
GitLab


From 91df2c942ebf4bd048edba055418467cae510431 Mon Sep 17 00:00:00 2001
From: Fred Reiss <frreiss@us.ibm.com>
Date: Thu, 5 Oct 2017 14:22:26 -0700
Subject: [PATCH 080/909] Give accumulate_n op a gradient (version 2) (#13325)

* Changed accumulate_n ==> accumulate_n_v2 and moved to contrib

* Moving source files to contrib.

* Better startup message.

* Fixing up build

* Removal of temporary code.

* Reduce logging output

* Fixing build issues.

* CI sanity fixes.

* Cleanup prior to PR

* Cleanup

* Cleanup.

* Cleanup.

* Cleanup.

* Moved AccumulateNV2 to main build and added fallback to AddN for eager mode

* Fixing CI issues
---
 tensorflow/contrib/framework/BUILD            |  29 ++-
 .../framework/python/ops/accumulate_n_v2.py   | 111 ++++++++++
 .../python/ops/accumulate_n_v2_eager_test.py  |  84 ++++++++
 .../python/ops/accumulate_n_v2_test.py        | 123 +++++++++++
 tensorflow/core/BUILD                         |   1 +
 .../common_runtime/accumulate_n_optimizer.cc  | 191 ++++++++++++++++++
 tensorflow/core/ops/math_ops.cc               |  32 +++
 tensorflow/python/ops/hidden_ops.txt          |   2 +
 8 files changed, 571 insertions(+), 2 deletions(-)
 create mode 100644 tensorflow/contrib/framework/python/ops/accumulate_n_v2.py
 create mode 100644 tensorflow/contrib/framework/python/ops/accumulate_n_v2_eager_test.py
 create mode 100644 tensorflow/contrib/framework/python/ops/accumulate_n_v2_test.py
 create mode 100644 tensorflow/core/common_runtime/accumulate_n_optimizer.cc

diff --git a/tensorflow/contrib/framework/BUILD b/tensorflow/contrib/framework/BUILD
index 6b0599ddd2..dd882acb8e 100644
--- a/tensorflow/contrib/framework/BUILD
+++ b/tensorflow/contrib/framework/BUILD
@@ -10,9 +10,8 @@ package(default_visibility = [
     "//tensorflow:__subpackages__",
 ])
 
-load("//tensorflow:tensorflow.bzl", "cuda_py_test")
-load("//tensorflow:tensorflow.bzl", "tf_custom_op_py_library")
 load("//tensorflow:tensorflow.bzl", "py_test")
+load("//tensorflow:tensorflow.bzl", "tf_custom_op_py_library")
 load("//tensorflow:tensorflow.bzl", "tf_custom_op_library")
 load("//tensorflow:tensorflow.bzl", "tf_gen_op_wrapper_py")
 load("//tensorflow:tensorflow.bzl", "tf_gen_op_libs")
@@ -27,6 +26,7 @@ tf_custom_op_py_library(
         "python/framework/experimental.py",
         "python/framework/tensor_util.py",
         "python/ops/__init__.py",
+        "python/ops/accumulate_n_v2.py",
         "python/ops/arg_scope.py",
         "python/ops/audio_ops.py",
         "python/ops/checkpoint_ops.py",
@@ -149,6 +149,31 @@ py_test(
     ],
 )
 
+py_test(
+    name = "accumulate_n_v2_test",
+    size = "small",
+    srcs = ["python/ops/accumulate_n_v2_test.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":framework_py",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework_for_generated_wrappers",
+    ],
+)
+
+py_test(
+    name = "accumulate_n_v2_eager_test",
+    size = "small",
+    srcs = ["python/ops/accumulate_n_v2_eager_test.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":framework_py",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework_for_generated_wrappers",
+        "//tensorflow/python/eager:backprop",
+    ],
+)
+
 py_test(
     name = "ops_test",
     size = "small",
diff --git a/tensorflow/contrib/framework/python/ops/accumulate_n_v2.py b/tensorflow/contrib/framework/python/ops/accumulate_n_v2.py
new file mode 100644
index 0000000000..a0667bd489
--- /dev/null
+++ b/tensorflow/contrib/framework/python/ops/accumulate_n_v2.py
@@ -0,0 +1,111 @@
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Ops that will eventually be folded into tensorflow/python/ops/math_ops.py
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+
+from tensorflow.python.eager import context
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import tensor_shape
+from tensorflow.python.ops import gen_math_ops
+from tensorflow.python.ops import math_ops
+
+
+
+def accumulate_n_v2(inputs, shape=None, tensor_dtype=None, name=None):
+  """Returns the element-wise sum of a list of tensors.
+
+  Optionally, pass `shape` and `tensor_dtype` for shape and type checking,
+  otherwise, these are inferred.
+
+  `tf.accumulate_n_v2` performs the same operation as `tf.add_n`, but does not
+  wait for all of its inputs to be ready before beginning to sum. This can
+  save memory if inputs are ready at different times, since minimum temporary
+  storage is proportional to the output size rather than the inputs size.
+
+  Unlike the original `accumulate_n`, `accumulate_n_v2` is differentiable.
+
+  For example:
+
+  ```python
+  a = tf.constant([[1, 2], [3, 4]])
+  b = tf.constant([[5, 0], [0, 6]])
+  tf.accumulate_n_v2([a, b, a])  # [[7, 4], [6, 14]]
+
+  # Explicitly pass shape and type
+  tf.accumulate_n_v2([a, b, a], shape=[2, 2], tensor_dtype=tf.int32)  
+                                                                   # [[7,  4],
+                                                                   #  [6, 14]]
+  ```
+
+  Args:
+    inputs: A list of `Tensor` objects, each with same shape and type.
+    shape: Shape of elements of `inputs`.
+    tensor_dtype: The type of `inputs`.
+    name: A name for the operation (optional).
+
+  Returns:
+    A `Tensor` of same shape and type as the elements of `inputs`.
+
+  Raises:
+    ValueError: If `inputs` don't all have same shape and dtype or the shape
+    cannot be inferred.
+  """
+  _INPUTS_ERR_MSG = ValueError("inputs must be a list of at least one Tensor"
+                               "with the same dtype and shape")
+  if not inputs or not isinstance(inputs, (list, tuple)):
+    raise _INPUTS_ERR_MSG
+  inputs = ops.convert_n_to_tensor_or_indexed_slices(inputs)
+  if not all(isinstance(x, ops.Tensor) for x in inputs):
+    raise _INPUTS_ERR_MSG
+  if not all(x.dtype == inputs[0].dtype for x in inputs):
+    raise _INPUTS_ERR_MSG
+  if shape is not None:
+    shape = tensor_shape.as_shape(shape)
+  else:
+    shape = tensor_shape.unknown_shape()
+  for input_tensor in inputs:
+    if isinstance(input_tensor, ops.Tensor):
+      shape = shape.merge_with(input_tensor.get_shape())
+
+  # tensor_dtype is for safety only; operator's output type computed in C++
+  if tensor_dtype is not None and tensor_dtype != inputs[0].dtype:
+    raise TypeError("tensor_dtype is {}, but input is of type {}"
+                    .format(tensor_dtype, inputs[0].dtype))
+
+  if len(inputs) == 1 and name is None:
+    return inputs[0]
+  elif len(inputs) == 1 and name is not None:
+    return array_ops.identity(inputs[0], name=name)
+  elif context.in_eager_mode():
+    # TemporaryVariable not currently supported in eager mode; fall back 
+    # onto AddN for now.
+    # TODO(frreiss) remove this once the lifetime of eager variables gets
+    # addressed
+    return math_ops.add_n(inputs, name=name)
+  else:
+    return gen_math_ops._accumulate_nv2(inputs, name=name, shape=shape)
+
+# The following code should eventually be merged into 
+# tensorflow/python/ops/math_grad.py
+@ops.RegisterGradient("AccumulateNV2")
+def _AddNGrad(op, grad):
+  """Same as gradient for AddN. Copies the gradient to all inputs."""
+  # Not broadcasting.
+  return [grad] * len(op.inputs)
+
diff --git a/tensorflow/contrib/framework/python/ops/accumulate_n_v2_eager_test.py b/tensorflow/contrib/framework/python/ops/accumulate_n_v2_eager_test.py
new file mode 100644
index 0000000000..8c618838bf
--- /dev/null
+++ b/tensorflow/contrib/framework/python/ops/accumulate_n_v2_eager_test.py
@@ -0,0 +1,84 @@
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for new version of accumulate_n op that will eventually go into 
+`ops.math_ops`.
+
+These test cases spefically exercise the `eager` APIs. They need to be in a 
+separate file from the remaining tests because eager mode is currently something
+you can turn on but can't turn off for the lifetime of the current process."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.contrib.framework.python.ops import accumulate_n_v2 as av2
+
+from tensorflow.python.eager import backprop
+from tensorflow.python.eager import context as eager_context
+from tensorflow.python.eager import tape
+
+
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes as dtypes_lib
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
+from tensorflow.python.ops import gradients
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import resource_variable_ops
+from tensorflow.python.platform import test
+
+
+
+class AccumulateNV2EagerTest(test_util.TensorFlowTestCase):
+  """Tests of the new, differentiable version of accumulate_n"""
+
+  def testMinimalEagerMode(self):
+    forty = constant_op.constant(40)
+    two = constant_op.constant(2)
+    answer = av2.accumulate_n_v2([forty, two])
+    self.assertEqual(42, answer.numpy())
+
+
+  def testFloat(self):
+    np.random.seed(12345)
+    x = [np.random.random((1, 2, 3, 4, 5)) - 0.5 for _ in range(5)]
+    tf_x = ops.convert_n_to_tensor(x)
+    with self.test_session(use_gpu=True):
+      self.assertAllClose(sum(x), av2.accumulate_n_v2(tf_x).numpy())
+      self.assertAllClose(x[0] * 5, av2.accumulate_n_v2([tf_x[0]] * 5).numpy())
+
+  def testGrad(self):
+    np.random.seed(42)
+    num_inputs = 3
+    input_vars = [
+        resource_variable_ops.ResourceVariable(10.0 * np.random.random())
+        for i in range(0, num_inputs)
+    ]
+
+    def fn(first, second, third):
+      return av2.accumulate_n_v2([first, second, third])
+
+    grad_fn = backprop.gradients_function(fn)      
+    grad = grad_fn(input_vars[0], input_vars[1], input_vars[2])
+    self.assertAllEqual(np.repeat(1.0, num_inputs), # d/dx (x + y + ...) = 1
+                        [elem.numpy() for elem in grad])
+
+
+
+if __name__ == "__main__":
+  eager_context.enable_eager_execution()
+  test.main()
+
diff --git a/tensorflow/contrib/framework/python/ops/accumulate_n_v2_test.py b/tensorflow/contrib/framework/python/ops/accumulate_n_v2_test.py
new file mode 100644
index 0000000000..3386e849d5
--- /dev/null
+++ b/tensorflow/contrib/framework/python/ops/accumulate_n_v2_test.py
@@ -0,0 +1,123 @@
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for new version of accumulate_n op that will eventually go into 
+`ops.math_ops`."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.contrib.framework.python.ops import accumulate_n_v2 as av2
+
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes as dtypes_lib
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
+from tensorflow.python.ops import gradients
+from tensorflow.python.ops import variables
+from tensorflow.python.platform import googletest
+
+
+
+class AccumulateNV2Test(test_util.TensorFlowTestCase):
+  """Tests of the new, differentiable version of accumulate_n"""
+
+  def testFloat(self):
+    np.random.seed(12345)
+    x = [np.random.random((1, 2, 3, 4, 5)) - 0.5 for _ in range(5)]
+    tf_x = ops.convert_n_to_tensor(x)
+    with self.test_session(use_gpu=True):
+      self.assertAllClose(sum(x), av2.accumulate_n_v2(tf_x).eval())
+      self.assertAllClose(x[0] * 5, av2.accumulate_n_v2([tf_x[0]] * 5).eval())
+
+  def testInt(self):
+    np.random.seed(54321)
+    x = [np.random.randint(-128, 128, (5, 4, 3, 2, 1)) for _ in range(6)]
+    tf_x = ops.convert_n_to_tensor(x)
+    with self.test_session(use_gpu=True):
+      self.assertAllEqual(sum(x), av2.accumulate_n_v2(tf_x).eval())
+      self.assertAllEqual(x[0] * 6, av2.accumulate_n_v2([tf_x[0]] * 6).eval())
+
+  def testGrad(self):
+    np.random.seed(42)
+    for num_inputs in range(1, 10):
+      with self.test_session(use_gpu=True) as sess:
+        input_vars = [
+            variables.Variable(10.0 * np.random.random())
+            for i in range(0, num_inputs)
+        ]
+        accum_n = av2.accumulate_n_v2(input_vars)
+        sess.run(variables.global_variables_initializer())
+        accum_n_grad = gradients.gradients(accum_n, input_vars)
+        self.assertAllEqual(np.repeat(1.0, num_inputs), # d/dx (x + y + ...) = 1
+                            [g.eval() for g in accum_n_grad])
+
+  # The tests below used to be in a separate class under cwise_ops_test.py,
+  # which did not run in the default test target.
+  # Putting them here so that everything that exercises AccumulateNV2 is in
+  # one place and the default build runs all unit tests.
+  def testSimple(self):
+    with self.test_session():
+      random_arrays = [
+          np.random.rand(16, 16, 16, 16).astype(np.float32) for _ in range(20)
+      ]
+      random_tensors = [
+          ops.convert_to_tensor(
+              x, dtype=dtypes_lib.float32) for x in random_arrays
+      ]
+      tf_val = av2.accumulate_n_v2(random_tensors)
+      np_val = random_arrays[0]
+      for random_array in random_arrays[1:]:
+        np_val += random_array
+      self.assertAllClose(np_val, tf_val.eval())
+
+  def testZeroArgs(self):
+    with self.test_session():
+      with self.assertRaises(ValueError):
+        tf_val = av2.accumulate_n_v2([])
+        tf_val.eval()
+
+  def testWrongShape(self):
+    with self.test_session():
+      with self.assertRaises(ValueError):
+        a = variables.Variable(0.2)
+        b = variables.Variable(0.1)
+        tf_val = av2.accumulate_n_v2([a,b], shape=[2,2]) # Should be shape=[]
+
+  def testIncompatibleShapes(self):
+    with self.test_session():
+      with self.assertRaises(ValueError):
+        a = variables.Variable(np.array([0.1,0.2]))
+        b = variables.Variable(np.array([[0.3],[0.4]]))
+        tf_val = av2.accumulate_n_v2([a,b]) 
+
+  def testWrongType(self):
+    with self.test_session():
+      with self.assertRaises(TypeError):
+        a = variables.Variable(0.2, dtype=np.float32)
+        b = variables.Variable(0.1, dtype=np.float32)
+        tf_val = av2.accumulate_n_v2([a,b], tensor_dtype=np.int32) 
+
+  def testWrongTypeOneInput(self):
+    # Scenario that used to trigger a bug, even when testWrongType() worked
+    with self.test_session():
+      with self.assertRaises(TypeError):
+        a = variables.Variable(0.2, dtype=np.float32)
+        tf_val = av2.accumulate_n_v2([a], tensor_dtype=np.int32) 
+
+
+if __name__ == "__main__":
+  googletest.main()
diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index aaede2a6bb..aff132134c 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -1938,6 +1938,7 @@ CORE_CPU_LIB_HEADERS = CORE_CPU_BASE_HDRS + [
 tf_cuda_library(
     name = "core_cpu_impl",
     srcs = [
+        "common_runtime/accumulate_n_optimizer.cc",
         "common_runtime/allocator_retry.cc",
         "common_runtime/bfc_allocator.cc",
         "common_runtime/build_graph_options.cc",
diff --git a/tensorflow/core/common_runtime/accumulate_n_optimizer.cc b/tensorflow/core/common_runtime/accumulate_n_optimizer.cc
new file mode 100644
index 0000000000..81cd44870e
--- /dev/null
+++ b/tensorflow/core/common_runtime/accumulate_n_optimizer.cc
@@ -0,0 +1,191 @@
+/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+
+#include "tensorflow/core/common_runtime/optimization_registry.h"
+#include "tensorflow/core/graph/node_builder.h"
+
+
+namespace tensorflow {
+namespace {
+
+Tensor make_zeros(const DataType& dtype, const TensorShapeProto& shape) {
+  Tensor tensor(dtype, TensorShape(shape));
+
+  // Conveniently, all numeric data types have 0x0 == zero.  Otherwise we would
+  // need a giant switch statement here.
+  memset(const_cast<char*>(tensor.tensor_data().data()), 0,
+         tensor.tensor_data().size());
+
+  return tensor;
+}
+
+// Replaces occurrences of the "AccumulateNV2" stub operator with a graph of
+// lower-level ops. The graph is equivalent (modulo certain corner cases)
+// to the semantics of the original accumulate_n() Python op in math_ops.py.
+// Implementing the op with a rewrite allows this new variant of accumulate_n 
+// to be differentiable.
+//
+// The binary code that generates AccumulateNV2 stub ops is located in a
+// dynamic library built out of tensorflow/contrib/framework. Ideally, this
+// class would also be in contrib, but calls to REGISTER_OPTIMIZATION() from
+// third-party libraries aren't currently supported.
+class AccumulateNV2RemovePass : public GraphOptimizationPass {
+ public:
+
+  Status Run(const GraphOptimizationPassOptions& options) override {
+    // TODO(freiss.oss@gmail.com): Substantial shared code with
+    // ParallelConcatRemovePass::Run(). Consider refactoring if someone makes
+    // a third similar rewrite.
+    if (options.graph == nullptr) {
+      // TODO(apassos) returning OK feels weird here as we can't do anything
+      // without a graph, but some tests require this.
+      return Status::OK();
+    }
+
+    Graph* g = options.graph->get();
+    if (g == nullptr) {
+      return errors::Internal(
+          "AccumulateNV2 removal should happen before partitioning and a "
+          "graph should be available.");
+    }
+
+    // Build up a todo list of ops to replace, *then* modify the graph
+    gtl::InlinedVector<Node*, 2> matches;
+    for (Node* n : g->op_nodes()) {
+      if (n->type_string() == "AccumulateNV2") {
+        matches.push_back(n);
+      }
+    }
+    for (Node* n : matches) {
+      TF_RETURN_IF_ERROR(rewriteNode(n, g));
+    }
+    return Status::OK();
+  }
+
+  Status rewriteNode(Node* n, Graph* g) {
+    AttrSlice n_attrs = n->attrs();
+    auto base_make_node = [n, g, &n_attrs](const string& op,
+                                           const string& name) {
+      NodeBuilder node_builder(name, op);
+
+      // The pieces of AccumulateNV2 should all be on the same node.
+      node_builder.Device(n->requested_device());
+      string colo;
+      if (GetNodeAttr(n_attrs, kColocationAttrName, &colo).ok()) {
+        node_builder.Attr(kColocationAttrName, colo);
+      }
+      return node_builder;
+    };
+    auto make_node = [n, g, &n_attrs, &base_make_node](string op) {
+      return base_make_node(
+          op, g->NewName(strings::StrCat(n->name(), "/Internal")));
+    };
+
+    DataType dtype;
+    TF_RETURN_IF_ERROR(GetNodeAttr(n_attrs, "T", &dtype));
+    TensorShapeProto shape;
+    TF_RETURN_IF_ERROR(GetNodeAttr(n_attrs, "shape", &shape));
+
+    std::vector<const Edge*> data_edges, control_edges;
+    for (const Edge* input_edge : n->in_edges()) {
+      if (input_edge->IsControlEdge()) {
+        control_edges.push_back(input_edge);
+      } else {
+        data_edges.push_back(input_edge);
+      }
+    }
+
+    // Create the following ops to replace the AccumulateNV2 placeholder:
+    Node* create_accumulator = nullptr;            // TemporaryVariable op
+    Node* initial_val = nullptr;                   // Const op
+    Node* initialize_accumulator = nullptr;        // Assign op
+    std::vector<Node*> add_values_to_accumulator;  // AssignAdd ops
+    Node* clean_up_accumulator = nullptr;          // DestroyTemporaryVariable
+
+    const string accumulator_name =
+        strings::StrCat(n->name(), "/Internal/Accumulator");
+    TF_RETURN_IF_ERROR(make_node("TemporaryVariable")
+                           .Attr("shape", shape)
+                           .Attr("dtype", dtype)
+                           .Attr("var_name", accumulator_name)
+                           .Finalize(g, &create_accumulator));
+    TF_RETURN_IF_ERROR(make_node("Const")
+                           .Attr("value", make_zeros(dtype, shape))
+                           .Attr("dtype", dtype)
+                           .Finalize(g, &initial_val));
+    TF_RETURN_IF_ERROR(make_node("Assign")
+                           .Attr("T", dtype)
+                           .Input(create_accumulator)  // ref: Ref(T)
+                           .Input(initial_val)         // value: T
+                           .Finalize(g, &initialize_accumulator));
+    for (int i = 0; i < data_edges.size(); ++i) {
+      Node* assignAdd;
+      TF_RETURN_IF_ERROR(make_node("AssignAdd")
+                             .Attr("T", dtype)
+                             .Attr("use_locking", true)
+                             .Input(initialize_accumulator)  // ref: Ref(T)
+                             .Input(data_edges[i]->src(),
+                                    data_edges[i]->src_output())  // value: T
+                             .Finalize(g, &assignAdd));
+
+      add_values_to_accumulator.push_back(assignAdd);
+    }
+
+    // Note that we use the original placeholder op's name here
+    TF_RETURN_IF_ERROR(base_make_node("DestroyTemporaryVariable", n->name())
+                           .Attr("T", dtype)
+                           .Attr("var_name", accumulator_name)
+                           .Input(initialize_accumulator)
+                           .Finalize(g, &clean_up_accumulator));
+
+    // Add edges to the graph to ensure that operations occur in the right
+    // order:
+    // 1. Do anything that had a control edge to the AccumulateNV2 placeholder
+    // 2. Initialize accumulator
+    // 3. Add input values to accumulator (already handled by data edges
+    //    added above)
+    // 4. Reclaim the buffer that held the accumulator
+    // 5. Do anything that depended on the AccumulateNV2 placeholder
+    for (const Edge* control_edge : control_edges) {
+      g->AddControlEdge(control_edge->src(), initialize_accumulator);
+    }
+
+    for (Node* assign_add : add_values_to_accumulator) {
+      g->AddControlEdge(assign_add, clean_up_accumulator);
+    }
+
+    for (const Edge* out_edge : n->out_edges()) {
+      if (out_edge->IsControlEdge()) {
+        g->AddControlEdge(clean_up_accumulator, out_edge->dst());
+      } else {
+        g->AddEdge(clean_up_accumulator, 0, out_edge->dst(),
+                   out_edge->dst_input());
+      }
+    }
+
+    // Remove the original AccumulateNV2 placeholder op.
+    // This removal modifies the op and must happen after we have finished
+    // using its incoming/outgoing edge sets.
+    g->RemoveNode(n);
+
+    return Status::OK();
+  }
+};
+REGISTER_OPTIMIZATION(OptimizationPassRegistry::PRE_PLACEMENT, 0,
+                      AccumulateNV2RemovePass);
+
+}  // namespace
+}  // namespace tensorflow
diff --git a/tensorflow/core/ops/math_ops.cc b/tensorflow/core/ops/math_ops.cc
index 015fd6e388..967b121a44 100644
--- a/tensorflow/core/ops/math_ops.cc
+++ b/tensorflow/core/ops/math_ops.cc
@@ -49,6 +49,38 @@ inputs: Must all be the same size and shape.
 
 // --------------------------------------------------------------------------
 
+// Note that the following operator is just a placeholder and has no
+// associated kernel. The code in accumulate_n_optimizer.cc replaces
+// this placeholder with a graph of operators that do have kernels.
+// The Python code that generates instances of this op is currently in
+// contrib/framework/python/ops/accumulate_n_v2.py
+REGISTER_OP("AccumulateNV2")
+    .Input("inputs: N * T")
+    .Output("sum: T")
+    .Attr("N: int >= 1")
+    .Attr("T: numbertype")
+    .Attr("shape: shape")
+    .SetIsCommutative()
+    .SetIsAggregate()
+    .SetShapeFn(shape_inference::ExplicitShape)
+    .Doc(R"doc(
+Returns the element-wise sum of a list of tensors.
+
+`tf.accumulate_n_v2` performs the same operation as `tf.add_n`, but does not
+wait for all of its inputs to be ready before beginning to sum. This can
+save memory if inputs are ready at different times, since minimum temporary
+storage is proportional to the output size rather than the inputs size.
+
+Unlike the original `accumulate_n`, `accumulate_n_v2` is differentiable.
+
+Returns a `Tensor` of same shape and type as the elements of `inputs`.
+
+inputs: A list of `Tensor` objects, each with same shape and type.
+shape: Shape of elements of `inputs`.
+)doc");
+
+// --------------------------------------------------------------------------
+
 REGISTER_OP("BatchMatMul")
     .Input("x: T")
     .Input("y: T")
diff --git a/tensorflow/python/ops/hidden_ops.txt b/tensorflow/python/ops/hidden_ops.txt
index d27e867583..a12f750ec1 100644
--- a/tensorflow/python/ops/hidden_ops.txt
+++ b/tensorflow/python/ops/hidden_ops.txt
@@ -42,6 +42,7 @@ UniformCandidateSampler
 GenerateVocabRemapping
 LoadAndRemapMatrix
 
+
 # control_flow_ops
 Switch
 Merge
@@ -240,6 +241,7 @@ TensorSummaryV2
 
 # math_ops
 Abs
+AccumulateNV2
 AddN
 All
 Any
-- 
GitLab


From ccc00be1b1e3ed9bbf1b47fec007ac3f06b8ce7c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Yan=20Facai=20=28=E9=A2=9C=E5=8F=91=E6=89=8D=29?=
 <facai.yan@gmail.com>
Date: Fri, 6 Oct 2017 05:22:44 +0800
Subject: [PATCH 081/909] PREP: migrate ErfGrad to c++ side (#12872)

* ENH: migrate ErfGrad

* TST: add test case for real value

* CLN: add semicolon

* DOC: add comment

* CLN: remove useless dependency

* CLN: remove useless dependency in LgmmaGrad

* TST: move lgamma test case

* TST: add test case for Erf

* TST: complex is unsupported for kernel

* TST: complex64 -> float

* ENH: use grad_scope

* ENH: fix grad_scope for TanhGrad and SigmoidGrad

* ENH: import M_PI
---
 tensorflow/cc/gradients/math_grad.cc      | 32 ++++++++++---
 tensorflow/cc/gradients/math_grad_test.cc | 58 ++++++++++++++++++-----
 2 files changed, 71 insertions(+), 19 deletions(-)

diff --git a/tensorflow/cc/gradients/math_grad.cc b/tensorflow/cc/gradients/math_grad.cc
index ac288b1d83..68410812c5 100644
--- a/tensorflow/cc/gradients/math_grad.cc
+++ b/tensorflow/cc/gradients/math_grad.cc
@@ -13,6 +13,9 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
+#define _USE_MATH_DEFINES
+#include <cmath>
+
 #include "tensorflow/cc/ops/array_ops_internal.h"
 #include "tensorflow/cc/ops/math_ops_internal.h"
 #include "tensorflow/cc/ops/standard_ops.h"
@@ -200,8 +203,8 @@ Status TanhGrad(const Scope& scope, const Operation& op,
   // evaluated.
   Scope grad_scope = scope.WithControlDependencies(grad);
   auto y = ConjugateHelper(grad_scope, op.output(0));
-  grad_outputs->push_back(internal::TanhGrad(scope, y, grad));
-  return scope.status();
+  grad_outputs->push_back(internal::TanhGrad(grad_scope, y, grad));
+  return grad_scope.status();
 }
 REGISTER_GRADIENT_OP("Tanh", TanhGrad);
 
@@ -256,8 +259,8 @@ Status SigmoidGrad(const Scope& scope, const Operation& op,
   // evaluated.
   Scope grad_scope = scope.WithControlDependencies(grad);
   auto y = ConjugateHelper(grad_scope, op.output(0));
-  grad_outputs->push_back(internal::SigmoidGrad(scope, y, grad));
-  return scope.status();
+  grad_outputs->push_back(internal::SigmoidGrad(grad_scope, y, grad));
+  return grad_scope.status();
 }
 REGISTER_GRADIENT_OP("Sigmoid", SigmoidGrad);
 
@@ -696,15 +699,32 @@ Status MeanGrad(const Scope& scope, const Operation& op,
 }
 REGISTER_GRADIENT_OP("Mean", MeanGrad);
 
+Status ErfGrad(const Scope& scope, const Operation& op,
+               const std::vector<Output>& grad_inputs,
+               std::vector<Output>* grad_outputs) {
+  auto grad = grad_inputs[0];
+  auto two_over_root_pi = Cast(scope, Const(scope, 2 / std::sqrt(M_PI)),
+                               grad.type());
+  Scope grad_scope = scope.WithControlDependencies(grad);
+  auto x = ConjugateHelper(grad_scope, op.input(0));
+  // grad * 2/sqrt(pi) * exp(-x**2)
+  auto dx = Mul(grad_scope,
+                Mul(grad_scope, grad, two_over_root_pi),
+                Exp(grad_scope, Neg(grad_scope, Square(grad_scope, x))));
+  grad_outputs->push_back(dx);
+  return grad_scope.status();
+}
+REGISTER_GRADIENT_OP("Erf", ErfGrad);
+
 Status LgammaGrad(const Scope& scope, const Operation& op,
                   const std::vector<Output>& grad_inputs,
                   std::vector<Output>* grad_outputs) {
   auto grad = grad_inputs[0];
   Scope grad_scope = scope.WithControlDependencies(grad);
   auto x = ConjugateHelper(grad_scope, op.input(0));
-  auto dx = Mul(scope, grad, Digamma(scope, x));
+  auto dx = Mul(grad_scope, grad, Digamma(grad_scope, x));
   grad_outputs->push_back(dx);
-  return scope.status();
+  return grad_scope.status();
 }
 REGISTER_GRADIENT_OP("Lgamma", LgammaGrad);
 
diff --git a/tensorflow/cc/gradients/math_grad_test.cc b/tensorflow/cc/gradients/math_grad_test.cc
index a174f223ad..6313f41da5 100644
--- a/tensorflow/cc/gradients/math_grad_test.cc
+++ b/tensorflow/cc/gradients/math_grad_test.cc
@@ -64,7 +64,9 @@ class CWiseUnaryGradTest : public ::testing::Test {
     IMAG,
     CONJ,
     COMPLEX,
-    ANGLE
+    ANGLE,
+    LGAMMA,
+    ERF
   };
 
   template <typename X_T, typename Y_T>
@@ -168,6 +170,12 @@ class CWiseUnaryGradTest : public ::testing::Test {
       case ANGLE:
         y = Angle(scope_, x);
         break;
+      case LGAMMA:
+        y = Lgamma(scope_, x);
+        break;
+      case ERF:
+        y = Erf(scope_, x);
+        break;
     }
 
     float max_error;
@@ -503,6 +511,42 @@ TEST_F(CWiseUnaryGradTest, Angle) {
   TestCWiseGrad<complex64, float>(ANGLE, x_fn);
 }
 
+TEST_F(CWiseUnaryGradTest, Lgamma) {
+  auto x_fn = [this](const int i) {
+    return RV({-3.5, -2.5, -1.5, 1.0, 2.0, 3.5});
+  };
+  TestCWiseGrad<float, float>(LGAMMA, x_fn);
+}
+
+TEST_F(CWiseUnaryGradTest, Lgamma_Complex) {
+  auto x_fn = [this](const int i) {
+    return CRV({{-3.5, 0.5}, {-1.5, -0.5}, {1.5, -1.0}, {3.5, 1.0}});
+  };
+  // TODO(kbsriram)
+  // Add test when the lgamma kernel supports complex numbers
+  if (false) {
+    TestCWiseGrad<complex64, complex64>(LGAMMA, x_fn);
+  }
+}
+
+TEST_F(CWiseUnaryGradTest, Erf) {
+  auto x_fn = [this](const int i) {
+    return RV({-1.2, -1.0, -0.5, 0.3, 0.5, 1.3});
+  };
+  TestCWiseGrad<float, float>(ERF, x_fn);
+}
+
+TEST_F(CWiseUnaryGradTest, Erf_Complex) {
+  auto x_fn = [this](const int i) {
+    return CRV({{-1.2, 0.5}, {-0.5, -0.5}, {0.5, 0.5}, {1.2, -0.5}});
+  };
+  // TODO(kbsriram)
+  // Add test when the erf kernel supports complex numbers
+  if (false) {
+    TestCWiseGrad<complex64, complex64>(ERF, x_fn);
+  }
+}
+
 class MathGradTest : public ::testing::Test {
  protected:
   MathGradTest() : root_(Scope::NewRootScope().WithDevice("/cpu:0")) {}
@@ -821,17 +865,5 @@ TEST_F(NaryGradTest, Minimum) {
   RunTest(x, x_init_value, y, shape);
 }
 
-TEST_F(NaryGradTest, Lgamma) {
-  TensorShape shape({3, 2});
-  auto x = Placeholder(scope_, DT_FLOAT, Placeholder::Shape(shape));
-  auto y = Lgamma(scope_, x);
-  // Select values to avoid instability when computing finite differences.
-  // Ref: https://en.wikipedia.org/wiki/File:Gamma_plot.svg
-  Tensor x_init_value =
-      test::AsTensor<float>({-3.5f, -2.5f, -1.5f, 1.0f, 2.0f, 3.5f}, {3, 2});
-  RunTest(x, x_init_value, y, shape);
-  // TODO(suharshs): add test case for complex values
-}
-
 }  // namespace
 }  // namespace tensorflow
-- 
GitLab


From 5ad997498ac60d72f0f8f92a8d413b2398466aa7 Mon Sep 17 00:00:00 2001
From: Scott Kirkland <srkirkland@gmail.com>
Date: Thu, 5 Oct 2017 14:23:04 -0700
Subject: [PATCH 082/909] model_dir keyword argument repeated (#13494)

In https://www.tensorflow.org/tutorials/wide#adding_regularization_to_prevent_overfitting, the code repeats the model_dir keyword argument, causing a syntax error if you try to run it (`SyntaxError: keyword argument repeated`).  This remove the second occurrence of the model_dir param.
---
 tensorflow/docs_src/tutorials/wide.md | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tensorflow/docs_src/tutorials/wide.md b/tensorflow/docs_src/tutorials/wide.md
index 3055c54021..6292c1a01e 100644
--- a/tensorflow/docs_src/tutorials/wide.md
+++ b/tensorflow/docs_src/tutorials/wide.md
@@ -426,8 +426,7 @@ m = tf.estimator.LinearClassifier(
     optimizer=tf.train.FtrlOptimizer(
       learning_rate=0.1,
       l1_regularization_strength=1.0,
-      l2_regularization_strength=1.0),
-    model_dir=model_dir)
+      l2_regularization_strength=1.0))
 ```
 
 One important difference between L1 and L2 regularization is that L1
-- 
GitLab


From 8b90d603a9359af361fc4dad7883f533dd365f32 Mon Sep 17 00:00:00 2001
From: Dhananjay Nakrani <dhananjayn@google.com>
Date: Thu, 5 Oct 2017 14:53:21 -0700
Subject: [PATCH 083/909] Fix ASAN test.

ASAN correctly complains about the overflow on `CT(Eigen::NumTraits<U>::highest())`. This fixes the issue by providing correct CT for half and floats.

PiperOrigin-RevId: 171212745
---
 tensorflow/core/kernels/random_poisson_op.cc | 17 -----------------
 1 file changed, 17 deletions(-)

diff --git a/tensorflow/core/kernels/random_poisson_op.cc b/tensorflow/core/kernels/random_poisson_op.cc
index 3f635dbbaf..bf1d83ec75 100644
--- a/tensorflow/core/kernels/random_poisson_op.cc
+++ b/tensorflow/core/kernels/random_poisson_op.cc
@@ -58,25 +58,8 @@ static constexpr int kReservedSamplesPerOutput = 256;
 
 typedef Eigen::ThreadPoolDevice CPUDevice;
 
-// We will compute half-precision Poisson samples with float precision
-// intermediate calculations.
 template <typename T>
 struct PoissonComputeType {
-  typedef T ComputeType;
-};
-
-template <>
-struct PoissonComputeType<Eigen::half> {
-  typedef float ComputeType;
-};
-
-template <>
-struct PoissonComputeType<int32> {
-  typedef double ComputeType;
-};
-
-template <>
-struct PoissonComputeType<int64> {
   typedef double ComputeType;
 };
 
-- 
GitLab


From 0e71ecaf9512cd8a69af01ac85e5e1632171c651 Mon Sep 17 00:00:00 2001
From: Jacques Pienaar <jpienaar@google.com>
Date: Thu, 5 Oct 2017 15:00:43 -0700
Subject: [PATCH 084/909] [TFXLA] Loops whose values are not consumed need no
 out edges.

If there is no exit node then there is not need to add output edges to it.

PiperOrigin-RevId: 171213900
---
 .../tf2xla/functionalize_control_flow.cc      |  27 +++--
 .../tf2xla/functionalize_control_flow_test.cc | 102 ++++++++++++++++++
 2 files changed, 115 insertions(+), 14 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/functionalize_control_flow.cc b/tensorflow/compiler/tf2xla/functionalize_control_flow.cc
index 56d8bb4f2c..b9b2b4be27 100644
--- a/tensorflow/compiler/tf2xla/functionalize_control_flow.cc
+++ b/tensorflow/compiler/tf2xla/functionalize_control_flow.cc
@@ -402,10 +402,6 @@ Status FunctionalizeLoop(Graph* graph, Frame* frame,
           arg.exit = edge->dst();
         }
       }
-      if (arg.exit == nullptr) {
-        return errors::InvalidArgument("Missing Exit successor to ",
-                                       arg.switch_node->name());
-      }
     }
   }
 
@@ -470,16 +466,19 @@ Status FunctionalizeLoop(Graph* graph, Frame* frame,
     }
 
     if (!arg.is_loop_invariant) {
-      std::vector<const Edge*> edges(arg.exit->out_edges().begin(),
-                                     arg.exit->out_edges().end());
-      for (const Edge* edge : edges) {
-        Node* dst = edge->dst();
-        int dst_input = edge->dst_input();
-        graph->RemoveEdge(edge);
-
-        int src_output =
-            dst_input == Graph::kControlSlot ? Graph::kControlSlot : i;
-        graph->AddEdge(while_node, src_output, dst, dst_input);
+      // Add output edges if the output of the loop is consumed.
+      if (arg.exit != nullptr) {
+        std::vector<const Edge*> edges(arg.exit->out_edges().begin(),
+                                       arg.exit->out_edges().end());
+        for (const Edge* edge : edges) {
+          Node* dst = edge->dst();
+          int dst_input = edge->dst_input();
+          graph->RemoveEdge(edge);
+
+          int src_output =
+              dst_input == Graph::kControlSlot ? Graph::kControlSlot : i;
+          graph->AddEdge(while_node, src_output, dst, dst_input);
+        }
       }
     }
   }
diff --git a/tensorflow/compiler/tf2xla/functionalize_control_flow_test.cc b/tensorflow/compiler/tf2xla/functionalize_control_flow_test.cc
index 8f155ca85e..4acdf1a26d 100644
--- a/tensorflow/compiler/tf2xla/functionalize_control_flow_test.cc
+++ b/tensorflow/compiler/tf2xla/functionalize_control_flow_test.cc
@@ -297,6 +297,108 @@ TEST(FunctionalizeControlFlow, OneLoopVar) {
   }
 }
 
+// Tests functionalizing OneLoopVar where the loop value is not used post the
+// loop.
+// Graph:
+// x = array_ops.placeholder(dtypes.int32)
+// control_flow_ops.while_loop(lambda i: i < 10, lambda i: i + 1, [x])
+TEST(FunctionalizeControlFlow, OneLoopVarWithoutExit) {
+  Graph graph(OpRegistry::Global());
+  {
+    Scope scope = Scope::NewRootScope().ExitOnError();
+
+    auto dummy = ops::Placeholder(scope.WithOpName("Dummy"), DT_INT32);
+
+    auto source = ops::Placeholder(scope.WithOpName("source"), DT_INT32);
+    auto enter =
+        ops::internal::Enter(scope.WithOpName("while/Enter"), source, "aloop");
+    auto merge = ops::Merge(scope.WithOpName("while/Merge"),
+                            std::initializer_list<Input>{enter, dummy});
+    auto ten = ops::Const<int32>(
+        scope.WithOpName("while/Less/y").WithControlDependencies(merge.output),
+        10);
+    auto less = ops::Less(scope.WithOpName("while/Less"), merge.output, ten);
+    auto loop_cond = ops::LoopCond(scope.WithOpName("while/LoopCond"), less);
+    auto switch_ =
+        ops::Switch(scope.WithOpName("while/Switch"), merge.output, loop_cond);
+    auto identity =
+        ops::Identity(scope.WithOpName("while/Identity"), switch_.output_true);
+    auto one = ops::Const<int32>(
+        scope.WithOpName("while/add/y").WithControlDependencies(identity), 1);
+    auto add = ops::Add(scope.WithOpName("while/add"), identity, one);
+    auto next_iteration =
+        ops::NextIteration(scope.WithOpName("while/NextIteration"), add);
+
+    // Remove the dummy node and add the loop backedge.
+    scope.graph()->RemoveNode(dummy.node());
+    scope.graph()->AddEdge(next_iteration.node(), 0, merge.output.node(), 1);
+
+    TF_EXPECT_OK(scope.ToGraph(&graph));
+  }
+
+  FunctionLibraryDefinition library(OpRegistry::Global(), {});
+  TF_ASSERT_OK(FunctionalizeControlFlow(&graph, &library));
+
+  GraphDef graph_def;
+  graph.ToGraphDef(&graph_def);
+
+  NameAttrList cond_fn, body_fn;
+  TF_EXPECT_OK(FindWhileCondAndBody(graph_def, &cond_fn, &body_fn));
+
+  // Outer graph
+  {
+    Scope scope = Scope::NewRootScope().ExitOnError();
+    auto source = ops::Placeholder(scope.WithOpName("source"), DT_INT32);
+    auto while_op =
+        ops::XlaWhile(scope.WithOpName("while/LoopCond"),
+                      std::initializer_list<Input>{source}, cond_fn, body_fn);
+    GraphDef expected;
+    TF_EXPECT_OK(scope.ToGraphDef(&expected));
+    TF_EXPECT_GRAPH_EQ(expected, graph_def);
+  }
+
+  // Condition graph
+  {
+    Scope scope = Scope::NewRootScope().ExitOnError();
+    auto arg = ops::_Arg(scope.WithOpName("_arg0"), DT_INT32, 0);
+    auto ten = ops::Const<int32>(
+        scope.WithOpName("while/Less/y").WithControlDependencies(arg), 10);
+    auto less = ops::Less(scope.WithOpName("while/Less"), arg, ten);
+    auto retval = ops::_Retval(scope.WithOpName("_retval0_RetVal"), less, 0);
+
+    GraphDef expected;
+    TF_EXPECT_OK(scope.ToGraphDef(&expected));
+
+    InstantiationResultForTest result;
+    TF_EXPECT_OK(InstantiateFunctionForTest(cond_fn.name(), library, &result));
+
+    EXPECT_EQ(DataTypeVector{DT_INT32}, result.arg_types);
+    EXPECT_EQ(DataTypeVector{DT_BOOL}, result.ret_types);
+    TF_EXPECT_GRAPH_EQ(expected, result.gdef);
+  }
+
+  // Body graph.
+  {
+    Scope scope = Scope::NewRootScope().ExitOnError();
+    auto arg = ops::_Arg(scope.WithOpName("_arg0"), DT_INT32, 0);
+    auto identity = ops::Identity(scope.WithOpName("while/Identity"), arg);
+    auto one = ops::Const<int32>(
+        scope.WithOpName("while/add/y").WithControlDependencies(identity), 1);
+    auto add = ops::Add(scope.WithOpName("while/add"), identity, one);
+    auto retval = ops::_Retval(scope.WithOpName("_retval0_RetVal"), add, 0);
+
+    GraphDef expected;
+    TF_EXPECT_OK(scope.ToGraphDef(&expected));
+
+    InstantiationResultForTest result;
+    TF_EXPECT_OK(InstantiateFunctionForTest(body_fn.name(), library, &result));
+
+    EXPECT_EQ(DataTypeVector{DT_INT32}, result.arg_types);
+    EXPECT_EQ(DataTypeVector{DT_INT32}, result.ret_types);
+    TF_EXPECT_GRAPH_EQ(expected, result.gdef);
+  }
+}
+
 // Graph:
 // x = array_ops.placeholder(dtypes.int32)
 // y = array_ops.placeholder(dtypes.int32)
-- 
GitLab


From fca432028808c3d17f74b2a80a2ab8f83a0a91b1 Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Thu, 5 Oct 2017 15:28:51 -0700
Subject: [PATCH 085/909] Internal private header file with eager C struct
 definitions.

PiperOrigin-RevId: 171218337
---
 tensorflow/c/eager/BUILD            | 22 ++++++-
 tensorflow/c/eager/c_api.cc         | 59 +-----------------
 tensorflow/c/eager/c_api_internal.h | 96 +++++++++++++++++++++++++++++
 3 files changed, 116 insertions(+), 61 deletions(-)
 create mode 100644 tensorflow/c/eager/c_api_internal.h

diff --git a/tensorflow/c/eager/BUILD b/tensorflow/c/eager/BUILD
index 52945d3239..d39f229b42 100644
--- a/tensorflow/c/eager/BUILD
+++ b/tensorflow/c/eager/BUILD
@@ -10,13 +10,15 @@ load(
 
 tf_cuda_library(
     name = "c_api",
-    srcs = ["c_api.cc"],
+    srcs = [
+        "c_api.cc",
+        "c_api_internal.h",
+    ],
     hdrs = ["c_api.h"],
     copts = tf_copts(),
     visibility = ["//visibility:public"],
     deps = select({
         "//tensorflow:android": [
-            ":c_api_internal",
             "//tensorflow/core:android_tensorflow_lib_lite",
         ],
         "//conditions:default": [
@@ -33,6 +35,21 @@ tf_cuda_library(
     }),
 )
 
+tf_cuda_library(
+    name = "c_api_internal",
+    hdrs = ["c_api_internal.h"],
+    deps = [
+        ":c_api",
+        ":runtime",
+        "//tensorflow/c:c_api",
+        "//tensorflow/c:c_api_internal",
+        "//tensorflow/core:core_cpu_lib",
+        "//tensorflow/core:framework_internal",
+        "//tensorflow/core:framework_lite",
+        "//tensorflow/core:lib_internal",
+    ],
+)
+
 tf_cc_test(
     name = "c_api_test",
     srcs = ["c_api_test.cc"],
@@ -53,7 +70,6 @@ tf_cuda_library(
     visibility = ["//tensorflow:internal"],
     deps = select({
         "//tensorflow:android": [
-            ":c_api_internal",
             "//tensorflow/core:android_tensorflow_lib_lite",
         ],
         "//conditions:default": [
diff --git a/tensorflow/c/eager/c_api.cc b/tensorflow/c/eager/c_api.cc
index 801d730749..74f2e4f342 100644
--- a/tensorflow/c/eager/c_api.cc
+++ b/tensorflow/c/eager/c_api.cc
@@ -23,6 +23,7 @@ limitations under the License.
 
 #include "tensorflow/c/c_api.h"
 #include "tensorflow/c/c_api_internal.h"
+#include "tensorflow/c/eager/c_api_internal.h"
 #include "tensorflow/c/eager/runtime.h"
 #include "tensorflow/core/common_runtime/device_factory.h"
 #include "tensorflow/core/common_runtime/device_mgr.h"
@@ -51,64 +52,6 @@ string DeviceName(tensorflow::Device* d) {
 }
 }  // namespace
 
-struct TFE_Context {
-  explicit TFE_Context(TF_Session* s) : session(s) {}
-
-  // TFE_Context is an extension of TF_Session. And TF_Session needs a TF_Graph.
-  TF_Session* session;
-  tensorflow::Rendezvous* rendezvous;
-
-  tensorflow::mutex functions_mu;
-  tensorflow::FunctionLibraryDefinition func_lib_def GUARDED_BY(functions_mu){
-      tensorflow::OpRegistry::Global(), {}};
-
-  // One FunctionLibraryRuntime per device.
-  // func_libs[i] is the FunctionLibraryRuntime corresponding to
-  // session->devices[i].
-  std::unique_ptr<tensorflow::ProcessFunctionLibraryRuntime> pflr;
-
-  std::unordered_map<tensorflow::Fprint128, tensorflow::KernelAndDevice*,
-                     tensorflow::Fprint128Hasher>
-      kernel_cache;
-
-  tensorflow::FunctionLibraryRuntime* func_lib(tensorflow::Device* d) {
-    return pflr->GetFLR(d->name());
-  }
-
-  const std::vector<tensorflow::Device*>& devices() { return session->devices; }
-};
-
-struct TFE_TensorHandle {
-  TFE_TensorHandle(const tensorflow::Tensor& t, tensorflow::Device* d)
-      : t(t), d(d) {}
-
-  tensorflow::Tensor t;
-  // TODO(ashankar): d == nullptr iff local CPU
-  // This was expedient, but perhaps worth revisiting ('d' should always be a
-  // valid pointer?)
-  // This can be done if TFE_NewOp() and the TFE_TensorHandle constructors are
-  // provided with the appropriate TFE_Context.
-  //
-  // TODO(ashankar): Reference count TFE_Context to ensure that 'd' of a
-  // TFE_TensorHandle does not outlive the TFE_Context from which it came?
-  tensorflow::Device* d;
-};
-
-struct TFE_Op {
-  TFE_Op(TFE_Context* ctx, const char* op, const tensorflow::AttrTypeMap* t)
-      : ctx(ctx), name(op), attrs(op), attr_types(t), device(nullptr) {}
-
-  bool const is_function() const { return attr_types == nullptr; }
-
-  TFE_Context* ctx;  // Must outlive the TFE_Op.
-  const string name;
-  tensorflow::AttrBuilder attrs;
-  const tensorflow::AttrTypeMap* attr_types;
-  std::vector<tensorflow::Tensor> inputs;
-  std::vector<tensorflow::Device*> input_devices;
-  tensorflow::Device* device;
-};
-
 extern "C" {
 
 TFE_Context* TFE_NewContext(const TF_SessionOptions* opts, TF_Status* status) {
diff --git a/tensorflow/c/eager/c_api_internal.h b/tensorflow/c/eager/c_api_internal.h
new file mode 100644
index 0000000000..712526f170
--- /dev/null
+++ b/tensorflow/c/eager/c_api_internal.h
@@ -0,0 +1,96 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_C_EAGER_C_API_INTERNAL_H_
+#define TENSORFLOW_C_EAGER_C_API_INTERNAL_H_
+
+#include "tensorflow/c/eager/c_api.h"
+
+#include <algorithm>
+#include <cstddef>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "tensorflow/c/c_api.h"
+#include "tensorflow/c/c_api_internal.h"
+#include "tensorflow/c/eager/runtime.h"
+#include "tensorflow/core/common_runtime/device_factory.h"
+#include "tensorflow/core/common_runtime/function.h"
+#include "tensorflow/core/common_runtime/rendezvous_mgr.h"
+#include "tensorflow/core/framework/rendezvous.h"
+#include "tensorflow/core/lib/gtl/map_util.h"
+#include "tensorflow/core/lib/gtl/stl_util.h"
+#include "tensorflow/core/platform/mutex.h"
+#include "tensorflow/core/platform/thread_annotations.h"
+
+struct TFE_Context {
+  explicit TFE_Context(TF_Session* s) : session(s) {}
+
+  // TFE_Context is an extension of TF_Session. And TF_Session needs a TF_Graph.
+  TF_Session* session;
+  tensorflow::Rendezvous* rendezvous;
+
+  tensorflow::mutex functions_mu;
+  tensorflow::FunctionLibraryDefinition func_lib_def GUARDED_BY(functions_mu){
+      tensorflow::OpRegistry::Global(), {}};
+
+  // One FunctionLibraryRuntime per device.
+  // func_libs[i] is the FunctionLibraryRuntime corresponding to
+  // session->devices[i].
+  std::unique_ptr<tensorflow::ProcessFunctionLibraryRuntime> pflr;
+
+  std::unordered_map<tensorflow::Fprint128, tensorflow::KernelAndDevice*,
+                     tensorflow::Fprint128Hasher>
+      kernel_cache;
+
+  tensorflow::FunctionLibraryRuntime* func_lib(tensorflow::Device* d) {
+    return pflr->GetFLR(d->name());
+  }
+
+  const std::vector<tensorflow::Device*>& devices() { return session->devices; }
+};
+
+struct TFE_TensorHandle {
+  TFE_TensorHandle(const tensorflow::Tensor& t, tensorflow::Device* d)
+      : t(t), d(d) {}
+
+  tensorflow::Tensor t;
+  // TODO(ashankar): d == nullptr iff local CPU
+  // This was expedient, but perhaps worth revisiting ('d' should always be a
+  // valid pointer?)
+  // This can be done if TFE_NewOp() and the TFE_TensorHandle constructors are
+  // provided with the appropriate TFE_Context.
+  //
+  // TODO(ashankar): Reference count TFE_Context to ensure that 'd' of a
+  // TFE_TensorHandle does not outlive the TFE_Context from which it came?
+  tensorflow::Device* d;
+};
+
+struct TFE_Op {
+  TFE_Op(TFE_Context* ctx, const char* op, const tensorflow::AttrTypeMap* t)
+      : ctx(ctx), name(op), attrs(op), attr_types(t), device(nullptr) {}
+
+  bool const is_function() const { return attr_types == nullptr; }
+
+  TFE_Context* ctx;  // Must outlive the TFE_Op.
+  const tensorflow::string name;
+  tensorflow::AttrBuilder attrs;
+  const tensorflow::AttrTypeMap* attr_types;
+  std::vector<tensorflow::Tensor> inputs;
+  std::vector<tensorflow::Device*> input_devices;
+  tensorflow::Device* device;
+};
+
+#endif  // TENSORFLOW_C_EAGER_C_API_INTERNAL_H_
-- 
GitLab


From e11b9fd32eb5b8f1eb9b8a30dbb08fc1f83fc1dd Mon Sep 17 00:00:00 2001
From: Jingyue Wu <jingyue@google.com>
Date: Thu, 5 Oct 2017 15:42:09 -0700
Subject: [PATCH 086/909] [Grappler] Fix a bug with multiple-output nodes.

TrySimplifyAndReshapeUses should return a tensor not a node. Added a regression
test that would have failed without this CL. ArithmeticOptimizer would have
redirected the second input of concat to Split rather than Split:1.

PiperOrigin-RevId: 171220303
---
 .../optimizers/arithmetic_optimizer.cc        | 28 +++++++--------
 .../optimizers/arithmetic_optimizer.h         | 11 ++++--
 .../optimizers/arithmetic_optimizer_test.cc   | 34 +++++++++++++++++++
 3 files changed, 56 insertions(+), 17 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
index ba4487b6fc..2d7cf3b182 100644
--- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
@@ -274,7 +274,7 @@ static bool SimplyReordersData(const NodeDef& node) {
   return node.op() == "Transpose";
 }
 
-const NodeDef* ArithmeticOptimizer::TrySimplifyAndReplaceUses(
+string ArithmeticOptimizer::TrySimplifyAndReplaceUses(
     const NodeDef* node, GraphDef* graph_def, NodeMap* node_map,
     std::vector<const NodeDef*>* new_nodes) const {
   // Remove inverse transposes.
@@ -288,7 +288,7 @@ const NodeDef* ArithmeticOptimizer::TrySimplifyAndReplaceUses(
       if (Int32ValuesFromNode(*node_perm, &node_perm_values) &&
           Int32ValuesFromNode(*input_perm, &input_perm_values) &&
           AreInversePermutations(node_perm_values, input_perm_values)) {
-        return node_map->GetNode(input->input(0));
+        return input->input(0);
       }
     }
   }
@@ -316,7 +316,7 @@ const NodeDef* ArithmeticOptimizer::TrySimplifyAndReplaceUses(
       reshape->set_input(0, input->input(0));
       node_map->UpdateInput(reshape->name(), input->name(), input->input(0));
       new_nodes->push_back(reshape);
-      return reshape;
+      return reshape->name();
     }
   }
 
@@ -409,14 +409,14 @@ const NodeDef* ArithmeticOptimizer::TrySimplifyAndReplaceUses(
             consumer_of_mul->set_input(0, mul->input(0));
             node_map->UpdateInput(consumer_of_mul->name(), mul->name(),
                                   other->name());
-            return conv;
+            return conv->name();
           }
         }
       }
     }
   }
 
-  return nullptr;
+  return "";
 }
 
 namespace {
@@ -459,28 +459,28 @@ void ArithmeticOptimizer::SimplifyArithmeticOps(
   while (!nodes_to_simplify.Empty()) {
     const NodeDef* node = nodes_to_simplify.PopBack();
     std::vector<const NodeDef*> new_nodes;
-    const NodeDef* simplified_node =
+    const string simplified_tensor =
         TrySimplifyAndReplaceUses(node, optimized_graph, &node_map, &new_nodes);
-    if (!simplified_node) {
+    if (simplified_tensor.empty()) {
       continue;
     }
 
-    if (simplified_node->name() != node->name()) {
+    if (NodeName(simplified_tensor) != node->name()) {
       // When `node` is simplifed to another node rather than in-place, the
-      // consumers of `node` are redirected to `simplified_node`. Re-push the
-      // consumers into `nodes_to_simplify` for further optimizations.
+      // consumers of `node` are already redirected to `simplified_tensor`.
+      // Re-push the consumers into `nodes_to_simplify` for further
+      // optimizations.
       std::set<NodeDef*> consumers = node_map.GetOutputs(node->name());
       for (NodeDef* consumer : consumers) {
         // Update `consumer`'s use of `node` to `input`'s operand.
         for (int i = 0; i < consumer->input_size(); ++i) {
           if (NodeName(consumer->input(i)) == node->name()) {
-            *consumer->mutable_input(i) = simplified_node->name();
+            *consumer->mutable_input(i) = simplified_tensor;
           }
         }
         VLOG(2) << "Update input " << node->name() << " of " << consumer->name()
-                << " to " << simplified_node->name();
-        node_map.UpdateInput(consumer->name(), node->name(),
-                             simplified_node->name());
+                << " to " << simplified_tensor;
+        node_map.UpdateInput(consumer->name(), node->name(), simplified_tensor);
         if (!nodes_to_simplify.Exists(consumer)) {
           nodes_to_simplify.PushBack(consumer);
         }
diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.h b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.h
index 55757086cd..fc381ec907 100644
--- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.h
+++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.h
@@ -45,8 +45,9 @@ class ArithmeticOptimizer : public GraphOptimizer {
   // transposes.
   void SimplifyArithmeticOps(GraphDef* optimized_graph) const;
   // Tries to simplify the expression that roots at `node` and replaces the uses
-  // of `node` to the simplified expression. Returns the simplified node or
-  // nullptr if no simplification is performed.
+  // of `node` to the simplified expression. Returns the name of the simplified
+  // tensor (e.g. "split:1") or an emtpy string if no simplification is
+  // performed.
   //
   // `node_map` stores the mapping from node names to NodeDef*, and will be
   // updated according to the rewrite.
@@ -54,7 +55,11 @@ class ArithmeticOptimizer : public GraphOptimizer {
   // `new_nodes` will be populated with the new nodes this function creates and
   // updates. The caller can push these nodes into the simplification queue to
   // optimize them further.
-  const NodeDef* TrySimplifyAndReplaceUses(
+  //
+  // TODO(jingyue): This interface is not suitable for optimizing nodes with
+  // multiple output tensors. We should pass in a tensor name instead of a
+  // NodeDef.
+  string TrySimplifyAndReplaceUses(
       const NodeDef* node, GraphDef* graph_def, NodeMap* node_map,
       std::vector<const NodeDef*>* new_nodes) const;
 
diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc b/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc
index c81ed5a414..c8bca4282b 100644
--- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc
+++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc
@@ -140,6 +140,40 @@ TEST_F(ArithmeticOptimizerTest, RemoveInverseTransposes) {
             std::set<string>({"inputs_shape", "inputs", "outputs"}));
 }
 
+TEST_F(ArithmeticOptimizerTest, RemoveInverseTransposesMultipleOutputs) {
+  tensorflow::Scope s = tensorflow::Scope::NewRootScope();
+  Output inputs_shape =
+      ops::Const(s.WithOpName("inputs_shape"), {8, 9, 28, 28}, {4});
+  Output inputs = ops::Placeholder(s.WithOpName("inputs"), DT_FLOAT,
+                                   ops::Placeholder::Shape({8, 12, 28, 28}));
+  OutputList split = ops::Split(s, ops::Const(s, 1), inputs, 3).output;
+  Output perm1 = ops::Const(s, {0, 2, 3, 1}, {4});
+  Output perm2 = ops::Const(s, {0, 3, 1, 2}, {4});
+  Output branch0 = split[0];
+  Output branch1 = ops::Transpose(s, ops::Transpose(s, split[1], perm1), perm2);
+  Output branch2 = split[2];
+  Output concat = ops::Concat(s, {branch0, branch1, branch2}, ops::Const(s, 1));
+  Output outputs = ops::Identity(s.WithOpName("outputs"), concat);
+
+  GrapplerItem item;
+  item.fetch = {"outputs"};
+  TF_CHECK_OK(s.ToGraphDef(&item.graph));
+
+  GraphDef output;
+  TF_EXPECT_OK(ArithmeticOptimizer().Optimize(nullptr, item, &output));
+
+  item.graph = output;
+  TF_EXPECT_OK(ModelPruner().Optimize(nullptr, item, &output));
+
+  for (const NodeDef& node : output.node()) {
+    if (node.op() == "Concat") {
+      EXPECT_EQ(node.input(0), "Split");
+      EXPECT_EQ(node.input(1), "Split:1");
+      EXPECT_EQ(node.input(2), "Split:2");
+    }
+  }
+}
+
 TEST_F(ArithmeticOptimizerTest, NotRemoveTransposes) {
   tensorflow::Scope s = tensorflow::Scope::NewRootScope();
   Output inputs_shape =
-- 
GitLab


From 95a7ea781025fe7509b09e9fcb23d02f35bcf2d7 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Thu, 5 Oct 2017 15:50:44 -0700
Subject: [PATCH 087/909] Automated g4 rollback of changelist 171084886

PiperOrigin-RevId: 171221629
---
 tensorflow/compiler/tf2xla/kernels/BUILD      |   5 +-
 .../kernels/gather_op_kernel_float_int32.cc   |   3 -
 .../kernels/gather_op_kernel_float_int64.cc   |   3 -
 .../index_ops_kernel_argmax_float_1d.cc       |   3 -
 .../index_ops_kernel_argmax_float_2d.cc       |   3 -
 tensorflow/compiler/xla/service/cpu/BUILD     |  12 --
 .../cpu/custom_call_target_registry.cc        |  39 ----
 .../service/cpu/custom_call_target_registry.h |  74 -------
 .../xla/service/cpu/simple_orc_jit.cc         | 195 ++++++++----------
 tensorflow/compiler/xla/tests/BUILD           |   3 +-
 .../compiler/xla/tests/custom_call_test.cc    |  14 +-
 tensorflow/compiler/xla/xla.bzl               |   8 +
 12 files changed, 96 insertions(+), 266 deletions(-)
 delete mode 100644 tensorflow/compiler/xla/service/cpu/custom_call_target_registry.cc
 delete mode 100644 tensorflow/compiler/xla/service/cpu/custom_call_target_registry.h

diff --git a/tensorflow/compiler/tf2xla/kernels/BUILD b/tensorflow/compiler/tf2xla/kernels/BUILD
index 915c95e945..6a0c4fef75 100644
--- a/tensorflow/compiler/tf2xla/kernels/BUILD
+++ b/tensorflow/compiler/tf2xla/kernels/BUILD
@@ -5,6 +5,7 @@ package(
 )
 
 load("//tensorflow:tensorflow.bzl", "tf_kernel_library")
+load("//tensorflow/compiler/xla:xla.bzl", "export_dynamic_linkopts")
 
 tf_kernel_library(
     name = "xla_ops",
@@ -154,7 +155,6 @@ cc_library(
     visibility = ["//visibility:public"],
     deps = [
         "//tensorflow/compiler/tf2xla:xla_local_runtime_context",
-        "//tensorflow/compiler/xla/service/cpu:custom_call_target_registry",
         "//tensorflow/core:framework_lite",
         "//tensorflow/core/kernels:bounds_check",
         "//tensorflow/core/kernels:gather_functor_hdr",
@@ -169,7 +169,6 @@ cc_library(
     visibility = ["//visibility:public"],
     deps = [
         "//tensorflow/compiler/tf2xla:xla_local_runtime_context",
-        "//tensorflow/compiler/xla/service/cpu:custom_call_target_registry",
         "//tensorflow/core:framework_lite",
         "//tensorflow/core/kernels:bounds_check",
         "//tensorflow/core/kernels:gather_functor_hdr",
@@ -183,7 +182,6 @@ cc_library(
     srcs = ["index_ops_kernel_argmax_float_1d.cc"],
     visibility = ["//visibility:public"],
     deps = [
-        "//tensorflow/compiler/xla/service/cpu:custom_call_target_registry",
         "//tensorflow/core:framework_lite",
         "//third_party/eigen3",
     ],
@@ -195,7 +193,6 @@ cc_library(
     srcs = ["index_ops_kernel_argmax_float_2d.cc"],
     visibility = ["//visibility:public"],
     deps = [
-        "//tensorflow/compiler/xla/service/cpu:custom_call_target_registry",
         "//tensorflow/core:framework_lite",
         "//third_party/eigen3",
     ],
diff --git a/tensorflow/compiler/tf2xla/kernels/gather_op_kernel_float_int32.cc b/tensorflow/compiler/tf2xla/kernels/gather_op_kernel_float_int32.cc
index 0b44e0c6f8..33b1b087d0 100644
--- a/tensorflow/compiler/tf2xla/kernels/gather_op_kernel_float_int32.cc
+++ b/tensorflow/compiler/tf2xla/kernels/gather_op_kernel_float_int32.cc
@@ -17,7 +17,6 @@ limitations under the License.
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 #include "tensorflow/compiler/tf2xla/xla_local_runtime_context.h"
-#include "tensorflow/compiler/xla/service/cpu/custom_call_target_registry.h"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/kernels/gather_functor.h"
 #include "tensorflow/core/platform/dynamic_annotations.h"
@@ -71,5 +70,3 @@ EIGEN_STRONG_INLINE void gather_float_int32_xla_impl(float* out, void** data) {
 extern "C" void TF_EXPORT gather_float_int32_xla_impl(float* out, void** data) {
   tensorflow::gather_float_int32_xla_impl(out, data);
 }
-
-REGISTER_CUSTOM_CALL_TARGET(gather_float_int32_xla_impl);
diff --git a/tensorflow/compiler/tf2xla/kernels/gather_op_kernel_float_int64.cc b/tensorflow/compiler/tf2xla/kernels/gather_op_kernel_float_int64.cc
index d7c7a7bf2c..5e2d872ce0 100644
--- a/tensorflow/compiler/tf2xla/kernels/gather_op_kernel_float_int64.cc
+++ b/tensorflow/compiler/tf2xla/kernels/gather_op_kernel_float_int64.cc
@@ -17,7 +17,6 @@ limitations under the License.
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 #include "tensorflow/compiler/tf2xla/xla_local_runtime_context.h"
-#include "tensorflow/compiler/xla/service/cpu/custom_call_target_registry.h"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/kernels/gather_functor.h"
 #include "tensorflow/core/platform/dynamic_annotations.h"
@@ -71,5 +70,3 @@ EIGEN_STRONG_INLINE void gather_float_int64_xla_impl(float* out, void** data) {
 extern "C" void TF_EXPORT gather_float_int64_xla_impl(float* out, void** data) {
   tensorflow::gather_float_int64_xla_impl(out, data);
 }
-
-REGISTER_CUSTOM_CALL_TARGET(gather_float_int64_xla_impl);
diff --git a/tensorflow/compiler/tf2xla/kernels/index_ops_kernel_argmax_float_1d.cc b/tensorflow/compiler/tf2xla/kernels/index_ops_kernel_argmax_float_1d.cc
index 47cf8c6675..afbd64ca50 100644
--- a/tensorflow/compiler/tf2xla/kernels/index_ops_kernel_argmax_float_1d.cc
+++ b/tensorflow/compiler/tf2xla/kernels/index_ops_kernel_argmax_float_1d.cc
@@ -16,7 +16,6 @@ limitations under the License.
 #define EIGEN_USE_THREADS
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
-#include "tensorflow/compiler/xla/service/cpu/custom_call_target_registry.h"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/platform/dynamic_annotations.h"
 #include "tensorflow/core/platform/macros.h"
@@ -48,5 +47,3 @@ EIGEN_STRONG_INLINE void argmax_float_1d_xla_impl(void* out, void** data) {
 extern "C" void TF_EXPORT argmax_float_1d_xla_impl(void* out, void** data) {
   tensorflow::argmax_float_1d_xla_impl(out, data);
 }
-
-REGISTER_CUSTOM_CALL_TARGET(argmax_float_1d_xla_impl);
diff --git a/tensorflow/compiler/tf2xla/kernels/index_ops_kernel_argmax_float_2d.cc b/tensorflow/compiler/tf2xla/kernels/index_ops_kernel_argmax_float_2d.cc
index 9b83392d8f..841ff2f4df 100644
--- a/tensorflow/compiler/tf2xla/kernels/index_ops_kernel_argmax_float_2d.cc
+++ b/tensorflow/compiler/tf2xla/kernels/index_ops_kernel_argmax_float_2d.cc
@@ -16,7 +16,6 @@ limitations under the License.
 #define EIGEN_USE_THREADS
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
-#include "tensorflow/compiler/xla/service/cpu/custom_call_target_registry.h"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/platform/dynamic_annotations.h"
 #include "tensorflow/core/platform/macros.h"
@@ -50,5 +49,3 @@ EIGEN_STRONG_INLINE void argmax_float_2d_xla_impl(void* out, void** data) {
 extern "C" void TF_EXPORT argmax_float_2d_xla_impl(void* out, void** data) {
   tensorflow::argmax_float_2d_xla_impl(out, data);
 }
-
-REGISTER_CUSTOM_CALL_TARGET(argmax_float_2d_xla_impl);
diff --git a/tensorflow/compiler/xla/service/cpu/BUILD b/tensorflow/compiler/xla/service/cpu/BUILD
index 5d13b82427..fa6e5b2313 100644
--- a/tensorflow/compiler/xla/service/cpu/BUILD
+++ b/tensorflow/compiler/xla/service/cpu/BUILD
@@ -129,7 +129,6 @@ cc_library(
         ":cpu_runtime_avx",
         ":cpu_runtime_neon",
         ":cpu_runtime_sse4_1",
-        ":custom_call_target_registry",
         ":disassembler",
         ":runtime_conv2d",
         ":runtime_matmul",
@@ -675,17 +674,6 @@ cc_library(
     ],
 )
 
-cc_library(
-    name = "custom_call_target_registry",
-    srcs = [
-        "custom_call_target_registry.cc",
-    ],
-    hdrs = [
-        "custom_call_target_registry.h",
-    ],
-    visibility = ["//visibility:public"],
-)
-
 # -----------------------------------------------------------------------------
 
 filegroup(
diff --git a/tensorflow/compiler/xla/service/cpu/custom_call_target_registry.cc b/tensorflow/compiler/xla/service/cpu/custom_call_target_registry.cc
deleted file mode 100644
index 5f5803874b..0000000000
--- a/tensorflow/compiler/xla/service/cpu/custom_call_target_registry.cc
+++ /dev/null
@@ -1,39 +0,0 @@
-/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#include "tensorflow/compiler/xla/service/cpu/custom_call_target_registry.h"
-
-namespace xla {
-namespace cpu {
-
-CustomCallTargetRegistry* CustomCallTargetRegistry::Global() {
-  static auto* registry = new CustomCallTargetRegistry;
-  return registry;
-}
-
-void CustomCallTargetRegistry::Register(const std::string& symbol,
-                                        void* address) {
-  std::lock_guard<std::mutex> lock(mu_);
-  registered_symbols_[symbol] = address;
-}
-
-void* CustomCallTargetRegistry::Lookup(const std::string& symbol) const {
-  std::lock_guard<std::mutex> lock(mu_);
-  auto it = registered_symbols_.find(symbol);
-  return it == registered_symbols_.end() ? nullptr : it->second;
-}
-
-}  // namespace cpu
-}  // namespace xla
diff --git a/tensorflow/compiler/xla/service/cpu/custom_call_target_registry.h b/tensorflow/compiler/xla/service/cpu/custom_call_target_registry.h
deleted file mode 100644
index 2994642356..0000000000
--- a/tensorflow/compiler/xla/service/cpu/custom_call_target_registry.h
+++ /dev/null
@@ -1,74 +0,0 @@
-/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-#ifndef THIRD_PARTY_TENSORFLOW_COMPILER_XLA_SERVICE_CPU_CUSTOM_CALL_TARGET_REGISTRY_H_
-#define THIRD_PARTY_TENSORFLOW_COMPILER_XLA_SERVICE_CPU_CUSTOM_CALL_TARGET_REGISTRY_H_
-
-// This file is depended on by kernels that have to build for mobile devices.
-// For this reason, we avoid relying on TensorFlow and instead only use the
-// standard C++ library.
-
-#include <mutex>  // NOLINT
-#include <string>
-#include <unordered_map>
-
-namespace xla {
-namespace cpu {
-
-// The CPU JIT compiler uses this registry to resolve symbolic CustomCall
-// targets; so when using the CPU JIT, CustomCall targets need to be registered
-// here with the symbol name used in the CustomCall.
-//
-// The XLA AOT compiler links using a standard offline linker; so when compiling
-// in AOT mode, you *also* need to make sure the name of the callee (presumably
-// implemented in C++) matches up with the symbolic name used in the CustomCall.
-//
-// We maintain the registry in both the JIT and the AOT cases for simplicity,
-// but we only use it when running in JIT mode.
-class CustomCallTargetRegistry {
- public:
-  static CustomCallTargetRegistry* Global();
-
-  void Register(const std::string& symbol, void* address);
-  void* Lookup(const std::string& symbol) const;
-
- private:
-  std::unordered_map<std::string, void*> registered_symbols_;
-  mutable std::mutex mu_;
-};
-
-class RegisterCustomCallTarget {
- public:
-  explicit RegisterCustomCallTarget(const std::string& name, void* address) {
-    CustomCallTargetRegistry::Global()->Register(name, address);
-  }
-};
-
-#define REGISTER_CUSTOM_CALL_CONCAT(a, b) a##b
-
-#define REGISTER_CUSTOM_CALL_TARGET_WITH_SYM_HELPER(symbol, address, counter) \
-  static ::xla::cpu::RegisterCustomCallTarget REGISTER_CUSTOM_CALL_CONCAT(    \
-      custom_call_target_register, counter)(symbol,                           \
-                                            reinterpret_cast<void*>(address))
-
-#define REGISTER_CUSTOM_CALL_TARGET_WITH_SYM(symbol, address) \
-  REGISTER_CUSTOM_CALL_TARGET_WITH_SYM_HELPER(symbol, address, __COUNTER__)
-
-#define REGISTER_CUSTOM_CALL_TARGET(function) \
-  REGISTER_CUSTOM_CALL_TARGET_WITH_SYM(#function, function)
-
-}  // namespace cpu
-}  // namespace xla
-
-#endif  // THIRD_PARTY_TENSORFLOW_COMPILER_XLA_SERVICE_CPU_CUSTOM_CALL_TARGET_REGISTRY_H_
diff --git a/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc b/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc
index 0711c9de27..c3c11df090 100644
--- a/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc
+++ b/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc
@@ -31,7 +31,6 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/cpu/cpu_runtime_avx.h"
 #include "tensorflow/compiler/xla/service/cpu/cpu_runtime_neon.h"
 #include "tensorflow/compiler/xla/service/cpu/cpu_runtime_sse4_1.h"
-#include "tensorflow/compiler/xla/service/cpu/custom_call_target_registry.h"
 #include "tensorflow/compiler/xla/service/cpu/runtime_conv2d.h"
 #include "tensorflow/compiler/xla/service/cpu/runtime_matmul.h"
 #include "tensorflow/compiler/xla/service/cpu/runtime_single_threaded_conv2d.h"
@@ -43,10 +42,90 @@ namespace xla {
 namespace cpu {
 namespace {
 
+// Converts a symbol 'name' into the form expected by dlsym().
+std::string CanonicalizeSymbol(const std::string& name) {
+#if defined(__APPLE__)
+  // On Mac OS X, dlsym() expects names not to be prefixed with a leading
+  // underscore.
+  if (!name.empty() && name.front() == '_') {
+    return name.substr(1);
+  }
+#endif
+  return name;
+}
+
+class JITSymbolTable {
+ public:
+  JITSymbolTable() { Populate(); }
+
+  void* Lookup(llvm::StringRef jit_symbol_name) const {
+    auto it = jit_symbol_table_.find(jit_symbol_name);
+    return it == jit_symbol_table_.end() ? nullptr : it->getValue();
+  }
+
+  static bool MustBeInTable(llvm::StringRef name) {
+    // In particular, names starting with
+    // runtime::kXlaCpuRuntimeSymbolNamePrefix should not be dlsym'ed.
+    return name.startswith(runtime::kXlaCpuRuntimeSymbolNamePrefix);
+  }
+
+ private:
+  void AddJITSymbolToTable(llvm::StringRef jit_symbol_name,
+                           llvm::StringRef cpp_symbol_name,
+                           void* jit_symbol_value) {
+    // The JIT symbol name and the C++ symbol name (with an extern "C" linkage)
+    // need to match, otherwise AOT links will fail.
+    CHECK(jit_symbol_name == cpp_symbol_name);
+    CHECK(jit_symbol_table_.insert({jit_symbol_name, jit_symbol_value}).second);
+  }
+
+  void Populate() {
+#define ADD_JIT_SYMBOL_TO_TABLE(base_name)                       \
+  do {                                                           \
+    AddJITSymbolToTable(                                         \
+        xla::cpu::runtime::k##base_name##SymbolName,             \
+        "__xla_cpu_runtime_" #base_name,                         \
+        reinterpret_cast<void*>(__xla_cpu_runtime_##base_name)); \
+  } while (false)
+
+    ADD_JIT_SYMBOL_TO_TABLE(AcquireInfeedBufferForDequeue);
+    ADD_JIT_SYMBOL_TO_TABLE(ReleaseInfeedBufferAfterDequeue);
+    ADD_JIT_SYMBOL_TO_TABLE(AcquireOutfeedBufferForPopulation);
+    ADD_JIT_SYMBOL_TO_TABLE(ReleaseOutfeedBufferAfterPopulation);
+    ADD_JIT_SYMBOL_TO_TABLE(ExpV8F32AVX);
+    ADD_JIT_SYMBOL_TO_TABLE(LogV8F32AVX);
+    ADD_JIT_SYMBOL_TO_TABLE(ExpV4F32SSE);
+    ADD_JIT_SYMBOL_TO_TABLE(LogV4F32SSE);
+    ADD_JIT_SYMBOL_TO_TABLE(ExpV4F32NEON);
+    ADD_JIT_SYMBOL_TO_TABLE(LogV4F32NEON);
+    ADD_JIT_SYMBOL_TO_TABLE(EigenConvF32);
+    ADD_JIT_SYMBOL_TO_TABLE(EigenMatMulF32);
+    ADD_JIT_SYMBOL_TO_TABLE(EigenMatMulF64);
+    ADD_JIT_SYMBOL_TO_TABLE(EigenSingleThreadedConvF32);
+    ADD_JIT_SYMBOL_TO_TABLE(EigenSingleThreadedMatMulF32);
+    ADD_JIT_SYMBOL_TO_TABLE(EigenSingleThreadedMatMulF64);
+
+#undef ADD_JIT_SYMBOL_TO_TABLE
+  }
+
+  llvm::StringMap<void*> jit_symbol_table_;
+};
+
+const JITSymbolTable& GetJITSymbolTable() {
+  static JITSymbolTable* symbol_table = new JITSymbolTable;
+  return *symbol_table;
+}
+
 // A simple SymbolResolver that delegates to the host dynamic linker.
 struct SimpleResolver : public llvm::JITSymbolResolver {
   llvm::JITSymbol findSymbol(const std::string& name) override {
-    void* func_addr = CustomCallTargetRegistry::Global()->Lookup(name);
+    std::string canonical_name = CanonicalizeSymbol(name);
+    const JITSymbolTable& jit_symbol_table = GetJITSymbolTable();
+
+    void* func_addr = JITSymbolTable::MustBeInTable(canonical_name)
+                          ? jit_symbol_table.Lookup(canonical_name)
+                          : dlsym(RTLD_DEFAULT, canonical_name.c_str());
+
     if (func_addr == nullptr) {
       return nullptr;
     }
@@ -159,117 +238,5 @@ llvm::JITSymbol SimpleOrcJIT::FindSymbol(const std::string& name) {
   return nullptr;
 }
 
-namespace {
-// Register some known symbols with the CustomCallTargetRegistry.
-bool RegisterKnownJITSymbols() {
-  CustomCallTargetRegistry* registry = CustomCallTargetRegistry::Global();
-
-#define REGISTER_CPU_RUNTIME_SYMBOL(base_name)                                \
-  do {                                                                        \
-    auto* function_address =                                                  \
-        reinterpret_cast<void*>(__xla_cpu_runtime_##base_name);               \
-    registry->Register(xla::cpu::runtime::k##base_name##SymbolName,           \
-                       function_address);                                     \
-    CHECK_EQ(                                                                 \
-        tensorflow::StringPiece(xla::cpu::runtime::k##base_name##SymbolName), \
-        "__xla_cpu_runtime_" #base_name);                                     \
-  } while (false)
-
-  REGISTER_CPU_RUNTIME_SYMBOL(AcquireInfeedBufferForDequeue);
-  REGISTER_CPU_RUNTIME_SYMBOL(ReleaseInfeedBufferAfterDequeue);
-  REGISTER_CPU_RUNTIME_SYMBOL(AcquireOutfeedBufferForPopulation);
-  REGISTER_CPU_RUNTIME_SYMBOL(ReleaseOutfeedBufferAfterPopulation);
-  REGISTER_CPU_RUNTIME_SYMBOL(ExpV8F32AVX);
-  REGISTER_CPU_RUNTIME_SYMBOL(LogV8F32AVX);
-  REGISTER_CPU_RUNTIME_SYMBOL(ExpV4F32SSE);
-  REGISTER_CPU_RUNTIME_SYMBOL(LogV4F32SSE);
-  REGISTER_CPU_RUNTIME_SYMBOL(ExpV4F32NEON);
-  REGISTER_CPU_RUNTIME_SYMBOL(LogV4F32NEON);
-  REGISTER_CPU_RUNTIME_SYMBOL(EigenConvF32);
-  REGISTER_CPU_RUNTIME_SYMBOL(EigenMatMulF32);
-  REGISTER_CPU_RUNTIME_SYMBOL(EigenMatMulF64);
-  REGISTER_CPU_RUNTIME_SYMBOL(EigenSingleThreadedConvF32);
-  REGISTER_CPU_RUNTIME_SYMBOL(EigenSingleThreadedMatMulF32);
-  REGISTER_CPU_RUNTIME_SYMBOL(EigenSingleThreadedMatMulF64);
-
-#undef REGISTER_CPU_RUNTIME_SYMBOL
-
-#define REGISTER_LIBM_SYMBOL(name)                                    \
-  do {                                                                \
-    /* Register both the F32 and F64 variants of the libm symbol.  */ \
-    registry->Register(#name "f", reinterpret_cast<void*>(name##f));  \
-    registry->Register(#name, reinterpret_cast<void*>(name));         \
-  } while (false)
-
-  REGISTER_LIBM_SYMBOL(acos);
-  REGISTER_LIBM_SYMBOL(acosh);
-  REGISTER_LIBM_SYMBOL(asin);
-  REGISTER_LIBM_SYMBOL(asinh);
-  REGISTER_LIBM_SYMBOL(atan);
-  REGISTER_LIBM_SYMBOL(atan2);
-  REGISTER_LIBM_SYMBOL(atanh);
-  REGISTER_LIBM_SYMBOL(cbrt);
-  REGISTER_LIBM_SYMBOL(ceil);
-  REGISTER_LIBM_SYMBOL(copysign);
-  REGISTER_LIBM_SYMBOL(cos);
-  REGISTER_LIBM_SYMBOL(cosh);
-  REGISTER_LIBM_SYMBOL(erf);
-  REGISTER_LIBM_SYMBOL(erfc);
-  REGISTER_LIBM_SYMBOL(exp);
-  REGISTER_LIBM_SYMBOL(exp2);
-  REGISTER_LIBM_SYMBOL(expm1);
-  REGISTER_LIBM_SYMBOL(fabs);
-  REGISTER_LIBM_SYMBOL(fdim);
-  REGISTER_LIBM_SYMBOL(floor);
-  REGISTER_LIBM_SYMBOL(fma);
-  REGISTER_LIBM_SYMBOL(fmax);
-  REGISTER_LIBM_SYMBOL(fmin);
-  REGISTER_LIBM_SYMBOL(fmod);
-  REGISTER_LIBM_SYMBOL(frexp);
-  REGISTER_LIBM_SYMBOL(hypot);
-  REGISTER_LIBM_SYMBOL(ilogb);
-  REGISTER_LIBM_SYMBOL(ldexp);
-  REGISTER_LIBM_SYMBOL(lgamma);
-  REGISTER_LIBM_SYMBOL(llrint);
-  REGISTER_LIBM_SYMBOL(llround);
-  REGISTER_LIBM_SYMBOL(log);
-  REGISTER_LIBM_SYMBOL(log10);
-  REGISTER_LIBM_SYMBOL(log1p);
-  REGISTER_LIBM_SYMBOL(log2);
-  REGISTER_LIBM_SYMBOL(logb);
-  REGISTER_LIBM_SYMBOL(lrint);
-  REGISTER_LIBM_SYMBOL(lround);
-  REGISTER_LIBM_SYMBOL(modf);
-  REGISTER_LIBM_SYMBOL(nan);
-  REGISTER_LIBM_SYMBOL(nearbyint);
-  REGISTER_LIBM_SYMBOL(nextafter);
-  REGISTER_LIBM_SYMBOL(nexttoward);
-  REGISTER_LIBM_SYMBOL(pow);
-  REGISTER_LIBM_SYMBOL(remainder);
-  REGISTER_LIBM_SYMBOL(remquo);
-  REGISTER_LIBM_SYMBOL(rint);
-  REGISTER_LIBM_SYMBOL(round);
-  REGISTER_LIBM_SYMBOL(scalbln);
-  REGISTER_LIBM_SYMBOL(scalbn);
-  REGISTER_LIBM_SYMBOL(sin);
-  REGISTER_LIBM_SYMBOL(sincos);
-  REGISTER_LIBM_SYMBOL(sinh);
-  REGISTER_LIBM_SYMBOL(sqrt);
-  REGISTER_LIBM_SYMBOL(tan);
-  REGISTER_LIBM_SYMBOL(tanh);
-  REGISTER_LIBM_SYMBOL(tgamma);
-  REGISTER_LIBM_SYMBOL(trunc);
-
-#undef REGISTER_LIBM_SYMBOL
-
-  registry->Register("memcpy", reinterpret_cast<void*>(memcpy));
-  registry->Register("memmove", reinterpret_cast<void*>(memmove));
-  registry->Register("memset", reinterpret_cast<void*>(memset));
-  return true;
-}
-
-bool unused = RegisterKnownJITSymbols();
-}  // namespace
-
 }  // namespace cpu
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/tests/BUILD b/tensorflow/compiler/xla/tests/BUILD
index 84bebd4708..e45b839afd 100644
--- a/tensorflow/compiler/xla/tests/BUILD
+++ b/tensorflow/compiler/xla/tests/BUILD
@@ -23,6 +23,7 @@ filegroup(
     ]),
 )
 
+load("//tensorflow/compiler/xla:xla.bzl", "export_dynamic_linkopts")
 load("//tensorflow/compiler/xla/tests:build_defs.bzl", "xla_test")
 load("//tensorflow/compiler/xla/tests:build_defs.bzl", "xla_test_library")
 load("//tensorflow/compiler/xla/tests:build_defs.bzl", "generate_backend_suites")
@@ -980,13 +981,13 @@ xla_test(
 xla_test(
     name = "custom_call_test",
     srcs = ["custom_call_test.cc"],
+    linkopts = export_dynamic_linkopts,
     deps = [
         "//tensorflow/compiler/xla:literal_util",
         "//tensorflow/compiler/xla:shape_util",
         "//tensorflow/compiler/xla:util",
         "//tensorflow/compiler/xla:xla_data_proto",
         "//tensorflow/compiler/xla/service:hlo",
-        "//tensorflow/compiler/xla/service/cpu:custom_call_target_registry",
         "//tensorflow/compiler/xla/tests:hlo_test_base",
         "//tensorflow/compiler/xla/tests:literal_test_util",
         "//tensorflow/compiler/xla/tests:xla_internal_test_main",
diff --git a/tensorflow/compiler/xla/tests/custom_call_test.cc b/tensorflow/compiler/xla/tests/custom_call_test.cc
index 74f73a1ddc..342478bc74 100644
--- a/tensorflow/compiler/xla/tests/custom_call_test.cc
+++ b/tensorflow/compiler/xla/tests/custom_call_test.cc
@@ -18,7 +18,6 @@ limitations under the License.
 
 #include "tensorflow/compiler/xla/literal_util.h"
 #include "tensorflow/compiler/xla/ptr_util.h"
-#include "tensorflow/compiler/xla/service/cpu/custom_call_target_registry.h"
 #include "tensorflow/compiler/xla/service/hlo_computation.h"
 #include "tensorflow/compiler/xla/service/hlo_instruction.h"
 #include "tensorflow/compiler/xla/service/hlo_module.h"
@@ -32,19 +31,19 @@ limitations under the License.
 #include "tensorflow/core/platform/macros.h"
 #include "tensorflow/core/platform/test.h"
 
-namespace {
-void R0F32Add2(float* out, float** in) {
+
+extern "C" void TF_EXPORT R0F32Add2(float* out, float** in) {
   TF_ANNOTATE_MEMORY_IS_INITIALIZED(in, sizeof(float*));
   *out = **in + 2.0f;
 }
 
-void R2F32ReduceSum(float* out, float** in) {
+extern "C" void TF_EXPORT R2F32ReduceSum(float* out, float** in) {
   TF_ANNOTATE_MEMORY_IS_INITIALIZED(in, sizeof(float) * 4);
   float* array = in[0];
   *out = array[0] + array[1] + array[2] + array[3];
 }
 
-void Add1ToValues(float* out, float** in) {
+extern "C" void TF_EXPORT Add1ToValues(float* out, float** in) {
   TF_ANNOTATE_MEMORY_IS_INITIALIZED(in, sizeof(float) * 4);
   float* array = in[0];
   out[0] = array[0] + 1;
@@ -52,11 +51,6 @@ void Add1ToValues(float* out, float** in) {
   out[2] = array[2] + 1;
   out[3] = array[3] + 1;
 }
-}  // namespace
-
-REGISTER_CUSTOM_CALL_TARGET(R0F32Add2);
-REGISTER_CUSTOM_CALL_TARGET(R2F32ReduceSum);
-REGISTER_CUSTOM_CALL_TARGET(Add1ToValues);
 
 namespace xla {
 namespace {
diff --git a/tensorflow/compiler/xla/xla.bzl b/tensorflow/compiler/xla/xla.bzl
index 3fa5bcc1df..22e70ec97a 100644
--- a/tensorflow/compiler/xla/xla.bzl
+++ b/tensorflow/compiler/xla/xla.bzl
@@ -17,3 +17,11 @@ def xla_proto_library(name, srcs=[], deps=[], visibility=None, testonly=0):
                    protoc="@protobuf_archive//:protoc",
                    testonly=testonly,
                    visibility=visibility,)
+
+# Flags required for modules that export symbols that are to be called by the
+# XLA CustomCall operator. CustomCall must be able to find symbols with dlsym(),
+# which on Linux requires we link with --export-dynamic.
+export_dynamic_linkopts = select({
+    "//tensorflow:darwin": [],
+    "//conditions:default": ["-Wl,--export-dynamic"],
+})
-- 
GitLab


From e4aa9dc317773ff66d85ac422b83e8952d4610b5 Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Thu, 5 Oct 2017 15:53:25 -0700
Subject: [PATCH 088/909] Start of work towards ordering access to resources in
 tfe.defun/graph_callable.

Still missing handling control flow and pessimistic alias analysis.

PiperOrigin-RevId: 171221946
---
 tensorflow/python/eager/BUILD                 |  1 -
 tensorflow/python/eager/function.py           | 28 +++++++++++++++++--
 tensorflow/python/eager/graph_callable.py     | 14 ++++------
 .../python/eager/graph_callable_test.py       | 13 +++++++++
 4 files changed, 44 insertions(+), 12 deletions(-)

diff --git a/tensorflow/python/eager/BUILD b/tensorflow/python/eager/BUILD
index 76d4f37e9a..963eaf0742 100644
--- a/tensorflow/python/eager/BUILD
+++ b/tensorflow/python/eager/BUILD
@@ -308,7 +308,6 @@ py_library(
         "//tensorflow/python:dtypes",
         "//tensorflow/python:errors",
         "//tensorflow/python:framework_ops",
-        "//tensorflow/python:graph_to_function_def",
         "//tensorflow/python:resource_variable_ops",
         "//tensorflow/python:util",
         "//tensorflow/python:variable_scope",
diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py
index 6ffc914f73..8a1936b3fe 100644
--- a/tensorflow/python/eager/function.py
+++ b/tensorflow/python/eager/function.py
@@ -47,6 +47,28 @@ _scoped_captures = threading.local()
 _scoped_captures.tensors = None
 
 
+def make_function_def(graph, operations, inputs, outputs):
+  """Makes function def where accesses to resources are serialized."""
+  last_op_using_resource_tensor = {}
+
+  # TODO(apassos) probably control flow has to be handled delicately here as in
+  # if a resource is accessed inside a control flow context we need the control
+  # dependency to point to something outside the context which is guaranteed to
+  # happen after the access.
+  #
+  # TODO(apassos) this should do some form of alias analysis as ops which
+  # forward the resources such as Identity and Switch can cause serialization to
+  # fail.
+  for op in operations:
+    for t in op.inputs:
+      if t.dtype == dtypes.resource:
+        if t.name in last_op_using_resource_tensor:
+          op._add_control_input(last_op_using_resource_tensor[t.name])  # pylint: disable=protected-access
+        last_op_using_resource_tensor[t.name] = op
+  return graph_to_function_def.graph_to_function_def(
+      graph, operations, inputs, outputs)
+
+
 @contextlib.contextmanager
 def capture_tensors(captures):
   old = _scoped_captures.__dict__.get("tensors", None)
@@ -217,14 +239,14 @@ class _GraphModeFunction(object):
             grad_ys=self._out_grad_placeholders)
         shapes = [x.shape for x in in_gradients if x is not None]
     captures = list(sorted(c.captured_tensors, key=lambda x: x.name))
-    forward_function_def = graph_to_function_def.graph_to_function_def(
+    forward_function_def = make_function_def(
         self._graph, self._ops, self._input_placeholders,
         filtered_outputs + captures)
     self._forward_fdef = _DefinedFunction(forward_function_def)
     _register_with_name(_forward_name(self._func_name), forward_function_def)
     backward_outputs = [x for x in in_gradients if x is not None]
     all_inputs = self._out_grad_placeholders + captures
-    backward_function_def = graph_to_function_def.graph_to_function_def(
+    backward_function_def = make_function_def(
         self._graph, [x.op for x in self._out_grad_placeholders
                      ] + list(sorted(c.known_ops, key=lambda x: x.name)),
         all_inputs, backward_outputs)
@@ -386,7 +408,7 @@ def _defun_internal(name, func, args, kwds):
   all_inputs = flat_inputs + list(extra_placeholders)
 
   func_def_outputs = [x for x in outputs_list if x is not None]
-  inference_function_def = graph_to_function_def.graph_to_function_def(
+  inference_function_def = make_function_def(
       tmp_graph, tmp_graph.get_operations(), all_inputs, func_def_outputs)
   # Register any other functions defined in the graph
   # TODO(ashankar): Oh lord, forgive me for this lint travesty.
diff --git a/tensorflow/python/eager/graph_callable.py b/tensorflow/python/eager/graph_callable.py
index 5933da7865..64d1659993 100644
--- a/tensorflow/python/eager/graph_callable.py
+++ b/tensorflow/python/eager/graph_callable.py
@@ -27,7 +27,6 @@ from tensorflow.python.eager import function
 from tensorflow.python.eager import tape
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
-from tensorflow.python.framework import graph_to_function_def
 from tensorflow.python.framework import ops as tf_ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import resource_variable_ops
@@ -186,11 +185,10 @@ class _VariableCapturingScope(object):
           shared_name=name, shape=shape, dtype=dtype)
       if initializer is None:
         initializer = _default_initializer(name, shape, dtype)
-      with tf_ops.control_dependencies(
-          [resource_variable_ops.assign_variable_op(
-              graph_mode_resource, initializer(shape, dtype))]):
-        handle = array_ops.identity(v.variable.handle)
-      return _VariableFromResource(handle, dtype, name, shape=v.shape)
+      resource_variable_ops.assign_variable_op(
+          graph_mode_resource, initializer(shape, dtype))
+      return _VariableFromResource(
+          graph_mode_resource, dtype, name, shape=v.shape)
 
     scope = variable_scope.get_variable_scope()
     with variable_scope.variable_scope(scope, custom_getter=_custom_getter):
@@ -357,7 +355,7 @@ def _graph_callable_internal(func, shape_and_dtypes):
   all_inputs = variable_placeholders + placeholder_inputs
 
   func_def_outputs = [x for x in outputs_list if isinstance(x, tf_ops.Tensor)]
-  initializer_function_def = graph_to_function_def.graph_to_function_def(
+  initializer_function_def = function.make_function_def(
       tmp_graph,
       initializing_operations,
       placeholder_inputs,
@@ -381,7 +379,7 @@ def _graph_callable_internal(func, shape_and_dtypes):
 
   capture_func_def_outputs = [
       x for x in captured_outlist if isinstance(x, tf_ops.Tensor)]
-  captured_function_def = graph_to_function_def.graph_to_function_def(
+  captured_function_def = function.make_function_def(
       tmp_graph,
       capturing_operations,
       all_inputs,
diff --git a/tensorflow/python/eager/graph_callable_test.py b/tensorflow/python/eager/graph_callable_test.py
index cee6adec04..4ad8f1f36e 100644
--- a/tensorflow/python/eager/graph_callable_test.py
+++ b/tensorflow/python/eager/graph_callable_test.py
@@ -100,6 +100,19 @@ class GraphCallableTest(test.TestCase):
                          constant_op.constant([2.],
                                               dtype=dtypes.float32)).numpy())
 
+  def testUpdatesAreOrdered(self):
+
+    @graph_callable.graph_callable(
+        [graph_callable.ShapeAndDtype(shape=(), dtype=dtypes.float32)])
+    def my_function(x):
+      v = variable_scope.get_variable(
+          "v", initializer=init_ops.zeros_initializer(), shape=())
+      v.assign(x + 1)
+      v.assign(v * x)
+      return v.read_value()
+
+    self.assertEqual(my_function(constant_op.constant(2.0)).numpy(), 6.0)
+
   def testEmptyInitializer(self):
 
     @graph_callable.graph_callable(
-- 
GitLab


From f5ac1f40c96e3d41464ce39d18d9f97b9acfadc7 Mon Sep 17 00:00:00 2001
From: Jianwei Xie <xiejw@google.com>
Date: Thu, 5 Oct 2017 16:18:33 -0700
Subject: [PATCH 089/909] Fixed the training_test on gpu-py3.

PiperOrigin-RevId: 171225190
---
 tensorflow/python/estimator/training_test.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/estimator/training_test.py b/tensorflow/python/estimator/training_test.py
index 8c00ebddf3..d88ca2c925 100644
--- a/tensorflow/python/estimator/training_test.py
+++ b/tensorflow/python/estimator/training_test.py
@@ -705,7 +705,7 @@ class TrainingExecutorRunMasterTest(test.TestCase):
     mock_est = test.mock.Mock(spec=estimator_lib.Estimator)
     mock_est.evaluate = lambda *args, **kw: {ops.GraphKeys.GLOBAL_STEP: 123}
     mock_est.config = self._run_config
-    mock_train_spec = test.mock.Mock(spec=training.TrainSpec)
+    mock_train_spec = test.mock.Mock(spec=training.TrainSpec, max_steps=123)
     mock_eval_spec = test.mock.Mock(spec=training.EvalSpec, exporters=[])
 
     executor = training._TrainingExecutor(mock_est, mock_train_spec,
@@ -750,7 +750,7 @@ class TrainingExecutorRunMasterTest(test.TestCase):
     mock_est = test.mock.Mock(spec=estimator_lib.Estimator)
     mock_est.evaluate = lambda *args, **kw: {ops.GraphKeys.GLOBAL_STEP: 123}
     mock_est.config = self._run_config
-    mock_train_spec = test.mock.Mock(spec=training.TrainSpec)
+    mock_train_spec = test.mock.Mock(spec=training.TrainSpec, max_steps=123)
     mock_eval_spec = test.mock.Mock(spec=training.EvalSpec, exporters=[])
 
     executor = training._TrainingExecutor(mock_est, mock_train_spec,
-- 
GitLab


From 073d90578904aa00dee34e27d9cc6bac68af2c47 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 5 Oct 2017 16:34:53 -0700
Subject: [PATCH 090/909] Respect container context when creating
 ResourceVariables in Eager mode.

PiperOrigin-RevId: 171227139
---
 .../kernel_tests/resource_variable_ops_test.py       | 11 +++++++++++
 tensorflow/python/ops/resource_variable_ops.py       | 12 ++++++------
 2 files changed, 17 insertions(+), 6 deletions(-)

diff --git a/tensorflow/python/kernel_tests/resource_variable_ops_test.py b/tensorflow/python/kernel_tests/resource_variable_ops_test.py
index 17ecb6faf5..8cf8286ed1 100644
--- a/tensorflow/python/kernel_tests/resource_variable_ops_test.py
+++ b/tensorflow/python/kernel_tests/resource_variable_ops_test.py
@@ -411,6 +411,17 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase):
       # Test operations
       self.assertAllEqual((v * 2).numpy(), (v + v).numpy())
 
+  def testContainerEager(self):
+    with context.eager_mode():
+      v1 = resource_variable_ops.ResourceVariable(initial_value=lambda: 1,
+                                                  name="same")
+      with ops.container("different"):
+        v2 = resource_variable_ops.ResourceVariable(initial_value=lambda: 0,
+                                                    name="same")
+      v2.assign(2)
+      self.assertEqual(1, v1.read_value().numpy())
+      self.assertEqual(2, v2.read_value().numpy())
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/ops/resource_variable_ops.py b/tensorflow/python/ops/resource_variable_ops.py
index 4ef9b05d51..cbfa141256 100644
--- a/tensorflow/python/ops/resource_variable_ops.py
+++ b/tensorflow/python/ops/resource_variable_ops.py
@@ -38,9 +38,11 @@ from tensorflow.python.ops.gen_resource_variable_ops import *
 from tensorflow.python.util import compat
 
 
-def _eager_safe_variable_handle(shape, dtype, shared_name, name, graph_mode,
-                                container=None):
+def _eager_safe_variable_handle(shape, dtype, shared_name, name, graph_mode):
   """Creates a variable handle with information to do shape inference."""
+  container = ops.get_default_graph()._container  # pylint: disable=protected-access
+  if container is None:
+    container = ""
   handle = gen_resource_variable_ops.var_handle_op(shape=shape, dtype=dtype,
                                                    shared_name=shared_name,
                                                    name=name,
@@ -305,8 +307,7 @@ class ResourceVariable(variables.Variable):
                 dtype=initial_value.dtype.base_dtype,
                 shared_name=handle_name,
                 name=name,
-                graph_mode=False,
-                container="")
+                graph_mode=False)
             self._handle_device = (
                 self._handle.device if self._in_graph_mode else
                 context.get_default_context().device_name)
@@ -332,8 +333,7 @@ class ResourceVariable(variables.Variable):
               dtype=initial_value.dtype.base_dtype,
               shared_name=handle_name,
               name=name,
-              graph_mode=self._in_graph_mode,
-              container="")
+              graph_mode=self._in_graph_mode)
           self._handle_device = (self._handle.device if self._in_graph_mode else
                                  context.get_default_context().device_name)
           self._graph_shape = initial_value.get_shape()
-- 
GitLab


From be2b3dcbb6f17d472fa60553ab149f4472b27643 Mon Sep 17 00:00:00 2001
From: Amit Patankar <amitpatankar@google.com>
Date: Thu, 5 Oct 2017 17:10:00 -0700
Subject: [PATCH 091/909] Build tests only by default for
 ci_parameterized_build.sh

PiperOrigin-RevId: 171231427
---
 tensorflow/tools/ci_build/ci_parameterized_build.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/tools/ci_build/ci_parameterized_build.sh b/tensorflow/tools/ci_build/ci_parameterized_build.sh
index 7a1479c150..f640f07585 100755
--- a/tensorflow/tools/ci_build/ci_parameterized_build.sh
+++ b/tensorflow/tools/ci_build/ci_parameterized_build.sh
@@ -129,7 +129,7 @@ BAZEL_CMD="bazel test"
 BAZEL_BUILD_ONLY_CMD="bazel build"
 BAZEL_CLEAN_CMD="bazel clean"
 
-DEFAULT_BAZEL_CONFIGS="--config=gcp --config=hdfs"
+DEFAULT_BAZEL_CONFIGS="--config=gcp --config=hdfs --build_tests_only"
 
 PIP_CMD="${CI_BUILD_DIR}/builds/pip.sh"
 PIP_TEST_TUTORIALS_FLAG="--test_tutorials"
-- 
GitLab


From d6513c8149d5b69faa250949c6bec6c796c553e8 Mon Sep 17 00:00:00 2001
From: Amit Patankar <amitpatankar@google.com>
Date: Thu, 5 Oct 2017 17:41:09 -0700
Subject: [PATCH 092/909] Automated g4 rollback of changelist 171231427

PiperOrigin-RevId: 171234659
---
 tensorflow/tools/ci_build/ci_parameterized_build.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/tools/ci_build/ci_parameterized_build.sh b/tensorflow/tools/ci_build/ci_parameterized_build.sh
index f640f07585..7a1479c150 100755
--- a/tensorflow/tools/ci_build/ci_parameterized_build.sh
+++ b/tensorflow/tools/ci_build/ci_parameterized_build.sh
@@ -129,7 +129,7 @@ BAZEL_CMD="bazel test"
 BAZEL_BUILD_ONLY_CMD="bazel build"
 BAZEL_CLEAN_CMD="bazel clean"
 
-DEFAULT_BAZEL_CONFIGS="--config=gcp --config=hdfs --build_tests_only"
+DEFAULT_BAZEL_CONFIGS="--config=gcp --config=hdfs"
 
 PIP_CMD="${CI_BUILD_DIR}/builds/pip.sh"
 PIP_TEST_TUTORIALS_FLAG="--test_tutorials"
-- 
GitLab


From bdbcde775f47d56a98b7f0f7dcd72bcb83867ae8 Mon Sep 17 00:00:00 2001
From: Mike Case <mikecase@google.com>
Date: Thu, 5 Oct 2017 18:41:51 -0700
Subject: [PATCH 093/909] Fix small typo in docs of learn runner.

---
 tensorflow/contrib/learn/python/learn/learn_runner.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/learn/python/learn/learn_runner.py b/tensorflow/contrib/learn/python/learn/learn_runner.py
index 9f9740ec49..2af723a0d6 100644
--- a/tensorflow/contrib/learn/python/learn/learn_runner.py
+++ b/tensorflow/contrib/learn/python/learn/learn_runner.py
@@ -165,7 +165,7 @@ def run(experiment_fn, output_dir=None, schedule=None, run_config=None,
       must be None.
       2) It accepts two arguments `run_config` and `hparams`, which should be
       used to create the `Estimator` (`run_config` passed as `config` to its
-      constructor; `hparams` used as the hyper-paremeters of the model).
+      constructor; `hparams` used as the hyper-parameters of the model).
       It must return an `Experiment`. For this case, `output_dir` must be None.
     output_dir: Base output directory [Deprecated].
     schedule: The name of the method in the `Experiment` to run.
-- 
GitLab


From 86238e8d09efce59de038b062a230030aa8bdd3a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 5 Oct 2017 18:38:03 -0700
Subject: [PATCH 094/909] Track memory allocation/deallocation history.

PiperOrigin-RevId: 171239477
---
 .../python/kernel_tests/core_rnn_cell_test.py |  16 ++-
 .../rnn/python/kernel_tests/core_rnn_test.py  |  26 ++--
 .../core/common_runtime/direct_session.cc     |   3 +
 tensorflow/core/common_runtime/executor.cc    | 119 ++++++++++--------
 .../common_runtime/step_stats_collector.cc    |  99 +++++++++++----
 .../common_runtime/step_stats_collector.h     |  51 +++++++-
 tensorflow/core/distributed_runtime/worker.cc |   1 +
 .../worker_cache_logger.cc                    |   2 +-
 tensorflow/core/framework/step_stats.proto    |  12 +-
 .../core/framework/tracking_allocator.cc      |  20 ++-
 .../core/framework/tracking_allocator.h       |  18 ++-
 .../core/framework/tracking_allocator_test.cc |  28 ++++-
 tensorflow/core/platform/gpu_tracer_test.cc   |   1 +
 .../profiler/internal/run_metadata_test.py    |  29 +++++
 14 files changed, 317 insertions(+), 108 deletions(-)

diff --git a/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_cell_test.py b/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_cell_test.py
index deebadc142..8349188f6f 100644
--- a/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_cell_test.py
+++ b/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_cell_test.py
@@ -450,6 +450,17 @@ class RNNCellTest(test.TestCase):
       outputs, _ = cell(x, m)
       self.assertTrue("cpu:14159" in outputs.device.lower())
 
+  def _retrieve_cpu_gpu_stats(self, run_metadata):
+    cpu_stats = None
+    gpu_stats = None
+    step_stats = run_metadata.step_stats
+    for ds in step_stats.dev_stats:
+      if "cpu:0" in ds.device[-5:].lower():
+        cpu_stats = ds.node_stats
+      if "gpu:0" == ds.device[-5:].lower():
+        gpu_stats = ds.node_stats
+    return cpu_stats, gpu_stats
+
   def testDeviceWrapperDynamicExecutionNodesAreAllProperlyLocated(self):
     if not test.is_gpu_available():
       # Can't perform this test w/o a GPU
@@ -471,10 +482,7 @@ class RNNCellTest(test.TestCase):
         sess.run([variables_lib.global_variables_initializer()])
         _ = sess.run(outputs, options=opts, run_metadata=run_metadata)
 
-      step_stats = run_metadata.step_stats
-      ix = 0 if gpu_dev in step_stats.dev_stats[0].device else 1
-      gpu_stats = step_stats.dev_stats[ix].node_stats
-      cpu_stats = step_stats.dev_stats[1 - ix].node_stats
+      cpu_stats, gpu_stats = self._retrieve_cpu_gpu_stats(run_metadata)
       self.assertFalse([s for s in cpu_stats if "gru_cell" in s.node_name])
       self.assertTrue([s for s in gpu_stats if "gru_cell" in s.node_name])
 
diff --git a/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_test.py b/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_test.py
index 40a3fb2fb0..2fa033632a 100644
--- a/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_test.py
+++ b/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_test.py
@@ -2203,6 +2203,17 @@ class TensorArrayOnCorrectDeviceTest(test.TestCase):
 
     return run_metadata
 
+  def _retrieve_cpu_gpu_stats(self, run_metadata):
+    cpu_stats = None
+    gpu_stats = None
+    step_stats = run_metadata.step_stats
+    for ds in step_stats.dev_stats:
+      if "cpu:0" in ds.device[-5:].lower():
+        cpu_stats = ds.node_stats
+      if "gpu:0" == ds.device[-5:].lower():
+        gpu_stats = ds.node_stats
+    return cpu_stats, gpu_stats
+
   def testRNNOnCPUCellOnGPU(self):
     if not test.is_gpu_available():
       return  # Test requires access to a GPU
@@ -2210,10 +2221,7 @@ class TensorArrayOnCorrectDeviceTest(test.TestCase):
     gpu_dev = test.gpu_device_name()
     run_metadata = self._execute_rnn_on(
         rnn_device="/cpu:0", cell_device=gpu_dev)
-    step_stats = run_metadata.step_stats
-    ix = 0 if (gpu_dev in step_stats.dev_stats[0].device) else 1
-    gpu_stats = step_stats.dev_stats[ix].node_stats
-    cpu_stats = step_stats.dev_stats[1 - ix].node_stats
+    cpu_stats, gpu_stats = self._retrieve_cpu_gpu_stats(run_metadata)
 
     def _assert_in(op_str, in_stats, out_stats):
       self.assertTrue(any(op_str in s.node_name for s in in_stats))
@@ -2236,10 +2244,7 @@ class TensorArrayOnCorrectDeviceTest(test.TestCase):
     run_metadata = self._execute_rnn_on(
         rnn_device="/cpu:0", cell_device="/cpu:0",
         input_device=gpu_dev)
-    step_stats = run_metadata.step_stats
-    ix = 0 if (gpu_dev in step_stats.dev_stats[0].device) else 1
-    gpu_stats = step_stats.dev_stats[ix].node_stats
-    cpu_stats = step_stats.dev_stats[1 - ix].node_stats
+    cpu_stats, gpu_stats = self._retrieve_cpu_gpu_stats(run_metadata)
 
     def _assert_in(op_str, in_stats, out_stats):
       self.assertTrue(any(op_str in s.node_name for s in in_stats))
@@ -2255,10 +2260,7 @@ class TensorArrayOnCorrectDeviceTest(test.TestCase):
     gpu_dev = test.gpu_device_name()
     run_metadata = self._execute_rnn_on(
         input_device=gpu_dev)
-    step_stats = run_metadata.step_stats
-    ix = 0 if (gpu_dev in step_stats.dev_stats[0].device) else 1
-    gpu_stats = step_stats.dev_stats[ix].node_stats
-    cpu_stats = step_stats.dev_stats[1 - ix].node_stats
+    cpu_stats, gpu_stats = self._retrieve_cpu_gpu_stats(run_metadata)
 
     def _assert_in(op_str, in_stats, out_stats):
       self.assertTrue(any(op_str in s.node_name for s in in_stats))
diff --git a/tensorflow/core/common_runtime/direct_session.cc b/tensorflow/core/common_runtime/direct_session.cc
index 8674831eac..316fb0ac16 100644
--- a/tensorflow/core/common_runtime/direct_session.cc
+++ b/tensorflow/core/common_runtime/direct_session.cc
@@ -652,6 +652,9 @@ Status DirectSession::Run(const RunOptions& run_options,
   // Save the output tensors of this run we choose to keep.
   TF_RETURN_IF_ERROR(
       run_state.tensor_store.SaveTensors(output_names, &session_state_));
+  if (args.stats_collector) {
+    args.stats_collector->Finalize();
+  }
 
   // Build and return the cost model as instructed.
   mutex_lock l(executor_lock_);
diff --git a/tensorflow/core/common_runtime/executor.cc b/tensorflow/core/common_runtime/executor.cc
index b1537eab01..f57834cfbe 100644
--- a/tensorflow/core/common_runtime/executor.cc
+++ b/tensorflow/core/common_runtime/executor.cc
@@ -74,10 +74,13 @@ bool IsInitializationOp(const Node* node) {
 // Returns true iff the node is a transfer node.
 // TODO(tucker): merge with the DetailText function in session.cc
 // in a common location.
-bool SetTimelineLabel(const Node* node, NodeExecStats* node_stats) {
+bool SetTimelineLabel(const Node* node, NodeExecStatsWrapper* stats) {
   bool is_transfer_node = false;
+  if (!stats) {
+    return is_transfer_node;
+  }
   string memory;
-  for (auto& all : node_stats->memory()) {
+  for (auto& all : stats->stats()->memory()) {
     int64 tot = all.total_bytes();
     if (tot >= 0.1 * 1048576.0) {
       int64 peak = all.peak_bytes();
@@ -115,7 +118,7 @@ bool SetTimelineLabel(const Node* node, NodeExecStats* node_stats) {
         strings::StrCat(memory, node->name(), " = ", node->type_string(), "(",
                         str_util::Join(node->requested_inputs(), ", "), ")");
   }
-  node_stats->set_timeline_label(text);
+  stats->stats()->set_timeline_label(text);
   return is_transfer_node;
 }
 
@@ -123,49 +126,52 @@ bool SetTimelineLabel(const Node* node, NodeExecStats* node_stats) {
 namespace nodestats {
 inline int64 NowInUsec() { return Env::Default()->NowMicros(); }
 
-void SetScheduled(NodeExecStats* nt, int64 t) { nt->set_scheduled_micros(t); }
+void SetScheduled(NodeExecStatsWrapper* stats, int64 t) {
+  if (!stats) return;
+  stats->stats()->set_scheduled_micros(t);
+}
 
-void SetAllStart(NodeExecStats* nt) { nt->set_all_start_micros(NowInUsec()); }
+void SetAllStart(NodeExecStatsWrapper* stats) {
+  if (!stats) return;
+  stats->stats()->set_all_start_micros(NowInUsec());
+}
 
-void SetOpStart(NodeExecStats* nt) {
+void SetOpStart(NodeExecStatsWrapper* stats) {
+  if (!stats) return;
+  NodeExecStats* nt = stats->stats();
   DCHECK_NE(nt->all_start_micros(), 0);
   nt->set_op_start_rel_micros(NowInUsec() - nt->all_start_micros());
 }
 
-void SetOpEnd(NodeExecStats* nt) {
+void SetOpEnd(NodeExecStatsWrapper* stats) {
+  if (!stats) return;
+  NodeExecStats* nt = stats->stats();
   DCHECK_NE(nt->all_start_micros(), 0);
   nt->set_op_end_rel_micros(NowInUsec() - nt->all_start_micros());
 }
 
-void SetAllEnd(NodeExecStats* nt) {
+void SetAllEnd(NodeExecStatsWrapper* stats) {
+  if (!stats) return;
+  NodeExecStats* nt = stats->stats();
   DCHECK_NE(nt->all_start_micros(), 0);
   nt->set_all_end_rel_micros(NowInUsec() - nt->all_start_micros());
 }
 
-void SetOutput(NodeExecStats* nt, int slot, const Tensor* v) {
+void SetOutput(NodeExecStatsWrapper* stats, int slot, const Tensor* v) {
+  if (!stats) return;
   DCHECK(v);
-  NodeOutput* no = nt->add_output();
+  NodeOutput* no = stats->stats()->add_output();
   no->set_slot(slot);
   v->FillDescription(no->mutable_tensor_description());
 }
 
-void SetMemory(NodeExecStats* nt, OpKernelContext* ctx) {
+void SetMemory(NodeExecStatsWrapper* stats, OpKernelContext* ctx) {
+  if (!stats) return;
+
   for (const auto& allocator_pair : ctx->wrapped_allocators()) {
-    AllocatorMemoryUsed* memory = nt->add_memory();
-    // retrieving the sizes from the wrapped allocator removes the
-    // executor's reference to it, so allocator_pair.second must not
-    // be dereferenced again after this statement
-    const auto sizes = allocator_pair.second->GetSizesAndUnRef();
-    memory->set_allocator_name(allocator_pair.first->Name());
-    memory->set_total_bytes(std::get<0>(sizes));
-    memory->set_peak_bytes(std::get<1>(sizes));
-    memory->set_live_bytes(std::get<2>(sizes));
-
-    AllocatorStats stats;
-    allocator_pair.first->GetStats(&stats);
-    memory->set_allocator_bytes_in_use(stats.bytes_in_use);
-  }
-  auto* ms = nt->mutable_memory_stats();
+    stats->AddAllocation(allocator_pair.first, allocator_pair.second);
+  }
+  auto* ms = stats->stats()->mutable_memory_stats();
   ms->set_host_temp_memory_size(ctx->host_temp_memory_size());
   ms->set_device_temp_memory_size(ctx->device_temp_memory_size());
   for (const auto& alloc_id : ctx->host_persistent_alloc_ids()) {
@@ -179,12 +185,14 @@ void SetMemory(NodeExecStats* nt, OpKernelContext* ctx) {
       ctx->device_persistent_memory_allocated());
 }
 
-void SetReferencedTensors(NodeExecStats* nt,
+void SetReferencedTensors(NodeExecStatsWrapper* stats,
                           const TensorReferenceVector& tensors) {
+  if (!stats) return;
   // be careful not to increment the reference count on any tensor
   // while recording the information
   for (size_t i = 0; i < tensors.size(); ++i) {
-    AllocationDescription* description = nt->add_referenced_tensor();
+    AllocationDescription* description =
+        stats->stats()->add_referenced_tensor();
     tensors.at(i).FillDescription(description);
   }
 }
@@ -1241,7 +1249,7 @@ class ExecutorState {
 
   // After item->kernel computation is done, processes its outputs.
   Status ProcessOutputs(const NodeItem& item, OpKernelContext* ctx,
-                        EntryVector* outputs, NodeExecStats* stats);
+                        EntryVector* outputs, NodeExecStatsWrapper* stats);
 
   // After processing the outputs, propagates the outputs to their dsts.
   // Contents of *outputs are left in an indeterminate state after
@@ -1252,7 +1260,8 @@ class ExecutorState {
   // "node" just finishes. Takes ownership of "stats". Returns true if
   // execution has completed.
   bool NodeDone(const Status& s, const Node* node, const TaggedNodeSeq& ready,
-                NodeExecStats* stats, TaggedNodeReadyQueue* inline_ready);
+                NodeExecStatsWrapper* stats,
+                TaggedNodeReadyQueue* inline_ready);
 
   // Schedule all the expensive nodes in 'ready', and put all the inexpensive
   // nodes in 'ready' into 'inline_ready'.
@@ -1448,7 +1457,8 @@ void ExecutorState::RunAsync(Executor::DoneCallback done) {
 // sync kernels because these vectors are kept on the stack.
 struct ExecutorState::AsyncState {
   AsyncState(const OpKernelContext::Params& p, const TaggedNode& _tagged_node,
-             const NodeItem* _item, Entry* _first_input, NodeExecStats* _stats)
+             const NodeItem* _item, Entry* _first_input,
+             NodeExecStatsWrapper* _stats)
       : saved_inputs(*p.inputs),
         saved_input_device_contexts(*p.input_device_contexts),
         saved_input_alloc_attrs(*p.input_alloc_attrs),
@@ -1473,7 +1483,7 @@ struct ExecutorState::AsyncState {
   const NodeItem* item;
   Entry* first_input;
   OpKernelContext ctx;
-  NodeExecStats* stats;
+  NodeExecStatsWrapper* stats;
 
  private:
   OpKernelContext::Params* ParamsButClearingEigenGPUDevice(
@@ -1517,7 +1527,7 @@ void ExecutorState::Process(TaggedNode tagged_node, int64 scheduled_usec) {
   params.stats_collector = stats_collector_;
 
   Status s;
-  NodeExecStats* stats = nullptr;
+  NodeExecStatsWrapper* stats = nullptr;
   EntryVector outputs;
   bool completed = false;
   inline_ready.push_back(tagged_node);
@@ -1547,8 +1557,8 @@ void ExecutorState::Process(TaggedNode tagged_node, int64 scheduled_usec) {
     if (stats_collector_ && !tagged_node.is_dead) {
       // track allocations if and only if we are collecting statistics
       params.track_allocations = true;
-      stats = new NodeExecStats;
-      stats->set_node_name(node->name());
+      stats = new NodeExecStatsWrapper;
+      stats->stats()->set_node_name(node->name());
       nodestats::SetScheduled(stats, scheduled_usec);
       nodestats::SetAllStart(stats);
     }
@@ -1604,17 +1614,17 @@ void ExecutorState::Process(TaggedNode tagged_node, int64 scheduled_usec) {
 
         auto done = [this, state]() {
           Device* device = impl_->params_.device;
-          NodeExecStats* stats = state->stats;      // Shorthand
+          NodeExecStatsWrapper* stats = state->stats;  // Shorthand
           Entry* first_input = state->first_input;  // Shorthand
 
           if (vlog_) {
             VLOG(2) << this << " Async kernel done: "
                     << SummarizeNode(*state->item->node);
           }
-          if (stats) nodestats::SetOpEnd(stats);
+          nodestats::SetOpEnd(stats);
           EntryVector outputs;
           Status s = ProcessOutputs(*state->item, &state->ctx, &outputs, stats);
-          if (stats) nodestats::SetMemory(stats, &state->ctx);
+          nodestats::SetMemory(stats, &state->ctx);
           // Clears inputs.
           const int num_inputs = state->item->num_inputs;
           for (int i = 0; i < num_inputs; ++i) {
@@ -1633,7 +1643,7 @@ void ExecutorState::Process(TaggedNode tagged_node, int64 scheduled_usec) {
             // Get the list of all tensors accessed during the execution
             TensorReferenceVector accessed;
             state->ctx.retrieve_accessed_tensors(&accessed);
-            if (stats) nodestats::SetReferencedTensors(stats, accessed);
+            nodestats::SetReferencedTensors(stats, accessed);
             // callee takes ownership of the vector
             device->ConsumeListOfAccessedTensors(state->ctx.op_device_context(),
                                                  accessed);
@@ -1643,22 +1653,21 @@ void ExecutorState::Process(TaggedNode tagged_node, int64 scheduled_usec) {
           delete state;
           if (completed) Finish();
         };
-        if (stats) nodestats::SetOpStart(stats);
+        nodestats::SetOpStart(stats);
         device->ComputeAsync(async, &state->ctx, done);
       } else {
         // Synchronous computes.
         OpKernelContext ctx(&params, item.num_outputs);
-        if (stats) nodestats::SetOpStart(stats);
+        nodestats::SetOpStart(stats);
         device->Compute(CHECK_NOTNULL(op_kernel), &ctx);
-        if (stats) nodestats::SetOpEnd(stats);
-
+        nodestats::SetOpEnd(stats);
         s = ProcessOutputs(item, &ctx, &outputs, stats);
         if (s.ok() && impl_->device_record_tensor_accesses_) {
           // Get the list of all tensors accessed during the execution
           ctx.retrieve_accessed_tensors(&accessed_tensors);
           device_context = ctx.op_device_context();
         }
-        if (stats) nodestats::SetMemory(stats, &ctx);
+        nodestats::SetMemory(stats, &ctx);
       }
     }
 
@@ -1675,7 +1684,7 @@ void ExecutorState::Process(TaggedNode tagged_node, int64 scheduled_usec) {
       }
       outputs.clear();
       if (!accessed_tensors.empty()) {
-        if (stats) nodestats::SetReferencedTensors(stats, accessed_tensors);
+        nodestats::SetReferencedTensors(stats, accessed_tensors);
         // device_context is set above in synchronous computes
         device->ConsumeListOfAccessedTensors(device_context, accessed_tensors);
       }
@@ -1772,7 +1781,7 @@ Status ExecutorState::PrepareInputs(const NodeItem& item, Entry* first_input,
 
 Status ExecutorState::ProcessOutputs(const NodeItem& item, OpKernelContext* ctx,
                                      EntryVector* outputs,
-                                     NodeExecStats* stats) {
+                                     NodeExecStatsWrapper* stats) {
   const Node* node = item.node;
   DCHECK_EQ(0, outputs->size());
   outputs->resize(item.num_outputs);
@@ -1995,16 +2004,16 @@ void ExecutorState::PropagateOutputs(const TaggedNode& tagged_node,
 }
 
 bool ExecutorState::NodeDone(const Status& s, const Node* node,
-                             const TaggedNodeSeq& ready, NodeExecStats* stats,
+                             const TaggedNodeSeq& ready,
+                             NodeExecStatsWrapper* stats,
                              TaggedNodeReadyQueue* inline_ready) {
-  if (stats) {
-    nodestats::SetAllEnd(stats);
-    if (!SetTimelineLabel(node, stats)) {
-      // Only record non-transfer nodes.
-      stats_collector_->Save(impl_->params_.device->name(), stats);
-    } else {
-      delete stats;
-    }
+  nodestats::SetAllEnd(stats);
+  if (!SetTimelineLabel(node, stats)) {
+    // Only record non-transfer nodes.
+    // Transfers 'stats' ownership to 'stats_collector_'.
+    stats_collector_->Save(impl_->params_.device->name(), stats);
+  } else if (stats) {
+    delete stats;
   }
 
   bool abort_run = false;
diff --git a/tensorflow/core/common_runtime/step_stats_collector.cc b/tensorflow/core/common_runtime/step_stats_collector.cc
index ee12624074..e7f58f9ecf 100644
--- a/tensorflow/core/common_runtime/step_stats_collector.cc
+++ b/tensorflow/core/common_runtime/step_stats_collector.cc
@@ -16,8 +16,8 @@ limitations under the License.
 #include "tensorflow/core/common_runtime/step_stats_collector.h"
 #include "tensorflow/core/common_runtime/costmodel_manager.h"
 #include "tensorflow/core/framework/allocation_description.pb.h"
-#include "tensorflow/core/framework/step_stats.pb.h"
 #include "tensorflow/core/framework/tensor_description.pb.h"
+#include "tensorflow/core/framework/tracking_allocator.h"
 #include "tensorflow/core/graph/costmodel.h"
 #include "tensorflow/core/lib/core/stringpiece.h"
 #include "tensorflow/core/lib/strings/scanner.h"
@@ -25,7 +25,40 @@ limitations under the License.
 
 namespace tensorflow {
 
-StepStatsCollector::StepStatsCollector(StepStats* ss) : step_stats_(ss) {}
+NodeExecStatsWrapper::NodeExecStatsWrapper()
+    : NodeExecStatsWrapper(new NodeExecStats) {}
+NodeExecStatsWrapper::NodeExecStatsWrapper(NodeExecStats* stats)
+    : stats_(stats) {}
+
+void NodeExecStatsWrapper::AddAllocation(
+    Allocator* allocator, TrackingAllocator* tracking_allocator) {
+  AllocatorMemoryUsed* memory = stats_->add_memory();
+  memory->set_allocator_name(allocator->Name());
+  auto sizes = tracking_allocator->GetSizes();
+  memory->set_total_bytes(std::get<0>(sizes));
+  memory->set_peak_bytes(std::get<1>(sizes));
+  memory->set_live_bytes(std::get<2>(sizes));
+
+  AllocatorStats stats;
+  allocator->GetStats(&stats);
+  memory->set_allocator_bytes_in_use(stats.bytes_in_use);
+  allocations_.push_back(std::make_pair(memory, tracking_allocator));
+}
+
+void NodeExecStatsWrapper::Finalize() {
+  for (auto& alloc : allocations_) {
+    AllocatorMemoryUsed* memory = alloc.first;
+    for (auto& record : alloc.second->GetRecordsAndUnRef()) {
+      auto* r = memory->add_allocation_records();
+      r->set_alloc_bytes(record.alloc_bytes);
+      r->set_alloc_micros(record.alloc_micros);
+    }
+  }
+  allocations_.clear();
+}
+
+StepStatsCollector::StepStatsCollector(StepStats* ss)
+    : finalized_(false), step_stats_(ss) {}
 
 static int ExtractGpuWithStreamAll(string device_name) {
   // Check if the device name matches the ".*gpu:(\\d+)/stream:all$" regexp,
@@ -92,6 +125,9 @@ void StepStatsCollector::BuildCostModel(
     const std::unordered_map<string, const Graph*>& device_map) {
   mutex_lock lock(mu_);
 
+  if (!finalized_) {
+    FinalizeInternal();
+  }
   // Hardware stats for gpu are available under a fake device named
   // "gpu:<id>/stream::all.
   // Use them instead of regular stats whenever they're available to extract
@@ -208,39 +244,60 @@ void StepStatsCollector::BuildCostModel(
 }
 
 void StepStatsCollector::Save(const string& device, NodeExecStats* nt) {
-  VLOG(1) << "Save dev " << device << " nt " << nt;
+  Save(device, new NodeExecStatsWrapper(nt));
+}
+
+void StepStatsCollector::Save(const string& device,
+                              NodeExecStatsWrapper* stats) {
+  if (!stats) return;
+  VLOG(1) << "Save dev " << device << " nt " << stats->stats();
   {
     mutex_lock l(mu_);
+    CHECK(!finalized_);
     if (!step_stats_ || collectedNodes >= kMaxCollectedNodes) {
       VLOG(1) << "step_stats_ nullptr or already collected too many nodes.";
-      delete nt;
+      delete stats;
       return;
     }
-    DeviceStepStats* dss = nullptr;
-    // Slow linear scan, but it should only be called
-    // by a Worker in a context with < ~10 devices.
-    // TODO(tucker): consider adding a std::unordered_map.
-    for (auto& ds : *step_stats_->mutable_dev_stats()) {
-      if (ds.device() == device) {
-        dss = &ds;
-        break;
-      }
-    }
-    if (dss == nullptr) {
-      dss = step_stats_->add_dev_stats();
-      dss->set_device(device);
-    }
-    nt->Swap(dss->add_node_stats());
+    auto& dss = dev_stats_[device];
+    dss.push_back(std::unique_ptr<NodeExecStatsWrapper>(stats));
     collectedNodes++;
   }
-  delete nt;
 }
 
-void StepStatsCollector::Swap(StepStats* ss) {
+void StepStatsCollector::Finalize() {
+  mutex_lock l(mu_);
+  FinalizeInternal();
+}
+
+void StepStatsCollector::FinalizeAndSwap(StepStats* ss) {
   mutex_lock l(mu_);
   CHECK(step_stats_);
+  FinalizeInternal();
   ss->Swap(step_stats_);
   collectedNodes = 0;
 }
 
+void StepStatsCollector::FinalizeInternal() {
+  if (!step_stats_ || finalized_) {
+    return;
+  }
+  finalized_ = true;
+  std::map<string, DeviceStepStats*> dev_stats_pb;
+  for (auto& ds : *step_stats_->mutable_dev_stats()) {
+    dev_stats_pb[ds.device()] = &ds;
+  }
+  for (const auto& dev_stat : dev_stats_) {
+    if (dev_stats_pb.find(dev_stat.first) == dev_stats_pb.end()) {
+      DeviceStepStats* ndev_stat = step_stats_->add_dev_stats();
+      ndev_stat->set_device(dev_stat.first);
+      dev_stats_pb[dev_stat.first] = ndev_stat;
+    }
+    DeviceStepStats* dss = dev_stats_pb.at(dev_stat.first);
+    for (auto& stats : dev_stat.second) {
+      stats->Finalize();
+      stats->stats()->Swap(dss->add_node_stats());
+    }
+  }
+}
 }  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/step_stats_collector.h b/tensorflow/core/common_runtime/step_stats_collector.h
index 37b1c4b308..b1fd28a982 100644
--- a/tensorflow/core/common_runtime/step_stats_collector.h
+++ b/tensorflow/core/common_runtime/step_stats_collector.h
@@ -15,23 +15,59 @@ limitations under the License.
 #ifndef TENSORFLOW_CORE_DISTRIBUTED_RUNTIME_STEP_STATS_COLLECTOR_H_
 #define TENSORFLOW_CORE_DISTRIBUTED_RUNTIME_STEP_STATS_COLLECTOR_H_
 
+#include <memory>
 #include <unordered_map>
+#include <vector>
+#include "tensorflow/core/framework/step_stats.pb.h"
+#include "tensorflow/core/lib/gtl/inlined_vector.h"
 #include "tensorflow/core/platform/mutex.h"
 #include "tensorflow/core/platform/thread_annotations.h"
 #include "tensorflow/core/platform/types.h"
 
 namespace tensorflow {
 
+class Allocator;
+class AllocatorMemoryUsed;
 class CostModelManager;
 class Graph;
 class NodeExecStats;
 class StepStats;
+class TrackingAllocator;
+
+// Wraps NodeExecStats and adds allocation to it.
+class NodeExecStatsWrapper {
+ public:
+  NodeExecStatsWrapper();
+  // Owns 'stats'.
+  NodeExecStatsWrapper(NodeExecStats* stats);
+
+  // Destructor calls Finalize() to release the TrackingAllocators.
+  ~NodeExecStatsWrapper() { Finalize(); }
+
+  NodeExecStats* stats() { return stats_.get(); }
+
+  // "Does not take ownership of the 'allocator'.
+  // Transfers ownership of the 'tracking_allocator' to *this."
+  void AddAllocation(Allocator* allocator,
+                     TrackingAllocator* tracking_allocator);
+
+ private:
+  friend class StepStatsCollector;
+
+  // Populates stats_ and releases TrackingAllocator.
+  void Finalize();
+
+  gtl::InlinedVector<std::pair<AllocatorMemoryUsed*, TrackingAllocator*>, 2>
+      allocations_;
+  std::unique_ptr<NodeExecStats> stats_;
+};
 
 // StepStatsCollector manages the collection of a StepStats object.
 // The StepStats object holds multiple DeviceStats.
 // Each DeviceStats object holds multiple NodeExecStats.
 class StepStatsCollector {
  public:
+  // Does not take ownership of `ss`.
   explicit StepStatsCollector(StepStats* ss);
 
   // BuildCostModel builds or updates a CostModel managed by cost_model_manager,
@@ -42,16 +78,27 @@ class StepStatsCollector {
       const std::unordered_map<string, const Graph*>& device_map);
 
   // Save saves nt to the DeviceStats object associated with device.
+  // Should be called before Finalize.
   void Save(const string& device, NodeExecStats* nt);
+  void Save(const string& device, NodeExecStatsWrapper* stats);
 
-  // Swap replaces the current step stats with ss.
-  void Swap(StepStats* ss);
+  // The following 2 Finalize methods populate the StepStats passed
+  // from the constructor. Calling it more than once won't have any effect.
+  // User shouldn't call Save() methods after Finalize.
+  void Finalize();
+  // swaps the content of StepStats* from constructor with 'ss'.
+  void FinalizeAndSwap(StepStats* ss);
 
  private:
+  void FinalizeInternal() EXCLUSIVE_LOCKS_REQUIRED(mu_);
+
+  typedef std::vector<std::unique_ptr<NodeExecStatsWrapper>> NodeExecStatsVec;
   // TODO(suharshs): Make this configurable if its not possible to find a value
   //                 that works for all cases.
   const uint64 kMaxCollectedNodes = 1 << 20;
   mutex mu_;
+  bool finalized_ GUARDED_BY(mu_);
+  std::unordered_map<string, NodeExecStatsVec> dev_stats_ GUARDED_BY(mu_);
   StepStats* step_stats_ GUARDED_BY(mu_);
   uint64 collectedNodes GUARDED_BY(mu_) = 0;
 };
diff --git a/tensorflow/core/distributed_runtime/worker.cc b/tensorflow/core/distributed_runtime/worker.cc
index 94c1dd0a93..b7c5793736 100644
--- a/tensorflow/core/distributed_runtime/worker.cc
+++ b/tensorflow/core/distributed_runtime/worker.cc
@@ -179,6 +179,7 @@ void Worker::DoRunGraph(CallOptions* opts, RunGraphRequestWrapper* request,
             response->AddRecv(key, val);
           }
         }
+        if (collector) collector->Finalize();
         delete collector;
         delete out;
         done(s);
diff --git a/tensorflow/core/distributed_runtime/worker_cache_logger.cc b/tensorflow/core/distributed_runtime/worker_cache_logger.cc
index 8e413b80f0..702af78c88 100644
--- a/tensorflow/core/distributed_runtime/worker_cache_logger.cc
+++ b/tensorflow/core/distributed_runtime/worker_cache_logger.cc
@@ -60,7 +60,7 @@ bool WorkerCacheLogger::RetrieveLogs(int64 step_id, StepStats* ss) {
   mutex_lock l(mu_);
   LogMap::iterator iter = log_map_.find(step_id);
   if (iter != log_map_.end()) {
-    iter->second.collector->Swap(ss);
+    iter->second.collector->FinalizeAndSwap(ss);
     delete iter->second.collector;
     log_map_.erase(iter);
     return true;
diff --git a/tensorflow/core/framework/step_stats.proto b/tensorflow/core/framework/step_stats.proto
index 3b3d62193c..99dee2257e 100644
--- a/tensorflow/core/framework/step_stats.proto
+++ b/tensorflow/core/framework/step_stats.proto
@@ -9,9 +9,13 @@ option java_package = "org.tensorflow.framework";
 import "tensorflow/core/framework/allocation_description.proto";
 import "tensorflow/core/framework/tensor_description.proto";
 
-// TODO(tucker): The next 4 message defs are very similar to
-// the *LogEntry messages in profile.proto.  They should be
-// unified in one place.
+// An allocation/de-allocation operation performed by the allocator.
+message AllocationRecord {
+  // The timestamp of the operation.
+  int64 alloc_micros = 1;
+  // Number of bytes allocated, or de-allocated if negative.
+  int64 alloc_bytes = 2;
+}
 
 message AllocatorMemoryUsed {
   string allocator_name = 1;
@@ -20,6 +24,8 @@ message AllocatorMemoryUsed {
   int64 peak_bytes = 3;
   // The bytes that are not deallocated.
   int64 live_bytes = 4;
+  // The allocation and deallocation timeline.
+  repeated AllocationRecord allocation_records = 6;
 
   // These are snapshots of the overall allocator memory stats.
   // The number of live bytes currently allocated by the allocator.
diff --git a/tensorflow/core/framework/tracking_allocator.cc b/tensorflow/core/framework/tracking_allocator.cc
index 1052ac0554..db996e31b0 100644
--- a/tensorflow/core/framework/tracking_allocator.cc
+++ b/tensorflow/core/framework/tracking_allocator.cc
@@ -15,6 +15,7 @@ limitations under the License.
 
 #include "tensorflow/core/framework/tracking_allocator.h"
 
+#include "tensorflow/core/platform/env.h"
 #include "tensorflow/core/platform/logging.h"
 
 namespace tensorflow {
@@ -44,6 +45,7 @@ void* TrackingAllocator::AllocateRaw(
       allocated_ += allocated_bytes;
       high_watermark_ = std::max(high_watermark_, allocated_);
       total_bytes_ += allocated_bytes;
+      allocations_.emplace_back(allocated_bytes, Env::Default()->NowMicros());
       ++ref_;
     }
   } else if (track_sizes_locally_) {
@@ -59,10 +61,12 @@ void* TrackingAllocator::AllocateRaw(
     allocated_ += allocated_bytes;
     high_watermark_ = std::max(high_watermark_, allocated_);
     total_bytes_ += allocated_bytes;
+    allocations_.emplace_back(allocated_bytes, Env::Default()->NowMicros());
     ++ref_;
   } else {
     mutex_lock lock(mu_);
     total_bytes_ += num_bytes;
+    allocations_.emplace_back(num_bytes, Env::Default()->NowMicros());
     ++ref_;
   }
   return ptr;
@@ -95,6 +99,7 @@ void TrackingAllocator::DeallocateRaw(void* ptr) {
     if (tracks_allocation_sizes) {
       CHECK_GE(allocated_, allocated_bytes);
       allocated_ -= allocated_bytes;
+      allocations_.emplace_back(-allocated_bytes, Env::Default()->NowMicros());
     }
     should_delete = UnRef();
   }
@@ -151,22 +156,31 @@ void TrackingAllocator::GetStats(AllocatorStats* stats) {
   allocator_->GetStats(stats);
 }
 
-std::tuple<size_t, size_t, size_t> TrackingAllocator::GetSizesAndUnRef() {
+std::tuple<size_t, size_t, size_t> TrackingAllocator::GetSizes() {
   size_t high_watermark;
   size_t total_bytes;
   size_t still_live_bytes;
-  bool should_delete;
   {
     mutex_lock lock(mu_);
     high_watermark = high_watermark_;
     total_bytes = total_bytes_;
     still_live_bytes = allocated_;
+  }
+  return std::make_tuple(total_bytes, high_watermark, still_live_bytes);
+}
+
+gtl::InlinedVector<AllocRecord, 4> TrackingAllocator::GetRecordsAndUnRef() {
+  bool should_delete;
+  gtl::InlinedVector<AllocRecord, 4> allocations;
+  {
+    mutex_lock lock(mu_);
+    allocations.swap(allocations_);
     should_delete = UnRef();
   }
   if (should_delete) {
     delete this;
   }
-  return std::make_tuple(total_bytes, high_watermark, still_live_bytes);
+  return allocations;
 }
 
 bool TrackingAllocator::UnRef() {
diff --git a/tensorflow/core/framework/tracking_allocator.h b/tensorflow/core/framework/tracking_allocator.h
index 92c89d30ac..d10b0cca51 100644
--- a/tensorflow/core/framework/tracking_allocator.h
+++ b/tensorflow/core/framework/tracking_allocator.h
@@ -18,7 +18,9 @@ limitations under the License.
 
 #include <unordered_map>
 #include "tensorflow/core/framework/allocator.h"
+#include "tensorflow/core/framework/step_stats.pb.h"
 #include "tensorflow/core/lib/core/refcount.h"
+#include "tensorflow/core/lib/gtl/inlined_vector.h"
 #include "tensorflow/core/platform/mutex.h"
 #include "tensorflow/core/platform/thread_annotations.h"
 #include "tensorflow/core/platform/types.h"
@@ -42,6 +44,15 @@ namespace tensorflow {
 // TrackingAllocator keeps track of outstanding calls using a
 // reference count, and deletes itself once the last call has been
 // received and the high watermark has been retrieved.
+struct AllocRecord {
+  AllocRecord(int64 a_btyes, int64 a_micros)
+      : alloc_bytes(a_btyes), alloc_micros(a_micros) {}
+  AllocRecord() : AllocRecord(0, 0) {}
+
+  int64 alloc_bytes;
+  int64 alloc_micros;
+};
+
 class TrackingAllocator : public Allocator {
  public:
   explicit TrackingAllocator(Allocator* allocator, bool track_ids);
@@ -67,12 +78,13 @@ class TrackingAllocator : public Allocator {
   // value is the total number of bytes requested through this wrapper
   // and the second and the third are 0.
   //
-  // After GetSizesAndUnref is called, the only further calls allowed
+  std::tuple<size_t, size_t, size_t> GetSizes();
+  // After GetRecordsAndUnRef is called, the only further calls allowed
   // on this wrapper are calls to DeallocateRaw with pointers that
   // were allocated by this wrapper and have not yet been
   // deallocated. After this call completes and all allocated pointers
   // have been deallocated the wrapper will delete itself.
-  std::tuple<size_t, size_t, size_t> GetSizesAndUnRef();
+  gtl::InlinedVector<AllocRecord, 4> GetRecordsAndUnRef();
 
  protected:
   ~TrackingAllocator() override {}
@@ -100,6 +112,8 @@ class TrackingAllocator : public Allocator {
   // this allocator.
   size_t total_bytes_ GUARDED_BY(mu_);
 
+  gtl::InlinedVector<AllocRecord, 4> allocations_ GUARDED_BY(mu_);
+
   // Track allocations locally if requested in the constructor and the
   // underlying allocator doesn't already do it for us.
   const bool track_sizes_locally_;
diff --git a/tensorflow/core/framework/tracking_allocator_test.cc b/tensorflow/core/framework/tracking_allocator_test.cc
index ae440cc28b..4e32a907f2 100644
--- a/tensorflow/core/framework/tracking_allocator_test.cc
+++ b/tensorflow/core/framework/tracking_allocator_test.cc
@@ -75,13 +75,16 @@ TEST(TrackingAllocatorTest, SimpleNoTracking) {
   ta->DeallocateRaw(p1);
   void* p2 = ta->AllocateRaw(4, 12);
 
-  std::tuple<size_t, size_t, size_t> sizes = ta->GetSizesAndUnRef();
+  std::tuple<size_t, size_t, size_t> sizes = ta->GetSizes();
 
   EXPECT_EQ(16, std::get<0>(sizes));
   EXPECT_EQ(0, std::get<1>(sizes));
   EXPECT_EQ(0, std::get<2>(sizes));
 
   ta->DeallocateRaw(p2);
+  auto records = ta->GetRecordsAndUnRef();
+  EXPECT_EQ(4, records[0].alloc_bytes);
+  EXPECT_EQ(12, records[1].alloc_bytes);
 
   // This time enable the tracking inside the tracking allocator
   ta = new TrackingAllocator(a, true);
@@ -96,13 +99,18 @@ TEST(TrackingAllocatorTest, SimpleNoTracking) {
   EXPECT_LE(12, ta->AllocatedSize(p2));
   EXPECT_EQ(2, ta->AllocationId(p2));
 
-  sizes = ta->GetSizesAndUnRef();
+  sizes = ta->GetSizes();
 
   EXPECT_LE(16, std::get<0>(sizes));
   EXPECT_LE(12, std::get<1>(sizes));
   EXPECT_LE(12, std::get<2>(sizes));
 
   ta->DeallocateRaw(p2);
+  records = ta->GetRecordsAndUnRef();
+  EXPECT_LE(4, records[0].alloc_bytes);
+  EXPECT_GE(-4, records[1].alloc_bytes);
+  EXPECT_LE(12, records[2].alloc_bytes);
+  EXPECT_GE(-12, records[3].alloc_bytes);
 }
 
 TEST(TrackingAllocatorTest, SimpleTracking) {
@@ -116,13 +124,19 @@ TEST(TrackingAllocatorTest, SimpleTracking) {
   ta->DeallocateRaw(p1);
   void* p2 = ta->AllocateRaw(4, 4);
 
-  std::tuple<size_t, size_t, size_t> sizes = ta->GetSizesAndUnRef();
+  std::tuple<size_t, size_t, size_t> sizes = ta->GetSizes();
 
   EXPECT_EQ(16, std::get<0>(sizes));
   EXPECT_EQ(12, std::get<1>(sizes));
   EXPECT_EQ(4, std::get<2>(sizes));
 
   ta->DeallocateRaw(p2);
+
+  auto records = ta->GetRecordsAndUnRef();
+  EXPECT_EQ(12, records[0].alloc_bytes);
+  EXPECT_EQ(-12, records[1].alloc_bytes);
+  EXPECT_EQ(4, records[2].alloc_bytes);
+  EXPECT_EQ(-4, records[3].alloc_bytes);
 }
 
 TEST(TrackingAllocatorTest, OutOfMemory) {
@@ -135,11 +149,13 @@ TEST(TrackingAllocatorTest, OutOfMemory) {
   void* p1 = ta->AllocateRaw(4, 12);
   EXPECT_EQ(nullptr, p1);
 
-  std::tuple<size_t, size_t, size_t> sizes = ta->GetSizesAndUnRef();
+  std::tuple<size_t, size_t, size_t> sizes = ta->GetSizes();
 
   EXPECT_EQ(0, std::get<0>(sizes));
   EXPECT_EQ(0, std::get<1>(sizes));
   EXPECT_EQ(0, std::get<2>(sizes));
+
+  EXPECT_EQ(0, ta->GetRecordsAndUnRef().size());
 }
 
 TEST(TrackingAllocatorTest, FreeNullPtr) {
@@ -151,11 +167,13 @@ TEST(TrackingAllocatorTest, FreeNullPtr) {
 
   ta->DeallocateRaw(nullptr);
 
-  std::tuple<size_t, size_t, size_t> sizes = ta->GetSizesAndUnRef();
+  std::tuple<size_t, size_t, size_t> sizes = ta->GetSizes();
 
   EXPECT_EQ(0, std::get<0>(sizes));
   EXPECT_EQ(0, std::get<1>(sizes));
   EXPECT_EQ(0, std::get<2>(sizes));
+
+  EXPECT_EQ(0, ta->GetRecordsAndUnRef().size());
 }
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/platform/gpu_tracer_test.cc b/tensorflow/core/platform/gpu_tracer_test.cc
index f6c2c6cb37..ce2985fd47 100644
--- a/tensorflow/core/platform/gpu_tracer_test.cc
+++ b/tensorflow/core/platform/gpu_tracer_test.cc
@@ -195,6 +195,7 @@ TEST_F(GPUTracerTest, TraceToStepStatsCollector) {
   StepStats stats;
   StepStatsCollector collector(&stats);
   TF_ASSERT_OK(tracer->Collect(&collector));
+  collector.Finalize();
   // Depending on whether this runs on CPU or GPU, we will have a
   // different number of devices.
   EXPECT_GE(stats.dev_stats_size(), 1);
diff --git a/tensorflow/python/profiler/internal/run_metadata_test.py b/tensorflow/python/profiler/internal/run_metadata_test.py
index 80df44f5f5..4ff09d3800 100644
--- a/tensorflow/python/profiler/internal/run_metadata_test.py
+++ b/tensorflow/python/profiler/internal/run_metadata_test.py
@@ -121,6 +121,35 @@ class RunMetadataTest(test.TestCase):
     self.assertEqual(len(ret['gpu:0']), 1)
     self.assertEqual(len(ret['gpu:0/stream:all']), 1, '%s' % run_meta)
 
+  def testAllocationHistory(self):
+    if not test.is_gpu_available(cuda_only=True):
+      return
+
+    gpu_dev = test.gpu_device_name()
+    ops.reset_default_graph()
+    with ops.device(gpu_dev):
+      _, run_meta = _run_model()
+
+    mm = _extract_node(run_meta, 'MatMul')['gpu:0'][0]
+    mm_allocs = mm.memory[0].allocation_records
+    # has allocation and deallocation.
+    self.assertEqual(len(mm_allocs), 2)
+    # first allocated.
+    self.assertGreater(mm_allocs[1].alloc_micros, mm_allocs[0].alloc_micros)
+    self.assertGreater(mm_allocs[0].alloc_bytes, 0)
+    # Then deallocated.
+    self.assertLess(mm_allocs[1].alloc_bytes, 0)
+    # All memory deallocated.
+    self.assertEqual(mm_allocs[0].alloc_bytes + mm_allocs[1].alloc_bytes, 0)
+
+    rand = _extract_node(
+        run_meta, 'random_normal/RandomStandardNormal')['gpu:0'][0]
+    random_allocs = rand.memory[0].allocation_records
+    # random normal must allocated first since matmul depends on it.
+    self.assertLess(random_allocs[0].alloc_micros, mm.all_start_micros)
+    # deallocates the memory after matmul started.
+    self.assertGreater(random_allocs[1].alloc_micros, mm.all_start_micros)
+
   def testCPU(self):
     ops.reset_default_graph()
     with ops.device('/cpu:0'):
-- 
GitLab


From 7bb0592ef2f5ee4ac9261448daf51446cfc19941 Mon Sep 17 00:00:00 2001
From: Michael Case <mikecase@google.com>
Date: Thu, 5 Oct 2017 20:29:46 -0700
Subject: [PATCH 095/909] Remove setting AWS logging for S3 file system.

Was causing issues with tests. Can repro test failures on Macs by running...

bazel test --config=s3  --cache_test_results=no --test_output=streamed
//tensorflow/core/kernels:control_flow_ops_test

Possible reason for error is symbol collision with AWS logging code.
One possible solution would be to split out another shared object for
the S3 filesystem op which does not link in libtensorflow_framework.so.
This is done, for example, by libforestprotos.so in
tensorflow/contrib/tensor_forest/BUILD

PiperOrigin-RevId: 171246381
---
 tensorflow/contrib/s3/s3_file_system.cc | 2 --
 1 file changed, 2 deletions(-)

diff --git a/tensorflow/contrib/s3/s3_file_system.cc b/tensorflow/contrib/s3/s3_file_system.cc
index b09cf81d46..daced83145 100644
--- a/tensorflow/contrib/s3/s3_file_system.cc
+++ b/tensorflow/contrib/s3/s3_file_system.cc
@@ -222,7 +222,6 @@ class S3ReadOnlyMemoryRegion : public ReadOnlyMemoryRegion {
 
 S3FileSystem::S3FileSystem() {
   Aws::SDKOptions options;
-  options.loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Info;
   options.cryptoOptions.sha256Factory_create_fn = []() {
     return Aws::MakeShared<S3SHA256Factory>(S3CryptoAllocationTag);
   };
@@ -234,7 +233,6 @@ S3FileSystem::S3FileSystem() {
 
 S3FileSystem::~S3FileSystem() {
   Aws::SDKOptions options;
-  options.loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Info;
   Aws::ShutdownAPI(options);
 }
 
-- 
GitLab


From 78af510b9aab4094a895851d61e2ea359a9b4985 Mon Sep 17 00:00:00 2001
From: Jacques Pienaar <jpienaar@google.com>
Date: Thu, 5 Oct 2017 20:42:05 -0700
Subject: [PATCH 096/909] Temporarily don't error out if the requested device
 name cannot be parsed.

PiperOrigin-RevId: 171246995
---
 .../compiler/tf2xla/xla_compilation_device.cc | 23 ++++++++-----------
 1 file changed, 10 insertions(+), 13 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/xla_compilation_device.cc b/tensorflow/compiler/tf2xla/xla_compilation_device.cc
index 890a9ccb83..3814a2b8b9 100644
--- a/tensorflow/compiler/tf2xla/xla_compilation_device.cc
+++ b/tensorflow/compiler/tf2xla/xla_compilation_device.cc
@@ -98,20 +98,17 @@ void XlaCompilationDevice::Compute(OpKernel* op_kernel,
   b->SetOpMetadata(metadata);
 
   DeviceNameUtils::ParsedName parsed;
-  OP_REQUIRES(
-      context,
-      DeviceNameUtils::ParseFullName(op_kernel->requested_device(), &parsed),
-      errors::Internal("Unable to parse device name: ",
-                       op_kernel->requested_device()));
-  xla::OpDeviceAssignment assignment;
-  // If no device ID assignment is found, XLA is free to use whatever device it
-  // wants. In practice this usually has the effect of placing things on
-  // device 0.
-  if (parsed.has_id) {
-    assignment.set_has_device(true);
-    assignment.set_device(parsed.id);
+  if (DeviceNameUtils::ParseFullName(op_kernel->requested_device(), &parsed)) {
+    // If no device ID assignment is found, XLA is free to use whatever device
+    // it wants. In practice this usually has the effect of placing things on
+    // device 0.
+    xla::OpDeviceAssignment assignment;
+    if (parsed.has_id) {
+      assignment.set_has_device(true);
+      assignment.set_device(parsed.id);
+    }
+    b->SetDeviceAssignment(assignment);
   }
-  b->SetDeviceAssignment(assignment);
 
   op_kernel->Compute(context);
 
-- 
GitLab


From 825a9f8d9a4cc3cce7cee2fb08dcc058b5a8e2a8 Mon Sep 17 00:00:00 2001
From: Peter Hawkins <phawkins@google.com>
Date: Fri, 6 Oct 2017 05:36:08 -0700
Subject: [PATCH 097/909] [TF:XLA] Make registration of an XlaDevice for
 autoclustering optional.

PiperOrigin-RevId: 171281666
---
 .../compiler/jit/mark_for_compilation_pass.cc |  1 +
 tensorflow/compiler/jit/xla_cpu_device.cc     |  6 +++---
 tensorflow/compiler/jit/xla_device.cc         | 21 +++++++++++--------
 tensorflow/compiler/jit/xla_device.h          |  1 +
 tensorflow/compiler/jit/xla_gpu_device.cc     |  6 +++---
 5 files changed, 20 insertions(+), 15 deletions(-)

diff --git a/tensorflow/compiler/jit/mark_for_compilation_pass.cc b/tensorflow/compiler/jit/mark_for_compilation_pass.cc
index db2ed16f95..78d0aa86a8 100644
--- a/tensorflow/compiler/jit/mark_for_compilation_pass.cc
+++ b/tensorflow/compiler/jit/mark_for_compilation_pass.cc
@@ -560,6 +560,7 @@ Status MarkForCompilationPass::RunImpl(
         name = strings::StrCat("cluster_", cluster_sequence_num++);
       }
       n->AddAttr(kXlaClusterAttr, name);
+      VLOG(3) << "Assigning node " << n->name() << " to cluster " << name;
     }
   }
 
diff --git a/tensorflow/compiler/jit/xla_cpu_device.cc b/tensorflow/compiler/jit/xla_cpu_device.cc
index 57b9d6b56b..2e33fdca65 100644
--- a/tensorflow/compiler/jit/xla_cpu_device.cc
+++ b/tensorflow/compiler/jit/xla_cpu_device.cc
@@ -39,9 +39,9 @@ Status XlaCpuDeviceFactory::CreateDevices(const SessionOptions& options,
   (void)registrations;
 
   std::unique_ptr<XlaDevice> device;
-  TF_RETURN_IF_ERROR(XlaDevice::Create("Host", DEVICE_XLA_CPU, 0,
-                                       DEVICE_CPU_XLA_JIT, options, name_prefix,
-                                       &device));
+  TF_RETURN_IF_ERROR(XlaDevice::Create(
+      "Host", DEVICE_XLA_CPU, 0, DEVICE_CPU_XLA_JIT, options, name_prefix,
+      /*register_device_for_compilation=*/true, &device));
   devices->push_back(device.release());
   return Status::OK();
 }
diff --git a/tensorflow/compiler/jit/xla_device.cc b/tensorflow/compiler/jit/xla_device.cc
index 888461611f..a2c91511ec 100644
--- a/tensorflow/compiler/jit/xla_device.cc
+++ b/tensorflow/compiler/jit/xla_device.cc
@@ -107,18 +107,21 @@ XlaDeviceAllocator* XlaDeviceAllocatorState::GetOrCreateXlaDeviceAllocator(
 /* static */ Status XlaDevice::Create(
     const string& platform_name, const string& device_name, int device_ordinal,
     const string& jit_device_name, const SessionOptions& options,
-    const string& name_prefix, std::unique_ptr<XlaDevice>* device) {
+    const string& name_prefix, bool register_device_for_compilation,
+    std::unique_ptr<XlaDevice>* device) {
   VLOG(1) << "XlaDevice::Create " << platform_name << " " << device_name << ":"
           << device_ordinal;
 
-  // These are no-ops if they have already been done previously for
-  // this device_name/compilation_device_name pair.
-  XlaOpRegistry::DeviceRegistration registration;
-  registration.compilation_device_name = jit_device_name;
-  registration.requires_compilation = true;
-  registration.enable_jit_by_default = false;
-  registration.compile_resource_ops = true;
-  XlaOpRegistry::RegisterCompilationDevice(device_name, registration);
+  if (register_device_for_compilation) {
+    // These are no-ops if they have already been done previously for
+    // this device_name/compilation_device_name pair.
+    XlaOpRegistry::DeviceRegistration registration;
+    registration.compilation_device_name = jit_device_name;
+    registration.requires_compilation = true;
+    registration.enable_jit_by_default = false;
+    registration.compile_resource_ops = true;
+    XlaOpRegistry::RegisterCompilationDevice(device_name, registration);
+  }
 
   auto platform = se::MultiPlatformManager::PlatformWithName(platform_name);
   if (!platform.ok()) {
diff --git a/tensorflow/compiler/jit/xla_device.h b/tensorflow/compiler/jit/xla_device.h
index 0d90b8b692..d2ec38293c 100644
--- a/tensorflow/compiler/jit/xla_device.h
+++ b/tensorflow/compiler/jit/xla_device.h
@@ -74,6 +74,7 @@ class XlaDevice : public LocalDevice {
   static Status Create(const string& platform_name, const string& device_name,
                        int device_ordinal, const string& jit_device_name,
                        const SessionOptions& options, const string& name_prefix,
+                       bool register_device_for_compilation,
                        std::unique_ptr<XlaDevice>* device);
 
   XlaDevice(const SessionOptions& options, const DeviceAttributes& attrs,
diff --git a/tensorflow/compiler/jit/xla_gpu_device.cc b/tensorflow/compiler/jit/xla_gpu_device.cc
index 4474d8f4eb..5233665ec2 100644
--- a/tensorflow/compiler/jit/xla_gpu_device.cc
+++ b/tensorflow/compiler/jit/xla_gpu_device.cc
@@ -39,9 +39,9 @@ Status XlaGpuDeviceFactory::CreateDevices(const SessionOptions& options,
   (void)registrations;
 
   std::unique_ptr<XlaDevice> device;
-  Status status =
-      XlaDevice::Create("CUDA", DEVICE_XLA_GPU, 0, DEVICE_GPU_XLA_JIT, options,
-                        name_prefix, &device);
+  Status status = XlaDevice::Create(
+      "CUDA", DEVICE_XLA_GPU, 0, DEVICE_GPU_XLA_JIT, options, name_prefix,
+      /*register_device_for_compilation=*/true, &device);
   if (!status.ok()) {
     // Treat failures as non-fatal; there might not be a GPU in the machine.
     VLOG(1) << "Failed to create XLA_GPU device: " << status;
-- 
GitLab


From ed2970634444d423261fd7b094084124ccc4f755 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 6 Oct 2017 06:45:38 -0700
Subject: [PATCH 098/909] Include resource variable ops in tensorflow/core:ops
 build target.

PiperOrigin-RevId: 171286346
---
 tensorflow/core/BUILD | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index c1b103c98b..eb66d8e329 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -651,14 +651,15 @@ cc_library(
         ":image_ops_op_lib",
         ":io_ops_op_lib",
         ":linalg_ops_op_lib",
-        ":lookup_ops_op_lib",
         ":logging_ops_op_lib",
+        ":lookup_ops_op_lib",
         ":math_ops_op_lib",
         ":nn_ops_op_lib",
         ":no_op_op_lib",
         ":parsing_ops_op_lib",
         ":random_ops_op_lib",
         ":remote_fused_graph_ops_op_lib",
+        ":resource_variable_ops_op_lib",
         ":script_ops_op_lib",
         ":sdca_ops_op_lib",
         ":sendrecv_ops_op_lib",
-- 
GitLab


From 0cfdb855483d98a8c42f078bae9b00281d05633a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 6 Oct 2017 07:06:26 -0700
Subject: [PATCH 099/909] Update ops-related pbtxt files.

PiperOrigin-RevId: 171288134
---
 .../core/ops/compat/ops_history.v1.pbtxt      | 231 +++++++++++++++
 tensorflow/core/ops/ops.pbtxt                 | 271 ++++++++++++++++++
 2 files changed, 502 insertions(+)

diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index 950422305e..a3321c26f3 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -2061,6 +2061,22 @@ op {
     }
   }
 }
+op {
+  name: "AssignAddVariableOp"
+  input_arg {
+    name: "resource"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "value"
+    type_attr: "dtype"
+  }
+  attr {
+    name: "dtype"
+    type: "type"
+  }
+  is_stateful: true
+}
 op {
   name: "AssignSub"
   input_arg {
@@ -2107,6 +2123,38 @@ op {
     }
   }
 }
+op {
+  name: "AssignSubVariableOp"
+  input_arg {
+    name: "resource"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "value"
+    type_attr: "dtype"
+  }
+  attr {
+    name: "dtype"
+    type: "type"
+  }
+  is_stateful: true
+}
+op {
+  name: "AssignVariableOp"
+  input_arg {
+    name: "resource"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "value"
+    type_attr: "dtype"
+  }
+  attr {
+    name: "dtype"
+    type: "type"
+  }
+  is_stateful: true
+}
 op {
   name: "Atan"
   input_arg {
@@ -7622,6 +7670,21 @@ op {
     type: "type"
   }
 }
+op {
+  name: "DestroyResourceOp"
+  input_arg {
+    name: "resource"
+    type: DT_RESOURCE
+  }
+  attr {
+    name: "ignore_lookup_error"
+    type: "bool"
+    default_value {
+      b: true
+    }
+  }
+  is_stateful: true
+}
 op {
   name: "DestroyTemporaryVariable"
   input_arg {
@@ -20716,6 +20779,22 @@ op {
     type: DT_STRING
   }
 }
+op {
+  name: "ReadVariableOp"
+  input_arg {
+    name: "resource"
+    type: DT_RESOURCE
+  }
+  output_arg {
+    name: "value"
+    type_attr: "dtype"
+  }
+  attr {
+    name: "dtype"
+    type: "type"
+  }
+  is_stateful: true
+}
 op {
   name: "ReaderNumRecordsProduced"
   input_arg {
@@ -22741,6 +22820,91 @@ op {
   }
   is_stateful: true
 }
+op {
+  name: "ResourceGather"
+  input_arg {
+    name: "resource"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "indices"
+    type_attr: "Tindices"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "dtype"
+  }
+  attr {
+    name: "validate_indices"
+    type: "bool"
+    default_value {
+      b: true
+    }
+  }
+  attr {
+    name: "dtype"
+    type: "type"
+  }
+  attr {
+    name: "Tindices"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+  is_stateful: true
+}
+op {
+  name: "ResourceScatterAdd"
+  input_arg {
+    name: "resource"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "indices"
+    type_attr: "Tindices"
+  }
+  input_arg {
+    name: "updates"
+    type_attr: "dtype"
+  }
+  attr {
+    name: "dtype"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
+      }
+    }
+  }
+  attr {
+    name: "Tindices"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+  is_stateful: true
+}
 op {
   name: "ResourceSparseApplyAdadelta"
   input_arg {
@@ -32719,6 +32883,48 @@ op {
   }
   is_stateful: true
 }
+op {
+  name: "VarHandleOp"
+  output_arg {
+    name: "resource"
+    type: DT_RESOURCE
+  }
+  attr {
+    name: "container"
+    type: "string"
+    default_value {
+      s: ""
+    }
+  }
+  attr {
+    name: "shared_name"
+    type: "string"
+    default_value {
+      s: ""
+    }
+  }
+  attr {
+    name: "dtype"
+    type: "type"
+  }
+  attr {
+    name: "shape"
+    type: "shape"
+  }
+  is_stateful: true
+}
+op {
+  name: "VarIsInitializedOp"
+  input_arg {
+    name: "resource"
+    type: DT_RESOURCE
+  }
+  output_arg {
+    name: "is_initialized"
+    type: DT_BOOL
+  }
+  is_stateful: true
+}
 op {
   name: "Variable"
   output_arg {
@@ -32750,6 +32956,31 @@ op {
   }
   is_stateful: true
 }
+op {
+  name: "VariableShape"
+  input_arg {
+    name: "input"
+    type: DT_RESOURCE
+  }
+  output_arg {
+    name: "output"
+    type_attr: "out_type"
+  }
+  attr {
+    name: "out_type"
+    type: "type"
+    default_value {
+      type: DT_INT32
+    }
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+  is_stateful: true
+}
 op {
   name: "VariableV2"
   output_arg {
diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index cbde462325..429000a058 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -2039,6 +2039,27 @@ op {
   summary: "Update \'ref\' by adding \'value\' to it."
   description: "This operation outputs \"ref\" after the update is done.\nThis makes it easier to chain operations that need to use the reset value."
 }
+op {
+  name: "AssignAddVariableOp"
+  input_arg {
+    name: "resource"
+    description: "handle to the resource in which to store the variable."
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "value"
+    description: "the value by which the variable will be incremented."
+    type_attr: "dtype"
+  }
+  attr {
+    name: "dtype"
+    type: "type"
+    description: "the dtype of the value."
+  }
+  summary: "Adds a value to the current value of a variable."
+  description: "Any ReadVariableOp which depends directly or indirectly on this assign is\nguaranteed to see the incremented value or a subsequent newer one.\n\nOutputs the incremented value, which can be used to totally order the\nincrements to this variable."
+  is_stateful: true
+}
 op {
   name: "AssignSub"
   input_arg {
@@ -2091,6 +2112,48 @@ op {
   summary: "Update \'ref\' by subtracting \'value\' from it."
   description: "This operation outputs \"ref\" after the update is done.\nThis makes it easier to chain operations that need to use the reset value."
 }
+op {
+  name: "AssignSubVariableOp"
+  input_arg {
+    name: "resource"
+    description: "handle to the resource in which to store the variable."
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "value"
+    description: "the value by which the variable will be incremented."
+    type_attr: "dtype"
+  }
+  attr {
+    name: "dtype"
+    type: "type"
+    description: "the dtype of the value."
+  }
+  summary: "Subtracts a value from the current value of a variable."
+  description: "Any ReadVariableOp which depends directly or indirectly on this assign is\nguaranteed to see the incremented value or a subsequent newer one.\n\nOutputs the incremented value, which can be used to totally order the\nincrements to this variable."
+  is_stateful: true
+}
+op {
+  name: "AssignVariableOp"
+  input_arg {
+    name: "resource"
+    description: "handle to the resource in which to store the variable."
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "value"
+    description: "the value to set the new tensor to use."
+    type_attr: "dtype"
+  }
+  attr {
+    name: "dtype"
+    type: "type"
+    description: "the dtype of the value."
+  }
+  summary: "Assigns a new value to a variable."
+  description: "Any ReadVariableOp with a control dependency on this op is guaranteed to return\nthis value or a subsequent newer value of the variable."
+  is_stateful: true
+}
 op {
   name: "Atan"
   input_arg {
@@ -6829,6 +6892,25 @@ op {
   summary: "Deserialize and concatenate `SparseTensors` from a serialized minibatch."
   description: "The input `serialized_sparse` must be a string matrix of shape `[N x 3]` where\n`N` is the minibatch size and the rows correspond to packed outputs of\n`SerializeSparse`.  The ranks of the original `SparseTensor` objects\nmust all match.  When the final `SparseTensor` is created, it has rank one\nhigher than the ranks of the incoming `SparseTensor` objects\n(they have been concatenated along a new row dimension).\n\nThe output `SparseTensor` object\'s shape values for all dimensions but the\nfirst are the max across the input `SparseTensor` objects\' shape values\nfor the corresponding dimensions.  Its first shape value is `N`, the minibatch\nsize.\n\nThe input `SparseTensor` objects\' indices are assumed ordered in\nstandard lexicographic order.  If this is not the case, after this\nstep run `SparseReorder` to restore index ordering.\n\nFor example, if the serialized input is a `[2 x 3]` matrix representing two\noriginal `SparseTensor` objects:\n\n    index = [ 0]\n            [10]\n            [20]\n    values = [1, 2, 3]\n    shape = [50]\n\nand\n\n    index = [ 2]\n            [10]\n    values = [4, 5]\n    shape = [30]\n\nthen the final deserialized `SparseTensor` will be:\n\n    index = [0  0]\n            [0 10]\n            [0 20]\n            [1  2]\n            [1 10]\n    values = [1, 2, 3, 4, 5]\n    shape = [2 50]"
 }
+op {
+  name: "DestroyResourceOp"
+  input_arg {
+    name: "resource"
+    description: "handle to the resource to delete."
+    type: DT_RESOURCE
+  }
+  attr {
+    name: "ignore_lookup_error"
+    type: "bool"
+    default_value {
+      b: true
+    }
+    description: "whether to ignore the error when the resource\ndoesn\'t exist."
+  }
+  summary: "Deletes the resource specified by the handle."
+  description: "All subsequent operations using the resource will result in a NotFound\nerror status."
+  is_stateful: true
+}
 op {
   name: "DestroyTemporaryVariable"
   input_arg {
@@ -19351,6 +19433,26 @@ op {
   }
   summary: "Reads and outputs the entire contents of the input filename."
 }
+op {
+  name: "ReadVariableOp"
+  input_arg {
+    name: "resource"
+    description: "handle to the resource in which to store the variable."
+    type: DT_RESOURCE
+  }
+  output_arg {
+    name: "value"
+    type_attr: "dtype"
+  }
+  attr {
+    name: "dtype"
+    type: "type"
+    description: "the dtype of the value."
+  }
+  summary: "Reads the value of a variable."
+  description: "The tensor returned by this operation is immutable.\n\nThe value returned by this operation is guaranteed to be influenced by all the\nwrites on which this operation depends directly or indirectly, and to not be\ninfluenced by any of the writes which depend directly or indirectly on this\noperation."
+  is_stateful: true
+}
 op {
   name: "ReaderNumRecordsProduced"
   input_arg {
@@ -21551,6 +21653,98 @@ op {
   description: "Note that in dense implementation of this algorithm, ms and mom will\nupdate even if the grad is zero, but in this sparse implementation, ms\nand mom will not update in iterations during which the grad is zero.\n\nmean_square = decay * mean_square + (1-decay) * gradient ** 2\nDelta = learning_rate * gradient / sqrt(mean_square + epsilon)\n\nms <- rho * ms_{t-1} + (1-rho) * grad * grad\nmom <- momentum * mom_{t-1} + lr * grad / sqrt(ms + epsilon)\nvar <- var - mom"
   is_stateful: true
 }
+op {
+  name: "ResourceGather"
+  input_arg {
+    name: "resource"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "indices"
+    type_attr: "Tindices"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "dtype"
+  }
+  attr {
+    name: "validate_indices"
+    type: "bool"
+    default_value {
+      b: true
+    }
+  }
+  attr {
+    name: "dtype"
+    type: "type"
+  }
+  attr {
+    name: "Tindices"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+  summary: "Gather slices from the variable pointed to by `resource` according to `indices`."
+  description: "`indices` must be an integer tensor of any dimension (usually 0-D or 1-D).\nProduces an output tensor with shape `indices.shape + params.shape[1:]` where:\n\n```python\n    # Scalar indices\n    output[:, ..., :] = params[indices, :, ... :]\n\n    # Vector indices\n    output[i, :, ..., :] = params[indices[i], :, ... :]\n\n    # Higher rank indices\n    output[i, ..., j, :, ... :] = params[indices[i, ..., j], :, ..., :]\n```"
+  is_stateful: true
+}
+op {
+  name: "ResourceScatterAdd"
+  input_arg {
+    name: "resource"
+    description: "Should be from a `Variable` node."
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "indices"
+    description: "A tensor of indices into the first dimension of `ref`."
+    type_attr: "Tindices"
+  }
+  input_arg {
+    name: "updates"
+    description: "A tensor of updated values to add to `ref`."
+    type_attr: "dtype"
+  }
+  attr {
+    name: "dtype"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
+      }
+    }
+  }
+  attr {
+    name: "Tindices"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+  summary: "Adds sparse updates to the variable referenced by `resource`."
+  description: "This operation computes\n\n    # Scalar indices\n    ref[indices, ...] += updates[...]\n\n    # Vector indices (for each i)\n    ref[indices[i], ...] += updates[i, ...]\n\n    # High rank indices (for each i, ..., j)\n    ref[indices[i, ..., j], ...] += updates[i, ..., j, ...]\n\nDuplicate entries are handled correctly: if multiple `indices` reference\nthe same location, their contributions add.\n\nRequires `updates.shape = indices.shape + ref.shape[1:]`.\n\n<div style=\"width:70%; margin:auto; margin-bottom:10px; margin-top:20px;\">\n<img style=\"width:100%\" src=\"https://www.tensorflow.org/images/ScatterAdd.png\" alt>\n</div>"
+  is_stateful: true
+}
 op {
   name: "ResourceSparseApplyAdadelta"
   input_arg {
@@ -31795,6 +31989,56 @@ op {
   description: "The basic functionality is similar to dequeue with many fewer\ncapabilities and options.  This Op is optimized for performance."
   is_stateful: true
 }
+op {
+  name: "VarHandleOp"
+  output_arg {
+    name: "resource"
+    type: DT_RESOURCE
+  }
+  attr {
+    name: "container"
+    type: "string"
+    default_value {
+      s: ""
+    }
+    description: "the container this variable is placed in."
+  }
+  attr {
+    name: "shared_name"
+    type: "string"
+    default_value {
+      s: ""
+    }
+    description: "the name by which this variable is referred to."
+  }
+  attr {
+    name: "dtype"
+    type: "type"
+    description: "the type of this variable. Must agree with the dtypes\nof all ops using this variable."
+  }
+  attr {
+    name: "shape"
+    type: "shape"
+    description: "The (possibly partially specified) shape of this variable."
+  }
+  summary: "Creates a handle to a Variable resource."
+  is_stateful: true
+}
+op {
+  name: "VarIsInitializedOp"
+  input_arg {
+    name: "resource"
+    description: "the input resource handle."
+    type: DT_RESOURCE
+  }
+  output_arg {
+    name: "is_initialized"
+    description: "a scalar boolean which is true if the variable has been\ninitialized."
+    type: DT_BOOL
+  }
+  summary: "Checks whether a resource handle-based variable has been initialized."
+  is_stateful: true
+}
 op {
   name: "Variable"
   output_arg {
@@ -31827,6 +32071,33 @@ op {
   summary: "Use VariableV2 instead."
   is_stateful: true
 }
+op {
+  name: "VariableShape"
+  input_arg {
+    name: "input"
+    type: DT_RESOURCE
+  }
+  output_arg {
+    name: "output"
+    type_attr: "out_type"
+  }
+  attr {
+    name: "out_type"
+    type: "type"
+    default_value {
+      type: DT_INT32
+    }
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+  summary: "Returns the shape of the variable pointed to by `resource`."
+  description: "This operation returns a 1-D integer tensor representing the shape of `input`.\n\nFor example:\n\n```\n# \'t\' is [[[1, 1, 1], [2, 2, 2]], [[3, 3, 3], [4, 4, 4]]]\nshape(t) ==> [2, 2, 3]\n```"
+  is_stateful: true
+}
 op {
   name: "VariableV2"
   output_arg {
-- 
GitLab


From bbf1085651fab743d17f74dde622c8d89ebbc102 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 6 Oct 2017 07:12:43 -0700
Subject: [PATCH 100/909] Go: Update generated wrapper functions for TensorFlow
 ops.

PiperOrigin-RevId: 171288708
---
 tensorflow/go/op/wrappers.go | 4846 +++++++++++++++++-----------------
 1 file changed, 2423 insertions(+), 2423 deletions(-)

diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go
index ef1f8a9df6..29c69b3c59 100644
--- a/tensorflow/go/op/wrappers.go
+++ b/tensorflow/go/op/wrappers.go
@@ -38,156 +38,6 @@ func makeOutputList(op *tf.Operation, start int, output string) ([]tf.Output, in
 	return list, start + size, nil
 }
 
-// Adds sparse updates to the variable referenced by `resource`.
-//
-// This operation computes
-//
-//     # Scalar indices
-//     ref[indices, ...] += updates[...]
-//
-//     # Vector indices (for each i)
-//     ref[indices[i], ...] += updates[i, ...]
-//
-//     # High rank indices (for each i, ..., j)
-//     ref[indices[i, ..., j], ...] += updates[i, ..., j, ...]
-//
-// Duplicate entries are handled correctly: if multiple `indices` reference
-// the same location, their contributions add.
-//
-// Requires `updates.shape = indices.shape + ref.shape[1:]`.
-//
-// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
-// <img style="width:100%" src="https://www.tensorflow.org/images/ScatterAdd.png" alt>
-// </div>
-//
-// Arguments:
-//	resource: Should be from a `Variable` node.
-//	indices: A tensor of indices into the first dimension of `ref`.
-//	updates: A tensor of updated values to add to `ref`.
-//
-// Returns the created operation.
-func ResourceScatterAdd(scope *Scope, resource tf.Output, indices tf.Output, updates tf.Output) (o *tf.Operation) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "ResourceScatterAdd",
-		Input: []tf.Input{
-			resource, indices, updates,
-		},
-	}
-	return scope.AddOperation(opspec)
-}
-
-// VariableShapeAttr is an optional argument to VariableShape.
-type VariableShapeAttr func(optionalAttr)
-
-// VariableShapeOutType sets the optional out_type attribute to value.
-// If not specified, defaults to DT_INT32
-func VariableShapeOutType(value tf.DataType) VariableShapeAttr {
-	return func(m optionalAttr) {
-		m["out_type"] = value
-	}
-}
-
-// Returns the shape of the variable pointed to by `resource`.
-//
-// This operation returns a 1-D integer tensor representing the shape of `input`.
-//
-// For example:
-//
-// ```
-// # 't' is [[[1, 1, 1], [2, 2, 2]], [[3, 3, 3], [4, 4, 4]]]
-// shape(t) ==> [2, 2, 3]
-// ```
-func VariableShape(scope *Scope, input tf.Output, optional ...VariableShapeAttr) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "VariableShape",
-		Input: []tf.Input{
-			input,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Assigns a new value to a variable.
-//
-// Any ReadVariableOp with a control dependency on this op is guaranteed to return
-// this value or a subsequent newer value of the variable.
-//
-// Arguments:
-//	resource: handle to the resource in which to store the variable.
-//	value: the value to set the new tensor to use.
-//
-// Returns the created operation.
-func AssignVariableOp(scope *Scope, resource tf.Output, value tf.Output) (o *tf.Operation) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "AssignVariableOp",
-		Input: []tf.Input{
-			resource, value,
-		},
-	}
-	return scope.AddOperation(opspec)
-}
-
-// VarHandleOpAttr is an optional argument to VarHandleOp.
-type VarHandleOpAttr func(optionalAttr)
-
-// VarHandleOpContainer sets the optional container attribute to value.
-//
-// value: the container this variable is placed in.
-// If not specified, defaults to ""
-func VarHandleOpContainer(value string) VarHandleOpAttr {
-	return func(m optionalAttr) {
-		m["container"] = value
-	}
-}
-
-// VarHandleOpSharedName sets the optional shared_name attribute to value.
-//
-// value: the name by which this variable is referred to.
-// If not specified, defaults to ""
-func VarHandleOpSharedName(value string) VarHandleOpAttr {
-	return func(m optionalAttr) {
-		m["shared_name"] = value
-	}
-}
-
-// Creates a handle to a Variable resource.
-//
-// Arguments:
-//	dtype: the type of this variable. Must agree with the dtypes
-// of all ops using this variable.
-//	shape: The (possibly partially specified) shape of this variable.
-func VarHandleOp(scope *Scope, dtype tf.DataType, shape tf.Shape, optional ...VarHandleOpAttr) (resource tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"dtype": dtype, "shape": shape}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "VarHandleOp",
-
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
 // Writes a `Summary` protocol buffer with scalar values.
 //
 // The input `tag` and `value` must have the scalars.
@@ -4047,73 +3897,6 @@ func TensorArrayWriteV2(scope *Scope, handle tf.Output, index tf.Output, value t
 	return op.Output(0)
 }
 
-// ResourceGatherAttr is an optional argument to ResourceGather.
-type ResourceGatherAttr func(optionalAttr)
-
-// ResourceGatherValidateIndices sets the optional validate_indices attribute to value.
-// If not specified, defaults to true
-func ResourceGatherValidateIndices(value bool) ResourceGatherAttr {
-	return func(m optionalAttr) {
-		m["validate_indices"] = value
-	}
-}
-
-// Gather slices from the variable pointed to by `resource` according to `indices`.
-//
-// `indices` must be an integer tensor of any dimension (usually 0-D or 1-D).
-// Produces an output tensor with shape `indices.shape + params.shape[1:]` where:
-//
-// ```python
-//     # Scalar indices
-//     output[:, ..., :] = params[indices, :, ... :]
-//
-//     # Vector indices
-//     output[i, :, ..., :] = params[indices[i], :, ... :]
-//
-//     # Higher rank indices
-//     output[i, ..., j, :, ... :] = params[indices[i, ..., j], :, ..., :]
-// ```
-func ResourceGather(scope *Scope, resource tf.Output, indices tf.Output, dtype tf.DataType, optional ...ResourceGatherAttr) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"dtype": dtype}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "ResourceGather",
-		Input: []tf.Input{
-			resource, indices,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Delete the TensorArray from its resource container.
-//
-// This enables the user to close and release the resource in the middle
-// of a step/run.
-//
-// Arguments:
-//	handle: The handle to a TensorArray (output of TensorArray or TensorArrayGrad).
-//
-// Returns the created operation.
-func TensorArrayCloseV3(scope *Scope, handle tf.Output) (o *tf.Operation) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "TensorArrayCloseV3",
-		Input: []tf.Input{
-			handle,
-		},
-	}
-	return scope.AddOperation(opspec)
-}
-
 // Get the current size of the TensorArray.
 //
 // Arguments:
@@ -7697,40 +7480,265 @@ func MergeV2Checkpoints(scope *Scope, checkpoint_prefixes tf.Output, destination
 	return scope.AddOperation(opspec)
 }
 
-// FusedBatchNormGradAttr is an optional argument to FusedBatchNormGrad.
-type FusedBatchNormGradAttr func(optionalAttr)
+// QueueEnqueueManyV2Attr is an optional argument to QueueEnqueueManyV2.
+type QueueEnqueueManyV2Attr func(optionalAttr)
 
-// FusedBatchNormGradEpsilon sets the optional epsilon attribute to value.
+// QueueEnqueueManyV2TimeoutMs sets the optional timeout_ms attribute to value.
 //
-// value: A small float number added to the variance of x.
-// If not specified, defaults to 0.0001
-func FusedBatchNormGradEpsilon(value float32) FusedBatchNormGradAttr {
+// value: If the queue is too full, this operation will block for up
+// to timeout_ms milliseconds.
+// Note: This option is not supported yet.
+// If not specified, defaults to -1
+func QueueEnqueueManyV2TimeoutMs(value int64) QueueEnqueueManyV2Attr {
 	return func(m optionalAttr) {
-		m["epsilon"] = value
+		m["timeout_ms"] = value
 	}
 }
 
-// FusedBatchNormGradDataFormat sets the optional data_format attribute to value.
+// Enqueues zero or more tuples of one or more tensors in the given queue.
 //
-// value: The data format for y_backprop, x, x_backprop.
-// Either "NHWC" (default) or "NCHW".
-// If not specified, defaults to "NHWC"
-func FusedBatchNormGradDataFormat(value string) FusedBatchNormGradAttr {
-	return func(m optionalAttr) {
-		m["data_format"] = value
-	}
-}
-
-// FusedBatchNormGradIsTraining sets the optional is_training attribute to value.
+// This operation slices each component tensor along the 0th dimension to
+// make multiple queue elements. All of the tuple components must have the
+// same size in the 0th dimension.
 //
-// value: A bool value to indicate the operation is for training (default)
-// or inference.
-// If not specified, defaults to true
-func FusedBatchNormGradIsTraining(value bool) FusedBatchNormGradAttr {
-	return func(m optionalAttr) {
-		m["is_training"] = value
-	}
-}
+// The components input has k elements, which correspond to the components of
+// tuples stored in the given queue.
+//
+// N.B. If the queue is full, this operation will block until the given
+// elements have been enqueued (or 'timeout_ms' elapses, if specified).
+//
+// Arguments:
+//	handle: The handle to a queue.
+//	components: One or more tensors from which the enqueued tensors should
+// be taken.
+//
+// Returns the created operation.
+func QueueEnqueueManyV2(scope *Scope, handle tf.Output, components []tf.Output, optional ...QueueEnqueueManyV2Attr) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "QueueEnqueueManyV2",
+		Input: []tf.Input{
+			handle, tf.OutputList(components),
+		},
+		Attrs: attrs,
+	}
+	return scope.AddOperation(opspec)
+}
+
+// SvdAttr is an optional argument to Svd.
+type SvdAttr func(optionalAttr)
+
+// SvdComputeUv sets the optional compute_uv attribute to value.
+//
+// value: If true, left and right singular vectors will be
+// computed and returned in `u` and `v`, respectively.
+// If false, `u` and `v` are not set and should never referenced.
+// If not specified, defaults to true
+func SvdComputeUv(value bool) SvdAttr {
+	return func(m optionalAttr) {
+		m["compute_uv"] = value
+	}
+}
+
+// SvdFullMatrices sets the optional full_matrices attribute to value.
+//
+// value: If true, compute full-sized `u` and `v`. If false
+// (the default), compute only the leading `P` singular vectors.
+// Ignored if `compute_uv` is `False`.
+// If not specified, defaults to false
+func SvdFullMatrices(value bool) SvdAttr {
+	return func(m optionalAttr) {
+		m["full_matrices"] = value
+	}
+}
+
+// Computes the singular value decompositions of one or more matrices.
+//
+// Computes the SVD of each inner matrix in `input` such that
+// `input[..., :, :] = u[..., :, :] * diag(s[..., :, :]) * transpose(v[..., :, :])`
+//
+// ```python
+// # a is a tensor containing a batch of matrices.
+// # s is a tensor of singular values for each matrix.
+// # u is the tensor containing of left singular vectors for each matrix.
+// # v is the tensor containing of right singular vectors for each matrix.
+// s, u, v = svd(a)
+// s, _, _ = svd(a, compute_uv=False)
+// ```
+//
+// Arguments:
+//	input: A tensor of shape `[..., M, N]` whose inner-most 2 dimensions
+// form matrices of size `[M, N]`. Let `P` be the minimum of `M` and `N`.
+//
+// Returns Singular values. Shape is `[..., P]`.Left singular vectors. If `full_matrices` is `False` then shape is
+// `[..., M, P]`; if `full_matrices` is `True` then shape is
+// `[..., M, M]`. Undefined if `compute_uv` is `False`.Left singular vectors. If `full_matrices` is `False` then shape is
+// `[..., N, P]`. If `full_matrices` is `True` then shape is `[..., N, N]`.
+// Undefined if `compute_uv` is false.
+func Svd(scope *Scope, input tf.Output, optional ...SvdAttr) (s tf.Output, u tf.Output, v tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "Svd",
+		Input: []tf.Input{
+			input,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1), op.Output(2)
+}
+
+// Converts one or more images from RGB to HSV.
+//
+// Outputs a tensor of the same shape as the `images` tensor, containing the HSV
+// value of the pixels. The output is only well defined if the value in `images`
+// are in `[0,1]`.
+//
+// `output[..., 0]` contains hue, `output[..., 1]` contains saturation, and
+// `output[..., 2]` contains value. All HSV values are in `[0,1]`. A hue of 0
+// corresponds to pure red, hue 1/3 is pure green, and 2/3 is pure blue.
+//
+// Arguments:
+//	images: 1-D or higher rank. RGB data to convert. Last dimension must be size 3.
+//
+// Returns `images` converted to HSV.
+func RGBToHSV(scope *Scope, images tf.Output) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "RGBToHSV",
+		Input: []tf.Input{
+			images,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// MatrixSolveLsAttr is an optional argument to MatrixSolveLs.
+type MatrixSolveLsAttr func(optionalAttr)
+
+// MatrixSolveLsFast sets the optional fast attribute to value.
+// If not specified, defaults to true
+func MatrixSolveLsFast(value bool) MatrixSolveLsAttr {
+	return func(m optionalAttr) {
+		m["fast"] = value
+	}
+}
+
+// Solves one or more linear least-squares problems.
+//
+// `matrix` is a tensor of shape `[..., M, N]` whose inner-most 2 dimensions
+// form real or complex matrices of size `[M, N]`. `Rhs` is a tensor of the same
+// type as `matrix` and shape `[..., M, K]`.
+// The output is a tensor shape `[..., N, K]` where each output matrix solves
+// each of the equations
+// `matrix[..., :, :]` * `output[..., :, :]` = `rhs[..., :, :]`
+// in the least squares sense.
+//
+// We use the following notation for (complex) matrix and right-hand sides
+// in the batch:
+//
+// `matrix`=\\(A \in \mathbb{C}^{m \times n}\\),
+// `rhs`=\\(B  \in \mathbb{C}^{m \times k}\\),
+// `output`=\\(X  \in \mathbb{C}^{n \times k}\\),
+// `l2_regularizer`=\\(\lambda \in \mathbb{R}\\).
+//
+// If `fast` is `True`, then the solution is computed by solving the normal
+// equations using Cholesky decomposition. Specifically, if \\(m \ge n\\) then
+// \\(X = (A^H A + \lambda I)^{-1} A^H B\\), which solves the least-squares
+// problem \\(X = \mathrm{argmin}_{Z \in \Re^{n \times k} } ||A Z - B||_F^2 +
+// \lambda ||Z||_F^2\\). If \\(m \lt n\\) then `output` is computed as
+// \\(X = A^H (A A^H + \lambda I)^{-1} B\\), which (for \\(\lambda = 0\\)) is the
+// minimum-norm solution to the under-determined linear system, i.e.
+// \\(X = \mathrm{argmin}_{Z \in \mathbb{C}^{n \times k} } ||Z||_F^2 \\),
+// subject to \\(A Z = B\\). Notice that the fast path is only numerically stable
+// when \\(A\\) is numerically full rank and has a condition number
+// \\(\mathrm{cond}(A) \lt \frac{1}{\sqrt{\epsilon_{mach} } }\\) or\\(\lambda\\) is
+// sufficiently large.
+//
+// If `fast` is `False` an algorithm based on the numerically robust complete
+// orthogonal decomposition is used. This computes the minimum-norm
+// least-squares solution, even when \\(A\\) is rank deficient. This path is
+// typically 6-7 times slower than the fast path. If `fast` is `False` then
+// `l2_regularizer` is ignored.
+//
+// Arguments:
+//	matrix: Shape is `[..., M, N]`.
+//	rhs: Shape is `[..., M, K]`.
+//	l2_regularizer: Scalar tensor.
+//
+// @compatibility(numpy)
+// Equivalent to np.linalg.lstsq
+// @end_compatibility
+//
+// Returns Shape is `[..., N, K]`.
+func MatrixSolveLs(scope *Scope, matrix tf.Output, rhs tf.Output, l2_regularizer tf.Output, optional ...MatrixSolveLsAttr) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "MatrixSolveLs",
+		Input: []tf.Input{
+			matrix, rhs, l2_regularizer,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// FusedBatchNormGradAttr is an optional argument to FusedBatchNormGrad.
+type FusedBatchNormGradAttr func(optionalAttr)
+
+// FusedBatchNormGradEpsilon sets the optional epsilon attribute to value.
+//
+// value: A small float number added to the variance of x.
+// If not specified, defaults to 0.0001
+func FusedBatchNormGradEpsilon(value float32) FusedBatchNormGradAttr {
+	return func(m optionalAttr) {
+		m["epsilon"] = value
+	}
+}
+
+// FusedBatchNormGradDataFormat sets the optional data_format attribute to value.
+//
+// value: The data format for y_backprop, x, x_backprop.
+// Either "NHWC" (default) or "NCHW".
+// If not specified, defaults to "NHWC"
+func FusedBatchNormGradDataFormat(value string) FusedBatchNormGradAttr {
+	return func(m optionalAttr) {
+		m["data_format"] = value
+	}
+}
+
+// FusedBatchNormGradIsTraining sets the optional is_training attribute to value.
+//
+// value: A bool value to indicate the operation is for training (default)
+// or inference.
+// If not specified, defaults to true
+func FusedBatchNormGradIsTraining(value bool) FusedBatchNormGradAttr {
+	return func(m optionalAttr) {
+		m["is_training"] = value
+	}
+}
 
 // Gradient for batch normalization.
 //
@@ -9346,41 +9354,12 @@ func DecodeCSV(scope *Scope, records tf.Output, record_defaults []tf.Output, opt
 	return output
 }
 
-// Convert JSON-encoded Example records to binary protocol buffer strings.
-//
-// This op translates a tensor containing Example records, encoded using
-// the [standard JSON
-// mapping](https://developers.google.com/protocol-buffers/docs/proto3#json),
-// into a tensor containing the same records encoded as binary protocol
-// buffers. The resulting tensor can then be fed to any of the other
-// Example-parsing ops.
+// Transforms a serialized tensorflow.TensorProto proto into a Tensor.
 //
 // Arguments:
-//	json_examples: Each string is a JSON object serialized according to the JSON
-// mapping of the Example proto.
-//
-// Returns Each string is a binary Example protocol buffer corresponding
-// to the respective element of `json_examples`.
-func DecodeJSONExample(scope *Scope, json_examples tf.Output) (binary_examples tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "DecodeJSONExample",
-		Input: []tf.Input{
-			json_examples,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Transforms a serialized tensorflow.TensorProto proto into a Tensor.
-//
-// Arguments:
-//	serialized: A scalar string containing a serialized TensorProto proto.
-//	out_type: The type of the serialized tensor.  The provided type must match the
-// type of the serialized tensor and no implicit conversion will take place.
+//	serialized: A scalar string containing a serialized TensorProto proto.
+//	out_type: The type of the serialized tensor.  The provided type must match the
+// type of the serialized tensor and no implicit conversion will take place.
 //
 // Returns A Tensor of type `out_type`.
 func ParseTensor(scope *Scope, serialized tf.Output, out_type tf.DataType) (output tf.Output) {
@@ -9521,172 +9500,157 @@ func DecodeRaw(scope *Scope, bytes tf.Output, out_type tf.DataType, optional ...
 	return op.Output(0)
 }
 
-// QueueDequeueV2Attr is an optional argument to QueueDequeueV2.
-type QueueDequeueV2Attr func(optionalAttr)
+// OrderedMapIncompleteSizeAttr is an optional argument to OrderedMapIncompleteSize.
+type OrderedMapIncompleteSizeAttr func(optionalAttr)
 
-// QueueDequeueV2TimeoutMs sets the optional timeout_ms attribute to value.
+// OrderedMapIncompleteSizeCapacity sets the optional capacity attribute to value.
+// If not specified, defaults to 0
 //
-// value: If the queue is empty, this operation will block for up to
-// timeout_ms milliseconds.
-// Note: This option is not supported yet.
-// If not specified, defaults to -1
-func QueueDequeueV2TimeoutMs(value int64) QueueDequeueV2Attr {
+// REQUIRES: value >= 0
+func OrderedMapIncompleteSizeCapacity(value int64) OrderedMapIncompleteSizeAttr {
 	return func(m optionalAttr) {
-		m["timeout_ms"] = value
+		m["capacity"] = value
 	}
 }
 
-// Dequeues a tuple of one or more tensors from the given queue.
-//
-// This operation has k outputs, where k is the number of components
-// in the tuples stored in the given queue, and output i is the ith
-// component of the dequeued tuple.
-//
-// N.B. If the queue is empty, this operation will block until an element
-// has been dequeued (or 'timeout_ms' elapses, if specified).
-//
-// Arguments:
-//	handle: The handle to a queue.
-//	component_types: The type of each component in a tuple.
+// OrderedMapIncompleteSizeMemoryLimit sets the optional memory_limit attribute to value.
+// If not specified, defaults to 0
 //
-// Returns One or more tensors that were dequeued as a tuple.
-func QueueDequeueV2(scope *Scope, handle tf.Output, component_types []tf.DataType, optional ...QueueDequeueV2Attr) (components []tf.Output) {
+// REQUIRES: value >= 0
+func OrderedMapIncompleteSizeMemoryLimit(value int64) OrderedMapIncompleteSizeAttr {
+	return func(m optionalAttr) {
+		m["memory_limit"] = value
+	}
+}
+
+// OrderedMapIncompleteSizeContainer sets the optional container attribute to value.
+// If not specified, defaults to ""
+func OrderedMapIncompleteSizeContainer(value string) OrderedMapIncompleteSizeAttr {
+	return func(m optionalAttr) {
+		m["container"] = value
+	}
+}
+
+// OrderedMapIncompleteSizeSharedName sets the optional shared_name attribute to value.
+// If not specified, defaults to ""
+func OrderedMapIncompleteSizeSharedName(value string) OrderedMapIncompleteSizeAttr {
+	return func(m optionalAttr) {
+		m["shared_name"] = value
+	}
+}
+
+// Op returns the number of incomplete elements in the underlying container.
+func OrderedMapIncompleteSize(scope *Scope, dtypes []tf.DataType, optional ...OrderedMapIncompleteSizeAttr) (size tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"component_types": component_types}
+	attrs := map[string]interface{}{"dtypes": dtypes}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "QueueDequeueV2",
-		Input: []tf.Input{
-			handle,
-		},
+		Type: "OrderedMapIncompleteSize",
+
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	if scope.Err() != nil {
-		return
-	}
-	var idx int
-	var err error
-	if components, idx, err = makeOutputList(op, idx, "components"); err != nil {
-		scope.UpdateErr("QueueDequeueV2", err)
-		return
-	}
-	return components
+	return op.Output(0)
 }
 
-// ParseSingleSequenceExampleAttr is an optional argument to ParseSingleSequenceExample.
-type ParseSingleSequenceExampleAttr func(optionalAttr)
+// RandomShuffleAttr is an optional argument to RandomShuffle.
+type RandomShuffleAttr func(optionalAttr)
 
-// ParseSingleSequenceExampleContextSparseTypes sets the optional context_sparse_types attribute to value.
-//
-// value: A list of Ncontext_sparse types; the data types of data in
-// each context Feature given in context_sparse_keys.
-// Currently the ParseSingleSequenceExample supports DT_FLOAT (FloatList),
-// DT_INT64 (Int64List), and DT_STRING (BytesList).
-// If not specified, defaults to <>
+// RandomShuffleSeed sets the optional seed attribute to value.
 //
-// REQUIRES: len(value) >= 0
-func ParseSingleSequenceExampleContextSparseTypes(value []tf.DataType) ParseSingleSequenceExampleAttr {
+// value: If either `seed` or `seed2` are set to be non-zero, the random number
+// generator is seeded by the given seed.  Otherwise, it is seeded by a
+// random seed.
+// If not specified, defaults to 0
+func RandomShuffleSeed(value int64) RandomShuffleAttr {
 	return func(m optionalAttr) {
-		m["context_sparse_types"] = value
+		m["seed"] = value
 	}
 }
 
-// ParseSingleSequenceExampleFeatureListDenseTypes sets the optional feature_list_dense_types attribute to value.
-// If not specified, defaults to <>
+// RandomShuffleSeed2 sets the optional seed2 attribute to value.
 //
-// REQUIRES: len(value) >= 0
-func ParseSingleSequenceExampleFeatureListDenseTypes(value []tf.DataType) ParseSingleSequenceExampleAttr {
+// value: A second seed to avoid seed collision.
+// If not specified, defaults to 0
+func RandomShuffleSeed2(value int64) RandomShuffleAttr {
 	return func(m optionalAttr) {
-		m["feature_list_dense_types"] = value
+		m["seed2"] = value
 	}
 }
 
-// ParseSingleSequenceExampleContextDenseShapes sets the optional context_dense_shapes attribute to value.
+// Randomly shuffles a tensor along its first dimension.
 //
-// value: A list of Ncontext_dense shapes; the shapes of data in
-// each context Feature given in context_dense_keys.
-// The number of elements in the Feature corresponding to context_dense_key[j]
-// must always equal context_dense_shapes[j].NumEntries().
-// The shape of context_dense_values[j] will match context_dense_shapes[j].
-// If not specified, defaults to <>
+//   The tensor is shuffled along dimension 0, such that each `value[j]` is mapped
+//   to one and only one `output[i]`. For example, a mapping that might occur for a
+//   3x2 tensor is:
 //
-// REQUIRES: len(value) >= 0
-func ParseSingleSequenceExampleContextDenseShapes(value []tf.Shape) ParseSingleSequenceExampleAttr {
-	return func(m optionalAttr) {
-		m["context_dense_shapes"] = value
+// ```
+// [[1, 2],       [[5, 6],
+//  [3, 4],  ==>   [1, 2],
+//  [5, 6]]        [3, 4]]
+// ```
+//
+// Arguments:
+//	value: The tensor to be shuffled.
+//
+// Returns A tensor of same shape and type as `value`, shuffled along its first
+// dimension.
+func RandomShuffle(scope *Scope, value tf.Output, optional ...RandomShuffleAttr) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "RandomShuffle",
+		Input: []tf.Input{
+			value,
+		},
+		Attrs: attrs,
 	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// ParseSingleSequenceExampleFeatureListSparseTypes sets the optional feature_list_sparse_types attribute to value.
-//
-// value: A list of Nfeature_list_sparse types; the data types
-// of data in each FeatureList given in feature_list_sparse_keys.
-// Currently the ParseSingleSequenceExample supports DT_FLOAT (FloatList),
-// DT_INT64 (Int64List), and DT_STRING (BytesList).
-// If not specified, defaults to <>
-//
-// REQUIRES: len(value) >= 0
-func ParseSingleSequenceExampleFeatureListSparseTypes(value []tf.DataType) ParseSingleSequenceExampleAttr {
+// FakeQuantWithMinMaxVarsPerChannelAttr is an optional argument to FakeQuantWithMinMaxVarsPerChannel.
+type FakeQuantWithMinMaxVarsPerChannelAttr func(optionalAttr)
+
+// FakeQuantWithMinMaxVarsPerChannelNumBits sets the optional num_bits attribute to value.
+// If not specified, defaults to 8
+func FakeQuantWithMinMaxVarsPerChannelNumBits(value int64) FakeQuantWithMinMaxVarsPerChannelAttr {
 	return func(m optionalAttr) {
-		m["feature_list_sparse_types"] = value
+		m["num_bits"] = value
 	}
 }
 
-// ParseSingleSequenceExampleFeatureListDenseShapes sets the optional feature_list_dense_shapes attribute to value.
-//
-// value: A list of Nfeature_list_dense shapes; the shapes of
-// data in each FeatureList given in feature_list_dense_keys.
-// The shape of each Feature in the FeatureList corresponding to
-// feature_list_dense_key[j] must always equal
-// feature_list_dense_shapes[j].NumEntries().
-// If not specified, defaults to <>
-//
-// REQUIRES: len(value) >= 0
-func ParseSingleSequenceExampleFeatureListDenseShapes(value []tf.Shape) ParseSingleSequenceExampleAttr {
+// FakeQuantWithMinMaxVarsPerChannelNarrowRange sets the optional narrow_range attribute to value.
+// If not specified, defaults to false
+func FakeQuantWithMinMaxVarsPerChannelNarrowRange(value bool) FakeQuantWithMinMaxVarsPerChannelAttr {
 	return func(m optionalAttr) {
-		m["feature_list_dense_shapes"] = value
+		m["narrow_range"] = value
 	}
 }
 
-// Transforms a scalar brain.SequenceExample proto (as strings) into typed tensors.
+// Fake-quantize the 'inputs' tensor of type float and one of the shapes: `[d]`,
 //
-// Arguments:
-//	serialized: A scalar containing a binary serialized SequenceExample proto.
-//	feature_list_dense_missing_assumed_empty: A vector listing the
-// FeatureList keys which may be missing from the SequenceExample.  If the
-// associated FeatureList is missing, it is treated as empty.  By default,
-// any FeatureList not listed in this vector must exist in the SequenceExample.
-//	context_sparse_keys: A list of Ncontext_sparse string Tensors (scalars).
-// The keys expected in the Examples' features associated with context_sparse
-// values.
-//	context_dense_keys: A list of Ncontext_dense string Tensors (scalars).
-// The keys expected in the SequenceExamples' context features associated with
-// dense values.
-//	feature_list_sparse_keys: A list of Nfeature_list_sparse string Tensors
-// (scalars).  The keys expected in the FeatureLists associated with sparse
+// `[b, d]` `[b, h, w, d]` via per-channel floats `min` and `max` of shape `[d]`
+// to 'outputs' tensor of same shape as `inputs`.
+//
+// `[min; max]` define the clamping range for the `inputs` data.
+// `inputs` values are quantized into the quantization range (`[0; 2^num_bits - 1]`
+// when `narrow_range` is false and `[1; 2^num_bits - 1]` when it is true) and
+// then de-quantized and output as floats in `[min; max]` interval.
+// `num_bits` is the bitwidth of the quantization; between 2 and 8, inclusive.
+//
+// This operation has a gradient and thus allows for training `min` and `max`
 // values.
-//	feature_list_dense_keys: A list of Nfeature_list_dense string Tensors (scalars).
-// The keys expected in the SequenceExamples' feature_lists associated
-// with lists of dense values.
-//	context_dense_defaults: A list of Ncontext_dense Tensors (some may be empty).
-// context_dense_defaults[j] provides default values
-// when the SequenceExample's context map lacks context_dense_key[j].
-// If an empty Tensor is provided for context_dense_defaults[j],
-// then the Feature context_dense_keys[j] is required.
-// The input type is inferred from context_dense_defaults[j], even when it's
-// empty.  If context_dense_defaults[j] is not empty, its shape must match
-// context_dense_shapes[j].
-//	debug_name: A scalar containing the name of the serialized proto.
-// May contain, for example, table key (descriptive) name for the
-// corresponding serialized proto.  This is purely useful for debugging
-// purposes, and the presence of values here has no effect on the output.
-// May also be an empty scalar if no name is available.
-func ParseSingleSequenceExample(scope *Scope, serialized tf.Output, feature_list_dense_missing_assumed_empty tf.Output, context_sparse_keys []tf.Output, context_dense_keys []tf.Output, feature_list_sparse_keys []tf.Output, feature_list_dense_keys []tf.Output, context_dense_defaults []tf.Output, debug_name tf.Output, optional ...ParseSingleSequenceExampleAttr) (context_sparse_indices []tf.Output, context_sparse_values []tf.Output, context_sparse_shapes []tf.Output, context_dense_values []tf.Output, feature_list_sparse_indices []tf.Output, feature_list_sparse_values []tf.Output, feature_list_sparse_shapes []tf.Output, feature_list_dense_values []tf.Output) {
+func FakeQuantWithMinMaxVarsPerChannel(scope *Scope, inputs tf.Output, min tf.Output, max tf.Output, optional ...FakeQuantWithMinMaxVarsPerChannelAttr) (outputs tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -9695,105 +9659,65 @@ func ParseSingleSequenceExample(scope *Scope, serialized tf.Output, feature_list
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "ParseSingleSequenceExample",
+		Type: "FakeQuantWithMinMaxVarsPerChannel",
 		Input: []tf.Input{
-			serialized, feature_list_dense_missing_assumed_empty, tf.OutputList(context_sparse_keys), tf.OutputList(context_dense_keys), tf.OutputList(feature_list_sparse_keys), tf.OutputList(feature_list_dense_keys), tf.OutputList(context_dense_defaults), debug_name,
+			inputs, min, max,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	if scope.Err() != nil {
-		return
-	}
-	var idx int
-	var err error
-	if context_sparse_indices, idx, err = makeOutputList(op, idx, "context_sparse_indices"); err != nil {
-		scope.UpdateErr("ParseSingleSequenceExample", err)
-		return
-	}
-	if context_sparse_values, idx, err = makeOutputList(op, idx, "context_sparse_values"); err != nil {
-		scope.UpdateErr("ParseSingleSequenceExample", err)
-		return
-	}
-	if context_sparse_shapes, idx, err = makeOutputList(op, idx, "context_sparse_shapes"); err != nil {
-		scope.UpdateErr("ParseSingleSequenceExample", err)
-		return
-	}
-	if context_dense_values, idx, err = makeOutputList(op, idx, "context_dense_values"); err != nil {
-		scope.UpdateErr("ParseSingleSequenceExample", err)
-		return
-	}
-	if feature_list_sparse_indices, idx, err = makeOutputList(op, idx, "feature_list_sparse_indices"); err != nil {
-		scope.UpdateErr("ParseSingleSequenceExample", err)
-		return
-	}
-	if feature_list_sparse_values, idx, err = makeOutputList(op, idx, "feature_list_sparse_values"); err != nil {
-		scope.UpdateErr("ParseSingleSequenceExample", err)
-		return
-	}
-	if feature_list_sparse_shapes, idx, err = makeOutputList(op, idx, "feature_list_sparse_shapes"); err != nil {
-		scope.UpdateErr("ParseSingleSequenceExample", err)
-		return
-	}
-	if feature_list_dense_values, idx, err = makeOutputList(op, idx, "feature_list_dense_values"); err != nil {
-		scope.UpdateErr("ParseSingleSequenceExample", err)
-		return
-	}
-	return context_sparse_indices, context_sparse_values, context_sparse_shapes, context_dense_values, feature_list_sparse_indices, feature_list_sparse_values, feature_list_sparse_shapes, feature_list_dense_values
+	return op.Output(0)
 }
 
-// RandomGammaAttr is an optional argument to RandomGamma.
-type RandomGammaAttr func(optionalAttr)
+// TruncatedNormalAttr is an optional argument to TruncatedNormal.
+type TruncatedNormalAttr func(optionalAttr)
 
-// RandomGammaSeed sets the optional seed attribute to value.
+// TruncatedNormalSeed sets the optional seed attribute to value.
 //
 // value: If either `seed` or `seed2` are set to be non-zero, the random number
 // generator is seeded by the given seed.  Otherwise, it is seeded by a
 // random seed.
 // If not specified, defaults to 0
-func RandomGammaSeed(value int64) RandomGammaAttr {
+func TruncatedNormalSeed(value int64) TruncatedNormalAttr {
 	return func(m optionalAttr) {
 		m["seed"] = value
 	}
 }
 
-// RandomGammaSeed2 sets the optional seed2 attribute to value.
+// TruncatedNormalSeed2 sets the optional seed2 attribute to value.
 //
 // value: A second seed to avoid seed collision.
 // If not specified, defaults to 0
-func RandomGammaSeed2(value int64) RandomGammaAttr {
+func TruncatedNormalSeed2(value int64) TruncatedNormalAttr {
 	return func(m optionalAttr) {
 		m["seed2"] = value
 	}
 }
 
-// Outputs random values from the Gamma distribution(s) described by alpha.
+// Outputs random values from a truncated normal distribution.
 //
-// This op uses the algorithm by Marsaglia et al. to acquire samples via
-// transformation-rejection from pairs of uniform and normal random variables.
-// See http://dl.acm.org/citation.cfm?id=358414
+// The generated values follow a normal distribution with mean 0 and standard
+// deviation 1, except that values whose magnitude is more than 2 standard
+// deviations from the mean are dropped and re-picked.
 //
 // Arguments:
-//	shape: 1-D integer tensor. Shape of independent samples to draw from each
-// distribution described by the shape parameters given in alpha.
-//	alpha: A tensor in which each scalar is a "shape" parameter describing the
-// associated gamma distribution.
+//	shape: The shape of the output tensor.
+//	dtype: The type of the output.
 //
-// Returns A tensor with shape `shape + shape(alpha)`. Each slice
-// `[:, ..., :, i0, i1, ...iN]` contains the samples drawn for
-// `alpha[i0, i1, ...iN]`. The dtype of the output matches the dtype of alpha.
-func RandomGamma(scope *Scope, shape tf.Output, alpha tf.Output, optional ...RandomGammaAttr) (output tf.Output) {
+// Returns A tensor of the specified shape filled with random truncated normal
+// values.
+func TruncatedNormal(scope *Scope, shape tf.Output, dtype tf.DataType, optional ...TruncatedNormalAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
+	attrs := map[string]interface{}{"dtype": dtype}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "RandomGamma",
+		Type: "TruncatedNormal",
 		Input: []tf.Input{
-			shape, alpha,
+			shape,
 		},
 		Attrs: attrs,
 	}
@@ -9801,106 +9725,163 @@ func RandomGamma(scope *Scope, shape tf.Output, alpha tf.Output, optional ...Ran
 	return op.Output(0)
 }
 
-// OrderedMapIncompleteSizeAttr is an optional argument to OrderedMapIncompleteSize.
-type OrderedMapIncompleteSizeAttr func(optionalAttr)
+// ResourceApplyFtrlV2Attr is an optional argument to ResourceApplyFtrlV2.
+type ResourceApplyFtrlV2Attr func(optionalAttr)
 
-// OrderedMapIncompleteSizeCapacity sets the optional capacity attribute to value.
-// If not specified, defaults to 0
+// ResourceApplyFtrlV2UseLocking sets the optional use_locking attribute to value.
 //
-// REQUIRES: value >= 0
-func OrderedMapIncompleteSizeCapacity(value int64) OrderedMapIncompleteSizeAttr {
+// value: If `True`, updating of the var and accum tensors will be protected
+// by a lock; otherwise the behavior is undefined, but may exhibit less
+// contention.
+// If not specified, defaults to false
+func ResourceApplyFtrlV2UseLocking(value bool) ResourceApplyFtrlV2Attr {
 	return func(m optionalAttr) {
-		m["capacity"] = value
+		m["use_locking"] = value
 	}
 }
 
-// OrderedMapIncompleteSizeMemoryLimit sets the optional memory_limit attribute to value.
-// If not specified, defaults to 0
+// Update '*var' according to the Ftrl-proximal scheme.
 //
-// REQUIRES: value >= 0
-func OrderedMapIncompleteSizeMemoryLimit(value int64) OrderedMapIncompleteSizeAttr {
+// grad_with_shrinkage = grad + 2 * l2_shrinkage * var
+// accum_new = accum + grad_with_shrinkage * grad_with_shrinkage
+// linear += grad_with_shrinkage +
+//     (accum_new^(-lr_power) - accum^(-lr_power)) / lr * var
+// quadratic = 1.0 / (accum_new^(lr_power) * lr) + 2 * l2
+// var = (sign(linear) * l1 - linear) / quadratic if |linear| > l1 else 0.0
+// accum = accum_new
+//
+// Arguments:
+//	var_: Should be from a Variable().
+//	accum: Should be from a Variable().
+//	linear: Should be from a Variable().
+//	grad: The gradient.
+//	lr: Scaling factor. Must be a scalar.
+//	l1: L1 regulariation. Must be a scalar.
+//	l2: L2 shrinkage regulariation. Must be a scalar.
+//
+//	lr_power: Scaling factor. Must be a scalar.
+//
+// Returns the created operation.
+func ResourceApplyFtrlV2(scope *Scope, var_ tf.Output, accum tf.Output, linear tf.Output, grad tf.Output, lr tf.Output, l1 tf.Output, l2 tf.Output, l2_shrinkage tf.Output, lr_power tf.Output, optional ...ResourceApplyFtrlV2Attr) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "ResourceApplyFtrlV2",
+		Input: []tf.Input{
+			var_, accum, linear, grad, lr, l1, l2, l2_shrinkage, lr_power,
+		},
+		Attrs: attrs,
+	}
+	return scope.AddOperation(opspec)
+}
+
+// SkipgramAttr is an optional argument to Skipgram.
+type SkipgramAttr func(optionalAttr)
+
+// SkipgramWindowSize sets the optional window_size attribute to value.
+//
+// value: The number of words to predict to the left and right of the target.
+// If not specified, defaults to 5
+func SkipgramWindowSize(value int64) SkipgramAttr {
 	return func(m optionalAttr) {
-		m["memory_limit"] = value
+		m["window_size"] = value
 	}
 }
 
-// OrderedMapIncompleteSizeContainer sets the optional container attribute to value.
-// If not specified, defaults to ""
-func OrderedMapIncompleteSizeContainer(value string) OrderedMapIncompleteSizeAttr {
+// SkipgramMinCount sets the optional min_count attribute to value.
+//
+// value: The minimum number of word occurrences for it to be included in the
+// vocabulary.
+// If not specified, defaults to 5
+func SkipgramMinCount(value int64) SkipgramAttr {
 	return func(m optionalAttr) {
-		m["container"] = value
+		m["min_count"] = value
 	}
 }
 
-// OrderedMapIncompleteSizeSharedName sets the optional shared_name attribute to value.
-// If not specified, defaults to ""
-func OrderedMapIncompleteSizeSharedName(value string) OrderedMapIncompleteSizeAttr {
+// SkipgramSubsample sets the optional subsample attribute to value.
+//
+// value: Threshold for word occurrence. Words that appear with higher
+// frequency will be randomly down-sampled. Set to 0 to disable.
+// If not specified, defaults to 0.001
+func SkipgramSubsample(value float32) SkipgramAttr {
 	return func(m optionalAttr) {
-		m["shared_name"] = value
+		m["subsample"] = value
 	}
 }
 
-// Op returns the number of incomplete elements in the underlying container.
-func OrderedMapIncompleteSize(scope *Scope, dtypes []tf.DataType, optional ...OrderedMapIncompleteSizeAttr) (size tf.Output) {
+// Parses a text file and creates a batch of examples.
+//
+// DEPRECATED at GraphDef version 19: Moving word2vec into tensorflow_models/tutorials and deprecating its ops here as a result
+//
+// Arguments:
+//	filename: The corpus's text file name.
+//	batch_size: The size of produced batch.
+//
+// Returns A vector of words in the corpus.Frequencies of words. Sorted in the non-ascending order.Number of words per epoch in the data file.The current epoch number.The total number of words processed so far.A vector of word ids.A vector of word ids.
+func Skipgram(scope *Scope, filename string, batch_size int64, optional ...SkipgramAttr) (vocab_word tf.Output, vocab_freq tf.Output, words_per_epoch tf.Output, current_epoch tf.Output, total_words_processed tf.Output, examples tf.Output, labels tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"dtypes": dtypes}
+	attrs := map[string]interface{}{"filename": filename, "batch_size": batch_size}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "OrderedMapIncompleteSize",
+		Type: "Skipgram",
 
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return op.Output(0), op.Output(1), op.Output(2), op.Output(3), op.Output(4), op.Output(5), op.Output(6)
 }
 
-// RandomShuffleAttr is an optional argument to RandomShuffle.
-type RandomShuffleAttr func(optionalAttr)
+// ParameterizedTruncatedNormalAttr is an optional argument to ParameterizedTruncatedNormal.
+type ParameterizedTruncatedNormalAttr func(optionalAttr)
 
-// RandomShuffleSeed sets the optional seed attribute to value.
+// ParameterizedTruncatedNormalSeed sets the optional seed attribute to value.
 //
 // value: If either `seed` or `seed2` are set to be non-zero, the random number
 // generator is seeded by the given seed.  Otherwise, it is seeded by a
 // random seed.
 // If not specified, defaults to 0
-func RandomShuffleSeed(value int64) RandomShuffleAttr {
+func ParameterizedTruncatedNormalSeed(value int64) ParameterizedTruncatedNormalAttr {
 	return func(m optionalAttr) {
 		m["seed"] = value
 	}
 }
 
-// RandomShuffleSeed2 sets the optional seed2 attribute to value.
+// ParameterizedTruncatedNormalSeed2 sets the optional seed2 attribute to value.
 //
 // value: A second seed to avoid seed collision.
 // If not specified, defaults to 0
-func RandomShuffleSeed2(value int64) RandomShuffleAttr {
+func ParameterizedTruncatedNormalSeed2(value int64) ParameterizedTruncatedNormalAttr {
 	return func(m optionalAttr) {
 		m["seed2"] = value
 	}
 }
 
-// Randomly shuffles a tensor along its first dimension.
-//
-//   The tensor is shuffled along dimension 0, such that each `value[j]` is mapped
-//   to one and only one `output[i]`. For example, a mapping that might occur for a
-//   3x2 tensor is:
+// Outputs random values from a normal distribution. The parameters may each be a
 //
-// ```
-// [[1, 2],       [[5, 6],
-//  [3, 4],  ==>   [1, 2],
-//  [5, 6]]        [3, 4]]
-// ```
+// scalar which applies to the entire output, or a vector of length shape[0] which
+// stores the parameters for each batch.
 //
 // Arguments:
-//	value: The tensor to be shuffled.
+//	shape: The shape of the output tensor. Batches are indexed by the 0th dimension.
+//	means: The mean parameter of each batch.
+//	stdevs: The standard deviation parameter of each batch. Must be greater than 0.
+//	minvals: The minimum cutoff. May be -infinity.
+//	maxvals: The maximum cutoff. May be +infinity, and must be more than the minval
+// for each batch.
 //
-// Returns A tensor of same shape and type as `value`, shuffled along its first
-// dimension.
-func RandomShuffle(scope *Scope, value tf.Output, optional ...RandomShuffleAttr) (output tf.Output) {
+// Returns A matrix of shape num_batches x samples_per_batch, filled with random
+// truncated normal values using the parameters for each row.
+func ParameterizedTruncatedNormal(scope *Scope, shape tf.Output, means tf.Output, stdevs tf.Output, minvals tf.Output, maxvals tf.Output, optional ...ParameterizedTruncatedNormalAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -9909,9 +9890,9 @@ func RandomShuffle(scope *Scope, value tf.Output, optional ...RandomShuffleAttr)
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "RandomShuffle",
+		Type: "ParameterizedTruncatedNormal",
 		Input: []tf.Input{
-			value,
+			shape, means, stdevs, minvals, maxvals,
 		},
 		Attrs: attrs,
 	}
@@ -9919,39 +9900,48 @@ func RandomShuffle(scope *Scope, value tf.Output, optional ...RandomShuffleAttr)
 	return op.Output(0)
 }
 
-// FakeQuantWithMinMaxVarsPerChannelAttr is an optional argument to FakeQuantWithMinMaxVarsPerChannel.
-type FakeQuantWithMinMaxVarsPerChannelAttr func(optionalAttr)
+// RandomUniformIntAttr is an optional argument to RandomUniformInt.
+type RandomUniformIntAttr func(optionalAttr)
 
-// FakeQuantWithMinMaxVarsPerChannelNumBits sets the optional num_bits attribute to value.
-// If not specified, defaults to 8
-func FakeQuantWithMinMaxVarsPerChannelNumBits(value int64) FakeQuantWithMinMaxVarsPerChannelAttr {
+// RandomUniformIntSeed sets the optional seed attribute to value.
+//
+// value: If either `seed` or `seed2` are set to be non-zero, the random number
+// generator is seeded by the given seed.  Otherwise, it is seeded by a
+// random seed.
+// If not specified, defaults to 0
+func RandomUniformIntSeed(value int64) RandomUniformIntAttr {
 	return func(m optionalAttr) {
-		m["num_bits"] = value
+		m["seed"] = value
 	}
 }
 
-// FakeQuantWithMinMaxVarsPerChannelNarrowRange sets the optional narrow_range attribute to value.
-// If not specified, defaults to false
-func FakeQuantWithMinMaxVarsPerChannelNarrowRange(value bool) FakeQuantWithMinMaxVarsPerChannelAttr {
+// RandomUniformIntSeed2 sets the optional seed2 attribute to value.
+//
+// value: A second seed to avoid seed collision.
+// If not specified, defaults to 0
+func RandomUniformIntSeed2(value int64) RandomUniformIntAttr {
 	return func(m optionalAttr) {
-		m["narrow_range"] = value
+		m["seed2"] = value
 	}
 }
 
-// Fake-quantize the 'inputs' tensor of type float and one of the shapes: `[d]`,
+// Outputs random integers from a uniform distribution.
 //
-// `[b, d]` `[b, h, w, d]` via per-channel floats `min` and `max` of shape `[d]`
-// to 'outputs' tensor of same shape as `inputs`.
+// The generated values are uniform integers in the range `[minval, maxval)`.
+// The lower bound `minval` is included in the range, while the upper bound
+// `maxval` is excluded.
 //
-// `[min; max]` define the clamping range for the `inputs` data.
-// `inputs` values are quantized into the quantization range (`[0; 2^num_bits - 1]`
-// when `narrow_range` is false and `[1; 2^num_bits - 1]` when it is true) and
-// then de-quantized and output as floats in `[min; max]` interval.
-// `num_bits` is the bitwidth of the quantization; between 2 and 8, inclusive.
+// The random integers are slightly biased unless `maxval - minval` is an exact
+// power of two.  The bias is small for values of `maxval - minval` significantly
+// smaller than the range of the output (either `2^32` or `2^64`).
 //
-// This operation has a gradient and thus allows for training `min` and `max`
-// values.
-func FakeQuantWithMinMaxVarsPerChannel(scope *Scope, inputs tf.Output, min tf.Output, max tf.Output, optional ...FakeQuantWithMinMaxVarsPerChannelAttr) (outputs tf.Output) {
+// Arguments:
+//	shape: The shape of the output tensor.
+//	minval: 0-D.  Inclusive lower bound on the generated integers.
+//	maxval: 0-D.  Exclusive upper bound on the generated integers.
+//
+// Returns A tensor of the specified shape filled with uniform random integers.
+func RandomUniformInt(scope *Scope, shape tf.Output, minval tf.Output, maxval tf.Output, optional ...RandomUniformIntAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -9960,9 +9950,9 @@ func FakeQuantWithMinMaxVarsPerChannel(scope *Scope, inputs tf.Output, min tf.Ou
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "FakeQuantWithMinMaxVarsPerChannel",
+		Type: "RandomUniformInt",
 		Input: []tf.Input{
-			inputs, min, max,
+			shape, minval, maxval,
 		},
 		Attrs: attrs,
 	}
@@ -9970,269 +9960,358 @@ func FakeQuantWithMinMaxVarsPerChannel(scope *Scope, inputs tf.Output, min tf.Ou
 	return op.Output(0)
 }
 
-// TruncatedNormalAttr is an optional argument to TruncatedNormal.
-type TruncatedNormalAttr func(optionalAttr)
-
-// TruncatedNormalSeed sets the optional seed attribute to value.
+// Convert JSON-encoded Example records to binary protocol buffer strings.
 //
-// value: If either `seed` or `seed2` are set to be non-zero, the random number
-// generator is seeded by the given seed.  Otherwise, it is seeded by a
-// random seed.
-// If not specified, defaults to 0
-func TruncatedNormalSeed(value int64) TruncatedNormalAttr {
-	return func(m optionalAttr) {
-		m["seed"] = value
+// This op translates a tensor containing Example records, encoded using
+// the [standard JSON
+// mapping](https://developers.google.com/protocol-buffers/docs/proto3#json),
+// into a tensor containing the same records encoded as binary protocol
+// buffers. The resulting tensor can then be fed to any of the other
+// Example-parsing ops.
+//
+// Arguments:
+//	json_examples: Each string is a JSON object serialized according to the JSON
+// mapping of the Example proto.
+//
+// Returns Each string is a binary Example protocol buffer corresponding
+// to the respective element of `json_examples`.
+func DecodeJSONExample(scope *Scope, json_examples tf.Output) (binary_examples tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "DecodeJSONExample",
+		Input: []tf.Input{
+			json_examples,
+		},
 	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// TruncatedNormalSeed2 sets the optional seed2 attribute to value.
+// Adds sparse updates to the variable referenced by `resource`.
 //
-// value: A second seed to avoid seed collision.
-// If not specified, defaults to 0
-func TruncatedNormalSeed2(value int64) TruncatedNormalAttr {
-	return func(m optionalAttr) {
-		m["seed2"] = value
+// This operation computes
+//
+//     # Scalar indices
+//     ref[indices, ...] += updates[...]
+//
+//     # Vector indices (for each i)
+//     ref[indices[i], ...] += updates[i, ...]
+//
+//     # High rank indices (for each i, ..., j)
+//     ref[indices[i, ..., j], ...] += updates[i, ..., j, ...]
+//
+// Duplicate entries are handled correctly: if multiple `indices` reference
+// the same location, their contributions add.
+//
+// Requires `updates.shape = indices.shape + ref.shape[1:]`.
+//
+// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
+// <img style="width:100%" src="https://www.tensorflow.org/images/ScatterAdd.png" alt>
+// </div>
+//
+// Arguments:
+//	resource: Should be from a `Variable` node.
+//	indices: A tensor of indices into the first dimension of `ref`.
+//	updates: A tensor of updated values to add to `ref`.
+//
+// Returns the created operation.
+func ResourceScatterAdd(scope *Scope, resource tf.Output, indices tf.Output, updates tf.Output) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "ResourceScatterAdd",
+		Input: []tf.Input{
+			resource, indices, updates,
+		},
 	}
+	return scope.AddOperation(opspec)
 }
 
-// Outputs random values from a truncated normal distribution.
+// Delete the TensorArray from its resource container.
 //
-// The generated values follow a normal distribution with mean 0 and standard
-// deviation 1, except that values whose magnitude is more than 2 standard
-// deviations from the mean are dropped and re-picked.
+// This enables the user to close and release the resource in the middle
+// of a step/run.
 //
 // Arguments:
-//	shape: The shape of the output tensor.
-//	dtype: The type of the output.
+//	handle: The handle to a TensorArray (output of TensorArray or TensorArrayGrad).
 //
-// Returns A tensor of the specified shape filled with random truncated normal
-// values.
-func TruncatedNormal(scope *Scope, shape tf.Output, dtype tf.DataType, optional ...TruncatedNormalAttr) (output tf.Output) {
+// Returns the created operation.
+func TensorArrayCloseV3(scope *Scope, handle tf.Output) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"dtype": dtype}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "TruncatedNormal",
+		Type: "TensorArrayCloseV3",
 		Input: []tf.Input{
-			shape,
+			handle,
 		},
-		Attrs: attrs,
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return scope.AddOperation(opspec)
 }
 
-// ResourceApplyFtrlV2Attr is an optional argument to ResourceApplyFtrlV2.
-type ResourceApplyFtrlV2Attr func(optionalAttr)
+// ResourceGatherAttr is an optional argument to ResourceGather.
+type ResourceGatherAttr func(optionalAttr)
 
-// ResourceApplyFtrlV2UseLocking sets the optional use_locking attribute to value.
-//
-// value: If `True`, updating of the var and accum tensors will be protected
-// by a lock; otherwise the behavior is undefined, but may exhibit less
-// contention.
-// If not specified, defaults to false
-func ResourceApplyFtrlV2UseLocking(value bool) ResourceApplyFtrlV2Attr {
+// ResourceGatherValidateIndices sets the optional validate_indices attribute to value.
+// If not specified, defaults to true
+func ResourceGatherValidateIndices(value bool) ResourceGatherAttr {
 	return func(m optionalAttr) {
-		m["use_locking"] = value
+		m["validate_indices"] = value
 	}
 }
 
-// Update '*var' according to the Ftrl-proximal scheme.
+// Gather slices from the variable pointed to by `resource` according to `indices`.
 //
-// grad_with_shrinkage = grad + 2 * l2_shrinkage * var
-// accum_new = accum + grad_with_shrinkage * grad_with_shrinkage
-// linear += grad_with_shrinkage +
-//     (accum_new^(-lr_power) - accum^(-lr_power)) / lr * var
-// quadratic = 1.0 / (accum_new^(lr_power) * lr) + 2 * l2
-// var = (sign(linear) * l1 - linear) / quadratic if |linear| > l1 else 0.0
-// accum = accum_new
+// `indices` must be an integer tensor of any dimension (usually 0-D or 1-D).
+// Produces an output tensor with shape `indices.shape + params.shape[1:]` where:
 //
-// Arguments:
-//	var_: Should be from a Variable().
-//	accum: Should be from a Variable().
-//	linear: Should be from a Variable().
-//	grad: The gradient.
-//	lr: Scaling factor. Must be a scalar.
-//	l1: L1 regulariation. Must be a scalar.
-//	l2: L2 shrinkage regulariation. Must be a scalar.
+// ```python
+//     # Scalar indices
+//     output[:, ..., :] = params[indices, :, ... :]
 //
-//	lr_power: Scaling factor. Must be a scalar.
+//     # Vector indices
+//     output[i, :, ..., :] = params[indices[i], :, ... :]
 //
-// Returns the created operation.
-func ResourceApplyFtrlV2(scope *Scope, var_ tf.Output, accum tf.Output, linear tf.Output, grad tf.Output, lr tf.Output, l1 tf.Output, l2 tf.Output, l2_shrinkage tf.Output, lr_power tf.Output, optional ...ResourceApplyFtrlV2Attr) (o *tf.Operation) {
+//     # Higher rank indices
+//     output[i, ..., j, :, ... :] = params[indices[i, ..., j], :, ..., :]
+// ```
+func ResourceGather(scope *Scope, resource tf.Output, indices tf.Output, dtype tf.DataType, optional ...ResourceGatherAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
+	attrs := map[string]interface{}{"dtype": dtype}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "ResourceApplyFtrlV2",
+		Type: "ResourceGather",
 		Input: []tf.Input{
-			var_, accum, linear, grad, lr, l1, l2, l2_shrinkage, lr_power,
+			resource, indices,
 		},
 		Attrs: attrs,
 	}
-	return scope.AddOperation(opspec)
-}
-
-// SkipgramAttr is an optional argument to Skipgram.
-type SkipgramAttr func(optionalAttr)
-
-// SkipgramWindowSize sets the optional window_size attribute to value.
-//
-// value: The number of words to predict to the left and right of the target.
-// If not specified, defaults to 5
-func SkipgramWindowSize(value int64) SkipgramAttr {
-	return func(m optionalAttr) {
-		m["window_size"] = value
-	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// SkipgramMinCount sets the optional min_count attribute to value.
-//
-// value: The minimum number of word occurrences for it to be included in the
-// vocabulary.
-// If not specified, defaults to 5
-func SkipgramMinCount(value int64) SkipgramAttr {
-	return func(m optionalAttr) {
-		m["min_count"] = value
-	}
-}
+// QuantizedConv2DAttr is an optional argument to QuantizedConv2D.
+type QuantizedConv2DAttr func(optionalAttr)
 
-// SkipgramSubsample sets the optional subsample attribute to value.
-//
-// value: Threshold for word occurrence. Words that appear with higher
-// frequency will be randomly down-sampled. Set to 0 to disable.
-// If not specified, defaults to 0.001
-func SkipgramSubsample(value float32) SkipgramAttr {
+// QuantizedConv2DOutType sets the optional out_type attribute to value.
+// If not specified, defaults to DT_QINT32
+func QuantizedConv2DOutType(value tf.DataType) QuantizedConv2DAttr {
 	return func(m optionalAttr) {
-		m["subsample"] = value
+		m["out_type"] = value
 	}
 }
 
-// Parses a text file and creates a batch of examples.
+// Computes a 2D convolution given quantized 4D input and filter tensors.
 //
-// DEPRECATED at GraphDef version 19: Moving word2vec into tensorflow_models/tutorials and deprecating its ops here as a result
+// The inputs are quantized tensors where the lowest value represents the real
+// number of the associated minimum, and the highest represents the maximum.
+// This means that you can only interpret the quantized output in the same way, by
+// taking the returned minimum and maximum values into account.
 //
 // Arguments:
-//	filename: The corpus's text file name.
-//	batch_size: The size of produced batch.
 //
-// Returns A vector of words in the corpus.Frequencies of words. Sorted in the non-ascending order.Number of words per epoch in the data file.The current epoch number.The total number of words processed so far.A vector of word ids.A vector of word ids.
-func Skipgram(scope *Scope, filename string, batch_size int64, optional ...SkipgramAttr) (vocab_word tf.Output, vocab_freq tf.Output, words_per_epoch tf.Output, current_epoch tf.Output, total_words_processed tf.Output, examples tf.Output, labels tf.Output) {
+//	filter: filter's input_depth dimension must match input's depth dimensions.
+//	min_input: The float value that the lowest quantized input value represents.
+//	max_input: The float value that the highest quantized input value represents.
+//	min_filter: The float value that the lowest quantized filter value represents.
+//	max_filter: The float value that the highest quantized filter value represents.
+//	strides: The stride of the sliding window for each dimension of the input
+// tensor.
+//	padding: The type of padding algorithm to use.
+//
+// Returns The float value that the lowest quantized output value represents.The float value that the highest quantized output value represents.
+func QuantizedConv2D(scope *Scope, input tf.Output, filter tf.Output, min_input tf.Output, max_input tf.Output, min_filter tf.Output, max_filter tf.Output, strides []int64, padding string, optional ...QuantizedConv2DAttr) (output tf.Output, min_output tf.Output, max_output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"filename": filename, "batch_size": batch_size}
+	attrs := map[string]interface{}{"strides": strides, "padding": padding}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "Skipgram",
-
+		Type: "QuantizedConv2D",
+		Input: []tf.Input{
+			input, filter, min_input, max_input, min_filter, max_filter,
+		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2), op.Output(3), op.Output(4), op.Output(5), op.Output(6)
+	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// ParameterizedTruncatedNormalAttr is an optional argument to ParameterizedTruncatedNormal.
-type ParameterizedTruncatedNormalAttr func(optionalAttr)
+// QueueDequeueV2Attr is an optional argument to QueueDequeueV2.
+type QueueDequeueV2Attr func(optionalAttr)
 
-// ParameterizedTruncatedNormalSeed sets the optional seed attribute to value.
+// QueueDequeueV2TimeoutMs sets the optional timeout_ms attribute to value.
 //
-// value: If either `seed` or `seed2` are set to be non-zero, the random number
-// generator is seeded by the given seed.  Otherwise, it is seeded by a
-// random seed.
-// If not specified, defaults to 0
-func ParameterizedTruncatedNormalSeed(value int64) ParameterizedTruncatedNormalAttr {
+// value: If the queue is empty, this operation will block for up to
+// timeout_ms milliseconds.
+// Note: This option is not supported yet.
+// If not specified, defaults to -1
+func QueueDequeueV2TimeoutMs(value int64) QueueDequeueV2Attr {
 	return func(m optionalAttr) {
-		m["seed"] = value
+		m["timeout_ms"] = value
 	}
 }
 
-// ParameterizedTruncatedNormalSeed2 sets the optional seed2 attribute to value.
+// Dequeues a tuple of one or more tensors from the given queue.
 //
-// value: A second seed to avoid seed collision.
-// If not specified, defaults to 0
-func ParameterizedTruncatedNormalSeed2(value int64) ParameterizedTruncatedNormalAttr {
-	return func(m optionalAttr) {
-		m["seed2"] = value
-	}
-}
-
-// Outputs random values from a normal distribution. The parameters may each be a
+// This operation has k outputs, where k is the number of components
+// in the tuples stored in the given queue, and output i is the ith
+// component of the dequeued tuple.
 //
-// scalar which applies to the entire output, or a vector of length shape[0] which
-// stores the parameters for each batch.
+// N.B. If the queue is empty, this operation will block until an element
+// has been dequeued (or 'timeout_ms' elapses, if specified).
 //
 // Arguments:
-//	shape: The shape of the output tensor. Batches are indexed by the 0th dimension.
-//	means: The mean parameter of each batch.
-//	stdevs: The standard deviation parameter of each batch. Must be greater than 0.
-//	minvals: The minimum cutoff. May be -infinity.
-//	maxvals: The maximum cutoff. May be +infinity, and must be more than the minval
-// for each batch.
+//	handle: The handle to a queue.
+//	component_types: The type of each component in a tuple.
 //
-// Returns A matrix of shape num_batches x samples_per_batch, filled with random
-// truncated normal values using the parameters for each row.
-func ParameterizedTruncatedNormal(scope *Scope, shape tf.Output, means tf.Output, stdevs tf.Output, minvals tf.Output, maxvals tf.Output, optional ...ParameterizedTruncatedNormalAttr) (output tf.Output) {
+// Returns One or more tensors that were dequeued as a tuple.
+func QueueDequeueV2(scope *Scope, handle tf.Output, component_types []tf.DataType, optional ...QueueDequeueV2Attr) (components []tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
+	attrs := map[string]interface{}{"component_types": component_types}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "ParameterizedTruncatedNormal",
+		Type: "QueueDequeueV2",
 		Input: []tf.Input{
-			shape, means, stdevs, minvals, maxvals,
+			handle,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	if scope.Err() != nil {
+		return
+	}
+	var idx int
+	var err error
+	if components, idx, err = makeOutputList(op, idx, "components"); err != nil {
+		scope.UpdateErr("QueueDequeueV2", err)
+		return
+	}
+	return components
 }
 
-// EncodePngAttr is an optional argument to EncodePng.
-type EncodePngAttr func(optionalAttr)
+// ParseSingleSequenceExampleAttr is an optional argument to ParseSingleSequenceExample.
+type ParseSingleSequenceExampleAttr func(optionalAttr)
 
-// EncodePngCompression sets the optional compression attribute to value.
+// ParseSingleSequenceExampleContextSparseTypes sets the optional context_sparse_types attribute to value.
 //
-// value: Compression level.
-// If not specified, defaults to -1
-func EncodePngCompression(value int64) EncodePngAttr {
+// value: A list of Ncontext_sparse types; the data types of data in
+// each context Feature given in context_sparse_keys.
+// Currently the ParseSingleSequenceExample supports DT_FLOAT (FloatList),
+// DT_INT64 (Int64List), and DT_STRING (BytesList).
+// If not specified, defaults to <>
+//
+// REQUIRES: len(value) >= 0
+func ParseSingleSequenceExampleContextSparseTypes(value []tf.DataType) ParseSingleSequenceExampleAttr {
 	return func(m optionalAttr) {
-		m["compression"] = value
+		m["context_sparse_types"] = value
 	}
 }
 
-// PNG-encode an image.
+// ParseSingleSequenceExampleFeatureListDenseTypes sets the optional feature_list_dense_types attribute to value.
+// If not specified, defaults to <>
 //
-// `image` is a 3-D uint8 or uint16 Tensor of shape `[height, width, channels]`
-// where `channels` is:
+// REQUIRES: len(value) >= 0
+func ParseSingleSequenceExampleFeatureListDenseTypes(value []tf.DataType) ParseSingleSequenceExampleAttr {
+	return func(m optionalAttr) {
+		m["feature_list_dense_types"] = value
+	}
+}
+
+// ParseSingleSequenceExampleContextDenseShapes sets the optional context_dense_shapes attribute to value.
 //
-// *   1: for grayscale.
-// *   2: for grayscale + alpha.
-// *   3: for RGB.
-// *   4: for RGBA.
+// value: A list of Ncontext_dense shapes; the shapes of data in
+// each context Feature given in context_dense_keys.
+// The number of elements in the Feature corresponding to context_dense_key[j]
+// must always equal context_dense_shapes[j].NumEntries().
+// The shape of context_dense_values[j] will match context_dense_shapes[j].
+// If not specified, defaults to <>
 //
-// The ZLIB compression level, `compression`, can be -1 for the PNG-encoder
-// default or a value from 0 to 9.  9 is the highest compression level, generating
-// the smallest output, but is slower.
+// REQUIRES: len(value) >= 0
+func ParseSingleSequenceExampleContextDenseShapes(value []tf.Shape) ParseSingleSequenceExampleAttr {
+	return func(m optionalAttr) {
+		m["context_dense_shapes"] = value
+	}
+}
+
+// ParseSingleSequenceExampleFeatureListSparseTypes sets the optional feature_list_sparse_types attribute to value.
 //
-// Arguments:
-//	image: 3-D with shape `[height, width, channels]`.
+// value: A list of Nfeature_list_sparse types; the data types
+// of data in each FeatureList given in feature_list_sparse_keys.
+// Currently the ParseSingleSequenceExample supports DT_FLOAT (FloatList),
+// DT_INT64 (Int64List), and DT_STRING (BytesList).
+// If not specified, defaults to <>
 //
-// Returns 0-D. PNG-encoded image.
-func EncodePng(scope *Scope, image tf.Output, optional ...EncodePngAttr) (contents tf.Output) {
+// REQUIRES: len(value) >= 0
+func ParseSingleSequenceExampleFeatureListSparseTypes(value []tf.DataType) ParseSingleSequenceExampleAttr {
+	return func(m optionalAttr) {
+		m["feature_list_sparse_types"] = value
+	}
+}
+
+// ParseSingleSequenceExampleFeatureListDenseShapes sets the optional feature_list_dense_shapes attribute to value.
+//
+// value: A list of Nfeature_list_dense shapes; the shapes of
+// data in each FeatureList given in feature_list_dense_keys.
+// The shape of each Feature in the FeatureList corresponding to
+// feature_list_dense_key[j] must always equal
+// feature_list_dense_shapes[j].NumEntries().
+// If not specified, defaults to <>
+//
+// REQUIRES: len(value) >= 0
+func ParseSingleSequenceExampleFeatureListDenseShapes(value []tf.Shape) ParseSingleSequenceExampleAttr {
+	return func(m optionalAttr) {
+		m["feature_list_dense_shapes"] = value
+	}
+}
+
+// Transforms a scalar brain.SequenceExample proto (as strings) into typed tensors.
+//
+// Arguments:
+//	serialized: A scalar containing a binary serialized SequenceExample proto.
+//	feature_list_dense_missing_assumed_empty: A vector listing the
+// FeatureList keys which may be missing from the SequenceExample.  If the
+// associated FeatureList is missing, it is treated as empty.  By default,
+// any FeatureList not listed in this vector must exist in the SequenceExample.
+//	context_sparse_keys: A list of Ncontext_sparse string Tensors (scalars).
+// The keys expected in the Examples' features associated with context_sparse
+// values.
+//	context_dense_keys: A list of Ncontext_dense string Tensors (scalars).
+// The keys expected in the SequenceExamples' context features associated with
+// dense values.
+//	feature_list_sparse_keys: A list of Nfeature_list_sparse string Tensors
+// (scalars).  The keys expected in the FeatureLists associated with sparse
+// values.
+//	feature_list_dense_keys: A list of Nfeature_list_dense string Tensors (scalars).
+// The keys expected in the SequenceExamples' feature_lists associated
+// with lists of dense values.
+//	context_dense_defaults: A list of Ncontext_dense Tensors (some may be empty).
+// context_dense_defaults[j] provides default values
+// when the SequenceExample's context map lacks context_dense_key[j].
+// If an empty Tensor is provided for context_dense_defaults[j],
+// then the Feature context_dense_keys[j] is required.
+// The input type is inferred from context_dense_defaults[j], even when it's
+// empty.  If context_dense_defaults[j] is not empty, its shape must match
+// context_dense_shapes[j].
+//	debug_name: A scalar containing the name of the serialized proto.
+// May contain, for example, table key (descriptive) name for the
+// corresponding serialized proto.  This is purely useful for debugging
+// purposes, and the presence of values here has no effect on the output.
+// May also be an empty scalar if no name is available.
+func ParseSingleSequenceExample(scope *Scope, serialized tf.Output, feature_list_dense_missing_assumed_empty tf.Output, context_sparse_keys []tf.Output, context_dense_keys []tf.Output, feature_list_sparse_keys []tf.Output, feature_list_dense_keys []tf.Output, context_dense_defaults []tf.Output, debug_name tf.Output, optional ...ParseSingleSequenceExampleAttr) (context_sparse_indices []tf.Output, context_sparse_values []tf.Output, context_sparse_shapes []tf.Output, context_dense_values []tf.Output, feature_list_sparse_indices []tf.Output, feature_list_sparse_values []tf.Output, feature_list_sparse_shapes []tf.Output, feature_list_dense_values []tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -10241,58 +10320,94 @@ func EncodePng(scope *Scope, image tf.Output, optional ...EncodePngAttr) (conten
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "EncodePng",
+		Type: "ParseSingleSequenceExample",
 		Input: []tf.Input{
-			image,
+			serialized, feature_list_dense_missing_assumed_empty, tf.OutputList(context_sparse_keys), tf.OutputList(context_dense_keys), tf.OutputList(feature_list_sparse_keys), tf.OutputList(feature_list_dense_keys), tf.OutputList(context_dense_defaults), debug_name,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	if scope.Err() != nil {
+		return
+	}
+	var idx int
+	var err error
+	if context_sparse_indices, idx, err = makeOutputList(op, idx, "context_sparse_indices"); err != nil {
+		scope.UpdateErr("ParseSingleSequenceExample", err)
+		return
+	}
+	if context_sparse_values, idx, err = makeOutputList(op, idx, "context_sparse_values"); err != nil {
+		scope.UpdateErr("ParseSingleSequenceExample", err)
+		return
+	}
+	if context_sparse_shapes, idx, err = makeOutputList(op, idx, "context_sparse_shapes"); err != nil {
+		scope.UpdateErr("ParseSingleSequenceExample", err)
+		return
+	}
+	if context_dense_values, idx, err = makeOutputList(op, idx, "context_dense_values"); err != nil {
+		scope.UpdateErr("ParseSingleSequenceExample", err)
+		return
+	}
+	if feature_list_sparse_indices, idx, err = makeOutputList(op, idx, "feature_list_sparse_indices"); err != nil {
+		scope.UpdateErr("ParseSingleSequenceExample", err)
+		return
+	}
+	if feature_list_sparse_values, idx, err = makeOutputList(op, idx, "feature_list_sparse_values"); err != nil {
+		scope.UpdateErr("ParseSingleSequenceExample", err)
+		return
+	}
+	if feature_list_sparse_shapes, idx, err = makeOutputList(op, idx, "feature_list_sparse_shapes"); err != nil {
+		scope.UpdateErr("ParseSingleSequenceExample", err)
+		return
+	}
+	if feature_list_dense_values, idx, err = makeOutputList(op, idx, "feature_list_dense_values"); err != nil {
+		scope.UpdateErr("ParseSingleSequenceExample", err)
+		return
+	}
+	return context_sparse_indices, context_sparse_values, context_sparse_shapes, context_dense_values, feature_list_sparse_indices, feature_list_sparse_values, feature_list_sparse_shapes, feature_list_dense_values
 }
 
-// RandomUniformIntAttr is an optional argument to RandomUniformInt.
-type RandomUniformIntAttr func(optionalAttr)
+// RandomGammaAttr is an optional argument to RandomGamma.
+type RandomGammaAttr func(optionalAttr)
 
-// RandomUniformIntSeed sets the optional seed attribute to value.
+// RandomGammaSeed sets the optional seed attribute to value.
 //
 // value: If either `seed` or `seed2` are set to be non-zero, the random number
 // generator is seeded by the given seed.  Otherwise, it is seeded by a
 // random seed.
 // If not specified, defaults to 0
-func RandomUniformIntSeed(value int64) RandomUniformIntAttr {
+func RandomGammaSeed(value int64) RandomGammaAttr {
 	return func(m optionalAttr) {
 		m["seed"] = value
 	}
 }
 
-// RandomUniformIntSeed2 sets the optional seed2 attribute to value.
+// RandomGammaSeed2 sets the optional seed2 attribute to value.
 //
 // value: A second seed to avoid seed collision.
 // If not specified, defaults to 0
-func RandomUniformIntSeed2(value int64) RandomUniformIntAttr {
+func RandomGammaSeed2(value int64) RandomGammaAttr {
 	return func(m optionalAttr) {
 		m["seed2"] = value
 	}
 }
 
-// Outputs random integers from a uniform distribution.
-//
-// The generated values are uniform integers in the range `[minval, maxval)`.
-// The lower bound `minval` is included in the range, while the upper bound
-// `maxval` is excluded.
+// Outputs random values from the Gamma distribution(s) described by alpha.
 //
-// The random integers are slightly biased unless `maxval - minval` is an exact
-// power of two.  The bias is small for values of `maxval - minval` significantly
-// smaller than the range of the output (either `2^32` or `2^64`).
+// This op uses the algorithm by Marsaglia et al. to acquire samples via
+// transformation-rejection from pairs of uniform and normal random variables.
+// See http://dl.acm.org/citation.cfm?id=358414
 //
 // Arguments:
-//	shape: The shape of the output tensor.
-//	minval: 0-D.  Inclusive lower bound on the generated integers.
-//	maxval: 0-D.  Exclusive upper bound on the generated integers.
+//	shape: 1-D integer tensor. Shape of independent samples to draw from each
+// distribution described by the shape parameters given in alpha.
+//	alpha: A tensor in which each scalar is a "shape" parameter describing the
+// associated gamma distribution.
 //
-// Returns A tensor of the specified shape filled with uniform random integers.
-func RandomUniformInt(scope *Scope, shape tf.Output, minval tf.Output, maxval tf.Output, optional ...RandomUniformIntAttr) (output tf.Output) {
+// Returns A tensor with shape `shape + shape(alpha)`. Each slice
+// `[:, ..., :, i0, i1, ...iN]` contains the samples drawn for
+// `alpha[i0, i1, ...iN]`. The dtype of the output matches the dtype of alpha.
+func RandomGamma(scope *Scope, shape tf.Output, alpha tf.Output, optional ...RandomGammaAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -10301,9 +10416,150 @@ func RandomUniformInt(scope *Scope, shape tf.Output, minval tf.Output, maxval tf
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "RandomUniformInt",
+		Type: "RandomGamma",
 		Input: []tf.Input{
-			shape, minval, maxval,
+			shape, alpha,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Computes the gradient for the inverse of `x` wrt its input.
+//
+// Specifically, `grad = -dy * y*y`, where `y = 1/x`, and `dy`
+// is the corresponding input gradient.
+func ReciprocalGrad(scope *Scope, y tf.Output, dy tf.Output) (z tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "ReciprocalGrad",
+		Input: []tf.Input{
+			y, dy,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Creates a dataset with a range of values. Corresponds to python's xrange.
+//
+// Arguments:
+//	start: corresponds to start in python's xrange().
+//	stop: corresponds to stop in python's xrange().
+//	step: corresponds to step in python's xrange().
+//
+//
+func RangeDataset(scope *Scope, start tf.Output, stop tf.Output, step tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
+	opspec := tf.OpSpec{
+		Type: "RangeDataset",
+		Input: []tf.Input{
+			start, stop, step,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Saves tensors in V2 checkpoint format.
+//
+// By default, saves the named tensors in full.  If the caller wishes to save
+// specific slices of full tensors, "shape_and_slices" should be non-empty strings
+// and correspondingly well-formed.
+//
+// Arguments:
+//	prefix: Must have a single element. The prefix of the V2 checkpoint to which we
+// write the tensors.
+//	tensor_names: shape {N}. The names of the tensors to be saved.
+//	shape_and_slices: shape {N}.  The slice specs of the tensors to be saved.
+// Empty strings indicate that they are non-partitioned tensors.
+//	tensors: `N` tensors to save.
+//
+// Returns the created operation.
+func SaveV2(scope *Scope, prefix tf.Output, tensor_names tf.Output, shape_and_slices tf.Output, tensors []tf.Output) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "SaveV2",
+		Input: []tf.Input{
+			prefix, tensor_names, shape_and_slices, tf.OutputList(tensors),
+		},
+	}
+	return scope.AddOperation(opspec)
+}
+
+// MatrixTriangularSolveAttr is an optional argument to MatrixTriangularSolve.
+type MatrixTriangularSolveAttr func(optionalAttr)
+
+// MatrixTriangularSolveLower sets the optional lower attribute to value.
+//
+// value: Boolean indicating whether the innermost matrices in `matrix` are
+// lower or upper triangular.
+// If not specified, defaults to true
+func MatrixTriangularSolveLower(value bool) MatrixTriangularSolveAttr {
+	return func(m optionalAttr) {
+		m["lower"] = value
+	}
+}
+
+// MatrixTriangularSolveAdjoint sets the optional adjoint attribute to value.
+//
+// value: Boolean indicating whether to solve with `matrix` or its (block-wise)
+//          adjoint.
+//
+// @compatibility(numpy)
+// Equivalent to np.linalg.triangular_solve
+// @end_compatibility
+// If not specified, defaults to false
+func MatrixTriangularSolveAdjoint(value bool) MatrixTriangularSolveAttr {
+	return func(m optionalAttr) {
+		m["adjoint"] = value
+	}
+}
+
+// Solves systems of linear equations with upper or lower triangular matrices by
+//
+// backsubstitution.
+//
+// `matrix` is a tensor of shape `[..., M, M]` whose inner-most 2 dimensions form
+// square matrices. If `lower` is `True` then the strictly upper triangular part
+// of each inner-most matrix is assumed to be zero and not accessed.
+// If `lower` is False then the strictly lower triangular part of each inner-most
+// matrix is assumed to be zero and not accessed.
+// `rhs` is a tensor of shape `[..., M, K]`.
+//
+// The output is a tensor of shape `[..., M, K]`. If `adjoint` is
+// `True` then the innermost matrices in `output` satisfy matrix equations
+// `matrix[..., :, :] * output[..., :, :] = rhs[..., :, :]`.
+// If `adjoint` is `False` then the strictly then the  innermost matrices in
+// `output` satisfy matrix equations
+// `adjoint(matrix[..., i, k]) * output[..., k, j] = rhs[..., i, j]`.
+//
+// Arguments:
+//	matrix: Shape is `[..., M, M]`.
+//	rhs: Shape is `[..., M, K]`.
+//
+// Returns Shape is `[..., M, K]`.
+func MatrixTriangularSolve(scope *Scope, matrix tf.Output, rhs tf.Output, optional ...MatrixTriangularSolveAttr) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "MatrixTriangularSolve",
+		Input: []tf.Input{
+			matrix, rhs,
 		},
 		Attrs: attrs,
 	}
@@ -10395,90 +10651,6 @@ func SparseMatMul(scope *Scope, a tf.Output, b tf.Output, optional ...SparseMatM
 	return op.Output(0)
 }
 
-// MaxPoolV2Attr is an optional argument to MaxPoolV2.
-type MaxPoolV2Attr func(optionalAttr)
-
-// MaxPoolV2DataFormat sets the optional data_format attribute to value.
-//
-// value: Specify the data format of the input and output data. With the
-// default format "NHWC", the data is stored in the order of:
-//     [batch, in_height, in_width, in_channels].
-// Alternatively, the format could be "NCHW", the data storage order of:
-//     [batch, in_channels, in_height, in_width].
-// If not specified, defaults to "NHWC"
-func MaxPoolV2DataFormat(value string) MaxPoolV2Attr {
-	return func(m optionalAttr) {
-		m["data_format"] = value
-	}
-}
-
-// Performs max pooling on the input.
-//
-// Arguments:
-//	input: 4-D input to pool over.
-//	ksize: The size of the window for each dimension of the input tensor.
-//	strides: The stride of the sliding window for each dimension of the
-// input tensor.
-//	padding: The type of padding algorithm to use.
-//
-// Returns The max pooled output tensor.
-func MaxPoolV2(scope *Scope, input tf.Output, ksize tf.Output, strides tf.Output, padding string, optional ...MaxPoolV2Attr) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"padding": padding}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "MaxPoolV2",
-		Input: []tf.Input{
-			input, ksize, strides,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Batch normalization.
-//
-// DEPRECATED at GraphDef version 9: Use tf.nn.batch_normalization()
-//
-// This op is deprecated. Prefer `tf.nn.batch_normalization`.
-//
-// Arguments:
-//	t: A 4D input Tensor.
-//	m: A 1D mean Tensor with size matching the last dimension of t.
-// This is the first output from tf.nn.moments,
-// or a saved moving average thereof.
-//	v: A 1D variance Tensor with size matching the last dimension of t.
-// This is the second output from tf.nn.moments,
-// or a saved moving average thereof.
-//	beta: A 1D beta Tensor with size matching the last dimension of t.
-// An offset to be added to the normalized tensor.
-//	gamma: A 1D gamma Tensor with size matching the last dimension of t.
-// If "scale_after_normalization" is true, this tensor will be multiplied
-// with the normalized tensor.
-//	variance_epsilon: A small float number to avoid dividing by 0.
-//	scale_after_normalization: A bool indicating whether the resulted tensor
-// needs to be multiplied with gamma.
-func BatchNormWithGlobalNormalization(scope *Scope, t tf.Output, m tf.Output, v tf.Output, beta tf.Output, gamma tf.Output, variance_epsilon float32, scale_after_normalization bool) (result tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"variance_epsilon": variance_epsilon, "scale_after_normalization": scale_after_normalization}
-	opspec := tf.OpSpec{
-		Type: "BatchNormWithGlobalNormalization",
-		Input: []tf.Input{
-			t, m, v, beta, gamma,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
 // SdcaOptimizerAttr is an optional argument to SdcaOptimizer.
 type SdcaOptimizerAttr func(optionalAttr)
 
@@ -10963,17 +11135,62 @@ func DenseToDenseSetOperation(scope *Scope, set1 tf.Output, set2 tf.Output, set_
 	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// Computes the sign and the log of the absolute value of the determinant of
+// SumAttr is an optional argument to Sum.
+type SumAttr func(optionalAttr)
+
+// SumKeepDims sets the optional keep_dims attribute to value.
 //
-// one or more square matrices.
+// value: If true, retain reduced dimensions with length 1.
+// If not specified, defaults to false
+func SumKeepDims(value bool) SumAttr {
+	return func(m optionalAttr) {
+		m["keep_dims"] = value
+	}
+}
+
+// Computes the sum of elements across dimensions of a tensor.
 //
-// The input is a tensor of shape `[N, M, M]` whose inner-most 2 dimensions
-// form square matrices. The outputs are two tensors containing the signs and
-// absolute values of the log determinants for all N input submatrices
-// `[..., :, :]` such that the determinant = sign*exp(log_abs_determinant).
-// The log_abs_determinant is computed as det(P)*sum(log(diag(LU))) where LU
-// is the LU decomposition of the input and P is the corresponding
-// permutation matrix.
+// Reduces `input` along the dimensions given in `reduction_indices`. Unless
+// `keep_dims` is true, the rank of the tensor is reduced by 1 for each entry in
+// `reduction_indices`. If `keep_dims` is true, the reduced dimensions are
+// retained with length 1.
+//
+// Arguments:
+//	input: The tensor to reduce.
+//	reduction_indices: The dimensions to reduce. Must be in the range
+// `[-rank(input), rank(input))`.
+//
+// Returns The reduced tensor.
+func Sum(scope *Scope, input tf.Output, reduction_indices tf.Output, optional ...SumAttr) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "Sum",
+		Input: []tf.Input{
+			input, reduction_indices,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Computes the sign and the log of the absolute value of the determinant of
+//
+// one or more square matrices.
+//
+// The input is a tensor of shape `[N, M, M]` whose inner-most 2 dimensions
+// form square matrices. The outputs are two tensors containing the signs and
+// absolute values of the log determinants for all N input submatrices
+// `[..., :, :]` such that the determinant = sign*exp(log_abs_determinant).
+// The log_abs_determinant is computed as det(P)*sum(log(diag(LU))) where LU
+// is the LU decomposition of the input and P is the corresponding
+// permutation matrix.
 //
 // Arguments:
 //	input: Shape is `[N, M, M]`.
@@ -11071,6 +11288,29 @@ func SparseFillEmptyRowsGrad(scope *Scope, reverse_index_map tf.Output, grad_val
 	return op.Output(0), op.Output(1)
 }
 
+// Assigns a new value to a variable.
+//
+// Any ReadVariableOp with a control dependency on this op is guaranteed to return
+// this value or a subsequent newer value of the variable.
+//
+// Arguments:
+//	resource: handle to the resource in which to store the variable.
+//	value: the value to set the new tensor to use.
+//
+// Returns the created operation.
+func AssignVariableOp(scope *Scope, resource tf.Output, value tf.Output) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "AssignVariableOp",
+		Input: []tf.Input{
+			resource, value,
+		},
+	}
+	return scope.AddOperation(opspec)
+}
+
 // Says whether the targets are in the top `K` predictions.
 //
 // This outputs a `batch_size` bool array, an entry `out[i]` is `true` if the
@@ -12763,6 +13003,90 @@ func SparseTensorDenseMatMul(scope *Scope, a_indices tf.Output, a_values tf.Outp
 	return op.Output(0)
 }
 
+// Batch normalization.
+//
+// DEPRECATED at GraphDef version 9: Use tf.nn.batch_normalization()
+//
+// This op is deprecated. Prefer `tf.nn.batch_normalization`.
+//
+// Arguments:
+//	t: A 4D input Tensor.
+//	m: A 1D mean Tensor with size matching the last dimension of t.
+// This is the first output from tf.nn.moments,
+// or a saved moving average thereof.
+//	v: A 1D variance Tensor with size matching the last dimension of t.
+// This is the second output from tf.nn.moments,
+// or a saved moving average thereof.
+//	beta: A 1D beta Tensor with size matching the last dimension of t.
+// An offset to be added to the normalized tensor.
+//	gamma: A 1D gamma Tensor with size matching the last dimension of t.
+// If "scale_after_normalization" is true, this tensor will be multiplied
+// with the normalized tensor.
+//	variance_epsilon: A small float number to avoid dividing by 0.
+//	scale_after_normalization: A bool indicating whether the resulted tensor
+// needs to be multiplied with gamma.
+func BatchNormWithGlobalNormalization(scope *Scope, t tf.Output, m tf.Output, v tf.Output, beta tf.Output, gamma tf.Output, variance_epsilon float32, scale_after_normalization bool) (result tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"variance_epsilon": variance_epsilon, "scale_after_normalization": scale_after_normalization}
+	opspec := tf.OpSpec{
+		Type: "BatchNormWithGlobalNormalization",
+		Input: []tf.Input{
+			t, m, v, beta, gamma,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// MaxPoolV2Attr is an optional argument to MaxPoolV2.
+type MaxPoolV2Attr func(optionalAttr)
+
+// MaxPoolV2DataFormat sets the optional data_format attribute to value.
+//
+// value: Specify the data format of the input and output data. With the
+// default format "NHWC", the data is stored in the order of:
+//     [batch, in_height, in_width, in_channels].
+// Alternatively, the format could be "NCHW", the data storage order of:
+//     [batch, in_channels, in_height, in_width].
+// If not specified, defaults to "NHWC"
+func MaxPoolV2DataFormat(value string) MaxPoolV2Attr {
+	return func(m optionalAttr) {
+		m["data_format"] = value
+	}
+}
+
+// Performs max pooling on the input.
+//
+// Arguments:
+//	input: 4-D input to pool over.
+//	ksize: The size of the window for each dimension of the input tensor.
+//	strides: The stride of the sliding window for each dimension of the
+// input tensor.
+//	padding: The type of padding algorithm to use.
+//
+// Returns The max pooled output tensor.
+func MaxPoolV2(scope *Scope, input tf.Output, ksize tf.Output, strides tf.Output, padding string, optional ...MaxPoolV2Attr) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"padding": padding}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "MaxPoolV2",
+		Input: []tf.Input{
+			input, ksize, strides,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // OrderedMapUnstageNoKeyAttr is an optional argument to OrderedMapUnstageNoKey.
 type OrderedMapUnstageNoKeyAttr func(optionalAttr)
 
@@ -12835,52 +13159,6 @@ func OrderedMapUnstageNoKey(scope *Scope, indices tf.Output, dtypes []tf.DataTyp
 	return key, values
 }
 
-// MaxPoolAttr is an optional argument to MaxPool.
-type MaxPoolAttr func(optionalAttr)
-
-// MaxPoolDataFormat sets the optional data_format attribute to value.
-//
-// value: Specify the data format of the input and output data. With the
-// default format "NHWC", the data is stored in the order of:
-//     [batch, in_height, in_width, in_channels].
-// Alternatively, the format could be "NCHW", the data storage order of:
-//     [batch, in_channels, in_height, in_width].
-// If not specified, defaults to "NHWC"
-func MaxPoolDataFormat(value string) MaxPoolAttr {
-	return func(m optionalAttr) {
-		m["data_format"] = value
-	}
-}
-
-// Performs max pooling on the input.
-//
-// Arguments:
-//	input: 4-D input to pool over.
-//	ksize: The size of the window for each dimension of the input tensor.
-//	strides: The stride of the sliding window for each dimension of the
-// input tensor.
-//	padding: The type of padding algorithm to use.
-//
-// Returns The max pooled output tensor.
-func MaxPool(scope *Scope, input tf.Output, ksize []int64, strides []int64, padding string, optional ...MaxPoolAttr) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"ksize": ksize, "strides": strides, "padding": padding}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "MaxPool",
-		Input: []tf.Input{
-			input,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
 // Merges summaries.
 //
 // This op creates a
@@ -13867,24 +14145,6 @@ func QuantizedRelu(scope *Scope, features tf.Output, min_features tf.Output, max
 	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// Computes the gradient for the inverse of `x` wrt its input.
-//
-// Specifically, `grad = -dy * y*y`, where `y = 1/x`, and `dy`
-// is the corresponding input gradient.
-func ReciprocalGrad(scope *Scope, y tf.Output, dy tf.Output) (z tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "ReciprocalGrad",
-		Input: []tf.Input{
-			y, dy,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
 // Reverses specific dimensions of a tensor.
 //
 // NOTE `tf.reverse` has now changed behavior in preparation for 1.0.
@@ -14077,35 +14337,6 @@ func LookupTableFindV2(scope *Scope, table_handle tf.Output, keys tf.Output, def
 	return op.Output(0)
 }
 
-// Computes second-order gradients of the maxpooling function.
-//
-// Arguments:
-//	input: The original input.
-//	grad: 4-D with shape `[batch, height, width, channels]`.  Gradients w.r.t. the
-// input of `max_pool`.
-//	argmax: The indices of the maximum values chosen for each output of `max_pool`.
-//	ksize: The size of the window for each dimension of the input tensor.
-//	strides: The stride of the sliding window for each dimension of the
-// input tensor.
-//	padding: The type of padding algorithm to use.
-//
-// Returns Gradients of gradients w.r.t. the input of `max_pool`.
-func MaxPoolGradGradWithArgmax(scope *Scope, input tf.Output, grad tf.Output, argmax tf.Output, ksize []int64, strides []int64, padding string) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"ksize": ksize, "strides": strides, "padding": padding}
-	opspec := tf.OpSpec{
-		Type: "MaxPoolGradGradWithArgmax",
-		Input: []tf.Input{
-			input, grad, argmax,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
 // DepthwiseConv2dNativeBackpropInputAttr is an optional argument to DepthwiseConv2dNativeBackpropInput.
 type DepthwiseConv2dNativeBackpropInputAttr func(optionalAttr)
 
@@ -14535,84 +14766,7 @@ func ResourceSparseApplyAdagradDA(scope *Scope, var_ tf.Output, gradient_accumul
 	return scope.AddOperation(opspec)
 }
 
-// MaxPoolWithArgmaxAttr is an optional argument to MaxPoolWithArgmax.
-type MaxPoolWithArgmaxAttr func(optionalAttr)
-
-// MaxPoolWithArgmaxTargmax sets the optional Targmax attribute to value.
-// If not specified, defaults to DT_INT64
-func MaxPoolWithArgmaxTargmax(value tf.DataType) MaxPoolWithArgmaxAttr {
-	return func(m optionalAttr) {
-		m["Targmax"] = value
-	}
-}
-
-// Performs max pooling on the input and outputs both max values and indices.
-//
-// The indices in `argmax` are flattened, so that a maximum value at position
-// `[b, y, x, c]` becomes flattened index
-// `((b * height + y) * width + x) * channels + c`.
-//
-// The indices returned are always in `[0, height) x [0, width)` before flattening,
-// even if padding is involved and the mathematically correct answer is outside
-// (either negative or too large).  This is a bug, but fixing it is difficult to do
-// in a safe backwards compatible way, especially due to flattening.
-//
-// Arguments:
-//	input: 4-D with shape `[batch, height, width, channels]`.  Input to pool over.
-//	ksize: The size of the window for each dimension of the input tensor.
-//	strides: The stride of the sliding window for each dimension of the
-// input tensor.
-//	padding: The type of padding algorithm to use.
-//
-// Returns The max pooled output tensor.4-D.  The flattened indices of the max values chosen for each output.
-func MaxPoolWithArgmax(scope *Scope, input tf.Output, ksize []int64, strides []int64, padding string, optional ...MaxPoolWithArgmaxAttr) (output tf.Output, argmax tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"ksize": ksize, "strides": strides, "padding": padding}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "MaxPoolWithArgmax",
-		Input: []tf.Input{
-			input,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1)
-}
-
-// Fast Fourier transform.
-//
-// Computes the 1-dimensional discrete Fourier transform over the inner-most
-// dimension of `input`.
-//
-// Arguments:
-//	input: A complex64 tensor.
-//
-// Returns A complex64 tensor of the same shape as `input`. The inner-most
-//   dimension of `input` is replaced with its 1D Fourier transform.
-//
-// @compatibility(numpy)
-// Equivalent to np.fft.fft
-// @end_compatibility
-func FFT(scope *Scope, input tf.Output) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "FFT",
-		Input: []tf.Input{
-			input,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Computes softmax cross entropy cost and gradients to backpropagate.
+// Computes softmax cross entropy cost and gradients to backpropagate.
 //
 // Unlike `SoftmaxCrossEntropyWithLogits`, this operation does not accept
 // a matrix of label probabilities, but rather a single label per row
@@ -14990,6 +15144,46 @@ func Conv2D(scope *Scope, input tf.Output, filter tf.Output, strides []int64, pa
 	return op.Output(0)
 }
 
+// VariableShapeAttr is an optional argument to VariableShape.
+type VariableShapeAttr func(optionalAttr)
+
+// VariableShapeOutType sets the optional out_type attribute to value.
+// If not specified, defaults to DT_INT32
+func VariableShapeOutType(value tf.DataType) VariableShapeAttr {
+	return func(m optionalAttr) {
+		m["out_type"] = value
+	}
+}
+
+// Returns the shape of the variable pointed to by `resource`.
+//
+// This operation returns a 1-D integer tensor representing the shape of `input`.
+//
+// For example:
+//
+// ```
+// # 't' is [[[1, 1, 1], [2, 2, 2]], [[3, 3, 3], [4, 4, 4]]]
+// shape(t) ==> [2, 2, 3]
+// ```
+func VariableShape(scope *Scope, input tf.Output, optional ...VariableShapeAttr) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "VariableShape",
+		Input: []tf.Input{
+			input,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // StringJoinAttr is an optional argument to StringJoin.
 type StringJoinAttr func(optionalAttr)
 
@@ -15600,132 +15794,6 @@ func TruncateDiv(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
 	return op.Output(0)
 }
 
-// MaxPoolGradGradV2Attr is an optional argument to MaxPoolGradGradV2.
-type MaxPoolGradGradV2Attr func(optionalAttr)
-
-// MaxPoolGradGradV2DataFormat sets the optional data_format attribute to value.
-//
-// value: Specify the data format of the input and output data. With the
-// default format "NHWC", the data is stored in the order of:
-//     [batch, in_height, in_width, in_channels].
-// Alternatively, the format could be "NCHW", the data storage order of:
-//     [batch, in_channels, in_height, in_width].
-// If not specified, defaults to "NHWC"
-func MaxPoolGradGradV2DataFormat(value string) MaxPoolGradGradV2Attr {
-	return func(m optionalAttr) {
-		m["data_format"] = value
-	}
-}
-
-// Computes second-order gradients of the maxpooling function.
-//
-// Arguments:
-//	orig_input: The original input tensor.
-//	orig_output: The original output tensor.
-//	grad: 4-D.  Gradients of gradients w.r.t. the input of `max_pool`.
-//	ksize: The size of the window for each dimension of the input tensor.
-//	strides: The stride of the sliding window for each dimension of the
-// input tensor.
-//	padding: The type of padding algorithm to use.
-//
-// Returns Gradients of gradients w.r.t. the input to `max_pool`.
-func MaxPoolGradGradV2(scope *Scope, orig_input tf.Output, orig_output tf.Output, grad tf.Output, ksize tf.Output, strides tf.Output, padding string, optional ...MaxPoolGradGradV2Attr) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"padding": padding}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "MaxPoolGradGradV2",
-		Input: []tf.Input{
-			orig_input, orig_output, grad, ksize, strides,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Adjust the saturation of one or more images.
-//
-// `images` is a tensor of at least 3 dimensions.  The last dimension is
-// interpretted as channels, and must be three.
-//
-// The input image is considered in the RGB colorspace. Conceptually, the RGB
-// colors are first mapped into HSV. A scale is then applied all the saturation
-// values, and then remapped back to RGB colorspace.
-//
-// Arguments:
-//	images: Images to adjust.  At least 3-D.
-//	scale: A float scale to add to the saturation.
-//
-// Returns The hue-adjusted image or images.
-func AdjustSaturation(scope *Scope, images tf.Output, scale tf.Output) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "AdjustSaturation",
-		Input: []tf.Input{
-			images, scale,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// SelfAdjointEigV2Attr is an optional argument to SelfAdjointEigV2.
-type SelfAdjointEigV2Attr func(optionalAttr)
-
-// SelfAdjointEigV2ComputeV sets the optional compute_v attribute to value.
-//
-// value: If `True` then eigenvectors will be computed and returned in `v`.
-// Otherwise, only the eigenvalues will be computed.
-// If not specified, defaults to true
-func SelfAdjointEigV2ComputeV(value bool) SelfAdjointEigV2Attr {
-	return func(m optionalAttr) {
-		m["compute_v"] = value
-	}
-}
-
-// Computes the eigen decomposition of one or more square self-adjoint matrices.
-//
-// Computes the eigenvalues and (optionally) eigenvectors of each inner matrix in
-// `input` such that `input[..., :, :] = v[..., :, :] * diag(e[..., :])`.
-//
-// ```python
-// # a is a tensor.
-// # e is a tensor of eigenvalues.
-// # v is a tensor of eigenvectors.
-// e, v = self_adjoint_eig(a)
-// e = self_adjoint_eig(a, compute_v=False)
-// ```
-//
-// Arguments:
-//	input: `Tensor` input of shape `[N, N]`.
-//
-// Returns Eigenvalues. Shape is `[N]`.Eigenvectors. Shape is `[N, N]`.
-func SelfAdjointEigV2(scope *Scope, input tf.Output, optional ...SelfAdjointEigV2Attr) (e tf.Output, v tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "SelfAdjointEigV2",
-		Input: []tf.Input{
-			input,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1)
-}
-
 // SampleDistortedBoundingBoxAttr is an optional argument to SampleDistortedBoundingBox.
 type SampleDistortedBoundingBoxAttr func(optionalAttr)
 
@@ -16519,28 +16587,74 @@ func EncodeBase64(scope *Scope, input tf.Output, optional ...EncodeBase64Attr) (
 	return op.Output(0)
 }
 
-// Output a fact about factorials.
-func Fact(scope *Scope) (fact tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "Fact",
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// StatelessRandomUniformAttr is an optional argument to StatelessRandomUniform.
-type StatelessRandomUniformAttr func(optionalAttr)
+// VarHandleOpAttr is an optional argument to VarHandleOp.
+type VarHandleOpAttr func(optionalAttr)
 
-// StatelessRandomUniformDtype sets the optional dtype attribute to value.
+// VarHandleOpContainer sets the optional container attribute to value.
 //
-// value: The type of the output.
-// If not specified, defaults to DT_FLOAT
-func StatelessRandomUniformDtype(value tf.DataType) StatelessRandomUniformAttr {
+// value: the container this variable is placed in.
+// If not specified, defaults to ""
+func VarHandleOpContainer(value string) VarHandleOpAttr {
 	return func(m optionalAttr) {
-		m["dtype"] = value
+		m["container"] = value
+	}
+}
+
+// VarHandleOpSharedName sets the optional shared_name attribute to value.
+//
+// value: the name by which this variable is referred to.
+// If not specified, defaults to ""
+func VarHandleOpSharedName(value string) VarHandleOpAttr {
+	return func(m optionalAttr) {
+		m["shared_name"] = value
+	}
+}
+
+// Creates a handle to a Variable resource.
+//
+// Arguments:
+//	dtype: the type of this variable. Must agree with the dtypes
+// of all ops using this variable.
+//	shape: The (possibly partially specified) shape of this variable.
+func VarHandleOp(scope *Scope, dtype tf.DataType, shape tf.Shape, optional ...VarHandleOpAttr) (resource tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"dtype": dtype, "shape": shape}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "VarHandleOp",
+
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Output a fact about factorials.
+func Fact(scope *Scope) (fact tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "Fact",
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// StatelessRandomUniformAttr is an optional argument to StatelessRandomUniform.
+type StatelessRandomUniformAttr func(optionalAttr)
+
+// StatelessRandomUniformDtype sets the optional dtype attribute to value.
+//
+// value: The type of the output.
+// If not specified, defaults to DT_FLOAT
+func StatelessRandomUniformDtype(value tf.DataType) StatelessRandomUniformAttr {
+	return func(m optionalAttr) {
+		m["dtype"] = value
 	}
 }
 
@@ -17121,129 +17235,6 @@ func ResourceSparseApplyAdagrad(scope *Scope, var_ tf.Output, accum tf.Output, l
 	return scope.AddOperation(opspec)
 }
 
-// LRNGradAttr is an optional argument to LRNGrad.
-type LRNGradAttr func(optionalAttr)
-
-// LRNGradDepthRadius sets the optional depth_radius attribute to value.
-//
-// value: A depth radius.
-// If not specified, defaults to 5
-func LRNGradDepthRadius(value int64) LRNGradAttr {
-	return func(m optionalAttr) {
-		m["depth_radius"] = value
-	}
-}
-
-// LRNGradBias sets the optional bias attribute to value.
-//
-// value: An offset (usually > 0 to avoid dividing by 0).
-// If not specified, defaults to 1
-func LRNGradBias(value float32) LRNGradAttr {
-	return func(m optionalAttr) {
-		m["bias"] = value
-	}
-}
-
-// LRNGradAlpha sets the optional alpha attribute to value.
-//
-// value: A scale factor, usually positive.
-// If not specified, defaults to 1
-func LRNGradAlpha(value float32) LRNGradAttr {
-	return func(m optionalAttr) {
-		m["alpha"] = value
-	}
-}
-
-// LRNGradBeta sets the optional beta attribute to value.
-//
-// value: An exponent.
-// If not specified, defaults to 0.5
-func LRNGradBeta(value float32) LRNGradAttr {
-	return func(m optionalAttr) {
-		m["beta"] = value
-	}
-}
-
-// Gradients for Local Response Normalization.
-//
-// Arguments:
-//	input_grads: 4-D with shape `[batch, height, width, channels]`.
-//	input_image: 4-D with shape `[batch, height, width, channels]`.
-//	output_image: 4-D with shape `[batch, height, width, channels]`.
-//
-// Returns The gradients for LRN.
-func LRNGrad(scope *Scope, input_grads tf.Output, input_image tf.Output, output_image tf.Output, optional ...LRNGradAttr) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "LRNGrad",
-		Input: []tf.Input{
-			input_grads, input_image, output_image,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// StringToNumberAttr is an optional argument to StringToNumber.
-type StringToNumberAttr func(optionalAttr)
-
-// StringToNumberOutType sets the optional out_type attribute to value.
-//
-// value: The numeric type to interpret each string in `string_tensor` as.
-// If not specified, defaults to DT_FLOAT
-func StringToNumberOutType(value tf.DataType) StringToNumberAttr {
-	return func(m optionalAttr) {
-		m["out_type"] = value
-	}
-}
-
-// Converts each string in the input Tensor to the specified numeric type.
-//
-// (Note that int32 overflow results in an error while float overflow
-// results in a rounded value.)
-//
-// Returns A Tensor of the same shape as the input `string_tensor`.
-func StringToNumber(scope *Scope, string_tensor tf.Output, optional ...StringToNumberAttr) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "StringToNumber",
-		Input: []tf.Input{
-			string_tensor,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Returns the truth value of NOT x element-wise.
-func LogicalNot(scope *Scope, x tf.Output) (y tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "LogicalNot",
-		Input: []tf.Input{
-			x,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
 // AvgPool3DGradAttr is an optional argument to AvgPool3DGrad.
 type AvgPool3DGradAttr func(optionalAttr)
 
@@ -17292,6 +17283,34 @@ func AvgPool3DGrad(scope *Scope, orig_input_shape tf.Output, grad tf.Output, ksi
 	return op.Output(0)
 }
 
+// Inverse fast Fourier transform.
+//
+// Computes the inverse 1-dimensional discrete Fourier transform over the
+// inner-most dimension of `input`.
+//
+// Arguments:
+//	input: A complex64 tensor.
+//
+// Returns A complex64 tensor of the same shape as `input`. The inner-most
+//   dimension of `input` is replaced with its inverse 1D Fourier transform.
+//
+// @compatibility(numpy)
+// Equivalent to np.fft.ifft
+// @end_compatibility
+func IFFT(scope *Scope, input tf.Output) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "IFFT",
+		Input: []tf.Input{
+			input,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Creates a dataset that batches `batch_size` elements from `input_dataset`.
 //
 // Arguments:
@@ -18381,34 +18400,6 @@ func StatelessRandomNormal(scope *Scope, shape tf.Output, seed tf.Output, option
 	return op.Output(0)
 }
 
-// Inverse fast Fourier transform.
-//
-// Computes the inverse 1-dimensional discrete Fourier transform over the
-// inner-most dimension of `input`.
-//
-// Arguments:
-//	input: A complex64 tensor.
-//
-// Returns A complex64 tensor of the same shape as `input`. The inner-most
-//   dimension of `input` is replaced with its inverse 1D Fourier transform.
-//
-// @compatibility(numpy)
-// Equivalent to np.fft.ifft
-// @end_compatibility
-func IFFT(scope *Scope, input tf.Output) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "IFFT",
-		Input: []tf.Input{
-			input,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
 // Computes the sum along sparse segments of a tensor divided by the sqrt of N.
 //
 // N is the size of the segment being reduced.
@@ -18785,122 +18776,17 @@ func SparseSegmentMeanGrad(scope *Scope, grad tf.Output, indices tf.Output, segm
 	return op.Output(0)
 }
 
-// Converts one or more images from RGB to HSV.
+// Applies sparse addition to `input` using individual values or slices
 //
-// Outputs a tensor of the same shape as the `images` tensor, containing the HSV
-// value of the pixels. The output is only well defined if the value in `images`
-// are in `[0,1]`.
+// from `updates` according to indices `indices`.  The updates are non-aliasing:
+// `input` is only modified in-place if no other operations will use it.
+// Otherwise, a copy of `input` is made.  This operation has a gradient with
+// respect to both `input` and `updates`.
 //
-// `output[..., 0]` contains hue, `output[..., 1]` contains saturation, and
-// `output[..., 2]` contains value. All HSV values are in `[0,1]`. A hue of 0
-// corresponds to pure red, hue 1/3 is pure green, and 2/3 is pure blue.
+// `input` is a `Tensor` with rank `P` and `indices` is a `Tensor` of rank `Q`.
 //
-// Arguments:
-//	images: 1-D or higher rank. RGB data to convert. Last dimension must be size 3.
-//
-// Returns `images` converted to HSV.
-func RGBToHSV(scope *Scope, images tf.Output) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "RGBToHSV",
-		Input: []tf.Input{
-			images,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// MatrixSolveLsAttr is an optional argument to MatrixSolveLs.
-type MatrixSolveLsAttr func(optionalAttr)
-
-// MatrixSolveLsFast sets the optional fast attribute to value.
-// If not specified, defaults to true
-func MatrixSolveLsFast(value bool) MatrixSolveLsAttr {
-	return func(m optionalAttr) {
-		m["fast"] = value
-	}
-}
-
-// Solves one or more linear least-squares problems.
-//
-// `matrix` is a tensor of shape `[..., M, N]` whose inner-most 2 dimensions
-// form real or complex matrices of size `[M, N]`. `Rhs` is a tensor of the same
-// type as `matrix` and shape `[..., M, K]`.
-// The output is a tensor shape `[..., N, K]` where each output matrix solves
-// each of the equations
-// `matrix[..., :, :]` * `output[..., :, :]` = `rhs[..., :, :]`
-// in the least squares sense.
-//
-// We use the following notation for (complex) matrix and right-hand sides
-// in the batch:
-//
-// `matrix`=\\(A \in \mathbb{C}^{m \times n}\\),
-// `rhs`=\\(B  \in \mathbb{C}^{m \times k}\\),
-// `output`=\\(X  \in \mathbb{C}^{n \times k}\\),
-// `l2_regularizer`=\\(\lambda \in \mathbb{R}\\).
-//
-// If `fast` is `True`, then the solution is computed by solving the normal
-// equations using Cholesky decomposition. Specifically, if \\(m \ge n\\) then
-// \\(X = (A^H A + \lambda I)^{-1} A^H B\\), which solves the least-squares
-// problem \\(X = \mathrm{argmin}_{Z \in \Re^{n \times k} } ||A Z - B||_F^2 +
-// \lambda ||Z||_F^2\\). If \\(m \lt n\\) then `output` is computed as
-// \\(X = A^H (A A^H + \lambda I)^{-1} B\\), which (for \\(\lambda = 0\\)) is the
-// minimum-norm solution to the under-determined linear system, i.e.
-// \\(X = \mathrm{argmin}_{Z \in \mathbb{C}^{n \times k} } ||Z||_F^2 \\),
-// subject to \\(A Z = B\\). Notice that the fast path is only numerically stable
-// when \\(A\\) is numerically full rank and has a condition number
-// \\(\mathrm{cond}(A) \lt \frac{1}{\sqrt{\epsilon_{mach} } }\\) or\\(\lambda\\) is
-// sufficiently large.
-//
-// If `fast` is `False` an algorithm based on the numerically robust complete
-// orthogonal decomposition is used. This computes the minimum-norm
-// least-squares solution, even when \\(A\\) is rank deficient. This path is
-// typically 6-7 times slower than the fast path. If `fast` is `False` then
-// `l2_regularizer` is ignored.
-//
-// Arguments:
-//	matrix: Shape is `[..., M, N]`.
-//	rhs: Shape is `[..., M, K]`.
-//	l2_regularizer: Scalar tensor.
-//
-// @compatibility(numpy)
-// Equivalent to np.linalg.lstsq
-// @end_compatibility
-//
-// Returns Shape is `[..., N, K]`.
-func MatrixSolveLs(scope *Scope, matrix tf.Output, rhs tf.Output, l2_regularizer tf.Output, optional ...MatrixSolveLsAttr) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "MatrixSolveLs",
-		Input: []tf.Input{
-			matrix, rhs, l2_regularizer,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Applies sparse addition to `input` using individual values or slices
-//
-// from `updates` according to indices `indices`.  The updates are non-aliasing:
-// `input` is only modified in-place if no other operations will use it.
-// Otherwise, a copy of `input` is made.  This operation has a gradient with
-// respect to both `input` and `updates`.
-//
-// `input` is a `Tensor` with rank `P` and `indices` is a `Tensor` of rank `Q`.
-//
-// `indices` must be integer tensor, containing indices into `input`.
-// It must be shape `[d_0, ..., d_{Q-2}, K]` where `0 < K <= P`.
+// `indices` must be integer tensor, containing indices into `input`.
+// It must be shape `[d_0, ..., d_{Q-2}, K]` where `0 < K <= P`.
 //
 // The innermost dimension of `indices` (with length `K`) corresponds to
 // indices into elements (if `K = P`) or `(P-K)`-dimensional slices
@@ -19191,185 +19077,502 @@ func ReduceJoin(scope *Scope, inputs tf.Output, reduction_indices tf.Output, opt
 	return op.Output(0)
 }
 
-// SvdAttr is an optional argument to Svd.
-type SvdAttr func(optionalAttr)
+// Computes element-wise population count (a.k.a. popcount, bitsum, bitcount).
+//
+// For each entry in `x`, calculates the number of `1` (on) bits in the binary
+// representation of that entry.
+//
+// **NOTE**: It is more efficient to first `tf.bitcast` your tensors into
+// `int32` or `int64` and perform the bitcount on the result, than to feed in
+// 8- or 16-bit inputs and then aggregate the resulting counts.
+func PopulationCount(scope *Scope, x tf.Output) (y tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "PopulationCount",
+		Input: []tf.Input{
+			x,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
 
-// SvdComputeUv sets the optional compute_uv attribute to value.
+// AssertAttr is an optional argument to Assert.
+type AssertAttr func(optionalAttr)
+
+// AssertSummarize sets the optional summarize attribute to value.
 //
-// value: If true, left and right singular vectors will be
-// computed and returned in `u` and `v`, respectively.
-// If false, `u` and `v` are not set and should never referenced.
-// If not specified, defaults to true
-func SvdComputeUv(value bool) SvdAttr {
+// value: Print this many entries of each tensor.
+// If not specified, defaults to 3
+func AssertSummarize(value int64) AssertAttr {
 	return func(m optionalAttr) {
-		m["compute_uv"] = value
+		m["summarize"] = value
 	}
 }
 
-// SvdFullMatrices sets the optional full_matrices attribute to value.
+// Asserts that the given condition is true.
 //
-// value: If true, compute full-sized `u` and `v`. If false
-// (the default), compute only the leading `P` singular vectors.
-// Ignored if `compute_uv` is `False`.
+// If `condition` evaluates to false, print the list of tensors in `data`.
+// `summarize` determines how many entries of the tensors to print.
+//
+// Arguments:
+//	condition: The condition to evaluate.
+//	data: The tensors to print out when condition is false.
+//
+// Returns the created operation.
+func Assert(scope *Scope, condition tf.Output, data []tf.Output, optional ...AssertAttr) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "Assert",
+		Input: []tf.Input{
+			condition, tf.OutputList(data),
+		},
+		Attrs: attrs,
+	}
+	return scope.AddOperation(opspec)
+}
+
+// RandomUniformAttr is an optional argument to RandomUniform.
+type RandomUniformAttr func(optionalAttr)
+
+// RandomUniformSeed sets the optional seed attribute to value.
+//
+// value: If either `seed` or `seed2` are set to be non-zero, the random number
+// generator is seeded by the given seed.  Otherwise, it is seeded by a
+// random seed.
+// If not specified, defaults to 0
+func RandomUniformSeed(value int64) RandomUniformAttr {
+	return func(m optionalAttr) {
+		m["seed"] = value
+	}
+}
+
+// RandomUniformSeed2 sets the optional seed2 attribute to value.
+//
+// value: A second seed to avoid seed collision.
+// If not specified, defaults to 0
+func RandomUniformSeed2(value int64) RandomUniformAttr {
+	return func(m optionalAttr) {
+		m["seed2"] = value
+	}
+}
+
+// Outputs random values from a uniform distribution.
+//
+// The generated values follow a uniform distribution in the range `[0, 1)`. The
+// lower bound 0 is included in the range, while the upper bound 1 is excluded.
+//
+// Arguments:
+//	shape: The shape of the output tensor.
+//	dtype: The type of the output.
+//
+// Returns A tensor of the specified shape filled with uniform random values.
+func RandomUniform(scope *Scope, shape tf.Output, dtype tf.DataType, optional ...RandomUniformAttr) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"dtype": dtype}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "RandomUniform",
+		Input: []tf.Input{
+			shape,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// ResourceApplyFtrlAttr is an optional argument to ResourceApplyFtrl.
+type ResourceApplyFtrlAttr func(optionalAttr)
+
+// ResourceApplyFtrlUseLocking sets the optional use_locking attribute to value.
+//
+// value: If `True`, updating of the var and accum tensors will be protected
+// by a lock; otherwise the behavior is undefined, but may exhibit less
+// contention.
 // If not specified, defaults to false
-func SvdFullMatrices(value bool) SvdAttr {
+func ResourceApplyFtrlUseLocking(value bool) ResourceApplyFtrlAttr {
+	return func(m optionalAttr) {
+		m["use_locking"] = value
+	}
+}
+
+// Update '*var' according to the Ftrl-proximal scheme.
+//
+// accum_new = accum + grad * grad
+// linear += grad - (accum_new^(-lr_power) - accum^(-lr_power)) / lr * var
+// quadratic = 1.0 / (accum_new^(lr_power) * lr) + 2 * l2
+// var = (sign(linear) * l1 - linear) / quadratic if |linear| > l1 else 0.0
+// accum = accum_new
+//
+// Arguments:
+//	var_: Should be from a Variable().
+//	accum: Should be from a Variable().
+//	linear: Should be from a Variable().
+//	grad: The gradient.
+//	lr: Scaling factor. Must be a scalar.
+//	l1: L1 regulariation. Must be a scalar.
+//	l2: L2 regulariation. Must be a scalar.
+//	lr_power: Scaling factor. Must be a scalar.
+//
+// Returns the created operation.
+func ResourceApplyFtrl(scope *Scope, var_ tf.Output, accum tf.Output, linear tf.Output, grad tf.Output, lr tf.Output, l1 tf.Output, l2 tf.Output, lr_power tf.Output, optional ...ResourceApplyFtrlAttr) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "ResourceApplyFtrl",
+		Input: []tf.Input{
+			var_, accum, linear, grad, lr, l1, l2, lr_power,
+		},
+		Attrs: attrs,
+	}
+	return scope.AddOperation(opspec)
+}
+
+// AnyAttr is an optional argument to Any.
+type AnyAttr func(optionalAttr)
+
+// AnyKeepDims sets the optional keep_dims attribute to value.
+//
+// value: If true, retain reduced dimensions with length 1.
+// If not specified, defaults to false
+func AnyKeepDims(value bool) AnyAttr {
+	return func(m optionalAttr) {
+		m["keep_dims"] = value
+	}
+}
+
+// Computes the "logical or" of elements across dimensions of a tensor.
+//
+// Reduces `input` along the dimensions given in `reduction_indices`. Unless
+// `keep_dims` is true, the rank of the tensor is reduced by 1 for each entry in
+// `reduction_indices`. If `keep_dims` is true, the reduced dimensions are
+// retained with length 1.
+//
+// Arguments:
+//	input: The tensor to reduce.
+//	reduction_indices: The dimensions to reduce. Must be in the range
+// `[-rank(input), rank(input))`.
+//
+// Returns The reduced tensor.
+func Any(scope *Scope, input tf.Output, reduction_indices tf.Output, optional ...AnyAttr) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "Any",
+		Input: []tf.Input{
+			input, reduction_indices,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Compute the Hurwitz zeta function \\(\zeta(x, q)\\).
+//
+// The Hurwitz zeta function is defined as:
+//
+//
+// \\(\zeta(x, q) = \sum_{n=0}^{\infty} (q + n)^{-x}\\)
+func Zeta(scope *Scope, x tf.Output, q tf.Output) (z tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "Zeta",
+		Input: []tf.Input{
+			x, q,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Inverse real-valued fast Fourier transform.
+//
+// Computes the inverse 1-dimensional discrete Fourier transform of a real-valued
+// signal over the inner-most dimension of `input`.
+//
+// The inner-most dimension of `input` is assumed to be the result of `RFFT`: the
+// `fft_length / 2 + 1` unique components of the DFT of a real-valued signal. If
+// `fft_length` is not provided, it is computed from the size of the inner-most
+// dimension of `input` (`fft_length = 2 * (inner - 1)`). If the FFT length used to
+// compute `input` is odd, it should be provided since it cannot be inferred
+// properly.
+//
+// Along the axis `IRFFT` is computed on, if `fft_length / 2 + 1` is smaller
+// than the corresponding dimension of `input`, the dimension is cropped. If it is
+// larger, the dimension is padded with zeros.
+//
+// Arguments:
+//	input: A complex64 tensor.
+//	fft_length: An int32 tensor of shape [1]. The FFT length.
+//
+// Returns A float32 tensor of the same rank as `input`. The inner-most
+//   dimension of `input` is replaced with the `fft_length` samples of its inverse
+//   1D Fourier transform.
+//
+// @compatibility(numpy)
+// Equivalent to np.fft.irfft
+// @end_compatibility
+func IRFFT(scope *Scope, input tf.Output, fft_length tf.Output) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "IRFFT",
+		Input: []tf.Input{
+			input, fft_length,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Adds a value to the current value of a variable.
+//
+// Any ReadVariableOp which depends directly or indirectly on this assign is
+// guaranteed to see the incremented value or a subsequent newer one.
+//
+// Outputs the incremented value, which can be used to totally order the
+// increments to this variable.
+//
+// Arguments:
+//	resource: handle to the resource in which to store the variable.
+//	value: the value by which the variable will be incremented.
+//
+// Returns the created operation.
+func AssignAddVariableOp(scope *Scope, resource tf.Output, value tf.Output) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "AssignAddVariableOp",
+		Input: []tf.Input{
+			resource, value,
+		},
+	}
+	return scope.AddOperation(opspec)
+}
+
+// Computes inverse hyperbolic sine of x element-wise.
+func Asinh(scope *Scope, x tf.Output) (y tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "Asinh",
+		Input: []tf.Input{
+			x,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Real-valued fast Fourier transform.
+//
+// Computes the 1-dimensional discrete Fourier transform of a real-valued signal
+// over the inner-most dimension of `input`.
+//
+// Since the DFT of a real signal is Hermitian-symmetric, `RFFT` only returns the
+// `fft_length / 2 + 1` unique components of the FFT: the zero-frequency term,
+// followed by the `fft_length / 2` positive-frequency terms.
+//
+// Along the axis `RFFT` is computed on, if `fft_length` is smaller than the
+// corresponding dimension of `input`, the dimension is cropped. If it is larger,
+// the dimension is padded with zeros.
+//
+// Arguments:
+//	input: A float32 tensor.
+//	fft_length: An int32 tensor of shape [1]. The FFT length.
+//
+// Returns A complex64 tensor of the same rank as `input`. The inner-most
+//   dimension of `input` is replaced with the `fft_length / 2 + 1` unique
+//   frequency components of its 1D Fourier transform.
+//
+// @compatibility(numpy)
+// Equivalent to np.fft.rfft
+// @end_compatibility
+func RFFT(scope *Scope, input tf.Output, fft_length tf.Output) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "RFFT",
+		Input: []tf.Input{
+			input, fft_length,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// OrderedMapStageAttr is an optional argument to OrderedMapStage.
+type OrderedMapStageAttr func(optionalAttr)
+
+// OrderedMapStageCapacity sets the optional capacity attribute to value.
+//
+// value: Maximum number of elements in the Staging Area. If > 0, inserts
+// on the container will block when the capacity is reached.
+// If not specified, defaults to 0
+//
+// REQUIRES: value >= 0
+func OrderedMapStageCapacity(value int64) OrderedMapStageAttr {
+	return func(m optionalAttr) {
+		m["capacity"] = value
+	}
+}
+
+// OrderedMapStageMemoryLimit sets the optional memory_limit attribute to value.
+// If not specified, defaults to 0
+//
+// REQUIRES: value >= 0
+func OrderedMapStageMemoryLimit(value int64) OrderedMapStageAttr {
+	return func(m optionalAttr) {
+		m["memory_limit"] = value
+	}
+}
+
+// OrderedMapStageContainer sets the optional container attribute to value.
+//
+// value: If non-empty, this queue is placed in the given container. Otherwise,
+// a default container is used.
+// If not specified, defaults to ""
+func OrderedMapStageContainer(value string) OrderedMapStageAttr {
 	return func(m optionalAttr) {
-		m["full_matrices"] = value
+		m["container"] = value
 	}
 }
 
-// Computes the singular value decompositions of one or more matrices.
+// OrderedMapStageSharedName sets the optional shared_name attribute to value.
 //
-// Computes the SVD of each inner matrix in `input` such that
-// `input[..., :, :] = u[..., :, :] * diag(s[..., :, :]) * transpose(v[..., :, :])`
+// value: It is necessary to match this name to the matching Unstage Op.
+// If not specified, defaults to ""
+func OrderedMapStageSharedName(value string) OrderedMapStageAttr {
+	return func(m optionalAttr) {
+		m["shared_name"] = value
+	}
+}
+
+// Stage (key, values) in the underlying container which behaves like a ordered
 //
-// ```python
-// # a is a tensor containing a batch of matrices.
-// # s is a tensor of singular values for each matrix.
-// # u is the tensor containing of left singular vectors for each matrix.
-// # v is the tensor containing of right singular vectors for each matrix.
-// s, u, v = svd(a)
-// s, _, _ = svd(a, compute_uv=False)
-// ```
+// associative container.   Elements are ordered by key.
 //
 // Arguments:
-//	input: A tensor of shape `[..., M, N]` whose inner-most 2 dimensions
-// form matrices of size `[M, N]`. Let `P` be the minimum of `M` and `N`.
+//	key: int64
 //
-// Returns Singular values. Shape is `[..., P]`.Left singular vectors. If `full_matrices` is `False` then shape is
-// `[..., M, P]`; if `full_matrices` is `True` then shape is
-// `[..., M, M]`. Undefined if `compute_uv` is `False`.Left singular vectors. If `full_matrices` is `False` then shape is
-// `[..., N, P]`. If `full_matrices` is `True` then shape is `[..., N, N]`.
-// Undefined if `compute_uv` is false.
-func Svd(scope *Scope, input tf.Output, optional ...SvdAttr) (s tf.Output, u tf.Output, v tf.Output) {
+//	values: a list of tensors
+// dtypes A list of data types that inserted values should adhere to.
+//
+//
+// Returns the created operation.
+func OrderedMapStage(scope *Scope, key tf.Output, indices tf.Output, values []tf.Output, dtypes []tf.DataType, optional ...OrderedMapStageAttr) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
+	attrs := map[string]interface{}{"dtypes": dtypes}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "Svd",
+		Type: "OrderedMapStage",
 		Input: []tf.Input{
-			input,
+			key, indices, tf.OutputList(values),
 		},
 		Attrs: attrs,
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
+	return scope.AddOperation(opspec)
 }
 
-// Computes element-wise population count (a.k.a. popcount, bitsum, bitcount).
-//
-// For each entry in `x`, calculates the number of `1` (on) bits in the binary
-// representation of that entry.
+// Computes the gradient for the tanh of `x` wrt its input.
 //
-// **NOTE**: It is more efficient to first `tf.bitcast` your tensors into
-// `int32` or `int64` and perform the bitcount on the result, than to feed in
-// 8- or 16-bit inputs and then aggregate the resulting counts.
-func PopulationCount(scope *Scope, x tf.Output) (y tf.Output) {
+// Specifically, `grad = dy * (1 - y*y)`, where `y = tanh(x)`, and `dy`
+// is the corresponding input gradient.
+func TanhGrad(scope *Scope, y tf.Output, dy tf.Output) (z tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "PopulationCount",
+		Type: "TanhGrad",
 		Input: []tf.Input{
-			x,
+			y, dy,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// AssertAttr is an optional argument to Assert.
-type AssertAttr func(optionalAttr)
-
-// AssertSummarize sets the optional summarize attribute to value.
+// Outputs all keys and values in the table.
 //
-// value: Print this many entries of each tensor.
-// If not specified, defaults to 3
-func AssertSummarize(value int64) AssertAttr {
-	return func(m optionalAttr) {
-		m["summarize"] = value
-	}
-}
-
-// Asserts that the given condition is true.
+// Arguments:
+//	table_handle: Handle to the table.
 //
-// If `condition` evaluates to false, print the list of tensors in `data`.
-// `summarize` determines how many entries of the tensors to print.
 //
-// Arguments:
-//	condition: The condition to evaluate.
-//	data: The tensors to print out when condition is false.
 //
-// Returns the created operation.
-func Assert(scope *Scope, condition tf.Output, data []tf.Output, optional ...AssertAttr) (o *tf.Operation) {
+// Returns Vector of all keys present in the table.Tensor of all values in the table. Indexed in parallel with `keys`.
+func LookupTableExportV2(scope *Scope, table_handle tf.Output, Tkeys tf.DataType, Tvalues tf.DataType) (keys tf.Output, values tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
+	attrs := map[string]interface{}{"Tkeys": Tkeys, "Tvalues": Tvalues}
 	opspec := tf.OpSpec{
-		Type: "Assert",
+		Type: "LookupTableExportV2",
 		Input: []tf.Input{
-			condition, tf.OutputList(data),
+			table_handle,
 		},
 		Attrs: attrs,
 	}
-	return scope.AddOperation(opspec)
-}
-
-// RandomUniformAttr is an optional argument to RandomUniform.
-type RandomUniformAttr func(optionalAttr)
-
-// RandomUniformSeed sets the optional seed attribute to value.
-//
-// value: If either `seed` or `seed2` are set to be non-zero, the random number
-// generator is seeded by the given seed.  Otherwise, it is seeded by a
-// random seed.
-// If not specified, defaults to 0
-func RandomUniformSeed(value int64) RandomUniformAttr {
-	return func(m optionalAttr) {
-		m["seed"] = value
-	}
-}
-
-// RandomUniformSeed2 sets the optional seed2 attribute to value.
-//
-// value: A second seed to avoid seed collision.
-// If not specified, defaults to 0
-func RandomUniformSeed2(value int64) RandomUniformAttr {
-	return func(m optionalAttr) {
-		m["seed2"] = value
-	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1)
 }
 
-// Outputs random values from a uniform distribution.
+// Converts each string in the input Tensor to its hash mod by a number of buckets.
 //
-// The generated values follow a uniform distribution in the range `[0, 1)`. The
-// lower bound 0 is included in the range, while the upper bound 1 is excluded.
+// The hash function is deterministic on the content of the string within the
+// process and will never change. However, it is not suitable for cryptography.
+// This function may be used when CPU time is scarce and inputs are trusted or
+// unimportant. There is a risk of adversaries constructing inputs that all hash
+// to the same bucket. To prevent this problem, use a strong hash function with
+// `tf.string_to_hash_bucket_strong`.
 //
 // Arguments:
-//	shape: The shape of the output tensor.
-//	dtype: The type of the output.
+//	input: The strings to assign a hash bucket.
+//	num_buckets: The number of buckets.
 //
-// Returns A tensor of the specified shape filled with uniform random values.
-func RandomUniform(scope *Scope, shape tf.Output, dtype tf.DataType, optional ...RandomUniformAttr) (output tf.Output) {
+// Returns A Tensor of the same shape as the input `string_tensor`.
+func StringToHashBucketFast(scope *Scope, input tf.Output, num_buckets int64) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"dtype": dtype}
-	for _, a := range optional {
-		a(attrs)
-	}
+	attrs := map[string]interface{}{"num_buckets": num_buckets}
 	opspec := tf.OpSpec{
-		Type: "RandomUniform",
+		Type: "StringToHashBucketFast",
 		Input: []tf.Input{
-			shape,
+			input,
 		},
 		Attrs: attrs,
 	}
@@ -19377,156 +19580,149 @@ func RandomUniform(scope *Scope, shape tf.Output, dtype tf.DataType, optional ..
 	return op.Output(0)
 }
 
-// ResourceApplyFtrlAttr is an optional argument to ResourceApplyFtrl.
-type ResourceApplyFtrlAttr func(optionalAttr)
+// TensorArrayGatherV3Attr is an optional argument to TensorArrayGatherV3.
+type TensorArrayGatherV3Attr func(optionalAttr)
 
-// ResourceApplyFtrlUseLocking sets the optional use_locking attribute to value.
+// TensorArrayGatherV3ElementShape sets the optional element_shape attribute to value.
 //
-// value: If `True`, updating of the var and accum tensors will be protected
-// by a lock; otherwise the behavior is undefined, but may exhibit less
-// contention.
-// If not specified, defaults to false
-func ResourceApplyFtrlUseLocking(value bool) ResourceApplyFtrlAttr {
+// value: The expected shape of an element, if known. Used to
+// validate the shapes of TensorArray elements. If this shape is not
+// fully specified, gathering zero-size TensorArrays is an error.
+// If not specified, defaults to <unknown_rank:true >
+func TensorArrayGatherV3ElementShape(value tf.Shape) TensorArrayGatherV3Attr {
 	return func(m optionalAttr) {
-		m["use_locking"] = value
+		m["element_shape"] = value
 	}
 }
 
-// Update '*var' according to the Ftrl-proximal scheme.
+// Gather specific elements from the TensorArray into output `value`.
 //
-// accum_new = accum + grad * grad
-// linear += grad - (accum_new^(-lr_power) - accum^(-lr_power)) / lr * var
-// quadratic = 1.0 / (accum_new^(lr_power) * lr) + 2 * l2
-// var = (sign(linear) * l1 - linear) / quadratic if |linear| > l1 else 0.0
-// accum = accum_new
+// All elements selected by `indices` must have the same shape.
 //
 // Arguments:
-//	var_: Should be from a Variable().
-//	accum: Should be from a Variable().
-//	linear: Should be from a Variable().
-//	grad: The gradient.
-//	lr: Scaling factor. Must be a scalar.
-//	l1: L1 regulariation. Must be a scalar.
-//	l2: L2 regulariation. Must be a scalar.
-//	lr_power: Scaling factor. Must be a scalar.
+//	handle: The handle to a TensorArray.
+//	indices: The locations in the TensorArray from which to read tensor elements.
+//	flow_in: A float scalar that enforces proper chaining of operations.
+//	dtype: The type of the elem that is returned.
 //
-// Returns the created operation.
-func ResourceApplyFtrl(scope *Scope, var_ tf.Output, accum tf.Output, linear tf.Output, grad tf.Output, lr tf.Output, l1 tf.Output, l2 tf.Output, lr_power tf.Output, optional ...ResourceApplyFtrlAttr) (o *tf.Operation) {
+// Returns All of the elements in the TensorArray, concatenated along a new
+// axis (the new dimension 0).
+func TensorArrayGatherV3(scope *Scope, handle tf.Output, indices tf.Output, flow_in tf.Output, dtype tf.DataType, optional ...TensorArrayGatherV3Attr) (value tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
+	attrs := map[string]interface{}{"dtype": dtype}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "ResourceApplyFtrl",
+		Type: "TensorArrayGatherV3",
 		Input: []tf.Input{
-			var_, accum, linear, grad, lr, l1, l2, lr_power,
+			handle, indices, flow_in,
 		},
 		Attrs: attrs,
 	}
-	return scope.AddOperation(opspec)
-}
-
-// AnyAttr is an optional argument to Any.
-type AnyAttr func(optionalAttr)
-
-// AnyKeepDims sets the optional keep_dims attribute to value.
-//
-// value: If true, retain reduced dimensions with length 1.
-// If not specified, defaults to false
-func AnyKeepDims(value bool) AnyAttr {
-	return func(m optionalAttr) {
-		m["keep_dims"] = value
-	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// Computes the "logical or" of elements across dimensions of a tensor.
-//
-// Reduces `input` along the dimensions given in `reduction_indices`. Unless
-// `keep_dims` is true, the rank of the tensor is reduced by 1 for each entry in
-// `reduction_indices`. If `keep_dims` is true, the reduced dimensions are
-// retained with length 1.
-//
-// Arguments:
-//	input: The tensor to reduce.
-//	reduction_indices: The dimensions to reduce. Must be in the range
-// `[-rank(input), rank(input))`.
+// Deprecated. Disallowed in GraphDef version >= 2.
 //
-// Returns The reduced tensor.
-func Any(scope *Scope, input tf.Output, reduction_indices tf.Output, optional ...AnyAttr) (output tf.Output) {
+// DEPRECATED at GraphDef version 2: Use AdjustContrastv2 instead
+func AdjustContrast(scope *Scope, images tf.Output, contrast_factor tf.Output, min_value tf.Output, max_value tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "Any",
+		Type: "AdjustContrast",
 		Input: []tf.Input{
-			input, reduction_indices,
+			images, contrast_factor, min_value, max_value,
 		},
-		Attrs: attrs,
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// MaxPoolGradGradAttr is an optional argument to MaxPoolGradGrad.
+type MaxPoolGradGradAttr func(optionalAttr)
+
+// MaxPoolGradGradDataFormat sets the optional data_format attribute to value.
+//
+// value: Specify the data format of the input and output data. With the
+// default format "NHWC", the data is stored in the order of:
+//     [batch, in_height, in_width, in_channels].
+// Alternatively, the format could be "NCHW", the data storage order of:
+//     [batch, in_channels, in_height, in_width].
+// If not specified, defaults to "NHWC"
+func MaxPoolGradGradDataFormat(value string) MaxPoolGradGradAttr {
+	return func(m optionalAttr) {
+		m["data_format"] = value
+	}
 }
 
-// Compute the Hurwitz zeta function \\(\zeta(x, q)\\).
-//
-// The Hurwitz zeta function is defined as:
+// Computes second-order gradients of the maxpooling function.
 //
+// Arguments:
+//	orig_input: The original input tensor.
+//	orig_output: The original output tensor.
+//	grad: 4-D.  Gradients of gradients w.r.t. the input of `max_pool`.
+//	ksize: The size of the window for each dimension of the input tensor.
+//	strides: The stride of the sliding window for each dimension of the
+// input tensor.
+//	padding: The type of padding algorithm to use.
 //
-// \\(\zeta(x, q) = \sum_{n=0}^{\infty} (q + n)^{-x}\\)
-func Zeta(scope *Scope, x tf.Output, q tf.Output) (z tf.Output) {
+// Returns Gradients of gradients w.r.t. the input to `max_pool`.
+func MaxPoolGradGrad(scope *Scope, orig_input tf.Output, orig_output tf.Output, grad tf.Output, ksize []int64, strides []int64, padding string, optional ...MaxPoolGradGradAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"ksize": ksize, "strides": strides, "padding": padding}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "Zeta",
+		Type: "MaxPoolGradGrad",
 		Input: []tf.Input{
-			x, q,
+			orig_input, orig_output, grad,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Inverse real-valued fast Fourier transform.
+// 3D real-valued fast Fourier transform.
 //
-// Computes the inverse 1-dimensional discrete Fourier transform of a real-valued
-// signal over the inner-most dimension of `input`.
+// Computes the 3-dimensional discrete Fourier transform of a real-valued signal
+// over the inner-most 3 dimensions of `input`.
 //
-// The inner-most dimension of `input` is assumed to be the result of `RFFT`: the
-// `fft_length / 2 + 1` unique components of the DFT of a real-valued signal. If
-// `fft_length` is not provided, it is computed from the size of the inner-most
-// dimension of `input` (`fft_length = 2 * (inner - 1)`). If the FFT length used to
-// compute `input` is odd, it should be provided since it cannot be inferred
-// properly.
+// Since the DFT of a real signal is Hermitian-symmetric, `RFFT3D` only returns the
+// `fft_length / 2 + 1` unique components of the FFT for the inner-most dimension
+// of `output`: the zero-frequency term, followed by the `fft_length / 2`
+// positive-frequency terms.
 //
-// Along the axis `IRFFT` is computed on, if `fft_length / 2 + 1` is smaller
-// than the corresponding dimension of `input`, the dimension is cropped. If it is
-// larger, the dimension is padded with zeros.
+// Along each axis `RFFT3D` is computed on, if `fft_length` is smaller than the
+// corresponding dimension of `input`, the dimension is cropped. If it is larger,
+// the dimension is padded with zeros.
 //
 // Arguments:
-//	input: A complex64 tensor.
-//	fft_length: An int32 tensor of shape [1]. The FFT length.
+//	input: A float32 tensor.
+//	fft_length: An int32 tensor of shape [3]. The FFT length for each dimension.
 //
-// Returns A float32 tensor of the same rank as `input`. The inner-most
-//   dimension of `input` is replaced with the `fft_length` samples of its inverse
-//   1D Fourier transform.
+// Returns A complex64 tensor of the same rank as `input`. The inner-most 3
+//   dimensions of `input` are replaced with the their 3D Fourier transform. The
+//   inner-most dimension contains `fft_length / 2 + 1` unique frequency
+//   components.
 //
 // @compatibility(numpy)
-// Equivalent to np.fft.irfft
+// Equivalent to np.fft.rfftn with 3 dimensions.
 // @end_compatibility
-func IRFFT(scope *Scope, input tf.Output, fft_length tf.Output) (output tf.Output) {
+func RFFT3D(scope *Scope, input tf.Output, fft_length tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "IRFFT",
+		Type: "RFFT3D",
 		Input: []tf.Input{
 			input, fft_length,
 		},
@@ -19535,111 +19731,85 @@ func IRFFT(scope *Scope, input tf.Output, fft_length tf.Output) (output tf.Outpu
 	return op.Output(0)
 }
 
-// Creates a dataset with a range of values. Corresponds to python's xrange.
-//
-// Arguments:
-//	start: corresponds to start in python's xrange().
-//	stop: corresponds to stop in python's xrange().
-//	step: corresponds to step in python's xrange().
-//
+// RestoreSliceAttr is an optional argument to RestoreSlice.
+type RestoreSliceAttr func(optionalAttr)
+
+// RestoreSlicePreferredShard sets the optional preferred_shard attribute to value.
 //
-func RangeDataset(scope *Scope, start tf.Output, stop tf.Output, step tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
-	opspec := tf.OpSpec{
-		Type: "RangeDataset",
-		Input: []tf.Input{
-			start, stop, step,
-		},
-		Attrs: attrs,
+// value: Index of file to open first if multiple files match
+// `file_pattern`. See the documentation for `Restore`.
+// If not specified, defaults to -1
+func RestoreSlicePreferredShard(value int64) RestoreSliceAttr {
+	return func(m optionalAttr) {
+		m["preferred_shard"] = value
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
 }
 
-// Saves tensors in V2 checkpoint format.
+// Restores a tensor from checkpoint files.
 //
-// By default, saves the named tensors in full.  If the caller wishes to save
-// specific slices of full tensors, "shape_and_slices" should be non-empty strings
-// and correspondingly well-formed.
+// This is like `Restore` except that restored tensor can be listed as filling
+// only a slice of a larger tensor.  `shape_and_slice` specifies the shape of the
+// larger tensor and the slice that the restored tensor covers.
+//
+// The `shape_and_slice` input has the same format as the
+// elements of the `shapes_and_slices` input of the `SaveSlices` op.
 //
 // Arguments:
-//	prefix: Must have a single element. The prefix of the V2 checkpoint to which we
-// write the tensors.
-//	tensor_names: shape {N}. The names of the tensors to be saved.
-//	shape_and_slices: shape {N}.  The slice specs of the tensors to be saved.
-// Empty strings indicate that they are non-partitioned tensors.
-//	tensors: `N` tensors to save.
+//	file_pattern: Must have a single element. The pattern of the files from
+// which we read the tensor.
+//	tensor_name: Must have a single element. The name of the tensor to be
+// restored.
+//	shape_and_slice: Scalar. The shapes and slice specifications to use when
+// restoring a tensors.
+//	dt: The type of the tensor to be restored.
 //
-// Returns the created operation.
-func SaveV2(scope *Scope, prefix tf.Output, tensor_names tf.Output, shape_and_slices tf.Output, tensors []tf.Output) (o *tf.Operation) {
+// Returns The restored tensor.
+func RestoreSlice(scope *Scope, file_pattern tf.Output, tensor_name tf.Output, shape_and_slice tf.Output, dt tf.DataType, optional ...RestoreSliceAttr) (tensor tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"dt": dt}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "SaveV2",
+		Type: "RestoreSlice",
 		Input: []tf.Input{
-			prefix, tensor_names, shape_and_slices, tf.OutputList(tensors),
+			file_pattern, tensor_name, shape_and_slice,
 		},
+		Attrs: attrs,
 	}
-	return scope.AddOperation(opspec)
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// MatrixTriangularSolveAttr is an optional argument to MatrixTriangularSolve.
-type MatrixTriangularSolveAttr func(optionalAttr)
-
-// MatrixTriangularSolveLower sets the optional lower attribute to value.
-//
-// value: Boolean indicating whether the innermost matrices in `matrix` are
-// lower or upper triangular.
-// If not specified, defaults to true
-func MatrixTriangularSolveLower(value bool) MatrixTriangularSolveAttr {
-	return func(m optionalAttr) {
-		m["lower"] = value
-	}
-}
+// StatelessTruncatedNormalAttr is an optional argument to StatelessTruncatedNormal.
+type StatelessTruncatedNormalAttr func(optionalAttr)
 
-// MatrixTriangularSolveAdjoint sets the optional adjoint attribute to value.
-//
-// value: Boolean indicating whether to solve with `matrix` or its (block-wise)
-//          adjoint.
+// StatelessTruncatedNormalDtype sets the optional dtype attribute to value.
 //
-// @compatibility(numpy)
-// Equivalent to np.linalg.triangular_solve
-// @end_compatibility
-// If not specified, defaults to false
-func MatrixTriangularSolveAdjoint(value bool) MatrixTriangularSolveAttr {
+// value: The type of the output.
+// If not specified, defaults to DT_FLOAT
+func StatelessTruncatedNormalDtype(value tf.DataType) StatelessTruncatedNormalAttr {
 	return func(m optionalAttr) {
-		m["adjoint"] = value
+		m["dtype"] = value
 	}
 }
 
-// Solves systems of linear equations with upper or lower triangular matrices by
-//
-// backsubstitution.
+// Outputs deterministic pseudorandom values from a truncated normal distribution.
 //
-// `matrix` is a tensor of shape `[..., M, M]` whose inner-most 2 dimensions form
-// square matrices. If `lower` is `True` then the strictly upper triangular part
-// of each inner-most matrix is assumed to be zero and not accessed.
-// If `lower` is False then the strictly lower triangular part of each inner-most
-// matrix is assumed to be zero and not accessed.
-// `rhs` is a tensor of shape `[..., M, K]`.
+// The generated values follow a normal distribution with mean 0 and standard
+// deviation 1, except that values whose magnitude is more than 2 standard
+// deviations from the mean are dropped and re-picked.
 //
-// The output is a tensor of shape `[..., M, K]`. If `adjoint` is
-// `True` then the innermost matrices in `output` satisfy matrix equations
-// `matrix[..., :, :] * output[..., :, :] = rhs[..., :, :]`.
-// If `adjoint` is `False` then the strictly then the  innermost matrices in
-// `output` satisfy matrix equations
-// `adjoint(matrix[..., i, k]) * output[..., k, j] = rhs[..., i, j]`.
+// The outputs are a deterministic function of `shape` and `seed`.
 //
 // Arguments:
-//	matrix: Shape is `[..., M, M]`.
-//	rhs: Shape is `[..., M, K]`.
+//	shape: The shape of the output tensor.
+//	seed: 2 seeds (shape [2]).
 //
-// Returns Shape is `[..., M, K]`.
-func MatrixTriangularSolve(scope *Scope, matrix tf.Output, rhs tf.Output, optional ...MatrixTriangularSolveAttr) (output tf.Output) {
+// Returns Random values with specified shape.
+func StatelessTruncatedNormal(scope *Scope, shape tf.Output, seed tf.Output, optional ...StatelessTruncatedNormalAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -19648,9 +19818,9 @@ func MatrixTriangularSolve(scope *Scope, matrix tf.Output, rhs tf.Output, option
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "MatrixTriangularSolve",
+		Type: "StatelessTruncatedNormal",
 		Input: []tf.Input{
-			matrix, rhs,
+			shape, seed,
 		},
 		Attrs: attrs,
 	}
@@ -19658,338 +19828,282 @@ func MatrixTriangularSolve(scope *Scope, matrix tf.Output, rhs tf.Output, option
 	return op.Output(0)
 }
 
-// Adds a value to the current value of a variable.
+// UniqueWithCountsAttr is an optional argument to UniqueWithCounts.
+type UniqueWithCountsAttr func(optionalAttr)
+
+// UniqueWithCountsOutIdx sets the optional out_idx attribute to value.
+// If not specified, defaults to DT_INT32
+func UniqueWithCountsOutIdx(value tf.DataType) UniqueWithCountsAttr {
+	return func(m optionalAttr) {
+		m["out_idx"] = value
+	}
+}
+
+// Finds unique elements in a 1-D tensor.
 //
-// Any ReadVariableOp which depends directly or indirectly on this assign is
-// guaranteed to see the incremented value or a subsequent newer one.
+// This operation returns a tensor `y` containing all of the unique elements of `x`
+// sorted in the same order that they occur in `x`. This operation also returns a
+// tensor `idx` the same size as `x` that contains the index of each value of `x`
+// in the unique output `y`. Finally, it returns a third tensor `count` that
+// contains the count of each element of `y` in `x`. In other words:
 //
-// Outputs the incremented value, which can be used to totally order the
-// increments to this variable.
+// `y[idx[i]] = x[i] for i in [0, 1,...,rank(x) - 1]`
 //
-// Arguments:
-//	resource: handle to the resource in which to store the variable.
-//	value: the value by which the variable will be incremented.
+// For example:
 //
-// Returns the created operation.
-func AssignAddVariableOp(scope *Scope, resource tf.Output, value tf.Output) (o *tf.Operation) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "AssignAddVariableOp",
-		Input: []tf.Input{
-			resource, value,
-		},
-	}
-	return scope.AddOperation(opspec)
-}
-
-// Computes inverse hyperbolic sine of x element-wise.
-func Asinh(scope *Scope, x tf.Output) (y tf.Output) {
+// ```
+// # tensor 'x' is [1, 1, 2, 4, 4, 4, 7, 8, 8]
+// y, idx, count = unique_with_counts(x)
+// y ==> [1, 2, 4, 7, 8]
+// idx ==> [0, 0, 1, 2, 2, 2, 3, 4, 4]
+// count ==> [2, 1, 3, 1, 2]
+// ```
+//
+// Arguments:
+//	x: 1-D.
+//
+// Returns 1-D.1-D.1-D.
+func UniqueWithCounts(scope *Scope, x tf.Output, optional ...UniqueWithCountsAttr) (y tf.Output, idx tf.Output, count tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "Asinh",
+		Type: "UniqueWithCounts",
 		Input: []tf.Input{
 			x,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// Real-valued fast Fourier transform.
-//
-// Computes the 1-dimensional discrete Fourier transform of a real-valued signal
-// over the inner-most dimension of `input`.
-//
-// Since the DFT of a real signal is Hermitian-symmetric, `RFFT` only returns the
-// `fft_length / 2 + 1` unique components of the FFT: the zero-frequency term,
-// followed by the `fft_length / 2` positive-frequency terms.
-//
-// Along the axis `RFFT` is computed on, if `fft_length` is smaller than the
-// corresponding dimension of `input`, the dimension is cropped. If it is larger,
-// the dimension is padded with zeros.
+// Creates a dataset that skips `count` elements from the `input_dataset`.
 //
 // Arguments:
-//	input: A float32 tensor.
-//	fft_length: An int32 tensor of shape [1]. The FFT length.
 //
-// Returns A complex64 tensor of the same rank as `input`. The inner-most
-//   dimension of `input` is replaced with the `fft_length / 2 + 1` unique
-//   frequency components of its 1D Fourier transform.
+//	count: A scalar representing the number of elements from the `input_dataset`
+// that should be skipped.  If count is -1, skips everything.
 //
-// @compatibility(numpy)
-// Equivalent to np.fft.rfft
-// @end_compatibility
-func RFFT(scope *Scope, input tf.Output, fft_length tf.Output) (output tf.Output) {
+//
+func SkipDataset(scope *Scope, input_dataset tf.Output, count tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
 	opspec := tf.OpSpec{
-		Type: "RFFT",
+		Type: "SkipDataset",
 		Input: []tf.Input{
-			input, fft_length,
+			input_dataset, count,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// OrderedMapStageAttr is an optional argument to OrderedMapStage.
-type OrderedMapStageAttr func(optionalAttr)
-
-// OrderedMapStageCapacity sets the optional capacity attribute to value.
-//
-// value: Maximum number of elements in the Staging Area. If > 0, inserts
-// on the container will block when the capacity is reached.
-// If not specified, defaults to 0
-//
-// REQUIRES: value >= 0
-func OrderedMapStageCapacity(value int64) OrderedMapStageAttr {
-	return func(m optionalAttr) {
-		m["capacity"] = value
-	}
-}
-
-// OrderedMapStageMemoryLimit sets the optional memory_limit attribute to value.
-// If not specified, defaults to 0
-//
-// REQUIRES: value >= 0
-func OrderedMapStageMemoryLimit(value int64) OrderedMapStageAttr {
-	return func(m optionalAttr) {
-		m["memory_limit"] = value
-	}
-}
-
-// OrderedMapStageContainer sets the optional container attribute to value.
-//
-// value: If non-empty, this queue is placed in the given container. Otherwise,
-// a default container is used.
-// If not specified, defaults to ""
-func OrderedMapStageContainer(value string) OrderedMapStageAttr {
-	return func(m optionalAttr) {
-		m["container"] = value
-	}
-}
+// ComplexAttr is an optional argument to Complex.
+type ComplexAttr func(optionalAttr)
 
-// OrderedMapStageSharedName sets the optional shared_name attribute to value.
-//
-// value: It is necessary to match this name to the matching Unstage Op.
-// If not specified, defaults to ""
-func OrderedMapStageSharedName(value string) OrderedMapStageAttr {
+// ComplexTout sets the optional Tout attribute to value.
+// If not specified, defaults to DT_COMPLEX64
+func ComplexTout(value tf.DataType) ComplexAttr {
 	return func(m optionalAttr) {
-		m["shared_name"] = value
+		m["Tout"] = value
 	}
 }
 
-// Stage (key, values) in the underlying container which behaves like a ordered
-//
-// associative container.   Elements are ordered by key.
+// Converts two real numbers to a complex number.
 //
-// Arguments:
-//	key: int64
+// Given a tensor `real` representing the real part of a complex number, and a
+// tensor `imag` representing the imaginary part of a complex number, this
+// operation returns complex numbers elementwise of the form \\(a + bj\\), where
+// *a* represents the `real` part and *b* represents the `imag` part.
 //
-//	values: a list of tensors
-// dtypes A list of data types that inserted values should adhere to.
+// The input tensors `real` and `imag` must have the same shape.
 //
+// For example:
 //
-// Returns the created operation.
-func OrderedMapStage(scope *Scope, key tf.Output, indices tf.Output, values []tf.Output, dtypes []tf.DataType, optional ...OrderedMapStageAttr) (o *tf.Operation) {
+// ```
+// # tensor 'real' is [2.25, 3.25]
+// # tensor `imag` is [4.75, 5.75]
+// tf.complex(real, imag) ==> [[2.25 + 4.75j], [3.25 + 5.75j]]
+// ```
+func Complex(scope *Scope, real tf.Output, imag tf.Output, optional ...ComplexAttr) (out tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"dtypes": dtypes}
+	attrs := map[string]interface{}{}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "OrderedMapStage",
+		Type: "Complex",
 		Input: []tf.Input{
-			key, indices, tf.OutputList(values),
+			real, imag,
 		},
 		Attrs: attrs,
 	}
-	return scope.AddOperation(opspec)
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// Computes the gradient for the tanh of `x` wrt its input.
+// ImagAttr is an optional argument to Imag.
+type ImagAttr func(optionalAttr)
+
+// ImagTout sets the optional Tout attribute to value.
+// If not specified, defaults to DT_FLOAT
+func ImagTout(value tf.DataType) ImagAttr {
+	return func(m optionalAttr) {
+		m["Tout"] = value
+	}
+}
+
+// Returns the imaginary part of a complex number.
 //
-// Specifically, `grad = dy * (1 - y*y)`, where `y = tanh(x)`, and `dy`
-// is the corresponding input gradient.
-func TanhGrad(scope *Scope, y tf.Output, dy tf.Output) (z tf.Output) {
+// Given a tensor `input` of complex numbers, this operation returns a tensor of
+// type `float` that is the imaginary part of each element in `input`. All
+// elements in `input` must be complex numbers of the form \\(a + bj\\), where *a*
+// is the real part and *b* is the imaginary part returned by this operation.
+//
+// For example:
+//
+// ```
+// # tensor 'input' is [-2.25 + 4.75j, 3.25 + 5.75j]
+// tf.imag(input) ==> [4.75, 5.75]
+// ```
+func Imag(scope *Scope, input tf.Output, optional ...ImagAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "TanhGrad",
+		Type: "Imag",
 		Input: []tf.Input{
-			y, dy,
+			input,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Outputs all keys and values in the table.
+// Creates a dataset that emits the lines of one or more text files.
 //
 // Arguments:
-//	table_handle: Handle to the table.
-//
-//
-//
-// Returns Vector of all keys present in the table.Tensor of all values in the table. Indexed in parallel with `keys`.
-func LookupTableExportV2(scope *Scope, table_handle tf.Output, Tkeys tf.DataType, Tvalues tf.DataType) (keys tf.Output, values tf.Output) {
+//	filenames: A scalar or a vector containing the name(s) of the file(s) to be
+// read.
+//	compression_type: A scalar containing either (i) the empty string (no
+// compression), (ii) "ZLIB", or (iii) "GZIP".
+//	buffer_size: A scalar containing the number of bytes to buffer.
+func TextLineDataset(scope *Scope, filenames tf.Output, compression_type tf.Output, buffer_size tf.Output) (handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"Tkeys": Tkeys, "Tvalues": Tvalues}
 	opspec := tf.OpSpec{
-		Type: "LookupTableExportV2",
+		Type: "TextLineDataset",
 		Input: []tf.Input{
-			table_handle,
+			filenames, compression_type, buffer_size,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1)
+	return op.Output(0)
 }
 
-// Converts each string in the input Tensor to its hash mod by a number of buckets.
+// Returns the number of records this Reader has produced.
 //
-// The hash function is deterministic on the content of the string within the
-// process and will never change. However, it is not suitable for cryptography.
-// This function may be used when CPU time is scarce and inputs are trusted or
-// unimportant. There is a risk of adversaries constructing inputs that all hash
-// to the same bucket. To prevent this problem, use a strong hash function with
-// `tf.string_to_hash_bucket_strong`.
+// This is the same as the number of ReaderRead executions that have
+// succeeded.
 //
 // Arguments:
-//	input: The strings to assign a hash bucket.
-//	num_buckets: The number of buckets.
-//
-// Returns A Tensor of the same shape as the input `string_tensor`.
-func StringToHashBucketFast(scope *Scope, input tf.Output, num_buckets int64) (output tf.Output) {
+//	reader_handle: Handle to a Reader.
+func ReaderNumRecordsProducedV2(scope *Scope, reader_handle tf.Output) (records_produced tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"num_buckets": num_buckets}
 	opspec := tf.OpSpec{
-		Type: "StringToHashBucketFast",
+		Type: "ReaderNumRecordsProducedV2",
 		Input: []tf.Input{
-			input,
+			reader_handle,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// TensorArrayGatherV3Attr is an optional argument to TensorArrayGatherV3.
-type TensorArrayGatherV3Attr func(optionalAttr)
-
-// TensorArrayGatherV3ElementShape sets the optional element_shape attribute to value.
-//
-// value: The expected shape of an element, if known. Used to
-// validate the shapes of TensorArray elements. If this shape is not
-// fully specified, gathering zero-size TensorArrays is an error.
-// If not specified, defaults to <unknown_rank:true >
-func TensorArrayGatherV3ElementShape(value tf.Shape) TensorArrayGatherV3Attr {
-	return func(m optionalAttr) {
-		m["element_shape"] = value
-	}
-}
-
-// Gather specific elements from the TensorArray into output `value`.
-//
-// All elements selected by `indices` must have the same shape.
-//
-// Arguments:
-//	handle: The handle to a TensorArray.
-//	indices: The locations in the TensorArray from which to read tensor elements.
-//	flow_in: A float scalar that enforces proper chaining of operations.
-//	dtype: The type of the elem that is returned.
+// Computes exponential of x - 1 element-wise.
 //
-// Returns All of the elements in the TensorArray, concatenated along a new
-// axis (the new dimension 0).
-func TensorArrayGatherV3(scope *Scope, handle tf.Output, indices tf.Output, flow_in tf.Output, dtype tf.DataType, optional ...TensorArrayGatherV3Attr) (value tf.Output) {
+// I.e., \\(y = (\exp x) - 1\\).
+func Expm1(scope *Scope, x tf.Output) (y tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"dtype": dtype}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "TensorArrayGatherV3",
+		Type: "Expm1",
 		Input: []tf.Input{
-			handle, indices, flow_in,
+			x,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Deprecated. Disallowed in GraphDef version >= 2.
+// Returns x - y element-wise.
 //
-// DEPRECATED at GraphDef version 2: Use AdjustContrastv2 instead
-func AdjustContrast(scope *Scope, images tf.Output, contrast_factor tf.Output, min_value tf.Output, max_value tf.Output) (output tf.Output) {
+// *NOTE*: `Sub` supports broadcasting. More about broadcasting
+// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
+func Sub(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "AdjustContrast",
+		Type: "Sub",
 		Input: []tf.Input{
-			images, contrast_factor, min_value, max_value,
+			x, y,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// MaxPoolGradGradAttr is an optional argument to MaxPoolGradGrad.
-type MaxPoolGradGradAttr func(optionalAttr)
+// StringToNumberAttr is an optional argument to StringToNumber.
+type StringToNumberAttr func(optionalAttr)
 
-// MaxPoolGradGradDataFormat sets the optional data_format attribute to value.
+// StringToNumberOutType sets the optional out_type attribute to value.
 //
-// value: Specify the data format of the input and output data. With the
-// default format "NHWC", the data is stored in the order of:
-//     [batch, in_height, in_width, in_channels].
-// Alternatively, the format could be "NCHW", the data storage order of:
-//     [batch, in_channels, in_height, in_width].
-// If not specified, defaults to "NHWC"
-func MaxPoolGradGradDataFormat(value string) MaxPoolGradGradAttr {
+// value: The numeric type to interpret each string in `string_tensor` as.
+// If not specified, defaults to DT_FLOAT
+func StringToNumberOutType(value tf.DataType) StringToNumberAttr {
 	return func(m optionalAttr) {
-		m["data_format"] = value
+		m["out_type"] = value
 	}
 }
 
-// Computes second-order gradients of the maxpooling function.
+// Converts each string in the input Tensor to the specified numeric type.
 //
-// Arguments:
-//	orig_input: The original input tensor.
-//	orig_output: The original output tensor.
-//	grad: 4-D.  Gradients of gradients w.r.t. the input of `max_pool`.
-//	ksize: The size of the window for each dimension of the input tensor.
-//	strides: The stride of the sliding window for each dimension of the
-// input tensor.
-//	padding: The type of padding algorithm to use.
+// (Note that int32 overflow results in an error while float overflow
+// results in a rounded value.)
 //
-// Returns Gradients of gradients w.r.t. the input to `max_pool`.
-func MaxPoolGradGrad(scope *Scope, orig_input tf.Output, orig_output tf.Output, grad tf.Output, ksize []int64, strides []int64, padding string, optional ...MaxPoolGradGradAttr) (output tf.Output) {
+// Returns A Tensor of the same shape as the input `string_tensor`.
+func StringToNumber(scope *Scope, string_tensor tf.Output, optional ...StringToNumberAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"ksize": ksize, "strides": strides, "padding": padding}
+	attrs := map[string]interface{}{}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "MaxPoolGradGrad",
+		Type: "StringToNumber",
 		Input: []tf.Input{
-			orig_input, orig_output, grad,
+			string_tensor,
 		},
 		Attrs: attrs,
 	}
@@ -19997,91 +20111,84 @@ func MaxPoolGradGrad(scope *Scope, orig_input tf.Output, orig_output tf.Output,
 	return op.Output(0)
 }
 
-// 3D real-valued fast Fourier transform.
-//
-// Computes the 3-dimensional discrete Fourier transform of a real-valued signal
-// over the inner-most 3 dimensions of `input`.
-//
-// Since the DFT of a real signal is Hermitian-symmetric, `RFFT3D` only returns the
-// `fft_length / 2 + 1` unique components of the FFT for the inner-most dimension
-// of `output`: the zero-frequency term, followed by the `fft_length / 2`
-// positive-frequency terms.
-//
-// Along each axis `RFFT3D` is computed on, if `fft_length` is smaller than the
-// corresponding dimension of `input`, the dimension is cropped. If it is larger,
-// the dimension is padded with zeros.
-//
-// Arguments:
-//	input: A float32 tensor.
-//	fft_length: An int32 tensor of shape [3]. The FFT length for each dimension.
-//
-// Returns A complex64 tensor of the same rank as `input`. The inner-most 3
-//   dimensions of `input` are replaced with the their 3D Fourier transform. The
-//   inner-most dimension contains `fft_length / 2 + 1` unique frequency
-//   components.
-//
-// @compatibility(numpy)
-// Equivalent to np.fft.rfftn with 3 dimensions.
-// @end_compatibility
-func RFFT3D(scope *Scope, input tf.Output, fft_length tf.Output) (output tf.Output) {
+// Returns the truth value of NOT x element-wise.
+func LogicalNot(scope *Scope, x tf.Output) (y tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "RFFT3D",
+		Type: "LogicalNot",
 		Input: []tf.Input{
-			input, fft_length,
+			x,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// RestoreSliceAttr is an optional argument to RestoreSlice.
-type RestoreSliceAttr func(optionalAttr)
+// LRNGradAttr is an optional argument to LRNGrad.
+type LRNGradAttr func(optionalAttr)
 
-// RestoreSlicePreferredShard sets the optional preferred_shard attribute to value.
+// LRNGradDepthRadius sets the optional depth_radius attribute to value.
 //
-// value: Index of file to open first if multiple files match
-// `file_pattern`. See the documentation for `Restore`.
-// If not specified, defaults to -1
-func RestoreSlicePreferredShard(value int64) RestoreSliceAttr {
+// value: A depth radius.
+// If not specified, defaults to 5
+func LRNGradDepthRadius(value int64) LRNGradAttr {
 	return func(m optionalAttr) {
-		m["preferred_shard"] = value
+		m["depth_radius"] = value
 	}
 }
 
-// Restores a tensor from checkpoint files.
+// LRNGradBias sets the optional bias attribute to value.
 //
-// This is like `Restore` except that restored tensor can be listed as filling
-// only a slice of a larger tensor.  `shape_and_slice` specifies the shape of the
-// larger tensor and the slice that the restored tensor covers.
+// value: An offset (usually > 0 to avoid dividing by 0).
+// If not specified, defaults to 1
+func LRNGradBias(value float32) LRNGradAttr {
+	return func(m optionalAttr) {
+		m["bias"] = value
+	}
+}
+
+// LRNGradAlpha sets the optional alpha attribute to value.
 //
-// The `shape_and_slice` input has the same format as the
-// elements of the `shapes_and_slices` input of the `SaveSlices` op.
+// value: A scale factor, usually positive.
+// If not specified, defaults to 1
+func LRNGradAlpha(value float32) LRNGradAttr {
+	return func(m optionalAttr) {
+		m["alpha"] = value
+	}
+}
+
+// LRNGradBeta sets the optional beta attribute to value.
+//
+// value: An exponent.
+// If not specified, defaults to 0.5
+func LRNGradBeta(value float32) LRNGradAttr {
+	return func(m optionalAttr) {
+		m["beta"] = value
+	}
+}
+
+// Gradients for Local Response Normalization.
 //
 // Arguments:
-//	file_pattern: Must have a single element. The pattern of the files from
-// which we read the tensor.
-//	tensor_name: Must have a single element. The name of the tensor to be
-// restored.
-//	shape_and_slice: Scalar. The shapes and slice specifications to use when
-// restoring a tensors.
-//	dt: The type of the tensor to be restored.
+//	input_grads: 4-D with shape `[batch, height, width, channels]`.
+//	input_image: 4-D with shape `[batch, height, width, channels]`.
+//	output_image: 4-D with shape `[batch, height, width, channels]`.
 //
-// Returns The restored tensor.
-func RestoreSlice(scope *Scope, file_pattern tf.Output, tensor_name tf.Output, shape_and_slice tf.Output, dt tf.DataType, optional ...RestoreSliceAttr) (tensor tf.Output) {
+// Returns The gradients for LRN.
+func LRNGrad(scope *Scope, input_grads tf.Output, input_image tf.Output, output_image tf.Output, optional ...LRNGradAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"dt": dt}
+	attrs := map[string]interface{}{}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "RestoreSlice",
+		Type: "LRNGrad",
 		Input: []tf.Input{
-			file_pattern, tensor_name, shape_and_slice,
+			input_grads, input_image, output_image,
 		},
 		Attrs: attrs,
 	}
@@ -20089,33 +20196,38 @@ func RestoreSlice(scope *Scope, file_pattern tf.Output, tensor_name tf.Output, s
 	return op.Output(0)
 }
 
-// StatelessTruncatedNormalAttr is an optional argument to StatelessTruncatedNormal.
-type StatelessTruncatedNormalAttr func(optionalAttr)
+// EncodePngAttr is an optional argument to EncodePng.
+type EncodePngAttr func(optionalAttr)
 
-// StatelessTruncatedNormalDtype sets the optional dtype attribute to value.
+// EncodePngCompression sets the optional compression attribute to value.
 //
-// value: The type of the output.
-// If not specified, defaults to DT_FLOAT
-func StatelessTruncatedNormalDtype(value tf.DataType) StatelessTruncatedNormalAttr {
+// value: Compression level.
+// If not specified, defaults to -1
+func EncodePngCompression(value int64) EncodePngAttr {
 	return func(m optionalAttr) {
-		m["dtype"] = value
+		m["compression"] = value
 	}
 }
 
-// Outputs deterministic pseudorandom values from a truncated normal distribution.
+// PNG-encode an image.
 //
-// The generated values follow a normal distribution with mean 0 and standard
-// deviation 1, except that values whose magnitude is more than 2 standard
-// deviations from the mean are dropped and re-picked.
+// `image` is a 3-D uint8 or uint16 Tensor of shape `[height, width, channels]`
+// where `channels` is:
 //
-// The outputs are a deterministic function of `shape` and `seed`.
+// *   1: for grayscale.
+// *   2: for grayscale + alpha.
+// *   3: for RGB.
+// *   4: for RGBA.
+//
+// The ZLIB compression level, `compression`, can be -1 for the PNG-encoder
+// default or a value from 0 to 9.  9 is the highest compression level, generating
+// the smallest output, but is slower.
 //
 // Arguments:
-//	shape: The shape of the output tensor.
-//	seed: 2 seeds (shape [2]).
+//	image: 3-D with shape `[height, width, channels]`.
 //
-// Returns Random values with specified shape.
-func StatelessTruncatedNormal(scope *Scope, shape tf.Output, seed tf.Output, optional ...StatelessTruncatedNormalAttr) (output tf.Output) {
+// Returns 0-D. PNG-encoded image.
+func EncodePng(scope *Scope, image tf.Output, optional ...EncodePngAttr) (contents tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -20124,9 +20236,9 @@ func StatelessTruncatedNormal(scope *Scope, shape tf.Output, seed tf.Output, opt
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "StatelessTruncatedNormal",
+		Type: "EncodePng",
 		Input: []tf.Input{
-			shape, seed,
+			image,
 		},
 		Attrs: attrs,
 	}
@@ -20134,166 +20246,170 @@ func StatelessTruncatedNormal(scope *Scope, shape tf.Output, seed tf.Output, opt
 	return op.Output(0)
 }
 
-// UniqueWithCountsAttr is an optional argument to UniqueWithCounts.
-type UniqueWithCountsAttr func(optionalAttr)
+// MaxPoolAttr is an optional argument to MaxPool.
+type MaxPoolAttr func(optionalAttr)
 
-// UniqueWithCountsOutIdx sets the optional out_idx attribute to value.
-// If not specified, defaults to DT_INT32
-func UniqueWithCountsOutIdx(value tf.DataType) UniqueWithCountsAttr {
+// MaxPoolDataFormat sets the optional data_format attribute to value.
+//
+// value: Specify the data format of the input and output data. With the
+// default format "NHWC", the data is stored in the order of:
+//     [batch, in_height, in_width, in_channels].
+// Alternatively, the format could be "NCHW", the data storage order of:
+//     [batch, in_channels, in_height, in_width].
+// If not specified, defaults to "NHWC"
+func MaxPoolDataFormat(value string) MaxPoolAttr {
 	return func(m optionalAttr) {
-		m["out_idx"] = value
+		m["data_format"] = value
 	}
 }
 
-// Finds unique elements in a 1-D tensor.
-//
-// This operation returns a tensor `y` containing all of the unique elements of `x`
-// sorted in the same order that they occur in `x`. This operation also returns a
-// tensor `idx` the same size as `x` that contains the index of each value of `x`
-// in the unique output `y`. Finally, it returns a third tensor `count` that
-// contains the count of each element of `y` in `x`. In other words:
-//
-// `y[idx[i]] = x[i] for i in [0, 1,...,rank(x) - 1]`
-//
-// For example:
-//
-// ```
-// # tensor 'x' is [1, 1, 2, 4, 4, 4, 7, 8, 8]
-// y, idx, count = unique_with_counts(x)
-// y ==> [1, 2, 4, 7, 8]
-// idx ==> [0, 0, 1, 2, 2, 2, 3, 4, 4]
-// count ==> [2, 1, 3, 1, 2]
-// ```
+// Performs max pooling on the input.
 //
 // Arguments:
-//	x: 1-D.
+//	input: 4-D input to pool over.
+//	ksize: The size of the window for each dimension of the input tensor.
+//	strides: The stride of the sliding window for each dimension of the
+// input tensor.
+//	padding: The type of padding algorithm to use.
 //
-// Returns 1-D.1-D.1-D.
-func UniqueWithCounts(scope *Scope, x tf.Output, optional ...UniqueWithCountsAttr) (y tf.Output, idx tf.Output, count tf.Output) {
+// Returns The max pooled output tensor.
+func MaxPool(scope *Scope, input tf.Output, ksize []int64, strides []int64, padding string, optional ...MaxPoolAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
+	attrs := map[string]interface{}{"ksize": ksize, "strides": strides, "padding": padding}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "UniqueWithCounts",
+		Type: "MaxPool",
 		Input: []tf.Input{
-			x,
+			input,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
+	return op.Output(0)
 }
 
-// Creates a dataset that skips `count` elements from the `input_dataset`.
+// Fast Fourier transform.
 //
-// Arguments:
+// Computes the 1-dimensional discrete Fourier transform over the inner-most
+// dimension of `input`.
 //
-//	count: A scalar representing the number of elements from the `input_dataset`
-// that should be skipped.  If count is -1, skips everything.
+// Arguments:
+//	input: A complex64 tensor.
 //
+// Returns A complex64 tensor of the same shape as `input`. The inner-most
+//   dimension of `input` is replaced with its 1D Fourier transform.
 //
-func SkipDataset(scope *Scope, input_dataset tf.Output, count tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
+// @compatibility(numpy)
+// Equivalent to np.fft.fft
+// @end_compatibility
+func FFT(scope *Scope, input tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
 	opspec := tf.OpSpec{
-		Type: "SkipDataset",
+		Type: "FFT",
 		Input: []tf.Input{
-			input_dataset, count,
+			input,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// ComplexAttr is an optional argument to Complex.
-type ComplexAttr func(optionalAttr)
+// MaxPoolWithArgmaxAttr is an optional argument to MaxPoolWithArgmax.
+type MaxPoolWithArgmaxAttr func(optionalAttr)
 
-// ComplexTout sets the optional Tout attribute to value.
-// If not specified, defaults to DT_COMPLEX64
-func ComplexTout(value tf.DataType) ComplexAttr {
+// MaxPoolWithArgmaxTargmax sets the optional Targmax attribute to value.
+// If not specified, defaults to DT_INT64
+func MaxPoolWithArgmaxTargmax(value tf.DataType) MaxPoolWithArgmaxAttr {
 	return func(m optionalAttr) {
-		m["Tout"] = value
+		m["Targmax"] = value
 	}
 }
 
-// Converts two real numbers to a complex number.
+// Performs max pooling on the input and outputs both max values and indices.
 //
-// Given a tensor `real` representing the real part of a complex number, and a
-// tensor `imag` representing the imaginary part of a complex number, this
-// operation returns complex numbers elementwise of the form \\(a + bj\\), where
-// *a* represents the `real` part and *b* represents the `imag` part.
+// The indices in `argmax` are flattened, so that a maximum value at position
+// `[b, y, x, c]` becomes flattened index
+// `((b * height + y) * width + x) * channels + c`.
 //
-// The input tensors `real` and `imag` must have the same shape.
+// The indices returned are always in `[0, height) x [0, width)` before flattening,
+// even if padding is involved and the mathematically correct answer is outside
+// (either negative or too large).  This is a bug, but fixing it is difficult to do
+// in a safe backwards compatible way, especially due to flattening.
 //
-// For example:
+// Arguments:
+//	input: 4-D with shape `[batch, height, width, channels]`.  Input to pool over.
+//	ksize: The size of the window for each dimension of the input tensor.
+//	strides: The stride of the sliding window for each dimension of the
+// input tensor.
+//	padding: The type of padding algorithm to use.
 //
-// ```
-// # tensor 'real' is [2.25, 3.25]
-// # tensor `imag` is [4.75, 5.75]
-// tf.complex(real, imag) ==> [[2.25 + 4.75j], [3.25 + 5.75j]]
-// ```
-func Complex(scope *Scope, real tf.Output, imag tf.Output, optional ...ComplexAttr) (out tf.Output) {
+// Returns The max pooled output tensor.4-D.  The flattened indices of the max values chosen for each output.
+func MaxPoolWithArgmax(scope *Scope, input tf.Output, ksize []int64, strides []int64, padding string, optional ...MaxPoolWithArgmaxAttr) (output tf.Output, argmax tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
+	attrs := map[string]interface{}{"ksize": ksize, "strides": strides, "padding": padding}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "Complex",
+		Type: "MaxPoolWithArgmax",
 		Input: []tf.Input{
-			real, imag,
+			input,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return op.Output(0), op.Output(1)
 }
 
-// ImagAttr is an optional argument to Imag.
-type ImagAttr func(optionalAttr)
+// MaxPoolGradGradV2Attr is an optional argument to MaxPoolGradGradV2.
+type MaxPoolGradGradV2Attr func(optionalAttr)
 
-// ImagTout sets the optional Tout attribute to value.
-// If not specified, defaults to DT_FLOAT
-func ImagTout(value tf.DataType) ImagAttr {
+// MaxPoolGradGradV2DataFormat sets the optional data_format attribute to value.
+//
+// value: Specify the data format of the input and output data. With the
+// default format "NHWC", the data is stored in the order of:
+//     [batch, in_height, in_width, in_channels].
+// Alternatively, the format could be "NCHW", the data storage order of:
+//     [batch, in_channels, in_height, in_width].
+// If not specified, defaults to "NHWC"
+func MaxPoolGradGradV2DataFormat(value string) MaxPoolGradGradV2Attr {
 	return func(m optionalAttr) {
-		m["Tout"] = value
+		m["data_format"] = value
 	}
 }
 
-// Returns the imaginary part of a complex number.
-//
-// Given a tensor `input` of complex numbers, this operation returns a tensor of
-// type `float` that is the imaginary part of each element in `input`. All
-// elements in `input` must be complex numbers of the form \\(a + bj\\), where *a*
-// is the real part and *b* is the imaginary part returned by this operation.
+// Computes second-order gradients of the maxpooling function.
 //
-// For example:
+// Arguments:
+//	orig_input: The original input tensor.
+//	orig_output: The original output tensor.
+//	grad: 4-D.  Gradients of gradients w.r.t. the input of `max_pool`.
+//	ksize: The size of the window for each dimension of the input tensor.
+//	strides: The stride of the sliding window for each dimension of the
+// input tensor.
+//	padding: The type of padding algorithm to use.
 //
-// ```
-// # tensor 'input' is [-2.25 + 4.75j, 3.25 + 5.75j]
-// tf.imag(input) ==> [4.75, 5.75]
-// ```
-func Imag(scope *Scope, input tf.Output, optional ...ImagAttr) (output tf.Output) {
+// Returns Gradients of gradients w.r.t. the input to `max_pool`.
+func MaxPoolGradGradV2(scope *Scope, orig_input tf.Output, orig_output tf.Output, grad tf.Output, ksize tf.Output, strides tf.Output, padding string, optional ...MaxPoolGradGradV2Attr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
+	attrs := map[string]interface{}{"padding": padding}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "Imag",
+		Type: "MaxPoolGradGradV2",
 		Input: []tf.Input{
-			input,
+			orig_input, orig_output, grad, ksize, strides,
 		},
 		Attrs: attrs,
 	}
@@ -20301,79 +20417,108 @@ func Imag(scope *Scope, input tf.Output, optional ...ImagAttr) (output tf.Output
 	return op.Output(0)
 }
 
-// Creates a dataset that emits the lines of one or more text files.
+// Adjust the saturation of one or more images.
+//
+// `images` is a tensor of at least 3 dimensions.  The last dimension is
+// interpretted as channels, and must be three.
+//
+// The input image is considered in the RGB colorspace. Conceptually, the RGB
+// colors are first mapped into HSV. A scale is then applied all the saturation
+// values, and then remapped back to RGB colorspace.
 //
 // Arguments:
-//	filenames: A scalar or a vector containing the name(s) of the file(s) to be
-// read.
-//	compression_type: A scalar containing either (i) the empty string (no
-// compression), (ii) "ZLIB", or (iii) "GZIP".
-//	buffer_size: A scalar containing the number of bytes to buffer.
-func TextLineDataset(scope *Scope, filenames tf.Output, compression_type tf.Output, buffer_size tf.Output) (handle tf.Output) {
+//	images: Images to adjust.  At least 3-D.
+//	scale: A float scale to add to the saturation.
+//
+// Returns The hue-adjusted image or images.
+func AdjustSaturation(scope *Scope, images tf.Output, scale tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "TextLineDataset",
+		Type: "AdjustSaturation",
 		Input: []tf.Input{
-			filenames, compression_type, buffer_size,
+			images, scale,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Returns the number of records this Reader has produced.
-//
-// This is the same as the number of ReaderRead executions that have
-// succeeded.
+// SelfAdjointEigV2Attr is an optional argument to SelfAdjointEigV2.
+type SelfAdjointEigV2Attr func(optionalAttr)
+
+// SelfAdjointEigV2ComputeV sets the optional compute_v attribute to value.
 //
-// Arguments:
-//	reader_handle: Handle to a Reader.
-func ReaderNumRecordsProducedV2(scope *Scope, reader_handle tf.Output) (records_produced tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "ReaderNumRecordsProducedV2",
-		Input: []tf.Input{
-			reader_handle,
-		},
+// value: If `True` then eigenvectors will be computed and returned in `v`.
+// Otherwise, only the eigenvalues will be computed.
+// If not specified, defaults to true
+func SelfAdjointEigV2ComputeV(value bool) SelfAdjointEigV2Attr {
+	return func(m optionalAttr) {
+		m["compute_v"] = value
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
 }
 
-// Computes exponential of x - 1 element-wise.
+// Computes the eigen decomposition of one or more square self-adjoint matrices.
 //
-// I.e., \\(y = (\exp x) - 1\\).
-func Expm1(scope *Scope, x tf.Output) (y tf.Output) {
+// Computes the eigenvalues and (optionally) eigenvectors of each inner matrix in
+// `input` such that `input[..., :, :] = v[..., :, :] * diag(e[..., :])`.
+//
+// ```python
+// # a is a tensor.
+// # e is a tensor of eigenvalues.
+// # v is a tensor of eigenvectors.
+// e, v = self_adjoint_eig(a)
+// e = self_adjoint_eig(a, compute_v=False)
+// ```
+//
+// Arguments:
+//	input: `Tensor` input of shape `[N, N]`.
+//
+// Returns Eigenvalues. Shape is `[N]`.Eigenvectors. Shape is `[N, N]`.
+func SelfAdjointEigV2(scope *Scope, input tf.Output, optional ...SelfAdjointEigV2Attr) (e tf.Output, v tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "Expm1",
+		Type: "SelfAdjointEigV2",
 		Input: []tf.Input{
-			x,
+			input,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return op.Output(0), op.Output(1)
 }
 
-// Returns x - y element-wise.
+// Computes second-order gradients of the maxpooling function.
 //
-// *NOTE*: `Sub` supports broadcasting. More about broadcasting
-// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
-func Sub(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
+// Arguments:
+//	input: The original input.
+//	grad: 4-D with shape `[batch, height, width, channels]`.  Gradients w.r.t. the
+// input of `max_pool`.
+//	argmax: The indices of the maximum values chosen for each output of `max_pool`.
+//	ksize: The size of the window for each dimension of the input tensor.
+//	strides: The stride of the sliding window for each dimension of the
+// input tensor.
+//	padding: The type of padding algorithm to use.
+//
+// Returns Gradients of gradients w.r.t. the input of `max_pool`.
+func MaxPoolGradGradWithArgmax(scope *Scope, input tf.Output, grad tf.Output, argmax tf.Output, ksize []int64, strides []int64, padding string) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"ksize": ksize, "strides": strides, "padding": padding}
 	opspec := tf.OpSpec{
-		Type: "Sub",
+		Type: "MaxPoolGradGradWithArgmax",
 		Input: []tf.Input{
-			x, y,
+			input, grad, argmax,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
@@ -21962,56 +22107,7 @@ func QuantizedBiasAdd(scope *Scope, input tf.Output, bias tf.Output, min_input t
 	opspec := tf.OpSpec{
 		Type: "QuantizedBiasAdd",
 		Input: []tf.Input{
-			input, bias, min_input, max_input, min_bias, max_bias,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
-}
-
-// QuantizedConv2DAttr is an optional argument to QuantizedConv2D.
-type QuantizedConv2DAttr func(optionalAttr)
-
-// QuantizedConv2DOutType sets the optional out_type attribute to value.
-// If not specified, defaults to DT_QINT32
-func QuantizedConv2DOutType(value tf.DataType) QuantizedConv2DAttr {
-	return func(m optionalAttr) {
-		m["out_type"] = value
-	}
-}
-
-// Computes a 2D convolution given quantized 4D input and filter tensors.
-//
-// The inputs are quantized tensors where the lowest value represents the real
-// number of the associated minimum, and the highest represents the maximum.
-// This means that you can only interpret the quantized output in the same way, by
-// taking the returned minimum and maximum values into account.
-//
-// Arguments:
-//
-//	filter: filter's input_depth dimension must match input's depth dimensions.
-//	min_input: The float value that the lowest quantized input value represents.
-//	max_input: The float value that the highest quantized input value represents.
-//	min_filter: The float value that the lowest quantized filter value represents.
-//	max_filter: The float value that the highest quantized filter value represents.
-//	strides: The stride of the sliding window for each dimension of the input
-// tensor.
-//	padding: The type of padding algorithm to use.
-//
-// Returns The float value that the lowest quantized output value represents.The float value that the highest quantized output value represents.
-func QuantizedConv2D(scope *Scope, input tf.Output, filter tf.Output, min_input tf.Output, max_input tf.Output, min_filter tf.Output, max_filter tf.Output, strides []int64, padding string, optional ...QuantizedConv2DAttr) (output tf.Output, min_output tf.Output, max_output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"strides": strides, "padding": padding}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "QuantizedConv2D",
-		Input: []tf.Input{
-			input, filter, min_input, max_input, min_filter, max_filter,
+			input, bias, min_input, max_input, min_bias, max_bias,
 		},
 		Attrs: attrs,
 	}
@@ -22201,6 +22297,25 @@ func ComplexAbs(scope *Scope, x tf.Output, optional ...ComplexAbsAttr) (y tf.Out
 	return op.Output(0)
 }
 
+// Computes the reciprocal of x element-wise.
+//
+// DEPRECATED at GraphDef version 17: Use Reciprocal
+//
+// I.e., \\(y = 1 / x\\).
+func Inv(scope *Scope, x tf.Output) (y tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "Inv",
+		Input: []tf.Input{
+			x,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // OrderedMapClearAttr is an optional argument to OrderedMapClear.
 type OrderedMapClearAttr func(optionalAttr)
 
@@ -25687,57 +25802,6 @@ func QuantizedMul(scope *Scope, x tf.Output, y tf.Output, min_x tf.Output, max_x
 	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// QueueEnqueueManyV2Attr is an optional argument to QueueEnqueueManyV2.
-type QueueEnqueueManyV2Attr func(optionalAttr)
-
-// QueueEnqueueManyV2TimeoutMs sets the optional timeout_ms attribute to value.
-//
-// value: If the queue is too full, this operation will block for up
-// to timeout_ms milliseconds.
-// Note: This option is not supported yet.
-// If not specified, defaults to -1
-func QueueEnqueueManyV2TimeoutMs(value int64) QueueEnqueueManyV2Attr {
-	return func(m optionalAttr) {
-		m["timeout_ms"] = value
-	}
-}
-
-// Enqueues zero or more tuples of one or more tensors in the given queue.
-//
-// This operation slices each component tensor along the 0th dimension to
-// make multiple queue elements. All of the tuple components must have the
-// same size in the 0th dimension.
-//
-// The components input has k elements, which correspond to the components of
-// tuples stored in the given queue.
-//
-// N.B. If the queue is full, this operation will block until the given
-// elements have been enqueued (or 'timeout_ms' elapses, if specified).
-//
-// Arguments:
-//	handle: The handle to a queue.
-//	components: One or more tensors from which the enqueued tensors should
-// be taken.
-//
-// Returns the created operation.
-func QueueEnqueueManyV2(scope *Scope, handle tf.Output, components []tf.Output, optional ...QueueEnqueueManyV2Attr) (o *tf.Operation) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "QueueEnqueueManyV2",
-		Input: []tf.Input{
-			handle, tf.OutputList(components),
-		},
-		Attrs: attrs,
-	}
-	return scope.AddOperation(opspec)
-}
-
 // Forwards the input to the output.
 //
 // This operator represents the loop termination condition used by the
@@ -25872,105 +25936,6 @@ func CompareAndBitpack(scope *Scope, input tf.Output, threshold tf.Output) (outp
 	return op.Output(0)
 }
 
-// Creates a dataset that asynchronously prefetches elements from `input_dataset`.
-//
-// Arguments:
-//
-//	buffer_size: The maximum number of elements to buffer in an iterator over
-// this dataset.
-//
-//
-func PrefetchDataset(scope *Scope, input_dataset tf.Output, buffer_size tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
-	opspec := tf.OpSpec{
-		Type: "PrefetchDataset",
-		Input: []tf.Input{
-			input_dataset, buffer_size,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Outputs a `Summary` protocol buffer with a tensor and per-plugin data.
-//
-// Arguments:
-//	tag: A string attached to this summary. Used for organization in TensorBoard.
-//	tensor: A tensor to serialize.
-//	serialized_summary_metadata: A serialized SummaryMetadata proto. Contains plugin
-// data.
-func TensorSummaryV2(scope *Scope, tag tf.Output, tensor tf.Output, serialized_summary_metadata tf.Output) (summary tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "TensorSummaryV2",
-		Input: []tf.Input{
-			tag, tensor, serialized_summary_metadata,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// AudioSummaryV2Attr is an optional argument to AudioSummaryV2.
-type AudioSummaryV2Attr func(optionalAttr)
-
-// AudioSummaryV2MaxOutputs sets the optional max_outputs attribute to value.
-//
-// value: Max number of batch elements to generate audio for.
-// If not specified, defaults to 3
-//
-// REQUIRES: value >= 1
-func AudioSummaryV2MaxOutputs(value int64) AudioSummaryV2Attr {
-	return func(m optionalAttr) {
-		m["max_outputs"] = value
-	}
-}
-
-// Outputs a `Summary` protocol buffer with audio.
-//
-// The summary has up to `max_outputs` summary values containing audio. The
-// audio is built from `tensor` which must be 3-D with shape `[batch_size,
-// frames, channels]` or 2-D with shape `[batch_size, frames]`. The values are
-// assumed to be in the range of `[-1.0, 1.0]` with a sample rate of `sample_rate`.
-//
-// The `tag` argument is a scalar `Tensor` of type `string`.  It is used to
-// build the `tag` of the summary values:
-//
-// *  If `max_outputs` is 1, the summary value tag is '*tag*/audio'.
-// *  If `max_outputs` is greater than 1, the summary value tags are
-//    generated sequentially as '*tag*/audio/0', '*tag*/audio/1', etc.
-//
-// Arguments:
-//	tag: Scalar. Used to build the `tag` attribute of the summary values.
-//	tensor: 2-D of shape `[batch_size, frames]`.
-//	sample_rate: The sample rate of the signal in hertz.
-//
-// Returns Scalar. Serialized `Summary` protocol buffer.
-func AudioSummaryV2(scope *Scope, tag tf.Output, tensor tf.Output, sample_rate tf.Output, optional ...AudioSummaryV2Attr) (summary tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "AudioSummaryV2",
-		Input: []tf.Input{
-			tag, tensor, sample_rate,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
 // Replaces the contents of the table with the specified keys and values.
 //
 // The tensor `keys` must be of the same type as the keys of the table.
@@ -26311,6 +26276,95 @@ func MutableHashTableOfTensorsV2(scope *Scope, key_dtype tf.DataType, value_dtyp
 	return op.Output(0)
 }
 
+// Partitions `data` into `num_partitions` tensors using indices from `partitions`.
+//
+// For each index tuple `js` of size `partitions.ndim`, the slice `data[js, ...]`
+// becomes part of `outputs[partitions[js]]`.  The slices with `partitions[js] = i`
+// are placed in `outputs[i]` in lexicographic order of `js`, and the first
+// dimension of `outputs[i]` is the number of entries in `partitions` equal to `i`.
+// In detail,
+//
+// ```python
+//     outputs[i].shape = [sum(partitions == i)] + data.shape[partitions.ndim:]
+//
+//     outputs[i] = pack([data[js, ...] for js if partitions[js] == i])
+// ```
+//
+// `data.shape` must start with `partitions.shape`.
+//
+// For example:
+//
+// ```python
+//     # Scalar partitions.
+//     partitions = 1
+//     num_partitions = 2
+//     data = [10, 20]
+//     outputs[0] = []  # Empty with shape [0, 2]
+//     outputs[1] = [[10, 20]]
+//
+//     # Vector partitions.
+//     partitions = [0, 0, 1, 1, 0]
+//     num_partitions = 2
+//     data = [10, 20, 30, 40, 50]
+//     outputs[0] = [10, 20, 50]
+//     outputs[1] = [30, 40]
+// ```
+//
+// See `dynamic_stitch` for an example on how to merge partitions back.
+//
+// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
+// <img style="width:100%" src="https://www.tensorflow.org/images/DynamicPartition.png" alt>
+// </div>
+//
+// Arguments:
+//
+//	partitions: Any shape.  Indices in the range `[0, num_partitions)`.
+//	num_partitions: The number of partitions to output.
+func DynamicPartition(scope *Scope, data tf.Output, partitions tf.Output, num_partitions int64) (outputs []tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"num_partitions": num_partitions}
+	opspec := tf.OpSpec{
+		Type: "DynamicPartition",
+		Input: []tf.Input{
+			data, partitions,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	if scope.Err() != nil {
+		return
+	}
+	var idx int
+	var err error
+	if outputs, idx, err = makeOutputList(op, idx, "outputs"); err != nil {
+		scope.UpdateErr("DynamicPartition", err)
+		return
+	}
+	return outputs
+}
+
+// Serialize a `SparseTensor` into a string 3-vector (1-D `Tensor`) object.
+//
+// Arguments:
+//	sparse_indices: 2-D.  The `indices` of the `SparseTensor`.
+//	sparse_values: 1-D.  The `values` of the `SparseTensor`.
+//	sparse_shape: 1-D.  The `shape` of the `SparseTensor`.
+func SerializeSparse(scope *Scope, sparse_indices tf.Output, sparse_values tf.Output, sparse_shape tf.Output) (serialized_sparse tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "SerializeSparse",
+		Input: []tf.Input{
+			sparse_indices, sparse_values, sparse_shape,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Table initializer that takes two tensors for keys and values respectively.
 //
 // Arguments:
@@ -26453,6 +26507,105 @@ func Minimum(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
 	return op.Output(0)
 }
 
+// Creates a dataset that asynchronously prefetches elements from `input_dataset`.
+//
+// Arguments:
+//
+//	buffer_size: The maximum number of elements to buffer in an iterator over
+// this dataset.
+//
+//
+func PrefetchDataset(scope *Scope, input_dataset tf.Output, buffer_size tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
+	opspec := tf.OpSpec{
+		Type: "PrefetchDataset",
+		Input: []tf.Input{
+			input_dataset, buffer_size,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Outputs a `Summary` protocol buffer with a tensor and per-plugin data.
+//
+// Arguments:
+//	tag: A string attached to this summary. Used for organization in TensorBoard.
+//	tensor: A tensor to serialize.
+//	serialized_summary_metadata: A serialized SummaryMetadata proto. Contains plugin
+// data.
+func TensorSummaryV2(scope *Scope, tag tf.Output, tensor tf.Output, serialized_summary_metadata tf.Output) (summary tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "TensorSummaryV2",
+		Input: []tf.Input{
+			tag, tensor, serialized_summary_metadata,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// AudioSummaryV2Attr is an optional argument to AudioSummaryV2.
+type AudioSummaryV2Attr func(optionalAttr)
+
+// AudioSummaryV2MaxOutputs sets the optional max_outputs attribute to value.
+//
+// value: Max number of batch elements to generate audio for.
+// If not specified, defaults to 3
+//
+// REQUIRES: value >= 1
+func AudioSummaryV2MaxOutputs(value int64) AudioSummaryV2Attr {
+	return func(m optionalAttr) {
+		m["max_outputs"] = value
+	}
+}
+
+// Outputs a `Summary` protocol buffer with audio.
+//
+// The summary has up to `max_outputs` summary values containing audio. The
+// audio is built from `tensor` which must be 3-D with shape `[batch_size,
+// frames, channels]` or 2-D with shape `[batch_size, frames]`. The values are
+// assumed to be in the range of `[-1.0, 1.0]` with a sample rate of `sample_rate`.
+//
+// The `tag` argument is a scalar `Tensor` of type `string`.  It is used to
+// build the `tag` of the summary values:
+//
+// *  If `max_outputs` is 1, the summary value tag is '*tag*/audio'.
+// *  If `max_outputs` is greater than 1, the summary value tags are
+//    generated sequentially as '*tag*/audio/0', '*tag*/audio/1', etc.
+//
+// Arguments:
+//	tag: Scalar. Used to build the `tag` attribute of the summary values.
+//	tensor: 2-D of shape `[batch_size, frames]`.
+//	sample_rate: The sample rate of the signal in hertz.
+//
+// Returns Scalar. Serialized `Summary` protocol buffer.
+func AudioSummaryV2(scope *Scope, tag tf.Output, tensor tf.Output, sample_rate tf.Output, optional ...AudioSummaryV2Attr) (summary tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "AudioSummaryV2",
+		Input: []tf.Input{
+			tag, tensor, sample_rate,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Computes the gradient for the sqrt of `x` wrt its input.
 //
 // Specifically, `grad = dy * 0.5 / y`, where `y = sqrt(x)`, and `dy`
@@ -26589,156 +26742,3 @@ func MatrixSolve(scope *Scope, matrix tf.Output, rhs tf.Output, optional ...Matr
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
-
-// SumAttr is an optional argument to Sum.
-type SumAttr func(optionalAttr)
-
-// SumKeepDims sets the optional keep_dims attribute to value.
-//
-// value: If true, retain reduced dimensions with length 1.
-// If not specified, defaults to false
-func SumKeepDims(value bool) SumAttr {
-	return func(m optionalAttr) {
-		m["keep_dims"] = value
-	}
-}
-
-// Computes the sum of elements across dimensions of a tensor.
-//
-// Reduces `input` along the dimensions given in `reduction_indices`. Unless
-// `keep_dims` is true, the rank of the tensor is reduced by 1 for each entry in
-// `reduction_indices`. If `keep_dims` is true, the reduced dimensions are
-// retained with length 1.
-//
-// Arguments:
-//	input: The tensor to reduce.
-//	reduction_indices: The dimensions to reduce. Must be in the range
-// `[-rank(input), rank(input))`.
-//
-// Returns The reduced tensor.
-func Sum(scope *Scope, input tf.Output, reduction_indices tf.Output, optional ...SumAttr) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "Sum",
-		Input: []tf.Input{
-			input, reduction_indices,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Partitions `data` into `num_partitions` tensors using indices from `partitions`.
-//
-// For each index tuple `js` of size `partitions.ndim`, the slice `data[js, ...]`
-// becomes part of `outputs[partitions[js]]`.  The slices with `partitions[js] = i`
-// are placed in `outputs[i]` in lexicographic order of `js`, and the first
-// dimension of `outputs[i]` is the number of entries in `partitions` equal to `i`.
-// In detail,
-//
-// ```python
-//     outputs[i].shape = [sum(partitions == i)] + data.shape[partitions.ndim:]
-//
-//     outputs[i] = pack([data[js, ...] for js if partitions[js] == i])
-// ```
-//
-// `data.shape` must start with `partitions.shape`.
-//
-// For example:
-//
-// ```python
-//     # Scalar partitions.
-//     partitions = 1
-//     num_partitions = 2
-//     data = [10, 20]
-//     outputs[0] = []  # Empty with shape [0, 2]
-//     outputs[1] = [[10, 20]]
-//
-//     # Vector partitions.
-//     partitions = [0, 0, 1, 1, 0]
-//     num_partitions = 2
-//     data = [10, 20, 30, 40, 50]
-//     outputs[0] = [10, 20, 50]
-//     outputs[1] = [30, 40]
-// ```
-//
-// See `dynamic_stitch` for an example on how to merge partitions back.
-//
-// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
-// <img style="width:100%" src="https://www.tensorflow.org/images/DynamicPartition.png" alt>
-// </div>
-//
-// Arguments:
-//
-//	partitions: Any shape.  Indices in the range `[0, num_partitions)`.
-//	num_partitions: The number of partitions to output.
-func DynamicPartition(scope *Scope, data tf.Output, partitions tf.Output, num_partitions int64) (outputs []tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"num_partitions": num_partitions}
-	opspec := tf.OpSpec{
-		Type: "DynamicPartition",
-		Input: []tf.Input{
-			data, partitions,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	if scope.Err() != nil {
-		return
-	}
-	var idx int
-	var err error
-	if outputs, idx, err = makeOutputList(op, idx, "outputs"); err != nil {
-		scope.UpdateErr("DynamicPartition", err)
-		return
-	}
-	return outputs
-}
-
-// Serialize a `SparseTensor` into a string 3-vector (1-D `Tensor`) object.
-//
-// Arguments:
-//	sparse_indices: 2-D.  The `indices` of the `SparseTensor`.
-//	sparse_values: 1-D.  The `values` of the `SparseTensor`.
-//	sparse_shape: 1-D.  The `shape` of the `SparseTensor`.
-func SerializeSparse(scope *Scope, sparse_indices tf.Output, sparse_values tf.Output, sparse_shape tf.Output) (serialized_sparse tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "SerializeSparse",
-		Input: []tf.Input{
-			sparse_indices, sparse_values, sparse_shape,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Computes the reciprocal of x element-wise.
-//
-// DEPRECATED at GraphDef version 17: Use Reciprocal
-//
-// I.e., \\(y = 1 / x\\).
-func Inv(scope *Scope, x tf.Output) (y tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "Inv",
-		Input: []tf.Input{
-			x,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-- 
GitLab


From e7ab55b01f25bc1c9023dcc9510667ea480c6186 Mon Sep 17 00:00:00 2001
From: Ian Langmore <langmore@google.com>
Date: Fri, 6 Oct 2017 08:14:46 -0700
Subject: [PATCH 101/909] SinhArcsinh distributions modified so that their
 skewness is symmetric.

Also, some doc-fixes/changes, and make SinhArcsinh bijector have same None kwargs and naming scheme as the distributions

PiperOrigin-RevId: 171294037
---
 .../bijectors/sinh_arcsinh_bijector_test.py   |  8 ++++-
 .../python/kernel_tests/sinh_arcsinh_test.py  | 16 ++++++++++
 .../vector_sinh_arcsinh_diag_test.py          | 16 ++++++++++
 .../python/ops/bijectors/sinh_arcsinh_impl.py | 20 ++++++++-----
 .../distributions/python/ops/sinh_arcsinh.py  | 29 ++++++++++++-------
 .../python/ops/vector_sinh_arcsinh_diag.py    | 28 ++++++++++++------
 6 files changed, 89 insertions(+), 28 deletions(-)

diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/sinh_arcsinh_bijector_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/sinh_arcsinh_bijector_test.py
index 230dd93a2a..172c180a44 100644
--- a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/sinh_arcsinh_bijector_test.py
+++ b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/sinh_arcsinh_bijector_test.py
@@ -41,7 +41,7 @@ class SinhArcsinhBijectorTest(test.TestCase):
           tailweight=tailweight,
           event_ndims=1,
           validate_args=True)
-      self.assertEqual("sinh_arcsinh", bijector.name)
+      self.assertEqual("SinhArcsinh", bijector.name)
       x = np.array([[[-2.01], [2.], [1e-4]]]).astype(np.float32)
       y = np.sinh((np.arcsinh(x) + skewness) * tailweight)
       self.assertAllClose(y, bijector.forward(x).eval())
@@ -170,6 +170,12 @@ class SinhArcsinhBijectorTest(test.TestCase):
       with self.assertRaisesOpError("not positive"):
         SinhArcsinh(tailweight=0., validate_args=True).forward(1.0).eval()
 
+  def testDefaultDtypeIsFloat32(self):
+    with self.test_session():
+      bijector = SinhArcsinh()
+      self.assertEqual(bijector.tailweight.dtype, np.float32)
+      self.assertEqual(bijector.skewness.dtype, np.float32)
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/contrib/distributions/python/kernel_tests/sinh_arcsinh_test.py b/tensorflow/contrib/distributions/python/kernel_tests/sinh_arcsinh_test.py
index 8ea3a59255..88b48736dd 100644
--- a/tensorflow/contrib/distributions/python/kernel_tests/sinh_arcsinh_test.py
+++ b/tensorflow/contrib/distributions/python/kernel_tests/sinh_arcsinh_test.py
@@ -200,6 +200,22 @@ class SinhArcsinhTest(test.TestCase):
       sasnorm_samps = sess.run(sasnorm.sample(10000, seed=4))
       np.testing.assert_array_less(loc, sasnorm_samps.mean(axis=0))
 
+  def test_pdf_reflected_for_negative_skewness(self):
+    with self.test_session() as sess:
+      sas_pos_skew = ds.SinhArcsinh(
+          loc=0.,
+          scale=1.,
+          skewness=2.,
+          validate_args=True)
+      sas_neg_skew = ds.SinhArcsinh(
+          loc=0.,
+          scale=1.,
+          skewness=-2.,
+          validate_args=True)
+      x = np.linspace(-2, 2, num=5).astype(np.float32)
+      self.assertAllClose(
+          *sess.run([sas_pos_skew.prob(x), sas_neg_skew.prob(x[::-1])]))
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/contrib/distributions/python/kernel_tests/vector_sinh_arcsinh_diag_test.py b/tensorflow/contrib/distributions/python/kernel_tests/vector_sinh_arcsinh_diag_test.py
index a7140cd98b..a5d837d454 100644
--- a/tensorflow/contrib/distributions/python/kernel_tests/vector_sinh_arcsinh_diag_test.py
+++ b/tensorflow/contrib/distributions/python/kernel_tests/vector_sinh_arcsinh_diag_test.py
@@ -251,6 +251,22 @@ class VectorSinhArcsinhDiagTest(test_util.VectorDistributionTestHelpers,
           center=0.15,
           rtol=0.1)
 
+  def test_pdf_reflected_for_negative_skewness(self):
+    with self.test_session() as sess:
+      sas_pos_skew = ds.VectorSinhArcsinhDiag(
+          loc=[0.],
+          scale_identity_multiplier=1.,
+          skewness=2.,
+          validate_args=True)
+      sas_neg_skew = ds.VectorSinhArcsinhDiag(
+          loc=[0.],
+          scale_identity_multiplier=1.,
+          skewness=-2.,
+          validate_args=True)
+      x = np.linspace(-2, 2, num=5).astype(np.float32).reshape(5, 1)
+      self.assertAllClose(
+          *sess.run([sas_pos_skew.prob(x), sas_neg_skew.prob(x[::-1])]))
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/sinh_arcsinh_impl.py b/tensorflow/contrib/distributions/python/ops/bijectors/sinh_arcsinh_impl.py
index dac3d812ee..3a75e4ae94 100644
--- a/tensorflow/contrib/distributions/python/ops/bijectors/sinh_arcsinh_impl.py
+++ b/tensorflow/contrib/distributions/python/ops/bijectors/sinh_arcsinh_impl.py
@@ -89,18 +89,18 @@ class SinhArcsinh(bijector.Bijector):
   """
 
   def __init__(self,
-               skewness=0.,
-               tailweight=1.,
+               skewness=None,
+               tailweight=None,
                event_ndims=0,
                validate_args=False,
-               name="sinh_arcsinh"):
+               name="SinhArcsinh"):
     """Instantiates the `SinhArcsinh` bijector.
 
     Args:
-      skewness:  Skewness parameter.  Float-type `Tensor`.
+      skewness:  Skewness parameter.  Float-type `Tensor`.  Default is `0`
+        of type `float32`.
       tailweight:  Tailweight parameter.  Positive `Tensor` of same `dtype` as
-        `skewness`
-        and broadcastable `shape`.
+        `skewness` and broadcastable `shape`.  Default is `1` of type `float32`.
       event_ndims: Python scalar indicating the number of dimensions associated
         with a particular draw from the distribution.
       validate_args: Python `bool` indicating whether arguments should be
@@ -111,8 +111,12 @@ class SinhArcsinh(bijector.Bijector):
     self._name = name
     self._validate_args = validate_args
     with self._name_scope("init", values=[skewness, tailweight]):
-      self._skewness = ops.convert_to_tensor(skewness, name="skewness")
-      self._tailweight = ops.convert_to_tensor(tailweight, name="tailweight")
+      tailweight = 1. if tailweight is None else tailweight
+      skewness = 0. if skewness is None else skewness
+      self._skewness = ops.convert_to_tensor(
+          skewness, name="skewness")
+      self._tailweight = ops.convert_to_tensor(
+          tailweight, name="tailweight", dtype=self._skewness.dtype)
       check_ops.assert_same_float_dtype([self._skewness, self._tailweight])
       if validate_args:
         self._tailweight = control_flow_ops.with_dependencies([
diff --git a/tensorflow/contrib/distributions/python/ops/sinh_arcsinh.py b/tensorflow/contrib/distributions/python/ops/sinh_arcsinh.py
index cdf81526da..b05f15771a 100644
--- a/tensorflow/contrib/distributions/python/ops/sinh_arcsinh.py
+++ b/tensorflow/contrib/distributions/python/ops/sinh_arcsinh.py
@@ -51,8 +51,9 @@ class SinhArcsinh(transformed_distribution.TransformedDistribution):
   `(loc, scale, skewness, tailweight)`, via the relation:
 
   ```
-  Y := loc + scale * F(Z) * (2 / F(2))
+  Y := loc + scale * F(Z) * (2 / F_0(2))
   F(Z) := Sinh( (Arcsinh(Z) + skewness) * tailweight )
+  F_0(Z) := Sinh( Arcsinh(Z) * tailweight )
   ```
 
   This distribution is similar to the location-scale transformation
@@ -61,7 +62,7 @@ class SinhArcsinh(transformed_distribution.TransformedDistribution):
   * If `skewness = 0` and `tailweight = 1` (the defaults), `F(Z) = Z`, and then
     `Y = L(Z)` exactly.
   * `loc` is used in both to shift the result by a constant factor.
-  * Our definition of `C` ensures that
+  * The multiplication of `scale` by `2 / F_0(2)` ensures that if `skewness = 0`
     `P[Y - loc <= 2 * scale] = P[L(Z) - loc <= 2 * scale]`.
     Thus it can be said that the weights in the tails of `Y` and `L(Z)` beyond
     `loc + 2 * scale` are the same.
@@ -84,12 +85,12 @@ class SinhArcsinh(transformed_distribution.TransformedDistribution):
   `|Z| >> (|skewness| * tailweight)**tailweight`, we have
   `Y approx 0.5 Z**tailweight e**(sign(Z) skewness * tailweight)`.
 
-  To see the argument about `C` and quantiles, note that
+  To see the argument regarding multiplying `scale` by `2 / F_0(2)`,
 
   ```
-  P[(Y - loc) / scale <= 2] = P[F(Z) <= 2 * scale / C]
-                             = P[Z <= F^{-1}(2 * scale / C)]
-                             = P[Z <= 2].
+  P[(Y - loc) / scale <= 2] = P[F(Z) * (2 / F_0(2)) <= 2]
+                            = P[F(Z) <= F_0(2)]
+                            = P[Z <= 2]  (if F = F_0).
   ```
   """
 
@@ -101,7 +102,7 @@ class SinhArcsinh(transformed_distribution.TransformedDistribution):
                distribution=None,
                validate_args=False,
                allow_nan_stats=True,
-               name="MultivariateNormalLinearOperator"):
+               name="SinhArcsinh"):
     """Construct SinhArcsinh distribution on `(-inf, inf)`.
 
     Arguments `(loc, scale, skewness, tailweight)` must have broadcastable shape
@@ -138,6 +139,7 @@ class SinhArcsinh(transformed_distribution.TransformedDistribution):
       dtype = loc.dtype
       scale = ops.convert_to_tensor(scale, name="scale", dtype=dtype)
       tailweight = 1. if tailweight is None else tailweight
+      has_default_skewness = skewness is None
       skewness = 0. if skewness is None else skewness
       tailweight = ops.convert_to_tensor(
           tailweight, name="tailweight", dtype=dtype)
@@ -149,7 +151,8 @@ class SinhArcsinh(transformed_distribution.TransformedDistribution):
       # Recall, with Z a random variable,
       #   Y := loc + C * F(Z),
       #   F(Z) := Sinh( (Arcsinh(Z) + skewness) * tailweight )
-      #   C := 2 * scale / F(2)
+      #   F_0(Z) := Sinh( Arcsinh(Z) * tailweight )
+      #   C := 2 * scale / F_0(2)
       if distribution is None:
         distribution = normal.Normal(
             loc=array_ops.zeros([], dtype=dtype),
@@ -164,9 +167,15 @@ class SinhArcsinh(transformed_distribution.TransformedDistribution):
       # Make the SAS bijector, 'F'.
       f = bijectors.SinhArcsinh(
           skewness=skewness, tailweight=tailweight, event_ndims=0)
+      if has_default_skewness:
+        f_noskew = f
+      else:
+        f_noskew = bijectors.SinhArcsinh(
+            skewness=skewness.dtype.as_numpy_dtype(0.),
+            tailweight=tailweight, event_ndims=0)
 
-      # Make the Affine bijector, Z --> loc + C * Z.
-      c = 2 * scale / f.forward(ops.convert_to_tensor(2, dtype=dtype))
+      # Make the Affine bijector, Z --> loc + scale * Z (2 / F_0(2))
+      c = 2 * scale / f_noskew.forward(ops.convert_to_tensor(2, dtype=dtype))
       affine = bijectors.Affine(
           shift=loc,
           scale_identity_multiplier=c,
diff --git a/tensorflow/contrib/distributions/python/ops/vector_sinh_arcsinh_diag.py b/tensorflow/contrib/distributions/python/ops/vector_sinh_arcsinh_diag.py
index 488724e80c..544a871070 100644
--- a/tensorflow/contrib/distributions/python/ops/vector_sinh_arcsinh_diag.py
+++ b/tensorflow/contrib/distributions/python/ops/vector_sinh_arcsinh_diag.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""SinhArcsinh transformation of a distribution."""
+"""Multi-dimensional (Vector) SinhArcsinh transformation of a distribution."""
 
 from __future__ import absolute_import
 from __future__ import division
@@ -52,8 +52,9 @@ class VectorSinhArcsinhDiag(transformed_distribution.TransformedDistribution):
   matrix multiplication):
 
   ```
-  Y := loc + scale @ F(Z) * (2 / F(2))
+  Y := loc + scale @ F(Z) * (2 / F_0(2))
   F(Z) := Sinh( (Arcsinh(Z) + skewness) * tailweight )
+  F_0(Z) := Sinh( Arcsinh(Z) * tailweight )
   ```
 
   This distribution is similar to the location-scale transformation
@@ -62,7 +63,7 @@ class VectorSinhArcsinhDiag(transformed_distribution.TransformedDistribution):
   * If `skewness = 0` and `tailweight = 1` (the defaults), `F(Z) = Z`, and then
     `Y = L(Z)` exactly.
   * `loc` is used in both to shift the result by a constant factor.
-  * Our definition of `C` ensures that
+  * The multiplication of `scale` by `2 / F_0(2)` ensures that if `skewness = 0`
     `P[Y - loc <= 2 * scale] = P[L(Z) - loc <= 2 * scale]`.
     Thus it can be said that the weights in the tails of `Y` and `L(Z)` beyond
     `loc + 2 * scale` are the same.
@@ -85,12 +86,12 @@ class VectorSinhArcsinhDiag(transformed_distribution.TransformedDistribution):
   `|Z| >> (|skewness| * tailweight)**tailweight`, we have
   `Y approx 0.5 Z**tailweight e**(sign(Z) skewness * tailweight)`.
 
-  To see the argument about `C` and quantiles, note that
+  To see the argument regarding multiplying `scale` by `2 / F_0(2)`,
 
   ```
-  P[(Y - loc) / scale <= 2] = P[F(Z) <= 2 * scale / C]
-                             = P[Z <= F^{-1}(2 * scale / C)]
-                             = P[Z <= 2].
+  P[(Y - loc) / scale <= 2] = P[F(Z) * (2 / F_0(2)) <= 2]
+                            = P[F(Z) <= F_0(2)]
+                            = P[Z <= 2]  (if F = F_0).
   ```
   """
 
@@ -171,12 +172,14 @@ class VectorSinhArcsinhDiag(transformed_distribution.TransformedDistribution):
         ]):
       loc = ops.convert_to_tensor(loc, name="loc") if loc is not None else loc
       tailweight = 1. if tailweight is None else tailweight
+      has_default_skewness = skewness is None
       skewness = 0. if skewness is None else skewness
 
       # Recall, with Z a random variable,
       #   Y := loc + C * F(Z),
       #   F(Z) := Sinh( (Arcsinh(Z) + skewness) * tailweight )
-      #   C := 2 * scale / F(2)
+      #   F_0(Z) := Sinh( Arcsinh(Z) * tailweight )
+      #   C := 2 * scale / F_0(2)
 
       # Construct shapes and 'scale' out of the scale_* and loc kwargs.
       # scale_linop is only an intermediary to:
@@ -213,9 +216,16 @@ class VectorSinhArcsinhDiag(transformed_distribution.TransformedDistribution):
           tailweight, dtype=dtype, name="tailweight")
       f = bijectors.SinhArcsinh(
           skewness=skewness, tailweight=tailweight, event_ndims=1)
+      if has_default_skewness:
+        f_noskew = f
+      else:
+        f_noskew = bijectors.SinhArcsinh(
+            skewness=skewness.dtype.as_numpy_dtype(0.),
+            tailweight=tailweight, event_ndims=0)
 
       # Make the Affine bijector, Z --> loc + C * Z.
-      c = 2 * scale_diag_part / f.forward(ops.convert_to_tensor(2, dtype=dtype))
+      c = 2 * scale_diag_part / f_noskew.forward(
+          ops.convert_to_tensor(2, dtype=dtype))
       affine = bijectors.Affine(
           shift=loc, scale_diag=c, validate_args=validate_args, event_ndims=1)
 
-- 
GitLab


From 9d8346a1204d05b2ab16c169a6a6077167fe162a Mon Sep 17 00:00:00 2001
From: Jingyue Wu <jingyue@google.com>
Date: Fri, 6 Oct 2017 08:15:48 -0700
Subject: [PATCH 102/909] [Grappler] Reorder cast and transpose.

A common pattern after the layout optimizer is casting an uint8 NHWC
image to float before transposing it to NCHW. It is beneficial to reorder
the cast and the transpose to make the transpose process smaller amount
of data. This optimization converts

  Transpose(Cast(image, dst_type), perm)

to

  Cast(Transpose(image, perm), dst_type)

when sizeof(image.type) < sizeof(dst_type).

PiperOrigin-RevId: 171294111
---
 .../optimizers/arithmetic_optimizer.cc        | 81 +++++++++++++++++++
 .../optimizers/arithmetic_optimizer_test.cc   | 66 +++++++++++++++
 2 files changed, 147 insertions(+)

diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
index 2d7cf3b182..343820de71 100644
--- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
@@ -24,6 +24,7 @@ limitations under the License.
 #include "tensorflow/core/grappler/op_types.h"
 #include "tensorflow/core/lib/strings/strcat.h"
 #include "tensorflow/core/platform/tensor_coding.h"
+#include "tensorflow/core/util/device_name_utils.h"
 
 namespace tensorflow {
 namespace grappler {
@@ -274,6 +275,26 @@ static bool SimplyReordersData(const NodeDef& node) {
   return node.op() == "Transpose";
 }
 
+// Returns the data type in attribute `attr_name` of `node`. If that attribute
+// doesn't exist, returns DT_INVALID.
+static DataType GetDataTypeFromAttr(const NodeDef& node,
+                                    const string& attr_name) {
+  if (!node.attr().count(attr_name)) {
+    return DT_INVALID;
+  }
+  const auto& attr = node.attr().at(attr_name);
+  if (attr.value_case() != AttrValue::kType) {
+    return DT_INVALID;
+  }
+  return attr.type();
+}
+
+static bool IsNumberType(DataType dtype) {
+  DataTypeVector number_types = NumberTypes();
+  return std::find(number_types.begin(), number_types.end(), dtype) !=
+         number_types.end();
+}
+
 string ArithmeticOptimizer::TrySimplifyAndReplaceUses(
     const NodeDef* node, GraphDef* graph_def, NodeMap* node_map,
     std::vector<const NodeDef*>* new_nodes) const {
@@ -320,6 +341,66 @@ string ArithmeticOptimizer::TrySimplifyAndReplaceUses(
     }
   }
 
+  if (node->op() == "Transpose") {
+    // Reorder Cast and Transpose if beneficial.
+    //
+    // A common pattern after the layout optimizer is casting an uint8 NHWC
+    // image to float before transposing it to NCHW. It is beneficial to reorder
+    // the cast and the transpose to make the transpose process smaller amount
+    // of data. This optimization converts
+    //   Transpose(Cast(image, dst_type), perm)
+    // to
+    //   Cast(Transpose(image, perm), dst_type)
+    // when sizeof(image.type) < sizeof(dst_type).
+    //
+    // TODO(jingyue): This optimization can be generalized to a cast followed by
+    // a chain of ops that merely reorder elements (e.g. Reshape and
+    // DepthToSpace).
+    const NodeDef* transpose = node;
+    string dontcare;
+    string device;
+    // This optimization can be dangerous on devices other than CPU and GPU. The
+    // transpose might not be implemented for image.type, or might be slower
+    // with image.type than with dst_type.
+    if (DeviceNameUtils::SplitDeviceName(transpose->device(), &dontcare,
+                                         &device) &&
+        (StringPiece(device).contains(DEVICE_CPU) ||
+         StringPiece(device).contains(DEVICE_GPU))) {
+      const NodeDef* cast = node_map->GetNode(transpose->input(0));
+      if (cast->op() == "Cast") {
+        const NodeDef* input = node_map->GetNode(cast->input(0));
+        const DataType src_type = GetDataTypeFromAttr(*cast, "SrcT");
+        const DataType dst_type = GetDataTypeFromAttr(*cast, "DstT");
+        if (IsNumberType(src_type) && IsNumberType(dst_type) &&
+            DataTypeSize(src_type) < DataTypeSize(dst_type)) {
+          NodeDef* new_transpose = graph_def->add_node();
+          *new_transpose = *transpose;
+          new_transpose->set_name(transpose->name() + "_" +
+                                  DataTypeString(src_type));
+          (*new_transpose->mutable_attr())["T"].set_type(src_type);
+          node_map->AddNode(new_transpose->name(), new_transpose);
+
+          new_transpose->set_input(0, cast->input(0));
+          node_map->AddOutput(input->name(), new_transpose->name());
+          node_map->AddOutput(NodeName(new_transpose->input(1)),
+                              new_transpose->name());
+
+          NodeDef* new_cast = graph_def->add_node();
+          *new_cast = *cast;
+          new_cast->set_name(cast->name() + "_new");
+          node_map->AddNode(new_cast->name(), new_cast);
+
+          new_cast->set_input(0, new_transpose->name());
+          node_map->AddOutput(new_transpose->name(), new_cast->name());
+
+          new_nodes->push_back(new_transpose);
+          new_nodes->push_back(new_cast);
+          return new_cast->name();
+        }
+      }
+    }
+  }
+
   // Fold a multiply of a scalar into the following convolution. This folding
   // can jump across nodes that merely reorders data (such as reshape and
   // transpose). For example, we can optimize
diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc b/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc
index c8bca4282b..b3405646eb 100644
--- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc
+++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc
@@ -109,6 +109,72 @@ TEST_F(ArithmeticOptimizerTest, CombineReshapes) {
                    [](const NodeDef& node) { return node.op() == "Reshape"; }));
 }
 
+TEST_F(ArithmeticOptimizerTest, ReorderTransposeCast) {
+  tensorflow::Scope s = tensorflow::Scope::NewRootScope().WithDevice("/gpu:0");
+  Output nhwc_uint8 =
+      ops::Placeholder(s, DT_UINT8, ops::Placeholder::Shape({8, 28, 28, 3}));
+  Output nhwc_fp32 = ops::Cast(s, nhwc_uint8, DT_FLOAT);
+  Output nchw_fp32 =
+      ops::Transpose(s, nhwc_fp32, ops::Const(s, {0, 3, 1, 2}, {4}));
+  Output outputs = ops::Identity(s.WithOpName("outputs"), nchw_fp32);
+
+  GrapplerItem item;
+  item.fetch = {"outputs"};
+  TF_CHECK_OK(s.ToGraphDef(&item.graph));
+
+  GraphDef output;
+  TF_EXPECT_OK(ArithmeticOptimizer().Optimize(nullptr, item, &output));
+
+  item.graph = output;
+  TF_EXPECT_OK(ModelPruner().Optimize(nullptr, item, &output));
+
+  const NodeDef* transpose_node = nullptr;
+  for (const NodeDef& node : output.node()) {
+    if (node.op() == "Transpose") {
+      EXPECT_EQ(transpose_node, nullptr);
+      EXPECT_EQ(DT_UINT8, node.attr().at("T").type());
+      transpose_node = &node;
+    }
+  }
+  EXPECT_NE(transpose_node, nullptr);
+
+  for (const NodeDef& node : output.node()) {
+    if (node.op() == "Cast") {
+      EXPECT_EQ(NodeName(node.input(0)), transpose_node->name());
+    }
+  }
+}
+
+TEST_F(ArithmeticOptimizerTest, NoReorderTransposeCast) {
+  tensorflow::Scope s = tensorflow::Scope::NewRootScope().WithDevice("/gpu:0");
+  Output nhwc_fp32 =
+      ops::Placeholder(s, DT_FLOAT, ops::Placeholder::Shape({8, 28, 28, 3}));
+  Output nhwc_uint8 = ops::Cast(s, nhwc_fp32, DT_UINT8);
+  Output nchw_uint8 =
+      ops::Transpose(s, nhwc_uint8, ops::Const(s, {0, 3, 1, 2}, {4}));
+  Output outputs = ops::Identity(s.WithOpName("outputs"), nchw_uint8);
+
+  GrapplerItem item;
+  item.fetch = {"outputs"};
+  TF_CHECK_OK(s.ToGraphDef(&item.graph));
+
+  GraphDef output;
+  TF_EXPECT_OK(ArithmeticOptimizer().Optimize(nullptr, item, &output));
+
+  item.graph = output;
+  TF_EXPECT_OK(ModelPruner().Optimize(nullptr, item, &output));
+
+  int num_transposes = 0;
+  for (const NodeDef& node : output.node()) {
+    if (node.op() == "Transpose") {
+      EXPECT_EQ(DT_UINT8, node.attr().at("T").type());
+      EXPECT_EQ(node.input(0), "Cast");
+      ++num_transposes;
+    }
+  }
+  EXPECT_EQ(1, num_transposes);
+}
+
 TEST_F(ArithmeticOptimizerTest, RemoveInverseTransposes) {
   tensorflow::Scope s = tensorflow::Scope::NewRootScope();
   Output inputs_shape =
-- 
GitLab


From 2226790bbf19638eb3535abe521df7b16a109147 Mon Sep 17 00:00:00 2001
From: Andrew Selle <aselle@google.com>
Date: Fri, 6 Oct 2017 08:23:46 -0700
Subject: [PATCH 103/909] Internal Change

PiperOrigin-RevId: 171294796
---
 tensorflow/leakr_file_type_recipe.ftrcp | 30 +++++++++++++++++++++++++
 1 file changed, 30 insertions(+)
 create mode 100644 tensorflow/leakr_file_type_recipe.ftrcp

diff --git a/tensorflow/leakr_file_type_recipe.ftrcp b/tensorflow/leakr_file_type_recipe.ftrcp
new file mode 100644
index 0000000000..0521a084c7
--- /dev/null
+++ b/tensorflow/leakr_file_type_recipe.ftrcp
@@ -0,0 +1,30 @@
+name: "TensorFlow filetype recipes"
+desc: "Copybara leakr checks, used by copy.bara.sky."
+
+file_config:{
+  name: "Image labels text file skip"
+  desc: "Generic text files."
+  pattern: ".*labels.txt"
+  compression: COMPRESSION_NONE
+  scan_mode: SCAN_SKIP
+  file_group: FG_PLAIN_TEXT_GENERIC
+}
+
+file_config:{
+  name: "[Mediafiles] Graphics"
+  desc: "All media files that are images, graphics and icons."
+  ext: "bmp"
+  ext: "gif"
+  ext: "icns"
+  ext: "ico"
+  ext: "jpeg"
+  ext: "jpg"
+  ext: "png"
+  ext: "svg"
+  ext: "tga"
+  ext: "tiff"
+  ext: "webp"
+  compression: COMPRESSION_NONE
+  scan_mode: SCAN_SKIP
+  file_group: FG_MEDIA_GRAPHICS
+}
\ No newline at end of file
-- 
GitLab


From fb0df6d9de9acb1d598c0400a705d16e8cd4f693 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Fri, 6 Oct 2017 09:12:22 -0700
Subject: [PATCH 104/909] [XLA:LLVM] Allow LLVM AA to work cross-functions.

Create our AA domain with createAliasScopeDomain rather than
createAnonymousAliasScopeDomain.  This way inlining does not duplicate
the domain (and thus prevent us from reasoning about loads/stores that
cross the inlined function boundary).

PiperOrigin-RevId: 171299706
---
 .../compiler/xla/service/llvm_ir/alias_analysis.cc    | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xla/service/llvm_ir/alias_analysis.cc b/tensorflow/compiler/xla/service/llvm_ir/alias_analysis.cc
index 5e28e37600..bdddc232ef 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/alias_analysis.cc
+++ b/tensorflow/compiler/xla/service/llvm_ir/alias_analysis.cc
@@ -92,7 +92,16 @@ void AliasAnalysis::AddAliasingInformationToIrArray(const HloInstruction& hlo,
 llvm::MDNode* AliasAnalysis::GetAliasDomain() {
   llvm::MDBuilder metadata_builder(*context_);
   if (alias_domain_ == nullptr) {
-    alias_domain_ = metadata_builder.createAnonymousAliasScopeDomain();
+    // We use createAliasScopeDomain rather than createAnonymousAliasScopeDomain
+    // so that when functions get inlined, we continue using the one domain,
+    // rather than duplicating it (and thus having two AA domains in one
+    // function).
+    //
+    // A side-effect of this is that if you ever compile two HLO modules in the
+    // same LLVM module, they'll have the same alias scope domain.  This isn't a
+    // problem because the two HLO modules will never interact with one another.
+    alias_domain_ =
+        metadata_builder.createAliasScopeDomain("XLA global AA domain");
   }
   return alias_domain_;
 }
-- 
GitLab


From 3251bc07927c6a60916fc274e11445d42e5ec193 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 6 Oct 2017 09:24:17 -0700
Subject: [PATCH 105/909] Fixed typo in DynamicRnnEstimator __init__
 documentation.

PiperOrigin-RevId: 171300981
---
 .../learn/python/learn/estimators/dynamic_rnn_estimator.py      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/learn/python/learn/estimators/dynamic_rnn_estimator.py b/tensorflow/contrib/learn/python/learn/estimators/dynamic_rnn_estimator.py
index 1724d7599d..69440e823e 100644
--- a/tensorflow/contrib/learn/python/learn/estimators/dynamic_rnn_estimator.py
+++ b/tensorflow/contrib/learn/python/learn/estimators/dynamic_rnn_estimator.py
@@ -639,7 +639,7 @@ class DynamicRnnEstimator(estimator.Estimator):
       ValueError: `problem_type` is not one of
         `ProblemType.LINEAR_REGRESSION` or `ProblemType.CLASSIFICATION`.
       ValueError: `problem_type` is `ProblemType.CLASSIFICATION` but
-        `num_classes` is not specifieProblemType
+        `num_classes` is not specified.
       ValueError: `prediction_type` is not one of
         `PredictionType.MULTIPLE_VALUE` or `PredictionType.SINGLE_VALUE`.
     """
-- 
GitLab


From 2daa40f9d096d47fc3add05a36fb7e41a00ba69d Mon Sep 17 00:00:00 2001
From: Yangzihao Wang <yangzihao@google.com>
Date: Fri, 6 Oct 2017 09:35:06 -0700
Subject: [PATCH 106/909] Fix transpose bug for large dimension. Add random
 tests of large shapes for better coverage. Update transpose benchmark with
 cases that swap one small dimension with one large dimension.

PiperOrigin-RevId: 171302097
---
 tensorflow/core/kernels/conv_ops_gpu_3.cu.cc  | 127 +++++++++++++++-
 tensorflow/python/BUILD                       |  20 +++
 .../python/kernel_tests/transpose_op_test.py  |  74 +++++++++
 tensorflow/python/ops/conv2d_benchmark.py     | 141 ++++++++++++++++++
 tensorflow/python/ops/transpose_benchmark.py  |  48 ++++--
 5 files changed, 393 insertions(+), 17 deletions(-)
 create mode 100644 tensorflow/python/ops/conv2d_benchmark.py

diff --git a/tensorflow/core/kernels/conv_ops_gpu_3.cu.cc b/tensorflow/core/kernels/conv_ops_gpu_3.cu.cc
index 3d4670c9ba..9083626fbf 100644
--- a/tensorflow/core/kernels/conv_ops_gpu_3.cu.cc
+++ b/tensorflow/core/kernels/conv_ops_gpu_3.cu.cc
@@ -272,6 +272,88 @@ __global__ void SwapDimension1And2InTensor3UsingTiles(const T* input,
   }
 }
 
+// Use shared memory tiles to swap dimension-1 and dimension-2 of a 3D tensor
+// when only one of the dimension sizes is smaller than 16,
+// where dimensions are zero-based: output[i][j][k] = input[i][k][j].
+//
+// small_dim = the_smaller_dimension_size
+// large_dim = the_larger_dimension_size
+// tile_num_per_block = blockDim.x
+// kTileLength = small_dim
+//
+// Each thread block operates on a single rectangle tile, where its width is
+// kTileLength (we currently set it to 64) and its height is small_dim,
+// We set the thread block's X dimension to be tile_num_per_block, and its Y
+// and Z to be one.
+template <typename T, int ShmemSize, bool SmallDim2>
+__global__ void SwapDimension1And2InTensor3SmallDim(const T* input,
+                                                    int batch_per_block,
+                                                    Dimension<3> input_dims,
+                                                    T* output) {
+  // TODO(yangzihao) avoid share memory bank conflict.
+  __shared__ T shared_memory_tile[ShmemSize];
+
+  eigen_assert(blockDim.y == 1);
+  eigen_assert(blockDim.z == 1);
+  eigen_assert(gridDim.z == 1);
+
+  int block_offset = blockIdx.x * blockDim.x;
+
+  int x = threadIdx.x;
+  int tile_height = blockDim.x;
+
+  // Get tile height, width, and thread/block origin indices.
+  int small_dim = SmallDim2 ? input_dims[2] : input_dims[1];
+  int large_dim = SmallDim2 ? input_dims[1] : input_dims[2];
+
+  int global_offset = small_dim * large_dim * (blockIdx.y * batch_per_block) +
+                      (SmallDim2 ? block_offset * small_dim : block_offset);
+  if (global_offset >= (input_dims[0] * input_dims[1] * input_dims[2])) return;
+
+  for (int batch = 0; batch < batch_per_block; ++batch) {
+    int block_origin_idx =
+        small_dim * large_dim * (blockIdx.y * batch_per_block + batch);
+    int thread_origin_idx =
+        block_origin_idx +
+        (SmallDim2 ? block_offset * small_dim : block_offset) + x;
+
+    if (block_offset + blockDim.x > large_dim) {
+      tile_height = large_dim - block_offset;
+    }
+
+    __syncthreads();
+
+    // Load a continuous memory region to shared memory tile.
+    if (x < tile_height) {
+      for (int y = 0; y < small_dim; y++) {
+        int shmem_index =
+            SmallDim2 ? (x + y * tile_height) : (x * small_dim + y);
+        shared_memory_tile[shmem_index] =
+            ldg(input + thread_origin_idx +
+                y * (SmallDim2 ? tile_height : large_dim));
+      }
+    }
+
+    __syncthreads();
+
+    // Get block origin index for output array.
+    int output_block_offset = block_origin_idx;
+    int output_block_idx = SmallDim2 ? block_offset : block_offset * small_dim;
+    int output_block_origin_idx = output_block_offset + output_block_idx;
+
+    // Store the tranposed memory region in shared memory to device.
+    if (x < tile_height) {
+      for (int y = 0; y < small_dim; y++) {
+        int output_idx = output_block_origin_idx + x +
+                         y * (SmallDim2 ? large_dim : tile_height);
+        int shmem_index =
+            SmallDim2 ? (x * small_dim + y) : (x + y * tile_height);
+        output[output_idx] = shared_memory_tile[shmem_index];
+      }
+    }
+  }
+}
+
 // A Cuda custom kernel that convert input to output, given proper padding on
 // the left and the top. The padded value is zero.
 template <typename T, int NDIMS>
@@ -420,25 +502,62 @@ template <typename T>
 void RunSwapDimension1And2InTensor3(const GPUDevice& d, const T* input,
                                     const Dimension<3>& input_dims, T* output) {
   // If both dimensions are not trivial, use tiles for the actual swapping.
+  // If one dimension is trivial, use SmallDim kernel for swapping.
   // Otherwise, the trivial swapping relying on the ldg cache is more efficient.
   static const int kMinDimensionToUseTiles = 16;
   bool use_tiles = (input_dims[1] >= kMinDimensionToUseTiles &&
                     input_dims[2] >= kMinDimensionToUseTiles);
+  bool use_small_dim = ((input_dims[1] >= kMinDimensionToUseTiles &&
+                         input_dims[2] < kMinDimensionToUseTiles)) ||
+                       ((input_dims[1] < kMinDimensionToUseTiles &&
+                         input_dims[2] >= kMinDimensionToUseTiles));
+  static const int NumSubTiles = 8;
+
   if (use_tiles) {
-    // We get best performance when TileSize is the number of threads in a warp
-    // (32 on our GPUs) and NumSubTiles is 8, so our block size is 8 * 32 = 256
-    // threads.
     static const int TileSize = 32;
-    static const int NumSubTiles = 8;
     Dimension<3> input_dims_in_tiles = {
         input_dims[0], (input_dims[1] + TileSize - 1) / TileSize,
         (input_dims[2] + TileSize - 1) / TileSize,
     };
     int total_tiles_count = input_dims_in_tiles[0] * input_dims_in_tiles[1] *
                             input_dims_in_tiles[2];
+    // We get best performance when TileSize is the number of threads in a warp
+    // (32 on our GPUs) and NumSubTiles is 8, so our block size is 8 * 32 = 256
+    // threads.
     SwapDimension1And2InTensor3UsingTiles<T, TileSize, NumSubTiles><<<
         total_tiles_count, dim3(TileSize, NumSubTiles), 0, d.stream()>>>(
         input, input_dims, output);
+  } else if (use_small_dim) {
+    // When only one of the dimensions is smaller than kMinDimensionToUseTiles,
+    // we use one block to process a rectangle region with the size of
+    // kTileLength * small_dim. We found that when set kTileLength to 64 on
+    // TitanX Maxwell GPU, it achieves the best performance.
+    //              large_dim
+    //            +---------------...--------+
+    //            |            |        |    |
+    // small_dim  |            |  ...   |    |
+    //            |            |        |    |
+    //            +--------------...---------+
+    //            \----- ------/         \- -/
+    //                  V                  V
+    //    kTileLength(tile_height)    tile_height
+    static const int kTileLength = 64;
+    static const int kGridDimY = 65535;
+    int large_dim = std::max(input_dims[2], input_dims[1]);
+    int tile_num_per_block = (large_dim + kTileLength - 1) / kTileLength;
+    int grid_dim_y = std::min(input_dims[0], kGridDimY);
+    int batch_per_block = (input_dims[0] + grid_dim_y - 1) / grid_dim_y;
+    if (input_dims[2] < input_dims[1]) {
+      SwapDimension1And2InTensor3SmallDim<
+          T, kTileLength * kMinDimensionToUseTiles, true>
+          <<<dim3(tile_num_per_block, grid_dim_y), kTileLength, 0,
+             d.stream()>>>(input, batch_per_block, input_dims, output);
+    } else {
+      SwapDimension1And2InTensor3SmallDim<
+          T, kTileLength * kMinDimensionToUseTiles, false>
+          <<<dim3(tile_num_per_block, grid_dim_y), kTileLength, 0,
+             d.stream()>>>(input, batch_per_block, input_dims, output);
+    }
   } else {
     int total_element_count = input_dims[0] * input_dims[1] * input_dims[2];
     CudaLaunchConfig config = GetCudaLaunchConfig(total_element_count, d);
diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index ab3b851ef8..bdbad14660 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -4060,6 +4060,26 @@ cuda_py_test(
     main = "ops/concat_benchmark.py",
 )
 
+cuda_py_test(
+    name = "conv2d_benchmark",
+    size = "large",
+    srcs = ["ops/conv2d_benchmark.py"],
+    additional_deps = [
+        ":client",
+        ":client_testlib",
+        ":control_flow_ops",
+        ":framework_for_generated_wrappers",
+        ":nn_ops",
+        ":platform",
+        ":platform_benchmark",
+        ":random_ops",
+        ":variables",
+        "//third_party/py/numpy",
+        "//tensorflow/core:protos_all_py",
+    ],
+    main = "ops/conv2d_benchmark.py",
+)
+
 cuda_py_test(
     name = "split_benchmark",
     srcs = ["ops/split_benchmark.py"],
diff --git a/tensorflow/python/kernel_tests/transpose_op_test.py b/tensorflow/python/kernel_tests/transpose_op_test.py
index 570fa79944..9e1f83395b 100644
--- a/tensorflow/python/kernel_tests/transpose_op_test.py
+++ b/tensorflow/python/kernel_tests/transpose_op_test.py
@@ -229,6 +229,80 @@ class TransposeTest(test.TestCase):
         self.assertAllEqual(np_ans, tf_ans)
         self.assertShapeEqual(np_ans, y)
 
+  def testLargeSizeGPU(self):
+    # If no GPU available, skip the test
+    if not test.is_gpu_available(cuda_only=True):
+      return
+
+    large_shapes = [[1000000, 31, 3], [3, 1000000, 31], [3, 31, 1000000],
+                    [10000, 310, 3], [3, 10000, 310], [3, 310, 10000],
+                    [2, 1000, 1000], [1000, 2, 1000], [1000, 1000, 2]]
+    perms = [[0, 2, 1]] * 9
+
+    for input_shape, perm in zip(large_shapes, perms):
+      total_size = np.prod(input_shape)
+      inp = np.arange(1, total_size + 1, dtype=np.float32).reshape(input_shape)
+      np_ans = self._np_transpose(inp, perm)
+      with self.test_session(use_gpu=True):
+        inx = ops.convert_to_tensor(inp)
+        y = array_ops.transpose(inx, perm)
+        tf_ans = y.eval()
+      self.assertAllEqual(np_ans, tf_ans)
+      self.assertShapeEqual(np_ans, y)
+
+  def testRandomizedSmallDimLargeSizeGPU(self):
+    # If no GPU available, skip the test
+    if not test.is_gpu_available(cuda_only=True):
+      return
+
+    # Draw 10 random shapes with large dimension sizes.
+    # 40% prob to generate dim[0] size within [1, 2047]
+    # 40% prob to generate dim[0] size within [2048, 4095]
+    # 20% prob to generate dim[0] size within [4096, 100000]
+    # 50% prob to use dim[1] as the small dim (<16)
+    num_samples = 10
+    total_size = 500000
+    small_size_limit = 2048
+    large_size_limit = 95905
+    small_size_percentage = 0.4
+    medium_size_percentage = 0.4
+    large_size_percentage = 0.2
+    perms = [[0, 2, 1]] * num_samples
+    dim_zero_sizes = []
+    dim_zero_sizes += list(
+        np.random.randint(
+            small_size_limit, size=int(small_size_percentage * num_samples)) +
+        1)
+    dim_zero_sizes += list(
+        np.random.randint(
+            small_size_limit, size=int(medium_size_percentage * num_samples)) +
+        small_size_limit)
+    dim_zero_sizes += list(
+        np.random.randint(
+            large_size_limit, size=int(large_size_percentage * num_samples)) +
+        small_size_limit * 2)
+    input_shapes = []
+    small_dim_limit = 16
+    for dim_zero_size in dim_zero_sizes:
+      small_dim_size = np.random.randint(small_dim_limit - 1) + 1
+      large_dim_size = int(
+          total_size / dim_zero_size / small_dim_size) + small_dim_limit
+      input_shapes += ([[dim_zero_size, small_dim_size, large_dim_size]]
+                       if np.random.randint(2) else
+                       [[dim_zero_size, large_dim_size, small_dim_size]])
+
+    for input_shape, perm in zip(input_shapes, perms):
+      # generate input data with random ints from 0 to 9.
+      inp = np.random.randint(10, size=input_shape)
+      np_ans = self._np_transpose(inp, perm)
+      with self.test_session(use_gpu=True):
+        inx = ops.convert_to_tensor(inp)
+        y = array_ops.transpose(inx, perm)
+        tf_ans = y.eval()
+      self.assertAllEqual(np_ans, tf_ans)
+      self.assertShapeEqual(np_ans, y)
+      self._ClearCachedSession()
+
   def testNop(self):
     self._compareCpu(np.arange(0, 6).reshape([3, 2]).astype(np.float32), [0, 1])
 
diff --git a/tensorflow/python/ops/conv2d_benchmark.py b/tensorflow/python/ops/conv2d_benchmark.py
new file mode 100644
index 0000000000..6992fa57ea
--- /dev/null
+++ b/tensorflow/python/ops/conv2d_benchmark.py
@@ -0,0 +1,141 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Benchmark for Conv2D op."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import itertools
+import time
+
+from tensorflow.python.client import session as session_lib
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import nn_ops
+from tensorflow.python.ops import random_ops
+from tensorflow.python.ops import variables
+from tensorflow.python.platform import test
+
+
+def build_graph(device, input_shape, filter_shape, strides, padding, num_iters):
+  """builds a graph containing a sequence of conv2d operations.
+
+  Args:
+    device: String, the device to run on.
+    input_shape: Shape of the input tensor.
+    filter_shape: Shape of the filter tensor.
+    strides: A list of ints. 1-D of length 4. The stride of sliding
+             window for each dimension of input.
+    padding: A string from: "SAME", "VALID". The type of padding
+             algorithm to use.
+    num_iters: number of iterations to run conv2d.
+
+  Returns:
+    An array of tensors to run()
+  """
+  with ops.device("/%s:0" % device):
+    inp = variables.Variable(random_ops.truncated_normal(input_shape))
+    filt = variables.Variable(random_ops.truncated_normal(filter_shape))
+
+    outputs = []
+    conv2d_op = nn_ops.conv2d(inp, filt, strides, padding, data_format="NHWC")
+    outputs.append(conv2d_op)
+    for _ in range(1, num_iters):
+      with ops.control_dependencies([conv2d_op]):
+        conv2d_op = nn_ops.conv2d(
+            inp, filt, strides, padding, data_format="NHWC")
+        outputs.append(conv2d_op)
+    return control_flow_ops.group(*outputs)
+
+
+class Conv2DBenchmark(test.Benchmark):
+  """Benchmark conv2d!"""
+
+  def _run_graph(self, device, input_shape, filter_shape, strides, padding,
+                 num_iters):
+    """runs the graph and print its execution time.
+
+    Args:
+      device: String, the device to run on.
+      input_shape: Shape of the input tensor.
+      filter_shape: Shape of the filter tensor.
+      strides: A list of ints. 1-D of length 4. The stride of sliding
+               window for each dimension of input.
+      padding: A string from: "SAME", "VALID". The type of padding
+               algorithm to use.  num_iters: Number of iterations to run the
+                 benchmark.
+      num_iters: number of iterations to run conv2d.
+
+    Returns:
+      The duration of the run in seconds.
+    """
+    graph = ops.Graph()
+    with graph.as_default():
+      outputs = build_graph(device, input_shape, filter_shape, strides, padding,
+                            num_iters)
+      with session_lib.Session(graph=graph) as session:
+        variables.global_variables_initializer().run()
+        # warmup runs
+        session.run(outputs)
+
+        start_time = time.time()
+        session.run(outputs)
+        duration = (time.time() - start_time) / num_iters
+
+        print("%s inputshape:%s filtershape:%s strides:%s padding:%s "
+              "%d iters: %.8f sec" %
+              (device, str(input_shape).replace(" ", ""),
+               str(filter_shape).replace(" ", ""),
+               str(strides).replace(" ", ""), padding, num_iters, duration))
+
+    name_template = (
+        "conv2d_{device}_input_shape_{inputshape}_filter_shape_{filtershape}_"
+        "strides_{strides}_padding_{padding}")
+
+    self.report_benchmark(
+        name=name_template.format(
+            device=device,
+            inputshape=str(input_shape).replace(" ", ""),
+            filtershape=str(filter_shape).replace(" ", ""),
+            strides=str(strides).replace(" ", ""),
+            padding=padding).replace(" ", ""),
+        iters=num_iters,
+        wall_time=duration / num_iters)
+
+    return duration
+
+  def benchmark_conv2d(self):
+    print("conv2d benchmark:")
+
+    h = 500
+    w = 500
+    fh = 3
+    fw = 3
+    input_shapes = []
+    filter_shapes = []
+    for b, c in itertools.product([4, 16, 32], [i for i in range(3, 16)]):
+      input_shapes += [[b, h, w, c]]
+      filter_shapes += [[fh, fw, c, b]]
+    strides = [[1, 2, 2, 1]]
+    paddings = ["VALID", "SAME"]
+    for ishape, fshape in zip(input_shapes, filter_shapes):
+      for stride in strides:
+        for padding in paddings:
+          self._run_graph("gpu", ishape, fshape, stride, padding, 80)
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/python/ops/transpose_benchmark.py b/tensorflow/python/ops/transpose_benchmark.py
index 63a314295e..6b5f0f20d8 100644
--- a/tensorflow/python/ops/transpose_benchmark.py
+++ b/tensorflow/python/ops/transpose_benchmark.py
@@ -1,4 +1,4 @@
-# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -32,7 +32,7 @@ from tensorflow.python.platform import test
 
 
 def build_graph(device, input_shape, perm, datatype, num_iters):
-  """Build a graph containing a sequence of conv2d operations.
+  """builds a graph containing a sequence of conv2d operations.
 
   Args:
     device: String, the device to run on.
@@ -50,10 +50,12 @@ def build_graph(device, input_shape, perm, datatype, num_iters):
     t = constant_op.constant(inp, shape=input_shape)
 
     outputs = []
-    outputs.append(array_ops.transpose(t, perm))
-    for i in range(1, num_iters):
-      with ops.control_dependencies([outputs[i - 1]]):
-        outputs.append(array_ops.transpose(t, perm))
+    transpose_op = array_ops.transpose(t, perm)
+    outputs.append(transpose_op)
+    for _ in range(1, num_iters):
+      with ops.control_dependencies([transpose_op]):
+        transpose_op = array_ops.transpose(t, perm)
+        outputs.append(transpose_op)
     return control_flow_ops.group(*outputs)
 
 
@@ -61,7 +63,7 @@ class TransposeBenchmark(test.Benchmark):
   """Benchmark transpose!"""
 
   def _run_graph(self, device, input_shape, perm, num_iters, datatype):
-    """Run the graph and print its execution time.
+    """runs the graph and print its execution time.
 
     Args:
       device: String, the device to run on.
@@ -82,9 +84,11 @@ class TransposeBenchmark(test.Benchmark):
         session.run(outputs)
         start_time = time.time()
         session.run(outputs)
+
         duration = (time.time() - start_time) / num_iters
         throughput = np.prod(
             np.array(input_shape)) * datatype().itemsize * 2 / duration / 1e9
+
         print("%s %s inputshape:%s perm:%s %d %.6fsec, %.4fGB/s." %
               (device, str(datatype), str(input_shape).replace(" ", ""),
                str(perm).replace(" ", ""), num_iters, duration, throughput))
@@ -108,12 +112,12 @@ class TransposeBenchmark(test.Benchmark):
 
     datatypes = [np.complex128, np.float64, np.float32, np.float16, np.int8]
 
-    small_shapes = [[2, 20, 20, 20, 16], [2, 16, 20, 20, 20]] * 2 + [[
-        2, 100, 100, 16
-    ], [2, 16, 100, 100]] * 2 + [[2, 5000, 16], [2, 16, 5000]] * 2
-    small_perms = [[0, 4, 1, 2, 3], [0, 2, 3, 4, 1]] + [[4, 1, 2, 3, 0]] * 2 + [
-        [0, 3, 1, 2], [0, 2, 3, 1]
-    ] + [[3, 1, 2, 0]] * 2 + [[0, 2, 1]] * 2 + [[2, 1, 0]] * 2
+    small_shapes = [[2, 20, 20, 20, 16], [2, 16, 20, 20, 20]] * 2
+    small_shapes += [[2, 100, 100, 16], [2, 16, 100, 100]] * 2
+    small_shapes += [[2, 5000, 16], [2, 16, 5000]] * 2
+    small_perms = [[0, 4, 1, 2, 3], [0, 2, 3, 4, 1]] + [[4, 1, 2, 3, 0]] * 2
+    small_perms += [[0, 3, 1, 2], [0, 2, 3, 1]] + [[3, 1, 2, 0]] * 2
+    small_perms += [[0, 2, 1]] * 2 + [[2, 1, 0]] * 2
 
     large_shapes = [[2, 40, 40, 40, 32], [2, 40, 40, 40, 64]] * 2 + [[
         2, 300, 300, 32
@@ -132,5 +136,23 @@ class TransposeBenchmark(test.Benchmark):
           for ishape, perm in zip(large_shapes, large_perms):
             self._run_graph("gpu", ishape, perm, num_iters, datatype)
 
+    small_dim_large_shapes = [[2, 10000, 3], [2, 3, 10000], [2, 10000, 8],
+                              [2, 8, 10000]]
+    small_dim_small_shapes = [[2, 5000, 3], [2, 3, 5000], [2, 5000, 8],
+                              [2, 8, 5000]]
+    small_dim_perms = [[0, 2, 1]] * 4
+
+    num_iters = 320
+    small_dim_large_shape_datatypes = [np.float64, np.float32, np.int8]
+    for datatype in small_dim_large_shape_datatypes:
+      for ishape, perm in zip(small_dim_large_shapes, small_dim_perms):
+        self._run_graph("gpu", ishape, perm, num_iters, datatype)
+
+    small_dim_small_shape_datatypes = [np.complex128, np.float16]
+    for datatype in small_dim_small_shape_datatypes:
+      for ishape, perm in zip(small_dim_small_shapes, small_dim_perms):
+        self._run_graph("gpu", ishape, perm, num_iters, datatype)
+
+
 if __name__ == "__main__":
   test.main()
-- 
GitLab


From 3acd57c2ffff6055b322ba08ba74fa1885fbba19 Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Fri, 6 Oct 2017 09:37:33 -0700
Subject: [PATCH 107/909] Fuse TFE_NewOp and TFE_OpGetAttrType to avoid leaking
 memory.

Removes TFE_NewOp and TFE_OpGetAttrType from pywrap_tensorflow, adds TFE_OpNameGetAttrType.

PiperOrigin-RevId: 171302338
---
 tensorflow/c/eager/c_api.cc         | 14 ++++++++++++++
 tensorflow/c/eager/c_api.h          |  6 ++++++
 tensorflow/python/eager/backprop.py |  4 ++--
 tensorflow/python/pywrap_tfe.i      |  3 +--
 4 files changed, 23 insertions(+), 4 deletions(-)

diff --git a/tensorflow/c/eager/c_api.cc b/tensorflow/c/eager/c_api.cc
index 74f2e4f342..514a4010bc 100644
--- a/tensorflow/c/eager/c_api.cc
+++ b/tensorflow/c/eager/c_api.cc
@@ -273,6 +273,20 @@ TF_AttrType TFE_OpGetAttrType(TFE_Op* op, const char* attr_name,
   return ret;
 }
 
+TF_AttrType TFE_OpNameGetAttrType(TFE_Context* ctx,
+                                  const char* op_or_function_name,
+                                  const char* attr_name, unsigned char* is_list,
+                                  TF_Status* status) {
+  TF_AttrType ret;
+  TFE_Op* op = TFE_NewOp(ctx, op_or_function_name, status);
+  if (!status->status.ok()) {
+    return TF_ATTR_INT;  // Same dummy return as TFE_OpGetAttrType.
+  }
+  ret = TFE_OpGetAttrType(op, attr_name, is_list, status);
+  TFE_DeleteOp(op);
+  return ret;
+}
+
 void TFE_OpSetAttrString(TFE_Op* op, const char* attr_name, const char* value) {
   op->attrs.Set(attr_name, value);
 }
diff --git a/tensorflow/c/eager/c_api.h b/tensorflow/c/eager/c_api.h
index a4f7d308fb..9bfa63711b 100644
--- a/tensorflow/c/eager/c_api.h
+++ b/tensorflow/c/eager/c_api.h
@@ -107,6 +107,12 @@ TF_CAPI_EXPORT extern void TFE_OpAddInput(TFE_Op* op, TFE_TensorHandle* h, TF_St
 
 TF_CAPI_EXPORT extern TF_AttrType TFE_OpGetAttrType(TFE_Op* op, const char* attr_name,
                                                     unsigned char* is_list, TF_Status* status);
+// Get an attribute type given an op name; a fusion of TFE_NewOp and
+// TFE_OpGetAttrType for use from Python without the overhead of the individual
+// calls and memory management of TFE_Op.
+TF_CAPI_EXPORT extern TF_AttrType TFE_OpNameGetAttrType(
+    TFE_Context* ctx, const char* op_or_function_name, const char* attr_name,
+    unsigned char* is_list, TF_Status* status);
 
 TF_CAPI_EXPORT extern void TFE_OpSetAttrString(TFE_Op* op, const char* attr_name,
                                                const char* value);
diff --git a/tensorflow/python/eager/backprop.py b/tensorflow/python/eager/backprop.py
index 3c84cbbd6f..cca8e47044 100644
--- a/tensorflow/python/eager/backprop.py
+++ b/tensorflow/python/eager/backprop.py
@@ -49,8 +49,8 @@ def op_attr_type(op_type, attr_name):
   except KeyError:
     with errors.raise_exception_on_not_ok_status() as status:
       h = context.context()._handle  # pylint: disable=protected-access
-      op = pywrap_tensorflow.TFE_NewOp(h, op_type, status)
-      attr_type = pywrap_tensorflow.TFE_OpGetAttrType(op, attr_name, status)
+      attr_type = pywrap_tensorflow.TFE_OpNameGetAttrType(
+          h, op_type, attr_name, status)
     _op_attr_type_cache[(op_type, attr_name)] = attr_type
     return attr_type
 
diff --git a/tensorflow/python/pywrap_tfe.i b/tensorflow/python/pywrap_tfe.i
index 128e46e6ce..d5b7294c82 100644
--- a/tensorflow/python/pywrap_tfe.i
+++ b/tensorflow/python/pywrap_tfe.i
@@ -19,8 +19,7 @@ limitations under the License.
 %rename("%s") TFE_DeleteContext;
 %rename("%s") TFE_ContextListDevices;
 %rename("%s") TFE_ContextAddFunctionDef;
-%rename("%s") TFE_NewOp;
-%rename("%s") TFE_OpGetAttrType;
+%rename("%s") TFE_OpNameGetAttrType;
 %rename("%s") TFE_Py_InitEagerTensor;
 %rename("%s") TFE_Py_RegisterExceptionClass;
 %rename("%s") TFE_Py_Execute;
-- 
GitLab


From 8fcbef3428ce69de9cedafd0d4c0f141c79d418c Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Fri, 6 Oct 2017 09:46:44 -0700
Subject: [PATCH 108/909] [XLA:LLVM] Annotate tuple instructions with AA
 metadata.

PiperOrigin-RevId: 171303412
---
 tensorflow/compiler/xla/service/llvm_ir/ops.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xla/service/llvm_ir/ops.cc b/tensorflow/compiler/xla/service/llvm_ir/ops.cc
index ac562e231c..3965433494 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/ops.cc
+++ b/tensorflow/compiler/xla/service/llvm_ir/ops.cc
@@ -73,12 +73,13 @@ void EmitTuple(IrArray tuple,
                tensorflow::gtl::ArraySlice<llvm::Value*> operands,
                llvm::IRBuilder<>* ir_builder) {
   for (size_t i = 0; i < operands.size(); ++i) {
-    ir_builder->CreateStore(
+    auto* store = ir_builder->CreateStore(
         ir_builder->CreatePointerCast(operands[i],
                                       PrimitiveTypeToIrType(TUPLE, ir_builder)),
         ir_builder->CreateInBoundsGEP(
             tuple.GetBasePointer(),
             {ir_builder->getInt64(0), ir_builder->getInt64(i)}));
+    tuple.AnnotateLoadStoreInstructionWithMetadata(store);
   }
 }
 
-- 
GitLab


From a9104e7529eb75454aaaa2ea29b8ebe40ee7bbd0 Mon Sep 17 00:00:00 2001
From: Brennan Saeta <saeta@google.com>
Date: Fri, 6 Oct 2017 09:46:44 -0700
Subject: [PATCH 109/909] Add documentation to sloppy_interleave function

PiperOrigin-RevId: 171303413
---
 tensorflow/contrib/data/python/ops/sloppy_ops.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/tensorflow/contrib/data/python/ops/sloppy_ops.py b/tensorflow/contrib/data/python/ops/sloppy_ops.py
index 058c497320..4f3da4320c 100644
--- a/tensorflow/contrib/data/python/ops/sloppy_ops.py
+++ b/tensorflow/contrib/data/python/ops/sloppy_ops.py
@@ -102,6 +102,17 @@ def sloppy_interleave(map_func, cycle_length, block_length=1):
   strictly obeys), producing an element from a different underlying
   dataset instead.
 
+  Example usage:
+
+  ```python
+  # Preprocess 4 files concurrently.
+  filenames = tf.data.Dataset.list_files("/path/to/data/train*.tfrecords")
+  dataset = filenames.apply(
+      tf.contrib.data.sloppy_interleave(
+          lambda filename: tf.data.TFRecordDataset(filename),
+          cycle_length=4))
+  ```
+
   WARNING: The order of elements in the resulting dataset is not
   deterministic. Use `Dataset.interleave()` if you want the elements to have a
   deterministic order.
-- 
GitLab


From 420d166e7f79d37d1be66d648dd99131068a8537 Mon Sep 17 00:00:00 2001
From: Yunxing Dai <yunxing@google.com>
Date: Fri, 6 Oct 2017 09:51:05 -0700
Subject: [PATCH 110/909] Use a serialized graph compiler to generate xla
 graph.

- Move away from previous TF graph executor, which contains few features that we need and also introduces indeterminism.
- Unlike previous executor, the new serial graph compiler doesn't recurse into a function and inlines it. Instead, it creates a computation of the function and then creates a `call` op to call into the newly created computation.
- Add a optional comparator in DFS algorithm, which is needed to make the compiler deterministic.

RELNOTES: Use a determinisitc executor to generate xla graph.
PiperOrigin-RevId: 171303938
---
 tensorflow/compiler/tf2xla/BUILD              |   2 +
 tensorflow/compiler/tf2xla/graph_compiler.cc  | 185 ++++++++++++++++++
 tensorflow/compiler/tf2xla/graph_compiler.h   | 103 ++++++++++
 tensorflow/compiler/tf2xla/xla_compiler.cc    |  98 +++++-----
 tensorflow/compiler/tf2xla/xla_compiler.h     |   2 +-
 .../compiler/tf2xla/xla_compiler_test.cc      |  69 ++++++-
 tensorflow/compiler/xla/service/service.cc    |   5 +-
 tensorflow/core/graph/algorithm.cc            |  64 ++++--
 tensorflow/core/graph/algorithm.h             |  43 +++-
 tensorflow/core/graph/algorithm_test.cc       |  35 ++++
 tensorflow/core/graph/graph.h                 |   4 +-
 11 files changed, 530 insertions(+), 80 deletions(-)
 create mode 100644 tensorflow/compiler/tf2xla/graph_compiler.cc
 create mode 100644 tensorflow/compiler/tf2xla/graph_compiler.h

diff --git a/tensorflow/compiler/tf2xla/BUILD b/tensorflow/compiler/tf2xla/BUILD
index 4da2ed722e..647bfd1849 100644
--- a/tensorflow/compiler/tf2xla/BUILD
+++ b/tensorflow/compiler/tf2xla/BUILD
@@ -102,11 +102,13 @@ cc_library(
         "xla_helpers.cc",
         "xla_op_kernel.cc",
         "xla_op_registry.cc",
+        "graph_compiler.cc",
         "xla_cpu_backend.cc",
     ] + if_cuda_is_configured([
         "xla_gpu_backend.cc",
     ]),
     hdrs = [
+        "graph_compiler.h",
         "xla_compilation_device.h",
         "xla_compiler.h",
         "xla_context.h",
diff --git a/tensorflow/compiler/tf2xla/graph_compiler.cc b/tensorflow/compiler/tf2xla/graph_compiler.cc
new file mode 100644
index 0000000000..c168266b16
--- /dev/null
+++ b/tensorflow/compiler/tf2xla/graph_compiler.cc
@@ -0,0 +1,185 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/tf2xla/graph_compiler.h"
+
+#include <deque>
+#include <numeric>
+#include <vector>
+
+#include "tensorflow/compiler/tf2xla/dump_graph.h"
+#include "tensorflow/compiler/tf2xla/functionalize_control_flow.h"
+#include "tensorflow/compiler/tf2xla/shape_util.h"
+#include "tensorflow/compiler/tf2xla/type_util.h"
+#include "tensorflow/compiler/tf2xla/xla_compilation_device.h"
+#include "tensorflow/compiler/tf2xla/xla_context.h"
+#include "tensorflow/compiler/tf2xla/xla_op_kernel.h"
+#include "tensorflow/compiler/xla/client/client_library.h"
+#include "tensorflow/core/common_runtime/device.h"
+#include "tensorflow/core/common_runtime/executor.h"
+#include "tensorflow/core/common_runtime/function.h"
+#include "tensorflow/core/common_runtime/graph_optimizer.h"
+#include "tensorflow/core/framework/attr_value_util.h"
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/graph/algorithm.h"
+#include "tensorflow/core/graph/graph_constructor.h"
+#include "tensorflow/core/graph/node_builder.h"
+#include "tensorflow/core/lib/hash/hash.h"
+#include "tensorflow/core/platform/logging.h"
+#include "tensorflow/core/public/version.h"
+
+namespace tensorflow {
+
+Status GraphCompiler::Compile() {
+  std::vector<NodeBinding> bindings(graph_->num_node_ids());
+  std::vector<Node*> topo_sorted_nodes;
+  // XLA requires determinism, generate a stable ordering from DFS.
+  GetReversePostOrder(*graph_, &topo_sorted_nodes,
+                      /*stable_comparator=*/NodeComparatorID());
+
+  OpKernelContext::Params params;
+  PartiallySetupParams(&params);
+
+  for (Node* n : topo_sorted_nodes) {
+    // Set up bindings.
+    NodeBinding& binding = bindings[n->id()];
+    binding.node = n;
+    Status s = flib_->CreateKernel(n->def(), &binding.op_kernel);
+    binding.output_attrs.resize(n->num_outputs());
+    if (!s.ok()) {
+      binding.op_kernel = nullptr;
+      s = AttachDef(s, *n);
+      LOG(ERROR) << "Executor failed to create kernel. " << s;
+      return s;
+    }
+  }
+
+  // Bindings are initialized by the size of graph_->num_node_ids. However, the
+  // graph may contain dead nodes that still hold a valid node id. Thus
+  // graph_->num_node_ids could be larger than number of topo sorted nodes.
+  TF_RET_CHECK(bindings.size() >= topo_sorted_nodes.size());
+
+  for (Node* n : topo_sorted_nodes) {
+    TF_RET_CHECK(!n->IsRecv() && !n->IsSend() && !n->IsSwitch())
+        << "Not supported node: " << n->DebugString();
+    NodeBinding& binding = bindings[n->id()];
+    params.op_kernel = binding.op_kernel;
+    params.output_attr_array = binding.output_attrs.data();
+
+    // tensor_inputs_ is a buffer reused across graph traversal. We clean up and
+    // reinitialize the buffer before we visit a new node.
+    tensor_inputs_.clear();
+    tensor_inputs_.resize(n->num_inputs());
+
+    // Set up inputs from outputs of previous nodes.
+    for (auto* e : n->in_edges()) {
+      if (e->IsControlEdge()) continue;
+      Node* src = e->src();
+      tensor_inputs_[e->dst_input()] =
+          bindings[src->id()].tensor_values[e->src_output()];
+    }
+
+    OpKernelContext op_context(&params, n->num_outputs());
+    if (IsFunctional(n)) {
+      TF_RETURN_IF_ERROR(CompileFunctionalNode(n, &op_context));
+    } else {
+      device_->Compute(CHECK_NOTNULL(params.op_kernel), &op_context);
+      Status s = op_context.status();
+      TF_RETURN_IF_ERROR(s);
+    }
+
+    // Set up outputs. Also check if outputs from the previous computation is
+    // valid.
+    for (int o = 0; o < n->num_outputs(); ++o) {
+      const auto tensor_val = op_context.release_output(o);
+      if (*op_context.is_output_dead() || tensor_val.tensor == nullptr) {
+        return errors::Internal("Missing xla_context ", o, "-th output from ",
+                                (*op_context.is_output_dead() ? "(dead)" : ""),
+                                SummarizeNode(*n));
+      }
+      binding.tensor_values.push_back(tensor_val);
+    }
+  }
+
+  // Clean up tensor data and op kernels.
+  for (NodeBinding& binding : bindings) {
+    delete binding.op_kernel;
+    for (auto& t : binding.tensor_values) {
+      if (!t.is_ref()) {
+        delete t.tensor;
+      }
+    }
+  }
+  return Status::OK();
+}
+
+bool GraphCompiler::IsFunctional(Node* n) {
+  return n->type_string() == FunctionLibraryDefinition::kGradientOp ||
+         (flib_->GetFunctionLibraryDefinition()->Find(n->def().op()) !=
+          nullptr);
+}
+
+Status GraphCompiler::CompileFunctionalNode(Node* n,
+                                            OpKernelContext* op_context) {
+  TF_RET_CHECK(IsFunctional(n));
+  // For functional nodes, compile them using compiler_ and call into the
+  // functions.
+  XlaOpKernelContext xla_op_context(op_context);
+
+  std::vector<XlaCompiler::Argument> arguments;
+  XlaCompiler::CompilationResult result;
+  NameAttrList func;
+  if (flib_->GetFunctionLibraryDefinition()->Find(n->def().op())) {
+    func.set_name(n->def().op());
+  } else {
+    func.set_name(FunctionLibraryDefinition::kGradientOp);
+  }
+  *func.mutable_attr() = n->def().attr();
+
+  // Compile the graph using the function compiler.
+  TF_ASSIGN_OR_RETURN(auto computation, compiler_(func, &xla_op_context));
+  XlaContext& context = XlaContext::Get(op_context);
+  auto* b = context.builder();
+
+  // Graph data handles from the inputs.
+  std::vector<xla::ComputationDataHandle> handles;
+  for (auto tensor : tensor_inputs_) {
+    auto expression =
+        reinterpret_cast<const XlaExpression*>(tensor->tensor_data().data());
+    // TODO(yunxing): Support two rare cases below where input is a resource or
+    // contains a null handle.
+    TF_RET_CHECK(expression->resource() == nullptr)
+        << "Input with resource is not supported.";
+    TF_RET_CHECK(expression->handle().handle() != 0)
+        << "Invalid computation handle.";
+    handles.push_back(expression->handle());
+  }
+  auto output_handle = b->Call(*computation, handles);
+  // The output handle of `Call` computation is a tuple type. Unzip it so
+  // that it can into fit future computations.
+  for (int64 idx = 0; idx < n->num_outputs(); ++idx) {
+    xla_op_context.SetOutput(idx, b->GetTupleElement(output_handle, idx));
+  }
+  return b->first_error();
+}
+
+void GraphCompiler::PartiallySetupParams(OpKernelContext::Params* params) {
+  params->device = device_;
+  params->inputs = &tensor_inputs_;
+  params->step_container = step_container_;
+  params->resource_manager = device_->resource_manager();
+}
+
+}  // namespace tensorflow
diff --git a/tensorflow/compiler/tf2xla/graph_compiler.h b/tensorflow/compiler/tf2xla/graph_compiler.h
new file mode 100644
index 0000000000..6fc0b18dcd
--- /dev/null
+++ b/tensorflow/compiler/tf2xla/graph_compiler.h
@@ -0,0 +1,103 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_COMPILER_TF2XLA_GRAPH_COMPILER_H_
+#define TENSORFLOW_COMPILER_TF2XLA_GRAPH_COMPILER_H_
+
+#include "tensorflow/compiler/tf2xla/xla_compilation_device.h"
+#include "tensorflow/compiler/tf2xla/xla_context.h"
+#include "tensorflow/compiler/xla/client/local_client.h"
+#include "tensorflow/core/common_runtime/device.h"
+#include "tensorflow/core/common_runtime/device_mgr.h"
+#include "tensorflow/core/common_runtime/function.h"
+#include "tensorflow/core/framework/function.h"
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/platform/env.h"
+#include "tensorflow/core/platform/mutex.h"
+#include "tensorflow/core/platform/notification.h"
+#include "tensorflow/core/platform/thread_annotations.h"
+#include "tensorflow/core/public/version.h"
+
+namespace tensorflow {
+
+using FunctionCompiler =
+    std::function<xla::StatusOr<std::shared_ptr<xla::Computation>>(
+        const NameAttrList& function, XlaOpKernelContext* xla_op_context)>;
+
+// GraphCompiler compiles the graph in topological order in the current
+// thread. It also resolves the nondeterminism in the graph by enforcing a total
+// order on all inputs to a node. This abstraction helps us create the same XLA
+// computation given two structurally equivalent TensorFlow graphs. If a
+// function call is visited during the graph traversal, it is then compiled
+// through the FunctionCompiler into a computation and a `Call` operation is
+// inserted to call into that computation.
+class GraphCompiler {
+ public:
+  GraphCompiler(XlaContext* xla_context, XlaCompilationDevice* device,
+                Graph* graph, FunctionLibraryRuntime* flib,
+                ScopedStepContainer* step_container,
+                const FunctionCompiler& compiler)
+      : xla_context_(xla_context),
+        device_(device),
+        graph_(graph),
+        flib_(flib),
+        step_container_(step_container),
+        compiler_(compiler) {}
+
+  // Compiles the graph. The results are written in `xla_context` that is passed
+  // into the compiler.
+  Status Compile();
+
+ private:
+  // NodeBinding is a wrapper on a `Node` that also contains computed
+  // TensorValue.
+  struct NodeBinding {
+    const Node* node;
+    // Kernel for this node, to be filled by CreateKernel.
+    OpKernel* op_kernel;
+    // Output values of this node.
+    std::vector<TensorValue> tensor_values;
+    // Attributes of the outputs.
+    gtl::InlinedVector<AllocatorAttributes, 4> output_attrs;
+  };
+
+  // Partially sets params. This partially set params can be reused
+  // across multple nodes visit.
+  void PartiallySetupParams(OpKernelContext::Params* params);
+
+  // Tests if a node is a functional node. A functional node represents a
+  // defined computation and should be compiled using `compiler_`.
+  bool IsFunctional(Node*);
+
+  // Compiles a functional node and writes result to OpkernelContext. A
+  // functional node represents a defined computation and should be compiled
+  // using `compiler_`.
+  Status CompileFunctionalNode(Node*, OpKernelContext*);
+
+  XlaContext* xla_context_;
+  XlaCompilationDevice* device_;
+  Graph* graph_;
+  FunctionLibraryRuntime* flib_;
+  ScopedStepContainer* step_container_;
+  FunctionCompiler compiler_;
+  // A buffer to hold tensor inputs to a node, this is reused across the graph
+  // traversal.
+  gtl::InlinedVector<TensorValue, 4> tensor_inputs_;
+};
+
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_COMPILER_TF2XLA_GRAPH_COMPILER_H_
diff --git a/tensorflow/compiler/tf2xla/xla_compiler.cc b/tensorflow/compiler/tf2xla/xla_compiler.cc
index 8521d4167a..9e405578aa 100644
--- a/tensorflow/compiler/tf2xla/xla_compiler.cc
+++ b/tensorflow/compiler/tf2xla/xla_compiler.cc
@@ -20,10 +20,12 @@ limitations under the License.
 
 #include "tensorflow/compiler/tf2xla/dump_graph.h"
 #include "tensorflow/compiler/tf2xla/functionalize_control_flow.h"
+#include "tensorflow/compiler/tf2xla/graph_compiler.h"
 #include "tensorflow/compiler/tf2xla/shape_util.h"
 #include "tensorflow/compiler/tf2xla/type_util.h"
 #include "tensorflow/compiler/tf2xla/xla_compilation_device.h"
 #include "tensorflow/compiler/tf2xla/xla_context.h"
+#include "tensorflow/compiler/tf2xla/xla_op_kernel.h"
 #include "tensorflow/compiler/xla/client/client_library.h"
 #include "tensorflow/core/common_runtime/device.h"
 #include "tensorflow/core/common_runtime/executor.h"
@@ -178,9 +180,34 @@ Status XlaCompiler::CompileFunction(
 
 namespace {
 
-Status ExecuteGraph(XlaContext* xla_context, std::unique_ptr<Graph> graph,
-                    XlaCompilationDevice* device, FunctionLibraryRuntime* flib,
-                    int64 step_id) {
+// Builds XlaCompiler argument descriptions `args` from `ctx`.
+Status MakeXlaCompilerArgumentsFromInputs(
+    XlaOpKernelContext* ctx, std::vector<XlaCompiler::Argument>* args) {
+  VLOG(2) << "Num inputs " << ctx->num_inputs();
+  args->resize(ctx->num_inputs());
+  for (int i = 0; i < ctx->num_inputs(); ++i) {
+    VLOG(2) << "  Input " << i
+            << " type: " << DataTypeString(ctx->input_type(i))
+            << " shape: " << ctx->InputShape(i).DebugString();
+    XlaCompiler::Argument& arg = (*args)[i];
+    DataType type = ctx->input_type(i);
+
+    if (type == DT_RESOURCE) {
+      return errors::InvalidArgument(
+          "Resource as function argument is not yet implemented.");
+    } else {
+      arg.kind = XlaCompiler::Argument::kParameter;
+      arg.type = ctx->input_type(i);
+      TF_RETURN_IF_ERROR(
+          TensorShapeToXLAShape(arg.type, ctx->InputShape(i), &arg.shape));
+    }
+  }
+  return Status::OK();
+}
+
+Status ExecuteGraph(XlaCompiler* compiler, XlaContext* xla_context,
+                    std::unique_ptr<Graph> graph, XlaCompilationDevice* device,
+                    FunctionLibraryRuntime* flib, int64 step_id) {
   // Resource cleanup is a bit messy. XlaContext is a ref-counted resource; the
   // resource manager takes ownership via Create, and unrefs via Cleanup.  We
   // explicitly add a reference to ensure the refcount at entry is maintained at
@@ -197,56 +224,27 @@ Status ExecuteGraph(XlaContext* xla_context, std::unique_ptr<Graph> graph,
   TF_RETURN_IF_ERROR(device->resource_manager()->Create(
       step_container->name(), XlaContext::kXlaContextResourceName,
       xla_context));
-
-  // Create a LocalExecutor that will own and run the graph.
-  // TODO(b/66947550): migrate away from using an Executor in order to guarantee
-  // determinism and thread-safety.
-  LocalExecutorParams exec_params;
-  exec_params.device = device;
-  exec_params.function_library = flib;
-  exec_params.create_kernel = [flib](const NodeDef& ndef, OpKernel** kernel) {
-    return flib->CreateKernel(ndef, kernel);
-  };
-  exec_params.delete_kernel = [](OpKernel* kernel) { delete kernel; };
-  Executor* exec_ptr = nullptr;
-  TF_RETURN_IF_ERROR(NewLocalExecutor(exec_params, graph.release(), &exec_ptr));
-  std::unique_ptr<Executor> exec(exec_ptr);
-  // At this point ownership of the graph has been transferred to exec.
-
-  // Run the graph symbolically, turning the graph into an XLA computation.
-  Executor::Args exec_args;
-  exec_args.step_id = step_id;
-  exec_args.step_container = step_container.get();
-
-  // Pushes closures to run onto `worklist`. We don't run the closures directly
-  // from 'runner' since that might lead to a stack overflow for large graphs.
-  std::deque<Executor::Args::Closure> worklist;
-  exec_args.runner = [&](Executor::Args::Closure c) {
-    worklist.push_back(std::move(c));
+  // Compile_func is used to tell the serial executor how to compile a function.
+  auto compile_func = [&](const NameAttrList& function,
+                          XlaOpKernelContext* xla_op_context)
+      -> xla::StatusOr<std::shared_ptr<xla::Computation>> {
+    std::vector<XlaCompiler::Argument> arguments;
+
+    TF_RETURN_IF_ERROR(
+        MakeXlaCompilerArgumentsFromInputs(xla_op_context, &arguments));
+
+    XlaCompiler::CompilationResult result;
+    TF_RETURN_IF_ERROR(compiler->CompileFunction(XlaCompiler::CompileOptions(),
+                                                 function, arguments, &result));
+    return result.computation;
   };
 
-  // The following code assumes there is only one thread involved and no
-  // concurrency, because we did not provide Executor a threaded runner. Async
-  // ops on the XlaCompilation device must not use threads or concurrency
-  // internally.
-  bool done = false;
-  exec->RunAsync(exec_args, [&](const Status& s) {
-    status = s;
-    done = true;
-  });
-  // Repeatedly run closures from the worklist until `done` is signalled.
-  while (!done) {
-    TF_RET_CHECK(!worklist.empty());
-    Executor::Args::Closure& c = worklist.front();
-    c();
-    worklist.pop_front();
-  }
-  TF_RETURN_WITH_CONTEXT_IF_ERROR(
-      status, "Conversion from TensorFlow graph to XLA computation failed.");
-
+  GraphCompiler graph_compiler(xla_context, device, graph.get(), flib,
+                               step_container.get(), compile_func);
+  TF_RETURN_IF_ERROR(graph_compiler.Compile());
   // Explicitly clean up the step container, to capture the cleanup status.
   step_container.reset();
-  return status;
+  return Status::OK();
 }
 
 // Builds XLA computations for each of the arguments to the computation.
@@ -494,7 +492,7 @@ Status XlaCompiler::CompileGraph(const XlaCompiler::CompileOptions& options,
       &result->input_mapping, &result->xla_input_shapes));
   context->set_args(std::move(arg_expressions));
 
-  TF_RETURN_IF_ERROR(ExecuteGraph(context, std::move(graph), device_,
+  TF_RETURN_IF_ERROR(ExecuteGraph(this, context, std::move(graph), device_,
                                   flib_runtime_, NextStepId()));
 
   int num_nonconst_outputs;
diff --git a/tensorflow/compiler/tf2xla/xla_compiler.h b/tensorflow/compiler/tf2xla/xla_compiler.h
index 35159dbad4..0435c619f8 100644
--- a/tensorflow/compiler/tf2xla/xla_compiler.h
+++ b/tensorflow/compiler/tf2xla/xla_compiler.h
@@ -240,7 +240,7 @@ class XlaCompiler {
     bool use_tuple_arg = false;
 
     // If 'return_updated_values_for_all_resources' is true, then updated
-    // values of all resource resources arguments will be included in the
+    // values of all resource arguments will be included in the
     // 'resource_updates' of the computation, even if the resource was not
     // modified by the computation. Used when compiling loop bodies to ensure
     // the input and output signatures match.
diff --git a/tensorflow/compiler/tf2xla/xla_compiler_test.cc b/tensorflow/compiler/tf2xla/xla_compiler_test.cc
index 531725a623..88ed3b89a6 100644
--- a/tensorflow/compiler/tf2xla/xla_compiler_test.cc
+++ b/tensorflow/compiler/tf2xla/xla_compiler_test.cc
@@ -63,6 +63,7 @@ class DummyReadResourceOp : public XlaOpKernel {
     dummy->Unref();
 
     ctx->SetOutput(0, ctx->Input(0));
+    ctx->SetOutput(1, ctx->Input(0));
   }
 };
 
@@ -80,22 +81,25 @@ class DummyReadResourceCC {
     if (!scope.ok()) return;
     scope.UpdateStatus(scope.DoShapeInference(ret));
     if (!scope.ok()) return;
-    this->output_ = Output(ret, 0);
+    this->output1_ = Output(ret, 0);
+    this->output2_ = Output(ret, 1);
   }
-  Node* node() const { return output_.node(); }
 
-  Output output_;
+  Output output1_;
+  Output output2_;
 };
 
 REGISTER_OP("DummyReadResource")
     .Input("input: int32")
-    .Output("output: int32")
+    .Output("output1: int32")
+    .Output("output2: int32")
     .SetShapeFn(shape_inference::UnknownShape)
     .Doc(R"doc(
 A dummy Op.
 
 input: dummy input.
-output: dummy output.
+output1: dummy output.
+output2: dummy output.
 )doc");
 
 REGISTER_XLA_OP(Name("DummyReadResource"), DummyReadResourceOp);
@@ -316,7 +320,8 @@ TEST_F(XlaCompilerTest, ResourceManager) {
   Scope scope = Scope::NewRootScope().ExitOnError();
   auto a = ops::_Arg(scope.WithOpName("A"), DT_INT32, 0);
   auto b = DummyReadResourceCC(scope.WithOpName("B"), a);
-  auto c = ops::_Retval(scope.WithOpName("C"), b.output_, 0);
+  auto c = ops::Add(scope.WithOpName("C"), b.output2_, b.output1_);
+  auto d = ops::_Retval(scope.WithOpName("D"), c, 0);
   std::unique_ptr<Graph> graph(new Graph(OpRegistry::Global()));
   TF_ASSERT_OK(scope.ToGraph(graph.get()));
 
@@ -349,6 +354,58 @@ TEST_F(XlaCompilerTest, ResourceManager) {
   resource->Unref();
 }
 
+// Tests compilation and execution of a graph that adds two tensors.
+TEST_F(XlaCompilerTest, DeterministicCompilation) {
+  // Builds a graph that contains a node with two output edges. The compiler
+  // should always traverse them in the same order.
+  const int64 test_count = 2;
+
+  std::vector<XlaCompiler::CompilationResult> results(test_count);
+
+  for (int64 i = 0; i < test_count; ++i) {
+    Scope scope = Scope::NewRootScope().ExitOnError();
+    auto a = ops::_Arg(scope.WithOpName("A"), DT_INT32, 0);
+    auto b = ops::Neg(scope.WithOpName("B"), a);
+    auto c = ops::Neg(scope.WithOpName("C"), a);
+    auto d = ops::Add(scope.WithOpName("D"), b, c);
+    auto e = ops::_Retval(scope.WithOpName("E"), d, 0);
+    std::unique_ptr<Graph> graph(new Graph(OpRegistry::Global()));
+    TF_ASSERT_OK(scope.ToGraph(graph.get()));
+
+    // Builds a description of the argument.
+    std::vector<XlaCompiler::Argument> args(1);
+    args[0].kind = XlaCompiler::Argument::kParameter;
+    args[0].type = DT_INT32;
+    args[0].shape = xla::ShapeUtil::MakeShape(xla::S32, {2});
+
+    // Compiles the graph.
+    auto options = DefaultOptions();
+    XlaCompiler compiler(options);
+
+    TF_ASSERT_OK(compiler.CompileGraph(XlaCompiler::CompileOptions(), "dummy",
+                                       std::move(graph), args, &results[i]));
+  }
+
+  for (int64 i = 1; i < test_count; ++i) {
+    auto m1 =
+        results[i - 1].computation->Snapshot().ValueOrDie()->entry().requests();
+    auto m2 =
+        results[i].computation->Snapshot().ValueOrDie()->entry().requests();
+    // Check if every entry is the same.
+    for (auto& entry1 : m1) {
+      int64 key = entry1.first;
+      auto value1 = entry1.second;
+      auto entry2 = m2.find(key);
+      auto value2 = entry2->second;
+      EXPECT_TRUE(entry2 != m2.end());
+      string str1, str2;
+      value1.AppendToString(&str1);
+      value2.AppendToString(&str2);
+      EXPECT_EQ(str1, str2);
+    }
+  }
+}
+
 // Tests a computation that receives a TensorArray resource as input and
 // updates it.
 TEST_F(XlaCompilerTest, CanPassTensorArraysToAndFromComputation) {
diff --git a/tensorflow/compiler/xla/service/service.cc b/tensorflow/compiler/xla/service/service.cc
index bd7898a41f..d279e1f50f 100644
--- a/tensorflow/compiler/xla/service/service.cc
+++ b/tensorflow/compiler/xla/service/service.cc
@@ -187,8 +187,9 @@ tensorflow::Status Service::Computation(const ComputationRequest* arg,
 
   *result->mutable_computation() =
       computation_tracker_.NewComputation(arg->name());
-  VLOG(1) << Printf("Created new computation %s on service %p",
-                    result->computation().ShortDebugString().c_str(), this);
+  VLOG(1) << Printf("Created new computation %s on service %p, name %s",
+                    result->computation().ShortDebugString().c_str(), this,
+                    arg->name().c_str());
   return tensorflow::Status::OK();
 }
 
diff --git a/tensorflow/core/graph/algorithm.cc b/tensorflow/core/graph/algorithm.cc
index 3bfba3fc4e..6ef51aa7df 100644
--- a/tensorflow/core/graph/algorithm.cc
+++ b/tensorflow/core/graph/algorithm.cc
@@ -24,7 +24,8 @@ limitations under the License.
 namespace tensorflow {
 
 void DFS(const Graph& g, const std::function<void(Node*)>& enter,
-         const std::function<void(Node*)>& leave) {
+         const std::function<void(Node*)>& leave,
+         const NodeComparator& stable_comparator) {
   // Stack of work to do.
   struct Work {
     Node* node;
@@ -51,24 +52,41 @@ void DFS(const Graph& g, const std::function<void(Node*)>& enter,
     // Arrange to call leave(n) when all done with descendants.
     if (leave) stack.push_back(Work{n, true});
 
-    // Arrange to work on descendants.
-    for (Node* out : n->out_nodes()) {
+    gtl::iterator_range<NeighborIter> nodes = n->out_nodes();
+    auto add_work = [&visited, &stack](Node* out) {
       if (!visited[out->id()]) {
         // Note; we must not mark as visited until we actually process it.
         stack.push_back(Work{out, false});
       }
+    };
+
+    if (stable_comparator) {
+      std::vector<Node*> nodes_sorted;
+      for (Node* out : nodes) {
+        nodes_sorted.emplace_back(out);
+      }
+      std::sort(nodes_sorted.begin(), nodes_sorted.end(), stable_comparator);
+      for (Node* out : nodes_sorted) {
+        add_work(out);
+      }
+    } else {
+      for (Node* out : nodes) {
+        add_work(out);
+      }
     }
   }
 }
 
 void ReverseDFS(const Graph& g, const std::function<void(Node*)>& enter,
-                const std::function<void(Node*)>& leave) {
-  ReverseDFSFrom(g, {g.sink_node()}, enter, leave);
+                const std::function<void(Node*)>& leave,
+                const NodeComparator& stable_comparator) {
+  ReverseDFSFrom(g, {g.sink_node()}, enter, leave, stable_comparator);
 }
 
 void ReverseDFSFrom(const Graph& g, gtl::ArraySlice<Node*> start,
                     const std::function<void(Node*)>& enter,
-                    const std::function<void(Node*)>& leave) {
+                    const std::function<void(Node*)>& leave,
+                    const NodeComparator& stable_comparator) {
   // Stack of work to do.
   struct Work {
     Node* node;
@@ -97,23 +115,41 @@ void ReverseDFSFrom(const Graph& g, gtl::ArraySlice<Node*> start,
     // Arrange to call leave(n) when all done with descendants.
     if (leave) stack.push_back(Work{n, true});
 
-    // Arrange to work on parents.
-    for (Node* in : n->in_nodes()) {
-      if (!visited[in->id()]) {
+    gtl::iterator_range<NeighborIter> nodes = n->in_nodes();
+
+    auto add_work = [&visited, &stack](Node* out) {
+      if (!visited[out->id()]) {
         // Note; we must not mark as visited until we actually process it.
-        stack.push_back(Work{in, false});
+        stack.push_back(Work{out, false});
+      }
+    };
+
+    if (stable_comparator) {
+      std::vector<Node*> nodes_sorted;
+      for (Node* in : nodes) {
+        nodes_sorted.emplace_back(in);
+      }
+      std::sort(nodes_sorted.begin(), nodes_sorted.end(), stable_comparator);
+      for (Node* in : nodes_sorted) {
+        add_work(in);
+      }
+    } else {
+      for (Node* in : nodes) {
+        add_work(in);
       }
     }
   }
 }
 
-void GetPostOrder(const Graph& g, std::vector<Node*>* order) {
+void GetPostOrder(const Graph& g, std::vector<Node*>* order,
+                  const NodeComparator& stable_comparator) {
   order->clear();
-  DFS(g, nullptr, [order](Node* n) { order->push_back(n); });
+  DFS(g, nullptr, [order](Node* n) { order->push_back(n); }, stable_comparator);
 }
 
-void GetReversePostOrder(const Graph& g, std::vector<Node*>* order) {
-  GetPostOrder(g, order);
+void GetReversePostOrder(const Graph& g, std::vector<Node*>* order,
+                         const NodeComparator& stable_comparator) {
+  GetPostOrder(g, order, stable_comparator);
   std::reverse(order->begin(), order->end());
 }
 
diff --git a/tensorflow/core/graph/algorithm.h b/tensorflow/core/graph/algorithm.h
index 01d36e0a12..5bb6041d98 100644
--- a/tensorflow/core/graph/algorithm.h
+++ b/tensorflow/core/graph/algorithm.h
@@ -25,24 +25,50 @@ limitations under the License.
 
 namespace tensorflow {
 
+// Comparator for two nodes. This is used in order to get a stable ording.
+using NodeComparator = std::function<bool(const Node*, const Node*)>;
+
+// Compares two node based on their ids.
+struct NodeComparatorID {
+  bool operator()(const Node* n1, const Node* n2) const {
+    return n1->id() < n2->id();
+  }
+};
+
+// Compare two nodes based on their names.
+struct NodeComparatorName {
+  bool operator()(const Node* n1, const Node* n2) const {
+    return n1->name() < n2->name();
+  }
+};
+
 // Perform a depth-first-search on g starting at the source node.
 // If enter is not empty, calls enter(n) before visiting any children of n.
 // If leave is not empty, calls leave(n) after visiting all children of n.
+// If stable_comparator is set, a stable ordering of visit is achieved by
+// sorting a node's neighbors first before visiting them.
 extern void DFS(const Graph& g, const std::function<void(Node*)>& enter,
-                const std::function<void(Node*)>& leave);
+                const std::function<void(Node*)>& leave,
+                const NodeComparator& stable_comparator = {});
 
 // Perform a reverse depth-first-search on g starting at the sink node.
 // If enter is not empty, calls enter(n) before visiting any parents of n.
 // If leave is not empty, calls leave(n) after visiting all parents of n.
+// If stable_comparator is set, a stable ordering of visit is achieved by
+// sorting a node's neighbors first before visiting them.
 extern void ReverseDFS(const Graph& g, const std::function<void(Node*)>& enter,
-                       const std::function<void(Node*)>& leave);
+                       const std::function<void(Node*)>& leave,
+                       const NodeComparator& stable_comparator = {});
 
 // Perform a reverse depth-first-search on g starting at the 'start' nodes.
 // If enter is not empty, calls enter(n) before visiting any parents of n.
 // If leave is not empty, calls leave(n) after visiting all parents of n.
+// If stable_comparator is set, a stable ordering of visit is achieved by
+// sorting a node's neighbors first before visiting them.
 extern void ReverseDFSFrom(const Graph& g, gtl::ArraySlice<Node*> start,
                            const std::function<void(Node*)>& enter,
-                           const std::function<void(Node*)>& leave);
+                           const std::function<void(Node*)>& leave,
+                           const NodeComparator& stable_comparator = {});
 
 // Stores in *order the post-order numbering of all nodes
 // in graph found via a depth first search starting at the source node.
@@ -50,11 +76,18 @@ extern void ReverseDFSFrom(const Graph& g, gtl::ArraySlice<Node*> start,
 // Note that this is equivalent to reverse topological sorting when the
 // graph does not have cycles.
 //
+// If stable_comparator is set, a stable ordering of visit is achieved by
+// sorting a node's neighbors first before visiting them.
+//
 // REQUIRES: order is not NULL.
-void GetPostOrder(const Graph& g, std::vector<Node*>* order);
+void GetPostOrder(const Graph& g, std::vector<Node*>* order,
+                  const NodeComparator& stable_comparator = {});
 
 // Stores in *order the reverse post-order numbering of all nodes
-void GetReversePostOrder(const Graph& g, std::vector<Node*>* order);
+// If stable_comparator is set, a stable ordering of visit is achieved by
+// sorting a node's neighbors first before visiting them.
+void GetReversePostOrder(const Graph& g, std::vector<Node*>* order,
+                         const NodeComparator& stable_comparator = {});
 
 // Prune nodes in "g" that are not in some path from the source node
 // to any node in 'nodes'. Returns true if changes were made to the graph.
diff --git a/tensorflow/core/graph/algorithm_test.cc b/tensorflow/core/graph/algorithm_test.cc
index a529760426..0cdcdb6685 100644
--- a/tensorflow/core/graph/algorithm_test.cc
+++ b/tensorflow/core/graph/algorithm_test.cc
@@ -112,5 +112,40 @@ TEST(AlgorithmTest, ReversePostOrder) {
   EXPECT_FALSE(ExpectBefore(orders, order, &error));
 }
 
+TEST(AlgorithmTest, ReversePostOrderStable) {
+  int64 run_count = 100;
+  using namespace ::tensorflow::ops;  // NOLINT(build/namespaces)
+
+  for (int64 i = 0; i < run_count; ++i) {
+    // One source of nondeterminism comes from unordered set with key of a
+    // pointer type, for example the order of FlatSet<Node*> depends on the
+    // raw pointer value of Node. Stable post order suppose to remove this
+    // nondeterminism by enforcing an ordering based on node ids.
+    GraphDefBuilder b(GraphDefBuilder::kFailImmediately);
+    string error;
+    Node* w1 = SourceOp("TestParams", b.opts().WithName("W1"));
+    Node* input =
+        SourceOp("TestInput", b.opts().WithName("input").WithControlInput(w1));
+    BinaryOp("TestMul", w1, {input, 1}, b.opts().WithName("t2"));
+    // Insert different number of nodes between the allocation of t2 and t3,
+    // this creates enough entropy in the memory distance between t2 and t3 thus
+    // forces them to have randomized ordering had stable DFS was not
+    // implemented correctly.
+    for (int64 j = 0; j < i; ++j) {
+      BinaryOp("TestMul", w1, {input, 1},
+               b.opts().WithName(strings::StrCat("internal", j)));
+    }
+
+    BinaryOp("TestMul", w1, {input, 1}, b.opts().WithName("t3"));
+
+    Graph g(OpRegistry::Global());
+    TF_ASSERT_OK(b.ToGraph(&g));
+    std::vector<Node*> order;
+
+    // Test reverse post order generates expected ordering.
+    GetReversePostOrder(g, &order, /*stable_comparator=*/NodeComparatorID());
+    EXPECT_TRUE(ExpectBefore({{"t3", "t2"}}, order, &error));
+  }
+}
 }  // namespace
 }  // namespace tensorflow
diff --git a/tensorflow/core/graph/graph.h b/tensorflow/core/graph/graph.h
index 5a31a6216b..54076ed1ab 100644
--- a/tensorflow/core/graph/graph.h
+++ b/tensorflow/core/graph/graph.h
@@ -298,12 +298,12 @@ class Edge {
   Node* dst() const { return dst_; }
   int id() const { return id_; }
 
-  // Return the number of the source output that produces the data
+  // Return the index of the source output that produces the data
   // carried by this edge.  The special value kControlSlot is used
   // for control dependencies.
   int src_output() const { return src_output_; }
 
-  // Return the number of the destination input that consumes the data
+  // Return the index of the destination input that consumes the data
   // carried by this edge.  The special value kControlSlot is used
   // for control dependencies.
   int dst_input() const { return dst_input_; }
-- 
GitLab


From bb6c863c10f0e9702fc29380f2ed598624897b18 Mon Sep 17 00:00:00 2001
From: Skye Wanderman-Milne <skyewm@google.com>
Date: Fri, 6 Oct 2017 10:01:43 -0700
Subject: [PATCH 111/909] Deprecate op_dict argument to import_graph_def

This semantics of this argument are unclear and don't seem usable (it
can effectively only be used to limit the available ops to be
imported).

PiperOrigin-RevId: 171305211
---
 tensorflow/python/framework/importer.py | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/tensorflow/python/framework/importer.py b/tensorflow/python/framework/importer.py
index eec7c4a463..c0d221ddfe 100644
--- a/tensorflow/python/framework/importer.py
+++ b/tensorflow/python/framework/importer.py
@@ -32,6 +32,7 @@ from tensorflow.python.framework import op_def_registry
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
 from tensorflow.python.util import compat
+from tensorflow.python.util.deprecation import deprecated_args
 
 
 # TODO(josh11b): SWIG the code from node_def_util instead of duplicating
@@ -153,6 +154,10 @@ def _FindAttrInOpDef(attr_name, op_def):
   return None
 
 
+@deprecated_args(None, 'Please file an issue at '
+                 'https://github.com/tensorflow/tensorflow/issues if you depend'
+                 ' on this feature.',
+                 'op_dict')
 def import_graph_def(graph_def, input_map=None, return_elements=None,
                      name=None, op_dict=None, producer_op_list=None):
   """Imports the graph from `graph_def` into the current default `Graph`.
@@ -177,15 +182,12 @@ def import_graph_def(graph_def, input_map=None, return_elements=None,
     name: (Optional.) A prefix that will be prepended to the names in
       `graph_def`. Note that this does not apply to imported function names.
       Defaults to `"import"`.
-    op_dict: (Optional.) A dictionary mapping op type names to `OpDef` protos.
-      Must contain an `OpDef` proto for each op type named in `graph_def`.
-      If omitted, uses the `OpDef` protos registered in the global registry.
+    op_dict: (Optional.) Deprecated, do not use.
     producer_op_list: (Optional.) An `OpList` proto with the (possibly stripped)
-      list of `OpDef`s used by the producer of the graph. If provided, attrs
-      for ops in `graph_def` that are not in `op_dict` that have their default
-      value according to `producer_op_list` will be removed. This will allow
-      some more `GraphDef`s produced by later binaries to be accepted by
-      earlier binaries.
+      list of `OpDef`s used by the producer of the graph. If provided,
+      unrecognized attrs for ops in `graph_def` that have their default value
+      according to `producer_op_list` will be removed. This will allow some more
+      `GraphDef`s produced by later binaries to be accepted by earlier binaries.
 
   Returns:
     A list of `Operation` and/or `Tensor` objects from the imported graph,
@@ -229,8 +231,7 @@ def import_graph_def(graph_def, input_map=None, return_elements=None,
 
   name_to_op = {}
 
-  if op_dict is None:
-    op_dict = op_def_registry.get_registered_ops()
+  op_dict = op_def_registry.get_registered_ops()
 
   if producer_op_list is None:
     producer_op_dict = None
-- 
GitLab


From 251a1e70dc04b10fb25e8013d1ad1f27d5eda30b Mon Sep 17 00:00:00 2001
From: Derek Murray <derek.murray@gmail.com>
Date: Fri, 6 Oct 2017 10:27:49 -0700
Subject: [PATCH 112/909] Add an actionable error message for build_info
 ImportError (#13528)

This `import` statement is now the first point where we attempt to import a generated file, and hence could see a failure if the user tries to `import tensorflow` from the root of the git repository source tree. When this `import` fails, raise a more actionable error message.

Fixes #13526.
---
 tensorflow/python/platform/self_check.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/platform/self_check.py b/tensorflow/python/platform/self_check.py
index 39d38d7bbc..966a094e55 100644
--- a/tensorflow/python/platform/self_check.py
+++ b/tensorflow/python/platform/self_check.py
@@ -21,7 +21,13 @@ from __future__ import print_function
 import os
 
 
-from tensorflow.python.platform import build_info
+try:
+  from tensorflow.python.platform import build_info
+except ImportError:
+  raise ImportError("Could not import tensorflow. Do not import tensorflow "
+                    "from its source directory; change directory to outside "
+                    "the TensorFlow source tree, and relaunch your Python "
+                    "interpreter from there.")
 
 
 def preload_check():
-- 
GitLab


From 08ea64c5a6748b66b310e73bb4591d091c227a33 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Fri, 6 Oct 2017 10:40:52 -0700
Subject: [PATCH 113/909] [XLA:CPU] Give parameter loads a meaningful LLVM
 name.

The typed parameter loads often get lost after optimization, but the
untyped loads tend to stick around.  Giving them a name helps with
readability of the IR.

PiperOrigin-RevId: 171310991
---
 tensorflow/compiler/xla/service/cpu/ir_emitter.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
index 8b777bcf84..4375f13a0e 100644
--- a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
@@ -1457,6 +1457,7 @@ Status IrEmitter::HandleParameter(HloInstruction* parameter) {
       llvm_ir::EmitBufferIndexingGEP(params, param_number, &ir_builder_);
   llvm::LoadInst* param_address_untyped =
       ir_builder_.CreateLoad(param_address_offset);
+  param_address_untyped->setName(AsStringRef(IrName(parameter, "untyped")));
   if (hlo_module_config_.debug_options()
           .xla_llvm_enable_invariant_load_metadata()) {
     // We never reassign parameters, so this load is invariant.
-- 
GitLab


From 368754d8a6f4be1772b4bec9dbef686570637c5d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 6 Oct 2017 10:53:51 -0700
Subject: [PATCH 114/909] Automated g4 rollback of changelist 171303938

PiperOrigin-RevId: 171313020
---
 tensorflow/compiler/tf2xla/BUILD              |   2 -
 tensorflow/compiler/tf2xla/graph_compiler.cc  | 185 ------------------
 tensorflow/compiler/tf2xla/graph_compiler.h   | 103 ----------
 tensorflow/compiler/tf2xla/xla_compiler.cc    |  98 +++++-----
 tensorflow/compiler/tf2xla/xla_compiler.h     |   2 +-
 .../compiler/tf2xla/xla_compiler_test.cc      |  69 +------
 tensorflow/compiler/xla/service/service.cc    |   5 +-
 tensorflow/core/graph/algorithm.cc            |  64 ++----
 tensorflow/core/graph/algorithm.h             |  43 +---
 tensorflow/core/graph/algorithm_test.cc       |  35 ----
 tensorflow/core/graph/graph.h                 |   4 +-
 11 files changed, 80 insertions(+), 530 deletions(-)
 delete mode 100644 tensorflow/compiler/tf2xla/graph_compiler.cc
 delete mode 100644 tensorflow/compiler/tf2xla/graph_compiler.h

diff --git a/tensorflow/compiler/tf2xla/BUILD b/tensorflow/compiler/tf2xla/BUILD
index 647bfd1849..4da2ed722e 100644
--- a/tensorflow/compiler/tf2xla/BUILD
+++ b/tensorflow/compiler/tf2xla/BUILD
@@ -102,13 +102,11 @@ cc_library(
         "xla_helpers.cc",
         "xla_op_kernel.cc",
         "xla_op_registry.cc",
-        "graph_compiler.cc",
         "xla_cpu_backend.cc",
     ] + if_cuda_is_configured([
         "xla_gpu_backend.cc",
     ]),
     hdrs = [
-        "graph_compiler.h",
         "xla_compilation_device.h",
         "xla_compiler.h",
         "xla_context.h",
diff --git a/tensorflow/compiler/tf2xla/graph_compiler.cc b/tensorflow/compiler/tf2xla/graph_compiler.cc
deleted file mode 100644
index c168266b16..0000000000
--- a/tensorflow/compiler/tf2xla/graph_compiler.cc
+++ /dev/null
@@ -1,185 +0,0 @@
-/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#include "tensorflow/compiler/tf2xla/graph_compiler.h"
-
-#include <deque>
-#include <numeric>
-#include <vector>
-
-#include "tensorflow/compiler/tf2xla/dump_graph.h"
-#include "tensorflow/compiler/tf2xla/functionalize_control_flow.h"
-#include "tensorflow/compiler/tf2xla/shape_util.h"
-#include "tensorflow/compiler/tf2xla/type_util.h"
-#include "tensorflow/compiler/tf2xla/xla_compilation_device.h"
-#include "tensorflow/compiler/tf2xla/xla_context.h"
-#include "tensorflow/compiler/tf2xla/xla_op_kernel.h"
-#include "tensorflow/compiler/xla/client/client_library.h"
-#include "tensorflow/core/common_runtime/device.h"
-#include "tensorflow/core/common_runtime/executor.h"
-#include "tensorflow/core/common_runtime/function.h"
-#include "tensorflow/core/common_runtime/graph_optimizer.h"
-#include "tensorflow/core/framework/attr_value_util.h"
-#include "tensorflow/core/framework/op_kernel.h"
-#include "tensorflow/core/graph/algorithm.h"
-#include "tensorflow/core/graph/graph_constructor.h"
-#include "tensorflow/core/graph/node_builder.h"
-#include "tensorflow/core/lib/hash/hash.h"
-#include "tensorflow/core/platform/logging.h"
-#include "tensorflow/core/public/version.h"
-
-namespace tensorflow {
-
-Status GraphCompiler::Compile() {
-  std::vector<NodeBinding> bindings(graph_->num_node_ids());
-  std::vector<Node*> topo_sorted_nodes;
-  // XLA requires determinism, generate a stable ordering from DFS.
-  GetReversePostOrder(*graph_, &topo_sorted_nodes,
-                      /*stable_comparator=*/NodeComparatorID());
-
-  OpKernelContext::Params params;
-  PartiallySetupParams(&params);
-
-  for (Node* n : topo_sorted_nodes) {
-    // Set up bindings.
-    NodeBinding& binding = bindings[n->id()];
-    binding.node = n;
-    Status s = flib_->CreateKernel(n->def(), &binding.op_kernel);
-    binding.output_attrs.resize(n->num_outputs());
-    if (!s.ok()) {
-      binding.op_kernel = nullptr;
-      s = AttachDef(s, *n);
-      LOG(ERROR) << "Executor failed to create kernel. " << s;
-      return s;
-    }
-  }
-
-  // Bindings are initialized by the size of graph_->num_node_ids. However, the
-  // graph may contain dead nodes that still hold a valid node id. Thus
-  // graph_->num_node_ids could be larger than number of topo sorted nodes.
-  TF_RET_CHECK(bindings.size() >= topo_sorted_nodes.size());
-
-  for (Node* n : topo_sorted_nodes) {
-    TF_RET_CHECK(!n->IsRecv() && !n->IsSend() && !n->IsSwitch())
-        << "Not supported node: " << n->DebugString();
-    NodeBinding& binding = bindings[n->id()];
-    params.op_kernel = binding.op_kernel;
-    params.output_attr_array = binding.output_attrs.data();
-
-    // tensor_inputs_ is a buffer reused across graph traversal. We clean up and
-    // reinitialize the buffer before we visit a new node.
-    tensor_inputs_.clear();
-    tensor_inputs_.resize(n->num_inputs());
-
-    // Set up inputs from outputs of previous nodes.
-    for (auto* e : n->in_edges()) {
-      if (e->IsControlEdge()) continue;
-      Node* src = e->src();
-      tensor_inputs_[e->dst_input()] =
-          bindings[src->id()].tensor_values[e->src_output()];
-    }
-
-    OpKernelContext op_context(&params, n->num_outputs());
-    if (IsFunctional(n)) {
-      TF_RETURN_IF_ERROR(CompileFunctionalNode(n, &op_context));
-    } else {
-      device_->Compute(CHECK_NOTNULL(params.op_kernel), &op_context);
-      Status s = op_context.status();
-      TF_RETURN_IF_ERROR(s);
-    }
-
-    // Set up outputs. Also check if outputs from the previous computation is
-    // valid.
-    for (int o = 0; o < n->num_outputs(); ++o) {
-      const auto tensor_val = op_context.release_output(o);
-      if (*op_context.is_output_dead() || tensor_val.tensor == nullptr) {
-        return errors::Internal("Missing xla_context ", o, "-th output from ",
-                                (*op_context.is_output_dead() ? "(dead)" : ""),
-                                SummarizeNode(*n));
-      }
-      binding.tensor_values.push_back(tensor_val);
-    }
-  }
-
-  // Clean up tensor data and op kernels.
-  for (NodeBinding& binding : bindings) {
-    delete binding.op_kernel;
-    for (auto& t : binding.tensor_values) {
-      if (!t.is_ref()) {
-        delete t.tensor;
-      }
-    }
-  }
-  return Status::OK();
-}
-
-bool GraphCompiler::IsFunctional(Node* n) {
-  return n->type_string() == FunctionLibraryDefinition::kGradientOp ||
-         (flib_->GetFunctionLibraryDefinition()->Find(n->def().op()) !=
-          nullptr);
-}
-
-Status GraphCompiler::CompileFunctionalNode(Node* n,
-                                            OpKernelContext* op_context) {
-  TF_RET_CHECK(IsFunctional(n));
-  // For functional nodes, compile them using compiler_ and call into the
-  // functions.
-  XlaOpKernelContext xla_op_context(op_context);
-
-  std::vector<XlaCompiler::Argument> arguments;
-  XlaCompiler::CompilationResult result;
-  NameAttrList func;
-  if (flib_->GetFunctionLibraryDefinition()->Find(n->def().op())) {
-    func.set_name(n->def().op());
-  } else {
-    func.set_name(FunctionLibraryDefinition::kGradientOp);
-  }
-  *func.mutable_attr() = n->def().attr();
-
-  // Compile the graph using the function compiler.
-  TF_ASSIGN_OR_RETURN(auto computation, compiler_(func, &xla_op_context));
-  XlaContext& context = XlaContext::Get(op_context);
-  auto* b = context.builder();
-
-  // Graph data handles from the inputs.
-  std::vector<xla::ComputationDataHandle> handles;
-  for (auto tensor : tensor_inputs_) {
-    auto expression =
-        reinterpret_cast<const XlaExpression*>(tensor->tensor_data().data());
-    // TODO(yunxing): Support two rare cases below where input is a resource or
-    // contains a null handle.
-    TF_RET_CHECK(expression->resource() == nullptr)
-        << "Input with resource is not supported.";
-    TF_RET_CHECK(expression->handle().handle() != 0)
-        << "Invalid computation handle.";
-    handles.push_back(expression->handle());
-  }
-  auto output_handle = b->Call(*computation, handles);
-  // The output handle of `Call` computation is a tuple type. Unzip it so
-  // that it can into fit future computations.
-  for (int64 idx = 0; idx < n->num_outputs(); ++idx) {
-    xla_op_context.SetOutput(idx, b->GetTupleElement(output_handle, idx));
-  }
-  return b->first_error();
-}
-
-void GraphCompiler::PartiallySetupParams(OpKernelContext::Params* params) {
-  params->device = device_;
-  params->inputs = &tensor_inputs_;
-  params->step_container = step_container_;
-  params->resource_manager = device_->resource_manager();
-}
-
-}  // namespace tensorflow
diff --git a/tensorflow/compiler/tf2xla/graph_compiler.h b/tensorflow/compiler/tf2xla/graph_compiler.h
deleted file mode 100644
index 6fc0b18dcd..0000000000
--- a/tensorflow/compiler/tf2xla/graph_compiler.h
+++ /dev/null
@@ -1,103 +0,0 @@
-/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#ifndef TENSORFLOW_COMPILER_TF2XLA_GRAPH_COMPILER_H_
-#define TENSORFLOW_COMPILER_TF2XLA_GRAPH_COMPILER_H_
-
-#include "tensorflow/compiler/tf2xla/xla_compilation_device.h"
-#include "tensorflow/compiler/tf2xla/xla_context.h"
-#include "tensorflow/compiler/xla/client/local_client.h"
-#include "tensorflow/core/common_runtime/device.h"
-#include "tensorflow/core/common_runtime/device_mgr.h"
-#include "tensorflow/core/common_runtime/function.h"
-#include "tensorflow/core/framework/function.h"
-#include "tensorflow/core/framework/op_kernel.h"
-#include "tensorflow/core/lib/core/status.h"
-#include "tensorflow/core/platform/env.h"
-#include "tensorflow/core/platform/mutex.h"
-#include "tensorflow/core/platform/notification.h"
-#include "tensorflow/core/platform/thread_annotations.h"
-#include "tensorflow/core/public/version.h"
-
-namespace tensorflow {
-
-using FunctionCompiler =
-    std::function<xla::StatusOr<std::shared_ptr<xla::Computation>>(
-        const NameAttrList& function, XlaOpKernelContext* xla_op_context)>;
-
-// GraphCompiler compiles the graph in topological order in the current
-// thread. It also resolves the nondeterminism in the graph by enforcing a total
-// order on all inputs to a node. This abstraction helps us create the same XLA
-// computation given two structurally equivalent TensorFlow graphs. If a
-// function call is visited during the graph traversal, it is then compiled
-// through the FunctionCompiler into a computation and a `Call` operation is
-// inserted to call into that computation.
-class GraphCompiler {
- public:
-  GraphCompiler(XlaContext* xla_context, XlaCompilationDevice* device,
-                Graph* graph, FunctionLibraryRuntime* flib,
-                ScopedStepContainer* step_container,
-                const FunctionCompiler& compiler)
-      : xla_context_(xla_context),
-        device_(device),
-        graph_(graph),
-        flib_(flib),
-        step_container_(step_container),
-        compiler_(compiler) {}
-
-  // Compiles the graph. The results are written in `xla_context` that is passed
-  // into the compiler.
-  Status Compile();
-
- private:
-  // NodeBinding is a wrapper on a `Node` that also contains computed
-  // TensorValue.
-  struct NodeBinding {
-    const Node* node;
-    // Kernel for this node, to be filled by CreateKernel.
-    OpKernel* op_kernel;
-    // Output values of this node.
-    std::vector<TensorValue> tensor_values;
-    // Attributes of the outputs.
-    gtl::InlinedVector<AllocatorAttributes, 4> output_attrs;
-  };
-
-  // Partially sets params. This partially set params can be reused
-  // across multple nodes visit.
-  void PartiallySetupParams(OpKernelContext::Params* params);
-
-  // Tests if a node is a functional node. A functional node represents a
-  // defined computation and should be compiled using `compiler_`.
-  bool IsFunctional(Node*);
-
-  // Compiles a functional node and writes result to OpkernelContext. A
-  // functional node represents a defined computation and should be compiled
-  // using `compiler_`.
-  Status CompileFunctionalNode(Node*, OpKernelContext*);
-
-  XlaContext* xla_context_;
-  XlaCompilationDevice* device_;
-  Graph* graph_;
-  FunctionLibraryRuntime* flib_;
-  ScopedStepContainer* step_container_;
-  FunctionCompiler compiler_;
-  // A buffer to hold tensor inputs to a node, this is reused across the graph
-  // traversal.
-  gtl::InlinedVector<TensorValue, 4> tensor_inputs_;
-};
-
-}  // namespace tensorflow
-
-#endif  // TENSORFLOW_COMPILER_TF2XLA_GRAPH_COMPILER_H_
diff --git a/tensorflow/compiler/tf2xla/xla_compiler.cc b/tensorflow/compiler/tf2xla/xla_compiler.cc
index 9e405578aa..8521d4167a 100644
--- a/tensorflow/compiler/tf2xla/xla_compiler.cc
+++ b/tensorflow/compiler/tf2xla/xla_compiler.cc
@@ -20,12 +20,10 @@ limitations under the License.
 
 #include "tensorflow/compiler/tf2xla/dump_graph.h"
 #include "tensorflow/compiler/tf2xla/functionalize_control_flow.h"
-#include "tensorflow/compiler/tf2xla/graph_compiler.h"
 #include "tensorflow/compiler/tf2xla/shape_util.h"
 #include "tensorflow/compiler/tf2xla/type_util.h"
 #include "tensorflow/compiler/tf2xla/xla_compilation_device.h"
 #include "tensorflow/compiler/tf2xla/xla_context.h"
-#include "tensorflow/compiler/tf2xla/xla_op_kernel.h"
 #include "tensorflow/compiler/xla/client/client_library.h"
 #include "tensorflow/core/common_runtime/device.h"
 #include "tensorflow/core/common_runtime/executor.h"
@@ -180,34 +178,9 @@ Status XlaCompiler::CompileFunction(
 
 namespace {
 
-// Builds XlaCompiler argument descriptions `args` from `ctx`.
-Status MakeXlaCompilerArgumentsFromInputs(
-    XlaOpKernelContext* ctx, std::vector<XlaCompiler::Argument>* args) {
-  VLOG(2) << "Num inputs " << ctx->num_inputs();
-  args->resize(ctx->num_inputs());
-  for (int i = 0; i < ctx->num_inputs(); ++i) {
-    VLOG(2) << "  Input " << i
-            << " type: " << DataTypeString(ctx->input_type(i))
-            << " shape: " << ctx->InputShape(i).DebugString();
-    XlaCompiler::Argument& arg = (*args)[i];
-    DataType type = ctx->input_type(i);
-
-    if (type == DT_RESOURCE) {
-      return errors::InvalidArgument(
-          "Resource as function argument is not yet implemented.");
-    } else {
-      arg.kind = XlaCompiler::Argument::kParameter;
-      arg.type = ctx->input_type(i);
-      TF_RETURN_IF_ERROR(
-          TensorShapeToXLAShape(arg.type, ctx->InputShape(i), &arg.shape));
-    }
-  }
-  return Status::OK();
-}
-
-Status ExecuteGraph(XlaCompiler* compiler, XlaContext* xla_context,
-                    std::unique_ptr<Graph> graph, XlaCompilationDevice* device,
-                    FunctionLibraryRuntime* flib, int64 step_id) {
+Status ExecuteGraph(XlaContext* xla_context, std::unique_ptr<Graph> graph,
+                    XlaCompilationDevice* device, FunctionLibraryRuntime* flib,
+                    int64 step_id) {
   // Resource cleanup is a bit messy. XlaContext is a ref-counted resource; the
   // resource manager takes ownership via Create, and unrefs via Cleanup.  We
   // explicitly add a reference to ensure the refcount at entry is maintained at
@@ -224,27 +197,56 @@ Status ExecuteGraph(XlaCompiler* compiler, XlaContext* xla_context,
   TF_RETURN_IF_ERROR(device->resource_manager()->Create(
       step_container->name(), XlaContext::kXlaContextResourceName,
       xla_context));
-  // Compile_func is used to tell the serial executor how to compile a function.
-  auto compile_func = [&](const NameAttrList& function,
-                          XlaOpKernelContext* xla_op_context)
-      -> xla::StatusOr<std::shared_ptr<xla::Computation>> {
-    std::vector<XlaCompiler::Argument> arguments;
-
-    TF_RETURN_IF_ERROR(
-        MakeXlaCompilerArgumentsFromInputs(xla_op_context, &arguments));
-
-    XlaCompiler::CompilationResult result;
-    TF_RETURN_IF_ERROR(compiler->CompileFunction(XlaCompiler::CompileOptions(),
-                                                 function, arguments, &result));
-    return result.computation;
+
+  // Create a LocalExecutor that will own and run the graph.
+  // TODO(b/66947550): migrate away from using an Executor in order to guarantee
+  // determinism and thread-safety.
+  LocalExecutorParams exec_params;
+  exec_params.device = device;
+  exec_params.function_library = flib;
+  exec_params.create_kernel = [flib](const NodeDef& ndef, OpKernel** kernel) {
+    return flib->CreateKernel(ndef, kernel);
+  };
+  exec_params.delete_kernel = [](OpKernel* kernel) { delete kernel; };
+  Executor* exec_ptr = nullptr;
+  TF_RETURN_IF_ERROR(NewLocalExecutor(exec_params, graph.release(), &exec_ptr));
+  std::unique_ptr<Executor> exec(exec_ptr);
+  // At this point ownership of the graph has been transferred to exec.
+
+  // Run the graph symbolically, turning the graph into an XLA computation.
+  Executor::Args exec_args;
+  exec_args.step_id = step_id;
+  exec_args.step_container = step_container.get();
+
+  // Pushes closures to run onto `worklist`. We don't run the closures directly
+  // from 'runner' since that might lead to a stack overflow for large graphs.
+  std::deque<Executor::Args::Closure> worklist;
+  exec_args.runner = [&](Executor::Args::Closure c) {
+    worklist.push_back(std::move(c));
   };
 
-  GraphCompiler graph_compiler(xla_context, device, graph.get(), flib,
-                               step_container.get(), compile_func);
-  TF_RETURN_IF_ERROR(graph_compiler.Compile());
+  // The following code assumes there is only one thread involved and no
+  // concurrency, because we did not provide Executor a threaded runner. Async
+  // ops on the XlaCompilation device must not use threads or concurrency
+  // internally.
+  bool done = false;
+  exec->RunAsync(exec_args, [&](const Status& s) {
+    status = s;
+    done = true;
+  });
+  // Repeatedly run closures from the worklist until `done` is signalled.
+  while (!done) {
+    TF_RET_CHECK(!worklist.empty());
+    Executor::Args::Closure& c = worklist.front();
+    c();
+    worklist.pop_front();
+  }
+  TF_RETURN_WITH_CONTEXT_IF_ERROR(
+      status, "Conversion from TensorFlow graph to XLA computation failed.");
+
   // Explicitly clean up the step container, to capture the cleanup status.
   step_container.reset();
-  return Status::OK();
+  return status;
 }
 
 // Builds XLA computations for each of the arguments to the computation.
@@ -492,7 +494,7 @@ Status XlaCompiler::CompileGraph(const XlaCompiler::CompileOptions& options,
       &result->input_mapping, &result->xla_input_shapes));
   context->set_args(std::move(arg_expressions));
 
-  TF_RETURN_IF_ERROR(ExecuteGraph(this, context, std::move(graph), device_,
+  TF_RETURN_IF_ERROR(ExecuteGraph(context, std::move(graph), device_,
                                   flib_runtime_, NextStepId()));
 
   int num_nonconst_outputs;
diff --git a/tensorflow/compiler/tf2xla/xla_compiler.h b/tensorflow/compiler/tf2xla/xla_compiler.h
index 0435c619f8..35159dbad4 100644
--- a/tensorflow/compiler/tf2xla/xla_compiler.h
+++ b/tensorflow/compiler/tf2xla/xla_compiler.h
@@ -240,7 +240,7 @@ class XlaCompiler {
     bool use_tuple_arg = false;
 
     // If 'return_updated_values_for_all_resources' is true, then updated
-    // values of all resource arguments will be included in the
+    // values of all resource resources arguments will be included in the
     // 'resource_updates' of the computation, even if the resource was not
     // modified by the computation. Used when compiling loop bodies to ensure
     // the input and output signatures match.
diff --git a/tensorflow/compiler/tf2xla/xla_compiler_test.cc b/tensorflow/compiler/tf2xla/xla_compiler_test.cc
index 88ed3b89a6..531725a623 100644
--- a/tensorflow/compiler/tf2xla/xla_compiler_test.cc
+++ b/tensorflow/compiler/tf2xla/xla_compiler_test.cc
@@ -63,7 +63,6 @@ class DummyReadResourceOp : public XlaOpKernel {
     dummy->Unref();
 
     ctx->SetOutput(0, ctx->Input(0));
-    ctx->SetOutput(1, ctx->Input(0));
   }
 };
 
@@ -81,25 +80,22 @@ class DummyReadResourceCC {
     if (!scope.ok()) return;
     scope.UpdateStatus(scope.DoShapeInference(ret));
     if (!scope.ok()) return;
-    this->output1_ = Output(ret, 0);
-    this->output2_ = Output(ret, 1);
+    this->output_ = Output(ret, 0);
   }
+  Node* node() const { return output_.node(); }
 
-  Output output1_;
-  Output output2_;
+  Output output_;
 };
 
 REGISTER_OP("DummyReadResource")
     .Input("input: int32")
-    .Output("output1: int32")
-    .Output("output2: int32")
+    .Output("output: int32")
     .SetShapeFn(shape_inference::UnknownShape)
     .Doc(R"doc(
 A dummy Op.
 
 input: dummy input.
-output1: dummy output.
-output2: dummy output.
+output: dummy output.
 )doc");
 
 REGISTER_XLA_OP(Name("DummyReadResource"), DummyReadResourceOp);
@@ -320,8 +316,7 @@ TEST_F(XlaCompilerTest, ResourceManager) {
   Scope scope = Scope::NewRootScope().ExitOnError();
   auto a = ops::_Arg(scope.WithOpName("A"), DT_INT32, 0);
   auto b = DummyReadResourceCC(scope.WithOpName("B"), a);
-  auto c = ops::Add(scope.WithOpName("C"), b.output2_, b.output1_);
-  auto d = ops::_Retval(scope.WithOpName("D"), c, 0);
+  auto c = ops::_Retval(scope.WithOpName("C"), b.output_, 0);
   std::unique_ptr<Graph> graph(new Graph(OpRegistry::Global()));
   TF_ASSERT_OK(scope.ToGraph(graph.get()));
 
@@ -354,58 +349,6 @@ TEST_F(XlaCompilerTest, ResourceManager) {
   resource->Unref();
 }
 
-// Tests compilation and execution of a graph that adds two tensors.
-TEST_F(XlaCompilerTest, DeterministicCompilation) {
-  // Builds a graph that contains a node with two output edges. The compiler
-  // should always traverse them in the same order.
-  const int64 test_count = 2;
-
-  std::vector<XlaCompiler::CompilationResult> results(test_count);
-
-  for (int64 i = 0; i < test_count; ++i) {
-    Scope scope = Scope::NewRootScope().ExitOnError();
-    auto a = ops::_Arg(scope.WithOpName("A"), DT_INT32, 0);
-    auto b = ops::Neg(scope.WithOpName("B"), a);
-    auto c = ops::Neg(scope.WithOpName("C"), a);
-    auto d = ops::Add(scope.WithOpName("D"), b, c);
-    auto e = ops::_Retval(scope.WithOpName("E"), d, 0);
-    std::unique_ptr<Graph> graph(new Graph(OpRegistry::Global()));
-    TF_ASSERT_OK(scope.ToGraph(graph.get()));
-
-    // Builds a description of the argument.
-    std::vector<XlaCompiler::Argument> args(1);
-    args[0].kind = XlaCompiler::Argument::kParameter;
-    args[0].type = DT_INT32;
-    args[0].shape = xla::ShapeUtil::MakeShape(xla::S32, {2});
-
-    // Compiles the graph.
-    auto options = DefaultOptions();
-    XlaCompiler compiler(options);
-
-    TF_ASSERT_OK(compiler.CompileGraph(XlaCompiler::CompileOptions(), "dummy",
-                                       std::move(graph), args, &results[i]));
-  }
-
-  for (int64 i = 1; i < test_count; ++i) {
-    auto m1 =
-        results[i - 1].computation->Snapshot().ValueOrDie()->entry().requests();
-    auto m2 =
-        results[i].computation->Snapshot().ValueOrDie()->entry().requests();
-    // Check if every entry is the same.
-    for (auto& entry1 : m1) {
-      int64 key = entry1.first;
-      auto value1 = entry1.second;
-      auto entry2 = m2.find(key);
-      auto value2 = entry2->second;
-      EXPECT_TRUE(entry2 != m2.end());
-      string str1, str2;
-      value1.AppendToString(&str1);
-      value2.AppendToString(&str2);
-      EXPECT_EQ(str1, str2);
-    }
-  }
-}
-
 // Tests a computation that receives a TensorArray resource as input and
 // updates it.
 TEST_F(XlaCompilerTest, CanPassTensorArraysToAndFromComputation) {
diff --git a/tensorflow/compiler/xla/service/service.cc b/tensorflow/compiler/xla/service/service.cc
index d279e1f50f..bd7898a41f 100644
--- a/tensorflow/compiler/xla/service/service.cc
+++ b/tensorflow/compiler/xla/service/service.cc
@@ -187,9 +187,8 @@ tensorflow::Status Service::Computation(const ComputationRequest* arg,
 
   *result->mutable_computation() =
       computation_tracker_.NewComputation(arg->name());
-  VLOG(1) << Printf("Created new computation %s on service %p, name %s",
-                    result->computation().ShortDebugString().c_str(), this,
-                    arg->name().c_str());
+  VLOG(1) << Printf("Created new computation %s on service %p",
+                    result->computation().ShortDebugString().c_str(), this);
   return tensorflow::Status::OK();
 }
 
diff --git a/tensorflow/core/graph/algorithm.cc b/tensorflow/core/graph/algorithm.cc
index 6ef51aa7df..3bfba3fc4e 100644
--- a/tensorflow/core/graph/algorithm.cc
+++ b/tensorflow/core/graph/algorithm.cc
@@ -24,8 +24,7 @@ limitations under the License.
 namespace tensorflow {
 
 void DFS(const Graph& g, const std::function<void(Node*)>& enter,
-         const std::function<void(Node*)>& leave,
-         const NodeComparator& stable_comparator) {
+         const std::function<void(Node*)>& leave) {
   // Stack of work to do.
   struct Work {
     Node* node;
@@ -52,41 +51,24 @@ void DFS(const Graph& g, const std::function<void(Node*)>& enter,
     // Arrange to call leave(n) when all done with descendants.
     if (leave) stack.push_back(Work{n, true});
 
-    gtl::iterator_range<NeighborIter> nodes = n->out_nodes();
-    auto add_work = [&visited, &stack](Node* out) {
+    // Arrange to work on descendants.
+    for (Node* out : n->out_nodes()) {
       if (!visited[out->id()]) {
         // Note; we must not mark as visited until we actually process it.
         stack.push_back(Work{out, false});
       }
-    };
-
-    if (stable_comparator) {
-      std::vector<Node*> nodes_sorted;
-      for (Node* out : nodes) {
-        nodes_sorted.emplace_back(out);
-      }
-      std::sort(nodes_sorted.begin(), nodes_sorted.end(), stable_comparator);
-      for (Node* out : nodes_sorted) {
-        add_work(out);
-      }
-    } else {
-      for (Node* out : nodes) {
-        add_work(out);
-      }
     }
   }
 }
 
 void ReverseDFS(const Graph& g, const std::function<void(Node*)>& enter,
-                const std::function<void(Node*)>& leave,
-                const NodeComparator& stable_comparator) {
-  ReverseDFSFrom(g, {g.sink_node()}, enter, leave, stable_comparator);
+                const std::function<void(Node*)>& leave) {
+  ReverseDFSFrom(g, {g.sink_node()}, enter, leave);
 }
 
 void ReverseDFSFrom(const Graph& g, gtl::ArraySlice<Node*> start,
                     const std::function<void(Node*)>& enter,
-                    const std::function<void(Node*)>& leave,
-                    const NodeComparator& stable_comparator) {
+                    const std::function<void(Node*)>& leave) {
   // Stack of work to do.
   struct Work {
     Node* node;
@@ -115,41 +97,23 @@ void ReverseDFSFrom(const Graph& g, gtl::ArraySlice<Node*> start,
     // Arrange to call leave(n) when all done with descendants.
     if (leave) stack.push_back(Work{n, true});
 
-    gtl::iterator_range<NeighborIter> nodes = n->in_nodes();
-
-    auto add_work = [&visited, &stack](Node* out) {
-      if (!visited[out->id()]) {
+    // Arrange to work on parents.
+    for (Node* in : n->in_nodes()) {
+      if (!visited[in->id()]) {
         // Note; we must not mark as visited until we actually process it.
-        stack.push_back(Work{out, false});
-      }
-    };
-
-    if (stable_comparator) {
-      std::vector<Node*> nodes_sorted;
-      for (Node* in : nodes) {
-        nodes_sorted.emplace_back(in);
-      }
-      std::sort(nodes_sorted.begin(), nodes_sorted.end(), stable_comparator);
-      for (Node* in : nodes_sorted) {
-        add_work(in);
-      }
-    } else {
-      for (Node* in : nodes) {
-        add_work(in);
+        stack.push_back(Work{in, false});
       }
     }
   }
 }
 
-void GetPostOrder(const Graph& g, std::vector<Node*>* order,
-                  const NodeComparator& stable_comparator) {
+void GetPostOrder(const Graph& g, std::vector<Node*>* order) {
   order->clear();
-  DFS(g, nullptr, [order](Node* n) { order->push_back(n); }, stable_comparator);
+  DFS(g, nullptr, [order](Node* n) { order->push_back(n); });
 }
 
-void GetReversePostOrder(const Graph& g, std::vector<Node*>* order,
-                         const NodeComparator& stable_comparator) {
-  GetPostOrder(g, order, stable_comparator);
+void GetReversePostOrder(const Graph& g, std::vector<Node*>* order) {
+  GetPostOrder(g, order);
   std::reverse(order->begin(), order->end());
 }
 
diff --git a/tensorflow/core/graph/algorithm.h b/tensorflow/core/graph/algorithm.h
index 5bb6041d98..01d36e0a12 100644
--- a/tensorflow/core/graph/algorithm.h
+++ b/tensorflow/core/graph/algorithm.h
@@ -25,50 +25,24 @@ limitations under the License.
 
 namespace tensorflow {
 
-// Comparator for two nodes. This is used in order to get a stable ording.
-using NodeComparator = std::function<bool(const Node*, const Node*)>;
-
-// Compares two node based on their ids.
-struct NodeComparatorID {
-  bool operator()(const Node* n1, const Node* n2) const {
-    return n1->id() < n2->id();
-  }
-};
-
-// Compare two nodes based on their names.
-struct NodeComparatorName {
-  bool operator()(const Node* n1, const Node* n2) const {
-    return n1->name() < n2->name();
-  }
-};
-
 // Perform a depth-first-search on g starting at the source node.
 // If enter is not empty, calls enter(n) before visiting any children of n.
 // If leave is not empty, calls leave(n) after visiting all children of n.
-// If stable_comparator is set, a stable ordering of visit is achieved by
-// sorting a node's neighbors first before visiting them.
 extern void DFS(const Graph& g, const std::function<void(Node*)>& enter,
-                const std::function<void(Node*)>& leave,
-                const NodeComparator& stable_comparator = {});
+                const std::function<void(Node*)>& leave);
 
 // Perform a reverse depth-first-search on g starting at the sink node.
 // If enter is not empty, calls enter(n) before visiting any parents of n.
 // If leave is not empty, calls leave(n) after visiting all parents of n.
-// If stable_comparator is set, a stable ordering of visit is achieved by
-// sorting a node's neighbors first before visiting them.
 extern void ReverseDFS(const Graph& g, const std::function<void(Node*)>& enter,
-                       const std::function<void(Node*)>& leave,
-                       const NodeComparator& stable_comparator = {});
+                       const std::function<void(Node*)>& leave);
 
 // Perform a reverse depth-first-search on g starting at the 'start' nodes.
 // If enter is not empty, calls enter(n) before visiting any parents of n.
 // If leave is not empty, calls leave(n) after visiting all parents of n.
-// If stable_comparator is set, a stable ordering of visit is achieved by
-// sorting a node's neighbors first before visiting them.
 extern void ReverseDFSFrom(const Graph& g, gtl::ArraySlice<Node*> start,
                            const std::function<void(Node*)>& enter,
-                           const std::function<void(Node*)>& leave,
-                           const NodeComparator& stable_comparator = {});
+                           const std::function<void(Node*)>& leave);
 
 // Stores in *order the post-order numbering of all nodes
 // in graph found via a depth first search starting at the source node.
@@ -76,18 +50,11 @@ extern void ReverseDFSFrom(const Graph& g, gtl::ArraySlice<Node*> start,
 // Note that this is equivalent to reverse topological sorting when the
 // graph does not have cycles.
 //
-// If stable_comparator is set, a stable ordering of visit is achieved by
-// sorting a node's neighbors first before visiting them.
-//
 // REQUIRES: order is not NULL.
-void GetPostOrder(const Graph& g, std::vector<Node*>* order,
-                  const NodeComparator& stable_comparator = {});
+void GetPostOrder(const Graph& g, std::vector<Node*>* order);
 
 // Stores in *order the reverse post-order numbering of all nodes
-// If stable_comparator is set, a stable ordering of visit is achieved by
-// sorting a node's neighbors first before visiting them.
-void GetReversePostOrder(const Graph& g, std::vector<Node*>* order,
-                         const NodeComparator& stable_comparator = {});
+void GetReversePostOrder(const Graph& g, std::vector<Node*>* order);
 
 // Prune nodes in "g" that are not in some path from the source node
 // to any node in 'nodes'. Returns true if changes were made to the graph.
diff --git a/tensorflow/core/graph/algorithm_test.cc b/tensorflow/core/graph/algorithm_test.cc
index 0cdcdb6685..a529760426 100644
--- a/tensorflow/core/graph/algorithm_test.cc
+++ b/tensorflow/core/graph/algorithm_test.cc
@@ -112,40 +112,5 @@ TEST(AlgorithmTest, ReversePostOrder) {
   EXPECT_FALSE(ExpectBefore(orders, order, &error));
 }
 
-TEST(AlgorithmTest, ReversePostOrderStable) {
-  int64 run_count = 100;
-  using namespace ::tensorflow::ops;  // NOLINT(build/namespaces)
-
-  for (int64 i = 0; i < run_count; ++i) {
-    // One source of nondeterminism comes from unordered set with key of a
-    // pointer type, for example the order of FlatSet<Node*> depends on the
-    // raw pointer value of Node. Stable post order suppose to remove this
-    // nondeterminism by enforcing an ordering based on node ids.
-    GraphDefBuilder b(GraphDefBuilder::kFailImmediately);
-    string error;
-    Node* w1 = SourceOp("TestParams", b.opts().WithName("W1"));
-    Node* input =
-        SourceOp("TestInput", b.opts().WithName("input").WithControlInput(w1));
-    BinaryOp("TestMul", w1, {input, 1}, b.opts().WithName("t2"));
-    // Insert different number of nodes between the allocation of t2 and t3,
-    // this creates enough entropy in the memory distance between t2 and t3 thus
-    // forces them to have randomized ordering had stable DFS was not
-    // implemented correctly.
-    for (int64 j = 0; j < i; ++j) {
-      BinaryOp("TestMul", w1, {input, 1},
-               b.opts().WithName(strings::StrCat("internal", j)));
-    }
-
-    BinaryOp("TestMul", w1, {input, 1}, b.opts().WithName("t3"));
-
-    Graph g(OpRegistry::Global());
-    TF_ASSERT_OK(b.ToGraph(&g));
-    std::vector<Node*> order;
-
-    // Test reverse post order generates expected ordering.
-    GetReversePostOrder(g, &order, /*stable_comparator=*/NodeComparatorID());
-    EXPECT_TRUE(ExpectBefore({{"t3", "t2"}}, order, &error));
-  }
-}
 }  // namespace
 }  // namespace tensorflow
diff --git a/tensorflow/core/graph/graph.h b/tensorflow/core/graph/graph.h
index 54076ed1ab..5a31a6216b 100644
--- a/tensorflow/core/graph/graph.h
+++ b/tensorflow/core/graph/graph.h
@@ -298,12 +298,12 @@ class Edge {
   Node* dst() const { return dst_; }
   int id() const { return id_; }
 
-  // Return the index of the source output that produces the data
+  // Return the number of the source output that produces the data
   // carried by this edge.  The special value kControlSlot is used
   // for control dependencies.
   int src_output() const { return src_output_; }
 
-  // Return the index of the destination input that consumes the data
+  // Return the number of the destination input that consumes the data
   // carried by this edge.  The special value kControlSlot is used
   // for control dependencies.
   int dst_input() const { return dst_input_; }
-- 
GitLab


From 9aad24f89ee9fbaa31f36087ec5fc527d7b728b6 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 6 Oct 2017 10:54:59 -0700
Subject: [PATCH 115/909] One last data_set race condition fix.

PiperOrigin-RevId: 171313226
---
 .../tensor_forest/kernels/stats_ops.cc        | 19 ++++++++-----------
 1 file changed, 8 insertions(+), 11 deletions(-)

diff --git a/tensorflow/contrib/tensor_forest/kernels/stats_ops.cc b/tensorflow/contrib/tensor_forest/kernels/stats_ops.cc
index b6d57ef952..f80a34ece6 100644
--- a/tensorflow/contrib/tensor_forest/kernels/stats_ops.cc
+++ b/tensorflow/contrib/tensor_forest/kernels/stats_ops.cc
@@ -235,9 +235,6 @@ class ProcessInputOp : public OpKernel {
     string serialized_proto;
     OP_REQUIRES_OK(context, context->GetAttr("input_spec", &serialized_proto));
     input_spec_.ParseFromString(serialized_proto);
-
-    data_set_ = std::unique_ptr<TensorDataSet>(
-        new TensorDataSet(input_spec_, random_seed_));
   }
 
   void Compute(OpKernelContext* context) override {
@@ -249,8 +246,9 @@ class ProcessInputOp : public OpKernel {
     const Tensor& input_weights = context->input(7);
     const Tensor& leaf_ids_tensor = context->input(8);
 
-    data_set_->set_input_tensors(input_data, sparse_input_indices,
-                                 sparse_input_values, sparse_input_shape);
+    std::unique_ptr<TensorDataSet> data_set(new TensorDataSet(input_spec_, 0));
+    data_set->set_input_tensors(input_data, sparse_input_indices,
+                                sparse_input_values, sparse_input_shape);
 
     FertileStatsResource* fertile_stats_resource;
     OP_REQUIRES_OK(context, LookupResource(context, HandleFromInput(context, 1),
@@ -264,7 +262,7 @@ class ProcessInputOp : public OpKernel {
     core::ScopedUnref unref_stats(fertile_stats_resource);
     core::ScopedUnref unref_tree(tree_resource);
 
-    const int32 num_data = data_set_->NumItems();
+    const int32 num_data = data_set->NumItems();
     auto worker_threads = context->device()->tensorflow_cpu_worker_threads();
     int num_threads = worker_threads->num_threads;
 
@@ -308,23 +306,23 @@ class ProcessInputOp : public OpKernel {
     // from a digits run on local desktop.  Heuristics might be necessary
     // if it really matters that much.
     const int64 costPerUpdate = 1000;
-    auto update = [this, &target, &leaf_ids_tensor, &num_targets,
+    auto update = [this, &target, &leaf_ids_tensor, &num_targets, &data_set,
                    fertile_stats_resource, &locks, &set_lock, &ready_to_split,
                    num_data](int64 start, int64 end) {
       CHECK(start <= end);
       CHECK(end <= num_data);
-      UpdateStats(fertile_stats_resource, data_set_, target, num_targets,
+      UpdateStats(fertile_stats_resource, data_set, target, num_targets,
                   leaf_ids_tensor, &locks, &set_lock, static_cast<int32>(start),
                   static_cast<int32>(end), &ready_to_split);
     };
 
     auto update_collated = [this, &target, &num_targets, fertile_stats_resource,
                             tree_resource, &leaf_examples, &set_lock,
-                            &ready_to_split,
+                            &ready_to_split, &data_set,
                             num_leaves](int64 start, int64 end) {
       CHECK(start <= end);
       CHECK(end <= num_leaves);
-      UpdateStatsCollated(fertile_stats_resource, tree_resource, data_set_,
+      UpdateStatsCollated(fertile_stats_resource, tree_resource, data_set,
                           target, num_targets, leaf_examples, &set_lock,
                           static_cast<int32>(start), static_cast<int32>(end),
                           &ready_to_split);
@@ -350,7 +348,6 @@ class ProcessInputOp : public OpKernel {
  private:
   int32 random_seed_;
   tensorforest::TensorForestDataSpec input_spec_;
-  std::unique_ptr<TensorDataSet> data_set_;
   TensorForestParams param_proto_;
 };
 
-- 
GitLab


From dc500c869721e93ae1f3036b677a1d9d424e9d23 Mon Sep 17 00:00:00 2001
From: Jacques Pienaar <jpienaar@google.com>
Date: Fri, 6 Oct 2017 11:03:06 -0700
Subject: [PATCH 116/909] [TF2XLA] Update device name in convert and redo check
 that name parsing is correct.

* Update ConvertGraphToXla to use the new form for setting the assigned device name.
* Remove some stale comments.
* Revert workaround that allowed the requested device name to not be parsed.

PiperOrigin-RevId: 171314671
---
 tensorflow/compiler/tf2xla/tf2xla.cc          |  5 ++--
 .../compiler/tf2xla/xla_compilation_device.cc | 23 +++++++++++--------
 2 files changed, 15 insertions(+), 13 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/tf2xla.cc b/tensorflow/compiler/tf2xla/tf2xla.cc
index b7213a6cc1..a14c93a2b9 100644
--- a/tensorflow/compiler/tf2xla/tf2xla.cc
+++ b/tensorflow/compiler/tf2xla/tf2xla.cc
@@ -255,11 +255,10 @@ Status CreateXlaArgs(const Graph& graph,
 Status ConvertGraphToXla(std::unique_ptr<Graph> graph, xla::Client* client,
                          xla::Computation* computation,
                          bool* requires_runtime_context) {
-  // Create a device and context to convert the graph into an XLA computation.
   XlaOpRegistry::RegisterCompilationKernels();
-  // Populate the context with args from the graph.
   for (Node* node : graph->nodes()) {
-    node->set_assigned_device_name(DEVICE_CPU_XLA_JIT);
+    node->set_assigned_device_name(
+        strings::StrCat("/device:", DEVICE_CPU_XLA_JIT));
   }
   std::vector<XlaCompiler::Argument> xla_args;
   TF_RETURN_IF_ERROR(CreateXlaArgs(*graph, &xla_args));
diff --git a/tensorflow/compiler/tf2xla/xla_compilation_device.cc b/tensorflow/compiler/tf2xla/xla_compilation_device.cc
index 3814a2b8b9..890a9ccb83 100644
--- a/tensorflow/compiler/tf2xla/xla_compilation_device.cc
+++ b/tensorflow/compiler/tf2xla/xla_compilation_device.cc
@@ -98,17 +98,20 @@ void XlaCompilationDevice::Compute(OpKernel* op_kernel,
   b->SetOpMetadata(metadata);
 
   DeviceNameUtils::ParsedName parsed;
-  if (DeviceNameUtils::ParseFullName(op_kernel->requested_device(), &parsed)) {
-    // If no device ID assignment is found, XLA is free to use whatever device
-    // it wants. In practice this usually has the effect of placing things on
-    // device 0.
-    xla::OpDeviceAssignment assignment;
-    if (parsed.has_id) {
-      assignment.set_has_device(true);
-      assignment.set_device(parsed.id);
-    }
-    b->SetDeviceAssignment(assignment);
+  OP_REQUIRES(
+      context,
+      DeviceNameUtils::ParseFullName(op_kernel->requested_device(), &parsed),
+      errors::Internal("Unable to parse device name: ",
+                       op_kernel->requested_device()));
+  xla::OpDeviceAssignment assignment;
+  // If no device ID assignment is found, XLA is free to use whatever device it
+  // wants. In practice this usually has the effect of placing things on
+  // device 0.
+  if (parsed.has_id) {
+    assignment.set_has_device(true);
+    assignment.set_device(parsed.id);
   }
+  b->SetDeviceAssignment(assignment);
 
   op_kernel->Compute(context);
 
-- 
GitLab


From 71a285922a4279fd35f73271e09b90d5787746a9 Mon Sep 17 00:00:00 2001
From: James Qin <jamesqin@google.com>
Date: Fri, 6 Oct 2017 11:04:52 -0700
Subject: [PATCH 117/909] Fix a minor issue w/ allreduce

PiperOrigin-RevId: 171314944
---
 tensorflow/contrib/all_reduce/python/all_reduce.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/all_reduce/python/all_reduce.py b/tensorflow/contrib/all_reduce/python/all_reduce.py
index 8e7f1791b8..22d7633ce2 100644
--- a/tensorflow/contrib/all_reduce/python/all_reduce.py
+++ b/tensorflow/contrib/all_reduce/python/all_reduce.py
@@ -762,6 +762,8 @@ def _reduce_non_singleton(input_tensors, red_f, un_op):
   if len(input_tensors) > 1:
     return red_f(input_tensors)
   else:
+    if not un_op:
+      return input_tensors
     output_tensors = []
     for t in input_tensors:
       with ops.colocate_with(t):
@@ -835,7 +837,7 @@ def _build_shuffle_hybrid(input_tensors, gather_devices, red_op, upper_level_f):
 
 
 def build_shuffle_then_ring(input_tensors, gather_devices, subdiv,
-                            red_n_op, red_op, un_op):
+                            red_n_op, red_op, un_op=None):
   """Construct hybrid of Shuffle within workers, Ring across workers."""
   def upper_builder(tensors):
     return build_ring_all_reduce(tensors, len(tensors), subdiv, [0],
-- 
GitLab


From b99457c2138482470ae976a6364ce0ba754503cf Mon Sep 17 00:00:00 2001
From: Chris Leary <leary@google.com>
Date: Fri, 6 Oct 2017 11:06:12 -0700
Subject: [PATCH 118/909] [XLA] Fix a bug in ComputationBuilder::Collapse and
 add more tests/docs.

Also updated test infrastructure so a shape mismatch does not cause a fatal
crash in index_util, but rather reports an appropriate test failure message.

PiperOrigin-RevId: 171315165
---
 tensorflow/compiler/xla/client/client.cc      |  1 +
 .../xla/client/computation_builder.cc         | 13 ++++
 .../compiler/xla/client/computation_builder.h | 10 +++
 .../compiler/xla/service/shape_inference.cc   |  9 ++-
 .../compiler/xla/tests/literal_test_util.cc   | 73 ++++++++++++++-----
 .../compiler/xla/tests/literal_test_util.h    |  2 +
 tensorflow/compiler/xla/tests/reshape_test.cc | 18 ++++-
 7 files changed, 105 insertions(+), 21 deletions(-)

diff --git a/tensorflow/compiler/xla/client/client.cc b/tensorflow/compiler/xla/client/client.cc
index 387253617e..7db2ea79fb 100644
--- a/tensorflow/compiler/xla/client/client.cc
+++ b/tensorflow/compiler/xla/client/client.cc
@@ -206,6 +206,7 @@ StatusOr<std::unique_ptr<GlobalData>> Client::Execute(
     *request.mutable_execution_options() = *execution_options;
   }
   for (GlobalData* argument : arguments) {
+    CHECK(argument != nullptr) << "Argument pointers must not be null.";
     *request.add_arguments() = argument->handle();
   }
 
diff --git a/tensorflow/compiler/xla/client/computation_builder.cc b/tensorflow/compiler/xla/client/computation_builder.cc
index 15a713513f..925dcd36c0 100644
--- a/tensorflow/compiler/xla/client/computation_builder.cc
+++ b/tensorflow/compiler/xla/client/computation_builder.cc
@@ -489,6 +489,16 @@ ComputationDataHandle ComputationBuilder::Collapse(
   }
   std::unique_ptr<Shape> original_shape = shape_or_status.ConsumeValueOrDie();
 
+  VLOG(3) << "original shape: " << ShapeUtil::HumanString(*original_shape);
+  VLOG(3) << "dims to collapse: "
+          << tensorflow::str_util::Join(dims_to_collapse, ",");
+
+  if (dims_to_collapse.size() <= 1) {
+    // Not collapsing anything, trivially we can return the operand versus
+    // enqueueing a trivial reshape.
+    return operand;
+  }
+
   std::vector<int64> new_sizes;
   for (int i = 0; i < ShapeUtil::Rank(*original_shape); ++i) {
     if (i <= dims_to_collapse.front() || i > dims_to_collapse.back()) {
@@ -498,6 +508,9 @@ ComputationDataHandle ComputationBuilder::Collapse(
     }
   }
 
+  VLOG(3) << "new sizes: [" << tensorflow::str_util::Join(new_sizes, ",")
+          << "]";
+
   return Reshape(operand, new_sizes);
 }
 
diff --git a/tensorflow/compiler/xla/client/computation_builder.h b/tensorflow/compiler/xla/client/computation_builder.h
index 73972c1290..7014685ea5 100644
--- a/tensorflow/compiler/xla/client/computation_builder.h
+++ b/tensorflow/compiler/xla/client/computation_builder.h
@@ -201,6 +201,16 @@ class ComputationBuilder {
   // {x=1024, y=32} by collapsing dims {0, 1, 2}. Collapsing dimensions must
   // be a consecutive, in-order subsequence of the operand dimensions.
   //
+  // Note that collapsing a single dimension does nothing:
+  //
+  //    {256} collapsing {0} => {256}
+  //    {1} collapsing {0} => {1}
+  //
+  // Collapsing multiple dimensions produces a single result dimension:
+  //
+  //    {256, 2} collapsing {0,1} => {512}
+  //    {256, 2, 3} collapsing {0,1} => {512, 3}
+  //
   // This could potentially cause data to be moved -- it provides a more
   // structured form of reshaping than an arbitrary Reshape operation.
   ComputationDataHandle Collapse(const ComputationDataHandle& operand,
diff --git a/tensorflow/compiler/xla/service/shape_inference.cc b/tensorflow/compiler/xla/service/shape_inference.cc
index ffd8018827..29221d2d29 100644
--- a/tensorflow/compiler/xla/service/shape_inference.cc
+++ b/tensorflow/compiler/xla/service/shape_inference.cc
@@ -1894,11 +1894,16 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(
 
   Shape inferred_shape =
       ShapeUtil::MakeShape(operand.element_type(), new_sizes);
+  VLOG(3) << "Reshape inferred shape: "
+          << ShapeUtil::HumanString(inferred_shape);
 
   if (ShapeUtil::ElementsIn(operand) != ShapeUtil::ElementsIn(inferred_shape)) {
     return InvalidArgument(
-        "reshape operation has mismatched element counts: from=%lld to=%lld",
-        ShapeUtil::ElementsIn(operand), ShapeUtil::ElementsIn(inferred_shape));
+        "reshape operation has mismatched element counts: from=%lld (%s) "
+        "to=%lld (%s)",
+        ShapeUtil::ElementsIn(operand), ShapeUtil::HumanString(operand).c_str(),
+        ShapeUtil::ElementsIn(inferred_shape),
+        ShapeUtil::HumanString(inferred_shape).c_str());
   }
 
   std::vector<int64> indices(ShapeUtil::Rank(operand));
diff --git a/tensorflow/compiler/xla/tests/literal_test_util.cc b/tensorflow/compiler/xla/tests/literal_test_util.cc
index 061a4e190f..2876a79dd8 100644
--- a/tensorflow/compiler/xla/tests/literal_test_util.cc
+++ b/tensorflow/compiler/xla/tests/literal_test_util.cc
@@ -39,30 +39,60 @@ limitations under the License.
 
 namespace xla {
 
-/* static */ void LiteralTestUtil::AssertEqualShapes(const Shape& expected,
-                                                     const Shape& actual) {
-  ASSERT_EQ(ShapeUtil::IsTuple(expected), ShapeUtil::IsTuple(actual));
+/* static */ ::testing::AssertionResult LiteralTestUtil::EqualShapes(
+    const Shape& expected, const Shape& actual) {
+  if (ShapeUtil::IsTuple(expected) != ShapeUtil::IsTuple(actual)) {
+    return ::testing::AssertionFailure()
+           << "tupleness-mismatch! want: " << ShapeUtil::HumanString(expected)
+           << " got: " << ShapeUtil::HumanString(actual);
+  }
   if (ShapeUtil::IsTuple(expected)) {
-    ASSERT_EQ(ShapeUtil::TupleElementCount(expected),
-              ShapeUtil::TupleElementCount(actual));
+    if (ShapeUtil::TupleElementCount(expected) !=
+        ShapeUtil::TupleElementCount(actual)) {
+      return ::testing::AssertionFailure()
+             << "want tuple element count: "
+             << ShapeUtil::TupleElementCount(expected)
+             << " got tuple element count: "
+             << ShapeUtil::TupleElementCount(actual);
+    }
     for (int i = 0; i < expected.tuple_shapes_size(); ++i) {
-      AssertEqualShapes(expected.tuple_shapes(i), actual.tuple_shapes(i));
+      ::testing::AssertionResult result =
+          EqualShapes(expected.tuple_shapes(i), actual.tuple_shapes(i));
+      if (!result) {
+        return result;
+      }
     }
   } else {
-    ASSERT_EQ(ShapeUtil::Rank(expected), ShapeUtil::Rank(actual))
-        << "want rank of: " << ShapeUtil::HumanString(expected)
-        << " got rank of: " << ShapeUtil::HumanString(actual);
-    ASSERT_EQ(expected.element_type(), actual.element_type())
-        << PrimitiveType_Name(expected.element_type()) << " vs "
-        << PrimitiveType_Name(actual.element_type());
-    ASSERT_EQ(expected.dimensions_size(), actual.dimensions_size());
+    if (ShapeUtil::Rank(expected) != ShapeUtil::Rank(actual)) {
+      return ::testing::AssertionFailure()
+             << "want rank of: " << ShapeUtil::HumanString(expected)
+             << " got rank of: " << ShapeUtil::HumanString(actual);
+    }
+    if (expected.element_type() != actual.element_type()) {
+      return ::testing::AssertionFailure()
+             << PrimitiveType_Name(expected.element_type()) << " vs "
+             << PrimitiveType_Name(actual.element_type());
+    }
+    if (expected.dimensions_size() != actual.dimensions_size()) {
+      return ::testing::AssertionFailure()
+             << "want dimensions_size " << expected.dimensions_size()
+             << " got dimensions_size " << actual.dimensions_size();
+    }
     for (int i = 0; i < expected.dimensions_size(); ++i) {
-      ASSERT_EQ(expected.dimensions(i), actual.dimensions(i))
-          << "mismatch in dimension #" << i
-          << " expected: " << ShapeUtil::HumanString(expected)
-          << " actual: " << ShapeUtil::HumanString(actual);
+      if (expected.dimensions(i) != actual.dimensions(i)) {
+        return ::testing::AssertionFailure()
+               << "mismatch in dimension #" << i
+               << " expected: " << ShapeUtil::HumanString(expected)
+               << " actual: " << ShapeUtil::HumanString(actual);
+      }
     }
   }
+  return ::testing::AssertionSuccess();
+}
+
+/* static */ void LiteralTestUtil::AssertEqualShapes(const Shape& expected,
+                                                     const Shape& actual) {
+  ASSERT_TRUE(EqualShapes(expected, actual));
 }
 
 /* static */ void LiteralTestUtil::AssertEqualShapesAndLayouts(
@@ -265,7 +295,14 @@ class NearComparator {
     VLOG(1) << "actual:";
     XLA_VLOG_LINES(1, actual.ToString());
 
-    LiteralTestUtil::AssertEqualShapes(expected.shape(), actual.shape());
+    // If the shapes mismatch, we simply fail the expectation instead of
+    // printing out data, as it's a type error rather than a value error.
+    ::testing::AssertionResult equal_shapes =
+        LiteralTestUtil::EqualShapes(expected.shape(), actual.shape());
+    if (!equal_shapes) {
+      EXPECT_TRUE(equal_shapes);
+      return false;
+    }
 
     // Set up members used during the comparison.
     num_miscompares_ = 0;
diff --git a/tensorflow/compiler/xla/tests/literal_test_util.h b/tensorflow/compiler/xla/tests/literal_test_util.h
index f645c4e8dc..467d44b857 100644
--- a/tensorflow/compiler/xla/tests/literal_test_util.h
+++ b/tensorflow/compiler/xla/tests/literal_test_util.h
@@ -50,6 +50,8 @@ class LiteralTestUtil {
  public:
   // Asserts that the given shapes have the same rank, dimension sizes, and
   // primitive types.
+  static ::testing::AssertionResult EqualShapes(const Shape& expected,
+                                                const Shape& actual);
   static void AssertEqualShapes(const Shape& expected, const Shape& actual);
 
   // Asserts that the provided shapes are equal as defined in AssertEqualShapes
diff --git a/tensorflow/compiler/xla/tests/reshape_test.cc b/tensorflow/compiler/xla/tests/reshape_test.cc
index bb7160e3a0..72c68f24a0 100644
--- a/tensorflow/compiler/xla/tests/reshape_test.cc
+++ b/tensorflow/compiler/xla/tests/reshape_test.cc
@@ -47,7 +47,7 @@ class ReshapeTest : public ClientLibraryTestBase {
 };
 
 // Collapses 2-dimensional pseudo-scalar (single-element array) to 1 dimension.
-XLA_TEST_F(ReshapeTest, Trivial1x1) {
+XLA_TEST_F(ReshapeTest, CollapseTrivial1x1) {
   ComputationBuilder builder(client_, TestName());
   auto a = builder.ConstantR2<float>({{1.0}});
   builder.Collapse(/*operand=*/a, /*dimensions=*/{0, 1});
@@ -55,6 +55,22 @@ XLA_TEST_F(ReshapeTest, Trivial1x1) {
   ComputeAndCompareR1<float>(&builder, {1.0f}, {}, zero_error_spec_);
 }
 
+XLA_TEST_F(ReshapeTest, CollapseTrivialR1EmptyDims) {
+  ComputationBuilder builder(client_, TestName());
+  auto a = builder.ConstantR1<float>({1.0});
+  builder.Collapse(/*operand=*/a, /*dimensions=*/{});
+
+  ComputeAndCompareR1<float>(&builder, {1.0f}, {}, zero_error_spec_);
+}
+
+XLA_TEST_F(ReshapeTest, CollapseTrivialR1OnlyDim) {
+  ComputationBuilder builder(client_, TestName());
+  auto a = builder.ConstantR1<float>({1.0});
+  builder.Collapse(/*operand=*/a, /*dimensions=*/{0});
+
+  ComputeAndCompareR1<float>(&builder, {1.0f}, {}, zero_error_spec_);
+}
+
 // Collapses 2-dimensional pseudo-scalar (single-element array) to scalar.
 XLA_TEST_F(ReshapeTest, SingleElementArrayToScalar) {
   ComputationBuilder builder(client_, TestName());
-- 
GitLab


From 32e044d333e85d535a27a3729ed836855383be1b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 6 Oct 2017 11:24:00 -0700
Subject: [PATCH 119/909] Fix stats_collector_ null pointer error.

PiperOrigin-RevId: 171318477
---
 tensorflow/core/common_runtime/executor.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/common_runtime/executor.cc b/tensorflow/core/common_runtime/executor.cc
index f57834cfbe..11e063d8d2 100644
--- a/tensorflow/core/common_runtime/executor.cc
+++ b/tensorflow/core/common_runtime/executor.cc
@@ -2008,7 +2008,7 @@ bool ExecutorState::NodeDone(const Status& s, const Node* node,
                              NodeExecStatsWrapper* stats,
                              TaggedNodeReadyQueue* inline_ready) {
   nodestats::SetAllEnd(stats);
-  if (!SetTimelineLabel(node, stats)) {
+  if (stats_collector_ != nullptr && !SetTimelineLabel(node, stats)) {
     // Only record non-transfer nodes.
     // Transfers 'stats' ownership to 'stats_collector_'.
     stats_collector_->Save(impl_->params_.device->name(), stats);
-- 
GitLab


From 549e651106e1e582dad0e8a6ea57b8f59ce95067 Mon Sep 17 00:00:00 2001
From: Jacques Pienaar <jpienaar@google.com>
Date: Fri, 6 Oct 2017 11:03:06 -0700
Subject: [PATCH 120/909] [TF2XLA] Update device name in convert and redo check
 that name parsing is correct.

* Update ConvertGraphToXla to use the new form for setting the assigned device name.
* Remove some stale comments.
* Revert workaround that allowed the requested device name to not be parsed.

PiperOrigin-RevId: 171314671
---
 tensorflow/compiler/xla/client/client.cc      |  1 -
 .../xla/client/computation_builder.cc         | 13 ----
 .../compiler/xla/client/computation_builder.h | 10 ---
 .../compiler/xla/service/shape_inference.cc   |  9 +--
 .../compiler/xla/tests/literal_test_util.cc   | 73 +++++--------------
 .../compiler/xla/tests/literal_test_util.h    |  2 -
 tensorflow/compiler/xla/tests/reshape_test.cc | 18 +----
 .../contrib/all_reduce/python/all_reduce.py   |  4 +-
 tensorflow/core/common_runtime/executor.cc    |  2 +-
 9 files changed, 23 insertions(+), 109 deletions(-)

diff --git a/tensorflow/compiler/xla/client/client.cc b/tensorflow/compiler/xla/client/client.cc
index 7db2ea79fb..387253617e 100644
--- a/tensorflow/compiler/xla/client/client.cc
+++ b/tensorflow/compiler/xla/client/client.cc
@@ -206,7 +206,6 @@ StatusOr<std::unique_ptr<GlobalData>> Client::Execute(
     *request.mutable_execution_options() = *execution_options;
   }
   for (GlobalData* argument : arguments) {
-    CHECK(argument != nullptr) << "Argument pointers must not be null.";
     *request.add_arguments() = argument->handle();
   }
 
diff --git a/tensorflow/compiler/xla/client/computation_builder.cc b/tensorflow/compiler/xla/client/computation_builder.cc
index 925dcd36c0..15a713513f 100644
--- a/tensorflow/compiler/xla/client/computation_builder.cc
+++ b/tensorflow/compiler/xla/client/computation_builder.cc
@@ -489,16 +489,6 @@ ComputationDataHandle ComputationBuilder::Collapse(
   }
   std::unique_ptr<Shape> original_shape = shape_or_status.ConsumeValueOrDie();
 
-  VLOG(3) << "original shape: " << ShapeUtil::HumanString(*original_shape);
-  VLOG(3) << "dims to collapse: "
-          << tensorflow::str_util::Join(dims_to_collapse, ",");
-
-  if (dims_to_collapse.size() <= 1) {
-    // Not collapsing anything, trivially we can return the operand versus
-    // enqueueing a trivial reshape.
-    return operand;
-  }
-
   std::vector<int64> new_sizes;
   for (int i = 0; i < ShapeUtil::Rank(*original_shape); ++i) {
     if (i <= dims_to_collapse.front() || i > dims_to_collapse.back()) {
@@ -508,9 +498,6 @@ ComputationDataHandle ComputationBuilder::Collapse(
     }
   }
 
-  VLOG(3) << "new sizes: [" << tensorflow::str_util::Join(new_sizes, ",")
-          << "]";
-
   return Reshape(operand, new_sizes);
 }
 
diff --git a/tensorflow/compiler/xla/client/computation_builder.h b/tensorflow/compiler/xla/client/computation_builder.h
index 7014685ea5..73972c1290 100644
--- a/tensorflow/compiler/xla/client/computation_builder.h
+++ b/tensorflow/compiler/xla/client/computation_builder.h
@@ -201,16 +201,6 @@ class ComputationBuilder {
   // {x=1024, y=32} by collapsing dims {0, 1, 2}. Collapsing dimensions must
   // be a consecutive, in-order subsequence of the operand dimensions.
   //
-  // Note that collapsing a single dimension does nothing:
-  //
-  //    {256} collapsing {0} => {256}
-  //    {1} collapsing {0} => {1}
-  //
-  // Collapsing multiple dimensions produces a single result dimension:
-  //
-  //    {256, 2} collapsing {0,1} => {512}
-  //    {256, 2, 3} collapsing {0,1} => {512, 3}
-  //
   // This could potentially cause data to be moved -- it provides a more
   // structured form of reshaping than an arbitrary Reshape operation.
   ComputationDataHandle Collapse(const ComputationDataHandle& operand,
diff --git a/tensorflow/compiler/xla/service/shape_inference.cc b/tensorflow/compiler/xla/service/shape_inference.cc
index 29221d2d29..ffd8018827 100644
--- a/tensorflow/compiler/xla/service/shape_inference.cc
+++ b/tensorflow/compiler/xla/service/shape_inference.cc
@@ -1894,16 +1894,11 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(
 
   Shape inferred_shape =
       ShapeUtil::MakeShape(operand.element_type(), new_sizes);
-  VLOG(3) << "Reshape inferred shape: "
-          << ShapeUtil::HumanString(inferred_shape);
 
   if (ShapeUtil::ElementsIn(operand) != ShapeUtil::ElementsIn(inferred_shape)) {
     return InvalidArgument(
-        "reshape operation has mismatched element counts: from=%lld (%s) "
-        "to=%lld (%s)",
-        ShapeUtil::ElementsIn(operand), ShapeUtil::HumanString(operand).c_str(),
-        ShapeUtil::ElementsIn(inferred_shape),
-        ShapeUtil::HumanString(inferred_shape).c_str());
+        "reshape operation has mismatched element counts: from=%lld to=%lld",
+        ShapeUtil::ElementsIn(operand), ShapeUtil::ElementsIn(inferred_shape));
   }
 
   std::vector<int64> indices(ShapeUtil::Rank(operand));
diff --git a/tensorflow/compiler/xla/tests/literal_test_util.cc b/tensorflow/compiler/xla/tests/literal_test_util.cc
index 2876a79dd8..061a4e190f 100644
--- a/tensorflow/compiler/xla/tests/literal_test_util.cc
+++ b/tensorflow/compiler/xla/tests/literal_test_util.cc
@@ -39,60 +39,30 @@ limitations under the License.
 
 namespace xla {
 
-/* static */ ::testing::AssertionResult LiteralTestUtil::EqualShapes(
-    const Shape& expected, const Shape& actual) {
-  if (ShapeUtil::IsTuple(expected) != ShapeUtil::IsTuple(actual)) {
-    return ::testing::AssertionFailure()
-           << "tupleness-mismatch! want: " << ShapeUtil::HumanString(expected)
-           << " got: " << ShapeUtil::HumanString(actual);
-  }
+/* static */ void LiteralTestUtil::AssertEqualShapes(const Shape& expected,
+                                                     const Shape& actual) {
+  ASSERT_EQ(ShapeUtil::IsTuple(expected), ShapeUtil::IsTuple(actual));
   if (ShapeUtil::IsTuple(expected)) {
-    if (ShapeUtil::TupleElementCount(expected) !=
-        ShapeUtil::TupleElementCount(actual)) {
-      return ::testing::AssertionFailure()
-             << "want tuple element count: "
-             << ShapeUtil::TupleElementCount(expected)
-             << " got tuple element count: "
-             << ShapeUtil::TupleElementCount(actual);
-    }
+    ASSERT_EQ(ShapeUtil::TupleElementCount(expected),
+              ShapeUtil::TupleElementCount(actual));
     for (int i = 0; i < expected.tuple_shapes_size(); ++i) {
-      ::testing::AssertionResult result =
-          EqualShapes(expected.tuple_shapes(i), actual.tuple_shapes(i));
-      if (!result) {
-        return result;
-      }
+      AssertEqualShapes(expected.tuple_shapes(i), actual.tuple_shapes(i));
     }
   } else {
-    if (ShapeUtil::Rank(expected) != ShapeUtil::Rank(actual)) {
-      return ::testing::AssertionFailure()
-             << "want rank of: " << ShapeUtil::HumanString(expected)
-             << " got rank of: " << ShapeUtil::HumanString(actual);
-    }
-    if (expected.element_type() != actual.element_type()) {
-      return ::testing::AssertionFailure()
-             << PrimitiveType_Name(expected.element_type()) << " vs "
-             << PrimitiveType_Name(actual.element_type());
-    }
-    if (expected.dimensions_size() != actual.dimensions_size()) {
-      return ::testing::AssertionFailure()
-             << "want dimensions_size " << expected.dimensions_size()
-             << " got dimensions_size " << actual.dimensions_size();
-    }
+    ASSERT_EQ(ShapeUtil::Rank(expected), ShapeUtil::Rank(actual))
+        << "want rank of: " << ShapeUtil::HumanString(expected)
+        << " got rank of: " << ShapeUtil::HumanString(actual);
+    ASSERT_EQ(expected.element_type(), actual.element_type())
+        << PrimitiveType_Name(expected.element_type()) << " vs "
+        << PrimitiveType_Name(actual.element_type());
+    ASSERT_EQ(expected.dimensions_size(), actual.dimensions_size());
     for (int i = 0; i < expected.dimensions_size(); ++i) {
-      if (expected.dimensions(i) != actual.dimensions(i)) {
-        return ::testing::AssertionFailure()
-               << "mismatch in dimension #" << i
-               << " expected: " << ShapeUtil::HumanString(expected)
-               << " actual: " << ShapeUtil::HumanString(actual);
-      }
+      ASSERT_EQ(expected.dimensions(i), actual.dimensions(i))
+          << "mismatch in dimension #" << i
+          << " expected: " << ShapeUtil::HumanString(expected)
+          << " actual: " << ShapeUtil::HumanString(actual);
     }
   }
-  return ::testing::AssertionSuccess();
-}
-
-/* static */ void LiteralTestUtil::AssertEqualShapes(const Shape& expected,
-                                                     const Shape& actual) {
-  ASSERT_TRUE(EqualShapes(expected, actual));
 }
 
 /* static */ void LiteralTestUtil::AssertEqualShapesAndLayouts(
@@ -295,14 +265,7 @@ class NearComparator {
     VLOG(1) << "actual:";
     XLA_VLOG_LINES(1, actual.ToString());
 
-    // If the shapes mismatch, we simply fail the expectation instead of
-    // printing out data, as it's a type error rather than a value error.
-    ::testing::AssertionResult equal_shapes =
-        LiteralTestUtil::EqualShapes(expected.shape(), actual.shape());
-    if (!equal_shapes) {
-      EXPECT_TRUE(equal_shapes);
-      return false;
-    }
+    LiteralTestUtil::AssertEqualShapes(expected.shape(), actual.shape());
 
     // Set up members used during the comparison.
     num_miscompares_ = 0;
diff --git a/tensorflow/compiler/xla/tests/literal_test_util.h b/tensorflow/compiler/xla/tests/literal_test_util.h
index 467d44b857..f645c4e8dc 100644
--- a/tensorflow/compiler/xla/tests/literal_test_util.h
+++ b/tensorflow/compiler/xla/tests/literal_test_util.h
@@ -50,8 +50,6 @@ class LiteralTestUtil {
  public:
   // Asserts that the given shapes have the same rank, dimension sizes, and
   // primitive types.
-  static ::testing::AssertionResult EqualShapes(const Shape& expected,
-                                                const Shape& actual);
   static void AssertEqualShapes(const Shape& expected, const Shape& actual);
 
   // Asserts that the provided shapes are equal as defined in AssertEqualShapes
diff --git a/tensorflow/compiler/xla/tests/reshape_test.cc b/tensorflow/compiler/xla/tests/reshape_test.cc
index 72c68f24a0..bb7160e3a0 100644
--- a/tensorflow/compiler/xla/tests/reshape_test.cc
+++ b/tensorflow/compiler/xla/tests/reshape_test.cc
@@ -47,7 +47,7 @@ class ReshapeTest : public ClientLibraryTestBase {
 };
 
 // Collapses 2-dimensional pseudo-scalar (single-element array) to 1 dimension.
-XLA_TEST_F(ReshapeTest, CollapseTrivial1x1) {
+XLA_TEST_F(ReshapeTest, Trivial1x1) {
   ComputationBuilder builder(client_, TestName());
   auto a = builder.ConstantR2<float>({{1.0}});
   builder.Collapse(/*operand=*/a, /*dimensions=*/{0, 1});
@@ -55,22 +55,6 @@ XLA_TEST_F(ReshapeTest, CollapseTrivial1x1) {
   ComputeAndCompareR1<float>(&builder, {1.0f}, {}, zero_error_spec_);
 }
 
-XLA_TEST_F(ReshapeTest, CollapseTrivialR1EmptyDims) {
-  ComputationBuilder builder(client_, TestName());
-  auto a = builder.ConstantR1<float>({1.0});
-  builder.Collapse(/*operand=*/a, /*dimensions=*/{});
-
-  ComputeAndCompareR1<float>(&builder, {1.0f}, {}, zero_error_spec_);
-}
-
-XLA_TEST_F(ReshapeTest, CollapseTrivialR1OnlyDim) {
-  ComputationBuilder builder(client_, TestName());
-  auto a = builder.ConstantR1<float>({1.0});
-  builder.Collapse(/*operand=*/a, /*dimensions=*/{0});
-
-  ComputeAndCompareR1<float>(&builder, {1.0f}, {}, zero_error_spec_);
-}
-
 // Collapses 2-dimensional pseudo-scalar (single-element array) to scalar.
 XLA_TEST_F(ReshapeTest, SingleElementArrayToScalar) {
   ComputationBuilder builder(client_, TestName());
diff --git a/tensorflow/contrib/all_reduce/python/all_reduce.py b/tensorflow/contrib/all_reduce/python/all_reduce.py
index 22d7633ce2..8e7f1791b8 100644
--- a/tensorflow/contrib/all_reduce/python/all_reduce.py
+++ b/tensorflow/contrib/all_reduce/python/all_reduce.py
@@ -762,8 +762,6 @@ def _reduce_non_singleton(input_tensors, red_f, un_op):
   if len(input_tensors) > 1:
     return red_f(input_tensors)
   else:
-    if not un_op:
-      return input_tensors
     output_tensors = []
     for t in input_tensors:
       with ops.colocate_with(t):
@@ -837,7 +835,7 @@ def _build_shuffle_hybrid(input_tensors, gather_devices, red_op, upper_level_f):
 
 
 def build_shuffle_then_ring(input_tensors, gather_devices, subdiv,
-                            red_n_op, red_op, un_op=None):
+                            red_n_op, red_op, un_op):
   """Construct hybrid of Shuffle within workers, Ring across workers."""
   def upper_builder(tensors):
     return build_ring_all_reduce(tensors, len(tensors), subdiv, [0],
diff --git a/tensorflow/core/common_runtime/executor.cc b/tensorflow/core/common_runtime/executor.cc
index 11e063d8d2..f57834cfbe 100644
--- a/tensorflow/core/common_runtime/executor.cc
+++ b/tensorflow/core/common_runtime/executor.cc
@@ -2008,7 +2008,7 @@ bool ExecutorState::NodeDone(const Status& s, const Node* node,
                              NodeExecStatsWrapper* stats,
                              TaggedNodeReadyQueue* inline_ready) {
   nodestats::SetAllEnd(stats);
-  if (stats_collector_ != nullptr && !SetTimelineLabel(node, stats)) {
+  if (!SetTimelineLabel(node, stats)) {
     // Only record non-transfer nodes.
     // Transfers 'stats' ownership to 'stats_collector_'.
     stats_collector_->Save(impl_->params_.device->name(), stats);
-- 
GitLab


From 84b579e1d14760fc2a313c8e1d7ca100f74945a1 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Fri, 6 Oct 2017 11:34:03 -0700
Subject: [PATCH 121/909] [XLA:CPU] Make EmitTargetAddressForOp return void
 (well, technically Status).

This is a general cleanup -- less repeated code -- but it's also part of
an effort to use IrArray more and llvm::Value less.  In particular, many
callsites would take the llvm::Value returned by EmitTargetAddressForOp
and create an IrArray out of it, but then never attach AA info to that
array.  Having this function return void forces you to call
GetIrArrayForOp(), which attaches the AA metadata appropriately.

This change also gets rid of an unused arg to EmitTargetAddressForOp.

PiperOrigin-RevId: 171320201
---
 tensorflow/compiler/xla/client/client.cc      |   1 +
 .../xla/client/computation_builder.cc         |  13 +
 .../compiler/xla/client/computation_builder.h |  10 +
 .../compiler/xla/service/cpu/ir_emitter.cc    | 242 ++++++------------
 .../compiler/xla/service/cpu/ir_emitter.h     |   9 +-
 .../compiler/xla/service/shape_inference.cc   |   9 +-
 .../compiler/xla/tests/literal_test_util.cc   |  73 ++++--
 .../compiler/xla/tests/literal_test_util.h    |   2 +
 tensorflow/compiler/xla/tests/reshape_test.cc |  18 +-
 .../contrib/all_reduce/python/all_reduce.py   |   4 +-
 tensorflow/core/common_runtime/executor.cc    |   2 +-
 11 files changed, 195 insertions(+), 188 deletions(-)

diff --git a/tensorflow/compiler/xla/client/client.cc b/tensorflow/compiler/xla/client/client.cc
index 387253617e..7db2ea79fb 100644
--- a/tensorflow/compiler/xla/client/client.cc
+++ b/tensorflow/compiler/xla/client/client.cc
@@ -206,6 +206,7 @@ StatusOr<std::unique_ptr<GlobalData>> Client::Execute(
     *request.mutable_execution_options() = *execution_options;
   }
   for (GlobalData* argument : arguments) {
+    CHECK(argument != nullptr) << "Argument pointers must not be null.";
     *request.add_arguments() = argument->handle();
   }
 
diff --git a/tensorflow/compiler/xla/client/computation_builder.cc b/tensorflow/compiler/xla/client/computation_builder.cc
index 15a713513f..925dcd36c0 100644
--- a/tensorflow/compiler/xla/client/computation_builder.cc
+++ b/tensorflow/compiler/xla/client/computation_builder.cc
@@ -489,6 +489,16 @@ ComputationDataHandle ComputationBuilder::Collapse(
   }
   std::unique_ptr<Shape> original_shape = shape_or_status.ConsumeValueOrDie();
 
+  VLOG(3) << "original shape: " << ShapeUtil::HumanString(*original_shape);
+  VLOG(3) << "dims to collapse: "
+          << tensorflow::str_util::Join(dims_to_collapse, ",");
+
+  if (dims_to_collapse.size() <= 1) {
+    // Not collapsing anything, trivially we can return the operand versus
+    // enqueueing a trivial reshape.
+    return operand;
+  }
+
   std::vector<int64> new_sizes;
   for (int i = 0; i < ShapeUtil::Rank(*original_shape); ++i) {
     if (i <= dims_to_collapse.front() || i > dims_to_collapse.back()) {
@@ -498,6 +508,9 @@ ComputationDataHandle ComputationBuilder::Collapse(
     }
   }
 
+  VLOG(3) << "new sizes: [" << tensorflow::str_util::Join(new_sizes, ",")
+          << "]";
+
   return Reshape(operand, new_sizes);
 }
 
diff --git a/tensorflow/compiler/xla/client/computation_builder.h b/tensorflow/compiler/xla/client/computation_builder.h
index 73972c1290..7014685ea5 100644
--- a/tensorflow/compiler/xla/client/computation_builder.h
+++ b/tensorflow/compiler/xla/client/computation_builder.h
@@ -201,6 +201,16 @@ class ComputationBuilder {
   // {x=1024, y=32} by collapsing dims {0, 1, 2}. Collapsing dimensions must
   // be a consecutive, in-order subsequence of the operand dimensions.
   //
+  // Note that collapsing a single dimension does nothing:
+  //
+  //    {256} collapsing {0} => {256}
+  //    {1} collapsing {0} => {1}
+  //
+  // Collapsing multiple dimensions produces a single result dimension:
+  //
+  //    {256, 2} collapsing {0,1} => {512}
+  //    {256, 2, 3} collapsing {0,1} => {512, 3}
+  //
   // This could potentially cause data to be moved -- it provides a more
   // structured form of reshaping than an arbitrary Reshape operation.
   ComputationDataHandle Collapse(const ComputationDataHandle& operand,
diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
index 4375f13a0e..e4fb7c0496 100644
--- a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
@@ -291,8 +291,7 @@ Status IrEmitter::HandleConstant(HloInstruction* constant,
 Status IrEmitter::HandleCopy(HloInstruction* copy) {
   if (ShapeUtil::IsTuple(copy->shape())) {
     // kCopy shallow copies a tuple so just memcpy the top-level buffer.
-    TF_ASSIGN_OR_RETURN(llvm::Value * copy_value, EmitTargetAddressForOp(copy));
-    emitted_value_[copy] = copy_value;
+    TF_RETURN_IF_ERROR(EmitTargetAddressForOp(copy));
     return EmitMemcpy(*(copy->operand(0)), *copy);
   } else {
     // Use the elemental emitter for non-tuple shapes.
@@ -395,9 +394,7 @@ Status IrEmitter::HandleSelect(HloInstruction* select, HloInstruction* pred,
   TF_RET_CHECK(pred->shape().element_type() == PRED);
 
   if (ShapeUtil::IsTuple(select->shape())) {
-    TF_ASSIGN_OR_RETURN(llvm::Value * output_address,
-                        EmitTargetAddressForOp(select));
-    emitted_value_[select] = output_address;
+    TF_RETURN_IF_ERROR(EmitTargetAddressForOp(select));
     llvm_ir::EmitTupleSelect(GetIrArrayForOp(select), GetIrArrayForOp(pred),
                              GetEmittedValueFor(on_true),
                              GetEmittedValueFor(on_false), &ir_builder_);
@@ -414,8 +411,8 @@ Status IrEmitter::HandleInfeed(HloInstruction* infeed) {
 
   // The infeed operation produces data (dequeued from the infeed queue) at this
   // address, which has been provided by buffer assignment.
-  TF_ASSIGN_OR_RETURN(llvm::Value * target_address,
-                      EmitTargetAddressForOp(infeed));
+  TF_RETURN_IF_ERROR(EmitTargetAddressForOp(infeed));
+  llvm_ir::IrArray infeed_array = GetIrArrayForOp(infeed);
 
   if (ShapeUtil::IsTuple(shape)) {
     TF_RET_CHECK(!ShapeUtil::IsNestedTuple(shape));
@@ -433,9 +430,9 @@ Status IrEmitter::HandleInfeed(HloInstruction* infeed) {
           ShapeUtil::GetTupleElementShape(shape, i);
 
       // Only the outer tuple buffer's target address is obtained from
-      // EmitTargetAddressForOp to handle the case when Infeed is the
-      // root instruction. Target addresses for internal elements can
-      // be obtained from EmitTempBufferPointer.
+      // GetEmittedValueFor, to handle the case when Infeed is the root
+      // instruction. Target addresses for internal elements can be obtained
+      // from EmitTempBufferPointer.
       llvm::Value* tuple_element_address =
           EmitTempBufferPointer(buffer, tuple_element_shape);
 
@@ -445,15 +442,12 @@ Status IrEmitter::HandleInfeed(HloInstruction* infeed) {
       tuple_element_addresses.push_back(tuple_element_address);
     }
 
-    llvm_ir::EmitTuple(llvm_ir::IrArray(target_address, shape),
-                       tuple_element_addresses, &ir_builder_);
+    llvm_ir::EmitTuple(infeed_array, tuple_element_addresses, &ir_builder_);
   } else {
-    TF_RETURN_IF_ERROR(
-        EmitXfeedTransfer(XfeedKind::kInfeed, shape, target_address));
+    TF_RETURN_IF_ERROR(EmitXfeedTransfer(XfeedKind::kInfeed, shape,
+                                         GetEmittedValueFor(infeed)));
   }
 
-  emitted_value_[infeed] = target_address;
-
   return Status::OK();
 }
 
@@ -567,15 +561,12 @@ Status IrEmitter::HandleSort(HloInstruction* sort, HloInstruction* operand) {
 Status IrEmitter::HandleTuple(
     HloInstruction* tuple,
     tensorflow::gtl::ArraySlice<HloInstruction*> operands) {
-  TF_ASSIGN_OR_RETURN(llvm::Value * target_address,
-                      EmitTargetAddressForOp(tuple));
+  TF_RETURN_IF_ERROR(EmitTargetAddressForOp(tuple));
   std::vector<llvm::Value*> base_ptrs;
   for (auto operand : operands) {
     base_ptrs.push_back(GetEmittedValueFor(operand));
   }
-  llvm_ir::EmitTuple(llvm_ir::IrArray(target_address, tuple->shape()),
-                     base_ptrs, &ir_builder_);
-  emitted_value_[tuple] = target_address;
+  llvm_ir::EmitTuple(GetIrArrayForOp(tuple), base_ptrs, &ir_builder_);
   return Status::OK();
 }
 
@@ -892,11 +883,8 @@ Status IrEmitter::HandleDot(HloInstruction* dot, HloInstruction* lhs,
   llvm_ir::IrArray lhs_array(GetIrArrayForOp(lhs));
   llvm_ir::IrArray rhs_array(GetIrArrayForOp(rhs));
 
-  Shape target_shape = dot->shape();
-  TF_ASSIGN_OR_RETURN(llvm::Value * target_address,
-                      EmitTargetAddressForOp(dot));
-  llvm_ir::IrArray target_array(target_address, target_shape);
-  AddAliasingInformationToIrArray(*dot, &target_array);
+  TF_RETURN_IF_ERROR(EmitTargetAddressForOp(dot));
+  llvm_ir::IrArray target_array = GetIrArrayForOp(dot);
 
   VLOG(2) << "HandleDot: ";
   VLOG(2) << "  lhs operand: "
@@ -907,13 +895,10 @@ Status IrEmitter::HandleDot(HloInstruction* dot, HloInstruction* lhs,
           << llvm_ir::DumpToString(*target_array.GetBasePointer());
 
   // Dot operation is complicated so we delegate to a helper class.
-  TF_RETURN_IF_ERROR(DotOpEmitter::EmitDotOperation(
+  return DotOpEmitter::EmitDotOperation(
       *dot, /*transpose_lhs=*/false, /*transpose_rhs=*/false, target_array,
       lhs_array, rhs_array, GetExecutableRunOptionsArgument(), &ir_builder_,
-      hlo_module_config_));
-
-  emitted_value_[dot] = target_address;
-  return Status::OK();
+      hlo_module_config_);
 }
 
 Status IrEmitter::HandleConvolution(HloInstruction* convolution,
@@ -941,8 +926,7 @@ Status IrEmitter::HandleConvolution(HloInstruction* convolution,
       bool one_dim_convolution = lhs_shape.dimensions_size() == 3;
       llvm::Value* lhs_address = GetEmittedValueFor(lhs);
       llvm::Value* rhs_address = GetEmittedValueFor(rhs);
-      TF_ASSIGN_OR_RETURN(llvm::Value * target_address,
-                          EmitTargetAddressForOp(convolution));
+      TF_RETURN_IF_ERROR(EmitTargetAddressForOp(convolution));
 
       const ConvolutionDimensionNumbers& dnums =
           convolution->convolution_dimension_numbers();
@@ -1024,35 +1008,33 @@ Status IrEmitter::HandleConvolution(HloInstruction* convolution,
       conv_func->setDoesNotThrow();
       conv_func->setOnlyAccessesArgMemory();
       ir_builder_.CreateCall(
-          conv_func,
-          {
-              GetExecutableRunOptionsArgument(),
-              ir_builder_.CreateBitCast(target_address, float_ptr_type),
-              ir_builder_.CreateBitCast(lhs_address, float_ptr_type),
-              ir_builder_.CreateBitCast(rhs_address, float_ptr_type),
-              ir_builder_.getInt64(input_batch),
-              ir_builder_.getInt64(input_rows),
-              ir_builder_.getInt64(input_cols),
-              ir_builder_.getInt64(input_channels),
-              ir_builder_.getInt64(kernel_rows),
-              ir_builder_.getInt64(kernel_cols),
-              ir_builder_.getInt64(kernel_channels),
-              ir_builder_.getInt64(kernel_filters),
-              ir_builder_.getInt64(output_rows),
-              ir_builder_.getInt64(output_cols),
-              ir_builder_.getInt64(row_stride),
-              ir_builder_.getInt64(col_stride),
-              ir_builder_.getInt64(padding_top),
-              ir_builder_.getInt64(padding_bottom),
-              ir_builder_.getInt64(padding_left),
-              ir_builder_.getInt64(padding_right),
-              ir_builder_.getInt64(lhs_row_dilation),
-              ir_builder_.getInt64(lhs_col_dilation),
-              ir_builder_.getInt64(rhs_row_dilation),
-              ir_builder_.getInt64(rhs_col_dilation),
-          });
-      target_address->setName(AsStringRef(IrName(convolution)));
-      emitted_value_[convolution] = target_address;
+          conv_func, {
+                         GetExecutableRunOptionsArgument(),
+                         ir_builder_.CreateBitCast(
+                             GetEmittedValueFor(convolution), float_ptr_type),
+                         ir_builder_.CreateBitCast(lhs_address, float_ptr_type),
+                         ir_builder_.CreateBitCast(rhs_address, float_ptr_type),
+                         ir_builder_.getInt64(input_batch),
+                         ir_builder_.getInt64(input_rows),
+                         ir_builder_.getInt64(input_cols),
+                         ir_builder_.getInt64(input_channels),
+                         ir_builder_.getInt64(kernel_rows),
+                         ir_builder_.getInt64(kernel_cols),
+                         ir_builder_.getInt64(kernel_channels),
+                         ir_builder_.getInt64(kernel_filters),
+                         ir_builder_.getInt64(output_rows),
+                         ir_builder_.getInt64(output_cols),
+                         ir_builder_.getInt64(row_stride),
+                         ir_builder_.getInt64(col_stride),
+                         ir_builder_.getInt64(padding_top),
+                         ir_builder_.getInt64(padding_bottom),
+                         ir_builder_.getInt64(padding_left),
+                         ir_builder_.getInt64(padding_right),
+                         ir_builder_.getInt64(lhs_row_dilation),
+                         ir_builder_.getInt64(lhs_col_dilation),
+                         ir_builder_.getInt64(rhs_row_dilation),
+                         ir_builder_.getInt64(rhs_col_dilation),
+                     });
 
       return Status::OK();
     }
@@ -1367,9 +1349,7 @@ Status IrEmitter::HandleBatchNormTraining(HloInstruction* batch_norm_training) {
           mean_array, &ir_builder_)
           .EmitLoop(IrName(batch_norm_training, "mean_var")));
 
-  TF_ASSIGN_OR_RETURN(llvm::Value * target_address,
-                      EmitTargetAddressForOp(batch_norm_training));
-
+  TF_RETURN_IF_ERROR(EmitTargetAddressForOp(batch_norm_training));
   TF_ASSIGN_OR_RETURN(
       const BufferAllocation::Slice slice,
       assignment_.GetUniqueSlice(batch_norm_training, /*index=*/{0}));
@@ -1425,11 +1405,8 @@ Status IrEmitter::HandleBatchNormTraining(HloInstruction* batch_norm_training) {
           target_array, &ir_builder_)
           .EmitLoop(IrName(batch_norm_training, "normalize")));
 
-  llvm_ir::EmitTuple(
-      llvm_ir::IrArray(target_address, batch_norm_training->shape()),
-      {normalized, mean, var}, &ir_builder_);
-  emitted_value_[batch_norm_training] = target_address;
-
+  llvm_ir::EmitTuple(GetIrArrayForOp(batch_norm_training),
+                     {normalized, mean, var}, &ir_builder_);
   return Status::OK();
 }
 
@@ -1789,6 +1766,7 @@ StatusOr<bool> IrEmitter::EmitVectorizedReduce(
   }
 
   CHECK(!ShapeUtil::IsTuple(reduce->shape()));
+  TF_RETURN_IF_ERROR(EmitTargetAddressForOp(reduce));
 
   // We know we're not reducing over the most minor dimension, which means we
   // can lower the reduction loop as:
@@ -1851,10 +1829,7 @@ StatusOr<bool> IrEmitter::EmitVectorizedReduce(
                             reduction_generator, array_index, vector_type,
                             init_value, arg, dimensions, element_alignment));
 
-    TF_ASSIGN_OR_RETURN(llvm::Value * target_address,
-                        EmitTargetAddressForOp(reduce));
-    llvm_ir::IrArray target_array(target_address, reduce->shape());
-    AddAliasingInformationToIrArray(*reduce, &target_array);
+    llvm_ir::IrArray target_array = GetIrArrayForOp(reduce);
     llvm::Value* output_address =
         target_array.EmitArrayElementAddress(array_index, &ir_builder_);
     EmitShardedVectorStore(output_address, accumulator, element_alignment,
@@ -1886,10 +1861,7 @@ StatusOr<bool> IrEmitter::EmitVectorizedReduce(
                             reduction_generator, array_index, vector_type,
                             init_value, arg, dimensions, element_alignment));
 
-    TF_ASSIGN_OR_RETURN(llvm::Value * target_address,
-                        EmitTargetAddressForOp(reduce));
-    llvm_ir::IrArray target_array(target_address, reduce->shape());
-    AddAliasingInformationToIrArray(*reduce, &target_array);
+    llvm_ir::IrArray target_array = GetIrArrayForOp(reduce);
     llvm::Value* output_address =
         target_array.EmitArrayElementAddress(array_index, &ir_builder_);
     EmitShardedVectorStore(output_address, accumulator, element_alignment,
@@ -1900,10 +1872,6 @@ StatusOr<bool> IrEmitter::EmitVectorizedReduce(
     ir_builder_.SetInsertPoint(outermost_loop_exit_block);
   }
 
-  TF_ASSIGN_OR_RETURN(llvm::Value * target_address,
-                      EmitTargetAddressForOp(reduce));
-
-  emitted_value_[reduce] = target_address;
   return true;
 }
 
@@ -2003,9 +1971,7 @@ Status IrEmitter::HandleSlice(HloInstruction* slice, HloInstruction* operand) {
     return DefaultAction(slice);
   }
 
-  TF_ASSIGN_OR_RETURN(llvm::Value * target_address,
-                      EmitTargetAddressForOp(slice));
-  emitted_value_[slice] = target_address;
+  TF_RETURN_IF_ERROR(EmitTargetAddressForOp(slice));
 
   if (ShapeUtil::HasZeroElements(slice->shape())) {
     return Status::OK();
@@ -2077,8 +2043,7 @@ Status IrEmitter::HandleSlice(HloInstruction* slice, HloInstruction* operand) {
     outer_dims.push_back(memcpy_dim);
   }
 
-  llvm_ir::IrArray target_array(target_address, slice->shape());
-  AddAliasingInformationToIrArray(*slice, &target_array);
+  llvm_ir::IrArray target_array = GetIrArrayForOp(slice);
 
   const int64 num_outer_loops = outer_dims.size();
   llvm_ir::ForLoopNest loops(IrName(slice), &ir_builder_);
@@ -2131,10 +2096,7 @@ Status IrEmitter::HandleDynamicSlice(HloInstruction* dynamic_slice,
                                      HloInstruction* operand,
                                      HloInstruction* /*start_indices*/) {
   if (ShapeUtil::IsScalar(dynamic_slice->shape())) {
-    TF_ASSIGN_OR_RETURN(llvm::Value * target_address,
-                        EmitTargetAddressForOp(dynamic_slice));
-    target_address->setName(AsStringRef(IrName(dynamic_slice)));
-    emitted_value_[dynamic_slice] = target_address;
+    TF_RETURN_IF_ERROR(EmitTargetAddressForOp(dynamic_slice));
     return EmitMemcpy(*operand, *dynamic_slice);
   }
   return DefaultAction(dynamic_slice);
@@ -2190,10 +2152,7 @@ Status IrEmitter::HandleDynamicUpdateSlice(HloInstruction* dynamic_update_slice,
                                            HloInstruction* update,
                                            HloInstruction* start_indices) {
   if (ShapeUtil::IsScalar(dynamic_update_slice->shape())) {
-    TF_ASSIGN_OR_RETURN(llvm::Value * target_address,
-                        EmitTargetAddressForOp(dynamic_update_slice));
-    target_address->setName(AsStringRef(IrName(dynamic_update_slice)));
-    emitted_value_[dynamic_update_slice] = target_address;
+    TF_RETURN_IF_ERROR(EmitTargetAddressForOp(dynamic_update_slice));
     return EmitMemcpy(*update, *dynamic_update_slice);
   } else if (CanUpdateDynamicSliceInPlace(assignment_, dynamic_update_slice)) {
     VLOG(2) << "Emitting HandleDynamicUpdateSlice in-place.";
@@ -2247,9 +2206,7 @@ Status IrEmitter::HandleDynamicUpdateSlice(HloInstruction* dynamic_update_slice,
         llvm_ir::LoopEmitter(loop_body_emitter, update->shape(), &ir_builder_)
             .EmitLoop(IrName(dynamic_update_slice, "in_place")));
 
-    TF_ASSIGN_OR_RETURN(llvm::Value * dynamic_update_slice_address,
-                        EmitTargetAddressForOp(dynamic_update_slice));
-    emitted_value_[dynamic_update_slice] = dynamic_update_slice_address;
+    TF_RETURN_IF_ERROR(EmitTargetAddressForOp(dynamic_update_slice));
     return Status::OK();
   }
   return DefaultAction(dynamic_update_slice);
@@ -2348,11 +2305,8 @@ Status IrEmitter::HandleFusion(HloInstruction* fusion) {
     llvm_ir::IrArray rhs_array(GetIrArrayForOp(rhs));
 
     Shape target_shape = fusion->shape();
-    TF_ASSIGN_OR_RETURN(llvm::Value * target_address,
-                        EmitTargetAddressForOp(fusion));
-    llvm_ir::IrArray target_array(target_address, target_shape);
-    AddAliasingInformationToIrArray(*fusion, &target_array);
-
+    TF_RETURN_IF_ERROR(EmitTargetAddressForOp(fusion));
+    llvm_ir::IrArray target_array = GetIrArrayForOp(fusion);
     VLOG(2) << "HandleFusion kTransposeDot: ";
     VLOG(2) << "  lhs operand: "
             << llvm_ir::DumpToString(*lhs_array.GetBasePointer());
@@ -2366,8 +2320,6 @@ Status IrEmitter::HandleFusion(HloInstruction* fusion) {
         *dot, dot->operand(0)->IsRank2Transpose(),
         dot->operand(1)->IsRank2Transpose(), target_array, lhs_array, rhs_array,
         GetExecutableRunOptionsArgument(), &ir_builder_, hlo_module_config_));
-
-    emitted_value_[fusion] = target_address;
     return Status::OK();
   } else if (fusion->fusion_kind() == HloInstruction::FusionKind::kLoop) {
     std::vector<llvm_ir::IrArray> parameter_arrays;
@@ -2393,14 +2345,9 @@ Status IrEmitter::HandleCall(HloInstruction* call) {
     parameter_addresses.push_back(GetEmittedValueFor(operand));
   }
 
-  TF_ASSIGN_OR_RETURN(llvm::Value * output_address,
-                      EmitTargetAddressForOp(call));
-  output_address->setName(AsStringRef(IrName(call)));
-
+  TF_RETURN_IF_ERROR(EmitTargetAddressForOp(call));
   EmitArrayFunctionCallInto(call_ir_function, parameter_addresses,
-                            output_address, computation->name());
-
-  emitted_value_[call] = output_address;
+                            emitted_value_[call], computation->name());
   return Status::OK();
 }
 
@@ -2429,17 +2376,13 @@ Status IrEmitter::HandleCustomCall(
               /*Params=*/{i8_ptr_type, operands_alloca->getType()},
               /*isVarArg=*/false)));
 
-  TF_ASSIGN_OR_RETURN(llvm::Value * output_address,
-                      EmitTargetAddressForOp(custom_call));
-  output_address->setName(AsStringRef(IrName(custom_call)));
-
-  auto* output_address_arg =
-      ir_builder_.CreatePointerCast(output_address, i8_ptr_type);
+  TF_RETURN_IF_ERROR(EmitTargetAddressForOp(custom_call));
+  auto* output_address_arg = ir_builder_.CreatePointerCast(
+      GetEmittedValueFor(custom_call), i8_ptr_type);
 
   ir_builder_.CreateCall(custom_call_ir_function,
                          {output_address_arg, operands_alloca});
 
-  emitted_value_[custom_call] = output_address;
   return Status::OK();
 }
 
@@ -2583,10 +2526,8 @@ StatusOr<bool> IrEmitter::EmitFastConcatenate(
   llvm::Type* i8_ptr_type = ir_builder_.getInt8PtrTy();
   llvm::Type* i8_type = ir_builder_.getInt8Ty();
 
-  TF_ASSIGN_OR_RETURN(llvm::Value * target_address,
-                      EmitTargetAddressForOp(concatenate));
-
-  llvm_ir::IrArray target_array(target_address, output_shape);
+  TF_RETURN_IF_ERROR(EmitTargetAddressForOp(concatenate));
+  llvm_ir::IrArray target_array = GetIrArrayForOp(concatenate);
 
   llvm_ir::ForLoopNest loops(IrName(concatenate), &ir_builder_);
   llvm_ir::IrArray::Index outer_dims_index =
@@ -2603,8 +2544,6 @@ StatusOr<bool> IrEmitter::EmitFastConcatenate(
   unsigned primitive_type_size =
       ShapeUtil::ByteSizeOfPrimitiveType(primitive_type);
 
-  AddAliasingInformationToIrArray(*concatenate, &target_array);
-
   // Contiguous subregions from each operand to the concatenate contribute to a
   // contiguous subregion in the target buffer starting at target_region_begin.
   llvm::Value* target_region_begin = ir_builder_.CreateBitCast(
@@ -2647,8 +2586,6 @@ StatusOr<bool> IrEmitter::EmitFastConcatenate(
     SetToFirstInsertPoint(loops.GetOuterLoopExitBasicBlock(), &ir_builder_);
   }
 
-  emitted_value_[concatenate] = target_address;
-
   return true;
 }
 
@@ -2842,15 +2779,6 @@ Status IrEmitter::Preprocess(HloInstruction* hlo) {
 }
 
 Status IrEmitter::Postprocess(HloInstruction* hlo) {
-  // Set the name of the emitted llvm::Value to IrName(hlo).  Outfeed and send
-  // the only ops that don't emit a value.
-  if (hlo->opcode() != HloOpcode::kOutfeed &&
-      hlo->opcode() != HloOpcode::kSend) {
-    auto it = emitted_value_.find(hlo);
-    CHECK(it != emitted_value_.end());
-    it->second->setName(AsStringRef(IrName(hlo)));
-  }
-
   if (auto* prof_counter = GetProfileCounterFor(hlo)) {
     profiling_state_.RecordCycleDelta(&ir_builder_, hlo, prof_counter);
   }
@@ -3027,10 +2955,10 @@ llvm::Value* IrEmitter::EmitArrayFunctionCall(
   return return_value_buffer;
 }
 
-StatusOr<llvm::Value*> IrEmitter::EmitTargetAddressForOp(
-    const HloInstruction* op, const ShapeIndex& shape_index) {
-  const Shape& target_shape = ShapeUtil::GetSubshape(op->shape(), shape_index);
-  if (op == op->parent()->root_instruction() && shape_index.empty()) {
+Status IrEmitter::EmitTargetAddressForOp(const HloInstruction* op) {
+  llvm::Value* addr;
+  const Shape& target_shape = op->shape();
+  if (op == op->parent()->root_instruction()) {
     // For the root node, we write directly to the output buffer of the
     // function.
     llvm::Argument* retval = GetResultArgument();
@@ -3040,15 +2968,18 @@ StatusOr<llvm::Value*> IrEmitter::EmitTargetAddressForOp(
       attr_builder.addDereferenceableAttr(ByteSizeOf(target_shape));
       retval->addAttrs(attr_builder);
     }
-    return ir_builder_.CreateBitCast(retval,
+    addr = ir_builder_.CreateBitCast(retval,
                                      IrShapeType(target_shape)->getPointerTo());
-  }
-
-  // For other nodes, we need the temporary buffer allocated for this node to
-  // write the result into.
-  TF_ASSIGN_OR_RETURN(const BufferAllocation::Slice slice,
-                      assignment_.GetUniqueTopLevelSlice(op));
-  return EmitTempBufferPointer(slice, target_shape);
+  } else {
+    // For other nodes, we need the temporary buffer allocated for this node to
+    // write the result into.
+    TF_ASSIGN_OR_RETURN(const BufferAllocation::Slice slice,
+                        assignment_.GetUniqueTopLevelSlice(op));
+    addr = EmitTempBufferPointer(slice, target_shape);
+  }
+  addr->setName(AsStringRef(IrName(op)));
+  emitted_value_[op] = addr;
+  return Status::OK();
 }
 
 Status IrEmitter::EmitTargetElementLoop(
@@ -3062,12 +2993,9 @@ Status IrEmitter::EmitTargetElementLoop(
     const llvm_ir::ElementGenerator& element_generator) {
   VLOG(2) << "EmitTargetElementLoop: " << target_op->ToString();
 
-  // target_address will hold the address of the target buffer we will write the
-  // result of the computation into.
   const Shape& target_shape = target_op->shape();
-  TF_ASSIGN_OR_RETURN(llvm::Value * target_address,
-                      EmitTargetAddressForOp(target_op));
-  VLOG(2) << "  target address: " << llvm_ir::DumpToString(*target_address);
+  TF_RETURN_IF_ERROR(EmitTargetAddressForOp(target_op));
+  llvm_ir::IrArray target_array = GetIrArrayForOp(target_op);
 
   if (target_op->IsMultiOutputFusion()) {
     // For multiple outputs fusion, we need to emit each operand and the root.
@@ -3090,13 +3018,9 @@ Status IrEmitter::EmitTargetElementLoop(
     for (int64 i = 0; i < output_arrays.size(); ++i) {
       tuple_operand_ptrs.push_back(output_arrays[i].GetBasePointer());
     }
-    llvm_ir::EmitTuple(llvm_ir::IrArray(target_address, target_shape),
-                       tuple_operand_ptrs, &ir_builder_);
+    llvm_ir::EmitTuple(target_array, tuple_operand_ptrs, &ir_builder_);
 
   } else {
-    llvm_ir::IrArray target_array(target_address, target_shape);
-    AddAliasingInformationToIrArray(*target_op, &target_array);
-
     if (ShouldEmitParallelLoopFor(*target_op)) {
       TF_RETURN_IF_ERROR(EmitParallelTargetElementLoop(
           target_shape, element_generator, IrName(target_op), &target_array));
@@ -3106,8 +3030,6 @@ Status IrEmitter::EmitTargetElementLoop(
               .EmitLoop(IrName(target_op)));
     }
   }
-
-  emitted_value_[target_op] = target_address;
   return Status::OK();
 }
 
diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.h b/tensorflow/compiler/xla/service/cpu/ir_emitter.h
index 05663b6038..fd9ee71799 100644
--- a/tensorflow/compiler/xla/service/cpu/ir_emitter.h
+++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.h
@@ -353,11 +353,10 @@ class IrEmitter : public DfsHloVisitorWithDefault {
   Status EmitMemcpy(const HloInstruction& source,
                     const HloInstruction& destination);
 
-  // Emit IR to compute the target address of the buffer for the given op.
-  // The returned Value is a pointer to a IR type that represents the op's
-  // element type.
-  StatusOr<llvm::Value*> EmitTargetAddressForOp(
-      const HloInstruction* op, const ShapeIndex& shape_index = {});
+  // Emits IR to compute the target address of the buffer for the given op.
+  // After calling this function, you can get a pointer to this buffer by
+  // calling GetIrArrayForOp or GetEmittedValueFor.
+  Status EmitTargetAddressForOp(const HloInstruction* op);
 
   // Structurizes "array_elements" into an MD array that represents "shape".
   // This is a recursive function, and "dimension_index" indicates the index of
diff --git a/tensorflow/compiler/xla/service/shape_inference.cc b/tensorflow/compiler/xla/service/shape_inference.cc
index ffd8018827..29221d2d29 100644
--- a/tensorflow/compiler/xla/service/shape_inference.cc
+++ b/tensorflow/compiler/xla/service/shape_inference.cc
@@ -1894,11 +1894,16 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(
 
   Shape inferred_shape =
       ShapeUtil::MakeShape(operand.element_type(), new_sizes);
+  VLOG(3) << "Reshape inferred shape: "
+          << ShapeUtil::HumanString(inferred_shape);
 
   if (ShapeUtil::ElementsIn(operand) != ShapeUtil::ElementsIn(inferred_shape)) {
     return InvalidArgument(
-        "reshape operation has mismatched element counts: from=%lld to=%lld",
-        ShapeUtil::ElementsIn(operand), ShapeUtil::ElementsIn(inferred_shape));
+        "reshape operation has mismatched element counts: from=%lld (%s) "
+        "to=%lld (%s)",
+        ShapeUtil::ElementsIn(operand), ShapeUtil::HumanString(operand).c_str(),
+        ShapeUtil::ElementsIn(inferred_shape),
+        ShapeUtil::HumanString(inferred_shape).c_str());
   }
 
   std::vector<int64> indices(ShapeUtil::Rank(operand));
diff --git a/tensorflow/compiler/xla/tests/literal_test_util.cc b/tensorflow/compiler/xla/tests/literal_test_util.cc
index 061a4e190f..2876a79dd8 100644
--- a/tensorflow/compiler/xla/tests/literal_test_util.cc
+++ b/tensorflow/compiler/xla/tests/literal_test_util.cc
@@ -39,30 +39,60 @@ limitations under the License.
 
 namespace xla {
 
-/* static */ void LiteralTestUtil::AssertEqualShapes(const Shape& expected,
-                                                     const Shape& actual) {
-  ASSERT_EQ(ShapeUtil::IsTuple(expected), ShapeUtil::IsTuple(actual));
+/* static */ ::testing::AssertionResult LiteralTestUtil::EqualShapes(
+    const Shape& expected, const Shape& actual) {
+  if (ShapeUtil::IsTuple(expected) != ShapeUtil::IsTuple(actual)) {
+    return ::testing::AssertionFailure()
+           << "tupleness-mismatch! want: " << ShapeUtil::HumanString(expected)
+           << " got: " << ShapeUtil::HumanString(actual);
+  }
   if (ShapeUtil::IsTuple(expected)) {
-    ASSERT_EQ(ShapeUtil::TupleElementCount(expected),
-              ShapeUtil::TupleElementCount(actual));
+    if (ShapeUtil::TupleElementCount(expected) !=
+        ShapeUtil::TupleElementCount(actual)) {
+      return ::testing::AssertionFailure()
+             << "want tuple element count: "
+             << ShapeUtil::TupleElementCount(expected)
+             << " got tuple element count: "
+             << ShapeUtil::TupleElementCount(actual);
+    }
     for (int i = 0; i < expected.tuple_shapes_size(); ++i) {
-      AssertEqualShapes(expected.tuple_shapes(i), actual.tuple_shapes(i));
+      ::testing::AssertionResult result =
+          EqualShapes(expected.tuple_shapes(i), actual.tuple_shapes(i));
+      if (!result) {
+        return result;
+      }
     }
   } else {
-    ASSERT_EQ(ShapeUtil::Rank(expected), ShapeUtil::Rank(actual))
-        << "want rank of: " << ShapeUtil::HumanString(expected)
-        << " got rank of: " << ShapeUtil::HumanString(actual);
-    ASSERT_EQ(expected.element_type(), actual.element_type())
-        << PrimitiveType_Name(expected.element_type()) << " vs "
-        << PrimitiveType_Name(actual.element_type());
-    ASSERT_EQ(expected.dimensions_size(), actual.dimensions_size());
+    if (ShapeUtil::Rank(expected) != ShapeUtil::Rank(actual)) {
+      return ::testing::AssertionFailure()
+             << "want rank of: " << ShapeUtil::HumanString(expected)
+             << " got rank of: " << ShapeUtil::HumanString(actual);
+    }
+    if (expected.element_type() != actual.element_type()) {
+      return ::testing::AssertionFailure()
+             << PrimitiveType_Name(expected.element_type()) << " vs "
+             << PrimitiveType_Name(actual.element_type());
+    }
+    if (expected.dimensions_size() != actual.dimensions_size()) {
+      return ::testing::AssertionFailure()
+             << "want dimensions_size " << expected.dimensions_size()
+             << " got dimensions_size " << actual.dimensions_size();
+    }
     for (int i = 0; i < expected.dimensions_size(); ++i) {
-      ASSERT_EQ(expected.dimensions(i), actual.dimensions(i))
-          << "mismatch in dimension #" << i
-          << " expected: " << ShapeUtil::HumanString(expected)
-          << " actual: " << ShapeUtil::HumanString(actual);
+      if (expected.dimensions(i) != actual.dimensions(i)) {
+        return ::testing::AssertionFailure()
+               << "mismatch in dimension #" << i
+               << " expected: " << ShapeUtil::HumanString(expected)
+               << " actual: " << ShapeUtil::HumanString(actual);
+      }
     }
   }
+  return ::testing::AssertionSuccess();
+}
+
+/* static */ void LiteralTestUtil::AssertEqualShapes(const Shape& expected,
+                                                     const Shape& actual) {
+  ASSERT_TRUE(EqualShapes(expected, actual));
 }
 
 /* static */ void LiteralTestUtil::AssertEqualShapesAndLayouts(
@@ -265,7 +295,14 @@ class NearComparator {
     VLOG(1) << "actual:";
     XLA_VLOG_LINES(1, actual.ToString());
 
-    LiteralTestUtil::AssertEqualShapes(expected.shape(), actual.shape());
+    // If the shapes mismatch, we simply fail the expectation instead of
+    // printing out data, as it's a type error rather than a value error.
+    ::testing::AssertionResult equal_shapes =
+        LiteralTestUtil::EqualShapes(expected.shape(), actual.shape());
+    if (!equal_shapes) {
+      EXPECT_TRUE(equal_shapes);
+      return false;
+    }
 
     // Set up members used during the comparison.
     num_miscompares_ = 0;
diff --git a/tensorflow/compiler/xla/tests/literal_test_util.h b/tensorflow/compiler/xla/tests/literal_test_util.h
index f645c4e8dc..467d44b857 100644
--- a/tensorflow/compiler/xla/tests/literal_test_util.h
+++ b/tensorflow/compiler/xla/tests/literal_test_util.h
@@ -50,6 +50,8 @@ class LiteralTestUtil {
  public:
   // Asserts that the given shapes have the same rank, dimension sizes, and
   // primitive types.
+  static ::testing::AssertionResult EqualShapes(const Shape& expected,
+                                                const Shape& actual);
   static void AssertEqualShapes(const Shape& expected, const Shape& actual);
 
   // Asserts that the provided shapes are equal as defined in AssertEqualShapes
diff --git a/tensorflow/compiler/xla/tests/reshape_test.cc b/tensorflow/compiler/xla/tests/reshape_test.cc
index bb7160e3a0..72c68f24a0 100644
--- a/tensorflow/compiler/xla/tests/reshape_test.cc
+++ b/tensorflow/compiler/xla/tests/reshape_test.cc
@@ -47,7 +47,7 @@ class ReshapeTest : public ClientLibraryTestBase {
 };
 
 // Collapses 2-dimensional pseudo-scalar (single-element array) to 1 dimension.
-XLA_TEST_F(ReshapeTest, Trivial1x1) {
+XLA_TEST_F(ReshapeTest, CollapseTrivial1x1) {
   ComputationBuilder builder(client_, TestName());
   auto a = builder.ConstantR2<float>({{1.0}});
   builder.Collapse(/*operand=*/a, /*dimensions=*/{0, 1});
@@ -55,6 +55,22 @@ XLA_TEST_F(ReshapeTest, Trivial1x1) {
   ComputeAndCompareR1<float>(&builder, {1.0f}, {}, zero_error_spec_);
 }
 
+XLA_TEST_F(ReshapeTest, CollapseTrivialR1EmptyDims) {
+  ComputationBuilder builder(client_, TestName());
+  auto a = builder.ConstantR1<float>({1.0});
+  builder.Collapse(/*operand=*/a, /*dimensions=*/{});
+
+  ComputeAndCompareR1<float>(&builder, {1.0f}, {}, zero_error_spec_);
+}
+
+XLA_TEST_F(ReshapeTest, CollapseTrivialR1OnlyDim) {
+  ComputationBuilder builder(client_, TestName());
+  auto a = builder.ConstantR1<float>({1.0});
+  builder.Collapse(/*operand=*/a, /*dimensions=*/{0});
+
+  ComputeAndCompareR1<float>(&builder, {1.0f}, {}, zero_error_spec_);
+}
+
 // Collapses 2-dimensional pseudo-scalar (single-element array) to scalar.
 XLA_TEST_F(ReshapeTest, SingleElementArrayToScalar) {
   ComputationBuilder builder(client_, TestName());
diff --git a/tensorflow/contrib/all_reduce/python/all_reduce.py b/tensorflow/contrib/all_reduce/python/all_reduce.py
index 8e7f1791b8..22d7633ce2 100644
--- a/tensorflow/contrib/all_reduce/python/all_reduce.py
+++ b/tensorflow/contrib/all_reduce/python/all_reduce.py
@@ -762,6 +762,8 @@ def _reduce_non_singleton(input_tensors, red_f, un_op):
   if len(input_tensors) > 1:
     return red_f(input_tensors)
   else:
+    if not un_op:
+      return input_tensors
     output_tensors = []
     for t in input_tensors:
       with ops.colocate_with(t):
@@ -835,7 +837,7 @@ def _build_shuffle_hybrid(input_tensors, gather_devices, red_op, upper_level_f):
 
 
 def build_shuffle_then_ring(input_tensors, gather_devices, subdiv,
-                            red_n_op, red_op, un_op):
+                            red_n_op, red_op, un_op=None):
   """Construct hybrid of Shuffle within workers, Ring across workers."""
   def upper_builder(tensors):
     return build_ring_all_reduce(tensors, len(tensors), subdiv, [0],
diff --git a/tensorflow/core/common_runtime/executor.cc b/tensorflow/core/common_runtime/executor.cc
index f57834cfbe..11e063d8d2 100644
--- a/tensorflow/core/common_runtime/executor.cc
+++ b/tensorflow/core/common_runtime/executor.cc
@@ -2008,7 +2008,7 @@ bool ExecutorState::NodeDone(const Status& s, const Node* node,
                              NodeExecStatsWrapper* stats,
                              TaggedNodeReadyQueue* inline_ready) {
   nodestats::SetAllEnd(stats);
-  if (!SetTimelineLabel(node, stats)) {
+  if (stats_collector_ != nullptr && !SetTimelineLabel(node, stats)) {
     // Only record non-transfer nodes.
     // Transfers 'stats' ownership to 'stats_collector_'.
     stats_collector_->Save(impl_->params_.device->name(), stats);
-- 
GitLab


From af6e00f7c661c7d93bacfc3adc40d17f0faeb9b4 Mon Sep 17 00:00:00 2001
From: James Qin <jamesqin@google.com>
Date: Fri, 6 Oct 2017 11:04:52 -0700
Subject: [PATCH 122/909] Fix a minor issue w/ allreduce

PiperOrigin-RevId: 171314944
---
 tensorflow/compiler/xla/client/client.cc      |   1 -
 .../xla/client/computation_builder.cc         |  13 -
 .../compiler/xla/client/computation_builder.h |  10 -
 .../compiler/xla/service/cpu/ir_emitter.cc    | 242 ++++++++++++------
 .../compiler/xla/service/cpu/ir_emitter.h     |   9 +-
 .../compiler/xla/service/shape_inference.cc   |   9 +-
 .../compiler/xla/tests/literal_test_util.cc   |  73 ++----
 .../compiler/xla/tests/literal_test_util.h    |   2 -
 tensorflow/compiler/xla/tests/reshape_test.cc |  18 +-
 tensorflow/core/common_runtime/executor.cc    |   2 +-
 10 files changed, 187 insertions(+), 192 deletions(-)

diff --git a/tensorflow/compiler/xla/client/client.cc b/tensorflow/compiler/xla/client/client.cc
index 7db2ea79fb..387253617e 100644
--- a/tensorflow/compiler/xla/client/client.cc
+++ b/tensorflow/compiler/xla/client/client.cc
@@ -206,7 +206,6 @@ StatusOr<std::unique_ptr<GlobalData>> Client::Execute(
     *request.mutable_execution_options() = *execution_options;
   }
   for (GlobalData* argument : arguments) {
-    CHECK(argument != nullptr) << "Argument pointers must not be null.";
     *request.add_arguments() = argument->handle();
   }
 
diff --git a/tensorflow/compiler/xla/client/computation_builder.cc b/tensorflow/compiler/xla/client/computation_builder.cc
index 925dcd36c0..15a713513f 100644
--- a/tensorflow/compiler/xla/client/computation_builder.cc
+++ b/tensorflow/compiler/xla/client/computation_builder.cc
@@ -489,16 +489,6 @@ ComputationDataHandle ComputationBuilder::Collapse(
   }
   std::unique_ptr<Shape> original_shape = shape_or_status.ConsumeValueOrDie();
 
-  VLOG(3) << "original shape: " << ShapeUtil::HumanString(*original_shape);
-  VLOG(3) << "dims to collapse: "
-          << tensorflow::str_util::Join(dims_to_collapse, ",");
-
-  if (dims_to_collapse.size() <= 1) {
-    // Not collapsing anything, trivially we can return the operand versus
-    // enqueueing a trivial reshape.
-    return operand;
-  }
-
   std::vector<int64> new_sizes;
   for (int i = 0; i < ShapeUtil::Rank(*original_shape); ++i) {
     if (i <= dims_to_collapse.front() || i > dims_to_collapse.back()) {
@@ -508,9 +498,6 @@ ComputationDataHandle ComputationBuilder::Collapse(
     }
   }
 
-  VLOG(3) << "new sizes: [" << tensorflow::str_util::Join(new_sizes, ",")
-          << "]";
-
   return Reshape(operand, new_sizes);
 }
 
diff --git a/tensorflow/compiler/xla/client/computation_builder.h b/tensorflow/compiler/xla/client/computation_builder.h
index 7014685ea5..73972c1290 100644
--- a/tensorflow/compiler/xla/client/computation_builder.h
+++ b/tensorflow/compiler/xla/client/computation_builder.h
@@ -201,16 +201,6 @@ class ComputationBuilder {
   // {x=1024, y=32} by collapsing dims {0, 1, 2}. Collapsing dimensions must
   // be a consecutive, in-order subsequence of the operand dimensions.
   //
-  // Note that collapsing a single dimension does nothing:
-  //
-  //    {256} collapsing {0} => {256}
-  //    {1} collapsing {0} => {1}
-  //
-  // Collapsing multiple dimensions produces a single result dimension:
-  //
-  //    {256, 2} collapsing {0,1} => {512}
-  //    {256, 2, 3} collapsing {0,1} => {512, 3}
-  //
   // This could potentially cause data to be moved -- it provides a more
   // structured form of reshaping than an arbitrary Reshape operation.
   ComputationDataHandle Collapse(const ComputationDataHandle& operand,
diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
index e4fb7c0496..4375f13a0e 100644
--- a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
@@ -291,7 +291,8 @@ Status IrEmitter::HandleConstant(HloInstruction* constant,
 Status IrEmitter::HandleCopy(HloInstruction* copy) {
   if (ShapeUtil::IsTuple(copy->shape())) {
     // kCopy shallow copies a tuple so just memcpy the top-level buffer.
-    TF_RETURN_IF_ERROR(EmitTargetAddressForOp(copy));
+    TF_ASSIGN_OR_RETURN(llvm::Value * copy_value, EmitTargetAddressForOp(copy));
+    emitted_value_[copy] = copy_value;
     return EmitMemcpy(*(copy->operand(0)), *copy);
   } else {
     // Use the elemental emitter for non-tuple shapes.
@@ -394,7 +395,9 @@ Status IrEmitter::HandleSelect(HloInstruction* select, HloInstruction* pred,
   TF_RET_CHECK(pred->shape().element_type() == PRED);
 
   if (ShapeUtil::IsTuple(select->shape())) {
-    TF_RETURN_IF_ERROR(EmitTargetAddressForOp(select));
+    TF_ASSIGN_OR_RETURN(llvm::Value * output_address,
+                        EmitTargetAddressForOp(select));
+    emitted_value_[select] = output_address;
     llvm_ir::EmitTupleSelect(GetIrArrayForOp(select), GetIrArrayForOp(pred),
                              GetEmittedValueFor(on_true),
                              GetEmittedValueFor(on_false), &ir_builder_);
@@ -411,8 +414,8 @@ Status IrEmitter::HandleInfeed(HloInstruction* infeed) {
 
   // The infeed operation produces data (dequeued from the infeed queue) at this
   // address, which has been provided by buffer assignment.
-  TF_RETURN_IF_ERROR(EmitTargetAddressForOp(infeed));
-  llvm_ir::IrArray infeed_array = GetIrArrayForOp(infeed);
+  TF_ASSIGN_OR_RETURN(llvm::Value * target_address,
+                      EmitTargetAddressForOp(infeed));
 
   if (ShapeUtil::IsTuple(shape)) {
     TF_RET_CHECK(!ShapeUtil::IsNestedTuple(shape));
@@ -430,9 +433,9 @@ Status IrEmitter::HandleInfeed(HloInstruction* infeed) {
           ShapeUtil::GetTupleElementShape(shape, i);
 
       // Only the outer tuple buffer's target address is obtained from
-      // GetEmittedValueFor, to handle the case when Infeed is the root
-      // instruction. Target addresses for internal elements can be obtained
-      // from EmitTempBufferPointer.
+      // EmitTargetAddressForOp to handle the case when Infeed is the
+      // root instruction. Target addresses for internal elements can
+      // be obtained from EmitTempBufferPointer.
       llvm::Value* tuple_element_address =
           EmitTempBufferPointer(buffer, tuple_element_shape);
 
@@ -442,12 +445,15 @@ Status IrEmitter::HandleInfeed(HloInstruction* infeed) {
       tuple_element_addresses.push_back(tuple_element_address);
     }
 
-    llvm_ir::EmitTuple(infeed_array, tuple_element_addresses, &ir_builder_);
+    llvm_ir::EmitTuple(llvm_ir::IrArray(target_address, shape),
+                       tuple_element_addresses, &ir_builder_);
   } else {
-    TF_RETURN_IF_ERROR(EmitXfeedTransfer(XfeedKind::kInfeed, shape,
-                                         GetEmittedValueFor(infeed)));
+    TF_RETURN_IF_ERROR(
+        EmitXfeedTransfer(XfeedKind::kInfeed, shape, target_address));
   }
 
+  emitted_value_[infeed] = target_address;
+
   return Status::OK();
 }
 
@@ -561,12 +567,15 @@ Status IrEmitter::HandleSort(HloInstruction* sort, HloInstruction* operand) {
 Status IrEmitter::HandleTuple(
     HloInstruction* tuple,
     tensorflow::gtl::ArraySlice<HloInstruction*> operands) {
-  TF_RETURN_IF_ERROR(EmitTargetAddressForOp(tuple));
+  TF_ASSIGN_OR_RETURN(llvm::Value * target_address,
+                      EmitTargetAddressForOp(tuple));
   std::vector<llvm::Value*> base_ptrs;
   for (auto operand : operands) {
     base_ptrs.push_back(GetEmittedValueFor(operand));
   }
-  llvm_ir::EmitTuple(GetIrArrayForOp(tuple), base_ptrs, &ir_builder_);
+  llvm_ir::EmitTuple(llvm_ir::IrArray(target_address, tuple->shape()),
+                     base_ptrs, &ir_builder_);
+  emitted_value_[tuple] = target_address;
   return Status::OK();
 }
 
@@ -883,8 +892,11 @@ Status IrEmitter::HandleDot(HloInstruction* dot, HloInstruction* lhs,
   llvm_ir::IrArray lhs_array(GetIrArrayForOp(lhs));
   llvm_ir::IrArray rhs_array(GetIrArrayForOp(rhs));
 
-  TF_RETURN_IF_ERROR(EmitTargetAddressForOp(dot));
-  llvm_ir::IrArray target_array = GetIrArrayForOp(dot);
+  Shape target_shape = dot->shape();
+  TF_ASSIGN_OR_RETURN(llvm::Value * target_address,
+                      EmitTargetAddressForOp(dot));
+  llvm_ir::IrArray target_array(target_address, target_shape);
+  AddAliasingInformationToIrArray(*dot, &target_array);
 
   VLOG(2) << "HandleDot: ";
   VLOG(2) << "  lhs operand: "
@@ -895,10 +907,13 @@ Status IrEmitter::HandleDot(HloInstruction* dot, HloInstruction* lhs,
           << llvm_ir::DumpToString(*target_array.GetBasePointer());
 
   // Dot operation is complicated so we delegate to a helper class.
-  return DotOpEmitter::EmitDotOperation(
+  TF_RETURN_IF_ERROR(DotOpEmitter::EmitDotOperation(
       *dot, /*transpose_lhs=*/false, /*transpose_rhs=*/false, target_array,
       lhs_array, rhs_array, GetExecutableRunOptionsArgument(), &ir_builder_,
-      hlo_module_config_);
+      hlo_module_config_));
+
+  emitted_value_[dot] = target_address;
+  return Status::OK();
 }
 
 Status IrEmitter::HandleConvolution(HloInstruction* convolution,
@@ -926,7 +941,8 @@ Status IrEmitter::HandleConvolution(HloInstruction* convolution,
       bool one_dim_convolution = lhs_shape.dimensions_size() == 3;
       llvm::Value* lhs_address = GetEmittedValueFor(lhs);
       llvm::Value* rhs_address = GetEmittedValueFor(rhs);
-      TF_RETURN_IF_ERROR(EmitTargetAddressForOp(convolution));
+      TF_ASSIGN_OR_RETURN(llvm::Value * target_address,
+                          EmitTargetAddressForOp(convolution));
 
       const ConvolutionDimensionNumbers& dnums =
           convolution->convolution_dimension_numbers();
@@ -1008,33 +1024,35 @@ Status IrEmitter::HandleConvolution(HloInstruction* convolution,
       conv_func->setDoesNotThrow();
       conv_func->setOnlyAccessesArgMemory();
       ir_builder_.CreateCall(
-          conv_func, {
-                         GetExecutableRunOptionsArgument(),
-                         ir_builder_.CreateBitCast(
-                             GetEmittedValueFor(convolution), float_ptr_type),
-                         ir_builder_.CreateBitCast(lhs_address, float_ptr_type),
-                         ir_builder_.CreateBitCast(rhs_address, float_ptr_type),
-                         ir_builder_.getInt64(input_batch),
-                         ir_builder_.getInt64(input_rows),
-                         ir_builder_.getInt64(input_cols),
-                         ir_builder_.getInt64(input_channels),
-                         ir_builder_.getInt64(kernel_rows),
-                         ir_builder_.getInt64(kernel_cols),
-                         ir_builder_.getInt64(kernel_channels),
-                         ir_builder_.getInt64(kernel_filters),
-                         ir_builder_.getInt64(output_rows),
-                         ir_builder_.getInt64(output_cols),
-                         ir_builder_.getInt64(row_stride),
-                         ir_builder_.getInt64(col_stride),
-                         ir_builder_.getInt64(padding_top),
-                         ir_builder_.getInt64(padding_bottom),
-                         ir_builder_.getInt64(padding_left),
-                         ir_builder_.getInt64(padding_right),
-                         ir_builder_.getInt64(lhs_row_dilation),
-                         ir_builder_.getInt64(lhs_col_dilation),
-                         ir_builder_.getInt64(rhs_row_dilation),
-                         ir_builder_.getInt64(rhs_col_dilation),
-                     });
+          conv_func,
+          {
+              GetExecutableRunOptionsArgument(),
+              ir_builder_.CreateBitCast(target_address, float_ptr_type),
+              ir_builder_.CreateBitCast(lhs_address, float_ptr_type),
+              ir_builder_.CreateBitCast(rhs_address, float_ptr_type),
+              ir_builder_.getInt64(input_batch),
+              ir_builder_.getInt64(input_rows),
+              ir_builder_.getInt64(input_cols),
+              ir_builder_.getInt64(input_channels),
+              ir_builder_.getInt64(kernel_rows),
+              ir_builder_.getInt64(kernel_cols),
+              ir_builder_.getInt64(kernel_channels),
+              ir_builder_.getInt64(kernel_filters),
+              ir_builder_.getInt64(output_rows),
+              ir_builder_.getInt64(output_cols),
+              ir_builder_.getInt64(row_stride),
+              ir_builder_.getInt64(col_stride),
+              ir_builder_.getInt64(padding_top),
+              ir_builder_.getInt64(padding_bottom),
+              ir_builder_.getInt64(padding_left),
+              ir_builder_.getInt64(padding_right),
+              ir_builder_.getInt64(lhs_row_dilation),
+              ir_builder_.getInt64(lhs_col_dilation),
+              ir_builder_.getInt64(rhs_row_dilation),
+              ir_builder_.getInt64(rhs_col_dilation),
+          });
+      target_address->setName(AsStringRef(IrName(convolution)));
+      emitted_value_[convolution] = target_address;
 
       return Status::OK();
     }
@@ -1349,7 +1367,9 @@ Status IrEmitter::HandleBatchNormTraining(HloInstruction* batch_norm_training) {
           mean_array, &ir_builder_)
           .EmitLoop(IrName(batch_norm_training, "mean_var")));
 
-  TF_RETURN_IF_ERROR(EmitTargetAddressForOp(batch_norm_training));
+  TF_ASSIGN_OR_RETURN(llvm::Value * target_address,
+                      EmitTargetAddressForOp(batch_norm_training));
+
   TF_ASSIGN_OR_RETURN(
       const BufferAllocation::Slice slice,
       assignment_.GetUniqueSlice(batch_norm_training, /*index=*/{0}));
@@ -1405,8 +1425,11 @@ Status IrEmitter::HandleBatchNormTraining(HloInstruction* batch_norm_training) {
           target_array, &ir_builder_)
           .EmitLoop(IrName(batch_norm_training, "normalize")));
 
-  llvm_ir::EmitTuple(GetIrArrayForOp(batch_norm_training),
-                     {normalized, mean, var}, &ir_builder_);
+  llvm_ir::EmitTuple(
+      llvm_ir::IrArray(target_address, batch_norm_training->shape()),
+      {normalized, mean, var}, &ir_builder_);
+  emitted_value_[batch_norm_training] = target_address;
+
   return Status::OK();
 }
 
@@ -1766,7 +1789,6 @@ StatusOr<bool> IrEmitter::EmitVectorizedReduce(
   }
 
   CHECK(!ShapeUtil::IsTuple(reduce->shape()));
-  TF_RETURN_IF_ERROR(EmitTargetAddressForOp(reduce));
 
   // We know we're not reducing over the most minor dimension, which means we
   // can lower the reduction loop as:
@@ -1829,7 +1851,10 @@ StatusOr<bool> IrEmitter::EmitVectorizedReduce(
                             reduction_generator, array_index, vector_type,
                             init_value, arg, dimensions, element_alignment));
 
-    llvm_ir::IrArray target_array = GetIrArrayForOp(reduce);
+    TF_ASSIGN_OR_RETURN(llvm::Value * target_address,
+                        EmitTargetAddressForOp(reduce));
+    llvm_ir::IrArray target_array(target_address, reduce->shape());
+    AddAliasingInformationToIrArray(*reduce, &target_array);
     llvm::Value* output_address =
         target_array.EmitArrayElementAddress(array_index, &ir_builder_);
     EmitShardedVectorStore(output_address, accumulator, element_alignment,
@@ -1861,7 +1886,10 @@ StatusOr<bool> IrEmitter::EmitVectorizedReduce(
                             reduction_generator, array_index, vector_type,
                             init_value, arg, dimensions, element_alignment));
 
-    llvm_ir::IrArray target_array = GetIrArrayForOp(reduce);
+    TF_ASSIGN_OR_RETURN(llvm::Value * target_address,
+                        EmitTargetAddressForOp(reduce));
+    llvm_ir::IrArray target_array(target_address, reduce->shape());
+    AddAliasingInformationToIrArray(*reduce, &target_array);
     llvm::Value* output_address =
         target_array.EmitArrayElementAddress(array_index, &ir_builder_);
     EmitShardedVectorStore(output_address, accumulator, element_alignment,
@@ -1872,6 +1900,10 @@ StatusOr<bool> IrEmitter::EmitVectorizedReduce(
     ir_builder_.SetInsertPoint(outermost_loop_exit_block);
   }
 
+  TF_ASSIGN_OR_RETURN(llvm::Value * target_address,
+                      EmitTargetAddressForOp(reduce));
+
+  emitted_value_[reduce] = target_address;
   return true;
 }
 
@@ -1971,7 +2003,9 @@ Status IrEmitter::HandleSlice(HloInstruction* slice, HloInstruction* operand) {
     return DefaultAction(slice);
   }
 
-  TF_RETURN_IF_ERROR(EmitTargetAddressForOp(slice));
+  TF_ASSIGN_OR_RETURN(llvm::Value * target_address,
+                      EmitTargetAddressForOp(slice));
+  emitted_value_[slice] = target_address;
 
   if (ShapeUtil::HasZeroElements(slice->shape())) {
     return Status::OK();
@@ -2043,7 +2077,8 @@ Status IrEmitter::HandleSlice(HloInstruction* slice, HloInstruction* operand) {
     outer_dims.push_back(memcpy_dim);
   }
 
-  llvm_ir::IrArray target_array = GetIrArrayForOp(slice);
+  llvm_ir::IrArray target_array(target_address, slice->shape());
+  AddAliasingInformationToIrArray(*slice, &target_array);
 
   const int64 num_outer_loops = outer_dims.size();
   llvm_ir::ForLoopNest loops(IrName(slice), &ir_builder_);
@@ -2096,7 +2131,10 @@ Status IrEmitter::HandleDynamicSlice(HloInstruction* dynamic_slice,
                                      HloInstruction* operand,
                                      HloInstruction* /*start_indices*/) {
   if (ShapeUtil::IsScalar(dynamic_slice->shape())) {
-    TF_RETURN_IF_ERROR(EmitTargetAddressForOp(dynamic_slice));
+    TF_ASSIGN_OR_RETURN(llvm::Value * target_address,
+                        EmitTargetAddressForOp(dynamic_slice));
+    target_address->setName(AsStringRef(IrName(dynamic_slice)));
+    emitted_value_[dynamic_slice] = target_address;
     return EmitMemcpy(*operand, *dynamic_slice);
   }
   return DefaultAction(dynamic_slice);
@@ -2152,7 +2190,10 @@ Status IrEmitter::HandleDynamicUpdateSlice(HloInstruction* dynamic_update_slice,
                                            HloInstruction* update,
                                            HloInstruction* start_indices) {
   if (ShapeUtil::IsScalar(dynamic_update_slice->shape())) {
-    TF_RETURN_IF_ERROR(EmitTargetAddressForOp(dynamic_update_slice));
+    TF_ASSIGN_OR_RETURN(llvm::Value * target_address,
+                        EmitTargetAddressForOp(dynamic_update_slice));
+    target_address->setName(AsStringRef(IrName(dynamic_update_slice)));
+    emitted_value_[dynamic_update_slice] = target_address;
     return EmitMemcpy(*update, *dynamic_update_slice);
   } else if (CanUpdateDynamicSliceInPlace(assignment_, dynamic_update_slice)) {
     VLOG(2) << "Emitting HandleDynamicUpdateSlice in-place.";
@@ -2206,7 +2247,9 @@ Status IrEmitter::HandleDynamicUpdateSlice(HloInstruction* dynamic_update_slice,
         llvm_ir::LoopEmitter(loop_body_emitter, update->shape(), &ir_builder_)
             .EmitLoop(IrName(dynamic_update_slice, "in_place")));
 
-    TF_RETURN_IF_ERROR(EmitTargetAddressForOp(dynamic_update_slice));
+    TF_ASSIGN_OR_RETURN(llvm::Value * dynamic_update_slice_address,
+                        EmitTargetAddressForOp(dynamic_update_slice));
+    emitted_value_[dynamic_update_slice] = dynamic_update_slice_address;
     return Status::OK();
   }
   return DefaultAction(dynamic_update_slice);
@@ -2305,8 +2348,11 @@ Status IrEmitter::HandleFusion(HloInstruction* fusion) {
     llvm_ir::IrArray rhs_array(GetIrArrayForOp(rhs));
 
     Shape target_shape = fusion->shape();
-    TF_RETURN_IF_ERROR(EmitTargetAddressForOp(fusion));
-    llvm_ir::IrArray target_array = GetIrArrayForOp(fusion);
+    TF_ASSIGN_OR_RETURN(llvm::Value * target_address,
+                        EmitTargetAddressForOp(fusion));
+    llvm_ir::IrArray target_array(target_address, target_shape);
+    AddAliasingInformationToIrArray(*fusion, &target_array);
+
     VLOG(2) << "HandleFusion kTransposeDot: ";
     VLOG(2) << "  lhs operand: "
             << llvm_ir::DumpToString(*lhs_array.GetBasePointer());
@@ -2320,6 +2366,8 @@ Status IrEmitter::HandleFusion(HloInstruction* fusion) {
         *dot, dot->operand(0)->IsRank2Transpose(),
         dot->operand(1)->IsRank2Transpose(), target_array, lhs_array, rhs_array,
         GetExecutableRunOptionsArgument(), &ir_builder_, hlo_module_config_));
+
+    emitted_value_[fusion] = target_address;
     return Status::OK();
   } else if (fusion->fusion_kind() == HloInstruction::FusionKind::kLoop) {
     std::vector<llvm_ir::IrArray> parameter_arrays;
@@ -2345,9 +2393,14 @@ Status IrEmitter::HandleCall(HloInstruction* call) {
     parameter_addresses.push_back(GetEmittedValueFor(operand));
   }
 
-  TF_RETURN_IF_ERROR(EmitTargetAddressForOp(call));
+  TF_ASSIGN_OR_RETURN(llvm::Value * output_address,
+                      EmitTargetAddressForOp(call));
+  output_address->setName(AsStringRef(IrName(call)));
+
   EmitArrayFunctionCallInto(call_ir_function, parameter_addresses,
-                            emitted_value_[call], computation->name());
+                            output_address, computation->name());
+
+  emitted_value_[call] = output_address;
   return Status::OK();
 }
 
@@ -2376,13 +2429,17 @@ Status IrEmitter::HandleCustomCall(
               /*Params=*/{i8_ptr_type, operands_alloca->getType()},
               /*isVarArg=*/false)));
 
-  TF_RETURN_IF_ERROR(EmitTargetAddressForOp(custom_call));
-  auto* output_address_arg = ir_builder_.CreatePointerCast(
-      GetEmittedValueFor(custom_call), i8_ptr_type);
+  TF_ASSIGN_OR_RETURN(llvm::Value * output_address,
+                      EmitTargetAddressForOp(custom_call));
+  output_address->setName(AsStringRef(IrName(custom_call)));
+
+  auto* output_address_arg =
+      ir_builder_.CreatePointerCast(output_address, i8_ptr_type);
 
   ir_builder_.CreateCall(custom_call_ir_function,
                          {output_address_arg, operands_alloca});
 
+  emitted_value_[custom_call] = output_address;
   return Status::OK();
 }
 
@@ -2526,8 +2583,10 @@ StatusOr<bool> IrEmitter::EmitFastConcatenate(
   llvm::Type* i8_ptr_type = ir_builder_.getInt8PtrTy();
   llvm::Type* i8_type = ir_builder_.getInt8Ty();
 
-  TF_RETURN_IF_ERROR(EmitTargetAddressForOp(concatenate));
-  llvm_ir::IrArray target_array = GetIrArrayForOp(concatenate);
+  TF_ASSIGN_OR_RETURN(llvm::Value * target_address,
+                      EmitTargetAddressForOp(concatenate));
+
+  llvm_ir::IrArray target_array(target_address, output_shape);
 
   llvm_ir::ForLoopNest loops(IrName(concatenate), &ir_builder_);
   llvm_ir::IrArray::Index outer_dims_index =
@@ -2544,6 +2603,8 @@ StatusOr<bool> IrEmitter::EmitFastConcatenate(
   unsigned primitive_type_size =
       ShapeUtil::ByteSizeOfPrimitiveType(primitive_type);
 
+  AddAliasingInformationToIrArray(*concatenate, &target_array);
+
   // Contiguous subregions from each operand to the concatenate contribute to a
   // contiguous subregion in the target buffer starting at target_region_begin.
   llvm::Value* target_region_begin = ir_builder_.CreateBitCast(
@@ -2586,6 +2647,8 @@ StatusOr<bool> IrEmitter::EmitFastConcatenate(
     SetToFirstInsertPoint(loops.GetOuterLoopExitBasicBlock(), &ir_builder_);
   }
 
+  emitted_value_[concatenate] = target_address;
+
   return true;
 }
 
@@ -2779,6 +2842,15 @@ Status IrEmitter::Preprocess(HloInstruction* hlo) {
 }
 
 Status IrEmitter::Postprocess(HloInstruction* hlo) {
+  // Set the name of the emitted llvm::Value to IrName(hlo).  Outfeed and send
+  // the only ops that don't emit a value.
+  if (hlo->opcode() != HloOpcode::kOutfeed &&
+      hlo->opcode() != HloOpcode::kSend) {
+    auto it = emitted_value_.find(hlo);
+    CHECK(it != emitted_value_.end());
+    it->second->setName(AsStringRef(IrName(hlo)));
+  }
+
   if (auto* prof_counter = GetProfileCounterFor(hlo)) {
     profiling_state_.RecordCycleDelta(&ir_builder_, hlo, prof_counter);
   }
@@ -2955,10 +3027,10 @@ llvm::Value* IrEmitter::EmitArrayFunctionCall(
   return return_value_buffer;
 }
 
-Status IrEmitter::EmitTargetAddressForOp(const HloInstruction* op) {
-  llvm::Value* addr;
-  const Shape& target_shape = op->shape();
-  if (op == op->parent()->root_instruction()) {
+StatusOr<llvm::Value*> IrEmitter::EmitTargetAddressForOp(
+    const HloInstruction* op, const ShapeIndex& shape_index) {
+  const Shape& target_shape = ShapeUtil::GetSubshape(op->shape(), shape_index);
+  if (op == op->parent()->root_instruction() && shape_index.empty()) {
     // For the root node, we write directly to the output buffer of the
     // function.
     llvm::Argument* retval = GetResultArgument();
@@ -2968,18 +3040,15 @@ Status IrEmitter::EmitTargetAddressForOp(const HloInstruction* op) {
       attr_builder.addDereferenceableAttr(ByteSizeOf(target_shape));
       retval->addAttrs(attr_builder);
     }
-    addr = ir_builder_.CreateBitCast(retval,
+    return ir_builder_.CreateBitCast(retval,
                                      IrShapeType(target_shape)->getPointerTo());
-  } else {
-    // For other nodes, we need the temporary buffer allocated for this node to
-    // write the result into.
-    TF_ASSIGN_OR_RETURN(const BufferAllocation::Slice slice,
-                        assignment_.GetUniqueTopLevelSlice(op));
-    addr = EmitTempBufferPointer(slice, target_shape);
-  }
-  addr->setName(AsStringRef(IrName(op)));
-  emitted_value_[op] = addr;
-  return Status::OK();
+  }
+
+  // For other nodes, we need the temporary buffer allocated for this node to
+  // write the result into.
+  TF_ASSIGN_OR_RETURN(const BufferAllocation::Slice slice,
+                      assignment_.GetUniqueTopLevelSlice(op));
+  return EmitTempBufferPointer(slice, target_shape);
 }
 
 Status IrEmitter::EmitTargetElementLoop(
@@ -2993,9 +3062,12 @@ Status IrEmitter::EmitTargetElementLoop(
     const llvm_ir::ElementGenerator& element_generator) {
   VLOG(2) << "EmitTargetElementLoop: " << target_op->ToString();
 
+  // target_address will hold the address of the target buffer we will write the
+  // result of the computation into.
   const Shape& target_shape = target_op->shape();
-  TF_RETURN_IF_ERROR(EmitTargetAddressForOp(target_op));
-  llvm_ir::IrArray target_array = GetIrArrayForOp(target_op);
+  TF_ASSIGN_OR_RETURN(llvm::Value * target_address,
+                      EmitTargetAddressForOp(target_op));
+  VLOG(2) << "  target address: " << llvm_ir::DumpToString(*target_address);
 
   if (target_op->IsMultiOutputFusion()) {
     // For multiple outputs fusion, we need to emit each operand and the root.
@@ -3018,9 +3090,13 @@ Status IrEmitter::EmitTargetElementLoop(
     for (int64 i = 0; i < output_arrays.size(); ++i) {
       tuple_operand_ptrs.push_back(output_arrays[i].GetBasePointer());
     }
-    llvm_ir::EmitTuple(target_array, tuple_operand_ptrs, &ir_builder_);
+    llvm_ir::EmitTuple(llvm_ir::IrArray(target_address, target_shape),
+                       tuple_operand_ptrs, &ir_builder_);
 
   } else {
+    llvm_ir::IrArray target_array(target_address, target_shape);
+    AddAliasingInformationToIrArray(*target_op, &target_array);
+
     if (ShouldEmitParallelLoopFor(*target_op)) {
       TF_RETURN_IF_ERROR(EmitParallelTargetElementLoop(
           target_shape, element_generator, IrName(target_op), &target_array));
@@ -3030,6 +3106,8 @@ Status IrEmitter::EmitTargetElementLoop(
               .EmitLoop(IrName(target_op)));
     }
   }
+
+  emitted_value_[target_op] = target_address;
   return Status::OK();
 }
 
diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.h b/tensorflow/compiler/xla/service/cpu/ir_emitter.h
index fd9ee71799..05663b6038 100644
--- a/tensorflow/compiler/xla/service/cpu/ir_emitter.h
+++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.h
@@ -353,10 +353,11 @@ class IrEmitter : public DfsHloVisitorWithDefault {
   Status EmitMemcpy(const HloInstruction& source,
                     const HloInstruction& destination);
 
-  // Emits IR to compute the target address of the buffer for the given op.
-  // After calling this function, you can get a pointer to this buffer by
-  // calling GetIrArrayForOp or GetEmittedValueFor.
-  Status EmitTargetAddressForOp(const HloInstruction* op);
+  // Emit IR to compute the target address of the buffer for the given op.
+  // The returned Value is a pointer to a IR type that represents the op's
+  // element type.
+  StatusOr<llvm::Value*> EmitTargetAddressForOp(
+      const HloInstruction* op, const ShapeIndex& shape_index = {});
 
   // Structurizes "array_elements" into an MD array that represents "shape".
   // This is a recursive function, and "dimension_index" indicates the index of
diff --git a/tensorflow/compiler/xla/service/shape_inference.cc b/tensorflow/compiler/xla/service/shape_inference.cc
index 29221d2d29..ffd8018827 100644
--- a/tensorflow/compiler/xla/service/shape_inference.cc
+++ b/tensorflow/compiler/xla/service/shape_inference.cc
@@ -1894,16 +1894,11 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(
 
   Shape inferred_shape =
       ShapeUtil::MakeShape(operand.element_type(), new_sizes);
-  VLOG(3) << "Reshape inferred shape: "
-          << ShapeUtil::HumanString(inferred_shape);
 
   if (ShapeUtil::ElementsIn(operand) != ShapeUtil::ElementsIn(inferred_shape)) {
     return InvalidArgument(
-        "reshape operation has mismatched element counts: from=%lld (%s) "
-        "to=%lld (%s)",
-        ShapeUtil::ElementsIn(operand), ShapeUtil::HumanString(operand).c_str(),
-        ShapeUtil::ElementsIn(inferred_shape),
-        ShapeUtil::HumanString(inferred_shape).c_str());
+        "reshape operation has mismatched element counts: from=%lld to=%lld",
+        ShapeUtil::ElementsIn(operand), ShapeUtil::ElementsIn(inferred_shape));
   }
 
   std::vector<int64> indices(ShapeUtil::Rank(operand));
diff --git a/tensorflow/compiler/xla/tests/literal_test_util.cc b/tensorflow/compiler/xla/tests/literal_test_util.cc
index 2876a79dd8..061a4e190f 100644
--- a/tensorflow/compiler/xla/tests/literal_test_util.cc
+++ b/tensorflow/compiler/xla/tests/literal_test_util.cc
@@ -39,60 +39,30 @@ limitations under the License.
 
 namespace xla {
 
-/* static */ ::testing::AssertionResult LiteralTestUtil::EqualShapes(
-    const Shape& expected, const Shape& actual) {
-  if (ShapeUtil::IsTuple(expected) != ShapeUtil::IsTuple(actual)) {
-    return ::testing::AssertionFailure()
-           << "tupleness-mismatch! want: " << ShapeUtil::HumanString(expected)
-           << " got: " << ShapeUtil::HumanString(actual);
-  }
+/* static */ void LiteralTestUtil::AssertEqualShapes(const Shape& expected,
+                                                     const Shape& actual) {
+  ASSERT_EQ(ShapeUtil::IsTuple(expected), ShapeUtil::IsTuple(actual));
   if (ShapeUtil::IsTuple(expected)) {
-    if (ShapeUtil::TupleElementCount(expected) !=
-        ShapeUtil::TupleElementCount(actual)) {
-      return ::testing::AssertionFailure()
-             << "want tuple element count: "
-             << ShapeUtil::TupleElementCount(expected)
-             << " got tuple element count: "
-             << ShapeUtil::TupleElementCount(actual);
-    }
+    ASSERT_EQ(ShapeUtil::TupleElementCount(expected),
+              ShapeUtil::TupleElementCount(actual));
     for (int i = 0; i < expected.tuple_shapes_size(); ++i) {
-      ::testing::AssertionResult result =
-          EqualShapes(expected.tuple_shapes(i), actual.tuple_shapes(i));
-      if (!result) {
-        return result;
-      }
+      AssertEqualShapes(expected.tuple_shapes(i), actual.tuple_shapes(i));
     }
   } else {
-    if (ShapeUtil::Rank(expected) != ShapeUtil::Rank(actual)) {
-      return ::testing::AssertionFailure()
-             << "want rank of: " << ShapeUtil::HumanString(expected)
-             << " got rank of: " << ShapeUtil::HumanString(actual);
-    }
-    if (expected.element_type() != actual.element_type()) {
-      return ::testing::AssertionFailure()
-             << PrimitiveType_Name(expected.element_type()) << " vs "
-             << PrimitiveType_Name(actual.element_type());
-    }
-    if (expected.dimensions_size() != actual.dimensions_size()) {
-      return ::testing::AssertionFailure()
-             << "want dimensions_size " << expected.dimensions_size()
-             << " got dimensions_size " << actual.dimensions_size();
-    }
+    ASSERT_EQ(ShapeUtil::Rank(expected), ShapeUtil::Rank(actual))
+        << "want rank of: " << ShapeUtil::HumanString(expected)
+        << " got rank of: " << ShapeUtil::HumanString(actual);
+    ASSERT_EQ(expected.element_type(), actual.element_type())
+        << PrimitiveType_Name(expected.element_type()) << " vs "
+        << PrimitiveType_Name(actual.element_type());
+    ASSERT_EQ(expected.dimensions_size(), actual.dimensions_size());
     for (int i = 0; i < expected.dimensions_size(); ++i) {
-      if (expected.dimensions(i) != actual.dimensions(i)) {
-        return ::testing::AssertionFailure()
-               << "mismatch in dimension #" << i
-               << " expected: " << ShapeUtil::HumanString(expected)
-               << " actual: " << ShapeUtil::HumanString(actual);
-      }
+      ASSERT_EQ(expected.dimensions(i), actual.dimensions(i))
+          << "mismatch in dimension #" << i
+          << " expected: " << ShapeUtil::HumanString(expected)
+          << " actual: " << ShapeUtil::HumanString(actual);
     }
   }
-  return ::testing::AssertionSuccess();
-}
-
-/* static */ void LiteralTestUtil::AssertEqualShapes(const Shape& expected,
-                                                     const Shape& actual) {
-  ASSERT_TRUE(EqualShapes(expected, actual));
 }
 
 /* static */ void LiteralTestUtil::AssertEqualShapesAndLayouts(
@@ -295,14 +265,7 @@ class NearComparator {
     VLOG(1) << "actual:";
     XLA_VLOG_LINES(1, actual.ToString());
 
-    // If the shapes mismatch, we simply fail the expectation instead of
-    // printing out data, as it's a type error rather than a value error.
-    ::testing::AssertionResult equal_shapes =
-        LiteralTestUtil::EqualShapes(expected.shape(), actual.shape());
-    if (!equal_shapes) {
-      EXPECT_TRUE(equal_shapes);
-      return false;
-    }
+    LiteralTestUtil::AssertEqualShapes(expected.shape(), actual.shape());
 
     // Set up members used during the comparison.
     num_miscompares_ = 0;
diff --git a/tensorflow/compiler/xla/tests/literal_test_util.h b/tensorflow/compiler/xla/tests/literal_test_util.h
index 467d44b857..f645c4e8dc 100644
--- a/tensorflow/compiler/xla/tests/literal_test_util.h
+++ b/tensorflow/compiler/xla/tests/literal_test_util.h
@@ -50,8 +50,6 @@ class LiteralTestUtil {
  public:
   // Asserts that the given shapes have the same rank, dimension sizes, and
   // primitive types.
-  static ::testing::AssertionResult EqualShapes(const Shape& expected,
-                                                const Shape& actual);
   static void AssertEqualShapes(const Shape& expected, const Shape& actual);
 
   // Asserts that the provided shapes are equal as defined in AssertEqualShapes
diff --git a/tensorflow/compiler/xla/tests/reshape_test.cc b/tensorflow/compiler/xla/tests/reshape_test.cc
index 72c68f24a0..bb7160e3a0 100644
--- a/tensorflow/compiler/xla/tests/reshape_test.cc
+++ b/tensorflow/compiler/xla/tests/reshape_test.cc
@@ -47,7 +47,7 @@ class ReshapeTest : public ClientLibraryTestBase {
 };
 
 // Collapses 2-dimensional pseudo-scalar (single-element array) to 1 dimension.
-XLA_TEST_F(ReshapeTest, CollapseTrivial1x1) {
+XLA_TEST_F(ReshapeTest, Trivial1x1) {
   ComputationBuilder builder(client_, TestName());
   auto a = builder.ConstantR2<float>({{1.0}});
   builder.Collapse(/*operand=*/a, /*dimensions=*/{0, 1});
@@ -55,22 +55,6 @@ XLA_TEST_F(ReshapeTest, CollapseTrivial1x1) {
   ComputeAndCompareR1<float>(&builder, {1.0f}, {}, zero_error_spec_);
 }
 
-XLA_TEST_F(ReshapeTest, CollapseTrivialR1EmptyDims) {
-  ComputationBuilder builder(client_, TestName());
-  auto a = builder.ConstantR1<float>({1.0});
-  builder.Collapse(/*operand=*/a, /*dimensions=*/{});
-
-  ComputeAndCompareR1<float>(&builder, {1.0f}, {}, zero_error_spec_);
-}
-
-XLA_TEST_F(ReshapeTest, CollapseTrivialR1OnlyDim) {
-  ComputationBuilder builder(client_, TestName());
-  auto a = builder.ConstantR1<float>({1.0});
-  builder.Collapse(/*operand=*/a, /*dimensions=*/{0});
-
-  ComputeAndCompareR1<float>(&builder, {1.0f}, {}, zero_error_spec_);
-}
-
 // Collapses 2-dimensional pseudo-scalar (single-element array) to scalar.
 XLA_TEST_F(ReshapeTest, SingleElementArrayToScalar) {
   ComputationBuilder builder(client_, TestName());
diff --git a/tensorflow/core/common_runtime/executor.cc b/tensorflow/core/common_runtime/executor.cc
index 11e063d8d2..f57834cfbe 100644
--- a/tensorflow/core/common_runtime/executor.cc
+++ b/tensorflow/core/common_runtime/executor.cc
@@ -2008,7 +2008,7 @@ bool ExecutorState::NodeDone(const Status& s, const Node* node,
                              NodeExecStatsWrapper* stats,
                              TaggedNodeReadyQueue* inline_ready) {
   nodestats::SetAllEnd(stats);
-  if (stats_collector_ != nullptr && !SetTimelineLabel(node, stats)) {
+  if (!SetTimelineLabel(node, stats)) {
     // Only record non-transfer nodes.
     // Transfers 'stats' ownership to 'stats_collector_'.
     stats_collector_->Save(impl_->params_.device->name(), stats);
-- 
GitLab


From d749f56a3e0b17a5fe5f3252446223b84e485f04 Mon Sep 17 00:00:00 2001
From: Chris Leary <leary@google.com>
Date: Fri, 6 Oct 2017 11:06:12 -0700
Subject: [PATCH 123/909] [XLA] Fix a bug in ComputationBuilder::Collapse and
 add more tests/docs.

Also updated test infrastructure so a shape mismatch does not cause a fatal
crash in index_util, but rather reports an appropriate test failure message.

PiperOrigin-RevId: 171315165
---
 tensorflow/compiler/xla/client/client.cc      |  1 +
 .../xla/client/computation_builder.cc         | 13 ++++
 .../compiler/xla/client/computation_builder.h | 10 +++
 .../compiler/xla/service/shape_inference.cc   |  9 ++-
 .../compiler/xla/tests/literal_test_util.cc   | 73 ++++++++++++++-----
 .../compiler/xla/tests/literal_test_util.h    |  2 +
 tensorflow/compiler/xla/tests/reshape_test.cc | 18 ++++-
 7 files changed, 105 insertions(+), 21 deletions(-)

diff --git a/tensorflow/compiler/xla/client/client.cc b/tensorflow/compiler/xla/client/client.cc
index 387253617e..7db2ea79fb 100644
--- a/tensorflow/compiler/xla/client/client.cc
+++ b/tensorflow/compiler/xla/client/client.cc
@@ -206,6 +206,7 @@ StatusOr<std::unique_ptr<GlobalData>> Client::Execute(
     *request.mutable_execution_options() = *execution_options;
   }
   for (GlobalData* argument : arguments) {
+    CHECK(argument != nullptr) << "Argument pointers must not be null.";
     *request.add_arguments() = argument->handle();
   }
 
diff --git a/tensorflow/compiler/xla/client/computation_builder.cc b/tensorflow/compiler/xla/client/computation_builder.cc
index 15a713513f..925dcd36c0 100644
--- a/tensorflow/compiler/xla/client/computation_builder.cc
+++ b/tensorflow/compiler/xla/client/computation_builder.cc
@@ -489,6 +489,16 @@ ComputationDataHandle ComputationBuilder::Collapse(
   }
   std::unique_ptr<Shape> original_shape = shape_or_status.ConsumeValueOrDie();
 
+  VLOG(3) << "original shape: " << ShapeUtil::HumanString(*original_shape);
+  VLOG(3) << "dims to collapse: "
+          << tensorflow::str_util::Join(dims_to_collapse, ",");
+
+  if (dims_to_collapse.size() <= 1) {
+    // Not collapsing anything, trivially we can return the operand versus
+    // enqueueing a trivial reshape.
+    return operand;
+  }
+
   std::vector<int64> new_sizes;
   for (int i = 0; i < ShapeUtil::Rank(*original_shape); ++i) {
     if (i <= dims_to_collapse.front() || i > dims_to_collapse.back()) {
@@ -498,6 +508,9 @@ ComputationDataHandle ComputationBuilder::Collapse(
     }
   }
 
+  VLOG(3) << "new sizes: [" << tensorflow::str_util::Join(new_sizes, ",")
+          << "]";
+
   return Reshape(operand, new_sizes);
 }
 
diff --git a/tensorflow/compiler/xla/client/computation_builder.h b/tensorflow/compiler/xla/client/computation_builder.h
index 73972c1290..7014685ea5 100644
--- a/tensorflow/compiler/xla/client/computation_builder.h
+++ b/tensorflow/compiler/xla/client/computation_builder.h
@@ -201,6 +201,16 @@ class ComputationBuilder {
   // {x=1024, y=32} by collapsing dims {0, 1, 2}. Collapsing dimensions must
   // be a consecutive, in-order subsequence of the operand dimensions.
   //
+  // Note that collapsing a single dimension does nothing:
+  //
+  //    {256} collapsing {0} => {256}
+  //    {1} collapsing {0} => {1}
+  //
+  // Collapsing multiple dimensions produces a single result dimension:
+  //
+  //    {256, 2} collapsing {0,1} => {512}
+  //    {256, 2, 3} collapsing {0,1} => {512, 3}
+  //
   // This could potentially cause data to be moved -- it provides a more
   // structured form of reshaping than an arbitrary Reshape operation.
   ComputationDataHandle Collapse(const ComputationDataHandle& operand,
diff --git a/tensorflow/compiler/xla/service/shape_inference.cc b/tensorflow/compiler/xla/service/shape_inference.cc
index ffd8018827..29221d2d29 100644
--- a/tensorflow/compiler/xla/service/shape_inference.cc
+++ b/tensorflow/compiler/xla/service/shape_inference.cc
@@ -1894,11 +1894,16 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(
 
   Shape inferred_shape =
       ShapeUtil::MakeShape(operand.element_type(), new_sizes);
+  VLOG(3) << "Reshape inferred shape: "
+          << ShapeUtil::HumanString(inferred_shape);
 
   if (ShapeUtil::ElementsIn(operand) != ShapeUtil::ElementsIn(inferred_shape)) {
     return InvalidArgument(
-        "reshape operation has mismatched element counts: from=%lld to=%lld",
-        ShapeUtil::ElementsIn(operand), ShapeUtil::ElementsIn(inferred_shape));
+        "reshape operation has mismatched element counts: from=%lld (%s) "
+        "to=%lld (%s)",
+        ShapeUtil::ElementsIn(operand), ShapeUtil::HumanString(operand).c_str(),
+        ShapeUtil::ElementsIn(inferred_shape),
+        ShapeUtil::HumanString(inferred_shape).c_str());
   }
 
   std::vector<int64> indices(ShapeUtil::Rank(operand));
diff --git a/tensorflow/compiler/xla/tests/literal_test_util.cc b/tensorflow/compiler/xla/tests/literal_test_util.cc
index 061a4e190f..2876a79dd8 100644
--- a/tensorflow/compiler/xla/tests/literal_test_util.cc
+++ b/tensorflow/compiler/xla/tests/literal_test_util.cc
@@ -39,30 +39,60 @@ limitations under the License.
 
 namespace xla {
 
-/* static */ void LiteralTestUtil::AssertEqualShapes(const Shape& expected,
-                                                     const Shape& actual) {
-  ASSERT_EQ(ShapeUtil::IsTuple(expected), ShapeUtil::IsTuple(actual));
+/* static */ ::testing::AssertionResult LiteralTestUtil::EqualShapes(
+    const Shape& expected, const Shape& actual) {
+  if (ShapeUtil::IsTuple(expected) != ShapeUtil::IsTuple(actual)) {
+    return ::testing::AssertionFailure()
+           << "tupleness-mismatch! want: " << ShapeUtil::HumanString(expected)
+           << " got: " << ShapeUtil::HumanString(actual);
+  }
   if (ShapeUtil::IsTuple(expected)) {
-    ASSERT_EQ(ShapeUtil::TupleElementCount(expected),
-              ShapeUtil::TupleElementCount(actual));
+    if (ShapeUtil::TupleElementCount(expected) !=
+        ShapeUtil::TupleElementCount(actual)) {
+      return ::testing::AssertionFailure()
+             << "want tuple element count: "
+             << ShapeUtil::TupleElementCount(expected)
+             << " got tuple element count: "
+             << ShapeUtil::TupleElementCount(actual);
+    }
     for (int i = 0; i < expected.tuple_shapes_size(); ++i) {
-      AssertEqualShapes(expected.tuple_shapes(i), actual.tuple_shapes(i));
+      ::testing::AssertionResult result =
+          EqualShapes(expected.tuple_shapes(i), actual.tuple_shapes(i));
+      if (!result) {
+        return result;
+      }
     }
   } else {
-    ASSERT_EQ(ShapeUtil::Rank(expected), ShapeUtil::Rank(actual))
-        << "want rank of: " << ShapeUtil::HumanString(expected)
-        << " got rank of: " << ShapeUtil::HumanString(actual);
-    ASSERT_EQ(expected.element_type(), actual.element_type())
-        << PrimitiveType_Name(expected.element_type()) << " vs "
-        << PrimitiveType_Name(actual.element_type());
-    ASSERT_EQ(expected.dimensions_size(), actual.dimensions_size());
+    if (ShapeUtil::Rank(expected) != ShapeUtil::Rank(actual)) {
+      return ::testing::AssertionFailure()
+             << "want rank of: " << ShapeUtil::HumanString(expected)
+             << " got rank of: " << ShapeUtil::HumanString(actual);
+    }
+    if (expected.element_type() != actual.element_type()) {
+      return ::testing::AssertionFailure()
+             << PrimitiveType_Name(expected.element_type()) << " vs "
+             << PrimitiveType_Name(actual.element_type());
+    }
+    if (expected.dimensions_size() != actual.dimensions_size()) {
+      return ::testing::AssertionFailure()
+             << "want dimensions_size " << expected.dimensions_size()
+             << " got dimensions_size " << actual.dimensions_size();
+    }
     for (int i = 0; i < expected.dimensions_size(); ++i) {
-      ASSERT_EQ(expected.dimensions(i), actual.dimensions(i))
-          << "mismatch in dimension #" << i
-          << " expected: " << ShapeUtil::HumanString(expected)
-          << " actual: " << ShapeUtil::HumanString(actual);
+      if (expected.dimensions(i) != actual.dimensions(i)) {
+        return ::testing::AssertionFailure()
+               << "mismatch in dimension #" << i
+               << " expected: " << ShapeUtil::HumanString(expected)
+               << " actual: " << ShapeUtil::HumanString(actual);
+      }
     }
   }
+  return ::testing::AssertionSuccess();
+}
+
+/* static */ void LiteralTestUtil::AssertEqualShapes(const Shape& expected,
+                                                     const Shape& actual) {
+  ASSERT_TRUE(EqualShapes(expected, actual));
 }
 
 /* static */ void LiteralTestUtil::AssertEqualShapesAndLayouts(
@@ -265,7 +295,14 @@ class NearComparator {
     VLOG(1) << "actual:";
     XLA_VLOG_LINES(1, actual.ToString());
 
-    LiteralTestUtil::AssertEqualShapes(expected.shape(), actual.shape());
+    // If the shapes mismatch, we simply fail the expectation instead of
+    // printing out data, as it's a type error rather than a value error.
+    ::testing::AssertionResult equal_shapes =
+        LiteralTestUtil::EqualShapes(expected.shape(), actual.shape());
+    if (!equal_shapes) {
+      EXPECT_TRUE(equal_shapes);
+      return false;
+    }
 
     // Set up members used during the comparison.
     num_miscompares_ = 0;
diff --git a/tensorflow/compiler/xla/tests/literal_test_util.h b/tensorflow/compiler/xla/tests/literal_test_util.h
index f645c4e8dc..467d44b857 100644
--- a/tensorflow/compiler/xla/tests/literal_test_util.h
+++ b/tensorflow/compiler/xla/tests/literal_test_util.h
@@ -50,6 +50,8 @@ class LiteralTestUtil {
  public:
   // Asserts that the given shapes have the same rank, dimension sizes, and
   // primitive types.
+  static ::testing::AssertionResult EqualShapes(const Shape& expected,
+                                                const Shape& actual);
   static void AssertEqualShapes(const Shape& expected, const Shape& actual);
 
   // Asserts that the provided shapes are equal as defined in AssertEqualShapes
diff --git a/tensorflow/compiler/xla/tests/reshape_test.cc b/tensorflow/compiler/xla/tests/reshape_test.cc
index bb7160e3a0..72c68f24a0 100644
--- a/tensorflow/compiler/xla/tests/reshape_test.cc
+++ b/tensorflow/compiler/xla/tests/reshape_test.cc
@@ -47,7 +47,7 @@ class ReshapeTest : public ClientLibraryTestBase {
 };
 
 // Collapses 2-dimensional pseudo-scalar (single-element array) to 1 dimension.
-XLA_TEST_F(ReshapeTest, Trivial1x1) {
+XLA_TEST_F(ReshapeTest, CollapseTrivial1x1) {
   ComputationBuilder builder(client_, TestName());
   auto a = builder.ConstantR2<float>({{1.0}});
   builder.Collapse(/*operand=*/a, /*dimensions=*/{0, 1});
@@ -55,6 +55,22 @@ XLA_TEST_F(ReshapeTest, Trivial1x1) {
   ComputeAndCompareR1<float>(&builder, {1.0f}, {}, zero_error_spec_);
 }
 
+XLA_TEST_F(ReshapeTest, CollapseTrivialR1EmptyDims) {
+  ComputationBuilder builder(client_, TestName());
+  auto a = builder.ConstantR1<float>({1.0});
+  builder.Collapse(/*operand=*/a, /*dimensions=*/{});
+
+  ComputeAndCompareR1<float>(&builder, {1.0f}, {}, zero_error_spec_);
+}
+
+XLA_TEST_F(ReshapeTest, CollapseTrivialR1OnlyDim) {
+  ComputationBuilder builder(client_, TestName());
+  auto a = builder.ConstantR1<float>({1.0});
+  builder.Collapse(/*operand=*/a, /*dimensions=*/{0});
+
+  ComputeAndCompareR1<float>(&builder, {1.0f}, {}, zero_error_spec_);
+}
+
 // Collapses 2-dimensional pseudo-scalar (single-element array) to scalar.
 XLA_TEST_F(ReshapeTest, SingleElementArrayToScalar) {
   ComputationBuilder builder(client_, TestName());
-- 
GitLab


From ce2f89c8bfdbef373c1b1ff9a1c6818f6bf462f8 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 6 Oct 2017 11:24:00 -0700
Subject: [PATCH 124/909] Fix stats_collector_ null pointer error.

PiperOrigin-RevId: 171318477
---
 tensorflow/core/common_runtime/executor.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/common_runtime/executor.cc b/tensorflow/core/common_runtime/executor.cc
index f57834cfbe..11e063d8d2 100644
--- a/tensorflow/core/common_runtime/executor.cc
+++ b/tensorflow/core/common_runtime/executor.cc
@@ -2008,7 +2008,7 @@ bool ExecutorState::NodeDone(const Status& s, const Node* node,
                              NodeExecStatsWrapper* stats,
                              TaggedNodeReadyQueue* inline_ready) {
   nodestats::SetAllEnd(stats);
-  if (!SetTimelineLabel(node, stats)) {
+  if (stats_collector_ != nullptr && !SetTimelineLabel(node, stats)) {
     // Only record non-transfer nodes.
     // Transfers 'stats' ownership to 'stats_collector_'.
     stats_collector_->Save(impl_->params_.device->name(), stats);
-- 
GitLab


From 7fceb8d879dd23a2fd15403d216367e5e8f52b56 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Fri, 6 Oct 2017 11:34:03 -0700
Subject: [PATCH 125/909] [XLA:CPU] Make EmitTargetAddressForOp return void
 (well, technically Status).

This is a general cleanup -- less repeated code -- but it's also part of
an effort to use IrArray more and llvm::Value less.  In particular, many
callsites would take the llvm::Value returned by EmitTargetAddressForOp
and create an IrArray out of it, but then never attach AA info to that
array.  Having this function return void forces you to call
GetIrArrayForOp(), which attaches the AA metadata appropriately.

This change also gets rid of an unused arg to EmitTargetAddressForOp.

PiperOrigin-RevId: 171320201
---
 .../compiler/xla/service/cpu/ir_emitter.cc    | 242 ++++++------------
 .../compiler/xla/service/cpu/ir_emitter.h     |   9 +-
 2 files changed, 86 insertions(+), 165 deletions(-)

diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
index 4375f13a0e..e4fb7c0496 100644
--- a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
@@ -291,8 +291,7 @@ Status IrEmitter::HandleConstant(HloInstruction* constant,
 Status IrEmitter::HandleCopy(HloInstruction* copy) {
   if (ShapeUtil::IsTuple(copy->shape())) {
     // kCopy shallow copies a tuple so just memcpy the top-level buffer.
-    TF_ASSIGN_OR_RETURN(llvm::Value * copy_value, EmitTargetAddressForOp(copy));
-    emitted_value_[copy] = copy_value;
+    TF_RETURN_IF_ERROR(EmitTargetAddressForOp(copy));
     return EmitMemcpy(*(copy->operand(0)), *copy);
   } else {
     // Use the elemental emitter for non-tuple shapes.
@@ -395,9 +394,7 @@ Status IrEmitter::HandleSelect(HloInstruction* select, HloInstruction* pred,
   TF_RET_CHECK(pred->shape().element_type() == PRED);
 
   if (ShapeUtil::IsTuple(select->shape())) {
-    TF_ASSIGN_OR_RETURN(llvm::Value * output_address,
-                        EmitTargetAddressForOp(select));
-    emitted_value_[select] = output_address;
+    TF_RETURN_IF_ERROR(EmitTargetAddressForOp(select));
     llvm_ir::EmitTupleSelect(GetIrArrayForOp(select), GetIrArrayForOp(pred),
                              GetEmittedValueFor(on_true),
                              GetEmittedValueFor(on_false), &ir_builder_);
@@ -414,8 +411,8 @@ Status IrEmitter::HandleInfeed(HloInstruction* infeed) {
 
   // The infeed operation produces data (dequeued from the infeed queue) at this
   // address, which has been provided by buffer assignment.
-  TF_ASSIGN_OR_RETURN(llvm::Value * target_address,
-                      EmitTargetAddressForOp(infeed));
+  TF_RETURN_IF_ERROR(EmitTargetAddressForOp(infeed));
+  llvm_ir::IrArray infeed_array = GetIrArrayForOp(infeed);
 
   if (ShapeUtil::IsTuple(shape)) {
     TF_RET_CHECK(!ShapeUtil::IsNestedTuple(shape));
@@ -433,9 +430,9 @@ Status IrEmitter::HandleInfeed(HloInstruction* infeed) {
           ShapeUtil::GetTupleElementShape(shape, i);
 
       // Only the outer tuple buffer's target address is obtained from
-      // EmitTargetAddressForOp to handle the case when Infeed is the
-      // root instruction. Target addresses for internal elements can
-      // be obtained from EmitTempBufferPointer.
+      // GetEmittedValueFor, to handle the case when Infeed is the root
+      // instruction. Target addresses for internal elements can be obtained
+      // from EmitTempBufferPointer.
       llvm::Value* tuple_element_address =
           EmitTempBufferPointer(buffer, tuple_element_shape);
 
@@ -445,15 +442,12 @@ Status IrEmitter::HandleInfeed(HloInstruction* infeed) {
       tuple_element_addresses.push_back(tuple_element_address);
     }
 
-    llvm_ir::EmitTuple(llvm_ir::IrArray(target_address, shape),
-                       tuple_element_addresses, &ir_builder_);
+    llvm_ir::EmitTuple(infeed_array, tuple_element_addresses, &ir_builder_);
   } else {
-    TF_RETURN_IF_ERROR(
-        EmitXfeedTransfer(XfeedKind::kInfeed, shape, target_address));
+    TF_RETURN_IF_ERROR(EmitXfeedTransfer(XfeedKind::kInfeed, shape,
+                                         GetEmittedValueFor(infeed)));
   }
 
-  emitted_value_[infeed] = target_address;
-
   return Status::OK();
 }
 
@@ -567,15 +561,12 @@ Status IrEmitter::HandleSort(HloInstruction* sort, HloInstruction* operand) {
 Status IrEmitter::HandleTuple(
     HloInstruction* tuple,
     tensorflow::gtl::ArraySlice<HloInstruction*> operands) {
-  TF_ASSIGN_OR_RETURN(llvm::Value * target_address,
-                      EmitTargetAddressForOp(tuple));
+  TF_RETURN_IF_ERROR(EmitTargetAddressForOp(tuple));
   std::vector<llvm::Value*> base_ptrs;
   for (auto operand : operands) {
     base_ptrs.push_back(GetEmittedValueFor(operand));
   }
-  llvm_ir::EmitTuple(llvm_ir::IrArray(target_address, tuple->shape()),
-                     base_ptrs, &ir_builder_);
-  emitted_value_[tuple] = target_address;
+  llvm_ir::EmitTuple(GetIrArrayForOp(tuple), base_ptrs, &ir_builder_);
   return Status::OK();
 }
 
@@ -892,11 +883,8 @@ Status IrEmitter::HandleDot(HloInstruction* dot, HloInstruction* lhs,
   llvm_ir::IrArray lhs_array(GetIrArrayForOp(lhs));
   llvm_ir::IrArray rhs_array(GetIrArrayForOp(rhs));
 
-  Shape target_shape = dot->shape();
-  TF_ASSIGN_OR_RETURN(llvm::Value * target_address,
-                      EmitTargetAddressForOp(dot));
-  llvm_ir::IrArray target_array(target_address, target_shape);
-  AddAliasingInformationToIrArray(*dot, &target_array);
+  TF_RETURN_IF_ERROR(EmitTargetAddressForOp(dot));
+  llvm_ir::IrArray target_array = GetIrArrayForOp(dot);
 
   VLOG(2) << "HandleDot: ";
   VLOG(2) << "  lhs operand: "
@@ -907,13 +895,10 @@ Status IrEmitter::HandleDot(HloInstruction* dot, HloInstruction* lhs,
           << llvm_ir::DumpToString(*target_array.GetBasePointer());
 
   // Dot operation is complicated so we delegate to a helper class.
-  TF_RETURN_IF_ERROR(DotOpEmitter::EmitDotOperation(
+  return DotOpEmitter::EmitDotOperation(
       *dot, /*transpose_lhs=*/false, /*transpose_rhs=*/false, target_array,
       lhs_array, rhs_array, GetExecutableRunOptionsArgument(), &ir_builder_,
-      hlo_module_config_));
-
-  emitted_value_[dot] = target_address;
-  return Status::OK();
+      hlo_module_config_);
 }
 
 Status IrEmitter::HandleConvolution(HloInstruction* convolution,
@@ -941,8 +926,7 @@ Status IrEmitter::HandleConvolution(HloInstruction* convolution,
       bool one_dim_convolution = lhs_shape.dimensions_size() == 3;
       llvm::Value* lhs_address = GetEmittedValueFor(lhs);
       llvm::Value* rhs_address = GetEmittedValueFor(rhs);
-      TF_ASSIGN_OR_RETURN(llvm::Value * target_address,
-                          EmitTargetAddressForOp(convolution));
+      TF_RETURN_IF_ERROR(EmitTargetAddressForOp(convolution));
 
       const ConvolutionDimensionNumbers& dnums =
           convolution->convolution_dimension_numbers();
@@ -1024,35 +1008,33 @@ Status IrEmitter::HandleConvolution(HloInstruction* convolution,
       conv_func->setDoesNotThrow();
       conv_func->setOnlyAccessesArgMemory();
       ir_builder_.CreateCall(
-          conv_func,
-          {
-              GetExecutableRunOptionsArgument(),
-              ir_builder_.CreateBitCast(target_address, float_ptr_type),
-              ir_builder_.CreateBitCast(lhs_address, float_ptr_type),
-              ir_builder_.CreateBitCast(rhs_address, float_ptr_type),
-              ir_builder_.getInt64(input_batch),
-              ir_builder_.getInt64(input_rows),
-              ir_builder_.getInt64(input_cols),
-              ir_builder_.getInt64(input_channels),
-              ir_builder_.getInt64(kernel_rows),
-              ir_builder_.getInt64(kernel_cols),
-              ir_builder_.getInt64(kernel_channels),
-              ir_builder_.getInt64(kernel_filters),
-              ir_builder_.getInt64(output_rows),
-              ir_builder_.getInt64(output_cols),
-              ir_builder_.getInt64(row_stride),
-              ir_builder_.getInt64(col_stride),
-              ir_builder_.getInt64(padding_top),
-              ir_builder_.getInt64(padding_bottom),
-              ir_builder_.getInt64(padding_left),
-              ir_builder_.getInt64(padding_right),
-              ir_builder_.getInt64(lhs_row_dilation),
-              ir_builder_.getInt64(lhs_col_dilation),
-              ir_builder_.getInt64(rhs_row_dilation),
-              ir_builder_.getInt64(rhs_col_dilation),
-          });
-      target_address->setName(AsStringRef(IrName(convolution)));
-      emitted_value_[convolution] = target_address;
+          conv_func, {
+                         GetExecutableRunOptionsArgument(),
+                         ir_builder_.CreateBitCast(
+                             GetEmittedValueFor(convolution), float_ptr_type),
+                         ir_builder_.CreateBitCast(lhs_address, float_ptr_type),
+                         ir_builder_.CreateBitCast(rhs_address, float_ptr_type),
+                         ir_builder_.getInt64(input_batch),
+                         ir_builder_.getInt64(input_rows),
+                         ir_builder_.getInt64(input_cols),
+                         ir_builder_.getInt64(input_channels),
+                         ir_builder_.getInt64(kernel_rows),
+                         ir_builder_.getInt64(kernel_cols),
+                         ir_builder_.getInt64(kernel_channels),
+                         ir_builder_.getInt64(kernel_filters),
+                         ir_builder_.getInt64(output_rows),
+                         ir_builder_.getInt64(output_cols),
+                         ir_builder_.getInt64(row_stride),
+                         ir_builder_.getInt64(col_stride),
+                         ir_builder_.getInt64(padding_top),
+                         ir_builder_.getInt64(padding_bottom),
+                         ir_builder_.getInt64(padding_left),
+                         ir_builder_.getInt64(padding_right),
+                         ir_builder_.getInt64(lhs_row_dilation),
+                         ir_builder_.getInt64(lhs_col_dilation),
+                         ir_builder_.getInt64(rhs_row_dilation),
+                         ir_builder_.getInt64(rhs_col_dilation),
+                     });
 
       return Status::OK();
     }
@@ -1367,9 +1349,7 @@ Status IrEmitter::HandleBatchNormTraining(HloInstruction* batch_norm_training) {
           mean_array, &ir_builder_)
           .EmitLoop(IrName(batch_norm_training, "mean_var")));
 
-  TF_ASSIGN_OR_RETURN(llvm::Value * target_address,
-                      EmitTargetAddressForOp(batch_norm_training));
-
+  TF_RETURN_IF_ERROR(EmitTargetAddressForOp(batch_norm_training));
   TF_ASSIGN_OR_RETURN(
       const BufferAllocation::Slice slice,
       assignment_.GetUniqueSlice(batch_norm_training, /*index=*/{0}));
@@ -1425,11 +1405,8 @@ Status IrEmitter::HandleBatchNormTraining(HloInstruction* batch_norm_training) {
           target_array, &ir_builder_)
           .EmitLoop(IrName(batch_norm_training, "normalize")));
 
-  llvm_ir::EmitTuple(
-      llvm_ir::IrArray(target_address, batch_norm_training->shape()),
-      {normalized, mean, var}, &ir_builder_);
-  emitted_value_[batch_norm_training] = target_address;
-
+  llvm_ir::EmitTuple(GetIrArrayForOp(batch_norm_training),
+                     {normalized, mean, var}, &ir_builder_);
   return Status::OK();
 }
 
@@ -1789,6 +1766,7 @@ StatusOr<bool> IrEmitter::EmitVectorizedReduce(
   }
 
   CHECK(!ShapeUtil::IsTuple(reduce->shape()));
+  TF_RETURN_IF_ERROR(EmitTargetAddressForOp(reduce));
 
   // We know we're not reducing over the most minor dimension, which means we
   // can lower the reduction loop as:
@@ -1851,10 +1829,7 @@ StatusOr<bool> IrEmitter::EmitVectorizedReduce(
                             reduction_generator, array_index, vector_type,
                             init_value, arg, dimensions, element_alignment));
 
-    TF_ASSIGN_OR_RETURN(llvm::Value * target_address,
-                        EmitTargetAddressForOp(reduce));
-    llvm_ir::IrArray target_array(target_address, reduce->shape());
-    AddAliasingInformationToIrArray(*reduce, &target_array);
+    llvm_ir::IrArray target_array = GetIrArrayForOp(reduce);
     llvm::Value* output_address =
         target_array.EmitArrayElementAddress(array_index, &ir_builder_);
     EmitShardedVectorStore(output_address, accumulator, element_alignment,
@@ -1886,10 +1861,7 @@ StatusOr<bool> IrEmitter::EmitVectorizedReduce(
                             reduction_generator, array_index, vector_type,
                             init_value, arg, dimensions, element_alignment));
 
-    TF_ASSIGN_OR_RETURN(llvm::Value * target_address,
-                        EmitTargetAddressForOp(reduce));
-    llvm_ir::IrArray target_array(target_address, reduce->shape());
-    AddAliasingInformationToIrArray(*reduce, &target_array);
+    llvm_ir::IrArray target_array = GetIrArrayForOp(reduce);
     llvm::Value* output_address =
         target_array.EmitArrayElementAddress(array_index, &ir_builder_);
     EmitShardedVectorStore(output_address, accumulator, element_alignment,
@@ -1900,10 +1872,6 @@ StatusOr<bool> IrEmitter::EmitVectorizedReduce(
     ir_builder_.SetInsertPoint(outermost_loop_exit_block);
   }
 
-  TF_ASSIGN_OR_RETURN(llvm::Value * target_address,
-                      EmitTargetAddressForOp(reduce));
-
-  emitted_value_[reduce] = target_address;
   return true;
 }
 
@@ -2003,9 +1971,7 @@ Status IrEmitter::HandleSlice(HloInstruction* slice, HloInstruction* operand) {
     return DefaultAction(slice);
   }
 
-  TF_ASSIGN_OR_RETURN(llvm::Value * target_address,
-                      EmitTargetAddressForOp(slice));
-  emitted_value_[slice] = target_address;
+  TF_RETURN_IF_ERROR(EmitTargetAddressForOp(slice));
 
   if (ShapeUtil::HasZeroElements(slice->shape())) {
     return Status::OK();
@@ -2077,8 +2043,7 @@ Status IrEmitter::HandleSlice(HloInstruction* slice, HloInstruction* operand) {
     outer_dims.push_back(memcpy_dim);
   }
 
-  llvm_ir::IrArray target_array(target_address, slice->shape());
-  AddAliasingInformationToIrArray(*slice, &target_array);
+  llvm_ir::IrArray target_array = GetIrArrayForOp(slice);
 
   const int64 num_outer_loops = outer_dims.size();
   llvm_ir::ForLoopNest loops(IrName(slice), &ir_builder_);
@@ -2131,10 +2096,7 @@ Status IrEmitter::HandleDynamicSlice(HloInstruction* dynamic_slice,
                                      HloInstruction* operand,
                                      HloInstruction* /*start_indices*/) {
   if (ShapeUtil::IsScalar(dynamic_slice->shape())) {
-    TF_ASSIGN_OR_RETURN(llvm::Value * target_address,
-                        EmitTargetAddressForOp(dynamic_slice));
-    target_address->setName(AsStringRef(IrName(dynamic_slice)));
-    emitted_value_[dynamic_slice] = target_address;
+    TF_RETURN_IF_ERROR(EmitTargetAddressForOp(dynamic_slice));
     return EmitMemcpy(*operand, *dynamic_slice);
   }
   return DefaultAction(dynamic_slice);
@@ -2190,10 +2152,7 @@ Status IrEmitter::HandleDynamicUpdateSlice(HloInstruction* dynamic_update_slice,
                                            HloInstruction* update,
                                            HloInstruction* start_indices) {
   if (ShapeUtil::IsScalar(dynamic_update_slice->shape())) {
-    TF_ASSIGN_OR_RETURN(llvm::Value * target_address,
-                        EmitTargetAddressForOp(dynamic_update_slice));
-    target_address->setName(AsStringRef(IrName(dynamic_update_slice)));
-    emitted_value_[dynamic_update_slice] = target_address;
+    TF_RETURN_IF_ERROR(EmitTargetAddressForOp(dynamic_update_slice));
     return EmitMemcpy(*update, *dynamic_update_slice);
   } else if (CanUpdateDynamicSliceInPlace(assignment_, dynamic_update_slice)) {
     VLOG(2) << "Emitting HandleDynamicUpdateSlice in-place.";
@@ -2247,9 +2206,7 @@ Status IrEmitter::HandleDynamicUpdateSlice(HloInstruction* dynamic_update_slice,
         llvm_ir::LoopEmitter(loop_body_emitter, update->shape(), &ir_builder_)
             .EmitLoop(IrName(dynamic_update_slice, "in_place")));
 
-    TF_ASSIGN_OR_RETURN(llvm::Value * dynamic_update_slice_address,
-                        EmitTargetAddressForOp(dynamic_update_slice));
-    emitted_value_[dynamic_update_slice] = dynamic_update_slice_address;
+    TF_RETURN_IF_ERROR(EmitTargetAddressForOp(dynamic_update_slice));
     return Status::OK();
   }
   return DefaultAction(dynamic_update_slice);
@@ -2348,11 +2305,8 @@ Status IrEmitter::HandleFusion(HloInstruction* fusion) {
     llvm_ir::IrArray rhs_array(GetIrArrayForOp(rhs));
 
     Shape target_shape = fusion->shape();
-    TF_ASSIGN_OR_RETURN(llvm::Value * target_address,
-                        EmitTargetAddressForOp(fusion));
-    llvm_ir::IrArray target_array(target_address, target_shape);
-    AddAliasingInformationToIrArray(*fusion, &target_array);
-
+    TF_RETURN_IF_ERROR(EmitTargetAddressForOp(fusion));
+    llvm_ir::IrArray target_array = GetIrArrayForOp(fusion);
     VLOG(2) << "HandleFusion kTransposeDot: ";
     VLOG(2) << "  lhs operand: "
             << llvm_ir::DumpToString(*lhs_array.GetBasePointer());
@@ -2366,8 +2320,6 @@ Status IrEmitter::HandleFusion(HloInstruction* fusion) {
         *dot, dot->operand(0)->IsRank2Transpose(),
         dot->operand(1)->IsRank2Transpose(), target_array, lhs_array, rhs_array,
         GetExecutableRunOptionsArgument(), &ir_builder_, hlo_module_config_));
-
-    emitted_value_[fusion] = target_address;
     return Status::OK();
   } else if (fusion->fusion_kind() == HloInstruction::FusionKind::kLoop) {
     std::vector<llvm_ir::IrArray> parameter_arrays;
@@ -2393,14 +2345,9 @@ Status IrEmitter::HandleCall(HloInstruction* call) {
     parameter_addresses.push_back(GetEmittedValueFor(operand));
   }
 
-  TF_ASSIGN_OR_RETURN(llvm::Value * output_address,
-                      EmitTargetAddressForOp(call));
-  output_address->setName(AsStringRef(IrName(call)));
-
+  TF_RETURN_IF_ERROR(EmitTargetAddressForOp(call));
   EmitArrayFunctionCallInto(call_ir_function, parameter_addresses,
-                            output_address, computation->name());
-
-  emitted_value_[call] = output_address;
+                            emitted_value_[call], computation->name());
   return Status::OK();
 }
 
@@ -2429,17 +2376,13 @@ Status IrEmitter::HandleCustomCall(
               /*Params=*/{i8_ptr_type, operands_alloca->getType()},
               /*isVarArg=*/false)));
 
-  TF_ASSIGN_OR_RETURN(llvm::Value * output_address,
-                      EmitTargetAddressForOp(custom_call));
-  output_address->setName(AsStringRef(IrName(custom_call)));
-
-  auto* output_address_arg =
-      ir_builder_.CreatePointerCast(output_address, i8_ptr_type);
+  TF_RETURN_IF_ERROR(EmitTargetAddressForOp(custom_call));
+  auto* output_address_arg = ir_builder_.CreatePointerCast(
+      GetEmittedValueFor(custom_call), i8_ptr_type);
 
   ir_builder_.CreateCall(custom_call_ir_function,
                          {output_address_arg, operands_alloca});
 
-  emitted_value_[custom_call] = output_address;
   return Status::OK();
 }
 
@@ -2583,10 +2526,8 @@ StatusOr<bool> IrEmitter::EmitFastConcatenate(
   llvm::Type* i8_ptr_type = ir_builder_.getInt8PtrTy();
   llvm::Type* i8_type = ir_builder_.getInt8Ty();
 
-  TF_ASSIGN_OR_RETURN(llvm::Value * target_address,
-                      EmitTargetAddressForOp(concatenate));
-
-  llvm_ir::IrArray target_array(target_address, output_shape);
+  TF_RETURN_IF_ERROR(EmitTargetAddressForOp(concatenate));
+  llvm_ir::IrArray target_array = GetIrArrayForOp(concatenate);
 
   llvm_ir::ForLoopNest loops(IrName(concatenate), &ir_builder_);
   llvm_ir::IrArray::Index outer_dims_index =
@@ -2603,8 +2544,6 @@ StatusOr<bool> IrEmitter::EmitFastConcatenate(
   unsigned primitive_type_size =
       ShapeUtil::ByteSizeOfPrimitiveType(primitive_type);
 
-  AddAliasingInformationToIrArray(*concatenate, &target_array);
-
   // Contiguous subregions from each operand to the concatenate contribute to a
   // contiguous subregion in the target buffer starting at target_region_begin.
   llvm::Value* target_region_begin = ir_builder_.CreateBitCast(
@@ -2647,8 +2586,6 @@ StatusOr<bool> IrEmitter::EmitFastConcatenate(
     SetToFirstInsertPoint(loops.GetOuterLoopExitBasicBlock(), &ir_builder_);
   }
 
-  emitted_value_[concatenate] = target_address;
-
   return true;
 }
 
@@ -2842,15 +2779,6 @@ Status IrEmitter::Preprocess(HloInstruction* hlo) {
 }
 
 Status IrEmitter::Postprocess(HloInstruction* hlo) {
-  // Set the name of the emitted llvm::Value to IrName(hlo).  Outfeed and send
-  // the only ops that don't emit a value.
-  if (hlo->opcode() != HloOpcode::kOutfeed &&
-      hlo->opcode() != HloOpcode::kSend) {
-    auto it = emitted_value_.find(hlo);
-    CHECK(it != emitted_value_.end());
-    it->second->setName(AsStringRef(IrName(hlo)));
-  }
-
   if (auto* prof_counter = GetProfileCounterFor(hlo)) {
     profiling_state_.RecordCycleDelta(&ir_builder_, hlo, prof_counter);
   }
@@ -3027,10 +2955,10 @@ llvm::Value* IrEmitter::EmitArrayFunctionCall(
   return return_value_buffer;
 }
 
-StatusOr<llvm::Value*> IrEmitter::EmitTargetAddressForOp(
-    const HloInstruction* op, const ShapeIndex& shape_index) {
-  const Shape& target_shape = ShapeUtil::GetSubshape(op->shape(), shape_index);
-  if (op == op->parent()->root_instruction() && shape_index.empty()) {
+Status IrEmitter::EmitTargetAddressForOp(const HloInstruction* op) {
+  llvm::Value* addr;
+  const Shape& target_shape = op->shape();
+  if (op == op->parent()->root_instruction()) {
     // For the root node, we write directly to the output buffer of the
     // function.
     llvm::Argument* retval = GetResultArgument();
@@ -3040,15 +2968,18 @@ StatusOr<llvm::Value*> IrEmitter::EmitTargetAddressForOp(
       attr_builder.addDereferenceableAttr(ByteSizeOf(target_shape));
       retval->addAttrs(attr_builder);
     }
-    return ir_builder_.CreateBitCast(retval,
+    addr = ir_builder_.CreateBitCast(retval,
                                      IrShapeType(target_shape)->getPointerTo());
-  }
-
-  // For other nodes, we need the temporary buffer allocated for this node to
-  // write the result into.
-  TF_ASSIGN_OR_RETURN(const BufferAllocation::Slice slice,
-                      assignment_.GetUniqueTopLevelSlice(op));
-  return EmitTempBufferPointer(slice, target_shape);
+  } else {
+    // For other nodes, we need the temporary buffer allocated for this node to
+    // write the result into.
+    TF_ASSIGN_OR_RETURN(const BufferAllocation::Slice slice,
+                        assignment_.GetUniqueTopLevelSlice(op));
+    addr = EmitTempBufferPointer(slice, target_shape);
+  }
+  addr->setName(AsStringRef(IrName(op)));
+  emitted_value_[op] = addr;
+  return Status::OK();
 }
 
 Status IrEmitter::EmitTargetElementLoop(
@@ -3062,12 +2993,9 @@ Status IrEmitter::EmitTargetElementLoop(
     const llvm_ir::ElementGenerator& element_generator) {
   VLOG(2) << "EmitTargetElementLoop: " << target_op->ToString();
 
-  // target_address will hold the address of the target buffer we will write the
-  // result of the computation into.
   const Shape& target_shape = target_op->shape();
-  TF_ASSIGN_OR_RETURN(llvm::Value * target_address,
-                      EmitTargetAddressForOp(target_op));
-  VLOG(2) << "  target address: " << llvm_ir::DumpToString(*target_address);
+  TF_RETURN_IF_ERROR(EmitTargetAddressForOp(target_op));
+  llvm_ir::IrArray target_array = GetIrArrayForOp(target_op);
 
   if (target_op->IsMultiOutputFusion()) {
     // For multiple outputs fusion, we need to emit each operand and the root.
@@ -3090,13 +3018,9 @@ Status IrEmitter::EmitTargetElementLoop(
     for (int64 i = 0; i < output_arrays.size(); ++i) {
       tuple_operand_ptrs.push_back(output_arrays[i].GetBasePointer());
     }
-    llvm_ir::EmitTuple(llvm_ir::IrArray(target_address, target_shape),
-                       tuple_operand_ptrs, &ir_builder_);
+    llvm_ir::EmitTuple(target_array, tuple_operand_ptrs, &ir_builder_);
 
   } else {
-    llvm_ir::IrArray target_array(target_address, target_shape);
-    AddAliasingInformationToIrArray(*target_op, &target_array);
-
     if (ShouldEmitParallelLoopFor(*target_op)) {
       TF_RETURN_IF_ERROR(EmitParallelTargetElementLoop(
           target_shape, element_generator, IrName(target_op), &target_array));
@@ -3106,8 +3030,6 @@ Status IrEmitter::EmitTargetElementLoop(
               .EmitLoop(IrName(target_op)));
     }
   }
-
-  emitted_value_[target_op] = target_address;
   return Status::OK();
 }
 
diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.h b/tensorflow/compiler/xla/service/cpu/ir_emitter.h
index 05663b6038..fd9ee71799 100644
--- a/tensorflow/compiler/xla/service/cpu/ir_emitter.h
+++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.h
@@ -353,11 +353,10 @@ class IrEmitter : public DfsHloVisitorWithDefault {
   Status EmitMemcpy(const HloInstruction& source,
                     const HloInstruction& destination);
 
-  // Emit IR to compute the target address of the buffer for the given op.
-  // The returned Value is a pointer to a IR type that represents the op's
-  // element type.
-  StatusOr<llvm::Value*> EmitTargetAddressForOp(
-      const HloInstruction* op, const ShapeIndex& shape_index = {});
+  // Emits IR to compute the target address of the buffer for the given op.
+  // After calling this function, you can get a pointer to this buffer by
+  // calling GetIrArrayForOp or GetEmittedValueFor.
+  Status EmitTargetAddressForOp(const HloInstruction* op);
 
   // Structurizes "array_elements" into an MD array that represents "shape".
   // This is a recursive function, and "dimension_index" indicates the index of
-- 
GitLab


From 3110185270e93e0b6a3e82be9199febed1239602 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 6 Oct 2017 11:37:42 -0700
Subject: [PATCH 126/909] Use the new Estimator.get_variable_value() method to
 get the kmeans cluster centers.

PiperOrigin-RevId: 171320755
---
 .../contrib/factorization/examples/mnist.py   |  2 +-
 .../python/ops/clustering_ops.py              |  8 ++++--
 .../factorization/python/ops/kmeans.py        | 28 +++----------------
 .../learn/python/learn/estimators/kmeans.py   |  2 +-
 4 files changed, 11 insertions(+), 29 deletions(-)

diff --git a/tensorflow/contrib/factorization/examples/mnist.py b/tensorflow/contrib/factorization/examples/mnist.py
index 9eefbccd4d..06a62db004 100644
--- a/tensorflow/contrib/factorization/examples/mnist.py
+++ b/tensorflow/contrib/factorization/examples/mnist.py
@@ -142,7 +142,7 @@ def inference(inp, num_clusters, hidden1_units, hidden2_units):
       # initial_clusters=tf.contrib.factorization.KMEANS_PLUS_PLUS_INIT,
       use_mini_batch=True)
 
-  (all_scores, _, clustering_scores, _, _, kmeans_init,
+  (all_scores, _, clustering_scores, _, kmeans_init,
    kmeans_training_op) = kmeans.training_graph()
   # Some heuristics to approximately whiten this output.
   all_scores = (all_scores[0] - 0.5) * 5
diff --git a/tensorflow/contrib/factorization/python/ops/clustering_ops.py b/tensorflow/contrib/factorization/python/ops/clustering_ops.py
index e5c9180662..d7320aeb3d 100644
--- a/tensorflow/contrib/factorization/python/ops/clustering_ops.py
+++ b/tensorflow/contrib/factorization/python/ops/clustering_ops.py
@@ -51,6 +51,9 @@ COSINE_DISTANCE = 'cosine'
 RANDOM_INIT = 'random'
 KMEANS_PLUS_PLUS_INIT = 'kmeans_plus_plus'
 
+# The name of the variable holding the cluster centers. Used by the Estimator.
+CLUSTERS_VAR_NAME = 'clusters'
+
 
 class KMeans(object):
   """Creates the graph for k-means clustering."""
@@ -279,7 +282,7 @@ class KMeans(object):
     """
     init_value = array_ops.constant([], dtype=dtypes.float32)
     cluster_centers = variable_scope.variable(
-        init_value, name='clusters', validate_shape=False)
+        init_value, name=CLUSTERS_VAR_NAME, validate_shape=False)
     cluster_centers_initialized = variable_scope.variable(
         False, dtype=dtypes.bool, name='initialized')
 
@@ -337,7 +340,6 @@ class KMeans(object):
         assigned cluster instead.
       cluster_centers_initialized: scalar indicating whether clusters have been
         initialized.
-      cluster_centers_var: a Variable holding the cluster centers.
       init_op: an op to initialize the clusters.
       training_op: an op that runs an iteration of training.
     """
@@ -381,7 +383,7 @@ class KMeans(object):
           inputs, num_clusters, cluster_idx, cluster_centers_var)
 
     return (all_scores, cluster_idx, scores, cluster_centers_initialized,
-            cluster_centers_var, init_op, training_op)
+            init_op, training_op)
 
   def _mini_batch_sync_updates_op(self, update_in_steps, cluster_centers_var,
                                   cluster_centers_updated, total_counts):
diff --git a/tensorflow/contrib/factorization/python/ops/kmeans.py b/tensorflow/contrib/factorization/python/ops/kmeans.py
index 6284768bdd..9a5413fc3f 100644
--- a/tensorflow/contrib/factorization/python/ops/kmeans.py
+++ b/tensorflow/contrib/factorization/python/ops/kmeans.py
@@ -21,12 +21,10 @@ from __future__ import division
 from __future__ import print_function
 
 import time
-import numpy as np
 
 from tensorflow.contrib.factorization.python.ops import clustering_ops
 from tensorflow.python.estimator import estimator
 from tensorflow.python.estimator import model_fn as model_fn_lib
-from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
@@ -161,8 +159,7 @@ class _ModelFn(object):
         * `eval_metric_ops`: Maps `SCORE` to `loss`.
         * `predictions`: Maps `ALL_DISTANCES` to the distance from each input
              point to each cluster center; maps `CLUSTER_INDEX` to the index of
-             the closest cluster center for each input point; maps `CLUSTERS` to
-             the cluster centers (which ignores the input points).
+             the closest cluster center for each input point.
     """
     # input_points is a single Tensor. Therefore, the sharding functionality
     # in clustering_ops is unused, and some of the values below are lists of a
@@ -184,8 +181,8 @@ class _ModelFn(object):
     # training_op: an op that runs an iteration of training, either an entire
     #   Lloyd iteration or a mini-batch of a Lloyd iteration. Multiple workers
     #   may execute this op, but only after is_initialized becomes True.
-    (all_distances, model_predictions, losses, is_initialized,
-     cluster_centers_var, init_op, training_op) = clustering_ops.KMeans(
+    (all_distances, model_predictions, losses, is_initialized, init_op,
+     training_op) = clustering_ops.KMeans(
          inputs=input_points,
          num_clusters=self._num_clusters,
          initial_clusters=self._initial_clusters,
@@ -215,7 +212,6 @@ class _ModelFn(object):
         predictions={
             KMeansClustering.ALL_DISTANCES: all_distances[0],
             KMeansClustering.CLUSTER_INDEX: model_predictions[0],
-            KMeansClustering.CLUSTERS: cluster_centers_var.value(),
         },
         loss=loss,
         train_op=training_op,
@@ -242,9 +238,7 @@ class KMeansClustering(estimator.Estimator):
   # Keys returned by predict().
   # ALL_DISTANCES: The distance from each input  point to each cluster center.
   # CLUSTER_INDEX: The index of the closest cluster center for each input point.
-  # CLUSTERS: The cluster centers (which ignores the input points).
   CLUSTER_INDEX = 'cluster_index'
-  CLUSTERS = 'clusters'
   ALL_DISTANCES = 'all_distances'
 
   def __init__(self,
@@ -400,18 +394,4 @@ class KMeansClustering(estimator.Estimator):
 
   def cluster_centers(self):
     """Returns the cluster centers."""
-
-    # TODO(ccolby): Fix this clunky code once cl/168262087 is submitted.
-    # Discussion: go/estimator-get-variable-value
-    class RunOnceHook(session_run_hook.SessionRunHook):
-      """Stops after a single run."""
-
-      def after_run(self, run_context, run_values):
-        del run_values  # unused
-        run_context.request_stop()
-
-    result = self.predict(
-        input_fn=lambda: (constant_op.constant([], shape=[0, 1]), None),
-        predict_keys=[KMeansClustering.CLUSTERS],
-        hooks=[RunOnceHook()])
-    return np.array([r[KMeansClustering.CLUSTERS] for r in result])
+    return self.get_variable_value(clustering_ops.CLUSTERS_VAR_NAME)
diff --git a/tensorflow/contrib/learn/python/learn/estimators/kmeans.py b/tensorflow/contrib/learn/python/learn/estimators/kmeans.py
index b4d9c3fc6f..a92302420f 100644
--- a/tensorflow/contrib/learn/python/learn/estimators/kmeans.py
+++ b/tensorflow/contrib/learn/python/learn/estimators/kmeans.py
@@ -106,7 +106,7 @@ def _kmeans_clustering_model_fn(features, labels, mode, params, config):
   """Model function for KMeansClustering estimator."""
   assert labels is None, labels
   (all_scores, model_predictions, losses,
-   is_initialized, _, init_op, training_op) = clustering_ops.KMeans(
+   is_initialized, init_op, training_op) = clustering_ops.KMeans(
        _parse_tensor_or_dict(features),
        params.get('num_clusters'),
        initial_clusters=params.get('training_initial_clusters'),
-- 
GitLab


From 5eaefbabce16bffeeb4b19cee9890b1aeccabb09 Mon Sep 17 00:00:00 2001
From: Frank Chen <frankchn@google.com>
Date: Fri, 6 Oct 2017 11:44:25 -0700
Subject: [PATCH 127/909] Merge changes from github. END_PUBLIC

---
Commit ee0fdc296 authored by Gunhan Gulsoy<gunan@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Add noasan tag to estimator_test

PiperOrigin-RevId: 171075499

---
Commit a02116882 authored by Justin Lebar<jlebar@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
[XLA:CPU] Put the HLO name in IR values that hold the HLO's value.

PiperOrigin-RevId: 171075449

---
Commit 89aaac4bc authored by A. Unique TensorFlower<gardener@tensorflow.org>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Allow Layer.add_update() in Eager mode.

PiperOrigin-RevId: 171070861

---
Commit 840dcae57 authored by Amit Patankar<amitpatankar@google.com>
Committed by gunan<gunan@google.com>:
Updating the install sources file with a supported configs table (#13450)

* Updating the install sources file with a supported configs page.

* Implementing Gunan's suggestions.

* Adding GCC string to Linux compiler.

* Updating the bazel/cmake column.

---
Commit 89df2e336 authored by Igor Saprykin<isaprykin@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Add the 'is_the_final_export' signal to Exporters. Use them in training.

When the training ends, the final export is performed via `Exporter.export()` call.  That final export is going to have is_the_final_export parameter being set to true.

If `TrainSpec.max_steps` is `None`, then "when training ends" is undefined.  We are going to train forever.  In that case, `is_the_final_export` is going to be always False.  I added a note about it.

PiperOrigin-RevId: 171070760

---
Commit 4486b4f69 authored by Akshay Agrawal<akshayka@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Make graph_callable compatible with functions that do not return anything

PiperOrigin-RevId: 171067061

---
Commit 39565c0cb authored by Martin Wicke<wicke@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Publish train_and_evaluate and associated classes.

PiperOrigin-RevId: 171066379

---
Commit 3b4477000 authored by Saurabh Saxena<srbs@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Make VariantTensorData::tensors_size() const.

PiperOrigin-RevId: 171063397

---
Commit 53cc63a2d authored by Dhananjay Nakrani<dhananjayn@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
[part 1] Add support for int32 & int64 in RandomPoissonOp.

This computes int32/int64-precision poisson samples with double precision intermediate calculations (same as it's done for `half`) respectively.

part 2 will switch over python calls to new op once forward compatibility period has passed.

PiperOrigin-RevId: 171058336

---
Commit 70fc9bf9b authored by Asim Shankar<ashankar@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Java: Add support for loading op libraries dynamically.

This change adds the equivalent of tf.load_op_library in Python to Java.
(https://github.com/tensorflow/tensorflow/commit/5c7f9e316d8c7735308a217310350d416d7498cc
 was required to make this possible)

Though, TensorFlow.loadLibrary() is likely to fail on Windows as symbols
required by custom op libraries (those exported by the tensorflow_framework library)
are not exported by the monolithic JNI library yet.

This should help with #10454 and #13476

PiperOrigin-RevId: 171054707

---
Commit e7c53698e authored by A. Unique TensorFlower<gardener@tensorflow.org>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Internal cleanup

PiperOrigin-RevId: 171053770

---
Commit cc8ee6c0f authored by Alexandre Passos<apassos@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Fast path for tf.conj when it should be pass-through.

PiperOrigin-RevId: 171053662

---
Commit c41dbc3c1 authored by A. Unique TensorFlower<gardener@tensorflow.org>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Adding TF Boosted trees regression example on boston dataset, minor fix for mnist example.

PiperOrigin-RevId: 171052367

---
Commit d66e77f7c authored by Mustafa Ispir<ispir@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Added get variable utils to tf.estimator.Estimator.

PiperOrigin-RevId: 171052121

---
Commit 083bd5dde authored by Asim Shankar<ashankar@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Java: Add support for loading op libraries dynamically.

This change adds the equivalent of tf.load_op_library in Python to Java.
(https://github.com/tensorflow/tensorflow/commit/5c7f9e316d8c7735308a217310350d416d7498cc
 was required to make this possible)

Though, TensorFlow.loadLibrary() is likely to fail on Windows as symbols
required by custom op libraries (those exported by the tensorflow_framework library)
are not exported by the monolithic JNI library yet.

This should help with #10454 and #13476

PiperOrigin-RevId: 171054707

---
Commit 2fe6cf285 authored by A. Unique TensorFlower<gardener@tensorflow.org>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Internal cleanup

PiperOrigin-RevId: 171053770

---
Commit 15155493b authored by Alexandre Passos<apassos@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Fast path for tf.conj when it should be pass-through.

PiperOrigin-RevId: 171053662

---
Commit 6c954d0b3 authored by A. Unique TensorFlower<gardener@tensorflow.org>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Adding TF Boosted trees regression example on boston dataset, minor fix for mnist example.

PiperOrigin-RevId: 171052367

---
Commit ad69076eb authored by Mustafa Ispir<ispir@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Added get variable utils to tf.estimator.Estimator.

PiperOrigin-RevId: 171052121

---
Commit 3cf41b2ed authored by A. Unique TensorFlower<gardener@tensorflow.org>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Test save/restore variable from graph_callable.

PiperOrigin-RevId: 171051237

---
Commit cf17ec96e authored by Yangzihao Wang<yangzihao@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Add V2 versions of output window size computation functions for convolution.
These V2 versions take arbitrary dilation rates.
In preparation for the support of native cudnn dilated convolution.

PiperOrigin-RevId: 171048878

---
Commit 491584ff4 authored by Asim Shankar<ashankar@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
eager: Always run dataset iterator operations on CPU.

It has no kernels for other devices.
With an explicit "tf.device()" before invoking the kernel we ensure
that Iterator.next() functions even when placed inside a:

with tf.device("/device:GPU:0")

PiperOrigin-RevId: 171048558

---
Commit 3b354016e authored by Igor Saprykin<isaprykin@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Rename SavedModelExporter to LatestExporter.

PiperOrigin-RevId: 171048345

---
Commit 943c6d7af authored by Jianwei Xie<xiejw@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
errors out if the evaluator has task id > 0.

PiperOrigin-RevId: 171047652

---
Commit 8c9ef4466 authored by Mark Heffernan<meheff@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Expand set of 64-bit type tests in LocalClientExecuteTest.ShapeBufferToLiteralConversion64bit and factor out into their own test.

PiperOrigin-RevId: 171043047

---
Commit cc521eb06 authored by Benoit Steiner<bsteiner@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Place all the nodes created by the trivial_test_graph_input_yielder

PiperOrigin-RevId: 171045878

---
Commit 9b9301240 authored by A. Unique TensorFlower<gardener@tensorflow.org>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
[XLA:CPU] Factor out parallel task assignment from cpu parallelization prep (no functional changes).

PiperOrigin-RevId: 171045137

---
Commit 558d878d9 authored by Allen Lavoie<allenl@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
TFTS: Move normalization to the base class, start using it for state space models

Preivously, state space models adjusted their priors based on the data
(e.g. setting initial variances to match sample variance) but did not normalize
the data itself. When the data has a rather extreme scale, this runs into
precision issues. After this CL, state space models will first normalize, then
use adjusted statistics on top of that normalization to estimate initial
observation/transition noise.

Also fixes an issue where start-of-series statistics were incorrect for the first
batch (which only shows up with large input scales).

PiperOrigin-RevId: 171044863

---
Commit 266f77156 authored by Mark Heffernan<meheff@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Expand set of 64-bit type tests in LocalClientExecuteTest.ShapeBufferToLiteralConversion64bit and factor out into their own test.

PiperOrigin-RevId: 171043047

---
Commit c9915d1a2 authored by Shanqing Cai<cais@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
[tf-signal] Fix pip tests by including test_util in signal_py

PiperOrigin-RevId: 171042732

---
Commit f8550f4e9 authored by Mark Heffernan<meheff@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Expand set of 64-bit type tests in LocalClientExecuteTest.ShapeBufferToLiteralConversion64bit and factor out into their own test.

PiperOrigin-RevId: 171043047

---
Commit 87dc532cd authored by Shanqing Cai<cais@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
[tf-signal] Fix pip tests by including test_util in signal_py

PiperOrigin-RevId: 171042732

---
Commit 0578dd65e authored by A. Unique TensorFlower<gardener@tensorflow.org>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Add more debugging output for XLA send/recv.

PiperOrigin-RevId: 171041978

---
Commit 23992bb09 authored by A. Unique TensorFlower<gardener@tensorflow.org>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Several minor documentation fixes.

PiperOrigin-RevId: 171038610

---
Commit af14ed3f3 authored by Jianwei Xie<xiejw@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Some docstring twists and argument validations.

PiperOrigin-RevId: 171037949

---
Commit 6b90a65f6 authored by Mark Heffernan<meheff@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Remove "hybrid" HloModuleConfig option. The option was used to generate executables which only generated the array values of tuple-shaped outputs, not the tuple index tables.. With cl/170133015, ShapedBuffers which hold the computation output now have materialized tuples with these index tables so this option is no longer desired or necessary.

No functional change. Just cleanup.

PiperOrigin-RevId: 171035738

---
Commit 41a0264ab authored by Mustafa Ispir<ispir@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Added utilities to make global step reading deterministic. Used them in Estimator.
Enabled/Fixed some tests.

PiperOrigin-RevId: 171035291

---
Commit 9d7843c0a authored by Skye Wanderman-Milne<skyewm@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Add optional unused_input_map_keys output param to ImportGraphDef

This is a more general feature than that in the Python importer, which
raises an exception if the input map contains unused names.

PiperOrigin-RevId: 171029316

---
Commit 4f10a6597 authored by Mark Heffernan<meheff@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Add vlogging of HloModule before and after fusion.

PiperOrigin-RevId: 171029054

---
Commit 9e658545a authored by Reed Wanderman-Milne<reedwm@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Document what dtype tf.image.resize_images returns.

For consistency, tf.image.resize_images now will always return a float32 when method != ResizeMethod.NEAREST_NEIGHBOR. Before, it returned the same dtype as its input if it could be determined statically that the height and width would not be changed.

PiperOrigin-RevId: 171028825

---
Commit 4d70239f0 authored by Jianwei Xie<xiejw@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Replace the contrib FC with core FC  in canned Estimator docstring.

PiperOrigin-RevId: 171027602

---
Commit 6a1b867ff authored by Jianwei Xie<xiejw@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Adds the docstring with details for tf.estimator.train_and_evaluate

PiperOrigin-RevId: 171027527

---
Commit 7209c1602 authored by Peter Hawkins<phawkins@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
[TF:XLA] Mark IdentityN as CompilationOnly().

PiperOrigin-RevId: 171025171

---
Commit 8e22eb874 authored by FAIJUL<md.faijul.amin@intel.com>
Committed by Benoit Steiner<benoitsteiner@users.noreply.github.com>:
Eigen BiasAdd and BiasAddGrad Fix for NCHW Format. (#13158)

---
Commit 7db7a890c authored by Jingyue Wu<jingyue@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
[Grappler] Move InferOutputShapes to GraphProperties.

So it can be used by other optimizers. No functional changes.

PiperOrigin-RevId: 171010106

---
Commit 2114fd51e authored by Peter Hawkins<phawkins@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
[TF:XLA] Improve numerical stability of SoftPlus.

PiperOrigin-RevId: 171003559

---
Commit 727d6270f authored by A. Unique TensorFlower<gardener@tensorflow.org>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Fix race condition in TensorForest tree traversal.

PiperOrigin-RevId: 170990425

---
Commit d016cb020 authored by Suharsh Sivakumar<suharshs@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Fix c++ gradients issue where multiple dependent outputs result in incorrect answer.

The issue is that we incorrectly calculate the pending num_expected_backprops for outputs nodes when one output transitively depends on another. this is because we use output nodes as an indicator of when we need to end our traversal. Instead we should only use output nodes that don't transitively get consumed by other output nodes as end indicators for our traversal. This change implements that fix.

Fixes #13190

PiperOrigin-RevId: 170971937

---
Commit 5405f3bd7 authored by gunan<gunan@google.com>
Committed by Frank Chen<frankchn@gmail.com>:
Fix tf-signal tests on pip packages. (#13483)

---
Commit f9f037c1c authored by Eugene Brevdo<ebrevdo@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Bugfix to LSTMBlockCell and friends: clipping is off by default.

* Rename broken API argu clip_cell boolean to cell_clip value.
* Make default no clipping.

PiperOrigin-RevId: 170960975

---
Commit bfaaefa9e authored by Frank Chen<frankchn@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Update APIs for TPU Cluster Resolver to remove the custom API definition and instead use a standard definition file stored in GCS.

PiperOrigin-RevId: 170960877

---
Commit c31c118a3 authored by Ian Langmore<langmore@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Extend tf.contrib.bijector API to handle some non-injective transforms.
AbsoluteValue Bijector added to contrib/distributions/bijectors/
TransformedDistribution udpated to handle some non-injective transforms.

PiperOrigin-RevId: 170960054

---
Commit 664dd0859 authored by Frank Chen<frankchn@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Disable cluster_function_library_runtime_test on Mac OS as it is currently failing with an Unimplemented error

PiperOrigin-RevId: 170958505

---
Commit 6af7ab97a authored by Mahmoud Abuzaina<mahmoud.abuzaina@intel.com>
Committed by gunan<gunan@google.com>:
MKL-DNN open source integration. (#13135)

* MKL-DNN conv and build integration

* Adding new files that were mistakenly missing from the PR

* Minor change in the pip package build file

* Added missing #include

* Fixed a linking failure when running the bazel test

* Fixing BUILD file format

* Using -fopenmp for building mkl_dnn only when running on linux

* Fixing build rule attribute value

* Removing unnecessary deps from mkl test rule

* Removed deps on mkl-dnn when not building with --config=mkl

---
Commit 93fa1af76 authored by Akshay Agrawal<akshayka@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Make graph_callable, defun tf_decorators

PiperOrigin-RevId: 170948777

---
Commit b39525785 authored by Mustafa Ispir<ispir@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Added comment re:behavior of listener in case of multiple saver hooks.

PiperOrigin-RevId: 170946536

---
Commit de14fcbb6 authored by Igor Saprykin<isaprykin@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Support evaluation in `_TrainingExecutor.run_master()`.

This CL aims to address the following TODO:

    # TODO(b/66720832): Once listener API is added into Estimator.train, the
    # eval and export process should be wrapped as a listener and passed to
    # _start_distributed_training. The expected behavior should be
    # 1. The export is invoked after each intermediate evaluation.
    # 2. The evaluation and export should be invoked correctly at the end of
    # training. This should be fine if the listener works as intended (it will
    # send the `after_save` signal for the final ckpt saving).

1. is achieved as follows:
  a. saving_evaluators are added to the CheckpointSaverHook's listeners inside the Estimator.
  b. MonitoredSession calls after_run() of CheckpointSaverHook, which in turn calls after_save on the listeners.

2. is achieved in a similar way, but when MonitoredSession calls .end() on CheckpointSaverHook.

PiperOrigin-RevId: 170945961

---
Commit d4ea993ca authored by Alexandre Passos<apassos@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Removes unnecessary eager-mode call to convert_to_tensor in record_gradient.

PiperOrigin-RevId: 170944265

---
Commit add6d2d03 authored by RJ Ryan<rjryan@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
[tf-signal] Use tf.spectral.dct in mfccs_from_log_mel_spectrograms instead of a private implementation.

PiperOrigin-RevId: 170943986

---
Commit b959da92f authored by Jiri Simsa<jsimsa@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Fixing CPU implementation of parallel_stack for tensors with non-zero rank.

PiperOrigin-RevId: 170942814

---
Commit 4cf61262a authored by A. Unique TensorFlower<gardener@tensorflow.org>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Improve TFGAN documentation.

PiperOrigin-RevId: 170940188

---
Commit 0068086b9 authored by A. Unique TensorFlower<gardener@tensorflow.org>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Introduce `tf.data` namespace.

PiperOrigin-RevId: 170939033

---
Commit 0c8dbc1fd authored by Alexandre Passos<apassos@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
matmul uses shape_tuple internally

PiperOrigin-RevId: 170938790

---
Commit ad37fa81f authored by Igor Saprykin<isaprykin@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Refactor ExportStrategies into Exporters.

This design eliminates some indirection.  Instead of combining an `export_fn` with `make_export_strategy` call to arrive at an ExportStrategy that is going to call the supplied `export_fn` inside its `export` call with Exporters one just defines the `export` call in an Exporter.

PiperOrigin-RevId: 170936640

---
Commit b925f8553 authored by Alexandre Passos<apassos@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Fast-path for EagerTensorBase.dtype

PiperOrigin-RevId: 170933005

---
Commit 08e266d9b authored by A. Unique TensorFlower<gardener@tensorflow.org>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Pass activity_regularizer to __init__ instead of using the (now
deprecated) property setter.

PiperOrigin-RevId: 170932807

---
Commit b002c8b7d authored by Jingyue Wu<jingyue@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
[Grappler] Fold chains of reshapes.

Reshape(Reshape(input, shape1), shape2) is equivalent to Reshape(input,
shape2).

PiperOrigin-RevId: 170932278

---
Commit 075d1d13b authored by horance<horance@aliyun.com>
Committed by Frank Chen<frankchn@gmail.com>:
remove warning for forward decl (#13459)

---
Commit 931609fcf authored by Ryohei Kuroki<ryohei.kuroki@gmail.com>
Committed by Frank Chen<frankchn@gmail.com>:
Remove unnecessary specification for default kernel name (#13465)

---
Commit 94463f521 authored by Akshay Agrawal<akshayka@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Preserve target function signature in custom_gradient decorator

PiperOrigin-RevId: 170931715

---
Commit 681056636 authored by A. Unique TensorFlower<gardener@tensorflow.org>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Internal change to simplify prediction ops.
 - it no longer returns predictions_no_dropout, which is mostly for debugging purpose.
 - as a consequence, MultipleAdditiveTrees::Predict() doesn't return prediction_no_dropout, and it accept trees_to_include indexes intead of trees_to_drop indexes.

PiperOrigin-RevId: 170926422

---
Commit d6e963b82 authored by Asim Shankar<ashankar@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
SYCL: Fix build breakage introduced in
https://github.com/tensorflow/tensorflow/commit/f0e8c545e0196b8b48ce0ad0f116df97d980d1f1

Fixes #13350

PiperOrigin-RevId: 170923862

---
Commit 5123f2971 authored by A. Unique TensorFlower<gardener@tensorflow.org>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Internal cleanup.

PiperOrigin-RevId: 170922297

---
Commit d0c76cd18 authored by Igor Saprykin<isaprykin@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Handle the absence of a fresh eval checkpoint in `run_local`.

It is ~unexpected condition for an eval checkpoint to not be available after a train call to the estimator.  There is a corner case when it is possible, but that's going to be resolved soon.

This case is handled for continuous (distributed) evaluation differently.  Instead of erroring out, we skip evaluation runs.  That behavior is captured in the `test_skip_evaluation_due_to_ckpt` test.

PiperOrigin-RevId: 170919925

---
Commit 435b31b9f authored by Gunhan Gulsoy<gunan@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
BEGIN_PUBLIC
Automated g4 rollback of changelist 170892257

PiperOrigin-RevId: 171321707
---
 README.md                                     |   6 +-
 .../compiler/jit/kernels/xla_launch_op.cc     |  15 +
 .../xla/service/gpu/convolution_thunk.cc      |  51 +-
 .../xla/service/gpu/convolution_thunk.h       |   4 +-
 .../android/TensorFlowInferenceInterface.java |  23 +-
 .../quantiles/weighted_quantiles_summary.h    |   2 +-
 .../kernel_tests/batch_dataset_op_test.py     |  40 ++
 .../contrib/data/python/ops/dataset_ops.py    |   2 +-
 tensorflow/contrib/deprecated/__init__.py     |   2 +-
 .../contrib/ffmpeg/default/ffmpeg_lib.cc      |  10 +-
 .../framework/python/framework/tensor_util.py |   6 +-
 .../fused_conv2d_bias_activation_op.cc        |  57 +--
 tensorflow/contrib/memory_stats/__init__.py   |   2 +
 .../memory_stats/kernels/memory_stats_ops.cc  |  22 +
 .../memory_stats/ops/memory_stats_ops.cc      |   4 +
 .../kernel_tests/memory_stats_ops_test.py     |  22 +-
 .../python/ops/memory_stats_ops.py            |   5 +
 .../resampler/kernels/resampler_ops.cc        |   2 +-
 tensorflow/contrib/rnn/python/ops/rnn_cell.py |  10 +-
 .../contrib/seq2seq/python/ops/helper.py      |   2 +-
 tensorflow/contrib/signal/BUILD               |   1 +
 .../python/slim/data/tfexample_decoder.py     |   5 +-
 .../slim/data/tfexample_decoder_test.py       |  45 +-
 .../timeseries/python/timeseries/BUILD        |  48 +-
 .../timeseries/python/timeseries/ar_model.py  |   2 +-
 .../python/timeseries/estimators.py           |   7 +-
 .../timeseries/python/timeseries/head.py      | 375 +++++++++++++++
 .../timeseries/python/timeseries/head_test.py | 267 +++++++++++
 .../python/timeseries/model_utils.py          | 319 -------------
 .../python/timeseries/model_utils_test.py     | 236 ---------
 .../python/timeseries/saved_model_utils.py    |   3 +-
 tensorflow/core/BUILD                         |  22 +-
 tensorflow/core/graph/mkl_graph_util.h        | 128 +++++
 tensorflow/core/graph/mkl_layout_pass.cc      |   2 +-
 tensorflow/core/graph/mkl_layout_pass_test.cc |   2 +-
 .../core/graph/mkl_tfconversion_pass.cc       |   2 +-
 .../core/graph/mkl_tfconversion_pass_test.cc  |   2 +-
 tensorflow/core/kernels/BUILD                 |  34 +-
 tensorflow/core/kernels/bias_op.cc            | 159 ++++---
 .../core/kernels/conv_grad_filter_ops.cc      |  55 +--
 .../core/kernels/conv_grad_input_ops.cc       |  53 +--
 tensorflow/core/kernels/conv_grad_ops_3d.cc   | 109 ++---
 tensorflow/core/kernels/conv_ops.cc           |  51 +-
 tensorflow/core/kernels/conv_ops_3d.cc        |  51 +-
 tensorflow/core/kernels/decode_csv_op.cc      |  19 +-
 .../dense_to_sparse_batch_dataset_op.cc       |  45 +-
 .../core/kernels/mkl_conv_grad_filter_ops.cc  | 181 +++++++
 .../core/kernels/mkl_conv_grad_input_ops.cc   | 190 +++++++-
 tensorflow/core/kernels/mkl_conv_ops.cc       | 213 +++++++++
 tensorflow/core/kernels/mkl_conv_ops.h        | 308 ++++++++++++
 .../core/kernels/mkl_cwise_ops_common.cc      |   2 +-
 tensorflow/core/lib/strings/numbers.cc        |   2 +-
 tensorflow/core/ops/dataset_ops.cc            |   3 +-
 tensorflow/core/ops/nn_ops.cc                 |  84 ++--
 tensorflow/core/ops/nn_ops_test.cc            |  49 --
 tensorflow/core/ops/parsing_ops.cc            |   2 +
 tensorflow/core/util/mkl_util.h               | 401 ++++++++++++----
 .../docs_src/install/install_sources.md       |  38 ++
 .../org/tensorflow/demo/SpeechActivity.java   |   8 +-
 .../tutorials/word2vec/word2vec_basic.py      |   2 +-
 .../go/example_inception_inference_test.go    |   2 +-
 tensorflow/go/tensor.go                       |  48 +-
 tensorflow/go/tensor_test.go                  |  10 +
 .../java/src/gen/perl/tftypes-runall.pl       |   2 +-
 tensorflow/java/src/gen/perl/tftypes.pl       | 102 ++--
 .../java/src/gen/resources/Tensors.java.tmpl  |  31 ++
 tensorflow/java/src/gen/resources/tftypes.csv |  42 +-
 .../main/java/org/tensorflow/DataType.java    |  39 +-
 .../src/main/java/org/tensorflow/Graph.java   |   7 +-
 .../src/main/java/org/tensorflow/Input.java   |   4 +-
 .../java/org/tensorflow/NativeLibrary.java    |   9 +-
 .../src/main/java/org/tensorflow/Operand.java |  12 +-
 .../main/java/org/tensorflow/Operation.java   |  18 +-
 .../java/org/tensorflow/OperationBuilder.java |  14 +-
 .../src/main/java/org/tensorflow/Output.java  |  12 +-
 .../java/org/tensorflow/SavedModelBundle.java |   5 +-
 .../src/main/java/org/tensorflow/Session.java |  34 +-
 .../src/main/java/org/tensorflow/Tensor.java  | 241 +++++++---
 .../src/main/java/org/tensorflow/Tensors.java | 447 ++++++++++++++++++
 .../org/tensorflow/examples/LabelImage.java   |  79 ++--
 .../main/java/org/tensorflow/op/Operands.java |   8 +-
 .../java/org/tensorflow/op/core/Constant.java |  34 +-
 .../main/java/org/tensorflow/types/UInt8.java |  21 +
 .../org/tensorflow/types/package-info.java    |  16 +-
 .../test/java/org/tensorflow/GraphTest.java   |   1 -
 .../org/tensorflow/OperationBuilderTest.java  |  25 +-
 .../java/org/tensorflow/OperationTest.java    |  19 +-
 .../test/java/org/tensorflow/SessionTest.java |  41 +-
 .../test/java/org/tensorflow/ShapeTest.java   |   2 +-
 .../test/java/org/tensorflow/TensorTest.java  |  99 ++--
 .../test/java/org/tensorflow/TestUtil.java    |  24 +-
 .../java/org/tensorflow/op/OperandsTest.java  |   7 +-
 .../org/tensorflow/op/PrimitiveOpTest.java    |   2 +-
 .../java/org/tensorflow/op/ScopeTest.java     | 128 +++--
 .../org/tensorflow/op/core/ConstantTest.java  |  22 +-
 tensorflow/python/debug/lib/debug_graphs.py   |   4 +-
 .../inputs/queues/feeding_functions.py        |   2 +-
 .../keras/_impl/keras/engine/topology_test.py |   2 +-
 .../kernel_tests/conv2d_transpose_test.py     |  14 +
 .../python/kernel_tests/decode_csv_op_test.py |  11 +
 .../kernel_tests/summary_tensor_op_test.py    |   2 +-
 tensorflow/python/ops/hidden_ops.txt          |   1 +
 tensorflow/python/ops/parsing_ops.py          |  39 ++
 tensorflow/stream_executor/cuda/cuda_dnn.cc   |  90 ++--
 tensorflow/stream_executor/cuda/cuda_dnn.h    |  12 +-
 tensorflow/stream_executor/dnn.cc             |  12 +-
 tensorflow/stream_executor/dnn.h              |  12 +-
 tensorflow/stream_executor/platform.h         |   2 +-
 tensorflow/stream_executor/stream.h           |   2 +-
 .../stream_executor/stream_executor_pimpl.cc  |  22 +-
 .../stream_executor/stream_executor_pimpl.h   |   9 +-
 tensorflow/tensorflow.bzl                     |  35 +-
 tensorflow/tools/api/golden/tensorflow.pbtxt  |   2 +-
 .../tools/ci_build/install/install_golang.sh  |   2 +-
 tensorflow/tools/docker/Dockerfile.devel-gpu  |   4 +-
 .../tools/docker/jupyter_notebook_config.py   |   1 -
 tensorflow/tools/docs/parser.py               |   4 +-
 .../gen_proto_text_functions_lib_test.cc      |   9 +-
 tensorflow/workspace.bzl                      |  17 +-
 third_party/gpus/cuda_configure.bzl           |   2 +-
 third_party/mkl_dnn/BUILD                     |   1 +
 third_party/mkl_dnn/mkldnn.BUILD              |  25 +
 122 files changed, 4102 insertions(+), 1655 deletions(-)
 create mode 100644 tensorflow/contrib/timeseries/python/timeseries/head.py
 create mode 100644 tensorflow/contrib/timeseries/python/timeseries/head_test.py
 create mode 100644 tensorflow/core/graph/mkl_graph_util.h
 create mode 100644 tensorflow/core/kernels/mkl_conv_ops.h
 create mode 100644 tensorflow/java/src/gen/resources/Tensors.java.tmpl
 create mode 100644 tensorflow/java/src/main/java/org/tensorflow/Tensors.java
 create mode 100644 tensorflow/java/src/main/java/org/tensorflow/types/UInt8.java
 create mode 100644 third_party/mkl_dnn/BUILD
 create mode 100644 third_party/mkl_dnn/mkldnn.BUILD

diff --git a/README.md b/README.md
index 4cc53096e0..6339c57c95 100644
--- a/README.md
+++ b/README.md
@@ -48,9 +48,9 @@ GPU packages on all platforms will arrive soon!
 * Linux CPU-only: [Python 2](https://ci.tensorflow.org/view/tf-nightly/job/tf-nightly-linux/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=cpu-slave/lastSuccessfulBuild/artifact/pip_test/whl/tf_nightly-1.head-cp27-none-linux_x86_64.whl) ([build history](https://ci.tensorflow.org/view/tf-nightly/job/tf-nightly-linux/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=cpu-slave/)) / [Python 3.4](https://ci.tensorflow.org/view/tf-nightly/job/tf-nightly-linux/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3,label=cpu-slave/lastSuccessfulBuild/artifact/pip_test/whl/tf_nightly-1.head-cp34-cp34m-linux_x86_64.whl) ([build history](https://ci.tensorflow.org/view/tf-nightly/job/tf-nightly-linux/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3,label=cpu-slave/)) / [Python 3.5](https://ci.tensorflow.org/view/tf-nightly/job/tf-nightly-linux/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3.5,label=cpu-slave/lastSuccessfulBuild/artifact/pip_test/whl/tf_nightly-1.head-cp35-cp35m-linux_x86_64.whl) ([build history](https://ci.tensorflow.org/view/tf-nightly/job/tf-nightly-linux/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3.5,label=cpu-slave/))
 * Linux GPU: [Python 2](https://ci.tensorflow.org/view/tf-nightly/job/tf-nightly-linux/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=gpu-linux/42/artifact/pip_test/whl/tf_nightly_gpu-1.head-cp27-none-linux_x86_64.whl) ([build history](https://ci.tensorflow.org/view/tf-nightly/job/tf-nightly-linux/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=gpu-linux/)) / [Python 3.4](https://ci.tensorflow.org/view/tf-nightly/job/tf-nightly-linux/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3,label=gpu-linux/lastSuccessfulBuild/artifact/pip_test/whl/tf_nightly_gpu-1.head-cp34-cp34m-linux_x86_64.whl) ([build history](https://ci.tensorflow.org/view/tf-nightly/job/tf-nightly-linux/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3,label=gpu-linux/)) / [Python 3.5](https://ci.tensorflow.org/view/tf-nightly/job/tf-nightly-linux/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3.5,label=gpu-linux/lastSuccessfulBuild/artifact/pip_test/whl/tf_nightly_gpu-1.head-cp35-cp35m-linux_x86_64.whl) ([build history](https://ci.tensorflow.org/view/tf-nightly/job/tf-nightly-linux/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3.5,label=gpu-linux/))
 * Mac CPU-only: [Python 2](https://ci.tensorflow.org/view/tf-nightly/job/tf-nightly-mac/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=mac-slave/lastSuccessfulBuild/artifact/pip_test/whl/tf_nightly-1.head-py2-none-any.whl) ([build history](https://ci.tensorflow.org/view/tf-nightly/job/tf-nightly-mac/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=mac-slave/)) / [Python 3](https://ci.tensorflow.org/view/tf-nightly/job/tf-nightly-mac/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3,label=mac-slave/lastSuccessfulBuild/artifact/pip_test/whl/tf_nightly-1.head-py3-none-any.whl) ([build history](https://ci.tensorflow.org/view/tf-nightly/job/tf-nightly-mac/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3,label=mac-slave/))
-* Windows CPU-only: [Python 3.5 64-bit](http://ci.tensorflow.org/view/tf-nightly/job/tf-nightly-windows/M=windows,PY=35/lastSuccessfulBuild/artifact/cmake_build/tf_python/dist/tf_nightly-1.head-cp35-cp35m-win_amd64.whl) ([build history](http://ci.tensorflow.org/view/tf-nightly/job/tf-nightly-windows/M=windows,PY=35/)) / [Python 3.6 64-bit](http://ci.tensorflow.org/view/tf-nightly/job/tf-nightly-windows/M=windows,PY=36/lastSuccessfulBuild/artifact/cmake_build/tf_python/dist/tf_nightly-1.head-cp36-cp36m-win_amd64.whl) ([build history](http://ci.tensorflow.org/view/tf-nightly/job/tf-nightly-windows/M=windows,PY=36/))
-* Windows GPU: Coming soon!
-* Android: [demo APK](https://ci.tensorflow.org/view/Nightly/job/nightly-android/lastSuccessfulBuild/artifact/out/tensorflow_demo.apk), [native libs](http://ci.tensorflow.org/view/Nightly/job/nightly-android/lastSuccessfulBuild/artifact/out/native/)
+* Windows CPU-only: [Python 3.5 64-bit](https://ci.tensorflow.org/view/tf-nightly/job/tf-nightly-windows/M=windows,PY=35/lastSuccessfulBuild/artifact/cmake_build/tf_python/dist/tf_nightly-1.head-cp35-cp35m-win_amd64.whl) ([build history](https://ci.tensorflow.org/view/tf-nightly/job/tf-nightly-windows/M=windows,PY=35/)) / [Python 3.6 64-bit](https://ci.tensorflow.org/view/tf-nightly/job/tf-nightly-windows/M=windows,PY=36/lastSuccessfulBuild/artifact/cmake_build/tf_python/dist/tf_nightly-1.head-cp36-cp36m-win_amd64.whl) ([build history](http://ci.tensorflow.org/view/tf-nightly/job/tf-nightly-windows/M=windows,PY=36/))
+* Windows GPU: [Python 3.5 64-bit](https://ci.tensorflow.org/view/tf-nightly/job/tf-nightly-windows/M=windows-gpu,PY=35/lastSuccessfulBuild/artifact/cmake_build/tf_python/dist/tf_nightly_gpu-1.head-cp35-cp35m-win_amd64.whl) ([build history](https://ci.tensorflow.org/view/tf-nightly/job/tf-nightly-windows/M=windows-gpu,PY=35/)) / [Python 3.6 64-bit](https://ci.tensorflow.org/view/tf-nightly/job/tf-nightly-windows/M=windows-gpu,PY=36/lastSuccessfulBuild/artifact/cmake_build/tf_python/dist/tf_nightly_gpu-1.head-cp36-cp36m-win_amd64.whl) ([build history](http://ci.tensorflow.org/view/tf-nightly/job/tf-nightly-windows/M=windows-gpu,PY=36/))
+* Android: [demo APK](https://ci.tensorflow.org/view/Nightly/job/nightly-android/lastSuccessfulBuild/artifact/out/tensorflow_demo.apk), [native libs](https://ci.tensorflow.org/view/Nightly/job/nightly-android/lastSuccessfulBuild/artifact/out/native/)
 ([build history](https://ci.tensorflow.org/view/Nightly/job/nightly-android/))
 
 #### *Try your first TensorFlow program*
diff --git a/tensorflow/compiler/jit/kernels/xla_launch_op.cc b/tensorflow/compiler/jit/kernels/xla_launch_op.cc
index 1b5dd558dd..27c5da08c1 100644
--- a/tensorflow/compiler/jit/kernels/xla_launch_op.cc
+++ b/tensorflow/compiler/jit/kernels/xla_launch_op.cc
@@ -52,6 +52,11 @@ class XlaAllocator : public xla::DeviceMemoryAllocator {
                                                 bool retry_on_failure) override;
   Status Deallocate(int device_ordinal, gpu::DeviceMemoryBase* mem) override;
 
+  // Register an Tensor (input or resource variable) with the allocator. If
+  // the operation returns an alias to one of its inputs, then the allocator
+  // needs to be able to handle it.
+  Status RegisterArgument(const Tensor* t);
+
   // Makes 'tensor' a wrapper around the data buffer at 'ptr'. The buffer is
   // interpreted as having data type 'dtype' and shape 'shape'.
   Status MakeTensorFromBuffer(gpu::DeviceMemoryBase buffer, DataType dtype,
@@ -103,6 +108,14 @@ xla::StatusOr<gpu::DeviceMemoryBase> XlaAllocator::Allocate(
   return gpu::DeviceMemoryBase(data, size);
 }
 
+Status XlaAllocator::RegisterArgument(const Tensor* t) {
+  void* data =
+      reinterpret_cast<void*>(const_cast<char*>(t->tensor_data().data()));
+  TF_RET_CHECK(data != nullptr);
+  tensors_[data] = *t;
+  return Status::OK();
+}
+
 Status XlaAllocator::Deallocate(int device_ordinal,
                                 gpu::DeviceMemoryBase* mem) {
   if (mem->opaque() != nullptr) {
@@ -284,6 +297,8 @@ void XlaLocalLaunchOp::Compute(OpKernelContext* ctx) {
             shape, client->platform(), client->default_device_ordinal(), dmem)
             .ConsumeValueOrDie();
     arg_ptrs[i] = arg_buffers[i].get();
+
+    OP_REQUIRES_OK(ctx, xla_allocator.RegisterArgument(t));
   }
 
   // Make the final parameter point at local_runtime_context.
diff --git a/tensorflow/compiler/xla/service/gpu/convolution_thunk.cc b/tensorflow/compiler/xla/service/gpu/convolution_thunk.cc
index 89145a9038..7dd242425c 100644
--- a/tensorflow/compiler/xla/service/gpu/convolution_thunk.cc
+++ b/tensorflow/compiler/xla/service/gpu/convolution_thunk.cc
@@ -256,9 +256,9 @@ tensorflow::Status ConvolutionThunk::Convolve(
       algorithm_config.algorithm_no_scratch().algo_id());
 }
 
-std::vector<AlgorithmDesc::Index> ConvolutionThunk::GetAlgorithms(
+std::vector<AlgorithmDesc> ConvolutionThunk::GetAlgorithms(
     se::StreamExecutor* stream_exec) const {
-  std::vector<AlgorithmDesc::Index> algorithms;
+  std::vector<AlgorithmDesc> algorithms;
   // TODO(yangzihao): Currently disable the use of winograd nonfused in XLA
   // by default. Should send in conv parameters and enable it when
   // ShouldIncludeWinogradNonfusedAlgo() returns true.
@@ -297,32 +297,27 @@ tensorflow::Status ConvolutionThunk::ConvolveWithTune(
 
     se::dnn::ProfileResult best_result;
     se::dnn::ProfileResult best_result_without_scratch;
-    std::vector<AlgorithmDesc::Index> algorithms =
-        GetAlgorithms(stream->parent());
-    for (bool use_tensor_ops : {false, true}) {
-      for (auto algo_index : algorithms) {
-        AlgorithmDesc algorithm(algo_index, use_tensor_ops);
-        ConvolveScratchAllocator scratch_allocator(
-            buffer_allocations.device_ordinal(),
-            buffer_allocations.memory_allocator());
-        se::dnn::ProfileResult profile_result;
-        bool launch_ok =
-            Convolve(input_descriptor, input_data, filter_descriptor,
-                     filter_data, output_descriptor, output_data,
-                     convolution_descriptor,
-                     se::dnn::AlgorithmConfig(algorithm, algorithm), stream,
-                     &scratch_allocator, &profile_result)
-                .ok();
-        if (launch_ok && profile_result.is_valid()) {
-          if (profile_result.elapsed_time_in_ms() <
-              best_result.elapsed_time_in_ms()) {
-            best_result = profile_result;
-          }
-          if (scratch_allocator.TotalAllocatedBytes() == 0 &&
-              profile_result.elapsed_time_in_ms() <
-                  best_result_without_scratch.elapsed_time_in_ms()) {
-            best_result_without_scratch = profile_result;
-          }
+    std::vector<AlgorithmDesc> algorithms = GetAlgorithms(stream->parent());
+    for (auto algorithm : algorithms) {
+      ConvolveScratchAllocator scratch_allocator(
+          buffer_allocations.device_ordinal(),
+          buffer_allocations.memory_allocator());
+      se::dnn::ProfileResult profile_result;
+      bool launch_ok =
+          Convolve(input_descriptor, input_data, filter_descriptor, filter_data,
+                   output_descriptor, output_data, convolution_descriptor,
+                   se::dnn::AlgorithmConfig(algorithm, algorithm), stream,
+                   &scratch_allocator, &profile_result)
+              .ok();
+      if (launch_ok && profile_result.is_valid()) {
+        if (profile_result.elapsed_time_in_ms() <
+            best_result.elapsed_time_in_ms()) {
+          best_result = profile_result;
+        }
+        if (scratch_allocator.TotalAllocatedBytes() == 0 &&
+            profile_result.elapsed_time_in_ms() <
+                best_result_without_scratch.elapsed_time_in_ms()) {
+          best_result_without_scratch = profile_result;
         }
       }
     }
diff --git a/tensorflow/compiler/xla/service/gpu/convolution_thunk.h b/tensorflow/compiler/xla/service/gpu/convolution_thunk.h
index 509719c1fe..13432301b2 100644
--- a/tensorflow/compiler/xla/service/gpu/convolution_thunk.h
+++ b/tensorflow/compiler/xla/service/gpu/convolution_thunk.h
@@ -115,9 +115,7 @@ class ConvolutionThunk : public Thunk {
       perftools::gputools::dnn::ProfileResult* profile_result);
 
   // Returns the convolve algorithms that can be used for this ConvolutionThunk.
-  // TODO(nluehr) GetAlgorithms should return AlgorithmDesc including both
-  // tensor-op and non-tensor-op variants.
-  std::vector<perftools::gputools::dnn::AlgorithmDesc::Index> GetAlgorithms(
+  std::vector<perftools::gputools::dnn::AlgorithmDesc> GetAlgorithms(
       perftools::gputools::StreamExecutor* stream_exec) const;
 
   // Fastest cuDNN convolution algorithm for this thunk learned from
diff --git a/tensorflow/contrib/android/java/org/tensorflow/contrib/android/TensorFlowInferenceInterface.java b/tensorflow/contrib/android/java/org/tensorflow/contrib/android/TensorFlowInferenceInterface.java
index 395dd6c5d2..80e03f2036 100644
--- a/tensorflow/contrib/android/java/org/tensorflow/contrib/android/TensorFlowInferenceInterface.java
+++ b/tensorflow/contrib/android/java/org/tensorflow/contrib/android/TensorFlowInferenceInterface.java
@@ -31,12 +31,13 @@ import java.nio.IntBuffer;
 import java.nio.LongBuffer;
 import java.util.ArrayList;
 import java.util.List;
-import org.tensorflow.DataType;
 import org.tensorflow.Graph;
 import org.tensorflow.Operation;
 import org.tensorflow.Session;
 import org.tensorflow.Tensor;
 import org.tensorflow.TensorFlow;
+import org.tensorflow.Tensors;
+import org.tensorflow.types.UInt8;
 
 /**
  * Wrapper over the TensorFlow API ({@link Graph}, {@link Session}) providing a smaller API surface
@@ -328,7 +329,7 @@ public class TensorFlowInferenceInterface {
    * destination has capacity, the copy is truncated.
    */
   public void feed(String inputName, byte[] src, long... dims) {
-    addFeed(inputName, Tensor.create(DataType.UINT8, dims, ByteBuffer.wrap(src)));
+    addFeed(inputName, Tensor.create(UInt8.class, dims, ByteBuffer.wrap(src)));
   }
 
   /**
@@ -337,7 +338,7 @@ public class TensorFlowInferenceInterface {
    * a Java {@code String} (which is a sequence of characters).
    */
   public void feedString(String inputName, byte[] src) {
-    addFeed(inputName, Tensor.create(src));
+    addFeed(inputName, Tensors.create(src));
   }
 
   /**
@@ -346,7 +347,7 @@ public class TensorFlowInferenceInterface {
    * arbitrary sequence of bytes, not a Java {@code String} (which is a sequence of characters).
    */
   public void feedString(String inputName, byte[][] src) {
-    addFeed(inputName, Tensor.create(src));
+    addFeed(inputName, Tensors.create(src));
   }
 
   // Methods for taking a native Tensor and filling it with src from Java native IO buffers.
@@ -403,7 +404,7 @@ public class TensorFlowInferenceInterface {
    * destination has capacity, the copy is truncated.
    */
   public void feed(String inputName, ByteBuffer src, long... dims) {
-    addFeed(inputName, Tensor.create(DataType.UINT8, dims, src));
+    addFeed(inputName, Tensor.create(UInt8.class, dims, src));
   }
 
   /**
@@ -544,7 +545,7 @@ public class TensorFlowInferenceInterface {
         "Model load took " + (endMs - startMs) + "ms, TensorFlow version: " + TensorFlow.version());
   }
 
-  private void addFeed(String inputName, Tensor t) {
+  private void addFeed(String inputName, Tensor<?> t) {
     // The string format accepted by TensorFlowInferenceInterface is node_name[:output_index].
     TensorId tid = TensorId.parse(inputName);
     runner.feed(tid.name, tid.outputIndex, t);
@@ -578,7 +579,7 @@ public class TensorFlowInferenceInterface {
     }
   }
 
-  private Tensor getTensor(String outputName) {
+  private Tensor<?> getTensor(String outputName) {
     int i = 0;
     for (String n : fetchNames) {
       if (n.equals(outputName)) {
@@ -591,7 +592,7 @@ public class TensorFlowInferenceInterface {
   }
 
   private void closeFeeds() {
-    for (Tensor t : feedTensors) {
+    for (Tensor<?> t : feedTensors) {
       t.close();
     }
     feedTensors.clear();
@@ -599,7 +600,7 @@ public class TensorFlowInferenceInterface {
   }
 
   private void closeFetches() {
-    for (Tensor t : fetchTensors) {
+    for (Tensor<?> t : fetchTensors) {
       t.close();
     }
     fetchTensors.clear();
@@ -614,9 +615,9 @@ public class TensorFlowInferenceInterface {
   // State reset on every call to run.
   private Session.Runner runner;
   private List<String> feedNames = new ArrayList<String>();
-  private List<Tensor> feedTensors = new ArrayList<Tensor>();
+  private List<Tensor<?>> feedTensors = new ArrayList<Tensor<?>>();
   private List<String> fetchNames = new ArrayList<String>();
-  private List<Tensor> fetchTensors = new ArrayList<Tensor>();
+  private List<Tensor<?>> fetchTensors = new ArrayList<Tensor<?>>();
 
   // Mutable state.
   private RunStats runStats;
diff --git a/tensorflow/contrib/boosted_trees/lib/quantiles/weighted_quantiles_summary.h b/tensorflow/contrib/boosted_trees/lib/quantiles/weighted_quantiles_summary.h
index dad3b4e10d..c329c6d4f7 100644
--- a/tensorflow/contrib/boosted_trees/lib/quantiles/weighted_quantiles_summary.h
+++ b/tensorflow/contrib/boosted_trees/lib/quantiles/weighted_quantiles_summary.h
@@ -36,7 +36,7 @@ class WeightedQuantilesSummary {
   struct SummaryEntry {
     SummaryEntry(const ValueType& v, const WeightType& w, const WeightType& min,
                  const WeightType& max) {
-      // Explicitely initialize all of memory (including padding from memory
+      // Explicitly initialize all of memory (including padding from memory
       // alignment) to allow the struct to be msan-resistant "plain old data".
       //
       // POD = http://en.cppreference.com/w/cpp/concept/PODType
diff --git a/tensorflow/contrib/data/python/kernel_tests/batch_dataset_op_test.py b/tensorflow/contrib/data/python/kernel_tests/batch_dataset_op_test.py
index 813c64d141..91f100e0f0 100644
--- a/tensorflow/contrib/data/python/kernel_tests/batch_dataset_op_test.py
+++ b/tensorflow/contrib/data/python/kernel_tests/batch_dataset_op_test.py
@@ -253,6 +253,46 @@ class BatchDatasetTest(test.TestCase):
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
+  def testDenseToSparseBatchDatasetWithUnknownShape(self):
+    components = np.random.randint(5, size=(40,)).astype(np.int32)
+    iterator = (dataset_ops.Dataset.from_tensor_slices(components)
+                .map(lambda x: array_ops.fill([x, x], x)).dense_to_sparse_batch(
+                    4, [5, -1]).make_initializable_iterator())
+    init_op = iterator.initializer
+    get_next = sparse_tensor.SparseTensor(*iterator.get_next())
+
+    with self.test_session() as sess:
+      sess.run(init_op)
+
+      for start in range(0, len(components), 4):
+        results = sess.run(get_next)
+        self.assertAllEqual(
+            [[i, j, z] for i, c in enumerate(components[start:start+4])
+             for j in range(c) for z in range(c)], results.indices)
+        self.assertAllEqual(
+            [c for c in components[start:start+4]
+             for _ in range(c) for _ in range(c)],
+            results.values)
+        self.assertAllEqual(
+            [min(4, len(components) - start),
+             5,
+             np.max(components[start:start+4])],
+            results.dense_shape)
+
+      with self.assertRaises(errors.OutOfRangeError):
+        sess.run(get_next)
+
+  def testDenseToSparseBatchDatasetWithInvalidShape(self):
+    input_tensor = array_ops.constant([[1]])
+    iterator = (dataset_ops.Dataset.from_tensors(input_tensor)
+                .dense_to_sparse_batch(4, [-2]).make_initializable_iterator())
+    init_op = iterator.initializer
+
+    with self.test_session() as sess:
+      with self.assertRaisesRegexp(errors.InvalidArgumentError,
+                                   "Dimension -2 must be >= -1"):
+        sess.run(init_op)
+
   def testDenseToSparseBatchDatasetShapeErrors(self):
     input_tensor = array_ops.placeholder(dtypes.int32)
     iterator = (dataset_ops.Dataset.from_tensors(input_tensor).apply(
diff --git a/tensorflow/contrib/data/python/ops/dataset_ops.py b/tensorflow/contrib/data/python/ops/dataset_ops.py
index ff89c47a2e..b74dcd3be2 100644
--- a/tensorflow/contrib/data/python/ops/dataset_ops.py
+++ b/tensorflow/contrib/data/python/ops/dataset_ops.py
@@ -653,7 +653,7 @@ class Dataset(dataset_ops.Dataset):
     ```python
     # Preprocess 4 files concurrently, and interleave blocks of 16 records from
     # each file.
-    filenames = ["/var/data/file1.txt", "/var/data/file2.txt", ..."]
+    filenames = ["/var/data/file1.txt", "/var/data/file2.txt", ...]
     dataset = (Dataset.from_tensor_slices(filenames)
                .interleave(lambda x:
                    TextLineDataset(x).map(parse_fn, num_parallel_calls=1),
diff --git a/tensorflow/contrib/deprecated/__init__.py b/tensorflow/contrib/deprecated/__init__.py
index bfea8445a7..7aff045de3 100644
--- a/tensorflow/contrib/deprecated/__init__.py
+++ b/tensorflow/contrib/deprecated/__init__.py
@@ -91,7 +91,7 @@ from __future__ import division
 from __future__ import print_function
 
 
-# pylint: disable=unused-import,line-too-long
+# pylint: disable=unused-import
 from tensorflow.python.ops.logging_ops import audio_summary
 from tensorflow.python.ops.logging_ops import histogram_summary
 from tensorflow.python.ops.logging_ops import image_summary
diff --git a/tensorflow/contrib/ffmpeg/default/ffmpeg_lib.cc b/tensorflow/contrib/ffmpeg/default/ffmpeg_lib.cc
index 888f5c38a2..b417a70b6e 100644
--- a/tensorflow/contrib/ffmpeg/default/ffmpeg_lib.cc
+++ b/tensorflow/contrib/ffmpeg/default/ffmpeg_lib.cc
@@ -208,7 +208,15 @@ string GetTempFilename(const string& extension) {
     }
     struct stat statbuf;
     if (!stat(dir, &statbuf) && S_ISDIR(statbuf.st_mode)) {
-      return io::JoinPath(dir, StrCat("tmp_file_", getpid(), ".", extension));
+      string tmp_filepath =
+          io::JoinPath(dir, StrCat("tmp_file_XXXXXX", ".", extension));
+      int fd = mkstemps(&tmp_filepath[0], extension.length() + 1);
+      if (fd < 0) {
+        LOG(FATAL) << "Failed to create temp file.";
+      } else {
+        close(fd);
+        return tmp_filepath;
+      }
     }
   }
   LOG(FATAL) << "No temp directory found.";
diff --git a/tensorflow/contrib/framework/python/framework/tensor_util.py b/tensorflow/contrib/framework/python/framework/tensor_util.py
index e595e4d90b..92a2a4ff2d 100644
--- a/tensorflow/contrib/framework/python/framework/tensor_util.py
+++ b/tensorflow/contrib/framework/python/framework/tensor_util.py
@@ -78,9 +78,9 @@ def reduce_sum_n(tensors, name=None):
     return math_ops.add_n(tensors, name=name_scope)
 
 @deprecated(None,
-    "Please switch to tf.confusion_matrix.remove_squeezable_dimensions. Note "
-    "that order of the inputs and ouputs of labels and predictions have also "
-    "been switched.")
+            'Please switch to tf.confusion_matrix.remove_squeezable_dimensions.'
+            'Note that order of the inputs and outputs of labels and '
+            'predictions have also been switched.')
 def remove_squeezable_dimensions(predictions, labels, name=None):
   """Squeeze last dim if ranks of `predictions` and `labels` differ by 1.
 
diff --git a/tensorflow/contrib/fused_conv/kernels/fused_conv2d_bias_activation_op.cc b/tensorflow/contrib/fused_conv/kernels/fused_conv2d_bias_activation_op.cc
index 9275d5a22b..256f200868 100644
--- a/tensorflow/contrib/fused_conv/kernels/fused_conv2d_bias_activation_op.cc
+++ b/tensorflow/contrib/fused_conv/kernels/fused_conv2d_bias_activation_op.cc
@@ -493,42 +493,37 @@ void LaunchFusedConv2DBiasActivationOp<GPUDevice, T, BiasType, ScaleType>::
   dnn::AlgorithmConfig algorithm_config;
   if (cudnn_use_autotune && !AutoTuneConvBiasActivation::GetInstance()->Find(
                                 fused_conv_parameters, &algorithm_config)) {
-    std::vector<dnn::AlgorithmDesc::Index> algorithms;
+    std::vector<dnn::AlgorithmDesc> algorithms;
     CHECK(stream->parent()->GetConvolveAlgorithms(
         fused_conv_parameters.ShouldIncludeWinogradNonfusedAlgo<T>(),
         &algorithms));
     dnn::ProfileResult best_result;
     dnn::ProfileResult best_result_no_scratch;
-    // TODO(benbarsdell): Ideally this should not attempt using tensor op math
-    // if it's not enabled.
-    for (bool use_tensor_ops : {false, true}) {
-      for (auto algo_index : algorithms) {
-        // TODO(zhengxq): profile each algorithm multiple times to better
-        // accuracy.
-        dnn::AlgorithmDesc profile_algorithm(algo_index, use_tensor_ops);
-        CudnnScratchAllocator scratch_allocator(ConvolveScratchSize, ctx);
-        dnn::ProfileResult profile_result;
-        bool cudnn_launch_status =
-            stream
-                ->ThenFusedConvolveWithAlgorithm(
-                    conv_input_desc, conv_input_ptr, conv_input_scale,
-                    filter_desc, filter_ptr, conv_desc, side_input_ptr,
-                    side_input_scale, bias_desc, bias_ptr,
-                    dnn::ActivationMode::kRelu, output_desc, &output_ptr,
-                    &scratch_allocator, dnn::AlgorithmConfig(profile_algorithm),
-                    &profile_result)
-                .ok();
-        if (cudnn_launch_status) {
-          if (profile_result.is_valid()) {
-            if (profile_result.elapsed_time_in_ms() <
-                best_result.elapsed_time_in_ms()) {
-              best_result = profile_result;
-            }
-            if (scratch_allocator.TotalByteSize() == 0 &&
-                profile_result.elapsed_time_in_ms() <
-                    best_result_no_scratch.elapsed_time_in_ms()) {
-              best_result_no_scratch = profile_result;
-            }
+    for (auto profile_algorithm : algorithms) {
+      // TODO(zhengxq): profile each algorithm multiple times to better
+      // accuracy.
+      CudnnScratchAllocator scratch_allocator(ConvolveScratchSize, ctx);
+      dnn::ProfileResult profile_result;
+      bool cudnn_launch_status =
+          stream
+              ->ThenFusedConvolveWithAlgorithm(
+                  conv_input_desc, conv_input_ptr, conv_input_scale,
+                  filter_desc, filter_ptr, conv_desc, side_input_ptr,
+                  side_input_scale, bias_desc, bias_ptr,
+                  dnn::ActivationMode::kRelu, output_desc, &output_ptr,
+                  &scratch_allocator, dnn::AlgorithmConfig(profile_algorithm),
+                  &profile_result)
+              .ok();
+      if (cudnn_launch_status) {
+        if (profile_result.is_valid()) {
+          if (profile_result.elapsed_time_in_ms() <
+              best_result.elapsed_time_in_ms()) {
+            best_result = profile_result;
+          }
+          if (scratch_allocator.TotalByteSize() == 0 &&
+              profile_result.elapsed_time_in_ms() <
+                  best_result_no_scratch.elapsed_time_in_ms()) {
+            best_result_no_scratch = profile_result;
           }
         }
       }
diff --git a/tensorflow/contrib/memory_stats/__init__.py b/tensorflow/contrib/memory_stats/__init__.py
index a2b2b65692..a32302c854 100644
--- a/tensorflow/contrib/memory_stats/__init__.py
+++ b/tensorflow/contrib/memory_stats/__init__.py
@@ -14,10 +14,12 @@
 # ==============================================================================
 """Ops for memory statistics.
 
+@@BytesInUse
 @@BytesLimit
 @@MaxBytesInUse
 """
 
+from tensorflow.contrib.memory_stats.python.ops.memory_stats_ops import BytesInUse
 from tensorflow.contrib.memory_stats.python.ops.memory_stats_ops import BytesLimit
 from tensorflow.contrib.memory_stats.python.ops.memory_stats_ops import MaxBytesInUse
 
diff --git a/tensorflow/contrib/memory_stats/kernels/memory_stats_ops.cc b/tensorflow/contrib/memory_stats/kernels/memory_stats_ops.cc
index 3b88535dce..7e2e96e160 100644
--- a/tensorflow/contrib/memory_stats/kernels/memory_stats_ops.cc
+++ b/tensorflow/contrib/memory_stats/kernels/memory_stats_ops.cc
@@ -40,6 +40,28 @@ class MemoryStatsOp : public OpKernel {
       const AllocatorStats& allocator_stats) const = 0;
 };
 
+// Op that measures current memory in bytes.
+class BytesInUseOp : public MemoryStatsOp {
+ public:
+  explicit BytesInUseOp(OpKernelConstruction* context)
+      : MemoryStatsOp(context) {}
+
+ private:
+  int64 ExtractAllocatorStats(
+      const AllocatorStats& allocator_stats) const override {
+    return allocator_stats.bytes_in_use;
+  }
+};
+
+// Register this op on GPU only, see comment for MaxBytesInUse for reason
+REGISTER_KERNEL_BUILDER(Name("BytesInUse").Device(DEVICE_GPU).HostMemory("out"),
+                        BytesInUseOp);
+
+#ifdef TENSORFLOW_USE_SYCL
+REGISTER_KERNEL_BUILDER(
+    Name("BytesInUse").Device(DEVICE_SYCL).HostMemory("out"), MaxBytesInUseOp);
+#endif  // TENSORFLOW_USE_SYCL
+
 // Op that measures the total memory (in bytes) of a device.
 class BytesLimitOp : public MemoryStatsOp {
  public:
diff --git a/tensorflow/contrib/memory_stats/ops/memory_stats_ops.cc b/tensorflow/contrib/memory_stats/ops/memory_stats_ops.cc
index 08859c8613..42020cf7f6 100644
--- a/tensorflow/contrib/memory_stats/ops/memory_stats_ops.cc
+++ b/tensorflow/contrib/memory_stats/ops/memory_stats_ops.cc
@@ -17,6 +17,10 @@ limitations under the License.
 
 namespace tensorflow {
 
+REGISTER_OP("BytesInUse")
+    .Output("out: int64")
+    .SetIsStateful()
+    .SetShapeFn(shape_inference::ScalarShape);
 REGISTER_OP("BytesLimit")
     .Output("out: int64")
     .SetIsStateful()
diff --git a/tensorflow/contrib/memory_stats/python/kernel_tests/memory_stats_ops_test.py b/tensorflow/contrib/memory_stats/python/kernel_tests/memory_stats_ops_test.py
index ec25c032f0..d1b430b803 100644
--- a/tensorflow/contrib/memory_stats/python/kernel_tests/memory_stats_ops_test.py
+++ b/tensorflow/contrib/memory_stats/python/kernel_tests/memory_stats_ops_test.py
@@ -20,6 +20,7 @@ from __future__ import print_function
 
 from tensorflow.contrib.memory_stats.python.ops import memory_stats_ops
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import math_ops
@@ -64,10 +65,29 @@ class MemoryStatsOpsTest(test_util.TensorFlowTestCase):
       d = math_ops.matmul(c, b)
       sess.run(d)
 
-      max_bytes_in_use = sess.run(memory_stats_ops.MaxBytesInUse())
+      max_bytes_in_use_op = memory_stats_ops.MaxBytesInUse()
+      max_bytes_in_use = sess.run(max_bytes_in_use_op)
       self.assertGreaterEqual(max_bytes_in_use, matrix_size_in_bytes * 3)
       self.assertLess(max_bytes_in_use, matrix_size_in_bytes * 4)
 
+      # run chain with 2 ops, make sure BytesInUse captures intermediate
+      # memory usage
+      a = random_ops.random_uniform(matrix_shape, dtype=dtype)
+      with ops.control_dependencies([a]):
+        bytes_in_use_op = memory_stats_ops.BytesInUse()
+      with ops.control_dependencies([bytes_in_use_op]):
+        b = random_ops.random_uniform(matrix_shape, dtype=dtype)
+
+      _, bytes_in_use, max_bytes_in_use = sess.run([a, bytes_in_use_op,
+                                                    max_bytes_in_use_op])
+
+      # intermediate result allocates 1 matrix, max usage is at least 2
+      self.assertGreaterEqual(bytes_in_use, matrix_size_in_bytes * 1)
+      self.assertLess(bytes_in_use, matrix_size_in_bytes * 2)
+
+      # max usage is still 3 because it reflects maxium from previous .run call
+      self.assertGreaterEqual(max_bytes_in_use, matrix_size_in_bytes * 3)
+
 
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/contrib/memory_stats/python/ops/memory_stats_ops.py b/tensorflow/contrib/memory_stats/python/ops/memory_stats_ops.py
index d35c6583ed..c0f7788c1c 100644
--- a/tensorflow/contrib/memory_stats/python/ops/memory_stats_ops.py
+++ b/tensorflow/contrib/memory_stats/python/ops/memory_stats_ops.py
@@ -26,6 +26,11 @@ _memory_stats_ops_so = loader.load_op_library(
     resource_loader.get_path_to_datafile("_memory_stats_ops.so"))
 
 
+def BytesInUse():
+  """Generates an op that computes the current memory of a device."""
+  return gen_memory_stats_ops.bytes_in_use()
+
+
 def BytesLimit():
   """Generates an op that measures the total memory (in bytes) of a device."""
   return gen_memory_stats_ops.bytes_limit()
diff --git a/tensorflow/contrib/resampler/kernels/resampler_ops.cc b/tensorflow/contrib/resampler/kernels/resampler_ops.cc
index afc8bcd446..7d9ef14cef 100644
--- a/tensorflow/contrib/resampler/kernels/resampler_ops.cc
+++ b/tensorflow/contrib/resampler/kernels/resampler_ops.cc
@@ -122,7 +122,7 @@ struct Resampler2DFunctor<CPUDevice, T>{
     };
     // Rough estimate of work for each batch entry.
     // From third_party/tensorflow/core/util/work_sharder.cc we gather that an
-    // estimate of the cost of each work unit is needed to correclty shard the
+    // estimate of the cost of each work unit is needed to correctly shard the
     // workload. Shard assumes each cost unit is 1ns, minimum cost per shard
     // being 10us.
     const int64 cost =  static_cast<int64>(num_sampling_points) *
diff --git a/tensorflow/contrib/rnn/python/ops/rnn_cell.py b/tensorflow/contrib/rnn/python/ops/rnn_cell.py
index 1b0327d62b..6702a89d22 100644
--- a/tensorflow/contrib/rnn/python/ops/rnn_cell.py
+++ b/tensorflow/contrib/rnn/python/ops/rnn_cell.py
@@ -525,7 +525,7 @@ class GridLSTMCell(rnn_cell_impl.RNNCell):
       self._state_tuple_type = collections.namedtuple(
           "GridLSTMStateTuple", state_names.strip(","))
       self._state_size = self._state_tuple_type(
-              *([num_units, num_units] * self._total_blocks))
+          *([num_units, num_units] * self._total_blocks))
     else:
       self._state_tuple_type = None
       self._state_size = num_units * self._total_blocks * 2
@@ -2082,9 +2082,11 @@ def _conv(args,
   shape_length = len(shapes[0])
   for shape in shapes:
     if len(shape) not in [3,4,5]:
-      raise ValueError("Conv Linear expects 3D, 4D or 5D arguments: %s" % str(shapes))
+      raise ValueError("Conv Linear expects 3D, 4D "
+                       "or 5D arguments: %s" % str(shapes))
     if len(shape) != len(shapes[0]):
-      raise ValueError("Conv Linear expects all args to be of same Dimensiton: %s" % str(shapes))
+      raise ValueError("Conv Linear expects all args "
+                       "to be of same Dimension: %s" % str(shapes))
     else:
       total_arg_size_depth += shape[-1]
   dtype = [a.dtype for a in args][0]
@@ -2102,7 +2104,7 @@ def _conv(args,
 
   # Now the computation.
   kernel = vs.get_variable(
-      "kernel", 
+      "kernel",
       filter_size + [total_arg_size_depth, num_features],
       dtype=dtype)
   if len(args) == 1:
diff --git a/tensorflow/contrib/seq2seq/python/ops/helper.py b/tensorflow/contrib/seq2seq/python/ops/helper.py
index 64e00c21c7..b55d90cbab 100644
--- a/tensorflow/contrib/seq2seq/python/ops/helper.py
+++ b/tensorflow/contrib/seq2seq/python/ops/helper.py
@@ -309,7 +309,7 @@ class ScheduledEmbeddingTrainingHelper(TrainingHelper):
           gen_array_ops.fill([self.batch_size], -1))
 
   def next_inputs(self, time, outputs, state, sample_ids, name=None):
-    with ops.name_scope(name, "ScheduledEmbeddingTrainingHelperSample",
+    with ops.name_scope(name, "ScheduledEmbeddingTrainingHelperNextInputs",
                         [time, outputs, state, sample_ids]):
       (finished, base_next_inputs, state) = (
           super(ScheduledEmbeddingTrainingHelper, self).next_inputs(
diff --git a/tensorflow/contrib/signal/BUILD b/tensorflow/contrib/signal/BUILD
index 43f24474ed..2204b684ac 100644
--- a/tensorflow/contrib/signal/BUILD
+++ b/tensorflow/contrib/signal/BUILD
@@ -5,6 +5,7 @@ licenses(["notice"])  # Apache 2.0
 exports_files(["LICENSE"])
 
 load("//tensorflow:tensorflow.bzl", "cuda_py_tests")
+load("//tensorflow:tensorflow.bzl", "py_test")  # @unused
 
 py_library(
     name = "signal_py",
diff --git a/tensorflow/contrib/slim/python/slim/data/tfexample_decoder.py b/tensorflow/contrib/slim/python/slim/data/tfexample_decoder.py
index f9449095be..094568389c 100644
--- a/tensorflow/contrib/slim/python/slim/data/tfexample_decoder.py
+++ b/tensorflow/contrib/slim/python/slim/data/tfexample_decoder.py
@@ -135,7 +135,10 @@ class BoundingBox(ItemHandler):
     """
     sides = []
     for key in self._full_keys:
-      side = array_ops.expand_dims(keys_to_tensors[key].values, 0)
+      side = keys_to_tensors[key]
+      if isinstance(side, sparse_tensor.SparseTensor):
+        side = side.values
+      side = array_ops.expand_dims(side, 0)
       sides.append(side)
 
     bounding_box = array_ops.concat(sides, 0)
diff --git a/tensorflow/contrib/slim/python/slim/data/tfexample_decoder_test.py b/tensorflow/contrib/slim/python/slim/data/tfexample_decoder_test.py
index 96606b9c0e..60d1eba07f 100644
--- a/tensorflow/contrib/slim/python/slim/data/tfexample_decoder_test.py
+++ b/tensorflow/contrib/slim/python/slim/data/tfexample_decoder_test.py
@@ -692,7 +692,7 @@ class TFExampleDecoderTest(test.TestCase):
         else:
           self.assertAllClose(image, decoded_image, atol=0)
 
-  def testDecodeExampleWithBoundingBox(self):
+  def testDecodeExampleWithBoundingBoxSparse(self):
     num_bboxes = 10
     np_ymin = np.random.rand(num_bboxes, 1)
     np_xmin = np.random.rand(num_bboxes, 1)
@@ -731,6 +731,49 @@ class TFExampleDecoderTest(test.TestCase):
 
     self.assertAllClose(np_bboxes, bboxes)
 
+  def testDecodeExampleWithBoundingBoxDense(self):
+    num_bboxes = 10
+    np_ymin = np.random.rand(num_bboxes, 1)
+    np_xmin = np.random.rand(num_bboxes, 1)
+    np_ymax = np.random.rand(num_bboxes, 1)
+    np_xmax = np.random.rand(num_bboxes, 1)
+    np_bboxes = np.hstack([np_ymin, np_xmin, np_ymax, np_xmax])
+
+    example = example_pb2.Example(features=feature_pb2.Features(feature={
+        'image/object/bbox/ymin': self._EncodedFloatFeature(np_ymin),
+        'image/object/bbox/xmin': self._EncodedFloatFeature(np_xmin),
+        'image/object/bbox/ymax': self._EncodedFloatFeature(np_ymax),
+        'image/object/bbox/xmax': self._EncodedFloatFeature(np_xmax),
+    }))
+    serialized_example = example.SerializeToString()
+
+    with self.test_session():
+      serialized_example = array_ops.reshape(serialized_example, shape=[])
+
+      keys_to_features = {
+          'image/object/bbox/ymin': parsing_ops.FixedLenSequenceFeature(
+              [], dtypes.float32, allow_missing=True),
+          'image/object/bbox/xmin': parsing_ops.FixedLenSequenceFeature(
+              [], dtypes.float32, allow_missing=True),
+          'image/object/bbox/ymax': parsing_ops.FixedLenSequenceFeature(
+              [], dtypes.float32, allow_missing=True),
+          'image/object/bbox/xmax': parsing_ops.FixedLenSequenceFeature(
+              [], dtypes.float32, allow_missing=True),
+      }
+
+      items_to_handlers = {
+          'object/bbox':
+              tfexample_decoder.BoundingBox(['ymin', 'xmin', 'ymax', 'xmax'],
+                                            'image/object/bbox/'),
+      }
+
+      decoder = tfexample_decoder.TFExampleDecoder(keys_to_features,
+                                                   items_to_handlers)
+      [tf_bboxes] = decoder.decode(serialized_example, ['object/bbox'])
+      bboxes = tf_bboxes.eval()
+
+    self.assertAllClose(np_bboxes, bboxes)
+
   def testDecodeExampleWithRepeatedImages(self):
     image_shape = (2, 3, 3)
     image_format = 'png'
diff --git a/tensorflow/contrib/timeseries/python/timeseries/BUILD b/tensorflow/contrib/timeseries/python/timeseries/BUILD
index 2c4bed5db1..da583a2ba0 100644
--- a/tensorflow/contrib/timeseries/python/timeseries/BUILD
+++ b/tensorflow/contrib/timeseries/python/timeseries/BUILD
@@ -42,6 +42,7 @@ py_library(
     srcs_version = "PY2AND3",
     deps = [
         ":feature_keys",
+        ":head",
         ":input_pipeline",
         ":model_utils",
         "//tensorflow/python:util",
@@ -78,8 +79,8 @@ py_library(
     deps = [
         ":ar_model",
         ":feature_keys",
+        ":head",
         ":math_utils",
-        ":model_utils",
         ":state_management",
         "//tensorflow/contrib/timeseries/python/timeseries/state_space_models:filtering_postprocessor",
         "//tensorflow/contrib/timeseries/python/timeseries/state_space_models:state_space_model",
@@ -123,9 +124,9 @@ py_test(
 )
 
 py_library(
-    name = "model_utils",
+    name = "head",
     srcs = [
-        "model_utils.py",
+        "head.py",
     ],
     srcs_version = "PY2AND3",
     deps = [
@@ -149,9 +150,9 @@ py_library(
 )
 
 py_test(
-    name = "model_utils_test",
+    name = "head_test",
     srcs = [
-        "model_utils_test.py",
+        "head_test.py",
     ],
     srcs_version = "PY2AND3",
     tags = [
@@ -159,8 +160,8 @@ py_test(
     ],
     deps = [
         ":feature_keys",
+        ":head",
         ":model",
-        ":model_utils",
         ":state_management",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:client_testlib",
@@ -174,6 +175,41 @@ py_test(
     ],
 )
 
+py_library(
+    name = "model_utils",
+    srcs = [
+        "model_utils.py",
+    ],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":feature_keys",
+        "//tensorflow/contrib/framework:framework_py",
+        "//tensorflow/python:dtypes",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:init_ops",
+        "//tensorflow/python:nn_ops",
+        "//tensorflow/python:variable_scope",
+        "//third_party/py/numpy",
+    ],
+)
+
+py_test(
+    name = "model_utils_test",
+    srcs = [
+        "model_utils_test.py",
+    ],
+    srcs_version = "PY2AND3",
+    tags = [
+        "no_pip_gpu",  # b/63391119
+    ],
+    deps = [
+        ":model_utils",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:variables",
+    ],
+)
+
 py_library(
     name = "state_management",
     srcs = [
diff --git a/tensorflow/contrib/timeseries/python/timeseries/ar_model.py b/tensorflow/contrib/timeseries/python/timeseries/ar_model.py
index 267a5f88da..ff140efd48 100644
--- a/tensorflow/contrib/timeseries/python/timeseries/ar_model.py
+++ b/tensorflow/contrib/timeseries/python/timeseries/ar_model.py
@@ -374,7 +374,7 @@ class ARModel(model.TimeSeriesModel):
     original_values = values
 
     # Extra shape checking for the window size (above that in
-    # model_utils.make_model_fn).
+    # `head.create_estimator_spec`).
     expected_times_shape = [None, self.window_size]
     if not times.get_shape().is_compatible_with(expected_times_shape):
       raise ValueError(
diff --git a/tensorflow/contrib/timeseries/python/timeseries/estimators.py b/tensorflow/contrib/timeseries/python/timeseries/estimators.py
index 4025a8f014..3738dfa154 100644
--- a/tensorflow/contrib/timeseries/python/timeseries/estimators.py
+++ b/tensorflow/contrib/timeseries/python/timeseries/estimators.py
@@ -20,8 +20,8 @@ from __future__ import print_function
 
 from tensorflow.contrib.timeseries.python.timeseries import ar_model
 from tensorflow.contrib.timeseries.python.timeseries import feature_keys
+from tensorflow.contrib.timeseries.python.timeseries import head as ts_head_lib
 from tensorflow.contrib.timeseries.python.timeseries import math_utils
-from tensorflow.contrib.timeseries.python.timeseries import model_utils
 from tensorflow.contrib.timeseries.python.timeseries import state_management
 from tensorflow.contrib.timeseries.python.timeseries.state_space_models import state_space_model
 from tensorflow.contrib.timeseries.python.timeseries.state_space_models import structural_ensemble
@@ -59,9 +59,10 @@ class TimeSeriesRegressor(estimator_lib.Estimator):
     if optimizer is None:
       optimizer = train.AdamOptimizer(0.02)
     self._model = model
-    model_fn = model_utils.make_model_fn(
+    ts_regression_head = ts_head_lib.time_series_regression_head(
         model, state_manager, optimizer,
         input_statistics_generator=input_statistics_generator)
+    model_fn = ts_regression_head.create_estimator_spec
     super(TimeSeriesRegressor, self).__init__(
         model_fn=model_fn,
         model_dir=model_dir,
@@ -132,7 +133,7 @@ class TimeSeriesRegressor(estimator_lib.Estimator):
       with ops.Graph().as_default():
         self._model.initialize_graph()
         model_start_state = self._model.get_start_state()
-      for prefixed_state_name, state_tensor in model_utils.state_to_dictionary(
+      for prefixed_state_name, state_tensor in ts_head_lib.state_to_dictionary(
           model_start_state).items():
         state_shape_with_batch = tensor_shape.TensorShape(
             (default_batch_size,)).concatenate(state_tensor.get_shape())
diff --git a/tensorflow/contrib/timeseries/python/timeseries/head.py b/tensorflow/contrib/timeseries/python/timeseries/head.py
new file mode 100644
index 0000000000..5896fc2a20
--- /dev/null
+++ b/tensorflow/contrib/timeseries/python/timeseries/head.py
@@ -0,0 +1,375 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Timeseries head."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import re
+
+from tensorflow.contrib.framework.python.ops import variables
+from tensorflow.contrib.layers.python.layers import optimizers
+
+from tensorflow.contrib.timeseries.python.timeseries import feature_keys
+
+from tensorflow.python.estimator import estimator_lib
+from tensorflow.python.estimator.canned import head as head_lib
+from tensorflow.python.estimator.export import export_lib
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import state_ops
+from tensorflow.python.ops import variable_scope
+from tensorflow.python.util import nest
+
+
+def time_series_regression_head(model,
+                                state_manager,
+                                optimizer,
+                                input_statistics_generator=None):
+  """Creates a `_Head` for time series regression.
+
+  Args:
+    model: A model for time series regression.
+    state_manager: A state manager.
+    optimizer: An optimizer.
+    input_statistics_generator: A input statistics generator.
+
+  Returns:
+    An instance of `_Head` for time series regression.
+  """
+  return _TimeSeriesRegressionHead(model, state_manager, optimizer,
+                                   input_statistics_generator)
+
+
+class _TimeSeriesRegressionHead(head_lib._Head):  # pylint:disable=protected-access
+  """See `time_series_regression_head`."""
+
+  def __init__(self,
+               model,
+               state_manager,
+               optimizer,
+               input_statistics_generator=None,
+               name=None):
+    self.model = model
+    self.state_manager = state_manager
+    self.optimizer = optimizer
+    self.input_statistics_generator = input_statistics_generator
+    self._name = name
+
+  def _train_ops(self, features):
+    """Add training ops to the graph."""
+    with variable_scope.variable_scope("model"):
+      model_outputs = self.state_manager.define_loss(
+          self.model, features, estimator_lib.ModeKeys.TRAIN)
+
+    train_op = optimizers.optimize_loss(
+        model_outputs.loss,
+        global_step=variables.get_global_step(),
+        optimizer=self.optimizer,
+        # Learning rate is set in the Optimizer object
+        learning_rate=None)
+    return estimator_lib.EstimatorSpec(
+        loss=model_outputs.loss,
+        mode=estimator_lib.ModeKeys.TRAIN,
+        train_op=train_op)
+
+  # TODO(terrytangyuan): suffix summary and metrics keys by `"/" + name`
+  @property
+  def name(self):
+    return self._name
+
+  # TODO(terrytangyuan): unused for now. Need to decouple
+  # `state_manager.define_loss` to satisfy the extendable return signature of
+  # `_Head.create_loss`.
+  def create_loss(self, features, mode, logits, labels):
+    """See `_Head`."""
+    return None
+
+  # TODO(terrytangyuan): check label dimension
+  @property
+  def logits_dimension(self):
+    return None
+
+  def _evaluate_ops(self, features):
+    """Add ops for evaluation (aka filtering) to the graph."""
+    with variable_scope.variable_scope("model"):
+      model_outputs = self.state_manager.define_loss(
+          self.model, features, estimator_lib.ModeKeys.EVAL)
+    metrics = {}
+    # Just output in-sample predictions for the last chunk seen
+    for prediction_key, prediction_value in model_outputs.predictions.items():
+      metrics[prediction_key] = _identity_metric_single(prediction_key,
+                                                        prediction_value)
+    metrics[feature_keys.FilteringResults.TIMES] = _identity_metric_single(
+        feature_keys.FilteringResults.TIMES, model_outputs.prediction_times)
+    metrics[feature_keys.FilteringResults.STATE_TUPLE] = (
+        _identity_metric_nested(feature_keys.FilteringResults.STATE_TUPLE,
+                                model_outputs.end_state))
+    return estimator_lib.EstimatorSpec(
+        loss=model_outputs.loss,
+        mode=estimator_lib.ModeKeys.EVAL,
+        eval_metric_ops=metrics,
+        predictions={})
+
+  def _predict_ops(self, features):
+    """Add ops for prediction to the graph."""
+    with variable_scope.variable_scope("model"):
+      prediction = self.model.predict(features=features)
+    prediction[feature_keys.PredictionResults.TIMES] = features[
+        feature_keys.PredictionFeatures.TIMES]
+    return estimator_lib.EstimatorSpec(
+        predictions=prediction, mode=estimator_lib.ModeKeys.PREDICT)
+
+  def _serving_ops(self, features):
+    """Add ops for serving to the graph."""
+    with variable_scope.variable_scope("model"):
+      prediction_outputs = self.model.predict(features=features)
+    with variable_scope.variable_scope("model", reuse=True):
+      filtering_outputs = self.state_manager.define_loss(
+          self.model, features, estimator_lib.ModeKeys.EVAL)
+
+    return estimator_lib.EstimatorSpec(
+        mode=estimator_lib.ModeKeys.PREDICT,
+        export_outputs={
+            feature_keys.SavedModelLabels.PREDICT:
+                export_lib.PredictOutput(prediction_outputs),
+            feature_keys.SavedModelLabels.FILTER:
+                export_lib.PredictOutput(
+                    state_to_dictionary(filtering_outputs.end_state))
+        },
+        # Likely unused, but it is necessary to return `predictions` to satisfy
+        # the Estimator's error checking.
+        predictions={})
+
+  def _convert_feature_to_tensor(self, name, value):
+    """Casts features to the correct dtype based on their name."""
+    if name in [
+        feature_keys.TrainEvalFeatures.TIMES,
+        feature_keys.PredictionFeatures.TIMES
+    ]:
+      return math_ops.cast(value, dtypes.int64)
+    if name == feature_keys.TrainEvalFeatures.VALUES:
+      return math_ops.cast(value, self.model.dtype)
+    if name == feature_keys.PredictionFeatures.STATE_TUPLE:
+      return value  # Correct dtypes are model-dependent
+    return ops.convert_to_tensor(value)
+
+  def _gather_state(self, features):
+    """Returns `features` with state packed, indicates if packing was done."""
+    prefixed_state_re = re.compile(r"^" + feature_keys.State.STATE_PREFIX +
+                                   r"_(\d+)$")
+    numbered_state = []
+    for key, tensor in features.items():
+      search_result = prefixed_state_re.search(key)
+      if search_result:
+        numbered_state.append((int(search_result.group(1)), key, tensor))
+    if not numbered_state:
+      return features, False
+    features = features.copy()
+    for _, key, _ in numbered_state:
+      del features[key]
+    numbered_state.sort(key=lambda number, *_: number)
+    features[feature_keys.State.STATE_TUPLE] = nest.pack_sequence_as(
+        structure=self.model.get_start_state(),
+        flat_sequence=[tensor for _, _, tensor in numbered_state])
+    return features, True
+
+  def create_estimator_spec(self, features, mode, labels=None):
+    """Performs basic error checking and returns an EstimatorSpec."""
+    with ops.name_scope("head"):
+      if labels:
+        raise ValueError(
+            "The model received a `labels` dictionary, which is "
+            "not supported. Pass '{}' and '{}' as "
+            "features.".format(feature_keys.TrainEvalFeatures.TIMES,
+                               feature_keys.TrainEvalFeatures.VALUES))
+      del labels
+      features = {
+          name: self._convert_feature_to_tensor(name=name, value=value)
+          for name, value in features.items()
+      }
+      if self.input_statistics_generator is not None:
+        input_statistics = self.input_statistics_generator.initialize_graph(
+            features, update_statistics=(mode == estimator_lib.ModeKeys.TRAIN))
+      else:
+        input_statistics = None
+      self.model.initialize_graph(input_statistics=input_statistics)
+
+      # _gather_state requires the model to have its graph initialized (so it
+      # has access to the structure of the model's state)
+      features, passed_flat_state = self._gather_state(features)
+      if (mode == estimator_lib.ModeKeys.TRAIN or
+          mode == estimator_lib.ModeKeys.EVAL):
+        _check_train_eval_features(features, self.model)
+      elif mode == estimator_lib.ModeKeys.PREDICT:
+        _check_predict_features(features)
+      else:
+        raise ValueError("Unknown mode '{}' passed to model_fn.".format(mode))
+
+      self.state_manager.initialize_graph(
+          model=self.model, input_statistics=input_statistics)
+
+      if mode == estimator_lib.ModeKeys.TRAIN:
+        return self._train_ops(features)
+      elif mode == estimator_lib.ModeKeys.EVAL:
+        return self._evaluate_ops(features)
+      elif mode == estimator_lib.ModeKeys.PREDICT and not passed_flat_state:
+        return self._predict_ops(features)
+      elif mode == estimator_lib.ModeKeys.PREDICT and passed_flat_state:
+        # The mode is PREDICT, but we're actually in export_savedmodel for
+        # serving. We want to return two graphs: one for filtering (state + data
+        # -> state) and one for predicting (state -> prediction).
+        return self._serving_ops(features)
+
+
+def _check_feature_shapes_compatible_with(features,
+                                          compatible_with_name,
+                                          compatible_with_value,
+                                          ignore=None):
+  """Checks all features are compatible with the given time-like feature."""
+  if ignore is None:
+    ignore = set()
+  for name, value in features.items():
+    if name in ignore:
+      continue
+    feature_shape = value.get_shape()
+    if feature_shape.ndims is None:
+      continue
+    if feature_shape.ndims < 2:
+      raise ValueError(
+          ("Features must have shape (batch dimension, window size, ...) "
+           "(got rank {} for feature '{}')").format(feature_shape.ndims, name))
+    if not feature_shape[:2].is_compatible_with(
+        compatible_with_value.get_shape()):
+      raise ValueError(
+          ("Features must have shape (batch dimension, window size, ...) "
+           "where batch dimension and window size match the "
+           "'{times_feature}' feature (got shape {feature_shape} for "
+           "feature '{feature_name}' but shape {times_shape} for feature "
+           "'{times_feature}')").format(
+               times_feature=compatible_with_name,
+               feature_shape=feature_shape,
+               feature_name=name,
+               times_shape=compatible_with_value.get_shape()))
+
+
+def _check_predict_features(features):
+  """Raises errors if features are not suitable for prediction."""
+  if feature_keys.PredictionFeatures.TIMES not in features:
+    raise ValueError("Expected a '{}' feature for prediction.".format(
+        feature_keys.PredictionFeatures.TIMES))
+  if feature_keys.PredictionFeatures.STATE_TUPLE not in features:
+    raise ValueError("Expected a '{}' feature for prediction.".format(
+        feature_keys.PredictionFeatures.STATE_TUPLE))
+  times_feature = features[feature_keys.PredictionFeatures.TIMES]
+  if not times_feature.get_shape().is_compatible_with([None, None]):
+    raise ValueError(
+        ("Expected shape (batch dimension, window size) for feature '{}' "
+         "(got shape {})").format(feature_keys.PredictionFeatures.TIMES,
+                                  times_feature.get_shape()))
+  _check_feature_shapes_compatible_with(
+      features=features,
+      compatible_with_name=feature_keys.PredictionFeatures.TIMES,
+      compatible_with_value=times_feature,
+      ignore=set([
+          feature_keys.PredictionFeatures.STATE_TUPLE  # Model-dependent shapes
+      ]))
+
+
+def _check_train_eval_features(features, model):
+  """Raise errors if features are not suitable for training/evaluation."""
+  if feature_keys.TrainEvalFeatures.TIMES not in features:
+    raise ValueError("Expected a '{}' feature for training/evaluation.".format(
+        feature_keys.TrainEvalFeatures.TIMES))
+  if feature_keys.TrainEvalFeatures.VALUES not in features:
+    raise ValueError("Expected a '{}' feature for training/evaluation.".format(
+        feature_keys.TrainEvalFeatures.VALUES))
+  times_feature = features[feature_keys.TrainEvalFeatures.TIMES]
+  if not times_feature.get_shape().is_compatible_with([None, None]):
+    raise ValueError(
+        ("Expected shape (batch dimension, window size) for feature '{}' "
+         "(got shape {})").format(feature_keys.TrainEvalFeatures.TIMES,
+                                  times_feature.get_shape()))
+  values_feature = features[feature_keys.TrainEvalFeatures.VALUES]
+  if not values_feature.get_shape().is_compatible_with(
+      [None, None, model.num_features]):
+    raise ValueError(
+        ("Expected shape (batch dimension, window size, {num_features}) "
+         "for feature '{feature_name}', since the model was configured "
+         "with num_features={num_features} (got shape {got_shape})").format(
+             num_features=model.num_features,
+             feature_name=feature_keys.TrainEvalFeatures.VALUES,
+             got_shape=times_feature.get_shape()))
+  _check_feature_shapes_compatible_with(
+      features=features,
+      compatible_with_name=feature_keys.TrainEvalFeatures.TIMES,
+      compatible_with_value=times_feature,
+      ignore=set([
+          feature_keys.State.STATE_TUPLE  # Model-dependent shapes
+      ]))
+
+
+def _identity_metric_single(name, input_tensor):
+  """A metric which takes on its last updated value.
+
+  This keeps evaluation metrics in sync with one another, since update ops are
+  run separately from their result Tensors. Simply returning (input_tensor,
+  no_op) as a metric with a value but no update means that a metric will come
+  from a different batch of data than metrics which cache values in a Variable
+  (e.g. the default loss metric).
+
+  Args:
+    name: A name for the metric.
+    input_tensor: Any Tensor.
+  Returns:
+    A tuple of (value, update_op).
+  """
+  metric_variable = variable_scope.variable(
+      name="{}_identity_metric".format(name),
+      initial_value=array_ops.zeros([], dtype=input_tensor.dtype),
+      collections=[ops.GraphKeys.LOCAL_VARIABLES],
+      validate_shape=False)
+  update_op = state_ops.assign(
+      metric_variable, input_tensor, validate_shape=False)
+  # This shape will be correct once the first update runs (but may be
+  # incomplete, so is not helpful for initializing the variable).
+  metric_variable.set_shape(input_tensor.get_shape())
+  return (metric_variable.value(), update_op)
+
+
+def _identity_metric_nested(name, input_tensors):
+  """Create identity metrics for a nested tuple of Tensors."""
+  update_ops = []
+  value_tensors = []
+  for tensor_number, tensor in enumerate(nest.flatten(input_tensors)):
+    value_tensor, update_op = _identity_metric_single(
+        name="{}_{}".format(name, tensor_number), input_tensor=tensor)
+    update_ops.append(update_op)
+    value_tensors.append(value_tensor)
+  return (nest.pack_sequence_as(input_tensors, value_tensors),
+          control_flow_ops.group(*update_ops))
+
+
+def state_to_dictionary(state_tuple):
+  """Flatten model state into a dictionary with string keys."""
+  flattened = {}
+  for state_number, state_value in enumerate(nest.flatten(state_tuple)):
+    prefixed_state_name = "{}_{:02d}".format(feature_keys.State.STATE_PREFIX,
+                                             state_number)
+    flattened[prefixed_state_name] = state_value
+  return flattened
diff --git a/tensorflow/contrib/timeseries/python/timeseries/head_test.py b/tensorflow/contrib/timeseries/python/timeseries/head_test.py
new file mode 100644
index 0000000000..3415061cfd
--- /dev/null
+++ b/tensorflow/contrib/timeseries/python/timeseries/head_test.py
@@ -0,0 +1,267 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for head."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.contrib.timeseries.python.timeseries import feature_keys
+from tensorflow.contrib.timeseries.python.timeseries import head as ts_head_lib
+from tensorflow.contrib.timeseries.python.timeseries import model
+from tensorflow.contrib.timeseries.python.timeseries import state_management
+
+from tensorflow.python.estimator import estimator_lib
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import metrics
+from tensorflow.python.ops import variables
+from tensorflow.python.platform import test
+from tensorflow.python.training import coordinator as coordinator_lib
+from tensorflow.python.training import queue_runner_impl
+from tensorflow.python.training import training as train
+
+
+class HeadTest(test.TestCase):
+
+  def test_labels_provided_error(self):
+    model_fn = _stub_model_fn()
+    for mode in [estimator_lib.ModeKeys.TRAIN, estimator_lib.ModeKeys.EVAL,
+                 estimator_lib.ModeKeys.PREDICT]:
+      with self.assertRaisesRegexp(ValueError, "labels"):
+        model_fn(features={}, labels={"a": "b"}, mode=mode)
+
+  def test_unknown_mode(self):
+    model_fn = _stub_model_fn()
+    with self.assertRaisesRegexp(ValueError, "Unknown mode 'Not a mode'"):
+      model_fn(features={}, labels={}, mode="Not a mode")
+
+
+class _TickerModel(object):
+  num_features = 1
+  dtype = dtypes.float32
+
+  def initialize_graph(self, input_statistics):
+    pass
+
+  def define_loss(self, features, mode):
+    del mode  # unused
+    return model.ModelOutputs(
+        loss=features["ticker"],
+        end_state=(features["ticker"], features["ticker"]),
+        prediction_times=array_ops.zeros(()),
+        predictions={"ticker": features["ticker"]})
+
+
+class EvaluationMetricsTests(test.TestCase):
+
+  def test_metrics_consistent(self):
+    # Tests that the identity metrics used to report in-sample predictions match
+    # the behavior of standard metrics.
+    g = ops.Graph()
+    with g.as_default():
+      features = {
+          feature_keys.TrainEvalFeatures.TIMES:
+              array_ops.zeros((1, 1)),
+          feature_keys.TrainEvalFeatures.VALUES:
+              array_ops.zeros((1, 1, 1)),
+          "ticker":
+              array_ops.reshape(
+                  math_ops.cast(
+                      variables.Variable(
+                          name="ticker",
+                          initial_value=0,
+                          dtype=dtypes.int64,
+                          collections=[ops.GraphKeys.LOCAL_VARIABLES])
+                      .count_up_to(10),
+                      dtype=dtypes.float32), (1, 1, 1))
+      }
+      model_fn = ts_head_lib.time_series_regression_head(
+          model=_TickerModel(),
+          state_manager=state_management.PassthroughStateManager(),
+          optimizer=train.GradientDescentOptimizer(0.001)).create_estimator_spec
+      outputs = model_fn(
+          features=features, labels=None, mode=estimator_lib.ModeKeys.EVAL)
+      metric_update_ops = [
+          metric[1] for metric in outputs.eval_metric_ops.values()]
+      loss_mean, loss_update = metrics.mean(outputs.loss)
+      metric_update_ops.append(loss_update)
+      with self.test_session() as sess:
+        coordinator = coordinator_lib.Coordinator()
+        queue_runner_impl.start_queue_runners(sess, coord=coordinator)
+        variables.local_variables_initializer().run()
+        sess.run(metric_update_ops)
+        loss_evaled, metric_evaled, nested_metric_evaled = sess.run(
+            (loss_mean, outputs.eval_metric_ops["ticker"][0],
+             outputs.eval_metric_ops[feature_keys.FilteringResults.STATE_TUPLE][
+                 0][0]))
+        # The custom model_utils metrics for in-sample predictions should be in
+        # sync with the Estimator's mean metric for model loss.
+        self.assertAllClose(0., loss_evaled)
+        self.assertAllClose((((0.,),),), metric_evaled)
+        self.assertAllClose((((0.,),),), nested_metric_evaled)
+        coordinator.request_stop()
+        coordinator.join()
+
+
+class _StubModel(object):
+  num_features = 3
+  dtype = dtypes.float64
+
+  def initialize_graph(self, input_statistics):
+    del input_statistics  # unused
+
+
+def _stub_model_fn():
+  return ts_head_lib.time_series_regression_head(
+      model=_StubModel(),
+      state_manager=state_management.PassthroughStateManager(),
+      optimizer=train.AdamOptimizer(0.001)).create_estimator_spec
+
+
+class TrainEvalFeatureCheckingTests(test.TestCase):
+
+  def test_no_time_feature(self):
+    model_fn = _stub_model_fn()
+    for mode in [estimator_lib.ModeKeys.TRAIN, estimator_lib.ModeKeys.EVAL]:
+      with self.assertRaisesRegexp(ValueError, "Expected a '{}' feature".format(
+          feature_keys.TrainEvalFeatures.TIMES)):
+        model_fn(
+            features={feature_keys.TrainEvalFeatures.VALUES: [[[1.]]]},
+            labels=None,
+            mode=mode)
+
+  def test_no_value_feature(self):
+    model_fn = _stub_model_fn()
+    for mode in [estimator_lib.ModeKeys.TRAIN, estimator_lib.ModeKeys.EVAL]:
+      with self.assertRaisesRegexp(ValueError, "Expected a '{}' feature".format(
+          feature_keys.TrainEvalFeatures.VALUES)):
+        model_fn(
+            features={feature_keys.TrainEvalFeatures.TIMES: [[1]]},
+            labels=None,
+            mode=mode)
+
+  def test_bad_time_rank(self):
+    model_fn = _stub_model_fn()
+    for mode in [estimator_lib.ModeKeys.TRAIN, estimator_lib.ModeKeys.EVAL]:
+      with self.assertRaisesRegexp(ValueError,
+                                   "Expected shape.*for feature '{}'".format(
+                                       feature_keys.TrainEvalFeatures.TIMES)):
+        model_fn(
+            features={
+                feature_keys.TrainEvalFeatures.TIMES: [[[1]]],
+                feature_keys.TrainEvalFeatures.VALUES: [[[1.]]]
+            },
+            labels=None,
+            mode=mode)
+
+  def test_bad_value_rank(self):
+    model_fn = _stub_model_fn()
+    for mode in [estimator_lib.ModeKeys.TRAIN, estimator_lib.ModeKeys.EVAL]:
+      with self.assertRaisesRegexp(ValueError,
+                                   "Expected shape.*for feature '{}'".format(
+                                       feature_keys.TrainEvalFeatures.VALUES)):
+        model_fn(
+            features={
+                feature_keys.TrainEvalFeatures.TIMES: [[1]],
+                feature_keys.TrainEvalFeatures.VALUES: [[1.]]
+            },
+            labels=None,
+            mode=mode)
+
+  def test_bad_value_num_features(self):
+    model_fn = _stub_model_fn()
+    for mode in [estimator_lib.ModeKeys.TRAIN, estimator_lib.ModeKeys.EVAL]:
+      with self.assertRaisesRegexp(
+          ValueError, "Expected shape.*, 3.*for feature '{}'".format(
+              feature_keys.TrainEvalFeatures.VALUES)):
+        model_fn(
+            features={
+                feature_keys.TrainEvalFeatures.TIMES: [[1]],
+                feature_keys.TrainEvalFeatures.VALUES: [[[1.]]]
+            },
+            labels=None,
+            mode=mode)
+
+  def test_bad_exogenous_shape(self):
+    model_fn = _stub_model_fn()
+    for mode in [estimator_lib.ModeKeys.TRAIN, estimator_lib.ModeKeys.EVAL]:
+      with self.assertRaisesRegexp(
+          ValueError,
+          "Features must have shape.*for feature 'exogenous'"):
+        model_fn(
+            features={
+                feature_keys.TrainEvalFeatures.TIMES: [[1]],
+                feature_keys.TrainEvalFeatures.VALUES: [[[1., 2., 3.]]],
+                "exogenous": [[1], [2]]
+            },
+            labels=None,
+            mode=mode)
+
+
+class PredictFeatureCheckingTests(test.TestCase):
+
+  def test_no_time_feature(self):
+    model_fn = _stub_model_fn()
+    with self.assertRaisesRegexp(ValueError, "Expected a '{}' feature".format(
+        feature_keys.PredictionFeatures.TIMES)):
+      model_fn(
+          features={
+              feature_keys.PredictionFeatures.STATE_TUPLE: ([[[1.]]], 1.)
+          },
+          labels=None,
+          mode=estimator_lib.ModeKeys.PREDICT)
+
+  def test_no_start_state_feature(self):
+    model_fn = _stub_model_fn()
+    with self.assertRaisesRegexp(ValueError, "Expected a '{}' feature".format(
+        feature_keys.PredictionFeatures.STATE_TUPLE)):
+      model_fn(
+          features={feature_keys.PredictionFeatures.TIMES: [[1]]},
+          labels=None,
+          mode=estimator_lib.ModeKeys.PREDICT)
+
+  def test_bad_time_rank(self):
+    model_fn = _stub_model_fn()
+    with self.assertRaisesRegexp(ValueError,
+                                 "Expected shape.*for feature '{}'".format(
+                                     feature_keys.PredictionFeatures.TIMES)):
+      model_fn(
+          features={
+              feature_keys.PredictionFeatures.TIMES: 1,
+              feature_keys.PredictionFeatures.STATE_TUPLE: (1, (2, 3.))
+          },
+          labels=None,
+          mode=estimator_lib.ModeKeys.PREDICT)
+
+  def test_bad_exogenous_shape(self):
+    model_fn = _stub_model_fn()
+    with self.assertRaisesRegexp(
+        ValueError,
+        "Features must have shape.*for feature 'exogenous'"):
+      model_fn(
+          features={
+              feature_keys.PredictionFeatures.TIMES: [[1]],
+              feature_keys.PredictionFeatures.STATE_TUPLE: (1, (2, 3.)),
+              "exogenous": 1.
+          },
+          labels=None,
+          mode=estimator_lib.ModeKeys.PREDICT)
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/contrib/timeseries/python/timeseries/model_utils.py b/tensorflow/contrib/timeseries/python/timeseries/model_utils.py
index addcdb0575..b5d7cb376b 100644
--- a/tensorflow/contrib/timeseries/python/timeseries/model_utils.py
+++ b/tensorflow/contrib/timeseries/python/timeseries/model_utils.py
@@ -18,334 +18,15 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import re
-
 import numpy
 
-from tensorflow.contrib.framework.python.ops import variables
-from tensorflow.contrib.layers.python.layers import optimizers
-
 from tensorflow.contrib.timeseries.python.timeseries import feature_keys
 
-from tensorflow.python.estimator import estimator_lib
-from tensorflow.python.estimator.export import export_lib
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
-from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import init_ops
-from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import nn_ops
-from tensorflow.python.ops import state_ops
 from tensorflow.python.ops import variable_scope
-from tensorflow.python.util import nest
-
-
-def _check_feature_shapes_compatible_with(
-    features, compatible_with_name, compatible_with_value, ignore=None):
-  """Checks all features are compatible with the given time-like feature."""
-  if ignore is None:
-    ignore = set()
-  for name, value in features.items():
-    if name in ignore:
-      continue
-    feature_shape = value.get_shape()
-    if feature_shape.ndims is None:
-      continue
-    if feature_shape.ndims < 2:
-      raise ValueError(
-          ("Features must have shape (batch dimension, window size, ...) "
-           "(got rank {} for feature '{}')").format(
-               feature_shape.ndims, name))
-    if not feature_shape[:2].is_compatible_with(
-        compatible_with_value.get_shape()):
-      raise ValueError(
-          ("Features must have shape (batch dimension, window size, ...) "
-           "where batch dimension and window size match the "
-           "'{times_feature}' feature (got shape {feature_shape} for "
-           "feature '{feature_name}' but shape {times_shape} for feature "
-           "'{times_feature}')").format(
-               times_feature=compatible_with_name,
-               feature_shape=feature_shape,
-               feature_name=name,
-               times_shape=compatible_with_value.get_shape()))
-
-
-def _check_predict_features(features):
-  """Raises errors if features are not suitable for prediction."""
-  if feature_keys.PredictionFeatures.TIMES not in features:
-    raise ValueError("Expected a '{}' feature for prediction.".format(
-        feature_keys.PredictionFeatures.TIMES))
-  if feature_keys.PredictionFeatures.STATE_TUPLE not in features:
-    raise ValueError("Expected a '{}' feature for prediction.".format(
-        feature_keys.PredictionFeatures.STATE_TUPLE))
-  times_feature = features[feature_keys.PredictionFeatures.TIMES]
-  if not times_feature.get_shape().is_compatible_with([None, None]):
-    raise ValueError(
-        ("Expected shape (batch dimension, window size) for feature '{}' "
-         "(got shape {})").format(feature_keys.PredictionFeatures.TIMES,
-                                  times_feature.get_shape()))
-  _check_feature_shapes_compatible_with(
-      features=features,
-      compatible_with_name=feature_keys.PredictionFeatures.TIMES,
-      compatible_with_value=times_feature,
-      ignore=set([
-          feature_keys.PredictionFeatures.STATE_TUPLE  # Model-dependent shapes
-      ]))
-
-
-def _check_train_eval_features(features, model):
-  """Raise errors if features are not suitable for training/evaluation."""
-  if feature_keys.TrainEvalFeatures.TIMES not in features:
-    raise ValueError("Expected a '{}' feature for training/evaluation.".format(
-        feature_keys.TrainEvalFeatures.TIMES))
-  if feature_keys.TrainEvalFeatures.VALUES not in features:
-    raise ValueError("Expected a '{}' feature for training/evaluation.".format(
-        feature_keys.TrainEvalFeatures.VALUES))
-  times_feature = features[feature_keys.TrainEvalFeatures.TIMES]
-  if not times_feature.get_shape().is_compatible_with([None, None]):
-    raise ValueError(
-        ("Expected shape (batch dimension, window size) for feature '{}' "
-         "(got shape {})").format(feature_keys.TrainEvalFeatures.TIMES,
-                                  times_feature.get_shape()))
-  values_feature = features[feature_keys.TrainEvalFeatures.VALUES]
-  if not values_feature.get_shape().is_compatible_with(
-      [None, None, model.num_features]):
-    raise ValueError(
-        ("Expected shape (batch dimension, window size, {num_features}) "
-         "for feature '{feature_name}', since the model was configured "
-         "with num_features={num_features} (got shape {got_shape})").format(
-             num_features=model.num_features,
-             feature_name=feature_keys.TrainEvalFeatures.VALUES,
-             got_shape=times_feature.get_shape()))
-  _check_feature_shapes_compatible_with(
-      features=features,
-      compatible_with_name=feature_keys.TrainEvalFeatures.TIMES,
-      compatible_with_value=times_feature,
-      ignore=set([
-          feature_keys.State.STATE_TUPLE  # Model-dependent shapes
-      ]))
-
-
-def _identity_metric_single(name, input_tensor):
-  """A metric which takes on its last updated value.
-
-  This keeps evaluation metrics in sync with one another, since update ops are
-  run separately from their result Tensors. Simply returning (input_tensor,
-  no_op) as a metric with a value but no update means that a metric will come
-  from a different batch of data than metrics which cache values in a Variable
-  (e.g. the default loss metric).
-
-  Args:
-    name: A name for the metric.
-    input_tensor: Any Tensor.
-  Returns:
-    A tuple of (value, update_op).
-  """
-  metric_variable = variable_scope.variable(
-      name="{}_identity_metric".format(name),
-      initial_value=array_ops.zeros([], dtype=input_tensor.dtype),
-      collections=[ops.GraphKeys.LOCAL_VARIABLES],
-      validate_shape=False)
-  update_op = state_ops.assign(metric_variable, input_tensor,
-                               validate_shape=False)
-  # This shape will be correct once the first update runs (but may be
-  # incomplete, so is not helpful for initializing the variable).
-  metric_variable.set_shape(input_tensor.get_shape())
-  return (metric_variable.value(), update_op)
-
-
-def _identity_metric_nested(name, input_tensors):
-  """Create identity metrics for a nested tuple of Tensors."""
-  update_ops = []
-  value_tensors = []
-  for tensor_number, tensor in enumerate(nest.flatten(input_tensors)):
-    value_tensor, update_op = _identity_metric_single(
-        name="{}_{}".format(name, tensor_number),
-        input_tensor=tensor)
-    update_ops.append(update_op)
-    value_tensors.append(value_tensor)
-  return (nest.pack_sequence_as(input_tensors, value_tensors),
-          control_flow_ops.group(*update_ops))
-
-
-def state_to_dictionary(state_tuple):
-  """Flatten model state into a dictionary with string keys."""
-  flattened = {}
-  for state_number, state_value in enumerate(nest.flatten(state_tuple)):
-    prefixed_state_name = "{}_{:02d}".format(feature_keys.State.STATE_PREFIX,
-                                             state_number)
-    flattened[prefixed_state_name] = state_value
-  return flattened
-
-
-def make_model_fn(
-    model, state_manager, optimizer, input_statistics_generator=None):
-  """Returns a model function suitable for use with a tf.estimator.
-
-  Args:
-    model: The object (inheriting from Model) to create a function for.
-    state_manager: A state manager to wrap the model with (or
-        PassthroughStateManager if no state needs to be managed).
-    optimizer: An instance of `tf.train.Optimizer` to use for training.
-    input_statistics_generator: An InputStatisticsFromMiniBatch object from
-        math_utils.py, used for collecting statistics about input data during
-        training.
-  Returns:
-    The model function, suitable for passing to a tf.estimator.Estimator.
-  """
-
-  def _convert_feature_to_tensor(name, value):
-    """Casts features to the correct dtype based on their name."""
-    if name in [
-        feature_keys.TrainEvalFeatures.TIMES,
-        feature_keys.PredictionFeatures.TIMES
-    ]:
-      return math_ops.cast(value, dtypes.int64)
-    if name == feature_keys.TrainEvalFeatures.VALUES:
-      return math_ops.cast(value, model.dtype)
-    if name == feature_keys.PredictionFeatures.STATE_TUPLE:
-      return value  # Correct dtypes are model-dependent
-    return ops.convert_to_tensor(value)
-
-  def _gather_state(features):
-    """Returns `features` with state packed, indicates if packing was done."""
-    prefixed_state_re = re.compile(r"^" + feature_keys.State.STATE_PREFIX +
-                                   r"_(\d+)$")
-    numbered_state = []
-    for key, tensor in features.items():
-      search_result = prefixed_state_re.search(key)
-      if search_result:
-        numbered_state.append((int(search_result.group(1)), key, tensor))
-    if not numbered_state:
-      return features, False
-    features = features.copy()
-    for _, key, _ in numbered_state:
-      del features[key]
-    numbered_state.sort(key=lambda number, *_: number)
-    features[feature_keys.State.STATE_TUPLE] = nest.pack_sequence_as(
-        structure=model.get_start_state(),
-        flat_sequence=[tensor for _, _, tensor in numbered_state])
-    return features, True
-
-  def _train(features):
-    """Add training ops to the graph."""
-    with variable_scope.variable_scope("model"):
-      model_outputs = state_manager.define_loss(model, features,
-                                                estimator_lib.ModeKeys.TRAIN)
-    train_op = optimizers.optimize_loss(
-        model_outputs.loss,
-        global_step=variables.get_global_step(),
-        optimizer=optimizer,
-        # Learning rate is set in the Optimizer object
-        learning_rate=None)
-    return estimator_lib.EstimatorSpec(
-        loss=model_outputs.loss,
-        mode=estimator_lib.ModeKeys.TRAIN,
-        train_op=train_op)
-
-  def _evaluate(features):
-    """Add ops for evaluation (aka filtering) to the graph."""
-    with variable_scope.variable_scope("model"):
-      model_outputs = state_manager.define_loss(model, features,
-                                                estimator_lib.ModeKeys.EVAL)
-    metrics = {}
-    # Just output in-sample predictions for the last chunk seen
-    for prediction_key, prediction_value in model_outputs.predictions.items():
-      metrics[prediction_key] = _identity_metric_single(prediction_key,
-                                                        prediction_value)
-    metrics[feature_keys.FilteringResults.TIMES] = _identity_metric_single(
-        feature_keys.FilteringResults.TIMES, model_outputs.prediction_times)
-    metrics[feature_keys.FilteringResults.STATE_TUPLE] = (
-        _identity_metric_nested(feature_keys.FilteringResults.STATE_TUPLE,
-                                model_outputs.end_state))
-    return estimator_lib.EstimatorSpec(
-        loss=model_outputs.loss,
-        mode=estimator_lib.ModeKeys.EVAL,
-        eval_metric_ops=metrics,
-        predictions={})
-
-  def _predict(features):
-    """Add ops for prediction to the graph."""
-    with variable_scope.variable_scope("model"):
-      prediction = model.predict(features=features)
-    prediction[feature_keys.PredictionResults.TIMES] = features[
-        feature_keys.PredictionFeatures.TIMES]
-    return estimator_lib.EstimatorSpec(
-        predictions=prediction, mode=estimator_lib.ModeKeys.PREDICT)
-
-  def _serving(features):
-    with variable_scope.variable_scope("model"):
-      prediction_outputs = model.predict(features=features)
-    with variable_scope.variable_scope("model", reuse=True):
-      filtering_outputs = state_manager.define_loss(model, features,
-                                                    estimator_lib.ModeKeys.EVAL)
-    return estimator_lib.EstimatorSpec(
-        mode=estimator_lib.ModeKeys.PREDICT,
-        export_outputs={
-            feature_keys.SavedModelLabels.PREDICT:
-                export_lib.PredictOutput(prediction_outputs),
-            feature_keys.SavedModelLabels.FILTER:
-                export_lib.PredictOutput(
-                    state_to_dictionary(filtering_outputs.end_state))
-        },
-        # Likely unused, but it is necessary to return `predictions` to satisfy
-        # the Estimator's error checking.
-        predictions={})
-
-  def _model_fn(features, labels, mode):
-    """Given a time series in `features`, define a loss for `mode`.
-
-    Args:
-      features: A dictionary, the output of a chunker (typically with keys
-          feature_keys.TrainEvalFeatures.TIMES and
-          feature_keys.TrainEvalFeatures.VALUES).
-      labels: Not used; included for compatibility with tf.learn.
-      mode: The tf.estimator.ModeKeys mode to use (TRAIN, EVAL, INFER).
-    Returns:
-      A tuple of predictions, a loss Tensor, and a train op.
-    Raises:
-      ValueError: If the model makes predictions which do not have static shape
-          information.
-    """
-    if labels:
-      raise ValueError("The model received a `labels` dictionary, which is not"
-                       " supported. Pass '{}' and '{}' as features.".format(
-                           feature_keys.TrainEvalFeatures.TIMES,
-                           feature_keys.TrainEvalFeatures.VALUES))
-    del labels
-    features = {name: _convert_feature_to_tensor(name=name, value=value)
-                for name, value in features.items()}
-    if input_statistics_generator is not None:
-      input_statistics = input_statistics_generator.initialize_graph(
-          features, update_statistics=(mode == estimator_lib.ModeKeys.TRAIN))
-    else:
-      input_statistics = None
-    model.initialize_graph(input_statistics=input_statistics)
-    # _gather_state requires the model to have its graph initialized (so it has
-    # access to the structure of the model's state)
-    features, passed_flat_state = _gather_state(features)
-    if (mode == estimator_lib.ModeKeys.TRAIN
-        or mode == estimator_lib.ModeKeys.EVAL):
-      _check_train_eval_features(features, model)
-    elif mode == estimator_lib.ModeKeys.PREDICT:
-      _check_predict_features(features)
-    else:
-      raise ValueError("Unknown mode '{}' passed to model_fn.".format(mode))
-    state_manager.initialize_graph(
-        model=model, input_statistics=input_statistics)
-    if mode == estimator_lib.ModeKeys.TRAIN:
-      return _train(features)
-    elif mode == estimator_lib.ModeKeys.EVAL:
-      return _evaluate(features)
-    elif mode == estimator_lib.ModeKeys.PREDICT and not passed_flat_state:
-      return _predict(features)
-    elif mode == estimator_lib.ModeKeys.PREDICT and passed_flat_state:
-      # The mode is PREDICT, but we're actually in export_savedmodel for
-      # serving. We want to return two graphs: one for filtering (state + data
-      # -> state) and one for predicting (state -> prediction).
-      return _serving(features)
-  return _model_fn
 
 
 # TODO(agarwal): Remove and replace with functionality from tf.slim
diff --git a/tensorflow/contrib/timeseries/python/timeseries/model_utils_test.py b/tensorflow/contrib/timeseries/python/timeseries/model_utils_test.py
index 2998689554..cfd31cc70d 100644
--- a/tensorflow/contrib/timeseries/python/timeseries/model_utils_test.py
+++ b/tensorflow/contrib/timeseries/python/timeseries/model_utils_test.py
@@ -18,22 +18,10 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.contrib.timeseries.python.timeseries import feature_keys
-from tensorflow.contrib.timeseries.python.timeseries import model
 from tensorflow.contrib.timeseries.python.timeseries import model_utils
-from tensorflow.contrib.timeseries.python.timeseries import state_management
 
-from tensorflow.python.estimator import estimator_lib
-from tensorflow.python.framework import dtypes
-from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import math_ops
-from tensorflow.python.ops import metrics
-from tensorflow.python.ops import variables
 from tensorflow.python.platform import test
-from tensorflow.python.training import coordinator as coordinator_lib
-from tensorflow.python.training import queue_runner_impl
-from tensorflow.python.training import training as train
 
 
 class ModelUtilsTest(test.TestCase):
@@ -46,230 +34,6 @@ class ModelUtilsTest(test.TestCase):
       self.assertEqual(5, getter(parameter))
       self.assertEqual(4, getter(overridden_parameter))
 
-  def test_labels_provided_error(self):
-    model_fn = _stub_model_fn()
-    for mode in [estimator_lib.ModeKeys.TRAIN, estimator_lib.ModeKeys.EVAL,
-                 estimator_lib.ModeKeys.PREDICT]:
-      with self.assertRaisesRegexp(ValueError, "labels"):
-        model_fn(features={}, labels={"a": "b"}, mode=mode)
-
-  def test_unknown_mode(self):
-    model_fn = _stub_model_fn()
-    with self.assertRaisesRegexp(ValueError, "Unknown mode 'Not a mode'"):
-      model_fn(features={}, labels={}, mode="Not a mode")
-
-
-class _TickerModel(object):
-  num_features = 1
-  dtype = dtypes.float32
-
-  def initialize_graph(self, input_statistics):
-    pass
-
-  def define_loss(self, features, mode):
-    del mode  # unused
-    return model.ModelOutputs(
-        loss=features["ticker"],
-        end_state=(features["ticker"], features["ticker"]),
-        prediction_times=array_ops.zeros(()),
-        predictions={"ticker": features["ticker"]})
-
-
-class EvaluationMetricsTests(test.TestCase):
-
-  def test_metrics_consistent(self):
-    # Tests that the identity metrics used to report in-sample predictions match
-    # the behavior of standard metrics.
-    g = ops.Graph()
-    with g.as_default():
-      features = {
-          feature_keys.TrainEvalFeatures.TIMES:
-              array_ops.zeros((1, 1)),
-          feature_keys.TrainEvalFeatures.VALUES:
-              array_ops.zeros((1, 1, 1)),
-          "ticker":
-              array_ops.reshape(
-                  math_ops.cast(
-                      variables.Variable(
-                          name="ticker",
-                          initial_value=0,
-                          dtype=dtypes.int64,
-                          collections=[ops.GraphKeys.LOCAL_VARIABLES])
-                      .count_up_to(10),
-                      dtype=dtypes.float32), (1, 1, 1))
-      }
-      model_fn = model_utils.make_model_fn(
-          model=_TickerModel(),
-          state_manager=state_management.PassthroughStateManager(),
-          optimizer=train.GradientDescentOptimizer(0.001))
-      outputs = model_fn(
-          features=features, labels=None, mode=estimator_lib.ModeKeys.EVAL)
-      metric_update_ops = [
-          metric[1] for metric in outputs.eval_metric_ops.values()]
-      loss_mean, loss_update = metrics.mean(outputs.loss)
-      metric_update_ops.append(loss_update)
-      with self.test_session() as sess:
-        coordinator = coordinator_lib.Coordinator()
-        queue_runner_impl.start_queue_runners(sess, coord=coordinator)
-        variables.local_variables_initializer().run()
-        sess.run(metric_update_ops)
-        loss_evaled, metric_evaled, nested_metric_evaled = sess.run(
-            (loss_mean, outputs.eval_metric_ops["ticker"][0],
-             outputs.eval_metric_ops[feature_keys.FilteringResults.STATE_TUPLE][
-                 0][0]))
-        # The custom model_utils metrics for in-sample predictions should be in
-        # sync with the Estimator's mean metric for model loss.
-        self.assertAllClose(0., loss_evaled)
-        self.assertAllClose((((0.,),),), metric_evaled)
-        self.assertAllClose((((0.,),),), nested_metric_evaled)
-        coordinator.request_stop()
-        coordinator.join()
-
-
-class _StubModel(object):
-  num_features = 3
-  dtype = dtypes.float64
-
-  def initialize_graph(self, input_statistics):
-    del input_statistics  # unused
-
-
-def _stub_model_fn():
-  return model_utils.make_model_fn(
-      model=_StubModel(),
-      state_manager=state_management.PassthroughStateManager(),
-      optimizer=train.AdamOptimizer(0.001))
-
-
-class TrainEvalFeatureCheckingTests(test.TestCase):
-
-  def test_no_time_feature(self):
-    model_fn = _stub_model_fn()
-    for mode in [estimator_lib.ModeKeys.TRAIN, estimator_lib.ModeKeys.EVAL]:
-      with self.assertRaisesRegexp(ValueError, "Expected a '{}' feature".format(
-          feature_keys.TrainEvalFeatures.TIMES)):
-        model_fn(
-            features={feature_keys.TrainEvalFeatures.VALUES: [[[1.]]]},
-            labels=None,
-            mode=mode)
-
-  def test_no_value_feature(self):
-    model_fn = _stub_model_fn()
-    for mode in [estimator_lib.ModeKeys.TRAIN, estimator_lib.ModeKeys.EVAL]:
-      with self.assertRaisesRegexp(ValueError, "Expected a '{}' feature".format(
-          feature_keys.TrainEvalFeatures.VALUES)):
-        model_fn(
-            features={feature_keys.TrainEvalFeatures.TIMES: [[1]]},
-            labels=None,
-            mode=mode)
-
-  def test_bad_time_rank(self):
-    model_fn = _stub_model_fn()
-    for mode in [estimator_lib.ModeKeys.TRAIN, estimator_lib.ModeKeys.EVAL]:
-      with self.assertRaisesRegexp(ValueError,
-                                   "Expected shape.*for feature '{}'".format(
-                                       feature_keys.TrainEvalFeatures.TIMES)):
-        model_fn(
-            features={
-                feature_keys.TrainEvalFeatures.TIMES: [[[1]]],
-                feature_keys.TrainEvalFeatures.VALUES: [[[1.]]]
-            },
-            labels=None,
-            mode=mode)
-
-  def test_bad_value_rank(self):
-    model_fn = _stub_model_fn()
-    for mode in [estimator_lib.ModeKeys.TRAIN, estimator_lib.ModeKeys.EVAL]:
-      with self.assertRaisesRegexp(ValueError,
-                                   "Expected shape.*for feature '{}'".format(
-                                       feature_keys.TrainEvalFeatures.VALUES)):
-        model_fn(
-            features={
-                feature_keys.TrainEvalFeatures.TIMES: [[1]],
-                feature_keys.TrainEvalFeatures.VALUES: [[1.]]
-            },
-            labels=None,
-            mode=mode)
-
-  def test_bad_value_num_features(self):
-    model_fn = _stub_model_fn()
-    for mode in [estimator_lib.ModeKeys.TRAIN, estimator_lib.ModeKeys.EVAL]:
-      with self.assertRaisesRegexp(
-          ValueError, "Expected shape.*, 3.*for feature '{}'".format(
-              feature_keys.TrainEvalFeatures.VALUES)):
-        model_fn(
-            features={
-                feature_keys.TrainEvalFeatures.TIMES: [[1]],
-                feature_keys.TrainEvalFeatures.VALUES: [[[1.]]]
-            },
-            labels=None,
-            mode=mode)
-
-  def test_bad_exogenous_shape(self):
-    model_fn = _stub_model_fn()
-    for mode in [estimator_lib.ModeKeys.TRAIN, estimator_lib.ModeKeys.EVAL]:
-      with self.assertRaisesRegexp(
-          ValueError,
-          "Features must have shape.*for feature 'exogenous'"):
-        model_fn(
-            features={
-                feature_keys.TrainEvalFeatures.TIMES: [[1]],
-                feature_keys.TrainEvalFeatures.VALUES: [[[1., 2., 3.]]],
-                "exogenous": [[1], [2]]
-            },
-            labels=None,
-            mode=mode)
-
-
-class PredictFeatureCheckingTests(test.TestCase):
-
-  def test_no_time_feature(self):
-    model_fn = _stub_model_fn()
-    with self.assertRaisesRegexp(ValueError, "Expected a '{}' feature".format(
-        feature_keys.PredictionFeatures.TIMES)):
-      model_fn(
-          features={
-              feature_keys.PredictionFeatures.STATE_TUPLE: ([[[1.]]], 1.)
-          },
-          labels=None,
-          mode=estimator_lib.ModeKeys.PREDICT)
-
-  def test_no_start_state_feature(self):
-    model_fn = _stub_model_fn()
-    with self.assertRaisesRegexp(ValueError, "Expected a '{}' feature".format(
-        feature_keys.PredictionFeatures.STATE_TUPLE)):
-      model_fn(
-          features={feature_keys.PredictionFeatures.TIMES: [[1]]},
-          labels=None,
-          mode=estimator_lib.ModeKeys.PREDICT)
-
-  def test_bad_time_rank(self):
-    model_fn = _stub_model_fn()
-    with self.assertRaisesRegexp(ValueError,
-                                 "Expected shape.*for feature '{}'".format(
-                                     feature_keys.PredictionFeatures.TIMES)):
-      model_fn(
-          features={
-              feature_keys.PredictionFeatures.TIMES: 1,
-              feature_keys.PredictionFeatures.STATE_TUPLE: (1, (2, 3.))
-          },
-          labels=None,
-          mode=estimator_lib.ModeKeys.PREDICT)
-
-  def test_bad_exogenous_shape(self):
-    model_fn = _stub_model_fn()
-    with self.assertRaisesRegexp(
-        ValueError,
-        "Features must have shape.*for feature 'exogenous'"):
-      model_fn(
-          features={
-              feature_keys.PredictionFeatures.TIMES: [[1]],
-              feature_keys.PredictionFeatures.STATE_TUPLE: (1, (2, 3.)),
-              "exogenous": 1.
-          },
-          labels=None,
-          mode=estimator_lib.ModeKeys.PREDICT)
-
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/contrib/timeseries/python/timeseries/saved_model_utils.py b/tensorflow/contrib/timeseries/python/timeseries/saved_model_utils.py
index 16e29f5e68..97f6d36a87 100644
--- a/tensorflow/contrib/timeseries/python/timeseries/saved_model_utils.py
+++ b/tensorflow/contrib/timeseries/python/timeseries/saved_model_utils.py
@@ -23,6 +23,7 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.contrib.timeseries.python.timeseries import feature_keys as _feature_keys
+from tensorflow.contrib.timeseries.python.timeseries import head as _head
 from tensorflow.contrib.timeseries.python.timeseries import input_pipeline as _input_pipeline
 from tensorflow.contrib.timeseries.python.timeseries import model_utils as _model_utils
 
@@ -34,7 +35,7 @@ def _colate_features_to_feeds_and_fetches(continue_from, signature, features,
   """Uses a saved model signature to construct feed and fetch dictionaries."""
   if _feature_keys.FilteringResults.STATE_TUPLE in continue_from:
     # We're continuing from an evaluation, so we need to unpack/flatten state.
-    state_values = _model_utils.state_to_dictionary(
+    state_values = _head.state_to_dictionary(
         continue_from[_feature_keys.FilteringResults.STATE_TUPLE])
   else:
     state_values = continue_from
diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index eb66d8e329..f3e43dd552 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -1773,6 +1773,7 @@ tf_cuda_library(
     ) + if_mkl(
         [
             "//third_party/mkl:intel_binary_blob",
+            "@mkl_dnn//:mkl_dnn",
         ],
     ),
     alwayslink = 1,
@@ -1933,7 +1934,7 @@ CORE_CPU_LIB_HEADERS = CORE_CPU_BASE_HDRS + [
     "common_runtime/visitable_allocator.h",
     "graph/gradients.h",
     "graph/quantize_training.h",
-]
+] + if_mkl(["graph/mkl_graph_util.h"])
 
 tf_cuda_library(
     name = "core_cpu_impl",
@@ -2034,7 +2035,10 @@ tf_cuda_library(
         "//third_party/eigen3",
         "//tensorflow/core/kernels:required",
     ] + if_mkl(
-        ["//third_party/mkl:intel_binary_blob"],
+        [
+            "//third_party/mkl:intel_binary_blob",
+            "@mkl_dnn//:mkl_dnn",
+        ],
     ) + tf_additional_core_deps() + if_static([":core_cpu_impl"]),
     alwayslink = 1,
 )
@@ -2670,7 +2674,7 @@ tf_cc_test_mkl(
         "graph/mkl_layout_pass_test.cc",
         "graph/mkl_tfconversion_pass_test.cc",
     ],
-    linkstatic = tf_kernel_tests_linkstatic(),
+    linkstatic = 1,
     deps = [
         ":core",
         ":core_cpu",
@@ -2688,18 +2692,6 @@ tf_cc_test_mkl(
         "//tensorflow/cc:cc_ops",
         "//tensorflow/cc:scope",
         "//tensorflow/cc:sendrecv_ops",
-        "//tensorflow/core/kernels:mkl_aggregate_ops",
-        "//tensorflow/core/kernels:mkl_concat_op",
-        "//tensorflow/core/kernels:mkl_conv_op",
-        "//tensorflow/core/kernels:mkl_cwise_ops_common",
-        "//tensorflow/core/kernels:mkl_fused_batch_norm_op",
-        "//tensorflow/core/kernels:mkl_identity_op",
-        "//tensorflow/core/kernels:mkl_input_conversion_op",
-        "//tensorflow/core/kernels:mkl_lrn_op",
-        "//tensorflow/core/kernels:mkl_pooling_ops",
-        "//tensorflow/core/kernels:mkl_relu_op",
-        "//tensorflow/core/kernels:mkl_reshape_op",
-        "//tensorflow/core/kernels:mkl_tfconv_op",
         "//tensorflow/core/kernels:ops_util",
         "//third_party/eigen3",
     ],
diff --git a/tensorflow/core/graph/mkl_graph_util.h b/tensorflow/core/graph/mkl_graph_util.h
new file mode 100644
index 0000000000..cb32d64334
--- /dev/null
+++ b/tensorflow/core/graph/mkl_graph_util.h
@@ -0,0 +1,128 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_CORE_GRAPH_MKL_GRAPH_UTIL_H_
+#define TENSORFLOW_CORE_GRAPH_MKL_GRAPH_UTIL_H_
+#ifdef INTEL_MKL
+
+#include <string>
+#include "tensorflow/core/framework/op_kernel.h"
+
+namespace tensorflow {
+// Since our ops are going to produce and also consume N addition tensors
+// (Mkl) for N Tensorflow tensors, we can have following different
+// orderings among these 2N tensors.
+//
+// E.g., for Tensorflow tensors A, B, and C, our ops will produce and
+// consume A_m, B_m, and C_m additionally.
+//
+// INTERLEAVED: in this case 2N tensors are interleaved. So for above
+//              example, the ordering looks like: A, A_m, B, B_m, C, C_m.
+//
+// CONTIGUOUS: in thi case N Tensorflow tensors are contiguous followed
+//             by N Mkl tensors. So for above example, the ordering looks
+//             like: A, B, C, A_m, B_m, C_m
+//
+// Following APIs map index of original Tensorflow tensors to their
+// appropriate position based on selected ordering. For contiguous ordering,
+// we need to know the total number of tensors (parameter total).
+//
+typedef enum { TENSORS_INTERLEAVED, TENSORS_CONTIGUOUS } MklTfTensorOrdering;
+// NOTE: Currently, we use contiguous ordering. If you change this, then you
+// would need to change Mkl op definitions in nn_ops.cc.
+static MklTfTensorOrdering kTensorOrdering = TENSORS_CONTIGUOUS;
+
+// Get index of MetaData tensor from index 'n' of Data tensor.
+inline int DataIndexToMetaDataIndex(int n, int total_tensors) {
+  if (kTensorOrdering == MklTfTensorOrdering::TENSORS_INTERLEAVED) {
+    // For interleaved ordering, Mkl tensor follows immediately after
+    // Tensorflow tensor.
+    return n + 1;
+  } else {
+    CHECK_EQ(kTensorOrdering, MklTfTensorOrdering::TENSORS_CONTIGUOUS);
+    // For contiguous ordering, Mkl tensor is n+total_tensors / 2 away.
+    return n + total_tensors / 2;
+  }
+}
+
+int inline GetTensorDataIndex(int n, int total_tensors) {
+  if (kTensorOrdering == MklTfTensorOrdering::TENSORS_INTERLEAVED) {
+    return 2 * n;  // index corresponding to nth input/output tensor
+  } else {
+    CHECK_EQ(kTensorOrdering, MklTfTensorOrdering::TENSORS_CONTIGUOUS);
+    return n;
+  }
+}
+
+int inline GetTensorMetaDataIndex(int n, int total_tensors) {
+  // Get index for TensorData first and then use mapping function
+  // to get TensorMetaData index from TensorData index.
+  int tidx = GetTensorDataIndex(n, total_tensors);
+  return DataIndexToMetaDataIndex(tidx, total_tensors);
+}
+
+namespace mkl_op_registry {
+static const char* kMklOpLabel = "MklOp";
+static const char* kMklOpLabelPattern = "label='MklOp'";
+
+// Get the name of Mkl op from original TensorFlow op
+// We prefix 'Mkl' to the original op to get Mkl op.
+inline string GetMklOpName(const string& name) {
+  // Prefix that we add to Tensorflow op name to construct Mkl op name.
+  const char* const kMklOpPrefix = "_Mkl";
+  return string(kMklOpPrefix) + name;
+}
+
+// Check whether opname with type T is registered as MKL-compliant.
+//
+// @input: name of the op
+// @input: T datatype to be used for checking op
+// @return: true if opname is registered as Mkl op; false otherwise
+static inline bool IsMklOp(const std::string& op_name, DataType T) {
+  string kernel = KernelsRegisteredForOp(op_name);
+  bool result =
+      kernel.find(kMklOpLabelPattern) != string::npos && (T == DT_FLOAT);
+  if (result) {
+    VLOG(1) << "mkl_op_registry::" << op_name << " is " << kMklOpLabel;
+  }
+  return result;
+}
+
+// Check whether opname with type T is registered as MKL-compliant and
+// is element-wise.
+//
+// @input: name of the op
+// @input: T datatype to be used for checking op
+// @return: true if opname is registered as element-wise Mkl op;
+// false otherwise
+static inline bool IsMklElementWiseOp(const std::string& op_name, DataType T) {
+  if (!IsMklOp(op_name, T)) {
+    return false;
+  }
+
+  bool result = (0 == op_name.compare(GetMklOpName("Add")) ||
+                 0 == op_name.compare(GetMklOpName("Sub")) ||
+                 0 == op_name.compare(GetMklOpName("Mul")) ||
+                 0 == op_name.compare(GetMklOpName("Maximum")) ||
+                 0 == op_name.compare(GetMklOpName("SquaredDifference")));
+
+  VLOG(1) << "mkl_op_registry::" << op_name
+          << " is elementwise MKL op: " << result;
+  return result;
+}
+}  // namespace mkl_op_registry
+}  // namespace tensorflow
+#endif  // INTEL_MKL
+#endif  // TENSORFLOW_CORE_GRAPH_MKL_GRAPH_UTIL_H_
diff --git a/tensorflow/core/graph/mkl_layout_pass.cc b/tensorflow/core/graph/mkl_layout_pass.cc
index 90377e54c7..f87a94a76a 100644
--- a/tensorflow/core/graph/mkl_layout_pass.cc
+++ b/tensorflow/core/graph/mkl_layout_pass.cc
@@ -37,8 +37,8 @@ limitations under the License.
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/util/tensor_format.h"
 
+#include "tensorflow/core/graph/mkl_graph_util.h"
 #include "tensorflow/core/graph/mkl_layout_pass.h"
-#include "tensorflow/core/util/mkl_util.h"
 
 namespace tensorflow {
 
diff --git a/tensorflow/core/graph/mkl_layout_pass_test.cc b/tensorflow/core/graph/mkl_layout_pass_test.cc
index 6a41e3965a..a2b2f6530d 100644
--- a/tensorflow/core/graph/mkl_layout_pass_test.cc
+++ b/tensorflow/core/graph/mkl_layout_pass_test.cc
@@ -16,7 +16,7 @@ limitations under the License.
 #ifdef INTEL_MKL
 
 #include "tensorflow/core/graph/mkl_layout_pass.h"
-#include "tensorflow/core/util/mkl_util.h"
+#include "tensorflow/core/graph/mkl_graph_util.h"
 
 #include <algorithm>
 #include <string>
diff --git a/tensorflow/core/graph/mkl_tfconversion_pass.cc b/tensorflow/core/graph/mkl_tfconversion_pass.cc
index 3f8b0e86d0..fe4588389e 100644
--- a/tensorflow/core/graph/mkl_tfconversion_pass.cc
+++ b/tensorflow/core/graph/mkl_tfconversion_pass.cc
@@ -33,8 +33,8 @@ limitations under the License.
 #include "tensorflow/core/lib/hash/hash.h"
 #include "tensorflow/core/platform/logging.h"
 
+#include "tensorflow/core/graph/mkl_graph_util.h"
 #include "tensorflow/core/graph/mkl_tfconversion_pass.h"
-#include "tensorflow/core/util/mkl_util.h"
 
 namespace tensorflow {
 
diff --git a/tensorflow/core/graph/mkl_tfconversion_pass_test.cc b/tensorflow/core/graph/mkl_tfconversion_pass_test.cc
index b01818f746..bbdbe78bbd 100644
--- a/tensorflow/core/graph/mkl_tfconversion_pass_test.cc
+++ b/tensorflow/core/graph/mkl_tfconversion_pass_test.cc
@@ -16,7 +16,7 @@ limitations under the License.
 #ifdef INTEL_MKL
 
 #include "tensorflow/core/graph/mkl_tfconversion_pass.h"
-#include "tensorflow/core/util/mkl_util.h"
+#include "tensorflow/core/graph/mkl_graph_util.h"
 
 #include <algorithm>
 #include <string>
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 36fbf6b023..bdc6faefbc 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -820,6 +820,7 @@ tf_kernel_library(
     hdrs = ["transpose_op.h"],
     deps = ARRAY_DEPS + if_mkl([
         "//third_party/mkl:intel_binary_blob",
+        "@mkl_dnn//:mkl_dnn",
     ]),
 )
 
@@ -2596,6 +2597,7 @@ tf_kernel_library(
         "//conditions:default": [],
     }) + if_mkl([
         "//third_party/mkl:intel_binary_blob",
+        "@mkl_dnn//:mkl_dnn",
     ]) + if_cuda([
         "//tensorflow/core/platform/default/build_config:cublas_plugin",
     ]),
@@ -5501,8 +5503,10 @@ tf_mkl_kernel_library(
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
         "//tensorflow/core:nn_ops_op_lib",
+    ] + if_mkl([
         "//third_party/mkl:intel_binary_blob",
-    ],
+        "@mkl_dnn//:mkl_dnn",
+    ]),
 )
 
 tf_mkl_kernel_library(
@@ -5516,8 +5520,10 @@ tf_mkl_kernel_library(
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
         "//tensorflow/core:nn_ops_op_lib",
+    ] + if_mkl([
         "//third_party/mkl:intel_binary_blob",
-    ],
+        "@mkl_dnn//:mkl_dnn",
+    ]),
 )
 
 tf_mkl_kernel_library(
@@ -5566,16 +5572,19 @@ tf_mkl_kernel_library(
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
         "//tensorflow/core:nn_ops_op_lib",
+    ] + if_mkl([
         "//third_party/mkl:intel_binary_blob",
-    ],
+        "@mkl_dnn//:mkl_dnn",
+    ]),
 )
 
 tf_mkl_kernel_library(
     name = "mkl_fused_batch_norm_op",
     srcs = ["mkl_fused_batch_norm_op.cc"],
-    deps = NN_DEPS + [
+    deps = NN_DEPS + if_mkl([
         "//third_party/mkl:intel_binary_blob",
-    ],
+        "@mkl_dnn//:mkl_dnn",
+    ]),
 )
 
 tf_mkl_kernel_library(
@@ -5589,9 +5598,10 @@ tf_mkl_kernel_library(
 tf_mkl_kernel_library(
     name = "mkl_concat_op",
     prefix = "mkl_concat_op",
-    deps = ARRAY_DEPS + [
+    deps = ARRAY_DEPS + if_mkl([
         "//third_party/mkl:intel_binary_blob",
-    ],
+        "@mkl_dnn//:mkl_dnn",
+    ]),
 )
 
 tf_mkl_kernel_library(
@@ -5605,17 +5615,19 @@ tf_mkl_kernel_library(
 tf_mkl_kernel_library(
     name = "mkl_identity_op",
     prefix = "mkl_identity_op",
-    deps = ARRAY_DEPS + [
+    deps = ARRAY_DEPS + if_mkl([
         "//third_party/mkl:intel_binary_blob",
-    ],
+        "@mkl_dnn//:mkl_dnn",
+    ]),
 )
 
 tf_mkl_kernel_library(
     name = "mkl_lrn_op",
     prefix = "mkl_lrn_op",
-    deps = NN_DEPS + [
+    deps = NN_DEPS + if_mkl([
         "//third_party/mkl:intel_binary_blob",
-    ],
+        "@mkl_dnn//:mkl_dnn",
+    ]),
 )
 
 tf_mkl_kernel_library(
diff --git a/tensorflow/core/kernels/bias_op.cc b/tensorflow/core/kernels/bias_op.cc
index 1bdfafb89b..368993c827 100644
--- a/tensorflow/core/kernels/bias_op.cc
+++ b/tensorflow/core/kernels/bias_op.cc
@@ -39,6 +39,48 @@ typedef Eigen::GpuDevice GPUDevice;
 typedef Eigen::SyclDevice SYCLDevice;
 #endif  // TENSORFLOW_USE_SYCL
 
+namespace {
+
+void GetBiasValueDims(const Tensor& value_tensor, TensorFormat data_format,
+                      int32* batch, int32* height, int32* width,
+                      int32* channel) {
+  *batch = 1;
+  *width = 1;
+  *height = 1;
+  *channel = 1;
+  if (data_format == FORMAT_NHWC) {
+    int32 channel_dim = value_tensor.dims() - 1;
+    *channel = static_cast<int32>(value_tensor.dim_size(channel_dim));
+    for (int32 i = 0; i < channel_dim; i++) {
+      *batch *= static_cast<int32>(value_tensor.dim_size(i));
+    }
+  } else if (data_format == FORMAT_NCHW) {
+    int32 channel_dim = value_tensor.dims() - 3;
+    int32 height_dim = value_tensor.dims() - 2;
+    int32 width_dim = value_tensor.dims() - 1;
+    *channel = static_cast<int32>(value_tensor.dim_size(channel_dim));
+    *height = static_cast<int32>(value_tensor.dim_size(height_dim));
+    *width = static_cast<int32>(value_tensor.dim_size(width_dim));
+    for (int32 i = 0; i < channel_dim; i++) {
+      *batch *= static_cast<int32>(value_tensor.dim_size(i));
+    }
+  }
+}
+
+template <class T>
+struct AccumulatorType {
+  typedef T type;
+};
+
+// float is faster on the CPU than half, and also more precise,
+// so use float for the temporary accumulators.
+template <>
+struct AccumulatorType<Eigen::half> {
+  typedef float type;
+};
+
+}  // namespace
+
 template <typename Device, typename T>
 class BiasOp : public BinaryOp<T> {
  public:
@@ -50,9 +92,6 @@ class BiasOp : public BinaryOp<T> {
     } else {
       data_format_ = FORMAT_NHWC;
     }
-    OP_REQUIRES(context, data_format_ == FORMAT_NHWC,
-                errors::InvalidArgument(context->device()->name() +
-                                        " BiasOp only supports NHWC."));
   }
 
   void Compute(OpKernelContext* context) override {
@@ -65,9 +104,21 @@ class BiasOp : public BinaryOp<T> {
     OP_REQUIRES(context, TensorShapeUtils::IsVector(bias.shape()),
                 errors::InvalidArgument("Biases must be 1D: ",
                                         bias.shape().DebugString()));
-    const auto last_dim = input.shape().dims() - 1;
+
+    // Added by intel_tf to support NCHW on CPU regardless of MKL used or not.
+    size_t channel_dim;
+    if (data_format_ == FORMAT_NCHW) {
+      OP_REQUIRES(context, input.dims() == 4,
+                  errors::InvalidArgument(
+                      "NCHW format supports only 4D input tensor."));
+      channel_dim = 1;
+    } else {
+      channel_dim = input.shape().dims() - 1;  // End of code by intel_tf.
+    }
+
     OP_REQUIRES(
-        context, bias.shape().dim_size(0) == input.shape().dim_size(last_dim),
+        context,
+        bias.shape().dim_size(0) == input.shape().dim_size(channel_dim),
         errors::InvalidArgument(
             "Must provide as many biases as the last dimension "
             "of the input tensor: ",
@@ -78,6 +129,19 @@ class BiasOp : public BinaryOp<T> {
                                 {0}, 0, input.shape(), &output));
     if (input.NumElements() == 0) return;
 
+    // Added by intel_tf to support NCHW on CPU regardless of MKL used or not.
+    if (data_format_ == FORMAT_NCHW) {
+      int32 batch, height, width, channel;
+      GetBiasValueDims(input, data_format_, &batch, &height, &width, &channel);
+      Eigen::DSizes<int32, 4> four_dims(1, channel, 1, 1);
+      Eigen::DSizes<int32, 4> broad_cast_dims(batch, 1, height, width);
+      const Device& d = context->eigen_device<Device>();
+      output->tensor<T, 4>().device(d) =
+          input.tensor<T, 4>() +
+          bias.tensor<T, 1>().reshape(four_dims).broadcast(broad_cast_dims);
+      return;
+    }  // End of code by intel_tf.
+
     switch (input.shape().dims()) {
       case 2:
         Compute<2>(context, input, bias, output);
@@ -137,48 +201,6 @@ REGISTER_KERNEL(double);
 #undef REGISTER_KERNEL
 #endif  // TENSORFLOW_USE_SYCL
 
-namespace {
-
-void GetBiasValueDims(const Tensor& value_tensor, TensorFormat data_format,
-                      int32* batch, int32* height, int32* width,
-                      int32* channel) {
-  *batch = 1;
-  *width = 1;
-  *height = 1;
-  *channel = 1;
-  if (data_format == FORMAT_NHWC) {
-    int32 channel_dim = value_tensor.dims() - 1;
-    *channel = static_cast<int32>(value_tensor.dim_size(channel_dim));
-    for (int32 i = 0; i < channel_dim; i++) {
-      *batch *= static_cast<int32>(value_tensor.dim_size(i));
-    }
-  } else if (data_format == FORMAT_NCHW) {
-    int32 channel_dim = value_tensor.dims() - 3;
-    int32 height_dim = value_tensor.dims() - 2;
-    int32 width_dim = value_tensor.dims() - 1;
-    *channel = static_cast<int32>(value_tensor.dim_size(channel_dim));
-    *height = static_cast<int32>(value_tensor.dim_size(height_dim));
-    *width = static_cast<int32>(value_tensor.dim_size(width_dim));
-    for (int32 i = 0; i < channel_dim; i++) {
-      *batch *= static_cast<int32>(value_tensor.dim_size(i));
-    }
-  }
-}
-
-template <class T>
-struct AccumulatorType {
-  typedef T type;
-};
-
-// float is faster on the CPU than half, and also more precise,
-// so use float for the temporary accumulators.
-template <>
-struct AccumulatorType<Eigen::half> {
-  typedef float type;
-};
-
-}  // namespace
-
 template <typename Device, typename T>
 class BiasGradOp : public OpKernel {
  public:
@@ -190,9 +212,6 @@ class BiasGradOp : public OpKernel {
     } else {
       data_format_ = FORMAT_NHWC;
     }
-    OP_REQUIRES(context, data_format_ == FORMAT_NHWC,
-                errors::InvalidArgument(context->device()->name() +
-                                        " BiasGradOp only supports NHWC."));
   }
 
   void Compute(OpKernelContext* context) override {
@@ -222,18 +241,40 @@ class BiasGradOp : public OpKernel {
       // Eigen often crashes by design on empty tensors, but setZero is safe
       output->template flat<T>().setZero();
     } else {
-      Eigen::DSizes<int, 2> two_dims(batch * height * width, channel);
+      // Added by intel_tf to support NCHW on CPU regardless of MKL used or not.
+      if (data_format_ == FORMAT_NCHW) {
+        OP_REQUIRES(context, output_backprop.dims() == 4,
+                    errors::InvalidArgument(
+                        "NCHW format supports only 4D input/output tensor."));
+        Eigen::DSizes<int, 4> four_dims(batch, channel, height, width);
+#ifdef EIGEN_HAS_INDEX_LIST
+        using idx0 = Eigen::type2index<0>;
+        using idx2 = Eigen::type2index<2>;
+        using idx3 = Eigen::type2index<3>;
+        Eigen::IndexList<idx0, idx2, idx3> reduction_axes;
+#else
+        Eigen::array<int, 3> reduction_axes = {0, 2, 3};
+#endif
+        output->template flat<T>().device(context->eigen_device<Device>()) =
+            output_backprop.flat<T>()
+                .template cast<typename AccumulatorType<T>::type>()
+                .reshape(four_dims)
+                .sum(reduction_axes)
+                .template cast<T>();  // End of code by intel_tf.
+      } else {
+        Eigen::DSizes<int, 2> two_dims(batch * height * width, channel);
 #ifdef EIGEN_HAS_INDEX_LIST
-      Eigen::IndexList<Eigen::type2index<0> > reduction_axis;
+        Eigen::IndexList<Eigen::type2index<0> > reduction_axis;
 #else
-      Eigen::array<int, 1> reduction_axis = {0};
+        Eigen::array<int, 1> reduction_axis = {0};
 #endif
-      output->template flat<T>().device(context->eigen_device<Device>()) =
-          output_backprop.flat<T>()
-              .template cast<typename AccumulatorType<T>::type>()
-              .reshape(two_dims)
-              .sum(reduction_axis)
-              .template cast<T>();
+        output->template flat<T>().device(context->eigen_device<Device>()) =
+            output_backprop.flat<T>()
+                .template cast<typename AccumulatorType<T>::type>()
+                .reshape(two_dims)
+                .sum(reduction_axis)
+                .template cast<T>();
+      }
     }
   }
 
diff --git a/tensorflow/core/kernels/conv_grad_filter_ops.cc b/tensorflow/core/kernels/conv_grad_filter_ops.cc
index 641077ca65..5e09963d2d 100644
--- a/tensorflow/core/kernels/conv_grad_filter_ops.cc
+++ b/tensorflow/core/kernels/conv_grad_filter_ops.cc
@@ -816,40 +816,35 @@ void LaunchConv2DBackpropFilterOp<Eigen::GpuDevice, T>::operator()(
   AlgorithmConfig algorithm_config;
   if (cudnn_use_autotune && !AutoTuneConvBwdFilter::GetInstance()->Find(
                                 conv_parameters, &algorithm_config)) {
-    std::vector<AlgorithmDesc::Index> algorithms;
+    std::vector<AlgorithmDesc> algorithms;
     CHECK(stream->parent()->GetConvolveBackwardFilterAlgorithms(
         conv_parameters.ShouldIncludeWinogradNonfusedAlgo<T>(), &algorithms));
     ProfileResult best_result;
     ProfileResult best_result_no_scratch;
-    // TODO(benbarsdell): Ideally this should not attempt using tensor op math
-    // if it's not enabled.
-    for (bool use_tensor_ops : {false, true}) {
-      for (auto algo_index : algorithms) {
-        // TODO(zhengxq): profile each algorithm multiple times to better
-        // accuracy.
-        AlgorithmDesc profile_algorithm(algo_index, use_tensor_ops);
-        CudnnScratchAllocator scratch_allocator(
-            ConvolveBackwardFilterScratchSize, ctx);
-        ProfileResult profile_result;
-        bool cudnn_launch_status =
-            stream
-                ->ThenConvolveBackwardFilterWithAlgorithm(
-                    input_desc, input_ptr, output_desc, out_backprop_ptr,
-                    conv_desc, filter_desc, &filter_backprop_ptr,
-                    &scratch_allocator, AlgorithmConfig(profile_algorithm),
-                    &profile_result)
-                .ok();
-        if (cudnn_launch_status) {
-          if (profile_result.is_valid()) {
-            if (profile_result.elapsed_time_in_ms() <
-                best_result.elapsed_time_in_ms()) {
-              best_result = profile_result;
-            }
-            if (scratch_allocator.TotalByteSize() == 0 &&
-                profile_result.elapsed_time_in_ms() <
-                    best_result_no_scratch.elapsed_time_in_ms()) {
-              best_result_no_scratch = profile_result;
-            }
+    for (auto profile_algorithm : algorithms) {
+      // TODO(zhengxq): profile each algorithm multiple times to better
+      // accuracy.
+      CudnnScratchAllocator scratch_allocator(ConvolveBackwardFilterScratchSize,
+                                              ctx);
+      ProfileResult profile_result;
+      bool cudnn_launch_status =
+          stream
+              ->ThenConvolveBackwardFilterWithAlgorithm(
+                  input_desc, input_ptr, output_desc, out_backprop_ptr,
+                  conv_desc, filter_desc, &filter_backprop_ptr,
+                  &scratch_allocator, AlgorithmConfig(profile_algorithm),
+                  &profile_result)
+              .ok();
+      if (cudnn_launch_status) {
+        if (profile_result.is_valid()) {
+          if (profile_result.elapsed_time_in_ms() <
+              best_result.elapsed_time_in_ms()) {
+            best_result = profile_result;
+          }
+          if (scratch_allocator.TotalByteSize() == 0 &&
+              profile_result.elapsed_time_in_ms() <
+                  best_result_no_scratch.elapsed_time_in_ms()) {
+            best_result_no_scratch = profile_result;
           }
         }
       }
diff --git a/tensorflow/core/kernels/conv_grad_input_ops.cc b/tensorflow/core/kernels/conv_grad_input_ops.cc
index 0732bf4046..0b2d01afa9 100644
--- a/tensorflow/core/kernels/conv_grad_input_ops.cc
+++ b/tensorflow/core/kernels/conv_grad_input_ops.cc
@@ -870,39 +870,34 @@ void LaunchConv2DBackpropInputOp<GPUDevice, T>::operator()(
   AlgorithmConfig algorithm_config;
   if (cudnn_use_autotune && !AutoTuneConvBwdData::GetInstance()->Find(
                                 conv_parameters, &algorithm_config)) {
-    std::vector<AlgorithmDesc::Index> algorithms;
+    std::vector<AlgorithmDesc> algorithms;
     CHECK(stream->parent()->GetConvolveBackwardDataAlgorithms(
         conv_parameters.ShouldIncludeWinogradNonfusedAlgo<T>(), &algorithms));
     ProfileResult best_result;
     ProfileResult best_result_no_scratch;
-    // TODO(benbarsdell): Ideally this should not attempt using tensor op math
-    // if it's not enabled.
-    for (bool use_tensor_ops : {false, true}) {
-      for (auto algo_index : algorithms) {
-        // TODO(zhengxq): profile each algorithm multiple times to better
-        // accuracy.
-        AlgorithmDesc profile_algorithm(algo_index, use_tensor_ops);
-        CudnnScratchAllocator scratch_allocator(ConvolveBackwardDataScratchSize,
-                                                ctx);
-        ProfileResult profile_result;
-        bool cudnn_launch_status =
-            stream
-                ->ThenConvolveBackwardDataWithAlgorithm(
-                    filter_desc, filter_ptr, output_desc, out_backprop_ptr,
-                    conv_desc, input_desc, &in_backprop_ptr, &scratch_allocator,
-                    AlgorithmConfig(profile_algorithm), &profile_result)
-                .ok();
-        if (cudnn_launch_status) {
-          if (profile_result.is_valid()) {
-            if (profile_result.elapsed_time_in_ms() <
-                best_result.elapsed_time_in_ms()) {
-              best_result = profile_result;
-            }
-            if (scratch_allocator.TotalByteSize() == 0 &&
-                profile_result.elapsed_time_in_ms() <
-                    best_result_no_scratch.elapsed_time_in_ms()) {
-              best_result_no_scratch = profile_result;
-            }
+    for (auto profile_algorithm : algorithms) {
+      // TODO(zhengxq): profile each algorithm multiple times to better
+      // accuracy.
+      CudnnScratchAllocator scratch_allocator(ConvolveBackwardDataScratchSize,
+                                              ctx);
+      ProfileResult profile_result;
+      bool cudnn_launch_status =
+          stream
+              ->ThenConvolveBackwardDataWithAlgorithm(
+                  filter_desc, filter_ptr, output_desc, out_backprop_ptr,
+                  conv_desc, input_desc, &in_backprop_ptr, &scratch_allocator,
+                  AlgorithmConfig(profile_algorithm), &profile_result)
+              .ok();
+      if (cudnn_launch_status) {
+        if (profile_result.is_valid()) {
+          if (profile_result.elapsed_time_in_ms() <
+              best_result.elapsed_time_in_ms()) {
+            best_result = profile_result;
+          }
+          if (scratch_allocator.TotalByteSize() == 0 &&
+              profile_result.elapsed_time_in_ms() <
+                  best_result_no_scratch.elapsed_time_in_ms()) {
+            best_result_no_scratch = profile_result;
           }
         }
       }
diff --git a/tensorflow/core/kernels/conv_grad_ops_3d.cc b/tensorflow/core/kernels/conv_grad_ops_3d.cc
index 8ad56053a8..21f5cb1716 100644
--- a/tensorflow/core/kernels/conv_grad_ops_3d.cc
+++ b/tensorflow/core/kernels/conv_grad_ops_3d.cc
@@ -654,40 +654,34 @@ class Conv3DBackpropInputOp<GPUDevice, T> : public OpKernel {
     AlgorithmConfig algorithm_config;
     if (cudnn_use_autotune_ && !AutoTuneConv3dBwdData::GetInstance()->Find(
                                    conv_parameters, &algorithm_config)) {
-      std::vector<AlgorithmDesc::Index> algorithms;
+      std::vector<AlgorithmDesc> algorithms;
       CHECK(stream->parent()->GetConvolveBackwardDataAlgorithms(
           conv_parameters.ShouldIncludeWinogradNonfusedAlgo<T>(), &algorithms));
       ProfileResult best_result;
       ProfileResult best_result_no_scratch;
-      // TODO(benbarsdell): Ideally this should not attempt using tensor op math
-      // if it's not enabled.
-      for (bool use_tensor_ops : {false, true}) {
-        for (auto algo_index : algorithms) {
-          // TODO(zhengxq): profile each algorithm multiple times to better
-          // accuracy.
-          AlgorithmDesc profile_algorithm(algo_index, use_tensor_ops);
-          CudnnScratchAllocator scratch_allocator(
-              ConvolveBackwardDataScratchSize, context);
-          ProfileResult profile_result;
-          bool cudnn_launch_status =
-              stream
-                  ->ThenConvolveBackwardDataWithAlgorithm(
-                      filter_desc, filter_ptr, output_desc, out_backprop_ptr,
-                      conv_desc, input_desc, &in_backprop_ptr,
-                      &scratch_allocator, AlgorithmConfig(profile_algorithm),
-                      &profile_result)
-                  .ok();
-          if (cudnn_launch_status) {
-            if (profile_result.is_valid()) {
-              if (profile_result.elapsed_time_in_ms() <
-                  best_result.elapsed_time_in_ms()) {
-                best_result = profile_result;
-              }
-              if (scratch_allocator.TotalByteSize() == 0 &&
-                  profile_result.elapsed_time_in_ms() <
-                      best_result_no_scratch.elapsed_time_in_ms()) {
-                best_result_no_scratch = profile_result;
-              }
+      for (auto profile_algorithm : algorithms) {
+        // TODO(zhengxq): profile each algorithm multiple times to better
+        // accuracy.
+        CudnnScratchAllocator scratch_allocator(ConvolveBackwardDataScratchSize,
+                                                context);
+        ProfileResult profile_result;
+        bool cudnn_launch_status =
+            stream
+                ->ThenConvolveBackwardDataWithAlgorithm(
+                    filter_desc, filter_ptr, output_desc, out_backprop_ptr,
+                    conv_desc, input_desc, &in_backprop_ptr, &scratch_allocator,
+                    AlgorithmConfig(profile_algorithm), &profile_result)
+                .ok();
+        if (cudnn_launch_status) {
+          if (profile_result.is_valid()) {
+            if (profile_result.elapsed_time_in_ms() <
+                best_result.elapsed_time_in_ms()) {
+              best_result = profile_result;
+            }
+            if (scratch_allocator.TotalByteSize() == 0 &&
+                profile_result.elapsed_time_in_ms() <
+                    best_result_no_scratch.elapsed_time_in_ms()) {
+              best_result_no_scratch = profile_result;
             }
           }
         }
@@ -1026,40 +1020,35 @@ class Conv3DBackpropFilterOp<GPUDevice, T> : public OpKernel {
     AlgorithmConfig algorithm_config;
     if (cudnn_use_autotune_ && !AutoTuneConv3dBwdFilter::GetInstance()->Find(
                                    conv_parameters, &algorithm_config)) {
-      std::vector<AlgorithmDesc::Index> algorithms;
+      std::vector<AlgorithmDesc> algorithms;
       CHECK(stream->parent()->GetConvolveBackwardFilterAlgorithms(
           conv_parameters.ShouldIncludeWinogradNonfusedAlgo<T>(), &algorithms));
       ProfileResult best_result;
       ProfileResult best_result_no_scratch;
-      // TODO(benbarsdell): Ideally this should not attempt using tensor op math
-      //                      if it's not enabled.
-      for (bool use_tensor_ops : {false, true}) {
-        for (auto algo_index : algorithms) {
-          // TODO(zhengxq): profile each algorithm multiple times to better
-          // accuracy.
-          AlgorithmDesc profile_algorithm(algo_index, use_tensor_ops);
-          CudnnScratchAllocator scratch_allocator(
-              ConvolveBackwardFilterScratchSize, context);
-          ProfileResult profile_result;
-          bool cudnn_launch_status =
-              stream
-                  ->ThenConvolveBackwardFilterWithAlgorithm(
-                      input_desc, input_ptr, output_desc, out_backprop_ptr,
-                      conv_desc, filter_desc, &filter_backprop_ptr,
-                      &scratch_allocator, AlgorithmConfig(profile_algorithm),
-                      &profile_result)
-                  .ok();
-          if (cudnn_launch_status) {
-            if (profile_result.is_valid()) {
-              if (profile_result.elapsed_time_in_ms() <
-                  best_result.elapsed_time_in_ms()) {
-                best_result = profile_result;
-              }
-              if (scratch_allocator.TotalByteSize() == 0 &&
-                  profile_result.elapsed_time_in_ms() <
-                      best_result_no_scratch.elapsed_time_in_ms()) {
-                best_result_no_scratch = profile_result;
-              }
+      for (auto profile_algorithm : algorithms) {
+        // TODO(zhengxq): profile each algorithm multiple times to better
+        // accuracy.
+        CudnnScratchAllocator scratch_allocator(
+            ConvolveBackwardFilterScratchSize, context);
+        ProfileResult profile_result;
+        bool cudnn_launch_status =
+            stream
+                ->ThenConvolveBackwardFilterWithAlgorithm(
+                    input_desc, input_ptr, output_desc, out_backprop_ptr,
+                    conv_desc, filter_desc, &filter_backprop_ptr,
+                    &scratch_allocator, AlgorithmConfig(profile_algorithm),
+                    &profile_result)
+                .ok();
+        if (cudnn_launch_status) {
+          if (profile_result.is_valid()) {
+            if (profile_result.elapsed_time_in_ms() <
+                best_result.elapsed_time_in_ms()) {
+              best_result = profile_result;
+            }
+            if (scratch_allocator.TotalByteSize() == 0 &&
+                profile_result.elapsed_time_in_ms() <
+                    best_result_no_scratch.elapsed_time_in_ms()) {
+              best_result_no_scratch = profile_result;
             }
           }
         }
diff --git a/tensorflow/core/kernels/conv_ops.cc b/tensorflow/core/kernels/conv_ops.cc
index dc03eeb658..bb67113fb0 100644
--- a/tensorflow/core/kernels/conv_ops.cc
+++ b/tensorflow/core/kernels/conv_ops.cc
@@ -662,38 +662,33 @@ void LaunchConv2DOp<GPUDevice, T>::operator()(
   AlgorithmConfig algorithm_config;
   if (cudnn_use_autotune &&
       !AutoTuneConv::GetInstance()->Find(conv_parameters, &algorithm_config)) {
-    std::vector<AlgorithmDesc::Index> algorithms;
+    std::vector<AlgorithmDesc> algorithms;
     CHECK(stream->parent()->GetConvolveAlgorithms(
         conv_parameters.ShouldIncludeWinogradNonfusedAlgo<T>(), &algorithms));
     ProfileResult best_result;
     ProfileResult best_result_no_scratch;
-    // TODO(benbarsdell): Ideally this should not attempt using tensor op math
-    // if it's not enabled.
-    for (bool use_tensor_ops : {false, true}) {
-      for (auto algo_index : algorithms) {
-        // TODO(zhengxq): profile each algorithm multiple times to better
-        // accuracy.
-        AlgorithmDesc profile_algorithm(algo_index, use_tensor_ops);
-        CudnnScratchAllocator scratch_allocator(ConvolveScratchSize, ctx);
-        ProfileResult profile_result;
-        bool cudnn_launch_status =
-            stream
-                ->ThenConvolveWithAlgorithm(
-                    input_desc, input_ptr, filter_desc, filter_ptr, conv_desc,
-                    output_desc, &output_ptr, &scratch_allocator,
-                    AlgorithmConfig(profile_algorithm), &profile_result)
-                .ok();
-        if (cudnn_launch_status) {
-          if (profile_result.is_valid()) {
-            if (profile_result.elapsed_time_in_ms() <
-                best_result.elapsed_time_in_ms()) {
-              best_result = profile_result;
-            }
-            if (scratch_allocator.TotalByteSize() == 0 &&
-                profile_result.elapsed_time_in_ms() <
-                    best_result_no_scratch.elapsed_time_in_ms()) {
-              best_result_no_scratch = profile_result;
-            }
+    for (auto profile_algorithm : algorithms) {
+      // TODO(zhengxq): profile each algorithm multiple times to better
+      // accuracy.
+      CudnnScratchAllocator scratch_allocator(ConvolveScratchSize, ctx);
+      ProfileResult profile_result;
+      bool cudnn_launch_status =
+          stream
+              ->ThenConvolveWithAlgorithm(
+                  input_desc, input_ptr, filter_desc, filter_ptr, conv_desc,
+                  output_desc, &output_ptr, &scratch_allocator,
+                  AlgorithmConfig(profile_algorithm), &profile_result)
+              .ok();
+      if (cudnn_launch_status) {
+        if (profile_result.is_valid()) {
+          if (profile_result.elapsed_time_in_ms() <
+              best_result.elapsed_time_in_ms()) {
+            best_result = profile_result;
+          }
+          if (scratch_allocator.TotalByteSize() == 0 &&
+              profile_result.elapsed_time_in_ms() <
+                  best_result_no_scratch.elapsed_time_in_ms()) {
+            best_result_no_scratch = profile_result;
           }
         }
       }
diff --git a/tensorflow/core/kernels/conv_ops_3d.cc b/tensorflow/core/kernels/conv_ops_3d.cc
index 72758f707a..8a89d564de 100644
--- a/tensorflow/core/kernels/conv_ops_3d.cc
+++ b/tensorflow/core/kernels/conv_ops_3d.cc
@@ -390,38 +390,33 @@ struct LaunchConvOp<GPUDevice, T> {
 
     if (cudnn_use_autotune && !AutoTuneConv3d::GetInstance()->Find(
                                   conv_parameters, &algorithm_config)) {
-      std::vector<AlgorithmDesc::Index> algorithms;
+      std::vector<AlgorithmDesc> algorithms;
       CHECK(stream->parent()->GetConvolveAlgorithms(
           conv_parameters.ShouldIncludeWinogradNonfusedAlgo<T>(), &algorithms));
       ProfileResult best_result;
       ProfileResult best_result_no_scratch;
-      // TODO(benbarsdell): Ideally this should not attempt using tensor op math
-      // if it's not enabled.
-      for (bool use_tensor_ops : {false, true}) {
-        for (auto algo_index : algorithms) {
-          AlgorithmDesc profile_algorithm(algo_index, use_tensor_ops);
-          // TODO(zhengxq): profile each algorithm multiple times to better
-          // accuracy.
-          CudnnScratchAllocator scratch_allocator(ConvolveScratchSize, ctx);
-          ProfileResult profile_result;
-          bool cudnn_launch_status =
-              stream
-                  ->ThenConvolveWithAlgorithm(
-                      input_desc, input_ptr, filter_desc, filter_ptr, conv_desc,
-                      output_desc, &output_ptr, &scratch_allocator,
-                      AlgorithmConfig(profile_algorithm), &profile_result)
-                  .ok();
-          if (cudnn_launch_status) {
-            if (profile_result.is_valid()) {
-              if (profile_result.elapsed_time_in_ms() <
-                  best_result.elapsed_time_in_ms()) {
-                best_result = profile_result;
-              }
-              if (scratch_allocator.TotalByteSize() == 0 &&
-                  profile_result.elapsed_time_in_ms() <
-                      best_result_no_scratch.elapsed_time_in_ms()) {
-                best_result_no_scratch = profile_result;
-              }
+      for (auto profile_algorithm : algorithms) {
+        // TODO(zhengxq): profile each algorithm multiple times to better
+        // accuracy.
+        CudnnScratchAllocator scratch_allocator(ConvolveScratchSize, ctx);
+        ProfileResult profile_result;
+        bool cudnn_launch_status =
+            stream
+                ->ThenConvolveWithAlgorithm(
+                    input_desc, input_ptr, filter_desc, filter_ptr, conv_desc,
+                    output_desc, &output_ptr, &scratch_allocator,
+                    AlgorithmConfig(profile_algorithm), &profile_result)
+                .ok();
+        if (cudnn_launch_status) {
+          if (profile_result.is_valid()) {
+            if (profile_result.elapsed_time_in_ms() <
+                best_result.elapsed_time_in_ms()) {
+              best_result = profile_result;
+            }
+            if (scratch_allocator.TotalByteSize() == 0 &&
+                profile_result.elapsed_time_in_ms() <
+                    best_result_no_scratch.elapsed_time_in_ms()) {
+              best_result_no_scratch = profile_result;
             }
           }
         }
diff --git a/tensorflow/core/kernels/decode_csv_op.cc b/tensorflow/core/kernels/decode_csv_op.cc
index 42ea23553b..5e48ae9766 100644
--- a/tensorflow/core/kernels/decode_csv_op.cc
+++ b/tensorflow/core/kernels/decode_csv_op.cc
@@ -36,8 +36,8 @@ class DecodeCSVOp : public OpKernel {
     OP_REQUIRES_OK(ctx, ctx->GetAttr("use_quote_delim", &use_quote_delim_));
     OP_REQUIRES(ctx, delim.size() == 1,
                 errors::InvalidArgument("field_delim should be only 1 char"));
-
     delim_ = delim[0];
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("na_value", &na_value_));
   }
 
   void Compute(OpKernelContext* ctx) override {
@@ -79,9 +79,9 @@ class DecodeCSVOp : public OpKernel {
         const DataType& dtype = out_type_[f];
         switch (dtype) {
           case DT_INT32: {
-            // If this field is empty, check if default is given:
+            // If this field is empty or NA value, check if default is given:
             // If yes, use default value; Otherwise report error.
-            if (fields[f].empty()) {
+            if (fields[f].empty() || fields[f] == na_value_) {
               OP_REQUIRES(ctx, record_defaults[f].NumElements() == 1,
                           errors::InvalidArgument(
                               "Field ", f,
@@ -99,9 +99,9 @@ class DecodeCSVOp : public OpKernel {
             break;
           }
           case DT_INT64: {
-            // If this field is empty, check if default is given:
+            // If this field is empty or NA value, check if default is given:
             // If yes, use default value; Otherwise report error.
-            if (fields[f].empty()) {
+            if (fields[f].empty() || fields[f] == na_value_) {
               OP_REQUIRES(ctx, record_defaults[f].NumElements() == 1,
                           errors::InvalidArgument(
                               "Field ", f,
@@ -119,9 +119,9 @@ class DecodeCSVOp : public OpKernel {
             break;
           }
           case DT_FLOAT: {
-            // If this field is empty, check if default is given:
+            // If this field is empty or NA value, check if default is given:
             // If yes, use default value; Otherwise report error.
-            if (fields[f].empty()) {
+            if (fields[f].empty() || fields[f] == na_value_) {
               OP_REQUIRES(ctx, record_defaults[f].NumElements() == 1,
                           errors::InvalidArgument(
                               "Field ", f,
@@ -138,9 +138,9 @@ class DecodeCSVOp : public OpKernel {
             break;
           }
           case DT_STRING: {
-            // If this field is empty, check if default is given:
+            // If this field is empty or NA value, check if default is given:
             // If yes, use default value; Otherwise report error.
-            if (fields[f].empty()) {
+            if (fields[f].empty() || fields[f] == na_value_) {
               OP_REQUIRES(ctx, record_defaults[f].NumElements() == 1,
                           errors::InvalidArgument(
                               "Field ", f,
@@ -165,6 +165,7 @@ class DecodeCSVOp : public OpKernel {
   std::vector<DataType> out_type_;
   char delim_;
   bool use_quote_delim_;
+  string na_value_;
 
   void ExtractFields(OpKernelContext* ctx, StringPiece input,
                      std::vector<string>* result) {
diff --git a/tensorflow/core/kernels/dense_to_sparse_batch_dataset_op.cc b/tensorflow/core/kernels/dense_to_sparse_batch_dataset_op.cc
index 25a6813d59..0174c8dfc8 100644
--- a/tensorflow/core/kernels/dense_to_sparse_batch_dataset_op.cc
+++ b/tensorflow/core/kernels/dense_to_sparse_batch_dataset_op.cc
@@ -49,10 +49,10 @@ class DenseToSparseBatchDatasetOp : public UnaryDatasetOpKernel {
     OP_REQUIRES_OK(ctx, ctx->input("row_shape", &row_shape_t));
     OP_REQUIRES(ctx, TensorShapeUtils::IsVector(row_shape_t->shape()),
                 errors::InvalidArgument("row_shape must be a vector"));
-    TensorShape row_shape;
-    for (size_t i = 0; i < row_shape_t->dim_size(0); ++i) {
-      row_shape.AddDim(row_shape_t->vec<int64>()(i));
-    }
+    PartialTensorShape row_shape;
+    OP_REQUIRES_OK(ctx, PartialTensorShape::MakePartialShape(
+                            row_shape_t->vec<int64>().data(),
+                            row_shape_t->NumElements(), &row_shape));
 
     *output = nullptr;
 
@@ -78,7 +78,7 @@ class DenseToSparseBatchDatasetOp : public UnaryDatasetOpKernel {
   template <class T>
   class Dataset : public DatasetBase {
    public:
-    Dataset(int64 batch_size, const TensorShape& row_shape,
+    Dataset(int64 batch_size, const PartialTensorShape& row_shape,
             const DatasetBase* input)
         : batch_size_(batch_size), row_shape_(row_shape), input_(input) {
       input_->Ref();
@@ -129,9 +129,22 @@ class DenseToSparseBatchDatasetOp : public UnaryDatasetOpKernel {
         int64 total_elements = 0;
         batch_elements.reserve(
             DatasetIterator<Dataset<T>>::dataset()->batch_size_);
-        const TensorShape& row_shape =
+        const PartialTensorShape& row_shape =
             DatasetIterator<Dataset<T>>::dataset()->row_shape_;
         const int row_ndims = row_shape.dims();
+
+        // Determine the size of the output tensors:
+        // * dense_shape will be [`row_shape + 1`].
+        Tensor dense_shape(cpu_allocator(), DT_INT64, {row_ndims + 1});
+        auto dense_shape_vec = dense_shape.vec<int64>();
+        for (size_t i = 0; i < row_ndims; ++i) {
+          if (row_shape.dim_size(i) == -1) {
+            dense_shape_vec(i + 1) = 0;
+          } else {
+            dense_shape_vec(i + 1) = row_shape.dim_size(i);
+          }
+        }
+
         {
           mutex_lock l(mu_);
           *end_of_sequence = false;
@@ -156,9 +169,14 @@ class DenseToSparseBatchDatasetOp : public UnaryDatasetOpKernel {
                     ") that is incompatible with the row shape (",
                     row_shape.DebugString(), ").");
               }
-              for (int i = 0; i < row_ndims; ++i) {
-                if (batch_element_tuple[0].shape().dim_size(i) >
-                    row_shape.dim_size(i)) {
+              for (int j = 0; j < row_ndims; ++j) {
+                // Take the maximum in the dimension if -1 is given.
+                if (row_shape.dim_size(j) == -1) {
+                  dense_shape_vec(j + 1) =
+                      std::max(batch_element_tuple[0].dim_size(j),
+                               dense_shape_vec(j + 1));
+                } else if (batch_element_tuple[0].dim_size(j) >
+                           row_shape.dim_size(j)) {
                   return errors::DataLoss(
                       "Input element had shape (",
                       batch_element_tuple[0].shape().DebugString(),
@@ -175,20 +193,16 @@ class DenseToSparseBatchDatasetOp : public UnaryDatasetOpKernel {
           return Status::OK();
         }
 
-        // Determine the size of the output tensors:
         // * indices will be [`total_elements`, `row_shape + 1`].
         // * values will be [`total_elements`].
-        // * dense_shape will be [`row_shape + 1`].
         Tensor indices(cpu_allocator(), DT_INT64,
                        {total_elements, row_ndims + 1});
         Tensor values(
             cpu_allocator(),
             DatasetIterator<Dataset<T>>::dataset()->output_dtypes()[1],
             {total_elements});
-        Tensor dense_shape(cpu_allocator(), DT_INT64, {row_ndims + 1});
         auto indices_matrix = indices.matrix<int64>();
         auto values_flat = values.flat<T>();
-        auto dense_shape_vec = dense_shape.vec<int64>();
 
         int64 current_position_in_values = 0;
         for (int64 i = 0; i < batch_elements.size(); ++i) {
@@ -220,9 +234,6 @@ class DenseToSparseBatchDatasetOp : public UnaryDatasetOpKernel {
         }
 
         dense_shape_vec(0) = batch_elements.size();
-        for (size_t i = 0; i < row_ndims; ++i) {
-          dense_shape_vec(i + 1) = row_shape.dim_size(i);
-        }
 
         out_tensors->push_back(std::move(indices));
         out_tensors->push_back(std::move(values));
@@ -239,7 +250,7 @@ class DenseToSparseBatchDatasetOp : public UnaryDatasetOpKernel {
     };
 
     const int64 batch_size_;
-    const TensorShape row_shape_;
+    const PartialTensorShape row_shape_;
     const DatasetBase* const input_;
     std::vector<PartialTensorShape> output_shapes_;
   };
diff --git a/tensorflow/core/kernels/mkl_conv_grad_filter_ops.cc b/tensorflow/core/kernels/mkl_conv_grad_filter_ops.cc
index f81a448e51..9080bf7be8 100644
--- a/tensorflow/core/kernels/mkl_conv_grad_filter_ops.cc
+++ b/tensorflow/core/kernels/mkl_conv_grad_filter_ops.cc
@@ -27,6 +27,7 @@ limitations under the License.
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/tensor_slice.h"
 #include "tensorflow/core/kernels/conv_grad_ops.h"
+#include "tensorflow/core/kernels/mkl_conv_ops.h"
 #include "tensorflow/core/kernels/ops_util.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/gtl/array_slice.h"
@@ -41,10 +42,24 @@ limitations under the License.
 #include "mkl_dnn.h"
 #include "mkl_dnn_types.h"
 
+#ifdef INTEL_MKL_DNN
+#include "mkldnn.hpp"
+
+using mkldnn::prop_kind;
+using mkldnn::stream;
+
+using mkldnn::convolution_backward_weights;
+using mkldnn::convolution_direct;
+using mkldnn::convolution_forward;
+
+#endif
+
 namespace tensorflow {
 
 typedef Eigen::ThreadPoolDevice CPUDevice;
 
+#ifndef INTEL_MKL_DNN
+
 template <typename Device, class T>
 class MklConv2DCustomBackpropFilterOp : public OpKernel {
  public:
@@ -411,6 +426,172 @@ class MklConv2DCustomBackpropFilterOp : public OpKernel {
   TensorFormat data_format_;
 };
 
+#else
+
+template <typename Device, class T>
+class MklConv2DCustomBackpropFilterOp : public OpKernel {
+ public:
+  explicit MklConv2DCustomBackpropFilterOp(OpKernelConstruction* context)
+      : OpKernel(context) {
+    string data_format;
+    OP_REQUIRES_OK(context, context->GetAttr("data_format", &data_format));
+    OP_REQUIRES(context, FormatFromString(data_format, &data_format_),
+                errors::InvalidArgument("Invalid data format"));
+
+    OP_REQUIRES_OK(context, context->GetAttr("strides", &strides_));
+    int stride_n = GetTensorDim(strides_, data_format_, 'N');
+    int stride_c = GetTensorDim(strides_, data_format_, 'C');
+    OP_REQUIRES(
+        context, (stride_n == 1 && stride_c == 1),
+        errors::InvalidArgument("Current implementation does not yet support "
+                                "strides in the batch and depth dimensions."));
+    OP_REQUIRES_OK(context, context->GetAttr("padding", &padding_));
+  }
+
+  void Compute(OpKernelContext* context) override {
+    try {
+      auto cpu_engine = engine(engine::cpu, 0);
+
+      MklDnnData<T> input(&cpu_engine);
+      MklDnnData<T> outbackprop(&cpu_engine);
+      MklDnnData<T> output(&cpu_engine);
+
+      // Input tensors
+      const Tensor& input_tensor = MklGetInput(context, 0);
+      const Tensor& filter_tensor = MklGetInput(context, 1);
+      const Tensor& obp_tensor = MklGetInput(context, 2);  // Outbackprop
+
+      // Generate input shapes.
+      TensorShape filter_shape;
+      OP_REQUIRES(
+          context, TensorShapeUtils::IsVector(filter_tensor.shape()),
+          errors::InvalidArgument(
+              "Conv2DBackpropFilter: filter_sizes input must be 1-dim, not ",
+              filter_tensor.dims()));
+      OP_REQUIRES_OK(context, TensorShapeUtils::MakeShape(
+                                  filter_tensor.vec<int32>(), &filter_shape));
+      TensorShape input_shape = input_tensor.shape();
+      TensorShape obp_shape = obp_tensor.shape();
+
+      // By default, all dims are in MKL order. Only dims in TF order
+      // are those with prefix tf_order.
+      memory::dims obp_dims, fwd_input_dims, fwd_filter_dims;
+      memory::dims padding_l, padding_r, strides, fwd_output_dims;
+      memory::dims fwd_output_dims_tf_order;
+
+      // Get forward convolution parameters.
+      MklDnnConvUtil conv_utl(context, strides_, padding_, data_format_);
+      conv_utl.GetConvFwdSizesInMklOrder(
+          input_shape, filter_shape, &fwd_input_dims, &fwd_filter_dims,
+          &strides, &fwd_output_dims_tf_order, &fwd_output_dims, &padding_l,
+          &padding_r);
+      if (!context->status().ok()) return;
+
+      // Create Convolution forward descriptor since Convolution backward
+      // API needs it. For that, we first need to create input, filter
+      // and output memory descriptors.
+      auto mkl_data_format = TFDataFormatToMklDnnDataFormat(data_format_);
+      auto fwd_src_md =
+          memory::desc(fwd_input_dims, MklDnnType<T>(), mkl_data_format);
+      auto fwd_filter_md =
+          memory::desc(fwd_filter_dims, MklDnnType<T>(), memory::format::hwio);
+      auto fwd_out_md =
+          memory::desc(fwd_output_dims, MklDnnType<T>(), mkl_data_format);
+      auto fwd_desc = convolution_forward::desc(
+          prop_kind::forward, convolution_direct, fwd_src_md, fwd_filter_md,
+          fwd_out_md, strides, padding_l, padding_r,
+          TFPaddingToMklDnnPadding(padding_));
+      auto fwd_pd = convolution_forward::primitive_desc(fwd_desc, cpu_engine);
+
+      // Allocate output tensor and shape
+      // TODO(nhasabni): Update this when support for MKL layout is added.
+      // Shape of output of Conv2DBackpropInput is same as 'input' of Conv2D.
+      TensorShape tf_output_shape(filter_shape);
+      MklShape mkl_output_mkl_shape;
+      mkl_output_mkl_shape.SetMklTensor(false);
+      Tensor* output_tensor = nullptr;
+      AllocateOutputSetMklShape(context, 0, &output_tensor, tf_output_shape,
+                                mkl_output_mkl_shape);
+
+      // Create memory for user data.
+      // Describe how the inputs and outputs of Convolution look like. Also
+      // specify buffers containing actual input and output data.
+      // Although input shape required is in MKL-DNN order, the layout is
+      // Tensorflow's layout (NHWC or NCHW depending on data format).
+      input.SetUsrMem(fwd_input_dims, mkl_data_format, &input_tensor);
+      // Outbackprop shape is NHWC or NCHW depending on data format. Since
+      // GetInputSizeInMklOrder function returns size in that order we just use
+      // use that function directly.
+      conv_utl.GetInputSizeInMklOrder(obp_shape, &obp_dims);
+      if (!context->status().ok()) return;
+      outbackprop.SetUsrMem(obp_dims, mkl_data_format, &obp_tensor);
+      // Although output shape required is in MKL-DNN order,
+      // layout is Tensorflow's filter layout (HWIO)
+      // Shape of output of Conv2DBackpropInput is same as shape of filter.
+      memory::dims bwd_output_dims = fwd_filter_dims;
+      output.SetUsrMem(bwd_output_dims, memory::format::hwio, output_tensor);
+
+      // Create memory descriptors for convolution data w/ no specified format.
+      input.SetOpMemDesc(fwd_input_dims, memory::format::any);
+      outbackprop.SetOpMemDesc(obp_dims, memory::format::any);
+      output.SetOpMemDesc(bwd_output_dims, memory::format::any);
+
+      // Create convolution backward weights primitive.
+      auto bwd_desc = convolution_backward_weights::desc(
+          convolution_direct, input.GetOpMemDesc(), output.GetOpMemDesc(),
+          outbackprop.GetOpMemDesc(), strides, padding_l, padding_r,
+          TFPaddingToMklDnnPadding(padding_));
+
+      auto bwd_pd = convolution_backward_weights::primitive_desc(
+          bwd_desc, cpu_engine, fwd_pd);
+
+      PrepareAndExecutePrimitive(bwd_pd, &input, &outbackprop, &output);
+    } catch (mkldnn::error& e) {
+      string error_msg = "Status: " + std::to_string(e.status) +
+                         ", message: " + string(e.message) + ", in file " +
+                         string(__FILE__) + ":" + std::to_string(__LINE__);
+      OP_REQUIRES_OK(
+          context,
+          errors::Aborted("Operation received an exception:", error_msg));
+    }
+  }
+
+ private:
+  std::vector<int32> strides_;
+  Padding padding_;
+  TensorFormat data_format_;
+
+  // Prepare and execute net - checks for input and output reorders.
+  void PrepareAndExecutePrimitive(
+      const convolution_backward_weights::primitive_desc& conv_pd,
+      MklDnnData<T>* input, MklDnnData<T>* obp, MklDnnData<T>* output) {
+    // Create reorders between user layout and MKL layout if it is needed and
+    // add it to the net before convolution.
+    std::vector<primitive> net;
+    input->CheckReorderToOpMem(conv_pd.src_primitive_desc(), &net);
+    obp->CheckReorderToOpMem(conv_pd.diff_dst_primitive_desc(), &net);
+
+    // Memory for output of convolution. Since we may need reorder on the
+    // output side, we will prepare reorder primitive in case output
+    // reorder to user memory is required.
+    bool output_reorder_required = output->PrepareReorderToUserMemIfReq(
+        conv_pd.diff_weights_primitive_desc());
+
+    net.push_back(convolution_backward_weights(
+        conv_pd, input->GetOpMem(), obp->GetOpMem(), output->GetOpMem()));
+
+    // Insert reorder primitive in the net for output reorder if reorder is
+    // required.
+    if (output_reorder_required) {
+      output->InsertReorderToUserMem(&net);
+    }
+
+    // Handle output reorder
+    stream(stream::kind::eager).submit(net).wait();
+  }
+};
+#endif
+
 #define REGISTER_MKL_FILTER_KERNELS(T)                              \
   REGISTER_KERNEL_BUILDER(Name("_MklConv2DBackpropFilter")          \
                               .Device(DEVICE_CPU)                   \
diff --git a/tensorflow/core/kernels/mkl_conv_grad_input_ops.cc b/tensorflow/core/kernels/mkl_conv_grad_input_ops.cc
index 00884d0981..4b6bf92e42 100644
--- a/tensorflow/core/kernels/mkl_conv_grad_input_ops.cc
+++ b/tensorflow/core/kernels/mkl_conv_grad_input_ops.cc
@@ -23,6 +23,8 @@ limitations under the License.
 #define EIGEN_USE_THREADS
 #include <algorithm>
 #include <vector>
+#include "mkl_dnn.h"
+#include "mkl_dnn_types.h"
 #include "tensorflow/core/framework/numeric_op.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
@@ -30,6 +32,7 @@ limitations under the License.
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/tensor_slice.h"
 #include "tensorflow/core/kernels/conv_grad_ops.h"
+#include "tensorflow/core/kernels/mkl_conv_ops.h"
 #include "tensorflow/core/kernels/ops_util.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/gtl/array_slice.h"
@@ -40,13 +43,24 @@ limitations under the License.
 #include "tensorflow/core/util/tensor_format.h"
 #include "tensorflow/core/util/use_cudnn.h"
 #include "tensorflow/core/util/work_sharder.h"
-#include "mkl_dnn.h"
-#include "mkl_dnn_types.h"
+
+#ifdef INTEL_MKL_DNN
+#include "mkldnn.hpp"
+
+using mkldnn::prop_kind;
+using mkldnn::stream;
+
+using mkldnn::convolution_backward_data;
+using mkldnn::convolution_direct;
+using mkldnn::convolution_forward;
+#endif
 
 namespace tensorflow {
 
 typedef Eigen::ThreadPoolDevice CPUDevice;
 
+#ifndef INTEL_MKL_DNN
+
 template <typename Device, class T>
 class MklConv2DCustomBackpropInputOp : public OpKernel {
  public:
@@ -345,6 +359,178 @@ class MklConv2DCustomBackpropInputOp : public OpKernel {
   TensorFormat data_format;
 };
 
+#else
+
+template <typename Device, class T>
+class MklConv2DCustomBackpropInputOp : public OpKernel {
+ public:
+  ~MklConv2DCustomBackpropInputOp() {}
+  explicit MklConv2DCustomBackpropInputOp(OpKernelConstruction* context)
+      : OpKernel(context) {
+    string data_format_str;
+    OP_REQUIRES_OK(context, context->GetAttr("data_format", &data_format_str));
+    OP_REQUIRES(context, FormatFromString(data_format_str, &data_format_),
+                errors::InvalidArgument("Invalid data format"));
+    OP_REQUIRES_OK(context, context->GetAttr("strides", &strides_));
+    int stride_n = GetTensorDim(strides_, data_format_, 'N');
+    int stride_c = GetTensorDim(strides_, data_format_, 'C');
+    OP_REQUIRES(
+        context, (stride_n == 1 && stride_c == 1),
+        errors::InvalidArgument("Current implementation does not yet support "
+                                "strides in the batch and depth dimensions."));
+
+    OP_REQUIRES_OK(context, context->GetAttr("padding", &padding_));
+  }
+
+  void Compute(OpKernelContext* context) override {
+    try {
+      auto cpu_engine = engine(engine::cpu, 0);
+
+      MklDnnData<T> filter(&cpu_engine);
+      MklDnnData<T> outbackprop(&cpu_engine);
+      MklDnnData<T> output(&cpu_engine);
+
+      // Input tensors
+      const Tensor& input_tensor = MklGetInput(context, 0);
+      const Tensor& filter_tensor = MklGetInput(context, 1);
+      const Tensor& obp_tensor = MklGetInput(context, 2);  // Outbackprop
+
+      // Generate input shape.
+      TensorShape input_shape;
+      OP_REQUIRES(
+          context, TensorShapeUtils::IsVector(input_tensor.shape()),
+          errors::InvalidArgument(
+              "Conv2DBackpropInput: input_sizes input must be 1-dim, not ",
+              input_tensor.dims()));
+      OP_REQUIRES_OK(context, TensorShapeUtils::MakeShape(
+                                  input_tensor.vec<int32>(), &input_shape));
+      TensorShape filter_shape = filter_tensor.shape();
+      TensorShape obp_shape = obp_tensor.shape();
+
+      // By default, all dims are in MKL order. Only dims in TF order
+      // are those with prefix tf_order.
+      memory::dims obp_dims, fwd_input_dims, fwd_filter_dims;
+      memory::dims padding_l, padding_r, strides, fwd_output_dims;
+      memory::dims fwd_output_dims_tf_order;
+
+      // Get forward convolution parameters.
+      MklDnnConvUtil conv_utl(context, strides_, padding_, data_format_);
+      conv_utl.GetConvFwdSizesInMklOrder(
+          input_shape, filter_shape, &fwd_input_dims, &fwd_filter_dims,
+          &strides, &fwd_output_dims_tf_order, &fwd_output_dims, &padding_l,
+          &padding_r);
+      if (!context->status().ok()) return;
+
+      // Create Convolution forward descriptor since Convolution backward
+      // API needs it. For that, we first need to create input, filter
+      // and output memory descriptors.
+      auto mkl_data_format = TFDataFormatToMklDnnDataFormat(data_format_);
+      auto fwd_src_md =
+          memory::desc(fwd_input_dims, MklDnnType<T>(), mkl_data_format);
+      auto fwd_filter_md =
+          memory::desc(fwd_filter_dims, MklDnnType<T>(), memory::format::hwio);
+      auto fwd_out_md =
+          memory::desc(fwd_output_dims, MklDnnType<T>(), mkl_data_format);
+      auto fwd_desc = convolution_forward::desc(
+          prop_kind::forward, convolution_direct, fwd_src_md, fwd_filter_md,
+          fwd_out_md, strides, padding_l, padding_r,
+          TFPaddingToMklDnnPadding(padding_));
+      auto fwd_pd = convolution_forward::primitive_desc(fwd_desc, cpu_engine);
+
+      // Allocate output tensor and shape
+      // TODO(nhasabni): Update this when support for MKL layout is added.
+      // Shape of output of Conv2DBackpropInput is same as 'input' of Conv2D.
+      TensorShape tf_output_shape(input_shape);
+      MklShape mkl_output_mkl_shape;
+      mkl_output_mkl_shape.SetMklTensor(false);
+      Tensor* output_tensor = nullptr;
+      AllocateOutputSetMklShape(context, 0, &output_tensor, tf_output_shape,
+                                mkl_output_mkl_shape);
+
+      // Create memory for user data.
+      // Describe how the inputs and outputs of Convolution look like. Also
+      // specify buffers containing actual input and output data.
+      // Although input shape required is in MKL-DNN order, the layout is
+      // Tensorflow's layout (NHWC or NCHW depending on data format).
+      // Although filter shape (filter_dims) required is in MKL-DNN order,
+      // the layout is Tensorflow's layout (HWIO).
+      // Shape of Conv2DBackpropInput's filter is same as that of Conv2D filter.
+      filter.SetUsrMem(fwd_filter_dims, memory::format::hwio, &filter_tensor);
+      // Outbackprop shape is NHWC or NCHW depending on data format. Since
+      // GetInputSizeInMklOrder function returns size in that order we just use
+      // use that function directly.
+      conv_utl.GetInputSizeInMklOrder(obp_shape, &obp_dims);
+      if (!context->status().ok()) return;
+      outbackprop.SetUsrMem(obp_dims, mkl_data_format, &obp_tensor);
+      // Although output shape required is in MKL-DNN order,
+      // layout is Tensorflow's layout (NHWC or NCHW depending on data format).
+      // Shape of output of Conv2DBackpropInput is same as shape of 'input'
+      // of Conv2D.
+      memory::dims bwd_output_dims = fwd_input_dims;
+      output.SetUsrMem(bwd_output_dims, mkl_data_format, output_tensor);
+
+      // Create memory descriptors for convolution data w/ no specified format.
+      filter.SetOpMemDesc(fwd_filter_dims, memory::format::any);
+      outbackprop.SetOpMemDesc(obp_dims, memory::format::any);
+      output.SetOpMemDesc(bwd_output_dims, memory::format::any);
+
+      // Create convolution backward data primitive.
+      auto bwd_desc = convolution_backward_data::desc(
+          convolution_direct, output.GetOpMemDesc(), filter.GetOpMemDesc(),
+          outbackprop.GetOpMemDesc(), strides, padding_l, padding_r,
+          TFPaddingToMklDnnPadding(padding_));
+
+      auto bwd_pd = convolution_backward_data::primitive_desc(
+          bwd_desc, cpu_engine, fwd_pd);
+
+      PrepareAndExecutePrimitive(bwd_pd, &filter, &outbackprop, &output);
+    } catch (mkldnn::error& e) {
+      string error_msg = "Status: " + std::to_string(e.status) +
+                         ", message: " + string(e.message) + ", in file " +
+                         string(__FILE__) + ":" + std::to_string(__LINE__);
+      OP_REQUIRES_OK(
+          context,
+          errors::Aborted("Operation received an exception:", error_msg));
+    }
+  }
+
+ private:
+  std::vector<int32> strides_;
+  Padding padding_;
+  TensorFormat data_format_;
+
+  // Prepare and execute net - checks for input and output reorders.
+  void PrepareAndExecutePrimitive(
+      const convolution_backward_data::primitive_desc& conv_pd,
+      MklDnnData<T>* filter, MklDnnData<T>* obp, MklDnnData<T>* output) {
+    // Create reorders between user layout and MKL layout if it is needed and
+    // add it to the net before convolution.
+    std::vector<primitive> net;
+    filter->CheckReorderToOpMem(conv_pd.weights_primitive_desc(), &net);
+    obp->CheckReorderToOpMem(conv_pd.diff_dst_primitive_desc(), &net);
+
+    // Memory for output of convolution. Since we may need reorder on the
+    // output side, we will prepare reorder primitive in case output
+    // reorder to user memory is required.
+    bool output_reorder_required =
+        output->PrepareReorderToUserMemIfReq(conv_pd.diff_src_primitive_desc());
+
+    net.push_back(convolution_backward_data(
+        conv_pd, obp->GetOpMem(), filter->GetOpMem(), output->GetOpMem()));
+
+    // Insert reorder primitive in the net for output reorder if reorder is
+    // required.
+    if (output_reorder_required) {
+      output->InsertReorderToUserMem(&net);
+    }
+
+    // Handle output reorder
+    stream(stream::kind::eager).submit(net).wait();
+  }
+};
+
+#endif  // INTEL_MKL_DNN
+
 #define REGISTER_MKL_CPU_KERNELS(T)                                 \
   REGISTER_KERNEL_BUILDER(Name("_MklConv2DBackpropInput")           \
                               .Device(DEVICE_CPU)                   \
diff --git a/tensorflow/core/kernels/mkl_conv_ops.cc b/tensorflow/core/kernels/mkl_conv_ops.cc
index 7f1555d325..57661e8b10 100644
--- a/tensorflow/core/kernels/mkl_conv_ops.cc
+++ b/tensorflow/core/kernels/mkl_conv_ops.cc
@@ -18,7 +18,9 @@ limitations under the License.
 
 #include <string.h>
 #include <map>
+#include <string>
 #include <vector>
+
 #include "tensorflow/core/framework/numeric_op.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
@@ -26,6 +28,7 @@ limitations under the License.
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/tensor_slice.h"
 #include "tensorflow/core/kernels/bounds_check.h"
+#include "tensorflow/core/kernels/mkl_conv_ops.h"
 #include "tensorflow/core/kernels/ops_util.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/gtl/array_slice.h"
@@ -40,10 +43,23 @@ limitations under the License.
 #include "mkl_dnn.h"
 #include "mkl_dnn_types.h"
 
+#ifdef INTEL_MKL_DNN
+#include "mkldnn.hpp"
+
+using mkldnn::prop_kind;
+using mkldnn::stream;
+
+using mkldnn::convolution_direct;
+using mkldnn::convolution_forward;
+#endif
+
 namespace tensorflow {
 
 typedef Eigen::ThreadPoolDevice CPUDevice;
 
+// For now, MKL-ML is default. So making MKL-DNN not a default choice.
+#ifndef INTEL_MKL_DNN
+
 template <typename Device, typename T, bool biasEnabled>
 class MklConv2DOp : public OpKernel {
  public:
@@ -461,6 +477,203 @@ class MklConv2DOp : public OpKernel {
   TensorFormat data_format_;
 };
 
+#else
+
+template <typename Device, typename T, bool biasEnabled>
+class MklConv2DOp : public OpKernel {
+ public:
+  ~MklConv2DOp() {}
+
+  explicit MklConv2DOp(OpKernelConstruction* context) : OpKernel(context) {
+    OP_REQUIRES_OK(context, context->GetAttr("strides", &strides_));
+    string data_format;
+    OP_REQUIRES_OK(context, context->GetAttr("data_format", &data_format));
+    OP_REQUIRES(context, FormatFromString(data_format, &data_format_),
+                errors::InvalidArgument("Invalid data format"));
+    OP_REQUIRES(context, strides_.size() == 4,
+                errors::InvalidArgument("Sliding window strides field must "
+                                        "specify 4 dimensions"));
+
+    const int64 stride_n = GetTensorDim(strides_, data_format_, 'N');
+    const int64 stride_c = GetTensorDim(strides_, data_format_, 'C');
+    OP_REQUIRES(
+        context, stride_n == 1 && stride_c == 1,
+        errors::InvalidArgument("Current implementation does not yet support "
+                                "strides in the batch and depth dimensions."));
+    OP_REQUIRES_OK(context, context->GetAttr("padding", &padding_));
+  }
+
+  void Compute(OpKernelContext* context) override {
+    try {
+      auto cpu_engine = engine(engine::cpu, 0);
+
+      // Input tensors
+      size_t src_idx = 0, filter_idx = 1;
+      const Tensor& src_tensor = MklGetInput(context, src_idx);
+      const Tensor& filter_tensor = MklGetInput(context, filter_idx);
+
+      MklDnnData<T> src(&cpu_engine);
+      MklDnnData<T> filter(&cpu_engine);
+      MklDnnData<T> output(&cpu_engine);
+
+      memory::dims src_dims, filter_dims, padding_l, padding_r, strides;
+      memory::dims output_dims_tf_order, output_dims_mkl_order;
+
+      // Get shapes of input tensors in MKL-DNN order
+      MklDnnConvUtil conv_utl(context, strides_, padding_, data_format_);
+      conv_utl.GetConvFwdSizesInMklOrder(
+          src_tensor.shape(), filter_tensor.shape(), &src_dims, &filter_dims,
+          &strides, &output_dims_tf_order, &output_dims_mkl_order, &padding_l,
+          &padding_r);
+      if (!context->status().ok()) return;
+
+      // Check for corner case - if there is nothing to compute, return.
+      TensorShape tf_output_shape(
+          {output_dims_tf_order[0], output_dims_tf_order[1],
+           output_dims_tf_order[2], output_dims_tf_order[3]});
+      Tensor* output_tensor = nullptr;
+      MklShape mkl_output_mkl_shape;
+      mkl_output_mkl_shape.SetMklTensor(false);
+      AllocateOutputSetMklShape(context, 0, &output_tensor, tf_output_shape,
+                                mkl_output_mkl_shape);
+
+      // Forward filter in TF format from input at index 1 to output at index 1.
+      ForwardTfTensorInToOut(context, 1, 1);
+
+      if (tf_output_shape.num_elements() == 0) {
+        // TODO(jbobba): Verify correctness here
+        //               Need semantics for Null MKL tensor
+        return;
+      }
+
+      // Corner case to handle 0 batch size.
+      if (output_dims_tf_order[0] == 0) {
+        // Nothing to do, allocate output tensor and return
+        // TODO(nhasabni): remove this code later once serialization
+        // in MKL-DNN is supported.
+        AllocateOutputSetMklShape(context, 0, &output_tensor,
+                                  src_tensor.shape(), mkl_output_mkl_shape);
+        return;
+      } else {
+        // Otherwise regular output tensor allocation
+        // Allocate output tensor.
+      }
+      CHECK_NOTNULL(output_tensor);
+
+      // Create memory for user data.
+      // Describe how the inputs and outputs of Convolution look like. Also
+      // specify buffers containing actual input and output data.
+      // Although input shape (src_dims) required is in MKL-DNN order,
+      // the layout is Tensorflow's layout (NHWC or NCHW depending on data
+      // format).
+      src.SetUsrMem(src_dims, TFDataFormatToMklDnnDataFormat(data_format_),
+                    const_cast<void*>(
+                        static_cast<const void*>(src_tensor.flat<T>().data())));
+      // Although filter shape (filter_dims) required is in MKL-DNN order,
+      // the layout is Tensorflow's layout (HWIO).
+      filter.SetUsrMem(filter_dims, memory::format::hwio,
+                       const_cast<void*>(static_cast<const void*>(
+                           filter_tensor.flat<T>().data())));
+      // Although output shape (output_dims) required is in MKL-DNN order,
+      // layout is Tensorflow's layout (NHWC or NCHW depending on data format).
+      output.SetUsrMem(output_dims_mkl_order,
+                       TFDataFormatToMklDnnDataFormat(data_format_),
+                       output_tensor->flat<T>().data());
+
+      // Create memory descriptors for convolution data w/ no specified format.
+      src.SetOpMemDesc(src_dims, memory::format::any);
+      filter.SetOpMemDesc(filter_dims, memory::format::any);
+      output.SetOpMemDesc(output_dims_mkl_order, memory::format::any);
+
+      // If bias is enabled, then do the same steps as above for bias.
+      if (biasEnabled) {
+        MklDnnData<T> bias(&cpu_engine);
+        memory::dims bias_size;
+        conv_utl.GetBiasSizeInMklOrder(2 /* bias idx */, &bias_size);
+        const Tensor& bias_tensor = MklGetInput(context, 2);
+        bias.SetUsrMem(bias_size, memory::format::x,
+                       const_cast<void*>(static_cast<const void*>(
+                           bias_tensor.flat<T>().data())));
+        bias.SetOpMemDesc(bias_size, memory::format::any);
+
+        // Create convolution primitive with Bias.
+        auto conv_desc = convolution_forward::desc(
+            prop_kind::forward, convolution_direct, src.GetOpMemDesc(),
+            filter.GetOpMemDesc(), bias.GetOpMemDesc(), output.GetOpMemDesc(),
+            strides, padding_l, padding_r, TFPaddingToMklDnnPadding(padding_));
+
+        auto conv_prim_desc =
+            convolution_forward::primitive_desc(conv_desc, cpu_engine);
+        PrepareAndExecuteNet(conv_prim_desc, &src, &filter, &bias, &output);
+      } else {
+        // Create convolution primitive without Bias.
+        auto conv_desc = convolution_forward::desc(
+            prop_kind::forward, convolution_direct, src.GetOpMemDesc(),
+            filter.GetOpMemDesc(), output.GetOpMemDesc(), strides, padding_l,
+            padding_r, TFPaddingToMklDnnPadding(padding_));
+
+        auto conv_prim_desc =
+            convolution_forward::primitive_desc(conv_desc, cpu_engine);
+        PrepareAndExecuteNet(conv_prim_desc, &src, &filter, nullptr, &output);
+      }
+    } catch (mkldnn::error& e) {
+      string error_msg = "Status: " + std::to_string(e.status) +
+                         ", message: " + std::string(e.message) + ", in file " +
+                         std::string(__FILE__) + ":" + std::to_string(__LINE__);
+      OP_REQUIRES_OK(
+          context,
+          errors::Aborted("Operation received an exception:", error_msg));
+    }
+  }
+
+ private:
+  std::vector<int32> strides_;
+  Padding padding_;
+  TensorFormat data_format_;
+
+  // Prepare and execute net - checks for input and output reorders.
+  void PrepareAndExecuteNet(
+      const convolution_forward::primitive_desc& conv_prim_desc,
+      MklDnnData<T>* src, MklDnnData<T>* filter, MklDnnData<T>* bias,
+      MklDnnData<T>* output) {
+    // Create reorders between user layout and MKL layout if it is needed and
+    // add it to the net before convolution.
+    std::vector<primitive> net;
+    src->CheckReorderToOpMem(conv_prim_desc.src_primitive_desc(), &net);
+    filter->CheckReorderToOpMem(conv_prim_desc.weights_primitive_desc(), &net);
+
+    // Memory for output of convolution. Since we may need reorder on the
+    // output side, we will prepare reorder primitive in case output
+    // reorder to user memory is required.
+    bool output_reorder_required = output->PrepareReorderToUserMemIfReq(
+        conv_prim_desc.dst_primitive_desc());
+
+    // Create convolution primitive and add it to net.
+    if (bias) {
+      CHECK_EQ(biasEnabled, true);
+      net.push_back(convolution_forward(conv_prim_desc, src->GetOpMem(),
+                                        filter->GetOpMem(), bias->GetOpMem(),
+                                        output->GetOpMem()));
+    } else {
+      CHECK_EQ(biasEnabled, false);
+      net.push_back(convolution_forward(conv_prim_desc, src->GetOpMem(),
+                                        filter->GetOpMem(),
+                                        output->GetOpMem()));
+    }
+
+    // Insert reorder primitive in the net for output reorder if reorder is
+    // required.
+    if (output_reorder_required) {
+      output->InsertReorderToUserMem(&net);
+    }
+
+    // Handle output reorder
+    stream(stream::kind::eager).submit(net).wait();
+  }
+};
+
+#endif
+
 #define REGISTER_MKL_CPU(T)                                         \
   REGISTER_KERNEL_BUILDER(Name("_MklConv2D")                        \
                               .Device(DEVICE_CPU)                   \
diff --git a/tensorflow/core/kernels/mkl_conv_ops.h b/tensorflow/core/kernels/mkl_conv_ops.h
new file mode 100644
index 0000000000..e29af19ca9
--- /dev/null
+++ b/tensorflow/core/kernels/mkl_conv_ops.h
@@ -0,0 +1,308 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_CORE_KERNELS_MKL_CONV_OPS_H_
+#define TENSORFLOW_CORE_KERNELS_MKL_CONV_OPS_H_
+
+#include <limits>
+#include <vector>
+
+#include "tensorflow/core/framework/numeric_op.h"
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/framework/register_types.h"
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/framework/tensor_shape.h"
+#include "tensorflow/core/framework/tensor_slice.h"
+#include "tensorflow/core/kernels/bounds_check.h"
+#include "tensorflow/core/kernels/conv_grad_ops.h"
+#include "tensorflow/core/kernels/ops_util.h"
+#include "tensorflow/core/lib/core/errors.h"
+#include "tensorflow/core/lib/gtl/array_slice.h"
+#include "tensorflow/core/lib/strings/numbers.h"
+#include "tensorflow/core/lib/strings/str_util.h"
+#include "tensorflow/core/platform/logging.h"
+#include "tensorflow/core/platform/macros.h"
+#include "tensorflow/core/util/padding.h"
+#include "tensorflow/core/util/tensor_format.h"
+
+#include "tensorflow/core/util/mkl_util.h"
+
+#ifdef INTEL_MKL_DNN
+#include "mkldnn.hpp"
+#endif
+
+namespace tensorflow {
+
+#ifdef INTEL_MKL_DNN
+
+class MklDnnConvUtil {
+ protected:
+  OpKernelContext *context_;  // We don't own this.
+  std::vector<int32> strides_;
+  Padding padding_;
+  TensorFormat data_format_;
+
+ public:
+  MklDnnConvUtil(OpKernelContext *context, const std::vector<int32> &strides,
+                 Padding pad, TensorFormat fm)
+      : context_(context), strides_(strides), padding_(pad), data_format_(fm) {}
+
+  virtual ~MklDnnConvUtil() { context_ = nullptr; }
+
+  // Calculate Convolution strides
+  virtual inline void GetStridesInMklOrder(memory::dims *strides) {
+    // For now we take the stride from the second and third dimensions only
+    // (we do not support striding on the batch or depth dimension).
+    CHECK_NOTNULL(strides);
+    int stride_rows = GetTensorDim(strides_, data_format_, 'H');
+    int stride_cols = GetTensorDim(strides_, data_format_, 'W');
+    *strides = {stride_rows, stride_cols};
+  }
+
+  // Calculate Convolution input size in MKL-DNN order. MKL-DNN
+  // requires input in NCHW format. Function does not return anything.
+  // But errors arising from sanity checks are returned in context's
+  // status.
+  virtual inline void GetInputSizeInMklOrder(const TensorShape &input_shape,
+                                             memory::dims *input_dims) {
+#define CHECK_BOUNDS(val, err_msg)                                     \
+  do {                                                                 \
+    OP_REQUIRES(context_,                                              \
+                FastBoundsCheck(val, std::numeric_limits<int>::max()), \
+                errors::InvalidArgument(err_msg));                     \
+  } while (0)
+
+    CHECK_NOTNULL(input_dims);
+
+    // Input channel
+    int64 input_depth_raw = GetTensorDim(input_shape, data_format_, 'C');
+    int input_depth = static_cast<int>(input_depth_raw);
+
+    // Input rows/height
+    int64 input_rows_raw = GetTensorDim(input_shape, data_format_, 'H');
+    CHECK_BOUNDS(input_rows_raw, "Input rows too large");
+    int input_rows = static_cast<int>(input_rows_raw);
+
+    // Input columns/width
+    int64 input_cols_raw = GetTensorDim(input_shape, data_format_, 'W');
+    CHECK_BOUNDS(input_cols_raw, "Input cols too large");
+    int input_cols = static_cast<int>(input_cols_raw);
+
+    // Input batch
+    int64 input_batch_raw = GetTensorDim(input_shape, data_format_, 'N');
+    CHECK_BOUNDS(input_batch_raw, "Input batch too large");
+    int input_batch = static_cast<int>(input_batch_raw);
+
+#undef CHECK_BOUNDS
+
+    // MKL-DNN always requires input in NCHW format.
+    *input_dims = {input_batch, input_depth, input_rows, input_cols};
+  }
+
+  // Calculate Convolution filter size in MKL-DNN order. MKL-DNN
+  // requires filter in OIHW format. Function does not return anything.
+  // But errors arising from sanity checks are returned in context's
+  // status.
+  //
+  // Calculate Convolution filter size in MKL-DNN order. MKL-DNN
+  // requires filter in OIHW format. Function does not return anything.
+  // But errors arising from sanity checks are returned in context's
+  // status. This function differs from GetConvFilterSizeInMklOrder in
+  // parameter for input - it accepts src_shape since Convolution Backward
+  // Input gets shape of input tensor rather than actual tensor (Convolution
+  // forward gets actual tensor as input).
+  //
+  // TODO(nhasabni): Add similar function for input and filter in MklShape.
+  virtual inline void GetFilterSizeInMklOrder(const TensorShape &input_shape,
+                                              const TensorShape &filter_shape,
+                                              memory::dims *filter_dims) {
+    CHECK_NOTNULL(filter_dims);
+
+    OP_REQUIRES(context_, filter_shape.dims() == 4,
+                errors::InvalidArgument("filter must be 4-dimensional: ",
+                                        filter_shape.DebugString()));
+
+    for (int i = 0; i < 3; i++) {
+      OP_REQUIRES(context_,
+                  FastBoundsCheck(filter_shape.dim_size(i),
+                                  std::numeric_limits<int>::max()),
+                  errors::InvalidArgument("filter too large"));
+    }
+
+    int input_depth = GetTensorDim(input_shape, data_format_, 'C');
+
+    OP_REQUIRES(context_, input_depth == filter_shape.dim_size(2),
+                errors::InvalidArgument(
+                    "input and filter must have the same depth: ", input_depth,
+                    " vs ", filter_shape.dim_size(2)));
+
+    // TF filter is always in (rows, cols, in_depth, out_depth) order.
+    int filter_rows = static_cast<int>(filter_shape.dim_size(0));
+    int filter_cols = static_cast<int>(filter_shape.dim_size(1));
+    int in_depth = static_cast<int>(filter_shape.dim_size(2));
+    int out_depth = static_cast<int>(filter_shape.dim_size(3));
+
+    // MKL-DNN always needs filter in OIHW format.
+    // OIHW = (out_depth, in_depth, rows, cols)
+    *filter_dims = {out_depth, in_depth, filter_rows, filter_cols};
+  }
+
+  // Calculate Convolution filter size in MKL-DNN order. MKL-DNN
+  // requires filter in OIHW format. Function does not return anything.
+  // But errors arising from sanity checks are returned in context's
+  // status.
+  virtual inline void GetFilterSizeInMklOrder(size_t src_index,
+                                              size_t filter_index,
+                                              memory::dims *filter_dims) {
+    CHECK_NOTNULL(filter_dims);
+    const Tensor &input = MklGetInput(context_, src_index);
+    const Tensor &filter = MklGetInput(context_, filter_index);
+    GetFilterSizeInMklOrder(input.shape(), filter.shape(), filter_dims);
+  }
+
+  // Calculate Bias size for 2D Convolution. Function does not return
+  // anything, but sets error in context status.
+  virtual inline void GetBiasSizeInMklOrder(size_t bias_index,
+                                            memory::dims *bias_dims) {
+    const Tensor &bias = MklGetInput(context_, bias_index);
+    OP_REQUIRES(context_, bias.dims() == 1,
+                errors::InvalidArgument("bias must be 1-dimensional: ",
+                                        bias.shape().DebugString()));
+
+    *bias_dims = {static_cast<int>(bias.dim_size(0))};
+  }
+
+  // Function to calculate output and padding size for 2D convolution.
+  //
+  // Calculate output shape of Convolution in MKL-DNN and TensorFlow order.
+  // MKL-DNN uses NCHW for output order. But TensorFlow output will be in
+  // NHWC or NCHW format depending on data format. Function also calculates
+  // left, right, top and bottom pads. Function does not return any status -
+  // status is returned via context status.
+  //
+  // TODO(nhasabni): Add similar function for input and filter in MklShape.
+  virtual inline void GetOutputAndPadSizeInMklOrder(
+      const TensorShape &input_shape, const TensorShape &filter_shape,
+      const memory::dims &strides, memory::dims *output_dims_tf_order,
+      memory::dims *output_dims_mkl_order, memory::dims *pad_l,
+      memory::dims *pad_r) {
+    CHECK_NOTNULL(output_dims_tf_order);
+    CHECK_NOTNULL(output_dims_mkl_order);
+    CHECK_NOTNULL(pad_l);
+    CHECK_NOTNULL(pad_r);
+
+    int input_rows = GetTensorDim(input_shape, data_format_, 'H');
+    int input_cols = GetTensorDim(input_shape, data_format_, 'W');
+
+    // The first dimension for filter is rows/height.
+    int filter_rows = filter_shape.dim_size(0);
+    // The second dimension for filter is cols/width.
+    int filter_cols = filter_shape.dim_size(1);
+
+    // Stride is vector of 2 elements: {s_r, s_c}
+    int stride_rows = strides[0];
+    int stride_cols = strides[1];
+
+    // Output batch is same as input batch.
+    int out_batch = GetTensorDim(input_shape, data_format_, 'N');
+    // Output depth is same as last dimension for filter.
+    int out_depth = filter_shape.dim_size(3);
+
+    int64 out_rows = 0, out_cols = 0;
+    int64 pad_top = 0, pad_bottom = 0, pad_left, pad_right;
+
+    OP_REQUIRES_OK(context_, GetWindowedOutputSizeVerbose(
+                                 input_rows, filter_rows, stride_rows, padding_,
+                                 &out_rows, &pad_top, &pad_bottom));
+    OP_REQUIRES_OK(context_, GetWindowedOutputSizeVerbose(
+                                 input_cols, filter_cols, stride_cols, padding_,
+                                 &out_cols, &pad_left, &pad_right));
+
+    // Tensorflow output is in data_format order. (NHWC or NCHW)
+    TensorShape out_shape =
+        ShapeFromFormat(data_format_, out_batch, out_rows, out_cols, out_depth);
+    *output_dims_tf_order = TFShapeToMklDnnDims(out_shape);
+
+    // MKL-DNN always needs output in NCHW format.
+    *output_dims_mkl_order = {out_batch, out_depth, static_cast<int>(out_rows),
+                              static_cast<int>(out_cols)};
+
+    // Now handle padding. MKL-DNN uses asymetric padding.
+    *pad_l = {static_cast<int>(pad_top), static_cast<int>(pad_left)};
+    *pad_r = {static_cast<int>(pad_bottom), static_cast<int>(pad_right)};
+  }
+
+  // Calculate output and pad size of forward Convolution operator.
+  // See comment on GetConvOutputAndPadSizeInMklOrder for parameters.
+  //
+  // Function does not return anything, but sets error in context status.
+  inline void GetOutputAndPadSizeInMklOrder(
+      size_t src_index, size_t filter_index, const memory::dims &strides,
+      memory::dims *output_dims_tf_order, memory::dims *output_dims_mkl_order,
+      memory::dims *pad_l, memory::dims *pad_r) {
+    CHECK_NOTNULL(output_dims_tf_order);
+    CHECK_NOTNULL(output_dims_mkl_order);
+    CHECK_NOTNULL(pad_l);
+    CHECK_NOTNULL(pad_r);
+
+    const Tensor &input = MklGetInput(context_, src_index);
+    const Tensor &filter = MklGetInput(context_, filter_index);
+
+    OP_REQUIRES(context_, input.dims() == 4,
+                errors::InvalidArgument("input must be 4-dimensional",
+                                        input.shape().DebugString()));
+
+    GetOutputAndPadSizeInMklOrder(input.shape(), filter.shape(), strides,
+                                  output_dims_tf_order, output_dims_mkl_order,
+                                  pad_l, pad_r);
+  }
+
+  // Wrapper function to calculate input, filter, and output sizes of
+  // 2D Convolution in MKL order (NCHW for input and output; OIHW for filter.)
+  // Function also calculates output shape in Tensorflow order. Additionally, it
+  // also calculates strides and paddings for 2D Convolution.
+  //
+  // Function does not return anything, but sets error in context status.
+  inline void GetConvFwdSizesInMklOrder(
+      const TensorShape &input_shape, const TensorShape &filter_shape,
+      memory::dims *input_dims, memory::dims *filter_dims,
+      memory::dims *strides, memory::dims *output_dims_tf_order,
+      memory::dims *output_dims_mkl_order, memory::dims *pad_l,
+      memory::dims *pad_r) {
+    CHECK_NOTNULL(input_dims);
+    CHECK_NOTNULL(filter_dims);
+    CHECK_NOTNULL(strides);
+    CHECK_NOTNULL(output_dims_tf_order);
+    CHECK_NOTNULL(output_dims_mkl_order);
+    CHECK_NOTNULL(pad_l);
+    CHECK_NOTNULL(pad_r);
+
+    GetInputSizeInMklOrder(input_shape, input_dims);
+    if (!context_->status().ok()) return;
+    GetFilterSizeInMklOrder(input_shape, filter_shape, filter_dims);
+    if (!context_->status().ok()) return;
+    GetStridesInMklOrder(strides);
+    GetOutputAndPadSizeInMklOrder(input_shape, filter_shape, *strides,
+                                  output_dims_tf_order, output_dims_mkl_order,
+                                  pad_l, pad_r);
+    if (!context_->status().ok()) return;
+  }
+};
+
+#endif  // INTEL_MKL_DNN
+
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_CORE_KERNELS_MKL_CONV_OPS_H_
diff --git a/tensorflow/core/kernels/mkl_cwise_ops_common.cc b/tensorflow/core/kernels/mkl_cwise_ops_common.cc
index 7fc633c254..c065724e0d 100644
--- a/tensorflow/core/kernels/mkl_cwise_ops_common.cc
+++ b/tensorflow/core/kernels/mkl_cwise_ops_common.cc
@@ -48,7 +48,7 @@ class MklBinaryOp : public BinaryOp<Device, Functor> {
     auto out = context->mutable_output(0);
     VLOG(1) << "Shapes (output): " << out->shape().DebugString();
 
-    // Pass input shape through to ouput shape
+    // Pass input shape through to output shape
     ForwardMklMetaDataInToOut(context, 0, 0);
 
     out = context->mutable_output(0);
diff --git a/tensorflow/core/lib/strings/numbers.cc b/tensorflow/core/lib/strings/numbers.cc
index 3c85737702..302a6967e3 100644
--- a/tensorflow/core/lib/strings/numbers.cc
+++ b/tensorflow/core/lib/strings/numbers.cc
@@ -340,7 +340,7 @@ char* FloatToBuffer(float value, char* buffer) {
   float parsed_value;
   if (!safe_strtof(buffer, &parsed_value) || parsed_value != value) {
     snprintf_result =
-        snprintf(buffer, kFastToBufferSize, "%.*g", FLT_DIG + 2, value);
+        snprintf(buffer, kFastToBufferSize, "%.*g", FLT_DIG + 3, value);
 
     // Should never overflow; see above.
     DCHECK(snprintf_result > 0 && snprintf_result < kFastToBufferSize);
diff --git a/tensorflow/core/ops/dataset_ops.cc b/tensorflow/core/ops/dataset_ops.cc
index df189af1b8..c0e84c8bb0 100644
--- a/tensorflow/core/ops/dataset_ops.cc
+++ b/tensorflow/core/ops/dataset_ops.cc
@@ -383,7 +383,8 @@ input_dataset: A handle to an input dataset. Must have a single component.
 batch_size: A scalar representing the number of elements to accumulate in a
   batch.
 row_shape: A vector representing the dense shape of each row in the produced
-  SparseTensor.
+  SparseTensor. The shape may be partially specified, using `-1` to indicate
+  that a particular dimension should use the maximum size of all batch elements.
 )doc");
 
 REGISTER_OP("RangeDataset")
diff --git a/tensorflow/core/ops/nn_ops.cc b/tensorflow/core/ops/nn_ops.cc
index 3dc16ac457..b34dc1a008 100644
--- a/tensorflow/core/ops/nn_ops.cc
+++ b/tensorflow/core/ops/nn_ops.cc
@@ -29,22 +29,6 @@ using shape_inference::ShapeHandle;
 
 namespace {
 
-// A shape function that uses the tensor value at <input_idx> as a shape for
-// output 0. If the tensor value is not available, it uses a shape with <ndims>
-// unknown dims.
-Status InputTensorShapeOrUnknown(InferenceContext* c, int input_idx,
-                                 int ndims) {
-  ShapeHandle out;
-  const Tensor* input = c->input_tensor(input_idx);
-  if (input == nullptr) {
-    out = c->UnknownShapeOfRank(ndims);
-  } else {
-    TF_RETURN_IF_ERROR(c->MakeShapeFromShapeTensor(input_idx, &out));
-  }
-  c->set_output(0, out);
-  return Status::OK();
-}
-
 Status FractionalPoolShapeFn(InferenceContext* c) {
   ShapeHandle input;
   TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 4, &input));
@@ -119,11 +103,11 @@ REGISTER_OP("AvgPoolGrad")
     .Attr(GetConvnetDataFormatAttrString())
     .Attr("T: {half, float, double}")
     .SetShapeFn([](InferenceContext* c) {
-      // NOTE(mrry): We could in principle work out the shape from the
-      // gradients and the attrs, but if we do not know orig_input_shape
-      // statically, then we are unlikely to know the shape of the
-      // gradients either.
-      return InputTensorShapeOrUnknown(c, 0 /* input_idx */, 4 /* ndims */);
+      ShapeHandle s;
+      TF_RETURN_IF_ERROR(c->MakeShapeFromShapeTensor(0, &s));
+      TF_RETURN_IF_ERROR(c->WithRank(s, 4, &s));
+      c->set_output(0, s);
+      return Status::OK();
     })
     .Doc(R"doc(
 Computes gradients of the average pooling function.
@@ -583,11 +567,11 @@ REGISTER_OP("Conv2DBackpropInput")
     .Attr(GetPaddingAttrString())
     .Attr(GetConvnetDataFormatAttrString())
     .SetShapeFn([](InferenceContext* c) {
-      // NOTE(mrry): We could in principle work out the shape from the
-      // gradients and the attrs, but if we do not know orig_input_shape
-      // statically, then we are unlikely to know the shape of the
-      // gradients either.
-      return InputTensorShapeOrUnknown(c, 0 /* input_idx */, 4 /* ndims */);
+      ShapeHandle s;
+      TF_RETURN_IF_ERROR(c->MakeShapeFromShapeTensor(0, &s));
+      TF_RETURN_IF_ERROR(c->WithRank(s, 4, &s));
+      c->set_output(0, s);
+      return Status::OK();
     })
     .Doc(R"doc(
 Computes the gradients of convolution with respect to the input.
@@ -625,11 +609,11 @@ REGISTER_OP("Conv2DBackpropFilter")
     .Attr(GetPaddingAttrString())
     .Attr(GetConvnetDataFormatAttrString())
     .SetShapeFn([](InferenceContext* c) {
-      // NOTE(mrry): We could in principle work out the shape from the
-      // gradients and the attrs, but if we do not know orig_input_shape
-      // statically, then we are unlikely to know the shape of the
-      // gradients either.
-      return InputTensorShapeOrUnknown(c, 1 /* input_idx */, 4 /* ndims */);
+      ShapeHandle s;
+      TF_RETURN_IF_ERROR(c->MakeShapeFromShapeTensor(1, &s));
+      TF_RETURN_IF_ERROR(c->WithRank(s, 4, &s));
+      c->set_output(0, s);
+      return Status::OK();
     })
     .Doc(R"doc(
 Computes the gradients of convolution with respect to the filter.
@@ -882,11 +866,11 @@ REGISTER_OP("DepthwiseConv2dNativeBackpropInput")
     .Attr(GetPaddingAttrString())
     .Attr(GetConvnetDataFormatAttrString())
     .SetShapeFn([](InferenceContext* c) {
-      // NOTE(mrry): We could in principle work out the shape from the
-      // gradients and the attrs, but if we do not know orig_input_shape
-      // statically, then we are unlikely to know the shape of the
-      // gradients either.
-      return InputTensorShapeOrUnknown(c, 0 /* input_idx */, 4 /* ndims */);
+      ShapeHandle s;
+      TF_RETURN_IF_ERROR(c->MakeShapeFromShapeTensor(0, &s));
+      TF_RETURN_IF_ERROR(c->WithRank(s, 4, &s));
+      c->set_output(0, s);
+      return Status::OK();
     })
     .Doc(R"doc(
 Computes the gradients of depthwise convolution with respect to the input.
@@ -924,11 +908,11 @@ REGISTER_OP("DepthwiseConv2dNativeBackpropFilter")
     .Attr(GetPaddingAttrString())
     .Attr(GetConvnetDataFormatAttrString())
     .SetShapeFn([](InferenceContext* c) {
-      // NOTE(mrry): We could in principle work out the shape from the
-      // gradients and the attrs, but if we do not know orig_input_shape
-      // statically, then we are unlikely to know the shape of the
-      // gradients either.
-      return InputTensorShapeOrUnknown(c, 1 /* input_idx */, 4 /* ndims */);
+      ShapeHandle s;
+      TF_RETURN_IF_ERROR(c->MakeShapeFromShapeTensor(1, &s));
+      TF_RETURN_IF_ERROR(c->WithRank(s, 4, &s));
+      c->set_output(0, s);
+      return Status::OK();
     })
     .Doc(R"doc(
 Computes the gradients of depthwise convolution with respect to the filter.
@@ -2870,7 +2854,11 @@ REGISTER_OP("_MklConv2DBackpropFilter")
     .Attr(GetPaddingAttrString())
     .Attr(GetConvnetDataFormatAttrString())
     .SetShapeFn([](InferenceContext* c) {
-      return InputTensorShapeOrUnknown(c, 1 /* input_idx */, 4 /* ndims */);
+      ShapeHandle s;
+      TF_RETURN_IF_ERROR(c->MakeShapeFromShapeTensor(1, &s));
+      TF_RETURN_IF_ERROR(c->WithRank(s, 4, &s));
+      c->set_output(0, s);
+      return Status::OK();
     })
     .Doc(R"doc(
 MKL version of Conv2DBackpropFilter. Uses MKL DNN APIs to compute the
@@ -2911,7 +2899,11 @@ REGISTER_OP("_MklConv2DBackpropInput")
     .Attr(GetPaddingAttrString())
     .Attr(GetConvnetDataFormatAttrString())
     .SetShapeFn([](InferenceContext* c) {
-      return InputTensorShapeOrUnknown(c, 0 /* input_idx */, 4 /* ndims */);
+      ShapeHandle s;
+      TF_RETURN_IF_ERROR(c->MakeShapeFromShapeTensor(0, &s));
+      TF_RETURN_IF_ERROR(c->WithRank(s, 4, &s));
+      c->set_output(0, s);
+      return Status::OK();
     })
     .Doc(R"doc(
 MKL version of Convolution2D backward input. Uses MKL DNN APIs to compute the
@@ -3034,7 +3026,11 @@ REGISTER_OP("_MklAvgPoolGrad")
     .Attr(GetConvnetDataFormatAttrString())
     .Attr("T: {float, half, double}")
     .SetShapeFn([](InferenceContext* c) {
-      return InputTensorShapeOrUnknown(c, 0 /* input_idx */, 4 /* ndims */);
+      ShapeHandle s;
+      TF_RETURN_IF_ERROR(c->MakeShapeFromShapeTensor(0, &s));
+      TF_RETURN_IF_ERROR(c->WithRank(s, 4, &s));
+      c->set_output(0, s);
+      return Status::OK();
     })
     .Doc(R"doc(
 MKL version of AvgPoolGrad operator. Uses MKL DNN APIs to compute gradients
diff --git a/tensorflow/core/ops/nn_ops_test.cc b/tensorflow/core/ops/nn_ops_test.cc
index 51e4f8bffe..4628b725f8 100644
--- a/tensorflow/core/ops/nn_ops_test.cc
+++ b/tensorflow/core/ops/nn_ops_test.cc
@@ -81,55 +81,6 @@ TEST(NNOpsTest, TopKV2_ShapeFn) {
       op, "[1,2,3,4];[]");
 }
 
-TEST(NNOpsTest, InputTensorShapeOrUnknown2D_ShapeFn) {
-  typedef std::pair<const char*, int> NameAndInputIndex;
-  for (const auto& p :
-       {NameAndInputIndex("AvgPoolGrad", 0),
-        NameAndInputIndex("Conv2DBackpropInput", 0),
-        NameAndInputIndex("Conv2DBackpropFilter", 1),
-        NameAndInputIndex("DepthwiseConv2dNativeBackpropInput", 0),
-        NameAndInputIndex("DepthwiseConv2dNativeBackpropFilter", 1)}) {
-    ShapeInferenceTestOp op(p.first);
-    op.input_tensors.resize(2);
-
-    // Conv and Depthwise conv have three inputs.
-    string extra_shapes = (op.name == "AvgPoolGrad" ? "" : ";?");
-
-    // When the input tensor is not known, the output is 4 unknown dims.
-    INFER_OK(op, "?;?" + extra_shapes, "[?,?,?,?]");
-    INFER_OK(op, "[4];?" + extra_shapes, "[?,?,?,?]");
-
-    // When input tensor is known, its values determine output shape.
-    std::vector<int32> shape{1, 2, 3, 4};
-    Tensor shape_t = test::AsTensor<int32>(shape);
-    op.input_tensors[p.second] = &shape_t;
-    INFER_OK(op, "[4];?" + extra_shapes, "[1,2,3,4]");
-  }
-}
-
-TEST(NNOpsTest, InputTensorShapeOrUnknown3D_ShapeFn) {
-  typedef std::pair<const char*, int> NameAndInputIndex;
-  for (const auto& p : {NameAndInputIndex("AvgPool3DGrad", 0),
-                        NameAndInputIndex("Conv3DBackpropInputV2", 0),
-                        NameAndInputIndex("Conv3DBackpropFilterV2", 1)}) {
-    ShapeInferenceTestOp op(p.first);
-    op.input_tensors.resize(2);
-
-    // Conv3D has an extra shape.
-    string extra_shapes = (op.name == "AvgPool3DGrad" ? "" : ";?");
-
-    // When the input tensor is not known, the output is 4 unknown dims.
-    INFER_OK(op, "?;?" + extra_shapes, "[?,?,?,?,?]");
-    INFER_OK(op, "[5];?" + extra_shapes, "[?,?,?,?,?]");
-
-    // When input tensor is known, its values determine output shape.
-    std::vector<int32> shape{1, 2, 3, 4, 5};
-    Tensor shape_t = test::AsTensor<int32>(shape);
-    op.input_tensors[p.second] = &shape_t;
-    INFER_OK(op, "[5];?" + extra_shapes, "[1,2,3,4,5]");
-  }
-}
-
 TEST(NNOpsTest, BatchNormWithGlobalNormalization_ShapeFn) {
   ShapeInferenceTestOp op("BatchNormWithGlobalNormalization");
 
diff --git a/tensorflow/core/ops/parsing_ops.cc b/tensorflow/core/ops/parsing_ops.cc
index f23ff083af..b44ea2e080 100644
--- a/tensorflow/core/ops/parsing_ops.cc
+++ b/tensorflow/core/ops/parsing_ops.cc
@@ -332,6 +332,7 @@ REGISTER_OP("DecodeCSV")
     .Attr("OUT_TYPE: list({float,int32,int64,string})")
     .Attr("field_delim: string = ','")
     .Attr("use_quote_delim: bool = true")
+    .Attr("na_value: string = ''")
     .SetShapeFn([](InferenceContext* c) {
       // Validate the record_defaults inputs.
       for (int i = 1; i < c->num_inputs(); ++i) {
@@ -362,6 +363,7 @@ field_delim: char delimiter to separate fields in a record.
 use_quote_delim: If false, treats double quotation marks as regular
   characters inside of the string fields (ignoring RFC 4180, Section 2,
   Bullet 5).
+na_value: Additional string to recognize as NA/NaN.
 output: Each tensor will have the same shape as records.
 )doc");
 
diff --git a/tensorflow/core/util/mkl_util.h b/tensorflow/core/util/mkl_util.h
index f4bec9524a..1bfa4f83a3 100644
--- a/tensorflow/core/util/mkl_util.h
+++ b/tensorflow/core/util/mkl_util.h
@@ -26,13 +26,19 @@ limitations under the License.
 #include "mkl_trans.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
-#include "tensorflow/core/util/tensor_format.h"
 
 #include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/graph/mkl_graph_util.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/gtl/array_slice.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/macros.h"
+#include "tensorflow/core/util/padding.h"
+#include "tensorflow/core/util/tensor_format.h"
+
+#ifdef INTEL_MKL_DNN
+#include "mkldnn.hpp"
+#endif
 
 // The file contains a number of utility classes and functions used by MKL
 // enabled kernels
@@ -219,19 +225,18 @@ class MklShape {
 // Location from start of buffer where isMklTensor_ is serialized
 #define DIMS_OFFSET \
   (IS_MKL_TENSOR_OFFSET + sizeof(size_t))  // Location of dimension_
-#define SIZES_OFFSET(dims) \
-  (DIMS_OFFSET +           \
-   sizeof(size_t))  // Location of sizes. Note dim is not used here, left here
-                    // to make macros consistent.
+// Location of sizes. Note dim is not used here, left here
+// to make macros consistent.
+#define SIZES_OFFSET(dims) (DIMS_OFFSET + sizeof(size_t))
 #define STRIDES_OFFSET(dims) \
   (SIZES_OFFSET(dims) + dims * sizeof(size_t))  // Location of strides
 #define MKL_LAYOUT_OFFSET(dims) \
   (STRIDES_OFFSET(dims) + dims * sizeof(size_t))  // Location of mklLayout_
 #define TF_LAYOUT_OFFSET(dims) \
   (MKL_LAYOUT_OFFSET(dims) + SIZE_OF_MKL_DNN_BUF)  // Location of tfLayout_
+// Location of tf_to_mkl_dim_map_
 #define TF_TO_MKL_DIM_MAP_OFFSET(dims) \
-  (TF_LAYOUT_OFFSET(dims) +            \
-   SIZE_OF_MKL_DNN_BUF)  // Location of tf_to_mkl_dim_map_
+  (TF_LAYOUT_OFFSET(dims) + SIZE_OF_MKL_DNN_BUF)
 
   // TODO(agramesh1) make sure to create a const to share with rewrite pass
   // for min size of MKL metadata tensor.
@@ -342,58 +347,6 @@ inline Tensor ConvertMklToTF(OpKernelContext* context, const Tensor& mkl_tensor,
   return output_tensor;
 }
 
-// Since our ops are going to produce and also consume N addition tensors
-// (Mkl) for N Tensorflow tensors, we can have following different
-// orderings among these 2N tensors.
-//
-// E.g., for Tensorflow tensors A, B, and C, our ops will produce and
-// consume A_m, B_m, and C_m additionally.
-//
-// INTERLEAVED: in this case 2N tensors are interleaved. So for above
-//              example, the ordering looks like: A, A_m, B, B_m, C, C_m.
-//
-// CONTIGUOUS: in thi case N Tensorflow tensors are contiguous followed
-//             by N Mkl tensors. So for above example, the ordering looks
-//             like: A, B, C, A_m, B_m, C_m
-//
-// Following APIs map index of original Tensorflow tensors to their appropriate
-// position based on selected ordering. For contiguous ordering, we need to know
-// the total number of tensors (parameter total).
-//
-typedef enum { TENSORS_INTERLEAVED, TENSORS_CONTIGUOUS } MklTfTensorOrdering;
-// NOTE: Currently, we use contiguous ordering. If you change this, then you
-// would need to change Mkl op definitions in nn_ops.cc.
-static MklTfTensorOrdering kTensorOrdering = TENSORS_CONTIGUOUS;
-
-// Get index of MetaData tensor from index 'n' of Data tensor.
-inline int DataIndexToMetaDataIndex(int n, int total_tensors) {
-  if (kTensorOrdering == MklTfTensorOrdering::TENSORS_INTERLEAVED) {
-    // For interleaved ordering, Mkl tensor follows immediately after
-    // Tensorflow tensor.
-    return n + 1;
-  } else {
-    CHECK_EQ(kTensorOrdering, MklTfTensorOrdering::TENSORS_CONTIGUOUS);
-    // For contiguous ordering, Mkl tensor is n+total_tensors / 2 away.
-    return n + total_tensors / 2;
-  }
-}
-
-int inline GetTensorDataIndex(int n, int total_tensors) {
-  if (kTensorOrdering == MklTfTensorOrdering::TENSORS_INTERLEAVED) {
-    return 2 * n;  // index corresponding to nth input/output tensor
-  } else {
-    CHECK_EQ(kTensorOrdering, MklTfTensorOrdering::TENSORS_CONTIGUOUS);
-    return n;
-  }
-}
-
-int inline GetTensorMetaDataIndex(int n, int total_tensors) {
-  // Get index for TensorData first and then use mapping function
-  // to get TensorMetaData index from TensorData index.
-  int tidx = GetTensorDataIndex(n, total_tensors);
-  return DataIndexToMetaDataIndex(tidx, total_tensors);
-}
-
 // Get the MKL shape from the second string tensor
 inline void GetMklShape(OpKernelContext* ctext, int n, MklShape* mklshape) {
   mklshape->DeSerializeMklShape(
@@ -480,6 +433,13 @@ inline void AllocTmpBuffer(OpKernelContext* context, Tensor* tensor_out,
   *buf_out = static_cast<void*>(tensor_out->flat<float>().data());
 }
 
+template <typename T>
+inline void AllocTmpBuffer(OpKernelContext* context, Tensor* tensor_out,
+                           TensorShape tf_shape) {
+  OP_REQUIRES_OK(context, context->allocate_temp(DataTypeToEnum<T>::v(),
+                                                 tf_shape, tensor_out));
+}
+
 inline void GetStridesFromSizes(TensorFormat data_format, size_t* strides,
                                 const size_t* sizes) {
   // MKL requires strides in NCHW
@@ -743,56 +703,299 @@ inline void MklNCHWToNHWC(const Tensor& input, Tensor** output) {
   }
 }
 
-namespace mkl_op_registry {
-static const char* kMklOpLabel = "MklOp";
-static const char* kMklOpLabelPattern = "label='MklOp'";
+// -------------------------------------------------------------------
+
+#ifdef INTEL_MKL_DNN
+
+using mkldnn::engine;
+using mkldnn::memory;
+using mkldnn::padding_kind;
+using mkldnn::primitive;
+using mkldnn::reorder;
+
+/// Return MKL-DNN data type (memory::data_type) for input type T
+///
+/// @input None
+/// @return memory::data_type corresponding to type T
+template <typename T>
+static memory::data_type MklDnnType();
+
+/// Instantiation for float type. Add similar instantiations for other
+/// type if needed.
+template <>
+memory::data_type MklDnnType<float>() {
+  return memory::data_type::f32;
+}
+
+/// Map TensorFlow's data format into MKL-DNN data format
+///
+/// @input: TensorFlow data format
+/// @return: memory::format corresponding to TensorFlow data format;
+///          Fails with an error if invalid data format.
+inline memory::format TFDataFormatToMklDnnDataFormat(TensorFormat format) {
+  if (format == FORMAT_NHWC)
+    return memory::format::nhwc;
+  else if (format == FORMAT_NCHW)
+    return memory::format::nchw;
+  TF_CHECK_OK(Status(error::Code::INVALID_ARGUMENT, "Unsupported data format"));
+  // Return to get rid of compiler warning
+  return memory::format::format_undef;
+}
 
-// Get the name of Mkl op from original TensorFlow op
-// We prefix 'Mkl' to the original op to get Mkl op.
-inline string GetMklOpName(const string& name) {
-  // Prefix that we add to Tensorflow op name to construct Mkl op name.
-  const char* const kMklOpPrefix = "_Mkl";
-  return string(kMklOpPrefix) + name;
+/// Map TensorShape object into memory::dims required by MKL-DNN
+///
+/// This function will simply map input TensorShape into MKL-DNN dims
+/// naively. So it will preserve the order of dimensions. E.g., if
+/// input tensor is in NHWC format, then dims will be in NHWC format
+/// also.
+///
+/// @input TensorShape object in shape
+/// @return memory::dims corresponding to TensorShape
+inline memory::dims TFShapeToMklDnnDims(const TensorShape& shape) {
+  memory::dims dims(shape.dims());
+  for (unsigned int d = 0; d < shape.dims(); ++d) {
+    dims[d] = shape.dim_size(d);
+  }
+  return dims;
 }
 
-// Check whether opname with type T is registered as MKL-compliant.
-//
-// @input: name of the op
-// @input: T datatype to be used for checking op
-// @return: true if opname is registered as Mkl op; false otherwise
-static inline bool IsMklOp(const std::string& op_name, DataType T) {
-  string kernel = KernelsRegisteredForOp(op_name);
-  bool result =
-      kernel.find(kMklOpLabelPattern) != string::npos && (T == DT_FLOAT);
-  if (result) {
-    VLOG(1) << "mkl_op_registry::" << op_name << " is " << kMklOpLabel;
-  }
-  return result;
+/// Map TensorShape object into memory::dims in NCHW format required by MKL-DNN
+///
+/// This function is a specific one than above function. It will map input
+/// TensorShape into MKL-DNN dims in NCHW format. So it may not preserve the
+/// order of dimensions. E.g., if input tensor is in NHWC format, then dims
+/// will be in NCHW format, and not in NHWC format.
+///
+/// @input TensorShape object in shape
+/// @return memory::dims in MKL-DNN required NCHW format
+inline memory::dims TFShapeToMklDnnDimsInNCHW(const TensorShape& shape,
+                                              TensorFormat format) {
+  // Check validity of format.
+  CHECK_NE(TFDataFormatToMklDnnDataFormat(format),
+           memory::format::format_undef);
+
+  int n = shape.dim_size(GetTensorDimIndex(format, 'N'));
+  int c = shape.dim_size(GetTensorDimIndex(format, 'C'));
+  int h = shape.dim_size(GetTensorDimIndex(format, 'H'));
+  int w = shape.dim_size(GetTensorDimIndex(format, 'W'));
+
+  // MKL-DNN requires dimensions in NCHW format.
+  return memory::dims({n, c, h, w});
 }
 
-// Check whether opname with type T is registered as MKL-compliant and
-// is element-wise.
-//
-// @input: name of the op
-// @input: T datatype to be used for checking op
-// @return: true if opname is registered as element-wise Mkl op; false otherwise
-static inline bool IsMklElementWiseOp(const std::string& op_name, DataType T) {
-  if (!IsMklOp(op_name, T)) {
+inline padding_kind TFPaddingToMklDnnPadding(Padding pad) {
+  // MKL-DNN only supports zero padding.
+  return padding_kind::zero;
+}
+
+/*
+ * Class to represent all the resources corresponding to a tensor in TensorFlow
+ * that are required to execute an operation (such as Convolution).
+ */
+template <typename T>
+class MklDnnData {
+ private:
+  /// MKL-DNN memory primitive for input user memory
+  memory* user_memory_;
+
+  /// MKL-DNN memory primitive in case input or output reorder is needed.
+  memory* reorder_memory_;
+
+  /// Operations memory descriptor
+  memory::desc* op_md_;
+
+  /// CPU engine on which operation will be executed
+  const engine* cpu_engine_;
+
+ public:
+  explicit MklDnnData(const engine* e)
+      : user_memory_(nullptr),
+        reorder_memory_(nullptr),
+        op_md_(nullptr),
+        cpu_engine_(e) {}
+
+  ~MklDnnData() {
+    cpu_engine_ = nullptr;  // We don't own this.
+    delete (user_memory_);
+    delete (reorder_memory_);
+    delete (op_md_);
+  }
+
+  void* GetTensorBuffer(const Tensor* tensor) {
+    CHECK_NOTNULL(tensor);
+    return const_cast<void*>(
+        static_cast<const void*>(tensor->flat<T>().data()));
+  }
+
+  /// Set user memory primitive using specified dimensions, memory format and
+  /// data_buffer. Function automatically uses element data type by using
+  /// input type T used for creating call object.
+  ///
+  /// In a nutshell, function allows user to describe the input tensor to
+  /// an operation. E.g., filter of Conv2D is of shape {1, 2, 3, 4}, and
+  /// memory format HWIO, and the buffer that contains actual values is
+  /// pointed by data_buffer.
+  void SetUsrMem(memory::dims dim, memory::format fm, void* data_buffer) {
+    CHECK_NOTNULL(data_buffer);
+    CHECK_NOTNULL(cpu_engine_);
+    // TODO(nhasabni): can we remove dynamic memory allocation?
+    user_memory_ =
+        new memory(memory::primitive_desc(
+                       memory::desc(dim, MklDnnType<T>(), fm), *cpu_engine_),
+                   data_buffer);
+  }
+
+  void SetUsrMem(memory::dims dim, memory::format fm, const Tensor* tensor) {
+    CHECK_NOTNULL(tensor);
+    SetUsrMem(dim, fm, GetTensorBuffer(tensor));
+  }
+
+  /// A version of function to set user memory primitive that accepts memory
+  /// descriptor directly, instead of accepting dimensions and format. This
+  /// function is more generic that the one above, but the function above is
+  /// sufficient in most cases.
+  void SetUsrMem(memory::desc md, void* data_buffer) {
+    CHECK_NOTNULL(data_buffer);
+    CHECK_NOTNULL(cpu_engine_);
+    // TODO(nhasabni): can we remove dynamic memory allocation?
+    user_memory_ =
+        new memory(memory::primitive_desc(md, *cpu_engine_), data_buffer);
+  }
+
+  /// A version of SetUsrMem with memory descriptor and tensor
+  void SetUsrMem(memory::desc md, const Tensor* tensor) {
+    CHECK_NOTNULL(tensor);
+    SetUsrMem(md, GetTensorBuffer(tensor));
+  }
+
+  /// A version of function to set user memory primitive that accepts primitive
+  /// descriptor directly, instead of accepting dimensions and format. This
+  /// function is more generic that the one above, but the function above is
+  /// sufficient in most cases.
+  void SetUsrMem(memory::primitive_desc pd, void* data_buffer) {
+    CHECK_NOTNULL(data_buffer);
+    CHECK_NOTNULL(cpu_engine_);
+    // TODO(nhasabni): can we remove dynamic memory allocation?
+    user_memory_ = new memory(pd, data_buffer);
+  }
+
+  /// A version of SetUsrMem with primitive descriptor and tensor
+  void SetUsrMem(memory::primitive_desc pd, const Tensor* tensor) {
+    CHECK_NOTNULL(tensor);
+    SetUsrMem(pd, GetTensorBuffer(tensor));
+  }
+
+  /// Get function for user memory primitive.
+  const memory* GetUsrMem() const { return user_memory_; }
+
+  /// Get function for primitive descriptor of user memory primitive.
+  const memory::primitive_desc GetUsrMemPrimDesc() const {
+    CHECK_NOTNULL(user_memory_);
+    return user_memory_->get_primitive_desc();
+  }
+
+  /// Get function for descriptor of user memory.
+  memory::desc GetUsrMemDesc() {
+    // This is ugly. Why MKL-DNN does not provide desc() method of const type??
+    const memory::primitive_desc pd = GetUsrMemPrimDesc();
+    return const_cast<memory::primitive_desc*>(&pd)->desc();
+  }
+
+  /// Get function for data buffer of user memory primitive.
+  void* GetUsrMemDataHandle() const {
+    CHECK_NOTNULL(user_memory_);
+    return user_memory_->get_data_handle();
+  }
+
+  /// Get the memory primitive for input and output of an op. If inputs
+  /// to an op require reorders, then this function returns memory primitive
+  /// for reorder. Otherwise, it will return memory primitive for user memory.
+  ///
+  /// E.g., Conv2D(I, F) is a primitive with I and F being inputs. Then to
+  /// execute Conv2D, we need memory primitive for I and F. Buf if reorder is
+  /// required for I and F (say I_r is reorder primitive for I; F_r is reorder
+  /// primitive for F), then we need I_r and F_r to perform Conv2D.
+  const memory& GetOpMem() const {
+    return reorder_memory_ ? *reorder_memory_ : *user_memory_;
+  }
+
+  /// Set memory descriptor of an operation in terms of dimensions and memory
+  /// format. E.g., For Conv2D, the dimensions would be same as user dimensions
+  /// but memory::format would be mkldnn::any because we want MKL-DNN to choose
+  /// best layout/format for given input dimensions.
+  void SetOpMemDesc(const memory::dims& dim, memory::format fm) {
+    // TODO(nhasabni): can we remove dynamic memory allocation?
+    op_md_ = new memory::desc(dim, MklDnnType<T>(), fm);
+  }
+
+  /// Get function for memory descriptor for an operation
+  const memory::desc& GetOpMemDesc() const { return *op_md_; }
+
+  /// Function to handle input reordering
+  ///
+  /// Check if we need to reorder this input of an operation.
+  /// Return true and allocate reorder memory primitive if reorder is needed.
+  /// Otherwise, return false and do not allocate reorder memory primitive.
+  ///
+  /// To check if reorder is needed, this function compares memory primitive
+  /// descriptor of an operation (op_pd) for the given input with the
+  /// user-specified memory primitive descriptor.
+  ///
+  /// @input: op_pd - memory primitive descriptor of the given input of an
+  ///               operation
+  /// @input: net - net to which to add reorder primitive in case it is needed.
+  /// @return: true in case reorder of input is needed; false, otherwise.
+  bool CheckReorderToOpMem(const memory::primitive_desc& op_pd,
+                           std::vector<primitive>* net) {
+    CHECK_NOTNULL(net);
+    CHECK_NOTNULL(user_memory_);
+    if (op_pd != user_memory_->get_primitive_desc()) {
+      // TODO(nhasabni): can we remove dynamic memory allocation?
+      reorder_memory_ = new memory(op_pd);
+      net->push_back(reorder(*user_memory_, *reorder_memory_));
+      return true;
+    }
     return false;
   }
 
-  bool result = (0 == op_name.compare(GetMklOpName("Add")) ||
-                 0 == op_name.compare(GetMklOpName("Sub")) ||
-                 0 == op_name.compare(GetMklOpName("Mul")) ||
-                 0 == op_name.compare(GetMklOpName("Maximum")) ||
-                 0 == op_name.compare(GetMklOpName("SquaredDifference")));
+  /// Function to handle output reorder
+  ///
+  /// This function performs very similar functionality as input reordering
+  /// function above. The only difference is that this function does not add
+  /// reorder primitive to the net. The reason for this is: the reorder
+  /// primitive for output needs to be added to the list only after operation
+  /// has executed. But we need to prepare a temporary buffer in case output
+  /// reorder is needed. And this temporary buffer will hold the output of
+  /// an operation before it is fed to reorder primitive.
+  ///
+  /// @input memory primitive descriptor for the given output of an operation
+  /// @return: true in case reorder of output is needed; false, otherwise.
+  bool PrepareReorderToUserMemIfReq(const memory::primitive_desc& op_pd) {
+    CHECK_NOTNULL(user_memory_);
+    if (op_pd != user_memory_->get_primitive_desc()) {
+      // TODO(nhasabni): can we remove dynamic memory allocation?
+      reorder_memory_ = new memory(op_pd);
+      return true;
+    }
+    return false;
+  }
 
-  VLOG(1) << "mkl_op_registry::" << op_name
-          << " is elementwise MKL op: " << result;
-  return result;
-}
+  /// Function to actually insert reorder primitive in the net
+  ///
+  /// This function completes remaining part of output reordering. It inserts
+  /// a reordering primitive from the temporary buffer that holds the output
+  /// to the user-specified output buffer.
+  ///
+  /// @input: net - net to which to add reorder primitive
+  void InsertReorderToUserMem(std::vector<primitive>* net) {
+    CHECK_NOTNULL(net);
+    CHECK_NOTNULL(user_memory_);
+    CHECK_NOTNULL(reorder_memory_);
+    net->push_back(reorder(*reorder_memory_, *user_memory_));
+  }
+};
 
-}  // namespace mkl_op_registry
+#endif  // INTEL_MKL_DNN
 
 }  // namespace tensorflow
 #endif  // INTEL_MKL
diff --git a/tensorflow/docs_src/install/install_sources.md b/tensorflow/docs_src/install/install_sources.md
index d8925d3909..e6a4088656 100644
--- a/tensorflow/docs_src/install/install_sources.md
+++ b/tensorflow/docs_src/install/install_sources.md
@@ -429,3 +429,41 @@ Stack Overflow and specify the `tensorflow` tag.
   <pre>ImportError: cannot import name pywrap_tensorflow</pre></td>
 </tr>
 </table>
+
+## Tested source configurations
+**Linux**
+<table>
+<tr><th>Version:</th><th>CPU/GPU:</th><th>Python Version:</th><th>Compiler:</th><th>Build Tools:</th><th>cuDNN:</th><th>CUDA:</th></tr>
+<tr><td>tensorflow-1.3.0</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>GCC 4.8</td><td>Bazel 0.4.5</td><td>N/A</td><td>N/A</td></tr>
+<tr><td>tensorflow_gpu-1.3.0</td><td>GPU</td><td>2.7, 3.3-3.6</td><td>GCC 4.8</td><td>Bazel 0.4.5</td><td>6</td><td>8</td></tr>
+<tr><td>tensorflow-1.2.0</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>GCC 4.8</td><td>Bazel 0.4.5</td><td>N/A</td><td>N/A</td></tr>
+<tr><td>tensorflow_gpu-1.2.0</td><td>GPU</td><td>2.7, 3.3-3.6</td><td>GCC 4.8</td><td>Bazel 0.4.5</td><td>5.1</td><td>8</td></tr>
+<tr><td>tensorflow-1.1.0</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>GCC 4.8</td><td>Bazel 0.4.2</td><td>N/A</td><td>N/A</td></tr>
+<tr><td>tensorflow_gpu-1.1.0</td><td>GPU</td><td>2.7, 3.3-3.6</td><td>GCC 4.8</td><td>Bazel 0.4.2</td><td>5.1</td><td>8</td></tr>
+<tr><td>tensorflow-1.0.0</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>GCC 4.8</td><td>Bazel 0.4.2</td><td>N/A</td><td>N/A</td></tr>
+<tr><td>tensorflow_gpu-1.0.0</td><td>GPU</td><td>2.7, 3.3-3.6</td><td>GCC 4.8</td><td>Bazel 0.4.2</td><td>5.1</td><td>8</td></tr>
+</table>
+
+**Mac**
+<table>
+<tr><th>Version:</th><th>CPU/GPU:</th><th>Python Version:</th><th>Compiler:</th><th>Build Tools:</th><th>cuDNN:</th><th>CUDA:</th></tr>
+<tr><td>tensorflow-1.3.0</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>Clang from xcode</td><td>Bazel 0.4.5</td><td>N/A</td><td>N/A</td></tr>
+<tr><td>ttensorflow-1.2.0</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>Clang from xcode</td><td>Bazel 0.4.5</td><td>N/A</td><td>N/A</td></tr>
+<tr><td>ttensorflow-1.1.0</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>Clang from xcode</td><td>Bazel 0.4.2</td><td>N/A</td><td>N/A</td></tr>
+<tr><td>ttensorflow_gpu-1.1.0</td><td>GPU</td><td>2.7, 3.3-3.6</td><td>Clang from xcode</td><td>Bazel 0.4.2</td><td>5.1</td><td>8</td></tr>
+<tr><td>ttensorflow-1.0.0</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>Clang from xcode</td><td>Bazel 0.4.2</td><td>N/A</td><td>N/A</td></tr>
+<tr><td>ttensorflow_gpu-1.0.0</td><td>GPU</td><td>2.7, 3.3-3.6</td><td>Clang from xcode</td><td>Bazel 0.4.2</td><td>5.1</td><td>8</td></tr>
+</table>
+
+**Windows**
+<table>
+<tr><th>Version:</th><th>CPU/GPU:</th><th>Python Version:</th><th>Compiler:</th><th>Build Tools:</th><th>cuDNN:</th><th>CUDA:</th></tr>
+<tr><td>tensorflow-1.3.0</td><td>CPU</td><td>3.5-3.6</td><td>MSVC 2015 update 3</td><td>Cmake v3.6.3</td><td>N/A</td><td>N/A</td></tr>
+<tr><td>tensorflow_gpu-1.3.0</td><td>GPU</td><td>3.5-3.6</td><td>MSVC 2015 update 3</td><td>Cmake v3.6.3</td><td>6</td><td>8</td></tr>
+<tr><td>tensorflow-1.2.0</td><td>CPU</td><td>3.5-3.6</td><td>MSVC 2015 update 3</td><td>Cmake v3.6.3</td><td>N/A</td><td>N/A</td></tr>
+<tr><td>tensorflow_gpu-1.2.0</td><td>GPU</td><td>3.5-3.6</td><td>MSVC 2015 update 3</td><td>Cmake v3.6.3</td><td>5.1</td><td>8</td></tr>
+<tr><td>tensorflow-1.1.0</td><td>CPU</td><td>3.5</td><td>MSVC 2015 update 3</td><td>Cmake v3.6.3</td><td>N/A</td><td>N/A</td></tr>
+<tr><td>tensorflow_gpu-1.1.0</td><td>GPU</td><td>3.5</td><td>MSVC 2015 update 3</td><td>Cmake v3.6.3</td><td>5.1</td><td>8</td></tr>
+<tr><td>tensorflow-1.0.0</td><td>CPU</td><td>3.5</td><td>MSVC 2015 update 3</td><td>Cmake v3.6.3</td><td>N/A</td><td>N/A</td></tr>
+<tr><td>tensorflow_gpu-1.0.0</td><td>GPU</td><td>3.5</td><td>MSVC 2015 update 3</td><td>Cmake v3.6.3</td><td>5.1</td><td>8</td></tr>
+</table>
diff --git a/tensorflow/examples/android/src/org/tensorflow/demo/SpeechActivity.java b/tensorflow/examples/android/src/org/tensorflow/demo/SpeechActivity.java
index eb4dc69d63..184df1bdb4 100644
--- a/tensorflow/examples/android/src/org/tensorflow/demo/SpeechActivity.java
+++ b/tensorflow/examples/android/src/org/tensorflow/demo/SpeechActivity.java
@@ -37,6 +37,7 @@ import android.content.pm.PackageManager;
 import android.media.AudioFormat;
 import android.media.AudioRecord;
 import android.media.MediaRecorder;
+import android.os.Build;
 import android.os.Bundle;
 import android.util.Log;
 import android.view.View;
@@ -151,12 +152,15 @@ public class SpeechActivity extends Activity {
 
     // Start the recording and recognition threads.
     requestMicrophonePermission();
+    startRecording();
     startRecognition();
   }
 
   private void requestMicrophonePermission() {
-    requestPermissions(
-        new String[] {android.Manifest.permission.RECORD_AUDIO}, REQUEST_RECORD_AUDIO);
+    if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) {
+      requestPermissions(
+          new String[]{android.Manifest.permission.RECORD_AUDIO}, REQUEST_RECORD_AUDIO);
+    }
   }
 
   @Override
diff --git a/tensorflow/examples/tutorials/word2vec/word2vec_basic.py b/tensorflow/examples/tutorials/word2vec/word2vec_basic.py
index 6d98c7b85d..1fa2b14869 100644
--- a/tensorflow/examples/tutorials/word2vec/word2vec_basic.py
+++ b/tensorflow/examples/tutorials/word2vec/word2vec_basic.py
@@ -89,7 +89,7 @@ def build_dataset(words, n_words):
 # Filling 4 global variables:
 # data - list of codes (integers from 0 to vocabulary_size-1).
 #   This is the original text but words are replaced by their codes
-# count - map of words(strings) to count of occurences
+# count - map of words(strings) to count of occurrences
 # dictionary - map of words(strings) to their codes(integers)
 # reverse_dictionary - maps codes(integers) to words(strings)
 data, count, dictionary, reverse_dictionary = build_dataset(vocabulary,
diff --git a/tensorflow/go/example_inception_inference_test.go b/tensorflow/go/example_inception_inference_test.go
index 2162fbe484..f84a588899 100644
--- a/tensorflow/go/example_inception_inference_test.go
+++ b/tensorflow/go/example_inception_inference_test.go
@@ -28,8 +28,8 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/tensorflow/tensorflow/tensorflow/go/op"
 	tf "github.com/tensorflow/tensorflow/tensorflow/go"
+	"github.com/tensorflow/tensorflow/tensorflow/go/op"
 )
 
 func Example() {
diff --git a/tensorflow/go/tensor.go b/tensorflow/go/tensor.go
index a534a0d659..e8fa21a62b 100644
--- a/tensorflow/go/tensor.go
+++ b/tensorflow/go/tensor.go
@@ -92,7 +92,7 @@ func NewTensor(value interface{}) (*Tensor, error) {
 	raw := tensorData(t.c)
 	buf := bytes.NewBuffer(raw[:0:len(raw)])
 	if dataType != String {
-		if err := encodeTensor(buf, val); err != nil {
+		if err := encodeTensor(buf, val, shape); err != nil {
 			return nil, err
 		}
 		if uintptr(buf.Len()) != nbytes {
@@ -100,7 +100,7 @@ func NewTensor(value interface{}) (*Tensor, error) {
 		}
 	} else {
 		e := stringEncoder{offsets: buf, data: raw[nflattened*8 : len(raw)], status: newStatus()}
-		if err := e.encode(reflect.ValueOf(value)); err != nil {
+		if err := e.encode(reflect.ValueOf(value), shape); err != nil {
 			return nil, err
 		}
 		if int64(buf.Len()) != nflattened*8 {
@@ -236,17 +236,11 @@ func shapeAndDataTypeOf(val reflect.Value) (shape []int64, dt DataType, err erro
 	typ := val.Type()
 	for typ.Kind() == reflect.Array || typ.Kind() == reflect.Slice {
 		shape = append(shape, int64(val.Len()))
-		// If slice elements are slices, verify that all of them have the same size.
-		// Go's type system makes that guarantee for arrays.
 		if val.Len() > 0 {
-			if val.Type().Elem().Kind() == reflect.Slice {
-				expected := val.Index(0).Len()
-				for i := 1; i < val.Len(); i++ {
-					if val.Index(i).Len() != expected {
-						return shape, dt, fmt.Errorf("mismatched slice lengths: %d and %d", val.Index(i).Len(), expected)
-					}
-				}
-			}
+			// In order to check tensor structure properly in general case we need to iterate over all slices of the tensor to check sizes match
+			// Since we already going to iterate over all elements in encodeTensor() let's
+			// 1) do the actual check in encodeTensor() to save some cpu cycles here
+			// 2) assume the shape is represented by lengths of elements with zero index in each dimension
 			val = val.Index(0)
 		}
 		typ = typ.Elem()
@@ -302,7 +296,7 @@ func byteSizeOfEncodedStrings(val interface{}) uintptr {
 
 // encodeTensor writes v to the specified buffer using the format specified in
 // c_api.h. Use stringEncoder for String tensors.
-func encodeTensor(w *bytes.Buffer, v reflect.Value) error {
+func encodeTensor(w *bytes.Buffer, v reflect.Value, shape []int64) error {
 	switch v.Kind() {
 	case reflect.Bool:
 		b := byte(0)
@@ -318,19 +312,18 @@ func encodeTensor(w *bytes.Buffer, v reflect.Value) error {
 		}
 
 	case reflect.Array, reflect.Slice:
-		// If slice elements are slices, verify that all of them have the same size.
+		// If current dimension is a slice, verify that it has the expected size
 		// Go's type system makes that guarantee for arrays.
-		if v.Len() > 0 && v.Type().Elem().Kind() == reflect.Slice {
-			expected := v.Index(0).Len()
-			for i := 1; i < v.Len(); i++ {
-				if v.Index(i).Len() != expected {
-					return fmt.Errorf("mismatched slice lengths: %d and %d", v.Index(i).Len(), expected)
-				}
+		if v.Kind() == reflect.Slice {
+			expected := int(shape[0])
+			if v.Len() != expected {
+				return fmt.Errorf("mismatched slice lengths: %d and %d", v.Len(), expected)
 			}
 		}
 
+		subShape := shape[1:]
 		for i := 0; i < v.Len(); i++ {
-			err := encodeTensor(w, v.Index(i))
+			err := encodeTensor(w, v.Index(i), subShape)
 			if err != nil {
 				return err
 			}
@@ -379,7 +372,7 @@ type stringEncoder struct {
 	status  *status
 }
 
-func (e *stringEncoder) encode(v reflect.Value) error {
+func (e *stringEncoder) encode(v reflect.Value, shape []int64) error {
 	if v.Kind() == reflect.String {
 		if err := binary.Write(e.offsets, nativeEndian, e.offset); err != nil {
 			return err
@@ -395,8 +388,17 @@ func (e *stringEncoder) encode(v reflect.Value) error {
 		C.free(unsafe.Pointer(src))
 		return e.status.Err()
 	}
+
+	if v.Kind() == reflect.Slice {
+		expected := int(shape[0])
+		if v.Len() != expected {
+			return fmt.Errorf("mismatched slice lengths: %d and %d", v.Len(), expected)
+		}
+	}
+
+	subShape := shape[1:]
 	for i := 0; i < v.Len(); i++ {
-		if err := e.encode(v.Index(i)); err != nil {
+		if err := e.encode(v.Index(i), subShape); err != nil {
 			return err
 		}
 	}
diff --git a/tensorflow/go/tensor_test.go b/tensorflow/go/tensor_test.go
index 2fc7553f87..35bd2fd9a5 100644
--- a/tensorflow/go/tensor_test.go
+++ b/tensorflow/go/tensor_test.go
@@ -42,6 +42,10 @@ func TestNewTensor(t *testing.T) {
 		{[]int64{2}, []bool{true, false}},
 		{[]int64{1}, []float64{1}},
 		{[]int64{1}, [1]float64{1}},
+		{[]int64{1, 1}, [1][1]float64{{1}}},
+		{[]int64{1, 1, 1}, [1][1][]float64{{{1}}}},
+		{[]int64{1, 1, 2}, [1][][2]float64{{{1, 2}}}},
+		{[]int64{1, 1, 1, 1}, [1][][1][]float64{{{{1}}}}},
 		{[]int64{2}, []string{"string", "slice"}},
 		{[]int64{2}, [2]string{"string", "array"}},
 		{[]int64{3, 2}, [][]float64{{1, 2}, {3, 4}, {5, 6}}},
@@ -74,6 +78,12 @@ func TestNewTensor(t *testing.T) {
 		[]uint64{5},
 		// Mismatched dimensions
 		[][]float32{{1, 2, 3}, {4}},
+		// Mismatched dimensions. Should return "mismatched slice lengths" error instead of "BUG"
+		[][][]float32{{{1, 2}, {3, 4}}, {{1}, {3}}},
+		// Mismatched dimensions. Should return error instead of valid tensor
+		[][][]float32{{{1, 2}, {3, 4}}, {{1}, {3}}, {{1, 2, 3}, {2, 3, 4}}},
+		// Mismatched dimensions for strings
+		[][]string{{"abc"}, {"abcd", "abcd"}},
 	}
 
 	for _, test := range tests {
diff --git a/tensorflow/java/src/gen/perl/tftypes-runall.pl b/tensorflow/java/src/gen/perl/tftypes-runall.pl
index 258c1ff836..a451ce92aa 100644
--- a/tensorflow/java/src/gen/perl/tftypes-runall.pl
+++ b/tensorflow/java/src/gen/perl/tftypes-runall.pl
@@ -37,4 +37,4 @@ sub locchk {
 &locchk("$rsrc/tftypes.csv");
 
 system("perl $dir/tftypes.pl -t $rsrc/tftypes.csv $pkg/types");
-# system("perl $dir/tftypes.pl -c $rsrc/tftypes.csv $rsrc/Tensors.java.tmpl > $pkg/op/Tensors.java");
+system("perl $dir/tftypes.pl -c $rsrc/tftypes.csv $rsrc/Tensors.java.tmpl > $pkg/Tensors.java");
diff --git a/tensorflow/java/src/gen/perl/tftypes.pl b/tensorflow/java/src/gen/perl/tftypes.pl
index 86867335cb..115723ac8a 100644
--- a/tensorflow/java/src/gen/perl/tftypes.pl
+++ b/tensorflow/java/src/gen/perl/tftypes.pl
@@ -75,15 +75,23 @@ open (TYPEDESC, $typedesc);
 
 my @info = ([]);
 
+sub trim {
+    (my $ret) = @_;
+    $ret =~ s/^\s*//g;
+    $ret =~ s/\s*$//g;
+    return $ret;
+}
+
 while (<TYPEDESC>) {
     chomp;
     my $line = $_;
     if ($line =~ m/^TF type/) { next }
     $line =~ s/\r$//;
-    (my $name, my $jtype, my $creat, my $default, my $desc) =
-        split /,/, $line, 5;
-    $desc =~ s/^ *//g;
-    $desc =~ s/ *$//g;
+    my @items = split /,/, $line, 6;
+    for (my $i = 0; $i <= $#items; $i++) {
+        $items[$i] = trim $items[$i];
+    }
+    my $jtype = $items[2];
     $jtypecount{$jtype}++;
     if ($jtypecount{$jtype} > 1) {
 # currently allowing Java types to stand for more than one TF type, but
@@ -92,63 +100,85 @@ while (<TYPEDESC>) {
 #       exit 1
     }
 
-    push @info, [$name, $jtype, $creat, $default, $desc];
+    push @info, \@items;
+}
+
+sub article {
+    (my $s) = @_;
+    if (substr($s, 0, 1) =~ m/^[aeoiu8]$/i) {
+        return "an $s"
+    } else {
+        return "a $s"
+    }
 }
 
 for (my $i = 1; $i <= $#info; $i++) {
-    (my $name, my $jtype, my $creat, my $default, my $desc) =
+    (my $name, my $builtin, my $jtype, my $creat, my $default, my $desc) =
         @{$info[$i]};
-    my $tfname = "TF".$name;
+    my $tfname = $name;
     my $ucname = uc $name;
 
+    print STDERR "$name $desc\n";
+
     if ($option eq '-t') {
         if ($jtype eq '') { next }
+        if ($builtin eq 'y') { next }
         # Generate class declarations
         # print STDERR "Creating $dirname/$tfname.java\n";
         open (CLASSFILE, ">$dirname/$tfname.java") || die "Can't open $tfname.java";
-        print CLASSFILE $copyright;
-        print CLASSFILE "// GENERATED FILE. To update, edit tftypes.pl instead.\n\n";
-
-        my $fulldesc = $desc;
-        if (substr($desc, 0, 1) =~ m/^[aeoiu8]$/i) {
-            $fulldesc = "an $desc"
-        } else {
-            $fulldesc = "a $desc"
-        }
-        print CLASSFILE  "package org.tensorflow.types;\n\n"
-                        ."import org.tensorflow.DataType;\n\n";
+        print CLASSFILE $copyright, "\n";
+        # print CLASSFILE "// GENERATED FILE. To update, edit tftypes.pl instead.\n\n";
+
+        my $fulldesc = article($desc);
+        print CLASSFILE  "package org.tensorflow.types;\n\n";
         print CLASSFILE  "/** Represents $fulldesc. */\n"
-                        ."public class $tfname implements TFType {\n"
-                        ."  private $tfname() {}\n"
-                        ."  static {\n"
-                        ."    Types.typeCodes.put($tfname.class, DataType.$ucname);\n"
-                        ."  }\n";
-        if ($default ne '') {
-            print CLASSFILE
-                         "  static {\n"
-                        ."    Types.scalars.put($tfname.class, $default);\n"
-                        ."  }\n";
-        }
-        print CLASSFILE  "}\n";
+                        ."public class $tfname {\n"
+                        ."  private $tfname() {\n"
+                        ."  }\n"
+                        ."}\n";
         close(CLASSFILE);
     } elsif ($option eq '-c') {
       # Generate creator declarations for Tensors.java
       if ($jtype ne '' && $creat eq 'y') {
-        for (my $brackets = ''; length $brackets <= 12; $brackets .= '[]') {
+        for (my $brackets = '', my $rank = 0; length $brackets <= 12; $brackets .= '[]', $rank++) {
+            my $datainfo = "   *  \@param data An array containing the values to put into the new tensor.\n"
+                          ."   *  The dimensions of the new tensor will match those of the array.\n";
+            if ($rank == 0) {
+                $datainfo = "   *  \@param data The value to put into the new scalar tensor.\n"
+            }
+
+            my $trank = $rank;
+            if ($tfname eq 'String') {
+                $trank = $rank-1;
+                next if $trank < 0;
+
+                $datainfo = "   *  \@param data An array containing the data to put into the new tensor.\n"
+                           ."   *  String elements are sequences of bytes from the last array dimension.\n";
+            }
+
+    
+            my $intro = ($trank > 0)
+                ?  "Creates a rank-$trank tensor of {\@code $jtype} elements."
+                :  "Creates a scalar tensor containing a single {\@code $jtype} element.";
             $typeinfo .=
-                "  public static Tensor<$tfname> create($jtype$brackets data) {\n"
-               ."    return Tensor.create(data, $tfname.class);\n"
-               ."  }\n";
+             "  /**\n"
+            ."   * $intro\n"
+            ."   * \n"
+            .$datainfo
+            ."   */\n"
+            ."  public static Tensor<$tfname> create($jtype$brackets data) {\n"
+            ."    return Tensor.create(data, $tfname.class);\n"
+            ."  }\n\n";
         }
       }
-      if ($text =~ m/\b$tfname\b/ || $creat eq 'y') {
+      if ($text =~ m/\b$tfname\b/ && $builtin eq 'n' && $creat eq 'y') {
             $imports .= "import org.tensorflow.types.$tfname;\n";
       }
     }
 }
 
 if ($option ne '-t') {
-  print "// GENERATED FILE. Edits to this file will be lost -- edit $tmpl instead.\n";
+# print "// GENERATED FILE. Edits to this file will be lost -- edit $tmpl instead.\n";
 
   $text =~ s/\@TYPEINFO\@/$typeinfo/;
   $text =~ s/\@IMPORTS\@/$imports/;
diff --git a/tensorflow/java/src/gen/resources/Tensors.java.tmpl b/tensorflow/java/src/gen/resources/Tensors.java.tmpl
new file mode 100644
index 0000000000..98e1588559
--- /dev/null
+++ b/tensorflow/java/src/gen/resources/Tensors.java.tmpl
@@ -0,0 +1,31 @@
+package org.tensorflow;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import org.tensorflow.Tensor;
+@IMPORTS@
+
+/**
+ * Type-safe factory methods for creating {@link Tensor} objects.
+ */
+public final class Tensors {
+  private Tensors() {}
+
+  /** Creates a scalar String tensor using the default, UTF-8 encoding.
+   * 
+   *  @param data  The string to put into the new scalar tensor.
+   */
+  public static Tensor<String> create(String data) {
+    return Tensor.create(data.getBytes(UTF_8), String.class);
+  }
+
+  /** Creates a scalar String tensor using a specified encoding.
+   * 
+   *  @param charset The encoding from String to bytes.
+   *  @param data    The string to put into the new scalar tensor.
+   */
+  public static Tensor<String> create(String data, java.nio.charset.Charset charset) {
+    return Tensor.create(data.getBytes(charset), String.class);
+  }
+
+@TYPEINFO@}
+
diff --git a/tensorflow/java/src/gen/resources/tftypes.csv b/tensorflow/java/src/gen/resources/tftypes.csv
index 88acaafd3c..6f26230f27 100644
--- a/tensorflow/java/src/gen/resources/tftypes.csv
+++ b/tensorflow/java/src/gen/resources/tftypes.csv
@@ -1,21 +1,21 @@
-TF type,Java type,Creator?,Zero value,Description
-Float,float,y,0f,32-bit single precision floating point number
-Double,double,y,0.0,64-bit double precision floating point number
-Int32,int,y,0,32-bit signed integer
-UInt8,byte,n,(byte)0,8-bit unsigned integer
-Int16,,n,(short)0,16-bit signed integer
-Int8,,n,(byte)0,8-bit signed integer
-String,byte,n,,arbitrary sequence of bytes
-Complex64,,n,,single-precision complex number
-Int64,long,y,0L,64-bit signed integer
-Bool,boolean,y,false,boolean
-QInt8,,n,,quantized int8
-QUInt8,,n,,quantized uint8
-QInt32,,n,,quantized int32
-BFloat16,,n,,float32 truncated to 16 bits. Only for cast ops.
-QInt16,,n,,quantized int16
-QUInt16,,n,,quantized uint16
-UInt16,,n,,16-bit unsigned integer
-Complex128,,n,,double-precision complex number
-Half,,n,,
-Resource,,n,,
+TF type,Builtin,Java type,Creator?,Zero value,Description
+Float,y,float,y,0f,32-bit single precision floating point number
+Double,y,double,y,0.0,64-bit double precision floating point number
+Integer,y,int,y,0,32-bit signed integer
+UInt8,n,byte,n,(byte)0,8-bit unsigned integer
+Short,y,,n,(short)0,16-bit signed integer
+Byte,y,,n,(byte)0,8-bit signed integer
+String,y,byte,y,,arbitrary sequence of bytes
+Complex64,n,,n,,single-precision complex number
+Long,y,long,y,0L,64-bit signed integer
+Boolean,y,boolean,y,false,boolean
+QInt8,n,,n,,quantized int8
+QUInt8,n,,n,,quantized uint8
+QInt32,n,,n,,quantized int32
+BFloat16,n,,n,,float32 truncated to 16 bits. Only for cast ops.
+QInt16,n,,n,,quantized int16
+QUInt16,n,,n,,quantized uint16
+UInt16,n,,n,,16-bit unsigned integer
+Complex128,n,,n,,double-precision complex number
+Half,n,,n,,
+Resource,n,,n,,
diff --git a/tensorflow/java/src/main/java/org/tensorflow/DataType.java b/tensorflow/java/src/main/java/org/tensorflow/DataType.java
index e67e266ff7..e835101d08 100644
--- a/tensorflow/java/src/main/java/org/tensorflow/DataType.java
+++ b/tensorflow/java/src/main/java/org/tensorflow/DataType.java
@@ -15,7 +15,13 @@ limitations under the License.
 
 package org.tensorflow;
 
-/** Type of elements in a {@link Tensor}. */
+import java.util.HashMap;
+import java.util.Map;
+import org.tensorflow.types.UInt8;
+
+/**
+ * Represents the type of elements in a {@link Tensor} as an enum.
+ */
 public enum DataType {
   /** 32-bit single precision floating point. */
   FLOAT(1),
@@ -55,14 +61,41 @@ public enum DataType {
   }
   
   // Cached to avoid copying it
-  final private static DataType[] values = values();
+  private static final DataType[] values = values();
 
   static DataType fromC(int c) {
     for (DataType t : values) {
-      if (t.value == c)
+      if (t.value == c) {
         return t;
+      }
     }
     throw new IllegalArgumentException(
         "DataType " + c + " is not recognized in Java (version " + TensorFlow.version() + ")");
   }
+
+  /**
+   * Returns the DataType of a Tensor whose elements have the type specified by class {@code c}.
+   *
+   * @param c The class describing the TensorFlow type of interest.
+   */
+  public static DataType fromClass(Class<?> c) {
+    DataType dtype = typeCodes.get(c);
+    if (dtype == null) {
+      throw new IllegalArgumentException(
+          c.getName() + " objects cannot be used as elements in a TensorFlow Tensor");
+    }
+    return dtype;
+  }
+
+  private static final Map<Class<?>, DataType> typeCodes = new HashMap<>();
+
+  static {
+    typeCodes.put(Float.class, DataType.FLOAT);
+    typeCodes.put(Double.class, DataType.DOUBLE);
+    typeCodes.put(Integer.class, DataType.INT32);
+    typeCodes.put(UInt8.class, DataType.UINT8);
+    typeCodes.put(Long.class, DataType.INT64);
+    typeCodes.put(Boolean.class, DataType.BOOL);
+    typeCodes.put(String.class, DataType.STRING);
+  }
 }
diff --git a/tensorflow/java/src/main/java/org/tensorflow/Graph.java b/tensorflow/java/src/main/java/org/tensorflow/Graph.java
index 58ad3ab193..d4fd3db5f7 100644
--- a/tensorflow/java/src/main/java/org/tensorflow/Graph.java
+++ b/tensorflow/java/src/main/java/org/tensorflow/Graph.java
@@ -81,8 +81,8 @@ public final class Graph implements AutoCloseable {
   /**
    * Iterator over all the {@link Operation}s in the graph.
    *
-   * The order of iteration is unspecified. Consumers of the iterator will received no notification
-   * should the underlying graph change during iteration.
+   * <p>The order of iteration is unspecified. Consumers of the iterator will receive no
+   * notification should the underlying graph change during iteration.
    */
   public Iterator<Operation> operations() {
     return new OperationIterator(this);
@@ -245,7 +245,8 @@ public final class Graph implements AutoCloseable {
 
   private static native long operation(long handle, String name);
 
-  // This method returns the Operation native handle at index 0 and the new value for pos at index 1 (see TF_GraphNextOperation)
+  // This method returns the Operation native handle at index 0 and the new value for pos at index 1
+  // (see TF_GraphNextOperation)
   private static native long[] nextOperation(long handle, int position);
 
   private static native void importGraphDef(long handle, byte[] graphDef, String prefix)
diff --git a/tensorflow/java/src/main/java/org/tensorflow/Input.java b/tensorflow/java/src/main/java/org/tensorflow/Input.java
index 8e6685ee0f..13bc463e7d 100644
--- a/tensorflow/java/src/main/java/org/tensorflow/Input.java
+++ b/tensorflow/java/src/main/java/org/tensorflow/Input.java
@@ -34,7 +34,7 @@ package org.tensorflow;
  * ops.array().concat(0, split);
  * }</pre>
  */
-public interface Input {
+public interface Input<T> {
 
   /**
    * Returns the symbolic handle of a tensor.
@@ -44,5 +44,5 @@ public interface Input {
    *
    * @see OperationBuilder#addInput(Output)
    */
-  Output asOutput();
+  Output<T> asOutput();
 }
diff --git a/tensorflow/java/src/main/java/org/tensorflow/NativeLibrary.java b/tensorflow/java/src/main/java/org/tensorflow/NativeLibrary.java
index d2d019babb..2b431eebf5 100644
--- a/tensorflow/java/src/main/java/org/tensorflow/NativeLibrary.java
+++ b/tensorflow/java/src/main/java/org/tensorflow/NativeLibrary.java
@@ -122,8 +122,7 @@ final class NativeLibrary {
   }
 
   private static String extractResource(
-      InputStream resource, String resourceName, String extractToDirectory)
-      throws IOException {
+      InputStream resource, String resourceName, String extractToDirectory) throws IOException {
     final File dst = new File(extractToDirectory, System.mapLibraryName(resourceName));
     dst.deleteOnExit();
     final String dstPath = dst.toString();
@@ -184,8 +183,7 @@ final class NativeLibrary {
   // compatibility.
   private static File createTemporaryDirectory() {
     File baseDirectory = new File(System.getProperty("java.io.tmpdir"));
-    String directoryName
-        = "tensorflow_native_libraries-" + System.currentTimeMillis() + "-";
+    String directoryName = "tensorflow_native_libraries-" + System.currentTimeMillis() + "-";
     for (int attempt = 0; attempt < 1000; attempt++) {
       File temporaryDirectory = new File(baseDirectory, directoryName + attempt);
       if (temporaryDirectory.mkdir()) {
@@ -194,7 +192,8 @@ final class NativeLibrary {
     }
     throw new IllegalStateException(
         "Could not create a temporary directory (tried to make "
-        + directoryName + "*) to extract TensorFlow native libraries.");
+            + directoryName
+            + "*) to extract TensorFlow native libraries.");
   }
 
   private NativeLibrary() {}
diff --git a/tensorflow/java/src/main/java/org/tensorflow/Operand.java b/tensorflow/java/src/main/java/org/tensorflow/Operand.java
index 695c4c1060..61082e83d5 100644
--- a/tensorflow/java/src/main/java/org/tensorflow/Operand.java
+++ b/tensorflow/java/src/main/java/org/tensorflow/Operand.java
@@ -22,19 +22,19 @@ package org.tensorflow;
  *
  * <pre>{@code
  * // The "decodeJpeg" operation can be used as an operand to the "cast" operation
- * Operand decodeJpeg = ops.image().decodeJpeg(...);
+ * Operand<UInt8> decodeJpeg = ops.image().decodeJpeg(...);
  * ops.math().cast(decodeJpeg, DataType.FLOAT);
  *
  * // The output "y" of the "unique" operation can be used as an operand to the "cast" operation
- * Output y = ops.array().unique(...).y();
- * ops.math().cast(y, DataType.FLOAT);
+ * Output<Integer> y = ops.array().unique(...).y();
+ * ops.math().cast(y, Float.class);
  *
  * // The "split" operation can be used as operand list to the "concat" operation
- * Iterable<? extends Operand> split = ops.array().split(...);
+ * Iterable<? extends Operand<Float>> split = ops.array().split(...);
  * ops.array().concat(0, split);
  * }</pre>
  */
-public interface Operand {
+public interface Operand<T> {
 
   /**
    * Returns the symbolic handle of a tensor.
@@ -44,5 +44,5 @@ public interface Operand {
    *
    * @see OperationBuilder#addInput(Output)
    */
-  Output asOutput();
+  Output<T> asOutput();
 }
diff --git a/tensorflow/java/src/main/java/org/tensorflow/Operation.java b/tensorflow/java/src/main/java/org/tensorflow/Operation.java
index ec26309fba..6b82e5780b 100644
--- a/tensorflow/java/src/main/java/org/tensorflow/Operation.java
+++ b/tensorflow/java/src/main/java/org/tensorflow/Operation.java
@@ -98,16 +98,26 @@ public final class Operation {
    * @param length number of tensors in the list
    * @return array of {@code Output}
    */
-  public Output[] outputList(int idx, int length) {
-    Output[] outputs = new Output[length];
+  public Output<?>[] outputList(int idx, int length) {
+    Output<?>[] outputs = new Output<?>[length];
     for (int i = 0; i < length; ++i) {
       outputs[i] = output(idx + i);
     }
     return outputs;
   }
 
-  /** Returns a symbolic handle to one of the tensors produced by this operation. */
-  public Output output(int idx) {
+  /**
+   * Returns a symbolic handle to one of the tensors produced by this operation.
+   *
+   * <p>Warning: Does not check that the type of the tensor matches T. It is recommended to call
+   * this method with an explicit type parameter rather than letting it be inferred, e.g. {@code
+   * operation.<Integer>output(0)}
+   *
+   * @param <T> The expected element type of the tensors produced by this output.
+   * @param idx The index of the output among the outputs produced by this operation.
+   */
+  @SuppressWarnings({"rawtypes", "unchecked"})
+  public <T> Output<T> output(int idx) {
     return new Output(this, idx);
   }
 
diff --git a/tensorflow/java/src/main/java/org/tensorflow/OperationBuilder.java b/tensorflow/java/src/main/java/org/tensorflow/OperationBuilder.java
index 15077ce439..9a1b7592b3 100644
--- a/tensorflow/java/src/main/java/org/tensorflow/OperationBuilder.java
+++ b/tensorflow/java/src/main/java/org/tensorflow/OperationBuilder.java
@@ -63,7 +63,6 @@ public final class OperationBuilder {
     }
   }
 
-
   /**
    * Returns the builder to create an operation.
    *
@@ -73,7 +72,7 @@ public final class OperationBuilder {
    * @param input {@link Output} supposed to be the input of the OperationBuilder.
    * @return the OperationBuilder instance for chaining.
    */
-  public OperationBuilder addInput(Output input) {
+  public OperationBuilder addInput(Output<?> input) {
     Graph.Reference r = graph.ref();
     try {
       addInput(unsafeNativeHandle, input.op().getUnsafeNativeHandle(), input.index());
@@ -106,7 +105,7 @@ public final class OperationBuilder {
     return this;
   }
 
-  public OperationBuilder addInputList(Output[] inputs) {
+  public OperationBuilder addInputList(Output<?>[] inputs) {
     Graph.Reference r = graph.ref();
     try {
       long[] opHandles = new long[inputs.length];
@@ -231,7 +230,7 @@ public final class OperationBuilder {
     return this;
   }
 
-  public OperationBuilder setAttr(String name, Tensor value) {
+  public OperationBuilder setAttr(String name, Tensor<?> value) {
     Graph.Reference r = graph.ref();
     try {
       setAttrTensor(unsafeNativeHandle, name, value.getNativeHandle());
@@ -241,10 +240,10 @@ public final class OperationBuilder {
     return this;
   }
 
-  public OperationBuilder setAttr(String name, Tensor[] value) {
+  public OperationBuilder setAttr(String name, Tensor<?>[] value) {
     long[] handles = new long[value.length];
     int idx = 0;
-    for (Tensor t : value) {
+    for (Tensor<?> t : value) {
       handles[idx++] = t.getNativeHandle();
     }
     Graph.Reference r = graph.ref();
@@ -266,7 +265,7 @@ public final class OperationBuilder {
     return this;
   }
 
-  public OperationBuilder setAttr(String name,  String[] value) {
+  public OperationBuilder setAttr(String name, String[] value) {
     Charset utf8 = Charset.forName("UTF-8");
     Object[] objects = new Object[value.length];
     for (int i = 0; i < value.length; ++i) {
@@ -326,5 +325,4 @@ public final class OperationBuilder {
   private static native void setAttrShape(long handle, String name, long[] shape, int numDims);
 
   private static native void setAttrStringList(long handle, String name, Object[] value);
-
 }
diff --git a/tensorflow/java/src/main/java/org/tensorflow/Output.java b/tensorflow/java/src/main/java/org/tensorflow/Output.java
index 8dff50fafb..0e17a722ff 100644
--- a/tensorflow/java/src/main/java/org/tensorflow/Output.java
+++ b/tensorflow/java/src/main/java/org/tensorflow/Output.java
@@ -20,13 +20,13 @@ import java.util.Objects;
 /**
  * A symbolic handle to a tensor produced by an {@link Operation}.
  *
- * <p>An Output is a symbolic handle to a tensor. The value of the Tensor is computed by executing
- * the {@link Operation} in a {@link Session}.
+ * <p>An Output<T> is a symbolic handle to a Tensor<T>. The value of the tensor is computed by
+ * executing the {@link Operation} in a {@link Session}.
  *
  * <p>By implementing the {@link Operand} interface, instances of this class also act as operands to
  * {@link org.tensorflow.op.Op Op} instances.
  */
-public final class Output implements Operand {
+public final class Output<T> implements Operand<T> {
 
   /** Handle to the idx-th output of the Operation {@code op}. */
   public Output(Operation op, int idx) {
@@ -55,7 +55,7 @@ public final class Output implements Operand {
   }
 
   @Override
-  public Output asOutput() {
+  public Output<T> asOutput() {
     return this;
   }
 
@@ -69,8 +69,8 @@ public final class Output implements Operand {
     if (o == this) {
       return true;
     }
-    if (o instanceof Output) {
-      Output that = (Output) o;
+    if (o instanceof Output<?>) {
+      Output<?> that = (Output<?>) o;
       return index == that.index && operation.equals(that.operation);
     }
     return false;
diff --git a/tensorflow/java/src/main/java/org/tensorflow/SavedModelBundle.java b/tensorflow/java/src/main/java/org/tensorflow/SavedModelBundle.java
index b4591dd869..c8b9126f03 100644
--- a/tensorflow/java/src/main/java/org/tensorflow/SavedModelBundle.java
+++ b/tensorflow/java/src/main/java/org/tensorflow/SavedModelBundle.java
@@ -27,8 +27,9 @@ package org.tensorflow;
 public class SavedModelBundle implements AutoCloseable {
 
   /**
-   * Load a saved model from an export directory. The model that is being loaded should be created using
-   * the <a href="https://www.tensorflow.org/api_docs/python/tf/saved_model">Saved Model API</a>.
+   * Load a saved model from an export directory. The model that is being loaded should be created
+   * using the <a href="https://www.tensorflow.org/api_docs/python/tf/saved_model">Saved Model
+   * API</a>.
    *
    * @param exportDir the directory path containing a saved model.
    * @param tags the tags identifying the specific metagraphdef to load.
diff --git a/tensorflow/java/src/main/java/org/tensorflow/Session.java b/tensorflow/java/src/main/java/org/tensorflow/Session.java
index 83a300a560..73324f23e6 100644
--- a/tensorflow/java/src/main/java/org/tensorflow/Session.java
+++ b/tensorflow/java/src/main/java/org/tensorflow/Session.java
@@ -127,7 +127,7 @@ public final class Session implements AutoCloseable {
      *     {@code SignatureDef} protocol buffer messages that are included in {@link
      *     SavedModelBundle#metaGraphDef()}.
      */
-    public Runner feed(String operation, Tensor t) {
+    public Runner feed(String operation, Tensor<?> t) {
       return feed(parseOutput(operation), t);
     }
 
@@ -138,7 +138,7 @@ public final class Session implements AutoCloseable {
      * <p>Operations in a {@link Graph} can have multiple outputs, {@code index} identifies which
      * one {@code t} is being provided for.
      */
-    public Runner feed(String operation, int index, Tensor t) {
+    public Runner feed(String operation, int index, Tensor<?> t) {
       Operation op = operationByName(operation);
       if (op != null) {
         inputs.add(op.output(index));
@@ -151,7 +151,7 @@ public final class Session implements AutoCloseable {
      * Use {@code t} instead of the Tensor referred to by executing the operation referred to by
      * {@code output}.
      */
-    public Runner feed(Output o, Tensor t) {
+    public Runner feed(Output<?> o, Tensor<?> t) {
       inputs.add(o);
       inputTensors.add(t);
       return this;
@@ -186,7 +186,7 @@ public final class Session implements AutoCloseable {
     }
 
     /** Makes {@link #run()} return the Tensor referred to by {@code output}. */
-    public Runner fetch(Output output) {
+    public Runner fetch(Output<?> output) {
       outputs.add(output);
       return this;
     }
@@ -240,8 +240,11 @@ public final class Session implements AutoCloseable {
      * easier for the caller to cleanup (perhaps returning something like AutoCloseableList in
      * SessionTest.java), and (b) Evaluate whether the return value should be a list, or maybe a
      * {@code Map<Output, Tensor>}?
+     *
+     * <p>TODO(andrewmyers): It would also be good if whatever is returned here made it easier to
+     * extract output tensors in a type-safe way.
      */
-    public List<Tensor> run() {
+    public List<Tensor<?>> run() {
       return runHelper(false).outputs;
     }
 
@@ -269,17 +272,17 @@ public final class Session implements AutoCloseable {
       // It's okay to use Operation.getUnsafeNativeHandle() here since the safety depends on the
       // validity of the Graph and graphRef ensures that.
       int idx = 0;
-      for (Tensor t : inputTensors) {
+      for (Tensor<?> t : inputTensors) {
         inputTensorHandles[idx++] = t.getNativeHandle();
       }
       idx = 0;
-      for (Output o : inputs) {
+      for (Output<?> o : inputs) {
         inputOpHandles[idx] = o.op().getUnsafeNativeHandle();
         inputOpIndices[idx] = o.index();
         idx++;
       }
       idx = 0;
-      for (Output o : outputs) {
+      for (Output<?> o : outputs) {
         outputOpHandles[idx] = o.op().getUnsafeNativeHandle();
         outputOpIndices[idx] = o.index();
         idx++;
@@ -306,12 +309,12 @@ public final class Session implements AutoCloseable {
       } finally {
         runRef.close();
       }
-      List<Tensor> outputs = new ArrayList<Tensor>();
+      List<Tensor<?>> outputs = new ArrayList<Tensor<?>>();
       for (long h : outputTensorHandles) {
         try {
           outputs.add(Tensor.fromHandle(h));
         } catch (Exception e) {
-          for (Tensor t : outputs) {
+          for (Tensor<?> t : outputs) {
             t.close();
           }
           outputs.clear();
@@ -355,7 +358,8 @@ public final class Session implements AutoCloseable {
       return op;
     }
 
-    private Output parseOutput(String opName) {
+    @SuppressWarnings("rawtypes")
+    private Output<?> parseOutput(String opName) {
       int colon = opName.lastIndexOf(':');
       if (colon == -1 || colon == opName.length() - 1) {
         return new Output(operationByName(opName), 0);
@@ -369,9 +373,9 @@ public final class Session implements AutoCloseable {
       }
     }
 
-    private ArrayList<Output> inputs = new ArrayList<Output>();
-    private ArrayList<Tensor> inputTensors = new ArrayList<Tensor>();
-    private ArrayList<Output> outputs = new ArrayList<Output>();
+    private ArrayList<Output<?>> inputs = new ArrayList<Output<?>>();
+    private ArrayList<Tensor<?>> inputTensors = new ArrayList<Tensor<?>>();
+    private ArrayList<Output<?>> outputs = new ArrayList<Output<?>>();
     private ArrayList<Operation> targets = new ArrayList<Operation>();
     private byte[] runOptions = null;
   }
@@ -388,7 +392,7 @@ public final class Session implements AutoCloseable {
    */
   public static final class Run {
     /** Tensors from requested fetches. */
-    public List<Tensor> outputs;
+    public List<Tensor<?>> outputs;
 
     /**
      * (Experimental): Metadata about the run.
diff --git a/tensorflow/java/src/main/java/org/tensorflow/Tensor.java b/tensorflow/java/src/main/java/org/tensorflow/Tensor.java
index c5ad1ee51c..d4b753628b 100644
--- a/tensorflow/java/src/main/java/org/tensorflow/Tensor.java
+++ b/tensorflow/java/src/main/java/org/tensorflow/Tensor.java
@@ -28,89 +28,117 @@ import java.util.Arrays;
 import java.util.HashMap;
 
 /**
- * A typed multi-dimensional array.
+ * A statically typed multi-dimensional array whose elements are of a type described by T.
  *
  * <p>Instances of a Tensor are <b>not</b> thread-safe.
  *
  * <p><b>WARNING:</b> Resources consumed by the Tensor object <b>must</b> be explicitly freed by
  * invoking the {@link #close()} method when the object is no longer needed. For example, using a
- * try-with-resources block like:
+ * try-with-resources block:
  *
  * <pre>{@code
- * try(Tensor t = Tensor.create(...)) {
+ * try (Tensor t = Tensor.create(...)) {
  *   doSomethingWith(t);
  * }
  * }</pre>
  */
-public final class Tensor implements AutoCloseable {
+public final class Tensor<T> implements AutoCloseable {
 
   /**
-   * Create a Tensor from a Java object.
+   * Creates a Tensor from a Java object.
    *
-   * <p>A Tensor is a multi-dimensional array of elements of a limited set of types ({@link
-   * DataType}). Thus, not all Java objects can be converted to a Tensor. In particular, {@code obj}
-   * must be either a primitive (float, double, int, long, boolean) or a multi-dimensional array of
-   * one of those primitives. For example:
+   * <p>A {@code Tensor} is a multi-dimensional array of elements of a limited set of types ({@link
+   * types}), so not all Java objects can be converted to a {@code Tensor}. In particular, the
+   * argument {@code obj} must be either a primitive (float, double, int, long, boolean, byte) or a
+   * multi-dimensional array of one of those primitives. The argument {@code type} specifies how to
+   * interpret the first argument as a TensorFlow type. For example:
    *
    * <pre>{@code
    * // Valid: A 64-bit integer scalar.
-   * Tensor s = Tensor.create(42L);
+   * Tensor<Long> s = Tensor.create(42L, Long.class);
    *
    * // Valid: A 3x2 matrix of floats.
    * float[][] matrix = new float[3][2];
-   * Tensor m = Tensor.create(matrix);
+   * Tensor<Float> m = Tensor.create(matrix, Float.class);
    *
    * // Invalid: Will throw an IllegalArgumentException as an arbitrary Object
    * // does not fit into the TensorFlow type system.
-   * Tensor o = Tensor.create(new Object());
+   * Tensor<?> o = Tensor.create(new Object())
    *
    * // Invalid: Will throw an IllegalArgumentException since there are
    * // a differing number of elements in each row of this 2-D array.
    * int[][] twoD = new int[2][];
    * twoD[0] = new int[1];
    * twoD[1] = new int[2];
-   * Tensor x = Tensor.create(twoD);
+   * Tensor<Integer> x = Tensor.create(twoD, Integer.class);
    * }</pre>
    *
-   * {@link DataType#STRING} typed Tensors are multi-dimensionary arrays of arbitrary byte sequences
-   * and thus have {@code byte[]} and not {@code String}-valued elements. For example:
+   * {@link String}-typed Tensors are multi-dimensional arrays of arbitrary byte sequences, so can
+   * be initialized from arrays of {@code byte[]} elements. For example:
    *
    * <pre>{@code
-   * // Valid: A DataType.STRING tensor.
-   * Tensor s = Tensor.create(new byte[]{1, 2, 3});
+   * // Valid: A String tensor.
+   * Tensor<String> s = Tensor.create(new byte[]{1, 2, 3}, String.class);
    *
    * // Java Strings will need to be encoded into a byte-sequence.
    * String mystring = "foo";
-   * Tensor s = Tensor.create(mystring.getBytes("UTF-8"));
+   * Tensor<String> s = Tensor.create(mystring.getBytes("UTF-8"), String.class);
    *
-   * // Valid: Matrix of DataType.STRING tensors.
+   * // Valid: Matrix of String tensors.
    * // Each element might have a different length.
    * byte[][][] matrix = new byte[2][2][];
    * matrix[0][0] = "this".getBytes("UTF-8");
    * matrix[0][1] = "is".getBytes("UTF-8");
    * matrix[1][0] = "a".getBytes("UTF-8");
    * matrix[1][1] = "matrix".getBytes("UTF-8");
-   * Tensor m = Tensor.create(matrix);
+   * Tensor<String> m = Tensor.create(matrix, String.class);
    * }</pre>
    *
+   * @param obj The object to convert to a Tensor<T>. Note that whether it is compatible with the
+   *     type T is not checked by the type system. For type-safe creation of tensors, use {@link
+   *     Tensors}.
+   * @param type The class object representing the type T.
    * @throws IllegalArgumentException if {@code obj} is not compatible with the TensorFlow type
-   *     system, or if obj does not disambiguate between multiple DataTypes. In that case, consider
-   *     using {@link #create(DataType, long[], ByteBuffer)} instead.
+   *     system.
    */
-  public static Tensor create(Object obj) {
+  @SuppressWarnings("unchecked")
+  public static <T> Tensor<T> create(Object obj, Class<T> type) {
+    DataType dtype = DataType.fromClass(type);
+    if (!objectCompatWithType(obj, dtype)) {
+      throw new IllegalArgumentException(
+          "DataType of object does not match T (expected "
+              + dtype
+              + ", got "
+              + dataTypeOf(obj)
+              + ")");
+    }
+    return (Tensor<T>) create(obj, dtype);
+  }
+
+  /**
+   * Creates a tensor from an object whose class is inspected to figure out what the underlying data
+   * type should be.
+   *
+   * @throws IllegalArgumentException if {@code obj} is not compatible with the TensorFlow type
+   *     system.
+   */
+  public static Tensor<?> create(Object obj) {
     return create(obj, dataTypeOf(obj));
   }
 
   /**
-   * Create a Tensor of data type {@code dtype} from a Java object.
+   * Create a Tensor of data type {@code dtype} from a Java object. Requires the parameter {@code T}
+   * to match {@code type}, but this condition is not checked.
    *
-   * @param dtype the intended tensor data type. It must match the the run-time type of the object.
+   * @param obj the object supplying the tensor data.
+   * @param dtype the data type of the tensor to create. It must be compatible with the run-time
+   *     type of the object.
+   * @return the new tensor
    */
-  static Tensor create(Object obj, DataType dtype) {
-    Tensor t = new Tensor();
-    t.dtype = dtype;
+  private static Tensor<?> create(Object obj, DataType dtype) {
+    @SuppressWarnings("rawtypes")
+    Tensor<?> t = new Tensor(dtype);
     t.shapeCopy = new long[numDimensions(obj, dtype)];
-    assert objectCompatWithType(obj, dtype);
     fillShape(obj, 0, t.shapeCopy);
     if (t.dtype != DataType.STRING) {
       int byteSize = elemByteSize(t.dtype) * numElements(t.shapeCopy);
@@ -125,7 +153,7 @@ public final class Tensor implements AutoCloseable {
   }
 
   /**
-   * Create an {@link DataType#INT32} Tensor with data from the given buffer.
+   * Create a {@link Integer} Tensor with data from the given buffer.
    *
    * <p>Creates a Tensor with the given shape by copying elements from the buffer (starting from its
    * current position) into the tensor. For example, if {@code shape = {2,3} } (which represents a
@@ -136,14 +164,14 @@ public final class Tensor implements AutoCloseable {
    * @param data a buffer containing the tensor data.
    * @throws IllegalArgumentException If the tensor shape is not compatible with the buffer
    */
-  public static Tensor create(long[] shape, IntBuffer data) {
-    Tensor t = allocateForBuffer(DataType.INT32, shape, data.remaining());
+  public static Tensor<Integer> create(long[] shape, IntBuffer data) {
+    Tensor<Integer> t = allocateForBuffer(DataType.INT32, shape, data.remaining());
     t.buffer().asIntBuffer().put(data);
     return t;
   }
 
   /**
-   * Create a {@link DataType#FLOAT} Tensor with data from the given buffer.
+   * Create a {@link Float} Tensor with data from the given buffer.
    *
    * <p>Creates a Tensor with the given shape by copying elements from the buffer (starting from its
    * current position) into the tensor. For example, if {@code shape = {2,3} } (which represents a
@@ -154,14 +182,14 @@ public final class Tensor implements AutoCloseable {
    * @param data a buffer containing the tensor data.
    * @throws IllegalArgumentException If the tensor shape is not compatible with the buffer
    */
-  public static Tensor create(long[] shape, FloatBuffer data) {
-    Tensor t = allocateForBuffer(DataType.FLOAT, shape, data.remaining());
+  public static Tensor<Float> create(long[] shape, FloatBuffer data) {
+    Tensor<Float> t = allocateForBuffer(DataType.FLOAT, shape, data.remaining());
     t.buffer().asFloatBuffer().put(data);
     return t;
   }
 
   /**
-   * Create a {@link DataType#DOUBLE} Tensor with data from the given buffer.
+   * Create a {@link Double} Tensor with data from the given buffer.
    *
    * <p>Creates a Tensor with the given shape by copying elements from the buffer (starting from its
    * current position) into the tensor. For example, if {@code shape = {2,3} } (which represents a
@@ -172,14 +200,14 @@ public final class Tensor implements AutoCloseable {
    * @param data a buffer containing the tensor data.
    * @throws IllegalArgumentException If the tensor shape is not compatible with the buffer
    */
-  public static Tensor create(long[] shape, DoubleBuffer data) {
-    Tensor t = allocateForBuffer(DataType.DOUBLE, shape, data.remaining());
+  public static Tensor<Double> create(long[] shape, DoubleBuffer data) {
+    Tensor<Double> t = allocateForBuffer(DataType.DOUBLE, shape, data.remaining());
     t.buffer().asDoubleBuffer().put(data);
     return t;
   }
 
   /**
-   * Create an {@link DataType#INT64} Tensor with data from the given buffer.
+   * Create an {@link Long} Tensor with data from the given buffer.
    *
    * <p>Creates a Tensor with the given shape by copying elements from the buffer (starting from its
    * current position) into the tensor. For example, if {@code shape = {2,3} } (which represents a
@@ -190,47 +218,87 @@ public final class Tensor implements AutoCloseable {
    * @param data a buffer containing the tensor data.
    * @throws IllegalArgumentException If the tensor shape is not compatible with the buffer
    */
-  public static Tensor create(long[] shape, LongBuffer data) {
-    Tensor t = allocateForBuffer(DataType.INT64, shape, data.remaining());
+  public static Tensor<Long> create(long[] shape, LongBuffer data) {
+    Tensor<Long> t = allocateForBuffer(DataType.INT64, shape, data.remaining());
     t.buffer().asLongBuffer().put(data);
     return t;
   }
 
   /**
-   * Create a Tensor with data from the given buffer.
+   * Create a Tensor of any type with data from the given buffer.
+   *
+   * <p>Creates a Tensor with the provided shape of any type where the tensor's data has been
+   * encoded into {@code data} as per the specification of the TensorFlow <a
+   * href="https://www.tensorflow.org/code/tensorflow/c/c_api.h">C API</a>.
+   *
+   * @param <T> the tensor element type
+   * @param type the tensor element type, represented as a class object.
+   * @param shape the tensor shape.
+   * @param data a buffer containing the tensor data.
+   * @throws IllegalArgumentException If the tensor datatype or shape is not compatible with the
+   *     buffer
+   */
+  public static <T> Tensor<T> create(Class<T> type, long[] shape, ByteBuffer data) {
+    @SuppressWarnings("unchecked")
+    Tensor<T> ret = (Tensor<T>) create(DataType.fromClass(type), shape, data);
+    return ret;
+  }
+
+  /**
+   * Creates a Tensor of any type with data from the given buffer.
    *
    * <p>Creates a Tensor with the provided shape of any type where the tensor's data has been
    * encoded into {@code data} as per the specification of the TensorFlow <a
    * href="https://www.tensorflow.org/code/tensorflow/c/c_api.h">C API</a>.
    *
-   * @param dataType the tensor datatype.
+   * @param <T> The tensor element type
+   * @param type the tensor element type, specified as a DataType. This must agree with T.
    * @param shape the tensor shape.
    * @param data a buffer containing the tensor data.
    * @throws IllegalArgumentException If the tensor datatype or shape is not compatible with the
    *     buffer
    */
-  public static Tensor create(DataType dataType, long[] shape, ByteBuffer data) {
+  private static Tensor<?> create(DataType dtype, long[] shape, ByteBuffer data) {
     int nremaining = 0;
-    if (dataType != DataType.STRING) {
-      int elemBytes = elemByteSize(dataType);
+    if (dtype != DataType.STRING) {
+      int elemBytes = elemByteSize(dtype);
       if (data.remaining() % elemBytes != 0) {
         throw new IllegalArgumentException(
             String.format(
                 "ByteBuffer with %d bytes is not compatible with a %s Tensor (%d bytes/element)",
-                data.remaining(), dataType.toString(), elemBytes));
+                data.remaining(), dtype.toString(), elemBytes));
       }
       nremaining = data.remaining() / elemBytes;
     } else {
       nremaining = data.remaining();
     }
-    Tensor t = allocateForBuffer(dataType, shape, nremaining);
+    Tensor<?> t = allocateForBuffer(dtype, shape, nremaining);
     t.buffer().put(data);
     return t;
   }
 
+  /**
+   * Returns this Tensor object with the type {@code Tensor<U>}. This method is useful when given a
+   * value of type {@code Tensor<?>}.
+   *
+   * @param type any (non-null) array of the correct type.
+   * @throws IllegalArgumentException if the actual data type of this object does not match the type
+   *     {@code U}.
+   */
+  @SuppressWarnings("unchecked")
+  public <U> Tensor<U> expect(Class<U> type) {
+    DataType dt = DataType.fromClass(type);
+    if (!dt.equals(dtype)) {
+      throw new IllegalArgumentException(
+          "Cannot cast from tensor of " + dtype + " to tensor of " + dt);
+    }
+    return ((Tensor<U>) this);
+  }
+
   // Helper function to allocate a Tensor for the create() methods that create a Tensor from
   // a java.nio.Buffer.
-  private static Tensor allocateForBuffer(DataType dataType, long[] shape, int nBuffered) {
+  // Requires: dataType matches T
+  private static <T> Tensor<T> allocateForBuffer(DataType dataType, long[] shape, int nBuffered) {
     final int nflattened = numElements(shape);
     int nbytes = 0;
     if (dataType != DataType.STRING) {
@@ -242,8 +310,7 @@ public final class Tensor implements AutoCloseable {
       // DT_STRING tensor encoded in a ByteBuffer.
       nbytes = nBuffered;
     }
-    Tensor t = new Tensor();
-    t.dtype = dataType;
+    Tensor<T> t = new Tensor<T>(dataType);
     t.shapeCopy = Arrays.copyOf(shape, shape.length);
     t.nativeHandle = allocate(t.dtype.c(), t.shapeCopy, nbytes);
     return t;
@@ -300,7 +367,7 @@ public final class Tensor implements AutoCloseable {
   }
 
   /**
-   * Returns the value in a scalar {@link DataType#FLOAT} tensor.
+   * Returns the value in a scalar {@link Float} tensor.
    *
    * @throws IllegalArgumentException if the Tensor does not represent a float scalar.
    */
@@ -309,7 +376,7 @@ public final class Tensor implements AutoCloseable {
   }
 
   /**
-   * Returns the value in a scalar {@link DataType#DOUBLE} tensor.
+   * Returns the value in a scalar {@link Double} tensor.
    *
    * @throws IllegalArgumentException if the Tensor does not represent a double scalar.
    */
@@ -318,7 +385,7 @@ public final class Tensor implements AutoCloseable {
   }
 
   /**
-   * Returns the value in a scalar {@link DataType#INT32} tensor.
+   * Returns the value in a scalar {@link Integer} tensor.
    *
    * @throws IllegalArgumentException if the Tensor does not represent a int scalar.
    */
@@ -327,7 +394,7 @@ public final class Tensor implements AutoCloseable {
   }
 
   /**
-   * Returns the value in a scalar {@link DataType#INT64} tensor.
+   * Returns the value in a scalar {@link Long} tensor.
    *
    * @throws IllegalArgumentException if the Tensor does not represent a long scalar.
    */
@@ -336,7 +403,7 @@ public final class Tensor implements AutoCloseable {
   }
 
   /**
-   * Returns the value in a scalar {@link DataType#BOOL} tensor.
+   * Returns the value in a scalar {@link Boolean} tensor.
    *
    * @throws IllegalArgumentException if the Tensor does not represent a boolean scalar.
    */
@@ -345,7 +412,7 @@ public final class Tensor implements AutoCloseable {
   }
 
   /**
-   * Returns the value in a scalar {@link DataType#STRING} tensor.
+   * Returns the value in a scalar {@link String} tensor.
    *
    * @throws IllegalArgumentException if the Tensor does not represent a boolean scalar.
    */
@@ -377,21 +444,21 @@ public final class Tensor implements AutoCloseable {
    * @throws IllegalArgumentException if the tensor is a scalar or if {@code dst} is not compatible
    *     with the tensor (for example, mismatched data types or shapes).
    */
-  public <T> T copyTo(T dst) {
+  public <U> U copyTo(U dst) {
     throwExceptionIfTypeIsIncompatible(dst);
     readNDArray(nativeHandle, dst);
     return dst;
   }
 
   /**
-   * Write the data of a {@link DataType#INT32} tensor into the given buffer.
+   * Write the data of a {@link Integer} tensor into the given buffer.
    *
    * <p>Copies {@code numElements()} elements to the buffer.
    *
    * @param dst the destination buffer
    * @throws BufferOverflowException If there is insufficient space in the given buffer for the data
    *     in this tensor
-   * @throws IllegalArgumentException If the tensor datatype is not {@link DataType#INT32}
+   * @throws IllegalArgumentException If the tensor data type is not {@link Integer}
    */
   public void writeTo(IntBuffer dst) {
     if (dtype != DataType.INT32) {
@@ -402,14 +469,14 @@ public final class Tensor implements AutoCloseable {
   }
 
   /**
-   * Write the data of a {@link DataType#FLOAT} tensor into the given buffer.
+   * Write the data of a {@link Float} tensor into the given buffer.
    *
    * <p>Copies {@code numElements()} elements to the buffer.
    *
    * @param dst the destination buffer
    * @throws BufferOverflowException If there is insufficient space in the given buffer for the data
    *     in this tensor
-   * @throws IllegalArgumentException If the tensor datatype is not {@link DataType#FLOAT}
+   * @throws IllegalArgumentException If the tensor datatype is not {@link Float}
    */
   public void writeTo(FloatBuffer dst) {
     if (dtype != DataType.FLOAT) {
@@ -420,14 +487,14 @@ public final class Tensor implements AutoCloseable {
   }
 
   /**
-   * Write the data of a {@link DataType#DOUBLE} tensor into the given buffer.
+   * Write the data of a {@link Double} tensor into the given buffer.
    *
    * <p>Copies {@code numElements()} elements to the buffer.
    *
    * @param dst the destination buffer
    * @throws BufferOverflowException If there is insufficient space in the given buffer for the data
    *     in this tensor
-   * @throws IllegalArgumentException If the tensor datatype is not {@link DataType#DOUBLE}
+   * @throws IllegalArgumentException If the tensor datatype is not {@link Double}
    */
   public void writeTo(DoubleBuffer dst) {
     if (dtype != DataType.DOUBLE) {
@@ -438,14 +505,14 @@ public final class Tensor implements AutoCloseable {
   }
 
   /**
-   * Write the data of a {@link DataType#INT64} tensor into the given buffer.
+   * Write the data of a {@link Long} tensor into the given buffer.
    *
    * <p>Copies {@code numElements()} elements to the buffer.
    *
    * @param dst the destination buffer
    * @throws BufferOverflowException If there is insufficient space in the given buffer for the data
    *     in this tensor
-   * @throws IllegalArgumentException If the tensor datatype is not {@link DataType#INT64}
+   * @throws IllegalArgumentException If the tensor datatype is not {@link Long}
    */
   public void writeTo(LongBuffer dst) {
     if (dtype != DataType.INT64) {
@@ -480,9 +547,9 @@ public final class Tensor implements AutoCloseable {
    *
    * <p>Takes ownership of the handle.
    */
-  static Tensor fromHandle(long handle) {
-    Tensor t = new Tensor();
-    t.dtype = DataType.fromC(dtype(handle));
+  static Tensor<?> fromHandle(long handle) {
+    @SuppressWarnings("rawtypes")
+    Tensor<?> t = new Tensor(DataType.fromC(dtype(handle)));
     t.shapeCopy = shape(handle);
     t.nativeHandle = handle;
     return t;
@@ -496,7 +563,9 @@ public final class Tensor implements AutoCloseable {
   private DataType dtype;
   private long[] shapeCopy = null;
 
-  private Tensor() {}
+  private Tensor(DataType t) {
+    dtype = t;
+  }
 
   private ByteBuffer buffer() {
     return buffer(nativeHandle).order(ByteOrder.nativeOrder());
@@ -564,11 +633,26 @@ public final class Tensor implements AutoCloseable {
     classDataTypes.put(Boolean.class, DataType.BOOL);
   }
 
-  private static DataType dataTypeOf(Object o) {
+  /** The class for the data type to which Java object o corresponds. */
+  private static Class<?> baseObjType(Object o) {
     Class<?> c = o.getClass();
     while (c.isArray()) {
       c = c.getComponentType();
     }
+    return c;
+  }
+
+  /**
+   * The default TensorFlow data type to which Java object o corresponds. Some Java objects
+   * represent more than one TensorFlow data type; for example, 'byte' can represent both {@code
+   * uint8} and {@code string}, with the latter being the default interpretation.
+   */
+  private static DataType dataTypeOf(Object o) {
+    Class<?> c = baseObjType(o);
+    return dataTypeFromClass(c);
+  }
+
+  private static DataType dataTypeFromClass(Class<?> c) {
     DataType ret = classDataTypes.get(c);
     if (ret != null) {
       return ret;
@@ -577,7 +661,12 @@ public final class Tensor implements AutoCloseable {
   }
 
   /**
-   * Returns the number of dimensions of a tensor of type dtype when represented by the object o.
+   * Return the number of dimensions of the tensor that object {@code o} represents as a tensor
+   * whose datatype is {@code dtype}. Normally this is the same as the number of dimensions of o
+   * itself, but is one smaller for tensors of strings.
+   *
+   * @param o The object to inspect. It must be a valid representation of the given data type.
+   * @param dtype The expected data type of the tensor.
    */
   private static int numDimensions(Object o, DataType dtype) {
     int ret = numArrayDimensions(o);
@@ -624,7 +713,13 @@ public final class Tensor implements AutoCloseable {
 
   /** Returns whether the object {@code obj} can represent a tensor with data type {@code dtype}. */
   private static boolean objectCompatWithType(Object obj, DataType dtype) {
-    DataType dto = dataTypeOf(obj);
+    Class<?> c = baseObjType(obj);
+    DataType dto = dataTypeFromClass(c);
+    int nd = numDimensions(obj, dto);
+    if (!c.isPrimitive() && c != String.class && nd != 0) {
+      throw new IllegalArgumentException(
+          "cannot create non-scalar Tensors from arrays of boxed values");
+    }
     if (dto.equals(dtype)) {
       return true;
     }
diff --git a/tensorflow/java/src/main/java/org/tensorflow/Tensors.java b/tensorflow/java/src/main/java/org/tensorflow/Tensors.java
new file mode 100644
index 0000000000..c828d23efc
--- /dev/null
+++ b/tensorflow/java/src/main/java/org/tensorflow/Tensors.java
@@ -0,0 +1,447 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+package org.tensorflow;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+/** Type-safe factory methods for creating {@link org.tensorflow.Tensor} objects. */
+public final class Tensors {
+  private Tensors() {}
+
+  /**
+   * Creates a scalar String tensor using the default, UTF-8 encoding.
+   *
+   * @param data The string to put into the new scalar tensor.
+   */
+  public static Tensor<String> create(String data) {
+    return Tensor.create(data.getBytes(UTF_8), String.class);
+  }
+
+  /**
+   * Creates a scalar String tensor using a specified encoding.
+   *
+   * @param charset The encoding from String to bytes.
+   * @param data The string to put into the new scalar tensor.
+   */
+  public static Tensor<String> create(String data, java.nio.charset.Charset charset) {
+    return Tensor.create(data.getBytes(charset), String.class);
+  }
+
+  /**
+   * Creates a scalar tensor containing a single {@code float} element.
+   *
+   * @param data The value to put into the new scalar tensor.
+   */
+  public static Tensor<Float> create(float data) {
+    return Tensor.create(data, Float.class);
+  }
+
+  /**
+   * Creates a rank-1 tensor of {@code float} elements.
+   *
+   * @param data An array containing the values to put into the new tensor. The dimensions of the
+   *     new tensor will match those of the array.
+   */
+  public static Tensor<Float> create(float[] data) {
+    return Tensor.create(data, Float.class);
+  }
+
+  /**
+   * Creates a rank-2 tensor of {@code float} elements.
+   *
+   * @param data An array containing the values to put into the new tensor. The dimensions of the
+   *     new tensor will match those of the array.
+   */
+  public static Tensor<Float> create(float[][] data) {
+    return Tensor.create(data, Float.class);
+  }
+
+  /**
+   * Creates a rank-3 tensor of {@code float} elements.
+   *
+   * @param data An array containing the values to put into the new tensor. The dimensions of the
+   *     new tensor will match those of the array.
+   */
+  public static Tensor<Float> create(float[][][] data) {
+    return Tensor.create(data, Float.class);
+  }
+
+  /**
+   * Creates a rank-4 tensor of {@code float} elements.
+   *
+   * @param data An array containing the values to put into the new tensor. The dimensions of the
+   *     new tensor will match those of the array.
+   */
+  public static Tensor<Float> create(float[][][][] data) {
+    return Tensor.create(data, Float.class);
+  }
+
+  /**
+   * Creates a rank-5 tensor of {@code float} elements.
+   *
+   * @param data An array containing the values to put into the new tensor. The dimensions of the
+   *     new tensor will match those of the array.
+   */
+  public static Tensor<Float> create(float[][][][][] data) {
+    return Tensor.create(data, Float.class);
+  }
+
+  /**
+   * Creates a rank-6 tensor of {@code float} elements.
+   *
+   * @param data An array containing the values to put into the new tensor. The dimensions of the
+   *     new tensor will match those of the array.
+   */
+  public static Tensor<Float> create(float[][][][][][] data) {
+    return Tensor.create(data, Float.class);
+  }
+
+  /**
+   * Creates a scalar tensor containing a single {@code double} element.
+   *
+   * @param data The value to put into the new scalar tensor.
+   */
+  public static Tensor<Double> create(double data) {
+    return Tensor.create(data, Double.class);
+  }
+
+  /**
+   * Creates a rank-1 tensor of {@code double} elements.
+   *
+   * @param data An array containing the values to put into the new tensor. The dimensions of the
+   *     new tensor will match those of the array.
+   */
+  public static Tensor<Double> create(double[] data) {
+    return Tensor.create(data, Double.class);
+  }
+
+  /**
+   * Creates a rank-2 tensor of {@code double} elements.
+   *
+   * @param data An array containing the values to put into the new tensor. The dimensions of the
+   *     new tensor will match those of the array.
+   */
+  public static Tensor<Double> create(double[][] data) {
+    return Tensor.create(data, Double.class);
+  }
+
+  /**
+   * Creates a rank-3 tensor of {@code double} elements.
+   *
+   * @param data An array containing the values to put into the new tensor. The dimensions of the
+   *     new tensor will match those of the array.
+   */
+  public static Tensor<Double> create(double[][][] data) {
+    return Tensor.create(data, Double.class);
+  }
+
+  /**
+   * Creates a rank-4 tensor of {@code double} elements.
+   *
+   * @param data An array containing the values to put into the new tensor. The dimensions of the
+   *     new tensor will match those of the array.
+   */
+  public static Tensor<Double> create(double[][][][] data) {
+    return Tensor.create(data, Double.class);
+  }
+
+  /**
+   * Creates a rank-5 tensor of {@code double} elements.
+   *
+   * @param data An array containing the values to put into the new tensor. The dimensions of the
+   *     new tensor will match those of the array.
+   */
+  public static Tensor<Double> create(double[][][][][] data) {
+    return Tensor.create(data, Double.class);
+  }
+
+  /**
+   * Creates a rank-6 tensor of {@code double} elements.
+   *
+   * @param data An array containing the values to put into the new tensor. The dimensions of the
+   *     new tensor will match those of the array.
+   */
+  public static Tensor<Double> create(double[][][][][][] data) {
+    return Tensor.create(data, Double.class);
+  }
+
+  /**
+   * Creates a scalar tensor containing a single {@code int} element.
+   *
+   * @param data The value to put into the new scalar tensor.
+   */
+  public static Tensor<Integer> create(int data) {
+    return Tensor.create(data, Integer.class);
+  }
+
+  /**
+   * Creates a rank-1 tensor of {@code int} elements.
+   *
+   * @param data An array containing the values to put into the new tensor. The dimensions of the
+   *     new tensor will match those of the array.
+   */
+  public static Tensor<Integer> create(int[] data) {
+    return Tensor.create(data, Integer.class);
+  }
+
+  /**
+   * Creates a rank-2 tensor of {@code int} elements.
+   *
+   * @param data An array containing the values to put into the new tensor. The dimensions of the
+   *     new tensor will match those of the array.
+   */
+  public static Tensor<Integer> create(int[][] data) {
+    return Tensor.create(data, Integer.class);
+  }
+
+  /**
+   * Creates a rank-3 tensor of {@code int} elements.
+   *
+   * @param data An array containing the values to put into the new tensor. The dimensions of the
+   *     new tensor will match those of the array.
+   */
+  public static Tensor<Integer> create(int[][][] data) {
+    return Tensor.create(data, Integer.class);
+  }
+
+  /**
+   * Creates a rank-4 tensor of {@code int} elements.
+   *
+   * @param data An array containing the values to put into the new tensor. The dimensions of the
+   *     new tensor will match those of the array.
+   */
+  public static Tensor<Integer> create(int[][][][] data) {
+    return Tensor.create(data, Integer.class);
+  }
+
+  /**
+   * Creates a rank-5 tensor of {@code int} elements.
+   *
+   * @param data An array containing the values to put into the new tensor. The dimensions of the
+   *     new tensor will match those of the array.
+   */
+  public static Tensor<Integer> create(int[][][][][] data) {
+    return Tensor.create(data, Integer.class);
+  }
+
+  /**
+   * Creates a rank-6 tensor of {@code int} elements.
+   *
+   * @param data An array containing the values to put into the new tensor. The dimensions of the
+   *     new tensor will match those of the array.
+   */
+  public static Tensor<Integer> create(int[][][][][][] data) {
+    return Tensor.create(data, Integer.class);
+  }
+
+  /**
+   * Creates a scalar tensor containing a single {@code byte} element.
+   *
+   * @param data An array containing the data to put into the new tensor. String elements are
+   *     sequences of bytes from the last array dimension.
+   */
+  public static Tensor<String> create(byte[] data) {
+    return Tensor.create(data, String.class);
+  }
+
+  /**
+   * Creates a rank-1 tensor of {@code byte} elements.
+   *
+   * @param data An array containing the data to put into the new tensor. String elements are
+   *     sequences of bytes from the last array dimension.
+   */
+  public static Tensor<String> create(byte[][] data) {
+    return Tensor.create(data, String.class);
+  }
+
+  /**
+   * Creates a rank-2 tensor of {@code byte} elements.
+   *
+   * @param data An array containing the data to put into the new tensor. String elements are
+   *     sequences of bytes from the last array dimension.
+   */
+  public static Tensor<String> create(byte[][][] data) {
+    return Tensor.create(data, String.class);
+  }
+
+  /**
+   * Creates a rank-3 tensor of {@code byte} elements.
+   *
+   * @param data An array containing the data to put into the new tensor. String elements are
+   *     sequences of bytes from the last array dimension.
+   */
+  public static Tensor<String> create(byte[][][][] data) {
+    return Tensor.create(data, String.class);
+  }
+
+  /**
+   * Creates a rank-4 tensor of {@code byte} elements.
+   *
+   * @param data An array containing the data to put into the new tensor. String elements are
+   *     sequences of bytes from the last array dimension.
+   */
+  public static Tensor<String> create(byte[][][][][] data) {
+    return Tensor.create(data, String.class);
+  }
+
+  /**
+   * Creates a rank-5 tensor of {@code byte} elements.
+   *
+   * @param data An array containing the data to put into the new tensor. String elements are
+   *     sequences of bytes from the last array dimension.
+   */
+  public static Tensor<String> create(byte[][][][][][] data) {
+    return Tensor.create(data, String.class);
+  }
+
+  /**
+   * Creates a scalar tensor containing a single {@code long} element.
+   *
+   * @param data The value to put into the new scalar tensor.
+   */
+  public static Tensor<Long> create(long data) {
+    return Tensor.create(data, Long.class);
+  }
+
+  /**
+   * Creates a rank-1 tensor of {@code long} elements.
+   *
+   * @param data An array containing the values to put into the new tensor. The dimensions of the
+   *     new tensor will match those of the array.
+   */
+  public static Tensor<Long> create(long[] data) {
+    return Tensor.create(data, Long.class);
+  }
+
+  /**
+   * Creates a rank-2 tensor of {@code long} elements.
+   *
+   * @param data An array containing the values to put into the new tensor. The dimensions of the
+   *     new tensor will match those of the array.
+   */
+  public static Tensor<Long> create(long[][] data) {
+    return Tensor.create(data, Long.class);
+  }
+
+  /**
+   * Creates a rank-3 tensor of {@code long} elements.
+   *
+   * @param data An array containing the values to put into the new tensor. The dimensions of the
+   *     new tensor will match those of the array.
+   */
+  public static Tensor<Long> create(long[][][] data) {
+    return Tensor.create(data, Long.class);
+  }
+
+  /**
+   * Creates a rank-4 tensor of {@code long} elements.
+   *
+   * @param data An array containing the values to put into the new tensor. The dimensions of the
+   *     new tensor will match those of the array.
+   */
+  public static Tensor<Long> create(long[][][][] data) {
+    return Tensor.create(data, Long.class);
+  }
+
+  /**
+   * Creates a rank-5 tensor of {@code long} elements.
+   *
+   * @param data An array containing the values to put into the new tensor. The dimensions of the
+   *     new tensor will match those of the array.
+   */
+  public static Tensor<Long> create(long[][][][][] data) {
+    return Tensor.create(data, Long.class);
+  }
+
+  /**
+   * Creates a rank-6 tensor of {@code long} elements.
+   *
+   * @param data An array containing the values to put into the new tensor. The dimensions of the
+   *     new tensor will match those of the array.
+   */
+  public static Tensor<Long> create(long[][][][][][] data) {
+    return Tensor.create(data, Long.class);
+  }
+
+  /**
+   * Creates a scalar tensor containing a single {@code boolean} element.
+   *
+   * @param data The value to put into the new scalar tensor.
+   */
+  public static Tensor<Boolean> create(boolean data) {
+    return Tensor.create(data, Boolean.class);
+  }
+
+  /**
+   * Creates a rank-1 tensor of {@code boolean} elements.
+   *
+   * @param data An array containing the values to put into the new tensor. The dimensions of the
+   *     new tensor will match those of the array.
+   */
+  public static Tensor<Boolean> create(boolean[] data) {
+    return Tensor.create(data, Boolean.class);
+  }
+
+  /**
+   * Creates a rank-2 tensor of {@code boolean} elements.
+   *
+   * @param data An array containing the values to put into the new tensor. The dimensions of the
+   *     new tensor will match those of the array.
+   */
+  public static Tensor<Boolean> create(boolean[][] data) {
+    return Tensor.create(data, Boolean.class);
+  }
+
+  /**
+   * Creates a rank-3 tensor of {@code boolean} elements.
+   *
+   * @param data An array containing the values to put into the new tensor. The dimensions of the
+   *     new tensor will match those of the array.
+   */
+  public static Tensor<Boolean> create(boolean[][][] data) {
+    return Tensor.create(data, Boolean.class);
+  }
+
+  /**
+   * Creates a rank-4 tensor of {@code boolean} elements.
+   *
+   * @param data An array containing the values to put into the new tensor. The dimensions of the
+   *     new tensor will match those of the array.
+   */
+  public static Tensor<Boolean> create(boolean[][][][] data) {
+    return Tensor.create(data, Boolean.class);
+  }
+
+  /**
+   * Creates a rank-5 tensor of {@code boolean} elements.
+   *
+   * @param data An array containing the values to put into the new tensor. The dimensions of the
+   *     new tensor will match those of the array.
+   */
+  public static Tensor<Boolean> create(boolean[][][][][] data) {
+    return Tensor.create(data, Boolean.class);
+  }
+
+  /**
+   * Creates a rank-6 tensor of {@code boolean} elements.
+   *
+   * @param data An array containing the values to put into the new tensor. The dimensions of the
+   *     new tensor will match those of the array.
+   */
+  public static Tensor<Boolean> create(boolean[][][][][][] data) {
+    return Tensor.create(data, Boolean.class);
+  }
+}
diff --git a/tensorflow/java/src/main/java/org/tensorflow/examples/LabelImage.java b/tensorflow/java/src/main/java/org/tensorflow/examples/LabelImage.java
index 19929188a5..489e95c310 100644
--- a/tensorflow/java/src/main/java/org/tensorflow/examples/LabelImage.java
+++ b/tensorflow/java/src/main/java/org/tensorflow/examples/LabelImage.java
@@ -29,6 +29,7 @@ import org.tensorflow.Output;
 import org.tensorflow.Session;
 import org.tensorflow.Tensor;
 import org.tensorflow.TensorFlow;
+import org.tensorflow.types.UInt8;
 
 /** Sample use of the TensorFlow Java API to label images using a pre-trained model. */
 public class LabelImage {
@@ -61,17 +62,17 @@ public class LabelImage {
         readAllLinesOrExit(Paths.get(modelDir, "imagenet_comp_graph_label_strings.txt"));
     byte[] imageBytes = readAllBytesOrExit(Paths.get(imageFile));
 
-    try (Tensor image = constructAndExecuteGraphToNormalizeImage(imageBytes)) {
+    try (Tensor<Float> image = constructAndExecuteGraphToNormalizeImage(imageBytes)) {
       float[] labelProbabilities = executeInceptionGraph(graphDef, image);
       int bestLabelIdx = maxIndex(labelProbabilities);
       System.out.println(
-          String.format(
-              "BEST MATCH: %s (%.2f%% likely)",
-              labels.get(bestLabelIdx), labelProbabilities[bestLabelIdx] * 100f));
+          String.format("BEST MATCH: %s (%.2f%% likely)",
+              labels.get(bestLabelIdx),
+              labelProbabilities[bestLabelIdx] * 100f));
     }
   }
 
-  private static Tensor constructAndExecuteGraphToNormalizeImage(byte[] imageBytes) {
+  private static Tensor<Float> constructAndExecuteGraphToNormalizeImage(byte[] imageBytes) {
     try (Graph g = new Graph()) {
       GraphBuilder b = new GraphBuilder(g);
       // Some constants specific to the pre-trained model at:
@@ -88,28 +89,29 @@ public class LabelImage {
       // Since the graph is being constructed once per execution here, we can use a constant for the
       // input image. If the graph were to be re-used for multiple input images, a placeholder would
       // have been more appropriate.
-      final Output input = b.constant("input", imageBytes);
-      final Output output =
+      final Output<String> input = b.constant("input", imageBytes);
+      final Output<Float> output =
           b.div(
               b.sub(
                   b.resizeBilinear(
                       b.expandDims(
-                          b.cast(b.decodeJpeg(input, 3), DataType.FLOAT),
+                          b.cast(b.decodeJpeg(input, 3), Float.class),
                           b.constant("make_batch", 0)),
                       b.constant("size", new int[] {H, W})),
                   b.constant("mean", mean)),
               b.constant("scale", scale));
       try (Session s = new Session(g)) {
-        return s.runner().fetch(output.op().name()).run().get(0);
+        return s.runner().fetch(output.op().name()).run().get(0).expect(Float.class);
       }
     }
   }
 
-  private static float[] executeInceptionGraph(byte[] graphDef, Tensor image) {
+  private static float[] executeInceptionGraph(byte[] graphDef, Tensor<Float> image) {
     try (Graph g = new Graph()) {
       g.importGraphDef(graphDef);
       try (Session s = new Session(g);
-          Tensor result = s.runner().feed("input", image).fetch("output").run().get(0)) {
+          Tensor<Float> result =
+              s.runner().feed("input", image).fetch("output").run().get(0).expect(Float.class)) {
         final long[] rshape = result.shape();
         if (result.numDimensions() != 2 || rshape[0] != 1) {
           throw new RuntimeException(
@@ -161,48 +163,71 @@ public class LabelImage {
       this.g = g;
     }
 
-    Output div(Output x, Output y) {
+    Output<Float> div(Output<Float> x, Output<Float> y) {
       return binaryOp("Div", x, y);
     }
 
-    Output sub(Output x, Output y) {
+    <T> Output<T> sub(Output<T> x, Output<T> y) {
       return binaryOp("Sub", x, y);
     }
 
-    Output resizeBilinear(Output images, Output size) {
-      return binaryOp("ResizeBilinear", images, size);
+    <T> Output<Float> resizeBilinear(Output<T> images, Output<Integer> size) {
+      return binaryOp3("ResizeBilinear", images, size);
     }
 
-    Output expandDims(Output input, Output dim) {
-      return binaryOp("ExpandDims", input, dim);
+    <T> Output<T> expandDims(Output<T> input, Output<Integer> dim) {
+      return binaryOp3("ExpandDims", input, dim);
     }
 
-    Output cast(Output value, DataType dtype) {
-      return g.opBuilder("Cast", "Cast").addInput(value).setAttr("DstT", dtype).build().output(0);
+    <T, U> Output<U> cast(Output<T> value, Class<U> type) {
+      DataType dtype = DataType.fromClass(type);
+      return g.opBuilder("Cast", "Cast")
+          .addInput(value)
+          .setAttr("DstT", dtype)
+          .build()
+          .<U>output(0);
     }
 
-    Output decodeJpeg(Output contents, long channels) {
+    Output<UInt8> decodeJpeg(Output<String> contents, long channels) {
       return g.opBuilder("DecodeJpeg", "DecodeJpeg")
           .addInput(contents)
           .setAttr("channels", channels)
           .build()
-          .output(0);
+          .<UInt8>output(0);
     }
 
-    Output constant(String name, Object value) {
-      try (Tensor t = Tensor.create(value)) {
+    <T> Output<T> constant(String name, Object value, Class<T> type) {
+      try (Tensor<T> t = Tensor.<T>create(value, type)) {
         return g.opBuilder("Const", name)
-            .setAttr("dtype", t.dataType())
+            .setAttr("dtype", DataType.fromClass(type))
             .setAttr("value", t)
             .build()
-            .output(0);
+            .<T>output(0);
       }
     }
+    Output<String> constant(String name, byte[] value) {
+      return this.constant(name, value, String.class);
+    }
 
-    private Output binaryOp(String type, Output in1, Output in2) {
-      return g.opBuilder(type, type).addInput(in1).addInput(in2).build().output(0);
+    Output<Integer> constant(String name, int value) {
+      return this.constant(name, value, Integer.class);
     }
 
+    Output<Integer> constant(String name, int[] value) {
+      return this.constant(name, value, Integer.class);
+    }
+
+    Output<Float> constant(String name, float value) {
+      return this.constant(name, value, Float.class);
+    }
+
+    private <T> Output<T> binaryOp(String type, Output<T> in1, Output<T> in2) {
+      return g.opBuilder(type, type).addInput(in1).addInput(in2).build().<T>output(0);
+    }
+
+    private <T, U, V> Output<T> binaryOp3(String type, Output<U> in1, Output<V> in2) {
+      return g.opBuilder(type, type).addInput(in1).addInput(in2).build().<T>output(0);
+    }
     private Graph g;
   }
 }
diff --git a/tensorflow/java/src/main/java/org/tensorflow/op/Operands.java b/tensorflow/java/src/main/java/org/tensorflow/op/Operands.java
index 5971103d6d..ac48da8032 100644
--- a/tensorflow/java/src/main/java/org/tensorflow/op/Operands.java
+++ b/tensorflow/java/src/main/java/org/tensorflow/op/Operands.java
@@ -33,12 +33,12 @@ public final class Operands {
    * @param inputs an iteration of input operands
    * @return an array of outputs
    */
-  public static Output[] asOutputs(Iterable<? extends Operand> inputs) {
-    List<Output> outputList = new ArrayList<>();
-    for (Operand input : inputs) {
+  public static Output<?>[] asOutputs(Iterable<? extends Operand<?>> inputs) {
+    List<Output<?>> outputList = new ArrayList<>();
+    for (Operand<?> input : inputs) {
       outputList.add(input.asOutput());
     }
-    return outputList.toArray(new Output[outputList.size()]);
+    return outputList.toArray(new Output<?>[outputList.size()]);
   }
 
   // Disabled constructor
diff --git a/tensorflow/java/src/main/java/org/tensorflow/op/core/Constant.java b/tensorflow/java/src/main/java/org/tensorflow/op/core/Constant.java
index cd7931d3bb..725c81765a 100644
--- a/tensorflow/java/src/main/java/org/tensorflow/op/core/Constant.java
+++ b/tensorflow/java/src/main/java/org/tensorflow/op/core/Constant.java
@@ -31,7 +31,7 @@ import org.tensorflow.op.annotation.Operator;
 
 /** An operator producing a constant value. */
 @Operator
-public final class Constant extends PrimitiveOp implements Operand {
+public final class Constant<T> extends PrimitiveOp implements Operand<T> {
   /**
    * Create a constant from a Java object.
    *
@@ -47,8 +47,8 @@ public final class Constant extends PrimitiveOp implements Operand {
    * @param object a Java object representing the constant.
    * @see org.tensorflow.Tensor#create(Object) Tensor.create
    */
-  public static Constant create(Scope scope, Object object) {
-    try (Tensor value = Tensor.create(object)) {
+  public static <T> Constant<T> create(Scope scope, Object object, Class<T> type) {
+    try (Tensor<T> value = Tensor.create(object, type)) {
       return createWithTensor(scope, value);
     }
   }
@@ -66,8 +66,8 @@ public final class Constant extends PrimitiveOp implements Operand {
    * @param data a buffer containing the tensor data.
    * @throws IllegalArgumentException If the tensor shape is not compatible with the buffer
    */
-  public static Constant create(Scope scope, long[] shape, IntBuffer data) {
-    try (Tensor value = Tensor.create(shape, data)) {
+  public static Constant<Integer> create(Scope scope, long[] shape, IntBuffer data) {
+    try (Tensor<Integer> value = Tensor.create(shape, data)) {
       return createWithTensor(scope, value);
     }
   }
@@ -85,8 +85,8 @@ public final class Constant extends PrimitiveOp implements Operand {
    * @param data a buffer containing the tensor data.
    * @throws IllegalArgumentException If the tensor shape is not compatible with the buffer
    */
-  public static Constant create(Scope scope, long[] shape, FloatBuffer data) {
-    try (Tensor value = Tensor.create(shape, data)) {
+  public static Constant<Float> create(Scope scope, long[] shape, FloatBuffer data) {
+    try (Tensor<Float> value = Tensor.create(shape, data)) {
       return createWithTensor(scope, value);
     }
   }
@@ -104,8 +104,8 @@ public final class Constant extends PrimitiveOp implements Operand {
    * @param data a buffer containing the tensor data.
    * @throws IllegalArgumentException If the tensor shape is not compatible with the buffer
    */
-  public static Constant create(Scope scope, long[] shape, DoubleBuffer data) {
-    try (Tensor value = Tensor.create(shape, data)) {
+  public static Constant<Double> create(Scope scope, long[] shape, DoubleBuffer data) {
+    try (Tensor<Double> value = Tensor.create(shape, data)) {
       return createWithTensor(scope, value);
     }
   }
@@ -123,8 +123,8 @@ public final class Constant extends PrimitiveOp implements Operand {
    * @param data a buffer containing the tensor data.
    * @throws IllegalArgumentException If the tensor shape is not compatible with the buffer
    */
-  public static Constant create(Scope scope, long[] shape, LongBuffer data) {
-    try (Tensor value = Tensor.create(shape, data)) {
+  public static Constant<Long> create(Scope scope, long[] shape, LongBuffer data) {
+    try (Tensor<Long> value = Tensor.create(shape, data)) {
       return createWithTensor(scope, value);
     }
   }
@@ -143,14 +143,14 @@ public final class Constant extends PrimitiveOp implements Operand {
    * @throws IllegalArgumentException If the tensor datatype or shape is not compatible with the
    *     buffer
    */
-  public static Constant create(Scope scope, DataType dataType, long[] shape, ByteBuffer data) {
-    try (Tensor value = Tensor.create(dataType, shape, data)) {
+  public static <T> Constant<T> create(Scope scope, Class<T> type, long[] shape, ByteBuffer data) {
+    try (Tensor<T> value = Tensor.create(type, shape, data)) {
       return createWithTensor(scope, value);
     }
   }
 
-  private static Constant createWithTensor(Scope scope, Tensor value) {
-    return new Constant(
+  private static <T> Constant<T> createWithTensor(Scope scope, Tensor<T> value) {
+    return new Constant<T>(
         scope
             .graph()
             .opBuilder("Const", scope.makeOpName("Const"))
@@ -160,7 +160,7 @@ public final class Constant extends PrimitiveOp implements Operand {
   }
 
   @Override
-  public Output asOutput() {
+  public Output<T> asOutput() {
     return output;
   }
 
@@ -169,5 +169,5 @@ public final class Constant extends PrimitiveOp implements Operand {
     output = operation.output(0);
   }
 
-  private final Output output;
+  private final Output<T> output;
 }
diff --git a/tensorflow/java/src/main/java/org/tensorflow/types/UInt8.java b/tensorflow/java/src/main/java/org/tensorflow/types/UInt8.java
new file mode 100644
index 0000000000..0c751aed9f
--- /dev/null
+++ b/tensorflow/java/src/main/java/org/tensorflow/types/UInt8.java
@@ -0,0 +1,21 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+package org.tensorflow.types;
+
+/** Represents an 8-bit unsigned integer. */
+public class UInt8 {
+  private UInt8() {}
+}
diff --git a/tensorflow/java/src/main/java/org/tensorflow/types/package-info.java b/tensorflow/java/src/main/java/org/tensorflow/types/package-info.java
index f1410a760e..96018c5366 100644
--- a/tensorflow/java/src/main/java/org/tensorflow/types/package-info.java
+++ b/tensorflow/java/src/main/java/org/tensorflow/types/package-info.java
@@ -15,13 +15,15 @@ limitations under the License.
 
 /**
  * Defines classes that represent TensorFlow data types. For each possible data type
- * that can be used in a tensor, there is a corresponding class in this package that
+ * that can be used in a tensor, there is a corresponding class that
  * is used to represent it. For example, the TensorFlow int32 type is represented by
- * the type TFInt32 and by the class object TFInt32.class. The former is used to
- * support compile-time checking of tensor data types and the latter is used for
- * run-time checking of data types. All such classes implement the TFType interface.
- * TensorFlow data types are also separately represented by the DataType enum, with
- * one enum value per data type. The enum representation should rarely be needed, but
- * the Types class can be used to obtain it from the class object representation.
+ * the type {@link Integer} and by the class object {@code Integer.class}. The former is used to
+ * support compile-time checking of tensor element types and the latter is used for
+ * run-time checking of element types. Classes appearing in this package, such as
+ * UInt8, represent TensorFlow data types for which there is no existing Java equivalent.
+ *
+ * <p>TensorFlow element types are also separately represented by the {@link DataType} enum, with
+ * one enum value per element type. The enum representation is not usually needed, but
+ * can be obtained using {@link DataType.fromClass}.
  */
 package org.tensorflow.types;
diff --git a/tensorflow/java/src/test/java/org/tensorflow/GraphTest.java b/tensorflow/java/src/test/java/org/tensorflow/GraphTest.java
index 4adc861bf1..c540299bdc 100644
--- a/tensorflow/java/src/test/java/org/tensorflow/GraphTest.java
+++ b/tensorflow/java/src/test/java/org/tensorflow/GraphTest.java
@@ -22,7 +22,6 @@ import static org.junit.Assert.assertTrue;
 
 import java.util.HashSet;
 import java.util.Iterator;
-
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
diff --git a/tensorflow/java/src/test/java/org/tensorflow/OperationBuilderTest.java b/tensorflow/java/src/test/java/org/tensorflow/OperationBuilderTest.java
index b3bc3aaef9..6dc233987b 100644
--- a/tensorflow/java/src/test/java/org/tensorflow/OperationBuilderTest.java
+++ b/tensorflow/java/src/test/java/org/tensorflow/OperationBuilderTest.java
@@ -34,8 +34,8 @@ public class OperationBuilderTest {
   public void failWhenMixingOperationsOnDifferentGraphs() {
     try (Graph g1 = new Graph();
         Graph g2 = new Graph()) {
-      Output c1 = TestUtil.constant(g1, "C1", 3);
-      Output c2 = TestUtil.constant(g2, "C2", 3);
+      Output<Integer> c1 = TestUtil.constant(g1, "C1", 3);
+      Output<Integer> c2 = TestUtil.constant(g2, "C2", 3);
       TestUtil.addN(g1, c1, c1);
       try {
         TestUtil.addN(g2, c1, c2);
@@ -48,7 +48,7 @@ public class OperationBuilderTest {
   @Test
   public void failOnUseAfterBuild() {
     try (Graph g = new Graph();
-        Tensor t = Tensor.create(1)) {
+        Tensor<Integer> t = Tensors.create(1)) {
       OperationBuilder b =
           g.opBuilder("Const", "Const").setAttr("dtype", t.dataType()).setAttr("value", t);
       b.build();
@@ -64,7 +64,7 @@ public class OperationBuilderTest {
   public void failOnUseAfterGraphClose() {
     OperationBuilder b = null;
     try (Graph g = new Graph();
-        Tensor t = Tensor.create(1)) {
+        Tensor<Integer> t = Tensors.create(1)) {
       b = g.opBuilder("Const", "Const").setAttr("dtype", t.dataType()).setAttr("value", t);
     }
     try {
@@ -85,7 +85,7 @@ public class OperationBuilderTest {
     // types that aren't inferred from the input arguments.
     try (Graph g = new Graph()) {
       // dtype, tensor attributes.
-      try (Tensor t = Tensor.create(1)) {
+      try (Tensor<Integer> t = Tensors.create(1)) {
         g.opBuilder("Const", "DataTypeAndTensor")
             .setAttr("dtype", DataType.INT32)
             .setAttr("value", t)
@@ -101,7 +101,7 @@ public class OperationBuilderTest {
       assertTrue(hasNode(g, "StringAndBool"));
       // int (TF "int" attributes are 64-bit signed, so a Java long).
       g.opBuilder("RandomUniform", "Int")
-          .addInput(TestUtil.constant(g, "RandomUniformShape", new int[]{1}))
+          .addInput(TestUtil.constant(g, "RandomUniformShape", new int[] {1}))
           .setAttr("seed", 10)
           .setAttr("dtype", DataType.FLOAT)
           .build();
@@ -127,7 +127,7 @@ public class OperationBuilderTest {
   @Test
   public void setAttrShape() {
     try (Graph g = new Graph()) {
-      Output n =
+      Output<?> n =
           g.opBuilder("Placeholder", "unknown")
               .setAttr("dtype", DataType.FLOAT)
               .setAttr("shape", Shape.unknown())
@@ -136,8 +136,7 @@ public class OperationBuilderTest {
       assertEquals(-1, n.shape().numDimensions());
       assertEquals(DataType.FLOAT, n.dataType());
 
-      n =
-          g.opBuilder("Placeholder", "batch_of_vectors")
+      n = g.opBuilder("Placeholder", "batch_of_vectors")
               .setAttr("dtype", DataType.FLOAT)
               .setAttr("shape", Shape.make(-1, 784))
               .build()
@@ -153,13 +152,13 @@ public class OperationBuilderTest {
   public void addControlInput() {
     try (Graph g = new Graph();
         Session s = new Session(g);
-        Tensor yes = Tensor.create(true);
-        Tensor no = Tensor.create(false)) {
-      Output placeholder = TestUtil.placeholder(g, "boolean", DataType.BOOL);
+        Tensor<Boolean> yes = Tensors.create(true);
+        Tensor<Boolean> no = Tensors.create(false)) {
+      Output<Boolean> placeholder = TestUtil.placeholder(g, "boolean", Boolean.class);
       Operation check =
           g.opBuilder("Assert", "assert")
               .addInput(placeholder)
-              .addInputList(new Output[] {placeholder})
+              .addInputList(new Output<?>[] {placeholder})
               .build();
       Operation noop = g.opBuilder("NoOp", "noop").addControlInput(check).build();
 
diff --git a/tensorflow/java/src/test/java/org/tensorflow/OperationTest.java b/tensorflow/java/src/test/java/org/tensorflow/OperationTest.java
index aade375db8..6fe3b3c327 100644
--- a/tensorflow/java/src/test/java/org/tensorflow/OperationTest.java
+++ b/tensorflow/java/src/test/java/org/tensorflow/OperationTest.java
@@ -24,7 +24,6 @@ import static org.junit.Assert.fail;
 import java.util.Arrays;
 import java.util.HashSet;
 import java.util.Set;
-
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
@@ -104,9 +103,9 @@ public class OperationTest {
   @Test
   public void outputEquality() {
     try (Graph g = new Graph()) {
-      Output output = TestUtil.constant(g, "c", 1);
-      Output output1 = output.op().output(0);
-      Output output2 = g.operation("c").output(0);
+      Output<Integer> output = TestUtil.constant(g, "c", 1);
+      Output<Integer> output1 = output.op().<Integer>output(0);
+      Output<Integer> output2 = g.operation("c").<Integer>output(0);
       assertEquals(output, output1);
       assertEquals(output.hashCode(), output1.hashCode());
       assertEquals(output, output2);
@@ -117,10 +116,10 @@ public class OperationTest {
   @Test
   public void outputCollection() {
     try (Graph g = new Graph()) {
-      Output output = TestUtil.constant(g, "c", 1);
-      Output output1 = output.op().output(0);
-      Output output2 = g.operation("c").output(0);
-      Set<Output> ops = new HashSet<>();
+      Output<Integer> output = TestUtil.constant(g, "c", 1);
+      Output<Integer> output1 = output.op().<Integer>output(0);
+      Output<Integer> output2 = g.operation("c").<Integer>output(0);
+      Set<Output<Integer>> ops = new HashSet<>();
       ops.addAll(Arrays.asList(output, output1, output2));
       assertEquals(1, ops.size());
       assertTrue(ops.contains(output));
@@ -132,7 +131,7 @@ public class OperationTest {
   @Test
   public void outputToString() {
     try (Graph g = new Graph()) {
-      Output output = TestUtil.constant(g, "c", new int[] {1});
+      Output<Integer> output = TestUtil.constant(g, "c", new int[] {1});
       assertNotNull(output.toString());
     }
   }
@@ -158,7 +157,7 @@ public class OperationTest {
   public void outputList() {
     try (Graph g = new Graph()) {
       Operation split = TestUtil.split(g, "split", new int[] {0, 1, 2}, 3);
-      Output[] outputs = split.outputList(1, 2);
+      Output<?>[] outputs = split.outputList(1, 2);
       assertNotNull(outputs);
       assertEquals(2, outputs.length);
       for (int i = 0; i < outputs.length; ++i) {
diff --git a/tensorflow/java/src/test/java/org/tensorflow/SessionTest.java b/tensorflow/java/src/test/java/org/tensorflow/SessionTest.java
index 50bdf351e3..a86b4dd117 100644
--- a/tensorflow/java/src/test/java/org/tensorflow/SessionTest.java
+++ b/tensorflow/java/src/test/java/org/tensorflow/SessionTest.java
@@ -35,9 +35,9 @@ public class SessionTest {
     try (Graph g = new Graph();
         Session s = new Session(g)) {
       TestUtil.transpose_A_times_X(g, new int[][] {{2}, {3}});
-      try (Tensor x = Tensor.create(new int[][] {{5}, {7}});
-          AutoCloseableList<Tensor> outputs =
-              new AutoCloseableList<Tensor>(s.runner().feed("X", x).fetch("Y").run())) {
+      try (Tensor<Integer> x = Tensors.create(new int[][] {{5}, {7}});
+          AutoCloseableList<Tensor<?>> outputs =
+              new AutoCloseableList<Tensor<?>>(s.runner().feed("X", x).fetch("Y").run())) {
         assertEquals(1, outputs.size());
         final int[][] expected = {{31}};
         assertArrayEquals(expected, outputs.get(0).copyTo(new int[1][1]));
@@ -50,11 +50,11 @@ public class SessionTest {
     try (Graph g = new Graph();
         Session s = new Session(g)) {
       TestUtil.transpose_A_times_X(g, new int[][] {{2}, {3}});
-      Output feed = g.operation("X").output(0);
-      Output fetch = g.operation("Y").output(0);
-      try (Tensor x = Tensor.create(new int[][] {{5}, {7}});
-          AutoCloseableList<Tensor> outputs =
-              new AutoCloseableList<Tensor>(s.runner().feed(feed, x).fetch(fetch).run())) {
+      Output<Integer> feed = g.operation("X").output(0);
+      Output<Integer> fetch = g.operation("Y").output(0);
+      try (Tensor<Integer> x = Tensors.create(new int[][] {{5}, {7}});
+          AutoCloseableList<Tensor<?>> outputs =
+              new AutoCloseableList<Tensor<?>>(s.runner().feed(feed, x).fetch(fetch).run())) {
         assertEquals(1, outputs.size());
         final int[][] expected = {{31}};
         assertArrayEquals(expected, outputs.get(0).copyTo(new int[1][1]));
@@ -78,14 +78,21 @@ public class SessionTest {
           .build()
           .output(0);
       // Fetch using colon separated names.
-      try (Tensor fetched = s.runner().fetch("Split:1").run().get(0)) {
+      try (Tensor<Integer> fetched =
+          s.runner().fetch("Split:1").run().get(0).expect(Integer.class)) {
         final int[] expected = {3, 4};
         assertArrayEquals(expected, fetched.copyTo(new int[2]));
       }
       // Feed using colon separated names.
-      try (Tensor fed = Tensor.create(new int[] {4, 3, 2, 1});
-          Tensor fetched =
-              s.runner().feed("Split:0", fed).feed("Split:1", fed).fetch("Add").run().get(0)) {
+      try (Tensor<Integer> fed = Tensors.create(new int[] {4, 3, 2, 1});
+          Tensor<Integer> fetched =
+              s.runner()
+                  .feed("Split:0", fed)
+                  .feed("Split:1", fed)
+                  .fetch("Add")
+                  .run()
+                  .get(0)
+                  .expect(Integer.class)) {
         final int[] expected = {8, 6, 4, 2};
         assertArrayEquals(expected, fetched.copyTo(new int[4]));
       }
@@ -97,7 +104,7 @@ public class SessionTest {
     try (Graph g = new Graph();
         Session s = new Session(g)) {
       TestUtil.transpose_A_times_X(g, new int[][] {{2}, {3}});
-      try (Tensor x = Tensor.create(new int[][] {{5}, {7}})) {
+      try (Tensor<Integer> x = Tensors.create(new int[][] {{5}, {7}})) {
         Session.Run result =
             s.runner()
                 .feed("X", x)
@@ -105,7 +112,7 @@ public class SessionTest {
                 .setOptions(fullTraceRunOptions())
                 .runAndFetchMetadata();
         // Sanity check on outputs.
-        AutoCloseableList<Tensor> outputs = new AutoCloseableList<Tensor>(result.outputs);
+        AutoCloseableList<Tensor<?>> outputs = new AutoCloseableList<Tensor<?>>(result.outputs);
         assertEquals(1, outputs.size());
         final int[][] expected = {{31}};
         assertArrayEquals(expected, outputs.get(0).copyTo(new int[1][1]));
@@ -117,6 +124,7 @@ public class SessionTest {
             assertTrue(md.toString(), md.hasStepStats());
         */
         assertTrue(result.metadata.length > 0);
+        outputs.close();
       }
     }
   }
@@ -127,11 +135,12 @@ public class SessionTest {
         Session s = new Session(g)) {
       TestUtil.constant(g, "c1", 2718);
       TestUtil.constant(g, "c2", 31415);
-      AutoCloseableList<Tensor> outputs =
-          new AutoCloseableList<Tensor>(s.runner().fetch("c2").fetch("c1").run());
+      AutoCloseableList<Tensor<?>> outputs =
+          new AutoCloseableList<Tensor<?>>(s.runner().fetch("c2").fetch("c1").run());
       assertEquals(2, outputs.size());
       assertEquals(31415, outputs.get(0).intValue());
       assertEquals(2718, outputs.get(1).intValue());
+      outputs.close();
     }
   }
 
diff --git a/tensorflow/java/src/test/java/org/tensorflow/ShapeTest.java b/tensorflow/java/src/test/java/org/tensorflow/ShapeTest.java
index fe46c0184c..3b027700c5 100644
--- a/tensorflow/java/src/test/java/org/tensorflow/ShapeTest.java
+++ b/tensorflow/java/src/test/java/org/tensorflow/ShapeTest.java
@@ -61,7 +61,7 @@ public class ShapeTest {
   @Test
   public void nodesInAGraph() {
     try (Graph g = new Graph()) {
-      Output n = TestUtil.placeholder(g, "feed", DataType.FLOAT);
+      Output<Float> n = TestUtil.placeholder(g, "feed", Float.class);
       assertEquals(-1, n.shape().numDimensions());
 
       n = TestUtil.constant(g, "scalar", 3);
diff --git a/tensorflow/java/src/test/java/org/tensorflow/TensorTest.java b/tensorflow/java/src/test/java/org/tensorflow/TensorTest.java
index 036db04503..6538359d11 100644
--- a/tensorflow/java/src/test/java/org/tensorflow/TensorTest.java
+++ b/tensorflow/java/src/test/java/org/tensorflow/TensorTest.java
@@ -30,6 +30,7 @@ import java.nio.LongBuffer;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
+import org.tensorflow.types.UInt8;
 
 /** Unit tests for {@link org.tensorflow.Tensor}. */
 @RunWith(JUnit4.class)
@@ -47,7 +48,7 @@ public class TensorTest {
     byte[] strings = "test".getBytes(UTF_8);
     long[] strings_shape = {};
     byte[] strings_; // raw TF_STRING
-    try (Tensor t = Tensor.create(strings)) {
+    try (Tensor<String> t = Tensors.create(strings)) {
       ByteBuffer to = ByteBuffer.allocate(t.numBytes());
       t.writeTo(to);
       strings_ = to.array();
@@ -55,7 +56,7 @@ public class TensorTest {
 
     // validate creating a tensor using a byte buffer
     {
-      try (Tensor t = Tensor.create(DataType.BOOL, bools_shape, ByteBuffer.wrap(bools_))) {
+      try (Tensor<Boolean> t = Tensor.create(Boolean.class, bools_shape, ByteBuffer.wrap(bools_))) {
         boolean[] actual = t.copyTo(new boolean[bools_.length]);
         for (int i = 0; i < bools.length; ++i) {
           assertEquals("" + i, bools[i], actual[i]);
@@ -63,7 +64,8 @@ public class TensorTest {
       }
 
       // note: the buffer is expected to contain raw TF_STRING (as per C API)
-      try (Tensor t = Tensor.create(DataType.STRING, strings_shape, ByteBuffer.wrap(strings_))) {
+      try (Tensor<String> t =
+          Tensor.create(String.class, strings_shape, ByteBuffer.wrap(strings_))) {
         assertArrayEquals(strings, t.bytesValue());
       }
     }
@@ -72,15 +74,15 @@ public class TensorTest {
     {
       ByteBuffer buf = ByteBuffer.allocateDirect(8 * doubles.length).order(ByteOrder.nativeOrder());
       buf.asDoubleBuffer().put(doubles);
-      try (Tensor t = Tensor.create(DataType.DOUBLE, doubles_shape, buf)) {
+      try (Tensor<Double> t = Tensor.create(Double.class, doubles_shape, buf)) {
         double[] actual = new double[doubles.length];
         assertArrayEquals(doubles, t.copyTo(actual), EPSILON);
       }
     }
 
     // validate shape checking
-    try (Tensor t =
-        Tensor.create(DataType.BOOL, new long[bools_.length * 2], ByteBuffer.wrap(bools_))) {
+    try (Tensor<Boolean> t =
+        Tensor.create(Boolean.class, new long[bools_.length * 2], ByteBuffer.wrap(bools_))) {
       fail("should have failed on incompatible buffer");
     } catch (IllegalArgumentException e) {
       // expected
@@ -99,7 +101,7 @@ public class TensorTest {
             .asDoubleBuffer()
             .put(doubles);
     buf.flip();
-    try (Tensor t = Tensor.create(new long[] {doubles.length}, buf)) {
+    try (Tensor<Double> t = Tensor.create(new long[] {doubles.length}, buf)) {
       double[] actual = new double[doubles.length];
       assertArrayEquals(doubles, t.copyTo(actual), EPSILON);
     }
@@ -115,19 +117,19 @@ public class TensorTest {
 
     // validate creating a tensor using a typed buffer
     {
-      try (Tensor t = Tensor.create(shape, DoubleBuffer.wrap(doubles))) {
+      try (Tensor<Double> t = Tensor.create(shape, DoubleBuffer.wrap(doubles))) {
         double[] actual = new double[doubles.length];
         assertArrayEquals(doubles, t.copyTo(actual), EPSILON);
       }
-      try (Tensor t = Tensor.create(shape, FloatBuffer.wrap(floats))) {
+      try (Tensor<Float> t = Tensor.create(shape, FloatBuffer.wrap(floats))) {
         float[] actual = new float[floats.length];
         assertArrayEquals(floats, t.copyTo(actual), EPSILON_F);
       }
-      try (Tensor t = Tensor.create(shape, IntBuffer.wrap(ints))) {
+      try (Tensor<Integer> t = Tensor.create(shape, IntBuffer.wrap(ints))) {
         int[] actual = new int[ints.length];
         assertArrayEquals(ints, t.copyTo(actual));
       }
-      try (Tensor t = Tensor.create(shape, LongBuffer.wrap(longs))) {
+      try (Tensor<Long> t = Tensor.create(shape, LongBuffer.wrap(longs))) {
         long[] actual = new long[longs.length];
         assertArrayEquals(longs, t.copyTo(actual));
       }
@@ -135,22 +137,23 @@ public class TensorTest {
 
     // validate shape-checking
     {
-      try (Tensor t = Tensor.create(new long[doubles.length + 1], DoubleBuffer.wrap(doubles))) {
+      try (Tensor<Double> t =
+          Tensor.create(new long[doubles.length + 1], DoubleBuffer.wrap(doubles))) {
         fail("should have failed on incompatible buffer");
       } catch (IllegalArgumentException e) {
         // expected
       }
-      try (Tensor t = Tensor.create(new long[floats.length + 1], FloatBuffer.wrap(floats))) {
+      try (Tensor<Float> t = Tensor.create(new long[floats.length + 1], FloatBuffer.wrap(floats))) {
         fail("should have failed on incompatible buffer");
       } catch (IllegalArgumentException e) {
         // expected
       }
-      try (Tensor t = Tensor.create(new long[ints.length + 1], IntBuffer.wrap(ints))) {
+      try (Tensor<Integer> t = Tensor.create(new long[ints.length + 1], IntBuffer.wrap(ints))) {
         fail("should have failed on incompatible buffer");
       } catch (IllegalArgumentException e) {
         // expected
       }
-      try (Tensor t = Tensor.create(new long[longs.length + 1], LongBuffer.wrap(longs))) {
+      try (Tensor<Long> t = Tensor.create(new long[longs.length + 1], LongBuffer.wrap(longs))) {
         fail("should have failed on incompatible buffer");
       } catch (IllegalArgumentException e) {
         // expected
@@ -166,11 +169,11 @@ public class TensorTest {
     long[] longs = {1L, 2L, 3L};
     boolean[] bools = {true, false, true};
 
-    try (Tensor tints = Tensor.create(ints);
-        Tensor tfloats = Tensor.create(floats);
-        Tensor tdoubles = Tensor.create(doubles);
-        Tensor tlongs = Tensor.create(longs);
-        Tensor tbools = Tensor.create(bools)) {
+    try (Tensor<Integer> tints = Tensors.create(ints);
+        Tensor<Float> tfloats = Tensors.create(floats);
+        Tensor<Double> tdoubles = Tensors.create(doubles);
+        Tensor<Long> tlongs = Tensors.create(longs);
+        Tensor<Boolean> tbools = Tensors.create(bools)) {
 
       // validate that any datatype is readable with ByteBuffer (content, position)
       {
@@ -293,35 +296,35 @@ public class TensorTest {
 
   @Test
   public void scalars() {
-    try (Tensor t = Tensor.create(2.718f)) {
+    try (Tensor<Float> t = Tensors.create(2.718f)) {
       assertEquals(DataType.FLOAT, t.dataType());
       assertEquals(0, t.numDimensions());
       assertEquals(0, t.shape().length);
       assertEquals(2.718f, t.floatValue(), EPSILON_F);
     }
 
-    try (Tensor t = Tensor.create(3.1415)) {
+    try (Tensor<Double> t = Tensors.create(3.1415)) {
       assertEquals(DataType.DOUBLE, t.dataType());
       assertEquals(0, t.numDimensions());
       assertEquals(0, t.shape().length);
       assertEquals(3.1415, t.doubleValue(), EPSILON);
     }
 
-    try (Tensor t = Tensor.create(-33)) {
+    try (Tensor<Integer> t = Tensors.create(-33)) {
       assertEquals(DataType.INT32, t.dataType());
       assertEquals(0, t.numDimensions());
       assertEquals(0, t.shape().length);
       assertEquals(-33, t.intValue());
     }
 
-    try (Tensor t = Tensor.create(8589934592L)) {
+    try (Tensor<Long> t = Tensors.create(8589934592L)) {
       assertEquals(DataType.INT64, t.dataType());
       assertEquals(0, t.numDimensions());
       assertEquals(0, t.shape().length);
       assertEquals(8589934592L, t.longValue());
     }
 
-    try (Tensor t = Tensor.create(true)) {
+    try (Tensor<Boolean> t = Tensors.create(true)) {
       assertEquals(DataType.BOOL, t.dataType());
       assertEquals(0, t.numDimensions());
       assertEquals(0, t.shape().length);
@@ -329,7 +332,7 @@ public class TensorTest {
     }
 
     final byte[] bytes = {1, 2, 3, 4};
-    try (Tensor t = Tensor.create(bytes)) {
+    try (Tensor<String> t = Tensors.create(bytes)) {
       assertEquals(DataType.STRING, t.dataType());
       assertEquals(0, t.numDimensions());
       assertEquals(0, t.shape().length);
@@ -340,7 +343,7 @@ public class TensorTest {
   @Test
   public void nDimensional() {
     double[] vector = {1.414, 2.718, 3.1415};
-    try (Tensor t = Tensor.create(vector)) {
+    try (Tensor<Double> t = Tensors.create(vector)) {
       assertEquals(DataType.DOUBLE, t.dataType());
       assertEquals(1, t.numDimensions());
       assertArrayEquals(new long[] {3}, t.shape());
@@ -350,7 +353,7 @@ public class TensorTest {
     }
 
     int[][] matrix = {{1, 2, 3}, {4, 5, 6}};
-    try (Tensor t = Tensor.create(matrix)) {
+    try (Tensor<Integer> t = Tensors.create(matrix)) {
       assertEquals(DataType.INT32, t.dataType());
       assertEquals(2, t.numDimensions());
       assertArrayEquals(new long[] {2, 3}, t.shape());
@@ -362,7 +365,7 @@ public class TensorTest {
     long[][][] threeD = {
       {{1}, {3}, {5}, {7}, {9}}, {{2}, {4}, {6}, {8}, {0}},
     };
-    try (Tensor t = Tensor.create(threeD)) {
+    try (Tensor<Long> t = Tensors.create(threeD)) {
       assertEquals(DataType.INT64, t.dataType());
       assertEquals(3, t.numDimensions());
       assertArrayEquals(new long[] {2, 5, 1}, t.shape());
@@ -376,7 +379,7 @@ public class TensorTest {
       {{{false, false, true, true}, {false, true, false, false}}},
       {{{false, true, false, true}, {false, true, true, false}}},
     };
-    try (Tensor t = Tensor.create(fourD)) {
+    try (Tensor<Boolean> t = Tensors.create(fourD)) {
       assertEquals(DataType.BOOL, t.dataType());
       assertEquals(4, t.numDimensions());
       assertArrayEquals(new long[] {3, 1, 2, 4}, t.shape());
@@ -394,7 +397,7 @@ public class TensorTest {
         matrix[i][j] = String.format("(%d, %d) = %d", i, j, i << j).getBytes(UTF_8);
       }
     }
-    try (Tensor t = Tensor.create(matrix)) {
+    try (Tensor<String> t = Tensors.create(matrix)) {
       assertEquals(DataType.STRING, t.dataType());
       assertEquals(2, t.numDimensions());
       assertArrayEquals(new long[] {4, 3}, t.shape());
@@ -412,14 +415,24 @@ public class TensorTest {
 
   @Test
   public void testUInt8Tensor() {
-    byte[] vector = new byte[] { 1, 2, 3, 4 };
-    try (Tensor t = Tensor.create(vector, DataType.UINT8)) {
+    byte[] vector = new byte[] {1, 2, 3, 4};
+    try (Tensor<UInt8> t = Tensor.create(vector, UInt8.class)) {
       assertEquals(DataType.UINT8, t.dataType());
       assertEquals(1, t.numDimensions());
       assertArrayEquals(new long[] {4}, t.shape());
 
       byte[] got = t.copyTo(new byte[4]);
-      assertArrayEquals(got, vector);
+      assertArrayEquals(vector, got);
+    }
+  }
+
+  @Test
+  public void testCreateFromArrayOfBoxed() {
+    Integer[] vector = new Integer[] {1, 2, 3, 4};
+    try (Tensor<Integer> t = Tensor.create(vector, Integer.class)) {
+      fail("Tensor.create() should fail because it was given an array of boxed values");
+    } catch (IllegalArgumentException e) {
+        // The expected exception
     }
   }
 
@@ -431,7 +444,7 @@ public class TensorTest {
         invalid[x][y] = new int[x + y + 1];
       }
     }
-    try (Tensor t = Tensor.create(invalid)) {
+    try (Tensor<?> t = Tensor.create(invalid)) {
       fail("Tensor.create() should fail because of differing sizes in the 3rd dimension");
     } catch (IllegalArgumentException e) {
       // The expected exception.
@@ -440,7 +453,7 @@ public class TensorTest {
 
   @Test
   public void failCopyToOnIncompatibleDestination() {
-    try (final Tensor matrix = Tensor.create(new int[][] {{1, 2}, {3, 4}})) {
+    try (final Tensor<Integer> matrix = Tensors.create(new int[][] {{1, 2}, {3, 4}})) {
       try {
         matrix.copyTo(new int[2]);
         fail("should have failed on dimension mismatch");
@@ -466,7 +479,7 @@ public class TensorTest {
 
   @Test
   public void failCopyToOnScalar() {
-    try (final Tensor scalar = Tensor.create(3)) {
+    try (final Tensor<Integer> scalar = Tensors.create(3)) {
       try {
         scalar.copyTo(3);
         fail("copyTo should fail on scalar tensors, suggesting use of primitive accessors instead");
@@ -478,8 +491,8 @@ public class TensorTest {
 
   @Test
   public void failOnArbitraryObject() {
-    try (Tensor t = Tensor.create(new Object())) {
-      fail("should fail on creating a Tensor with a Java object that has not equivalent DataType");
+    try (Tensor<?> t = Tensor.create(new Object())) {
+      fail("should fail on creating a Tensor with a Java object that has no equivalent DataType");
     } catch (IllegalArgumentException e) {
       // The expected exception.
     }
@@ -487,7 +500,7 @@ public class TensorTest {
 
   @Test
   public void failOnZeroDimension() {
-    try (Tensor t = Tensor.create(new int[3][0][1])) {
+    try (Tensor<Integer> t = Tensors.create(new int[3][0][1])) {
       fail("should fail on creating a Tensor where one of the dimensions is 0");
     } catch (IllegalArgumentException e) {
       // The expected exception.
@@ -497,7 +510,7 @@ public class TensorTest {
   @Test
   public void useAfterClose() {
     int n = 4;
-    Tensor t = Tensor.create(n);
+    Tensor<?> t = Tensor.create(n);
     t.close();
     try {
       t.intValue();
@@ -515,8 +528,8 @@ public class TensorTest {
     // An exception is made for this test, where the pitfalls of this is avoided by not calling
     // close() on both Tensors.
     final float[][] matrix = {{1, 2, 3}, {4, 5, 6}};
-    try (Tensor src = Tensor.create(matrix)) {
-      Tensor cpy = Tensor.fromHandle(src.getNativeHandle());
+    try (Tensor<Float> src = Tensors.create(matrix)) {
+      Tensor<Float> cpy = Tensor.fromHandle(src.getNativeHandle()).expect(Float.class);
       assertEquals(src.dataType(), cpy.dataType());
       assertEquals(src.numDimensions(), cpy.numDimensions());
       assertArrayEquals(src.shape(), cpy.shape());
diff --git a/tensorflow/java/src/test/java/org/tensorflow/TestUtil.java b/tensorflow/java/src/test/java/org/tensorflow/TestUtil.java
index e3415a696d..c973b5a3d8 100644
--- a/tensorflow/java/src/test/java/org/tensorflow/TestUtil.java
+++ b/tensorflow/java/src/test/java/org/tensorflow/TestUtil.java
@@ -19,33 +19,36 @@ import java.lang.reflect.Array;
 
 /** Static utility functions. */
 public class TestUtil {
-  public static Output constant(Graph g, String name, Object value) {
-    try (Tensor t = Tensor.create(value)) {
+  public static <T> Output<T> constant(Graph g, String name, Object value) {
+    try (Tensor<?> t = Tensor.create(value)) {
       return g.opBuilder("Const", name)
           .setAttr("dtype", t.dataType())
           .setAttr("value", t)
           .build()
-          .output(0);
+          .<T>output(0);
     }
   }
 
-  public static Output placeholder(Graph g, String name, DataType dtype) {
-    return g.opBuilder("Placeholder", name).setAttr("dtype", dtype).build().output(0);
+  public static <T> Output<T> placeholder(Graph g, String name, Class<T> type) {
+    return g.opBuilder("Placeholder", name)
+        .setAttr("dtype", DataType.fromClass(type))
+        .build()
+        .<T>output(0);
   }
 
-  public static Output addN(Graph g, Output... inputs) {
+  public static Output<?> addN(Graph g, Output<?>... inputs) {
     return g.opBuilder("AddN", "AddN").addInputList(inputs).build().output(0);
   }
 
-  public static Output matmul(
-      Graph g, String name, Output a, Output b, boolean transposeA, boolean transposeB) {
+  public static <T> Output<T> matmul(
+      Graph g, String name, Output<T> a, Output<T> b, boolean transposeA, boolean transposeB) {
     return g.opBuilder("MatMul", name)
         .addInput(a)
         .addInput(b)
         .setAttr("transpose_a", transposeA)
         .setAttr("transpose_b", transposeB)
         .build()
-        .output(0);
+        .<T>output(0);
   }
 
   public static Operation split(Graph g, String name, int[] values, int numSplit) {
@@ -57,7 +60,8 @@ public class TestUtil {
   }
 
   public static void transpose_A_times_X(Graph g, int[][] a) {
-    matmul(g, "Y", constant(g, "A", a), placeholder(g, "X", DataType.INT32), true, false);
+    Output<Integer> aa = constant(g, "A", a);
+    matmul(g, "Y", aa, placeholder(g, "X", Integer.class), true, false);
   }
 
   /**
diff --git a/tensorflow/java/src/test/java/org/tensorflow/op/OperandsTest.java b/tensorflow/java/src/test/java/org/tensorflow/op/OperandsTest.java
index 4fdd150acc..79bfcc8354 100644
--- a/tensorflow/java/src/test/java/org/tensorflow/op/OperandsTest.java
+++ b/tensorflow/java/src/test/java/org/tensorflow/op/OperandsTest.java
@@ -4,7 +4,7 @@ Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
 
-    http://www.apache.org/licenses/LICENSE-2.0
+http://www.apache.org/licenses/LICENSE-2.0
 
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
@@ -36,8 +36,9 @@ public class OperandsTest {
   public void createOutputArrayFromOperandList() {
     try (Graph g = new Graph()) {
       Operation split = TestUtil.split(g, "split", new int[] {0, 1, 2}, 3);
-      List<Output> list = Arrays.asList(split.output(0), split.output(2));
-      Output[] array = Operands.asOutputs(list);
+      List<Output<Integer>> list =
+          Arrays.asList(split.<Integer>output(0), split.<Integer>output(2));
+      Output<?>[] array = Operands.asOutputs(list);
       assertEquals(list.size(), array.length);
       assertSame(array[0], list.get(0));
       assertSame(array[1], list.get(1));
diff --git a/tensorflow/java/src/test/java/org/tensorflow/op/PrimitiveOpTest.java b/tensorflow/java/src/test/java/org/tensorflow/op/PrimitiveOpTest.java
index b24bf5a476..e02c38ed22 100644
--- a/tensorflow/java/src/test/java/org/tensorflow/op/PrimitiveOpTest.java
+++ b/tensorflow/java/src/test/java/org/tensorflow/op/PrimitiveOpTest.java
@@ -36,7 +36,7 @@ public class PrimitiveOpTest {
   @Test
   public void equalsHashcode() {
     try (Graph g = new Graph()) {
-      Output array = TestUtil.constant(g, "array", new int[2]);
+      Output<Integer> array = TestUtil.constant(g, "array", new int[2]);
 
       PrimitiveOp test1 =
           new PrimitiveOp(g.opBuilder("Shape", "shape1").addInput(array).build()) {};
diff --git a/tensorflow/java/src/test/java/org/tensorflow/op/ScopeTest.java b/tensorflow/java/src/test/java/org/tensorflow/op/ScopeTest.java
index 9256cb281d..125de73554 100644
--- a/tensorflow/java/src/test/java/org/tensorflow/op/ScopeTest.java
+++ b/tensorflow/java/src/test/java/org/tensorflow/op/ScopeTest.java
@@ -19,6 +19,8 @@ import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.fail;
 
+import java.util.HashMap;
+import java.util.Map;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
@@ -26,6 +28,8 @@ import org.tensorflow.Graph;
 import org.tensorflow.Output;
 import org.tensorflow.Session;
 import org.tensorflow.Tensor;
+import org.tensorflow.Tensors;
+import org.tensorflow.types.UInt8;
 
 /** Unit tests for {@link org.tensorflow.Scope}. */
 @RunWith(JUnit4.class)
@@ -122,13 +126,13 @@ public class ScopeTest {
   public void basic() {
     try (Graph g = new Graph()) {
       Scope s = new Scope(g);
-      Const c1 = Const.create(s, 42);
+      Const<Integer> c1 = Const.create(s, 42);
       assertEquals("Const", c1.output().op().name());
-      Const c2 = Const.create(s, 7);
+      Const<Integer> c2 = Const.create(s, 7);
       assertEquals("Const_1", c2.output().op().name());
-      Const c3 = Const.create(s.withName("four"), 4);
+      Const<Integer> c3 = Const.create(s.withName("four"), 4);
       assertEquals("four", c3.output().op().name());
-      Const c4 = Const.create(s.withName("four"), 4);
+      Const<Integer> c4 = Const.create(s.withName("four"), 4);
       assertEquals("four_1", c4.output().op().name());
     }
   }
@@ -148,122 +152,164 @@ public class ScopeTest {
     try (Graph g = new Graph();
         Session sess = new Session(g)) {
       Scope s = new Scope(g);
-      Output data = Const.create(s.withName("data"), new int[] {600, 470, 170, 430, 300}).output();
+      Output<Integer> data =
+          Const.create(s.withName("data"), new int[] {600, 470, 170, 430, 300}).output();
 
       // Create a composite op with a customized name
-      Variance var1 = Variance.create(s.withName("example"), data);
+      Variance<Integer> var1 = Variance.create(s.withName("example"), data, Integer.class);
       assertEquals("example/variance", var1.output().op().name());
 
       // Confirm internally added ops have the right names.
       assertNotNull(g.operation("example/squared_deviation"));
       assertNotNull(g.operation("example/Mean"));
-      assertNotNull(g.operation("example/zero"));
+      // assertNotNull(g.operation("example/zero"));
 
       // Same composite op with a default name
-      Variance var2 = Variance.create(s, data);
+      Variance<Integer> var2 = Variance.create(s, data, Integer.class);
       assertEquals("variance/variance", var2.output().op().name());
 
       // Confirm internally added ops have the right names.
       assertNotNull(g.operation("variance/squared_deviation"));
       assertNotNull(g.operation("variance/Mean"));
-      assertNotNull(g.operation("variance/zero"));
+      // assertNotNull(g.operation("variance/zero"));
 
       // Verify correct results as well.
-      Tensor result = sess.runner().fetch(var1.output()).run().get(0);
+      Tensor<Integer> result =
+          sess.runner().fetch(var1.output()).run().get(0).expect(Integer.class);
       assertEquals(21704, result.intValue());
-      result = sess.runner().fetch(var2.output()).run().get(0);
+      result = sess.runner().fetch(var2.output()).run().get(0).expect(Integer.class);
       assertEquals(21704, result.intValue());
     }
   }
 
   // "handwritten" sample operator classes
-  private static final class Const {
-    private final Output output;
+  private static final class Const<T> {
+    private final Output<T> output;
 
-    static Const create(Scope s, Object v) {
-      try (Tensor value = Tensor.create(v)) {
-        return new Const(
+    static Const<Integer> create(Scope s, int v) {
+      return create(s, Tensors.create(v));
+    }
+
+    static Const<Integer> create(Scope s, int[] v) {
+      return create(s, Tensors.create(v));
+    }
+
+    static <T> Const<T> create(Scope s, Tensor<T> value) {
+      return new Const<T>(
+          s.graph()
+              .opBuilder("Const", s.makeOpName("Const"))
+              .setAttr("dtype", value.dataType())
+              .setAttr("value", value)
+              .build()
+              .<T>output(0));
+    }
+
+    static <T> Const<T> create(Scope s, Object v, Class<T> type) {
+      try (Tensor<T> value = Tensor.create(v, type)) {
+        return new Const<T>(
             s.graph()
                 .opBuilder("Const", s.makeOpName("Const"))
                 .setAttr("dtype", value.dataType())
                 .setAttr("value", value)
                 .build()
-                .output(0));
+                .<T>output(0));
       }
     }
 
-    Const(Output o) {
+    Const(Output<T> o) {
       output = o;
     }
 
-    Output output() {
+    Output<T> output() {
       return output;
     }
   }
 
-  private static final class Mean {
-    private final Output output;
+  private static final class Mean<T> {
+    private final Output<T> output;
 
-    static Mean create(Scope s, Output input, Output reductionIndices) {
-      return new Mean(
+    static <T> Mean<T> create(Scope s, Output<T> input, Output<T> reductionIndices) {
+      return new Mean<T>(
           s.graph()
               .opBuilder("Mean", s.makeOpName("Mean"))
               .addInput(input)
               .addInput(reductionIndices)
               .build()
-              .output(0));
+              .<T>output(0));
     }
 
-    Mean(Output o) {
+    Mean(Output<T> o) {
       output = o;
     }
 
-    Output output() {
+    Output<T> output() {
       return output;
     }
   }
 
-  private static final class SquaredDifference {
-    private final Output output;
+  private static final class SquaredDifference<T> {
+    private final Output<T> output;
 
-    static SquaredDifference create(Scope s, Output x, Output y) {
-      return new SquaredDifference(
+    static <T> SquaredDifference<T> create(Scope s, Output<T> x, Output<T> y) {
+      return new SquaredDifference<T>(
           s.graph()
               .opBuilder("SquaredDifference", s.makeOpName("SquaredDifference"))
               .addInput(x)
               .addInput(y)
               .build()
-              .output(0));
+              .<T>output(0));
     }
 
-    SquaredDifference(Output o) {
+    SquaredDifference(Output<T> o) {
       output = o;
     }
 
-    Output output() {
+    Output<T> output() {
       return output;
     }
   }
 
-  private static final class Variance {
-    private final Output output;
+  /**
+   * Returns the zero value of type described by {@code c}, or null if the type (e.g., string) is
+   * not numeric and therefore has no zero value.
+   *
+   * @param c The class describing the TensorFlow type of interest.
+   */
+  public static Object zeroValue(Class<?> c) {
+    return zeros.get(c);
+  }
+
+  private static final Map<Class<?>, Object> zeros = new HashMap<>();
+
+  static {
+    zeros.put(Float.class, 0.0f);
+    zeros.put(Double.class, 0.0);
+    zeros.put(Integer.class, 0);
+    zeros.put(UInt8.class, (byte) 0);
+    zeros.put(Long.class, 0L);
+    zeros.put(Boolean.class, false);
+    zeros.put(String.class, null); // no zero value
+  }
+
+  private static final class Variance<T> {
+    private final Output<T> output;
 
-    static Variance create(Scope base, Output x) {
+    static <T> Variance<T> create(Scope base, Output<T> x, Class<T> type) {
       Scope s = base.withSubScope("variance");
-      Output zero = Const.create(s.withName("zero"), new int[] {0}).output();
-      Output sqdiff =
+      Output<T> zero = Const.create(base, zeroValue(type), type).output();
+      Output<T> sqdiff =
           SquaredDifference.create(
                   s.withName("squared_deviation"), x, Mean.create(s, x, zero).output())
               .output();
 
-      return new Variance(Mean.create(s.withName("variance"), sqdiff, zero).output());
+      return new Variance<T>(Mean.create(s.withName("variance"), sqdiff, zero).output());
     }
 
-    Variance(Output o) {
+    Variance(Output<T> o) {
       output = o;
     }
 
-    Output output() {
+    Output<T> output() {
       return output;
     }
   }
diff --git a/tensorflow/java/src/test/java/org/tensorflow/op/core/ConstantTest.java b/tensorflow/java/src/test/java/org/tensorflow/op/core/ConstantTest.java
index ec23792485..ca54214e06 100644
--- a/tensorflow/java/src/test/java/org/tensorflow/op/core/ConstantTest.java
+++ b/tensorflow/java/src/test/java/org/tensorflow/op/core/ConstantTest.java
@@ -29,7 +29,6 @@ import java.nio.LongBuffer;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
-import org.tensorflow.DataType;
 import org.tensorflow.Graph;
 import org.tensorflow.Session;
 import org.tensorflow.Tensor;
@@ -47,8 +46,9 @@ public class ConstantTest {
     try (Graph g = new Graph();
         Session sess = new Session(g)) {
       Scope scope = new Scope(g);
-      Constant op = Constant.create(scope, shape, IntBuffer.wrap(ints));
-      Tensor result = sess.runner().fetch(op.asOutput()).run().get(0);
+      Constant<Integer> op = Constant.create(scope, shape, IntBuffer.wrap(ints));
+      Tensor<Integer> result = sess.runner().fetch(op.asOutput())
+          .run().get(0).expect(Integer.class);
       int[] actual = new int[ints.length];
       assertArrayEquals(ints, result.copyTo(actual));
     }
@@ -62,8 +62,8 @@ public class ConstantTest {
     try (Graph g = new Graph();
         Session sess = new Session(g)) {
       Scope scope = new Scope(g);
-      Constant op = Constant.create(scope, shape, FloatBuffer.wrap(floats));
-      Tensor result = sess.runner().fetch(op.asOutput()).run().get(0);
+      Constant<Float> op = Constant.create(scope, shape, FloatBuffer.wrap(floats));
+      Tensor<Float> result = sess.runner().fetch(op.asOutput()).run().get(0).expect(Float.class);
       float[] actual = new float[floats.length];
       assertArrayEquals(floats, result.copyTo(actual), EPSILON);
     }
@@ -77,8 +77,8 @@ public class ConstantTest {
     try (Graph g = new Graph();
         Session sess = new Session(g)) {
       Scope scope = new Scope(g);
-      Constant op = Constant.create(scope, shape, DoubleBuffer.wrap(doubles));
-      Tensor result = sess.runner().fetch(op.asOutput()).run().get(0);
+      Constant<Double> op = Constant.create(scope, shape, DoubleBuffer.wrap(doubles));
+      Tensor<Double> result = sess.runner().fetch(op.asOutput()).run().get(0).expect(Double.class);
       double[] actual = new double[doubles.length];
       assertArrayEquals(doubles, result.copyTo(actual), EPSILON);
     }
@@ -92,8 +92,8 @@ public class ConstantTest {
     try (Graph g = new Graph();
         Session sess = new Session(g)) {
       Scope scope = new Scope(g);
-      Constant op = Constant.create(scope, shape, LongBuffer.wrap(longs));
-      Tensor result = sess.runner().fetch(op.asOutput()).run().get(0);
+      Constant<Long> op = Constant.create(scope, shape, LongBuffer.wrap(longs));
+      Tensor<Long> result = sess.runner().fetch(op.asOutput()).run().get(0).expect(Long.class);
       long[] actual = new long[longs.length];
       assertArrayEquals(longs, result.copyTo(actual));
     }
@@ -123,8 +123,8 @@ public class ConstantTest {
     try (Graph g = new Graph();
         Session sess = new Session(g)) {
       Scope scope = new Scope(g);
-      Constant op = Constant.create(scope, DataType.STRING, shape, ByteBuffer.wrap(content));
-      Tensor result = sess.runner().fetch(op.asOutput()).run().get(0);
+      Constant<String> op = Constant.create(scope, String.class, shape, ByteBuffer.wrap(content));
+      Tensor<String> result = sess.runner().fetch(op.asOutput()).run().get(0).expect(String.class);
       assertArrayEquals(data, result.bytesValue());
     }
   }
diff --git a/tensorflow/python/debug/lib/debug_graphs.py b/tensorflow/python/debug/lib/debug_graphs.py
index 486e659158..87033d53a4 100644
--- a/tensorflow/python/debug/lib/debug_graphs.py
+++ b/tensorflow/python/debug/lib/debug_graphs.py
@@ -231,8 +231,8 @@ def _infer_device_name(graph_def):
       break
   if device_name is None:
     logging.warn(
-        "Failed to infer device name from partiton GraphDef: none of the nodes "
-        "of the GraphDef has a non-empty device name.")
+        "Failed to infer device name from partition GraphDef: none of the "
+        "nodes of the GraphDef has a non-empty device name.")
   return device_name
 
 
diff --git a/tensorflow/python/estimator/inputs/queues/feeding_functions.py b/tensorflow/python/estimator/inputs/queues/feeding_functions.py
index d7fe4bbfa1..c0a287e922 100644
--- a/tensorflow/python/estimator/inputs/queues/feeding_functions.py
+++ b/tensorflow/python/estimator/inputs/queues/feeding_functions.py
@@ -49,7 +49,7 @@ except ImportError:
 def _fill_array(arr, seq, fillvalue=0):
   """ 
   Recursively fills padded arr with elements from seq. 
-  If lenght of seq is less then arr padded length, fillvalue used.
+  If length of seq is less than arr padded length, fillvalue used.
 
   Args:
     arr: Padded tensor of shape [batch_size, ..., max_padded_dim_len].
diff --git a/tensorflow/python/keras/_impl/keras/engine/topology_test.py b/tensorflow/python/keras/_impl/keras/engine/topology_test.py
index 97bef2965c..32e692ba7c 100644
--- a/tensorflow/python/keras/_impl/keras/engine/topology_test.py
+++ b/tensorflow/python/keras/_impl/keras/engine/topology_test.py
@@ -200,7 +200,7 @@ class TopologyConstructionTest(test.TestCase):
     with self.assertRaises(ValueError):
       _ = keras.layers.Input(shape=(32,), batch_shape=(10, 32))
     with self.assertRaises(ValueError):
-      _ = keras.layers.Input(shape=(32,), unknwon_kwarg=None)
+      _ = keras.layers.Input(shape=(32,), unknown_kwarg=None)
 
     self.assertListEqual(a.get_shape().as_list(), [None, 32])
     a_layer, a_node_index, a_tensor_index = a._keras_history
diff --git a/tensorflow/python/kernel_tests/conv2d_transpose_test.py b/tensorflow/python/kernel_tests/conv2d_transpose_test.py
index 18184a0ee0..7d0bc54b69 100644
--- a/tensorflow/python/kernel_tests/conv2d_transpose_test.py
+++ b/tensorflow/python/kernel_tests/conv2d_transpose_test.py
@@ -24,8 +24,12 @@ from six.moves import xrange  # pylint: disable=redefined-builtin
 from tensorflow.python.client import device_lib
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
+from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gradient_checker
 from tensorflow.python.ops import nn_ops
+from tensorflow.python.ops import random_ops
+from tensorflow.python.ops import variable_scope
+from tensorflow.python.ops import variables
 import tensorflow.python.ops.nn_grad  # pylint: disable=unused-import
 from tensorflow.python.platform import test
 
@@ -289,6 +293,16 @@ class Conv2DTransposeTest(test.TestCase):
 
         self.assertAllClose(cache_values, value)
 
+  def testConv2DTransposeShapeInference(self):
+    # Test case for 8972
+    initializer = random_ops.truncated_normal(
+        [3, 3, 5, 1], mean=0.0, stddev=0.01, dtype=dtypes.float32)
+    x = variables.Variable(random_ops.random_normal([3, 10, 5, 1]))
+    f = variable_scope.get_variable("f", initializer=initializer)
+    f_shape = array_ops.stack([array_ops.shape(x)[0], 10, 5, 5])
+    output = nn_ops.conv2d_transpose(
+        x, f, f_shape, strides=[1, 1, 1, 1], padding="SAME")
+    self.assertEqual(output.get_shape().as_list(), [None, 10, 5, 5])
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/kernel_tests/decode_csv_op_test.py b/tensorflow/python/kernel_tests/decode_csv_op_test.py
index 3853379328..7d9e57c8e5 100644
--- a/tensorflow/python/kernel_tests/decode_csv_op_test.py
+++ b/tensorflow/python/kernel_tests/decode_csv_op_test.py
@@ -116,6 +116,17 @@ class DecodeCSVOpTest(test.TestCase):
 
     self._test(args, expected_out)
 
+  def testNA(self):
+    args = {
+        "records": ["2.0,NA,aa", "NA,5,bb", "3,6,NA"],
+        "record_defaults": [[0.0], [0], [""]],
+        "na_value": "NA"
+    }
+
+    expected_out = [[2.0, 0.0, 3], [0, 5, 6], [b"aa", b"bb", b""]]
+
+    self._test(args, expected_out)
+
   def testWithDefaults(self):
     args = {
         "records": [",1,", "0.2,3,bcd", "3.0,,"],
diff --git a/tensorflow/python/kernel_tests/summary_tensor_op_test.py b/tensorflow/python/kernel_tests/summary_tensor_op_test.py
index 3584637865..d534aadb79 100644
--- a/tensorflow/python/kernel_tests/summary_tensor_op_test.py
+++ b/tensorflow/python/kernel_tests/summary_tensor_op_test.py
@@ -154,7 +154,7 @@ class SummaryOpsTest(test.TestCase):
       self.assertEqual(descr.display_name, "my name")
       self.assertEqual(descr.summary_description, "my description")
 
-      # If both SummmaryMetadata and explicit args are provided, the args win
+      # If both SummaryMetadata and explicit args are provided, the args win
       overwrite = summary_ops.tensor_summary(
           "simple",
           const,
diff --git a/tensorflow/python/ops/hidden_ops.txt b/tensorflow/python/ops/hidden_ops.txt
index 6e7122db5e..d27e867583 100644
--- a/tensorflow/python/ops/hidden_ops.txt
+++ b/tensorflow/python/ops/hidden_ops.txt
@@ -207,6 +207,7 @@ TextLineReaderV2
 TFRecordReaderV2
 WholeFileReaderV2
 LMDBReader
+DecodeCSV
 
 # linalg_ops
 BatchCholesky
diff --git a/tensorflow/python/ops/parsing_ops.py b/tensorflow/python/ops/parsing_ops.py
index c5fd15bae4..ea7132791c 100644
--- a/tensorflow/python/ops/parsing_ops.py
+++ b/tensorflow/python/ops/parsing_ops.py
@@ -1166,3 +1166,42 @@ def _parse_single_sequence_example_raw(serialized,
             feature_list_sparse_tensors + feature_list_dense_values))
 
     return (context_output, feature_list_output)
+
+
+# Swap `name` and `na_value` for backward compatibility.
+def decode_csv(records, record_defaults, field_delim=",",
+               use_quote_delim=True, name=None, na_value=""):
+  # pylint: disable=protected-access
+  """Convert CSV records to tensors. Each column maps to one tensor.
+
+  RFC 4180 format is expected for the CSV records.
+  (https://tools.ietf.org/html/rfc4180)
+  Note that we allow leading and trailing spaces with int or float field.
+
+  Args:
+    records: A `Tensor` of type `string`.
+      Each string is a record/row in the csv and all records should have
+      the same format.
+    record_defaults: A list of `Tensor` objects with specific types.
+      Acceptable types are `float32`, `int32`, `int64`, `string`.
+      One tensor per column of the input record, with either a
+      scalar default value for that column or empty if the column is required.
+    field_delim: An optional `string`. Defaults to `","`.
+      char delimiter to separate fields in a record.
+    use_quote_delim: An optional `bool`. Defaults to `True`.
+      If false, treats double quotation marks as regular
+      characters inside of the string fields (ignoring RFC 4180, Section 2,
+      Bullet 5).
+    name: A name for the operation (optional).
+    na_value: Additional string to recognize as NA/NaN.
+
+  Returns:
+    A list of `Tensor` objects. Has the same type as `record_defaults`.
+    Each tensor will have the same shape as records.
+  """
+  # TODO(martinwicke), remove the wrapper when new Python API generator is done.
+  return gen_parsing_ops._decode_csv(
+      records=records, record_defaults=record_defaults,
+      field_delim=field_delim, use_quote_delim=use_quote_delim,
+      na_value=na_value, name=name)
+  # pylint: enable=protected-access
diff --git a/tensorflow/stream_executor/cuda/cuda_dnn.cc b/tensorflow/stream_executor/cuda/cuda_dnn.cc
index bf8380ebbd..0a1a748c40 100644
--- a/tensorflow/stream_executor/cuda/cuda_dnn.cc
+++ b/tensorflow/stream_executor/cuda/cuda_dnn.cc
@@ -562,7 +562,7 @@ static bool TensorOpMathEnabled() {
     bool ret;
     TF_CHECK_OK(tensorflow::ReadBoolFromEnvVar("TF_DISABLE_TENSOR_OP_MATH",
                                                /*default=*/false, &ret));
-    return ret;
+    return !ret;
   }();
   return is_enabled;
 }
@@ -2474,58 +2474,73 @@ struct WinogradNonfused {
 };
 
 bool CudnnSupport::GetConvolveAlgorithms(
-    bool with_winograd_nonfused,
-    std::vector<dnn::AlgorithmDesc::Index>* out_algorithms) {
-  out_algorithms->assign({
-      // clang-format off
-      CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_GEMM,
-      CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_PRECOMP_GEMM,
-      CUDNN_CONVOLUTION_FWD_ALGO_GEMM,
-      CUDNN_CONVOLUTION_FWD_ALGO_DIRECT,
-      CUDNN_CONVOLUTION_FWD_ALGO_FFT,
+    bool with_winograd_nonfused, int cc_major, int cc_minor,
+    std::vector<dnn::AlgorithmDesc>* out_algorithms) {
+  std::vector<dnn::AlgorithmDesc::Index> algo_types = {
+    // clang-format off
+    CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_GEMM,
+    CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_PRECOMP_GEMM,
+    CUDNN_CONVOLUTION_FWD_ALGO_GEMM,
+    CUDNN_CONVOLUTION_FWD_ALGO_DIRECT,
+    CUDNN_CONVOLUTION_FWD_ALGO_FFT,
 #if CUDNN_VERSION >= 5000
-      CUDNN_CONVOLUTION_FWD_ALGO_WINOGRAD,
+    CUDNN_CONVOLUTION_FWD_ALGO_WINOGRAD,
 #endif
-      // clang-format on
-  });
+    // clang-format on
+  };
   if (CudnnEnvVar<FftTilingForward>::IsEnabled()) {
-    out_algorithms->push_back(CUDNN_CONVOLUTION_FWD_ALGO_FFT_TILING);
+    algo_types.push_back(CUDNN_CONVOLUTION_FWD_ALGO_FFT_TILING);
   }
 #if CUDNN_VERSION >= 5100
   if (CudnnEnvVar<WinogradNonfused>::IsEnabled() && with_winograd_nonfused) {
-    out_algorithms->push_back(CUDNN_CONVOLUTION_FWD_ALGO_WINOGRAD_NONFUSED);
+    algo_types.push_back(CUDNN_CONVOLUTION_FWD_ALGO_WINOGRAD_NONFUSED);
   }
 #endif
+
+  out_algorithms->clear();
+  for (auto i : algo_types) {
+    out_algorithms->push_back({i, /*use_tensor_ops=*/false});
+    if (cc_major >= 7 && CUDNN_VERSION >= 7000 && TensorOpMathEnabled()) {
+      out_algorithms->push_back({i, /*use_tensor_ops=*/true});
+    }
+  }
   return true;
 }
 
 bool CudnnSupport::GetConvolveBackwardDataAlgorithms(
-    bool with_winograd_nonfused,
-    std::vector<dnn::AlgorithmDesc::Index>* out_algorithms) {
-  out_algorithms->assign({
-      // clang-format off
-      CUDNN_CONVOLUTION_BWD_DATA_ALGO_0,
-      CUDNN_CONVOLUTION_BWD_DATA_ALGO_1,
-      CUDNN_CONVOLUTION_BWD_DATA_ALGO_FFT,
-      CUDNN_CONVOLUTION_BWD_DATA_ALGO_FFT_TILING,
+    bool with_winograd_nonfused, int cc_major, int cc_minor,
+    std::vector<dnn::AlgorithmDesc>* out_algorithms) {
+  std::vector<dnn::AlgorithmDesc::Index> algo_types = {
+    // clang-format off
+    CUDNN_CONVOLUTION_BWD_DATA_ALGO_0,
+    CUDNN_CONVOLUTION_BWD_DATA_ALGO_1,
+    CUDNN_CONVOLUTION_BWD_DATA_ALGO_FFT,
+    CUDNN_CONVOLUTION_BWD_DATA_ALGO_FFT_TILING,
 #if CUDNN_VERSION >= 5000
-      CUDNN_CONVOLUTION_BWD_DATA_ALGO_WINOGRAD,
+    CUDNN_CONVOLUTION_BWD_DATA_ALGO_WINOGRAD,
 #endif
-      // clang-format on
-  });
+    // clang-format on
+  };
 #if CUDNN_VERSION >= 5100
   if (CudnnEnvVar<WinogradNonfused>::IsEnabled() && with_winograd_nonfused) {
-    out_algorithms->push_back(
-        CUDNN_CONVOLUTION_BWD_DATA_ALGO_WINOGRAD_NONFUSED);
+    algo_types.push_back(CUDNN_CONVOLUTION_BWD_DATA_ALGO_WINOGRAD_NONFUSED);
   }
 #endif
+
+  out_algorithms->clear();
+  for (auto i : algo_types) {
+    out_algorithms->push_back({i, /*use_tensor_ops=*/false});
+    if (cc_major >= 7 && CUDNN_VERSION >= 7000 && TensorOpMathEnabled()) {
+      out_algorithms->push_back({i, /*use_tensor_ops=*/true});
+    }
+  }
   return true;
 }
 
 bool CudnnSupport::GetConvolveBackwardFilterAlgorithms(
-    bool with_winograd_nonfused,
-    std::vector<dnn::AlgorithmDesc::Index>* out_algorithms) {
-  out_algorithms->assign({
+    bool with_winograd_nonfused, int cc_major, int cc_minor,
+    std::vector<dnn::AlgorithmDesc>* out_algorithms) {
+  std::vector<dnn::AlgorithmDesc::Index> algo_types = {
       // clang-format off
       CUDNN_CONVOLUTION_BWD_FILTER_ALGO_0,
       CUDNN_CONVOLUTION_BWD_FILTER_ALGO_1,
@@ -2534,13 +2549,20 @@ bool CudnnSupport::GetConvolveBackwardFilterAlgorithms(
       // Based on cudnn.h, the following is not implemented.
       // CUDNN_CONVOLUTION_BWD_FILTER_ALGO_WINOGRAD,
       // clang-format on
-  });
+  };
 #if CUDNN_VERSION >= 5110
   if (CudnnEnvVar<WinogradNonfused>::IsEnabled() && with_winograd_nonfused) {
-    out_algorithms->push_back(
-        CUDNN_CONVOLUTION_BWD_FILTER_ALGO_WINOGRAD_NONFUSED);
+    algo_types.push_back(CUDNN_CONVOLUTION_BWD_FILTER_ALGO_WINOGRAD_NONFUSED);
   }
 #endif
+
+  out_algorithms->clear();
+  for (auto i : algo_types) {
+    out_algorithms->push_back({i, /*use_tensor_ops=*/false});
+    if (cc_major >= 7 && CUDNN_VERSION >= 7000 && TensorOpMathEnabled()) {
+      out_algorithms->push_back({i, /*use_tensor_ops=*/true});
+    }
+  }
   return true;
 }
 
diff --git a/tensorflow/stream_executor/cuda/cuda_dnn.h b/tensorflow/stream_executor/cuda/cuda_dnn.h
index beb2f7d050..8d7069a902 100644
--- a/tensorflow/stream_executor/cuda/cuda_dnn.h
+++ b/tensorflow/stream_executor/cuda/cuda_dnn.h
@@ -145,16 +145,16 @@ class CudnnSupport : public dnn::DnnSupport {
                      ScratchAllocator* workspace_allocator) override;
 
   bool GetConvolveAlgorithms(
-      bool with_winograd_nonfused,
-      std::vector<dnn::AlgorithmDesc::Index>* out_algorithms) override;
+      bool with_winograd_nonfused, int cc_major, int cc_minor,
+      std::vector<dnn::AlgorithmDesc>* out_algorithms) override;
 
   bool GetConvolveBackwardDataAlgorithms(
-      bool with_winograd_nonfused,
-      std::vector<dnn::AlgorithmDesc::Index>* out_algorithms) override;
+      bool with_winograd_nonfused, int cc_major, int cc_minor,
+      std::vector<dnn::AlgorithmDesc>* out_algorithms) override;
 
   bool GetConvolveBackwardFilterAlgorithms(
-      bool with_winograd_nonfused,
-      std::vector<dnn::AlgorithmDesc::Index>* out_algorithms) override;
+      bool with_winograd_nonfused, int cc_major, int cc_minor,
+      std::vector<dnn::AlgorithmDesc>* out_algorithms) override;
 
   bool DoBatchNormalizationForward(
       Stream* stream, const DeviceMemory<float>& x,
diff --git a/tensorflow/stream_executor/dnn.cc b/tensorflow/stream_executor/dnn.cc
index 2c40e18f5c..07fe8a85f4 100644
--- a/tensorflow/stream_executor/dnn.cc
+++ b/tensorflow/stream_executor/dnn.cc
@@ -23,20 +23,20 @@ namespace gputools {
 namespace dnn {
 
 bool DnnSupport::GetConvolveAlgorithms(
-    bool with_winograd_nonfused,
-    std::vector<AlgorithmDesc::Index>* out_algorithms) {
+    bool with_winograd_nonfused, int cc_major, int cc_minor,
+    std::vector<AlgorithmDesc>* out_algorithms) {
   return false;
 }
 
 bool DnnSupport::GetConvolveBackwardDataAlgorithms(
-    bool with_winograd_nonfused,
-    std::vector<AlgorithmDesc::Index>* out_algorithms) {
+    bool with_winograd_nonfused, int cc_major, int cc_minor,
+    std::vector<AlgorithmDesc>* out_algorithms) {
   return false;
 }
 
 bool DnnSupport::GetConvolveBackwardFilterAlgorithms(
-    bool with_winograd_nonfused,
-    std::vector<AlgorithmDesc::Index>* out_algorithms) {
+    bool with_winograd_nonfused, int cc_major, int cc_minor,
+    std::vector<AlgorithmDesc>* out_algorithms) {
   return false;
 }
 
diff --git a/tensorflow/stream_executor/dnn.h b/tensorflow/stream_executor/dnn.h
index 5fe523602a..624357b82f 100644
--- a/tensorflow/stream_executor/dnn.h
+++ b/tensorflow/stream_executor/dnn.h
@@ -1183,8 +1183,8 @@ class DnnSupport {
 
   // Return a list of algorithms supported by the forward convolution pass.
   virtual bool GetConvolveAlgorithms(
-      bool with_winograd_nonfused,
-      std::vector<AlgorithmDesc::Index>* out_algorithms);
+      bool with_winograd_nonfused, int cc_major, int cc_minor,
+      std::vector<AlgorithmDesc>* out_algorithms);
 
   // Version of DoConvolve that uses pre-quantized 8 bit coefficients.
   // coefficient_scales specifies the scaling of each column of coefficients:
@@ -1263,8 +1263,8 @@ class DnnSupport {
   // Return a list of algorithms supported by the backward convolution pass for
   // data.
   virtual bool GetConvolveBackwardDataAlgorithms(
-      bool with_winograd_nonfused,
-      std::vector<AlgorithmDesc::Index>* out_algorithms);
+      bool with_winograd_nonfused, int cc_major, int cc_minor,
+      std::vector<AlgorithmDesc>* out_algorithms);
 
   virtual bool DoConvolveBackwardData(
       Stream* stream, const FilterDescriptor& filter_descriptor,
@@ -1312,8 +1312,8 @@ class DnnSupport {
   // Return a list of algorithms supported by the backward convolution pass for
   // filters.
   virtual bool GetConvolveBackwardFilterAlgorithms(
-      bool with_winograd_nonfused,
-      std::vector<AlgorithmDesc::Index>* out_algorithms);
+      bool with_winograd_nonfused, int cc_major, int cc_minor,
+      std::vector<AlgorithmDesc>* out_algorithms);
 
   virtual bool DoConvolveBackwardFilter(
       Stream* stream, const BatchDescriptor& input_descriptor,
diff --git a/tensorflow/stream_executor/platform.h b/tensorflow/stream_executor/platform.h
index ed12982e30..f0a0e60e02 100644
--- a/tensorflow/stream_executor/platform.h
+++ b/tensorflow/stream_executor/platform.h
@@ -96,7 +96,7 @@ class Platform {
   // each platform is required to expose an ID to ensure unique registration and
   // as a target against which plugins can register.
   //
-  // The macro below is provided to help generate a [process-unique] identifer.
+  // The macro below is provided to help generate a [process-unique] identifier.
   using Id = void*;
 
 // Helper macro to define a plugin ID. To be used only inside plugin
diff --git a/tensorflow/stream_executor/stream.h b/tensorflow/stream_executor/stream.h
index a72ee804c1..21172d5a16 100644
--- a/tensorflow/stream_executor/stream.h
+++ b/tensorflow/stream_executor/stream.h
@@ -70,7 +70,7 @@ class BatchDescriptor;
 class FilterDescriptor;
 class ConvolutionDescriptor;
 class ProfileResult;
-struct AlgorithmDesc;
+class AlgorithmDesc;
 }  // namespace dnn
 
 class StreamExecutor;
diff --git a/tensorflow/stream_executor/stream_executor_pimpl.cc b/tensorflow/stream_executor/stream_executor_pimpl.cc
index 199a908914..9bbfe7f04a 100644
--- a/tensorflow/stream_executor/stream_executor_pimpl.cc
+++ b/tensorflow/stream_executor/stream_executor_pimpl.cc
@@ -286,35 +286,41 @@ bool StreamExecutor::SupportsDnn() const {
 
 bool StreamExecutor::GetConvolveAlgorithms(
     bool with_winograd_nonfused,
-    std::vector<dnn::AlgorithmDesc::Index> *out_algorithms) {
+    std::vector<dnn::AlgorithmDesc> *out_algorithms) {
   dnn::DnnSupport *dnn_support = AsDnn();
   if (!dnn_support) {
     return false;
   }
-  return dnn_support->GetConvolveAlgorithms(with_winograd_nonfused,
-                                            out_algorithms);
+  int cc_major, cc_minor;
+  GetDeviceDescription().cuda_compute_capability(&cc_major, &cc_minor);
+  return dnn_support->GetConvolveAlgorithms(with_winograd_nonfused, cc_major,
+                                            cc_minor, out_algorithms);
 }
 
 bool StreamExecutor::GetConvolveBackwardDataAlgorithms(
     bool with_winograd_nonfused,
-    std::vector<dnn::AlgorithmDesc::Index> *out_algorithms) {
+    std::vector<dnn::AlgorithmDesc> *out_algorithms) {
   dnn::DnnSupport *dnn_support = AsDnn();
   if (!dnn_support) {
     return false;
   }
-  return dnn_support->GetConvolveBackwardDataAlgorithms(with_winograd_nonfused,
-                                                        out_algorithms);
+  int cc_major, cc_minor;
+  GetDeviceDescription().cuda_compute_capability(&cc_major, &cc_minor);
+  return dnn_support->GetConvolveBackwardDataAlgorithms(
+      with_winograd_nonfused, cc_major, cc_minor, out_algorithms);
 }
 
 bool StreamExecutor::GetConvolveBackwardFilterAlgorithms(
     bool with_winograd_nonfused,
-    std::vector<dnn::AlgorithmDesc::Index> *out_algorithms) {
+    std::vector<dnn::AlgorithmDesc> *out_algorithms) {
   dnn::DnnSupport *dnn_support = AsDnn();
   if (!dnn_support) {
     return false;
   }
+  int cc_major, cc_minor;
+  GetDeviceDescription().cuda_compute_capability(&cc_major, &cc_minor);
   return dnn_support->GetConvolveBackwardFilterAlgorithms(
-      with_winograd_nonfused, out_algorithms);
+      with_winograd_nonfused, cc_major, cc_minor, out_algorithms);
 }
 
 bool StreamExecutor::GetBlasGemmAlgorithms(
diff --git a/tensorflow/stream_executor/stream_executor_pimpl.h b/tensorflow/stream_executor/stream_executor_pimpl.h
index 98136a92a0..f354317a6e 100644
--- a/tensorflow/stream_executor/stream_executor_pimpl.h
+++ b/tensorflow/stream_executor/stream_executor_pimpl.h
@@ -343,20 +343,19 @@ class StreamExecutor {
   bool SupportsDnn() const;
 
   // Get the list of supported algorithms for the forward convolution opeartion.
-  bool GetConvolveAlgorithms(
-      bool with_winograd_nonfused,
-      std::vector<dnn::AlgorithmDesc::Index> *out_algorithms);
+  bool GetConvolveAlgorithms(bool with_winograd_nonfused,
+                             std::vector<dnn::AlgorithmDesc> *out_algorithms);
 
   // Get the list of supported algorithms for the backward convolution on data.
   bool GetConvolveBackwardDataAlgorithms(
       bool with_winograd_nonfused,
-      std::vector<dnn::AlgorithmDesc::Index> *out_algorithms);
+      std::vector<dnn::AlgorithmDesc> *out_algorithms);
 
   // Get the list of supported algorithms for the backward convolution on the
   // filter.
   bool GetConvolveBackwardFilterAlgorithms(
       bool with_winograd_nonfused,
-      std::vector<dnn::AlgorithmDesc::Index> *out_algorithms);
+      std::vector<dnn::AlgorithmDesc> *out_algorithms);
 
   // Get the list of supported algorithms for BLAS gemm.
   bool GetBlasGemmAlgorithms(std::vector<blas::AlgorithmType> *out_algorithms);
diff --git a/tensorflow/tensorflow.bzl b/tensorflow/tensorflow.bzl
index a308688790..0f074151db 100644
--- a/tensorflow/tensorflow.bzl
+++ b/tensorflow/tensorflow.bzl
@@ -526,6 +526,7 @@ def tf_cc_test(name,
                extra_copts=[],
                suffix="",
                linkopts=[],
+               nocopts=None,
                **kwargs):
   native.cc_test(
       name="%s%s" % (name, suffix),
@@ -547,6 +548,7 @@ def tf_cc_test(name,
           clean_dep("//tensorflow:darwin"): 1,
           "//conditions:default": 0,
       }),
+      nocopts=nocopts,
       **kwargs)
 
 
@@ -649,7 +651,8 @@ def tf_cc_tests(srcs,
                 tags=[],
                 size="medium",
                 args=None,
-                linkopts=[]):
+                linkopts=[],
+                nocopts=None):
   for src in srcs:
     tf_cc_test(
         name=src_to_test_name(src),
@@ -659,7 +662,8 @@ def tf_cc_tests(srcs,
         tags=tags,
         size=size,
         args=args,
-        linkopts=linkopts)
+        linkopts=linkopts,
+        nocopts=nocopts)
 
 
 def tf_cc_test_mkl(srcs,
@@ -669,7 +673,7 @@ def tf_cc_test_mkl(srcs,
                    tags=[],
                    size="medium",
                    args=None):
-  if_mkl(tf_cc_tests(srcs, deps, linkstatic, tags=tags, size=size, args=args))
+  if_mkl(tf_cc_tests(srcs, deps, name, linkstatic=linkstatic, tags=tags, size=size, args=args, nocopts="-fno-exceptions"))
 
 
 def tf_cc_tests_gpu(srcs,
@@ -867,18 +871,33 @@ def tf_mkl_kernel_library(name,
                           deps=None,
                           alwayslink=1,
                           copts=tf_copts(),
+                          nocopts="-fno-exceptions",
                           **kwargs):
+  """A rule to build MKL-based TensorFlow kernel libraries."""
+  gpu_srcs = gpu_srcs  # unused argument
+  kwargs = kwargs  # unused argument
+
+  if not bool(srcs):
+    srcs = []
+  if not bool(hdrs):
+    hdrs = []
+
+  if prefix:
+    srcs = srcs + native.glob(
+        [prefix + "*.cc"])
+    hdrs = hdrs + native.glob(
+        [prefix + "*.h"])
+
   if_mkl(
-      tf_kernel_library(
-          name,
-          prefix=prefix,
+      native.cc_library(
+          name=name,
           srcs=srcs,
-          gpu_srcs=gpu_srcs,
           hdrs=hdrs,
           deps=deps,
           alwayslink=alwayslink,
           copts=copts,
-          **kwargs))
+          nocopts=nocopts
+      ))
 
 
 # Bazel rules for building swig files.
diff --git a/tensorflow/tools/api/golden/tensorflow.pbtxt b/tensorflow/tools/api/golden/tensorflow.pbtxt
index 32a86e420a..6e03f9e8fb 100644
--- a/tensorflow/tools/api/golden/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.pbtxt
@@ -874,7 +874,7 @@ tf_module {
   }
   member_method {
     name: "decode_csv"
-    argspec: "args=[\'records\', \'record_defaults\', \'field_delim\', \'use_quote_delim\', \'name\'], varargs=None, keywords=None, defaults=[\',\', \'True\', \'None\'], "
+    argspec: "args=[\'records\', \'record_defaults\', \'field_delim\', \'use_quote_delim\', \'name\', \'na_value\'], varargs=None, keywords=None, defaults=[\',\', \'True\', \'None\', \'\'], "
   }
   member_method {
     name: "decode_json_example"
diff --git a/tensorflow/tools/ci_build/install/install_golang.sh b/tensorflow/tools/ci_build/install/install_golang.sh
index 88bc2960e3..596265b069 100755
--- a/tensorflow/tools/ci_build/install/install_golang.sh
+++ b/tensorflow/tools/ci_build/install/install_golang.sh
@@ -16,7 +16,7 @@
 
 set -ex
 
-GOLANG_URL="https://storage.googleapis.com/golang/go1.8.3.linux-amd64.tar.gz"
+GOLANG_URL="https://storage.googleapis.com/golang/go1.9.linux-amd64.tar.gz"
 
 sudo mkdir -p /usr/local
 wget -q -O - "${GOLANG_URL}" | sudo tar -C /usr/local -xz
diff --git a/tensorflow/tools/docker/Dockerfile.devel-gpu b/tensorflow/tools/docker/Dockerfile.devel-gpu
index f5364d803a..04773376e9 100644
--- a/tensorflow/tools/docker/Dockerfile.devel-gpu
+++ b/tensorflow/tools/docker/Dockerfile.devel-gpu
@@ -78,10 +78,12 @@ WORKDIR /tensorflow
 
 # Configure the build for our CUDA configuration.
 ENV CI_BUILD_PYTHON python
-ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH
+ENV LD_LIBRARY_PATH /usr/local/cuda/lib64/stubs:/usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH
 ENV TF_NEED_CUDA 1
 ENV TF_CUDA_COMPUTE_CAPABILITIES=3.0,3.5,5.2,6.0,6.1
 
+RUN ln -s /usr/local/cuda/lib64/stubs/libcuda.so /usr/local/cuda/lib64/stubs/libcuda.so.1
+
 RUN tensorflow/tools/ci_build/builds/configured GPU \
     bazel build -c opt --config=cuda --cxxopt="-D_GLIBCXX_USE_CXX11_ABI=0" \
         tensorflow/tools/pip_package:build_pip_package && \
diff --git a/tensorflow/tools/docker/jupyter_notebook_config.py b/tensorflow/tools/docker/jupyter_notebook_config.py
index 747beb8251..0acbf6fcee 100644
--- a/tensorflow/tools/docker/jupyter_notebook_config.py
+++ b/tensorflow/tools/docker/jupyter_notebook_config.py
@@ -18,7 +18,6 @@ from IPython.lib import passwd
 c.NotebookApp.ip = '*'
 c.NotebookApp.port = int(os.getenv('PORT', 8888))
 c.NotebookApp.open_browser = False
-c.MultiKernelManager.default_kernel_name = 'python2'
 
 # sets a password if PASSWORD is set in the environment
 if 'PASSWORD' in os.environ:
diff --git a/tensorflow/tools/docs/parser.py b/tensorflow/tools/docs/parser.py
index ca3b778c29..1015103077 100644
--- a/tensorflow/tools/docs/parser.py
+++ b/tensorflow/tools/docs/parser.py
@@ -923,7 +923,7 @@ class _ClassPageInfo(object):
     """Sets the `aliases` list.
 
     Args:
-      aliases: A list of strings. Containing all the obejct's full names.
+      aliases: A list of strings. Containing all the object's full names.
     """
     assert self.aliases is None
     self._aliases = aliases
@@ -1438,7 +1438,7 @@ class _PythonBuiltin(object):
 class _PythonFile(object):
   """This class indicates that the object is defined in a regular python file.
 
-  This can be used for the `defined_in` slot of the `PageInfo` obejcts.
+  This can be used for the `defined_in` slot of the `PageInfo` objects.
   """
 
   def __init__(self, path, parser_config):
diff --git a/tensorflow/tools/proto_text/gen_proto_text_functions_lib_test.cc b/tensorflow/tools/proto_text/gen_proto_text_functions_lib_test.cc
index 81f85e0009..6f0b4f47de 100644
--- a/tensorflow/tools/proto_text/gen_proto_text_functions_lib_test.cc
+++ b/tensorflow/tools/proto_text/gen_proto_text_functions_lib_test.cc
@@ -93,13 +93,15 @@ TEST(CreateProtoDebugStringLibTest, ValidSimpleTypes) {
   proto.set_optional_int64(std::numeric_limits<protobuf_int64>::max());
   proto.set_optional_uint32(std::numeric_limits<uint32>::max());
   proto.set_optional_uint64(std::numeric_limits<uint64>::max());
-  proto.set_optional_float(std::numeric_limits<float>::max());
+  // TODO(b/67475677): Re-enable after resolving float precision issue
+  // proto.set_optional_float(std::numeric_limits<float>::max());
   proto.set_optional_double(std::numeric_limits<double>::max());
   EXPECT_TEXT_TRANSFORMS_MATCH();
 
   // Least positive numeric values.
   proto.Clear();
-  proto.set_optional_float(std::numeric_limits<float>::min());
+  // TODO(b/67475677): Re-enable after resolving float precision issue
+  // proto.set_optional_float(std::numeric_limits<float>::min());
   proto.set_optional_double(std::numeric_limits<double>::min());
   EXPECT_TEXT_TRANSFORMS_MATCH();
 
@@ -107,7 +109,8 @@ TEST(CreateProtoDebugStringLibTest, ValidSimpleTypes) {
   proto.Clear();
   proto.set_optional_int32(std::numeric_limits<int32>::lowest());
   proto.set_optional_int64(std::numeric_limits<protobuf_int64>::lowest());
-  proto.set_optional_float(std::numeric_limits<float>::lowest());
+  // TODO(b/67475677): Re-enable after resolving float precision issue
+  // proto.set_optional_float(std::numeric_limits<float>::lowest());
   proto.set_optional_double(std::numeric_limits<double>::lowest());
   EXPECT_TEXT_TRANSFORMS_MATCH();
 
diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index b226184261..de0084613b 100644
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -170,6 +170,17 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
     print("path_prefix was specified to tf_workspace but is no longer used " +
           "and will be removed in the future.")
 
+  native.new_http_archive(
+      name = "mkl_dnn",
+      urls = [
+          "https://github.com/01org/mkl-dnn/archive/b01e3a55a07be62172e713bcd2644c5176360212.tar.gz",
+          "http://mirror.bazel.build/github.com/01org/mkl-dnn/archive/b01e3a55a07be62172e713bcd2644c5176360212.tar.gz",
+      ],
+      sha256 = "0d529ad4c49dc799e6df07c2b88b115d0668735da15fb3b3862d28d33fa68165",
+      strip_prefix = "mkl-dnn-b01e3a55a07be62172e713bcd2644c5176360212",
+      build_file = str(Label("//third_party/mkl_dnn:mkldnn.BUILD")),
+  )
+
   native.new_http_archive(
       name = "eigen_archive",
       urls = [
@@ -373,10 +384,10 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   patched_http_archive(
       name = "protobuf_archive",
       urls = [
-          "http://mirror.bazel.build/github.com/google/protobuf/archive/0b059a3d8a8f8aa40dde7bea55edca4ec5dfea66.tar.gz",
+          "http://mirror.bazel.build/github.com/google/protobuf/archive/b04e5cba356212e4e8c66c61bbe0c3a20537c5b9.tar.gz",
       ],
-      sha256 = "6d43b9d223ce09e5d4ce8b0060cb8a7513577a35a64c7e3dad10f0703bf3ad93",
-      strip_prefix = "protobuf-0b059a3d8a8f8aa40dde7bea55edca4ec5dfea66",
+      sha256 = "e178a25c52efcb6b05988bdbeace4c0d3f2d2fe5b46696d1d9898875c3803d6a",
+      strip_prefix = "protobuf-b04e5cba356212e4e8c66c61bbe0c3a20537c5b9",
       # TODO: remove patching when tensorflow stops linking same protos into
       #       multiple shared libraries loaded in runtime by python.
       #       This patch fixes a runtime crash when tensorflow is compiled
diff --git a/third_party/gpus/cuda_configure.bzl b/third_party/gpus/cuda_configure.bzl
index baa6e01bca..31a4bfabf6 100644
--- a/third_party/gpus/cuda_configure.bzl
+++ b/third_party/gpus/cuda_configure.bzl
@@ -117,7 +117,7 @@ def get_cxx_inc_directories(repository_ctx, cc):
   includes_cpp = _get_cxx_inc_directories_impl(repository_ctx, cc, True)
   includes_c = _get_cxx_inc_directories_impl(repository_ctx, cc, False)
 
-  includes_cpp_set = set(includes_cpp)
+  includes_cpp_set = depset(includes_cpp)
   return includes_cpp + [inc for inc in includes_c
                          if inc not in includes_cpp_set]
 
diff --git a/third_party/mkl_dnn/BUILD b/third_party/mkl_dnn/BUILD
new file mode 100644
index 0000000000..5b01f6e3e4
--- /dev/null
+++ b/third_party/mkl_dnn/BUILD
@@ -0,0 +1 @@
+licenses(["notice"])
diff --git a/third_party/mkl_dnn/mkldnn.BUILD b/third_party/mkl_dnn/mkldnn.BUILD
new file mode 100644
index 0000000000..58bb7a6a5d
--- /dev/null
+++ b/third_party/mkl_dnn/mkldnn.BUILD
@@ -0,0 +1,25 @@
+exports_files(["LICENSE"])
+
+cc_library(
+    name = "mkl_dnn",
+    srcs = glob([
+        "src/common/*.cpp",
+        "src/cpu/*.cpp",
+    ]),
+    hdrs = glob(["include/*"]),
+    copts = ["-fexceptions"] + select({
+        "@org_tensorflow//tensorflow:linux_x86_64": [
+            "-fopenmp",
+        ],
+        "//conditions:default": [],
+    }),
+    includes = [
+        "include",
+        "src",
+        "src/common",
+        "src/cpu",
+        "src/cpu/xbyak",
+    ],
+    nocopts = "-fno-exceptions",
+    visibility = ["//visibility:public"],
+)
-- 
GitLab


From aa20fc1aea6d3fdf4e0ba821e8e4ef5c08cfd282 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Fri, 6 Oct 2017 12:12:10 -0700
Subject: [PATCH 128/909] [XLA:CPU] Rename GetIrArrayForOp to GetIrArrayFor.

This makes it consistent with the other similar functions in IrEmitter.

PiperOrigin-RevId: 171325815
---
 .../compiler/xla/service/cpu/ir_emitter.cc    | 78 +++++++++----------
 .../compiler/xla/service/cpu/ir_emitter.h     |  6 +-
 2 files changed, 42 insertions(+), 42 deletions(-)

diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
index e4fb7c0496..ec9a69709d 100644
--- a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
@@ -395,7 +395,7 @@ Status IrEmitter::HandleSelect(HloInstruction* select, HloInstruction* pred,
 
   if (ShapeUtil::IsTuple(select->shape())) {
     TF_RETURN_IF_ERROR(EmitTargetAddressForOp(select));
-    llvm_ir::EmitTupleSelect(GetIrArrayForOp(select), GetIrArrayForOp(pred),
+    llvm_ir::EmitTupleSelect(GetIrArrayFor(select), GetIrArrayFor(pred),
                              GetEmittedValueFor(on_true),
                              GetEmittedValueFor(on_false), &ir_builder_);
     return Status::OK();
@@ -412,7 +412,7 @@ Status IrEmitter::HandleInfeed(HloInstruction* infeed) {
   // The infeed operation produces data (dequeued from the infeed queue) at this
   // address, which has been provided by buffer assignment.
   TF_RETURN_IF_ERROR(EmitTargetAddressForOp(infeed));
-  llvm_ir::IrArray infeed_array = GetIrArrayForOp(infeed);
+  llvm_ir::IrArray infeed_array = GetIrArrayFor(infeed);
 
   if (ShapeUtil::IsTuple(shape)) {
     TF_RET_CHECK(!ShapeUtil::IsNestedTuple(shape));
@@ -566,7 +566,7 @@ Status IrEmitter::HandleTuple(
   for (auto operand : operands) {
     base_ptrs.push_back(GetEmittedValueFor(operand));
   }
-  llvm_ir::EmitTuple(GetIrArrayForOp(tuple), base_ptrs, &ir_builder_);
+  llvm_ir::EmitTuple(GetIrArrayFor(tuple), base_ptrs, &ir_builder_);
   return Status::OK();
 }
 
@@ -581,7 +581,7 @@ Status IrEmitter::HandleMap(
                                         const llvm_ir::IrArray::Index& index) {
     std::vector<llvm::Value*> parameter_addresses;
     for (const HloInstruction* operand : operands) {
-      const llvm_ir::IrArray& array = GetIrArrayForOp(operand);
+      const llvm_ir::IrArray& array = GetIrArrayFor(operand);
       parameter_addresses.push_back(
           array.EmitArrayElementAddress(index, &ir_builder_));
     }
@@ -677,7 +677,7 @@ Status IrEmitter::HandleReduceWindow(HloInstruction* reduce_window,
         SetToFirstInsertPoint(if_data.true_block, &ir_builder_);
 
         // We are not in the padding, so carry out the computation.
-        llvm_ir::IrArray input_array(GetIrArrayForOp(operand));
+        llvm_ir::IrArray input_array(GetIrArrayFor(operand));
         llvm::Value* input_value_address =
             input_array.EmitArrayElementAddress(input_index, &ir_builder_);
         llvm::Value* result = EmitElementFunctionCall(
@@ -814,7 +814,7 @@ Status IrEmitter::HandleSelectAndScatter(HloInstruction* select_and_scatter) {
       ir_builder_.CreateStore(operand_index[i], selected_index_address_slot);
     }
   };
-  llvm_ir::IrArray operand_array(GetIrArrayForOp(operand));
+  llvm_ir::IrArray operand_array(GetIrArrayFor(operand));
   llvm::Value* operand_data =
       operand_array.EmitReadArrayElement(operand_index, &ir_builder_);
   ir_builder_.CreateStore(operand_data, selected_value_address);
@@ -857,10 +857,10 @@ Status IrEmitter::HandleSelectAndScatter(HloInstruction* select_and_scatter) {
     selected_index.push_back(
         ir_builder_.CreateLoad(selected_index_address_slot));
   }
-  llvm_ir::IrArray source_array(GetIrArrayForOp(source));
+  llvm_ir::IrArray source_array(GetIrArrayFor(source));
   llvm::Value* source_value_address =
       source_array.EmitArrayElementAddress(source_index, &ir_builder_);
-  llvm_ir::IrArray output_array(GetIrArrayForOp(select_and_scatter));
+  llvm_ir::IrArray output_array(GetIrArrayFor(select_and_scatter));
   llvm::Value* output_value_address =
       output_array.EmitArrayElementAddress(selected_index, &ir_builder_);
   llvm::Value* scatter_value = EmitElementFunctionCall(
@@ -880,11 +880,11 @@ Status IrEmitter::HandleDot(HloInstruction* dot, HloInstruction* lhs,
       /*instruction=*/*dot, /*operands=*/{lhs, rhs},
       /*supported_types=*/{F32, F64}));
 
-  llvm_ir::IrArray lhs_array(GetIrArrayForOp(lhs));
-  llvm_ir::IrArray rhs_array(GetIrArrayForOp(rhs));
+  llvm_ir::IrArray lhs_array(GetIrArrayFor(lhs));
+  llvm_ir::IrArray rhs_array(GetIrArrayFor(rhs));
 
   TF_RETURN_IF_ERROR(EmitTargetAddressForOp(dot));
-  llvm_ir::IrArray target_array = GetIrArrayForOp(dot);
+  llvm_ir::IrArray target_array = GetIrArrayFor(dot);
 
   VLOG(2) << "HandleDot: ";
   VLOG(2) << "  lhs operand: "
@@ -1163,7 +1163,7 @@ Status IrEmitter::HandleConvolution(HloInstruction* convolution,
         input_index[dnums.feature_dimension()] = input_feature;
         input_index[dnums.batch_dimension()] = batch;
 
-        llvm_ir::IrArray kernel_array(GetIrArrayForOp(rhs));
+        llvm_ir::IrArray kernel_array(GetIrArrayFor(rhs));
         llvm_ir::IrArray::Index kernel_index(num_dims);
         for (int i = 0; i < num_spatial_dims; ++i) {
           kernel_index[dnums.kernel_spatial_dimensions(i)] = kernel_spatial[i];
@@ -1171,7 +1171,7 @@ Status IrEmitter::HandleConvolution(HloInstruction* convolution,
         kernel_index[dnums.kernel_input_feature_dimension()] = input_feature;
         kernel_index[dnums.kernel_output_feature_dimension()] = output_feature;
 
-        llvm_ir::IrArray input_array(GetIrArrayForOp(lhs));
+        llvm_ir::IrArray input_array(GetIrArrayFor(lhs));
         llvm::Value* product = ir_builder_.CreateFMul(
             input_array.EmitReadArrayElement(input_index, &ir_builder_),
             kernel_array.EmitReadArrayElement(kernel_index, &ir_builder_));
@@ -1305,7 +1305,7 @@ Status IrEmitter::HandleBatchNormTraining(HloInstruction* batch_norm_training) {
             SetToFirstInsertPoint(loops.GetInnerLoopBodyBasicBlock(),
                                   &ir_builder_);
 
-            llvm_ir::IrArray operand_array(GetIrArrayForOp(operand));
+            llvm_ir::IrArray operand_array(GetIrArrayFor(operand));
             llvm_ir::IrArray::Index input_index =
                 FillReducedDimensionIndex(reduced_dims_index, index);
             llvm::Value* new_value =
@@ -1379,7 +1379,7 @@ Status IrEmitter::HandleBatchNormTraining(HloInstruction* batch_norm_training) {
             llvm::Value* var = var_array.EmitReadArrayElement(
                 feature_index_value, &ir_builder_);
 
-            llvm_ir::IrArray operand_array(GetIrArrayForOp(operand));
+            llvm_ir::IrArray operand_array(GetIrArrayFor(operand));
             llvm::Value* input =
                 operand_array.EmitReadArrayElement(index, &ir_builder_);
 
@@ -1391,10 +1391,10 @@ Status IrEmitter::HandleBatchNormTraining(HloInstruction* batch_norm_training) {
                 ir_builder_.CreateCall(func_llvm_sqrt, {variance_with_epsilon});
             llvm::Value* normalized = ir_builder_.CreateFDiv(
                 ir_builder_.CreateFSub(input, mean), variance_sqrt);
-            llvm_ir::IrArray offset_array(GetIrArrayForOp(offset));
+            llvm_ir::IrArray offset_array(GetIrArrayFor(offset));
             llvm::Value* offset = offset_array.EmitReadArrayElement(
                 feature_index_value, &ir_builder_);
-            llvm_ir::IrArray scale_array(GetIrArrayForOp(scale));
+            llvm_ir::IrArray scale_array(GetIrArrayFor(scale));
             llvm::Value* scale = scale_array.EmitReadArrayElement(
                 feature_index_value, &ir_builder_);
             llvm::Value* result = ir_builder_.CreateFAdd(
@@ -1405,7 +1405,7 @@ Status IrEmitter::HandleBatchNormTraining(HloInstruction* batch_norm_training) {
           target_array, &ir_builder_)
           .EmitLoop(IrName(batch_norm_training, "normalize")));
 
-  llvm_ir::EmitTuple(GetIrArrayForOp(batch_norm_training),
+  llvm_ir::EmitTuple(GetIrArrayFor(batch_norm_training),
                      {normalized, mean, var}, &ir_builder_);
   return Status::OK();
 }
@@ -1653,7 +1653,7 @@ IrEmitter::EmitInnerLoopForVectorizedReduction(
   SetToFirstInsertPoint(reduction_loop_nest.GetInnerLoopBodyBasicBlock(),
                         &ir_builder_);
 
-  llvm_ir::IrArray arg_array(GetIrArrayForOp(arg));
+  llvm_ir::IrArray arg_array(GetIrArrayFor(arg));
   llvm_ir::IrArray::Index input_index = reduced_dims_index;
   llvm_ir::IrArray::Index::const_iterator it = output_index.begin();
 
@@ -1829,7 +1829,7 @@ StatusOr<bool> IrEmitter::EmitVectorizedReduce(
                             reduction_generator, array_index, vector_type,
                             init_value, arg, dimensions, element_alignment));
 
-    llvm_ir::IrArray target_array = GetIrArrayForOp(reduce);
+    llvm_ir::IrArray target_array = GetIrArrayFor(reduce);
     llvm::Value* output_address =
         target_array.EmitArrayElementAddress(array_index, &ir_builder_);
     EmitShardedVectorStore(output_address, accumulator, element_alignment,
@@ -1861,7 +1861,7 @@ StatusOr<bool> IrEmitter::EmitVectorizedReduce(
                             reduction_generator, array_index, vector_type,
                             init_value, arg, dimensions, element_alignment));
 
-    llvm_ir::IrArray target_array = GetIrArrayForOp(reduce);
+    llvm_ir::IrArray target_array = GetIrArrayFor(reduce);
     llvm::Value* output_address =
         target_array.EmitArrayElementAddress(array_index, &ir_builder_);
     EmitShardedVectorStore(output_address, accumulator, element_alignment,
@@ -1928,7 +1928,7 @@ Status IrEmitter::HandleReduce(HloInstruction* reduce, HloInstruction* arg,
         // filled in. We fill in the rest of the dimensions with induction
         // Value*s taken from 'index' which iterates over the target array.
         // See the high-level description in the XLA documentation for details.
-        llvm_ir::IrArray arg_array(GetIrArrayForOp(arg));
+        llvm_ir::IrArray arg_array(GetIrArrayFor(arg));
         llvm_ir::IrArray::Index input_index = reduced_dims_index;
         llvm_ir::IrArray::Index::const_iterator it = index.begin();
 
@@ -2043,7 +2043,7 @@ Status IrEmitter::HandleSlice(HloInstruction* slice, HloInstruction* operand) {
     outer_dims.push_back(memcpy_dim);
   }
 
-  llvm_ir::IrArray target_array = GetIrArrayForOp(slice);
+  llvm_ir::IrArray target_array = GetIrArrayFor(slice);
 
   const int64 num_outer_loops = outer_dims.size();
   llvm_ir::ForLoopNest loops(IrName(slice), &ir_builder_);
@@ -2061,7 +2061,7 @@ Status IrEmitter::HandleSlice(HloInstruction* slice, HloInstruction* operand) {
     SetToFirstInsertPoint(loops.GetInnerLoopBodyBasicBlock(), &ir_builder_);
   }
 
-  llvm_ir::IrArray source_array = GetIrArrayForOp(operand);
+  llvm_ir::IrArray source_array = GetIrArrayFor(operand);
   const llvm_ir::IrArray::Index source_index = target_index.SourceIndexOfSlice(
       /*shape=*/slice->shape(), /*starts=*/slice->slice_starts(),
       /*strides=*/slice->slice_strides(), /*builder=*/&ir_builder_);
@@ -2166,7 +2166,7 @@ Status IrEmitter::HandleDynamicUpdateSlice(HloInstruction* dynamic_update_slice,
     llvm_ir::IrArray::Index start_index(rank);
     for (int64 i = 0; i < rank; ++i) {
       llvm_ir::IrArray::Index dim_index({ir_builder_.getInt64(i)});
-      llvm_ir::IrArray start_indices_array(GetIrArrayForOp(start_indices));
+      llvm_ir::IrArray start_indices_array(GetIrArrayFor(start_indices));
       start_index[i] =
           start_indices_array.EmitReadArrayElement(dim_index, &ir_builder_);
     }
@@ -2192,13 +2192,13 @@ Status IrEmitter::HandleDynamicUpdateSlice(HloInstruction* dynamic_update_slice,
       }
 
       // Read value from 'update'.
-      llvm_ir::IrArray update_array(GetIrArrayForOp(update));
+      llvm_ir::IrArray update_array(GetIrArrayFor(update));
       llvm::Value* update_data =
           update_array.EmitReadArrayElement(index, &ir_builder_);
 
       // Write value to output array.
-      GetIrArrayForOp(operand).EmitWriteArrayElement(output_index, update_data,
-                                                     &ir_builder_);
+      GetIrArrayFor(operand).EmitWriteArrayElement(output_index, update_data,
+                                                   &ir_builder_);
       return Status::OK();
     };
 
@@ -2249,7 +2249,7 @@ Status IrEmitter::HandlePad(HloInstruction* pad) {
   SetToFirstInsertPoint(loops.GetInnerLoopBodyBasicBlock(), &ir_builder_);
 
   // Load an element from the operand.
-  llvm_ir::IrArray operand_array(GetIrArrayForOp(operand));
+  llvm_ir::IrArray operand_array(GetIrArrayFor(operand));
   llvm::Value* operand_data =
       operand_array.EmitReadArrayElement(operand_index, &ir_builder_);
 
@@ -2269,7 +2269,7 @@ Status IrEmitter::HandlePad(HloInstruction* pad) {
   }
 
   // Store the operand element to the computed output location.
-  llvm_ir::IrArray output_array(GetIrArrayForOp(pad));
+  llvm_ir::IrArray output_array(GetIrArrayFor(pad));
   output_array.EmitWriteArrayElement(output_index, operand_data, &ir_builder_);
 
   SetToFirstInsertPoint(loops.GetOuterLoopExitBasicBlock(), &ir_builder_);
@@ -2301,12 +2301,12 @@ Status IrEmitter::HandleFusion(HloInstruction* fusion) {
         /*instruction=*/*dot, /*operands=*/{lhs, rhs},
         /*supported_types=*/{F32}));
 
-    llvm_ir::IrArray lhs_array(GetIrArrayForOp(lhs));
-    llvm_ir::IrArray rhs_array(GetIrArrayForOp(rhs));
+    llvm_ir::IrArray lhs_array(GetIrArrayFor(lhs));
+    llvm_ir::IrArray rhs_array(GetIrArrayFor(rhs));
 
     Shape target_shape = fusion->shape();
     TF_RETURN_IF_ERROR(EmitTargetAddressForOp(fusion));
-    llvm_ir::IrArray target_array = GetIrArrayForOp(fusion);
+    llvm_ir::IrArray target_array = GetIrArrayFor(fusion);
     VLOG(2) << "HandleFusion kTransposeDot: ";
     VLOG(2) << "  lhs operand: "
             << llvm_ir::DumpToString(*lhs_array.GetBasePointer());
@@ -2324,7 +2324,7 @@ Status IrEmitter::HandleFusion(HloInstruction* fusion) {
   } else if (fusion->fusion_kind() == HloInstruction::FusionKind::kLoop) {
     std::vector<llvm_ir::IrArray> parameter_arrays;
     for (HloInstruction* operand : fusion->operands()) {
-      parameter_arrays.push_back(GetIrArrayForOp(operand));
+      parameter_arrays.push_back(GetIrArrayFor(operand));
     }
     CpuElementalIrEmitter elemental_emitter(hlo_module_config_, this, module_);
     FusedIrEmitter fused_emitter(parameter_arrays, &elemental_emitter);
@@ -2527,7 +2527,7 @@ StatusOr<bool> IrEmitter::EmitFastConcatenate(
   llvm::Type* i8_type = ir_builder_.getInt8Ty();
 
   TF_RETURN_IF_ERROR(EmitTargetAddressForOp(concatenate));
-  llvm_ir::IrArray target_array = GetIrArrayForOp(concatenate);
+  llvm_ir::IrArray target_array = GetIrArrayFor(concatenate);
 
   llvm_ir::ForLoopNest loops(IrName(concatenate), &ir_builder_);
   llvm_ir::IrArray::Index outer_dims_index =
@@ -2562,7 +2562,7 @@ StatusOr<bool> IrEmitter::EmitFastConcatenate(
   // equal to the product of inner dimensions.
   for (HloInstruction* operand : operands) {
     const Shape& input_shape = operand->shape();
-    llvm_ir::IrArray source_array = GetIrArrayForOp(operand);
+    llvm_ir::IrArray source_array = GetIrArrayFor(operand);
     llvm::Value* copy_source_address = ir_builder_.CreateBitCast(
         source_array.EmitArrayElementAddress(outer_dims_index, &ir_builder_,
                                              "src_addr"),
@@ -2785,7 +2785,7 @@ Status IrEmitter::Postprocess(HloInstruction* hlo) {
   return Status::OK();
 }
 
-llvm_ir::IrArray IrEmitter::GetIrArrayForOp(const HloInstruction* hlo) {
+llvm_ir::IrArray IrEmitter::GetIrArrayFor(const HloInstruction* hlo) {
   llvm::Value* value_for_op = GetEmittedValueFor(hlo);
 
   llvm_ir::IrArray array(value_for_op, hlo->shape());
@@ -2995,7 +2995,7 @@ Status IrEmitter::EmitTargetElementLoop(
 
   const Shape& target_shape = target_op->shape();
   TF_RETURN_IF_ERROR(EmitTargetAddressForOp(target_op));
-  llvm_ir::IrArray target_array = GetIrArrayForOp(target_op);
+  llvm_ir::IrArray target_array = GetIrArrayFor(target_op);
 
   if (target_op->IsMultiOutputFusion()) {
     // For multiple outputs fusion, we need to emit each operand and the root.
@@ -3121,7 +3121,7 @@ Status IrEmitter::DefaultAction(HloInstruction* hlo) {
   ElementalIrEmitter::HloToElementGeneratorMap operand_to_generator;
   for (const HloInstruction* operand : hlo->operands()) {
     operand_to_generator[operand] = [=](const llvm_ir::IrArray::Index& index) {
-      return GetIrArrayForOp(operand).EmitReadArrayElement(index, &ir_builder_);
+      return GetIrArrayFor(operand).EmitReadArrayElement(index, &ir_builder_);
     };
   }
   CpuElementalIrEmitter elemental_emitter(hlo_module_config_, this, module_);
diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.h b/tensorflow/compiler/xla/service/cpu/ir_emitter.h
index fd9ee71799..b15026b6da 100644
--- a/tensorflow/compiler/xla/service/cpu/ir_emitter.h
+++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.h
@@ -220,8 +220,8 @@ class IrEmitter : public DfsHloVisitorWithDefault {
 
   // Gets the IR Value emitted previously for the given hlo.
   //
-  // Prefer calling GetIrArrayForOp if the value you're reading is a buffer,
-  // because GetIrArrayForOp annotates buffer's loads/stores with noalias
+  // Prefer calling GetIrArrayFor if the value you're reading is a buffer,
+  // because GetIrArrayFor annotates buffer's loads/stores with noalias
   // metadata.
   //
   // Make sure to call this only when you're certain a value *was* emitted - if
@@ -229,7 +229,7 @@ class IrEmitter : public DfsHloVisitorWithDefault {
   llvm::Value* GetEmittedValueFor(const HloInstruction* hlo);
 
   // Gets an IrArray representing the given hlo.
-  llvm_ir::IrArray GetIrArrayForOp(const HloInstruction* hlo);
+  llvm_ir::IrArray GetIrArrayFor(const HloInstruction* hlo);
 
   // Augments IrArray with aliasing information.
   void AddAliasingInformationToIrArray(const HloInstruction& hlo,
-- 
GitLab


From e35372fe3e8a5de4a90a42cdd5a62c5e0fe452ff Mon Sep 17 00:00:00 2001
From: Jeff Carpenter <jeffcarp@chromium.org>
Date: Fri, 6 Oct 2017 12:20:05 -0700
Subject: [PATCH 129/909] Fix unevaluated link in "Reading data" docs

---
 tensorflow/docs_src/api_guides/python/reading_data.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/docs_src/api_guides/python/reading_data.md b/tensorflow/docs_src/api_guides/python/reading_data.md
index 8b6196ea34..e7fb05f9b5 100644
--- a/tensorflow/docs_src/api_guides/python/reading_data.md
+++ b/tensorflow/docs_src/api_guides/python/reading_data.md
@@ -58,7 +58,7 @@ A typical pipeline for reading records from files has the following stages:
 8.  Example queue
 
 Note: This section discusses implementing input pipelines using the
-queue-based APIs which can be cleanly replaced by the ${$datasets$Dataset API}.
+queue-based APIs which can be cleanly replaced by the @{$datasets$Datasets API}.
 
 ### Filenames, shuffling, and epoch limits
 
-- 
GitLab


From 1d3d4ed02feca370e9009193946cd7efb458b7b6 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 6 Oct 2017 12:26:42 -0700
Subject: [PATCH 130/909] Update ops-related pbtxt files.

PiperOrigin-RevId: 171327794
---
 .../core/ops/compat/ops_history.v1.pbtxt      | 50 +++++++++++++++++++
 tensorflow/core/ops/ops.pbtxt                 | 10 +++-
 2 files changed, 59 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index a3321c26f3..f8667177cc 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -6831,6 +6831,56 @@ op {
     }
   }
 }
+op {
+  name: "DecodeCSV"
+  input_arg {
+    name: "records"
+    type: DT_STRING
+  }
+  input_arg {
+    name: "record_defaults"
+    type_list_attr: "OUT_TYPE"
+  }
+  output_arg {
+    name: "output"
+    type_list_attr: "OUT_TYPE"
+  }
+  attr {
+    name: "OUT_TYPE"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_STRING
+      }
+    }
+  }
+  attr {
+    name: "field_delim"
+    type: "string"
+    default_value {
+      s: ","
+    }
+  }
+  attr {
+    name: "use_quote_delim"
+    type: "bool"
+    default_value {
+      b: true
+    }
+  }
+  attr {
+    name: "na_value"
+    type: "string"
+    default_value {
+      s: ""
+    }
+  }
+}
 op {
   name: "DecodeGif"
   input_arg {
diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index 429000a058..9cda34a8c8 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -6219,6 +6219,14 @@ op {
     }
     description: "If false, treats double quotation marks as regular\ncharacters inside of the string fields (ignoring RFC 4180, Section 2,\nBullet 5)."
   }
+  attr {
+    name: "na_value"
+    type: "string"
+    default_value {
+      s: ""
+    }
+    description: "Additional string to recognize as NA/NaN."
+  }
   summary: "Convert CSV records to tensors. Each column maps to one tensor."
   description: "RFC 4180 format is expected for the CSV records.\n(https://tools.ietf.org/html/rfc4180)\nNote that we allow leading and trailing spaces with int or float field."
 }
@@ -6505,7 +6513,7 @@ op {
   }
   input_arg {
     name: "row_shape"
-    description: "A vector representing the dense shape of each row in the produced\nSparseTensor."
+    description: "A vector representing the dense shape of each row in the produced\nSparseTensor. The shape may be partially specified, using `-1` to indicate\nthat a particular dimension should use the maximum size of all batch elements."
     type: DT_INT64
   }
   output_arg {
-- 
GitLab


From 958a321b0e7a9e5ba07b536024c41615188b547d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 6 Oct 2017 12:33:19 -0700
Subject: [PATCH 131/909] Go: Update generated wrapper functions for TensorFlow
 ops.

PiperOrigin-RevId: 171328576
---
 tensorflow/go/op/wrappers.go | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go
index 29c69b3c59..f2ee710a9e 100644
--- a/tensorflow/go/op/wrappers.go
+++ b/tensorflow/go/op/wrappers.go
@@ -5720,7 +5720,8 @@ func TensorArrayGradV2(scope *Scope, handle tf.Output, flow_in tf.Output, source
 //	batch_size: A scalar representing the number of elements to accumulate in a
 // batch.
 //	row_shape: A vector representing the dense shape of each row in the produced
-// SparseTensor.
+// SparseTensor. The shape may be partially specified, using `-1` to indicate
+// that a particular dimension should use the maximum size of all batch elements.
 //
 //
 func DenseToSparseBatchDataset(scope *Scope, input_dataset tf.Output, batch_size tf.Output, row_shape tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
@@ -9313,6 +9314,16 @@ func DecodeCSVUseQuoteDelim(value bool) DecodeCSVAttr {
 	}
 }
 
+// DecodeCSVNaValue sets the optional na_value attribute to value.
+//
+// value: Additional string to recognize as NA/NaN.
+// If not specified, defaults to ""
+func DecodeCSVNaValue(value string) DecodeCSVAttr {
+	return func(m optionalAttr) {
+		m["na_value"] = value
+	}
+}
+
 // Convert CSV records to tensors. Each column maps to one tensor.
 //
 // RFC 4180 format is expected for the CSV records.
-- 
GitLab


From e2e57bd0bb122abec220bcb399ebeaefdb61e5b2 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Fri, 6 Oct 2017 12:52:31 -0700
Subject: [PATCH 132/909] [XLA:LLVM] Remove SetTbaaForInstruction.

This was made a nop some time ago because it was broken; this patch
removes it entirely.

I don't think we can sensibly use HLO types for alias analysis -- a
buffer may store values of different HLO types over its lifetime.  This
isn't an indictment against LLVM TBAA in general; we may be able to use
it for something other than AA based on HLO types.

PiperOrigin-RevId: 171330686
---
 tensorflow/compiler/xla/service/cpu/ir_emitter.cc    | 5 -----
 tensorflow/compiler/xla/service/layout_assignment.cc | 2 --
 tensorflow/compiler/xla/service/llvm_ir/ir_array.cc  | 4 ----
 tensorflow/compiler/xla/service/llvm_ir/llvm_util.cc | 7 -------
 tensorflow/compiler/xla/service/llvm_ir/llvm_util.h  | 6 ------
 tensorflow/compiler/xla/service/llvm_ir/ops.cc       | 1 -
 6 files changed, 25 deletions(-)

diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
index ec9a69709d..85f790a717 100644
--- a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
@@ -1447,9 +1447,6 @@ Status IrEmitter::HandleParameter(HloInstruction* parameter) {
       param_address_untyped, IrShapeType(param_shape)->getPointerTo());
   emitted_value_[parameter] = param_address_typed;
 
-  // Parameters of different types may not alias one another.
-  llvm_ir::SetTbaaForInstruction(param_address_untyped, param_shape,
-                                 /*is_pointer_to=*/true);
   if (!ShapeUtil::IsOpaque(param_shape)) {
     AttachAlignmentMetadataForLoad(param_address_untyped, param_shape);
     AttachDereferenceableMetadataForLoad(param_address_untyped, param_shape);
@@ -2867,8 +2864,6 @@ llvm::Value* IrEmitter::EmitTempBufferPointer(
         llvm::LLVMContext::MD_invariant_load,
         llvm::MDNode::get(tempbuf_address_base->getContext(), /*MDs=*/{}));
   }
-  llvm_ir::SetTbaaForInstruction(tempbuf_address_base, target_shape,
-                                 /*is_pointer_to=*/true);
   AttachAlignmentMetadataForLoad(tempbuf_address_base, allocation.size());
   AttachDereferenceableMetadataForLoad(tempbuf_address_base, allocation.size());
 
diff --git a/tensorflow/compiler/xla/service/layout_assignment.cc b/tensorflow/compiler/xla/service/layout_assignment.cc
index 8fd330fda7..2058706f11 100644
--- a/tensorflow/compiler/xla/service/layout_assignment.cc
+++ b/tensorflow/compiler/xla/service/layout_assignment.cc
@@ -1180,8 +1180,6 @@ Status CopyOperandIfLayoutsDiffer(const ShapeLayout& operand_layout,
 // to match the layout of its corresponding fusion instruction operand. Also,
 // set the layout of the fused root to match the layout of the fusion
 // instruction itself.
-// Fused GetTupleElement requires a layout so that TBAA metadata for the tuple
-// element array pointer load can be added.
 Status SetFusionLayouts(HloInstruction* fusion) {
   TF_RET_CHECK(fusion->opcode() == HloOpcode::kFusion);
   for (auto* fused_instruction : fusion->fused_instructions()) {
diff --git a/tensorflow/compiler/xla/service/llvm_ir/ir_array.cc b/tensorflow/compiler/xla/service/llvm_ir/ir_array.cc
index e36c791c1a..6a00a565c6 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/ir_array.cc
+++ b/tensorflow/compiler/xla/service/llvm_ir/ir_array.cc
@@ -268,8 +268,6 @@ llvm::Value* IrArray::EmitReadArrayElement(const Index& index,
   llvm::Value* element_address =
       EmitArrayElementAddress(index, ir_builder, name);
   llvm::LoadInst* load = ir_builder->CreateLoad(element_address);
-  llvm_ir::SetTbaaForInstruction(load, GetShape(),
-                                 /*is_pointer_to=*/false);
   AnnotateLoadStoreInstructionWithMetadata(load);
   return load;
 }
@@ -278,8 +276,6 @@ void IrArray::EmitWriteArrayElement(const Index& index, llvm::Value* value,
                                     llvm::IRBuilder<>* ir_builder) const {
   llvm::Value* element_address = EmitArrayElementAddress(index, ir_builder);
   llvm::StoreInst* store = ir_builder->CreateStore(value, element_address);
-  llvm_ir::SetTbaaForInstruction(store, GetShape(),
-                                 /*is_pointer_to=*/false);
   AnnotateLoadStoreInstructionWithMetadata(store);
 }
 
diff --git a/tensorflow/compiler/xla/service/llvm_ir/llvm_util.cc b/tensorflow/compiler/xla/service/llvm_ir/llvm_util.cc
index 4a7d2b48f7..8e188e7ae8 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/llvm_util.cc
+++ b/tensorflow/compiler/xla/service/llvm_ir/llvm_util.cc
@@ -402,13 +402,6 @@ void EmitLogging(const char* tag, llvm::Value* value,
       {ir_builder->getInt64(tensorflow::bit_cast<int64>(tag)), value});
 }
 
-void SetTbaaForInstruction(llvm::Instruction* instruction, Shape shape,
-                           bool is_pointer_to) {
-  // TODO(b/62903316): TBAA metadata causes LLVM to miscompile generated code,
-  // most likely because the generated metadata is incorrect.  Disable TBAA
-  // metadata while we resolve this.
-}
-
 void SetAlignmentMetadataForLoad(llvm::LoadInst* load, uint64_t alignment) {
   llvm::LLVMContext& context = load->getContext();
   llvm::Type* int64_ty = llvm::Type::getInt64Ty(context);
diff --git a/tensorflow/compiler/xla/service/llvm_ir/llvm_util.h b/tensorflow/compiler/xla/service/llvm_ir/llvm_util.h
index 5af62b056e..7a7d14da1e 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/llvm_util.h
+++ b/tensorflow/compiler/xla/service/llvm_ir/llvm_util.h
@@ -227,12 +227,6 @@ llvm::Value* EmitComparison(llvm::CmpInst::Predicate predicate,
 void EmitLogging(const char* tag, llvm::Value* value,
                  llvm::IRBuilder<>* ir_builder);
 
-// Adds TBAA metadata to a load or store instruction using the given shape as
-// it's type.  The is_pointer_to parameter is used to indicate whether or not
-// this instruction loads or stores a pointer to an array.
-void SetTbaaForInstruction(llvm::Instruction* instruction, Shape shape,
-                           bool is_pointer_to);
-
 // Adds alignment metadata to a load instruction using the given alignment.
 // The alignment refers to the result of the load, not the load itself.
 void SetAlignmentMetadataForLoad(llvm::LoadInst* load, uint64_t alignment);
diff --git a/tensorflow/compiler/xla/service/llvm_ir/ops.cc b/tensorflow/compiler/xla/service/llvm_ir/ops.cc
index 3965433494..60777bc8a8 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/ops.cc
+++ b/tensorflow/compiler/xla/service/llvm_ir/ops.cc
@@ -89,7 +89,6 @@ llvm::Value* EmitGetTupleElement(const Shape& target_shape, int64 index,
   llvm::Value* element_ptr = ir_builder->CreateInBoundsGEP(
       operand, {ir_builder->getInt64(0), ir_builder->getInt64(index)});
   llvm::LoadInst* src_buffer = ir_builder->CreateLoad(element_ptr);
-  SetTbaaForInstruction(src_buffer, target_shape, /*is_pointer_to=*/true);
   SetAlignmentMetadataForLoad(src_buffer, alignment);
   llvm::Type* element_type = ShapeToIrType(target_shape, ir_builder);
   llvm::Value* ret_val =
-- 
GitLab


From b1c095a28a7aa9bbee4af4d9a7e9d0c60567765b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 6 Oct 2017 13:01:33 -0700
Subject: [PATCH 133/909] Bugfix: Ensure tf.distributions.Multinomial doesn't
 underflow in log_prob.

PiperOrigin-RevId: 171331659
---
 .../python/kernel_tests/distributions/multinomial_test.py  | 7 +++++++
 tensorflow/python/ops/distributions/multinomial.py         | 3 ++-
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/kernel_tests/distributions/multinomial_test.py b/tensorflow/python/kernel_tests/distributions/multinomial_test.py
index 80caf10391..614a34f077 100644
--- a/tensorflow/python/kernel_tests/distributions/multinomial_test.py
+++ b/tensorflow/python/kernel_tests/distributions/multinomial_test.py
@@ -76,6 +76,13 @@ class MultinomialTest(test.TestCase):
       self.assertAllClose(p, multinom.probs.eval())
       self.assertAllClose(logits, multinom.logits.eval())
 
+  def testPmfUnderflow(self):
+    logits = np.array([[-200, 0]], dtype=np.float32)
+    with self.test_session():
+      dist = multinomial.Multinomial(total_count=1., logits=logits)
+      lp = dist.log_prob([1., 0.]).eval()[0]
+      self.assertAllClose(-200, lp, atol=0, rtol=1e-6)
+
   def testPmfandCountsAgree(self):
     p = [[0.1, 0.2, 0.7]]
     n = [[5.]]
diff --git a/tensorflow/python/ops/distributions/multinomial.py b/tensorflow/python/ops/distributions/multinomial.py
index 9b15d4c76e..00b5697c83 100644
--- a/tensorflow/python/ops/distributions/multinomial.py
+++ b/tensorflow/python/ops/distributions/multinomial.py
@@ -24,6 +24,7 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import check_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import nn_ops
 from tensorflow.python.ops import random_ops
 from tensorflow.python.ops.distributions import distribution
 from tensorflow.python.ops.distributions import util as distribution_util
@@ -260,7 +261,7 @@ class Multinomial(distribution.Distribution):
 
   def _log_unnormalized_prob(self, counts):
     counts = self._maybe_assert_valid_sample(counts)
-    return math_ops.reduce_sum(counts * math_ops.log(self.probs), -1)
+    return math_ops.reduce_sum(counts * nn_ops.log_softmax(self.logits), -1)
 
   def _log_normalization(self, counts):
     counts = self._maybe_assert_valid_sample(counts)
-- 
GitLab


From 129947535edd50225b7a6bbe620ea58c6d32953c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 6 Oct 2017 13:15:12 -0700
Subject: [PATCH 134/909] Fixed a typo in a message from the debugger.

PiperOrigin-RevId: 171333405
---
 tensorflow/python/debug/cli/cli_shared.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/debug/cli/cli_shared.py b/tensorflow/python/debug/cli/cli_shared.py
index c3c9a332a7..df972eacf7 100644
--- a/tensorflow/python/debug/cli/cli_shared.py
+++ b/tensorflow/python/debug/cli/cli_shared.py
@@ -347,7 +347,7 @@ def get_run_start_intro(run_call_count,
 
   out = debugger_cli_common.RichTextLines(_HORIZONTAL_BAR)
   if is_callable_runner:
-    out.append("Running a runner returned by Session.make_callabe()")
+    out.append("Running a runner returned by Session.make_callable()")
   else:
     out.append("Session.run() call #%d:" % run_call_count)
     out.append("")
-- 
GitLab


From 2a90713ef70f01392ac59899ca92376549c57126 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Fri, 6 Oct 2017 13:25:28 -0700
Subject: [PATCH 135/909] [XLA:CPU] Mark pointers loaded via get-tuple-element
 as dereferenceable.

PiperOrigin-RevId: 171334827
---
 tensorflow/compiler/xla/service/llvm_ir/ops.cc | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/tensorflow/compiler/xla/service/llvm_ir/ops.cc b/tensorflow/compiler/xla/service/llvm_ir/ops.cc
index 60777bc8a8..ae5c666b7d 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/ops.cc
+++ b/tensorflow/compiler/xla/service/llvm_ir/ops.cc
@@ -89,7 +89,15 @@ llvm::Value* EmitGetTupleElement(const Shape& target_shape, int64 index,
   llvm::Value* element_ptr = ir_builder->CreateInBoundsGEP(
       operand, {ir_builder->getInt64(0), ir_builder->getInt64(index)});
   llvm::LoadInst* src_buffer = ir_builder->CreateLoad(element_ptr);
+
+  // Mark the loaded pointer as dereferenceable if we know its shape.
+  if (!ShapeUtil::IsOpaque(target_shape)) {
+    SetDereferenceableMetadataForLoad(
+        src_buffer,
+        ByteSizeOf(target_shape, src_buffer->getModule()->getDataLayout()));
+  }
   SetAlignmentMetadataForLoad(src_buffer, alignment);
+
   llvm::Type* element_type = ShapeToIrType(target_shape, ir_builder);
   llvm::Value* ret_val =
       ir_builder->CreateBitCast(src_buffer, element_type->getPointerTo());
-- 
GitLab


From 30c5f4347b722961a40eab483f2391a92d9088bb Mon Sep 17 00:00:00 2001
From: Michael Case <mikecase@google.com>
Date: Fri, 6 Oct 2017 13:52:17 -0700
Subject: [PATCH 136/909] Fix float32 precision causing test failure in gcs
 cloud TF tests.

The time in nanoseconds was being cast to float32 which caused loss
of precision. Because floats are used when parsing the time, the time
calculation can still be rounded incorrectly. Also changing EXPECT_EQ
to EXPECT_NEAR(,,1).

PiperOrigin-RevId: 171338952
---
 tensorflow/core/platform/cloud/BUILD                   | 2 --
 tensorflow/core/platform/cloud/gcs_file_system_test.cc | 4 ++--
 tensorflow/core/platform/cloud/time_util.cc            | 3 ++-
 tensorflow/core/platform/cloud/time_util_test.cc       | 2 +-
 4 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/tensorflow/core/platform/cloud/BUILD b/tensorflow/core/platform/cloud/BUILD
index c06004e747..c937fea049 100644
--- a/tensorflow/core/platform/cloud/BUILD
+++ b/tensorflow/core/platform/cloud/BUILD
@@ -228,7 +228,6 @@ tf_cc_test(
     name = "gcs_file_system_test",
     size = "small",
     srcs = ["gcs_file_system_test.cc"],
-    tags = ["nomac"],  # b/67103845
     deps = [
         ":gcs_file_system",
         ":http_request_fake",
@@ -304,7 +303,6 @@ tf_cc_test(
     name = "time_util_test",
     size = "small",
     srcs = ["time_util_test.cc"],
-    tags = ["nomac"],  # b/67103845
     deps = [
         ":time_util",
         "//tensorflow/core:test",
diff --git a/tensorflow/core/platform/cloud/gcs_file_system_test.cc b/tensorflow/core/platform/cloud/gcs_file_system_test.cc
index b8573e335d..911176365f 100644
--- a/tensorflow/core/platform/cloud/gcs_file_system_test.cc
+++ b/tensorflow/core/platform/cloud/gcs_file_system_test.cc
@@ -1637,7 +1637,7 @@ TEST(GcsFileSystemTest, Stat_Object) {
   FileStatistics stat;
   TF_EXPECT_OK(fs.Stat("gs://bucket/file.txt", &stat));
   EXPECT_EQ(1010, stat.length);
-  EXPECT_EQ(1461971724896, stat.mtime_nsec / 1000 / 1000);
+  EXPECT_NEAR(1461971724896, stat.mtime_nsec / 1000 / 1000, 1);
   EXPECT_FALSE(stat.is_directory);
 }
 
@@ -1771,7 +1771,7 @@ TEST(GcsFileSystemTest, Stat_Cache) {
     FileStatistics stat;
     TF_EXPECT_OK(fs.Stat("gs://bucket/file.txt", &stat));
     EXPECT_EQ(1010, stat.length);
-    EXPECT_EQ(1461971724896, stat.mtime_nsec / 1000 / 1000);
+    EXPECT_NEAR(1461971724896, stat.mtime_nsec / 1000 / 1000, 1);
     EXPECT_FALSE(stat.is_directory);
     TF_EXPECT_OK(fs.Stat("gs://bucket/subfolder", &stat));
     EXPECT_EQ(0, stat.length);
diff --git a/tensorflow/core/platform/cloud/time_util.cc b/tensorflow/core/platform/cloud/time_util.cc
index 633733a21c..2f8643f3c7 100644
--- a/tensorflow/core/platform/cloud/time_util.cc
+++ b/tensorflow/core/platform/cloud/time_util.cc
@@ -44,7 +44,8 @@ Status ParseRfc3339Time(const string& time, int64* mtime_nsec) {
   parsed.tm_sec = int_seconds;
 
   *mtime_nsec = timegm(&parsed) * kNanosecondsPerSecond +
-                floor((seconds - int_seconds) * kNanosecondsPerSecond);
+                static_cast<int64>(
+                    floor((seconds - int_seconds) * kNanosecondsPerSecond));
 
   return Status::OK();
 }
diff --git a/tensorflow/core/platform/cloud/time_util_test.cc b/tensorflow/core/platform/cloud/time_util_test.cc
index 3fd8fcdab0..1f975f7325 100644
--- a/tensorflow/core/platform/cloud/time_util_test.cc
+++ b/tensorflow/core/platform/cloud/time_util_test.cc
@@ -23,7 +23,7 @@ TEST(TimeUtil, ParseRfc3339Time) {
   int64 mtime_nsec;
   TF_EXPECT_OK(ParseRfc3339Time("2016-04-29T23:15:24.896Z", &mtime_nsec));
   // Compare milliseconds instead of nanoseconds.
-  EXPECT_EQ(1461971724896, mtime_nsec / 1000 / 1000);
+  EXPECT_NEAR(1461971724896, mtime_nsec / 1000 / 1000, 1);
 }
 
 TEST(TimeUtil, ParseRfc3339Time_ParseError) {
-- 
GitLab


From ac2e086d1811be3d41b14f79d9c5c71ec98a1105 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Fri, 6 Oct 2017 14:20:41 -0700
Subject: [PATCH 137/909] Explicitly tag constants in LLVM IR with required
 alignment

(We are most likely getting lucky with this today, but this will eventually blow up.)

PiperOrigin-RevId: 171343275
---
 tensorflow/compiler/xla/service/cpu/ir_emitter.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
index 85f790a717..8132207699 100644
--- a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
@@ -281,6 +281,7 @@ Status IrEmitter::HandleConstant(HloInstruction* constant,
       /*Linkage=*/llvm::GlobalValue::PrivateLinkage,
       /*Initializer=*/initializer,
       /*Name=*/"");
+  global_for_const->setAlignment(MinimumAlignmentForShape(literal.shape()));
   emitted_value_[constant] = global_for_const;
   VLOG(2) << "  emitted value: " << llvm_ir::DumpToString(*global_for_const);
   VLOG(2) << "  its type: "
-- 
GitLab


From bbfef93661ebf8ec23c7b9ad920313be9898bbbc Mon Sep 17 00:00:00 2001
From: Akshay Agrawal <akshayka@google.com>
Date: Fri, 6 Oct 2017 14:47:55 -0700
Subject: [PATCH 138/909] Convert shape to TensorShape when creating
 _VariableFromResource

Ensures that variable shapes are TensorShapes when accessed in
graph_callable functions.

PiperOrigin-RevId: 171347097
---
 tensorflow/python/eager/graph_callable.py      |  3 ++-
 tensorflow/python/eager/graph_callable_test.py | 10 ++++++++++
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/eager/graph_callable.py b/tensorflow/python/eager/graph_callable.py
index 64d1659993..e3aacbd140 100644
--- a/tensorflow/python/eager/graph_callable.py
+++ b/tensorflow/python/eager/graph_callable.py
@@ -28,6 +28,7 @@ from tensorflow.python.eager import tape
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops as tf_ops
+from tensorflow.python.framework import tensor_shape
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.ops import variable_scope
@@ -54,7 +55,7 @@ class _VariableFromResource(resource_variable_ops.ResourceVariable):
 
   def __init__(self, resource, dtype, name, shape):
     self._handle = resource
-    self._graph_shape = shape
+    self._graph_shape = tensor_shape.as_shape(shape)
     self._handle_device = resource.device
     self._handle_name = name
     self._cached_value = None
diff --git a/tensorflow/python/eager/graph_callable_test.py b/tensorflow/python/eager/graph_callable_test.py
index 4ad8f1f36e..104e019391 100644
--- a/tensorflow/python/eager/graph_callable_test.py
+++ b/tensorflow/python/eager/graph_callable_test.py
@@ -22,6 +22,7 @@ from tensorflow.python.eager import test
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import function
+from tensorflow.python.framework import tensor_shape
 from tensorflow.python.ops import init_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import variable_scope
@@ -209,6 +210,15 @@ class GraphCallableTest(test.TestCase):
     ret = my_op(inputs)
     self.assertEqual(ret[1].numpy(), 11.)
 
+  def testVariableShapeIsTensorShape(self):
+    @graph_callable.graph_callable([])
+    def my_function():
+      v = variable_scope.get_variable(
+          "v", initializer=init_ops.zeros_initializer(), shape=())
+      self.assertIsInstance(v.get_shape(), tensor_shape.TensorShape)
+
+    my_function()
+
 
 if __name__ == "__main__":
   test.main()
-- 
GitLab


From eb1a0a5294b9b7b209d419b4113fb57d6443b45f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 6 Oct 2017 14:53:56 -0700
Subject: [PATCH 139/909] (1) Adds broadcasting to scaled_softplus (2) Adds the
 ability to clip (so we can get a soft version of relu6)

PiperOrigin-RevId: 171347879
---
 .../contrib/nn/python/ops/scaled_softplus.py  | 82 ++++++++++++++-----
 .../nn/python/ops/scaled_softplus_test.py     | 23 ++++--
 2 files changed, 77 insertions(+), 28 deletions(-)

diff --git a/tensorflow/contrib/nn/python/ops/scaled_softplus.py b/tensorflow/contrib/nn/python/ops/scaled_softplus.py
index 5fc11d8ec6..fcbfbc239c 100644
--- a/tensorflow/contrib/nn/python/ops/scaled_softplus.py
+++ b/tensorflow/contrib/nn/python/ops/scaled_softplus.py
@@ -20,58 +20,96 @@ from __future__ import print_function
 
 from tensorflow.python.framework import function
 from tensorflow.python.framework import ops
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import gen_array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import nn
 
 
-def scaled_softplus(x, alpha, name=None):
-  """Returns `alpha * ln(1 + exp(x / alpha))`, for scalar `alpha > 0`.
+def _reduce_and_reshape_grad(g, t):
+  """Returns the gradient, sum-reduced and reshaped to `t`'s shape."""
+  shape = array_ops.shape(t)
+  g_shape = array_ops.shape(g)
+  # pylint: disable=protected-access
+  bcast_dims, _ = gen_array_ops._broadcast_gradient_args(shape, g_shape)
+  # pylint: enable=protected-access
+  return array_ops.reshape(math_ops.reduce_sum(g, bcast_dims), shape)
+
+
+def scaled_softplus(x, alpha, clip=None, name=None):
+  """Returns `y = alpha * ln(1 + exp(x / alpha))` or `min(y, clip)`.
 
   This can be seen as a softplus applied to the scaled input, with the output
   appropriately scaled. As `alpha` tends to 0, `scaled_softplus(x, alpha)` tends
-  to `relu(x)`.
+  to `relu(x)`. The clipping is optional. As alpha->0, scaled_softplus(x, alpha)
+  tends to relu(x), and scaled_softplus(x, alpha, clip=6) tends to relu6(x).
 
   Note: the gradient for this operation is defined to depend on the backprop
   inputs as well as the outputs of this operation.
 
   Args:
     x: A `Tensor` of inputs.
-    alpha: A scalar `Tensor`, indicating the amount of smoothness. The caller
+    alpha: A `Tensor`, indicating the amount of smoothness. The caller
         must ensure that `alpha > 0`.
+    clip: (optional) A `Tensor`, the upper bound to clip the values.
     name: A name for the scope of the operations (optional).
 
   Returns:
-    A tensor of same size and type as `x`.
+    A tensor of the size and type determined by broadcasting of the inputs.
 
   """
-  with ops.name_scope(name, 'scaled_softplus', [x, alpha]):
+  clipping = clip is not None
+  with ops.name_scope(name, 'scaled_softplus',
+                      [x, alpha] + ([clip] if clipping else [])):
     x = ops.convert_to_tensor(x, name='x')
     dtype = x.dtype
     alpha = ops.convert_to_tensor(alpha, dtype=dtype, name='alpha')
-    # Verify that alpha is a scalar.
-    alpha.get_shape().assert_has_rank(0)
+    # Compute the forward value.
+    y = alpha * nn.softplus(x / alpha)
+    if clipping:
+      clip = ops.convert_to_tensor(clip, dtype=dtype, name='clip')
+      y = math_ops.minimum(y, clip)
 
     def _grad(op, g):
-      """Backprop for scaled softplus."""
-      y = op.outputs[0]
-      alpha = op.inputs[1]
-      # Prevent the expensive computations from happening before g is available.
+      """Backprop for scaled softplus, with optional clipping."""
+      y, x, alpha = op.inputs[:3]
+      # Prevent the memory-expensive computations from happening before g is
+      # available.
       with ops.control_dependencies([g]):
-        y /= alpha
+        y = array_ops.identity(y)
+      clip_grad = []
+      if clipping:
+        clip = op.inputs[3]
+        unclipped = math_ops.cast(y < clip, g.dtype)
+        clip_grad = [_reduce_and_reshape_grad(g * (1. - unclipped), clip)]
+        g *= unclipped
+      y /= alpha
       emy = math_ops.exp(-y)
       dy_dx = 1. - emy
       # The eps below avoids log(0). Note that t*log(t) -> 0 as t->0.
       eps = 1e-8
       dy_dalpha = y * emy - dy_dx * math_ops.log(dy_dx + eps)
-      return g * dy_dx, math_ops.reduce_sum(g * dy_dalpha)
+      # Backprop to the actual inputs, but not to the output.
+      return [None,
+              _reduce_and_reshape_grad(g * dy_dx, x),
+              _reduce_and_reshape_grad(g * dy_dalpha, alpha)] + clip_grad
 
-    @function.Defun(dtype, dtype,
-                    func_name='ScaledSoftplus_%s' % dtype.name,
-                    shape_func=lambda op: [op.inputs[0].get_shape()],
+    if clipping:
+      @function.Defun(dtype, dtype, dtype, dtype,
+                      func_name='ScaledSoftplusHelper_clip_%s' % dtype.name,
+                      shape_func=lambda op: [op.inputs[0].shape],
+                      python_grad_func=_grad)
+      def _forward_helper_clip(y, x, alpha, clip):
+        del x, alpha, clip  # Unused.
+        return y
+      return _forward_helper_clip(y, x, alpha, clip)
+    # No clipping.
+    @function.Defun(dtype, dtype, dtype,
+                    func_name='ScaledSoftplusHelper_%s' % dtype.name,
+                    shape_func=lambda op: [op.inputs[0].shape],
                     python_grad_func=_grad)
-    def _forward(x, alpha):
-      """Forward computation of scaled softplus."""
-      return alpha * nn.softplus(x / alpha)
-
-    return _forward(x, alpha)
+    def _forward_helper(y, x, alpha):
+      del x, alpha  # Unused.
+      return y
+    return _forward_helper(y, x, alpha)
 
diff --git a/tensorflow/contrib/nn/python/ops/scaled_softplus_test.py b/tensorflow/contrib/nn/python/ops/scaled_softplus_test.py
index 3a459330ce..b978343c6a 100644
--- a/tensorflow/contrib/nn/python/ops/scaled_softplus_test.py
+++ b/tensorflow/contrib/nn/python/ops/scaled_softplus_test.py
@@ -33,10 +33,11 @@ class ScaledSoftplusTest(test.TestCase):
     x = np.random.randn(3, 4).astype(np.float32)
     x64 = np.random.randn(3, 4).astype(np.float64)
     alpha = np.random.rand() + 0.01
-    y = alpha * np.log(1. + np.exp(x / alpha))
+    clip = np.float32(0.1)
+    y = np.minimum(alpha * np.log(1. + np.exp(x / alpha)), clip)
     y64 = alpha * np.log(1. + np.exp(x64 / alpha))
     with self.test_session(use_gpu=True) as sess:
-      z = scaled_softplus(constant_op.constant(x), alpha)
+      z = scaled_softplus(constant_op.constant(x), alpha, clip)
       z64 = scaled_softplus(constant_op.constant(x64), alpha)
       z, z64 = sess.run([z, z64])
       eps = 1e-6
@@ -47,18 +48,28 @@ class ScaledSoftplusTest(test.TestCase):
     np.random.seed(1)  # Make it reproducible.
     x_shape = [5, 10]
     x_np = np.random.randn(*x_shape).astype(np.float32)
-    alpha_np = np.float32(np.random.rand() + 0.01)
+    alpha_np = np.float32(np.random.rand(1, x_shape[1]) + 0.01)
+    clip_np = np.float32(np.random.rand(x_shape[0], 1) * 5.)
     with self.test_session(use_gpu=True):
       x_tf = constant_op.constant(x_np)
       alpha_tf = constant_op.constant(alpha_np)
+      clip_tf = constant_op.constant(clip_np)
       y_tf = scaled_softplus(x_tf, alpha_tf)
+      z_tf = scaled_softplus(x_tf, alpha_tf, clip_tf * 0.1)
       err = gradient_checker.compute_gradient_error([x_tf, alpha_tf],
-                                                    [x_shape, []],
+                                                    [x_shape, alpha_np.shape],
                                                     y_tf, x_shape,
                                                     [x_np, alpha_np],
-                                                    delta=1e-2)
-    eps = 1e-4
+                                                    delta=0.002)
+      err_clip = gradient_checker.compute_gradient_error(
+          [x_tf, alpha_tf, clip_tf],
+          [x_shape, alpha_np.shape, clip_np.shape],
+          z_tf, x_shape,
+          [x_np, alpha_np, clip_np],
+          delta=0.002)
+    eps = 2e-4
     self.assertLess(err, eps)
+    self.assertLess(err_clip, eps)
 
 
 if __name__ == '__main__':
-- 
GitLab


From e744cca9861b175f93e3e2bd72b38731a9f1fca7 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 6 Oct 2017 14:55:17 -0700
Subject: [PATCH 140/909] Changes Relu6Grad to depend on relu6's output rather
 than its input, for consistency with relu. This would result in memory
 savings when training conv->relu6->bn and conv->bn->relu6->conv models, as
 the inputs to bn and conv are already retained for backprop.

PiperOrigin-RevId: 171348086
---
 tensorflow/core/kernels/relu_op_functor.h | 7 ++++---
 tensorflow/core/ops/nn_ops.cc             | 3 ++-
 tensorflow/python/ops/nn_grad.py          | 2 +-
 3 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/tensorflow/core/kernels/relu_op_functor.h b/tensorflow/core/kernels/relu_op_functor.h
index 9577b963c6..24b789c543 100644
--- a/tensorflow/core/kernels/relu_op_functor.h
+++ b/tensorflow/core/kernels/relu_op_functor.h
@@ -76,14 +76,15 @@ struct Relu6Grad {
   // Computes Relu6Grad backprops.
   //
   // gradients: gradients backpropagated to the Relu6 op.
-  // features: inputs that where passed to the Relu6 op.
+  // features: inputs that where passed to the Relu6 op, or its outputs.
   // backprops: gradients to backpropagate to the Relu6 inputs.
   void operator()(const Device& d, typename TTypes<T>::ConstTensor gradients,
                   typename TTypes<T>::ConstTensor features,
                   typename TTypes<T>::Tensor backprops) {
     // NOTE: When the activation is exactly zero or six, we
-    // arbitrarily choose to not propagate the associated gradient
-    // value.
+    // make sure not to propagate the associated gradient
+    // value. This allows "features" to be either the input or the output of
+    // the relu6.
     backprops.device(d) =
         gradients *
         ((features > static_cast<T>(0)) * (features < static_cast<T>(6)))
diff --git a/tensorflow/core/ops/nn_ops.cc b/tensorflow/core/ops/nn_ops.cc
index b34dc1a008..5efa55b496 100644
--- a/tensorflow/core/ops/nn_ops.cc
+++ b/tensorflow/core/ops/nn_ops.cc
@@ -1851,7 +1851,8 @@ REGISTER_OP("Relu6Grad")
 Computes rectified linear 6 gradients for a Relu6 operation.
 
 gradients: The backpropagated gradients to the corresponding Relu6 operation.
-features: The features passed as input to the corresponding Relu6 operation.
+features: The features passed as input to the corresponding Relu6 operation, or
+  its output; using either one produces the same result.
 backprops: The gradients:
   `gradients * (features > 0) * (features < 6)`.
 )doc");
diff --git a/tensorflow/python/ops/nn_grad.py b/tensorflow/python/ops/nn_grad.py
index 7dcd72968a..af610d8fdb 100644
--- a/tensorflow/python/ops/nn_grad.py
+++ b/tensorflow/python/ops/nn_grad.py
@@ -349,7 +349,7 @@ def _SeluGradGrad(op, grad):
 
 @ops.RegisterGradient("Relu6")
 def _Relu6Grad(op, grad):
-  return gen_nn_ops._relu6_grad(grad, op.inputs[0])
+  return gen_nn_ops._relu6_grad(grad, op.outputs[0])  # pylint: disable=protected-access
 
 
 @ops.RegisterGradient("Elu")
-- 
GitLab


From 25e6d2331b9e79df9e7a1f296ecc02064ff7c43e Mon Sep 17 00:00:00 2001
From: Vinu Rajashekhar <vinuraja@google.com>
Date: Fri, 6 Oct 2017 15:09:16 -0700
Subject: [PATCH 141/909] Adds helpers for bucketing strategies for TF
 monitoring samplers.

- Adds explicit and exponential strategies for now.

PiperOrigin-RevId: 171350246
---
 .../monitoring/collection_registry_test.cc    |   4 +-
 .../core/lib/monitoring/mobile_sampler.h      |  37 +++++-
 tensorflow/core/lib/monitoring/sampler.cc     | 112 ++++++++++++++++++
 tensorflow/core/lib/monitoring/sampler.h      |  66 ++++++-----
 .../core/lib/monitoring/sampler_test.cc       |  35 +++++-
 5 files changed, 216 insertions(+), 38 deletions(-)
 create mode 100644 tensorflow/core/lib/monitoring/sampler.cc

diff --git a/tensorflow/core/lib/monitoring/collection_registry_test.cc b/tensorflow/core/lib/monitoring/collection_registry_test.cc
index 34a480b07d..5b9c100690 100644
--- a/tensorflow/core/lib/monitoring/collection_registry_test.cc
+++ b/tensorflow/core/lib/monitoring/collection_registry_test.cc
@@ -188,10 +188,10 @@ TEST(CollectMetricsTest, Sampler) {
   auto sampler_with_labels = std::unique_ptr<Sampler<2>>(
       Sampler<2>::New({"/tensorflow/test/sampler_with_labels",
                        "Sampler with labels.", "MyLabel0", "MyLabel1"},
-                      {1.0, 2.0}));
+                      Buckets::Explicit({1.0, 2.0})));
   auto sampler_without_labels = std::unique_ptr<Sampler<0>>(Sampler<0>::New(
       {"/tensorflow/test/sampler_without_labels", "Sampler without labels."},
-      {0.0}));
+      Buckets::Explicit({0.0})));
 
   Histogram with_labels0({1.0, 2.0, DBL_MAX});
   sampler_with_labels->GetCell("Label00", "Label10")->Add(0.7);
diff --git a/tensorflow/core/lib/monitoring/mobile_sampler.h b/tensorflow/core/lib/monitoring/mobile_sampler.h
index 5499237347..cf390e5c7f 100644
--- a/tensorflow/core/lib/monitoring/mobile_sampler.h
+++ b/tensorflow/core/lib/monitoring/mobile_sampler.h
@@ -18,7 +18,10 @@ limitations under the License.
 #ifndef THIRD_PARTY_TENSORFLOW_CORE_LIB_MONITORING_MOBILE_SAMPLER_H_
 #define THIRD_PARTY_TENSORFLOW_CORE_LIB_MONITORING_MOBILE_SAMPLER_H_
 
+#include <memory>
+
 #include "tensorflow/core/framework/summary.pb.h"
+#include "tensorflow/core/lib/monitoring/metric_def.h"
 #include "tensorflow/core/platform/macros.h"
 #include "tensorflow/core/platform/types.h"
 
@@ -38,6 +41,33 @@ class SamplerCell {
   TF_DISALLOW_COPY_AND_ASSIGN(SamplerCell);
 };
 
+// Buckets which has a null implementation.
+class Buckets {
+ public:
+  Buckets() = default;
+  ~Buckets() = default;
+
+  static std::unique_ptr<Buckets> Explicit(
+      std::initializer_list<double> bucket_limits) {
+    return std::unique_ptr<Buckets>(new Buckets());
+  }
+
+  static std::unique_ptr<Buckets> Exponential(double scale,
+                                              double growth_factor,
+                                              int bucket_count) {
+    return std::unique_ptr<Buckets>(new Buckets());
+  }
+
+  const std::vector<double>& explicit_bounds() const {
+    return explicit_bounds_;
+  }
+
+ private:
+  std::vector<double> explicit_bounds_;
+
+  TF_DISALLOW_COPY_AND_ASSIGN(Buckets);
+};
+
 // Sampler which has a null implementation.
 template <int NumLabels>
 class Sampler {
@@ -47,8 +77,8 @@ class Sampler {
   template <typename... MetricDefArgs>
   static Sampler* New(const MetricDef<MetricKind::kCumulative, HistogramProto,
                                       NumLabels>& metric_def,
-                      const std::vector<double>& explicit_bucket_limits) {
-    return new Sampler<NumLabels>();
+                      std::unique_ptr<Buckets> buckets) {
+    return new Sampler<NumLabels>(std::move(buckets));
   }
 
   template <typename... Labels>
@@ -57,9 +87,10 @@ class Sampler {
   }
 
  private:
-  Sampler() {}
+  Sampler(std::unique_ptr<Buckets> buckets) : buckets_(std::move(buckets)) {}
 
   SamplerCell default_sampler_cell_;
+  std::unique_ptr<Buckets> buckets_;
 
   TF_DISALLOW_COPY_AND_ASSIGN(Sampler);
 };
diff --git a/tensorflow/core/lib/monitoring/sampler.cc b/tensorflow/core/lib/monitoring/sampler.cc
new file mode 100644
index 0000000000..23d3668fbd
--- /dev/null
+++ b/tensorflow/core/lib/monitoring/sampler.cc
@@ -0,0 +1,112 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/lib/monitoring/sampler.h"
+
+// We replace this implementation with a null implementation for mobile
+// platforms.
+#include "tensorflow/core/platform/platform.h"
+#ifdef IS_MOBILE_PLATFORM
+// Do nothing.
+#else
+
+namespace tensorflow {
+namespace monitoring {
+namespace {
+
+class ExplicitBuckets : public Buckets {
+ public:
+  ~ExplicitBuckets() override = default;
+
+  explicit ExplicitBuckets(std::vector<double> bucket_limits)
+      : bucket_limits_(std::move(bucket_limits)) {
+    CHECK_GT(bucket_limits_.size(), 0);
+    // Verify that the bucket boundaries are strictly increasing
+    for (size_t i = 1; i < bucket_limits_.size(); i++) {
+      CHECK_GT(bucket_limits_[i], bucket_limits_[i - 1]);
+    }
+    // We augment the bucket limits so that all boundaries are within [-DBL_MAX,
+    // DBL_MAX].
+    //
+    // Since we use ThreadSafeHistogram, we don't have to explicitly add
+    // -DBL_MAX, because it uses these limits as upper-bounds, so
+    // bucket_count[0] is always the number of elements in
+    // [-DBL_MAX, bucket_limits[0]).
+    if (bucket_limits_.back() != DBL_MAX) {
+      bucket_limits_.push_back(DBL_MAX);
+    }
+  }
+
+  const std::vector<double>& explicit_bounds() const override {
+    return bucket_limits_;
+  }
+
+ private:
+  std::vector<double> bucket_limits_;
+
+  TF_DISALLOW_COPY_AND_ASSIGN(ExplicitBuckets);
+};
+
+class ExponentialBuckets : public Buckets {
+ public:
+  ~ExponentialBuckets() override = default;
+
+  ExponentialBuckets(double scale, double growth_factor, int bucket_count)
+      : explicit_buckets_(
+            ComputeBucketLimits(scale, growth_factor, bucket_count)) {}
+
+  const std::vector<double>& explicit_bounds() const override {
+    return explicit_buckets_.explicit_bounds();
+  }
+
+ private:
+  static std::vector<double> ComputeBucketLimits(double scale,
+                                                 double growth_factor,
+                                                 int bucket_count) {
+    CHECK_GT(bucket_count, 0);
+    std::vector<double> bucket_limits;
+    double bound = scale;
+    for (int i = 0; i < bucket_count; i++) {
+      bucket_limits.push_back(bound);
+      bound *= growth_factor;
+    }
+    return bucket_limits;
+  }
+
+  ExplicitBuckets explicit_buckets_;
+
+  TF_DISALLOW_COPY_AND_ASSIGN(ExponentialBuckets);
+};
+
+}  // namespace
+
+// static
+std::unique_ptr<Buckets> Buckets::Explicit(
+    std::initializer_list<double> bucket_limits) {
+  return std::unique_ptr<Buckets>(new ExplicitBuckets(bucket_limits));
+}
+
+// static
+std::unique_ptr<Buckets> Buckets::Exponential(double scale,
+                                              double growth_factor,
+                                              int bucket_count) {
+  return std::unique_ptr<Buckets>(
+      new ExponentialBuckets(scale, growth_factor, bucket_count));
+}
+
+}  // namespace monitoring
+}  // namespace tensorflow
+
+#endif  // IS_MOBILE_PLATFORM
diff --git a/tensorflow/core/lib/monitoring/sampler.h b/tensorflow/core/lib/monitoring/sampler.h
index 3932f8d1a7..5a4d49d5d4 100644
--- a/tensorflow/core/lib/monitoring/sampler.h
+++ b/tensorflow/core/lib/monitoring/sampler.h
@@ -65,12 +65,40 @@ class SamplerCell {
   TF_DISALLOW_COPY_AND_ASSIGN(SamplerCell);
 };
 
+// Bucketing strategies for the samplers.
+//
+// We automatically add -DBL_MAX and DBL_MAX to the ranges, so that no sample
+// goes out of bounds.
+//
+// WARNING: If you are changing the interface here, please do change the same in
+// mobile_sampler.h.
+class Buckets {
+ public:
+  virtual ~Buckets() = default;
+
+  // Sets up buckets of the form:
+  // [-DBL_MAX, ..., scale * growth^i,
+  //   scale * growth_factor^(i + 1), ..., DBL_MAX].
+  //
+  // So for powers of 2 with a bucket count of 10, you would say (1, 2, 10)
+  static std::unique_ptr<Buckets> Exponential(double scale,
+                                              double growth_factor,
+                                              int bucket_count);
+
+  // Sets up buckets of the form:
+  // [-DBL_MAX, ..., bucket_limits[i], bucket_limits[i + 1], ..., DBL_MAX].
+  static std::unique_ptr<Buckets> Explicit(
+      std::initializer_list<double> bucket_limits);
+
+  virtual const std::vector<double>& explicit_bounds() const = 0;
+};
+
 // A stateful class for updating a cumulative histogram metric.
 //
 // This class encapsulates a set of histograms (or a single histogram for a
 // label-less metric) configured with a list of increasing bucket boundaries.
-// Each histogram is identified by a tuple of labels. The class allows the user
-// to add a sample to each histogram value.
+// Each histogram is identified by a tuple of labels. The class allows the
+// user to add a sample to each histogram value.
 //
 // Sampler allocates storage and maintains a cell for each value. You can
 // retrieve an individual cell using a label-tuple and update it separately.
@@ -86,21 +114,14 @@ class Sampler {
     registration_handle_.reset();
   }
 
-  // Creates the metric based on the metric-definition arguments.
+  // Creates the metric based on the metric-definition arguments and buckets.
   //
   // Example;
   // auto* sampler_with_label = Sampler<1>::New({"/tensorflow/sampler",
   //   "Tensorflow sampler", "MyLabelName"}, {10.0, 20.0, 30.0});
-  //
-  // We automatically add -DBL_MAX and DBL_MAX to the list of bucket limits, so
-  // that no sample goes out of bounds. So for the above example, the ranges end
-  // up being: [-DBL_Max, 10.0, 20.0, 30.0, DBL_MAX]
-  //
-  // REQUIRES: bucket_limits[i] values are monotonically increasing.
-  // REQUIRES: bucket_limits is not empty().
   static Sampler* New(const MetricDef<MetricKind::kCumulative, HistogramProto,
                                       NumLabels>& metric_def,
-                      const std::vector<double>& bucket_limits);
+                      std::unique_ptr<Buckets> buckets);
 
   // Retrieves the cell for the specified labels, creating it on demand if
   // not already present.
@@ -112,9 +133,9 @@ class Sampler {
 
   Sampler(const MetricDef<MetricKind::kCumulative, HistogramProto, NumLabels>&
               metric_def,
-          const std::vector<double>& bucket_limits)
+          std::unique_ptr<Buckets> buckets)
       : metric_def_(metric_def),
-        bucket_limits_(bucket_limits),
+        buckets_(std::move(buckets)),
         registration_handle_(CollectionRegistry::Default()->Register(
             &metric_def_, [&](MetricCollectorGetter getter) {
               auto metric_collector = getter.Get(&metric_def_);
@@ -133,7 +154,7 @@ class Sampler {
       metric_def_;
 
   // Bucket limits for the histograms in the cells.
-  const std::vector<double> bucket_limits_;
+  std::unique_ptr<Buckets> buckets_;
 
   // Registration handle with the CollectionRegistry.
   std::unique_ptr<CollectionRegistry::RegistrationHandle> registration_handle_;
@@ -162,19 +183,8 @@ template <int NumLabels>
 Sampler<NumLabels>* Sampler<NumLabels>::New(
     const MetricDef<MetricKind::kCumulative, HistogramProto, NumLabels>&
         metric_def,
-    const std::vector<double>& bucket_limits) {
-  CHECK_GT(bucket_limits.size(), 0);
-  // Verify that the bucket boundaries are strictly increasing
-  for (size_t i = 1; i < bucket_limits.size(); i++) {
-    CHECK_GT(bucket_limits[i], bucket_limits[i - 1]);
-  }
-  std::vector<double> augmented_bucket_limits(bucket_limits);
-  // We add DBL_MAX to the end so that all boundaries are within [-DBL_MAX,
-  // DBL_MAX].
-  if (bucket_limits.back() != DBL_MAX) {
-    augmented_bucket_limits.push_back(DBL_MAX);
-  }
-  return new Sampler<NumLabels>(metric_def, augmented_bucket_limits);
+    std::unique_ptr<Buckets> buckets) {
+  return new Sampler<NumLabels>(metric_def, std::move(buckets));
 }
 
 template <int NumLabels>
@@ -196,7 +206,7 @@ SamplerCell* Sampler<NumLabels>::GetCell(const Labels&... labels)
   return &(cells_
                .emplace(std::piecewise_construct,
                         std::forward_as_tuple(label_array),
-                        std::forward_as_tuple(bucket_limits_))
+                        std::forward_as_tuple(buckets_->explicit_bounds()))
                .first->second);
 }
 
diff --git a/tensorflow/core/lib/monitoring/sampler_test.cc b/tensorflow/core/lib/monitoring/sampler_test.cc
index 27e1ccca3c..d61d858b6b 100644
--- a/tensorflow/core/lib/monitoring/sampler_test.cc
+++ b/tensorflow/core/lib/monitoring/sampler_test.cc
@@ -34,14 +34,14 @@ void EqHistograms(const Histogram& expected,
 auto* sampler_with_labels =
     Sampler<1>::New({"/tensorflow/test/sampler_with_labels",
                      "Sampler with one label.", "MyLabel"},
-                    {10.0, 20.0});
+                    Buckets::Explicit({10.0, 20.0}));
 
 TEST(LabeledSamplerTest, InitializedEmpty) {
   Histogram empty;
   EqHistograms(empty, sampler_with_labels->GetCell("Empty")->value());
 }
 
-TEST(LabeledSamplerTest, BucketBoundaries) {
+TEST(LabeledSamplerTest, ExplicitBucketBoundaries) {
   // Sampler automatically adds DBL_MAX to the list of buckets.
   Histogram expected({10.0, 20.0, DBL_MAX});
   auto* cell = sampler_with_labels->GetCell("BucketBoundaries");
@@ -61,7 +61,7 @@ TEST(LabeledSamplerTest, BucketBoundaries) {
 auto* init_sampler_without_labels =
     Sampler<0>::New({"/tensorflow/test/init_sampler_without_labels",
                      "Sampler without labels initialized as empty."},
-                    {1.5, 2.8});
+                    Buckets::Explicit({1.5, 2.8}));
 
 TEST(UnlabeledSamplerTest, InitializedEmpty) {
   Histogram empty;
@@ -71,9 +71,9 @@ TEST(UnlabeledSamplerTest, InitializedEmpty) {
 auto* sampler_without_labels =
     Sampler<0>::New({"/tensorflow/test/sampler_without_labels",
                      "Sampler without labels initialized as empty."},
-                    {1.5, 2.8});
+                    Buckets::Explicit({1.5, 2.8}));
 
-TEST(UnlabeledSamplerTest, BucketBoundaries) {
+TEST(UnlabeledSamplerTest, ExplicitBucketBoundaries) {
   // Sampler automatically adds DBL_MAX to the list of buckets.
   Histogram expected({1.5, 2.8, DBL_MAX});
   auto* cell = sampler_without_labels->GetCell();
@@ -87,6 +87,31 @@ TEST(UnlabeledSamplerTest, BucketBoundaries) {
   EqHistograms(expected, cell->value());
 }
 
+auto* sampler_with_exponential =
+    Sampler<1>::New({"/tensorflow/test/sampler_with_exponential",
+                     "Sampler with exponential buckets.", "MyLabel"},
+                    // So limits are {1, 2, 4}.
+                    Buckets::Exponential(1, 2, 3));
+
+TEST(ExponentialSamplerTest, ExponentialBucketBoundaries) {
+  // Sampler automatically adds DBL_MAX to the list of buckets.
+  Histogram expected({1.0, 2.0, 4.0, DBL_MAX});
+  auto* cell = sampler_with_exponential->GetCell("BucketBoundaries");
+  sampler_with_exponential->GetCell("AddedToCheckPreviousCellValidity");
+  cell->Add(-1.0);
+  expected.Add(-1.0);
+  cell->Add(0.5);
+  expected.Add(0.5);
+  cell->Add(1.001);
+  expected.Add(1.001);
+  cell->Add(3.999);
+  expected.Add(3.999);
+  cell->Add(6.0);
+  expected.Add(6.0);
+
+  EqHistograms(expected, cell->value());
+}
+
 }  // namespace
 }  // namespace monitoring
 }  // namespace tensorflow
-- 
GitLab


From ea513ed3ec78531af1ebdb25b2daf52bd688b4d0 Mon Sep 17 00:00:00 2001
From: ZxYuan <lvp0526@gmail.com>
Date: Fri, 6 Oct 2017 17:16:28 -0500
Subject: [PATCH 142/909] Update word2vec_basic.py (#13531)

Use random.sample to simplify random selection of context words
---
 tensorflow/examples/tutorials/word2vec/word2vec_basic.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/tensorflow/examples/tutorials/word2vec/word2vec_basic.py b/tensorflow/examples/tutorials/word2vec/word2vec_basic.py
index 1fa2b14869..142e45a2e8 100644
--- a/tensorflow/examples/tutorials/word2vec/word2vec_basic.py
+++ b/tensorflow/examples/tutorials/word2vec/word2vec_basic.py
@@ -115,11 +115,9 @@ def generate_batch(batch_size, num_skips, skip_window):
   data_index += span
   for i in range(batch_size // num_skips):
     context_words = [w for w in range(span) if w != skip_window]
-    random.shuffle(context_words)
-    words_to_use = collections.deque(context_words)
-    for j in range(num_skips):
+    words_to_use = random.sample(context_words, num_skips)
+    for j, context_word in enumerate(words_to_use):
       batch[i * num_skips + j] = buffer[skip_window]
-      context_word = words_to_use.pop()
       labels[i * num_skips + j, 0] = buffer[context_word]
     if data_index == len(data):
       buffer[:] = data[:span]
-- 
GitLab


From c5f715f62e7d8c4fbf9244eefb9379f188e06b98 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 6 Oct 2017 15:20:39 -0700
Subject: [PATCH 143/909] Update ops-related pbtxt files.

PiperOrigin-RevId: 171351986
---
 tensorflow/core/ops/ops.pbtxt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index 9cda34a8c8..9abb4f7a5e 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -20208,7 +20208,7 @@ op {
   }
   input_arg {
     name: "features"
-    description: "The features passed as input to the corresponding Relu6 operation."
+    description: "The features passed as input to the corresponding Relu6 operation, or\nits output; using either one produces the same result."
     type_attr: "T"
   }
   output_arg {
-- 
GitLab


From 710efeecbffad94259bdcf5d19ca3a83043cf145 Mon Sep 17 00:00:00 2001
From: Igor Ganichev <iga@google.com>
Date: Fri, 6 Oct 2017 15:25:16 -0700
Subject: [PATCH 144/909] Bump min graph consumer version when adding functions
 to it

PiperOrigin-RevId: 171352662
---
 tensorflow/core/graph/graph.cc                | 9 +++++++++
 tensorflow/core/graph/graph_partition_test.cc | 5 ++---
 2 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/tensorflow/core/graph/graph.cc b/tensorflow/core/graph/graph.cc
index 2ad0081e1f..daefb6b1fb 100644
--- a/tensorflow/core/graph/graph.cc
+++ b/tensorflow/core/graph/graph.cc
@@ -293,6 +293,11 @@ Graph::Graph(const OpRegistryInterface* ops)
 
 Graph::Graph(const FunctionLibraryDefinition& flib_def)
     : Graph(flib_def.default_registry()) {
+  // Need a new-enough consumer to support the functions we add to the graph.
+  if (flib_def.ToProto().function_size() > 0 &&
+      versions_->min_consumer() < 12) {
+    versions_->set_min_consumer(12);
+  }
   Status s = ops_.AddLibrary(flib_def);
   CHECK(s.ok()) << s.error_message();
 }
@@ -448,6 +453,10 @@ const Edge* Graph::FindEdge(const Node* dst, int index) {
 }
 
 Status Graph::AddFunctionLibrary(const FunctionDefLibrary& fdef_lib) {
+  // Need a new-enough consumer to support the functions we add to the graph.
+  if (fdef_lib.function_size() > 0 && versions_->min_consumer() < 12) {
+    versions_->set_min_consumer(12);
+  }
   return ops_.AddLibrary(fdef_lib);
 }
 
diff --git a/tensorflow/core/graph/graph_partition_test.cc b/tensorflow/core/graph/graph_partition_test.cc
index 858ef8ac01..20822ecb1d 100644
--- a/tensorflow/core/graph/graph_partition_test.cc
+++ b/tensorflow/core/graph/graph_partition_test.cc
@@ -91,10 +91,9 @@ void Partition(const GraphDef& graph_def,
   Status s = Partition(popts, &g, partitions);
   CHECK(s.ok()) << s;
 
-  // Check versions
+  // Check versions.
   EXPECT_EQ(graph_def.versions().producer(), TF_GRAPH_DEF_VERSION);
-  EXPECT_EQ(graph_def.versions().min_consumer(),
-            TF_GRAPH_DEF_VERSION_MIN_CONSUMER);
+  // Partitions must inherit the versions of the original graph.
   for (auto& it : *partitions) {
     EXPECT_EQ(graph_def.versions().producer(), it.second.versions().producer());
     EXPECT_EQ(graph_def.versions().min_consumer(),
-- 
GitLab


From a713e49e8662b90eea3b5cda9bd50ae4c7546fef Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 6 Oct 2017 15:27:37 -0700
Subject: [PATCH 145/909] Go: Update generated wrapper functions for TensorFlow
 ops.

PiperOrigin-RevId: 171352952
---
 tensorflow/go/op/wrappers.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go
index f2ee710a9e..804275dda6 100644
--- a/tensorflow/go/op/wrappers.go
+++ b/tensorflow/go/op/wrappers.go
@@ -22810,7 +22810,8 @@ func ResizeBicubic(scope *Scope, images tf.Output, size tf.Output, optional ...R
 //
 // Arguments:
 //	gradients: The backpropagated gradients to the corresponding Relu6 operation.
-//	features: The features passed as input to the corresponding Relu6 operation.
+//	features: The features passed as input to the corresponding Relu6 operation, or
+// its output; using either one produces the same result.
 //
 // Returns The gradients:
 // `gradients * (features > 0) * (features < 6)`.
-- 
GitLab


From d9a969c84b56fc5bca7ddbb58761303cafee94bd Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Fri, 6 Oct 2017 15:48:17 -0700
Subject: [PATCH 146/909] Disable some tests on tsan.

PiperOrigin-RevId: 171355854
---
 tensorflow/python/estimator/BUILD | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/estimator/BUILD b/tensorflow/python/estimator/BUILD
index 3507d9fedc..22de474013 100644
--- a/tensorflow/python/estimator/BUILD
+++ b/tensorflow/python/estimator/BUILD
@@ -242,7 +242,10 @@ py_test(
     srcs = ["canned/dnn_test.py"],
     shard_count = 4,
     srcs_version = "PY2AND3",
-    tags = ["no_pip"],
+    tags = [
+        "no_pip",
+        "notsan",  # b/67510291
+    ],
     deps = [
         ":dnn",
         ":dnn_testing_utils",
@@ -296,7 +299,10 @@ py_test(
     srcs = ["canned/dnn_linear_combined_test.py"],
     shard_count = 8,
     srcs_version = "PY2AND3",
-    tags = ["no_pip"],
+    tags = [
+        "no_pip",
+        "notsan",  # b/67510291
+    ],
     deps = [
         ":dnn_linear_combined",
         ":dnn_testing_utils",
@@ -373,6 +379,7 @@ py_test(
     name = "estimator_test",
     srcs = ["estimator_test.py"],
     srcs_version = "PY2AND3",
+    tags = ["notsan"],  # b/67510291
     deps = [
         ":estimator",
         ":export_export",
@@ -646,7 +653,10 @@ py_test(
     srcs = ["canned/linear_test.py"],
     shard_count = 4,
     srcs_version = "PY2AND3",
-    tags = ["no_pip"],
+    tags = [
+        "no_pip",
+        "notsan",  # b/67510291
+    ],
     deps = [
         ":linear",
         ":linear_testing_utils",
-- 
GitLab


From be893ac19b13a77c645e168b6ab3f835062c4280 Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenlavoie@gmail.com>
Date: Fri, 6 Oct 2017 15:53:53 -0700
Subject: [PATCH 147/909] Clean up our libcuda stub when building the GPU
 Docker container (#13456)

---
 tensorflow/tools/docker/Dockerfile.devel-gpu | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/tensorflow/tools/docker/Dockerfile.devel-gpu b/tensorflow/tools/docker/Dockerfile.devel-gpu
index 04773376e9..a607e5e27b 100644
--- a/tensorflow/tools/docker/Dockerfile.devel-gpu
+++ b/tensorflow/tools/docker/Dockerfile.devel-gpu
@@ -78,15 +78,18 @@ WORKDIR /tensorflow
 
 # Configure the build for our CUDA configuration.
 ENV CI_BUILD_PYTHON python
-ENV LD_LIBRARY_PATH /usr/local/cuda/lib64/stubs:/usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH
+ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH
 ENV TF_NEED_CUDA 1
 ENV TF_CUDA_COMPUTE_CAPABILITIES=3.0,3.5,5.2,6.0,6.1
 
-RUN ln -s /usr/local/cuda/lib64/stubs/libcuda.so /usr/local/cuda/lib64/stubs/libcuda.so.1
 
-RUN tensorflow/tools/ci_build/builds/configured GPU \
-    bazel build -c opt --config=cuda --cxxopt="-D_GLIBCXX_USE_CXX11_ABI=0" \
+RUN ln -s /usr/local/cuda/lib64/stubs/libcuda.so /usr/local/cuda/lib64/stubs/libcuda.so.1 && \
+    LD_LIBRARY_PATH=/usr/local/cuda/lib64/stubs:${LD_LIBRARY_PATH} \
+    tensorflow/tools/ci_build/builds/configured GPU \
+    bazel build -c opt --config=cuda \
+	--cxxopt="-D_GLIBCXX_USE_CXX11_ABI=0" \
         tensorflow/tools/pip_package:build_pip_package && \
+    rm /usr/local/cuda/lib64/stubs/libcuda.so.1 && \
     bazel-bin/tensorflow/tools/pip_package/build_pip_package /tmp/pip && \
     pip --no-cache-dir install --upgrade /tmp/pip/tensorflow-*.whl && \
     rm -rf /tmp/pip && \
-- 
GitLab


From febf2e69608acae22f9b33e54e1088b7e1e0749c Mon Sep 17 00:00:00 2001
From: Amit Patankar <amitpatankar@google.com>
Date: Fri, 6 Oct 2017 15:54:01 -0700
Subject: [PATCH 148/909] Update README.md with tf-nightly-gpu

---
 README.md | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index 6339c57c95..24bbb6cec1 100644
--- a/README.md
+++ b/README.md
@@ -38,10 +38,11 @@ People who are a little more adventurous can also try our nightly binaries:
 
 **Nightly pip packages**
 * We are pleased to announce that TensorFlow now offers nightly pip packages
-under the [tf-nightly](https://pypi.python.org/pypi/tf-nightly) project on pypi.
-Simply run `pip install tf-nightly` in a clean environment to install the nightly
-tensorflow  build. We currently only support CPU packages on Linux, Mac, and Windows.
-GPU packages on all platforms will arrive soon!
+under the [tf-nightly](https://pypi.python.org/pypi/tf-nightly) and
+[tf-nightly-gpu](https://pypi.python.org/pypi/tf-nightly-gpu) project on pypi.
+Simply run `pip install tf-nightly` or `pip install tf-nightly-gpu` in a clean
+environment to install the nightly TensorFlow build. We support CPU and GPU
+packages on Linux, Mac, and Windows.
 
 
 **Individual whl files**
-- 
GitLab


From 09369376b4ee41eafc674ce7a699fd74ee9468d5 Mon Sep 17 00:00:00 2001
From: melvyniandrag <melvyniandrag@gmail.com>
Date: Fri, 6 Oct 2017 19:35:34 -0400
Subject: [PATCH 149/909] modified readme (#13515)

---
 tensorflow/tools/docker/README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/tools/docker/README.md b/tensorflow/tools/docker/README.md
index 3780bde2be..2e5a0038ed 100644
--- a/tensorflow/tools/docker/README.md
+++ b/tensorflow/tools/docker/README.md
@@ -41,6 +41,7 @@ Note: If you would have a problem running nvidia-docker you may try the old meth
 we have used. But it is not recommended. If you find a bug in nvidia-docker, please report
 it there and try using nvidia-docker as described above.
 
+    $ # The old, not recommended way to run docker with gpu support: 
     $ export CUDA_SO=$(\ls /usr/lib/x86_64-linux-gnu/libcuda.* | xargs -I{} echo '-v {}:{}')
     $ export DEVICES=$(\ls /dev/nvidia* | xargs -I{} echo '--device {}:{}')
     $ docker run -it -p 8888:8888 $CUDA_SO $DEVICES gcr.io/tensorflow/tensorflow:latest-gpu
-- 
GitLab


From 8018fc9385647876b3ce954e4d9a345316526b0b Mon Sep 17 00:00:00 2001
From: "Dr. Kashif Rasul" <kashif.rasul@gmail.com>
Date: Sat, 7 Oct 2017 01:36:45 +0200
Subject: [PATCH 150/909] instructions for libcupti for CUDA 8 (#13414)

---
 tensorflow/docs_src/install/install_linux.md   | 14 +++++++++++++-
 tensorflow/docs_src/install/install_sources.md | 11 +++++++++--
 2 files changed, 22 insertions(+), 3 deletions(-)

diff --git a/tensorflow/docs_src/install/install_linux.md b/tensorflow/docs_src/install/install_linux.md
index 576099f054..14cc1f733c 100644
--- a/tensorflow/docs_src/install/install_linux.md
+++ b/tensorflow/docs_src/install/install_linux.md
@@ -42,8 +42,20 @@ must be installed on your system:
     a list of supported GPU cards.
   * The libcupti-dev library, which is the NVIDIA CUDA Profile Tools Interface.
     This library provides advanced profiling support. To install this library,
-    issue the following command:
+    issue the following command for CUDA Toolkit >= 8.0:
 
+    <pre>
+    $ <b>sudo apt-get install cuda-command-line-tools</b>
+    </pre>
+    
+    and add its path to your `LD_LIBRARY_PATH` environment variable:
+
+    <pre> 
+    $ <b>export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda/extras/CUPTI/lib64</b> 
+    </pre>
+
+    For CUDA Toolkit <= 7.5 do:
+    
     <pre>
     $ <b>sudo apt-get install libcupti-dev</b>
     </pre>
diff --git a/tensorflow/docs_src/install/install_sources.md b/tensorflow/docs_src/install/install_sources.md
index e6a4088656..3d143506f0 100644
--- a/tensorflow/docs_src/install/install_sources.md
+++ b/tensorflow/docs_src/install/install_sources.md
@@ -137,8 +137,15 @@ The following NVIDIA <i>software</i> must be installed on your system:
     particularly the description of appending the appropriate pathname
     to your `LD_LIBRARY_PATH` environment variable.
 
-Finally, you must also install `libcupti-dev` by invoking the following
-command:
+Finally, you must also install `libcupti` which for Cuda Toolkit >= 8.0 you do via 
+
+<pre> $ <b>sudo apt-get install cuda-command-line-tools</b> </pre>
+
+and add its path to your `LD_LIBRARY_PATH` environment variable:
+
+<pre> $ <b>export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda/extras/CUPTI/lib64</b> </pre>
+
+For Cuda Toolkit <= 7.5, you install `libcupti-dev` by invoking the following command:
 
 <pre> $ <b>sudo apt-get install libcupti-dev</b> </pre>
 
-- 
GitLab


From 6fc7de9522e0d1ed6f1e1d5fd095fdeb6a31b197 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 6 Oct 2017 16:41:17 -0700
Subject: [PATCH 151/909] Define object-oriented metrics classes that are
 Eager-safe.

PiperOrigin-RevId: 171363240
---
 tensorflow/contrib/eager/python/BUILD         |  31 +++
 tensorflow/contrib/eager/python/metrics.py    |  26 +++
 .../contrib/eager/python/metrics_impl.py      | 197 ++++++++++++++++++
 .../contrib/eager/python/metrics_test.py      |  59 ++++++
 4 files changed, 313 insertions(+)
 create mode 100644 tensorflow/contrib/eager/python/metrics.py
 create mode 100644 tensorflow/contrib/eager/python/metrics_impl.py
 create mode 100644 tensorflow/contrib/eager/python/metrics_test.py

diff --git a/tensorflow/contrib/eager/python/BUILD b/tensorflow/contrib/eager/python/BUILD
index 9185c963f7..1a63c901a2 100644
--- a/tensorflow/contrib/eager/python/BUILD
+++ b/tensorflow/contrib/eager/python/BUILD
@@ -11,6 +11,7 @@ py_library(
     srcs_version = "PY2AND3",
     deps = [
         ":datasets",
+        ":metrics",
         ":saver",
         ":summary_writer",
         "//tensorflow/python:framework_ops",
@@ -116,6 +117,36 @@ cuda_py_test(
     ],
 )
 
+py_library(
+    name = "metrics",
+    srcs = [
+        "metrics.py",
+        "metrics_impl.py",
+    ],
+    srcs_version = "PY2AND3",
+    visibility = ["//tensorflow:internal"],
+    deps = [
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:dtypes",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:init_ops",
+        "//tensorflow/python:layers_base",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:util",
+        "//tensorflow/python:variable_scope",
+    ],
+)
+
+py_test(
+    name = "metrics_test",
+    srcs = ["metrics_test.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":metrics",
+        "//tensorflow/python/eager:test",
+    ],
+)
+
 filegroup(
     name = "all_files",
     srcs = glob(
diff --git a/tensorflow/contrib/eager/python/metrics.py b/tensorflow/contrib/eager/python/metrics.py
new file mode 100644
index 0000000000..3e31004273
--- /dev/null
+++ b/tensorflow/contrib/eager/python/metrics.py
@@ -0,0 +1,26 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Metrics namespace."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+# pylint:disable=wildcard-import
+from tensorflow.contrib.eager.python.metrics_impl import *
+from tensorflow.python.util.all_util import remove_undocumented
+
+_allowed_symbols = ['Accuracy', 'Mean', 'Metric']
+remove_undocumented(__name__, _allowed_symbols)
diff --git a/tensorflow/contrib/eager/python/metrics_impl.py b/tensorflow/contrib/eager/python/metrics_impl.py
new file mode 100644
index 0000000000..6bc0ce6dce
--- /dev/null
+++ b/tensorflow/contrib/eager/python/metrics_impl.py
@@ -0,0 +1,197 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Metrics classes for computing the output of an evaluation."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.framework import dtypes
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import init_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import variable_scope
+
+
+class Metric(object):
+  """A metric holds state for aggregating statistics over an evaluation run.
+
+  Users will use Network.add_metric() to add Metric objects to their
+  evaluation network, call them in each step, and then use
+  Network.all_metric_results() at the end.
+
+  Descendants will implement:
+  * call(): Should follow this pattern:
+      if not self.built:
+        self.var = self.add_variable(...)
+      self.add_update(self.var.assign_add(...))
+  * aggregate(): Adds in the state from a list of metrics of the same type
+    as `self`.  (Default of summing all the variables will be fine for most
+    descendants.)
+  * result(): Computes and returns a final value for the metric
+    from the variables in `self`.
+  """
+
+  def __init__(self, name=None):
+    self.built = False
+    self._vars = []
+    self._updates = []
+    self._name = name or self.__class__.__name__
+    # TODO(josh11b): Need some way to make sure two Metrics in the same
+    # Network have distinct names. Maybe we can get a unique name from
+    # a name/variable scope?
+    # TODO(josh11b): self._in_graph_mode = context.in_graph_mode()
+
+  # ---- API for users ---
+  def __call__(self, *args, **kwargs):
+    # TODO(josh11b): If self._in_graph_mode is true, make self.call() into a
+    # graph callable here, so that variable updates happen without requiring
+    # a separate fetch.
+    # TODO(josh11b): Do we need a separate build() method to separate
+    # initialization from each update? If so, how do we get the arguments
+    # to it?  We *could* just pass in *args and **kwargs...
+    if not self.built:
+      # TODO(ashankar): Set up container isolation so there is no chance
+      # distinct metrics objects accidentally share variables.
+      with variable_scope.variable_scope(
+          self._name, use_resource=True, reuse=False):
+        ret = self.call(*args, **kwargs)
+      self.built = True
+    else:
+      ret = self.call(*args, **kwargs)
+    return ret
+
+  @property
+  def name(self):
+    return self._name
+
+  @property
+  def variables(self):
+    return self._vars
+
+  # ---- To be implemented by descendants ---
+  def call(self, *args, **kwargs):
+    """Accumulates statistics for the metric."""
+    raise NotImplementedError("Metrics must define a call() member function")
+
+  # We can support two different strategies of for doing data-parallel
+  # distributed metric computations:
+  # * Put metric variables on the first device and rely on small
+  #   bandwidth needed to do updates. (Doesn't require any particular
+  #   code in Metric implementations.)
+  # * Ask each type of metric to define an aggregation method to run
+  #   at the end of eval to merge across devices. Note: this is good
+  #   for the use case where they want to record the metric's state
+  #   for each example and then later decide which examples they want
+  #   to aggregate over. (Recommended -- not too much harder and adds
+  #   flexibilty over previous option.)
+  # I'm going with the second strategy since we can define a default
+  # implementation of aggregate() that will work for most descendants.
+  def aggregate(self, metrics):
+    """Adds in the state from a list of metrics.
+
+    Default implementation sums all the metric variables.
+
+    Args:
+      metrics: A list of metrics with the same type as `self`.
+
+    Raises:
+      ValueError: If metrics contains invalid data.
+    """
+    for m in metrics:
+      if type(self) != type(m):  # pylint: disable=unidiomatic-typecheck
+        raise TypeError("All metrics must be the same type, '%s' != '%s'." %
+                        (type(self), type(m)))
+    # pylint: disable=protected-access
+    for i in range(len(self._vars)):
+      if any(m._vars[i].name != self._vars[i].name for m in metrics):
+        raise ValueError("All metrics must have variables in the same order.")
+      self._vars[i].assign_add(math_ops.add_n([m._vars[i] for m in metrics]))
+    # pylint: enable=protected-access
+
+  def result(self):
+    """Computes and returns a final value for the metric."""
+    raise NotImplementedError("Metrics must define a result() member function")
+
+  # ---- For use by descendants ---
+  def add_variable(self, name, shape=None, dtype=None, initializer=None):
+    """***Only for use by descendants of Metric***."""
+    if self.built:
+      raise RuntimeError("Can't call add_variable() after a Metric has been "
+                         "built in the first call().")
+    v = variable_scope.get_variable(name, shape, dtype, initializer,
+                                    trainable=False, use_resource=True)
+    self._vars.append(v)
+    return v
+
+
+class Mean(Metric):
+  """Computes the (weighted) mean of the given values."""
+  # TODO(josh11b): Maybe have a dtype argument that defaults to tf.float64?
+  # Or defaults to type of the input if it is tf.float32, else tf.float64?
+
+  def call(self, values, weights=None):
+    """Accumulate statistics for computing the mean.
+
+    For example, if values is [1, 3, 5, 7] then the mean is 4.
+    If the weights were specified as [1, 1, 0, 0] then the mean would be 2.
+
+    Args:
+      values: Tensor with the per-example value.
+      weights: Optional weighting of each example. Defaults to 1.
+    """
+    if not self.built:  # False only in the first call().
+      self.numer = self.add_variable(name="numer", shape=(),
+                                     dtype=dtypes.float64,
+                                     initializer=init_ops.zeros_initializer)
+      self.denom = self.add_variable(name="denom", shape=(),
+                                     dtype=dtypes.float64,
+                                     initializer=init_ops.zeros_initializer)
+    if weights is None:
+      self.denom.assign_add(
+          math_ops.cast(array_ops.size(values), dtypes.float64))
+      values = math_ops.reduce_sum(values)
+      self.numer.assign_add(math_ops.cast(values, dtypes.float64))
+    else:
+      weights = math_ops.cast(weights, dtypes.float64)
+      self.denom.assign_add(math_ops.reduce_sum(weights))
+      values = math_ops.cast(values, dtypes.float64) * weights
+      self.numer.assign_add(math_ops.reduce_sum(values))
+
+  def result(self):
+    return self.numer / self.denom
+
+
+class Accuracy(Mean):
+  """Calculates how often `predictions` matches `labels`."""
+
+  def call(self, labels, predictions, weights=None):
+    """Accumulate accuracy statistics.
+
+    For example, if labels is [1, 2, 3, 4] and predictions is [0, 2, 3, 4]
+    then the accuracy is 3/4 or .75.  If the weights were specified as
+    [1, 1, 0, 0] then the accuracy would be 1/2 or .5.
+
+    `labels` and `predictions` should have the same shape and type.
+
+    Args:
+      labels: Tensor with the true labels for each example.  One example
+        per element of the Tensor.
+      predictions: Tensor with the predicted label for each example.
+      weights: Optional weighting of each example. Defaults to 1.
+    """
+    matches = math_ops.equal(labels, predictions)
+    matches = math_ops.cast(matches, dtypes.float64)
+    super(Accuracy, self).call(matches, weights=weights)
diff --git a/tensorflow/contrib/eager/python/metrics_test.py b/tensorflow/contrib/eager/python/metrics_test.py
new file mode 100644
index 0000000000..8c2d8081ba
--- /dev/null
+++ b/tensorflow/contrib/eager/python/metrics_test.py
@@ -0,0 +1,59 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.contrib.eager.python import metrics
+from tensorflow.python.eager import test
+
+
+class MetricsTest(test.TestCase):
+
+  def testMean(self):
+    m = metrics.Mean()
+    m([1, 10, 100])
+    m(1000)
+    m([10000.0, 100000.0])
+    self.assertEqual(111111.0/6, m.result().numpy())
+
+  def testWeightedMean(self):
+    m = metrics.Mean()
+    m([1, 100, 100000], weights=[1, 0.2, 0.3])
+    m([500000, 5000, 500])  # weights of 1 each
+    self.assertNear(535521/4.5, m.result().numpy(), 0.001)
+
+  def testAccuracy(self):
+    m = metrics.Accuracy()
+    m([0, 1, 2, 3], [0, 0, 0, 0])  # 1 correct
+    m([4], [4])  # 1 correct
+    m([5], [0])  # 0 correct
+    m([6], [6])  # 1 correct
+    m([7], [2])  # 0 correct
+    self.assertEqual(3.0/8, m.result().numpy())
+
+  def testWeightedAccuracy(self):
+    m = metrics.Accuracy()
+    # 1 correct, total weight of 2
+    m([0, 1, 2, 3], [0, 0, 0, 0], weights=[1, 1, 0, 0])
+    m([4], [4], weights=[0.5])  # 1 correct with a weight of 0.5
+    m([5], [0], weights=[0.5])  # 0 correct, weight 0.5
+    m([6], [6])  # 1 correct, weight 1
+    m([7], [2])  # 0 correct, weight 1
+    self.assertEqual(2.5/5, m.result().numpy())
+
+
+if __name__ == "__main__":
+  test.main()
-- 
GitLab


From c26542cdaeb4cd815406a8175251ff76cdfbc20a Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Fri, 6 Oct 2017 17:08:19 -0700
Subject: [PATCH 152/909] [XLA] Don't clone and throw away instructions without
 calling DetachFromOperands.

If you clone an instruction and then don't insert it into a computation,
it's on you to call DetachFromOperands before destroying it.  Otherwise
the instruction will stay in its operands' use lists.

PiperOrigin-RevId: 171367649
---
 .../compiler/xla/service/algebraic_simplifier.cc    | 13 ++++---------
 tensorflow/compiler/xla/service/hlo_evaluator.cc    | 13 +++++++++++--
 2 files changed, 15 insertions(+), 11 deletions(-)

diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier.cc b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
index 4858f47c59..dd97f3d876 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
@@ -1782,7 +1782,7 @@ static const HloInstruction* NonConstantOperand(const HloInstruction* instr) {
 
 // Tries to determine the number of times the given loop executes.  Currently
 // simply returns 0, 1, or "can't tell" (nullopt).
-static optional<int64> GetLoopTripCount(const HloInstruction* while_op) {
+static optional<int64> GetLoopTripCount(HloInstruction* while_op) {
   CHECK_EQ(while_op->opcode(), HloOpcode::kWhile);
   VLOG(2) << "Getting trip count for loop " << while_op->ToString();
 
@@ -1803,15 +1803,10 @@ static optional<int64> GetLoopTripCount(const HloInstruction* while_op) {
   // compute how many times the loop executes.  Start by computing the induction
   // variable's initial value.
   HloEvaluator evaluator;
-  auto* while_init = while_op->operand(0);
-  auto* indvar_init = while_init->operand(*indvar_tuple_idx);
-  // TODO(b/67157142): This should not be redundant, remove this when the
-  // underlying issue has been addressed.
-  if (!hlo_query::AllOperandsAreConstants(*indvar_init)) {
-    return nullopt;
-  }
+  auto* while_init = while_op->mutable_operand(0);
+  auto* indvar_init = while_init->mutable_operand(*indvar_tuple_idx);
   StatusOr<std::unique_ptr<Literal>> indvar_init_result =
-      evaluator.Evaluate(indvar_init->Clone().get());
+      evaluator.Evaluate(indvar_init);
   if (!indvar_init_result.ok()) {
     VLOG(2) << "Couldn't evaluate induction variable init: "
             << indvar_init_result.status();
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator.cc b/tensorflow/compiler/xla/service/hlo_evaluator.cc
index 4f9d6c0096..61c59987f5 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator.cc
+++ b/tensorflow/compiler/xla/service/hlo_evaluator.cc
@@ -1285,8 +1285,17 @@ StatusOr<std::unique_ptr<Literal>> HloEvaluator::EvaluateWithSubstitutions(
     operands.push_back(operand.get());
   }
 
-  return Evaluate(
-      instruction->CloneWithNewOperands(instruction->shape(), operands).get());
+  std::unique_ptr<HloInstruction> cloned_instruction =
+      instruction->CloneWithNewOperands(instruction->shape(), operands);
+  auto result = Evaluate(cloned_instruction.get());
+
+  // Clean up our cloned instructions before returning.
+  cloned_instruction->DetachFromOperands();
+  for (auto& operand : owned_operands) {
+    operand->DetachFromOperands();
+  }
+
+  return result;
 }
 
 Status HloEvaluator::HandleParameter(HloInstruction* parameter) {
-- 
GitLab


From fb3c68db3fd9d1f18f8c5f8d6b005523dfcdf34d Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Fri, 6 Oct 2017 17:30:25 -0700
Subject: [PATCH 153/909] Disable keras:models_test in tsan mode.

PiperOrigin-RevId: 171369892
---
 tensorflow/python/keras/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index f1266cdf9e..03bf9d2177 100644
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -654,6 +654,7 @@ py_test(
     size = "small",
     srcs = ["_impl/keras/models_test.py"],
     srcs_version = "PY2AND3",
+    tags = ["notsan"],  # b/67509773
     deps = [
         ":keras",
         "//tensorflow/python:client_testlib",
-- 
GitLab


From 646db3e3f91cdfcb1d00eb2bd8bc510ce453e7d3 Mon Sep 17 00:00:00 2001
From: Asim Shankar <ashankar@google.com>
Date: Fri, 6 Oct 2017 18:07:17 -0700
Subject: [PATCH 154/909] eager: Compute num_gpus() correctly.

Without this change, if TensorFlow is compiled with support for other devices
(such with XLA, which makes XLA_CPU and XLA_GPU devices available), then
tfe.num_gpus() was incorrectly overcounting the number of available GPUs.

PiperOrigin-RevId: 171373389
---
 tensorflow/python/eager/context.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/eager/context.py b/tensorflow/python/eager/context.py
index 02ff567e9e..be3d535271 100644
--- a/tensorflow/python/eager/context.py
+++ b/tensorflow/python/eager/context.py
@@ -95,11 +95,18 @@ class Context(object):
         device_list = pywrap_tensorflow.TFE_ContextListDevices(
             self._context_handle, status)
       try:
+        self._num_gpus = 0
         for i in range(pywrap_tensorflow.TF_DeviceListCount(device_list)):
           with errors.raise_exception_on_not_ok_status() as status:
             dev_name = pywrap_tensorflow.TF_DeviceListName(
                 device_list, i, status)
           self._context_devices.append(pydev.canonical_name(dev_name))
+          with errors.raise_exception_on_not_ok_status() as status:
+            dev_type = pywrap_tensorflow.TF_DeviceListType(
+                device_list, i, status)
+          if dev_type == "GPU":
+            self._num_gpus += 1
+
       finally:
         pywrap_tensorflow.TF_DeleteDeviceList(device_list)
 
@@ -238,8 +245,8 @@ class Context(object):
 
   def num_gpus(self):
     """The number of GPUs available to execute operations."""
-    # TODO(ashankar): Use TF_DeviceListType to count GPU devices.
-    return len(self._devices) - 1
+    self._initialize_handle_and_devices()
+    return self._num_gpus
 
   def add_function_def(self, fdef):
     """Add a function definition to the context.
-- 
GitLab


From 96d276fe4db70a79a9283f35442b5e37dbfd66c6 Mon Sep 17 00:00:00 2001
From: Max Galkin <maxgalkin@google.com>
Date: Fri, 6 Oct 2017 18:20:24 -0700
Subject: [PATCH 155/909] Improvements and fixes in VirtualPlacer:

- fixed a recent regression where VirtualPlacer stopped placing onto non-default devices like "device:TPU", added a test for this, verified that the test failed without the fix;
- fixed a number of problems with uppercase/lowercase mismatch in VirtualPlacer code, before that a slight difference between VirtualCluster device and node device ("/tpu:0" vs "/device:TPU:0") could cause fallback to default device, new code should be more resilient.

PiperOrigin-RevId: 171374421
---
 .../core/grappler/costs/virtual_placer.cc     | 134 +++++++++++-------
 .../core/grappler/costs/virtual_placer.h      |  21 ++-
 .../grappler/costs/virtual_placer_test.cc     |  28 ++++
 3 files changed, 122 insertions(+), 61 deletions(-)

diff --git a/tensorflow/core/grappler/costs/virtual_placer.cc b/tensorflow/core/grappler/costs/virtual_placer.cc
index 24c45235ff..965a2d2517 100644
--- a/tensorflow/core/grappler/costs/virtual_placer.cc
+++ b/tensorflow/core/grappler/costs/virtual_placer.cc
@@ -26,18 +26,27 @@ namespace grappler {
 VirtualPlacer::VirtualPlacer(const Cluster* cluster) {
   CHECK(cluster);
   devices_ = cluster->GetDevices();
+  lfqn_map_.reserve(devices_.size());
+  for (const auto& kv : devices_) {
+    const auto lfqn = to_lfqn_or_empty(kv.first);
+    if (lfqn.empty()) {
+      LOG(ERROR) << "VirtualPlacer couldn't parse device name from cluster: "
+                 << kv.first;
+    } else {
+      lfqn_map_[lfqn] = kv.first;
+    }
+  }
 
   if (devices_.empty()) {
     // If there are no devices in the cluster, add a single device, "UNKNOWN" to
     // the cluster.
-    default_device_ = "UNKNOWN";
+    default_device_name_ = "UNKNOWN";
     DeviceProperties& prop = devices_["UNKNOWN"];
     prop.set_type("UNKNOWN");
-
   } else if (devices_.size() == 1) {
     // If there is only one device in the cluster, use it as default device,
     // whatever it is.
-    default_device_ = devices_.begin()->first;
+    default_device_name_ = devices_.begin()->first;
   } else {
     // Default device is set from the devices in the cluster in the following
     // priority: /gpu:0, /cpu:0, or any device.
@@ -46,41 +55,48 @@ VirtualPlacer::VirtualPlacer(const Cluster* cluster) {
     // other than CPU and GPU.
     std::map<int, string> cpu_devices;  // CPU device map: id -> device name.
     std::map<int, string> gpu_devices;  // GPU device map: id -> device name.
-    for (const auto& device : devices_) {
+    for (const auto& kv : lfqn_map_) {
+      const auto& lfqn = kv.first;
+      const auto& cluster_device_name = kv.second;
       DeviceNameUtils::ParsedName parsed_name;
-      bool parsed = DeviceNameUtils::ParseFullName(device.first, &parsed_name);
+      bool parsed = DeviceNameUtils::ParseFullName(lfqn, &parsed_name);
       if (parsed) {
         // Parsed devices are stored to cpu_devices or gpu_devices map,
-        // addressed (and orderd) by device id.
-        if (str_util::Lowercase(parsed_name.type) == "gpu") {
-          gpu_devices[parsed_name.id] = device.first;
-        } else if (str_util::Lowercase(parsed_name.type) == "cpu") {
-          cpu_devices[parsed_name.id] = device.first;
+        // addressed (and ordered) by device id.
+        const auto type = str_util::Lowercase(parsed_name.type);
+        if (type == "gpu") {
+          gpu_devices[parsed_name.id] = cluster_device_name;
+        } else if (type == "cpu") {
+          cpu_devices[parsed_name.id] = cluster_device_name;
         }
       }
     }
+
     if (!gpu_devices.empty()) {
       // GPU:0 (or GPU with smallest device id).
-      default_device_ = gpu_devices.begin()->second;
+      default_device_name_ = gpu_devices.begin()->second;
     } else if (!cpu_devices.empty()) {
       // CPU:0 (or CPU with smallest device id).
-      default_device_ = cpu_devices.begin()->second;
+      default_device_name_ = cpu_devices.begin()->second;
     } else {
-      default_device_ = devices_.begin()->first;  // Any device.
+      default_device_name_ = devices_.begin()->first;  // Any device.
     }
   }
 
   // Default job name for canonical device name.
-  default_job_name_ = "localhost";
+  default_job_name_lowercase_ = "localhost";
   // Scan the device names from the cluster, and if there is one job name used,
   // use it for canonical device name.
   std::unordered_set<string> job_names_from_cluster;
-  for (const auto& device : devices_) {
-    const auto& device_name = device.first;
+  for (const auto& device : lfqn_map_) {
+    const auto& lfqn = device.first;
     DeviceNameUtils::ParsedName parsed_name;
-    bool parsed = DeviceNameUtils::ParseFullName(device_name, &parsed_name);
+    bool parsed = DeviceNameUtils::ParseFullName(lfqn, &parsed_name);
     if (parsed && !parsed_name.job.empty()) {
       job_names_from_cluster.insert(parsed_name.job);
+      if (job_names_from_cluster.size() > 1) {
+        break;
+      }
     }
   }
   // If there is only  type of job name in all the devices in the cluster, use
@@ -89,60 +105,68 @@ VirtualPlacer::VirtualPlacer(const Cluster* cluster) {
   // composed of multiple worker, PS, and other types of jobs.
   if (job_names_from_cluster.size() == 1) {
     auto it = job_names_from_cluster.begin();
-    default_job_name_ = *it;
+    default_job_name_lowercase_ = *it;
   }
 }
 
 const DeviceProperties& VirtualPlacer::get_device(const NodeDef& node) const {
   string device = get_canonical_device_name(node);
-  VLOG(3) << "Device name: " << device;
+  VLOG(3) << "node.name=" << node.name() << " node.device=" << node.device()
+          << " is placed on: " << device;
   auto it = devices_.find(device);
   DCHECK(it != devices_.end());
   return it->second;
 }
 
 string VirtualPlacer::get_canonical_device_name(const NodeDef& node) const {
-  string device;
-  if (!node.device().empty()) {
-    if (devices_.find(node.device()) != devices_.end()) {
-      return node.device();
-    }
-    DeviceNameUtils::ParsedName parsed_name;
-    bool parsed = DeviceNameUtils::ParseFullName(node.device(), &parsed_name);
-    if (!parsed) {
-      parsed = DeviceNameUtils::ParseLocalName(node.device(), &parsed_name);
+  if (node.device().empty()) {
+    return default_device_name_;
+  }
+
+  const auto lfqn = to_lfqn_or_empty(node.device());
+  if (lfqn.empty()) {
+    return default_device_name_;
+  }
+
+  const auto it = lfqn_map_.find(lfqn);
+  if (it != lfqn_map_.end()) {
+    return it->second;
+  }
+
+  return default_device_name_;
+}
+
+string VirtualPlacer::to_lfqn_or_empty(const string& device_name) const {
+  DeviceNameUtils::ParsedName parsed_name;
+  const auto lowercase_name = str_util::Lowercase(device_name);
+  bool parsed = DeviceNameUtils::ParseFullName(lowercase_name, &parsed_name);
+  if (!parsed) {
+    parsed = DeviceNameUtils::ParseLocalName(lowercase_name, &parsed_name);
+    parsed_name.job = "localhost";
+  }
+  if (!parsed) {
+    if (lowercase_name == "gpu" || lowercase_name == "cpu") {
       parsed_name.job = "localhost";
+      parsed_name.type = lowercase_name;
+      parsed = true;
     }
-    if (!parsed) {
-      if (node.device() == "GPU" || node.device() == "CPU" ||
-          node.device() == "gpu" || node.device() == "cpu") {
-        parsed_name.job = "localhost";
-        parsed_name.type = node.device();
-        parsed = true;
-      }
-    }
-    if (!parsed) {
-      return get_default_device_name();
-    } else {
-      if (parsed_name.job.empty()) {
-        parsed_name.job = default_job_name_;
-      }
-      device = strings::StrCat(
-          "/job:", parsed_name.job, "/replica:", parsed_name.replica,
-          "/task:", parsed_name.task, "/",
-          str_util::Lowercase(parsed_name.type), ":", parsed_name.id);
-    }
-  } else {
-    return get_default_device_name();
   }
-  if (devices_.find(device) == devices_.end()) {
-    return get_default_device_name();
+  if (!parsed) {
+    return {};
   }
-  return device;
-}
 
-const string& VirtualPlacer::get_default_device_name() const {
-  return default_device_;
+  if (parsed_name.job.empty()) {
+    parsed_name.job = default_job_name_lowercase_;
+  }
+
+  // Have to do this, because parser returns uppercase types for CPU and GPU.
+  parsed_name.type = str_util::Lowercase(parsed_name.type);
+
+  string lfqn = strings::StrCat(
+      "/job:", parsed_name.job, "/replica:", parsed_name.replica,
+      "/task:", parsed_name.task, "/device:", parsed_name.type, ":",
+      parsed_name.id);
+  return lfqn;
 }
 
 }  // end namespace grappler
diff --git a/tensorflow/core/grappler/costs/virtual_placer.h b/tensorflow/core/grappler/costs/virtual_placer.h
index 75ee496329..7ccb1ebb99 100644
--- a/tensorflow/core/grappler/costs/virtual_placer.h
+++ b/tensorflow/core/grappler/costs/virtual_placer.h
@@ -33,16 +33,25 @@ class VirtualPlacer {
 
   const DeviceProperties& get_device(const NodeDef& node) const;
 
-  // Returns canonical device name that has a corresponding device in the
-  // cluster; returns empty string if no device found or the node.device() can
-  // not be parsed.
+  // Returns device name from cluster, which best matches the node.device()
+  // specification. Returns default device if no match was found or the
+  // node.device() could not be parsed.
   string get_canonical_device_name(const NodeDef& node) const;
 
  private:
+  // Converts given device name to Lowercase Fully-Qualified Name (LFQN) string.
+  // This helps us disambiguate device names internally and simplify matching.
+  // If device_name couldn't be parsed succesfully, returns empty string.
+  string to_lfqn_or_empty(const string& device_name) const;
+
+  // Map based on the cluster info: cluster device name -> device properties.
   std::unordered_map<string, DeviceProperties> devices_;
-  string default_device_;
-  string default_job_name_;
-  const string& get_default_device_name() const;
+
+  // Maps LFQN to original device name as it was declared in cluster.
+  std::unordered_map<string, string> lfqn_map_;
+
+  string default_device_name_;
+  string default_job_name_lowercase_;
 };
 
 }  // namespace grappler
diff --git a/tensorflow/core/grappler/costs/virtual_placer_test.cc b/tensorflow/core/grappler/costs/virtual_placer_test.cc
index 3a0510c44a..1c2e2815a6 100644
--- a/tensorflow/core/grappler/costs/virtual_placer_test.cc
+++ b/tensorflow/core/grappler/costs/virtual_placer_test.cc
@@ -53,6 +53,34 @@ TEST(VirtualPlacerTest, LocalDevices) {
             placer.get_canonical_device_name(node));
 }
 
+TEST(VirtualPlacerTest, PlacementOnNonDefaultDevice) {
+  // Create a virtual cluster with a CPU and a device:TPU
+  // Test that placement on TPU works
+  // In contrast with GPU, TPU is not selected as default device at the moment.
+
+  std::unordered_map<string, DeviceProperties> devices;
+  DeviceProperties cpu_device;
+  cpu_device.set_type("CPU");
+  devices["/job:localhost/replica:0/task:0/cpu:0"] = cpu_device;
+  DeviceProperties tpu_device;
+  tpu_device.set_type("TPU");
+  devices["/job:localhost/replica:0/task:0/device:TPU:0"] = tpu_device;
+  VirtualCluster cluster(devices);
+  VirtualPlacer placer(&cluster);
+
+  NodeDef node;
+  node.set_op("Conv2D");
+  // node.device() is empty, and CPU is default device.
+  EXPECT_EQ("CPU", placer.get_device(node).type());
+  EXPECT_EQ("/job:localhost/replica:0/task:0/cpu:0",
+            placer.get_canonical_device_name(node));
+
+  node.set_device("/device:TPU:0");
+  EXPECT_EQ("TPU", placer.get_device(node).type());
+  EXPECT_EQ("/job:localhost/replica:0/task:0/device:TPU:0",
+            placer.get_canonical_device_name(node));
+}
+
 TEST(VirtualPlacerTest, EmptyJobName) {
   // Virtual placer choose job name from the devices in cluster if a device name
   // of an op is empty. In case there are more than one kind of job name
-- 
GitLab


From 010dd39b949a57f80122ea7fdca8a0937f6fbb65 Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Fri, 6 Oct 2017 18:24:03 -0700
Subject: [PATCH 156/909] Disable predict_test under tsan.

PiperOrigin-RevId: 171374722
---
 tensorflow/contrib/timeseries/examples/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/contrib/timeseries/examples/BUILD b/tensorflow/contrib/timeseries/examples/BUILD
index 015d0eba29..8ed812f9d1 100644
--- a/tensorflow/contrib/timeseries/examples/BUILD
+++ b/tensorflow/contrib/timeseries/examples/BUILD
@@ -25,6 +25,7 @@ py_test(
     srcs = ["predict_test.py"],
     data = ["data/period_trend.csv"],
     srcs_version = "PY2AND3",
+    tags = ["notsan"],  # b/67513579
     deps = [
         ":predict",
         "//tensorflow/python:client_testlib",
-- 
GitLab


From 5a107a9a278e98f2fcb77c8ac6c224d40c06e8c2 Mon Sep 17 00:00:00 2001
From: Neal Wu <wun@google.com>
Date: Fri, 6 Oct 2017 18:33:41 -0700
Subject: [PATCH 157/909] Fix broken docs links to other TensorFlow interfaces
 in tf.contrib.learn.Experiment

PiperOrigin-RevId: 171375351
---
 tensorflow/contrib/learn/python/learn/experiment.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/learn/python/learn/experiment.py b/tensorflow/contrib/learn/python/learn/experiment.py
index 9b55826e62..307db76afe 100644
--- a/tensorflow/contrib/learn/python/learn/experiment.py
+++ b/tensorflow/contrib/learn/python/learn/experiment.py
@@ -149,16 +149,16 @@ class Experiment(object):
 
     Args:
       estimator: Object implementing Estimator interface, which could be a
-        combination of ${tf.contrib.learn.Trainable} and
-        ${tf.contrib.learn.Evaluable} (deprecated), or
-        ${tf.estimator.`Estimator}.
+        combination of @{tf.contrib.learn.Trainable} and
+        @{tf.contrib.learn.Evaluable} (deprecated), or
+        @{tf.estimator.Estimator}.
       train_input_fn: function, returns features and labels for training.
       eval_input_fn: function, returns features and labels for evaluation. If
         `eval_steps` is `None`, this should be configured only to produce for a
         finite number of batches (generally, 1 epoch over the evaluation data).
       eval_metrics: `dict` of string, metric function. If `None`, default set
         is used. This should be `None` if the `estimator` is
-        ${tf.estimator.Estimator}. If metrics are provided they will be
+        @{tf.estimator.Estimator}. If metrics are provided they will be
         *appended* to the default set.
       train_steps: Perform this many steps of training. `None`, the default,
         means train forever.
-- 
GitLab


From 394e5601c13da603237063d436d87867727ecf68 Mon Sep 17 00:00:00 2001
From: Mark Daoust <markdaoust@google.com>
Date: Fri, 6 Oct 2017 18:34:17 -0700
Subject: [PATCH 158/909] Add a custom estimator example to the regression
 cookbook.

PiperOrigin-RevId: 171375399
---
 .../docs_src/get_started/linear_regression.md |  27 +++
 .../examples/get_started/regression/BUILD     |   1 +
 .../regression/custom_regression.py           | 163 ++++++++++++++++++
 .../get_started/regression/imports85.py       |   6 +-
 .../examples/get_started/regression/test.py   |   7 +
 5 files changed, 201 insertions(+), 3 deletions(-)
 create mode 100644 tensorflow/examples/get_started/regression/custom_regression.py

diff --git a/tensorflow/docs_src/get_started/linear_regression.md b/tensorflow/docs_src/get_started/linear_regression.md
index b12bbd770f..7cfff8db15 100644
--- a/tensorflow/docs_src/get_started/linear_regression.md
+++ b/tensorflow/docs_src/get_started/linear_regression.md
@@ -27,6 +27,13 @@ to implement regression in Estimators:
         regression model on discrete data with a deep neural network.</td>
   </tr>
 
+  <tr>
+    <td><a href="https://www.tensorflow.org/code/tensorflow/examples/get_started/regression/custom_regression.py">custom_regression.py</a></td>
+    <td>[imports85](https://archive.ics.uci.edu/ml/datasets/automobile)</td>
+    <td>Use @{tf.estimator.Estimator} to train a customized dnn
+        regression model.</td>
+  </tr>
+
 </table>
 
 The preceding examples rely on the following data set utility:
@@ -207,3 +214,23 @@ in a deep neural network.
 
 After printing loss values, the program outputs the Mean Square Error
 on a test set.
+
+
+<a name="dnn"></a>
+## custom_regression.py
+
+The `custom_regression.py` example also trains a model that predicts the price
+of a car based on mixed real-valued and categorical input features, described by
+feature_columns. Unlike `linear_regression_categorical.py`, and
+`dnn_regression.py` this example does not use a pre-made estimator, but defines
+a custom model using the base @{tf.estimator.Estimator$`Estimator`} class. The
+custom model is quite similar to the model defined by `dnn_regression.py`.
+
+The custom model is defined by the `model_fn` argument to the constructor. The
+customization is made more reusable through `params` dictionary, which is later
+passed through to the `model_fn` when the `model_fn` is called.
+
+The `model_fn` returns an
+@{tf.estimator.EstimatorSpec$`EstimatorSpec`} which is a simple structure
+indicating to the `Estimator` which operations should be run to accomplish
+varions tasks.
diff --git a/tensorflow/examples/get_started/regression/BUILD b/tensorflow/examples/get_started/regression/BUILD
index 334c8096c1..577b970c90 100644
--- a/tensorflow/examples/get_started/regression/BUILD
+++ b/tensorflow/examples/get_started/regression/BUILD
@@ -18,6 +18,7 @@ py_test(
     name = "test",
     size = "medium",
     srcs = [
+        "custom_regression.py",
         "dnn_regression.py",
         "imports85.py",
         "linear_regression.py",
diff --git a/tensorflow/examples/get_started/regression/custom_regression.py b/tensorflow/examples/get_started/regression/custom_regression.py
new file mode 100644
index 0000000000..2e34362c5c
--- /dev/null
+++ b/tensorflow/examples/get_started/regression/custom_regression.py
@@ -0,0 +1,163 @@
+# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Regression using the DNNRegressor Estimator."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import tensorflow as tf
+
+import imports85  # pylint: disable=g-bad-import-order
+
+STEPS = 1000
+PRICE_NORM_FACTOR = 1000
+
+
+def my_dnn_regression_fn(features, labels, mode, params):
+  """A model function implementing DNN regression for a custom Estimator."""
+
+  # Extract the input into a dense layer, according to the feature_columns.
+  top = tf.feature_column.input_layer(features, params["feature_columns"])
+
+  # Iterate over the "hidden_units" list of layer sizes, default is [20].
+  for units in params.get("hidden_units", [20]):
+    # Add a hidden layer, densely connected on top of the previous layer.
+    top = tf.layers.dense(inputs=top, units=units, activation=tf.nn.relu)
+
+  # Connect a linear output layer on top.
+  output_layer = tf.layers.dense(inputs=top, units=1)
+
+  # Reshape the output layer to a 1-dim Tensor to return predictions
+  predictions = tf.squeeze(output_layer, 1)
+
+  if mode == tf.estimator.ModeKeys.PREDICT:
+    # In `PREDICT` mode we only need to return predictions.
+    return tf.estimator.EstimatorSpec(
+        mode=mode, predictions={"price": predictions})
+
+  # Calculate loss using mean squared error
+  average_loss = tf.losses.mean_squared_error(labels, predictions)
+
+  # Pre-made estimators use the total_loss instead of the average,
+  # so report total_loss for compatibility.
+  batch_size = tf.shape(labels)[0]
+  total_loss = tf.to_float(batch_size) * average_loss
+
+  if mode == tf.estimator.ModeKeys.TRAIN:
+    optimizer = params.get("optimizer", tf.train.AdamOptimizer)
+    optimizer = optimizer(params.get("learning_rate", None))
+    train_op = optimizer.minimize(
+        loss=average_loss, global_step=tf.train.get_global_step())
+
+    return tf.estimator.EstimatorSpec(
+        mode=mode, loss=total_loss, train_op=train_op)
+
+  # In evaluation mode we will calculate evaluation metrics.
+  assert mode == tf.estimator.ModeKeys.EVAL
+
+  # Calculate root mean squared error
+  rmse = tf.metrics.root_mean_squared_error(labels, predictions)
+
+  # Add the rmse to the collection of evaluation metrics.
+  eval_metrics = {"rmse": rmse}
+
+  return tf.estimator.EstimatorSpec(
+      mode=mode,
+      # Report sum of error for compatibility with pre-made estimators
+      loss=total_loss,
+      eval_metric_ops=eval_metrics)
+
+
+def main(argv):
+  """Builds, trains, and evaluates the model."""
+  assert len(argv) == 1
+  (train, test) = imports85.dataset()
+
+  # Switch the labels to units of thousands for better convergence.
+  def normalize_price(features, labels):
+    return features, labels / PRICE_NORM_FACTOR
+
+  train = train.map(normalize_price)
+  test = test.map(normalize_price)
+
+  # Build the training input_fn.
+  def input_train():
+    return (
+        # Shuffling with a buffer larger than the data set ensures
+        # that the examples are well mixed.
+        train.shuffle(1000).batch(128)
+        # Repeat forever
+        .repeat().make_one_shot_iterator().get_next())
+
+  # Build the validation input_fn.
+  def input_test():
+    return (test.shuffle(1000).batch(128)
+            .make_one_shot_iterator().get_next())
+
+  # The first way assigns a unique weight to each category. To do this you must
+  # specify the category's vocabulary (values outside this specification will
+  # receive a weight of zero). Here we specify the vocabulary using a list of
+  # options. The vocabulary can also be specified with a vocabulary file (using
+  # `categorical_column_with_vocabulary_file`). For features covering a
+  # range of positive integers use `categorical_column_with_identity`.
+  body_style_vocab = ["hardtop", "wagon", "sedan", "hatchback", "convertible"]
+  body_style = tf.feature_column.categorical_column_with_vocabulary_list(
+      key="body-style", vocabulary_list=body_style_vocab)
+  make = tf.feature_column.categorical_column_with_hash_bucket(
+      key="make", hash_bucket_size=50)
+
+  feature_columns = [
+      tf.feature_column.numeric_column(key="curb-weight"),
+      tf.feature_column.numeric_column(key="highway-mpg"),
+      # Since this is a DNN model, convert categorical columns from sparse
+      # to dense.
+      # Wrap them in an `indicator_column` to create a
+      # one-hot vector from the input.
+      tf.feature_column.indicator_column(body_style),
+      # Or use an `embedding_column` to create a trainable vector for each
+      # index.
+      tf.feature_column.embedding_column(make, dimension=3),
+  ]
+
+  # Build a custom Estimator, using the model_fn.
+  # `params` is passed through to the `model_fn`.
+  model = tf.estimator.Estimator(
+      model_fn=my_dnn_regression_fn,
+      params={
+          "feature_columns": feature_columns,
+          "learning_rate": 0.001,
+          "optimizer": tf.train.AdamOptimizer,
+          "hidden_units": [20, 20]
+      })
+
+  # Train the model.
+  model.train(input_fn=input_train, steps=STEPS)
+
+  # Evaluate how the model performs on data it has not yet seen.
+  eval_result = model.evaluate(input_fn=input_test)
+
+  # Print the Root Mean Square Error (RMSE).
+  print("\n" + 80 * "*")
+  print("\nRMS error for the test set: ${:.0f}"
+        .format(PRICE_NORM_FACTOR * eval_result["rmse"]))
+
+  print()
+
+
+if __name__ == "__main__":
+  # The Estimator periodically generates "INFO" logs; make these logs visible.
+  tf.logging.set_verbosity(tf.logging.INFO)
+  tf.app.run(main=main)
diff --git a/tensorflow/examples/get_started/regression/imports85.py b/tensorflow/examples/get_started/regression/imports85.py
index c165f0175d..96a464920a 100644
--- a/tensorflow/examples/get_started/regression/imports85.py
+++ b/tensorflow/examples/get_started/regression/imports85.py
@@ -140,10 +140,10 @@ def dataset(y_name="price", train_fraction=0.7):
   train = (base_dataset
            # Take only the training-set lines.
            .filter(in_training_set)
-           # Cache data so you only read the file once.
-           .cache()
            # Decode each line into a (features_dict, label) pair.
-           .map(decode_line))
+           .map(decode_line)
+           # Cache data so you only decode the file once.
+           .cache())
 
   # Do the same for the test-set.
   test = (base_dataset.filter(in_test_set).cache().map(decode_line))
diff --git a/tensorflow/examples/get_started/regression/test.py b/tensorflow/examples/get_started/regression/test.py
index fa06dde9ae..652b44f543 100644
--- a/tensorflow/examples/get_started/regression/test.py
+++ b/tensorflow/examples/get_started/regression/test.py
@@ -34,6 +34,7 @@ import tensorflow.contrib.data as data
 import tensorflow.examples.get_started.regression.dnn_regression as dnn_regression
 import tensorflow.examples.get_started.regression.linear_regression as linear_regression
 import tensorflow.examples.get_started.regression.linear_regression_categorical as linear_regression_categorical
+import tensorflow.examples.get_started.regression.custom_regression as custom_regression
 
 from tensorflow.python.platform import googletest
 from tensorflow.python.platform import test
@@ -86,6 +87,12 @@ class RegressionTest(googletest.TestCase):
   def test_dnn_regression(self):
     dnn_regression.main([""])
 
+  @test.mock.patch.dict(data.__dict__, {"TextLineDataset": four_lines_dataset})
+  @test.mock.patch.dict(imports85.__dict__, {"_get_imports85": (lambda: None)})
+  @test.mock.patch.dict(custom_regression.__dict__, {"STEPS": 1})
+  def test_custom_regression(self):
+    custom_regression.main([""])
+
 
 if __name__ == "__main__":
   googletest.main()
-- 
GitLab


From f8f1ccefb6afc9de0b07e8c1392ecf2abe3391e4 Mon Sep 17 00:00:00 2001
From: Reed Wanderman-Milne <reedwm@google.com>
Date: Fri, 6 Oct 2017 19:32:10 -0700
Subject: [PATCH 159/909] Log in executor when a synchronous node is finished.

Also log more info when an asynchronous node is finished.

This is useful for debugging deadlocks and issues where a kernel does not return.

PiperOrigin-RevId: 171379066
---
 tensorflow/core/common_runtime/executor.cc | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/tensorflow/core/common_runtime/executor.cc b/tensorflow/core/common_runtime/executor.cc
index 11e063d8d2..ada29ff287 100644
--- a/tensorflow/core/common_runtime/executor.cc
+++ b/tensorflow/core/common_runtime/executor.cc
@@ -1617,14 +1617,17 @@ void ExecutorState::Process(TaggedNode tagged_node, int64 scheduled_usec) {
           NodeExecStatsWrapper* stats = state->stats;  // Shorthand
           Entry* first_input = state->first_input;  // Shorthand
 
-          if (vlog_) {
-            VLOG(2) << this << " Async kernel done: "
-                    << SummarizeNode(*state->item->node);
-          }
           nodestats::SetOpEnd(stats);
           EntryVector outputs;
           Status s = ProcessOutputs(*state->item, &state->ctx, &outputs, stats);
           nodestats::SetMemory(stats, &state->ctx);
+          if (vlog_) {
+            VLOG(2) << "Async kernel done: " << state->item->node->id()
+                    << " step " << step_id_ << " "
+                    << SummarizeNode(*state->item->node)
+                    << " is dead: " << state->tagged_node.is_dead;
+          }
+
           // Clears inputs.
           const int num_inputs = state->item->num_inputs;
           for (int i = 0; i < num_inputs; ++i) {
@@ -1672,6 +1675,12 @@ void ExecutorState::Process(TaggedNode tagged_node, int64 scheduled_usec) {
     }
 
     if (!launched_asynchronously) {
+      if (vlog_) {
+        VLOG(2) << "Synchronous kernel done: " << id << " step "
+                << params.step_id << " " << SummarizeNode(*node)
+                << " is dead: " << tagged_node.is_dead;
+      }
+
       // Clears inputs.
       const int num_inputs = item.num_inputs;
       for (int i = 0; i < num_inputs; ++i) {
-- 
GitLab


From 843394627a43fd48b2cf77cb434948122e75858b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 6 Oct 2017 21:01:02 -0700
Subject: [PATCH 160/909] Make name scopes consistent.

PiperOrigin-RevId: 171382508
---
 .../gan/python/losses/python/losses_impl.py   | 80 +++++++++++--------
 .../python/losses/python/losses_impl_test.py  |  6 +-
 2 files changed, 49 insertions(+), 37 deletions(-)

diff --git a/tensorflow/contrib/gan/python/losses/python/losses_impl.py b/tensorflow/contrib/gan/python/losses/python/losses_impl.py
index 2a40dbade6..b4a74fc49c 100644
--- a/tensorflow/contrib/gan/python/losses/python/losses_impl.py
+++ b/tensorflow/contrib/gan/python/losses/python/losses_impl.py
@@ -217,21 +217,25 @@ def acgan_discriminator_loss(
   Raises:
     TypeError: If the discriminator does not output a tuple.
   """
-  loss_on_generated = losses.softmax_cross_entropy(
-      one_hot_labels, discriminator_gen_classification_logits,
-      weights=generated_weights, scope=scope, loss_collection=None,
-      reduction=reduction)
-  loss_on_real = losses.softmax_cross_entropy(
-      one_hot_labels, discriminator_real_classification_logits,
-      weights=real_weights, label_smoothing=label_smoothing, scope=scope,
-      loss_collection=None, reduction=reduction)
-  loss = loss_on_generated + loss_on_real
-  util.add_loss(loss, loss_collection)
+  with ops.name_scope(
+      scope, 'acgan_discriminator_loss',
+      (discriminator_real_classification_logits,
+       discriminator_gen_classification_logits, one_hot_labels)) as scope:
+    loss_on_generated = losses.softmax_cross_entropy(
+        one_hot_labels, discriminator_gen_classification_logits,
+        weights=generated_weights, scope=scope, loss_collection=None,
+        reduction=reduction)
+    loss_on_real = losses.softmax_cross_entropy(
+        one_hot_labels, discriminator_real_classification_logits,
+        weights=real_weights, label_smoothing=label_smoothing, scope=scope,
+        loss_collection=None, reduction=reduction)
+    loss = loss_on_generated + loss_on_real
+    util.add_loss(loss, loss_collection)
 
-  if add_summaries:
-    summary.scalar('discriminator_gen_ac_loss', loss_on_generated)
-    summary.scalar('discriminator_real_ac_loss', loss_on_real)
-    summary.scalar('discriminator_ac_loss', loss)
+    if add_summaries:
+      summary.scalar('discriminator_gen_ac_loss', loss_on_generated)
+      summary.scalar('discriminator_real_ac_loss', loss_on_real)
+      summary.scalar('discriminator_ac_loss', loss)
 
   return loss
 
@@ -275,12 +279,16 @@ def acgan_generator_loss(
     ValueError: if arg module not either `generator` or `discriminator`
     TypeError: if the discriminator does not output a tuple.
   """
-  loss = losses.softmax_cross_entropy(
-      one_hot_labels, discriminator_gen_classification_logits, weights=weights,
-      scope=scope, loss_collection=loss_collection, reduction=reduction)
+  with ops.name_scope(
+      scope, 'acgan_generator_loss',
+      (discriminator_gen_classification_logits, one_hot_labels)) as scope:
+    loss = losses.softmax_cross_entropy(
+        one_hot_labels, discriminator_gen_classification_logits,
+        weights=weights, scope=scope, loss_collection=loss_collection,
+        reduction=reduction)
 
-  if add_summaries:
-    summary.scalar('generator_ac_loss', loss)
+    if add_summaries:
+      summary.scalar('generator_ac_loss', loss)
 
   return loss
 
@@ -546,7 +554,7 @@ def modified_generator_loss(
     discriminator_gen_outputs,
     label_smoothing=0.0,
     weights=1.0,
-    scope='generator_modified_loss',
+    scope=None,
     loss_collection=ops.GraphKeys.LOSSES,
     reduction=losses.Reduction.SUM_BY_NONZERO_WEIGHTS,
     add_summaries=False):
@@ -576,12 +584,15 @@ def modified_generator_loss(
   Returns:
     A loss Tensor. The shape depends on `reduction`.
   """
-  loss = losses.sigmoid_cross_entropy(
-      array_ops.ones_like(discriminator_gen_outputs), discriminator_gen_outputs,
-      weights, label_smoothing, scope, loss_collection, reduction)
+  with ops.name_scope(scope, 'generator_modified_loss',
+                      [discriminator_gen_outputs]) as scope:
+    loss = losses.sigmoid_cross_entropy(
+        array_ops.ones_like(discriminator_gen_outputs),
+        discriminator_gen_outputs, weights, label_smoothing, scope,
+        loss_collection, reduction)
 
-  if add_summaries:
-    summary.scalar('generator_modified_loss', loss)
+    if add_summaries:
+      summary.scalar('generator_modified_loss', loss)
 
   return loss
 
@@ -739,7 +750,7 @@ def mutual_information_penalty(
     structured_generator_inputs,
     predicted_distributions,
     weights=1.0,
-    scope='generator_modified_loss',
+    scope=None,
     loss_collection=ops.GraphKeys.LOSSES,
     reduction=losses.Reduction.SUM_BY_NONZERO_WEIGHTS,
     add_summaries=False):
@@ -767,15 +778,16 @@ def mutual_information_penalty(
   _validate_information_penalty_inputs(
       structured_generator_inputs, predicted_distributions)
 
-  # Calculate the negative log-likelihood of the reconstructed noise.
-  log_probs = [math_ops.reduce_mean(dist.log_prob(noise)) for dist, noise in
-               zip(predicted_distributions, structured_generator_inputs)]
-  loss = -1 * losses.compute_weighted_loss(
-      log_probs, weights, scope, loss_collection=loss_collection,
-      reduction=reduction)
+  with ops.name_scope(scope, 'mutual_information_loss') as scope:
+    # Calculate the negative log-likelihood of the reconstructed noise.
+    log_probs = [math_ops.reduce_mean(dist.log_prob(noise)) for dist, noise in
+                 zip(predicted_distributions, structured_generator_inputs)]
+    loss = -1 * losses.compute_weighted_loss(
+        log_probs, weights, scope, loss_collection=loss_collection,
+        reduction=reduction)
 
-  if add_summaries:
-    summary.scalar('mutual_information_penalty', loss)
+    if add_summaries:
+      summary.scalar('mutual_information_penalty', loss)
 
   return loss
 
diff --git a/tensorflow/contrib/gan/python/losses/python/losses_impl_test.py b/tensorflow/contrib/gan/python/losses/python/losses_impl_test.py
index 3e003dd0f8..c15ce5baae 100644
--- a/tensorflow/contrib/gan/python/losses/python/losses_impl_test.py
+++ b/tensorflow/contrib/gan/python/losses/python/losses_impl_test.py
@@ -274,8 +274,8 @@ class ACGANLossTest(test.TestCase):
         self._discriminator_real_classification_logits,
         'one_hot_labels': self._one_hot_labels,
     }
-    self._generator_loss_name = 'softmax_cross_entropy_loss/value'
-    self._discriminator_loss_name = 'add'
+    self._generator_loss_name = 'acgan_generator_loss/value'
+    self._discriminator_loss_name = 'acgan_discriminator_loss/add'
     self._expected_g_loss = 3.84974
     self._expected_d_loss = 9.43950
 
@@ -504,7 +504,7 @@ class MutualInformationPenaltyTest(test.TestCase, _PenaltyTest):
         'predicted_distributions': self._predicted_distributions,
     }
     self._expected_loss = 1.61610
-    self._expected_op_name = 'mul'
+    self._expected_op_name = 'mutual_information_loss/mul'
     self._batch_size = 2
 
 
-- 
GitLab


From d43911058b63c7e91fac01b8b18bffa4cd936868 Mon Sep 17 00:00:00 2001
From: Taehoon Lee <me@taehoonlee.com>
Date: Sat, 7 Oct 2017 15:04:58 +0900
Subject: [PATCH 161/909] Fix typos

---
 tensorflow/contrib/meta_graph_transform/meta_graph_transform.py | 2 +-
 tensorflow/core/framework/rendezvous.cc                         | 2 +-
 tensorflow/core/profiler/g3doc/options.md                       | 2 +-
 tensorflow/examples/get_started/regression/imports85.py         | 2 +-
 tensorflow/python/debug/cli/tensor_format.py                    | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/tensorflow/contrib/meta_graph_transform/meta_graph_transform.py b/tensorflow/contrib/meta_graph_transform/meta_graph_transform.py
index 303c02dfa4..2932ae1c8d 100644
--- a/tensorflow/contrib/meta_graph_transform/meta_graph_transform.py
+++ b/tensorflow/contrib/meta_graph_transform/meta_graph_transform.py
@@ -749,7 +749,7 @@ def meta_graph_transform(
         base_meta_graph_def, meta_graph_def, collection_name,
         removed_op_names)
 
-  # Append newly added initalizers to collection.
+  # Append newly added initializers to collection.
   _add_new_inits_to_collection(meta_graph_def, updated_initializer_names)
 
   # Copy signature_defs, excluding any pruned nodes
diff --git a/tensorflow/core/framework/rendezvous.cc b/tensorflow/core/framework/rendezvous.cc
index 90426defa0..a9e4c1cfb1 100644
--- a/tensorflow/core/framework/rendezvous.cc
+++ b/tensorflow/core/framework/rendezvous.cc
@@ -210,7 +210,7 @@ class LocalRendezvousImpl : public Rendezvous {
     ItemQueue* queue = &table_[key_hash];
     if (queue->empty() || !queue->front()->IsSendValue()) {
       // There is no message to pick up.
-      // Only recv-related fileds need to be filled.
+      // Only recv-related fields need to be filled.
       Item* item = new Item;
       item->waiter = std::move(done);
       item->recv_args = recv_args;
diff --git a/tensorflow/core/profiler/g3doc/options.md b/tensorflow/core/profiler/g3doc/options.md
index ddee63ad42..4c73e372e3 100644
--- a/tensorflow/core/profiler/g3doc/options.md
+++ b/tensorflow/core/profiler/g3doc/options.md
@@ -43,7 +43,7 @@ In graph view, in means the number of hops in the <b>graph</b>.
 
 ### Times
 
-Most machines have mutli-core CPUs. Some installs one or more accelerators.
+Most machines have multi-core CPUs. Some installs one or more accelerators.
 Each accelerator usually performs massive parallel processing. The profiler
 tracks the accumulated processing times. Hence, the accumulated processing
 time is likely larger than the time of each step.
diff --git a/tensorflow/examples/get_started/regression/imports85.py b/tensorflow/examples/get_started/regression/imports85.py
index c165f0175d..56d19f0d0a 100644
--- a/tensorflow/examples/get_started/regression/imports85.py
+++ b/tensorflow/examples/get_started/regression/imports85.py
@@ -127,7 +127,7 @@ def dataset(y_name="price", train_fraction=0.7):
   def in_test_set(line):
     """Returns a boolean tensor, true if the line is in the training set."""
     # Items not in the training set are in the test set.
-    # This line must use `~` instead of `not` beacuse `not` only works on python
+    # This line must use `~` instead of `not` because `not` only works on python
     # booleans but we are dealing with symbolic tensors.
     return ~in_training_set(line)
 
diff --git a/tensorflow/python/debug/cli/tensor_format.py b/tensorflow/python/debug/cli/tensor_format.py
index 7a5597db12..05ccf93f15 100644
--- a/tensorflow/python/debug/cli/tensor_format.py
+++ b/tensorflow/python/debug/cli/tensor_format.py
@@ -480,7 +480,7 @@ def _pad_string_to_length(string, length):
 
 
 def numeric_summary(tensor):
-  """Get a text summmary of a numeric tensor.
+  """Get a text summary of a numeric tensor.
 
   This summary is only available for numeric (int*, float*, complex*) and
   Boolean tensors.
-- 
GitLab


From f59ef8a3e5c79ed97813b136d900ade31c0c11a7 Mon Sep 17 00:00:00 2001
From: Hanmin Qin <qinhanmin2005@sina.com>
Date: Sat, 7 Oct 2017 15:04:59 +0800
Subject: [PATCH 162/909] small typo

---
 .../get_started/regression/linear_regression_categorical.py     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/examples/get_started/regression/linear_regression_categorical.py b/tensorflow/examples/get_started/regression/linear_regression_categorical.py
index 860d0e437c..e2ad415fbc 100644
--- a/tensorflow/examples/get_started/regression/linear_regression_categorical.py
+++ b/tensorflow/examples/get_started/regression/linear_regression_categorical.py
@@ -67,7 +67,7 @@ def main(argv):
 
   # The second way, appropriate for an unspecified vocabulary, is to create a
   # hashed column. It will create a fixed length list of weights, and
-  # automatically assign each input categort to a weight. Due to the
+  # automatically assign each input category to a weight. Due to the
   # pseudo-randomness of the process, some weights may be shared between
   # categories, while others will remain unused.
   make_column = tf.feature_column.categorical_column_with_hash_bucket(
-- 
GitLab


From 188297f80e0341f2480071c85a671c6c0abdbf8e Mon Sep 17 00:00:00 2001
From: "Yuan (Terry) Tang" <terrytangyuan@users.noreply.github.com>
Date: Sat, 7 Oct 2017 11:08:19 -0400
Subject: [PATCH 163/909] Added missing `` in train_and_evaluate doc

---
 tensorflow/python/estimator/training.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/estimator/training.py b/tensorflow/python/estimator/training.py
index 64b014a6b5..45bff233ea 100644
--- a/tensorflow/python/estimator/training.py
+++ b/tensorflow/python/estimator/training.py
@@ -408,8 +408,8 @@ def train_and_evaluate(estimator, train_spec, eval_spec):
 
   Args:
     estimator: An `Estimator` instance to train and evaluate.
-    train_spec: A `TrainSpec instance to specify the training specification.
-    eval_spec: A `EvalSpec instance to specify the evaluation and export
+    train_spec: A `TrainSpec` instance to specify the training specification.
+    eval_spec: A `EvalSpec` instance to specify the evaluation and export
       specification.
 
   Raises:
-- 
GitLab


From e81fbdf719f39d82afb5c6e27c99cd006fb5f689 Mon Sep 17 00:00:00 2001
From: Armen Donigian <donigian@gmail.com>
Date: Sat, 7 Oct 2017 09:38:14 -0700
Subject: [PATCH 164/909] This branch updates the installation instructions for
 conda install to include pip as well, in order to prevent the usage of the
 pip installed in the root conda environment.

---
 tensorflow/docs_src/install/install_linux.md   | 2 +-
 tensorflow/docs_src/install/install_mac.md     | 2 +-
 tensorflow/docs_src/install/install_windows.md | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/docs_src/install/install_linux.md b/tensorflow/docs_src/install/install_linux.md
index 14cc1f733c..2b488cc4f5 100644
--- a/tensorflow/docs_src/install/install_linux.md
+++ b/tensorflow/docs_src/install/install_linux.md
@@ -457,7 +457,7 @@ Take the following steps to install TensorFlow in an Anaconda environment:
   2. Create a conda environment named <tt>tensorflow</tt> to run a version
      of Python by invoking the following command:
 
-     <pre>$ <b>conda create -n tensorflow python=2.7 # or python=3.3, etc.</b></pre>
+     <pre>$ <b>conda create -n tensorflow pip python=2.7 # or python=3.3, etc.</b></pre>
 
   3. Activate the conda environment by issuing the following command:
 
diff --git a/tensorflow/docs_src/install/install_mac.md b/tensorflow/docs_src/install/install_mac.md
index b6daeb0dd6..efd977089b 100644
--- a/tensorflow/docs_src/install/install_mac.md
+++ b/tensorflow/docs_src/install/install_mac.md
@@ -321,7 +321,7 @@ Take the following steps to install TensorFlow in an Anaconda environment:
   2. Create a conda environment named `tensorflow`
      by invoking the following command:
 
-     <pre>$ <b>conda create -n tensorflow python=2.7 # or python=3.3, etc.</b></pre>
+     <pre>$ <b>conda create -n tensorflow pip python=2.7 # or python=3.3, etc.</b></pre>
 
   3. Activate the conda environment by issuing the following command:
 
diff --git a/tensorflow/docs_src/install/install_windows.md b/tensorflow/docs_src/install/install_windows.md
index ae8749c231..f0d580d803 100644
--- a/tensorflow/docs_src/install/install_windows.md
+++ b/tensorflow/docs_src/install/install_windows.md
@@ -105,7 +105,7 @@ Take the following steps to install TensorFlow in an Anaconda environment:
   2. Create a conda environment named <tt>tensorflow</tt>
      by invoking the following command:
 
-     <pre>C:\> <b>conda create -n tensorflow python=3.5</b> </pre>
+     <pre>C:\> <b>conda create -n tensorflow pip python=3.5</b> </pre>
 
   3. Activate the conda environment by issuing the following command:
 
-- 
GitLab


From 0652d7aced72f795c494cd371d9e6aa8e082d0c8 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Sat, 7 Oct 2017 17:07:40 +0000
Subject: [PATCH 165/909] Fix broken link in performance guide

This fix fixes broken link in performance guide as models repo moved
slim to `models/research/slim`
`https://github.com/tensorflow/models/tree/master/slim#Data`
->
`https://github.com/tensorflow/models/tree/master/research/slim#Data`

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/docs_src/performance/performance_guide.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/docs_src/performance/performance_guide.md b/tensorflow/docs_src/performance/performance_guide.md
index 30fb91f9d9..d3aa901bec 100644
--- a/tensorflow/docs_src/performance/performance_guide.md
+++ b/tensorflow/docs_src/performance/performance_guide.md
@@ -93,7 +93,7 @@ Reading large numbers of small files significantly impacts I/O performance.
 One approach to get maximum I/O throughput is to preprocess input data into
 larger (~100MB) `TFRecord` files. For smaller data sets (200MB-1GB), the best
 approach is often to load the entire data set into memory. The document
-[Downloading and converting to TFRecord format](https://github.com/tensorflow/models/tree/master/slim#Data)
+[Downloading and converting to TFRecord format](https://github.com/tensorflow/models/tree/master/research/slim#Data)
 includes information and scripts for creating `TFRecords` and this
 [script](https://github.com/tensorflow/models/tree/master/tutorials/image/cifar10_estimator/generate_cifar10_tfrecords.py)
 converts the CIFAR-10 data set into `TFRecords`.
-- 
GitLab


From b3a286301beb68d6809f892b7f252204eb02b880 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Sat, 7 Oct 2017 17:12:43 +0000
Subject: [PATCH 166/909] Fix broken link in performance models This fix fixes
 broken link in performance models as models repo moved inception to
 `models/research/inception`:
 `https://github.com/tensorflow/models/tree/master/inception#getting-started`
 ->
 `https://github.com/tensorflow/models/tree/master/research/inception#getting-started`

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/docs_src/performance/performance_models.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/docs_src/performance/performance_models.md b/tensorflow/docs_src/performance/performance_models.md
index 183bbc75a9..fcda19e74c 100644
--- a/tensorflow/docs_src/performance/performance_models.md
+++ b/tensorflow/docs_src/performance/performance_models.md
@@ -345,7 +345,7 @@ executing the main script
 *   **`num_gpus`**: Number of GPUs to use.
 *   **`data_dir`**: Path to data to process. If not set, synthetic data is used.
     To use Imagenet data use these
-    [instructions](https://github.com/tensorflow/models/tree/master/inception#getting-started)
+    [instructions](https://github.com/tensorflow/models/tree/master/research/inception#getting-started)
     as a starting point.
 *   **`batch_size`**: Batch size for each GPU.
 *   **`variable_update`**: The method for managing variables: `parameter_server`
-- 
GitLab


From 54b8c7b8d2d44d862a7ecb297c835d60fca427ad Mon Sep 17 00:00:00 2001
From: Justine Tunney <jart@google.com>
Date: Sat, 7 Oct 2017 22:49:33 -0700
Subject: [PATCH 167/909] Mirror SQLite zip file

PiperOrigin-RevId: 171441141
---
 tensorflow/workspace.bzl | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index de0084613b..6151dc6241 100644
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -313,7 +313,10 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
 
   native.new_http_archive(
       name = "sqlite_archive",
-      urls = ["http://www.sqlite.org/2017/sqlite-amalgamation-3200000.zip"],
+      urls = [
+          "http://mirror.bazel.build/www.sqlite.org/2017/sqlite-amalgamation-3200000.zip",
+          "http://www.sqlite.org/2017/sqlite-amalgamation-3200000.zip",
+      ],
       sha256 = "208780b3616f9de0aeb50822b7a8f5482f6515193859e91ed61637be6ad74fd4",
       strip_prefix = "sqlite-amalgamation-3200000",
       build_file = str(Label("//third_party:sqlite.BUILD"))
-- 
GitLab


From a1ab2a3b5263c535bfece377f1bdd77c7ade3240 Mon Sep 17 00:00:00 2001
From: Justine Tunney <jart@google.com>
Date: Sat, 7 Oct 2017 22:55:05 -0700
Subject: [PATCH 168/909] Pin TensorBoard 0.4 to tf-nightly (#13545)

---
 tensorflow/tools/pip_package/setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/tools/pip_package/setup.py b/tensorflow/tools/pip_package/setup.py
index a7a0706d0b..f476fe766f 100644
--- a/tensorflow/tools/pip_package/setup.py
+++ b/tensorflow/tools/pip_package/setup.py
@@ -36,7 +36,7 @@ REQUIRED_PACKAGES = [
     'numpy >= 1.12.1',
     'six >= 1.10.0',
     'protobuf >= 3.3.0',
-    'tensorflow-tensorboard >= 0.1.0, < 0.2.0',
+    'tensorflow-tensorboard >= 0.4.0rc1, < 0.5.0',
 ]
 
 project_name = 'tensorflow'
-- 
GitLab


From 3431602bdf00038a87522b3afb08095d20e9a064 Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Sat, 7 Oct 2017 23:11:20 -0700
Subject: [PATCH 169/909] Disable kmeans test in tsan.

PiperOrigin-RevId: 171441927
---
 tensorflow/contrib/factorization/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/contrib/factorization/BUILD b/tensorflow/contrib/factorization/BUILD
index 8a7825c614..c741815042 100644
--- a/tensorflow/contrib/factorization/BUILD
+++ b/tensorflow/contrib/factorization/BUILD
@@ -206,6 +206,7 @@ py_test(
     size = "medium",
     srcs = ["python/ops/kmeans_test.py"],
     srcs_version = "PY2AND3",
+    tags = ["notsan"],  # b/67512932
     deps = [
         ":factorization_py",
         ":factorization_py_CYCLIC_DEPENDENCIES_THAT_NEED_TO_GO",
-- 
GitLab


From 074b66af3415cb3c60336b0a94f23aec04a715e3 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Sun, 8 Oct 2017 14:19:49 -0700
Subject: [PATCH 170/909] Change `dim` to `axis` for cosine_distance (#12801)

* Change `dim` to `axis` for cosine_distance

This fix changes  `dim` to `axis` for cosine_distance
so that the args are consistent with other methods in TensorFlow.

The backward-compatibility has been maintained in the fix.

This fix fixes 8205.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Change `dim` to `axis` for tf.losses.cosine_distance

so that args are consistent with other TensorFlow methods.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Update API goldens and address review feedback

This commit updates API goldens so that
`//tensorflow/tools/api/tests:api_compatibility_test`
could pass. Review feedback has also been addressed.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 .../contrib/losses/python/losses/loss_ops.py  | 17 +++++++++-----
 tensorflow/python/ops/losses/losses_impl.py   | 22 +++++++++++++------
 .../tools/api/golden/tensorflow.losses.pbtxt  |  2 +-
 3 files changed, 28 insertions(+), 13 deletions(-)

diff --git a/tensorflow/contrib/losses/python/losses/loss_ops.py b/tensorflow/contrib/losses/python/losses/loss_ops.py
index 1d2477b8b7..7c523ad492 100644
--- a/tensorflow/contrib/losses/python/losses/loss_ops.py
+++ b/tensorflow/contrib/losses/python/losses/loss_ops.py
@@ -28,6 +28,7 @@ from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import nn
 from tensorflow.python.ops import nn_ops
 from tensorflow.python.util.deprecation import deprecated
+from tensorflow.python.util.deprecation import deprecated_args
 
 __all__ = ["absolute_difference",
            "add_loss",
@@ -623,8 +624,9 @@ def mean_pairwise_squared_error(
 
 
 @deprecated("2016-12-30", "Use tf.losses.cosine_distance instead.")
+@deprecated_args(None, "dim is deprecated, use axis instead", "dim")
 def cosine_distance(
-    predictions, labels=None, dim=None, weights=1.0, scope=None):
+    predictions, labels=None, axis=None, weights=1.0, scope=None, dim=None):
   """Adds a cosine-distance loss to the training procedure.
 
   Note that the function assumes that `predictions` and `labels` are already
@@ -633,10 +635,11 @@ def cosine_distance(
   Args:
     predictions: An arbitrary matrix.
     labels: A `Tensor` whose shape matches 'predictions'
-    dim: The dimension along which the cosine distance is computed.
+    axis: The dimension along which the cosine distance is computed.
     weights: Coefficients for the loss a scalar, a tensor of shape
       [batch_size] or a tensor whose shape matches `predictions`.
     scope: The scope for the operations performed in computing the loss.
+    dim: The old (deprecated) name for `axis`.
 
   Returns:
     A scalar `Tensor` representing the loss value.
@@ -645,8 +648,12 @@ def cosine_distance(
     ValueError: If `predictions` shape doesn't match `labels` shape, or
       `weights` is `None`.
   """
-  if dim is None:
-    raise ValueError("`dim` cannot be None.")
+  if dim is not None:
+    if axis is not None:
+      raise ValueError("Cannot specify both 'axis' and 'dim'")
+    axis = dim
+  if axis is None and dim is None:
+    raise ValueError("You must specify 'axis'.")
   with ops.name_scope(scope, "cosine_distance_loss",
                       [predictions, labels, weights]) as scope:
     predictions.get_shape().assert_is_compatible_with(labels.get_shape())
@@ -655,5 +662,5 @@ def cosine_distance(
     labels = math_ops.to_float(labels)
 
     radial_diffs = math_ops.multiply(predictions, labels)
-    losses = 1 - math_ops.reduce_sum(radial_diffs, reduction_indices=[dim,])
+    losses = 1 - math_ops.reduce_sum(radial_diffs, reduction_indices=[axis,])
     return compute_weighted_loss(losses, weights, scope=scope)
diff --git a/tensorflow/python/ops/losses/losses_impl.py b/tensorflow/python/ops/losses/losses_impl.py
index 752d260fba..55a18d28ca 100644
--- a/tensorflow/python/ops/losses/losses_impl.py
+++ b/tensorflow/python/ops/losses/losses_impl.py
@@ -27,6 +27,7 @@ from tensorflow.python.ops import nn
 from tensorflow.python.ops import nn_ops
 from tensorflow.python.ops import weights_broadcast_ops
 from tensorflow.python.ops.losses import util
+from tensorflow.python.util.deprecation import deprecated_args
 
 
 class Reduction(object):
@@ -230,10 +231,12 @@ def absolute_difference(
         losses, weights, scope, loss_collection, reduction=reduction)
 
 
+@deprecated_args(None, "dim is deprecated, use axis instead", "dim")
 def cosine_distance(
-    labels, predictions, dim=None, weights=1.0, scope=None,
+    labels, predictions, axis=None, weights=1.0, scope=None,
     loss_collection=ops.GraphKeys.LOSSES,
-    reduction=Reduction.SUM_BY_NONZERO_WEIGHTS):
+    reduction=Reduction.SUM_BY_NONZERO_WEIGHTS,
+    dim=None):
   """Adds a cosine-distance loss to the training procedure.
 
   Note that the function assumes that `predictions` and `labels` are already
@@ -242,13 +245,14 @@ def cosine_distance(
   Args:
     labels: `Tensor` whose shape matches 'predictions'
     predictions: An arbitrary matrix.
-    dim: The dimension along which the cosine distance is computed.
+    axis: The dimension along which the cosine distance is computed.
     weights: Optional `Tensor` whose rank is either 0, or the same rank as
       `labels`, and must be broadcastable to `labels` (i.e., all dimensions must
       be either `1`, or the same as the corresponding `losses` dimension).
     scope: The scope for the operations performed in computing the loss.
     loss_collection: collection to which this loss will be added.
     reduction: Type of reduction to apply to loss.
+    dim: The old (deprecated) name for `axis`.
 
   Returns:
     Weighted loss float `Tensor`. If `reduction` is `NONE`, this has the same
@@ -256,10 +260,14 @@ def cosine_distance(
 
   Raises:
     ValueError: If `predictions` shape doesn't match `labels` shape, or
-      `dim`, `labels`, `predictions` or `weights` is `None`.
+      `axis`, `labels`, `predictions` or `weights` is `None`.
   """
-  if dim is None:
-    raise ValueError("`dim` cannot be None.")
+  if dim is not None:
+    if axis is not None:
+      raise ValueError("Cannot specify both 'axis' and 'dim'")
+    axis = dim
+  if axis is None and dim is None:
+    raise ValueError("You must specify 'axis'.")
   if labels is None:
     raise ValueError("labels must not be None.")
   if predictions is None:
@@ -271,7 +279,7 @@ def cosine_distance(
     predictions.get_shape().assert_is_compatible_with(labels.get_shape())
 
     radial_diffs = math_ops.multiply(predictions, labels)
-    losses = 1 - math_ops.reduce_sum(radial_diffs, axis=(dim,), keep_dims=True)
+    losses = 1 - math_ops.reduce_sum(radial_diffs, axis=(axis,), keep_dims=True)
     return compute_weighted_loss(
         losses, weights, scope, loss_collection, reduction=reduction)
 
diff --git a/tensorflow/tools/api/golden/tensorflow.losses.pbtxt b/tensorflow/tools/api/golden/tensorflow.losses.pbtxt
index 79443839b9..c1d190ae11 100644
--- a/tensorflow/tools/api/golden/tensorflow.losses.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.losses.pbtxt
@@ -18,7 +18,7 @@ tf_module {
   }
   member_method {
     name: "cosine_distance"
-    argspec: "args=[\'labels\', \'predictions\', \'dim\', \'weights\', \'scope\', \'loss_collection\', \'reduction\'], varargs=None, keywords=None, defaults=[\'None\', \'1.0\', \'None\', \'losses\', \'weighted_sum_by_nonzero_weights\'], "
+    argspec: "args=[\'labels\', \'predictions\', \'axis\', \'weights\', \'scope\', \'loss_collection\', \'reduction\', \'dim\'], varargs=None, keywords=None, defaults=[\'None\', \'1.0\', \'None\', \'losses\', \'weighted_sum_by_nonzero_weights\', \'None\'], "
   }
   member_method {
     name: "get_losses"
-- 
GitLab


From cab4f6f615e259546a1c0719a32d019730b2ee71 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Sun, 8 Oct 2017 15:50:43 -0700
Subject: [PATCH 171/909] Improve invalid size vocab ValueError by appending
 the vocab file. This is helpful to identify erroneous vocab file for the
 common case of training programs with multiple vocabs.

PiperOrigin-RevId: 171476954
---
 .../python/kernel_tests/lookup_ops_test.py    | 21 +++++++++++++++++++
 tensorflow/python/ops/lookup_ops.py           |  7 ++++++-
 2 files changed, 27 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/kernel_tests/lookup_ops_test.py b/tensorflow/python/kernel_tests/lookup_ops_test.py
index 1d92a08f5c..76c790a0a2 100644
--- a/tensorflow/python/kernel_tests/lookup_ops_test.py
+++ b/tensorflow/python/kernel_tests/lookup_ops_test.py
@@ -378,6 +378,27 @@ class IndexTableFromFile(test.TestCase):
     self.assertRaises(
         ValueError, lookup_ops.index_table_from_file, vocabulary_file=None)
 
+  def test_index_table_from_file_str_fails_with_zero_size_vocabulary(self):
+    vocabulary_file = self._createVocabFile("zero_vocab_str.txt")
+    self.assertRaisesRegexp(
+        ValueError,
+        "vocab_size must be greater than 0, got 0. "
+        "vocabulary_file: .*zero_vocab_str.txt",
+        lookup_ops.index_table_from_file,
+        vocabulary_file=vocabulary_file,
+        vocab_size=0)
+
+  def test_index_table_from_file_tensor_fails_with_zero_size_vocabulary(self):
+    vocabulary_file = constant_op.constant(
+        self._createVocabFile("zero_vocab_tensor.txt"))
+    self.assertRaisesRegexp(
+        ValueError,
+        "vocab_size must be greater than 0, got 0. "
+        "vocabulary_file: .*zero_vocab_tensor.txt",
+        lookup_ops.index_table_from_file,
+        vocabulary_file=vocabulary_file,
+        vocab_size=0)
+
   def test_index_table_from_file_with_vocab_size_too_small(self):
     vocabulary_file = self._createVocabFile("f2i_vocab6.txt")
     with self.test_session():
diff --git a/tensorflow/python/ops/lookup_ops.py b/tensorflow/python/ops/lookup_ops.py
index bbfa38aa17..7f00344be2 100644
--- a/tensorflow/python/ops/lookup_ops.py
+++ b/tensorflow/python/ops/lookup_ops.py
@@ -27,6 +27,7 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import sparse_tensor
 from tensorflow.python.framework import tensor_shape
+from tensorflow.python.framework import tensor_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import gen_lookup_ops
@@ -927,7 +928,11 @@ def index_table_from_file(vocabulary_file=None,
     raise ValueError("num_oov_buckets must be greater or equal than 0, got %d."
                      % num_oov_buckets)
   if vocab_size is not None and vocab_size < 1:
-    raise ValueError("vocab_size must be greater than 0, got %d." % vocab_size)
+    vocab_file_value = vocabulary_file
+    if isinstance(vocabulary_file, ops.Tensor):
+      vocab_file_value = tensor_util.constant_value(vocabulary_file) or "?"
+    raise ValueError("vocab_size must be greater than 0, got %d. "
+                     "vocabulary_file: %s" % (vocab_size, vocab_file_value))
   if (not key_dtype.is_integer) and (dtypes.string != key_dtype.base_dtype):
     raise TypeError("Only integer and string keys are supported.")
 
-- 
GitLab


From e0924e0577fe42b455be5fb881647fa64ea5b7c3 Mon Sep 17 00:00:00 2001
From: Jacques Pienaar <jpienaar@google.com>
Date: Sun, 8 Oct 2017 16:18:24 -0700
Subject: [PATCH 172/909] [TFXLA] Don't discard status unless it is NotFound.

PiperOrigin-RevId: 171477807
---
 tensorflow/compiler/tf2xla/xla_compiler.cc    | 19 +++-
 tensorflow/compiler/tf2xla/xla_compiler.h     |  2 +
 .../compiler/tf2xla/xla_compiler_test.cc      | 99 ++++++++++++++-----
 3 files changed, 90 insertions(+), 30 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/xla_compiler.cc b/tensorflow/compiler/tf2xla/xla_compiler.cc
index 8521d4167a..1cd96fc4e2 100644
--- a/tensorflow/compiler/tf2xla/xla_compiler.cc
+++ b/tensorflow/compiler/tf2xla/xla_compiler.cc
@@ -92,7 +92,6 @@ XlaCompiler::XlaCompiler(XlaCompiler::Options options)
   }
 
   local_flib_def_.reset(new FunctionLibraryDefinition(OpRegistry::Global(),
-
                                                       FunctionDefLibrary{}));
   local_pflr_.reset(new ProcessFunctionLibraryRuntime(
       &device_mgr_, Env::Default(), options.graph_def_version,
@@ -142,8 +141,17 @@ Status XlaCompiler::CompileFunction(
   }
 
   const FunctionBody* fbody;
-  if (!GetFunctionBody(function, local_flib_runtime_, &fbody).ok()) {
-    TF_RETURN_IF_ERROR(GetFunctionBody(function, flib_runtime_, &fbody));
+  // The function may be in either the local_flib_runtime_ or flib_runtime_.
+  // Look up the function in local first and if it is not found then look up the
+  // function in flib_runtime_.
+  auto status = GetFunctionBody(function, local_flib_runtime_, &fbody);
+  if (!status.ok()) {
+    if (!errors::IsNotFound(status)) {
+      return status;
+    }
+    TF_RETURN_WITH_CONTEXT_IF_ERROR(
+        GetFunctionBody(function, flib_runtime_, &fbody),
+        "Local lookup failed with: ", status.error_message());
   }
 
   TF_RETURN_IF_ERROR(CheckSignature(fbody->arg_types, args));
@@ -509,7 +517,7 @@ Status XlaCompiler::CompileGraph(const XlaCompiler::CompileOptions& options,
   result->requires_runtime_context = context->has_context_parameter();
 
   // Tuple arguments and runtime context parameters are incompatible.
-  CHECK(!(options.use_tuple_arg && result->requires_runtime_context));
+  TF_RET_CHECK(!(options.use_tuple_arg && result->requires_runtime_context));
 
   VLOG(2) << "Outputs: total: " << context->retvals().size()
           << " nonconstant: " << num_nonconst_outputs;
@@ -546,7 +554,8 @@ Status XlaCompiler::CompileGraph(const XlaCompiler::CompileOptions& options,
        i < context->retvals().size(); ++i) {
     const XlaExpression& retval = context->retvals()[i];
     if (!retval.has_constant_value()) {
-      CHECK_LT(computation_output, num_computation_outputs);
+      TF_RET_CHECK(computation_output < num_computation_outputs)
+          << "Computation has more outputs than expected";
       OutputDescription& output = result->outputs[i];
       output.is_constant = false;
       TF_RETURN_IF_ERROR(XLAShapeToTensorShape(
diff --git a/tensorflow/compiler/tf2xla/xla_compiler.h b/tensorflow/compiler/tf2xla/xla_compiler.h
index 35159dbad4..addea74fc2 100644
--- a/tensorflow/compiler/tf2xla/xla_compiler.h
+++ b/tensorflow/compiler/tf2xla/xla_compiler.h
@@ -287,6 +287,8 @@ class XlaCompiler {
   FunctionLibraryRuntime* flib_runtime() const { return flib_runtime_; }
 
  private:
+  friend class XlaCompilerTest;
+
   Options options_;
 
   // Status set to non-OK in the constructor if initialization fails.
diff --git a/tensorflow/compiler/tf2xla/xla_compiler_test.cc b/tensorflow/compiler/tf2xla/xla_compiler_test.cc
index 531725a623..9af557e23c 100644
--- a/tensorflow/compiler/tf2xla/xla_compiler_test.cc
+++ b/tensorflow/compiler/tf2xla/xla_compiler_test.cc
@@ -27,6 +27,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/tests/literal_test_util.h"
 #include "tensorflow/core/common_runtime/function.h"
 #include "tensorflow/core/framework/common_shape_fns.h"
+#include "tensorflow/core/framework/function_testlib.h"
 #include "tensorflow/core/framework/resource_mgr.h"
 #include "tensorflow/core/framework/tensor_testutil.h"
 #include "tensorflow/core/graph/graph.h"
@@ -36,6 +37,37 @@ limitations under the License.
 #include "tensorflow/core/public/version.h"
 
 namespace tensorflow {
+
+class XlaCompilerTest : public ::testing::Test {
+ protected:
+  XlaCompilerTest() : cpu_device_type_(DEVICE_CPU_XLA_JIT) {}
+
+  void SetUp() override {
+    client_ = xla::ClientLibrary::LocalClientOrDie();
+
+    XlaOpRegistry::RegisterCompilationKernels();
+
+    FunctionDefLibrary flib;
+    flib_def_.reset(new FunctionLibraryDefinition(OpRegistry::Global(), flib));
+  }
+
+  XlaCompiler::Options DefaultOptions() {
+    XlaCompiler::Options options;
+    options.device_type = &cpu_device_type_;
+    options.client = client_;
+    options.flib_def = flib_def_.get();
+    return options;
+  }
+
+  FunctionLibraryDefinition* LocalFlibDef(XlaCompiler* compiler) {
+    return compiler->local_flib_def_.get();
+  }
+
+  DeviceType cpu_device_type_;
+  xla::Client* client_;
+  std::unique_ptr<FunctionLibraryDefinition> flib_def_;
+};
+
 namespace {
 
 // Helper class to test the ability to pass resources through to XLA
@@ -125,31 +157,6 @@ REGISTER_XLA_OP(Name("DummyDuplicateOp").Device(DEVICE_CPU_XLA_JIT),
 REGISTER_XLA_OP(Name("DummyDuplicateOp").Device(DEVICE_GPU_XLA_JIT),
                 DummyDuplicateOp);
 
-class XlaCompilerTest : public ::testing::Test {
- protected:
-  XlaCompilerTest() : cpu_device_type_(DEVICE_CPU_XLA_JIT) {}
-
-  void SetUp() override {
-    client_ = xla::ClientLibrary::LocalClientOrDie();
-
-    XlaOpRegistry::RegisterCompilationKernels();
-
-    FunctionDefLibrary flib;
-    flib_def_.reset(new FunctionLibraryDefinition(OpRegistry::Global(), flib));
-  }
-
-  XlaCompiler::Options DefaultOptions() {
-    XlaCompiler::Options options;
-    options.device_type = &cpu_device_type_;
-    options.client = client_;
-    options.flib_def = flib_def_.get();
-    return options;
-  }
-
-  DeviceType cpu_device_type_;
-  xla::Client* client_;
-  std::unique_ptr<FunctionLibraryDefinition> flib_def_;
-};
 
 // Tests compilation and execution of an empty graph.
 TEST_F(XlaCompilerTest, EmptyReturnValues) {
@@ -489,5 +496,47 @@ TEST_F(XlaCompilerTest, NewTensorArrayGradientsAreComputationOutputs) {
   EXPECT_EQ(1, result.resource_updates.size());
 }
 
+// Tests CompileFunction with undefined function fails.
+TEST_F(XlaCompilerTest, UndefinedFunctionFails) {
+  XlaCompiler compiler(DefaultOptions());
+
+  std::unique_ptr<Graph> graph(new Graph(OpRegistry::Global()));
+  XlaCompiler::CompilationResult result;
+  NameAttrList name_attr;
+  name_attr.set_name("Function_NotDefined_");
+  Status status =
+      compiler.CompileFunction(XlaCompiler::CompileOptions(), name_attr,
+                               /*args=*/{}, &result);
+  EXPECT_FALSE(status.ok());
+  EXPECT_TRUE(StringPiece(status.error_message()).contains("is not defined."))
+      << status.error_message();
+}
+
+// Tests CompileFunction with a local function lookup failing, fails with
+// informative error about both lookups.
+TEST_F(XlaCompilerTest, LocalFunctionWithWrongArgumentsFail) {
+  XlaCompiler compiler(DefaultOptions());
+
+  auto local_flib_def = LocalFlibDef(&compiler);
+  TF_ASSERT_OK(local_flib_def->AddFunctionDef(test::function::XTimesTwo()));
+
+  std::unique_ptr<Graph> graph(new Graph(OpRegistry::Global()));
+  XlaCompiler::CompilationResult result;
+  NameAttrList name_attr;
+  name_attr.set_name("XTimesTwo");
+  Status status =
+      compiler.CompileFunction(XlaCompiler::CompileOptions(), name_attr,
+                               /*args=*/{}, &result);
+
+  ASSERT_FALSE(status.ok());
+  // Flib lookup failure.
+  EXPECT_TRUE(StringPiece(status.error_message()).contains("is not defined."))
+      << status.error_message();
+  // Local flib lookup failure.
+  EXPECT_TRUE(
+      StringPiece(status.error_message()).contains("Attr T is not found"))
+      << status.error_message();
+}
+
 }  // namespace
 }  // namespace tensorflow
-- 
GitLab


From 21da2369596e8d21aab6a562c747f4ea8a72480b Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Sun, 8 Oct 2017 20:47:49 -0700
Subject: [PATCH 173/909] Disable flaky cluster_function_library_runtime_test
 in opensource.

PiperOrigin-RevId: 171489827
---
 tensorflow/core/distributed_runtime/BUILD | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/distributed_runtime/BUILD b/tensorflow/core/distributed_runtime/BUILD
index 87c56b66a5..26e82fbb9a 100644
--- a/tensorflow/core/distributed_runtime/BUILD
+++ b/tensorflow/core/distributed_runtime/BUILD
@@ -121,7 +121,10 @@ tf_cc_test(
     name = "cluster_function_library_runtime_test",
     srcs = ["cluster_function_library_runtime_test.cc"],
     linkstatic = tf_kernel_tests_linkstatic(),
-    tags = ["nomac"],
+    tags = [
+        "no_oss",
+        "nomac",
+    ],
     deps = [
         ":worker_session",
         "//tensorflow/core:framework_internal",
-- 
GitLab


From 159dfb5e0b8e2b393ac6fa24a38c707bca154c1e Mon Sep 17 00:00:00 2001
From: Scott Mudge <19617165+scottmudge@users.noreply.github.com>
Date: Mon, 9 Oct 2017 09:27:00 -0400
Subject: [PATCH 174/909] Fix for AVX2 support in Visual Studio (#13525)

* Fixed AVX2 support for Visual Studio 2015.

* Fixed for portability.
---
 .../CXX11/src/FixedPoint/PacketMathAVX2.h     | 51 +++++++++++++++----
 1 file changed, 40 insertions(+), 11 deletions(-)

diff --git a/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/PacketMathAVX2.h b/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/PacketMathAVX2.h
index 078be83e0d..c210b1712c 100644
--- a/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/PacketMathAVX2.h
+++ b/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/PacketMathAVX2.h
@@ -1,6 +1,35 @@
 #ifndef THIRD_PARTY_EIGEN3_UNSUPPORTED_EIGEN_CXX11_SRC_FIXEDPOINT_PACKETMATHAVX2_H_
 #define THIRD_PARTY_EIGEN3_UNSUPPORTED_EIGEN_CXX11_SRC_FIXEDPOINT_PACKETMATHAVX2_H_
 
+#ifdef _MSC_VER
+
+#include <immintrin.h>
+#include <emmintrin.h>
+#include <smmintrin.h>
+
+#endif
+
+inline int _mm256_extract_epi16_N0(const __m256i X)
+{
+	return _mm_extract_epi16(_mm256_extractf128_si256(X, 0 >> 3), 0 % 8);
+}
+
+inline int _mm256_extract_epi16_N1(const __m256i X)
+{
+	return _mm_extract_epi16(_mm256_extractf128_si256(X, 1 >> 3), 1 % 8);
+}
+
+inline int _mm256_extract_epi8_N0(const __m256i X)
+{
+	return _mm_extract_epi8(_mm256_extractf128_si256((X), 0 >> 4), 0 % 16);
+}
+
+inline int _mm256_extract_epi8_N1(const __m256i X)
+{
+	return _mm_extract_epi8(_mm256_extractf128_si256((X), 1 >> 4), 1 % 16);
+}
+
+
 namespace Eigen {
 namespace internal {
 
@@ -271,15 +300,15 @@ EIGEN_STRONG_INLINE QInt32 pfirst<Packet8q32i>(const Packet8q32i& a) {
 }
 template <>
 EIGEN_STRONG_INLINE QInt16 pfirst<Packet16q16i>(const Packet16q16i& a) {
-  return _mm256_extract_epi16(a.val, 0);
+  return _mm256_extract_epi16_N0(a.val);
 }
 template <>
 EIGEN_STRONG_INLINE QUInt8 pfirst<Packet32q8u>(const Packet32q8u& a) {
-  return static_cast<uint8_t>(_mm256_extract_epi8(a.val, 0));
+  return static_cast<uint8_t>(_mm256_extract_epi8_N0(a.val));
 }
 template <>
 EIGEN_STRONG_INLINE QInt8 pfirst<Packet32q8i>(const Packet32q8i& a) {
-  return _mm256_extract_epi8(a.val, 0);
+  return _mm256_extract_epi8_N0(a.val);
 }
 
 // Initialize to constant value.
@@ -391,7 +420,7 @@ EIGEN_STRONG_INLINE QInt16 predux_min<Packet16q16i>(const Packet16q16i& a) {
   tmp =
       _mm256_min_epi16(tmp, _mm256_shuffle_epi32(tmp, _MM_SHUFFLE(1, 0, 3, 2)));
   tmp = _mm256_min_epi16(tmp, _mm256_shuffle_epi32(tmp, 1));
-  return std::min(_mm256_extract_epi16(tmp, 0), _mm256_extract_epi16(tmp, 1));
+  return std::min(_mm256_extract_epi16_N0(tmp), _mm256_extract_epi16_N1(tmp));
 }
 template <>
 EIGEN_STRONG_INLINE QInt16 predux_max<Packet16q16i>(const Packet16q16i& a) {
@@ -399,7 +428,7 @@ EIGEN_STRONG_INLINE QInt16 predux_max<Packet16q16i>(const Packet16q16i& a) {
   tmp =
       _mm256_max_epi16(tmp, _mm256_shuffle_epi32(tmp, _MM_SHUFFLE(1, 0, 3, 2)));
   tmp = _mm256_max_epi16(tmp, _mm256_shuffle_epi32(tmp, 1));
-  return std::max(_mm256_extract_epi16(tmp, 0), _mm256_extract_epi16(tmp, 1));
+  return std::max(_mm256_extract_epi16_N0(tmp), _mm256_extract_epi16_N1(tmp));
 }
 
 template <>
@@ -410,8 +439,8 @@ EIGEN_STRONG_INLINE QUInt8 predux_min<Packet32q8u>(const Packet32q8u& a) {
   tmp = _mm256_min_epu8(tmp, _mm256_shuffle_epi32(tmp, 1));
   tmp = _mm256_min_epu8(tmp,
                         _mm256_shufflelo_epi16(tmp, _MM_SHUFFLE(1, 0, 3, 2)));
-  return std::min(static_cast<uint8_t>(_mm256_extract_epi8(tmp, 0)),
-                  static_cast<uint8_t>(_mm256_extract_epi8(tmp, 1)));
+  return std::min(static_cast<uint8_t>(_mm256_extract_epi8_N0(tmp)),
+                  static_cast<uint8_t>(_mm256_extract_epi8_N1(tmp)));
 }
 template <>
 EIGEN_STRONG_INLINE QUInt8 predux_max<Packet32q8u>(const Packet32q8u& a) {
@@ -421,8 +450,8 @@ EIGEN_STRONG_INLINE QUInt8 predux_max<Packet32q8u>(const Packet32q8u& a) {
   tmp = _mm256_max_epu8(tmp, _mm256_shuffle_epi32(tmp, 1));
   tmp = _mm256_max_epu8(tmp,
                         _mm256_shufflelo_epi16(tmp, _MM_SHUFFLE(1, 0, 3, 2)));
-  return std::max(static_cast<uint8_t>(_mm256_extract_epi8(tmp, 0)),
-                  static_cast<uint8_t>(_mm256_extract_epi8(tmp, 1)));
+  return std::max(static_cast<uint8_t>(_mm256_extract_epi8_N0(tmp)),
+                  static_cast<uint8_t>(_mm256_extract_epi8_N1(tmp)));
 }
 
 template <>
@@ -431,7 +460,7 @@ EIGEN_STRONG_INLINE QInt8 predux_min<Packet32q8i>(const Packet32q8i& a) {
   tmp = _mm256_min_epi8(tmp, _mm256_shuffle_epi32(tmp, _MM_SHUFFLE(1, 0, 3, 2)));
   tmp = _mm256_min_epi8(tmp, _mm256_shuffle_epi32(tmp, 1));
   tmp = _mm256_min_epi8(tmp, _mm256_shufflelo_epi16(tmp, _MM_SHUFFLE(1, 0, 3, 2)));
-  return std::min(_mm256_extract_epi8(tmp, 0), _mm256_extract_epi8(tmp, 1));
+  return std::min(_mm256_extract_epi8_N0(tmp), _mm256_extract_epi8_N1(tmp));
 }
 template <>
 EIGEN_STRONG_INLINE QInt8 predux_max<Packet32q8i>(const Packet32q8i& a) {
@@ -439,7 +468,7 @@ EIGEN_STRONG_INLINE QInt8 predux_max<Packet32q8i>(const Packet32q8i& a) {
   tmp = _mm256_max_epi8(tmp, _mm256_shuffle_epi32(tmp, _MM_SHUFFLE(1, 0, 3, 2)));
   tmp = _mm256_max_epi8(tmp, _mm256_shuffle_epi32(tmp, 1));
   tmp = _mm256_max_epi8(tmp, _mm256_shufflelo_epi16(tmp, _MM_SHUFFLE(1, 0, 3, 2)));
-  return std::max(_mm256_extract_epi8(tmp, 0), _mm256_extract_epi8(tmp, 1));
+  return std::max(_mm256_extract_epi8_N0(tmp), _mm256_extract_epi8_N1(tmp));
 }
 
 // Vectorized scaling of Packet32q8i by float.
-- 
GitLab


From bb789adc1543684512aab1c83b13872b9ca27c63 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 9 Oct 2017 08:14:04 -0700
Subject: [PATCH 175/909] [TF:XLA] Rename HloOpcode::kLogicalX to kX

PiperOrigin-RevId: 171536686
---
 .../compiler/xla/service/cpu/ir_emitter.cc    |  4 +--
 .../xla/service/elemental_ir_emitter.cc       | 12 +++----
 .../compiler/xla/service/hlo_graph_dumper.cc  |  6 ++--
 .../compiler/xla/service/hlo_instruction.cc   | 34 +++++++++----------
 .../compiler/xla/service/hlo_matchers.h       |  6 ++--
 tensorflow/compiler/xla/service/hlo_opcode.cc | 12 +++----
 tensorflow/compiler/xla/service/hlo_opcode.h  |  6 ++--
 .../xla/service/instruction_fusion.cc         |  6 ++--
 .../compiler/xla/service/shape_inference.cc   |  6 ++--
 .../compiler/xla/service/user_computation.cc  |  6 ++--
 10 files changed, 49 insertions(+), 49 deletions(-)

diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
index 8132207699..c9c87f065b 100644
--- a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
@@ -1511,11 +1511,11 @@ IrEmitter::ReductionGenerator IrEmitter::MatchReductionGenerator(
                                 : ir_builder->CreateFMul(lhs, rhs);
       };
 
-    case HloOpcode::kLogicalAnd:
+    case HloOpcode::kAnd:
       return [](llvm::IRBuilder<>* ir_builder, llvm::Value* lhs,
                 llvm::Value* rhs) { return ir_builder->CreateAnd(lhs, rhs); };
 
-    case HloOpcode::kLogicalOr:
+    case HloOpcode::kOr:
       return [](llvm::IRBuilder<>* ir_builder, llvm::Value* lhs,
                 llvm::Value* rhs) { return ir_builder->CreateOr(lhs, rhs); };
 
diff --git a/tensorflow/compiler/xla/service/elemental_ir_emitter.cc b/tensorflow/compiler/xla/service/elemental_ir_emitter.cc
index 7117ecb08b..12fb88f39c 100644
--- a/tensorflow/compiler/xla/service/elemental_ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/elemental_ir_emitter.cc
@@ -126,7 +126,7 @@ StatusOr<llvm::Value*> ElementalIrEmitter::EmitIntegerUnaryOp(
     }
     case HloOpcode::kNegate:
       return ir_builder_->CreateNeg(operand_value);
-    case HloOpcode::kLogicalNot:
+    case HloOpcode::kNot:
       // It is not sufficient to just call CreateNot() here because a PRED is
       // represented as an i8 and the truth value is stored only in the bottom
       // bit.
@@ -557,9 +557,9 @@ StatusOr<llvm::Value*> ElementalIrEmitter::EmitIntegerBinaryOp(
               is_signed ? llvm::ICmpInst::ICMP_SGE : llvm::ICmpInst::ICMP_UGE,
               lhs_value, rhs_value),
           lhs_value, rhs_value);
-    case HloOpcode::kLogicalAnd:
+    case HloOpcode::kAnd:
       return ir_builder_->CreateAnd(lhs_value, rhs_value);
-    case HloOpcode::kLogicalOr:
+    case HloOpcode::kOr:
       return ir_builder_->CreateOr(lhs_value, rhs_value);
     default:
       return Unimplemented("binary integer op '%s'",
@@ -799,7 +799,7 @@ llvm_ir::ElementGenerator ElementalIrEmitter::MakeElementGenerator(
     case HloOpcode::kSign:
     case HloOpcode::kSin:
     case HloOpcode::kTanh:
-    case HloOpcode::kLogicalNot:
+    case HloOpcode::kNot:
       return [this, hlo, &operand_to_generator](
                  const IrArray::Index& index) -> StatusOr<llvm::Value*> {
         TF_ASSIGN_OR_RETURN(llvm::Value * operand_value,
@@ -821,8 +821,8 @@ llvm_ir::ElementGenerator ElementalIrEmitter::MakeElementGenerator(
     case HloOpcode::kPower:
     case HloOpcode::kRemainder:
     case HloOpcode::kSubtract:
-    case HloOpcode::kLogicalAnd:
-    case HloOpcode::kLogicalOr:
+    case HloOpcode::kAnd:
+    case HloOpcode::kOr:
       return [this, hlo, &operand_to_generator](
                  const IrArray::Index& index) -> StatusOr<llvm::Value*> {
         const HloInstruction* lhs = hlo->operand(0);
diff --git a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
index 9b4a2f1048..20fc85c0e9 100644
--- a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
+++ b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
@@ -777,9 +777,9 @@ ColorScheme HloDotDumper::GetInstructionColor(const HloInstruction* instr) {
     case HloOpcode::kIsFinite:
     case HloOpcode::kLe:
     case HloOpcode::kLog:
-    case HloOpcode::kLogicalAnd:
-    case HloOpcode::kLogicalNot:
-    case HloOpcode::kLogicalOr:
+    case HloOpcode::kAnd:
+    case HloOpcode::kNot:
+    case HloOpcode::kOr:
     case HloOpcode::kLt:
     case HloOpcode::kMaximum:
     case HloOpcode::kMinimum:
diff --git a/tensorflow/compiler/xla/service/hlo_instruction.cc b/tensorflow/compiler/xla/service/hlo_instruction.cc
index 7419ab8704..77a748163e 100644
--- a/tensorflow/compiler/xla/service/hlo_instruction.cc
+++ b/tensorflow/compiler/xla/service/hlo_instruction.cc
@@ -126,7 +126,7 @@ HloInstruction::CreateGetTupleElement(const Shape& shape,
     case HloOpcode::kFloor:
     case HloOpcode::kIsFinite:
     case HloOpcode::kLog:
-    case HloOpcode::kLogicalNot:
+    case HloOpcode::kNot:
     case HloOpcode::kNegate:
     case HloOpcode::kSign:
     case HloOpcode::kSin:
@@ -161,8 +161,8 @@ HloInstruction::CreateGetTupleElement(const Shape& shape,
     case (HloOpcode::kPower):
     case (HloOpcode::kRemainder):
     case (HloOpcode::kSubtract):
-    case (HloOpcode::kLogicalAnd):
-    case (HloOpcode::kLogicalOr):
+    case (HloOpcode::kAnd):
+    case (HloOpcode::kOr):
       break;
     default:
       LOG(FATAL) << "Invalid binary instruction opcode "
@@ -879,7 +879,7 @@ std::unique_ptr<HloInstruction> HloInstruction::CloneWithNewOperands(
     case HloOpcode::kIsFinite:
     case HloOpcode::kFloor:
     case HloOpcode::kLog:
-    case HloOpcode::kLogicalNot:
+    case HloOpcode::kNot:
     case HloOpcode::kNegate:
     case HloOpcode::kSign:
     case HloOpcode::kSin:
@@ -903,8 +903,8 @@ std::unique_ptr<HloInstruction> HloInstruction::CloneWithNewOperands(
     case HloOpcode::kMinimum:
     case HloOpcode::kPower:
     case HloOpcode::kRemainder:
-    case HloOpcode::kLogicalAnd:
-    case HloOpcode::kLogicalOr:
+    case HloOpcode::kAnd:
+    case HloOpcode::kOr:
       CHECK_EQ(new_operands.size(), 2);
       return CreateBinary(shape, opcode_, new_operands[0], new_operands[1]);
     // Ternary ops.
@@ -1258,9 +1258,9 @@ bool HloInstruction::IdenticalSlowPath(
     case HloOpcode::kIsFinite:
     case HloOpcode::kLe:
     case HloOpcode::kLog:
-    case HloOpcode::kLogicalAnd:
-    case HloOpcode::kLogicalNot:
-    case HloOpcode::kLogicalOr:
+    case HloOpcode::kAnd:
+    case HloOpcode::kNot:
+    case HloOpcode::kOr:
     case HloOpcode::kLt:
     case HloOpcode::kMaximum:
     case HloOpcode::kMinimum:
@@ -1957,9 +1957,9 @@ Status HloInstruction::Visit(DfsHloVisitor* visitor) {
       return visitor->HandleMaximum(this);
     case HloOpcode::kMinimum:
       return visitor->HandleMinimum(this);
-    case HloOpcode::kLogicalAnd:
+    case HloOpcode::kAnd:
       return visitor->HandleLogicalAnd(this, operands_[0], operands_[1]);
-    case HloOpcode::kLogicalOr:
+    case HloOpcode::kOr:
       return visitor->HandleLogicalOr(this, operands_[0], operands_[1]);
     case HloOpcode::kConcatenate:
       return visitor->HandleConcatenate(this, operands_);
@@ -2016,7 +2016,7 @@ Status HloInstruction::Visit(DfsHloVisitor* visitor) {
       return visitor->HandleSin(this, operands_[0]);
     case HloOpcode::kIsFinite:
       return visitor->HandleIsFinite(this, operands_[0]);
-    case HloOpcode::kLogicalNot:
+    case HloOpcode::kNot:
       return visitor->HandleLogicalNot(this, operands_[0]);
     case HloOpcode::kBitcast:
       return visitor->HandleBitcast(this);
@@ -2319,8 +2319,8 @@ bool HloInstruction::IsElementwiseBinary() const {
     case HloOpcode::kPower:
     case HloOpcode::kRemainder:
     case HloOpcode::kSubtract:
-    case HloOpcode::kLogicalAnd:
-    case HloOpcode::kLogicalOr:
+    case HloOpcode::kAnd:
+    case HloOpcode::kOr:
       return true;
     default:
       return false;
@@ -2344,7 +2344,7 @@ bool HloInstruction::IsElementwise() const {
     case HloOpcode::kFloor:
     case HloOpcode::kIsFinite:
     case HloOpcode::kLog:
-    case HloOpcode::kLogicalNot:
+    case HloOpcode::kNot:
     case HloOpcode::kNegate:
     case HloOpcode::kReducePrecision:
     case HloOpcode::kSign:
@@ -2368,8 +2368,8 @@ bool HloInstruction::IsElementwise() const {
     case HloOpcode::kPower:
     case HloOpcode::kRemainder:
     case HloOpcode::kSubtract:
-    case HloOpcode::kLogicalAnd:
-    case HloOpcode::kLogicalOr:
+    case HloOpcode::kAnd:
+    case HloOpcode::kOr:
       return true;
 
     // Ternary elementwise operations.
diff --git a/tensorflow/compiler/xla/service/hlo_matchers.h b/tensorflow/compiler/xla/service/hlo_matchers.h
index b1b3dd61a6..ab5e5463fa 100644
--- a/tensorflow/compiler/xla/service/hlo_matchers.h
+++ b/tensorflow/compiler/xla/service/hlo_matchers.h
@@ -79,9 +79,9 @@ HLO_MATCHER(Infeed);
 HLO_MATCHER(IsFinite);
 HLO_MATCHER(Le);
 HLO_MATCHER(Log);
-HLO_MATCHER(LogicalAnd);
-HLO_MATCHER(LogicalNot);
-HLO_MATCHER(LogicalOr);
+HLO_MATCHER(And);
+HLO_MATCHER(Not);
+HLO_MATCHER(Or);
 HLO_MATCHER(Lt);
 HLO_MATCHER(Map);
 HLO_MATCHER(Maximum);
diff --git a/tensorflow/compiler/xla/service/hlo_opcode.cc b/tensorflow/compiler/xla/service/hlo_opcode.cc
index 83fe6ef6c9..d3d78f4a99 100644
--- a/tensorflow/compiler/xla/service/hlo_opcode.cc
+++ b/tensorflow/compiler/xla/service/hlo_opcode.cc
@@ -95,12 +95,12 @@ string HloOpcodeString(HloOpcode opcode) {
       return "less-than-or-equal-to";
     case HloOpcode::kLog:
       return "log";
-    case HloOpcode::kLogicalAnd:
-      return "logical-and";
-    case HloOpcode::kLogicalOr:
-      return "logical-or";
-    case HloOpcode::kLogicalNot:
-      return "logical-not";
+    case HloOpcode::kAnd:
+      return "and";
+    case HloOpcode::kOr:
+      return "or";
+    case HloOpcode::kNot:
+      return "not";
     case HloOpcode::kLt:
       return "less-than";
     case HloOpcode::kMap:
diff --git a/tensorflow/compiler/xla/service/hlo_opcode.h b/tensorflow/compiler/xla/service/hlo_opcode.h
index 7b23249640..9c26f360fb 100644
--- a/tensorflow/compiler/xla/service/hlo_opcode.h
+++ b/tensorflow/compiler/xla/service/hlo_opcode.h
@@ -62,9 +62,9 @@ enum class HloOpcode {
   kIsFinite,
   kLe,
   kLog,
-  kLogicalAnd,
-  kLogicalNot,
-  kLogicalOr,
+  kAnd,
+  kNot,
+  kOr,
   kLt,
   kMap,
   kMaximum,
diff --git a/tensorflow/compiler/xla/service/instruction_fusion.cc b/tensorflow/compiler/xla/service/instruction_fusion.cc
index 7a27381642..e08e4e4d69 100644
--- a/tensorflow/compiler/xla/service/instruction_fusion.cc
+++ b/tensorflow/compiler/xla/service/instruction_fusion.cc
@@ -53,9 +53,9 @@ namespace xla {
     case HloOpcode::kInfeed:
     case HloOpcode::kIsFinite:
     case HloOpcode::kLe:
-    case HloOpcode::kLogicalAnd:
-    case HloOpcode::kLogicalNot:
-    case HloOpcode::kLogicalOr:
+    case HloOpcode::kAnd:
+    case HloOpcode::kNot:
+    case HloOpcode::kOr:
     case HloOpcode::kLt:
     case HloOpcode::kMaximum:
     case HloOpcode::kMinimum:
diff --git a/tensorflow/compiler/xla/service/shape_inference.cc b/tensorflow/compiler/xla/service/shape_inference.cc
index 29221d2d29..06a68c81e4 100644
--- a/tensorflow/compiler/xla/service/shape_inference.cc
+++ b/tensorflow/compiler/xla/service/shape_inference.cc
@@ -57,7 +57,7 @@ UnaryOperation OpcodeToUnaryOperation(HloOpcode opcode) {
       return UNOP_IS_FINITE;
     case HloOpcode::kLog:
       return UNOP_LOG;
-    case HloOpcode::kLogicalNot:
+    case HloOpcode::kNot:
       return UNOP_LOGICAL_NOT;
     case HloOpcode::kNegate:
       return UNOP_NEGATE;
@@ -113,9 +113,9 @@ BinaryOperation OpcodeToBinaryOperation(HloOpcode opcode) {
       return BINOP_POW;
     case HloOpcode::kRemainder:
       return BINOP_REM;
-    case HloOpcode::kLogicalOr:
+    case HloOpcode::kOr:
       return BINOP_LOGICAL_OR;
-    case HloOpcode::kLogicalAnd:
+    case HloOpcode::kAnd:
       return BINOP_LOGICAL_AND;
     default:
       LOG(FATAL) << "unhandled opcode " << opcode;
diff --git a/tensorflow/compiler/xla/service/user_computation.cc b/tensorflow/compiler/xla/service/user_computation.cc
index 3f62501bb5..05f5476b88 100644
--- a/tensorflow/compiler/xla/service/user_computation.cc
+++ b/tensorflow/compiler/xla/service/user_computation.cc
@@ -59,7 +59,7 @@ HloOpcode UnaryOperationToHloOpcode(UnaryOperation unop) {
     case UNOP_LOG:
       return HloOpcode::kLog;
     case UNOP_LOGICAL_NOT:
-      return HloOpcode::kLogicalNot;
+      return HloOpcode::kNot;
     case UNOP_NEGATE:
       return HloOpcode::kNegate;
     case UNOP_ROUND_NEAREST_AFZ:
@@ -112,9 +112,9 @@ HloOpcode BinaryOperationToHloOpcode(BinaryOperation binop) {
     case BINOP_REM:
       return HloOpcode::kRemainder;
     case BINOP_LOGICAL_OR:
-      return HloOpcode::kLogicalOr;
+      return HloOpcode::kOr;
     case BINOP_LOGICAL_AND:
-      return HloOpcode::kLogicalAnd;
+      return HloOpcode::kAnd;
     default:
       LOG(FATAL) << "unhandled operation " << binop;
   }
-- 
GitLab


From edfb9bb100f9814bf1bbcff2e8a32f12f049bfcc Mon Sep 17 00:00:00 2001
From: Asim Shankar <ashankar@google.com>
Date: Mon, 9 Oct 2017 08:56:08 -0700
Subject: [PATCH 176/909] Correct documentation typo.

Fixes #13576

PiperOrigin-RevId: 171540987
---
 tensorflow/python/ops/nn_ops.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/ops/nn_ops.py b/tensorflow/python/ops/nn_ops.py
index babe2efba0..8876591e53 100644
--- a/tensorflow/python/ops/nn_ops.py
+++ b/tensorflow/python/ops/nn_ops.py
@@ -1844,7 +1844,7 @@ def sparse_softmax_cross_entropy_with_logits(_sentinel=None,  # pylint: disable=
 
   Raises:
     ValueError: If logits are scalars (need to have rank >= 1) or if the rank
-      of the labels is not equal to the rank of the labels minus one.
+      of the labels is not equal to the rank of the logits minus one.
   """
   _ensure_xent_args("sparse_softmax_cross_entropy_with_logits", _sentinel,
                     labels, logits)
-- 
GitLab


From b0b92fd60b44808925fa554190b80d09ced67677 Mon Sep 17 00:00:00 2001
From: Derek Murray <mrry@google.com>
Date: Mon, 9 Oct 2017 09:06:45 -0700
Subject: [PATCH 177/909] [tf.data] Add new custom transformation:
 `tf.contrib.data.scan()`.

`scan()` is similar to `Dataset.map()`, with the addition of a generic piece of
state that is accumulated across the elements of the input, and that may be
used in the computation of the output elements.

This change also updates `rejection_resample()` to use `scan()` rather than a
local `tf.ResourceVariable` for accumulating the number of times each class
has been encountered.

PiperOrigin-RevId: 171542274
---
 .../contrib/data/python/kernel_tests/BUILD    |  27 +++
 .../data/python/kernel_tests/resample_test.py |  22 --
 .../kernel_tests/scan_dataset_op_test.py      | 128 +++++++++++
 tensorflow/contrib/data/python/ops/BUILD      |   1 +
 .../contrib/data/python/ops/resampling.py     |  49 ++--
 .../contrib/data/python/ops/scan_ops.py       | 182 +++++++++++++++
 tensorflow/core/kernels/BUILD                 |  15 ++
 tensorflow/core/kernels/scan_dataset_op.cc    | 213 ++++++++++++++++++
 tensorflow/core/ops/dataset_ops.cc            |  15 ++
 9 files changed, 603 insertions(+), 49 deletions(-)
 create mode 100644 tensorflow/contrib/data/python/kernel_tests/scan_dataset_op_test.py
 create mode 100644 tensorflow/contrib/data/python/ops/scan_ops.py
 create mode 100644 tensorflow/core/kernels/scan_dataset_op.cc

diff --git a/tensorflow/contrib/data/python/kernel_tests/BUILD b/tensorflow/contrib/data/python/kernel_tests/BUILD
index c34c9dad9b..faf051203c 100644
--- a/tensorflow/contrib/data/python/kernel_tests/BUILD
+++ b/tensorflow/contrib/data/python/kernel_tests/BUILD
@@ -237,6 +237,33 @@ py_test(
     ],
 )
 
+py_test(
+    name = "scan_dataset_op_test",
+    size = "small",
+    srcs = ["scan_dataset_op_test.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        "//tensorflow/contrib/data/python/ops:dataset_ops",
+        "//tensorflow/contrib/data/python/ops:transformation_ops",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:constant_op",
+        "//tensorflow/python:data_flow_ops",
+        "//tensorflow/python:dtypes",
+        "//tensorflow/python:errors",
+        "//tensorflow/python:functional_ops",
+        "//tensorflow/python:io_ops",
+        "//tensorflow/python:lookup_ops",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:random_ops",
+        "//tensorflow/python:script_ops",
+        "//tensorflow/python:string_ops",
+        "//tensorflow/python:util",
+        "//tensorflow/python:variable_scope",
+        "//third_party/py/numpy",
+    ],
+)
+
 py_test(
     name = "range_dataset_op_test",
     size = "small",
diff --git a/tensorflow/contrib/data/python/kernel_tests/resample_test.py b/tensorflow/contrib/data/python/kernel_tests/resample_test.py
index a19c917075..0ac8d7359f 100644
--- a/tensorflow/contrib/data/python/kernel_tests/resample_test.py
+++ b/tensorflow/contrib/data/python/kernel_tests/resample_test.py
@@ -22,11 +22,8 @@ import numpy as np
 from tensorflow.contrib.data.python.ops import dataset_ops
 from tensorflow.contrib.data.python.ops import resampling
 from tensorflow.python.framework import errors
-from tensorflow.python.framework import ops
 from tensorflow.python.ops import string_ops
-from tensorflow.python.ops import variables
 from tensorflow.python.platform import test
-from tensorflow.python.training import device_setter
 from tensorflow.python.util import compat
 
 
@@ -51,10 +48,8 @@ class ResampleTest(test.TestCase):
                 seed=27)).make_initializable_iterator())
     init_op = iterator.initializer
     get_next = iterator.get_next()
-    variable_init_op = variables.local_variables_initializer()
 
     with self.test_session() as sess:
-      sess.run(variable_init_op)
       sess.run(init_op)
       returned = []
       with self.assertRaises(errors.OutOfRangeError):
@@ -75,23 +70,6 @@ class ResampleTest(test.TestCase):
     returned_dist = class_counts / total_returned
     self.assertAllClose(target_dist, returned_dist, atol=1e-2)
 
-  def testVariableDevicePlacement(self):
-    classes = np.random.randint(5, size=(20000,))  # Uniformly sampled
-    target_dist = [0.9, 0.05, 0.05, 0.0, 0.0]
-    with ops.device(
-        device_setter.replica_device_setter(ps_tasks=1, ps_device="/cpu:0")):
-      _ = (dataset_ops.Dataset.from_tensor_slices(classes).shuffle(
-          200, seed=21).map(lambda c: (c, string_ops.as_string(c))).apply(
-              resampling.rejection_resample(
-                  target_dist=target_dist,
-                  initial_dist=None,
-                  class_func=lambda c, _: c,
-                  seed=27)))
-
-      self.assertEqual(1, len(variables.local_variables()))
-      self.assertEqual(b"",
-                       compat.as_bytes(variables.local_variables()[0].device))
-
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/contrib/data/python/kernel_tests/scan_dataset_op_test.py b/tensorflow/contrib/data/python/kernel_tests/scan_dataset_op_test.py
new file mode 100644
index 0000000000..5338ec56bf
--- /dev/null
+++ b/tensorflow/contrib/data/python/kernel_tests/scan_dataset_op_test.py
@@ -0,0 +1,128 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for the experimental input pipeline ops."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import itertools
+
+import numpy as np
+
+from tensorflow.contrib.data.python.ops import scan_ops
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import errors
+from tensorflow.python.ops import array_ops
+from tensorflow.python.platform import test
+
+
+class ScanDatasetTest(test.TestCase):
+
+  def _count(self, start, step):
+    return dataset_ops.Dataset.from_tensors(0).repeat(None).apply(
+        scan_ops.scan(start, lambda state, _: (state + step, state)))
+
+  def testCount(self):
+    start = array_ops.placeholder(dtypes.int32, shape=[])
+    step = array_ops.placeholder(dtypes.int32, shape=[])
+    take = array_ops.placeholder(dtypes.int64, shape=[])
+    iterator = self._count(start, step).take(take).make_initializable_iterator()
+    next_element = iterator.get_next()
+
+    with self.test_session() as sess:
+
+      for start_val, step_val, take_val in [(0, 1, 10), (0, 1, 0), (10, 1, 10),
+                                            (10, 2, 10), (10, -1, 10),
+                                            (10, -2, 10)]:
+        sess.run(iterator.initializer,
+                 feed_dict={start: start_val, step: step_val, take: take_val})
+        for expected, _ in zip(
+            itertools.count(start_val, step_val), range(take_val)):
+          self.assertEqual(expected, sess.run(next_element))
+        with self.assertRaises(errors.OutOfRangeError):
+          sess.run(next_element)
+
+  def testFibonacci(self):
+    iterator = dataset_ops.Dataset.from_tensors(1).repeat(None).apply(
+        scan_ops.scan([0, 1], lambda a, _: ([a[1], a[0] + a[1]], a[1]))
+    ).make_one_shot_iterator()
+    next_element = iterator.get_next()
+
+    with self.test_session() as sess:
+      self.assertEqual(1, sess.run(next_element))
+      self.assertEqual(1, sess.run(next_element))
+      self.assertEqual(2, sess.run(next_element))
+      self.assertEqual(3, sess.run(next_element))
+      self.assertEqual(5, sess.run(next_element))
+      self.assertEqual(8, sess.run(next_element))
+
+  def testChangingStateShape(self):
+    # Test the fixed-point shape invariant calculations: start with
+    # initial values with known shapes, and use a scan function that
+    # changes the size of the state on each element.
+    def _scan_fn(state, input_value):
+      # Statically known rank, but dynamic length.
+      ret_longer_vector = array_ops.concat([state[0], state[0]], 0)
+      # Statically unknown rank.
+      ret_larger_rank = array_ops.expand_dims(state[1], 0)
+      return (ret_longer_vector, ret_larger_rank), (state, input_value)
+
+    dataset = dataset_ops.Dataset.from_tensors(0).repeat(5).apply(
+        scan_ops.scan(([0], 1), _scan_fn))
+    self.assertEqual([None], dataset.output_shapes[0][0].as_list())
+    self.assertIs(None, dataset.output_shapes[0][1].ndims)
+    self.assertEqual([], dataset.output_shapes[1].as_list())
+
+    iterator = dataset.make_one_shot_iterator()
+    next_element = iterator.get_next()
+
+    with self.test_session() as sess:
+      for i in range(5):
+        (longer_vector_val, larger_rank_val), _ = sess.run(next_element)
+        self.assertAllEqual([0] * (2**i), longer_vector_val)
+        self.assertAllEqual(np.array(1, ndmin=i), larger_rank_val)
+      with self.assertRaises(errors.OutOfRangeError):
+        sess.run(next_element)
+
+  def testIncorrectStateType(self):
+
+    def _scan_fn(state, _):
+      return constant_op.constant(1, dtype=dtypes.int64), state
+
+    dataset = dataset_ops.Dataset.range(10)
+    with self.assertRaisesRegexp(
+        TypeError,
+        "The element types for the new state must match the initial state."):
+      dataset.apply(
+          scan_ops.scan(constant_op.constant(1, dtype=dtypes.int32), _scan_fn))
+
+  def testIncorrectReturnType(self):
+
+    def _scan_fn(unused_state, unused_input_value):
+      return constant_op.constant(1, dtype=dtypes.int64)
+
+    dataset = dataset_ops.Dataset.range(10)
+    with self.assertRaisesRegexp(
+        TypeError,
+        "The scan function must return a pair comprising the new state and the "
+        "output value."):
+      dataset.apply(
+          scan_ops.scan(constant_op.constant(1, dtype=dtypes.int32), _scan_fn))
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/contrib/data/python/ops/BUILD b/tensorflow/contrib/data/python/ops/BUILD
index 690cccbea3..2a9b41d6df 100644
--- a/tensorflow/contrib/data/python/ops/BUILD
+++ b/tensorflow/contrib/data/python/ops/BUILD
@@ -49,6 +49,7 @@ py_library(
         "error_ops.py",
         "grouping.py",
         "resampling.py",
+        "scan_ops.py",
         "sloppy_ops.py",
     ],
     srcs_version = "PY2AND3",
diff --git a/tensorflow/contrib/data/python/ops/resampling.py b/tensorflow/contrib/data/python/ops/resampling.py
index ee46f3e852..56f526a330 100644
--- a/tensorflow/contrib/data/python/ops/resampling.py
+++ b/tensorflow/contrib/data/python/ops/resampling.py
@@ -20,6 +20,7 @@ from __future__ import print_function
 import numpy as np
 
 from tensorflow.contrib.data.python.ops import batching
+from tensorflow.contrib.data.python.ops import scan_ops
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
@@ -28,7 +29,6 @@ from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import logging_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import random_ops
-from tensorflow.python.ops import resource_variable_ops
 
 
 def rejection_resample(class_func, target_dist, initial_dist=None, seed=None):
@@ -68,26 +68,20 @@ def rejection_resample(class_func, target_dist, initial_dist=None, seed=None):
       num_classes = (target_dist_t.shape[0].value or
                      array_ops.shape(target_dist_t)[0])
       smoothing_constant = 10
-      # Disable device functions and colocation constraints so that the variable
-      # will be placed with the eventual DT_VARIANT dataset tensor.
-      with ops.colocate_with(None, ignore_existing=True):
-        num_examples_per_class_seen = resource_variable_ops.ResourceVariable(
-            initial_value=array_ops.fill([num_classes],
-                                         np.int64(smoothing_constant)),
-            trainable=False,
-            collections=[ops.GraphKeys.LOCAL_VARIABLES],
-            name="local_class_count",
-            dtype=dtypes.int64)
-
-      def update_estimate_and_tile(c):
-        return array_ops.tile(
-            array_ops.expand_dims(
-                _estimate_data_distribution(c, num_examples_per_class_seen), 0),
-            [dist_estimation_batch_size, 1])
+      initial_examples_per_class_seen = array_ops.fill(
+          [num_classes], np.int64(smoothing_constant))
+
+      def update_estimate_and_tile(num_examples_per_class_seen, c):
+        updated_examples_per_class_seen, dist = _estimate_data_distribution(
+            c, num_examples_per_class_seen)
+        tiled_dist = array_ops.tile(
+            array_ops.expand_dims(dist, 0), [dist_estimation_batch_size, 1])
+        return updated_examples_per_class_seen, tiled_dist
 
       initial_dist_ds = (class_values_ds.batch(dist_estimation_batch_size)
-                         .map(update_estimate_and_tile).apply(batching
-                                                              .unbatch()))
+                         .apply(scan_ops.scan(initial_examples_per_class_seen,
+                                              update_estimate_and_tile))
+                         .apply(batching.unbatch()))
       acceptance_dist_ds = initial_dist_ds.map(
           lambda initial: _calculate_acceptance_probs(initial, target_dist_t))
 
@@ -174,20 +168,21 @@ def _estimate_data_distribution(c, num_examples_per_class_seen):
 
   Args:
     c: The class labels.  Type `int32`, shape `[batch_size]`.
-    num_examples_per_class_seen: A `ResourceVariable` containing counts.
-      Type `int64`, shape `[num_classes]`.
+    num_examples_per_class_seen: Type `int64`, shape `[num_classes]`,
+      containing counts.
 
   Returns:
+    num_examples_per_lass_seen: Updated counts.  Type `int64`, shape
+      `[num_classes]`.
     dist: The updated distribution.  Type `float32`, shape `[num_classes]`.
   """
   num_classes = num_examples_per_class_seen.get_shape()[0].value
-  # Update the class-count based on what labels are seen in
-  # batch.  But do this asynchronously to avoid performing a
-  # cross-device round-trip.  Just use the cached value.
-  num_examples_per_class_seen = num_examples_per_class_seen.assign_add(
-      math_ops.reduce_sum(
+  # Update the class-count based on what labels are seen in batch.
+  num_examples_per_class_seen = math_ops.add(
+      num_examples_per_class_seen, math_ops.reduce_sum(
           array_ops.one_hot(c, num_classes, dtype=dtypes.int64), 0))
   init_prob_estimate = math_ops.truediv(
       num_examples_per_class_seen,
       math_ops.reduce_sum(num_examples_per_class_seen))
-  return math_ops.cast(init_prob_estimate, dtypes.float32)
+  dist = math_ops.cast(init_prob_estimate, dtypes.float32)
+  return num_examples_per_class_seen, dist
diff --git a/tensorflow/contrib/data/python/ops/scan_ops.py b/tensorflow/contrib/data/python/ops/scan_ops.py
new file mode 100644
index 0000000000..5acaed48a3
--- /dev/null
+++ b/tensorflow/contrib/data/python/ops/scan_ops.py
@@ -0,0 +1,182 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Scan dataset transformation."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import collections
+
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.data.util import nest
+from tensorflow.python.framework import function
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import gen_dataset_ops
+
+
+class _ScanDataset(dataset_ops.Dataset):
+  """A dataset that scans a function across its input."""
+
+  def __init__(self, input_dataset, initial_state, scan_func):
+    """See `scan()` for details."""
+    super(_ScanDataset, self).__init__()
+    self._input_dataset = input_dataset
+
+    with ops.name_scope("initial_state"):
+      self._initial_state = nest.pack_sequence_as(initial_state, [
+          ops.convert_to_tensor(t, name="component_%d" % i)
+          for i, t in enumerate(nest.flatten(initial_state))
+      ])
+
+    # Compute initial values for the state shapes and types based on
+    # the initial state. These will be refined by running
+    # `tf_scan_func` one or more times below.
+    self._state_shapes = nest.pack_sequence_as(
+        self._initial_state,
+        [t.shape for t in nest.flatten(self._initial_state)])
+    self._state_types = nest.pack_sequence_as(
+        self._initial_state,
+        [t.dtype for t in nest.flatten(self._initial_state)])
+
+    # Will be populated by calling `tf_scan_func`.
+    self._output_shapes = None
+    self._output_types = None
+
+    # Iteratively rerun the scan function until reaching a fixed pont on
+    # `self._state_shapes`.
+    need_to_rerun = True
+    while need_to_rerun:
+
+      flat_state_shapes = nest.flatten(self._state_shapes)
+      flat_state_types = nest.flatten(self._state_types)
+
+      # Create a list in which `tf_scan_func` will store the s
+      flat_new_state_shapes = []
+
+      @function.Defun(
+          *(flat_state_types + nest.flatten(input_dataset.output_types)))
+      def tf_scan_func(*args):
+        """A wrapper for Defun that facilitates shape inference."""
+        # Pass in shape information from the state and input_dataset.
+        for arg, shape in zip(
+            args,
+            flat_state_shapes + nest.flatten(input_dataset.output_shapes)):
+          arg.set_shape(shape)
+
+        pivot = len(flat_state_shapes)
+        old_state = nest.pack_sequence_as(self._initial_state, args[:pivot])
+        input_value = nest.pack_sequence_as(input_dataset.output_types,
+                                            args[pivot:])
+
+        ret = scan_func(old_state, input_value)
+        if not isinstance(ret, collections.Sequence) or len(ret) != 2:
+          raise TypeError("The scan function must return a pair comprising the "
+                          "new state and the output value.")
+        new_state, output_value = ret
+
+        flat_new_state = [
+            ops.convert_to_tensor(t) for t in nest.flatten(new_state)
+        ]
+        flat_output_value = [
+            ops.convert_to_tensor(t) for t in nest.flatten(output_value)
+        ]
+
+        # Extract shape information from the returned values.
+        flat_new_state_shapes.extend([t.shape for t in flat_new_state])
+        self._output_shapes = nest.pack_sequence_as(
+            output_value, [t.shape for t in flat_output_value])
+
+        # Extract and validate type information from the returned values.
+        for t, dtype in zip(flat_new_state, flat_state_types):
+          if t.dtype != dtype:
+            raise TypeError(
+                "The element types for the new state must match the initial "
+                "state. Expected %s; got %s." %
+                (self._state_types, nest.pack_sequence_as(
+                    self._state_types, [t.dtype for t in flat_new_state])))
+        self._output_types = nest.pack_sequence_as(
+            output_value, [t.dtype for t in flat_output_value])
+
+        return flat_new_state + flat_output_value
+
+      # Use the private method that will execute `tf_scan_func` but delay
+      # adding it to the graph in case we need to rerun the function.
+      tf_scan_func._create_definition_if_needed()  # pylint: disable=protected-access
+
+      weakened_state_shapes = [
+          original.most_specific_compatible_shape(new)
+          for original, new in zip(flat_state_shapes, flat_new_state_shapes)
+      ]
+
+      need_to_rerun = False
+      for original_shape, weakened_shape in zip(flat_state_shapes,
+                                                weakened_state_shapes):
+        if original_shape.ndims is not None and (
+            weakened_shape.ndims is None or
+            original_shape.as_list() != weakened_shape.as_list()):
+          need_to_rerun = True
+          break
+
+      if need_to_rerun:
+        # NOTE(mrry): `self._output_shapes` will be overwritten when we rerun
+        # `tf_scan_func`.
+        self._state_shapes = nest.pack_sequence_as(self._state_shapes,
+                                                   weakened_state_shapes)
+
+    self._scan_func = tf_scan_func
+
+  def _as_variant_tensor(self):
+    input_t = self._input_dataset._as_variant_tensor()  # pylint: disable=protected-access
+    return gen_dataset_ops.scan_dataset(
+        input_t,
+        nest.flatten(self._initial_state),
+        self._scan_func.captured_inputs,
+        f=self._scan_func,
+        output_types=nest.flatten(self.output_types),
+        output_shapes=nest.flatten(self.output_shapes))
+
+  @property
+  def output_shapes(self):
+    return self._output_shapes
+
+  @property
+  def output_types(self):
+    return self._output_types
+
+
+def scan(initial_state, scan_func):
+  """A transformation that scans a function across an input dataset.
+
+  This transformation is a stateful relative of @{tf.data.Dataset.map}.
+  In addition to mapping `scan_func` across the elements of the input dataset,
+  `scan()` accumulates one or more state tensors, whose initial values are
+  `initial_state`.
+
+  Args:
+    initial_state: A nested structure of tensors, representing the initial state
+      of the accumulator.
+    scan_func: A function that maps `(old_state, input_element)` to
+      `(new_state, output_element). It must take two arguments and return a
+      pair of nested structures of tensors. The `new_state` must match the
+      structure of `initial_state`.
+
+  Returns:
+    A `Dataset` transformation function, which can be passed to
+    @{tf.contrib.data.Dataset.apply}.
+  """
+  def _apply_fn(dataset):
+    return _ScanDataset(dataset, initial_state, scan_func)
+
+  return _apply_fn
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index bdc6faefbc..a3aa905415 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -5790,6 +5790,20 @@ tf_kernel_library(
     ],
 )
 
+tf_kernel_library(
+    name = "scan_dataset_op",
+    srcs = ["scan_dataset_op.cc"],
+    deps = [
+        ":captured_function",
+        ":dataset",
+        "//tensorflow/core:core_cpu_internal",
+        "//tensorflow/core:dataset_ops_op_lib",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:lib_internal",
+    ],
+)
+
 tf_kernel_library(
     name = "flat_map_dataset_op",
     srcs = ["flat_map_dataset_op.cc"],
@@ -6061,6 +6075,7 @@ tf_kernel_library(
         ":range_dataset_op",
         ":reader_dataset_ops",
         ":repeat_dataset_op",
+        ":scan_dataset_op",
         ":shuffle_dataset_op",
         ":skip_dataset_op",
         ":sloppy_interleave_dataset_op",
diff --git a/tensorflow/core/kernels/scan_dataset_op.cc b/tensorflow/core/kernels/scan_dataset_op.cc
new file mode 100644
index 0000000000..76c219f1ae
--- /dev/null
+++ b/tensorflow/core/kernels/scan_dataset_op.cc
@@ -0,0 +1,213 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include <iterator>
+#include <vector>
+
+#include "tensorflow/core/common_runtime/function.h"
+#include "tensorflow/core/framework/partial_tensor_shape.h"
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/kernels/captured_function.h"
+#include "tensorflow/core/kernels/dataset.h"
+#include "tensorflow/core/lib/random/random.h"
+
+namespace tensorflow {
+
+namespace {
+
+// See documentation in ../ops/dataset_ops.cc for a high-level
+// description of the following op.
+
+class ScanDatasetOp : public UnaryDatasetOpKernel {
+ public:
+  explicit ScanDatasetOp(OpKernelConstruction* ctx)
+      : UnaryDatasetOpKernel(ctx),
+        graph_def_version_(ctx->graph_def_version()) {
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("f", &func_));
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("Tstate", &state_types_));
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("output_types", &output_types_));
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("output_shapes", &output_shapes_));
+  }
+
+  void MakeDataset(OpKernelContext* ctx, DatasetBase* input,
+                   DatasetBase** output) override {
+    OpInputList initial_state_inputs;
+    OP_REQUIRES_OK(ctx,
+                   ctx->input_list("initial_state", &initial_state_inputs));
+    std::vector<Tensor> initial_state;
+    initial_state.reserve(initial_state_inputs.size());
+    for (const Tensor& t : initial_state_inputs) {
+      initial_state.push_back(t);
+    }
+
+    OpInputList inputs;
+    OP_REQUIRES_OK(ctx, ctx->input_list("other_arguments", &inputs));
+    std::vector<Tensor> other_arguments;
+    other_arguments.reserve(inputs.size());
+    for (const Tensor& t : inputs) {
+      other_arguments.push_back(t);
+    }
+
+    std::unique_ptr<CapturedFunction> captured_func;
+    OP_REQUIRES_OK(ctx, CapturedFunction::Create(ctx, func_, graph_def_version_,
+                                                 std::move(other_arguments),
+                                                 &captured_func));
+
+    *output =
+        new Dataset(input, std::move(initial_state), std::move(captured_func),
+                    state_types_, output_types_, output_shapes_);
+  }
+
+ private:
+  class Dataset : public DatasetBase {
+   public:
+    Dataset(const DatasetBase* input, std::vector<Tensor> initial_state,
+            std::unique_ptr<CapturedFunction> captured_func,
+            const DataTypeVector& state_types,
+            const DataTypeVector& output_types,
+            const std::vector<PartialTensorShape>& output_shapes)
+        : input_(input),
+          initial_state_(std::move(initial_state)),
+          captured_func_(std::move(captured_func)),
+          state_types_(state_types),
+          output_types_(output_types),
+          output_shapes_(output_shapes) {
+      input_->Ref();
+    }
+
+    ~Dataset() override { input_->Unref(); }
+
+    std::unique_ptr<IteratorBase> MakeIterator(
+        const string& prefix) const override {
+      return std::unique_ptr<IteratorBase>(
+          new Iterator({this, strings::StrCat(prefix, "::Scan")}));
+    }
+
+    const DataTypeVector& output_dtypes() const override {
+      return output_types_;
+    }
+    const std::vector<PartialTensorShape>& output_shapes() const override {
+      return output_shapes_;
+    }
+
+    string DebugString() override { return "ScanDatasetOp::Dataset"; }
+
+   private:
+    class Iterator : public DatasetIterator<Dataset> {
+     public:
+      explicit Iterator(const Params& params)
+          : DatasetIterator<Dataset>(params),
+            input_impl_(params.dataset->input_->MakeIterator(params.prefix)),
+            state_(params.dataset->initial_state_) {}
+
+      Status GetNextInternal(IteratorContext* ctx,
+                             std::vector<Tensor>* out_tensors,
+                             bool* end_of_sequence) override {
+        mutex_lock l(mu_);
+
+        std::vector<Tensor> next_element;
+        TF_RETURN_IF_ERROR(
+            input_impl_->GetNext(ctx, &next_element, end_of_sequence));
+        if (*end_of_sequence) {
+          return Status::OK();
+        }
+
+        std::vector<Tensor> args;
+        args.reserve(state_.size() + next_element.size());
+        std::copy(state_.begin(), state_.end(), std::back_inserter(args));
+        std::copy(next_element.begin(), next_element.end(),
+                  std::back_inserter(args));
+
+        FunctionLibraryRuntime::Options opts;
+        opts.step_id = CapturedFunction::generate_step_id();
+        ScopedStepContainer step_container(
+            opts.step_id, [this, ctx](const string& name) {
+              dataset()
+                  ->captured_func_->resource_manager()
+                  ->Cleanup(name)
+                  .IgnoreError();
+            });
+        opts.step_container = &step_container;
+        opts.runner = ctx->runner();
+        std::vector<Tensor> state_and_output;
+        state_and_output.reserve(dataset()->state_types_.size() +
+                                 output_dtypes().size());
+        Status s =
+            dataset()->captured_func_->Run(opts, args, &state_and_output);
+        if (s.ok()) {
+          state_.clear();
+          size_t i = 0;
+          for (; i < dataset()->state_types_.size(); ++i) {
+            if (state_and_output[i].dtype() != dataset()->state_types_[i]) {
+              return errors::InvalidArgument(
+                  "Got wrong type for scan_func return value ", i,
+                  " (expected ", DataTypeString(dataset()->state_types_[i]),
+                  ", got ", DataTypeString(state_and_output[i].dtype()), ").");
+            }
+            state_.push_back(std::move(state_and_output[i]));
+          }
+          for (; i < state_and_output.size(); ++i) {
+            const size_t output_index = i - dataset()->state_types_.size();
+            if (state_and_output[i].dtype() != output_dtypes()[output_index]) {
+              return errors::InvalidArgument(
+                  "Got wrong type for scan_func return value ", i,
+                  " (expected ",
+                  DataTypeString(dataset()->state_types_[output_index]),
+                  ", got ", DataTypeString(state_and_output[i].dtype()), ").");
+            }
+            if (!output_shapes()[output_index].IsCompatibleWith(
+                    state_and_output[i].shape())) {
+              return errors::InvalidArgument(
+                  "Got wrong shape for scan_func return value ", i,
+                  " (expected ", output_shapes()[output_index].DebugString(),
+                  ", got ", state_and_output[i].shape().DebugString(), ").");
+            }
+
+            out_tensors->push_back(std::move(state_and_output[i]));
+          }
+        } else if (errors::IsOutOfRange(s)) {
+          // `f` may deliberately raise `errors::OutOfRange` to indicate
+          // that we should terminate the iteration early.
+          *end_of_sequence = true;
+          return Status::OK();
+        }
+        return s;
+      }
+
+     private:
+      mutex mu_;
+      const std::unique_ptr<IteratorBase> input_impl_ GUARDED_BY(mu_);
+      std::vector<Tensor> state_ GUARDED_BY(mu_);
+    };
+
+    const DatasetBase* const input_;
+    const std::vector<Tensor> initial_state_;
+    const std::unique_ptr<CapturedFunction> captured_func_;
+    const DataTypeVector state_types_;
+    const DataTypeVector output_types_;
+    const std::vector<PartialTensorShape> output_shapes_;
+  };
+
+  const int graph_def_version_;
+  DataTypeVector state_types_;
+  DataTypeVector output_types_;
+  std::vector<PartialTensorShape> output_shapes_;
+  NameAttrList func_;
+};
+
+REGISTER_KERNEL_BUILDER(Name("ScanDataset").Device(DEVICE_CPU), ScanDatasetOp);
+
+}  // namespace
+
+}  // namespace tensorflow
diff --git a/tensorflow/core/ops/dataset_ops.cc b/tensorflow/core/ops/dataset_ops.cc
index c0e84c8bb0..ac15a3f71b 100644
--- a/tensorflow/core/ops/dataset_ops.cc
+++ b/tensorflow/core/ops/dataset_ops.cc
@@ -198,6 +198,21 @@ buffer_size: The maximum number of elements to buffer in an iterator over
   this dataset.
 )doc");
 
+REGISTER_OP("ScanDataset")
+    .Input("input_dataset: variant")
+    .Input("initial_state: Tstate")
+    .Input("other_arguments: Targuments")
+    .Output("handle: variant")
+    .Attr("f: func")
+    .Attr("Tstate: list(type) >= 1")
+    .Attr("Targuments: list(type) >= 0")
+    .Attr("output_types: list(type) >= 1")
+    .Attr("output_shapes: list(shape) >= 1")
+    .SetShapeFn(shape_inference::ScalarShape)
+    .Doc(R"doc(
+Creates a dataset successively reduces `f` over the elements of `input_dataset`.
+)doc");
+
 REGISTER_OP("FlatMapDataset")
     .Input("input_dataset: variant")
     .Input("other_arguments: Targuments")
-- 
GitLab


From 4878a28ac3e5b63cd820c9aa13cb0c4f0025ec23 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 9 Oct 2017 09:20:06 -0700
Subject: [PATCH 178/909] Update ops-related pbtxt files.

PiperOrigin-RevId: 171543801
---
 .../core/ops/compat/ops_history.v1.pbtxt      | 46 ++++++++++++++++++
 tensorflow/core/ops/ops.pbtxt                 | 47 +++++++++++++++++++
 2 files changed, 93 insertions(+)

diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index f8667177cc..a449fc1452 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -24743,6 +24743,52 @@ op {
     }
   }
 }
+op {
+  name: "ScanDataset"
+  input_arg {
+    name: "input_dataset"
+    type: DT_VARIANT
+  }
+  input_arg {
+    name: "initial_state"
+    type_list_attr: "Tstate"
+  }
+  input_arg {
+    name: "other_arguments"
+    type_list_attr: "Targuments"
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "f"
+    type: "func"
+  }
+  attr {
+    name: "Tstate"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "Targuments"
+    type: "list(type)"
+    has_minimum: true
+  }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
+  }
+}
 op {
   name: "ScatterAdd"
   input_arg {
diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index 9abb4f7a5e..88e57ea0cb 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -23407,6 +23407,53 @@ op {
   summary: "Outputs a `Summary` protocol buffer with scalar values."
   description: "The input `tags` and `values` must have the same shape.  The generated summary\nhas a summary value for each tag-value pair in `tags` and `values`."
 }
+op {
+  name: "ScanDataset"
+  input_arg {
+    name: "input_dataset"
+    type: DT_VARIANT
+  }
+  input_arg {
+    name: "initial_state"
+    type_list_attr: "Tstate"
+  }
+  input_arg {
+    name: "other_arguments"
+    type_list_attr: "Targuments"
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "f"
+    type: "func"
+  }
+  attr {
+    name: "Tstate"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "Targuments"
+    type: "list(type)"
+    has_minimum: true
+  }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
+  }
+  summary: "Creates a dataset successively reduces `f` over the elements of `input_dataset`."
+}
 op {
   name: "ScatterAdd"
   input_arg {
-- 
GitLab


From 022b25cd72af3127180728baf20351630a294609 Mon Sep 17 00:00:00 2001
From: Sylvus <Sylvus@users.noreply.github.com>
Date: Mon, 9 Oct 2017 17:48:14 +0100
Subject: [PATCH 179/909] Fix for the IOU metric (#12709)

* Fixed mean iou case when a class does not appear in the labels nor in the prediction.

* Added 3 tests for both mean_iou and streaming_mean_iou, 2 of which would fail with the previous code and one to make sure the behavior is still correct in the normal case. Fixed broken tests as well.

* Added check for div by 0 in iou metric.

* Add space around operator

As per style guide.
---
 .../metrics/python/ops/metric_ops_test.py     | 54 ++++++++++++++++++-
 .../python/kernel_tests/metrics_test.py       | 51 +++++++++++++++++-
 tensorflow/python/ops/metrics_impl.py         | 14 ++++-
 3 files changed, 116 insertions(+), 3 deletions(-)

diff --git a/tensorflow/contrib/metrics/python/ops/metric_ops_test.py b/tensorflow/contrib/metrics/python/ops/metric_ops_test.py
index 9b959b43a9..0f7f83f764 100644
--- a/tensorflow/contrib/metrics/python/ops/metric_ops_test.py
+++ b/tensorflow/contrib/metrics/python/ops/metric_ops_test.py
@@ -4978,7 +4978,7 @@ class StreamingMeanIOUTest(test.TestCase):
       sess.run(variables.local_variables_initializer())
       for _ in range(5):
         sess.run(update_op)
-      desired_output = np.mean([1.0 / 3.0, 2.0 / 4.0, 0.])
+      desired_output = np.mean([1.0 / 3.0, 2.0 / 4.0])
       self.assertAlmostEqual(desired_output, miou.eval())
 
   def testUpdateOpEvalIsAccumulatedConfusionMatrix(self):
@@ -5060,6 +5060,58 @@ class StreamingMeanIOUTest(test.TestCase):
       desired_miou = np.mean([2. / 4., 4. / 6.])
       self.assertAlmostEqual(desired_miou, miou.eval())
 
+  def testMissingClassInLabels(self):
+    labels = constant_op.constant([
+      [[0, 0, 1, 1, 0, 0],
+       [1, 0, 0, 0, 0, 1]],
+      [[1, 1, 1, 1, 1, 1],
+       [0, 0, 0, 0, 0, 0]]])
+    predictions = constant_op.constant([
+      [[0, 0, 2, 1, 1, 0],
+       [0, 1, 2, 2, 0, 1]],
+      [[0, 0, 2, 1, 1, 1],
+       [1, 1, 2, 0, 0, 0]]])
+    num_classes = 3
+    with self.test_session() as sess:
+      miou, update_op = metrics.streaming_mean_iou(
+          predictions, labels, num_classes)
+      sess.run(variables.local_variables_initializer())
+      self.assertAllEqual([[7, 4, 3], [3, 5, 2], [0, 0, 0]], update_op.eval())
+      self.assertAlmostEqual(
+        1 / 3 * (7 / (7 + 3 + 7) + 5 / (5 + 4 + 5) + 0 / (0 + 5 + 0)),
+        miou.eval())
+
+  def testMissingClassOverallSmall(self):
+    labels = constant_op.constant([0])
+    predictions = constant_op.constant([0])
+    num_classes = 2
+    with self.test_session() as sess:
+      miou, update_op = metrics.streaming_mean_iou(
+          predictions, labels, num_classes)
+      sess.run(variables.local_variables_initializer())
+      self.assertAllEqual([[1, 0], [0, 0]], update_op.eval())
+      self.assertAlmostEqual(1, miou.eval())
+
+  def testMissingClassOverallLarge(self):
+    labels = constant_op.constant([
+      [[0, 0, 1, 1, 0, 0],
+       [1, 0, 0, 0, 0, 1]],
+      [[1, 1, 1, 1, 1, 1],
+       [0, 0, 0, 0, 0, 0]]])
+    predictions = constant_op.constant([
+      [[0, 0, 1, 1, 0, 0],
+       [1, 1, 0, 0, 1, 1]],
+      [[0, 0, 0, 1, 1, 1],
+       [1, 1, 1, 0, 0, 0]]])
+    num_classes = 3
+    with self.test_session() as sess:
+      miou, update_op = metrics.streaming_mean_iou(
+          predictions, labels, num_classes)
+      sess.run(variables.local_variables_initializer())
+      self.assertAllEqual([[9, 5, 0], [3, 7, 0], [0, 0, 0]], update_op.eval())
+      self.assertAlmostEqual(
+        1 / 2 * (9 / (9 + 3 + 5) + 7 / (7 + 5 + 3)), miou.eval())
+
 
 class StreamingConcatTest(test.TestCase):
 
diff --git a/tensorflow/python/kernel_tests/metrics_test.py b/tensorflow/python/kernel_tests/metrics_test.py
index 2472b2a2a6..804346e6e7 100644
--- a/tensorflow/python/kernel_tests/metrics_test.py
+++ b/tensorflow/python/kernel_tests/metrics_test.py
@@ -3331,7 +3331,7 @@ class MeanIOUTest(test.TestCase):
       sess.run(variables.local_variables_initializer())
       for _ in range(5):
         sess.run(update_op)
-      desired_output = np.mean([1.0 / 3.0, 2.0 / 4.0, 0.])
+      desired_output = np.mean([1.0 / 3.0, 2.0 / 4.0])
       self.assertAlmostEqual(desired_output, miou.eval())
 
   def testUpdateOpEvalIsAccumulatedConfusionMatrix(self):
@@ -3410,6 +3410,55 @@ class MeanIOUTest(test.TestCase):
       desired_miou = np.mean([2. / 4., 4. / 6.])
       self.assertAlmostEqual(desired_miou, miou.eval())
 
+  def testMissingClassInLabels(self):
+    labels = constant_op.constant([
+      [[0, 0, 1, 1, 0, 0],
+       [1, 0, 0, 0, 0, 1]],
+      [[1, 1, 1, 1, 1, 1],
+       [0, 0, 0, 0, 0, 0]]])
+    predictions = constant_op.constant([
+      [[0, 0, 2, 1, 1, 0],
+       [0, 1, 2, 2, 0, 1]],
+      [[0, 0, 2, 1, 1, 1],
+       [1, 1, 2, 0, 0, 0]]])
+    num_classes = 3
+    with self.test_session() as sess:
+      miou, update_op = metrics.mean_iou(labels, predictions, num_classes)
+      sess.run(variables.local_variables_initializer())
+      self.assertAllEqual([[7, 4, 3], [3, 5, 2], [0, 0, 0]], update_op.eval())
+      self.assertAlmostEqual(
+        1 / 3 * (7 / (7 + 3 + 7) + 5 / (5 + 4 + 5) + 0 / (0 + 5 + 0)),
+        miou.eval())
+
+  def testMissingClassOverallSmall(self):
+    labels = constant_op.constant([0])
+    predictions = constant_op.constant([0])
+    num_classes = 2
+    with self.test_session() as sess:
+      miou, update_op = metrics.mean_iou(labels, predictions, num_classes)
+      sess.run(variables.local_variables_initializer())
+      self.assertAllEqual([[1, 0], [0, 0]], update_op.eval())
+      self.assertAlmostEqual(1, miou.eval())
+
+  def testMissingClassOverallLarge(self):
+    labels = constant_op.constant([
+      [[0, 0, 1, 1, 0, 0],
+       [1, 0, 0, 0, 0, 1]],
+      [[1, 1, 1, 1, 1, 1],
+       [0, 0, 0, 0, 0, 0]]])
+    predictions = constant_op.constant([
+      [[0, 0, 1, 1, 0, 0],
+       [1, 1, 0, 0, 1, 1]],
+      [[0, 0, 0, 1, 1, 1],
+       [1, 1, 1, 0, 0, 0]]])
+    num_classes = 3
+    with self.test_session() as sess:
+      miou, update_op = metrics.mean_iou(labels, predictions, num_classes)
+      sess.run(variables.local_variables_initializer())
+      self.assertAllEqual([[9, 5, 0], [3, 7, 0], [0, 0, 0]], update_op.eval())
+      self.assertAlmostEqual(
+        1 / 2 * (9 / (9 + 3 + 5) + 7 / (7 + 5 + 3)), miou.eval())
+
 
 class MeanPerClassAccuracyTest(test.TestCase):
 
diff --git a/tensorflow/python/ops/metrics_impl.py b/tensorflow/python/ops/metrics_impl.py
index 4c3ebb3aae..c40273b047 100644
--- a/tensorflow/python/ops/metrics_impl.py
+++ b/tensorflow/python/ops/metrics_impl.py
@@ -949,6 +949,12 @@ def mean_iou(labels,
       cm_diag = math_ops.to_float(array_ops.diag_part(total_cm))
       denominator = sum_over_row + sum_over_col - cm_diag
 
+      # The mean is only computed over classes that appear in the
+      # label or prediction tensor. If the denominator is 0, we need to
+      # ignore the class.
+      num_valid_entries = math_ops.reduce_sum(math_ops.cast(
+          math_ops.not_equal(denominator, 0), dtype=dtypes.float32))
+
       # If the value of the denominator is 0, set it to 1 to avoid
       # zero division.
       denominator = array_ops.where(
@@ -956,7 +962,13 @@ def mean_iou(labels,
           denominator,
           array_ops.ones_like(denominator))
       iou = math_ops.div(cm_diag, denominator)
-      return math_ops.reduce_mean(iou, name=name)
+
+      # If the number of valid entries is 0 (no classes) we return 0.
+      result = array_ops.where(
+          math_ops.greater(num_valid_entries, 0),
+          math_ops.reduce_sum(iou, name=name) / num_valid_entries,
+          0)
+      return result
 
     mean_iou_v = compute_mean_iou('mean_iou')
 
-- 
GitLab


From 7e2b50d8490f573b470ca97bd06a4677830db738 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 9 Oct 2017 09:45:16 -0700
Subject: [PATCH 180/909] Update docs of MomentumOptimizer about use_nesterov
 and of RMSProp about momentum

PiperOrigin-RevId: 171546603
---
 tensorflow/python/training/momentum.py | 5 ++++-
 tensorflow/python/training/rmsprop.py  | 2 ++
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/training/momentum.py b/tensorflow/python/training/momentum.py
index f34ff22f07..7c00e219fd 100644
--- a/tensorflow/python/training/momentum.py
+++ b/tensorflow/python/training/momentum.py
@@ -53,7 +53,10 @@ class MomentumOptimizer(optimizer.Optimizer):
         gradients.  Defaults to "Momentum".
       use_nesterov: If `True` use Nesterov Momentum.
         See [Sutskever et al., 2013](
-        http://jmlr.org/proceedings/papers/v28/sutskever13.pdf)
+        http://jmlr.org/proceedings/papers/v28/sutskever13.pdf).
+        This implementation always computes gradients at the value of the
+        variable(s) passed to the optimizer. Using Nesterov Momentum makes the
+        variable(s) track the values called `theta_t + mu*v_t` in the paper.
 
     """
     super(MomentumOptimizer, self).__init__(use_locking, name)
diff --git a/tensorflow/python/training/rmsprop.py b/tensorflow/python/training/rmsprop.py
index d046456c85..ebec725b7b 100644
--- a/tensorflow/python/training/rmsprop.py
+++ b/tensorflow/python/training/rmsprop.py
@@ -26,6 +26,8 @@ mean_square = decay * mean_square{t-1} + (1-decay) * gradient ** 2
 mom = momentum * mom{t-1} + learning_rate * g_t / sqrt(mean_square + epsilon)
 delta = - mom
 
+This implementation of RMSProp uses plain momentum, not Nesterov momentum.
+
 The centered version additionally maintains a moving (discounted) average of the
 gradients, and uses that average to estimate the variance:
 
-- 
GitLab


From 5bba158bbeea684c3e87de28a61004dbef28e00d Mon Sep 17 00:00:00 2001
From: Akshay Agrawal <akshayka@google.com>
Date: Mon, 9 Oct 2017 10:07:05 -0700
Subject: [PATCH 181/909] Print numpy value for variables when in Eager mode

PiperOrigin-RevId: 171549468
---
 tensorflow/python/framework/ops.py            | 24 ++++++++++---------
 .../python/kernel_tests/variables_test.py     |  2 +-
 tensorflow/python/ops/variables.py            | 10 +++++---
 3 files changed, 21 insertions(+), 15 deletions(-)

diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index e6e6b9c6ca..0257f094d7 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -174,6 +174,17 @@ def uid():
   return c_api.TFE_Py_UID()
 
 
+def numpy_text(tensor, is_repr=False):
+  """Human readable representation of a tensor's numpy value."""
+  if tensor.dtype.is_numpy_compatible:
+    text = repr(tensor.numpy()) if is_repr else str(tensor.numpy())
+  else:
+    text = "<unprintable>"
+  if "\n" in text:
+    text = "\n" + text
+  return text
+
+
 # NOTE(ebrevdo): Do not subclass this.  If you do, I will break you on purpose.
 class _TensorLike(object):
   """Internal cls for grouping Tensor, SparseTensor, ..., for is_instance."""
@@ -590,15 +601,6 @@ class _EagerTensorBase(Tensor):
     # performance-sensitive in some models.
     return dtypes._INTERN_TABLE[self._datatype_enum()]  # pylint: disable=protected-access
 
-  def _numpy_text(self, is_repr=False):
-    if self.dtype.is_numpy_compatible:
-      numpy_text = repr(self.numpy()) if is_repr else str(self.numpy())
-    else:
-      numpy_text = "<unprintable>"
-    if "\n" in numpy_text:
-      numpy_text = "\n" + numpy_text
-    return numpy_text
-
   def numpy(self):
     """Returns a numpy array with the same contents as the Tensor.
 
@@ -640,13 +642,13 @@ class _EagerTensorBase(Tensor):
     raise NotImplementedError()
 
   def __str__(self):
-    return "tf.Tensor(%s, shape=%s, dtype=%s)" % (self._numpy_text(),
+    return "tf.Tensor(%s, shape=%s, dtype=%s)" % (numpy_text(self),
                                                   self.shape,
                                                   self.dtype.name)
 
   def __repr__(self):
     return "<tf.Tensor: id=%s, shape=%s, dtype=%s, numpy=%s>" % (
-        self._id, self.shape, self.dtype.name, self._numpy_text(is_repr=True))
+        self._id, self.shape, self.dtype.name, numpy_text(self, is_repr=True))
 
   @staticmethod
   def _override_operator(name, func):
diff --git a/tensorflow/python/kernel_tests/variables_test.py b/tensorflow/python/kernel_tests/variables_test.py
index 7718710c69..f60ebf58f6 100644
--- a/tensorflow/python/kernel_tests/variables_test.py
+++ b/tensorflow/python/kernel_tests/variables_test.py
@@ -504,7 +504,7 @@ class VariablesTestCase(test.TestCase):
       self.assertAllClose(np.ones((5, 5), np.float32), var.eval())
 
   def testRepr(self):
-    var = variables.Variable(np.zeros((5, 5), np.float32), name='noop')
+    var = variables.Variable(np.zeros((5, 5), np.float32), name="noop")
     self.assertEqual(
         "<tf.Variable 'noop:0' shape=(5, 5) dtype=float32_ref>",
         repr(var))
diff --git a/tensorflow/python/ops/variables.py b/tensorflow/python/ops/variables.py
index a27f26e303..90b4f25d81 100644
--- a/tensorflow/python/ops/variables.py
+++ b/tensorflow/python/ops/variables.py
@@ -213,9 +213,13 @@ class Variable(object):
           constraint=constraint)
 
   def __repr__(self):
-    return "<tf.Variable '%s' shape=%s dtype=%s>" % (self.name,
-                                                     self.get_shape(),
-                                                     self.dtype.name)
+    if context.in_eager_mode():
+      return "<tf.Variable '%s' shape=%s dtype=%s, numpy=%s>" % (
+          self.name, self.get_shape(), self.dtype.name,
+          ops.numpy_text(self.read_value(), is_repr=True))
+    else:
+      return "<tf.Variable '%s' shape=%s dtype=%s>" % (
+          self.name, self.get_shape(), self.dtype.name)
 
   def _init_from_args(self,
                       initial_value=None,
-- 
GitLab


From ff8019199722f516968ba2867c7f090dc73a734f Mon Sep 17 00:00:00 2001
From: Jianwei Xie <xiejw@google.com>
Date: Mon, 9 Oct 2017 10:27:18 -0700
Subject: [PATCH 182/909] Estimator.predict should not generate warning if user
 uses TF dataset.

PiperOrigin-RevId: 171552443
---
 tensorflow/python/estimator/BUILD             |  1 +
 tensorflow/python/estimator/estimator.py      | 27 +++++++++--
 tensorflow/python/estimator/estimator_test.py | 46 ++++++++++++++++++-
 3 files changed, 68 insertions(+), 6 deletions(-)

diff --git a/tensorflow/python/estimator/BUILD b/tensorflow/python/estimator/BUILD
index 22de474013..2040d45cb6 100644
--- a/tensorflow/python/estimator/BUILD
+++ b/tensorflow/python/estimator/BUILD
@@ -406,6 +406,7 @@ py_test(
         "//tensorflow/python:training",
         "//tensorflow/python:util",
         "//tensorflow/python:variables",
+        "//tensorflow/python/data",
         "//tensorflow/python/ops/losses",
         "//tensorflow/python/saved_model:loader",
         "//tensorflow/python/saved_model:tag_constants",
diff --git a/tensorflow/python/estimator/estimator.py b/tensorflow/python/estimator/estimator.py
index 1197366256..4dfc53aadf 100644
--- a/tensorflow/python/estimator/estimator.py
+++ b/tensorflow/python/estimator/estimator.py
@@ -51,6 +51,7 @@ from tensorflow.python.training import saver
 from tensorflow.python.training import training
 from tensorflow.python.training import training_util
 from tensorflow.python.util import compat
+from tensorflow.python.util import nest
 from tensorflow.python.util import tf_inspect
 
 
@@ -564,13 +565,16 @@ class Estimator(object):
       return export_dir
 
   def _get_features_from_input_fn(self, input_fn, mode):
+    """Extracts the `features` from return values of `input_fn`."""
     result = self._call_input_fn(input_fn, mode)
-    if not ops.get_default_graph().get_collection(ops.GraphKeys.QUEUE_RUNNERS):
-      logging.warning('Input graph does not contain a QueueRunner. '
-                      'That means predict yields forever. '
-                      'This is probably a mistake.')
     if isinstance(result, (list, tuple)):
-      return result[0]
+      # Unconditionally drop the label (the second element of result).
+      result = result[0]
+
+    if not _has_dataset_or_queue_runner(result):
+      logging.warning('Input graph does not use tf.data.Dataset or contain a '
+                      'QueueRunner. That means predict yields forever. '
+                      'This is probably a mistake.')
     return result
 
   def _get_features_and_labels_from_input_fn(self, input_fn, mode):
@@ -1005,3 +1009,16 @@ def _write_dict_to_summary(output_dir,
           key)
   summary_writer.add_summary(summary_proto, current_global_step)
   summary_writer.flush()
+
+
+def _has_dataset_or_queue_runner(maybe_tensor):
+  """Returns True if TF dataset or QueueRunner has been used."""
+  # Check TF dataset first. Here, we use a simple algorithm to check the top
+  # level Tensors only, which should be sufficient for most users.
+  tensors = [x for x in nest.flatten(maybe_tensor) if isinstance(x, ops.Tensor)]
+  if any([t.op.type == 'IteratorGetNext' for t in tensors]):
+    return True
+
+  # Now, check queue.
+  return ops.get_default_graph().get_collection(ops.GraphKeys.QUEUE_RUNNERS)
+
diff --git a/tensorflow/python/estimator/estimator_test.py b/tensorflow/python/estimator/estimator_test.py
index cdffe3378f..0040ec3650 100644
--- a/tensorflow/python/estimator/estimator_test.py
+++ b/tensorflow/python/estimator/estimator_test.py
@@ -29,6 +29,7 @@ import six
 from google.protobuf import text_format
 
 from tensorflow.python.client import session
+from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.estimator import estimator
 from tensorflow.python.estimator import model_fn as model_fn_lib
 from tensorflow.python.estimator import run_config
@@ -1212,7 +1213,50 @@ class EstimatorPredictTest(test.TestCase):
       next(est.predict(dummy_input_fn))
       self.assertRegexpMatches(
           str(mock_log.call_args),
-          'Input graph does not contain a QueueRunner.')
+          'Input graph does not.*contain a QueueRunner.')
+
+  def test_skip_warn_if_dataset_returns_features(self):
+
+    def _model_fn(features, labels, mode):
+      _, _ = features, labels
+      return model_fn_lib.EstimatorSpec(
+          mode,
+          loss=constant_op.constant(0.),
+          train_op=state_ops.assign_add(training.get_global_step(), 1),
+          predictions=constant_op.constant([[10.]]))
+
+    def _input_fn():
+      it = dataset_ops.Dataset.from_tensors([1]).make_one_shot_iterator()
+      return it.get_next()
+
+    est = estimator.Estimator(model_fn=_model_fn)
+    est.train(dummy_input_fn, steps=1)
+    with test.mock.patch.object(logging, 'warning') as mock_log:
+      next(est.predict(_input_fn))
+      # The warning should not have keyword QueueRunner.
+      self.assertRegexpMatches(str(mock_log.call_args), '^((?!QueueRunner).)*$')
+
+  def test_skip_warn_if_dataset_returns_features_dict(self):
+
+    def _model_fn(features, labels, mode):
+      _, _ = features, labels
+      return model_fn_lib.EstimatorSpec(
+          mode,
+          loss=constant_op.constant(0.),
+          train_op=state_ops.assign_add(training.get_global_step(), 1),
+          predictions=constant_op.constant([[10.]]))
+
+    def _input_fn():
+      it = dataset_ops.Dataset.from_tensors([1]).make_one_shot_iterator()
+      features = {'age': it.get_next()}
+      return features
+
+    est = estimator.Estimator(model_fn=_model_fn)
+    est.train(dummy_input_fn, steps=1)
+    with test.mock.patch.object(logging, 'warning') as mock_log:
+      next(est.predict(_input_fn))
+      # The warning should not have keyword QueueRunner.
+      self.assertRegexpMatches(str(mock_log.call_args), '^((?!QueueRunner).)*$')
 
   def test_input_fn_can_return_just_features(self):
 
-- 
GitLab


From 9ff05e9e7f471a8487cdd8a7bb6fdd554055e2dd Mon Sep 17 00:00:00 2001
From: Amit Patankar <amitpatankar@google.com>
Date: Mon, 9 Oct 2017 10:48:57 -0700
Subject: [PATCH 183/909] Fixing the name of the disabled test. (#13593)

---
 tensorflow/contrib/cmake/tf_tests.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/cmake/tf_tests.cmake b/tensorflow/contrib/cmake/tf_tests.cmake
index 4cf22a9c47..0e61cd6539 100644
--- a/tensorflow/contrib/cmake/tf_tests.cmake
+++ b/tensorflow/contrib/cmake/tf_tests.cmake
@@ -229,7 +229,7 @@ if (tensorflow_BUILD_PYTHON_TESTS)
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/cholesky_op_test.py"
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/diag_op_test.py"
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/linalg_ops_test.py"
-      "${tensorflow_source_dir}/tensorflow/python/ops/init_ops.py"
+      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/init_ops_test.py"
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/self_adjoint_eig_op_test.py"
       # misc
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/variable_scope_test.py"
-- 
GitLab


From 15dd5fd0b2e0b39d87b1cb873ae84225d86173db Mon Sep 17 00:00:00 2001
From: Yuefeng Zhou <yuefengz@google.com>
Date: Mon, 9 Oct 2017 11:00:55 -0700
Subject: [PATCH 184/909] Track persistent memory in constant op.

PiperOrigin-RevId: 171557547
---
 tensorflow/core/kernels/BUILD               |  2 +-
 tensorflow/core/kernels/constant_op.cc      | 12 +++-
 tensorflow/core/kernels/constant_op_test.cc | 65 +++++++++++++++++++++
 3 files changed, 77 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index a3aa905415..ad6f84304d 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -900,7 +900,7 @@ tf_cc_test(
     ],
 )
 
-tf_cc_test(
+tf_cuda_cc_test(
     name = "constant_op_test",
     size = "small",
     srcs = ["constant_op_test.cc"],
diff --git a/tensorflow/core/kernels/constant_op.cc b/tensorflow/core/kernels/constant_op.cc
index 618d4f580b..018ace5485 100644
--- a/tensorflow/core/kernels/constant_op.cc
+++ b/tensorflow/core/kernels/constant_op.cc
@@ -54,7 +54,17 @@ ConstantOp::ConstantOp(OpKernelConstruction* ctx)
                               DataTypeString(ctx->output_type(0)), ")"));
 }
 
-void ConstantOp::Compute(OpKernelContext* ctx) { ctx->set_output(0, tensor_); }
+void ConstantOp::Compute(OpKernelContext* ctx) {
+  ctx->set_output(0, tensor_);
+  if (TF_PREDICT_FALSE(ctx->track_allocations())) {
+    AllocatorAttributes attr;
+    if (ctx->allocate_on_host(attr)) {
+      ctx->record_host_persistent_memory_allocation(tensor_.AllocatedBytes());
+    } else {
+      ctx->record_device_persistent_memory_allocation(tensor_.AllocatedBytes());
+    }
+  }
+}
 
 ConstantOp::~ConstantOp() {}
 
diff --git a/tensorflow/core/kernels/constant_op_test.cc b/tensorflow/core/kernels/constant_op_test.cc
index 2d44140b72..62cc67c736 100644
--- a/tensorflow/core/kernels/constant_op_test.cc
+++ b/tensorflow/core/kernels/constant_op_test.cc
@@ -14,17 +14,82 @@ limitations under the License.
 ==============================================================================*/
 
 #include "tensorflow/core/common_runtime/kernel_benchmark_testlib.h"
+#include "tensorflow/core/framework/node_def_builder.h"
 #include "tensorflow/core/framework/op.h"
+#include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/framework/types.pb.h"
 #include "tensorflow/core/graph/graph.h"
+#include "tensorflow/core/kernels/ops_testutil.h"
 #include "tensorflow/core/platform/test_benchmark.h"
 #include "tensorflow/core/platform/types.h"
 
 namespace tensorflow {
 
+class ConstantOpTest : public OpsTestBase {
+ protected:
+  void PersistentMemoryTrackingTest(bool on_gpu);
+};
+
+void ConstantOpTest::PersistentMemoryTrackingTest(bool on_gpu) {
+  DataType data_type = DT_INT32;
+  std::initializer_list<int64> dims = {2, 3, 4, 5};
+  Tensor tensor(data_type, TensorShape(dims));
+  for (int i = 0; i < 2 * 3 * 4 * 5; ++i) {
+    tensor.flat<int32>()(i) = i;
+  }
+
+  NodeDef const_node;
+  TF_ASSERT_OK(NodeDefBuilder("some_node", "Const")
+                   .Attr("dtype", data_type)
+                   .Attr("value", tensor)
+                   .Finalize(&const_node));
+
+  string device_string = "CPU";
+  DeviceType device_type = DEVICE_CPU;
+  if (on_gpu) {
+    device_string = "GPU";
+    DeviceType device_type = DEVICE_GPU;
+  }
+  std::unique_ptr<Device> device(DeviceFactory::NewDevice(
+      device_string, {}, "/job:worker/replica:0/task:0"));
+
+  Status status;
+  std::unique_ptr<OpKernel> op(CreateOpKernel(device_type, device.get(),
+                                              cpu_allocator(), const_node,
+                                              TF_GRAPH_DEF_VERSION, &status));
+
+  OpKernelContext::Params params;
+  params.device = device.get();
+  params.frame_iter = FrameAndIter(0, 0);
+  params.op_kernel = op.get();
+  params.track_allocations = true;
+
+  OpKernelContext ctx(&params);
+  op->Compute(&ctx);
+  TF_EXPECT_OK(ctx.status());
+
+  if (on_gpu) {
+    EXPECT_EQ(ctx.device_persistent_memory_allocated(), 512);
+  } else {
+    EXPECT_EQ(ctx.host_persistent_memory_allocated(), 480);
+  }
+
+  // Remove memry leak errors.
+  for (auto allocator_pair : ctx.wrapped_allocators()) {
+    allocator_pair.second->GetRecordsAndUnRef();
+  }
+}
+
+TEST_F(ConstantOpTest, PersistentMemoryTracking) {
+  PersistentMemoryTrackingTest(false);
+#if GOOGLE_CUDA
+  PersistentMemoryTrackingTest(true);
+#endif  // GOOGLE_CUDA
+}
+
 // Returns graph containing "num" const nodes.  If 'sequential' is
 // true, make sure all constants are executed sequentially in the
 // graph by adding control dependencies.
-- 
GitLab


From e56628b085ffa7922e5238537f6ebd6deee0f0cc Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 9 Oct 2017 11:30:51 -0700
Subject: [PATCH 185/909] [TF:XLA] Rename ComputationBuilder::LogicalX to X

PiperOrigin-RevId: 171562764
---
 .../compiler/tf2xla/kernels/binary_ops.cc     |  4 ++--
 .../compiler/tf2xla/kernels/random_ops.cc     |  2 +-
 .../compiler/tf2xla/kernels/reduction_ops.cc  |  4 ++--
 tensorflow/compiler/tf2xla/kernels/relu_op.cc |  6 ++---
 .../compiler/tf2xla/kernels/softmax_op.cc     |  2 +-
 .../compiler/tf2xla/kernels/unary_ops.cc      |  8 +++----
 .../xla/client/computation_builder.cc         |  6 ++---
 .../compiler/xla/client/computation_builder.h |  6 ++---
 .../compiler/xla/client/lib/arithmetic.cc     |  4 ++--
 .../xla/tests/array_elementwise_ops_test.cc   | 24 +++++++++----------
 .../xla/tests/broadcast_simple_test.cc        |  4 ++--
 .../xla/tests/scalar_computations_test.cc     | 14 +++++------
 tensorflow/compiler/xla/tests/while_test.cc   |  4 ++--
 13 files changed, 43 insertions(+), 45 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/kernels/binary_ops.cc b/tensorflow/compiler/tf2xla/kernels/binary_ops.cc
index 58538b4513..a180f1e4d9 100644
--- a/tensorflow/compiler/tf2xla/kernels/binary_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/binary_ops.cc
@@ -96,8 +96,8 @@ static xla::ComputationDataHandle FloorModImpl(xla::ComputationBuilder* b,
 XLA_MAKE_BINARY(FloorMod,
                 FloorModImpl(b, input_type(0), lhs, rhs, broadcast_helper));
 
-XLA_MAKE_BINARY(LogicalAnd, b->LogicalAnd(lhs, rhs, extend_dimensions));
-XLA_MAKE_BINARY(LogicalOr, b->LogicalOr(lhs, rhs, extend_dimensions));
+XLA_MAKE_BINARY(LogicalAnd, b->And(lhs, rhs, extend_dimensions));
+XLA_MAKE_BINARY(LogicalOr, b->Or(lhs, rhs, extend_dimensions));
 XLA_MAKE_BINARY(Mod, b->Rem(lhs, rhs, extend_dimensions));
 XLA_MAKE_BINARY(Maximum, b->Max(lhs, rhs, extend_dimensions));
 XLA_MAKE_BINARY(Minimum, b->Min(lhs, rhs, extend_dimensions));
diff --git a/tensorflow/compiler/tf2xla/kernels/random_ops.cc b/tensorflow/compiler/tf2xla/kernels/random_ops.cc
index 66b99665cb..2421825ead 100644
--- a/tensorflow/compiler/tf2xla/kernels/random_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/random_ops.cc
@@ -140,7 +140,7 @@ class TruncatedNormalOp : public XlaOpKernel {
                                       xla::ComputationBuilder* b) {
       xla::ComputationDataHandle too_large = b->Gt(candidate, two_sd(false, b));
       xla::ComputationDataHandle too_small = b->Lt(candidate, two_sd(true, b));
-      return b->LogicalOr(too_large, too_small);
+      return b->Or(too_large, too_small);
     };
 
     // The algorithm we're using is roughly:
diff --git a/tensorflow/compiler/tf2xla/kernels/reduction_ops.cc b/tensorflow/compiler/tf2xla/kernels/reduction_ops.cc
index dae2eb9d2a..647b627408 100644
--- a/tensorflow/compiler/tf2xla/kernels/reduction_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/reduction_ops.cc
@@ -129,7 +129,7 @@ class AllOp : public XlaReductionOp {
   void BuildReducer(xla::ComputationBuilder* builder,
                     const xla::ComputationDataHandle& scalar_lhs,
                     const xla::ComputationDataHandle& scalar_rhs) override {
-    builder->LogicalAnd(scalar_lhs, scalar_rhs);
+    builder->And(scalar_lhs, scalar_rhs);
   }
 };
 
@@ -147,7 +147,7 @@ class AnyOp : public XlaReductionOp {
   void BuildReducer(xla::ComputationBuilder* builder,
                     const xla::ComputationDataHandle& scalar_lhs,
                     const xla::ComputationDataHandle& scalar_rhs) override {
-    builder->LogicalOr(scalar_lhs, scalar_rhs);
+    builder->Or(scalar_lhs, scalar_rhs);
   }
 };
 
diff --git a/tensorflow/compiler/tf2xla/kernels/relu_op.cc b/tensorflow/compiler/tf2xla/kernels/relu_op.cc
index a137d28118..12a3552999 100644
--- a/tensorflow/compiler/tf2xla/kernels/relu_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/relu_op.cc
@@ -77,9 +77,9 @@ class Relu6GradOp : public XlaOpKernel {
         b->Broadcast(XlaHelpers::Zero(b, input_type(0)), shape.dim_sizes());
     const auto six = b->Broadcast(
         XlaHelpers::IntegerLiteral(b, input_type(0), 6), shape.dim_sizes());
-    auto out = b->Select(
-        b->LogicalAnd(b->Lt(ctx->Input(1), six), b->Gt(ctx->Input(1), zero)),
-        ctx->Input(0), zero);
+    auto out =
+        b->Select(b->And(b->Lt(ctx->Input(1), six), b->Gt(ctx->Input(1), zero)),
+                  ctx->Input(0), zero);
     ctx->SetOutput(0, out);
   }
 };
diff --git a/tensorflow/compiler/tf2xla/kernels/softmax_op.cc b/tensorflow/compiler/tf2xla/kernels/softmax_op.cc
index a0d8ab4d73..750a4c2dec 100644
--- a/tensorflow/compiler/tf2xla/kernels/softmax_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/softmax_op.cc
@@ -202,7 +202,7 @@ class SparseSoftmaxXentWithLogitsOp : public XlaOpKernel {
     // NaN otherwise; then add that vector to the labels to force out-of-range
     // values to NaNs.
     xla::ComputationDataHandle nan_or_zero = builder->Select(
-        builder->LogicalAnd(
+        builder->And(
             builder->Le(XlaHelpers::Zero(builder, indices_type), indices),
             builder->Lt(indices, XlaHelpers::IntegerLiteral(
                                      builder, indices_type, depth))),
diff --git a/tensorflow/compiler/tf2xla/kernels/unary_ops.cc b/tensorflow/compiler/tf2xla/kernels/unary_ops.cc
index 3e4a0f5950..8f04fc94be 100644
--- a/tensorflow/compiler/tf2xla/kernels/unary_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/unary_ops.cc
@@ -87,7 +87,7 @@ XLAJIT_MAKE_UNARY(Log, b->Log(x));
 // TODO(b/34703906): use a more accurate implementation of log1p.
 XLAJIT_MAKE_UNARY(Log1p, b->Log(b->Add(XlaHelpers::One(b, input_type(0)), x)));
 
-XLAJIT_MAKE_UNARY(LogicalNot, b->LogicalNot(x));
+XLAJIT_MAKE_UNARY(LogicalNot, b->Not(x));
 XLAJIT_MAKE_UNARY(Neg, b->Neg(x));
 
 // Implements Banker's rounding: numbers that are equidistant between two
@@ -104,9 +104,9 @@ static xla::ComputationDataHandle Round(xla::ComputationBuilder* b,
   auto nearest_even_int =
       b->Sub(round_val, b->Mul(two, b->Floor(b->Mul(half, x))));
   auto is_odd = b->Eq(nearest_even_int, one);
-  return b->Select(b->LogicalOr(b->Gt(fraction, half),
-                                b->LogicalAnd(b->Eq(fraction, half), is_odd)),
-                   b->Add(round_val, one), round_val);
+  return b->Select(
+      b->Or(b->Gt(fraction, half), b->And(b->Eq(fraction, half), is_odd)),
+      b->Add(round_val, one), round_val);
 }
 
 XLAJIT_MAKE_UNARY(Rint, Round(b, input_type(0), x));
diff --git a/tensorflow/compiler/xla/client/computation_builder.cc b/tensorflow/compiler/xla/client/computation_builder.cc
index 925dcd36c0..4757e8b0d2 100644
--- a/tensorflow/compiler/xla/client/computation_builder.cc
+++ b/tensorflow/compiler/xla/client/computation_builder.cc
@@ -955,19 +955,19 @@ ComputationDataHandle ComputationBuilder::Min(
   return BinaryOp(BINOP_MIN, lhs, rhs, broadcast_dimensions);
 }
 
-ComputationDataHandle ComputationBuilder::LogicalAnd(
+ComputationDataHandle ComputationBuilder::And(
     const ComputationDataHandle& lhs, const ComputationDataHandle& rhs,
     tensorflow::gtl::ArraySlice<int64> broadcast_dimensions) {
   return BinaryOp(BINOP_LOGICAL_AND, lhs, rhs, broadcast_dimensions);
 }
 
-ComputationDataHandle ComputationBuilder::LogicalOr(
+ComputationDataHandle ComputationBuilder::Or(
     const ComputationDataHandle& lhs, const ComputationDataHandle& rhs,
     tensorflow::gtl::ArraySlice<int64> broadcast_dimensions) {
   return BinaryOp(BINOP_LOGICAL_OR, lhs, rhs, broadcast_dimensions);
 }
 
-ComputationDataHandle ComputationBuilder::LogicalNot(
+ComputationDataHandle ComputationBuilder::Not(
     const ComputationDataHandle& operand) {
   return UnaryOp(UNOP_LOGICAL_NOT, operand);
 }
diff --git a/tensorflow/compiler/xla/client/computation_builder.h b/tensorflow/compiler/xla/client/computation_builder.h
index 7014685ea5..23769f0afc 100644
--- a/tensorflow/compiler/xla/client/computation_builder.h
+++ b/tensorflow/compiler/xla/client/computation_builder.h
@@ -461,15 +461,15 @@ class ComputationBuilder {
       tensorflow::gtl::ArraySlice<int64> broadcast_dimensions = {});
 
   // Element-wise logical operators
-  ComputationDataHandle LogicalAnd(
+  ComputationDataHandle And(
       const ComputationDataHandle& lhs, const ComputationDataHandle& rhs,
       tensorflow::gtl::ArraySlice<int64> broadcast_dimensions = {});
 
-  ComputationDataHandle LogicalOr(
+  ComputationDataHandle Or(
       const ComputationDataHandle& lhs, const ComputationDataHandle& rhs,
       tensorflow::gtl::ArraySlice<int64> broadcast_dimensions = {});
 
-  ComputationDataHandle LogicalNot(const ComputationDataHandle& lhs);
+  ComputationDataHandle Not(const ComputationDataHandle& operand);
 
   // Reduces an array among the provided dimensions, given "computation" as a
   // reduction operator.
diff --git a/tensorflow/compiler/xla/client/lib/arithmetic.cc b/tensorflow/compiler/xla/client/lib/arithmetic.cc
index 969b0eee1d..99e9f2dbb2 100644
--- a/tensorflow/compiler/xla/client/lib/arithmetic.cc
+++ b/tensorflow/compiler/xla/client/lib/arithmetic.cc
@@ -93,14 +93,14 @@ Computation CreateScalarLogicalAndComputation(ComputationBuilder* builder) {
   return CreateScalarComputation(
       "logical_and", PRED, builder,
       [](ComputationBuilder* b, const ComputationDataHandle& lhs,
-         const ComputationDataHandle& rhs) { return b->LogicalAnd(lhs, rhs); });
+         const ComputationDataHandle& rhs) { return b->And(lhs, rhs); });
 }
 
 Computation CreateScalarLogicalOrComputation(ComputationBuilder* builder) {
   return CreateScalarComputation(
       "logical_or", PRED, builder,
       [](ComputationBuilder* b, const ComputationDataHandle& lhs,
-         const ComputationDataHandle& rhs) { return b->LogicalOr(lhs, rhs); });
+         const ComputationDataHandle& rhs) { return b->Or(lhs, rhs); });
 }
 
 StatusOr<ComputationDataHandle> Any(const ComputationDataHandle& predicates,
diff --git a/tensorflow/compiler/xla/tests/array_elementwise_ops_test.cc b/tensorflow/compiler/xla/tests/array_elementwise_ops_test.cc
index 24bccf6863..08b39b6379 100644
--- a/tensorflow/compiler/xla/tests/array_elementwise_ops_test.cc
+++ b/tensorflow/compiler/xla/tests/array_elementwise_ops_test.cc
@@ -496,54 +496,54 @@ XLA_TEST_F(ArrayElementwiseOpTest, MulTwoConstantU32s) {
   ComputeAndCompareR1<uint32>(&builder, expected, {});
 }
 
-XLA_TEST_F(ArrayElementwiseOpTest, LogicalAnd) {
+XLA_TEST_F(ArrayElementwiseOpTest, BooleanAnd) {
   ComputationBuilder builder(client_, TestName());
   auto a = builder.ConstantR1<bool>({false, false, true, true});
   auto b = builder.ConstantR1<bool>({false, true, false, true});
-  auto out = builder.LogicalAnd(a, b);
+  auto out = builder.And(a, b);
 
   ComputeAndCompareR1<bool>(&builder, {false, false, false, true}, {});
 }
 
-XLA_TEST_F(ArrayElementwiseOpTest, LogicalAndZeroElement) {
+XLA_TEST_F(ArrayElementwiseOpTest, BooleanAndZeroElement) {
   ComputationBuilder builder(client_, TestName());
   auto a = builder.ConstantR1<bool>({});
   auto b = builder.ConstantR1<bool>({});
-  auto out = builder.LogicalAnd(a, b);
+  auto out = builder.And(a, b);
 
   ComputeAndCompareR1<bool>(&builder, {}, {});
 }
 
-XLA_TEST_F(ArrayElementwiseOpTest, LogicalOr) {
+XLA_TEST_F(ArrayElementwiseOpTest, BooleanOr) {
   ComputationBuilder builder(client_, TestName());
   auto a = builder.ConstantR1<bool>({false, false, true, true});
   auto b = builder.ConstantR1<bool>({false, true, false, true});
-  auto out = builder.LogicalOr(a, b);
+  auto out = builder.Or(a, b);
 
   ComputeAndCompareR1<bool>(&builder, {false, true, true, true}, {});
 }
 
-XLA_TEST_F(ArrayElementwiseOpTest, LogicalOrZeroElement) {
+XLA_TEST_F(ArrayElementwiseOpTest, BooleanOrZeroElement) {
   ComputationBuilder builder(client_, TestName());
   auto a = builder.ConstantR1<bool>({});
   auto b = builder.ConstantR1<bool>({});
-  auto out = builder.LogicalOr(a, b);
+  auto out = builder.Or(a, b);
 
   ComputeAndCompareR1<bool>(&builder, {}, {});
 }
 
-XLA_TEST_F(ArrayElementwiseOpTest, LogicalNot) {
+XLA_TEST_F(ArrayElementwiseOpTest, BooleanNot) {
   ComputationBuilder builder(client_, TestName());
   auto a = builder.ConstantR1<bool>({false, true, true, false});
-  auto out = builder.LogicalNot(a);
+  auto out = builder.Not(a);
 
   ComputeAndCompareR1<bool>(&builder, {true, false, false, true}, {});
 }
 
-XLA_TEST_F(ArrayElementwiseOpTest, LogicalNotZeroElement) {
+XLA_TEST_F(ArrayElementwiseOpTest, BooleanNotZeroElement) {
   ComputationBuilder builder(client_, TestName());
   auto a = builder.ConstantR1<bool>({});
-  auto out = builder.LogicalNot(a);
+  auto out = builder.Not(a);
 
   ComputeAndCompareR1<bool>(&builder, {}, {});
 }
diff --git a/tensorflow/compiler/xla/tests/broadcast_simple_test.cc b/tensorflow/compiler/xla/tests/broadcast_simple_test.cc
index 505fa059f2..03f5e08315 100644
--- a/tensorflow/compiler/xla/tests/broadcast_simple_test.cc
+++ b/tensorflow/compiler/xla/tests/broadcast_simple_test.cc
@@ -159,7 +159,7 @@ XLA_TEST_F(BroadcastSimpleTest, 1DTo2D) {
 }
 
 // Tests implicit broadcasting of PREDs.
-XLA_TEST_F(BroadcastSimpleTest, LogicalAnd2DTo3D_Pred) {
+XLA_TEST_F(BroadcastSimpleTest, BooleanAnd2DTo3D_Pred) {
   ComputationBuilder b(client_, TestName());
 
   Array2D<bool> x_vals(2, 1);
@@ -174,7 +174,7 @@ XLA_TEST_F(BroadcastSimpleTest, LogicalAnd2DTo3D_Pred) {
   ComputationDataHandle x, y;
   auto x_data = CreateR2Parameter<bool>(x_vals, 0, "x", &b, &x);
   auto y_data = CreateR3Parameter<bool>(y_vals, 1, "y", &b, &y);
-  b.LogicalAnd(x, y, /*broadcast_dimensions=*/{1, 2});
+  b.And(x, y, /*broadcast_dimensions=*/{1, 2});
 
   Array3D<bool> expected(2, 2, 1);
   expected(0, 0, 0) = false;
diff --git a/tensorflow/compiler/xla/tests/scalar_computations_test.cc b/tensorflow/compiler/xla/tests/scalar_computations_test.cc
index 77d1c019f3..da84d185ca 100644
--- a/tensorflow/compiler/xla/tests/scalar_computations_test.cc
+++ b/tensorflow/compiler/xla/tests/scalar_computations_test.cc
@@ -459,34 +459,32 @@ XLA_TEST_F(ScalarComputationsTest, RemTwoScalarsU32) {
   ComputeAndCompareR0<uint32>(&builder, 2, {});
 }
 
-XLA_TEST_F(ScalarComputationsTest, LogicalAnd) {
+XLA_TEST_F(ScalarComputationsTest, BooleanAnd) {
   for (bool x : {false, true}) {
     for (bool y : {false, true}) {
       ComputationBuilder builder(client_, TestName());
-      builder.LogicalAnd(builder.ConstantR0<bool>(x),
-                         builder.ConstantR0<bool>(y));
+      builder.And(builder.ConstantR0<bool>(x), builder.ConstantR0<bool>(y));
 
       ComputeAndCompareR0<bool>(&builder, x && y, {});
     }
   }
 }
 
-XLA_TEST_F(ScalarComputationsTest, LogicalOr) {
+XLA_TEST_F(ScalarComputationsTest, BooleanOr) {
   for (bool x : {false, true}) {
     for (bool y : {false, true}) {
       ComputationBuilder builder(client_, TestName());
-      builder.LogicalOr(builder.ConstantR0<bool>(x),
-                        builder.ConstantR0<bool>(y));
+      builder.Or(builder.ConstantR0<bool>(x), builder.ConstantR0<bool>(y));
 
       ComputeAndCompareR0<bool>(&builder, x || y, {});
     }
   }
 }
 
-XLA_TEST_F(ScalarComputationsTest, LogicalNot) {
+XLA_TEST_F(ScalarComputationsTest, BooleanNot) {
   for (bool x : {false, true}) {
     ComputationBuilder builder(client_, TestName());
-    builder.LogicalNot(builder.ConstantR0<bool>(x));
+    builder.Not(builder.ConstantR0<bool>(x));
 
     ComputeAndCompareR0<bool>(&builder, !x, {});
   }
diff --git a/tensorflow/compiler/xla/tests/while_test.cc b/tensorflow/compiler/xla/tests/while_test.cc
index bb2d90fa94..71a1b0abee 100644
--- a/tensorflow/compiler/xla/tests/while_test.cc
+++ b/tensorflow/compiler/xla/tests/while_test.cc
@@ -169,7 +169,7 @@ TEST_F(WhileTest, WhileWithPredicateResult) {
   {
     ComputationBuilder builder(client_, "body");
     auto prev = builder.Parameter(0, result_shape, "prev");
-    auto result = builder.LogicalOr(prev, builder.ConstantR0<bool>(true));
+    auto result = builder.Or(prev, builder.ConstantR0<bool>(true));
     body = builder.Build().ConsumeValueOrDie();
   }
 
@@ -437,7 +437,7 @@ TEST_F(WhileTest, WhileWithPredicateTupleResult) {
     auto prev = builder.Parameter(0, result_shape, "prev");
     auto iteration = builder.GetTupleElement(prev, 0);
     auto pred = builder.GetTupleElement(prev, 1);
-    auto new_pred = builder.LogicalOr(pred, builder.ConstantR0<bool>(true));
+    auto new_pred = builder.Or(pred, builder.ConstantR0<bool>(true));
     auto result = builder.Tuple(
         {builder.Add(iteration, builder.ConstantR0<int32>(1)), new_pred});
     body = builder.Build().ConsumeValueOrDie();
-- 
GitLab


From 4a97a8210ce31fe9a3081a3afacdf12f2feeefad Mon Sep 17 00:00:00 2001
From: Akshay Agrawal <akshayka@google.com>
Date: Mon, 9 Oct 2017 12:02:17 -0700
Subject: [PATCH 186/909] Validate input shapes for the graph_callable
 decorator

PiperOrigin-RevId: 171567580
---
 tensorflow/python/eager/graph_callable.py     | 30 +++++++++++++++----
 .../python/eager/graph_callable_test.py       | 14 +++++++++
 2 files changed, 38 insertions(+), 6 deletions(-)

diff --git a/tensorflow/python/eager/graph_callable.py b/tensorflow/python/eager/graph_callable.py
index e3aacbd140..a1bdba6e4e 100644
--- a/tensorflow/python/eager/graph_callable.py
+++ b/tensorflow/python/eager/graph_callable.py
@@ -19,7 +19,6 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import collections
 import contextlib
 
 from tensorflow.python.eager import context
@@ -241,15 +240,27 @@ class _InitializingFunctionObject(object):
   from the graph, which might not be possible in general.
   """
 
-  def __init__(self, call_fn, init_fn):
+  def __init__(self, call_fn, init_fn, shape_and_dtypes):
     self._init_fn = init_fn
     self._call_fn = call_fn
+    self.shape_and_dtypes = shape_and_dtypes
+    self.flattened_shapes = [tensor_shape.as_shape(sd.shape) for sd in
+                             nest.flatten(self.shape_and_dtypes)]
 
   @property
   def variables(self):
     return self._call_fn.variables
 
   def __call__(self, *args):
+    nest.assert_same_structure(self.shape_and_dtypes, args, check_types=False)
+    if not all([
+        shape.is_compatible_with(arg.shape)
+        for shape, arg in zip(self.flattened_shapes, nest.flatten(args))
+    ]):
+      raise ValueError(
+          "Declared shapes do not match argument shapes: Expected %s, found %s."
+          % (self.flattened_shapes, [arg.shape for arg in nest.flatten(args)]))
+
     initialized = [resource_variable_ops.var_is_initialized_op(
         v.handle).numpy() for v in self._call_fn.variables]
     if all(x for x in initialized):
@@ -398,12 +409,19 @@ def _graph_callable_internal(func, shape_and_dtypes):
       function._map_sequence_obj_to_idx(capture_func_def_outputs),  # pylint: disable=protected-access
       output_shapes)
 
-  return _InitializingFunctionObject(captured_function, initializer_function)
+  return _InitializingFunctionObject(captured_function, initializer_function,
+                                     shape_and_dtypes)
+
+
+class ShapeAndDtype(object):
+  """Data type that packages together shape and type information.
 
+  Used for arguments to graph callables. See graph_callable() for an example.
+  """
 
-# Data type that packages together shape and type information for arguments to
-# graph callables. See graph_callable() for an example.
-ShapeAndDtype = collections.namedtuple("ShapeAndDtype", ["shape", "dtype"])
+  def __init__(self, shape, dtype):
+    self.shape = shape
+    self.dtype = dtype
 
 
 def graph_callable(shape_and_dtypes):
diff --git a/tensorflow/python/eager/graph_callable_test.py b/tensorflow/python/eager/graph_callable_test.py
index 104e019391..57e1a062e1 100644
--- a/tensorflow/python/eager/graph_callable_test.py
+++ b/tensorflow/python/eager/graph_callable_test.py
@@ -219,6 +219,20 @@ class GraphCallableTest(test.TestCase):
 
     my_function()
 
+  def testIncorrectlyShapedInputs(self):
+    @graph_callable.graph_callable(
+        [graph_callable.ShapeAndDtype(shape=(3), dtype=dtypes.float32)])
+    def my_function(x):
+      v = variable_scope.get_variable(
+          "v", initializer=init_ops.zeros_initializer(), shape=())
+      return v + x
+
+    with self.assertRaises(ValueError):
+      my_function([1, 2])
+
+    self.assertTrue(([1, 2, 3] == my_function(
+        constant_op.constant([1, 2, 3], dtype=dtypes.float32)).numpy()).all())
+
 
 if __name__ == "__main__":
   test.main()
-- 
GitLab


From 8ed8e220017c13049490d2c4188e1eaf3ab068b0 Mon Sep 17 00:00:00 2001
From: Skye Wanderman-Milne <skyewm@google.com>
Date: Mon, 9 Oct 2017 12:08:22 -0700
Subject: [PATCH 187/909] Make ops_test.py work with the C API enabled.

This mostly involves adding custom ops to the test_ops library to
replace the ad-hoc ops previously used in the tests (it's not possible
to create new ops on the fly using the C API). In addition, this
change modifies importer_test.py to use the new custom ops as well.

PiperOrigin-RevId: 171568617
---
 tensorflow/python/framework/importer_test.py | 213 ++-----
 tensorflow/python/framework/ops.py           |  21 +-
 tensorflow/python/framework/ops_test.py      | 625 ++++++++++---------
 tensorflow/python/framework/test_ops.cc      | 161 +++++
 4 files changed, 557 insertions(+), 463 deletions(-)

diff --git a/tensorflow/python/framework/importer_test.py b/tensorflow/python/framework/importer_test.py
index 8ce8e76629..e447f9a3e8 100644
--- a/tensorflow/python/framework/importer_test.py
+++ b/tensorflow/python/framework/importer_test.py
@@ -29,9 +29,7 @@ from tensorflow.python.framework import device
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import function
 from tensorflow.python.framework import importer
-from tensorflow.python.framework import op_def_registry
 from tensorflow.python.framework import ops
-from tensorflow.python.framework import tensor_shape
 from tensorflow.python.framework import test_ops  # pylint: disable=unused-import
 from tensorflow.python.framework import versions
 from tensorflow.python.ops import array_ops
@@ -44,117 +42,6 @@ import tensorflow.python.ops.nn_grad  # pylint: disable=unused-import
 from tensorflow.python.platform import test
 
 
-def _UnknownShape(op):
-  return [tensor_shape.unknown_shape() for _ in op.outputs]
-
-
-# NOTE(cwhipkey): Dummy shape registration for ops used in the tests, since they
-# don't have C++ op registrations on which to attach C++ shape fns.
-ops.RegisterShape("If")(_UnknownShape)
-ops.RegisterShape("Iff")(_UnknownShape)
-ops.RegisterShape("Ii")(_UnknownShape)
-ops.RegisterShape("Iif")(_UnknownShape)
-ops.RegisterShape("Iii")(_UnknownShape)
-ops.RegisterShape("In")(_UnknownShape)
-ops.RegisterShape("Iri")(_UnknownShape)
-ops.RegisterShape("None")(_UnknownShape)
-ops.RegisterShape("Of")(_UnknownShape)
-ops.RegisterShape("Oi")(_UnknownShape)
-ops.RegisterShape("Oif")(_UnknownShape)
-ops.RegisterShape("Oii")(_UnknownShape)
-ops.RegisterShape("OpWithDefaultAttr")(_UnknownShape)
-ops.RegisterShape("OpWithFutureDefaultAttr")(_UnknownShape)
-ops.RegisterShape("Or")(_UnknownShape)
-ops.RegisterShape("Otl")(_UnknownShape)
-ops.RegisterShape("Unary")(_UnknownShape)
-
-_op_list = op_def_pb2.OpList()
-text_format.Merge("""
-  op {
-    name: 'None'
-  }
-  op {
-    name: 'Oi'
-    output_arg { name: 'a' type: DT_INT32 }
-  }
-  op {
-    name: 'Or'
-    output_arg { name: 'a' type: DT_INT32 is_ref: true }
-  }
-  op {
-    name: 'Of'
-    output_arg { name: 'a' type: DT_FLOAT }
-  }
-  op {
-    name: 'Ii'
-    input_arg { name: 'a' type: DT_INT32 }
-  }
-  op {
-    name: 'If'
-    input_arg { name: 'a' type: DT_FLOAT }
-  }
-  op {
-    name: 'Oii'
-    output_arg { name: 'a' type: DT_INT32 }
-    output_arg { name: 'b' type: DT_INT32 }
-  }
-  op {
-    name: 'Oif'
-    output_arg { name: 'a' type: DT_INT32 }
-    output_arg { name: 'b' type: DT_FLOAT }
-  }
-  op {
-    name: 'Iii'
-    input_arg { name: 'a' type: DT_INT32 }
-    input_arg { name: 'b' type: DT_INT32 }
-  }
-  op {
-    name: 'Iff'
-    input_arg { name: 'a' type: DT_FLOAT }
-    input_arg { name: 'b' type: DT_FLOAT }
-  }
-  op {
-    name: 'Iif'
-    input_arg { name: 'a' type: DT_INT32 }
-    input_arg { name: 'b' type: DT_FLOAT }
-  }
-  op {
-    name: 'Iri'
-    input_arg { name: 'a' type: DT_INT32 is_ref: true }
-    input_arg { name: 'b' type: DT_INT32 }
-  }
-  op {
-    name: 'In'
-    input_arg { name: 'a' number_attr: 'N' type_attr: 'T' }
-    attr { name: 'N' type: 'int' minimum: 1 }
-    attr { name: 'T' type: 'type' }
-  }
-  op {
-    name: 'Otl'
-    output_arg { name: 'a' type_list_attr: 't' }
-    attr { name: 'T' type: 'list(type)' minimum: 1 }
-  }
-  op {
-    name: 'Unary'
-    input_arg { name: 'a' type_attr: 'T' }
-    output_arg { name: 'b' type_attr: 'T' }
-    attr { name: 'T' type: 'type' }
-  }
-  op {
-    name: 'OpWithDefaultAttr'
-    output_arg { name: 'a' type: DT_INT32 }
-    attr { name: 'default_float' type: 'float' default_value { f: 123.0 } }
-  }
-  op {
-    name: 'OpWithFutureDefaultAttr'
-  }
-""", _op_list)
-op_def_registry.register_op_list(_op_list)
-# NOTE(mrry): Dummy shape registrations for ops used in the tests.
-for op_def in _op_list.op:
-  ops.RegisterShape(op_def.name)(None)
-
-
 class ImportGraphDefTest(test.TestCase):
 
   def _MakeGraphDef(self,
@@ -172,15 +59,15 @@ class ImportGraphDefTest(test.TestCase):
     with ops.Graph().as_default():
       a, b, c, d = importer.import_graph_def(
           self._MakeGraphDef("""
-          node { name: 'A' op: 'Oif' }
-          node { name: 'B' op: 'Otl'
-                 attr { key: 't'
+          node { name: 'A' op: 'IntOutputFloatOutput' }
+          node { name: 'B' op: 'ListOutput'
+                 attr { key: 'T'
                         value { list { type: DT_INT32 type: DT_FLOAT } } } }
-          node { name: 'C' op: 'In'
+          node { name: 'C' op: 'ListInput'
                  attr { key: 'N' value { i: 2 } }
                  attr { key: 'T' value { type: DT_INT32 } }
                  input: 'A:0' input: 'B:0' }
-          node { name: 'D' op: 'In'
+          node { name: 'D' op: 'ListInput'
                  attr { key: 'N' value { i: 2 } }
                  attr { key: 'T' value { type: DT_FLOAT } }
                  input: 'A:1' input: 'B:1' }
@@ -203,10 +90,10 @@ class ImportGraphDefTest(test.TestCase):
       self.assertEqual(d.inputs[1], b.outputs[1])
 
       # Check the types of the returned ops and tensors.
-      self.assertEqual(a.type, "Oif")
-      self.assertEqual(b.type, "Otl")
-      self.assertEqual(c.type, "In")
-      self.assertEqual(d.type, "In")
+      self.assertEqual(a.type, "IntOutputFloatOutput")
+      self.assertEqual(b.type, "ListOutput")
+      self.assertEqual(c.type, "ListInput")
+      self.assertEqual(d.type, "ListInput")
       self.assertEqual(a.outputs[0].dtype, dtypes.int32)
       self.assertEqual(a.outputs[1].dtype, dtypes.float32)
       self.assertEqual(b.outputs[0].dtype, dtypes.int32)
@@ -228,13 +115,13 @@ class ImportGraphDefTest(test.TestCase):
 
       a, b, c, d = importer.import_graph_def(
           self._MakeGraphDef("""
-          node { name: 'A' op: 'Oii' }
-          node { name: 'B' op: 'Oii' }
-          node { name: 'C' op: 'In'
+          node { name: 'A' op: 'TwoIntOutputs' }
+          node { name: 'B' op: 'TwoIntOutputs' }
+          node { name: 'C' op: 'ListInput'
                  attr { key: 'N' value { i: 2 } }
                  attr { key: 'T' value { type: DT_INT32 } }
                  input: 'A:0' input: 'B:0' }
-          node { name: 'D' op: 'In'
+          node { name: 'D' op: 'ListInput'
                  attr { key: 'N' value { i: 2 } }
                  attr { key: 'T' value { type: DT_INT32 } }
                  input: 'A:1' input: 'B:1' }
@@ -255,13 +142,13 @@ class ImportGraphDefTest(test.TestCase):
 
       a, b, c, d = importer.import_graph_def(
           self._MakeGraphDef("""
-          node { name: 'A' op: 'Oii' }
-          node { name: 'B' op: 'Oii' }
-          node { name: 'C' op: 'In'
+          node { name: 'A' op: 'TwoIntOutputs' }
+          node { name: 'B' op: 'TwoIntOutputs' }
+          node { name: 'C' op: 'ListInput'
                  attr { key: 'N' value { i: 2 } }
                  attr { key: 'T' value { type: DT_INT32 } }
                  input: 'A:0' input: 'B:0' }
-          node { name: 'D' op: 'In'
+          node { name: 'D' op: 'ListInput'
                  attr { key: 'N' value { i: 2 } }
                  attr { key: 'T' value { type: DT_INT32 } }
                  input: 'A:1' input: 'B:1' }
@@ -282,13 +169,13 @@ class ImportGraphDefTest(test.TestCase):
 
       a, b, c, d = importer.import_graph_def(
           self._MakeGraphDef("""
-          node { name: 'A' op: 'Oii' }
-          node { name: 'B' op: 'Oii' }
-          node { name: 'C' op: 'In'
+          node { name: 'A' op: 'TwoIntOutputs' }
+          node { name: 'B' op: 'TwoIntOutputs' }
+          node { name: 'C' op: 'ListInput'
                  attr { key: 'N' value { i: 2 } }
                  attr { key: 'T' value { type: DT_INT32 } }
                  input: 'A:0' input: 'B:0' }
-          node { name: 'D' op: 'In'
+          node { name: 'D' op: 'ListInput'
                  attr { key: 'N' value { i: 2 } }
                  attr { key: 'T' value { type: DT_INT32 } }
                  input: 'A:1' input: 'B:1' }
@@ -306,8 +193,8 @@ class ImportGraphDefTest(test.TestCase):
     with ops.Graph().as_default():
       a, b = importer.import_graph_def(
           self._MakeGraphDef("""
-          node { name: 'A' op: 'Oii' }
-          node { name: 'B' op: 'Ii' input: 'A' }
+          node { name: 'A' op: 'TwoIntOutputs' }
+          node { name: 'B' op: 'IntInput' input: 'A' }
           """),
           return_elements=["A", "B"])
 
@@ -318,8 +205,8 @@ class ImportGraphDefTest(test.TestCase):
       feed_a_0 = constant_op.constant(0, dtype=dtypes.int32)
       b, = importer.import_graph_def(
           self._MakeGraphDef("""
-          node { name: 'A' op: 'Oii' }
-          node { name: 'B' op: 'Ii' input: 'A:0' }
+          node { name: 'A' op: 'TwoIntOutputs' }
+          node { name: 'B' op: 'IntInput' input: 'A:0' }
           """),
           input_map={"A": feed_a_0},
           return_elements=["B"])
@@ -341,10 +228,10 @@ class ImportGraphDefTest(test.TestCase):
     with ops.Graph().as_default():
       a, b, c, d = importer.import_graph_def(
           self._MakeGraphDef("""
-          node { name: 'A' op: 'Or' }
-          node { name: 'B' op: 'Oi' }
-          node { name: 'C' op: 'Iii' input: 'A:0' input: 'B:0' }
-          node { name: 'D' op: 'Iri' input: 'A:0' input: 'B:0' }
+          node { name: 'A' op: 'RefOutput' }
+          node { name: 'B' op: 'IntOutput' }
+          node { name: 'C' op: 'TwoIntInputs' input: 'A:0' input: 'B:0' }
+          node { name: 'D' op: 'RefInputIntInput' input: 'A:0' input: 'B:0' }
           """),
           return_elements=["A", "B", "C", "D"])
 
@@ -378,8 +265,8 @@ class ImportGraphDefTest(test.TestCase):
       with self.assertRaises(ValueError) as e:
         importer.import_graph_def(
             self._MakeGraphDef("""
-            node { name: 'A' op: 'Oi' }
-            node { name: 'B' op: 'If' input: 'A:0' }
+            node { name: 'A' op: 'IntOutput' }
+            node { name: 'B' op: 'FloatInput' input: 'A:0' }
             """))
       self.assertTrue(
           "Cannot convert a tensor of type int32 to an input of type float" in
@@ -405,7 +292,7 @@ class ImportGraphDefTest(test.TestCase):
       with self.assertRaises(ValueError) as e:
         _ = importer.import_graph_def(
             self._MakeGraphDef("""
-              node { name: 'A' op: 'Of' }
+              node { name: 'A' op: 'FloatOutput' }
               node { name: 'B' op: 'L2Loss'
                      input: 'A:0'
                      attr { key: 'T' value { type: DT_FLOAT } }
@@ -422,7 +309,7 @@ class ImportGraphDefTest(test.TestCase):
       with self.assertRaises(ValueError) as e:
         importer.import_graph_def(
             self._MakeGraphDef("""
-            node { name: 'A' op: 'Oi' }
+            node { name: 'A' op: 'IntOutput' }
             node { name: 'B' op: 'None' input: 'A:0' }
             """))
       self.assertTrue("More inputs specified ('A:0') than the op expects" in
@@ -433,8 +320,8 @@ class ImportGraphDefTest(test.TestCase):
       with self.assertRaises(ValueError) as e:
         importer.import_graph_def(
             self._MakeGraphDef("""
-            node { name: 'A' op: 'Oi' }
-            node { name: 'B' op: 'Iif' input: 'A:0' }
+            node { name: 'A' op: 'IntOutput' }
+            node { name: 'B' op: 'IntInputFloatInput' input: 'A:0' }
             """))
       self.assertTrue("Input types mismatch (expected 'int32, float32' but "
                       "got 'int32')" in str(e.exception))
@@ -444,7 +331,7 @@ class ImportGraphDefTest(test.TestCase):
       with self.assertRaises(ValueError) as e:
         importer.import_graph_def(
             self._MakeGraphDef("""
-            node { name: 'B' op: 'If' input: 'A:0' }
+            node { name: 'B' op: 'FloatInput' input: 'A:0' }
             """))
       self.assertTrue("Input tensor 'A:0' not found" in str(e.exception))
 
@@ -453,7 +340,7 @@ class ImportGraphDefTest(test.TestCase):
       feed_a_0 = constant_op.constant(5.0)
       b, = importer.import_graph_def(
           self._MakeGraphDef("""
-          node { name: 'B' op: 'If' input: 'A:0' }
+          node { name: 'B' op: 'FloatInput' input: 'A:0' }
           """),
           input_map={"A:0": feed_a_0},
           return_elements=["B"])
@@ -464,8 +351,8 @@ class ImportGraphDefTest(test.TestCase):
       with self.assertRaises(ValueError) as e:
         importer.import_graph_def(
             self._MakeGraphDef("""
-            node { name: 'A' op: 'Of' }
-            node { name: 'B' op: 'If' input: 'A:1' }
+            node { name: 'A' op: 'FloatOutput' }
+            node { name: 'B' op: 'FloatInput' input: 'A:1' }
             """))
       self.assertTrue("Input tensor 'A:1' not found" in str(e.exception))
 
@@ -514,7 +401,7 @@ class ImportGraphDefTest(test.TestCase):
       with self.assertRaises(ValueError) as e:
         importer.import_graph_def(
             self._MakeGraphDef("""
-            node { name: 'A' op: 'Oi' }
+            node { name: 'A' op: 'IntOutput' }
             """),
             return_elements=["A:1"])
       self.assertTrue(
@@ -523,7 +410,7 @@ class ImportGraphDefTest(test.TestCase):
       with self.assertRaises(ValueError) as e:
         importer.import_graph_def(
             self._MakeGraphDef("""
-            node { name: 'A' op: 'Oi' }
+            node { name: 'A' op: 'IntOutput' }
             """),
             return_elements=["B:0"])
       self.assertTrue(
@@ -532,7 +419,7 @@ class ImportGraphDefTest(test.TestCase):
       with self.assertRaises(ValueError) as e:
         importer.import_graph_def(
             self._MakeGraphDef("""
-            node { name: 'A' op: 'Oi' }
+            node { name: 'A' op: 'IntOutput' }
             """),
             return_elements=["A:B:0"])
       self.assertTrue(
@@ -553,7 +440,7 @@ class ImportGraphDefTest(test.TestCase):
       # Mapping an unused node output should succeed.
       importer.import_graph_def(
           self._MakeGraphDef("""
-          node { name: 'A' op: 'Oi' }
+          node { name: 'A' op: 'IntOutput' }
           """),
           input_map={"A:0": constant_op.constant(5.0)})
 
@@ -561,7 +448,7 @@ class ImportGraphDefTest(test.TestCase):
       with self.assertRaises(ValueError) as e:
         importer.import_graph_def(
             self._MakeGraphDef("""
-            node { name: 'A' op: 'Oi' }
+            node { name: 'A' op: 'IntOutput' }
             """),
             input_map={"A:2": constant_op.constant(5.0)})
       self.assertTrue("not found in graph_def: [A:2]" in str(e.exception))
@@ -571,8 +458,8 @@ class ImportGraphDefTest(test.TestCase):
       with self.assertRaises(ValueError) as e:
         importer.import_graph_def(
             self._MakeGraphDef("""
-            node { name: 'A' op: 'Oi' }
-            node { name: 'B' op: 'Ii' input: 'A:0' }
+            node { name: 'A' op: 'IntOutput' }
+            node { name: 'B' op: 'IntInput' input: 'A:0' }
             """),
             input_map={"A:0": constant_op.constant(5.0)})
       self.assertTrue(
@@ -826,9 +713,9 @@ class ImportGraphDefTest(test.TestCase):
       with self.assertRaises(ValueError) as e:
         importer.import_graph_def(
             self._MakeGraphDef("""
-            node { name: 'A' op: 'Oi' }
-            node { name: 'B' op: 'Oi' }
-            node { name: 'A' op: 'Oi' }
+            node { name: 'A' op: 'IntOutput' }
+            node { name: 'B' op: 'IntOutput' }
+            node { name: 'A' op: 'IntOutput' }
             """))
       self.assertEqual("Duplicate name 'A' in GraphDef.", str(e.exception))
 
@@ -962,7 +849,7 @@ class ImportGraphDefTest(test.TestCase):
         with ops.Graph().as_default():
           a, = importer.import_graph_def(
               self._MakeGraphDef(
-                  "node { name: 'A' op: 'Oii' }",
+                  "node { name: 'A' op: 'TwoIntOutputs' }",
                   producer=producer,
                   min_consumer=min_consumer),
               return_elements=["A"])
diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index 0257f094d7..669588ace0 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -415,6 +415,7 @@ class Tensor(_TensorLike):
       ValueError: If `shape` is not compatible with the current shape of
         this tensor.
     """
+    # TODO(skyewm): call C API
     self._shape = self._shape.merge_with(shape)
 
   @property
@@ -1873,6 +1874,7 @@ class Operation(object):
     """The list of `Tensor` objects representing the data inputs of this op."""
     if self._c_op:
       tf_outputs = c_api.GetOperationInputs(self._c_op)
+      # TODO(skyewm): return Operation._InputList
       # pylint: disable=protected-access
       return [self.graph._get_tensor_by_tf_output(tf_output)
               for tf_output in tf_outputs]
@@ -4340,14 +4342,17 @@ class _DefaultStack(threading.local):
       self.stack.append(default)
       yield default
     finally:
-      if self._enforce_nesting:
-        if self.stack[-1] is not default:
-          raise AssertionError(
-              "Nesting violated for default stack of %s objects" %
-              type(default))
-        self.stack.pop()
-      else:
-        self.stack.remove(default)
+      # stack may be empty if reset() was called
+      if self.stack:
+        if self._enforce_nesting:
+          if self.stack[-1] is not default:
+            raise AssertionError(
+                "Nesting violated for default stack of %s objects" %
+                type(default))
+          self.stack.pop()
+        else:
+          self.stack.remove(default)
+
 
 _default_session_stack = _DefaultStack()  # pylint: disable=protected-access
 
diff --git a/tensorflow/python/framework/ops_test.py b/tensorflow/python/framework/ops_test.py
index 5c39dc192e..9ef7f59529 100644
--- a/tensorflow/python/framework/ops_test.py
+++ b/tensorflow/python/framework/ops_test.py
@@ -51,6 +51,7 @@ from tensorflow.python.util import compat
 ops._set_call_cpp_shape_fn(common_shapes.call_cpp_shape_fn)
 
 
+@test_util.with_c_api
 class ResourceTest(test_util.TensorFlowTestCase):
 
   def testBuildGraph(self):
@@ -76,11 +77,12 @@ class ResourceTest(test_util.TensorFlowTestCase):
                   resources.shared_resources()).eval()), 0)
 
 
+@test_util.with_c_api
 class TensorTest(test_util.TensorFlowTestCase):
 
   def testShape(self):
     op = ops.Operation(
-        ops._NodeDef("noop", "myop"), ops.Graph(), [], [dtypes.float32])
+        ops._NodeDef("FloatOutput", "myop"), ops.Graph(), [], [dtypes.float32])
     t = op.outputs[0]
     self.assertEqual(tensor_shape.unknown_shape(), t.get_shape())
     t.set_shape([1, 2, 3])
@@ -88,7 +90,7 @@ class TensorTest(test_util.TensorFlowTestCase):
 
   def testIterable(self):
     op = ops.Operation(
-        ops._NodeDef("noop", "myop"), ops.Graph(), [], [dtypes.float32])
+        ops._NodeDef("FloatOutput", "myop"), ops.Graph(), [], [dtypes.float32])
     t = op.outputs[0]
     self.assertTrue(isinstance(t, ops.Tensor))
     with self.assertRaisesRegexp(TypeError, "not iterable"):
@@ -96,6 +98,7 @@ class TensorTest(test_util.TensorFlowTestCase):
         pass
 
 
+@test_util.with_c_api
 class IndexedSlicesTest(test_util.TensorFlowTestCase):
 
   def testToTensor(self):
@@ -124,11 +127,12 @@ class IndexedSlicesTest(test_util.TensorFlowTestCase):
       self.assertAllEqual(x.indices.eval(), [0, 2])
 
 
+@test_util.with_c_api
 class NodeDefConstructorTest(test_util.TensorFlowTestCase):
 
   def testNoArgs(self):
-    nodedef = ops._NodeDef("noop", "bar")
-    self.assertProtoEquals("op: 'noop' name: 'bar'", nodedef)
+    nodedef = ops._NodeDef("None", "bar")
+    self.assertProtoEquals("op: 'None' name: 'bar'", nodedef)
 
   def testArgs(self):
     nodedef = ops._NodeDef("foo", "bar", device="/device:baz:*")
@@ -138,23 +142,6 @@ class NodeDefConstructorTest(test_util.TensorFlowTestCase):
     self.assertProtoEquals("op:'foo' name:'bar' device:'/job:j'", nodedef)
 
 
-# NOTE(mrry): Dummy shape registrations for ops used in the tests, since they
-# don't have C++ op registrations on which to attach C++ shape fns.
-ops.RegisterShape("a")(common_shapes.unknown_shape)
-ops.RegisterShape("b")(common_shapes.unknown_shape)
-ops.RegisterShape("c")(common_shapes.unknown_shape)
-ops.RegisterShape("add")(common_shapes.unknown_shape)
-ops.RegisterShape("an_op")(common_shapes.unknown_shape)
-ops.RegisterShape("const")(common_shapes.unknown_shape)
-ops.RegisterShape("copy")(common_shapes.unknown_shape)
-ops.RegisterShape("foo")(common_shapes.unknown_shape)
-ops.RegisterShape("identity")(common_shapes.unknown_shape)
-ops.RegisterShape("mul")(common_shapes.unknown_shape)
-ops.RegisterShape("nonrefop")(common_shapes.unknown_shape)
-ops.RegisterShape("noop")(common_shapes.unknown_shape)
-ops.RegisterShape("refop")(common_shapes.unknown_shape)
-
-
 def _apply_op(g, *args, **kwargs):
   op = g.create_op(*args, **kwargs)
   if len(op.outputs) == 1:
@@ -163,12 +150,11 @@ def _apply_op(g, *args, **kwargs):
     return op.outputs
 
 
+@test_util.with_c_api
 class OperationTest(test_util.TensorFlowTestCase):
 
   def testNoInputs(self):
-    op = ops.Operation(
-        ops._NodeDef("noop", "myop"),
-        ops.Graph(), [], [dtypes.float32, dtypes.string])
+    op = test_ops.float_output_string_output(name="myop").a.op
     self.assertEqual(2, len(op.values()))
     self.assertEqual(0, len(op.inputs))
     self.assertEqual("myop", op.name)
@@ -186,13 +172,13 @@ class OperationTest(test_util.TensorFlowTestCase):
     self.assertEqual(0, len(label_str_t._consumers))
     self.assertEqual("myop:1", label_str_t._as_node_def_input())
 
-    self.assertProtoEquals("op:'noop' name:'myop'", op.node_def)
+    self.assertProtoEquals("op:'FloatOutputStringOutput' name:'myop'",
+                           op.node_def)
 
   def testNoOutputs(self):
-    g = ops.Graph()
-    op1 = ops.Operation(ops._NodeDef("noop", "myop1"), g, [], [dtypes.float32])
+    op1 = test_ops.float_output(name="myop1").op
     float_t, = op1.values()
-    op2 = ops.Operation(ops._NodeDef("reop", "myop2"), g, [float_t], [])
+    op2 = test_ops.float_input(float_t, name="myop2")
     self.assertEqual(0, len(op2.values()))
     self.assertEqual(1, len(op2.inputs))
     self.assertIs(float_t, op2.inputs[0])
@@ -200,24 +186,21 @@ class OperationTest(test_util.TensorFlowTestCase):
     self.assertEqual(1, len(float_t._consumers))
     self.assertEqual(op2, float_t._consumers[0])
 
-    self.assertProtoEquals("op:'noop' name:'myop1'", op1.node_def)
-    self.assertProtoEquals("op:'reop' name:'myop2' input:'myop1'", op2.node_def)
+    self.assertProtoEquals("op:'FloatOutput' name:'myop1'", op1.node_def)
+    self.assertProtoEquals("op:'FloatInput' name:'myop2' input:'myop1'",
+                           op2.node_def)
 
   def testInputsAndOutputs(self):
-    g = ops.Graph()
-    op1 = ops.Operation(ops._NodeDef("noop", "myop1"), g, [], [dtypes.float32])
+    op1 = test_ops.float_output(name="myop1").op
     self.assertEqual(1, len(op1.values()))
     float1_t, = op1.values()
 
-    op2 = ops.Operation(
-        ops._NodeDef("reop", "myop2"), g, [], [dtypes.float32, dtypes.string])
+    op2 = test_ops.float_output_string_output(name="myop2").a.op
     self.assertEqual(2, len(op2.values()))
     float2_t, label2_str_t = op2.values()
 
     # Note that we consume label2_str_t twice here.
-    op3 = ops.Operation(
-        ops._NodeDef("add", "myop3"), g, [float1_t, label2_str_t, label2_str_t],
-        [dtypes.float32, dtypes.int32])
+    op3 = test_ops.foo2(float1_t, label2_str_t, label2_str_t, name="myop3").d.op
     self.assertEqual(2, len(op3.values()))
 
     self.assertEqual(1, len(float1_t._consumers))
@@ -230,40 +213,42 @@ class OperationTest(test_util.TensorFlowTestCase):
     self.assertEqual(op3, label2_str_t._consumers[1])
 
     self.assertProtoEquals("""
-    op:'add' name:'myop3'
+    op:'Foo2' name:'myop3'
     input:'myop1' input:'myop2:1' input:'myop2:1'
     """, op3.node_def)
 
   def testDeviceObject(self):
-    op = ops.Operation(ops._NodeDef("noop", "myop"), ops.Graph(), [], [])
+    op = ops.Operation(ops._NodeDef("None", "myop"), ops.Graph(), [], [])
     op._set_device("/job:goo/device:GPU:0")
     self.assertProtoEquals(
-        "op:'noop' name:'myop' device:'/job:goo/device:GPU:0' ", op.node_def)
-    op = ops.Operation(ops._NodeDef("noop", "op2"), ops.Graph(), [], [])
+        "op:'None' name:'myop' device:'/job:goo/device:GPU:0' ", op.node_def)
+    op = ops.Operation(ops._NodeDef("None", "op2"), ops.Graph(), [], [])
     op._set_device(
         pydev.DeviceSpec(
             job="muu", device_type="CPU", device_index=0))
     self.assertProtoEquals(
-        "op:'noop' name:'op2' device:'/job:muu/device:CPU:0'", op.node_def)
+        "op:'None' name:'op2' device:'/job:muu/device:CPU:0'", op.node_def)
 
   def testReferenceInput(self):
     g = ops.Graph()
     op1 = ops.Operation(
-        ops._NodeDef("noop", "op1"), g, [],
+        ops._NodeDef("RefOutputFloatOutput", "op1"), g, [],
         [dtypes.float32_ref, dtypes.float32])
-    self.assertProtoEquals("op:'noop' name:'op1'", op1.node_def)
+    self.assertProtoEquals("op:'RefOutputFloatOutput' name:'op1'", op1.node_def)
     ref_t, nonref_t = op1.values()
     # NOTE(mrry): Must specify input_types to preserve ref-typed input.
     op2 = ops.Operation(
-        ops._NodeDef("refop", "op2"),
+        ops._NodeDef("RefInputFloatInput", "op2"),
         g, [ref_t, nonref_t], [],
         input_types=[dtypes.float32_ref, dtypes.float32])
-    self.assertProtoEquals("op:'refop' name:'op2' input:'op1' input:'op1:1'",
-                           op2.node_def)
+    self.assertProtoEquals(
+        "op:'RefInputFloatInput' name:'op2' input:'op1' input:'op1:1'",
+        op2.node_def)
     op3 = ops.Operation(
-        ops._NodeDef("nonrefop", "op3"), g, [ref_t, nonref_t], [])
-    self.assertProtoEquals("op:'nonrefop' name:'op3' input:'op1' input:'op1:1'",
-                           op3.node_def)
+        ops._NodeDef("TwoFloatInputs", "op3"), g, [ref_t, nonref_t], [])
+    self.assertProtoEquals(
+        "op:'TwoFloatInputs' name:'op3' input:'op1' input:'op1:1'",
+        op3.node_def)
 
   def testInvalidNames(self):
     g = ops.Graph()
@@ -279,10 +264,8 @@ class OperationTest(test_util.TensorFlowTestCase):
       ops.Operation(ops._NodeDef("op", "invalid:0"), g)
 
   def testNoShapeFunction(self):
-    g = ops.Graph()
-    ops.Operation(ops._NodeDef("op", "an_op"), g, output_types=[dtypes.float32])
-    self.assertEqual(tensor_shape.unknown_shape(),
-                     _apply_op(g, "an_op", [], [dtypes.float32]).get_shape())
+    op = test_ops.a()
+    self.assertEqual(tensor_shape.unknown_shape(), op.get_shape())
 
   def testConvertToTensorNestedArray(self):
     with self.test_session():
@@ -364,22 +347,25 @@ class OperationTest(test_util.TensorFlowTestCase):
       ops.convert_to_tensor(op)
 
   def testStr(self):
-    node_def = ops._NodeDef("noop", "op1")
+    node_def = ops._NodeDef("None", "op1")
     op = ops.Operation(node_def, ops.Graph(), [], [dtypes.float32])
     self.assertEqual(str(node_def), str(op))
 
   def testRepr(self):
     op = ops.Operation(
-        ops._NodeDef("noop", "op1"), ops.Graph(), [], [dtypes.float32])
-    self.assertEqual("<tf.Operation 'op1' type=noop>", repr(op))
+        ops._NodeDef("None", "op1"), ops.Graph(), [], [dtypes.float32])
+    self.assertEqual("<tf.Operation 'op1' type=None>", repr(op))
 
   def testGetAttr(self):
+    # TODO(skyewm): implement get_attr with C API
+    if ops._USE_C_API: return
+
     list_value = attr_value_pb2.AttrValue.ListValue()
     list_value.type.append(types_pb2.DT_STRING)
     list_value.type.append(types_pb2.DT_DOUBLE)
     op = ops.Operation(
         ops._NodeDef(
-            "noop",
+            "None",
             "op1",
             attrs={
                 "value": attr_value_pb2.AttrValue(i=32),
@@ -403,7 +389,6 @@ class OperationTest(test_util.TensorFlowTestCase):
     self.assertEqual([dtypes.string, dtypes.double], l)
 
   # TODO(nolivia): test all error cases
-  @test_util.enable_c_api
   def testAddControlInput(self):
     with ops.Graph().as_default():
       x = constant_op.constant(1).op
@@ -411,8 +396,9 @@ class OperationTest(test_util.TensorFlowTestCase):
     y._add_control_input(x)  # pylint: disable=protected-access
     self.assertEqual(y.control_inputs, [x])
 
-  @test_util.enable_c_api
   def testControlInputCycle(self):
+    # Non-C API path has a different error message
+    if not ops._USE_C_API: return
     graph = ops.Graph()
     with graph.as_default():
       z = constant_op.constant(0)
@@ -427,7 +413,6 @@ class OperationTest(test_util.TensorFlowTestCase):
           "Graph is invalid, contains a cycle with 2 nodes"):
         sess.run(x)
 
-  @test_util.enable_c_api
   def testUpdateInput(self):
     g = ops.Graph()
     with g.as_default():
@@ -436,21 +421,20 @@ class OperationTest(test_util.TensorFlowTestCase):
       z = x + y
 
     z.op._update_input(0, y)  # pylint: disable=protected-access
-    self.assertEquals(z.op.inputs, [y, y])
+    self.assertEquals(list(z.op.inputs), [y, y])
     with session.Session(graph=g) as sess:
       self.assertEquals(sess.run(z), 4)
 
     z.op._update_input(0, x)  # pylint: disable=protected-access
-    self.assertEquals(z.op.inputs, [x, y])
+    self.assertEquals(list(z.op.inputs), [x, y])
     with session.Session(graph=g) as sess:
       self.assertEquals(sess.run(z), 3)
 
     z.op._update_input(1, y)  # pylint: disable=protected-access
-    self.assertEquals(z.op.inputs, [x, y])
+    self.assertEquals(list(z.op.inputs), [x, y])
     with session.Session(graph=g) as sess:
       self.assertEquals(sess.run(z), 3)
 
-  @test_util.enable_c_api
   def testUpdateInputGraphError(self):
     g_0 = ops.Graph()
     g_1 = ops.Graph()
@@ -464,7 +448,6 @@ class OperationTest(test_util.TensorFlowTestCase):
 
   # TODO(nolivia): check the shape/type in _update_input() instead of depending
   # on run to do that.
-  @test_util.enable_c_api
   def testUpdateInputTypeError(self):
     g = ops.Graph()
     with g.as_default():
@@ -480,34 +463,39 @@ class OperationTest(test_util.TensorFlowTestCase):
           "with expected int32"):
         sess.run(z)
 
-  # C-API throws the error differently.
   def testUpdateInputOutOfRange(self):
+    # C-API throws the error differently.
+    if ops._USE_C_API: return
     g = ops.Graph()
     with g.as_default():
       x = constant_op.constant(1)
-    with self.assertRaises(IndexError):
+    with self.assertRaisesRegexp(IndexError, "list index out of range"):
       x.op._update_input(1, x)  # pylint: disable=protected-access
 
-  @test_util.enable_c_api
   def testUpdateInputOutOfRangeC(self):
+    # C-API throws the error differently.
+    if not ops._USE_C_API: return
     g = ops.Graph()
     with g.as_default():
       x = constant_op.constant(1)
     with self.assertRaisesRegexp(errors.OutOfRangeError,
-                                 "does not have input 1"):
+                                 r"Node 'Const' \(type: 'Const', "
+                                 r"num of inputs: 0\) does not have input 1"):
       x.op._update_input(1, x)  # pylint: disable=protected-access
 
 
+@test_util.with_c_api
 class CreateOpTest(test_util.TensorFlowTestCase):
 
   def testNodeDefArgs(self):
     g = ops.Graph()
-    op1 = g.create_op("const", [], [dtypes.float32], None, name="myop1")
+    op1 = g.create_op("FloatOutput", [], [dtypes.float32], None, name="myop1")
     with g.device("/device:GPU:0"):
       op2 = g.create_op(
-          "add", [], [dtypes.float32, dtypes.string], None, name="myop2")
+          "FloatOutputStringOutput", [], [dtypes.float32, dtypes.string], None,
+          name="myop2")
     op3 = g.create_op(
-        "foo",
+        "Foo3",
         [list(op1.values())[0], list(op2.values())[1], list(op2.values())[0]],
         [dtypes.float32, dtypes.int32],
         None,
@@ -515,52 +503,57 @@ class CreateOpTest(test_util.TensorFlowTestCase):
     self.assertDeviceEqual(None, op1.device)
     self.assertDeviceEqual("/device:GPU:0", op2.device)
     self.assertDeviceEqual(None, op3.device)
-    self.assertProtoEquals("name:'myop1' op:'const'", op1.node_def)
-    self.assertProtoEquals("name:'myop2' op:'add' device:'/device:GPU:0'",
-                           op2.node_def)
+    self.assertProtoEquals("name:'myop1' op:'FloatOutput'", op1.node_def)
     self.assertProtoEquals(
-        "name:'myop3' input:'myop1' input:'myop2:1' input:'myop2' op:'foo'",
+        "name:'myop2' op:'FloatOutputStringOutput' device:'/device:GPU:0'",
+        op2.node_def)
+    self.assertProtoEquals(
+        "name:'myop3' input:'myop1' input:'myop2:1' input:'myop2' op:'Foo3'",
         op3.node_def)
 
   def testReferenceInput(self):
     g = ops.Graph()
     op1 = g.create_op(
-        "noop", [], [dtypes.float32_ref, dtypes.float32], name="op1")
-    self.assertProtoEquals("op:'noop' name:'op1'", op1.node_def)
+        "RefOutputFloatOutput", [], [dtypes.float32_ref, dtypes.float32],
+        name="op1")
+    self.assertProtoEquals("op:'RefOutputFloatOutput' name:'op1'", op1.node_def)
     ref_t, nonref_t = op1.values()
     # NOTE(mrry): Must specify input_types to preserve ref-typed input.
     op2 = g.create_op(
-        "refop", [ref_t, nonref_t], [],
+        "RefInputFloatInput", [ref_t, nonref_t], [],
         input_types=[dtypes.float32_ref, dtypes.float32],
         name="op2")
-    self.assertProtoEquals("op:'refop' name:'op2' input:'op1' input:'op1:1'",
-                           op2.node_def)
-    op3 = g.create_op("nonrefop", [ref_t, nonref_t], [], name="op3")
-    self.assertProtoEquals("op:'nonrefop' name:'op3' input:'op1' input:'op1:1'",
-                           op3.node_def)
+    self.assertProtoEquals(
+        "op:'RefInputFloatInput' name:'op2' input:'op1' input:'op1:1'",
+        op2.node_def)
+    op3 = g.create_op("TwoFloatInputs", [ref_t, nonref_t], [], name="op3")
+    self.assertProtoEquals(
+        "op:'TwoFloatInputs' name:'op3' input:'op1' input:'op1:1'",
+        op3.node_def)
 
   def testFinalized(self):
     g = ops.Graph()
     g.finalize()
     with self.assertRaises(RuntimeError):
-      g.create_op("const", [], [dtypes.float32], None, name="myop1")
+      g.create_op("FloatOutput", [], [dtypes.float32], None, name="myop1")
 
     # Test unfinalize.
     g._unsafe_unfinalize()
-    g.create_op("const", [], [dtypes.float32], None, name="myop1")
+    g.create_op("FloatOutput", [], [dtypes.float32], None, name="myop1")
 
 
+@test_util.with_c_api
 class ApplyOpTest(test_util.TensorFlowTestCase):
 
   def testNodeDefArgs(self):
     g = ops.Graph()
-    t1 = _apply_op(g, "const", [], [dtypes.float32], name="myop1")
+    t1 = _apply_op(g, "FloatOutput", [], [dtypes.float32], name="myop1")
     with g.device("/device:GPU:0"):
       t2 = _apply_op(
-          g, "add", [], [dtypes.float32, dtypes.string], name="myop2")
+          g, "TwoIntOutputs", [], [dtypes.int32, dtypes.int32], name="myop2")
     t3 = _apply_op(
         g,
-        "foo", [t1, t2[1], t2[0]], [dtypes.float32, dtypes.int32],
+        "Foo1", [t1, t2[1], t2[0]], [dtypes.float32, dtypes.int32],
         name="myop3")
     self.assertTrue(isinstance(t1, ops.Tensor))
     self.assertTrue(isinstance(t2, list))
@@ -571,32 +564,39 @@ class ApplyOpTest(test_util.TensorFlowTestCase):
     self.assertEqual("myop2:1", t2[1]._as_node_def_input())
     self.assertEqual("myop3", t3[0]._as_node_def_input())
     # Validate that we got the right ops as well
-    self.assertProtoEquals("name:'myop1' op:'const'", t1.op.node_def)
-    self.assertProtoEquals("name:'myop2' op:'add' device:'/device:GPU:0'",
-                           t2[0].op.node_def)
+    self.assertProtoEquals("name:'myop1' op:'FloatOutput'", t1.op.node_def)
+    self.assertProtoEquals(
+        "name:'myop2' op:'TwoIntOutputs' device:'/device:GPU:0'",
+        t2[0].op.node_def)
     self.assertProtoEquals(
-        "name:'myop3' input:'myop1' input:'myop2:1' input:'myop2' op:'foo'",
+        "name:'myop3' input:'myop1' input:'myop2:1' input:'myop2' op:'Foo1'",
         t3[0].op.node_def)
 
   def testReferenceInput(self):
     g = ops.Graph()
     ref_t, nonref_t = _apply_op(
-        g, "noop", [], [dtypes.float32_ref, dtypes.float32], name="op1")
-    self.assertProtoEquals("op:'noop' name:'op1'", ref_t.op.node_def)
+        g, "RefOutputFloatOutput", [], [dtypes.float32_ref, dtypes.float32],
+        name="op1")
+    self.assertProtoEquals("op:'RefOutputFloatOutput' name:'op1'",
+                           ref_t.op.node_def)
     # NOTE(mrry): Must specify input_types to preserve ref-typed input.
     out_2 = _apply_op(
         g,
-        "refop", [ref_t, nonref_t], [dtypes.int32],
+        "RefInputFloatInputIntOutput", [ref_t, nonref_t], [dtypes.int32],
         input_types=[dtypes.float32_ref, dtypes.float32],
         name="op2")
-    self.assertProtoEquals("op:'refop' name:'op2' input:'op1' input:'op1:1'",
-                           out_2.op.node_def)
+    self.assertProtoEquals(
+        "op:'RefInputFloatInputIntOutput' name:'op2' input:'op1' input:'op1:1'",
+        out_2.op.node_def)
     out_3 = _apply_op(
-        g, "nonrefop", [ref_t, nonref_t], [dtypes.int32], name="op3")
-    self.assertProtoEquals("op:'nonrefop' name:'op3' input:'op1' input:'op1:1'",
-                           out_3.op.node_def)
+        g, "TwoFloatInputsIntOutput", [ref_t, nonref_t], [dtypes.int32],
+        name="op3")
+    self.assertProtoEquals(
+        "op:'TwoFloatInputsIntOutput' name:'op3' input:'op1' input:'op1:1'",
+        out_3.op.node_def)
 
 
+@test_util.with_c_api
 class NameStackTest(test_util.TensorFlowTestCase):
 
   def testBasics(self):
@@ -695,22 +695,27 @@ class NameStackTest(test_util.TensorFlowTestCase):
         pass
 
 
+@test_util.with_c_api
 class NameTest(test_util.TensorFlowTestCase):
 
   def testGenerateName(self):
     g = ops.Graph()
-    op0 = g.create_op("const", [], [dtypes.float32, dtypes.float32])
-    self.assertEqual("const", op0.name)
-    self.assertEqual("const:0", op0.outputs[0].name)
-    self.assertEqual("const:1", op0.outputs[1].name)
+    op0 = g.create_op("TwoFloatOutputs", [], [dtypes.float32, dtypes.float32])
+    self.assertEqual("TwoFloatOutputs", op0.name)
+    self.assertEqual("TwoFloatOutputs:0", op0.outputs[0].name)
+    self.assertEqual("TwoFloatOutputs:1", op0.outputs[1].name)
+
+    op1 = g.create_op("FloatOutput", [], [dtypes.float32])
+    self.assertEqual("FloatOutput", op1.name)
+    self.assertEqual("FloatOutput:0", op1.outputs[0].name)
 
-    op1 = g.create_op("const", [], [dtypes.float32])
-    self.assertEqual("const_1", op1.name)
-    self.assertEqual("const_1:0", op1.outputs[0].name)
+    op2 = g.create_op("FloatOutput", [], [dtypes.float32])
+    self.assertEqual("FloatOutput_1", op2.name)
+    self.assertEqual("FloatOutput_1:0", op2.outputs[0].name)
 
-    op2 = g.create_op("const", [], [dtypes.float32], name="my_op")
-    self.assertEqual("my_op", op2.name)
-    self.assertEqual("my_op:0", op2.outputs[0].name)
+    op3 = g.create_op("FloatOutput", [], [dtypes.float32], name="my_op")
+    self.assertEqual("my_op", op3.name)
+    self.assertEqual("my_op:0", op3.outputs[0].name)
 
   def testNameScope(self):
     g = ops.Graph()
@@ -726,57 +731,60 @@ class NameTest(test_util.TensorFlowTestCase):
       with g.name_scope("") as empty2:
         self.assertEqual("", empty2)
 
-    self.assertEqual("const", g.create_op("const", [], [dtypes.float32]).name)
+    self.assertEqual("FloatOutput",
+                     g.create_op("FloatOutput", [], [dtypes.float32]).name)
     with g.name_scope("bar") as scope:
-      self.assertEqual("bar/const",
-                       g.create_op("const", [], [dtypes.float32]).name)
-      self.assertEqual("bar/const_1",
-                       g.create_op("const", [], [dtypes.float32]).name)
+      self.assertEqual("bar/FloatOutput",
+                       g.create_op("FloatOutput", [], [dtypes.float32]).name)
+      self.assertEqual("bar/FloatOutput_1",
+                       g.create_op("FloatOutput", [], [dtypes.float32]).name)
       # If you use the value from "with .. as", that values is used as-is.
       self.assertEqual(
           "bar", g.create_op(
-              "const", [], [dtypes.float32], name=scope).name)
+              "FloatOutput", [], [dtypes.float32], name=scope).name)
     with g.name_scope("baz") as scope:
       with g.name_scope("quux"):
-        self.assertEqual("baz/quux/const",
-                         g.create_op("const", [], [dtypes.float32]).name)
+        self.assertEqual("baz/quux/FloatOutput",
+                         g.create_op("FloatOutput", [], [dtypes.float32]).name)
       # If you use the value from the enclosing "with .. as", nothing is pushed.
       with g.name_scope(scope):
-        self.assertEqual("baz/const",
-                         g.create_op("const", [], [dtypes.float32]).name)
+        self.assertEqual("baz/FloatOutput",
+                         g.create_op("FloatOutput", [], [dtypes.float32]).name)
         self.assertEqual(
             "baz", g.create_op(
-                "const", [], [dtypes.float32], name=scope).name)
+                "FloatOutput", [], [dtypes.float32], name=scope).name)
         self.assertEqual(
             "trailing",
             g.create_op(
-                "const", [], [dtypes.float32], name="trailing/").name)
+                "FloatOutput", [], [dtypes.float32], name="trailing/").name)
     with g.name_scope("bar"):
-      self.assertEqual("bar_1/const",
-                       g.create_op("const", [], [dtypes.float32]).name)
+      self.assertEqual("bar_1/FloatOutput",
+                       g.create_op("FloatOutput", [], [dtypes.float32]).name)
     with g.name_scope("bar/"):
-      self.assertEqual("bar/const_2",
-                       g.create_op("const", [], [dtypes.float32]).name)
+      self.assertEqual("bar/FloatOutput_2",
+                       g.create_op("FloatOutput", [], [dtypes.float32]).name)
 
 
+@test_util.with_c_api
 class DeviceTest(test_util.TensorFlowTestCase):
 
   def testNoDevice(self):
     g = ops.Graph()
-    op = g.create_op("an_op", [], [dtypes.float32])
+    op = g.create_op("FloatOutput", [], [dtypes.float32])
     self.assertDeviceEqual(None, op.device)
     gd = g.as_graph_def()
     self.assertProtoEqualsVersion("""
-      node { name: "an_op" op: "an_op" }
+      node { name: "FloatOutput" op: "FloatOutput" }
     """, gd)
 
   def testDevicePartialString(self):
     g = ops.Graph()
     with g.device("/job:worker/replica:2"):
-      g.create_op("an_op", [], [dtypes.float32])
+      g.create_op("FloatOutput", [], [dtypes.float32])
     gd = g.as_graph_def()
     self.assertProtoEqualsVersion("""
-      node { name: "an_op" op: "an_op" device: "/job:worker/replica:2" }
+      node { name: "FloatOutput" op: "FloatOutput"
+             device: "/job:worker/replica:2" }
     """, gd)
 
   def testDeviceFull(self):
@@ -785,61 +793,61 @@ class DeviceTest(test_util.TensorFlowTestCase):
         pydev.DeviceSpec(
             job="worker", replica=2, task=0, device_type="CPU",
             device_index=3)):
-      g.create_op("an_op", [], [dtypes.float32])
+      g.create_op("FloatOutput", [], [dtypes.float32])
     gd = g.as_graph_def()
     self.assertProtoEqualsVersion("""
-      node { name: "an_op" op: "an_op"
+      node { name: "FloatOutput" op: "FloatOutput"
              device: "/job:worker/replica:2/task:0/device:CPU:3" }
     """, gd)
 
   def testNesting(self):
     g = ops.Graph()
     with g.device("/job:worker/replica:2"):
-      g.create_op("an_op", [], [dtypes.float32])
+      g.create_op("FloatOutput", [], [dtypes.float32])
       with g.device("/job:worker/replica:3/task:0"):
-        g.create_op("an_op", [], [dtypes.float32])
-      g.create_op("an_op", [], [dtypes.float32])
+        g.create_op("FloatOutput", [], [dtypes.float32])
+      g.create_op("FloatOutput", [], [dtypes.float32])
     gd = g.as_graph_def()
     self.assertProtoEqualsVersion("""
-      node { name: "an_op" op: "an_op"
+      node { name: "FloatOutput" op: "FloatOutput"
              device: "/job:worker/replica:2" }
-      node { name: "an_op_1" op: "an_op"
+      node { name: "FloatOutput_1" op: "FloatOutput"
              device: "/job:worker/replica:3/task:0" }
-      node { name: "an_op_2" op: "an_op"
+      node { name: "FloatOutput_2" op: "FloatOutput"
              device: "/job:worker/replica:2" }
     """, gd)
 
   def testNestingString(self):
     g = ops.Graph()
     with g.device("/job:worker/replica:2"):
-      g.create_op("an_op", [], [dtypes.float32])
+      g.create_op("FloatOutput", [], [dtypes.float32])
       with g.device("/job:worker/replica:3/task:0"):
-        g.create_op("an_op", [], [dtypes.float32])
-      g.create_op("an_op", [], [dtypes.float32])
+        g.create_op("FloatOutput", [], [dtypes.float32])
+      g.create_op("FloatOutput", [], [dtypes.float32])
     gd = g.as_graph_def()
     self.assertProtoEqualsVersion("""
-      node { name: "an_op" op: "an_op"
+      node { name: "FloatOutput" op: "FloatOutput"
              device: "/job:worker/replica:2" }
-      node { name: "an_op_1" op: "an_op"
+      node { name: "FloatOutput_1" op: "FloatOutput"
              device: "/job:worker/replica:3/task:0" }
-      node { name: "an_op_2" op: "an_op"
+      node { name: "FloatOutput_2" op: "FloatOutput"
              device: "/job:worker/replica:2" }
     """, gd)
 
   def testNestingOverrideGpuCpu(self):
     g = ops.Graph()
     with g.device("/job:worker/replica:2/device:CPU:1"):
-      g.create_op("an_op", [], [dtypes.float32])
+      g.create_op("FloatOutput", [], [dtypes.float32])
       with g.device("/job:worker/replica:2/device:GPU:2"):
-        g.create_op("an_op", [], [dtypes.float32])
-      g.create_op("an_op", [], [dtypes.float32])
+        g.create_op("FloatOutput", [], [dtypes.float32])
+      g.create_op("FloatOutput", [], [dtypes.float32])
     gd = g.as_graph_def()
     self.assertProtoEqualsVersion("""
-      node { name: "an_op" op: "an_op"
+      node { name: "FloatOutput" op: "FloatOutput"
              device: "/job:worker/replica:2/device:CPU:1"  }
-      node { name: "an_op_1" op: "an_op"
+      node { name: "FloatOutput_1" op: "FloatOutput"
              device: "/job:worker/replica:2/device:GPU:2" }
-      node { name: "an_op_2" op: "an_op"
+      node { name: "FloatOutput_2" op: "FloatOutput"
              device: "/job:worker/replica:2/device:CPU:1" }
     """, gd)
 
@@ -847,27 +855,27 @@ class DeviceTest(test_util.TensorFlowTestCase):
     g = ops.Graph()
 
     with g.device(pydev.merge_device("/device:GPU:0")):
-      g.create_op("an_op", [], [dtypes.float32])
+      g.create_op("FloatOutput", [], [dtypes.float32])
       with g.device(pydev.merge_device("/job:worker")):
-        g.create_op("an_op", [], [dtypes.float32])
+        g.create_op("FloatOutput", [], [dtypes.float32])
         with g.device(pydev.merge_device("/device:CPU:0")):
-          g.create_op("an_op", [], [dtypes.float32])
+          g.create_op("FloatOutput", [], [dtypes.float32])
           with g.device(pydev.merge_device("/job:ps")):
-            g.create_op("an_op", [], [dtypes.float32])
+            g.create_op("FloatOutput", [], [dtypes.float32])
             with g.device(pydev.merge_device(None)):
-              g.create_op("an_op", [], [dtypes.float32])
+              g.create_op("FloatOutput", [], [dtypes.float32])
 
     gd = g.as_graph_def()
     self.assertProtoEqualsVersion("""
-      node { name: "an_op" op: "an_op"
+      node { name: "FloatOutput" op: "FloatOutput"
              device: "/device:GPU:0" }
-      node { name: "an_op_1" op: "an_op"
+      node { name: "FloatOutput_1" op: "FloatOutput"
              device: "/job:worker/device:GPU:0" }
-      node { name: "an_op_2" op: "an_op"
+      node { name: "FloatOutput_2" op: "FloatOutput"
              device: "/job:worker/device:CPU:0" }
-      node { name: "an_op_3" op: "an_op"
+      node { name: "FloatOutput_3" op: "FloatOutput"
              device: "/job:ps/device:CPU:0" }
-      node { name: "an_op_4" op: "an_op"
+      node { name: "FloatOutput_4" op: "FloatOutput"
              device: "/job:ps/device:CPU:0" }
     """, gd)
 
@@ -875,27 +883,27 @@ class DeviceTest(test_util.TensorFlowTestCase):
     g = ops.Graph()
 
     with g.device("/device:GPU:0"):
-      g.create_op("an_op", [], [dtypes.float32])
+      g.create_op("FloatOutput", [], [dtypes.float32])
       with g.device("/job:worker"):
-        g.create_op("an_op", [], [dtypes.float32])
+        g.create_op("FloatOutput", [], [dtypes.float32])
         with g.device("/device:CPU:0"):
-          g.create_op("an_op", [], [dtypes.float32])
+          g.create_op("FloatOutput", [], [dtypes.float32])
           with g.device("/job:ps"):
-            g.create_op("an_op", [], [dtypes.float32])
+            g.create_op("FloatOutput", [], [dtypes.float32])
             with g.device(""):
-              g.create_op("an_op", [], [dtypes.float32])
+              g.create_op("FloatOutput", [], [dtypes.float32])
 
     gd = g.as_graph_def()
     self.assertProtoEqualsVersion("""
-      node { name: "an_op" op: "an_op"
+      node { name: "FloatOutput" op: "FloatOutput"
              device: "/device:GPU:0" }
-      node { name: "an_op_1" op: "an_op"
+      node { name: "FloatOutput_1" op: "FloatOutput"
              device: "/job:worker/device:GPU:0" }
-      node { name: "an_op_2" op: "an_op"
+      node { name: "FloatOutput_2" op: "FloatOutput"
              device: "/job:worker/device:CPU:0" }
-      node { name: "an_op_3" op: "an_op"
+      node { name: "FloatOutput_3" op: "FloatOutput"
              device: "/job:ps/device:CPU:0" }
-      node { name: "an_op_4" op: "an_op"
+      node { name: "FloatOutput_4" op: "FloatOutput"
              device: "/job:ps/device:CPU:0" }
     """, gd)
 
@@ -903,56 +911,56 @@ class DeviceTest(test_util.TensorFlowTestCase):
     g = ops.Graph()
 
     with g.device("/device:GPU:7"):
-      g.create_op("an_op", [], [dtypes.float32])
+      g.create_op("FloatOutput", [], [dtypes.float32])
       with g.device("/device:GPU:*"):
-        g.create_op("an_op", [], [dtypes.float32])
+        g.create_op("FloatOutput", [], [dtypes.float32])
 
     with g.device("/device:CPU:*"):
-      g.create_op("an_op", [], [dtypes.float32])
+      g.create_op("FloatOutput", [], [dtypes.float32])
       with g.device("/device:CPU:5"):
-        g.create_op("an_op", [], [dtypes.float32])
+        g.create_op("FloatOutput", [], [dtypes.float32])
 
     gd = g.as_graph_def()
     self.assertProtoEqualsVersion("""
-      node { name: "an_op" op: "an_op"
+      node { name: "FloatOutput" op: "FloatOutput"
              device: "/device:GPU:7" }
-      node { name: "an_op_1" op: "an_op"
+      node { name: "FloatOutput_1" op: "FloatOutput"
              device: "/device:GPU:7" }
-      node { name: "an_op_2" op: "an_op"
+      node { name: "FloatOutput_2" op: "FloatOutput"
              device: "/device:CPU:*" }
-      node { name: "an_op_3" op: "an_op"
+      node { name: "FloatOutput_3" op: "FloatOutput"
              device: "/device:CPU:5" }
     """, gd)
 
   def testNoneClearsDefault(self):
     g = ops.Graph()
     with g.device("/job:worker/replica:2/device:CPU:1"):
-      g.create_op("an_op", [], [dtypes.float32])
+      g.create_op("FloatOutput", [], [dtypes.float32])
       with g.device(None):
-        g.create_op("an_op", [], [dtypes.float32])
-      g.create_op("an_op", [], [dtypes.float32])
+        g.create_op("FloatOutput", [], [dtypes.float32])
+      g.create_op("FloatOutput", [], [dtypes.float32])
     gd = g.as_graph_def()
     self.assertProtoEqualsVersion("""
-      node { name: "an_op" op: "an_op"
+      node { name: "FloatOutput" op: "FloatOutput"
              device: "/job:worker/replica:2/device:CPU:1" }
-      node { name: "an_op_1" op: "an_op" }
-      node { name: "an_op_2" op: "an_op"
+      node { name: "FloatOutput_1" op: "FloatOutput" }
+      node { name: "FloatOutput_2" op: "FloatOutput"
              device: "/job:worker/replica:2/device:CPU:1" }
     """, gd)
 
   def testNoneIgnoresOuterDeviceFunction(self):
     g = ops.Graph()
     with g.device(lambda op: "/job:worker/replica:2/device:CPU:1"):
-      g.create_op("an_op", [], [dtypes.float32])
+      g.create_op("FloatOutput", [], [dtypes.float32])
       with g.device(None):
-        g.create_op("an_op", [], [dtypes.float32])
-      g.create_op("an_op", [], [dtypes.float32])
+        g.create_op("FloatOutput", [], [dtypes.float32])
+      g.create_op("FloatOutput", [], [dtypes.float32])
     gd = g.as_graph_def()
     self.assertProtoEqualsVersion("""
-      node { name: "an_op" op: "an_op"
+      node { name: "FloatOutput" op: "FloatOutput"
              device: "/job:worker/replica:2/device:CPU:1" }
-      node { name: "an_op_1" op: "an_op" }
-      node { name: "an_op_2" op: "an_op"
+      node { name: "FloatOutput_1" op: "FloatOutput" }
+      node { name: "FloatOutput_2" op: "FloatOutput"
              device: "/job:worker/replica:2/device:CPU:1" }
     """, gd)
 
@@ -968,32 +976,33 @@ class DeviceTest(test_util.TensorFlowTestCase):
   def testOverwritingBehavior(self):
     g = ops.Graph()
     with g.device(self._overwritingDeviceFunction):
-      g.create_op("an_op", [], [dtypes.float32])
+      g.create_op("FloatOutput", [], [dtypes.float32])
       with g.device("/job:ps"):  # Will be overwritten.
-        g.create_op("an_op", [], [dtypes.float32])
+        g.create_op("FloatOutput", [], [dtypes.float32])
       with g.device(pydev.merge_device("/job:ps")):  # Will be overwritten.
-        g.create_op("an_op", [], [dtypes.float32])
+        g.create_op("FloatOutput", [], [dtypes.float32])
       with g.device(None):  # Disables overwriting device function
         with g.device("/job:ps"):
-          g.create_op("an_op", [], [dtypes.float32])
+          g.create_op("FloatOutput", [], [dtypes.float32])
       with g.device(None):  # Disables overwriting device function
         with g.device(pydev.merge_device("/job:ps")):
-          g.create_op("an_op", [], [dtypes.float32])
+          g.create_op("FloatOutput", [], [dtypes.float32])
     gd = g.as_graph_def()
     self.assertProtoEqualsVersion("""
-      node { name: "an_op" op: "an_op"
+      node { name: "FloatOutput" op: "FloatOutput"
              device: "/job:overwrite" }
-      node { name: "an_op_1" op: "an_op"
+      node { name: "FloatOutput_1" op: "FloatOutput"
              device: "/job:overwrite" }
-      node { name: "an_op_2" op: "an_op"
+      node { name: "FloatOutput_2" op: "FloatOutput"
              device: "/job:overwrite" }
-      node { name: "an_op_3" op: "an_op"
+      node { name: "FloatOutput_3" op: "FloatOutput"
              device: "/job:ps" }
-      node { name: "an_op_4" op: "an_op"
+      node { name: "FloatOutput_4" op: "FloatOutput"
              device: "/job:ps" }
     """, gd)
 
 
+@test_util.with_c_api
 class ObjectWithName(object):
 
   def __init__(self, name):
@@ -1004,6 +1013,7 @@ class ObjectWithName(object):
     return self._name
 
 
+@test_util.with_c_api
 class CollectionTest(test_util.TensorFlowTestCase):
 
   def test_get_collections(self):
@@ -1112,18 +1122,10 @@ class CollectionTest(test_util.TensorFlowTestCase):
       self.assertEqual([90, 100], ops.get_collection("key"))
 
 
-def an_op(g):
-  return _apply_op(g, "an_op", [], [dtypes.float32])
-
+ops.NotDifferentiable("FloatOutput")
 
-ops.NotDifferentiable("an_op")
 
-
-def copy_op(x):
-  return _apply_op(x.graph, "copy", [x], [x.dtype])
-
-
-@ops.RegisterGradient("copy")
+@ops.RegisterGradient("CopyOp")
 def _CopyGrad(op, x_grad):  # pylint: disable=invalid-name
   _ = op
   return x_grad
@@ -1135,44 +1137,48 @@ def _CopyOverrideGrad(op, x_grad):  # pylint: disable=invalid-name
   return x_grad
 
 
+@test_util.with_c_api
 class RegistrationTest(test_util.TensorFlowTestCase):
 
   def testRegisterGradients(self):
-    g = ops.Graph()
-    x = an_op(g)
-    y = copy_op(x)
+    x = test_ops.float_output()
+    y = test_ops.copy_op(x)
     fn = ops.get_gradient_function(y.op)
     self.assertEqual(_CopyGrad, fn)
 
   def testOverrideGradients(self):
     g = ops.Graph()
-    x = an_op(g)
-    with g.gradient_override_map({"copy": "copy_override"}):
-      y = copy_op(x)
-    fn = ops.get_gradient_function(y.op)
-    self.assertEqual(_CopyOverrideGrad, fn)
+    with g.as_default():
+      x = test_ops.float_output()
+      with g.gradient_override_map({"CopyOp": "copy_override"}):
+        y = test_ops.copy_op(x)
+      fn = ops.get_gradient_function(y.op)
+      self.assertEqual(_CopyOverrideGrad, fn)
 
   def testNonExistentOverride(self):
     g = ops.Graph()
-    x = an_op(g)
-    with g.gradient_override_map({"copy": "unknown_override"}):
-      y = copy_op(x)
-    with self.assertRaisesRegexp(LookupError, "unknown_override"):
-      ops.get_gradient_function(y.op)
+    with g.as_default():
+      x = test_ops.float_output()
+      with g.gradient_override_map({"CopyOp": "unknown_override"}):
+        y = test_ops.copy_op(x)
+      with self.assertRaisesRegexp(LookupError, "unknown_override"):
+        ops.get_gradient_function(y.op)
 
 
+@test_util.with_c_api
 class ComparisonTest(test_util.TensorFlowTestCase):
 
   def testMembershipAllowed(self):
     g = ops.Graph()
-    t1 = _apply_op(g, "const", [], [dtypes.float32], name="myop1")
-    t2 = _apply_op(g, "const", [], [dtypes.float32], name="myop2")
+    t1 = _apply_op(g, "FloatOutput", [], [dtypes.float32], name="myop1")
+    t2 = _apply_op(g, "FloatOutput", [], [dtypes.float32], name="myop2")
     self.assertTrue(isinstance(t1, ops.Tensor))
     self.assertTrue(isinstance(t2, ops.Tensor))
     self.assertTrue(t1 in [t1])
     self.assertTrue(t1 not in [t2])
 
 
+@test_util.with_c_api
 class ControlDependenciesTest(test_util.TensorFlowTestCase):
 
   @test_util.enable_c_api
@@ -1198,7 +1204,7 @@ class ControlDependenciesTest(test_util.TensorFlowTestCase):
 
   def testBasicWithConversion(self):
     g = ops.Graph()
-    a = _apply_op(g, "const", [], [dtypes.float32])
+    a = _apply_op(g, "FloatOutput", [], [dtypes.float32])
 
     class ConvertibleObj(object):
 
@@ -1206,25 +1212,25 @@ class ControlDependenciesTest(test_util.TensorFlowTestCase):
         return a
 
     with g.control_dependencies([ConvertibleObj()]):
-      c = _apply_op(g, "const", [], [dtypes.float32])
+      c = _apply_op(g, "FloatOutput", [], [dtypes.float32])
 
     self.assertEqual(c.op.control_inputs, [a.op])
 
   def testNested(self):
     g = ops.Graph()
-    a_1 = _apply_op(g, "const", [], [dtypes.float32])
-    a_2 = _apply_op(g, "const", [], [dtypes.float32])
-    a_3 = _apply_op(g, "const", [], [dtypes.float32])
-    a_4 = _apply_op(g, "const", [], [dtypes.float32])
+    a_1 = _apply_op(g, "FloatOutput", [], [dtypes.float32])
+    a_2 = _apply_op(g, "FloatOutput", [], [dtypes.float32])
+    a_3 = _apply_op(g, "FloatOutput", [], [dtypes.float32])
+    a_4 = _apply_op(g, "FloatOutput", [], [dtypes.float32])
 
     with g.control_dependencies([a_1, a_2, a_3, a_4]):
-      b_1 = _apply_op(g, "const", [], [dtypes.float32])
+      b_1 = _apply_op(g, "FloatOutput", [], [dtypes.float32])
 
     with g.control_dependencies([a_1]):
       with g.control_dependencies([a_2]):
         with g.control_dependencies([a_3]):
           with g.control_dependencies([a_4]):
-            b_2 = _apply_op(g, "const", [], [dtypes.float32])
+            b_2 = _apply_op(g, "FloatOutput", [], [dtypes.float32])
 
     self.assertItemsEqual([a_1.op, a_2.op, a_3.op, a_4.op],
                           b_1.op.control_inputs)
@@ -1232,10 +1238,10 @@ class ControlDependenciesTest(test_util.TensorFlowTestCase):
 
   def testClear(self):
     g = ops.Graph()
-    a_1 = _apply_op(g, "const", [], [dtypes.float32])
-    a_2 = _apply_op(g, "const", [], [dtypes.float32])
-    a_3 = _apply_op(g, "const", [], [dtypes.float32])
-    a_4 = _apply_op(g, "const", [], [dtypes.float32])
+    a_1 = _apply_op(g, "FloatOutput", [], [dtypes.float32])
+    a_2 = _apply_op(g, "FloatOutput", [], [dtypes.float32])
+    a_3 = _apply_op(g, "FloatOutput", [], [dtypes.float32])
+    a_4 = _apply_op(g, "FloatOutput", [], [dtypes.float32])
 
     with g.control_dependencies([a_1]):
       with g.control_dependencies([a_2]):
@@ -1243,18 +1249,18 @@ class ControlDependenciesTest(test_util.TensorFlowTestCase):
           with g.control_dependencies([a_3]):
             with g.control_dependencies([a_4]):
               # deps [a_3, a_4]
-              b_3_4 = _apply_op(g, "const", [], [dtypes.float32])
+              b_3_4 = _apply_op(g, "FloatOutput", [], [dtypes.float32])
             # deps = [a_3]
-            b_3 = _apply_op(g, "const", [], [dtypes.float32])
+            b_3 = _apply_op(g, "FloatOutput", [], [dtypes.float32])
           # deps back to None
-          b_none = _apply_op(g, "const", [], [dtypes.float32])
+          b_none = _apply_op(g, "FloatOutput", [], [dtypes.float32])
         # deps back to [a_1, a_2]
-        b_1_2 = _apply_op(g, "const", [], [dtypes.float32])
+        b_1_2 = _apply_op(g, "FloatOutput", [], [dtypes.float32])
       # deps back to [a_1]
-      b_1 = _apply_op(g, "const", [], [dtypes.float32])
+      b_1 = _apply_op(g, "FloatOutput", [], [dtypes.float32])
       with g.control_dependencies(None):
         # deps are None again
-        b_none2 = _apply_op(g, "const", [], [dtypes.float32])
+        b_none2 = _apply_op(g, "FloatOutput", [], [dtypes.float32])
 
     self.assertItemsEqual([a_3.op, a_4.op], b_3_4.op.control_inputs)
     self.assertItemsEqual([a_3.op], b_3.op.control_inputs)
@@ -1274,31 +1280,46 @@ class ControlDependenciesTest(test_util.TensorFlowTestCase):
     # * Nodes d_i are defined as Mul(b_i, c_i) at each scope.
     # * Nodes e_i are defined as Mul(e_i-1, e_i-1) at each scope i > 1.
 
-    a_1 = _apply_op(g, "const", [], [dtypes.float32])
-    a_2 = _apply_op(g, "const", [], [dtypes.float32])
-    a_3 = _apply_op(g, "const", [], [dtypes.float32])
-    a_4 = _apply_op(g, "const", [], [dtypes.float32])
+    a_1 = _apply_op(g, "FloatOutput", [], [dtypes.float32])
+    a_2 = _apply_op(g, "FloatOutput", [], [dtypes.float32])
+    a_3 = _apply_op(g, "FloatOutput", [], [dtypes.float32])
+    a_4 = _apply_op(g, "FloatOutput", [], [dtypes.float32])
 
     with g.control_dependencies([a_1]):
-      b_1 = _apply_op(g, "mul", [a_3, a_4], [dtypes.float32])
-      c_1 = _apply_op(g, "mul", [a_1, b_1], [dtypes.float32])
-      d_1 = _apply_op(g, "mul", [b_1, c_1], [dtypes.float32])
-      e_1 = _apply_op(g, "const", [], [dtypes.float32])
+      b_1 = _apply_op(g, "TwoFloatInputsFloatOutput", [a_3, a_4],
+                      [dtypes.float32])
+      c_1 = _apply_op(g, "TwoFloatInputsFloatOutput", [a_1, b_1],
+                      [dtypes.float32])
+      d_1 = _apply_op(g, "TwoFloatInputsFloatOutput", [b_1, c_1],
+                      [dtypes.float32])
+      e_1 = _apply_op(g, "FloatOutput", [], [dtypes.float32])
       with g.control_dependencies([a_2]):
-        b_2 = _apply_op(g, "mul", [a_3, a_4], [dtypes.float32])
-        c_2 = _apply_op(g, "mul", [a_1, b_1], [dtypes.float32])
-        d_2 = _apply_op(g, "mul", [b_2, c_2], [dtypes.float32])
-        e_2 = _apply_op(g, "mul", [e_1, e_1], [dtypes.float32])
+        b_2 = _apply_op(g, "TwoFloatInputsFloatOutput", [a_3, a_4],
+                        [dtypes.float32])
+        c_2 = _apply_op(g, "TwoFloatInputsFloatOutput", [a_1, b_1],
+                        [dtypes.float32])
+        d_2 = _apply_op(g, "TwoFloatInputsFloatOutput", [b_2, c_2],
+                        [dtypes.float32])
+        e_2 = _apply_op(g, "TwoFloatInputsFloatOutput", [e_1, e_1],
+                        [dtypes.float32])
         with g.control_dependencies([a_3]):
-          b_3 = _apply_op(g, "mul", [a_3, a_4], [dtypes.float32])
-          c_3 = _apply_op(g, "mul", [a_1, b_1], [dtypes.float32])
-          d_3 = _apply_op(g, "mul", [b_3, c_3], [dtypes.float32])
-          e_3 = _apply_op(g, "mul", [e_2, e_2], [dtypes.float32])
+          b_3 = _apply_op(g, "TwoFloatInputsFloatOutput", [a_3, a_4],
+                          [dtypes.float32])
+          c_3 = _apply_op(g, "TwoFloatInputsFloatOutput", [a_1, b_1],
+                          [dtypes.float32])
+          d_3 = _apply_op(g, "TwoFloatInputsFloatOutput", [b_3, c_3],
+                          [dtypes.float32])
+          e_3 = _apply_op(g, "TwoFloatInputsFloatOutput", [e_2, e_2],
+                          [dtypes.float32])
           with g.control_dependencies([a_4]):
-            b_4 = _apply_op(g, "mul", [a_3, a_4], [dtypes.float32])
-            c_4 = _apply_op(g, "mul", [a_1, b_1], [dtypes.float32])
-            d_4 = _apply_op(g, "mul", [b_4, c_4], [dtypes.float32])
-            e_4 = _apply_op(g, "mul", [e_3, e_3], [dtypes.float32])
+            b_4 = _apply_op(g, "TwoFloatInputsFloatOutput", [a_3, a_4],
+                            [dtypes.float32])
+            c_4 = _apply_op(g, "TwoFloatInputsFloatOutput", [a_1, b_1],
+                            [dtypes.float32])
+            d_4 = _apply_op(g, "TwoFloatInputsFloatOutput", [b_4, c_4],
+                            [dtypes.float32])
+            e_4 = _apply_op(g, "TwoFloatInputsFloatOutput", [e_3, e_3],
+                            [dtypes.float32])
 
     self.assertItemsEqual([a_1.op], b_1.op.control_inputs)
     self.assertItemsEqual([a_1.op, a_2.op], b_2.op.control_inputs)
@@ -1322,25 +1343,26 @@ class ControlDependenciesTest(test_util.TensorFlowTestCase):
 
   def testRepeatedDependency(self):
     g = ops.Graph()
-    a = g.create_op("foo", [], [dtypes.float32, dtypes.float32])
+    a = g.create_op("TwoFloatOutputs", [], [dtypes.float32, dtypes.float32])
     a_0, a_1 = a.outputs
     with g.control_dependencies([a_0]):
-      b = _apply_op(g, "const", [], [dtypes.float32])
+      b = _apply_op(g, "FloatOutput", [], [dtypes.float32])
       with g.control_dependencies([a_1]):
-        c = _apply_op(g, "const", [], [dtypes.float32])
+        c = _apply_op(g, "FloatOutput", [], [dtypes.float32])
 
     self.assertEqual(b.op.control_inputs, [a])
     self.assertEqual(c.op.control_inputs, [a])
 
   def testNoControlDependencyWithDataDependency(self):
     g = ops.Graph()
-    a = _apply_op(g, "const", [], [dtypes.float32])
+    a = _apply_op(g, "FloatOutput", [], [dtypes.float32])
     with g.control_dependencies([a]):
-      b = _apply_op(g, "identity", [a], [dtypes.float32])
+      b = _apply_op(g, "Identity", [a], [dtypes.float32])
 
     self.assertEqual(b.op.control_inputs, [])
 
 
+@test_util.with_c_api
 class OpScopeTest(test_util.TensorFlowTestCase):
 
   @test_util.run_in_graph_and_eager_modes()
@@ -1353,8 +1375,8 @@ class OpScopeTest(test_util.TensorFlowTestCase):
   def testNoScopeName(self):
     g0 = ops.Graph()
     values = [
-        g0.create_op("a", [], [dtypes.float32]),
-        g0.create_op("b", [], [dtypes.float32])
+        g0.create_op("A", [], [dtypes.float32]),
+        g0.create_op("B", [], [dtypes.float32])
     ]
     with self.assertRaises(ValueError):
       with ops.name_scope(None, values=values):
@@ -1365,8 +1387,8 @@ class OpScopeTest(test_util.TensorFlowTestCase):
 
   def testEmptyScopeName(self):
     g0 = ops.Graph()
-    a = g0.create_op("a", [], [dtypes.float32])
-    b = g0.create_op("b", [], [dtypes.float32])
+    a = g0.create_op("A", [], [dtypes.float32])
+    b = g0.create_op("B", [], [dtypes.float32])
     with ops.name_scope("", values=[a, b]) as scope:
       self.assertEqual("", scope)
       self.assertEqual(g0, ops.get_default_graph())
@@ -1376,8 +1398,8 @@ class OpScopeTest(test_util.TensorFlowTestCase):
 
   def testDefaultScopeName(self):
     g0 = ops.Graph()
-    a = g0.create_op("a", [], [dtypes.float32])
-    b = g0.create_op("b", [], [dtypes.float32])
+    a = g0.create_op("A", [], [dtypes.float32])
+    b = g0.create_op("B", [], [dtypes.float32])
     scope_name = "my_scope"
     default_scope_name = "my_default_scope"
     with ops.name_scope(scope_name, default_scope_name, [a, b]) as scope:
@@ -1393,36 +1415,37 @@ class OpScopeTest(test_util.TensorFlowTestCase):
       self.assertEqual("%s/" % scope_name, scope)
       self.assertEqual(graph_elements[0].graph, ops.get_default_graph())
     g1 = ops.Graph()
-    c = g1.create_op("c", [], [dtypes.float32])
+    a = g1.create_op("A", [], [dtypes.float32])
     with self.assertRaises(ValueError):
-      with ops.name_scope(scope_name, values=graph_elements + [c]):
+      with ops.name_scope(scope_name, values=graph_elements + [a]):
         pass
 
   def testTensor(self):
     g0 = ops.Graph()
-    a = g0.create_op("a", [], [dtypes.float32])
-    b = g0.create_op("b", [], [dtypes.float32])
+    a = g0.create_op("A", [], [dtypes.float32])
+    b = g0.create_op("B", [], [dtypes.float32])
     self._testGraphElements([a, b])
 
   def testSparseTensor(self):
     g0 = ops.Graph()
-    a = g0.create_op("a", [], [dtypes.float32])
-    b = g0.create_op("b", [], [dtypes.float32])
+    a = g0.create_op("A", [], [dtypes.float32])
+    b = g0.create_op("B", [], [dtypes.float32])
     sparse = sparse_tensor.SparseTensor(
-        _apply_op(g0, "const", [], [dtypes.int64]),
-        _apply_op(g0, "const", [], [dtypes.float32]),
-        _apply_op(g0, "const", [], [dtypes.int64]))
+        _apply_op(g0, "Int64Output", [], [dtypes.int64]),
+        _apply_op(g0, "FloatOutput", [], [dtypes.float32]),
+        _apply_op(g0, "Int64Output", [], [dtypes.int64]))
     self._testGraphElements([a, sparse, b])
 
   def testVariable(self):
     g0 = ops.Graph()
     with g0.as_default():
       variable = variables.Variable([1.0])
-    a = g0.create_op("a", [], [dtypes.float32])
-    b = g0.create_op("b", [], [dtypes.float32])
+    a = g0.create_op("A", [], [dtypes.float32])
+    b = g0.create_op("B", [], [dtypes.float32])
     self._testGraphElements([a, variable, b])
 
 
+@test_util.with_c_api
 class GraphTest(test_util.TensorFlowTestCase):
 
   def setUp(self):
@@ -1461,14 +1484,14 @@ class GraphTest(test_util.TensorFlowTestCase):
     class ConvertibleObj(object):
 
       def _as_graph_element(self):
-        return "const:0"
+        return "FloatOutput:0"
 
     class NonConvertibleObj(object):
 
       pass
 
     g = ops.Graph()
-    a = _apply_op(g, "const", [], [dtypes.float32])
+    a = _apply_op(g, "FloatOutput", [], [dtypes.float32])
     self.assertEqual(a, g.as_graph_element(ConvertibleObj()))
     with self.assertRaises(TypeError):
       g.as_graph_element(NonConvertibleObj())
@@ -1500,6 +1523,7 @@ class GraphTest(test_util.TensorFlowTestCase):
     self.assertIsNone(g_ref())
 
 
+@test_util.with_c_api
 class AttrScopeTest(test_util.TensorFlowTestCase):
 
   def _get_test_attrs(self):
@@ -1551,8 +1575,10 @@ class AttrScopeTest(test_util.TensorFlowTestCase):
 ops.RegisterShape("KernelLabel")(common_shapes.scalar_shape)
 
 
+@test_util.with_c_api
 class KernelLabelTest(test_util.TensorFlowTestCase):
 
+  @test_util.enable_c_api
   def testNoLabel(self):
     with self.test_session():
       self.assertAllEqual(b"My label is: default",
@@ -1594,7 +1620,8 @@ class AsGraphDefTest(test_util.TensorFlowTestCase):
 
   def testAddShapes(self):
     with ops.Graph().as_default() as g:
-      t1, t2, t3, t4, t5 = _apply_op(g, "an_op", [], [dtypes.float32] * 5)
+      t1, t2, t3, t4, t5 = _apply_op(g, "FiveFloatOutputs", [],
+                                     [dtypes.float32] * 5)
       t1.set_shape(None)
       t2.set_shape([])
       t3.set_shape([None])
@@ -1603,7 +1630,7 @@ class AsGraphDefTest(test_util.TensorFlowTestCase):
 
       gd = g.as_graph_def(add_shapes=True)
       self.assertProtoEqualsVersion("""
-      node { name: "an_op" op: "an_op"
+      node { name: "FiveFloatOutputs" op: "FiveFloatOutputs"
         attr {
           key: "_output_shapes"
           value {
@@ -1625,6 +1652,7 @@ def _calc_a_forward_flops(unused_graph, unused_node):
   return ops.OpStats("flops", 20)
 
 
+@test_util.with_c_api
 class StatisticsTest(test_util.TensorFlowTestCase):
 
   def testRegisteredNode(self):
@@ -1649,6 +1677,7 @@ class StatisticsTest(test_util.TensorFlowTestCase):
     self.assertEqual(3, flops_total.value)
 
 
+@test_util.with_c_api
 class ColocationGroupTest(test_util.TensorFlowTestCase):
 
   def testBasic(self):
@@ -1773,9 +1802,13 @@ class ColocationGroupTest(test_util.TensorFlowTestCase):
     self.assertEqual("/device:CPU:0", b.device)
 
 
+@test_util.with_c_api
 class DeprecatedTest(test_util.TensorFlowTestCase):
 
   def testSuccess(self):
+    # TODO(skyewm): make g.graph_def_versions work with the C API enabled
+    if ops._USE_C_API: return
+
     with ops.Graph().as_default() as g:
       g.graph_def_versions.producer = 7
       old = test_ops.old()
@@ -1793,6 +1826,9 @@ class DeprecatedTest(test_util.TensorFlowTestCase):
         test_ops.old()
 
   def testGraphExecutionFail(self):
+    # TODO(skyewm): make g.graph_def_versions work with the C API enabled
+    if ops._USE_C_API: return
+
     with ops.Graph().as_default() as g:
       g.graph_def_versions.producer = 7
       old = test_ops.old()
@@ -1802,11 +1838,12 @@ class DeprecatedTest(test_util.TensorFlowTestCase):
           old.run()
 
 
+@test_util.with_c_api
 class DenseTensorLikeTypeTest(test_util.TensorFlowTestCase):
 
   def testSuccess(self):
     op = ops.Operation(
-        ops._NodeDef("noop", "myop"), ops.Graph(), [], [dtypes.float32])
+        ops._NodeDef("None", "myop"), ops.Graph(), [], [dtypes.float32])
     t = op.outputs[0]
     self.assertTrue(ops.is_dense_tensor_like(t))
 
@@ -1851,6 +1888,7 @@ class DenseTensorLikeTypeTest(test_util.TensorFlowTestCase):
           DenseTensorLikeTypeTest.BadClassBadDtype)
 
 
+@test_util.with_c_api
 class NameScopeTest(test_util.TensorFlowTestCase):
 
   def testStripAndPrependScope(self):
@@ -1889,6 +1927,7 @@ class NameScopeTest(test_util.TensorFlowTestCase):
       self.assertEqual("", g.get_name_scope())
 
 
+@test_util.with_c_api
 class TracebackTest(test_util.TensorFlowTestCase):
 
   def testTracebackWithStartLines(self):
@@ -1910,6 +1949,7 @@ class TracebackTest(test_util.TensorFlowTestCase):
           self.assertEquals(frame, frame_with_start_line[:-1])
 
 
+@test_util.with_c_api
 class OutputTypesTest(test_util.TensorFlowTestCase):
   """Tests Operation._output_types property.
 
@@ -1959,6 +1999,7 @@ class OutputTypesTest(test_util.TensorFlowTestCase):
       # pylint: enable=protected-access
 
 
+@test_util.with_c_api
 class InputTypesTest(test_util.TensorFlowTestCase):
   """Tests Operation._input_dtypes and Operation._input_types properties.
 
diff --git a/tensorflow/python/framework/test_ops.cc b/tensorflow/python/framework/test_ops.cc
index d22b5b3e25..ead756a0a1 100644
--- a/tensorflow/python/framework/test_ops.cc
+++ b/tensorflow/python/framework/test_ops.cc
@@ -170,4 +170,165 @@ class ResourceUsingOp : public OpKernel {
 REGISTER_KERNEL_BUILDER(Name("ResourceUsingOp").Device(DEVICE_CPU),
                         ResourceUsingOp);
 
+// Various test ops without kernels. These are used to test graph construction.
+
+REGISTER_OP("A")
+    .Output("out: float32")
+    .SetShapeFn(shape_inference::UnknownShape);
+
+REGISTER_OP("B")
+    .Output("out: float32")
+    .SetShapeFn(shape_inference::UnknownShape);
+
+REGISTER_OP("Foo1")
+    .Input("a: float32")
+    .Input("b: int32")
+    .Input("c: int32")
+    .Output("d: float32")
+    .Output("e: int32")
+    .SetShapeFn(shape_inference::UnknownShape);
+
+REGISTER_OP("Foo2")
+    .Input("a: float32")
+    .Input("b: string")
+    .Input("c: string")
+    .Output("d: float32")
+    .Output("e: int32")
+    .SetShapeFn(shape_inference::UnknownShape);
+
+REGISTER_OP("Foo3")
+    .Input("a: float32")
+    .Input("b: string")
+    .Input("c: float32")
+    .Output("d: float32")
+    .Output("e: int32")
+    .SetShapeFn(shape_inference::UnknownShape);
+
+REGISTER_OP("CopyOp").Input("a: T").Output("b: T").Attr("T: type").SetShapeFn(
+    shape_inference::UnknownShape);
+
+REGISTER_OP("None").SetShapeFn(shape_inference::UnknownShape);
+
+REGISTER_OP("IntOutput")
+    .Output("a: int32")
+    .SetShapeFn(shape_inference::UnknownShape);
+
+REGISTER_OP("Int64Output")
+    .Output("out: int64")
+    .SetShapeFn(shape_inference::UnknownShape);
+
+REGISTER_OP("RefOutput")
+    .Output("a: Ref(int32)")
+    .SetShapeFn(shape_inference::UnknownShape);
+
+REGISTER_OP("FloatOutput")
+    .Output("a: float32")
+    .SetShapeFn(shape_inference::UnknownShape);
+
+REGISTER_OP("TwoFloatOutputs")
+    .Output("a: float32")
+    .Output("b: float32")
+    .SetShapeFn(shape_inference::UnknownShape);
+
+REGISTER_OP("FiveFloatOutputs")
+    .Output("a: float32")
+    .Output("b: float32")
+    .Output("c: float32")
+    .Output("d: float32")
+    .Output("e: float32")
+    .SetShapeFn(shape_inference::UnknownShape);
+
+REGISTER_OP("RefOutputFloatOutput")
+    .Output("a: Ref(float32)")
+    .Output("b: float32")
+    .SetShapeFn(shape_inference::UnknownShape);
+
+REGISTER_OP("RefInputFloatInput")
+    .Input("a: Ref(float)")
+    .Input("b: float")
+    .SetShapeFn(shape_inference::UnknownShape);
+
+REGISTER_OP("IntInput")
+    .Input("a: int32")
+    .SetShapeFn(shape_inference::UnknownShape);
+
+REGISTER_OP("FloatInput")
+    .Input("a: float32")
+    .SetShapeFn(shape_inference::UnknownShape);
+
+REGISTER_OP("TwoIntOutputs")
+    .Output("a: int32")
+    .Output("b: int32")
+    .SetShapeFn(shape_inference::UnknownShape);
+
+REGISTER_OP("IntOutputFloatOutput")
+    .Output("a: int32")
+    .Output("b: float32")
+    .SetShapeFn(shape_inference::UnknownShape);
+
+REGISTER_OP("FloatOutputStringOutput")
+    .Output("a: float32")
+    .Output("b: string")
+    .SetShapeFn(shape_inference::UnknownShape);
+
+REGISTER_OP("TwoIntInputs")
+    .Input("a: int32")
+    .Input("b: int32")
+    .SetShapeFn(shape_inference::UnknownShape);
+
+REGISTER_OP("TwoFloatInputs")
+    .Input("a: float32")
+    .Input("b: float32")
+    .SetShapeFn(shape_inference::UnknownShape);
+
+REGISTER_OP("IntInputFloatInput")
+    .Input("a: int32")
+    .Input("b: float32")
+    .SetShapeFn(shape_inference::UnknownShape);
+
+REGISTER_OP("RefInputIntInput")
+    .Input("a: Ref(int32)")
+    .Input("b: int32")
+    .SetShapeFn(shape_inference::UnknownShape);
+
+REGISTER_OP("TwoFloatInputsFloatOutput")
+    .Input("a: float32")
+    .Input("b: float32")
+    .Output("c: float32")
+    .SetShapeFn(shape_inference::UnknownShape);
+
+REGISTER_OP("TwoFloatInputsIntOutput")
+    .Input("a: float32")
+    .Input("b: float32")
+    .Output("c: int32")
+    .SetShapeFn(shape_inference::UnknownShape);
+
+REGISTER_OP("RefInputFloatInputIntOutput")
+    .Input("a: Ref(float32)")
+    .Input("b: float32")
+    .Output("c: int32")
+    .SetShapeFn(shape_inference::UnknownShape);
+
+REGISTER_OP("ListInput")
+    .Input("a: N * T")
+    .Attr("N: int >= 1")
+    .Attr("T: type")
+    .SetShapeFn(shape_inference::UnknownShape);
+
+REGISTER_OP("ListOutput")
+    .Output("a: T")
+    .Attr("T: list(type) >= 1")
+    .SetShapeFn(shape_inference::UnknownShape);
+
+REGISTER_OP("Unary").Input("a: T").Output("b: T").Attr("T: type").SetShapeFn(
+    shape_inference::UnknownShape);
+
+REGISTER_OP("OpWithDefaultAttr")
+    .Output("a: int32")
+    .Attr("default_float: float = 123.0")
+    .SetShapeFn(shape_inference::UnknownShape);
+
+REGISTER_OP("OpWithFutureDefaultAttr")
+    .SetShapeFn(shape_inference::UnknownShape);
+
 }  // end namespace tensorflow
-- 
GitLab


From 1ba562a6878905c9967e999a73e749b59de56e21 Mon Sep 17 00:00:00 2001
From: Benoit Steiner <bsteiner@google.com>
Date: Mon, 9 Oct 2017 12:48:22 -0700
Subject: [PATCH 188/909] Rewrote the clip_by_norm op to avoid generating
 infinite intermediate results when processing tensors of zeros.

PiperOrigin-RevId: 171573629
---
 tensorflow/python/BUILD                | 15 ++++++++
 tensorflow/python/ops/clip_ops.py      |  8 ++---
 tensorflow/python/ops/clip_ops_test.py | 50 ++++++++++++++++++++++++++
 3 files changed, 68 insertions(+), 5 deletions(-)
 create mode 100644 tensorflow/python/ops/clip_ops_test.py

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index bdbad14660..1099611f37 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -1485,6 +1485,21 @@ py_library(
     ],
 )
 
+py_test(
+    name = "clip_ops_test",
+    size = "small",
+    srcs = ["ops/clip_ops_test.py"],
+    srcs_version = "PY2AND3",
+    tags = ["no_windows"],
+    deps = [
+        ":client_testlib",
+        ":clip_ops",
+        ":framework_for_generated_wrappers",
+        ":numerics",
+        "//third_party/py/numpy",
+    ],
+)
+
 py_library(
     name = "control_flow_grad",
     srcs = ["ops/control_flow_grad.py"],
diff --git a/tensorflow/python/ops/clip_ops.py b/tensorflow/python/ops/clip_ops.py
index 7430c28583..80803530c1 100644
--- a/tensorflow/python/ops/clip_ops.py
+++ b/tensorflow/python/ops/clip_ops.py
@@ -107,15 +107,13 @@ def clip_by_norm(t, clip_norm, axes=None, name=None):
     t = ops.convert_to_tensor(t, name="t")
 
     # Calculate L2-norm, clip elements by ratio of clip_norm to L2-norm
-    l2norm_inv = math_ops.rsqrt(
-        math_ops.reduce_sum(t * t, axes, keep_dims=True))
+    l2norm = math_ops.sqrt(math_ops.reduce_sum(t * t, axes, keep_dims=True))
     intermediate = t * clip_norm
     # Assert that the shape is compatible with the initial shape,
     # to prevent unintentional broadcasting.
     _ = t.shape.merge_with(intermediate.shape)
-    tclip = array_ops.identity(intermediate * math_ops.minimum(
-        l2norm_inv, constant_op.constant(1.0, dtype=t.dtype) / clip_norm),
-                               name=name)
+    tclip = array_ops.identity(
+        intermediate / math_ops.maximum(l2norm, clip_norm), name=name)
 
   return tclip
 
diff --git a/tensorflow/python/ops/clip_ops_test.py b/tensorflow/python/ops/clip_ops_test.py
new file mode 100644
index 0000000000..7d8dc90491
--- /dev/null
+++ b/tensorflow/python/ops/clip_ops_test.py
@@ -0,0 +1,50 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for Clip Operations."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.framework import constant_op
+from tensorflow.python.ops import clip_ops
+from tensorflow.python.ops import numerics
+from tensorflow.python.platform import test
+
+
+class ClipOpsTest(test.TestCase):
+
+  def __init__(self, method_name="runTest"):
+    super(ClipOpsTest, self).__init__(method_name)
+
+  def _testClipByNorm(self, inputs, max_norm, expected):
+    with self.test_session() as sess:
+      input_op = constant_op.constant(inputs)
+      clipped = clip_ops.clip_by_norm(input_op, max_norm)
+      check_op = numerics.add_check_numerics_ops()
+      result, _ = sess.run([clipped, check_op])
+    self.assertAllClose(result, expected)
+
+  def testClipByNorm(self):
+    # Simple example
+    self._testClipByNorm([[-3.0, 0.0, 0.0], [4.0, 0.0, 0.0]], 4.0,
+                         [[-2.4, 0.0, 0.0], [3.2, 0.0, 0.0]])
+    # Zero norm
+    self._testClipByNorm([[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]], 4.0,
+                         [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]])
+
+
+if __name__ == "__main__":
+  test.main()
-- 
GitLab


From 27df639673ae2bfe63b82862008da9bec488f0db Mon Sep 17 00:00:00 2001
From: Jingyue Wu <jingyue@google.com>
Date: Mon, 9 Oct 2017 13:00:39 -0700
Subject: [PATCH 189/909] [Grappler] Correctly replace control-dependency uses.

When redirecting the use of node A to node B, old code incorrectly replace
control dependencies with data dependencies.

PiperOrigin-RevId: 171575072
---
 .../optimizers/arithmetic_optimizer.cc        | 14 ++++++++---
 .../optimizers/arithmetic_optimizer_test.cc   | 25 +++++++++++++++++++
 2 files changed, 35 insertions(+), 4 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
index 343820de71..5c9073f049 100644
--- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
@@ -555,12 +555,18 @@ void ArithmeticOptimizer::SimplifyArithmeticOps(
       for (NodeDef* consumer : consumers) {
         // Update `consumer`'s use of `node` to `input`'s operand.
         for (int i = 0; i < consumer->input_size(); ++i) {
-          if (NodeName(consumer->input(i)) == node->name()) {
-            *consumer->mutable_input(i) = simplified_tensor;
+          int operand_pos;
+          string operand_node_name =
+              ParseNodeName(consumer->input(i), &operand_pos);
+          if (operand_node_name == node->name()) {
+            *consumer->mutable_input(i) =
+                (operand_pos < 0
+                     ? AsControlDependency(NodeName(simplified_tensor))
+                     : simplified_tensor);
           }
+          VLOG(2) << "Update input " << consumer->input(i) << " of "
+                  << consumer->name() << " to " << simplified_tensor;
         }
-        VLOG(2) << "Update input " << node->name() << " of " << consumer->name()
-                << " to " << simplified_tensor;
         node_map.UpdateInput(consumer->name(), node->name(), simplified_tensor);
         if (!nodes_to_simplify.Exists(consumer)) {
           nodes_to_simplify.PushBack(consumer);
diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc b/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc
index b3405646eb..7965419ea2 100644
--- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc
+++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc
@@ -240,6 +240,31 @@ TEST_F(ArithmeticOptimizerTest, RemoveInverseTransposesMultipleOutputs) {
   }
 }
 
+TEST_F(ArithmeticOptimizerTest, RemoveTransposesWithControlDependency) {
+  tensorflow::Scope s = tensorflow::Scope::NewRootScope();
+  Output inputs =
+      ops::Placeholder(s, DT_FLOAT, ops::Placeholder::Shape({2, 3}));
+  Output transpose1 = ops::Transpose(s, inputs, ops::Const(s, {1, 0}));
+  Output transpose2 = ops::Transpose(s, transpose1, ops::Const(s, {1, 0}));
+  Output outputs =
+      ops::Identity(s.WithOpName("outputs").WithControlDependencies(transpose2),
+                    ops::Const(s.WithOpName("outputs_const"), 1.0f));
+
+  GrapplerItem item;
+  item.fetch = {"outputs"};
+  TF_CHECK_OK(s.ToGraphDef(&item.graph));
+  GraphDef output;
+  TF_EXPECT_OK(ArithmeticOptimizer().Optimize(nullptr, item, &output));
+  item.graph = output;
+  TF_EXPECT_OK(ModelPruner().Optimize(nullptr, item, &output));
+
+  NodeMap node_map(&output);
+  const NodeDef* outputs_node = node_map.GetNode("outputs");
+  EXPECT_EQ(2, outputs_node->input_size());
+  EXPECT_EQ(outputs_node->input(0), "outputs_const");
+  EXPECT_EQ(outputs_node->input(1), "^Placeholder");
+}
+
 TEST_F(ArithmeticOptimizerTest, NotRemoveTransposes) {
   tensorflow::Scope s = tensorflow::Scope::NewRootScope();
   Output inputs_shape =
-- 
GitLab


From 11c123b43bd26d7829a927f2150622be84d57ef2 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 9 Oct 2017 13:19:07 -0700
Subject: [PATCH 190/909] [TF:XLA] Rename HLO visitor methods from LogicalX to
 X

PiperOrigin-RevId: 171577639
---
 tensorflow/compiler/tests/randomized_tests.cc | 12 ++++-----
 .../compiler/xla/client/lib/arithmetic.cc     | 10 +++----
 .../compiler/xla/client/lib/arithmetic.h      |  4 +--
 .../compiler/xla/service/dfs_hlo_visitor.h    | 17 ++++++------
 .../compiler/xla/service/hlo_evaluator.cc     | 27 +++++++++----------
 .../compiler/xla/service/hlo_instruction.cc   |  6 ++---
 tensorflow/compiler/xla/tests/reduce_test.cc  | 18 ++++++-------
 7 files changed, 45 insertions(+), 49 deletions(-)

diff --git a/tensorflow/compiler/tests/randomized_tests.cc b/tensorflow/compiler/tests/randomized_tests.cc
index 7e307f16af..fef12d9397 100644
--- a/tensorflow/compiler/tests/randomized_tests.cc
+++ b/tensorflow/compiler/tests/randomized_tests.cc
@@ -1791,28 +1791,28 @@ TEST_F(OpTest, Log1p) {
   });
 }
 
-TEST_F(OpTest, LogicalAnd) {
+TEST_F(OpTest, BooleanAnd) {
   Repeatedly([this]() {
     auto dims = BroadcastableDims();
     return ExpectTfAndXlaOutputsAreClose(
-        OpTestBuilder("LogicalAnd")
+        OpTestBuilder("BooleanAnd")
             .RandomInput(DT_BOOL, dims.first)
             .RandomInput(DT_BOOL, dims.second));
   });
 }
 
-TEST_F(OpTest, LogicalNot) {
+TEST_F(OpTest, BooleanNot) {
   Repeatedly([this]() {
     return ExpectTfAndXlaOutputsAreClose(
-        OpTestBuilder("LogicalNot").RandomInput(DT_BOOL));
+        OpTestBuilder("BooleanNot").RandomInput(DT_BOOL));
   });
 }
 
-TEST_F(OpTest, LogicalOr) {
+TEST_F(OpTest, BooleanOr) {
   Repeatedly([this]() {
     auto dims = BroadcastableDims();
     return ExpectTfAndXlaOutputsAreClose(
-        OpTestBuilder("LogicalOr")
+        OpTestBuilder("BooleanOr")
             .RandomInput(DT_BOOL, dims.first)
             .RandomInput(DT_BOOL, dims.second));
   });
diff --git a/tensorflow/compiler/xla/client/lib/arithmetic.cc b/tensorflow/compiler/xla/client/lib/arithmetic.cc
index 99e9f2dbb2..24048a1e5a 100644
--- a/tensorflow/compiler/xla/client/lib/arithmetic.cc
+++ b/tensorflow/compiler/xla/client/lib/arithmetic.cc
@@ -89,16 +89,16 @@ Computation CreateScalarMinComputation(PrimitiveType type,
          const ComputationDataHandle& rhs) { return b->Min(lhs, rhs); });
 }
 
-Computation CreateScalarLogicalAndComputation(ComputationBuilder* builder) {
+Computation CreateScalarAndComputation(ComputationBuilder* builder) {
   return CreateScalarComputation(
-      "logical_and", PRED, builder,
+      "and", PRED, builder,
       [](ComputationBuilder* b, const ComputationDataHandle& lhs,
          const ComputationDataHandle& rhs) { return b->And(lhs, rhs); });
 }
 
-Computation CreateScalarLogicalOrComputation(ComputationBuilder* builder) {
+Computation CreateScalarOrComputation(ComputationBuilder* builder) {
   return CreateScalarComputation(
-      "logical_or", PRED, builder,
+      "or", PRED, builder,
       [](ComputationBuilder* b, const ComputationDataHandle& lhs,
          const ComputationDataHandle& rhs) { return b->Or(lhs, rhs); });
 }
@@ -106,7 +106,7 @@ Computation CreateScalarLogicalOrComputation(ComputationBuilder* builder) {
 StatusOr<ComputationDataHandle> Any(const ComputationDataHandle& predicates,
                                     ComputationBuilder* builder) {
   auto f = builder->ConstantR0<bool>(false);
-  Computation logical_or = CreateScalarLogicalOrComputation(builder);
+  Computation logical_or = CreateScalarOrComputation(builder);
   TF_ASSIGN_OR_RETURN(std::unique_ptr<Shape> predicates_shape,
                       builder->GetShape(predicates));
   std::vector<int64> all_dimensions(ShapeUtil::Rank(*predicates_shape));
diff --git a/tensorflow/compiler/xla/client/lib/arithmetic.h b/tensorflow/compiler/xla/client/lib/arithmetic.h
index f43d35fe4a..ae89784bc2 100644
--- a/tensorflow/compiler/xla/client/lib/arithmetic.h
+++ b/tensorflow/compiler/xla/client/lib/arithmetic.h
@@ -45,10 +45,10 @@ Computation CreateScalarMinComputation(PrimitiveType type,
                                        ComputationBuilder* builder);
 
 // Creates a scalar logical AND computation and returns it.
-Computation CreateScalarLogicalAndComputation(ComputationBuilder* builder);
+Computation CreateScalarAndComputation(ComputationBuilder* builder);
 
 // Creates a scalar logical OR computation and returns it.
-Computation CreateScalarLogicalOrComputation(ComputationBuilder* builder);
+Computation CreateScalarOrComputation(ComputationBuilder* builder);
 
 // Returns whether any predicate in "predicates" is set.
 //
diff --git a/tensorflow/compiler/xla/service/dfs_hlo_visitor.h b/tensorflow/compiler/xla/service/dfs_hlo_visitor.h
index 2c16a1b903..8c864f3d07 100644
--- a/tensorflow/compiler/xla/service/dfs_hlo_visitor.h
+++ b/tensorflow/compiler/xla/service/dfs_hlo_visitor.h
@@ -156,17 +156,16 @@ class DfsHloVisitor {
                                 HloInstruction* operand) {
     return HandleElementwiseUnary(is_finite);
   }
-  virtual Status HandleLogicalAnd(HloInstruction* logical_and,
-                                  HloInstruction* lhs, HloInstruction* rhs) {
-    return HandleElementwiseBinary(logical_and);
+  virtual Status HandleAnd(HloInstruction* and_, HloInstruction* lhs,
+                           HloInstruction* rhs) {
+    return HandleElementwiseBinary(and_);
   }
-  virtual Status HandleLogicalNot(HloInstruction* logical_not,
-                                  HloInstruction* operand) {
-    return HandleElementwiseUnary(logical_not);
+  virtual Status HandleNot(HloInstruction* not_, HloInstruction* operand) {
+    return HandleElementwiseUnary(not_);
   }
-  virtual Status HandleLogicalOr(HloInstruction* logical_or,
-                                 HloInstruction* lhs, HloInstruction* rhs) {
-    return HandleElementwiseBinary(logical_or);
+  virtual Status HandleOr(HloInstruction* or_, HloInstruction* lhs,
+                          HloInstruction* rhs) {
+    return HandleElementwiseBinary(or_);
   }
   virtual Status HandleReducePrecision(HloInstruction* reduce_precision) {
     return HandleElementwiseUnary(reduce_precision);
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator.cc b/tensorflow/compiler/xla/service/hlo_evaluator.cc
index 61c59987f5..53e33c9fd0 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator.cc
+++ b/tensorflow/compiler/xla/service/hlo_evaluator.cc
@@ -255,12 +255,11 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault {
     return Status::OK();
   };
 
-  Status HandleLogicalNot(HloInstruction* logical_not,
-                          HloInstruction* operand) override {
-    TF_ASSIGN_OR_RETURN(
-        parent_->evaluated_[logical_not],
-        ElementWiseUnaryOp(logical_not,
-                           [](ReturnT elem_operand) { return !elem_operand; }));
+  Status HandleNot(HloInstruction* not_, HloInstruction* operand) override {
+    TF_ASSIGN_OR_RETURN(parent_->evaluated_[not_],
+                        ElementWiseUnaryOp(not_, [](ReturnT elem_operand) {
+                          return !elem_operand;
+                        }));
     return Status::OK();
   };
 
@@ -368,21 +367,21 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault {
     return Status::OK();
   };
 
-  Status HandleLogicalAnd(HloInstruction* logical_and, HloInstruction* lhs,
-                          HloInstruction* rhs) override {
+  Status HandleAnd(HloInstruction* and_, HloInstruction* lhs,
+                   HloInstruction* rhs) override {
     TF_ASSIGN_OR_RETURN(
-        parent_->evaluated_[logical_and],
-        ElementWiseBinaryOp(logical_and, [](ReturnT lhs_el, ReturnT rhs_el) {
+        parent_->evaluated_[and_],
+        ElementWiseBinaryOp(and_, [](ReturnT lhs_el, ReturnT rhs_el) {
           return lhs_el && rhs_el;
         }));
     return Status::OK();
   };
 
-  Status HandleLogicalOr(HloInstruction* logical_or, HloInstruction* lhs,
-                         HloInstruction* rhs) override {
+  Status HandleOr(HloInstruction* or_, HloInstruction* lhs,
+                  HloInstruction* rhs) override {
     TF_ASSIGN_OR_RETURN(
-        parent_->evaluated_[logical_or],
-        ElementWiseBinaryOp(logical_or, [](ReturnT lhs_el, ReturnT rhs_el) {
+        parent_->evaluated_[or_],
+        ElementWiseBinaryOp(or_, [](ReturnT lhs_el, ReturnT rhs_el) {
           return lhs_el || rhs_el;
         }));
     return Status::OK();
diff --git a/tensorflow/compiler/xla/service/hlo_instruction.cc b/tensorflow/compiler/xla/service/hlo_instruction.cc
index 77a748163e..81bccfddbb 100644
--- a/tensorflow/compiler/xla/service/hlo_instruction.cc
+++ b/tensorflow/compiler/xla/service/hlo_instruction.cc
@@ -1958,9 +1958,9 @@ Status HloInstruction::Visit(DfsHloVisitor* visitor) {
     case HloOpcode::kMinimum:
       return visitor->HandleMinimum(this);
     case HloOpcode::kAnd:
-      return visitor->HandleLogicalAnd(this, operands_[0], operands_[1]);
+      return visitor->HandleAnd(this, operands_[0], operands_[1]);
     case HloOpcode::kOr:
-      return visitor->HandleLogicalOr(this, operands_[0], operands_[1]);
+      return visitor->HandleOr(this, operands_[0], operands_[1]);
     case HloOpcode::kConcatenate:
       return visitor->HandleConcatenate(this, operands_);
     case HloOpcode::kConvert:
@@ -2017,7 +2017,7 @@ Status HloInstruction::Visit(DfsHloVisitor* visitor) {
     case HloOpcode::kIsFinite:
       return visitor->HandleIsFinite(this, operands_[0]);
     case HloOpcode::kNot:
-      return visitor->HandleLogicalNot(this, operands_[0]);
+      return visitor->HandleNot(this, operands_[0]);
     case HloOpcode::kBitcast:
       return visitor->HandleBitcast(this);
     case HloOpcode::kBroadcast:
diff --git a/tensorflow/compiler/xla/tests/reduce_test.cc b/tensorflow/compiler/xla/tests/reduce_test.cc
index 2271f32c59..b48b3a2bdb 100644
--- a/tensorflow/compiler/xla/tests/reduce_test.cc
+++ b/tensorflow/compiler/xla/tests/reduce_test.cc
@@ -120,10 +120,10 @@ class ReduceTest : public ClientLibraryTestBase {
     Computation reduce;
     if (and_reduce) {
       init_value = builder.ConstantR0<bool>(true);
-      reduce = CreateScalarLogicalAndComputation(&builder);
+      reduce = CreateScalarAndComputation(&builder);
     } else {
       init_value = builder.ConstantR0<bool>(false);
-      reduce = CreateScalarLogicalOrComputation(&builder);
+      reduce = CreateScalarOrComputation(&builder);
     }
     builder.Reduce(pred_values, init_value, reduce,
                    /*dimensions_to_reduce=*/{0});
@@ -729,16 +729,14 @@ XLA_TEST_F(ReduceTest, VectorizedReduce_Min) {
                           std::numeric_limits<uint32>::max());
 }
 
-XLA_TEST_F(ReduceTest, VectorizedReduce_LogicalAnd) {
-  RunVectorizedReduceTestForType<bool>(CreateScalarLogicalAndComputation,
-                                       [](bool a, bool b) { return a && b; },
-                                       true);
+XLA_TEST_F(ReduceTest, VectorizedReduce_BooleanAnd) {
+  RunVectorizedReduceTestForType<bool>(
+      CreateScalarAndComputation, [](bool a, bool b) { return a && b; }, true);
 }
 
-XLA_TEST_F(ReduceTest, VectorizedReduce_LogicalOr) {
-  RunVectorizedReduceTestForType<bool>(CreateScalarLogicalOrComputation,
-                                       [](bool a, bool b) { return a || b; },
-                                       false);
+XLA_TEST_F(ReduceTest, VectorizedReduce_BooleanOr) {
+  RunVectorizedReduceTestForType<bool>(
+      CreateScalarOrComputation, [](bool a, bool b) { return a || b; }, false);
 }
 
 class ReduceR3ToR2Test : public ReduceTest,
-- 
GitLab


From 0ac688a18cc56816d8c767f7fcbce97b05b2319e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 9 Oct 2017 13:21:22 -0700
Subject: [PATCH 191/909] Adding a binary classification example

PiperOrigin-RevId: 171577979
---
 tensorflow/contrib/boosted_trees/README.md    |  11 ++
 .../boosted_trees/examples/binary_mnist.py    | 169 ++++++++++++++++++
 .../contrib/boosted_trees/examples/boston.py  |   2 -
 3 files changed, 180 insertions(+), 2 deletions(-)
 create mode 100644 tensorflow/contrib/boosted_trees/README.md
 create mode 100644 tensorflow/contrib/boosted_trees/examples/binary_mnist.py

diff --git a/tensorflow/contrib/boosted_trees/README.md b/tensorflow/contrib/boosted_trees/README.md
new file mode 100644
index 0000000000..9ce700f1a1
--- /dev/null
+++ b/tensorflow/contrib/boosted_trees/README.md
@@ -0,0 +1,11 @@
+# TF Boosted Trees (TFBT)
+
+TF Boosted trees is an implementation of a gradient boosting algorithm with
+trees used as week learners.
+
+## Examples
+Folder "examples" demonstrates how TFBT estimators can be used for various
+problems. Namely, it contains:
+* binary_mnist.py - an example on how to use TFBT for binary classification.
+* mnist.py - a multiclass example.
+* boston.py - a regression example.
\ No newline at end of file
diff --git a/tensorflow/contrib/boosted_trees/examples/binary_mnist.py b/tensorflow/contrib/boosted_trees/examples/binary_mnist.py
new file mode 100644
index 0000000000..9be362f5c8
--- /dev/null
+++ b/tensorflow/contrib/boosted_trees/examples/binary_mnist.py
@@ -0,0 +1,169 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+r"""Demonstrates multiclass MNIST TF Boosted trees example.
+
+  This example demonstrates how to run experiments with TF Boosted Trees on
+  a binary dataset. We use digits 4 and 9 from the original MNIST dataset.
+
+  Example Usage:
+  python tensorflow/contrib/boosted_trees/examples/binary_mnist.py \
+  --output_dir="/tmp/binary_mnist" --depth=4 --learning_rate=0.3 \
+  --batch_size=10761 --examples_per_layer=10761 --eval_batch_size=1030 \
+  --num_eval_steps=1 --num_trees=10 --l2=1 --vmodule=training_ops=1 \
+
+  When training is done, accuracy on eval data is reported. Point tensorboard
+  to the directory for the run to see how the training progresses:
+
+  tensorboard --logdir=/tmp/binary_mnist
+
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import argparse
+import sys
+
+import numpy as np
+import tensorflow as tf
+from tensorflow.contrib.boosted_trees.estimator_batch.estimator import GradientBoostedDecisionTreeClassifier
+from tensorflow.contrib.boosted_trees.proto import learner_pb2
+from tensorflow.contrib.learn import learn_runner
+
+
+def get_input_fn(data,
+                 batch_size,
+                 capacity=10000,
+                 min_after_dequeue=3000):
+  """Input function over MNIST data."""
+  # Keep only 4 and 9 digits.
+  ids = np.where((data.labels == 4) | (data.labels == 9))
+  images = data.images[ids]
+  labels = data.labels[ids]
+  # Make digit 4 label 0, 9 is 1.
+  labels = labels == 4
+
+  def _input_fn():
+    """Prepare features and labels."""
+    images_batch, labels_batch = tf.train.shuffle_batch(
+        tensors=[images,
+                 labels.astype(np.int32)],
+        batch_size=batch_size,
+        capacity=capacity,
+        min_after_dequeue=min_after_dequeue,
+        enqueue_many=True,
+        num_threads=4)
+    features_map = {"images": images_batch}
+    return features_map, labels_batch
+
+  return _input_fn
+
+
+# Main config - creates a TF Boosted Trees Estimator based on flags.
+def _get_tfbt(output_dir):
+  """Configures TF Boosted Trees estimator based on flags."""
+  learner_config = learner_pb2.LearnerConfig()
+
+  learner_config.learning_rate_tuner.fixed.learning_rate = FLAGS.learning_rate
+  learner_config.regularization.l1 = 0.0
+  learner_config.regularization.l2 = FLAGS.l2 / FLAGS.examples_per_layer
+  learner_config.constraints.max_tree_depth = FLAGS.depth
+
+  growing_mode = learner_pb2.LearnerConfig.LAYER_BY_LAYER
+  learner_config.growing_mode = growing_mode
+  run_config = tf.contrib.learn.RunConfig(save_checkpoints_secs=300)
+
+  # Create a TF Boosted trees estimator that can take in custom loss.
+  estimator = GradientBoostedDecisionTreeClassifier(
+      learner_config=learner_config,
+      examples_per_layer=FLAGS.examples_per_layer,
+      model_dir=output_dir,
+      num_trees=FLAGS.num_trees,
+      center_bias=False,
+      config=run_config)
+  return estimator
+
+
+def _make_experiment_fn(output_dir):
+  """Creates experiment for gradient boosted decision trees."""
+  data = tf.contrib.learn.datasets.mnist.load_mnist()
+  train_input_fn = get_input_fn(data.train, FLAGS.batch_size)
+  eval_input_fn = get_input_fn(data.validation, FLAGS.eval_batch_size)
+
+  return tf.contrib.learn.Experiment(
+      estimator=_get_tfbt(output_dir),
+      train_input_fn=train_input_fn,
+      eval_input_fn=eval_input_fn,
+      train_steps=None,
+      eval_steps=FLAGS.num_eval_steps,
+      eval_metrics=None)
+
+
+def main(unused_argv):
+  learn_runner.run(
+      experiment_fn=_make_experiment_fn,
+      output_dir=FLAGS.output_dir,
+      schedule="train_and_evaluate")
+
+
+if __name__ == "__main__":
+  tf.logging.set_verbosity(tf.logging.INFO)
+  parser = argparse.ArgumentParser()
+  # Define the list of flags that users can change.
+  parser.add_argument(
+      "--output_dir",
+      type=str,
+      required=True,
+      help="Choose the dir for the output.")
+  parser.add_argument(
+      "--batch_size",
+      type=int,
+      default=1000,
+      help="The batch size for reading data.")
+  parser.add_argument(
+      "--eval_batch_size",
+      type=int,
+      default=1000,
+      help="Size of the batch for eval.")
+  parser.add_argument(
+      "--num_eval_steps",
+      type=int,
+      default=1,
+      help="The number of steps to run evaluation for.")
+  # Flags for gradient boosted trees config.
+  parser.add_argument(
+      "--depth", type=int, default=4, help="Maximum depth of weak learners.")
+  parser.add_argument(
+      "--l2", type=float, default=1.0, help="l2 regularization per batch.")
+  parser.add_argument(
+      "--learning_rate",
+      type=float,
+      default=0.1,
+      help="Learning rate (shrinkage weight) with which each new tree is added."
+  )
+  parser.add_argument(
+      "--examples_per_layer",
+      type=int,
+      default=1000,
+      help="Number of examples to accumulate stats for per layer.")
+  parser.add_argument(
+      "--num_trees",
+      type=int,
+      default=None,
+      required=True,
+      help="Number of trees to grow before stopping.")
+
+  FLAGS, unparsed = parser.parse_known_args()
+  tf.app.run(main=main, argv=[sys.argv[0]] + unparsed)
diff --git a/tensorflow/contrib/boosted_trees/examples/boston.py b/tensorflow/contrib/boosted_trees/examples/boston.py
index 0cb9e956ef..2c0a3c4912 100644
--- a/tensorflow/contrib/boosted_trees/examples/boston.py
+++ b/tensorflow/contrib/boosted_trees/examples/boston.py
@@ -44,8 +44,6 @@ from tensorflow.contrib.boosted_trees.proto import learner_pb2
 from tensorflow.contrib.layers.python.layers import feature_column
 from tensorflow.contrib.learn import learn_runner
 
-_TEST_SPLIT_RATIO = 0.2
-_TEST_SPLIT_SEED = 42
 _BOSTON_NUM_FEATURES = 13
 
 
-- 
GitLab


From 7e4e336ce5b874fadf8024b6a9c90e1bc8ed2867 Mon Sep 17 00:00:00 2001
From: Michael Case <mikecase@google.com>
Date: Mon, 9 Oct 2017 13:31:15 -0700
Subject: [PATCH 192/909] Relanding change to add config to enable S3 file
 system support.

Pass --config=s3 argument to Bazel to build with S3 file system support.
Change was originally rolled back due to a failure it caused in
//tensorflow/core/kernels:control_flow_ops_test on Macs which is now fixed.

PiperOrigin-RevId: 171579378
---
 configure.py                                      | 2 ++
 tensorflow/BUILD                                  | 6 ++++++
 tensorflow/core/platform/default/build_config.bzl | 5 +++++
 3 files changed, 13 insertions(+)

diff --git a/configure.py b/configure.py
index 9ca614f8f9..9da49b628d 100644
--- a/configure.py
+++ b/configure.py
@@ -991,6 +991,8 @@ def main():
                 'with_gcp_support', False, 'gcp')
   set_build_var(environ_cp, 'TF_NEED_HDFS', 'Hadoop File System',
                 'with_hdfs_support', False, 'hdfs')
+  set_build_var(environ_cp, 'TF_NEED_S3', 'Amazon S3 File System',
+                'with_s3_support', True, 's3')
   set_build_var(environ_cp, 'TF_ENABLE_XLA', 'XLA JIT', 'with_xla_support',
                 False, 'xla')
   set_build_var(environ_cp, 'TF_NEED_GDR', 'GDR', 'with_gdr_support',
diff --git a/tensorflow/BUILD b/tensorflow/BUILD
index 56d0939023..1620bb5f2a 100644
--- a/tensorflow/BUILD
+++ b/tensorflow/BUILD
@@ -185,6 +185,12 @@ config_setting(
     visibility = ["//visibility:public"],
 )
 
+config_setting(
+    name = "with_s3_support",
+    values = {"define": "with_s3_support=true"},
+    visibility = ["//visibility:public"],
+)
+
 config_setting(
     name = "with_xla_support",
     values = {"define": "with_xla_support=true"},
diff --git a/tensorflow/core/platform/default/build_config.bzl b/tensorflow/core/platform/default/build_config.bzl
index 51d37291ee..2c14ea917c 100644
--- a/tensorflow/core/platform/default/build_config.bzl
+++ b/tensorflow/core/platform/default/build_config.bzl
@@ -465,6 +465,11 @@ def tf_additional_core_deps():
           "//tensorflow/core/platform/hadoop:hadoop_file_system",
       ],
       "//conditions:default": [],
+  }) + select({
+      "//tensorflow:with_s3_support": [
+          "//tensorflow/contrib/s3:s3_file_system",
+      ],
+      "//conditions:default": [],
   })
 
 # TODO(jart, jhseu): Delete when GCP is default on.
-- 
GitLab


From 7c74d2f68a9d4737c85606c41435555189d3dc44 Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Mon, 9 Oct 2017 13:44:11 -0700
Subject: [PATCH 193/909] Expose tfe.test, tfe.in_eager_mode, tfe.in_graph_mode

All are useful for library writers.

PiperOrigin-RevId: 171581311
---
 tensorflow/contrib/eager/python/tfe.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/tensorflow/contrib/eager/python/tfe.py b/tensorflow/contrib/eager/python/tfe.py
index 249aaebea2..fbdc576739 100644
--- a/tensorflow/contrib/eager/python/tfe.py
+++ b/tensorflow/contrib/eager/python/tfe.py
@@ -47,6 +47,9 @@ To use, at program startup, call `tfe.enable_eager_execution()`.
 @@SummaryWriter
 @@restore_variables_on_create
 @@Variable
+
+@@in_eager_mode
+@@in_graph_mode
 """
 
 from __future__ import absolute_import
@@ -65,6 +68,8 @@ from tensorflow.python.eager import backprop
 from tensorflow.python.eager.custom_gradient import custom_gradient
 from tensorflow.python.eager import function
 from tensorflow.python.eager.context import enable_eager_execution
+from tensorflow.python.eager.context import in_eager_mode
+from tensorflow.python.eager.context import in_graph_mode
 from tensorflow.python.eager.context import list_devices
 from tensorflow.python.eager.context import num_gpus
 from tensorflow.python.eager.context import run
-- 
GitLab


From be69f13a074013a9c0322822e83b6320ef6c52bc Mon Sep 17 00:00:00 2001
From: Peter Hawkins <phawkins@google.com>
Date: Mon, 9 Oct 2017 14:21:44 -0700
Subject: [PATCH 194/909] [TF:XLA] Fix broken build of xla_interpreter_device.

PiperOrigin-RevId: 171586211
---
 tensorflow/compiler/jit/xla_interpreter_device.cc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/compiler/jit/xla_interpreter_device.cc b/tensorflow/compiler/jit/xla_interpreter_device.cc
index 4e4cbe200a..2614deefd8 100644
--- a/tensorflow/compiler/jit/xla_interpreter_device.cc
+++ b/tensorflow/compiler/jit/xla_interpreter_device.cc
@@ -42,9 +42,9 @@ Status XlaInterpreterDeviceFactory::CreateDevices(
   (void)registrations;
 
   std::unique_ptr<XlaDevice> device;
-  TF_RETURN_IF_ERROR(XlaDevice::Create("Interpreter", DEVICE_XLA_INTERPRETER, 0,
-                                       DEVICE_INTERPRETER_XLA_JIT, options,
-                                       name_prefix, &device));
+  TF_RETURN_IF_ERROR(XlaDevice::Create(
+      "Interpreter", DEVICE_XLA_INTERPRETER, 0, DEVICE_INTERPRETER_XLA_JIT,
+      options, name_prefix, /*register_device_for_compilation=*/true, &device));
   devices->push_back(device.release());
   return Status::OK();
 }
-- 
GitLab


From 33d55122d994d12f2a066f9ec4f0f03094a59579 Mon Sep 17 00:00:00 2001
From: Jingyue Wu <jingyue@google.com>
Date: Mon, 9 Oct 2017 15:18:44 -0700
Subject: [PATCH 195/909] [Grappler] Fixed two bugs in ArithmeticOptimizer.

1. The data type of Mul should be stored in key "T" instead of "dtype".
2. Add consumer_of_mul to new_nodes because it is modified. This caused
  Grappler to miss some optimizations.

PiperOrigin-RevId: 171594972
---
 tensorflow/core/grappler/optimizers/BUILD     |  1 +
 .../optimizers/arithmetic_optimizer.cc        |  3 +-
 .../optimizers/arithmetic_optimizer_test.cc   | 53 +++++++++++++++++++
 3 files changed, 56 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/grappler/optimizers/BUILD b/tensorflow/core/grappler/optimizers/BUILD
index c4def6cf23..06a62f2a00 100644
--- a/tensorflow/core/grappler/optimizers/BUILD
+++ b/tensorflow/core/grappler/optimizers/BUILD
@@ -178,6 +178,7 @@ tf_cc_test(
     srcs = ["arithmetic_optimizer_test.cc"],
     deps = [
         ":arithmetic_optimizer",
+        ":constant_folding",
         ":model_pruner",
         "//tensorflow/cc:cc_ops",
         "//tensorflow/core:protos_all_cc",
diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
index 5c9073f049..3ec62b5a00 100644
--- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
@@ -465,7 +465,7 @@ string ArithmeticOptimizer::TrySimplifyAndReplaceUses(
             scaled_weights->set_name(weights->name() + "_scaled");
             scaled_weights->set_op("Mul");
             scaled_weights->set_device(weights->device());
-            (*scaled_weights->mutable_attr())["dtype"] =
+            (*scaled_weights->mutable_attr())["T"] =
                 weights->attr().at("dtype");
             node_map->AddNode(scaled_weights->name(), scaled_weights);
             new_nodes->push_back(scaled_weights);
@@ -490,6 +490,7 @@ string ArithmeticOptimizer::TrySimplifyAndReplaceUses(
             consumer_of_mul->set_input(0, mul->input(0));
             node_map->UpdateInput(consumer_of_mul->name(), mul->name(),
                                   other->name());
+            new_nodes->push_back(consumer_of_mul);
             return conv->name();
           }
         }
diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc b/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc
index 7965419ea2..234c096073 100644
--- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc
+++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc
@@ -18,6 +18,7 @@ limitations under the License.
 #include "tensorflow/core/framework/node_def.pb.h"
 #include "tensorflow/core/grappler/grappler_item.h"
 #include "tensorflow/core/grappler/inputs/trivial_test_graph_input_yielder.h"
+#include "tensorflow/core/grappler/optimizers/constant_folding.h"
 #include "tensorflow/core/grappler/optimizers/model_pruner.h"
 #include "tensorflow/core/grappler/utils.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
@@ -397,6 +398,58 @@ TEST_F(ArithmeticOptimizerTest, FoldMulToConv) {
   CHECK_EQ(node_map.GetNode(NodeName(folded_conv->input(1)))->op(), "Mul");
 }
 
+TEST_F(ArithmeticOptimizerTest, OptimizeCastMulTransposeConv) {
+  // This unit test exercises two optimizations, folding mul into conv, and
+  // reordering cast and transpose.
+  //
+  //   Conv2D(Transpose(Mul(Cast(I), S)), W)
+  //     =>
+  //   Conv2D(Transpose(Cast(I)), W*S)
+  //     =>
+  //   Conv2D(Cast(Transpose(I)), W*S)
+  tensorflow::Scope s = tensorflow::Scope::NewRootScope().WithDevice("/gpu:0");
+  Output inputs =
+      ops::Placeholder(s, DT_UINT8, ops::Placeholder::Shape({8, 28, 28, 3}));
+  Output cast = ops::Cast(s, inputs, DT_FLOAT);
+  Output mul = ops::Mul(s, cast, ops::Const(s, 1.0f / 255.0f));
+  Output transpose =
+      ops::Transpose(s, mul, ops::Const(s.WithOpName("perm"), {0, 3, 1, 2}));
+  Output weights = ops::Const(s.WithOpName("weights"),
+                              Input::Initializer(127.0f, {5, 5, 3, 16}));
+  Output conv = ops::Conv2D(s, transpose, weights, {1, 1, 1, 1}, "VALID",
+                            ops::Conv2D::DataFormat("NCHW"));
+  Output outputs = ops::Identity(s.WithOpName("outputs"), conv);
+
+  GrapplerItem item;
+  item.fetch = {"outputs"};
+  TF_CHECK_OK(s.ToGraphDef(&item.graph));
+
+  GraphDef output;
+  TF_EXPECT_OK(ArithmeticOptimizer().Optimize(nullptr, item, &output));
+
+  item.graph = output;
+  TF_EXPECT_OK(
+      ConstantFolding(/*cpu_device=*/nullptr).Optimize(nullptr, item, &output));
+
+  item.graph = output;
+  TF_EXPECT_OK(ModelPruner().Optimize(nullptr, item, &output));
+
+  NodeMap node_map(&output);
+  const NodeDef* inputs_node = CHECK_NOTNULL(node_map.GetNode("Placeholder"));
+  const NodeDef* transpose_node =
+      CHECK_NOTNULL(node_map.GetNode("Transpose_uint8"));
+  const NodeDef* cast_node = CHECK_NOTNULL(node_map.GetNode("Cast_new"));
+  const NodeDef* weights_node =
+      CHECK_NOTNULL(node_map.GetNode("weights_scaled"));
+  const NodeDef* conv_node = CHECK_NOTNULL(node_map.GetNode("Conv2D"));
+
+  EXPECT_EQ(output.node_size(), 7);
+  EXPECT_EQ(transpose_node->input(0), inputs_node->name());
+  EXPECT_EQ(cast_node->input(0), transpose_node->name());
+  EXPECT_EQ(conv_node->input(0), cast_node->name());
+  EXPECT_EQ(conv_node->input(1), weights_node->name());
+}
+
 }  // namespace
 }  // namespace grappler
 }  // namespace tensorflow
-- 
GitLab


From 88145023cea47b4a96cc04f8febe205d50a0d0d6 Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Mon, 9 Oct 2017 16:24:05 -0700
Subject: [PATCH 196/909] Removing side outputs from tape code.

They belong better in future function objects (simplifies tape move to C)

PiperOrigin-RevId: 171603665
---
 tensorflow/python/eager/backprop.py        |  2 +-
 tensorflow/python/eager/custom_gradient.py |  1 -
 tensorflow/python/eager/function.py        |  8 +++++---
 tensorflow/python/eager/imperative_grad.py |  3 +--
 tensorflow/python/eager/tape.py            | 19 +++----------------
 tensorflow/python/framework/ops.py         |  2 +-
 6 files changed, 11 insertions(+), 24 deletions(-)

diff --git a/tensorflow/python/eager/backprop.py b/tensorflow/python/eager/backprop.py
index cca8e47044..554b9a818c 100644
--- a/tensorflow/python/eager/backprop.py
+++ b/tensorflow/python/eager/backprop.py
@@ -281,7 +281,7 @@ def _record_gradient(op_name, inputs, attrs, results, name):
             "output_grads", orig_outputs, "gradients", result)
     return nest.flatten(result)
 
-  tape.record_operation(op_name, results, inputs, [], grad_fn)
+  tape.record_operation(op_name, results, inputs, grad_fn)
   if _tracing:
     print("Computed op", (name if name else op_name), "inputs", inputs,
           "outputs", results)
diff --git a/tensorflow/python/eager/custom_gradient.py b/tensorflow/python/eager/custom_gradient.py
index 4360e53225..87348e87b1 100644
--- a/tensorflow/python/eager/custom_gradient.py
+++ b/tensorflow/python/eager/custom_gradient.py
@@ -85,7 +85,6 @@ def custom_gradient(f):
         f.__name__,
         flat_result,
         input_tensors,
-        [],
         actual_grad_fn)
     flat_result = list(flat_result)
     return result
diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py
index 8a1936b3fe..da49517cf9 100644
--- a/tensorflow/python/eager/function.py
+++ b/tensorflow/python/eager/function.py
@@ -109,7 +109,7 @@ def _convert_to_graph_tensor(value, dtype=None, name=None, as_ref=False):
     tensor_map[ops.tensor_id(value)] = (value, captured_value)
   else:
     captured_value = captured_value[1]
-  tape.record_operation("captured_value", [captured_value], [value], [],
+  tape.record_operation("captured_value", [captured_value], [value],
                         lambda x: [x])
   return captured_value
 
@@ -288,12 +288,14 @@ class _GraphModeFunction(object):
     real_outputs = outputs[:len(self._returns)]
     side_outputs = outputs[len(self._returns):]
 
+    def backward_function(*args):
+      return self._backward_function(*(list(args) + side_outputs))
+
     tape.record_operation(
         signature.name,
         real_outputs,
         (args + self._extra_inputs),
-        side_outputs,
-        self._backward_function)
+        backward_function)
 
     return self._build_call_outputs(self._returns, real_outputs)
 
diff --git a/tensorflow/python/eager/imperative_grad.py b/tensorflow/python/eager/imperative_grad.py
index b81f5bba14..ab6eb87a07 100644
--- a/tensorflow/python/eager/imperative_grad.py
+++ b/tensorflow/python/eager/imperative_grad.py
@@ -182,8 +182,7 @@ def imperative_grad(
       else:
         out_gradients[i] = vspace.aggregate_fn(out_gradients[i])
 
-    in_gradients = op_trace.backward_function(
-        *(out_gradients + op_trace.side_outputs))
+    in_gradients = op_trace.backward_function(*(out_gradients))
     for i, t in enumerate(op_trace.input_ids):
       if in_gradients[i] is not None:
         vspace.add_new_grads_fn(gradients, gradients_size, t, in_gradients[i])
diff --git a/tensorflow/python/eager/tape.py b/tensorflow/python/eager/tape.py
index 84814d48fd..4578a7190d 100644
--- a/tensorflow/python/eager/tape.py
+++ b/tensorflow/python/eager/tape.py
@@ -32,7 +32,7 @@ def tid(tensor):
 class TapeEntry(
     collections.namedtuple("TapeEntry", [
         "op_type",
-        "output_ids", "input_ids", "side_outputs", "backward_function",
+        "output_ids", "input_ids", "backward_function",
         "output_shape_and_dtype",
     ])):
   """Entry in the gradient tape.
@@ -43,8 +43,6 @@ class TapeEntry(
   Args:
    output_ids: tensor_id(t) for each output tensor T
    input_ids: tensor_id(t) for each input tensor T
-   side_outputs: optional tensors (not IDs) which need to be provided to the
-    backward function.
    backward_function: function to be called with the downstream gradients and
     side outputs as arguments which computes the backward pass.
    output_shape_and_dtype: a list of (shape_tuple, dtype) for every output
@@ -69,8 +67,6 @@ class Tape(object):
     self._op_tape = {}
     # next operation ID
     self._next_op_id = 0
-    # List of directly watched tensors
-    self._watched = []
     # Set of directly watched variables
     self._watched_variables = set()
 
@@ -91,14 +87,13 @@ class Tape(object):
     if i not in self._tensor_tape:
       self._tensor_tape[i] = None
       self._tensor_usage[i] = 1
-      self._watched.append(tensor)
 
   def watch_variable(self, v):
     self._watched_variables.add(v)
     self.watch(v.handle)
 
   def record_operation(self, op_type, output_tensors, input_tensors,
-                       side_outputs, backward_function):
+                       backward_function):
     """Records an operation in the tape."""
     if not self.should_record(input_tensors):
       return output_tensors
@@ -113,7 +108,6 @@ class Tape(object):
         op_type,
         [tid(t) for t in output_tensors],
         [tid(t) for t in input_tensors],
-        side_outputs,
         backward_function,
         [(_tensor_shape(t), t.dtype) for t in output_tensors])
     self._next_op_id += 1
@@ -227,13 +221,11 @@ def should_record(tensors):
   return any(x.should_record(tensors) for x in _tape_stack.stack)
 
 
-def record_operation(op_type, output_tensors, input_tensors, side_outputs,
-                     backward_function):
+def record_operation(op_type, output_tensors, input_tensors, backward_function):
   """Records the operation on all tapes in the stack."""
   for t in _tape_stack.stack:
     t.record_operation(op_type, output_tensors,
                        input_tensors,
-                       side_outputs,
                        backward_function)
 
 
@@ -243,11 +235,6 @@ def delete_trace(tensor_id):
     t.delete_trace(tensor_id)
 
 
-def top_tape_watched_tensors():
-  t = _tape_stack.stack[-1]
-  return t._watched  # pylint: disable=protected-access
-
-
 def top_tape_watched_variables():
   t = _tape_stack.stack[-1]
   return t._watched_variables  # pylint: disable=protected-access
diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index 669588ace0..7f5f60e599 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -679,7 +679,7 @@ class _EagerTensorBase(Tensor):
       self_device = self.device
       def grad_fun(dresult):
         return [dresult._copy(device_name=self_device)]
-      tape.record_operation("_copy", [new_tensor], [self], [], grad_fun)
+      tape.record_operation("_copy", [new_tensor], [self], grad_fun)
     return new_tensor
     # pylint: enable=protected-access
 
-- 
GitLab


From f49f6cd1758b9ecc92eedd377983e8047b05d964 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 9 Oct 2017 16:39:21 -0700
Subject: [PATCH 197/909] Replace CHECK() with a WARNING in StepStatsCollector
 so that Save after Finalize won't crash.

PiperOrigin-RevId: 171605724
---
 tensorflow/core/common_runtime/step_stats_collector.cc | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/common_runtime/step_stats_collector.cc b/tensorflow/core/common_runtime/step_stats_collector.cc
index e7f58f9ecf..e6403df97f 100644
--- a/tensorflow/core/common_runtime/step_stats_collector.cc
+++ b/tensorflow/core/common_runtime/step_stats_collector.cc
@@ -253,7 +253,9 @@ void StepStatsCollector::Save(const string& device,
   VLOG(1) << "Save dev " << device << " nt " << stats->stats();
   {
     mutex_lock l(mu_);
-    CHECK(!finalized_);
+    if (finalized_) {
+      LOG(WARNING) << "stats saved after finalize will not be collected.";
+    }
     if (!step_stats_ || collectedNodes >= kMaxCollectedNodes) {
       VLOG(1) << "step_stats_ nullptr or already collected too many nodes.";
       delete stats;
-- 
GitLab


From 0cbd8c74a3c4833733d7e69ff31c3e7ba50cc413 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 9 Oct 2017 17:01:17 -0700
Subject: [PATCH 198/909] New CUDA kernel for LSTMBlockCell's forward
 propagation.

PiperOrigin-RevId: 171608367
---
 tensorflow/contrib/rnn/kernels/lstm_ops.cc    |  82 ++++++-
 tensorflow/contrib/rnn/kernels/lstm_ops.h     |  82 -------
 .../contrib/rnn/kernels/lstm_ops_gpu.cu.cc    | 202 +++++++++++++++++-
 3 files changed, 279 insertions(+), 87 deletions(-)

diff --git a/tensorflow/contrib/rnn/kernels/lstm_ops.cc b/tensorflow/contrib/rnn/kernels/lstm_ops.cc
index ffeb9953c5..2b56c6f95a 100644
--- a/tensorflow/contrib/rnn/kernels/lstm_ops.cc
+++ b/tensorflow/contrib/rnn/kernels/lstm_ops.cc
@@ -41,6 +41,86 @@ typedef Eigen::GpuDevice GPUDevice;
 
 namespace functor {
 
+template <typename T>
+void LSTMBlockCellFpropWithEigen(
+    const LSTMBlockCell& cell, OpKernelContext* ctx, const CPUDevice& d,
+    const T forget_bias, const T cell_clip, bool use_peephole,
+    typename TTypes<T>::ConstMatrix x, typename TTypes<T>::ConstMatrix cs_prev,
+    typename TTypes<T>::ConstMatrix h_prev, typename TTypes<T>::ConstMatrix w,
+    typename TTypes<T>::ConstVec wci, typename TTypes<T>::ConstVec wcf,
+    typename TTypes<T>::ConstVec wco, typename TTypes<T>::ConstVec b,
+    typename TTypes<T>::Matrix xh, typename TTypes<T>::Matrix i,
+    typename TTypes<T>::Matrix cs, typename TTypes<T>::Matrix f,
+    typename TTypes<T>::Matrix o, typename TTypes<T>::Matrix ci,
+    typename TTypes<T>::Matrix co, typename TTypes<T>::Matrix icfo,
+    typename TTypes<T>::Matrix h) {
+  // Concat xh = [x, h].
+  xh.slice(cell.xh_x_offsets(), cell.xh_x_extents()).device(d) = x;
+  xh.slice(cell.xh_h_offsets(), cell.xh_h_extents()).device(d) = h_prev;
+
+  // states1 = xh * w + b
+  typename TTypes<T>::ConstMatrix const_xh(xh.data(), xh.dimensions());
+  TensorBlasGemm<CPUDevice, T, false /* USE_CUBLAS */>::compute(
+      ctx, d, false, false, T(1), const_xh, w, T(0), icfo);
+  Eigen::array<Eigen::DenseIndex, 2> b_shape({1, b.dimensions()[0]});
+  Eigen::array<Eigen::DenseIndex, 2> broadcast_shape({cell.batch_size(), 1});
+  icfo.device(d) += b.reshape(b_shape).broadcast(broadcast_shape);
+
+  Eigen::array<Eigen::DenseIndex, 2> p_shape({1, cell.cell_size()});
+  Eigen::array<Eigen::DenseIndex, 2> p_broadcast_shape({cell.batch_size(), 1});
+
+  // Input gate.
+  if (use_peephole) {
+    auto i_peep = cs_prev * wci.reshape(p_shape).broadcast(p_broadcast_shape);
+    i.device(d) =
+        (icfo.slice(cell.icfo_i_offsets(), cell.cell_extents()) + i_peep)
+            .sigmoid();
+  } else {
+    i.device(d) =
+        icfo.slice(cell.icfo_i_offsets(), cell.cell_extents()).sigmoid();
+  }
+
+  // Cell input.
+  ci.device(d) = icfo.slice(cell.icfo_c_offsets(), cell.cell_extents()).tanh();
+
+  // Forget gate (w/ bias).
+  if (use_peephole) {
+    auto f_peep = cs_prev * wcf.reshape(p_shape).broadcast(p_broadcast_shape);
+    f.device(d) = (icfo.slice(cell.icfo_f_offsets(), cell.cell_extents()) +
+                   f.constant(forget_bias) + f_peep)
+                      .sigmoid();
+  } else {
+    f.device(d) = (icfo.slice(cell.icfo_f_offsets(), cell.cell_extents()) +
+                   f.constant(forget_bias))
+                      .sigmoid();
+  }
+
+  // cs = ci .* i + f .* cs_prev
+  cs.device(d) = i * ci + f * cs_prev;
+
+  if (cell_clip > 0.0f) {
+    cs.device(d) =
+        cs.binaryExpr(cs.constant(cell_clip), Eigen::scalar_clip_op<T>());
+  }
+
+  // co = tanh(cs)
+  co.device(d) = cs.tanh();
+
+  // Output gate.
+  if (use_peephole) {
+    auto o_peep = cs * wco.reshape(p_shape).broadcast(p_broadcast_shape);
+    o.device(d) =
+        (icfo.slice(cell.icfo_o_offsets(), cell.cell_extents()) + o_peep)
+            .sigmoid();
+  } else {
+    o.device(d) =
+        icfo.slice(cell.icfo_o_offsets(), cell.cell_extents()).sigmoid();
+  }
+
+  // h = o .* co
+  h.device(d) = o * co;
+}
+
 #define DEFINE_CPU_SPECS(T)                                                    \
   template <>                                                                  \
   void LSTMBlockCellFprop<CPUDevice, T, false /* USE_CUBLAS */>::operator()(   \
@@ -55,7 +135,7 @@ namespace functor {
       typename TTypes<T>::Matrix f, typename TTypes<T>::Matrix o,              \
       typename TTypes<T>::Matrix ci, typename TTypes<T>::Matrix co,            \
       typename TTypes<T>::Matrix icfo, typename TTypes<T>::Matrix h) {         \
-    LSTMBlockCellFpropWithEigen<CPUDevice, T, false /* USE_CUBLAS */>(         \
+    LSTMBlockCellFpropWithEigen<T>(                                            \
         *this, ctx, d, forget_bias, cell_clip, use_peephole, x, cs_prev,       \
         h_prev, w, wci, wcf, wco, b, xh, i, cs, f, o, ci, co, icfo, h);        \
   }                                                                            \
diff --git a/tensorflow/contrib/rnn/kernels/lstm_ops.h b/tensorflow/contrib/rnn/kernels/lstm_ops.h
index 30a4b44706..53641ff47e 100644
--- a/tensorflow/contrib/rnn/kernels/lstm_ops.h
+++ b/tensorflow/contrib/rnn/kernels/lstm_ops.h
@@ -169,88 +169,6 @@ struct LSTMBlockCellFprop : public LSTMBlockCell {
       typename TTypes<T>::Matrix h);
 };
 
-// TODO(b/63339763): Once GPUDevice implementation no longer relies on Eigen,
-// move into lstm_ops.cc.
-template <typename Device, typename T, bool USE_CUBLAS>
-void LSTMBlockCellFpropWithEigen(
-    const LSTMBlockCell& cell, OpKernelContext* ctx, const Device& d,
-    const T forget_bias, const T cell_clip, bool use_peephole,
-    typename TTypes<T>::ConstMatrix x, typename TTypes<T>::ConstMatrix cs_prev,
-    typename TTypes<T>::ConstMatrix h_prev, typename TTypes<T>::ConstMatrix w,
-    typename TTypes<T>::ConstVec wci, typename TTypes<T>::ConstVec wcf,
-    typename TTypes<T>::ConstVec wco, typename TTypes<T>::ConstVec b,
-    typename TTypes<T>::Matrix xh, typename TTypes<T>::Matrix i,
-    typename TTypes<T>::Matrix cs, typename TTypes<T>::Matrix f,
-    typename TTypes<T>::Matrix o, typename TTypes<T>::Matrix ci,
-    typename TTypes<T>::Matrix co, typename TTypes<T>::Matrix icfo,
-    typename TTypes<T>::Matrix h) {
-  // Concat xh = [x, h].
-  xh.slice(cell.xh_x_offsets(), cell.xh_x_extents()).device(d) = x;
-  xh.slice(cell.xh_h_offsets(), cell.xh_h_extents()).device(d) = h_prev;
-
-  // states1 = xh * w + b
-  typename TTypes<T>::ConstMatrix const_xh(xh.data(), xh.dimensions());
-  TensorBlasGemm<Device, T, USE_CUBLAS>::compute(ctx, d, false, false, T(1),
-                                                 const_xh, w, T(0), icfo);
-  Eigen::array<Eigen::DenseIndex, 2> b_shape({1, b.dimensions()[0]});
-  Eigen::array<Eigen::DenseIndex, 2> broadcast_shape({cell.batch_size(), 1});
-  icfo.device(d) += b.reshape(b_shape).broadcast(broadcast_shape);
-
-  Eigen::array<Eigen::DenseIndex, 2> p_shape({1, cell.cell_size()});
-  Eigen::array<Eigen::DenseIndex, 2> p_broadcast_shape({cell.batch_size(), 1});
-
-  // Input gate.
-  if (use_peephole) {
-    auto i_peep = cs_prev * wci.reshape(p_shape).broadcast(p_broadcast_shape);
-    i.device(d) =
-        (icfo.slice(cell.icfo_i_offsets(), cell.cell_extents()) + i_peep)
-            .sigmoid();
-  } else {
-    i.device(d) =
-        icfo.slice(cell.icfo_i_offsets(), cell.cell_extents()).sigmoid();
-  }
-
-  // Cell input.
-  ci.device(d) = icfo.slice(cell.icfo_c_offsets(), cell.cell_extents()).tanh();
-
-  // Forget gate (w/ bias).
-  if (use_peephole) {
-    auto f_peep = cs_prev * wcf.reshape(p_shape).broadcast(p_broadcast_shape);
-    f.device(d) = (icfo.slice(cell.icfo_f_offsets(), cell.cell_extents()) +
-                   f.constant(forget_bias) + f_peep)
-                      .sigmoid();
-  } else {
-    f.device(d) = (icfo.slice(cell.icfo_f_offsets(), cell.cell_extents()) +
-                   f.constant(forget_bias))
-                      .sigmoid();
-  }
-
-  // cs = ci .* i + f .* cs_prev
-  cs.device(d) = i * ci + f * cs_prev;
-
-  if (cell_clip > 0.0f) {
-    cs.device(d) =
-        cs.binaryExpr(cs.constant(cell_clip), Eigen::scalar_clip_op<T>());
-  }
-
-  // co = tanh(cs)
-  co.device(d) = cs.tanh();
-
-  // Output gate.
-  if (use_peephole) {
-    auto o_peep = cs * wco.reshape(p_shape).broadcast(p_broadcast_shape);
-    o.device(d) =
-        (icfo.slice(cell.icfo_o_offsets(), cell.cell_extents()) + o_peep)
-            .sigmoid();
-  } else {
-    o.device(d) =
-        icfo.slice(cell.icfo_o_offsets(), cell.cell_extents()).sigmoid();
-  }
-
-  // h = o .* co
-  h.device(d) = o * co;
-}
-
 // See lstm_ops.cc for CPUDevice implementation and lstm_ops_gpu.cu.cc for
 // GPUDevice implementation.
 template <typename Device, typename T, bool USE_CUBLAS>
diff --git a/tensorflow/contrib/rnn/kernels/lstm_ops_gpu.cu.cc b/tensorflow/contrib/rnn/kernels/lstm_ops_gpu.cu.cc
index e18f8079a3..90990fe452 100644
--- a/tensorflow/contrib/rnn/kernels/lstm_ops_gpu.cu.cc
+++ b/tensorflow/contrib/rnn/kernels/lstm_ops_gpu.cu.cc
@@ -20,15 +20,208 @@ limitations under the License.
 #include "tensorflow/contrib/rnn/kernels/lstm_ops.h"
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "tensorflow/core/kernels/eigen_activations.h"
 #include "tensorflow/core/platform/logging.h"
+#include "tensorflow/core/util/cuda_kernel_helper.h"
 
 namespace tensorflow {
 namespace functor {
 
 typedef Eigen::GpuDevice GPUDevice;
 
+namespace {
+
+// Adds bias, applies non-linearities and gates.
+//
+// Launch with a 2D setup such that there is one thread per (example,
+// activation) with 'x' governing example index and 'y' governing activation.
+//
+// Launch with blocks of (batch x 32)
+//
+// TODO(b/67600500): Try making 'use_peephole' a template parameter.
+template <typename T>
+__global__ void lstm_gates(const T* icfo, const T* b, const T* cs_prev,
+                           const T* wci, const T* wcf, const T* wco, T* o, T* h,
+                           T* ci, T* cs, T* co, T* i, T* f, const T forget_bias,
+                           const T cell_clip, const bool use_peephole,
+                           const int batch_size, const int cell_size) {
+  const int batch_id = blockIdx.x * blockDim.x + threadIdx.x;
+  const int act_id = blockIdx.y * blockDim.y + threadIdx.y;
+
+  if (batch_id >= batch_size || act_id >= cell_size) return;
+
+  // The following code assumes the input arrays are of the following
+  // shapes and interpretations.
+  //
+  // 1) 'icfo' is a matrix such that,
+  //
+  //   cell_size  cell_size  cell_size  cell_size
+  //  +----------+----------+----------+----------+
+  //  |          |          |          |          |
+  //  |    i     |    c     |    f     |    o     |  batch_size
+  //  |          |          |          |          |
+  //  +----------+----------+----------+----------+
+  //
+  // 'gid' is the index assigned to this thread for 'icfo' in the 'i' submatrix.
+  //
+  // 2) 'b' is a vector such that,
+  //
+  //   cell_size  cell_size  cell_size  cell_size
+  //  +----------+----------+----------+----------+
+  //  |    i     |    c     |    f     |    o     |  1
+  //  +----------+----------+----------+----------+
+  //
+  // 'act_id' is the index assigned to this thread for 'b' in the 'i' subvector.
+  //
+  // 3) 'wc{i,f,o}' are vectors such that,
+  //
+  //   cell_size
+  //  +----------+
+  //  |    i     |  1
+  //  +----------+
+  //
+  //  'act_id' is the index to this thread.
+  //
+  // 4) All other matrices have the form,
+  //
+  //   cell_size
+  //  +----------+
+  //  |          |
+  //  |    i     |  batch_size
+  //  |          |
+  //  +----------+
+  //
+  // 'cid' is the index assigned to this thread.
+  //
+  const int gid = batch_id * cell_size * 4 + act_id;
+  const int cid = batch_id * cell_size + act_id;
+  Eigen::internal::scalar_sigmoid_op<T> sigmoid_op;
+  Eigen::internal::scalar_tanh_op<T> tanh_op;
+  Eigen::scalar_clip_op<T> clip_op;
+
+  T i_local;
+  if (use_peephole) {
+    i_local = sigmoid_op(icfo[0 * cell_size + gid] + b[0 * cell_size + act_id] +
+                         cs_prev[cid] * wci[act_id]);
+  } else {
+    i_local = sigmoid_op(icfo[0 * cell_size + gid] + b[0 * cell_size + act_id]);
+  }
+  i[cid] = i_local;
+
+  T ci_local = tanh_op(icfo[1 * cell_size + gid] + b[1 * cell_size + act_id]);
+  ci[cid] = ci_local;
+
+  T f_local;
+  if (use_peephole) {
+    f_local = sigmoid_op(icfo[2 * cell_size + gid] + b[2 * cell_size + act_id] +
+                         forget_bias + cs_prev[cid] * wcf[act_id]);
+  } else {
+    f_local = sigmoid_op(icfo[2 * cell_size + gid] + b[2 * cell_size + act_id] +
+                         forget_bias);
+  }
+  f[cid] = f_local;
+
+  T cs_local = i_local * ci_local + f_local * cs_prev[cid];
+  if (cell_clip > 0.0) {
+    cs_local = clip_op(cs_local, cell_clip);
+  }
+  cs[cid] = cs_local;
+
+  T co_local = tanh_op(cs_local);
+  co[cid] = co_local;
+
+  T o_local;
+  if (use_peephole) {
+    o_local = sigmoid_op(icfo[3 * cell_size + gid] + b[3 * cell_size + act_id] +
+                         cs_local * wco[act_id]);
+  } else {
+    o_local = sigmoid_op(icfo[3 * cell_size + gid] + b[3 * cell_size + act_id]);
+  }
+  o[cid] = o_local;
+
+  h[cid] = o_local * co_local;
+}
+
+// Concatenate 'x' and 'h' and copy their contents into 'xh'.
+template <typename T>
+__global__ void concat_xh(T* xh, const T* x, const T* h_prev,
+                          const int batch_size, const int cell_size,
+                          const int input_size) {
+  // Assumes 'x', 'h', and 'xh' are of the following shape,
+  //
+  //   input_size  cell_size
+  //  +----------+----------+
+  //  |          |          |
+  //  |    x     |    h     |  batch_size
+  //  |          |          |
+  //  +----------+----------+
+  //
+  const int gid = blockDim.x * blockIdx.x + threadIdx.x;
+  const int width = input_size + cell_size;
+
+  if (gid >= width * batch_size) return;
+
+  const int output_row = gid / width;
+  const int output_col = gid % width;
+
+  if (output_col < input_size) {  // x
+    xh[gid] = x[output_row * input_size + output_col];
+  } else {  // h
+    xh[gid] = h_prev[output_row * cell_size + output_col - input_size];
+  }
+}
+
+template <typename T>
+void LSTMBlockCellFpropWithCUDA(
+    OpKernelContext* ctx, const GPUDevice& d, const T forget_bias,
+    const T cell_clip, bool use_peephole, typename TTypes<T>::ConstMatrix x,
+    typename TTypes<T>::ConstMatrix cs_prev,
+    typename TTypes<T>::ConstMatrix h_prev, typename TTypes<T>::ConstMatrix w,
+    typename TTypes<T>::ConstVec wci, typename TTypes<T>::ConstVec wcf,
+    typename TTypes<T>::ConstVec wco, typename TTypes<T>::ConstVec b,
+    typename TTypes<T>::Matrix xh, typename TTypes<T>::Matrix i,
+    typename TTypes<T>::Matrix cs, typename TTypes<T>::Matrix f,
+    typename TTypes<T>::Matrix o, typename TTypes<T>::Matrix ci,
+    typename TTypes<T>::Matrix co, typename TTypes<T>::Matrix icfo,
+    typename TTypes<T>::Matrix h, int batch_size, int cell_size,
+    int input_size) {
+  const cudaStream_t& cu_stream = GetCudaStream(ctx);
+
+  // Concatenate xh = [x, h].
+  //
+  // Each block is assigned 128 threads. Good values are in [128, 1024] and are
+  // divisible by 32 (the size of a warp). The number of blocks is such that
+  // there are enough to process all the data.
+  const int block_dim = 128;
+  const int grid_dim =
+      Eigen::divup(batch_size * (cell_size + input_size), block_dim);
+  concat_xh<<<grid_dim, block_dim, 0, cu_stream>>>(
+      xh.data(), x.data(), h_prev.data(), batch_size, cell_size, input_size);
+
+  // states1 = xh * w
+  typename TTypes<T>::ConstMatrix const_xh(xh.data(), xh.dimensions());
+  TensorBlasGemm<GPUDevice, T, true /* USE_CUBLAS */>::compute(
+      ctx, d, false, false, T(1), const_xh, w, T(0), icfo);
+
+  // Add bias, apply non-linearities and gating.
+  //
+  // Use 2D blocks. The number of threads per block is equal to x * y, where x =
+  // min(batch_size, 8) and y = 32. See above for guidance on number of
+  // threads.
+  dim3 block_dim_2d(min(batch_size, 8), 32);
+  dim3 grid_dim_2d(Eigen::divup(batch_size, static_cast<int>(block_dim_2d.x)),
+                   Eigen::divup(cell_size, static_cast<int>(block_dim_2d.y)));
+
+  lstm_gates<<<grid_dim_2d, block_dim_2d, 0, cu_stream>>>(
+      icfo.data(), b.data(), cs_prev.data(), wci.data(), wcf.data(), wco.data(),
+      o.data(), h.data(), ci.data(), cs.data(), co.data(), i.data(), f.data(),
+      forget_bias, cell_clip, use_peephole, batch_size, cell_size);
+}
+
+}  // namespace
+
 // TODO(b/63339763): Provide an alternative implementation for
-// LSTMBlockCell{F,B}prop that doesn't rely on Eigen.
+// LSTMBlockCellBprop that doesn't rely on Eigen.
 #define DEFINE_GPU_SPECS(T)                                                    \
   template struct TensorZero<GPUDevice, T>;                                    \
   template struct TensorUnalignedZero<GPUDevice, T>;                           \
@@ -49,9 +242,10 @@ typedef Eigen::GpuDevice GPUDevice;
       typename TTypes<T>::Matrix f, typename TTypes<T>::Matrix o,              \
       typename TTypes<T>::Matrix ci, typename TTypes<T>::Matrix co,            \
       typename TTypes<T>::Matrix icfo, typename TTypes<T>::Matrix h) {         \
-    LSTMBlockCellFpropWithEigen<GPUDevice, T, true /* USE_CUBLAS */>(          \
-        *this, ctx, d, forget_bias, cell_clip, use_peephole, x, cs_prev,       \
-        h_prev, w, wci, wcf, wco, b, xh, i, cs, f, o, ci, co, icfo, h);        \
+    LSTMBlockCellFpropWithCUDA(ctx, d, forget_bias, cell_clip, use_peephole,   \
+                               x, cs_prev, h_prev, w, wci, wcf, wco, b, xh, i, \
+                               cs, f, o, ci, co, icfo, h, batch_size_,         \
+                               cell_size_, input_size_);                       \
   }                                                                            \
   template <>                                                                  \
   void LSTMBlockCellBprop<GPUDevice, T, true /* USE_CUBLAS */>::operator()(    \
-- 
GitLab


From 319d823a09e8c3f1c0850b9d146f7e4d7e5bd310 Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Mon, 9 Oct 2017 17:01:25 -0700
Subject: [PATCH 199/909] TFE: Fix reference counts when copying to Numpy
 arrays.

PiperOrigin-RevId: 171608395
---
 tensorflow/python/eager/pywrap_tensor.cc | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tensorflow/python/eager/pywrap_tensor.cc b/tensorflow/python/eager/pywrap_tensor.cc
index 18337bdd45..157e87d387 100644
--- a/tensorflow/python/eager/pywrap_tensor.cc
+++ b/tensorflow/python/eager/pywrap_tensor.cc
@@ -326,6 +326,9 @@ void EagerTensor_dealloc(EagerTensor* self) {
   Py_DECREF(self->keras_mask);
   TFE_DeleteTensorHandle(self->handle);
   self->handle = nullptr;
+  // We have the global interpreter lock, so use this chance to perform delayed
+  // refcount decrements.
+  tensorflow::ClearDecrefCache();
   PyObject* id = PyLong_FromLongLong(self->id);
   PyObject* func = PyObject_GetAttrString(reinterpret_cast<PyObject*>(self),
                                           "_delete_trace");
-- 
GitLab


From 3a52d39b41486d2c7d19a47e5a246b6a446aa76c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 9 Oct 2017 17:01:17 -0700
Subject: [PATCH 200/909] New CUDA kernel for LSTMBlockCell's forward
 propagation.

PiperOrigin-RevId: 171608367
---
 tensorflow/python/eager/pywrap_tensor.cc | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/tensorflow/python/eager/pywrap_tensor.cc b/tensorflow/python/eager/pywrap_tensor.cc
index 157e87d387..18337bdd45 100644
--- a/tensorflow/python/eager/pywrap_tensor.cc
+++ b/tensorflow/python/eager/pywrap_tensor.cc
@@ -326,9 +326,6 @@ void EagerTensor_dealloc(EagerTensor* self) {
   Py_DECREF(self->keras_mask);
   TFE_DeleteTensorHandle(self->handle);
   self->handle = nullptr;
-  // We have the global interpreter lock, so use this chance to perform delayed
-  // refcount decrements.
-  tensorflow::ClearDecrefCache();
   PyObject* id = PyLong_FromLongLong(self->id);
   PyObject* func = PyObject_GetAttrString(reinterpret_cast<PyObject*>(self),
                                           "_delete_trace");
-- 
GitLab


From fdb2b12d1ad84392df09dc5dcd457ca7e96cb423 Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Mon, 9 Oct 2017 17:01:25 -0700
Subject: [PATCH 201/909] TFE: Fix reference counts when copying to Numpy
 arrays.

PiperOrigin-RevId: 171608395
---
 tensorflow/python/eager/pywrap_tensor.cc | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tensorflow/python/eager/pywrap_tensor.cc b/tensorflow/python/eager/pywrap_tensor.cc
index 18337bdd45..157e87d387 100644
--- a/tensorflow/python/eager/pywrap_tensor.cc
+++ b/tensorflow/python/eager/pywrap_tensor.cc
@@ -326,6 +326,9 @@ void EagerTensor_dealloc(EagerTensor* self) {
   Py_DECREF(self->keras_mask);
   TFE_DeleteTensorHandle(self->handle);
   self->handle = nullptr;
+  // We have the global interpreter lock, so use this chance to perform delayed
+  // refcount decrements.
+  tensorflow::ClearDecrefCache();
   PyObject* id = PyLong_FromLongLong(self->id);
   PyObject* func = PyObject_GetAttrString(reinterpret_cast<PyObject*>(self),
                                           "_delete_trace");
-- 
GitLab


From 8ff5070392bd0066930d11e3e39d21d3fa84bb2e Mon Sep 17 00:00:00 2001
From: Jingyue Wu <jingyue@google.com>
Date: Mon, 9 Oct 2017 17:05:20 -0700
Subject: [PATCH 202/909] [Grappler] Optimize bitcasts.

Two optimizations:
1. If dst_type == type(x), Bitcast(x, dst_type) => No-op
2. Bitcast(Bitcast(x, type1), type2) => Bitcast(x, type2)

PiperOrigin-RevId: 171608976
---
 .../optimizers/arithmetic_optimizer.cc        | 68 ++++++++++++++++++-
 .../optimizers/arithmetic_optimizer_test.cc   | 61 +++++++++++++++++
 2 files changed, 127 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
index 3ec62b5a00..971163eadf 100644
--- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
@@ -289,6 +289,44 @@ static DataType GetDataTypeFromAttr(const NodeDef& node,
   return attr.type();
 }
 
+static void SetDataTypeToAttr(DataType dtype, const string& attr_name,
+                              NodeDef* node) {
+  (*node->mutable_attr())[attr_name].set_type(dtype);
+}
+
+static string SourceDataTypeAttrName(const NodeDef& node) {
+  if (node.op() == "Bitcast") {
+    return "T";
+  } else if (node.op() == "Cast") {
+    return "SrcT";
+  } else {
+    LOG(FATAL) << "SourceDataTypeAttrName not implemented for op " << node.op();
+  }
+}
+
+static string DestinationDataTypeAttrName(const NodeDef& node) {
+  if (node.op() == "Bitcast") {
+    return "type";
+  } else if (node.op() == "Cast") {
+    return "DstT";
+  } else {
+    LOG(FATAL) << "DestinationDataTypeAttrName not implemented for op "
+               << node.op();
+  }
+}
+
+static DataType GetSourceDataType(const NodeDef& node) {
+  return GetDataTypeFromAttr(node, SourceDataTypeAttrName(node));
+}
+
+static DataType GetDestinationDataType(const NodeDef& node) {
+  return GetDataTypeFromAttr(node, DestinationDataTypeAttrName(node));
+}
+
+static void SetSourceDataType(DataType dtype, NodeDef* node) {
+  SetDataTypeToAttr(dtype, SourceDataTypeAttrName(*node), node);
+}
+
 static bool IsNumberType(DataType dtype) {
   DataTypeVector number_types = NumberTypes();
   return std::find(number_types.begin(), number_types.end(), dtype) !=
@@ -369,8 +407,8 @@ string ArithmeticOptimizer::TrySimplifyAndReplaceUses(
       const NodeDef* cast = node_map->GetNode(transpose->input(0));
       if (cast->op() == "Cast") {
         const NodeDef* input = node_map->GetNode(cast->input(0));
-        const DataType src_type = GetDataTypeFromAttr(*cast, "SrcT");
-        const DataType dst_type = GetDataTypeFromAttr(*cast, "DstT");
+        const DataType src_type = GetSourceDataType(*cast);
+        const DataType dst_type = GetDestinationDataType(*cast);
         if (IsNumberType(src_type) && IsNumberType(dst_type) &&
             DataTypeSize(src_type) < DataTypeSize(dst_type)) {
           NodeDef* new_transpose = graph_def->add_node();
@@ -401,6 +439,32 @@ string ArithmeticOptimizer::TrySimplifyAndReplaceUses(
     }
   }
 
+  if (node->op() == "Bitcast") {
+    NodeDef* bitcast = node_map->GetNode(node->name());
+    // Bypass bitcasts whose source type and destination type are equal.
+    if (GetSourceDataType(*bitcast) == GetDestinationDataType(*bitcast)) {
+      return bitcast->input(0);
+    }
+
+    const NodeDef* operand = node_map->GetNode(bitcast->input(0));
+    if (operand->op() == bitcast->op()) {
+      // Bitcast(Bitcast(x, type1), type2) => Bitcast(x, type2)
+      bitcast->set_input(0, operand->input(0));
+      SetSourceDataType(GetSourceDataType(*operand), bitcast);
+      node_map->UpdateInput(bitcast->name(), bitcast->input(0),
+                            operand->input(0));
+      new_nodes->push_back(bitcast);
+      return bitcast->name();
+    }
+  }
+
+  if (node->op() == "Cast") {
+    // Bypass casts whose source type and destination type are equal.
+    if (GetSourceDataType(*node) == GetDestinationDataType(*node)) {
+      return node->input(0);
+    }
+  }
+
   // Fold a multiply of a scalar into the following convolution. This folding
   // can jump across nodes that merely reorders data (such as reshape and
   // transpose). For example, we can optimize
diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc b/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc
index 234c096073..39b4999808 100644
--- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc
+++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc
@@ -450,6 +450,67 @@ TEST_F(ArithmeticOptimizerTest, OptimizeCastMulTransposeConv) {
   EXPECT_EQ(conv_node->input(1), weights_node->name());
 }
 
+TEST_F(ArithmeticOptimizerTest, CombineBitcasts) {
+  tensorflow::Scope s = tensorflow::Scope::NewRootScope();
+  Output inputs =
+      ops::Placeholder(s, DT_UINT8, ops::Placeholder::Shape({2, 3}));
+  Output bc1 = ops::Bitcast(s, inputs, DT_QINT8);
+  Output bc2 = ops::Bitcast(s, bc1, DT_INT8);
+  Output outputs = ops::Identity(s.WithOpName("outputs"), bc2);
+
+  GrapplerItem item;
+  item.fetch = {"outputs"};
+  TF_CHECK_OK(s.ToGraphDef(&item.graph));
+
+  GraphDef output;
+  TF_EXPECT_OK(ArithmeticOptimizer().Optimize(nullptr, item, &output));
+  item.graph = output;
+  TF_EXPECT_OK(ModelPruner().Optimize(nullptr, item, &output));
+
+  EXPECT_EQ(1, std::count_if(
+                   output.node().begin(), output.node().end(),
+                   [](const NodeDef& node) { return node.op() == "Bitcast"; }));
+}
+
+TEST_F(ArithmeticOptimizerTest, CombineAndRemoveBitcasts) {
+  tensorflow::Scope s = tensorflow::Scope::NewRootScope();
+  Output inputs = ops::Placeholder(s, DT_INT8, ops::Placeholder::Shape({2, 3}));
+  Output bc1 = ops::Bitcast(s, inputs, DT_QINT8);
+  Output bc2 = ops::Bitcast(s, bc1, DT_INT8);
+  Output outputs = ops::Identity(s.WithOpName("outputs"), bc2);
+
+  GrapplerItem item;
+  item.fetch = {"outputs"};
+  TF_CHECK_OK(s.ToGraphDef(&item.graph));
+  GraphDef output;
+  TF_EXPECT_OK(ArithmeticOptimizer().Optimize(nullptr, item, &output));
+  item.graph = output;
+  TF_EXPECT_OK(ModelPruner().Optimize(nullptr, item, &output));
+
+  EXPECT_EQ(0, std::count_if(
+                   output.node().begin(), output.node().end(),
+                   [](const NodeDef& node) { return node.op() == "Bitcast"; }));
+}
+
+TEST_F(ArithmeticOptimizerTest, RemoveRedundantCast) {
+  tensorflow::Scope s = tensorflow::Scope::NewRootScope();
+  Output inputs = ops::Placeholder(s, DT_INT8, ops::Placeholder::Shape({2, 3}));
+  Output cast = ops::Cast(s, inputs, DT_INT8);
+  Output outputs = ops::Identity(s.WithOpName("outputs"), cast);
+
+  GrapplerItem item;
+  item.fetch = {"outputs"};
+  TF_CHECK_OK(s.ToGraphDef(&item.graph));
+  GraphDef output;
+  TF_EXPECT_OK(ArithmeticOptimizer().Optimize(nullptr, item, &output));
+  item.graph = output;
+  TF_EXPECT_OK(ModelPruner().Optimize(nullptr, item, &output));
+
+  EXPECT_EQ(0, std::count_if(
+                   output.node().begin(), output.node().end(),
+                   [](const NodeDef& node) { return node.op() == "Cast"; }));
+}
+
 }  // namespace
 }  // namespace grappler
 }  // namespace tensorflow
-- 
GitLab


From 319a359fba508d5012dd4d9f6362c349c7c88367 Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Mon, 9 Oct 2017 17:21:55 -0700
Subject: [PATCH 203/909] Create a cuda9 cudnn 7 docker file, simpler, using
 ARGS.

PiperOrigin-RevId: 171610904
---
 .../docker/Dockerfile.devel-gpu-cuda9-cudnn7  | 107 ++++++++++++++++++
 1 file changed, 107 insertions(+)
 create mode 100644 tensorflow/tools/docker/Dockerfile.devel-gpu-cuda9-cudnn7

diff --git a/tensorflow/tools/docker/Dockerfile.devel-gpu-cuda9-cudnn7 b/tensorflow/tools/docker/Dockerfile.devel-gpu-cuda9-cudnn7
new file mode 100644
index 0000000000..ac1a437031
--- /dev/null
+++ b/tensorflow/tools/docker/Dockerfile.devel-gpu-cuda9-cudnn7
@@ -0,0 +1,107 @@
+FROM nvidia/cuda:9.0-cudnn7-devel-ubuntu16.04
+
+MAINTAINER Gunhan Gulsoy <gunan@google.com>
+
+# It is possible to override these for releases.
+ARG TF_BRANCH=master
+ARG BAZEL_VERSION=0.5.4
+ARG TF_AVAILABLE_CPUS=32
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        build-essential \
+        curl \
+        git \
+        golang \
+        libcurl3-dev \
+        libfreetype6-dev \
+        libpng12-dev \
+        libzmq3-dev \
+        pkg-config \
+        python-dev \
+        python-pip \
+        rsync \
+        software-properties-common \
+        unzip \
+        zip \
+        zlib1g-dev \
+        openjdk-8-jdk \
+        openjdk-8-jre-headless \
+        wget \
+        && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN pip --no-cache-dir install --upgrade \
+        pip setuptools
+
+RUN pip --no-cache-dir install \
+        ipykernel \
+        jupyter \
+        matplotlib \
+        numpy \
+        scipy \
+        sklearn \
+        pandas \
+        && \
+    python -m ipykernel.kernelspec
+
+# Set up our notebook config.
+COPY jupyter_notebook_config.py /root/.jupyter/
+
+# Jupyter has issues with being run directly:
+#   https://github.com/ipython/ipython/issues/7062
+# We just add a little wrapper script.
+COPY run_jupyter.sh /
+
+# Set up Bazel.
+
+# Running bazel inside a `docker build` command causes trouble, cf:
+#   https://github.com/bazelbuild/bazel/issues/134
+# The easiest solution is to set up a bazelrc file forcing --batch.
+RUN echo "startup --batch" >>/etc/bazel.bazelrc
+# Similarly, we need to workaround sandboxing issues:
+#   https://github.com/bazelbuild/bazel/issues/418
+RUN echo "build --spawn_strategy=standalone --genrule_strategy=standalone" \
+    >>/etc/bazel.bazelrc
+WORKDIR /
+RUN mkdir /bazel && \
+    cd /bazel && \
+    wget --quiet https://github.com/bazelbuild/bazel/releases/download/$BAZEL_VERSION/bazel-$BAZEL_VERSION-installer-linux-x86_64.sh && \
+    wget --quiet https://raw.githubusercontent.com/bazelbuild/bazel/master/LICENSE && \
+    chmod +x bazel-*.sh && \
+    ./bazel-$BAZEL_VERSION-installer-linux-x86_64.sh && \
+    rm -f /bazel/bazel-$BAZEL_VERSION-installer-linux-x86_64.sh
+
+# Download and build TensorFlow.
+WORKDIR /
+RUN git clone https://github.com/tensorflow/tensorflow.git && \
+    cd tensorflow && \
+    git checkout ${TF_BRANCH}
+WORKDIR /tensorflow
+
+# Configure the build for our CUDA configuration.
+ENV CI_BUILD_PYTHON python
+ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH
+ENV TF_NEED_CUDA 1
+ENV TF_CUDA_COMPUTE_CAPABILITIES 3.0,3.5,5.2,6.0,6.1
+ENV TF_CUDA_VERSION 9.0
+ENV TF_CUDNN_VERSION 7.0
+RUN ./configure
+
+RUN LD_LIBRARY_PATH=/usr/local/cuda/lib64/stubs:${LD_LIBRARY_PATH} \
+    bazel build -c opt --config=cuda --cxxopt="-D_GLIBCXX_USE_CXX11_ABI=0" \
+        --jobs=${TF_AVAILABLE_CPUS} \
+        tensorflow/tools/pip_package:build_pip_package && \
+    mkdir -p /pip_pkg && \
+    bazel-bin/tensorflow/tools/pip_package/build_pip_package /pip_pkg
+
+RUN pip --no-cache-dir install --upgrade /pip_pkg/tensorflow-*.whl && \
+
+WORKDIR /root
+
+# TensorBoard
+EXPOSE 6006
+# IPython
+EXPOSE 8888
+
+RUN ["/bin/bash"]
-- 
GitLab


From 52d3a842463d11990600bb65f9752b59f6d8f418 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 9 Oct 2017 17:22:20 -0700
Subject: [PATCH 204/909] Fix wasserstein gradient penalty name scope issue and
 add the proper name scope.

PiperOrigin-RevId: 171610946
---
 .../gan/python/losses/python/losses_impl.py   | 83 ++++++++++---------
 .../python/losses/python/losses_impl_test.py  | 23 ++++-
 2 files changed, 64 insertions(+), 42 deletions(-)

diff --git a/tensorflow/contrib/gan/python/losses/python/losses_impl.py b/tensorflow/contrib/gan/python/losses/python/losses_impl.py
index b4a74fc49c..940762cf2a 100644
--- a/tensorflow/contrib/gan/python/losses/python/losses_impl.py
+++ b/tensorflow/contrib/gan/python/losses/python/losses_impl.py
@@ -297,7 +297,6 @@ def acgan_generator_loss(
 # GANs` (https://arxiv.org/abs/1704.00028).
 
 
-# TODO(joelshor): Figure out why this function can't be inside a name scope.
 def wasserstein_gradient_penalty(
     real_data,
     generated_data,
@@ -339,48 +338,50 @@ def wasserstein_gradient_penalty(
   Raises:
     ValueError: If the rank of data Tensors is unknown.
   """
-  real_data = ops.convert_to_tensor(real_data)
-  generated_data = ops.convert_to_tensor(generated_data)
-  if real_data.shape.ndims is None:
-    raise ValueError('`real_data` can\'t have unknown rank.')
-  if generated_data.shape.ndims is None:
-    raise ValueError('`generated_data` can\'t have unknown rank.')
-
-  differences = generated_data - real_data
-  batch_size = differences.shape[0].value or array_ops.shape(differences)[0]
-  alpha_shape = [batch_size] + [1] * (differences.shape.ndims - 1)
-  alpha = random_ops.random_uniform(shape=alpha_shape)
-  interpolates = real_data + (alpha * differences)
-
-  # Reuse variables if a discriminator scope already exists.
-  reuse = False if discriminator_scope is None else True
-  with variable_scope.variable_scope(discriminator_scope, 'gpenalty_dscope',
-                                     reuse=reuse):
-    disc_interpolates = discriminator_fn(interpolates, generator_inputs)
-
-  if isinstance(disc_interpolates, tuple):
-    # ACGAN case: disc outputs more than one tensor
-    disc_interpolates = disc_interpolates[0]
-
-  gradients = gradients_impl.gradients(disc_interpolates, interpolates)[0]
-  gradient_squares = math_ops.reduce_sum(
-      math_ops.square(gradients), axis=list(range(1, gradients.shape.ndims)))
-  # Propagate shape information, if possible.
-  if isinstance(batch_size, int):
-    gradient_squares.set_shape([
-        batch_size] + gradient_squares.shape.as_list()[1:])
-  # For numerical stability, add epsilon to the sum before taking the square
-  # root. Note tf.norm does not add epsilon.
-  slopes = math_ops.sqrt(gradient_squares + epsilon)
-  penalties = math_ops.square(slopes - 1.0)
-  penalty = losses.compute_weighted_loss(
-      penalties, weights, scope=scope, loss_collection=loss_collection,
-      reduction=reduction)
+  with ops.name_scope(scope, 'wasserstein_gradient_penalty',
+                      (real_data, generated_data)) as scope:
+    real_data = ops.convert_to_tensor(real_data)
+    generated_data = ops.convert_to_tensor(generated_data)
+    if real_data.shape.ndims is None:
+      raise ValueError('`real_data` can\'t have unknown rank.')
+    if generated_data.shape.ndims is None:
+      raise ValueError('`generated_data` can\'t have unknown rank.')
+
+    differences = generated_data - real_data
+    batch_size = differences.shape[0].value or array_ops.shape(differences)[0]
+    alpha_shape = [batch_size] + [1] * (differences.shape.ndims - 1)
+    alpha = random_ops.random_uniform(shape=alpha_shape)
+    interpolates = real_data + (alpha * differences)
+
+    with ops.name_scope(None):  # Clear scope so update ops are added properly.
+      # Reuse variables if variables already exists.
+      with variable_scope.variable_scope(discriminator_scope, 'gpenalty_dscope',
+                                         reuse=variable_scope.AUTO_REUSE):
+        disc_interpolates = discriminator_fn(interpolates, generator_inputs)
+
+    if isinstance(disc_interpolates, tuple):
+      # ACGAN case: disc outputs more than one tensor
+      disc_interpolates = disc_interpolates[0]
+
+    gradients = gradients_impl.gradients(disc_interpolates, interpolates)[0]
+    gradient_squares = math_ops.reduce_sum(
+        math_ops.square(gradients), axis=list(range(1, gradients.shape.ndims)))
+    # Propagate shape information, if possible.
+    if isinstance(batch_size, int):
+      gradient_squares.set_shape([
+          batch_size] + gradient_squares.shape.as_list()[1:])
+    # For numerical stability, add epsilon to the sum before taking the square
+    # root. Note tf.norm does not add epsilon.
+    slopes = math_ops.sqrt(gradient_squares + epsilon)
+    penalties = math_ops.square(slopes - 1.0)
+    penalty = losses.compute_weighted_loss(
+        penalties, weights, scope=scope, loss_collection=loss_collection,
+        reduction=reduction)
 
-  if add_summaries:
-    summary.scalar('gradient_penalty_loss', penalty)
+    if add_summaries:
+      summary.scalar('gradient_penalty_loss', penalty)
 
-  return penalty
+    return penalty
 
 
 # Original losses from `Generative Adversarial Nets`
diff --git a/tensorflow/contrib/gan/python/losses/python/losses_impl_test.py b/tensorflow/contrib/gan/python/losses/python/losses_impl_test.py
index c15ce5baae..b5cd8c92ba 100644
--- a/tensorflow/contrib/gan/python/losses/python/losses_impl_test.py
+++ b/tensorflow/contrib/gan/python/losses/python/losses_impl_test.py
@@ -453,10 +453,11 @@ class GradientPenaltyTest(test.TestCase, _PenaltyTest):
         'discriminator_scope': self._scope,
     }
     self._expected_loss = 9.00000
-    self._expected_op_name = 'weighted_loss/value'
+    self._expected_op_name = 'wasserstein_gradient_penalty/value'
     self._batch_size = 1
 
   def _discriminator_fn(self, inputs, _):
+    ops.add_to_collection('fake_update_ops', constant_op.constant(1.0))
     return variable_scope.get_variable('dummy_d', initializer=2.0) * inputs
 
   def test_loss_with_placeholder(self):
@@ -487,6 +488,26 @@ class GradientPenaltyTest(test.TestCase, _PenaltyTest):
     self.assertEqual(
         num_vars, len(ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)))
 
+  def test_works_with_get_collection(self):
+    """Tests that gradient penalty works inside other scopes."""
+    # We ran the discriminator once in the setup, so there should be an op
+    # already in the collection.
+    self.assertEqual(1, len(ops.get_collection(
+        'fake_update_ops', self._kwargs['discriminator_scope'].name)))
+
+    # Make sure the op is added to the collection even if it's in a name scope.
+    with ops.name_scope('loss'):
+      tfgan_losses.wasserstein_gradient_penalty(**self._kwargs)
+    self.assertEqual(2, len(ops.get_collection(
+        'fake_update_ops', self._kwargs['discriminator_scope'].name)))
+
+    # Make sure the op is added to the collection even if it's in a variable
+    # scope.
+    with variable_scope.variable_scope('loss_vscope'):
+      tfgan_losses.wasserstein_gradient_penalty(**self._kwargs)
+    self.assertEqual(3, len(ops.get_collection(
+        'fake_update_ops', self._kwargs['discriminator_scope'].name)))
+
 
 class MutualInformationPenaltyTest(test.TestCase, _PenaltyTest):
   """Tests for mutual_information_penalty."""
-- 
GitLab


From 485cb179ea84c8de26263628510f930d07a98c4a Mon Sep 17 00:00:00 2001
From: Neal Wu <wun@google.com>
Date: Mon, 9 Oct 2017 17:23:25 -0700
Subject: [PATCH 205/909] Fix the example in the RNN tutorial which left out
 one of the pieces of data.

PiperOrigin-RevId: 171611082
---
 tensorflow/docs_src/tutorials/recurrent.md | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/tensorflow/docs_src/tutorials/recurrent.md b/tensorflow/docs_src/tutorials/recurrent.md
index 73d40575d7..3bae9bb457 100644
--- a/tensorflow/docs_src/tutorials/recurrent.md
+++ b/tensorflow/docs_src/tutorials/recurrent.md
@@ -51,10 +51,10 @@ The core of the model consists of an LSTM cell that processes one word at a
 time and computes probabilities of the possible values for the next word in the
 sentence. The memory state of the network is initialized with a vector of zeros
 and gets updated after reading each word. For computational reasons, we will
-process data in mini-batches of size `batch_size`.  In this example, it is important 
-to note that `current_batch_of_words` does not correspond to a "sentence" of words.  
-Every word in a batch should correspond to time t.  Tensorflow will automatically sum 
-the gradients of each batch for you.
+process data in mini-batches of size `batch_size`.  In this example, it is
+important to note that `current_batch_of_words` does not correspond to a
+"sentence" of words.  Every word in a batch should correspond to a time t.
+TensorFlow will automatically sum the gradients of each batch for you.
 
 For example:
 ```
@@ -63,16 +63,17 @@ For example:
 [The, red,   fox, jumped, high]
 
 words_in_dataset[0] = [The, The]
-words_in_dataset[1] = [fox, fox]
-words_in_dataset[2] = [is, jumped]
-words_in_dataset[3] = [quick, high]
-num_batches = 4, batch_size = 2, time_steps = 5
+words_in_dataset[1] = [brown, red]
+words_in_dataset[2] = [fox, fox]
+words_in_dataset[3] = [is, jumped]
+words_in_dataset[4] = [quick, high]
+batch_size = 2, time_steps = 5
 ```
 
 The basic pseudocode is as follows:
 
 ```python
-words_in_dataset = tf.placeholder(tf.float32, [num_batches, batch_size, num_features])
+words_in_dataset = tf.placeholder(tf.float32, [time_steps, batch_size, num_features])
 lstm = tf.contrib.rnn.BasicLSTMCell(lstm_size)
 # Initial state of the LSTM memory.
 hidden_state = tf.zeros([batch_size, lstm.state_size])
-- 
GitLab


From 07d78ddeafe41bc0363ac92efd7ca8ea60478989 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 9 Oct 2017 17:31:28 -0700
Subject: [PATCH 206/909] Removes the use of tf.cond in the SweepHook used in
 the WALSMatrixFactorization estimator, to prevent a rare but possible race
 condition.

PiperOrigin-RevId: 171612114
---
 tensorflow/contrib/factorization/BUILD        |   1 -
 .../contrib/factorization/python/ops/wals.py  | 250 ++++++++----------
 .../factorization/python/ops/wals_test.py     |  14 +-
 3 files changed, 111 insertions(+), 154 deletions(-)

diff --git a/tensorflow/contrib/factorization/BUILD b/tensorflow/contrib/factorization/BUILD
index c741815042..44095bd00a 100644
--- a/tensorflow/contrib/factorization/BUILD
+++ b/tensorflow/contrib/factorization/BUILD
@@ -246,7 +246,6 @@ tf_py_test(
         "manual",
         "noasan",  # times out b/63678675
         "nomsan",
-        "notsan",
     ],
 )
 
diff --git a/tensorflow/contrib/factorization/python/ops/wals.py b/tensorflow/contrib/factorization/python/ops/wals.py
index 3e3ee5fa57..3976395d78 100644
--- a/tensorflow/contrib/factorization/python/ops/wals.py
+++ b/tensorflow/contrib/factorization/python/ops/wals.py
@@ -26,7 +26,6 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
-from tensorflow.python.ops import gen_math_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import state_ops
 from tensorflow.python.ops import variable_scope
@@ -38,31 +37,30 @@ from tensorflow.python.training import session_run_hook
 class _SweepHook(session_run_hook.SessionRunHook):
   """Keeps track of row/col sweeps, and runs prep ops before each sweep."""
 
-  def __init__(self, is_row_sweep_var, train_op, num_rows, num_cols,
-               processed_row_indices, processed_col_indices, row_prep_ops,
-               col_prep_ops, cache_init_ops, completed_sweeps_var):
+  def __init__(self, is_row_sweep_var, train_ops, num_rows, num_cols,
+               input_row_indices, input_col_indices, row_prep_ops,
+               col_prep_ops, init_op, completed_sweeps_var):
     """Initializes SweepHook.
 
     Args:
       is_row_sweep_var: A Boolean tf.Variable, determines whether we are
         currently doing a row or column sweep. It is updated by the hook.
-      train_op: An op. All the ops created by the hook will have
-        control_dependencies on train_op.
+      train_ops: A list of ops. The ops created by this hook will have
+        control dependencies on `train_ops`.
       num_rows: int, the total number of rows to be processed.
       num_cols: int, the total number of columns to be processed.
-      processed_row_indices: A Tensor of type int64. The indices of the input
-        rows that are processed during the current sweep. All elements of
-        processed_row_indices must be in [0, num_rows).
-      processed_col_indices: A Tensor of type int64. The indices of the input
+      input_row_indices: A Tensor of type int64. The indices of the input rows
+        that are processed during the current sweep. All elements of
+        `input_row_indices` must be in [0, num_rows).
+      input_col_indices: A Tensor of type int64. The indices of the input
         columns that are processed during the current sweep. All elements of
-        processed_col_indices must be in [0, num_cols).
+        `input_col_indices` must be in [0, num_cols).
       row_prep_ops: list of ops, to be run before the beginning of each row
         sweep, in the given order.
       col_prep_ops: list of ops, to be run before the beginning of each column
         sweep, in the given order.
-      cache_init_ops: list of ops, to be run once before training, in the given
-        order. These are typically local initialization ops (such as cache
-        initialization).
+      init_op: op to be run once before training. This is typically a local
+        initialization op (such as cache initialization).
       completed_sweeps_var: An integer tf.Variable, indicates the number of
         completed sweeps. It is updated by the hook.
     """
@@ -70,55 +68,45 @@ class _SweepHook(session_run_hook.SessionRunHook):
     self._num_cols = num_cols
     self._row_prep_ops = row_prep_ops
     self._col_prep_ops = col_prep_ops
-    self._cache_init_ops = cache_init_ops
+    self._init_op = init_op
     self._is_row_sweep_var = is_row_sweep_var
     self._completed_sweeps_var = completed_sweeps_var
-    # Boolean variable that determines whether the cache_init_ops have been run.
+    # Boolean variable that determines whether the init_ops have been run.
     self._is_initialized = False
-    # Boolean variable that is set to True when a sweep is completed.
-    # Used to run the prep_ops at the beginning of a sweep, in before_run().
-    self._is_sweep_done = False
-    # Ops to run jointly with train_op, responsible for updating
-    # _is_row_sweep_var and incrementing the global_step and completed_sweeps
-    # counters. They have control_dependencies on train_op.
-    self._fetches = self._create_switch_ops(processed_row_indices,
-                                            processed_col_indices, train_op)
-
-  def _create_switch_ops(self, processed_row_indices, processed_col_indices,
-                         train_op):
+    # Ops to run jointly with train_ops, responsible for updating
+    # `is_row_sweep_var` and incrementing the `global_step` and
+    # `completed_sweeps` counters.
+    self._update_op, self._is_sweep_done_var, self._switch_op = (
+        self._create_hook_ops(input_row_indices, input_col_indices, train_ops))
+
+  def _create_hook_ops(self, input_row_indices, input_col_indices, train_ops):
     """Creates ops to update is_row_sweep_var, global_step and completed_sweeps.
 
-    Creates two boolean tensors processed_rows and processed_cols, which keep
-    track of which rows/cols have been processed during the current sweep.
+    Creates two boolean tensors `processed_rows` and `processed_cols`, which
+    keep track of which rows/cols have been processed during the current sweep.
     Returns ops that should be run after each row / col update.
-      - When is_row_sweep_var is True, it sets
-        processed_rows[processed_row_indices] to True.
-      - When is_row_sweep_var is False, it sets
-        processed_cols[processed_col_indices] to True .
-    When all rows or all cols have been processed, negates is_row_sweep_var,
-    increments the completed_sweeps counter, and resets processed_rows and
-    processed_cols to False.
-    All of the ops created by this function have control_dependencies on
-    train_op.
+      - When `self._is_row_sweep_var` is True, it sets
+        processed_rows[input_row_indices] to True.
+      - When `self._is_row_sweep_var` is False, it sets
+        processed_cols[input_col_indices] to True.
 
     Args:
-      processed_row_indices: A Tensor. The indices of the input rows that are
+      input_row_indices: A Tensor. The indices of the input rows that are
         processed during the current sweep.
-      processed_col_indices: A Tensor. The indices of the input columns that
+      input_col_indices: A Tensor. The indices of the input columns that
         are processed during the current sweep.
-      train_op: An op. All the ops created by this function have
-        control_dependencies on train_op.
+      train_ops: A list of ops. The ops created by this function have control
+        dependencies on `train_ops`.
+
     Returns:
-      A list consisting of:
-        is_sweep_done: A Boolean tensor, determines whether the sweep is done,
-          i.e. all rows (during a row sweep) or all columns (during a column
-          sweep) have been processed.
-        switch_ops: An op that updates is_row_sweep_var when is_sweep_done is
-          True. Has control_dependencies on train_op.
-        incr_ops: An op that increments the global_step and completed_sweeps
-          counters. Has control_dependenciens on switch_ops.
+      A tuple consisting of:
+        update_op: An op to be run jointly with training. It updates the state
+          and increments counters (global step and completed sweeps).
+        is_sweep_done_var: A Boolean tf.Variable, specifies whether the sweep is
+          done, i.e. all rows (during a row sweep) or all columns (during a
+          column sweep) have been processed.
+        switch_op: An op to be run in `self.before_run` when the sweep is done.
     """
-
     processed_rows_init = array_ops.fill(dims=[self._num_rows], value=False)
     with ops.colocate_with(processed_rows_init):
       processed_rows = variable_scope.variable(
@@ -133,97 +121,72 @@ class _SweepHook(session_run_hook.SessionRunHook):
           collections=[ops.GraphKeys.GLOBAL_VARIABLES],
           trainable=False,
           name="sweep_hook_processed_cols")
-    # After running the train_op, update processed_rows or processed_cols
-    # tensors, depending on whether we are currently doing a row or a col sweep
-    with ops.control_dependencies([train_op]):
-
-      def get_row_update_op():
-        with ops.colocate_with(processed_rows):
-          return state_ops.scatter_update(processed_rows, processed_row_indices,
-                                          array_ops.ones_like(
-                                              processed_row_indices,
-                                              dtype=dtypes.bool))
-
-      def get_col_update_op():
-        with ops.colocate_with(processed_cols):
-          return state_ops.scatter_update(processed_cols, processed_col_indices,
-                                          array_ops.ones_like(
-                                              processed_col_indices,
-                                              dtype=dtypes.bool))
-
-      update_processed_op = control_flow_ops.cond(
-          self._is_row_sweep_var, get_row_update_op, get_col_update_op)
-
-      # After update_processed_op, check whether we have completed a sweep.
-      # If this is the case, flip the is_row_sweep_var and reset processed_rows
-      # and processed_cols tensors.
-      with ops.control_dependencies([update_processed_op]):
-
-        def get_switch_op():
-          return state_ops.assign(
-              self._is_row_sweep_var,
-              gen_math_ops.logical_not(self._is_row_sweep_var)).op
-
-        def get_reset_op():
-          return control_flow_ops.group(
-              state_ops.assign(processed_rows, processed_rows_init).op,
-              state_ops.assign(processed_cols, processed_cols_init).op)
-
-        is_sweep_done = control_flow_ops.cond(
+    switch_ops = control_flow_ops.group(
+        state_ops.assign(
             self._is_row_sweep_var,
-            lambda: math_ops.reduce_all(processed_rows),
-            lambda: math_ops.reduce_all(processed_cols),
-            name="sweep_hook_is_sweep_done")
-        switch_op = control_flow_ops.cond(
-            is_sweep_done,
-            get_switch_op,
-            control_flow_ops.no_op,
-            name="sweep_hook_switch_op")
-        reset_op = control_flow_ops.cond(
-            is_sweep_done,
-            get_reset_op,
-            control_flow_ops.no_op,
-            name="sweep_hook_reset_op")
-        switch_ops = control_flow_ops.group(
-            switch_op, reset_op, name="sweep_hook_switch_ops")
-
-        with ops.control_dependencies([switch_ops]):
-          # Op to increment the completed_sweeps counter.
-          completed_sweeps_incr_op = control_flow_ops.cond(
-              is_sweep_done,
-              lambda: state_ops.assign_add(self._completed_sweeps_var, 1).op,
-              control_flow_ops.no_op,
-              name="completed_sweeps_incr")
-
-          # Op to increment the global_step counter.
-          global_step = framework_variables.get_global_step()
-          if global_step is not None:
-            global_step_incr_op = state_ops.assign_add(
-                global_step, 1, name="global_step_incr").op
-          else:
-            global_step_incr_op = control_flow_ops.no_op(
-                name="global_step_incr")
-
-          incr_ops = control_flow_ops.group(
-              completed_sweeps_incr_op,
-              global_step_incr_op,
-              name="counter_incr_ops")
-
-    return [is_sweep_done, switch_ops, incr_ops]
+            math_ops.logical_not(self._is_row_sweep_var)),
+        state_ops.assign(processed_rows, processed_rows_init),
+        state_ops.assign(processed_cols, processed_cols_init))
+    is_sweep_done_var = variable_scope.variable(
+        False,
+        collections=[ops.GraphKeys.GLOBAL_VARIABLES],
+        trainable=False,
+        name="is_sweep_done")
+
+    # After running the `train_ops`, updates `processed_rows` or
+    # `processed_cols` tensors, depending on whether this is a row or col sweep.
+    with ops.control_dependencies(train_ops):
+      with ops.colocate_with(processed_rows):
+        update_processed_rows = state_ops.scatter_update(
+            processed_rows,
+            input_row_indices,
+            math_ops.logical_and(
+                self._is_row_sweep_var,
+                array_ops.ones_like(input_row_indices, dtype=dtypes.bool)))
+      with ops.colocate_with(processed_cols):
+        update_processed_cols = state_ops.scatter_update(
+            processed_cols,
+            input_col_indices,
+            math_ops.logical_and(
+                math_ops.logical_not(self._is_row_sweep_var),
+                array_ops.ones_like(input_col_indices, dtype=dtypes.bool)))
+      update_processed_op = control_flow_ops.group(
+          update_processed_rows, update_processed_cols)
 
-  def begin(self):
-    pass
+      with ops.control_dependencies([update_processed_op]):
+        is_sweep_done = math_ops.logical_or(
+            math_ops.reduce_all(processed_rows),
+            math_ops.reduce_all(processed_cols))
+        # Increments global step.
+        global_step = framework_variables.get_global_step()
+        if global_step is not None:
+          global_step_incr_op = state_ops.assign_add(
+              global_step, 1, name="global_step_incr").op
+        else:
+          global_step_incr_op = control_flow_ops.no_op()
+        # Increments completed sweeps.
+        completed_sweeps_incr_op = state_ops.assign_add(
+            self._completed_sweeps_var,
+            math_ops.cast(is_sweep_done, dtypes.int32),
+            use_locking=True).op
+        update_ops = control_flow_ops.group(
+            global_step_incr_op,
+            completed_sweeps_incr_op,
+            state_ops.assign(is_sweep_done_var, is_sweep_done))
+
+    return update_ops, is_sweep_done_var, switch_ops
 
   def before_run(self, run_context):
     """Runs the appropriate prep ops, and requests running update ops."""
-    # Run the appropriate cache_init and prep ops
+    # Runs the appropriate init ops and prep ops.
     sess = run_context.session
+    is_sweep_done = sess.run(self._is_sweep_done_var)
     if not self._is_initialized:
-      logging.info("SweepHook running cache init ops.")
-      for init_op in self._cache_init_ops:
-        sess.run(init_op)
-
-    if self._is_sweep_done or not self._is_initialized:
+      logging.info("SweepHook running cache init op.")
+      sess.run(self._init_op)
+    if is_sweep_done:
+      sess.run(self._switch_op)
+    if is_sweep_done or not self._is_initialized:
       logging.info("SweepHook running sweep prep ops.")
       row_sweep = sess.run(self._is_row_sweep_var)
       prep_ops = self._row_prep_ops if row_sweep else self._col_prep_ops
@@ -232,13 +195,12 @@ class _SweepHook(session_run_hook.SessionRunHook):
 
     self._is_initialized = True
 
-    # Request running the switch_ops and the incr_ops
-    logging.info("Partial fit starting.")
-    return session_run_hook.SessionRunArgs(fetches=self._fetches)
+    # Requests running `self._update_op` jointly with the training op.
+    logging.info("Next fit step starting.")
+    return session_run_hook.SessionRunArgs(fetches=[self._update_op])
 
   def after_run(self, run_context, run_values):
-    self._is_sweep_done = run_values.results[0]
-    logging.info("Partial fit done.")
+    logging.info("Fit step done.")
 
 
 class _StopAtSweepHook(session_run_hook.SessionRunHook):
@@ -360,19 +322,19 @@ def _wals_factorization_model_function(features, labels, mode, params):
   col_prep_ops = [
       model.col_update_prep_gramian_op, model.initialize_col_update_op
   ]
-  cache_init_ops = [model.worker_init]
+  init_ops = [model.worker_init]
 
   sweep_hook = _SweepHook(
       is_row_sweep_var,
-      train_op,
+      [train_op, loss],
       params["num_rows"],
       params["num_cols"],
       input_row_indices,
       input_col_indices,
       row_prep_ops,
       col_prep_ops,
-      cache_init_ops,
-      completed_sweeps_var,)
+      init_ops,
+      completed_sweeps_var)
   training_hooks = [sweep_hook]
   if max_sweeps is not None:
     training_hooks.append(_StopAtSweepHook(max_sweeps))
diff --git a/tensorflow/contrib/factorization/python/ops/wals_test.py b/tensorflow/contrib/factorization/python/ops/wals_test.py
index b5c1bb1151..8bd72b7025 100644
--- a/tensorflow/contrib/factorization/python/ops/wals_test.py
+++ b/tensorflow/contrib/factorization/python/ops/wals_test.py
@@ -357,7 +357,7 @@ class WALSMatrixFactorizationTest(test.TestCase):
 
     self.assertNear(
         loss, true_loss, err=.001,
-        msg="""After row update, eval loss = {}, does not match the true
+        msg="""After col update, eval loss = {}, does not match the true
         loss = {}.""".format(loss, true_loss))
 
 
@@ -442,7 +442,7 @@ class SweepHookTest(test.TestCase):
       completed_sweeps_var = variables.Variable(0)
       sweep_hook = wals_lib._SweepHook(
           is_row_sweep_var,
-          self._train_op,
+          [self._train_op],
           self._num_rows,
           self._num_cols,
           self._input_row_indices_ph,
@@ -465,11 +465,9 @@ class SweepHookTest(test.TestCase):
                       'False.')
       # Row sweep completed.
       mon_sess.run(self._train_op, ind_feed([3, 4], [0, 1, 2, 3, 4, 5, 6]))
-      self.assertFalse(sess.run(is_row_sweep_var),
-                       msg='Row sweep is complete but is_row_sweep is True.')
       self.assertTrue(sess.run(completed_sweeps_var) == 1,
                       msg='Completed sweeps should be equal to 1.')
-      self.assertTrue(sweep_hook._is_sweep_done,
+      self.assertTrue(sess.run(sweep_hook._is_sweep_done_var),
                       msg='Sweep is complete but is_sweep_done is False.')
       # Col init ops should run. Col sweep not completed.
       mon_sess.run(self._train_op, ind_feed([], [0, 1, 2, 3, 4]))
@@ -478,13 +476,11 @@ class SweepHookTest(test.TestCase):
       self.assertFalse(sess.run(is_row_sweep_var),
                        msg='Col sweep is not complete but is_row_sweep is '
                        'True.')
-      self.assertFalse(sweep_hook._is_sweep_done,
+      self.assertFalse(sess.run(sweep_hook._is_sweep_done_var),
                        msg='Sweep is not complete but is_sweep_done is True.')
       # Col sweep completed.
       mon_sess.run(self._train_op, ind_feed([], [4, 5, 6]))
-      self.assertTrue(sess.run(is_row_sweep_var),
-                      msg='Col sweep is complete but is_row_sweep is False')
-      self.assertTrue(sweep_hook._is_sweep_done,
+      self.assertTrue(sess.run(sweep_hook._is_sweep_done_var),
                       msg='Sweep is complete but is_sweep_done is False.')
       self.assertTrue(sess.run(completed_sweeps_var) == 2,
                       msg='Completed sweeps should be equal to 2.')
-- 
GitLab


From 2cdd0647e08c1dc7948f70416ee8311c09598e59 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 9 Oct 2017 17:49:32 -0700
Subject: [PATCH 207/909] Make error message more explicit when running
 FusedConv2DBiasActivationOp with type int8 on a GPU that doesn't support it.
 Old error message: "No algorithm worked!" New error message:
 "FusedConv2DBiasActivation is only supported on GPUs with compute capability
 6.1 or later."

PiperOrigin-RevId: 171614032
---
 .../kernels/fused_conv2d_bias_activation_op.cc      | 11 +++++++++++
 tensorflow/stream_executor/cuda/cuda_dnn.cc         | 13 ++++++++++---
 2 files changed, 21 insertions(+), 3 deletions(-)

diff --git a/tensorflow/contrib/fused_conv/kernels/fused_conv2d_bias_activation_op.cc b/tensorflow/contrib/fused_conv/kernels/fused_conv2d_bias_activation_op.cc
index 256f200868..e4c39739f7 100644
--- a/tensorflow/contrib/fused_conv/kernels/fused_conv2d_bias_activation_op.cc
+++ b/tensorflow/contrib/fused_conv/kernels/fused_conv2d_bias_activation_op.cc
@@ -298,6 +298,17 @@ void LaunchFusedConv2DBiasActivationOp<GPUDevice, T, BiasType, ScaleType>::
   constexpr int rank = is_int8x4 ? 5 : 4;
   constexpr int vect = is_int8x4 ? 4 : 1;
 
+  if (is_int8x4) {
+    int cc_major, cc_minor;
+    stream->parent()->GetDeviceDescription().cuda_compute_capability(&cc_major,
+                                                                     &cc_minor);
+    OP_REQUIRES(
+        ctx, cc_major >= 6 && cc_minor >= 1,
+        errors::Unimplemented(
+            "FusedConv2DBiasActivation for int8 is only supported on GPUs with "
+            "compute capability 6.1 or later."));
+  }
+
   const int batch_size = GetTensorDim(conv_input_param, data_format, 'N');
   int conv_input_rows = GetTensorDim(conv_input_param, data_format, 'H');
   int conv_input_cols = GetTensorDim(conv_input_param, data_format, 'W');
diff --git a/tensorflow/stream_executor/cuda/cuda_dnn.cc b/tensorflow/stream_executor/cuda/cuda_dnn.cc
index 0a1a748c40..46516cc445 100644
--- a/tensorflow/stream_executor/cuda/cuda_dnn.cc
+++ b/tensorflow/stream_executor/cuda/cuda_dnn.cc
@@ -2864,10 +2864,18 @@ bool CudnnSupport::DoFusedConvolve(
     const dnn::AlgorithmConfig& algorithm_config,
     dnn::ProfileResult* output_profile_result) {
 #if CUDNN_VERSION < 6000
-  LOG(ERROR) << "cudnnConvolutionBiasActivationForward() is only "
-                "supported for cuDNN version >= 6";
+  LOG(WARNING) << "cudnnConvolutionBiasActivationForward() is only "
+                  "supported for cuDNN version >= 6";
   return false;
 #else
+  int cc_major, cc_minor;
+  stream->parent()->GetDeviceDescription().cuda_compute_capability(&cc_major,
+                                                                   &cc_minor);
+  if (cc_major < 6 || (cc_major == 6 && cc_minor < 1)) {
+    LOG(WARNING) << "cudnnConvolutionBiasActivationForward() for int8 is only "
+                    "supported on GPUs with compute capability 6.1 or later.";
+    return false;
+  }
   return DoFusedConvolveImpl<int8, float, float, CUDNN_DATA_INT8x4,
                              CUDNN_DATA_INT32>(
       stream, conv_input_descriptor, conv_input_data, conv_input_scale,
@@ -2875,7 +2883,6 @@ bool CudnnSupport::DoFusedConvolve(
       side_input_scale, bias_descriptor, biases, activation_mode,
       output_descriptor, output_data, scratch_allocator, algorithm_config,
       output_profile_result);
-  return true;
 #endif
 }
 
-- 
GitLab


From cd37dbb8d8cdf1c8ae70f3aa8f588b85ce00a0ec Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 9 Oct 2017 18:22:25 -0700
Subject: [PATCH 208/909] Benchmark for LSTMBlockCell's forward propagation.

PiperOrigin-RevId: 171616821
---
 tensorflow/contrib/rnn/BUILD                  |  11 ++
 .../rnn/python/kernel_tests/benchmarking.py   |  66 ++++++++
 .../rnn/python/kernel_tests/gru_ops_test.py   | 157 +++++++++---------
 .../rnn/python/kernel_tests/lstm_ops_test.py  |  52 ++++++
 4 files changed, 211 insertions(+), 75 deletions(-)
 create mode 100644 tensorflow/contrib/rnn/python/kernel_tests/benchmarking.py

diff --git a/tensorflow/contrib/rnn/BUILD b/tensorflow/contrib/rnn/BUILD
index 3e6c09662f..7dc76cf622 100644
--- a/tensorflow/contrib/rnn/BUILD
+++ b/tensorflow/contrib/rnn/BUILD
@@ -42,6 +42,7 @@ tf_custom_op_py_library(
     srcs_version = "PY2AND3",
     visibility = ["//visibility:public"],
     deps = [
+        ":benchmarking",
         ":gru_ops",
         ":lstm_ops",
         "//tensorflow/contrib/compiler:compiler_py",
@@ -386,3 +387,13 @@ py_test(
         "//tensorflow/python:variables",
     ],
 )
+
+py_library(
+    name = "benchmarking",
+    srcs = ["python/kernel_tests/benchmarking.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        "//tensorflow/python:framework_ops",
+        "//third_party/py/numpy",
+    ],
+)
diff --git a/tensorflow/contrib/rnn/python/kernel_tests/benchmarking.py b/tensorflow/contrib/rnn/python/kernel_tests/benchmarking.py
new file mode 100644
index 0000000000..a48cd58706
--- /dev/null
+++ b/tensorflow/contrib/rnn/python/kernel_tests/benchmarking.py
@@ -0,0 +1,66 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Library for benchmarking OpKernels."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import itertools
+import time
+
+from tensorflow.python.framework import ops
+
+
+def device(use_gpu=False):
+  """TensorFlow device to assign ops to."""
+  if use_gpu:
+    return ops.device("/gpu:0")
+  return ops.device("/cpu:0")
+
+
+def seconds_per_run(op, sess, num_runs=50):
+  """Number of seconds taken to execute 'op' once on average."""
+  for _ in range(2):
+    sess.run(op)
+
+  start_time = time.time()
+  for _ in range(num_runs):
+    sess.run(op)
+
+  end_time = time.time()
+  time_taken = (end_time - start_time) / num_runs
+  return time_taken
+
+
+def dict_product(dicts):
+  """Constructs iterator over outer product of entries in a dict-of-lists.
+
+  Example:
+    >>> dict_products({"a": [1,2], "b": [3, 4]})
+    >>> [{"a": 1, "b": 3},
+         {"a": 1, "b": 4},
+         {"a": 2, "b": 3},
+         {"a": 2, "b": 4}]
+
+  Args:
+    dicts: dictionary with string keys and list values.
+
+  Yields:
+    Individual dicts from outer product.
+  """
+  keys, values = zip(*dicts.items())
+  for config_values in itertools.product(*values):
+    yield dict(zip(keys, config_values))
diff --git a/tensorflow/contrib/rnn/python/kernel_tests/gru_ops_test.py b/tensorflow/contrib/rnn/python/kernel_tests/gru_ops_test.py
index 4239e32ab9..b865466cc7 100644
--- a/tensorflow/contrib/rnn/python/kernel_tests/gru_ops_test.py
+++ b/tensorflow/contrib/rnn/python/kernel_tests/gru_ops_test.py
@@ -18,10 +18,9 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import time
-
 import numpy as np
 
+from tensorflow.contrib.rnn.python.kernel_tests import benchmarking
 from tensorflow.contrib.rnn.python.ops import gru_ops
 from tensorflow.python.client import session
 from tensorflow.python.framework import dtypes
@@ -333,20 +332,6 @@ class GRUBlockCellTest(test.TestCase):
 #### Benchmarking GRUBlockCell vs GRUCell.
 
 
-def time_taken_by_op(op, sess, num_runs=50):
-  """Time taken by the Op."""
-  for _ in range(2):
-    sess.run([op])
-
-  start_time = time.time()
-  for _ in range(num_runs):
-    sess.run([op])
-
-  end_time = time.time()
-  time_taken = end_time - start_time
-  return time_taken
-
-
 def training_gru_block_vs_gru_cell(batch_size,
                                    cell_size,
                                    input_size,
@@ -357,7 +342,7 @@ def training_gru_block_vs_gru_cell(batch_size,
   ops.reset_default_graph()
   with session.Session(graph=ops.Graph()) as sess:
     # Specify the device which is been used.
-    with ops.device("/cpu:0" if not use_gpu else "/device:GPU:0"):
+    with benchmarking.device(use_gpu):
 
       # Random initializers.
       seed = 1994
@@ -387,7 +372,8 @@ def training_gru_block_vs_gru_cell(batch_size,
             learning_rate).minimize(cost)
 
         # time for a training step.
-        basic_time_training = time_taken_by_op(optimizer, sess, iters)
+        basic_time_training = benchmarking.seconds_per_run(
+            optimizer, sess, iters)
 
       # Output from the basic GRU cell implementation.
       with vs.variable_scope("block", initializer=initializer):
@@ -406,7 +392,8 @@ def training_gru_block_vs_gru_cell(batch_size,
             learning_rate).minimize(cost)
 
         # time for a training step.
-        block_time_training = time_taken_by_op(optimizer, sess, iters)
+        block_time_training = benchmarking.seconds_per_run(
+            optimizer, sess, iters)
 
     performance_training = (
         basic_time_training - block_time_training) * 100 / basic_time_training
@@ -429,7 +416,7 @@ def inference_gru_block_vs_gru_cell(batch_size,
   """Benchmark inference speed between GRUBlockCell vs GRUCell."""
   ops.reset_default_graph()
   with session.Session(graph=ops.Graph()) as sess:
-    with ops.device("/cpu:0" if not use_gpu else "/device:GPU:0"):
+    with benchmarking.device(use_gpu):
 
       # Random initializers.
       seed = 1994
@@ -451,7 +438,8 @@ def inference_gru_block_vs_gru_cell(batch_size,
             time_major=True,
             dtype=dtypes.float32)
         sess.run([variables.global_variables_initializer()])
-        basic_time_inference = time_taken_by_op(outputs_dynamic, sess, iters)
+        basic_time_inference = benchmarking.seconds_per_run(
+            outputs_dynamic, sess, iters)
 
       # Output from the block GRU cell implementation.
       with vs.variable_scope("block", initializer=initializer):
@@ -463,7 +451,8 @@ def inference_gru_block_vs_gru_cell(batch_size,
             time_major=True,
             dtype=dtypes.float32)
         sess.run([variables.global_variables_initializer()])
-        block_time_inference = time_taken_by_op(outputs_dynamic, sess, iters)
+        block_time_inference = benchmarking.seconds_per_run(
+            outputs_dynamic, sess, iters)
 
     performance_inference = (basic_time_inference - block_time_inference
                             ) * 100 / basic_time_inference
@@ -484,7 +473,7 @@ def single_bprop_step_gru_block_vs_gru_cell(batch_size,
   """Benchmark single bprop step speed between GRUBlockCell vs GRUCell."""
   ops.reset_default_graph()
   with session.Session(graph=ops.Graph()) as sess:
-    with ops.device("/cpu:0" if not use_gpu else "/device:GPU:0"):
+    with benchmarking.device(use_gpu):
       initializer = init_ops.random_uniform_initializer(-1, 1, seed=1989)
       # Inputs
       x = vs.get_variable("x", [batch_size, input_size])
@@ -496,7 +485,8 @@ def single_bprop_step_gru_block_vs_gru_cell(batch_size,
                                              array_ops.identity(h))
         sess.run([variables.global_variables_initializer()])
         grad_output_wrt_input = gradients_impl.gradients([output], h)
-        basic_time_bprop = time_taken_by_op(grad_output_wrt_input, sess, iters)
+        basic_time_bprop = benchmarking.seconds_per_run(grad_output_wrt_input,
+                                                        sess, iters)
 
       # Output from the block GRU cell implementation.
       with vs.variable_scope("block", initializer=initializer):
@@ -504,7 +494,8 @@ def single_bprop_step_gru_block_vs_gru_cell(batch_size,
                                                  array_ops.identity(h))
         sess.run([variables.global_variables_initializer()])
         grad_output_wrt_input = gradients_impl.gradients([output], h)
-        block_time_bprop = time_taken_by_op(grad_output_wrt_input, sess, iters)
+        block_time_bprop = benchmarking.seconds_per_run(grad_output_wrt_input,
+                                                        sess, iters)
 
   performance_inference = (
       basic_time_bprop - block_time_bprop) * 100 / basic_time_bprop
@@ -526,23 +517,29 @@ class BenchmarkGRUBlock(test.Benchmark):
     print("batch_size, cell_size, input_size, time_steps, GPU, "
           "basic_time_training, block_time_training, performance_training[%]")
     iters = 10
-    for use_gpu in [True, False]:
-      for batch_size in [1, 32, 128]:
-        for cell_size in [128, 512]:
-          for input_size in [128, 512]:
-            for time_steps in [50]:
-              basic_time, block_time = training_gru_block_vs_gru_cell(
-                  batch_size, cell_size, input_size, time_steps, use_gpu, iters)
-              self.report_benchmark(
-                  name="GRUCell_training_time_BS%i_CS%i_IS%i_TS%i_gpu_%s" %
-                  (batch_size, cell_size, input_size, time_steps, use_gpu),
-                  iters=iters,
-                  wall_time=basic_time)
-              self.report_benchmark(
-                  name="GRUBlockCell_training_time_BS%i_CS%i_IS%i_TS%i_gpu_%s" %
-                  (batch_size, cell_size, input_size, time_steps, use_gpu),
-                  iters=iters,
-                  wall_time=block_time)
+
+    for config in benchmarking.dict_product({
+        "use_gpu": [True, False],
+        "batch_size": [1, 32, 128],
+        "cell_size": [128, 512],
+        "input_size": [128, 512],
+        "time_steps": [50]
+    }):
+      basic_time, block_time = training_gru_block_vs_gru_cell(
+          config["batch_size"], config["cell_size"], config["input_size"],
+          config["time_steps"], config["use_gpu"], iters)
+      self.report_benchmark(
+          name="GRUCell_training_time_BS%i_CS%i_IS%i_TS%i_gpu_%s" %
+          (config["batch_size"], config["cell_size"], config["input_size"],
+           config["time_steps"], config["use_gpu"]),
+          iters=iters,
+          wall_time=basic_time)
+      self.report_benchmark(
+          name="GRUBlockCell_training_time_BS%i_CS%i_IS%i_TS%i_gpu_%s" %
+          (config["batch_size"], config["cell_size"], config["input_size"],
+           config["time_steps"], config["use_gpu"]),
+          iters=iters,
+          wall_time=block_time)
 
   def benchmarkInferenceBlockGRUVsGRUCell(self):
     print("--------------------------------------------------------------")
@@ -551,23 +548,28 @@ class BenchmarkGRUBlock(test.Benchmark):
         "batch_size, cell_size, input_size, time_steps, GPU, "
         "basic_time_inference, block_time_inference, performance_inference[%]")
     iters = 10
-    for use_gpu in [True, False]:
-      for batch_size in [1, 32, 128]:
-        for cell_size in [128, 512]:
-          for input_size in [128, 512]:
-            for time_steps in [50]:
-              basic_time, block_time = inference_gru_block_vs_gru_cell(
-                  batch_size, cell_size, input_size, time_steps, use_gpu, iters)
-              self.report_benchmark(
-                  name="GRUCell_inference_time_BS%i_CS%i_IS%i_TS%i_gpu_%s" %
-                  (batch_size, cell_size, input_size, time_steps, use_gpu),
-                  iters=iters,
-                  wall_time=basic_time)
-              self.report_benchmark(
-                  name="GRUBlockCell_inference_time_BS%i_CS%i_IS%i_TS%i_gpu_%s"
-                  % (batch_size, cell_size, input_size, time_steps, use_gpu),
-                  iters=iters,
-                  wall_time=block_time)
+    for config in benchmarking.dict_product({
+        "use_gpu": [True, False],
+        "batch_size": [1, 32, 128],
+        "cell_size": [128, 512],
+        "input_size": [128, 512],
+        "time_steps": [50]
+    }):
+      basic_time, block_time = inference_gru_block_vs_gru_cell(
+          config["batch_size"], config["cell_size"], config["input_size"],
+          config["time_steps"], config["use_gpu"], iters)
+      self.report_benchmark(
+          name="GRUCell_inference_time_BS%i_CS%i_IS%i_TS%i_gpu_%s" %
+          (config["batch_size"], config["cell_size"], config["input_size"],
+           config["time_steps"], config["use_gpu"]),
+          iters=iters,
+          wall_time=basic_time)
+      self.report_benchmark(
+          name="GRUBlockCell_inference_time_BS%i_CS%i_IS%i_TS%i_gpu_%s" %
+          (config["batch_size"], config["cell_size"], config["input_size"],
+           config["time_steps"], config["use_gpu"]),
+          iters=iters,
+          wall_time=block_time)
 
   def benchmarkSingleBpropStepBlockGRUVsGRUCell(self):
     print("--------------------------------------------------------------")
@@ -575,22 +577,27 @@ class BenchmarkGRUBlock(test.Benchmark):
     print("batch_size, cell_size, input_size, GPU, basic_time, "
           "block_time, performance_inference[%]")
     iters = 10
-    for use_gpu in [True, False]:
-      for batch_size in [1, 32, 128]:
-        for cell_size in [128, 512]:
-          for input_size in [128, 512]:
-            basic_time, block_time = single_bprop_step_gru_block_vs_gru_cell(
-                batch_size, cell_size, input_size, use_gpu, iters)
-            self.report_benchmark(
-                name="GRUCell_Bprop_single_step_time_BS%i_CS%i_IS%i_gpu_%s" %
-                (batch_size, cell_size, input_size, use_gpu),
-                iters=iters,
-                wall_time=basic_time)
-            self.report_benchmark(
-                name="GRUBlockCell_Bprop_single_step_time_BS%i_CS%i_IS%i_gpu_%s"
-                % (batch_size, cell_size, input_size, use_gpu),
-                iters=iters,
-                wall_time=block_time)
+    for config in benchmarking.dict_product({
+        "use_gpu": [True, False],
+        "batch_size": [1, 32, 128],
+        "cell_size": [128, 512],
+        "input_size": [128, 512]
+    }):
+      basic_time, block_time = single_bprop_step_gru_block_vs_gru_cell(
+          config["batch_size"], config["cell_size"], config["input_size"],
+          config["use_gpu"], iters)
+      self.report_benchmark(
+          name="GRUCell_Bprop_single_step_time_BS%i_CS%i_IS%i_gpu_%s" %
+          (config["batch_size"], config["cell_size"], config["input_size"],
+           config["use_gpu"]),
+          iters=iters,
+          wall_time=basic_time)
+      self.report_benchmark(
+          name="GRUBlockCell_Bprop_single_step_time_BS%i_CS%i_IS%i_gpu_%s" %
+          (config["batch_size"], config["cell_size"], config["input_size"],
+           config["use_gpu"]),
+          iters=iters,
+          wall_time=block_time)
 
     print("--------------------------------------------------------------")
 
diff --git a/tensorflow/contrib/rnn/python/kernel_tests/lstm_ops_test.py b/tensorflow/contrib/rnn/python/kernel_tests/lstm_ops_test.py
index 0ec37411f5..3016821b74 100644
--- a/tensorflow/contrib/rnn/python/kernel_tests/lstm_ops_test.py
+++ b/tensorflow/contrib/rnn/python/kernel_tests/lstm_ops_test.py
@@ -20,7 +20,9 @@ from __future__ import print_function
 
 import numpy as np
 
+from tensorflow.contrib.rnn.python.kernel_tests import benchmarking
 from tensorflow.contrib.rnn.python.ops import lstm_ops
+from tensorflow.python.client import session
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
@@ -467,6 +469,56 @@ class LSTMBlockCellTest(test.TestCase):
       for basic, unfused in zip(basic_wgrads, unfused_wgrads):
         self.assertAllClose(basic, unfused, rtol=1e-2, atol=1e-2)
 
+#### Benchmarking.
+
+
+class BenchmarkLSTMBlock(test.Benchmark):
+
+  def benchmarkLSTMBlockCellFpropWithDynamicRNN(self):
+    print("BlockLSTMCell forward propagation via dynamic_rnn().")
+    print("--------------------------------------------------------------")
+    print("LSTMBlockCell Seconds per inference.")
+    print("batch_size,cell_size,input_size,time_steps,use_gpu,wall_time")
+    iters = 10
+    for config in benchmarking.dict_product({
+        "batch_size": [1, 32, 128],
+        "cell_size": [32, 128, 512],
+        "input_size": [128, 512],
+        "time_steps": [10, 25, 100],
+        "use_gpu": [True, False]
+    }):
+      with ops.Graph().as_default():
+        with benchmarking.device(use_gpu=config["use_gpu"]):
+          inputs = variable_scope.get_variable("x", [
+              config["time_steps"], config["batch_size"], config["input_size"]
+          ])
+          cell = lstm_ops.LSTMBlockCell(config["cell_size"])
+          outputs = rnn.dynamic_rnn(
+              cell, inputs, time_major=True, dtype=dtypes.float32)
+          init_op = variables.global_variables_initializer()
+
+        with session.Session() as sess:
+          sess.run(init_op)
+          wall_time = benchmarking.seconds_per_run(outputs, sess, iters)
+
+        # Print to stdout. If the TEST_REPORT_FILE_PREFIX environment variable
+        # is set, this will produce a copy-paste-able CSV file.
+        print(",".join(
+            map(str, [
+                config["batch_size"], config["cell_size"], config["input_size"],
+                config["time_steps"], config["use_gpu"], wall_time
+            ])))
+        benchmark_name_template = "_".join([
+            "LSTMBlockCell_fprop", "BS%(batch_size)i", "CS%(cell_size)i",
+            "IS%(input_size)i", "TS%(time_steps)i", "gpu_%(use_gpu)s"
+        ])
+
+        self.report_benchmark(
+            name=benchmark_name_template % config,
+            iters=iters,
+            wall_time=wall_time,
+            extras=config)
+
 
 if __name__ == "__main__":
   test.main()
-- 
GitLab


From 103d383a6c73363d16034c57fa7da6aea7876912 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 9 Oct 2017 18:41:05 -0700
Subject: [PATCH 209/909] Add scaled_softplus to the documented symbols so it
 can be accessed as tf.contrib.nn.scaled_softplus.

PiperOrigin-RevId: 171618233
---
 tensorflow/contrib/nn/__init__.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/contrib/nn/__init__.py b/tensorflow/contrib/nn/__init__.py
index be0957f473..7007e26bac 100644
--- a/tensorflow/contrib/nn/__init__.py
+++ b/tensorflow/contrib/nn/__init__.py
@@ -19,6 +19,7 @@
 @@deprecated_flipped_sparse_softmax_cross_entropy_with_logits
 @@deprecated_flipped_sigmoid_cross_entropy_with_logits
 @@rank_sampled_softmax_loss
+@@scaled_softplus
 """
 
 from __future__ import absolute_import
-- 
GitLab


From d08cb107e6eeedd74c44f0d3654753b141cfa645 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 9 Oct 2017 18:59:24 -0700
Subject: [PATCH 210/909] Scheduler exports tensor size info to RunMetadata. In
 addition, tensor size histogram is printed out optionally (use
 vmodule=analytical_cost_estimator=1 or 2).

PiperOrigin-RevId: 171619454
---
 .../costs/analytical_cost_estimator.cc        |  14 +-
 tensorflow/core/grappler/costs/utils.cc       | 164 ++++++++++++++++++
 tensorflow/core/grappler/costs/utils.h        |  48 +++++
 tensorflow/core/grappler/costs/utils_test.cc  | 113 ++++++++++++
 .../core/grappler/costs/virtual_scheduler.cc  |  71 ++++++--
 .../core/grappler/costs/virtual_scheduler.h   |   2 +-
 .../grappler/costs/virtual_scheduler_test.cc  |  10 +-
 7 files changed, 395 insertions(+), 27 deletions(-)

diff --git a/tensorflow/core/grappler/costs/analytical_cost_estimator.cc b/tensorflow/core/grappler/costs/analytical_cost_estimator.cc
index 91b6686971..ca66f7c75a 100644
--- a/tensorflow/core/grappler/costs/analytical_cost_estimator.cc
+++ b/tensorflow/core/grappler/costs/analytical_cost_estimator.cc
@@ -102,12 +102,20 @@ Status AnalyticalCostEstimator::PredictCosts(const GraphDef& optimized_graph,
     }
   } while (scheduler.MarkCurrNodeExecuted(node_costs));
 
-  *costs = scheduler.Summary();
+  RunMetadata run_metadata;
+  *costs = scheduler.Summary(&run_metadata);
   VLOG(1) << inaccurate_nodes.size() << " out of "
           << optimized_graph.node_size()
           << " nodes have inaccurate time estimation";
-  for (const auto& node : inaccurate_nodes) {
-    VLOG(2) << "Node with inaccurate time estimation: " << node;
+  if (VLOG_IS_ON(3)) {
+    for (const auto& node : inaccurate_nodes) {
+      VLOG(4) << "Node with inaccurate time estimation: " << node;
+    }
+  }
+
+  if (VLOG_IS_ON(1)) {
+    bool verbosity = VLOG_IS_ON(2);
+    VLOG(1) << GetStatsStringFromRunMetadata(run_metadata, verbosity);
   }
   return Status::OK();
 }
diff --git a/tensorflow/core/grappler/costs/utils.cc b/tensorflow/core/grappler/costs/utils.cc
index ff65aca13d..1504d6b74b 100644
--- a/tensorflow/core/grappler/costs/utils.cc
+++ b/tensorflow/core/grappler/costs/utils.cc
@@ -26,21 +26,27 @@ limitations under the License.
 #include "cuda/include/cudnn.h"
 #endif
 
+#include "tensorflow/core/framework/allocation_description.pb.h"
 #include "tensorflow/core/framework/attr_value.pb.h"
 #include "tensorflow/core/framework/op.h"
 #include "tensorflow/core/framework/op_def.pb.h"
+#include "tensorflow/core/framework/step_stats.pb.h"
 #include "tensorflow/core/framework/tensor.pb.h"
+#include "tensorflow/core/framework/tensor_description.pb.h"
 #include "tensorflow/core/framework/tensor_shape.pb.h"
 #include "tensorflow/core/framework/types.pb.h"
 #include "tensorflow/core/graph/graph.h"
 #include "tensorflow/core/graph/tensor_id.h"
 #include "tensorflow/core/grappler/clusters/utils.h"
 #include "tensorflow/core/grappler/utils.h"
+#include "tensorflow/core/lib/core/bits.h"
 #include "tensorflow/core/lib/strings/numbers.h"
 #include "tensorflow/core/lib/strings/strcat.h"
 #include "tensorflow/core/platform/cpu_info.h"
 #include "tensorflow/core/platform/env.h"
+#include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/protobuf.h"
+#include "tensorflow/core/protobuf/config.pb.h"
 #include "tensorflow/core/util/device_name_utils.h"
 
 namespace tensorflow {
@@ -291,5 +297,163 @@ OpPerformanceList CostGraphToOpPerformanceData(const CostGraphDef& cost_graph,
   return ret;
 }
 
+void TensorSizeHistogram::Add(const uint64 value) {
+  num_elem_++;
+  sum_elem_ += value;
+  min_ = std::min(min_, value);
+  max_ = std::max(max_, value);
+  buckets_[Index(value)]++;
+}
+
+void TensorSizeHistogram::Merge(const TensorSizeHistogram& src) {
+  num_elem_ += src.num_elem_;
+  sum_elem_ += src.sum_elem_;
+  min_ = std::min(min_, src.min_);
+  max_ = std::max(max_, src.max_);
+  std::transform(buckets_.begin(), buckets_.end(), src.buckets_.begin(),
+                 buckets_.begin(), std::plus<uint64>());
+}
+
+std::string TensorSizeHistogram::ToString() const {
+  std::string r;
+  char buf[200];
+  snprintf(buf, sizeof(buf), "Count: %lld, Average: ", num_elem_);
+  r.append(buf);
+  r.append(strings::HumanReadableNumBytes(Average()));
+  r.append(", Min: ");
+  r.append(strings::HumanReadableNumBytes(min_));
+  r.append(", Max: ");
+  r.append(strings::HumanReadableNumBytes(max_));
+  r.append("\n------------------------------------------------------\n");
+  const double mult = num_elem_ > 0 ? 100.0 / num_elem_ : 0.0;
+  uint64 cumul_sum = 0;
+
+  const int size_string_width = 12;
+  for (int i = 0; i < buckets_.size(); i++) {
+    if (buckets_[i] == 0) continue;
+    cumul_sum += buckets_[i];
+    r.append("[ ");
+    if (i == 0) {
+      r.append(size_string_width - 2, ' ');
+      r.append("0B");
+    } else {
+      uint64 left = 1ULL << (i - 1);
+      const auto left_string = strings::HumanReadableNumBytes(left);
+      r.append(size_string_width - left_string.size(), ' ');
+      r.append(left_string);
+    }
+    r.append(", ");
+    uint64 right = 1ULL << i;
+    const auto right_string = strings::HumanReadableNumBytes(right);
+    r.append(size_string_width - right_string.size(), ' ');
+    r.append(right_string);
+    snprintf(buf, sizeof(buf), ") %7lld %7.3f%% %7.3f%% ",
+             buckets_[i],         // count
+             mult * buckets_[i],  // percentage
+             mult * cumul_sum);   // cum percentage
+    r.append(buf);
+
+    // Add hash marks based on percentage; 40 marks for 100%.
+    auto marks = static_cast<int>(
+        (static_cast<double>(40 * buckets_[i] + (num_elem_ >> 1)) / num_elem_));
+    r.append(marks, '#');
+    r.push_back('\n');
+  }
+  return r;
+}
+
+const int TensorSizeHistogram::Index(const uint64 value) const {
+  // Log2Floor64 returns -1 for 0, 0 for 1, 1 for 2-3, 2 for 4-7, ...
+  const auto index = Log2Floor64(value) + 1;
+  return std::min(index, kMaxBuckets - 1);
+}
+
+string GetDeviceClassForNonChannelDevice(const string& device_name) {
+  DeviceNameUtils::ParsedName parsed_name;
+  bool parsed = DeviceNameUtils::ParseFullName(device_name, &parsed_name);
+  if (parsed) {
+    const string& jobname = parsed_name.has_job ? parsed_name.job : "";
+    return strings::StrCat("/", jobname, "/", parsed_name.type);
+  } else {
+    return "Unclassified";
+  }
+}
+
+string GetDeviceClass(const string& device_name) {
+  // TODO(dyoon): channel device name follows the convention we currently have
+  // in VirtualScheduler. This should be revised with VirtualScheduler as well
+  // as VirtualPlacer in the future.
+  if (device_name.find("Channel") != string::npos) {
+    const string from = " from ";
+    const string to = " to ";
+    const auto from_loc = device_name.find(from);
+    const auto to_loc = device_name.find(to);
+    const auto src_device_full = device_name.substr(
+        from_loc + from.size(), to_loc - (from_loc + from.size()));
+    const auto dst_device_full = device_name.substr(to_loc + to.size());
+    return strings::StrCat(
+        "Channel", ": ", GetDeviceClassForNonChannelDevice(src_device_full),
+        " -> ", GetDeviceClassForNonChannelDevice(dst_device_full));
+  } else {
+    return GetDeviceClassForNonChannelDevice(device_name);
+  }
+}
+
+string GetStatsStringFromRunMetadata(const RunMetadata& run_metadata,
+                                     bool verbosity) {
+  // TODO(dyoon): print out other stats as needed.
+  std::ostringstream output;
+
+  // Tensor size histogram:
+  // if verbosity, it outputs per-device histogram,
+  // otherwise, only per-class histogram.
+  std::unordered_map<string, TensorSizeHistogram> device_to_hist_map;
+  const auto& step_stats = run_metadata.step_stats();
+  for (const auto& dev_stat : step_stats.dev_stats()) {
+    const auto& device_name = dev_stat.device();
+    auto& hist = device_to_hist_map[device_name];
+    for (const auto& node_stat : dev_stat.node_stats()) {
+      for (const auto& node_output : node_stat.output()) {
+        // TODO(dyoon): Calculate tensor size from tensor_description's dtype
+        // and shape, instead of using optional allocation_description.
+        const auto size = node_output.tensor_description()
+                              .allocation_description()
+                              .allocated_bytes();
+        hist.Add(size);
+      }
+    }
+  }
+  if (verbosity) {
+    output << "\n";
+    output << "Per device tensor size histogram.\n";
+  }
+
+  std::unordered_map<string, TensorSizeHistogram> device_class_to_hist_map;
+  for (const auto& device_hist : device_to_hist_map) {
+    const auto& device_name = device_hist.first;
+    const auto& hist = device_hist.second;
+    if (verbosity) {
+      output << "Device: " << device_name << "\n" << hist.ToString() << "\n";
+    }
+    const auto device_class = GetDeviceClass(device_name);
+    auto it = device_class_to_hist_map.find(device_class);
+    if (it == device_class_to_hist_map.end()) {
+      device_class_to_hist_map.emplace(device_class, TensorSizeHistogram(hist));
+    } else {
+      it->second.Merge(hist);
+    }
+  }
+  output << "\n";
+  output << "Aggregated per device / channel type tensor size histogram:\n";
+  for (const auto& device_hist : device_class_to_hist_map) {
+    const auto& device_name = device_hist.first;
+    const auto& hist = device_hist.second;
+    output << "Device: " << device_name << "\n" << hist.ToString() << "\n";
+  }
+  output << "\n";
+
+  return output.str();
+}
+
 }  // end namespace grappler
 }  // end namespace tensorflow
diff --git a/tensorflow/core/grappler/costs/utils.h b/tensorflow/core/grappler/costs/utils.h
index 96f2935951..409f07b28b 100644
--- a/tensorflow/core/grappler/costs/utils.h
+++ b/tensorflow/core/grappler/costs/utils.h
@@ -27,6 +27,7 @@ limitations under the License.
 #include "tensorflow/core/graph/types.h"
 #include "tensorflow/core/grappler/costs/op_performance_data.pb.h"
 #include "tensorflow/core/platform/types.h"
+#include "tensorflow/core/protobuf/config.pb.h"
 #include "tensorflow/core/protobuf/device_properties.pb.h"
 
 namespace tensorflow {
@@ -60,6 +61,53 @@ OpInfo BuildOpInfoWithoutDevice(
 OpPerformanceList CostGraphToOpPerformanceData(const CostGraphDef& cost_graph,
                                                const GraphDef& graph);
 
+// Simple histogram for profiling Tensor size; histogram uses logarithmic
+// buckets.
+class TensorSizeHistogram {
+ public:
+  TensorSizeHistogram() : buckets_(kMaxBuckets, 0) {}
+
+  void Add(const uint64 value);
+  void Merge(const TensorSizeHistogram& src);
+  double Average() const {
+    if (num_elem_ > 0) {
+      return static_cast<double>(sum_elem_) / num_elem_;
+    } else {
+      return 0.0;
+    }
+  }
+  uint64 Min() const { return min_; }
+  uint64 Max() const { return max_; }
+  uint64 NumElem() const { return num_elem_; }
+  uint64 SumElem() const { return sum_elem_; }
+  std::string ToString() const;
+
+ protected:
+  const int Index(const uint64 value) const;
+  const std::vector<uint64>& GetBuckets() const { return buckets_; }
+
+ private:
+  const int kMaxBuckets = 64;
+  uint64 num_elem_ = 0;
+  uint64 sum_elem_ = 0;
+  // min_ and max_ are initialized to a very large value and zero, respectively,
+  // so that any value added can replace the initial min_ and max_.
+  uint64 min_ = kuint64max;
+  uint64 max_ = 0;
+  // Buckets are logarithmic:
+  // 0B, 1B, 2-3B, 4-7B, 8-15B, ..., 2^N - 2^(N+1)-1B, ...
+  std::vector<uint64> buckets_;
+};
+
+// Helper functions for aggregating per-device stats into per-device-class
+// stats.
+string GetDeviceClassForNonChannelDevice(const string& device_name);
+string GetDeviceClass(const string& device_name);
+
+// Get stats in string format from RunMetadata.
+string GetStatsStringFromRunMetadata(const RunMetadata& run_metadata,
+                                     bool verbosity);
+
 }  // end namespace grappler
 }  // end namespace tensorflow
 
diff --git a/tensorflow/core/grappler/costs/utils_test.cc b/tensorflow/core/grappler/costs/utils_test.cc
index 00cd967fc8..bd0af79029 100644
--- a/tensorflow/core/grappler/costs/utils_test.cc
+++ b/tensorflow/core/grappler/costs/utils_test.cc
@@ -172,5 +172,118 @@ TEST_F(UtilsTest, TestSkipControlInput) {
   EXPECT_TRUE(node_found);
 }
 
+// Class for testing TensorSizeHistogram.
+class TestTensorSizeHistogram : public TensorSizeHistogram {
+ public:
+  FRIEND_TEST(TensorSizeHistogramTest, Constructor);
+  FRIEND_TEST(TensorSizeHistogramTest, Index);
+  FRIEND_TEST(TensorSizeHistogramTest, Add);
+  FRIEND_TEST(TensorSizeHistogramTest, Merge);
+};
+
+TEST(TensorSizeHistogramTest, Constructor) {
+  TestTensorSizeHistogram hist;
+  EXPECT_EQ(0, hist.NumElem());
+  EXPECT_EQ(0, hist.SumElem());
+  EXPECT_LT(1000000000, hist.Min());  // Initially, min_ is a very large value.
+  EXPECT_EQ(0, hist.Max());
+  EXPECT_EQ(0.0, hist.Average());
+  const auto& buckets = hist.GetBuckets();
+  for (const auto& bucket : buckets) {
+    EXPECT_EQ(0, bucket);
+  }
+}
+
+TEST(TensorSizeHistogramTest, Index) {
+  TestTensorSizeHistogram hist;
+  EXPECT_EQ(0, hist.Index(0));
+  EXPECT_EQ(1, hist.Index(1));
+  EXPECT_EQ(2, hist.Index(2));
+  EXPECT_EQ(2, hist.Index(3));
+  EXPECT_EQ(3, hist.Index(4));
+  EXPECT_EQ(3, hist.Index(5));
+  EXPECT_EQ(3, hist.Index(6));
+  EXPECT_EQ(3, hist.Index(7));
+  EXPECT_EQ(4, hist.Index(8));
+  EXPECT_EQ(4, hist.Index(15));
+  EXPECT_EQ(5, hist.Index(16));
+  EXPECT_EQ(5, hist.Index(31));
+  EXPECT_EQ(6, hist.Index(32));
+  EXPECT_EQ(11, hist.Index(1025));
+}
+
+TEST(TensorSizeHistogramTest, Add) {
+  TestTensorSizeHistogram hist;
+  hist.Add(1037);
+  hist.Add(1038);
+  hist.Add(1039);
+
+  const auto& buckets = hist.GetBuckets();
+  EXPECT_EQ(3, hist.NumElem());
+  EXPECT_EQ(1037 + 1038 + 1039, hist.SumElem());
+  EXPECT_DOUBLE_EQ(1038.0, hist.Average());
+  EXPECT_EQ(1037, hist.Min());
+  EXPECT_EQ(1039, hist.Max());
+  EXPECT_EQ(3, buckets.at(11));
+}
+
+TEST(TensorSizeHistogramTest, Merge) {
+  TestTensorSizeHistogram hist1;
+  const auto& buckets = hist1.GetBuckets();
+  hist1.Add(1037);
+  hist1.Add(1038);
+  hist1.Add(1039);
+
+  TestTensorSizeHistogram hist2(hist1);
+  hist1.Merge(hist2);
+  EXPECT_EQ(6, hist1.NumElem());
+  EXPECT_EQ(2 * (1037 + 1038 + 1039), hist1.SumElem());
+  EXPECT_DOUBLE_EQ(1038.0, hist1.Average());
+  EXPECT_EQ(1037, hist1.Min());
+  EXPECT_EQ(1039, hist1.Max());
+  EXPECT_EQ(6, buckets.at(11));
+
+  TestTensorSizeHistogram hist3;
+  hist3.Add(1);
+  hist3.Add(2);
+  hist3.Add(4);
+
+  hist1.Merge(hist3);
+  EXPECT_EQ(9, hist1.NumElem());
+  EXPECT_EQ(2 * (1037 + 1038 + 1039) + 1 + 2 + 4, hist1.SumElem());
+  EXPECT_DOUBLE_EQ((2 * (1037 + 1038 + 1039) + 1 + 2 + 4) / 9.0,
+                   hist1.Average());
+  EXPECT_EQ(1, hist1.Min());
+  EXPECT_EQ(1039, hist1.Max());
+  EXPECT_EQ(1, buckets.at(1));
+  EXPECT_EQ(1, buckets.at(2));
+  EXPECT_EQ(1, buckets.at(3));
+  EXPECT_EQ(6, buckets.at(11));
+}
+
+TEST(DeviceClassTest, GetDeviceClass) {
+  EXPECT_EQ(
+      "Channel: /ps/CPU -> /worker/GPU",
+      GetDeviceClass("Channel from /job:ps/replica:0/task:0/device:CPU:0 to "
+                     "/job:worker/replica:7/task:0/device:GPU:7"));
+  EXPECT_EQ(
+      "Channel: /worker_train/CPU -> /ps/GPU",
+      GetDeviceClass(
+          "Channel from /job:worker_train/replica:0/task:0/device:CPU:0 to "
+          "/job:ps/replica:7/task:0/device:GPU:7"));
+}
+
+TEST(DeviceClassTest, GetDeviceClassForNonChannelDevice) {
+  EXPECT_EQ("Unclassified",
+            GetDeviceClassForNonChannelDevice("SOMETHING_WEIRD_DEVICE_NAME"));
+  EXPECT_EQ("/worker/GPU", GetDeviceClassForNonChannelDevice(
+                               "/job:worker/replica:0/task:0/device:GPU:0"));
+  EXPECT_EQ("/worker/CPU", GetDeviceClassForNonChannelDevice(
+                               "/job:worker/replica:0/task:0/device:CPU:0"));
+  EXPECT_EQ("/worker_train/CPU", GetDeviceClassForNonChannelDevice(
+                                     "/job:worker_train/replica:7/CPU:0"));
+  EXPECT_EQ("//GPU", GetDeviceClassForNonChannelDevice("/device:GPU:7"));
+}
+
 }  // end namespace grappler
 }  // end namespace tensorflow
diff --git a/tensorflow/core/grappler/costs/virtual_scheduler.cc b/tensorflow/core/grappler/costs/virtual_scheduler.cc
index 99ea75f703..1ae6fac8c8 100644
--- a/tensorflow/core/grappler/costs/virtual_scheduler.cc
+++ b/tensorflow/core/grappler/costs/virtual_scheduler.cc
@@ -17,6 +17,7 @@ limitations under the License.
 
 #include <math.h>
 
+#include "tensorflow/core/framework/allocation_description.pb.h"
 #include "tensorflow/core/framework/attr_value.pb.h"
 #include "tensorflow/core/framework/node_def.pb.h"
 #include "tensorflow/core/framework/tensor.pb.h"
@@ -26,7 +27,9 @@ limitations under the License.
 #include "tensorflow/core/grappler/costs/utils.h"
 #include "tensorflow/core/grappler/op_types.h"
 #include "tensorflow/core/grappler/utils.h"
+#include "tensorflow/core/lib/strings/numbers.h"
 #include "tensorflow/core/lib/strings/str_util.h"
+#include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/util/device_name_utils.h"
 
 namespace tensorflow {
@@ -51,7 +54,7 @@ Costs CombineCosts(const Costs& left, const Costs& right) {
     result.max_per_op_streaming =
         std::max(left.max_per_op_streaming, right.max_per_op_streaming);
   }
-  VLOG(3) << "costs execution_time=" << result.execution_time.count()
+  VLOG(4) << "costs execution_time=" << result.execution_time.count()
           << " max_memory=" << result.max_memory
           << " max_per_op_buffers=" << result.max_per_op_buffers
           << " max_per_op_streaming=" << result.max_per_op_streaming;
@@ -544,7 +547,7 @@ bool VirtualScheduler::MarkCurrNodeExecuted(const Costs& node_costs) {
   auto& device_op_cost = FindOrCreateZero(op_name, &device.op_to_cost);
   device_op_cost = CombineCosts(device_op_cost, node_costs);
 
-  VLOG(2) << "Op scheduled -- name: " << node->name() << ", op: " << node->op()
+  VLOG(3) << "Op scheduled -- name: " << node->name() << ", op: " << node->op()
           << ", device: " << node->device()
           << ", ready: " << node_state.time_ready.count()
           << ", scheduled: " << node_state.time_scheduled.count()
@@ -649,12 +652,12 @@ Costs VirtualScheduler::Summary() const {
             << ", execution_time = " << state.GetCurrTime().count()
             << ", memory usage: "
             << "persistenst = "
-            << Round2(persistent_memory_usage / 1024.0 / 1024.0 / 1024.0)
-            << " GB, peak = "
-            << Round2(state.max_memory_usage / 1024.0 / 1024.0 / 1024.0)
-            << " GB, total = "
-            << Round2(max_memory_usage / 1024.0 / 1024.0 / 1024.0)
-            << " GB, at the end: " << state.memory_usage << " B";
+            << strings::HumanReadableNumBytes(persistent_memory_usage)
+            << ", peak = "
+            << strings::HumanReadableNumBytes(state.max_memory_usage)
+            << ", total = " << strings::HumanReadableNumBytes(max_memory_usage)
+            << ", at the end: "
+            << strings::HumanReadableNumBytes(state.memory_usage);
 
     VLOG(1) << "Per-op execution time (and memory usage at peak memory usage):";
 
@@ -668,16 +671,20 @@ Costs VirtualScheduler::Summary() const {
     for (const auto& op_cost_pair : state.op_to_cost) {
       const auto& op = op_cost_pair.first;
       const auto& cost = op_cost_pair.second.execution_time.count();
-      const float mem_usage_gb =
-          Round2(op_to_memory[op] / 1024.0 / 1024.0 / 1024.0);
-      int64 op_mem_usage = op_to_memory.at(op);
+      int64 op_mem_usage = 0;
+      auto it = op_to_memory.find(op);
+      if (it != op_to_memory.end()) {
+        op_mem_usage = it->second;
+      }
+
       const float mem_usage_percent =
           max_memory_usage > 0 ? Round2(100.0 * op_mem_usage / max_memory_usage)
                                : 0.0;
       if (cost || mem_usage_percent > 1.0) {
         // Print out only non-zero cost ops or ops with > 1% memory usage.
-        VLOG(1) << " + " << op << " : " << cost << " (" << mem_usage_gb
-                << " GB [" << mem_usage_percent << "%] "
+        VLOG(1) << " + " << op << " : " << cost << " ("
+                << strings::HumanReadableNumBytes(op_mem_usage) << " ["
+                << mem_usage_percent << "%] "
                 << (persisent_ops.count(op) > 0 ? ": persistent op)" : ")");
       }
     }
@@ -686,11 +693,13 @@ Costs VirtualScheduler::Summary() const {
     }
   }
 
-  // Also log the op description and their corresponding counts.
-  VLOG(2) << "Node description, counts, cost:";
-  for (const auto& item : op_counts_) {
-    VLOG(2) << "Node: " << item.first << ", Count: " << item.second
-            << ", Individual Cost: " << op_costs_.at(item.first);
+  if (VLOG_IS_ON(2)) {
+    // Also log the op description and their corresponding counts.
+    VLOG(2) << "Node description, counts, cost:";
+    for (const auto& item : op_counts_) {
+      VLOG(2) << "Node: " << item.first << ", Count: " << item.second
+              << ", Individual Cost: " << op_costs_.at(item.first);
+    }
   }
 
   VLOG(1) << "Critical path execution time: "
@@ -709,6 +718,7 @@ Costs VirtualScheduler::Summary(RunMetadata* metadata) {
       for (const auto& node_def : device.second.nodes_executed) {
         const NodeState& nodestate = node_map_.at(node_def);
         NodeExecStats* node_stats = device_stepstats->add_node_stats();
+        uint64 total_output_size = 0;
         for (int slot = 0; slot < nodestate.output_properties.size(); slot++) {
           const auto& properties = nodestate.output_properties[slot];
           NodeOutput* no = node_stats->add_output();
@@ -716,6 +726,14 @@ Costs VirtualScheduler::Summary(RunMetadata* metadata) {
           TensorDescription* tensor_descr = no->mutable_tensor_description();
           tensor_descr->set_dtype(properties.dtype());
           *tensor_descr->mutable_shape() = properties.shape();
+          // Optional allocation description.
+          const auto tensor_size =
+              CalculateOutputSize(nodestate.output_properties, slot);
+          total_output_size += tensor_size;
+          tensor_descr->mutable_allocation_description()->set_requested_bytes(
+              tensor_size);
+          tensor_descr->mutable_allocation_description()->set_allocated_bytes(
+              tensor_size);
         }
         node_stats->set_timeline_label(node_def->op());
         node_stats->set_node_name(node_def->name());
@@ -728,6 +746,23 @@ Costs VirtualScheduler::Summary(RunMetadata* metadata) {
         node_stats->set_all_end_rel_micros(
             nodestate.time_finished.asMicroSeconds().count() -
             nodestate.time_scheduled.asMicroSeconds().count());
+        auto* mem_stats = node_stats->mutable_memory_stats();
+        // VirtualScheduler does not specify scratch pad memory usage.
+        mem_stats->set_host_temp_memory_size(0);
+        mem_stats->set_device_temp_memory_size(0);
+        int64 host_persistent_memory_size = 0;
+        int64 device_persistent_memory_size = 0;
+        if (IsPersistentNode(node_def)) {
+          if (device.first.find("cpu") != string::npos ||
+              device.first.find("CPU") != string::npos) {
+            host_persistent_memory_size = total_output_size;
+          } else {
+            device_persistent_memory_size = total_output_size;
+          }
+        }
+        mem_stats->set_host_persistent_memory_size(host_persistent_memory_size);
+        mem_stats->set_device_persistent_memory_size(
+            device_persistent_memory_size);
         *device_partition_graph->mutable_node()->Add() = *node_def;
       }
     }
diff --git a/tensorflow/core/grappler/costs/virtual_scheduler.h b/tensorflow/core/grappler/costs/virtual_scheduler.h
index 767b91677f..8741afff7d 100644
--- a/tensorflow/core/grappler/costs/virtual_scheduler.h
+++ b/tensorflow/core/grappler/costs/virtual_scheduler.h
@@ -327,7 +327,7 @@ class VirtualScheduler {
 
   // Auxilliary data structures for constructing NodeState and DeviceState.
   GraphProperties graph_properties_;
-  Cluster* cluster_;                   // Not owned.
+  Cluster* cluster_;  // Not owned.
 
   const GrapplerItem* grappler_item_;  // Not owned.
   bool use_static_shapes_;
diff --git a/tensorflow/core/grappler/costs/virtual_scheduler_test.cc b/tensorflow/core/grappler/costs/virtual_scheduler_test.cc
index 64fb626422..5656aab4b4 100644
--- a/tensorflow/core/grappler/costs/virtual_scheduler_test.cc
+++ b/tensorflow/core/grappler/costs/virtual_scheduler_test.cc
@@ -1235,7 +1235,7 @@ TEST_F(VirtualSchedulerTest, CalculateOutputSize) {
   EXPECT_EQ(2 * 10 * 10 * 10, scheduler_->CalculateOutputSize(output, 2));
   EXPECT_EQ(4 * 100 * 7 * 8 * 99, scheduler_->CalculateOutputSize(output, 3));
 
-  // Any uknown shape (-1) shall yield zero output size.
+  // Any unknown shape (-1) shall yield zero output size.
   EXPECT_EQ(0, scheduler_->CalculateOutputSize(output, 4));
   EXPECT_EQ(0, scheduler_->CalculateOutputSize(output, 5));
 
@@ -1320,8 +1320,10 @@ TEST_F(VirtualSchedulerTest, ComplexDependency) {
         return std::make_pair(node_port.first->name(), node_port.second);
       });
   std::set<std::pair<string, int>> expected = {
-      std::make_pair("bn", -1), std::make_pair("bn", 0),
-      std::make_pair("bn", 2), std::make_pair("x", 0),
+      std::make_pair("bn", -1),
+      std::make_pair("bn", 0),
+      std::make_pair("bn", 2),
+      std::make_pair("x", 0),
   };
   ExpectSetEq(expected, nodes_in_memory);
 
@@ -1512,7 +1514,6 @@ TEST_F(VirtualSchedulerTest, InterDeviceTransfer) {
       output_properties.push_back(output_property);
     }
     return scheduler_->CalculateOutputSize(output_properties, 0);
-
   };
 
   // Validate transfer size.
@@ -1529,6 +1530,5 @@ TEST_F(VirtualSchedulerTest, InterDeviceTransfer) {
   EXPECT_EQ(get_output_size(recv_op_names[-1]), 4);
   EXPECT_EQ(get_output_size(send_op_names[-1]), 4);
 }
-
 }  // end namespace grappler
 }  // end namespace tensorflow
-- 
GitLab


From 403e51018b3c47cd5989d6b50776e235221fade4 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Mon, 9 Oct 2017 19:12:48 -0700
Subject: [PATCH 211/909] [XLA] Factor out repeated
 LatestNonGteAncestorAndIndex helper.

PiperOrigin-RevId: 171620470
---
 .../compiler/xla/service/cpu/ir_emitter.cc    | 18 ++--------
 .../xla/service/gpu/hlo_to_ir_bindings.cc     |  2 +-
 .../xla/service/gpu/ir_emission_utils.cc      |  7 ----
 .../xla/service/gpu/ir_emission_utils.h       |  4 ---
 .../xla/service/gpu/ir_emitter_unnested.cc    | 33 +++++--------------
 .../compiler/xla/service/hlo_instruction.cc   | 23 +++++++++++++
 .../compiler/xla/service/hlo_instruction.h    | 20 +++++++++++
 7 files changed, 55 insertions(+), 52 deletions(-)

diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
index c9c87f065b..a58db883d3 100644
--- a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
@@ -2102,19 +2102,6 @@ Status IrEmitter::HandleDynamicSlice(HloInstruction* dynamic_slice,
 
 namespace {
 
-// Returns the first non-GetTupleElement ancestor instruction of 'hlo'.
-// If the first non-GTE ancestor is tuple-shaped, populates 'index' with the
-// (possibly nested) tuple indices used on the path from ancestor to 'hlo'.
-const HloInstruction* LatestNonGteAncestorAndIndex(const HloInstruction* hlo,
-                                                   ShapeIndex* index) {
-  if (hlo->opcode() == HloOpcode::kGetTupleElement) {
-    const auto* operand = LatestNonGteAncestorAndIndex(hlo->operand(0), index);
-    index->push_back(hlo->tuple_index());
-    return operand;
-  }
-  return hlo;
-}
-
 // Checks if we can emit code for DynamicUpdateSlice to update data in-place.
 // Returns true if operand 0 of DynamicUpdateSlice and its output buffer
 // share the same buffer allocation.
@@ -2126,9 +2113,10 @@ bool CanUpdateDynamicSliceInPlace(const BufferAssignment& assignment,
 
   // Walk DynamicUpdateSlice operand(0) to parameter and get its
   // associated operand. See if it shares an allocation with this operand.
+  HloInstruction* operand;
   ShapeIndex index;
-  auto* operand =
-      LatestNonGteAncestorAndIndex(dynamic_update_slice->operand(0), &index);
+  std::tie(operand, index) =
+      dynamic_update_slice->mutable_operand(0)->LatestNonGteAncestorAndIndex();
   if (operand->opcode() != HloOpcode::kParameter) {
     return false;
   }
diff --git a/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.cc b/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.cc
index 373c1aa5f9..0bf66a4bc8 100644
--- a/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.cc
+++ b/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.cc
@@ -67,7 +67,7 @@ void HloToIrBindings::EmitBasePointersForHlos(
         // Lookup allocation GetTupleElement operand.
         const BufferAllocation::Slice slice =
             buffer_assignment_
-                ->GetUniqueTopLevelSlice(LatestNonGteAncestor(non_io_hlo))
+                ->GetUniqueTopLevelSlice(non_io_hlo->LatestNonGteAncestor())
                 .ConsumeValueOrDie();
         // We are not in a nested context, so check non-thread-local allocation.
         CHECK(!slice.allocation()->is_thread_local());
diff --git a/tensorflow/compiler/xla/service/gpu/ir_emission_utils.cc b/tensorflow/compiler/xla/service/gpu/ir_emission_utils.cc
index 6be26dde8f..8fb7a6adda 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emission_utils.cc
+++ b/tensorflow/compiler/xla/service/gpu/ir_emission_utils.cc
@@ -214,12 +214,5 @@ llvm::Value* EmitShuffleDown(llvm::Value* value, llvm::Value* offset,
       value->getType());
 }
 
-const HloInstruction* LatestNonGteAncestor(const HloInstruction* hlo) {
-  while (hlo->opcode() == HloOpcode::kGetTupleElement) {
-    hlo = hlo->operand(0);
-  }
-  return hlo;
-}
-
 }  // namespace gpu
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/service/gpu/ir_emission_utils.h b/tensorflow/compiler/xla/service/gpu/ir_emission_utils.h
index 422972762e..06c3205296 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emission_utils.h
+++ b/tensorflow/compiler/xla/service/gpu/ir_emission_utils.h
@@ -53,10 +53,6 @@ llvm::Value* EmitPrintf(tensorflow::StringPiece fmt,
 llvm::Value* EmitShuffleDown(llvm::Value* value, llvm::Value* offset,
                              llvm::IRBuilder<>* builder);
 
-// Resolves GetTupleElement instruction operands starting with 'hlo'.
-// Returns the first ancestor instruction which is not a GetTupleElement.
-const HloInstruction* LatestNonGteAncestor(const HloInstruction* hlo);
-
 }  // namespace gpu
 }  // namespace xla
 
diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
index 4e6b109b80..88ea5760cb 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
+++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
@@ -254,27 +254,11 @@ Status IrEmitterUnnested::HandleConvolution(HloInstruction* convolution,
                                       rhs_instruction, window);
 }
 
-namespace {
-
-// Returns the first non-GetTupleElement ancestor instruction of 'hlo'.
-// If the first non-GTE ancestor is tuple-shaped, populates 'index' with the
-// (possibly nested) tuple indices used on the path from ancestor to 'hlo'.
-const HloInstruction* LatestNonGteAncestorAndIndex(const HloInstruction* hlo,
-                                                   ShapeIndex* index) {
-  if (hlo->opcode() == HloOpcode::kGetTupleElement) {
-    const auto* operand = LatestNonGteAncestorAndIndex(hlo->operand(0), index);
-    index->push_back(hlo->tuple_index());
-    return operand;
-  }
-  return hlo;
-}
-
 // Checks if we can emit code for DynamicUpdateSlice to update data in-place.
 // Returns true if operand 0 of DynamicUpdateSlice and its output buffer
 // share the same buffer allocation.
-// Returns false otherwise.
-bool CanUpdateDynamicSliceInPlace(const BufferAssignment& assignment,
-                                  HloInstruction* fusion) {
+static bool CanUpdateDynamicSliceInPlace(const BufferAssignment& assignment,
+                                         HloInstruction* fusion) {
   CHECK_EQ(HloOpcode::kFusion, fusion->opcode());
   HloInstruction* fused_root = fusion->fused_expression_root();
   if (fused_root->opcode() != HloOpcode::kDynamicUpdateSlice) {
@@ -282,9 +266,10 @@ bool CanUpdateDynamicSliceInPlace(const BufferAssignment& assignment,
   }
   // Walk DynamicUpdateSlice operand(0) to fused parameter and get its
   // associated operand. See if it shares an allocation with this operand.
+  HloInstruction* fusion_operand;
   ShapeIndex index;
-  auto* fusion_operand =
-      LatestNonGteAncestorAndIndex(fused_root->operand(0), &index);
+  std::tie(fusion_operand, index) =
+      fused_root->mutable_operand(0)->LatestNonGteAncestorAndIndex();
   if (fusion_operand->opcode() != HloOpcode::kParameter) {
     return false;
   }
@@ -292,8 +277,6 @@ bool CanUpdateDynamicSliceInPlace(const BufferAssignment& assignment,
   return assignment.SharesSliceAtIndex(fusion, {}, operand, index);
 }
 
-}  // namespace
-
 Status IrEmitterUnnested::HandleFusion(HloInstruction* fusion) {
   HloInstruction* root = fusion->fused_expression_root();
   // HandleFusion specializes reduction from a multi-dimensional array to a 1D
@@ -386,7 +369,7 @@ Status IrEmitterUnnested::HandleFusion(HloInstruction* fusion) {
     TF_RETURN_IF_ERROR(root->Accept(&fused_emitter));
 
     // Recursively lookup 'fusion_operand' for DynamicUpdateSlice operand 0.
-    auto* fusion_operand = LatestNonGteAncestor(root->operand(0));
+    auto* fusion_operand = root->operand(0)->LatestNonGteAncestor();
     CHECK_EQ(HloOpcode::kParameter, fusion_operand->opcode());
 
     // Operand(0) the input array which shares an allocation with the output.
@@ -1625,7 +1608,7 @@ llvm::Function* IrEmitterUnnested::EmitBasePointersForHloAndItsOperands(
   // with their operand buffer in 'io_hlos' and 'non_io_hlos' below.
   std::vector<const HloInstruction*> non_io_hlos;
   for (const HloInstruction* operand : hlo.operands()) {
-    const HloInstruction* to_lookup = LatestNonGteAncestor(operand);
+    const HloInstruction* to_lookup = operand->LatestNonGteAncestor();
     if (buffer_assignment.HasTopLevelAllocation(to_lookup) &&
         buffer_assignment.GetUniqueTopLevelSlice(to_lookup)
             .ConsumeValueOrDie()
@@ -1665,7 +1648,7 @@ std::unique_ptr<Thunk> IrEmitterUnnested::BuildKernelThunk(
   std::vector<BufferAllocation::Slice> io_buffers;
   io_buffers.reserve(io_hlos.size());
   for (const HloInstruction* io_hlo : io_hlos) {
-    io_buffers.push_back(GetAllocationSlice(*LatestNonGteAncestor(io_hlo)));
+    io_buffers.push_back(GetAllocationSlice(*io_hlo->LatestNonGteAncestor()));
   }
 
   // Create a KernelThunk that launches the kernel that implements "inst".
diff --git a/tensorflow/compiler/xla/service/hlo_instruction.cc b/tensorflow/compiler/xla/service/hlo_instruction.cc
index 81bccfddbb..e3e482cf85 100644
--- a/tensorflow/compiler/xla/service/hlo_instruction.cc
+++ b/tensorflow/compiler/xla/service/hlo_instruction.cc
@@ -1131,6 +1131,29 @@ std::unique_ptr<HloInstruction> HloInstruction::CloneFusionWithNewOperands(
   return new_instruction;
 }
 
+std::pair<const HloInstruction*, ShapeIndex>
+HloInstruction::LatestNonGteAncestorAndIndex() const {
+  const HloInstruction* hlo = this;
+  ShapeIndex index;
+  while (hlo->opcode() == HloOpcode::kGetTupleElement) {
+    index.push_back(hlo->tuple_index());
+    hlo = hlo->operand(0);
+  }
+
+  // We built up index in the reverse order from what we want.
+  std::reverse(index.begin(), index.end());
+
+  return {hlo, index};
+}
+
+const HloInstruction* HloInstruction::LatestNonGteAncestor() const {
+  const HloInstruction* hlo = this;
+  while (hlo->opcode() == HloOpcode::kGetTupleElement) {
+    hlo = hlo->operand(0);
+  }
+  return hlo;
+}
+
 const Literal& HloInstruction::literal() const {
   CHECK_EQ(HloOpcode::kConstant, opcode_);
   return *literal_;
diff --git a/tensorflow/compiler/xla/service/hlo_instruction.h b/tensorflow/compiler/xla/service/hlo_instruction.h
index 73c4ebd9f1..011cc8f742 100644
--- a/tensorflow/compiler/xla/service/hlo_instruction.h
+++ b/tensorflow/compiler/xla/service/hlo_instruction.h
@@ -508,6 +508,26 @@ class HloInstruction {
   // Precondition: opcode() == HloOpcode::kGetTupleElement
   int64 tuple_index() const;
 
+  // Returns the first non-GetTupleElement ancestor instruction of 'hlo'.
+  // If the first non-GTE ancestor is tuple-shaped, populates 'index' with the
+  // (possibly nested) tuple indices used on the path from ancestor to 'hlo'.
+  std::pair<const HloInstruction*, ShapeIndex> LatestNonGteAncestorAndIndex()
+      const;
+
+  std::pair<HloInstruction*, ShapeIndex> LatestNonGteAncestorAndIndex() {
+    auto rv =
+        const_cast<const HloInstruction*>(this)->LatestNonGteAncestorAndIndex();
+    return {const_cast<HloInstruction*>(rv.first), rv.second};
+  }
+
+  // Same as LatestNonGteAncestorAndIndex, but just returns the HloInstruction.
+  const HloInstruction* LatestNonGteAncestor() const;
+
+  HloInstruction* LatestNonGteAncestor() {
+    return const_cast<HloInstruction*>(
+        const_cast<const HloInstruction*>(this)->LatestNonGteAncestor());
+  }
+
   // Gets/sets the to_apply HloComputation for Call, Map, Reduce, etc.
   // The setter should only be called by HloModule or HloComputation methods.
   //
-- 
GitLab


From 84f1b9049de86ba5614ce73f91232fd72eefbd1f Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Mon, 9 Oct 2017 19:47:07 -0700
Subject: [PATCH 212/909] [XLA:LLVM] Rename ops.h to tuple_ops.h.

I would like to reclaim ops.h for a different purpose in a later patch.
It doesn't make sense to shove it all in the same header because
FusedIrEmitter uses (tuple_)ops.h, but my new functions will use
FusedIrEmitter.

PiperOrigin-RevId: 171622776
---
 tensorflow/compiler/xla/service/cpu/BUILD                | 2 +-
 tensorflow/compiler/xla/service/cpu/ir_emitter.cc        | 2 +-
 tensorflow/compiler/xla/service/gpu/BUILD                | 4 ++--
 .../compiler/xla/service/gpu/convolution_folding.cc      | 2 +-
 .../compiler/xla/service/gpu/hlo_to_ir_bindings.cc       | 2 +-
 tensorflow/compiler/xla/service/gpu/ir_emitter.cc        | 2 +-
 .../compiler/xla/service/gpu/ir_emitter_unnested.cc      | 2 +-
 tensorflow/compiler/xla/service/llvm_ir/BUILD            | 9 ++++-----
 .../compiler/xla/service/llvm_ir/fused_ir_emitter.cc     | 2 +-
 tensorflow/compiler/xla/service/llvm_ir/loop_emitter.cc  | 1 -
 .../xla/service/llvm_ir/{ops.cc => tuple_ops.cc}         | 2 +-
 .../compiler/xla/service/llvm_ir/{ops.h => tuple_ops.h}  | 8 +++++---
 12 files changed, 19 insertions(+), 19 deletions(-)
 rename tensorflow/compiler/xla/service/llvm_ir/{ops.cc => tuple_ops.cc} (98%)
 rename tensorflow/compiler/xla/service/llvm_ir/{ops.h => tuple_ops.h} (93%)

diff --git a/tensorflow/compiler/xla/service/cpu/BUILD b/tensorflow/compiler/xla/service/cpu/BUILD
index fa6e5b2313..0daaa122f4 100644
--- a/tensorflow/compiler/xla/service/cpu/BUILD
+++ b/tensorflow/compiler/xla/service/cpu/BUILD
@@ -237,7 +237,7 @@ cc_library(
         "//tensorflow/compiler/xla/service/llvm_ir:llvm_loop",
         "//tensorflow/compiler/xla/service/llvm_ir:llvm_util",
         "//tensorflow/compiler/xla/service/llvm_ir:loop_emitter",
-        "//tensorflow/compiler/xla/service/llvm_ir:ops",
+        "//tensorflow/compiler/xla/service/llvm_ir:tuple_ops",
         "//tensorflow/core:lib",
         "@llvm//:core",
         "@llvm//:support",
diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
index a58db883d3..5474862e45 100644
--- a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
@@ -48,7 +48,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/llvm_ir/fused_ir_emitter.h"
 #include "tensorflow/compiler/xla/service/llvm_ir/llvm_loop.h"
 #include "tensorflow/compiler/xla/service/llvm_ir/llvm_util.h"
-#include "tensorflow/compiler/xla/service/llvm_ir/ops.h"
+#include "tensorflow/compiler/xla/service/llvm_ir/tuple_ops.h"
 #include "tensorflow/compiler/xla/shape_util.h"
 #include "tensorflow/compiler/xla/status_macros.h"
 #include "tensorflow/compiler/xla/types.h"
diff --git a/tensorflow/compiler/xla/service/gpu/BUILD b/tensorflow/compiler/xla/service/gpu/BUILD
index 82c32407d3..1d980405dd 100644
--- a/tensorflow/compiler/xla/service/gpu/BUILD
+++ b/tensorflow/compiler/xla/service/gpu/BUILD
@@ -104,7 +104,7 @@ cc_library(
         "//tensorflow/compiler/xla/service/llvm_ir:alias_analysis",
         "//tensorflow/compiler/xla/service/llvm_ir:ir_array",
         "//tensorflow/compiler/xla/service/llvm_ir:llvm_util",
-        "//tensorflow/compiler/xla/service/llvm_ir:ops",
+        "//tensorflow/compiler/xla/service/llvm_ir:tuple_ops",
         "//tensorflow/core:lib",
         "@llvm//:core",
     ],
@@ -146,7 +146,7 @@ cc_library(
         "//tensorflow/compiler/xla/service/llvm_ir:llvm_loop",
         "//tensorflow/compiler/xla/service/llvm_ir:llvm_util",
         "//tensorflow/compiler/xla/service/llvm_ir:loop_emitter",
-        "//tensorflow/compiler/xla/service/llvm_ir:ops",
+        "//tensorflow/compiler/xla/service/llvm_ir:tuple_ops",
         "//tensorflow/core:lib",
         "//tensorflow/core:stream_executor_no_cuda",
         "@llvm//:core",
diff --git a/tensorflow/compiler/xla/service/gpu/convolution_folding.cc b/tensorflow/compiler/xla/service/gpu/convolution_folding.cc
index 7cf5613ce5..edd04773d1 100644
--- a/tensorflow/compiler/xla/service/gpu/convolution_folding.cc
+++ b/tensorflow/compiler/xla/service/gpu/convolution_folding.cc
@@ -275,7 +275,7 @@ MatchBackwardInput(HloInstruction* conv) {
   Window new_window = old_window;
   for (size_t i = 0; i < spatial_dims.size(); ++i) {
     // Restore backward convolution's padding config from the matched pattern.
-    // See the comment in tensorflow/core/kernels/conv_grad_ops.cc
+    // See the comment in tensorflow/core/kernels/conv_grad_tuple_ops.cc
     // for how we convert backward input convolution to a variant of forward
     // convolution.
     //
diff --git a/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.cc b/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.cc
index 0bf66a4bc8..152d226ab0 100644
--- a/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.cc
+++ b/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.cc
@@ -21,7 +21,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/gpu/ir_emission_utils.h"
 #include "tensorflow/compiler/xla/service/hlo_opcode.h"
 #include "tensorflow/compiler/xla/service/llvm_ir/llvm_util.h"
-#include "tensorflow/compiler/xla/service/llvm_ir/ops.h"
+#include "tensorflow/compiler/xla/service/llvm_ir/tuple_ops.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/types.h"
 
diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter.cc b/tensorflow/compiler/xla/service/gpu/ir_emitter.cc
index a76d217cac..3862c2190b 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/gpu/ir_emitter.cc
@@ -34,7 +34,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/llvm_ir/llvm_loop.h"
 #include "tensorflow/compiler/xla/service/llvm_ir/llvm_util.h"
 #include "tensorflow/compiler/xla/service/llvm_ir/loop_emitter.h"
-#include "tensorflow/compiler/xla/service/llvm_ir/ops.h"
+#include "tensorflow/compiler/xla/service/llvm_ir/tuple_ops.h"
 #include "tensorflow/compiler/xla/service/name_uniquer.h"
 #include "tensorflow/compiler/xla/shape_util.h"
 #include "tensorflow/compiler/xla/status_macros.h"
diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
index 88ea5760cb..cf41623a9b 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
+++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
@@ -50,7 +50,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/hlo_opcode.h"
 #include "tensorflow/compiler/xla/service/llvm_ir/fused_ir_emitter.h"
 #include "tensorflow/compiler/xla/service/llvm_ir/llvm_util.h"
-#include "tensorflow/compiler/xla/service/llvm_ir/ops.h"
+#include "tensorflow/compiler/xla/service/llvm_ir/tuple_ops.h"
 #include "tensorflow/compiler/xla/service/name_uniquer.h"
 #include "tensorflow/compiler/xla/shape_util.h"
 #include "tensorflow/compiler/xla/status_macros.h"
diff --git a/tensorflow/compiler/xla/service/llvm_ir/BUILD b/tensorflow/compiler/xla/service/llvm_ir/BUILD
index f498f95057..62e404bd82 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/BUILD
+++ b/tensorflow/compiler/xla/service/llvm_ir/BUILD
@@ -93,7 +93,6 @@ cc_library(
     deps = [
         ":ir_array",
         ":llvm_loop",
-        ":ops",
         "//tensorflow/compiler/xla:shape_util",
         "//tensorflow/compiler/xla:status_macros",
         "//tensorflow/compiler/xla:statusor",
@@ -112,7 +111,7 @@ cc_library(
         ":ir_array",
         ":llvm_util",
         ":loop_emitter",
-        ":ops",
+        ":tuple_ops",
         "//tensorflow/compiler/xla:status_macros",
         "//tensorflow/compiler/xla:statusor",
         "//tensorflow/compiler/xla:util",
@@ -125,9 +124,9 @@ cc_library(
 )
 
 cc_library(
-    name = "ops",
-    srcs = ["ops.cc"],
-    hdrs = ["ops.h"],
+    name = "tuple_ops",
+    srcs = ["tuple_ops.cc"],
+    hdrs = ["tuple_ops.h"],
     deps = [
         ":ir_array",
         ":llvm_util",
diff --git a/tensorflow/compiler/xla/service/llvm_ir/fused_ir_emitter.cc b/tensorflow/compiler/xla/service/llvm_ir/fused_ir_emitter.cc
index 7d1fad753e..d286c49d68 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/fused_ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/llvm_ir/fused_ir_emitter.cc
@@ -22,7 +22,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/elemental_ir_emitter.h"
 #include "tensorflow/compiler/xla/service/llvm_ir/ir_array.h"
 #include "tensorflow/compiler/xla/service/llvm_ir/llvm_util.h"
-#include "tensorflow/compiler/xla/service/llvm_ir/ops.h"
+#include "tensorflow/compiler/xla/service/llvm_ir/tuple_ops.h"
 #include "tensorflow/compiler/xla/status_macros.h"
 #include "tensorflow/compiler/xla/statusor.h"
 #include "tensorflow/compiler/xla/util.h"
diff --git a/tensorflow/compiler/xla/service/llvm_ir/loop_emitter.cc b/tensorflow/compiler/xla/service/llvm_ir/loop_emitter.cc
index 8bba1776d1..6fa4cd08c9 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/loop_emitter.cc
+++ b/tensorflow/compiler/xla/service/llvm_ir/loop_emitter.cc
@@ -19,7 +19,6 @@ limitations under the License.
 #include <utility>
 
 #include "tensorflow/compiler/xla/service/llvm_ir/llvm_loop.h"
-#include "tensorflow/compiler/xla/service/llvm_ir/ops.h"
 #include "tensorflow/compiler/xla/shape_util.h"
 #include "tensorflow/compiler/xla/status_macros.h"
 #include "tensorflow/compiler/xla/types.h"
diff --git a/tensorflow/compiler/xla/service/llvm_ir/ops.cc b/tensorflow/compiler/xla/service/llvm_ir/tuple_ops.cc
similarity index 98%
rename from tensorflow/compiler/xla/service/llvm_ir/ops.cc
rename to tensorflow/compiler/xla/service/llvm_ir/tuple_ops.cc
index ae5c666b7d..6051cbfc6f 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/ops.cc
+++ b/tensorflow/compiler/xla/service/llvm_ir/tuple_ops.cc
@@ -13,7 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/compiler/xla/service/llvm_ir/ops.h"
+#include "tensorflow/compiler/xla/service/llvm_ir/tuple_ops.h"
 
 #include <stddef.h>
 #include <string>
diff --git a/tensorflow/compiler/xla/service/llvm_ir/ops.h b/tensorflow/compiler/xla/service/llvm_ir/tuple_ops.h
similarity index 93%
rename from tensorflow/compiler/xla/service/llvm_ir/ops.h
rename to tensorflow/compiler/xla/service/llvm_ir/tuple_ops.h
index 4e1d9d1080..a75cdc8158 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/ops.h
+++ b/tensorflow/compiler/xla/service/llvm_ir/tuple_ops.h
@@ -13,8 +13,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#ifndef TENSORFLOW_COMPILER_XLA_SERVICE_LLVM_IR_OPS_H_
-#define TENSORFLOW_COMPILER_XLA_SERVICE_LLVM_IR_OPS_H_
+#ifndef TENSORFLOW_COMPILER_XLA_SERVICE_LLVM_IR_TUPLE_OPS_H_
+#define TENSORFLOW_COMPILER_XLA_SERVICE_LLVM_IR_TUPLE_OPS_H_
 
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/Value.h"
@@ -22,6 +22,8 @@ limitations under the License.
 #include "tensorflow/core/lib/gtl/array_slice.h"
 #include "tensorflow/core/platform/types.h"
 
+// Utilities for emitting LLVM IR related to HLO tuples.
+
 namespace xla {
 namespace llvm_ir {
 
@@ -76,4 +78,4 @@ llvm::Value* EmitGetTupleElement(const Shape& target_shape, int64 index,
 }  // namespace llvm_ir
 }  // namespace xla
 
-#endif  // TENSORFLOW_COMPILER_XLA_SERVICE_LLVM_IR_OPS_H_
+#endif  // TENSORFLOW_COMPILER_XLA_SERVICE_LLVM_IR_TUPLE_OPS_H_
-- 
GitLab


From d98519bf80c3a7fc26b41139bf3e753510efffb2 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Mon, 9 Oct 2017 20:22:07 -0700
Subject: [PATCH 213/909] [XLA:CPU] Let the elementwise concat op handle being
 emitted into a degenerate BB.

It's possible to create a graph such that an elementwise concat is
emitted into an LLVM basic block which lacks a terminator.  In this case
it's an error to call splitBasicBlock(), so we need to handle this (as
is done elsewhere in this file).

PiperOrigin-RevId: 171624976
---
 .../xla/service/elemental_ir_emitter.cc       | 22 +++++++++++++++----
 1 file changed, 18 insertions(+), 4 deletions(-)

diff --git a/tensorflow/compiler/xla/service/elemental_ir_emitter.cc b/tensorflow/compiler/xla/service/elemental_ir_emitter.cc
index 12fb88f39c..3a8f70a8ef 100644
--- a/tensorflow/compiler/xla/service/elemental_ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/elemental_ir_emitter.cc
@@ -879,17 +879,31 @@ llvm_ir::ElementGenerator ElementalIrEmitter::MakeElementGenerator(
         const int64 concat_dim = hlo->dimensions(0);
         auto source_index = target_index;
 
+        llvm::BasicBlock* init_block = ir_builder_->GetInsertBlock();
+
+        // A terminator should be present iff we're emitting code
+        // into the middle (as opposed to the end) of a basic block.
+        CHECK_EQ(ir_builder_->GetInsertPoint() == init_block->end(),
+                 init_block->getTerminator() == nullptr);
+
+        llvm::BasicBlock* exit_block;
+        if (ir_builder_->GetInsertPoint() == init_block->end()) {
+          exit_block = llvm_ir::CreateBasicBlock(
+              /*insert_before=*/nullptr, IrName(hlo, "merge"), ir_builder_);
+        } else {
+          exit_block = init_block->splitBasicBlock(
+              ir_builder_->GetInsertPoint(), AsStringRef(IrName(hlo, "merge")));
+          init_block->getTerminator()->eraseFromParent();
+        }
+
+        llvm_ir::SetToFirstInsertPoint(exit_block, ir_builder_);
         llvm::PHINode* output = ir_builder_->CreatePHI(
             llvm_ir::PrimitiveTypeToIrType(hlo->shape().element_type(),
                                            ir_builder_),
             hlo->operands().size());
-        llvm::BasicBlock* init_block = ir_builder_->GetInsertBlock();
         auto prior_insert_point = ir_builder_->GetInsertPoint();
-        llvm::BasicBlock* exit_block =
-            init_block->splitBasicBlock(output, "concat_merge");
 
         ir_builder_->SetInsertPoint(init_block);
-        init_block->getTerminator()->eraseFromParent();
 
         for (int64 operand_idx = 0; operand_idx < hlo->operand_count();
              ++operand_idx) {
-- 
GitLab


From 4f102ffd12d56a2c41dc8b5a5324873ecc0f07e4 Mon Sep 17 00:00:00 2001
From: Igor Ganichev <iga@google.com>
Date: Mon, 9 Oct 2017 20:34:06 -0700
Subject: [PATCH 214/909] Cache last zero tensor in eager gradient computation

SPINN and probably other models commonly split large tensors into many
equal parts (e.g. along the batch dimension). When we compute the
gradient of such split, we often don't have gradients comming from all
parts and end up creating zero tensors. This change caches the last
created zero tensor and reuses it. It reduces SPINN training time by
over 13%.

PiperOrigin-RevId: 171625608
---
 tensorflow/python/eager/imperative_grad.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/eager/imperative_grad.py b/tensorflow/python/eager/imperative_grad.py
index ab6eb87a07..f388d0a148 100644
--- a/tensorflow/python/eager/imperative_grad.py
+++ b/tensorflow/python/eager/imperative_grad.py
@@ -171,14 +171,23 @@ def imperative_grad(
     op = ready_ops.pop()
     op_trace = op_to_entry.pop(op)
     out_gradients = [gradients.pop(t, None) for t in op_trace.output_ids]
+
+    # Cache the last used zero tensor. We reuse it if the next one
+    # we need is of the same shape and dtype. This is very helpful in
+    # large splits and should have negligible overhead in other cases.
+    last_shape_and_dtype = None
+    last_zeros = None
     for i in range(len(out_gradients)):
       if out_gradients[i] is None:
         # TODO(apassos) this should be in the right device
         none_indices = _grad_fn_accepts_none_for_indices.get(
             op_trace.op_type, None)
         if none_indices is None or i not in none_indices:
-          out_gradients[i] = vspace.zeros(
-              *op_trace.output_shape_and_dtype[i])
+          shape_and_dtype = op_trace.output_shape_and_dtype[i]
+          if shape_and_dtype != last_shape_and_dtype:
+            last_shape_and_dtype = shape_and_dtype
+            last_zeros = vspace.zeros(*shape_and_dtype)
+          out_gradients[i] = last_zeros
       else:
         out_gradients[i] = vspace.aggregate_fn(out_gradients[i])
 
-- 
GitLab


From effb22e8a44763901ee2cf55c30290f0b1edb570 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Mon, 9 Oct 2017 20:41:00 -0700
Subject: [PATCH 215/909] Use an external constant pool to reduce LLVM compile
 times

LLVM does not deal well with huge arrays emitted inline into the IR.  In JIT
mode, this change teaches XLA to emit large constant tensors onto a side data
structure, which are then symbolically linked to the generated executable.  It
is important to note that this works only in JIT mode, and my current
understanding is that making this work reliably in AOT will be somewhat more
difficult.

PiperOrigin-RevId: 171626043
---
 tensorflow/compiler/xla/service/cpu/BUILD     | 25 ++++++
 .../compiler/xla/service/cpu/cpu_compiler.cc  |  9 +-
 .../xla/service/cpu/external_constant_pool.cc | 53 ++++++++++++
 .../xla/service/cpu/external_constant_pool.h  | 64 +++++++++++++++
 .../cpu/external_constant_pool_test.cc        | 82 +++++++++++++++++++
 .../compiler/xla/service/cpu/ir_emitter.cc    | 49 ++++++++---
 .../compiler/xla/service/cpu/ir_emitter.h     | 10 ++-
 .../xla/service/cpu/simple_orc_jit.cc         | 19 ++++-
 .../compiler/xla/service/cpu/simple_orc_jit.h |  6 ++
 9 files changed, 299 insertions(+), 18 deletions(-)
 create mode 100644 tensorflow/compiler/xla/service/cpu/external_constant_pool.cc
 create mode 100644 tensorflow/compiler/xla/service/cpu/external_constant_pool.h
 create mode 100644 tensorflow/compiler/xla/service/cpu/external_constant_pool_test.cc

diff --git a/tensorflow/compiler/xla/service/cpu/BUILD b/tensorflow/compiler/xla/service/cpu/BUILD
index 0daaa122f4..7933e226bf 100644
--- a/tensorflow/compiler/xla/service/cpu/BUILD
+++ b/tensorflow/compiler/xla/service/cpu/BUILD
@@ -48,6 +48,29 @@ cc_library(
     alwayslink = True,  # Contains per-platform transfer manager registration
 )
 
+cc_library(
+    name = "external_constant_pool",
+    srcs = ["external_constant_pool.cc"],
+    hdrs = ["external_constant_pool.h"],
+    deps = [
+        "//tensorflow/compiler/xla:literal_util",
+        "//tensorflow/compiler/xla:shape_util",
+        "//tensorflow/compiler/xla:util",
+        "//tensorflow/core:lib",
+    ],
+)
+
+tf_cc_test(
+    name = "external_constant_pool_test",
+    srcs = ["external_constant_pool_test.cc"],
+    deps = [
+        ":external_constant_pool",
+        "//tensorflow/compiler/xla:shape_util",
+        "//tensorflow/compiler/xla/tests:xla_internal_test_main",
+        "//tensorflow/core:test",
+    ],
+)
+
 cc_library(
     name = "cpu_compiler",
     srcs = ["cpu_compiler.cc"],
@@ -130,6 +153,7 @@ cc_library(
         ":cpu_runtime_neon",
         ":cpu_runtime_sse4_1",
         ":disassembler",
+        ":external_constant_pool",
         ":runtime_conv2d",
         ":runtime_matmul",
         ":runtime_single_threaded_conv2d",
@@ -217,6 +241,7 @@ cc_library(
         ":cpu_options",
         ":cpu_runtime",
         ":dot_op_emitter",
+        ":external_constant_pool",
         ":ir_emission_utils",
         ":simple_orc_jit",
         "//tensorflow/compiler/xla:shape_util",
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
index 2ad3578969..d0e366de57 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
@@ -522,7 +522,8 @@ StatusOr<std::unique_ptr<Executable>> CpuCompiler::Compile(
     }
 
     IrEmitter ir_emitter(*module, *assignment, llvm_module.get(),
-                         &hlo_to_profile_idx, jit->target_machine());
+                         &hlo_to_profile_idx, jit->target_machine(),
+                         jit->external_constant_pool());
 
     std::unique_ptr<std::map<HloInstruction*, string>> function_names(
         new std::map<HloInstruction*, string>());
@@ -602,7 +603,8 @@ StatusOr<std::unique_ptr<Executable>> CpuCompiler::Compile(
     // GetEmbeddedComputations guarantees that a called computation occurs
     // before a caller computation.
     IrEmitter ir_emitter(*module, *assignment, llvm_module.get(),
-                         &hlo_to_profile_idx, jit->target_machine());
+                         &hlo_to_profile_idx, jit->target_machine(),
+                         jit->external_constant_pool());
 
     for (auto embedded_computation :
          computation->MakeEmbeddedComputationsList()) {
@@ -771,7 +773,8 @@ CpuCompiler::CompileAheadOfTime(std::vector<std::unique_ptr<HloModule>> modules,
     }
 
     IrEmitter ir_emitter(*module, *assignment, &llvm_module,
-                         /*hlo_to_profile_idx=*/nullptr, target_machine.get());
+                         /*hlo_to_profile_idx=*/nullptr, target_machine.get(),
+                         /*external_constant_pool=*/nullptr);
     HloComputation* computation = module->entry_computation();
     for (auto embedded_computation :
          computation->MakeEmbeddedComputationsList()) {
diff --git a/tensorflow/compiler/xla/service/cpu/external_constant_pool.cc b/tensorflow/compiler/xla/service/cpu/external_constant_pool.cc
new file mode 100644
index 0000000000..c9f8e55849
--- /dev/null
+++ b/tensorflow/compiler/xla/service/cpu/external_constant_pool.cc
@@ -0,0 +1,53 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/service/cpu/external_constant_pool.h"
+
+#include <algorithm>
+#include <cstdlib>
+#include <cstring>
+
+#include "tensorflow/compiler/xla/map_util.h"
+#include "tensorflow/compiler/xla/ptr_util.h"
+#include "tensorflow/compiler/xla/shape_util.h"
+#include "tensorflow/core/lib/gtl/flatset.h"
+
+namespace xla {
+namespace cpu {
+void ExternalConstantPool::Insert(string name, const Literal& literal,
+                                  int64 alignment) {
+  CHECK(!ShapeUtil::IsTuple(literal.shape()));
+  CHECK(alignment > 0 && IsPowerOfTwo(static_cast<uint64>(alignment)));
+  CHECK(entries_.find(name) == entries_.end());
+
+  int64 literal_size = ShapeUtil::ByteSizeOf(literal.shape());
+  void* raw_pointer;
+  CHECK_EQ(
+      posix_memalign(&raw_pointer, std::max<size_t>(alignment, sizeof(void*)),
+                     literal_size),
+      0)
+      << "failed to allocate " << literal_size << " bytes with alignment of "
+      << alignment;
+
+  std::memcpy(raw_pointer, literal.InternalData(), literal_size);
+  entries_.emplace(std::move(name), static_cast<uint8*>(raw_pointer));
+}
+
+const uint8* ExternalConstantPool::Find(const string& name) {
+  auto it = entries_.find(name);
+  return it == entries_.end() ? nullptr : it->second.get();
+}
+}  // namespace cpu
+}  // namespace xla
diff --git a/tensorflow/compiler/xla/service/cpu/external_constant_pool.h b/tensorflow/compiler/xla/service/cpu/external_constant_pool.h
new file mode 100644
index 0000000000..ade28cbcbc
--- /dev/null
+++ b/tensorflow/compiler/xla/service/cpu/external_constant_pool.h
@@ -0,0 +1,64 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef THIRD_PARTY_TENSORFLOW_COMPILER_XLA_SERVICE_CPU_EXTERNAL_CONSTANT_POOL_H_
+#define THIRD_PARTY_TENSORFLOW_COMPILER_XLA_SERVICE_CPU_EXTERNAL_CONSTANT_POOL_H_
+
+#include <memory>
+
+#include "tensorflow/compiler/xla/literal_util.h"
+#include "tensorflow/core/lib/gtl/flatmap.h"
+
+namespace xla {
+namespace cpu {
+// An ExternalConstantPool maintains a set of constants kept external to
+// generated LLVM IR. These constants are accessed from the IR via globals with
+// extern linkage.  This current incarnation of ExternalConstantPool only
+// supports the JIT CPU backend; the AOT backend is not supported.
+//
+// Implementation-wise, this is a simple wrapper around a map of strings to byte
+// buffers.  This simply implementation works in a JIT scenario.  This class
+// will have to become smarter if we decide to support external constant pools
+// on AOT compiles in the future.
+class ExternalConstantPool {
+ public:
+  // Inserts a buffer with the contents of `literal` into the constant pool with
+  // the name `name`.  It is an error to try to insert two constants with the
+  // same `name` into the same constant pool.  The buffer for literal is aligned
+  // to `aligment` bytes, and `alignment` must be a power of 2.
+  //
+  // The constant pool copies out the contents of `literal` into a buffer it
+  // owns -- it does not keep pointers to `literal`, or to memory owned by
+  // `literal`.
+  void Insert(string name, const Literal& literal, int64 alignment);
+
+  // Find the constant with name `name` in this constant pool.  If there isn't
+  // such constant, return nullptr.
+  const uint8* Find(const string& name);
+
+ private:
+  // We need to `free()` pointers allocated into `entries_` since we allocate
+  // them with `posix_memalign`.
+  struct FreeDeleter {
+    void operator()(void* ptr) { free(ptr); }
+  };
+
+  tensorflow::gtl::FlatMap<string, std::unique_ptr<uint8, FreeDeleter>>
+      entries_;
+};
+}  // namespace cpu
+}  // namespace xla
+
+#endif  // THIRD_PARTY_TENSORFLOW_COMPILER_XLA_SERVICE_CPU_EXTERNAL_CONSTANT_POOL_H_
diff --git a/tensorflow/compiler/xla/service/cpu/external_constant_pool_test.cc b/tensorflow/compiler/xla/service/cpu/external_constant_pool_test.cc
new file mode 100644
index 0000000000..9290a4e5df
--- /dev/null
+++ b/tensorflow/compiler/xla/service/cpu/external_constant_pool_test.cc
@@ -0,0 +1,82 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/service/cpu/external_constant_pool.h"
+#include "tensorflow/compiler/xla/layout_util.h"
+#include "tensorflow/core/platform/test.h"
+
+namespace xla {
+namespace cpu {
+namespace {
+class ExternalConstantPoolTest : public ::testing::Test {};
+
+template <typename T>
+T GetFromBuffer(const uint8* buffer, int64 index) {
+  T result;
+  std::memcpy(&result, buffer + index * sizeof(T), sizeof(T));
+  return result;
+}
+
+TEST(ExternalConstantPoolTest, Basic) {
+  ExternalConstantPool constant_pool;
+  EXPECT_EQ(constant_pool.Find("name-0"), nullptr);
+  const auto literal = Literal::CreateR2({{1, 2}, {3, 4}});
+  constant_pool.Insert("name-0", *literal, 4);
+  const uint8* constant = constant_pool.Find("name-0");
+  ASSERT_NE(constant, nullptr);
+
+  EXPECT_EQ(GetFromBuffer<int32>(constant, 0), 1);
+  EXPECT_EQ(GetFromBuffer<int32>(constant, 1), 2);
+  EXPECT_EQ(GetFromBuffer<int32>(constant, 2), 3);
+  EXPECT_EQ(GetFromBuffer<int32>(constant, 3), 4);
+
+  EXPECT_EQ(constant_pool.Find("name-1"), nullptr);
+}
+
+TEST(ExternalConstantPoolTest, RowMinorLayout) {
+  ExternalConstantPool constant_pool;
+  EXPECT_EQ(constant_pool.Find("name-0"), nullptr);
+  const auto literal = Literal::CreateR2WithLayout(
+      {{1, 2}, {3, 4}}, LayoutUtil::MakeLayout({0, 1}));
+  constant_pool.Insert("name-0", *literal, 4);
+  const uint8* constant = constant_pool.Find("name-0");
+  ASSERT_NE(constant, nullptr);
+
+  EXPECT_EQ(GetFromBuffer<int32>(constant, 0), 1);
+  EXPECT_EQ(GetFromBuffer<int32>(constant, 1), 3);
+  EXPECT_EQ(GetFromBuffer<int32>(constant, 2), 2);
+  EXPECT_EQ(GetFromBuffer<int32>(constant, 3), 4);
+}
+
+TEST(ExternalConstantPoolTest, Alignment) {
+  ExternalConstantPool constant_pool;
+  EXPECT_EQ(constant_pool.Find("name-0"), nullptr);
+
+  for (int i = 0; i < 8; i++) {
+    int64 alignment = 1 << i;
+    string name = tensorflow::strings::StrCat("name-", i);
+
+    const auto literal = Literal::CreateR2({{1, 2}, {3, 4}});
+    constant_pool.Insert(name, *literal, alignment);
+
+    const uint8* constant = constant_pool.Find(name);
+    ASSERT_NE(constant, nullptr);
+    EXPECT_EQ(reinterpret_cast<intptr_t>(constant) % alignment, 0);
+  }
+}
+
+}  // namespace
+}  // namespace cpu
+}  // namespace xla
diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
index 5474862e45..89a911d070 100644
--- a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
@@ -75,7 +75,8 @@ IrEmitter::IrEmitter(
     const HloModule& hlo_module, const BufferAssignment& assignment,
     llvm::Module* llvm_module,
     const std::unordered_map<const HloInstruction*, size_t>* hlo_to_profile_idx,
-    llvm::TargetMachine* target_machine)
+    llvm::TargetMachine* target_machine,
+    ExternalConstantPool* external_constant_pool)
     : assignment_(assignment),
       module_(llvm_module),
       arch_type_(llvm::Triple(llvm_module->getTargetTriple()).getArch()),
@@ -86,7 +87,8 @@ IrEmitter::IrEmitter(
       parallel_cpu_backend_(
           options::CpuParallelBackendRequested(hlo_module_config_)),
       is_top_level_computation_(false),
-      target_machine_features_(target_machine) {
+      target_machine_features_(target_machine),
+      external_constant_pool_(external_constant_pool) {
   ir_builder_.setFastMathFlags(llvm_ir::GetFastMathFlags(
       /*fast_math_enabled=*/hlo_module_config_.debug_options()
           .xla_enable_fast_math()));
@@ -272,16 +274,39 @@ Status IrEmitter::HandleBitcast(HloInstruction* bitcast) {
 Status IrEmitter::HandleConstant(HloInstruction* constant,
                                  const Literal& literal) {
   VLOG(2) << "HandleConstant: " << constant->ToString();
-  llvm::Constant* initializer =
-      llvm_ir::ConvertLiteralToIrConstant(literal, &ir_builder_);
-  llvm::GlobalVariable* global_for_const = new llvm::GlobalVariable(
-      /*Module=*/*module_,
-      /*Type=*/initializer->getType(),
-      /*isConstant=*/true,
-      /*Linkage=*/llvm::GlobalValue::PrivateLinkage,
-      /*Initializer=*/initializer,
-      /*Name=*/"");
-  global_for_const->setAlignment(MinimumAlignmentForShape(literal.shape()));
+  llvm::GlobalVariable* global_for_const;
+
+  // We avoid creating large constants in the LLVM IR since LLVM is not
+  // efficient for large constant arrays.  We still emit "small enough" constant
+  // arrays into the Ir, in the off chance the LLVM optimizer can do something
+  // interesting with it.
+  const int kMaxInternalConstantSizeInBytes = 128;
+  if (external_constant_pool_ &&
+      ByteSizeOf(literal.shape()) >= kMaxInternalConstantSizeInBytes) {
+    string global_name = tensorflow::strings::StrCat(
+        "constant_global_", external_global_constant_counter_++);
+    global_for_const = new llvm::GlobalVariable(
+        /*Module=*/*module_,
+        /*Type=*/IrShapeType(literal.shape()),
+        /*isConstant=*/true,
+        /*Linkage=*/llvm::GlobalValue::ExternalLinkage,
+        /*Initializer=*/nullptr,
+        /*Name=*/AsStringRef(global_name));
+    global_for_const->setAlignment(MinimumAlignmentForShape(literal.shape()));
+    external_constant_pool_->Insert(global_name, literal,
+                                    MinimumAlignmentForShape(literal.shape()));
+  } else {
+    llvm::Constant* initializer =
+        llvm_ir::ConvertLiteralToIrConstant(literal, &ir_builder_);
+    global_for_const = new llvm::GlobalVariable(
+        /*Module=*/*module_,
+        /*Type=*/initializer->getType(),
+        /*isConstant=*/true,
+        /*Linkage=*/llvm::GlobalValue::PrivateLinkage,
+        /*Initializer=*/initializer,
+        /*Name=*/"");
+    global_for_const->setAlignment(MinimumAlignmentForShape(literal.shape()));
+  }
   emitted_value_[constant] = global_for_const;
   VLOG(2) << "  emitted value: " << llvm_ir::DumpToString(*global_for_const);
   VLOG(2) << "  its type: "
diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.h b/tensorflow/compiler/xla/service/cpu/ir_emitter.h
index b15026b6da..ba02f5f778 100644
--- a/tensorflow/compiler/xla/service/cpu/ir_emitter.h
+++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.h
@@ -29,6 +29,7 @@ limitations under the License.
 #include "llvm/IR/Value.h"
 #include "llvm/Target/TargetMachine.h"
 #include "tensorflow/compiler/xla/service/buffer_assignment.h"
+#include "tensorflow/compiler/xla/service/cpu/external_constant_pool.h"
 #include "tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h"
 #include "tensorflow/compiler/xla/service/hlo_computation.h"
 #include "tensorflow/compiler/xla/service/hlo_instruction.h"
@@ -104,11 +105,15 @@ class IrEmitter : public DfsHloVisitorWithDefault {
   // llvm_module: the LLVM module to emit IR into.
   // hlo_to_profile_idx: the mapping from HLO to its index in the profiling
   //                     array.
+  // external_constant_pool: if non-null, points to an ExternalConstantPool
+  //                         instance into which the Ir emitter can spill
+  //                         constants.
   IrEmitter(const HloModule& hlo_module, const BufferAssignment& assignment,
             llvm::Module* llvm_module,
             const std::unordered_map<const HloInstruction*, size_t>*
                 hlo_to_profile_idx,
-            llvm::TargetMachine* target_machine);
+            llvm::TargetMachine* target_machine,
+            ExternalConstantPool* external_constant_pool);
   ~IrEmitter() override;
 
   // Emit and return the given HLO computation as an LLVM IR
@@ -601,6 +606,9 @@ class IrEmitter : public DfsHloVisitorWithDefault {
 
   TargetMachineFeatures target_machine_features_;
 
+  int64 external_global_constant_counter_ = 0;
+  ExternalConstantPool* external_constant_pool_;
+
   TF_DISALLOW_COPY_AND_ASSIGN(IrEmitter);
 };
 
diff --git a/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc b/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc
index c3c11df090..c614e334a8 100644
--- a/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc
+++ b/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc
@@ -117,8 +117,20 @@ const JITSymbolTable& GetJITSymbolTable() {
 }
 
 // A simple SymbolResolver that delegates to the host dynamic linker.
-struct SimpleResolver : public llvm::JITSymbolResolver {
+class SimpleResolver : public llvm::JITSymbolResolver {
+ public:
+  explicit SimpleResolver(ExternalConstantPool* external_constant_pool)
+      : external_constant_pool_(external_constant_pool) {}
+
   llvm::JITSymbol findSymbol(const std::string& name) override {
+    string name_as_string(name);
+    if (const uint8* from_constant_pool =
+            external_constant_pool_->Find(string(name))) {
+      return llvm::JITEvaluatedSymbol(
+          reinterpret_cast<uint64_t>(from_constant_pool),
+          llvm::JITSymbolFlags::None);
+    }
+
     std::string canonical_name = CanonicalizeSymbol(name);
     const JITSymbolTable& jit_symbol_table = GetJITSymbolTable();
 
@@ -136,6 +148,9 @@ struct SimpleResolver : public llvm::JITSymbolResolver {
   llvm::JITSymbol findSymbolInLogicalDylib(const std::string& name) override {
     return nullptr;
   }
+
+ private:
+  ExternalConstantPool* external_constant_pool_;
 };
 
 llvm::SmallVector<std::string, 0> DetectMachineAttributes() {
@@ -205,7 +220,7 @@ SimpleOrcJIT::SimpleOrcJIT(const llvm::TargetOptions& target_options,
 SimpleOrcJIT::ModuleHandleT SimpleOrcJIT::AddModule(
     std::unique_ptr<llvm::Module> module) {
   auto handle = cantFail(compile_layer_.addModule(
-      std::move(module), MakeUnique<SimpleResolver>()));
+      std::move(module), MakeUnique<SimpleResolver>(external_constant_pool())));
   module_handles_.push_back(handle);
   return handle;
 }
diff --git a/tensorflow/compiler/xla/service/cpu/simple_orc_jit.h b/tensorflow/compiler/xla/service/cpu/simple_orc_jit.h
index e476c0e381..ded01e9e4d 100644
--- a/tensorflow/compiler/xla/service/cpu/simple_orc_jit.h
+++ b/tensorflow/compiler/xla/service/cpu/simple_orc_jit.h
@@ -27,6 +27,7 @@ limitations under the License.
 #include "llvm/Target/TargetMachine.h"
 #include "tensorflow/compiler/xla/service/cpu/compiler_functor.h"
 #include "tensorflow/compiler/xla/service/cpu/disassembler.h"
+#include "tensorflow/compiler/xla/service/cpu/external_constant_pool.h"
 #include "tensorflow/compiler/xla/types.h"
 
 namespace xla {
@@ -90,6 +91,10 @@ class SimpleOrcJIT {
 
   llvm::TargetMachine* target_machine() const { return target_machine_.get(); }
 
+  ExternalConstantPool* external_constant_pool() {
+    return &external_constant_pool_;
+  }
+
  private:
   std::vector<ModuleHandleT> module_handles_;
   std::unique_ptr<llvm::TargetMachine> target_machine_;
@@ -97,6 +102,7 @@ class SimpleOrcJIT {
   const llvm::DataLayout data_layout_;
   ObjLayerT object_layer_;
   CompileLayerT compile_layer_;
+  ExternalConstantPool external_constant_pool_;
 };
 
 }  // namespace cpu
-- 
GitLab


From 1be36dd6d675998842824f69285f146b95615042 Mon Sep 17 00:00:00 2001
From: Peter Hawkins <phawkins@google.com>
Date: Mon, 9 Oct 2017 21:01:13 -0700
Subject: [PATCH 216/909] [TF:XLA] Re-enable strided slice tests that now pass.

PiperOrigin-RevId: 171627028
---
 tensorflow/compiler/tests/BUILD | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/tensorflow/compiler/tests/BUILD b/tensorflow/compiler/tests/BUILD
index c8269b3d5b..eded6dc463 100644
--- a/tensorflow/compiler/tests/BUILD
+++ b/tensorflow/compiler/tests/BUILD
@@ -208,11 +208,6 @@ tf_xla_py_test(
     name = "slice_ops_test",
     size = "small",
     srcs = ["slice_ops_test.py"],
-    # TODO(b/62962492): Test fails with assertion error.
-    tags = [
-        "manual",
-        "notap",
-    ],
     deps = [
         ":xla_test",
         "//tensorflow/python:array_ops",
-- 
GitLab


From 90f257e0fc12e54d96d1e8a2afd374d1a2723577 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 9 Oct 2017 21:28:14 -0700
Subject: [PATCH 217/909] Fix ReshapeMover bug with reshaped constants; add
 HloVerifiedTestBase.

An example of a bad ReshapeMover rewrite:

BEFORE
  %reshape.1 = f32[1,1,128] reshape(f32[1,128] %dot)
  %constant = f32[128] constant({...})
  %reshape.2 = f32[1,1,128] reshape(f32[128] %constant)
  %add = f32[1,1,128] add(f32[1,1,128] %reshape.1, f32[1,1,128] %reshape.2)

AFTER
  %constant = f32[128] constant({...})
  %add = f32[1,128] add(f32[1,128] %dot, f32[128] %constant)
  %reshape = f32[1,1,128] reshape(f32[1,128] %add)

The problem in AFTER is the add now contains an implicit broadcast. One way to
fix this is to re-shape the %constant to f32[1,128] before the %add.

Instead of that, the fix introduced in this CL is to simply prevent the
ReshapeMover from moving the reshapes in this case. A comment in
reshape_mover.cc describes the complexities that led to this choice.

Also added HloVerifiedTestBase, which keeps track of a default HloModule, and
automatically runs HloVerifier at the end of every test. This is useful for many
HLO tests; the tests of various passes can probably all use this. Three existing
issues in reshape_mover_test.cc were found and fixed as a result.

PiperOrigin-RevId: 171628656
---
 tensorflow/compiler/xla/service/BUILD         |   2 +-
 .../compiler/xla/service/cpu/cpu_compiler.cc  |   2 +
 .../compiler/xla/service/reshape_mover.cc     | 275 +++++++++---------
 .../compiler/xla/service/reshape_mover.h      |   2 +-
 .../xla/service/reshape_mover_test.cc         | 124 +++++---
 tensorflow/compiler/xla/tests/BUILD           |  16 +
 .../xla/tests/hlo_verified_test_base.cc       |  69 +++++
 .../xla/tests/hlo_verified_test_base.h        |  63 ++++
 8 files changed, 371 insertions(+), 182 deletions(-)
 create mode 100644 tensorflow/compiler/xla/tests/hlo_verified_test_base.cc
 create mode 100644 tensorflow/compiler/xla/tests/hlo_verified_test_base.h

diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD
index 4b28467725..0c20a05714 100644
--- a/tensorflow/compiler/xla/service/BUILD
+++ b/tensorflow/compiler/xla/service/BUILD
@@ -1118,7 +1118,7 @@ tf_cc_test(
         "//tensorflow/compiler/xla:types",
         "//tensorflow/compiler/xla:util",
         "//tensorflow/compiler/xla:xla_data_proto",
-        "//tensorflow/compiler/xla/tests:hlo_test_base",
+        "//tensorflow/compiler/xla/tests:hlo_verified_test_base",
         "//tensorflow/compiler/xla/tests:xla_internal_test_main",
         "//tensorflow/core:lib",
     ],
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
index d0e366de57..386800d221 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
@@ -269,6 +269,8 @@ Status CpuCompiler::RunHloPasses(HloModule* module) {
   {
     auto& pass =
         pipeline.AddPass<HloPassFix<HloPassPipeline>>("simplification");
+    pass.AddInvariantChecker<HloVerifier>(ShapeSizeBytesFunction());
+
     pass.AddPass<BatchNormRewriter>(
         /*rewrite_training_op=*/true,
         /*rewrite_inference_op=*/true,
diff --git a/tensorflow/compiler/xla/service/reshape_mover.cc b/tensorflow/compiler/xla/service/reshape_mover.cc
index 404fd3e6d7..0fb90230f2 100644
--- a/tensorflow/compiler/xla/service/reshape_mover.cc
+++ b/tensorflow/compiler/xla/service/reshape_mover.cc
@@ -48,23 +48,28 @@ namespace xla {
 
 namespace {
 
-// Checks if an instruction can change its shape simply by adjusting metadata.
-// This is the case if it is:
-//
-// - an instruction does not have any producers like Constants
-// or Rng instruction, or is a scalar.
-//
-// Or
-//
-// - an reshape/transpose instruction with an operand that can trivially change
-// its shape.
-bool InstructionCanTriviallyChangeShape(const HloInstruction* instruction) {
-  // Reshape/Transposes are only trivial if their operand is trivial.
-  if (instruction->opcode() == HloOpcode::kReshape ||
-      instruction->opcode() == HloOpcode::kTranspose) {
-    CHECK_EQ(instruction->operand_count(), 1);
-    return InstructionCanTriviallyChangeShape(instruction->operand(0));
-  }
+bool IsReshapeOrTranspose(const HloInstruction* instruction) {
+  return instruction->opcode() == HloOpcode::kReshape ||
+         instruction->opcode() == HloOpcode::kTranspose;
+}
+
+// Returns true iff `instruction` can change its shape simply by adjusting
+// metadata.
+bool CanTriviallyChangeShape(const HloInstruction* instruction) {
+  // NOTE: Technically a sequence of reshape(reshape(constant)) is also
+  // trivially reshapable, so we might be tempted to simply recurse if
+  // IsReshapeOrTranspose(instruction)==true.
+  //
+  // But it's not that simple. E.g. reshape(reshape(rng)) is only trivially
+  // reshapable if *all* instructions in the chain have user_count == 1. And
+  // reshape(scalar) isn't trivial at all if the reshape itself isn't scalar; we
+  // rely on implicit scalar broadcast for scalars to be trivial. In addition,
+  // these cases make it harder to maintain correctness of the UpdateOperand
+  // logic below.
+  //
+  // So don't handle these chains, unless you update the tests and code to deal
+  // with these properly. One idea is to add a pass immediately beforehand that
+  // collapses trivial runs of reshapes / transposes.
 
   // Scalars can operate with any shape.
   if (ShapeUtil::IsScalar(instruction->shape())) {
@@ -93,9 +98,8 @@ HloInstruction* FirstNonScalarAndNonTrivialReshapeOperand(
     const HloInstruction* hlo) {
   for (HloInstruction* operand : hlo->operands()) {
     if (!ShapeUtil::IsScalar(operand->shape()) &&
-        ((operand->opcode() == HloOpcode::kReshape ||
-          operand->opcode() == HloOpcode::kTranspose) &&
-         !InstructionCanTriviallyChangeShape(operand->operand(0)))) {
+        IsReshapeOrTranspose(operand) &&
+        !CanTriviallyChangeShape(operand->operand(0))) {
       VLOG(5) << "Found first non-scalar and non-trivial reshape operand of "
               << hlo->ToStringNoMetadata() << ":\n\t"
               << operand->ToStringNoMetadata();
@@ -122,28 +126,15 @@ bool AreEquivalentReshapes(const HloInstruction* a, const HloInstruction* b) {
   }
 }
 
-// Returns true if an elementwise operation has all operands that can easily
-// change shape. Operands can easily change shape if they are all
-// reshapes/transposes to and from the same shape. Additionally, operands like
-// constant, rng, and any scalar change shape with only an adjustment of
-// metadata.
-bool IsElementwiseOfEquivalentReshapesOrTransposes(
-    const HloInstruction* instruction) {
-  const auto& operands = instruction->operands();
-  HloInstruction* first_reshape_operand =
-      FirstNonScalarAndNonTrivialReshapeOperand(instruction);
-  // If there are no non-trivial reshapes or transposes, then there is nothing
-  // to sink below the elementwise operation.
-  if (!first_reshape_operand) {
-    return false;
-  }
-  VLOG(3) << "** Checking whether instruction is an elementwise operation of "
-             "equivalent reshapes/transposes: "
+// Returns true if all operands of `instruction` can easily change shape.
+// Operands can easily change shape if they are all reshapes/transposes to and
+// from the same shape. Additionally, operands like constant, rng, and any
+// scalar change shape with only an adjustment of metadata.
+bool AllOperandsHaveEasyShapeChanges(
+    const HloInstruction* instruction,
+    const HloInstruction* first_reshape_operand) {
+  VLOG(3) << "** Checking whether all operands have easy shape changes: "
           << instruction->ToStringNoMetadata();
-  bool result = (instruction->user_count() > 0 ||
-                 instruction == instruction->parent()->root_instruction()) &&
-                instruction->IsElementwise() && !operands.empty();
-
   // Check whether all operands:
   //    0. Have the same dimensions as the output -- if not, it may be
   //       implicitly broadcast, which can confound the movement's
@@ -155,66 +146,117 @@ bool IsElementwiseOfEquivalentReshapesOrTransposes(
   //     or
   //    2. Are one of kConstant, kRng, and scalars that can change shape
   //    trivially,
-  if (result) {
-    for (auto& operand : operands) {
-      if (!ShapeUtil::SameDimensions(operand->shape(), instruction->shape())) {
-        VLOG(5) << "Operand shape differs from output shape; may be "
-                   "implicitly broadcast, so preventing "
-                   "movement\n\toperand: "
-                << operand->ToStringNoMetadata()
-                << "\n\tinstruction: " << instruction->ToStringNoMetadata();
-        result = false;
-        break;
-      }
-
-      if (AreEquivalentReshapes(first_reshape_operand, operand)) {
-        VLOG(5) << "Are equivalent reshapes:\n\tfirst_reshape_operand: "
-                << first_reshape_operand->ToStringNoMetadata()
-                << "\n\toperand: " << operand->ToStringNoMetadata();
-        continue;
-      }
+  for (const HloInstruction* operand : instruction->operands()) {
+    if (!ShapeUtil::SameDimensions(operand->shape(), instruction->shape())) {
+      VLOG(5) << "Operand shape differs from output shape; may be "
+                 "implicitly broadcast, so preventing "
+                 "movement\n\toperand: "
+              << operand->ToStringNoMetadata()
+              << "\n\tinstruction: " << instruction->ToStringNoMetadata();
+      return false;
+    }
 
-      if (InstructionCanTriviallyChangeShape(operand)) {
-        VLOG(5) << "Operand can trivially change shape: "
-                << operand->ToStringNoMetadata();
-        continue;
-      }
+    if (AreEquivalentReshapes(first_reshape_operand, operand)) {
+      VLOG(5) << "Are equivalent reshapes:\n\tfirst_reshape_operand: "
+              << first_reshape_operand->ToStringNoMetadata()
+              << "\n\toperand: " << operand->ToStringNoMetadata();
+      continue;
+    }
 
-      // TODO(someone): Look into supporting general ops for the operands as
-      // well.
-      VLOG(5) << "Operand is neither equalivant to the first Reshape operand"
-                 "nor can trivially change shape: "
+    if (CanTriviallyChangeShape(operand)) {
+      VLOG(5) << "Operand can trivially change shape: "
               << operand->ToStringNoMetadata();
-      result = false;
-      break;
+      continue;
     }
+
+    // TODO(someone): Look into supporting general ops for the operands as
+    // well.
+    VLOG(5) << "Operand is neither equalivant to the first Reshape operand"
+               "nor can trivially change shape: "
+            << operand->ToStringNoMetadata();
+    return false;
   }
 
-  VLOG(3) << "ElementwiseOfEquivalentReshapesOrTransposes result for "
-          << instruction->ToStringNoMetadata() << ": " << result;
-  return result;
+  VLOG(3) << "All operands have easy shape changes: "
+          << instruction->ToStringNoMetadata();
+  return true;
+}
+
+// This function is called once we've decided to sink reshape/transpose operands
+// across an instruction. It returns an updated `operand` with a shape that
+// plays nicely with `new_operand_shape`; either it has the same shape (of the
+// correct type), or it is a scalar that may be implicitly broadcast.
+HloInstruction* UpdateOperand(HloComputation* computation,
+                              const HloInstruction* first_reshape_operand,
+                              const Shape& new_operand_shape,
+                              HloInstruction* operand) {
+  const PrimitiveType element_type = operand->shape().element_type();
+  const Shape new_shape =
+      ShapeUtil::ChangeElementType(new_operand_shape, element_type);
+
+  switch (operand->opcode()) {
+    case HloOpcode::kConstant: {
+      if (first_reshape_operand->opcode() == HloOpcode::kReshape) {
+        VLOG(5) << "Adding reshape to kConstant operand";
+        return computation->AddInstruction(
+            HloInstruction::CreateReshape(new_shape, operand));
+      } else {
+        CHECK(first_reshape_operand->opcode() == HloOpcode::kTranspose);
+        VLOG(5) << "Adding transpose to kConstant operand";
+        std::vector<int64> inverse_permutation =
+            InversePermutation(first_reshape_operand->dimensions());
+        return computation->AddInstruction(HloInstruction::CreateTranspose(
+            new_shape, operand, inverse_permutation));
+      }
+    }
+    case HloOpcode::kRng: {
+      CHECK_EQ(operand->user_count(), 1);
+      VLOG(5) << "Cloning kRng operand with new shape";
+      return computation->AddInstruction(
+          operand->CloneWithNewOperands(new_shape, operand->operands()));
+    }
+    case HloOpcode::kReshape:
+    case HloOpcode::kTranspose: {
+      VLOG(5) << "Using existing operand of kReshape or kTranspose";
+      return operand->mutable_operand(0);
+    }
+    default:
+      LOG(FATAL) << "Unexpected operand opcode during update: " << operand;
+  }
 }
 
 // Try to sink any reshape or transpose operands of `instruction` across it. We
 // do so if `instruction` is elementwise and all operands are either equivalent
-// reshapes/transposes or are trivially reshapable. Note that no move is
-// performend if there is no nontrivial reshapes/transposes.
+// reshapes/transposes or are trivially reshapable.
 StatusOr<bool> TrySinkReshapeOrTranspose(HloComputation* computation,
                                          HloInstruction* instruction) {
-  if (!IsElementwiseOfEquivalentReshapesOrTransposes(instruction)) {
+  // Only perform sinks for live elementwise instructions with operands.
+  const bool is_dead = instruction->user_count() == 0 &&
+                       instruction != computation->root_instruction();
+  if (!instruction->IsElementwise() || instruction->operands().empty() ||
+      is_dead) {
     return false;
   }
 
-  HloInstruction* old_reshape =
+  // Only perform sinks if there are any nontrivial reshape/transpose operands.
+  const HloInstruction* first_reshape_operand =
       FirstNonScalarAndNonTrivialReshapeOperand(instruction);
-  TF_RET_CHECK(old_reshape != nullptr);
-  Shape new_elementwise_shape = old_reshape->operand(0)->shape();
+  if (!first_reshape_operand) {
+    return false;
+  }
+
+  // Only perform sinks if all operands can easily change shape.
+  if (!AllOperandsHaveEasyShapeChanges(instruction, first_reshape_operand)) {
+    return false;
+  }
 
-  VLOG(3) << "** Trying to sink reshape or transpose: "
-          << instruction->ToStringNoMetadata()
-          << "\n\told reshape: " << old_reshape->ToStringNoMetadata()
-          << "\n\tnew elementwise shape: "
-          << ShapeUtil::HumanString(new_elementwise_shape);
+  // At this point we've decided to sink reshape/transpose operands.
+  const Shape& new_operand_shape = first_reshape_operand->operand(0)->shape();
+  VLOG(3) << "** Sinking reshape or transpose: "
+          << instruction->ToStringNoMetadata() << "\n\tfirst reshape operand: "
+          << first_reshape_operand->ToStringNoMetadata()
+          << "\n\tnew operand shape: "
+          << ShapeUtil::HumanString(new_operand_shape);
 
   auto operands = instruction->operands();
   for (size_t i = 0; i < operands.size(); ++i) {
@@ -224,55 +266,19 @@ StatusOr<bool> TrySinkReshapeOrTranspose(HloComputation* computation,
     if (ShapeUtil::IsScalar(operands[i]->shape())) {
       continue;
     }
-    PrimitiveType element_type = operands[i]->shape().element_type();
-    switch (operands[i]->opcode()) {
-      case HloOpcode::kConstant: {
-        if (old_reshape->opcode() == HloOpcode::kReshape) {
-          VLOG(3) << "Creating reshape for kConstant operand " << i << ": "
-                  << operands[i]->ToStringNoMetadata();
-          operands[i] = instruction->parent()->AddInstruction(
-              HloInstruction::CreateReshape(
-                  ShapeUtil::ChangeElementType(new_elementwise_shape,
-                                               element_type),
-                  operands[i]));
-        } else {
-          TF_RET_CHECK(old_reshape->opcode() == HloOpcode::kTranspose);
-          std::vector<int64> inverse_permutation =
-              InversePermutation(old_reshape->dimensions());
-          operands[i] = instruction->parent()->AddInstruction(
-              HloInstruction::CreateTranspose(
-                  ShapeUtil::ChangeElementType(new_elementwise_shape,
-                                               element_type),
-                  operands[i], inverse_permutation));
-        }
-        break;
-      }
-      case HloOpcode::kRng: {
-        CHECK_EQ(operands[i]->user_count(), 1);
-        operands[i] = instruction->parent()->AddInstruction(
-            operands[i]->CloneWithNewOperands(
-                ShapeUtil::ChangeElementType(new_elementwise_shape,
-                                             element_type),
-                operands[i]->operands()));
-        break;
-      }
-      case HloOpcode::kReshape:
-      case HloOpcode::kTranspose:
-        operands[i] = operands[i]->mutable_operand(0);
-        break;
-      default:
-        LOG(FATAL) << "Unexpected opcode while trying to sink reshapes or "
-                      "transposes.";
-    }
+    VLOG(3) << "Updating operand #" << i << ": "
+            << operands[i]->ToStringNoMetadata();
+    operands[i] = UpdateOperand(computation, first_reshape_operand,
+                                new_operand_shape, operands[i]);
   }
   if (HloOpcode::kFusion == instruction->opcode()) {
     // Here we already know `instruction` is elementwise, and no operand is
-    // implicit broadcast as if it were the operands would not be equivalent
-    // reshapes, so all the fused instructions have the same dimensions.
+    // implicit broadcast as if it were the operands would not have easy shape
+    // changes, so all the fused instructions have the same dimensions.
     for (const auto& fused_instruction : instruction->fused_instructions()) {
       Shape* shape = fused_instruction->mutable_shape();
-      *shape->mutable_dimensions() = new_elementwise_shape.dimensions();
-      *shape->mutable_layout() = new_elementwise_shape.layout();
+      *shape->mutable_dimensions() = new_operand_shape.dimensions();
+      *shape->mutable_layout() = new_operand_shape.layout();
     }
   }
   HloInstruction* new_elementwise =
@@ -284,12 +290,12 @@ StatusOr<bool> TrySinkReshapeOrTranspose(HloComputation* computation,
           //
           // In this case, convert' should have the same element type as
           // `convert` and the same dimensions as operands[0].
-          ShapeUtil::ChangeElementType(new_elementwise_shape,
+          ShapeUtil::ChangeElementType(new_operand_shape,
                                        instruction->shape().element_type()),
           operands));
 
   std::unique_ptr<HloInstruction> new_reshape;
-  switch (old_reshape->opcode()) {
+  switch (first_reshape_operand->opcode()) {
     case HloOpcode::kReshape:
       VLOG(3) << "Creating new reshape for new elementwise op: "
               << new_elementwise->ToStringNoMetadata();
@@ -297,8 +303,9 @@ StatusOr<bool> TrySinkReshapeOrTranspose(HloComputation* computation,
           HloInstruction::CreateReshape(instruction->shape(), new_elementwise);
       break;
     case HloOpcode::kTranspose:
-      new_reshape = HloInstruction::CreateTranspose(
-          instruction->shape(), new_elementwise, old_reshape->dimensions());
+      new_reshape =
+          HloInstruction::CreateTranspose(instruction->shape(), new_elementwise,
+                                          first_reshape_operand->dimensions());
       break;
     default:
       LOG(FATAL) << "Bad opcode";
@@ -312,6 +319,8 @@ StatusOr<bool> TrySinkReshapeOrTranspose(HloComputation* computation,
 
 StatusOr<bool> ReshapeMover::Run(HloModule* module) {
   bool changed = false;
+  VLOG(2) << "Pre ReshapeMover HLO:";
+  XLA_VLOG_LINES(2, module->ToString());
   for (auto* comp : module->MakeNonfusionComputations()) {
     for (HloInstruction* instruction : comp->MakeInstructionPostOrder()) {
       TF_ASSIGN_OR_RETURN(bool did_change,
@@ -319,6 +328,8 @@ StatusOr<bool> ReshapeMover::Run(HloModule* module) {
       changed |= did_change;
     }
   }
+  VLOG(2) << "Post ReshapeMover HLO:";
+  XLA_VLOG_LINES(2, module->ToString());
   return changed;
 }
 
diff --git a/tensorflow/compiler/xla/service/reshape_mover.h b/tensorflow/compiler/xla/service/reshape_mover.h
index b7e0a46939..1f59e3b314 100644
--- a/tensorflow/compiler/xla/service/reshape_mover.h
+++ b/tensorflow/compiler/xla/service/reshape_mover.h
@@ -26,7 +26,7 @@ namespace xla {
 // them inputward also.
 class ReshapeMover : public HloPassInterface {
  public:
-  tensorflow::StringPiece name() const override { return "reshape-motion"; }
+  tensorflow::StringPiece name() const override { return "reshape-mover"; }
 
   StatusOr<bool> Run(HloModule* module) override;
 };
diff --git a/tensorflow/compiler/xla/service/reshape_mover_test.cc b/tensorflow/compiler/xla/service/reshape_mover_test.cc
index a81d3f4eb3..aac8638a54 100644
--- a/tensorflow/compiler/xla/service/reshape_mover_test.cc
+++ b/tensorflow/compiler/xla/service/reshape_mover_test.cc
@@ -25,7 +25,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/shape_util.h"
 #include "tensorflow/compiler/xla/test.h"
 #include "tensorflow/compiler/xla/test_helpers.h"
-#include "tensorflow/compiler/xla/tests/hlo_test_base.h"
+#include "tensorflow/compiler/xla/tests/hlo_verified_test_base.h"
 #include "tensorflow/compiler/xla/types.h"
 #include "tensorflow/compiler/xla/xla_data.pb.h"
 #include "tensorflow/core/lib/strings/str_util.h"
@@ -34,7 +34,7 @@ namespace op = xla::testing::opcode_matchers;
 
 namespace xla {
 namespace {
-using ReshapeMoverTest = HloTestBase;
+using ReshapeMoverTest = HloVerifiedTestBase;
 
 TEST_F(ReshapeMoverTest, ReshapesWithDifferentInputShapesNotMoved) {
   HloComputation::Builder builder(TestName());
@@ -50,13 +50,12 @@ TEST_F(ReshapeMoverTest, ReshapesWithDifferentInputShapesNotMoved) {
   builder.AddInstruction(HloInstruction::CreateBinary(
       root_shape, HloOpcode::kAdd, reshape0, reshape1));
 
-  auto module = CreateNewModule();
-  auto computation = module->AddEntryComputation(builder.Build());
+  auto computation = module().AddEntryComputation(builder.Build());
 
   EXPECT_THAT(computation->root_instruction(),
               op::Add(op::Reshape(param0), op::Reshape(param1)));
 
-  EXPECT_FALSE(ReshapeMover().Run(module.get()).ValueOrDie());
+  EXPECT_FALSE(ReshapeMover().Run(&module()).ValueOrDie());
 
   EXPECT_THAT(computation->root_instruction(),
               op::Add(op::Reshape(param0), op::Reshape(param1)));
@@ -89,13 +88,12 @@ TEST_F(ReshapeMoverTest, 1ConstantAnd1ReshapesOnRngNotMoved) {
   builder.AddInstruction(HloInstruction::CreateBinary(
       root_shape, HloOpcode::kAdd, reshape0, const1));
 
-  auto module = CreateNewModule();
-  auto computation = module->AddEntryComputation(builder.Build());
+  auto computation = module().AddEntryComputation(builder.Build());
 
   EXPECT_THAT(computation->root_instruction(),
               op::Add(op::Reshape(rng0), const1));
 
-  EXPECT_FALSE(ReshapeMover().Run(module.get()).ValueOrDie());
+  EXPECT_FALSE(ReshapeMover().Run(&module()).ValueOrDie());
 
   EXPECT_THAT(computation->root_instruction(),
               op::Add(op::Reshape(rng0), const1));
@@ -115,13 +113,12 @@ TEST_F(ReshapeMoverTest, ScalarReshapesNotMoved) {
   builder.AddInstruction(HloInstruction::CreateBinary(
       root_shape, HloOpcode::kAdd, reshape0, reshape1));
 
-  auto module = CreateNewModule();
-  auto computation = module->AddEntryComputation(builder.Build());
+  auto computation = module().AddEntryComputation(builder.Build());
 
   EXPECT_THAT(computation->root_instruction(),
               op::Add(op::Reshape(param0), op::Reshape(param1)));
 
-  EXPECT_FALSE(ReshapeMover().Run(module.get()).ValueOrDie());
+  EXPECT_FALSE(ReshapeMover().Run(&module()).ValueOrDie());
 
   EXPECT_THAT(
       computation->root_instruction(),
@@ -142,12 +139,11 @@ TEST_F(ReshapeMoverTest, EquivalentReshapesMoved) {
   builder.AddInstruction(HloInstruction::CreateBinary(
       root_shape, HloOpcode::kAdd, reshape0, reshape1));
 
-  auto module = CreateNewModule();
-  auto computation = module->AddEntryComputation(builder.Build());
+  auto computation = module().AddEntryComputation(builder.Build());
 
   EXPECT_THAT(computation->root_instruction(),
               op::Add(op::Reshape(param0), op::Reshape(param1)));
-  EXPECT_TRUE(ReshapeMover().Run(module.get()).ValueOrDie());
+  EXPECT_TRUE(ReshapeMover().Run(&module()).ValueOrDie());
 
   EXPECT_THAT(computation->root_instruction(),
               op::Reshape(op::Add(param0, param1)));
@@ -193,21 +189,19 @@ TEST_F(ReshapeMoverTest, 1ConstantAnd2ReshapesMoved) {
       builder.AddInstruction(HloInstruction::CreateReshape(root_shape, param2));
 
   builder.AddInstruction(HloInstruction::CreateTernary(
-      ShapeUtil::MakeShape(PRED, {2, 3}), HloOpcode::kSelect, const0, reshape1,
-      reshape2));
+      root_shape, HloOpcode::kSelect, const0, reshape1, reshape2));
 
-  auto module = CreateNewModule();
-  auto computation = module->AddEntryComputation(builder.Build());
+  auto computation = module().AddEntryComputation(builder.Build());
 
   EXPECT_THAT(computation->root_instruction(),
               op::Select(const0, reshape1, reshape2));
 
-  EXPECT_TRUE(ReshapeMover().Run(module.get()).ValueOrDie());
+  EXPECT_TRUE(ReshapeMover().Run(&module()).ValueOrDie());
 
   EXPECT_THAT(computation->root_instruction(),
               op::Reshape(op::Select(op::Reshape(const0), param1, param2)));
 
-  EXPECT_EQ(const0->shape().DebugString(),
+  EXPECT_EQ(root_shape.DebugString(),
             computation->root_instruction()->shape().DebugString());
 }
 
@@ -228,17 +222,16 @@ TEST_F(ReshapeMoverTest, 1ParameterAnd1ReshapeNotMoved) {
       0, ShapeUtil::MakeShape(F32, {1, 8, 1, 7}), "param0"));
   auto reshape0 =
       builder.AddInstruction(HloInstruction::CreateReshape(root_shape, param0));
-  auto param1 = builder.AddInstruction(HloInstruction::CreateParameter(
-      1, ShapeUtil::MakeShape(F32, {1, 8, 1, 7}), "param1"));
+  auto param1 = builder.AddInstruction(
+      HloInstruction::CreateParameter(1, root_shape, "param1"));
   builder.AddInstruction(HloInstruction::CreateBinary(
       root_shape, HloOpcode::kAdd, reshape0, param1));
 
-  auto module = CreateNewModule();
-  auto computation = module->AddEntryComputation(builder.Build());
+  auto computation = module().AddEntryComputation(builder.Build());
 
   EXPECT_THAT(computation->root_instruction(),
               op::Add(op::Reshape(param0), param1));
-  EXPECT_FALSE(ReshapeMover().Run(module.get()).ValueOrDie());
+  EXPECT_FALSE(ReshapeMover().Run(&module()).ValueOrDie());
 
   EXPECT_THAT(computation->root_instruction(),
               op::Add(op::Reshape(param0), param1));
@@ -260,7 +253,7 @@ TEST_F(ReshapeMoverTest, 1ParameterAnd1ReshapeNotMoved) {
 // trivial reshapes.
 TEST_F(ReshapeMoverTest, 2TrivialConstantReshapeNotMoved) {
   HloComputation::Builder builder(TestName());
-  auto root_shape = ShapeUtil::MakeShape(F32, {2, 3});
+  auto root_shape = ShapeUtil::MakeShape(F32, {3, 2});
   auto const0 = builder.AddInstruction(HloInstruction::CreateConstant(
       Literal::CreateR2<float>({{1, 2, 3}, {4, 5, 6}})));
   auto reshape0 =
@@ -272,18 +265,17 @@ TEST_F(ReshapeMoverTest, 2TrivialConstantReshapeNotMoved) {
       builder.AddInstruction(HloInstruction::CreateReshape(root_shape, const1));
 
   auto pred = builder.AddInstruction(HloInstruction::CreateParameter(
-      0, ShapeUtil::MakeShape(PRED, {1, 3, 1, 2}), "pred"));
+      0, ShapeUtil::MakeShape(PRED, {3, 2}), "pred"));
 
   builder.AddInstruction(HloInstruction::CreateTernary(
       root_shape, HloOpcode::kSelect, pred, reshape0, reshape1));
 
-  auto module = CreateNewModule();
-  auto computation = module->AddEntryComputation(builder.Build());
+  auto computation = module().AddEntryComputation(builder.Build());
 
   EXPECT_THAT(computation->root_instruction(),
               op::Select(pred, op::Reshape(const0), op::Reshape(const1)));
 
-  EXPECT_FALSE(ReshapeMover().Run(module.get()).ValueOrDie());
+  EXPECT_FALSE(ReshapeMover().Run(&module()).ValueOrDie());
 
   EXPECT_THAT(computation->root_instruction(),
               op::Select(pred, op::Reshape(const0), op::Reshape(const1)));
@@ -323,13 +315,12 @@ TEST_F(ReshapeMoverTest, 1NonTrivialReshapeMoved) {
   builder.AddInstruction(HloInstruction::CreateBinary(
       root_shape, HloOpcode::kAdd, reshape0, const1));
 
-  auto module = CreateNewModule();
-  auto computation = module->AddEntryComputation(builder.Build());
+  auto computation = module().AddEntryComputation(builder.Build());
 
   EXPECT_THAT(computation->root_instruction(),
               op::Add(op::Reshape(param0), const1));
 
-  EXPECT_TRUE(ReshapeMover().Run(module.get()).ValueOrDie());
+  EXPECT_TRUE(ReshapeMover().Run(&module()).ValueOrDie());
 
   EXPECT_THAT(computation->root_instruction(),
               op::Reshape(op::Add(param0, op::Reshape(const1))));
@@ -337,6 +328,48 @@ TEST_F(ReshapeMoverTest, 1NonTrivialReshapeMoved) {
             computation->root_instruction()->shape().DebugString());
 }
 
+// For a graph that looks like:
+//
+// +- reshape0 - param0 (shape A)
+// |
+// +- reshape1 - const1 (shape B)
+// |
+// add
+//
+// There is 1 non-trivial reshape (reshape0). It's not clear whether reshape1
+// should be trivial or not; conceptually it's trivial, but handling it would
+// complicate the rest of our logic.
+//
+// For now we treat it as non-trivial, so we verify that we don't sink the
+// reshapes in this case.
+TEST_F(ReshapeMoverTest, 1NonTrivialReshapeWith1ReshapedConstNotMoved) {
+  HloComputation::Builder builder(TestName());
+  auto root_shape = ShapeUtil::MakeShape(F32, {1, 1, 3});
+  auto param0 = builder.AddInstruction(HloInstruction::CreateParameter(
+      0, ShapeUtil::MakeShape(F32, {1, 3}), "param0"));
+  auto const1 = builder.AddInstruction(
+      HloInstruction::CreateConstant(Literal::CreateR1<float>({9, 8, 7})));
+  auto reshape0 =
+      builder.AddInstruction(HloInstruction::CreateReshape(root_shape, param0));
+  auto reshape1 =
+      builder.AddInstruction(HloInstruction::CreateReshape(root_shape, const1));
+
+  builder.AddInstruction(HloInstruction::CreateBinary(
+      root_shape, HloOpcode::kAdd, reshape0, reshape1));
+
+  auto computation = module().AddEntryComputation(builder.Build());
+
+  EXPECT_THAT(computation->root_instruction(),
+              op::Add(op::Reshape(param0), op::Reshape(const1)));
+
+  EXPECT_FALSE(ReshapeMover().Run(&module()).ValueOrDie());
+
+  EXPECT_THAT(computation->root_instruction(),
+              op::Add(op::Reshape(param0), op::Reshape(const1)));
+  EXPECT_EQ(root_shape.DebugString(),
+            computation->root_instruction()->shape().DebugString());
+}
+
 TEST_F(ReshapeMoverTest, EquivalentReshapesMovedAcrossFusion) {
   HloComputation::Builder builder(TestName());
   auto root_shape = ShapeUtil::MakeShape(F32, {8, 7});
@@ -351,15 +384,14 @@ TEST_F(ReshapeMoverTest, EquivalentReshapesMovedAcrossFusion) {
   auto add = builder.AddInstruction(HloInstruction::CreateBinary(
       root_shape, HloOpcode::kAdd, reshape0, reshape1));
 
-  HloModule module(TestName());
-  auto computation = module.AddEntryComputation(builder.Build());
+  auto computation = module().AddEntryComputation(builder.Build());
   computation->CreateFusionInstruction({add},
                                        HloInstruction::FusionKind::kLoop);
 
   EXPECT_THAT(computation->root_instruction(),
               op::Fusion(op::Reshape(param0), op::Reshape(param1)));
 
-  EXPECT_TRUE(ReshapeMover().Run(&module).ValueOrDie());
+  EXPECT_TRUE(ReshapeMover().Run(&module()).ValueOrDie());
 
   EXPECT_THAT(computation->root_instruction(),
               op::Reshape(op::Fusion(param0, param1)));
@@ -386,14 +418,13 @@ TEST_F(ReshapeMoverTest, EquivalentReshapesMovedAcrossSelect) {
   builder.AddInstruction(HloInstruction::CreateTernary(
       root_shape, HloOpcode::kSelect, reshape_pred, reshape0, reshape1));
 
-  auto module = CreateNewModule();
-  auto computation = module->AddEntryComputation(builder.Build());
+  auto computation = module().AddEntryComputation(builder.Build());
 
   EXPECT_THAT(
       computation->root_instruction(),
       op::Select(op::Reshape(pred), op::Reshape(param0), op::Reshape(param1)));
 
-  EXPECT_TRUE(ReshapeMover().Run(module.get()).ValueOrDie());
+  EXPECT_TRUE(ReshapeMover().Run(&module()).ValueOrDie());
 
   EXPECT_THAT(computation->root_instruction(),
               op::Reshape(op::Select(pred, param0, param1)));
@@ -416,12 +447,11 @@ TEST_F(ReshapeMoverTest, ScalarReshapeNotMovedAcrossSelect) {
   auto select = builder.AddInstruction(HloInstruction::CreateTernary(
       root_shape, HloOpcode::kSelect, reshape_pred, param0, param1));
 
-  auto module = CreateNewModule();
-  auto computation = module->AddEntryComputation(builder.Build());
+  auto computation = module().AddEntryComputation(builder.Build());
   EXPECT_THAT(computation->root_instruction(),
               op::Select(op::Reshape(pred), param0, param1));
 
-  EXPECT_FALSE(ReshapeMover().Run(module.get()).ValueOrDie());
+  EXPECT_FALSE(ReshapeMover().Run(&module()).ValueOrDie());
 
   EXPECT_THAT(computation->root_instruction(),
               op::Select(op::Reshape(pred), param0, param1));
@@ -468,12 +498,11 @@ TEST_F(ReshapeMoverTest, ImplicitlyBroadcastReshapeIsNotMovedBug37787999) {
   auto multiply = builder.AddInstruction(HloInstruction::CreateBinary(
       constant->shape(), HloOpcode::kMultiply, constant, reshape));
 
-  auto module = CreateNewModule();
-  auto computation = module->AddEntryComputation(builder.Build());
+  auto computation = module().AddEntryComputation(builder.Build());
   EXPECT_THAT(computation->root_instruction(),
               op::Multiply(op::Constant(), op::Reshape(param0)));
 
-  EXPECT_FALSE(ReshapeMover().Run(module.get()).ValueOrDie());
+  EXPECT_FALSE(ReshapeMover().Run(&module()).ValueOrDie());
 
   EXPECT_THAT(computation->root_instruction(),
               op::Multiply(op::Constant(), op::Reshape(param0)));
@@ -517,15 +546,14 @@ TEST_F(ReshapeMoverTest, MultiplePasses) {
   builder.AddInstruction(HloInstruction::CreateBinary(shape3, HloOpcode::kAdd,
                                                       reshape2, reshape3));
 
-  auto module = CreateNewModule();
-  auto computation = module->AddEntryComputation(builder.Build());
+  auto computation = module().AddEntryComputation(builder.Build());
 
   EXPECT_THAT(
       computation->root_instruction(),
       op::Add(op::Reshape(param2),
               op::Reshape(op::Add(op::Reshape(param0), op::Reshape(param1)))));
 
-  EXPECT_TRUE(ReshapeMover().Run(module.get()).ValueOrDie());
+  EXPECT_TRUE(ReshapeMover().Run(&module()).ValueOrDie());
 
   EXPECT_THAT(
       computation->root_instruction(),
diff --git a/tensorflow/compiler/xla/tests/BUILD b/tensorflow/compiler/xla/tests/BUILD
index e45b839afd..f37a331a72 100644
--- a/tensorflow/compiler/xla/tests/BUILD
+++ b/tensorflow/compiler/xla/tests/BUILD
@@ -127,6 +127,22 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "hlo_verified_test_base",
+    testonly = True,
+    srcs = ["hlo_verified_test_base.cc"],
+    hdrs = ["hlo_verified_test_base.h"],
+    deps = [
+        ":hlo_test_base",
+        "//tensorflow/compiler/xla:shape_util",
+        "//tensorflow/compiler/xla:status_macros",
+        "//tensorflow/compiler/xla/service:hlo",
+        "//tensorflow/compiler/xla/service:hlo_verifier",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:test",
+    ],
+)
+
 tf_cc_binary(
     name = "local_client_aot_test_helper",
     srcs = ["local_client_aot_test_helper.cc"],
diff --git a/tensorflow/compiler/xla/tests/hlo_verified_test_base.cc b/tensorflow/compiler/xla/tests/hlo_verified_test_base.cc
new file mode 100644
index 0000000000..31060b9e80
--- /dev/null
+++ b/tensorflow/compiler/xla/tests/hlo_verified_test_base.cc
@@ -0,0 +1,69 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/tests/hlo_verified_test_base.h"
+
+#include "tensorflow/compiler/xla/service/hlo_verifier.h"
+#include "tensorflow/compiler/xla/shape_util.h"
+#include "tensorflow/compiler/xla/status_macros.h"
+#include "tensorflow/core/platform/logging.h"
+#include "tensorflow/core/platform/test.h"
+
+namespace xla {
+
+/*static*/ int64 HloVerifiedTestBase::DefaultShapeSize(const Shape& shape) {
+  constexpr int64 kPointerSize = sizeof(void*);
+  if (ShapeUtil::IsOpaque(shape)) {
+    return kPointerSize;
+  }
+  return ShapeUtil::ByteSizeOf(shape, kPointerSize);
+}
+
+HloVerifiedTestBase::HloVerifiedTestBase() : shape_size_fn_(DefaultShapeSize) {}
+
+HloVerifiedTestBase::~HloVerifiedTestBase() {
+  // We can't call the ASSERT or EXPECT test macros in destructors, so we
+  // perform HLO verification in TearDown, and use the CHECK here to ensure
+  // users don't accidentally override the verification.
+  CHECK(tear_down_called_)
+      << "TearDown was never called; subclasses of HloVerifiedTestBase that "
+      << "override TearDown must call the superclass TearDown.";
+}
+
+void HloVerifiedTestBase::TearDown() {
+  EXPECT_FALSE(tear_down_called_)
+      << "TearDown called more than once; it should be called exactly once.";
+  tear_down_called_ = true;
+  if (module_) {
+    HloVerifier verifier(shape_size_fn_);
+    xla::StatusOr<bool> mutated = verifier.Run(module_.get());
+    if (!mutated.ok()) {
+      ADD_FAILURE() << "HloVerifier failed: " << mutated.status();
+    } else {
+      EXPECT_FALSE(mutated.ValueOrDie())
+          << "HloVerifier should never mutate the HloModule";
+    }
+  }
+  HloTestBase::TearDown();
+}
+
+HloModule& HloVerifiedTestBase::module() {
+  if (!module_) {
+    module_ = CreateNewModule();
+  }
+  return *module_;
+}
+
+}  // namespace xla
diff --git a/tensorflow/compiler/xla/tests/hlo_verified_test_base.h b/tensorflow/compiler/xla/tests/hlo_verified_test_base.h
new file mode 100644
index 0000000000..b3d6b5af3b
--- /dev/null
+++ b/tensorflow/compiler/xla/tests/hlo_verified_test_base.h
@@ -0,0 +1,63 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_COMPILER_XLA_TESTS_HLO_VERIFIED_TEST_BASE_H_
+#define TENSORFLOW_COMPILER_XLA_TESTS_HLO_VERIFIED_TEST_BASE_H_
+
+#include <functional>
+#include <memory>
+#include <utility>
+
+#include "tensorflow/compiler/xla/service/hlo_module.h"
+#include "tensorflow/compiler/xla/tests/hlo_test_base.h"
+
+namespace xla {
+
+// A base class for HLO tests that stores a default HloModule, and automatically
+// performs verification on that module on tear-down.
+class HloVerifiedTestBase : public HloTestBase {
+ public:
+  // Returns the size in bytes of the given shape, using a default pointer size.
+  static int64 DefaultShapeSize(const Shape& shape);
+
+ protected:
+  HloVerifiedTestBase();
+  ~HloVerifiedTestBase() override;
+
+  // Performs verification on the default HloModule returned by module().
+  // Automatically called by the testing framework for each test.
+  //
+  // REQUIRED: subclasses that override TearDown() must call this explicitly.
+  void TearDown() override;
+
+  // Returns the default HloModule, lazily creating it if necessary via
+  // HloTestBase::CreateNewModule().
+  HloModule& module();
+
+  // Sets the shape-size function used during hlo verification. If this isn't
+  // called, DefaultShapeSize is used instead.
+  void SetShapeSizeFn(std::function<int64(const Shape&)> shape_size_fn) {
+    shape_size_fn_ = std::move(shape_size_fn);
+  }
+
+ private:
+  std::unique_ptr<HloModule> module_;  // Lazily populated. Access via module().
+  std::function<int64(const Shape&)> shape_size_fn_;
+  bool tear_down_called_ = false;
+};
+
+}  // namespace xla
+
+#endif  // TENSORFLOW_COMPILER_XLA_TESTS_HLO_VERIFIED_TEST_BASE_H_
-- 
GitLab


From 4b6eacbcdb8ca5182f83eee89edad24d87420b8e Mon Sep 17 00:00:00 2001
From: Taehoon Lee <me@taehoonlee.com>
Date: Tue, 10 Oct 2017 22:27:05 +0900
Subject: [PATCH 218/909] Fix typos

---
 tensorflow/c/c_api.h                                        | 2 +-
 .../contrib/cudnn_rnn/python/kernel_tests/cudnn_rnn_test.py | 6 +++---
 tensorflow/contrib/mpi_collectives/__init__.py              | 2 +-
 tensorflow/core/graph/graph.h                               | 2 +-
 tensorflow/core/grappler/optimizers/model_pruner.cc         | 2 +-
 tensorflow/core/profiler/README.md                          | 2 +-
 6 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/tensorflow/c/c_api.h b/tensorflow/c/c_api.h
index db94828e1a..7c31b04ed1 100644
--- a/tensorflow/c/c_api.h
+++ b/tensorflow/c/c_api.h
@@ -1144,7 +1144,7 @@ TF_CAPI_EXPORT extern TF_Function* TF_FunctionImportFunctionDef(
     const void* proto, size_t proto_len, TF_Status* status);
 
 // Sets function attribute named `attr_name` to value stored in `proto`.
-// If this attribute is already set to another value, it is overriden.
+// If this attribute is already set to another value, it is overridden.
 // `proto` should point to a sequence of bytes of length `proto_len`
 // representing a binary serialization of an AttrValue protocol
 // buffer.
diff --git a/tensorflow/contrib/cudnn_rnn/python/kernel_tests/cudnn_rnn_test.py b/tensorflow/contrib/cudnn_rnn/python/kernel_tests/cudnn_rnn_test.py
index 9e627bcaf4..1ce8954bb0 100644
--- a/tensorflow/contrib/cudnn_rnn/python/kernel_tests/cudnn_rnn_test.py
+++ b/tensorflow/contrib/cudnn_rnn/python/kernel_tests/cudnn_rnn_test.py
@@ -385,7 +385,7 @@ class CudnnRNNTestSaveRestore(TensorFlowTestCase):
       reset_op = state_ops.assign(
           opaque_params,
           array_ops.zeros(array_ops.shape(opaque_params), dtype=dtype))
-      # Passing graph explictly, otherwise an old sess would be reused.
+      # Passing graph explicitly, otherwise an old sess would be reused.
       with self.test_session(use_gpu=True, graph=g) as sess:
         sess.run(variables.global_variables_initializer())
         val = saver.save(sess, save_path)
@@ -436,7 +436,7 @@ class CudnnRNNTestSaveRestore(TensorFlowTestCase):
       save_path = os.path.join(self.get_temp_dir(),
                                "save-restore-variable-test2")
       saver = saver_lib.Saver()
-      # Passing graph explictly, otherwise an old sess would be reused.
+      # Passing graph explicitly, otherwise an old sess would be reused.
       with self.test_session(use_gpu=True, graph=g) as sess:
         sess.run(variables.global_variables_initializer())
         val = saver.save(sess, save_path)
@@ -484,7 +484,7 @@ class CudnnRNNTestSaveRestore(TensorFlowTestCase):
           array_ops.zeros(
               array_ops.shape(rnn.trainable_variables[0]), dtype=dtype))
 
-      # Passing graph explictly, otherwise an old sess would be reused.
+      # Passing graph explicitly, otherwise an old sess would be reused.
       with self.test_session(use_gpu=True, graph=g) as sess:
         sess.run(variables.global_variables_initializer())
         inputs, initial_state = model.SynthesizeInput(seq_length, batch_size)
diff --git a/tensorflow/contrib/mpi_collectives/__init__.py b/tensorflow/contrib/mpi_collectives/__init__.py
index b94f7b0a35..9ed16a6f07 100644
--- a/tensorflow/contrib/mpi_collectives/__init__.py
+++ b/tensorflow/contrib/mpi_collectives/__init__.py
@@ -194,7 +194,7 @@ class DistributedOptimizer(tf.train.Optimizer):
 
     See Optimizer.compute_gradients() for more info.
 
-    In DistributedOptimizer, compute_gradients() is overriden to also
+    In DistributedOptimizer, compute_gradients() is overridden to also
     allreduce the gradients before returning them.
     """
     gradients = (super(DistributedOptimizer, self)
diff --git a/tensorflow/core/graph/graph.h b/tensorflow/core/graph/graph.h
index 5a31a6216b..418ce63bcb 100644
--- a/tensorflow/core/graph/graph.h
+++ b/tensorflow/core/graph/graph.h
@@ -631,7 +631,7 @@ class Graph {
   std::unordered_map<string, int> device_names_map_;
 
   // All the while contexts owned by this graph, keyed by frame name,
-  // corresonding to all the while loops contained in this graph (including
+  // corresponding to all the while loops contained in this graph (including
   // nested loops). The stored contexts are usually accessed via
   // AddWhileContext() or Node::while_ctx(), but this manages the lifetime.
   std::map<string, WhileContext> while_ctxs_;
diff --git a/tensorflow/core/grappler/optimizers/model_pruner.cc b/tensorflow/core/grappler/optimizers/model_pruner.cc
index e087621c3b..b9df196f83 100644
--- a/tensorflow/core/grappler/optimizers/model_pruner.cc
+++ b/tensorflow/core/grappler/optimizers/model_pruner.cc
@@ -104,7 +104,7 @@ Status ModelPruner::Optimize(Cluster* cluster, const GrapplerItem& item,
     // - Don't remove nodes that receive reference values, as those can be
     //   converting references to non-references. It is important to preserve
     //   these non-references since the partitioner will avoid sending
-    //   non-references accross partitions more than once.
+    //   non-references across partitions more than once.
     if (!rewriter.DrivesControlDependency(node) &&
         !rewriter.IsDrivenByControlDependency(node) &&
         !rewriter.IsConnectedToFunction(node) &&
diff --git a/tensorflow/core/profiler/README.md b/tensorflow/core/profiler/README.md
index 92bce9c1ce..8ca26fa5dc 100644
--- a/tensorflow/core/profiler/README.md
+++ b/tensorflow/core/profiler/README.md
@@ -48,7 +48,7 @@ bazel-bin/tensorflow/python/profiler/profiler_ui \
 # Create options to profile the time and memory information.
 builder = tf.profiler.ProfileOptionBuilder
 opts = builder(builder.time_and_memory()).order_by('micros').build()
-# Create a profiling context, set contructor argument `trace_steps`, 
+# Create a profiling context, set constructor argument `trace_steps`, 
 # `dump_steps` to empty for explicit control.
 with tf.contrib.tfprof.ProfileContext('/tmp/train_dir',
                                       trace_steps=[],
-- 
GitLab


From 5a26d1ede506825455d1199267d88caeba7d206a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 10 Oct 2017 07:02:27 -0700
Subject: [PATCH 219/909] Minor cleanup (remove unused inclusions, NULL =>
 nullptr)

PiperOrigin-RevId: 171672655
---
 tensorflow/contrib/boosted_trees/kernels/model_ops.cc | 1 -
 tensorflow/core/kernels/cuda_solvers.cc               | 9 +++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/tensorflow/contrib/boosted_trees/kernels/model_ops.cc b/tensorflow/contrib/boosted_trees/kernels/model_ops.cc
index d63be3d041..4b5d5ba0de 100644
--- a/tensorflow/contrib/boosted_trees/kernels/model_ops.cc
+++ b/tensorflow/contrib/boosted_trees/kernels/model_ops.cc
@@ -15,7 +15,6 @@
 #include <string>
 
 #include "tensorflow/contrib/boosted_trees/lib/utils/tensor_utils.h"
-#include "tensorflow/contrib/boosted_trees/proto/tree_config.pb.h"
 #include "tensorflow/contrib/boosted_trees/resources/decision_tree_ensemble_resource.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/resource_mgr.h"
diff --git a/tensorflow/core/kernels/cuda_solvers.cc b/tensorflow/core/kernels/cuda_solvers.cc
index 6c12a0e218..a83671a471 100644
--- a/tensorflow/core/kernels/cuda_solvers.cc
+++ b/tensorflow/core/kernels/cuda_solvers.cc
@@ -617,10 +617,11 @@ static inline Status GesvdImpl(
   // kernel on the stream, it is not a big performance hit.
   mutex_lock lock(handle_map_mutex);
   /* Launch the solver kernel. */
-  TF_RETURN_IF_CUSOLVER_ERROR(solver(
-      cusolver_dn_handle, jobu, jobvt, m, n, CUDAComplex(A), lda, S,
-      CUDAComplex(U), ldu, CUDAComplex(VT), ldvt,
-      CUDAComplex(dev_workspace.mutable_data()), lwork, NULL, dev_lapack_info));
+  TF_RETURN_IF_CUSOLVER_ERROR(solver(cusolver_dn_handle, jobu, jobvt, m, n,
+                                     CUDAComplex(A), lda, S, CUDAComplex(U),
+                                     ldu, CUDAComplex(VT), ldvt,
+                                     CUDAComplex(dev_workspace.mutable_data()),
+                                     lwork, nullptr, dev_lapack_info));
   return Status::OK();
 }
 
-- 
GitLab


From 3bafe0a86f67dd54197c6d60bdb5053f510de7d8 Mon Sep 17 00:00:00 2001
From: Peter Hawkins <phawkins@google.com>
Date: Tue, 10 Oct 2017 08:36:23 -0700
Subject: [PATCH 220/909] Add uint32 and uint64 types to TensorFlow.

This change merely creates the types, but does not register kernels that act on uint32/uint64 values. It also does not alter most op registration lists to include uint32/uint64 values. If desirable, that can be done in a subsequent change, although binary size will likely prove problematic if adding more kernels.

The intent of the change is so XLA-compiled code can make use uint32/uint64 types. Since XLA does not use traditional TensorFlow kernels, using uint32/uint64 operators from XLA will require only uint32/uint64 op registrations, but will require few new kernel registrations.

PiperOrigin-RevId: 171681867
---
 tensorflow/c/c_api.h                          |  2 ++
 tensorflow/compiler/tf2xla/type_util.cc       |  6 ++++
 tensorflow/compiler/tf2xla/xla_op_registry.h  | 13 ++++---
 .../python/learn/learn_io/data_feeder_test.py | 20 +++++------
 .../core/framework/op_def_builder_test.cc     | 13 +++----
 tensorflow/core/framework/register_types.h    |  6 ++++
 tensorflow/core/framework/tensor.cc           | 23 +++++++++++++
 tensorflow/core/framework/tensor.proto        |  6 ++++
 tensorflow/core/framework/types.cc            | 34 +++++++++++++++----
 tensorflow/core/framework/types.h             |  2 ++
 tensorflow/core/framework/types.proto         | 16 ++++++---
 tensorflow/go/tensor.go                       |  4 +++
 tensorflow/python/__init__.py                 |  2 ++
 tensorflow/python/framework/dtypes.py         | 20 +++++++++++
 tensorflow/python/framework/dtypes_test.py    |  3 ++
 tensorflow/python/framework/function.py       |  2 ++
 tensorflow/python/lib/core/ndarray_tensor.cc  |  6 ++++
 .../python/lib/core/ndarray_tensor_bridge.cc  |  6 ++++
 tensorflow/tools/api/golden/tensorflow.pbtxt  |  8 +++++
 19 files changed, 159 insertions(+), 33 deletions(-)

diff --git a/tensorflow/c/c_api.h b/tensorflow/c/c_api.h
index db94828e1a..68a758498d 100644
--- a/tensorflow/c/c_api.h
+++ b/tensorflow/c/c_api.h
@@ -118,6 +118,8 @@ typedef enum TF_DataType {
   TF_HALF = 19,
   TF_RESOURCE = 20,
   TF_VARIANT = 21,
+  TF_UINT32 = 22,
+  TF_UINT64 = 23,
 } TF_DataType;
 
 // TF_DataTypeSize returns the sizeof() for the underlying type corresponding
diff --git a/tensorflow/compiler/tf2xla/type_util.cc b/tensorflow/compiler/tf2xla/type_util.cc
index b54848f342..c698488776 100644
--- a/tensorflow/compiler/tf2xla/type_util.cc
+++ b/tensorflow/compiler/tf2xla/type_util.cc
@@ -43,6 +43,12 @@ Status DataTypeToPrimitiveType(DataType data_type, xla::PrimitiveType* type) {
     case tensorflow::DT_UINT16:
       *type = xla::U16;
       return Status::OK();
+    case tensorflow::DT_UINT32:
+      *type = xla::U32;
+      return Status::OK();
+    case tensorflow::DT_UINT64:
+      *type = xla::U64;
+      return Status::OK();
     case tensorflow::DT_HALF:
       *type = xla::F16;
       return Status::OK();
diff --git a/tensorflow/compiler/tf2xla/xla_op_registry.h b/tensorflow/compiler/tf2xla/xla_op_registry.h
index 1a8d03757a..2144868646 100644
--- a/tensorflow/compiler/tf2xla/xla_op_registry.h
+++ b/tensorflow/compiler/tf2xla/xla_op_registry.h
@@ -45,17 +45,16 @@ extern const char* const DEVICE_GPU_XLA_JIT;  // "GPU_XLA_JIT"
 extern const char* const DEVICE_XLA_CPU;
 extern const char* const DEVICE_XLA_GPU;
 
-constexpr std::array<DataType, 2> kIntTypes = {{DT_INT32, DT_INT64}};
 constexpr std::array<DataType, 3> kFloatTypes = {
     {DT_HALF, DT_FLOAT, DT_DOUBLE}};
-constexpr std::array<DataType, 5> kNumericTypes = {
-    {DT_INT32, DT_INT64, DT_HALF, DT_FLOAT, DT_DOUBLE}};
+constexpr std::array<DataType, 7> kNumericTypes = {
+    {DT_UINT32, DT_UINT64, DT_INT32, DT_INT64, DT_HALF, DT_FLOAT, DT_DOUBLE}};
 
-constexpr std::array<DataType, 5> kCpuAllTypes = {
-    {DT_INT32, DT_INT64, DT_FLOAT, DT_DOUBLE, DT_BOOL}};
+constexpr std::array<DataType, 7> kCpuAllTypes = {
+    {DT_UINT32, DT_UINT64, DT_INT32, DT_INT64, DT_FLOAT, DT_DOUBLE, DT_BOOL}};
 
-constexpr std::array<DataType, 5> kGpuAllTypes = {
-    {DT_INT32, DT_INT64, DT_FLOAT, DT_DOUBLE, DT_BOOL}};
+constexpr std::array<DataType, 7> kGpuAllTypes = {
+    {DT_UINT32, DT_UINT64, DT_INT32, DT_INT64, DT_FLOAT, DT_DOUBLE, DT_BOOL}};
 
 // Class that manages registrations of operators and devices for the XLA JIT.
 // Not thread-safe.
diff --git a/tensorflow/contrib/learn/python/learn/learn_io/data_feeder_test.py b/tensorflow/contrib/learn/python/learn/learn_io/data_feeder_test.py
index eaf6ae4ed7..82848be7df 100644
--- a/tensorflow/contrib/learn/python/learn/learn_io/data_feeder_test.py
+++ b/tensorflow/contrib/learn/python/learn/learn_io/data_feeder_test.py
@@ -42,16 +42,6 @@ class DataFeederTest(test.TestCase):
     with self.assertRaisesRegexp(TypeError, 'annot convert'):
       data_feeder.DataFeeder(input_data, None, n_classes=0, batch_size=1)
 
-  def test_input_uint32(self):
-    data = np.matrix([[1, 2], [3, 4]], dtype=np.uint32)
-    self._assert_raises(data)
-    self._assert_raises(self._wrap_dict(data))
-
-  def test_input_uint64(self):
-    data = np.matrix([[1, 2], [3, 4]], dtype=np.uint64)
-    self._assert_raises(data)
-    self._assert_raises(self._wrap_dict(data))
-
   def _assert_dtype(self, expected_np_dtype, expected_tf_dtype, input_data):
     feeder = data_feeder.DataFeeder(input_data, None, n_classes=0, batch_size=1)
     if isinstance(input_data, dict):
@@ -87,6 +77,16 @@ class DataFeederTest(test.TestCase):
     self._assert_dtype(np.int64, dtypes.int64, data)
     self._assert_dtype(np.int64, dtypes.int64, self._wrap_dict(data))
 
+  def test_input_uint32(self):
+    data = np.matrix([[1, 2], [3, 4]], dtype=np.uint32)
+    self._assert_dtype(np.uint32, dtypes.uint32, data)
+    self._assert_dtype(np.uint32, dtypes.uint32, self._wrap_dict(data))
+
+  def test_input_uint64(self):
+    data = np.matrix([[1, 2], [3, 4]], dtype=np.uint64)
+    self._assert_dtype(np.uint64, dtypes.uint64, data)
+    self._assert_dtype(np.uint64, dtypes.uint64, self._wrap_dict(data))
+
   def test_input_uint8(self):
     data = np.matrix([[1, 2], [3, 4]], dtype=np.uint8)
     self._assert_dtype(np.uint8, dtypes.uint8, data)
diff --git a/tensorflow/core/framework/op_def_builder_test.cc b/tensorflow/core/framework/op_def_builder_test.cc
index efedb221e7..c1511ebe34 100644
--- a/tensorflow/core/framework/op_def_builder_test.cc
+++ b/tensorflow/core/framework/op_def_builder_test.cc
@@ -124,21 +124,22 @@ TEST_F(OpDefBuilderTest, AttrWithRestrictions) {
       "attr: { name: 'a' type: 'type' allowed_values { list { type: "
       "[DT_HALF, DT_FLOAT, DT_DOUBLE, DT_INT64, DT_INT32, DT_UINT8, DT_INT16, "
       "DT_UINT16, DT_INT8, DT_COMPLEX64, DT_COMPLEX128, DT_QINT8, DT_QUINT8, "
-      "DT_QINT32] } } }");
+      "DT_QINT32, DT_UINT32, DT_UINT64] } } }");
   ExpectSuccess(
       b().Attr("a:{numbertype, variant}"),
       "attr: { name: 'a' type: 'type' allowed_values { list { type: "
       "[DT_HALF, DT_FLOAT, DT_DOUBLE, DT_INT64, DT_INT32, DT_UINT8, DT_INT16, "
       "DT_UINT16, DT_INT8, DT_COMPLEX64, DT_COMPLEX128, DT_QINT8, DT_QUINT8, "
-      "DT_QINT32, DT_VARIANT] } } }");
+      "DT_QINT32, DT_UINT32, DT_UINT64, DT_VARIANT] } } }");
   ExpectSuccess(b().Attr("a:realnumbertype"),
                 "attr: { name: 'a' type: 'type' allowed_values { list { type: "
                 "[DT_HALF, DT_FLOAT, DT_DOUBLE, DT_INT64, DT_INT32, DT_UINT8, "
-                "DT_INT16, DT_UINT16, DT_INT8] } } }");
+                "DT_INT16, DT_UINT16, DT_INT8, DT_UINT32, DT_UINT64] } } }");
   ExpectSuccess(b().Attr("a:{realnumbertype,  variant , string, }"),
                 "attr: { name: 'a' type: 'type' allowed_values { list { type: "
                 "[DT_HALF, DT_FLOAT, DT_DOUBLE, DT_INT64, DT_INT32, DT_UINT8, "
-                "DT_INT16, DT_UINT16, DT_INT8, DT_VARIANT, DT_STRING] } } }");
+                "DT_INT16, DT_UINT16, DT_INT8, DT_UINT32, DT_UINT64, "
+                "DT_VARIANT, DT_STRING] } } }");
   ExpectSuccess(b().Attr("a:quantizedtype"),
                 "attr: { name: 'a' type: 'type' allowed_values { list { type: "
                 "[DT_QINT8, DT_QUINT8, DT_QINT32, DT_QINT16, DT_QUINT16]} } }");
@@ -215,12 +216,12 @@ TEST_F(OpDefBuilderTest, AttrListOfRestricted) {
       b().Attr("a:list(realnumbertype)"),
       "attr: { name: 'a' type: 'list(type)' allowed_values { list { type: "
       "[DT_FLOAT, DT_DOUBLE, DT_INT64, DT_INT32, DT_UINT8, DT_INT16, "
-      "DT_UINT16, DT_INT8, DT_HALF] } } }");
+      "DT_UINT16, DT_INT8, DT_HALF, DT_UINT32, DT_UINT64] } } }");
   ExpectSuccess(
       b().Attr("a:list({realnumbertype, variant})"),
       "attr: { name: 'a' type: 'list(type)' allowed_values { list { type: "
       "[DT_FLOAT, DT_DOUBLE, DT_INT64, DT_INT32, DT_UINT8, DT_INT16, "
-      "DT_UINT16, DT_INT8, DT_HALF, DT_VARIANT] } } }");
+      "DT_UINT16, DT_INT8, DT_HALF, DT_UINT32, DT_UINT64, DT_VARIANT] } } }");
   ExpectSuccess(
       b().Attr("a:list(quantizedtype)"),
       "attr: { name: 'a' type: 'list(type)' allowed_values { list { type: "
diff --git a/tensorflow/core/framework/register_types.h b/tensorflow/core/framework/register_types.h
index 030c00cb8e..3f9c307d03 100644
--- a/tensorflow/core/framework/register_types.h
+++ b/tensorflow/core/framework/register_types.h
@@ -60,6 +60,7 @@ limitations under the License.
 #define TF_CALL_float(m) m(float)
 #define TF_CALL_double(m) m(double)
 #define TF_CALL_int32(m) m(::tensorflow::int32)
+#define TF_CALL_uint32(m) m(::tensorflow::uint32)
 #define TF_CALL_uint8(m) m(::tensorflow::uint8)
 #define TF_CALL_int16(m) m(::tensorflow::int16)
 
@@ -68,6 +69,7 @@ limitations under the License.
 #define TF_CALL_resource(m) m(::tensorflow::ResourceHandle)
 #define TF_CALL_complex64(m) m(::tensorflow::complex64)
 #define TF_CALL_int64(m) m(::tensorflow::int64)
+#define TF_CALL_uint64(m) m(::tensorflow::uint64)
 #define TF_CALL_bool(m) m(bool)
 
 #define TF_CALL_qint8(m) m(::tensorflow::qint8)
@@ -87,6 +89,7 @@ limitations under the License.
 #define TF_CALL_float(m) m(float)
 #define TF_CALL_double(m)
 #define TF_CALL_int32(m) m(::tensorflow::int32)
+#define TF_CALL_uint32(m)
 #define TF_CALL_uint8(m)
 #define TF_CALL_int16(m)
 
@@ -95,6 +98,7 @@ limitations under the License.
 #define TF_CALL_resource(m)
 #define TF_CALL_complex64(m)
 #define TF_CALL_int64(m) m(::tensorflow::int64)
+#define TF_CALL_uint64(m)
 #define TF_CALL_bool(m) m(bool)
 
 #define TF_CALL_qint8(m) m(::tensorflow::qint8)
@@ -114,6 +118,7 @@ limitations under the License.
 #define TF_CALL_float(m) m(float)
 #define TF_CALL_double(m)
 #define TF_CALL_int32(m) m(::tensorflow::int32)
+#define TF_CALL_uint32(m)
 #define TF_CALL_uint8(m)
 #define TF_CALL_int16(m)
 
@@ -122,6 +127,7 @@ limitations under the License.
 #define TF_CALL_resource(m)
 #define TF_CALL_complex64(m)
 #define TF_CALL_int64(m)
+#define TF_CALL_uint64(m)
 #define TF_CALL_bool(m) m(bool)
 
 #define TF_CALL_qint8(m)
diff --git a/tensorflow/core/framework/tensor.cc b/tensorflow/core/framework/tensor.cc
index a5b5ef0acc..24b7b08ebc 100644
--- a/tensorflow/core/framework/tensor.cc
+++ b/tensorflow/core/framework/tensor.cc
@@ -288,6 +288,7 @@ PROTO_TRAITS(double, double, double);
 PROTO_TRAITS(int32, int32, int);
 PROTO_TRAITS(uint8, int32, int);
 PROTO_TRAITS(uint16, int32, int);
+PROTO_TRAITS(uint32, uint32, uint32);
 PROTO_TRAITS(int16, int32, int);
 PROTO_TRAITS(int8, int32, int);
 PROTO_TRAITS(bool, bool, bool);
@@ -312,6 +313,20 @@ struct ProtoHelper<int64> {
   }
 };
 
+template <>
+struct ProtoHelper<uint64> {
+  static const uint64* Begin(const TensorProto& proto) {
+    return reinterpret_cast<const uint64*>(proto.uint64_val().begin());
+  }
+  static size_t NumElements(const TensorProto& proto) {
+    return proto.uint64_val().size();
+  }
+  static void Fill(const uint64* data, size_t n, TensorProto* proto) {
+    protobuf::RepeatedField<protobuf_uint64> copy(data, data + n);
+    proto->mutable_uint64_val()->Swap(&copy);
+  }
+};
+
 template <>
 struct ProtoHelper<ResourceHandle> {
   static protobuf::RepeatedPtrField<ResourceHandleProto>::const_iterator Begin(
@@ -649,6 +664,8 @@ bool Tensor::RefCountIsOne() const {
     CASE(int32, SINGLE_ARG(STMTS))                             \
     CASE(uint8, SINGLE_ARG(STMTS))                             \
     CASE(uint16, SINGLE_ARG(STMTS))                            \
+    CASE(uint32, SINGLE_ARG(STMTS))                            \
+    CASE(uint64, SINGLE_ARG(STMTS))                            \
     CASE(int16, SINGLE_ARG(STMTS))                             \
     CASE(int8, SINGLE_ARG(STMTS))                              \
     CASE(string, SINGLE_ARG(STMTS))                            \
@@ -925,6 +942,9 @@ string Tensor::SummarizeValue(int64 max_entries) const {
     case DT_DOUBLE:
       return SummarizeArray<double>(limit, num_elts, shape_, data);
       break;
+    case DT_UINT32:
+      return SummarizeArray<uint32>(limit, num_elts, shape_, data);
+      break;
     case DT_INT32:
       return SummarizeArray<int32>(limit, num_elts, shape_, data);
       break;
@@ -944,6 +964,9 @@ string Tensor::SummarizeValue(int64 max_entries) const {
     case DT_QINT8:
       return SummarizeArray<int8>(limit, num_elts, shape_, data);
       break;
+    case DT_UINT64:
+      return SummarizeArray<uint64>(limit, num_elts, shape_, data);
+      break;
     case DT_INT64:
       return SummarizeArray<int64>(limit, num_elts, shape_, data);
       break;
diff --git a/tensorflow/core/framework/tensor.proto b/tensorflow/core/framework/tensor.proto
index 7e4af7a645..6dab325969 100644
--- a/tensorflow/core/framework/tensor.proto
+++ b/tensorflow/core/framework/tensor.proto
@@ -75,6 +75,12 @@ message TensorProto {
 
   // DT_VARIANT
   repeated VariantTensorDataProto variant_val = 15;
+
+  // DT_UINT32
+  repeated uint32 uint32_val = 16 [packed = true];
+
+  // DT_UINT64
+  repeated uint64 uint64_val = 17 [packed = true];
 };
 
 // Protocol buffer representing the serialization format of DT_VARIANT tensors.
diff --git a/tensorflow/core/framework/types.cc b/tensorflow/core/framework/types.cc
index 1a5fd10f52..cc86871cae 100644
--- a/tensorflow/core/framework/types.cc
+++ b/tensorflow/core/framework/types.cc
@@ -61,6 +61,8 @@ string DataTypeString(DataType dtype) {
       return "double";
     case DT_INT32:
       return "int32";
+    case DT_UINT32:
+      return "uint32";
     case DT_UINT8:
       return "uint8";
     case DT_UINT16:
@@ -77,6 +79,8 @@ string DataTypeString(DataType dtype) {
       return "complex128";
     case DT_INT64:
       return "int64";
+    case DT_UINT64:
+      return "uint64";
     case DT_BOOL:
       return "bool";
     case DT_QINT8:
@@ -124,6 +128,9 @@ bool DataTypeFromString(StringPiece sp, DataType* dt) {
   } else if (sp == "int32") {
     *dt = DT_INT32;
     return true;
+  } else if (sp == "uint32") {
+    *dt = DT_UINT32;
+    return true;
   } else if (sp == "uint8") {
     *dt = DT_UINT8;
     return true;
@@ -148,6 +155,9 @@ bool DataTypeFromString(StringPiece sp, DataType* dt) {
   } else if (sp == "int64") {
     *dt = DT_INT64;
     return true;
+  } else if (sp == "uint64") {
+    *dt = DT_UINT64;
+    return true;
   } else if (sp == "bool") {
     *dt = DT_BOOL;
     return true;
@@ -199,14 +209,15 @@ DataTypeVector AllTypes() {
   return {DT_FLOAT,   DT_DOUBLE, DT_INT32,  DT_UINT8,     DT_INT16,
           DT_UINT16,  DT_INT8,   DT_STRING, DT_COMPLEX64, DT_COMPLEX128,
           DT_INT64,   DT_BOOL,   DT_QINT8,  DT_QUINT8,    DT_QINT16,
-          DT_QUINT16, DT_QINT32, DT_HALF,   DT_RESOURCE,  DT_VARIANT};
+          DT_QUINT16, DT_QINT32, DT_HALF,   DT_RESOURCE,  DT_VARIANT,
+          DT_UINT32,  DT_UINT64};
 }
 
 #if !defined(IS_MOBILE_PLATFORM) || defined(SUPPORT_SELECTIVE_REGISTRATION)
 
 DataTypeVector RealNumberTypes() {
-  return {DT_FLOAT, DT_DOUBLE, DT_INT32,  DT_INT64, DT_UINT8,
-          DT_INT16, DT_INT8,   DT_UINT16, DT_HALF};
+  return {DT_FLOAT, DT_DOUBLE, DT_INT32, DT_INT64,  DT_UINT8, DT_INT16,
+          DT_INT8,  DT_UINT16, DT_HALF,  DT_UINT32, DT_UINT64};
 }
 
 DataTypeVector QuantizedTypes() {
@@ -220,9 +231,10 @@ DataTypeVector RealAndQuantizedTypes() {
 }
 
 DataTypeVector NumberTypes() {
-  return {DT_FLOAT,  DT_DOUBLE, DT_INT64,  DT_INT32,     DT_UINT8,
-          DT_UINT16, DT_INT16,  DT_INT8,   DT_COMPLEX64, DT_COMPLEX128,
-          DT_QINT8,  DT_QUINT8, DT_QINT32, DT_HALF};
+  return {DT_FLOAT,     DT_DOUBLE,     DT_INT64,  DT_INT32,
+          DT_UINT8,     DT_UINT16,     DT_INT16,  DT_INT8,
+          DT_COMPLEX64, DT_COMPLEX128, DT_QINT8,  DT_QUINT8,
+          DT_QINT32,    DT_HALF,       DT_UINT32, DT_UINT64};
 }
 
 #elif defined(__ANDROID_TYPES_FULL__)
@@ -271,6 +283,7 @@ bool DataTypeCanUseMemcpy(DataType dt) {
     case DT_FLOAT:
     case DT_DOUBLE:
     case DT_INT32:
+    case DT_UINT32:
     case DT_UINT8:
     case DT_UINT16:
     case DT_INT16:
@@ -278,6 +291,7 @@ bool DataTypeCanUseMemcpy(DataType dt) {
     case DT_COMPLEX64:
     case DT_COMPLEX128:
     case DT_INT64:
+    case DT_UINT64:
     case DT_BOOL:
     case DT_QINT8:
     case DT_QUINT8:
@@ -312,7 +326,9 @@ bool DataTypeIsInteger(DataType dt) {
     case DT_INT16:
     case DT_UINT16:
     case DT_INT32:
+    case DT_UINT32:
     case DT_INT64:
+    case DT_UINT64:
       return true;
     default:
       return false;
@@ -331,6 +347,12 @@ int DataTypeSize(DataType dt) {
     // bitcast.
     TF_CALL_qint16(CASE);
     TF_CALL_quint16(CASE);
+
+    // uint32 and uint64 aren't included in TF_CALL_POD_TYPES because we
+    // don't want to define kernels for them at this stage to avoid binary
+    // bloat.
+    TF_CALL_uint32(CASE);
+    TF_CALL_uint64(CASE);
     default:
       return 0;
   }
diff --git a/tensorflow/core/framework/types.h b/tensorflow/core/framework/types.h
index 3b4362bcc9..300a57e948 100644
--- a/tensorflow/core/framework/types.h
+++ b/tensorflow/core/framework/types.h
@@ -187,6 +187,7 @@ struct EnumToDataType {};  // Specializations below
 MATCH_TYPE_AND_ENUM(float, DT_FLOAT);
 MATCH_TYPE_AND_ENUM(double, DT_DOUBLE);
 MATCH_TYPE_AND_ENUM(int32, DT_INT32);
+MATCH_TYPE_AND_ENUM(uint32, DT_UINT32);
 MATCH_TYPE_AND_ENUM(uint16, DT_UINT16);
 MATCH_TYPE_AND_ENUM(uint8, DT_UINT8);
 MATCH_TYPE_AND_ENUM(int16, DT_INT16);
@@ -195,6 +196,7 @@ MATCH_TYPE_AND_ENUM(string, DT_STRING);
 MATCH_TYPE_AND_ENUM(complex64, DT_COMPLEX64);
 MATCH_TYPE_AND_ENUM(complex128, DT_COMPLEX128);
 MATCH_TYPE_AND_ENUM(int64, DT_INT64);
+MATCH_TYPE_AND_ENUM(uint64, DT_UINT64);
 MATCH_TYPE_AND_ENUM(bool, DT_BOOL);
 MATCH_TYPE_AND_ENUM(qint8, DT_QINT8);
 MATCH_TYPE_AND_ENUM(quint8, DT_QUINT8);
diff --git a/tensorflow/core/framework/types.proto b/tensorflow/core/framework/types.proto
index 1beb2a1aa2..e003fd0010 100644
--- a/tensorflow/core/framework/types.proto
+++ b/tensorflow/core/framework/types.proto
@@ -35,9 +35,8 @@ enum DataType {
   DT_HALF = 19;
   DT_RESOURCE = 20;
   DT_VARIANT = 21;  // Arbitrary C++ data types
-
-  // TODO(josh11b): DT_GENERIC_PROTO = ??;
-  // TODO(jeff,josh11b): DT_UINT64?  DT_UINT32?
+  DT_UINT32 = 22;
+  DT_UINT64 = 23;
 
   // Do not use!  These are only for parameters.  Every enum above
   // should have a corresponding value below (verified by types_test).
@@ -62,5 +61,14 @@ enum DataType {
   DT_HALF_REF = 119;
   DT_RESOURCE_REF = 120;
   DT_VARIANT_REF = 121;
+  DT_UINT32_REF = 122;
+  DT_UINT64_REF = 123;
 }
-// LINT.ThenChange(https://www.tensorflow.org/code/tensorflow/c/c_api.h,https://www.tensorflow.org/code/tensorflow/go/tensor.go)
+// LINT.ThenChange(
+//    https://www.tensorflow.org/code/tensorflow/c/c_api.h,
+//    https://www.tensorflow.org/code/tensorflow/go/tensor.go,
+//    https://www.tensorflow.org/code/tensorflow/core/framework/tensor.cc,
+//    https://www.tensorflow.org/code/tensorflow/core/framework/types.h,
+//    https://www.tensorflow.org/code/tensorflow/core/framework/types.cc,
+//    https://www.tensorflow.org/code/tensorflow/python/framework/dtypes.py,
+//    https://www.tensorflow.org/code/tensorflow/python/framework/function.py)
diff --git a/tensorflow/go/tensor.go b/tensorflow/go/tensor.go
index e8fa21a62b..36a74c0081 100644
--- a/tensorflow/go/tensor.go
+++ b/tensorflow/go/tensor.go
@@ -39,6 +39,7 @@ const (
 	Float      DataType = C.TF_FLOAT
 	Double     DataType = C.TF_DOUBLE
 	Int32      DataType = C.TF_INT32
+	Uint32     DataType = C.TF_UINT32
 	Uint8      DataType = C.TF_UINT8
 	Int16      DataType = C.TF_INT16
 	Int8       DataType = C.TF_INT8
@@ -46,6 +47,7 @@ const (
 	Complex64  DataType = C.TF_COMPLEX64
 	Complex    DataType = C.TF_COMPLEX
 	Int64      DataType = C.TF_INT64
+	Uint64     DataType = C.TF_UINT64
 	Bool       DataType = C.TF_BOOL
 	Qint8      DataType = C.TF_QINT8
 	Quint8     DataType = C.TF_QUINT8
@@ -217,12 +219,14 @@ var types = []struct {
 	{reflect.TypeOf(float32(0)), C.TF_FLOAT},
 	{reflect.TypeOf(float64(0)), C.TF_DOUBLE},
 	{reflect.TypeOf(int32(0)), C.TF_INT32},
+	{reflect.TypeOf(uint32(0)), C.TF_UINT32},
 	{reflect.TypeOf(uint8(0)), C.TF_UINT8},
 	{reflect.TypeOf(int16(0)), C.TF_INT16},
 	{reflect.TypeOf(int8(0)), C.TF_INT8},
 	{reflect.TypeOf(""), C.TF_STRING},
 	{reflect.TypeOf(complex(float32(0), float32(0))), C.TF_COMPLEX64},
 	{reflect.TypeOf(int64(0)), C.TF_INT64},
+	{reflect.TypeOf(uint64(0)), C.TF_UINT64},
 	{reflect.TypeOf(false), C.TF_BOOL},
 	{reflect.TypeOf(uint16(0)), C.TF_UINT16},
 	{reflect.TypeOf(complex(float64(0), float64(0))), C.TF_COMPLEX128},
diff --git a/tensorflow/python/__init__.py b/tensorflow/python/__init__.py
index f3bdea92dd..f21f1f822c 100644
--- a/tensorflow/python/__init__.py
+++ b/tensorflow/python/__init__.py
@@ -213,6 +213,8 @@ _allowed_symbols.extend([
     'quint16',
     'quint8',
     'string',
+    'uint64',
+    'uint32',
     'uint16',
     'uint8',
     'resource',
diff --git a/tensorflow/python/framework/dtypes.py b/tensorflow/python/framework/dtypes.py
index 43535a593e..db124ab12a 100644
--- a/tensorflow/python/framework/dtypes.py
+++ b/tensorflow/python/framework/dtypes.py
@@ -37,6 +37,8 @@ class DType(object):
   * `tf.int8`: 8-bit signed integer.
   * `tf.uint8`: 8-bit unsigned integer.
   * `tf.uint16`: 16-bit unsigned integer.
+  * `tf.uint32`: 32-bit unsigned integer.
+  * `tf.uint64`: 64-bit unsigned integer.
   * `tf.int16`: 16-bit signed integer.
   * `tf.int32`: 32-bit signed integer.
   * `tf.int64`: 64-bit signed integer.
@@ -318,6 +320,8 @@ double = float64
 int32 = DType(types_pb2.DT_INT32)
 uint8 = DType(types_pb2.DT_UINT8)
 uint16 = DType(types_pb2.DT_UINT16)
+uint32 = DType(types_pb2.DT_UINT32)
+uint64 = DType(types_pb2.DT_UINT64)
 int16 = DType(types_pb2.DT_INT16)
 int8 = DType(types_pb2.DT_INT8)
 string = DType(types_pb2.DT_STRING)
@@ -339,6 +343,7 @@ float32_ref = DType(types_pb2.DT_FLOAT_REF)
 float64_ref = DType(types_pb2.DT_DOUBLE_REF)
 double_ref = float64_ref
 int32_ref = DType(types_pb2.DT_INT32_REF)
+uint32_ref = DType(types_pb2.DT_UINT32_REF)
 uint8_ref = DType(types_pb2.DT_UINT8_REF)
 uint16_ref = DType(types_pb2.DT_UINT16_REF)
 int16_ref = DType(types_pb2.DT_INT16_REF)
@@ -347,6 +352,7 @@ string_ref = DType(types_pb2.DT_STRING_REF)
 complex64_ref = DType(types_pb2.DT_COMPLEX64_REF)
 complex128_ref = DType(types_pb2.DT_COMPLEX128_REF)
 int64_ref = DType(types_pb2.DT_INT64_REF)
+uint64_ref = DType(types_pb2.DT_UINT64_REF)
 bool_ref = DType(types_pb2.DT_BOOL_REF)
 qint8_ref = DType(types_pb2.DT_QINT8_REF)
 quint8_ref = DType(types_pb2.DT_QUINT8_REF)
@@ -365,6 +371,8 @@ _INTERN_TABLE = {
     types_pb2.DT_INT32: int32,
     types_pb2.DT_UINT8: uint8,
     types_pb2.DT_UINT16: uint16,
+    types_pb2.DT_UINT32: uint32,
+    types_pb2.DT_UINT64: uint64,
     types_pb2.DT_INT16: int16,
     types_pb2.DT_INT8: int8,
     types_pb2.DT_STRING: string,
@@ -384,6 +392,7 @@ _INTERN_TABLE = {
     types_pb2.DT_FLOAT_REF: float32_ref,
     types_pb2.DT_DOUBLE_REF: float64_ref,
     types_pb2.DT_INT32_REF: int32_ref,
+    types_pb2.DT_UINT32_REF: uint32_ref,
     types_pb2.DT_UINT8_REF: uint8_ref,
     types_pb2.DT_UINT16_REF: uint16_ref,
     types_pb2.DT_INT16_REF: int16_ref,
@@ -392,6 +401,7 @@ _INTERN_TABLE = {
     types_pb2.DT_COMPLEX64_REF: complex64_ref,
     types_pb2.DT_COMPLEX128_REF: complex128_ref,
     types_pb2.DT_INT64_REF: int64_ref,
+    types_pb2.DT_UINT64_REF: uint64_ref,
     types_pb2.DT_BOOL_REF: bool_ref,
     types_pb2.DT_QINT8_REF: qint8_ref,
     types_pb2.DT_QUINT8_REF: quint8_ref,
@@ -412,6 +422,8 @@ _TYPE_TO_STRING = {
     types_pb2.DT_INT32: "int32",
     types_pb2.DT_UINT8: "uint8",
     types_pb2.DT_UINT16: "uint16",
+    types_pb2.DT_UINT32: "uint32",
+    types_pb2.DT_UINT64: "uint64",
     types_pb2.DT_INT16: "int16",
     types_pb2.DT_INT8: "int8",
     types_pb2.DT_STRING: "string",
@@ -431,6 +443,7 @@ _TYPE_TO_STRING = {
     types_pb2.DT_FLOAT_REF: "float32_ref",
     types_pb2.DT_DOUBLE_REF: "float64_ref",
     types_pb2.DT_INT32_REF: "int32_ref",
+    types_pb2.DT_UINT32_REF: "uint32_ref",
     types_pb2.DT_UINT8_REF: "uint8_ref",
     types_pb2.DT_UINT16_REF: "uint16_ref",
     types_pb2.DT_INT16_REF: "int16_ref",
@@ -439,6 +452,7 @@ _TYPE_TO_STRING = {
     types_pb2.DT_COMPLEX64_REF: "complex64_ref",
     types_pb2.DT_COMPLEX128_REF: "complex128_ref",
     types_pb2.DT_INT64_REF: "int64_ref",
+    types_pb2.DT_UINT64_REF: "uint64_ref",
     types_pb2.DT_BOOL_REF: "bool_ref",
     types_pb2.DT_QINT8_REF: "qint8_ref",
     types_pb2.DT_QUINT8_REF: "quint8_ref",
@@ -484,6 +498,8 @@ _NP_TO_TF = frozenset([
     (np.int64, int64),
     (np.uint8, uint8),
     (np.uint16, uint16),
+    (np.uint32, uint32),
+    (np.uint64, uint64),
     (np.int16, int16),
     (np.int8, int8),
     (np.complex64, complex64),
@@ -504,6 +520,8 @@ _TF_TO_NP = {
     types_pb2.DT_INT32: np.int32,
     types_pb2.DT_UINT8: np.uint8,
     types_pb2.DT_UINT16: np.uint16,
+    types_pb2.DT_UINT32: np.uint32,
+    types_pb2.DT_UINT64: np.uint64,
     types_pb2.DT_INT16: np.int16,
     types_pb2.DT_INT8: np.int8,
     # NOTE(touts): For strings we use np.object as it supports variable length
@@ -525,6 +543,7 @@ _TF_TO_NP = {
     types_pb2.DT_FLOAT_REF: np.float32,
     types_pb2.DT_DOUBLE_REF: np.float64,
     types_pb2.DT_INT32_REF: np.int32,
+    types_pb2.DT_UINT32_REF: np.uint32,
     types_pb2.DT_UINT8_REF: np.uint8,
     types_pb2.DT_UINT16_REF: np.uint16,
     types_pb2.DT_INT16_REF: np.int16,
@@ -533,6 +552,7 @@ _TF_TO_NP = {
     types_pb2.DT_COMPLEX64_REF: np.complex64,
     types_pb2.DT_COMPLEX128_REF: np.complex128,
     types_pb2.DT_INT64_REF: np.int64,
+    types_pb2.DT_UINT64_REF: np.uint64,
     types_pb2.DT_BOOL_REF: np.bool,
     types_pb2.DT_QINT8_REF: _np_qint8,
     types_pb2.DT_QUINT8_REF: _np_quint8,
diff --git a/tensorflow/python/framework/dtypes_test.py b/tensorflow/python/framework/dtypes_test.py
index 1e84f1b656..67842e14b1 100644
--- a/tensorflow/python/framework/dtypes_test.py
+++ b/tensorflow/python/framework/dtypes_test.py
@@ -268,6 +268,9 @@ class TypesTest(test_util.TensorFlowTestCase):
           self.assertEquals(dtype.min, 0)
           self.assertEquals(dtype.max, 4294967295)
       if numpy_dtype == np.uint32:
+        self.assertEquals(dtype.min, 0)
+        self.assertEquals(dtype.max, 4294967295)
+      if numpy_dtype == np.uint64:
         self.assertEquals(dtype.min, 0)
         self.assertEquals(dtype.max, 18446744073709551615)
       if numpy_dtype in (np.float16, np.float32, np.float64):
diff --git a/tensorflow/python/framework/function.py b/tensorflow/python/framework/function.py
index 7068e72009..cef3f8d4c4 100644
--- a/tensorflow/python/framework/function.py
+++ b/tensorflow/python/framework/function.py
@@ -1002,6 +1002,8 @@ _DTYPE_TO_STR = {
     dtypes.int32: "i32",
     dtypes.uint8: "i8",
     dtypes.uint16: "u16",
+    dtypes.uint32: "u32",
+    dtypes.uint64: "u64",
     dtypes.int16: "i16",
     dtypes.int8: "i8",
     dtypes.string: "s",
diff --git a/tensorflow/python/lib/core/ndarray_tensor.cc b/tensorflow/python/lib/core/ndarray_tensor.cc
index b1a5a37924..cf2c2e6eb0 100644
--- a/tensorflow/python/lib/core/ndarray_tensor.cc
+++ b/tensorflow/python/lib/core/ndarray_tensor.cc
@@ -88,6 +88,12 @@ Status PyArray_TYPE_to_TF_DataType(PyArrayObject* array,
     case NPY_UINT16:
       *out_tf_datatype = TF_UINT16;
       break;
+    case NPY_UINT32:
+      *out_tf_datatype = TF_UINT32;
+      break;
+    case NPY_UINT64:
+      *out_tf_datatype = TF_UINT64;
+      break;
     case NPY_INT8:
       *out_tf_datatype = TF_INT8;
       break;
diff --git a/tensorflow/python/lib/core/ndarray_tensor_bridge.cc b/tensorflow/python/lib/core/ndarray_tensor_bridge.cc
index f468e0b70e..82c45f5a31 100644
--- a/tensorflow/python/lib/core/ndarray_tensor_bridge.cc
+++ b/tensorflow/python/lib/core/ndarray_tensor_bridge.cc
@@ -120,6 +120,9 @@ Status TF_DataType_to_PyArray_TYPE(TF_DataType tf_datatype,
     case TF_INT32:
       *out_pyarray_type = NPY_INT32;
       break;
+    case TF_UINT32:
+      *out_pyarray_type = NPY_UINT32;
+      break;
     case TF_UINT8:
       *out_pyarray_type = NPY_UINT8;
       break;
@@ -135,6 +138,9 @@ Status TF_DataType_to_PyArray_TYPE(TF_DataType tf_datatype,
     case TF_INT64:
       *out_pyarray_type = NPY_INT64;
       break;
+    case TF_UINT64:
+      *out_pyarray_type = NPY_UINT64;
+      break;
     case TF_BOOL:
       *out_pyarray_type = NPY_BOOL;
       break;
diff --git a/tensorflow/tools/api/golden/tensorflow.pbtxt b/tensorflow/tools/api/golden/tensorflow.pbtxt
index 6e03f9e8fb..d77f8fd253 100644
--- a/tensorflow/tools/api/golden/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.pbtxt
@@ -500,6 +500,14 @@ tf_module {
     name: "uint16"
     mtype: "<class \'tensorflow.python.framework.dtypes.DType\'>"
   }
+  member {
+    name: "uint32"
+    mtype: "<class \'tensorflow.python.framework.dtypes.DType\'>"
+  }
+  member {
+    name: "uint64"
+    mtype: "<class \'tensorflow.python.framework.dtypes.DType\'>"
+  }
   member {
     name: "uint8"
     mtype: "<class \'tensorflow.python.framework.dtypes.DType\'>"
-- 
GitLab


From 8776bfdf07be8ce95b9f1f75742b7bb8c9e30e35 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 10 Oct 2017 08:51:00 -0700
Subject: [PATCH 221/909] Internal change

PiperOrigin-RevId: 171683977
---
 tensorflow/contrib/eager/python/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/contrib/eager/python/BUILD b/tensorflow/contrib/eager/python/BUILD
index 1a63c901a2..7ef163c707 100644
--- a/tensorflow/contrib/eager/python/BUILD
+++ b/tensorflow/contrib/eager/python/BUILD
@@ -9,6 +9,7 @@ py_library(
     name = "tfe",
     srcs = ["tfe.py"],
     srcs_version = "PY2AND3",
+    visibility = ["//visibility:public"],
     deps = [
         ":datasets",
         ":metrics",
-- 
GitLab


From cf3cddc2089d310360f2332ac4df2b14344f6cde Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 10 Oct 2017 09:19:09 -0700
Subject: [PATCH 222/909] Update ops-related pbtxt files.

PiperOrigin-RevId: 171688013
---
 .../core/ops/compat/ops_history.v1.pbtxt      | 15800 ++++++++++++----
 tensorflow/core/ops/ops.pbtxt                 |   252 +
 2 files changed, 11862 insertions(+), 4190 deletions(-)

diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index a449fc1452..1eafbe138c 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -77,6 +77,46 @@ op {
     }
   }
 }
+op {
+  name: "AccumulatorApplyGradient"
+  input_arg {
+    name: "handle"
+    type: DT_STRING
+    is_ref: true
+  }
+  input_arg {
+    name: "local_step"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "gradient"
+    type_attr: "dtype"
+  }
+  attr {
+    name: "dtype"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+}
 op {
   name: "AccumulatorNumAccumulated"
   input_arg {
@@ -139,6 +179,46 @@ op {
     }
   }
 }
+op {
+  name: "AccumulatorTakeGradient"
+  input_arg {
+    name: "handle"
+    type: DT_STRING
+    is_ref: true
+  }
+  input_arg {
+    name: "num_required"
+    type: DT_INT32
+  }
+  output_arg {
+    name: "average"
+    type_attr: "dtype"
+  }
+  attr {
+    name: "dtype"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+}
 op {
   name: "Acos"
   input_arg {
@@ -346,6 +426,51 @@ op {
   is_aggregate: true
   is_commutative: true
 }
+op {
+  name: "AddN"
+  input_arg {
+    name: "inputs"
+    type_attr: "T"
+    number_attr: "N"
+  }
+  output_arg {
+    name: "sum"
+    type_attr: "T"
+  }
+  attr {
+    name: "N"
+    type: "int"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+        type: DT_VARIANT
+      }
+    }
+  }
+  is_aggregate: true
+  is_commutative: true
+}
 op {
   name: "AddSparseToTensorsMap"
   input_arg {
@@ -745,7 +870,7 @@ op {
   }
 }
 op {
-  name: "ApplyAdagrad"
+  name: "ApplyAdadelta"
   input_arg {
     name: "var"
     type_attr: "T"
@@ -756,10 +881,23 @@ op {
     type_attr: "T"
     is_ref: true
   }
+  input_arg {
+    name: "accum_update"
+    type_attr: "T"
+    is_ref: true
+  }
   input_arg {
     name: "lr"
     type_attr: "T"
   }
+  input_arg {
+    name: "rho"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "epsilon"
+    type_attr: "T"
+  }
   input_arg {
     name: "grad"
     type_attr: "T"
@@ -788,6 +926,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -800,42 +940,25 @@ op {
   }
 }
 op {
-  name: "ApplyAdagradDA"
+  name: "ApplyAdagrad"
   input_arg {
     name: "var"
     type_attr: "T"
     is_ref: true
   }
   input_arg {
-    name: "gradient_accumulator"
-    type_attr: "T"
-    is_ref: true
-  }
-  input_arg {
-    name: "gradient_squared_accumulator"
+    name: "accum"
     type_attr: "T"
     is_ref: true
   }
-  input_arg {
-    name: "grad"
-    type_attr: "T"
-  }
   input_arg {
     name: "lr"
     type_attr: "T"
   }
   input_arg {
-    name: "l1"
-    type_attr: "T"
-  }
-  input_arg {
-    name: "l2"
+    name: "grad"
     type_attr: "T"
   }
-  input_arg {
-    name: "global_step"
-    type: DT_INT64
-  }
   output_arg {
     name: "out"
     type_attr: "T"
@@ -872,46 +995,21 @@ op {
   }
 }
 op {
-  name: "ApplyAdam"
+  name: "ApplyAdagrad"
   input_arg {
     name: "var"
     type_attr: "T"
     is_ref: true
   }
   input_arg {
-    name: "m"
-    type_attr: "T"
-    is_ref: true
-  }
-  input_arg {
-    name: "v"
+    name: "accum"
     type_attr: "T"
     is_ref: true
   }
-  input_arg {
-    name: "beta1_power"
-    type_attr: "T"
-  }
-  input_arg {
-    name: "beta2_power"
-    type_attr: "T"
-  }
   input_arg {
     name: "lr"
     type_attr: "T"
   }
-  input_arg {
-    name: "beta1"
-    type_attr: "T"
-  }
-  input_arg {
-    name: "beta2"
-    type_attr: "T"
-  }
-  input_arg {
-    name: "epsilon"
-    type_attr: "T"
-  }
   input_arg {
     name: "grad"
     type_attr: "T"
@@ -940,6 +1038,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -952,28 +1052,24 @@ op {
   }
 }
 op {
-  name: "ApplyAdam"
+  name: "ApplyAdagradDA"
   input_arg {
     name: "var"
     type_attr: "T"
     is_ref: true
   }
   input_arg {
-    name: "m"
+    name: "gradient_accumulator"
     type_attr: "T"
     is_ref: true
   }
   input_arg {
-    name: "v"
+    name: "gradient_squared_accumulator"
     type_attr: "T"
     is_ref: true
   }
   input_arg {
-    name: "beta1_power"
-    type_attr: "T"
-  }
-  input_arg {
-    name: "beta2_power"
+    name: "grad"
     type_attr: "T"
   }
   input_arg {
@@ -981,20 +1077,16 @@ op {
     type_attr: "T"
   }
   input_arg {
-    name: "beta1"
-    type_attr: "T"
-  }
-  input_arg {
-    name: "beta2"
+    name: "l1"
     type_attr: "T"
   }
   input_arg {
-    name: "epsilon"
+    name: "l2"
     type_attr: "T"
   }
   input_arg {
-    name: "grad"
-    type_attr: "T"
+    name: "global_step"
+    type: DT_INT64
   }
   output_arg {
     name: "out"
@@ -1030,55 +1122,43 @@ op {
       b: false
     }
   }
-  attr {
-    name: "use_nesterov"
-    type: "bool"
-    default_value {
-      b: false
-    }
-  }
 }
 op {
-  name: "ApplyCenteredRMSProp"
+  name: "ApplyAdagradDA"
   input_arg {
     name: "var"
     type_attr: "T"
     is_ref: true
   }
   input_arg {
-    name: "mg"
+    name: "gradient_accumulator"
     type_attr: "T"
     is_ref: true
   }
   input_arg {
-    name: "ms"
+    name: "gradient_squared_accumulator"
     type_attr: "T"
     is_ref: true
   }
   input_arg {
-    name: "mom"
+    name: "grad"
     type_attr: "T"
-    is_ref: true
   }
   input_arg {
     name: "lr"
     type_attr: "T"
   }
   input_arg {
-    name: "rho"
-    type_attr: "T"
-  }
-  input_arg {
-    name: "momentum"
+    name: "l1"
     type_attr: "T"
   }
   input_arg {
-    name: "epsilon"
+    name: "l2"
     type_attr: "T"
   }
   input_arg {
-    name: "grad"
-    type_attr: "T"
+    name: "global_step"
+    type: DT_INT64
   }
   output_arg {
     name: "out"
@@ -1104,6 +1184,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -1116,24 +1198,28 @@ op {
   }
 }
 op {
-  name: "ApplyFtrl"
+  name: "ApplyAdam"
   input_arg {
     name: "var"
     type_attr: "T"
     is_ref: true
   }
   input_arg {
-    name: "accum"
+    name: "m"
     type_attr: "T"
     is_ref: true
   }
   input_arg {
-    name: "linear"
+    name: "v"
     type_attr: "T"
     is_ref: true
   }
   input_arg {
-    name: "grad"
+    name: "beta1_power"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "beta2_power"
     type_attr: "T"
   }
   input_arg {
@@ -1141,15 +1227,19 @@ op {
     type_attr: "T"
   }
   input_arg {
-    name: "l1"
+    name: "beta1"
     type_attr: "T"
   }
   input_arg {
-    name: "l2"
+    name: "beta2"
     type_attr: "T"
   }
   input_arg {
-    name: "lr_power"
+    name: "epsilon"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "grad"
     type_attr: "T"
   }
   output_arg {
@@ -1188,24 +1278,28 @@ op {
   }
 }
 op {
-  name: "ApplyFtrlV2"
+  name: "ApplyAdam"
   input_arg {
     name: "var"
     type_attr: "T"
     is_ref: true
   }
   input_arg {
-    name: "accum"
+    name: "m"
     type_attr: "T"
     is_ref: true
   }
   input_arg {
-    name: "linear"
+    name: "v"
     type_attr: "T"
     is_ref: true
   }
   input_arg {
-    name: "grad"
+    name: "beta1_power"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "beta2_power"
     type_attr: "T"
   }
   input_arg {
@@ -1213,19 +1307,19 @@ op {
     type_attr: "T"
   }
   input_arg {
-    name: "l1"
+    name: "beta1"
     type_attr: "T"
   }
   input_arg {
-    name: "l2"
+    name: "beta2"
     type_attr: "T"
   }
   input_arg {
-    name: "l2_shrinkage"
+    name: "epsilon"
     type_attr: "T"
   }
   input_arg {
-    name: "lr_power"
+    name: "grad"
     type_attr: "T"
   }
   output_arg {
@@ -1262,20 +1356,57 @@ op {
       b: false
     }
   }
+  attr {
+    name: "use_nesterov"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
 }
 op {
-  name: "ApplyGradientDescent"
+  name: "ApplyAdam"
   input_arg {
     name: "var"
     type_attr: "T"
     is_ref: true
   }
   input_arg {
-    name: "alpha"
+    name: "m"
+    type_attr: "T"
+    is_ref: true
+  }
+  input_arg {
+    name: "v"
     type_attr: "T"
+    is_ref: true
   }
   input_arg {
-    name: "delta"
+    name: "beta1_power"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "beta2_power"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "lr"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "beta1"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "beta2"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "epsilon"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "grad"
     type_attr: "T"
   }
   output_arg {
@@ -1302,6 +1433,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -1312,16 +1445,33 @@ op {
       b: false
     }
   }
+  attr {
+    name: "use_nesterov"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
 }
 op {
-  name: "ApplyMomentum"
+  name: "ApplyCenteredRMSProp"
   input_arg {
     name: "var"
     type_attr: "T"
     is_ref: true
   }
   input_arg {
-    name: "accum"
+    name: "mg"
+    type_attr: "T"
+    is_ref: true
+  }
+  input_arg {
+    name: "ms"
+    type_attr: "T"
+    is_ref: true
+  }
+  input_arg {
+    name: "mom"
     type_attr: "T"
     is_ref: true
   }
@@ -1330,13 +1480,21 @@ op {
     type_attr: "T"
   }
   input_arg {
-    name: "grad"
+    name: "rho"
     type_attr: "T"
   }
   input_arg {
     name: "momentum"
     type_attr: "T"
   }
+  input_arg {
+    name: "epsilon"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "grad"
+    type_attr: "T"
+  }
   output_arg {
     name: "out"
     type_attr: "T"
@@ -1371,23 +1529,26 @@ op {
       b: false
     }
   }
-  attr {
-    name: "use_nesterov"
-    type: "bool"
-    default_value {
-      b: false
-    }
-  }
 }
 op {
-  name: "ApplyProximalAdagrad"
+  name: "ApplyCenteredRMSProp"
   input_arg {
     name: "var"
     type_attr: "T"
     is_ref: true
   }
   input_arg {
-    name: "accum"
+    name: "mg"
+    type_attr: "T"
+    is_ref: true
+  }
+  input_arg {
+    name: "ms"
+    type_attr: "T"
+    is_ref: true
+  }
+  input_arg {
+    name: "mom"
     type_attr: "T"
     is_ref: true
   }
@@ -1396,11 +1557,15 @@ op {
     type_attr: "T"
   }
   input_arg {
-    name: "l1"
+    name: "rho"
     type_attr: "T"
   }
   input_arg {
-    name: "l2"
+    name: "momentum"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "epsilon"
     type_attr: "T"
   }
   input_arg {
@@ -1431,6 +1596,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -1443,14 +1610,28 @@ op {
   }
 }
 op {
-  name: "ApplyProximalGradientDescent"
+  name: "ApplyFtrl"
   input_arg {
     name: "var"
     type_attr: "T"
     is_ref: true
   }
   input_arg {
-    name: "alpha"
+    name: "accum"
+    type_attr: "T"
+    is_ref: true
+  }
+  input_arg {
+    name: "linear"
+    type_attr: "T"
+    is_ref: true
+  }
+  input_arg {
+    name: "grad"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "lr"
     type_attr: "T"
   }
   input_arg {
@@ -1462,7 +1643,7 @@ op {
     type_attr: "T"
   }
   input_arg {
-    name: "delta"
+    name: "lr_power"
     type_attr: "T"
   }
   output_arg {
@@ -1501,40 +1682,40 @@ op {
   }
 }
 op {
-  name: "ApplyRMSProp"
+  name: "ApplyFtrl"
   input_arg {
     name: "var"
     type_attr: "T"
     is_ref: true
   }
   input_arg {
-    name: "ms"
+    name: "accum"
     type_attr: "T"
     is_ref: true
   }
   input_arg {
-    name: "mom"
+    name: "linear"
     type_attr: "T"
     is_ref: true
   }
   input_arg {
-    name: "lr"
+    name: "grad"
     type_attr: "T"
   }
   input_arg {
-    name: "rho"
+    name: "lr"
     type_attr: "T"
   }
   input_arg {
-    name: "momentum"
+    name: "l1"
     type_attr: "T"
   }
   input_arg {
-    name: "epsilon"
+    name: "l2"
     type_attr: "T"
   }
   input_arg {
-    name: "grad"
+    name: "lr_power"
     type_attr: "T"
   }
   output_arg {
@@ -1561,6 +1742,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -1573,18 +1756,50 @@ op {
   }
 }
 op {
-  name: "ApproximateEqual"
+  name: "ApplyFtrlV2"
   input_arg {
-    name: "x"
+    name: "var"
     type_attr: "T"
+    is_ref: true
   }
   input_arg {
-    name: "y"
+    name: "accum"
+    type_attr: "T"
+    is_ref: true
+  }
+  input_arg {
+    name: "linear"
+    type_attr: "T"
+    is_ref: true
+  }
+  input_arg {
+    name: "grad"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "lr"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "l1"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "l2"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "l2_shrinkage"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "lr_power"
     type_attr: "T"
   }
   output_arg {
-    name: "z"
-    type: DT_BOOL
+    name: "out"
+    type_attr: "T"
+    is_ref: true
   }
   attr {
     name: "T"
@@ -1609,27 +1824,58 @@ op {
     }
   }
   attr {
-    name: "tolerance"
-    type: "float"
+    name: "use_locking"
+    type: "bool"
     default_value {
-      f: 1e-05
+      b: false
     }
   }
-  is_commutative: true
 }
 op {
-  name: "ArgMax"
+  name: "ApplyFtrlV2"
   input_arg {
-    name: "input"
+    name: "var"
     type_attr: "T"
+    is_ref: true
   }
   input_arg {
-    name: "dimension"
-    type_attr: "Tidx"
+    name: "accum"
+    type_attr: "T"
+    is_ref: true
+  }
+  input_arg {
+    name: "linear"
+    type_attr: "T"
+    is_ref: true
+  }
+  input_arg {
+    name: "grad"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "lr"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "l1"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "l2"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "l2_shrinkage"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "lr_power"
+    type_attr: "T"
   }
   output_arg {
-    name: "output"
-    type: DT_INT64
+    name: "out"
+    type_attr: "T"
+    is_ref: true
   }
   attr {
     name: "T"
@@ -1650,36 +1896,38 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
   attr {
-    name: "Tidx"
-    type: "type"
+    name: "use_locking"
+    type: "bool"
     default_value {
-      type: DT_INT32
-    }
-    allowed_values {
-      list {
-        type: DT_INT32
-        type: DT_INT64
-      }
+      b: false
     }
   }
 }
 op {
-  name: "ArgMax"
+  name: "ApplyGradientDescent"
   input_arg {
-    name: "input"
+    name: "var"
     type_attr: "T"
+    is_ref: true
   }
   input_arg {
-    name: "dimension"
-    type_attr: "Tidx"
+    name: "alpha"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "delta"
+    type_attr: "T"
   }
   output_arg {
-    name: "output"
-    type_attr: "output_type"
+    name: "out"
+    type_attr: "T"
+    is_ref: true
   }
   attr {
     name: "T"
@@ -1704,45 +1952,32 @@ op {
     }
   }
   attr {
-    name: "Tidx"
-    type: "type"
-    default_value {
-      type: DT_INT32
-    }
-    allowed_values {
-      list {
-        type: DT_INT32
-        type: DT_INT64
-      }
-    }
-  }
-  attr {
-    name: "output_type"
-    type: "type"
+    name: "use_locking"
+    type: "bool"
     default_value {
-      type: DT_INT64
-    }
-    allowed_values {
-      list {
-        type: DT_INT32
-        type: DT_INT64
-      }
+      b: false
     }
   }
 }
 op {
-  name: "ArgMin"
+  name: "ApplyGradientDescent"
   input_arg {
-    name: "input"
+    name: "var"
+    type_attr: "T"
+    is_ref: true
+  }
+  input_arg {
+    name: "alpha"
     type_attr: "T"
   }
   input_arg {
-    name: "dimension"
-    type_attr: "Tidx"
+    name: "delta"
+    type_attr: "T"
   }
   output_arg {
-    name: "output"
-    type: DT_INT64
+    name: "out"
+    type_attr: "T"
+    is_ref: true
   }
   attr {
     name: "T"
@@ -1763,36 +1998,47 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
   attr {
-    name: "Tidx"
-    type: "type"
+    name: "use_locking"
+    type: "bool"
     default_value {
-      type: DT_INT32
-    }
-    allowed_values {
-      list {
-        type: DT_INT32
-        type: DT_INT64
-      }
+      b: false
     }
   }
 }
 op {
-  name: "ArgMin"
+  name: "ApplyMomentum"
   input_arg {
-    name: "input"
+    name: "var"
     type_attr: "T"
+    is_ref: true
   }
   input_arg {
-    name: "dimension"
-    type_attr: "Tidx"
+    name: "accum"
+    type_attr: "T"
+    is_ref: true
+  }
+  input_arg {
+    name: "lr"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "grad"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "momentum"
+    type_attr: "T"
   }
   output_arg {
-    name: "output"
-    type_attr: "output_type"
+    name: "out"
+    type_attr: "T"
+    is_ref: true
   }
   attr {
     name: "T"
@@ -1817,217 +2063,299 @@ op {
     }
   }
   attr {
-    name: "Tidx"
-    type: "type"
+    name: "use_locking"
+    type: "bool"
     default_value {
-      type: DT_INT32
-    }
-    allowed_values {
-      list {
-        type: DT_INT32
-        type: DT_INT64
-      }
+      b: false
     }
   }
   attr {
-    name: "output_type"
-    type: "type"
+    name: "use_nesterov"
+    type: "bool"
     default_value {
-      type: DT_INT64
-    }
-    allowed_values {
-      list {
-        type: DT_INT32
-        type: DT_INT64
-      }
+      b: false
     }
   }
 }
 op {
-  name: "AsString"
+  name: "ApplyMomentum"
   input_arg {
-    name: "input"
+    name: "var"
+    type_attr: "T"
+    is_ref: true
+  }
+  input_arg {
+    name: "accum"
+    type_attr: "T"
+    is_ref: true
+  }
+  input_arg {
+    name: "lr"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "grad"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "momentum"
     type_attr: "T"
   }
   output_arg {
-    name: "output"
-    type: DT_STRING
+    name: "out"
+    type_attr: "T"
+    is_ref: true
   }
   attr {
     name: "T"
     type: "type"
     allowed_values {
       list {
-        type: DT_INT32
-        type: DT_INT64
-        type: DT_COMPLEX64
         type: DT_FLOAT
         type: DT_DOUBLE
-        type: DT_BOOL
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
         type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
   attr {
-    name: "precision"
-    type: "int"
-    default_value {
-      i: -1
-    }
-  }
-  attr {
-    name: "scientific"
+    name: "use_locking"
     type: "bool"
     default_value {
       b: false
     }
   }
   attr {
-    name: "shortest"
+    name: "use_nesterov"
     type: "bool"
     default_value {
       b: false
     }
   }
-  attr {
-    name: "width"
-    type: "int"
-    default_value {
-      i: -1
-    }
-  }
-  attr {
-    name: "fill"
-    type: "string"
-    default_value {
-      s: ""
-    }
-  }
 }
 op {
-  name: "Asin"
+  name: "ApplyProximalAdagrad"
   input_arg {
-    name: "x"
+    name: "var"
+    type_attr: "T"
+    is_ref: true
+  }
+  input_arg {
+    name: "accum"
+    type_attr: "T"
+    is_ref: true
+  }
+  input_arg {
+    name: "lr"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "l1"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "l2"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "grad"
     type_attr: "T"
   }
   output_arg {
-    name: "y"
+    name: "out"
     type_attr: "T"
+    is_ref: true
   }
   attr {
     name: "T"
     type: "type"
     allowed_values {
       list {
-        type: DT_HALF
         type: DT_FLOAT
         type: DT_DOUBLE
-        type: DT_INT32
         type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
         type: DT_COMPLEX64
         type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
       }
     }
   }
+  attr {
+    name: "use_locking"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
 }
 op {
-  name: "Asinh"
+  name: "ApplyProximalAdagrad"
   input_arg {
-    name: "x"
+    name: "var"
     type_attr: "T"
+    is_ref: true
   }
-  output_arg {
-    name: "y"
+  input_arg {
+    name: "accum"
     type_attr: "T"
+    is_ref: true
+  }
+  input_arg {
+    name: "lr"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "l1"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "l2"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "grad"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "out"
+    type_attr: "T"
+    is_ref: true
   }
   attr {
     name: "T"
     type: "type"
     allowed_values {
       list {
-        type: DT_HALF
         type: DT_FLOAT
         type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
         type: DT_COMPLEX64
         type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
-}
-op {
-  name: "Assert"
-  input_arg {
-    name: "condition"
-    type: DT_BOOL
-  }
-  input_arg {
-    name: "data"
-    type_list_attr: "T"
-  }
-  attr {
-    name: "T"
-    type: "list(type)"
-    has_minimum: true
-    minimum: 1
-  }
   attr {
-    name: "summarize"
-    type: "int"
+    name: "use_locking"
+    type: "bool"
     default_value {
-      i: 3
+      b: false
     }
   }
-  is_stateful: true
 }
 op {
-  name: "Assign"
+  name: "ApplyProximalGradientDescent"
   input_arg {
-    name: "ref"
+    name: "var"
     type_attr: "T"
     is_ref: true
   }
   input_arg {
-    name: "value"
+    name: "alpha"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "l1"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "l2"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "delta"
     type_attr: "T"
   }
   output_arg {
-    name: "output_ref"
+    name: "out"
     type_attr: "T"
     is_ref: true
   }
   attr {
     name: "T"
     type: "type"
-  }
-  attr {
-    name: "validate_shape"
-    type: "bool"
-    default_value {
-      b: true
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
+      }
     }
   }
   attr {
     name: "use_locking"
     type: "bool"
     default_value {
-      b: true
+      b: false
     }
   }
-  allows_uninitialized_input: true
 }
 op {
-  name: "AssignAdd"
+  name: "ApplyProximalGradientDescent"
   input_arg {
-    name: "ref"
+    name: "var"
     type_attr: "T"
     is_ref: true
   }
   input_arg {
-    name: "value"
+    name: "alpha"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "l1"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "l2"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "delta"
     type_attr: "T"
   }
   output_arg {
-    name: "output_ref"
+    name: "out"
     type_attr: "T"
     is_ref: true
   }
@@ -2050,6 +2378,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -2062,34 +2392,44 @@ op {
   }
 }
 op {
-  name: "AssignAddVariableOp"
+  name: "ApplyRMSProp"
   input_arg {
-    name: "resource"
-    type: DT_RESOURCE
+    name: "var"
+    type_attr: "T"
+    is_ref: true
   }
   input_arg {
-    name: "value"
-    type_attr: "dtype"
-  }
-  attr {
-    name: "dtype"
-    type: "type"
+    name: "ms"
+    type_attr: "T"
+    is_ref: true
   }
-  is_stateful: true
-}
-op {
-  name: "AssignSub"
   input_arg {
-    name: "ref"
+    name: "mom"
     type_attr: "T"
     is_ref: true
   }
   input_arg {
-    name: "value"
+    name: "lr"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "rho"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "momentum"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "epsilon"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "grad"
     type_attr: "T"
   }
   output_arg {
-    name: "output_ref"
+    name: "out"
     type_attr: "T"
     is_ref: true
   }
@@ -2124,76 +2464,92 @@ op {
   }
 }
 op {
-  name: "AssignSubVariableOp"
+  name: "ApplyRMSProp"
   input_arg {
-    name: "resource"
-    type: DT_RESOURCE
+    name: "var"
+    type_attr: "T"
+    is_ref: true
   }
   input_arg {
-    name: "value"
-    type_attr: "dtype"
+    name: "ms"
+    type_attr: "T"
+    is_ref: true
   }
-  attr {
-    name: "dtype"
-    type: "type"
+  input_arg {
+    name: "mom"
+    type_attr: "T"
+    is_ref: true
   }
-  is_stateful: true
-}
-op {
-  name: "AssignVariableOp"
   input_arg {
-    name: "resource"
-    type: DT_RESOURCE
+    name: "lr"
+    type_attr: "T"
   }
   input_arg {
-    name: "value"
-    type_attr: "dtype"
+    name: "rho"
+    type_attr: "T"
   }
-  attr {
-    name: "dtype"
-    type: "type"
+  input_arg {
+    name: "momentum"
+    type_attr: "T"
   }
-  is_stateful: true
-}
-op {
-  name: "Atan"
   input_arg {
-    name: "x"
+    name: "epsilon"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "grad"
     type_attr: "T"
   }
   output_arg {
-    name: "y"
+    name: "out"
     type_attr: "T"
+    is_ref: true
   }
   attr {
     name: "T"
     type: "type"
     allowed_values {
       list {
-        type: DT_HALF
         type: DT_FLOAT
         type: DT_DOUBLE
-        type: DT_INT32
         type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
         type: DT_COMPLEX64
         type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
+  attr {
+    name: "use_locking"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
 }
 op {
-  name: "Atan2"
+  name: "ApproximateEqual"
   input_arg {
-    name: "y"
+    name: "x"
     type_attr: "T"
   }
   input_arg {
-    name: "x"
+    name: "y"
     type_attr: "T"
   }
   output_arg {
     name: "z"
-    type_attr: "T"
+    type: DT_BOOL
   }
   attr {
     name: "T"
@@ -2202,606 +2558,621 @@ op {
       list {
         type: DT_FLOAT
         type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
       }
     }
   }
+  attr {
+    name: "tolerance"
+    type: "float"
+    default_value {
+      f: 1e-05
+    }
+  }
+  is_commutative: true
 }
 op {
-  name: "Atanh"
+  name: "ApproximateEqual"
   input_arg {
     name: "x"
     type_attr: "T"
   }
-  output_arg {
+  input_arg {
     name: "y"
     type_attr: "T"
   }
+  output_arg {
+    name: "z"
+    type: DT_BOOL
+  }
   attr {
     name: "T"
     type: "type"
     allowed_values {
       list {
-        type: DT_HALF
         type: DT_FLOAT
         type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
         type: DT_COMPLEX64
         type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
-}
-op {
-  name: "AudioSpectrogram"
-  input_arg {
-    name: "input"
-    type: DT_FLOAT
-  }
-  output_arg {
-    name: "spectrogram"
-    type: DT_FLOAT
-  }
-  attr {
-    name: "window_size"
-    type: "int"
-  }
-  attr {
-    name: "stride"
-    type: "int"
-  }
   attr {
-    name: "magnitude_squared"
-    type: "bool"
+    name: "tolerance"
+    type: "float"
     default_value {
-      b: false
+      f: 1e-05
     }
   }
+  is_commutative: true
 }
 op {
-  name: "AudioSummary"
+  name: "ArgMax"
   input_arg {
-    name: "tag"
-    type: DT_STRING
+    name: "input"
+    type_attr: "T"
   }
   input_arg {
-    name: "tensor"
-    type: DT_FLOAT
+    name: "dimension"
+    type_attr: "Tidx"
   }
   output_arg {
-    name: "summary"
-    type: DT_STRING
-  }
-  attr {
-    name: "sample_rate"
-    type: "float"
+    name: "output"
+    type: DT_INT64
   }
   attr {
-    name: "max_outputs"
-    type: "int"
-    default_value {
-      i: 3
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
+      }
     }
-    has_minimum: true
-    minimum: 1
-  }
-  deprecation {
-    version: 15
-  }
-}
-op {
-  name: "AudioSummaryV2"
-  input_arg {
-    name: "tag"
-    type: DT_STRING
-  }
-  input_arg {
-    name: "tensor"
-    type: DT_FLOAT
-  }
-  input_arg {
-    name: "sample_rate"
-    type: DT_FLOAT
-  }
-  output_arg {
-    name: "summary"
-    type: DT_STRING
   }
   attr {
-    name: "max_outputs"
-    type: "int"
+    name: "Tidx"
+    type: "type"
     default_value {
-      i: 3
+      type: DT_INT32
+    }
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
     }
-    has_minimum: true
-    minimum: 1
   }
 }
 op {
-  name: "AvgPool"
+  name: "ArgMax"
   input_arg {
-    name: "value"
+    name: "input"
     type_attr: "T"
   }
+  input_arg {
+    name: "dimension"
+    type_attr: "Tidx"
+  }
   output_arg {
     name: "output"
-    type_attr: "T"
-  }
-  attr {
-    name: "ksize"
-    type: "list(int)"
-    has_minimum: true
-    minimum: 4
-  }
-  attr {
-    name: "strides"
-    type: "list(int)"
-    has_minimum: true
-    minimum: 4
+    type_attr: "output_type"
   }
   attr {
-    name: "padding"
-    type: "string"
+    name: "T"
+    type: "type"
     allowed_values {
       list {
-        s: "SAME"
-        s: "VALID"
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
       }
     }
   }
   attr {
-    name: "data_format"
-    type: "string"
+    name: "Tidx"
+    type: "type"
     default_value {
-      s: "NHWC"
+      type: DT_INT32
     }
     allowed_values {
       list {
-        s: "NHWC"
-        s: "NCHW"
+        type: DT_INT32
+        type: DT_INT64
       }
     }
   }
   attr {
-    name: "T"
+    name: "output_type"
     type: "type"
+    default_value {
+      type: DT_INT64
+    }
     allowed_values {
       list {
-        type: DT_FLOAT
-        type: DT_HALF
-        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_INT64
       }
     }
   }
 }
 op {
-  name: "AvgPool"
+  name: "ArgMax"
   input_arg {
-    name: "value"
+    name: "input"
     type_attr: "T"
   }
+  input_arg {
+    name: "dimension"
+    type_attr: "Tidx"
+  }
   output_arg {
     name: "output"
-    type_attr: "T"
-  }
-  attr {
-    name: "ksize"
-    type: "list(int)"
-    has_minimum: true
-    minimum: 4
-  }
-  attr {
-    name: "strides"
-    type: "list(int)"
-    has_minimum: true
-    minimum: 4
+    type_attr: "output_type"
   }
   attr {
-    name: "padding"
-    type: "string"
+    name: "T"
+    type: "type"
     allowed_values {
       list {
-        s: "SAME"
-        s: "VALID"
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
   attr {
-    name: "data_format"
-    type: "string"
+    name: "Tidx"
+    type: "type"
     default_value {
-      s: "NHWC"
+      type: DT_INT32
     }
     allowed_values {
       list {
-        s: "NHWC"
-        s: "NCHW"
+        type: DT_INT32
+        type: DT_INT64
       }
     }
   }
   attr {
-    name: "T"
+    name: "output_type"
     type: "type"
+    default_value {
+      type: DT_INT64
+    }
     allowed_values {
       list {
-        type: DT_FLOAT
-        type: DT_DOUBLE
-        type: DT_HALF
+        type: DT_INT32
+        type: DT_INT64
       }
     }
   }
 }
 op {
-  name: "AvgPool"
+  name: "ArgMin"
   input_arg {
-    name: "value"
+    name: "input"
     type_attr: "T"
   }
+  input_arg {
+    name: "dimension"
+    type_attr: "Tidx"
+  }
   output_arg {
     name: "output"
-    type_attr: "T"
-  }
-  attr {
-    name: "ksize"
-    type: "list(int)"
-    has_minimum: true
-    minimum: 4
-  }
-  attr {
-    name: "strides"
-    type: "list(int)"
-    has_minimum: true
-    minimum: 4
+    type: DT_INT64
   }
   attr {
-    name: "padding"
-    type: "string"
+    name: "T"
+    type: "type"
     allowed_values {
       list {
-        s: "SAME"
-        s: "VALID"
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
       }
     }
   }
   attr {
-    name: "data_format"
-    type: "string"
+    name: "Tidx"
+    type: "type"
     default_value {
-      s: "NHWC"
-    }
-    allowed_values {
-      list {
-        s: "NHWC"
-        s: "NCHW"
-      }
+      type: DT_INT32
     }
-  }
-  attr {
-    name: "T"
-    type: "type"
     allowed_values {
       list {
-        type: DT_HALF
-        type: DT_FLOAT
-        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_INT64
       }
     }
   }
 }
 op {
-  name: "AvgPool3D"
+  name: "ArgMin"
   input_arg {
     name: "input"
     type_attr: "T"
   }
+  input_arg {
+    name: "dimension"
+    type_attr: "Tidx"
+  }
   output_arg {
     name: "output"
-    type_attr: "T"
-  }
-  attr {
-    name: "ksize"
-    type: "list(int)"
-    has_minimum: true
-    minimum: 5
+    type_attr: "output_type"
   }
   attr {
-    name: "strides"
-    type: "list(int)"
-    has_minimum: true
-    minimum: 5
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
+      }
+    }
   }
   attr {
-    name: "padding"
-    type: "string"
+    name: "Tidx"
+    type: "type"
+    default_value {
+      type: DT_INT32
+    }
     allowed_values {
       list {
-        s: "SAME"
-        s: "VALID"
+        type: DT_INT32
+        type: DT_INT64
       }
     }
   }
   attr {
-    name: "T"
+    name: "output_type"
     type: "type"
+    default_value {
+      type: DT_INT64
+    }
     allowed_values {
       list {
-        type: DT_FLOAT
-        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_INT64
       }
     }
   }
 }
 op {
-  name: "AvgPool3D"
+  name: "ArgMin"
   input_arg {
     name: "input"
     type_attr: "T"
   }
+  input_arg {
+    name: "dimension"
+    type_attr: "Tidx"
+  }
   output_arg {
     name: "output"
-    type_attr: "T"
-  }
-  attr {
-    name: "ksize"
-    type: "list(int)"
-    has_minimum: true
-    minimum: 5
-  }
-  attr {
-    name: "strides"
-    type: "list(int)"
-    has_minimum: true
-    minimum: 5
+    type_attr: "output_type"
   }
   attr {
-    name: "padding"
-    type: "string"
+    name: "T"
+    type: "type"
     allowed_values {
       list {
-        s: "SAME"
-        s: "VALID"
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
   attr {
-    name: "data_format"
-    type: "string"
+    name: "Tidx"
+    type: "type"
     default_value {
-      s: "NDHWC"
+      type: DT_INT32
     }
     allowed_values {
       list {
-        s: "NDHWC"
-        s: "NCDHW"
+        type: DT_INT32
+        type: DT_INT64
       }
     }
   }
   attr {
-    name: "T"
+    name: "output_type"
     type: "type"
+    default_value {
+      type: DT_INT64
+    }
     allowed_values {
       list {
-        type: DT_FLOAT
-        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_INT64
       }
     }
   }
 }
 op {
-  name: "AvgPool3DGrad"
-  input_arg {
-    name: "orig_input_shape"
-    type: DT_INT32
-  }
+  name: "AsString"
   input_arg {
-    name: "grad"
+    name: "input"
     type_attr: "T"
   }
   output_arg {
     name: "output"
-    type_attr: "T"
-  }
-  attr {
-    name: "ksize"
-    type: "list(int)"
-    has_minimum: true
-    minimum: 5
-  }
-  attr {
-    name: "strides"
-    type: "list(int)"
-    has_minimum: true
-    minimum: 5
-  }
-  attr {
-    name: "padding"
-    type: "string"
-    allowed_values {
-      list {
-        s: "SAME"
-        s: "VALID"
-      }
-    }
+    type: DT_STRING
   }
   attr {
     name: "T"
     type: "type"
     allowed_values {
       list {
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_COMPLEX64
         type: DT_FLOAT
         type: DT_DOUBLE
+        type: DT_BOOL
+        type: DT_INT8
       }
     }
   }
-}
-op {
-  name: "AvgPool3DGrad"
-  input_arg {
-    name: "orig_input_shape"
-    type: DT_INT32
-  }
-  input_arg {
-    name: "grad"
-    type_attr: "T"
-  }
-  output_arg {
-    name: "output"
-    type_attr: "T"
+  attr {
+    name: "precision"
+    type: "int"
+    default_value {
+      i: -1
+    }
   }
   attr {
-    name: "ksize"
-    type: "list(int)"
-    has_minimum: true
-    minimum: 5
+    name: "scientific"
+    type: "bool"
+    default_value {
+      b: false
+    }
   }
   attr {
-    name: "strides"
-    type: "list(int)"
-    has_minimum: true
-    minimum: 5
+    name: "shortest"
+    type: "bool"
+    default_value {
+      b: false
+    }
   }
   attr {
-    name: "padding"
-    type: "string"
-    allowed_values {
-      list {
-        s: "SAME"
-        s: "VALID"
-      }
+    name: "width"
+    type: "int"
+    default_value {
+      i: -1
     }
   }
   attr {
-    name: "data_format"
+    name: "fill"
     type: "string"
     default_value {
-      s: "NDHWC"
-    }
-    allowed_values {
-      list {
-        s: "NDHWC"
-        s: "NCDHW"
-      }
+      s: ""
     }
   }
+}
+op {
+  name: "Asin"
+  input_arg {
+    name: "x"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "y"
+    type_attr: "T"
+  }
   attr {
     name: "T"
     type: "type"
     allowed_values {
       list {
+        type: DT_HALF
         type: DT_FLOAT
         type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
       }
     }
   }
 }
 op {
-  name: "AvgPoolGrad"
-  input_arg {
-    name: "orig_input_shape"
-    type: DT_INT32
-  }
+  name: "Asinh"
   input_arg {
-    name: "grad"
+    name: "x"
     type_attr: "T"
   }
   output_arg {
-    name: "output"
+    name: "y"
     type_attr: "T"
   }
   attr {
-    name: "ksize"
-    type: "list(int)"
-    has_minimum: true
-    minimum: 4
-  }
-  attr {
-    name: "strides"
-    type: "list(int)"
-    has_minimum: true
-    minimum: 4
-  }
-  attr {
-    name: "padding"
-    type: "string"
+    name: "T"
+    type: "type"
     allowed_values {
       list {
-        s: "SAME"
-        s: "VALID"
+        type: DT_HALF
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
       }
     }
   }
-  attr {
-    name: "data_format"
-    type: "string"
-    default_value {
-      s: "NHWC"
-    }
-    allowed_values {
-      list {
-        s: "NHWC"
-        s: "NCHW"
-      }
-    }
+}
+op {
+  name: "Assert"
+  input_arg {
+    name: "condition"
+    type: DT_BOOL
+  }
+  input_arg {
+    name: "data"
+    type_list_attr: "T"
   }
   attr {
     name: "T"
-    type: "type"
-    allowed_values {
-      list {
-        type: DT_FLOAT
-        type: DT_HALF
-        type: DT_DOUBLE
-      }
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "summarize"
+    type: "int"
+    default_value {
+      i: 3
     }
   }
+  is_stateful: true
 }
 op {
-  name: "AvgPoolGrad"
+  name: "Assign"
   input_arg {
-    name: "orig_input_shape"
-    type: DT_INT32
+    name: "ref"
+    type_attr: "T"
+    is_ref: true
   }
   input_arg {
-    name: "grad"
+    name: "value"
     type_attr: "T"
   }
   output_arg {
-    name: "output"
+    name: "output_ref"
     type_attr: "T"
+    is_ref: true
   }
   attr {
-    name: "ksize"
-    type: "list(int)"
-    has_minimum: true
-    minimum: 4
-  }
-  attr {
-    name: "strides"
-    type: "list(int)"
-    has_minimum: true
-    minimum: 4
+    name: "T"
+    type: "type"
   }
   attr {
-    name: "padding"
-    type: "string"
-    allowed_values {
-      list {
-        s: "SAME"
-        s: "VALID"
-      }
+    name: "validate_shape"
+    type: "bool"
+    default_value {
+      b: true
     }
   }
   attr {
-    name: "data_format"
-    type: "string"
+    name: "use_locking"
+    type: "bool"
     default_value {
-      s: "NHWC"
-    }
-    allowed_values {
-      list {
-        s: "NHWC"
-        s: "NCHW"
-      }
+      b: true
     }
   }
+  allows_uninitialized_input: true
+}
+op {
+  name: "AssignAdd"
+  input_arg {
+    name: "ref"
+    type_attr: "T"
+    is_ref: true
+  }
+  input_arg {
+    name: "value"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "output_ref"
+    type_attr: "T"
+    is_ref: true
+  }
   attr {
     name: "T"
     type: "type"
@@ -2809,126 +3180,133 @@ op {
       list {
         type: DT_FLOAT
         type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
         type: DT_HALF
       }
     }
   }
+  attr {
+    name: "use_locking"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
 }
 op {
-  name: "AvgPoolGrad"
+  name: "AssignAdd"
   input_arg {
-    name: "orig_input_shape"
-    type: DT_INT32
+    name: "ref"
+    type_attr: "T"
+    is_ref: true
   }
   input_arg {
-    name: "grad"
+    name: "value"
     type_attr: "T"
   }
   output_arg {
-    name: "output"
+    name: "output_ref"
     type_attr: "T"
-  }
-  attr {
-    name: "ksize"
-    type: "list(int)"
-    has_minimum: true
-    minimum: 4
-  }
-  attr {
-    name: "strides"
-    type: "list(int)"
-    has_minimum: true
-    minimum: 4
-  }
-  attr {
-    name: "padding"
-    type: "string"
-    allowed_values {
-      list {
-        s: "SAME"
-        s: "VALID"
-      }
-    }
-  }
-  attr {
-    name: "data_format"
-    type: "string"
-    default_value {
-      s: "NHWC"
-    }
-    allowed_values {
-      list {
-        s: "NHWC"
-        s: "NCHW"
-      }
-    }
+    is_ref: true
   }
   attr {
     name: "T"
     type: "type"
     allowed_values {
       list {
-        type: DT_HALF
         type: DT_FLOAT
         type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
+  attr {
+    name: "use_locking"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
 }
 op {
-  name: "Barrier"
-  output_arg {
-    name: "handle"
-    type: DT_STRING
-    is_ref: true
-  }
-  attr {
-    name: "component_types"
-    type: "list(type)"
-    has_minimum: true
-    minimum: 1
-  }
-  attr {
-    name: "shapes"
-    type: "list(shape)"
-    default_value {
-      list {
-      }
-    }
-    has_minimum: true
-  }
-  attr {
-    name: "capacity"
-    type: "int"
-    default_value {
-      i: -1
-    }
+  name: "AssignAddVariableOp"
+  input_arg {
+    name: "resource"
+    type: DT_RESOURCE
   }
-  attr {
-    name: "container"
-    type: "string"
-    default_value {
-      s: ""
-    }
+  input_arg {
+    name: "value"
+    type_attr: "dtype"
   }
   attr {
-    name: "shared_name"
-    type: "string"
-    default_value {
-      s: ""
-    }
+    name: "dtype"
+    type: "type"
   }
   is_stateful: true
 }
 op {
-  name: "BarrierClose"
+  name: "AssignSub"
   input_arg {
-    name: "handle"
-    type: DT_STRING
+    name: "ref"
+    type_attr: "T"
+    is_ref: true
+  }
+  input_arg {
+    name: "value"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "output_ref"
+    type_attr: "T"
     is_ref: true
   }
   attr {
-    name: "cancel_pending_enqueues"
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
+      }
+    }
+  }
+  attr {
+    name: "use_locking"
     type: "bool"
     default_value {
       b: false
@@ -2936,112 +3314,93 @@ op {
   }
 }
 op {
-  name: "BarrierIncompleteSize"
-  input_arg {
-    name: "handle"
-    type: DT_STRING
-    is_ref: true
-  }
-  output_arg {
-    name: "size"
-    type: DT_INT32
-  }
-}
-op {
-  name: "BarrierInsertMany"
+  name: "AssignSub"
   input_arg {
-    name: "handle"
-    type: DT_STRING
+    name: "ref"
+    type_attr: "T"
     is_ref: true
   }
   input_arg {
-    name: "keys"
-    type: DT_STRING
+    name: "value"
+    type_attr: "T"
   }
-  input_arg {
-    name: "values"
+  output_arg {
+    name: "output_ref"
     type_attr: "T"
+    is_ref: true
   }
   attr {
     name: "T"
     type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
   }
   attr {
-    name: "component_index"
-    type: "int"
+    name: "use_locking"
+    type: "bool"
+    default_value {
+      b: false
+    }
   }
 }
 op {
-  name: "BarrierReadySize"
+  name: "AssignSubVariableOp"
   input_arg {
-    name: "handle"
-    type: DT_STRING
-    is_ref: true
+    name: "resource"
+    type: DT_RESOURCE
   }
-  output_arg {
-    name: "size"
-    type: DT_INT32
+  input_arg {
+    name: "value"
+    type_attr: "dtype"
+  }
+  attr {
+    name: "dtype"
+    type: "type"
   }
+  is_stateful: true
 }
 op {
-  name: "BarrierTakeMany"
+  name: "AssignVariableOp"
   input_arg {
-    name: "handle"
-    type: DT_STRING
-    is_ref: true
+    name: "resource"
+    type: DT_RESOURCE
   }
   input_arg {
-    name: "num_elements"
-    type: DT_INT32
-  }
-  output_arg {
-    name: "indices"
-    type: DT_INT64
-  }
-  output_arg {
-    name: "keys"
-    type: DT_STRING
-  }
-  output_arg {
-    name: "values"
-    type_list_attr: "component_types"
-  }
-  attr {
-    name: "component_types"
-    type: "list(type)"
-    has_minimum: true
-    minimum: 1
-  }
-  attr {
-    name: "allow_small_batch"
-    type: "bool"
-    default_value {
-      b: false
-    }
-  }
-  attr {
-    name: "wait_for_incomplete"
-    type: "bool"
-    default_value {
-      b: false
-    }
+    name: "value"
+    type_attr: "dtype"
   }
   attr {
-    name: "timeout_ms"
-    type: "int"
-    default_value {
-      i: -1
-    }
+    name: "dtype"
+    type: "type"
   }
+  is_stateful: true
 }
 op {
-  name: "BatchCholesky"
+  name: "Atan"
   input_arg {
-    name: "input"
+    name: "x"
     type_attr: "T"
   }
   output_arg {
-    name: "output"
+    name: "y"
     type_attr: "T"
   }
   attr {
@@ -3049,27 +3408,29 @@ op {
     type: "type"
     allowed_values {
       list {
-        type: DT_DOUBLE
+        type: DT_HALF
         type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
       }
     }
   }
-  deprecation {
-    version: 13
-  }
 }
 op {
-  name: "BatchCholeskyGrad"
+  name: "Atan2"
   input_arg {
-    name: "l"
+    name: "y"
     type_attr: "T"
   }
   input_arg {
-    name: "grad"
+    name: "x"
     type_attr: "T"
   }
   output_arg {
-    name: "output"
+    name: "z"
     type_attr: "T"
   }
   attr {
@@ -3082,157 +3443,120 @@ op {
       }
     }
   }
-  deprecation {
-    version: 13
-  }
 }
 op {
-  name: "BatchDataset"
-  input_arg {
-    name: "input_dataset"
-    type: DT_VARIANT
-  }
+  name: "Atanh"
   input_arg {
-    name: "batch_size"
-    type: DT_INT64
+    name: "x"
+    type_attr: "T"
   }
   output_arg {
-    name: "handle"
-    type: DT_VARIANT
-  }
-  attr {
-    name: "output_types"
-    type: "list(type)"
-    has_minimum: true
-    minimum: 1
+    name: "y"
+    type_attr: "T"
   }
   attr {
-    name: "output_shapes"
-    type: "list(shape)"
-    has_minimum: true
-    minimum: 1
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_HALF
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+      }
+    }
   }
-  is_stateful: true
 }
 op {
-  name: "BatchDataset"
-  input_arg {
-    name: "input_dataset"
-    type: DT_VARIANT
-  }
+  name: "AudioSpectrogram"
   input_arg {
-    name: "batch_size"
-    type: DT_INT64
+    name: "input"
+    type: DT_FLOAT
   }
   output_arg {
-    name: "handle"
-    type: DT_VARIANT
+    name: "spectrogram"
+    type: DT_FLOAT
   }
   attr {
-    name: "output_types"
-    type: "list(type)"
-    has_minimum: true
-    minimum: 1
+    name: "window_size"
+    type: "int"
   }
   attr {
-    name: "output_shapes"
-    type: "list(shape)"
-    has_minimum: true
-    minimum: 1
-  }
-}
-op {
-  name: "BatchFFT"
-  input_arg {
-    name: "input"
-    type: DT_COMPLEX64
-  }
-  output_arg {
-    name: "output"
-    type: DT_COMPLEX64
+    name: "stride"
+    type: "int"
   }
-  deprecation {
-    version: 15
+  attr {
+    name: "magnitude_squared"
+    type: "bool"
+    default_value {
+      b: false
+    }
   }
 }
 op {
-  name: "BatchFFT2D"
+  name: "AudioSummary"
   input_arg {
-    name: "input"
-    type: DT_COMPLEX64
-  }
-  output_arg {
-    name: "output"
-    type: DT_COMPLEX64
-  }
-  deprecation {
-    version: 15
+    name: "tag"
+    type: DT_STRING
   }
-}
-op {
-  name: "BatchFFT3D"
   input_arg {
-    name: "input"
-    type: DT_COMPLEX64
+    name: "tensor"
+    type: DT_FLOAT
   }
   output_arg {
-    name: "output"
-    type: DT_COMPLEX64
-  }
-  deprecation {
-    version: 15
+    name: "summary"
+    type: DT_STRING
   }
-}
-op {
-  name: "BatchIFFT"
-  input_arg {
-    name: "input"
-    type: DT_COMPLEX64
+  attr {
+    name: "sample_rate"
+    type: "float"
   }
-  output_arg {
-    name: "output"
-    type: DT_COMPLEX64
+  attr {
+    name: "max_outputs"
+    type: "int"
+    default_value {
+      i: 3
+    }
+    has_minimum: true
+    minimum: 1
   }
   deprecation {
     version: 15
   }
 }
 op {
-  name: "BatchIFFT2D"
+  name: "AudioSummaryV2"
   input_arg {
-    name: "input"
-    type: DT_COMPLEX64
-  }
-  output_arg {
-    name: "output"
-    type: DT_COMPLEX64
+    name: "tag"
+    type: DT_STRING
   }
-  deprecation {
-    version: 15
+  input_arg {
+    name: "tensor"
+    type: DT_FLOAT
   }
-}
-op {
-  name: "BatchIFFT3D"
   input_arg {
-    name: "input"
-    type: DT_COMPLEX64
+    name: "sample_rate"
+    type: DT_FLOAT
   }
   output_arg {
-    name: "output"
-    type: DT_COMPLEX64
+    name: "summary"
+    type: DT_STRING
   }
-  deprecation {
-    version: 15
+  attr {
+    name: "max_outputs"
+    type: "int"
+    default_value {
+      i: 3
+    }
+    has_minimum: true
+    minimum: 1
   }
 }
 op {
-  name: "BatchMatMul"
-  input_arg {
-    name: "x"
-    type_attr: "T"
-  }
+  name: "AvgPool"
   input_arg {
-    name: "y"
+    name: "value"
     type_attr: "T"
   }
   output_arg {
@@ -3240,88 +3564,56 @@ op {
     type_attr: "T"
   }
   attr {
-    name: "T"
-    type: "type"
+    name: "ksize"
+    type: "list(int)"
+    has_minimum: true
+    minimum: 4
+  }
+  attr {
+    name: "strides"
+    type: "list(int)"
+    has_minimum: true
+    minimum: 4
+  }
+  attr {
+    name: "padding"
+    type: "string"
     allowed_values {
       list {
-        type: DT_HALF
-        type: DT_FLOAT
-        type: DT_DOUBLE
-        type: DT_INT32
-        type: DT_COMPLEX64
-        type: DT_COMPLEX128
+        s: "SAME"
+        s: "VALID"
       }
     }
   }
   attr {
-    name: "adj_x"
-    type: "bool"
+    name: "data_format"
+    type: "string"
     default_value {
-      b: false
+      s: "NHWC"
     }
-  }
-  attr {
-    name: "adj_y"
-    type: "bool"
-    default_value {
-      b: false
+    allowed_values {
+      list {
+        s: "NHWC"
+        s: "NCHW"
+      }
     }
   }
-}
-op {
-  name: "BatchMatrixBandPart"
-  input_arg {
-    name: "input"
-    type_attr: "T"
-  }
-  input_arg {
-    name: "num_lower"
-    type: DT_INT64
-  }
-  input_arg {
-    name: "num_upper"
-    type: DT_INT64
-  }
-  output_arg {
-    name: "band"
-    type_attr: "T"
-  }
-  attr {
-    name: "T"
-    type: "type"
-  }
-  deprecation {
-    version: 14
-  }
-}
-op {
-  name: "BatchMatrixDeterminant"
-  input_arg {
-    name: "input"
-    type_attr: "T"
-  }
-  output_arg {
-    name: "output"
-    type_attr: "T"
-  }
   attr {
     name: "T"
     type: "type"
     allowed_values {
       list {
         type: DT_FLOAT
+        type: DT_HALF
         type: DT_DOUBLE
       }
     }
   }
-  deprecation {
-    version: 13
-  }
 }
 op {
-  name: "BatchMatrixDeterminant"
+  name: "AvgPool"
   input_arg {
-    name: "input"
+    name: "value"
     type_attr: "T"
   }
   output_arg {
@@ -3329,72 +3621,95 @@ op {
     type_attr: "T"
   }
   attr {
-    name: "T"
-    type: "type"
+    name: "ksize"
+    type: "list(int)"
+    has_minimum: true
+    minimum: 4
+  }
+  attr {
+    name: "strides"
+    type: "list(int)"
+    has_minimum: true
+    minimum: 4
+  }
+  attr {
+    name: "padding"
+    type: "string"
     allowed_values {
       list {
-        type: DT_FLOAT
-        type: DT_DOUBLE
-        type: DT_COMPLEX64
-        type: DT_COMPLEX128
+        s: "SAME"
+        s: "VALID"
       }
     }
   }
-  deprecation {
-    version: 13
-  }
-}
-op {
-  name: "BatchMatrixDiag"
-  input_arg {
-    name: "diagonal"
-    type_attr: "T"
-  }
-  output_arg {
-    name: "output"
-    type_attr: "T"
+  attr {
+    name: "data_format"
+    type: "string"
+    default_value {
+      s: "NHWC"
+    }
+    allowed_values {
+      list {
+        s: "NHWC"
+        s: "NCHW"
+      }
+    }
   }
   attr {
     name: "T"
     type: "type"
-  }
-  deprecation {
-    version: 14
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_HALF
+      }
+    }
   }
 }
 op {
-  name: "BatchMatrixDiagPart"
+  name: "AvgPool"
   input_arg {
-    name: "input"
+    name: "value"
     type_attr: "T"
   }
   output_arg {
-    name: "diagonal"
+    name: "output"
     type_attr: "T"
   }
   attr {
-    name: "T"
-    type: "type"
-  }
-  deprecation {
-    version: 14
+    name: "ksize"
+    type: "list(int)"
+    has_minimum: true
+    minimum: 4
   }
-}
-op {
-  name: "BatchMatrixInverse"
-  input_arg {
-    name: "input"
-    type_attr: "T"
+  attr {
+    name: "strides"
+    type: "list(int)"
+    has_minimum: true
+    minimum: 4
   }
-  output_arg {
-    name: "output"
-    type_attr: "T"
+  attr {
+    name: "padding"
+    type: "string"
+    allowed_values {
+      list {
+        s: "SAME"
+        s: "VALID"
+      }
+    }
   }
   attr {
-    name: "adjoint"
-    type: "bool"
+    name: "data_format"
+    type: "string"
     default_value {
-      b: false
+      s: "NHWC"
+    }
+    allowed_values {
+      list {
+        s: "NHWC"
+        s: "NCHW"
+      }
     }
   }
   attr {
@@ -3402,119 +3717,60 @@ op {
     type: "type"
     allowed_values {
       list {
-        type: DT_DOUBLE
+        type: DT_HALF
         type: DT_FLOAT
+        type: DT_DOUBLE
       }
     }
   }
-  deprecation {
-    version: 13
-  }
 }
 op {
-  name: "BatchMatrixSetDiag"
+  name: "AvgPool3D"
   input_arg {
     name: "input"
     type_attr: "T"
   }
-  input_arg {
-    name: "diagonal"
-    type_attr: "T"
-  }
   output_arg {
     name: "output"
     type_attr: "T"
   }
   attr {
-    name: "T"
-    type: "type"
-  }
-  deprecation {
-    version: 14
-  }
-}
-op {
-  name: "BatchMatrixSolve"
-  input_arg {
-    name: "matrix"
-    type_attr: "T"
-  }
-  input_arg {
-    name: "rhs"
-    type_attr: "T"
-  }
-  output_arg {
-    name: "output"
-    type_attr: "T"
+    name: "ksize"
+    type: "list(int)"
+    has_minimum: true
+    minimum: 5
   }
   attr {
-    name: "adjoint"
-    type: "bool"
-    default_value {
-      b: false
-    }
+    name: "strides"
+    type: "list(int)"
+    has_minimum: true
+    minimum: 5
   }
   attr {
-    name: "T"
-    type: "type"
+    name: "padding"
+    type: "string"
     allowed_values {
       list {
-        type: DT_DOUBLE
-        type: DT_FLOAT
+        s: "SAME"
+        s: "VALID"
       }
     }
   }
-  deprecation {
-    version: 13
-  }
-}
-op {
-  name: "BatchMatrixSolveLs"
-  input_arg {
-    name: "matrix"
-    type_attr: "T"
-  }
-  input_arg {
-    name: "rhs"
-    type_attr: "T"
-  }
-  input_arg {
-    name: "l2_regularizer"
-    type: DT_DOUBLE
-  }
-  output_arg {
-    name: "output"
-    type_attr: "T"
-  }
   attr {
     name: "T"
     type: "type"
     allowed_values {
       list {
-        type: DT_DOUBLE
         type: DT_FLOAT
+        type: DT_DOUBLE
       }
     }
   }
-  attr {
-    name: "fast"
-    type: "bool"
-    default_value {
-      b: true
-    }
-  }
-  deprecation {
-    version: 13
-  }
 }
 op {
-  name: "BatchMatrixTriangularSolve"
-  input_arg {
-    name: "matrix"
-    type_attr: "T"
-  }
+  name: "AvgPool3D"
   input_arg {
-    name: "rhs"
+    name: "input"
     type_attr: "T"
   }
   output_arg {
@@ -3522,17 +3778,38 @@ op {
     type_attr: "T"
   }
   attr {
-    name: "lower"
-    type: "bool"
-    default_value {
-      b: true
+    name: "ksize"
+    type: "list(int)"
+    has_minimum: true
+    minimum: 5
+  }
+  attr {
+    name: "strides"
+    type: "list(int)"
+    has_minimum: true
+    minimum: 5
+  }
+  attr {
+    name: "padding"
+    type: "string"
+    allowed_values {
+      list {
+        s: "SAME"
+        s: "VALID"
+      }
     }
   }
   attr {
-    name: "adjoint"
-    type: "bool"
+    name: "data_format"
+    type: "string"
     default_value {
-      b: false
+      s: "NDHWC"
+    }
+    allowed_values {
+      list {
+        s: "NDHWC"
+        s: "NCDHW"
+      }
     }
   }
   attr {
@@ -3540,40 +3817,47 @@ op {
     type: "type"
     allowed_values {
       list {
-        type: DT_DOUBLE
         type: DT_FLOAT
+        type: DT_DOUBLE
       }
     }
   }
-  deprecation {
-    version: 13
-  }
 }
 op {
-  name: "BatchNormWithGlobalNormalization"
+  name: "AvgPool3DGrad"
   input_arg {
-    name: "t"
-    type_attr: "T"
+    name: "orig_input_shape"
+    type: DT_INT32
   }
   input_arg {
-    name: "m"
+    name: "grad"
     type_attr: "T"
   }
-  input_arg {
-    name: "v"
+  output_arg {
+    name: "output"
     type_attr: "T"
   }
-  input_arg {
-    name: "beta"
-    type_attr: "T"
+  attr {
+    name: "ksize"
+    type: "list(int)"
+    has_minimum: true
+    minimum: 5
   }
-  input_arg {
-    name: "gamma"
-    type_attr: "T"
+  attr {
+    name: "strides"
+    type: "list(int)"
+    has_minimum: true
+    minimum: 5
   }
-  output_arg {
-    name: "result"
-    type_attr: "T"
+  attr {
+    name: "padding"
+    type: "string"
+    allowed_values {
+      list {
+        s: "SAME"
+        s: "VALID"
+      }
+    }
   }
   attr {
     name: "T"
@@ -3582,74 +3866,58 @@ op {
       list {
         type: DT_FLOAT
         type: DT_DOUBLE
-        type: DT_INT64
-        type: DT_INT32
-        type: DT_UINT8
-        type: DT_UINT16
-        type: DT_INT16
-        type: DT_INT8
-        type: DT_COMPLEX64
-        type: DT_COMPLEX128
-        type: DT_QINT8
-        type: DT_QUINT8
-        type: DT_QINT32
-        type: DT_HALF
       }
     }
   }
-  attr {
-    name: "variance_epsilon"
-    type: "float"
-  }
-  attr {
-    name: "scale_after_normalization"
-    type: "bool"
-  }
-  deprecation {
-    version: 9
-  }
 }
 op {
-  name: "BatchNormWithGlobalNormalizationGrad"
-  input_arg {
-    name: "t"
-    type_attr: "T"
-  }
-  input_arg {
-    name: "m"
-    type_attr: "T"
-  }
-  input_arg {
-    name: "v"
-    type_attr: "T"
-  }
+  name: "AvgPool3DGrad"
   input_arg {
-    name: "gamma"
-    type_attr: "T"
+    name: "orig_input_shape"
+    type: DT_INT32
   }
   input_arg {
-    name: "backprop"
+    name: "grad"
     type_attr: "T"
   }
   output_arg {
-    name: "dx"
+    name: "output"
     type_attr: "T"
   }
-  output_arg {
-    name: "dm"
-    type_attr: "T"
+  attr {
+    name: "ksize"
+    type: "list(int)"
+    has_minimum: true
+    minimum: 5
   }
-  output_arg {
-    name: "dv"
-    type_attr: "T"
+  attr {
+    name: "strides"
+    type: "list(int)"
+    has_minimum: true
+    minimum: 5
   }
-  output_arg {
-    name: "db"
-    type_attr: "T"
+  attr {
+    name: "padding"
+    type: "string"
+    allowed_values {
+      list {
+        s: "SAME"
+        s: "VALID"
+      }
+    }
   }
-  output_arg {
-    name: "dg"
-    type_attr: "T"
+  attr {
+    name: "data_format"
+    type: "string"
+    default_value {
+      s: "NDHWC"
+    }
+    allowed_values {
+      list {
+        s: "NDHWC"
+        s: "NCDHW"
+      }
+    }
   }
   attr {
     name: "T"
@@ -3658,37 +3926,18 @@ op {
       list {
         type: DT_FLOAT
         type: DT_DOUBLE
-        type: DT_INT64
-        type: DT_INT32
-        type: DT_UINT8
-        type: DT_UINT16
-        type: DT_INT16
-        type: DT_INT8
-        type: DT_COMPLEX64
-        type: DT_COMPLEX128
-        type: DT_QINT8
-        type: DT_QUINT8
-        type: DT_QINT32
-        type: DT_HALF
       }
     }
   }
-  attr {
-    name: "variance_epsilon"
-    type: "float"
-  }
-  attr {
-    name: "scale_after_normalization"
-    type: "bool"
-  }
-  deprecation {
-    version: 9
-  }
 }
 op {
-  name: "BatchSelfAdjointEig"
+  name: "AvgPoolGrad"
   input_arg {
-    name: "input"
+    name: "orig_input_shape"
+    type: DT_INT32
+  }
+  input_arg {
+    name: "grad"
     type_attr: "T"
   }
   output_arg {
@@ -3696,38 +3945,38 @@ op {
     type_attr: "T"
   }
   attr {
-    name: "T"
-    type: "type"
+    name: "ksize"
+    type: "list(int)"
+    has_minimum: true
+    minimum: 4
+  }
+  attr {
+    name: "strides"
+    type: "list(int)"
+    has_minimum: true
+    minimum: 4
+  }
+  attr {
+    name: "padding"
+    type: "string"
     allowed_values {
       list {
-        type: DT_DOUBLE
-        type: DT_FLOAT
+        s: "SAME"
+        s: "VALID"
       }
     }
   }
-  deprecation {
-    version: 11
-  }
-}
-op {
-  name: "BatchSelfAdjointEigV2"
-  input_arg {
-    name: "input"
-    type_attr: "T"
-  }
-  output_arg {
-    name: "e"
-    type_attr: "T"
-  }
-  output_arg {
-    name: "v"
-    type_attr: "T"
-  }
   attr {
-    name: "compute_v"
-    type: "bool"
+    name: "data_format"
+    type: "string"
     default_value {
-      b: true
+      s: "NHWC"
+    }
+    allowed_values {
+      list {
+        s: "NHWC"
+        s: "NCHW"
+      }
     }
   }
   attr {
@@ -3735,45 +3984,60 @@ op {
     type: "type"
     allowed_values {
       list {
-        type: DT_DOUBLE
         type: DT_FLOAT
+        type: DT_HALF
+        type: DT_DOUBLE
       }
     }
   }
-  deprecation {
-    version: 13
-  }
 }
 op {
-  name: "BatchSvd"
+  name: "AvgPoolGrad"
   input_arg {
-    name: "input"
-    type_attr: "T"
+    name: "orig_input_shape"
+    type: DT_INT32
   }
-  output_arg {
-    name: "s"
+  input_arg {
+    name: "grad"
     type_attr: "T"
   }
   output_arg {
-    name: "u"
+    name: "output"
     type_attr: "T"
   }
-  output_arg {
-    name: "v"
-    type_attr: "T"
+  attr {
+    name: "ksize"
+    type: "list(int)"
+    has_minimum: true
+    minimum: 4
   }
   attr {
-    name: "compute_uv"
-    type: "bool"
-    default_value {
-      b: true
+    name: "strides"
+    type: "list(int)"
+    has_minimum: true
+    minimum: 4
+  }
+  attr {
+    name: "padding"
+    type: "string"
+    allowed_values {
+      list {
+        s: "SAME"
+        s: "VALID"
+      }
     }
   }
   attr {
-    name: "full_matrices"
-    type: "bool"
+    name: "data_format"
+    type: "string"
     default_value {
-      b: false
+      s: "NHWC"
+    }
+    allowed_values {
+      list {
+        s: "NHWC"
+        s: "NCHW"
+      }
     }
   }
   attr {
@@ -3781,71 +4045,159 @@ op {
     type: "type"
     allowed_values {
       list {
-        type: DT_DOUBLE
         type: DT_FLOAT
-        type: DT_COMPLEX64
-        type: DT_COMPLEX128
+        type: DT_DOUBLE
+        type: DT_HALF
       }
     }
   }
-  deprecation {
-    version: 13
-  }
 }
 op {
-  name: "BatchToSpace"
+  name: "AvgPoolGrad"
   input_arg {
-    name: "input"
-    type_attr: "T"
+    name: "orig_input_shape"
+    type: DT_INT32
   }
   input_arg {
-    name: "crops"
-    type_attr: "Tidx"
+    name: "grad"
+    type_attr: "T"
   }
   output_arg {
     name: "output"
     type_attr: "T"
   }
   attr {
-    name: "T"
-    type: "type"
-  }
-  attr {
-    name: "block_size"
-    type: "int"
+    name: "ksize"
+    type: "list(int)"
     has_minimum: true
-    minimum: 2
+    minimum: 4
   }
   attr {
-    name: "Tidx"
-    type: "type"
+    name: "strides"
+    type: "list(int)"
+    has_minimum: true
+    minimum: 4
+  }
+  attr {
+    name: "padding"
+    type: "string"
+    allowed_values {
+      list {
+        s: "SAME"
+        s: "VALID"
+      }
+    }
+  }
+  attr {
+    name: "data_format"
+    type: "string"
     default_value {
-      type: DT_INT32
+      s: "NHWC"
     }
     allowed_values {
       list {
-        type: DT_INT32
-        type: DT_INT64
+        s: "NHWC"
+        s: "NCHW"
+      }
+    }
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_HALF
+        type: DT_FLOAT
+        type: DT_DOUBLE
       }
     }
   }
 }
 op {
-  name: "BatchToSpaceND"
-  input_arg {
-    name: "input"
-    type_attr: "T"
+  name: "Barrier"
+  output_arg {
+    name: "handle"
+    type: DT_STRING
+    is_ref: true
+  }
+  attr {
+    name: "component_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "shapes"
+    type: "list(shape)"
+    default_value {
+      list {
+      }
+    }
+    has_minimum: true
+  }
+  attr {
+    name: "capacity"
+    type: "int"
+    default_value {
+      i: -1
+    }
+  }
+  attr {
+    name: "container"
+    type: "string"
+    default_value {
+      s: ""
+    }
+  }
+  attr {
+    name: "shared_name"
+    type: "string"
+    default_value {
+      s: ""
+    }
   }
+  is_stateful: true
+}
+op {
+  name: "BarrierClose"
   input_arg {
-    name: "block_shape"
-    type_attr: "Tblock_shape"
+    name: "handle"
+    type: DT_STRING
+    is_ref: true
+  }
+  attr {
+    name: "cancel_pending_enqueues"
+    type: "bool"
+    default_value {
+      b: false
+    }
   }
+}
+op {
+  name: "BarrierIncompleteSize"
   input_arg {
-    name: "crops"
-    type_attr: "Tcrops"
+    name: "handle"
+    type: DT_STRING
+    is_ref: true
   }
   output_arg {
-    name: "output"
+    name: "size"
+    type: DT_INT32
+  }
+}
+op {
+  name: "BarrierInsertMany"
+  input_arg {
+    name: "handle"
+    type: DT_STRING
+    is_ref: true
+  }
+  input_arg {
+    name: "keys"
+    type: DT_STRING
+  }
+  input_arg {
+    name: "values"
     type_attr: "T"
   }
   attr {
@@ -3853,48 +4205,81 @@ op {
     type: "type"
   }
   attr {
-    name: "Tblock_shape"
-    type: "type"
+    name: "component_index"
+    type: "int"
+  }
+}
+op {
+  name: "BarrierReadySize"
+  input_arg {
+    name: "handle"
+    type: DT_STRING
+    is_ref: true
+  }
+  output_arg {
+    name: "size"
+    type: DT_INT32
+  }
+}
+op {
+  name: "BarrierTakeMany"
+  input_arg {
+    name: "handle"
+    type: DT_STRING
+    is_ref: true
+  }
+  input_arg {
+    name: "num_elements"
+    type: DT_INT32
+  }
+  output_arg {
+    name: "indices"
+    type: DT_INT64
+  }
+  output_arg {
+    name: "keys"
+    type: DT_STRING
+  }
+  output_arg {
+    name: "values"
+    type_list_attr: "component_types"
+  }
+  attr {
+    name: "component_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "allow_small_batch"
+    type: "bool"
     default_value {
-      type: DT_INT32
-    }
-    allowed_values {
-      list {
-        type: DT_INT32
-        type: DT_INT64
-      }
+      b: false
     }
   }
   attr {
-    name: "Tcrops"
-    type: "type"
+    name: "wait_for_incomplete"
+    type: "bool"
     default_value {
-      type: DT_INT32
+      b: false
     }
-    allowed_values {
-      list {
-        type: DT_INT32
-        type: DT_INT64
-      }
+  }
+  attr {
+    name: "timeout_ms"
+    type: "int"
+    default_value {
+      i: -1
     }
   }
 }
 op {
-  name: "Betainc"
-  input_arg {
-    name: "a"
-    type_attr: "T"
-  }
-  input_arg {
-    name: "b"
-    type_attr: "T"
-  }
+  name: "BatchCholesky"
   input_arg {
-    name: "x"
+    name: "input"
     type_attr: "T"
   }
   output_arg {
-    name: "z"
+    name: "output"
     type_attr: "T"
   }
   attr {
@@ -3902,20 +4287,23 @@ op {
     type: "type"
     allowed_values {
       list {
-        type: DT_FLOAT
         type: DT_DOUBLE
+        type: DT_FLOAT
       }
     }
   }
+  deprecation {
+    version: 13
+  }
 }
 op {
-  name: "BiasAdd"
+  name: "BatchCholeskyGrad"
   input_arg {
-    name: "value"
+    name: "l"
     type_attr: "T"
   }
   input_arg {
-    name: "bias"
+    name: "grad"
     type_attr: "T"
   }
   output_arg {
@@ -3929,39 +4317,160 @@ op {
       list {
         type: DT_FLOAT
         type: DT_DOUBLE
-        type: DT_INT64
-        type: DT_INT32
-        type: DT_UINT8
-        type: DT_UINT16
-        type: DT_INT16
-        type: DT_INT8
-        type: DT_COMPLEX64
-        type: DT_COMPLEX128
-        type: DT_QINT8
-        type: DT_QUINT8
-        type: DT_QINT32
-        type: DT_HALF
       }
     }
   }
+  deprecation {
+    version: 13
+  }
+}
+op {
+  name: "BatchDataset"
+  input_arg {
+    name: "input_dataset"
+    type: DT_VARIANT
+  }
+  input_arg {
+    name: "batch_size"
+    type: DT_INT64
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
   attr {
-    name: "data_format"
-    type: "string"
-    default_value {
-      s: "NHWC"
-    }
-    allowed_values {
-      list {
-        s: "NHWC"
-        s: "NCHW"
-      }
-    }
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
   }
+  is_stateful: true
 }
 op {
-  name: "BiasAddGrad"
+  name: "BatchDataset"
   input_arg {
-    name: "out_backprop"
+    name: "input_dataset"
+    type: DT_VARIANT
+  }
+  input_arg {
+    name: "batch_size"
+    type: DT_INT64
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
+  }
+}
+op {
+  name: "BatchFFT"
+  input_arg {
+    name: "input"
+    type: DT_COMPLEX64
+  }
+  output_arg {
+    name: "output"
+    type: DT_COMPLEX64
+  }
+  deprecation {
+    version: 15
+  }
+}
+op {
+  name: "BatchFFT2D"
+  input_arg {
+    name: "input"
+    type: DT_COMPLEX64
+  }
+  output_arg {
+    name: "output"
+    type: DT_COMPLEX64
+  }
+  deprecation {
+    version: 15
+  }
+}
+op {
+  name: "BatchFFT3D"
+  input_arg {
+    name: "input"
+    type: DT_COMPLEX64
+  }
+  output_arg {
+    name: "output"
+    type: DT_COMPLEX64
+  }
+  deprecation {
+    version: 15
+  }
+}
+op {
+  name: "BatchIFFT"
+  input_arg {
+    name: "input"
+    type: DT_COMPLEX64
+  }
+  output_arg {
+    name: "output"
+    type: DT_COMPLEX64
+  }
+  deprecation {
+    version: 15
+  }
+}
+op {
+  name: "BatchIFFT2D"
+  input_arg {
+    name: "input"
+    type: DT_COMPLEX64
+  }
+  output_arg {
+    name: "output"
+    type: DT_COMPLEX64
+  }
+  deprecation {
+    version: 15
+  }
+}
+op {
+  name: "BatchIFFT3D"
+  input_arg {
+    name: "input"
+    type: DT_COMPLEX64
+  }
+  output_arg {
+    name: "output"
+    type: DT_COMPLEX64
+  }
+  deprecation {
+    version: 15
+  }
+}
+op {
+  name: "BatchMatMul"
+  input_arg {
+    name: "x"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "y"
     type_attr: "T"
   }
   output_arg {
@@ -3973,90 +4482,64 @@ op {
     type: "type"
     allowed_values {
       list {
+        type: DT_HALF
         type: DT_FLOAT
         type: DT_DOUBLE
-        type: DT_INT64
         type: DT_INT32
-        type: DT_UINT8
-        type: DT_UINT16
-        type: DT_INT16
-        type: DT_INT8
         type: DT_COMPLEX64
         type: DT_COMPLEX128
-        type: DT_QINT8
-        type: DT_QUINT8
-        type: DT_QINT32
-        type: DT_HALF
       }
     }
   }
   attr {
-    name: "data_format"
-    type: "string"
+    name: "adj_x"
+    type: "bool"
     default_value {
-      s: "NHWC"
+      b: false
     }
-    allowed_values {
-      list {
-        s: "NHWC"
-        s: "NCHW"
-      }
+  }
+  attr {
+    name: "adj_y"
+    type: "bool"
+    default_value {
+      b: false
     }
   }
 }
 op {
-  name: "BiasAddV1"
+  name: "BatchMatrixBandPart"
   input_arg {
-    name: "value"
+    name: "input"
     type_attr: "T"
   }
   input_arg {
-    name: "bias"
-    type_attr: "T"
+    name: "num_lower"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "num_upper"
+    type: DT_INT64
   }
   output_arg {
-    name: "output"
+    name: "band"
     type_attr: "T"
   }
   attr {
     name: "T"
     type: "type"
-    allowed_values {
-      list {
-        type: DT_FLOAT
-        type: DT_DOUBLE
-        type: DT_INT64
-        type: DT_INT32
-        type: DT_UINT8
-        type: DT_UINT16
-        type: DT_INT16
-        type: DT_INT8
-        type: DT_COMPLEX64
-        type: DT_COMPLEX128
-        type: DT_QINT8
-        type: DT_QUINT8
-        type: DT_QINT32
-        type: DT_HALF
-      }
-    }
+  }
+  deprecation {
+    version: 14
   }
 }
 op {
-  name: "Bincount"
-  input_arg {
-    name: "arr"
-    type: DT_INT32
-  }
-  input_arg {
-    name: "size"
-    type: DT_INT32
-  }
+  name: "BatchMatrixDeterminant"
   input_arg {
-    name: "weights"
+    name: "input"
     type_attr: "T"
   }
   output_arg {
-    name: "bins"
+    name: "output"
     type_attr: "T"
   }
   attr {
@@ -4064,23 +4547,24 @@ op {
     type: "type"
     allowed_values {
       list {
-        type: DT_INT32
-        type: DT_INT64
         type: DT_FLOAT
         type: DT_DOUBLE
       }
     }
   }
+  deprecation {
+    version: 13
+  }
 }
 op {
-  name: "Bitcast"
+  name: "BatchMatrixDeterminant"
   input_arg {
     name: "input"
     type_attr: "T"
   }
   output_arg {
     name: "output"
-    type_attr: "type"
+    type_attr: "T"
   }
   attr {
     name: "T"
@@ -4089,175 +4573,155 @@ op {
       list {
         type: DT_FLOAT
         type: DT_DOUBLE
-        type: DT_INT64
-        type: DT_INT32
-        type: DT_UINT8
-        type: DT_UINT16
-        type: DT_INT16
-        type: DT_INT8
         type: DT_COMPLEX64
         type: DT_COMPLEX128
-        type: DT_QINT8
-        type: DT_QUINT8
-        type: DT_QINT32
-        type: DT_HALF
       }
     }
   }
+  deprecation {
+    version: 13
+  }
+}
+op {
+  name: "BatchMatrixDiag"
+  input_arg {
+    name: "diagonal"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
   attr {
-    name: "type"
+    name: "T"
     type: "type"
-    allowed_values {
-      list {
-        type: DT_FLOAT
-        type: DT_DOUBLE
-        type: DT_INT64
-        type: DT_INT32
-        type: DT_UINT8
-        type: DT_UINT16
-        type: DT_INT16
-        type: DT_INT8
-        type: DT_COMPLEX64
-        type: DT_COMPLEX128
-        type: DT_QINT8
-        type: DT_QUINT8
-        type: DT_QINT32
-        type: DT_HALF
-      }
-    }
+  }
+  deprecation {
+    version: 14
   }
 }
 op {
-  name: "Bitcast"
+  name: "BatchMatrixDiagPart"
   input_arg {
     name: "input"
     type_attr: "T"
   }
   output_arg {
-    name: "output"
-    type_attr: "type"
+    name: "diagonal"
+    type_attr: "T"
   }
   attr {
     name: "T"
     type: "type"
-    allowed_values {
-      list {
-        type: DT_FLOAT
-        type: DT_DOUBLE
-        type: DT_INT64
-        type: DT_INT32
-        type: DT_UINT8
-        type: DT_UINT16
-        type: DT_INT8
-        type: DT_INT16
-        type: DT_COMPLEX64
-        type: DT_COMPLEX128
-        type: DT_QINT8
-        type: DT_QUINT8
-        type: DT_QINT16
-        type: DT_QUINT16
-        type: DT_QINT32
-        type: DT_HALF
-      }
+  }
+  deprecation {
+    version: 14
+  }
+}
+op {
+  name: "BatchMatrixInverse"
+  input_arg {
+    name: "input"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  attr {
+    name: "adjoint"
+    type: "bool"
+    default_value {
+      b: false
     }
   }
   attr {
-    name: "type"
+    name: "T"
     type: "type"
     allowed_values {
       list {
-        type: DT_FLOAT
         type: DT_DOUBLE
-        type: DT_INT64
-        type: DT_INT32
-        type: DT_UINT8
-        type: DT_UINT16
-        type: DT_INT8
-        type: DT_INT16
-        type: DT_COMPLEX64
-        type: DT_COMPLEX128
-        type: DT_QINT8
-        type: DT_QUINT8
-        type: DT_QINT16
-        type: DT_QUINT16
-        type: DT_QINT32
-        type: DT_HALF
+        type: DT_FLOAT
       }
     }
   }
+  deprecation {
+    version: 13
+  }
 }
 op {
-  name: "BitwiseAnd"
+  name: "BatchMatrixSetDiag"
   input_arg {
-    name: "x"
+    name: "input"
     type_attr: "T"
   }
   input_arg {
-    name: "y"
+    name: "diagonal"
     type_attr: "T"
   }
   output_arg {
-    name: "z"
+    name: "output"
     type_attr: "T"
   }
   attr {
     name: "T"
     type: "type"
-    allowed_values {
-      list {
-        type: DT_INT8
-        type: DT_INT16
-        type: DT_INT32
-        type: DT_INT64
-        type: DT_UINT8
-        type: DT_UINT16
-      }
-    }
   }
-  is_commutative: true
+  deprecation {
+    version: 14
+  }
 }
 op {
-  name: "BitwiseOr"
+  name: "BatchMatrixSolve"
   input_arg {
-    name: "x"
+    name: "matrix"
     type_attr: "T"
   }
   input_arg {
-    name: "y"
+    name: "rhs"
     type_attr: "T"
   }
   output_arg {
-    name: "z"
+    name: "output"
     type_attr: "T"
   }
+  attr {
+    name: "adjoint"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
   attr {
     name: "T"
     type: "type"
     allowed_values {
       list {
-        type: DT_INT8
-        type: DT_INT16
-        type: DT_INT32
-        type: DT_INT64
-        type: DT_UINT8
-        type: DT_UINT16
+        type: DT_DOUBLE
+        type: DT_FLOAT
       }
     }
   }
-  is_commutative: true
+  deprecation {
+    version: 13
+  }
 }
 op {
-  name: "BitwiseXor"
+  name: "BatchMatrixSolveLs"
   input_arg {
-    name: "x"
+    name: "matrix"
     type_attr: "T"
   }
   input_arg {
-    name: "y"
+    name: "rhs"
     type_attr: "T"
   }
+  input_arg {
+    name: "l2_regularizer"
+    type: DT_DOUBLE
+  }
   output_arg {
-    name: "z"
+    name: "output"
     type_attr: "T"
   }
   attr {
@@ -4265,358 +4729,575 @@ op {
     type: "type"
     allowed_values {
       list {
-        type: DT_INT8
-        type: DT_INT16
-        type: DT_INT32
-        type: DT_INT64
-        type: DT_UINT8
-        type: DT_UINT16
+        type: DT_DOUBLE
+        type: DT_FLOAT
       }
     }
   }
-  is_commutative: true
+  attr {
+    name: "fast"
+    type: "bool"
+    default_value {
+      b: true
+    }
+  }
+  deprecation {
+    version: 13
+  }
 }
 op {
-  name: "BroadcastArgs"
+  name: "BatchMatrixTriangularSolve"
   input_arg {
-    name: "s0"
+    name: "matrix"
     type_attr: "T"
   }
   input_arg {
-    name: "s1"
+    name: "rhs"
     type_attr: "T"
   }
   output_arg {
-    name: "r0"
+    name: "output"
     type_attr: "T"
   }
   attr {
-    name: "T"
-    type: "type"
+    name: "lower"
+    type: "bool"
     default_value {
-      type: DT_INT32
+      b: true
+    }
+  }
+  attr {
+    name: "adjoint"
+    type: "bool"
+    default_value {
+      b: false
     }
+  }
+  attr {
+    name: "T"
+    type: "type"
     allowed_values {
       list {
-        type: DT_INT32
-        type: DT_INT64
+        type: DT_DOUBLE
+        type: DT_FLOAT
       }
     }
   }
+  deprecation {
+    version: 13
+  }
 }
 op {
-  name: "BroadcastGradientArgs"
+  name: "BatchNormWithGlobalNormalization"
   input_arg {
-    name: "s0"
+    name: "t"
     type_attr: "T"
   }
   input_arg {
-    name: "s1"
+    name: "m"
     type_attr: "T"
   }
-  output_arg {
-    name: "r0"
+  input_arg {
+    name: "v"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "beta"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "gamma"
     type_attr: "T"
   }
   output_arg {
-    name: "r1"
+    name: "result"
     type_attr: "T"
   }
   attr {
     name: "T"
     type: "type"
-    default_value {
-      type: DT_INT32
-    }
     allowed_values {
       list {
-        type: DT_INT32
+        type: DT_FLOAT
+        type: DT_DOUBLE
         type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
       }
     }
   }
+  attr {
+    name: "variance_epsilon"
+    type: "float"
+  }
+  attr {
+    name: "scale_after_normalization"
+    type: "bool"
+  }
+  deprecation {
+    version: 9
+  }
 }
 op {
-  name: "Bucketize"
+  name: "BatchNormWithGlobalNormalization"
   input_arg {
-    name: "input"
+    name: "t"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "m"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "v"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "beta"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "gamma"
     type_attr: "T"
   }
   output_arg {
-    name: "output"
-    type: DT_INT32
+    name: "result"
+    type_attr: "T"
   }
   attr {
     name: "T"
     type: "type"
     allowed_values {
       list {
-        type: DT_INT32
-        type: DT_INT64
         type: DT_FLOAT
         type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
   attr {
-    name: "boundaries"
-    type: "list(float)"
+    name: "variance_epsilon"
+    type: "float"
+  }
+  attr {
+    name: "scale_after_normalization"
+    type: "bool"
+  }
+  deprecation {
+    version: 9
   }
 }
 op {
-  name: "CTCBeamSearchDecoder"
+  name: "BatchNormWithGlobalNormalizationGrad"
   input_arg {
-    name: "inputs"
-    type: DT_FLOAT
+    name: "t"
+    type_attr: "T"
   }
   input_arg {
-    name: "sequence_length"
-    type: DT_INT32
+    name: "m"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "v"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "gamma"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "backprop"
+    type_attr: "T"
   }
   output_arg {
-    name: "decoded_indices"
-    type: DT_INT64
-    number_attr: "top_paths"
+    name: "dx"
+    type_attr: "T"
   }
   output_arg {
-    name: "decoded_values"
-    type: DT_INT64
-    number_attr: "top_paths"
+    name: "dm"
+    type_attr: "T"
   }
   output_arg {
-    name: "decoded_shape"
-    type: DT_INT64
-    number_attr: "top_paths"
+    name: "dv"
+    type_attr: "T"
   }
   output_arg {
-    name: "log_probability"
-    type: DT_FLOAT
+    name: "db"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "dg"
+    type_attr: "T"
   }
   attr {
-    name: "beam_width"
-    type: "int"
-    has_minimum: true
-    minimum: 1
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
+      }
+    }
   }
   attr {
-    name: "top_paths"
-    type: "int"
-    has_minimum: true
-    minimum: 1
+    name: "variance_epsilon"
+    type: "float"
   }
   attr {
-    name: "merge_repeated"
+    name: "scale_after_normalization"
     type: "bool"
-    default_value {
-      b: true
-    }
+  }
+  deprecation {
+    version: 9
   }
 }
 op {
-  name: "CTCGreedyDecoder"
+  name: "BatchNormWithGlobalNormalizationGrad"
   input_arg {
-    name: "inputs"
-    type: DT_FLOAT
+    name: "t"
+    type_attr: "T"
   }
   input_arg {
-    name: "sequence_length"
-    type: DT_INT32
+    name: "m"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "v"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "gamma"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "backprop"
+    type_attr: "T"
   }
   output_arg {
-    name: "decoded_indices"
-    type: DT_INT64
+    name: "dx"
+    type_attr: "T"
   }
   output_arg {
-    name: "decoded_values"
-    type: DT_INT64
+    name: "dm"
+    type_attr: "T"
   }
   output_arg {
-    name: "decoded_shape"
-    type: DT_INT64
+    name: "dv"
+    type_attr: "T"
   }
   output_arg {
-    name: "log_probability"
-    type: DT_FLOAT
+    name: "db"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "dg"
+    type_attr: "T"
   }
   attr {
-    name: "merge_repeated"
-    type: "bool"
-    default_value {
-      b: false
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
     }
   }
+  attr {
+    name: "variance_epsilon"
+    type: "float"
+  }
+  attr {
+    name: "scale_after_normalization"
+    type: "bool"
+  }
+  deprecation {
+    version: 9
+  }
 }
 op {
-  name: "CTCLoss"
+  name: "BatchSelfAdjointEig"
   input_arg {
-    name: "inputs"
-    type: DT_FLOAT
+    name: "input"
+    type_attr: "T"
   }
-  input_arg {
-    name: "labels_indices"
-    type: DT_INT64
+  output_arg {
+    name: "output"
+    type_attr: "T"
   }
-  input_arg {
-    name: "labels_values"
-    type: DT_INT32
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_DOUBLE
+        type: DT_FLOAT
+      }
+    }
+  }
+  deprecation {
+    version: 11
   }
+}
+op {
+  name: "BatchSelfAdjointEigV2"
   input_arg {
-    name: "sequence_length"
-    type: DT_INT32
+    name: "input"
+    type_attr: "T"
   }
   output_arg {
-    name: "loss"
-    type: DT_FLOAT
+    name: "e"
+    type_attr: "T"
   }
   output_arg {
-    name: "gradient"
-    type: DT_FLOAT
+    name: "v"
+    type_attr: "T"
   }
   attr {
-    name: "preprocess_collapse_repeated"
+    name: "compute_v"
     type: "bool"
     default_value {
-      b: false
+      b: true
     }
   }
   attr {
-    name: "ctc_merge_repeated"
-    type: "bool"
-    default_value {
-      b: true
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_DOUBLE
+        type: DT_FLOAT
+      }
     }
   }
+  deprecation {
+    version: 13
+  }
 }
 op {
-  name: "CTCLoss"
+  name: "BatchSvd"
   input_arg {
-    name: "inputs"
-    type: DT_FLOAT
-  }
-  input_arg {
-    name: "labels_indices"
-    type: DT_INT64
-  }
-  input_arg {
-    name: "labels_values"
-    type: DT_INT32
+    name: "input"
+    type_attr: "T"
   }
-  input_arg {
-    name: "sequence_length"
-    type: DT_INT32
+  output_arg {
+    name: "s"
+    type_attr: "T"
   }
   output_arg {
-    name: "loss"
-    type: DT_FLOAT
+    name: "u"
+    type_attr: "T"
   }
   output_arg {
-    name: "gradient"
-    type: DT_FLOAT
+    name: "v"
+    type_attr: "T"
   }
   attr {
-    name: "preprocess_collapse_repeated"
+    name: "compute_uv"
     type: "bool"
     default_value {
-      b: false
+      b: true
     }
   }
   attr {
-    name: "ctc_merge_repeated"
+    name: "full_matrices"
     type: "bool"
     default_value {
-      b: true
+      b: false
     }
   }
   attr {
-    name: "ignore_longer_outputs_than_inputs"
-    type: "bool"
-    default_value {
-      b: false
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_DOUBLE
+        type: DT_FLOAT
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+      }
     }
   }
+  deprecation {
+    version: 13
+  }
 }
 op {
-  name: "CacheDataset"
+  name: "BatchToSpace"
   input_arg {
-    name: "input_dataset"
-    type: DT_VARIANT
+    name: "input"
+    type_attr: "T"
   }
   input_arg {
-    name: "filename"
-    type: DT_STRING
+    name: "crops"
+    type_attr: "Tidx"
   }
   output_arg {
-    name: "handle"
-    type: DT_VARIANT
+    name: "output"
+    type_attr: "T"
   }
   attr {
-    name: "output_types"
-    type: "list(type)"
-    has_minimum: true
-    minimum: 1
+    name: "T"
+    type: "type"
   }
   attr {
-    name: "output_shapes"
-    type: "list(shape)"
+    name: "block_size"
+    type: "int"
     has_minimum: true
-    minimum: 1
+    minimum: 2
+  }
+  attr {
+    name: "Tidx"
+    type: "type"
+    default_value {
+      type: DT_INT32
+    }
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
   }
-  is_stateful: true
 }
 op {
-  name: "CacheDataset"
+  name: "BatchToSpaceND"
   input_arg {
-    name: "input_dataset"
-    type: DT_VARIANT
+    name: "input"
+    type_attr: "T"
   }
   input_arg {
-    name: "filename"
-    type: DT_STRING
+    name: "block_shape"
+    type_attr: "Tblock_shape"
+  }
+  input_arg {
+    name: "crops"
+    type_attr: "Tcrops"
   }
   output_arg {
-    name: "handle"
-    type: DT_VARIANT
+    name: "output"
+    type_attr: "T"
   }
   attr {
-    name: "output_types"
-    type: "list(type)"
-    has_minimum: true
-    minimum: 1
+    name: "T"
+    type: "type"
   }
   attr {
-    name: "output_shapes"
-    type: "list(shape)"
-    has_minimum: true
-    minimum: 1
+    name: "Tblock_shape"
+    type: "type"
+    default_value {
+      type: DT_INT32
+    }
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+  attr {
+    name: "Tcrops"
+    type: "type"
+    default_value {
+      type: DT_INT32
+    }
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
   }
 }
 op {
-  name: "Cast"
+  name: "Betainc"
+  input_arg {
+    name: "a"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "b"
+    type_attr: "T"
+  }
   input_arg {
     name: "x"
-    type_attr: "SrcT"
+    type_attr: "T"
   }
   output_arg {
-    name: "y"
-    type_attr: "DstT"
-  }
-  attr {
-    name: "SrcT"
-    type: "type"
+    name: "z"
+    type_attr: "T"
   }
   attr {
-    name: "DstT"
+    name: "T"
     type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+      }
+    }
   }
 }
 op {
-  name: "Ceil"
+  name: "BiasAdd"
   input_arg {
-    name: "x"
+    name: "value"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "bias"
     type_attr: "T"
   }
   output_arg {
-    name: "y"
+    name: "output"
     type_attr: "T"
   }
   attr {
@@ -4624,17 +5305,45 @@ op {
     type: "type"
     allowed_values {
       list {
-        type: DT_HALF
         type: DT_FLOAT
         type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
+      }
+    }
+  }
+  attr {
+    name: "data_format"
+    type: "string"
+    default_value {
+      s: "NHWC"
+    }
+    allowed_values {
+      list {
+        s: "NHWC"
+        s: "NCHW"
       }
     }
   }
 }
 op {
-  name: "CheckNumerics"
+  name: "BiasAdd"
   input_arg {
-    name: "tensor"
+    name: "value"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "bias"
     type_attr: "T"
   }
   output_arg {
@@ -4646,21 +5355,43 @@ op {
     type: "type"
     allowed_values {
       list {
-        type: DT_HALF
         type: DT_FLOAT
         type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
   attr {
-    name: "message"
+    name: "data_format"
     type: "string"
+    default_value {
+      s: "NHWC"
+    }
+    allowed_values {
+      list {
+        s: "NHWC"
+        s: "NCHW"
+      }
+    }
   }
 }
 op {
-  name: "Cholesky"
+  name: "BiasAddGrad"
   input_arg {
-    name: "input"
+    name: "out_backprop"
     type_attr: "T"
   }
   output_arg {
@@ -4672,16 +5403,41 @@ op {
     type: "type"
     allowed_values {
       list {
-        type: DT_DOUBLE
         type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
+      }
+    }
+  }
+  attr {
+    name: "data_format"
+    type: "string"
+    default_value {
+      s: "NHWC"
+    }
+    allowed_values {
+      list {
+        s: "NHWC"
+        s: "NCHW"
       }
     }
   }
 }
 op {
-  name: "Cholesky"
+  name: "BiasAddGrad"
   input_arg {
-    name: "input"
+    name: "out_backprop"
     type_attr: "T"
   }
   output_arg {
@@ -4693,22 +5449,47 @@ op {
     type: "type"
     allowed_values {
       list {
-        type: DT_DOUBLE
         type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
         type: DT_COMPLEX64
         type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+  attr {
+    name: "data_format"
+    type: "string"
+    default_value {
+      s: "NHWC"
+    }
+    allowed_values {
+      list {
+        s: "NHWC"
+        s: "NCHW"
       }
     }
   }
 }
 op {
-  name: "CholeskyGrad"
+  name: "BiasAddV1"
   input_arg {
-    name: "l"
+    name: "value"
     type_attr: "T"
   }
   input_arg {
-    name: "grad"
+    name: "bias"
     type_attr: "T"
   }
   output_arg {
@@ -4722,236 +5503,344 @@ op {
       list {
         type: DT_FLOAT
         type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
       }
     }
   }
 }
 op {
-  name: "CompareAndBitpack"
+  name: "BiasAddV1"
   input_arg {
-    name: "input"
+    name: "value"
     type_attr: "T"
   }
   input_arg {
-    name: "threshold"
+    name: "bias"
     type_attr: "T"
   }
   output_arg {
     name: "output"
-    type: DT_UINT8
+    type_attr: "T"
   }
   attr {
     name: "T"
     type: "type"
     allowed_values {
       list {
-        type: DT_BOOL
-        type: DT_HALF
         type: DT_FLOAT
         type: DT_DOUBLE
-        type: DT_INT8
-        type: DT_INT16
-        type: DT_INT32
         type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
 }
 op {
-  name: "Complex"
+  name: "Bincount"
   input_arg {
-    name: "real"
-    type_attr: "T"
+    name: "arr"
+    type: DT_INT32
   }
   input_arg {
-    name: "imag"
+    name: "size"
+    type: DT_INT32
+  }
+  input_arg {
+    name: "weights"
     type_attr: "T"
   }
   output_arg {
-    name: "out"
-    type_attr: "Tout"
+    name: "bins"
+    type_attr: "T"
   }
   attr {
     name: "T"
     type: "type"
-    default_value {
-      type: DT_FLOAT
-    }
     allowed_values {
       list {
+        type: DT_INT32
+        type: DT_INT64
         type: DT_FLOAT
         type: DT_DOUBLE
       }
     }
   }
+}
+op {
+  name: "Bitcast"
+  input_arg {
+    name: "input"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "type"
+  }
   attr {
-    name: "Tout"
+    name: "T"
     type: "type"
-    default_value {
-      type: DT_COMPLEX64
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
+      }
     }
+  }
+  attr {
+    name: "type"
+    type: "type"
     allowed_values {
       list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
         type: DT_COMPLEX64
         type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
       }
     }
   }
 }
 op {
-  name: "ComplexAbs"
+  name: "Bitcast"
   input_arg {
-    name: "x"
+    name: "input"
     type_attr: "T"
   }
   output_arg {
-    name: "y"
-    type_attr: "Tout"
+    name: "output"
+    type_attr: "type"
   }
   attr {
     name: "T"
     type: "type"
-    default_value {
-      type: DT_COMPLEX64
-    }
     allowed_values {
       list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT8
+        type: DT_INT16
         type: DT_COMPLEX64
         type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT16
+        type: DT_QUINT16
+        type: DT_QINT32
+        type: DT_HALF
       }
     }
   }
   attr {
-    name: "Tout"
+    name: "type"
     type: "type"
-    default_value {
-      type: DT_FLOAT
-    }
     allowed_values {
       list {
         type: DT_FLOAT
         type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT8
+        type: DT_INT16
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT16
+        type: DT_QUINT16
+        type: DT_QINT32
+        type: DT_HALF
       }
     }
   }
 }
 op {
-  name: "ComputeAccidentalHits"
+  name: "BitwiseAnd"
   input_arg {
-    name: "true_classes"
-    type: DT_INT64
+    name: "x"
+    type_attr: "T"
   }
   input_arg {
-    name: "sampled_candidates"
-    type: DT_INT64
-  }
-  output_arg {
-    name: "indices"
-    type: DT_INT32
-  }
-  output_arg {
-    name: "ids"
-    type: DT_INT64
+    name: "y"
+    type_attr: "T"
   }
   output_arg {
-    name: "weights"
-    type: DT_FLOAT
-  }
-  attr {
-    name: "num_true"
-    type: "int"
-  }
-  attr {
-    name: "seed"
-    type: "int"
-    default_value {
-      i: 0
-    }
+    name: "z"
+    type_attr: "T"
   }
   attr {
-    name: "seed2"
-    type: "int"
-    default_value {
-      i: 0
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT8
+        type: DT_INT16
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_UINT8
+        type: DT_UINT16
+      }
     }
   }
+  is_commutative: true
 }
 op {
-  name: "Concat"
+  name: "BitwiseOr"
   input_arg {
-    name: "concat_dim"
-    type: DT_INT32
+    name: "x"
+    type_attr: "T"
   }
   input_arg {
-    name: "values"
+    name: "y"
     type_attr: "T"
-    number_attr: "N"
   }
   output_arg {
-    name: "output"
+    name: "z"
     type_attr: "T"
   }
-  attr {
-    name: "N"
-    type: "int"
-    has_minimum: true
-    minimum: 2
-  }
   attr {
     name: "T"
     type: "type"
+    allowed_values {
+      list {
+        type: DT_INT8
+        type: DT_INT16
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_UINT8
+        type: DT_UINT16
+      }
+    }
   }
+  is_commutative: true
 }
 op {
-  name: "ConcatOffset"
+  name: "BitwiseXor"
   input_arg {
-    name: "concat_dim"
-    type: DT_INT32
+    name: "x"
+    type_attr: "T"
   }
   input_arg {
-    name: "shape"
-    type: DT_INT32
-    number_attr: "N"
+    name: "y"
+    type_attr: "T"
   }
   output_arg {
-    name: "offset"
-    type: DT_INT32
-    number_attr: "N"
+    name: "z"
+    type_attr: "T"
   }
   attr {
-    name: "N"
-    type: "int"
-    has_minimum: true
-    minimum: 2
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT8
+        type: DT_INT16
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_UINT8
+        type: DT_UINT16
+      }
+    }
   }
+  is_commutative: true
 }
 op {
-  name: "ConcatV2"
+  name: "BroadcastArgs"
   input_arg {
-    name: "values"
+    name: "s0"
     type_attr: "T"
-    number_attr: "N"
   }
   input_arg {
-    name: "axis"
-    type_attr: "Tidx"
+    name: "s1"
+    type_attr: "T"
   }
   output_arg {
-    name: "output"
+    name: "r0"
     type_attr: "T"
   }
-  attr {
-    name: "N"
-    type: "int"
-    has_minimum: true
-    minimum: 2
-  }
   attr {
     name: "T"
     type: "type"
+    default_value {
+      type: DT_INT32
+    }
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+}
+op {
+  name: "BroadcastGradientArgs"
+  input_arg {
+    name: "s0"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "s1"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "r0"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "r1"
+    type_attr: "T"
   }
   attr {
-    name: "Tidx"
+    name: "T"
     type: "type"
     default_value {
       type: DT_INT32
@@ -4965,14 +5854,213 @@ op {
   }
 }
 op {
-  name: "ConcatenateDataset"
+  name: "Bucketize"
+  input_arg {
+    name: "input"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "output"
+    type: DT_INT32
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_FLOAT
+        type: DT_DOUBLE
+      }
+    }
+  }
+  attr {
+    name: "boundaries"
+    type: "list(float)"
+  }
+}
+op {
+  name: "CTCBeamSearchDecoder"
+  input_arg {
+    name: "inputs"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "sequence_length"
+    type: DT_INT32
+  }
+  output_arg {
+    name: "decoded_indices"
+    type: DT_INT64
+    number_attr: "top_paths"
+  }
+  output_arg {
+    name: "decoded_values"
+    type: DT_INT64
+    number_attr: "top_paths"
+  }
+  output_arg {
+    name: "decoded_shape"
+    type: DT_INT64
+    number_attr: "top_paths"
+  }
+  output_arg {
+    name: "log_probability"
+    type: DT_FLOAT
+  }
+  attr {
+    name: "beam_width"
+    type: "int"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "top_paths"
+    type: "int"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "merge_repeated"
+    type: "bool"
+    default_value {
+      b: true
+    }
+  }
+}
+op {
+  name: "CTCGreedyDecoder"
+  input_arg {
+    name: "inputs"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "sequence_length"
+    type: DT_INT32
+  }
+  output_arg {
+    name: "decoded_indices"
+    type: DT_INT64
+  }
+  output_arg {
+    name: "decoded_values"
+    type: DT_INT64
+  }
+  output_arg {
+    name: "decoded_shape"
+    type: DT_INT64
+  }
+  output_arg {
+    name: "log_probability"
+    type: DT_FLOAT
+  }
+  attr {
+    name: "merge_repeated"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
+}
+op {
+  name: "CTCLoss"
+  input_arg {
+    name: "inputs"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "labels_indices"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "labels_values"
+    type: DT_INT32
+  }
+  input_arg {
+    name: "sequence_length"
+    type: DT_INT32
+  }
+  output_arg {
+    name: "loss"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "gradient"
+    type: DT_FLOAT
+  }
+  attr {
+    name: "preprocess_collapse_repeated"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
+  attr {
+    name: "ctc_merge_repeated"
+    type: "bool"
+    default_value {
+      b: true
+    }
+  }
+}
+op {
+  name: "CTCLoss"
+  input_arg {
+    name: "inputs"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "labels_indices"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "labels_values"
+    type: DT_INT32
+  }
+  input_arg {
+    name: "sequence_length"
+    type: DT_INT32
+  }
+  output_arg {
+    name: "loss"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "gradient"
+    type: DT_FLOAT
+  }
+  attr {
+    name: "preprocess_collapse_repeated"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
+  attr {
+    name: "ctc_merge_repeated"
+    type: "bool"
+    default_value {
+      b: true
+    }
+  }
+  attr {
+    name: "ignore_longer_outputs_than_inputs"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
+}
+op {
+  name: "CacheDataset"
   input_arg {
     name: "input_dataset"
     type: DT_VARIANT
   }
   input_arg {
-    name: "another_dataset"
-    type: DT_VARIANT
+    name: "filename"
+    type: DT_STRING
   }
   output_arg {
     name: "handle"
@@ -4993,14 +6081,14 @@ op {
   is_stateful: true
 }
 op {
-  name: "ConcatenateDataset"
+  name: "CacheDataset"
   input_arg {
     name: "input_dataset"
     type: DT_VARIANT
   }
   input_arg {
-    name: "another_dataset"
-    type: DT_VARIANT
+    name: "filename"
+    type: DT_STRING
   }
   output_arg {
     name: "handle"
@@ -5020,56 +6108,74 @@ op {
   }
 }
 op {
-  name: "ConditionalAccumulator"
+  name: "Cast"
+  input_arg {
+    name: "x"
+    type_attr: "SrcT"
+  }
   output_arg {
-    name: "handle"
-    type: DT_STRING
-    is_ref: true
+    name: "y"
+    type_attr: "DstT"
   }
   attr {
-    name: "dtype"
+    name: "SrcT"
+    type: "type"
+  }
+  attr {
+    name: "DstT"
+    type: "type"
+  }
+}
+op {
+  name: "Ceil"
+  input_arg {
+    name: "x"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "y"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
     type: "type"
     allowed_values {
       list {
+        type: DT_HALF
         type: DT_FLOAT
         type: DT_DOUBLE
-        type: DT_INT64
-        type: DT_INT32
-        type: DT_UINT8
-        type: DT_UINT16
-        type: DT_INT16
-        type: DT_INT8
-        type: DT_COMPLEX64
-        type: DT_COMPLEX128
-        type: DT_QINT8
-        type: DT_QUINT8
-        type: DT_QINT32
-        type: DT_HALF
       }
     }
   }
-  attr {
-    name: "shape"
-    type: "shape"
+}
+op {
+  name: "CheckNumerics"
+  input_arg {
+    name: "tensor"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
   }
   attr {
-    name: "container"
-    type: "string"
-    default_value {
-      s: ""
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_HALF
+        type: DT_FLOAT
+        type: DT_DOUBLE
+      }
     }
   }
   attr {
-    name: "shared_name"
+    name: "message"
     type: "string"
-    default_value {
-      s: ""
-    }
   }
-  is_stateful: true
 }
 op {
-  name: "Conj"
+  name: "Cholesky"
   input_arg {
     name: "input"
     type_attr: "T"
@@ -5081,19 +6187,16 @@ op {
   attr {
     name: "T"
     type: "type"
-    default_value {
-      type: DT_COMPLEX64
-    }
     allowed_values {
       list {
-        type: DT_COMPLEX64
-        type: DT_COMPLEX128
+        type: DT_DOUBLE
+        type: DT_FLOAT
       }
     }
   }
 }
 op {
-  name: "Conj"
+  name: "Cholesky"
   input_arg {
     name: "input"
     type_attr: "T"
@@ -5105,130 +6208,595 @@ op {
   attr {
     name: "T"
     type: "type"
-    default_value {
-      type: DT_COMPLEX64
-    }
     allowed_values {
       list {
+        type: DT_DOUBLE
+        type: DT_FLOAT
         type: DT_COMPLEX64
         type: DT_COMPLEX128
-        type: DT_VARIANT
       }
     }
   }
 }
 op {
-  name: "Const"
+  name: "CholeskyGrad"
+  input_arg {
+    name: "l"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "grad"
+    type_attr: "T"
+  }
   output_arg {
     name: "output"
-    type_attr: "dtype"
-  }
-  attr {
-    name: "value"
-    type: "tensor"
+    type_attr: "T"
   }
   attr {
-    name: "dtype"
+    name: "T"
     type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+      }
+    }
   }
 }
 op {
-  name: "ControlTrigger"
-}
-op {
-  name: "Conv2D"
+  name: "CompareAndBitpack"
   input_arg {
     name: "input"
     type_attr: "T"
   }
   input_arg {
-    name: "filter"
+    name: "threshold"
     type_attr: "T"
   }
   output_arg {
     name: "output"
-    type_attr: "T"
+    type: DT_UINT8
   }
   attr {
     name: "T"
     type: "type"
     allowed_values {
       list {
+        type: DT_BOOL
         type: DT_HALF
         type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT8
+        type: DT_INT16
+        type: DT_INT32
+        type: DT_INT64
       }
     }
   }
-  attr {
-    name: "strides"
-    type: "list(int)"
+}
+op {
+  name: "Complex"
+  input_arg {
+    name: "real"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "imag"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "out"
+    type_attr: "Tout"
   }
   attr {
-    name: "use_cudnn_on_gpu"
-    type: "bool"
+    name: "T"
+    type: "type"
     default_value {
-      b: true
+      type: DT_FLOAT
     }
-  }
-  attr {
-    name: "padding"
-    type: "string"
     allowed_values {
       list {
-        s: "SAME"
-        s: "VALID"
+        type: DT_FLOAT
+        type: DT_DOUBLE
       }
     }
   }
   attr {
-    name: "data_format"
-    type: "string"
+    name: "Tout"
+    type: "type"
     default_value {
-      s: "NHWC"
+      type: DT_COMPLEX64
     }
     allowed_values {
       list {
-        s: "NHWC"
-        s: "NCHW"
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
       }
     }
   }
 }
 op {
-  name: "Conv2DBackpropFilter"
-  input_arg {
-    name: "input"
-    type_attr: "T"
-  }
-  input_arg {
-    name: "filter_sizes"
-    type: DT_INT32
-  }
+  name: "ComplexAbs"
   input_arg {
-    name: "out_backprop"
+    name: "x"
     type_attr: "T"
   }
   output_arg {
-    name: "output"
-    type_attr: "T"
+    name: "y"
+    type_attr: "Tout"
   }
   attr {
     name: "T"
     type: "type"
+    default_value {
+      type: DT_COMPLEX64
+    }
     allowed_values {
       list {
-        type: DT_HALF
-        type: DT_FLOAT
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
       }
     }
   }
   attr {
-    name: "strides"
-    type: "list(int)"
-  }
-  attr {
-    name: "use_cudnn_on_gpu"
-    type: "bool"
+    name: "Tout"
+    type: "type"
+    default_value {
+      type: DT_FLOAT
+    }
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+      }
+    }
+  }
+}
+op {
+  name: "ComputeAccidentalHits"
+  input_arg {
+    name: "true_classes"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "sampled_candidates"
+    type: DT_INT64
+  }
+  output_arg {
+    name: "indices"
+    type: DT_INT32
+  }
+  output_arg {
+    name: "ids"
+    type: DT_INT64
+  }
+  output_arg {
+    name: "weights"
+    type: DT_FLOAT
+  }
+  attr {
+    name: "num_true"
+    type: "int"
+  }
+  attr {
+    name: "seed"
+    type: "int"
+    default_value {
+      i: 0
+    }
+  }
+  attr {
+    name: "seed2"
+    type: "int"
+    default_value {
+      i: 0
+    }
+  }
+}
+op {
+  name: "Concat"
+  input_arg {
+    name: "concat_dim"
+    type: DT_INT32
+  }
+  input_arg {
+    name: "values"
+    type_attr: "T"
+    number_attr: "N"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  attr {
+    name: "N"
+    type: "int"
+    has_minimum: true
+    minimum: 2
+  }
+  attr {
+    name: "T"
+    type: "type"
+  }
+}
+op {
+  name: "ConcatOffset"
+  input_arg {
+    name: "concat_dim"
+    type: DT_INT32
+  }
+  input_arg {
+    name: "shape"
+    type: DT_INT32
+    number_attr: "N"
+  }
+  output_arg {
+    name: "offset"
+    type: DT_INT32
+    number_attr: "N"
+  }
+  attr {
+    name: "N"
+    type: "int"
+    has_minimum: true
+    minimum: 2
+  }
+}
+op {
+  name: "ConcatV2"
+  input_arg {
+    name: "values"
+    type_attr: "T"
+    number_attr: "N"
+  }
+  input_arg {
+    name: "axis"
+    type_attr: "Tidx"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  attr {
+    name: "N"
+    type: "int"
+    has_minimum: true
+    minimum: 2
+  }
+  attr {
+    name: "T"
+    type: "type"
+  }
+  attr {
+    name: "Tidx"
+    type: "type"
+    default_value {
+      type: DT_INT32
+    }
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+}
+op {
+  name: "ConcatenateDataset"
+  input_arg {
+    name: "input_dataset"
+    type: DT_VARIANT
+  }
+  input_arg {
+    name: "another_dataset"
+    type: DT_VARIANT
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
+  }
+  is_stateful: true
+}
+op {
+  name: "ConcatenateDataset"
+  input_arg {
+    name: "input_dataset"
+    type: DT_VARIANT
+  }
+  input_arg {
+    name: "another_dataset"
+    type: DT_VARIANT
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
+  }
+}
+op {
+  name: "ConditionalAccumulator"
+  output_arg {
+    name: "handle"
+    type: DT_STRING
+    is_ref: true
+  }
+  attr {
+    name: "dtype"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
+      }
+    }
+  }
+  attr {
+    name: "shape"
+    type: "shape"
+  }
+  attr {
+    name: "container"
+    type: "string"
+    default_value {
+      s: ""
+    }
+  }
+  attr {
+    name: "shared_name"
+    type: "string"
+    default_value {
+      s: ""
+    }
+  }
+  is_stateful: true
+}
+op {
+  name: "ConditionalAccumulator"
+  output_arg {
+    name: "handle"
+    type: DT_STRING
+    is_ref: true
+  }
+  attr {
+    name: "dtype"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+  attr {
+    name: "shape"
+    type: "shape"
+  }
+  attr {
+    name: "container"
+    type: "string"
+    default_value {
+      s: ""
+    }
+  }
+  attr {
+    name: "shared_name"
+    type: "string"
+    default_value {
+      s: ""
+    }
+  }
+  is_stateful: true
+}
+op {
+  name: "Conj"
+  input_arg {
+    name: "input"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    default_value {
+      type: DT_COMPLEX64
+    }
+    allowed_values {
+      list {
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+      }
+    }
+  }
+}
+op {
+  name: "Conj"
+  input_arg {
+    name: "input"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    default_value {
+      type: DT_COMPLEX64
+    }
+    allowed_values {
+      list {
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_VARIANT
+      }
+    }
+  }
+}
+op {
+  name: "Const"
+  output_arg {
+    name: "output"
+    type_attr: "dtype"
+  }
+  attr {
+    name: "value"
+    type: "tensor"
+  }
+  attr {
+    name: "dtype"
+    type: "type"
+  }
+}
+op {
+  name: "ControlTrigger"
+}
+op {
+  name: "Conv2D"
+  input_arg {
+    name: "input"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "filter"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_HALF
+        type: DT_FLOAT
+      }
+    }
+  }
+  attr {
+    name: "strides"
+    type: "list(int)"
+  }
+  attr {
+    name: "use_cudnn_on_gpu"
+    type: "bool"
+    default_value {
+      b: true
+    }
+  }
+  attr {
+    name: "padding"
+    type: "string"
+    allowed_values {
+      list {
+        s: "SAME"
+        s: "VALID"
+      }
+    }
+  }
+  attr {
+    name: "data_format"
+    type: "string"
+    default_value {
+      s: "NHWC"
+    }
+    allowed_values {
+      list {
+        s: "NHWC"
+        s: "NCHW"
+      }
+    }
+  }
+}
+op {
+  name: "Conv2DBackpropFilter"
+  input_arg {
+    name: "input"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "filter_sizes"
+    type: DT_INT32
+  }
+  input_arg {
+    name: "out_backprop"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_HALF
+        type: DT_FLOAT
+      }
+    }
+  }
+  attr {
+    name: "strides"
+    type: "list(int)"
+  }
+  attr {
+    name: "use_cudnn_on_gpu"
+    type: "bool"
     default_value {
       b: true
     }
@@ -6086,6 +7654,40 @@ op {
     }
   }
 }
+op {
+  name: "Cross"
+  input_arg {
+    name: "a"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "b"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "product"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_UINT16
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+}
 op {
   name: "Cumprod"
   input_arg {
@@ -6151,7 +7753,7 @@ op {
   }
 }
 op {
-  name: "Cumsum"
+  name: "Cumprod"
   input_arg {
     name: "x"
     type_attr: "T"
@@ -6197,6 +7799,138 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+  attr {
+    name: "Tidx"
+    type: "type"
+    default_value {
+      type: DT_INT32
+    }
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+}
+op {
+  name: "Cumsum"
+  input_arg {
+    name: "x"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "axis"
+    type_attr: "Tidx"
+  }
+  output_arg {
+    name: "out"
+    type_attr: "T"
+  }
+  attr {
+    name: "exclusive"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
+  attr {
+    name: "reverse"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
+      }
+    }
+  }
+  attr {
+    name: "Tidx"
+    type: "type"
+    default_value {
+      type: DT_INT32
+    }
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+}
+op {
+  name: "Cumsum"
+  input_arg {
+    name: "x"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "axis"
+    type_attr: "Tidx"
+  }
+  output_arg {
+    name: "out"
+    type_attr: "T"
+  }
+  attr {
+    name: "exclusive"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
+  attr {
+    name: "reverse"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -7881,6 +9615,62 @@ op {
     }
   }
 }
+op {
+  name: "Dilation2D"
+  input_arg {
+    name: "input"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "filter"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_UINT16
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+  attr {
+    name: "strides"
+    type: "list(int)"
+    has_minimum: true
+    minimum: 4
+  }
+  attr {
+    name: "rates"
+    type: "list(int)"
+    has_minimum: true
+    minimum: 4
+  }
+  attr {
+    name: "padding"
+    type: "string"
+    allowed_values {
+      list {
+        s: "SAME"
+        s: "VALID"
+      }
+    }
+  }
+}
 op {
   name: "Dilation2DBackpropFilter"
   input_arg {
@@ -7939,6 +9729,124 @@ op {
     }
   }
 }
+op {
+  name: "Dilation2DBackpropFilter"
+  input_arg {
+    name: "input"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "filter"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "out_backprop"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "filter_backprop"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_UINT16
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+  attr {
+    name: "strides"
+    type: "list(int)"
+    has_minimum: true
+    minimum: 4
+  }
+  attr {
+    name: "rates"
+    type: "list(int)"
+    has_minimum: true
+    minimum: 4
+  }
+  attr {
+    name: "padding"
+    type: "string"
+    allowed_values {
+      list {
+        s: "SAME"
+        s: "VALID"
+      }
+    }
+  }
+}
+op {
+  name: "Dilation2DBackpropInput"
+  input_arg {
+    name: "input"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "filter"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "out_backprop"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "in_backprop"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_UINT16
+        type: DT_HALF
+      }
+    }
+  }
+  attr {
+    name: "strides"
+    type: "list(int)"
+    has_minimum: true
+    minimum: 4
+  }
+  attr {
+    name: "rates"
+    type: "list(int)"
+    has_minimum: true
+    minimum: 4
+  }
+  attr {
+    name: "padding"
+    type: "string"
+    allowed_values {
+      list {
+        s: "SAME"
+        s: "VALID"
+      }
+    }
+  }
+}
 op {
   name: "Dilation2DBackpropInput"
   input_arg {
@@ -7971,6 +9879,8 @@ op {
         type: DT_INT8
         type: DT_UINT16
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -8708,6 +10618,64 @@ op {
     }
   }
 }
+op {
+  name: "ExtractImagePatches"
+  input_arg {
+    name: "images"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "patches"
+    type_attr: "T"
+  }
+  attr {
+    name: "ksizes"
+    type: "list(int)"
+    has_minimum: true
+    minimum: 4
+  }
+  attr {
+    name: "strides"
+    type: "list(int)"
+    has_minimum: true
+    minimum: 4
+  }
+  attr {
+    name: "rates"
+    type: "list(int)"
+    has_minimum: true
+    minimum: 4
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_UINT16
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+  attr {
+    name: "padding"
+    type: "string"
+    allowed_values {
+      list {
+        s: "SAME"
+        s: "VALID"
+      }
+    }
+  }
+}
 op {
   name: "ExtractJpegShape"
   input_arg {
@@ -11078,6 +13046,40 @@ op {
     }
   }
 }
+op {
+  name: "Greater"
+  input_arg {
+    name: "x"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "y"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "z"
+    type: DT_BOOL
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_UINT16
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+}
 op {
   name: "GreaterEqual"
   input_arg {
@@ -11110,6 +13112,40 @@ op {
     }
   }
 }
+op {
+  name: "GreaterEqual"
+  input_arg {
+    name: "x"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "y"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "z"
+    type: DT_BOOL
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_UINT16
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+}
 op {
   name: "GroupByWindowDataset"
   input_arg {
@@ -11369,6 +13405,43 @@ op {
     }
   }
 }
+op {
+  name: "HistogramSummary"
+  input_arg {
+    name: "tag"
+    type: DT_STRING
+  }
+  input_arg {
+    name: "values"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "summary"
+    type: DT_STRING
+  }
+  attr {
+    name: "T"
+    type: "type"
+    default_value {
+      type: DT_FLOAT
+    }
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_UINT16
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+}
 op {
   name: "IFFT"
   input_arg {
@@ -12722,6 +14795,72 @@ op {
     }
   }
 }
+op {
+  name: "Less"
+  input_arg {
+    name: "x"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "y"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "z"
+    type: DT_BOOL
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_UINT16
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+}
+op {
+  name: "LessEqual"
+  input_arg {
+    name: "x"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "y"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "z"
+    type: DT_BOOL
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_UINT16
+        type: DT_HALF
+      }
+    }
+  }
+}
 op {
   name: "LessEqual"
   input_arg {
@@ -12750,6 +14889,8 @@ op {
         type: DT_INT8
         type: DT_UINT16
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -14278,6 +16419,65 @@ op {
     }
   }
 }
+op {
+  name: "Max"
+  input_arg {
+    name: "input"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "reduction_indices"
+    type_attr: "Tidx"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  attr {
+    name: "keep_dims"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+  attr {
+    name: "Tidx"
+    type: "type"
+    default_value {
+      type: DT_INT32
+    }
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+}
 op {
   name: "MaxPool"
   input_arg {
@@ -14964,7 +17164,7 @@ op {
   }
 }
 op {
-  name: "MaxPoolGradGrad"
+  name: "MaxPoolGrad"
   input_arg {
     name: "orig_input"
     type_attr: "T"
@@ -15019,6 +17219,9 @@ op {
   attr {
     name: "T"
     type: "type"
+    default_value {
+      type: DT_FLOAT
+    }
     allowed_values {
       list {
         type: DT_FLOAT
@@ -15030,12 +17233,14 @@ op {
         type: DT_INT8
         type: DT_UINT16
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
 }
 op {
-  name: "MaxPoolGradGradV2"
+  name: "MaxPoolGradGrad"
   input_arg {
     name: "orig_input"
     type_attr: "T"
@@ -15048,18 +17253,22 @@ op {
     name: "grad"
     type_attr: "T"
   }
-  input_arg {
-    name: "ksize"
-    type: DT_INT32
-  }
-  input_arg {
-    name: "strides"
-    type: DT_INT32
-  }
   output_arg {
     name: "output"
     type_attr: "T"
   }
+  attr {
+    name: "ksize"
+    type: "list(int)"
+    has_minimum: true
+    minimum: 4
+  }
+  attr {
+    name: "strides"
+    type: "list(int)"
+    has_minimum: true
+    minimum: 4
+  }
   attr {
     name: "padding"
     type: "string"
@@ -15102,18 +17311,18 @@ op {
   }
 }
 op {
-  name: "MaxPoolGradGradWithArgmax"
+  name: "MaxPoolGradGrad"
   input_arg {
-    name: "input"
+    name: "orig_input"
     type_attr: "T"
   }
   input_arg {
-    name: "grad"
+    name: "orig_output"
     type_attr: "T"
   }
   input_arg {
-    name: "argmax"
-    type_attr: "Targmax"
+    name: "grad"
+    type_attr: "T"
   }
   output_arg {
     name: "output"
@@ -15142,12 +17351,84 @@ op {
     }
   }
   attr {
-    name: "Targmax"
+    name: "data_format"
+    type: "string"
+    default_value {
+      s: "NHWC"
+    }
+    allowed_values {
+      list {
+        s: "NHWC"
+        s: "NCHW"
+      }
+    }
+  }
+  attr {
+    name: "T"
     type: "type"
     allowed_values {
       list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
         type: DT_INT32
         type: DT_INT64
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_UINT16
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+}
+op {
+  name: "MaxPoolGradGradV2"
+  input_arg {
+    name: "orig_input"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "orig_output"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "grad"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "ksize"
+    type: DT_INT32
+  }
+  input_arg {
+    name: "strides"
+    type: DT_INT32
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  attr {
+    name: "padding"
+    type: "string"
+    allowed_values {
+      list {
+        s: "SAME"
+        s: "VALID"
+      }
+    }
+  }
+  attr {
+    name: "data_format"
+    type: "string"
+    default_value {
+      s: "NHWC"
+    }
+    allowed_values {
+      list {
+        s: "NHWC"
+        s: "NCHW"
       }
     }
   }
@@ -15170,7 +17451,7 @@ op {
   }
 }
 op {
-  name: "MaxPoolGradV2"
+  name: "MaxPoolGradGradV2"
   input_arg {
     name: "orig_input"
     type_attr: "T"
@@ -15221,9 +17502,6 @@ op {
   attr {
     name: "T"
     type: "type"
-    default_value {
-      type: DT_FLOAT
-    }
     allowed_values {
       list {
         type: DT_FLOAT
@@ -15235,12 +17513,14 @@ op {
         type: DT_INT8
         type: DT_UINT16
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
 }
 op {
-  name: "MaxPoolGradWithArgmax"
+  name: "MaxPoolGradGradWithArgmax"
   input_arg {
     name: "input"
     type_attr: "T"
@@ -15292,19 +17572,23 @@ op {
   attr {
     name: "T"
     type: "type"
-    default_value {
-      type: DT_FLOAT
-    }
     allowed_values {
       list {
         type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_UINT16
         type: DT_HALF
       }
     }
   }
 }
 op {
-  name: "MaxPoolGradWithArgmax"
+  name: "MaxPoolGradGradWithArgmax"
   input_arg {
     name: "input"
     type_attr: "T"
@@ -15367,14 +17651,24 @@ op {
         type: DT_INT8
         type: DT_UINT16
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
 }
 op {
-  name: "MaxPoolV2"
+  name: "MaxPoolGradV2"
   input_arg {
-    name: "input"
+    name: "orig_input"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "orig_output"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "grad"
     type_attr: "T"
   }
   input_arg {
@@ -15389,26 +17683,6 @@ op {
     name: "output"
     type_attr: "T"
   }
-  attr {
-    name: "T"
-    type: "type"
-    default_value {
-      type: DT_FLOAT
-    }
-    allowed_values {
-      list {
-        type: DT_FLOAT
-        type: DT_DOUBLE
-        type: DT_INT32
-        type: DT_INT64
-        type: DT_UINT8
-        type: DT_INT16
-        type: DT_INT8
-        type: DT_UINT16
-        type: DT_HALF
-      }
-    }
-  }
   attr {
     name: "padding"
     type: "string"
@@ -15432,25 +17706,6 @@ op {
       }
     }
   }
-}
-op {
-  name: "MaxPoolV2"
-  input_arg {
-    name: "input"
-    type_attr: "T"
-  }
-  input_arg {
-    name: "ksize"
-    type: DT_INT32
-  }
-  input_arg {
-    name: "strides"
-    type: DT_INT32
-  }
-  output_arg {
-    name: "output"
-    type_attr: "T"
-  }
   attr {
     name: "T"
     type: "type"
@@ -15468,10 +17723,36 @@ op {
         type: DT_INT8
         type: DT_UINT16
         type: DT_HALF
-        type: DT_QINT8
       }
     }
   }
+}
+op {
+  name: "MaxPoolGradV2"
+  input_arg {
+    name: "orig_input"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "orig_output"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "grad"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "ksize"
+    type: DT_INT32
+  }
+  input_arg {
+    name: "strides"
+    type: DT_INT32
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
   attr {
     name: "padding"
     type: "string"
@@ -15492,25 +17773,50 @@ op {
       list {
         s: "NHWC"
         s: "NCHW"
-        s: "NCHW_VECT_C"
+      }
+    }
+  }
+  attr {
+    name: "T"
+    type: "type"
+    default_value {
+      type: DT_FLOAT
+    }
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_UINT16
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
 }
 op {
-  name: "MaxPoolWithArgmax"
+  name: "MaxPoolGradWithArgmax"
   input_arg {
     name: "input"
     type_attr: "T"
   }
-  output_arg {
-    name: "output"
+  input_arg {
+    name: "grad"
     type_attr: "T"
   }
-  output_arg {
+  input_arg {
     name: "argmax"
     type_attr: "Targmax"
   }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
   attr {
     name: "ksize"
     type: "list(int)"
@@ -15524,25 +17830,22 @@ op {
     minimum: 4
   }
   attr {
-    name: "Targmax"
-    type: "type"
-    default_value {
-      type: DT_INT64
-    }
+    name: "padding"
+    type: "string"
     allowed_values {
       list {
-        type: DT_INT32
-        type: DT_INT64
+        s: "SAME"
+        s: "VALID"
       }
     }
   }
   attr {
-    name: "padding"
-    type: "string"
+    name: "Targmax"
+    type: "type"
     allowed_values {
       list {
-        s: "SAME"
-        s: "VALID"
+        type: DT_INT32
+        type: DT_INT64
       }
     }
   }
@@ -15561,19 +17864,23 @@ op {
   }
 }
 op {
-  name: "MaxPoolWithArgmax"
+  name: "MaxPoolGradWithArgmax"
   input_arg {
     name: "input"
     type_attr: "T"
   }
-  output_arg {
-    name: "output"
+  input_arg {
+    name: "grad"
     type_attr: "T"
   }
-  output_arg {
+  input_arg {
     name: "argmax"
     type_attr: "Targmax"
   }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
   attr {
     name: "ksize"
     type: "list(int)"
@@ -15587,25 +17894,22 @@ op {
     minimum: 4
   }
   attr {
-    name: "Targmax"
-    type: "type"
-    default_value {
-      type: DT_INT64
-    }
+    name: "padding"
+    type: "string"
     allowed_values {
       list {
-        type: DT_INT32
-        type: DT_INT64
+        s: "SAME"
+        s: "VALID"
       }
     }
   }
   attr {
-    name: "padding"
-    type: "string"
+    name: "Targmax"
+    type: "type"
     allowed_values {
       list {
-        s: "SAME"
-        s: "VALID"
+        type: DT_INT32
+        type: DT_INT64
       }
     }
   }
@@ -15627,6 +17931,401 @@ op {
     }
   }
 }
+op {
+  name: "MaxPoolGradWithArgmax"
+  input_arg {
+    name: "input"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "grad"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "argmax"
+    type_attr: "Targmax"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  attr {
+    name: "ksize"
+    type: "list(int)"
+    has_minimum: true
+    minimum: 4
+  }
+  attr {
+    name: "strides"
+    type: "list(int)"
+    has_minimum: true
+    minimum: 4
+  }
+  attr {
+    name: "padding"
+    type: "string"
+    allowed_values {
+      list {
+        s: "SAME"
+        s: "VALID"
+      }
+    }
+  }
+  attr {
+    name: "Targmax"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_UINT16
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+}
+op {
+  name: "MaxPoolV2"
+  input_arg {
+    name: "input"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "ksize"
+    type: DT_INT32
+  }
+  input_arg {
+    name: "strides"
+    type: DT_INT32
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    default_value {
+      type: DT_FLOAT
+    }
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_UINT16
+        type: DT_HALF
+      }
+    }
+  }
+  attr {
+    name: "padding"
+    type: "string"
+    allowed_values {
+      list {
+        s: "SAME"
+        s: "VALID"
+      }
+    }
+  }
+  attr {
+    name: "data_format"
+    type: "string"
+    default_value {
+      s: "NHWC"
+    }
+    allowed_values {
+      list {
+        s: "NHWC"
+        s: "NCHW"
+      }
+    }
+  }
+}
+op {
+  name: "MaxPoolV2"
+  input_arg {
+    name: "input"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "ksize"
+    type: DT_INT32
+  }
+  input_arg {
+    name: "strides"
+    type: DT_INT32
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    default_value {
+      type: DT_FLOAT
+    }
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_UINT16
+        type: DT_HALF
+        type: DT_QINT8
+      }
+    }
+  }
+  attr {
+    name: "padding"
+    type: "string"
+    allowed_values {
+      list {
+        s: "SAME"
+        s: "VALID"
+      }
+    }
+  }
+  attr {
+    name: "data_format"
+    type: "string"
+    default_value {
+      s: "NHWC"
+    }
+    allowed_values {
+      list {
+        s: "NHWC"
+        s: "NCHW"
+        s: "NCHW_VECT_C"
+      }
+    }
+  }
+}
+op {
+  name: "MaxPoolWithArgmax"
+  input_arg {
+    name: "input"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "argmax"
+    type_attr: "Targmax"
+  }
+  attr {
+    name: "ksize"
+    type: "list(int)"
+    has_minimum: true
+    minimum: 4
+  }
+  attr {
+    name: "strides"
+    type: "list(int)"
+    has_minimum: true
+    minimum: 4
+  }
+  attr {
+    name: "Targmax"
+    type: "type"
+    default_value {
+      type: DT_INT64
+    }
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+  attr {
+    name: "padding"
+    type: "string"
+    allowed_values {
+      list {
+        s: "SAME"
+        s: "VALID"
+      }
+    }
+  }
+  attr {
+    name: "T"
+    type: "type"
+    default_value {
+      type: DT_FLOAT
+    }
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_HALF
+      }
+    }
+  }
+}
+op {
+  name: "MaxPoolWithArgmax"
+  input_arg {
+    name: "input"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "argmax"
+    type_attr: "Targmax"
+  }
+  attr {
+    name: "ksize"
+    type: "list(int)"
+    has_minimum: true
+    minimum: 4
+  }
+  attr {
+    name: "strides"
+    type: "list(int)"
+    has_minimum: true
+    minimum: 4
+  }
+  attr {
+    name: "Targmax"
+    type: "type"
+    default_value {
+      type: DT_INT64
+    }
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+  attr {
+    name: "padding"
+    type: "string"
+    allowed_values {
+      list {
+        s: "SAME"
+        s: "VALID"
+      }
+    }
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_UINT16
+        type: DT_HALF
+      }
+    }
+  }
+}
+op {
+  name: "MaxPoolWithArgmax"
+  input_arg {
+    name: "input"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "argmax"
+    type_attr: "Targmax"
+  }
+  attr {
+    name: "ksize"
+    type: "list(int)"
+    has_minimum: true
+    minimum: 4
+  }
+  attr {
+    name: "strides"
+    type: "list(int)"
+    has_minimum: true
+    minimum: 4
+  }
+  attr {
+    name: "Targmax"
+    type: "type"
+    default_value {
+      type: DT_INT64
+    }
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+  attr {
+    name: "padding"
+    type: "string"
+    allowed_values {
+      list {
+        s: "SAME"
+        s: "VALID"
+      }
+    }
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_UINT16
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+}
 op {
   name: "Maximum"
   input_arg {
@@ -15713,6 +18412,65 @@ op {
     }
   }
 }
+op {
+  name: "Mean"
+  input_arg {
+    name: "input"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "reduction_indices"
+    type_attr: "Tidx"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  attr {
+    name: "keep_dims"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+  attr {
+    name: "Tidx"
+    type: "type"
+    default_value {
+      type: DT_INT32
+    }
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+}
 op {
   name: "Merge"
   input_arg {
@@ -15894,6 +18652,65 @@ op {
     }
   }
 }
+op {
+  name: "Min"
+  input_arg {
+    name: "input"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "reduction_indices"
+    type_attr: "Tidx"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  attr {
+    name: "keep_dims"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+  attr {
+    name: "Tidx"
+    type: "type"
+    default_value {
+      type: DT_INT32
+    }
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+}
 op {
   name: "Minimum"
   input_arg {
@@ -16117,12 +18934,61 @@ op {
   is_stateful: true
 }
 op {
-  name: "MutableDenseHashTable"
+  name: "Multinomial"
   input_arg {
-    name: "empty_key"
-    type_attr: "key_dtype"
+    name: "logits"
+    type_attr: "T"
   }
-  output_arg {
+  input_arg {
+    name: "num_samples"
+    type: DT_INT32
+  }
+  output_arg {
+    name: "output"
+    type: DT_INT64
+  }
+  attr {
+    name: "seed"
+    type: "int"
+    default_value {
+      i: 0
+    }
+  }
+  attr {
+    name: "seed2"
+    type: "int"
+    default_value {
+      i: 0
+    }
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_UINT16
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+  is_stateful: true
+}
+op {
+  name: "MutableDenseHashTable"
+  input_arg {
+    name: "empty_key"
+    type_attr: "key_dtype"
+  }
+  output_arg {
     name: "table_handle"
     type: DT_STRING
     is_ref: true
@@ -18216,6 +21082,65 @@ op {
     }
   }
 }
+op {
+  name: "Prod"
+  input_arg {
+    name: "input"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "reduction_indices"
+    type_attr: "Tidx"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  attr {
+    name: "keep_dims"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+  attr {
+    name: "Tidx"
+    type: "type"
+    default_value {
+      type: DT_INT32
+    }
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+}
 op {
   name: "PyFunc"
   input_arg {
@@ -19494,250 +22419,1277 @@ op {
     }
   }
   attr {
-    name: "T2"
-    type: "type"
-    allowed_values {
-      list {
-        type: DT_QINT8
-        type: DT_QUINT8
-        type: DT_QINT16
-        type: DT_QUINT16
-        type: DT_QINT32
-      }
+    name: "T2"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT16
+        type: DT_QUINT16
+        type: DT_QINT32
+      }
+    }
+  }
+  attr {
+    name: "Toutput"
+    type: "type"
+    default_value {
+      type: DT_QINT32
+    }
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT16
+        type: DT_QUINT16
+        type: DT_QINT32
+      }
+    }
+  }
+  is_commutative: true
+}
+op {
+  name: "QuantizedRelu"
+  input_arg {
+    name: "features"
+    type_attr: "Tinput"
+  }
+  input_arg {
+    name: "min_features"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "max_features"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "activations"
+    type_attr: "out_type"
+  }
+  output_arg {
+    name: "min_activations"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "max_activations"
+    type: DT_FLOAT
+  }
+  attr {
+    name: "Tinput"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT16
+        type: DT_QUINT16
+        type: DT_QINT32
+      }
+    }
+  }
+  attr {
+    name: "out_type"
+    type: "type"
+    default_value {
+      type: DT_QUINT8
+    }
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT16
+        type: DT_QUINT16
+        type: DT_QINT32
+      }
+    }
+  }
+}
+op {
+  name: "QuantizedRelu6"
+  input_arg {
+    name: "features"
+    type_attr: "Tinput"
+  }
+  input_arg {
+    name: "min_features"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "max_features"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "activations"
+    type_attr: "out_type"
+  }
+  output_arg {
+    name: "min_activations"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "max_activations"
+    type: DT_FLOAT
+  }
+  attr {
+    name: "Tinput"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT16
+        type: DT_QUINT16
+        type: DT_QINT32
+      }
+    }
+  }
+  attr {
+    name: "out_type"
+    type: "type"
+    default_value {
+      type: DT_QUINT8
+    }
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT16
+        type: DT_QUINT16
+        type: DT_QINT32
+      }
+    }
+  }
+}
+op {
+  name: "QuantizedReluX"
+  input_arg {
+    name: "features"
+    type_attr: "Tinput"
+  }
+  input_arg {
+    name: "max_value"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "min_features"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "max_features"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "activations"
+    type_attr: "out_type"
+  }
+  output_arg {
+    name: "min_activations"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "max_activations"
+    type: DT_FLOAT
+  }
+  attr {
+    name: "Tinput"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT16
+        type: DT_QUINT16
+        type: DT_QINT32
+      }
+    }
+  }
+  attr {
+    name: "out_type"
+    type: "type"
+    default_value {
+      type: DT_QUINT8
+    }
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT16
+        type: DT_QUINT16
+        type: DT_QINT32
+      }
+    }
+  }
+}
+op {
+  name: "QuantizedReshape"
+  input_arg {
+    name: "tensor"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "shape"
+    type_attr: "Tshape"
+  }
+  input_arg {
+    name: "input_min"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "input_max"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "output_min"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "output_max"
+    type: DT_FLOAT
+  }
+  attr {
+    name: "T"
+    type: "type"
+  }
+  attr {
+    name: "Tshape"
+    type: "type"
+    default_value {
+      type: DT_INT32
+    }
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+}
+op {
+  name: "QuantizedResizeBilinear"
+  input_arg {
+    name: "images"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "size"
+    type: DT_INT32
+  }
+  input_arg {
+    name: "min"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "max"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "resized_images"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "out_min"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "out_max"
+    type: DT_FLOAT
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_FLOAT
+      }
+    }
+  }
+  attr {
+    name: "align_corners"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
+}
+op {
+  name: "QueueClose"
+  input_arg {
+    name: "handle"
+    type: DT_STRING
+    is_ref: true
+  }
+  attr {
+    name: "cancel_pending_enqueues"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
+}
+op {
+  name: "QueueCloseV2"
+  input_arg {
+    name: "handle"
+    type: DT_RESOURCE
+  }
+  attr {
+    name: "cancel_pending_enqueues"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
+  is_stateful: true
+}
+op {
+  name: "QueueDequeue"
+  input_arg {
+    name: "handle"
+    type: DT_STRING
+    is_ref: true
+  }
+  output_arg {
+    name: "components"
+    type_list_attr: "component_types"
+  }
+  attr {
+    name: "component_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "timeout_ms"
+    type: "int"
+    default_value {
+      i: -1
+    }
+  }
+}
+op {
+  name: "QueueDequeueMany"
+  input_arg {
+    name: "handle"
+    type: DT_STRING
+    is_ref: true
+  }
+  input_arg {
+    name: "n"
+    type: DT_INT32
+  }
+  output_arg {
+    name: "components"
+    type_list_attr: "component_types"
+  }
+  attr {
+    name: "component_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "timeout_ms"
+    type: "int"
+    default_value {
+      i: -1
+    }
+  }
+}
+op {
+  name: "QueueDequeueManyV2"
+  input_arg {
+    name: "handle"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "n"
+    type: DT_INT32
+  }
+  output_arg {
+    name: "components"
+    type_list_attr: "component_types"
+  }
+  attr {
+    name: "component_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "timeout_ms"
+    type: "int"
+    default_value {
+      i: -1
+    }
+  }
+  is_stateful: true
+}
+op {
+  name: "QueueDequeueUpTo"
+  input_arg {
+    name: "handle"
+    type: DT_STRING
+    is_ref: true
+  }
+  input_arg {
+    name: "n"
+    type: DT_INT32
+  }
+  output_arg {
+    name: "components"
+    type_list_attr: "component_types"
+  }
+  attr {
+    name: "component_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "timeout_ms"
+    type: "int"
+    default_value {
+      i: -1
+    }
+  }
+}
+op {
+  name: "QueueDequeueUpToV2"
+  input_arg {
+    name: "handle"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "n"
+    type: DT_INT32
+  }
+  output_arg {
+    name: "components"
+    type_list_attr: "component_types"
+  }
+  attr {
+    name: "component_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "timeout_ms"
+    type: "int"
+    default_value {
+      i: -1
+    }
+  }
+  is_stateful: true
+}
+op {
+  name: "QueueDequeueV2"
+  input_arg {
+    name: "handle"
+    type: DT_RESOURCE
+  }
+  output_arg {
+    name: "components"
+    type_list_attr: "component_types"
+  }
+  attr {
+    name: "component_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "timeout_ms"
+    type: "int"
+    default_value {
+      i: -1
+    }
+  }
+  is_stateful: true
+}
+op {
+  name: "QueueEnqueue"
+  input_arg {
+    name: "handle"
+    type: DT_STRING
+    is_ref: true
+  }
+  input_arg {
+    name: "components"
+    type_list_attr: "Tcomponents"
+  }
+  attr {
+    name: "Tcomponents"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "timeout_ms"
+    type: "int"
+    default_value {
+      i: -1
+    }
+  }
+}
+op {
+  name: "QueueEnqueueMany"
+  input_arg {
+    name: "handle"
+    type: DT_STRING
+    is_ref: true
+  }
+  input_arg {
+    name: "components"
+    type_list_attr: "Tcomponents"
+  }
+  attr {
+    name: "Tcomponents"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "timeout_ms"
+    type: "int"
+    default_value {
+      i: -1
+    }
+  }
+}
+op {
+  name: "QueueEnqueueManyV2"
+  input_arg {
+    name: "handle"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "components"
+    type_list_attr: "Tcomponents"
+  }
+  attr {
+    name: "Tcomponents"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "timeout_ms"
+    type: "int"
+    default_value {
+      i: -1
+    }
+  }
+  is_stateful: true
+}
+op {
+  name: "QueueEnqueueV2"
+  input_arg {
+    name: "handle"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "components"
+    type_list_attr: "Tcomponents"
+  }
+  attr {
+    name: "Tcomponents"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "timeout_ms"
+    type: "int"
+    default_value {
+      i: -1
+    }
+  }
+  is_stateful: true
+}
+op {
+  name: "QueueIsClosed"
+  input_arg {
+    name: "handle"
+    type: DT_STRING
+    is_ref: true
+  }
+  output_arg {
+    name: "is_closed"
+    type: DT_BOOL
+  }
+}
+op {
+  name: "QueueIsClosedV2"
+  input_arg {
+    name: "handle"
+    type: DT_RESOURCE
+  }
+  output_arg {
+    name: "is_closed"
+    type: DT_BOOL
+  }
+  is_stateful: true
+}
+op {
+  name: "QueueSize"
+  input_arg {
+    name: "handle"
+    type: DT_STRING
+    is_ref: true
+  }
+  output_arg {
+    name: "size"
+    type: DT_INT32
+  }
+}
+op {
+  name: "QueueSizeV2"
+  input_arg {
+    name: "handle"
+    type: DT_RESOURCE
+  }
+  output_arg {
+    name: "size"
+    type: DT_INT32
+  }
+  is_stateful: true
+}
+op {
+  name: "RFFT"
+  input_arg {
+    name: "input"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "fft_length"
+    type: DT_INT32
+  }
+  output_arg {
+    name: "output"
+    type: DT_COMPLEX64
+  }
+}
+op {
+  name: "RFFT2D"
+  input_arg {
+    name: "input"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "fft_length"
+    type: DT_INT32
+  }
+  output_arg {
+    name: "output"
+    type: DT_COMPLEX64
+  }
+}
+op {
+  name: "RFFT3D"
+  input_arg {
+    name: "input"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "fft_length"
+    type: DT_INT32
+  }
+  output_arg {
+    name: "output"
+    type: DT_COMPLEX64
+  }
+}
+op {
+  name: "RGBToHSV"
+  input_arg {
+    name: "images"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    default_value {
+      type: DT_FLOAT
+    }
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+      }
+    }
+  }
+}
+op {
+  name: "RandomCrop"
+  input_arg {
+    name: "image"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "size"
+    type: DT_INT64
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_UINT8
+        type: DT_INT8
+        type: DT_INT16
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_FLOAT
+        type: DT_DOUBLE
+      }
+    }
+  }
+  attr {
+    name: "seed"
+    type: "int"
+    default_value {
+      i: 0
+    }
+  }
+  attr {
+    name: "seed2"
+    type: "int"
+    default_value {
+      i: 0
+    }
+  }
+  deprecation {
+    version: 8
+  }
+  is_stateful: true
+}
+op {
+  name: "RandomGamma"
+  input_arg {
+    name: "shape"
+    type_attr: "S"
+  }
+  input_arg {
+    name: "alpha"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  attr {
+    name: "seed"
+    type: "int"
+    default_value {
+      i: 0
+    }
+  }
+  attr {
+    name: "seed2"
+    type: "int"
+    default_value {
+      i: 0
+    }
+  }
+  attr {
+    name: "S"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_HALF
+        type: DT_FLOAT
+        type: DT_DOUBLE
+      }
+    }
+  }
+  is_stateful: true
+}
+op {
+  name: "RandomPoisson"
+  input_arg {
+    name: "shape"
+    type_attr: "S"
+  }
+  input_arg {
+    name: "rate"
+    type_attr: "dtype"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "dtype"
+  }
+  attr {
+    name: "seed"
+    type: "int"
+    default_value {
+      i: 0
+    }
+  }
+  attr {
+    name: "seed2"
+    type: "int"
+    default_value {
+      i: 0
+    }
+  }
+  attr {
+    name: "S"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+  attr {
+    name: "dtype"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_HALF
+        type: DT_FLOAT
+        type: DT_DOUBLE
+      }
+    }
+  }
+  is_stateful: true
+}
+op {
+  name: "RandomPoissonV2"
+  input_arg {
+    name: "shape"
+    type_attr: "S"
+  }
+  input_arg {
+    name: "rate"
+    type_attr: "R"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "dtype"
+  }
+  attr {
+    name: "seed"
+    type: "int"
+    default_value {
+      i: 0
+    }
+  }
+  attr {
+    name: "seed2"
+    type: "int"
+    default_value {
+      i: 0
+    }
+  }
+  attr {
+    name: "S"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+  attr {
+    name: "R"
+    type: "type"
+    default_value {
+      type: DT_DOUBLE
+    }
+    allowed_values {
+      list {
+        type: DT_HALF
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+  attr {
+    name: "dtype"
+    type: "type"
+    default_value {
+      type: DT_INT64
+    }
+    allowed_values {
+      list {
+        type: DT_HALF
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+  is_stateful: true
+}
+op {
+  name: "RandomShuffle"
+  input_arg {
+    name: "value"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  attr {
+    name: "seed"
+    type: "int"
+    default_value {
+      i: 0
+    }
+  }
+  attr {
+    name: "seed2"
+    type: "int"
+    default_value {
+      i: 0
+    }
+  }
+  attr {
+    name: "T"
+    type: "type"
+  }
+  is_stateful: true
+}
+op {
+  name: "RandomShuffleQueue"
+  output_arg {
+    name: "handle"
+    type: DT_STRING
+    is_ref: true
+  }
+  attr {
+    name: "component_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "shapes"
+    type: "list(shape)"
+    default_value {
+      list {
+      }
+    }
+    has_minimum: true
+  }
+  attr {
+    name: "capacity"
+    type: "int"
+    default_value {
+      i: -1
+    }
+  }
+  attr {
+    name: "min_after_dequeue"
+    type: "int"
+    default_value {
+      i: 0
+    }
+  }
+  attr {
+    name: "seed"
+    type: "int"
+    default_value {
+      i: 0
+    }
+  }
+  attr {
+    name: "seed2"
+    type: "int"
+    default_value {
+      i: 0
+    }
+  }
+  attr {
+    name: "container"
+    type: "string"
+    default_value {
+      s: ""
+    }
+  }
+  attr {
+    name: "shared_name"
+    type: "string"
+    default_value {
+      s: ""
+    }
+  }
+  is_stateful: true
+}
+op {
+  name: "RandomShuffleQueueV2"
+  output_arg {
+    name: "handle"
+    type: DT_RESOURCE
+  }
+  attr {
+    name: "component_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "shapes"
+    type: "list(shape)"
+    default_value {
+      list {
+      }
+    }
+    has_minimum: true
+  }
+  attr {
+    name: "capacity"
+    type: "int"
+    default_value {
+      i: -1
+    }
+  }
+  attr {
+    name: "min_after_dequeue"
+    type: "int"
+    default_value {
+      i: 0
+    }
+  }
+  attr {
+    name: "seed"
+    type: "int"
+    default_value {
+      i: 0
+    }
+  }
+  attr {
+    name: "seed2"
+    type: "int"
+    default_value {
+      i: 0
     }
   }
   attr {
-    name: "Toutput"
-    type: "type"
+    name: "container"
+    type: "string"
     default_value {
-      type: DT_QINT32
+      s: ""
     }
-    allowed_values {
-      list {
-        type: DT_QINT8
-        type: DT_QUINT8
-        type: DT_QINT16
-        type: DT_QUINT16
-        type: DT_QINT32
-      }
+  }
+  attr {
+    name: "shared_name"
+    type: "string"
+    default_value {
+      s: ""
     }
   }
-  is_commutative: true
+  is_stateful: true
 }
 op {
-  name: "QuantizedRelu"
-  input_arg {
-    name: "features"
-    type_attr: "Tinput"
-  }
-  input_arg {
-    name: "min_features"
-    type: DT_FLOAT
-  }
+  name: "RandomStandardNormal"
   input_arg {
-    name: "max_features"
-    type: DT_FLOAT
+    name: "shape"
+    type_attr: "T"
   }
   output_arg {
-    name: "activations"
-    type_attr: "out_type"
+    name: "output"
+    type_attr: "dtype"
   }
-  output_arg {
-    name: "min_activations"
-    type: DT_FLOAT
+  attr {
+    name: "seed"
+    type: "int"
+    default_value {
+      i: 0
+    }
   }
-  output_arg {
-    name: "max_activations"
-    type: DT_FLOAT
+  attr {
+    name: "seed2"
+    type: "int"
+    default_value {
+      i: 0
+    }
   }
   attr {
-    name: "Tinput"
+    name: "dtype"
     type: "type"
     allowed_values {
       list {
-        type: DT_QINT8
-        type: DT_QUINT8
-        type: DT_QINT16
-        type: DT_QUINT16
-        type: DT_QINT32
+        type: DT_HALF
+        type: DT_FLOAT
+        type: DT_DOUBLE
       }
     }
   }
   attr {
-    name: "out_type"
+    name: "T"
     type: "type"
-    default_value {
-      type: DT_QUINT8
-    }
     allowed_values {
       list {
-        type: DT_QINT8
-        type: DT_QUINT8
-        type: DT_QINT16
-        type: DT_QUINT16
-        type: DT_QINT32
+        type: DT_INT32
+        type: DT_INT64
       }
     }
   }
+  is_stateful: true
 }
 op {
-  name: "QuantizedRelu6"
-  input_arg {
-    name: "features"
-    type_attr: "Tinput"
-  }
-  input_arg {
-    name: "min_features"
-    type: DT_FLOAT
-  }
+  name: "RandomUniform"
   input_arg {
-    name: "max_features"
-    type: DT_FLOAT
+    name: "shape"
+    type_attr: "T"
   }
   output_arg {
-    name: "activations"
-    type_attr: "out_type"
+    name: "output"
+    type_attr: "dtype"
   }
-  output_arg {
-    name: "min_activations"
-    type: DT_FLOAT
+  attr {
+    name: "seed"
+    type: "int"
+    default_value {
+      i: 0
+    }
   }
-  output_arg {
-    name: "max_activations"
-    type: DT_FLOAT
+  attr {
+    name: "seed2"
+    type: "int"
+    default_value {
+      i: 0
+    }
   }
   attr {
-    name: "Tinput"
+    name: "dtype"
     type: "type"
     allowed_values {
       list {
-        type: DT_QINT8
-        type: DT_QUINT8
-        type: DT_QINT16
-        type: DT_QUINT16
-        type: DT_QINT32
+        type: DT_HALF
+        type: DT_FLOAT
+        type: DT_DOUBLE
       }
     }
   }
   attr {
-    name: "out_type"
+    name: "T"
     type: "type"
-    default_value {
-      type: DT_QUINT8
-    }
     allowed_values {
       list {
-        type: DT_QINT8
-        type: DT_QUINT8
-        type: DT_QINT16
-        type: DT_QUINT16
-        type: DT_QINT32
+        type: DT_INT32
+        type: DT_INT64
       }
     }
   }
+  is_stateful: true
 }
 op {
-  name: "QuantizedReluX"
-  input_arg {
-    name: "features"
-    type_attr: "Tinput"
-  }
+  name: "RandomUniformInt"
   input_arg {
-    name: "max_value"
-    type: DT_FLOAT
+    name: "shape"
+    type_attr: "T"
   }
   input_arg {
-    name: "min_features"
-    type: DT_FLOAT
+    name: "minval"
+    type_attr: "Tout"
   }
   input_arg {
-    name: "max_features"
-    type: DT_FLOAT
+    name: "maxval"
+    type_attr: "Tout"
   }
   output_arg {
-    name: "activations"
-    type_attr: "out_type"
+    name: "output"
+    type_attr: "Tout"
   }
-  output_arg {
-    name: "min_activations"
-    type: DT_FLOAT
+  attr {
+    name: "seed"
+    type: "int"
+    default_value {
+      i: 0
+    }
   }
-  output_arg {
-    name: "max_activations"
-    type: DT_FLOAT
+  attr {
+    name: "seed2"
+    type: "int"
+    default_value {
+      i: 0
+    }
   }
   attr {
-    name: "Tinput"
+    name: "Tout"
     type: "type"
     allowed_values {
       list {
-        type: DT_QINT8
-        type: DT_QUINT8
-        type: DT_QINT16
-        type: DT_QUINT16
-        type: DT_QINT32
+        type: DT_INT32
+        type: DT_INT64
       }
     }
   }
   attr {
-    name: "out_type"
+    name: "T"
     type: "type"
-    default_value {
-      type: DT_QUINT8
-    }
     allowed_values {
       list {
-        type: DT_QINT8
-        type: DT_QUINT8
-        type: DT_QINT16
-        type: DT_QUINT16
-        type: DT_QINT32
+        type: DT_INT32
+        type: DT_INT64
       }
     }
   }
+  is_stateful: true
 }
 op {
-  name: "QuantizedReshape"
-  input_arg {
-    name: "tensor"
-    type_attr: "T"
-  }
+  name: "Range"
   input_arg {
-    name: "shape"
-    type_attr: "Tshape"
+    name: "start"
+    type_attr: "Tidx"
   }
   input_arg {
-    name: "input_min"
-    type: DT_FLOAT
+    name: "limit"
+    type_attr: "Tidx"
   }
   input_arg {
-    name: "input_max"
-    type: DT_FLOAT
+    name: "delta"
+    type_attr: "Tidx"
   }
   output_arg {
     name: "output"
-    type_attr: "T"
-  }
-  output_arg {
-    name: "output_min"
-    type: DT_FLOAT
-  }
-  output_arg {
-    name: "output_max"
-    type: DT_FLOAT
-  }
-  attr {
-    name: "T"
-    type: "type"
+    type_attr: "Tidx"
   }
   attr {
-    name: "Tshape"
+    name: "Tidx"
     type: "type"
     default_value {
       type: DT_INT32
     }
     allowed_values {
       list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
         type: DT_INT32
         type: DT_INT64
       }
@@ -19745,456 +23697,307 @@ op {
   }
 }
 op {
-  name: "QuantizedResizeBilinear"
-  input_arg {
-    name: "images"
-    type_attr: "T"
-  }
+  name: "RangeDataset"
   input_arg {
-    name: "size"
-    type: DT_INT32
+    name: "start"
+    type: DT_INT64
   }
   input_arg {
-    name: "min"
-    type: DT_FLOAT
+    name: "stop"
+    type: DT_INT64
   }
   input_arg {
-    name: "max"
-    type: DT_FLOAT
-  }
-  output_arg {
-    name: "resized_images"
-    type_attr: "T"
-  }
-  output_arg {
-    name: "out_min"
-    type: DT_FLOAT
+    name: "step"
+    type: DT_INT64
   }
   output_arg {
-    name: "out_max"
-    type: DT_FLOAT
-  }
-  attr {
-    name: "T"
-    type: "type"
-    allowed_values {
-      list {
-        type: DT_QUINT8
-        type: DT_QINT32
-        type: DT_FLOAT
-      }
-    }
-  }
-  attr {
-    name: "align_corners"
-    type: "bool"
-    default_value {
-      b: false
-    }
-  }
-}
-op {
-  name: "QueueClose"
-  input_arg {
     name: "handle"
-    type: DT_STRING
-    is_ref: true
+    type: DT_VARIANT
   }
   attr {
-    name: "cancel_pending_enqueues"
-    type: "bool"
-    default_value {
-      b: false
-    }
-  }
-}
-op {
-  name: "QueueCloseV2"
-  input_arg {
-    name: "handle"
-    type: DT_RESOURCE
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
   }
   attr {
-    name: "cancel_pending_enqueues"
-    type: "bool"
-    default_value {
-      b: false
-    }
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
   }
   is_stateful: true
 }
 op {
-  name: "QueueDequeue"
+  name: "Rank"
   input_arg {
-    name: "handle"
-    type: DT_STRING
-    is_ref: true
+    name: "input"
+    type_attr: "T"
   }
   output_arg {
-    name: "components"
-    type_list_attr: "component_types"
-  }
-  attr {
-    name: "component_types"
-    type: "list(type)"
-    has_minimum: true
-    minimum: 1
+    name: "output"
+    type: DT_INT32
   }
   attr {
-    name: "timeout_ms"
-    type: "int"
-    default_value {
-      i: -1
-    }
+    name: "T"
+    type: "type"
   }
 }
 op {
-  name: "QueueDequeueMany"
+  name: "ReadFile"
   input_arg {
-    name: "handle"
+    name: "filename"
     type: DT_STRING
-    is_ref: true
-  }
-  input_arg {
-    name: "n"
-    type: DT_INT32
   }
   output_arg {
-    name: "components"
-    type_list_attr: "component_types"
-  }
-  attr {
-    name: "component_types"
-    type: "list(type)"
-    has_minimum: true
-    minimum: 1
-  }
-  attr {
-    name: "timeout_ms"
-    type: "int"
-    default_value {
-      i: -1
-    }
+    name: "contents"
+    type: DT_STRING
   }
 }
 op {
-  name: "QueueDequeueManyV2"
+  name: "ReadVariableOp"
   input_arg {
-    name: "handle"
+    name: "resource"
     type: DT_RESOURCE
   }
-  input_arg {
-    name: "n"
-    type: DT_INT32
-  }
   output_arg {
-    name: "components"
-    type_list_attr: "component_types"
-  }
-  attr {
-    name: "component_types"
-    type: "list(type)"
-    has_minimum: true
-    minimum: 1
+    name: "value"
+    type_attr: "dtype"
   }
   attr {
-    name: "timeout_ms"
-    type: "int"
-    default_value {
-      i: -1
-    }
+    name: "dtype"
+    type: "type"
   }
   is_stateful: true
 }
 op {
-  name: "QueueDequeueUpTo"
+  name: "ReaderNumRecordsProduced"
   input_arg {
-    name: "handle"
+    name: "reader_handle"
     type: DT_STRING
     is_ref: true
   }
-  input_arg {
-    name: "n"
-    type: DT_INT32
-  }
   output_arg {
-    name: "components"
-    type_list_attr: "component_types"
-  }
-  attr {
-    name: "component_types"
-    type: "list(type)"
-    has_minimum: true
-    minimum: 1
-  }
-  attr {
-    name: "timeout_ms"
-    type: "int"
-    default_value {
-      i: -1
-    }
+    name: "records_produced"
+    type: DT_INT64
   }
 }
 op {
-  name: "QueueDequeueUpToV2"
+  name: "ReaderNumRecordsProducedV2"
   input_arg {
-    name: "handle"
+    name: "reader_handle"
     type: DT_RESOURCE
   }
-  input_arg {
-    name: "n"
-    type: DT_INT32
-  }
   output_arg {
-    name: "components"
-    type_list_attr: "component_types"
-  }
-  attr {
-    name: "component_types"
-    type: "list(type)"
-    has_minimum: true
-    minimum: 1
-  }
-  attr {
-    name: "timeout_ms"
-    type: "int"
-    default_value {
-      i: -1
-    }
+    name: "records_produced"
+    type: DT_INT64
   }
   is_stateful: true
 }
 op {
-  name: "QueueDequeueV2"
+  name: "ReaderNumWorkUnitsCompleted"
   input_arg {
-    name: "handle"
-    type: DT_RESOURCE
+    name: "reader_handle"
+    type: DT_STRING
+    is_ref: true
   }
   output_arg {
-    name: "components"
-    type_list_attr: "component_types"
+    name: "units_completed"
+    type: DT_INT64
   }
-  attr {
-    name: "component_types"
-    type: "list(type)"
-    has_minimum: true
-    minimum: 1
+}
+op {
+  name: "ReaderNumWorkUnitsCompletedV2"
+  input_arg {
+    name: "reader_handle"
+    type: DT_RESOURCE
   }
-  attr {
-    name: "timeout_ms"
-    type: "int"
-    default_value {
-      i: -1
-    }
+  output_arg {
+    name: "units_completed"
+    type: DT_INT64
   }
   is_stateful: true
 }
 op {
-  name: "QueueEnqueue"
+  name: "ReaderRead"
   input_arg {
-    name: "handle"
+    name: "reader_handle"
     type: DT_STRING
     is_ref: true
   }
   input_arg {
-    name: "components"
-    type_list_attr: "Tcomponents"
-  }
-  attr {
-    name: "Tcomponents"
-    type: "list(type)"
-    has_minimum: true
-    minimum: 1
+    name: "queue_handle"
+    type: DT_STRING
+    is_ref: true
   }
-  attr {
-    name: "timeout_ms"
-    type: "int"
-    default_value {
-      i: -1
-    }
+  output_arg {
+    name: "key"
+    type: DT_STRING
+  }
+  output_arg {
+    name: "value"
+    type: DT_STRING
   }
 }
 op {
-  name: "QueueEnqueueMany"
+  name: "ReaderReadUpTo"
   input_arg {
-    name: "handle"
+    name: "reader_handle"
     type: DT_STRING
     is_ref: true
   }
   input_arg {
-    name: "components"
-    type_list_attr: "Tcomponents"
+    name: "queue_handle"
+    type: DT_STRING
+    is_ref: true
   }
-  attr {
-    name: "Tcomponents"
-    type: "list(type)"
-    has_minimum: true
-    minimum: 1
+  input_arg {
+    name: "num_records"
+    type: DT_INT64
   }
-  attr {
-    name: "timeout_ms"
-    type: "int"
-    default_value {
-      i: -1
-    }
+  output_arg {
+    name: "keys"
+    type: DT_STRING
+  }
+  output_arg {
+    name: "values"
+    type: DT_STRING
   }
 }
 op {
-  name: "QueueEnqueueManyV2"
+  name: "ReaderReadUpToV2"
   input_arg {
-    name: "handle"
+    name: "reader_handle"
     type: DT_RESOURCE
   }
   input_arg {
-    name: "components"
-    type_list_attr: "Tcomponents"
+    name: "queue_handle"
+    type: DT_RESOURCE
   }
-  attr {
-    name: "Tcomponents"
-    type: "list(type)"
-    has_minimum: true
-    minimum: 1
+  input_arg {
+    name: "num_records"
+    type: DT_INT64
   }
-  attr {
-    name: "timeout_ms"
-    type: "int"
-    default_value {
-      i: -1
-    }
+  output_arg {
+    name: "keys"
+    type: DT_STRING
+  }
+  output_arg {
+    name: "values"
+    type: DT_STRING
   }
   is_stateful: true
 }
 op {
-  name: "QueueEnqueueV2"
+  name: "ReaderReadV2"
   input_arg {
-    name: "handle"
+    name: "reader_handle"
     type: DT_RESOURCE
   }
   input_arg {
-    name: "components"
-    type_list_attr: "Tcomponents"
+    name: "queue_handle"
+    type: DT_RESOURCE
   }
-  attr {
-    name: "Tcomponents"
-    type: "list(type)"
-    has_minimum: true
-    minimum: 1
+  output_arg {
+    name: "key"
+    type: DT_STRING
   }
-  attr {
-    name: "timeout_ms"
-    type: "int"
-    default_value {
-      i: -1
-    }
+  output_arg {
+    name: "value"
+    type: DT_STRING
   }
   is_stateful: true
 }
 op {
-  name: "QueueIsClosed"
+  name: "ReaderReset"
   input_arg {
-    name: "handle"
+    name: "reader_handle"
     type: DT_STRING
     is_ref: true
   }
-  output_arg {
-    name: "is_closed"
-    type: DT_BOOL
-  }
 }
 op {
-  name: "QueueIsClosedV2"
+  name: "ReaderResetV2"
   input_arg {
-    name: "handle"
+    name: "reader_handle"
     type: DT_RESOURCE
   }
-  output_arg {
-    name: "is_closed"
-    type: DT_BOOL
-  }
   is_stateful: true
 }
 op {
-  name: "QueueSize"
+  name: "ReaderRestoreState"
   input_arg {
-    name: "handle"
+    name: "reader_handle"
     type: DT_STRING
     is_ref: true
   }
-  output_arg {
-    name: "size"
-    type: DT_INT32
+  input_arg {
+    name: "state"
+    type: DT_STRING
   }
 }
 op {
-  name: "QueueSizeV2"
+  name: "ReaderRestoreStateV2"
   input_arg {
-    name: "handle"
+    name: "reader_handle"
     type: DT_RESOURCE
   }
-  output_arg {
-    name: "size"
-    type: DT_INT32
+  input_arg {
+    name: "state"
+    type: DT_STRING
   }
   is_stateful: true
 }
 op {
-  name: "RFFT"
-  input_arg {
-    name: "input"
-    type: DT_FLOAT
-  }
+  name: "ReaderSerializeState"
   input_arg {
-    name: "fft_length"
-    type: DT_INT32
+    name: "reader_handle"
+    type: DT_STRING
+    is_ref: true
   }
   output_arg {
-    name: "output"
-    type: DT_COMPLEX64
+    name: "state"
+    type: DT_STRING
   }
 }
 op {
-  name: "RFFT2D"
-  input_arg {
-    name: "input"
-    type: DT_FLOAT
-  }
+  name: "ReaderSerializeStateV2"
   input_arg {
-    name: "fft_length"
-    type: DT_INT32
+    name: "reader_handle"
+    type: DT_RESOURCE
   }
   output_arg {
-    name: "output"
-    type: DT_COMPLEX64
+    name: "state"
+    type: DT_STRING
   }
+  is_stateful: true
 }
 op {
-  name: "RFFT3D"
+  name: "Real"
   input_arg {
     name: "input"
-    type: DT_FLOAT
-  }
-  input_arg {
-    name: "fft_length"
-    type: DT_INT32
-  }
-  output_arg {
-    name: "output"
-    type: DT_COMPLEX64
-  }
-}
-op {
-  name: "RGBToHSV"
-  input_arg {
-    name: "images"
     type_attr: "T"
   }
   output_arg {
     name: "output"
-    type_attr: "T"
+    type_attr: "Tout"
   }
   attr {
     name: "T"
     type: "type"
+    default_value {
+      type: DT_COMPLEX64
+    }
+    allowed_values {
+      list {
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+      }
+    }
+  }
+  attr {
+    name: "Tout"
+    type: "type"
     default_value {
       type: DT_FLOAT
     }
@@ -20207,17 +24010,17 @@ op {
   }
 }
 op {
-  name: "RandomCrop"
+  name: "RealDiv"
   input_arg {
-    name: "image"
+    name: "x"
     type_attr: "T"
   }
   input_arg {
-    name: "size"
-    type: DT_INT64
+    name: "y"
+    type_attr: "T"
   }
   output_arg {
-    name: "output"
+    name: "z"
     type_attr: "T"
   }
   attr {
@@ -20225,73 +24028,31 @@ op {
     type: "type"
     allowed_values {
       list {
+        type: DT_HALF
+        type: DT_FLOAT
+        type: DT_DOUBLE
         type: DT_UINT8
         type: DT_INT8
+        type: DT_UINT16
         type: DT_INT16
         type: DT_INT32
         type: DT_INT64
-        type: DT_FLOAT
-        type: DT_DOUBLE
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
       }
     }
   }
-  attr {
-    name: "seed"
-    type: "int"
-    default_value {
-      i: 0
-    }
-  }
-  attr {
-    name: "seed2"
-    type: "int"
-    default_value {
-      i: 0
-    }
-  }
-  deprecation {
-    version: 8
-  }
-  is_stateful: true
 }
 op {
-  name: "RandomGamma"
-  input_arg {
-    name: "shape"
-    type_attr: "S"
-  }
+  name: "Reciprocal"
   input_arg {
-    name: "alpha"
+    name: "x"
     type_attr: "T"
   }
   output_arg {
-    name: "output"
+    name: "y"
     type_attr: "T"
   }
-  attr {
-    name: "seed"
-    type: "int"
-    default_value {
-      i: 0
-    }
-  }
-  attr {
-    name: "seed2"
-    type: "int"
-    default_value {
-      i: 0
-    }
-  }
-  attr {
-    name: "S"
-    type: "type"
-    allowed_values {
-      list {
-        type: DT_INT32
-        type: DT_INT64
-      }
-    }
-  }
   attr {
     name: "T"
     type: "type"
@@ -20300,832 +24061,892 @@ op {
         type: DT_HALF
         type: DT_FLOAT
         type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
       }
     }
   }
-  is_stateful: true
 }
 op {
-  name: "RandomPoisson"
+  name: "ReciprocalGrad"
   input_arg {
-    name: "shape"
-    type_attr: "S"
+    name: "x"
+    type_attr: "T"
   }
   input_arg {
-    name: "rate"
-    type_attr: "dtype"
+    name: "y"
+    type_attr: "T"
   }
   output_arg {
-    name: "output"
-    type_attr: "dtype"
-  }
-  attr {
-    name: "seed"
-    type: "int"
-    default_value {
-      i: 0
-    }
-  }
-  attr {
-    name: "seed2"
-    type: "int"
-    default_value {
-      i: 0
-    }
-  }
-  attr {
-    name: "S"
-    type: "type"
-    allowed_values {
-      list {
-        type: DT_INT32
-        type: DT_INT64
-      }
-    }
+    name: "z"
+    type_attr: "T"
   }
   attr {
-    name: "dtype"
+    name: "T"
     type: "type"
     allowed_values {
       list {
         type: DT_HALF
         type: DT_FLOAT
         type: DT_DOUBLE
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
       }
     }
   }
-  is_stateful: true
 }
 op {
-  name: "RandomPoissonV2"
+  name: "ReciprocalGrad"
   input_arg {
-    name: "shape"
-    type_attr: "S"
+    name: "y"
+    type_attr: "T"
   }
   input_arg {
-    name: "rate"
-    type_attr: "R"
+    name: "dy"
+    type_attr: "T"
   }
   output_arg {
-    name: "output"
-    type_attr: "dtype"
-  }
-  attr {
-    name: "seed"
-    type: "int"
-    default_value {
-      i: 0
-    }
-  }
-  attr {
-    name: "seed2"
-    type: "int"
-    default_value {
-      i: 0
-    }
-  }
-  attr {
-    name: "S"
-    type: "type"
-    allowed_values {
-      list {
-        type: DT_INT32
-        type: DT_INT64
-      }
-    }
-  }
-  attr {
-    name: "R"
-    type: "type"
-    default_value {
-      type: DT_DOUBLE
-    }
-    allowed_values {
-      list {
-        type: DT_HALF
-        type: DT_FLOAT
-        type: DT_DOUBLE
-        type: DT_INT32
-        type: DT_INT64
-      }
-    }
+    name: "z"
+    type_attr: "T"
   }
   attr {
-    name: "dtype"
+    name: "T"
     type: "type"
-    default_value {
-      type: DT_INT64
-    }
     allowed_values {
       list {
         type: DT_HALF
         type: DT_FLOAT
         type: DT_DOUBLE
-        type: DT_INT32
-        type: DT_INT64
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
       }
     }
   }
-  is_stateful: true
-}
-op {
-  name: "RandomShuffle"
-  input_arg {
-    name: "value"
-    type_attr: "T"
-  }
-  output_arg {
-    name: "output"
-    type_attr: "T"
-  }
-  attr {
-    name: "seed"
-    type: "int"
-    default_value {
-      i: 0
-    }
-  }
-  attr {
-    name: "seed2"
-    type: "int"
-    default_value {
-      i: 0
-    }
-  }
-  attr {
-    name: "T"
-    type: "type"
-  }
-  is_stateful: true
 }
 op {
-  name: "RandomShuffleQueue"
+  name: "RecordInput"
   output_arg {
-    name: "handle"
+    name: "records"
     type: DT_STRING
-    is_ref: true
-  }
-  attr {
-    name: "component_types"
-    type: "list(type)"
-    has_minimum: true
-    minimum: 1
   }
   attr {
-    name: "shapes"
-    type: "list(shape)"
-    default_value {
-      list {
-      }
-    }
-    has_minimum: true
+    name: "file_pattern"
+    type: "string"
   }
   attr {
-    name: "capacity"
+    name: "file_random_seed"
     type: "int"
     default_value {
-      i: -1
+      i: 301
     }
   }
   attr {
-    name: "min_after_dequeue"
-    type: "int"
+    name: "file_shuffle_shift_ratio"
+    type: "float"
     default_value {
-      i: 0
+      f: 0
     }
   }
   attr {
-    name: "seed"
+    name: "file_buffer_size"
     type: "int"
     default_value {
-      i: 0
+      i: 10000
     }
   }
   attr {
-    name: "seed2"
+    name: "file_parallelism"
     type: "int"
     default_value {
-      i: 0
-    }
-  }
-  attr {
-    name: "container"
-    type: "string"
-    default_value {
-      s: ""
+      i: 16
     }
   }
   attr {
-    name: "shared_name"
-    type: "string"
+    name: "batch_size"
+    type: "int"
     default_value {
-      s: ""
+      i: 32
     }
   }
   is_stateful: true
 }
 op {
-  name: "RandomShuffleQueueV2"
-  output_arg {
-    name: "handle"
-    type: DT_RESOURCE
-  }
-  attr {
-    name: "component_types"
-    type: "list(type)"
-    has_minimum: true
-    minimum: 1
-  }
-  attr {
-    name: "shapes"
-    type: "list(shape)"
-    default_value {
-      list {
-      }
-    }
-    has_minimum: true
-  }
-  attr {
-    name: "capacity"
-    type: "int"
-    default_value {
-      i: -1
-    }
-  }
-  attr {
-    name: "min_after_dequeue"
-    type: "int"
-    default_value {
-      i: 0
-    }
+  name: "ReduceJoin"
+  input_arg {
+    name: "inputs"
+    type: DT_STRING
   }
-  attr {
-    name: "seed"
-    type: "int"
-    default_value {
-      i: 0
-    }
+  input_arg {
+    name: "reduction_indices"
+    type: DT_INT32
   }
-  attr {
-    name: "seed2"
-    type: "int"
-    default_value {
-      i: 0
-    }
+  output_arg {
+    name: "output"
+    type: DT_STRING
   }
   attr {
-    name: "container"
-    type: "string"
+    name: "keep_dims"
+    type: "bool"
     default_value {
-      s: ""
+      b: false
     }
   }
   attr {
-    name: "shared_name"
+    name: "separator"
     type: "string"
     default_value {
       s: ""
     }
   }
-  is_stateful: true
 }
 op {
-  name: "RandomStandardNormal"
+  name: "RefEnter"
   input_arg {
-    name: "shape"
+    name: "data"
     type_attr: "T"
+    is_ref: true
   }
   output_arg {
     name: "output"
-    type_attr: "dtype"
+    type_attr: "T"
+    is_ref: true
   }
   attr {
-    name: "seed"
-    type: "int"
+    name: "T"
+    type: "type"
+  }
+  attr {
+    name: "frame_name"
+    type: "string"
+  }
+  attr {
+    name: "is_constant"
+    type: "bool"
     default_value {
-      i: 0
+      b: false
     }
   }
   attr {
-    name: "seed2"
+    name: "parallel_iterations"
     type: "int"
     default_value {
-      i: 0
+      i: 10
     }
   }
+}
+op {
+  name: "RefExit"
+  input_arg {
+    name: "data"
+    type_attr: "T"
+    is_ref: true
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+    is_ref: true
+  }
   attr {
-    name: "dtype"
+    name: "T"
     type: "type"
-    allowed_values {
-      list {
-        type: DT_HALF
-        type: DT_FLOAT
-        type: DT_DOUBLE
-      }
-    }
+  }
+}
+op {
+  name: "RefIdentity"
+  input_arg {
+    name: "input"
+    type_attr: "T"
+    is_ref: true
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+    is_ref: true
   }
   attr {
     name: "T"
     type: "type"
-    allowed_values {
-      list {
-        type: DT_INT32
-        type: DT_INT64
-      }
-    }
   }
-  is_stateful: true
+  allows_uninitialized_input: true
 }
 op {
-  name: "RandomUniform"
+  name: "RefMerge"
   input_arg {
-    name: "shape"
+    name: "inputs"
     type_attr: "T"
+    number_attr: "N"
+    is_ref: true
   }
   output_arg {
     name: "output"
-    type_attr: "dtype"
+    type_attr: "T"
+    is_ref: true
+  }
+  output_arg {
+    name: "value_index"
+    type: DT_INT32
   }
   attr {
-    name: "seed"
-    type: "int"
-    default_value {
-      i: 0
-    }
+    name: "T"
+    type: "type"
   }
   attr {
-    name: "seed2"
+    name: "N"
     type: "int"
-    default_value {
-      i: 0
-    }
+    has_minimum: true
+    minimum: 1
+  }
+}
+op {
+  name: "RefNextIteration"
+  input_arg {
+    name: "data"
+    type_attr: "T"
+    is_ref: true
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+    is_ref: true
   }
   attr {
-    name: "dtype"
+    name: "T"
     type: "type"
-    allowed_values {
-      list {
-        type: DT_HALF
-        type: DT_FLOAT
-        type: DT_DOUBLE
-      }
-    }
+  }
+}
+op {
+  name: "RefSelect"
+  input_arg {
+    name: "index"
+    type: DT_INT32
+  }
+  input_arg {
+    name: "inputs"
+    type_attr: "T"
+    number_attr: "N"
+    is_ref: true
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+    is_ref: true
   }
   attr {
     name: "T"
     type: "type"
-    allowed_values {
-      list {
-        type: DT_INT32
-        type: DT_INT64
-      }
-    }
   }
-  is_stateful: true
+  attr {
+    name: "N"
+    type: "int"
+    has_minimum: true
+    minimum: 1
+  }
 }
 op {
-  name: "RandomUniformInt"
+  name: "RefSwitch"
   input_arg {
-    name: "shape"
+    name: "data"
     type_attr: "T"
+    is_ref: true
   }
   input_arg {
-    name: "minval"
-    type_attr: "Tout"
+    name: "pred"
+    type: DT_BOOL
   }
-  input_arg {
-    name: "maxval"
-    type_attr: "Tout"
+  output_arg {
+    name: "output_false"
+    type_attr: "T"
+    is_ref: true
   }
   output_arg {
-    name: "output"
-    type_attr: "Tout"
+    name: "output_true"
+    type_attr: "T"
+    is_ref: true
   }
   attr {
-    name: "seed"
-    type: "int"
-    default_value {
-      i: 0
-    }
+    name: "T"
+    type: "type"
   }
-  attr {
-    name: "seed2"
-    type: "int"
-    default_value {
-      i: 0
-    }
+  allows_uninitialized_input: true
+}
+op {
+  name: "Relu"
+  input_arg {
+    name: "features"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "activations"
+    type_attr: "T"
   }
   attr {
-    name: "Tout"
+    name: "T"
     type: "type"
     allowed_values {
       list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
         type: DT_INT32
         type: DT_INT64
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_UINT16
+        type: DT_HALF
       }
     }
   }
+}
+op {
+  name: "Relu"
+  input_arg {
+    name: "features"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "activations"
+    type_attr: "T"
+  }
   attr {
     name: "T"
     type: "type"
     allowed_values {
       list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
         type: DT_INT32
         type: DT_INT64
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_UINT16
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
-  is_stateful: true
 }
 op {
-  name: "Range"
-  input_arg {
-    name: "start"
-    type_attr: "Tidx"
-  }
-  input_arg {
-    name: "limit"
-    type_attr: "Tidx"
-  }
+  name: "Relu6"
   input_arg {
-    name: "delta"
-    type_attr: "Tidx"
+    name: "features"
+    type_attr: "T"
   }
   output_arg {
-    name: "output"
-    type_attr: "Tidx"
+    name: "activations"
+    type_attr: "T"
   }
   attr {
-    name: "Tidx"
+    name: "T"
     type: "type"
-    default_value {
-      type: DT_INT32
-    }
     allowed_values {
       list {
         type: DT_FLOAT
         type: DT_DOUBLE
         type: DT_INT32
         type: DT_INT64
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_UINT16
+        type: DT_HALF
       }
     }
   }
 }
 op {
-  name: "RangeDataset"
-  input_arg {
-    name: "start"
-    type: DT_INT64
-  }
-  input_arg {
-    name: "stop"
-    type: DT_INT64
-  }
+  name: "Relu6"
   input_arg {
-    name: "step"
-    type: DT_INT64
+    name: "features"
+    type_attr: "T"
   }
   output_arg {
-    name: "handle"
-    type: DT_VARIANT
-  }
-  attr {
-    name: "output_types"
-    type: "list(type)"
-    has_minimum: true
-    minimum: 1
+    name: "activations"
+    type_attr: "T"
   }
   attr {
-    name: "output_shapes"
-    type: "list(shape)"
-    has_minimum: true
-    minimum: 1
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_UINT16
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
   }
-  is_stateful: true
 }
 op {
-  name: "Rank"
+  name: "Relu6Grad"
   input_arg {
-    name: "input"
+    name: "gradients"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "features"
     type_attr: "T"
   }
   output_arg {
-    name: "output"
-    type: DT_INT32
+    name: "backprops"
+    type_attr: "T"
   }
   attr {
     name: "T"
     type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_UINT16
+        type: DT_HALF
+      }
+    }
   }
 }
 op {
-  name: "ReadFile"
+  name: "Relu6Grad"
   input_arg {
-    name: "filename"
-    type: DT_STRING
-  }
-  output_arg {
-    name: "contents"
-    type: DT_STRING
+    name: "gradients"
+    type_attr: "T"
   }
-}
-op {
-  name: "ReadVariableOp"
   input_arg {
-    name: "resource"
-    type: DT_RESOURCE
+    name: "features"
+    type_attr: "T"
   }
   output_arg {
-    name: "value"
-    type_attr: "dtype"
+    name: "backprops"
+    type_attr: "T"
   }
   attr {
-    name: "dtype"
+    name: "T"
     type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_UINT16
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
   }
-  is_stateful: true
 }
 op {
-  name: "ReaderNumRecordsProduced"
+  name: "ReluGrad"
   input_arg {
-    name: "reader_handle"
-    type: DT_STRING
-    is_ref: true
-  }
-  output_arg {
-    name: "records_produced"
-    type: DT_INT64
+    name: "gradients"
+    type_attr: "T"
   }
-}
-op {
-  name: "ReaderNumRecordsProducedV2"
   input_arg {
-    name: "reader_handle"
-    type: DT_RESOURCE
+    name: "features"
+    type_attr: "T"
   }
   output_arg {
-    name: "records_produced"
-    type: DT_INT64
+    name: "backprops"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_UINT16
+        type: DT_HALF
+      }
+    }
   }
-  is_stateful: true
 }
 op {
-  name: "ReaderNumWorkUnitsCompleted"
+  name: "ReluGrad"
   input_arg {
-    name: "reader_handle"
-    type: DT_STRING
-    is_ref: true
-  }
-  output_arg {
-    name: "units_completed"
-    type: DT_INT64
+    name: "gradients"
+    type_attr: "T"
   }
-}
-op {
-  name: "ReaderNumWorkUnitsCompletedV2"
   input_arg {
-    name: "reader_handle"
-    type: DT_RESOURCE
+    name: "features"
+    type_attr: "T"
   }
   output_arg {
-    name: "units_completed"
-    type: DT_INT64
+    name: "backprops"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_UINT16
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
   }
-  is_stateful: true
 }
 op {
-  name: "ReaderRead"
+  name: "RemoteCall"
   input_arg {
-    name: "reader_handle"
+    name: "target"
     type: DT_STRING
-    is_ref: true
   }
   input_arg {
-    name: "queue_handle"
-    type: DT_STRING
-    is_ref: true
+    name: "args"
+    type_list_attr: "Tin"
   }
   output_arg {
-    name: "key"
-    type: DT_STRING
+    name: "output"
+    type_list_attr: "Tout"
   }
-  output_arg {
-    name: "value"
-    type: DT_STRING
+  attr {
+    name: "Tin"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
   }
-}
-op {
-  name: "ReaderReadUpTo"
-  input_arg {
-    name: "reader_handle"
-    type: DT_STRING
-    is_ref: true
+  attr {
+    name: "Tout"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
   }
-  input_arg {
-    name: "queue_handle"
-    type: DT_STRING
-    is_ref: true
+  attr {
+    name: "f"
+    type: "func"
   }
+}
+op {
+  name: "RemoteFusedGraphExecute"
   input_arg {
-    name: "num_records"
-    type: DT_INT64
+    name: "inputs"
+    type_list_attr: "Tinputs"
   }
   output_arg {
-    name: "keys"
-    type: DT_STRING
+    name: "outputs"
+    type_list_attr: "Toutputs"
   }
-  output_arg {
-    name: "values"
-    type: DT_STRING
+  attr {
+    name: "Tinputs"
+    type: "list(type)"
+    has_minimum: true
   }
-}
-op {
-  name: "ReaderReadUpToV2"
-  input_arg {
-    name: "reader_handle"
-    type: DT_RESOURCE
+  attr {
+    name: "Toutputs"
+    type: "list(type)"
+    has_minimum: true
+  }
+  attr {
+    name: "serialized_remote_fused_graph_execute_info"
+    type: "string"
   }
+}
+op {
+  name: "RepeatDataset"
   input_arg {
-    name: "queue_handle"
-    type: DT_RESOURCE
+    name: "input_dataset"
+    type: DT_VARIANT
   }
   input_arg {
-    name: "num_records"
+    name: "count"
     type: DT_INT64
   }
   output_arg {
-    name: "keys"
-    type: DT_STRING
+    name: "handle"
+    type: DT_VARIANT
   }
-  output_arg {
-    name: "values"
-    type: DT_STRING
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
   }
   is_stateful: true
 }
 op {
-  name: "ReaderReadV2"
+  name: "RepeatDataset"
   input_arg {
-    name: "reader_handle"
-    type: DT_RESOURCE
+    name: "input_dataset"
+    type: DT_VARIANT
   }
   input_arg {
-    name: "queue_handle"
-    type: DT_RESOURCE
+    name: "count"
+    type: DT_INT64
   }
   output_arg {
-    name: "key"
-    type: DT_STRING
+    name: "handle"
+    type: DT_VARIANT
   }
-  output_arg {
-    name: "value"
-    type: DT_STRING
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
   }
-  is_stateful: true
 }
 op {
-  name: "ReaderReset"
+  name: "RequantizationRange"
   input_arg {
-    name: "reader_handle"
-    type: DT_STRING
-    is_ref: true
+    name: "input"
+    type_attr: "Tinput"
   }
-}
-op {
-  name: "ReaderResetV2"
   input_arg {
-    name: "reader_handle"
-    type: DT_RESOURCE
+    name: "input_min"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "input_max"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "output_min"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "output_max"
+    type: DT_FLOAT
+  }
+  attr {
+    name: "Tinput"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT16
+        type: DT_QUINT16
+        type: DT_QINT32
+      }
+    }
   }
-  is_stateful: true
 }
 op {
-  name: "ReaderRestoreState"
+  name: "Requantize"
   input_arg {
-    name: "reader_handle"
-    type: DT_STRING
-    is_ref: true
+    name: "input"
+    type_attr: "Tinput"
   }
   input_arg {
-    name: "state"
-    type: DT_STRING
+    name: "input_min"
+    type: DT_FLOAT
   }
-}
-op {
-  name: "ReaderRestoreStateV2"
   input_arg {
-    name: "reader_handle"
-    type: DT_RESOURCE
+    name: "input_max"
+    type: DT_FLOAT
   }
   input_arg {
-    name: "state"
-    type: DT_STRING
+    name: "requested_output_min"
+    type: DT_FLOAT
   }
-  is_stateful: true
-}
-op {
-  name: "ReaderSerializeState"
   input_arg {
-    name: "reader_handle"
-    type: DT_STRING
-    is_ref: true
+    name: "requested_output_max"
+    type: DT_FLOAT
   }
   output_arg {
-    name: "state"
-    type: DT_STRING
+    name: "output"
+    type_attr: "out_type"
   }
-}
-op {
-  name: "ReaderSerializeStateV2"
-  input_arg {
-    name: "reader_handle"
-    type: DT_RESOURCE
+  output_arg {
+    name: "output_min"
+    type: DT_FLOAT
   }
   output_arg {
-    name: "state"
-    type: DT_STRING
+    name: "output_max"
+    type: DT_FLOAT
+  }
+  attr {
+    name: "Tinput"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT16
+        type: DT_QUINT16
+        type: DT_QINT32
+      }
+    }
+  }
+  attr {
+    name: "out_type"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT16
+        type: DT_QUINT16
+        type: DT_QINT32
+      }
+    }
   }
-  is_stateful: true
 }
 op {
-  name: "Real"
+  name: "Reshape"
   input_arg {
-    name: "input"
+    name: "tensor"
     type_attr: "T"
   }
+  input_arg {
+    name: "shape"
+    type_attr: "Tshape"
+  }
   output_arg {
     name: "output"
-    type_attr: "Tout"
+    type_attr: "T"
   }
   attr {
     name: "T"
     type: "type"
+  }
+  attr {
+    name: "Tshape"
+    type: "type"
     default_value {
-      type: DT_COMPLEX64
+      type: DT_INT32
     }
     allowed_values {
       list {
-        type: DT_COMPLEX64
-        type: DT_COMPLEX128
+        type: DT_INT32
+        type: DT_INT64
       }
     }
   }
+}
+op {
+  name: "ResizeArea"
+  input_arg {
+    name: "images"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "size"
+    type: DT_INT32
+  }
+  output_arg {
+    name: "resized_images"
+    type: DT_FLOAT
+  }
   attr {
-    name: "Tout"
+    name: "T"
     type: "type"
-    default_value {
-      type: DT_FLOAT
-    }
     allowed_values {
       list {
+        type: DT_UINT8
+        type: DT_INT8
+        type: DT_INT16
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_HALF
         type: DT_FLOAT
         type: DT_DOUBLE
       }
     }
   }
+  attr {
+    name: "align_corners"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
 }
 op {
-  name: "RealDiv"
+  name: "ResizeBicubic"
   input_arg {
-    name: "x"
+    name: "images"
     type_attr: "T"
   }
   input_arg {
-    name: "y"
-    type_attr: "T"
+    name: "size"
+    type: DT_INT32
   }
   output_arg {
-    name: "z"
-    type_attr: "T"
+    name: "resized_images"
+    type: DT_FLOAT
   }
   attr {
     name: "T"
     type: "type"
     allowed_values {
       list {
-        type: DT_HALF
-        type: DT_FLOAT
-        type: DT_DOUBLE
         type: DT_UINT8
         type: DT_INT8
-        type: DT_UINT16
         type: DT_INT16
         type: DT_INT32
         type: DT_INT64
-        type: DT_COMPLEX64
-        type: DT_COMPLEX128
+        type: DT_HALF
+        type: DT_FLOAT
+        type: DT_DOUBLE
       }
     }
   }
+  attr {
+    name: "align_corners"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
 }
 op {
-  name: "Reciprocal"
+  name: "ResizeBicubicGrad"
   input_arg {
-    name: "x"
+    name: "grads"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "original_image"
     type_attr: "T"
   }
   output_arg {
-    name: "y"
+    name: "output"
     type_attr: "T"
   }
   attr {
@@ -21133,57 +24954,69 @@ op {
     type: "type"
     allowed_values {
       list {
-        type: DT_HALF
         type: DT_FLOAT
         type: DT_DOUBLE
-        type: DT_INT32
-        type: DT_INT64
-        type: DT_COMPLEX64
-        type: DT_COMPLEX128
       }
     }
   }
+  attr {
+    name: "align_corners"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
 }
 op {
-  name: "ReciprocalGrad"
+  name: "ResizeBilinear"
   input_arg {
-    name: "x"
+    name: "images"
     type_attr: "T"
   }
   input_arg {
-    name: "y"
-    type_attr: "T"
+    name: "size"
+    type: DT_INT32
   }
   output_arg {
-    name: "z"
-    type_attr: "T"
+    name: "resized_images"
+    type: DT_FLOAT
   }
   attr {
     name: "T"
     type: "type"
     allowed_values {
       list {
+        type: DT_UINT8
+        type: DT_INT8
+        type: DT_INT16
+        type: DT_INT32
+        type: DT_INT64
         type: DT_HALF
         type: DT_FLOAT
         type: DT_DOUBLE
-        type: DT_COMPLEX64
-        type: DT_COMPLEX128
       }
     }
   }
+  attr {
+    name: "align_corners"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
 }
 op {
-  name: "ReciprocalGrad"
+  name: "ResizeBilinearGrad"
   input_arg {
-    name: "y"
-    type_attr: "T"
+    name: "grads"
+    type: DT_FLOAT
   }
   input_arg {
-    name: "dy"
+    name: "original_image"
     type_attr: "T"
   }
   output_arg {
-    name: "z"
+    name: "output"
     type_attr: "T"
   }
   attr {
@@ -21191,271 +25024,417 @@ op {
     type: "type"
     allowed_values {
       list {
-        type: DT_HALF
         type: DT_FLOAT
+        type: DT_HALF
         type: DT_DOUBLE
-        type: DT_COMPLEX64
-        type: DT_COMPLEX128
       }
     }
   }
+  attr {
+    name: "align_corners"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
 }
 op {
-  name: "RecordInput"
+  name: "ResizeNearestNeighbor"
+  input_arg {
+    name: "images"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "size"
+    type: DT_INT32
+  }
   output_arg {
-    name: "records"
-    type: DT_STRING
+    name: "resized_images"
+    type_attr: "T"
   }
   attr {
-    name: "file_pattern"
-    type: "string"
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_UINT8
+        type: DT_INT8
+        type: DT_INT16
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_HALF
+        type: DT_FLOAT
+        type: DT_DOUBLE
+      }
+    }
   }
   attr {
-    name: "file_random_seed"
-    type: "int"
+    name: "align_corners"
+    type: "bool"
     default_value {
-      i: 301
+      b: false
     }
   }
+}
+op {
+  name: "ResizeNearestNeighborGrad"
+  input_arg {
+    name: "grads"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "size"
+    type: DT_INT32
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
   attr {
-    name: "file_shuffle_shift_ratio"
-    type: "float"
-    default_value {
-      f: 0
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_UINT8
+        type: DT_INT8
+        type: DT_INT32
+        type: DT_HALF
+        type: DT_FLOAT
+        type: DT_DOUBLE
+      }
     }
   }
   attr {
-    name: "file_buffer_size"
-    type: "int"
+    name: "align_corners"
+    type: "bool"
     default_value {
-      i: 10000
+      b: false
     }
   }
+}
+op {
+  name: "ResourceApplyAdadelta"
+  input_arg {
+    name: "var"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "accum"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "accum_update"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "lr"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "rho"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "epsilon"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "grad"
+    type_attr: "T"
+  }
   attr {
-    name: "file_parallelism"
-    type: "int"
-    default_value {
-      i: 16
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
+      }
     }
   }
   attr {
-    name: "batch_size"
-    type: "int"
+    name: "use_locking"
+    type: "bool"
     default_value {
-      i: 32
+      b: false
     }
   }
   is_stateful: true
 }
 op {
-  name: "ReduceJoin"
+  name: "ResourceApplyAdadelta"
   input_arg {
-    name: "inputs"
-    type: DT_STRING
+    name: "var"
+    type: DT_RESOURCE
   }
   input_arg {
-    name: "reduction_indices"
-    type: DT_INT32
+    name: "accum"
+    type: DT_RESOURCE
   }
-  output_arg {
-    name: "output"
-    type: DT_STRING
+  input_arg {
+    name: "accum_update"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "lr"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "rho"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "epsilon"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "grad"
+    type_attr: "T"
   }
   attr {
-    name: "keep_dims"
-    type: "bool"
-    default_value {
-      b: false
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
     }
   }
   attr {
-    name: "separator"
-    type: "string"
+    name: "use_locking"
+    type: "bool"
     default_value {
-      s: ""
+      b: false
     }
   }
+  is_stateful: true
 }
 op {
-  name: "RefEnter"
+  name: "ResourceApplyAdagrad"
   input_arg {
-    name: "data"
+    name: "var"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "accum"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "lr"
     type_attr: "T"
-    is_ref: true
   }
-  output_arg {
-    name: "output"
+  input_arg {
+    name: "grad"
     type_attr: "T"
-    is_ref: true
   }
   attr {
     name: "T"
     type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
+      }
+    }
   }
   attr {
-    name: "frame_name"
-    type: "string"
-  }
-  attr {
-    name: "is_constant"
+    name: "use_locking"
     type: "bool"
     default_value {
       b: false
     }
   }
-  attr {
-    name: "parallel_iterations"
-    type: "int"
-    default_value {
-      i: 10
-    }
-  }
+  is_stateful: true
 }
 op {
-  name: "RefExit"
+  name: "ResourceApplyAdagrad"
   input_arg {
-    name: "data"
-    type_attr: "T"
-    is_ref: true
-  }
-  output_arg {
-    name: "output"
-    type_attr: "T"
-    is_ref: true
+    name: "var"
+    type: DT_RESOURCE
   }
-  attr {
-    name: "T"
-    type: "type"
+  input_arg {
+    name: "accum"
+    type: DT_RESOURCE
   }
-}
-op {
-  name: "RefIdentity"
   input_arg {
-    name: "input"
+    name: "lr"
     type_attr: "T"
-    is_ref: true
   }
-  output_arg {
-    name: "output"
+  input_arg {
+    name: "grad"
     type_attr: "T"
-    is_ref: true
   }
   attr {
     name: "T"
     type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
   }
-  allows_uninitialized_input: true
+  attr {
+    name: "use_locking"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
+  is_stateful: true
 }
 op {
-  name: "RefMerge"
+  name: "ResourceApplyAdagradDA"
   input_arg {
-    name: "inputs"
-    type_attr: "T"
-    number_attr: "N"
-    is_ref: true
-  }
-  output_arg {
-    name: "output"
-    type_attr: "T"
-    is_ref: true
-  }
-  output_arg {
-    name: "value_index"
-    type: DT_INT32
+    name: "var"
+    type: DT_RESOURCE
   }
-  attr {
-    name: "T"
-    type: "type"
+  input_arg {
+    name: "gradient_accumulator"
+    type: DT_RESOURCE
   }
-  attr {
-    name: "N"
-    type: "int"
-    has_minimum: true
-    minimum: 1
+  input_arg {
+    name: "gradient_squared_accumulator"
+    type: DT_RESOURCE
   }
-}
-op {
-  name: "RefNextIteration"
   input_arg {
-    name: "data"
+    name: "grad"
     type_attr: "T"
-    is_ref: true
   }
-  output_arg {
-    name: "output"
+  input_arg {
+    name: "lr"
     type_attr: "T"
-    is_ref: true
-  }
-  attr {
-    name: "T"
-    type: "type"
   }
-}
-op {
-  name: "RefSelect"
   input_arg {
-    name: "index"
-    type: DT_INT32
+    name: "l1"
+    type_attr: "T"
   }
   input_arg {
-    name: "inputs"
+    name: "l2"
     type_attr: "T"
-    number_attr: "N"
-    is_ref: true
   }
-  output_arg {
-    name: "output"
-    type_attr: "T"
-    is_ref: true
+  input_arg {
+    name: "global_step"
+    type: DT_INT64
   }
   attr {
     name: "T"
     type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
+      }
+    }
   }
   attr {
-    name: "N"
-    type: "int"
-    has_minimum: true
-    minimum: 1
+    name: "use_locking"
+    type: "bool"
+    default_value {
+      b: false
+    }
   }
+  is_stateful: true
 }
 op {
-  name: "RefSwitch"
+  name: "ResourceApplyAdagradDA"
   input_arg {
-    name: "data"
-    type_attr: "T"
-    is_ref: true
+    name: "var"
+    type: DT_RESOURCE
   }
   input_arg {
-    name: "pred"
-    type: DT_BOOL
+    name: "gradient_accumulator"
+    type: DT_RESOURCE
   }
-  output_arg {
-    name: "output_false"
-    type_attr: "T"
-    is_ref: true
+  input_arg {
+    name: "gradient_squared_accumulator"
+    type: DT_RESOURCE
   }
-  output_arg {
-    name: "output_true"
+  input_arg {
+    name: "grad"
     type_attr: "T"
-    is_ref: true
   }
-  attr {
-    name: "T"
-    type: "type"
+  input_arg {
+    name: "lr"
+    type_attr: "T"
   }
-  allows_uninitialized_input: true
-}
-op {
-  name: "Relu"
   input_arg {
-    name: "features"
+    name: "l1"
     type_attr: "T"
   }
-  output_arg {
-    name: "activations"
+  input_arg {
+    name: "l2"
     type_attr: "T"
   }
+  input_arg {
+    name: "global_step"
+    type: DT_INT64
+  }
   attr {
     name: "T"
     type: "type"
@@ -21463,25 +25442,72 @@ op {
       list {
         type: DT_FLOAT
         type: DT_DOUBLE
-        type: DT_INT32
         type: DT_INT64
+        type: DT_INT32
         type: DT_UINT8
+        type: DT_UINT16
         type: DT_INT16
         type: DT_INT8
-        type: DT_UINT16
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
+  attr {
+    name: "use_locking"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
+  is_stateful: true
 }
 op {
-  name: "Relu6"
+  name: "ResourceApplyAdam"
   input_arg {
-    name: "features"
+    name: "var"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "m"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "v"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "beta1_power"
     type_attr: "T"
   }
-  output_arg {
-    name: "activations"
+  input_arg {
+    name: "beta2_power"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "lr"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "beta1"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "beta2"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "epsilon"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "grad"
     type_attr: "T"
   }
   attr {
@@ -21491,29 +25517,70 @@ op {
       list {
         type: DT_FLOAT
         type: DT_DOUBLE
-        type: DT_INT32
         type: DT_INT64
+        type: DT_INT32
         type: DT_UINT8
+        type: DT_UINT16
         type: DT_INT16
         type: DT_INT8
-        type: DT_UINT16
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
         type: DT_HALF
       }
     }
   }
+  attr {
+    name: "use_locking"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
+  is_stateful: true
 }
 op {
-  name: "Relu6Grad"
+  name: "ResourceApplyAdam"
   input_arg {
-    name: "gradients"
+    name: "var"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "m"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "v"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "beta1_power"
     type_attr: "T"
   }
   input_arg {
-    name: "features"
+    name: "beta2_power"
     type_attr: "T"
   }
-  output_arg {
-    name: "backprops"
+  input_arg {
+    name: "lr"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "beta1"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "beta2"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "epsilon"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "grad"
     type_attr: "T"
   }
   attr {
@@ -21523,29 +25590,77 @@ op {
       list {
         type: DT_FLOAT
         type: DT_DOUBLE
-        type: DT_INT32
         type: DT_INT64
+        type: DT_INT32
         type: DT_UINT8
+        type: DT_UINT16
         type: DT_INT16
         type: DT_INT8
-        type: DT_UINT16
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
         type: DT_HALF
       }
     }
   }
+  attr {
+    name: "use_locking"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
+  attr {
+    name: "use_nesterov"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
+  is_stateful: true
 }
 op {
-  name: "ReluGrad"
+  name: "ResourceApplyAdam"
   input_arg {
-    name: "gradients"
+    name: "var"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "m"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "v"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "beta1_power"
     type_attr: "T"
   }
   input_arg {
-    name: "features"
+    name: "beta2_power"
     type_attr: "T"
   }
-  output_arg {
-    name: "backprops"
+  input_arg {
+    name: "lr"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "beta1"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "beta2"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "epsilon"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "grad"
     type_attr: "T"
   }
   attr {
@@ -21555,345 +25670,463 @@ op {
       list {
         type: DT_FLOAT
         type: DT_DOUBLE
-        type: DT_INT32
         type: DT_INT64
+        type: DT_INT32
         type: DT_UINT8
+        type: DT_UINT16
         type: DT_INT16
         type: DT_INT8
-        type: DT_UINT16
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
-}
-op {
-  name: "RemoteCall"
-  input_arg {
-    name: "target"
-    type: DT_STRING
-  }
-  input_arg {
-    name: "args"
-    type_list_attr: "Tin"
-  }
-  output_arg {
-    name: "output"
-    type_list_attr: "Tout"
-  }
-  attr {
-    name: "Tin"
-    type: "list(type)"
-    has_minimum: true
-    minimum: 1
-  }
   attr {
-    name: "Tout"
-    type: "list(type)"
-    has_minimum: true
-    minimum: 1
+    name: "use_locking"
+    type: "bool"
+    default_value {
+      b: false
+    }
   }
   attr {
-    name: "f"
-    type: "func"
+    name: "use_nesterov"
+    type: "bool"
+    default_value {
+      b: false
+    }
   }
+  is_stateful: true
 }
 op {
-  name: "RemoteFusedGraphExecute"
+  name: "ResourceApplyCenteredRMSProp"
   input_arg {
-    name: "inputs"
-    type_list_attr: "Tinputs"
+    name: "var"
+    type: DT_RESOURCE
   }
-  output_arg {
-    name: "outputs"
-    type_list_attr: "Toutputs"
+  input_arg {
+    name: "mg"
+    type: DT_RESOURCE
   }
-  attr {
-    name: "Tinputs"
-    type: "list(type)"
-    has_minimum: true
+  input_arg {
+    name: "ms"
+    type: DT_RESOURCE
   }
-  attr {
-    name: "Toutputs"
-    type: "list(type)"
-    has_minimum: true
+  input_arg {
+    name: "mom"
+    type: DT_RESOURCE
   }
-  attr {
-    name: "serialized_remote_fused_graph_execute_info"
-    type: "string"
+  input_arg {
+    name: "lr"
+    type_attr: "T"
   }
-}
-op {
-  name: "RepeatDataset"
   input_arg {
-    name: "input_dataset"
-    type: DT_VARIANT
+    name: "rho"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "momentum"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "epsilon"
+    type_attr: "T"
   }
   input_arg {
-    name: "count"
-    type: DT_INT64
-  }
-  output_arg {
-    name: "handle"
-    type: DT_VARIANT
+    name: "grad"
+    type_attr: "T"
   }
   attr {
-    name: "output_types"
-    type: "list(type)"
-    has_minimum: true
-    minimum: 1
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
+      }
+    }
   }
   attr {
-    name: "output_shapes"
-    type: "list(shape)"
-    has_minimum: true
-    minimum: 1
+    name: "use_locking"
+    type: "bool"
+    default_value {
+      b: false
+    }
   }
   is_stateful: true
 }
 op {
-  name: "RepeatDataset"
+  name: "ResourceApplyCenteredRMSProp"
   input_arg {
-    name: "input_dataset"
-    type: DT_VARIANT
+    name: "var"
+    type: DT_RESOURCE
   }
   input_arg {
-    name: "count"
-    type: DT_INT64
-  }
-  output_arg {
-    name: "handle"
-    type: DT_VARIANT
+    name: "mg"
+    type: DT_RESOURCE
   }
-  attr {
-    name: "output_types"
-    type: "list(type)"
-    has_minimum: true
-    minimum: 1
+  input_arg {
+    name: "ms"
+    type: DT_RESOURCE
   }
-  attr {
-    name: "output_shapes"
-    type: "list(shape)"
-    has_minimum: true
-    minimum: 1
+  input_arg {
+    name: "mom"
+    type: DT_RESOURCE
   }
-}
-op {
-  name: "RequantizationRange"
   input_arg {
-    name: "input"
-    type_attr: "Tinput"
+    name: "lr"
+    type_attr: "T"
   }
   input_arg {
-    name: "input_min"
-    type: DT_FLOAT
+    name: "rho"
+    type_attr: "T"
   }
   input_arg {
-    name: "input_max"
-    type: DT_FLOAT
+    name: "momentum"
+    type_attr: "T"
   }
-  output_arg {
-    name: "output_min"
-    type: DT_FLOAT
+  input_arg {
+    name: "epsilon"
+    type_attr: "T"
   }
-  output_arg {
-    name: "output_max"
-    type: DT_FLOAT
+  input_arg {
+    name: "grad"
+    type_attr: "T"
   }
   attr {
-    name: "Tinput"
+    name: "T"
     type: "type"
     allowed_values {
       list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
         type: DT_QINT8
         type: DT_QUINT8
-        type: DT_QINT16
-        type: DT_QUINT16
         type: DT_QINT32
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
+  attr {
+    name: "use_locking"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
+  is_stateful: true
 }
 op {
-  name: "Requantize"
+  name: "ResourceApplyFtrl"
   input_arg {
-    name: "input"
-    type_attr: "Tinput"
+    name: "var"
+    type: DT_RESOURCE
   }
   input_arg {
-    name: "input_min"
-    type: DT_FLOAT
+    name: "accum"
+    type: DT_RESOURCE
   }
   input_arg {
-    name: "input_max"
-    type: DT_FLOAT
+    name: "linear"
+    type: DT_RESOURCE
   }
   input_arg {
-    name: "requested_output_min"
-    type: DT_FLOAT
+    name: "grad"
+    type_attr: "T"
   }
   input_arg {
-    name: "requested_output_max"
-    type: DT_FLOAT
+    name: "lr"
+    type_attr: "T"
   }
-  output_arg {
-    name: "output"
-    type_attr: "out_type"
+  input_arg {
+    name: "l1"
+    type_attr: "T"
   }
-  output_arg {
-    name: "output_min"
-    type: DT_FLOAT
+  input_arg {
+    name: "l2"
+    type_attr: "T"
   }
-  output_arg {
-    name: "output_max"
-    type: DT_FLOAT
+  input_arg {
+    name: "lr_power"
+    type_attr: "T"
   }
   attr {
-    name: "Tinput"
+    name: "T"
     type: "type"
     allowed_values {
       list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
         type: DT_QINT8
         type: DT_QUINT8
-        type: DT_QINT16
-        type: DT_QUINT16
         type: DT_QINT32
+        type: DT_HALF
       }
     }
   }
   attr {
-    name: "out_type"
-    type: "type"
-    allowed_values {
-      list {
-        type: DT_QINT8
-        type: DT_QUINT8
-        type: DT_QINT16
-        type: DT_QUINT16
-        type: DT_QINT32
-      }
+    name: "use_locking"
+    type: "bool"
+    default_value {
+      b: false
     }
   }
+  is_stateful: true
 }
 op {
-  name: "Reshape"
+  name: "ResourceApplyFtrl"
   input_arg {
-    name: "tensor"
+    name: "var"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "accum"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "linear"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "grad"
     type_attr: "T"
   }
   input_arg {
-    name: "shape"
-    type_attr: "Tshape"
+    name: "lr"
+    type_attr: "T"
   }
-  output_arg {
-    name: "output"
+  input_arg {
+    name: "l1"
     type_attr: "T"
   }
-  attr {
-    name: "T"
-    type: "type"
+  input_arg {
+    name: "l2"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "lr_power"
+    type_attr: "T"
   }
   attr {
-    name: "Tshape"
+    name: "T"
     type: "type"
-    default_value {
-      type: DT_INT32
-    }
     allowed_values {
       list {
-        type: DT_INT32
+        type: DT_FLOAT
+        type: DT_DOUBLE
         type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
+  attr {
+    name: "use_locking"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
+  is_stateful: true
 }
 op {
-  name: "ResizeArea"
+  name: "ResourceApplyFtrlV2"
   input_arg {
-    name: "images"
+    name: "var"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "accum"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "linear"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "grad"
     type_attr: "T"
   }
   input_arg {
-    name: "size"
-    type: DT_INT32
+    name: "lr"
+    type_attr: "T"
   }
-  output_arg {
-    name: "resized_images"
-    type: DT_FLOAT
+  input_arg {
+    name: "l1"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "l2"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "l2_shrinkage"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "lr_power"
+    type_attr: "T"
   }
   attr {
     name: "T"
     type: "type"
     allowed_values {
       list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
         type: DT_UINT8
-        type: DT_INT8
+        type: DT_UINT16
         type: DT_INT16
-        type: DT_INT32
-        type: DT_INT64
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
         type: DT_HALF
-        type: DT_FLOAT
-        type: DT_DOUBLE
       }
     }
   }
   attr {
-    name: "align_corners"
+    name: "use_locking"
     type: "bool"
     default_value {
       b: false
     }
   }
+  is_stateful: true
 }
 op {
-  name: "ResizeBicubic"
+  name: "ResourceApplyFtrlV2"
   input_arg {
-    name: "images"
+    name: "var"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "accum"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "linear"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "grad"
     type_attr: "T"
   }
   input_arg {
-    name: "size"
-    type: DT_INT32
+    name: "lr"
+    type_attr: "T"
   }
-  output_arg {
-    name: "resized_images"
-    type: DT_FLOAT
+  input_arg {
+    name: "l1"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "l2"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "l2_shrinkage"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "lr_power"
+    type_attr: "T"
   }
   attr {
     name: "T"
     type: "type"
     allowed_values {
       list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
         type: DT_UINT8
-        type: DT_INT8
+        type: DT_UINT16
         type: DT_INT16
-        type: DT_INT32
-        type: DT_INT64
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
         type: DT_HALF
-        type: DT_FLOAT
-        type: DT_DOUBLE
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
   attr {
-    name: "align_corners"
+    name: "use_locking"
     type: "bool"
     default_value {
       b: false
     }
   }
+  is_stateful: true
 }
 op {
-  name: "ResizeBicubicGrad"
+  name: "ResourceApplyGradientDescent"
   input_arg {
-    name: "grads"
-    type: DT_FLOAT
+    name: "var"
+    type: DT_RESOURCE
   }
   input_arg {
-    name: "original_image"
+    name: "alpha"
     type_attr: "T"
   }
-  output_arg {
-    name: "output"
+  input_arg {
+    name: "delta"
     type_attr: "T"
   }
   attr {
@@ -21903,67 +26136,97 @@ op {
       list {
         type: DT_FLOAT
         type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
       }
     }
   }
   attr {
-    name: "align_corners"
+    name: "use_locking"
     type: "bool"
     default_value {
       b: false
     }
   }
+  is_stateful: true
 }
 op {
-  name: "ResizeBilinear"
+  name: "ResourceApplyGradientDescent"
   input_arg {
-    name: "images"
-    type_attr: "T"
+    name: "var"
+    type: DT_RESOURCE
   }
   input_arg {
-    name: "size"
-    type: DT_INT32
+    name: "alpha"
+    type_attr: "T"
   }
-  output_arg {
-    name: "resized_images"
-    type: DT_FLOAT
+  input_arg {
+    name: "delta"
+    type_attr: "T"
   }
   attr {
     name: "T"
     type: "type"
     allowed_values {
       list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
         type: DT_UINT8
-        type: DT_INT8
+        type: DT_UINT16
         type: DT_INT16
-        type: DT_INT32
-        type: DT_INT64
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
         type: DT_HALF
-        type: DT_FLOAT
-        type: DT_DOUBLE
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
   attr {
-    name: "align_corners"
+    name: "use_locking"
     type: "bool"
     default_value {
       b: false
     }
   }
+  is_stateful: true
 }
 op {
-  name: "ResizeBilinearGrad"
+  name: "ResourceApplyMomentum"
   input_arg {
-    name: "grads"
-    type: DT_FLOAT
+    name: "var"
+    type: DT_RESOURCE
   }
   input_arg {
-    name: "original_image"
+    name: "accum"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "lr"
     type_attr: "T"
   }
-  output_arg {
-    name: "output"
+  input_arg {
+    name: "grad"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "momentum"
     type_attr: "T"
   }
   attr {
@@ -21972,31 +26235,58 @@ op {
     allowed_values {
       list {
         type: DT_FLOAT
-        type: DT_HALF
         type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
       }
     }
   }
   attr {
-    name: "align_corners"
+    name: "use_locking"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
+  attr {
+    name: "use_nesterov"
     type: "bool"
     default_value {
       b: false
     }
   }
+  is_stateful: true
 }
 op {
-  name: "ResizeNearestNeighbor"
+  name: "ResourceApplyMomentum"
   input_arg {
-    name: "images"
+    name: "var"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "accum"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "lr"
     type_attr: "T"
   }
   input_arg {
-    name: "size"
-    type: DT_INT32
+    name: "grad"
+    type_attr: "T"
   }
-  output_arg {
-    name: "resized_images"
+  input_arg {
+    name: "momentum"
     type_attr: "T"
   }
   attr {
@@ -22004,37 +26294,65 @@ op {
     type: "type"
     allowed_values {
       list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
         type: DT_UINT8
-        type: DT_INT8
+        type: DT_UINT16
         type: DT_INT16
-        type: DT_INT32
-        type: DT_INT64
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
         type: DT_HALF
-        type: DT_FLOAT
-        type: DT_DOUBLE
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
   attr {
-    name: "align_corners"
+    name: "use_locking"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
+  attr {
+    name: "use_nesterov"
     type: "bool"
     default_value {
       b: false
     }
   }
+  is_stateful: true
 }
 op {
-  name: "ResizeNearestNeighborGrad"
+  name: "ResourceApplyProximalAdagrad"
   input_arg {
-    name: "grads"
+    name: "var"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "accum"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "lr"
     type_attr: "T"
   }
   input_arg {
-    name: "size"
-    type: DT_INT32
+    name: "l1"
+    type_attr: "T"
   }
-  output_arg {
-    name: "output"
+  input_arg {
+    name: "l2"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "grad"
     type_attr: "T"
   }
   attr {
@@ -22042,25 +26360,34 @@ op {
     type: "type"
     allowed_values {
       list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
         type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
         type: DT_INT8
-        type: DT_INT32
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
         type: DT_HALF
-        type: DT_FLOAT
-        type: DT_DOUBLE
       }
     }
   }
   attr {
-    name: "align_corners"
+    name: "use_locking"
     type: "bool"
     default_value {
       b: false
     }
   }
+  is_stateful: true
 }
 op {
-  name: "ResourceApplyAdadelta"
+  name: "ResourceApplyProximalAdagrad"
   input_arg {
     name: "var"
     type: DT_RESOURCE
@@ -22069,20 +26396,16 @@ op {
     name: "accum"
     type: DT_RESOURCE
   }
-  input_arg {
-    name: "accum_update"
-    type: DT_RESOURCE
-  }
   input_arg {
     name: "lr"
     type_attr: "T"
   }
   input_arg {
-    name: "rho"
+    name: "l1"
     type_attr: "T"
   }
   input_arg {
-    name: "epsilon"
+    name: "l2"
     type_attr: "T"
   }
   input_arg {
@@ -22108,6 +26431,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -22121,21 +26446,25 @@ op {
   is_stateful: true
 }
 op {
-  name: "ResourceApplyAdagrad"
+  name: "ResourceApplyProximalGradientDescent"
   input_arg {
     name: "var"
     type: DT_RESOURCE
   }
   input_arg {
-    name: "accum"
-    type: DT_RESOURCE
+    name: "alpha"
+    type_attr: "T"
   }
   input_arg {
-    name: "lr"
+    name: "l1"
     type_attr: "T"
   }
   input_arg {
-    name: "grad"
+    name: "l2"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "delta"
     type_attr: "T"
   }
   attr {
@@ -22170,25 +26499,13 @@ op {
   is_stateful: true
 }
 op {
-  name: "ResourceApplyAdagradDA"
+  name: "ResourceApplyProximalGradientDescent"
   input_arg {
     name: "var"
     type: DT_RESOURCE
   }
   input_arg {
-    name: "gradient_accumulator"
-    type: DT_RESOURCE
-  }
-  input_arg {
-    name: "gradient_squared_accumulator"
-    type: DT_RESOURCE
-  }
-  input_arg {
-    name: "grad"
-    type_attr: "T"
-  }
-  input_arg {
-    name: "lr"
+    name: "alpha"
     type_attr: "T"
   }
   input_arg {
@@ -22200,8 +26517,8 @@ op {
     type_attr: "T"
   }
   input_arg {
-    name: "global_step"
-    type: DT_INT64
+    name: "delta"
+    type_attr: "T"
   }
   attr {
     name: "T"
@@ -22222,6 +26539,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -22235,37 +26554,29 @@ op {
   is_stateful: true
 }
 op {
-  name: "ResourceApplyAdam"
+  name: "ResourceApplyRMSProp"
   input_arg {
     name: "var"
     type: DT_RESOURCE
   }
   input_arg {
-    name: "m"
+    name: "ms"
     type: DT_RESOURCE
   }
   input_arg {
-    name: "v"
+    name: "mom"
     type: DT_RESOURCE
   }
-  input_arg {
-    name: "beta1_power"
-    type_attr: "T"
-  }
-  input_arg {
-    name: "beta2_power"
-    type_attr: "T"
-  }
   input_arg {
     name: "lr"
     type_attr: "T"
   }
   input_arg {
-    name: "beta1"
+    name: "rho"
     type_attr: "T"
   }
   input_arg {
-    name: "beta2"
+    name: "momentum"
     type_attr: "T"
   }
   input_arg {
@@ -22308,37 +26619,29 @@ op {
   is_stateful: true
 }
 op {
-  name: "ResourceApplyAdam"
+  name: "ResourceApplyRMSProp"
   input_arg {
     name: "var"
     type: DT_RESOURCE
   }
   input_arg {
-    name: "m"
+    name: "ms"
     type: DT_RESOURCE
   }
   input_arg {
-    name: "v"
+    name: "mom"
     type: DT_RESOURCE
   }
-  input_arg {
-    name: "beta1_power"
-    type_attr: "T"
-  }
-  input_arg {
-    name: "beta2_power"
-    type_attr: "T"
-  }
   input_arg {
     name: "lr"
     type_attr: "T"
   }
   input_arg {
-    name: "beta1"
+    name: "rho"
     type_attr: "T"
   }
   input_arg {
-    name: "beta2"
+    name: "momentum"
     type_attr: "T"
   }
   input_arg {
@@ -22368,6 +26671,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -22378,55 +26683,109 @@ op {
       b: false
     }
   }
+  is_stateful: true
+}
+op {
+  name: "ResourceGather"
+  input_arg {
+    name: "resource"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "indices"
+    type_attr: "Tindices"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "dtype"
+  }
   attr {
-    name: "use_nesterov"
+    name: "validate_indices"
     type: "bool"
     default_value {
-      b: false
+      b: true
+    }
+  }
+  attr {
+    name: "dtype"
+    type: "type"
+  }
+  attr {
+    name: "Tindices"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
     }
   }
   is_stateful: true
 }
 op {
-  name: "ResourceApplyCenteredRMSProp"
-  input_arg {
-    name: "var"
-    type: DT_RESOURCE
-  }
+  name: "ResourceScatterAdd"
   input_arg {
-    name: "mg"
+    name: "resource"
     type: DT_RESOURCE
   }
   input_arg {
-    name: "ms"
-    type: DT_RESOURCE
+    name: "indices"
+    type_attr: "Tindices"
   }
   input_arg {
-    name: "mom"
-    type: DT_RESOURCE
+    name: "updates"
+    type_attr: "dtype"
   }
-  input_arg {
-    name: "lr"
-    type_attr: "T"
+  attr {
+    name: "dtype"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
+      }
+    }
   }
-  input_arg {
-    name: "rho"
-    type_attr: "T"
+  attr {
+    name: "Tindices"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
   }
+  is_stateful: true
+}
+op {
+  name: "ResourceScatterAdd"
   input_arg {
-    name: "momentum"
-    type_attr: "T"
+    name: "resource"
+    type: DT_RESOURCE
   }
   input_arg {
-    name: "epsilon"
-    type_attr: "T"
+    name: "indices"
+    type_attr: "Tindices"
   }
   input_arg {
-    name: "grad"
-    type_attr: "T"
+    name: "updates"
+    type_attr: "dtype"
   }
   attr {
-    name: "T"
+    name: "dtype"
     type: "type"
     allowed_values {
       list {
@@ -22444,20 +26803,25 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
   attr {
-    name: "use_locking"
-    type: "bool"
-    default_value {
-      b: false
+    name: "Tindices"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
     }
   }
   is_stateful: true
 }
 op {
-  name: "ResourceApplyFtrl"
+  name: "ResourceSparseApplyAdadelta"
   input_arg {
     name: "var"
     type: DT_RESOURCE
@@ -22467,28 +26831,28 @@ op {
     type: DT_RESOURCE
   }
   input_arg {
-    name: "linear"
+    name: "accum_update"
     type: DT_RESOURCE
   }
   input_arg {
-    name: "grad"
+    name: "lr"
     type_attr: "T"
   }
   input_arg {
-    name: "lr"
+    name: "rho"
     type_attr: "T"
   }
   input_arg {
-    name: "l1"
+    name: "epsilon"
     type_attr: "T"
   }
   input_arg {
-    name: "l2"
+    name: "grad"
     type_attr: "T"
   }
   input_arg {
-    name: "lr_power"
-    type_attr: "T"
+    name: "indices"
+    type_attr: "Tindices"
   }
   attr {
     name: "T"
@@ -22512,6 +26876,16 @@ op {
       }
     }
   }
+  attr {
+    name: "Tindices"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
   attr {
     name: "use_locking"
     type: "bool"
@@ -22522,7 +26896,7 @@ op {
   is_stateful: true
 }
 op {
-  name: "ResourceApplyFtrlV2"
+  name: "ResourceSparseApplyAdadelta"
   input_arg {
     name: "var"
     type: DT_RESOURCE
@@ -22532,32 +26906,28 @@ op {
     type: DT_RESOURCE
   }
   input_arg {
-    name: "linear"
+    name: "accum_update"
     type: DT_RESOURCE
   }
-  input_arg {
-    name: "grad"
-    type_attr: "T"
-  }
   input_arg {
     name: "lr"
     type_attr: "T"
   }
   input_arg {
-    name: "l1"
+    name: "rho"
     type_attr: "T"
   }
   input_arg {
-    name: "l2"
+    name: "epsilon"
     type_attr: "T"
   }
   input_arg {
-    name: "l2_shrinkage"
+    name: "grad"
     type_attr: "T"
   }
   input_arg {
-    name: "lr_power"
-    type_attr: "T"
+    name: "indices"
+    type_attr: "Tindices"
   }
   attr {
     name: "T"
@@ -22578,6 +26948,18 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+  attr {
+    name: "Tindices"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
       }
     }
   }
@@ -22591,19 +26973,27 @@ op {
   is_stateful: true
 }
 op {
-  name: "ResourceApplyGradientDescent"
+  name: "ResourceSparseApplyAdagrad"
   input_arg {
     name: "var"
     type: DT_RESOURCE
   }
   input_arg {
-    name: "alpha"
+    name: "accum"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "lr"
     type_attr: "T"
   }
   input_arg {
-    name: "delta"
+    name: "grad"
     type_attr: "T"
   }
+  input_arg {
+    name: "indices"
+    type_attr: "Tindices"
+  }
   attr {
     name: "T"
     type: "type"
@@ -22626,6 +27016,16 @@ op {
       }
     }
   }
+  attr {
+    name: "Tindices"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
   attr {
     name: "use_locking"
     type: "bool"
@@ -22636,7 +27036,7 @@ op {
   is_stateful: true
 }
 op {
-  name: "ResourceApplyMomentum"
+  name: "ResourceSparseApplyAdagrad"
   input_arg {
     name: "var"
     type: DT_RESOURCE
@@ -22654,8 +27054,8 @@ op {
     type_attr: "T"
   }
   input_arg {
-    name: "momentum"
-    type_attr: "T"
+    name: "indices"
+    type_attr: "Tindices"
   }
   attr {
     name: "T"
@@ -22676,18 +27076,23 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
   attr {
-    name: "use_locking"
-    type: "bool"
-    default_value {
-      b: false
+    name: "Tindices"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
     }
   }
   attr {
-    name: "use_nesterov"
+    name: "use_locking"
     type: "bool"
     default_value {
       b: false
@@ -22696,15 +27101,27 @@ op {
   is_stateful: true
 }
 op {
-  name: "ResourceApplyProximalAdagrad"
+  name: "ResourceSparseApplyAdagradDA"
   input_arg {
     name: "var"
     type: DT_RESOURCE
   }
   input_arg {
-    name: "accum"
+    name: "gradient_accumulator"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "gradient_squared_accumulator"
     type: DT_RESOURCE
   }
+  input_arg {
+    name: "grad"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "indices"
+    type_attr: "Tindices"
+  }
   input_arg {
     name: "lr"
     type_attr: "T"
@@ -22718,8 +27135,8 @@ op {
     type_attr: "T"
   }
   input_arg {
-    name: "grad"
-    type_attr: "T"
+    name: "global_step"
+    type: DT_INT64
   }
   attr {
     name: "T"
@@ -22743,6 +27160,16 @@ op {
       }
     }
   }
+  attr {
+    name: "Tindices"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
   attr {
     name: "use_locking"
     type: "bool"
@@ -22753,13 +27180,29 @@ op {
   is_stateful: true
 }
 op {
-  name: "ResourceApplyProximalGradientDescent"
+  name: "ResourceSparseApplyAdagradDA"
   input_arg {
     name: "var"
     type: DT_RESOURCE
   }
   input_arg {
-    name: "alpha"
+    name: "gradient_accumulator"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "gradient_squared_accumulator"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "grad"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "indices"
+    type_attr: "Tindices"
+  }
+  input_arg {
+    name: "lr"
     type_attr: "T"
   }
   input_arg {
@@ -22771,8 +27214,8 @@ op {
     type_attr: "T"
   }
   input_arg {
-    name: "delta"
-    type_attr: "T"
+    name: "global_step"
+    type: DT_INT64
   }
   attr {
     name: "T"
@@ -22793,6 +27236,18 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+  attr {
+    name: "Tindices"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
       }
     }
   }
@@ -22806,11 +27261,15 @@ op {
   is_stateful: true
 }
 op {
-  name: "ResourceApplyRMSProp"
+  name: "ResourceSparseApplyCenteredRMSProp"
   input_arg {
     name: "var"
     type: DT_RESOURCE
   }
+  input_arg {
+    name: "mg"
+    type: DT_RESOURCE
+  }
   input_arg {
     name: "ms"
     type: DT_RESOURCE
@@ -22839,6 +27298,10 @@ op {
     name: "grad"
     type_attr: "T"
   }
+  input_arg {
+    name: "indices"
+    type_attr: "Tindices"
+  }
   attr {
     name: "T"
     type: "type"
@@ -22861,6 +27324,16 @@ op {
       }
     }
   }
+  attr {
+    name: "Tindices"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
   attr {
     name: "use_locking"
     type: "bool"
@@ -22871,58 +27344,49 @@ op {
   is_stateful: true
 }
 op {
-  name: "ResourceGather"
+  name: "ResourceSparseApplyCenteredRMSProp"
   input_arg {
-    name: "resource"
+    name: "var"
     type: DT_RESOURCE
   }
   input_arg {
-    name: "indices"
-    type_attr: "Tindices"
+    name: "mg"
+    type: DT_RESOURCE
   }
-  output_arg {
-    name: "output"
-    type_attr: "dtype"
+  input_arg {
+    name: "ms"
+    type: DT_RESOURCE
   }
-  attr {
-    name: "validate_indices"
-    type: "bool"
-    default_value {
-      b: true
-    }
+  input_arg {
+    name: "mom"
+    type: DT_RESOURCE
   }
-  attr {
-    name: "dtype"
-    type: "type"
+  input_arg {
+    name: "lr"
+    type_attr: "T"
   }
-  attr {
-    name: "Tindices"
-    type: "type"
-    allowed_values {
-      list {
-        type: DT_INT32
-        type: DT_INT64
-      }
-    }
+  input_arg {
+    name: "rho"
+    type_attr: "T"
   }
-  is_stateful: true
-}
-op {
-  name: "ResourceScatterAdd"
   input_arg {
-    name: "resource"
-    type: DT_RESOURCE
+    name: "momentum"
+    type_attr: "T"
   }
   input_arg {
-    name: "indices"
-    type_attr: "Tindices"
+    name: "epsilon"
+    type_attr: "T"
   }
   input_arg {
-    name: "updates"
-    type_attr: "dtype"
+    name: "grad"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "indices"
+    type_attr: "Tindices"
   }
   attr {
-    name: "dtype"
+    name: "T"
     type: "type"
     allowed_values {
       list {
@@ -22940,6 +27404,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -22953,10 +27419,17 @@ op {
       }
     }
   }
+  attr {
+    name: "use_locking"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
   is_stateful: true
 }
 op {
-  name: "ResourceSparseApplyAdadelta"
+  name: "ResourceSparseApplyFtrl"
   input_arg {
     name: "var"
     type: DT_RESOURCE
@@ -22966,28 +27439,32 @@ op {
     type: DT_RESOURCE
   }
   input_arg {
-    name: "accum_update"
+    name: "linear"
     type: DT_RESOURCE
   }
   input_arg {
-    name: "lr"
+    name: "grad"
     type_attr: "T"
   }
   input_arg {
-    name: "rho"
+    name: "indices"
+    type_attr: "Tindices"
+  }
+  input_arg {
+    name: "lr"
     type_attr: "T"
   }
   input_arg {
-    name: "epsilon"
+    name: "l1"
     type_attr: "T"
   }
   input_arg {
-    name: "grad"
+    name: "l2"
     type_attr: "T"
   }
   input_arg {
-    name: "indices"
-    type_attr: "Tindices"
+    name: "lr_power"
+    type_attr: "T"
   }
   attr {
     name: "T"
@@ -23031,7 +27508,7 @@ op {
   is_stateful: true
 }
 op {
-  name: "ResourceSparseApplyAdagrad"
+  name: "ResourceSparseApplyFtrl"
   input_arg {
     name: "var"
     type: DT_RESOURCE
@@ -23041,8 +27518,8 @@ op {
     type: DT_RESOURCE
   }
   input_arg {
-    name: "lr"
-    type_attr: "T"
+    name: "linear"
+    type: DT_RESOURCE
   }
   input_arg {
     name: "grad"
@@ -23052,6 +27529,22 @@ op {
     name: "indices"
     type_attr: "Tindices"
   }
+  input_arg {
+    name: "lr"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "l1"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "l2"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "lr_power"
+    type_attr: "T"
+  }
   attr {
     name: "T"
     type: "type"
@@ -23071,6 +27564,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -23094,17 +27589,17 @@ op {
   is_stateful: true
 }
 op {
-  name: "ResourceSparseApplyAdagradDA"
+  name: "ResourceSparseApplyFtrlV2"
   input_arg {
     name: "var"
     type: DT_RESOURCE
   }
   input_arg {
-    name: "gradient_accumulator"
+    name: "accum"
     type: DT_RESOURCE
   }
   input_arg {
-    name: "gradient_squared_accumulator"
+    name: "linear"
     type: DT_RESOURCE
   }
   input_arg {
@@ -23128,8 +27623,12 @@ op {
     type_attr: "T"
   }
   input_arg {
-    name: "global_step"
-    type: DT_INT64
+    name: "l2_shrinkage"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "lr_power"
+    type_attr: "T"
   }
   attr {
     name: "T"
@@ -23173,46 +27672,46 @@ op {
   is_stateful: true
 }
 op {
-  name: "ResourceSparseApplyCenteredRMSProp"
+  name: "ResourceSparseApplyFtrlV2"
   input_arg {
     name: "var"
     type: DT_RESOURCE
   }
   input_arg {
-    name: "mg"
+    name: "accum"
     type: DT_RESOURCE
   }
   input_arg {
-    name: "ms"
+    name: "linear"
     type: DT_RESOURCE
   }
   input_arg {
-    name: "mom"
-    type: DT_RESOURCE
+    name: "grad"
+    type_attr: "T"
   }
   input_arg {
-    name: "lr"
-    type_attr: "T"
+    name: "indices"
+    type_attr: "Tindices"
   }
   input_arg {
-    name: "rho"
+    name: "lr"
     type_attr: "T"
   }
   input_arg {
-    name: "momentum"
+    name: "l1"
     type_attr: "T"
   }
   input_arg {
-    name: "epsilon"
+    name: "l2"
     type_attr: "T"
   }
   input_arg {
-    name: "grad"
+    name: "l2_shrinkage"
     type_attr: "T"
   }
   input_arg {
-    name: "indices"
-    type_attr: "Tindices"
+    name: "lr_power"
+    type_attr: "T"
   }
   attr {
     name: "T"
@@ -23233,6 +27732,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -23256,7 +27757,7 @@ op {
   is_stateful: true
 }
 op {
-  name: "ResourceSparseApplyFtrl"
+  name: "ResourceSparseApplyMomentum"
   input_arg {
     name: "var"
     type: DT_RESOURCE
@@ -23266,8 +27767,8 @@ op {
     type: DT_RESOURCE
   }
   input_arg {
-    name: "linear"
-    type: DT_RESOURCE
+    name: "lr"
+    type_attr: "T"
   }
   input_arg {
     name: "grad"
@@ -23278,19 +27779,7 @@ op {
     type_attr: "Tindices"
   }
   input_arg {
-    name: "lr"
-    type_attr: "T"
-  }
-  input_arg {
-    name: "l1"
-    type_attr: "T"
-  }
-  input_arg {
-    name: "l2"
-    type_attr: "T"
-  }
-  input_arg {
-    name: "lr_power"
+    name: "momentum"
     type_attr: "T"
   }
   attr {
@@ -23332,10 +27821,17 @@ op {
       b: false
     }
   }
+  attr {
+    name: "use_nesterov"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
   is_stateful: true
 }
 op {
-  name: "ResourceSparseApplyFtrlV2"
+  name: "ResourceSparseApplyMomentum"
   input_arg {
     name: "var"
     type: DT_RESOURCE
@@ -23345,8 +27841,8 @@ op {
     type: DT_RESOURCE
   }
   input_arg {
-    name: "linear"
-    type: DT_RESOURCE
+    name: "lr"
+    type_attr: "T"
   }
   input_arg {
     name: "grad"
@@ -23356,6 +27852,70 @@ op {
     name: "indices"
     type_attr: "Tindices"
   }
+  input_arg {
+    name: "momentum"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+  attr {
+    name: "Tindices"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+  attr {
+    name: "use_locking"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
+  attr {
+    name: "use_nesterov"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
+  is_stateful: true
+}
+op {
+  name: "ResourceSparseApplyProximalAdagrad"
+  input_arg {
+    name: "var"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "accum"
+    type: DT_RESOURCE
+  }
   input_arg {
     name: "lr"
     type_attr: "T"
@@ -23369,12 +27929,12 @@ op {
     type_attr: "T"
   }
   input_arg {
-    name: "l2_shrinkage"
+    name: "grad"
     type_attr: "T"
   }
   input_arg {
-    name: "lr_power"
-    type_attr: "T"
+    name: "indices"
+    type_attr: "Tindices"
   }
   attr {
     name: "T"
@@ -23418,7 +27978,7 @@ op {
   is_stateful: true
 }
 op {
-  name: "ResourceSparseApplyMomentum"
+  name: "ResourceSparseApplyProximalAdagrad"
   input_arg {
     name: "var"
     type: DT_RESOURCE
@@ -23432,17 +27992,21 @@ op {
     type_attr: "T"
   }
   input_arg {
-    name: "grad"
+    name: "l1"
     type_attr: "T"
   }
   input_arg {
-    name: "indices"
-    type_attr: "Tindices"
+    name: "l2"
+    type_attr: "T"
   }
   input_arg {
-    name: "momentum"
+    name: "grad"
     type_attr: "T"
   }
+  input_arg {
+    name: "indices"
+    type_attr: "Tindices"
+  }
   attr {
     name: "T"
     type: "type"
@@ -23462,6 +28026,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -23482,8 +28048,68 @@ op {
       b: false
     }
   }
+  is_stateful: true
+}
+op {
+  name: "ResourceSparseApplyProximalGradientDescent"
+  input_arg {
+    name: "var"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "alpha"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "l1"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "l2"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "grad"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "indices"
+    type_attr: "Tindices"
+  }
   attr {
-    name: "use_nesterov"
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
+      }
+    }
+  }
+  attr {
+    name: "Tindices"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+  attr {
+    name: "use_locking"
     type: "bool"
     default_value {
       b: false
@@ -23492,17 +28118,13 @@ op {
   is_stateful: true
 }
 op {
-  name: "ResourceSparseApplyProximalAdagrad"
+  name: "ResourceSparseApplyProximalGradientDescent"
   input_arg {
     name: "var"
     type: DT_RESOURCE
   }
   input_arg {
-    name: "accum"
-    type: DT_RESOURCE
-  }
-  input_arg {
-    name: "lr"
+    name: "alpha"
     type_attr: "T"
   }
   input_arg {
@@ -23540,6 +28162,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -23563,21 +28187,33 @@ op {
   is_stateful: true
 }
 op {
-  name: "ResourceSparseApplyProximalGradientDescent"
+  name: "ResourceSparseApplyRMSProp"
   input_arg {
     name: "var"
     type: DT_RESOURCE
   }
   input_arg {
-    name: "alpha"
+    name: "ms"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "mom"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "lr"
     type_attr: "T"
   }
   input_arg {
-    name: "l1"
+    name: "rho"
     type_attr: "T"
   }
   input_arg {
-    name: "l2"
+    name: "momentum"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "epsilon"
     type_attr: "T"
   }
   input_arg {
@@ -23686,6 +28322,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -24743,6 +29381,40 @@ op {
     }
   }
 }
+op {
+  name: "ScalarSummary"
+  input_arg {
+    name: "tags"
+    type: DT_STRING
+  }
+  input_arg {
+    name: "values"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "summary"
+    type: DT_STRING
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_UINT16
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+}
 op {
   name: "ScanDataset"
   input_arg {
@@ -24849,6 +29521,68 @@ op {
     }
   }
 }
+op {
+  name: "ScatterAdd"
+  input_arg {
+    name: "ref"
+    type_attr: "T"
+    is_ref: true
+  }
+  input_arg {
+    name: "indices"
+    type_attr: "Tindices"
+  }
+  input_arg {
+    name: "updates"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "output_ref"
+    type_attr: "T"
+    is_ref: true
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+  attr {
+    name: "Tindices"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+  attr {
+    name: "use_locking"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
+}
 op {
   name: "ScatterDiv"
   input_arg {
@@ -24910,11 +29644,349 @@ op {
   }
 }
 op {
-  name: "ScatterMul"
+  name: "ScatterDiv"
+  input_arg {
+    name: "ref"
+    type_attr: "T"
+    is_ref: true
+  }
+  input_arg {
+    name: "indices"
+    type_attr: "Tindices"
+  }
+  input_arg {
+    name: "updates"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "output_ref"
+    type_attr: "T"
+    is_ref: true
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+  attr {
+    name: "Tindices"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+  attr {
+    name: "use_locking"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
+}
+op {
+  name: "ScatterMul"
+  input_arg {
+    name: "ref"
+    type_attr: "T"
+    is_ref: true
+  }
+  input_arg {
+    name: "indices"
+    type_attr: "Tindices"
+  }
+  input_arg {
+    name: "updates"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "output_ref"
+    type_attr: "T"
+    is_ref: true
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
+      }
+    }
+  }
+  attr {
+    name: "Tindices"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+  attr {
+    name: "use_locking"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
+}
+op {
+  name: "ScatterMul"
+  input_arg {
+    name: "ref"
+    type_attr: "T"
+    is_ref: true
+  }
+  input_arg {
+    name: "indices"
+    type_attr: "Tindices"
+  }
+  input_arg {
+    name: "updates"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "output_ref"
+    type_attr: "T"
+    is_ref: true
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+  attr {
+    name: "Tindices"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+  attr {
+    name: "use_locking"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
+}
+op {
+  name: "ScatterNd"
+  input_arg {
+    name: "indices"
+    type_attr: "Tindices"
+  }
+  input_arg {
+    name: "updates"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "shape"
+    type_attr: "Tindices"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+  }
+  attr {
+    name: "Tindices"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+}
+op {
+  name: "ScatterNdAdd"
+  input_arg {
+    name: "ref"
+    type_attr: "T"
+    is_ref: true
+  }
+  input_arg {
+    name: "indices"
+    type_attr: "Tindices"
+  }
+  input_arg {
+    name: "updates"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "output_ref"
+    type_attr: "T"
+    is_ref: true
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
+      }
+    }
+  }
+  attr {
+    name: "Tindices"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+  attr {
+    name: "use_locking"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
+}
+op {
+  name: "ScatterNdAdd"
+  input_arg {
+    name: "ref"
+    type_attr: "T"
+    is_ref: true
+  }
+  input_arg {
+    name: "indices"
+    type_attr: "Tindices"
+  }
+  input_arg {
+    name: "updates"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "output_ref"
+    type_attr: "T"
+    is_ref: true
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+  attr {
+    name: "Tindices"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+  attr {
+    name: "use_locking"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
+}
+op {
+  name: "ScatterNdNonAliasingAdd"
   input_arg {
-    name: "ref"
+    name: "input"
     type_attr: "T"
-    is_ref: true
   }
   input_arg {
     name: "indices"
@@ -24925,9 +29997,8 @@ op {
     type_attr: "T"
   }
   output_arg {
-    name: "output_ref"
+    name: "output"
     type_attr: "T"
-    is_ref: true
   }
   attr {
     name: "T"
@@ -24961,16 +30032,13 @@ op {
       }
     }
   }
-  attr {
-    name: "use_locking"
-    type: "bool"
-    default_value {
-      b: false
-    }
-  }
 }
 op {
-  name: "ScatterNd"
+  name: "ScatterNdNonAliasingAdd"
+  input_arg {
+    name: "input"
+    type_attr: "T"
+  }
   input_arg {
     name: "indices"
     type_attr: "Tindices"
@@ -24979,10 +30047,6 @@ op {
     name: "updates"
     type_attr: "T"
   }
-  input_arg {
-    name: "shape"
-    type_attr: "Tindices"
-  }
   output_arg {
     name: "output"
     type_attr: "T"
@@ -24990,6 +30054,26 @@ op {
   attr {
     name: "T"
     type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
   }
   attr {
     name: "Tindices"
@@ -25003,7 +30087,7 @@ op {
   }
 }
 op {
-  name: "ScatterNdAdd"
+  name: "ScatterNdSub"
   input_arg {
     name: "ref"
     type_attr: "T"
@@ -25063,10 +30147,11 @@ op {
   }
 }
 op {
-  name: "ScatterNdNonAliasingAdd"
+  name: "ScatterNdSub"
   input_arg {
-    name: "input"
+    name: "ref"
     type_attr: "T"
+    is_ref: true
   }
   input_arg {
     name: "indices"
@@ -25077,8 +30162,9 @@ op {
     type_attr: "T"
   }
   output_arg {
-    name: "output"
+    name: "output_ref"
     type_attr: "T"
+    is_ref: true
   }
   attr {
     name: "T"
@@ -25099,6 +30185,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -25112,9 +30200,16 @@ op {
       }
     }
   }
+  attr {
+    name: "use_locking"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
 }
 op {
-  name: "ScatterNdSub"
+  name: "ScatterNdUpdate"
   input_arg {
     name: "ref"
     type_attr: "T"
@@ -25136,24 +30231,6 @@ op {
   attr {
     name: "T"
     type: "type"
-    allowed_values {
-      list {
-        type: DT_FLOAT
-        type: DT_DOUBLE
-        type: DT_INT64
-        type: DT_INT32
-        type: DT_UINT8
-        type: DT_UINT16
-        type: DT_INT16
-        type: DT_INT8
-        type: DT_COMPLEX64
-        type: DT_COMPLEX128
-        type: DT_QINT8
-        type: DT_QUINT8
-        type: DT_QINT32
-        type: DT_HALF
-      }
-    }
   }
   attr {
     name: "Tindices"
@@ -25169,12 +30246,12 @@ op {
     name: "use_locking"
     type: "bool"
     default_value {
-      b: false
+      b: true
     }
   }
 }
 op {
-  name: "ScatterNdUpdate"
+  name: "ScatterSub"
   input_arg {
     name: "ref"
     type_attr: "T"
@@ -25196,6 +30273,24 @@ op {
   attr {
     name: "T"
     type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
+      }
+    }
   }
   attr {
     name: "Tindices"
@@ -25211,7 +30306,7 @@ op {
     name: "use_locking"
     type: "bool"
     default_value {
-      b: true
+      b: false
     }
   }
 }
@@ -25254,6 +30349,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -25440,36 +30537,250 @@ op {
     minimum: 1
   }
   attr {
-    name: "num_inner_iterations"
-    type: "int"
-    has_minimum: true
-    minimum: 1
+    name: "num_inner_iterations"
+    type: "int"
+    has_minimum: true
+    minimum: 1
+  }
+}
+op {
+  name: "SdcaShrinkL1"
+  input_arg {
+    name: "weights"
+    type: DT_FLOAT
+    number_attr: "num_features"
+    is_ref: true
+  }
+  attr {
+    name: "num_features"
+    type: "int"
+    has_minimum: true
+  }
+  attr {
+    name: "l1"
+    type: "float"
+  }
+  attr {
+    name: "l2"
+    type: "float"
+  }
+}
+op {
+  name: "SegmentMax"
+  input_arg {
+    name: "data"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "segment_ids"
+    type_attr: "Tindices"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_UINT16
+        type: DT_HALF
+      }
+    }
+  }
+  attr {
+    name: "Tindices"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+}
+op {
+  name: "SegmentMax"
+  input_arg {
+    name: "data"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "segment_ids"
+    type_attr: "Tindices"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_UINT16
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+  attr {
+    name: "Tindices"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+}
+op {
+  name: "SegmentMean"
+  input_arg {
+    name: "data"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "segment_ids"
+    type_attr: "Tindices"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_UINT16
+        type: DT_HALF
+      }
+    }
+  }
+  attr {
+    name: "Tindices"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+}
+op {
+  name: "SegmentMean"
+  input_arg {
+    name: "data"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "segment_ids"
+    type_attr: "Tindices"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_UINT16
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+  attr {
+    name: "Tindices"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
   }
 }
 op {
-  name: "SdcaShrinkL1"
+  name: "SegmentMin"
   input_arg {
-    name: "weights"
-    type: DT_FLOAT
-    number_attr: "num_features"
-    is_ref: true
+    name: "data"
+    type_attr: "T"
   }
-  attr {
-    name: "num_features"
-    type: "int"
-    has_minimum: true
+  input_arg {
+    name: "segment_ids"
+    type_attr: "Tindices"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
   }
   attr {
-    name: "l1"
-    type: "float"
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_UINT16
+        type: DT_HALF
+      }
+    }
   }
   attr {
-    name: "l2"
-    type: "float"
+    name: "Tindices"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
   }
 }
 op {
-  name: "SegmentMax"
+  name: "SegmentMin"
   input_arg {
     name: "data"
     type_attr: "T"
@@ -25496,6 +30807,8 @@ op {
         type: DT_INT8
         type: DT_UINT16
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -25511,7 +30824,7 @@ op {
   }
 }
 op {
-  name: "SegmentMean"
+  name: "SegmentProd"
   input_arg {
     name: "data"
     type_attr: "T"
@@ -25531,12 +30844,17 @@ op {
       list {
         type: DT_FLOAT
         type: DT_DOUBLE
-        type: DT_INT32
         type: DT_INT64
+        type: DT_INT32
         type: DT_UINT8
+        type: DT_UINT16
         type: DT_INT16
         type: DT_INT8
-        type: DT_UINT16
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
         type: DT_HALF
       }
     }
@@ -25553,7 +30871,7 @@ op {
   }
 }
 op {
-  name: "SegmentMin"
+  name: "SegmentProd"
   input_arg {
     name: "data"
     type_attr: "T"
@@ -25573,13 +30891,20 @@ op {
       list {
         type: DT_FLOAT
         type: DT_DOUBLE
-        type: DT_INT32
         type: DT_INT64
+        type: DT_INT32
         type: DT_UINT8
+        type: DT_UINT16
         type: DT_INT16
         type: DT_INT8
-        type: DT_UINT16
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -25595,7 +30920,7 @@ op {
   }
 }
 op {
-  name: "SegmentProd"
+  name: "SegmentSum"
   input_arg {
     name: "data"
     type_attr: "T"
@@ -25674,6 +30999,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -26163,13 +31490,436 @@ op {
   }
 }
 op {
-  name: "Sigmoid"
+  name: "Sigmoid"
+  input_arg {
+    name: "x"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "y"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_HALF
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+      }
+    }
+  }
+}
+op {
+  name: "SigmoidGrad"
+  input_arg {
+    name: "x"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "y"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "z"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_HALF
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+      }
+    }
+  }
+}
+op {
+  name: "SigmoidGrad"
+  input_arg {
+    name: "y"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "dy"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "z"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_HALF
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+      }
+    }
+  }
+}
+op {
+  name: "Sign"
+  input_arg {
+    name: "x"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "y"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_HALF
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+      }
+    }
+  }
+}
+op {
+  name: "Sin"
+  input_arg {
+    name: "x"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "y"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_HALF
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+      }
+    }
+  }
+}
+op {
+  name: "Sinh"
+  input_arg {
+    name: "x"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "y"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_HALF
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+      }
+    }
+  }
+}
+op {
+  name: "Size"
+  input_arg {
+    name: "input"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "out_type"
+  }
+  attr {
+    name: "T"
+    type: "type"
+  }
+  attr {
+    name: "out_type"
+    type: "type"
+    default_value {
+      type: DT_INT32
+    }
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+}
+op {
+  name: "SkipDataset"
+  input_arg {
+    name: "input_dataset"
+    type: DT_VARIANT
+  }
+  input_arg {
+    name: "count"
+    type: DT_INT64
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
+  }
+  is_stateful: true
+}
+op {
+  name: "SkipDataset"
+  input_arg {
+    name: "input_dataset"
+    type: DT_VARIANT
+  }
+  input_arg {
+    name: "count"
+    type: DT_INT64
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
+  }
+}
+op {
+  name: "Skipgram"
+  output_arg {
+    name: "vocab_word"
+    type: DT_STRING
+  }
+  output_arg {
+    name: "vocab_freq"
+    type: DT_INT32
+  }
+  output_arg {
+    name: "words_per_epoch"
+    type: DT_INT64
+  }
+  output_arg {
+    name: "current_epoch"
+    type: DT_INT32
+  }
+  output_arg {
+    name: "total_words_processed"
+    type: DT_INT64
+  }
+  output_arg {
+    name: "examples"
+    type: DT_INT32
+  }
+  output_arg {
+    name: "labels"
+    type: DT_INT32
+  }
+  attr {
+    name: "filename"
+    type: "string"
+  }
+  attr {
+    name: "batch_size"
+    type: "int"
+  }
+  attr {
+    name: "window_size"
+    type: "int"
+    default_value {
+      i: 5
+    }
+  }
+  attr {
+    name: "min_count"
+    type: "int"
+    default_value {
+      i: 5
+    }
+  }
+  attr {
+    name: "subsample"
+    type: "float"
+    default_value {
+      f: 0.001
+    }
+  }
+  deprecation {
+    version: 19
+  }
+  is_stateful: true
+}
+op {
+  name: "Slice"
+  input_arg {
+    name: "input"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "begin"
+    type_attr: "Index"
+  }
+  input_arg {
+    name: "size"
+    type_attr: "Index"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+  }
+  attr {
+    name: "Index"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+}
+op {
+  name: "SloppyInterleaveDataset"
+  input_arg {
+    name: "input_dataset"
+    type: DT_VARIANT
+  }
+  input_arg {
+    name: "other_arguments"
+    type_list_attr: "Targuments"
+  }
+  input_arg {
+    name: "cycle_length"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "block_length"
+    type: DT_INT64
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "f"
+    type: "func"
+  }
+  attr {
+    name: "Targuments"
+    type: "list(type)"
+    has_minimum: true
+  }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
+  }
+  is_stateful: true
+}
+op {
+  name: "SloppyInterleaveDataset"
+  input_arg {
+    name: "input_dataset"
+    type: DT_VARIANT
+  }
+  input_arg {
+    name: "other_arguments"
+    type_list_attr: "Targuments"
+  }
+  input_arg {
+    name: "cycle_length"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "block_length"
+    type: DT_INT64
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "f"
+    type: "func"
+  }
+  attr {
+    name: "Targuments"
+    type: "list(type)"
+    has_minimum: true
+  }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
+  }
+}
+op {
+  name: "Softmax"
   input_arg {
-    name: "x"
+    name: "logits"
     type_attr: "T"
   }
   output_arg {
-    name: "y"
+    name: "softmax"
     type_attr: "T"
   }
   attr {
@@ -26180,24 +31930,26 @@ op {
         type: DT_HALF
         type: DT_FLOAT
         type: DT_DOUBLE
-        type: DT_COMPLEX64
-        type: DT_COMPLEX128
       }
     }
   }
 }
 op {
-  name: "SigmoidGrad"
+  name: "SoftmaxCrossEntropyWithLogits"
   input_arg {
-    name: "x"
+    name: "features"
     type_attr: "T"
   }
   input_arg {
-    name: "y"
+    name: "labels"
     type_attr: "T"
   }
   output_arg {
-    name: "z"
+    name: "loss"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "backprop"
     type_attr: "T"
   }
   attr {
@@ -26208,24 +31960,46 @@ op {
         type: DT_HALF
         type: DT_FLOAT
         type: DT_DOUBLE
-        type: DT_COMPLEX64
-        type: DT_COMPLEX128
       }
     }
   }
 }
 op {
-  name: "SigmoidGrad"
+  name: "Softplus"
   input_arg {
-    name: "y"
+    name: "features"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "activations"
     type_attr: "T"
   }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_UINT16
+        type: DT_HALF
+      }
+    }
+  }
+}
+op {
+  name: "Softplus"
   input_arg {
-    name: "dy"
+    name: "features"
     type_attr: "T"
   }
   output_arg {
-    name: "z"
+    name: "activations"
     type_attr: "T"
   }
   attr {
@@ -26233,23 +32007,33 @@ op {
     type: "type"
     allowed_values {
       list {
-        type: DT_HALF
         type: DT_FLOAT
         type: DT_DOUBLE
-        type: DT_COMPLEX64
-        type: DT_COMPLEX128
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_UINT16
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
 }
 op {
-  name: "Sign"
+  name: "SoftplusGrad"
   input_arg {
-    name: "x"
+    name: "gradients"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "features"
     type_attr: "T"
   }
   output_arg {
-    name: "y"
+    name: "backprops"
     type_attr: "T"
   }
   attr {
@@ -26257,25 +32041,61 @@ op {
     type: "type"
     allowed_values {
       list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_UINT16
         type: DT_HALF
+      }
+    }
+  }
+}
+op {
+  name: "SoftplusGrad"
+  input_arg {
+    name: "gradients"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "features"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "backprops"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
         type: DT_FLOAT
         type: DT_DOUBLE
         type: DT_INT32
         type: DT_INT64
-        type: DT_COMPLEX64
-        type: DT_COMPLEX128
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_UINT16
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
 }
 op {
-  name: "Sin"
+  name: "Softsign"
   input_arg {
-    name: "x"
+    name: "features"
     type_attr: "T"
   }
   output_arg {
-    name: "y"
+    name: "activations"
     type_attr: "T"
   }
   attr {
@@ -26283,23 +32103,61 @@ op {
     type: "type"
     allowed_values {
       list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_UINT16
         type: DT_HALF
+      }
+    }
+  }
+}
+op {
+  name: "Softsign"
+  input_arg {
+    name: "features"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "activations"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
         type: DT_FLOAT
         type: DT_DOUBLE
-        type: DT_COMPLEX64
-        type: DT_COMPLEX128
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_UINT16
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
 }
 op {
-  name: "Sinh"
+  name: "SoftsignGrad"
   input_arg {
-    name: "x"
+    name: "gradients"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "features"
     type_attr: "T"
   }
   output_arg {
-    name: "y"
+    name: "backprops"
     type_attr: "T"
   }
   attr {
@@ -26307,31 +32165,73 @@ op {
     type: "type"
     allowed_values {
       list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_UINT16
         type: DT_HALF
+      }
+    }
+  }
+}
+op {
+  name: "SoftsignGrad"
+  input_arg {
+    name: "gradients"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "features"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "backprops"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
         type: DT_FLOAT
         type: DT_DOUBLE
-        type: DT_COMPLEX64
-        type: DT_COMPLEX128
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_UINT16
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
 }
 op {
-  name: "Size"
+  name: "SpaceToBatch"
   input_arg {
     name: "input"
     type_attr: "T"
   }
+  input_arg {
+    name: "paddings"
+    type_attr: "Tpaddings"
+  }
   output_arg {
     name: "output"
-    type_attr: "out_type"
+    type_attr: "T"
   }
   attr {
     name: "T"
     type: "type"
   }
   attr {
-    name: "out_type"
+    name: "Tpaddings"
     type: "type"
     default_value {
       type: DT_INT32
@@ -26343,256 +32243,506 @@ op {
       }
     }
   }
+  attr {
+    name: "block_size"
+    type: "int"
+    has_minimum: true
+    minimum: 2
+  }
 }
 op {
-  name: "SkipDataset"
+  name: "SpaceToBatchND"
   input_arg {
-    name: "input_dataset"
-    type: DT_VARIANT
+    name: "input"
+    type_attr: "T"
   }
   input_arg {
-    name: "count"
-    type: DT_INT64
+    name: "block_shape"
+    type_attr: "Tblock_shape"
+  }
+  input_arg {
+    name: "paddings"
+    type_attr: "Tpaddings"
   }
   output_arg {
-    name: "handle"
-    type: DT_VARIANT
+    name: "output"
+    type_attr: "T"
   }
   attr {
-    name: "output_types"
-    type: "list(type)"
+    name: "T"
+    type: "type"
+  }
+  attr {
+    name: "Tblock_shape"
+    type: "type"
+    default_value {
+      type: DT_INT32
+    }
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+  attr {
+    name: "Tpaddings"
+    type: "type"
+    default_value {
+      type: DT_INT32
+    }
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+}
+op {
+  name: "SpaceToDepth"
+  input_arg {
+    name: "input"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+  }
+  attr {
+    name: "block_size"
+    type: "int"
     has_minimum: true
-    minimum: 1
+    minimum: 2
+  }
+}
+op {
+  name: "SpaceToDepth"
+  input_arg {
+    name: "input"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
   }
   attr {
-    name: "output_shapes"
-    type: "list(shape)"
+    name: "T"
+    type: "type"
+  }
+  attr {
+    name: "block_size"
+    type: "int"
     has_minimum: true
-    minimum: 1
+    minimum: 2
+  }
+  attr {
+    name: "data_format"
+    type: "string"
+    default_value {
+      s: "NHWC"
+    }
+    allowed_values {
+      list {
+        s: "NHWC"
+        s: "NCHW"
+        s: "NCHW_VECT_C"
+      }
+    }
   }
-  is_stateful: true
 }
 op {
-  name: "SkipDataset"
+  name: "SparseAccumulatorApplyGradient"
   input_arg {
-    name: "input_dataset"
-    type: DT_VARIANT
+    name: "handle"
+    type: DT_STRING
+    is_ref: true
   }
   input_arg {
-    name: "count"
+    name: "local_step"
     type: DT_INT64
   }
-  output_arg {
+  input_arg {
+    name: "gradient_indices"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "gradient_values"
+    type_attr: "dtype"
+  }
+  input_arg {
+    name: "gradient_shape"
+    type: DT_INT64
+  }
+  attr {
+    name: "dtype"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
+      }
+    }
+  }
+  attr {
+    name: "has_known_shape"
+    type: "bool"
+  }
+}
+op {
+  name: "SparseAccumulatorApplyGradient"
+  input_arg {
     name: "handle"
-    type: DT_VARIANT
+    type: DT_STRING
+    is_ref: true
+  }
+  input_arg {
+    name: "local_step"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "gradient_indices"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "gradient_values"
+    type_attr: "dtype"
+  }
+  input_arg {
+    name: "gradient_shape"
+    type: DT_INT64
   }
   attr {
-    name: "output_types"
-    type: "list(type)"
-    has_minimum: true
-    minimum: 1
+    name: "dtype"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
   }
   attr {
-    name: "output_shapes"
-    type: "list(shape)"
-    has_minimum: true
-    minimum: 1
+    name: "has_known_shape"
+    type: "bool"
   }
 }
 op {
-  name: "Skipgram"
-  output_arg {
-    name: "vocab_word"
+  name: "SparseAccumulatorTakeGradient"
+  input_arg {
+    name: "handle"
     type: DT_STRING
+    is_ref: true
   }
-  output_arg {
-    name: "vocab_freq"
+  input_arg {
+    name: "num_required"
     type: DT_INT32
   }
   output_arg {
-    name: "words_per_epoch"
+    name: "indices"
     type: DT_INT64
   }
   output_arg {
-    name: "current_epoch"
-    type: DT_INT32
+    name: "values"
+    type_attr: "dtype"
   }
   output_arg {
-    name: "total_words_processed"
+    name: "shape"
     type: DT_INT64
   }
-  output_arg {
-    name: "examples"
+  attr {
+    name: "dtype"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
+      }
+    }
+  }
+}
+op {
+  name: "SparseAccumulatorTakeGradient"
+  input_arg {
+    name: "handle"
+    type: DT_STRING
+    is_ref: true
+  }
+  input_arg {
+    name: "num_required"
     type: DT_INT32
   }
   output_arg {
-    name: "labels"
-    type: DT_INT32
+    name: "indices"
+    type: DT_INT64
   }
-  attr {
-    name: "filename"
-    type: "string"
+  output_arg {
+    name: "values"
+    type_attr: "dtype"
   }
-  attr {
-    name: "batch_size"
-    type: "int"
+  output_arg {
+    name: "shape"
+    type: DT_INT64
   }
   attr {
-    name: "window_size"
-    type: "int"
-    default_value {
-      i: 5
+    name: "dtype"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
     }
   }
-  attr {
-    name: "min_count"
-    type: "int"
-    default_value {
-      i: 5
-    }
+}
+op {
+  name: "SparseAdd"
+  input_arg {
+    name: "a_indices"
+    type: DT_INT64
   }
-  attr {
-    name: "subsample"
-    type: "float"
-    default_value {
-      f: 0.001
-    }
+  input_arg {
+    name: "a_values"
+    type_attr: "T"
   }
-  deprecation {
-    version: 19
+  input_arg {
+    name: "a_shape"
+    type: DT_INT64
   }
-  is_stateful: true
-}
-op {
-  name: "Slice"
   input_arg {
-    name: "input"
+    name: "b_indices"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "b_values"
     type_attr: "T"
   }
   input_arg {
-    name: "begin"
-    type_attr: "Index"
+    name: "b_shape"
+    type: DT_INT64
   }
   input_arg {
-    name: "size"
-    type_attr: "Index"
+    name: "thresh"
+    type_attr: "Treal"
   }
   output_arg {
-    name: "output"
+    name: "sum_indices"
+    type: DT_INT64
+  }
+  output_arg {
+    name: "sum_values"
     type_attr: "T"
   }
+  output_arg {
+    name: "sum_shape"
+    type: DT_INT64
+  }
   attr {
     name: "T"
     type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
+      }
+    }
   }
   attr {
-    name: "Index"
+    name: "Treal"
     type: "type"
     allowed_values {
       list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
         type: DT_INT32
         type: DT_INT64
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_UINT16
+        type: DT_HALF
       }
     }
   }
 }
 op {
-  name: "SloppyInterleaveDataset"
+  name: "SparseAdd"
   input_arg {
-    name: "input_dataset"
-    type: DT_VARIANT
+    name: "a_indices"
+    type: DT_INT64
   }
   input_arg {
-    name: "other_arguments"
-    type_list_attr: "Targuments"
+    name: "a_values"
+    type_attr: "T"
   }
   input_arg {
-    name: "cycle_length"
+    name: "a_shape"
     type: DT_INT64
   }
   input_arg {
-    name: "block_length"
+    name: "b_indices"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "b_values"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "b_shape"
     type: DT_INT64
   }
+  input_arg {
+    name: "thresh"
+    type_attr: "Treal"
+  }
   output_arg {
-    name: "handle"
-    type: DT_VARIANT
+    name: "sum_indices"
+    type: DT_INT64
   }
-  attr {
-    name: "f"
-    type: "func"
+  output_arg {
+    name: "sum_values"
+    type_attr: "T"
   }
-  attr {
-    name: "Targuments"
-    type: "list(type)"
-    has_minimum: true
+  output_arg {
+    name: "sum_shape"
+    type: DT_INT64
   }
   attr {
-    name: "output_types"
-    type: "list(type)"
-    has_minimum: true
-    minimum: 1
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
   }
   attr {
-    name: "output_shapes"
-    type: "list(shape)"
-    has_minimum: true
-    minimum: 1
+    name: "Treal"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_UINT16
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
   }
-  is_stateful: true
 }
 op {
-  name: "SloppyInterleaveDataset"
+  name: "SparseAddGrad"
   input_arg {
-    name: "input_dataset"
-    type: DT_VARIANT
+    name: "backprop_val_grad"
+    type_attr: "T"
   }
   input_arg {
-    name: "other_arguments"
-    type_list_attr: "Targuments"
+    name: "a_indices"
+    type: DT_INT64
   }
   input_arg {
-    name: "cycle_length"
+    name: "b_indices"
     type: DT_INT64
   }
   input_arg {
-    name: "block_length"
+    name: "sum_indices"
     type: DT_INT64
   }
   output_arg {
-    name: "handle"
-    type: DT_VARIANT
-  }
-  attr {
-    name: "f"
-    type: "func"
-  }
-  attr {
-    name: "Targuments"
-    type: "list(type)"
-    has_minimum: true
-  }
-  attr {
-    name: "output_types"
-    type: "list(type)"
-    has_minimum: true
-    minimum: 1
-  }
-  attr {
-    name: "output_shapes"
-    type: "list(shape)"
-    has_minimum: true
-    minimum: 1
-  }
-}
-op {
-  name: "Softmax"
-  input_arg {
-    name: "logits"
+    name: "a_val_grad"
     type_attr: "T"
   }
   output_arg {
-    name: "softmax"
+    name: "b_val_grad"
     type_attr: "T"
   }
   attr {
@@ -26600,29 +32750,48 @@ op {
     type: "type"
     allowed_values {
       list {
-        type: DT_HALF
         type: DT_FLOAT
         type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
       }
     }
   }
 }
 op {
-  name: "SoftmaxCrossEntropyWithLogits"
+  name: "SparseAddGrad"
   input_arg {
-    name: "features"
+    name: "backprop_val_grad"
     type_attr: "T"
   }
   input_arg {
-    name: "labels"
-    type_attr: "T"
+    name: "a_indices"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "b_indices"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "sum_indices"
+    type: DT_INT64
   }
   output_arg {
-    name: "loss"
+    name: "a_val_grad"
     type_attr: "T"
   }
   output_arg {
-    name: "backprop"
+    name: "b_val_grad"
     type_attr: "T"
   }
   attr {
@@ -26630,22 +32799,67 @@ op {
     type: "type"
     allowed_values {
       list {
-        type: DT_HALF
         type: DT_FLOAT
         type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
 }
 op {
-  name: "Softplus"
+  name: "SparseApplyAdadelta"
   input_arg {
-    name: "features"
+    name: "var"
+    type_attr: "T"
+    is_ref: true
+  }
+  input_arg {
+    name: "accum"
+    type_attr: "T"
+    is_ref: true
+  }
+  input_arg {
+    name: "accum_update"
+    type_attr: "T"
+    is_ref: true
+  }
+  input_arg {
+    name: "lr"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "rho"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "epsilon"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "grad"
     type_attr: "T"
   }
+  input_arg {
+    name: "indices"
+    type_attr: "Tindices"
+  }
   output_arg {
-    name: "activations"
+    name: "out"
     type_attr: "T"
+    is_ref: true
   }
   attr {
     name: "T"
@@ -26654,30 +32868,80 @@ op {
       list {
         type: DT_FLOAT
         type: DT_DOUBLE
-        type: DT_INT32
         type: DT_INT64
+        type: DT_INT32
         type: DT_UINT8
+        type: DT_UINT16
         type: DT_INT16
         type: DT_INT8
-        type: DT_UINT16
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
         type: DT_HALF
       }
     }
   }
+  attr {
+    name: "Tindices"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+  attr {
+    name: "use_locking"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
 }
 op {
-  name: "SoftplusGrad"
+  name: "SparseApplyAdadelta"
   input_arg {
-    name: "gradients"
+    name: "var"
+    type_attr: "T"
+    is_ref: true
+  }
+  input_arg {
+    name: "accum"
     type_attr: "T"
+    is_ref: true
   }
   input_arg {
-    name: "features"
+    name: "accum_update"
+    type_attr: "T"
+    is_ref: true
+  }
+  input_arg {
+    name: "lr"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "rho"
     type_attr: "T"
   }
+  input_arg {
+    name: "epsilon"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "grad"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "indices"
+    type_attr: "Tindices"
+  }
   output_arg {
-    name: "backprops"
+    name: "out"
     type_attr: "T"
+    is_ref: true
   }
   attr {
     name: "T"
@@ -26686,26 +32950,69 @@ op {
       list {
         type: DT_FLOAT
         type: DT_DOUBLE
-        type: DT_INT32
         type: DT_INT64
+        type: DT_INT32
         type: DT_UINT8
+        type: DT_UINT16
         type: DT_INT16
         type: DT_INT8
-        type: DT_UINT16
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+  attr {
+    name: "Tindices"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
       }
     }
   }
+  attr {
+    name: "use_locking"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
 }
 op {
-  name: "Softsign"
+  name: "SparseApplyAdagrad"
   input_arg {
-    name: "features"
+    name: "var"
+    type_attr: "T"
+    is_ref: true
+  }
+  input_arg {
+    name: "accum"
+    type_attr: "T"
+    is_ref: true
+  }
+  input_arg {
+    name: "lr"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "grad"
     type_attr: "T"
   }
+  input_arg {
+    name: "indices"
+    type_attr: "Tindices"
+  }
   output_arg {
-    name: "activations"
+    name: "out"
     type_attr: "T"
+    is_ref: true
   }
   attr {
     name: "T"
@@ -26714,30 +33021,67 @@ op {
       list {
         type: DT_FLOAT
         type: DT_DOUBLE
-        type: DT_INT32
         type: DT_INT64
+        type: DT_INT32
         type: DT_UINT8
+        type: DT_UINT16
         type: DT_INT16
         type: DT_INT8
-        type: DT_UINT16
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
         type: DT_HALF
       }
     }
   }
+  attr {
+    name: "Tindices"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+  attr {
+    name: "use_locking"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
 }
 op {
-  name: "SoftsignGrad"
+  name: "SparseApplyAdagrad"
   input_arg {
-    name: "gradients"
+    name: "var"
     type_attr: "T"
+    is_ref: true
   }
   input_arg {
-    name: "features"
+    name: "accum"
+    type_attr: "T"
+    is_ref: true
+  }
+  input_arg {
+    name: "lr"
     type_attr: "T"
   }
+  input_arg {
+    name: "grad"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "indices"
+    type_attr: "Tindices"
+  }
   output_arg {
-    name: "backprops"
+    name: "out"
     type_attr: "T"
+    is_ref: true
   }
   attr {
     name: "T"
@@ -26746,96 +33090,112 @@ op {
       list {
         type: DT_FLOAT
         type: DT_DOUBLE
-        type: DT_INT32
         type: DT_INT64
+        type: DT_INT32
         type: DT_UINT8
+        type: DT_UINT16
         type: DT_INT16
         type: DT_INT8
-        type: DT_UINT16
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+  attr {
+    name: "Tindices"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
       }
     }
   }
+  attr {
+    name: "use_locking"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
 }
 op {
-  name: "SpaceToBatch"
+  name: "SparseApplyAdagradDA"
   input_arg {
-    name: "input"
+    name: "var"
     type_attr: "T"
+    is_ref: true
   }
   input_arg {
-    name: "paddings"
-    type_attr: "Tpaddings"
+    name: "gradient_accumulator"
+    type_attr: "T"
+    is_ref: true
   }
-  output_arg {
-    name: "output"
+  input_arg {
+    name: "gradient_squared_accumulator"
+    type_attr: "T"
+    is_ref: true
+  }
+  input_arg {
+    name: "grad"
     type_attr: "T"
   }
-  attr {
-    name: "T"
-    type: "type"
-  }
-  attr {
-    name: "Tpaddings"
-    type: "type"
-    default_value {
-      type: DT_INT32
-    }
-    allowed_values {
-      list {
-        type: DT_INT32
-        type: DT_INT64
-      }
-    }
+  input_arg {
+    name: "indices"
+    type_attr: "Tindices"
   }
-  attr {
-    name: "block_size"
-    type: "int"
-    has_minimum: true
-    minimum: 2
+  input_arg {
+    name: "lr"
+    type_attr: "T"
   }
-}
-op {
-  name: "SpaceToBatchND"
   input_arg {
-    name: "input"
+    name: "l1"
     type_attr: "T"
   }
   input_arg {
-    name: "block_shape"
-    type_attr: "Tblock_shape"
+    name: "l2"
+    type_attr: "T"
   }
   input_arg {
-    name: "paddings"
-    type_attr: "Tpaddings"
+    name: "global_step"
+    type: DT_INT64
   }
   output_arg {
-    name: "output"
+    name: "out"
     type_attr: "T"
+    is_ref: true
   }
   attr {
     name: "T"
     type: "type"
-  }
-  attr {
-    name: "Tblock_shape"
-    type: "type"
-    default_value {
-      type: DT_INT32
-    }
     allowed_values {
       list {
-        type: DT_INT32
+        type: DT_FLOAT
+        type: DT_DOUBLE
         type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
       }
     }
   }
   attr {
-    name: "Tpaddings"
+    name: "Tindices"
     type: "type"
-    default_value {
-      type: DT_INT32
-    }
     allowed_values {
       list {
         type: DT_INT32
@@ -26843,88 +33203,62 @@ op {
       }
     }
   }
+  attr {
+    name: "use_locking"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
 }
 op {
-  name: "SpaceToDepth"
+  name: "SparseApplyAdagradDA"
   input_arg {
-    name: "input"
+    name: "var"
     type_attr: "T"
+    is_ref: true
   }
-  output_arg {
-    name: "output"
+  input_arg {
+    name: "gradient_accumulator"
     type_attr: "T"
+    is_ref: true
   }
-  attr {
-    name: "T"
-    type: "type"
-  }
-  attr {
-    name: "block_size"
-    type: "int"
-    has_minimum: true
-    minimum: 2
-  }
-}
-op {
-  name: "SpaceToDepth"
   input_arg {
-    name: "input"
+    name: "gradient_squared_accumulator"
     type_attr: "T"
+    is_ref: true
   }
-  output_arg {
-    name: "output"
+  input_arg {
+    name: "grad"
     type_attr: "T"
   }
-  attr {
-    name: "T"
-    type: "type"
-  }
-  attr {
-    name: "block_size"
-    type: "int"
-    has_minimum: true
-    minimum: 2
-  }
-  attr {
-    name: "data_format"
-    type: "string"
-    default_value {
-      s: "NHWC"
-    }
-    allowed_values {
-      list {
-        s: "NHWC"
-        s: "NCHW"
-        s: "NCHW_VECT_C"
-      }
-    }
-  }
-}
-op {
-  name: "SparseAccumulatorApplyGradient"
   input_arg {
-    name: "handle"
-    type: DT_STRING
-    is_ref: true
+    name: "indices"
+    type_attr: "Tindices"
   }
   input_arg {
-    name: "local_step"
-    type: DT_INT64
+    name: "lr"
+    type_attr: "T"
   }
   input_arg {
-    name: "gradient_indices"
-    type: DT_INT64
+    name: "l1"
+    type_attr: "T"
   }
   input_arg {
-    name: "gradient_values"
-    type_attr: "dtype"
+    name: "l2"
+    type_attr: "T"
   }
   input_arg {
-    name: "gradient_shape"
+    name: "global_step"
     type: DT_INT64
   }
+  output_arg {
+    name: "out"
+    type_attr: "T"
+    is_ref: true
+  }
   attr {
-    name: "dtype"
+    name: "T"
     type: "type"
     allowed_values {
       list {
@@ -26942,39 +33276,82 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
   attr {
-    name: "has_known_shape"
+    name: "Tindices"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+  attr {
+    name: "use_locking"
     type: "bool"
+    default_value {
+      b: false
+    }
   }
 }
 op {
-  name: "SparseAccumulatorTakeGradient"
+  name: "SparseApplyCenteredRMSProp"
   input_arg {
-    name: "handle"
-    type: DT_STRING
+    name: "var"
+    type_attr: "T"
     is_ref: true
   }
   input_arg {
-    name: "num_required"
-    type: DT_INT32
+    name: "mg"
+    type_attr: "T"
+    is_ref: true
   }
-  output_arg {
-    name: "indices"
-    type: DT_INT64
+  input_arg {
+    name: "ms"
+    type_attr: "T"
+    is_ref: true
   }
-  output_arg {
-    name: "values"
-    type_attr: "dtype"
+  input_arg {
+    name: "mom"
+    type_attr: "T"
+    is_ref: true
+  }
+  input_arg {
+    name: "lr"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "rho"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "momentum"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "epsilon"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "grad"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "indices"
+    type_attr: "Tindices"
   }
   output_arg {
-    name: "shape"
-    type: DT_INT64
+    name: "out"
+    type_attr: "T"
+    is_ref: true
   }
   attr {
-    name: "dtype"
+    name: "T"
     type: "type"
     allowed_values {
       list {
@@ -26995,48 +33372,74 @@ op {
       }
     }
   }
+  attr {
+    name: "Tindices"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+  attr {
+    name: "use_locking"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
 }
 op {
-  name: "SparseAdd"
+  name: "SparseApplyCenteredRMSProp"
   input_arg {
-    name: "a_indices"
-    type: DT_INT64
+    name: "var"
+    type_attr: "T"
+    is_ref: true
   }
   input_arg {
-    name: "a_values"
+    name: "mg"
     type_attr: "T"
+    is_ref: true
   }
   input_arg {
-    name: "a_shape"
-    type: DT_INT64
+    name: "ms"
+    type_attr: "T"
+    is_ref: true
   }
   input_arg {
-    name: "b_indices"
-    type: DT_INT64
+    name: "mom"
+    type_attr: "T"
+    is_ref: true
   }
   input_arg {
-    name: "b_values"
+    name: "lr"
     type_attr: "T"
   }
   input_arg {
-    name: "b_shape"
-    type: DT_INT64
+    name: "rho"
+    type_attr: "T"
   }
   input_arg {
-    name: "thresh"
-    type_attr: "Treal"
+    name: "momentum"
+    type_attr: "T"
   }
-  output_arg {
-    name: "sum_indices"
-    type: DT_INT64
+  input_arg {
+    name: "epsilon"
+    type_attr: "T"
   }
-  output_arg {
-    name: "sum_values"
+  input_arg {
+    name: "grad"
     type_attr: "T"
   }
+  input_arg {
+    name: "indices"
+    type_attr: "Tindices"
+  }
   output_arg {
-    name: "sum_shape"
-    type: DT_INT64
+    name: "out"
+    type_attr: "T"
+    is_ref: true
   }
   attr {
     name: "T"
@@ -27057,52 +33460,74 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
   attr {
-    name: "Treal"
+    name: "Tindices"
     type: "type"
     allowed_values {
       list {
-        type: DT_FLOAT
-        type: DT_DOUBLE
         type: DT_INT32
         type: DT_INT64
-        type: DT_UINT8
-        type: DT_INT16
-        type: DT_INT8
-        type: DT_UINT16
-        type: DT_HALF
       }
     }
   }
+  attr {
+    name: "use_locking"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
 }
 op {
-  name: "SparseAddGrad"
+  name: "SparseApplyFtrl"
   input_arg {
-    name: "backprop_val_grad"
+    name: "var"
     type_attr: "T"
+    is_ref: true
   }
   input_arg {
-    name: "a_indices"
-    type: DT_INT64
+    name: "accum"
+    type_attr: "T"
+    is_ref: true
   }
   input_arg {
-    name: "b_indices"
-    type: DT_INT64
+    name: "linear"
+    type_attr: "T"
+    is_ref: true
   }
   input_arg {
-    name: "sum_indices"
-    type: DT_INT64
+    name: "grad"
+    type_attr: "T"
   }
-  output_arg {
-    name: "a_val_grad"
+  input_arg {
+    name: "indices"
+    type_attr: "Tindices"
+  }
+  input_arg {
+    name: "lr"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "l1"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "l2"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "lr_power"
     type_attr: "T"
   }
   output_arg {
-    name: "b_val_grad"
+    name: "out"
     type_attr: "T"
+    is_ref: true
   }
   attr {
     name: "T"
@@ -27126,9 +33551,26 @@ op {
       }
     }
   }
+  attr {
+    name: "Tindices"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+  attr {
+    name: "use_locking"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
 }
 op {
-  name: "SparseApplyAdadelta"
+  name: "SparseApplyFtrl"
   input_arg {
     name: "var"
     type_attr: "T"
@@ -27140,29 +33582,33 @@ op {
     is_ref: true
   }
   input_arg {
-    name: "accum_update"
+    name: "linear"
     type_attr: "T"
     is_ref: true
   }
   input_arg {
-    name: "lr"
+    name: "grad"
     type_attr: "T"
   }
   input_arg {
-    name: "rho"
+    name: "indices"
+    type_attr: "Tindices"
+  }
+  input_arg {
+    name: "lr"
     type_attr: "T"
   }
   input_arg {
-    name: "epsilon"
+    name: "l1"
     type_attr: "T"
   }
   input_arg {
-    name: "grad"
+    name: "l2"
     type_attr: "T"
   }
   input_arg {
-    name: "indices"
-    type_attr: "Tindices"
+    name: "lr_power"
+    type_attr: "T"
   }
   output_arg {
     name: "out"
@@ -27188,6 +33634,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -27210,7 +33658,7 @@ op {
   }
 }
 op {
-  name: "SparseApplyAdagrad"
+  name: "SparseApplyFtrlV2"
   input_arg {
     name: "var"
     type_attr: "T"
@@ -27222,8 +33670,9 @@ op {
     is_ref: true
   }
   input_arg {
-    name: "lr"
+    name: "linear"
     type_attr: "T"
+    is_ref: true
   }
   input_arg {
     name: "grad"
@@ -27233,6 +33682,26 @@ op {
     name: "indices"
     type_attr: "Tindices"
   }
+  input_arg {
+    name: "lr"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "l1"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "l2"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "l2_shrinkage"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "lr_power"
+    type_attr: "T"
+  }
   output_arg {
     name: "out"
     type_attr: "T"
@@ -27279,19 +33748,19 @@ op {
   }
 }
 op {
-  name: "SparseApplyAdagradDA"
+  name: "SparseApplyFtrlV2"
   input_arg {
     name: "var"
     type_attr: "T"
     is_ref: true
   }
   input_arg {
-    name: "gradient_accumulator"
+    name: "accum"
     type_attr: "T"
     is_ref: true
   }
   input_arg {
-    name: "gradient_squared_accumulator"
+    name: "linear"
     type_attr: "T"
     is_ref: true
   }
@@ -27316,8 +33785,12 @@ op {
     type_attr: "T"
   }
   input_arg {
-    name: "global_step"
-    type: DT_INT64
+    name: "l2_shrinkage"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "lr_power"
+    type_attr: "T"
   }
   output_arg {
     name: "out"
@@ -27343,6 +33816,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -27365,41 +33840,99 @@ op {
   }
 }
 op {
-  name: "SparseApplyCenteredRMSProp"
+  name: "SparseApplyMomentum"
   input_arg {
     name: "var"
     type_attr: "T"
     is_ref: true
   }
   input_arg {
-    name: "mg"
+    name: "accum"
     type_attr: "T"
     is_ref: true
   }
   input_arg {
-    name: "ms"
+    name: "lr"
     type_attr: "T"
-    is_ref: true
   }
   input_arg {
-    name: "mom"
+    name: "grad"
     type_attr: "T"
-    is_ref: true
   }
   input_arg {
-    name: "lr"
+    name: "indices"
+    type_attr: "Tindices"
+  }
+  input_arg {
+    name: "momentum"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "out"
     type_attr: "T"
+    is_ref: true
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
+      }
+    }
+  }
+  attr {
+    name: "Tindices"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+  attr {
+    name: "use_locking"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
+  attr {
+    name: "use_nesterov"
+    type: "bool"
+    default_value {
+      b: false
+    }
   }
+}
+op {
+  name: "SparseApplyMomentum"
   input_arg {
-    name: "rho"
+    name: "var"
     type_attr: "T"
+    is_ref: true
   }
   input_arg {
-    name: "momentum"
+    name: "accum"
     type_attr: "T"
+    is_ref: true
   }
   input_arg {
-    name: "epsilon"
+    name: "lr"
     type_attr: "T"
   }
   input_arg {
@@ -27410,6 +33943,10 @@ op {
     name: "indices"
     type_attr: "Tindices"
   }
+  input_arg {
+    name: "momentum"
+    type_attr: "T"
+  }
   output_arg {
     name: "out"
     type_attr: "T"
@@ -27434,6 +33971,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -27454,9 +33993,16 @@ op {
       b: false
     }
   }
+  attr {
+    name: "use_nesterov"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
 }
 op {
-  name: "SparseApplyFtrl"
+  name: "SparseApplyProximalAdagrad"
   input_arg {
     name: "var"
     type_attr: "T"
@@ -27467,19 +34013,6 @@ op {
     type_attr: "T"
     is_ref: true
   }
-  input_arg {
-    name: "linear"
-    type_attr: "T"
-    is_ref: true
-  }
-  input_arg {
-    name: "grad"
-    type_attr: "T"
-  }
-  input_arg {
-    name: "indices"
-    type_attr: "Tindices"
-  }
   input_arg {
     name: "lr"
     type_attr: "T"
@@ -27493,9 +34026,13 @@ op {
     type_attr: "T"
   }
   input_arg {
-    name: "lr_power"
+    name: "grad"
     type_attr: "T"
   }
+  input_arg {
+    name: "indices"
+    type_attr: "Tindices"
+  }
   output_arg {
     name: "out"
     type_attr: "T"
@@ -27542,7 +34079,7 @@ op {
   }
 }
 op {
-  name: "SparseApplyFtrlV2"
+  name: "SparseApplyProximalAdagrad"
   input_arg {
     name: "var"
     type_attr: "T"
@@ -27553,19 +34090,6 @@ op {
     type_attr: "T"
     is_ref: true
   }
-  input_arg {
-    name: "linear"
-    type_attr: "T"
-    is_ref: true
-  }
-  input_arg {
-    name: "grad"
-    type_attr: "T"
-  }
-  input_arg {
-    name: "indices"
-    type_attr: "Tindices"
-  }
   input_arg {
     name: "lr"
     type_attr: "T"
@@ -27579,12 +34103,12 @@ op {
     type_attr: "T"
   }
   input_arg {
-    name: "l2_shrinkage"
+    name: "grad"
     type_attr: "T"
   }
   input_arg {
-    name: "lr_power"
-    type_attr: "T"
+    name: "indices"
+    type_attr: "Tindices"
   }
   output_arg {
     name: "out"
@@ -27610,6 +34134,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -27632,19 +34158,22 @@ op {
   }
 }
 op {
-  name: "SparseApplyMomentum"
+  name: "SparseApplyProximalGradientDescent"
   input_arg {
     name: "var"
     type_attr: "T"
     is_ref: true
   }
   input_arg {
-    name: "accum"
+    name: "alpha"
     type_attr: "T"
-    is_ref: true
   }
   input_arg {
-    name: "lr"
+    name: "l1"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "l2"
     type_attr: "T"
   }
   input_arg {
@@ -27655,10 +34184,6 @@ op {
     name: "indices"
     type_attr: "Tindices"
   }
-  input_arg {
-    name: "momentum"
-    type_attr: "T"
-  }
   output_arg {
     name: "out"
     type_attr: "T"
@@ -27703,28 +34228,16 @@ op {
       b: false
     }
   }
-  attr {
-    name: "use_nesterov"
-    type: "bool"
-    default_value {
-      b: false
-    }
-  }
 }
 op {
-  name: "SparseApplyProximalAdagrad"
+  name: "SparseApplyProximalGradientDescent"
   input_arg {
     name: "var"
     type_attr: "T"
     is_ref: true
   }
   input_arg {
-    name: "accum"
-    type_attr: "T"
-    is_ref: true
-  }
-  input_arg {
-    name: "lr"
+    name: "alpha"
     type_attr: "T"
   }
   input_arg {
@@ -27767,6 +34280,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -27789,22 +34304,36 @@ op {
   }
 }
 op {
-  name: "SparseApplyProximalGradientDescent"
+  name: "SparseApplyRMSProp"
   input_arg {
     name: "var"
     type_attr: "T"
     is_ref: true
   }
   input_arg {
-    name: "alpha"
+    name: "ms"
     type_attr: "T"
+    is_ref: true
   }
   input_arg {
-    name: "l1"
+    name: "mom"
     type_attr: "T"
+    is_ref: true
   }
   input_arg {
-    name: "l2"
+    name: "lr"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "rho"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "momentum"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "epsilon"
     type_attr: "T"
   }
   input_arg {
@@ -27925,6 +34454,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -28039,6 +34570,57 @@ op {
   }
   is_stateful: true
 }
+op {
+  name: "SparseConditionalAccumulator"
+  output_arg {
+    name: "handle"
+    type: DT_STRING
+    is_ref: true
+  }
+  attr {
+    name: "dtype"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+  attr {
+    name: "shape"
+    type: "shape"
+  }
+  attr {
+    name: "container"
+    type: "string"
+    default_value {
+      s: ""
+    }
+  }
+  attr {
+    name: "shared_name"
+    type: "string"
+    default_value {
+      s: ""
+    }
+  }
+  is_stateful: true
+}
 op {
   name: "SparseCross"
   input_arg {
@@ -28177,6 +34759,53 @@ op {
     }
   }
 }
+op {
+  name: "SparseDenseCwiseAdd"
+  input_arg {
+    name: "sp_indices"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "sp_values"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "sp_shape"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "dense"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+}
 op {
   name: "SparseDenseCwiseDiv"
   input_arg {
@@ -28222,6 +34851,98 @@ op {
     }
   }
 }
+op {
+  name: "SparseDenseCwiseDiv"
+  input_arg {
+    name: "sp_indices"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "sp_values"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "sp_shape"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "dense"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+}
+op {
+  name: "SparseDenseCwiseMul"
+  input_arg {
+    name: "sp_indices"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "sp_values"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "sp_shape"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "dense"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
+      }
+    }
+  }
+}
 op {
   name: "SparseDenseCwiseMul"
   input_arg {
@@ -28263,6 +34984,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -28434,19 +35157,232 @@ op {
       list {
         type: DT_FLOAT
         type: DT_DOUBLE
-        type: DT_INT32
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_UINT16
+        type: DT_HALF
+      }
+    }
+  }
+}
+op {
+  name: "SparseReduceMax"
+  input_arg {
+    name: "input_indices"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "input_values"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "input_shape"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "reduction_axes"
+    type: DT_INT32
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  attr {
+    name: "keep_dims"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_UINT16
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+}
+op {
+  name: "SparseReduceMaxSparse"
+  input_arg {
+    name: "input_indices"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "input_values"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "input_shape"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "reduction_axes"
+    type: DT_INT32
+  }
+  output_arg {
+    name: "output_indices"
+    type: DT_INT64
+  }
+  output_arg {
+    name: "output_values"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "output_shape"
+    type: DT_INT64
+  }
+  attr {
+    name: "keep_dims"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_UINT16
+        type: DT_HALF
+      }
+    }
+  }
+}
+op {
+  name: "SparseReduceMaxSparse"
+  input_arg {
+    name: "input_indices"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "input_values"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "input_shape"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "reduction_axes"
+    type: DT_INT32
+  }
+  output_arg {
+    name: "output_indices"
+    type: DT_INT64
+  }
+  output_arg {
+    name: "output_values"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "output_shape"
+    type: DT_INT64
+  }
+  attr {
+    name: "keep_dims"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_UINT16
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+}
+op {
+  name: "SparseReduceSum"
+  input_arg {
+    name: "input_indices"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "input_values"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "input_shape"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "reduction_axes"
+    type: DT_INT32
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  attr {
+    name: "keep_dims"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
         type: DT_INT64
+        type: DT_INT32
         type: DT_UINT8
+        type: DT_UINT16
         type: DT_INT16
         type: DT_INT8
-        type: DT_UINT16
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
         type: DT_HALF
       }
     }
   }
 }
 op {
-  name: "SparseReduceMaxSparse"
+  name: "SparseReduceSum"
   input_arg {
     name: "input_indices"
     type: DT_INT64
@@ -28464,17 +35400,9 @@ op {
     type: DT_INT32
   }
   output_arg {
-    name: "output_indices"
-    type: DT_INT64
-  }
-  output_arg {
-    name: "output_values"
+    name: "output"
     type_attr: "T"
   }
-  output_arg {
-    name: "output_shape"
-    type: DT_INT64
-  }
   attr {
     name: "keep_dims"
     type: "bool"
@@ -28489,19 +35417,26 @@ op {
       list {
         type: DT_FLOAT
         type: DT_DOUBLE
-        type: DT_INT32
         type: DT_INT64
+        type: DT_INT32
         type: DT_UINT8
+        type: DT_UINT16
         type: DT_INT16
         type: DT_INT8
-        type: DT_UINT16
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
 }
 op {
-  name: "SparseReduceSum"
+  name: "SparseReduceSumSparse"
   input_arg {
     name: "input_indices"
     type: DT_INT64
@@ -28519,9 +35454,17 @@ op {
     type: DT_INT32
   }
   output_arg {
-    name: "output"
+    name: "output_indices"
+    type: DT_INT64
+  }
+  output_arg {
+    name: "output_values"
     type_attr: "T"
   }
+  output_arg {
+    name: "output_shape"
+    type: DT_INT64
+  }
   attr {
     name: "keep_dims"
     type: "bool"
@@ -28608,6 +35551,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -28887,6 +35832,57 @@ op {
     }
   }
 }
+op {
+  name: "SparseSegmentSum"
+  input_arg {
+    name: "data"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "indices"
+    type_attr: "Tidx"
+  }
+  input_arg {
+    name: "segment_ids"
+    type: DT_INT32
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_UINT16
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+  attr {
+    name: "Tidx"
+    type: "type"
+    default_value {
+      type: DT_INT32
+    }
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+}
 op {
   name: "SparseSlice"
   input_arg {
@@ -29050,6 +36046,60 @@ op {
     }
   }
 }
+op {
+  name: "SparseSparseMaximum"
+  input_arg {
+    name: "a_indices"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "a_values"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "a_shape"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "b_indices"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "b_values"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "b_shape"
+    type: DT_INT64
+  }
+  output_arg {
+    name: "output_indices"
+    type: DT_INT64
+  }
+  output_arg {
+    name: "output_values"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_UINT16
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+}
 op {
   name: "SparseSparseMinimum"
   input_arg {
@@ -29107,6 +36157,65 @@ op {
     }
   }
 }
+op {
+  name: "SparseSparseMinimum"
+  input_arg {
+    name: "a_indices"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "a_values"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "a_shape"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "b_indices"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "b_values"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "b_shape"
+    type: DT_INT64
+  }
+  output_arg {
+    name: "output_indices"
+    type: DT_INT64
+  }
+  output_arg {
+    name: "output_values"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+}
 op {
   name: "SparseSplit"
   input_arg {
@@ -29206,6 +36315,63 @@ op {
     }
   }
 }
+op {
+  name: "SparseTensorDenseAdd"
+  input_arg {
+    name: "a_indices"
+    type_attr: "Tindices"
+  }
+  input_arg {
+    name: "a_values"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "a_shape"
+    type_attr: "Tindices"
+  }
+  input_arg {
+    name: "b"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+  attr {
+    name: "Tindices"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+}
 op {
   name: "SparseTensorDenseMatMul"
   input_arg {
@@ -30622,60 +37788,117 @@ op {
   }
 }
 op {
-  name: "Sub"
+  name: "Sub"
+  input_arg {
+    name: "x"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "y"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "z"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_HALF
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_UINT8
+        type: DT_INT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+      }
+    }
+  }
+}
+op {
+  name: "Substr"
   input_arg {
-    name: "x"
+    name: "input"
+    type: DT_STRING
+  }
+  input_arg {
+    name: "pos"
     type_attr: "T"
   }
   input_arg {
-    name: "y"
+    name: "len"
     type_attr: "T"
   }
   output_arg {
-    name: "z"
-    type_attr: "T"
+    name: "output"
+    type: DT_STRING
   }
   attr {
     name: "T"
     type: "type"
     allowed_values {
       list {
-        type: DT_HALF
-        type: DT_FLOAT
-        type: DT_DOUBLE
-        type: DT_UINT8
-        type: DT_INT8
-        type: DT_UINT16
-        type: DT_INT16
         type: DT_INT32
         type: DT_INT64
-        type: DT_COMPLEX64
-        type: DT_COMPLEX128
       }
     }
   }
 }
 op {
-  name: "Substr"
+  name: "Sum"
   input_arg {
     name: "input"
-    type: DT_STRING
-  }
-  input_arg {
-    name: "pos"
     type_attr: "T"
   }
   input_arg {
-    name: "len"
-    type_attr: "T"
+    name: "reduction_indices"
+    type_attr: "Tidx"
   }
   output_arg {
     name: "output"
-    type: DT_STRING
+    type_attr: "T"
+  }
+  attr {
+    name: "keep_dims"
+    type: "bool"
+    default_value {
+      b: false
+    }
   }
   attr {
     name: "T"
     type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
+      }
+    }
+  }
+  attr {
+    name: "Tidx"
+    type: "type"
+    default_value {
+      type: DT_INT32
+    }
     allowed_values {
       list {
         type: DT_INT32
@@ -30724,6 +37947,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -32420,6 +39645,98 @@ op {
     version: 7
   }
 }
+op {
+  name: "TopK"
+  input_arg {
+    name: "input"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "values"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "indices"
+    type: DT_INT32
+  }
+  attr {
+    name: "k"
+    type: "int"
+    has_minimum: true
+  }
+  attr {
+    name: "sorted"
+    type: "bool"
+    default_value {
+      b: true
+    }
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_UINT16
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+  deprecation {
+    version: 7
+  }
+}
+op {
+  name: "TopKV2"
+  input_arg {
+    name: "input"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "k"
+    type: DT_INT32
+  }
+  output_arg {
+    name: "values"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "indices"
+    type: DT_INT32
+  }
+  attr {
+    name: "sorted"
+    type: "bool"
+    default_value {
+      b: true
+    }
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_UINT16
+        type: DT_HALF
+      }
+    }
+  }
+}
 op {
   name: "TopKV2"
   input_arg {
@@ -32459,6 +39776,8 @@ op {
         type: DT_INT8
         type: DT_UINT16
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -32856,6 +40175,105 @@ op {
     }
   }
 }
+op {
+  name: "UnsortedSegmentMax"
+  input_arg {
+    name: "data"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "segment_ids"
+    type_attr: "Tindices"
+  }
+  input_arg {
+    name: "num_segments"
+    type: DT_INT32
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_UINT16
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+  attr {
+    name: "Tindices"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+}
+op {
+  name: "UnsortedSegmentSum"
+  input_arg {
+    name: "data"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "segment_ids"
+    type_attr: "Tindices"
+  }
+  input_arg {
+    name: "num_segments"
+    type: DT_INT32
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
+      }
+    }
+  }
+  attr {
+    name: "Tindices"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+}
 op {
   name: "UnsortedSegmentSum"
   input_arg {
@@ -32893,6 +40311,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index 88e57ea0cb..53d99178e5 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -82,6 +82,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -157,6 +159,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -334,6 +338,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
         type: DT_VARIANT
       }
     }
@@ -738,6 +744,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -801,6 +809,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -885,6 +895,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -978,6 +990,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -1075,6 +1089,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -1159,6 +1175,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -1247,6 +1265,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -1304,6 +1324,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -1371,6 +1393,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -1452,6 +1476,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -1519,6 +1545,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -1602,6 +1630,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -1649,6 +1679,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -1696,6 +1728,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -1762,6 +1796,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -2025,6 +2061,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -2098,6 +2136,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -3337,6 +3377,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -3428,6 +3470,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -3722,6 +3766,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -3773,6 +3819,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -3829,6 +3877,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -4773,6 +4823,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -5723,6 +5775,8 @@ op {
         type: DT_INT8
         type: DT_UINT16
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -5780,6 +5834,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -5850,6 +5906,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -7055,6 +7113,8 @@ op {
         type: DT_INT8
         type: DT_UINT16
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -7122,6 +7182,8 @@ op {
         type: DT_INT8
         type: DT_UINT16
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -7188,6 +7250,8 @@ op {
         type: DT_INT8
         type: DT_UINT16
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -7960,6 +8024,8 @@ op {
         type: DT_INT8
         type: DT_UINT16
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -9925,6 +9991,8 @@ op {
         type: DT_INT8
         type: DT_UINT16
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -9959,6 +10027,8 @@ op {
         type: DT_INT8
         type: DT_UINT16
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -10183,6 +10253,8 @@ op {
         type: DT_INT8
         type: DT_UINT16
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -11460,6 +11532,8 @@ op {
         type: DT_INT8
         type: DT_UINT16
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -11494,6 +11568,8 @@ op {
         type: DT_INT8
         type: DT_UINT16
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -12969,6 +13045,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -13361,6 +13439,8 @@ op {
         type: DT_INT8
         type: DT_UINT16
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -13441,6 +13521,8 @@ op {
         type: DT_INT8
         type: DT_UINT16
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -13517,6 +13599,8 @@ op {
         type: DT_INT8
         type: DT_UINT16
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -13593,6 +13677,8 @@ op {
         type: DT_INT8
         type: DT_UINT16
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -13672,6 +13758,8 @@ op {
         type: DT_INT8
         type: DT_UINT16
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -13748,6 +13836,8 @@ op {
         type: DT_INT8
         type: DT_UINT16
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -13893,6 +13983,8 @@ op {
         type: DT_INT8
         type: DT_UINT16
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -13974,6 +14066,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -14165,6 +14259,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -14424,6 +14520,8 @@ op {
         type: DT_INT8
         type: DT_UINT16
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -16612,6 +16710,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -20165,6 +20265,8 @@ op {
         type: DT_INT8
         type: DT_UINT16
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -20194,6 +20296,8 @@ op {
         type: DT_INT8
         type: DT_UINT16
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -20230,6 +20334,8 @@ op {
         type: DT_INT8
         type: DT_UINT16
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -20266,6 +20372,8 @@ op {
         type: DT_INT8
         type: DT_UINT16
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -20864,6 +20972,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -20920,6 +21030,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -20996,6 +21108,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -21081,6 +21195,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -21169,6 +21285,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -21245,6 +21363,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -21325,6 +21445,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -21376,6 +21498,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -21436,6 +21560,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -21510,6 +21636,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -21571,6 +21699,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -21646,6 +21776,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -21736,6 +21868,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -21813,6 +21947,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -21883,6 +22019,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -21974,6 +22112,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -22068,6 +22208,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -22159,6 +22301,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -22254,6 +22398,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -22330,6 +22476,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -22419,6 +22567,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -22495,6 +22645,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -22585,6 +22737,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -23401,6 +23555,8 @@ op {
         type: DT_INT8
         type: DT_UINT16
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -23497,6 +23653,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -23564,6 +23722,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -23631,6 +23791,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -23737,6 +23899,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -23802,6 +23966,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -23861,6 +24027,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -23977,6 +24145,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -24263,6 +24433,8 @@ op {
         type: DT_INT8
         type: DT_UINT16
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -24309,6 +24481,8 @@ op {
         type: DT_INT8
         type: DT_UINT16
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -24355,6 +24529,8 @@ op {
         type: DT_INT8
         type: DT_UINT16
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -24406,6 +24582,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -24457,6 +24635,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -25338,6 +25518,8 @@ op {
         type: DT_INT8
         type: DT_UINT16
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -25374,6 +25556,8 @@ op {
         type: DT_INT8
         type: DT_UINT16
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -25403,6 +25587,8 @@ op {
         type: DT_INT8
         type: DT_UINT16
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -25439,6 +25625,8 @@ op {
         type: DT_INT8
         type: DT_UINT16
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -25626,6 +25814,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -25685,6 +25875,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -25759,6 +25951,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -25776,6 +25970,8 @@ op {
         type: DT_INT8
         type: DT_UINT16
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -25833,6 +26029,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -25908,6 +26106,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -25985,6 +26185,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -26084,6 +26286,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -26187,6 +26391,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -26286,6 +26492,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -26389,6 +26597,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -26472,6 +26682,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -26568,6 +26780,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -26650,6 +26864,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -26748,6 +26964,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -26853,6 +27071,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -27032,6 +27252,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -27084,6 +27306,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -27136,6 +27360,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -27339,6 +27565,8 @@ op {
         type: DT_INT8
         type: DT_UINT16
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -27401,6 +27629,8 @@ op {
         type: DT_INT8
         type: DT_UINT16
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -27461,6 +27691,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -27528,6 +27760,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -27831,6 +28065,8 @@ op {
         type: DT_INT8
         type: DT_UINT16
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -28038,6 +28274,8 @@ op {
         type: DT_INT8
         type: DT_UINT16
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -28105,6 +28343,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -28209,6 +28449,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -29732,6 +29974,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -31500,6 +31744,8 @@ op {
         type: DT_INT8
         type: DT_UINT16
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -31554,6 +31800,8 @@ op {
         type: DT_INT8
         type: DT_UINT16
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -31927,6 +32175,8 @@ op {
         type: DT_INT8
         type: DT_UINT16
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -31982,6 +32232,8 @@ op {
         type: DT_QUINT8
         type: DT_QINT32
         type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
-- 
GitLab


From 091504af57f70df13ebf1db9946dc59482e1190a Mon Sep 17 00:00:00 2001
From: Eugene Brevdo <ebrevdo@google.com>
Date: Tue, 10 Oct 2017 10:29:43 -0700
Subject: [PATCH 223/909] Fix gradient behavior of fully dynamic tensor arrays
 + stop_gradients on tf.scan.

Added a test checking that this fixes a bug with tf.stop_gradient of tf.scan output.

PiperOrigin-RevId: 171697920
---
 tensorflow/core/kernels/tensor_array.h        | 49 +++++++++++++++++--
 .../kernel_tests/functional_ops_test.py       | 12 +++++
 .../kernel_tests/tensor_array_ops_test.py     |  8 ++-
 tensorflow/python/ops/functional_ops.py       |  8 +--
 tensorflow/python/ops/tensor_array_ops.py     | 28 +++++------
 5 files changed, 80 insertions(+), 25 deletions(-)

diff --git a/tensorflow/core/kernels/tensor_array.h b/tensorflow/core/kernels/tensor_array.h
index b43fafe921..6882a8a0e5 100644
--- a/tensorflow/core/kernels/tensor_array.h
+++ b/tensorflow/core/kernels/tensor_array.h
@@ -460,8 +460,9 @@ Status TensorArray::LockedWriteOrAggregate(OpKernelContext* ctx,
         "TensorArray ", handle_.vec<string>()(1),
         ": Could not write to TensorArray index ", index,
         " because the value shape is ", value_t->shape().DebugString(),
-        " which is incompatible with the TensorArray's element shape: ",
-        element_shape_.DebugString(), ".");
+        " which is incompatible with the TensorArray's inferred element "
+        "shape: ",
+        element_shape_.DebugString(), " (consider setting infer_shape=False).");
   }
 
   if (t.read) {
@@ -530,11 +531,53 @@ template <typename Device, typename T>
 Status TensorArray::LockedRead(OpKernelContext* ctx, const int32 index,
                                PersistentTensor* value) {
   TF_RETURN_IF_ERROR(LockedReturnIfClosed());
-  if (index < 0 || static_cast<size_t>(index) >= tensors_.size()) {
+  if ((index < 0) ||
+      (!is_grad_ && (static_cast<size_t>(index) >= tensors_.size()))) {
     return errors::InvalidArgument("Tried to read from index ", index,
                                    " but array size is: ", tensors_.size());
   }
+  size_t index_t = static_cast<size_t>(index);
+  if (is_grad_ && (index_t >= tensors_.size() || !tensors_[index].written)) {
+    // Special case returning zeros if this is a gradient read that happens
+    // after a stop_gradients call with dynamic forward TensorArrays.
+    // There is sometimes a race condition where the gradient is not
+    // written due to stop_gradients, but is later read.
+    TensorShape element_shape;
+    if (index_t < tensors_.size() && tensors_[index].shape.dims() > 0) {
+      element_shape = tensors_[index].shape;
+    } else if (!element_shape_.IsFullyDefined()) {
+      return errors::InvalidArgument(
+          "TensorArray ", handle_.vec<string>()(1),
+          ": Could not read from gradient TensorArray index ", index,
+          ".  Furthermore, the element shape is not fully defined: ",
+          element_shape_.DebugString(),
+          ".  "
+          "It is likely you are working with a resizeable TensorArray and "
+          "stop_gradients "
+          "is not allowing the gradients to be written.  If you set the full "
+          "element_shape "
+          "property on the forward TensorArray, the proper all-zeros tensor "
+          "will be "
+          "returned instead of incurring this error.");
+    } else {
+      DCHECK(element_shape_.AsTensorShape(&element_shape));
+    }
+    if (index_t >= tensors_.size()) {
+      // Fill in tensors_ up to index to have known shape.
+      size_t old_tensors_size = tensors_.size();
+      tensors_.resize(index + 1);
+      for (size_t i = old_tensors_size; i < index + 1; ++i) {
+        tensors_[i].shape = element_shape;
+        tensors_[i].written = true;
+      }
+    } else {
+      tensors_[index].shape = element_shape;
+      tensors_[index].written = true;
+    }
+  }
+
   TensorAndState& t = tensors_[index];
+
   if (!t.written) {
     return errors::InvalidArgument("TensorArray ", handle_.vec<string>()(1),
                                    ": Could not read from TensorArray index ",
diff --git a/tensorflow/python/kernel_tests/functional_ops_test.py b/tensorflow/python/kernel_tests/functional_ops_test.py
index 429b6c2e83..21fe588ac1 100644
--- a/tensorflow/python/kernel_tests/functional_ops_test.py
+++ b/tensorflow/python/kernel_tests/functional_ops_test.py
@@ -371,6 +371,18 @@ class FunctionalOpsTest(test.TestCase):
       r = gradients_impl.gradients(r, v)[0]
       self.assertAllEqual(873.0, r.eval())
 
+  def testScanGradientWithPartStopGradient(self):
+    a = variables.Variable(0.0, name="a")
+    b = variables.Variable(0.0, name="b")
+    elems = array_ops.zeros(5)
+    l0, l1 = functional_ops.scan(
+        lambda elem_, input_: (a, b), elems, initializer=(0., 0.))
+    loss = l0 + array_ops.stop_gradient(l1)
+    grad = gradients_impl.gradients(ys=[loss], xs=[a, b])
+    with self.test_session(use_gpu=True) as sess:
+      variables.global_variables_initializer().run()
+      sess.run(grad)
+
   def testFoldShape(self):
     with self.test_session():
       x = constant_op.constant([[1, 2, 3], [4, 5, 6]])
diff --git a/tensorflow/python/kernel_tests/tensor_array_ops_test.py b/tensorflow/python/kernel_tests/tensor_array_ops_test.py
index cffedf63f7..fc4f9b22b9 100644
--- a/tensorflow/python/kernel_tests/tensor_array_ops_test.py
+++ b/tensorflow/python/kernel_tests/tensor_array_ops_test.py
@@ -1066,7 +1066,10 @@ class TensorArrayTest(test.TestCase):
           infer_shape=True)
       w0 = ta1.split(value, [1, 2])
       r0 = w0.read(0)
-      self.assertAllEqual(r0.get_shape(), tensor_shape.unknown_shape())
+      self.assertEqual(r0.get_shape().ndims, None)
+      self.assertEqual(
+          tensor_shape.TensorShape(
+              ta1.handle.op.get_attr("element_shape")).ndims, None)
 
   def testWriteUnknownShape(self):
     with self.test_session(use_gpu=True):
@@ -1142,10 +1145,11 @@ class TensorArrayTest(test.TestCase):
       # Don't actually perform the pack.  This stores the static shape.
       ta.unstack(array_ops.zeros([0, 3, 5])).mark_used()
       packed = ta.stack()
+      concatenated = ta.concat()
       self.assertAllEqual([0, 3, 5], packed.eval().shape)
       # Concatenating zero tensors along their first dimension gives a
       # first dimension of zero
-      self.assertAllEqual([0, 5], ta.concat().eval().shape)
+      self.assertAllEqual([0, 5], concatenated.eval().shape)
 
   def testTensorArrayEvalEmptyWithDefault(self):
     self._testTensorArrayEvalEmptyWithDefault()
diff --git a/tensorflow/python/ops/functional_ops.py b/tensorflow/python/ops/functional_ops.py
index 413c29850e..96b799f610 100644
--- a/tensorflow/python/ops/functional_ops.py
+++ b/tensorflow/python/ops/functional_ops.py
@@ -545,9 +545,11 @@ def scan(fn, elems, initializer=None, parallel_iterations=10, back_prop=True,
 
     # Create a tensor array to store the intermediate values.
     accs_ta = [
-        tensor_array_ops.TensorArray(dtype=init.dtype, size=n,
-                                     dynamic_size=False,
-                                     infer_shape=infer_shape)
+        tensor_array_ops.TensorArray(
+            dtype=init.dtype, size=n,
+            element_shape=init.shape if infer_shape else None,
+            dynamic_size=False,
+            infer_shape=infer_shape)
         for init in a_flat]
 
     if initializer is None:
diff --git a/tensorflow/python/ops/tensor_array_ops.py b/tensorflow/python/ops/tensor_array_ops.py
index 08325ba771..37b4b3bcf9 100644
--- a/tensorflow/python/ops/tensor_array_ops.py
+++ b/tensorflow/python/ops/tensor_array_ops.py
@@ -301,6 +301,8 @@ class TensorArray(object):
     """
     with ops.name_scope(name, "TensorArrayWrite", [self._handle, index, value]):
       value = ops.convert_to_tensor(value, name="value")
+      if self._infer_shape:
+        self._merge_element_shape(value.shape)
       with self._maybe_colocate_with(value):
         flow_out = gen_data_flow_ops._tensor_array_write_v3(
             handle=self._handle,
@@ -314,8 +316,6 @@ class TensorArray(object):
       ta._infer_shape = self._infer_shape
       ta._element_shape = self._element_shape
       ta._colocate_with = self._colocate_with
-      if ta._infer_shape:
-        ta._merge_element_shape(value.get_shape())
       return ta
 
   def stack(self, name=None):
@@ -433,6 +433,8 @@ class TensorArray(object):
     with ops.name_scope(name, "TensorArrayScatter",
                         [self._handle, value, indices]):
       value = ops.convert_to_tensor(value, name="value")
+      if self._infer_shape and context.in_graph_mode():
+        self._merge_element_shape(value.shape[1:])
       with self._maybe_colocate_with(value):
         flow_out = gen_data_flow_ops._tensor_array_scatter_v3(
             handle=self._handle,
@@ -446,12 +448,6 @@ class TensorArray(object):
       ta._infer_shape = self._infer_shape
       ta._element_shape = self._element_shape
       ta._colocate_with = self._colocate_with
-      if ta._infer_shape and context.in_graph_mode():
-        val_shape = flow_out.op.inputs[2].get_shape()
-        element_shape = tensor_shape.unknown_shape()
-        if val_shape.dims is not None:
-          element_shape = tensor_shape.TensorShape(val_shape.dims[1:])
-        ta._merge_element_shape(element_shape)
       return ta
 
   @tf_should_use.should_use_result
@@ -476,6 +472,13 @@ class TensorArray(object):
       value = ops.convert_to_tensor(value, name="value")
       with self._maybe_colocate_with(value):
         lengths_64 = math_ops.to_int64(lengths)
+        if self._infer_shape and context.in_graph_mode():
+          clengths = tensor_util.constant_value(lengths_64)
+          if value.shape.dims is not None:
+            if clengths is not None and clengths.max() == clengths.min():
+              self._merge_element_shape(
+                  tensor_shape.TensorShape([clengths[0]]).concatenate(
+                      value.shape[1:]))
         flow_out = gen_data_flow_ops._tensor_array_split_v3(
             handle=self._handle,
             value=value,
@@ -488,15 +491,6 @@ class TensorArray(object):
       ta._infer_shape = self._infer_shape
       ta._element_shape = self._element_shape
       ta._colocate_with = self._colocate_with
-      if ta._infer_shape and context.in_graph_mode():
-        val_shape = flow_out.op.inputs[1].get_shape()
-        clengths = tensor_util.constant_value(flow_out.op.inputs[2])
-        element_shape = tensor_shape.unknown_shape()
-        if val_shape.dims is not None:
-          if clengths is not None and clengths.max() == clengths.min():
-            element_shape = tensor_shape.TensorShape([clengths[0]] +
-                                                     val_shape.dims[1:])
-        ta._merge_element_shape(element_shape)
       return ta
 
   def size(self, name=None):
-- 
GitLab


From a83154967bb2955acc234f4a64b63b505508b728 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 10 Oct 2017 10:31:15 -0700
Subject: [PATCH 224/909] Improve Eager mode random numbers.

PiperOrigin-RevId: 171698189
---
 tensorflow/python/eager/context.py            | 36 +++++++++++++++++++
 tensorflow/python/framework/random_seed.py    | 24 ++++++++++---
 .../python/framework/random_seed_test.py      | 11 +++++-
 tensorflow/python/framework/test_util.py      |  2 +-
 .../kernel_tests/multinomial_op_test.py       | 13 +++++--
 5 files changed, 77 insertions(+), 9 deletions(-)

diff --git a/tensorflow/python/eager/context.py b/tensorflow/python/eager/context.py
index be3d535271..996748a870 100644
--- a/tensorflow/python/eager/context.py
+++ b/tensorflow/python/eager/context.py
@@ -20,6 +20,7 @@ from __future__ import print_function
 
 import contextlib
 import copy
+import random
 import threading
 
 from tensorflow.python import pywrap_tensorflow
@@ -42,6 +43,8 @@ _default_mode = GRAPH_MODE
 # and the idempotent nature of writes to provide thread safety.
 _device_parsing_cache = {}
 
+_MAXINT32 = 2**31 - 1
+
 
 # TODO(agarwal): better name ?
 class _EagerContext(threading.local):
@@ -76,8 +79,26 @@ class Context(object):
     self._summary_writer_resource = None
     self._post_execution_callbacks = []
     self._config = config
+    self._seed = None
     self._initialize_lock = threading.Lock()
 
+  def _set_global_seed(self, seed):
+    """Set a global eager mode seed for random ops."""
+    self._seed = seed
+    self._rng = random.Random(self._seed)
+
+  def _internal_operation_seed(self):
+    """Returns a fake operation seed.
+
+      In eager mode, user shouldn't set or depend on operation seed.
+      Here, we generate a random seed based on global seed to make
+      operation's randomness different and depend on the global seed.
+
+    Returns:
+      A fake operation seed based on global seed.
+    """
+    return self._rng.randint(0, _MAXINT32)
+
   def _initialize_handle_and_devices(self):
     """Initialize handle and devices."""
     with self._initialize_lock:
@@ -326,6 +347,21 @@ def get_default_context():
   return _context
 
 
+def set_global_seed(seed):
+  """Sets the eager mode seed."""
+  context()._set_global_seed(seed)  # pylint: disable=protected-access
+
+
+def global_seed():
+  """Returns the eager mode seed."""
+  return context()._seed  # pylint: disable=protected-access
+
+
+def internal_operation_seed():
+  """Returns the operation seed generated based on global seed."""
+  return context()._internal_operation_seed()  # pylint: disable=protected-access
+
+
 def in_graph_mode():
   """Returns True if current thread is in GRAPH mode for default context."""
   return context().in_graph_mode()
diff --git a/tensorflow/python/framework/random_seed.py b/tensorflow/python/framework/random_seed.py
index 0d8bd4bcf1..5f1130570d 100644
--- a/tensorflow/python/framework/random_seed.py
+++ b/tensorflow/python/framework/random_seed.py
@@ -20,6 +20,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from tensorflow.python.eager import context
 from tensorflow.python.framework import ops
 
 
@@ -49,12 +50,22 @@ def get_seed(op_seed):
     A tuple of two integers that should be used for the local seed of this
     operation.
   """
-  graph_seed = ops.get_default_graph().seed
-  if graph_seed is not None:
+  is_graph_mode = context.in_graph_mode()
+
+  if is_graph_mode:
+    global_seed = ops.get_default_graph().seed
+  else:
+    global_seed = context.global_seed()
+
+  if global_seed is not None:
     if op_seed is None:
       # pylint: disable=protected-access
-      op_seed = ops.get_default_graph()._last_id
-    seeds = _truncate_seed(graph_seed), _truncate_seed(op_seed)
+      if is_graph_mode:
+        op_seed = ops.get_default_graph()._last_id
+      else:
+        op_seed = context.internal_operation_seed()
+
+    seeds = _truncate_seed(global_seed), _truncate_seed(op_seed)
   else:
     if op_seed is not None:
       seeds = DEFAULT_GRAPH_SEED, _truncate_seed(op_seed)
@@ -162,4 +173,7 @@ def set_random_seed(seed):
   Args:
     seed: integer.
   """
-  ops.get_default_graph().seed = seed
+  if context.in_graph_mode():
+    ops.get_default_graph().seed = seed
+  else:
+    context.set_global_seed(seed)
diff --git a/tensorflow/python/framework/random_seed_test.py b/tensorflow/python/framework/random_seed_test.py
index c1d2b05b0b..b4c98ab8b2 100644
--- a/tensorflow/python/framework/random_seed_test.py
+++ b/tensorflow/python/framework/random_seed_test.py
@@ -18,12 +18,15 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from tensorflow.python.eager import context
 from tensorflow.python.framework import random_seed
+from tensorflow.python.framework import test_util
 from tensorflow.python.platform import test
 
 
 class RandomSeedTest(test.TestCase):
 
+  @test_util.run_in_graph_and_eager_modes()
   def testRandomSeed(self):
     test_cases = [
         # Each test case is a tuple with input to get_seed:
@@ -32,12 +35,18 @@ class RandomSeedTest(test.TestCase):
         # (output_graph_seed, output_op_seed)
         ((None, None), (None, None)),
         ((None, 1), (random_seed.DEFAULT_GRAPH_SEED, 1)),
-        ((1, None), (1, 0)),  # 0 will be the default_graph._lastid.
         ((1, 1), (1, 1)),
         ((0, 0), (0, 2**31 - 1)),  # Avoid nondeterministic (0, 0) output
         ((2**31 - 1, 0), (0, 2**31 - 1)),  # Don't wrap to (0, 0) either
         ((0, 2**31 - 1), (0, 2**31 - 1)),  # Wrapping for the other argument
     ]
+    if context.in_graph_mode():
+      # 0 will be the default_graph._lastid.
+      test_cases.append(((1, None), (1, 0)))
+    else:
+      # operation seed is random number generated based on global seed.
+      # it's not tested due to possibility of platform or version difference.
+      pass
     for tc in test_cases:
       tinput, toutput = tc[0], tc[1]
       random_seed.set_random_seed(tinput[0])
diff --git a/tensorflow/python/framework/test_util.py b/tensorflow/python/framework/test_util.py
index ef733136f4..c681ffb514 100644
--- a/tensorflow/python/framework/test_util.py
+++ b/tensorflow/python/framework/test_util.py
@@ -519,7 +519,7 @@ class TensorFlowTestCase(googletest.TestCase):
     # cleared first.
     ops._default_graph_stack.reset()  # pylint: disable=protected-access
     ops.reset_default_graph()
-    ops.get_default_graph().seed = random_seed.DEFAULT_GRAPH_SEED
+    random_seed.set_random_seed(random_seed.DEFAULT_GRAPH_SEED)
 
   def tearDown(self):
     for thread in self._threads:
diff --git a/tensorflow/python/kernel_tests/multinomial_op_test.py b/tensorflow/python/kernel_tests/multinomial_op_test.py
index d6e1b2b4c0..ca48ba6cad 100644
--- a/tensorflow/python/kernel_tests/multinomial_op_test.py
+++ b/tensorflow/python/kernel_tests/multinomial_op_test.py
@@ -25,9 +25,11 @@ import numpy as np
 
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.python.client import session
+from tensorflow.python.eager import context
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import random_seed
 from tensorflow.python.framework import tensor_shape
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import math_ops
@@ -52,13 +54,14 @@ native_sampler = random_ops.multinomial
 
 class MultinomialTest(test.TestCase):
 
+  @test_util.run_in_graph_and_eager_modes()
   def testSmallEntropy(self):
     random_seed.set_random_seed(1618)
-    with self.test_session(use_gpu=True):
+    with test_util.device(use_gpu=True):
       # A logit value of -10 corresponds to a probability of ~5e-5.
       logits = constant_op.constant([[-10., 10., -10.], [-10., -10., 10.]])
       num_samples = 1000
-      samples = random_ops.multinomial(logits, num_samples).eval()
+      samples = self.evaluate(random_ops.multinomial(logits, num_samples))
       self.assertAllEqual([[1] * num_samples, [2] * num_samples], samples)
 
   def testOneOpMultipleStepsIndependent(self):
@@ -69,6 +72,12 @@ class MultinomialTest(test.TestCase):
       sample1b = sess.run(sample_op1)
       self.assertFalse(np.equal(sample1a, sample1b).all())
 
+  def testEagerOneOpMultipleStepsIndependent(self):
+    with context.eager_mode(), test_util.device(use_gpu=True):
+      sample1, sample2 = self._make_ops(10)
+      # Consecutive runs shouldn't yield identical output.
+      self.assertFalse(np.equal(sample1.numpy(), sample2.numpy()).all())
+
   def testTwoOpsIndependent(self):
     with self.test_session(use_gpu=True) as sess:
       sample_op1, sample_op2 = self._make_ops(32)
-- 
GitLab


From 1bd776c9c217474b07c29dcd9d8fbbb6eba93ea0 Mon Sep 17 00:00:00 2001
From: Eugene Brevdo <ebrevdo@google.com>
Date: Tue, 10 Oct 2017 10:45:09 -0700
Subject: [PATCH 225/909] Automated g4 rollback of changelist 170772848

PiperOrigin-RevId: 171700278
---
 tensorflow/core/kernels/BUILD                 |  12 +-
 tensorflow/core/kernels/where_op.cc           | 140 ++++++++-----
 tensorflow/core/kernels/where_op.h            |  20 +-
 .../{where_op_gpu.cu.cc => where_op_gpu.cu.h} | 186 +++++++++++++-----
 .../core/kernels/where_op_gpu_impl_1.cu.cc    |  18 ++
 .../core/kernels/where_op_gpu_impl_2.cu.cc    |  18 ++
 .../core/kernels/where_op_gpu_impl_3.cu.cc    |  18 ++
 .../core/kernels/where_op_gpu_impl_4.cu.cc    |  18 ++
 .../core/kernels/where_op_gpu_impl_5.cu.cc    |  18 ++
 tensorflow/core/ops/array_ops.cc              |  33 +++-
 tensorflow/python/kernel_tests/BUILD          |   2 +-
 .../python/kernel_tests/where_op_test.py      |  38 ++++
 tensorflow/python/ops/array_ops.py            |   4 +-
 13 files changed, 422 insertions(+), 103 deletions(-)
 rename tensorflow/core/kernels/{where_op_gpu.cu.cc => where_op_gpu.cu.h} (53%)
 create mode 100644 tensorflow/core/kernels/where_op_gpu_impl_1.cu.cc
 create mode 100644 tensorflow/core/kernels/where_op_gpu_impl_2.cu.cc
 create mode 100644 tensorflow/core/kernels/where_op_gpu_impl_3.cu.cc
 create mode 100644 tensorflow/core/kernels/where_op_gpu_impl_4.cu.cc
 create mode 100644 tensorflow/core/kernels/where_op_gpu_impl_5.cu.cc

diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index ad6f84304d..3b7d803bea 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -838,7 +838,17 @@ tf_kernel_library(
 
 tf_kernel_library(
     name = "where_op",
-    prefix = "where_op",
+    srcs = ["where_op.cc"],
+    hdrs = ["where_op.h"],
+    gpu_srcs = [
+        "where_op.h",
+        "where_op_gpu.cu.h",
+        "where_op_gpu_impl_1.cu.cc",
+        "where_op_gpu_impl_2.cu.cc",
+        "where_op_gpu_impl_3.cu.cc",
+        "where_op_gpu_impl_4.cu.cc",
+        "where_op_gpu_impl_5.cu.cc",
+    ],
     deps = if_cuda([
         ":cuda_solvers",
         "@cub_archive//:cub",
diff --git a/tensorflow/core/kernels/where_op.cc b/tensorflow/core/kernels/where_op.cc
index 59b474e41c..42d1365e64 100644
--- a/tensorflow/core/kernels/where_op.cc
+++ b/tensorflow/core/kernels/where_op.cc
@@ -52,19 +52,33 @@ typedef Eigen::GpuDevice GPUDevice;
 
 namespace functor {
 
+namespace {
+template <typename T>
+int64 CountAccumulator(const T* begin, const T* end) {
+  return std::accumulate(begin, end, 0L, [](int64 accum, const T& val) {
+    return accum + (val != T(0));
+  });
+}
+
 template <>
-struct NumTrue<CPUDevice, int64> {
+int64 CountAccumulator<bool>(const bool* begin, const bool* end) {
+  return std::accumulate(begin, end, 0L);
+}
+
+}  // namespace
+
+template <typename T>
+struct NumTrue<CPUDevice, T, int64> {
   static Status Compute(OpKernelContext* ctx, const CPUDevice& d,
-                        TTypes<bool>::ConstFlat input,
+                        typename TTypes<T>::ConstFlat input,
                         TTypes<int64>::Scalar num_true) {
-    *num_true.data() =
-        std::accumulate(input.data(), input.data() + input.size(), 0);
+    num_true() = CountAccumulator<T>(input.data(), input.data() + input.size());
     return Status::OK();
   }
 };
 
-template <int DIMS, typename TIndex>
-struct Where<CPUDevice, DIMS, TIndex> {
+template <int DIMS, typename T, typename TIndex>
+struct Where<CPUDevice, DIMS, T, TIndex> {
   EIGEN_ALWAYS_INLINE static void WriteIndexRowMajor(
       typename TTypes<int64>::Matrix output,
       const typename Eigen::DSizes<TIndex, DIMS>& strides, TIndex true_n,
@@ -77,7 +91,7 @@ struct Where<CPUDevice, DIMS, TIndex> {
 
   EIGEN_ALWAYS_INLINE static Status Compute(
       OpKernelContext* ctx, const CPUDevice& d,
-      typename TTypes<bool, DIMS>::ConstTensor input,
+      typename TTypes<T, DIMS>::ConstTensor input,
       typename TTypes<int64>::Matrix output, TIndex* found_true) {
     Eigen::DSizes<Eigen::DenseIndex, DIMS> dims = input.dimensions();
     Eigen::DSizes<TIndex, DIMS> strides;
@@ -93,7 +107,7 @@ struct Where<CPUDevice, DIMS, TIndex> {
 
     Eigen::DenseIndex output_size = output.dimension(0);
     for (Eigen::DenseIndex n = 0; n < input.size(); ++n) {
-      if (input.data()[n]) {
+      if (input.data()[n] != T(0)) {
         if (FastBoundsCheck(*found_true, output_size)) {
           WriteIndexRowMajor(output, strides, *found_true, n);
         }
@@ -106,6 +120,7 @@ struct Where<CPUDevice, DIMS, TIndex> {
 
 }  // namespace functor
 
+template <typename T>
 class WhereCPUOp : public OpKernel {
  public:
   explicit WhereCPUOp(OpKernelConstruction* context) : OpKernel(context) {}
@@ -113,6 +128,12 @@ class WhereCPUOp : public OpKernel {
   void Compute(OpKernelContext* context) override {
     const Tensor& input = context->input(0);
 
+    OP_REQUIRES(
+        context, input.dtype() != DT_HALF,
+        errors::Unimplemented("No WhereOp available for float16/half type on "
+                              "GPU; dying in CPU WhereOp to avoid silently "
+                              "creating costly copies from device."));
+
     const int input_dims = input.dims();
 
     Tensor num_true;
@@ -120,8 +141,8 @@ class WhereCPUOp : public OpKernel {
         context, context->allocate_temp(DT_INT64, TensorShape({}), &num_true));
     auto num_true_t = num_true.scalar<int64>();
 
-    Status s = functor::NumTrue<CPUDevice, int64>::Compute(
-        context, context->eigen_device<CPUDevice>(), input.flat<bool>(),
+    Status s = functor::NumTrue<CPUDevice, T, int64>::Compute(
+        context, context->eigen_device<CPUDevice>(), input.flat<T>(),
         num_true_t);
     OP_REQUIRES_OK(context, s);
     TensorShape output_shape({num_true_t(), input_dims});
@@ -134,12 +155,12 @@ class WhereCPUOp : public OpKernel {
     // separate threads below.
     int64 found_true = 0;
 
-#define HANDLE_DIM(NDIM)                                                   \
-  case NDIM: {                                                             \
-    Status s = functor::Where<CPUDevice, NDIM, int64>::Compute(            \
-        context, context->eigen_device<CPUDevice>(),                       \
-        input.tensor<bool, NDIM>(), output->matrix<int64>(), &found_true); \
-    OP_REQUIRES_OK(context, s);                                            \
+#define HANDLE_DIM(NDIM)                                                      \
+  case NDIM: {                                                                \
+    Status s = functor::Where<CPUDevice, NDIM, T, int64>::Compute(            \
+        context, context->eigen_device<CPUDevice>(), input.tensor<T, NDIM>(), \
+        output->matrix<int64>(), &found_true);                                \
+    OP_REQUIRES_OK(context, s);                                               \
   } break;
 
     switch (input_dims) {
@@ -169,44 +190,63 @@ class WhereCPUOp : public OpKernel {
   TF_DISALLOW_COPY_AND_ASSIGN(WhereCPUOp);
 };
 
-REGISTER_KERNEL_BUILDER(Name("Where").Device(DEVICE_CPU), WhereCPUOp);
+#define REGISTER_WHERE_OP(T) \
+  REGISTER_KERNEL_BUILDER(   \
+      Name("Where").Device(DEVICE_CPU).TypeConstraint<T>("T"), WhereCPUOp<T>);
+
+TF_CALL_NUMBER_TYPES(REGISTER_WHERE_OP);
+TF_CALL_bool(REGISTER_WHERE_OP);
+
+#undef REGISTER_WHERE_OP
 
 #if GOOGLE_CUDA
 
 namespace functor {
 
-#define DECLARE_GPU_NUMTRUE(Tindex)                                            \
-  template <>                                                                  \
-  Status NumTrue<GPUDevice, Tindex>::Compute(                                  \
-      OpKernelContext* ctx, const GPUDevice& d, TTypes<bool>::ConstFlat input, \
-      TTypes<Tindex>::Scalar num_true);                                        \
-  extern template struct NumTrue<GPUDevice, Tindex>
+#define DECLARE_GPU_NUMTRUE(T, Tindex)                                      \
+  template <>                                                               \
+  Status NumTrue<GPUDevice, T, Tindex>::Compute(                            \
+      OpKernelContext* ctx, const GPUDevice& d, TTypes<T>::ConstFlat input, \
+      TTypes<Tindex>::Scalar num_true);                                     \
+  extern template struct NumTrue<GPUDevice, T, Tindex>
 
-DECLARE_GPU_NUMTRUE(int32);
-DECLARE_GPU_NUMTRUE(int64);
+#define DECLARE_GPU_NUMTRUE_TYPE(T) \
+  DECLARE_GPU_NUMTRUE(T, int32);    \
+  DECLARE_GPU_NUMTRUE(T, int64);
+
+TF_CALL_NUMBER_TYPES(DECLARE_GPU_NUMTRUE_TYPE);
+TF_CALL_bool(DECLARE_GPU_NUMTRUE_TYPE);
+
+#undef DECLARE_GPU_NUMTRUE_TYPE
 #undef DECLARE_GPU_NUMTRUE
 
-#define DECLARE_GPU_WHERE_INDEX(Dims, Tindex)                     \
+#define DECLARE_GPU_WHERE_INDEX(Dims, T, Tindex)                  \
   template <>                                                     \
-  Status Where<GPUDevice, Dims, Tindex>::Compute(                 \
+  Status Where<GPUDevice, Dims, T, Tindex>::Compute(              \
       OpKernelContext* ctx, const GPUDevice& d,                   \
-      typename TTypes<bool, Dims>::ConstTensor input,             \
+      typename TTypes<T, Dims>::ConstTensor input,                \
       typename TTypes<int64>::Matrix output, Tindex* found_true); \
-  extern template struct Where<GPUDevice, Dims, Tindex>;
-#define DECLARE_GPU_WHERE(Dims)         \
-  DECLARE_GPU_WHERE_INDEX(Dims, int32); \
-  DECLARE_GPU_WHERE_INDEX(Dims, int64);
-
-DECLARE_GPU_WHERE(1);
-DECLARE_GPU_WHERE(2);
-DECLARE_GPU_WHERE(3);
-DECLARE_GPU_WHERE(4);
-DECLARE_GPU_WHERE(5);
+  extern template struct Where<GPUDevice, Dims, T, Tindex>;
+#define DECLARE_GPU_WHERE(Dims, T)         \
+  DECLARE_GPU_WHERE_INDEX(Dims, T, int32); \
+  DECLARE_GPU_WHERE_INDEX(Dims, T, int64);
+
+#define DECLARE_GPU_WHERE_TYPES(T) \
+  DECLARE_GPU_WHERE(1, T);         \
+  DECLARE_GPU_WHERE(2, T);         \
+  DECLARE_GPU_WHERE(3, T);         \
+  DECLARE_GPU_WHERE(4, T);         \
+  DECLARE_GPU_WHERE(5, T);
+
+TF_CALL_WHERE_GPU_TYPES(DECLARE_GPU_WHERE_TYPES);
+
+#undef DECLARE_GPU_WHERE_TYPES
 #undef DECLARE_GPU_WHERE
 #undef DECLARE_GPU_WHERE_INDEX
 
 }  // namespace functor
 
+template <typename T>
 class WhereGPUOp : public AsyncOpKernel {
  public:
   explicit WhereGPUOp(OpKernelConstruction* context) : AsyncOpKernel(context) {}
@@ -242,8 +282,8 @@ class WhereGPUOp : public AsyncOpKernel {
         static_cast<void*>(num_true_t.data()));
     // Push kernel to stream to get number of true elements.
     const GPUDevice& d = context->eigen_device<GPUDevice>();
-    Status s = functor::NumTrue<GPUDevice, Tindex>::Compute(
-        context, d, input.flat<bool>(), num_true_t);
+    Status s = functor::NumTrue<GPUDevice, T, Tindex>::Compute(
+        context, d, input.flat<T>(), num_true_t);
     OP_REQUIRES_OK_ASYNC(context, s, done);
 
     // Copy num_true to host;
@@ -279,12 +319,12 @@ class WhereGPUOp : public AsyncOpKernel {
                                0, TensorShape({num_true, input_dims}), &output),
                            done);
 
-#define HANDLE_DIM(NDIM)                                                 \
-  case NDIM: {                                                           \
-    Status s = functor::Where<GPUDevice, NDIM, Tindex>::Compute(         \
-        context, d, input.tensor<bool, NDIM>(), output->matrix<int64>(), \
-        &found_true);                                                    \
-    OP_REQUIRES_OK_ASYNC(context, s, done);                              \
+#define HANDLE_DIM(NDIM)                                              \
+  case NDIM: {                                                        \
+    Status s = functor::Where<GPUDevice, NDIM, T, Tindex>::Compute(   \
+        context, d, input.tensor<T, NDIM>(), output->matrix<int64>(), \
+        &found_true);                                                 \
+    OP_REQUIRES_OK_ASYNC(context, s, done);                           \
   } break;
 
       switch (input_dims) {
@@ -324,7 +364,13 @@ class WhereGPUOp : public AsyncOpKernel {
   TF_DISALLOW_COPY_AND_ASSIGN(WhereGPUOp);
 };
 
-REGISTER_KERNEL_BUILDER(Name("Where").Device(DEVICE_GPU), WhereGPUOp);
+#define REGISTER_GPU_WHERE_OP(T) \
+  REGISTER_KERNEL_BUILDER(       \
+      Name("Where").Device(DEVICE_GPU).TypeConstraint<T>("T"), WhereGPUOp<T>);
+
+TF_CALL_WHERE_GPU_TYPES(REGISTER_GPU_WHERE_OP);
+
+#undef REGISTER_GPU_WHERE_OP
 
 #endif  // GOOGLE_CUDA
 
diff --git a/tensorflow/core/kernels/where_op.h b/tensorflow/core/kernels/where_op.h
index e040325e3d..d26849c8bd 100644
--- a/tensorflow/core/kernels/where_op.h
+++ b/tensorflow/core/kernels/where_op.h
@@ -24,16 +24,28 @@ limitations under the License.
 
 namespace tensorflow {
 
+#define TF_CALL_WHERE_GPU_TYPES(m) \
+  TF_CALL_int8(m);                 \
+  TF_CALL_uint8(m);                \
+  TF_CALL_int32(m);                \
+  TF_CALL_int64(m);                \
+  TF_CALL_float(m);                \
+  TF_CALL_double(m);               \
+  TF_CALL_complex64(m);            \
+  TF_CALL_complex128(m);           \
+  TF_CALL_bool(m);
+
 namespace functor {
 
-template <typename Device, typename TIndex>
+template <typename Device, typename T, typename TIndex>
 struct NumTrue {
   EIGEN_ALWAYS_INLINE static Status Compute(
-      OpKernelContext* ctx, const Device& d, TTypes<bool>::ConstFlat input,
+      OpKernelContext* ctx, const Device& d,
+      typename TTypes<T>::ConstFlat input,
       typename TTypes<TIndex>::Scalar num_true);
 };
 
-template <typename Device, int NDIM, typename TIndex>
+template <typename Device, int NDIM, typename T, typename TIndex>
 struct Where {
   // Copies indices of true values in input into output.  The pointer
   // found_true should sit on the host.  Compute should copy the
@@ -43,7 +55,7 @@ struct Where {
   // the true values and the call to Where.
   EIGEN_ALWAYS_INLINE static Status Compute(
       OpKernelContext* ctx, const Device& d,
-      typename TTypes<bool, NDIM>::ConstTensor input,
+      typename TTypes<T, NDIM>::ConstTensor input,
       typename TTypes<int64>::Matrix output, TIndex* found_true);
 };
 
diff --git a/tensorflow/core/kernels/where_op_gpu.cu.cc b/tensorflow/core/kernels/where_op_gpu.cu.h
similarity index 53%
rename from tensorflow/core/kernels/where_op_gpu.cu.cc
rename to tensorflow/core/kernels/where_op_gpu.cu.h
index c7c54ccbb4..ce8e435c95 100644
--- a/tensorflow/core/kernels/where_op_gpu.cu.cc
+++ b/tensorflow/core/kernels/where_op_gpu.cu.h
@@ -21,6 +21,8 @@ limitations under the License.
 #include "external/cub_archive/cub/device/device_reduce.cuh"
 #include "external/cub_archive/cub/device/device_select.cuh"
 #include "external/cub_archive/cub/iterator/counting_input_iterator.cuh"
+#include "external/cub_archive/cub/iterator/transform_input_iterator.cuh"
+#include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/kernels/where_op.h"
@@ -51,23 +53,103 @@ __global__ void PropagateWhereIndicesKernel(
   }
 }
 
+namespace {
+
+template <typename T>
+struct IsNonzero {
+  EIGEN_DEVICE_FUNC IsNonzero() : zero(T(0)) {}
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool operator()(const T& x) const {
+    return (x != zero);
+  }
+  const T zero;
+};
+
+template <typename T, typename TIndex>
+struct CubDeviceReduceCount {
+  cudaError_t operator()(void* d_temp_storage, size_t& temp_storage_bytes,
+                         const T* d_in, TIndex* d_out, int num_items,
+                         cudaStream_t stream = 0,
+                         bool debug_synchronous = false) {
+    IsNonzero<T> is_nonzero;
+    cub::TransformInputIterator<bool, IsNonzero<T>, const T*> is_nonzero_iter(
+        d_in, is_nonzero);
+    return cub::DeviceReduce::Sum(d_temp_storage, temp_storage_bytes,
+                                  is_nonzero_iter, d_out, num_items, stream,
+                                  debug_synchronous);
+  }
+};
+
 template <typename TIndex>
-struct NumTrue<GPUDevice, TIndex> {
+struct CubDeviceReduceCount<bool, TIndex> {
+  cudaError_t operator()(void* d_temp_storage, size_t& temp_storage_bytes,
+                         const bool* d_in, TIndex* d_out, int num_items,
+                         cudaStream_t stream = 0,
+                         bool debug_synchronous = false) {
+    return cub::DeviceReduce::Sum(d_temp_storage, temp_storage_bytes, d_in,
+                                  d_out, num_items, stream, debug_synchronous);
+  }
+};
+
+template <typename T, typename TIndex, typename OutputIterator,
+          bool IsConvertibleToBool>
+struct CubDeviceSelectFlaggedCounter;
+
+template <typename T, typename TIndex, typename OutputIterator>
+struct CubDeviceSelectFlaggedCounter<T, TIndex, OutputIterator,
+                                     false /*IsConvertibleToBool*/> {
+  cudaError_t operator()(void* d_temp_storage, size_t& temp_storage_bytes,
+                         const T* d_flags, OutputIterator d_out,
+                         TIndex* d_num_selected_out, int num_items,
+                         cudaStream_t stream = 0,
+                         bool debug_synchronous = false) {
+    cub::CountingInputIterator<TIndex> select_counter(0);
+    IsNonzero<T> is_nonzero;
+    cub::TransformInputIterator<bool, IsNonzero<T>, const T*> is_nonzero_iter(
+        d_flags, is_nonzero);
+    return cub::DeviceSelect::Flagged(
+        d_temp_storage, temp_storage_bytes, select_counter /*d_in*/,
+        is_nonzero_iter /*d_flags*/, d_out, d_num_selected_out, num_items,
+        stream, debug_synchronous);
+  }
+};
+
+template <typename T, typename TIndex, typename OutputIterator>
+struct CubDeviceSelectFlaggedCounter<T, TIndex, OutputIterator,
+                                     true /*IsConvertibleToBool*/> {
+  cudaError_t operator()(void* d_temp_storage, size_t& temp_storage_bytes,
+                         const T* d_flags, OutputIterator d_out,
+                         TIndex* d_num_selected_out, int num_items,
+                         cudaStream_t stream = 0,
+                         bool debug_synchronous = false) {
+    cub::CountingInputIterator<TIndex> select_counter(0);
+    return cub::DeviceSelect::Flagged(
+        d_temp_storage, temp_storage_bytes, select_counter /*d_in*/, d_flags,
+        d_out, d_num_selected_out, num_items, stream, debug_synchronous);
+  }
+};
+
+}  // namespace
+
+template <typename T, typename TIndex>
+struct NumTrue<GPUDevice, T, TIndex> {
   EIGEN_ALWAYS_INLINE static Status Compute(
-      OpKernelContext* ctx, const GPUDevice& d, TTypes<bool>::ConstFlat input,
+      OpKernelContext* ctx, const GPUDevice& d,
+      typename TTypes<T>::ConstFlat input,
       typename TTypes<TIndex>::Scalar num_true) {
     const cudaStream_t& cu_stream = GetCudaStream(ctx);
 
     std::size_t temp_storage_bytes = 0;
-    const bool* input_data = input.data();
+    const T* input_data = input.data();
     TIndex* num_true_data = num_true.data();
 
-    auto first_success =
-        cub::DeviceReduce::Sum(/*temp_storage*/ nullptr, temp_storage_bytes,
-                               /*d_in*/ input_data,
-                               /*d_out*/ num_true_data,
-                               /*num_items*/ input.size(),
-                               /*stream*/ cu_stream);
+    // TODO(ebrevdo): sum doesn't work; perhaps need a different
+    // iterator?
+    auto reducer = CubDeviceReduceCount<T, TIndex>();
+    auto first_success = reducer(/*temp_storage*/ nullptr, temp_storage_bytes,
+                                 /*d_in*/ input_data,
+                                 /*d_out*/ num_true_data,
+                                 /*num_items*/ input.size(),
+                                 /*stream*/ cu_stream);
 
     if (first_success != cudaSuccess) {
       return errors::Internal(
@@ -81,7 +163,7 @@ struct NumTrue<GPUDevice, TIndex> {
         DT_INT8, TensorShape({static_cast<int64>(temp_storage_bytes)}),
         &temp_storage));
 
-    auto second_success = cub::DeviceReduce::Sum(
+    auto second_success = reducer(
         /*temp_storage*/ temp_storage.flat<int8>().data(), temp_storage_bytes,
         /*d_in*/ input_data,
         /*d_out*/ num_true_data,
@@ -91,7 +173,7 @@ struct NumTrue<GPUDevice, TIndex> {
     if (second_success != cudaSuccess) {
       return errors::Internal(
           "WhereOp: Could not launch cub::DeviceReduce::Sum to count "
-          "number of true indices.  temp_storage_bytes: ",
+          "number of true / nonzero indices.  temp_storage_bytes: ",
           temp_storage_bytes, ", status: ", cudaGetErrorString(second_success));
     }
 
@@ -99,8 +181,20 @@ struct NumTrue<GPUDevice, TIndex> {
   }
 };
 
-template struct NumTrue<GPUDevice, int32>;
-template struct NumTrue<GPUDevice, int64>;
+#define NUMTRUE_GPU_FUNCTOR(T)                  \
+  template struct NumTrue<GPUDevice, T, int32>; \
+  template struct NumTrue<GPUDevice, T, int64>;
+
+// We only need to declare the NumTrue functor once, but this file is
+// included from where_op_gpu_impl_X.cu.cc for X=1,2,...
+// Only declare for X = 1.
+#if GPU_PROVIDED_DIM == 1
+
+TF_CALL_WHERE_GPU_TYPES(NUMTRUE_GPU_FUNCTOR);
+
+#endif  // GPU_PROVIDED_DIM == 1
+
+#undef NUMTRUE_GPU_FUNCTOR
 
 template <int NDIM>
 class WhereOutputIterator {
@@ -143,9 +237,9 @@ class WhereOutputIterator {
   const Eigen::DenseIndex max_row_;
 };
 
-template <typename TIndex, int NDIM>
+template <typename TIndex, typename T, int NDIM>
 Eigen::array<TIndex, NDIM> CalculateStrides(
-    typename TTypes<bool, NDIM>::ConstTensor input) {
+    typename TTypes<T, NDIM>::ConstTensor input) {
   const Eigen::DSizes<Eigen::DenseIndex, NDIM> dims = input.dimensions();
   Eigen::array<TIndex, NDIM> strides;
   EIGEN_STATIC_ASSERT((static_cast<int>(decltype(input)::Layout) ==
@@ -158,12 +252,12 @@ Eigen::array<TIndex, NDIM> CalculateStrides(
   return strides;
 }
 
-template <int NDIM, typename Tindex>
-struct Where<GPUDevice, NDIM, Tindex> {
+template <int NDIM, typename T, typename TIndex>
+struct Where<GPUDevice, NDIM, T, TIndex> {
   EIGEN_ALWAYS_INLINE static Status Compute(
       OpKernelContext* ctx, const GPUDevice& d,
-      typename TTypes<bool, NDIM>::ConstTensor input,
-      typename TTypes<int64>::Matrix output, Tindex* found_true_host) {
+      typename TTypes<T, NDIM>::ConstTensor input,
+      typename TTypes<int64>::Matrix output, TIndex* found_true_host) {
     if (output.dimension(0) == 0) {
       // Nothing to do.
       return Status::OK();
@@ -173,25 +267,26 @@ struct Where<GPUDevice, NDIM, Tindex> {
 
     std::size_t temp_storage_bytes = 0;
 
-    cub::CountingInputIterator<Tindex> select_counter(0);
-
     Tensor found_true_t;
-    TF_RETURN_IF_ERROR(ctx->allocate_temp(DataTypeToEnum<Tindex>::v(),
+    TF_RETURN_IF_ERROR(ctx->allocate_temp(DataTypeToEnum<TIndex>::v(),
                                           TensorShape({}), &found_true_t));
-    Tindex* found_true_device = found_true_t.scalar<Tindex>().data();
+    TIndex* found_true_device = found_true_t.scalar<TIndex>().data();
 
     WhereOutputIterator<NDIM> output_iterator(
         output.data(),
         /* max_row */ output.dimension(0));
 
-    auto first_success =
-        cub::DeviceSelect::Flagged(/*temp_storage*/ nullptr, temp_storage_bytes,
-                                   /*d_in*/ select_counter,
-                                   /*d_flags*/ input.data(),
-                                   /*d_out*/ output_iterator,
-                                   /*d_num_selected_out*/ found_true_device,
-                                   /*num_items*/ input.size(),
-                                   /*stream*/ cu_stream);
+    typedef std::decay<T> DT;
+    CubDeviceSelectFlaggedCounter<
+        T, TIndex, typeof(output_iterator) /*OutputIterator*/,
+        std::is_convertible<DT, bool>::value /*IsConvertibleToBool*/>
+        counter;
+    auto first_success = counter(/*temp_storage*/ nullptr, temp_storage_bytes,
+                                 /*d_flags*/ input.data(),
+                                 /*d_out*/ output_iterator,
+                                 /*d_num_selected_out*/ found_true_device,
+                                 /*num_items*/ input.size(),
+                                 /*stream*/ cu_stream);
     if (first_success != cudaSuccess) {
       return errors::Internal(
           "WhereOp: Could not launch cub::DeviceSelect::Flagged to calculate "
@@ -204,9 +299,8 @@ struct Where<GPUDevice, NDIM, Tindex> {
         DT_INT8, TensorShape({static_cast<int64>(temp_storage_bytes)}),
         &temp_storage));
 
-    auto second_success = cub::DeviceSelect::Flagged(
+    auto second_success = counter(
         /*temp_storage*/ temp_storage.flat<int8>().data(), temp_storage_bytes,
-        /*d_in*/ select_counter,
         /*d_flags*/ input.data(),
         /*d_out*/ output_iterator,
         /*d_num_selected_out*/ found_true_device,
@@ -223,11 +317,11 @@ struct Where<GPUDevice, NDIM, Tindex> {
     // TODO(ebrevdo): Find a way to synchronously copy back data from
     // found_true_device to *found_true_host.
 
-    const Eigen::array<Tindex, NDIM> strides =
-        CalculateStrides<Tindex, NDIM>(input);
-    const Tindex output_rows = output.dimension(0);
+    const Eigen::array<TIndex, NDIM> strides =
+        CalculateStrides<TIndex, T, NDIM>(input);
+    const TIndex output_rows = output.dimension(0);
     CudaLaunchConfig config = GetCudaLaunchConfig(output_rows, d);
-    PropagateWhereIndicesKernel<NDIM, Tindex>
+    PropagateWhereIndicesKernel<NDIM, TIndex>
         <<<config.block_count, config.thread_per_block, 0, d.stream()>>>(
             output_rows, strides, output.data());
 
@@ -235,17 +329,14 @@ struct Where<GPUDevice, NDIM, Tindex> {
   }
 };
 
-#define DECLARE_GPU_SPEC_INDEX(Dims, Tindex) \
-  template struct Where<GPUDevice, Dims, Tindex>
-#define DECLARE_GPU_SPEC(Dims)         \
-  DECLARE_GPU_SPEC_INDEX(Dims, int32); \
-  DECLARE_GPU_SPEC_INDEX(Dims, int64)
+#define DECLARE_GPU_SPEC_INDEX(Dims, T, TIndex) \
+  template struct Where<GPUDevice, Dims, T, TIndex>
+
+#define DECLARE_GPU_SPEC(T)                           \
+  DECLARE_GPU_SPEC_INDEX(GPU_PROVIDED_DIM, T, int32); \
+  DECLARE_GPU_SPEC_INDEX(GPU_PROVIDED_DIM, T, int64)
 
-DECLARE_GPU_SPEC(1);
-DECLARE_GPU_SPEC(2);
-DECLARE_GPU_SPEC(3);
-DECLARE_GPU_SPEC(4);
-DECLARE_GPU_SPEC(5);
+TF_CALL_WHERE_GPU_TYPES(DECLARE_GPU_SPEC);
 
 #undef DECLARE_GPU_SPEC
 #undef DECLARE_GPU_SPEC_INDEX
@@ -253,4 +344,5 @@ DECLARE_GPU_SPEC(5);
 }  // namespace functor
 
 }  // namespace tensorflow
+
 #endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/where_op_gpu_impl_1.cu.cc b/tensorflow/core/kernels/where_op_gpu_impl_1.cu.cc
new file mode 100644
index 0000000000..75ddfa76ea
--- /dev/null
+++ b/tensorflow/core/kernels/where_op_gpu_impl_1.cu.cc
@@ -0,0 +1,18 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#define GPU_PROVIDED_DIM 1
+#include "tensorflow/core/kernels/where_op_gpu.cu.h"
+#undef GPU_PROVIDED_DIM
diff --git a/tensorflow/core/kernels/where_op_gpu_impl_2.cu.cc b/tensorflow/core/kernels/where_op_gpu_impl_2.cu.cc
new file mode 100644
index 0000000000..3a62259608
--- /dev/null
+++ b/tensorflow/core/kernels/where_op_gpu_impl_2.cu.cc
@@ -0,0 +1,18 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#define GPU_PROVIDED_DIM 2
+#include "tensorflow/core/kernels/where_op_gpu.cu.h"
+#undef GPU_PROVIDED_DIM
diff --git a/tensorflow/core/kernels/where_op_gpu_impl_3.cu.cc b/tensorflow/core/kernels/where_op_gpu_impl_3.cu.cc
new file mode 100644
index 0000000000..2ae5447175
--- /dev/null
+++ b/tensorflow/core/kernels/where_op_gpu_impl_3.cu.cc
@@ -0,0 +1,18 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#define GPU_PROVIDED_DIM 3
+#include "tensorflow/core/kernels/where_op_gpu.cu.h"
+#undef GPU_PROVIDED_DIM
diff --git a/tensorflow/core/kernels/where_op_gpu_impl_4.cu.cc b/tensorflow/core/kernels/where_op_gpu_impl_4.cu.cc
new file mode 100644
index 0000000000..e976bb4331
--- /dev/null
+++ b/tensorflow/core/kernels/where_op_gpu_impl_4.cu.cc
@@ -0,0 +1,18 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#define GPU_PROVIDED_DIM 4
+#include "tensorflow/core/kernels/where_op_gpu.cu.h"
+#undef GPU_PROVIDED_DIM
diff --git a/tensorflow/core/kernels/where_op_gpu_impl_5.cu.cc b/tensorflow/core/kernels/where_op_gpu_impl_5.cu.cc
new file mode 100644
index 0000000000..ccbe2d6499
--- /dev/null
+++ b/tensorflow/core/kernels/where_op_gpu_impl_5.cu.cc
@@ -0,0 +1,18 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#define GPU_PROVIDED_DIM 5
+#include "tensorflow/core/kernels/where_op_gpu.cu.h"
+#undef GPU_PROVIDED_DIM
diff --git a/tensorflow/core/ops/array_ops.cc b/tensorflow/core/ops/array_ops.cc
index ad111fc6b8..fec27c7c1c 100644
--- a/tensorflow/core/ops/array_ops.cc
+++ b/tensorflow/core/ops/array_ops.cc
@@ -2715,14 +2715,15 @@ each repeated tile of `input` into `output`.
 
 // --------------------------------------------------------------------------
 REGISTER_OP("Where")
-    .Input("input: bool")
+    .Input("input: T")
+    .Attr("T: {numbertype, bool} = DT_BOOL")
     .Output("index: int64")
     .SetShapeFn([](InferenceContext* c) {
       c->set_output(0, c->Matrix(c->UnknownDim(), c->Rank(c->input(0))));
       return Status::OK();
     })
     .Doc(R"doc(
-Returns locations of true values in a boolean tensor.
+Returns locations of nonzero / true values in a tensor.
 
 This operation returns the coordinates of true elements in `input`. The
 coordinates are returned in a 2-D tensor where the first dimension (rows)
@@ -2749,6 +2750,34 @@ where(input) ==> [[0, 0],
 #                     [False, True]]]
 # 'input' has 5 true values, so output has 5 coordinates.
 # 'input' has rank of 3, so coordinates have three indices.
+where(input) ==> [[0, 0, 0],
+                  [0, 1, 0],
+                  [1, 0, 1],
+                  [1, 1, 1],
+                  [2, 1, 1]]
+
+# `input` tensor is [[[1.5,  0.0]
+#                     [-0.5, 0.0]]
+#                    [[0.0,  0.25]
+#                     [0.0,  0.75]]
+#                    [[0.0,  0.0]
+#                     [0.0,  0.01]]]
+# 'input' has 5 nonzero values, so output has 5 coordinates.
+# 'input' has rank of 3, so coordinates have three indices.
+where(input) ==> [[0, 0, 0],
+                  [0, 1, 0],
+                  [1, 0, 1],
+                  [1, 1, 1],
+                  [2, 1, 1]]
+
+# `input` tensor is [[[1.5 + 0.0j, 0.0  + 0.0j]
+#                     [0.0 + 0.5j, 0.0  + 0.0j]]
+#                    [[0.0 + 0.0j, 0.25 + 1.5j]
+#                     [0.0 + 0.0j, 0.75 + 0.0j]]
+#                    [[0.0 + 0.0j, 0.0  + 0.0j]
+#                     [0.0 + 0.0j, 0.01 + 0.0j]]]
+# 'input' has 5 nonzero magnitude values, so output has 5 coordinates.
+# 'input' has rank of 3, so coordinates have three indices.
 where(input) ==> [[0, 0, 0],
                   [0, 1, 0],
                   [1, 0, 1],
diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD
index 6f618217f5..206c6a5692 100644
--- a/tensorflow/python/kernel_tests/BUILD
+++ b/tensorflow/python/kernel_tests/BUILD
@@ -971,7 +971,7 @@ tf_py_test(
 
 cuda_py_test(
     name = "where_op_test",
-    size = "small",
+    size = "medium",
     srcs = ["where_op_test.py"],
     additional_deps = [
         "//third_party/py/numpy",
diff --git a/tensorflow/python/kernel_tests/where_op_test.py b/tensorflow/python/kernel_tests/where_op_test.py
index 3e1fa0a287..17575da6f1 100644
--- a/tensorflow/python/kernel_tests/where_op_test.py
+++ b/tensorflow/python/kernel_tests/where_op_test.py
@@ -90,6 +90,44 @@ class WhereOpTest(test.TestCase):
 
     self._testWhere(x, truth)
 
+  def _testRandom(self, dtype, expected_err_re=None):
+    shape = [127, 33, 53]
+    x = np.random.randn(*shape) + 1j * np.random.randn(*shape)
+    x = (np.random.randn(*shape) > 0).astype(dtype)
+    truth = np.where(np.abs(x) > 0)  # Tuples of indices by axis.
+    truth = np.vstack(truth).T  # Convert to [num_true, indices].
+    self._testWhere(x, truth, expected_err_re)
+
+  def testRandomBool(self):
+    self._testRandom(np.bool)
+
+  def testRandomInt32(self):
+    self._testRandom(np.int32)
+
+  def testRandomInt64(self):
+    self._testRandom(np.int64)
+
+  def testRandomFloat(self):
+    self._testRandom(np.float32)
+
+  def testRandomDouble(self):
+    self._testRandom(np.float64)
+
+  def testRandomComplex64(self):
+    self._testRandom(np.complex64)
+
+  def testRandomComplex128(self):
+    self._testRandom(np.complex128)
+
+  def testRandomUint8(self):
+    self._testRandom(np.uint8)
+
+  def testRandomInt8(self):
+    self._testRandom(np.int8)
+
+  def testRandomInt16(self):
+    self._testRandom(np.int16)
+
   def testThreeArgument(self):
     x = np.array([[-2, 3, -1], [1, -3, -3]])
     np_val = np.where(x > 0, x * x, -x)
diff --git a/tensorflow/python/ops/array_ops.py b/tensorflow/python/ops/array_ops.py
index 5065217f33..3e0cfba90d 100644
--- a/tensorflow/python/ops/array_ops.py
+++ b/tensorflow/python/ops/array_ops.py
@@ -2436,7 +2436,9 @@ def where(condition, x=None, y=None, name=None):
     ValueError: When exactly one of `x` or `y` is non-None.
   """
   if x is None and y is None:
-    return gen_array_ops.where(input=condition, name=name)
+    with ops.name_scope(name, "Where", [condition]) as name:
+      condition = ops.convert_to_tensor(condition, dtype=dtypes.bool)
+      return gen_array_ops.where(input=condition, name=name)
   elif x is not None and y is not None:
     return gen_math_ops._select(condition=condition, t=x, e=y, name=name)
   else:
-- 
GitLab


From 697262d4ff781fdfb8f70226514d127adad74112 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 10 Oct 2017 10:48:41 -0700
Subject: [PATCH 226/909] Update ops-related pbtxt files.

PiperOrigin-RevId: 171700908
---
 .../core/ops/compat/ops_history.v1.pbtxt      | 39 +++++++++++++++++++
 tensorflow/core/ops/ops.pbtxt                 | 34 ++++++++++++++--
 2 files changed, 70 insertions(+), 3 deletions(-)

diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index 1eafbe138c..2097c587d5 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -40539,6 +40539,45 @@ op {
     type: DT_INT64
   }
 }
+op {
+  name: "Where"
+  input_arg {
+    name: "input"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "index"
+    type: DT_INT64
+  }
+  attr {
+    name: "T"
+    type: "type"
+    default_value {
+      type: DT_BOOL
+    }
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+        type: DT_BOOL
+      }
+    }
+  }
+}
 op {
   name: "WholeFileReader"
   output_arg {
diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index 53d99178e5..fc22594ea4 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -32447,14 +32447,42 @@ op {
   name: "Where"
   input_arg {
     name: "input"
-    type: DT_BOOL
+    type_attr: "T"
   }
   output_arg {
     name: "index"
     type: DT_INT64
   }
-  summary: "Returns locations of true values in a boolean tensor."
-  description: "This operation returns the coordinates of true elements in `input`. The\ncoordinates are returned in a 2-D tensor where the first dimension (rows)\nrepresents the number of true elements, and the second dimension (columns)\nrepresents the coordinates of the true elements. Keep in mind, the shape of\nthe output tensor can vary depending on how many true values there are in\n`input`. Indices are output in row-major order.\n\nFor example:\n\n```\n# \'input\' tensor is [[True, False]\n#                    [True, False]]\n# \'input\' has two true values, so output has two coordinates.\n# \'input\' has rank of 2, so coordinates have two indices.\nwhere(input) ==> [[0, 0],\n                  [1, 0]]\n\n# `input` tensor is [[[True, False]\n#                     [True, False]]\n#                    [[False, True]\n#                     [False, True]]\n#                    [[False, False]\n#                     [False, True]]]\n# \'input\' has 5 true values, so output has 5 coordinates.\n# \'input\' has rank of 3, so coordinates have three indices.\nwhere(input) ==> [[0, 0, 0],\n                  [0, 1, 0],\n                  [1, 0, 1],\n                  [1, 1, 1],\n                  [2, 1, 1]]\n```"
+  attr {
+    name: "T"
+    type: "type"
+    default_value {
+      type: DT_BOOL
+    }
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+        type: DT_BOOL
+      }
+    }
+  }
+  summary: "Returns locations of nonzero / true values in a tensor."
+  description: "This operation returns the coordinates of true elements in `input`. The\ncoordinates are returned in a 2-D tensor where the first dimension (rows)\nrepresents the number of true elements, and the second dimension (columns)\nrepresents the coordinates of the true elements. Keep in mind, the shape of\nthe output tensor can vary depending on how many true values there are in\n`input`. Indices are output in row-major order.\n\nFor example:\n\n```\n# \'input\' tensor is [[True, False]\n#                    [True, False]]\n# \'input\' has two true values, so output has two coordinates.\n# \'input\' has rank of 2, so coordinates have two indices.\nwhere(input) ==> [[0, 0],\n                  [1, 0]]\n\n# `input` tensor is [[[True, False]\n#                     [True, False]]\n#                    [[False, True]\n#                     [False, True]]\n#                    [[False, False]\n#                     [False, True]]]\n# \'input\' has 5 true values, so output has 5 coordinates.\n# \'input\' has rank of 3, so coordinates have three indices.\nwhere(input) ==> [[0, 0, 0],\n                  [0, 1, 0],\n                  [1, 0, 1],\n                  [1, 1, 1],\n                  [2, 1, 1]]\n\n# `input` tensor is [[[1.5,  0.0]\n#                     [-0.5, 0.0]]\n#                    [[0.0,  0.25]\n#                     [0.0,  0.75]]\n#                    [[0.0,  0.0]\n#                     [0.0,  0.01]]]\n# \'input\' has 5 nonzero values, so output has 5 coordinates.\n# \'input\' has rank of 3, so coordinates have three indices.\nwhere(input) ==> [[0, 0, 0],\n                  [0, 1, 0],\n                  [1, 0, 1],\n                  [1, 1, 1],\n                  [2, 1, 1]]\n\n# `input` tensor is [[[1.5 + 0.0j, 0.0  + 0.0j]\n#                     [0.0 + 0.5j, 0.0  + 0.0j]]\n#                    [[0.0 + 0.0j, 0.25 + 1.5j]\n#                     [0.0 + 0.0j, 0.75 + 0.0j]]\n#                    [[0.0 + 0.0j, 0.0  + 0.0j]\n#                     [0.0 + 0.0j, 0.01 + 0.0j]]]\n# \'input\' has 5 nonzero magnitude values, so output has 5 coordinates.\n# \'input\' has rank of 3, so coordinates have three indices.\nwhere(input) ==> [[0, 0, 0],\n                  [0, 1, 0],\n                  [1, 0, 1],\n                  [1, 1, 1],\n                  [2, 1, 1]]\n```"
 }
 op {
   name: "WholeFileReader"
-- 
GitLab


From 1fe440b368a19d0cf003bb7e4056a93937c57ada Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 10 Oct 2017 10:55:14 -0700
Subject: [PATCH 227/909] Go: Update generated wrapper functions for TensorFlow
 ops.

PiperOrigin-RevId: 171701981
---
 tensorflow/go/op/wrappers.go | 30 +++++++++++++++++++++++++++++-
 1 file changed, 29 insertions(+), 1 deletion(-)

diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go
index 804275dda6..9417de3932 100644
--- a/tensorflow/go/op/wrappers.go
+++ b/tensorflow/go/op/wrappers.go
@@ -1262,7 +1262,7 @@ func BroadcastArgs(scope *Scope, s0 tf.Output, s1 tf.Output) (r0 tf.Output) {
 	return op.Output(0)
 }
 
-// Returns locations of true values in a boolean tensor.
+// Returns locations of nonzero / true values in a tensor.
 //
 // This operation returns the coordinates of true elements in `input`. The
 // coordinates are returned in a 2-D tensor where the first dimension (rows)
@@ -1294,6 +1294,34 @@ func BroadcastArgs(scope *Scope, s0 tf.Output, s1 tf.Output) (r0 tf.Output) {
 //                   [1, 0, 1],
 //                   [1, 1, 1],
 //                   [2, 1, 1]]
+//
+// # `input` tensor is [[[1.5,  0.0]
+// #                     [-0.5, 0.0]]
+// #                    [[0.0,  0.25]
+// #                     [0.0,  0.75]]
+// #                    [[0.0,  0.0]
+// #                     [0.0,  0.01]]]
+// # 'input' has 5 nonzero values, so output has 5 coordinates.
+// # 'input' has rank of 3, so coordinates have three indices.
+// where(input) ==> [[0, 0, 0],
+//                   [0, 1, 0],
+//                   [1, 0, 1],
+//                   [1, 1, 1],
+//                   [2, 1, 1]]
+//
+// # `input` tensor is [[[1.5 + 0.0j, 0.0  + 0.0j]
+// #                     [0.0 + 0.5j, 0.0  + 0.0j]]
+// #                    [[0.0 + 0.0j, 0.25 + 1.5j]
+// #                     [0.0 + 0.0j, 0.75 + 0.0j]]
+// #                    [[0.0 + 0.0j, 0.0  + 0.0j]
+// #                     [0.0 + 0.0j, 0.01 + 0.0j]]]
+// # 'input' has 5 nonzero magnitude values, so output has 5 coordinates.
+// # 'input' has rank of 3, so coordinates have three indices.
+// where(input) ==> [[0, 0, 0],
+//                   [0, 1, 0],
+//                   [1, 0, 1],
+//                   [1, 1, 1],
+//                   [2, 1, 1]]
 // ```
 func Where(scope *Scope, input tf.Output) (index tf.Output) {
 	if scope.Err() != nil {
-- 
GitLab


From 46f0650df68214a3544ec00c1473a7ab14a0f99f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 10 Oct 2017 11:00:30 -0700
Subject: [PATCH 228/909] `name_scope('')` -> `name_scope(None)`.

PiperOrigin-RevId: 171702882
---
 .../contrib/gan/python/estimator/python/gan_estimator_impl.py   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/gan/python/estimator/python/gan_estimator_impl.py b/tensorflow/contrib/gan/python/estimator/python/gan_estimator_impl.py
index 6e1ee730aa..e89993991a 100644
--- a/tensorflow/contrib/gan/python/estimator/python/gan_estimator_impl.py
+++ b/tensorflow/contrib/gan/python/estimator/python/gan_estimator_impl.py
@@ -238,7 +238,7 @@ def _make_train_gan_model(generator_fn, discriminator_fn, real_data,
   if add_summaries:
     if not isinstance(add_summaries, (tuple, list)):
       add_summaries = [add_summaries]
-    with ops.name_scope(''):
+    with ops.name_scope(None):
       for summary_type in add_summaries:
         _summary_type_map[summary_type](gan_model)
 
-- 
GitLab


From 90121d582dbad4bd13dd2a9750c3a908e89469dd Mon Sep 17 00:00:00 2001
From: Dan Ringwalt <dringw@gmail.com>
Date: Tue, 10 Oct 2017 14:11:03 -0400
Subject: [PATCH 229/909] Add a tf.contrib.image.translate function (#12306)

* Add a tf.contrib.image.translate function

* Remove redundant checks from tf.contrib.image.translate.

* Add translate and translations_to_projective_transforms to the docstring.

* Fix lint errors for tf.contrib.image.translate

* Add name_scopes in image_ops.

Indicate in the docstrings when the static shape of the arguments must
have a known rank.

* Fix pyformat's weird docstring indentation.

* tf.name_scope -> ops.name_scope

* Move the test session inside the _DTYPES loop.

* Use the default_name arg of name_scope.

* Check for ndims == None

* Fix translate docstring and add a comment.

* s/vector/matrix/ for the multiple translations.
---
 tensorflow/contrib/image/__init__.py          |   4 +
 .../python/kernel_tests/image_ops_test.py     |  33 +-
 .../contrib/image/python/ops/image_ops.py     | 294 ++++++++++++------
 3 files changed, 224 insertions(+), 107 deletions(-)

diff --git a/tensorflow/contrib/image/__init__.py b/tensorflow/contrib/image/__init__.py
index 59a322d3ca..d030dffade 100755
--- a/tensorflow/contrib/image/__init__.py
+++ b/tensorflow/contrib/image/__init__.py
@@ -26,6 +26,8 @@ projective transforms (including rotation) are supported.
 @@random_yiq_hsv
 @@rotate
 @@transform
+@@translate
+@@translations_to_projective_transforms
 @@bipartite_match
 @@single_image_random_dot_stereograms
 """
@@ -41,6 +43,8 @@ from tensorflow.contrib.image.python.ops.image_ops import angles_to_projective_t
 from tensorflow.contrib.image.python.ops.image_ops import compose_transforms
 from tensorflow.contrib.image.python.ops.image_ops import rotate
 from tensorflow.contrib.image.python.ops.image_ops import transform
+from tensorflow.contrib.image.python.ops.image_ops import translate
+from tensorflow.contrib.image.python.ops.image_ops import translations_to_projective_transforms
 from tensorflow.contrib.image.python.ops.single_image_random_dot_stereograms import single_image_random_dot_stereograms
 
 from tensorflow.python.util.all_util import remove_undocumented
diff --git a/tensorflow/contrib/image/python/kernel_tests/image_ops_test.py b/tensorflow/contrib/image/python/kernel_tests/image_ops_test.py
index b8a0706b61..b50177ae56 100644
--- a/tensorflow/contrib/image/python/kernel_tests/image_ops_test.py
+++ b/tensorflow/contrib/image/python/kernel_tests/image_ops_test.py
@@ -36,8 +36,8 @@ _DTYPES = set(
 class ImageOpsTest(test_util.TensorFlowTestCase):
 
   def test_zeros(self):
-    with self.test_session():
-      for dtype in _DTYPES:
+    for dtype in _DTYPES:
+      with self.test_session():
         for shape in [(5, 5), (24, 24), (2, 24, 24, 3)]:
           for angle in [0, 1, np.pi / 2.0]:
             image = array_ops.zeros(shape, dtype)
@@ -46,8 +46,8 @@ class ImageOpsTest(test_util.TensorFlowTestCase):
                 np.zeros(shape, dtype.as_numpy_dtype()))
 
   def test_rotate_even(self):
-    with self.test_session():
-      for dtype in _DTYPES:
+    for dtype in _DTYPES:
+      with self.test_session():
         image = array_ops.reshape(
             math_ops.cast(math_ops.range(36), dtype), (6, 6))
         image_rep = array_ops.tile(image[None, :, :, None], [3, 1, 1, 1])
@@ -68,8 +68,8 @@ class ImageOpsTest(test_util.TensorFlowTestCase):
                               [1, 7, 13, 19, 25, 31], [0, 6, 12, 18, 24, 30]]])
 
   def test_rotate_odd(self):
-    with self.test_session():
-      for dtype in _DTYPES:
+    for dtype in _DTYPES:
+      with self.test_session():
         image = array_ops.reshape(
             math_ops.cast(math_ops.range(25), dtype), (5, 5))
         image_rep = array_ops.tile(image[None, :, :, None], [3, 1, 1, 1])
@@ -87,9 +87,25 @@ class ImageOpsTest(test_util.TensorFlowTestCase):
                               [22, 17, 12, 7, 2], [23, 18, 13, 8, 3],
                               [24, 19, 14, 9, 4]]])
 
+  def test_translate(self):
+    for dtype in _DTYPES:
+      with self.test_session():
+        image = constant_op.constant(
+            [[1, 0, 1, 0],
+             [0, 1, 0, 1],
+             [1, 0, 1, 0],
+             [0, 1, 0, 1]], dtype=dtype)
+        translation = constant_op.constant([-1, -1], dtypes.float32)
+        image_translated = image_ops.translate(image, translation)
+        self.assertAllEqual(image_translated.eval(),
+                            [[1, 0, 1, 0],
+                             [0, 1, 0, 0],
+                             [1, 0, 1, 0],
+                             [0, 0, 0, 0]])
+
   def test_compose(self):
-    with self.test_session():
-      for dtype in _DTYPES:
+    for dtype in _DTYPES:
+      with self.test_session():
         image = constant_op.constant(
             [[1, 1, 1, 0],
              [1, 0, 0, 0],
@@ -246,4 +262,3 @@ class BipartiteMatchTest(test_util.TensorFlowTestCase):
 
 if __name__ == "__main__":
   googletest.main()
-
diff --git a/tensorflow/contrib/image/python/ops/image_ops.py b/tensorflow/contrib/image/python/ops/image_ops.py
index aef3e385b5..011ddeaa9a 100644
--- a/tensorflow/contrib/image/python/ops/image_ops.py
+++ b/tensorflow/contrib/image/python/ops/image_ops.py
@@ -37,16 +37,18 @@ _IMAGE_DTYPES = set(
 ops.RegisterShape("ImageProjectiveTransform")(common_shapes.call_cpp_shape_fn)
 
 
-def rotate(images, angles, interpolation="NEAREST"):
+def rotate(images, angles, interpolation="NEAREST", name=None):
   """Rotate image(s) by the passed angle(s) in radians.
 
   Args:
     images: A tensor of shape (num_images, num_rows, num_columns, num_channels)
        (NHWC), (num_rows, num_columns, num_channels) (HWC), or
-       (num_rows, num_columns) (HW).
+       (num_rows, num_columns) (HW). The rank must be statically known (the
+       shape is not `TensorShape(None)`.
     angles: A scalar angle to rotate all images by, or (if images has rank 4)
        a vector of length num_images, with an angle for each image in the batch.
     interpolation: Interpolation mode. Supported values: "NEAREST", "BILINEAR".
+    name: The name of the op.
 
   Returns:
     Image(s) with the same type and shape as `images`, rotated by the given
@@ -55,38 +57,77 @@ def rotate(images, angles, interpolation="NEAREST"):
   Raises:
     TypeError: If `image` is an invalid type.
   """
-  image_or_images = ops.convert_to_tensor(images, name="images")
-  if image_or_images.dtype.base_dtype not in _IMAGE_DTYPES:
-    raise TypeError("Invalid dtype %s." % image_or_images.dtype)
-  if len(image_or_images.get_shape()) == 2:
-    images = image_or_images[None, :, :, None]
-  elif len(image_or_images.get_shape()) == 3:
-    images = image_or_images[None, :, :, :]
-  elif len(image_or_images.get_shape()) == 4:
-    images = image_or_images
-  else:
-    raise TypeError("Images should have rank between 2 and 4.")
-
-  image_height = math_ops.cast(array_ops.shape(images)[1], dtypes.float32)[None]
-  image_width = math_ops.cast(array_ops.shape(images)[2], dtypes.float32)[None]
-  output = transform(
-      images,
-      angles_to_projective_transforms(angles, image_height, image_width),
-      interpolation=interpolation)
-  if len(image_or_images.get_shape()) == 2:
-    return output[0, :, :, 0]
-  elif len(image_or_images.get_shape()) == 3:
-    return output[0, :, :, :]
-  else:
-    return output
+  with ops.name_scope(name, "rotate"):
+    image_or_images = ops.convert_to_tensor(images)
+    if image_or_images.dtype.base_dtype not in _IMAGE_DTYPES:
+      raise TypeError("Invalid dtype %s." % image_or_images.dtype)
+    elif image_or_images.get_shape().ndims is None:
+      raise TypeError("image_or_images rank must be statically known")
+    elif len(image_or_images.get_shape()) == 2:
+      images = image_or_images[None, :, :, None]
+    elif len(image_or_images.get_shape()) == 3:
+      images = image_or_images[None, :, :, :]
+    elif len(image_or_images.get_shape()) == 4:
+      images = image_or_images
+    else:
+      raise TypeError("Images should have rank between 2 and 4.")
+
+    image_height = math_ops.cast(array_ops.shape(images)[1],
+                                 dtypes.float32)[None]
+    image_width = math_ops.cast(array_ops.shape(images)[2],
+                                dtypes.float32)[None]
+    output = transform(
+        images,
+        angles_to_projective_transforms(angles, image_height, image_width),
+        interpolation=interpolation)
+    if image_or_images.get_shape().ndims is None:
+      raise TypeError("image_or_images rank must be statically known")
+    elif len(image_or_images.get_shape()) == 2:
+      return output[0, :, :, 0]
+    elif len(image_or_images.get_shape()) == 3:
+      return output[0, :, :, :]
+    else:
+      return output
+
+
+def translate(images, translations, interpolation="NEAREST", name=None):
+  """Translate image(s) by the passed vectors(s).
 
+  Args:
+    images: A tensor of shape (num_images, num_rows, num_columns, num_channels)
+        (NHWC), (num_rows, num_columns, num_channels) (HWC), or
+        (num_rows, num_columns) (HW). The rank must be statically known (the
+        shape is not `TensorShape(None)`.
+    translations: A vector representing [dx, dy] or (if images has rank 4)
+        a matrix of length num_images, with a [dx, dy] vector for each image in
+        the batch.
+    interpolation: Interpolation mode. Supported values: "NEAREST", "BILINEAR".
+    name: The name of the op.
 
-def angles_to_projective_transforms(angles, image_height, image_width):
+  Returns:
+    Image(s) with the same type and shape as `images`, translated by the given
+        vector(s). Empty space due to the translation will be filled with zeros.
+
+  Raises:
+    TypeError: If `image` is an invalid type.
+  """
+  with ops.name_scope(name, "translate"):
+    return transform(
+        images,
+        translations_to_projective_transforms(translations),
+        interpolation=interpolation)
+
+
+def angles_to_projective_transforms(angles,
+                                    image_height,
+                                    image_width,
+                                    name=None):
   """Returns projective transform(s) for the given angle(s).
 
   Args:
     angles: A scalar angle to rotate all images by, or (for batches of images)
-      a vector with an angle to rotate each image in the batch.
+        a vector with an angle to rotate each image in the batch. The rank must
+        be statically known (the shape is not `TensorShape(None)`.
     image_height: Height of the image(s) to be transformed.
     image_width: Width of the image(s) to be transformed.
 
@@ -94,41 +135,89 @@ def angles_to_projective_transforms(angles, image_height, image_width):
     A tensor of shape (num_images, 8). Projective transforms which can be given
       to `tf.contrib.image.transform`.
   """
-  angle_or_angles = ops.convert_to_tensor(
-      angles, name="angles", dtype=dtypes.float32)
-  if len(angle_or_angles.get_shape()) == 0:  # pylint: disable=g-explicit-length-test
-    angles = angle_or_angles[None]
-  elif len(angle_or_angles.get_shape()) == 1:
-    angles = angle_or_angles
-  else:
-    raise TypeError("Angles should have rank 0 or 1.")
-  x_offset = ((image_width - 1) - (math_ops.cos(angles) *
-                                   (image_width - 1) - math_ops.sin(angles) *
-                                   (image_height - 1))) / 2.0
-  y_offset = ((image_height - 1) - (math_ops.sin(angles) *
-                                    (image_width - 1) + math_ops.cos(angles) *
-                                    (image_height - 1))) / 2.0
-  num_angles = array_ops.shape(angles)[0]
-  return array_ops.concat(
-      values=[
-          math_ops.cos(angles)[:, None],
-          -math_ops.sin(angles)[:, None],
-          x_offset[:, None],
-          math_ops.sin(angles)[:, None],
-          math_ops.cos(angles)[:, None],
-          y_offset[:, None],
-          array_ops.zeros((num_angles, 2), dtypes.float32),
-      ],
-      axis=1)
-
-
-def transform(images, transforms, interpolation="NEAREST"):
+  with ops.name_scope(name, "angles_to_projective_transforms"):
+    angle_or_angles = ops.convert_to_tensor(
+        angles, name="angles", dtype=dtypes.float32)
+    if len(angle_or_angles.get_shape()) == 0:  # pylint: disable=g-explicit-length-test
+      angles = angle_or_angles[None]
+    elif len(angle_or_angles.get_shape()) == 1:
+      angles = angle_or_angles
+    else:
+      raise TypeError("Angles should have rank 0 or 1.")
+    x_offset = ((image_width - 1) - (math_ops.cos(angles) *
+                                     (image_width - 1) - math_ops.sin(angles) *
+                                     (image_height - 1))) / 2.0
+    y_offset = ((image_height - 1) - (math_ops.sin(angles) *
+                                      (image_width - 1) + math_ops.cos(angles) *
+                                      (image_height - 1))) / 2.0
+    num_angles = array_ops.shape(angles)[0]
+    return array_ops.concat(
+        values=[
+            math_ops.cos(angles)[:, None],
+            -math_ops.sin(angles)[:, None],
+            x_offset[:, None],
+            math_ops.sin(angles)[:, None],
+            math_ops.cos(angles)[:, None],
+            y_offset[:, None],
+            array_ops.zeros((num_angles, 2), dtypes.float32),
+        ],
+        axis=1)
+
+
+def translations_to_projective_transforms(translations, name=None):
+  """Returns projective transform(s) for the given translation(s).
+
+  Args:
+      translations: A 2-element list representing [dx, dy] or a matrix of
+          2-element lists representing [dx, dy] to translate for each image
+          (for a batch of images). The rank must be statically known (the shape
+          is not `TensorShape(None)`.
+      name: The name of the op.
+
+  Returns:
+      A tensor of shape (num_images, 8) projective transforms which can be given
+          to `tf.contrib.image.transform`.
+  """
+  with ops.name_scope(name, "translations_to_projective_transforms"):
+    translation_or_translations = ops.convert_to_tensor(
+        translations, name="translations", dtype=dtypes.float32)
+    if translation_or_translations.get_shape().ndims is None:
+      raise TypeError(
+          "translation_or_translations rank must be statically known")
+    elif len(translation_or_translations.get_shape()) == 1:
+      translations = translation_or_translations[None]
+    elif len(translation_or_translations.get_shape()) == 2:
+      translations = translation_or_translations
+    else:
+      raise TypeError("Translations should have rank 1 or 2.")
+    num_translations = array_ops.shape(translations)[0]
+    # The translation matrix looks like:
+    #     [[1 0 -dx]
+    #      [0 1 -dy]
+    #      [0 0 1]]
+    # where the last entry is implicit.
+    # Translation matrices are always float32.
+    return array_ops.concat(
+        values=[
+            array_ops.ones((num_translations, 1), dtypes.float32),
+            array_ops.zeros((num_translations, 1), dtypes.float32),
+            -translations[:, 0, None],
+            array_ops.zeros((num_translations, 1), dtypes.float32),
+            array_ops.ones((num_translations, 1), dtypes.float32),
+            -translations[:, 1, None],
+            array_ops.zeros((num_translations, 2), dtypes.float32),
+        ],
+        axis=1)
+
+
+def transform(images, transforms, interpolation="NEAREST", name=None):
   """Applies the given transform(s) to the image(s).
 
   Args:
     images: A tensor of shape (num_images, num_rows, num_columns, num_channels)
        (NHWC), (num_rows, num_columns, num_channels) (HWC), or
-       (num_rows, num_columns) (HW).
+       (num_rows, num_columns) (HW). The rank must be statically known (the
+       shape is not `TensorShape(None)`.
     transforms: Projective transform matrix/matrices. A vector of length 8 or
        tensor of size N x 8. If one row of transforms is
        [a0, a1, a2, b0, b1, b2, c0, c1], then it maps the *output* point
@@ -146,34 +235,40 @@ def transform(images, transforms, interpolation="NEAREST"):
   Raises:
     TypeError: If `image` is an invalid type.
   """
-  image_or_images = ops.convert_to_tensor(images, name="images")
-  transform_or_transforms = ops.convert_to_tensor(
-      transforms, name="transforms", dtype=dtypes.float32)
-  if image_or_images.dtype.base_dtype not in _IMAGE_DTYPES:
-    raise TypeError("Invalid dtype %s." % image_or_images.dtype)
-  if len(image_or_images.get_shape()) == 2:
-    images = image_or_images[None, :, :, None]
-  elif len(image_or_images.get_shape()) == 3:
-    images = image_or_images[None, :, :, :]
-  elif len(image_or_images.get_shape()) == 4:
-    images = image_or_images
-  else:
-    raise TypeError("Images should have rank between 2 and 4.")
-
-  if len(transform_or_transforms.get_shape()) == 1:
-    transforms = transform_or_transforms[None]
-  elif len(transform_or_transforms.get_shape()) == 2:
-    transforms = transform_or_transforms
-  else:
-    raise TypeError("Transforms should have rank 1 or 2.")
-  output = gen_image_ops.image_projective_transform(
-      images, transforms, interpolation=interpolation.upper())
-  if len(image_or_images.get_shape()) == 2:
-    return output[0, :, :, 0]
-  elif len(image_or_images.get_shape()) == 3:
-    return output[0, :, :, :]
-  else:
-    return output
+  with ops.name_scope(name, "transform"):
+    image_or_images = ops.convert_to_tensor(images, name="images")
+    transform_or_transforms = ops.convert_to_tensor(
+        transforms, name="transforms", dtype=dtypes.float32)
+    if image_or_images.dtype.base_dtype not in _IMAGE_DTYPES:
+      raise TypeError("Invalid dtype %s." % image_or_images.dtype)
+    elif image_or_images.get_shape().ndims is None:
+      raise TypeError("image_or_images rank must be statically known")
+    elif len(image_or_images.get_shape()) == 2:
+      images = image_or_images[None, :, :, None]
+    elif len(image_or_images.get_shape()) == 3:
+      images = image_or_images[None, :, :, :]
+    elif len(image_or_images.get_shape()) == 4:
+      images = image_or_images
+    else:
+      raise TypeError("Images should have rank between 2 and 4.")
+
+    if len(transform_or_transforms.get_shape()) == 1:
+      transforms = transform_or_transforms[None]
+    elif transform_or_transforms.get_shape().ndims is None:
+      raise TypeError(
+          "transform_or_transforms rank must be statically known")
+    elif len(transform_or_transforms.get_shape()) == 2:
+      transforms = transform_or_transforms
+    else:
+      raise TypeError("Transforms should have rank 1 or 2.")
+    output = gen_image_ops.image_projective_transform(
+        images, transforms, interpolation=interpolation.upper())
+    if len(image_or_images.get_shape()) == 2:
+      return output[0, :, :, 0]
+    elif len(image_or_images.get_shape()) == 3:
+      return output[0, :, :, :]
+    else:
+      return output
 
 
 def compose_transforms(*transforms):
@@ -191,11 +286,12 @@ def compose_transforms(*transforms):
         order.
   """
   assert transforms, "transforms cannot be empty"
-  composed = _flat_transforms_to_matrices(transforms[0])
-  for tr in transforms[1:]:
-    # Multiply batches of matrices.
-    composed = math_ops.matmul(composed, _flat_transforms_to_matrices(tr))
-  return _transform_matrices_to_flat(composed)
+  with ops.name_scope("compose_transforms"):
+    composed = _flat_transforms_to_matrices(transforms[0])
+    for tr in transforms[1:]:
+      # Multiply batches of matrices.
+      composed = math_ops.matmul(composed, _flat_transforms_to_matrices(tr))
+    return _transform_matrices_to_flat(composed)
 
 
 def _flat_transforms_to_matrices(transforms):
@@ -211,8 +307,8 @@ def _flat_transforms_to_matrices(transforms):
 
 def _transform_matrices_to_flat(transform_matrices):
   # Flatten each matrix.
-  transforms = array_ops.reshape(
-      transform_matrices, constant_op.constant([-1, 9]))
+  transforms = array_ops.reshape(transform_matrices,
+                                 constant_op.constant([-1, 9]))
   # Divide each matrix by the last entry (normally 1).
   transforms /= transforms[:, 8:9]
   return transforms[:, :8]
@@ -260,10 +356,10 @@ def _image_projective_transform_grad(op, grad):
     return [output, None]
 
 
-def bipartite_match(
-    distance_mat,
-    num_valid_rows,
-    top_k=-1):
+def bipartite_match(distance_mat,
+                    num_valid_rows,
+                    top_k=-1,
+                    name="bipartite_match"):
   """Find bipartite matching based on a given distance matrix.
 
   A greedy bi-partite matching algorithm is used to obtain the matching with
@@ -282,6 +378,7 @@ def bipartite_match(
     top_k: A scalar that specifies the number of top-k matches to retrieve.
       If set to be negative, then is set according to the maximum number of
       matches from `distance_mat`.
+    name: The name of the op.
 
   Returns:
     row_to_col_match_indices: A vector of length num_rows, which is the number
@@ -292,7 +389,8 @@ def bipartite_match(
       If `col_to_row_match_indices[j]` is not -1, column j is matched to row
       `col_to_row_match_indices[j]`.
   """
-  result = gen_image_ops.bipartite_match(distance_mat, num_valid_rows, top_k)
+  result = gen_image_ops.bipartite_match(
+      distance_mat, num_valid_rows, top_k, name=name)
   return result
 
 
-- 
GitLab


From cbd2974ed583ed725c33c22000a1a357cc30e46b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 10 Oct 2017 11:16:59 -0700
Subject: [PATCH 230/909] Adding comment to documentation of
 tf.image.crop_and_resize about it being corner aligned.

PiperOrigin-RevId: 171706213
---
 tensorflow/core/ops/image_ops.cc | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/ops/image_ops.cc b/tensorflow/core/ops/image_ops.cc
index 1453943d78..a44bac60bf 100644
--- a/tensorflow/core/ops/image_ops.cc
+++ b/tensorflow/core/ops/image_ops.cc
@@ -1101,7 +1101,10 @@ slice from the input image and does not allow resizing or aspect ratio change.
 Returns a tensor with `crops` from the input `image` at positions defined at the
 bounding box locations in `boxes`. The cropped boxes are all resized (with
 bilinear interpolation) to a fixed `size = [crop_height, crop_width]`. The
-result is a 4-D tensor `[num_boxes, crop_height, crop_width, depth]`.
+result is a 4-D tensor `[num_boxes, crop_height, crop_width, depth]`. The
+resizing is corner aligned. In particular, if `boxes = [[0, 0, 1, 1]]`, the
+method will give identical results to using `tf.image.resize_bilinear()`
+with `align_corners=True`.
 
 image: A 4-D tensor of shape `[batch, image_height, image_width, depth]`.
   Both `image_height` and `image_width` need to be positive.
-- 
GitLab


From 2446c53c8c9510f881f6193c91be21b8e8a9a488 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 10 Oct 2017 11:31:20 -0700
Subject: [PATCH 231/909] Update the base image for TF CPU remote build image

* `clang-debian8` from Cloud Launcher will be used directly , without building from source
  https://console.cloud.google.com/launcher/details/google/clang-debian8?filter=category:developer-tools&q=clang

PiperOrigin-RevId: 171708832
---
 .../tools/ci_build/remote/Dockerfile.cpu      |  2 +-
 .../ci_build/remote/remote_docker_build.sh    | 32 +++++++++----------
 2 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/tensorflow/tools/ci_build/remote/Dockerfile.cpu b/tensorflow/tools/ci_build/remote/Dockerfile.cpu
index 04365f12d6..7b01d8320d 100644
--- a/tensorflow/tools/ci_build/remote/Dockerfile.cpu
+++ b/tensorflow/tools/ci_build/remote/Dockerfile.cpu
@@ -1,4 +1,4 @@
-FROM debian8-clang:latest
+FROM launcher.gcr.io/google/clang-debian8:latest
 
 RUN apt-get update && apt-get --no-install-recommends install -y \
     binutils \
diff --git a/tensorflow/tools/ci_build/remote/remote_docker_build.sh b/tensorflow/tools/ci_build/remote/remote_docker_build.sh
index 0ac1165dcd..3ac6840f4e 100755
--- a/tensorflow/tools/ci_build/remote/remote_docker_build.sh
+++ b/tensorflow/tools/ci_build/remote/remote_docker_build.sh
@@ -88,25 +88,25 @@ function print_usage {
 }
 
 
+# Build nvidia-cuba-clang base image for GPU image.
+# For CPU the `clang-debian8` from Cloud Launcher will be used directly:
+# https://console.cloud.google.com/launcher/details/google/clang-debian8?filter=category:developer-tools&q=clang
 function build_base_image {
-  if [ "$cpu_build" = true ] ; then
-    base_image="debian8"
-  else
+  if [ "$gpu_build" = true ] ; then
     base_image="nvidia-cuda"
+    # Run a 2-stage build for clang base image, see
+    # https://github.com/llvm-mirror/llvm/blob/master/docs/Docker.rst
+    $base_image_build_script \
+      --source $base_image \
+      --branch branches/google/stable \
+      --docker-repository ${base_image}-clang --docker-tag "latest" \
+      -p clang -i stage2-install-clang -i stage2-install-clang-headers \
+      -- \
+      -DLLVM_TARGETS_TO_BUILD=Native -DCMAKE_BUILD_TYPE=Release \
+      -DBOOTSTRAP_CMAKE_BUILD_TYPE=Release \
+      -DCLANG_ENABLE_BOOTSTRAP=ON \
+      -DCLANG_BOOTSTRAP_TARGETS="install-clang;install-clang-headers"
   fi
-
-  # Run a 2-stage build for clang base image, see
-  # https://github.com/llvm-mirror/llvm/blob/master/docs/Docker.rst
-  $base_image_build_script \
-    --source $base_image \
-    --branch branches/google/stable \
-    --docker-repository ${base_image}-clang --docker-tag "latest" \
-    -p clang -i stage2-install-clang -i stage2-install-clang-headers \
-    -- \
-    -DLLVM_TARGETS_TO_BUILD=Native -DCMAKE_BUILD_TYPE=Release \
-    -DBOOTSTRAP_CMAKE_BUILD_TYPE=Release \
-    -DCLANG_ENABLE_BOOTSTRAP=ON \
-    -DCLANG_BOOTSTRAP_TARGETS="install-clang;install-clang-headers"
 }
 
 
-- 
GitLab


From afdfb5ac9807223cf3c21515a794ae7216f59700 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 10 Oct 2017 11:35:11 -0700
Subject: [PATCH 232/909] Update ops-related pbtxt files.

PiperOrigin-RevId: 171709536
---
 tensorflow/core/ops/ops.pbtxt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index fc22594ea4..fcb5792e5c 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -5631,7 +5631,7 @@ op {
     description: "Value used for extrapolation, when applicable."
   }
   summary: "Extracts crops from the input image tensor and bilinearly resizes them (possibly"
-  description: "with aspect ratio change) to a common output size specified by `crop_size`. This\nis more general than the `crop_to_bounding_box` op which extracts a fixed size\nslice from the input image and does not allow resizing or aspect ratio change.\n\nReturns a tensor with `crops` from the input `image` at positions defined at the\nbounding box locations in `boxes`. The cropped boxes are all resized (with\nbilinear interpolation) to a fixed `size = [crop_height, crop_width]`. The\nresult is a 4-D tensor `[num_boxes, crop_height, crop_width, depth]`."
+  description: "with aspect ratio change) to a common output size specified by `crop_size`. This\nis more general than the `crop_to_bounding_box` op which extracts a fixed size\nslice from the input image and does not allow resizing or aspect ratio change.\n\nReturns a tensor with `crops` from the input `image` at positions defined at the\nbounding box locations in `boxes`. The cropped boxes are all resized (with\nbilinear interpolation) to a fixed `size = [crop_height, crop_width]`. The\nresult is a 4-D tensor `[num_boxes, crop_height, crop_width, depth]`. The\nresizing is corner aligned. In particular, if `boxes = [[0, 0, 1, 1]]`, the\nmethod will give identical results to using `tf.image.resize_bilinear()`\nwith `align_corners=True`."
 }
 op {
   name: "CropAndResizeGradBoxes"
-- 
GitLab


From 651b7d587bc366bf93b551b3df2b44cf9fb53c71 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 10 Oct 2017 11:43:14 -0700
Subject: [PATCH 233/909] Go: Update generated wrapper functions for TensorFlow
 ops.

PiperOrigin-RevId: 171710900
---
 tensorflow/go/op/wrappers.go | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go
index 9417de3932..96a1c2695a 100644
--- a/tensorflow/go/op/wrappers.go
+++ b/tensorflow/go/op/wrappers.go
@@ -8523,7 +8523,10 @@ func CropAndResizeExtrapolationValue(value float32) CropAndResizeAttr {
 // Returns a tensor with `crops` from the input `image` at positions defined at the
 // bounding box locations in `boxes`. The cropped boxes are all resized (with
 // bilinear interpolation) to a fixed `size = [crop_height, crop_width]`. The
-// result is a 4-D tensor `[num_boxes, crop_height, crop_width, depth]`.
+// result is a 4-D tensor `[num_boxes, crop_height, crop_width, depth]`. The
+// resizing is corner aligned. In particular, if `boxes = [[0, 0, 1, 1]]`, the
+// method will give identical results to using `tf.image.resize_bilinear()`
+// with `align_corners=True`.
 //
 // Arguments:
 //	image: A 4-D tensor of shape `[batch, image_height, image_width, depth]`.
-- 
GitLab


From 253f5386cb6478dba6d9b99286775c6cbbe86a9a Mon Sep 17 00:00:00 2001
From: Asim Shankar <ashankar@google.com>
Date: Tue, 10 Oct 2017 12:09:42 -0700
Subject: [PATCH 234/909] eager: Fix an issue with tf.identity.

Like with graph execution, tf.identity should accept an input that
is not a Tensor instance but can be converted to one.

PiperOrigin-RevId: 171714919
---
 tensorflow/python/eager/ops_test.py | 3 +++
 tensorflow/python/ops/array_ops.py  | 8 +++++++-
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/eager/ops_test.py b/tensorflow/python/eager/ops_test.py
index 6d17c7eeff..7d54b8d2d8 100644
--- a/tensorflow/python/eager/ops_test.py
+++ b/tensorflow/python/eager/ops_test.py
@@ -299,6 +299,9 @@ class OpsTest(test_util.TensorFlowTestCase):
     y = flatten_layer(x)
     self.assertAllEqual([[-10, -20, -30, -40], [10, 20, 30, 40]], y.numpy())
 
+  def testIdentity(self):
+    self.assertEqual(2, array_ops.identity(2).numpy())
+
 
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/python/ops/array_ops.py b/tensorflow/python/ops/array_ops.py
index 3e0cfba90d..61405e3f45 100644
--- a/tensorflow/python/ops/array_ops.py
+++ b/tensorflow/python/ops/array_ops.py
@@ -124,7 +124,13 @@ def identity(input, name=None):  # pylint: disable=redefined-builtin
   if context.in_graph_mode():
     return gen_array_ops.identity(input, name=name)
   else:
-    if context.context().device_name != input.device:
+    try:
+      in_device = input.device
+    except AttributeError:
+      input = ops.convert_to_tensor(input)
+      in_device = input.device
+    # TODO(ashankar): Does 'identity' need to invoke execution callbacks?
+    if context.context().device_name != in_device:
       return input._copy()  # pylint: disable=protected-access
     return input
 
-- 
GitLab


From 9954458183ebd8d0ab5f7d06f063c8372dbcf6fb Mon Sep 17 00:00:00 2001
From: Andrew Selle <aselle@google.com>
Date: Tue, 10 Oct 2017 12:14:35 -0700
Subject: [PATCH 235/909] Define truncatemod in terms of tf.truncatediv to be
 explicit.

PiperOrigin-RevId: 171715629
---
 tensorflow/core/ops/math_ops.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/ops/math_ops.cc b/tensorflow/core/ops/math_ops.cc
index 015fd6e388..ab0bc258f7 100644
--- a/tensorflow/core/ops/math_ops.cc
+++ b/tensorflow/core/ops/math_ops.cc
@@ -692,8 +692,8 @@ REGISTER_OP("Mod")
     .SetShapeFn(shape_inference::BroadcastBinaryOpShapeFn)
     .Doc(R"doc(
 Returns element-wise remainder of division. This emulates C semantics in that
-the result here is consistent with a truncating divide. E.g. `truncate(x / y) *
-y + truncate_mod(x, y) = x`.
+the result here is consistent with a truncating divide. E.g.
+`tf.truncatediv(x, y) * y + truncate_mod(x, y) = x`.
 
 *NOTE*: `Mod` supports broadcasting. More about broadcasting
 [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
-- 
GitLab


From 721fbda83fc0cb00c9bf9ed461c8fc3084f42fe1 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 10 Oct 2017 12:20:59 -0700
Subject: [PATCH 236/909] [TF:XLA] Rename BINOP_LOGICAL_X to BINOP_X

PiperOrigin-RevId: 171716540
---
 .../compiler/xla/client/computation_builder.cc       |  6 +++---
 tensorflow/compiler/xla/service/shape_inference.cc   | 12 ++++++------
 tensorflow/compiler/xla/service/user_computation.cc  |  6 +++---
 tensorflow/compiler/xla/xla_data.proto               |  6 +++---
 4 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/tensorflow/compiler/xla/client/computation_builder.cc b/tensorflow/compiler/xla/client/computation_builder.cc
index 4757e8b0d2..cbd71dad86 100644
--- a/tensorflow/compiler/xla/client/computation_builder.cc
+++ b/tensorflow/compiler/xla/client/computation_builder.cc
@@ -958,18 +958,18 @@ ComputationDataHandle ComputationBuilder::Min(
 ComputationDataHandle ComputationBuilder::And(
     const ComputationDataHandle& lhs, const ComputationDataHandle& rhs,
     tensorflow::gtl::ArraySlice<int64> broadcast_dimensions) {
-  return BinaryOp(BINOP_LOGICAL_AND, lhs, rhs, broadcast_dimensions);
+  return BinaryOp(BINOP_AND, lhs, rhs, broadcast_dimensions);
 }
 
 ComputationDataHandle ComputationBuilder::Or(
     const ComputationDataHandle& lhs, const ComputationDataHandle& rhs,
     tensorflow::gtl::ArraySlice<int64> broadcast_dimensions) {
-  return BinaryOp(BINOP_LOGICAL_OR, lhs, rhs, broadcast_dimensions);
+  return BinaryOp(BINOP_OR, lhs, rhs, broadcast_dimensions);
 }
 
 ComputationDataHandle ComputationBuilder::Not(
     const ComputationDataHandle& operand) {
-  return UnaryOp(UNOP_LOGICAL_NOT, operand);
+  return UnaryOp(UNOP_NOT, operand);
 }
 
 ComputationDataHandle ComputationBuilder::Abs(
diff --git a/tensorflow/compiler/xla/service/shape_inference.cc b/tensorflow/compiler/xla/service/shape_inference.cc
index 06a68c81e4..b333d232a7 100644
--- a/tensorflow/compiler/xla/service/shape_inference.cc
+++ b/tensorflow/compiler/xla/service/shape_inference.cc
@@ -58,7 +58,7 @@ UnaryOperation OpcodeToUnaryOperation(HloOpcode opcode) {
     case HloOpcode::kLog:
       return UNOP_LOG;
     case HloOpcode::kNot:
-      return UNOP_LOGICAL_NOT;
+      return UNOP_NOT;
     case HloOpcode::kNegate:
       return UNOP_NEGATE;
     case HloOpcode::kRoundNearestAfz:
@@ -114,9 +114,9 @@ BinaryOperation OpcodeToBinaryOperation(HloOpcode opcode) {
     case HloOpcode::kRemainder:
       return BINOP_REM;
     case HloOpcode::kOr:
-      return BINOP_LOGICAL_OR;
+      return BINOP_OR;
     case HloOpcode::kAnd:
-      return BINOP_LOGICAL_AND;
+      return BINOP_AND;
     default:
       LOG(FATAL) << "unhandled opcode " << opcode;
   }
@@ -322,7 +322,7 @@ StatusOr<Shape> InferWindowOutputShape(const Shape& base_shape,
     case UNOP_SORT:
       return arg;
 
-    case UNOP_LOGICAL_NOT:
+    case UNOP_NOT:
       if (arg.element_type() != PRED) {
         return InvalidArgument(
             "expected pred element type in argument to logical-not operation; "
@@ -750,8 +750,8 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(
       return InferElementwiseBinaryOpShape(operation, lhs, rhs,
                                            broadcast_dimensions);
 
-    case BINOP_LOGICAL_AND:
-    case BINOP_LOGICAL_OR:
+    case BINOP_AND:
+    case BINOP_OR:
       if (lhs.element_type() != PRED) {
         return InvalidArgument(
             "expected pred element type in argument to logical and/or "
diff --git a/tensorflow/compiler/xla/service/user_computation.cc b/tensorflow/compiler/xla/service/user_computation.cc
index 05f5476b88..317817d022 100644
--- a/tensorflow/compiler/xla/service/user_computation.cc
+++ b/tensorflow/compiler/xla/service/user_computation.cc
@@ -58,7 +58,7 @@ HloOpcode UnaryOperationToHloOpcode(UnaryOperation unop) {
       return HloOpcode::kIsFinite;
     case UNOP_LOG:
       return HloOpcode::kLog;
-    case UNOP_LOGICAL_NOT:
+    case UNOP_NOT:
       return HloOpcode::kNot;
     case UNOP_NEGATE:
       return HloOpcode::kNegate;
@@ -111,9 +111,9 @@ HloOpcode BinaryOperationToHloOpcode(BinaryOperation binop) {
       return HloOpcode::kPower;
     case BINOP_REM:
       return HloOpcode::kRemainder;
-    case BINOP_LOGICAL_OR:
+    case BINOP_OR:
       return HloOpcode::kOr;
-    case BINOP_LOGICAL_AND:
+    case BINOP_AND:
       return HloOpcode::kAnd;
     default:
       LOG(FATAL) << "unhandled operation " << binop;
diff --git a/tensorflow/compiler/xla/xla_data.proto b/tensorflow/compiler/xla/xla_data.proto
index 1771a3d5de..3f26b88809 100644
--- a/tensorflow/compiler/xla/xla_data.proto
+++ b/tensorflow/compiler/xla/xla_data.proto
@@ -618,7 +618,7 @@ enum UnaryOperation {
   UNOP_INVALID = 0;
 
   // Elementwise, logical negation
-  UNOP_LOGICAL_NOT = 1;
+  UNOP_NOT = 1;
 
   // Elementwise, computes e^x.
   UNOP_EXP = 2;
@@ -707,8 +707,8 @@ enum BinaryOperation {
   BINOP_REM = 17;
 
   // Logical operators
-  BINOP_LOGICAL_AND = 18;
-  BINOP_LOGICAL_OR = 19;
+  BINOP_AND = 18;
+  BINOP_OR = 19;
 }
 
 message BinaryOpRequest {
-- 
GitLab


From 803707b01fdc3048347f6e1b3aca751cf699b1e8 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 10 Oct 2017 12:21:24 -0700
Subject: [PATCH 237/909] Update ops-related pbtxt files.

PiperOrigin-RevId: 171716595
---
 tensorflow/core/ops/ops.pbtxt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index fcb5792e5c..7579aef259 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -14434,7 +14434,7 @@ op {
     }
   }
   summary: "Returns element-wise remainder of division. This emulates C semantics in that"
-  description: "the result here is consistent with a truncating divide. E.g. `truncate(x / y) *\ny + truncate_mod(x, y) = x`.\n\n*NOTE*: `Mod` supports broadcasting. More about broadcasting\n[here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)"
+  description: "the result here is consistent with a truncating divide. E.g.\n`tf.truncatediv(x, y) * y + truncate_mod(x, y) = x`.\n\n*NOTE*: `Mod` supports broadcasting. More about broadcasting\n[here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)"
 }
 op {
   name: "Mul"
-- 
GitLab


From 35c4177d9e2349e4b5c6875e85220fc3f8ddc17c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 10 Oct 2017 12:22:58 -0700
Subject: [PATCH 238/909] Allow tensorflow devices to report their load. This
 may be used to improve batch scheduling.

PiperOrigin-RevId: 171716813
---
 tensorflow/stream_executor/stream_executor_internal.h | 2 ++
 tensorflow/stream_executor/stream_executor_pimpl.cc   | 4 ++++
 tensorflow/stream_executor/stream_executor_pimpl.h    | 4 ++++
 3 files changed, 10 insertions(+)

diff --git a/tensorflow/stream_executor/stream_executor_internal.h b/tensorflow/stream_executor/stream_executor_internal.h
index 802ef755eb..12593e31d4 100644
--- a/tensorflow/stream_executor/stream_executor_internal.h
+++ b/tensorflow/stream_executor/stream_executor_internal.h
@@ -225,6 +225,8 @@ class StreamExecutorInterface {
   virtual port::Status SetDeviceSharedMemoryConfig(
       SharedMemoryConfig config) = 0;
 
+  virtual int64 GetDeviceLoad() { return -1; }
+
   virtual bool DeviceMemoryUsage(int64 *free, int64 *total) const {
     return false;
   }
diff --git a/tensorflow/stream_executor/stream_executor_pimpl.cc b/tensorflow/stream_executor/stream_executor_pimpl.cc
index 9bbfe7f04a..9dc1749327 100644
--- a/tensorflow/stream_executor/stream_executor_pimpl.cc
+++ b/tensorflow/stream_executor/stream_executor_pimpl.cc
@@ -268,6 +268,10 @@ const DeviceDescription &StreamExecutor::GetDeviceDescription() const {
   return *device_description_;
 }
 
+int64 StreamExecutor::GetDeviceLoad() const {
+  return implementation_->GetDeviceLoad();
+}
+
 int StreamExecutor::PlatformDeviceCount() const {
   return implementation_->PlatformDeviceCount();
 }
diff --git a/tensorflow/stream_executor/stream_executor_pimpl.h b/tensorflow/stream_executor/stream_executor_pimpl.h
index f354317a6e..9c225e5fae 100644
--- a/tensorflow/stream_executor/stream_executor_pimpl.h
+++ b/tensorflow/stream_executor/stream_executor_pimpl.h
@@ -312,6 +312,10 @@ class StreamExecutor {
   // The value is cached on first use.
   const DeviceDescription &GetDeviceDescription() const;
 
+  // If implemented, returns device specific measurement of load
+  // (e.g. pending requests).
+  int64 GetDeviceLoad() const;
+
   // Returns the underlying device memory usage information, if it is available.
   // If it is not available (false is returned), free/total may not be
   // initialized.
-- 
GitLab


From 97fa3e4b87e20ecf6c68225812056345aca5f4cc Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 10 Oct 2017 12:28:53 -0700
Subject: [PATCH 239/909] Go: Update generated wrapper functions for TensorFlow
 ops.

PiperOrigin-RevId: 171717474
---
 tensorflow/go/op/wrappers.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go
index 96a1c2695a..cf842f3808 100644
--- a/tensorflow/go/op/wrappers.go
+++ b/tensorflow/go/op/wrappers.go
@@ -8203,8 +8203,8 @@ func OnesLike(scope *Scope, x tf.Output) (y tf.Output) {
 
 // Returns element-wise remainder of division. This emulates C semantics in that
 //
-// the result here is consistent with a truncating divide. E.g. `truncate(x / y) *
-// y + truncate_mod(x, y) = x`.
+// the result here is consistent with a truncating divide. E.g.
+// `tf.truncatediv(x, y) * y + truncate_mod(x, y) = x`.
 //
 // *NOTE*: `Mod` supports broadcasting. More about broadcasting
 // [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
-- 
GitLab


From 70e2cbfeb6dc9ba9c01a93405cd64fab90ef0b2e Mon Sep 17 00:00:00 2001
From: Yangzihao Wang <yangzihao@google.com>
Date: Tue, 10 Oct 2017 12:29:36 -0700
Subject: [PATCH 240/909] Add an env-var to choose between FP16 and FP32 as the
 internal compute type for conv when input data is FP16. The env-var is set to
 use FP32 by default.

PiperOrigin-RevId: 171717550
---
 tensorflow/stream_executor/cuda/cuda_dnn.cc | 147 +++++++++++---------
 1 file changed, 83 insertions(+), 64 deletions(-)

diff --git a/tensorflow/stream_executor/cuda/cuda_dnn.cc b/tensorflow/stream_executor/cuda/cuda_dnn.cc
index 46516cc445..039f7ea029 100644
--- a/tensorflow/stream_executor/cuda/cuda_dnn.cc
+++ b/tensorflow/stream_executor/cuda/cuda_dnn.cc
@@ -2079,6 +2079,85 @@ dnn::AlgorithmDesc GetCudnnConvolutionForwardAlgorithm(
   return dnn::AlgorithmDesc(algo, use_tensor_ops);
 }
 
+// A helper class to set env-vars and choose options for cudnn-related
+// algorithms.
+template <typename EnvVar>
+class CudnnEnvVar {
+ public:
+  static bool IsEnabled() {
+    static bool is_enabled = IsEnabledImpl();
+    return is_enabled;
+  }
+
+ private:
+  static bool IsEnabledImpl() {
+    const char* tf_env_var_val = getenv(EnvVar::kName);
+    if (tf_env_var_val != nullptr) {
+      port::StringPiece tf_env_var_val_str(tf_env_var_val);
+      if (tf_env_var_val_str == "0") {
+        return false;
+      }
+      return true;
+    }
+    return EnvVar::kDefaultFlag;
+  }
+};
+
+// A helper struct to decide whether to enable the FFT_TILING algorithms for
+// forward convolution. Before cudnn v5.1 it works fine but since cudnn v5.1
+// it is turned off due to memory corruption caused by some shapes with this
+// algorithm.
+// Before NVIDIA fixes the memory corruption bug, users can explicitly
+// enable the algorithm through an env-var "TF_ENABLE_FFT_TILING_FORWARD=1".
+struct FftTilingForward {
+  static constexpr const char* kName = "TF_ENABLE_FFT_TILING_FORWARD";
+  // TODO(yangzihao): turn the default to True when the memory corruption bug
+  // is fixed.
+  static constexpr bool kDefaultFlag = CUDNN_VERSION < 5100;
+};
+
+// A helper struct to decide whether to enable the WINOGRAD_NONFUSED algorithms.
+// By default it is turned on, users can explicitly disable them through an
+// env-var "TF_ENABLE_WINOGRAD_NONFUSED=0".
+// https://github.com/tensorflow/tensorflow/pull/4901
+struct WinogradNonfused {
+  static constexpr const char* kName = "TF_ENABLE_WINOGRAD_NONFUSED";
+  // NVIDIA has fixed winograd nonfused bug for cudnn v>=7.
+  // For cudnn v>=5.1, we have a workaround and for any lower version, we
+  // disable it by default.
+  static constexpr bool kDefaultFlag = CUDNN_VERSION >= 5100;
+};
+
+// A helper struct to decide whether to use FP32 as the internal compute type
+// for convolution when the input data type is FP16. By default it is turned on,
+// users can explicitly disable them (choose to use FP16 as the internal compute
+// type) through an env-var "TF_FP16_CONV_USE_FP32_COMPUTE=0".
+struct ConvDoFP32ComputationFP16Input {
+  static constexpr const char* kName = "TF_FP16_CONV_USE_FP32_COMPUTE";
+  // Using FP16 as the internal compute type for convolution when the input data
+  // type is FP16 is only supported on architectures with true fp16 support
+  // (compute capability 5.3 and 6.0). Setting this to false in an unsupported
+  // architecture will cause internal errors.
+  static constexpr bool kDefaultFlag = true;
+};
+
+// A group of helper functions to return the internal compute type for
+// convolutions in cudnn.
+// TODO(yangzihao): Add support for float64.
+template <typename T>
+cudnnDataType_t GetConvComputeType() {
+  return CUDNN_DATA_FLOAT;
+}
+
+template <>
+cudnnDataType_t GetConvComputeType<Eigen::half>() {
+  if (CudnnEnvVar<ConvDoFP32ComputationFP16Input>::IsEnabled()) {
+    return CUDNN_DATA_FLOAT;
+  } else {
+    return CUDNN_DATA_HALF;
+  }
+}
+
 }  // namespace
 
 template <class T>
@@ -2098,12 +2177,8 @@ bool CudnnSupport::DoConvolveImpl(
       static_cast<cudnnDataType_t>(cudnn_type)};
   ScopedFilterDescriptor filter{parent_, filter_descriptor, batch_descriptor,
       static_cast<cudnnDataType_t>(cudnn_type)};
-  // TODO(sesse): Figure out under what circumstances cuDNN would
-  // accept CUDNN_DATA_HALF here; probably related to compute capability
-  // and cuDNN version; at least cuDNN 4 on TITAN X only supports
-  // CUDNN_DATA_FLOAT even for half input.
   ScopedConvolutionDescriptor conv{parent_, convolution_descriptor,
-      CUDNN_DATA_FLOAT};
+                                   GetConvComputeType<T>()};
 
   mutex_lock lock{dnn_handle_mutex_};
   auto status = wrap::cudnnSetStream(parent_, ToHandle(dnn_handle_),
@@ -2424,55 +2499,6 @@ bool CudnnSupport::DoFusedConvolveImpl(
 #endif  // CUDNN_VERSION < 6000
 }
 
-// A helper class to set env-vars and choose options for cudnn-related
-// algorithms.
-template <typename EnvVar>
-class CudnnEnvVar {
- public:
-  static bool IsEnabled() {
-    static bool is_enabled = IsEnabledImpl();
-    return is_enabled;
-  }
-
- private:
-  static bool IsEnabledImpl() {
-    const char* tf_env_var_val = getenv(EnvVar::kName);
-    if (tf_env_var_val != nullptr) {
-      port::StringPiece tf_env_var_val_str(tf_env_var_val);
-      if (tf_env_var_val_str == "0") {
-        return false;
-      }
-      return true;
-    }
-    return EnvVar::kDefaultFlag;
-  }
-};
-
-// A helper struct to decide whether to enable the FFT_TILING algorithms for
-// forward convolution. Before cudnn v5.1 it works fine but since cudnn v5.1
-// it is turned off due to memory corruption caused by some shapes with this
-// algorithm.
-// Before NVIDIA fixes the memory corruption bug, users can explicitly
-// enable the algorithm through an env-var "TF_ENABLE_FFT_TILING_FORWARD=1".
-struct FftTilingForward {
-  static constexpr const char* kName = "TF_ENABLE_FFT_TILING_FORWARD";
-  // TODO(yangzihao): turn the default to True when the memory corruption bug
-  // is fixed.
-  static constexpr bool kDefaultFlag = CUDNN_VERSION < 5100;
-};
-
-// A helper struct to decide whether to enable the WINOGRAD_NONFUSED algorithms.
-// By default it is turned on, users can explicitly disable them through an
-// env-var "TF_ENABLE_WINOGRAD_NONFUSED=0".
-// https://github.com/tensorflow/tensorflow/pull/4901
-struct WinogradNonfused {
-  static constexpr const char* kName = "TF_ENABLE_WINOGRAD_NONFUSED";
-  // NVIDIA has fixed winograd nonfused bug for cudnn v>=7.
-  // For cudnn v>=5.1, we have a workaround and for any lower version, we
-  // disable it by default.
-  static constexpr bool kDefaultFlag = CUDNN_VERSION >= 5100;
-};
-
 bool CudnnSupport::GetConvolveAlgorithms(
     bool with_winograd_nonfused, int cc_major, int cc_minor,
     std::vector<dnn::AlgorithmDesc>* out_algorithms) {
@@ -2990,12 +3016,8 @@ bool CudnnSupport::DoConvolveBackwardDataImpl(
                                     static_cast<cudnnDataType_t>(cudnn_type)};
   ScopedFilterDescriptor filter{parent_, filter_descriptor, input_descriptor,
                                 static_cast<cudnnDataType_t>(cudnn_type)};
-  // TODO(sesse): Figure out under what circumstances cuDNN would
-  // accept CUDNN_DATA_HALF here; probably related to compute capability
-  // and cuDNN version; at least cuDNN 4 on TITAN X only supports
-  // CUDNN_DATA_FLOAT even for half input.
   ScopedConvolutionDescriptor conv{parent_, convolution_descriptor,
-                                   CUDNN_DATA_FLOAT};
+                                   GetConvComputeType<T>()};
 
   const bool is_profiling = output_profile_result != nullptr;
   cudnnConvolutionBwdDataAlgo_t algo;
@@ -3245,12 +3267,8 @@ bool CudnnSupport::DoConvolveBackwardFilterImpl(
           static_cast<cudnnDataType_t>(cudnn_type)};
   ScopedFilterDescriptor filter{parent_, filter_descriptor, input_descriptor,
         static_cast<cudnnDataType_t>(cudnn_type)};
-  // TODO(sesse): Figure out under what circumstances cuDNN would
-  // accept CUDNN_DATA_HALF here; probably related to compute capability
-  // and cuDNN version; at least cuDNN 4 on TITAN X only supports
-  // CUDNN_DATA_FLOAT even for half input.
   ScopedConvolutionDescriptor conv{parent_, convolution_descriptor,
-      CUDNN_DATA_FLOAT};
+                                   GetConvComputeType<T>()};
 
   const bool is_profiling = output_profile_result != nullptr;
   cudnnConvolutionBwdFilterAlgo_t algo;
@@ -3403,6 +3421,7 @@ bool CudnnSupport::DoConvolveBackwardFilterImpl(
       /*beta=*/&beta,
       /*gradDesc=*/filter.handle(),
       /*gradData=*/backward_filter_data->opaque());
+
   if (is_profiling) {
     timer->Stop(AsCUDAStream(stream));
     if (status == CUDNN_STATUS_SUCCESS) {
-- 
GitLab


From 30e40833147f04467b791b9faad3284504194eb8 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 10 Oct 2017 12:29:45 -0700
Subject: [PATCH 241/909] Fix bug in peephole implementation of BlockLSTM Cell.
  Fix tests.

PiperOrigin-RevId: 171717566
---
 .../rnn/python/kernel_tests/lstm_ops_test.py  | 106 +++++++++++-------
 tensorflow/contrib/rnn/python/ops/lstm_ops.py |  36 +++---
 2 files changed, 82 insertions(+), 60 deletions(-)

diff --git a/tensorflow/contrib/rnn/python/kernel_tests/lstm_ops_test.py b/tensorflow/contrib/rnn/python/kernel_tests/lstm_ops_test.py
index 3016821b74..3f72203594 100644
--- a/tensorflow/contrib/rnn/python/kernel_tests/lstm_ops_test.py
+++ b/tensorflow/contrib/rnn/python/kernel_tests/lstm_ops_test.py
@@ -304,7 +304,7 @@ class LSTMBlockCellTest(test.TestCase):
       batch_size = 2
       input_size = 3
       cell_size = 4
-      sequence_length = 5
+      sequence_length = 4
 
       inputs = []
       for _ in range(sequence_length):
@@ -314,38 +314,49 @@ class LSTMBlockCellTest(test.TestCase):
 
       initializer = init_ops.random_uniform_initializer(
           -0.01, 0.01, seed=19890212)
-      with variable_scope.variable_scope("basic", initializer=initializer):
-        cell = rnn_cell.LSTMCell(
-            cell_size, use_peepholes=True, state_is_tuple=True)
-        outputs, state = rnn.static_rnn(cell, inputs, dtype=dtypes.float32)
 
-        sess.run([variables.global_variables_initializer()])
-        basic_outputs, basic_state = sess.run([outputs, state[0]])
-        basic_grads = sess.run(gradients_impl.gradients(outputs, inputs))
-        basic_wgrads = sess.run(
-            gradients_impl.gradients(outputs, variables.trainable_variables()))
+      with variable_scope.variable_scope("test", initializer=initializer):
+        # magic naming so that the cells pick up these variables and resuse them
+        wci = variable_scope.get_variable(
+            "rnn/lstm_cell/w_i_diag", shape=[cell_size], dtype=dtypes.float32)
+        wcf = variable_scope.get_variable(
+            "rnn/lstm_cell/w_f_diag", shape=[cell_size], dtype=dtypes.float32)
+        wco = variable_scope.get_variable(
+            "rnn/lstm_cell/w_o_diag", shape=[cell_size], dtype=dtypes.float32)
 
-      with variable_scope.variable_scope("block", initializer=initializer):
         w = variable_scope.get_variable(
-            "w",
+            "rnn/lstm_cell/kernel",
             shape=[input_size + cell_size, cell_size * 4],
             dtype=dtypes.float32)
         b = variable_scope.get_variable(
-            "b",
+            "rnn/lstm_cell/bias",
             shape=[cell_size * 4],
             dtype=dtypes.float32,
             initializer=init_ops.zeros_initializer())
 
-        wci = variable_scope.get_variable(
-            "wci", shape=[cell_size], dtype=dtypes.float32)
-        wcf = variable_scope.get_variable(
-            "wcf", shape=[cell_size], dtype=dtypes.float32)
-        wco = variable_scope.get_variable(
-            "wco", shape=[cell_size], dtype=dtypes.float32)
-
-        _, _, _, _, _, _, outputs = block_lstm(
-            ops.convert_to_tensor(
-                sequence_length, dtype=dtypes.int64),
+        wci_block = variable_scope.get_variable(
+            "rnn/lstm_cell/lstm_block_wrapper/w_i_diag",
+            initializer=wci.initialized_value())
+        wcf_block = variable_scope.get_variable(
+            "rnn/lstm_cell/lstm_block_wrapper/w_f_diag",
+            initializer=wcf.initialized_value())
+        wco_block = variable_scope.get_variable(
+            "rnn/lstm_cell/lstm_block_wrapper/w_o_diag",
+            initializer=wco.initialized_value())
+        w_block = variable_scope.get_variable(
+            "rnn/lstm_cell/lstm_block_wrapper/kernel",
+            initializer=w.initialized_value())
+        b_block = variable_scope.get_variable(
+            "rnn/lstm_cell/lstm_block_wrapper/bias",
+            initializer=b.initialized_value())
+
+        basic_cell = rnn_cell.LSTMCell(
+            cell_size, use_peepholes=True, state_is_tuple=True, reuse=True)
+        basic_outputs_op, basic_state_op = rnn.static_rnn(
+            basic_cell, inputs, dtype=dtypes.float32)
+
+        _, _, _, _, _, _, block_outputs_op = block_lstm(
+            ops.convert_to_tensor(sequence_length, dtype=dtypes.int64),
             inputs,
             w,
             b,
@@ -355,36 +366,45 @@ class LSTMBlockCellTest(test.TestCase):
             cell_clip=0,
             use_peephole=True)
 
+        with variable_scope.variable_scope("rnn/lstm_cell", reuse=True):
+          fused_cell = lstm_ops.LSTMBlockFusedCell(
+              cell_size, cell_clip=0, use_peephole=True)
+          fused_outputs_op, fused_state_op = fused_cell(
+              inputs, dtype=dtypes.float32)
+
         sess.run([variables.global_variables_initializer()])
-        block_outputs = sess.run(outputs)
-        block_grads = sess.run(gradients_impl.gradients(outputs, inputs))
+        basic_outputs, basic_state = sess.run(
+            [basic_outputs_op, basic_state_op[0]])
+        basic_grads = sess.run(
+            gradients_impl.gradients(basic_outputs_op, inputs))
+        basic_wgrads = sess.run(
+            gradients_impl.gradients(basic_outputs_op, [w, b, wci, wcf, wco]))
+
+        block_outputs = sess.run(block_outputs_op)
+        block_grads = sess.run(
+            gradients_impl.gradients(block_outputs_op, inputs))
         block_wgrads = sess.run(
-            gradients_impl.gradients(outputs, [w, b, wci, wcf, wco]))
+            gradients_impl.gradients(block_outputs_op, [w, b, wci, wcf, wco]))
+
+        fused_outputs, fused_state = sess.run(
+            [fused_outputs_op, fused_state_op[0]])
+        fused_grads = sess.run(
+            gradients_impl.gradients(fused_outputs_op, inputs))
+        fused_wgrads = sess.run(
+            gradients_impl.gradients(
+                fused_outputs_op,
+                [w_block, b_block, wci_block, wcf_block, wco_block]))
 
       self.assertAllClose(basic_outputs, block_outputs)
       self.assertAllClose(basic_grads, block_grads)
       for basic, block in zip(basic_wgrads, block_wgrads):
-        self.assertAllClose(basic, block, rtol=1e-2, atol=1e-2)
-
-      with variable_scope.variable_scope("fused", initializer=initializer):
-        cell = lstm_ops.LSTMBlockFusedCell(
-            cell_size, cell_clip=0, use_peephole=True)
-        outputs, state = cell(inputs, dtype=dtypes.float32)
-
-        sess.run([variables.global_variables_initializer()])
-        fused_outputs, fused_state = sess.run([outputs, state[0]])
-        fused_grads = sess.run(gradients_impl.gradients(outputs, inputs))
-        fused_vars = [
-            v for v in variables.trainable_variables()
-            if v.name.startswith("fused/")
-        ]
-        fused_wgrads = sess.run(gradients_impl.gradients(outputs, fused_vars))
+        self.assertAllClose(basic, block, rtol=1e-6, atol=1e-6)
 
       self.assertAllClose(basic_outputs, fused_outputs)
       self.assertAllClose(basic_state, fused_state)
       self.assertAllClose(basic_grads, fused_grads)
-      for basic, fused in zip(basic_wgrads, fused_wgrads):
-        self.assertAllClose(basic, fused, rtol=1e-2, atol=1e-2)
+      for basic, fused in zip(block_wgrads, fused_wgrads):
+        self.assertAllClose(basic, fused, rtol=1e-6, atol=1e-6)
 
   def testLSTMFusedSequenceLengths(self):
     """Verify proper support for sequence lengths in LSTMBlockFusedCell."""
diff --git a/tensorflow/contrib/rnn/python/ops/lstm_ops.py b/tensorflow/contrib/rnn/python/ops/lstm_ops.py
index 352dae3acf..df910a3423 100644
--- a/tensorflow/contrib/rnn/python/ops/lstm_ops.py
+++ b/tensorflow/contrib/rnn/python/ops/lstm_ops.py
@@ -116,8 +116,8 @@ def _lstm_block_cell(x,
     if cell_size is None:
       raise ValueError("cell_size from `cs_prev` should not be None.")
     wci = array_ops.constant(0, dtype=dtypes.float32, shape=[cell_size])
-    wco = wci
     wcf = wci
+    wco = wci
 
   # pylint: disable=protected-access
   return gen_lstm_ops.lstm_block_cell(
@@ -126,8 +126,8 @@ def _lstm_block_cell(x,
       h_prev=h_prev,
       w=w,
       wci=wci,
-      wco=wco,
       wcf=wcf,
+      wco=wco,
       b=b,
       forget_bias=forget_bias,
       cell_clip=cell_clip if cell_clip is not None else -1,
@@ -201,8 +201,8 @@ def _block_lstm(seq_len_max,
     h_prev = zero_state
   if wci is None:
     wci = array_ops.constant(0, dtype=dtypes.float32, shape=[cell_size])
-    wco = wci
     wcf = wci
+    wco = wci
 
   # pylint: disable=protected-access
   i, cs, f, o, ci, co, h = gen_lstm_ops.block_lstm(
@@ -212,8 +212,8 @@ def _block_lstm(seq_len_max,
       h_prev=h_prev,
       w=w,
       wci=wci,
-      wco=wco,
       wcf=wcf,
+      wco=wco,
       b=b,
       forget_bias=forget_bias,
       cell_clip=cell_clip if cell_clip is not None else -1,
@@ -233,7 +233,7 @@ _lstm_block_cell_grad_outputs = ["cs_prev_grad", "dicfo"]
 @ops.RegisterGradient("LSTMBlockCell")
 def _LSTMBlockCellGrad(op, *grad):
   """Gradient for LSTMBlockCell."""
-  (x, cs_prev, h_prev, w, wci, wco, wcf, b) = op.inputs
+  (x, cs_prev, h_prev, w, wci, wcf, wco, b) = op.inputs
   (i, cs, f, o, ci, co, _) = op.outputs
   (_, cs_grad, _, _, _, _, h_grad) = grad
 
@@ -293,13 +293,13 @@ def _LSTMBlockCellGrad(op, *grad):
 @ops.RegisterGradient("BlockLSTM")
 def _BlockLSTMGrad(op, *grad):
   """Gradient for BlockLSTM."""
-  seq_len_max, x, cs_prev, h_prev, w, wci, wco, wcf, b = op.inputs
+  seq_len_max, x, cs_prev, h_prev, w, wci, wcf, wco, b = op.inputs
   i, cs, f, o, ci, co, h = op.outputs
 
   cs_grad = grad[1]
   h_grad = grad[6]
 
-  (x_grad, cs_prev_grad, h_prev_grad, w_grad, wci_grad, wco_grad, wcf_grad,
+  (x_grad, cs_prev_grad, h_prev_grad, w_grad, wci_grad, wcf_grad, wco_grad,
    b_grad) = gen_lstm_ops.block_lstm_grad(
        seq_len_max,
        x,
@@ -307,8 +307,8 @@ def _BlockLSTMGrad(op, *grad):
        h_prev,
        w,
        wci,
-       wco,
        wcf,
+       wco,
        b,
        i,
        cs,
@@ -321,8 +321,10 @@ def _BlockLSTMGrad(op, *grad):
        h_grad,
        use_peephole=op.get_attr("use_peephole"))
 
-  return [None, x_grad, cs_prev_grad, h_prev_grad, w_grad, wci_grad, wco_grad,
-          wcf_grad, b_grad]
+  return [
+      None, x_grad, cs_prev_grad, h_prev_grad, w_grad, wci_grad, wcf_grad,
+      wco_grad, b_grad
+  ]
 
 
 class LSTMBlockCell(rnn_cell_impl.RNNCell):
@@ -367,8 +369,8 @@ class LSTMBlockCell(rnn_cell_impl.RNNCell):
         "W": "kernel",
         "b": "bias",
         "wci": "w_i_diag",
-        "wco": "w_o_diag",
         "wcf": "w_f_diag",
+        "wco": "w_o_diag",
         "scope": "lstm_cell"
     }
 
@@ -396,10 +398,10 @@ class LSTMBlockCell(rnn_cell_impl.RNNCell):
           initializer=init_ops.constant_initializer(0.0))
       if self._use_peephole:
         wci = vs.get_variable(self._names["wci"], [self._num_units])
-        wco = vs.get_variable(self._names["wco"], [self._num_units])
         wcf = vs.get_variable(self._names["wcf"], [self._num_units])
+        wco = vs.get_variable(self._names["wco"], [self._num_units])
       else:
-        wci = wco = wcf = array_ops.zeros([self._num_units])
+        wci = wcf = wco = array_ops.zeros([self._num_units])
       (cs_prev, h_prev) = states_prev
       (_, cs, _, _, _, _, h) = _lstm_block_cell(
           x,
@@ -408,8 +410,8 @@ class LSTMBlockCell(rnn_cell_impl.RNNCell):
           w,
           b,
           wci=wci,
-          wco=wco,
           wcf=wcf,
+          wco=wco,
           forget_bias=self._forget_bias,
           cell_clip=self._cell_clip,
           use_peephole=self._use_peephole)
@@ -644,10 +646,10 @@ class LSTMBlockFusedCell(LSTMBlockWrapper):
         dtype=dtype)
     if self._use_peephole:
       wci = vs.get_variable("w_i_diag", [self._num_units], dtype=dtype)
-      wco = vs.get_variable("w_o_diag", [self._num_units], dtype=dtype)
       wcf = vs.get_variable("w_f_diag", [self._num_units], dtype=dtype)
+      wco = vs.get_variable("w_o_diag", [self._num_units], dtype=dtype)
     else:
-      wci = wco = wcf = array_ops.zeros([self._num_units], dtype=dtype)
+      wci = wcf = wco = array_ops.zeros([self._num_units], dtype=dtype)
 
     if sequence_length is None:
       max_seq_len = math_ops.to_int64(time_len)
@@ -661,8 +663,8 @@ class LSTMBlockFusedCell(LSTMBlockWrapper):
         h_prev=initial_output,
         w=w,
         wci=wci,
-        wco=wco,
         wcf=wcf,
+        wco=wco,
         b=b,
         forget_bias=self._forget_bias,
         cell_clip=self._cell_clip,
-- 
GitLab


From e74adb670920dd6f41306a4a40784a535ea7b878 Mon Sep 17 00:00:00 2001
From: Michael Case <mikecase@google.com>
Date: Tue, 10 Oct 2017 12:33:27 -0700
Subject: [PATCH 242/909] Fix S3 BUILD not including files explicitly.

This causes remote builds to fail since they AWS headers were missing.

PiperOrigin-RevId: 171718021
---
 third_party/aws.BUILD | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/third_party/aws.BUILD b/third_party/aws.BUILD
index 858a55ee07..38b7e0e543 100644
--- a/third_party/aws.BUILD
+++ b/third_party/aws.BUILD
@@ -19,6 +19,7 @@ cc_library(
             "aws-cpp-sdk-core/source/platform/linux-shared/*.cpp",
         ]),
     }) + glob([
+        "aws-cpp-sdk-core/include/**/*.h",
         "aws-cpp-sdk-core/source/*.cpp",
         "aws-cpp-sdk-core/source/auth/**/*.cpp",
         "aws-cpp-sdk-core/source/config/**/*.cpp",
@@ -38,6 +39,7 @@ cc_library(
         "aws-cpp-sdk-core/source/utils/xml/**/*.cpp",
         "aws-cpp-sdk-core/source/utils/crypto/*.cpp",
         "aws-cpp-sdk-core/source/utils/crypto/factory/**/*.cpp",
+        "aws-cpp-sdk-s3/include/**/*.h",
         "aws-cpp-sdk-s3/source/**/*.cpp",
     ]),
     hdrs = [
-- 
GitLab


From 0ffb522f02129c5d23a8b20ef56d0fefd7be91fe Mon Sep 17 00:00:00 2001
From: Max Galkin <maxgalkin@google.com>
Date: Tue, 10 Oct 2017 13:06:16 -0700
Subject: [PATCH 243/909] Add a flag to erase "_noinline" attribute to allow
 total inlining in Grappler.

PiperOrigin-RevId: 171722354
---
 .../core/grappler/grappler_item_builder.cc    | 26 ++++++++++++-------
 .../core/grappler/grappler_item_builder.h     | 20 +++++++-------
 2 files changed, 26 insertions(+), 20 deletions(-)

diff --git a/tensorflow/core/grappler/grappler_item_builder.cc b/tensorflow/core/grappler/grappler_item_builder.cc
index cb7d7f7330..d23facf81a 100644
--- a/tensorflow/core/grappler/grappler_item_builder.cc
+++ b/tensorflow/core/grappler/grappler_item_builder.cc
@@ -74,7 +74,7 @@ void InitializeTensor(DataType type, Tensor* tensor) {
 // of the cluster type (E.g: single cpu, multiple gpu, etc)  being simulated in
 // order to get the correct session options and environment, and performing the
 // correct optimizations.
-Status OptimizeGraph(const GraphDef& graph_def, GraphDef* output_graph_def,
+Status OptimizeGraph(const GraphDef& graph_def_arg, GraphDef* output_graph_def,
                      const ItemConfig& cfg) {
   if (!cfg.apply_optimizations && !cfg.inline_functions) {
     return Status::OK();
@@ -83,8 +83,16 @@ Status OptimizeGraph(const GraphDef& graph_def, GraphDef* output_graph_def,
   // Create a session option for a single GPU device.
   SessionOptions options;
 
-  // Inline all functions.
-  GraphDef inlined_graph_def(graph_def);
+  // Make a local copy of graph def, because we need to change some things.
+  GraphDef graph_def(graph_def_arg);
+
+  if (cfg.inline_functions && cfg.erase_noinline_attributes) {
+    // TF optimizer doesn't inline functions with "_noinline" attribute,
+    // so let's go over the function library and erase it.
+    for (auto& func : *graph_def.mutable_library()->mutable_function()) {
+      func.mutable_attr()->erase("_noinline");
+    }
+  }
 
   // Instantiate all variables for function library runtime creation.
   std::vector<Device*> devices;
@@ -92,7 +100,7 @@ Status OptimizeGraph(const GraphDef& graph_def, GraphDef* output_graph_def,
       options, "/job:localhost/replica:0/task:0", &devices));
   std::unique_ptr<DeviceMgr> dvc_mgr(new DeviceMgr(devices));
   FunctionLibraryDefinition function_library(OpRegistry::Global(),
-                                             inlined_graph_def.library());
+                                             graph_def.library());
   Env* env = Env::Default();
 
   // Optimizer options: L1 and inlining. L1 is default.
@@ -108,7 +116,7 @@ Status OptimizeGraph(const GraphDef& graph_def, GraphDef* output_graph_def,
   // Create the function library runtime.
   std::unique_ptr<ProcessFunctionLibraryRuntime> pflr(
       new ProcessFunctionLibraryRuntime(dvc_mgr.get(), env,
-                                        inlined_graph_def.versions().producer(),
+                                        graph_def.versions().producer(),
                                         &function_library, *optimizer_opts));
   FunctionLibraryRuntime* flr = pflr->GetFLR(devices[0]->name());
 
@@ -118,11 +126,11 @@ Status OptimizeGraph(const GraphDef& graph_def, GraphDef* output_graph_def,
   graph_ctor_opts.expect_device_spec = false;
   std::unique_ptr<Graph> graphptr(new Graph(function_library));
   // Populate default attrs to the NodeDefs in the GraphDef.
-  TF_RETURN_IF_ERROR(AddDefaultAttrsToGraphDef(&inlined_graph_def,
-                                               *graphptr->op_registry(), 0));
+  TF_RETURN_IF_ERROR(
+      AddDefaultAttrsToGraphDef(&graph_def, *graphptr->op_registry(), 0));
 
-  TF_RETURN_IF_ERROR(ConvertGraphDefToGraph(graph_ctor_opts, inlined_graph_def,
-                                            graphptr.get()));
+  TF_RETURN_IF_ERROR(
+      ConvertGraphDefToGraph(graph_ctor_opts, graph_def, graphptr.get()));
 
   // Optimize the graph.
   GraphOptimizer optimizer(*optimizer_opts);
diff --git a/tensorflow/core/grappler/grappler_item_builder.h b/tensorflow/core/grappler/grappler_item_builder.h
index 4ce5055e7a..9a7f52228b 100644
--- a/tensorflow/core/grappler/grappler_item_builder.h
+++ b/tensorflow/core/grappler/grappler_item_builder.h
@@ -27,24 +27,22 @@ class MetaGraphDef;
 namespace grappler {
 
 struct ItemConfig {
-  ItemConfig()
-      : ignore_user_placement(true),
-        ignore_colocation(true),
-        placeholder_unknown_output_shape_dim(-1),
-        apply_optimizations(false),
-        inline_functions(false) {}
+  ItemConfig() {}
 
   // If true, ignore all user specified node placement.
-  bool ignore_user_placement;
+  bool ignore_user_placement = true;
   // If true, ignore all user specified colocation attributes.
-  bool ignore_colocation;
+  bool ignore_colocation = true;
   // Dimension to use if a placeholder node has an _output_shapes attribute with
   // a dimension of -1.
-  int placeholder_unknown_output_shape_dim;
+  int placeholder_unknown_output_shape_dim = -1;
   // If true, does L1 optimizations.
-  bool apply_optimizations;
+  bool apply_optimizations = false;
   // If true, does inlining.
-  bool inline_functions;
+  bool inline_functions = false;
+  // If true, erases all "_noinline" attributes from user-defined functions.
+  // Has no effect if "inline_functions" is disabled.
+  bool erase_noinline_attributes = false;
   // If non-empty, override the directory of asset paths.
   string assets_directory_override;
 };
-- 
GitLab


From 3f4c6ccadf51475050549d4d3445e75869768aac Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Tue, 10 Oct 2017 14:11:27 -0700
Subject: [PATCH 244/909] Internal change.

PiperOrigin-RevId: 171731884
---
 tensorflow/contrib/estimator/BUILD | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/estimator/BUILD b/tensorflow/contrib/estimator/BUILD
index 596f68844b..3b61afe45e 100644
--- a/tensorflow/contrib/estimator/BUILD
+++ b/tensorflow/contrib/estimator/BUILD
@@ -50,7 +50,10 @@ py_test(
     size = "small",
     srcs = ["python/estimator/dnn_test.py"],
     srcs_version = "PY2AND3",
-    tags = ["no_pip"],
+    tags = [
+        "no_pip",
+        "notsan",
+    ],
     deps = [
         ":dnn",
         ":head",
-- 
GitLab


From 23418e4317b9e2c4a5148368daec873592a0de9e Mon Sep 17 00:00:00 2001
From: Eugene Brevdo <ebrevdo@google.com>
Date: Tue, 10 Oct 2017 14:16:21 -0700
Subject: [PATCH 245/909] Move LinearOperator to tf.linalg (with backwards
 compatibility support in contrib.linalg.)

PiperOrigin-RevId: 171732711
---
 tensorflow/BUILD                              |   2 +
 tensorflow/contrib/cmake/tf_python.cmake      |   2 +
 .../bijectors/affine_linear_operator_test.py  |   2 +-
 .../kernel_tests/distribution_util_test.py    |   2 +-
 .../kernel_tests/vector_diffeomixture_test.py |   4 +-
 .../python/ops/bijectors/affine_impl.py       |   2 +-
 .../bijectors/affine_linear_operator_impl.py  |   6 +-
 .../python/ops/distribution_util.py           |   2 +-
 .../python/ops/mvn_diag_plus_low_rank.py      |   2 +-
 .../python/ops/mvn_full_covariance.py         |   4 +-
 .../python/ops/mvn_linear_operator.py         |  14 +-
 .../distributions/python/ops/mvn_tril.py      |   6 +-
 .../python/ops/vector_diffeomixture.py        |  14 +-
 .../python/ops/vector_exponential_diag.py     |   2 +-
 .../ops/vector_exponential_linear_operator.py |   8 +-
 .../ops/vector_laplace_linear_operator.py     |  14 +-
 .../distributions/python/ops/wishart.py       |  12 +-
 tensorflow/contrib/linalg/BUILD               | 165 ++----------------
 tensorflow/contrib/linalg/__init__.py         |  18 +-
 .../linear_operator_addition_test.py          |  27 +--
 .../python/ops/linear_operator_addition.py    |  15 +-
 .../api_guides/python/contrib.linalg.md       |   4 +-
 tensorflow/python/BUILD                       |  18 +-
 tensorflow/python/__init__.py                 |   2 +-
 tensorflow/python/kernel_tests/BUILD          |   2 +-
 tensorflow/python/kernel_tests/linalg/BUILD   | 149 ++++++++++++++++
 .../python/kernel_tests/linalg/__init__.py    |  18 ++
 .../linear_operator_composition_test.py       |   4 +-
 .../linalg}/linear_operator_diag_test.py      |   4 +-
 .../linear_operator_full_matrix_test.py       |   4 +-
 .../linalg}/linear_operator_identity_test.py  |   4 +-
 .../linear_operator_low_rank_update_test.py}  |  49 +++---
 .../linear_operator_lower_triangular_test.py} |  16 +-
 .../linalg}/linear_operator_test.py           |   3 +-
 .../linalg}/linear_operator_util_test.py      |   4 +-
 .../python/kernel_tests/linalg_ops_test.py    |   2 +-
 tensorflow/python/ops/distributions/util.py   |   4 +-
 tensorflow/python/ops/linalg/BUILD            |  38 ++++
 .../python/ops/{ => linalg}/__init__.py       |   0
 .../ops/{linalg_ns.py => linalg/linalg.py}    |  14 +-
 .../python/ops/{ => linalg}/linalg_impl.py    |   0
 .../ops/linalg}/linear_operator.py            |   6 +-
 .../linalg}/linear_operator_composition.py    |   2 +-
 .../ops/linalg}/linear_operator_diag.py       |   4 +-
 .../linalg}/linear_operator_full_matrix.py    |   2 +-
 .../ops/linalg}/linear_operator_identity.py   |   4 +-
 .../linear_operator_low_rank_update.py}       |  30 ++--
 .../linear_operator_lower_triangular.py}      |  27 +--
 .../ops/linalg}/linear_operator_test_util.py  |   6 +-
 .../ops/linalg}/linear_operator_util.py       |   0
 ...r-operator-composition.__metaclass__.pbtxt |  14 ++
 ....linalg.-linear-operator-composition.pbtxt | 134 ++++++++++++++
 ....-linear-operator-diag.__metaclass__.pbtxt |  14 ++
 ...sorflow.linalg.-linear-operator-diag.pbtxt | 134 ++++++++++++++
 ...r-operator-full-matrix.__metaclass__.pbtxt |  14 ++
 ....linalg.-linear-operator-full-matrix.pbtxt | 130 ++++++++++++++
 ...near-operator-identity.__metaclass__.pbtxt |  14 ++
 ...low.linalg.-linear-operator-identity.pbtxt | 131 ++++++++++++++
 ...erator-low-rank-update.__metaclass__.pbtxt |  14 ++
 ...alg.-linear-operator-low-rank-update.pbtxt | 154 ++++++++++++++++
 ...rator-lower-triangular.__metaclass__.pbtxt |  14 ++
 ...lg.-linear-operator-lower-triangular.pbtxt | 130 ++++++++++++++
 ...erator-scaled-identity.__metaclass__.pbtxt |  14 ++
 ...alg.-linear-operator-scaled-identity.pbtxt | 135 ++++++++++++++
 ...inalg.-linear-operator.__metaclass__.pbtxt |  14 ++
 .../tensorflow.linalg.-linear-operator.pbtxt  | 129 ++++++++++++++
 .../tools/api/golden/tensorflow.linalg.pbtxt  |  32 ++++
 67 files changed, 1631 insertions(+), 333 deletions(-)
 create mode 100644 tensorflow/python/kernel_tests/linalg/BUILD
 create mode 100644 tensorflow/python/kernel_tests/linalg/__init__.py
 rename tensorflow/{contrib/linalg/python/kernel_tests => python/kernel_tests/linalg}/linear_operator_composition_test.py (98%)
 rename tensorflow/{contrib/linalg/python/kernel_tests => python/kernel_tests/linalg}/linear_operator_diag_test.py (97%)
 rename tensorflow/{contrib/linalg/python/kernel_tests => python/kernel_tests/linalg}/linear_operator_full_matrix_test.py (98%)
 rename tensorflow/{contrib/linalg/python/kernel_tests => python/kernel_tests/linalg}/linear_operator_identity_test.py (99%)
 rename tensorflow/{contrib/linalg/python/kernel_tests/linear_operator_udvh_update_test.py => python/kernel_tests/linalg/linear_operator_low_rank_update_test.py} (88%)
 rename tensorflow/{contrib/linalg/python/kernel_tests/linear_operator_tril_test.py => python/kernel_tests/linalg/linear_operator_lower_triangular_test.py} (86%)
 rename tensorflow/{contrib/linalg/python/kernel_tests => python/kernel_tests/linalg}/linear_operator_test.py (99%)
 rename tensorflow/{contrib/linalg/python/kernel_tests => python/kernel_tests/linalg}/linear_operator_util_test.py (98%)
 create mode 100644 tensorflow/python/ops/linalg/BUILD
 rename tensorflow/python/ops/{ => linalg}/__init__.py (100%)
 rename tensorflow/python/ops/{linalg_ns.py => linalg/linalg.py} (78%)
 rename tensorflow/python/ops/{ => linalg}/linalg_impl.py (100%)
 rename tensorflow/{contrib/linalg/python/ops => python/ops/linalg}/linear_operator.py (99%)
 rename tensorflow/{contrib/linalg/python/ops => python/ops/linalg}/linear_operator_composition.py (99%)
 rename tensorflow/{contrib/linalg/python/ops => python/ops/linalg}/linear_operator_diag.py (98%)
 rename tensorflow/{contrib/linalg/python/ops => python/ops/linalg}/linear_operator_full_matrix.py (98%)
 rename tensorflow/{contrib/linalg/python/ops => python/ops/linalg}/linear_operator_identity.py (99%)
 rename tensorflow/{contrib/linalg/python/ops/linear_operator_udvh_update.py => python/ops/linalg/linear_operator_low_rank_update.py} (95%)
 rename tensorflow/{contrib/linalg/python/ops/linear_operator_tril.py => python/ops/linalg/linear_operator_lower_triangular.py} (90%)
 rename tensorflow/{contrib/linalg/python/ops => python/ops/linalg}/linear_operator_test_util.py (99%)
 rename tensorflow/{contrib/linalg/python/ops => python/ops/linalg}/linear_operator_util.py (100%)
 create mode 100644 tensorflow/tools/api/golden/tensorflow.linalg.-linear-operator-composition.__metaclass__.pbtxt
 create mode 100644 tensorflow/tools/api/golden/tensorflow.linalg.-linear-operator-composition.pbtxt
 create mode 100644 tensorflow/tools/api/golden/tensorflow.linalg.-linear-operator-diag.__metaclass__.pbtxt
 create mode 100644 tensorflow/tools/api/golden/tensorflow.linalg.-linear-operator-diag.pbtxt
 create mode 100644 tensorflow/tools/api/golden/tensorflow.linalg.-linear-operator-full-matrix.__metaclass__.pbtxt
 create mode 100644 tensorflow/tools/api/golden/tensorflow.linalg.-linear-operator-full-matrix.pbtxt
 create mode 100644 tensorflow/tools/api/golden/tensorflow.linalg.-linear-operator-identity.__metaclass__.pbtxt
 create mode 100644 tensorflow/tools/api/golden/tensorflow.linalg.-linear-operator-identity.pbtxt
 create mode 100644 tensorflow/tools/api/golden/tensorflow.linalg.-linear-operator-low-rank-update.__metaclass__.pbtxt
 create mode 100644 tensorflow/tools/api/golden/tensorflow.linalg.-linear-operator-low-rank-update.pbtxt
 create mode 100644 tensorflow/tools/api/golden/tensorflow.linalg.-linear-operator-lower-triangular.__metaclass__.pbtxt
 create mode 100644 tensorflow/tools/api/golden/tensorflow.linalg.-linear-operator-lower-triangular.pbtxt
 create mode 100644 tensorflow/tools/api/golden/tensorflow.linalg.-linear-operator-scaled-identity.__metaclass__.pbtxt
 create mode 100644 tensorflow/tools/api/golden/tensorflow.linalg.-linear-operator-scaled-identity.pbtxt
 create mode 100644 tensorflow/tools/api/golden/tensorflow.linalg.-linear-operator.__metaclass__.pbtxt
 create mode 100644 tensorflow/tools/api/golden/tensorflow.linalg.-linear-operator.pbtxt

diff --git a/tensorflow/BUILD b/tensorflow/BUILD
index 1620bb5f2a..5bb31d7df1 100644
--- a/tensorflow/BUILD
+++ b/tensorflow/BUILD
@@ -490,7 +490,9 @@ filegroup(
         "//tensorflow/python/keras:all_files",
         "//tensorflow/python/kernel_tests:all_files",
         "//tensorflow/python/kernel_tests/distributions:all_files",
+        "//tensorflow/python/kernel_tests/linalg:all_files",
         "//tensorflow/python/ops/distributions:all_files",
+        "//tensorflow/python/ops/linalg:all_files",
         "//tensorflow/python/profiler:all_files",
         "//tensorflow/python/profiler/internal:all_files",
         "//tensorflow/python/saved_model:all_files",
diff --git a/tensorflow/contrib/cmake/tf_python.cmake b/tensorflow/contrib/cmake/tf_python.cmake
index bb3e69d53c..883b36b3fb 100755
--- a/tensorflow/contrib/cmake/tf_python.cmake
+++ b/tensorflow/contrib/cmake/tf_python.cmake
@@ -266,12 +266,14 @@ add_python_module("tensorflow/python/keras/_impl/keras/utils")
 add_python_module("tensorflow/python/keras/_impl/keras/wrappers")
 add_python_module("tensorflow/python/kernel_tests")
 add_python_module("tensorflow/python/kernel_tests/distributions")
+add_python_module("tensorflow/python/kernel_tests/linalg")
 add_python_module("tensorflow/python/layers")
 add_python_module("tensorflow/python/lib")
 add_python_module("tensorflow/python/lib/core")
 add_python_module("tensorflow/python/lib/io")
 add_python_module("tensorflow/python/ops")
 add_python_module("tensorflow/python/ops/distributions")
+add_python_module("tensorflow/python/ops/linalg")
 add_python_module("tensorflow/python/ops/losses")
 add_python_module("tensorflow/python/platform")
 add_python_module("tensorflow/python/platform/default")
diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/affine_linear_operator_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/affine_linear_operator_test.py
index 0738754b21..405ddd292c 100644
--- a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/affine_linear_operator_test.py
+++ b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/affine_linear_operator_test.py
@@ -72,7 +72,7 @@ class AffineLinearOperatorTest(test.TestCase):
                         [3, -2, 0],
                         [4, 3, 2]]],
                       dtype=np.float32)
-      scale = linalg.LinearOperatorTriL(tril, is_non_singular=True)
+      scale = linalg.LinearOperatorLowerTriangular(tril, is_non_singular=True)
       affine = AffineLinearOperator(
           shift=shift, scale=scale, validate_args=True)
 
diff --git a/tensorflow/contrib/distributions/python/kernel_tests/distribution_util_test.py b/tensorflow/contrib/distributions/python/kernel_tests/distribution_util_test.py
index d10312d667..2d74aa1f32 100644
--- a/tensorflow/contrib/distributions/python/kernel_tests/distribution_util_test.py
+++ b/tensorflow/contrib/distributions/python/kernel_tests/distribution_util_test.py
@@ -23,11 +23,11 @@ import itertools
 import numpy as np
 
 from tensorflow.contrib.distributions.python.ops import distribution_util
-from tensorflow.contrib.linalg.python.ops import linear_operator_diag
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import tensor_shape
 from tensorflow.python.ops import array_ops
+from tensorflow.python.ops.linalg import linear_operator_diag
 import tensorflow.python.ops.nn_grad  # pylint: disable=unused-import
 from tensorflow.python.platform import test
 
diff --git a/tensorflow/contrib/distributions/python/kernel_tests/vector_diffeomixture_test.py b/tensorflow/contrib/distributions/python/kernel_tests/vector_diffeomixture_test.py
index 070ee61be3..aea4d42503 100644
--- a/tensorflow/contrib/distributions/python/kernel_tests/vector_diffeomixture_test.py
+++ b/tensorflow/contrib/distributions/python/kernel_tests/vector_diffeomixture_test.py
@@ -22,9 +22,9 @@ import numpy as np
 
 from tensorflow.contrib.distributions.python.ops import test_util
 from tensorflow.contrib.distributions.python.ops import vector_diffeomixture as vector_diffeomixture_lib
-from tensorflow.contrib.linalg.python.ops import linear_operator_diag as linop_diag_lib
-from tensorflow.contrib.linalg.python.ops import linear_operator_identity as linop_identity_lib
 from tensorflow.python.ops.distributions import normal as normal_lib
+from tensorflow.python.ops.linalg import linear_operator_diag as linop_diag_lib
+from tensorflow.python.ops.linalg import linear_operator_identity as linop_identity_lib
 from tensorflow.python.platform import test
 
 
diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/affine_impl.py b/tensorflow/contrib/distributions/python/ops/bijectors/affine_impl.py
index f74d699a43..05bb9c2f9b 100644
--- a/tensorflow/contrib/distributions/python/ops/bijectors/affine_impl.py
+++ b/tensorflow/contrib/distributions/python/ops/bijectors/affine_impl.py
@@ -326,7 +326,7 @@ class Affine(bijector.Bijector):
         shape_hint=shape_hint)
 
     if perturb_factor is not None:
-      return linalg.LinearOperatorUDVHUpdate(
+      return linalg.LinearOperatorLowRankUpdate(
           scale,
           u=perturb_factor,
           diag_update=perturb_diag,
diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/affine_linear_operator_impl.py b/tensorflow/contrib/distributions/python/ops/bijectors/affine_linear_operator_impl.py
index ae380b5cb2..89043b1410 100644
--- a/tensorflow/contrib/distributions/python/ops/bijectors/affine_linear_operator_impl.py
+++ b/tensorflow/contrib/distributions/python/ops/bijectors/affine_linear_operator_impl.py
@@ -19,7 +19,6 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.contrib.distributions.python.ops.shape import _DistributionShape
-from tensorflow.contrib.linalg.python.ops import linear_operator
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
@@ -27,6 +26,7 @@ from tensorflow.python.framework import tensor_util
 from tensorflow.python.ops import check_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops.distributions import bijector
+from tensorflow.python.ops.linalg import linear_operator
 
 
 __all__ = [
@@ -66,7 +66,7 @@ class AffineLinearOperator(bijector.Bijector):
   Example Use:
 
   ```python
-  linalg = tf.contrib.linalg
+  linalg = tf.linalg
 
   x = [1., 2, 3]
 
@@ -82,7 +82,7 @@ class AffineLinearOperator(bijector.Bijector):
   tril = [[1., 0, 0],
           [2, 1, 0],
           [3, 2, 1]]
-  scale = linalg.LinearOperatorTriL(tril)
+  scale = linalg.LinearOperatorLowerTriangular(tril)
   affine = AffineLinearOperator(shift, scale)
   # In this case, `forward` is equivalent to:
   # np.squeeze(np.matmul(tril, np.expand_dims(x, -1)), -1) + shift
diff --git a/tensorflow/contrib/distributions/python/ops/distribution_util.py b/tensorflow/contrib/distributions/python/ops/distribution_util.py
index 3ed5592bf9..869b5698e5 100644
--- a/tensorflow/contrib/distributions/python/ops/distribution_util.py
+++ b/tensorflow/contrib/distributions/python/ops/distribution_util.py
@@ -160,7 +160,7 @@ def make_tril_scale(
 
     scale_tril = array_ops.matrix_set_diag(scale_tril, tril_diag)
 
-    return linalg.LinearOperatorTriL(
+    return linalg.LinearOperatorLowerTriangular(
         tril=_maybe_attach_assertion(scale_tril),
         is_non_singular=True,
         is_self_adjoint=False,
diff --git a/tensorflow/contrib/distributions/python/ops/mvn_diag_plus_low_rank.py b/tensorflow/contrib/distributions/python/ops/mvn_diag_plus_low_rank.py
index ee3e02e020..040bc23072 100644
--- a/tensorflow/contrib/distributions/python/ops/mvn_diag_plus_low_rank.py
+++ b/tensorflow/contrib/distributions/python/ops/mvn_diag_plus_low_rank.py
@@ -237,7 +237,7 @@ class MultivariateNormalDiagPlusLowRank(
             scale_perturb_diag,
             name="scale_perturb_diag")
         if has_low_rank:
-          scale = linalg.LinearOperatorUDVHUpdate(
+          scale = linalg.LinearOperatorLowRankUpdate(
               scale,
               u=scale_perturb_factor,
               diag_update=scale_perturb_diag,
diff --git a/tensorflow/contrib/distributions/python/ops/mvn_full_covariance.py b/tensorflow/contrib/distributions/python/ops/mvn_full_covariance.py
index 221eed547b..f9952b2069 100644
--- a/tensorflow/contrib/distributions/python/ops/mvn_full_covariance.py
+++ b/tensorflow/contrib/distributions/python/ops/mvn_full_covariance.py
@@ -174,8 +174,8 @@ class MultivariateNormalFullCovariance(mvn_tril.MultivariateNormalTriL):
             covariance_matrix = control_flow_ops.with_dependencies(
                 [assert_symmetric], covariance_matrix)
           # No need to validate that covariance_matrix is non-singular.
-          # LinearOperatorTriL has an assert_non_singular method that is called
-          # by the Bijector.
+          # LinearOperatorLowerTriangular has an assert_non_singular method that
+          # is called by the Bijector.
           # However, cholesky() ignores the upper triangular part, so we do need
           # to separately assert symmetric.
           scale_tril = linalg_ops.cholesky(covariance_matrix)
diff --git a/tensorflow/contrib/distributions/python/ops/mvn_linear_operator.py b/tensorflow/contrib/distributions/python/ops/mvn_linear_operator.py
index 50c7ba418b..251c2dbdfa 100644
--- a/tensorflow/contrib/distributions/python/ops/mvn_linear_operator.py
+++ b/tensorflow/contrib/distributions/python/ops/mvn_linear_operator.py
@@ -18,7 +18,6 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.contrib import linalg
 from tensorflow.contrib.distributions.python.ops import distribution_util
 from tensorflow.contrib.distributions.python.ops.bijectors import AffineLinearOperator
 from tensorflow.python.framework import ops
@@ -28,6 +27,7 @@ from tensorflow.python.ops import math_ops
 from tensorflow.python.ops.distributions import kullback_leibler
 from tensorflow.python.ops.distributions import normal
 from tensorflow.python.ops.distributions import transformed_distribution
+from tensorflow.python.ops.linalg import linalg
 
 
 __all__ = [
@@ -92,7 +92,7 @@ class MultivariateNormalLinearOperator(
 
   ```python
   ds = tf.contrib.distributions
-  la = tf.contrib.linalg
+  la = tf.linalg
 
   # Initialize a single 3-variate Gaussian.
   mu = [1., 2, 3]
@@ -106,7 +106,7 @@ class MultivariateNormalLinearOperator(
 
   mvn = ds.MultivariateNormalLinearOperator(
       loc=mu,
-      scale=la.LinearOperatorTriL(scale))
+      scale=la.LinearOperatorLowerTriangular(scale))
 
   # Covariance agrees with cholesky(cov) parameterization.
   mvn.covariance().eval()
@@ -243,8 +243,8 @@ class MultivariateNormalLinearOperator(
   def _variance(self):
     if distribution_util.is_diagonal_scale(self.scale):
       return math_ops.square(self.scale.diag_part())
-    elif (isinstance(self.scale, linalg.LinearOperatorUDVHUpdate)
-          and self.scale.is_self_adjoint):
+    elif (isinstance(self.scale, linalg.LinearOperatorLowRankUpdate) and
+          self.scale.is_self_adjoint):
       return array_ops.matrix_diag_part(
           self.scale.matmul(self.scale.to_dense()))
     else:
@@ -254,8 +254,8 @@ class MultivariateNormalLinearOperator(
   def _stddev(self):
     if distribution_util.is_diagonal_scale(self.scale):
       return math_ops.abs(self.scale.diag_part())
-    elif (isinstance(self.scale, linalg.LinearOperatorUDVHUpdate)
-          and self.scale.is_self_adjoint):
+    elif (isinstance(self.scale, linalg.LinearOperatorLowRankUpdate) and
+          self.scale.is_self_adjoint):
       return math_ops.sqrt(array_ops.matrix_diag_part(
           self.scale.matmul(self.scale.to_dense())))
     else:
diff --git a/tensorflow/contrib/distributions/python/ops/mvn_tril.py b/tensorflow/contrib/distributions/python/ops/mvn_tril.py
index 48c4dddc81..e3d68f6b4c 100644
--- a/tensorflow/contrib/distributions/python/ops/mvn_tril.py
+++ b/tensorflow/contrib/distributions/python/ops/mvn_tril.py
@@ -188,9 +188,9 @@ class MultivariateNormalTriL(
               assert_proper_shapes=validate_args)
         else:
           # No need to validate that scale_tril is non-singular.
-          # LinearOperatorTriL has an assert_non_singular method that is called
-          # by the Bijector.
-          scale = linalg.LinearOperatorTriL(
+          # LinearOperatorLowerTriangular has an assert_non_singular
+          # method that is called by the Bijector.
+          scale = linalg.LinearOperatorLowerTriangular(
               scale_tril,
               is_non_singular=True,
               is_self_adjoint=False,
diff --git a/tensorflow/contrib/distributions/python/ops/vector_diffeomixture.py b/tensorflow/contrib/distributions/python/ops/vector_diffeomixture.py
index 6d297ea1f1..438d628da4 100644
--- a/tensorflow/contrib/distributions/python/ops/vector_diffeomixture.py
+++ b/tensorflow/contrib/distributions/python/ops/vector_diffeomixture.py
@@ -23,10 +23,6 @@ import numpy as np
 from tensorflow.contrib.distributions.python.ops import distribution_util
 from tensorflow.contrib.distributions.python.ops.bijectors.affine_linear_operator import AffineLinearOperator
 from tensorflow.contrib.linalg.python.ops import linear_operator_addition as linop_add_lib
-from tensorflow.contrib.linalg.python.ops import linear_operator_diag as linop_diag_lib
-from tensorflow.contrib.linalg.python.ops import linear_operator_full_matrix as linop_full_lib
-from tensorflow.contrib.linalg.python.ops import linear_operator_identity as linop_identity_lib
-from tensorflow.contrib.linalg.python.ops import linear_operator_tril as linop_tril_lib
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
@@ -37,6 +33,10 @@ from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops.distributions import categorical as categorical_lib
 from tensorflow.python.ops.distributions import distribution as distribution_lib
+from tensorflow.python.ops.linalg import linear_operator_diag as linop_diag_lib
+from tensorflow.python.ops.linalg import linear_operator_full_matrix as linop_full_lib
+from tensorflow.python.ops.linalg import linear_operator_identity as linop_identity_lib
+from tensorflow.python.ops.linalg import linear_operator_lower_triangular as linop_tril_lib
 
 static_value = distribution_util.static_value
 
@@ -185,7 +185,7 @@ class VectorDiffeomixture(distribution_lib.Distribution):
 
   ```python
   ds = tf.contrib.distributions
-  la = tf.contrib.linalg
+  la = tf.linalg
 
   # Create two batches of VectorDiffeomixtures, one with mix_loc=[0.] and
   # another with mix_loc=[1]. In both cases, `K=2` and the affine
@@ -772,8 +772,8 @@ def linop_scale(w, op):
           is_non_singular=op.is_non_singular,
           is_self_adjoint=op.is_self_adjoint,
           is_positive_definite=op.is_positive_definite)
-    if isinstance(op, linop_tril_lib.LinearOperatorTriL):
-      return linop_tril_lib.LinearOperatorTriL(
+    if isinstance(op, linop_tril_lib.LinearOperatorLowerTriangular):
+      return linop_tril_lib.LinearOperatorLowerTriangular(
           tril=w[..., array_ops.newaxis, array_ops.newaxis] * op.to_dense(),
           is_non_singular=op.is_non_singular,
           is_self_adjoint=op.is_self_adjoint,
diff --git a/tensorflow/contrib/distributions/python/ops/vector_exponential_diag.py b/tensorflow/contrib/distributions/python/ops/vector_exponential_diag.py
index c88572e17f..356d78b67a 100644
--- a/tensorflow/contrib/distributions/python/ops/vector_exponential_diag.py
+++ b/tensorflow/contrib/distributions/python/ops/vector_exponential_diag.py
@@ -90,7 +90,7 @@ class VectorExponentialDiag(
 
   ```python
   ds = tf.contrib.distributions
-  la = tf.contrib.linalg
+  la = tf.linalg
 
   # Initialize a single 2-variate VectorExponential, supported on
   # {(x, y) in R^2 : x > 0, y > 0}.
diff --git a/tensorflow/contrib/distributions/python/ops/vector_exponential_linear_operator.py b/tensorflow/contrib/distributions/python/ops/vector_exponential_linear_operator.py
index 7123165417..b313a851b3 100644
--- a/tensorflow/contrib/distributions/python/ops/vector_exponential_linear_operator.py
+++ b/tensorflow/contrib/distributions/python/ops/vector_exponential_linear_operator.py
@@ -18,7 +18,6 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.contrib import linalg
 from tensorflow.contrib.distributions.python.ops import bijectors
 from tensorflow.contrib.distributions.python.ops import distribution_util
 from tensorflow.python.framework import ops
@@ -26,6 +25,7 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops.distributions import exponential
 from tensorflow.python.ops.distributions import transformed_distribution
+from tensorflow.python.ops.linalg import linalg
 
 __all__ = ["VectorExponentialLinearOperator"]
 
@@ -108,7 +108,7 @@ class VectorExponentialLinearOperator(
 
   ```python
   ds = tf.contrib.distributions
-  la = tf.contrib.linalg
+  la = tf.linalg
 
   # Initialize a single 2-variate VectorExponential, supported on
   # {(x, y) in R^2 : x > 0, y > 0}.
@@ -247,7 +247,7 @@ class VectorExponentialLinearOperator(
   def _variance(self):
     if distribution_util.is_diagonal_scale(self.scale):
       return math_ops.square(self.scale.diag_part())
-    elif (isinstance(self.scale, linalg.LinearOperatorUDVHUpdate) and
+    elif (isinstance(self.scale, linalg.LinearOperatorLowRankUpdate) and
           self.scale.is_self_adjoint):
       return array_ops.matrix_diag_part(
           self.scale.matmul(self.scale.to_dense()))
@@ -258,7 +258,7 @@ class VectorExponentialLinearOperator(
   def _stddev(self):
     if distribution_util.is_diagonal_scale(self.scale):
       return math_ops.abs(self.scale.diag_part())
-    elif (isinstance(self.scale, linalg.LinearOperatorUDVHUpdate) and
+    elif (isinstance(self.scale, linalg.LinearOperatorLowRankUpdate) and
           self.scale.is_self_adjoint):
       return math_ops.sqrt(
           array_ops.matrix_diag_part(self.scale.matmul(self.scale.to_dense())))
diff --git a/tensorflow/contrib/distributions/python/ops/vector_laplace_linear_operator.py b/tensorflow/contrib/distributions/python/ops/vector_laplace_linear_operator.py
index fdee57695e..c7abdbb4ca 100644
--- a/tensorflow/contrib/distributions/python/ops/vector_laplace_linear_operator.py
+++ b/tensorflow/contrib/distributions/python/ops/vector_laplace_linear_operator.py
@@ -20,7 +20,6 @@ from __future__ import print_function
 
 import numpy as np
 
-from tensorflow.contrib import linalg
 from tensorflow.contrib.distributions.python.ops import bijectors
 from tensorflow.contrib.distributions.python.ops import distribution_util
 from tensorflow.python.framework import ops
@@ -28,6 +27,7 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops.distributions import laplace
 from tensorflow.python.ops.distributions import transformed_distribution
+from tensorflow.python.ops.linalg import linalg
 
 
 __all__ = [
@@ -110,7 +110,7 @@ class VectorLaplaceLinearOperator(
 
   ```python
   ds = tf.contrib.distributions
-  la = tf.contrib.linalg
+  la = tf.linalg
 
   # Initialize a single 3-variate VectorLaplace with some desired covariance.
   mu = [1., 2, 3]
@@ -126,7 +126,7 @@ class VectorLaplaceLinearOperator(
   # Divide scale by sqrt(2) so that the final covariance will be what we want.
   vla = ds.VectorLaplaceLinearOperator(
       loc=mu,
-      scale=la.LinearOperatorTriL(scale / tf.sqrt(2)))
+      scale=la.LinearOperatorLowerTriangular(scale / tf.sqrt(2)))
 
   # Covariance agrees with cholesky(cov) parameterization.
   vla.covariance().eval()
@@ -271,8 +271,8 @@ class VectorLaplaceLinearOperator(
   def _variance(self):
     if distribution_util.is_diagonal_scale(self.scale):
       return 2. * math_ops.square(self.scale.diag_part())
-    elif (isinstance(self.scale, linalg.LinearOperatorUDVHUpdate)
-          and self.scale.is_self_adjoint):
+    elif (isinstance(self.scale, linalg.LinearOperatorLowRankUpdate) and
+          self.scale.is_self_adjoint):
       return array_ops.matrix_diag_part(
           2. * self.scale.matmul(self.scale.to_dense()))
     else:
@@ -282,8 +282,8 @@ class VectorLaplaceLinearOperator(
   def _stddev(self):
     if distribution_util.is_diagonal_scale(self.scale):
       return np.sqrt(2) * math_ops.abs(self.scale.diag_part())
-    elif (isinstance(self.scale, linalg.LinearOperatorUDVHUpdate)
-          and self.scale.is_self_adjoint):
+    elif (isinstance(self.scale, linalg.LinearOperatorLowRankUpdate) and
+          self.scale.is_self_adjoint):
       return np.sqrt(2) * math_ops.sqrt(array_ops.matrix_diag_part(
           self.scale.matmul(self.scale.to_dense())))
     else:
diff --git a/tensorflow/contrib/distributions/python/ops/wishart.py b/tensorflow/contrib/distributions/python/ops/wishart.py
index 9d30ce6719..e4ac65012b 100644
--- a/tensorflow/contrib/distributions/python/ops/wishart.py
+++ b/tensorflow/contrib/distributions/python/ops/wishart.py
@@ -251,8 +251,8 @@ class _WishartLinearOperator(distribution.Distribution):
 
     # Complexity: O(nbM) where M is the complexity of the operator solving a
     # vector system. E.g., for LinearOperatorDiag, each matmul is O(k**2), so
-    # this complexity is O(nbk**2). For LinearOperatorTriL, each matmul is
-    # O(k^3) so this step has complexity O(nbk^3).
+    # this complexity is O(nbk**2). For LinearOperatorLowerTriangular,
+    # each matmul is O(k^3) so this step has complexity O(nbk^3).
     x = self.scale_operator.matmul(x)
 
     # Undo make batch-op ready.
@@ -307,8 +307,8 @@ class _WishartLinearOperator(distribution.Distribution):
 
     # Complexity: O(nbM*k) where M is the complexity of the operator solving
     # a vector system. E.g., for LinearOperatorDiag, each solve is O(k), so
-    # this complexity is O(nbk**2). For LinearOperatorTriL, each solve is
-    # O(k**2) so this step has complexity O(nbk^3).
+    # this complexity is O(nbk**2). For LinearOperatorLowerTriangular,
+    # each solve is O(k**2) so this step has complexity O(nbk^3).
     scale_sqrt_inv_x_sqrt = self.scale_operator.solve(
         scale_sqrt_inv_x_sqrt)
 
@@ -544,7 +544,7 @@ class WishartCholesky(_WishartLinearOperator):
 
       super(WishartCholesky, self).__init__(
           df=df,
-          scale_operator=linalg.LinearOperatorTriL(
+          scale_operator=linalg.LinearOperatorLowerTriangular(
               tril=scale,
               is_non_singular=True,
               is_positive_definite=True,
@@ -655,7 +655,7 @@ class WishartFull(_WishartLinearOperator):
         ] if validate_args else [], chol)
     super(WishartFull, self).__init__(
         df=df,
-        scale_operator=linalg.LinearOperatorTriL(
+        scale_operator=linalg.LinearOperatorLowerTriangular(
             tril=chol,
             is_non_singular=True,
             is_positive_definite=True,
diff --git a/tensorflow/contrib/linalg/BUILD b/tensorflow/contrib/linalg/BUILD
index 810a3d34ee..734bac17dc 100644
--- a/tensorflow/contrib/linalg/BUILD
+++ b/tensorflow/contrib/linalg/BUILD
@@ -10,152 +10,7 @@ exports_files(["LICENSE"])
 
 package(default_visibility = ["//tensorflow:__subpackages__"])
 
-load("//tensorflow:tensorflow.bzl", "cuda_py_tests")
-
-cuda_py_tests(
-    name = "linear_operator_test",
-    size = "small",
-    srcs = ["python/kernel_tests/linear_operator_test.py"],
-    additional_deps = [
-        ":linalg_py",
-        "//third_party/py/numpy",
-        "//tensorflow/python:array_ops",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:framework_for_generated_wrappers",
-        "//tensorflow/python:framework_test_lib",
-        "//tensorflow/python:math_ops",
-        "//tensorflow/python:platform_test",
-    ],
-)
-
-cuda_py_tests(
-    name = "linear_operator_addition_test",
-    size = "small",
-    srcs = ["python/kernel_tests/linear_operator_addition_test.py"],
-    additional_deps = [
-        ":linalg_py",
-        "//third_party/py/numpy",
-        "//tensorflow/python:array_ops",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:framework",
-        "//tensorflow/python:framework_for_generated_wrappers",
-        "//tensorflow/python:framework_test_lib",
-        "//tensorflow/python:math_ops",
-        "//tensorflow/python:platform_test",
-    ],
-)
-
-cuda_py_tests(
-    name = "linear_operator_composition_test",
-    size = "medium",
-    srcs = ["python/kernel_tests/linear_operator_composition_test.py"],
-    additional_deps = [
-        ":linalg_py",
-        "//third_party/py/numpy",
-        "//tensorflow/python:array_ops",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:framework",
-        "//tensorflow/python:framework_for_generated_wrappers",
-        "//tensorflow/python:framework_test_lib",
-        "//tensorflow/python:math_ops",
-        "//tensorflow/python:platform_test",
-    ],
-    tags = ["noasan"],  # times out b/63678675
-)
-
-cuda_py_tests(
-    name = "linear_operator_diag_test",
-    size = "medium",
-    srcs = ["python/kernel_tests/linear_operator_diag_test.py"],
-    additional_deps = [
-        ":linalg_py",
-        "//tensorflow/python:array_ops",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:framework",
-        "//tensorflow/python:framework_test_lib",
-        "//tensorflow/python:linalg_ops",
-        "//tensorflow/python:math_ops",
-        "//tensorflow/python:platform_test",
-        "//tensorflow/python:random_ops",
-    ],
-)
-
-cuda_py_tests(
-    name = "linear_operator_identity_test",
-    size = "medium",
-    srcs = ["python/kernel_tests/linear_operator_identity_test.py"],
-    additional_deps = [
-        ":linalg_py",
-        "//tensorflow/python:array_ops",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:framework",
-        "//tensorflow/python:framework_test_lib",
-        "//tensorflow/python:linalg_ops",
-        "//tensorflow/python:platform_test",
-        "//tensorflow/python:random_ops",
-    ],
-)
-
-cuda_py_tests(
-    name = "linear_operator_full_matrix_test",
-    size = "medium",
-    srcs = ["python/kernel_tests/linear_operator_full_matrix_test.py"],
-    additional_deps = [
-        ":linalg_py",
-        "//tensorflow/python:array_ops",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:framework",
-        "//tensorflow/python:framework_for_generated_wrappers",
-        "//tensorflow/python:framework_test_lib",
-        "//tensorflow/python:platform_test",
-    ],
-)
-
-cuda_py_tests(
-    name = "linear_operator_tril_test",
-    size = "medium",
-    srcs = ["python/kernel_tests/linear_operator_tril_test.py"],
-    additional_deps = [
-        ":linalg_py",
-        "//tensorflow/python:array_ops",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:framework",
-        "//tensorflow/python:framework_for_generated_wrappers",
-        "//tensorflow/python:framework_test_lib",
-        "//tensorflow/python:platform_test",
-    ],
-)
-
-cuda_py_tests(
-    name = "linear_operator_udvh_update_test",
-    size = "medium",
-    srcs = ["python/kernel_tests/linear_operator_udvh_update_test.py"],
-    additional_deps = [
-        ":linalg_py",
-        "//tensorflow/python:array_ops",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:framework",
-        "//tensorflow/python:framework_for_generated_wrappers",
-        "//tensorflow/python:framework_test_lib",
-        "//tensorflow/python:platform_test",
-    ],
-    shard_count = 5,
-)
-
-cuda_py_tests(
-    name = "linear_operator_util_test",
-    size = "medium",
-    srcs = ["python/kernel_tests/linear_operator_util_test.py"],
-    additional_deps = [
-        ":linalg_py",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:framework",
-        "//tensorflow/python:framework_for_generated_wrappers",
-        "//tensorflow/python:framework_test_lib",
-        "//tensorflow/python:math_ops",
-        "//tensorflow/python:platform_test",
-    ],
-)
+load("//tensorflow:tensorflow.bzl", "cuda_py_test")
 
 py_library(
     name = "linalg_py",
@@ -176,11 +31,29 @@ py_library(
         "//tensorflow/python:random_seed",
         "//tensorflow/python:tensor_util",
         "//tensorflow/python:util",
+        "//tensorflow/python/ops/linalg",
         "//third_party/py/numpy",
         "@six_archive//:six",
     ],
 )
 
+cuda_py_test(
+    name = "linear_operator_addition_test",
+    size = "small",
+    srcs = ["python/kernel_tests/linear_operator_addition_test.py"],
+    additional_deps = [
+        ":linalg_py",
+        "//third_party/py/numpy",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework",
+        "//tensorflow/python:framework_for_generated_wrappers",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:platform_test",
+    ],
+)
+
 filegroup(
     name = "all_files",
     srcs = glob(
diff --git a/tensorflow/contrib/linalg/__init__.py b/tensorflow/contrib/linalg/__init__.py
index 44421a6b7d..4720692c33 100644
--- a/tensorflow/contrib/linalg/__init__.py
+++ b/tensorflow/contrib/linalg/__init__.py
@@ -21,8 +21,8 @@ See the @{$python/contrib.linalg} guide.
 @@LinearOperatorIdentity
 @@LinearOperatorScaledIdentity
 @@LinearOperatorFullMatrix
-@@LinearOperatorTriL
-@@LinearOperatorUDVHUpdate
+@@LinearOperatorLowerTriangular
+@@LinearOperatorLowRankUpdate
 @@LinearOperatorComposition
 @@add_operators
 
@@ -33,14 +33,14 @@ from __future__ import print_function
 
 # pylint: disable=unused-import,wildcard-import,line-too-long,g-importing-member
 
-from tensorflow.contrib.linalg.python.ops.linear_operator import *
 from tensorflow.contrib.linalg.python.ops.linear_operator_addition import *
-from tensorflow.contrib.linalg.python.ops.linear_operator_composition import *
-from tensorflow.contrib.linalg.python.ops.linear_operator_diag import *
-from tensorflow.contrib.linalg.python.ops.linear_operator_full_matrix import *
-from tensorflow.contrib.linalg.python.ops.linear_operator_identity import *
-from tensorflow.contrib.linalg.python.ops.linear_operator_tril import *
-from tensorflow.contrib.linalg.python.ops.linear_operator_udvh_update import *
+from tensorflow.python.ops.linalg.linear_operator import *
+from tensorflow.python.ops.linalg.linear_operator_composition import *
+from tensorflow.python.ops.linalg.linear_operator_diag import *
+from tensorflow.python.ops.linalg.linear_operator_full_matrix import *
+from tensorflow.python.ops.linalg.linear_operator_identity import *
+from tensorflow.python.ops.linalg.linear_operator_low_rank_update import *
+from tensorflow.python.ops.linalg.linear_operator_lower_triangular import *
 
 # pylint: enable=unused-import,wildcard-import,line-too-long,g-importing-member
 
diff --git a/tensorflow/contrib/linalg/python/kernel_tests/linear_operator_addition_test.py b/tensorflow/contrib/linalg/python/kernel_tests/linear_operator_addition_test.py
index 4746484755..6a72df6dfd 100644
--- a/tensorflow/contrib/linalg/python/kernel_tests/linear_operator_addition_test.py
+++ b/tensorflow/contrib/linalg/python/kernel_tests/linear_operator_addition_test.py
@@ -19,10 +19,10 @@ from __future__ import print_function
 
 import numpy as np
 
-from tensorflow.contrib import linalg as linalg_lib
 from tensorflow.contrib.linalg.python.ops import linear_operator_addition
 from tensorflow.python.framework import random_seed
 from tensorflow.python.ops import linalg_ops
+from tensorflow.python.ops.linalg import linalg as linalg_lib
 from tensorflow.python.platform import test
 
 linalg = linalg_lib
@@ -114,7 +114,7 @@ class LinearOperatorAdditionCorrectnessTest(test.TestCase):
   def test_diag_tril_diag(self):
     op1 = linalg.LinearOperatorDiag(
         [1., 1.], is_non_singular=True, name="diag_a")
-    op2 = linalg.LinearOperatorTriL(
+    op2 = linalg.LinearOperatorLowerTriangular(
         [[2., 0.], [0., 2.]],
         is_self_adjoint=True,
         is_non_singular=True,
@@ -125,7 +125,7 @@ class LinearOperatorAdditionCorrectnessTest(test.TestCase):
       op_sum = add_operators([op1, op2, op3])
       self.assertEqual(1, len(op_sum))
       op = op_sum[0]
-      self.assertTrue(isinstance(op, linalg_lib.LinearOperatorTriL))
+      self.assertTrue(isinstance(op, linalg_lib.LinearOperatorLowerTriangular))
       self.assertAllClose([[6., 0.], [0., 6.]], op.to_dense().eval())
 
       # The diag operators will be self-adjoint (because real and diagonal).
@@ -140,7 +140,8 @@ class LinearOperatorAdditionCorrectnessTest(test.TestCase):
     op0 = linalg.LinearOperatorFullMatrix(
         [[-1., -1.], [-1., -1.]], name="matrix")
     op1 = linalg.LinearOperatorDiag([1., 1.], name="diag_a")
-    op2 = linalg.LinearOperatorTriL([[2., 0.], [1.5, 2.]], name="tril")
+    op2 = linalg.LinearOperatorLowerTriangular(
+        [[2., 0.], [1.5, 2.]], name="tril")
     op3 = linalg.LinearOperatorDiag([3., 3.], name="diag_b")
     with self.test_session():
       op_sum = add_operators([op0, op1, op2, op3], operator_name="my_operator")
@@ -189,7 +190,7 @@ class LinearOperatorOrderOfAdditionTest(test.TestCase):
   def test_tier_1_additions_done_by_tier_1(self):
     diag1 = linalg.LinearOperatorDiag([1.])
     diag2 = linalg.LinearOperatorDiag([1.])
-    tril = linalg.LinearOperatorTriL([[1.]])
+    tril = linalg.LinearOperatorLowerTriangular([[1.]])
     addition_tiers = [
         [linear_operator_addition._AddAndReturnDiag()],
         [linear_operator_addition._AddAndReturnTriL()],
@@ -199,12 +200,12 @@ class LinearOperatorOrderOfAdditionTest(test.TestCase):
     # _BadAdder) was never reached.
     op_sum = add_operators([diag1, diag2, tril], addition_tiers=addition_tiers)
     self.assertEqual(1, len(op_sum))
-    self.assertTrue(isinstance(op_sum[0], linalg.LinearOperatorTriL))
+    self.assertTrue(isinstance(op_sum[0], linalg.LinearOperatorLowerTriangular))
 
   def test_tier_1_additions_done_by_tier_1_with_order_flipped(self):
     diag1 = linalg.LinearOperatorDiag([1.])
     diag2 = linalg.LinearOperatorDiag([1.])
-    tril = linalg.LinearOperatorTriL([[1.]])
+    tril = linalg.LinearOperatorLowerTriangular([[1.]])
     addition_tiers = [
         [linear_operator_addition._AddAndReturnTriL()],
         [linear_operator_addition._AddAndReturnDiag()],
@@ -216,12 +217,12 @@ class LinearOperatorOrderOfAdditionTest(test.TestCase):
     # Tier 2 was never used (therefore, _BadAdder didn't raise).
     op_sum = add_operators([diag1, diag2, tril], addition_tiers=addition_tiers)
     self.assertEqual(1, len(op_sum))
-    self.assertTrue(isinstance(op_sum[0], linalg.LinearOperatorTriL))
+    self.assertTrue(isinstance(op_sum[0], linalg.LinearOperatorLowerTriangular))
 
   def test_cannot_add_everything_so_return_more_than_one_operator(self):
     diag1 = linalg.LinearOperatorDiag([1.])
     diag2 = linalg.LinearOperatorDiag([2.])
-    tril5 = linalg.LinearOperatorTriL([[5.]])
+    tril5 = linalg.LinearOperatorLowerTriangular([[5.]])
     addition_tiers = [
         [linear_operator_addition._AddAndReturnDiag()],
     ]
@@ -237,7 +238,7 @@ class LinearOperatorOrderOfAdditionTest(test.TestCase):
         if isinstance(op, linalg.LinearOperatorDiag):
           found_diag = True
           self.assertAllClose([[3.]], op.to_dense().eval())
-        if isinstance(op, linalg.LinearOperatorTriL):
+        if isinstance(op, linalg.LinearOperatorLowerTriangular):
           found_tril = True
           self.assertAllClose([[5.]], op.to_dense().eval())
       self.assertTrue(found_diag and found_tril)
@@ -245,7 +246,7 @@ class LinearOperatorOrderOfAdditionTest(test.TestCase):
   def test_intermediate_tier_is_not_skipped(self):
     diag1 = linalg.LinearOperatorDiag([1.])
     diag2 = linalg.LinearOperatorDiag([1.])
-    tril = linalg.LinearOperatorTriL([[1.]])
+    tril = linalg.LinearOperatorLowerTriangular([[1.]])
     addition_tiers = [
         [linear_operator_addition._AddAndReturnDiag()],
         [_BadAdder()],
@@ -369,14 +370,14 @@ class AddAndReturnTriLTest(test.TestCase):
 
   def test_diag_plus_tril(self):
     diag = linalg.LinearOperatorDiag([1., 2.])
-    tril = linalg.LinearOperatorTriL([[10., 0.], [30., 0.]])
+    tril = linalg.LinearOperatorLowerTriangular([[10., 0.], [30., 0.]])
     hints = linear_operator_addition._Hints(
         is_positive_definite=True, is_non_singular=True)
 
     self.assertTrue(self._adder.can_add(diag, diag))
     self.assertTrue(self._adder.can_add(diag, tril))
     operator = self._adder.add(diag, tril, "my_operator", hints)
-    self.assertTrue(isinstance(operator, linalg.LinearOperatorTriL))
+    self.assertTrue(isinstance(operator, linalg.LinearOperatorLowerTriangular))
 
     with self.test_session():
       self.assertAllClose([[11., 0.], [30., 2.]], operator.to_dense().eval())
diff --git a/tensorflow/contrib/linalg/python/ops/linear_operator_addition.py b/tensorflow/contrib/linalg/python/ops/linear_operator_addition.py
index 16c4c6e6d6..86130a2c07 100644
--- a/tensorflow/contrib/linalg/python/ops/linear_operator_addition.py
+++ b/tensorflow/contrib/linalg/python/ops/linear_operator_addition.py
@@ -22,14 +22,14 @@ import abc
 
 import six
 
-from tensorflow.contrib.linalg.python.ops import linear_operator
-from tensorflow.contrib.linalg.python.ops import linear_operator_diag
-from tensorflow.contrib.linalg.python.ops import linear_operator_full_matrix
-from tensorflow.contrib.linalg.python.ops import linear_operator_identity
-from tensorflow.contrib.linalg.python.ops import linear_operator_tril
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import check_ops
+from tensorflow.python.ops.linalg import linear_operator
+from tensorflow.python.ops.linalg import linear_operator_diag
+from tensorflow.python.ops.linalg import linear_operator_full_matrix
+from tensorflow.python.ops.linalg import linear_operator_identity
+from tensorflow.python.ops.linalg import linear_operator_lower_triangular
 
 __all__ = []
 
@@ -347,7 +347,7 @@ class _AddAndReturnTriL(_Adder):
     else:
       op_add_to_tensor, op_other = op2, op1
 
-    return linear_operator_tril.LinearOperatorTriL(
+    return linear_operator_lower_triangular.LinearOperatorLowerTriangular(
         tril=op_add_to_tensor.add_to_tensor(op_other.to_dense()),
         is_non_singular=hints.is_non_singular,
         is_self_adjoint=hints.is_self_adjoint,
@@ -397,7 +397,8 @@ def _type(operator):
   """Returns the type name constant (e.g. _TRIL) for operator."""
   if isinstance(operator, linear_operator_diag.LinearOperatorDiag):
     return _DIAG
-  if isinstance(operator, linear_operator_tril.LinearOperatorTriL):
+  if isinstance(operator,
+                linear_operator_lower_triangular.LinearOperatorLowerTriangular):
     return _TRIL
   if isinstance(operator, linear_operator_full_matrix.LinearOperatorFullMatrix):
     return _MATRIX
diff --git a/tensorflow/docs_src/api_guides/python/contrib.linalg.md b/tensorflow/docs_src/api_guides/python/contrib.linalg.md
index 5f1db6c6af..c0cb2b195c 100644
--- a/tensorflow/docs_src/api_guides/python/contrib.linalg.md
+++ b/tensorflow/docs_src/api_guides/python/contrib.linalg.md
@@ -22,8 +22,8 @@ Subclasses of `LinearOperator` provide a access to common methods on a
 *   @{tf.contrib.linalg.LinearOperatorIdentity}
 *   @{tf.contrib.linalg.LinearOperatorScaledIdentity}
 *   @{tf.contrib.linalg.LinearOperatorFullMatrix}
-*   @{tf.contrib.linalg.LinearOperatorTriL}
-*   @{tf.contrib.linalg.LinearOperatorUDVHUpdate}
+*   @{tf.contrib.linalg.LinearOperatorLowerTriangular}
+*   @{tf.contrib.linalg.LinearOperatorLowRankUpdate}
 
 ### Transformations and Combinations of operators
 
diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 1099611f37..b9b85909a3 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -70,7 +70,6 @@ py_library(
         ":io_ops",
         ":layers",
         ":lib",
-        ":linalg_ns",
         ":math_ops",
         ":metrics",
         ":nn",
@@ -104,6 +103,7 @@ py_library(
         "//tensorflow/python/keras",
         "//tensorflow/python/ops/losses",
         "//tensorflow/python/ops/distributions",
+        "//tensorflow/python/ops/linalg",
         "//tensorflow/python/profiler",
         "//tensorflow/python/saved_model",
     ] + if_not_windows([
@@ -1710,21 +1710,6 @@ py_library(
     ],
 )
 
-py_library(
-    name = "linalg_ns",
-    srcs = [
-        "ops/linalg_impl.py",
-        "ops/linalg_ns.py",
-    ],
-    srcs_version = "PY2AND3",
-    deps = [
-        ":array_ops",
-        ":linalg_ops",
-        ":math_ops",
-        ":special_math_ops",
-    ],
-)
-
 py_library(
     name = "linalg_grad",
     srcs = ["ops/linalg_grad.py"],
@@ -2223,6 +2208,7 @@ py_library(
         ":variable_scope",
         ":variables",
         "//tensorflow/python/ops/distributions",
+        "//tensorflow/python/ops/linalg",
     ],
 )
 
diff --git a/tensorflow/python/__init__.py b/tensorflow/python/__init__.py
index f21f1f822c..8d9c5de9ad 100644
--- a/tensorflow/python/__init__.py
+++ b/tensorflow/python/__init__.py
@@ -73,7 +73,6 @@ from tensorflow.python.ops.standard_ops import *
 
 # Namespaces
 from tensorflow.python.ops import initializers_ns as initializers
-from tensorflow.python.ops import linalg_ns as linalg
 
 # pylint: enable=wildcard-import
 
@@ -90,6 +89,7 @@ from tensorflow.python.ops import nn
 from tensorflow.python.ops import sets
 from tensorflow.python.ops import spectral_ops as spectral
 from tensorflow.python.ops.distributions import distributions
+from tensorflow.python.ops.linalg import linalg
 from tensorflow.python.ops.losses import losses
 from tensorflow.python.profiler import profiler
 from tensorflow.python.saved_model import saved_model
diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD
index 206c6a5692..b8a7444f45 100644
--- a/tensorflow/python/kernel_tests/BUILD
+++ b/tensorflow/python/kernel_tests/BUILD
@@ -1485,8 +1485,8 @@ cuda_py_test(
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:framework_for_generated_wrappers",
         "//tensorflow/python:linalg_ops",
-        "//tensorflow/python:linalg_ns",
         "//tensorflow/python:math_ops",
+        "//tensorflow/python/ops/linalg",
     ],
     tags = ["no_windows_gpu"],
 )
diff --git a/tensorflow/python/kernel_tests/linalg/BUILD b/tensorflow/python/kernel_tests/linalg/BUILD
new file mode 100644
index 0000000000..4e18eaa4e8
--- /dev/null
+++ b/tensorflow/python/kernel_tests/linalg/BUILD
@@ -0,0 +1,149 @@
+# Tests of TensorFlow kernels written using the Python API.
+
+package(
+    default_visibility = ["//tensorflow:internal"],
+)
+
+licenses(["notice"])  # Apache 2.0
+
+load("//tensorflow:tensorflow.bzl", "cuda_py_test")
+
+cuda_py_test(
+    name = "linear_operator_test",
+    size = "small",
+    srcs = ["linear_operator_test.py"],
+    additional_deps = [
+        "//tensorflow/python/ops/linalg",
+        "//third_party/py/numpy",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework_for_generated_wrappers",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:platform_test",
+    ],
+)
+
+cuda_py_test(
+    name = "linear_operator_composition_test",
+    size = "medium",
+    srcs = ["linear_operator_composition_test.py"],
+    additional_deps = [
+        "//tensorflow/python/ops/linalg",
+        "//third_party/py/numpy",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework",
+        "//tensorflow/python:framework_for_generated_wrappers",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:platform_test",
+    ],
+    tags = ["noasan"],  # times out b/63678675
+)
+
+cuda_py_test(
+    name = "linear_operator_diag_test",
+    size = "medium",
+    srcs = ["linear_operator_diag_test.py"],
+    additional_deps = [
+        "//tensorflow/python/ops/linalg",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:linalg_ops",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:platform_test",
+        "//tensorflow/python:random_ops",
+    ],
+)
+
+cuda_py_test(
+    name = "linear_operator_identity_test",
+    size = "medium",
+    srcs = ["linear_operator_identity_test.py"],
+    additional_deps = [
+        "//tensorflow/python/ops/linalg",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:linalg_ops",
+        "//tensorflow/python:platform_test",
+        "//tensorflow/python:random_ops",
+    ],
+)
+
+cuda_py_test(
+    name = "linear_operator_full_matrix_test",
+    size = "medium",
+    srcs = ["linear_operator_full_matrix_test.py"],
+    additional_deps = [
+        "//tensorflow/python/ops/linalg",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework",
+        "//tensorflow/python:framework_for_generated_wrappers",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:platform_test",
+    ],
+)
+
+cuda_py_test(
+    name = "linear_operator_lower_triangular_test",
+    size = "medium",
+    srcs = ["linear_operator_lower_triangular_test.py"],
+    additional_deps = [
+        "//tensorflow/python/ops/linalg",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework",
+        "//tensorflow/python:framework_for_generated_wrappers",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:platform_test",
+    ],
+)
+
+cuda_py_test(
+    name = "linear_operator_low_rank_update_test",
+    size = "medium",
+    srcs = ["linear_operator_low_rank_update_test.py"],
+    additional_deps = [
+        "//tensorflow/python/ops/linalg",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework",
+        "//tensorflow/python:framework_for_generated_wrappers",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:platform_test",
+    ],
+    shard_count = 5,
+)
+
+cuda_py_test(
+    name = "linear_operator_util_test",
+    size = "medium",
+    srcs = ["linear_operator_util_test.py"],
+    additional_deps = [
+        "//tensorflow/python/ops/linalg",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework",
+        "//tensorflow/python:framework_for_generated_wrappers",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:platform_test",
+    ],
+)
+
+filegroup(
+    name = "all_files",
+    srcs = glob(
+        ["**/*"],
+        exclude = [
+            "**/METADATA",
+            "**/OWNERS",
+        ],
+    ),
+    visibility = ["//tensorflow:__subpackages__"],
+)
diff --git a/tensorflow/python/kernel_tests/linalg/__init__.py b/tensorflow/python/kernel_tests/linalg/__init__.py
new file mode 100644
index 0000000000..1f6cb4a020
--- /dev/null
+++ b/tensorflow/python/kernel_tests/linalg/__init__.py
@@ -0,0 +1,18 @@
+# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Kernel tests for tf.linalg."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
diff --git a/tensorflow/contrib/linalg/python/kernel_tests/linear_operator_composition_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_composition_test.py
similarity index 98%
rename from tensorflow/contrib/linalg/python/kernel_tests/linear_operator_composition_test.py
rename to tensorflow/python/kernel_tests/linalg/linear_operator_composition_test.py
index e2a7f5fbe1..4d79365dbe 100644
--- a/tensorflow/contrib/linalg/python/kernel_tests/linear_operator_composition_test.py
+++ b/tensorflow/python/kernel_tests/linalg/linear_operator_composition_test.py
@@ -19,13 +19,13 @@ from __future__ import print_function
 
 import numpy as np
 
-from tensorflow.contrib import linalg as linalg_lib
-from tensorflow.contrib.linalg.python.ops import linear_operator_test_util
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import random_seed
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
+from tensorflow.python.ops.linalg import linalg as linalg_lib
+from tensorflow.python.ops.linalg import linear_operator_test_util
 from tensorflow.python.platform import test
 
 linalg = linalg_lib
diff --git a/tensorflow/contrib/linalg/python/kernel_tests/linear_operator_diag_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_diag_test.py
similarity index 97%
rename from tensorflow/contrib/linalg/python/kernel_tests/linear_operator_diag_test.py
rename to tensorflow/python/kernel_tests/linalg/linear_operator_diag_test.py
index 397bfa2215..343d158498 100644
--- a/tensorflow/contrib/linalg/python/kernel_tests/linear_operator_diag_test.py
+++ b/tensorflow/python/kernel_tests/linalg/linear_operator_diag_test.py
@@ -17,13 +17,13 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.contrib import linalg as linalg_lib
-from tensorflow.contrib.linalg.python.ops import linear_operator_test_util
 from tensorflow.python.framework import random_seed
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import linalg_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import random_ops
+from tensorflow.python.ops.linalg import linalg as linalg_lib
+from tensorflow.python.ops.linalg import linear_operator_test_util
 from tensorflow.python.platform import test
 
 linalg = linalg_lib
diff --git a/tensorflow/contrib/linalg/python/kernel_tests/linear_operator_full_matrix_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_full_matrix_test.py
similarity index 98%
rename from tensorflow/contrib/linalg/python/kernel_tests/linear_operator_full_matrix_test.py
rename to tensorflow/python/kernel_tests/linalg/linear_operator_full_matrix_test.py
index 528bc3ed12..50d6f524e9 100644
--- a/tensorflow/contrib/linalg/python/kernel_tests/linear_operator_full_matrix_test.py
+++ b/tensorflow/python/kernel_tests/linalg/linear_operator_full_matrix_test.py
@@ -19,13 +19,13 @@ from __future__ import print_function
 
 import numpy as np
 
-from tensorflow.contrib import linalg as linalg_lib
-from tensorflow.contrib.linalg.python.ops import linear_operator_test_util
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import random_seed
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
+from tensorflow.python.ops.linalg import linalg as linalg_lib
+from tensorflow.python.ops.linalg import linear_operator_test_util
 from tensorflow.python.platform import test
 
 linalg = linalg_lib
diff --git a/tensorflow/contrib/linalg/python/kernel_tests/linear_operator_identity_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_identity_test.py
similarity index 99%
rename from tensorflow/contrib/linalg/python/kernel_tests/linear_operator_identity_test.py
rename to tensorflow/python/kernel_tests/linalg/linear_operator_identity_test.py
index 5faf2c432b..6d63570768 100644
--- a/tensorflow/contrib/linalg/python/kernel_tests/linear_operator_identity_test.py
+++ b/tensorflow/python/kernel_tests/linalg/linear_operator_identity_test.py
@@ -19,13 +19,13 @@ from __future__ import print_function
 
 import numpy as np
 
-from tensorflow.contrib import linalg as linalg_lib
-from tensorflow.contrib.linalg.python.ops import linear_operator_test_util
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import random_seed
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import linalg_ops
 from tensorflow.python.ops import random_ops
+from tensorflow.python.ops.linalg import linalg as linalg_lib
+from tensorflow.python.ops.linalg import linear_operator_test_util
 from tensorflow.python.platform import test
 
 
diff --git a/tensorflow/contrib/linalg/python/kernel_tests/linear_operator_udvh_update_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_low_rank_update_test.py
similarity index 88%
rename from tensorflow/contrib/linalg/python/kernel_tests/linear_operator_udvh_update_test.py
rename to tensorflow/python/kernel_tests/linalg/linear_operator_low_rank_update_test.py
index f28213096b..d3a47da946 100644
--- a/tensorflow/contrib/linalg/python/kernel_tests/linear_operator_udvh_update_test.py
+++ b/tensorflow/python/kernel_tests/linalg/linear_operator_low_rank_update_test.py
@@ -19,12 +19,12 @@ from __future__ import print_function
 
 import numpy as np
 
-from tensorflow.contrib import linalg as linalg_lib
-from tensorflow.contrib.linalg.python.ops import linear_operator_test_util
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import random_seed
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
+from tensorflow.python.ops.linalg import linalg as linalg_lib
+from tensorflow.python.ops.linalg import linear_operator_test_util
 from tensorflow.python.platform import test
 
 linalg = linalg_lib
@@ -32,7 +32,7 @@ random_seed.set_random_seed(23)
 rng = np.random.RandomState(0)
 
 
-class BaseLinearOperatorUDVHUpdatetest(object):
+class BaseLinearOperatorLowRankUpdatetest(object):
   """Base test for this type of operator."""
 
   # Subclasses should set these attributes to either True or False.
@@ -51,7 +51,7 @@ class BaseLinearOperatorUDVHUpdatetest(object):
   @property
   def _dtypes_to_test(self):
     # TODO(langmore) Test complex types once cholesky works with them.
-    # See comment in LinearOperatorUDVHUpdate.__init__.
+    # See comment in LinearOperatorLowRankUpdate.__init__.
     return [dtypes.float32, dtypes.float64]
 
   @property
@@ -108,7 +108,7 @@ class BaseLinearOperatorUDVHUpdatetest(object):
       base_operator = linalg.LinearOperatorDiag(
           base_diag_ph, is_positive_definite=True)
 
-      operator = linalg.LinearOperatorUDVHUpdate(
+      operator = linalg.LinearOperatorLowRankUpdate(
           base_operator,
           u=u_ph,
           v=v_ph if self._use_v else None,
@@ -122,7 +122,7 @@ class BaseLinearOperatorUDVHUpdatetest(object):
     else:
       base_operator = linalg.LinearOperatorDiag(
           base_diag, is_positive_definite=True)
-      operator = linalg.LinearOperatorUDVHUpdate(
+      operator = linalg.LinearOperatorLowRankUpdate(
           base_operator,
           u,
           v=v if self._use_v else None,
@@ -164,8 +164,8 @@ class BaseLinearOperatorUDVHUpdatetest(object):
     return operator, mat, feed_dict
 
 
-class LinearOperatorUDVHUpdatetestWithDiagUseCholesky(
-    BaseLinearOperatorUDVHUpdatetest,
+class LinearOperatorLowRankUpdatetestWithDiagUseCholesky(
+    BaseLinearOperatorLowRankUpdatetest,
     linear_operator_test_util.SquareLinearOperatorDerivedClassTest):
   """A = L + UDU^H, D > 0, L > 0 ==> A > 0 and we can use a Cholesky."""
 
@@ -182,8 +182,8 @@ class LinearOperatorUDVHUpdatetestWithDiagUseCholesky(
     self._rtol[dtypes.float64] = 1e-10
 
 
-class LinearOperatorUDVHUpdatetestWithDiagCannotUseCholesky(
-    BaseLinearOperatorUDVHUpdatetest,
+class LinearOperatorLowRankUpdatetestWithDiagCannotUseCholesky(
+    BaseLinearOperatorLowRankUpdatetest,
     linear_operator_test_util.SquareLinearOperatorDerivedClassTest):
   """A = L + UDU^H, D !> 0, L > 0 ==> A !> 0 and we cannot use a Cholesky."""
 
@@ -201,8 +201,8 @@ class LinearOperatorUDVHUpdatetestWithDiagCannotUseCholesky(
     self._rtol[dtypes.float64] = 1e-9
 
 
-class LinearOperatorUDVHUpdatetestNoDiagUseCholesky(
-    BaseLinearOperatorUDVHUpdatetest,
+class LinearOperatorLowRankUpdatetestNoDiagUseCholesky(
+    BaseLinearOperatorLowRankUpdatetest,
     linear_operator_test_util.SquareLinearOperatorDerivedClassTest):
   """A = L + UU^H, L > 0 ==> A > 0 and we can use a Cholesky."""
 
@@ -219,8 +219,8 @@ class LinearOperatorUDVHUpdatetestNoDiagUseCholesky(
     self._rtol[dtypes.float64] = 1e-10
 
 
-class LinearOperatorUDVHUpdatetestNoDiagCannotUseCholesky(
-    BaseLinearOperatorUDVHUpdatetest,
+class LinearOperatorLowRankUpdatetestNoDiagCannotUseCholesky(
+    BaseLinearOperatorLowRankUpdatetest,
     linear_operator_test_util.SquareLinearOperatorDerivedClassTest):
   """A = L + UV^H, L > 0 ==> A is not symmetric and we cannot use a Cholesky."""
 
@@ -238,8 +238,8 @@ class LinearOperatorUDVHUpdatetestNoDiagCannotUseCholesky(
     self._rtol[dtypes.float64] = 1e-9
 
 
-class LinearOperatorUDVHUpdatetestWithDiagNotSquare(
-    BaseLinearOperatorUDVHUpdatetest,
+class LinearOperatorLowRankUpdatetestWithDiagNotSquare(
+    BaseLinearOperatorLowRankUpdatetest,
     linear_operator_test_util.NonSquareLinearOperatorDerivedClassTest):
   """A = L + UDU^H, D > 0, L > 0 ==> A > 0 and we can use a Cholesky."""
 
@@ -248,7 +248,7 @@ class LinearOperatorUDVHUpdatetestWithDiagNotSquare(
   _use_v = True
 
 
-class LinearOpearatorUDVHUpdateBroadcastsShape(test.TestCase):
+class LinearOpearatorLowRankUpdateBroadcastsShape(test.TestCase):
   """Test that the operator's shape is the broadcast of arguments."""
 
   def test_static_shape_broadcasts_up_from_operator_to_other_args(self):
@@ -256,8 +256,7 @@ class LinearOpearatorUDVHUpdateBroadcastsShape(test.TestCase):
     u = array_ops.ones(shape=[2, 3, 2])
     diag = array_ops.ones(shape=[2, 2])
 
-    operator = linalg.LinearOperatorUDVHUpdate(
-        base_operator, u, diag)
+    operator = linalg.LinearOperatorLowRankUpdate(base_operator, u, diag)
 
     # domain_dimension is 3
     self.assertAllEqual([2, 3, 3], operator.shape)
@@ -272,7 +271,7 @@ class LinearOpearatorUDVHUpdateBroadcastsShape(test.TestCase):
     u_shape_ph = array_ops.placeholder(dtypes.int32)
     u = array_ops.ones(shape=u_shape_ph)
 
-    operator = linalg.LinearOperatorUDVHUpdate(base_operator, u)
+    operator = linalg.LinearOperatorLowRankUpdate(base_operator, u)
 
     feed_dict = {
         num_rows_ph: 3,
@@ -290,34 +289,34 @@ class LinearOpearatorUDVHUpdateBroadcastsShape(test.TestCase):
     u = rng.rand(5, 3, 2)
     v = rng.rand(4, 3, 2)
     with self.assertRaisesRegexp(ValueError, "Incompatible shapes"):
-      linalg.LinearOperatorUDVHUpdate(base_operator, u=u, v=v)
+      linalg.LinearOperatorLowRankUpdate(base_operator, u=u, v=v)
 
   def test_u_and_base_operator_incompatible_batch_shape_raises(self):
     base_operator = linalg.LinearOperatorIdentity(
         num_rows=3, batch_shape=[4], dtype=np.float64)
     u = rng.rand(5, 3, 2)
     with self.assertRaisesRegexp(ValueError, "Incompatible shapes"):
-      linalg.LinearOperatorUDVHUpdate(base_operator, u=u)
+      linalg.LinearOperatorLowRankUpdate(base_operator, u=u)
 
   def test_u_and_base_operator_incompatible_domain_dimension(self):
     base_operator = linalg.LinearOperatorIdentity(num_rows=3, dtype=np.float64)
     u = rng.rand(5, 4, 2)
     with self.assertRaisesRegexp(ValueError, "not compatible"):
-      linalg.LinearOperatorUDVHUpdate(base_operator, u=u)
+      linalg.LinearOperatorLowRankUpdate(base_operator, u=u)
 
   def test_u_and_diag_incompatible_low_rank_raises(self):
     base_operator = linalg.LinearOperatorIdentity(num_rows=3, dtype=np.float64)
     u = rng.rand(5, 3, 2)
     diag = rng.rand(5, 4)  # Last dimension should be 2
     with self.assertRaisesRegexp(ValueError, "not compatible"):
-      linalg.LinearOperatorUDVHUpdate(base_operator, u=u, diag_update=diag)
+      linalg.LinearOperatorLowRankUpdate(base_operator, u=u, diag_update=diag)
 
   def test_diag_incompatible_batch_shape_raises(self):
     base_operator = linalg.LinearOperatorIdentity(num_rows=3, dtype=np.float64)
     u = rng.rand(5, 3, 2)
     diag = rng.rand(4, 2)  # First dimension should be 5
     with self.assertRaisesRegexp(ValueError, "Incompatible shapes"):
-      linalg.LinearOperatorUDVHUpdate(base_operator, u=u, diag_update=diag)
+      linalg.LinearOperatorLowRankUpdate(base_operator, u=u, diag_update=diag)
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/contrib/linalg/python/kernel_tests/linear_operator_tril_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_lower_triangular_test.py
similarity index 86%
rename from tensorflow/contrib/linalg/python/kernel_tests/linear_operator_tril_test.py
rename to tensorflow/python/kernel_tests/linalg/linear_operator_lower_triangular_test.py
index 9f5f2856f1..db3918f998 100644
--- a/tensorflow/contrib/linalg/python/kernel_tests/linear_operator_tril_test.py
+++ b/tensorflow/python/kernel_tests/linalg/linear_operator_lower_triangular_test.py
@@ -17,18 +17,18 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.contrib import linalg as linalg_lib
-from tensorflow.contrib.linalg.python.ops import linear_operator_test_util
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import random_seed
 from tensorflow.python.ops import array_ops
+from tensorflow.python.ops.linalg import linalg as linalg_lib
+from tensorflow.python.ops.linalg import linear_operator_test_util
 from tensorflow.python.platform import test
 
 linalg = linalg_lib
 random_seed.set_random_seed(23)
 
 
-class LinearOperatorTriLTest(
+class LinearOperatorLowerTriangularTest(
     linear_operator_test_util.SquareLinearOperatorDerivedClassTest):
   """Most tests done in the base class LinearOperatorDerivedClassTest."""
 
@@ -50,10 +50,10 @@ class LinearOperatorTriLTest(
       # tril is random and we want the same value used for both mat and
       # feed_dict.
       tril = tril.eval()
-      operator = linalg.LinearOperatorTriL(tril_ph)
+      operator = linalg.LinearOperatorLowerTriangular(tril_ph)
       feed_dict = {tril_ph: tril}
     else:
-      operator = linalg.LinearOperatorTriL(tril)
+      operator = linalg.LinearOperatorLowerTriangular(tril)
       feed_dict = None
 
     mat = array_ops.matrix_band_part(tril, -1, 0)
@@ -64,14 +64,14 @@ class LinearOperatorTriLTest(
     # Singlular matrix with one positive eigenvalue and one zero eigenvalue.
     with self.test_session():
       tril = [[1., 0.], [1., 0.]]
-      operator = linalg.LinearOperatorTriL(tril)
+      operator = linalg.LinearOperatorLowerTriangular(tril)
       with self.assertRaisesOpError("Singular operator"):
         operator.assert_non_singular().run()
 
   def test_is_x_flags(self):
     # Matrix with two positive eigenvalues.
     tril = [[1., 0.], [1., 1.]]
-    operator = linalg.LinearOperatorTriL(
+    operator = linalg.LinearOperatorLowerTriangular(
         tril,
         is_positive_definite=True,
         is_non_singular=True,
@@ -82,7 +82,7 @@ class LinearOperatorTriLTest(
 
   def test_tril_must_have_at_least_two_dims_or_raises(self):
     with self.assertRaisesRegexp(ValueError, "at least 2 dimensions"):
-      linalg.LinearOperatorTriL([1.])
+      linalg.LinearOperatorLowerTriangular([1.])
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/contrib/linalg/python/kernel_tests/linear_operator_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_test.py
similarity index 99%
rename from tensorflow/contrib/linalg/python/kernel_tests/linear_operator_test.py
rename to tensorflow/python/kernel_tests/linalg/linear_operator_test.py
index 78a4822c17..8e9f0150a2 100644
--- a/tensorflow/contrib/linalg/python/kernel_tests/linear_operator_test.py
+++ b/tensorflow/python/kernel_tests/linalg/linear_operator_test.py
@@ -17,7 +17,7 @@ from __future__ import division
 from __future__ import print_function
 
 import numpy as np
-from tensorflow.contrib import linalg as linalg_lib
+
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
@@ -25,6 +25,7 @@ from tensorflow.python.framework import tensor_shape
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import linalg_ops
 from tensorflow.python.ops import math_ops
+from tensorflow.python.ops.linalg import linalg as linalg_lib
 from tensorflow.python.platform import test
 
 linalg = linalg_lib
diff --git a/tensorflow/contrib/linalg/python/kernel_tests/linear_operator_util_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_util_test.py
similarity index 98%
rename from tensorflow/contrib/linalg/python/kernel_tests/linear_operator_util_test.py
rename to tensorflow/python/kernel_tests/linalg/linear_operator_util_test.py
index f047f4b978..ca3c8647db 100644
--- a/tensorflow/contrib/linalg/python/kernel_tests/linear_operator_util_test.py
+++ b/tensorflow/python/kernel_tests/linalg/linear_operator_util_test.py
@@ -19,16 +19,14 @@ from __future__ import print_function
 
 import numpy as np
 
-from tensorflow.contrib import linalg as linalg_lib
-from tensorflow.contrib.linalg.python.ops import linear_operator_util
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import random_seed
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
+from tensorflow.python.ops.linalg import linear_operator_util
 from tensorflow.python.platform import test
 
-linalg = linalg_lib
 random_seed.set_random_seed(23)
 rng = np.random.RandomState(0)
 
diff --git a/tensorflow/python/kernel_tests/linalg_ops_test.py b/tensorflow/python/kernel_tests/linalg_ops_test.py
index be15e49f60..8bb583ce1b 100644
--- a/tensorflow/python/kernel_tests/linalg_ops_test.py
+++ b/tensorflow/python/kernel_tests/linalg_ops_test.py
@@ -22,9 +22,9 @@ import numpy as np
 
 from tensorflow.python.framework import dtypes
 from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import linalg_ns as linalg
 from tensorflow.python.ops import linalg_ops
 from tensorflow.python.ops import math_ops
+from tensorflow.python.ops.linalg import linalg
 from tensorflow.python.platform import test
 
 
diff --git a/tensorflow/python/ops/distributions/util.py b/tensorflow/python/ops/distributions/util.py
index 089ec49f06..f261d996b5 100644
--- a/tensorflow/python/ops/distributions/util.py
+++ b/tensorflow/python/ops/distributions/util.py
@@ -523,8 +523,8 @@ def matrix_diag_transform(matrix, transform=None, name=None):
   # valid Cholesky factor.
   chol = matrix_diag_transform(matrix, transform=tf.nn.softplus)
 
-  # LinearOperatorTriL ignores the upper triangle.
-  operator = LinearOperatorTriL(chol)
+  # LinearOperatorLowerTriangular ignores the upper triangle.
+  operator = LinearOperatorLowerTriangular(chol)
   ```
 
   Example of heteroskedastic 2-D linear regression.
diff --git a/tensorflow/python/ops/linalg/BUILD b/tensorflow/python/ops/linalg/BUILD
new file mode 100644
index 0000000000..a36e0a4be1
--- /dev/null
+++ b/tensorflow/python/ops/linalg/BUILD
@@ -0,0 +1,38 @@
+package(
+    default_visibility = ["//tensorflow:internal"],
+)
+
+licenses(["notice"])  # Apache 2.0
+
+py_library(
+    name = "linalg",
+    srcs = glob(["*.py"]),
+    srcs_version = "PY2AND3",
+    deps = [
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:check_ops",
+        "//tensorflow/python:control_flow_ops",
+        "//tensorflow/python:framework_for_generated_wrappers",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:nn",
+        "//tensorflow/python:nn_ops",
+        "//tensorflow/python:random_ops",
+        "//tensorflow/python:special_math_ops",
+        "//tensorflow/python:tensor_util",
+        "//tensorflow/python:util",
+        "//third_party/py/numpy",
+        "@six_archive//:six",
+    ],
+)
+
+filegroup(
+    name = "all_files",
+    srcs = glob(
+        ["**/*"],
+        exclude = [
+            "**/METADATA",
+            "**/OWNERS",
+        ],
+    ),
+    visibility = ["//tensorflow:__subpackages__"],
+)
diff --git a/tensorflow/python/ops/__init__.py b/tensorflow/python/ops/linalg/__init__.py
similarity index 100%
rename from tensorflow/python/ops/__init__.py
rename to tensorflow/python/ops/linalg/__init__.py
diff --git a/tensorflow/python/ops/linalg_ns.py b/tensorflow/python/ops/linalg/linalg.py
similarity index 78%
rename from tensorflow/python/ops/linalg_ns.py
rename to tensorflow/python/ops/linalg/linalg.py
index 92e488a6ce..02ceb65e2a 100644
--- a/tensorflow/python/ops/linalg_ns.py
+++ b/tensorflow/python/ops/linalg/linalg.py
@@ -12,10 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Public API for tf.linalg namespace.
-
-@@logdet
-"""
+"""Public API for tf.linalg namespace."""
 
 from __future__ import absolute_import
 from __future__ import division
@@ -29,7 +26,14 @@ from tensorflow.python.ops import special_math_ops
 
 # go/tf-wildcard-import
 # pylint: disable=wildcard-import,unused-import
-from tensorflow.python.ops.linalg_impl import *
+from tensorflow.python.ops.linalg.linalg_impl import *
+from tensorflow.python.ops.linalg.linear_operator import *
+from tensorflow.python.ops.linalg.linear_operator_composition import *
+from tensorflow.python.ops.linalg.linear_operator_diag import *
+from tensorflow.python.ops.linalg.linear_operator_full_matrix import *
+from tensorflow.python.ops.linalg.linear_operator_identity import *
+from tensorflow.python.ops.linalg.linear_operator_low_rank_update import *
+from tensorflow.python.ops.linalg.linear_operator_lower_triangular import *
 # pylint: enable=wildcard-import
 
 # Linear algebra ops.
diff --git a/tensorflow/python/ops/linalg_impl.py b/tensorflow/python/ops/linalg/linalg_impl.py
similarity index 100%
rename from tensorflow/python/ops/linalg_impl.py
rename to tensorflow/python/ops/linalg/linalg_impl.py
diff --git a/tensorflow/contrib/linalg/python/ops/linear_operator.py b/tensorflow/python/ops/linalg/linear_operator.py
similarity index 99%
rename from tensorflow/contrib/linalg/python/ops/linear_operator.py
rename to tensorflow/python/ops/linalg/linear_operator.py
index 91c0938e39..17c338ec75 100644
--- a/tensorflow/contrib/linalg/python/ops/linear_operator.py
+++ b/tensorflow/python/ops/linalg/linear_operator.py
@@ -23,13 +23,13 @@ import contextlib
 
 import numpy as np
 
-from tensorflow.contrib import framework as contrib_framework
-from tensorflow.contrib.linalg.python.ops import linear_operator_util
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import tensor_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import check_ops
 from tensorflow.python.ops import linalg_ops
 from tensorflow.python.ops import math_ops
+from tensorflow.python.ops.linalg import linear_operator_util
 from tensorflow.python.platform import tf_logging as logging
 
 __all__ = ["LinearOperator"]
@@ -192,7 +192,7 @@ class LinearOperator(object):
 
     graph_parents = [] if graph_parents is None else graph_parents
     for i, t in enumerate(graph_parents):
-      if t is None or not contrib_framework.is_tensor(t):
+      if t is None or not tensor_util.is_tensor(t):
         raise ValueError("Graph parent item %d is not a Tensor; %s." % (i, t))
     self._dtype = dtype
     self._graph_parents = graph_parents
diff --git a/tensorflow/contrib/linalg/python/ops/linear_operator_composition.py b/tensorflow/python/ops/linalg/linear_operator_composition.py
similarity index 99%
rename from tensorflow/contrib/linalg/python/ops/linear_operator_composition.py
rename to tensorflow/python/ops/linalg/linear_operator_composition.py
index 0a71a73a9c..14411291d4 100644
--- a/tensorflow/contrib/linalg/python/ops/linear_operator_composition.py
+++ b/tensorflow/python/ops/linalg/linear_operator_composition.py
@@ -18,13 +18,13 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.contrib.linalg.python.ops import linear_operator
 from tensorflow.python.framework import common_shapes
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import check_ops
+from tensorflow.python.ops.linalg import linear_operator
 
 __all__ = ["LinearOperatorComposition"]
 
diff --git a/tensorflow/contrib/linalg/python/ops/linear_operator_diag.py b/tensorflow/python/ops/linalg/linear_operator_diag.py
similarity index 98%
rename from tensorflow/contrib/linalg/python/ops/linear_operator_diag.py
rename to tensorflow/python/ops/linalg/linear_operator_diag.py
index 29184483bf..e1558a351d 100644
--- a/tensorflow/contrib/linalg/python/ops/linear_operator_diag.py
+++ b/tensorflow/python/ops/linalg/linear_operator_diag.py
@@ -18,13 +18,13 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.contrib.linalg.python.ops import linear_operator
-from tensorflow.contrib.linalg.python.ops import linear_operator_util
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import check_ops
 from tensorflow.python.ops import math_ops
+from tensorflow.python.ops.linalg import linear_operator
+from tensorflow.python.ops.linalg import linear_operator_util
 
 __all__ = ["LinearOperatorDiag",]
 
diff --git a/tensorflow/contrib/linalg/python/ops/linear_operator_full_matrix.py b/tensorflow/python/ops/linalg/linear_operator_full_matrix.py
similarity index 98%
rename from tensorflow/contrib/linalg/python/ops/linear_operator_full_matrix.py
rename to tensorflow/python/ops/linalg/linear_operator_full_matrix.py
index 52b40eaf8d..dd4c7cb041 100644
--- a/tensorflow/contrib/linalg/python/ops/linear_operator_full_matrix.py
+++ b/tensorflow/python/ops/linalg/linear_operator_full_matrix.py
@@ -18,11 +18,11 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.contrib.linalg.python.ops import linear_operator
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
+from tensorflow.python.ops.linalg import linear_operator
 
 __all__ = ["LinearOperatorFullMatrix"]
 
diff --git a/tensorflow/contrib/linalg/python/ops/linear_operator_identity.py b/tensorflow/python/ops/linalg/linear_operator_identity.py
similarity index 99%
rename from tensorflow/contrib/linalg/python/ops/linear_operator_identity.py
rename to tensorflow/python/ops/linalg/linear_operator_identity.py
index b9ac90ff33..18bd2f9f6d 100644
--- a/tensorflow/contrib/linalg/python/ops/linear_operator_identity.py
+++ b/tensorflow/python/ops/linalg/linear_operator_identity.py
@@ -20,8 +20,6 @@ from __future__ import print_function
 
 import numpy as np
 
-from tensorflow.contrib.linalg.python.ops import linear_operator
-from tensorflow.contrib.linalg.python.ops import linear_operator_util
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
@@ -30,6 +28,8 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import check_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import math_ops
+from tensorflow.python.ops.linalg import linear_operator
+from tensorflow.python.ops.linalg import linear_operator_util
 
 __all__ = [
     "LinearOperatorIdentity",
diff --git a/tensorflow/contrib/linalg/python/ops/linear_operator_udvh_update.py b/tensorflow/python/ops/linalg/linear_operator_low_rank_update.py
similarity index 95%
rename from tensorflow/contrib/linalg/python/ops/linear_operator_udvh_update.py
rename to tensorflow/python/ops/linalg/linear_operator_low_rank_update.py
index 9c9c359574..ad3bb2efa9 100644
--- a/tensorflow/contrib/linalg/python/ops/linear_operator_udvh_update.py
+++ b/tensorflow/python/ops/linalg/linear_operator_low_rank_update.py
@@ -18,20 +18,22 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.contrib.framework.python.framework import tensor_util as contrib_tensor_util
-from tensorflow.contrib.linalg.python.ops import linear_operator
-from tensorflow.contrib.linalg.python.ops import linear_operator_diag
-from tensorflow.contrib.linalg.python.ops import linear_operator_identity
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import check_ops
 from tensorflow.python.ops import linalg_ops
 from tensorflow.python.ops import math_ops
+from tensorflow.python.ops.linalg import linear_operator
+from tensorflow.python.ops.linalg import linear_operator_diag
+from tensorflow.python.ops.linalg import linear_operator_identity
 
-__all__ = ["LinearOperatorUDVHUpdate",]
+__all__ = [
+    "LinearOperatorLowRankUpdate",
+]
 
 
-class LinearOperatorUDVHUpdate(linear_operator.LinearOperator):
+class LinearOperatorLowRankUpdate(linear_operator.LinearOperator):
   """Perturb a `LinearOperator` with a rank `K` update.
 
   This operator acts like a [batch] matrix `A` with shape
@@ -39,7 +41,7 @@ class LinearOperatorUDVHUpdate(linear_operator.LinearOperator):
   batch member.  For every batch index `(i1,...,ib)`, `A[i1,...,ib, : :]` is
   an `M x N` matrix.
 
-  `LinearOperatorUDVHUpdate` represents `A = L + U D V^H`, where
+  `LinearOperatorLowRankUpdate` represents `A = L + U D V^H`, where
 
   ```
   L, is a LinearOperator representing [batch] M x N matrices
@@ -65,7 +67,7 @@ class LinearOperatorUDVHUpdate(linear_operator.LinearOperator):
       is_positive_definite=True)
 
   # Perturb with a rank 2 perturbation
-  operator = LinearOperatorUDVHUpdate(
+  operator = LinearOperatorLowRankUpdate(
       operator=diag_operator,
       u=[[1., 2.], [-1., 3.], [0., 0.]],
       diag_update=[11., 12.],
@@ -94,7 +96,7 @@ class LinearOperatorUDVHUpdate(linear_operator.LinearOperator):
 
   ### Performance
 
-  Suppose `operator` is a `LinearOperatorUDVHUpdate` of shape `[M, N]`,
+  Suppose `operator` is a `LinearOperatorLowRankUpdate` of shape `[M, N]`,
   made from a rank `K` update of `base_operator` which performs `.matmul(x)` on
   `x` having `x.shape = [N, R]` with `O(L_matmul*N*R)` complexity (and similarly
   for `solve`, `determinant`.  Then, if `x.shape = [N, R]`,
@@ -134,8 +136,8 @@ class LinearOperatorUDVHUpdate(linear_operator.LinearOperator):
                is_self_adjoint=None,
                is_positive_definite=None,
                is_square=None,
-               name="LinearOperatorUDVHUpdate"):
-    """Initialize a `LinearOperatorUDVHUpdate`.
+               name="LinearOperatorLowRankUpdate"):
+    """Initialize a `LinearOperatorLowRankUpdate`.
 
     This creates a `LinearOperator` of the form `A = L + U D V^H`, with
     `L` a `LinearOperator`, `U, V` both [batch] matrices, and `D` a [batch]
@@ -249,7 +251,7 @@ class LinearOperatorUDVHUpdate(linear_operator.LinearOperator):
           self.u, self._diag_update, self.v]
       graph_parents = [p for p in graph_parents if p is not None]
 
-      super(LinearOperatorUDVHUpdate, self).__init__(
+      super(LinearOperatorLowRankUpdate, self).__init__(
           dtype=self._base_operator.dtype,
           graph_parents=graph_parents,
           is_non_singular=is_non_singular,
@@ -262,8 +264,8 @@ class LinearOperatorUDVHUpdate(linear_operator.LinearOperator):
       self._set_diag_operators(diag_update, is_diag_update_positive)
       self._is_diag_update_positive = is_diag_update_positive
 
-      contrib_tensor_util.assert_same_float_dtype(
-          (base_operator, self.u, self.v, self._diag_update))
+      check_ops.assert_same_float_dtype((base_operator, self.u, self.v,
+                                         self._diag_update))
       self._check_shapes()
 
       # Pre-compute the so-called "capacitance" matrix
diff --git a/tensorflow/contrib/linalg/python/ops/linear_operator_tril.py b/tensorflow/python/ops/linalg/linear_operator_lower_triangular.py
similarity index 90%
rename from tensorflow/contrib/linalg/python/ops/linear_operator_tril.py
rename to tensorflow/python/ops/linalg/linear_operator_lower_triangular.py
index 22ccf6f131..4b074f5cec 100644
--- a/tensorflow/contrib/linalg/python/ops/linear_operator_tril.py
+++ b/tensorflow/python/ops/linalg/linear_operator_lower_triangular.py
@@ -18,18 +18,20 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.contrib.linalg.python.ops import linear_operator
-from tensorflow.contrib.linalg.python.ops import linear_operator_util
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import linalg_ops
 from tensorflow.python.ops import math_ops
+from tensorflow.python.ops.linalg import linear_operator
+from tensorflow.python.ops.linalg import linear_operator_util
 
-__all__ = ["LinearOperatorTriL",]
+__all__ = [
+    "LinearOperatorLowerTriangular",
+]
 
 
-class LinearOperatorTriL(linear_operator.LinearOperator):
+class LinearOperatorLowerTriangular(linear_operator.LinearOperator):
   """`LinearOperator` acting like a [batch] square lower triangular matrix.
 
   This operator acts like a [batch] lower triangular matrix `A` with shape
@@ -37,13 +39,14 @@ class LinearOperatorTriL(linear_operator.LinearOperator):
   batch member.  For every batch index `(i1,...,ib)`, `A[i1,...,ib, : :]` is
   an `N x N` matrix.
 
-  `LinearOperatorTriL` is initialized with a `Tensor` having dimensions
-  `[B1,...,Bb, N, N]`. The upper triangle of the last two dimensions is ignored.
+  `LinearOperatorLowerTriangular` is initialized with a `Tensor` having
+  dimensions `[B1,...,Bb, N, N]`. The upper triangle of the last two
+  dimensions is ignored.
 
   ```python
   # Create a 2 x 2 lower-triangular linear operator.
   tril = [[1., 2.], [3., 4.]]
-  operator = LinearOperatorTriL(tril)
+  operator = LinearOperatorLowerTriangular(tril)
 
   # The upper triangle is ignored.
   operator.to_dense()
@@ -62,7 +65,7 @@ class LinearOperatorTriL(linear_operator.LinearOperator):
 
   # Create a [2, 3] batch of 4 x 4 linear operators.
   tril = tf.random_normal(shape=[2, 3, 4, 4])
-  operator = LinearOperatorTriL(tril)
+  operator = LinearOperatorLowerTriangular(tril)
   ```
 
   #### Shape compatibility
@@ -77,7 +80,7 @@ class LinearOperatorTriL(linear_operator.LinearOperator):
 
   #### Performance
 
-  Suppose `operator` is a `LinearOperatorTriL` of shape `[N, N]`,
+  Suppose `operator` is a `LinearOperatorLowerTriangular` of shape `[N, N]`,
   and `x.shape = [N, R]`.  Then
 
   * `operator.matmul(x)` involves `N^2 * R` multiplications.
@@ -108,8 +111,8 @@ class LinearOperatorTriL(linear_operator.LinearOperator):
                is_self_adjoint=None,
                is_positive_definite=None,
                is_square=None,
-               name="LinearOperatorTriL"):
-    r"""Initialize a `LinearOperatorTriL`.
+               name="LinearOperatorLowerTriangular"):
+    r"""Initialize a `LinearOperatorLowerTriangular`.
 
     Args:
       tril:  Shape `[B1,...,Bb, N, N]` with `b >= 0`, `N >= 0`.
@@ -147,7 +150,7 @@ class LinearOperatorTriL(linear_operator.LinearOperator):
       self._tril = array_ops.matrix_band_part(tril, -1, 0)
       self._diag = array_ops.matrix_diag_part(self._tril)
 
-      super(LinearOperatorTriL, self).__init__(
+      super(LinearOperatorLowerTriangular, self).__init__(
           dtype=self._tril.dtype,
           graph_parents=[self._tril],
           is_non_singular=is_non_singular,
diff --git a/tensorflow/contrib/linalg/python/ops/linear_operator_test_util.py b/tensorflow/python/ops/linalg/linear_operator_test_util.py
similarity index 99%
rename from tensorflow/contrib/linalg/python/ops/linear_operator_test_util.py
rename to tensorflow/python/ops/linalg/linear_operator_test_util.py
index af14f34600..b86cb6d84d 100644
--- a/tensorflow/contrib/linalg/python/ops/linear_operator_test_util.py
+++ b/tensorflow/python/ops/linalg/linear_operator_test_util.py
@@ -22,16 +22,16 @@ import abc
 import numpy as np
 import six
 
-from tensorflow.contrib.framework.python.framework import tensor_util as contrib_tensor_util
-from tensorflow.contrib.linalg.python.ops import linear_operator_util
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import random_seed
 from tensorflow.python.framework import tensor_shape
+from tensorflow.python.framework import tensor_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import linalg_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import random_ops
+from tensorflow.python.ops.linalg import linear_operator_util
 from tensorflow.python.platform import test
 
 
@@ -428,7 +428,7 @@ def random_positive_definite_matrix(shape, dtype, force_well_conditioned=False):
     `Tensor` with desired shape and dtype.
   """
   dtype = dtypes.as_dtype(dtype)
-  if not contrib_tensor_util.is_tensor(shape):
+  if not tensor_util.is_tensor(shape):
     shape = tensor_shape.TensorShape(shape)
     # Matrix must be square.
     shape[-1].assert_is_compatible_with(shape[-2])
diff --git a/tensorflow/contrib/linalg/python/ops/linear_operator_util.py b/tensorflow/python/ops/linalg/linear_operator_util.py
similarity index 100%
rename from tensorflow/contrib/linalg/python/ops/linear_operator_util.py
rename to tensorflow/python/ops/linalg/linear_operator_util.py
diff --git a/tensorflow/tools/api/golden/tensorflow.linalg.-linear-operator-composition.__metaclass__.pbtxt b/tensorflow/tools/api/golden/tensorflow.linalg.-linear-operator-composition.__metaclass__.pbtxt
new file mode 100644
index 0000000000..1adbcb41ad
--- /dev/null
+++ b/tensorflow/tools/api/golden/tensorflow.linalg.-linear-operator-composition.__metaclass__.pbtxt
@@ -0,0 +1,14 @@
+path: "tensorflow.linalg.LinearOperatorComposition.__metaclass__"
+tf_class {
+  is_instance: "<class \'abc.ABCMeta\'>"
+  member_method {
+    name: "__init__"
+  }
+  member_method {
+    name: "mro"
+  }
+  member_method {
+    name: "register"
+    argspec: "args=[\'cls\', \'subclass\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/tensorflow.linalg.-linear-operator-composition.pbtxt b/tensorflow/tools/api/golden/tensorflow.linalg.-linear-operator-composition.pbtxt
new file mode 100644
index 0000000000..42d22bce42
--- /dev/null
+++ b/tensorflow/tools/api/golden/tensorflow.linalg.-linear-operator-composition.pbtxt
@@ -0,0 +1,134 @@
+path: "tensorflow.linalg.LinearOperatorComposition"
+tf_class {
+  is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator_composition.LinearOperatorComposition\'>"
+  is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator.LinearOperator\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "batch_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "domain_dimension"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "graph_parents"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "is_non_singular"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "is_positive_definite"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "is_self_adjoint"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "is_square"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "operators"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "range_dimension"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "tensor_rank"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'operators\', \'is_non_singular\', \'is_self_adjoint\', \'is_positive_definite\', \'is_square\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_to_tensor"
+    argspec: "args=[\'self\', \'x\', \'name\'], varargs=None, keywords=None, defaults=[\'add_to_tensor\'], "
+  }
+  member_method {
+    name: "assert_non_singular"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_non_singular\'], "
+  }
+  member_method {
+    name: "assert_positive_definite"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_positive_definite\'], "
+  }
+  member_method {
+    name: "assert_self_adjoint"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_self_adjoint\'], "
+  }
+  member_method {
+    name: "batch_shape_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'batch_shape_tensor\'], "
+  }
+  member_method {
+    name: "determinant"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'det\'], "
+  }
+  member_method {
+    name: "diag_part"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'diag_part\'], "
+  }
+  member_method {
+    name: "domain_dimension_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'domain_dimension_tensor\'], "
+  }
+  member_method {
+    name: "log_abs_determinant"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'log_abs_det\'], "
+  }
+  member_method {
+    name: "matmul"
+    argspec: "args=[\'self\', \'x\', \'adjoint\', \'adjoint_arg\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'False\', \'matmul\'], "
+  }
+  member_method {
+    name: "matvec"
+    argspec: "args=[\'self\', \'x\', \'adjoint\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'matvec\'], "
+  }
+  member_method {
+    name: "range_dimension_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'range_dimension_tensor\'], "
+  }
+  member_method {
+    name: "shape_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'shape_tensor\'], "
+  }
+  member_method {
+    name: "solve"
+    argspec: "args=[\'self\', \'rhs\', \'adjoint\', \'adjoint_arg\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'False\', \'solve\'], "
+  }
+  member_method {
+    name: "solvevec"
+    argspec: "args=[\'self\', \'rhs\', \'adjoint\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'solve\'], "
+  }
+  member_method {
+    name: "tensor_rank_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'tensor_rank_tensor\'], "
+  }
+  member_method {
+    name: "to_dense"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'to_dense\'], "
+  }
+  member_method {
+    name: "trace"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'trace\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/tensorflow.linalg.-linear-operator-diag.__metaclass__.pbtxt b/tensorflow/tools/api/golden/tensorflow.linalg.-linear-operator-diag.__metaclass__.pbtxt
new file mode 100644
index 0000000000..023d90ccdb
--- /dev/null
+++ b/tensorflow/tools/api/golden/tensorflow.linalg.-linear-operator-diag.__metaclass__.pbtxt
@@ -0,0 +1,14 @@
+path: "tensorflow.linalg.LinearOperatorDiag.__metaclass__"
+tf_class {
+  is_instance: "<class \'abc.ABCMeta\'>"
+  member_method {
+    name: "__init__"
+  }
+  member_method {
+    name: "mro"
+  }
+  member_method {
+    name: "register"
+    argspec: "args=[\'cls\', \'subclass\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/tensorflow.linalg.-linear-operator-diag.pbtxt b/tensorflow/tools/api/golden/tensorflow.linalg.-linear-operator-diag.pbtxt
new file mode 100644
index 0000000000..d6749fdcec
--- /dev/null
+++ b/tensorflow/tools/api/golden/tensorflow.linalg.-linear-operator-diag.pbtxt
@@ -0,0 +1,134 @@
+path: "tensorflow.linalg.LinearOperatorDiag"
+tf_class {
+  is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator_diag.LinearOperatorDiag\'>"
+  is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator.LinearOperator\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "batch_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "diag"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "domain_dimension"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "graph_parents"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "is_non_singular"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "is_positive_definite"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "is_self_adjoint"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "is_square"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "range_dimension"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "tensor_rank"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'diag\', \'is_non_singular\', \'is_self_adjoint\', \'is_positive_definite\', \'is_square\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'LinearOperatorDiag\'], "
+  }
+  member_method {
+    name: "add_to_tensor"
+    argspec: "args=[\'self\', \'x\', \'name\'], varargs=None, keywords=None, defaults=[\'add_to_tensor\'], "
+  }
+  member_method {
+    name: "assert_non_singular"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_non_singular\'], "
+  }
+  member_method {
+    name: "assert_positive_definite"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_positive_definite\'], "
+  }
+  member_method {
+    name: "assert_self_adjoint"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_self_adjoint\'], "
+  }
+  member_method {
+    name: "batch_shape_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'batch_shape_tensor\'], "
+  }
+  member_method {
+    name: "determinant"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'det\'], "
+  }
+  member_method {
+    name: "diag_part"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'diag_part\'], "
+  }
+  member_method {
+    name: "domain_dimension_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'domain_dimension_tensor\'], "
+  }
+  member_method {
+    name: "log_abs_determinant"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'log_abs_det\'], "
+  }
+  member_method {
+    name: "matmul"
+    argspec: "args=[\'self\', \'x\', \'adjoint\', \'adjoint_arg\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'False\', \'matmul\'], "
+  }
+  member_method {
+    name: "matvec"
+    argspec: "args=[\'self\', \'x\', \'adjoint\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'matvec\'], "
+  }
+  member_method {
+    name: "range_dimension_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'range_dimension_tensor\'], "
+  }
+  member_method {
+    name: "shape_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'shape_tensor\'], "
+  }
+  member_method {
+    name: "solve"
+    argspec: "args=[\'self\', \'rhs\', \'adjoint\', \'adjoint_arg\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'False\', \'solve\'], "
+  }
+  member_method {
+    name: "solvevec"
+    argspec: "args=[\'self\', \'rhs\', \'adjoint\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'solve\'], "
+  }
+  member_method {
+    name: "tensor_rank_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'tensor_rank_tensor\'], "
+  }
+  member_method {
+    name: "to_dense"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'to_dense\'], "
+  }
+  member_method {
+    name: "trace"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'trace\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/tensorflow.linalg.-linear-operator-full-matrix.__metaclass__.pbtxt b/tensorflow/tools/api/golden/tensorflow.linalg.-linear-operator-full-matrix.__metaclass__.pbtxt
new file mode 100644
index 0000000000..381072e76c
--- /dev/null
+++ b/tensorflow/tools/api/golden/tensorflow.linalg.-linear-operator-full-matrix.__metaclass__.pbtxt
@@ -0,0 +1,14 @@
+path: "tensorflow.linalg.LinearOperatorFullMatrix.__metaclass__"
+tf_class {
+  is_instance: "<class \'abc.ABCMeta\'>"
+  member_method {
+    name: "__init__"
+  }
+  member_method {
+    name: "mro"
+  }
+  member_method {
+    name: "register"
+    argspec: "args=[\'cls\', \'subclass\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/tensorflow.linalg.-linear-operator-full-matrix.pbtxt b/tensorflow/tools/api/golden/tensorflow.linalg.-linear-operator-full-matrix.pbtxt
new file mode 100644
index 0000000000..d9f363d133
--- /dev/null
+++ b/tensorflow/tools/api/golden/tensorflow.linalg.-linear-operator-full-matrix.pbtxt
@@ -0,0 +1,130 @@
+path: "tensorflow.linalg.LinearOperatorFullMatrix"
+tf_class {
+  is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator_full_matrix.LinearOperatorFullMatrix\'>"
+  is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator.LinearOperator\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "batch_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "domain_dimension"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "graph_parents"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "is_non_singular"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "is_positive_definite"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "is_self_adjoint"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "is_square"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "range_dimension"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "tensor_rank"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'matrix\', \'is_non_singular\', \'is_self_adjoint\', \'is_positive_definite\', \'is_square\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'LinearOperatorFullMatrix\'], "
+  }
+  member_method {
+    name: "add_to_tensor"
+    argspec: "args=[\'self\', \'x\', \'name\'], varargs=None, keywords=None, defaults=[\'add_to_tensor\'], "
+  }
+  member_method {
+    name: "assert_non_singular"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_non_singular\'], "
+  }
+  member_method {
+    name: "assert_positive_definite"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_positive_definite\'], "
+  }
+  member_method {
+    name: "assert_self_adjoint"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_self_adjoint\'], "
+  }
+  member_method {
+    name: "batch_shape_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'batch_shape_tensor\'], "
+  }
+  member_method {
+    name: "determinant"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'det\'], "
+  }
+  member_method {
+    name: "diag_part"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'diag_part\'], "
+  }
+  member_method {
+    name: "domain_dimension_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'domain_dimension_tensor\'], "
+  }
+  member_method {
+    name: "log_abs_determinant"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'log_abs_det\'], "
+  }
+  member_method {
+    name: "matmul"
+    argspec: "args=[\'self\', \'x\', \'adjoint\', \'adjoint_arg\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'False\', \'matmul\'], "
+  }
+  member_method {
+    name: "matvec"
+    argspec: "args=[\'self\', \'x\', \'adjoint\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'matvec\'], "
+  }
+  member_method {
+    name: "range_dimension_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'range_dimension_tensor\'], "
+  }
+  member_method {
+    name: "shape_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'shape_tensor\'], "
+  }
+  member_method {
+    name: "solve"
+    argspec: "args=[\'self\', \'rhs\', \'adjoint\', \'adjoint_arg\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'False\', \'solve\'], "
+  }
+  member_method {
+    name: "solvevec"
+    argspec: "args=[\'self\', \'rhs\', \'adjoint\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'solve\'], "
+  }
+  member_method {
+    name: "tensor_rank_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'tensor_rank_tensor\'], "
+  }
+  member_method {
+    name: "to_dense"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'to_dense\'], "
+  }
+  member_method {
+    name: "trace"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'trace\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/tensorflow.linalg.-linear-operator-identity.__metaclass__.pbtxt b/tensorflow/tools/api/golden/tensorflow.linalg.-linear-operator-identity.__metaclass__.pbtxt
new file mode 100644
index 0000000000..5d115b35fb
--- /dev/null
+++ b/tensorflow/tools/api/golden/tensorflow.linalg.-linear-operator-identity.__metaclass__.pbtxt
@@ -0,0 +1,14 @@
+path: "tensorflow.linalg.LinearOperatorIdentity.__metaclass__"
+tf_class {
+  is_instance: "<class \'abc.ABCMeta\'>"
+  member_method {
+    name: "__init__"
+  }
+  member_method {
+    name: "mro"
+  }
+  member_method {
+    name: "register"
+    argspec: "args=[\'cls\', \'subclass\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/tensorflow.linalg.-linear-operator-identity.pbtxt b/tensorflow/tools/api/golden/tensorflow.linalg.-linear-operator-identity.pbtxt
new file mode 100644
index 0000000000..aac7ee31ed
--- /dev/null
+++ b/tensorflow/tools/api/golden/tensorflow.linalg.-linear-operator-identity.pbtxt
@@ -0,0 +1,131 @@
+path: "tensorflow.linalg.LinearOperatorIdentity"
+tf_class {
+  is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator_identity.LinearOperatorIdentity\'>"
+  is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator_identity.BaseLinearOperatorIdentity\'>"
+  is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator.LinearOperator\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "batch_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "domain_dimension"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "graph_parents"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "is_non_singular"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "is_positive_definite"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "is_self_adjoint"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "is_square"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "range_dimension"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "tensor_rank"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'num_rows\', \'batch_shape\', \'dtype\', \'is_non_singular\', \'is_self_adjoint\', \'is_positive_definite\', \'is_square\', \'assert_proper_shapes\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'True\', \'True\', \'True\', \'True\', \'False\', \'LinearOperatorIdentity\'], "
+  }
+  member_method {
+    name: "add_to_tensor"
+    argspec: "args=[\'self\', \'mat\', \'name\'], varargs=None, keywords=None, defaults=[\'add_to_tensor\'], "
+  }
+  member_method {
+    name: "assert_non_singular"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_non_singular\'], "
+  }
+  member_method {
+    name: "assert_positive_definite"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_positive_definite\'], "
+  }
+  member_method {
+    name: "assert_self_adjoint"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_self_adjoint\'], "
+  }
+  member_method {
+    name: "batch_shape_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'batch_shape_tensor\'], "
+  }
+  member_method {
+    name: "determinant"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'det\'], "
+  }
+  member_method {
+    name: "diag_part"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'diag_part\'], "
+  }
+  member_method {
+    name: "domain_dimension_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'domain_dimension_tensor\'], "
+  }
+  member_method {
+    name: "log_abs_determinant"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'log_abs_det\'], "
+  }
+  member_method {
+    name: "matmul"
+    argspec: "args=[\'self\', \'x\', \'adjoint\', \'adjoint_arg\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'False\', \'matmul\'], "
+  }
+  member_method {
+    name: "matvec"
+    argspec: "args=[\'self\', \'x\', \'adjoint\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'matvec\'], "
+  }
+  member_method {
+    name: "range_dimension_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'range_dimension_tensor\'], "
+  }
+  member_method {
+    name: "shape_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'shape_tensor\'], "
+  }
+  member_method {
+    name: "solve"
+    argspec: "args=[\'self\', \'rhs\', \'adjoint\', \'adjoint_arg\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'False\', \'solve\'], "
+  }
+  member_method {
+    name: "solvevec"
+    argspec: "args=[\'self\', \'rhs\', \'adjoint\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'solve\'], "
+  }
+  member_method {
+    name: "tensor_rank_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'tensor_rank_tensor\'], "
+  }
+  member_method {
+    name: "to_dense"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'to_dense\'], "
+  }
+  member_method {
+    name: "trace"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'trace\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/tensorflow.linalg.-linear-operator-low-rank-update.__metaclass__.pbtxt b/tensorflow/tools/api/golden/tensorflow.linalg.-linear-operator-low-rank-update.__metaclass__.pbtxt
new file mode 100644
index 0000000000..1f0d33298a
--- /dev/null
+++ b/tensorflow/tools/api/golden/tensorflow.linalg.-linear-operator-low-rank-update.__metaclass__.pbtxt
@@ -0,0 +1,14 @@
+path: "tensorflow.linalg.LinearOperatorLowRankUpdate.__metaclass__"
+tf_class {
+  is_instance: "<class \'abc.ABCMeta\'>"
+  member_method {
+    name: "__init__"
+  }
+  member_method {
+    name: "mro"
+  }
+  member_method {
+    name: "register"
+    argspec: "args=[\'cls\', \'subclass\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/tensorflow.linalg.-linear-operator-low-rank-update.pbtxt b/tensorflow/tools/api/golden/tensorflow.linalg.-linear-operator-low-rank-update.pbtxt
new file mode 100644
index 0000000000..3ee800269e
--- /dev/null
+++ b/tensorflow/tools/api/golden/tensorflow.linalg.-linear-operator-low-rank-update.pbtxt
@@ -0,0 +1,154 @@
+path: "tensorflow.linalg.LinearOperatorLowRankUpdate"
+tf_class {
+  is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator_low_rank_update.LinearOperatorLowRankUpdate\'>"
+  is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator.LinearOperator\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "base_operator"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "batch_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "diag_operator"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "diag_update"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "domain_dimension"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "graph_parents"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "is_diag_update_positive"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "is_non_singular"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "is_positive_definite"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "is_self_adjoint"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "is_square"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "range_dimension"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "tensor_rank"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "u"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "v"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'base_operator\', \'u\', \'diag_update\', \'v\', \'is_diag_update_positive\', \'is_non_singular\', \'is_self_adjoint\', \'is_positive_definite\', \'is_square\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'LinearOperatorLowRankUpdate\'], "
+  }
+  member_method {
+    name: "add_to_tensor"
+    argspec: "args=[\'self\', \'x\', \'name\'], varargs=None, keywords=None, defaults=[\'add_to_tensor\'], "
+  }
+  member_method {
+    name: "assert_non_singular"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_non_singular\'], "
+  }
+  member_method {
+    name: "assert_positive_definite"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_positive_definite\'], "
+  }
+  member_method {
+    name: "assert_self_adjoint"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_self_adjoint\'], "
+  }
+  member_method {
+    name: "batch_shape_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'batch_shape_tensor\'], "
+  }
+  member_method {
+    name: "determinant"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'det\'], "
+  }
+  member_method {
+    name: "diag_part"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'diag_part\'], "
+  }
+  member_method {
+    name: "domain_dimension_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'domain_dimension_tensor\'], "
+  }
+  member_method {
+    name: "log_abs_determinant"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'log_abs_det\'], "
+  }
+  member_method {
+    name: "matmul"
+    argspec: "args=[\'self\', \'x\', \'adjoint\', \'adjoint_arg\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'False\', \'matmul\'], "
+  }
+  member_method {
+    name: "matvec"
+    argspec: "args=[\'self\', \'x\', \'adjoint\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'matvec\'], "
+  }
+  member_method {
+    name: "range_dimension_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'range_dimension_tensor\'], "
+  }
+  member_method {
+    name: "shape_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'shape_tensor\'], "
+  }
+  member_method {
+    name: "solve"
+    argspec: "args=[\'self\', \'rhs\', \'adjoint\', \'adjoint_arg\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'False\', \'solve\'], "
+  }
+  member_method {
+    name: "solvevec"
+    argspec: "args=[\'self\', \'rhs\', \'adjoint\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'solve\'], "
+  }
+  member_method {
+    name: "tensor_rank_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'tensor_rank_tensor\'], "
+  }
+  member_method {
+    name: "to_dense"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'to_dense\'], "
+  }
+  member_method {
+    name: "trace"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'trace\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/tensorflow.linalg.-linear-operator-lower-triangular.__metaclass__.pbtxt b/tensorflow/tools/api/golden/tensorflow.linalg.-linear-operator-lower-triangular.__metaclass__.pbtxt
new file mode 100644
index 0000000000..2683430f4f
--- /dev/null
+++ b/tensorflow/tools/api/golden/tensorflow.linalg.-linear-operator-lower-triangular.__metaclass__.pbtxt
@@ -0,0 +1,14 @@
+path: "tensorflow.linalg.LinearOperatorLowerTriangular.__metaclass__"
+tf_class {
+  is_instance: "<class \'abc.ABCMeta\'>"
+  member_method {
+    name: "__init__"
+  }
+  member_method {
+    name: "mro"
+  }
+  member_method {
+    name: "register"
+    argspec: "args=[\'cls\', \'subclass\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/tensorflow.linalg.-linear-operator-lower-triangular.pbtxt b/tensorflow/tools/api/golden/tensorflow.linalg.-linear-operator-lower-triangular.pbtxt
new file mode 100644
index 0000000000..63a1bc2321
--- /dev/null
+++ b/tensorflow/tools/api/golden/tensorflow.linalg.-linear-operator-lower-triangular.pbtxt
@@ -0,0 +1,130 @@
+path: "tensorflow.linalg.LinearOperatorLowerTriangular"
+tf_class {
+  is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator_lower_triangular.LinearOperatorLowerTriangular\'>"
+  is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator.LinearOperator\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "batch_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "domain_dimension"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "graph_parents"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "is_non_singular"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "is_positive_definite"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "is_self_adjoint"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "is_square"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "range_dimension"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "tensor_rank"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'tril\', \'is_non_singular\', \'is_self_adjoint\', \'is_positive_definite\', \'is_square\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'LinearOperatorLowerTriangular\'], "
+  }
+  member_method {
+    name: "add_to_tensor"
+    argspec: "args=[\'self\', \'x\', \'name\'], varargs=None, keywords=None, defaults=[\'add_to_tensor\'], "
+  }
+  member_method {
+    name: "assert_non_singular"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_non_singular\'], "
+  }
+  member_method {
+    name: "assert_positive_definite"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_positive_definite\'], "
+  }
+  member_method {
+    name: "assert_self_adjoint"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_self_adjoint\'], "
+  }
+  member_method {
+    name: "batch_shape_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'batch_shape_tensor\'], "
+  }
+  member_method {
+    name: "determinant"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'det\'], "
+  }
+  member_method {
+    name: "diag_part"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'diag_part\'], "
+  }
+  member_method {
+    name: "domain_dimension_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'domain_dimension_tensor\'], "
+  }
+  member_method {
+    name: "log_abs_determinant"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'log_abs_det\'], "
+  }
+  member_method {
+    name: "matmul"
+    argspec: "args=[\'self\', \'x\', \'adjoint\', \'adjoint_arg\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'False\', \'matmul\'], "
+  }
+  member_method {
+    name: "matvec"
+    argspec: "args=[\'self\', \'x\', \'adjoint\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'matvec\'], "
+  }
+  member_method {
+    name: "range_dimension_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'range_dimension_tensor\'], "
+  }
+  member_method {
+    name: "shape_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'shape_tensor\'], "
+  }
+  member_method {
+    name: "solve"
+    argspec: "args=[\'self\', \'rhs\', \'adjoint\', \'adjoint_arg\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'False\', \'solve\'], "
+  }
+  member_method {
+    name: "solvevec"
+    argspec: "args=[\'self\', \'rhs\', \'adjoint\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'solve\'], "
+  }
+  member_method {
+    name: "tensor_rank_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'tensor_rank_tensor\'], "
+  }
+  member_method {
+    name: "to_dense"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'to_dense\'], "
+  }
+  member_method {
+    name: "trace"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'trace\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/tensorflow.linalg.-linear-operator-scaled-identity.__metaclass__.pbtxt b/tensorflow/tools/api/golden/tensorflow.linalg.-linear-operator-scaled-identity.__metaclass__.pbtxt
new file mode 100644
index 0000000000..38bf7ad586
--- /dev/null
+++ b/tensorflow/tools/api/golden/tensorflow.linalg.-linear-operator-scaled-identity.__metaclass__.pbtxt
@@ -0,0 +1,14 @@
+path: "tensorflow.linalg.LinearOperatorScaledIdentity.__metaclass__"
+tf_class {
+  is_instance: "<class \'abc.ABCMeta\'>"
+  member_method {
+    name: "__init__"
+  }
+  member_method {
+    name: "mro"
+  }
+  member_method {
+    name: "register"
+    argspec: "args=[\'cls\', \'subclass\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/tensorflow.linalg.-linear-operator-scaled-identity.pbtxt b/tensorflow/tools/api/golden/tensorflow.linalg.-linear-operator-scaled-identity.pbtxt
new file mode 100644
index 0000000000..e2c5a505a7
--- /dev/null
+++ b/tensorflow/tools/api/golden/tensorflow.linalg.-linear-operator-scaled-identity.pbtxt
@@ -0,0 +1,135 @@
+path: "tensorflow.linalg.LinearOperatorScaledIdentity"
+tf_class {
+  is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator_identity.LinearOperatorScaledIdentity\'>"
+  is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator_identity.BaseLinearOperatorIdentity\'>"
+  is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator.LinearOperator\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "batch_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "domain_dimension"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "graph_parents"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "is_non_singular"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "is_positive_definite"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "is_self_adjoint"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "is_square"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "multiplier"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "range_dimension"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "tensor_rank"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'num_rows\', \'multiplier\', \'is_non_singular\', \'is_self_adjoint\', \'is_positive_definite\', \'is_square\', \'assert_proper_shapes\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'False\', \'LinearOperatorScaledIdentity\'], "
+  }
+  member_method {
+    name: "add_to_tensor"
+    argspec: "args=[\'self\', \'mat\', \'name\'], varargs=None, keywords=None, defaults=[\'add_to_tensor\'], "
+  }
+  member_method {
+    name: "assert_non_singular"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_non_singular\'], "
+  }
+  member_method {
+    name: "assert_positive_definite"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_positive_definite\'], "
+  }
+  member_method {
+    name: "assert_self_adjoint"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_self_adjoint\'], "
+  }
+  member_method {
+    name: "batch_shape_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'batch_shape_tensor\'], "
+  }
+  member_method {
+    name: "determinant"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'det\'], "
+  }
+  member_method {
+    name: "diag_part"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'diag_part\'], "
+  }
+  member_method {
+    name: "domain_dimension_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'domain_dimension_tensor\'], "
+  }
+  member_method {
+    name: "log_abs_determinant"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'log_abs_det\'], "
+  }
+  member_method {
+    name: "matmul"
+    argspec: "args=[\'self\', \'x\', \'adjoint\', \'adjoint_arg\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'False\', \'matmul\'], "
+  }
+  member_method {
+    name: "matvec"
+    argspec: "args=[\'self\', \'x\', \'adjoint\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'matvec\'], "
+  }
+  member_method {
+    name: "range_dimension_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'range_dimension_tensor\'], "
+  }
+  member_method {
+    name: "shape_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'shape_tensor\'], "
+  }
+  member_method {
+    name: "solve"
+    argspec: "args=[\'self\', \'rhs\', \'adjoint\', \'adjoint_arg\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'False\', \'solve\'], "
+  }
+  member_method {
+    name: "solvevec"
+    argspec: "args=[\'self\', \'rhs\', \'adjoint\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'solve\'], "
+  }
+  member_method {
+    name: "tensor_rank_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'tensor_rank_tensor\'], "
+  }
+  member_method {
+    name: "to_dense"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'to_dense\'], "
+  }
+  member_method {
+    name: "trace"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'trace\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/tensorflow.linalg.-linear-operator.__metaclass__.pbtxt b/tensorflow/tools/api/golden/tensorflow.linalg.-linear-operator.__metaclass__.pbtxt
new file mode 100644
index 0000000000..38da809b36
--- /dev/null
+++ b/tensorflow/tools/api/golden/tensorflow.linalg.-linear-operator.__metaclass__.pbtxt
@@ -0,0 +1,14 @@
+path: "tensorflow.linalg.LinearOperator.__metaclass__"
+tf_class {
+  is_instance: "<class \'abc.ABCMeta\'>"
+  member_method {
+    name: "__init__"
+  }
+  member_method {
+    name: "mro"
+  }
+  member_method {
+    name: "register"
+    argspec: "args=[\'cls\', \'subclass\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/tensorflow.linalg.-linear-operator.pbtxt b/tensorflow/tools/api/golden/tensorflow.linalg.-linear-operator.pbtxt
new file mode 100644
index 0000000000..6d849dc040
--- /dev/null
+++ b/tensorflow/tools/api/golden/tensorflow.linalg.-linear-operator.pbtxt
@@ -0,0 +1,129 @@
+path: "tensorflow.linalg.LinearOperator"
+tf_class {
+  is_instance: "<class \'tensorflow.python.ops.linalg.linear_operator.LinearOperator\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "batch_shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "domain_dimension"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "dtype"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "graph_parents"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "is_non_singular"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "is_positive_definite"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "is_self_adjoint"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "is_square"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "range_dimension"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "shape"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "tensor_rank"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'dtype\', \'graph_parents\', \'is_non_singular\', \'is_self_adjoint\', \'is_positive_definite\', \'is_square\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "add_to_tensor"
+    argspec: "args=[\'self\', \'x\', \'name\'], varargs=None, keywords=None, defaults=[\'add_to_tensor\'], "
+  }
+  member_method {
+    name: "assert_non_singular"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_non_singular\'], "
+  }
+  member_method {
+    name: "assert_positive_definite"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_positive_definite\'], "
+  }
+  member_method {
+    name: "assert_self_adjoint"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'assert_self_adjoint\'], "
+  }
+  member_method {
+    name: "batch_shape_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'batch_shape_tensor\'], "
+  }
+  member_method {
+    name: "determinant"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'det\'], "
+  }
+  member_method {
+    name: "diag_part"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'diag_part\'], "
+  }
+  member_method {
+    name: "domain_dimension_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'domain_dimension_tensor\'], "
+  }
+  member_method {
+    name: "log_abs_determinant"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'log_abs_det\'], "
+  }
+  member_method {
+    name: "matmul"
+    argspec: "args=[\'self\', \'x\', \'adjoint\', \'adjoint_arg\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'False\', \'matmul\'], "
+  }
+  member_method {
+    name: "matvec"
+    argspec: "args=[\'self\', \'x\', \'adjoint\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'matvec\'], "
+  }
+  member_method {
+    name: "range_dimension_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'range_dimension_tensor\'], "
+  }
+  member_method {
+    name: "shape_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'shape_tensor\'], "
+  }
+  member_method {
+    name: "solve"
+    argspec: "args=[\'self\', \'rhs\', \'adjoint\', \'adjoint_arg\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'False\', \'solve\'], "
+  }
+  member_method {
+    name: "solvevec"
+    argspec: "args=[\'self\', \'rhs\', \'adjoint\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'solve\'], "
+  }
+  member_method {
+    name: "tensor_rank_tensor"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'tensor_rank_tensor\'], "
+  }
+  member_method {
+    name: "to_dense"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'to_dense\'], "
+  }
+  member_method {
+    name: "trace"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'trace\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/tensorflow.linalg.pbtxt b/tensorflow/tools/api/golden/tensorflow.linalg.pbtxt
index 51b409bf80..4c94863caa 100644
--- a/tensorflow/tools/api/golden/tensorflow.linalg.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.linalg.pbtxt
@@ -1,5 +1,37 @@
 path: "tensorflow.linalg"
 tf_module {
+  member {
+    name: "LinearOperator"
+    mtype: "<class \'abc.ABCMeta\'>"
+  }
+  member {
+    name: "LinearOperatorComposition"
+    mtype: "<class \'abc.ABCMeta\'>"
+  }
+  member {
+    name: "LinearOperatorDiag"
+    mtype: "<class \'abc.ABCMeta\'>"
+  }
+  member {
+    name: "LinearOperatorFullMatrix"
+    mtype: "<class \'abc.ABCMeta\'>"
+  }
+  member {
+    name: "LinearOperatorIdentity"
+    mtype: "<class \'abc.ABCMeta\'>"
+  }
+  member {
+    name: "LinearOperatorLowRankUpdate"
+    mtype: "<class \'abc.ABCMeta\'>"
+  }
+  member {
+    name: "LinearOperatorLowerTriangular"
+    mtype: "<class \'abc.ABCMeta\'>"
+  }
+  member {
+    name: "LinearOperatorScaledIdentity"
+    mtype: "<class \'abc.ABCMeta\'>"
+  }
   member_method {
     name: "band_part"
     argspec: "args=[\'input\', \'num_lower\', \'num_upper\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
-- 
GitLab


From e540a893f14d9b0beea9161962694bf7d139caf3 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Tue, 10 Oct 2017 14:32:02 -0700
Subject: [PATCH 246/909] [XLA] Fix setting of changed_ in AlgebraicSimplifier.

Due to this bug, sometimes AlgebraicSimplifier would make a change but
say that it didn't.  This would cause us to run the HLO simplification
pipeline fewer times than we should.

PiperOrigin-RevId: 171735154
---
 .../xla/service/algebraic_simplifier.cc       | 10 ++--
 .../xla/service/algebraic_simplifier_test.cc  | 48 +++++++++++++++++++
 2 files changed, 54 insertions(+), 4 deletions(-)

diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier.cc b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
index dd97f3d876..a197a2accc 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
@@ -912,9 +912,10 @@ Status AlgebraicSimplifierVisitor::HandleBroadcast(HloInstruction* broadcast) {
   // A Broadcast that feeds a unary element-wise operation can sink the
   // broadcast after the unary element-wise operation.
   TF_ASSIGN_OR_RETURN(
-      changed_,
+      bool sink_succeeded,
       TryToSinkReshapeOrBroadcastAfterOpWithUniqueNonScalarOperand(broadcast));
-  if (changed_) {
+  changed_ |= sink_succeeded;
+  if (sink_succeeded) {
     return Status::OK();
   }
 
@@ -1217,9 +1218,10 @@ Status AlgebraicSimplifierVisitor::HandleReshape(HloInstruction* reshape) {
   // A Reshape that feeds a unary element-wise operation can sink the
   // reshape after the unary element-wise operation.
   TF_ASSIGN_OR_RETURN(
-      changed_,
+      bool sink_succeeded,
       TryToSinkReshapeOrBroadcastAfterOpWithUniqueNonScalarOperand(reshape));
-  if (changed_) {
+  changed_ |= sink_succeeded;
+  if (sink_succeeded) {
     return Status::OK();
   }
 
diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
index cf97a261da..52231b53d4 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
@@ -1077,6 +1077,54 @@ TEST_F(AlgebraicSimplifierTest, ReshapeToScalarNotHoistedAfterEffectiveUnary) {
               op::Maximum(op::Reshape(param), zero));
 }
 
+// Regression test for a bug where if we failed to sink a reshape, we'd set the
+// 'changed' bit in AlgebraicSimplifier to false.
+TEST_F(AlgebraicSimplifierTest, FailureToSinkReshapeDoesntAffectChangedBit) {
+  HloComputation::Builder builder(TestName());
+
+  // This add (param0 + 0) can be simplified.
+  Shape shape = ShapeUtil::MakeShape(F32, {2, 2});
+  HloInstruction* add = builder.AddInstruction(HloInstruction::CreateBinary(
+      shape, HloOpcode::kAdd,
+      builder.AddInstruction(
+          HloInstruction::CreateParameter(0, shape, "param0")),
+      builder.AddInstruction(HloInstruction::CreateConstant(
+          Literal::CreateR2<float>({{0, 0}, {0, 0}})))));
+
+  builder.AddInstruction(
+      HloInstruction::CreateReshape(ShapeUtil::MakeShape(F32, {4}), add));
+
+  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
+                                 bitcasting_callback());
+  auto module = CreateNewModule();
+  module->AddEntryComputation(builder.Build());
+  EXPECT_TRUE(simplifier.Run(module.get()).ValueOrDie());
+}
+
+// Regression test for a bug where if we failed to sink a reshape, we'd set the
+// 'changed' bit in AlgebraicSimplifier to false.
+TEST_F(AlgebraicSimplifierTest, FailureToSinkBroadcastDoesntAffectChangedBit) {
+  HloComputation::Builder builder(TestName());
+
+  // This add (param0 + 0) can be simplified.
+  Shape shape = ShapeUtil::MakeShape(F32, {2, 2});
+  HloInstruction* add = builder.AddInstruction(HloInstruction::CreateBinary(
+      shape, HloOpcode::kAdd,
+      builder.AddInstruction(
+          HloInstruction::CreateParameter(0, shape, "param0")),
+      builder.AddInstruction(HloInstruction::CreateConstant(
+          Literal::CreateR2<float>({{0, 0}, {0, 0}})))));
+
+  builder.AddInstruction(HloInstruction::CreateBroadcast(
+      ShapeUtil::MakeShape(F32, {2, 2, 2}), add, /*broadcast_dimensions=*/{0}));
+
+  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
+                                 bitcasting_callback());
+  auto module = CreateNewModule();
+  module->AddEntryComputation(builder.Build());
+  EXPECT_TRUE(simplifier.Run(module.get()).ValueOrDie());
+}
+
 TEST_F(AlgebraicSimplifierTest, TransposeEqualsBitcast1) {
   HloComputation::Builder builder(TestName());
   HloInstruction* param =
-- 
GitLab


From e3be40d099e1c5da869b7dfaf8d5891a8c2af312 Mon Sep 17 00:00:00 2001
From: "Jeffrey A. Dean" <jeff@google.com>
Date: Tue, 10 Oct 2017 15:36:59 -0700
Subject: [PATCH 247/909] Slightly rework tf.matmul to be more efficient
 (important for eager mode)

PiperOrigin-RevId: 171745141
---
 tensorflow/python/ops/math_ops.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/tensorflow/python/ops/math_ops.py b/tensorflow/python/ops/math_ops.py
index 9383d72f14..b572377e2f 100644
--- a/tensorflow/python/ops/math_ops.py
+++ b/tensorflow/python/ops/math_ops.py
@@ -1870,11 +1870,12 @@ def matmul(a,
       b = conj(b)
       transpose_b = True
 
-    sparse_matmul_types = [dtypes.bfloat16, dtypes.float32]
-    use_sparse_matmul = (a.dtype in sparse_matmul_types and
-                         b.dtype in sparse_matmul_types and
-                         (a_is_sparse or b_is_sparse))
-    if dtypes.bfloat16 in (a.dtype, b.dtype):
+    use_sparse_matmul = False
+    if a_is_sparse or b_is_sparse:
+      sparse_matmul_types = [dtypes.bfloat16, dtypes.float32]
+      use_sparse_matmul = (a.dtype in sparse_matmul_types and
+                           b.dtype in sparse_matmul_types)
+    if a.dtype == dtypes.bfloat16 or b.dtype == dtypes.bfloat16:
       # matmul currently doesn't handle bfloat16 inputs.
       use_sparse_matmul = True
     if use_sparse_matmul:
-- 
GitLab


From abf9e8cd35e9e83371f3c3ec8e08a8a2d933c82b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 10 Oct 2017 16:42:50 -0700
Subject: [PATCH 248/909] BUILD cleanup

PiperOrigin-RevId: 171753811
---
 tensorflow/contrib/boosted_trees/lib/BUILD | 33 ++++++++++++++++++++--
 1 file changed, 31 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/boosted_trees/lib/BUILD b/tensorflow/contrib/boosted_trees/lib/BUILD
index d4d405c3a9..9b3ffa98e3 100644
--- a/tensorflow/contrib/boosted_trees/lib/BUILD
+++ b/tensorflow/contrib/boosted_trees/lib/BUILD
@@ -262,6 +262,8 @@ py_library(
     srcs = ["learner/batch/base_split_handler.py"],
     srcs_version = "PY2AND3",
     deps = [
+        "//tensorflow/contrib/boosted_trees:batch_ops_utils_py",
+        "//tensorflow/python:control_flow_ops",
     ],
 )
 
@@ -271,9 +273,13 @@ py_library(
     srcs_version = "PY2AND3",
     deps = [
         ":base_split_handler",
-        "//tensorflow/contrib/boosted_trees:quantile_ops_py",
         "//tensorflow/contrib/boosted_trees:split_handler_ops_py",
         "//tensorflow/contrib/boosted_trees:stats_accumulator_ops_py",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:constant_op",
+        "//tensorflow/python:control_flow_ops",
+        "//tensorflow/python:dtypes",
+        "//tensorflow/python:math_ops",
     ],
 )
 
@@ -285,7 +291,15 @@ py_test(
         ":categorical_split_handler",
         "//tensorflow/contrib/boosted_trees/proto:learner_proto_py",
         "//tensorflow/contrib/boosted_trees/proto:split_info_proto_py",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:constant_op",
+        "//tensorflow/python:dtypes",
+        "//tensorflow/python:framework_ops",
         "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:platform_test",
+        "//tensorflow/python:resources",
+        "//tensorflow/python:sparse_tensor",
+        "//tensorflow/python:tensor_shape",
     ],
 )
 
@@ -298,7 +312,14 @@ py_library(
         "//tensorflow/contrib/boosted_trees:quantile_ops_py",
         "//tensorflow/contrib/boosted_trees:split_handler_ops_py",
         "//tensorflow/contrib/boosted_trees:stats_accumulator_ops_py",
-        "//tensorflow/contrib/boosted_trees/proto:quantiles_proto_py",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:constant_op",
+        "//tensorflow/python:control_flow_ops",
+        "//tensorflow/python:dtypes",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:function",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:sparse_tensor",
     ],
 )
 
@@ -310,7 +331,15 @@ py_test(
         ":ordinal_split_handler",
         "//tensorflow/contrib/boosted_trees/proto:learner_proto_py",
         "//tensorflow/contrib/boosted_trees/proto:split_info_proto_py",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:constant_op",
+        "//tensorflow/python:dtypes",
+        "//tensorflow/python:framework_ops",
         "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:platform_test",
+        "//tensorflow/python:resources",
+        "//tensorflow/python:sparse_tensor",
+        "//tensorflow/python:tensor_shape",
     ],
 )
 
-- 
GitLab


From 010506f4feb93ff210fe92d5b48b8b6da56fea9b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 10 Oct 2017 17:01:56 -0700
Subject: [PATCH 249/909] Fix docstring typos in
 tf.distributions.bijectors.Bijector.

PiperOrigin-RevId: 171756150
---
 tensorflow/python/ops/distributions/bijector_impl.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/ops/distributions/bijector_impl.py b/tensorflow/python/ops/distributions/bijector_impl.py
index 1f07b0c91d..8f6d18d91a 100644
--- a/tensorflow/python/ops/distributions/bijector_impl.py
+++ b/tensorflow/python/ops/distributions/bijector_impl.py
@@ -158,7 +158,7 @@ class Bijector(object):
   # Evaluate forward transformation.
   fwd_x = my_bijector.forward(x)
   x == my_bijector.inverse(fwd_x)
-  x != my_bijector.forward(fwd_x)  # Not equal because g(x) != g(g(x)).
+  x != my_bijector.forward(fwd_x)  # Not equal because x != g(g(x)).
   ```
 
   - Computing a log-likelihood:
@@ -275,7 +275,7 @@ class Bijector(object):
       implies `g^{-1}` is differentiable in the image of `g`.
       Applying the chain rule to `y = g(x) = g(g^{-1}(y))` yields
       `I = g'(g^{-1}(y))*g^{-1}'(y)`.
-      The same theorem also implies `g{-1}'` is non-singular therefore:
+      The same theorem also implies `g^{-1}'` is non-singular therefore:
       `inv[ g'(g^{-1}(y)) ] = g^{-1}'(y)`.
       The claim follows from [properties of determinant](
   https://en.wikipedia.org/wiki/Determinant#Multiplicativity_and_matrix_groups).
-- 
GitLab


From 36019666303cd474f5afd0235272c004536fb810 Mon Sep 17 00:00:00 2001
From: Max Galkin <maxgalkin@google.com>
Date: Tue, 10 Oct 2017 17:22:48 -0700
Subject: [PATCH 250/909] Add an option to apply ModelPruner when building a
 grappler item and an option to provide specific feed nodes to the item
 builder.

PiperOrigin-RevId: 171758733
---
 tensorflow/core/grappler/BUILD                |  1 +
 .../core/grappler/grappler_item_builder.cc    | 53 ++++++++++++++++++-
 .../core/grappler/grappler_item_builder.h     |  5 ++
 3 files changed, 57 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/grappler/BUILD b/tensorflow/core/grappler/BUILD
index 3f2cd2ddbf..678f8da298 100644
--- a/tensorflow/core/grappler/BUILD
+++ b/tensorflow/core/grappler/BUILD
@@ -100,6 +100,7 @@ cc_library(
         "//tensorflow/core:lib_internal",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core/grappler/inputs:utils",
+        "//tensorflow/core/grappler/optimizers:model_pruner",
     ],
 )
 
diff --git a/tensorflow/core/grappler/grappler_item_builder.cc b/tensorflow/core/grappler/grappler_item_builder.cc
index d23facf81a..54d60cd7aa 100644
--- a/tensorflow/core/grappler/grappler_item_builder.cc
+++ b/tensorflow/core/grappler/grappler_item_builder.cc
@@ -36,6 +36,7 @@ limitations under the License.
 #include "tensorflow/core/graph/graph_constructor.h"
 #include "tensorflow/core/grappler/inputs/utils.h"
 #include "tensorflow/core/grappler/op_types.h"
+#include "tensorflow/core/grappler/optimizers/model_pruner.h"
 #include "tensorflow/core/grappler/utils.h"
 #include "tensorflow/core/lib/io/path.h"
 #include "tensorflow/core/platform/protobuf_internal.h"
@@ -133,12 +134,24 @@ Status OptimizeGraph(const GraphDef& graph_def_arg, GraphDef* output_graph_def,
       ConvertGraphDefToGraph(graph_ctor_opts, graph_def, graphptr.get()));
 
   // Optimize the graph.
-  GraphOptimizer optimizer(*optimizer_opts);
+  ::tensorflow::GraphOptimizer optimizer(*optimizer_opts);
   optimizer.Optimize(flr, env, devices[0], &graphptr, /*shape_map=*/nullptr);
   graphptr->ToGraphDef(output_graph_def);
 
   return Status::OK();
 }
+
+// Applies the same graph pruning logic to the graph as Session.Run in TF.
+// If the returned status is not OK, item state may be inconsistent.
+Status PruneGraph(GrapplerItem* item) {
+  ModelPruner pruner;
+  GraphDef pruned_graph;
+  Cluster* cluster = nullptr;  // ModelPruner doesn't check cluster.
+  TF_RETURN_IF_ERROR(pruner.Optimize(cluster, *item, &pruned_graph));
+  item->graph = std::move(pruned_graph);
+  return Status::OK();
+}
+
 }  // namespace
 
 // static
@@ -152,6 +165,18 @@ std::unique_ptr<GrapplerItem> GrapplerItemFromMetaGraphDef(
   new_item->id = id;
   new_item->graph = meta_graph.graph_def();
 
+  // Fill in feed nodes from config, if any provided.
+  for (const auto& feed_node : cfg.feed_nodes) {
+    const string feed_name = NodeName(feed_node);
+    if (feed_name.empty()) {
+      LOG(ERROR) << "Invalid feed node name " << feed_node
+                 << ", skipping this input.";
+      return nullptr;
+    }
+    LOG(INFO) << "Will use feed node " << feed_name;
+    new_item->feed.emplace_back(feed_name, Tensor());
+  }
+
   // Attempt to detect the fetch node(s).
   if (meta_graph.collection_def().count("train_op") > 0) {
     const CollectionDef& nodes = meta_graph.collection_def().at("train_op");
@@ -339,9 +364,23 @@ std::unique_ptr<GrapplerItem> GrapplerItemFromMetaGraphDef(
           }
         }
       }
+
       Tensor fake_input(type, shape);
       InitializeTensor(type, &fake_input);
-      new_item->feed.emplace_back(node.name(), fake_input);
+
+      if (cfg.feed_nodes.empty()) {
+        // No specific feed nodes were given. Assume all placeholders are fed.
+        new_item->feed.emplace_back(node.name(), fake_input);
+      } else if (cfg.feed_nodes.count(node.name()) > 0) {
+        // If specific feed nodes were given, only update their tensors.
+        auto it = find_if(new_item->feed.begin(), new_item->feed.end(),
+                          [&node](std::pair<string, Tensor>& f) {
+                            return f.first == node.name();
+                          });
+        QCHECK(it != new_item->feed.end());
+        it->second = fake_input;
+      }
+
       // Set the shape of the node in the graph. This is needed for statically
       // inferring shapes and is a no-op when dynamically inferring shapes as
       // the Placeholder shape will match the shape passed from new_item->feed.
@@ -418,6 +457,16 @@ std::unique_ptr<GrapplerItem> GrapplerItemFromMetaGraphDef(
     return nullptr;
   }
 
+  if (cfg.prune_graph) {
+    VLOG(1) << "Pruning graph...";
+    auto status = PruneGraph(new_item.get());
+    if (!status.ok()) {
+      LOG(ERROR) << "Pruning failed: " << status.error_message();
+      return nullptr;
+    }
+    VLOG(1) << "Pruning ran succesfully.";
+  }
+
   // Validate feed, fetch and init nodes
   std::unordered_set<string> nodes;
   for (const auto& node : new_item->graph.node()) {
diff --git a/tensorflow/core/grappler/grappler_item_builder.h b/tensorflow/core/grappler/grappler_item_builder.h
index 9a7f52228b..85151aabea 100644
--- a/tensorflow/core/grappler/grappler_item_builder.h
+++ b/tensorflow/core/grappler/grappler_item_builder.h
@@ -17,6 +17,7 @@ limitations under the License.
 #define TENSORFLOW_GRAPPLER_GRAPPLER_ITEM_BUILDER_H_
 
 #include <memory>
+#include <set>
 #include <string>
 #include "tensorflow/core/grappler/grappler_item.h"
 
@@ -45,6 +46,10 @@ struct ItemConfig {
   bool erase_noinline_attributes = false;
   // If non-empty, override the directory of asset paths.
   string assets_directory_override;
+  // If true, runs ModelPruner on the graph.
+  bool prune_graph = false;
+  // Override feed nodes list.
+  std::set<string> feed_nodes;
 };
 
 // Factory method for creating a GrapplerItem from a MetaGraphDef.
-- 
GitLab


From d4d5e1510f2404ff1dafaa83171b0dcaec5fdfeb Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Tue, 10 Oct 2017 17:30:35 -0700
Subject: [PATCH 251/909] [XLA] Simplify trivial dynamic-slices.

Also make the dynamic-update-slice simplification respect the
is_layout_sensitive_ flag in algebraic-simplifier

While we're here, make the algebraic-simplifier test use the new
HloVerifiedTestBase class.

PiperOrigin-RevId: 171759708
---
 tensorflow/compiler/xla/service/BUILD         |  2 +-
 .../xla/service/algebraic_simplifier.cc       |  8 ++++--
 .../xla/service/algebraic_simplifier_test.cc  | 27 +++++++++++++++++--
 3 files changed, 32 insertions(+), 5 deletions(-)

diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD
index 0c20a05714..c1bb7107b6 100644
--- a/tensorflow/compiler/xla/service/BUILD
+++ b/tensorflow/compiler/xla/service/BUILD
@@ -1053,7 +1053,7 @@ tf_cc_test(
         "//tensorflow/compiler/xla:types",
         "//tensorflow/compiler/xla:util",
         "//tensorflow/compiler/xla:xla_data_proto",
-        "//tensorflow/compiler/xla/tests:hlo_test_base",
+        "//tensorflow/compiler/xla/tests:hlo_verified_test_base",
         "//tensorflow/compiler/xla/tests:xla_internal_test_main",  # fixdeps: keep
         "//tensorflow/core:lib",
         "//tensorflow/core:test",
diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier.cc b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
index a197a2accc..90ab7700ea 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
@@ -1264,6 +1264,11 @@ Status AlgebraicSimplifierVisitor::HandleDynamicSlice(
   if (ShapeUtil::IsScalar(dynamic_slice->shape())) {
     return ReplaceInstruction(dynamic_slice, operand);
   }
+  // DynamicSlice where operand has the same size as the output and
+  // start_indices are all zero is simply equal to operand.
+  if (IsAll(start_indices, 0) && SameShape(operand, dynamic_slice)) {
+    return ReplaceInstruction(dynamic_slice, operand);
+  }
   return Status::OK();
 }
 
@@ -1282,8 +1287,7 @@ Status AlgebraicSimplifierVisitor::HandleDynamicUpdateSlice(
   // not to affect the visible behavior of this op even when the indices are out
   // of range.  Currently dynamic-update-slice wraps out-of-range indices, so
   // we can only remove the op if its indices never wrap.)
-  if (start_indices->IsConstant() && start_indices->literal().IsAll(0) &&
-      ShapeUtil::Compatible(dynamic_update_slice->shape(), update->shape())) {
+  if (IsAll(start_indices, 0) && SameShape(dynamic_update_slice, update)) {
     return ReplaceInstruction(dynamic_update_slice, update);
   }
   return Status::OK();
diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
index 52231b53d4..f45e541b2c 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
@@ -28,7 +28,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/hlo_pass_fix.h"
 #include "tensorflow/compiler/xla/shape_util.h"
 #include "tensorflow/compiler/xla/test.h"
-#include "tensorflow/compiler/xla/tests/hlo_test_base.h"
+#include "tensorflow/compiler/xla/tests/hlo_verified_test_base.h"
 #include "tensorflow/compiler/xla/types.h"
 #include "tensorflow/compiler/xla/xla_data.pb.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
@@ -47,7 +47,7 @@ AlgebraicSimplifier::ValidBitcastCallback non_bitcasting_callback() {
   return [](const Shape&, const Shape&) { return false; };
 }
 
-class AlgebraicSimplifierTest : public HloTestBase {
+class AlgebraicSimplifierTest : public HloVerifiedTestBase {
  public:
   // Makes a computation that contains a loop that runs num_iters times.
   HloComputation* MakeSimpleLoop(HloModule* module, int num_iters);
@@ -2213,6 +2213,29 @@ TEST_F(AlgebraicSimplifierTest, NotRemovedIfContainsNonRemovableInstruction) {
   EXPECT_FALSE(simplifier.Run(&module).ValueOrDie());
 }
 
+// A dynamic-slice is trivial if its start indices are all zeroes and the size
+// of its input equals the size of its output.  In this case, the dynamic slice
+// is equal to its input.
+TEST_F(AlgebraicSimplifierTest, TrivialDynamicSlice) {
+  HloComputation::Builder builder(TestName());
+
+  Shape shape = ShapeUtil::MakeShape(F32, {10, 100, 1000});
+  builder.AddInstruction(HloInstruction::CreateDynamicSlice(
+      shape,
+      builder.AddInstruction(
+          HloInstruction::CreateParameter(0, shape, "slice_from")),
+      builder.AddInstruction(
+          HloInstruction::CreateConstant(Literal::CreateR1<int>({0, 0, 0}))),
+      /*slice_sizes=*/{10, 100, 1000}));
+
+  auto module = CreateNewModule();
+  auto computation = module->AddEntryComputation(builder.Build());
+  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
+                                 non_bitcasting_callback());
+  ASSERT_TRUE(simplifier.Run(module.get()).ValueOrDie());
+  EXPECT_THAT(computation->root_instruction(), op::Parameter());
+}
+
 // A dynamic-update-slice is trivial if its start indices are all zeroes and the
 // size of its "update" equals the size of its output.  In this case, the
 // dynamic-update-slice is equal to its update.
-- 
GitLab


From 9a7e849472c954470de889cc8873223e4db1e4df Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 10 Oct 2017 18:40:50 -0700
Subject: [PATCH 252/909] * Passing `training_features` (without weight column)
 instead of `features` into GradientBoostedDecisionTreeModel. * Export GTFlow
 model into generic format with features defined in proto.

PiperOrigin-RevId: 171766066
---
 .../estimator_batch/custom_export_strategy.py            | 9 +++++++--
 .../contrib/boosted_trees/estimator_batch/model.py       | 2 +-
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/tensorflow/contrib/boosted_trees/estimator_batch/custom_export_strategy.py b/tensorflow/contrib/boosted_trees/estimator_batch/custom_export_strategy.py
index 7773125c16..a800c3ddc7 100644
--- a/tensorflow/contrib/boosted_trees/estimator_batch/custom_export_strategy.py
+++ b/tensorflow/contrib/boosted_trees/estimator_batch/custom_export_strategy.py
@@ -96,7 +96,8 @@ def make_custom_export_strategy(name,
 
 def convert_to_universal_format(dtec, sorted_feature_names,
                                 num_dense, num_sparse_float,
-                                num_sparse_int):
+                                num_sparse_int,
+                                feature_name_to_proto=None):
   """Convert GTFlow trees to universal format."""
   del num_sparse_int  # unused.
   model_and_features = generic_tree_model_pb2.ModelAndFeatures()
@@ -104,7 +105,11 @@ def convert_to_universal_format(dtec, sorted_feature_names,
   # feature is processed before it's fed to the model (e.g. bucketing
   # information). As of now, this serves as a list of features the model uses.
   for feature_name in sorted_feature_names:
-    model_and_features.features[feature_name].SetInParent()
+    if not feature_name_to_proto:
+      model_and_features.features[feature_name].SetInParent()
+    else:
+      model_and_features.features[feature_name].CopyFrom(
+          feature_name_to_proto[feature_name])
   model = model_and_features.model
   model.ensemble.summation_combination_technique.SetInParent()
   for tree_idx in range(len(dtec.trees)):
diff --git a/tensorflow/contrib/boosted_trees/estimator_batch/model.py b/tensorflow/contrib/boosted_trees/estimator_batch/model.py
index 8cda5c8f2b..c6455a7ea3 100644
--- a/tensorflow/contrib/boosted_trees/estimator_batch/model.py
+++ b/tensorflow/contrib/boosted_trees/estimator_batch/model.py
@@ -93,7 +93,7 @@ def model_builder(features, labels, mode, params, config):
       learner_config=learner_config,
       feature_columns=feature_columns,
       logits_dimension=head.logits_dimension,
-      features=features)
+      features=training_features)
   with ops.name_scope("gbdt", "gbdt_optimizer"):
     predictions_dict = gbdt_model.predict(mode)
     logits = predictions_dict["predictions"]
-- 
GitLab


From 9885aa8636c51bdd4a155b504b7c8c22bdf22289 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Tue, 10 Oct 2017 19:27:45 -0700
Subject: [PATCH 253/909] Add some CPU specific test cases

PiperOrigin-RevId: 171769504
---
 tensorflow/BUILD                              |   1 +
 tensorflow/compiler/xla/tests/cpu/BUILD       |  99 ++++++
 .../xla/tests/cpu/cpu_bytesizeof_test.cc      |  37 ++
 .../compiler/xla/tests/cpu/cpu_codegen_test.h |  30 ++
 .../tests/cpu/cpu_external_constants_test.cc  |  73 ++++
 .../compiler/xla/tests/cpu/cpu_fusion_test.cc | 330 ++++++++++++++++++
 .../xla/tests/cpu/cpu_intrinsic_test.cc       | 150 ++++++++
 7 files changed, 720 insertions(+)
 create mode 100644 tensorflow/compiler/xla/tests/cpu/BUILD
 create mode 100644 tensorflow/compiler/xla/tests/cpu/cpu_bytesizeof_test.cc
 create mode 100644 tensorflow/compiler/xla/tests/cpu/cpu_codegen_test.h
 create mode 100644 tensorflow/compiler/xla/tests/cpu/cpu_external_constants_test.cc
 create mode 100644 tensorflow/compiler/xla/tests/cpu/cpu_fusion_test.cc
 create mode 100644 tensorflow/compiler/xla/tests/cpu/cpu_intrinsic_test.cc

diff --git a/tensorflow/BUILD b/tensorflow/BUILD
index 5bb31d7df1..065e61efca 100644
--- a/tensorflow/BUILD
+++ b/tensorflow/BUILD
@@ -330,6 +330,7 @@ filegroup(
         "//tensorflow/compiler/xla/service/interpreter:all_files",
         "//tensorflow/compiler/xla/service/llvm_ir:all_files",
         "//tensorflow/compiler/xla/tests:all_files",
+        "//tensorflow/compiler/xla/tests/cpu:all_files",
         "//tensorflow/compiler/xla/tools:all_files",
         "//tensorflow/contrib:all_files",
         "//tensorflow/contrib/all_reduce:all_files",
diff --git a/tensorflow/compiler/xla/tests/cpu/BUILD b/tensorflow/compiler/xla/tests/cpu/BUILD
new file mode 100644
index 0000000000..e0253b6a6b
--- /dev/null
+++ b/tensorflow/compiler/xla/tests/cpu/BUILD
@@ -0,0 +1,99 @@
+# Description:
+#   Tests for CPU, in C++, against the XLA API, using the in-process
+#   client library.
+
+licenses(["notice"])  # Apache 2.0
+
+package(
+    default_visibility = [":friends"],
+)
+
+package_group(
+    name = "friends",
+    includes = [
+        "//tensorflow/compiler/xla:friends",
+    ],
+)
+
+load("//tensorflow:tensorflow.bzl", "tf_cc_test")
+
+tf_cc_test(
+    name = "cpu_fusion_test",
+    srcs = ["cpu_fusion_test.cc"],
+    deps = [
+        "//tensorflow/compiler/xla:literal_util",
+        "//tensorflow/compiler/xla:shape_util",
+        "//tensorflow/compiler/xla:util",
+        "//tensorflow/compiler/xla:xla_data_proto",
+        "//tensorflow/compiler/xla/service:cpu_plugin",
+        "//tensorflow/compiler/xla/service:hlo",
+        "//tensorflow/compiler/xla/service/cpu:cpu_instruction_fusion",
+        "//tensorflow/compiler/xla/tests:hlo_test_base",
+        "//tensorflow/compiler/xla/tests:literal_test_util",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+    ],
+)
+
+tf_cc_test(
+    name = "cpu_bytesizeof_test",
+    srcs = ["cpu_bytesizeof_test.cc"],
+    deps = [
+        "//tensorflow/compiler/xla:shape_util",
+        "//tensorflow/compiler/xla/service/llvm_ir:llvm_util",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+    ],
+)
+
+tf_cc_test(
+    name = "cpu_external_constants_test",
+    srcs = ["cpu_external_constants_test.cc"],
+    deps = [
+        ":cpu_codegen_test",
+        "//tensorflow/compiler/xla:array2d",
+        "//tensorflow/compiler/xla:shape_util",
+        "//tensorflow/compiler/xla/service:hlo",
+        "//tensorflow/compiler/xla/tests:filecheck",
+        "//tensorflow/core:test",
+    ],
+)
+
+cc_library(
+    name = "cpu_codegen_test",
+    testonly = True,
+    hdrs = ["cpu_codegen_test.h"],
+    deps = [
+        "//tensorflow/compiler/xla/service:cpu_plugin",
+        "//tensorflow/compiler/xla/tests:llvm_irgen_test_base",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+    ],
+)
+
+tf_cc_test(
+    name = "cpu_intrinsic_test",
+    srcs = ["cpu_intrinsic_test.cc"],
+    deps = [
+        ":cpu_codegen_test",
+        "//tensorflow/compiler/xla/service:hlo",
+        "//tensorflow/compiler/xla/service/cpu:cpu_compiler",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+    ],
+)
+
+# -----------------------------------------------------------------------------
+
+filegroup(
+    name = "all_files",
+    srcs = glob(
+        ["**/*"],
+        exclude = [
+            "**/METADATA",
+            "**/OWNERS",
+        ],
+    ),
+    visibility = ["//tensorflow:__subpackages__"],
+)
diff --git a/tensorflow/compiler/xla/tests/cpu/cpu_bytesizeof_test.cc b/tensorflow/compiler/xla/tests/cpu/cpu_bytesizeof_test.cc
new file mode 100644
index 0000000000..3f2bbbd076
--- /dev/null
+++ b/tensorflow/compiler/xla/tests/cpu/cpu_bytesizeof_test.cc
@@ -0,0 +1,37 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/service/llvm_ir/llvm_util.h"
+#include "tensorflow/compiler/xla/shape_util.h"
+#include "tensorflow/core/platform/test.h"
+
+class CpuByteSizeOfTest : public ::testing::Test {};
+
+TEST_F(CpuByteSizeOfTest, ARM32) {
+  llvm::DataLayout data_layout(
+      "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64");
+  auto tuple_shape =
+      xla::ShapeUtil::MakeTupleShape({xla::ShapeUtil::MakeShape(xla::F32, {})});
+  EXPECT_EQ(xla::llvm_ir::ByteSizeOf(tuple_shape, data_layout),
+            data_layout.getPointerSize());
+}
+
+TEST_F(CpuByteSizeOfTest, ARM64) {
+  llvm::DataLayout data_layout("e-m:e-i64:64-i128:128-n32:64-S128");
+  auto tuple_shape =
+      xla::ShapeUtil::MakeTupleShape({xla::ShapeUtil::MakeShape(xla::F32, {})});
+  EXPECT_EQ(xla::llvm_ir::ByteSizeOf(tuple_shape, data_layout),
+            data_layout.getPointerSize());
+}
diff --git a/tensorflow/compiler/xla/tests/cpu/cpu_codegen_test.h b/tensorflow/compiler/xla/tests/cpu/cpu_codegen_test.h
new file mode 100644
index 0000000000..a6ca00b07d
--- /dev/null
+++ b/tensorflow/compiler/xla/tests/cpu/cpu_codegen_test.h
@@ -0,0 +1,30 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef PLATFORMS_XLA_TESTS_CPU_CPU_CODEGEN_TEST_H_
+#define PLATFORMS_XLA_TESTS_CPU_CPU_CODEGEN_TEST_H_
+
+#include "tensorflow/compiler/xla/tests/llvm_irgen_test_base.h"
+
+namespace xla {
+namespace cpu {
+
+// Tests that verify IR emitted by the CPU backend is as expected.
+class CpuCodegenTest : public LLVMIRGenTestBase {};
+
+}  // namespace cpu
+}  // namespace xla
+
+#endif  // PLATFORMS_XLA_TESTS_CPU_CPU_CODEGEN_TEST_H_
diff --git a/tensorflow/compiler/xla/tests/cpu/cpu_external_constants_test.cc b/tensorflow/compiler/xla/tests/cpu/cpu_external_constants_test.cc
new file mode 100644
index 0000000000..14f223e05e
--- /dev/null
+++ b/tensorflow/compiler/xla/tests/cpu/cpu_external_constants_test.cc
@@ -0,0 +1,73 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include <memory>
+#include <utility>
+
+#include "tensorflow/compiler/xla/array2d.h"
+#include "tensorflow/compiler/xla/service/hlo_computation.h"
+#include "tensorflow/compiler/xla/service/hlo_instruction.h"
+#include "tensorflow/compiler/xla/service/hlo_module.h"
+#include "tensorflow/compiler/xla/shape_util.h"
+#include "tensorflow/compiler/xla/tests/cpu/cpu_codegen_test.h"
+#include "tensorflow/compiler/xla/tests/filecheck.h"
+#include "tensorflow/core/platform/test.h"
+
+namespace xla {
+namespace cpu {
+namespace {
+class CpuExternalConstantsTest : public CpuCodegenTest {
+ public:
+  void TestWithArray(int64 rows, int64 cols, const char* filecheck_pattern) {
+    HloComputation::Builder builder(TestName());
+
+    Array2D<float> backing_array(rows, cols);
+    backing_array.FillUnique();
+
+    auto shape = ShapeUtil::MakeShape(F32, {rows, cols});
+
+    HloInstruction* constant =
+        builder.AddInstruction(HloInstruction::CreateConstant(
+            Literal::CreateR2FromArray2D(backing_array)));
+    HloInstruction* param =
+        builder.AddInstruction(HloInstruction::CreateParameter(0, shape, "x"));
+    builder.AddInstruction(
+        HloInstruction::CreateBinary(shape, HloOpcode::kAdd, param, constant));
+
+    std::unique_ptr<HloModule> module = CreateNewModule();
+    module->AddEntryComputation(builder.Build());
+
+    CompileAndVerifyIr(std::move(module), filecheck_pattern,
+                       /*match_optimized_ir=*/false);
+  }
+};
+
+TEST_F(CpuExternalConstantsTest, Basic) {
+  TestWithArray(/*rows=*/1024, /*cols=*/1024, R"(
+CHECK: @constant_global_0 = external constant [1024 x [1024 x float]], align 16
+)");
+}
+
+TEST_F(CpuExternalConstantsTest, BasicNegative) {
+  // The constant array in this test case is small enough that there is no need
+  // to externalize it.
+  TestWithArray(/*rows=*/4, /*cols=*/4, R"(
+CHECK-NOT: @constant_global_0 = external constant [4 x [4 x float]], align 8
+CHECK: @0 = private constant [4 x [4 x float]] {{.*}}, align 8
+)");
+}
+}  // namespace
+}  // namespace cpu
+}  // namespace xla
diff --git a/tensorflow/compiler/xla/tests/cpu/cpu_fusion_test.cc b/tensorflow/compiler/xla/tests/cpu/cpu_fusion_test.cc
new file mode 100644
index 0000000000..9231d3960e
--- /dev/null
+++ b/tensorflow/compiler/xla/tests/cpu/cpu_fusion_test.cc
@@ -0,0 +1,330 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include <memory>
+#include <utility>
+#include <vector>
+
+#include "tensorflow/compiler/xla/literal_util.h"
+#include "tensorflow/compiler/xla/ptr_util.h"
+#include "tensorflow/compiler/xla/service/cpu/cpu_instruction_fusion.h"
+#include "tensorflow/compiler/xla/service/hlo_computation.h"
+#include "tensorflow/compiler/xla/service/hlo_instruction.h"
+#include "tensorflow/compiler/xla/service/hlo_module.h"
+#include "tensorflow/compiler/xla/service/hlo_opcode.h"
+#include "tensorflow/compiler/xla/shape_util.h"
+#include "tensorflow/compiler/xla/tests/hlo_test_base.h"
+#include "tensorflow/compiler/xla/tests/literal_test_util.h"
+#include "tensorflow/compiler/xla/xla_data.pb.h"
+#include "tensorflow/core/platform/test.h"
+
+namespace xla {
+namespace cpu {
+namespace {
+
+class CpuFusionTest : public HloTestBase {
+ protected:
+  CpuFusionTest() {}
+
+  ErrorSpec error_spec_{0.0001, 1e-5};
+};
+
+TEST_F(CpuFusionTest, FuseTwoElementwiseOps) {
+  auto builder = HloComputation::Builder(TestName());
+  auto input_literal1 = Literal::CreateR1<float>({1.0, 2.0, 3.0});
+  auto input_literal2 = Literal::CreateR1<float>({-2.0, -42.0, 2.0});
+  Shape vshape = input_literal1->shape();
+
+  auto input1 = builder.AddInstruction(
+      HloInstruction::CreateConstant(std::move(input_literal1)));
+  auto input2 = builder.AddInstruction(
+      HloInstruction::CreateConstant(std::move(input_literal2)));
+
+  auto add1 = builder.AddInstruction(
+      HloInstruction::CreateBinary(vshape, HloOpcode::kAdd, input1, input2));
+  builder.AddInstruction(
+      HloInstruction::CreateUnary(vshape, HloOpcode::kNegate, add1));
+
+  auto module = CreateNewModule();
+  module->AddEntryComputation(builder.Build());
+
+  CpuInstructionFusion fusion;
+  EXPECT_TRUE(fusion.Run(module.get()).ValueOrDie());
+
+  // The computation root instruction was fused. Verify the fusion instruction
+  // is now the root.
+  auto computation = module->entry_computation();
+  auto fusion_instruction = computation->root_instruction();
+  EXPECT_EQ(HloOpcode::kFusion, fusion_instruction->opcode());
+  EXPECT_EQ(HloOpcode::kNegate,
+            fusion_instruction->fused_expression_root()->opcode());
+  // There should be four fused instructions: 2 parameters, the add, and the
+  // negate.
+  EXPECT_EQ(4, fusion_instruction->fused_instruction_count());
+
+  // Compile and execute the computation.
+  auto result = ExecuteAndTransfer(std::move(module), {});
+
+  // Check the output correctness.
+  LiteralTestUtil::ExpectR1Near<float>({1.0, 40.0, -5.0}, *result, error_spec_);
+}
+
+TEST_F(CpuFusionTest, FuseElementwiseOpChain) {
+  auto builder = HloComputation::Builder(TestName());
+  auto input_literal = Literal::CreateR1<float>({-1.5, -2.5, -3.0});
+  Shape vshape = input_literal->shape();
+
+  auto input = builder.AddInstruction(
+      HloInstruction::CreateConstant(std::move(input_literal)));
+  auto negate = builder.AddInstruction(
+      HloInstruction::CreateUnary(vshape, HloOpcode::kNegate, input));
+  auto ceil = builder.AddInstruction(
+      HloInstruction::CreateUnary(vshape, HloOpcode::kCeil, negate));
+  auto exp = builder.AddInstruction(
+      HloInstruction::CreateUnary(vshape, HloOpcode::kExp, ceil));
+  auto floor = builder.AddInstruction(
+      HloInstruction::CreateUnary(vshape, HloOpcode::kFloor, exp));
+  auto two = builder.AddInstruction(
+      HloInstruction::CreateConstant(Literal::CreateR0<float>(2.0)));
+  builder.AddInstruction(
+      HloInstruction::CreateBinary(vshape, HloOpcode::kMultiply, two, floor));
+
+  auto module = CreateNewModule();
+  module->AddEntryComputation(builder.Build());
+
+  CpuInstructionFusion fusion;
+  EXPECT_TRUE(fusion.Run(module.get()).ValueOrDie());
+
+  // The computation root instruction was fused. Verify the fusion instruction
+  // is now the root.
+  auto computation = module->entry_computation();
+  auto fusion_instruction = computation->root_instruction();
+  EXPECT_EQ(HloOpcode::kFusion, fusion_instruction->opcode());
+  EXPECT_EQ(HloOpcode::kMultiply,
+            fusion_instruction->fused_expression_root()->opcode());
+  // There should be 7 fused instructions: 2 parameters and the fused
+  // operations.
+  EXPECT_EQ(7, fusion_instruction->fused_instruction_count());
+
+  // Compile and execute the computation.
+  auto result = ExecuteAndTransfer(std::move(module), {});
+
+  // Check the output correctness.
+  LiteralTestUtil::ExpectR1Near<float>({14.0, 40.0, 40.0}, *result,
+                                       error_spec_);
+}
+
+TEST_F(CpuFusionTest, ElementwiseOpChainWithNonfusableInstruction) {
+  // Test a chain of fusable ops with a non-fusable op (a reduce) thrown in the
+  // middle.
+  auto module = CreateNewModule();
+  auto builder = HloComputation::Builder(TestName());
+  auto input_literal = Literal::CreateR1<float>({-1.5, -2.5, -3.0});
+  Shape vshape = input_literal->shape();
+
+  auto input = builder.AddInstruction(
+      HloInstruction::CreateConstant(std::move(input_literal)));
+  auto negate = builder.AddInstruction(
+      HloInstruction::CreateUnary(vshape, HloOpcode::kNegate, input));
+  auto ceil = builder.AddInstruction(
+      HloInstruction::CreateUnary(vshape, HloOpcode::kCeil, negate));
+
+  auto cshape = ShapeUtil::MakeShape(F32, {6});
+  auto concatenate = builder.AddInstruction(
+      HloInstruction::CreateConcatenate(cshape, {ceil, ceil}, /*dimension=*/0));
+
+  // Build an x+y computation to use in a reduce.
+  Shape r0f32 = ShapeUtil::MakeShape(F32, {});
+  auto embedded_builder = HloComputation::Builder("f32+f32");
+  embedded_builder.AddInstruction(HloInstruction::CreateBinary(
+      r0f32, HloOpcode::kAdd,
+      embedded_builder.AddInstruction(
+          HloInstruction::CreateParameter(0, r0f32, "x")),
+      embedded_builder.AddInstruction(
+          HloInstruction::CreateParameter(1, r0f32, "y"))));
+  auto add_f32 = module->AddEmbeddedComputation(embedded_builder.Build());
+
+  // This is a nop reduction.
+  auto reduce = builder.AddInstruction(HloInstruction::CreateReduce(
+      cshape,
+      builder.AddInstruction(HloInstruction::CreateReshape(
+          ShapeUtil::MakeShape(F32, {6, 1}), concatenate)),
+      /*init_value=*/
+      builder.AddInstruction(
+          HloInstruction::CreateConstant(Literal::CreateR0<float>(0))),
+      /*dimensions_to_reduce=*/{1}, add_f32));
+
+  auto exp = builder.AddInstruction(
+      HloInstruction::CreateUnary(cshape, HloOpcode::kExp, reduce));
+  auto floor = builder.AddInstruction(
+      HloInstruction::CreateUnary(cshape, HloOpcode::kFloor, exp));
+  auto two = builder.AddInstruction(
+      HloInstruction::CreateConstant(Literal::CreateR0<float>(2.0)));
+  builder.AddInstruction(
+      HloInstruction::CreateBinary(cshape, HloOpcode::kMultiply, two, floor));
+
+  module->AddEntryComputation(builder.Build());
+
+  CpuInstructionFusion fusion;
+  EXPECT_TRUE(fusion.Run(module.get()).ValueOrDie());
+
+  // The computation root instruction was fused. Verify the fusion instruction
+  // is now the root.
+  auto computation = module->entry_computation();
+
+  auto fusion_instruction1 = computation->root_instruction();
+  EXPECT_EQ(HloOpcode::kFusion, fusion_instruction1->opcode());
+  EXPECT_EQ(HloOpcode::kMultiply,
+            fusion_instruction1->fused_expression_root()->opcode());
+  // There should be 5 fused instructions in the root fusion instruction: 2
+  // parameters, multiply, floor, and exp.
+  EXPECT_EQ(5, fusion_instruction1->fused_instruction_count())
+      << fusion_instruction1->fused_instructions_computation()->ToString();
+
+  auto fusion_instruction2 = reduce->operand(0);
+  EXPECT_EQ(HloOpcode::kFusion, fusion_instruction1->opcode());
+  EXPECT_EQ(HloOpcode::kReshape,
+            fusion_instruction2->fused_expression_root()->opcode());
+  // There should be 5 fused instructions in the second fusion instruction: 1
+  // parameter, negate, ceil, concat, and reshape.
+  EXPECT_EQ(5, fusion_instruction2->fused_instruction_count())
+      << fusion_instruction2->fused_instructions_computation()->ToString();
+
+  // Compile and execute the computation.
+  auto result = ExecuteAndTransfer(std::move(module), {});
+
+  // Check the output correctness.
+  LiteralTestUtil::ExpectR1Near<float>({14.0, 40.0, 40.0, 14.0, 40.0, 40.0},
+                                       *result, error_spec_);
+}
+
+TEST_F(CpuFusionTest, TestOperandOrderToAvoidDuplication) {
+  // Test that the operands of an instruction to be fused are considered in the
+  // proper order to avoid duplication. Test input:
+  //
+  //   constant = {...}
+  //   negate    = neg(constant)
+  //   ceil      = ceil(negate)
+  //   add1      = add(negate, ceil)
+  //   add2      = add(ceil, negate)
+  //
+  // In this example, the operands of both add1 and add2 should be fused in the
+  // order {ceil, negate} even though they have different orders in their
+  // operand vectors. Test for this problem by counting the number of nodes in
+  // each fusion instruction to ensure that negate is not duplicated.
+  auto builder = HloComputation::Builder(TestName());
+  auto input_literal = Literal::CreateR1<float>({1.0, 2.0, 3.0});
+  Shape vshape = input_literal->shape();
+
+  auto constant = builder.AddInstruction(
+      HloInstruction::CreateConstant(std::move(input_literal)));
+  auto negate = builder.AddInstruction(
+      HloInstruction::CreateUnary(vshape, HloOpcode::kNegate, constant));
+  auto ceil = builder.AddInstruction(
+      HloInstruction::CreateUnary(vshape, HloOpcode::kCeil, negate));
+
+  auto add1 = builder.AddInstruction(
+      HloInstruction::CreateBinary(vshape, HloOpcode::kMultiply, negate, ceil));
+  auto add2 = builder.AddInstruction(
+      HloInstruction::CreateBinary(vshape, HloOpcode::kMultiply, ceil, negate));
+
+  // Tie together the two adds with a tuple to create a single root.
+  auto result =
+      builder.AddInstruction(HloInstruction::CreateTuple({add1, add2}));
+
+  // Create computation and module.
+  auto module = CreateNewModule();
+  module->AddEntryComputation(builder.Build());
+
+  // Run fusion.
+  CpuInstructionFusion fusion;
+  EXPECT_TRUE(fusion.Run(module.get()).ValueOrDie());
+
+  auto fusion1 = result->operand(0);
+  auto fusion2 = result->operand(1);
+  EXPECT_EQ(HloOpcode::kFusion, fusion1->opcode());
+  EXPECT_EQ(HloOpcode::kFusion, fusion2->opcode());
+
+  // Each fusion instruction should have 4 fused instruction inside: add, ceil,
+  // negate, and the fused parameter.
+  EXPECT_EQ(4, fusion1->fused_instruction_count());
+  EXPECT_EQ(4, fusion2->fused_instruction_count());
+
+  // Each fusion instruction should have one parameter and the parameter should
+  // be the constant.
+  EXPECT_EQ(1, fusion1->operand_count());
+  EXPECT_EQ(constant, fusion1->operand(0));
+  EXPECT_EQ(1, fusion2->operand_count());
+  EXPECT_EQ(constant, fusion2->operand(0));
+}
+
+TEST_F(CpuFusionTest, DoNotDuplicateExpensiveOps) {
+  // Verify that expensive operations will not be fused if the fusion results in
+  // duplication. Test code:
+  //
+  //   constant = 42.0
+  //   exp1 = exp(constant)
+  //   negate1 = negate(exp1)
+  //   exp2 = exp(constant)
+  //   negate2 = negate(exp2)
+  //   tuple = tuple(negate1, negate2, exp2)
+  //
+  // exp1 should be fused down into negate1, but exp2 will not be fused into
+  // negate2 because this will result in duplication of the expensive exp
+  // computation. The duplication is caused by the other use of exp2 in the
+  // tuple.
+  auto builder = HloComputation::Builder(TestName());
+  auto input_literal1 = Literal::CreateR1<float>({1.0, 2.0, 3.0});
+  auto input_literal2 = Literal::CreateR1<float>({-2.0, -42.0, 2.0});
+  auto constant = builder.AddInstruction(
+      HloInstruction::CreateConstant(Literal::CreateR0<float>(42.0)));
+  Shape shape = constant->shape();
+
+  auto exp1 = builder.AddInstruction(
+      HloInstruction::CreateUnary(shape, HloOpcode::kExp, constant));
+  auto negate1 = builder.AddInstruction(
+      HloInstruction::CreateUnary(shape, HloOpcode::kNegate, exp1));
+
+  auto exp2 = builder.AddInstruction(
+      HloInstruction::CreateUnary(shape, HloOpcode::kExp, constant));
+  auto negate2 = builder.AddInstruction(
+      HloInstruction::CreateUnary(shape, HloOpcode::kNegate, exp2));
+
+  auto tuple = builder.AddInstruction(
+      HloInstruction::CreateTuple({negate1, negate2, exp2}));
+
+  auto module = CreateNewModule();
+  module->AddEntryComputation(builder.Build());
+
+  CpuInstructionFusion fusion;
+  EXPECT_TRUE(fusion.Run(module.get()).ValueOrDie());
+
+  // The only fusion instruction should be operand 0 of the tuple (formerly
+  // negate1).
+  EXPECT_EQ(HloOpcode::kFusion, tuple->operand(0)->opcode());
+  EXPECT_EQ(HloOpcode::kNegate, tuple->operand(1)->opcode());
+  EXPECT_EQ(HloOpcode::kExp, tuple->operand(2)->opcode());
+
+  auto fusion_inst = tuple->operand(0);
+  // There should be three fused instructions: negate2, exp2, and the fused
+  // parameter.
+  EXPECT_EQ(3, fusion_inst->fused_instruction_count());
+  EXPECT_EQ(1, fusion_inst->operand_count());
+  EXPECT_EQ(constant, fusion_inst->operand(0));
+}
+
+}  // namespace
+}  // namespace cpu
+}  // namespace xla
diff --git a/tensorflow/compiler/xla/tests/cpu/cpu_intrinsic_test.cc b/tensorflow/compiler/xla/tests/cpu/cpu_intrinsic_test.cc
new file mode 100644
index 0000000000..15a8a44e4c
--- /dev/null
+++ b/tensorflow/compiler/xla/tests/cpu/cpu_intrinsic_test.cc
@@ -0,0 +1,150 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include <algorithm>
+#include <cctype>
+#include <string>
+
+#include "tensorflow/compiler/xla/service/cpu/cpu_compiler.h"
+#include "tensorflow/compiler/xla/service/hlo_computation.h"
+#include "tensorflow/compiler/xla/tests/cpu/cpu_codegen_test.h"
+#include "tensorflow/core/lib/strings/strcat.h"
+#include "tensorflow/core/platform/test.h"
+
+namespace xla {
+namespace cpu {
+namespace {
+
+const char* const kTriple_x86_64 = "x86_64-pc-linux";
+const char* const kTriple_android_arm = "armv7-none-android";
+
+struct IntrinsicTestSpec {
+  HloOpcode opcode;
+  tensorflow::StringPiece triple;
+  tensorflow::StringPiece features;
+  tensorflow::StringPiece check_lines;
+};
+
+// Tests that unary functions get lowered using intrinsic calls.
+class CpuUnaryIntrinsicTest
+    : public CpuCodegenTest,
+      public ::testing::WithParamInterface<IntrinsicTestSpec> {
+ public:
+  static string Name(const ::testing::TestParamInfo<IntrinsicTestSpec>& info) {
+    auto spec = info.param;
+
+    string opcode = HloOpcodeString(spec.opcode);
+    opcode[0] = toupper(opcode[0]);
+
+    string triple{spec.triple.data(), spec.triple.size()};
+    if (triple == kTriple_x86_64) {
+      triple = "x86_64";
+    } else if (triple == kTriple_android_arm) {
+      triple = "android_arm";
+    } else {
+      triple = "Unknown";
+    }
+
+    string features{spec.features.data(), spec.features.size()};
+    if (!features.empty()) {
+      std::replace_if(features.begin(), features.end(),
+                      [](char c) { return c != '_' && !isalnum(c); }, '_');
+    } else {
+      features = "";
+    }
+
+    return tensorflow::strings::StrCat(opcode.c_str(), "_On_", triple.c_str(),
+                                       features.empty() ? "" : "_With",
+                                       features.c_str());
+  }
+};
+
+// Creates a module with a call to the unary op, and tests if the
+// compiler replaced it with a call to the intrinsic.
+TEST_P(CpuUnaryIntrinsicTest, DoIt) {
+  HloComputation::Builder builder(TestName());
+  IntrinsicTestSpec spec = GetParam();
+
+  auto param_shape = ShapeUtil::MakeShape(F32, {1024});
+  HloInstruction* param = builder.AddInstruction(
+      HloInstruction::CreateParameter(0, param_shape, "input"));
+  builder.AddInstruction(
+      HloInstruction::CreateUnary(param_shape, spec.opcode, param));
+  std::unique_ptr<HloComputation> computation = builder.Build();
+
+  string triple{spec.triple.data(), spec.triple.size()};
+  string features{spec.features.data(), spec.features.size()};
+
+  CpuAotCompilationOptions options{
+      /*triple=*/triple, /*cpu_name=*/"", /*features=*/features,
+      /*entry_point_name=*/"entry",
+      /*relocation_model=*/CpuAotCompilationOptions::RelocationModel::Static};
+
+  auto hlo_module = CreateNewModule();
+  hlo_module->AddEntryComputation(std::move(computation));
+
+  string check_lines{spec.check_lines.data(), spec.check_lines.size()};
+
+  CompileAheadOfTimeAndVerifyIr(std::move(hlo_module), options, check_lines,
+                                /*match_optimized_ir=*/true);
+}
+
+IntrinsicTestSpec CpuUnaryIntrinsicTestCases[] = {
+    IntrinsicTestSpec{
+        HloOpcode::kExp, kTriple_x86_64, "+sse4.1",
+        R"(CHECK: call fast <4 x float> @__xla_cpu_runtime_ExpV4F32SSE(<4 x float> %wide.load))"},
+
+    IntrinsicTestSpec{
+        HloOpcode::kExp, kTriple_x86_64, "+avx",
+        R"(CHECK: call fast <8 x float> @__xla_cpu_runtime_ExpV8F32AVX(<8 x float> %wide.load))"},
+
+    IntrinsicTestSpec{
+        HloOpcode::kExp, kTriple_android_arm, "+neon",
+        R"(CHECK: call fast <4 x float> @__xla_cpu_runtime_ExpV4F32NEON(<4 x float> %wide.load))"},
+
+    IntrinsicTestSpec{
+        HloOpcode::kLog, kTriple_x86_64, "+sse4.1",
+        R"(CHECK: call fast <4 x float> @__xla_cpu_runtime_LogV4F32SSE(<4 x float> %wide.load))"},
+
+    IntrinsicTestSpec{
+        HloOpcode::kLog, kTriple_x86_64, "+avx",
+        R"(CHECK: call fast <8 x float> @__xla_cpu_runtime_LogV8F32AVX(<8 x float> %wide.load))"},
+
+    IntrinsicTestSpec{
+        HloOpcode::kLog, kTriple_android_arm, "+neon",
+        R"(CHECK: call fast <4 x float> @__xla_cpu_runtime_LogV4F32NEON(<4 x float> %wide.load))"},
+
+    // Tanh is inlined, so we match a line from it instead of a function call.
+
+    IntrinsicTestSpec{
+        HloOpcode::kTanh, kTriple_x86_64, "",
+        R"(CHECK: fcmp fast uge <4 x float> %wide.load, <float -9.000000e+00, float -9.000000e+00, float -9.000000e+00, float -9.000000e+00>)"},
+
+    IntrinsicTestSpec{
+        HloOpcode::kTanh, kTriple_x86_64, "+avx",
+        R"(CHECK: fcmp fast uge <8 x float> %wide.load, <float -9.000000e+00, float -9.000000e+00, float -9.000000e+00, float -9.000000e+00, float -9.000000e+00, float -9.000000e+00, float -9.000000e+00, float -9.000000e+00>)"},
+
+    IntrinsicTestSpec{
+        HloOpcode::kTanh, kTriple_android_arm, "",
+        R"(CHECK: fcmp fast uge <4 x float> %wide.load, <float -9.000000e+00, float -9.000000e+00, float -9.000000e+00, float -9.000000e+00>)"}};
+
+INSTANTIATE_TEST_CASE_P(CpuUnaryIntrinsicTestInstantiation,
+                        CpuUnaryIntrinsicTest,
+                        ::testing::ValuesIn(CpuUnaryIntrinsicTestCases),
+                        CpuUnaryIntrinsicTest::Name);
+
+}  // namespace
+}  // namespace cpu
+}  // namespace xla
-- 
GitLab


From 00b368966c8c3e003d2a7ddf3c36165185ed0079 Mon Sep 17 00:00:00 2001
From: Max Galkin <maxgalkin@google.com>
Date: Tue, 10 Oct 2017 20:22:50 -0700
Subject: [PATCH 254/909] Minor code cleanup in grappler cost estimation.

PiperOrigin-RevId: 171772766
---
 .../grappler/costs/op_level_cost_estimator.cc | 27 ++++++++++---------
 .../grappler/costs/op_level_cost_estimator.h  | 13 +++++----
 2 files changed, 22 insertions(+), 18 deletions(-)

diff --git a/tensorflow/core/grappler/costs/op_level_cost_estimator.cc b/tensorflow/core/grappler/costs/op_level_cost_estimator.cc
index b25def7612..7a1295c91e 100644
--- a/tensorflow/core/grappler/costs/op_level_cost_estimator.cc
+++ b/tensorflow/core/grappler/costs/op_level_cost_estimator.cc
@@ -292,21 +292,21 @@ Costs OpLevelCostEstimator::PredictCosts(const OpContext& op_context) const {
   return costs;
 }
 
-std::pair<double, double> OpLevelCostEstimator::GetDeviceInfo(
+OpLevelCostEstimator::DeviceInfo OpLevelCostEstimator::GetDeviceInfo(
     const DeviceProperties& device) const {
   double gflops = -1;
-  double bandwidth = -1;
+  double gb_per_sec = -1;
 
   if (device.type() == "CPU") {
     // Check if vector instructions are available, and refine performance
     // prediction based on this.
     // Frequencies are stored in MHz in the DeviceProperties.
     gflops = device.num_cores() * device.frequency() * 1e-3;
-    if (bandwidth < 0) {
+    if (gb_per_sec < 0) {
       if (device.bandwidth() > 0) {
-        bandwidth = device.bandwidth() / 1e6;
+        gb_per_sec = device.bandwidth() / 1e6;
       } else {
-        bandwidth = 32;
+        gb_per_sec = 32;
       }
     }
   } else if (device.type() == "GPU") {
@@ -328,15 +328,15 @@ std::pair<double, double> OpLevelCostEstimator::GetDeviceInfo(
     gflops = device.num_cores() * device.frequency() * 1e-3 *
              cores_per_multiprocessor * kOpsPerMac;
     if (device.bandwidth() > 0) {
-      bandwidth = device.bandwidth() / 1e6;
+      gb_per_sec = device.bandwidth() / 1e6;
     } else {
-      bandwidth = 100;
+      gb_per_sec = 100;
     }
   }
-  VLOG(1) << "Device: " << device.type() << " GFLOPS: " << gflops
-          << " Bandwidth: " << bandwidth;
+  VLOG(1) << "Device: " << device.type() << " gflops: " << gflops
+          << " gb_per_sec: " << gb_per_sec;
 
-  return std::make_pair(gflops, bandwidth);
+  return {gflops, gb_per_sec};
 }
 
 Costs OpLevelCostEstimator::PredictCwiseOp(const OpContext& op_context) const {
@@ -382,8 +382,8 @@ Costs OpLevelCostEstimator::DummyExecutionTime(
 
 Costs OpLevelCostEstimator::PredictOpCountBasedCost(
     double operations, const OpInfo& op_features) const {
-  std::pair<double, double> device_perf = GetDeviceInfo(op_features.device());
-  Costs::NanoSeconds compute_cost(std::ceil(operations / device_perf.first));
+  DeviceInfo device_perf = GetDeviceInfo(op_features.device());
+  Costs::NanoSeconds compute_cost(std::ceil(operations / device_perf.gigaops));
   VLOG(1) << "Op:" << op_features.op() << " GOps:" << operations / 1e9
           << " Execution Time (ns):" << compute_cost.count();
 
@@ -394,7 +394,8 @@ Costs OpLevelCostEstimator::PredictOpCountBasedCost(
       CalculateOutputSize(op_features, &found_unknown_shapes);
   double total_io_size = total_input_size + total_output_size;
 
-  Costs::NanoSeconds memory_cost(std::ceil(total_io_size / device_perf.second));
+  Costs::NanoSeconds memory_cost(
+      std::ceil(total_io_size / device_perf.gb_per_sec));
   VLOG(1) << "Op:" << op_features.op() << " Size (KB):" << (total_io_size) / 1e3
           << " Memory Time (ns):" << memory_cost.count();
 
diff --git a/tensorflow/core/grappler/costs/op_level_cost_estimator.h b/tensorflow/core/grappler/costs/op_level_cost_estimator.h
index 0e63299bcb..3a8385dd73 100644
--- a/tensorflow/core/grappler/costs/op_level_cost_estimator.h
+++ b/tensorflow/core/grappler/costs/op_level_cost_estimator.h
@@ -36,11 +36,14 @@ class OpLevelCostEstimator {
   virtual Costs PredictCosts(const OpContext& op_context) const;
 
  protected:
-  // Returns an estimate of device performance (in billions of operations
-  // executed per second) and memory bandwidth (in GigaBytes/second) for the
-  // specified device.
-  virtual std::pair<double, double> GetDeviceInfo(
-      const DeviceProperties& device) const;
+  // Basic device performance info, sufficient for roofline estimate.
+  struct DeviceInfo {
+    double gigaops;     // Billions of operations executed per second.
+    double gb_per_sec;  // Bandwidth to main memory in GB per second.
+  };
+
+  // Returns basic device performance info.
+  virtual DeviceInfo GetDeviceInfo(const DeviceProperties& device) const;
 
   // For operations for which we haven't yet built estimates, returns a dummy
   // value based on input size.
-- 
GitLab


From 4385bb907f3decea03d73b3f0a725613fa49a8f4 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Tue, 10 Oct 2017 20:58:01 -0700
Subject: [PATCH 255/909] Automated g4 rollback of changelist 171769504

PiperOrigin-RevId: 171774816
---
 tensorflow/BUILD                              |   1 -
 tensorflow/compiler/xla/tests/cpu/BUILD       |  99 ------
 .../xla/tests/cpu/cpu_bytesizeof_test.cc      |  37 --
 .../compiler/xla/tests/cpu/cpu_codegen_test.h |  30 --
 .../tests/cpu/cpu_external_constants_test.cc  |  73 ----
 .../compiler/xla/tests/cpu/cpu_fusion_test.cc | 330 ------------------
 .../xla/tests/cpu/cpu_intrinsic_test.cc       | 150 --------
 7 files changed, 720 deletions(-)
 delete mode 100644 tensorflow/compiler/xla/tests/cpu/BUILD
 delete mode 100644 tensorflow/compiler/xla/tests/cpu/cpu_bytesizeof_test.cc
 delete mode 100644 tensorflow/compiler/xla/tests/cpu/cpu_codegen_test.h
 delete mode 100644 tensorflow/compiler/xla/tests/cpu/cpu_external_constants_test.cc
 delete mode 100644 tensorflow/compiler/xla/tests/cpu/cpu_fusion_test.cc
 delete mode 100644 tensorflow/compiler/xla/tests/cpu/cpu_intrinsic_test.cc

diff --git a/tensorflow/BUILD b/tensorflow/BUILD
index 065e61efca..5bb31d7df1 100644
--- a/tensorflow/BUILD
+++ b/tensorflow/BUILD
@@ -330,7 +330,6 @@ filegroup(
         "//tensorflow/compiler/xla/service/interpreter:all_files",
         "//tensorflow/compiler/xla/service/llvm_ir:all_files",
         "//tensorflow/compiler/xla/tests:all_files",
-        "//tensorflow/compiler/xla/tests/cpu:all_files",
         "//tensorflow/compiler/xla/tools:all_files",
         "//tensorflow/contrib:all_files",
         "//tensorflow/contrib/all_reduce:all_files",
diff --git a/tensorflow/compiler/xla/tests/cpu/BUILD b/tensorflow/compiler/xla/tests/cpu/BUILD
deleted file mode 100644
index e0253b6a6b..0000000000
--- a/tensorflow/compiler/xla/tests/cpu/BUILD
+++ /dev/null
@@ -1,99 +0,0 @@
-# Description:
-#   Tests for CPU, in C++, against the XLA API, using the in-process
-#   client library.
-
-licenses(["notice"])  # Apache 2.0
-
-package(
-    default_visibility = [":friends"],
-)
-
-package_group(
-    name = "friends",
-    includes = [
-        "//tensorflow/compiler/xla:friends",
-    ],
-)
-
-load("//tensorflow:tensorflow.bzl", "tf_cc_test")
-
-tf_cc_test(
-    name = "cpu_fusion_test",
-    srcs = ["cpu_fusion_test.cc"],
-    deps = [
-        "//tensorflow/compiler/xla:literal_util",
-        "//tensorflow/compiler/xla:shape_util",
-        "//tensorflow/compiler/xla:util",
-        "//tensorflow/compiler/xla:xla_data_proto",
-        "//tensorflow/compiler/xla/service:cpu_plugin",
-        "//tensorflow/compiler/xla/service:hlo",
-        "//tensorflow/compiler/xla/service/cpu:cpu_instruction_fusion",
-        "//tensorflow/compiler/xla/tests:hlo_test_base",
-        "//tensorflow/compiler/xla/tests:literal_test_util",
-        "//tensorflow/core:test",
-        "//tensorflow/core:test_main",
-    ],
-)
-
-tf_cc_test(
-    name = "cpu_bytesizeof_test",
-    srcs = ["cpu_bytesizeof_test.cc"],
-    deps = [
-        "//tensorflow/compiler/xla:shape_util",
-        "//tensorflow/compiler/xla/service/llvm_ir:llvm_util",
-        "//tensorflow/core:test",
-        "//tensorflow/core:test_main",
-    ],
-)
-
-tf_cc_test(
-    name = "cpu_external_constants_test",
-    srcs = ["cpu_external_constants_test.cc"],
-    deps = [
-        ":cpu_codegen_test",
-        "//tensorflow/compiler/xla:array2d",
-        "//tensorflow/compiler/xla:shape_util",
-        "//tensorflow/compiler/xla/service:hlo",
-        "//tensorflow/compiler/xla/tests:filecheck",
-        "//tensorflow/core:test",
-    ],
-)
-
-cc_library(
-    name = "cpu_codegen_test",
-    testonly = True,
-    hdrs = ["cpu_codegen_test.h"],
-    deps = [
-        "//tensorflow/compiler/xla/service:cpu_plugin",
-        "//tensorflow/compiler/xla/tests:llvm_irgen_test_base",
-        "//tensorflow/core:test",
-        "//tensorflow/core:test_main",
-    ],
-)
-
-tf_cc_test(
-    name = "cpu_intrinsic_test",
-    srcs = ["cpu_intrinsic_test.cc"],
-    deps = [
-        ":cpu_codegen_test",
-        "//tensorflow/compiler/xla/service:hlo",
-        "//tensorflow/compiler/xla/service/cpu:cpu_compiler",
-        "//tensorflow/core:lib",
-        "//tensorflow/core:test",
-        "//tensorflow/core:test_main",
-    ],
-)
-
-# -----------------------------------------------------------------------------
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/compiler/xla/tests/cpu/cpu_bytesizeof_test.cc b/tensorflow/compiler/xla/tests/cpu/cpu_bytesizeof_test.cc
deleted file mode 100644
index 3f2bbbd076..0000000000
--- a/tensorflow/compiler/xla/tests/cpu/cpu_bytesizeof_test.cc
+++ /dev/null
@@ -1,37 +0,0 @@
-/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#include "tensorflow/compiler/xla/service/llvm_ir/llvm_util.h"
-#include "tensorflow/compiler/xla/shape_util.h"
-#include "tensorflow/core/platform/test.h"
-
-class CpuByteSizeOfTest : public ::testing::Test {};
-
-TEST_F(CpuByteSizeOfTest, ARM32) {
-  llvm::DataLayout data_layout(
-      "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64");
-  auto tuple_shape =
-      xla::ShapeUtil::MakeTupleShape({xla::ShapeUtil::MakeShape(xla::F32, {})});
-  EXPECT_EQ(xla::llvm_ir::ByteSizeOf(tuple_shape, data_layout),
-            data_layout.getPointerSize());
-}
-
-TEST_F(CpuByteSizeOfTest, ARM64) {
-  llvm::DataLayout data_layout("e-m:e-i64:64-i128:128-n32:64-S128");
-  auto tuple_shape =
-      xla::ShapeUtil::MakeTupleShape({xla::ShapeUtil::MakeShape(xla::F32, {})});
-  EXPECT_EQ(xla::llvm_ir::ByteSizeOf(tuple_shape, data_layout),
-            data_layout.getPointerSize());
-}
diff --git a/tensorflow/compiler/xla/tests/cpu/cpu_codegen_test.h b/tensorflow/compiler/xla/tests/cpu/cpu_codegen_test.h
deleted file mode 100644
index a6ca00b07d..0000000000
--- a/tensorflow/compiler/xla/tests/cpu/cpu_codegen_test.h
+++ /dev/null
@@ -1,30 +0,0 @@
-/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#ifndef PLATFORMS_XLA_TESTS_CPU_CPU_CODEGEN_TEST_H_
-#define PLATFORMS_XLA_TESTS_CPU_CPU_CODEGEN_TEST_H_
-
-#include "tensorflow/compiler/xla/tests/llvm_irgen_test_base.h"
-
-namespace xla {
-namespace cpu {
-
-// Tests that verify IR emitted by the CPU backend is as expected.
-class CpuCodegenTest : public LLVMIRGenTestBase {};
-
-}  // namespace cpu
-}  // namespace xla
-
-#endif  // PLATFORMS_XLA_TESTS_CPU_CPU_CODEGEN_TEST_H_
diff --git a/tensorflow/compiler/xla/tests/cpu/cpu_external_constants_test.cc b/tensorflow/compiler/xla/tests/cpu/cpu_external_constants_test.cc
deleted file mode 100644
index 14f223e05e..0000000000
--- a/tensorflow/compiler/xla/tests/cpu/cpu_external_constants_test.cc
+++ /dev/null
@@ -1,73 +0,0 @@
-/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#include <memory>
-#include <utility>
-
-#include "tensorflow/compiler/xla/array2d.h"
-#include "tensorflow/compiler/xla/service/hlo_computation.h"
-#include "tensorflow/compiler/xla/service/hlo_instruction.h"
-#include "tensorflow/compiler/xla/service/hlo_module.h"
-#include "tensorflow/compiler/xla/shape_util.h"
-#include "tensorflow/compiler/xla/tests/cpu/cpu_codegen_test.h"
-#include "tensorflow/compiler/xla/tests/filecheck.h"
-#include "tensorflow/core/platform/test.h"
-
-namespace xla {
-namespace cpu {
-namespace {
-class CpuExternalConstantsTest : public CpuCodegenTest {
- public:
-  void TestWithArray(int64 rows, int64 cols, const char* filecheck_pattern) {
-    HloComputation::Builder builder(TestName());
-
-    Array2D<float> backing_array(rows, cols);
-    backing_array.FillUnique();
-
-    auto shape = ShapeUtil::MakeShape(F32, {rows, cols});
-
-    HloInstruction* constant =
-        builder.AddInstruction(HloInstruction::CreateConstant(
-            Literal::CreateR2FromArray2D(backing_array)));
-    HloInstruction* param =
-        builder.AddInstruction(HloInstruction::CreateParameter(0, shape, "x"));
-    builder.AddInstruction(
-        HloInstruction::CreateBinary(shape, HloOpcode::kAdd, param, constant));
-
-    std::unique_ptr<HloModule> module = CreateNewModule();
-    module->AddEntryComputation(builder.Build());
-
-    CompileAndVerifyIr(std::move(module), filecheck_pattern,
-                       /*match_optimized_ir=*/false);
-  }
-};
-
-TEST_F(CpuExternalConstantsTest, Basic) {
-  TestWithArray(/*rows=*/1024, /*cols=*/1024, R"(
-CHECK: @constant_global_0 = external constant [1024 x [1024 x float]], align 16
-)");
-}
-
-TEST_F(CpuExternalConstantsTest, BasicNegative) {
-  // The constant array in this test case is small enough that there is no need
-  // to externalize it.
-  TestWithArray(/*rows=*/4, /*cols=*/4, R"(
-CHECK-NOT: @constant_global_0 = external constant [4 x [4 x float]], align 8
-CHECK: @0 = private constant [4 x [4 x float]] {{.*}}, align 8
-)");
-}
-}  // namespace
-}  // namespace cpu
-}  // namespace xla
diff --git a/tensorflow/compiler/xla/tests/cpu/cpu_fusion_test.cc b/tensorflow/compiler/xla/tests/cpu/cpu_fusion_test.cc
deleted file mode 100644
index 9231d3960e..0000000000
--- a/tensorflow/compiler/xla/tests/cpu/cpu_fusion_test.cc
+++ /dev/null
@@ -1,330 +0,0 @@
-/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#include <memory>
-#include <utility>
-#include <vector>
-
-#include "tensorflow/compiler/xla/literal_util.h"
-#include "tensorflow/compiler/xla/ptr_util.h"
-#include "tensorflow/compiler/xla/service/cpu/cpu_instruction_fusion.h"
-#include "tensorflow/compiler/xla/service/hlo_computation.h"
-#include "tensorflow/compiler/xla/service/hlo_instruction.h"
-#include "tensorflow/compiler/xla/service/hlo_module.h"
-#include "tensorflow/compiler/xla/service/hlo_opcode.h"
-#include "tensorflow/compiler/xla/shape_util.h"
-#include "tensorflow/compiler/xla/tests/hlo_test_base.h"
-#include "tensorflow/compiler/xla/tests/literal_test_util.h"
-#include "tensorflow/compiler/xla/xla_data.pb.h"
-#include "tensorflow/core/platform/test.h"
-
-namespace xla {
-namespace cpu {
-namespace {
-
-class CpuFusionTest : public HloTestBase {
- protected:
-  CpuFusionTest() {}
-
-  ErrorSpec error_spec_{0.0001, 1e-5};
-};
-
-TEST_F(CpuFusionTest, FuseTwoElementwiseOps) {
-  auto builder = HloComputation::Builder(TestName());
-  auto input_literal1 = Literal::CreateR1<float>({1.0, 2.0, 3.0});
-  auto input_literal2 = Literal::CreateR1<float>({-2.0, -42.0, 2.0});
-  Shape vshape = input_literal1->shape();
-
-  auto input1 = builder.AddInstruction(
-      HloInstruction::CreateConstant(std::move(input_literal1)));
-  auto input2 = builder.AddInstruction(
-      HloInstruction::CreateConstant(std::move(input_literal2)));
-
-  auto add1 = builder.AddInstruction(
-      HloInstruction::CreateBinary(vshape, HloOpcode::kAdd, input1, input2));
-  builder.AddInstruction(
-      HloInstruction::CreateUnary(vshape, HloOpcode::kNegate, add1));
-
-  auto module = CreateNewModule();
-  module->AddEntryComputation(builder.Build());
-
-  CpuInstructionFusion fusion;
-  EXPECT_TRUE(fusion.Run(module.get()).ValueOrDie());
-
-  // The computation root instruction was fused. Verify the fusion instruction
-  // is now the root.
-  auto computation = module->entry_computation();
-  auto fusion_instruction = computation->root_instruction();
-  EXPECT_EQ(HloOpcode::kFusion, fusion_instruction->opcode());
-  EXPECT_EQ(HloOpcode::kNegate,
-            fusion_instruction->fused_expression_root()->opcode());
-  // There should be four fused instructions: 2 parameters, the add, and the
-  // negate.
-  EXPECT_EQ(4, fusion_instruction->fused_instruction_count());
-
-  // Compile and execute the computation.
-  auto result = ExecuteAndTransfer(std::move(module), {});
-
-  // Check the output correctness.
-  LiteralTestUtil::ExpectR1Near<float>({1.0, 40.0, -5.0}, *result, error_spec_);
-}
-
-TEST_F(CpuFusionTest, FuseElementwiseOpChain) {
-  auto builder = HloComputation::Builder(TestName());
-  auto input_literal = Literal::CreateR1<float>({-1.5, -2.5, -3.0});
-  Shape vshape = input_literal->shape();
-
-  auto input = builder.AddInstruction(
-      HloInstruction::CreateConstant(std::move(input_literal)));
-  auto negate = builder.AddInstruction(
-      HloInstruction::CreateUnary(vshape, HloOpcode::kNegate, input));
-  auto ceil = builder.AddInstruction(
-      HloInstruction::CreateUnary(vshape, HloOpcode::kCeil, negate));
-  auto exp = builder.AddInstruction(
-      HloInstruction::CreateUnary(vshape, HloOpcode::kExp, ceil));
-  auto floor = builder.AddInstruction(
-      HloInstruction::CreateUnary(vshape, HloOpcode::kFloor, exp));
-  auto two = builder.AddInstruction(
-      HloInstruction::CreateConstant(Literal::CreateR0<float>(2.0)));
-  builder.AddInstruction(
-      HloInstruction::CreateBinary(vshape, HloOpcode::kMultiply, two, floor));
-
-  auto module = CreateNewModule();
-  module->AddEntryComputation(builder.Build());
-
-  CpuInstructionFusion fusion;
-  EXPECT_TRUE(fusion.Run(module.get()).ValueOrDie());
-
-  // The computation root instruction was fused. Verify the fusion instruction
-  // is now the root.
-  auto computation = module->entry_computation();
-  auto fusion_instruction = computation->root_instruction();
-  EXPECT_EQ(HloOpcode::kFusion, fusion_instruction->opcode());
-  EXPECT_EQ(HloOpcode::kMultiply,
-            fusion_instruction->fused_expression_root()->opcode());
-  // There should be 7 fused instructions: 2 parameters and the fused
-  // operations.
-  EXPECT_EQ(7, fusion_instruction->fused_instruction_count());
-
-  // Compile and execute the computation.
-  auto result = ExecuteAndTransfer(std::move(module), {});
-
-  // Check the output correctness.
-  LiteralTestUtil::ExpectR1Near<float>({14.0, 40.0, 40.0}, *result,
-                                       error_spec_);
-}
-
-TEST_F(CpuFusionTest, ElementwiseOpChainWithNonfusableInstruction) {
-  // Test a chain of fusable ops with a non-fusable op (a reduce) thrown in the
-  // middle.
-  auto module = CreateNewModule();
-  auto builder = HloComputation::Builder(TestName());
-  auto input_literal = Literal::CreateR1<float>({-1.5, -2.5, -3.0});
-  Shape vshape = input_literal->shape();
-
-  auto input = builder.AddInstruction(
-      HloInstruction::CreateConstant(std::move(input_literal)));
-  auto negate = builder.AddInstruction(
-      HloInstruction::CreateUnary(vshape, HloOpcode::kNegate, input));
-  auto ceil = builder.AddInstruction(
-      HloInstruction::CreateUnary(vshape, HloOpcode::kCeil, negate));
-
-  auto cshape = ShapeUtil::MakeShape(F32, {6});
-  auto concatenate = builder.AddInstruction(
-      HloInstruction::CreateConcatenate(cshape, {ceil, ceil}, /*dimension=*/0));
-
-  // Build an x+y computation to use in a reduce.
-  Shape r0f32 = ShapeUtil::MakeShape(F32, {});
-  auto embedded_builder = HloComputation::Builder("f32+f32");
-  embedded_builder.AddInstruction(HloInstruction::CreateBinary(
-      r0f32, HloOpcode::kAdd,
-      embedded_builder.AddInstruction(
-          HloInstruction::CreateParameter(0, r0f32, "x")),
-      embedded_builder.AddInstruction(
-          HloInstruction::CreateParameter(1, r0f32, "y"))));
-  auto add_f32 = module->AddEmbeddedComputation(embedded_builder.Build());
-
-  // This is a nop reduction.
-  auto reduce = builder.AddInstruction(HloInstruction::CreateReduce(
-      cshape,
-      builder.AddInstruction(HloInstruction::CreateReshape(
-          ShapeUtil::MakeShape(F32, {6, 1}), concatenate)),
-      /*init_value=*/
-      builder.AddInstruction(
-          HloInstruction::CreateConstant(Literal::CreateR0<float>(0))),
-      /*dimensions_to_reduce=*/{1}, add_f32));
-
-  auto exp = builder.AddInstruction(
-      HloInstruction::CreateUnary(cshape, HloOpcode::kExp, reduce));
-  auto floor = builder.AddInstruction(
-      HloInstruction::CreateUnary(cshape, HloOpcode::kFloor, exp));
-  auto two = builder.AddInstruction(
-      HloInstruction::CreateConstant(Literal::CreateR0<float>(2.0)));
-  builder.AddInstruction(
-      HloInstruction::CreateBinary(cshape, HloOpcode::kMultiply, two, floor));
-
-  module->AddEntryComputation(builder.Build());
-
-  CpuInstructionFusion fusion;
-  EXPECT_TRUE(fusion.Run(module.get()).ValueOrDie());
-
-  // The computation root instruction was fused. Verify the fusion instruction
-  // is now the root.
-  auto computation = module->entry_computation();
-
-  auto fusion_instruction1 = computation->root_instruction();
-  EXPECT_EQ(HloOpcode::kFusion, fusion_instruction1->opcode());
-  EXPECT_EQ(HloOpcode::kMultiply,
-            fusion_instruction1->fused_expression_root()->opcode());
-  // There should be 5 fused instructions in the root fusion instruction: 2
-  // parameters, multiply, floor, and exp.
-  EXPECT_EQ(5, fusion_instruction1->fused_instruction_count())
-      << fusion_instruction1->fused_instructions_computation()->ToString();
-
-  auto fusion_instruction2 = reduce->operand(0);
-  EXPECT_EQ(HloOpcode::kFusion, fusion_instruction1->opcode());
-  EXPECT_EQ(HloOpcode::kReshape,
-            fusion_instruction2->fused_expression_root()->opcode());
-  // There should be 5 fused instructions in the second fusion instruction: 1
-  // parameter, negate, ceil, concat, and reshape.
-  EXPECT_EQ(5, fusion_instruction2->fused_instruction_count())
-      << fusion_instruction2->fused_instructions_computation()->ToString();
-
-  // Compile and execute the computation.
-  auto result = ExecuteAndTransfer(std::move(module), {});
-
-  // Check the output correctness.
-  LiteralTestUtil::ExpectR1Near<float>({14.0, 40.0, 40.0, 14.0, 40.0, 40.0},
-                                       *result, error_spec_);
-}
-
-TEST_F(CpuFusionTest, TestOperandOrderToAvoidDuplication) {
-  // Test that the operands of an instruction to be fused are considered in the
-  // proper order to avoid duplication. Test input:
-  //
-  //   constant = {...}
-  //   negate    = neg(constant)
-  //   ceil      = ceil(negate)
-  //   add1      = add(negate, ceil)
-  //   add2      = add(ceil, negate)
-  //
-  // In this example, the operands of both add1 and add2 should be fused in the
-  // order {ceil, negate} even though they have different orders in their
-  // operand vectors. Test for this problem by counting the number of nodes in
-  // each fusion instruction to ensure that negate is not duplicated.
-  auto builder = HloComputation::Builder(TestName());
-  auto input_literal = Literal::CreateR1<float>({1.0, 2.0, 3.0});
-  Shape vshape = input_literal->shape();
-
-  auto constant = builder.AddInstruction(
-      HloInstruction::CreateConstant(std::move(input_literal)));
-  auto negate = builder.AddInstruction(
-      HloInstruction::CreateUnary(vshape, HloOpcode::kNegate, constant));
-  auto ceil = builder.AddInstruction(
-      HloInstruction::CreateUnary(vshape, HloOpcode::kCeil, negate));
-
-  auto add1 = builder.AddInstruction(
-      HloInstruction::CreateBinary(vshape, HloOpcode::kMultiply, negate, ceil));
-  auto add2 = builder.AddInstruction(
-      HloInstruction::CreateBinary(vshape, HloOpcode::kMultiply, ceil, negate));
-
-  // Tie together the two adds with a tuple to create a single root.
-  auto result =
-      builder.AddInstruction(HloInstruction::CreateTuple({add1, add2}));
-
-  // Create computation and module.
-  auto module = CreateNewModule();
-  module->AddEntryComputation(builder.Build());
-
-  // Run fusion.
-  CpuInstructionFusion fusion;
-  EXPECT_TRUE(fusion.Run(module.get()).ValueOrDie());
-
-  auto fusion1 = result->operand(0);
-  auto fusion2 = result->operand(1);
-  EXPECT_EQ(HloOpcode::kFusion, fusion1->opcode());
-  EXPECT_EQ(HloOpcode::kFusion, fusion2->opcode());
-
-  // Each fusion instruction should have 4 fused instruction inside: add, ceil,
-  // negate, and the fused parameter.
-  EXPECT_EQ(4, fusion1->fused_instruction_count());
-  EXPECT_EQ(4, fusion2->fused_instruction_count());
-
-  // Each fusion instruction should have one parameter and the parameter should
-  // be the constant.
-  EXPECT_EQ(1, fusion1->operand_count());
-  EXPECT_EQ(constant, fusion1->operand(0));
-  EXPECT_EQ(1, fusion2->operand_count());
-  EXPECT_EQ(constant, fusion2->operand(0));
-}
-
-TEST_F(CpuFusionTest, DoNotDuplicateExpensiveOps) {
-  // Verify that expensive operations will not be fused if the fusion results in
-  // duplication. Test code:
-  //
-  //   constant = 42.0
-  //   exp1 = exp(constant)
-  //   negate1 = negate(exp1)
-  //   exp2 = exp(constant)
-  //   negate2 = negate(exp2)
-  //   tuple = tuple(negate1, negate2, exp2)
-  //
-  // exp1 should be fused down into negate1, but exp2 will not be fused into
-  // negate2 because this will result in duplication of the expensive exp
-  // computation. The duplication is caused by the other use of exp2 in the
-  // tuple.
-  auto builder = HloComputation::Builder(TestName());
-  auto input_literal1 = Literal::CreateR1<float>({1.0, 2.0, 3.0});
-  auto input_literal2 = Literal::CreateR1<float>({-2.0, -42.0, 2.0});
-  auto constant = builder.AddInstruction(
-      HloInstruction::CreateConstant(Literal::CreateR0<float>(42.0)));
-  Shape shape = constant->shape();
-
-  auto exp1 = builder.AddInstruction(
-      HloInstruction::CreateUnary(shape, HloOpcode::kExp, constant));
-  auto negate1 = builder.AddInstruction(
-      HloInstruction::CreateUnary(shape, HloOpcode::kNegate, exp1));
-
-  auto exp2 = builder.AddInstruction(
-      HloInstruction::CreateUnary(shape, HloOpcode::kExp, constant));
-  auto negate2 = builder.AddInstruction(
-      HloInstruction::CreateUnary(shape, HloOpcode::kNegate, exp2));
-
-  auto tuple = builder.AddInstruction(
-      HloInstruction::CreateTuple({negate1, negate2, exp2}));
-
-  auto module = CreateNewModule();
-  module->AddEntryComputation(builder.Build());
-
-  CpuInstructionFusion fusion;
-  EXPECT_TRUE(fusion.Run(module.get()).ValueOrDie());
-
-  // The only fusion instruction should be operand 0 of the tuple (formerly
-  // negate1).
-  EXPECT_EQ(HloOpcode::kFusion, tuple->operand(0)->opcode());
-  EXPECT_EQ(HloOpcode::kNegate, tuple->operand(1)->opcode());
-  EXPECT_EQ(HloOpcode::kExp, tuple->operand(2)->opcode());
-
-  auto fusion_inst = tuple->operand(0);
-  // There should be three fused instructions: negate2, exp2, and the fused
-  // parameter.
-  EXPECT_EQ(3, fusion_inst->fused_instruction_count());
-  EXPECT_EQ(1, fusion_inst->operand_count());
-  EXPECT_EQ(constant, fusion_inst->operand(0));
-}
-
-}  // namespace
-}  // namespace cpu
-}  // namespace xla
diff --git a/tensorflow/compiler/xla/tests/cpu/cpu_intrinsic_test.cc b/tensorflow/compiler/xla/tests/cpu/cpu_intrinsic_test.cc
deleted file mode 100644
index 15a8a44e4c..0000000000
--- a/tensorflow/compiler/xla/tests/cpu/cpu_intrinsic_test.cc
+++ /dev/null
@@ -1,150 +0,0 @@
-/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#include <algorithm>
-#include <cctype>
-#include <string>
-
-#include "tensorflow/compiler/xla/service/cpu/cpu_compiler.h"
-#include "tensorflow/compiler/xla/service/hlo_computation.h"
-#include "tensorflow/compiler/xla/tests/cpu/cpu_codegen_test.h"
-#include "tensorflow/core/lib/strings/strcat.h"
-#include "tensorflow/core/platform/test.h"
-
-namespace xla {
-namespace cpu {
-namespace {
-
-const char* const kTriple_x86_64 = "x86_64-pc-linux";
-const char* const kTriple_android_arm = "armv7-none-android";
-
-struct IntrinsicTestSpec {
-  HloOpcode opcode;
-  tensorflow::StringPiece triple;
-  tensorflow::StringPiece features;
-  tensorflow::StringPiece check_lines;
-};
-
-// Tests that unary functions get lowered using intrinsic calls.
-class CpuUnaryIntrinsicTest
-    : public CpuCodegenTest,
-      public ::testing::WithParamInterface<IntrinsicTestSpec> {
- public:
-  static string Name(const ::testing::TestParamInfo<IntrinsicTestSpec>& info) {
-    auto spec = info.param;
-
-    string opcode = HloOpcodeString(spec.opcode);
-    opcode[0] = toupper(opcode[0]);
-
-    string triple{spec.triple.data(), spec.triple.size()};
-    if (triple == kTriple_x86_64) {
-      triple = "x86_64";
-    } else if (triple == kTriple_android_arm) {
-      triple = "android_arm";
-    } else {
-      triple = "Unknown";
-    }
-
-    string features{spec.features.data(), spec.features.size()};
-    if (!features.empty()) {
-      std::replace_if(features.begin(), features.end(),
-                      [](char c) { return c != '_' && !isalnum(c); }, '_');
-    } else {
-      features = "";
-    }
-
-    return tensorflow::strings::StrCat(opcode.c_str(), "_On_", triple.c_str(),
-                                       features.empty() ? "" : "_With",
-                                       features.c_str());
-  }
-};
-
-// Creates a module with a call to the unary op, and tests if the
-// compiler replaced it with a call to the intrinsic.
-TEST_P(CpuUnaryIntrinsicTest, DoIt) {
-  HloComputation::Builder builder(TestName());
-  IntrinsicTestSpec spec = GetParam();
-
-  auto param_shape = ShapeUtil::MakeShape(F32, {1024});
-  HloInstruction* param = builder.AddInstruction(
-      HloInstruction::CreateParameter(0, param_shape, "input"));
-  builder.AddInstruction(
-      HloInstruction::CreateUnary(param_shape, spec.opcode, param));
-  std::unique_ptr<HloComputation> computation = builder.Build();
-
-  string triple{spec.triple.data(), spec.triple.size()};
-  string features{spec.features.data(), spec.features.size()};
-
-  CpuAotCompilationOptions options{
-      /*triple=*/triple, /*cpu_name=*/"", /*features=*/features,
-      /*entry_point_name=*/"entry",
-      /*relocation_model=*/CpuAotCompilationOptions::RelocationModel::Static};
-
-  auto hlo_module = CreateNewModule();
-  hlo_module->AddEntryComputation(std::move(computation));
-
-  string check_lines{spec.check_lines.data(), spec.check_lines.size()};
-
-  CompileAheadOfTimeAndVerifyIr(std::move(hlo_module), options, check_lines,
-                                /*match_optimized_ir=*/true);
-}
-
-IntrinsicTestSpec CpuUnaryIntrinsicTestCases[] = {
-    IntrinsicTestSpec{
-        HloOpcode::kExp, kTriple_x86_64, "+sse4.1",
-        R"(CHECK: call fast <4 x float> @__xla_cpu_runtime_ExpV4F32SSE(<4 x float> %wide.load))"},
-
-    IntrinsicTestSpec{
-        HloOpcode::kExp, kTriple_x86_64, "+avx",
-        R"(CHECK: call fast <8 x float> @__xla_cpu_runtime_ExpV8F32AVX(<8 x float> %wide.load))"},
-
-    IntrinsicTestSpec{
-        HloOpcode::kExp, kTriple_android_arm, "+neon",
-        R"(CHECK: call fast <4 x float> @__xla_cpu_runtime_ExpV4F32NEON(<4 x float> %wide.load))"},
-
-    IntrinsicTestSpec{
-        HloOpcode::kLog, kTriple_x86_64, "+sse4.1",
-        R"(CHECK: call fast <4 x float> @__xla_cpu_runtime_LogV4F32SSE(<4 x float> %wide.load))"},
-
-    IntrinsicTestSpec{
-        HloOpcode::kLog, kTriple_x86_64, "+avx",
-        R"(CHECK: call fast <8 x float> @__xla_cpu_runtime_LogV8F32AVX(<8 x float> %wide.load))"},
-
-    IntrinsicTestSpec{
-        HloOpcode::kLog, kTriple_android_arm, "+neon",
-        R"(CHECK: call fast <4 x float> @__xla_cpu_runtime_LogV4F32NEON(<4 x float> %wide.load))"},
-
-    // Tanh is inlined, so we match a line from it instead of a function call.
-
-    IntrinsicTestSpec{
-        HloOpcode::kTanh, kTriple_x86_64, "",
-        R"(CHECK: fcmp fast uge <4 x float> %wide.load, <float -9.000000e+00, float -9.000000e+00, float -9.000000e+00, float -9.000000e+00>)"},
-
-    IntrinsicTestSpec{
-        HloOpcode::kTanh, kTriple_x86_64, "+avx",
-        R"(CHECK: fcmp fast uge <8 x float> %wide.load, <float -9.000000e+00, float -9.000000e+00, float -9.000000e+00, float -9.000000e+00, float -9.000000e+00, float -9.000000e+00, float -9.000000e+00, float -9.000000e+00>)"},
-
-    IntrinsicTestSpec{
-        HloOpcode::kTanh, kTriple_android_arm, "",
-        R"(CHECK: fcmp fast uge <4 x float> %wide.load, <float -9.000000e+00, float -9.000000e+00, float -9.000000e+00, float -9.000000e+00>)"}};
-
-INSTANTIATE_TEST_CASE_P(CpuUnaryIntrinsicTestInstantiation,
-                        CpuUnaryIntrinsicTest,
-                        ::testing::ValuesIn(CpuUnaryIntrinsicTestCases),
-                        CpuUnaryIntrinsicTest::Name);
-
-}  // namespace
-}  // namespace cpu
-}  // namespace xla
-- 
GitLab


From ff8f26d5968f01016428e1755adf514362bf880b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 10 Oct 2017 21:06:53 -0700
Subject: [PATCH 256/909] Improves "SparseTensor labels are not supported"
 error message.

PiperOrigin-RevId: 171775503
---
 tensorflow/python/estimator/canned/head.py | 26 +++++++++++++---------
 1 file changed, 16 insertions(+), 10 deletions(-)

diff --git a/tensorflow/python/estimator/canned/head.py b/tensorflow/python/estimator/canned/head.py
index 43baaece4b..e53626fc54 100644
--- a/tensorflow/python/estimator/canned/head.py
+++ b/tensorflow/python/estimator/canned/head.py
@@ -188,9 +188,6 @@ class _Head(object):
 def _maybe_expand_dim(tensor):
   """Expand the dim of `tensor` with static rank 1."""
   with ops.name_scope(None, 'maybe_expand_dim', (tensor,)):
-    tensor = sparse_tensor.convert_to_tensor_or_sparse_tensor(tensor)
-    if isinstance(tensor, sparse_tensor.SparseTensor):
-      raise ValueError('SparseTensor labels are not supported.')
     static_shape = tensor.shape
     if static_shape is None:
       return tensor
@@ -199,12 +196,21 @@ def _maybe_expand_dim(tensor):
             else tensor)
 
 
-def _check_labels(labels, expected_labels_dimension):
-  """Check labels type and shape."""
+def _check_and_reshape_dense_labels(labels, expected_labels_dimension):
+  """Checks dense labels type and shape and reshapes to 2D Tensor."""
   with ops.name_scope(None, 'labels', (labels,)) as scope:
     labels = sparse_tensor.convert_to_tensor_or_sparse_tensor(labels)
     if isinstance(labels, sparse_tensor.SparseTensor):
-      raise ValueError('SparseTensor labels are not supported.')
+      raise ValueError(
+          'SparseTensor labels are not supported. '
+          'labels must be a Tensor of shape [batch_size, %s]. '
+          'Suggested Fix (1): Check the label feature in your data. '
+          'Each example must contain %s value(s). If not, your choice of label '
+          'was probably incorrect. '
+          'Suggested Fix (2): In your input_fn, use '
+          'tf.sparse_tensor_to_dense() to turn labels into a Tensor.'
+          '' % (expected_labels_dimension, expected_labels_dimension))
+    labels = _maybe_expand_dim(labels)
     labels_shape = array_ops.shape(labels)
     err_msg = 'labels shape must be [batch_size, {}]'.format(
         expected_labels_dimension)
@@ -430,7 +436,7 @@ class _MultiClassHeadWithSoftmaxCrossEntropyLoss(_Head):
   def create_loss(self, features, mode, logits, labels):
     """See `Head`."""
     del mode, features  # Unused for this head.
-    label_ids = self._label_ids(_check_labels(_maybe_expand_dim(labels), 1))
+    label_ids = self._label_ids(_check_and_reshape_dense_labels(labels, 1))
     unweighted_loss = losses.sparse_softmax_cross_entropy(
         labels=label_ids, logits=logits, reduction=losses.Reduction.NONE)
     # Restore the squeezed dim, so unweighted_loss matches the weights shape.
@@ -674,7 +680,7 @@ class _BinaryLogisticHeadWithSigmoidCrossEntropyLoss(_Head):
   def create_loss(self, features, mode, logits, labels):
     """See `Head`."""
     del mode, features  # Unused for this head.
-    labels = _check_labels(_maybe_expand_dim(labels), self.logits_dimension)
+    labels = _check_and_reshape_dense_labels(labels, self.logits_dimension)
     if self._label_vocabulary is not None:
       labels = lookup_ops.index_table_from_tensor(
           vocabulary_list=tuple(self._label_vocabulary),
@@ -823,8 +829,8 @@ class _RegressionHeadWithMeanSquaredErrorLoss(_Head):
   def create_loss(self, features, mode, logits, labels):
     """See `Head`."""
     del mode, features  # Unused for this head.
-    labels = _check_labels(
-        _maybe_expand_dim(math_ops.to_float(labels)), self._logits_dimension)
+    labels = _check_and_reshape_dense_labels(
+        math_ops.to_float(labels), self._logits_dimension)
     return LossAndLabels(
         unweighted_loss=losses.mean_squared_error(
             labels=labels, predictions=logits, reduction=losses.Reduction.NONE),
-- 
GitLab


From 1ad5e692e2fc218ca0b2a9a461c19762fdc9674b Mon Sep 17 00:00:00 2001
From: Pete Warden <pete@petewarden.com>
Date: Tue, 10 Oct 2017 23:50:29 -0700
Subject: [PATCH 257/909] Added support for Python3 Raspberry Pi CI builds
 (#13612)

* Fix for RTLD_GLOBAL breakage of Pi builds, and removed Eigen version change for Pi that's no longer needed

* Fixed Pi Zero OpenBLAS build problems and tidied up directories used

* More robust checks in Pi build script

* Changed output directory for Pi CI build to fix permissions problem

* Added support for Python3 Raspberry Pi CI builds

* Tidied up comments and updated Python tool template

* Cleaned up Python include path logic
---
 tensorflow/tools/ci_build/Dockerfile.pi       |  3 ++
 .../tools/ci_build/Dockerfile.pi-python3      | 23 +++++++++++++++
 .../install/install_pi_python3_toolchain.sh   | 29 +++++++++++++++++++
 .../ci_build/install/install_pi_toolchain.sh  |  2 +-
 third_party/toolchains/cpus/arm/CROSSTOOL.tpl |  2 +-
 .../cpus/arm/arm_compiler_configure.bzl       | 11 +++++++
 6 files changed, 68 insertions(+), 2 deletions(-)
 create mode 100644 tensorflow/tools/ci_build/Dockerfile.pi-python3
 create mode 100755 tensorflow/tools/ci_build/install/install_pi_python3_toolchain.sh

diff --git a/tensorflow/tools/ci_build/Dockerfile.pi b/tensorflow/tools/ci_build/Dockerfile.pi
index 9d12ededb8..2fddd6a2c0 100644
--- a/tensorflow/tools/ci_build/Dockerfile.pi
+++ b/tensorflow/tools/ci_build/Dockerfile.pi
@@ -14,6 +14,9 @@ RUN /install/install_proto3.sh
 RUN /install/install_buildifier.sh
 RUN /install/install_auditwheel.sh
 RUN /install/install_golang.sh
+
+# The following line installs the Python cross-compilation toolchain. All the
+# preceding dependencies should be kept in sync with the main CPU docker file.
 RUN /install/install_pi_toolchain.sh
 
 # Set up the master bazelrc configuration file.
diff --git a/tensorflow/tools/ci_build/Dockerfile.pi-python3 b/tensorflow/tools/ci_build/Dockerfile.pi-python3
new file mode 100644
index 0000000000..18b131ea19
--- /dev/null
+++ b/tensorflow/tools/ci_build/Dockerfile.pi-python3
@@ -0,0 +1,23 @@
+FROM ubuntu:14.04
+
+MAINTAINER Jan Prach <jendap@google.com>
+
+# Copy and run the install scripts.
+COPY install/*.sh /install/
+RUN /install/install_bootstrap_deb_packages.sh
+RUN add-apt-repository -y ppa:openjdk-r/ppa && \
+    add-apt-repository -y ppa:george-edison55/cmake-3.x
+RUN /install/install_deb_packages.sh
+RUN /install/install_pip_packages.sh
+RUN /install/install_bazel.sh
+RUN /install/install_proto3.sh
+RUN /install/install_buildifier.sh
+RUN /install/install_auditwheel.sh
+RUN /install/install_golang.sh
+
+# The following line installs the Python cross-compilation toolchain. All the
+# preceding dependencies should be kept in sync with the main CPU docker file.
+RUN /install/install_pi_python3_toolchain.sh
+
+# Set up the master bazelrc configuration file.
+COPY install/.bazelrc /etc/bazel.bazelrc
diff --git a/tensorflow/tools/ci_build/install/install_pi_python3_toolchain.sh b/tensorflow/tools/ci_build/install/install_pi_python3_toolchain.sh
new file mode 100755
index 0000000000..9d8e3df3b5
--- /dev/null
+++ b/tensorflow/tools/ci_build/install/install_pi_python3_toolchain.sh
@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+dpkg --add-architecture armhf
+echo 'deb [arch=armhf] http://ports.ubuntu.com/ trusty main restricted universe multiverse' >> /etc/apt/sources.list.d/armhf.list
+echo 'deb [arch=armhf] http://ports.ubuntu.com/ trusty-updates main restricted universe multiverse' >> /etc/apt/sources.list.d/armhf.list
+echo 'deb [arch=armhf] http://ports.ubuntu.com/ trusty-security main restricted universe multiverse' >> /etc/apt/sources.list.d/armhf.list
+echo 'deb [arch=armhf] http://ports.ubuntu.com/ trusty-backports main restricted universe multiverse' >> /etc/apt/sources.list.d/armhf.list
+sed -i 's#deb http://archive.ubuntu.com/ubuntu/#deb [arch=amd64] http://archive.ubuntu.com/ubuntu/#g' /etc/apt/sources.list
+apt-get update
+apt-get install -y libpython3-all-dev:armhf
+echo "deb [arch=amd64] http://storage.googleapis.com/bazel-apt stable jdk1.8" | sudo tee /etc/apt/sources.list.d/bazel.list
+curl https://bazel.build/bazel-release.pub.gpg | sudo apt-key add -
+apt-get update
+rm -rf /usr/local/bin/bazel
+apt-get install -y bazel python3 python3-numpy python3-dev python3-pip
diff --git a/tensorflow/tools/ci_build/install/install_pi_toolchain.sh b/tensorflow/tools/ci_build/install/install_pi_toolchain.sh
index ef30ba58c2..03c43cc838 100755
--- a/tensorflow/tools/ci_build/install/install_pi_toolchain.sh
+++ b/tensorflow/tools/ci_build/install/install_pi_toolchain.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
diff --git a/third_party/toolchains/cpus/arm/CROSSTOOL.tpl b/third_party/toolchains/cpus/arm/CROSSTOOL.tpl
index ad7f5596d0..f0e17d1fe0 100644
--- a/third_party/toolchains/cpus/arm/CROSSTOOL.tpl
+++ b/third_party/toolchains/cpus/arm/CROSSTOOL.tpl
@@ -87,7 +87,7 @@ toolchain {
   cxx_flag: "-isystem"
   cxx_flag: "/usr/include/arm-linux-gnueabihf"
   cxx_flag: "-isystem"
-  cxx_flag: "/usr/include/python2.7"
+  cxx_flag: "%{PYTHON_INCLUDE_PATH}%"
   cxx_flag: "-isystem"
   cxx_flag: "/usr/include/"
   linker_flag: "-lstdc++"
diff --git a/third_party/toolchains/cpus/arm/arm_compiler_configure.bzl b/third_party/toolchains/cpus/arm/arm_compiler_configure.bzl
index 5eb3b7bb1c..ab6eac115c 100644
--- a/third_party/toolchains/cpus/arm/arm_compiler_configure.bzl
+++ b/third_party/toolchains/cpus/arm/arm_compiler_configure.bzl
@@ -11,9 +11,20 @@ def _tpl(repository_ctx, tpl, substitutions={}, out=None):
 
 
 def _arm_compiler_configure_impl(repository_ctx):
+  # We need to find a cross-compilation include directory for Python, so look
+  # for an environment variable. Be warned, this crosstool template is only
+  # regenerated on the first run of Bazel, so if you change the variable after
+  # it may not be reflected in later builds. Doing a shutdown and clean of Bazel
+  # doesn't fix this, you'll need to delete the generated file at something like:
+  # external/local_config_arm_compiler/CROSSTOOL in your Bazel install.
+  if "CROSSTOOL_PYTHON_INCLUDE_PATH" in repository_ctx.os.environ:
+    python_include_path = repository_ctx.os.environ["CROSSTOOL_PYTHON_INCLUDE_PATH"]
+  else:
+    python_include_path = "/usr/include/python2.7"
   _tpl(repository_ctx, "CROSSTOOL", {
       "%{ARM_COMPILER_PATH}%": str(repository_ctx.path(
           repository_ctx.attr.remote_config_repo)),
+      "%{PYTHON_INCLUDE_PATH}%": python_include_path,
   })
   repository_ctx.symlink(repository_ctx.attr.build_file, "BUILD")
 
-- 
GitLab


From 0ed44c0144c9dfae8a53dd3b4f943f23c5a57e37 Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Wed, 11 Oct 2017 00:22:33 -0700
Subject: [PATCH 258/909] TensorFlow base ApiDefs and tests to make sure they
 are kept in sync.

PiperOrigin-RevId: 171788007
---
 tensorflow/core/BUILD                         |   30 +
 tensorflow/core/api_def/api_test.cc           |  206 ++
 .../core/api_def/base_api/api_def_A.pbtxt     |  670 +++++
 .../core/api_def/base_api/api_def_B.pbtxt     |  448 +++
 .../core/api_def/base_api/api_def_C.pbtxt     |  513 ++++
 .../core/api_def/base_api/api_def_D.pbtxt     |  790 +++++
 .../core/api_def/base_api/api_def_E.pbtxt     |  261 ++
 .../core/api_def/base_api/api_def_F.pbtxt     |  411 +++
 .../core/api_def/base_api/api_def_G.pbtxt     |  257 ++
 .../core/api_def/base_api/api_def_H.pbtxt     |   52 +
 .../core/api_def/base_api/api_def_I.pbtxt     |  518 ++++
 .../core/api_def/base_api/api_def_L.pbtxt     |  392 +++
 .../core/api_def/base_api/api_def_M.pbtxt     |  749 +++++
 .../core/api_def/base_api/api_def_N.pbtxt     |   94 +
 .../core/api_def/base_api/api_def_O.pbtxt     |  195 ++
 .../core/api_def/base_api/api_def_P.pbtxt     |  431 +++
 .../core/api_def/base_api/api_def_Q.pbtxt     |  609 ++++
 .../core/api_def/base_api/api_def_R.pbtxt     | 1392 +++++++++
 .../core/api_def/base_api/api_def_S.pbtxt     | 2678 +++++++++++++++++
 .../core/api_def/base_api/api_def_T.pbtxt     |  619 ++++
 .../core/api_def/base_api/api_def_U.pbtxt     |  150 +
 .../core/api_def/base_api/api_def_V.pbtxt     |   19 +
 .../core/api_def/base_api/api_def_W.pbtxt     |   72 +
 .../core/api_def/base_api/api_def_Z.pbtxt     |   27 +
 tensorflow/core/api_def/update_api_def.sh     |   28 +
 tensorflow/core/framework/op.h                |    3 +-
 26 files changed, 11613 insertions(+), 1 deletion(-)
 create mode 100644 tensorflow/core/api_def/api_test.cc
 create mode 100644 tensorflow/core/api_def/base_api/api_def_A.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_B.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_C.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_D.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_E.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_F.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_G.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_H.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_I.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_L.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_M.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_N.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_O.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_P.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_Q.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_R.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_S.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_T.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_U.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_V.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_W.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_Z.pbtxt
 create mode 100755 tensorflow/core/api_def/update_api_def.sh

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index f3e43dd552..74aecbc1f2 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -3316,6 +3316,36 @@ tf_cc_test(
     ],
 )
 
+filegroup(
+    name = "base_api_def",
+    data = glob(["api_def/base_api/*"]),
+)
+
+tf_cc_test(
+    name = "api_test",
+    srcs = ["api_def/api_test.cc"],
+    data = [
+        ":base_api_def",
+        "//tensorflow/cc:ops/op_gen_overrides.pbtxt",
+    ],
+    tags = [
+        "manual",
+        "notap",
+    ],
+    deps = [
+        ":framework",
+        ":framework_internal",
+        ":lib",
+        ":lib_internal",
+        ":lib_test_internal",
+        ":op_gen_lib",
+        ":op_gen_overrides_proto_cc",
+        ":ops",
+        ":protos_all_cc",
+        ":test",
+    ],
+)
+
 tf_cc_test_gpu(
     name = "gpu_tracer_test",
     size = "small",
diff --git a/tensorflow/core/api_def/api_test.cc b/tensorflow/core/api_def/api_test.cc
new file mode 100644
index 0000000000..ceeb172fa0
--- /dev/null
+++ b/tensorflow/core/api_def/api_test.cc
@@ -0,0 +1,206 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+// Test that verifies tensorflow/core/api_def/base_api/api_def*.pbtxt files
+// are correct. If api_def*.pbtxt do not match expected contents, run
+// tensorflow/core/api_def/base_api/update_api_def.sh script to update them.
+
+#include <ctype.h>
+#include <algorithm>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "tensorflow/core/framework/api_def.pb.h"
+#include "tensorflow/core/framework/attr_value.pb.h"
+#include "tensorflow/core/framework/op.h"
+#include "tensorflow/core/framework/op_def.pb.h"
+#include "tensorflow/core/framework/op_gen_lib.h"
+#include "tensorflow/core/framework/op_gen_overrides.pb.h"
+#include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/lib/core/status_test_util.h"
+#include "tensorflow/core/lib/io/path.h"
+#include "tensorflow/core/lib/strings/stringprintf.h"
+#include "tensorflow/core/platform/env.h"
+#include "tensorflow/core/platform/init_main.h"
+#include "tensorflow/core/platform/protobuf.h"
+#include "tensorflow/core/platform/test.h"
+#include "tensorflow/core/platform/types.h"
+#include "tensorflow/core/util/command_line_flags.h"
+
+namespace tensorflow {
+namespace {
+constexpr char kDefaultApiDefDir[] =
+    "tensorflow/core/api_def/base_api";
+constexpr char kOverridesFilePath[] =
+    "tensorflow/cc/ops/op_gen_overrides.pbtxt";
+constexpr char kApiDefFileFormat[] = "api_def_%c.pbtxt";
+constexpr char kAlphabet[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
+
+// Get map from first character to ApiDefs for ops
+// that start with that character.
+std::unordered_map<char, ApiDefs> GenerateApiDef(
+    const OpList& ops, const OpGenOverrides& overrides) {
+  std::unordered_map<string, OpGenOverride> name_to_override;
+  for (const auto& op_override : overrides.op()) {
+    name_to_override[op_override.name()] = op_override;
+  }
+
+  std::unordered_map<char, ApiDefs> api_defs_map;
+
+  for (const auto& op : ops.op()) {
+    CHECK(!op.name().empty())
+        << "Encountered empty op name: %s" << op.DebugString();
+    const char file_id = toupper(op.name()[0]);
+    CHECK(isalpha(file_id)) << "Unexpected op name: " << op.name();
+    ApiDef* api_def = api_defs_map[file_id].add_op();
+    api_def->set_graph_op_name(op.name());
+
+    if (name_to_override.find(op.name()) != name_to_override.end()) {
+      const auto& op_override = name_to_override[op.name()];
+      // Set visibility
+      if (op_override.skip()) {
+        api_def->set_visibility(ApiDef_Visibility_SKIP);
+      } else if (op_override.hide()) {
+        api_def->set_visibility(ApiDef_Visibility_HIDDEN);
+      }
+      // Add endpoints
+      if (!op_override.rename_to().empty()) {
+        auto* endpoint = api_def->add_endpoint();
+        endpoint->set_name(op_override.rename_to());
+      } else {
+        auto* endpoint = api_def->add_endpoint();
+        endpoint->set_name(op.name());
+      }
+      for (auto& alias : op_override.alias()) {
+        auto* endpoint = api_def->add_endpoint();
+        endpoint->set_name(alias);
+      }
+      // Add attributes
+      for (auto& attr : op.attr()) {
+        auto* api_def_attr = api_def->add_attr();
+        api_def_attr->set_name(attr.name());
+        for (auto& attr_override : op_override.attr_default()) {
+          if (attr.name() == attr_override.name()) {
+            *(api_def_attr->mutable_default_value()) = attr_override.value();
+          }
+        }
+        for (auto& attr_rename : op_override.attr_rename()) {
+          if (attr.name() == attr_rename.from()) {
+            api_def_attr->set_rename_to(attr_rename.to());
+          }
+        }
+      }
+    } else {
+      auto* endpoint = api_def->add_endpoint();
+      endpoint->set_name(op.name());
+    }
+    // Add docs
+    api_def->set_summary(op.summary());
+    api_def->set_description(op.description());
+  }
+  return api_defs_map;
+}
+
+// Reads golden api defs file with the given suffix.
+string GetGoldenApiDefsStr(Env* env, const string& api_files_dir, char suffix) {
+  string file_path = strings::Printf(
+      io::JoinPath(api_files_dir, kApiDefFileFormat).c_str(), suffix);
+  if (env->FileExists(file_path).ok()) {
+    string file_contents;
+    TF_EXPECT_OK(ReadFileToString(env, file_path, &file_contents));
+    return file_contents;
+  }
+  return "";
+}
+
+void RunApiTest(bool update_api_def, const string& api_files_dir) {
+  // Read C++ overrides file
+  string overrides_file_contents;
+  Env* env = Env::Default();
+  TF_EXPECT_OK(
+      ReadFileToString(env, kOverridesFilePath, &overrides_file_contents));
+
+  // Read all ops
+  OpList ops;
+  OpRegistry::Global()->Export(false, &ops);
+  const std::vector<string> multi_line_fields = {"description"};
+
+  // Get expected ApiDefs
+  OpGenOverrides overrides;
+  auto new_api_defs_map = GenerateApiDef(ops, overrides);
+
+  bool updated_at_least_one_file = false;
+
+  for (char c : kAlphabet) {
+    string golden_api_defs_str = GetGoldenApiDefsStr(env, api_files_dir, c);
+    string new_api_defs_str = new_api_defs_map[c].DebugString();
+    new_api_defs_str = PBTxtToMultiline(new_api_defs_str, multi_line_fields);
+    if (golden_api_defs_str == new_api_defs_str) {
+      continue;
+    }
+    if (update_api_def) {
+      string output_file_path =
+          io::JoinPath(api_files_dir, strings::Printf(kApiDefFileFormat, c));
+      if (new_api_defs_str.empty()) {
+        std::cout << "Deleting " << output_file_path << "..." << std::endl;
+        TF_EXPECT_OK(env->DeleteFile(output_file_path));
+      } else {
+        std::cout << "Updating " << output_file_path << "..." << std::endl;
+        TF_EXPECT_OK(
+            WriteStringToFile(env, output_file_path, new_api_defs_str));
+      }
+      updated_at_least_one_file = true;
+    } else {
+      EXPECT_EQ(golden_api_defs_str, new_api_defs_str)
+          << "To update golden API files, run "
+          << "tensorflow/core/api_def/update_api_def.sh.";
+    }
+  }
+
+  if (update_api_def && !updated_at_least_one_file) {
+    std::cout << "Api def files are already up to date." << std::endl;
+  }
+}
+
+TEST(ApiTest, GenerateBaseAPIDef) { RunApiTest(false, kDefaultApiDefDir); }
+}  // namespace
+}  // namespace tensorflow
+
+int main(int argc, char** argv) {
+  bool update_api_def = false;
+  tensorflow::string api_files_dir = tensorflow::kDefaultApiDefDir;
+  std::vector<tensorflow::Flag> flag_list = {
+      tensorflow::Flag(
+          "update_api_def", &update_api_def,
+          "Whether to update tensorflow/core/api_def/base_api/api_def*.pbtxt "
+          "files if they differ from expected API."),
+      tensorflow::Flag("api_def_dir", &api_files_dir,
+                       "Base directory of api_def*.pbtxt files.")};
+  std::string usage = tensorflow::Flags::Usage(argv[0], flag_list);
+  bool parsed_values_ok = tensorflow::Flags::Parse(&argc, argv, flag_list);
+  if (!parsed_values_ok) {
+    std::cerr << usage << std::endl;
+    return 2;
+  }
+  if (update_api_def) {
+    tensorflow::port::InitMain(argv[0], &argc, &argv);
+    tensorflow::RunApiTest(update_api_def, api_files_dir);
+    return 0;
+  }
+  testing::InitGoogleTest(&argc, argv);
+  // Run tests
+  return RUN_ALL_TESTS();
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_A.pbtxt b/tensorflow/core/api_def/base_api/api_def_A.pbtxt
new file mode 100644
index 0000000000..8193d1bc62
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_A.pbtxt
@@ -0,0 +1,670 @@
+op {
+  graph_op_name: "Abort"
+  endpoint {
+    name: "Abort"
+  }
+  summary: "Raise a exception to abort the process when called."
+  description: <<END
+If exit_without_error is true, the process will exit normally,
+otherwise it will exit with a SIGABORT signal.
+
+Returns nothing but an exception.
+END
+}
+op {
+  graph_op_name: "Abs"
+  endpoint {
+    name: "Abs"
+  }
+  summary: "Computes the absolute value of a tensor."
+  description: <<END
+Given a tensor `x`, this operation returns a tensor containing the absolute
+value of each element in `x`. For example, if x is an input element and y is
+an output element, this operation computes \\(y = |x|\\).
+END
+}
+op {
+  graph_op_name: "AccumulatorApplyGradient"
+  endpoint {
+    name: "AccumulatorApplyGradient"
+  }
+  summary: "Applies a gradient to a given accumulator."
+  description: <<END
+Does not add if local_step is lesser than the accumulator's global_step.
+END
+}
+op {
+  graph_op_name: "AccumulatorNumAccumulated"
+  endpoint {
+    name: "AccumulatorNumAccumulated"
+  }
+  summary: "Returns the number of gradients aggregated in the given accumulators."
+}
+op {
+  graph_op_name: "AccumulatorSetGlobalStep"
+  endpoint {
+    name: "AccumulatorSetGlobalStep"
+  }
+  summary: "Updates the accumulator with a new value for global_step."
+  description: <<END
+Logs warning if the accumulator's value is already higher than
+new_global_step.
+END
+}
+op {
+  graph_op_name: "AccumulatorTakeGradient"
+  endpoint {
+    name: "AccumulatorTakeGradient"
+  }
+  summary: "Extracts the average gradient in the given ConditionalAccumulator."
+  description: <<END
+The op blocks until sufficient (i.e., more than num_required)
+gradients have been accumulated.  If the accumulator has already
+aggregated more than num_required gradients, it returns the average of
+the accumulated gradients.  Also automatically increments the recorded
+global_step in the accumulator by 1, and resets the aggregate to 0.
+END
+}
+op {
+  graph_op_name: "Acos"
+  endpoint {
+    name: "Acos"
+  }
+  summary: "Computes acos of x element-wise."
+}
+op {
+  graph_op_name: "Acosh"
+  endpoint {
+    name: "Acosh"
+  }
+  summary: "Computes inverse hyperbolic cosine of x element-wise."
+}
+op {
+  graph_op_name: "Add"
+  endpoint {
+    name: "Add"
+  }
+  summary: "Returns x + y element-wise."
+  description: <<END
+*NOTE*: `Add` supports broadcasting. `AddN` does not. More about broadcasting
+[here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
+END
+}
+op {
+  graph_op_name: "AddManySparseToTensorsMap"
+  endpoint {
+    name: "AddManySparseToTensorsMap"
+  }
+  summary: "Add an `N`-minibatch `SparseTensor` to a `SparseTensorsMap`, return `N` handles."
+  description: <<END
+A `SparseTensor` of rank `R` is represented by three tensors: `sparse_indices`,
+`sparse_values`, and `sparse_shape`, where
+
+```sparse_indices.shape[1] == sparse_shape.shape[0] == R```
+
+An `N`-minibatch of `SparseTensor` objects is represented as a `SparseTensor`
+having a first `sparse_indices` column taking values between `[0, N)`, where
+the minibatch size `N == sparse_shape[0]`.
+
+The input `SparseTensor` must have rank `R` greater than 1, and the first
+dimension is treated as the minibatch dimension.  Elements of the `SparseTensor`
+must be sorted in increasing order of this first dimension.  The stored
+`SparseTensor` objects pointed to by each row of the output `sparse_handles`
+will have rank `R-1`.
+
+The `SparseTensor` values can then be read out as part of a minibatch by passing
+the given keys as vector elements to `TakeManySparseFromTensorsMap`.  To ensure
+the correct `SparseTensorsMap` is accessed, ensure that the same
+`container` and `shared_name` are passed to that Op.  If no `shared_name`
+is provided here, instead use the *name* of the Operation created by calling
+`AddManySparseToTensorsMap` as the `shared_name` passed to
+`TakeManySparseFromTensorsMap`.  Ensure the Operations are colocated.
+END
+}
+op {
+  graph_op_name: "AddN"
+  endpoint {
+    name: "AddN"
+  }
+  summary: "Add all input tensors element wise."
+}
+op {
+  graph_op_name: "AddSparseToTensorsMap"
+  endpoint {
+    name: "AddSparseToTensorsMap"
+  }
+  summary: "Add a `SparseTensor` to a `SparseTensorsMap` return its handle."
+  description: <<END
+A `SparseTensor` is represented by three tensors: `sparse_indices`,
+`sparse_values`, and `sparse_shape`.
+
+This operator takes the given `SparseTensor` and adds it to a container
+object (a `SparseTensorsMap`).  A unique key within this container is generated
+in the form of an `int64`, and this is the value that is returned.
+
+The `SparseTensor` can then be read out as part of a minibatch by passing
+the key as a vector element to `TakeManySparseFromTensorsMap`.  To ensure
+the correct `SparseTensorsMap` is accessed, ensure that the same
+`container` and `shared_name` are passed to that Op.  If no `shared_name`
+is provided here, instead use the *name* of the Operation created by calling
+`AddSparseToTensorsMap` as the `shared_name` passed to
+`TakeManySparseFromTensorsMap`.  Ensure the Operations are colocated.
+END
+}
+op {
+  graph_op_name: "AdjustContrast"
+  endpoint {
+    name: "AdjustContrast"
+  }
+  summary: "Deprecated. Disallowed in GraphDef version >= 2."
+}
+op {
+  graph_op_name: "AdjustContrastv2"
+  endpoint {
+    name: "AdjustContrastv2"
+  }
+  summary: "Adjust the contrast of one or more images."
+  description: <<END
+`images` is a tensor of at least 3 dimensions.  The last 3 dimensions are
+interpreted as `[height, width, channels]`.  The other dimensions only
+represent a collection of images, such as `[batch, height, width, channels].`
+
+Contrast is adjusted independently for each channel of each image.
+
+For each channel, the Op first computes the mean of the image pixels in the
+channel and then adjusts each component of each pixel to
+`(x - mean) * contrast_factor + mean`.
+END
+}
+op {
+  graph_op_name: "AdjustHue"
+  endpoint {
+    name: "AdjustHue"
+  }
+  summary: "Adjust the hue of one or more images."
+  description: <<END
+`images` is a tensor of at least 3 dimensions.  The last dimension is
+interpretted as channels, and must be three.
+
+The input image is considered in the RGB colorspace. Conceptually, the RGB
+colors are first mapped into HSV. A delta is then applied all the hue values,
+and then remapped back to RGB colorspace.
+END
+}
+op {
+  graph_op_name: "AdjustSaturation"
+  endpoint {
+    name: "AdjustSaturation"
+  }
+  summary: "Adjust the saturation of one or more images."
+  description: <<END
+`images` is a tensor of at least 3 dimensions.  The last dimension is
+interpretted as channels, and must be three.
+
+The input image is considered in the RGB colorspace. Conceptually, the RGB
+colors are first mapped into HSV. A scale is then applied all the saturation
+values, and then remapped back to RGB colorspace.
+END
+}
+op {
+  graph_op_name: "All"
+  endpoint {
+    name: "All"
+  }
+  summary: "Computes the \"logical and\" of elements across dimensions of a tensor."
+  description: <<END
+Reduces `input` along the dimensions given in `reduction_indices`. Unless
+`keep_dims` is true, the rank of the tensor is reduced by 1 for each entry in
+`reduction_indices`. If `keep_dims` is true, the reduced dimensions are
+retained with length 1.
+END
+}
+op {
+  graph_op_name: "AllCandidateSampler"
+  endpoint {
+    name: "AllCandidateSampler"
+  }
+  summary: "Generates labels for candidate sampling with a learned unigram distribution."
+  description: <<END
+See explanations of candidate sampling and the data formats at
+go/candidate-sampling.
+
+For each batch, this op picks a single set of sampled candidate labels.
+
+The advantages of sampling candidates per-batch are simplicity and the
+possibility of efficient dense matrix multiplication. The disadvantage is that
+the sampled candidates must be chosen independently of the context and of the
+true labels.
+END
+}
+op {
+  graph_op_name: "Angle"
+  endpoint {
+    name: "Angle"
+  }
+  summary: "Returns the argument of a complex number."
+  description: <<END
+Given a tensor `input` of complex numbers, this operation returns a tensor of
+type `float` that is the argument of each element in `input`. All elements in
+`input` must be complex numbers of the form \\(a + bj\\), where *a*
+is the real part and *b* is the imaginary part.
+
+The argument returned by this operation is of the form \\(atan2(b, a)\\).
+
+For example:
+
+```
+# tensor 'input' is [-2.25 + 4.75j, 3.25 + 5.75j]
+tf.angle(input) ==> [2.0132, 1.056]
+```
+
+@compatibility(numpy)
+Equivalent to np.angle.
+@end_compatibility
+END
+}
+op {
+  graph_op_name: "Any"
+  endpoint {
+    name: "Any"
+  }
+  summary: "Computes the \"logical or\" of elements across dimensions of a tensor."
+  description: <<END
+Reduces `input` along the dimensions given in `reduction_indices`. Unless
+`keep_dims` is true, the rank of the tensor is reduced by 1 for each entry in
+`reduction_indices`. If `keep_dims` is true, the reduced dimensions are
+retained with length 1.
+END
+}
+op {
+  graph_op_name: "ApplyAdadelta"
+  endpoint {
+    name: "ApplyAdadelta"
+  }
+  summary: "Update \'*var\' according to the adadelta scheme."
+  description: <<END
+accum = rho() * accum + (1 - rho()) * grad.square();
+update = (update_accum + epsilon).sqrt() * (accum + epsilon()).rsqrt() * grad;
+update_accum = rho() * update_accum + (1 - rho()) * update.square();
+var -= update;
+END
+}
+op {
+  graph_op_name: "ApplyAdagrad"
+  endpoint {
+    name: "ApplyAdagrad"
+  }
+  summary: "Update \'*var\' according to the adagrad scheme."
+  description: <<END
+accum += grad * grad
+var -= lr * grad * (1 / sqrt(accum))
+END
+}
+op {
+  graph_op_name: "ApplyAdagradDA"
+  endpoint {
+    name: "ApplyAdagradDA"
+  }
+  summary: "Update \'*var\' according to the proximal adagrad scheme."
+}
+op {
+  graph_op_name: "ApplyAdam"
+  endpoint {
+    name: "ApplyAdam"
+  }
+  summary: "Update \'*var\' according to the Adam algorithm."
+  description: <<END
+lr_t <- learning_rate * sqrt(1 - beta2^t) / (1 - beta1^t)
+m_t <- beta1 * m_{t-1} + (1 - beta1) * g_t
+v_t <- beta2 * v_{t-1} + (1 - beta2) * g_t * g_t
+variable <- variable - lr_t * m_t / (sqrt(v_t) + epsilon)
+END
+}
+op {
+  graph_op_name: "ApplyCenteredRMSProp"
+  endpoint {
+    name: "ApplyCenteredRMSProp"
+  }
+  summary: "Update \'*var\' according to the centered RMSProp algorithm."
+  description: <<END
+The centered RMSProp algorithm uses an estimate of the centered second moment
+(i.e., the variance) for normalization, as opposed to regular RMSProp, which
+uses the (uncentered) second moment. This often helps with training, but is
+slightly more expensive in terms of computation and memory.
+
+Note that in dense implementation of this algorithm, mg, ms, and mom will
+update even if the grad is zero, but in this sparse implementation, mg, ms,
+and mom will not update in iterations during which the grad is zero.
+
+mean_square = decay * mean_square + (1-decay) * gradient ** 2
+mean_grad = decay * mean_grad + (1-decay) * gradient
+
+Delta = learning_rate * gradient / sqrt(mean_square + epsilon - mean_grad ** 2)
+
+mg <- rho * mg_{t-1} + (1-rho) * grad
+ms <- rho * ms_{t-1} + (1-rho) * grad * grad
+mom <- momentum * mom_{t-1} + lr * grad / sqrt(ms - mg * mg + epsilon)
+var <- var - mom
+END
+}
+op {
+  graph_op_name: "ApplyFtrl"
+  endpoint {
+    name: "ApplyFtrl"
+  }
+  summary: "Update \'*var\' according to the Ftrl-proximal scheme."
+  description: <<END
+accum_new = accum + grad * grad
+linear += grad + (accum_new^(-lr_power) - accum^(-lr_power)) / lr * var
+quadratic = 1.0 / (accum_new^(lr_power) * lr) + 2 * l2
+var = (sign(linear) * l1 - linear) / quadratic if |linear| > l1 else 0.0
+accum = accum_new
+END
+}
+op {
+  graph_op_name: "ApplyFtrlV2"
+  endpoint {
+    name: "ApplyFtrlV2"
+  }
+  summary: "Update \'*var\' according to the Ftrl-proximal scheme."
+  description: <<END
+grad_with_shrinkage = grad + 2 * l2_shrinkage * var
+accum_new = accum + grad_with_shrinkage * grad_with_shrinkage
+linear += grad_with_shrinkage +
+    (accum_new^(-lr_power) - accum^(-lr_power)) / lr * var
+quadratic = 1.0 / (accum_new^(lr_power) * lr) + 2 * l2
+var = (sign(linear) * l1 - linear) / quadratic if |linear| > l1 else 0.0
+accum = accum_new
+END
+}
+op {
+  graph_op_name: "ApplyGradientDescent"
+  endpoint {
+    name: "ApplyGradientDescent"
+  }
+  summary: "Update \'*var\' by subtracting \'alpha\' * \'delta\' from it."
+}
+op {
+  graph_op_name: "ApplyMomentum"
+  endpoint {
+    name: "ApplyMomentum"
+  }
+  summary: "Update \'*var\' according to the momentum scheme. Set use_nesterov = True if you"
+  description: <<END
+want to use Nesterov momentum.
+
+accum = accum * momentum + grad
+var -= lr * accum
+END
+}
+op {
+  graph_op_name: "ApplyProximalAdagrad"
+  endpoint {
+    name: "ApplyProximalAdagrad"
+  }
+  summary: "Update \'*var\' and \'*accum\' according to FOBOS with Adagrad learning rate."
+  description: <<END
+accum += grad * grad
+prox_v = var - lr * grad * (1 / sqrt(accum))
+var = sign(prox_v)/(1+lr*l2) * max{|prox_v|-lr*l1,0}
+END
+}
+op {
+  graph_op_name: "ApplyProximalGradientDescent"
+  endpoint {
+    name: "ApplyProximalGradientDescent"
+  }
+  summary: "Update \'*var\' as FOBOS algorithm with fixed learning rate."
+  description: <<END
+prox_v = var - alpha * delta
+var = sign(prox_v)/(1+alpha*l2) * max{|prox_v|-alpha*l1,0}
+END
+}
+op {
+  graph_op_name: "ApplyRMSProp"
+  endpoint {
+    name: "ApplyRMSProp"
+  }
+  summary: "Update \'*var\' according to the RMSProp algorithm."
+  description: <<END
+Note that in dense implementation of this algorithm, ms and mom will
+update even if the grad is zero, but in this sparse implementation, ms
+and mom will not update in iterations during which the grad is zero.
+
+mean_square = decay * mean_square + (1-decay) * gradient ** 2
+Delta = learning_rate * gradient / sqrt(mean_square + epsilon)
+
+ms <- rho * ms_{t-1} + (1-rho) * grad * grad
+mom <- momentum * mom_{t-1} + lr * grad / sqrt(ms + epsilon)
+var <- var - mom
+END
+}
+op {
+  graph_op_name: "ApproximateEqual"
+  endpoint {
+    name: "ApproximateEqual"
+  }
+  summary: "Returns the truth value of abs(x-y) < tolerance element-wise."
+}
+op {
+  graph_op_name: "ArgMax"
+  endpoint {
+    name: "ArgMax"
+  }
+  summary: "Returns the index with the largest value across dimensions of a tensor."
+  description: <<END
+Note that in case of ties the identity of the return value is not guaranteed.
+END
+}
+op {
+  graph_op_name: "ArgMin"
+  endpoint {
+    name: "ArgMin"
+  }
+  summary: "Returns the index with the smallest value across dimensions of a tensor."
+  description: <<END
+Note that in case of ties the identity of the return value is not guaranteed.
+END
+}
+op {
+  graph_op_name: "AsString"
+  endpoint {
+    name: "AsString"
+  }
+  summary: "Converts each entry in the given tensor to strings.  Supports many numeric"
+  description: <<END
+types and boolean.
+END
+}
+op {
+  graph_op_name: "Asin"
+  endpoint {
+    name: "Asin"
+  }
+  summary: "Computes asin of x element-wise."
+}
+op {
+  graph_op_name: "Asinh"
+  endpoint {
+    name: "Asinh"
+  }
+  summary: "Computes inverse hyperbolic sine of x element-wise."
+}
+op {
+  graph_op_name: "Assert"
+  endpoint {
+    name: "Assert"
+  }
+  summary: "Asserts that the given condition is true."
+  description: <<END
+If `condition` evaluates to false, print the list of tensors in `data`.
+`summarize` determines how many entries of the tensors to print.
+END
+}
+op {
+  graph_op_name: "Assign"
+  endpoint {
+    name: "Assign"
+  }
+  summary: "Update \'ref\' by assigning \'value\' to it."
+  description: <<END
+This operation outputs "ref" after the assignment is done.
+This makes it easier to chain operations that need to use the reset value.
+END
+}
+op {
+  graph_op_name: "AssignAdd"
+  endpoint {
+    name: "AssignAdd"
+  }
+  summary: "Update \'ref\' by adding \'value\' to it."
+  description: <<END
+This operation outputs "ref" after the update is done.
+This makes it easier to chain operations that need to use the reset value.
+END
+}
+op {
+  graph_op_name: "AssignSub"
+  endpoint {
+    name: "AssignSub"
+  }
+  summary: "Update \'ref\' by subtracting \'value\' from it."
+  description: <<END
+This operation outputs "ref" after the update is done.
+This makes it easier to chain operations that need to use the reset value.
+END
+}
+op {
+  graph_op_name: "Atan"
+  endpoint {
+    name: "Atan"
+  }
+  summary: "Computes atan of x element-wise."
+}
+op {
+  graph_op_name: "Atan2"
+  endpoint {
+    name: "Atan2"
+  }
+  summary: "Computes arctangent of `y/x` element-wise, respecting signs of the arguments."
+  description: <<END
+This is the angle \( \theta \in [-\pi, \pi] \) such that
+\[ x = r \cos(\theta) \]
+and
+\[ y = r \sin(\theta) \]
+where \(r = \sqrt(x^2 + y^2) \).
+END
+}
+op {
+  graph_op_name: "Atanh"
+  endpoint {
+    name: "Atanh"
+  }
+  summary: "Computes inverse hyperbolic tangent of x element-wise."
+}
+op {
+  graph_op_name: "AudioSpectrogram"
+  endpoint {
+    name: "AudioSpectrogram"
+  }
+  summary: "Produces a visualization of audio data over time."
+  description: <<END
+Spectrograms are a standard way of representing audio information as a series of
+slices of frequency information, one slice for each window of time. By joining
+these together into a sequence, they form a distinctive fingerprint of the sound
+over time.
+
+This op expects to receive audio data as an input, stored as floats in the range
+-1 to 1, together with a window width in samples, and a stride specifying how
+far to move the window between slices. From this it generates a three
+dimensional output. The lowest dimension has an amplitude value for each
+frequency during that time slice. The next dimension is time, with successive
+frequency slices. The final dimension is for the channels in the input, so a
+stereo audio input would have two here for example.
+
+This means the layout when converted and saved as an image is rotated 90 degrees
+clockwise from a typical spectrogram. Time is descending down the Y axis, and
+the frequency decreases from left to right.
+
+Each value in the result represents the square root of the sum of the real and
+imaginary parts of an FFT on the current window of samples. In this way, the
+lowest dimension represents the power of each frequency in the current window,
+and adjacent windows are concatenated in the next dimension.
+
+To get a more intuitive and visual look at what this operation does, you can run
+tensorflow/examples/wav_to_spectrogram to read in an audio file and save out the
+resulting spectrogram as a PNG image.
+END
+}
+op {
+  graph_op_name: "AudioSummary"
+  endpoint {
+    name: "AudioSummary"
+  }
+  summary: "Outputs a `Summary` protocol buffer with audio."
+  description: <<END
+The summary has up to `max_outputs` summary values containing audio. The
+audio is built from `tensor` which must be 3-D with shape `[batch_size,
+frames, channels]` or 2-D with shape `[batch_size, frames]`. The values are
+assumed to be in the range of `[-1.0, 1.0]` with a sample rate of `sample_rate`.
+
+The `tag` argument is a scalar `Tensor` of type `string`.  It is used to
+build the `tag` of the summary values:
+
+*  If `max_outputs` is 1, the summary value tag is '*tag*/audio'.
+*  If `max_outputs` is greater than 1, the summary value tags are
+   generated sequentially as '*tag*/audio/0', '*tag*/audio/1', etc.
+END
+}
+op {
+  graph_op_name: "AudioSummaryV2"
+  endpoint {
+    name: "AudioSummaryV2"
+  }
+  summary: "Outputs a `Summary` protocol buffer with audio."
+  description: <<END
+The summary has up to `max_outputs` summary values containing audio. The
+audio is built from `tensor` which must be 3-D with shape `[batch_size,
+frames, channels]` or 2-D with shape `[batch_size, frames]`. The values are
+assumed to be in the range of `[-1.0, 1.0]` with a sample rate of `sample_rate`.
+
+The `tag` argument is a scalar `Tensor` of type `string`.  It is used to
+build the `tag` of the summary values:
+
+*  If `max_outputs` is 1, the summary value tag is '*tag*/audio'.
+*  If `max_outputs` is greater than 1, the summary value tags are
+   generated sequentially as '*tag*/audio/0', '*tag*/audio/1', etc.
+END
+}
+op {
+  graph_op_name: "AvgPool"
+  endpoint {
+    name: "AvgPool"
+  }
+  summary: "Performs average pooling on the input."
+  description: <<END
+Each entry in `output` is the mean of the corresponding size `ksize`
+window in `value`.
+END
+}
+op {
+  graph_op_name: "AvgPool3D"
+  endpoint {
+    name: "AvgPool3D"
+  }
+  summary: "Performs 3D average pooling on the input."
+}
+op {
+  graph_op_name: "AvgPool3DGrad"
+  endpoint {
+    name: "AvgPool3DGrad"
+  }
+  summary: "Computes gradients of average pooling function."
+}
+op {
+  graph_op_name: "AvgPoolGrad"
+  endpoint {
+    name: "AvgPoolGrad"
+  }
+  summary: "Computes gradients of the average pooling function."
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_B.pbtxt b/tensorflow/core/api_def/base_api/api_def_B.pbtxt
new file mode 100644
index 0000000000..716d397f9a
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_B.pbtxt
@@ -0,0 +1,448 @@
+op {
+  graph_op_name: "Barrier"
+  endpoint {
+    name: "Barrier"
+  }
+  summary: "Defines a barrier that persists across different graph executions."
+  description: <<END
+A barrier represents a key-value map, where each key is a string, and
+each value is a tuple of tensors.
+
+At runtime, the barrier contains 'complete' and 'incomplete'
+elements. A complete element has defined tensors for all components of
+its value tuple, and may be accessed using BarrierTakeMany. An
+incomplete element has some undefined components in its value tuple,
+and may be updated using BarrierInsertMany.
+END
+}
+op {
+  graph_op_name: "BarrierClose"
+  endpoint {
+    name: "BarrierClose"
+  }
+  summary: "Closes the given barrier."
+  description: <<END
+This operation signals that no more new elements will be inserted in the
+given barrier. Subsequent InsertMany that try to introduce a new key will fail.
+Subsequent InsertMany operations that just add missing components to already
+existing elements will continue to succeed. Subsequent TakeMany operations will
+continue to succeed if sufficient completed elements remain in the barrier.
+Subsequent TakeMany operations that would block will fail immediately.
+END
+}
+op {
+  graph_op_name: "BarrierIncompleteSize"
+  endpoint {
+    name: "BarrierIncompleteSize"
+  }
+  summary: "Computes the number of incomplete elements in the given barrier."
+}
+op {
+  graph_op_name: "BarrierInsertMany"
+  endpoint {
+    name: "BarrierInsertMany"
+  }
+  summary: "For each key, assigns the respective value to the specified component."
+  description: <<END
+If a key is not found in the barrier, this operation will create a new
+incomplete element. If a key is found in the barrier, and the element
+already has a value at component_index, this operation will fail with
+INVALID_ARGUMENT, and leave the barrier in an undefined state.
+END
+}
+op {
+  graph_op_name: "BarrierReadySize"
+  endpoint {
+    name: "BarrierReadySize"
+  }
+  summary: "Computes the number of complete elements in the given barrier."
+}
+op {
+  graph_op_name: "BarrierTakeMany"
+  endpoint {
+    name: "BarrierTakeMany"
+  }
+  summary: "Takes the given number of completed elements from a barrier."
+  description: <<END
+This operation concatenates completed-element component tensors along
+the 0th dimension to make a single component tensor.
+
+Elements come out of the barrier when they are complete, and in the order
+in which they were placed into the barrier.  The indices output provides
+information about the batch in which each element was originally inserted
+into the barrier.
+END
+}
+op {
+  graph_op_name: "BatchCholesky"
+  endpoint {
+    name: "BatchCholesky"
+  }
+}
+op {
+  graph_op_name: "BatchCholeskyGrad"
+  endpoint {
+    name: "BatchCholeskyGrad"
+  }
+}
+op {
+  graph_op_name: "BatchDataset"
+  endpoint {
+    name: "BatchDataset"
+  }
+  summary: "Creates a dataset that batches `batch_size` elements from `input_dataset`."
+}
+op {
+  graph_op_name: "BatchFFT"
+  endpoint {
+    name: "BatchFFT"
+  }
+}
+op {
+  graph_op_name: "BatchFFT2D"
+  endpoint {
+    name: "BatchFFT2D"
+  }
+}
+op {
+  graph_op_name: "BatchFFT3D"
+  endpoint {
+    name: "BatchFFT3D"
+  }
+}
+op {
+  graph_op_name: "BatchIFFT"
+  endpoint {
+    name: "BatchIFFT"
+  }
+}
+op {
+  graph_op_name: "BatchIFFT2D"
+  endpoint {
+    name: "BatchIFFT2D"
+  }
+}
+op {
+  graph_op_name: "BatchIFFT3D"
+  endpoint {
+    name: "BatchIFFT3D"
+  }
+}
+op {
+  graph_op_name: "BatchMatMul"
+  endpoint {
+    name: "BatchMatMul"
+  }
+  summary: "Multiplies slices of two tensors in batches."
+  description: <<END
+Multiplies all slices of `Tensor` `x` and `y` (each slice can be
+viewed as an element of a batch), and arranges the individual results
+in a single output tensor of the same batch size. Each of the
+individual slices can optionally be adjointed (to adjoint a matrix
+means to transpose and conjugate it) before multiplication by setting
+the `adj_x` or `adj_y` flag to `True`, which are by default `False`.
+
+The input tensors `x` and `y` are 2-D or higher with shape `[..., r_x, c_x]`
+and `[..., r_y, c_y]`.
+
+The output tensor is 2-D or higher with shape `[..., r_o, c_o]`, where:
+
+    r_o = c_x if adj_x else r_x
+    c_o = r_y if adj_y else c_y
+
+It is computed as:
+
+    output[..., :, :] = matrix(x[..., :, :]) * matrix(y[..., :, :])
+END
+}
+op {
+  graph_op_name: "BatchMatrixBandPart"
+  endpoint {
+    name: "BatchMatrixBandPart"
+  }
+}
+op {
+  graph_op_name: "BatchMatrixDeterminant"
+  endpoint {
+    name: "BatchMatrixDeterminant"
+  }
+}
+op {
+  graph_op_name: "BatchMatrixDiag"
+  endpoint {
+    name: "BatchMatrixDiag"
+  }
+}
+op {
+  graph_op_name: "BatchMatrixDiagPart"
+  endpoint {
+    name: "BatchMatrixDiagPart"
+  }
+}
+op {
+  graph_op_name: "BatchMatrixInverse"
+  endpoint {
+    name: "BatchMatrixInverse"
+  }
+}
+op {
+  graph_op_name: "BatchMatrixSetDiag"
+  endpoint {
+    name: "BatchMatrixSetDiag"
+  }
+}
+op {
+  graph_op_name: "BatchMatrixSolve"
+  endpoint {
+    name: "BatchMatrixSolve"
+  }
+}
+op {
+  graph_op_name: "BatchMatrixSolveLs"
+  endpoint {
+    name: "BatchMatrixSolveLs"
+  }
+}
+op {
+  graph_op_name: "BatchMatrixTriangularSolve"
+  endpoint {
+    name: "BatchMatrixTriangularSolve"
+  }
+}
+op {
+  graph_op_name: "BatchNormWithGlobalNormalization"
+  endpoint {
+    name: "BatchNormWithGlobalNormalization"
+  }
+  summary: "Batch normalization."
+  description: <<END
+This op is deprecated. Prefer `tf.nn.batch_normalization`.
+END
+}
+op {
+  graph_op_name: "BatchNormWithGlobalNormalizationGrad"
+  endpoint {
+    name: "BatchNormWithGlobalNormalizationGrad"
+  }
+  summary: "Gradients for batch normalization."
+  description: <<END
+This op is deprecated. See `tf.nn.batch_normalization`.
+END
+}
+op {
+  graph_op_name: "BatchSelfAdjointEig"
+  endpoint {
+    name: "BatchSelfAdjointEig"
+  }
+}
+op {
+  graph_op_name: "BatchSelfAdjointEigV2"
+  endpoint {
+    name: "BatchSelfAdjointEigV2"
+  }
+}
+op {
+  graph_op_name: "BatchSvd"
+  endpoint {
+    name: "BatchSvd"
+  }
+}
+op {
+  graph_op_name: "BatchToSpace"
+  endpoint {
+    name: "BatchToSpace"
+  }
+  summary: "BatchToSpace for 4-D tensors of type T."
+  description: <<END
+This is a legacy version of the more general BatchToSpaceND.
+
+Rearranges (permutes) data from batch into blocks of spatial data, followed by
+cropping. This is the reverse transformation of SpaceToBatch. More specifically,
+this op outputs a copy of the input tensor where values from the `batch`
+dimension are moved in spatial blocks to the `height` and `width` dimensions,
+followed by cropping along the `height` and `width` dimensions.
+END
+}
+op {
+  graph_op_name: "BatchToSpaceND"
+  endpoint {
+    name: "BatchToSpaceND"
+  }
+  summary: "BatchToSpace for N-D tensors of type T."
+  description: <<END
+This operation reshapes the "batch" dimension 0 into `M + 1` dimensions of shape
+`block_shape + [batch]`, interleaves these blocks back into the grid defined by
+the spatial dimensions `[1, ..., M]`, to obtain a result with the same rank as
+the input.  The spatial dimensions of this intermediate result are then
+optionally cropped according to `crops` to produce the output.  This is the
+reverse of SpaceToBatch.  See below for a precise description.
+END
+}
+op {
+  graph_op_name: "Betainc"
+  endpoint {
+    name: "Betainc"
+  }
+  summary: "Compute the regularized incomplete beta integral \\\\(I_x(a, b)\\\\)."
+  description: <<END
+The regularized incomplete beta integral is defined as:
+
+
+\\(I_x(a, b) = \frac{B(x; a, b)}{B(a, b)}\\)
+
+where
+
+
+\\(B(x; a, b) = \int_0^x t^{a-1} (1 - t)^{b-1} dt\\)
+
+
+is the incomplete beta function and \\(B(a, b)\\) is the *complete*
+beta function.
+END
+}
+op {
+  graph_op_name: "BiasAdd"
+  endpoint {
+    name: "BiasAdd"
+  }
+  summary: "Adds `bias` to `value`."
+  description: <<END
+This is a special case of `tf.add` where `bias` is restricted to be 1-D.
+Broadcasting is supported, so `value` may have any number of dimensions.
+END
+}
+op {
+  graph_op_name: "BiasAddGrad"
+  endpoint {
+    name: "BiasAddGrad"
+  }
+  summary: "The backward operation for \"BiasAdd\" on the \"bias\" tensor."
+  description: <<END
+It accumulates all the values from out_backprop into the feature dimension.
+For NHWC data format, the feature dimension is the last. For NCHW data format,
+the feature dimension is the third-to-last.
+END
+}
+op {
+  graph_op_name: "BiasAddV1"
+  endpoint {
+    name: "BiasAddV1"
+  }
+  summary: "Adds `bias` to `value`."
+  description: <<END
+This is a deprecated version of BiasAdd and will be soon removed.
+
+This is a special case of `tf.add` where `bias` is restricted to be 1-D.
+Broadcasting is supported, so `value` may have any number of dimensions.
+END
+}
+op {
+  graph_op_name: "Bincount"
+  endpoint {
+    name: "Bincount"
+  }
+  summary: "Counts the number of occurrences of each value in an integer array."
+  description: <<END
+Outputs a vector with length `size` and the same dtype as `weights`. If
+`weights` are empty, then index `i` stores the number of times the value `i` is
+counted in `arr`. If `weights` are non-empty, then index `i` stores the sum of
+the value in `weights` at each index where the corresponding value in `arr` is
+`i`.
+
+Values in `arr` outside of the range [0, size) are ignored.
+END
+}
+op {
+  graph_op_name: "Bitcast"
+  endpoint {
+    name: "Bitcast"
+  }
+  summary: "Bitcasts a tensor from one type to another without copying data."
+  description: <<END
+Given a tensor `input`, this operation returns a tensor that has the same buffer
+data as `input` with datatype `type`.
+
+If the input datatype `T` is larger than the output datatype `type` then the
+shape changes from [...] to [..., sizeof(`T`)/sizeof(`type`)].
+
+If `T` is smaller than `type`, the operator requires that the rightmost
+dimension be equal to sizeof(`type`)/sizeof(`T`). The shape then goes from
+[..., sizeof(`type`)/sizeof(`T`)] to [...].
+
+*NOTE*: Bitcast is implemented as a low-level cast, so machines with different
+endian orderings will give different results.
+END
+}
+op {
+  graph_op_name: "BitwiseAnd"
+  endpoint {
+    name: "BitwiseAnd"
+  }
+  summary: "Elementwise computes the bitwise AND of `x` and `y`."
+  description: <<END
+The result will have those bits set, that are set in both `x` and `y`. The
+computation is performed on the underlying representations of `x` and `y`.
+END
+}
+op {
+  graph_op_name: "BitwiseOr"
+  endpoint {
+    name: "BitwiseOr"
+  }
+  summary: "Elementwise computes the bitwise OR of `x` and `y`."
+  description: <<END
+The result will have those bits set, that are set in `x`, `y` or both. The
+computation is performed on the underlying representations of `x` and `y`.
+END
+}
+op {
+  graph_op_name: "BitwiseXor"
+  endpoint {
+    name: "BitwiseXor"
+  }
+  summary: "Elementwise computes the bitwise XOR of `x` and `y`."
+  description: <<END
+The result will have those bits set, that are different in `x` and `y`. The
+computation is performed on the underlying representations of `x` and `y`.
+END
+}
+op {
+  graph_op_name: "BroadcastArgs"
+  endpoint {
+    name: "BroadcastArgs"
+  }
+  summary: "Return the shape of s0 op s1 with broadcast."
+  description: <<END
+Given `s0` and `s1`, tensors that represent shapes, compute `r0`, the
+broadcasted shape. `s0`, `s1` and `r0` are all integer vectors.
+END
+}
+op {
+  graph_op_name: "BroadcastGradientArgs"
+  endpoint {
+    name: "BroadcastGradientArgs"
+  }
+  summary: "Return the reduction indices for computing gradients of s0 op s1 with broadcast."
+  description: <<END
+This is typically used by gradient computations for a broadcasting operation.
+END
+}
+op {
+  graph_op_name: "Bucketize"
+  endpoint {
+    name: "Bucketize"
+  }
+  summary: "Bucketizes \'input\' based on \'boundaries\'."
+  description: <<END
+For example, if the inputs are
+    boundaries = [0, 10, 100]
+    input = [[-5, 10000]
+             [150,   10]
+             [5,    100]]
+
+then the output will be
+    output = [[0, 3]
+              [3, 2]
+              [1, 3]]
+END
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_C.pbtxt b/tensorflow/core/api_def/base_api/api_def_C.pbtxt
new file mode 100644
index 0000000000..48b04b7971
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_C.pbtxt
@@ -0,0 +1,513 @@
+op {
+  graph_op_name: "CTCBeamSearchDecoder"
+  endpoint {
+    name: "CTCBeamSearchDecoder"
+  }
+  summary: "Performs beam search decoding on the logits given in input."
+  description: <<END
+A note about the attribute merge_repeated: For the beam search decoder,
+this means that if consecutive entries in a beam are the same, only
+the first of these is emitted.  That is, when the top path is "A B B B B",
+"A B" is returned if merge_repeated = True but "A B B B B" is
+returned if merge_repeated = False.
+END
+}
+op {
+  graph_op_name: "CTCGreedyDecoder"
+  endpoint {
+    name: "CTCGreedyDecoder"
+  }
+  summary: "Performs greedy decoding on the logits given in inputs."
+  description: <<END
+A note about the attribute merge_repeated: if enabled, when
+consecutive logits' maximum indices are the same, only the first of
+these is emitted.  Labeling the blank '*', the sequence "A B B * B B"
+becomes "A B B" if merge_repeated = True and "A B B B B" if
+merge_repeated = False.
+
+Regardless of the value of merge_repeated, if the maximum index of a given
+time and batch corresponds to the blank, index `(num_classes - 1)`, no new
+element is emitted.
+END
+}
+op {
+  graph_op_name: "CTCLoss"
+  endpoint {
+    name: "CTCLoss"
+  }
+  summary: "Calculates the CTC Loss (log probability) for each batch entry.  Also calculates"
+  description: <<END
+the gradient.  This class performs the softmax operation for you, so inputs
+should be e.g. linear projections of outputs by an LSTM.
+END
+}
+op {
+  graph_op_name: "CacheDataset"
+  endpoint {
+    name: "CacheDataset"
+  }
+  summary: "Creates a dataset that caches elements from `input_dataset`."
+  description: <<END
+A CacheDataset will iterate over the input_dataset, and store tensors. If the
+cache already exists, the cache will be used. If the cache is inappropriate
+(e.g. cannot be opened, contains tensors of the wrong shape / size), an error
+will the returned when used.
+END
+}
+op {
+  graph_op_name: "Cast"
+  endpoint {
+    name: "Cast"
+  }
+  summary: "Cast x of type SrcT to y of DstT."
+}
+op {
+  graph_op_name: "Ceil"
+  endpoint {
+    name: "Ceil"
+  }
+  summary: "Returns element-wise smallest integer in not less than x."
+}
+op {
+  graph_op_name: "CheckNumerics"
+  endpoint {
+    name: "CheckNumerics"
+  }
+  summary: "Checks a tensor for NaN and Inf values."
+  description: <<END
+When run, reports an `InvalidArgument` error if `tensor` has any values
+that are not a number (NaN) or infinity (Inf). Otherwise, passes `tensor` as-is.
+END
+}
+op {
+  graph_op_name: "Cholesky"
+  endpoint {
+    name: "Cholesky"
+  }
+  summary: "Computes the Cholesky decomposition of one or more square matrices."
+  description: <<END
+The input is a tensor of shape `[..., M, M]` whose inner-most 2 dimensions
+form square matrices.
+
+The input has to be symmetric and positive definite. Only the lower-triangular
+part of the input will be used for this operation. The upper-triangular part
+will not be read.
+
+The output is a tensor of the same shape as the input
+containing the Cholesky decompositions for all input submatrices `[..., :, :]`.
+
+**Note**: The gradient computation on GPU is faster for large matrices but
+not for large batch dimensions when the submatrices are small. In this
+case it might be faster to use the CPU.
+END
+}
+op {
+  graph_op_name: "CholeskyGrad"
+  endpoint {
+    name: "CholeskyGrad"
+  }
+  summary: "Computes the reverse mode backpropagated gradient of the Cholesky algorithm."
+  description: <<END
+For an explanation see "Differentiation of the Cholesky algorithm" by
+Iain Murray http://arxiv.org/abs/1602.07527.
+END
+}
+op {
+  graph_op_name: "CompareAndBitpack"
+  endpoint {
+    name: "CompareAndBitpack"
+  }
+  summary: "Compare values of `input` to `threshold` and pack resulting bits into a `uint8`."
+  description: <<END
+Each comparison returns a boolean `true` (if `input_value > threshold`)
+or and `false` otherwise.
+
+This operation is useful for Locality-Sensitive-Hashing (LSH) and other
+algorithms that use hashing approximations of cosine and `L2` distances;
+codes can be generated from an input via:
+
+```python
+codebook_size = 50
+codebook_bits = codebook_size * 32
+codebook = tf.get_variable('codebook', [x.shape[-1].value, codebook_bits],
+                           dtype=x.dtype,
+                           initializer=tf.orthogonal_initializer())
+codes = compare_and_threshold(tf.matmul(x, codebook), threshold=0.)
+codes = tf.bitcast(codes, tf.int32)  # go from uint8 to int32
+# now codes has shape x.shape[:-1] + [codebook_size]
+```
+
+**NOTE**: Currently, the innermost dimension of the tensor must be divisible
+by 8.
+
+Given an `input` shaped `[s0, s1, ..., s_n]`, the output is
+a `uint8` tensor shaped `[s0, s1, ..., s_n / 8]`.
+END
+}
+op {
+  graph_op_name: "Complex"
+  endpoint {
+    name: "Complex"
+  }
+  summary: "Converts two real numbers to a complex number."
+  description: <<END
+Given a tensor `real` representing the real part of a complex number, and a
+tensor `imag` representing the imaginary part of a complex number, this
+operation returns complex numbers elementwise of the form \\(a + bj\\), where
+*a* represents the `real` part and *b* represents the `imag` part.
+
+The input tensors `real` and `imag` must have the same shape.
+
+For example:
+
+```
+# tensor 'real' is [2.25, 3.25]
+# tensor `imag` is [4.75, 5.75]
+tf.complex(real, imag) ==> [[2.25 + 4.75j], [3.25 + 5.75j]]
+```
+END
+}
+op {
+  graph_op_name: "ComplexAbs"
+  endpoint {
+    name: "ComplexAbs"
+  }
+  summary: "Computes the complex absolute value of a tensor."
+  description: <<END
+Given a tensor `x` of complex numbers, this operation returns a tensor of type
+`float` or `double` that is the absolute value of each element in `x`. All
+elements in `x` must be complex numbers of the form \\(a + bj\\). The absolute
+value is computed as \\( \sqrt{a^2 + b^2}\\).
+END
+}
+op {
+  graph_op_name: "ComputeAccidentalHits"
+  endpoint {
+    name: "ComputeAccidentalHits"
+  }
+  summary: "Computes the ids of the positions in sampled_candidates that match true_labels."
+  description: <<END
+When doing log-odds NCE, the result of this op should be passed through a
+SparseToDense op, then added to the logits of the sampled candidates. This has
+the effect of 'removing' the sampled labels that match the true labels by
+making the classifier sure that they are sampled labels.
+END
+}
+op {
+  graph_op_name: "Concat"
+  endpoint {
+    name: "Concat"
+  }
+  summary: "Concatenates tensors along one dimension."
+}
+op {
+  graph_op_name: "ConcatOffset"
+  endpoint {
+    name: "ConcatOffset"
+  }
+  summary: "Computes offsets of concat inputs within its output."
+  description: <<END
+For example:
+
+```
+# 'x' is [2, 2, 7]
+# 'y' is [2, 3, 7]
+# 'z' is [2, 5, 7]
+concat_offset(2, [x, y, z]) => [0, 0, 0], [0, 2, 0], [0, 5, 0]
+```
+
+This is typically used by gradient computations for a concat operation.
+END
+}
+op {
+  graph_op_name: "ConcatV2"
+  endpoint {
+    name: "ConcatV2"
+  }
+  summary: "Concatenates tensors along one dimension."
+}
+op {
+  graph_op_name: "ConcatenateDataset"
+  endpoint {
+    name: "ConcatenateDataset"
+  }
+  summary: "Creates a dataset that concatenates `input_dataset` with `another_dataset`."
+}
+op {
+  graph_op_name: "ConditionalAccumulator"
+  endpoint {
+    name: "ConditionalAccumulator"
+  }
+  summary: "A conditional accumulator for aggregating gradients."
+  description: <<END
+The accumulator accepts gradients marked with local_step greater or
+equal to the most recent global_step known to the accumulator. The
+average can be extracted from the accumulator, provided sufficient
+gradients have been accumulated. Extracting the average automatically
+resets the aggregate to 0, and increments the global_step recorded by
+the accumulator.
+END
+}
+op {
+  graph_op_name: "Conj"
+  endpoint {
+    name: "Conj"
+  }
+  summary: "Returns the complex conjugate of a complex number."
+  description: <<END
+Given a tensor `input` of complex numbers, this operation returns a tensor of
+complex numbers that are the complex conjugate of each element in `input`. The
+complex numbers in `input` must be of the form \\(a + bj\\), where *a* is the
+real part and *b* is the imaginary part.
+
+The complex conjugate returned by this operation is of the form \\(a - bj\\).
+
+For example:
+
+```
+# tensor 'input' is [-2.25 + 4.75j, 3.25 + 5.75j]
+tf.conj(input) ==> [-2.25 - 4.75j, 3.25 - 5.75j]
+```
+END
+}
+op {
+  graph_op_name: "Const"
+  endpoint {
+    name: "Const"
+  }
+  summary: "Returns a constant tensor."
+}
+op {
+  graph_op_name: "ControlTrigger"
+  endpoint {
+    name: "ControlTrigger"
+  }
+  summary: "Does nothing. Serves as a control trigger for scheduling."
+  description: <<END
+Only useful as a placeholder for control edges.
+END
+}
+op {
+  graph_op_name: "Conv2D"
+  endpoint {
+    name: "Conv2D"
+  }
+  summary: "Computes a 2-D convolution given 4-D `input` and `filter` tensors."
+  description: <<END
+Given an input tensor of shape `[batch, in_height, in_width, in_channels]`
+and a filter / kernel tensor of shape
+`[filter_height, filter_width, in_channels, out_channels]`, this op
+performs the following:
+
+1. Flattens the filter to a 2-D matrix with shape
+   `[filter_height * filter_width * in_channels, output_channels]`.
+2. Extracts image patches from the input tensor to form a *virtual*
+   tensor of shape `[batch, out_height, out_width,
+   filter_height * filter_width * in_channels]`.
+3. For each patch, right-multiplies the filter matrix and the image patch
+   vector.
+
+In detail, with the default NHWC format,
+
+    output[b, i, j, k] =
+        sum_{di, dj, q} input[b, strides[1] * i + di, strides[2] * j + dj, q] *
+                        filter[di, dj, q, k]
+
+Must have `strides[0] = strides[3] = 1`.  For the most common case of the same
+horizontal and vertices strides, `strides = [1, stride, stride, 1]`.
+END
+}
+op {
+  graph_op_name: "Conv2DBackpropFilter"
+  endpoint {
+    name: "Conv2DBackpropFilter"
+  }
+  summary: "Computes the gradients of convolution with respect to the filter."
+}
+op {
+  graph_op_name: "Conv2DBackpropInput"
+  endpoint {
+    name: "Conv2DBackpropInput"
+  }
+  summary: "Computes the gradients of convolution with respect to the input."
+}
+op {
+  graph_op_name: "Conv3D"
+  endpoint {
+    name: "Conv3D"
+  }
+  summary: "Computes a 3-D convolution given 5-D `input` and `filter` tensors."
+  description: <<END
+In signal processing, cross-correlation is a measure of similarity of
+two waveforms as a function of a time-lag applied to one of them. This
+is also known as a sliding dot product or sliding inner-product.
+
+Our Conv3D implements a form of cross-correlation.
+END
+}
+op {
+  graph_op_name: "Conv3DBackpropFilter"
+  endpoint {
+    name: "Conv3DBackpropFilter"
+  }
+  summary: "Computes the gradients of 3-D convolution with respect to the filter."
+}
+op {
+  graph_op_name: "Conv3DBackpropFilterV2"
+  endpoint {
+    name: "Conv3DBackpropFilterV2"
+  }
+  summary: "Computes the gradients of 3-D convolution with respect to the filter."
+}
+op {
+  graph_op_name: "Conv3DBackpropInput"
+  endpoint {
+    name: "Conv3DBackpropInput"
+  }
+  summary: "Computes the gradients of 3-D convolution with respect to the input."
+}
+op {
+  graph_op_name: "Conv3DBackpropInputV2"
+  endpoint {
+    name: "Conv3DBackpropInputV2"
+  }
+  summary: "Computes the gradients of 3-D convolution with respect to the input."
+}
+op {
+  graph_op_name: "Cos"
+  endpoint {
+    name: "Cos"
+  }
+  summary: "Computes cos of x element-wise."
+}
+op {
+  graph_op_name: "Cosh"
+  endpoint {
+    name: "Cosh"
+  }
+  summary: "Computes hyperbolic cosine of x element-wise."
+}
+op {
+  graph_op_name: "CountUpTo"
+  endpoint {
+    name: "CountUpTo"
+  }
+  summary: "Increments \'ref\' until it reaches \'limit\'."
+}
+op {
+  graph_op_name: "CropAndResize"
+  endpoint {
+    name: "CropAndResize"
+  }
+  summary: "Extracts crops from the input image tensor and bilinearly resizes them (possibly"
+  description: <<END
+with aspect ratio change) to a common output size specified by `crop_size`. This
+is more general than the `crop_to_bounding_box` op which extracts a fixed size
+slice from the input image and does not allow resizing or aspect ratio change.
+
+Returns a tensor with `crops` from the input `image` at positions defined at the
+bounding box locations in `boxes`. The cropped boxes are all resized (with
+bilinear interpolation) to a fixed `size = [crop_height, crop_width]`. The
+result is a 4-D tensor `[num_boxes, crop_height, crop_width, depth]`.
+END
+}
+op {
+  graph_op_name: "CropAndResizeGradBoxes"
+  endpoint {
+    name: "CropAndResizeGradBoxes"
+  }
+  summary: "Computes the gradient of the crop_and_resize op wrt the input boxes tensor."
+}
+op {
+  graph_op_name: "CropAndResizeGradImage"
+  endpoint {
+    name: "CropAndResizeGradImage"
+  }
+  summary: "Computes the gradient of the crop_and_resize op wrt the input image tensor."
+}
+op {
+  graph_op_name: "Cross"
+  endpoint {
+    name: "Cross"
+  }
+  summary: "Compute the pairwise cross product."
+  description: <<END
+`a` and `b` must be the same shape; they can either be simple 3-element vectors,
+or any shape where the innermost dimension is 3. In the latter case, each pair
+of corresponding 3-element vectors is cross-multiplied independently.
+END
+}
+op {
+  graph_op_name: "Cumprod"
+  endpoint {
+    name: "Cumprod"
+  }
+  summary: "Compute the cumulative product of the tensor `x` along `axis`."
+  description: <<END
+By default, this op performs an inclusive cumprod, which means that the first
+element of the input is identical to the first element of the output:
+
+```python
+tf.cumprod([a, b, c])  # => [a, a * b, a * b * c]
+```
+
+By setting the `exclusive` kwarg to `True`, an exclusive cumprod is
+performed instead:
+
+```python
+tf.cumprod([a, b, c], exclusive=True)  # => [1, a, a * b]
+```
+
+By setting the `reverse` kwarg to `True`, the cumprod is performed in the
+opposite direction:
+
+```python
+tf.cumprod([a, b, c], reverse=True)  # => [a * b * c, b * c, c]
+```
+
+This is more efficient than using separate `tf.reverse` ops.
+
+The `reverse` and `exclusive` kwargs can also be combined:
+
+```python
+tf.cumprod([a, b, c], exclusive=True, reverse=True)  # => [b * c, c, 1]
+```
+END
+}
+op {
+  graph_op_name: "Cumsum"
+  endpoint {
+    name: "Cumsum"
+  }
+  summary: "Compute the cumulative sum of the tensor `x` along `axis`."
+  description: <<END
+By default, this op performs an inclusive cumsum, which means that the first
+element of the input is identical to the first element of the output:
+
+```python
+tf.cumsum([a, b, c])  # => [a, a + b, a + b + c]
+```
+
+By setting the `exclusive` kwarg to `True`, an exclusive cumsum is
+performed instead:
+
+```python
+tf.cumsum([a, b, c], exclusive=True)  # => [0, a, a + b]
+```
+
+By setting the `reverse` kwarg to `True`, the cumsum is performed in the
+opposite direction:
+
+```python
+tf.cumsum([a, b, c], reverse=True)  # => [a + b + c, b + c, c]
+```
+
+This is more efficient than using separate `tf.reverse` ops.
+
+The `reverse` and `exclusive` kwargs can also be combined:
+
+```python
+tf.cumsum([a, b, c], exclusive=True, reverse=True)  # => [b + c, c, 0]
+```
+END
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_D.pbtxt b/tensorflow/core/api_def/base_api/api_def_D.pbtxt
new file mode 100644
index 0000000000..ff8a7223c7
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_D.pbtxt
@@ -0,0 +1,790 @@
+op {
+  graph_op_name: "DebugGradientIdentity"
+  endpoint {
+    name: "DebugGradientIdentity"
+  }
+  summary: "Identity op for gradient debugging."
+  description: <<END
+This op is hidden from public in Python. It is used by TensorFlow Debugger to
+register gradient tensors for gradient debugging.
+END
+}
+op {
+  graph_op_name: "DecodeAndCropJpeg"
+  endpoint {
+    name: "DecodeAndCropJpeg"
+  }
+  summary: "Decode and Crop a JPEG-encoded image to a uint8 tensor."
+  description: <<END
+The attr `channels` indicates the desired number of color channels for the
+decoded image.
+
+Accepted values are:
+
+*   0: Use the number of channels in the JPEG-encoded image.
+*   1: output a grayscale image.
+*   3: output an RGB image.
+
+If needed, the JPEG-encoded image is transformed to match the requested number
+of color channels.
+
+The attr `ratio` allows downscaling the image by an integer factor during
+decoding.  Allowed values are: 1, 2, 4, and 8.  This is much faster than
+downscaling the image later.
+
+
+It is equivalent to a combination of decode and crop, but much faster by only
+decoding partial jpeg image.
+END
+}
+op {
+  graph_op_name: "DecodeBase64"
+  endpoint {
+    name: "DecodeBase64"
+  }
+  summary: "Decode web-safe base64-encoded strings."
+  description: <<END
+Input may or may not have padding at the end. See EncodeBase64 for padding.
+Web-safe means that input must use - and _ instead of + and /.
+END
+}
+op {
+  graph_op_name: "DecodeBmp"
+  endpoint {
+    name: "DecodeBmp"
+  }
+  summary: "Decode the first frame of a BMP-encoded image to a uint8 tensor."
+  description: <<END
+The attr `channels` indicates the desired number of color channels for the
+decoded image.
+
+Accepted values are:
+
+*   0: Use the number of channels in the BMP-encoded image.
+*   3: output an RGB image.
+*   4: output an RGBA image.
+END
+}
+op {
+  graph_op_name: "DecodeCSV"
+  endpoint {
+    name: "DecodeCSV"
+  }
+  summary: "Convert CSV records to tensors. Each column maps to one tensor."
+  description: <<END
+RFC 4180 format is expected for the CSV records.
+(https://tools.ietf.org/html/rfc4180)
+Note that we allow leading and trailing spaces with int or float field.
+END
+}
+op {
+  graph_op_name: "DecodeGif"
+  endpoint {
+    name: "DecodeGif"
+  }
+  summary: "Decode the first frame of a GIF-encoded image to a uint8 tensor."
+  description: <<END
+GIF with frame or transparency compression are not supported
+convert animated GIF from compressed to uncompressed by:
+
+    convert $src.gif -coalesce $dst.gif
+
+This op also supports decoding JPEGs and PNGs, though it is cleaner to use
+`tf.image.decode_image`.
+END
+}
+op {
+  graph_op_name: "DecodeJSONExample"
+  endpoint {
+    name: "DecodeJSONExample"
+  }
+  summary: "Convert JSON-encoded Example records to binary protocol buffer strings."
+  description: <<END
+This op translates a tensor containing Example records, encoded using
+the [standard JSON
+mapping](https://developers.google.com/protocol-buffers/docs/proto3#json),
+into a tensor containing the same records encoded as binary protocol
+buffers. The resulting tensor can then be fed to any of the other
+Example-parsing ops.
+END
+}
+op {
+  graph_op_name: "DecodeJpeg"
+  endpoint {
+    name: "DecodeJpeg"
+  }
+  summary: "Decode a JPEG-encoded image to a uint8 tensor."
+  description: <<END
+The attr `channels` indicates the desired number of color channels for the
+decoded image.
+
+Accepted values are:
+
+*   0: Use the number of channels in the JPEG-encoded image.
+*   1: output a grayscale image.
+*   3: output an RGB image.
+
+If needed, the JPEG-encoded image is transformed to match the requested number
+of color channels.
+
+The attr `ratio` allows downscaling the image by an integer factor during
+decoding.  Allowed values are: 1, 2, 4, and 8.  This is much faster than
+downscaling the image later.
+
+
+This op also supports decoding PNGs and non-animated GIFs since the interface is
+the same, though it is cleaner to use `tf.image.decode_image`.
+END
+}
+op {
+  graph_op_name: "DecodePng"
+  endpoint {
+    name: "DecodePng"
+  }
+  summary: "Decode a PNG-encoded image to a uint8 or uint16 tensor."
+  description: <<END
+The attr `channels` indicates the desired number of color channels for the
+decoded image.
+
+Accepted values are:
+
+*   0: Use the number of channels in the PNG-encoded image.
+*   1: output a grayscale image.
+*   3: output an RGB image.
+*   4: output an RGBA image.
+
+If needed, the PNG-encoded image is transformed to match the requested number
+of color channels.
+
+This op also supports decoding JPEGs and non-animated GIFs since the interface
+is the same, though it is cleaner to use `tf.image.decode_image`.
+END
+}
+op {
+  graph_op_name: "DecodeRaw"
+  endpoint {
+    name: "DecodeRaw"
+  }
+  summary: "Reinterpret the bytes of a string as a vector of numbers."
+}
+op {
+  graph_op_name: "DecodeWav"
+  endpoint {
+    name: "DecodeWav"
+  }
+  summary: "Decode a 16-bit PCM WAV file to a float tensor."
+  description: <<END
+The -32768 to 32767 signed 16-bit values will be scaled to -1.0 to 1.0 in float.
+
+When desired_channels is set, if the input contains fewer channels than this
+then the last channel will be duplicated to give the requested number, else if
+the input has more channels than requested then the additional channels will be
+ignored.
+
+If desired_samples is set, then the audio will be cropped or padded with zeroes
+to the requested length.
+
+The first output contains a Tensor with the content of the audio samples. The
+lowest dimension will be the number of channels, and the second will be the
+number of samples. For example, a ten-sample-long stereo WAV file should give an
+output shape of [10, 2].
+END
+}
+op {
+  graph_op_name: "DeleteSessionTensor"
+  endpoint {
+    name: "DeleteSessionTensor"
+  }
+  summary: "Delete the tensor specified by its handle in the session."
+}
+op {
+  graph_op_name: "DenseToDenseSetOperation"
+  endpoint {
+    name: "DenseToDenseSetOperation"
+  }
+  summary: "Applies set operation along last dimension of 2 `Tensor` inputs."
+  description: <<END
+See SetOperationOp::SetOperationFromContext for values of `set_operation`.
+
+Output `result` is a `SparseTensor` represented by `result_indices`,
+`result_values`, and `result_shape`. For `set1` and `set2` ranked `n`, this
+has rank `n` and the same 1st `n-1` dimensions as `set1` and `set2`. The `nth`
+dimension contains the result of `set_operation` applied to the corresponding
+`[0...n-1]` dimension of `set`.
+END
+}
+op {
+  graph_op_name: "DenseToSparseBatchDataset"
+  endpoint {
+    name: "DenseToSparseBatchDataset"
+  }
+  summary: "Creates a dataset that yields a SparseTensor for each element of the input."
+}
+op {
+  graph_op_name: "DenseToSparseSetOperation"
+  endpoint {
+    name: "DenseToSparseSetOperation"
+  }
+  summary: "Applies set operation along last dimension of `Tensor` and `SparseTensor`."
+  description: <<END
+See SetOperationOp::SetOperationFromContext for values of `set_operation`.
+
+Input `set2` is a `SparseTensor` represented by `set2_indices`, `set2_values`,
+and `set2_shape`. For `set2` ranked `n`, 1st `n-1` dimensions must be the same
+as `set1`. Dimension `n` contains values in a set, duplicates are allowed but
+ignored.
+
+If `validate_indices` is `True`, this op validates the order and range of `set2`
+indices.
+
+Output `result` is a `SparseTensor` represented by `result_indices`,
+`result_values`, and `result_shape`. For `set1` and `set2` ranked `n`, this
+has rank `n` and the same 1st `n-1` dimensions as `set1` and `set2`. The `nth`
+dimension contains the result of `set_operation` applied to the corresponding
+`[0...n-1]` dimension of `set`.
+END
+}
+op {
+  graph_op_name: "DepthToSpace"
+  endpoint {
+    name: "DepthToSpace"
+  }
+  summary: "DepthToSpace for tensors of type T."
+  description: <<END
+Rearranges data from depth into blocks of spatial data.
+This is the reverse transformation of SpaceToDepth. More specifically,
+this op outputs a copy of the input tensor where values from the `depth`
+dimension are moved in spatial blocks to the `height` and `width` dimensions.
+The attr `block_size` indicates the input block size and how the data is moved.
+
+  * Chunks of data of size `block_size * block_size` from depth are rearranged
+    into non-overlapping blocks of size `block_size x block_size`
+  * The width the output tensor is `input_depth * block_size`, whereas the
+    height is `input_height * block_size`.
+  * The Y, X coordinates within each block of the output image are determined
+    by the high order component of the input channel index.
+  * The depth of the input tensor must be divisible by
+    `block_size * block_size`.
+
+The `data_format` attr specifies the layout of the input and output tensors
+with the following options:
+  "NHWC": `[ batch, height, width, channels ]`
+  "NCHW": `[ batch, channels, height, width ]`
+  "NCHW_VECT_C":
+      `qint8 [ batch, channels / 4, height, width, channels % 4 ]`
+
+It is useful to consider the operation as transforming a 6-D Tensor.
+e.g. for data_format = NHWC,
+     Each element in the input tensor can be specified via 6 coordinates,
+     ordered by decreasing memory layout significance as:
+     n,iY,iX,bY,bX,oC  (where n=batch index, iX, iY means X or Y coordinates
+                        within the input image, bX, bY means coordinates
+                        within the output block, oC means output channels).
+     The output would be the input transposed to the following layout:
+     n,iY,bY,iX,bX,oC
+
+This operation is useful for resizing the activations between convolutions
+(but keeping all data), e.g. instead of pooling. It is also useful for training
+purely convolutional models.
+
+For example, given an input of shape `[1, 1, 1, 4]`, data_format = "NHWC" and
+block_size = 2:
+
+```
+x = [[[[1, 2, 3, 4]]]]
+
+```
+
+This operation will output a tensor of shape `[1, 2, 2, 1]`:
+
+```
+   [[[[1], [2]],
+     [[3], [4]]]]
+```
+
+Here, the input has a batch of 1 and each batch element has shape `[1, 1, 4]`,
+the corresponding output will have 2x2 elements and will have a depth of
+1 channel (1 = `4 / (block_size * block_size)`).
+The output element shape is `[2, 2, 1]`.
+
+For an input tensor with larger depth, here of shape `[1, 1, 1, 12]`, e.g.
+
+```
+x = [[[[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]]]]
+```
+
+This operation, for block size of 2, will return the following tensor of shape
+`[1, 2, 2, 3]`
+
+```
+   [[[[1, 2, 3], [4, 5, 6]],
+     [[7, 8, 9], [10, 11, 12]]]]
+
+```
+
+Similarly, for the following input of shape `[1 2 2 4]`, and a block size of 2:
+
+```
+x =  [[[[1, 2, 3, 4],
+       [5, 6, 7, 8]],
+      [[9, 10, 11, 12],
+       [13, 14, 15, 16]]]]
+```
+
+the operator will return the following tensor of shape `[1 4 4 1]`:
+
+```
+x = [[[ [1],   [2],  [5],  [6]],
+      [ [3],   [4],  [7],  [8]],
+      [ [9],  [10], [13],  [14]],
+      [ [11], [12], [15],  [16]]]]
+
+```
+END
+}
+op {
+  graph_op_name: "DepthwiseConv2dNative"
+  endpoint {
+    name: "DepthwiseConv2dNative"
+  }
+  summary: "Computes a 2-D depthwise convolution given 4-D `input` and `filter` tensors."
+  description: <<END
+Given an input tensor of shape `[batch, in_height, in_width, in_channels]`
+and a filter / kernel tensor of shape
+`[filter_height, filter_width, in_channels, channel_multiplier]`, containing
+`in_channels` convolutional filters of depth 1, `depthwise_conv2d` applies
+a different filter to each input channel (expanding from 1 channel to
+`channel_multiplier` channels for each), then concatenates the results
+together. Thus, the output has `in_channels * channel_multiplier` channels.
+
+```
+for k in 0..in_channels-1
+  for q in 0..channel_multiplier-1
+    output[b, i, j, k * channel_multiplier + q] =
+      sum_{di, dj} input[b, strides[1] * i + di, strides[2] * j + dj, k] *
+                        filter[di, dj, k, q]
+```
+
+Must have `strides[0] = strides[3] = 1`.  For the most common case of the same
+horizontal and vertices strides, `strides = [1, stride, stride, 1]`.
+END
+}
+op {
+  graph_op_name: "DepthwiseConv2dNativeBackpropFilter"
+  endpoint {
+    name: "DepthwiseConv2dNativeBackpropFilter"
+  }
+  summary: "Computes the gradients of depthwise convolution with respect to the filter."
+}
+op {
+  graph_op_name: "DepthwiseConv2dNativeBackpropInput"
+  endpoint {
+    name: "DepthwiseConv2dNativeBackpropInput"
+  }
+  summary: "Computes the gradients of depthwise convolution with respect to the input."
+}
+op {
+  graph_op_name: "Dequantize"
+  endpoint {
+    name: "Dequantize"
+  }
+  summary: "Dequantize the \'input\' tensor into a float Tensor."
+  description: <<END
+[min_range, max_range] are scalar floats that specify the range for
+the 'input' data. The 'mode' attribute controls exactly which calculations are
+used to convert the float values to their quantized equivalents.
+
+In 'MIN_COMBINED' mode, each value of the tensor will undergo the following:
+
+```
+if T == qint8, in[i] += (range(T) + 1)/ 2.0
+out[i] = min_range + (in[i]* (max_range - min_range) / range(T))
+```
+here `range(T) = numeric_limits<T>::max() - numeric_limits<T>::min()`
+
+*MIN_COMBINED Mode Example*
+
+If the input comes from a QuantizedRelu6, the output type is
+quint8 (range of 0-255) but the possible range of QuantizedRelu6 is
+0-6.  The min_range and max_range values are therefore 0.0 and 6.0.
+Dequantize on quint8 will take each value, cast to float, and multiply
+by 6 / 255.
+Note that if quantizedtype is qint8, the operation will additionally add
+each value by 128 prior to casting.
+
+If the mode is 'MIN_FIRST', then this approach is used:
+
+```c++
+number_of_steps = 1 << (# of bits in T)
+range_adjust = number_of_steps / (number_of_steps - 1)
+range = (range_max - range_min) * range_adjust
+range_scale = range / number_of_steps
+const double offset_input = static_cast<double>(input) - lowest_quantized;
+result = range_min + ((input - numeric_limits<T>::min()) * range_scale)
+```
+
+*SCALED mode Example*
+
+`SCALED` mode matches the quantization approach used in
+`QuantizeAndDequantize{V2|V3}`.
+
+If the mode is `SCALED`, we do not use the full range of the output type,
+choosing to elide the lowest possible value for symmetry (e.g., output range is
+-127 to 127, not -128 to 127 for signed 8 bit quantization), so that 0.0 maps to
+0.
+
+We first find the range of values in our tensor. The
+range we use is always centered on 0, so we find m such that
+```c++
+  m = max(abs(input_min), abs(input_max))
+```
+
+Our input tensor range is then `[-m, m]`.
+
+Next, we choose our fixed-point quantization buckets, `[min_fixed, max_fixed]`.
+If T is signed, this is
+```
+  num_bits = sizeof(T) * 8
+  [min_fixed, max_fixed] =
+      [-(1 << (num_bits - 1) - 1), (1 << (num_bits - 1)) - 1]
+```
+
+Otherwise, if T is unsigned, the fixed-point range is
+```
+  [min_fixed, max_fixed] = [0, (1 << num_bits) - 1]
+```
+
+From this we compute our scaling factor, s:
+```c++
+  s = (2 * m) / (max_fixed - min_fixed)
+```
+
+Now we can dequantize the elements of our tensor:
+```c++
+result = input * s
+```
+END
+}
+op {
+  graph_op_name: "DeserializeManySparse"
+  endpoint {
+    name: "DeserializeManySparse"
+  }
+  summary: "Deserialize and concatenate `SparseTensors` from a serialized minibatch."
+  description: <<END
+The input `serialized_sparse` must be a string matrix of shape `[N x 3]` where
+`N` is the minibatch size and the rows correspond to packed outputs of
+`SerializeSparse`.  The ranks of the original `SparseTensor` objects
+must all match.  When the final `SparseTensor` is created, it has rank one
+higher than the ranks of the incoming `SparseTensor` objects
+(they have been concatenated along a new row dimension).
+
+The output `SparseTensor` object's shape values for all dimensions but the
+first are the max across the input `SparseTensor` objects' shape values
+for the corresponding dimensions.  Its first shape value is `N`, the minibatch
+size.
+
+The input `SparseTensor` objects' indices are assumed ordered in
+standard lexicographic order.  If this is not the case, after this
+step run `SparseReorder` to restore index ordering.
+
+For example, if the serialized input is a `[2 x 3]` matrix representing two
+original `SparseTensor` objects:
+
+    index = [ 0]
+            [10]
+            [20]
+    values = [1, 2, 3]
+    shape = [50]
+
+and
+
+    index = [ 2]
+            [10]
+    values = [4, 5]
+    shape = [30]
+
+then the final deserialized `SparseTensor` will be:
+
+    index = [0  0]
+            [0 10]
+            [0 20]
+            [1  2]
+            [1 10]
+    values = [1, 2, 3, 4, 5]
+    shape = [2 50]
+END
+}
+op {
+  graph_op_name: "DestroyTemporaryVariable"
+  endpoint {
+    name: "DestroyTemporaryVariable"
+  }
+  summary: "Destroys the temporary variable and returns its final value."
+  description: <<END
+Sets output to the value of the Tensor pointed to by 'ref', then destroys
+the temporary variable called 'var_name'.
+All other uses of 'ref' *must* have executed before this op.
+This is typically achieved by chaining the ref through each assign op, or by
+using control dependencies.
+
+Outputs the final value of the tensor pointed to by 'ref'.
+END
+}
+op {
+  graph_op_name: "Diag"
+  endpoint {
+    name: "Diag"
+  }
+  summary: "Returns a diagonal tensor with a given diagonal values."
+  description: <<END
+Given a `diagonal`, this operation returns a tensor with the `diagonal` and
+everything else padded with zeros. The diagonal is computed as follows:
+
+Assume `diagonal` has dimensions [D1,..., Dk], then the output is a tensor of
+rank 2k with dimensions [D1,..., Dk, D1,..., Dk] where:
+
+`output[i1,..., ik, i1,..., ik] = diagonal[i1, ..., ik]` and 0 everywhere else.
+
+For example:
+
+```
+# 'diagonal' is [1, 2, 3, 4]
+tf.diag(diagonal) ==> [[1, 0, 0, 0]
+                       [0, 2, 0, 0]
+                       [0, 0, 3, 0]
+                       [0, 0, 0, 4]]
+```
+END
+}
+op {
+  graph_op_name: "DiagPart"
+  endpoint {
+    name: "DiagPart"
+  }
+  summary: "Returns the diagonal part of the tensor."
+  description: <<END
+This operation returns a tensor with the `diagonal` part
+of the `input`. The `diagonal` part is computed as follows:
+
+Assume `input` has dimensions `[D1,..., Dk, D1,..., Dk]`, then the output is a
+tensor of rank `k` with dimensions `[D1,..., Dk]` where:
+
+`diagonal[i1,..., ik] = input[i1, ..., ik, i1,..., ik]`.
+
+For example:
+
+```
+# 'input' is [[1, 0, 0, 0]
+              [0, 2, 0, 0]
+              [0, 0, 3, 0]
+              [0, 0, 0, 4]]
+
+tf.diag_part(input) ==> [1, 2, 3, 4]
+```
+END
+}
+op {
+  graph_op_name: "Digamma"
+  endpoint {
+    name: "Digamma"
+  }
+  summary: "Computes Psi, the derivative of Lgamma (the log of the absolute value of"
+  description: <<END
+`Gamma(x)`), element-wise.
+END
+}
+op {
+  graph_op_name: "Dilation2D"
+  endpoint {
+    name: "Dilation2D"
+  }
+  summary: "Computes the grayscale dilation of 4-D `input` and 3-D `filter` tensors."
+  description: <<END
+The `input` tensor has shape `[batch, in_height, in_width, depth]` and the
+`filter` tensor has shape `[filter_height, filter_width, depth]`, i.e., each
+input channel is processed independently of the others with its own structuring
+function. The `output` tensor has shape
+`[batch, out_height, out_width, depth]`. The spatial dimensions of the output
+tensor depend on the `padding` algorithm. We currently only support the default
+"NHWC" `data_format`.
+
+In detail, the grayscale morphological 2-D dilation is the max-sum correlation
+(for consistency with `conv2d`, we use unmirrored filters):
+
+    output[b, y, x, c] =
+       max_{dy, dx} input[b,
+                          strides[1] * y + rates[1] * dy,
+                          strides[2] * x + rates[2] * dx,
+                          c] +
+                    filter[dy, dx, c]
+
+Max-pooling is a special case when the filter has size equal to the pooling
+kernel size and contains all zeros.
+
+Note on duality: The dilation of `input` by the `filter` is equal to the
+negation of the erosion of `-input` by the reflected `filter`.
+END
+}
+op {
+  graph_op_name: "Dilation2DBackpropFilter"
+  endpoint {
+    name: "Dilation2DBackpropFilter"
+  }
+  summary: "Computes the gradient of morphological 2-D dilation with respect to the filter."
+}
+op {
+  graph_op_name: "Dilation2DBackpropInput"
+  endpoint {
+    name: "Dilation2DBackpropInput"
+  }
+  summary: "Computes the gradient of morphological 2-D dilation with respect to the input."
+}
+op {
+  graph_op_name: "Div"
+  endpoint {
+    name: "Div"
+  }
+  summary: "Returns x / y element-wise."
+  description: <<END
+*NOTE*: `Div` supports broadcasting. More about broadcasting
+[here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
+END
+}
+op {
+  graph_op_name: "DrawBoundingBoxes"
+  endpoint {
+    name: "DrawBoundingBoxes"
+  }
+  summary: "Draw bounding boxes on a batch of images."
+  description: <<END
+Outputs a copy of `images` but draws on top of the pixels zero or more bounding
+boxes specified by the locations in `boxes`. The coordinates of the each
+bounding box in `boxes` are encoded as `[y_min, x_min, y_max, x_max]`. The
+bounding box coordinates are floats in `[0.0, 1.0]` relative to the width and
+height of the underlying image.
+
+For example, if an image is 100 x 200 pixels (height x width) and the bounding
+box is `[0.1, 0.2, 0.5, 0.9]`, the upper-left and bottom-right coordinates of
+the bounding box will be `(40, 10)` to `(100, 50)` (in (x,y) coordinates).
+
+Parts of the bounding box may fall outside the image.
+END
+}
+op {
+  graph_op_name: "DynamicPartition"
+  endpoint {
+    name: "DynamicPartition"
+  }
+  summary: "Partitions `data` into `num_partitions` tensors using indices from `partitions`."
+  description: <<END
+For each index tuple `js` of size `partitions.ndim`, the slice `data[js, ...]`
+becomes part of `outputs[partitions[js]]`.  The slices with `partitions[js] = i`
+are placed in `outputs[i]` in lexicographic order of `js`, and the first
+dimension of `outputs[i]` is the number of entries in `partitions` equal to `i`.
+In detail,
+
+```python
+    outputs[i].shape = [sum(partitions == i)] + data.shape[partitions.ndim:]
+
+    outputs[i] = pack([data[js, ...] for js if partitions[js] == i])
+```
+
+`data.shape` must start with `partitions.shape`.
+
+For example:
+
+```python
+    # Scalar partitions.
+    partitions = 1
+    num_partitions = 2
+    data = [10, 20]
+    outputs[0] = []  # Empty with shape [0, 2]
+    outputs[1] = [[10, 20]]
+
+    # Vector partitions.
+    partitions = [0, 0, 1, 1, 0]
+    num_partitions = 2
+    data = [10, 20, 30, 40, 50]
+    outputs[0] = [10, 20, 50]
+    outputs[1] = [30, 40]
+```
+
+See `dynamic_stitch` for an example on how to merge partitions back.
+
+<div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
+<img style="width:100%" src="https://www.tensorflow.org/images/DynamicPartition.png" alt>
+</div>
+END
+}
+op {
+  graph_op_name: "DynamicStitch"
+  endpoint {
+    name: "DynamicStitch"
+  }
+  summary: "Interleave the values from the `data` tensors into a single tensor."
+  description: <<END
+Builds a merged tensor such that
+
+```python
+    merged[indices[m][i, ..., j], ...] = data[m][i, ..., j, ...]
+```
+
+For example, if each `indices[m]` is scalar or vector, we have
+
+```python
+    # Scalar indices:
+    merged[indices[m], ...] = data[m][...]
+
+    # Vector indices:
+    merged[indices[m][i], ...] = data[m][i, ...]
+```
+
+Each `data[i].shape` must start with the corresponding `indices[i].shape`,
+and the rest of `data[i].shape` must be constant w.r.t. `i`.  That is, we
+must have `data[i].shape = indices[i].shape + constant`.  In terms of this
+`constant`, the output shape is
+
+    merged.shape = [max(indices)] + constant
+
+Values are merged in order, so if an index appears in both `indices[m][i]` and
+`indices[n][j]` for `(m,i) < (n,j)` the slice `data[n][j]` will appear in the
+merged result. If you do not need this guarantee, ParallelDynamicStitch might
+perform better on some devices.
+
+For example:
+
+```python
+    indices[0] = 6
+    indices[1] = [4, 1]
+    indices[2] = [[5, 2], [0, 3]]
+    data[0] = [61, 62]
+    data[1] = [[41, 42], [11, 12]]
+    data[2] = [[[51, 52], [21, 22]], [[1, 2], [31, 32]]]
+    merged = [[1, 2], [11, 12], [21, 22], [31, 32], [41, 42],
+              [51, 52], [61, 62]]
+```
+
+This method can be used to merge partitions created by `dynamic_partition`
+as illustrated on the following example:
+
+```python
+    # Apply function (increments x_i) on elements for which a certain condition
+    # apply (x_i != -1 in this example).
+    x=tf.constant([0.1, -1., 5.2, 4.3, -1., 7.4])
+    condition_mask=tf.not_equal(x,tf.constant(-1.))
+    partitioned_data = tf.dynamic_partition(
+        x, tf.cast(condition_mask, tf.int32) , 2)
+    partitioned_data[1] = partitioned_data[1] + 1.0
+    condition_indices = tf.dynamic_partition(
+        tf.range(tf.shape(x)[0]), tf.cast(condition_mask, tf.int32) , 2)
+    x = tf.dynamic_stitch(condition_indices, partitioned_data)
+    # Here x=[1.1, -1., 6.2, 5.3, -1, 8.4], the -1. values remain
+    # unchanged.
+```
+
+<div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
+<img style="width:100%" src="https://www.tensorflow.org/images/DynamicStitch.png" alt>
+</div>
+END
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_E.pbtxt b/tensorflow/core/api_def/base_api/api_def_E.pbtxt
new file mode 100644
index 0000000000..b49146f7c4
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_E.pbtxt
@@ -0,0 +1,261 @@
+op {
+  graph_op_name: "EditDistance"
+  endpoint {
+    name: "EditDistance"
+  }
+  summary: "Computes the (possibly normalized) Levenshtein Edit Distance."
+  description: <<END
+The inputs are variable-length sequences provided by SparseTensors
+  (hypothesis_indices, hypothesis_values, hypothesis_shape)
+and
+  (truth_indices, truth_values, truth_shape).
+
+The inputs are:
+END
+}
+op {
+  graph_op_name: "Elu"
+  endpoint {
+    name: "Elu"
+  }
+  summary: "Computes exponential linear: `exp(features) - 1` if < 0, `features` otherwise."
+  description: <<END
+See [Fast and Accurate Deep Network Learning by Exponential Linear Units (ELUs)
+](http://arxiv.org/abs/1511.07289)
+END
+}
+op {
+  graph_op_name: "EluGrad"
+  endpoint {
+    name: "EluGrad"
+  }
+  summary: "Computes gradients for the exponential linear (Elu) operation."
+}
+op {
+  graph_op_name: "EncodeBase64"
+  endpoint {
+    name: "EncodeBase64"
+  }
+  summary: "Encode strings into web-safe base64 format."
+  description: <<END
+Refer to the following article for more information on base64 format:
+en.wikipedia.org/wiki/Base64. Base64 strings may have padding with '=' at the
+end so that the encoded has length multiple of 4. See Padding section of the
+link above.
+
+Web-safe means that the encoder uses - and _ instead of + and /.
+END
+}
+op {
+  graph_op_name: "EncodeJpeg"
+  endpoint {
+    name: "EncodeJpeg"
+  }
+  summary: "JPEG-encode an image."
+  description: <<END
+`image` is a 3-D uint8 Tensor of shape `[height, width, channels]`.
+
+The attr `format` can be used to override the color format of the encoded
+output.  Values can be:
+
+*   `''`: Use a default format based on the number of channels in the image.
+*   `grayscale`: Output a grayscale JPEG image.  The `channels` dimension
+    of `image` must be 1.
+*   `rgb`: Output an RGB JPEG image. The `channels` dimension
+    of `image` must be 3.
+
+If `format` is not specified or is the empty string, a default format is picked
+in function of the number of channels in `image`:
+
+*   1: Output a grayscale image.
+*   3: Output an RGB image.
+END
+}
+op {
+  graph_op_name: "EncodePng"
+  endpoint {
+    name: "EncodePng"
+  }
+  summary: "PNG-encode an image."
+  description: <<END
+`image` is a 3-D uint8 or uint16 Tensor of shape `[height, width, channels]`
+where `channels` is:
+
+*   1: for grayscale.
+*   2: for grayscale + alpha.
+*   3: for RGB.
+*   4: for RGBA.
+
+The ZLIB compression level, `compression`, can be -1 for the PNG-encoder
+default or a value from 0 to 9.  9 is the highest compression level, generating
+the smallest output, but is slower.
+END
+}
+op {
+  graph_op_name: "EncodeWav"
+  endpoint {
+    name: "EncodeWav"
+  }
+  summary: "Encode audio data using the WAV file format."
+  description: <<END
+This operation will generate a string suitable to be saved out to create a .wav
+audio file. It will be encoded in the 16-bit PCM format. It takes in float
+values in the range -1.0f to 1.0f, and any outside that value will be clamped to
+that range.
+
+`audio` is a 2-D float Tensor of shape `[length, channels]`.
+`sample_rate` is a scalar Tensor holding the rate to use (e.g. 44100).
+END
+}
+op {
+  graph_op_name: "Enter"
+  endpoint {
+    name: "Enter"
+  }
+  summary: "Creates or finds a child frame, and makes `data` available to the child frame."
+  description: <<END
+This op is used together with `Exit` to create loops in the graph.
+The unique `frame_name` is used by the `Executor` to identify frames. If
+`is_constant` is true, `output` is a constant in the child frame; otherwise
+it may be changed in the child frame. At most `parallel_iterations` iterations
+are run in parallel in the child frame.
+END
+}
+op {
+  graph_op_name: "Equal"
+  endpoint {
+    name: "Equal"
+  }
+  summary: "Returns the truth value of (x == y) element-wise."
+  description: <<END
+*NOTE*: `Equal` supports broadcasting. More about broadcasting
+[here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
+END
+}
+op {
+  graph_op_name: "Erf"
+  endpoint {
+    name: "Erf"
+  }
+  summary: "Computes the Gauss error function of `x` element-wise."
+}
+op {
+  graph_op_name: "Erfc"
+  endpoint {
+    name: "Erfc"
+  }
+  summary: "Computes the complementary error function of `x` element-wise."
+}
+op {
+  graph_op_name: "Exit"
+  endpoint {
+    name: "Exit"
+  }
+  summary: "Exits the current frame to its parent frame."
+  description: <<END
+Exit makes its input `data` available to the parent frame.
+END
+}
+op {
+  graph_op_name: "Exp"
+  endpoint {
+    name: "Exp"
+  }
+  summary: "Computes exponential of x element-wise.  \\\\(y = e^x\\\\)."
+}
+op {
+  graph_op_name: "ExpandDims"
+  endpoint {
+    name: "ExpandDims"
+  }
+  summary: "Inserts a dimension of 1 into a tensor\'s shape."
+  description: <<END
+Given a tensor `input`, this operation inserts a dimension of 1 at the
+dimension index `dim` of `input`'s shape. The dimension index `dim` starts at
+zero; if you specify a negative number for `dim` it is counted backward from
+the end.
+
+This operation is useful if you want to add a batch dimension to a single
+element. For example, if you have a single image of shape `[height, width,
+channels]`, you can make it a batch of 1 image with `expand_dims(image, 0)`,
+which will make the shape `[1, height, width, channels]`.
+
+Other examples:
+
+```
+# 't' is a tensor of shape [2]
+shape(expand_dims(t, 0)) ==> [1, 2]
+shape(expand_dims(t, 1)) ==> [2, 1]
+shape(expand_dims(t, -1)) ==> [2, 1]
+
+# 't2' is a tensor of shape [2, 3, 5]
+shape(expand_dims(t2, 0)) ==> [1, 2, 3, 5]
+shape(expand_dims(t2, 2)) ==> [2, 3, 1, 5]
+shape(expand_dims(t2, 3)) ==> [2, 3, 5, 1]
+```
+
+This operation requires that:
+
+`-1-input.dims() <= dim <= input.dims()`
+
+This operation is related to `squeeze()`, which removes dimensions of
+size 1.
+END
+}
+op {
+  graph_op_name: "Expm1"
+  endpoint {
+    name: "Expm1"
+  }
+  summary: "Computes exponential of x - 1 element-wise."
+  description: <<END
+I.e., \\(y = (\exp x) - 1\\).
+END
+}
+op {
+  graph_op_name: "ExtractGlimpse"
+  endpoint {
+    name: "ExtractGlimpse"
+  }
+  summary: "Extracts a glimpse from the input tensor."
+  description: <<END
+Returns a set of windows called glimpses extracted at location
+`offsets` from the input tensor. If the windows only partially
+overlaps the inputs, the non overlapping areas will be filled with
+random noise.
+
+The result is a 4-D tensor of shape `[batch_size, glimpse_height,
+glimpse_width, channels]`. The channels and batch dimensions are the
+same as that of the input tensor. The height and width of the output
+windows are specified in the `size` parameter.
+
+The argument `normalized` and `centered` controls how the windows are built:
+
+* If the coordinates are normalized but not centered, 0.0 and 1.0
+  correspond to the minimum and maximum of each height and width
+  dimension.
+* If the coordinates are both normalized and centered, they range from
+  -1.0 to 1.0. The coordinates (-1.0, -1.0) correspond to the upper
+  left corner, the lower right corner is located at (1.0, 1.0) and the
+  center is at (0, 0).
+* If the coordinates are not normalized they are interpreted as
+  numbers of pixels.
+END
+}
+op {
+  graph_op_name: "ExtractImagePatches"
+  endpoint {
+    name: "ExtractImagePatches"
+  }
+  summary: "Extract `patches` from `images` and put them in the \"depth\" output dimension."
+}
+op {
+  graph_op_name: "ExtractJpegShape"
+  endpoint {
+    name: "ExtractJpegShape"
+  }
+  summary: "Extract the shape information of a JPEG-encoded image."
+  description: <<END
+This op only parses the image header, so it is much faster than DecodeJpeg.
+END
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_F.pbtxt b/tensorflow/core/api_def/base_api/api_def_F.pbtxt
new file mode 100644
index 0000000000..8c073d3369
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_F.pbtxt
@@ -0,0 +1,411 @@
+op {
+  graph_op_name: "FFT"
+  endpoint {
+    name: "FFT"
+  }
+  summary: "Fast Fourier transform."
+  description: <<END
+Computes the 1-dimensional discrete Fourier transform over the inner-most
+dimension of `input`.
+END
+}
+op {
+  graph_op_name: "FFT2D"
+  endpoint {
+    name: "FFT2D"
+  }
+  summary: "2D fast Fourier transform."
+  description: <<END
+Computes the 2-dimensional discrete Fourier transform over the inner-most
+2 dimensions of `input`.
+END
+}
+op {
+  graph_op_name: "FFT3D"
+  endpoint {
+    name: "FFT3D"
+  }
+  summary: "3D fast Fourier transform."
+  description: <<END
+Computes the 3-dimensional discrete Fourier transform over the inner-most 3
+dimensions of `input`.
+END
+}
+op {
+  graph_op_name: "FIFOQueue"
+  endpoint {
+    name: "FIFOQueue"
+  }
+  summary: "A queue that produces elements in first-in first-out order."
+}
+op {
+  graph_op_name: "FIFOQueueV2"
+  endpoint {
+    name: "FIFOQueueV2"
+  }
+  summary: "A queue that produces elements in first-in first-out order."
+}
+op {
+  graph_op_name: "Fact"
+  endpoint {
+    name: "Fact"
+  }
+  summary: "Output a fact about factorials."
+}
+op {
+  graph_op_name: "FakeQuantWithMinMaxArgs"
+  endpoint {
+    name: "FakeQuantWithMinMaxArgs"
+  }
+  summary: "Fake-quantize the \'inputs\' tensor, type float to \'outputs\' tensor of same type."
+  description: <<END
+Attributes `[min; max]` define the clamping range for the `inputs` data.
+`inputs` values are quantized into the quantization range (`[0; 2^num_bits - 1]`
+when `narrow_range` is false and `[1; 2^num_bits - 1]` when it is true) and
+then de-quantized and output as floats in `[min; max]` interval.
+`num_bits` is the bitwidth of the quantization; between 2 and 8, inclusive.
+
+Quantization is called fake since the output is still in floating point.
+END
+}
+op {
+  graph_op_name: "FakeQuantWithMinMaxArgsGradient"
+  endpoint {
+    name: "FakeQuantWithMinMaxArgsGradient"
+  }
+  summary: "Compute gradients for a FakeQuantWithMinMaxArgs operation."
+}
+op {
+  graph_op_name: "FakeQuantWithMinMaxVars"
+  endpoint {
+    name: "FakeQuantWithMinMaxVars"
+  }
+  summary: "Fake-quantize the \'inputs\' tensor of type float via global float scalars `min`"
+  description: <<END
+and `max` to 'outputs' tensor of same shape as `inputs`.
+
+`[min; max]` define the clamping range for the `inputs` data.
+`inputs` values are quantized into the quantization range (`[0; 2^num_bits - 1]`
+when `narrow_range` is false and `[1; 2^num_bits - 1]` when it is true) and
+then de-quantized and output as floats in `[min; max]` interval.
+`num_bits` is the bitwidth of the quantization; between 2 and 8, inclusive.
+
+This operation has a gradient and thus allows for training `min` and `max`
+values.
+END
+}
+op {
+  graph_op_name: "FakeQuantWithMinMaxVarsGradient"
+  endpoint {
+    name: "FakeQuantWithMinMaxVarsGradient"
+  }
+  summary: "Compute gradients for a FakeQuantWithMinMaxVars operation."
+}
+op {
+  graph_op_name: "FakeQuantWithMinMaxVarsPerChannel"
+  endpoint {
+    name: "FakeQuantWithMinMaxVarsPerChannel"
+  }
+  summary: "Fake-quantize the \'inputs\' tensor of type float and one of the shapes: `[d]`,"
+  description: <<END
+`[b, d]` `[b, h, w, d]` via per-channel floats `min` and `max` of shape `[d]`
+to 'outputs' tensor of same shape as `inputs`.
+
+`[min; max]` define the clamping range for the `inputs` data.
+`inputs` values are quantized into the quantization range (`[0; 2^num_bits - 1]`
+when `narrow_range` is false and `[1; 2^num_bits - 1]` when it is true) and
+then de-quantized and output as floats in `[min; max]` interval.
+`num_bits` is the bitwidth of the quantization; between 2 and 8, inclusive.
+
+This operation has a gradient and thus allows for training `min` and `max`
+values.
+END
+}
+op {
+  graph_op_name: "FakeQuantWithMinMaxVarsPerChannelGradient"
+  endpoint {
+    name: "FakeQuantWithMinMaxVarsPerChannelGradient"
+  }
+  summary: "Compute gradients for a FakeQuantWithMinMaxVarsPerChannel operation."
+}
+op {
+  graph_op_name: "FakeQueue"
+  endpoint {
+    name: "FakeQueue"
+  }
+  summary: "Deprecated. Do not use."
+}
+op {
+  graph_op_name: "Fill"
+  endpoint {
+    name: "Fill"
+  }
+  summary: "Creates a tensor filled with a scalar value."
+  description: <<END
+This operation creates a tensor of shape `dims` and fills it with `value`.
+
+For example:
+
+```
+# Output tensor has shape [2, 3].
+fill([2, 3], 9) ==> [[9, 9, 9]
+                     [9, 9, 9]]
+```
+END
+}
+op {
+  graph_op_name: "FilterDataset"
+  endpoint {
+    name: "FilterDataset"
+  }
+  summary: "Creates a dataset containing elements of `input_dataset` matching `predicate`."
+  description: <<END
+The `predicate` function must return a scalar boolean and accept the
+following arguments:
+
+* One tensor for each component of an element of `input_dataset`.
+* One tensor for each value in `other_arguments`.
+END
+}
+op {
+  graph_op_name: "FixedLengthRecordDataset"
+  endpoint {
+    name: "FixedLengthRecordDataset"
+  }
+  summary: "Creates a dataset that emits the records from one or more binary files."
+}
+op {
+  graph_op_name: "FixedLengthRecordReader"
+  endpoint {
+    name: "FixedLengthRecordReader"
+  }
+  summary: "A Reader that outputs fixed-length records from a file."
+}
+op {
+  graph_op_name: "FixedLengthRecordReaderV2"
+  endpoint {
+    name: "FixedLengthRecordReaderV2"
+  }
+  summary: "A Reader that outputs fixed-length records from a file."
+}
+op {
+  graph_op_name: "FixedUnigramCandidateSampler"
+  endpoint {
+    name: "FixedUnigramCandidateSampler"
+  }
+  summary: "Generates labels for candidate sampling with a learned unigram distribution."
+  description: <<END
+A unigram sampler could use a fixed unigram distribution read from a
+file or passed in as an in-memory array instead of building up the distribution
+from data on the fly. There is also an option to skew the distribution by
+applying a distortion power to the weights.
+
+The vocabulary file should be in CSV-like format, with the last field
+being the weight associated with the word.
+
+For each batch, this op picks a single set of sampled candidate labels.
+
+The advantages of sampling candidates per-batch are simplicity and the
+possibility of efficient dense matrix multiplication. The disadvantage is that
+the sampled candidates must be chosen independently of the context and of the
+true labels.
+END
+}
+op {
+  graph_op_name: "FlatMapDataset"
+  endpoint {
+    name: "FlatMapDataset"
+  }
+  summary: "Creates a dataset that applies `f` to the outputs of `input_dataset`."
+  description: <<END
+Unlike MapDataset, the `f` in FlatMapDataset is expected to return a
+Dataset variant, and FlatMapDataset will flatten successive results
+into a single Dataset.
+END
+}
+op {
+  graph_op_name: "Floor"
+  endpoint {
+    name: "Floor"
+  }
+  summary: "Returns element-wise largest integer not greater than x."
+}
+op {
+  graph_op_name: "FloorDiv"
+  endpoint {
+    name: "FloorDiv"
+  }
+  summary: "Returns x // y element-wise."
+  description: <<END
+*NOTE*: `FloorDiv` supports broadcasting. More about broadcasting
+[here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
+END
+}
+op {
+  graph_op_name: "FloorMod"
+  endpoint {
+    name: "FloorMod"
+  }
+  summary: "Returns element-wise remainder of division. When `x < 0` xor `y < 0` is"
+  description: <<END
+true, this follows Python semantics in that the result here is consistent
+with a flooring divide. E.g. `floor(x / y) * y + mod(x, y) = x`.
+
+*NOTE*: `FloorMod` supports broadcasting. More about broadcasting
+[here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
+END
+}
+op {
+  graph_op_name: "FractionalAvgPool"
+  endpoint {
+    name: "FractionalAvgPool"
+  }
+  summary: "Performs fractional average pooling on the input."
+  description: <<END
+Fractional average pooling is similar to Fractional max pooling in the pooling
+region generation step. The only difference is that after pooling regions are
+generated, a mean operation is performed instead of a max operation in each
+pooling region.
+END
+}
+op {
+  graph_op_name: "FractionalAvgPoolGrad"
+  endpoint {
+    name: "FractionalAvgPoolGrad"
+  }
+  summary: "Computes gradient of the FractionalAvgPool function."
+  description: <<END
+Unlike FractionalMaxPoolGrad, we don't need to find arg_max for
+FractionalAvgPoolGrad, we just need to evenly back-propagate each element of
+out_backprop to those indices that form the same pooling cell. Therefore, we
+just need to know the shape of original input tensor, instead of the whole
+tensor.
+END
+}
+op {
+  graph_op_name: "FractionalMaxPool"
+  endpoint {
+    name: "FractionalMaxPool"
+  }
+  summary: "Performs fractional max pooling on the input."
+  description: <<END
+Fractional max pooling is slightly different than regular max pooling.  In
+regular max pooling, you downsize an input set by taking the maximum value of
+smaller N x N subsections of the set (often 2x2), and try to reduce the set by
+a factor of N, where N is an integer.  Fractional max pooling, as you might
+expect from the word "fractional", means that the overall reduction ratio N
+does not have to be an integer.
+
+The sizes of the pooling regions are generated randomly but are fairly uniform.
+For example, let's look at the height dimension, and the constraints on the
+list of rows that will be pool boundaries.
+
+First we define the following:
+
+1.  input_row_length : the number of rows from the input set
+2.  output_row_length : which will be smaller than the input
+3.  alpha = input_row_length / output_row_length : our reduction ratio
+4.  K = floor(alpha)
+5.  row_pooling_sequence : this is the result list of pool boundary rows
+
+Then, row_pooling_sequence should satisfy:
+
+1.  a[0] = 0 : the first value of the sequence is 0
+2.  a[end] = input_row_length : the last value of the sequence is the size
+3.  K <= (a[i+1] - a[i]) <= K+1 : all intervals are K or K+1 size
+4.  length(row_pooling_sequence) = output_row_length+1
+
+For more details on fractional max pooling, see this paper:
+[Benjamin Graham, Fractional Max-Pooling](http://arxiv.org/abs/1412.6071)
+END
+}
+op {
+  graph_op_name: "FractionalMaxPoolGrad"
+  endpoint {
+    name: "FractionalMaxPoolGrad"
+  }
+  summary: "Computes gradient of the FractionalMaxPool function."
+}
+op {
+  graph_op_name: "FusedBatchNorm"
+  endpoint {
+    name: "FusedBatchNorm"
+  }
+  summary: "Batch normalization."
+  description: <<END
+Note that the size of 4D Tensors are defined by either "NHWC" or "NCHW".
+The size of 1D Tensors matches the dimension C of the 4D Tensors.
+END
+}
+op {
+  graph_op_name: "FusedBatchNormGrad"
+  endpoint {
+    name: "FusedBatchNormGrad"
+  }
+  summary: "Gradient for batch normalization."
+  description: <<END
+Note that the size of 4D Tensors are defined by either "NHWC" or "NCHW".
+The size of 1D Tensors matches the dimension C of the 4D Tensors.
+END
+}
+op {
+  graph_op_name: "FusedBatchNormGradV2"
+  endpoint {
+    name: "FusedBatchNormGradV2"
+  }
+  summary: "Gradient for batch normalization."
+  description: <<END
+Note that the size of 4D Tensors are defined by either "NHWC" or "NCHW".
+The size of 1D Tensors matches the dimension C of the 4D Tensors.
+END
+}
+op {
+  graph_op_name: "FusedBatchNormV2"
+  endpoint {
+    name: "FusedBatchNormV2"
+  }
+  summary: "Batch normalization."
+  description: <<END
+Note that the size of 4D Tensors are defined by either "NHWC" or "NCHW".
+The size of 1D Tensors matches the dimension C of the 4D Tensors.
+END
+}
+op {
+  graph_op_name: "FusedPadConv2D"
+  endpoint {
+    name: "FusedPadConv2D"
+  }
+  summary: "Performs a padding as a preprocess during a convolution."
+  description: <<END
+Similar to FusedResizeAndPadConv2d, this op allows for an optimized
+implementation where the spatial padding transformation stage is fused with the
+im2col lookup, but in this case without the bilinear filtering required for
+resizing. Fusing the padding prevents the need to write out the intermediate
+results as whole tensors, reducing memory pressure, and we can get some latency
+gains by merging the transformation calculations.
+The data_format attribute for Conv2D isn't supported by this op, and 'NHWC'
+order is used instead.
+Internally this op uses a single per-graph scratch buffer, which means that it
+will block if multiple versions are being run in parallel. This is because this
+operator is primarily an optimization to minimize memory usage.
+END
+}
+op {
+  graph_op_name: "FusedResizeAndPadConv2D"
+  endpoint {
+    name: "FusedResizeAndPadConv2D"
+  }
+  summary: "Performs a resize and padding as a preprocess during a convolution."
+  description: <<END
+It's often possible to do spatial transformations more efficiently as part of
+the packing stage of a convolution, so this op allows for an optimized
+implementation where these stages are fused together. This prevents the need to
+write out the intermediate results as whole tensors, reducing memory pressure,
+and we can get some latency gains by merging the transformation calculations.
+The data_format attribute for Conv2D isn't supported by this op, and defaults to
+'NHWC' order.
+Internally this op uses a single per-graph scratch buffer, which means that it
+will block if multiple versions are being run in parallel. This is because this
+operator is primarily an optimization to minimize memory usage.
+END
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_G.pbtxt b/tensorflow/core/api_def/base_api/api_def_G.pbtxt
new file mode 100644
index 0000000000..343d505718
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_G.pbtxt
@@ -0,0 +1,257 @@
+op {
+  graph_op_name: "Gather"
+  endpoint {
+    name: "Gather"
+  }
+  summary: "Gather slices from `params` according to `indices`."
+  description: <<END
+`indices` must be an integer tensor of any dimension (usually 0-D or 1-D).
+Produces an output tensor with shape `indices.shape + params.shape[1:]` where:
+
+```python
+    # Scalar indices
+    output[:, ..., :] = params[indices, :, ... :]
+
+    # Vector indices
+    output[i, :, ..., :] = params[indices[i], :, ... :]
+
+    # Higher rank indices
+    output[i, ..., j, :, ... :] = params[indices[i, ..., j], :, ..., :]
+```
+
+If `indices` is a permutation and `len(indices) == params.shape[0]` then
+this operation will permute `params` accordingly.
+
+`validate_indices`: DEPRECATED. If this operation is assigned to CPU, values in
+`indices` are always validated to be within range. If assigned to GPU,
+out-of-bound indices result in safe but unspecified behavior, which may include
+raising an error.
+
+<div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
+<img style="width:100%" src="https://www.tensorflow.org/images/Gather.png" alt>
+</div>
+END
+}
+op {
+  graph_op_name: "GatherNd"
+  endpoint {
+    name: "GatherNd"
+  }
+  summary: "Gather slices from `params` into a Tensor with shape specified by `indices`."
+  description: <<END
+`indices` is an K-dimensional integer tensor, best thought of as a
+(K-1)-dimensional tensor of indices into `params`, where each element defines a
+slice of `params`:
+
+    output[i_0, ..., i_{K-2}] = params[indices[i0, ..., i_{K-2}]]
+
+Whereas in @{tf.gather} `indices` defines slices into the first
+dimension of `params`, in `tf.gather_nd`, `indices` defines slices into the
+first `N` dimensions of `params`, where `N = indices.shape[-1]`.
+
+The last dimension of `indices` can be at most the rank of
+`params`:
+
+    indices.shape[-1] <= params.rank
+
+The last dimension of `indices` corresponds to elements
+(if `indices.shape[-1] == params.rank`) or slices
+(if `indices.shape[-1] < params.rank`) along dimension `indices.shape[-1]`
+of `params`.  The output tensor has shape
+
+    indices.shape[:-1] + params.shape[indices.shape[-1]:]
+
+Some examples below.
+
+Simple indexing into a matrix:
+
+```python
+    indices = [[0, 0], [1, 1]]
+    params = [['a', 'b'], ['c', 'd']]
+    output = ['a', 'd']
+```
+
+Slice indexing into a matrix:
+
+```python
+    indices = [[1], [0]]
+    params = [['a', 'b'], ['c', 'd']]
+    output = [['c', 'd'], ['a', 'b']]
+```
+
+Indexing into a 3-tensor:
+
+```python
+    indices = [[1]]
+    params = [[['a0', 'b0'], ['c0', 'd0']],
+              [['a1', 'b1'], ['c1', 'd1']]]
+    output = [[['a1', 'b1'], ['c1', 'd1']]]
+
+
+    indices = [[0, 1], [1, 0]]
+    params = [[['a0', 'b0'], ['c0', 'd0']],
+              [['a1', 'b1'], ['c1', 'd1']]]
+    output = [['c0', 'd0'], ['a1', 'b1']]
+
+
+    indices = [[0, 0, 1], [1, 0, 1]]
+    params = [[['a0', 'b0'], ['c0', 'd0']],
+              [['a1', 'b1'], ['c1', 'd1']]]
+    output = ['b0', 'b1']
+```
+
+Batched indexing into a matrix:
+
+```python
+    indices = [[[0, 0]], [[0, 1]]]
+    params = [['a', 'b'], ['c', 'd']]
+    output = [['a'], ['b']]
+```
+
+Batched slice indexing into a matrix:
+
+```python
+    indices = [[[1]], [[0]]]
+    params = [['a', 'b'], ['c', 'd']]
+    output = [[['c', 'd']], [['a', 'b']]]
+```
+
+Batched indexing into a 3-tensor:
+
+```python
+    indices = [[[1]], [[0]]]
+    params = [[['a0', 'b0'], ['c0', 'd0']],
+              [['a1', 'b1'], ['c1', 'd1']]]
+    output = [[[['a1', 'b1'], ['c1', 'd1']]],
+              [[['a0', 'b0'], ['c0', 'd0']]]]
+
+    indices = [[[0, 1], [1, 0]], [[0, 0], [1, 1]]]
+    params = [[['a0', 'b0'], ['c0', 'd0']],
+              [['a1', 'b1'], ['c1', 'd1']]]
+    output = [[['c0', 'd0'], ['a1', 'b1']],
+              [['a0', 'b0'], ['c1', 'd1']]]
+
+
+    indices = [[[0, 0, 1], [1, 0, 1]], [[0, 1, 1], [1, 1, 0]]]
+    params = [[['a0', 'b0'], ['c0', 'd0']],
+              [['a1', 'b1'], ['c1', 'd1']]]
+    output = [['b0', 'b1'], ['d0', 'c1']]
+```
+END
+}
+op {
+  graph_op_name: "GatherV2"
+  endpoint {
+    name: "GatherV2"
+  }
+  summary: "Gather slices from `params` axis `axis` according to `indices`."
+  description: <<END
+`indices` must be an integer tensor of any dimension (usually 0-D or 1-D).
+Produces an output tensor with shape `params.shape[:axis] + indices.shape +
+params.shape[axis + 1:]` where:
+
+```python
+    # Scalar indices (output is rank(params) - 1).
+    output[a_0, ..., a_n, b_0, ..., b_n] =
+      params[a_0, ..., a_n, indices, b_0, ..., b_n]
+
+    # Vector indices (output is rank(params)).
+    output[a_0, ..., a_n, i, b_0, ..., b_n] =
+      params[a_0, ..., a_n, indices[i], b_0, ..., b_n]
+
+    # Higher rank indices (output is rank(params) + rank(indices) - 1).
+    output[a_0, ..., a_n, i, ..., j, b_0, ... b_n] =
+      params[a_0, ..., a_n, indices[i, ..., j], b_0, ..., b_n]
+```
+
+<div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
+<img style="width:100%" src="https://www.tensorflow.org/images/Gather.png" alt>
+</div>
+END
+}
+op {
+  graph_op_name: "GenerateVocabRemapping"
+  endpoint {
+    name: "GenerateVocabRemapping"
+  }
+  summary: "Given a path to new and old vocabulary files, returns a remapping Tensor of"
+  description: <<END
+length `num_new_vocab`, where `remapping[i]` contains the row number in the old
+vocabulary that corresponds to row `i` in the new vocabulary (starting at line
+`new_vocab_offset` and up to `num_new_vocab` entities), or `-1` if entry `i`
+in the new vocabulary is not in the old vocabulary.  `num_vocab_offset` enables
+use in the partitioned variable case, and should generally be set through
+examining partitioning info.  The format of the files should be a text file,
+with each line containing a single entity within the vocabulary.
+
+For example, with `new_vocab_file` a text file containing each of the following
+elements on a single line: `[f0, f1, f2, f3]`, old_vocab_file = [f1, f0, f3],
+`num_new_vocab = 3, new_vocab_offset = 1`, the returned remapping would be
+`[0, -1, 2]`.
+
+The op also returns a count of how many entries in the new vocabulary
+were present in the old vocabulary, which is used to calculate the number of
+values to initialize in a weight matrix remapping
+
+This functionality can be used to remap both row vocabularies (typically,
+features) and column vocabularies (typically, classes) from TensorFlow
+checkpoints.  Note that the partitioning logic relies on contiguous vocabularies
+corresponding to div-partitioned variables.  Moreover, the underlying remapping
+uses an IndexTable (as opposed to an inexact CuckooTable), so client code should
+use the corresponding index_table_from_file() as the FeatureColumn framework
+does (as opposed to tf.feature_to_id(), which uses a CuckooTable).
+END
+}
+op {
+  graph_op_name: "GetSessionHandle"
+  endpoint {
+    name: "GetSessionHandle"
+  }
+  summary: "Store the input tensor in the state of the current session."
+}
+op {
+  graph_op_name: "GetSessionHandleV2"
+  endpoint {
+    name: "GetSessionHandleV2"
+  }
+  summary: "Store the input tensor in the state of the current session."
+}
+op {
+  graph_op_name: "GetSessionTensor"
+  endpoint {
+    name: "GetSessionTensor"
+  }
+  summary: "Get the value of the tensor specified by its handle."
+}
+op {
+  graph_op_name: "Greater"
+  endpoint {
+    name: "Greater"
+  }
+  summary: "Returns the truth value of (x > y) element-wise."
+  description: <<END
+*NOTE*: `Greater` supports broadcasting. More about broadcasting
+[here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
+END
+}
+op {
+  graph_op_name: "GreaterEqual"
+  endpoint {
+    name: "GreaterEqual"
+  }
+  summary: "Returns the truth value of (x >= y) element-wise."
+  description: <<END
+*NOTE*: `GreaterEqual` supports broadcasting. More about broadcasting
+[here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
+END
+}
+op {
+  graph_op_name: "GroupByWindowDataset"
+  endpoint {
+    name: "GroupByWindowDataset"
+  }
+  summary: "Creates a dataset that computes a windowed group-by on `input_dataset`."
+  description: <<END
+// TODO(mrry): Support non-int64 keys.
+END
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_H.pbtxt b/tensorflow/core/api_def/base_api/api_def_H.pbtxt
new file mode 100644
index 0000000000..71282e7def
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_H.pbtxt
@@ -0,0 +1,52 @@
+op {
+  graph_op_name: "HSVToRGB"
+  endpoint {
+    name: "HSVToRGB"
+  }
+  summary: "Convert one or more images from HSV to RGB."
+  description: <<END
+Outputs a tensor of the same shape as the `images` tensor, containing the RGB
+value of the pixels. The output is only well defined if the value in `images`
+are in `[0,1]`.
+
+See `rgb_to_hsv` for a description of the HSV encoding.
+END
+}
+op {
+  graph_op_name: "HashTable"
+  endpoint {
+    name: "HashTable"
+  }
+  summary: "Creates a non-initialized hash table."
+  description: <<END
+This op creates a hash table, specifying the type of its keys and values.
+Before using the table you will have to initialize it.  After initialization the
+table will be immutable.
+END
+}
+op {
+  graph_op_name: "HashTableV2"
+  endpoint {
+    name: "HashTableV2"
+  }
+  summary: "Creates a non-initialized hash table."
+  description: <<END
+This op creates a hash table, specifying the type of its keys and values.
+Before using the table you will have to initialize it.  After initialization the
+table will be immutable.
+END
+}
+op {
+  graph_op_name: "HistogramSummary"
+  endpoint {
+    name: "HistogramSummary"
+  }
+  summary: "Outputs a `Summary` protocol buffer with a histogram."
+  description: <<END
+The generated
+[`Summary`](https://www.tensorflow.org/code/tensorflow/core/framework/summary.proto)
+has one summary value containing a histogram for `values`.
+
+This op reports an `InvalidArgument` error if any value is not finite.
+END
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_I.pbtxt b/tensorflow/core/api_def/base_api/api_def_I.pbtxt
new file mode 100644
index 0000000000..caaf93bf88
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_I.pbtxt
@@ -0,0 +1,518 @@
+op {
+  graph_op_name: "IFFT"
+  endpoint {
+    name: "IFFT"
+  }
+  summary: "Inverse fast Fourier transform."
+  description: <<END
+Computes the inverse 1-dimensional discrete Fourier transform over the
+inner-most dimension of `input`.
+END
+}
+op {
+  graph_op_name: "IFFT2D"
+  endpoint {
+    name: "IFFT2D"
+  }
+  summary: "Inverse 2D fast Fourier transform."
+  description: <<END
+Computes the inverse 2-dimensional discrete Fourier transform over the
+inner-most 2 dimensions of `input`.
+END
+}
+op {
+  graph_op_name: "IFFT3D"
+  endpoint {
+    name: "IFFT3D"
+  }
+  summary: "Inverse 3D fast Fourier transform."
+  description: <<END
+Computes the inverse 3-dimensional discrete Fourier transform over the
+inner-most 3 dimensions of `input`.
+END
+}
+op {
+  graph_op_name: "IRFFT"
+  endpoint {
+    name: "IRFFT"
+  }
+  summary: "Inverse real-valued fast Fourier transform."
+  description: <<END
+Computes the inverse 1-dimensional discrete Fourier transform of a real-valued
+signal over the inner-most dimension of `input`.
+
+The inner-most dimension of `input` is assumed to be the result of `RFFT`: the
+`fft_length / 2 + 1` unique components of the DFT of a real-valued signal. If
+`fft_length` is not provided, it is computed from the size of the inner-most
+dimension of `input` (`fft_length = 2 * (inner - 1)`). If the FFT length used to
+compute `input` is odd, it should be provided since it cannot be inferred
+properly.
+
+Along the axis `IRFFT` is computed on, if `fft_length / 2 + 1` is smaller
+than the corresponding dimension of `input`, the dimension is cropped. If it is
+larger, the dimension is padded with zeros.
+END
+}
+op {
+  graph_op_name: "IRFFT2D"
+  endpoint {
+    name: "IRFFT2D"
+  }
+  summary: "Inverse 2D real-valued fast Fourier transform."
+  description: <<END
+Computes the inverse 2-dimensional discrete Fourier transform of a real-valued
+signal over the inner-most 2 dimensions of `input`.
+
+The inner-most 2 dimensions of `input` are assumed to be the result of `RFFT2D`:
+The inner-most dimension contains the `fft_length / 2 + 1` unique components of
+the DFT of a real-valued signal. If `fft_length` is not provided, it is computed
+from the size of the inner-most 2 dimensions of `input`. If the FFT length used
+to compute `input` is odd, it should be provided since it cannot be inferred
+properly.
+
+Along each axis `IRFFT2D` is computed on, if `fft_length` (or
+`fft_length / 2 + 1` for the inner-most dimension) is smaller than the
+corresponding dimension of `input`, the dimension is cropped. If it is larger,
+the dimension is padded with zeros.
+END
+}
+op {
+  graph_op_name: "IRFFT3D"
+  endpoint {
+    name: "IRFFT3D"
+  }
+  summary: "Inverse 3D real-valued fast Fourier transform."
+  description: <<END
+Computes the inverse 3-dimensional discrete Fourier transform of a real-valued
+signal over the inner-most 3 dimensions of `input`.
+
+The inner-most 3 dimensions of `input` are assumed to be the result of `RFFT3D`:
+The inner-most dimension contains the `fft_length / 2 + 1` unique components of
+the DFT of a real-valued signal. If `fft_length` is not provided, it is computed
+from the size of the inner-most 3 dimensions of `input`. If the FFT length used
+to compute `input` is odd, it should be provided since it cannot be inferred
+properly.
+
+Along each axis `IRFFT3D` is computed on, if `fft_length` (or
+`fft_length / 2 + 1` for the inner-most dimension) is smaller than the
+corresponding dimension of `input`, the dimension is cropped. If it is larger,
+the dimension is padded with zeros.
+END
+}
+op {
+  graph_op_name: "Identity"
+  endpoint {
+    name: "Identity"
+  }
+  summary: "Return a tensor with the same shape and contents as the input tensor or value."
+}
+op {
+  graph_op_name: "IdentityN"
+  endpoint {
+    name: "IdentityN"
+  }
+  summary: "Returns a list of tensors with the same shapes and contents as the input"
+  description: <<END
+tensors.
+
+This op can be used to override the gradient for complicated functions. For
+example, suppose y = f(x) and we wish to apply a custom function g for backprop
+such that dx = g(dy). In Python,
+
+```python
+with tf.get_default_graph().gradient_override_map(
+    {'IdentityN': 'OverrideGradientWithG'}):
+  y, _ = identity_n([f(x), x])
+
+@tf.RegisterGradient('OverrideGradientWithG')
+def ApplyG(op, dy, _):
+  return [None, g(dy)]  # Do not backprop to f(x).
+```
+END
+}
+op {
+  graph_op_name: "IdentityReader"
+  endpoint {
+    name: "IdentityReader"
+  }
+  summary: "A Reader that outputs the queued work as both the key and value."
+  description: <<END
+To use, enqueue strings in a Queue.  ReaderRead will take the front
+work string and output (work, work).
+END
+}
+op {
+  graph_op_name: "IdentityReaderV2"
+  endpoint {
+    name: "IdentityReaderV2"
+  }
+  summary: "A Reader that outputs the queued work as both the key and value."
+  description: <<END
+To use, enqueue strings in a Queue.  ReaderRead will take the front
+work string and output (work, work).
+END
+}
+op {
+  graph_op_name: "Igamma"
+  endpoint {
+    name: "Igamma"
+  }
+  summary: "Compute the lower regularized incomplete Gamma function `Q(a, x)`."
+  description: <<END
+The lower regularized incomplete Gamma function is defined as:
+
+
+\\(P(a, x) = gamma(a, x) / Gamma(a) = 1 - Q(a, x)\\)
+
+where
+
+\\(gamma(a, x) = int_{0}^{x} t^{a-1} exp(-t) dt\\)
+
+is the lower incomplete Gamma function.
+
+Note, above `Q(a, x)` (`Igammac`) is the upper regularized complete
+Gamma function.
+END
+}
+op {
+  graph_op_name: "Igammac"
+  endpoint {
+    name: "Igammac"
+  }
+  summary: "Compute the upper regularized incomplete Gamma function `Q(a, x)`."
+  description: <<END
+The upper regularized incomplete Gamma function is defined as:
+
+\\(Q(a, x) = Gamma(a, x) / Gamma(a) = 1 - P(a, x)\\)
+
+where
+
+\\(Gamma(a, x) = int_{x}^{\infty} t^{a-1} exp(-t) dt\\)
+
+is the upper incomplete Gama function.
+
+Note, above `P(a, x)` (`Igamma`) is the lower regularized complete
+Gamma function.
+END
+}
+op {
+  graph_op_name: "IgnoreErrorsDataset"
+  endpoint {
+    name: "IgnoreErrorsDataset"
+  }
+  summary: "Creates a dataset that contains the elements of `input_dataset` ignoring errors."
+}
+op {
+  graph_op_name: "Imag"
+  endpoint {
+    name: "Imag"
+  }
+  summary: "Returns the imaginary part of a complex number."
+  description: <<END
+Given a tensor `input` of complex numbers, this operation returns a tensor of
+type `float` that is the imaginary part of each element in `input`. All
+elements in `input` must be complex numbers of the form \\(a + bj\\), where *a*
+is the real part and *b* is the imaginary part returned by this operation.
+
+For example:
+
+```
+# tensor 'input' is [-2.25 + 4.75j, 3.25 + 5.75j]
+tf.imag(input) ==> [4.75, 5.75]
+```
+END
+}
+op {
+  graph_op_name: "ImageSummary"
+  endpoint {
+    name: "ImageSummary"
+  }
+  summary: "Outputs a `Summary` protocol buffer with images."
+  description: <<END
+The summary has up to `max_images` summary values containing images. The
+images are built from `tensor` which must be 4-D with shape `[batch_size,
+height, width, channels]` and where `channels` can be:
+
+*  1: `tensor` is interpreted as Grayscale.
+*  3: `tensor` is interpreted as RGB.
+*  4: `tensor` is interpreted as RGBA.
+
+The images have the same number of channels as the input tensor. For float
+input, the values are normalized one image at a time to fit in the range
+`[0, 255]`.  `uint8` values are unchanged.  The op uses two different
+normalization algorithms:
+
+*  If the input values are all positive, they are rescaled so the largest one
+   is 255.
+
+*  If any input value is negative, the values are shifted so input value 0.0
+   is at 127.  They are then rescaled so that either the smallest value is 0,
+   or the largest one is 255.
+
+The `tag` argument is a scalar `Tensor` of type `string`.  It is used to
+build the `tag` of the summary values:
+
+*  If `max_images` is 1, the summary value tag is '*tag*/image'.
+*  If `max_images` is greater than 1, the summary value tags are
+   generated sequentially as '*tag*/image/0', '*tag*/image/1', etc.
+
+The `bad_color` argument is the color to use in the generated images for
+non-finite input values.  It is a `unit8` 1-D tensor of length `channels`.
+Each element must be in the range `[0, 255]` (It represents the value of a
+pixel in the output image).  Non-finite values in the input tensor are
+replaced by this tensor in the output image.  The default value is the color
+red.
+END
+}
+op {
+  graph_op_name: "ImmutableConst"
+  endpoint {
+    name: "ImmutableConst"
+  }
+  summary: "Returns immutable tensor from memory region."
+  description: <<END
+The current implementation memmaps the tensor from a file.
+END
+}
+op {
+  graph_op_name: "InTopK"
+  endpoint {
+    name: "InTopK"
+  }
+  summary: "Says whether the targets are in the top `K` predictions."
+  description: <<END
+This outputs a `batch_size` bool array, an entry `out[i]` is `true` if the
+prediction for the target class is among the top `k` predictions among
+all predictions for example `i`. Note that the behavior of `InTopK` differs
+from the `TopK` op in its handling of ties; if multiple classes have the
+same prediction value and straddle the top-`k` boundary, all of those
+classes are considered to be in the top `k`.
+
+More formally, let
+
+  \\(predictions_i\\) be the predictions for all classes for example `i`,
+  \\(targets_i\\) be the target class for example `i`,
+  \\(out_i\\) be the output for example `i`,
+
+$$out_i = predictions_{i, targets_i} \in TopKIncludingTies(predictions_i)$$
+END
+}
+op {
+  graph_op_name: "InTopKV2"
+  endpoint {
+    name: "InTopKV2"
+  }
+  summary: "Says whether the targets are in the top `K` predictions."
+  description: <<END
+This outputs a `batch_size` bool array, an entry `out[i]` is `true` if the
+prediction for the target class is among the top `k` predictions among
+all predictions for example `i`. Note that the behavior of `InTopK` differs
+from the `TopK` op in its handling of ties; if multiple classes have the
+same prediction value and straddle the top-`k` boundary, all of those
+classes are considered to be in the top `k`.
+
+More formally, let
+
+  \\(predictions_i\\) be the predictions for all classes for example `i`,
+  \\(targets_i\\) be the target class for example `i`,
+  \\(out_i\\) be the output for example `i`,
+
+$$out_i = predictions_{i, targets_i} \in TopKIncludingTies(predictions_i)$$
+END
+}
+op {
+  graph_op_name: "InitializeTable"
+  endpoint {
+    name: "InitializeTable"
+  }
+  summary: "Table initializer that takes two tensors for keys and values respectively."
+}
+op {
+  graph_op_name: "InitializeTableFromTextFile"
+  endpoint {
+    name: "InitializeTableFromTextFile"
+  }
+  summary: "Initializes a table from a text file."
+  description: <<END
+It inserts one key-value pair into the table for each line of the file.
+The key and value is extracted from the whole line content, elements from the
+split line based on `delimiter` or the line number (starting from zero).
+Where to extract the key and value from a line is specified by `key_index` and
+`value_index`.
+
+- A value of -1 means use the line number(starting from zero), expects `int64`.
+- A value of -2 means use the whole line content, expects `string`.
+- A value >= 0 means use the index (starting at zero) of the split line based
+  on `delimiter`.
+END
+}
+op {
+  graph_op_name: "InitializeTableFromTextFileV2"
+  endpoint {
+    name: "InitializeTableFromTextFileV2"
+  }
+  summary: "Initializes a table from a text file."
+  description: <<END
+It inserts one key-value pair into the table for each line of the file.
+The key and value is extracted from the whole line content, elements from the
+split line based on `delimiter` or the line number (starting from zero).
+Where to extract the key and value from a line is specified by `key_index` and
+`value_index`.
+
+- A value of -1 means use the line number(starting from zero), expects `int64`.
+- A value of -2 means use the whole line content, expects `string`.
+- A value >= 0 means use the index (starting at zero) of the split line based
+  on `delimiter`.
+END
+}
+op {
+  graph_op_name: "InitializeTableV2"
+  endpoint {
+    name: "InitializeTableV2"
+  }
+  summary: "Table initializer that takes two tensors for keys and values respectively."
+}
+op {
+  graph_op_name: "InterleaveDataset"
+  endpoint {
+    name: "InterleaveDataset"
+  }
+  summary: "Creates a dataset that applies `f` to the outputs of `input_dataset`."
+  description: <<END
+Unlike MapDataset, the `f` in InterleaveDataset is expected to return
+a Dataset variant, and InterleaveDataset will flatten successive
+results into a single Dataset. Unlike FlatMapDataset,
+InterleaveDataset will interleave sequences of up to `block_length`
+consecutive elements from `cycle_length` input elements.
+END
+}
+op {
+  graph_op_name: "Inv"
+  endpoint {
+    name: "Inv"
+  }
+  summary: "Computes the reciprocal of x element-wise."
+  description: <<END
+I.e., \\(y = 1 / x\\).
+END
+}
+op {
+  graph_op_name: "InvGrad"
+  endpoint {
+    name: "InvGrad"
+  }
+  summary: "Computes the gradient for the inverse of `x` wrt its input."
+  description: <<END
+Specifically, `grad = -dy * y*y`, where `y = 1/x`, and `dy`
+is the corresponding input gradient.
+END
+}
+op {
+  graph_op_name: "Invert"
+  endpoint {
+    name: "Invert"
+  }
+  summary: "Flips all bits elementwise."
+  description: <<END
+The result will have exactly those bits set, that are not set in `x`. The
+computation is performed on the underlying representation of x.
+END
+}
+op {
+  graph_op_name: "InvertPermutation"
+  endpoint {
+    name: "InvertPermutation"
+  }
+  summary: "Computes the inverse permutation of a tensor."
+  description: <<END
+This operation computes the inverse of an index permutation. It takes a 1-D
+integer tensor `x`, which represents the indices of a zero-based array, and
+swaps each value with its index position. In other words, for an output tensor
+`y` and an input tensor `x`, this operation computes the following:
+
+`y[x[i]] = i for i in [0, 1, ..., len(x) - 1]`
+
+The values must include 0. There can be no duplicate values or negative values.
+
+For example:
+
+```
+# tensor `x` is [3, 4, 0, 2, 1]
+invert_permutation(x) ==> [2, 4, 3, 0, 1]
+```
+END
+}
+op {
+  graph_op_name: "IsFinite"
+  endpoint {
+    name: "IsFinite"
+  }
+  summary: "Returns which elements of x are finite."
+  description: <<END
+@compatibility(numpy)
+Equivalent to np.isfinite
+@end_compatibility
+END
+}
+op {
+  graph_op_name: "IsInf"
+  endpoint {
+    name: "IsInf"
+  }
+  summary: "Returns which elements of x are Inf."
+  description: <<END
+@compatibility(numpy)
+Equivalent to np.isinf
+@end_compatibility
+END
+}
+op {
+  graph_op_name: "IsNan"
+  endpoint {
+    name: "IsNan"
+  }
+  summary: "Returns which elements of x are NaN."
+  description: <<END
+@compatibility(numpy)
+Equivalent to np.isnan
+@end_compatibility
+END
+}
+op {
+  graph_op_name: "IsVariableInitialized"
+  endpoint {
+    name: "IsVariableInitialized"
+  }
+  summary: "Checks whether a tensor has been initialized."
+  description: <<END
+Outputs boolean scalar indicating whether the tensor has been initialized.
+END
+}
+op {
+  graph_op_name: "Iterator"
+  endpoint {
+    name: "Iterator"
+  }
+  summary: "A container for an iterator resource."
+}
+op {
+  graph_op_name: "IteratorFromStringHandle"
+  endpoint {
+    name: "IteratorFromStringHandle"
+  }
+  summary: "Converts the given string representing a handle to an iterator to a resource."
+}
+op {
+  graph_op_name: "IteratorGetNext"
+  endpoint {
+    name: "IteratorGetNext"
+  }
+  summary: "Gets the next output from the given iterator."
+}
+op {
+  graph_op_name: "IteratorToStringHandle"
+  endpoint {
+    name: "IteratorToStringHandle"
+  }
+  summary: "Converts the given `resource_handle` representing an iterator to a string."
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_L.pbtxt b/tensorflow/core/api_def/base_api/api_def_L.pbtxt
new file mode 100644
index 0000000000..09e55eacc7
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_L.pbtxt
@@ -0,0 +1,392 @@
+op {
+  graph_op_name: "L2Loss"
+  endpoint {
+    name: "L2Loss"
+  }
+  summary: "L2 Loss."
+  description: <<END
+Computes half the L2 norm of a tensor without the `sqrt`:
+
+    output = sum(t ** 2) / 2
+END
+}
+op {
+  graph_op_name: "LMDBReader"
+  endpoint {
+    name: "LMDBReader"
+  }
+  summary: "A Reader that outputs the records from a LMDB file."
+}
+op {
+  graph_op_name: "LRN"
+  endpoint {
+    name: "LRN"
+  }
+  summary: "Local Response Normalization."
+  description: <<END
+The 4-D `input` tensor is treated as a 3-D array of 1-D vectors (along the last
+dimension), and each vector is normalized independently.  Within a given vector,
+each component is divided by the weighted, squared sum of inputs within
+`depth_radius`.  In detail,
+
+    sqr_sum[a, b, c, d] =
+        sum(input[a, b, c, d - depth_radius : d + depth_radius + 1] ** 2)
+    output = input / (bias + alpha * sqr_sum) ** beta
+
+For details, see [Krizhevsky et al., ImageNet classification with deep
+convolutional neural networks (NIPS 2012)](http://papers.nips.cc/paper/4824-imagenet-classification-with-deep-convolutional-neural-networks).
+END
+}
+op {
+  graph_op_name: "LRNGrad"
+  endpoint {
+    name: "LRNGrad"
+  }
+  summary: "Gradients for Local Response Normalization."
+}
+op {
+  graph_op_name: "LearnedUnigramCandidateSampler"
+  endpoint {
+    name: "LearnedUnigramCandidateSampler"
+  }
+  summary: "Generates labels for candidate sampling with a learned unigram distribution."
+  description: <<END
+See explanations of candidate sampling and the data formats at
+go/candidate-sampling.
+
+For each batch, this op picks a single set of sampled candidate labels.
+
+The advantages of sampling candidates per-batch are simplicity and the
+possibility of efficient dense matrix multiplication. The disadvantage is that
+the sampled candidates must be chosen independently of the context and of the
+true labels.
+END
+}
+op {
+  graph_op_name: "Less"
+  endpoint {
+    name: "Less"
+  }
+  summary: "Returns the truth value of (x < y) element-wise."
+  description: <<END
+*NOTE*: `Less` supports broadcasting. More about broadcasting
+[here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
+END
+}
+op {
+  graph_op_name: "LessEqual"
+  endpoint {
+    name: "LessEqual"
+  }
+  summary: "Returns the truth value of (x <= y) element-wise."
+  description: <<END
+*NOTE*: `LessEqual` supports broadcasting. More about broadcasting
+[here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
+END
+}
+op {
+  graph_op_name: "Lgamma"
+  endpoint {
+    name: "Lgamma"
+  }
+  summary: "Computes the log of the absolute value of `Gamma(x)` element-wise."
+}
+op {
+  graph_op_name: "LinSpace"
+  endpoint {
+    name: "LinSpace"
+  }
+  summary: "Generates values in an interval."
+  description: <<END
+A sequence of `num` evenly-spaced values are generated beginning at `start`.
+If `num > 1`, the values in the sequence increase by `stop - start / num - 1`,
+so that the last one is exactly `stop`.
+
+For example:
+
+```
+tf.linspace(10.0, 12.0, 3, name="linspace") => [ 10.0  11.0  12.0]
+```
+END
+}
+op {
+  graph_op_name: "ListDiff"
+  endpoint {
+    name: "ListDiff"
+  }
+  summary: "Computes the difference between two lists of numbers or strings."
+  description: <<END
+Given a list `x` and a list `y`, this operation returns a list `out` that
+represents all values that are in `x` but not in `y`. The returned list `out`
+is sorted in the same order that the numbers appear in `x` (duplicates are
+preserved). This operation also returns a list `idx` that represents the
+position of each `out` element in `x`. In other words:
+
+`out[i] = x[idx[i]] for i in [0, 1, ..., len(out) - 1]`
+
+For example, given this input:
+
+```
+x = [1, 2, 3, 4, 5, 6]
+y = [1, 3, 5]
+```
+
+This operation would return:
+
+```
+out ==> [2, 4, 6]
+idx ==> [1, 3, 5]
+```
+END
+}
+op {
+  graph_op_name: "LoadAndRemapMatrix"
+  endpoint {
+    name: "LoadAndRemapMatrix"
+  }
+  summary: "Loads a 2-D (matrix) `Tensor` with name `old_tensor_name` from the checkpoint"
+  description: <<END
+at `ckpt_path` and potentially reorders its rows and columns using the
+specified remappings.
+
+Most users should use one of the wrapper initializers (such as
+`tf.contrib.framework.load_and_remap_matrix_initializer`) instead of this
+function directly.
+
+The remappings are 1-D tensors with the following properties:
+
+* `row_remapping` must have exactly `num_rows` entries. Row `i` of the output
+  matrix will be initialized from the row corresponding to index
+  `row_remapping[i]` in the old `Tensor` from the checkpoint.
+* `col_remapping` must have either 0 entries (indicating that no column
+  reordering is needed) or `num_cols` entries. If specified, column `j` of the
+  output matrix will be initialized from the column corresponding to index
+  `col_remapping[j]` in the old `Tensor` from the checkpoint.
+* A value of -1 in either of the remappings signifies a "missing" entry. In that
+  case, values from the `initializing_values` tensor will be used to fill that
+  missing row or column. If `row_remapping` has `r` missing entries and
+  `col_remapping` has `c` missing entries, then the following condition must be
+  true:
+
+`(r * num_cols) + (c * num_rows) - (r * c) == len(initializing_values)`
+
+The remapping tensors can be generated using the GenerateVocabRemapping op.
+
+As an example, with row_remapping = [1, 0, -1], col_remapping = [0, 2, -1],
+initializing_values = [0.5, -0.5, 0.25, -0.25, 42], and w(i, j) representing
+the value from row i, column j of the old tensor in the checkpoint, the output
+matrix will look like the following:
+
+[[w(1, 0),  w(1, 2),  0.5],
+ [w(0, 0),  w(0, 2), -0.5],
+ [0.25,    -0.25,      42]]
+END
+}
+op {
+  graph_op_name: "Log"
+  endpoint {
+    name: "Log"
+  }
+  summary: "Computes natural logarithm of x element-wise."
+  description: <<END
+I.e., \\(y = \log_e x\\).
+END
+}
+op {
+  graph_op_name: "Log1p"
+  endpoint {
+    name: "Log1p"
+  }
+  summary: "Computes natural logarithm of (1 + x) element-wise."
+  description: <<END
+I.e., \\(y = \log_e (1 + x)\\).
+END
+}
+op {
+  graph_op_name: "LogMatrixDeterminant"
+  endpoint {
+    name: "LogMatrixDeterminant"
+  }
+  summary: "Computes the sign and the log of the absolute value of the determinant of"
+  description: <<END
+one or more square matrices.
+
+The input is a tensor of shape `[N, M, M]` whose inner-most 2 dimensions
+form square matrices. The outputs are two tensors containing the signs and
+absolute values of the log determinants for all N input submatrices
+`[..., :, :]` such that the determinant = sign*exp(log_abs_determinant).
+The log_abs_determinant is computed as det(P)*sum(log(diag(LU))) where LU
+is the LU decomposition of the input and P is the corresponding
+permutation matrix.
+END
+}
+op {
+  graph_op_name: "LogSoftmax"
+  endpoint {
+    name: "LogSoftmax"
+  }
+  summary: "Computes log softmax activations."
+  description: <<END
+For each batch `i` and class `j` we have
+
+    logsoftmax[i, j] = logits[i, j] - log(sum(exp(logits[i])))
+END
+}
+op {
+  graph_op_name: "LogUniformCandidateSampler"
+  endpoint {
+    name: "LogUniformCandidateSampler"
+  }
+  summary: "Generates labels for candidate sampling with a log-uniform distribution."
+  description: <<END
+See explanations of candidate sampling and the data formats at
+go/candidate-sampling.
+
+For each batch, this op picks a single set of sampled candidate labels.
+
+The advantages of sampling candidates per-batch are simplicity and the
+possibility of efficient dense matrix multiplication. The disadvantage is that
+the sampled candidates must be chosen independently of the context and of the
+true labels.
+END
+}
+op {
+  graph_op_name: "LogicalAnd"
+  endpoint {
+    name: "LogicalAnd"
+  }
+  summary: "Returns the truth value of x AND y element-wise."
+  description: <<END
+*NOTE*: `LogicalAnd` supports broadcasting. More about broadcasting
+[here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
+END
+}
+op {
+  graph_op_name: "LogicalNot"
+  endpoint {
+    name: "LogicalNot"
+  }
+  summary: "Returns the truth value of NOT x element-wise."
+}
+op {
+  graph_op_name: "LogicalOr"
+  endpoint {
+    name: "LogicalOr"
+  }
+  summary: "Returns the truth value of x OR y element-wise."
+  description: <<END
+*NOTE*: `LogicalOr` supports broadcasting. More about broadcasting
+[here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
+END
+}
+op {
+  graph_op_name: "LookupTableExport"
+  endpoint {
+    name: "LookupTableExport"
+  }
+  summary: "Outputs all keys and values in the table."
+}
+op {
+  graph_op_name: "LookupTableExportV2"
+  endpoint {
+    name: "LookupTableExportV2"
+  }
+  summary: "Outputs all keys and values in the table."
+}
+op {
+  graph_op_name: "LookupTableFind"
+  endpoint {
+    name: "LookupTableFind"
+  }
+  summary: "Looks up keys in a table, outputs the corresponding values."
+  description: <<END
+The tensor `keys` must of the same type as the keys of the table.
+The output `values` is of the type of the table values.
+
+The scalar `default_value` is the value output for keys not present in the
+table. It must also be of the same type as the table values.
+END
+}
+op {
+  graph_op_name: "LookupTableFindV2"
+  endpoint {
+    name: "LookupTableFindV2"
+  }
+  summary: "Looks up keys in a table, outputs the corresponding values."
+  description: <<END
+The tensor `keys` must of the same type as the keys of the table.
+The output `values` is of the type of the table values.
+
+The scalar `default_value` is the value output for keys not present in the
+table. It must also be of the same type as the table values.
+END
+}
+op {
+  graph_op_name: "LookupTableImport"
+  endpoint {
+    name: "LookupTableImport"
+  }
+  summary: "Replaces the contents of the table with the specified keys and values."
+  description: <<END
+The tensor `keys` must be of the same type as the keys of the table.
+The tensor `values` must be of the type of the table values.
+END
+}
+op {
+  graph_op_name: "LookupTableImportV2"
+  endpoint {
+    name: "LookupTableImportV2"
+  }
+  summary: "Replaces the contents of the table with the specified keys and values."
+  description: <<END
+The tensor `keys` must be of the same type as the keys of the table.
+The tensor `values` must be of the type of the table values.
+END
+}
+op {
+  graph_op_name: "LookupTableInsert"
+  endpoint {
+    name: "LookupTableInsert"
+  }
+  summary: "Updates the table to associates keys with values."
+  description: <<END
+The tensor `keys` must be of the same type as the keys of the table.
+The tensor `values` must be of the type of the table values.
+END
+}
+op {
+  graph_op_name: "LookupTableInsertV2"
+  endpoint {
+    name: "LookupTableInsertV2"
+  }
+  summary: "Updates the table to associates keys with values."
+  description: <<END
+The tensor `keys` must be of the same type as the keys of the table.
+The tensor `values` must be of the type of the table values.
+END
+}
+op {
+  graph_op_name: "LookupTableSize"
+  endpoint {
+    name: "LookupTableSize"
+  }
+  summary: "Computes the number of elements in the given table."
+}
+op {
+  graph_op_name: "LookupTableSizeV2"
+  endpoint {
+    name: "LookupTableSizeV2"
+  }
+  summary: "Computes the number of elements in the given table."
+}
+op {
+  graph_op_name: "LoopCond"
+  endpoint {
+    name: "LoopCond"
+  }
+  summary: "Forwards the input to the output."
+  description: <<END
+This operator represents the loop termination condition used by the
+"pivot" switches of a loop.
+END
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_M.pbtxt b/tensorflow/core/api_def/base_api/api_def_M.pbtxt
new file mode 100644
index 0000000000..7295928bad
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_M.pbtxt
@@ -0,0 +1,749 @@
+op {
+  graph_op_name: "MakeIterator"
+  endpoint {
+    name: "MakeIterator"
+  }
+  summary: "Makes a new iterator from the given `dataset` and stores it in `iterator`."
+  description: <<END
+This operation may be executed multiple times. Each execution will reset the
+iterator in `iterator` to the first element of `dataset`.
+END
+}
+op {
+  graph_op_name: "MapClear"
+  endpoint {
+    name: "MapClear"
+  }
+  summary: "Op removes all elements in the underlying container."
+}
+op {
+  graph_op_name: "MapDataset"
+  endpoint {
+    name: "MapDataset"
+  }
+  summary: "Creates a dataset that applies `f` to the outputs of `input_dataset`."
+}
+op {
+  graph_op_name: "MapIncompleteSize"
+  endpoint {
+    name: "MapIncompleteSize"
+  }
+  summary: "Op returns the number of incomplete elements in the underlying container."
+}
+op {
+  graph_op_name: "MapPeek"
+  endpoint {
+    name: "MapPeek"
+  }
+  summary: "Op peeks at the values at the specified key.  If the"
+  description: <<END
+underlying container does not contain this key
+this op will block until it does.
+END
+}
+op {
+  graph_op_name: "MapSize"
+  endpoint {
+    name: "MapSize"
+  }
+  summary: "Op returns the number of elements in the underlying container."
+}
+op {
+  graph_op_name: "MapStage"
+  endpoint {
+    name: "MapStage"
+  }
+  summary: "Stage (key, values) in the underlying container which behaves like a hashtable."
+}
+op {
+  graph_op_name: "MapUnstage"
+  endpoint {
+    name: "MapUnstage"
+  }
+  summary: "Op removes and returns the values associated with the key"
+  description: <<END
+from the underlying container.   If the underlying container
+does not contain this key, the op will block until it does.
+END
+}
+op {
+  graph_op_name: "MapUnstageNoKey"
+  endpoint {
+    name: "MapUnstageNoKey"
+  }
+  summary: "Op removes and returns a random (key, value)"
+  description: <<END
+from the underlying container.   If the underlying container
+does not contain elements, the op will block until it does.
+END
+}
+op {
+  graph_op_name: "MatMul"
+  endpoint {
+    name: "MatMul"
+  }
+  summary: "Multiply the matrix \"a\" by the matrix \"b\"."
+  description: <<END
+The inputs must be two-dimensional matrices and the inner dimension of
+"a" (after being transposed if transpose_a is true) must match the
+outer dimension of "b" (after being transposed if transposed_b is
+true).
+
+*Note*: The default kernel implementation for MatMul on GPUs uses
+cublas.
+END
+}
+op {
+  graph_op_name: "MatchingFiles"
+  endpoint {
+    name: "MatchingFiles"
+  }
+  summary: "Returns the set of files matching one or more glob patterns."
+  description: <<END
+Note that this routine only supports wildcard characters in the
+basename portion of the pattern, not in the directory portion.
+END
+}
+op {
+  graph_op_name: "MatrixBandPart"
+  endpoint {
+    name: "MatrixBandPart"
+  }
+  summary: "Copy a tensor setting everything outside a central band in each innermost matrix"
+  description: <<END
+to zero.
+
+The `band` part is computed as follows:
+Assume `input` has `k` dimensions `[I, J, K, ..., M, N]`, then the output is a
+tensor with the same shape where
+
+`band[i, j, k, ..., m, n] = in_band(m, n) * input[i, j, k, ..., m, n]`.
+
+The indicator function
+
+`in_band(m, n) = (num_lower < 0 || (m-n) <= num_lower)) &&
+                 (num_upper < 0 || (n-m) <= num_upper)`.
+
+For example:
+
+```
+# if 'input' is [[ 0,  1,  2, 3]
+                 [-1,  0,  1, 2]
+                 [-2, -1,  0, 1]
+                 [-3, -2, -1, 0]],
+
+tf.matrix_band_part(input, 1, -1) ==> [[ 0,  1,  2, 3]
+                                       [-1,  0,  1, 2]
+                                       [ 0, -1,  0, 1]
+                                       [ 0,  0, -1, 0]],
+
+tf.matrix_band_part(input, 2, 1) ==> [[ 0,  1,  0, 0]
+                                      [-1,  0,  1, 0]
+                                      [-2, -1,  0, 1]
+                                      [ 0, -2, -1, 0]]
+```
+
+Useful special cases:
+
+```
+ tf.matrix_band_part(input, 0, -1) ==> Upper triangular part.
+ tf.matrix_band_part(input, -1, 0) ==> Lower triangular part.
+ tf.matrix_band_part(input, 0, 0) ==> Diagonal.
+```
+END
+}
+op {
+  graph_op_name: "MatrixDeterminant"
+  endpoint {
+    name: "MatrixDeterminant"
+  }
+  summary: "Computes the determinant of one or more square matrices."
+  description: <<END
+The input is a tensor of shape `[..., M, M]` whose inner-most 2 dimensions
+form square matrices. The output is a tensor containing the determinants
+for all input submatrices `[..., :, :]`.
+END
+}
+op {
+  graph_op_name: "MatrixDiag"
+  endpoint {
+    name: "MatrixDiag"
+  }
+  summary: "Returns a batched diagonal tensor with a given batched diagonal values."
+  description: <<END
+Given a `diagonal`, this operation returns a tensor with the `diagonal` and
+everything else padded with zeros. The diagonal is computed as follows:
+
+Assume `diagonal` has `k` dimensions `[I, J, K, ..., N]`, then the output is a
+tensor of rank `k+1` with dimensions [I, J, K, ..., N, N]` where:
+
+`output[i, j, k, ..., m, n] = 1{m=n} * diagonal[i, j, k, ..., n]`.
+
+For example:
+
+```
+# 'diagonal' is [[1, 2, 3, 4], [5, 6, 7, 8]]
+
+and diagonal.shape = (2, 4)
+
+tf.matrix_diag(diagonal) ==> [[[1, 0, 0, 0]
+                                     [0, 2, 0, 0]
+                                     [0, 0, 3, 0]
+                                     [0, 0, 0, 4]],
+                                    [[5, 0, 0, 0]
+                                     [0, 6, 0, 0]
+                                     [0, 0, 7, 0]
+                                     [0, 0, 0, 8]]]
+
+which has shape (2, 4, 4)
+```
+END
+}
+op {
+  graph_op_name: "MatrixDiagPart"
+  endpoint {
+    name: "MatrixDiagPart"
+  }
+  summary: "Returns the batched diagonal part of a batched tensor."
+  description: <<END
+This operation returns a tensor with the `diagonal` part
+of the batched `input`. The `diagonal` part is computed as follows:
+
+Assume `input` has `k` dimensions `[I, J, K, ..., M, N]`, then the output is a
+tensor of rank `k - 1` with dimensions `[I, J, K, ..., min(M, N)]` where:
+
+`diagonal[i, j, k, ..., n] = input[i, j, k, ..., n, n]`.
+
+The input must be at least a matrix.
+
+For example:
+
+```
+# 'input' is [[[1, 0, 0, 0]
+               [0, 2, 0, 0]
+               [0, 0, 3, 0]
+               [0, 0, 0, 4]],
+              [[5, 0, 0, 0]
+               [0, 6, 0, 0]
+               [0, 0, 7, 0]
+               [0, 0, 0, 8]]]
+
+and input.shape = (2, 4, 4)
+
+tf.matrix_diag_part(input) ==> [[1, 2, 3, 4], [5, 6, 7, 8]]
+
+which has shape (2, 4)
+```
+END
+}
+op {
+  graph_op_name: "MatrixInverse"
+  endpoint {
+    name: "MatrixInverse"
+  }
+  summary: "Computes the inverse of one or more square invertible matrices or their"
+  description: <<END
+adjoints (conjugate transposes).
+
+The input is a tensor of shape `[..., M, M]` whose inner-most 2 dimensions
+form square matrices. The output is a tensor of the same shape as the input
+containing the inverse for all input submatrices `[..., :, :]`.
+
+The op uses LU decomposition with partial pivoting to compute the inverses.
+
+If a matrix is not invertible there is no guarantee what the op does. It
+may detect the condition and raise an exception or it may simply return a
+garbage result.
+END
+}
+op {
+  graph_op_name: "MatrixSetDiag"
+  endpoint {
+    name: "MatrixSetDiag"
+  }
+  summary: "Returns a batched matrix tensor with new batched diagonal values."
+  description: <<END
+Given `input` and `diagonal`, this operation returns a tensor with the
+same shape and values as `input`, except for the main diagonal of the
+innermost matrices.  These will be overwritten by the values in `diagonal`.
+
+The output is computed as follows:
+
+Assume `input` has `k+1` dimensions `[I, J, K, ..., M, N]` and `diagonal` has
+`k` dimensions `[I, J, K, ..., min(M, N)]`.  Then the output is a
+tensor of rank `k+1` with dimensions `[I, J, K, ..., M, N]` where:
+
+  * `output[i, j, k, ..., m, n] = diagonal[i, j, k, ..., n]` for `m == n`.
+  * `output[i, j, k, ..., m, n] = input[i, j, k, ..., m, n]` for `m != n`.
+END
+}
+op {
+  graph_op_name: "MatrixSolve"
+  endpoint {
+    name: "MatrixSolve"
+  }
+  summary: "Solves systems of linear equations."
+  description: <<END
+`Matrix` is a tensor of shape `[..., M, M]` whose inner-most 2 dimensions
+form square matrices. `Rhs` is a tensor of shape `[..., M, K]`. The `output` is
+a tensor shape `[..., M, K]`.  If `adjoint` is `False` then each output matrix
+satisfies `matrix[..., :, :] * output[..., :, :] = rhs[..., :, :]`.
+If `adjoint` is `True` then each output matrix satisfies
+`adjoint(matrix[..., :, :]) * output[..., :, :] = rhs[..., :, :]`.
+END
+}
+op {
+  graph_op_name: "MatrixSolveLs"
+  endpoint {
+    name: "MatrixSolveLs"
+  }
+  summary: "Solves one or more linear least-squares problems."
+  description: <<END
+`matrix` is a tensor of shape `[..., M, N]` whose inner-most 2 dimensions
+form real or complex matrices of size `[M, N]`. `Rhs` is a tensor of the same
+type as `matrix` and shape `[..., M, K]`.
+The output is a tensor shape `[..., N, K]` where each output matrix solves
+each of the equations
+`matrix[..., :, :]` * `output[..., :, :]` = `rhs[..., :, :]`
+in the least squares sense.
+
+We use the following notation for (complex) matrix and right-hand sides
+in the batch:
+
+`matrix`=\\(A \in \mathbb{C}^{m \times n}\\),
+`rhs`=\\(B  \in \mathbb{C}^{m \times k}\\),
+`output`=\\(X  \in \mathbb{C}^{n \times k}\\),
+`l2_regularizer`=\\(\lambda \in \mathbb{R}\\).
+
+If `fast` is `True`, then the solution is computed by solving the normal
+equations using Cholesky decomposition. Specifically, if \\(m \ge n\\) then
+\\(X = (A^H A + \lambda I)^{-1} A^H B\\), which solves the least-squares
+problem \\(X = \mathrm{argmin}_{Z \in \Re^{n \times k} } ||A Z - B||_F^2 +
+\lambda ||Z||_F^2\\). If \\(m \lt n\\) then `output` is computed as
+\\(X = A^H (A A^H + \lambda I)^{-1} B\\), which (for \\(\lambda = 0\\)) is the
+minimum-norm solution to the under-determined linear system, i.e.
+\\(X = \mathrm{argmin}_{Z \in \mathbb{C}^{n \times k} } ||Z||_F^2 \\),
+subject to \\(A Z = B\\). Notice that the fast path is only numerically stable
+when \\(A\\) is numerically full rank and has a condition number
+\\(\mathrm{cond}(A) \lt \frac{1}{\sqrt{\epsilon_{mach} } }\\) or\\(\lambda\\) is
+sufficiently large.
+
+If `fast` is `False` an algorithm based on the numerically robust complete
+orthogonal decomposition is used. This computes the minimum-norm
+least-squares solution, even when \\(A\\) is rank deficient. This path is
+typically 6-7 times slower than the fast path. If `fast` is `False` then
+`l2_regularizer` is ignored.
+END
+}
+op {
+  graph_op_name: "MatrixTriangularSolve"
+  endpoint {
+    name: "MatrixTriangularSolve"
+  }
+  summary: "Solves systems of linear equations with upper or lower triangular matrices by"
+  description: <<END
+backsubstitution.
+
+`matrix` is a tensor of shape `[..., M, M]` whose inner-most 2 dimensions form
+square matrices. If `lower` is `True` then the strictly upper triangular part
+of each inner-most matrix is assumed to be zero and not accessed.
+If `lower` is False then the strictly lower triangular part of each inner-most
+matrix is assumed to be zero and not accessed.
+`rhs` is a tensor of shape `[..., M, K]`.
+
+The output is a tensor of shape `[..., M, K]`. If `adjoint` is
+`True` then the innermost matrices in `output` satisfy matrix equations
+`matrix[..., :, :] * output[..., :, :] = rhs[..., :, :]`.
+If `adjoint` is `False` then the strictly then the  innermost matrices in
+`output` satisfy matrix equations
+`adjoint(matrix[..., i, k]) * output[..., k, j] = rhs[..., i, j]`.
+END
+}
+op {
+  graph_op_name: "Max"
+  endpoint {
+    name: "Max"
+  }
+  summary: "Computes the maximum of elements across dimensions of a tensor."
+  description: <<END
+Reduces `input` along the dimensions given in `reduction_indices`. Unless
+`keep_dims` is true, the rank of the tensor is reduced by 1 for each entry in
+`reduction_indices`. If `keep_dims` is true, the reduced dimensions are
+retained with length 1.
+END
+}
+op {
+  graph_op_name: "MaxPool"
+  endpoint {
+    name: "MaxPool"
+  }
+  summary: "Performs max pooling on the input."
+}
+op {
+  graph_op_name: "MaxPool3D"
+  endpoint {
+    name: "MaxPool3D"
+  }
+  summary: "Performs 3D max pooling on the input."
+}
+op {
+  graph_op_name: "MaxPool3DGrad"
+  endpoint {
+    name: "MaxPool3DGrad"
+  }
+  summary: "Computes gradients of max pooling function."
+}
+op {
+  graph_op_name: "MaxPool3DGradGrad"
+  endpoint {
+    name: "MaxPool3DGradGrad"
+  }
+  summary: "Computes second-order gradients of the maxpooling function."
+}
+op {
+  graph_op_name: "MaxPoolGrad"
+  endpoint {
+    name: "MaxPoolGrad"
+  }
+  summary: "Computes gradients of the maxpooling function."
+}
+op {
+  graph_op_name: "MaxPoolGradGrad"
+  endpoint {
+    name: "MaxPoolGradGrad"
+  }
+  summary: "Computes second-order gradients of the maxpooling function."
+}
+op {
+  graph_op_name: "MaxPoolGradGradV2"
+  endpoint {
+    name: "MaxPoolGradGradV2"
+  }
+  summary: "Computes second-order gradients of the maxpooling function."
+}
+op {
+  graph_op_name: "MaxPoolGradGradWithArgmax"
+  endpoint {
+    name: "MaxPoolGradGradWithArgmax"
+  }
+  summary: "Computes second-order gradients of the maxpooling function."
+}
+op {
+  graph_op_name: "MaxPoolGradV2"
+  endpoint {
+    name: "MaxPoolGradV2"
+  }
+  summary: "Computes gradients of the maxpooling function."
+}
+op {
+  graph_op_name: "MaxPoolGradWithArgmax"
+  endpoint {
+    name: "MaxPoolGradWithArgmax"
+  }
+  summary: "Computes gradients of the maxpooling function."
+}
+op {
+  graph_op_name: "MaxPoolV2"
+  endpoint {
+    name: "MaxPoolV2"
+  }
+  summary: "Performs max pooling on the input."
+}
+op {
+  graph_op_name: "MaxPoolWithArgmax"
+  endpoint {
+    name: "MaxPoolWithArgmax"
+  }
+  summary: "Performs max pooling on the input and outputs both max values and indices."
+  description: <<END
+The indices in `argmax` are flattened, so that a maximum value at position
+`[b, y, x, c]` becomes flattened index
+`((b * height + y) * width + x) * channels + c`.
+
+The indices returned are always in `[0, height) x [0, width)` before flattening,
+even if padding is involved and the mathematically correct answer is outside
+(either negative or too large).  This is a bug, but fixing it is difficult to do
+in a safe backwards compatible way, especially due to flattening.
+END
+}
+op {
+  graph_op_name: "Maximum"
+  endpoint {
+    name: "Maximum"
+  }
+  summary: "Returns the max of x and y (i.e. x > y ? x : y) element-wise."
+  description: <<END
+*NOTE*: `Maximum` supports broadcasting. More about broadcasting
+[here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
+END
+}
+op {
+  graph_op_name: "Mean"
+  endpoint {
+    name: "Mean"
+  }
+  summary: "Computes the mean of elements across dimensions of a tensor."
+  description: <<END
+Reduces `input` along the dimensions given in `reduction_indices`. Unless
+`keep_dims` is true, the rank of the tensor is reduced by 1 for each entry in
+`reduction_indices`. If `keep_dims` is true, the reduced dimensions are
+retained with length 1.
+END
+}
+op {
+  graph_op_name: "Merge"
+  endpoint {
+    name: "Merge"
+  }
+  summary: "Forwards the value of an available tensor from `inputs` to `output`."
+  description: <<END
+`Merge` waits for at least one of the tensors in `inputs` to become available.
+It is usually combined with `Switch` to implement branching.
+
+`Merge` forwards the first tensor to become available to `output`, and sets
+`value_index` to its index in `inputs`.
+END
+}
+op {
+  graph_op_name: "MergeSummary"
+  endpoint {
+    name: "MergeSummary"
+  }
+  summary: "Merges summaries."
+  description: <<END
+This op creates a
+[`Summary`](https://www.tensorflow.org/code/tensorflow/core/framework/summary.proto)
+protocol buffer that contains the union of all the values in the input
+summaries.
+
+When the Op is run, it reports an `InvalidArgument` error if multiple values
+in the summaries to merge use the same tag.
+END
+}
+op {
+  graph_op_name: "MergeV2Checkpoints"
+  endpoint {
+    name: "MergeV2Checkpoints"
+  }
+  summary: "V2 format specific: merges the metadata files of sharded checkpoints.  The"
+  description: <<END
+result is one logical checkpoint, with one physical metadata file and renamed
+data files.
+
+Intended for "grouping" multiple checkpoints in a sharded checkpoint setup.
+
+If delete_old_dirs is true, attempts to delete recursively the dirname of each
+path in the input checkpoint_prefixes.  This is useful when those paths are non
+user-facing temporary locations.
+END
+}
+op {
+  graph_op_name: "Mfcc"
+  endpoint {
+    name: "Mfcc"
+  }
+  summary: "Transforms a spectrogram into a form that\'s useful for speech recognition."
+  description: <<END
+Mel Frequency Cepstral Coefficients are a way of representing audio data that's
+been effective as an input feature for machine learning. They are created by
+taking the spectrum of a spectrogram (a 'cepstrum'), and discarding some of the
+higher frequencies that are less significant to the human ear. They have a long
+history in the speech recognition world, and https://en.wikipedia.org/wiki/Mel-frequency_cepstrum
+is a good resource to learn more.
+END
+}
+op {
+  graph_op_name: "Min"
+  endpoint {
+    name: "Min"
+  }
+  summary: "Computes the minimum of elements across dimensions of a tensor."
+  description: <<END
+Reduces `input` along the dimensions given in `reduction_indices`. Unless
+`keep_dims` is true, the rank of the tensor is reduced by 1 for each entry in
+`reduction_indices`. If `keep_dims` is true, the reduced dimensions are
+retained with length 1.
+END
+}
+op {
+  graph_op_name: "Minimum"
+  endpoint {
+    name: "Minimum"
+  }
+  summary: "Returns the min of x and y (i.e. x < y ? x : y) element-wise."
+  description: <<END
+*NOTE*: `Minimum` supports broadcasting. More about broadcasting
+[here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
+END
+}
+op {
+  graph_op_name: "MirrorPad"
+  endpoint {
+    name: "MirrorPad"
+  }
+  summary: "Pads a tensor with mirrored values."
+  description: <<END
+This operation pads a `input` with mirrored values according to the `paddings`
+you specify. `paddings` is an integer tensor with shape `[n, 2]`, where n is
+the rank of `input`. For each dimension D of `input`, `paddings[D, 0]` indicates
+how many values to add before the contents of `input` in that dimension, and
+`paddings[D, 1]` indicates how many values to add after the contents of `input`
+in that dimension. Both `paddings[D, 0]` and `paddings[D, 1]` must be no greater
+than `input.dim_size(D)` (or `input.dim_size(D) - 1`) if `copy_border` is true
+(if false, respectively).
+
+The padded size of each dimension D of the output is:
+
+`paddings(D, 0) + input.dim_size(D) + paddings(D, 1)`
+
+For example:
+
+```
+# 't' is [[1, 2, 3], [4, 5, 6]].
+# 'paddings' is [[1, 1]], [2, 2]].
+# 'mode' is SYMMETRIC.
+# rank of 't' is 2.
+pad(t, paddings) ==> [[2, 1, 1, 2, 3, 3, 2]
+                      [2, 1, 1, 2, 3, 3, 2]
+                      [5, 4, 4, 5, 6, 6, 5]
+                      [5, 4, 4, 5, 6, 6, 5]]
+```
+END
+}
+op {
+  graph_op_name: "MirrorPadGrad"
+  endpoint {
+    name: "MirrorPadGrad"
+  }
+  summary: "Gradient op for `MirrorPad` op. This op folds a mirror-padded tensor."
+  description: <<END
+This operation folds the padded areas of `input` by `MirrorPad` according to the
+`paddings` you specify. `paddings` must be the same as `paddings` argument
+given to the corresponding `MirrorPad` op.
+
+The folded size of each dimension D of the output is:
+
+`input.dim_size(D) - paddings(D, 0) - paddings(D, 1)`
+
+For example:
+
+```
+# 't' is [[1, 2, 3], [4, 5, 6], [7, 8, 9]].
+# 'paddings' is [[0, 1]], [0, 1]].
+# 'mode' is SYMMETRIC.
+# rank of 't' is 2.
+pad(t, paddings) ==> [[ 1,  5]
+                      [11, 28]]
+```
+END
+}
+op {
+  graph_op_name: "Mod"
+  endpoint {
+    name: "Mod"
+  }
+  summary: "Returns element-wise remainder of division. This emulates C semantics in that"
+  description: <<END
+the result here is consistent with a truncating divide. E.g. `truncate(x / y) *
+y + truncate_mod(x, y) = x`.
+
+*NOTE*: `Mod` supports broadcasting. More about broadcasting
+[here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
+END
+}
+op {
+  graph_op_name: "Mul"
+  endpoint {
+    name: "Mul"
+  }
+  summary: "Returns x * y element-wise."
+  description: <<END
+*NOTE*: `Mul` supports broadcasting. More about broadcasting
+[here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
+END
+}
+op {
+  graph_op_name: "Multinomial"
+  endpoint {
+    name: "Multinomial"
+  }
+  summary: "Draws samples from a multinomial distribution."
+}
+op {
+  graph_op_name: "MutableDenseHashTable"
+  endpoint {
+    name: "MutableDenseHashTable"
+  }
+  summary: "Creates an empty hash table that uses tensors as the backing store."
+  description: <<END
+It uses "open addressing" with quadratic reprobing to resolve
+collisions.
+
+This op creates a mutable hash table, specifying the type of its keys and
+values. Each value must be a scalar. Data can be inserted into the table using
+the insert operations. It does not support the initialization operation.
+END
+}
+op {
+  graph_op_name: "MutableDenseHashTableV2"
+  endpoint {
+    name: "MutableDenseHashTableV2"
+  }
+  summary: "Creates an empty hash table that uses tensors as the backing store."
+  description: <<END
+It uses "open addressing" with quadratic reprobing to resolve
+collisions.
+
+This op creates a mutable hash table, specifying the type of its keys and
+values. Each value must be a scalar. Data can be inserted into the table using
+the insert operations. It does not support the initialization operation.
+END
+}
+op {
+  graph_op_name: "MutableHashTable"
+  endpoint {
+    name: "MutableHashTable"
+  }
+  summary: "Creates an empty hash table."
+  description: <<END
+This op creates a mutable hash table, specifying the type of its keys and
+values. Each value must be a scalar. Data can be inserted into the table using
+the insert operations. It does not support the initialization operation.
+END
+}
+op {
+  graph_op_name: "MutableHashTableOfTensors"
+  endpoint {
+    name: "MutableHashTableOfTensors"
+  }
+  summary: "Creates an empty hash table."
+  description: <<END
+This op creates a mutable hash table, specifying the type of its keys and
+values. Each value must be a vector. Data can be inserted into the table using
+the insert operations. It does not support the initialization operation.
+END
+}
+op {
+  graph_op_name: "MutableHashTableOfTensorsV2"
+  endpoint {
+    name: "MutableHashTableOfTensorsV2"
+  }
+  summary: "Creates an empty hash table."
+  description: <<END
+This op creates a mutable hash table, specifying the type of its keys and
+values. Each value must be a vector. Data can be inserted into the table using
+the insert operations. It does not support the initialization operation.
+END
+}
+op {
+  graph_op_name: "MutableHashTableV2"
+  endpoint {
+    name: "MutableHashTableV2"
+  }
+  summary: "Creates an empty hash table."
+  description: <<END
+This op creates a mutable hash table, specifying the type of its keys and
+values. Each value must be a scalar. Data can be inserted into the table using
+the insert operations. It does not support the initialization operation.
+END
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_N.pbtxt b/tensorflow/core/api_def/base_api/api_def_N.pbtxt
new file mode 100644
index 0000000000..0298a42cab
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_N.pbtxt
@@ -0,0 +1,94 @@
+op {
+  graph_op_name: "Neg"
+  endpoint {
+    name: "Neg"
+  }
+  summary: "Computes numerical negative value element-wise."
+  description: <<END
+I.e., \\(y = -x\\).
+END
+}
+op {
+  graph_op_name: "NegTrain"
+  endpoint {
+    name: "NegTrain"
+  }
+  summary: "Training via negative sampling."
+}
+op {
+  graph_op_name: "NextIteration"
+  endpoint {
+    name: "NextIteration"
+  }
+  summary: "Makes its input available to the next iteration."
+}
+op {
+  graph_op_name: "NoOp"
+  endpoint {
+    name: "NoOp"
+  }
+  summary: "Does nothing. Only useful as a placeholder for control edges."
+}
+op {
+  graph_op_name: "NonMaxSuppression"
+  endpoint {
+    name: "NonMaxSuppression"
+  }
+  summary: "Greedily selects a subset of bounding boxes in descending order of score,"
+  description: <<END
+pruning away boxes that have high intersection-over-union (IOU) overlap
+with previously selected boxes.  Bounding boxes are supplied as
+[y1, x1, y2, x2], where (y1, x1) and (y2, x2) are the coordinates of any
+diagonal pair of box corners and the coordinates can be provided as normalized
+(i.e., lying in the interval [0, 1]) or absolute.  Note that this algorithm
+is agnostic to where the origin is in the coordinate system.  Note that this
+algorithm is invariant to orthogonal transformations and translations
+of the coordinate system; thus translating or reflections of the coordinate
+system result in the same boxes being selected by the algorithm.
+The output of this operation is a set of integers indexing into the input
+collection of bounding boxes representing the selected boxes.  The bounding
+box coordinates corresponding to the selected indices can then be obtained
+using the `tf.gather operation`.  For example:
+  selected_indices = tf.image.non_max_suppression(
+      boxes, scores, max_output_size, iou_threshold)
+  selected_boxes = tf.gather(boxes, selected_indices)
+END
+}
+op {
+  graph_op_name: "NonMaxSuppressionV2"
+  endpoint {
+    name: "NonMaxSuppressionV2"
+  }
+  summary: "Greedily selects a subset of bounding boxes in descending order of score,"
+  description: <<END
+pruning away boxes that have high intersection-over-union (IOU) overlap
+with previously selected boxes.  Bounding boxes are supplied as
+[y1, x1, y2, x2], where (y1, x1) and (y2, x2) are the coordinates of any
+diagonal pair of box corners and the coordinates can be provided as normalized
+(i.e., lying in the interval [0, 1]) or absolute.  Note that this algorithm
+is agnostic to where the origin is in the coordinate system.  Note that this
+algorithm is invariant to orthogonal transformations and translations
+of the coordinate system; thus translating or reflections of the coordinate
+system result in the same boxes being selected by the algorithm.
+
+The output of this operation is a set of integers indexing into the input
+collection of bounding boxes representing the selected boxes.  The bounding
+box coordinates corresponding to the selected indices can then be obtained
+using the `tf.gather operation`.  For example:
+
+  selected_indices = tf.image.non_max_suppression_v2(
+      boxes, scores, max_output_size, iou_threshold)
+  selected_boxes = tf.gather(boxes, selected_indices)
+END
+}
+op {
+  graph_op_name: "NotEqual"
+  endpoint {
+    name: "NotEqual"
+  }
+  summary: "Returns the truth value of (x != y) element-wise."
+  description: <<END
+*NOTE*: `NotEqual` supports broadcasting. More about broadcasting
+[here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
+END
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_O.pbtxt b/tensorflow/core/api_def/base_api/api_def_O.pbtxt
new file mode 100644
index 0000000000..3c62335da9
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_O.pbtxt
@@ -0,0 +1,195 @@
+op {
+  graph_op_name: "OneHot"
+  endpoint {
+    name: "OneHot"
+  }
+  summary: "Returns a one-hot tensor."
+  description: <<END
+The locations represented by indices in `indices` take value `on_value`,
+while all other locations take value `off_value`.
+
+If the input `indices` is rank `N`, the output will have rank `N+1`,
+The new axis is created at dimension `axis` (default: the new axis is
+appended at the end).
+
+If `indices` is a scalar the output shape will be a vector of length `depth`.
+
+If `indices` is a vector of length `features`, the output shape will be:
+```
+  features x depth if axis == -1
+  depth x features if axis == 0
+```
+
+If `indices` is a matrix (batch) with shape `[batch, features]`,
+the output shape will be:
+```
+  batch x features x depth if axis == -1
+  batch x depth x features if axis == 1
+  depth x batch x features if axis == 0
+```
+
+
+Examples
+=========
+
+Suppose that
+
+```
+  indices = [0, 2, -1, 1]
+  depth = 3
+  on_value = 5.0
+  off_value = 0.0
+  axis = -1
+```
+
+Then output is `[4 x 3]`:
+
+    ```output =
+      [5.0 0.0 0.0]  // one_hot(0)
+      [0.0 0.0 5.0]  // one_hot(2)
+      [0.0 0.0 0.0]  // one_hot(-1)
+      [0.0 5.0 0.0]  // one_hot(1)
+    ```
+
+Suppose that
+
+```
+  indices = [0, 2, -1, 1]
+  depth = 3
+  on_value = 0.0
+  off_value = 3.0
+  axis = 0
+```
+
+Then output is `[3 x 4]`:
+
+    ```output =
+      [0.0 3.0 3.0 3.0]
+      [3.0 3.0 3.0 0.0]
+      [3.0 3.0 3.0 3.0]
+      [3.0 0.0 3.0 3.0]
+    //  ^                one_hot(0)
+    //      ^            one_hot(2)
+    //          ^        one_hot(-1)
+    //              ^    one_hot(1)
+    ```
+Suppose that
+
+```
+  indices = [[0, 2], [1, -1]]
+  depth = 3
+  on_value = 1.0
+  off_value = 0.0
+  axis = -1
+```
+
+Then output is `[2 x 2 x 3]`:
+
+    ```output =
+      [
+        [1.0, 0.0, 0.0]  // one_hot(0)
+        [0.0, 0.0, 1.0]  // one_hot(2)
+      ][
+        [0.0, 1.0, 0.0]  // one_hot(1)
+        [0.0, 0.0, 0.0]  // one_hot(-1)
+      ]```
+END
+}
+op {
+  graph_op_name: "OneShotIterator"
+  endpoint {
+    name: "OneShotIterator"
+  }
+  summary: "Makes a \"one-shot\" iterator that can be iterated only once."
+  description: <<END
+A one-shot iterator bundles the logic for defining the dataset and
+the state of the iterator in a single op, which allows simple input
+pipelines to be defined without an additional initialization
+("MakeIterator") step.
+
+One-shot iterators have the following limitations:
+
+* They do not support parameterization: all logic for creating the underlying
+  dataset must be bundled in the `dataset_factory` function.
+* They are not resettable. Once a one-shot iterator reaches the end of its
+  underlying dataset, subsequent "IteratorGetNext" operations on that
+  iterator will always produce an `OutOfRange` error.
+
+For greater flexibility, use "Iterator" and "MakeIterator" to define
+an iterator using an arbitrary subgraph, which may capture tensors
+(including fed values) as parameters, and which may be reset multiple
+times by rerunning "MakeIterator".
+END
+}
+op {
+  graph_op_name: "OnesLike"
+  endpoint {
+    name: "OnesLike"
+  }
+  summary: "Returns a tensor of ones with the same shape and type as x."
+}
+op {
+  graph_op_name: "OrderedMapClear"
+  endpoint {
+    name: "OrderedMapClear"
+  }
+  summary: "Op removes all elements in the underlying container."
+}
+op {
+  graph_op_name: "OrderedMapIncompleteSize"
+  endpoint {
+    name: "OrderedMapIncompleteSize"
+  }
+  summary: "Op returns the number of incomplete elements in the underlying container."
+}
+op {
+  graph_op_name: "OrderedMapPeek"
+  endpoint {
+    name: "OrderedMapPeek"
+  }
+  summary: "Op peeks at the values at the specified key.  If the"
+  description: <<END
+underlying container does not contain this key
+this op will block until it does.   This Op is optimized for
+performance.
+END
+}
+op {
+  graph_op_name: "OrderedMapSize"
+  endpoint {
+    name: "OrderedMapSize"
+  }
+  summary: "Op returns the number of elements in the underlying container."
+}
+op {
+  graph_op_name: "OrderedMapStage"
+  endpoint {
+    name: "OrderedMapStage"
+  }
+  summary: "Stage (key, values) in the underlying container which behaves like a ordered"
+  description: <<END
+associative container.   Elements are ordered by key.
+END
+}
+op {
+  graph_op_name: "OrderedMapUnstage"
+  endpoint {
+    name: "OrderedMapUnstage"
+  }
+  summary: "Op removes and returns the values associated with the key"
+  description: <<END
+from the underlying container.   If the underlying container
+does not contain this key, the op will block until it does.
+END
+}
+op {
+  graph_op_name: "OrderedMapUnstageNoKey"
+  endpoint {
+    name: "OrderedMapUnstageNoKey"
+  }
+  summary: "Op removes and returns the (key, value) element with the smallest"
+  description: <<END
+key from the underlying container.   If the underlying container
+does not contain elements, the op will block until it does.
+END
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_P.pbtxt b/tensorflow/core/api_def/base_api/api_def_P.pbtxt
new file mode 100644
index 0000000000..a3abb079e9
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_P.pbtxt
@@ -0,0 +1,431 @@
+op {
+  graph_op_name: "Pack"
+  endpoint {
+    name: "Pack"
+  }
+  summary: "Packs a list of `N` rank-`R` tensors into one rank-`(R+1)` tensor."
+  description: <<END
+Packs the `N` tensors in `values` into a tensor with rank one higher than each
+tensor in `values`, by packing them along the `axis` dimension.
+Given a list of tensors of shape `(A, B, C)`;
+
+if `axis == 0` then the `output` tensor will have the shape `(N, A, B, C)`.
+if `axis == 1` then the `output` tensor will have the shape `(A, N, B, C)`.
+Etc.
+
+For example:
+
+```
+# 'x' is [1, 4]
+# 'y' is [2, 5]
+# 'z' is [3, 6]
+pack([x, y, z]) => [[1, 4], [2, 5], [3, 6]]  # Pack along first dim.
+pack([x, y, z], axis=1) => [[1, 2, 3], [4, 5, 6]]
+```
+
+This is the opposite of `unpack`.
+END
+}
+op {
+  graph_op_name: "Pad"
+  endpoint {
+    name: "Pad"
+  }
+  summary: "Pads a tensor with zeros."
+  description: <<END
+This operation pads a `input` with zeros according to the `paddings` you
+specify. `paddings` is an integer tensor with shape `[Dn, 2]`, where n is the
+rank of `input`. For each dimension D of `input`, `paddings[D, 0]` indicates
+how many zeros to add before the contents of `input` in that dimension, and
+`paddings[D, 1]` indicates how many zeros to add after the contents of `input`
+in that dimension.
+
+The padded size of each dimension D of the output is:
+
+`paddings(D, 0) + input.dim_size(D) + paddings(D, 1)`
+
+For example:
+
+```
+# 't' is [[1, 1], [2, 2]]
+# 'paddings' is [[1, 1], [2, 2]]
+# rank of 't' is 2
+pad(t, paddings) ==> [[0, 0, 0, 0, 0, 0]
+                      [0, 0, 1, 1, 0, 0]
+                      [0, 0, 2, 2, 0, 0]
+                      [0, 0, 0, 0, 0, 0]]
+```
+END
+}
+op {
+  graph_op_name: "PadV2"
+  endpoint {
+    name: "PadV2"
+  }
+  summary: "Pads a tensor."
+  description: <<END
+This operation pads `input` according to the `paddings` and `constant_values`
+you specify. `paddings` is an integer tensor with shape `[Dn, 2]`, where n is
+the rank of `input`. For each dimension D of `input`, `paddings[D, 0]` indicates
+how many padding values to add before the contents of `input` in that dimension,
+and `paddings[D, 1]` indicates how many padding values to add after the contents
+of `input` in that dimension. `constant_values` is a scalar tensor of the same
+type as `input` that indicates the value to use for padding `input`.
+
+The padded size of each dimension D of the output is:
+
+`paddings(D, 0) + input.dim_size(D) + paddings(D, 1)`
+
+For example:
+
+```
+# 't' is [[1, 1], [2, 2]]
+# 'paddings' is [[1, 1], [2, 2]]
+# 'constant_values' is 0
+# rank of 't' is 2
+pad(t, paddings) ==> [[0, 0, 0, 0, 0, 0]
+                      [0, 0, 1, 1, 0, 0]
+                      [0, 0, 2, 2, 0, 0]
+                      [0, 0, 0, 0, 0, 0]]
+```
+END
+}
+op {
+  graph_op_name: "PaddedBatchDataset"
+  endpoint {
+    name: "PaddedBatchDataset"
+  }
+  summary: "Creates a dataset that batches and pads `batch_size` elements from the input."
+}
+op {
+  graph_op_name: "PaddingFIFOQueue"
+  endpoint {
+    name: "PaddingFIFOQueue"
+  }
+  summary: "A queue that produces elements in first-in first-out order."
+  description: <<END
+Variable-size shapes are allowed by setting the corresponding shape dimensions
+to 0 in the shape attr.  In this case DequeueMany will pad up to the maximum
+size of any given element in the minibatch.  See below for details.
+END
+}
+op {
+  graph_op_name: "PaddingFIFOQueueV2"
+  endpoint {
+    name: "PaddingFIFOQueueV2"
+  }
+  summary: "A queue that produces elements in first-in first-out order."
+  description: <<END
+Variable-size shapes are allowed by setting the corresponding shape dimensions
+to 0 in the shape attr.  In this case DequeueMany will pad up to the maximum
+size of any given element in the minibatch.  See below for details.
+END
+}
+op {
+  graph_op_name: "ParallelConcat"
+  endpoint {
+    name: "ParallelConcat"
+  }
+  summary: "Concatenates a list of `N` tensors along the first dimension."
+  description: <<END
+The input tensors are all required to have size 1 in the first dimension.
+
+For example:
+
+```
+# 'x' is [[1, 4]]
+# 'y' is [[2, 5]]
+# 'z' is [[3, 6]]
+parallel_concat([x, y, z]) => [[1, 4], [2, 5], [3, 6]]  # Pack along first dim.
+```
+
+The difference between concat and parallel_concat is that concat requires all
+of the inputs be computed before the operation will begin but doesn't require
+that the input shapes be known during graph construction.  Parallel concat
+will copy pieces of the input into the output as they become available, in
+some situations this can provide a performance benefit.
+END
+}
+op {
+  graph_op_name: "ParallelDynamicStitch"
+  endpoint {
+    name: "ParallelDynamicStitch"
+  }
+  summary: "Interleave the values from the `data` tensors into a single tensor."
+  description: <<END
+Builds a merged tensor such that
+
+```python
+    merged[indices[m][i, ..., j], ...] = data[m][i, ..., j, ...]
+```
+
+For example, if each `indices[m]` is scalar or vector, we have
+
+```python
+    # Scalar indices:
+    merged[indices[m], ...] = data[m][...]
+
+    # Vector indices:
+    merged[indices[m][i], ...] = data[m][i, ...]
+```
+
+Each `data[i].shape` must start with the corresponding `indices[i].shape`,
+and the rest of `data[i].shape` must be constant w.r.t. `i`.  That is, we
+must have `data[i].shape = indices[i].shape + constant`.  In terms of this
+`constant`, the output shape is
+
+    merged.shape = [max(indices)] + constant
+
+Values may be merged in parallel, so if an index appears in both `indices[m][i]`
+and `indices[n][j]`, the result may be invalid. This differs from the normal
+DynamicStitch operator that defines the behavior in that case.
+
+For example:
+
+```python
+    indices[0] = 6
+    indices[1] = [4, 1]
+    indices[2] = [[5, 2], [0, 3]]
+    data[0] = [61, 62]
+    data[1] = [[41, 42], [11, 12]]
+    data[2] = [[[51, 52], [21, 22]], [[1, 2], [31, 32]]]
+    merged = [[1, 2], [11, 12], [21, 22], [31, 32], [41, 42],
+              [51, 52], [61, 62]]
+```
+
+This method can be used to merge partitions created by `dynamic_partition`
+as illustrated on the following example:
+
+```python
+    # Apply function (increments x_i) on elements for which a certain condition
+    # apply (x_i != -1 in this example).
+    x=tf.constant([0.1, -1., 5.2, 4.3, -1., 7.4])
+    condition_mask=tf.not_equal(x,tf.constant(-1.))
+    partitioned_data = tf.dynamic_partition(
+        x, tf.cast(condition_mask, tf.int32) , 2)
+    partitioned_data[1] = partitioned_data[1] + 1.0
+    condition_indices = tf.dynamic_partition(
+        tf.range(tf.shape(x)[0]), tf.cast(condition_mask, tf.int32) , 2)
+    x = tf.dynamic_stitch(condition_indices, partitioned_data)
+    # Here x=[1.1, -1., 6.2, 5.3, -1, 8.4], the -1. values remain
+    # unchanged.
+```
+
+<div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
+<img style="width:100%" src="https://www.tensorflow.org/images/DynamicStitch.png" alt>
+</div>
+END
+}
+op {
+  graph_op_name: "ParallelMapDataset"
+  endpoint {
+    name: "ParallelMapDataset"
+  }
+  summary: "Creates a dataset that applies `f` to the outputs of `input_dataset`."
+  description: <<END
+Unlike a "MapDataset", which applies `f` sequentially, this dataset invokes up
+to `num_parallel_calls` copies of `f` in parallel.
+END
+}
+op {
+  graph_op_name: "ParameterizedTruncatedNormal"
+  endpoint {
+    name: "ParameterizedTruncatedNormal"
+  }
+  summary: "Outputs random values from a normal distribution. The parameters may each be a"
+  description: <<END
+scalar which applies to the entire output, or a vector of length shape[0] which
+stores the parameters for each batch.
+END
+}
+op {
+  graph_op_name: "ParseExample"
+  endpoint {
+    name: "ParseExample"
+  }
+  summary: "Transforms a vector of brain.Example protos (as strings) into typed tensors."
+}
+op {
+  graph_op_name: "ParseSingleSequenceExample"
+  endpoint {
+    name: "ParseSingleSequenceExample"
+  }
+  summary: "Transforms a scalar brain.SequenceExample proto (as strings) into typed tensors."
+}
+op {
+  graph_op_name: "ParseTensor"
+  endpoint {
+    name: "ParseTensor"
+  }
+  summary: "Transforms a serialized tensorflow.TensorProto proto into a Tensor."
+}
+op {
+  graph_op_name: "Placeholder"
+  endpoint {
+    name: "Placeholder"
+  }
+  summary: "A placeholder op for a value that will be fed into the computation."
+  description: <<END
+N.B. This operation will fail with an error if it is executed. It is
+intended as a way to represent a value that will always be fed, and to
+provide attrs that enable the fed value to be checked at runtime.
+END
+}
+op {
+  graph_op_name: "PlaceholderV2"
+  endpoint {
+    name: "PlaceholderV2"
+  }
+  summary: "A placeholder op for a value that will be fed into the computation."
+  description: <<END
+N.B. This operation will fail with an error if it is executed. It is
+intended as a way to represent a value that will always be fed, and to
+provide attrs that enable the fed value to be checked at runtime.
+END
+}
+op {
+  graph_op_name: "PlaceholderWithDefault"
+  endpoint {
+    name: "PlaceholderWithDefault"
+  }
+  summary: "A placeholder op that passes through `input` when its output is not fed."
+}
+op {
+  graph_op_name: "Polygamma"
+  endpoint {
+    name: "Polygamma"
+  }
+  summary: "Compute the polygamma function \\\\(\\psi^{(n)}(x)\\\\)."
+  description: <<END
+The polygamma function is defined as:
+
+
+\\(\psi^{(n)}(x) = \frac{d^n}{dx^n} \psi(x)\\)
+
+where \\(\psi(x)\\) is the digamma function.
+END
+}
+op {
+  graph_op_name: "PopulationCount"
+  endpoint {
+    name: "PopulationCount"
+  }
+  summary: "Computes element-wise population count (a.k.a. popcount, bitsum, bitcount)."
+  description: <<END
+For each entry in `x`, calculates the number of `1` (on) bits in the binary
+representation of that entry.
+
+**NOTE**: It is more efficient to first `tf.bitcast` your tensors into
+`int32` or `int64` and perform the bitcount on the result, than to feed in
+8- or 16-bit inputs and then aggregate the resulting counts.
+END
+}
+op {
+  graph_op_name: "Pow"
+  endpoint {
+    name: "Pow"
+  }
+  summary: "Computes the power of one value to another."
+  description: <<END
+Given a tensor `x` and a tensor `y`, this operation computes \\(x^y\\) for
+corresponding elements in `x` and `y`. For example:
+
+```
+# tensor 'x' is [[2, 2]], [3, 3]]
+# tensor 'y' is [[8, 16], [2, 3]]
+tf.pow(x, y) ==> [[256, 65536], [9, 27]]
+```
+END
+}
+op {
+  graph_op_name: "PrefetchDataset"
+  endpoint {
+    name: "PrefetchDataset"
+  }
+  summary: "Creates a dataset that asynchronously prefetches elements from `input_dataset`."
+}
+op {
+  graph_op_name: "PreventGradient"
+  endpoint {
+    name: "PreventGradient"
+  }
+  summary: "An identity op that triggers an error if a gradient is requested."
+  description: <<END
+When executed in a graph, this op outputs its input tensor as-is.
+
+When building ops to compute gradients, the TensorFlow gradient system
+will return an error when trying to lookup the gradient of this op,
+because no gradient must ever be registered for this function.  This
+op exists to prevent subtle bugs from silently returning unimplemented
+gradients in some corner cases.
+END
+}
+op {
+  graph_op_name: "Print"
+  endpoint {
+    name: "Print"
+  }
+  summary: "Prints a list of tensors."
+  description: <<END
+Passes `input` through to `output` and prints `data` when evaluating.
+END
+}
+op {
+  graph_op_name: "PriorityQueue"
+  endpoint {
+    name: "PriorityQueue"
+  }
+  summary: "A queue that produces elements sorted by the first component value."
+  description: <<END
+Note that the PriorityQueue requires the first component of any element
+to be a scalar int64, in addition to the other elements declared by
+component_types.  Therefore calls to Enqueue and EnqueueMany (resp. Dequeue
+and DequeueMany) on a PriorityQueue will all require (resp. output) one extra
+entry in their input (resp. output) lists.
+END
+}
+op {
+  graph_op_name: "PriorityQueueV2"
+  endpoint {
+    name: "PriorityQueueV2"
+  }
+  summary: "A queue that produces elements sorted by the first component value."
+  description: <<END
+Note that the PriorityQueue requires the first component of any element
+to be a scalar int64, in addition to the other elements declared by
+component_types.  Therefore calls to Enqueue and EnqueueMany (resp. Dequeue
+and DequeueMany) on a PriorityQueue will all require (resp. output) one extra
+entry in their input (resp. output) lists.
+END
+}
+op {
+  graph_op_name: "Prod"
+  endpoint {
+    name: "Prod"
+  }
+  summary: "Computes the product of elements across dimensions of a tensor."
+  description: <<END
+Reduces `input` along the dimensions given in `reduction_indices`. Unless
+`keep_dims` is true, the rank of the tensor is reduced by 1 for each entry in
+`reduction_indices`. If `keep_dims` is true, the reduced dimensions are
+retained with length 1.
+END
+}
+op {
+  graph_op_name: "PyFunc"
+  endpoint {
+    name: "PyFunc"
+  }
+  summary: "Invokes a python function to compute func(input)->output."
+  description: <<END
+This operation is considered stateful. For a stateless version, see
+PyFuncStateless.
+END
+}
+op {
+  graph_op_name: "PyFuncStateless"
+  endpoint {
+    name: "PyFuncStateless"
+  }
+  summary: "A stateless version of PyFunc."
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_Q.pbtxt b/tensorflow/core/api_def/base_api/api_def_Q.pbtxt
new file mode 100644
index 0000000000..4af60a1841
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_Q.pbtxt
@@ -0,0 +1,609 @@
+op {
+  graph_op_name: "Qr"
+  endpoint {
+    name: "Qr"
+  }
+  summary: "Computes the QR decompositions of one or more matrices."
+  description: <<END
+Computes the QR decomposition of each inner matrix in `tensor` such that
+`tensor[..., :, :] = q[..., :, :] * r[..., :,:])`
+
+```python
+# a is a tensor.
+# q is a tensor of orthonormal matrices.
+# r is a tensor of upper triangular matrices.
+q, r = qr(a)
+q_full, r_full = qr(a, full_matrices=True)
+```
+END
+}
+op {
+  graph_op_name: "QuantizeAndDequantize"
+  endpoint {
+    name: "QuantizeAndDequantize"
+  }
+  summary: "Use QuantizeAndDequantizeV2 instead."
+}
+op {
+  graph_op_name: "QuantizeAndDequantizeV2"
+  endpoint {
+    name: "QuantizeAndDequantizeV2"
+  }
+  summary: "Quantizes then dequantizes a tensor."
+  description: <<END
+This op simulates the precision loss from the quantized forward pass by:
+1. Quantizing the tensor to fixed point numbers, which should match the target
+   quantization method when it is used in inference.
+2. Dequantizing it back to floating point numbers for the following ops, most
+   likely matmul.
+
+There are different ways to quantize. This version does not use the full range
+of the output type, choosing to elide the lowest possible value for symmetry
+(e.g., output range is -127 to 127, not -128 to 127 for signed 8 bit
+quantization), so that 0.0 maps to 0.
+
+To perform this op, we first find the range of values in our tensor. The range
+we use is always centered on 0, so we find m such that
+
+1. m = max(abs(input_min), abs(input_max)) if range_given is true,
+2. m = max(abs(min_elem(input)), abs(max_elem(input))) otherwise.
+
+Our input tensor range is then [-m, m].
+
+Next, we choose our fixed-point quantization buckets, [min_fixed, max_fixed].
+If signed_input is true, this is
+
+  [min_fixed, max_fixed ] =
+      [-(1 << (num_bits - 1) - 1), (1 << (num_bits - 1)) - 1].
+
+Otherwise, if signed_input is false, the fixed-point range is
+
+  [min_fixed, max_fixed] = [0, (1 << num_bits) - 1].
+
+From this we compute our scaling factor, s:
+
+  s = (max_fixed - min_fixed) / (2 * m).
+
+Now we can quantize and dequantize the elements of our tensor.  An element e
+is transformed into e':
+
+  e' = (e * s).round_to_nearest() / s.
+
+Note that we have a different number of buckets in the signed vs. unsigned
+cases.  For example, if num_bits == 8, we get 254 buckets in the signed case
+vs. 255 in the unsigned case.
+
+For example, suppose num_bits = 8 and m = 1.  Then
+
+  [min_fixed, max_fixed] = [-127, 127], and
+  s = (127 + 127) / 2 = 127.
+
+Given the vector {-1, -0.5, 0, 0.3}, this is quantized to
+{-127, -63, 0, 38}, and dequantized to {-1, -63.0/127, 0, 38.0/127}.
+END
+}
+op {
+  graph_op_name: "QuantizeAndDequantizeV3"
+  endpoint {
+    name: "QuantizeAndDequantizeV3"
+  }
+  summary: "Quantizes then dequantizes a tensor."
+  description: <<END
+This is almost identical to QuantizeAndDequantizeV2, except that num_bits is a
+tensor, so its value can change during training.
+END
+}
+op {
+  graph_op_name: "QuantizeDownAndShrinkRange"
+  endpoint {
+    name: "QuantizeDownAndShrinkRange"
+  }
+  summary: "Convert the quantized \'input\' tensor into a lower-precision \'output\', using the"
+  description: <<END
+actual distribution of the values to maximize the usage of the lower bit depth
+and adjusting the output min and max ranges accordingly.
+
+[input_min, input_max] are scalar floats that specify the range for the float
+interpretation of the 'input' data. For example, if input_min is -1.0f and
+input_max is 1.0f, and we are dealing with quint16 quantized data, then a 0
+value in the 16-bit data should be interpreted as -1.0f, and a 65535 means 1.0f.
+
+This operator tries to squeeze as much precision as possible into an output with
+a lower bit depth by calculating the actual min and max values found in the
+data. For example, maybe that quint16 input has no values lower than 16,384 and
+none higher than 49,152. That means only half the range is actually needed, all
+the float interpretations are between -0.5f and 0.5f, so if we want to compress
+the data into a quint8 output, we can use that range rather than the theoretical
+-1.0f to 1.0f that is suggested by the input min and max.
+
+In practice, this is most useful for taking output from operations like
+QuantizedMatMul that can produce higher bit-depth outputs than their inputs and
+may have large potential output ranges, but in practice have a distribution of
+input values that only uses a small fraction of the possible range. By feeding
+that output into this operator, we can reduce it from 32 bits down to 8 with
+minimal loss of accuracy.
+END
+}
+op {
+  graph_op_name: "QuantizeV2"
+  endpoint {
+    name: "QuantizeV2"
+  }
+  summary: "Quantize the \'input\' tensor of type float to \'output\' tensor of type \'T\'."
+  description: <<END
+[min_range, max_range] are scalar floats that specify the range for
+the 'input' data. The 'mode' attribute controls exactly which calculations are
+used to convert the float values to their quantized equivalents.
+
+In 'MIN_COMBINED' mode, each value of the tensor will undergo the following:
+
+```
+out[i] = (in[i] - min_range) * range(T) / (max_range - min_range)
+if T == qint8, out[i] -= (range(T) + 1) / 2.0
+```
+here `range(T) = numeric_limits<T>::max() - numeric_limits<T>::min()`
+
+*MIN_COMBINED Mode Example*
+
+Assume the input is type float and has a possible range of [0.0, 6.0] and the
+output type is quint8 ([0, 255]). The min_range and max_range values should be
+specified as 0.0 and 6.0. Quantizing from float to quint8 will multiply each
+value of the input by 255/6 and cast to quint8.
+
+If the output type was qint8 ([-128, 127]), the operation will additionally
+subtract each value by 128 prior to casting, so that the range of values aligns
+with the range of qint8.
+
+If the mode is 'MIN_FIRST', then this approach is used:
+
+```
+number_of_steps = 1 << (# of bits in T)
+range_adjust = number_of_steps / (number_of_steps - 1)
+range = (range_max - range_min) * range_adjust
+range_scale = number_of_steps / range
+quantized = round(input * range_scale) - round(range_min * range_scale) +
+  numeric_limits<T>::min()
+quantized = max(quantized, numeric_limits<T>::min())
+quantized = min(quantized, numeric_limits<T>::max())
+```
+
+The biggest difference between this and MIN_COMBINED is that the minimum range
+is rounded first, before it's subtracted from the rounded value. With
+MIN_COMBINED, a small bias is introduced where repeated iterations of quantizing
+and dequantizing will introduce a larger and larger error.
+
+*SCALED mode Example*
+
+`SCALED` mode matches the quantization approach used in
+`QuantizeAndDequantize{V2|V3}`.
+
+If the mode is `SCALED`, we do not use the full range of the output type,
+choosing to elide the lowest possible value for symmetry (e.g., output range is
+-127 to 127, not -128 to 127 for signed 8 bit quantization), so that 0.0 maps to
+0.
+
+We first find the range of values in our tensor. The
+range we use is always centered on 0, so we find m such that
+```c++
+  m = max(abs(input_min), abs(input_max))
+```
+
+Our input tensor range is then `[-m, m]`.
+
+Next, we choose our fixed-point quantization buckets, `[min_fixed, max_fixed]`.
+If T is signed, this is
+```
+  num_bits = sizeof(T) * 8
+  [min_fixed, max_fixed] =
+      [-(1 << (num_bits - 1) - 1), (1 << (num_bits - 1)) - 1]
+```
+
+Otherwise, if T is unsigned, the fixed-point range is
+```
+  [min_fixed, max_fixed] = [0, (1 << num_bits) - 1]
+```
+
+From this we compute our scaling factor, s:
+```c++
+  s = (max_fixed - min_fixed) / (2 * m)
+```
+
+Now we can quantize the elements of our tensor:
+```c++
+result = (input * s).round_to_nearest()
+```
+
+One thing to watch out for is that the operator may choose to adjust the
+requested minimum and maximum values slightly during the quantization process,
+so you should always use the output ports as the range for further calculations.
+For example, if the requested minimum and maximum values are close to equal,
+they will be separated by a small epsilon value to prevent ill-formed quantized
+buffers from being created. Otherwise, you can end up with buffers where all the
+quantized values map to the same float value, which causes problems for
+operations that have to perform further calculations on them.
+END
+}
+op {
+  graph_op_name: "QuantizedAdd"
+  endpoint {
+    name: "QuantizedAdd"
+  }
+  summary: "Returns x + y element-wise, working on quantized buffers."
+}
+op {
+  graph_op_name: "QuantizedAvgPool"
+  endpoint {
+    name: "QuantizedAvgPool"
+  }
+  summary: "Produces the average pool of the input tensor for quantized types."
+}
+op {
+  graph_op_name: "QuantizedBatchNormWithGlobalNormalization"
+  endpoint {
+    name: "QuantizedBatchNormWithGlobalNormalization"
+  }
+  summary: "Quantized Batch normalization."
+  description: <<END
+This op is deprecated and will be removed in the future. Prefer
+`tf.nn.batch_normalization`.
+END
+}
+op {
+  graph_op_name: "QuantizedBiasAdd"
+  endpoint {
+    name: "QuantizedBiasAdd"
+  }
+  summary: "Adds Tensor \'bias\' to Tensor \'input\' for Quantized types."
+  description: <<END
+Broadcasts the values of bias on dimensions 0..N-2 of 'input'.
+END
+}
+op {
+  graph_op_name: "QuantizedConcat"
+  endpoint {
+    name: "QuantizedConcat"
+  }
+  summary: "Concatenates quantized tensors along one dimension."
+}
+op {
+  graph_op_name: "QuantizedConv2D"
+  endpoint {
+    name: "QuantizedConv2D"
+  }
+  summary: "Computes a 2D convolution given quantized 4D input and filter tensors."
+  description: <<END
+The inputs are quantized tensors where the lowest value represents the real
+number of the associated minimum, and the highest represents the maximum.
+This means that you can only interpret the quantized output in the same way, by
+taking the returned minimum and maximum values into account.
+END
+}
+op {
+  graph_op_name: "QuantizedInstanceNorm"
+  endpoint {
+    name: "QuantizedInstanceNorm"
+  }
+  summary: "Quantized Instance normalization."
+}
+op {
+  graph_op_name: "QuantizedMatMul"
+  endpoint {
+    name: "QuantizedMatMul"
+  }
+  summary: "Perform a quantized matrix multiplication of  `a` by the matrix `b`."
+  description: <<END
+The inputs must be two-dimensional matrices and the inner dimension of
+`a` (after being transposed if `transpose_a` is non-zero) must match the
+outer dimension of `b` (after being transposed if `transposed_b` is
+non-zero).
+END
+}
+op {
+  graph_op_name: "QuantizedMaxPool"
+  endpoint {
+    name: "QuantizedMaxPool"
+  }
+  summary: "Produces the max pool of the input tensor for quantized types."
+}
+op {
+  graph_op_name: "QuantizedMul"
+  endpoint {
+    name: "QuantizedMul"
+  }
+  summary: "Returns x * y element-wise, working on quantized buffers."
+}
+op {
+  graph_op_name: "QuantizedRelu"
+  endpoint {
+    name: "QuantizedRelu"
+  }
+  summary: "Computes Quantized Rectified Linear: `max(features, 0)`"
+}
+op {
+  graph_op_name: "QuantizedRelu6"
+  endpoint {
+    name: "QuantizedRelu6"
+  }
+  summary: "Computes Quantized Rectified Linear 6: `min(max(features, 0), 6)`"
+}
+op {
+  graph_op_name: "QuantizedReluX"
+  endpoint {
+    name: "QuantizedReluX"
+  }
+  summary: "Computes Quantized Rectified Linear X: `min(max(features, 0), max_value)`"
+}
+op {
+  graph_op_name: "QuantizedReshape"
+  endpoint {
+    name: "QuantizedReshape"
+  }
+  summary: "Reshapes a quantized tensor as per the Reshape op."
+  description: <<END
+```
+END
+}
+op {
+  graph_op_name: "QuantizedResizeBilinear"
+  endpoint {
+    name: "QuantizedResizeBilinear"
+  }
+  summary: "Resize quantized `images` to `size` using quantized bilinear interpolation."
+  description: <<END
+Input images and output images must be quantized types.
+END
+}
+op {
+  graph_op_name: "QueueClose"
+  endpoint {
+    name: "QueueClose"
+  }
+  summary: "Closes the given queue."
+  description: <<END
+This operation signals that no more elements will be enqueued in the
+given queue. Subsequent Enqueue(Many) operations will fail.
+Subsequent Dequeue(Many) operations will continue to succeed if
+sufficient elements remain in the queue. Subsequent Dequeue(Many)
+operations that would block will fail immediately.
+END
+}
+op {
+  graph_op_name: "QueueCloseV2"
+  endpoint {
+    name: "QueueCloseV2"
+  }
+  summary: "Closes the given queue."
+  description: <<END
+This operation signals that no more elements will be enqueued in the
+given queue. Subsequent Enqueue(Many) operations will fail.
+Subsequent Dequeue(Many) operations will continue to succeed if
+sufficient elements remain in the queue. Subsequent Dequeue(Many)
+operations that would block will fail immediately.
+END
+}
+op {
+  graph_op_name: "QueueDequeue"
+  endpoint {
+    name: "QueueDequeue"
+  }
+  summary: "Dequeues a tuple of one or more tensors from the given queue."
+  description: <<END
+This operation has k outputs, where k is the number of components
+in the tuples stored in the given queue, and output i is the ith
+component of the dequeued tuple.
+
+N.B. If the queue is empty, this operation will block until an element
+has been dequeued (or 'timeout_ms' elapses, if specified).
+END
+}
+op {
+  graph_op_name: "QueueDequeueMany"
+  endpoint {
+    name: "QueueDequeueMany"
+  }
+  summary: "Dequeues `n` tuples of one or more tensors from the given queue."
+  description: <<END
+If the queue is closed and there are fewer than `n` elements, then an
+OutOfRange error is returned.
+
+This operation concatenates queue-element component tensors along the
+0th dimension to make a single component tensor.  All of the components
+in the dequeued tuple will have size `n` in the 0th dimension.
+
+This operation has `k` outputs, where `k` is the number of components in
+the tuples stored in the given queue, and output `i` is the ith
+component of the dequeued tuple.
+
+N.B. If the queue is empty, this operation will block until `n` elements
+have been dequeued (or 'timeout_ms' elapses, if specified).
+END
+}
+op {
+  graph_op_name: "QueueDequeueManyV2"
+  endpoint {
+    name: "QueueDequeueManyV2"
+  }
+  summary: "Dequeues `n` tuples of one or more tensors from the given queue."
+  description: <<END
+If the queue is closed and there are fewer than `n` elements, then an
+OutOfRange error is returned.
+
+This operation concatenates queue-element component tensors along the
+0th dimension to make a single component tensor.  All of the components
+in the dequeued tuple will have size `n` in the 0th dimension.
+
+This operation has `k` outputs, where `k` is the number of components in
+the tuples stored in the given queue, and output `i` is the ith
+component of the dequeued tuple.
+
+N.B. If the queue is empty, this operation will block until `n` elements
+have been dequeued (or 'timeout_ms' elapses, if specified).
+END
+}
+op {
+  graph_op_name: "QueueDequeueUpTo"
+  endpoint {
+    name: "QueueDequeueUpTo"
+  }
+  summary: "Dequeues `n` tuples of one or more tensors from the given queue."
+  description: <<END
+This operation is not supported by all queues.  If a queue does not support
+DequeueUpTo, then an Unimplemented error is returned.
+
+If the queue is closed and there are more than 0 but less than `n`
+elements remaining, then instead of returning an OutOfRange error like
+QueueDequeueMany, less than `n` elements are returned immediately.  If
+the queue is closed and there are 0 elements left in the queue, then
+an OutOfRange error is returned just like in QueueDequeueMany.
+Otherwise the behavior is identical to QueueDequeueMany:
+
+This operation concatenates queue-element component tensors along the
+0th dimension to make a single component tensor.  All of the components
+in the dequeued tuple will have size `n` in the 0th dimension.
+
+This operation has k outputs, where `k` is the number of components in
+the tuples stored in the given queue, and output `i` is the ith
+component of the dequeued tuple.
+END
+}
+op {
+  graph_op_name: "QueueDequeueUpToV2"
+  endpoint {
+    name: "QueueDequeueUpToV2"
+  }
+  summary: "Dequeues `n` tuples of one or more tensors from the given queue."
+  description: <<END
+This operation is not supported by all queues.  If a queue does not support
+DequeueUpTo, then an Unimplemented error is returned.
+
+If the queue is closed and there are more than 0 but less than `n`
+elements remaining, then instead of returning an OutOfRange error like
+QueueDequeueMany, less than `n` elements are returned immediately.  If
+the queue is closed and there are 0 elements left in the queue, then
+an OutOfRange error is returned just like in QueueDequeueMany.
+Otherwise the behavior is identical to QueueDequeueMany:
+
+This operation concatenates queue-element component tensors along the
+0th dimension to make a single component tensor.  All of the components
+in the dequeued tuple will have size n in the 0th dimension.
+
+This operation has `k` outputs, where `k` is the number of components in
+the tuples stored in the given queue, and output `i` is the ith
+component of the dequeued tuple.
+END
+}
+op {
+  graph_op_name: "QueueDequeueV2"
+  endpoint {
+    name: "QueueDequeueV2"
+  }
+  summary: "Dequeues a tuple of one or more tensors from the given queue."
+  description: <<END
+This operation has k outputs, where k is the number of components
+in the tuples stored in the given queue, and output i is the ith
+component of the dequeued tuple.
+
+N.B. If the queue is empty, this operation will block until an element
+has been dequeued (or 'timeout_ms' elapses, if specified).
+END
+}
+op {
+  graph_op_name: "QueueEnqueue"
+  endpoint {
+    name: "QueueEnqueue"
+  }
+  summary: "Enqueues a tuple of one or more tensors in the given queue."
+  description: <<END
+The components input has k elements, which correspond to the components of
+tuples stored in the given queue.
+
+N.B. If the queue is full, this operation will block until the given
+element has been enqueued (or 'timeout_ms' elapses, if specified).
+END
+}
+op {
+  graph_op_name: "QueueEnqueueMany"
+  endpoint {
+    name: "QueueEnqueueMany"
+  }
+  summary: "Enqueues zero or more tuples of one or more tensors in the given queue."
+  description: <<END
+This operation slices each component tensor along the 0th dimension to
+make multiple queue elements. All of the tuple components must have the
+same size in the 0th dimension.
+
+The components input has k elements, which correspond to the components of
+tuples stored in the given queue.
+
+N.B. If the queue is full, this operation will block until the given
+elements have been enqueued (or 'timeout_ms' elapses, if specified).
+END
+}
+op {
+  graph_op_name: "QueueEnqueueManyV2"
+  endpoint {
+    name: "QueueEnqueueManyV2"
+  }
+  summary: "Enqueues zero or more tuples of one or more tensors in the given queue."
+  description: <<END
+This operation slices each component tensor along the 0th dimension to
+make multiple queue elements. All of the tuple components must have the
+same size in the 0th dimension.
+
+The components input has k elements, which correspond to the components of
+tuples stored in the given queue.
+
+N.B. If the queue is full, this operation will block until the given
+elements have been enqueued (or 'timeout_ms' elapses, if specified).
+END
+}
+op {
+  graph_op_name: "QueueEnqueueV2"
+  endpoint {
+    name: "QueueEnqueueV2"
+  }
+  summary: "Enqueues a tuple of one or more tensors in the given queue."
+  description: <<END
+The components input has k elements, which correspond to the components of
+tuples stored in the given queue.
+
+N.B. If the queue is full, this operation will block until the given
+element has been enqueued (or 'timeout_ms' elapses, if specified).
+END
+}
+op {
+  graph_op_name: "QueueIsClosed"
+  endpoint {
+    name: "QueueIsClosed"
+  }
+  summary: "Returns true if queue is closed."
+  description: <<END
+This operation returns true if the queue is closed and false if the queue
+is open.
+END
+}
+op {
+  graph_op_name: "QueueIsClosedV2"
+  endpoint {
+    name: "QueueIsClosedV2"
+  }
+  summary: "Returns true if queue is closed."
+  description: <<END
+This operation returns true if the queue is closed and false if the queue
+is open.
+END
+}
+op {
+  graph_op_name: "QueueSize"
+  endpoint {
+    name: "QueueSize"
+  }
+  summary: "Computes the number of elements in the given queue."
+}
+op {
+  graph_op_name: "QueueSizeV2"
+  endpoint {
+    name: "QueueSizeV2"
+  }
+  summary: "Computes the number of elements in the given queue."
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_R.pbtxt b/tensorflow/core/api_def/base_api/api_def_R.pbtxt
new file mode 100644
index 0000000000..4c398c9771
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_R.pbtxt
@@ -0,0 +1,1392 @@
+op {
+  graph_op_name: "RFFT"
+  endpoint {
+    name: "RFFT"
+  }
+  summary: "Real-valued fast Fourier transform."
+  description: <<END
+Computes the 1-dimensional discrete Fourier transform of a real-valued signal
+over the inner-most dimension of `input`.
+
+Since the DFT of a real signal is Hermitian-symmetric, `RFFT` only returns the
+`fft_length / 2 + 1` unique components of the FFT: the zero-frequency term,
+followed by the `fft_length / 2` positive-frequency terms.
+
+Along the axis `RFFT` is computed on, if `fft_length` is smaller than the
+corresponding dimension of `input`, the dimension is cropped. If it is larger,
+the dimension is padded with zeros.
+END
+}
+op {
+  graph_op_name: "RFFT2D"
+  endpoint {
+    name: "RFFT2D"
+  }
+  summary: "2D real-valued fast Fourier transform."
+  description: <<END
+Computes the 2-dimensional discrete Fourier transform of a real-valued signal
+over the inner-most 2 dimensions of `input`.
+
+Since the DFT of a real signal is Hermitian-symmetric, `RFFT2D` only returns the
+`fft_length / 2 + 1` unique components of the FFT for the inner-most dimension
+of `output`: the zero-frequency term, followed by the `fft_length / 2`
+positive-frequency terms.
+
+Along each axis `RFFT2D` is computed on, if `fft_length` is smaller than the
+corresponding dimension of `input`, the dimension is cropped. If it is larger,
+the dimension is padded with zeros.
+END
+}
+op {
+  graph_op_name: "RFFT3D"
+  endpoint {
+    name: "RFFT3D"
+  }
+  summary: "3D real-valued fast Fourier transform."
+  description: <<END
+Computes the 3-dimensional discrete Fourier transform of a real-valued signal
+over the inner-most 3 dimensions of `input`.
+
+Since the DFT of a real signal is Hermitian-symmetric, `RFFT3D` only returns the
+`fft_length / 2 + 1` unique components of the FFT for the inner-most dimension
+of `output`: the zero-frequency term, followed by the `fft_length / 2`
+positive-frequency terms.
+
+Along each axis `RFFT3D` is computed on, if `fft_length` is smaller than the
+corresponding dimension of `input`, the dimension is cropped. If it is larger,
+the dimension is padded with zeros.
+END
+}
+op {
+  graph_op_name: "RGBToHSV"
+  endpoint {
+    name: "RGBToHSV"
+  }
+  summary: "Converts one or more images from RGB to HSV."
+  description: <<END
+Outputs a tensor of the same shape as the `images` tensor, containing the HSV
+value of the pixels. The output is only well defined if the value in `images`
+are in `[0,1]`.
+
+`output[..., 0]` contains hue, `output[..., 1]` contains saturation, and
+`output[..., 2]` contains value. All HSV values are in `[0,1]`. A hue of 0
+corresponds to pure red, hue 1/3 is pure green, and 2/3 is pure blue.
+END
+}
+op {
+  graph_op_name: "RandomCrop"
+  endpoint {
+    name: "RandomCrop"
+  }
+  summary: "Randomly crop `image`."
+  description: <<END
+`size` is a 1-D int64 tensor with 2 elements representing the crop height and
+width.  The values must be non negative.
+
+This Op picks a random location in `image` and crops a `height` by `width`
+rectangle from that location.  The random location is picked so the cropped
+area will fit inside the original image.
+END
+}
+op {
+  graph_op_name: "RandomGamma"
+  endpoint {
+    name: "RandomGamma"
+  }
+  summary: "Outputs random values from the Gamma distribution(s) described by alpha."
+  description: <<END
+This op uses the algorithm by Marsaglia et al. to acquire samples via
+transformation-rejection from pairs of uniform and normal random variables.
+See http://dl.acm.org/citation.cfm?id=358414
+END
+}
+op {
+  graph_op_name: "RandomPoisson"
+  endpoint {
+    name: "RandomPoisson"
+  }
+  summary: "Outputs random values from the Poisson distribution(s) described by rate."
+  description: <<END
+This op uses two algorithms, depending on rate. If rate >= 10, then
+the algorithm by Hormann is used to acquire samples via
+transformation-rejection.
+See http://www.sciencedirect.com/science/article/pii/0167668793909974.
+
+Otherwise, Knuth's algorithm is used to acquire samples via multiplying uniform
+random variables.
+See Donald E. Knuth (1969). Seminumerical Algorithms. The Art of Computer
+Programming, Volume 2. Addison Wesley
+END
+}
+op {
+  graph_op_name: "RandomPoissonV2"
+  endpoint {
+    name: "RandomPoissonV2"
+  }
+  summary: "Outputs random values from the Poisson distribution(s) described by rate."
+  description: <<END
+This op uses two algorithms, depending on rate. If rate >= 10, then
+the algorithm by Hormann is used to acquire samples via
+transformation-rejection.
+See http://www.sciencedirect.com/science/article/pii/0167668793909974.
+
+Otherwise, Knuth's algorithm is used to acquire samples via multiplying uniform
+random variables.
+See Donald E. Knuth (1969). Seminumerical Algorithms. The Art of Computer
+Programming, Volume 2. Addison Wesley
+END
+}
+op {
+  graph_op_name: "RandomShuffle"
+  endpoint {
+    name: "RandomShuffle"
+  }
+  summary: "Randomly shuffles a tensor along its first dimension."
+  description: <<END
+  The tensor is shuffled along dimension 0, such that each `value[j]` is mapped
+  to one and only one `output[i]`. For example, a mapping that might occur for a
+  3x2 tensor is:
+
+```
+[[1, 2],       [[5, 6],
+ [3, 4],  ==>   [1, 2],
+ [5, 6]]        [3, 4]]
+```
+END
+}
+op {
+  graph_op_name: "RandomShuffleQueue"
+  endpoint {
+    name: "RandomShuffleQueue"
+  }
+  summary: "A queue that randomizes the order of elements."
+}
+op {
+  graph_op_name: "RandomShuffleQueueV2"
+  endpoint {
+    name: "RandomShuffleQueueV2"
+  }
+  summary: "A queue that randomizes the order of elements."
+}
+op {
+  graph_op_name: "RandomStandardNormal"
+  endpoint {
+    name: "RandomStandardNormal"
+  }
+  summary: "Outputs random values from a normal distribution."
+  description: <<END
+The generated values will have mean 0 and standard deviation 1.
+END
+}
+op {
+  graph_op_name: "RandomUniform"
+  endpoint {
+    name: "RandomUniform"
+  }
+  summary: "Outputs random values from a uniform distribution."
+  description: <<END
+The generated values follow a uniform distribution in the range `[0, 1)`. The
+lower bound 0 is included in the range, while the upper bound 1 is excluded.
+END
+}
+op {
+  graph_op_name: "RandomUniformInt"
+  endpoint {
+    name: "RandomUniformInt"
+  }
+  summary: "Outputs random integers from a uniform distribution."
+  description: <<END
+The generated values are uniform integers in the range `[minval, maxval)`.
+The lower bound `minval` is included in the range, while the upper bound
+`maxval` is excluded.
+
+The random integers are slightly biased unless `maxval - minval` is an exact
+power of two.  The bias is small for values of `maxval - minval` significantly
+smaller than the range of the output (either `2^32` or `2^64`).
+END
+}
+op {
+  graph_op_name: "Range"
+  endpoint {
+    name: "Range"
+  }
+  summary: "Creates a sequence of numbers."
+  description: <<END
+This operation creates a sequence of numbers that begins at `start` and
+extends by increments of `delta` up to but not including `limit`.
+
+For example:
+
+```
+# 'start' is 3
+# 'limit' is 18
+# 'delta' is 3
+tf.range(start, limit, delta) ==> [3, 6, 9, 12, 15]
+```
+END
+}
+op {
+  graph_op_name: "RangeDataset"
+  endpoint {
+    name: "RangeDataset"
+  }
+  summary: "Creates a dataset with a range of values. Corresponds to python\'s xrange."
+}
+op {
+  graph_op_name: "Rank"
+  endpoint {
+    name: "Rank"
+  }
+  summary: "Returns the rank of a tensor."
+  description: <<END
+This operation returns an integer representing the rank of `input`.
+
+For example:
+
+```
+# 't' is [[[1, 1, 1], [2, 2, 2]], [[3, 3, 3], [4, 4, 4]]]
+# shape of tensor 't' is [2, 2, 3]
+rank(t) ==> 3
+```
+
+**Note**: The rank of a tensor is not the same as the rank of a matrix. The rank
+of a tensor is the number of indices required to uniquely select each element
+of the tensor. Rank is also known as "order", "degree", or "ndims."
+END
+}
+op {
+  graph_op_name: "ReadFile"
+  endpoint {
+    name: "ReadFile"
+  }
+  summary: "Reads and outputs the entire contents of the input filename."
+}
+op {
+  graph_op_name: "ReaderNumRecordsProduced"
+  endpoint {
+    name: "ReaderNumRecordsProduced"
+  }
+  summary: "Returns the number of records this Reader has produced."
+  description: <<END
+This is the same as the number of ReaderRead executions that have
+succeeded.
+END
+}
+op {
+  graph_op_name: "ReaderNumRecordsProducedV2"
+  endpoint {
+    name: "ReaderNumRecordsProducedV2"
+  }
+  summary: "Returns the number of records this Reader has produced."
+  description: <<END
+This is the same as the number of ReaderRead executions that have
+succeeded.
+END
+}
+op {
+  graph_op_name: "ReaderNumWorkUnitsCompleted"
+  endpoint {
+    name: "ReaderNumWorkUnitsCompleted"
+  }
+  summary: "Returns the number of work units this Reader has finished processing."
+}
+op {
+  graph_op_name: "ReaderNumWorkUnitsCompletedV2"
+  endpoint {
+    name: "ReaderNumWorkUnitsCompletedV2"
+  }
+  summary: "Returns the number of work units this Reader has finished processing."
+}
+op {
+  graph_op_name: "ReaderRead"
+  endpoint {
+    name: "ReaderRead"
+  }
+  summary: "Returns the next record (key, value pair) produced by a Reader."
+  description: <<END
+Will dequeue from the input queue if necessary (e.g. when the
+Reader needs to start reading from a new file since it has finished
+with the previous file).
+END
+}
+op {
+  graph_op_name: "ReaderReadUpTo"
+  endpoint {
+    name: "ReaderReadUpTo"
+  }
+  summary: "Returns up to `num_records` (key, value) pairs produced by a Reader."
+  description: <<END
+Will dequeue from the input queue if necessary (e.g. when the
+Reader needs to start reading from a new file since it has finished
+with the previous file).
+It may return less than `num_records` even before the last batch.
+END
+}
+op {
+  graph_op_name: "ReaderReadUpToV2"
+  endpoint {
+    name: "ReaderReadUpToV2"
+  }
+  summary: "Returns up to `num_records` (key, value) pairs produced by a Reader."
+  description: <<END
+Will dequeue from the input queue if necessary (e.g. when the
+Reader needs to start reading from a new file since it has finished
+with the previous file).
+It may return less than `num_records` even before the last batch.
+END
+}
+op {
+  graph_op_name: "ReaderReadV2"
+  endpoint {
+    name: "ReaderReadV2"
+  }
+  summary: "Returns the next record (key, value pair) produced by a Reader."
+  description: <<END
+Will dequeue from the input queue if necessary (e.g. when the
+Reader needs to start reading from a new file since it has finished
+with the previous file).
+END
+}
+op {
+  graph_op_name: "ReaderReset"
+  endpoint {
+    name: "ReaderReset"
+  }
+  summary: "Restore a Reader to its initial clean state."
+}
+op {
+  graph_op_name: "ReaderResetV2"
+  endpoint {
+    name: "ReaderResetV2"
+  }
+  summary: "Restore a Reader to its initial clean state."
+}
+op {
+  graph_op_name: "ReaderRestoreState"
+  endpoint {
+    name: "ReaderRestoreState"
+  }
+  summary: "Restore a reader to a previously saved state."
+  description: <<END
+Not all Readers support being restored, so this can produce an
+Unimplemented error.
+END
+}
+op {
+  graph_op_name: "ReaderRestoreStateV2"
+  endpoint {
+    name: "ReaderRestoreStateV2"
+  }
+  summary: "Restore a reader to a previously saved state."
+  description: <<END
+Not all Readers support being restored, so this can produce an
+Unimplemented error.
+END
+}
+op {
+  graph_op_name: "ReaderSerializeState"
+  endpoint {
+    name: "ReaderSerializeState"
+  }
+  summary: "Produce a string tensor that encodes the state of a Reader."
+  description: <<END
+Not all Readers support being serialized, so this can produce an
+Unimplemented error.
+END
+}
+op {
+  graph_op_name: "ReaderSerializeStateV2"
+  endpoint {
+    name: "ReaderSerializeStateV2"
+  }
+  summary: "Produce a string tensor that encodes the state of a Reader."
+  description: <<END
+Not all Readers support being serialized, so this can produce an
+Unimplemented error.
+END
+}
+op {
+  graph_op_name: "Real"
+  endpoint {
+    name: "Real"
+  }
+  summary: "Returns the real part of a complex number."
+  description: <<END
+Given a tensor `input` of complex numbers, this operation returns a tensor of
+type `float` that is the real part of each element in `input`. All elements in
+`input` must be complex numbers of the form \\(a + bj\\), where *a* is the real
+ part returned by this operation and *b* is the imaginary part.
+
+For example:
+
+```
+# tensor 'input' is [-2.25 + 4.75j, 3.25 + 5.75j]
+tf.real(input) ==> [-2.25, 3.25]
+```
+END
+}
+op {
+  graph_op_name: "RealDiv"
+  endpoint {
+    name: "RealDiv"
+  }
+  summary: "Returns x / y element-wise for real types."
+  description: <<END
+If `x` and `y` are reals, this will return the floating-point division.
+
+*NOTE*: `Div` supports broadcasting. More about broadcasting
+[here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
+END
+}
+op {
+  graph_op_name: "Reciprocal"
+  endpoint {
+    name: "Reciprocal"
+  }
+  summary: "Computes the reciprocal of x element-wise."
+  description: <<END
+I.e., \\(y = 1 / x\\).
+END
+}
+op {
+  graph_op_name: "ReciprocalGrad"
+  endpoint {
+    name: "ReciprocalGrad"
+  }
+  summary: "Computes the gradient for the inverse of `x` wrt its input."
+  description: <<END
+Specifically, `grad = -dy * y*y`, where `y = 1/x`, and `dy`
+is the corresponding input gradient.
+END
+}
+op {
+  graph_op_name: "RecordInput"
+  endpoint {
+    name: "RecordInput"
+  }
+  summary: "Emits randomized records."
+}
+op {
+  graph_op_name: "ReduceJoin"
+  endpoint {
+    name: "ReduceJoin"
+  }
+  summary: "Joins a string Tensor across the given dimensions."
+  description: <<END
+Computes the string join across dimensions in the given string Tensor of shape
+`[d_0, d_1, ..., d_n-1]`.  Returns a new Tensor created by joining the input
+strings with the given separator (default: empty string).  Negative indices are
+counted backwards from the end, with `-1` being equivalent to `n - 1`.
+
+For example:
+
+```python
+# tensor `a` is [["a", "b"], ["c", "d"]]
+tf.reduce_join(a, 0) ==> ["ac", "bd"]
+tf.reduce_join(a, 1) ==> ["ab", "cd"]
+tf.reduce_join(a, -2) = tf.reduce_join(a, 0) ==> ["ac", "bd"]
+tf.reduce_join(a, -1) = tf.reduce_join(a, 1) ==> ["ab", "cd"]
+tf.reduce_join(a, 0, keep_dims=True) ==> [["ac", "bd"]]
+tf.reduce_join(a, 1, keep_dims=True) ==> [["ab"], ["cd"]]
+tf.reduce_join(a, 0, separator=".") ==> ["a.c", "b.d"]
+tf.reduce_join(a, [0, 1]) ==> ["acbd"]
+tf.reduce_join(a, [1, 0]) ==> ["abcd"]
+tf.reduce_join(a, []) ==> ["abcd"]
+```
+END
+}
+op {
+  graph_op_name: "RefEnter"
+  endpoint {
+    name: "RefEnter"
+  }
+  summary: "Creates or finds a child frame, and makes `data` available to the child frame."
+  description: <<END
+The unique `frame_name` is used by the `Executor` to identify frames. If
+`is_constant` is true, `output` is a constant in the child frame; otherwise
+it may be changed in the child frame. At most `parallel_iterations` iterations
+are run in parallel in the child frame.
+END
+}
+op {
+  graph_op_name: "RefExit"
+  endpoint {
+    name: "RefExit"
+  }
+  summary: "Exits the current frame to its parent frame."
+  description: <<END
+Exit makes its input `data` available to the parent frame.
+END
+}
+op {
+  graph_op_name: "RefIdentity"
+  endpoint {
+    name: "RefIdentity"
+  }
+  summary: "Return the same ref tensor as the input ref tensor."
+}
+op {
+  graph_op_name: "RefMerge"
+  endpoint {
+    name: "RefMerge"
+  }
+  summary: "Forwards the value of an available tensor from `inputs` to `output`."
+  description: <<END
+`Merge` waits for at least one of the tensors in `inputs` to become available.
+It is usually combined with `Switch` to implement branching.
+
+`Merge` forwards the first tensor for become available to `output`, and sets
+`value_index` to its index in `inputs`.
+END
+}
+op {
+  graph_op_name: "RefNextIteration"
+  endpoint {
+    name: "RefNextIteration"
+  }
+  summary: "Makes its input available to the next iteration."
+}
+op {
+  graph_op_name: "RefSelect"
+  endpoint {
+    name: "RefSelect"
+  }
+  summary: "Forwards the `index`th element of `inputs` to `output`."
+}
+op {
+  graph_op_name: "RefSwitch"
+  endpoint {
+    name: "RefSwitch"
+  }
+  summary: "Forwards the ref tensor `data` to the output port determined by `pred`."
+  description: <<END
+If `pred` is true, the `data` input is forwarded to `output_true`. Otherwise,
+the data goes to `output_false`.
+
+See also `Switch` and `Merge`.
+END
+}
+op {
+  graph_op_name: "Relu"
+  endpoint {
+    name: "Relu"
+  }
+  summary: "Computes rectified linear: `max(features, 0)`."
+}
+op {
+  graph_op_name: "Relu6"
+  endpoint {
+    name: "Relu6"
+  }
+  summary: "Computes rectified linear 6: `min(max(features, 0), 6)`."
+}
+op {
+  graph_op_name: "Relu6Grad"
+  endpoint {
+    name: "Relu6Grad"
+  }
+  summary: "Computes rectified linear 6 gradients for a Relu6 operation."
+}
+op {
+  graph_op_name: "ReluGrad"
+  endpoint {
+    name: "ReluGrad"
+  }
+  summary: "Computes rectified linear gradients for a Relu operation."
+}
+op {
+  graph_op_name: "RemoteCall"
+  endpoint {
+    name: "RemoteCall"
+  }
+  summary: "Runs function `f` on a remote device indicated by `target`."
+}
+op {
+  graph_op_name: "RemoteFusedGraphExecute"
+  endpoint {
+    name: "RemoteFusedGraphExecute"
+  }
+  summary: "Execute a sub graph on a remote processor."
+  description: <<END
+The graph specifications(such as graph itself, input tensors and output names)
+are stored as a serialized protocol buffer of RemoteFusedGraphExecuteInfo
+as serialized_remote_fused_graph_execute_info.
+The specifications will be passed to a dedicated registered
+remote fused graph executor.  The executor will send the graph specifications
+to a remote processor and execute that graph.  The execution results
+will be passed to consumer nodes as outputs of this node.
+END
+}
+op {
+  graph_op_name: "RepeatDataset"
+  endpoint {
+    name: "RepeatDataset"
+  }
+  summary: "Creates a dataset that emits the outputs of `input_dataset` `count` times."
+}
+op {
+  graph_op_name: "RequantizationRange"
+  endpoint {
+    name: "RequantizationRange"
+  }
+  summary: "Given a quantized tensor described by (input, input_min, input_max), outputs a"
+  description: <<END
+range that covers the actual values present in that tensor.  This op is
+typically used to produce the requested_output_min and requested_output_max for
+Requantize.
+END
+}
+op {
+  graph_op_name: "Requantize"
+  endpoint {
+    name: "Requantize"
+  }
+  summary: "Convert the quantized \'input\' tensor into a lower-precision \'output\', using the"
+  description: <<END
+output range specified with 'requested_output_min' and 'requested_output_max'.
+
+[input_min, input_max] are scalar floats that specify the range for the float
+interpretation of the 'input' data. For example, if input_min is -1.0f and
+input_max is 1.0f, and we are dealing with quint16 quantized data, then a 0
+value in the 16-bit data should be interpreted as -1.0f, and a 65535 means 1.0f.
+END
+}
+op {
+  graph_op_name: "Reshape"
+  endpoint {
+    name: "Reshape"
+  }
+  summary: "Reshapes a tensor."
+  description: <<END
+Given `tensor`, this operation returns a tensor that has the same values
+as `tensor` with shape `shape`.
+
+If one component of `shape` is the special value -1, the size of that dimension
+is computed so that the total size remains constant.  In particular, a `shape`
+of `[-1]` flattens into 1-D.  At most one component of `shape` can be -1.
+
+If `shape` is 1-D or higher, then the operation returns a tensor with shape
+`shape` filled with the values of `tensor`. In this case, the number of elements
+implied by `shape` must be the same as the number of elements in `tensor`.
+
+For example:
+
+```
+# tensor 't' is [1, 2, 3, 4, 5, 6, 7, 8, 9]
+# tensor 't' has shape [9]
+reshape(t, [3, 3]) ==> [[1, 2, 3],
+                        [4, 5, 6],
+                        [7, 8, 9]]
+
+# tensor 't' is [[[1, 1], [2, 2]],
+#                [[3, 3], [4, 4]]]
+# tensor 't' has shape [2, 2, 2]
+reshape(t, [2, 4]) ==> [[1, 1, 2, 2],
+                        [3, 3, 4, 4]]
+
+# tensor 't' is [[[1, 1, 1],
+#                 [2, 2, 2]],
+#                [[3, 3, 3],
+#                 [4, 4, 4]],
+#                [[5, 5, 5],
+#                 [6, 6, 6]]]
+# tensor 't' has shape [3, 2, 3]
+# pass '[-1]' to flatten 't'
+reshape(t, [-1]) ==> [1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 5, 6, 6, 6]
+
+# -1 can also be used to infer the shape
+
+# -1 is inferred to be 9:
+reshape(t, [2, -1]) ==> [[1, 1, 1, 2, 2, 2, 3, 3, 3],
+                         [4, 4, 4, 5, 5, 5, 6, 6, 6]]
+# -1 is inferred to be 2:
+reshape(t, [-1, 9]) ==> [[1, 1, 1, 2, 2, 2, 3, 3, 3],
+                         [4, 4, 4, 5, 5, 5, 6, 6, 6]]
+# -1 is inferred to be 3:
+reshape(t, [ 2, -1, 3]) ==> [[[1, 1, 1],
+                              [2, 2, 2],
+                              [3, 3, 3]],
+                             [[4, 4, 4],
+                              [5, 5, 5],
+                              [6, 6, 6]]]
+
+# tensor 't' is [7]
+# shape `[]` reshapes to a scalar
+reshape(t, []) ==> 7
+```
+END
+}
+op {
+  graph_op_name: "ResizeArea"
+  endpoint {
+    name: "ResizeArea"
+  }
+  summary: "Resize `images` to `size` using area interpolation."
+  description: <<END
+Input images can be of different types but output images are always float.
+
+Each output pixel is computed by first transforming the pixel's footprint into
+the input tensor and then averaging the pixels that intersect the footprint. An
+input pixel's contribution to the average is weighted by the fraction of its
+area that intersects the footprint.  This is the same as OpenCV's INTER_AREA.
+END
+}
+op {
+  graph_op_name: "ResizeBicubic"
+  endpoint {
+    name: "ResizeBicubic"
+  }
+  summary: "Resize `images` to `size` using bicubic interpolation."
+  description: <<END
+Input images can be of different types but output images are always float.
+END
+}
+op {
+  graph_op_name: "ResizeBicubicGrad"
+  endpoint {
+    name: "ResizeBicubicGrad"
+  }
+  summary: "Computes the gradient of bicubic interpolation."
+}
+op {
+  graph_op_name: "ResizeBilinear"
+  endpoint {
+    name: "ResizeBilinear"
+  }
+  summary: "Resize `images` to `size` using bilinear interpolation."
+  description: <<END
+Input images can be of different types but output images are always float.
+END
+}
+op {
+  graph_op_name: "ResizeBilinearGrad"
+  endpoint {
+    name: "ResizeBilinearGrad"
+  }
+  summary: "Computes the gradient of bilinear interpolation."
+}
+op {
+  graph_op_name: "ResizeNearestNeighbor"
+  endpoint {
+    name: "ResizeNearestNeighbor"
+  }
+  summary: "Resize `images` to `size` using nearest neighbor interpolation."
+}
+op {
+  graph_op_name: "ResizeNearestNeighborGrad"
+  endpoint {
+    name: "ResizeNearestNeighborGrad"
+  }
+  summary: "Computes the gradient of nearest neighbor interpolation."
+}
+op {
+  graph_op_name: "ResourceApplyAdadelta"
+  endpoint {
+    name: "ResourceApplyAdadelta"
+  }
+  summary: "Update \'*var\' according to the adadelta scheme."
+  description: <<END
+accum = rho() * accum + (1 - rho()) * grad.square();
+update = (update_accum + epsilon).sqrt() * (accum + epsilon()).rsqrt() * grad;
+update_accum = rho() * update_accum + (1 - rho()) * update.square();
+var -= update;
+END
+}
+op {
+  graph_op_name: "ResourceApplyAdagrad"
+  endpoint {
+    name: "ResourceApplyAdagrad"
+  }
+  summary: "Update \'*var\' according to the adagrad scheme."
+  description: <<END
+accum += grad * grad
+var -= lr * grad * (1 / sqrt(accum))
+END
+}
+op {
+  graph_op_name: "ResourceApplyAdagradDA"
+  endpoint {
+    name: "ResourceApplyAdagradDA"
+  }
+  summary: "Update \'*var\' according to the proximal adagrad scheme."
+}
+op {
+  graph_op_name: "ResourceApplyAdam"
+  endpoint {
+    name: "ResourceApplyAdam"
+  }
+  summary: "Update \'*var\' according to the Adam algorithm."
+  description: <<END
+lr_t <- learning_rate * sqrt(1 - beta2^t) / (1 - beta1^t)
+m_t <- beta1 * m_{t-1} + (1 - beta1) * g_t
+v_t <- beta2 * v_{t-1} + (1 - beta2) * g_t * g_t
+variable <- variable - lr_t * m_t / (sqrt(v_t) + epsilon)
+END
+}
+op {
+  graph_op_name: "ResourceApplyCenteredRMSProp"
+  endpoint {
+    name: "ResourceApplyCenteredRMSProp"
+  }
+  summary: "Update \'*var\' according to the centered RMSProp algorithm."
+  description: <<END
+The centered RMSProp algorithm uses an estimate of the centered second moment
+(i.e., the variance) for normalization, as opposed to regular RMSProp, which
+uses the (uncentered) second moment. This often helps with training, but is
+slightly more expensive in terms of computation and memory.
+
+Note that in dense implementation of this algorithm, mg, ms, and mom will
+update even if the grad is zero, but in this sparse implementation, mg, ms,
+and mom will not update in iterations during which the grad is zero.
+
+mean_square = decay * mean_square + (1-decay) * gradient ** 2
+mean_grad = decay * mean_grad + (1-decay) * gradient
+
+Delta = learning_rate * gradient / sqrt(mean_square + epsilon - mean_grad ** 2)
+
+mg <- rho * mg_{t-1} + (1-rho) * grad
+ms <- rho * ms_{t-1} + (1-rho) * grad * grad
+mom <- momentum * mom_{t-1} + lr * grad / sqrt(ms - mg * mg + epsilon)
+var <- var - mom
+END
+}
+op {
+  graph_op_name: "ResourceApplyFtrl"
+  endpoint {
+    name: "ResourceApplyFtrl"
+  }
+  summary: "Update \'*var\' according to the Ftrl-proximal scheme."
+  description: <<END
+accum_new = accum + grad * grad
+linear += grad - (accum_new^(-lr_power) - accum^(-lr_power)) / lr * var
+quadratic = 1.0 / (accum_new^(lr_power) * lr) + 2 * l2
+var = (sign(linear) * l1 - linear) / quadratic if |linear| > l1 else 0.0
+accum = accum_new
+END
+}
+op {
+  graph_op_name: "ResourceApplyFtrlV2"
+  endpoint {
+    name: "ResourceApplyFtrlV2"
+  }
+  summary: "Update \'*var\' according to the Ftrl-proximal scheme."
+  description: <<END
+grad_with_shrinkage = grad + 2 * l2_shrinkage * var
+accum_new = accum + grad_with_shrinkage * grad_with_shrinkage
+linear += grad_with_shrinkage +
+    (accum_new^(-lr_power) - accum^(-lr_power)) / lr * var
+quadratic = 1.0 / (accum_new^(lr_power) * lr) + 2 * l2
+var = (sign(linear) * l1 - linear) / quadratic if |linear| > l1 else 0.0
+accum = accum_new
+END
+}
+op {
+  graph_op_name: "ResourceApplyGradientDescent"
+  endpoint {
+    name: "ResourceApplyGradientDescent"
+  }
+  summary: "Update \'*var\' by subtracting \'alpha\' * \'delta\' from it."
+}
+op {
+  graph_op_name: "ResourceApplyMomentum"
+  endpoint {
+    name: "ResourceApplyMomentum"
+  }
+  summary: "Update \'*var\' according to the momentum scheme. Set use_nesterov = True if you"
+  description: <<END
+want to use Nesterov momentum.
+
+accum = accum * momentum + grad
+var -= lr * accum
+END
+}
+op {
+  graph_op_name: "ResourceApplyProximalAdagrad"
+  endpoint {
+    name: "ResourceApplyProximalAdagrad"
+  }
+  summary: "Update \'*var\' and \'*accum\' according to FOBOS with Adagrad learning rate."
+  description: <<END
+accum += grad * grad
+prox_v = var - lr * grad * (1 / sqrt(accum))
+var = sign(prox_v)/(1+lr*l2) * max{|prox_v|-lr*l1,0}
+END
+}
+op {
+  graph_op_name: "ResourceApplyProximalGradientDescent"
+  endpoint {
+    name: "ResourceApplyProximalGradientDescent"
+  }
+  summary: "Update \'*var\' as FOBOS algorithm with fixed learning rate."
+  description: <<END
+prox_v = var - alpha * delta
+var = sign(prox_v)/(1+alpha*l2) * max{|prox_v|-alpha*l1,0}
+END
+}
+op {
+  graph_op_name: "ResourceApplyRMSProp"
+  endpoint {
+    name: "ResourceApplyRMSProp"
+  }
+  summary: "Update \'*var\' according to the RMSProp algorithm."
+  description: <<END
+Note that in dense implementation of this algorithm, ms and mom will
+update even if the grad is zero, but in this sparse implementation, ms
+and mom will not update in iterations during which the grad is zero.
+
+mean_square = decay * mean_square + (1-decay) * gradient ** 2
+Delta = learning_rate * gradient / sqrt(mean_square + epsilon)
+
+ms <- rho * ms_{t-1} + (1-rho) * grad * grad
+mom <- momentum * mom_{t-1} + lr * grad / sqrt(ms + epsilon)
+var <- var - mom
+END
+}
+op {
+  graph_op_name: "ResourceSparseApplyAdadelta"
+  endpoint {
+    name: "ResourceSparseApplyAdadelta"
+  }
+  summary: "var: Should be from a Variable()."
+}
+op {
+  graph_op_name: "ResourceSparseApplyAdagrad"
+  endpoint {
+    name: "ResourceSparseApplyAdagrad"
+  }
+  summary: "Update relevant entries in \'*var\' and \'*accum\' according to the adagrad scheme."
+  description: <<END
+That is for rows we have grad for, we update var and accum as follows:
+accum += grad * grad
+var -= lr * grad * (1 / sqrt(accum))
+END
+}
+op {
+  graph_op_name: "ResourceSparseApplyAdagradDA"
+  endpoint {
+    name: "ResourceSparseApplyAdagradDA"
+  }
+  summary: "Update entries in \'*var\' and \'*accum\' according to the proximal adagrad scheme."
+}
+op {
+  graph_op_name: "ResourceSparseApplyCenteredRMSProp"
+  endpoint {
+    name: "ResourceSparseApplyCenteredRMSProp"
+  }
+  summary: "Update \'*var\' according to the centered RMSProp algorithm."
+  description: <<END
+The centered RMSProp algorithm uses an estimate of the centered second moment
+(i.e., the variance) for normalization, as opposed to regular RMSProp, which
+uses the (uncentered) second moment. This often helps with training, but is
+slightly more expensive in terms of computation and memory.
+
+Note that in dense implementation of this algorithm, mg, ms, and mom will
+update even if the grad is zero, but in this sparse implementation, mg, ms,
+and mom will not update in iterations during which the grad is zero.
+
+mean_square = decay * mean_square + (1-decay) * gradient ** 2
+mean_grad = decay * mean_grad + (1-decay) * gradient
+Delta = learning_rate * gradient / sqrt(mean_square + epsilon - mean_grad ** 2)
+
+ms <- rho * ms_{t-1} + (1-rho) * grad * grad
+mom <- momentum * mom_{t-1} + lr * grad / sqrt(ms + epsilon)
+var <- var - mom
+END
+}
+op {
+  graph_op_name: "ResourceSparseApplyFtrl"
+  endpoint {
+    name: "ResourceSparseApplyFtrl"
+  }
+  summary: "Update relevant entries in \'*var\' according to the Ftrl-proximal scheme."
+  description: <<END
+That is for rows we have grad for, we update var, accum and linear as follows:
+accum_new = accum + grad * grad
+linear += grad + (accum_new^(-lr_power) - accum^(-lr_power)) / lr * var
+quadratic = 1.0 / (accum_new^(lr_power) * lr) + 2 * l2
+var = (sign(linear) * l1 - linear) / quadratic if |linear| > l1 else 0.0
+accum = accum_new
+END
+}
+op {
+  graph_op_name: "ResourceSparseApplyFtrlV2"
+  endpoint {
+    name: "ResourceSparseApplyFtrlV2"
+  }
+  summary: "Update relevant entries in \'*var\' according to the Ftrl-proximal scheme."
+  description: <<END
+That is for rows we have grad for, we update var, accum and linear as follows:
+grad_with_shrinkage = grad + 2 * l2_shrinkage * var
+accum_new = accum + grad_with_shrinkage * grad_with_shrinkage
+linear += grad_with_shrinkage +
+    (accum_new^(-lr_power) - accum^(-lr_power)) / lr * var
+quadratic = 1.0 / (accum_new^(lr_power) * lr) + 2 * l2
+var = (sign(linear) * l1 - linear) / quadratic if |linear| > l1 else 0.0
+accum = accum_new
+END
+}
+op {
+  graph_op_name: "ResourceSparseApplyMomentum"
+  endpoint {
+    name: "ResourceSparseApplyMomentum"
+  }
+  summary: "Update relevant entries in \'*var\' and \'*accum\' according to the momentum scheme."
+  description: <<END
+Set use_nesterov = True if you want to use Nesterov momentum.
+
+That is for rows we have grad for, we update var and accum as follows:
+
+accum = accum * momentum + grad
+var -= lr * accum
+END
+}
+op {
+  graph_op_name: "ResourceSparseApplyProximalAdagrad"
+  endpoint {
+    name: "ResourceSparseApplyProximalAdagrad"
+  }
+  summary: "Sparse update entries in \'*var\' and \'*accum\' according to FOBOS algorithm."
+  description: <<END
+That is for rows we have grad for, we update var and accum as follows:
+accum += grad * grad
+prox_v = var
+prox_v -= lr * grad * (1 / sqrt(accum))
+var = sign(prox_v)/(1+lr*l2) * max{|prox_v|-lr*l1,0}
+END
+}
+op {
+  graph_op_name: "ResourceSparseApplyProximalGradientDescent"
+  endpoint {
+    name: "ResourceSparseApplyProximalGradientDescent"
+  }
+  summary: "Sparse update \'*var\' as FOBOS algorithm with fixed learning rate."
+  description: <<END
+That is for rows we have grad for, we update var as follows:
+prox_v = var - alpha * grad
+var = sign(prox_v)/(1+alpha*l2) * max{|prox_v|-alpha*l1,0}
+END
+}
+op {
+  graph_op_name: "ResourceSparseApplyRMSProp"
+  endpoint {
+    name: "ResourceSparseApplyRMSProp"
+  }
+  summary: "Update \'*var\' according to the RMSProp algorithm."
+  description: <<END
+Note that in dense implementation of this algorithm, ms and mom will
+update even if the grad is zero, but in this sparse implementation, ms
+and mom will not update in iterations during which the grad is zero.
+
+mean_square = decay * mean_square + (1-decay) * gradient ** 2
+Delta = learning_rate * gradient / sqrt(mean_square + epsilon)
+
+ms <- rho * ms_{t-1} + (1-rho) * grad * grad
+mom <- momentum * mom_{t-1} + lr * grad / sqrt(ms + epsilon)
+var <- var - mom
+END
+}
+op {
+  graph_op_name: "ResourceStridedSliceAssign"
+  endpoint {
+    name: "ResourceStridedSliceAssign"
+  }
+  summary: "Assign `value` to the sliced l-value reference of `ref`."
+  description: <<END
+The values of `value` are assigned to the positions in the variable
+`ref` that are selected by the slice parameters. The slice parameters
+`begin, `end`, `strides`, etc. work exactly as in `StridedSlice`.
+
+NOTE this op currently does not support broadcasting and so `value`'s
+shape must be exactly the shape produced by the slice of `ref`.
+END
+}
+op {
+  graph_op_name: "Restore"
+  endpoint {
+    name: "Restore"
+  }
+  summary: "Restores a tensor from checkpoint files."
+  description: <<END
+Reads a tensor stored in one or several files. If there are several files (for
+instance because a tensor was saved as slices), `file_pattern` may contain
+wildcard symbols (`*` and `?`) in the filename portion only, not in the
+directory portion.
+
+If a `file_pattern` matches several files, `preferred_shard` can be used to hint
+in which file the requested tensor is likely to be found. This op will first
+open the file at index `preferred_shard` in the list of matching files and try
+to restore tensors from that file.  Only if some tensors or tensor slices are
+not found in that first file, then the Op opens all the files. Setting
+`preferred_shard` to match the value passed as the `shard` input
+of a matching `Save` Op may speed up Restore.  This attribute only affects
+performance, not correctness.  The default value -1 means files are processed in
+order.
+
+See also `RestoreSlice`.
+END
+}
+op {
+  graph_op_name: "RestoreIterator"
+  endpoint {
+    name: "RestoreIterator"
+  }
+  summary: "Restores the state of the `iterator` from the checkpoint saved at `path` using \"SaveIterator\"."
+}
+op {
+  graph_op_name: "RestoreSlice"
+  endpoint {
+    name: "RestoreSlice"
+  }
+  summary: "Restores a tensor from checkpoint files."
+  description: <<END
+This is like `Restore` except that restored tensor can be listed as filling
+only a slice of a larger tensor.  `shape_and_slice` specifies the shape of the
+larger tensor and the slice that the restored tensor covers.
+
+The `shape_and_slice` input has the same format as the
+elements of the `shapes_and_slices` input of the `SaveSlices` op.
+END
+}
+op {
+  graph_op_name: "RestoreV2"
+  endpoint {
+    name: "RestoreV2"
+  }
+  summary: "Restores tensors from a V2 checkpoint."
+  description: <<END
+For backward compatibility with the V1 format, this Op currently allows
+restoring from a V1 checkpoint as well:
+  - This Op first attempts to find the V2 index file pointed to by "prefix", and
+    if found proceed to read it as a V2 checkpoint;
+  - Otherwise the V1 read path is invoked.
+Relying on this behavior is not recommended, as the ability to fall back to read
+V1 might be deprecated and eventually removed.
+
+By default, restores the named tensors in full.  If the caller wishes to restore
+specific slices of stored tensors, "shape_and_slices" should be non-empty
+strings and correspondingly well-formed.
+
+Callers must ensure all the named tensors are indeed stored in the checkpoint.
+END
+}
+op {
+  graph_op_name: "Reverse"
+  endpoint {
+    name: "Reverse"
+  }
+  summary: "Reverses specific dimensions of a tensor."
+  description: <<END
+Given a `tensor`, and a `bool` tensor `dims` representing the dimensions
+of `tensor`, this operation reverses each dimension i of `tensor` where
+`dims[i]` is `True`.
+
+`tensor` can have up to 8 dimensions. The number of dimensions
+of `tensor` must equal the number of elements in `dims`. In other words:
+
+`rank(tensor) = size(dims)`
+
+For example:
+
+```
+# tensor 't' is [[[[ 0,  1,  2,  3],
+#                  [ 4,  5,  6,  7],
+#                  [ 8,  9, 10, 11]],
+#                 [[12, 13, 14, 15],
+#                  [16, 17, 18, 19],
+#                  [20, 21, 22, 23]]]]
+# tensor 't' shape is [1, 2, 3, 4]
+
+# 'dims' is [False, False, False, True]
+reverse(t, dims) ==> [[[[ 3,  2,  1,  0],
+                        [ 7,  6,  5,  4],
+                        [ 11, 10, 9, 8]],
+                       [[15, 14, 13, 12],
+                        [19, 18, 17, 16],
+                        [23, 22, 21, 20]]]]
+
+# 'dims' is [False, True, False, False]
+reverse(t, dims) ==> [[[[12, 13, 14, 15],
+                        [16, 17, 18, 19],
+                        [20, 21, 22, 23]
+                       [[ 0,  1,  2,  3],
+                        [ 4,  5,  6,  7],
+                        [ 8,  9, 10, 11]]]]
+
+# 'dims' is [False, False, True, False]
+reverse(t, dims) ==> [[[[8, 9, 10, 11],
+                        [4, 5, 6, 7],
+                        [0, 1, 2, 3]]
+                       [[20, 21, 22, 23],
+                        [16, 17, 18, 19],
+                        [12, 13, 14, 15]]]]
+```
+END
+}
+op {
+  graph_op_name: "ReverseSequence"
+  endpoint {
+    name: "ReverseSequence"
+  }
+  summary: "Reverses variable length slices."
+  description: <<END
+This op first slices `input` along the dimension `batch_dim`, and for each
+slice `i`, reverses the first `seq_lengths[i]` elements along
+the dimension `seq_dim`.
+
+The elements of `seq_lengths` must obey `seq_lengths[i] <= input.dims[seq_dim]`,
+and `seq_lengths` must be a vector of length `input.dims[batch_dim]`.
+
+The output slice `i` along dimension `batch_dim` is then given by input
+slice `i`, with the first `seq_lengths[i]` slices along dimension
+`seq_dim` reversed.
+
+For example:
+
+```
+# Given this:
+batch_dim = 0
+seq_dim = 1
+input.dims = (4, 8, ...)
+seq_lengths = [7, 2, 3, 5]
+
+# then slices of input are reversed on seq_dim, but only up to seq_lengths:
+output[0, 0:7, :, ...] = input[0, 7:0:-1, :, ...]
+output[1, 0:2, :, ...] = input[1, 2:0:-1, :, ...]
+output[2, 0:3, :, ...] = input[2, 3:0:-1, :, ...]
+output[3, 0:5, :, ...] = input[3, 5:0:-1, :, ...]
+
+# while entries past seq_lens are copied through:
+output[0, 7:, :, ...] = input[0, 7:, :, ...]
+output[1, 2:, :, ...] = input[1, 2:, :, ...]
+output[2, 3:, :, ...] = input[2, 3:, :, ...]
+output[3, 2:, :, ...] = input[3, 2:, :, ...]
+```
+
+In contrast, if:
+
+```
+# Given this:
+batch_dim = 2
+seq_dim = 0
+input.dims = (8, ?, 4, ...)
+seq_lengths = [7, 2, 3, 5]
+
+# then slices of input are reversed on seq_dim, but only up to seq_lengths:
+output[0:7, :, 0, :, ...] = input[7:0:-1, :, 0, :, ...]
+output[0:2, :, 1, :, ...] = input[2:0:-1, :, 1, :, ...]
+output[0:3, :, 2, :, ...] = input[3:0:-1, :, 2, :, ...]
+output[0:5, :, 3, :, ...] = input[5:0:-1, :, 3, :, ...]
+
+# while entries past seq_lens are copied through:
+output[7:, :, 0, :, ...] = input[7:, :, 0, :, ...]
+output[2:, :, 1, :, ...] = input[2:, :, 1, :, ...]
+output[3:, :, 2, :, ...] = input[3:, :, 2, :, ...]
+output[2:, :, 3, :, ...] = input[2:, :, 3, :, ...]
+```
+END
+}
+op {
+  graph_op_name: "ReverseV2"
+  endpoint {
+    name: "ReverseV2"
+  }
+  summary: "Reverses specific dimensions of a tensor."
+  description: <<END
+NOTE `tf.reverse` has now changed behavior in preparation for 1.0.
+`tf.reverse_v2` is currently an alias that will be deprecated before TF 1.0.
+
+Given a `tensor`, and a `int32` tensor `axis` representing the set of
+dimensions of `tensor` to reverse. This operation reverses each dimension
+`i` for which there exists `j` s.t. `axis[j] == i`.
+
+`tensor` can have up to 8 dimensions. The number of dimensions specified
+in `axis` may be 0 or more entries. If an index is specified more than
+once, a InvalidArgument error is raised.
+
+For example:
+
+```
+# tensor 't' is [[[[ 0,  1,  2,  3],
+#                  [ 4,  5,  6,  7],
+#                  [ 8,  9, 10, 11]],
+#                 [[12, 13, 14, 15],
+#                  [16, 17, 18, 19],
+#                  [20, 21, 22, 23]]]]
+# tensor 't' shape is [1, 2, 3, 4]
+
+# 'dims' is [3] or 'dims' is -1
+reverse(t, dims) ==> [[[[ 3,  2,  1,  0],
+                        [ 7,  6,  5,  4],
+                        [ 11, 10, 9, 8]],
+                       [[15, 14, 13, 12],
+                        [19, 18, 17, 16],
+                        [23, 22, 21, 20]]]]
+
+# 'dims' is '[1]' (or 'dims' is '[-3]')
+reverse(t, dims) ==> [[[[12, 13, 14, 15],
+                        [16, 17, 18, 19],
+                        [20, 21, 22, 23]
+                       [[ 0,  1,  2,  3],
+                        [ 4,  5,  6,  7],
+                        [ 8,  9, 10, 11]]]]
+
+# 'dims' is '[2]' (or 'dims' is '[-2]')
+reverse(t, dims) ==> [[[[8, 9, 10, 11],
+                        [4, 5, 6, 7],
+                        [0, 1, 2, 3]]
+                       [[20, 21, 22, 23],
+                        [16, 17, 18, 19],
+                        [12, 13, 14, 15]]]]
+```
+END
+}
+op {
+  graph_op_name: "Rint"
+  endpoint {
+    name: "Rint"
+  }
+  summary: "Returns element-wise integer closest to x."
+  description: <<END
+If the result is midway between two representable values,
+the even representable is chosen.
+For example:
+
+```
+rint(-1.5) ==> -2.0
+rint(0.5000001) ==> 1.0
+rint([-1.7, -1.5, -0.2, 0.2, 1.5, 1.7, 2.0]) ==> [-2., -2., -0., 0., 2., 2., 2.]
+```
+END
+}
+op {
+  graph_op_name: "Round"
+  endpoint {
+    name: "Round"
+  }
+  summary: "Rounds the values of a tensor to the nearest integer, element-wise."
+  description: <<END
+Rounds half to even.  Also known as bankers rounding. If you want to round
+according to the current system rounding mode use std::cint.
+END
+}
+op {
+  graph_op_name: "Rsqrt"
+  endpoint {
+    name: "Rsqrt"
+  }
+  summary: "Computes reciprocal of square root of x element-wise."
+  description: <<END
+I.e., \\(y = 1 / \sqrt{x}\\).
+END
+}
+op {
+  graph_op_name: "RsqrtGrad"
+  endpoint {
+    name: "RsqrtGrad"
+  }
+  summary: "Computes the gradient for the rsqrt of `x` wrt its input."
+  description: <<END
+Specifically, `grad = dy * -0.5 * y^3`, where `y = rsqrt(x)`, and `dy`
+is the corresponding input gradient.
+END
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_S.pbtxt b/tensorflow/core/api_def/base_api/api_def_S.pbtxt
new file mode 100644
index 0000000000..9c53f9ac62
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_S.pbtxt
@@ -0,0 +1,2678 @@
+op {
+  graph_op_name: "SampleDistortedBoundingBox"
+  endpoint {
+    name: "SampleDistortedBoundingBox"
+  }
+  summary: "Generate a single randomly distorted bounding box for an image."
+  description: <<END
+Bounding box annotations are often supplied in addition to ground-truth labels
+in image recognition or object localization tasks. A common technique for
+training such a system is to randomly distort an image while preserving
+its content, i.e. *data augmentation*. This Op outputs a randomly distorted
+localization of an object, i.e. bounding box, given an `image_size`,
+`bounding_boxes` and a series of constraints.
+
+The output of this Op is a single bounding box that may be used to crop the
+original image. The output is returned as 3 tensors: `begin`, `size` and
+`bboxes`. The first 2 tensors can be fed directly into `tf.slice` to crop the
+image. The latter may be supplied to `tf.image.draw_bounding_boxes` to visualize
+what the bounding box looks like.
+
+Bounding boxes are supplied and returned as `[y_min, x_min, y_max, x_max]`. The
+bounding box coordinates are floats in `[0.0, 1.0]` relative to the width and
+height of the underlying image.
+
+For example,
+
+```python
+    # Generate a single distorted bounding box.
+    begin, size, bbox_for_draw = tf.image.sample_distorted_bounding_box(
+        tf.shape(image),
+        bounding_boxes=bounding_boxes)
+
+    # Draw the bounding box in an image summary.
+    image_with_box = tf.image.draw_bounding_boxes(tf.expand_dims(image, 0),
+                                                  bbox_for_draw)
+    tf.image_summary('images_with_box', image_with_box)
+
+    # Employ the bounding box to distort the image.
+    distorted_image = tf.slice(image, begin, size)
+```
+
+Note that if no bounding box information is available, setting
+`use_image_if_no_bounding_boxes = true` will assume there is a single implicit
+bounding box covering the whole image. If `use_image_if_no_bounding_boxes` is
+false and no bounding boxes are supplied, an error is raised.
+END
+}
+op {
+  graph_op_name: "SampleDistortedBoundingBoxV2"
+  endpoint {
+    name: "SampleDistortedBoundingBoxV2"
+  }
+  summary: "Generate a single randomly distorted bounding box for an image."
+  description: <<END
+Bounding box annotations are often supplied in addition to ground-truth labels
+in image recognition or object localization tasks. A common technique for
+training such a system is to randomly distort an image while preserving
+its content, i.e. *data augmentation*. This Op outputs a randomly distorted
+localization of an object, i.e. bounding box, given an `image_size`,
+`bounding_boxes` and a series of constraints.
+
+The output of this Op is a single bounding box that may be used to crop the
+original image. The output is returned as 3 tensors: `begin`, `size` and
+`bboxes`. The first 2 tensors can be fed directly into `tf.slice` to crop the
+image. The latter may be supplied to `tf.image.draw_bounding_boxes` to visualize
+what the bounding box looks like.
+
+Bounding boxes are supplied and returned as `[y_min, x_min, y_max, x_max]`. The
+bounding box coordinates are floats in `[0.0, 1.0]` relative to the width and
+height of the underlying image.
+
+For example,
+
+```python
+    # Generate a single distorted bounding box.
+    begin, size, bbox_for_draw = tf.image.sample_distorted_bounding_box(
+        tf.shape(image),
+        bounding_boxes=bounding_boxes)
+
+    # Draw the bounding box in an image summary.
+    image_with_box = tf.image.draw_bounding_boxes(tf.expand_dims(image, 0),
+                                                  bbox_for_draw)
+    tf.image_summary('images_with_box', image_with_box)
+
+    # Employ the bounding box to distort the image.
+    distorted_image = tf.slice(image, begin, size)
+```
+
+Note that if no bounding box information is available, setting
+`use_image_if_no_bounding_boxes = true` will assume there is a single implicit
+bounding box covering the whole image. If `use_image_if_no_bounding_boxes` is
+false and no bounding boxes are supplied, an error is raised.
+END
+}
+op {
+  graph_op_name: "Save"
+  endpoint {
+    name: "Save"
+  }
+  summary: "Saves the input tensors to disk."
+  description: <<END
+The size of `tensor_names` must match the number of tensors in `data`. `data[i]`
+is written to `filename` with name `tensor_names[i]`.
+
+See also `SaveSlices`.
+END
+}
+op {
+  graph_op_name: "SaveIterator"
+  endpoint {
+    name: "SaveIterator"
+  }
+  summary: "Saves the state of the `iterator` at `path`."
+  description: <<END
+This state can be restored using "RestoreIterator".
+END
+}
+op {
+  graph_op_name: "SaveSlices"
+  endpoint {
+    name: "SaveSlices"
+  }
+  summary: "Saves input tensors slices to disk."
+  description: <<END
+This is like `Save` except that tensors can be listed in the saved file as being
+a slice of a larger tensor.  `shapes_and_slices` specifies the shape of the
+larger tensor and the slice that this tensor covers. `shapes_and_slices` must
+have as many elements as `tensor_names`.
+
+Elements of the `shapes_and_slices` input must either be:
+
+*  The empty string, in which case the corresponding tensor is
+   saved normally.
+*  A string of the form `dim0 dim1 ... dimN-1 slice-spec` where the
+   `dimI` are the dimensions of the larger tensor and `slice-spec`
+   specifies what part is covered by the tensor to save.
+
+`slice-spec` itself is a `:`-separated list: `slice0:slice1:...:sliceN-1`
+where each `sliceI` is either:
+
+*  The string `-` meaning that the slice covers all indices of this dimension
+*  `start,length` where `start` and `length` are integers.  In that
+   case the slice covers `length` indices starting at `start`.
+
+See also `Save`.
+END
+}
+op {
+  graph_op_name: "SaveV2"
+  endpoint {
+    name: "SaveV2"
+  }
+  summary: "Saves tensors in V2 checkpoint format."
+  description: <<END
+By default, saves the named tensors in full.  If the caller wishes to save
+specific slices of full tensors, "shape_and_slices" should be non-empty strings
+and correspondingly well-formed.
+END
+}
+op {
+  graph_op_name: "ScalarSummary"
+  endpoint {
+    name: "ScalarSummary"
+  }
+  summary: "Outputs a `Summary` protocol buffer with scalar values."
+  description: <<END
+The input `tags` and `values` must have the same shape.  The generated summary
+has a summary value for each tag-value pair in `tags` and `values`.
+END
+}
+op {
+  graph_op_name: "ScatterAdd"
+  endpoint {
+    name: "ScatterAdd"
+  }
+  summary: "Adds sparse updates to a variable reference."
+  description: <<END
+This operation computes
+
+    # Scalar indices
+    ref[indices, ...] += updates[...]
+
+    # Vector indices (for each i)
+    ref[indices[i], ...] += updates[i, ...]
+
+    # High rank indices (for each i, ..., j)
+    ref[indices[i, ..., j], ...] += updates[i, ..., j, ...]
+
+This operation outputs `ref` after the update is done.
+This makes it easier to chain operations that need to use the reset value.
+
+Duplicate entries are handled correctly: if multiple `indices` reference
+the same location, their contributions add.
+
+Requires `updates.shape = indices.shape + ref.shape[1:]`.
+
+<div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
+<img style="width:100%" src="https://www.tensorflow.org/images/ScatterAdd.png" alt>
+</div>
+END
+}
+op {
+  graph_op_name: "ScatterDiv"
+  endpoint {
+    name: "ScatterDiv"
+  }
+  summary: "Divides a variable reference by sparse updates."
+  description: <<END
+This operation computes
+
+```python
+    # Scalar indices
+    ref[indices, ...] /= updates[...]
+
+    # Vector indices (for each i)
+    ref[indices[i], ...] /= updates[i, ...]
+
+    # High rank indices (for each i, ..., j)
+    ref[indices[i, ..., j], ...] /= updates[i, ..., j, ...]
+```
+
+This operation outputs `ref` after the update is done.
+This makes it easier to chain operations that need to use the reset value.
+
+Duplicate entries are handled correctly: if multiple `indices` reference
+the same location, their contributions divide.
+
+Requires `updates.shape = indices.shape + ref.shape[1:]`.
+END
+}
+op {
+  graph_op_name: "ScatterMul"
+  endpoint {
+    name: "ScatterMul"
+  }
+  summary: "Multiplies sparse updates into a variable reference."
+  description: <<END
+This operation computes
+
+```python
+    # Scalar indices
+    ref[indices, ...] *= updates[...]
+
+    # Vector indices (for each i)
+    ref[indices[i], ...] *= updates[i, ...]
+
+    # High rank indices (for each i, ..., j)
+    ref[indices[i, ..., j], ...] *= updates[i, ..., j, ...]
+```
+
+This operation outputs `ref` after the update is done.
+This makes it easier to chain operations that need to use the reset value.
+
+Duplicate entries are handled correctly: if multiple `indices` reference
+the same location, their contributions multiply.
+
+Requires `updates.shape = indices.shape + ref.shape[1:]`.
+END
+}
+op {
+  graph_op_name: "ScatterNd"
+  endpoint {
+    name: "ScatterNd"
+  }
+  summary: "Scatter `updates` into a new (initially zero) tensor according to `indices`."
+  description: <<END
+Creates a new tensor by applying sparse `updates` to individual
+values or slices within a zero tensor of the given `shape` according to
+indices.  This operator is the inverse of the @{tf.gather_nd} operator which
+extracts values or slices from a given tensor.
+
+**WARNING**: The order in which updates are applied is nondeterministic, so the
+output will be nondeterministic if `indices` contains duplicates.
+
+`indices` is an integer tensor containing indices into a new tensor of shape
+`shape`.  The last dimension of `indices` can be at most the rank of `shape`:
+
+    indices.shape[-1] <= shape.rank
+
+The last dimension of `indices` corresponds to indices into elements
+(if `indices.shape[-1] = shape.rank`) or slices
+(if `indices.shape[-1] < shape.rank`) along dimension `indices.shape[-1]` of
+`shape`.  `updates` is a tensor with shape
+
+    indices.shape[:-1] + shape[indices.shape[-1]:]
+
+The simplest form of scatter is to insert individual elements in a tensor by
+index. For example, say we want to insert 4 scattered elements in a rank-1
+tensor with 8 elements.
+
+<div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
+<img style="width:100%" src="https://www.tensorflow.org/images/ScatterNd1.png" alt>
+</div>
+
+In Python, this scatter operation would look like this:
+
+```python
+    indices = tf.constant([[4], [3], [1], [7]])
+    updates = tf.constant([9, 10, 11, 12])
+    shape = tf.constant([8])
+    scatter = tf.scatter_nd(indices, updates, shape)
+    with tf.Session() as sess:
+      print(sess.run(scatter))
+```
+
+The resulting tensor would look like this:
+
+    [0, 11, 0, 10, 9, 0, 0, 12]
+
+We can also, insert entire slices of a higher rank tensor all at once. For
+example, if we wanted to insert two slices in the first dimension of a
+rank-3 tensor with two matrices of new values.
+
+<div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
+<img style="width:100%" src="https://www.tensorflow.org/images/ScatterNd2.png" alt>
+</div>
+
+In Python, this scatter operation would look like this:
+
+```python
+    indices = tf.constant([[0], [2]])
+    updates = tf.constant([[[5, 5, 5, 5], [6, 6, 6, 6],
+                            [7, 7, 7, 7], [8, 8, 8, 8]],
+                           [[5, 5, 5, 5], [6, 6, 6, 6],
+                            [7, 7, 7, 7], [8, 8, 8, 8]]])
+    shape = tf.constant([4, 4, 4])
+    scatter = tf.scatter_nd(indices, updates, shape)
+    with tf.Session() as sess:
+      print(sess.run(scatter))
+```
+
+The resulting tensor would look like this:
+
+    [[[5, 5, 5, 5], [6, 6, 6, 6], [7, 7, 7, 7], [8, 8, 8, 8]],
+     [[0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0]],
+     [[5, 5, 5, 5], [6, 6, 6, 6], [7, 7, 7, 7], [8, 8, 8, 8]],
+     [[0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0]]]
+END
+}
+op {
+  graph_op_name: "ScatterNdAdd"
+  endpoint {
+    name: "ScatterNdAdd"
+  }
+  summary: "Applies sparse addition between `updates` and individual values or slices"
+  description: <<END
+within a given variable according to `indices`.
+
+`ref` is a `Tensor` with rank `P` and `indices` is a `Tensor` of rank `Q`.
+
+`indices` must be integer tensor, containing indices into `ref`.
+It must be shape `[d_0, ..., d_{Q-2}, K]` where `0 < K <= P`.
+
+The innermost dimension of `indices` (with length `K`) corresponds to
+indices into elements (if `K = P`) or slices (if `K < P`) along the `K`th
+dimension of `ref`.
+
+`updates` is `Tensor` of rank `Q-1+P-K` with shape:
+
+```
+[d_0, ..., d_{Q-2}, ref.shape[K], ..., ref.shape[P-1]].
+```
+
+For example, say we want to add 4 scattered elements to a rank-1 tensor to 8
+elements. In Python, that addition would look like this:
+
+    ref = tf.Variable([1, 2, 3, 4, 5, 6, 7, 8])
+    indices = tf.constant([[4], [3], [1], [7]])
+    updates = tf.constant([9, 10, 11, 12])
+    add = tf.scatter_nd_add(ref, indices, updates)
+    with tf.Session() as sess:
+      print sess.run(add)
+
+The resulting update to ref would look like this:
+
+    [1, 13, 3, 14, 14, 6, 7, 20]
+
+See @{tf.scatter_nd} for more details about how to make updates to
+slices.
+END
+}
+op {
+  graph_op_name: "ScatterNdNonAliasingAdd"
+  endpoint {
+    name: "ScatterNdNonAliasingAdd"
+  }
+  summary: "Applies sparse addition to `input` using individual values or slices"
+  description: <<END
+from `updates` according to indices `indices`.  The updates are non-aliasing:
+`input` is only modified in-place if no other operations will use it.
+Otherwise, a copy of `input` is made.  This operation has a gradient with
+respect to both `input` and `updates`.
+
+`input` is a `Tensor` with rank `P` and `indices` is a `Tensor` of rank `Q`.
+
+`indices` must be integer tensor, containing indices into `input`.
+It must be shape `[d_0, ..., d_{Q-2}, K]` where `0 < K <= P`.
+
+The innermost dimension of `indices` (with length `K`) corresponds to
+indices into elements (if `K = P`) or `(P-K)`-dimensional slices
+(if `K < P`) along the `K`th dimension of `input`.
+
+`updates` is `Tensor` of rank `Q-1+P-K` with shape:
+
+```
+[d_0, ..., d_{Q-2}, input.shape[K], ..., input.shape[P-1]].
+```
+
+For example, say we want to add 4 scattered elements to a rank-1 tensor to 8
+elements. In Python, that addition would look like this:
+
+    input = tf.constant([1, 2, 3, 4, 5, 6, 7, 8])
+    indices = tf.constant([[4], [3], [1], [7]])
+    updates = tf.constant([9, 10, 11, 12])
+    output = tf.scatter_nd_non_aliasing_add(input, indices, updates)
+    with tf.Session() as sess:
+      print(sess.run(output))
+
+The resulting value `output` would look like this:
+
+    [1, 13, 3, 14, 14, 6, 7, 20]
+
+See @{tf.scatter_nd} for more details about how to make updates to slices.
+END
+}
+op {
+  graph_op_name: "ScatterNdSub"
+  endpoint {
+    name: "ScatterNdSub"
+  }
+  summary: "Applies sparse subtraction between `updates` and individual values or slices"
+  description: <<END
+within a given variable according to `indices`.
+
+`ref` is a `Tensor` with rank `P` and `indices` is a `Tensor` of rank `Q`.
+
+`indices` must be integer tensor, containing indices into `ref`.
+It must be shape `[d_0, ..., d_{Q-2}, K]` where `0 < K <= P`.
+
+The innermost dimension of `indices` (with length `K`) corresponds to
+indices into elements (if `K = P`) or slices (if `K < P`) along the `K`th
+dimension of `ref`.
+
+`updates` is `Tensor` of rank `Q-1+P-K` with shape:
+
+```
+[d_0, ..., d_{Q-2}, ref.shape[K], ..., ref.shape[P-1]].
+```
+
+For example, say we want to subtract 4 scattered elements from a rank-1 tensor
+with 8 elements. In Python, that subtraction would look like this:
+
+    ref = tf.Variable([1, 2, 3, 4, 5, 6, 7, 8])
+    indices = tf.constant([[4], [3], [1], [7]])
+    updates = tf.constant([9, 10, 11, 12])
+    sub = tf.scatter_nd_sub(ref, indices, updates)
+    with tf.Session() as sess:
+      print sess.run(sub)
+
+The resulting update to ref would look like this:
+
+    [1, -9, 3, -6, -4, 6, 7, -4]
+
+See @{tf.scatter_nd} for more details about how to make updates to
+slices.
+END
+}
+op {
+  graph_op_name: "ScatterNdUpdate"
+  endpoint {
+    name: "ScatterNdUpdate"
+  }
+  summary: "Applies sparse `updates` to individual values or slices within a given"
+  description: <<END
+variable according to `indices`.
+
+`ref` is a `Tensor` with rank `P` and `indices` is a `Tensor` of rank `Q`.
+
+`indices` must be integer tensor, containing indices into `ref`.
+It must be shape `[d_0, ..., d_{Q-2}, K]` where `0 < K <= P`.
+
+The innermost dimension of `indices` (with length `K`) corresponds to
+indices into elements (if `K = P`) or slices (if `K < P`) along the `K`th
+dimension of `ref`.
+
+`updates` is `Tensor` of rank `Q-1+P-K` with shape:
+
+```
+[d_0, ..., d_{Q-2}, ref.shape[K], ..., ref.shape[P-1]].
+```
+
+For example, say we want to update 4 scattered elements to a rank-1 tensor to
+8 elements. In Python, that update would look like this:
+
+```python
+    ref = tf.Variable([1, 2, 3, 4, 5, 6, 7, 8])
+    indices = tf.constant([[4], [3], [1] ,[7]])
+    updates = tf.constant([9, 10, 11, 12])
+    update = tf.scatter_nd_update(ref, indices, updates)
+    with tf.Session() as sess:
+      print sess.run(update)
+```
+
+The resulting update to ref would look like this:
+
+    [1, 11, 3, 10, 9, 6, 7, 12]
+
+See @{tf.scatter_nd} for more details about how to make updates to
+slices.
+END
+}
+op {
+  graph_op_name: "ScatterSub"
+  endpoint {
+    name: "ScatterSub"
+  }
+  summary: "Subtracts sparse updates to a variable reference."
+  description: <<END
+```python
+    # Scalar indices
+    ref[indices, ...] -= updates[...]
+
+    # Vector indices (for each i)
+    ref[indices[i], ...] -= updates[i, ...]
+
+    # High rank indices (for each i, ..., j)
+    ref[indices[i, ..., j], ...] -= updates[i, ..., j, ...]
+```
+
+This operation outputs `ref` after the update is done.
+This makes it easier to chain operations that need to use the reset value.
+
+Duplicate entries are handled correctly: if multiple `indices` reference
+the same location, their (negated) contributions add.
+
+Requires `updates.shape = indices.shape + ref.shape[1:]`.
+
+<div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
+<img style="width:100%" src="https://www.tensorflow.org/images/ScatterSub.png" alt>
+</div>
+END
+}
+op {
+  graph_op_name: "ScatterUpdate"
+  endpoint {
+    name: "ScatterUpdate"
+  }
+  summary: "Applies sparse updates to a variable reference."
+  description: <<END
+This operation computes
+
+```python
+    # Scalar indices
+    ref[indices, ...] = updates[...]
+
+    # Vector indices (for each i)
+    ref[indices[i], ...] = updates[i, ...]
+
+    # High rank indices (for each i, ..., j)
+    ref[indices[i, ..., j], ...] = updates[i, ..., j, ...]
+```
+
+This operation outputs `ref` after the update is done.
+This makes it easier to chain operations that need to use the reset value.
+
+If values in `ref` is to be updated more than once, because there are
+duplicate entries in `indices`, the order at which the updates happen
+for each value is undefined.
+
+Requires `updates.shape = indices.shape + ref.shape[1:]`.
+
+<div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
+<img style="width:100%" src="https://www.tensorflow.org/images/ScatterUpdate.png" alt>
+</div>
+END
+}
+op {
+  graph_op_name: "SdcaFprint"
+  endpoint {
+    name: "SdcaFprint"
+  }
+  summary: "Computes fingerprints of the input strings."
+}
+op {
+  graph_op_name: "SdcaOptimizer"
+  endpoint {
+    name: "SdcaOptimizer"
+  }
+  summary: "Distributed version of Stochastic Dual Coordinate Ascent (SDCA) optimizer for"
+  description: <<END
+linear models with L1 + L2 regularization. As global optimization objective is
+strongly-convex, the optimizer optimizes the dual objective at each step. The
+optimizer applies each update one example at a time. Examples are sampled
+uniformly, and the optimizer is learning rate free and enjoys linear convergence
+rate.
+
+[Proximal Stochastic Dual Coordinate Ascent](http://arxiv.org/pdf/1211.2717v1.pdf).<br>
+Shai Shalev-Shwartz, Tong Zhang. 2012
+
+$$Loss Objective = \sum f_{i} (wx_{i}) + (l2 / 2) * |w|^2 + l1 * |w|$$
+
+[Adding vs. Averaging in Distributed Primal-Dual Optimization](http://arxiv.org/abs/1502.03508).<br>
+Chenxin Ma, Virginia Smith, Martin Jaggi, Michael I. Jordan,
+Peter Richtarik, Martin Takac. 2015
+
+[Stochastic Dual Coordinate Ascent with Adaptive Probabilities](https://arxiv.org/abs/1502.08053).<br>
+Dominik Csiba, Zheng Qu, Peter Richtarik. 2015
+END
+}
+op {
+  graph_op_name: "SdcaShrinkL1"
+  endpoint {
+    name: "SdcaShrinkL1"
+  }
+  summary: "Applies L1 regularization shrink step on the parameters."
+}
+op {
+  graph_op_name: "SegmentMax"
+  endpoint {
+    name: "SegmentMax"
+  }
+  summary: "Computes the maximum along segments of a tensor."
+  description: <<END
+Read @{$math_ops#segmentation$the section on segmentation} for an explanation of
+segments.
+
+Computes a tensor such that
+\\(output_i = \max_j(data_j)\\) where `max` is over `j` such
+that `segment_ids[j] == i`.
+
+If the max is empty for a given segment ID `i`, `output[i] = 0`.
+
+<div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
+<img style="width:100%" src="https://www.tensorflow.org/images/SegmentMax.png" alt>
+</div>
+END
+}
+op {
+  graph_op_name: "SegmentMean"
+  endpoint {
+    name: "SegmentMean"
+  }
+  summary: "Computes the mean along segments of a tensor."
+  description: <<END
+Read @{$math_ops#segmentation$the section on segmentation} for an explanation of
+segments.
+
+Computes a tensor such that
+\\(output_i = \frac{\sum_j data_j}{N}\\) where `mean` is
+over `j` such that `segment_ids[j] == i` and `N` is the total number of
+values summed.
+
+If the mean is empty for a given segment ID `i`, `output[i] = 0`.
+
+<div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
+<img style="width:100%" src="https://www.tensorflow.org/images/SegmentMean.png" alt>
+</div>
+END
+}
+op {
+  graph_op_name: "SegmentMin"
+  endpoint {
+    name: "SegmentMin"
+  }
+  summary: "Computes the minimum along segments of a tensor."
+  description: <<END
+Read @{$math_ops#segmentation$the section on segmentation} for an explanation of
+segments.
+
+Computes a tensor such that
+\\(output_i = \min_j(data_j)\\) where `min` is over `j` such
+that `segment_ids[j] == i`.
+
+If the min is empty for a given segment ID `i`, `output[i] = 0`.
+
+<div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
+<img style="width:100%" src="https://www.tensorflow.org/images/SegmentMin.png" alt>
+</div>
+END
+}
+op {
+  graph_op_name: "SegmentProd"
+  endpoint {
+    name: "SegmentProd"
+  }
+  summary: "Computes the product along segments of a tensor."
+  description: <<END
+Read @{$math_ops#segmentation$the section on segmentation} for an explanation of
+segments.
+
+Computes a tensor such that
+\\(output_i = \prod_j data_j\\) where the product is over `j` such
+that `segment_ids[j] == i`.
+
+If the product is empty for a given segment ID `i`, `output[i] = 1`.
+
+<div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
+<img style="width:100%" src="https://www.tensorflow.org/images/SegmentProd.png" alt>
+</div>
+END
+}
+op {
+  graph_op_name: "SegmentSum"
+  endpoint {
+    name: "SegmentSum"
+  }
+  summary: "Computes the sum along segments of a tensor."
+  description: <<END
+Read @{$math_ops#segmentation$the section on segmentation} for an explanation of
+segments.
+
+Computes a tensor such that
+\\(output_i = \sum_j data_j\\) where sum is over `j` such
+that `segment_ids[j] == i`.
+
+If the sum is empty for a given segment ID `i`, `output[i] = 0`.
+
+<div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
+<img style="width:100%" src="https://www.tensorflow.org/images/SegmentSum.png" alt>
+</div>
+END
+}
+op {
+  graph_op_name: "Select"
+  endpoint {
+    name: "Select"
+  }
+  summary: "Selects elements from `t` or `e`, depending on `condition`."
+  description: <<END
+The `t`, and `e` tensors must all have the same shape, and the
+output will also have that shape.
+
+The `condition` tensor must be a scalar if `t` and `e` are scalars.
+If `t` and `e` are vectors or higher rank, then `condition` must be either a
+scalar, a vector with size matching the first dimension of `t`, or must have
+the same shape as `t`.
+
+The `condition` tensor acts as a mask that chooses, based on the value at each
+element, whether the corresponding element / row in the output should be
+taken from `t` (if true) or `e` (if false).
+
+If `condition` is a vector and `t` and `e` are higher rank matrices, then
+it chooses which row (outer dimension) to copy from `t` and `e`.
+If `condition` has the same shape as `t` and `e`, then it chooses which
+element to copy from `t` and `e`.
+
+For example:
+
+```python
+# 'condition' tensor is [[True,  False]
+#                        [False, True]]
+# 't' is [[1, 2],
+#         [3, 4]]
+# 'e' is [[5, 6],
+#         [7, 8]]
+select(condition, t, e)  # => [[1, 6], [7, 4]]
+
+
+# 'condition' tensor is [True, False]
+# 't' is [[1, 2],
+#         [3, 4]]
+# 'e' is [[5, 6],
+#         [7, 8]]
+select(condition, t, e) ==> [[1, 2],
+                             [7, 8]]
+
+```
+END
+}
+op {
+  graph_op_name: "SelfAdjointEig"
+  endpoint {
+    name: "SelfAdjointEig"
+  }
+  summary: "Computes the Eigen Decomposition of a batch of square self-adjoint matrices."
+  description: <<END
+The input is a tensor of shape `[..., M, M]` whose inner-most 2 dimensions
+form square matrices, with the same constraints as the single matrix
+SelfAdjointEig.
+
+The result is a [..., M+1, M] matrix with [..., 0,:] containing the
+eigenvalues, and subsequent [...,1:, :] containing the eigenvectors.
+END
+}
+op {
+  graph_op_name: "SelfAdjointEigV2"
+  endpoint {
+    name: "SelfAdjointEigV2"
+  }
+  summary: "Computes the eigen decomposition of one or more square self-adjoint matrices."
+  description: <<END
+Computes the eigenvalues and (optionally) eigenvectors of each inner matrix in
+`input` such that `input[..., :, :] = v[..., :, :] * diag(e[..., :])`.
+
+```python
+# a is a tensor.
+# e is a tensor of eigenvalues.
+# v is a tensor of eigenvectors.
+e, v = self_adjoint_eig(a)
+e = self_adjoint_eig(a, compute_v=False)
+```
+END
+}
+op {
+  graph_op_name: "Selu"
+  endpoint {
+    name: "Selu"
+  }
+  summary: "Computes scaled exponential linear: `scale * alpha * (exp(features) - 1)`"
+  description: <<END
+if < 0, `scale * features` otherwise.
+
+See [Self-Normalizing Neural Networks](https://arxiv.org/abs/1706.02515)
+END
+}
+op {
+  graph_op_name: "SeluGrad"
+  endpoint {
+    name: "SeluGrad"
+  }
+  summary: "Computes gradients for the scaled exponential linear (Selu) operation."
+}
+op {
+  graph_op_name: "SerializeManySparse"
+  endpoint {
+    name: "SerializeManySparse"
+  }
+  summary: "Serialize an `N`-minibatch `SparseTensor` into an `[N, 3]` string `Tensor`."
+  description: <<END
+The `SparseTensor` must have rank `R` greater than 1, and the first dimension
+is treated as the minibatch dimension.  Elements of the `SparseTensor`
+must be sorted in increasing order of this first dimension.  The serialized
+`SparseTensor` objects going into each row of `serialized_sparse` will have
+rank `R-1`.
+
+The minibatch size `N` is extracted from `sparse_shape[0]`.
+END
+}
+op {
+  graph_op_name: "SerializeSparse"
+  endpoint {
+    name: "SerializeSparse"
+  }
+  summary: "Serialize a `SparseTensor` into a string 3-vector (1-D `Tensor`) object."
+}
+op {
+  graph_op_name: "SerializeTensor"
+  endpoint {
+    name: "SerializeTensor"
+  }
+  summary: "Transforms a Tensor into a serialized TensorProto proto."
+}
+op {
+  graph_op_name: "SetSize"
+  endpoint {
+    name: "SetSize"
+  }
+  summary: "Number of unique elements along last dimension of input `set`."
+  description: <<END
+Input `set` is a `SparseTensor` represented by `set_indices`, `set_values`,
+and `set_shape`. The last dimension contains values in a set, duplicates are
+allowed but ignored.
+
+If `validate_indices` is `True`, this op validates the order and range of `set`
+indices.
+END
+}
+op {
+  graph_op_name: "Shape"
+  endpoint {
+    name: "Shape"
+  }
+  summary: "Returns the shape of a tensor."
+  description: <<END
+This operation returns a 1-D integer tensor representing the shape of `input`.
+
+For example:
+
+```
+# 't' is [[[1, 1, 1], [2, 2, 2]], [[3, 3, 3], [4, 4, 4]]]
+shape(t) ==> [2, 2, 3]
+```
+END
+}
+op {
+  graph_op_name: "ShapeN"
+  endpoint {
+    name: "ShapeN"
+  }
+  summary: "Returns shape of tensors."
+  description: <<END
+This operation returns N 1-D integer tensors representing shape of `input[i]s`.
+END
+}
+op {
+  graph_op_name: "ShardedFilename"
+  endpoint {
+    name: "ShardedFilename"
+  }
+  summary: "Generate a sharded filename. The filename is printf formatted as"
+  description: <<END
+   %s-%05d-of-%05d, basename, shard, num_shards.
+END
+}
+op {
+  graph_op_name: "ShardedFilespec"
+  endpoint {
+    name: "ShardedFilespec"
+  }
+  summary: "Generate a glob pattern matching all sharded file names."
+}
+op {
+  graph_op_name: "ShuffleDataset"
+  endpoint {
+    name: "ShuffleDataset"
+  }
+  summary: "Creates a dataset that shuffles elements from `input_dataset` pseudorandomly."
+}
+op {
+  graph_op_name: "Sigmoid"
+  endpoint {
+    name: "Sigmoid"
+  }
+  summary: "Computes sigmoid of `x` element-wise."
+  description: <<END
+Specifically, `y = 1 / (1 + exp(-x))`.
+END
+}
+op {
+  graph_op_name: "SigmoidGrad"
+  endpoint {
+    name: "SigmoidGrad"
+  }
+  summary: "Computes the gradient of the sigmoid of `x` wrt its input."
+  description: <<END
+Specifically, `grad = dy * y * (1 - y)`, where `y = sigmoid(x)`, and
+`dy` is the corresponding input gradient.
+END
+}
+op {
+  graph_op_name: "Sign"
+  endpoint {
+    name: "Sign"
+  }
+  summary: "Returns an element-wise indication of the sign of a number."
+  description: <<END
+`y = sign(x) = -1` if `x < 0`; 0 if `x == 0`; 1 if `x > 0`.
+
+For complex numbers, `y = sign(x) = x / |x|` if `x != 0`, otherwise `y = 0`.
+END
+}
+op {
+  graph_op_name: "Sin"
+  endpoint {
+    name: "Sin"
+  }
+  summary: "Computes sin of x element-wise."
+}
+op {
+  graph_op_name: "Sinh"
+  endpoint {
+    name: "Sinh"
+  }
+  summary: "Computes hyperbolic sine of x element-wise."
+}
+op {
+  graph_op_name: "Size"
+  endpoint {
+    name: "Size"
+  }
+  summary: "Returns the size of a tensor."
+  description: <<END
+This operation returns an integer representing the number of elements in
+`input`.
+
+For example:
+
+```
+# 't' is [[[1, 1,, 1], [2, 2, 2]], [[3, 3, 3], [4, 4, 4]]]]
+size(t) ==> 12
+```
+END
+}
+op {
+  graph_op_name: "SkipDataset"
+  endpoint {
+    name: "SkipDataset"
+  }
+  summary: "Creates a dataset that skips `count` elements from the `input_dataset`."
+}
+op {
+  graph_op_name: "Skipgram"
+  endpoint {
+    name: "Skipgram"
+  }
+  summary: "Parses a text file and creates a batch of examples."
+}
+op {
+  graph_op_name: "Slice"
+  endpoint {
+    name: "Slice"
+  }
+  summary: "Return a slice from \'input\'."
+  description: <<END
+The output tensor is a tensor with dimensions described by 'size'
+whose values are extracted from 'input' starting at the offsets in
+'begin'.
+
+*Requirements*:
+  0 <= begin[i] <= begin[i] + size[i] <= Di  for i in [0, n)
+END
+}
+op {
+  graph_op_name: "SloppyInterleaveDataset"
+  endpoint {
+    name: "SloppyInterleaveDataset"
+  }
+  summary: "Creates a dataset that applies `f` to the outputs of `input_dataset`."
+  description: <<END
+The resulting dataset is similar to the `InterleaveDataset`, with the exception
+that if retrieving the next value from a dataset would cause the requester to
+block, it will skip that input dataset. This dataset is especially useful
+when loading data from a variable-latency datastores (e.g. HDFS, GCS), as it
+allows the training step to proceed so long as some data is available.
+
+!! WARNING !! This dataset is not deterministic!
+END
+}
+op {
+  graph_op_name: "Softmax"
+  endpoint {
+    name: "Softmax"
+  }
+  summary: "Computes softmax activations."
+  description: <<END
+For each batch `i` and class `j` we have
+
+    softmax[i, j] = exp(logits[i, j]) / sum_j(exp(logits[i, j]))
+END
+}
+op {
+  graph_op_name: "SoftmaxCrossEntropyWithLogits"
+  endpoint {
+    name: "SoftmaxCrossEntropyWithLogits"
+  }
+  summary: "Computes softmax cross entropy cost and gradients to backpropagate."
+  description: <<END
+Inputs are the logits, not probabilities.
+END
+}
+op {
+  graph_op_name: "Softplus"
+  endpoint {
+    name: "Softplus"
+  }
+  summary: "Computes softplus: `log(exp(features) + 1)`."
+}
+op {
+  graph_op_name: "SoftplusGrad"
+  endpoint {
+    name: "SoftplusGrad"
+  }
+  summary: "Computes softplus gradients for a softplus operation."
+}
+op {
+  graph_op_name: "Softsign"
+  endpoint {
+    name: "Softsign"
+  }
+  summary: "Computes softsign: `features / (abs(features) + 1)`."
+}
+op {
+  graph_op_name: "SoftsignGrad"
+  endpoint {
+    name: "SoftsignGrad"
+  }
+  summary: "Computes softsign gradients for a softsign operation."
+}
+op {
+  graph_op_name: "SpaceToBatch"
+  endpoint {
+    name: "SpaceToBatch"
+  }
+  summary: "SpaceToBatch for 4-D tensors of type T."
+  description: <<END
+This is a legacy version of the more general SpaceToBatchND.
+
+Zero-pads and then rearranges (permutes) blocks of spatial data into batch.
+More specifically, this op outputs a copy of the input tensor where values from
+the `height` and `width` dimensions are moved to the `batch` dimension. After
+the zero-padding, both `height` and `width` of the input must be divisible by the
+block size.
+END
+}
+op {
+  graph_op_name: "SpaceToBatchND"
+  endpoint {
+    name: "SpaceToBatchND"
+  }
+  summary: "SpaceToBatch for N-D tensors of type T."
+  description: <<END
+This operation divides "spatial" dimensions `[1, ..., M]` of the input into a
+grid of blocks of shape `block_shape`, and interleaves these blocks with the
+"batch" dimension (0) such that in the output, the spatial dimensions
+`[1, ..., M]` correspond to the position within the grid, and the batch
+dimension combines both the position within a spatial block and the original
+batch position.  Prior to division into blocks, the spatial dimensions of the
+input are optionally zero padded according to `paddings`.  See below for a
+precise description.
+END
+}
+op {
+  graph_op_name: "SpaceToDepth"
+  endpoint {
+    name: "SpaceToDepth"
+  }
+  summary: "SpaceToDepth for tensors of type T."
+  description: <<END
+Rearranges blocks of spatial data, into depth. More specifically,
+this op outputs a copy of the input tensor where values from the `height`
+and `width` dimensions are moved to the `depth` dimension.
+The attr `block_size` indicates the input block size.
+
+  * Non-overlapping blocks of size `block_size x block size` are rearranged
+    into depth at each location.
+  * The depth of the output tensor is `block_size * block_size * input_depth`.
+  * The Y, X coordinates within each block of the input become the high order
+    component of the output channel index.
+  * The input tensor's height and width must be divisible by block_size.
+
+The `data_format` attr specifies the layout of the input and output tensors
+with the following options:
+  "NHWC": `[ batch, height, width, channels ]`
+  "NCHW": `[ batch, channels, height, width ]`
+  "NCHW_VECT_C":
+      `qint8 [ batch, channels / 4, height, width, channels % 4 ]`
+
+It is useful to consider the operation as transforming a 6-D Tensor.
+e.g. for data_format = NHWC,
+     Each element in the input tensor can be specified via 6 coordinates,
+     ordered by decreasing memory layout significance as:
+     n,oY,bY,oX,bX,iC  (where n=batch index, oX, oY means X or Y coordinates
+                        within the output image, bX, bY means coordinates
+                        within the input block, iC means input channels).
+     The output would be a transpose to the following layout:
+     n,oY,oX,bY,bX,iC
+
+This operation is useful for resizing the activations between convolutions
+(but keeping all data), e.g. instead of pooling. It is also useful for training
+purely convolutional models.
+
+For example, given an input of shape `[1, 2, 2, 1]`, data_format = "NHWC" and
+block_size = 2:
+
+```
+x = [[[[1], [2]],
+      [[3], [4]]]]
+```
+
+This operation will output a tensor of shape `[1, 1, 1, 4]`:
+
+```
+[[[[1, 2, 3, 4]]]]
+```
+
+Here, the input has a batch of 1 and each batch element has shape `[2, 2, 1]`,
+the corresponding output will have a single element (i.e. width and height are
+both 1) and will have a depth of 4 channels (1 * block_size * block_size).
+The output element shape is `[1, 1, 4]`.
+
+For an input tensor with larger depth, here of shape `[1, 2, 2, 3]`, e.g.
+
+```
+x = [[[[1, 2, 3], [4, 5, 6]],
+      [[7, 8, 9], [10, 11, 12]]]]
+```
+
+This operation, for block_size of 2, will return the following tensor of shape
+`[1, 1, 1, 12]`
+
+```
+[[[[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]]]]
+```
+
+Similarly, for the following input of shape `[1 4 4 1]`, and a block size of 2:
+
+```
+x = [[[[1],   [2],  [5],  [6]],
+      [[3],   [4],  [7],  [8]],
+      [[9],  [10], [13],  [14]],
+      [[11], [12], [15],  [16]]]]
+```
+
+the operator will return the following tensor of shape `[1 2 2 4]`:
+
+```
+x = [[[[1, 2, 3, 4],
+       [5, 6, 7, 8]],
+      [[9, 10, 11, 12],
+       [13, 14, 15, 16]]]]
+```
+END
+}
+op {
+  graph_op_name: "SparseAccumulatorApplyGradient"
+  endpoint {
+    name: "SparseAccumulatorApplyGradient"
+  }
+  summary: "Applies a sparse gradient to a given accumulator."
+  description: <<END
+Does not add if local_step is smaller than the accumulator's
+global_step.
+END
+}
+op {
+  graph_op_name: "SparseAccumulatorTakeGradient"
+  endpoint {
+    name: "SparseAccumulatorTakeGradient"
+  }
+  summary: "Extracts the average sparse gradient in a SparseConditionalAccumulator."
+  description: <<END
+The op will blocks until sufficient (i.e., more than num_required)
+gradients have been accumulated. If the accumulator has already
+aggregated more than num_required gradients, it will return its
+average of the accumulated gradients.  Also automatically increments
+the recorded global_step in the accumulator by 1, and resets the
+aggregate to 0.
+END
+}
+op {
+  graph_op_name: "SparseAdd"
+  endpoint {
+    name: "SparseAdd"
+  }
+  summary: "Adds two `SparseTensor` objects to produce another `SparseTensor`."
+  description: <<END
+The input `SparseTensor` objects' indices are assumed ordered in standard
+lexicographic order.  If this is not the case, before this step run
+`SparseReorder` to restore index ordering.
+
+By default, if two values sum to zero at some index, the output `SparseTensor`
+would still include that particular location in its index, storing a zero in the
+corresponding value slot.  To override this, callers can specify `thresh`,
+indicating that if the sum has a magnitude strictly smaller than `thresh`, its
+corresponding value and index would then not be included.  In particular,
+`thresh == 0` (default) means everything is kept and actual thresholding happens
+only for a positive value.
+
+In the following shapes, `nnz` is the count after taking `thresh` into account.
+END
+}
+op {
+  graph_op_name: "SparseAddGrad"
+  endpoint {
+    name: "SparseAddGrad"
+  }
+  summary: "The gradient operator for the SparseAdd op."
+  description: <<END
+The SparseAdd op calculates A + B, where A, B, and the sum are all represented
+as `SparseTensor` objects.  This op takes in the upstream gradient w.r.t.
+non-empty values of the sum, and outputs the gradients w.r.t. the non-empty
+values of A and B.
+END
+}
+op {
+  graph_op_name: "SparseApplyAdadelta"
+  endpoint {
+    name: "SparseApplyAdadelta"
+  }
+  summary: "var: Should be from a Variable()."
+}
+op {
+  graph_op_name: "SparseApplyAdagrad"
+  endpoint {
+    name: "SparseApplyAdagrad"
+  }
+  summary: "Update relevant entries in \'*var\' and \'*accum\' according to the adagrad scheme."
+  description: <<END
+That is for rows we have grad for, we update var and accum as follows:
+accum += grad * grad
+var -= lr * grad * (1 / sqrt(accum))
+END
+}
+op {
+  graph_op_name: "SparseApplyAdagradDA"
+  endpoint {
+    name: "SparseApplyAdagradDA"
+  }
+  summary: "Update entries in \'*var\' and \'*accum\' according to the proximal adagrad scheme."
+}
+op {
+  graph_op_name: "SparseApplyCenteredRMSProp"
+  endpoint {
+    name: "SparseApplyCenteredRMSProp"
+  }
+  summary: "Update \'*var\' according to the centered RMSProp algorithm."
+  description: <<END
+The centered RMSProp algorithm uses an estimate of the centered second moment
+(i.e., the variance) for normalization, as opposed to regular RMSProp, which
+uses the (uncentered) second moment. This often helps with training, but is
+slightly more expensive in terms of computation and memory.
+
+Note that in dense implementation of this algorithm, mg, ms, and mom will
+update even if the grad is zero, but in this sparse implementation, mg, ms,
+and mom will not update in iterations during which the grad is zero.
+
+mean_square = decay * mean_square + (1-decay) * gradient ** 2
+mean_grad = decay * mean_grad + (1-decay) * gradient
+Delta = learning_rate * gradient / sqrt(mean_square + epsilon - mean_grad ** 2)
+
+ms <- rho * ms_{t-1} + (1-rho) * grad * grad
+mom <- momentum * mom_{t-1} + lr * grad / sqrt(ms + epsilon)
+var <- var - mom
+END
+}
+op {
+  graph_op_name: "SparseApplyFtrl"
+  endpoint {
+    name: "SparseApplyFtrl"
+  }
+  summary: "Update relevant entries in \'*var\' according to the Ftrl-proximal scheme."
+  description: <<END
+That is for rows we have grad for, we update var, accum and linear as follows:
+accum_new = accum + grad * grad
+linear += grad + (accum_new^(-lr_power) - accum^(-lr_power)) / lr * var
+quadratic = 1.0 / (accum_new^(lr_power) * lr) + 2 * l2
+var = (sign(linear) * l1 - linear) / quadratic if |linear| > l1 else 0.0
+accum = accum_new
+END
+}
+op {
+  graph_op_name: "SparseApplyFtrlV2"
+  endpoint {
+    name: "SparseApplyFtrlV2"
+  }
+  summary: "Update relevant entries in \'*var\' according to the Ftrl-proximal scheme."
+  description: <<END
+That is for rows we have grad for, we update var, accum and linear as follows:
+grad_with_shrinkage = grad + 2 * l2_shrinkage * var
+accum_new = accum + grad_with_shrinkage * grad_with_shrinkage
+linear += grad_with_shrinkage +
+    (accum_new^(-lr_power) - accum^(-lr_power)) / lr * var
+quadratic = 1.0 / (accum_new^(lr_power) * lr) + 2 * l2
+var = (sign(linear) * l1 - linear) / quadratic if |linear| > l1 else 0.0
+accum = accum_new
+END
+}
+op {
+  graph_op_name: "SparseApplyMomentum"
+  endpoint {
+    name: "SparseApplyMomentum"
+  }
+  summary: "Update relevant entries in \'*var\' and \'*accum\' according to the momentum scheme."
+  description: <<END
+Set use_nesterov = True if you want to use Nesterov momentum.
+
+That is for rows we have grad for, we update var and accum as follows:
+
+accum = accum * momentum + grad
+var -= lr * accum
+END
+}
+op {
+  graph_op_name: "SparseApplyProximalAdagrad"
+  endpoint {
+    name: "SparseApplyProximalAdagrad"
+  }
+  summary: "Sparse update entries in \'*var\' and \'*accum\' according to FOBOS algorithm."
+  description: <<END
+That is for rows we have grad for, we update var and accum as follows:
+accum += grad * grad
+prox_v = var
+prox_v -= lr * grad * (1 / sqrt(accum))
+var = sign(prox_v)/(1+lr*l2) * max{|prox_v|-lr*l1,0}
+END
+}
+op {
+  graph_op_name: "SparseApplyProximalGradientDescent"
+  endpoint {
+    name: "SparseApplyProximalGradientDescent"
+  }
+  summary: "Sparse update \'*var\' as FOBOS algorithm with fixed learning rate."
+  description: <<END
+That is for rows we have grad for, we update var as follows:
+prox_v = var - alpha * grad
+var = sign(prox_v)/(1+alpha*l2) * max{|prox_v|-alpha*l1,0}
+END
+}
+op {
+  graph_op_name: "SparseApplyRMSProp"
+  endpoint {
+    name: "SparseApplyRMSProp"
+  }
+  summary: "Update \'*var\' according to the RMSProp algorithm."
+  description: <<END
+Note that in dense implementation of this algorithm, ms and mom will
+update even if the grad is zero, but in this sparse implementation, ms
+and mom will not update in iterations during which the grad is zero.
+
+mean_square = decay * mean_square + (1-decay) * gradient ** 2
+Delta = learning_rate * gradient / sqrt(mean_square + epsilon)
+
+ms <- rho * ms_{t-1} + (1-rho) * grad * grad
+mom <- momentum * mom_{t-1} + lr * grad / sqrt(ms + epsilon)
+var <- var - mom
+END
+}
+op {
+  graph_op_name: "SparseConcat"
+  endpoint {
+    name: "SparseConcat"
+  }
+  summary: "Concatenates a list of `SparseTensor` along the specified dimension."
+  description: <<END
+Concatenation is with respect to the dense versions of these sparse tensors.
+It is assumed that each input is a `SparseTensor` whose elements are ordered
+along increasing dimension number.
+
+All inputs' shapes must match, except for the concat dimension.  The
+`indices`, `values`, and `shapes` lists must have the same length.
+
+The output shape is identical to the inputs', except along the concat
+dimension, where it is the sum of the inputs' sizes along that dimension.
+
+The output elements will be resorted to preserve the sort order along
+increasing dimension number.
+
+This op runs in `O(M log M)` time, where `M` is the total number of non-empty
+values across all inputs. This is due to the need for an internal sort in
+order to concatenate efficiently across an arbitrary dimension.
+
+For example, if `concat_dim = 1` and the inputs are
+
+    sp_inputs[0]: shape = [2, 3]
+    [0, 2]: "a"
+    [1, 0]: "b"
+    [1, 1]: "c"
+
+    sp_inputs[1]: shape = [2, 4]
+    [0, 1]: "d"
+    [0, 2]: "e"
+
+then the output will be
+
+    shape = [2, 7]
+    [0, 2]: "a"
+    [0, 4]: "d"
+    [0, 5]: "e"
+    [1, 0]: "b"
+    [1, 1]: "c"
+
+Graphically this is equivalent to doing
+
+    [    a] concat [  d e  ] = [    a   d e  ]
+    [b c  ]        [       ]   [b c          ]
+END
+}
+op {
+  graph_op_name: "SparseConditionalAccumulator"
+  endpoint {
+    name: "SparseConditionalAccumulator"
+  }
+  summary: "A conditional accumulator for aggregating sparse gradients."
+  description: <<END
+The accumulator accepts gradients marked with local_step greater or
+equal to the most recent global_step known to the accumulator. The
+average can be extracted from the accumulator, provided sufficient
+gradients have been accumulated. Extracting the average automatically
+resets the aggregate to 0, and increments the global_step recorded by
+the accumulator.
+END
+}
+op {
+  graph_op_name: "SparseCross"
+  endpoint {
+    name: "SparseCross"
+  }
+  summary: "Generates sparse cross from a list of sparse and dense tensors."
+  description: <<END
+The op takes two lists, one of 2D `SparseTensor` and one of 2D `Tensor`, each
+representing features of one feature column. It outputs a 2D `SparseTensor` with
+the batchwise crosses of these features.
+
+For example, if the inputs are
+
+    inputs[0]: SparseTensor with shape = [2, 2]
+    [0, 0]: "a"
+    [1, 0]: "b"
+    [1, 1]: "c"
+
+    inputs[1]: SparseTensor with shape = [2, 1]
+    [0, 0]: "d"
+    [1, 0]: "e"
+
+    inputs[2]: Tensor [["f"], ["g"]]
+
+then the output will be
+
+    shape = [2, 2]
+    [0, 0]: "a_X_d_X_f"
+    [1, 0]: "b_X_e_X_g"
+    [1, 1]: "c_X_e_X_g"
+
+if hashed_output=true then the output will be
+
+    shape = [2, 2]
+    [0, 0]: FingerprintCat64(
+                Fingerprint64("f"), FingerprintCat64(
+                    Fingerprint64("d"), Fingerprint64("a")))
+    [1, 0]: FingerprintCat64(
+                Fingerprint64("g"), FingerprintCat64(
+                    Fingerprint64("e"), Fingerprint64("b")))
+    [1, 1]: FingerprintCat64(
+                Fingerprint64("g"), FingerprintCat64(
+                    Fingerprint64("e"), Fingerprint64("c")))
+END
+}
+op {
+  graph_op_name: "SparseDenseCwiseAdd"
+  endpoint {
+    name: "SparseDenseCwiseAdd"
+  }
+  summary: "Adds up a SparseTensor and a dense Tensor, using these special rules:"
+  description: <<END
+(1) Broadcasts the dense side to have the same shape as the sparse side, if
+    eligible;
+(2) Then, only the dense values pointed to by the indices of the SparseTensor
+    participate in the cwise addition.
+
+By these rules, the result is a logical SparseTensor with exactly the same
+indices and shape, but possibly with different non-zero values.  The output of
+this Op is the resultant non-zero values.
+END
+}
+op {
+  graph_op_name: "SparseDenseCwiseDiv"
+  endpoint {
+    name: "SparseDenseCwiseDiv"
+  }
+  summary: "Component-wise divides a SparseTensor by a dense Tensor."
+  description: <<END
+*Limitation*: this Op only broadcasts the dense side to the sparse side, but not
+the other direction.
+END
+}
+op {
+  graph_op_name: "SparseDenseCwiseMul"
+  endpoint {
+    name: "SparseDenseCwiseMul"
+  }
+  summary: "Component-wise multiplies a SparseTensor by a dense Tensor."
+  description: <<END
+The output locations corresponding to the implicitly zero elements in the sparse
+tensor will be zero (i.e., will not take up storage space), regardless of the
+contents of the dense tensor (even if it's +/-INF and that INF*0 == NaN).
+
+*Limitation*: this Op only broadcasts the dense side to the sparse side, but not
+the other direction.
+END
+}
+op {
+  graph_op_name: "SparseFillEmptyRows"
+  endpoint {
+    name: "SparseFillEmptyRows"
+  }
+  summary: "Fills empty rows in the input 2-D `SparseTensor` with a default value."
+  description: <<END
+The input `SparseTensor` is represented via the tuple of inputs
+(`indices`, `values`, `dense_shape`).  The output `SparseTensor` has the
+same `dense_shape` but with indices `output_indices` and values
+`output_values`.
+
+This op inserts a single entry for every row that doesn't have any values.
+The index is created as `[row, 0, ..., 0]` and the inserted value
+is `default_value`.
+
+For example, suppose `sp_input` has shape `[5, 6]` and non-empty values:
+
+    [0, 1]: a
+    [0, 3]: b
+    [2, 0]: c
+    [3, 1]: d
+
+Rows 1 and 4 are empty, so the output will be of shape `[5, 6]` with values:
+
+    [0, 1]: a
+    [0, 3]: b
+    [1, 0]: default_value
+    [2, 0]: c
+    [3, 1]: d
+    [4, 0]: default_value
+
+The output `SparseTensor` will be in row-major order and will have the
+same shape as the input.
+
+This op also returns an indicator vector shaped `[dense_shape[0]]` such that
+
+    empty_row_indicator[i] = True iff row i was an empty row.
+
+And a reverse index map vector shaped `[indices.shape[0]]` that is used during
+backpropagation,
+
+    reverse_index_map[j] = out_j s.t. indices[j, :] == output_indices[out_j, :]
+END
+}
+op {
+  graph_op_name: "SparseFillEmptyRowsGrad"
+  endpoint {
+    name: "SparseFillEmptyRowsGrad"
+  }
+  summary: "The gradient of SparseFillEmptyRows."
+  description: <<END
+Takes vectors reverse_index_map, shaped `[N]`, and grad_values,
+shaped `[N_full]`, where `N_full >= N` and copies data into either
+`d_values` or `d_default_value`.  Here `d_values` is shaped `[N]` and
+`d_default_value` is a scalar.
+
+  d_values[j] = grad_values[reverse_index_map[j]]
+  d_default_value = sum_{k : 0 .. N_full - 1} (
+     grad_values[k] * 1{k not in reverse_index_map})
+END
+}
+op {
+  graph_op_name: "SparseMatMul"
+  endpoint {
+    name: "SparseMatMul"
+  }
+  summary: "Multiply matrix \"a\" by matrix \"b\"."
+  description: <<END
+The inputs must be two-dimensional matrices and the inner dimension of "a" must
+match the outer dimension of "b". This op is optimized for the case where at
+least one of "a" or "b" is sparse. The breakeven for using this versus a dense
+matrix multiply on one platform was 30% zero values in the sparse matrix.
+
+The gradient computation of this operation will only take advantage of sparsity
+in the input gradient when that gradient comes from a Relu.
+END
+}
+op {
+  graph_op_name: "SparseReduceMax"
+  endpoint {
+    name: "SparseReduceMax"
+  }
+  summary: "Computes the max of elements across dimensions of a SparseTensor."
+  description: <<END
+This Op takes a SparseTensor and is the sparse counterpart to
+`tf.reduce_max()`.  In particular, this Op also returns a dense `Tensor`
+instead of a sparse one.
+
+Reduces `sp_input` along the dimensions given in `reduction_axes`.  Unless
+`keep_dims` is true, the rank of the tensor is reduced by 1 for each entry in
+`reduction_axes`. If `keep_dims` is true, the reduced dimensions are retained
+with length 1.
+
+If `reduction_axes` has no entries, all dimensions are reduced, and a tensor
+with a single element is returned.  Additionally, the axes can be negative,
+which are interpreted according to the indexing rules in Python.
+END
+}
+op {
+  graph_op_name: "SparseReduceMaxSparse"
+  endpoint {
+    name: "SparseReduceMaxSparse"
+  }
+  summary: "Computes the max of elements across dimensions of a SparseTensor."
+  description: <<END
+This Op takes a SparseTensor and is the sparse counterpart to
+`tf.reduce_max()`.  In contrast to SparseReduceMax, this Op returns a
+SparseTensor.
+
+Reduces `sp_input` along the dimensions given in `reduction_axes`.  Unless
+`keep_dims` is true, the rank of the tensor is reduced by 1 for each entry in
+`reduction_axes`. If `keep_dims` is true, the reduced dimensions are retained
+with length 1.
+
+If `reduction_axes` has no entries, all dimensions are reduced, and a tensor
+with a single element is returned.  Additionally, the axes can be negative,
+which are interpreted according to the indexing rules in Python.
+END
+}
+op {
+  graph_op_name: "SparseReduceSum"
+  endpoint {
+    name: "SparseReduceSum"
+  }
+  summary: "Computes the sum of elements across dimensions of a SparseTensor."
+  description: <<END
+This Op takes a SparseTensor and is the sparse counterpart to
+`tf.reduce_sum()`.  In particular, this Op also returns a dense `Tensor`
+instead of a sparse one.
+
+Reduces `sp_input` along the dimensions given in `reduction_axes`.  Unless
+`keep_dims` is true, the rank of the tensor is reduced by 1 for each entry in
+`reduction_axes`. If `keep_dims` is true, the reduced dimensions are retained
+with length 1.
+
+If `reduction_axes` has no entries, all dimensions are reduced, and a tensor
+with a single element is returned.  Additionally, the axes can be negative,
+which are interpreted according to the indexing rules in Python.
+END
+}
+op {
+  graph_op_name: "SparseReduceSumSparse"
+  endpoint {
+    name: "SparseReduceSumSparse"
+  }
+  summary: "Computes the sum of elements across dimensions of a SparseTensor."
+  description: <<END
+This Op takes a SparseTensor and is the sparse counterpart to
+`tf.reduce_sum()`.  In contrast to SparseReduceSum, this Op returns a
+SparseTensor.
+
+Reduces `sp_input` along the dimensions given in `reduction_axes`.  Unless
+`keep_dims` is true, the rank of the tensor is reduced by 1 for each entry in
+`reduction_axes`. If `keep_dims` is true, the reduced dimensions are retained
+with length 1.
+
+If `reduction_axes` has no entries, all dimensions are reduced, and a tensor
+with a single element is returned.  Additionally, the axes can be negative,
+which are interpreted according to the indexing rules in Python.
+END
+}
+op {
+  graph_op_name: "SparseReorder"
+  endpoint {
+    name: "SparseReorder"
+  }
+  summary: "Reorders a SparseTensor into the canonical, row-major ordering."
+  description: <<END
+Note that by convention, all sparse ops preserve the canonical ordering along
+increasing dimension number. The only time ordering can be violated is during
+manual manipulation of the indices and values vectors to add entries.
+
+Reordering does not affect the shape of the SparseTensor.
+
+If the tensor has rank `R` and `N` non-empty values, `input_indices` has
+shape `[N, R]`, input_values has length `N`, and input_shape has length `R`.
+END
+}
+op {
+  graph_op_name: "SparseReshape"
+  endpoint {
+    name: "SparseReshape"
+  }
+  summary: "Reshapes a SparseTensor to represent values in a new dense shape."
+  description: <<END
+This operation has the same semantics as reshape on the represented dense
+tensor.  The `input_indices` are recomputed based on the requested `new_shape`.
+
+If one component of `new_shape` is the special value -1, the size of that
+dimension is computed so that the total dense size remains constant.  At
+most one component of `new_shape` can be -1.  The number of dense elements
+implied by `new_shape` must be the same as the number of dense elements
+originally implied by `input_shape`.
+
+Reshaping does not affect the order of values in the SparseTensor.
+
+If the input tensor has rank `R_in` and `N` non-empty values, and `new_shape`
+has length `R_out`, then `input_indices` has shape `[N, R_in]`,
+`input_shape` has length `R_in`, `output_indices` has shape `[N, R_out]`, and
+`output_shape` has length `R_out`.
+END
+}
+op {
+  graph_op_name: "SparseSegmentMean"
+  endpoint {
+    name: "SparseSegmentMean"
+  }
+  summary: "Computes the mean along sparse segments of a tensor."
+  description: <<END
+Read @{$math_ops#segmentation$the section on segmentation} for an explanation of
+segments.
+
+Like `SegmentMean`, but `segment_ids` can have rank less than `data`'s first
+dimension, selecting a subset of dimension 0, specified by `indices`.
+END
+}
+op {
+  graph_op_name: "SparseSegmentMeanGrad"
+  endpoint {
+    name: "SparseSegmentMeanGrad"
+  }
+  summary: "Computes gradients for SparseSegmentMean."
+  description: <<END
+Returns tensor "output" with same shape as grad, except for dimension 0 whose
+value is output_dim0.
+END
+}
+op {
+  graph_op_name: "SparseSegmentSqrtN"
+  endpoint {
+    name: "SparseSegmentSqrtN"
+  }
+  summary: "Computes the sum along sparse segments of a tensor divided by the sqrt of N."
+  description: <<END
+N is the size of the segment being reduced.
+
+Read @{$math_ops#segmentation$the section on segmentation} for an explanation of
+segments.
+END
+}
+op {
+  graph_op_name: "SparseSegmentSqrtNGrad"
+  endpoint {
+    name: "SparseSegmentSqrtNGrad"
+  }
+  summary: "Computes gradients for SparseSegmentSqrtN."
+  description: <<END
+Returns tensor "output" with same shape as grad, except for dimension 0 whose
+value is output_dim0.
+END
+}
+op {
+  graph_op_name: "SparseSegmentSum"
+  endpoint {
+    name: "SparseSegmentSum"
+  }
+  summary: "Computes the sum along sparse segments of a tensor."
+  description: <<END
+Read @{$math_ops#segmentation$the section on segmentation} for an explanation of
+segments.
+
+Like `SegmentSum`, but `segment_ids` can have rank less than `data`'s first
+dimension, selecting a subset of dimension 0, specified by `indices`.
+
+For example:
+
+```python
+c = tf.constant([[1,2,3,4], [-1,-2,-3,-4], [5,6,7,8]])
+
+# Select two rows, one segment.
+tf.sparse_segment_sum(c, tf.constant([0, 1]), tf.constant([0, 0]))
+# => [[0 0 0 0]]
+
+# Select two rows, two segment.
+tf.sparse_segment_sum(c, tf.constant([0, 1]), tf.constant([0, 1]))
+# => [[ 1  2  3  4]
+#     [-1 -2 -3 -4]]
+
+# Select all rows, two segments.
+tf.sparse_segment_sum(c, tf.constant([0, 1, 2]), tf.constant([0, 0, 1]))
+# => [[0 0 0 0]
+#     [5 6 7 8]]
+
+# Which is equivalent to:
+tf.segment_sum(c, tf.constant([0, 0, 1]))
+```
+END
+}
+op {
+  graph_op_name: "SparseSlice"
+  endpoint {
+    name: "SparseSlice"
+  }
+  summary: "Slice a `SparseTensor` based on the `start` and `size`."
+  description: <<END
+For example, if the input is
+
+    input_tensor = shape = [2, 7]
+    [    a   d e  ]
+    [b c          ]
+
+Graphically the output tensors are:
+
+    sparse_slice([0, 0], [2, 4]) = shape = [2, 4]
+    [    a  ]
+    [b c    ]
+
+    sparse_slice([0, 4], [2, 3]) = shape = [2, 3]
+    [ d e  ]
+    [      ]
+END
+}
+op {
+  graph_op_name: "SparseSoftmax"
+  endpoint {
+    name: "SparseSoftmax"
+  }
+  summary: "Applies softmax to a batched N-D `SparseTensor`."
+  description: <<END
+The inputs represent an N-D SparseTensor  with logical shape `[..., B, C]`
+(where `N >= 2`), and with indices sorted in the canonical lexicographic order.
+
+This op is equivalent to applying the normal `tf.nn.softmax()` to each innermost
+logical submatrix with shape `[B, C]`, but with the catch that *the implicitly
+zero elements do not participate*.  Specifically, the algorithm is equivalent
+to the following:
+
+  (1) Applies `tf.nn.softmax()` to a densified view of each innermost submatrix
+      with shape `[B, C]`, along the size-C dimension;
+  (2) Masks out the original implicitly-zero locations;
+  (3) Renormalizes the remaining elements.
+
+Hence, the `SparseTensor` result has exactly the same non-zero indices and
+shape.
+END
+}
+op {
+  graph_op_name: "SparseSoftmaxCrossEntropyWithLogits"
+  endpoint {
+    name: "SparseSoftmaxCrossEntropyWithLogits"
+  }
+  summary: "Computes softmax cross entropy cost and gradients to backpropagate."
+  description: <<END
+Unlike `SoftmaxCrossEntropyWithLogits`, this operation does not accept
+a matrix of label probabilities, but rather a single label per row
+of features.  This label is considered to have probability 1.0 for the
+given row.
+
+Inputs are the logits, not probabilities.
+END
+}
+op {
+  graph_op_name: "SparseSparseMaximum"
+  endpoint {
+    name: "SparseSparseMaximum"
+  }
+  summary: "Returns the element-wise max of two SparseTensors."
+  description: <<END
+Assumes the two SparseTensors have the same shape, i.e., no broadcasting.
+END
+}
+op {
+  graph_op_name: "SparseSparseMinimum"
+  endpoint {
+    name: "SparseSparseMinimum"
+  }
+  summary: "Returns the element-wise min of two SparseTensors."
+  description: <<END
+Assumes the two SparseTensors have the same shape, i.e., no broadcasting.
+END
+}
+op {
+  graph_op_name: "SparseSplit"
+  endpoint {
+    name: "SparseSplit"
+  }
+  summary: "Split a `SparseTensor` into `num_split` tensors along one dimension."
+  description: <<END
+If the `shape[split_dim]` is not an integer multiple of `num_split`. Slices
+`[0 : shape[split_dim] % num_split]` gets one extra dimension.
+For example, if `split_dim = 1` and `num_split = 2` and the input is
+
+    input_tensor = shape = [2, 7]
+    [    a   d e  ]
+    [b c          ]
+
+Graphically the output tensors are:
+
+    output_tensor[0] = shape = [2, 4]
+    [    a  ]
+    [b c    ]
+
+    output_tensor[1] = shape = [2, 3]
+    [ d e  ]
+    [      ]
+END
+}
+op {
+  graph_op_name: "SparseTensorDenseAdd"
+  endpoint {
+    name: "SparseTensorDenseAdd"
+  }
+  summary: "Adds up a `SparseTensor` and a dense `Tensor`, producing a dense `Tensor`."
+  description: <<END
+This Op does not require `a_indices` be sorted in standard lexicographic order.
+END
+}
+op {
+  graph_op_name: "SparseTensorDenseMatMul"
+  endpoint {
+    name: "SparseTensorDenseMatMul"
+  }
+  summary: "Multiply SparseTensor (of rank 2) \"A\" by dense matrix \"B\"."
+  description: <<END
+No validity checking is performed on the indices of A.  However, the following
+input format is recommended for optimal behavior:
+
+if adjoint_a == false:
+  A should be sorted in lexicographically increasing order.  Use SparseReorder
+  if you're not sure.
+if adjoint_a == true:
+  A should be sorted in order of increasing dimension 1 (i.e., "column major"
+  order instead of "row major" order).
+END
+}
+op {
+  graph_op_name: "SparseTensorSliceDataset"
+  endpoint {
+    name: "SparseTensorSliceDataset"
+  }
+  summary: "Creates a dataset that splits a SparseTensor into elements row-wise."
+}
+op {
+  graph_op_name: "SparseToDense"
+  endpoint {
+    name: "SparseToDense"
+  }
+  summary: "Converts a sparse representation into a dense tensor."
+  description: <<END
+Builds an array `dense` with shape `output_shape` such that
+
+```
+# If sparse_indices is scalar
+dense[i] = (i == sparse_indices ? sparse_values : default_value)
+
+# If sparse_indices is a vector, then for each i
+dense[sparse_indices[i]] = sparse_values[i]
+
+# If sparse_indices is an n by d matrix, then for each i in [0, n)
+dense[sparse_indices[i][0], ..., sparse_indices[i][d-1]] = sparse_values[i]
+```
+
+All other values in `dense` are set to `default_value`.  If `sparse_values` is a
+scalar, all sparse indices are set to this single value.
+
+Indices should be sorted in lexicographic order, and indices must not
+contain any repeats. If `validate_indices` is true, these properties
+are checked during execution.
+END
+}
+op {
+  graph_op_name: "SparseToSparseSetOperation"
+  endpoint {
+    name: "SparseToSparseSetOperation"
+  }
+  summary: "Applies set operation along last dimension of 2 `SparseTensor` inputs."
+  description: <<END
+See SetOperationOp::SetOperationFromContext for values of `set_operation`.
+
+If `validate_indices` is `True`, `SparseToSparseSetOperation` validates the
+order and range of `set1` and `set2` indices.
+
+Input `set1` is a `SparseTensor` represented by `set1_indices`, `set1_values`,
+and `set1_shape`. For `set1` ranked `n`, 1st `n-1` dimensions must be the same
+as `set2`. Dimension `n` contains values in a set, duplicates are allowed but
+ignored.
+
+Input `set2` is a `SparseTensor` represented by `set2_indices`, `set2_values`,
+and `set2_shape`. For `set2` ranked `n`, 1st `n-1` dimensions must be the same
+as `set1`. Dimension `n` contains values in a set, duplicates are allowed but
+ignored.
+
+If `validate_indices` is `True`, this op validates the order and range of `set1`
+and `set2` indices.
+
+Output `result` is a `SparseTensor` represented by `result_indices`,
+`result_values`, and `result_shape`. For `set1` and `set2` ranked `n`, this
+has rank `n` and the same 1st `n-1` dimensions as `set1` and `set2`. The `nth`
+dimension contains the result of `set_operation` applied to the corresponding
+`[0...n-1]` dimension of `set`.
+END
+}
+op {
+  graph_op_name: "Split"
+  endpoint {
+    name: "Split"
+  }
+  summary: "Splits a tensor into `num_split` tensors along one dimension."
+}
+op {
+  graph_op_name: "SplitV"
+  endpoint {
+    name: "SplitV"
+  }
+  summary: "Splits a tensor into `num_split` tensors along one dimension."
+}
+op {
+  graph_op_name: "SqlDataset"
+  endpoint {
+    name: "SqlDataset"
+  }
+  summary: "Creates a dataset that executes a SQL query and emits rows of the result set."
+}
+op {
+  graph_op_name: "Sqrt"
+  endpoint {
+    name: "Sqrt"
+  }
+  summary: "Computes square root of x element-wise."
+  description: <<END
+I.e., \\(y = \sqrt{x} = x^{1/2}\\).
+END
+}
+op {
+  graph_op_name: "SqrtGrad"
+  endpoint {
+    name: "SqrtGrad"
+  }
+  summary: "Computes the gradient for the sqrt of `x` wrt its input."
+  description: <<END
+Specifically, `grad = dy * 0.5 / y`, where `y = sqrt(x)`, and `dy`
+is the corresponding input gradient.
+END
+}
+op {
+  graph_op_name: "Square"
+  endpoint {
+    name: "Square"
+  }
+  summary: "Computes square of x element-wise."
+  description: <<END
+I.e., \\(y = x * x = x^2\\).
+END
+}
+op {
+  graph_op_name: "SquaredDifference"
+  endpoint {
+    name: "SquaredDifference"
+  }
+  summary: "Returns (x - y)(x - y) element-wise."
+  description: <<END
+*NOTE*: `SquaredDifference` supports broadcasting. More about broadcasting
+[here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
+END
+}
+op {
+  graph_op_name: "Squeeze"
+  endpoint {
+    name: "Squeeze"
+  }
+  summary: "Removes dimensions of size 1 from the shape of a tensor."
+  description: <<END
+Given a tensor `input`, this operation returns a tensor of the same type with
+all dimensions of size 1 removed. If you don't want to remove all size 1
+dimensions, you can remove specific size 1 dimensions by specifying
+`squeeze_dims`.
+
+For example:
+
+```
+# 't' is a tensor of shape [1, 2, 1, 3, 1, 1]
+shape(squeeze(t)) ==> [2, 3]
+```
+
+Or, to remove specific size 1 dimensions:
+
+```
+# 't' is a tensor of shape [1, 2, 1, 3, 1, 1]
+shape(squeeze(t, [2, 4])) ==> [1, 2, 3, 1]
+```
+END
+}
+op {
+  graph_op_name: "Stack"
+  endpoint {
+    name: "Stack"
+  }
+  summary: "Deprecated, use StackV2."
+}
+op {
+  graph_op_name: "StackClose"
+  endpoint {
+    name: "StackClose"
+  }
+  summary: "Deprecated, use StackCloseV2."
+}
+op {
+  graph_op_name: "StackCloseV2"
+  endpoint {
+    name: "StackCloseV2"
+  }
+  summary: "Delete the stack from its resource container."
+}
+op {
+  graph_op_name: "StackPop"
+  endpoint {
+    name: "StackPop"
+  }
+  summary: "Deprecated, use StackPopV2."
+}
+op {
+  graph_op_name: "StackPopV2"
+  endpoint {
+    name: "StackPopV2"
+  }
+  summary: "Pop the element at the top of the stack."
+}
+op {
+  graph_op_name: "StackPush"
+  endpoint {
+    name: "StackPush"
+  }
+  summary: "Deprecated, use StackPushV2."
+}
+op {
+  graph_op_name: "StackPushV2"
+  endpoint {
+    name: "StackPushV2"
+  }
+  summary: "Push an element onto the stack."
+}
+op {
+  graph_op_name: "StackV2"
+  endpoint {
+    name: "StackV2"
+  }
+  summary: "A stack that produces elements in first-in last-out order."
+}
+op {
+  graph_op_name: "Stage"
+  endpoint {
+    name: "Stage"
+  }
+  summary: "Stage values similar to a lightweight Enqueue."
+  description: <<END
+The basic functionality of this Op is similar to a queue with many
+fewer capabilities and options.  This Op is optimized for performance.
+END
+}
+op {
+  graph_op_name: "StageClear"
+  endpoint {
+    name: "StageClear"
+  }
+  summary: "Op removes all elements in the underlying container."
+}
+op {
+  graph_op_name: "StagePeek"
+  endpoint {
+    name: "StagePeek"
+  }
+  summary: "Op peeks at the values at the specified index.  If the"
+  description: <<END
+underlying container does not contain sufficient elements
+this op will block until it does.   This Op is optimized for
+performance.
+END
+}
+op {
+  graph_op_name: "StageSize"
+  endpoint {
+    name: "StageSize"
+  }
+  summary: "Op returns the number of elements in the underlying container."
+}
+op {
+  graph_op_name: "StatelessRandomNormal"
+  endpoint {
+    name: "StatelessRandomNormal"
+  }
+  summary: "Outputs deterministic pseudorandom values from a normal distribution."
+  description: <<END
+The generated values will have mean 0 and standard deviation 1.
+
+The outputs are a deterministic function of `shape` and `seed`.
+END
+}
+op {
+  graph_op_name: "StatelessRandomUniform"
+  endpoint {
+    name: "StatelessRandomUniform"
+  }
+  summary: "Outputs deterministic pseudorandom random values from a uniform distribution."
+  description: <<END
+The generated values follow a uniform distribution in the range `[0, 1)`. The
+lower bound 0 is included in the range, while the upper bound 1 is excluded.
+
+The outputs are a deterministic function of `shape` and `seed`.
+END
+}
+op {
+  graph_op_name: "StatelessTruncatedNormal"
+  endpoint {
+    name: "StatelessTruncatedNormal"
+  }
+  summary: "Outputs deterministic pseudorandom values from a truncated normal distribution."
+  description: <<END
+The generated values follow a normal distribution with mean 0 and standard
+deviation 1, except that values whose magnitude is more than 2 standard
+deviations from the mean are dropped and re-picked.
+
+The outputs are a deterministic function of `shape` and `seed`.
+END
+}
+op {
+  graph_op_name: "StopGradient"
+  endpoint {
+    name: "StopGradient"
+  }
+  summary: "Stops gradient computation."
+  description: <<END
+When executed in a graph, this op outputs its input tensor as-is.
+
+When building ops to compute gradients, this op prevents the contribution of
+its inputs to be taken into account.  Normally, the gradient generator adds ops
+to a graph to compute the derivatives of a specified 'loss' by recursively
+finding out inputs that contributed to its computation.  If you insert this op
+in the graph it inputs are masked from the gradient generator.  They are not
+taken into account for computing gradients.
+
+This is useful any time you want to compute a value with TensorFlow but need
+to pretend that the value was a constant. Some examples include:
+
+*  The *EM* algorithm where the *M-step* should not involve backpropagation
+   through the output of the *E-step*.
+*  Contrastive divergence training of Boltzmann machines where, when
+   differentiating the energy function, the training must not backpropagate
+   through the graph that generated the samples from the model.
+*  Adversarial training, where no backprop should happen through the adversarial
+   example generation process.
+END
+}
+op {
+  graph_op_name: "StridedSlice"
+  endpoint {
+    name: "StridedSlice"
+  }
+  summary: "Return a strided slice from `input`."
+  description: <<END
+Note, most python users will want to use the Python `Tensor.__getitem__`
+or `Variable.__getitem__` rather than this op directly.
+
+The goal of this op is to produce a new tensor with a subset of
+the elements from the `n` dimensional `input` tensor. The subset is chosen using
+a sequence of `m` sparse range specifications encoded into the arguments
+of this function. Note, in some cases
+`m` could be equal to `n`, but this need not be the case. Each
+range specification entry can be one of the following:
+
+- An ellipsis (...). Ellipses are used to imply zero or more
+  dimensions of full-dimension selection and are produced using
+  `ellipsis_mask`. For example, `foo[...]` is the identity slice.
+
+- A new axis. This is used to insert a new shape=1 dimension and is
+  produced using `new_axis_mask`. For example, `foo[:, ...]` where
+  `foo` is shape `(3, 4)` produces a `(1, 3, 4)` tensor.
+
+
+- A range `begin:end:stride`. This is used to specify how much to choose from
+  a given dimension. `stride` can be any integer but 0.  `begin` is an integer
+  which represents the index of the first value to select while `end` represents
+  the index of the last value to select. The number of values selected in each
+  dimension is `end - begin` if `stride > 0` and `begin - end` if `stride < 0`.
+  `begin` and `end` can be negative where `-1` is the last element, `-2` is
+  the second to last. `begin_mask` controls whether to replace the explicitly
+  given `begin` with an implicit effective value of `0` if `stride > 0` and
+  `-1` if `stride < 0`. `end_mask` is analogous but produces the number
+  required to create the largest open interval. For example, given a shape
+  `(3,)` tensor `foo[:]`, the effective `begin` and `end` are `0` and `3`. Do
+  not assume this is equivalent to `foo[0:-1]` which has an effective `begin`
+  and `end` of `0` and `2`. Another example is `foo[-2::-1]` which reverses the
+  first dimension of a tensor while dropping the last two (in the original
+  order elements). For example `foo = [1,2,3,4]; foo[-2::-1]` is `[4,3]`.
+
+- A single index. This is used to keep only elements that have a given
+  index. For example (`foo[2, :]` on a shape `(5,6)` tensor produces a
+  shape `(6,)` tensor. This is encoded in `begin` and `end` and
+  `shrink_axis_mask`.
+
+Each conceptual range specification is encoded in the op's argument. This
+encoding is best understand by considering a non-trivial example. In
+particular,
+`foo[1, 2:4, None, ..., :-3:-1, :]` will be encoded as
+
+```
+begin = [1, 2, x, x, 0, x] # x denotes don't care (usually 0)
+end = [2, 4, x, x, -3, x]
+strides = [1, 1, x, x, -1, 1]
+begin_mask = 1<<4 | 1 << 5 = 48
+end_mask = 1<<5 = 32
+ellipsis_mask = 1<<3 = 8
+new_axis_mask = 1<<2 4
+shrink_axis_mask = 1<<0
+```
+
+In this case if `foo.shape` is (5, 5, 5, 5, 5, 5) the final shape of
+the slice becomes (2, 1, 5, 5, 2, 5).
+Let us walk step by step through each argument specification.
+
+1.  The first argument in the example slice is turned into `begin = 1` and
+`end = begin + 1 = 2`. To disambiguate from the original spec `2:4` we
+also set the appropriate bit in `shrink_axis_mask`.
+
+2. `2:4` is contributes 2, 4, 1 to begin, end, and stride. All masks have
+zero bits contributed.
+
+3. None is a synonym for `tf.newaxis`. This means insert a dimension of size 1
+dimension in the final shape. Dummy values are contributed to begin,
+end and stride, while the new_axis_mask bit is set.
+
+4. `...` grab the full ranges from as many dimensions as needed to
+fully specify a slice for every dimension of the input shape.
+
+5. `:-3:-1` shows the use of negative indices. A negative index `i` associated
+with a dimension that has shape `s` is converted to a positive index
+`s + i`. So `-1` becomes `s-1` (i.e. the last element). This conversion
+is done internally so begin, end and strides receive x, -3, and -1.
+The appropriate begin_mask bit is set to indicate the start range is the
+full range (ignoring the x).
+
+6. `:` indicates that the entire contents of the corresponding dimension
+is selected. This is equivalent to `::` or `0::1`. begin, end, and strides
+receive 0, 0, and 1, respectively. The appropriate bits in `begin_mask` and
+`end_mask` are also set.
+
+*Requirements*:
+  `0 != strides[i] for i in [0, m)`
+  `ellipsis_mask must be a power of two (only one ellipsis)`
+END
+}
+op {
+  graph_op_name: "StridedSliceAssign"
+  endpoint {
+    name: "StridedSliceAssign"
+  }
+  summary: "Assign `value` to the sliced l-value reference of `ref`."
+  description: <<END
+The values of `value` are assigned to the positions in the variable
+`ref` that are selected by the slice parameters. The slice parameters
+`begin, `end`, `strides`, etc. work exactly as in `StridedSlice`.
+
+NOTE this op currently does not support broadcasting and so `value`'s
+shape must be exactly the shape produced by the slice of `ref`.
+END
+}
+op {
+  graph_op_name: "StridedSliceGrad"
+  endpoint {
+    name: "StridedSliceGrad"
+  }
+  summary: "Returns the gradient of `StridedSlice`."
+  description: <<END
+Since `StridedSlice` cuts out pieces of its `input` which is size
+`shape`, its gradient will have the same shape (which is passed here
+as `shape`). The gradient will be zero in any element that the slice
+does not select.
+
+Arguments are the same as StridedSliceGrad with the exception that
+`dy` is the input gradient to be propagated and `shape` is the
+shape of `StridedSlice`'s `input`.
+END
+}
+op {
+  graph_op_name: "StringJoin"
+  endpoint {
+    name: "StringJoin"
+  }
+  summary: "Joins the strings in the given list of string tensors into one tensor;"
+  description: <<END
+with the given separator (default is an empty separator).
+END
+}
+op {
+  graph_op_name: "StringSplit"
+  endpoint {
+    name: "StringSplit"
+  }
+  summary: "Split elements of `input` based on `delimiter` into a `SparseTensor`."
+  description: <<END
+Let N be the size of source (typically N will be the batch size). Split each
+element of `input` based on `delimiter` and return a `SparseTensor`
+containing the splitted tokens. Empty tokens are ignored.
+
+`delimiter` can be empty, or a string of split characters. If `delimiter` is an
+ empty string, each element of `input` is split into individual single-byte
+ character strings, including splitting of UTF-8 multibyte sequences. Otherwise
+ every character of `delimiter` is a potential split point.
+
+For example:
+  N = 2, input[0] is 'hello world' and input[1] is 'a b c', then the output
+  will be
+
+  indices = [0, 0;
+             0, 1;
+             1, 0;
+             1, 1;
+             1, 2]
+  shape = [2, 3]
+  values = ['hello', 'world', 'a', 'b', 'c']
+END
+}
+op {
+  graph_op_name: "StringToHashBucket"
+  endpoint {
+    name: "StringToHashBucket"
+  }
+  summary: "Converts each string in the input Tensor to its hash mod by a number of buckets."
+  description: <<END
+The hash function is deterministic on the content of the string within the
+process.
+
+Note that the hash function may change from time to time.
+This functionality will be deprecated and it's recommended to use
+`tf.string_to_hash_bucket_fast()` or `tf.string_to_hash_bucket_strong()`.
+END
+}
+op {
+  graph_op_name: "StringToHashBucketFast"
+  endpoint {
+    name: "StringToHashBucketFast"
+  }
+  summary: "Converts each string in the input Tensor to its hash mod by a number of buckets."
+  description: <<END
+The hash function is deterministic on the content of the string within the
+process and will never change. However, it is not suitable for cryptography.
+This function may be used when CPU time is scarce and inputs are trusted or
+unimportant. There is a risk of adversaries constructing inputs that all hash
+to the same bucket. To prevent this problem, use a strong hash function with
+`tf.string_to_hash_bucket_strong`.
+END
+}
+op {
+  graph_op_name: "StringToHashBucketStrong"
+  endpoint {
+    name: "StringToHashBucketStrong"
+  }
+  summary: "Converts each string in the input Tensor to its hash mod by a number of buckets."
+  description: <<END
+The hash function is deterministic on the content of the string within the
+process. The hash function is a keyed hash function, where attribute `key`
+defines the key of the hash function. `key` is an array of 2 elements.
+
+A strong hash is important when inputs may be malicious, e.g. URLs with
+additional components. Adversaries could try to make their inputs hash to the
+same bucket for a denial-of-service attack or to skew the results. A strong
+hash prevents this by making it difficult, if not infeasible, to compute inputs
+that hash to the same bucket. This comes at a cost of roughly 4x higher compute
+time than `tf.string_to_hash_bucket_fast`.
+END
+}
+op {
+  graph_op_name: "StringToNumber"
+  endpoint {
+    name: "StringToNumber"
+  }
+  summary: "Converts each string in the input Tensor to the specified numeric type."
+  description: <<END
+(Note that int32 overflow results in an error while float overflow
+results in a rounded value.)
+END
+}
+op {
+  graph_op_name: "Sub"
+  endpoint {
+    name: "Sub"
+  }
+  summary: "Returns x - y element-wise."
+  description: <<END
+*NOTE*: `Sub` supports broadcasting. More about broadcasting
+[here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
+END
+}
+op {
+  graph_op_name: "Substr"
+  endpoint {
+    name: "Substr"
+  }
+  summary: "Return substrings from `Tensor` of strings."
+  description: <<END
+For each string in the input `Tensor`, creates a substring starting at index
+`pos` with a total length of `len`.
+
+If `len` defines a substring that would extend beyond the length of the input
+string, then as many characters as possible are used.
+
+If `pos` is negative or specifies a character index larger than any of the input
+strings, then an `InvalidArgumentError` is thrown.
+
+`pos` and `len` must have the same shape, otherwise a `ValueError` is thrown on
+Op creation.
+
+*NOTE*: `Substr` supports broadcasting up to two dimensions. More about
+broadcasting
+[here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
+
+---
+
+Examples
+
+Using scalar `pos` and `len`:
+
+```python
+input = [b'Hello', b'World']
+position = 1
+length = 3
+
+output = [b'ell', b'orl']
+```
+
+Using `pos` and `len` with same shape as `input`:
+
+```python
+input = [[b'ten', b'eleven', b'twelve'],
+         [b'thirteen', b'fourteen', b'fifteen'],
+         [b'sixteen', b'seventeen', b'eighteen']]
+position = [[1, 2, 3],
+            [1, 2, 3],
+            [1, 2, 3]]
+length =   [[2, 3, 4],
+            [4, 3, 2],
+            [5, 5, 5]]
+
+output = [[b'en', b'eve', b'lve'],
+          [b'hirt', b'urt', b'te'],
+          [b'ixtee', b'vente', b'hteen']]
+```
+
+Broadcasting `pos` and `len` onto `input`:
+
+```
+input = [[b'ten', b'eleven', b'twelve'],
+         [b'thirteen', b'fourteen', b'fifteen'],
+         [b'sixteen', b'seventeen', b'eighteen'],
+         [b'nineteen', b'twenty', b'twentyone']]
+position = [1, 2, 3]
+length =   [1, 2, 3]
+
+output = [[b'e', b'ev', b'lve'],
+          [b'h', b'ur', b'tee'],
+          [b'i', b've', b'hte'],
+          [b'i', b'en', b'nty']]
+```
+
+Broadcasting `input` onto `pos` and `len`:
+
+```
+input = b'thirteen'
+position = [1, 5, 7]
+length =   [3, 2, 1]
+
+output = [b'hir', b'ee', b'n']
+```
+END
+}
+op {
+  graph_op_name: "Sum"
+  endpoint {
+    name: "Sum"
+  }
+  summary: "Computes the sum of elements across dimensions of a tensor."
+  description: <<END
+Reduces `input` along the dimensions given in `reduction_indices`. Unless
+`keep_dims` is true, the rank of the tensor is reduced by 1 for each entry in
+`reduction_indices`. If `keep_dims` is true, the reduced dimensions are
+retained with length 1.
+END
+}
+op {
+  graph_op_name: "Svd"
+  endpoint {
+    name: "Svd"
+  }
+  summary: "Computes the singular value decompositions of one or more matrices."
+  description: <<END
+Computes the SVD of each inner matrix in `input` such that
+`input[..., :, :] = u[..., :, :] * diag(s[..., :, :]) * transpose(v[..., :, :])`
+
+```python
+# a is a tensor containing a batch of matrices.
+# s is a tensor of singular values for each matrix.
+# u is the tensor containing of left singular vectors for each matrix.
+# v is the tensor containing of right singular vectors for each matrix.
+s, u, v = svd(a)
+s, _, _ = svd(a, compute_uv=False)
+```
+END
+}
+op {
+  graph_op_name: "Switch"
+  endpoint {
+    name: "Switch"
+  }
+  summary: "Forwards `data` to the output port determined by `pred`."
+  description: <<END
+If `pred` is true, the `data` input is forwarded to `output_true`. Otherwise,
+the data goes to `output_false`.
+
+See also `RefSwitch` and `Merge`.
+END
+}
+op {
+  graph_op_name: "SymbolicGradient"
+  endpoint {
+    name: "SymbolicGradient"
+  }
+  summary: "Computes the gradient function for function f via backpropagation."
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_T.pbtxt b/tensorflow/core/api_def/base_api/api_def_T.pbtxt
new file mode 100644
index 0000000000..8d1cbbcc06
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_T.pbtxt
@@ -0,0 +1,619 @@
+op {
+  graph_op_name: "TFRecordDataset"
+  endpoint {
+    name: "TFRecordDataset"
+  }
+  summary: "Creates a dataset that emits the records from one or more TFRecord files."
+}
+op {
+  graph_op_name: "TFRecordReader"
+  endpoint {
+    name: "TFRecordReader"
+  }
+  summary: "A Reader that outputs the records from a TensorFlow Records file."
+}
+op {
+  graph_op_name: "TFRecordReaderV2"
+  endpoint {
+    name: "TFRecordReaderV2"
+  }
+  summary: "A Reader that outputs the records from a TensorFlow Records file."
+}
+op {
+  graph_op_name: "TakeDataset"
+  endpoint {
+    name: "TakeDataset"
+  }
+  summary: "Creates a dataset that contains `count` elements from the `input_dataset`."
+}
+op {
+  graph_op_name: "TakeManySparseFromTensorsMap"
+  endpoint {
+    name: "TakeManySparseFromTensorsMap"
+  }
+  summary: "Read `SparseTensors` from a `SparseTensorsMap` and concatenate them."
+  description: <<END
+The input `sparse_handles` must be an `int64` matrix of shape `[N, 1]` where
+`N` is the minibatch size and the rows correspond to the output handles of
+`AddSparseToTensorsMap` or `AddManySparseToTensorsMap`.  The ranks of the
+original `SparseTensor` objects that went into the given input ops must all
+match.  When the final `SparseTensor` is created, it has rank one
+higher than the ranks of the incoming `SparseTensor` objects
+(they have been concatenated along a new row dimension on the left).
+
+The output `SparseTensor` object's shape values for all dimensions but the
+first are the max across the input `SparseTensor` objects' shape values
+for the corresponding dimensions.  Its first shape value is `N`, the minibatch
+size.
+
+The input `SparseTensor` objects' indices are assumed ordered in
+standard lexicographic order.  If this is not the case, after this
+step run `SparseReorder` to restore index ordering.
+
+For example, if the handles represent an input, which is a `[2, 3]` matrix
+representing two original `SparseTensor` objects:
+
+```
+    index = [ 0]
+            [10]
+            [20]
+    values = [1, 2, 3]
+    shape = [50]
+```
+
+and
+
+```
+    index = [ 2]
+            [10]
+    values = [4, 5]
+    shape = [30]
+```
+
+then the final `SparseTensor` will be:
+
+```
+    index = [0  0]
+            [0 10]
+            [0 20]
+            [1  2]
+            [1 10]
+    values = [1, 2, 3, 4, 5]
+    shape = [2 50]
+```
+END
+}
+op {
+  graph_op_name: "Tan"
+  endpoint {
+    name: "Tan"
+  }
+  summary: "Computes tan of x element-wise."
+}
+op {
+  graph_op_name: "Tanh"
+  endpoint {
+    name: "Tanh"
+  }
+  summary: "Computes hyperbolic tangent of `x` element-wise."
+}
+op {
+  graph_op_name: "TanhGrad"
+  endpoint {
+    name: "TanhGrad"
+  }
+  summary: "Computes the gradient for the tanh of `x` wrt its input."
+  description: <<END
+Specifically, `grad = dy * (1 - y*y)`, where `y = tanh(x)`, and `dy`
+is the corresponding input gradient.
+END
+}
+op {
+  graph_op_name: "TemporaryVariable"
+  endpoint {
+    name: "TemporaryVariable"
+  }
+  summary: "Returns a tensor that may be mutated, but only persists within a single step."
+  description: <<END
+This is an experimental op for internal use only and it is possible to use this
+op in unsafe ways.  DO NOT USE unless you fully understand the risks.
+
+It is the caller's responsibility to ensure that 'ref' is eventually passed to a
+matching 'DestroyTemporaryVariable' op after all other uses have completed.
+
+Outputs a ref to the tensor state so it may be read or modified.
+
+  E.g.
+      var = state_ops._temporary_variable([1, 2], types.float_)
+      var_name = var.op.name
+      var = state_ops.assign(var, [[4.0, 5.0]])
+      var = state_ops.assign_add(var, [[6.0, 7.0]])
+      final = state_ops._destroy_temporary_variable(var, var_name=var_name)
+END
+}
+op {
+  graph_op_name: "TensorArray"
+  endpoint {
+    name: "TensorArray"
+  }
+}
+op {
+  graph_op_name: "TensorArrayClose"
+  endpoint {
+    name: "TensorArrayClose"
+  }
+}
+op {
+  graph_op_name: "TensorArrayCloseV2"
+  endpoint {
+    name: "TensorArrayCloseV2"
+  }
+  summary: "Deprecated. Use TensorArrayCloseV3"
+}
+op {
+  graph_op_name: "TensorArrayCloseV3"
+  endpoint {
+    name: "TensorArrayCloseV3"
+  }
+  summary: "Delete the TensorArray from its resource container."
+  description: <<END
+This enables the user to close and release the resource in the middle
+of a step/run.
+END
+}
+op {
+  graph_op_name: "TensorArrayConcat"
+  endpoint {
+    name: "TensorArrayConcat"
+  }
+}
+op {
+  graph_op_name: "TensorArrayConcatV2"
+  endpoint {
+    name: "TensorArrayConcatV2"
+  }
+  summary: "Deprecated. Use TensorArrayConcatV3"
+}
+op {
+  graph_op_name: "TensorArrayConcatV3"
+  endpoint {
+    name: "TensorArrayConcatV3"
+  }
+  summary: "Concat the elements from the TensorArray into value `value`."
+  description: <<END
+Takes `T` elements of shapes
+
+  ```
+  (n0 x d0 x d1 x ...), (n1 x d0 x d1 x ...), ..., (n(T-1) x d0 x d1 x ...)
+  ```
+
+and concatenates them into a Tensor of shape:
+
+  ```(n0 + n1 + ... + n(T-1) x d0 x d1 x ...)```
+
+All elements must have the same shape (excepting the first dimension).
+END
+}
+op {
+  graph_op_name: "TensorArrayGather"
+  endpoint {
+    name: "TensorArrayGather"
+  }
+}
+op {
+  graph_op_name: "TensorArrayGatherV2"
+  endpoint {
+    name: "TensorArrayGatherV2"
+  }
+  summary: "Deprecated. Use TensorArrayGatherV3"
+}
+op {
+  graph_op_name: "TensorArrayGatherV3"
+  endpoint {
+    name: "TensorArrayGatherV3"
+  }
+  summary: "Gather specific elements from the TensorArray into output `value`."
+  description: <<END
+All elements selected by `indices` must have the same shape.
+END
+}
+op {
+  graph_op_name: "TensorArrayGrad"
+  endpoint {
+    name: "TensorArrayGrad"
+  }
+}
+op {
+  graph_op_name: "TensorArrayGradV2"
+  endpoint {
+    name: "TensorArrayGradV2"
+  }
+  summary: "Deprecated. Use TensorArrayGradV3"
+}
+op {
+  graph_op_name: "TensorArrayGradV3"
+  endpoint {
+    name: "TensorArrayGradV3"
+  }
+  summary: "Creates a TensorArray for storing the gradients of values in the given handle."
+  description: <<END
+If the given TensorArray gradient already exists, returns a reference to it.
+
+Locks the size of the original TensorArray by disabling its dynamic size flag.
+
+**A note about the input flow_in:**
+
+The handle flow_in forces the execution of the gradient lookup to occur
+only after certain other operations have occurred.  For example, when
+the forward TensorArray is dynamically sized, writes to this TensorArray
+may resize the object.  The gradient TensorArray is statically sized based
+on the size of the forward TensorArray when this operation executes.
+Furthermore, the size of the forward TensorArray is frozen by this call.
+As a result, the flow is used to ensure that the call to generate the gradient
+TensorArray only happens after all writes are executed.
+
+In the case of dynamically sized TensorArrays, gradient computation should
+only be performed on read operations that have themselves been chained via
+flow to occur only after all writes have executed. That way the final size
+of the forward TensorArray is known when this operation is called.
+
+**A note about the source attribute:**
+
+TensorArray gradient calls use an accumulator TensorArray object.  If
+multiple gradients are calculated and run in the same session, the multiple
+gradient nodes may accidentally flow through the same accumulator TensorArray.
+This double counts and generally breaks the TensorArray gradient flow.
+
+The solution is to identify which gradient call this particular
+TensorArray gradient is being called in.  This is performed by identifying
+a unique string (e.g. "gradients", "gradients_1", ...) from the input
+gradient Tensor's name.  This string is used as a suffix when creating
+the TensorArray gradient object here (the attribute `source`).
+
+The attribute `source` is added as a suffix to the forward TensorArray's
+name when performing the creation / lookup, so that each separate gradient
+calculation gets its own TensorArray accumulator.
+END
+}
+op {
+  graph_op_name: "TensorArrayPack"
+  endpoint {
+    name: "TensorArrayPack"
+  }
+}
+op {
+  graph_op_name: "TensorArrayRead"
+  endpoint {
+    name: "TensorArrayRead"
+  }
+}
+op {
+  graph_op_name: "TensorArrayReadV2"
+  endpoint {
+    name: "TensorArrayReadV2"
+  }
+  summary: "Deprecated. Use TensorArrayReadV3"
+}
+op {
+  graph_op_name: "TensorArrayReadV3"
+  endpoint {
+    name: "TensorArrayReadV3"
+  }
+  summary: "Read an element from the TensorArray into output `value`."
+}
+op {
+  graph_op_name: "TensorArrayScatter"
+  endpoint {
+    name: "TensorArrayScatter"
+  }
+}
+op {
+  graph_op_name: "TensorArrayScatterV2"
+  endpoint {
+    name: "TensorArrayScatterV2"
+  }
+  summary: "Deprecated. Use TensorArrayScatterV3"
+}
+op {
+  graph_op_name: "TensorArrayScatterV3"
+  endpoint {
+    name: "TensorArrayScatterV3"
+  }
+  summary: "Scatter the data from the input value into specific TensorArray elements."
+  description: <<END
+`indices` must be a vector, its length must match the first dim of `value`.
+END
+}
+op {
+  graph_op_name: "TensorArraySize"
+  endpoint {
+    name: "TensorArraySize"
+  }
+}
+op {
+  graph_op_name: "TensorArraySizeV2"
+  endpoint {
+    name: "TensorArraySizeV2"
+  }
+  summary: "Deprecated. Use TensorArraySizeV3"
+}
+op {
+  graph_op_name: "TensorArraySizeV3"
+  endpoint {
+    name: "TensorArraySizeV3"
+  }
+  summary: "Get the current size of the TensorArray."
+}
+op {
+  graph_op_name: "TensorArraySplit"
+  endpoint {
+    name: "TensorArraySplit"
+  }
+}
+op {
+  graph_op_name: "TensorArraySplitV2"
+  endpoint {
+    name: "TensorArraySplitV2"
+  }
+  summary: "Deprecated. Use TensorArraySplitV3"
+}
+op {
+  graph_op_name: "TensorArraySplitV3"
+  endpoint {
+    name: "TensorArraySplitV3"
+  }
+  summary: "Split the data from the input value into TensorArray elements."
+  description: <<END
+Assuming that `lengths` takes on values
+
+  ```(n0, n1, ..., n(T-1))```
+
+and that `value` has shape
+
+  ```(n0 + n1 + ... + n(T-1) x d0 x d1 x ...)```,
+
+this splits values into a TensorArray with T tensors.
+
+TensorArray index t will be the subtensor of values with starting position
+
+  ```(n0 + n1 + ... + n(t-1), 0, 0, ...)```
+
+and having size
+
+  ```nt x d0 x d1 x ...```
+END
+}
+op {
+  graph_op_name: "TensorArrayUnpack"
+  endpoint {
+    name: "TensorArrayUnpack"
+  }
+}
+op {
+  graph_op_name: "TensorArrayV2"
+  endpoint {
+    name: "TensorArrayV2"
+  }
+  summary: "Deprecated. Use TensorArrayV3"
+}
+op {
+  graph_op_name: "TensorArrayV3"
+  endpoint {
+    name: "TensorArrayV3"
+  }
+  summary: "An array of Tensors of given size."
+  description: <<END
+Write data via Write and read via Read or Pack.
+END
+}
+op {
+  graph_op_name: "TensorArrayWrite"
+  endpoint {
+    name: "TensorArrayWrite"
+  }
+}
+op {
+  graph_op_name: "TensorArrayWriteV2"
+  endpoint {
+    name: "TensorArrayWriteV2"
+  }
+  summary: "Deprecated. Use TensorArrayGradV3"
+}
+op {
+  graph_op_name: "TensorArrayWriteV3"
+  endpoint {
+    name: "TensorArrayWriteV3"
+  }
+  summary: "Push an element onto the tensor_array."
+}
+op {
+  graph_op_name: "TensorDataset"
+  endpoint {
+    name: "TensorDataset"
+  }
+  summary: "Creates a dataset that emits `components` as a tuple of tensors once."
+}
+op {
+  graph_op_name: "TensorSliceDataset"
+  endpoint {
+    name: "TensorSliceDataset"
+  }
+  summary: "Creates a dataset that emits each dim-0 slice of `components` once."
+}
+op {
+  graph_op_name: "TensorSummary"
+  endpoint {
+    name: "TensorSummary"
+  }
+  summary: "Outputs a `Summary` protocol buffer with a tensor."
+  description: <<END
+This op is being phased out in favor of TensorSummaryV2, which lets callers pass
+a tag as well as a serialized SummaryMetadata proto string that contains
+plugin-specific data. We will keep this op to maintain backwards compatibility.
+END
+}
+op {
+  graph_op_name: "TensorSummaryV2"
+  endpoint {
+    name: "TensorSummaryV2"
+  }
+  summary: "Outputs a `Summary` protocol buffer with a tensor and per-plugin data."
+}
+op {
+  graph_op_name: "TextLineDataset"
+  endpoint {
+    name: "TextLineDataset"
+  }
+  summary: "Creates a dataset that emits the lines of one or more text files."
+}
+op {
+  graph_op_name: "TextLineReader"
+  endpoint {
+    name: "TextLineReader"
+  }
+  summary: "A Reader that outputs the lines of a file delimited by \'\\n\'."
+}
+op {
+  graph_op_name: "TextLineReaderV2"
+  endpoint {
+    name: "TextLineReaderV2"
+  }
+  summary: "A Reader that outputs the lines of a file delimited by \'\\n\'."
+}
+op {
+  graph_op_name: "ThreadUnsafeUnigramCandidateSampler"
+  endpoint {
+    name: "ThreadUnsafeUnigramCandidateSampler"
+  }
+  summary: "Generates labels for candidate sampling with a learned unigram distribution."
+  description: <<END
+See explanations of candidate sampling and the data formats at
+go/candidate-sampling.
+
+For each batch, this op picks a single set of sampled candidate labels.
+
+The advantages of sampling candidates per-batch are simplicity and the
+possibility of efficient dense matrix multiplication. The disadvantage is that
+the sampled candidates must be chosen independently of the context and of the
+true labels.
+END
+}
+op {
+  graph_op_name: "Tile"
+  endpoint {
+    name: "Tile"
+  }
+  summary: "Constructs a tensor by tiling a given tensor."
+  description: <<END
+This operation creates a new tensor by replicating `input` `multiples` times.
+The output tensor's i'th dimension has `input.dims(i) * multiples[i]` elements,
+and the values of `input` are replicated `multiples[i]` times along the 'i'th
+dimension. For example, tiling `[a b c d]` by `[2]` produces
+`[a b c d a b c d]`.
+END
+}
+op {
+  graph_op_name: "TileGrad"
+  endpoint {
+    name: "TileGrad"
+  }
+  summary: "Returns the gradient of `Tile`."
+  description: <<END
+Since `Tile` takes an input and repeats the input `multiples` times
+along each dimension, `TileGrad` takes in `multiples` and aggregates
+each repeated tile of `input` into `output`.
+END
+}
+op {
+  graph_op_name: "TopK"
+  endpoint {
+    name: "TopK"
+  }
+  summary: "Finds values and indices of the `k` largest elements for the last dimension."
+  description: <<END
+If the input is a vector (rank-1), finds the `k` largest entries in the vector
+and outputs their values and indices as vectors.  Thus `values[j]` is the
+`j`-th largest entry in `input`, and its index is `indices[j]`.
+
+For matrices (resp. higher rank input), computes the top `k` entries in each
+row (resp. vector along the last dimension).  Thus,
+
+    values.shape = indices.shape = input.shape[:-1] + [k]
+
+If two elements are equal, the lower-index element appears first.
+
+If `k` varies dynamically, use `TopKV2` below.
+END
+}
+op {
+  graph_op_name: "TopKV2"
+  endpoint {
+    name: "TopKV2"
+  }
+  summary: "Finds values and indices of the `k` largest elements for the last dimension."
+  description: <<END
+If the input is a vector (rank-1), finds the `k` largest entries in the vector
+and outputs their values and indices as vectors.  Thus `values[j]` is the
+`j`-th largest entry in `input`, and its index is `indices[j]`.
+
+For matrices (resp. higher rank input), computes the top `k` entries in each
+row (resp. vector along the last dimension).  Thus,
+
+    values.shape = indices.shape = input.shape[:-1] + [k]
+
+If two elements are equal, the lower-index element appears first.
+END
+}
+op {
+  graph_op_name: "Transpose"
+  endpoint {
+    name: "Transpose"
+  }
+  summary: "Shuffle dimensions of x according to a permutation."
+  description: <<END
+The output `y` has the same rank as `x`. The shapes of `x` and `y` satisfy:
+  `y.shape[i] == x.shape[perm[i]] for i in [0, 1, ..., rank(x) - 1]`
+END
+}
+op {
+  graph_op_name: "TruncateDiv"
+  endpoint {
+    name: "TruncateDiv"
+  }
+  summary: "Returns x / y element-wise for integer types."
+  description: <<END
+Truncation designates that negative numbers will round fractional quantities
+toward zero. I.e. -7 / 5 = 1. This matches C semantics but it is different
+than Python semantics. See `FloorDiv` for a division function that matches
+Python Semantics.
+
+*NOTE*: `TruncateDiv` supports broadcasting. More about broadcasting
+[here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
+END
+}
+op {
+  graph_op_name: "TruncateMod"
+  endpoint {
+    name: "TruncateMod"
+  }
+  summary: "Returns element-wise remainder of division. This emulates C semantics in that"
+  description: <<END
+the result here is consistent with a truncating divide. E.g. `truncate(x / y) *
+y + truncate_mod(x, y) = x`.
+
+*NOTE*: `TruncateMod` supports broadcasting. More about broadcasting
+[here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
+END
+}
+op {
+  graph_op_name: "TruncatedNormal"
+  endpoint {
+    name: "TruncatedNormal"
+  }
+  summary: "Outputs random values from a truncated normal distribution."
+  description: <<END
+The generated values follow a normal distribution with mean 0 and standard
+deviation 1, except that values whose magnitude is more than 2 standard
+deviations from the mean are dropped and re-picked.
+END
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_U.pbtxt b/tensorflow/core/api_def/base_api/api_def_U.pbtxt
new file mode 100644
index 0000000000..6699efc0e0
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_U.pbtxt
@@ -0,0 +1,150 @@
+op {
+  graph_op_name: "UniformCandidateSampler"
+  endpoint {
+    name: "UniformCandidateSampler"
+  }
+  summary: "Generates labels for candidate sampling with a uniform distribution."
+  description: <<END
+See explanations of candidate sampling and the data formats at
+go/candidate-sampling.
+
+For each batch, this op picks a single set of sampled candidate labels.
+
+The advantages of sampling candidates per-batch are simplicity and the
+possibility of efficient dense matrix multiplication. The disadvantage is that
+the sampled candidates must be chosen independently of the context and of the
+true labels.
+END
+}
+op {
+  graph_op_name: "Unique"
+  endpoint {
+    name: "Unique"
+  }
+  summary: "Finds unique elements in a 1-D tensor."
+  description: <<END
+This operation returns a tensor `y` containing all of the unique elements of `x`
+sorted in the same order that they occur in `x`. This operation also returns a
+tensor `idx` the same size as `x` that contains the index of each value of `x`
+in the unique output `y`. In other words:
+
+`y[idx[i]] = x[i] for i in [0, 1,...,rank(x) - 1]`
+
+For example:
+
+```
+# tensor 'x' is [1, 1, 2, 4, 4, 4, 7, 8, 8]
+y, idx = unique(x)
+y ==> [1, 2, 4, 7, 8]
+idx ==> [0, 0, 1, 2, 2, 2, 3, 4, 4]
+```
+END
+}
+op {
+  graph_op_name: "UniqueWithCounts"
+  endpoint {
+    name: "UniqueWithCounts"
+  }
+  summary: "Finds unique elements in a 1-D tensor."
+  description: <<END
+This operation returns a tensor `y` containing all of the unique elements of `x`
+sorted in the same order that they occur in `x`. This operation also returns a
+tensor `idx` the same size as `x` that contains the index of each value of `x`
+in the unique output `y`. Finally, it returns a third tensor `count` that
+contains the count of each element of `y` in `x`. In other words:
+
+`y[idx[i]] = x[i] for i in [0, 1,...,rank(x) - 1]`
+
+For example:
+
+```
+# tensor 'x' is [1, 1, 2, 4, 4, 4, 7, 8, 8]
+y, idx, count = unique_with_counts(x)
+y ==> [1, 2, 4, 7, 8]
+idx ==> [0, 0, 1, 2, 2, 2, 3, 4, 4]
+count ==> [2, 1, 3, 1, 2]
+```
+END
+}
+op {
+  graph_op_name: "Unpack"
+  endpoint {
+    name: "Unpack"
+  }
+  summary: "Unpacks a given dimension of a rank-`R` tensor into `num` rank-`(R-1)` tensors."
+  description: <<END
+Unpacks `num` tensors from `value` by chipping it along the `axis` dimension.
+For example, given a tensor of shape `(A, B, C, D)`;
+
+If `axis == 0` then the i'th tensor in `output` is the slice `value[i, :, :, :]`
+  and each tensor in `output` will have shape `(B, C, D)`. (Note that the
+  dimension unpacked along is gone, unlike `split`).
+
+If `axis == 1` then the i'th tensor in `output` is the slice `value[:, i, :, :]`
+  and each tensor in `output` will have shape `(A, C, D)`.
+Etc.
+
+This is the opposite of `pack`.
+END
+}
+op {
+  graph_op_name: "UnsortedSegmentMax"
+  endpoint {
+    name: "UnsortedSegmentMax"
+  }
+  summary: "Computes the Max along segments of a tensor."
+  description: <<END
+Read @{$math_ops#segmentation$the section on segmentation} for an explanation of
+segments.
+
+This operator is similar to the [unsorted segment sum operator](../../../api_docs/python/math_ops.md#UnsortedSegmentSum).
+Instead of computing the sum over segments, it computes the maximum
+such that:
+
+\\(output_i = \max_j data_j\\) where max is over `j` such
+that `segment_ids[j] == i`.
+
+If the maximum is empty for a given segment ID `i`, it outputs the smallest possible value for specific numeric type,
+ `output[i] = numeric_limits<T>::min()`.
+
+<div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
+<img style="width:100%" src="https://www.tensorflow.org/images/UnsortedSegmentMax.png" alt>
+</div>
+END
+}
+op {
+  graph_op_name: "UnsortedSegmentSum"
+  endpoint {
+    name: "UnsortedSegmentSum"
+  }
+  summary: "Computes the sum along segments of a tensor."
+  description: <<END
+Read @{$math_ops#segmentation$the section on segmentation} for an explanation of
+segments.
+
+Computes a tensor such that
+`(output[i] = sum_{j...} data[j...]` where the sum is over tuples `j...` such
+that `segment_ids[j...] == i`.  Unlike `SegmentSum`, `segment_ids`
+need not be sorted and need not cover all values in the full
+range of valid values.
+
+If the sum is empty for a given segment ID `i`, `output[i] = 0`.
+
+`num_segments` should equal the number of distinct segment IDs.
+
+<div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
+<img style="width:100%" src="https://www.tensorflow.org/images/UnsortedSegmentSum.png" alt>
+</div>
+END
+}
+op {
+  graph_op_name: "Unstage"
+  endpoint {
+    name: "Unstage"
+  }
+  summary: "Op is similar to a lightweight Dequeue."
+  description: <<END
+The basic functionality is similar to dequeue with many fewer
+capabilities and options.  This Op is optimized for performance.
+END
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_V.pbtxt b/tensorflow/core/api_def/base_api/api_def_V.pbtxt
new file mode 100644
index 0000000000..31cc147900
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_V.pbtxt
@@ -0,0 +1,19 @@
+op {
+  graph_op_name: "Variable"
+  endpoint {
+    name: "Variable"
+  }
+  summary: "Use VariableV2 instead."
+}
+op {
+  graph_op_name: "VariableV2"
+  endpoint {
+    name: "VariableV2"
+  }
+  summary: "Holds state in the form of a tensor that persists across steps."
+  description: <<END
+Outputs a ref to the tensor state so it may be read or modified.
+TODO(zhifengc/mrry): Adds a pointer to a more detail document
+about sharing states in tensorflow.
+END
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_W.pbtxt b/tensorflow/core/api_def/base_api/api_def_W.pbtxt
new file mode 100644
index 0000000000..9120fe334e
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_W.pbtxt
@@ -0,0 +1,72 @@
+op {
+  graph_op_name: "Where"
+  endpoint {
+    name: "Where"
+  }
+  summary: "Returns locations of true values in a boolean tensor."
+  description: <<END
+This operation returns the coordinates of true elements in `input`. The
+coordinates are returned in a 2-D tensor where the first dimension (rows)
+represents the number of true elements, and the second dimension (columns)
+represents the coordinates of the true elements. Keep in mind, the shape of
+the output tensor can vary depending on how many true values there are in
+`input`. Indices are output in row-major order.
+
+For example:
+
+```
+# 'input' tensor is [[True, False]
+#                    [True, False]]
+# 'input' has two true values, so output has two coordinates.
+# 'input' has rank of 2, so coordinates have two indices.
+where(input) ==> [[0, 0],
+                  [1, 0]]
+
+# `input` tensor is [[[True, False]
+#                     [True, False]]
+#                    [[False, True]
+#                     [False, True]]
+#                    [[False, False]
+#                     [False, True]]]
+# 'input' has 5 true values, so output has 5 coordinates.
+# 'input' has rank of 3, so coordinates have three indices.
+where(input) ==> [[0, 0, 0],
+                  [0, 1, 0],
+                  [1, 0, 1],
+                  [1, 1, 1],
+                  [2, 1, 1]]
+```
+END
+}
+op {
+  graph_op_name: "WholeFileReader"
+  endpoint {
+    name: "WholeFileReader"
+  }
+  summary: "A Reader that outputs the entire contents of a file as a value."
+  description: <<END
+To use, enqueue filenames in a Queue.  The output of ReaderRead will
+be a filename (key) and the contents of that file (value).
+END
+}
+op {
+  graph_op_name: "WholeFileReaderV2"
+  endpoint {
+    name: "WholeFileReaderV2"
+  }
+  summary: "A Reader that outputs the entire contents of a file as a value."
+  description: <<END
+To use, enqueue filenames in a Queue.  The output of ReaderRead will
+be a filename (key) and the contents of that file (value).
+END
+}
+op {
+  graph_op_name: "WriteFile"
+  endpoint {
+    name: "WriteFile"
+  }
+  summary: "Writes contents to the file at input filename. Creates file and recursively"
+  description: <<END
+creates directory if not existing.
+END
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_Z.pbtxt b/tensorflow/core/api_def/base_api/api_def_Z.pbtxt
new file mode 100644
index 0000000000..f83fef054c
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_Z.pbtxt
@@ -0,0 +1,27 @@
+op {
+  graph_op_name: "ZerosLike"
+  endpoint {
+    name: "ZerosLike"
+  }
+  summary: "Returns a tensor of zeros with the same shape and type as x."
+}
+op {
+  graph_op_name: "Zeta"
+  endpoint {
+    name: "Zeta"
+  }
+  summary: "Compute the Hurwitz zeta function \\\\(\\zeta(x, q)\\\\)."
+  description: <<END
+The Hurwitz zeta function is defined as:
+
+
+\\(\zeta(x, q) = \sum_{n=0}^{\infty} (q + n)^{-x}\\)
+END
+}
+op {
+  graph_op_name: "ZipDataset"
+  endpoint {
+    name: "ZipDataset"
+  }
+  summary: "Creates a dataset that zips together `input_datasets`."
+}
diff --git a/tensorflow/core/api_def/update_api_def.sh b/tensorflow/core/api_def/update_api_def.sh
new file mode 100755
index 0000000000..07c76e6562
--- /dev/null
+++ b/tensorflow/core/api_def/update_api_def.sh
@@ -0,0 +1,28 @@
+#!/bin/bash
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+# Script to update tensorflow/core/api_def/base_api/api_def*.pbtxt files.
+
+set -e
+
+current_file="$(readlink -f "$0")"
+current_dir="$(dirname "$current_file")"
+
+bazel build //tensorflow/core:api_test
+bazel-bin/tensorflow/core/api_test \
+  --update_api_def \
+  --api_def_dir="${current_dir}/base_api"
+
diff --git a/tensorflow/core/framework/op.h b/tensorflow/core/framework/op.h
index c765bc915f..f7f1ed2a88 100644
--- a/tensorflow/core/framework/op.h
+++ b/tensorflow/core/framework/op.h
@@ -75,7 +75,8 @@ class OpRegistry : public OpRegistryInterface {
                 const OpRegistrationData** op_reg_data) const override;
 
   // Fills *ops with all registered OpDefs (except those with names
-  // starting with '_' if include_internal == false).
+  // starting with '_' if include_internal == false) sorted in
+  // ascending alphabetical order.
   void Export(bool include_internal, OpList* ops) const;
 
   // Returns ASCII-format OpList for all registered OpDefs (except
-- 
GitLab


From 13d6d3b51441d11dca141e1f17630b7448835f78 Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Wed, 11 Oct 2017 00:43:52 -0700
Subject: [PATCH 259/909] Internal change.

PiperOrigin-RevId: 171789232
---
 tensorflow/contrib/timeseries/examples/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/contrib/timeseries/examples/BUILD b/tensorflow/contrib/timeseries/examples/BUILD
index 8ed812f9d1..222a77c489 100644
--- a/tensorflow/contrib/timeseries/examples/BUILD
+++ b/tensorflow/contrib/timeseries/examples/BUILD
@@ -97,6 +97,7 @@ py_test(
     timeout = "long",  # Moderate but for asan
     srcs = ["lstm_test.py"],
     srcs_version = "PY2AND3",
+    tags = ["notsan"],
     deps = [":lstm"],
 )
 
-- 
GitLab


From f5af28ac36a9642e1c220b376ec25cc192086e85 Mon Sep 17 00:00:00 2001
From: Atlas7 <johnnychan0302@gmail.com>
Date: Wed, 11 Oct 2017 14:35:52 +0100
Subject: [PATCH 260/909] add the missing closing parenthesis to code snippet

---
 tensorflow/docs_src/programmers_guide/graphs.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/docs_src/programmers_guide/graphs.md b/tensorflow/docs_src/programmers_guide/graphs.md
index 6ba8bb7a34..10f53fe8f2 100644
--- a/tensorflow/docs_src/programmers_guide/graphs.md
+++ b/tensorflow/docs_src/programmers_guide/graphs.md
@@ -404,8 +404,8 @@ y = tf.square(x)
 
 with tf.Session() as sess:
   # Feeding a value changes the result that is returned when you evaluate `y`.
-  print(sess.run(y, {x: [1.0, 2.0, 3.0]})  # => "[1.0, 4.0, 9.0]"
-  print(sess.run(y, {x: [0.0, 0.0, 5.0]})  # => "[0.0, 0.0, 25.0]"
+  print(sess.run(y, {x: [1.0, 2.0, 3.0]}))  # => "[1.0, 4.0, 9.0]"
+  print(sess.run(y, {x: [0.0, 0.0, 5.0]}))  # => "[0.0, 0.0, 25.0]"
 
   # Raises `tf.errors.InvalidArgumentError`, because you must feed a value for
   # a `tf.placeholder()` when evaluating a tensor that depends on it.
-- 
GitLab


From 9e15937f966a83957552cc95c1e5baba2e16fa51 Mon Sep 17 00:00:00 2001
From: Dan Ringwalt <ringwalt@google.com>
Date: Wed, 11 Oct 2017 07:53:40 -0700
Subject: [PATCH 261/909] Copy public
 tf.contrib.graph_editor.reroute_{inputs,outputs} docs.

There are multiple references "see reroute_inputs" which are unhelpful because the full docstring now only exists on _reroute_sgv_inputs (likewise for reroute_outputs). Copy most of the docstring to reroute_{inputs,outputs} so that it is outputted in the docs.

Update some other dangling doc references from _reroute to _reroute_sgv, but that docstring will not be included the docs.

PiperOrigin-RevId: 171821659
---
 tensorflow/contrib/graph_editor/reroute.py | 40 +++++++++++++++++++---
 1 file changed, 35 insertions(+), 5 deletions(-)

diff --git a/tensorflow/contrib/graph_editor/reroute.py b/tensorflow/contrib/graph_editor/reroute.py
index 42968ae63b..7ffdbb7139 100644
--- a/tensorflow/contrib/graph_editor/reroute.py
+++ b/tensorflow/contrib/graph_editor/reroute.py
@@ -397,27 +397,57 @@ def swap_inputs(sgv0, sgv1):
 
 
 def reroute_inputs(sgv0, sgv1):
-  """Re-route all the inputs of sgv0 to sgv1 (see reroute_inputs)."""
+  """Re-route all the inputs of two subgraphs.
+
+  Args:
+    sgv0: the first subgraph to have its inputs swapped. This argument is
+      converted to a subgraph using the same rules than the function
+      subgraph.make_view.
+    sgv1: the second subgraph to have its inputs swapped. This argument is
+      converted to a subgraph using the same rules than the function
+      subgraph.make_view.
+  Returns:
+    A tuple `(sgv0, sgv1)` of subgraph views with their inputs swapped.
+      Note that the function argument sgv0 and sgv1 are also modified in place.
+  Raises:
+    StandardError: if sgv0 or sgv1 cannot be converted to a SubGraphView using
+      the same rules than the function subgraph.make_view.
+  """
   return _reroute_sgv_inputs(sgv0, sgv1, _RerouteMode.a2b)
 
 
 def swap_outputs(sgv0, sgv1):
-  """Swap all the outputs of sgv0 and sgv1 (see _reroute_outputs)."""
+  """Swap all the outputs of sgv0 and sgv1 (see reroute_outputs)."""
   return _reroute_sgv_outputs(sgv0, sgv1, _RerouteMode.swap)
 
 
 def reroute_outputs(sgv0, sgv1):
-  """Re-route all the outputs of sgv0 to sgv1 (see _reroute_outputs)."""
+  """Re-route all the outputs of two operations.
+
+  Args:
+    sgv0: the first subgraph to have its outputs swapped. This argument is
+      converted to a subgraph using the same rules than the function
+      subgraph.make_view.
+    sgv1: the second subgraph to have its outputs swapped. This argument is
+      converted to a subgraph using the same rules than the function
+      subgraph.make_view.
+  Returns:
+    A tuple `(sgv0, sgv1)` of subgraph views with their outputs swapped.
+      Note that the function argument sgv0 and sgv1 are also modified in place.
+  Raises:
+    StandardError: if sgv0 or sgv1 cannot be converted to a SubGraphView using
+      the same rules than the function subgraph.make_view.
+  """
   return _reroute_sgv_outputs(sgv0, sgv1, _RerouteMode.a2b)
 
 
 def swap_ios(sgv0, sgv1):
-  """Swap the inputs and outputs of sgv1 to sgv0 (see _reroute)."""
+  """Swap the inputs and outputs of sgv1 to sgv0 (see _reroute_sgv)."""
   return _reroute_sgv(sgv0, sgv1, _RerouteMode.swap)
 
 
 def reroute_ios(sgv0, sgv1):
-  """Re-route the inputs and outputs of sgv0 to sgv1 (see _reroute)."""
+  """Re-route the inputs and outputs of sgv0 to sgv1 (see _reroute_sgv)."""
   return _reroute_sgv(sgv0, sgv1, _RerouteMode.a2b)
 
 
-- 
GitLab


From 7ac56674663923b0c9d0ae7521b405f586a7a9a2 Mon Sep 17 00:00:00 2001
From: Till Hoffmann <tillahoffmann@gmail.com>
Date: Wed, 11 Oct 2017 16:56:18 +0100
Subject: [PATCH 262/909] Add ReceptiveField class and coordinate conversion
 methods. (#13239)

* Add ReceptiveField class and coordinate conversion methods.

* Add ReceptiveField class and coordinate conversion methods.

* Allow propagation of receptive field to be stopped explicitly.

* Improve documentation.

* Add periods to comments.

* Rename arguments of conversion methods.
---
 .../python/util/receptive_field.py            | 134 ++++++++++++++++--
 .../python/util/receptive_field_test.py       |  56 ++++++++
 2 files changed, 178 insertions(+), 12 deletions(-)

diff --git a/tensorflow/contrib/receptive_field/python/util/receptive_field.py b/tensorflow/contrib/receptive_field/python/util/receptive_field.py
index db190a1a41..8b34465d21 100644
--- a/tensorflow/contrib/receptive_field/python/util/receptive_field.py
+++ b/tensorflow/contrib/receptive_field/python/util/receptive_field.py
@@ -27,13 +27,15 @@ import math
 from tensorflow.contrib.receptive_field.python.util import graph_compute_order
 from tensorflow.contrib.util import make_ndarray
 from tensorflow.python.platform import tf_logging as logging
+from tensorflow.python.framework import ops as framework_ops
+import numpy as np
 
 # White-listed layer operations, which do not affect the receptive field
 # computation.
 _UNCHANGED_RF_LAYER_OPS = [
-    "Softplus", "Relu", "BiasAdd", "Mul", "Add", "Const", "Identity",
-    "VariableV2", "Sub", "Rsqrt", "ConcatV2"
-]
+  'Add', 'BiasAdd', 'Ceil', 'ConcatV2', 'Const', 'Floor', 'Identity', 'Log',
+  'Mul', 'Pow', 'RealDiv', 'Relu', 'Round', 'Rsqrt', 'Softplus', 'Sub',
+  'VariableV2']
 
 # Different ways in which padding modes may be spelled.
 _VALID_PADDING = ["VALID", b"VALID"]
@@ -238,7 +240,8 @@ def _get_layer_params(node, name_to_order_node):
     padding_x = 0
     padding_y = 0
   else:
-    raise ValueError("Unknown layer op: %s" % node.op)
+    raise ValueError("Unknown layer for operation '%s': %s" %
+                     (node.name, node.op))
   return kernel_size_x, kernel_size_y, stride_x, stride_y, padding_x, padding_y
 
 
@@ -304,13 +307,103 @@ def _get_effective_padding_node_input(stride, padding,
   return stride * effective_padding_output + padding
 
 
-def compute_receptive_field_from_graph_def(graph_def, input_node, output_node):
-  """Computes receptive field (RF) parameters from a GraphDef object.
+class ReceptiveField:
+  """
+  Receptive field of a convolutional neural network.
+
+  Args:
+    size: Receptive field size.
+    stride: Effective stride.
+    padding: Effective padding.
+  """
+  def __init__(self, size, stride, padding):
+    self.size = np.asarray(size)
+    self.stride = np.asarray(stride)
+    self.padding = np.asarray(padding)
+
+  def compute_input_center_coordinates(self, y, axis=None):
+    """
+    Computes the center of the receptive field that generated a feature.
+
+    Args:
+      y: An array of feature coordinates with shape `(..., d)`, where `d` is the
+        number of dimensions of the coordinates.
+      axis: The dimensions for which to compute the input center coordinates.
+        If `None` (the default), compute the input center coordinates for all
+        dimensions.
+
+    Returns:
+      x: Center of the receptive field that generated the features, at the input
+        of the network.
+
+    Raises:
+      ValueError: If the number of dimensions of the feature coordinates does
+        not match the number of elements in `axis`.
+    """
+    # Use all dimensions.
+    if axis is None:
+      axis = range(self.size.size)
+    # Ensure axis is a list because tuples have different indexing behavior.
+    axis = list(axis)
+    y = np.asarray(y)
+    if y.shape[-1] != len(axis):
+      raise ValueError("Dimensionality of the feature coordinates `y` (%d) "
+                       "does not match dimensionality of `axis` (%d)" %
+                       (y.shape[-1], len(axis)))
+    return - self.padding[axis] + y * self.stride[axis] + \
+      (self.size[axis] - 1) / 2
+
+  def compute_feature_coordinates(self, x, axis=None):
+    """
+    Computes the position of a feature given the center of a receptive field.
+
+    Args:
+      x: An array of input center coordinates with shape `(..., d)`, where `d`
+        is the number of dimensions of the coordinates.
+      axis: The dimensions for which to compute the feature coordinates.
+        If `None` (the default), compute the feature coordinates for all
+        dimensions.
+
+    Returns:
+      y: Coordinates of the features.
+
+    Raises:
+      ValueError: If the number of dimensions of the input center coordinates
+        does not match the number of elements in `axis`.
+    """
+    # Use all dimensions.
+    if axis is None:
+      axis = range(self.size.size)
+    # Ensure axis is a list because tuples have different indexing behavior.
+    axis = list(axis)
+    x = np.asarray(x)
+    if x.shape[-1] != len(axis):
+      raise ValueError("Dimensionality of the input center coordinates `x` "
+                       "(%d) does not match dimensionality of `axis` (%d)" %
+                       (x.shape[-1], len(axis)))
+    return (x + self.padding[axis] + (1 - self.size[axis]) / 2) / \
+      self.stride[axis]
+
+  def __iter__(self):
+    return iter(np.concatenate([self.size, self.stride, self.padding]))
+
+
+def compute_receptive_field_from_graph_def(graph_def, input_node, output_node,
+                                           stop_propagation=None):
+  """Computes receptive field (RF) parameters from a Graph or GraphDef object.
+
+  The algorithm stops the calculation of the receptive field whenever it
+  encounters an operation in the list `stop_propagation`. Stopping the
+  calculation early can be useful to calculate the receptive field of a
+  subgraph such as a single branch of the
+  [inception network](https://arxiv.org/abs/1512.00567).
 
   Args:
-    graph_def: GraphDef object.
-    input_node: Name of the input node from graph.
-    output_node: Name of the output node from graph.
+    graph_def: Graph or GraphDef object.
+    input_node: Name of the input node or Tensor object from graph.
+    output_node: Name of the output node or Tensor object from graph.
+    stop_propagation: List of operation or scope names for which to stop the
+      propagation of the receptive field.
 
   Returns:
     rf_size_x: Receptive field size of network in the horizontal direction, with
@@ -331,6 +424,18 @@ def compute_receptive_field_from_graph_def(graph_def, input_node, output_node):
       cannot be found. For network criterion alignment, see
       photos/vision/features/delf/g3doc/rf_computation.md
   """
+  # Convert a graph to graph_def if necessary.
+  if isinstance(graph_def, framework_ops.Graph):
+    graph_def = graph_def.as_graph_def()
+
+  # Convert tensors to names.
+  if isinstance(input_node, framework_ops.Tensor):
+    input_node = input_node.op.name
+  if isinstance(output_node, framework_ops.Tensor):
+    output_node = output_node.op.name
+
+  stop_propagation = stop_propagation or []
+
   # Computes order of computation for a given graph.
   name_to_order_node = graph_compute_order.get_compute_order(
       graph_def=graph_def)
@@ -422,6 +527,10 @@ def compute_receptive_field_from_graph_def(graph_def, input_node, output_node):
 
       # Loop over this node's inputs and potentially propagate information down.
       for inp_name in node.input:
+        # Stop the propagation of the receptive field.
+        if any(inp_name.startswith(stop) for stop in stop_propagation):
+          logging.vlog(3, "Skipping explicitly ignored node %s.", node.name)
+          continue
         logging.vlog(4, "inp_name = %s", inp_name)
         inp_node = name_to_order_node[inp_name].node
         logging.vlog(4, "inp_node = \n%s", inp_node)
@@ -480,6 +589,7 @@ def compute_receptive_field_from_graph_def(graph_def, input_node, output_node):
     raise ValueError("Output node was not found")
   if input_node not in rf_sizes_x:
     raise ValueError("Input node was not found")
-  return (rf_sizes_x[input_node], rf_sizes_y[input_node],
-          effective_strides_x[input_node], effective_strides_y[input_node],
-          effective_paddings_x[input_node], effective_paddings_y[input_node])
+  return ReceptiveField(
+    (rf_sizes_x[input_node], rf_sizes_y[input_node]),
+    (effective_strides_x[input_node], effective_strides_y[input_node]),
+    (effective_paddings_x[input_node], effective_paddings_y[input_node]))
diff --git a/tensorflow/contrib/receptive_field/python/util/receptive_field_test.py b/tensorflow/contrib/receptive_field/python/util/receptive_field_test.py
index 2771389250..8d7d5440f6 100644
--- a/tensorflow/contrib/receptive_field/python/util/receptive_field_test.py
+++ b/tensorflow/contrib/receptive_field/python/util/receptive_field_test.py
@@ -25,6 +25,7 @@ from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import nn
 from tensorflow.python.platform import test
+import numpy as np
 
 
 def create_test_network_1():
@@ -150,6 +151,31 @@ def create_test_network_5():
   return g
 
 
+def create_test_network_6():
+  """Aligned network with dropout for test.
+
+  The graph is similar to create_test_network_1(), except that the right branch
+  has dropout normalization.
+
+  Returns:
+    g: Tensorflow graph object (Graph proto).
+  """
+  g = ops.Graph()
+  with g.as_default():
+    # An 8x8 test image.
+    x = array_ops.placeholder(dtypes.float32, (1, 8, 8, 1), name='input_image')
+    # Left branch.
+    l1 = slim.conv2d(x, 1, [1, 1], stride=4, scope='L1', padding='VALID')
+    # Right branch.
+    l2_pad = array_ops.pad(x, [[0, 0], [1, 0], [1, 0], [0, 0]])
+    l2 = slim.conv2d(l2_pad, 1, [3, 3], stride=2, scope='L2', padding='VALID')
+    l3 = slim.conv2d(l2, 1, [1, 1], stride=2, scope='L3', padding='VALID')
+    dropout = slim.dropout(l3)
+    # Addition.
+    nn.relu(l1 + dropout, name='output')
+  return g
+
+
 class RfUtilsTest(test.TestCase):
 
   def testComputeRFFromGraphDefAligned(self):
@@ -220,6 +246,36 @@ class RfUtilsTest(test.TestCase):
     self.assertEqual(effective_padding_x, 0)
     self.assertEqual(effective_padding_y, 0)
 
+  def testComputeRFFromGraphDefStopPropagation(self):
+    graph_def = create_test_network_6().as_graph_def()
+    input_node = 'input_image'
+    output_node = 'output'
+    # Compute the receptive field but stop the propagation for the random
+    # uniform variable of the dropout.
+    (receptive_field_x, receptive_field_y, effective_stride_x,
+     effective_stride_y, effective_padding_x, effective_padding_y) = (
+         receptive_field.compute_receptive_field_from_graph_def(
+             graph_def, input_node, output_node,
+             ['Dropout/dropout/random_uniform']))
+    self.assertEqual(receptive_field_x, 3)
+    self.assertEqual(receptive_field_y, 3)
+    self.assertEqual(effective_stride_x, 4)
+    self.assertEqual(effective_stride_y, 4)
+    self.assertEqual(effective_padding_x, 1)
+    self.assertEqual(effective_padding_y, 1)
+
+  def testComputeCoordinatesRoundtrip(self):
+    graph_def = create_test_network_1()
+    input_node = 'input_image'
+    output_node = 'output'
+    rf = receptive_field.compute_receptive_field_from_graph_def(
+      graph_def, input_node, output_node)
+
+    x = np.random.randint(0, 100, (50, 2))
+    y = rf.compute_feature_coordinates(x)
+    x2 = rf.compute_input_center_coordinates(y)
+
+    self.assertAllEqual(x, x2)
 
 if __name__ == '__main__':
   test.main()
-- 
GitLab


From acaf5c41ea0d875f2a610c334281326c9fee9b6c Mon Sep 17 00:00:00 2001
From: "Joshua V. Dillon" <jvdillon@google.com>
Date: Wed, 11 Oct 2017 09:39:59 -0700
Subject: [PATCH 263/909] Add tf.contrib.distributions.bijectors.Permute.

PiperOrigin-RevId: 171833156
---
 tensorflow/contrib/distributions/BUILD        |  16 ++
 .../kernel_tests/bijectors/permute_test.py    |  87 +++++++++++
 .../python/ops/bijectors/__init__.py          |   2 +
 .../python/ops/bijectors/permute.py           |  29 ++++
 .../python/ops/bijectors/permute_impl.py      | 138 ++++++++++++++++++
 5 files changed, 272 insertions(+)
 create mode 100644 tensorflow/contrib/distributions/python/kernel_tests/bijectors/permute_test.py
 create mode 100644 tensorflow/contrib/distributions/python/ops/bijectors/permute.py
 create mode 100644 tensorflow/contrib/distributions/python/ops/bijectors/permute_impl.py

diff --git a/tensorflow/contrib/distributions/BUILD b/tensorflow/contrib/distributions/BUILD
index dcdfbbeba2..93770c37de 100644
--- a/tensorflow/contrib/distributions/BUILD
+++ b/tensorflow/contrib/distributions/BUILD
@@ -835,6 +835,22 @@ cuda_py_test(
     ],
 )
 
+cuda_py_test(
+    name = "permute_test",
+    size = "small",
+    srcs = ["python/kernel_tests/bijectors/permute_test.py"],
+    additional_deps = [
+        ":bijectors_py",
+        ":distributions_py",
+        "//third_party/py/numpy",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework_for_generated_wrappers",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:platform_test",
+    ],
+)
+
 cuda_py_test(
     name = "power_transform_test",
     size = "small",
diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/permute_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/permute_test.py
new file mode 100644
index 0000000000..54590de373
--- /dev/null
+++ b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/permute_test.py
@@ -0,0 +1,87 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for Permute bijector."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.contrib.distributions.python.ops.bijectors.permute import Permute
+from tensorflow.python.framework import dtypes
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops.distributions.bijector_test_util import assert_bijective_and_finite
+from tensorflow.python.platform import test
+
+
+class PermuteBijectorTest(test.TestCase):
+  """Tests correctness of the Permute bijector."""
+
+  def setUp(self):
+    self._rng = np.random.RandomState(42)
+
+  def testBijector(self):
+    expected_permutation = np.int32([2, 0, 1])
+    expected_x = np.random.randn(4, 2, 3)
+    expected_y = expected_x[..., expected_permutation]
+
+    with self.test_session() as sess:
+      permutation_ph = array_ops.placeholder(dtype=dtypes.int32)
+      bijector = Permute(
+          permutation=permutation_ph,
+          validate_args=True)
+      [
+          permutation_,
+          x_,
+          y_,
+          fldj,
+          ildj,
+      ] = sess.run([
+          bijector.permutation,
+          bijector.inverse(expected_y),
+          bijector.forward(expected_x),
+          bijector.forward_log_det_jacobian(expected_x),
+          bijector.inverse_log_det_jacobian(expected_y),
+      ], feed_dict={permutation_ph: expected_permutation})
+      self.assertEqual("permute", bijector.name)
+      self.assertAllEqual(expected_permutation, permutation_)
+      self.assertAllClose(expected_y, y_, rtol=1e-6, atol=0)
+      self.assertAllClose(expected_x, x_, rtol=1e-6, atol=0)
+      self.assertAllClose(0., fldj, rtol=1e-6, atol=0)
+      self.assertAllClose(0., ildj, rtol=1e-6, atol=0)
+
+  def testRaisesOpError(self):
+    with self.test_session() as sess:
+      with self.assertRaisesOpError("Permutation over `d` must contain"):
+        permutation_ph = array_ops.placeholder(dtype=dtypes.int32)
+        bijector = Permute(
+            permutation=permutation_ph,
+            validate_args=True)
+        sess.run(bijector.inverse([1.]),
+                 feed_dict={permutation_ph: [1, 2]})
+
+  def testBijectiveAndFinite(self):
+    permutation = np.int32([2, 0, 1])
+    x = np.random.randn(4, 2, 3)
+    y = x[..., permutation]
+    with self.test_session():
+      bijector = Permute(
+          permutation=permutation,
+          validate_args=True)
+      assert_bijective_and_finite(bijector, x, y, rtol=1e-6, atol=0)
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/__init__.py b/tensorflow/contrib/distributions/python/ops/bijectors/__init__.py
index 4541701109..c9ed546a34 100644
--- a/tensorflow/contrib/distributions/python/ops/bijectors/__init__.py
+++ b/tensorflow/contrib/distributions/python/ops/bijectors/__init__.py
@@ -25,6 +25,7 @@
 @@Identity
 @@Inline
 @@Invert
+@@Permute
 @@PowerTransform
 @@Sigmoid
 @@SigmoidCentered
@@ -49,6 +50,7 @@ from tensorflow.contrib.distributions.python.ops.bijectors.conditional_bijector
 from tensorflow.contrib.distributions.python.ops.bijectors.exp import *
 from tensorflow.contrib.distributions.python.ops.bijectors.inline import *
 from tensorflow.contrib.distributions.python.ops.bijectors.invert import *
+from tensorflow.contrib.distributions.python.ops.bijectors.permute import *
 from tensorflow.contrib.distributions.python.ops.bijectors.power_transform import *
 from tensorflow.contrib.distributions.python.ops.bijectors.sigmoid import *
 from tensorflow.contrib.distributions.python.ops.bijectors.sigmoid_centered import *
diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/permute.py b/tensorflow/contrib/distributions/python/ops/bijectors/permute.py
new file mode 100644
index 0000000000..a187ce22d6
--- /dev/null
+++ b/tensorflow/contrib/distributions/python/ops/bijectors/permute.py
@@ -0,0 +1,29 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Permute bijector."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+# go/tf-wildcard-import
+# pylint: disable=wildcard-import
+from tensorflow.contrib.distributions.python.ops.bijectors.permute_impl import *
+# pylint: enable=wildcard-import
+from tensorflow.python.util.all_util import remove_undocumented
+
+_allowed_symbols = ["Permute"]
+
+remove_undocumented(__name__, _allowed_symbols)
diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/permute_impl.py b/tensorflow/contrib/distributions/python/ops/bijectors/permute_impl.py
new file mode 100644
index 0000000000..b1d8f2f41b
--- /dev/null
+++ b/tensorflow/contrib/distributions/python/ops/bijectors/permute_impl.py
@@ -0,0 +1,138 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Permutation bijectors."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import tensor_util
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import check_ops
+from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import nn_ops
+from tensorflow.python.ops.distributions import bijector as bijector_lib
+
+
+__all__ = [
+    "Permute",
+]
+
+
+class Permute(bijector_lib.Bijector):
+  """Permutes the rightmost dimension of a `Tensor`.
+
+  ```python
+  bs = tf.contrib.distributions.bijectors
+
+  reverse = bs.Permute(permutation=[2, 1, 0])
+
+  reverse.forward([-1., 0., 1.])
+  # ==> [1., 0., -1]
+
+  reverse.inverse([1., 0., -1])
+  # ==> [-1., 0., 1.]
+
+  reverse.forward_log_det_jacobian(any_value)
+  # ==> 0.
+
+  reverse.inverse_log_det_jacobian(any_value)
+  # ==> 0.
+  ```
+
+  Warning: `tf.estimator` may repeatedly build the graph thus
+  `Permute(np.random.permutation(event_size)).astype("int32"))` is not a
+  reliable parameterization (nor would it be even if using `tf.constant`). A
+  safe alternative is to use `tf.get_variable` to achieve "init once" behavior,
+  i.e.,
+
+  ```python
+  def init_once(x, name):
+    return tf.get_variable(name, initializer=x, trainable=False)
+
+  Permute(permutation=init_once(
+      np.random.permutation(event_size).astype("int32"),
+      name="permutation"))
+  ```
+
+  """
+
+  def __init__(self, permutation, validate_args=False, name=None):
+    """Creates the `Permute` bijector.
+
+    Args:
+      permutation: An `int`-like vector-shaped `Tensor` representing the
+        permutation to apply to the rightmost dimension of the transformed
+        `Tensor`.
+      validate_args: Python `bool` indicating whether arguments should be
+        checked for correctness.
+      name: Python `str`, name given to ops managed by this object.
+
+    Raises:
+      TypeError: if `not permutation.dtype.is_integer`.
+      ValueError: if `permutation` does not contain exactly one of each of
+        `{0, 1, ..., d}`.
+    """
+    with ops.name_scope(name, "permute", values=[permutation]):
+      permutation = ops.convert_to_tensor(
+          permutation,
+          name="permutation")
+      if not permutation.dtype.is_integer:
+        raise TypeError("permutation.dtype ({}) should be `int`-like.".format(
+            permutation.dtype.name))
+      p = tensor_util.constant_value(permutation)
+      if p is not None:
+        if set(p) != set(np.arange(p.size)):
+          raise ValueError("Permutation over `d` must contain exactly one of "
+                           "each of `{0, 1, ..., d}`.")
+      elif validate_args:
+        p, _ = nn_ops.top_k(-permutation,
+                            k=array_ops.shape(permutation)[-1],
+                            sorted=True)
+        permutation = control_flow_ops.with_dependencies([
+            check_ops.assert_equal(
+                -p, math_ops.range(array_ops.size(p)),
+                message=("Permutation over `d` must contain exactly one of "
+                         "each of `{0, 1, ..., d}`.")),
+        ], permutation)
+      self._permutation = permutation
+      super(Permute, self).__init__(
+          is_constant_jacobian=True,
+          validate_args=validate_args,
+          name=name or "permute")
+
+  @property
+  def permutation(self):
+    return self._permutation
+
+  def _forward(self, x):
+    return array_ops.gather(x, self.permutation, axis=-1)
+
+  def _inverse(self, y):
+    return array_ops.gather(
+        y,
+        array_ops.invert_permutation(self.permutation),
+        axis=-1)
+
+  def _inverse_log_det_jacobian(self, y):
+    return constant_op.constant(0., dtype=y.dtype)
+
+  def _forward_log_det_jacobian(self, x):
+    return constant_op.constant(0., dtype=x.dtype)
-- 
GitLab


From e1d9e4ed05420ab9486bf18ec331e90e59e51982 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 11 Oct 2017 10:03:01 -0700
Subject: [PATCH 264/909] Add gradient for SVD. This is based on draft code by
 Catalin Ionescu (cdi@google.com), using the algorithm outlined in Mike Giles'
 paper: http://eprints.maths.ox.ac.uk/1079/1/NA-08-01.pdf.

This initial version has the following restrictions:
  Only supports statically known inner matrix dimensions m and n.

Backpropagating through U and V (i.e. backpropagating through SVD nodes with compute_uv=True) has further restrictions:
  a) Only supports real tensors.
  b) Only supports square and "almost square" matrices where the number of rows and columns differ by at most 1.
  c) full_matrices must be true also. This does not currently have severe implications, given the restriction in b).

Feature request on Github:
#6503

This CL also adds support for calling tf.real, tf.imag, and tf.angle with real arguments.

PiperOrigin-RevId: 171836140
---
 .../python/kernel_tests/cwise_ops_test.py     |  20 +--
 .../kernel_tests/self_adjoint_eig_op_test.py  |  12 +-
 tensorflow/python/kernel_tests/svd_op_test.py |  83 ++++++++++++
 tensorflow/python/ops/linalg_grad.py          | 120 ++++++++++++++++++
 tensorflow/python/ops/math_ops.py             |  56 ++++----
 5 files changed, 253 insertions(+), 38 deletions(-)

diff --git a/tensorflow/python/kernel_tests/cwise_ops_test.py b/tensorflow/python/kernel_tests/cwise_ops_test.py
index 4feb3b64e2..e0c53950e6 100644
--- a/tensorflow/python/kernel_tests/cwise_ops_test.py
+++ b/tensorflow/python/kernel_tests/cwise_ops_test.py
@@ -1927,15 +1927,17 @@ class ComplexMakeRealImagTest(test.TestCase):
 
   def _compareRealImag(self, cplx, use_gpu):
     np_real, np_imag = np.real(cplx), np.imag(cplx)
-    with self.test_session(use_gpu=use_gpu) as sess:
+    np_zeros = np_real * 0
+    with self.test_session(use_gpu=use_gpu):
       inx = ops.convert_to_tensor(cplx)
       tf_real = math_ops.real(inx)
       tf_imag = math_ops.imag(inx)
-      tf_real_val, tf_imag_val = sess.run([tf_real, tf_imag])
-    self.assertAllEqual(np_real, tf_real_val)
-    self.assertAllEqual(np_imag, tf_imag_val)
-    self.assertShapeEqual(np_real, tf_real)
-    self.assertShapeEqual(np_imag, tf_imag)
+      tf_real_real = math_ops.real(tf_real)
+      tf_imag_real = math_ops.imag(tf_real)
+      self.assertAllEqual(np_real, tf_real.eval())
+      self.assertAllEqual(np_imag, tf_imag.eval())
+      self.assertAllEqual(np_real, tf_real_real.eval())
+      self.assertAllEqual(np_zeros, tf_imag_real.eval())
 
   def testRealImag64(self):
     real = (np.arange(-3, 3) / 4.).reshape([1, 3, 2]).astype(np.float32)
@@ -2116,21 +2118,21 @@ class AccumulateTest(test.TestCase):
       with self.assertRaises(ValueError):
         a = variables.Variable(0.2)
         b = variables.Variable(0.1)
-        tf_val = math_ops.accumulate_n([a,b], shape=[2,2]) # Should be shape=[]
+        math_ops.accumulate_n([a, b], shape=[2, 2])  # Should be shape=[]
 
   def testWrongType(self):
     with self.test_session():
       with self.assertRaises(TypeError):
         a = variables.Variable(0.2, dtype=np.float32)
         b = variables.Variable(0.1, dtype=np.float32)
-        tf_val = math_ops.accumulate_n([a,b], tensor_dtype=np.int32) 
+        math_ops.accumulate_n([a, b], tensor_dtype=np.int32)
 
   def testWrongTypeOneInput(self):
     # Scenario that used to trigger a bug, even when testWrongType() worked
     with self.test_session():
       with self.assertRaises(TypeError):
         a = variables.Variable(0.2, dtype=np.float32)
-        tf_val = math_ops.accumulate_n([a], tensor_dtype=np.int32) 
+        math_ops.accumulate_n([a], tensor_dtype=np.int32)
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/kernel_tests/self_adjoint_eig_op_test.py b/tensorflow/python/kernel_tests/self_adjoint_eig_op_test.py
index 33032f0e59..4de5f4e4db 100644
--- a/tensorflow/python/kernel_tests/self_adjoint_eig_op_test.py
+++ b/tensorflow/python/kernel_tests/self_adjoint_eig_op_test.py
@@ -190,13 +190,17 @@ def _GetSelfAdjointEigGradTest(dtype_, shape_, compute_v_):
         tf_e, tf_v = linalg_ops.self_adjoint_eig(tf_a)
         # (complex) Eigenvectors are only unique up to an arbitrary phase
         # We normalize the vectors such that the first component has phase 0.
-        reference = tf_v / linalg_ops.norm(
-            tf_v[..., 0:1, :], axis=-1, keep_dims=True)
-        tf_v *= math_ops.conj(reference)
+        top_rows = tf_v[..., 0:1, :]
+        if tf_a.dtype.is_complex:
+          angle = -math_ops.angle(top_rows)
+          phase = math_ops.complex(math_ops.cos(angle), math_ops.sin(angle))
+        else:
+          phase = math_ops.sign(top_rows)
+        tf_v *= phase
         outputs = [tf_e, tf_v]
       else:
         tf_e = linalg_ops.self_adjoint_eigvals(tf_a)
-        outputs = [tf_e,]
+        outputs = [tf_e]
       for b in outputs:
         x_init = np.random.uniform(
             low=-1.0, high=1.0, size=n * n).reshape([n, n]).astype(np_dtype)
diff --git a/tensorflow/python/kernel_tests/svd_op_test.py b/tensorflow/python/kernel_tests/svd_op_test.py
index bda31f2892..9871eacb03 100644
--- a/tensorflow/python/kernel_tests/svd_op_test.py
+++ b/tensorflow/python/kernel_tests/svd_op_test.py
@@ -22,6 +22,7 @@ import numpy as np
 
 from tensorflow.python.framework import constant_op
 from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import gradient_checker
 from tensorflow.python.ops import linalg_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import random_ops
@@ -185,6 +186,74 @@ def _GetSvdOpTest(dtype_, shape_, use_static_shape_, compute_uv_,
   return Test
 
 
+class SvdGradOpTest(test.TestCase):
+  pass  # Filled in below
+
+
+def _GetSvdGradOpTest(dtype_, shape_, compute_uv_):
+
+  def _NormalizingSvd(tf_a):
+    tf_s, tf_u, tf_v = linalg_ops.svd(tf_a, compute_uv=True, full_matrices=True)
+    # Singular vectors are only unique up to an arbitrary phase. We normalize
+    # the vectors such that the first component of u (if m >=n) or v (if n > m)
+    # have phase 0.
+    m = tf_a.shape[-2]
+    n = tf_a.shape[-1]
+    if m >= n:
+      top_rows = tf_u[..., 0:1, :]
+    else:
+      top_rows = tf_v[..., 0:1, :]
+    if tf_u.dtype.is_complex:
+      angle = -math_ops.angle(top_rows)
+      phase = math_ops.complex(math_ops.cos(angle), math_ops.sin(angle))
+    else:
+      phase = math_ops.sign(top_rows)
+    tf_u *= phase[..., :m]
+    tf_v *= phase[..., :n]
+    return tf_s, tf_u, tf_v
+
+  def Test(self):
+    np.random.seed(42)
+    a = np.random.uniform(low=-1.0, high=1.0, size=shape_).astype(dtype_)
+    if dtype_ in [np.complex64, np.complex128]:
+      a += 1j * np.random.uniform(
+          low=-1.0, high=1.0, size=shape_).astype(dtype_)
+    # Optimal stepsize for central difference is O(epsilon^{1/3}).
+    # See Equation (21) in:
+    # http://www.karenkopecky.net/Teaching/eco613614/Notes_NumericalDifferentiation.pdf
+    # TODO(rmlarsen): Move step size control to gradient checker.
+    epsilon = np.finfo(dtype_).eps
+    delta = 0.1 * epsilon**(1.0 / 3.0)
+    if dtype_ in [np.float32, np.complex64]:
+      tol = 3e-2
+    else:
+      tol = 1e-6
+    with self.test_session(use_gpu=True):
+      tf_a = constant_op.constant(a)
+      if compute_uv_:
+        tf_s, tf_u, tf_v = _NormalizingSvd(tf_a)
+        outputs = [tf_s, tf_u, tf_v]
+      else:
+        tf_s = linalg_ops.svd(tf_a, compute_uv=False)
+        outputs = [tf_s]
+      for b in outputs:
+        x_init = np.random.uniform(
+            low=-1.0, high=1.0, size=shape_).astype(dtype_)
+        if dtype_ in [np.complex64, np.complex128]:
+          x_init += 1j * np.random.uniform(
+              low=-1.0, high=1.0, size=shape_).astype(dtype_)
+        theoretical, numerical = gradient_checker.compute_gradient(
+            tf_a,
+            tf_a.get_shape().as_list(),
+            b,
+            b.get_shape().as_list(),
+            x_init_value=x_init,
+            delta=delta)
+        self.assertAllClose(theoretical, numerical, atol=tol, rtol=tol)
+
+  return Test
+
+
 if __name__ == "__main__":
   for compute_uv in False, True:
     for full_matrices in False, True:
@@ -200,4 +269,18 @@ if __name__ == "__main__":
                 _AddTest(SvdOpTest, "Svd", name,
                          _GetSvdOpTest(dtype, shape, use_static_shape,
                                        compute_uv, full_matrices))
+  for compute_uv in False, True:
+    dtypes = ([np.float32, np.float64] + [np.complex64, np.complex128] *
+              (not compute_uv))
+    for dtype in dtypes:
+      mat_shapes = ([(10, 11), (11, 10),
+                     (11, 11)] + [(5, 11), (11, 5)] * (not compute_uv))
+      for mat_shape in mat_shapes:
+        for batch_dims in [(), (3,)]:
+          shape = batch_dims + mat_shape
+          name = "%s_%s_compute_uv_%s" % (dtype.__name__,
+                                          "_".join(map(str, shape)), compute_uv)
+          _AddTest(SvdGradOpTest, "SvdGrad", name,
+                   _GetSvdGradOpTest(dtype, shape, compute_uv))
+
   test.main()
diff --git a/tensorflow/python/ops/linalg_grad.py b/tensorflow/python/ops/linalg_grad.py
index b5e4e0e0af..1752164d7a 100644
--- a/tensorflow/python/ops/linalg_grad.py
+++ b/tensorflow/python/ops/linalg_grad.py
@@ -234,3 +234,123 @@ def _SelfAdjointEigV2Grad(op, grad_e, grad_v):
     grad_a = array_ops.matrix_set_diag(grad_a,
                                        0.5 * array_ops.matrix_diag_part(grad_a))
     return grad_a
+
+
+@ops.RegisterGradient("Svd")
+def _SvdGrad(op, grad_s, grad_u, grad_v):
+  """Gradient for Svd based on Giles' algorithm. Reference at top of file."""
+
+  def _Adjoint(x):
+    return math_ops.conj(array_ops.matrix_transpose(x))
+
+  if op.get_attr("compute_uv") and not op.get_attr("full_matrices"):
+    raise NotImplementedError(
+        "SVD gradient is not implemented for compute_uv=True and "
+        "full_matrices=False.")
+
+  a = op.inputs[0]
+  a_shape = a.get_shape().with_rank_at_least(2)
+
+  if op.get_attr("compute_uv"):
+    # TODO(rmlarsen): Make this work with complex types.
+    if a.dtype.is_complex:
+      raise NotImplementedError(
+          "SVD gradient is not implemented for complex types and "
+          "compute_uv=True.")
+    grad_u_shape = grad_u.get_shape().with_rank_at_least(2)
+    grad_v_shape = grad_v.get_shape().with_rank_at_least(2)
+    m = a_shape[-2].merge_with(grad_u_shape[-2])
+    n = a_shape[-1].merge_with(grad_v_shape[-2])
+    batch_shape = a_shape[:-2].merge_with(grad_u_shape[:-2]).merge_with(
+        grad_v_shape[:-2])
+    a_shape = batch_shape.concatenate([m, n])
+
+  m = a_shape[-2].value
+  n = a_shape[-1].value
+  # TODO(rmlarsen): Make this work with placeholders.
+  if m is None or n is None:
+    raise NotImplementedError(
+        "SVD gradient has not been implemented for input with unknown "
+        "inner matrix shape.")
+
+  if not op.get_attr("full_matrices") or not op.get_attr("compute_uv"):
+    s, u, v = linalg_ops.svd(a, compute_uv=True, full_matrices=True)
+  else:
+    s = op.outputs[0]
+    u = op.outputs[1]
+    v = op.outputs[2]
+
+  use_adjoint = False
+  if m > n:
+    # Compute the gradient for A^H = V * S^T * U^H, and (implicitly) take the
+    # Hermitian transpose of the gradient at the end.
+    use_adjoint = True
+    m, n = n, m
+    u, v = v, u
+    grad_u, grad_v = grad_v, grad_u
+
+  with ops.control_dependencies([grad_s, grad_u, grad_v]):
+    grad_s_mat = array_ops.matrix_diag(grad_s)
+    if not op.get_attr("compute_uv"):
+      if use_adjoint:
+        grad_a = math_ops.matmul(
+            v[..., :, :m], math_ops.matmul(u, grad_s_mat), adjoint_b=True)
+      else:
+        grad_a = math_ops.matmul(u,
+                                 math_ops.matmul(
+                                     grad_s_mat, v[..., :, :m], adjoint_b=True))
+      grad_a.set_shape(a_shape)
+      return grad_a
+
+    # TODO(rmlarsen): Define a gradient that is numerically stable for
+    # abs(m-n) > 1. Currently this does not work because there are effectively
+    # multiple singular values with value zero. I am not sure if this is a true
+    # instability or if it simply throws off the finite difference gradient
+    # checker.
+    if abs(m - n) > 1:
+      raise NotImplementedError(
+          "svd gradient is not implemented for abs(m - n) > 1")
+    s_mat = array_ops.matrix_diag(s)
+    s2 = math_ops.square(s)
+
+    # NOTICE: Because of the term involving f, the gradient becomes
+    # infinite (or NaN in practice) when singular values are not unique.
+    # Mathematically this should not be surprising, since for (k-fold)
+    # degenerate singular values, the corresponding singular vectors are
+    # only defined up a (k-dimensional) subspace. In practice, this can
+    # lead to numerical instability when singular values are close but not
+    # exactly equal.
+    f = array_ops.matrix_set_diag(
+        math_ops.reciprocal(
+            array_ops.expand_dims(s2, -2) - array_ops.expand_dims(s2, -1)),
+        array_ops.zeros_like(s))
+    s_inv_mat = array_ops.matrix_diag(math_ops.reciprocal(s))
+    u_gu = math_ops.matmul(u, grad_u, adjoint_a=True)
+    v_gv = math_ops.matmul(v, grad_v, adjoint_a=True)
+
+    if m == n:
+      f_u = f * u_gu
+      f_v = f * v_gv
+    else:
+      dv2 = array_ops.matrix_transpose(v_gv[..., m:n, :m]) - v_gv[..., :m, m:n]
+      f_u = f * u_gu
+      f_v = f * v_gv[..., :m, :m]
+
+    grad_a_nouv = (
+        grad_s_mat + math_ops.matmul(f_u + _Adjoint(f_u), s_mat) +
+        math_ops.matmul(s_mat, f_v + _Adjoint(f_v)))
+
+    if m != n:
+      grad_a_nouv = array_ops.concat(
+          [grad_a_nouv, math_ops.matmul(s_inv_mat, dv2)], -1)
+
+    if use_adjoint:
+      # Use (U X V^H)^H = V (U X)^H.
+      grad_a = math_ops.matmul(
+          v, math_ops.matmul(u, grad_a_nouv), adjoint_b=True)
+    else:
+      grad_a = math_ops.matmul(u,
+                               math_ops.matmul(grad_a_nouv, v, adjoint_b=True))
+
+    grad_a.set_shape(a_shape)
+    return grad_a
diff --git a/tensorflow/python/ops/math_ops.py b/tensorflow/python/ops/math_ops.py
index b572377e2f..101eee95f1 100644
--- a/tensorflow/python/ops/math_ops.py
+++ b/tensorflow/python/ops/math_ops.py
@@ -589,13 +589,10 @@ def complex(real, imag, name=None):
 
 
 def real(input, name=None):
-  r"""Returns the real part of a complex number.
+  r"""Returns the real part of a complex (or real) tensor.
 
-  Given a tensor `input` of complex numbers, this operation returns a tensor of
-  type `float32` or `float64` that is the real part of each element in `input`.
-  All elements in `input` must be complex numbers of the form \\(a + bj\\),
-  where *a* is the real part returned by this operation and *b* is the
-  imaginary part.
+  Given a tensor `input`, this operation returns a tensor of type `float` that
+  is the real part of each element in `input` considered as a complex number.
 
   For example:
 
@@ -614,19 +611,19 @@ def real(input, name=None):
     A `Tensor` of type `float32` or `float64`.
   """
   with ops.name_scope(name, "Real", [input]) as name:
-    real_dtype = input.dtype.real_dtype
-    if input.dtype.base_dtype == real_dtype:
+    if input.dtype.is_complex:
+      real_dtype = input.dtype.real_dtype
+      return gen_math_ops.real(input, Tout=real_dtype, name=name)
+    else:
       return input
-    return gen_math_ops.real(input, Tout=real_dtype, name=name)
 
 
 def imag(input, name=None):
-  r"""Returns the imaginary part of a complex number.
+  r"""Returns the imaginary part of a complex (or real) tensor.
 
-  Given a tensor `input` of complex numbers, this operation returns a tensor of
-  type `float` that is the argument of each element in `input`. All elements in
-  `input` must be complex numbers of the form \\(a + bj\\), where *a*
-  is the real part and *b* is the imaginary part returned by the operation.
+  Given a tensor `input`, this operation returns a tensor of type `float` that
+  is the imaginary part of each element in `input` considered as a complex
+  number. If `input` is real, a tensor of all zeros is returned.
 
   For example:
 
@@ -636,26 +633,32 @@ def imag(input, name=None):
   ```
 
   Args:
-    input: A `Tensor`. Must be one of the following types: `complex64`,
-      `complex128`.
+    input: A `Tensor`. Must be one of the following types: `float`, `double`,
+      `complex64`, `complex128`.
     name: A name for the operation (optional).
 
   Returns:
     A `Tensor` of type `float32` or `float64`.
   """
   with ops.name_scope(name, "Imag", [input]) as name:
-    return gen_math_ops.imag(input, Tout=input.dtype.real_dtype, name=name)
+    if input.dtype.is_complex:
+      return gen_math_ops.imag(input, Tout=input.dtype.real_dtype, name=name)
+    else:
+      return array_ops.zeros_like(input)
 
 
 def angle(input, name=None):
-  r"""Returns the argument of a complex number.
+  r"""Returns the element-wise argument of a complex (or real) tensor.
 
-  Given a tensor `input` of complex numbers, this operation returns a tensor of
-  type `float32` or `float64` that is the argument of each element in `input`.
-  All elements in `input` must be complex numbers of the form \\(a + bj\\),
-  where *a* is the real part and *b* is the imaginary part.
+  Given a tensor `input`, this operation returns a tensor of type `float` that
+  is the argument of each element in `input` considered as a complex number.
+
+  The elements in `input` are considered to be complex numbers of the form
+  \\(a + bj\\), where *a* is the real part and *b* is the imaginary part.
+  If `input` is real then *b* is zero by definition.
 
   The argument returned by this function is of the form \\(atan2(b, a)\\).
+  If `input` is real, a tensor of all zeros is returned.
 
   For example:
 
@@ -665,15 +668,18 @@ def angle(input, name=None):
   ```
 
   Args:
-    input: A `Tensor`. Must be one of the following types: `complex64`,
-      `complex128`.
+    input: A `Tensor`. Must be one of the following types: `float`, `double`,
+      `complex64`, `complex128`.
     name: A name for the operation (optional).
 
   Returns:
     A `Tensor` of type `float32` or `float64`.
   """
   with ops.name_scope(name, "Angle", [input]) as name:
-    return gen_math_ops.angle(input, Tout=input.dtype.real_dtype, name=name)
+    if input.dtype.is_complex:
+      return gen_math_ops.angle(input, Tout=input.dtype.real_dtype, name=name)
+    else:
+      return array_ops.zeros_like(input)
 
 
 # pylint: enable=redefined-outer-name,redefined-builtin
-- 
GitLab


From f4bcfc00ce4b43675969a04960878d27a89a3af7 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 11 Oct 2017 10:48:48 -0700
Subject: [PATCH 265/909] Fixes typo in hparams comment

PiperOrigin-RevId: 171842961
---
 tensorflow/contrib/training/python/training/hparam.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/training/python/training/hparam.py b/tensorflow/contrib/training/python/training/hparam.py
index 119fa3824b..c95a73ce44 100644
--- a/tensorflow/contrib/training/python/training/hparam.py
+++ b/tensorflow/contrib/training/python/training/hparam.py
@@ -138,7 +138,7 @@ def _process_list_value(name, parse_fn, var_type, m_dict, values,
 
 
 def parse_values(values, type_map):
-  """Parses hyperparameter values from a string into a python map..
+  """Parses hyperparameter values from a string into a python map.
 
   `values` is a string containing comma-separated `name=value` pairs.
   For each pair, the value of the hyperparameter named `name` is set to
-- 
GitLab


From 4d69d0408da946096163ee1d8ea068ae6698ae9d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 11 Oct 2017 10:51:59 -0700
Subject: [PATCH 266/909] Implement NCHW_VECT_C support for tf.space_to_depth
 on GPU.

PiperOrigin-RevId: 171843463
---
 tensorflow/core/kernels/spacetodepth_op.cc    | 65 ++++++++++------
 tensorflow/core/kernels/spacetodepth_op.h     |  4 +
 .../core/kernels/spacetodepth_op_gpu.cu.cc    | 11 +++
 tensorflow/core/ops/array_ops.cc              |  6 +-
 .../kernel_tests/spacetodepth_op_test.py      | 74 +++++++++++--------
 5 files changed, 104 insertions(+), 56 deletions(-)

diff --git a/tensorflow/core/kernels/spacetodepth_op.cc b/tensorflow/core/kernels/spacetodepth_op.cc
index 14510add56..23df1c35e5 100644
--- a/tensorflow/core/kernels/spacetodepth_op.cc
+++ b/tensorflow/core/kernels/spacetodepth_op.cc
@@ -50,6 +50,9 @@ class SpaceToDepthOp : public OpKernel {
                 errors::InvalidArgument("Invalid data format"));
 
     OP_REQUIRES_OK(context, context->GetAttr("block_size", &block_size_));
+    OP_REQUIRES(context, block_size_ > 1,
+                errors::InvalidArgument("Block size should be > 1, but was: ",
+                                        block_size_));
 
     if (std::is_same<Device, CPUDevice>::value) {
       OP_REQUIRES(
@@ -57,21 +60,22 @@ class SpaceToDepthOp : public OpKernel {
           errors::InvalidArgument(
               "Only NHWC data_format supported on CPU. Got ", data_format_str));
     }
-
-    OP_REQUIRES(
-        context, block_size_ > 1,
-        errors::InvalidArgument("Block size should be > 1: ", block_size_));
   }
 
   void Compute(OpKernelContext* context) override {
     const Tensor& input = context->input(0);
     const int dims = input.dims();
 
-    // Check on the input dimensions first.
-    // The input is presumed to be [batch, height, width, depth]
-    constexpr int kRequiredDims = 4;
-    OP_REQUIRES(context, kRequiredDims == dims,
-                errors::InvalidArgument("Input rank should be: ", kRequiredDims,
+    // Assuming qint8 <--> NCHW_VECT_C, OIHW_VECT_I (int8x4) here.
+    constexpr bool is_int8x4 = std::is_same<T, qint8>::value;
+    OP_REQUIRES(context, (is_int8x4 == (data_format_ == FORMAT_NCHW_VECT_C)),
+                errors::InvalidArgument(
+                    "qint8 should be used with data_format NCHW_VECT_C."));
+
+    constexpr int kVect = is_int8x4 ? 4 : 1;
+    constexpr int kDims = is_int8x4 ? 5 : 4;
+    OP_REQUIRES(context, kDims == dims,
+                errors::InvalidArgument("Input rank should be: ", kDims,
                                         " instead of: ", dims));
 
     constexpr int kNumSpatialDims = 2;
@@ -82,7 +86,8 @@ class SpaceToDepthOp : public OpKernel {
     const int width =
         input.dim_size(GetTensorDimIndex<kNumSpatialDims>(data_format_, 'W'));
     const int input_depth =
-        input.dim_size(GetTensorDimIndex<kNumSpatialDims>(data_format_, 'C'));
+        input.dim_size(GetTensorDimIndex<kNumSpatialDims>(data_format_, 'C')) *
+        kVect;
 
     // Both width and height must be divisible by block_size.
     OP_REQUIRES(context,
@@ -91,11 +96,9 @@ class SpaceToDepthOp : public OpKernel {
                     "Image width ", width, " and height ", height,
                     " should be divisible by block_size: ", block_size_));
 
-    const int block_size_sq = block_size_ * block_size_;
-
     // The 'spatial' block of size block_size_ X block_size_ will be moved
     // to depth.
-    const int output_depth = input_depth * block_size_sq;
+    const int output_depth = input_depth * block_size_ * block_size_;
     const int output_width = width / block_size_;
     const int output_height = height / block_size_;
 
@@ -108,17 +111,30 @@ class SpaceToDepthOp : public OpKernel {
                                        output_width, output_depth),
                        &outputs_tensor));
 
-    auto Toutput = outputs_tensor->tensor<T, 4>();
-    auto Tinput = input.tensor<T, 4>();
+    auto Tinput = input.tensor<T, kDims>();
+    auto Toutput = outputs_tensor->tensor<T, kDims>();
+
+    if (std::is_same<Device, GPUDevice>::value) {
+      if (is_int8x4) {
+        // NCHW_VECT_C with 4 x qint8 can be treated as NCHW int32.
+        auto Tinput_v = input.template reinterpret_last_dimension<int32, 4>();
+        auto Toutput_v = outputs_tensor->reinterpret_last_dimension<int32, 4>();
+        functor::SpaceToDepthOpFunctor<GPUDevice, int32, FORMAT_NCHW> functor;
+        functor(context->eigen_device<GPUDevice>(), Tinput_v, block_size_,
+                Toutput_v);
+        return;
+      } else if (data_format_ == FORMAT_NCHW) {
+        functor::SpaceToDepthOpFunctor<GPUDevice, T, FORMAT_NCHW> functor;
+        functor(context->eigen_device<GPUDevice>(), Tinput, block_size_,
+                Toutput);
+        return;
+      }
+    }
 
-    if (std::is_same<Device, GPUDevice>::value && data_format_ == FORMAT_NCHW) {
-      functor::SpaceToDepthOpFunctor<Device, T, FORMAT_NCHW> functor;
-      functor(context->eigen_device<Device>(), Tinput, block_size_, Toutput);
-    } else {
-      // TODO(pauldonnelly): Implement NCHW_VECT_C version for GPU.
-      OP_REQUIRES(
-          context, data_format_ == FORMAT_NHWC,
-          errors::InvalidArgument(ToString(data_format_), " not implemented"));
+    // NOTE: Assumes data_format_ == FORMAT_NHWC here, since we have rejected
+    // (CPU && data_format_ != FORMAT_NHWC) in the constructor.
+
+    if (!is_int8x4) {
       functor::SpaceToDepthOpFunctor<Device, T, FORMAT_NHWC> functor;
       functor(context->eigen_device<Device>(), Tinput, block_size_, Toutput);
     }
@@ -171,6 +187,9 @@ TF_CALL_ALL_TYPES(REGISTER);
 REGISTER_KERNEL_BUILDER(
     Name("SpaceToDepth").Device(DEVICE_GPU).TypeConstraint<float>("T"),
     SpaceToDepthOp<GPUDevice, float>);
+REGISTER_KERNEL_BUILDER(
+    Name("SpaceToDepth").Device(DEVICE_GPU).TypeConstraint<qint8>("T"),
+    SpaceToDepthOp<GPUDevice, qint8>);
 #endif  // GOOGLE_CUDA
 
 }  // end namespace tensorflow
diff --git a/tensorflow/core/kernels/spacetodepth_op.h b/tensorflow/core/kernels/spacetodepth_op.h
index 11321633ab..9d28be5feb 100644
--- a/tensorflow/core/kernels/spacetodepth_op.h
+++ b/tensorflow/core/kernels/spacetodepth_op.h
@@ -45,6 +45,10 @@ template <typename Device, typename T, TensorFormat data_format>
 struct SpaceToDepthOpFunctor {
   void operator()(const Device& d, typename TTypes<T, 4>::ConstTensor input,
                   int block_size, typename TTypes<T, 4>::Tensor output);
+
+  // This 5-D version is to support NCHW_VECT_C.
+  void operator()(const Device& d, typename TTypes<T, 5>::ConstTensor input,
+                  int block_size, typename TTypes<T, 5>::Tensor output);
 };
 
 }  // namespace functor
diff --git a/tensorflow/core/kernels/spacetodepth_op_gpu.cu.cc b/tensorflow/core/kernels/spacetodepth_op_gpu.cu.cc
index b2e45d346d..94c7a0a3f6 100644
--- a/tensorflow/core/kernels/spacetodepth_op_gpu.cu.cc
+++ b/tensorflow/core/kernels/spacetodepth_op_gpu.cu.cc
@@ -122,6 +122,10 @@ struct SpaceToDepthOpFunctor<GPUDevice, T, FORMAT_NHWC> {
         input_height, input_width, input_depth, output_height, output_width,
         output_depth, output.data());
   }
+  void operator()(const GPUDevice& d, typename TTypes<T, 5>::ConstTensor input,
+                  int block_size, typename TTypes<T, 5>::Tensor output) {
+    LOG(FATAL) << "5-D tensors should not be used with NHWC format";
+  }
 };
 
 template <typename T>
@@ -141,6 +145,10 @@ struct SpaceToDepthOpFunctor<GPUDevice, T, FORMAT_NCHW> {
         config.virtual_thread_count, input.data(), block_size, output_width,
         input_depth * output_height, output.data());
   }
+  void operator()(const GPUDevice& d, typename TTypes<T, 5>::ConstTensor input,
+                  int block_size, typename TTypes<T, 5>::Tensor output) {
+    LOG(FATAL) << "5-D tensors should not be used with NCHW format";
+  }
 };
 }  // end namespace functor
 
@@ -148,6 +156,9 @@ struct SpaceToDepthOpFunctor<GPUDevice, T, FORMAT_NCHW> {
 template struct functor::SpaceToDepthOpFunctor<GPUDevice, float, FORMAT_NCHW>;
 template struct functor::SpaceToDepthOpFunctor<GPUDevice, float, FORMAT_NHWC>;
 
+// NCHW_VECT_C with 4 x qint8 can be treated as NCHW int32.
+template struct functor::SpaceToDepthOpFunctor<GPUDevice, int32, FORMAT_NCHW>;
+
 }  // end namespace tensorflow
 
 #endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/ops/array_ops.cc b/tensorflow/core/ops/array_ops.cc
index fec27c7c1c..108c29ed6e 100644
--- a/tensorflow/core/ops/array_ops.cc
+++ b/tensorflow/core/ops/array_ops.cc
@@ -4084,13 +4084,15 @@ REGISTER_OP("SpaceToDepth")
       TensorFormat data_format;
       FormatFromString(data_format_str, &data_format);
 
+      constexpr int num_spatial_dims = 2;
+      const int dims =
+          GetTensorDimsFromSpatialDims(num_spatial_dims, data_format);
       ShapeHandle input;
-      TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 4, &input));
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(0), dims, &input));
 
       int32 block_size;
       TF_RETURN_IF_ERROR(c->GetAttr("block_size", &block_size));
 
-      constexpr int num_spatial_dims = 2;
       DimensionHandle batch_size =
           c->Dim(input, GetTensorDimIndex<num_spatial_dims>(data_format, 'N'));
       DimensionHandle input_height =
diff --git a/tensorflow/python/kernel_tests/spacetodepth_op_test.py b/tensorflow/python/kernel_tests/spacetodepth_op_test.py
index 195cca6325..4a9353d6bf 100644
--- a/tensorflow/python/kernel_tests/spacetodepth_op_test.py
+++ b/tensorflow/python/kernel_tests/spacetodepth_op_test.py
@@ -25,9 +25,11 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import gen_array_ops
 from tensorflow.python.ops import gradient_checker
 from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import test
+from tensorflow.python.platform import tf_logging
 
 
 class SpaceToDepthTest(test.TestCase):
@@ -222,44 +224,54 @@ class SpaceToDepthTest(test.TestCase):
       tensor = array_ops.reshape(tensor, [b, oc, oh, ow])
     return tensor
 
-  def compareToTranspose(self, data_format, use_gpu):
-    if use_gpu and not test.is_gpu_available():
-      print("gpu not available")
-      return
-
-    dtype = dtypes.float32
-    batch_size = 3
-    height = 4
-    width = 6
-    channels = 4
-    block_size = 2
-
-    if data_format == "NHWC":
-      input_shape = [batch_size, height, width, channels]
-    elif data_format == "NCHW":
-      input_shape = [batch_size, channels, height, width]
+  def compareToTranspose(self, batch_size, out_height, out_width, in_channels,
+                         block_size, data_format, use_gpu):
+    in_height = out_height * block_size
+    in_width = out_width * block_size
+    nhwc_input_shape = [batch_size, in_height, in_width, in_channels]
+    nchw_input_shape = [batch_size, in_channels, in_height, in_width]
+    total_size = np.prod(nhwc_input_shape)
+
+    if data_format == "NCHW_VECT_C":
+      # Initialize the input tensor with qint8 values that circle -127..127.
+      x = [((f + 128) % 255) - 127 for f in range(total_size)]
+      t = constant_op.constant(x, shape=nhwc_input_shape, dtype=dtypes.float32)
+      expected = self.spaceToDepthUsingTranspose(t, block_size, "NHWC")
+      t = test_util.NHWCToNCHW_VECT_C(t)
+      t, _, _ = gen_array_ops.quantize_v2(t, -128.0, 127.0, dtypes.qint8)
+      t = array_ops.space_to_depth(t, block_size, data_format="NCHW_VECT_C")
+      t = gen_array_ops.dequantize(t, -128, 127)
+      actual = test_util.NCHW_VECT_CToNHWC(t)
     else:
-      print("unsupported format")
-
-    # Initialize the input tensor with ascending whole numbers.
-    total_size = 1
-    for dim_size in input_shape:
-      total_size *= dim_size
-    x = [f for f in range(total_size)]
-    inputs = constant_op.constant(x, shape=input_shape, dtype=dtype)
-
-    expected = self.spaceToDepthUsingTranspose(inputs, block_size, data_format)
-    actual = array_ops.space_to_depth(
-        inputs, block_size, data_format=data_format)
+      # Initialize the input tensor with ascending whole numbers as floats.
+      x = [f * 1.0 for f in range(total_size)]
+      shape = nchw_input_shape if data_format == "NCHW" else nhwc_input_shape
+      t = constant_op.constant(x, shape=shape, dtype=dtypes.float32)
+      expected = self.spaceToDepthUsingTranspose(t, block_size, data_format)
+      actual = array_ops.space_to_depth(t, block_size, data_format=data_format)
 
     with self.test_session(use_gpu=use_gpu) as sess:
       actual_vals, expected_vals = sess.run([actual, expected])
       self.assertTrue(np.array_equal(actual_vals, expected_vals))
 
   def testAgainstTranspose(self):
-    self.compareToTranspose("NHWC", False)
-    self.compareToTranspose("NHWC", True)
-    self.compareToTranspose("NCHW", True)
+    self.compareToTranspose(3, 2, 3, 1, 2, "NHWC", False)
+    self.compareToTranspose(1, 2, 3, 2, 2, "NHWC", False)
+    self.compareToTranspose(1, 2, 3, 2, 3, "NHWC", False)
+
+    if not test.is_gpu_available():
+      tf_logging.info("skipping gpu tests since gpu not available")
+      return
+
+    self.compareToTranspose(3, 2, 3, 1, 2, "NHWC", True)
+    self.compareToTranspose(3, 2, 3, 2, 2, "NHWC", True)
+    self.compareToTranspose(3, 2, 3, 1, 2, "NCHW", True)
+    self.compareToTranspose(3, 2, 3, 2, 3, "NCHW", True)
+    self.compareToTranspose(5, 7, 11, 3, 2, "NCHW", True)
+
+    self.compareToTranspose(3, 2, 3, 4, 2, "NCHW_VECT_C", True)
+    self.compareToTranspose(3, 2, 3, 8, 3, "NCHW_VECT_C", True)
+    self.compareToTranspose(5, 7, 11, 12, 2, "NCHW_VECT_C", True)
 
 
 class SpaceToDepthGradientTest(test.TestCase):
-- 
GitLab


From 4a139397e8f4b3cbd50240cfd914bac9db476965 Mon Sep 17 00:00:00 2001
From: Eugene Brevdo <ebrevdo@google.com>
Date: Wed, 11 Oct 2017 11:01:29 -0700
Subject: [PATCH 267/909] More Variant cross-device support:

* Remove HostConstraint for ops taking Variants; they can now be copied from/to Device.
* Add ResourceVariable assign operations that support variants.

PiperOrigin-RevId: 171845029
---
 tensorflow/core/framework/register_types.h    |  4 ++
 tensorflow/core/kernels/aggregate_ops.cc      | 10 +---
 tensorflow/core/kernels/constant_op.cc        |  9 +---
 tensorflow/core/kernels/cwise_op_conj.cc      | 11 ++--
 .../core/kernels/resource_variable_ops.cc     | 53 ++++++++++++++++++-
 tensorflow/core/kernels/shape_op_test.cc      |  4 +-
 tensorflow/core/kernels/shape_ops.cc          | 44 ++-------------
 tensorflow/python/ops/state_ops.py            |  2 +-
 8 files changed, 68 insertions(+), 69 deletions(-)

diff --git a/tensorflow/core/framework/register_types.h b/tensorflow/core/framework/register_types.h
index 3f9c307d03..61e722e57b 100644
--- a/tensorflow/core/framework/register_types.h
+++ b/tensorflow/core/framework/register_types.h
@@ -19,6 +19,7 @@ limitations under the License.
 
 #include "tensorflow/core/framework/numeric_types.h"
 #include "tensorflow/core/framework/resource_handle.h"
+#include "tensorflow/core/framework/variant.h"
 #include "tensorflow/core/platform/types.h"
 
 // Two sets of macros:
@@ -67,6 +68,7 @@ limitations under the License.
 #define TF_CALL_int8(m) m(::tensorflow::int8)
 #define TF_CALL_string(m) m(string)
 #define TF_CALL_resource(m) m(::tensorflow::ResourceHandle)
+#define TF_CALL_variant(m) m(::tensorflow::Variant)
 #define TF_CALL_complex64(m) m(::tensorflow::complex64)
 #define TF_CALL_int64(m) m(::tensorflow::int64)
 #define TF_CALL_uint64(m) m(::tensorflow::uint64)
@@ -96,6 +98,7 @@ limitations under the License.
 #define TF_CALL_int8(m)
 #define TF_CALL_string(m)
 #define TF_CALL_resource(m)
+#define TF_CALL_variant(m)
 #define TF_CALL_complex64(m)
 #define TF_CALL_int64(m) m(::tensorflow::int64)
 #define TF_CALL_uint64(m)
@@ -125,6 +128,7 @@ limitations under the License.
 #define TF_CALL_int8(m)
 #define TF_CALL_string(m)
 #define TF_CALL_resource(m)
+#define TF_CALL_variant(m)
 #define TF_CALL_complex64(m)
 #define TF_CALL_int64(m)
 #define TF_CALL_uint64(m)
diff --git a/tensorflow/core/kernels/aggregate_ops.cc b/tensorflow/core/kernels/aggregate_ops.cc
index 0099984f69..2f125312d0 100644
--- a/tensorflow/core/kernels/aggregate_ops.cc
+++ b/tensorflow/core/kernels/aggregate_ops.cc
@@ -229,6 +229,7 @@ REGISTER_ADDN_CPU(Variant);
 TF_CALL_GPU_NUMBER_TYPES(REGISTER_ADDN_GPU);
 TF_CALL_complex64(REGISTER_ADDN_GPU);
 TF_CALL_complex128(REGISTER_ADDN_GPU);
+TF_CALL_variant(REGISTER_ADDN_GPU);
 #undef REGISTER_ADDN_GPU
 
 // A special GPU kernel for int32.
@@ -241,15 +242,6 @@ REGISTER_KERNEL_BUILDER(Name("AddN")
                             .HostMemory("sum"),
                         AddNOp<CPUDevice, int32>);
 
-// TODO(ebrevdo): Once rendezvous has been properly set up for
-// Variants, we'll no longer need a HostMemory attribute for this case.
-REGISTER_KERNEL_BUILDER(Name("AddN")
-                            .Device(DEVICE_GPU)
-                            .TypeConstraint<Variant>("T")
-                            .HostMemory("inputs")
-                            .HostMemory("sum"),
-                        AddNOp<GPUDevice, Variant>);
-
 #endif  // GOOGLE_CUDA
 
 #ifdef TENSORFLOW_USE_SYCL
diff --git a/tensorflow/core/kernels/constant_op.cc b/tensorflow/core/kernels/constant_op.cc
index 018ace5485..72132574a4 100644
--- a/tensorflow/core/kernels/constant_op.cc
+++ b/tensorflow/core/kernels/constant_op.cc
@@ -333,19 +333,12 @@ REGISTER_KERNEL(double, GPU);
 REGISTER_KERNEL(complex64, GPU);
 REGISTER_KERNEL(complex128, GPU);
 REGISTER_KERNEL(int64, GPU);
+REGISTER_KERNEL(Variant, GPU);
 REGISTER_KERNEL_BUILDER(Name("ZerosLike")
                             .Device(DEVICE_GPU)
                             .TypeConstraint<int32>("T")
                             .HostMemory("y"),
                         ZerosLikeOp<CPUDevice, int32>);
-// TODO(ebrevdo): Once rendezvous has been properly set up for
-// Variants, we'll no longer need a HostMemory attribute for this case.
-REGISTER_KERNEL_BUILDER(Name("ZerosLike")
-                            .Device(DEVICE_GPU)
-                            .TypeConstraint<Variant>("T")
-                            .HostMemory("x")
-                            .HostMemory("y"),
-                        ZerosLikeOp<GPUDevice, Variant>);
 #endif  // GOOGLE_CUDA
 
 #undef REGISTER_KERNEL
diff --git a/tensorflow/core/kernels/cwise_op_conj.cc b/tensorflow/core/kernels/cwise_op_conj.cc
index 2ab8c42c53..929c54a9a1 100644
--- a/tensorflow/core/kernels/cwise_op_conj.cc
+++ b/tensorflow/core/kernels/cwise_op_conj.cc
@@ -23,14 +23,9 @@ REGISTER2(UnaryOp, CPU, "Conj", functor::conj, complex64, complex128);
 REGISTER_VARIANT(UnaryVariantOp, CPU, "Conj", CONJ_VARIANT_UNARY_OP);
 
 #if GOOGLE_CUDA
-// TODO(ebrevdo): Once rendezvous has been properly set up for
-// Variants, we'll no longer need a HostMemory attribute for this case.
-REGISTER_KERNEL_BUILDER(Name("Conj")
-                            .Device(DEVICE_GPU)
-                            .TypeConstraint<Variant>("T")
-                            .HostMemory("input")
-                            .HostMemory("output"),
-                        UnaryVariantOp<GPUDevice, CONJ_VARIANT_UNARY_OP>);
+REGISTER_KERNEL_BUILDER(
+    Name("Conj").Device(DEVICE_GPU).TypeConstraint<Variant>("T"),
+    UnaryVariantOp<GPUDevice, CONJ_VARIANT_UNARY_OP>);
 REGISTER_KERNEL_BUILDER(
     Name("Conj").Device(DEVICE_GPU).TypeConstraint<complex64>("T"),
     UnaryOp<GPUDevice, functor::conj<complex64>>);
diff --git a/tensorflow/core/kernels/resource_variable_ops.cc b/tensorflow/core/kernels/resource_variable_ops.cc
index e45abb6c56..3cca493972 100644
--- a/tensorflow/core/kernels/resource_variable_ops.cc
+++ b/tensorflow/core/kernels/resource_variable_ops.cc
@@ -127,7 +127,7 @@ REGISTER_KERNEL_BUILDER(
                               .Device(DEVICE_GPU)              \
                               .HostMemory("resource")          \
                               .TypeConstraint<type>("dtype"),  \
-                          ResourceHandleOp<Var>)               \
+                          ResourceHandleOp<Var>)
 
 TF_CALL_GPU_ALL_TYPES(REGISTER_GPU_KERNELS);
 #undef REGISTER_GPU_KERNELS
@@ -272,6 +272,56 @@ class AssignVariableOp : public OpKernel {
   DataType dtype_;
 };
 
+template <typename Device>
+class AssignVariableOp<Device, Variant> : public OpKernel {
+ public:
+  explicit AssignVariableOp(OpKernelConstruction* c) : OpKernel(c) {
+    OP_REQUIRES_OK(c, c->GetAttr("dtype", &dtype_));
+    OP_REQUIRES(c, dtype_ == DT_VARIANT,
+                errors::Internal("Variant kernel called with dtype: ",
+                                 DataTypeString(dtype_)));
+  }
+
+  void Compute(OpKernelContext* context) override {
+    const Tensor& value = context->input(1);
+    OP_REQUIRES(context, dtype_ == value.dtype(),
+                errors::InvalidArgument(
+                    "Variable and value dtypes don't match; respectively, ",
+                    dtype_, " and ", context->input(1).dtype()));
+
+    Var* variable = nullptr;
+    OP_REQUIRES_OK(context, LookupOrCreateResource<Var>(
+                                context, HandleFromInput(context, 0), &variable,
+                                [this, context](Var** ptr) {
+                                  *ptr = new Var(dtype_);
+                                  // Create an empty new Variant tensor.
+                                  return Status::OK();
+                                }));
+    core::ScopedUnref s(variable);
+
+    OP_REQUIRES(context, variable->tensor()->dtype() == DT_VARIANT,
+                errors::InvalidArgument(
+                    "Trying to assign variable with wrong dtype. Expected ",
+                    DataTypeString(variable->tensor()->dtype()), " got ",
+                    DataTypeString(DT_VARIANT)));
+
+    mutex_lock ml(*variable->mu());
+    // TODO(ebrevdo): Add a proper Variant deep copy / assign registry
+    // entry and use that here.  For now, use a serialization
+    // roundtrip to perform the copy on CPU.  This is OK because this
+    // op is not registered for GPU.
+    *variable->tensor() = Tensor();
+    TensorProto tmp;
+    value.AsProtoTensorContent(&tmp);
+    OP_REQUIRES(context, variable->tensor()->FromProto(tmp),
+                errors::Internal("Could not properly reserialize values "
+                                 "Variant.  Check logs for more details."));
+  }
+
+ private:
+  DataType dtype_;
+};
+
 #define REGISTER_KERNELS(type)                                \
   REGISTER_KERNEL_BUILDER(Name("AssignVariableOp")            \
                               .Device(DEVICE_CPU)             \
@@ -280,6 +330,7 @@ class AssignVariableOp : public OpKernel {
 
 TF_CALL_ALL_TYPES(REGISTER_KERNELS);
 TF_CALL_QUANTIZED_TYPES(REGISTER_KERNELS);
+TF_CALL_variant(REGISTER_KERNELS);
 #undef REGISTER_KERNELS
 
 #if GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/shape_op_test.cc b/tensorflow/core/kernels/shape_op_test.cc
index 96eaa4ac75..a545fb146c 100644
--- a/tensorflow/core/kernels/shape_op_test.cc
+++ b/tensorflow/core/kernels/shape_op_test.cc
@@ -68,7 +68,9 @@ static void ExpectHasError(const Status& s, const string& substr) {
 }
 
 TEST_F(ShapeOpTest, Simple) {
-  Scope root = Scope::NewRootScope();
+  // Ensure the ops run on CPU, as we have no device copy registration
+  // for NoKnownShape and KnownVecSize objects.
+  Scope root = Scope::NewRootScope().WithDevice("/cpu:0");
 
   // Use a placeholder so the graph optimizer doesn't optimize away
   // the shape function.
diff --git a/tensorflow/core/kernels/shape_ops.cc b/tensorflow/core/kernels/shape_ops.cc
index 98cd208576..721f9b949b 100644
--- a/tensorflow/core/kernels/shape_ops.cc
+++ b/tensorflow/core/kernels/shape_ops.cc
@@ -84,6 +84,7 @@ REGISTER_KERNEL_BUILDER(Name("Shape")
 
 TF_CALL_NUMBER_TYPES_NO_INT32(REGISTER_GPU_KERNEL);
 TF_CALL_bool(REGISTER_GPU_KERNEL);
+TF_CALL_variant(REGISTER_GPU_KERNEL);
 #undef REGISTER_GPU_KERNEL
 
 // A special GPU kernel for int32.
@@ -104,23 +105,6 @@ REGISTER_KERNEL_BUILDER(Name("Shape")
                             .TypeConstraint<int64>("out_type"),
                         ShapeOp<int64>);
 
-// TODO(ebrevdo): Once rendezvous has been properly set up for
-// Variants, we'll no longer need a HostMemory attribute for this case.
-REGISTER_KERNEL_BUILDER(Name("Shape")
-                            .Device(DEVICE_GPU)
-                            .HostMemory("input")
-                            .HostMemory("output")
-                            .TypeConstraint<Variant>("T")
-                            .TypeConstraint<int32>("out_type"),
-                        ShapeOp<int32>);
-REGISTER_KERNEL_BUILDER(Name("Shape")
-                            .Device(DEVICE_GPU)
-                            .HostMemory("input")
-                            .HostMemory("output")
-                            .TypeConstraint<Variant>("T")
-                            .TypeConstraint<int64>("out_type"),
-                        ShapeOp<int64>);
-
 #endif  // GOOGLE_CUDA
 
 // ShapeN ---------------------------------------
@@ -245,6 +229,7 @@ REGISTER_KERNEL_BUILDER(Name("Rank")
                               .HostMemory("output"),     \
                           RankOp);
 TF_CALL_NUMBER_TYPES_NO_INT32(REGISTER_GPU_KERNEL);
+TF_CALL_variant(REGISTER_GPU_KERNEL);
 #undef REGISTER_GPU_KERNEL
 
 // A special GPU kernel for int32 and bool.
@@ -264,14 +249,6 @@ REGISTER_KERNEL_BUILDER(Name("Rank")
                             .HostMemory("output"),
                         RankOp);
 
-// TODO(ebrevdo): Once rendezvous has been properly set up for
-// Variants, we'll no longer need a HostMemory attribute for this case.
-REGISTER_KERNEL_BUILDER(Name("Rank")
-                            .Device(DEVICE_GPU)
-                            .TypeConstraint<Variant>("T")
-                            .HostMemory("input")
-                            .HostMemory("output"),
-                        RankOp);
 #endif  // GOOGLE_CUDA
 
 // Size ------------------------------------------
@@ -302,6 +279,7 @@ REGISTER_KERNEL_BUILDER(Name("Size")
                           SizeOp<int64>);
 TF_CALL_NUMBER_TYPES_NO_INT32(REGISTER_GPU_KERNEL);
 TF_CALL_bool(REGISTER_GPU_KERNEL);
+TF_CALL_variant(REGISTER_GPU_KERNEL);
 #undef REGISTER_GPU_KERNEL
 
 // A special GPU kernel for int32.
@@ -322,22 +300,6 @@ REGISTER_KERNEL_BUILDER(Name("Size")
                             .HostMemory("output"),
                         SizeOp<int64>);
 
-// TODO(ebrevdo): Once rendezvous has been properly set up for
-// Variants, we'll no longer need a HostMemory attribute for this case.
-REGISTER_KERNEL_BUILDER(Name("Size")
-                            .Device(DEVICE_GPU)
-                            .TypeConstraint<Variant>("T")
-                            .TypeConstraint<int32>("out_type")
-                            .HostMemory("input")
-                            .HostMemory("output"),
-                        SizeOp<int32>);
-REGISTER_KERNEL_BUILDER(Name("Size")
-                            .Device(DEVICE_GPU)
-                            .TypeConstraint<Variant>("T")
-                            .TypeConstraint<int64>("out_type")
-                            .HostMemory("input")
-                            .HostMemory("output"),
-                        SizeOp<int64>);
 #endif  // GOOGLE_CUDA
 
 #ifdef TENSORFLOW_USE_SYCL
diff --git a/tensorflow/python/ops/state_ops.py b/tensorflow/python/ops/state_ops.py
index f54bbfe90e..65ec2d4b77 100644
--- a/tensorflow/python/ops/state_ops.py
+++ b/tensorflow/python/ops/state_ops.py
@@ -124,7 +124,7 @@ def variable_op_v2(shape, dtype, name="Variable", container="", shared_name=""):
       with this shared_name. Otherwise, the node name is used instead.
 
   Returns:
-    A variable tensor.1;5A
+    A variable tensor.
   """
   return gen_state_ops._variable_v2(shape=shape,
                                     dtype=dtype,
-- 
GitLab


From 208dabe771f049b0b331f14adf5d8728a7eae931 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 11 Oct 2017 11:48:23 -0700
Subject: [PATCH 268/909] Build file cleanup for iOS.

PiperOrigin-RevId: 171853263
---
 tensorflow/BUILD | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/tensorflow/BUILD b/tensorflow/BUILD
index 5bb31d7df1..8d9089115d 100644
--- a/tensorflow/BUILD
+++ b/tensorflow/BUILD
@@ -120,6 +120,15 @@ config_setting(
     visibility = ["//visibility:public"],
 )
 
+config_setting(
+    name = "ios_x86_64",
+    values = {
+        "cc_target_os": "apple",
+        "cpu": "ios_x86_64",
+    },
+    visibility = ["//visibility:public"],
+)
+
 config_setting(
     name = "linux_x86_64",
     values = {"cpu": "k8"},
-- 
GitLab


From 8c857092026c67d3868664daa4c2ee2d39f1b4dd Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Wed, 11 Oct 2017 13:04:02 -0700
Subject: [PATCH 269/909] [XLA:CPU] Add an in-place implementation of
 fused-dynamic-update-slice.

This implementation, which applies when a loop-fusion node's root is a
dynamic-update-slice whose input operand and output share the same
buffer slice, is much faster than the out-of-place implementation.

This patch also unifies the implementation of the CPU and GPU versions
of this algorithm.

PiperOrigin-RevId: 171863142
---
 .../compiler/xla/service/buffer_assignment.h  |   7 +
 tensorflow/compiler/xla/service/cpu/BUILD     |   1 +
 .../compiler/xla/service/cpu/ir_emitter.cc    | 137 ++++---------
 .../compiler/xla/service/cpu/ir_emitter.h     |   4 +
 tensorflow/compiler/xla/service/gpu/BUILD     |   1 +
 .../xla/service/gpu/ir_emitter_unnested.cc    | 113 ++---------
 tensorflow/compiler/xla/service/llvm_ir/BUILD |  18 ++
 .../compiler/xla/service/llvm_ir/ops.cc       | 181 ++++++++++++++++++
 tensorflow/compiler/xla/service/llvm_ir/ops.h |  80 ++++++++
 9 files changed, 348 insertions(+), 194 deletions(-)
 create mode 100644 tensorflow/compiler/xla/service/llvm_ir/ops.cc
 create mode 100644 tensorflow/compiler/xla/service/llvm_ir/ops.h

diff --git a/tensorflow/compiler/xla/service/buffer_assignment.h b/tensorflow/compiler/xla/service/buffer_assignment.h
index 688aff8912..08a53af8ba 100644
--- a/tensorflow/compiler/xla/service/buffer_assignment.h
+++ b/tensorflow/compiler/xla/service/buffer_assignment.h
@@ -320,6 +320,13 @@ class BufferAssignment {
                           const HloInstruction* hlo_b,
                           const ShapeIndex& shape_index_b) const;
 
+  // Returns true if the top-level buffers of hlo_a and hlo_b are the same.
+  // REQUIRES: HasTopLevelAllocation(hlo_a) && HasTopLevelAllocation(hlo_b).
+  bool SharesTopLevelSlice(const HloInstruction* hlo_a,
+                           const HloInstruction* hlo_b) const {
+    return SharesSliceAtIndex(hlo_a, {}, hlo_b, {});
+  }
+
   // Returns the underlying points-to analysis used for this assignment.
   const TuplePointsToAnalysis& points_to_analysis() const {
     return liveness_->points_to_analysis();
diff --git a/tensorflow/compiler/xla/service/cpu/BUILD b/tensorflow/compiler/xla/service/cpu/BUILD
index 7933e226bf..8ab358fe17 100644
--- a/tensorflow/compiler/xla/service/cpu/BUILD
+++ b/tensorflow/compiler/xla/service/cpu/BUILD
@@ -262,6 +262,7 @@ cc_library(
         "//tensorflow/compiler/xla/service/llvm_ir:llvm_loop",
         "//tensorflow/compiler/xla/service/llvm_ir:llvm_util",
         "//tensorflow/compiler/xla/service/llvm_ir:loop_emitter",
+        "//tensorflow/compiler/xla/service/llvm_ir:ops",
         "//tensorflow/compiler/xla/service/llvm_ir:tuple_ops",
         "//tensorflow/core:lib",
         "@llvm//:core",
diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
index 89a911d070..633ad0290c 100644
--- a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
@@ -48,6 +48,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/llvm_ir/fused_ir_emitter.h"
 #include "tensorflow/compiler/xla/service/llvm_ir/llvm_loop.h"
 #include "tensorflow/compiler/xla/service/llvm_ir/llvm_util.h"
+#include "tensorflow/compiler/xla/service/llvm_ir/ops.h"
 #include "tensorflow/compiler/xla/service/llvm_ir/tuple_ops.h"
 #include "tensorflow/compiler/xla/shape_util.h"
 #include "tensorflow/compiler/xla/status_macros.h"
@@ -2125,39 +2126,6 @@ Status IrEmitter::HandleDynamicSlice(HloInstruction* dynamic_slice,
   return DefaultAction(dynamic_slice);
 }
 
-namespace {
-
-// Checks if we can emit code for DynamicUpdateSlice to update data in-place.
-// Returns true if operand 0 of DynamicUpdateSlice and its output buffer
-// share the same buffer allocation.
-// Returns false otherwise.
-// TODO(b/64142684) Share code with GPU implementation.
-bool CanUpdateDynamicSliceInPlace(const BufferAssignment& assignment,
-                                  HloInstruction* dynamic_update_slice) {
-  CHECK_EQ(HloOpcode::kDynamicUpdateSlice, dynamic_update_slice->opcode());
-
-  // Walk DynamicUpdateSlice operand(0) to parameter and get its
-  // associated operand. See if it shares an allocation with this operand.
-  HloInstruction* operand;
-  ShapeIndex index;
-  std::tie(operand, index) =
-      dynamic_update_slice->mutable_operand(0)->LatestNonGteAncestorAndIndex();
-  if (operand->opcode() != HloOpcode::kParameter) {
-    return false;
-  }
-
-  BufferAllocation::Slice operand_slice =
-      assignment.GetUniqueSlice(operand, index).ConsumeValueOrDie();
-
-  BufferAllocation::Slice dynamic_update_slice_slice =
-      assignment.GetUniqueTopLevelSlice(dynamic_update_slice)
-          .ConsumeValueOrDie();
-
-  return operand_slice == dynamic_update_slice_slice;
-}
-
-}  // namespace
-
 Status IrEmitter::HandleDynamicUpdateSlice(HloInstruction* dynamic_update_slice,
                                            HloInstruction* operand,
                                            HloInstruction* update,
@@ -2165,60 +2133,13 @@ Status IrEmitter::HandleDynamicUpdateSlice(HloInstruction* dynamic_update_slice,
   if (ShapeUtil::IsScalar(dynamic_update_slice->shape())) {
     TF_RETURN_IF_ERROR(EmitTargetAddressForOp(dynamic_update_slice));
     return EmitMemcpy(*update, *dynamic_update_slice);
-  } else if (CanUpdateDynamicSliceInPlace(assignment_, dynamic_update_slice)) {
-    VLOG(2) << "Emitting HandleDynamicUpdateSlice in-place.";
-    // DynamicUpdateSlice's operand(0) and 'fusion' output share the same
-    // BufferAllocation::Slice, so it is safe to emit code to update the slice
-    // 'in-place'. This avoids copying data outside of the slice update region.
-    // TODO(b/64142684) Implement in-place update for fused DynamicUpdateSlice.
-
-    // Emit IR to read dynamic start indices from 'start_indices'.
-    const int64 rank = ShapeUtil::Rank(operand->shape());
-    llvm_ir::IrArray::Index start_index(rank);
-    for (int64 i = 0; i < rank; ++i) {
-      llvm_ir::IrArray::Index dim_index({ir_builder_.getInt64(i)});
-      llvm_ir::IrArray start_indices_array(GetIrArrayFor(start_indices));
-      start_index[i] =
-          start_indices_array.EmitReadArrayElement(dim_index, &ir_builder_);
-    }
-
-    // Create loop body emitter which emits code to do the following:
-    // *) Map requested 'index' and slice 'start_index' to input/output shape
-    //    as 'output_index'.
-    // *) Reads value from 'update'.
-    // *) Writes value to input/output array at 'output_index'.
-    auto loop_body_emitter =
-        [&](const llvm_ir::IrArray::Index& index) -> Status {
-      // Calculate 'output_index' at which to write value from update.
-      llvm_ir::IrArray::Index output_index(rank);
-      for (int64 i = 0; i < rank; ++i) {
-        // Emit IR which computes:
-        //   output_index = (start_index + index) % dim_size
-        llvm::Value* dim_size = llvm::ConstantInt::get(
-            index[i]->getType(), operand->shape().dimensions(i));
-        llvm::Value* start_index0 = ir_builder_.CreateZExtOrBitCast(
-            start_index[i], index[i]->getType());
-        output_index[i] = ir_builder_.CreateURem(
-            ir_builder_.CreateAdd(start_index0, index[i]), dim_size);
-      }
-
-      // Read value from 'update'.
-      llvm_ir::IrArray update_array(GetIrArrayFor(update));
-      llvm::Value* update_data =
-          update_array.EmitReadArrayElement(index, &ir_builder_);
-
-      // Write value to output array.
-      GetIrArrayFor(operand).EmitWriteArrayElement(output_index, update_data,
-                                                   &ir_builder_);
-      return Status::OK();
-    };
-
-    TF_RETURN_IF_ERROR(
-        llvm_ir::LoopEmitter(loop_body_emitter, update->shape(), &ir_builder_)
-            .EmitLoop(IrName(dynamic_update_slice, "in_place")));
-
+  } else if (llvm_ir::CanUpdateDynamicSliceInPlace(dynamic_update_slice,
+                                                   assignment_)) {
     TF_RETURN_IF_ERROR(EmitTargetAddressForOp(dynamic_update_slice));
-    return Status::OK();
+    auto operands = GetIrArraysForOperandsOf(dynamic_update_slice);
+    return llvm_ir::EmitDynamicUpdateSliceInPlace(
+        operands, GetIrArrayFor(dynamic_update_slice),
+        IrName(dynamic_update_slice, "in_place"), &ir_builder_);
   }
   return DefaultAction(dynamic_update_slice);
 }
@@ -2296,11 +2217,11 @@ static const HloInstruction* StripTranspose(const HloInstruction& hlo) {
 }
 
 Status IrEmitter::HandleFusion(HloInstruction* fusion) {
+  auto* root = fusion->fused_expression_root();
   if (fusion->fusion_kind() == HloInstruction::FusionKind::kTransposeDot) {
-    const HloInstruction* dot = fusion->fused_expression_root();
-    DCHECK(dot->opcode() == HloOpcode::kDot);
-    const HloInstruction* lhs_parameter = StripTranspose(*dot->operand(0));
-    const HloInstruction* rhs_parameter = StripTranspose(*dot->operand(1));
+    DCHECK(root->opcode() == HloOpcode::kDot);
+    const HloInstruction* lhs_parameter = StripTranspose(*root->operand(0));
+    const HloInstruction* rhs_parameter = StripTranspose(*root->operand(1));
     DCHECK(lhs_parameter->opcode() == HloOpcode::kParameter &&
            rhs_parameter->opcode() == HloOpcode::kParameter);
     const HloInstruction* lhs =
@@ -2309,7 +2230,7 @@ Status IrEmitter::HandleFusion(HloInstruction* fusion) {
         fusion->operand(rhs_parameter->parameter_number());
 
     TF_RETURN_IF_ERROR(ElementTypesSameAndSupported(
-        /*instruction=*/*dot, /*operands=*/{lhs, rhs},
+        /*instruction=*/*root, /*operands=*/{lhs, rhs},
         /*supported_types=*/{F32}));
 
     llvm_ir::IrArray lhs_array(GetIrArrayFor(lhs));
@@ -2328,17 +2249,25 @@ Status IrEmitter::HandleFusion(HloInstruction* fusion) {
 
     // Dot operation is complicated so we delegate to a helper class.
     TF_RETURN_IF_ERROR(DotOpEmitter::EmitDotOperation(
-        *dot, dot->operand(0)->IsRank2Transpose(),
-        dot->operand(1)->IsRank2Transpose(), target_array, lhs_array, rhs_array,
-        GetExecutableRunOptionsArgument(), &ir_builder_, hlo_module_config_));
+        *root, root->operand(0)->IsRank2Transpose(),
+        root->operand(1)->IsRank2Transpose(), target_array, lhs_array,
+        rhs_array, GetExecutableRunOptionsArgument(), &ir_builder_,
+        hlo_module_config_));
     return Status::OK();
+  } else if (llvm_ir::CanEmitFusedDynamicUpdateSliceInPlace(fusion,
+                                                            assignment_)) {
+    CpuElementalIrEmitter elemental_emitter(hlo_module_config_, this, module_);
+    TF_RETURN_IF_ERROR(EmitTargetAddressForOp(fusion));
+
+    // Delegate to common implementation of fused in-place dynamic-update-slice.
+    auto operands = GetIrArraysForOperandsOf(fusion);
+    return llvm_ir::EmitFusedDynamicUpdateSliceInPlace(
+        fusion, operands, GetIrArrayFor(fusion), &elemental_emitter,
+        &ir_builder_);
   } else if (fusion->fusion_kind() == HloInstruction::FusionKind::kLoop) {
-    std::vector<llvm_ir::IrArray> parameter_arrays;
-    for (HloInstruction* operand : fusion->operands()) {
-      parameter_arrays.push_back(GetIrArrayFor(operand));
-    }
     CpuElementalIrEmitter elemental_emitter(hlo_module_config_, this, module_);
-    FusedIrEmitter fused_emitter(parameter_arrays, &elemental_emitter);
+    auto operands = GetIrArraysForOperandsOf(fusion);
+    FusedIrEmitter fused_emitter(operands, &elemental_emitter);
     TF_RETURN_IF_ERROR(fusion->fused_expression_root()->Accept(&fused_emitter));
 
     return EmitTargetElementLoop(fusion, fused_emitter.GetRootGenerator());
@@ -2804,6 +2733,16 @@ llvm_ir::IrArray IrEmitter::GetIrArrayFor(const HloInstruction* hlo) {
   return array;
 }
 
+std::vector<llvm_ir::IrArray> IrEmitter::GetIrArraysForOperandsOf(
+    const HloInstruction* hlo) {
+  std::vector<llvm_ir::IrArray> arrays;
+  std::transform(
+      hlo->operands().begin(), hlo->operands().end(),
+      std::back_inserter(arrays),
+      [&](const HloInstruction* operand) { return GetIrArrayFor(operand); });
+  return arrays;
+}
+
 llvm::Value* IrEmitter::GetEmittedValueFor(const HloInstruction* hlo) {
   auto it = emitted_value_.find(hlo);
   if (it == emitted_value_.end()) {
diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.h b/tensorflow/compiler/xla/service/cpu/ir_emitter.h
index ba02f5f778..53c4b6f241 100644
--- a/tensorflow/compiler/xla/service/cpu/ir_emitter.h
+++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.h
@@ -236,6 +236,10 @@ class IrEmitter : public DfsHloVisitorWithDefault {
   // Gets an IrArray representing the given hlo.
   llvm_ir::IrArray GetIrArrayFor(const HloInstruction* hlo);
 
+  // Gets a list of IrArrays, one for each of hlo's operands.
+  std::vector<llvm_ir::IrArray> GetIrArraysForOperandsOf(
+      const HloInstruction* hlo);
+
   // Augments IrArray with aliasing information.
   void AddAliasingInformationToIrArray(const HloInstruction& hlo,
                                        llvm_ir::IrArray* array) {
diff --git a/tensorflow/compiler/xla/service/gpu/BUILD b/tensorflow/compiler/xla/service/gpu/BUILD
index 1d980405dd..de84e06ceb 100644
--- a/tensorflow/compiler/xla/service/gpu/BUILD
+++ b/tensorflow/compiler/xla/service/gpu/BUILD
@@ -146,6 +146,7 @@ cc_library(
         "//tensorflow/compiler/xla/service/llvm_ir:llvm_loop",
         "//tensorflow/compiler/xla/service/llvm_ir:llvm_util",
         "//tensorflow/compiler/xla/service/llvm_ir:loop_emitter",
+        "//tensorflow/compiler/xla/service/llvm_ir:ops",
         "//tensorflow/compiler/xla/service/llvm_ir:tuple_ops",
         "//tensorflow/core:lib",
         "//tensorflow/core:stream_executor_no_cuda",
diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
index cf41623a9b..120d50ed25 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
+++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
@@ -50,6 +50,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/hlo_opcode.h"
 #include "tensorflow/compiler/xla/service/llvm_ir/fused_ir_emitter.h"
 #include "tensorflow/compiler/xla/service/llvm_ir/llvm_util.h"
+#include "tensorflow/compiler/xla/service/llvm_ir/ops.h"
 #include "tensorflow/compiler/xla/service/llvm_ir/tuple_ops.h"
 #include "tensorflow/compiler/xla/service/name_uniquer.h"
 #include "tensorflow/compiler/xla/shape_util.h"
@@ -254,29 +255,6 @@ Status IrEmitterUnnested::HandleConvolution(HloInstruction* convolution,
                                       rhs_instruction, window);
 }
 
-// Checks if we can emit code for DynamicUpdateSlice to update data in-place.
-// Returns true if operand 0 of DynamicUpdateSlice and its output buffer
-// share the same buffer allocation.
-static bool CanUpdateDynamicSliceInPlace(const BufferAssignment& assignment,
-                                         HloInstruction* fusion) {
-  CHECK_EQ(HloOpcode::kFusion, fusion->opcode());
-  HloInstruction* fused_root = fusion->fused_expression_root();
-  if (fused_root->opcode() != HloOpcode::kDynamicUpdateSlice) {
-    return false;
-  }
-  // Walk DynamicUpdateSlice operand(0) to fused parameter and get its
-  // associated operand. See if it shares an allocation with this operand.
-  HloInstruction* fusion_operand;
-  ShapeIndex index;
-  std::tie(fusion_operand, index) =
-      fused_root->mutable_operand(0)->LatestNonGteAncestorAndIndex();
-  if (fusion_operand->opcode() != HloOpcode::kParameter) {
-    return false;
-  }
-  auto* operand = fusion->operand(fusion_operand->parameter_number());
-  return assignment.SharesSliceAtIndex(fusion, {}, operand, index);
-}
-
 Status IrEmitterUnnested::HandleFusion(HloInstruction* fusion) {
   HloInstruction* root = fusion->fused_expression_root();
   // HandleFusion specializes reduction from a multi-dimensional array to a 1D
@@ -347,95 +325,40 @@ Status IrEmitterUnnested::HandleFusion(HloInstruction* fusion) {
         LOG(FATAL) << "Bad opcode for input fusion: "
                    << fusion->fused_expression_root()->opcode();
     }
-  } else if (HloInstruction::FusionKind::kLoop == fusion->fusion_kind() &&
-             root->opcode() == HloOpcode::kDynamicUpdateSlice &&
-             CanUpdateDynamicSliceInPlace(
-                 ir_emitter_context_->buffer_assignment(), fusion)) {
-    // Loop fusion instruction with DynamicUpdateSlice as fused root.
-    // DynamicUpdateSlice's operand(0) and 'fusion' output share the same
-    // BufferAllocation::Slice, so it is safe to emit code to update the slice
-    // 'in-place'. This avoids copying data outside of the slice update region.
+  } else if (llvm_ir::CanEmitFusedDynamicUpdateSliceInPlace(
+                 fusion, ir_emitter_context_->buffer_assignment())) {
+    // Fusion node with dynamic-update-slice as the root where the op's input
+    // (i.e. array to update) shares the same slice as its output.  In this case
+    // we have a special algorithm that modifies the output in place without
+    // touching the un-updated elements.
 
     // Set up kernel thunk and fused ir emitter.
     thunk_sequence_->emplace_back(BuildKernelThunk(fusion));
-    std::vector<llvm_ir::IrArray> parameter_arrays;
+    std::vector<llvm_ir::IrArray> operand_arrays;
     for (HloInstruction* operand : fusion->operands()) {
-      parameter_arrays.push_back(GetIrArray(*operand));
+      operand_arrays.push_back(GetIrArray(*operand));
     }
     GpuElementalIrEmitter elemental_emitter(hlo_module_config_,
                                             ir_emitter_context_->llvm_module(),
                                             &ir_builder_, GetNestedComputer());
-    FusedIrEmitter fused_emitter(parameter_arrays, &elemental_emitter);
-    TF_RETURN_IF_ERROR(root->Accept(&fused_emitter));
-
-    // Recursively lookup 'fusion_operand' for DynamicUpdateSlice operand 0.
-    auto* fusion_operand = root->operand(0)->LatestNonGteAncestor();
-    CHECK_EQ(HloOpcode::kParameter, fusion_operand->opcode());
-
-    // Operand(0) the input array which shares an allocation with the output.
-    const auto* input = root->operand(0);
-    llvm::Value* input_base_ptr = fused_emitter.GetIrValueForGTE(input);
-    // Operand(1) 'update' is slice with which to update input at operand(0).
-    const auto* update = root->operand(1);
-    Shape update_shape = update->shape();
-    TF_RETURN_IF_ERROR(
-        LayoutUtil::CopyLayoutBetweenShapes(fusion->shape(), &update_shape));
-    // Operand(2) the dynamic slice indices at which to write 'update'.
-    const auto* start_indices = root->operand(2);
-
-    // Create element generators for 'update' and 'start_indices'.
-    llvm_ir::ElementGenerator element_generator =
-        fused_emitter.GetGenerator(update);
-    llvm_ir::ElementGenerator start_generator =
-        fused_emitter.GetGenerator(start_indices);
-
-    // Create loop body emitter which emits code to do the following:
-    // *) Read dynamic slice start indices into 'start_index'.
-    // *) Map requested 'index' and slice 'start_index' to input/output shape
-    //    as 'output_index'.
-    // *) Reads value from 'update' element generator.
-    // *) Writes value to input/output array at 'output_index'.
-    auto loop_body_emitter =
-        [=](const llvm_ir::IrArray::Index& index) -> Status {
-      // Emit IR to read dynamic start indices from hlo->operand(2).
-      const int64 rank = ShapeUtil::Rank(input->shape());
-      llvm_ir::IrArray::Index start_index(rank);
-      for (int64 i = 0; i < rank; ++i) {
-        llvm_ir::IrArray::Index dim_index({ir_builder_.getInt64(i)});
-        TF_ASSIGN_OR_RETURN(start_index[i], start_generator(dim_index));
-      }
 
-      // Calculate 'output_index' at which to write value from update.
-      llvm_ir::IrArray::Index output_index(rank);
-      for (int64 i = 0; i < rank; ++i) {
-        // Emit IR which computes:
-        //   output_index = (start_index + index) % dim_size
-        llvm::Value* dim_size = llvm::ConstantInt::get(
-            index[i]->getType(), input->shape().dimensions(i));
-        llvm::Value* start_index0 = ir_builder_.CreateZExtOrBitCast(
-            start_index[i], index[i]->getType());
-        output_index[i] = ir_builder_.CreateURem(
-            ir_builder_.CreateAdd(start_index0, index[i]), dim_size);
-      }
+    // Shape of the dynamic-update-slice's "update" operand.
+    Shape update_shape = root->operand(1)->shape();
 
-      // Read value from 'update'.
-      TF_ASSIGN_OR_RETURN(llvm::Value * input_value, element_generator(index));
-      // Write value to output array.
-      llvm_ir::IrArray(input_base_ptr, input->shape())
-          .EmitWriteArrayElement(output_index, input_value, &ir_builder_);
-      return Status::OK();
-    };
+    // Array to write into.  Because this is an in-place operation, this is the
+    // same as operand 0's array.
+    llvm_ir::IrArray output_array = GetIrArray(*fusion);
 
-    // Create loop which iterates over 'update' shape.
     LaunchDimensions launch_dimensions = CalculateLaunchDimensions(
         update_shape, ir_emitter_context_->device_description());
     CHECK(Thunk::Kind::kKernel == LastThunk()->kind());
     UpdateLaunchDimensions(launch_dimensions,
                            static_cast<KernelThunk*>(LastThunk()),
                            ir_emitter_context_->llvm_module());
-    return ParallelLoopEmitter(loop_body_emitter, update_shape,
-                               launch_dimensions, &ir_builder_)
-        .EmitLoop(IrName(fusion));
+
+    return llvm_ir::EmitParallelFusedDynamicUpdateSliceInPlace(
+        fusion, operand_arrays, output_array, &elemental_emitter,
+        launch_dimensions, &ir_builder_);
   }
   if (ImplementedAsGemm(*fusion)) {
     thunk_sequence_->emplace_back(BuildGemmThunk(fusion));
diff --git a/tensorflow/compiler/xla/service/llvm_ir/BUILD b/tensorflow/compiler/xla/service/llvm_ir/BUILD
index 62e404bd82..70579e3273 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/BUILD
+++ b/tensorflow/compiler/xla/service/llvm_ir/BUILD
@@ -123,6 +123,24 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "ops",
+    srcs = ["ops.cc"],
+    hdrs = ["ops.h"],
+    deps = [
+        ":fused_ir_emitter",
+        ":ir_array",
+        ":llvm_util",
+        ":loop_emitter",
+        "//tensorflow/compiler/xla/service:buffer_assignment",
+        "//tensorflow/compiler/xla/service:elemental_ir_emitter",
+        "//tensorflow/compiler/xla/service:hlo",
+        "//tensorflow/compiler/xla/service/gpu:parallel_loop_emitter",
+        "//tensorflow/compiler/xla/service/gpu:partition_assignment",
+        "@llvm//:core",
+    ],
+)
+
 cc_library(
     name = "tuple_ops",
     srcs = ["tuple_ops.cc"],
diff --git a/tensorflow/compiler/xla/service/llvm_ir/ops.cc b/tensorflow/compiler/xla/service/llvm_ir/ops.cc
new file mode 100644
index 0000000000..34899b7400
--- /dev/null
+++ b/tensorflow/compiler/xla/service/llvm_ir/ops.cc
@@ -0,0 +1,181 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/service/llvm_ir/ops.h"
+#include "tensorflow/compiler/xla/service/gpu/parallel_loop_emitter.h"
+#include "tensorflow/compiler/xla/service/gpu/partition_assignment.h"
+#include "tensorflow/compiler/xla/service/llvm_ir/fused_ir_emitter.h"
+#include "tensorflow/compiler/xla/service/llvm_ir/llvm_util.h"
+#include "tensorflow/compiler/xla/service/llvm_ir/loop_emitter.h"
+
+namespace xla {
+namespace llvm_ir {
+
+bool CanUpdateDynamicSliceInPlace(HloInstruction* dynamic_update_slice,
+                                  const BufferAssignment& assignment) {
+  CHECK_EQ(HloOpcode::kDynamicUpdateSlice, dynamic_update_slice->opcode());
+  const HloInstruction* operand = dynamic_update_slice->operand(0);
+  return assignment.HasTopLevelAllocation(dynamic_update_slice) &&
+         assignment.HasTopLevelAllocation(operand) &&
+         assignment.SharesTopLevelSlice(dynamic_update_slice, operand);
+}
+
+// Shared implementation of EmitDynamicUpdateSliceInPlace and
+// EmitFusedDynamicUpdateSliceInPlace.
+//
+// Emits a sequential loop if launch_dimensions is null.
+static Status EmitDynamicUpdateSliceInPlaceImpl(
+    const Shape& update_shape, const ElementGenerator& start_indices_generator,
+    ElementGenerator update_array_generator, const IrArray& output_array,
+    const gpu::LaunchDimensions* launch_dimensions,
+    tensorflow::StringPiece name, llvm::IRBuilder<>* ir_builder) {
+  const Shape& output_shape = output_array.GetShape();
+
+  // Read start indices from start_indices_generator.
+  const int64 rank = ShapeUtil::Rank(output_shape);
+  IrArray::Index start_index(rank);
+  for (int64 i = 0; i < rank; ++i) {
+    IrArray::Index dim_index({ir_builder->getInt64(i)});
+    TF_ASSIGN_OR_RETURN(start_index[i], start_indices_generator(dim_index));
+  }
+
+  auto loop_body_emitter = [&](const IrArray::Index& update_index) -> Status {
+    // Calculate output_index, where we'll write the value from update.  For
+    // each dimension,
+    //
+    //   output_index[dim] = (start_index[dim] + update_index[dim]) % dim_size.
+    //
+    IrArray::Index output_index(rank);
+    for (int64 i = 0; i < rank; ++i) {
+      llvm::Value* dim_size = llvm::ConstantInt::get(
+          update_index[i]->getType(), output_shape.dimensions(i));
+      llvm::Value* start_index0 = ir_builder->CreateZExtOrBitCast(
+          start_index[i], update_index[i]->getType());
+      output_index[i] = ir_builder->CreateURem(
+          ir_builder->CreateAdd(start_index0, update_index[i]), dim_size);
+    }
+
+    // Do output[output_index] = update[update_index].
+    TF_ASSIGN_OR_RETURN(llvm::Value * update_data,
+                        update_array_generator(update_index));
+    output_array.EmitWriteArrayElement(output_index, update_data, ir_builder);
+    return Status::OK();
+  };
+
+  if (launch_dimensions != nullptr) {
+    return gpu::ParallelLoopEmitter(loop_body_emitter, update_shape,
+                                    *launch_dimensions, ir_builder)
+        .EmitLoop(name);
+  }
+  return LoopEmitter(loop_body_emitter, update_shape, ir_builder)
+      .EmitLoop(name);
+}
+
+Status EmitDynamicUpdateSliceInPlace(
+    tensorflow::gtl::ArraySlice<IrArray> operand_arrays,
+    const IrArray& output_array, tensorflow::StringPiece name,
+    llvm::IRBuilder<>* ir_builder) {
+  VLOG(2) << "EmitDynamicUpdateSliceInPlace for " << name;
+
+  // No need to use operand_arrays[0], the input array of the
+  // dynamic-update-slice, because we know it aliases the op's output.
+  IrArray update_array = operand_arrays[1];
+  IrArray start_indices_array = operand_arrays[2];
+  Shape output_shape = output_array.GetShape();
+  Shape update_shape = update_array.GetShape();
+
+  ElementGenerator start_indices_generator = [&](const IrArray::Index& index) {
+    return start_indices_array.EmitReadArrayElement(index, ir_builder);
+  };
+  ElementGenerator update_array_generator = [&](const IrArray::Index& index) {
+    return update_array.EmitReadArrayElement(index, ir_builder);
+  };
+
+  return EmitDynamicUpdateSliceInPlaceImpl(
+      update_shape, start_indices_generator, update_array_generator,
+      output_array, /*launch_dimensions=*/nullptr, name, ir_builder);
+}
+
+// Shared implementation for EmitFusedDynamicUpdateSliceInPlace and
+// EmitParallelFusedDynamicUpdateSliceInPlace.
+//
+// Emits a sequential loop if launch_dimensions is null.
+static Status EmitFusedDynamicUpdateSliceInPlaceImpl(
+    HloInstruction* fusion,
+    tensorflow::gtl::ArraySlice<IrArray> fusion_operand_arrays,
+    const IrArray& fusion_output_array, ElementalIrEmitter* elemental_emitter,
+    const gpu::LaunchDimensions* launch_dimensions,
+    llvm::IRBuilder<>* ir_builder) {
+  CHECK_EQ(fusion->opcode(), HloOpcode::kFusion);
+  VLOG(2) << "EmitFusedDynamicUpdateSliceInPlace for "
+          << fusion->ToShortString();
+
+  auto* dynamic_update_slice = fusion->fused_expression_root();
+
+  const auto* update = dynamic_update_slice->operand(1);
+  const auto* start_indices = dynamic_update_slice->operand(2);
+  Shape update_shape = update->shape();
+
+  // Our in-place dynamic-update-slice implementation emits a loop over
+  // update_shape.  To emit a cache-friendly loop, we need to know that shape's
+  // layout.
+  //
+  // update_shape is inside a fusion node -- it's never materialized in memory
+  // and thus doesn't have a layout.  In this case we use the layout of the
+  // fusion node for iteration, since that corresponds to the order in memory of
+  // the buffer we'll be writing to.
+  //
+  // (This isn't necessarily optimal; in some cases it might be faster to peek
+  // through the chain of ops that gives us the update operand and use the
+  // layout of its source buffer(s).  But this is no worse than we do with
+  // fusion elsewhere.)
+  TF_RETURN_IF_ERROR(
+      LayoutUtil::CopyLayoutBetweenShapes(fusion->shape(), &update_shape));
+
+  // Create element generators for update and start_indices.
+  FusedIrEmitter fused_emitter(fusion_operand_arrays, elemental_emitter);
+  TF_RETURN_IF_ERROR(dynamic_update_slice->Accept(&fused_emitter));
+  ElementGenerator update_array_generator = fused_emitter.GetGenerator(update);
+  ElementGenerator start_indices_generator =
+      fused_emitter.GetGenerator(start_indices);
+
+  return EmitDynamicUpdateSliceInPlaceImpl(
+      update_shape, start_indices_generator, update_array_generator,
+      fusion_output_array, launch_dimensions, IrName(fusion), ir_builder);
+}
+
+Status EmitFusedDynamicUpdateSliceInPlace(
+    HloInstruction* fusion,
+    tensorflow::gtl::ArraySlice<IrArray> fusion_operand_arrays,
+    const IrArray& fusion_output_array, ElementalIrEmitter* elemental_emitter,
+    llvm::IRBuilder<>* ir_builder) {
+  return EmitFusedDynamicUpdateSliceInPlaceImpl(
+      fusion, fusion_operand_arrays, fusion_output_array, elemental_emitter,
+      /*launch_dimensions=*/nullptr, ir_builder);
+}
+
+Status EmitParallelFusedDynamicUpdateSliceInPlace(
+    HloInstruction* fusion,
+    tensorflow::gtl::ArraySlice<IrArray> fusion_operand_arrays,
+    const IrArray& fusion_output_array, ElementalIrEmitter* elemental_emitter,
+    const gpu::LaunchDimensions& launch_dimensions,
+    llvm::IRBuilder<>* ir_builder) {
+  return EmitFusedDynamicUpdateSliceInPlaceImpl(
+      fusion, fusion_operand_arrays, fusion_output_array, elemental_emitter,
+      &launch_dimensions, ir_builder);
+}
+
+}  // namespace llvm_ir
+}  // namespace xla
diff --git a/tensorflow/compiler/xla/service/llvm_ir/ops.h b/tensorflow/compiler/xla/service/llvm_ir/ops.h
new file mode 100644
index 0000000000..11e84d9cb5
--- /dev/null
+++ b/tensorflow/compiler/xla/service/llvm_ir/ops.h
@@ -0,0 +1,80 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef THIRD_PARTY_TENSORFLOW_COMPILER_XLA_SERVICE_LLVM_IR_OPS_H_
+#define THIRD_PARTY_TENSORFLOW_COMPILER_XLA_SERVICE_LLVM_IR_OPS_H_
+
+#include "tensorflow/compiler/xla/service/buffer_assignment.h"
+#include "tensorflow/compiler/xla/service/elemental_ir_emitter.h"
+#include "tensorflow/compiler/xla/service/gpu/partition_assignment.h"
+#include "tensorflow/compiler/xla/service/hlo_instruction.h"
+#include "tensorflow/compiler/xla/service/llvm_ir/ir_array.h"
+
+// Utilities related to emitting LLVM IR for various HLO ops.
+
+namespace xla {
+namespace llvm_ir {
+
+// Checks if we can emit code for the given DynamicUpdateSlice node that updates
+// its input in place.  Returns true if the dynamic-update-slice's
+// array-to-be-updated and output share the same BufferAllocation::Slice.
+//
+// dynamic_update_slice must be a DynamicUpdateSlice op.
+bool CanUpdateDynamicSliceInPlace(HloInstruction* dynamic_update_slice,
+                                  const BufferAssignment& assignment);
+
+// Checks if the given fusion node is amenable to being implemented by
+// EmitFusedDynamicUpdateSliceInPlace.
+inline bool CanEmitFusedDynamicUpdateSliceInPlace(
+    HloInstruction* fusion, const BufferAssignment& assignment) {
+  CHECK_EQ(fusion->opcode(), HloOpcode::kFusion);
+  return fusion->fusion_kind() == HloInstruction::FusionKind::kLoop &&
+         fusion->fused_expression_root()->opcode() ==
+             HloOpcode::kDynamicUpdateSlice &&
+         CanUpdateDynamicSliceInPlace(fusion->fused_expression_root(),
+                                      assignment);
+}
+
+// Emits IR for running the given dynamic-update-slice op in-place -- that is,
+// where the input and output buffers share the same slice, so we can simply
+// modify the input/output buffer without touching any of the other elements.
+Status EmitDynamicUpdateSliceInPlace(
+    tensorflow::gtl::ArraySlice<IrArray> operand_arrays,
+    const IrArray& output_array, tensorflow::StringPiece name,
+    llvm::IRBuilder<>* ir_builder);
+
+// Given a loop-fusion node whose root is a dynamic-update-slice op whose
+// array-to-be-updated and output share the same buffer slice, emits
+// (sequential) code for a fusion node that does the dynamic-update-slice in
+// place.
+Status EmitFusedDynamicUpdateSliceInPlace(
+    HloInstruction* fusion,
+    tensorflow::gtl::ArraySlice<IrArray> fusion_operand_arrays,
+    const IrArray& fusion_output_array, ElementalIrEmitter* elemental_emitter,
+    llvm::IRBuilder<>* ir_builder);
+
+// Same as EmitFusedDynamicUpdateSliceInPlace, except emits a parallel loop with
+// the given launch dimensions.
+Status EmitParallelFusedDynamicUpdateSliceInPlace(
+    HloInstruction* fusion,
+    tensorflow::gtl::ArraySlice<IrArray> fusion_operand_arrays,
+    const IrArray& fusion_output_array, ElementalIrEmitter* elemental_emitter,
+    const gpu::LaunchDimensions& launch_dimensions,
+    llvm::IRBuilder<>* ir_builder);
+
+}  // namespace llvm_ir
+}  // namespace xla
+
+#endif  // THIRD_PARTY_TENSORFLOW_COMPILER_XLA_SERVICE_LLVM_IR_OPS_H_
-- 
GitLab


From f640c8980571d7578e891ea5ceab55978c8db9b4 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 11 Oct 2017 14:37:44 -0700
Subject: [PATCH 270/909] Define Eager-safe Network to hold the composition of
 Layers.

PiperOrigin-RevId: 171876670
---
 tensorflow/contrib/eager/python/BUILD         |  25 +++
 tensorflow/contrib/eager/python/network.py    | 199 ++++++++++++++++++
 .../contrib/eager/python/network_test.py      | 107 ++++++++++
 3 files changed, 331 insertions(+)
 create mode 100644 tensorflow/contrib/eager/python/network.py
 create mode 100644 tensorflow/contrib/eager/python/network_test.py

diff --git a/tensorflow/contrib/eager/python/BUILD b/tensorflow/contrib/eager/python/BUILD
index 7ef163c707..94f21808a3 100644
--- a/tensorflow/contrib/eager/python/BUILD
+++ b/tensorflow/contrib/eager/python/BUILD
@@ -13,6 +13,7 @@ py_library(
     deps = [
         ":datasets",
         ":metrics",
+        ":network",
         ":saver",
         ":summary_writer",
         "//tensorflow/python:framework_ops",
@@ -148,6 +149,30 @@ py_test(
     ],
 )
 
+py_library(
+    name = "network",
+    srcs = ["network.py"],
+    srcs_version = "PY2AND3",
+    visibility = ["//tensorflow:internal"],
+    deps = [
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:layers_base",
+        "//tensorflow/python:variable_scope",
+    ],
+)
+
+py_test(
+    name = "network_test",
+    srcs = ["network_test.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":network",
+        "//tensorflow/python:constant_op",
+        "//tensorflow/python:layers",
+        "//tensorflow/python/eager:test",
+    ],
+)
+
 filegroup(
     name = "all_files",
     srcs = glob(
diff --git a/tensorflow/contrib/eager/python/network.py b/tensorflow/contrib/eager/python/network.py
new file mode 100644
index 0000000000..bebc595df0
--- /dev/null
+++ b/tensorflow/contrib/eager/python/network.py
@@ -0,0 +1,199 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""A Network is a composition of Layers."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import collections
+import uuid
+
+import six
+
+from tensorflow.python.framework import ops
+from tensorflow.python.layers import base
+from tensorflow.python.ops import variable_scope
+
+
+class Network(base.Layer):
+  """Represents the composition of a set of Layers.
+
+  TODO(josh11b,ashankar):
+  - Should "trainable" be changeable on the Network object?
+  - Do we allow add_variable in Network?
+  - Layer.name and Layer.variables.names are not in sync today
+    d = tf.layers.Dense(1)
+    d(tf.constant([[1.]]))
+    print(d.name)
+    print(d.variables)
+  - Note that name provided to __init__ is only for error messages?
+  - Detect layers used in __call__ that weren't registered with add_layer.
+  - Convert inputs to __call__ to tensors.
+  - Prevent variables from being created after the first __call__?
+    (Think about restoring from a checkpoint).
+  - Save & restore
+  """
+
+  def __init__(self, name=None):
+    super(Network, self).__init__(name=name)
+    self._container = uuid.uuid4().hex
+    self._layers = collections.OrderedDict()
+
+  def add_layer(self, layer):
+    """Add a Layer to this Network.
+
+    `Network` requires that all `Layer`s used in `call()` be added so that the
+    `Network` can export a complete list of variables.
+
+    Args:
+      layer: A `tf.layers.Layer` object.
+
+    Returns:
+      The passed in `layer`.
+
+    Raises:
+      RuntimeError: If __init__ has not been called.
+      TypeError: If layer is the wrong type.
+      ValueError: If a layer with the same name has already been added.
+    """
+    if not hasattr(self, "_layers"):
+      raise RuntimeError("Need to call Network.__init__ before adding layers")
+    if not isinstance(layer, base.Layer):
+      raise TypeError(
+          "Network.add_layer() passed type %s, not a tf.layers.Layer" %
+          (type(layer),))
+    if layer.name in self._layers:
+      if self._layers[layer.name] is layer:
+        return layer
+      raise ValueError(
+          "Attempt to add two Layers with the name '%s' to the same Network "
+          "'%s'" % (layer.name, self.name))
+    self._layers[layer.name] = layer
+    return layer
+
+  def get_layer(self, name=None, index=None):
+    """Get a contained `tf.layers.Layer` either by name or index.
+
+    Args:
+      name: String matching one of the names of a contained `Layer`.
+      index: Integer in [0, number of layers). Layers are assigned an index
+        by the order they are added.
+
+    Returns:
+      A `tf.layers.Layer` object.
+
+    Raises:
+      ValueError: If neither or both of 'index' or 'name' is specified.
+    """
+    if index is not None:
+      if name is not None:
+        raise ValueError("Exactly one of 'index' or 'name' must be provided")
+      if len(self._layers) <= index:
+        raise ValueError("Was asked to retrieve layer at index " +
+                         str(index) + " but model only has " + str(
+                             len(self._layers)) + " layers.")
+      return list(self._layers.values())[index]
+    if name is None:
+      raise ValueError("Exactly one of 'index' or 'name' must be provided")
+    return self._layers[index]
+
+  # The following methods are for implementing the Layer interface.
+
+  @property
+  def weights(self):
+    # TODO(josh11b): Should this return a set or perform de-duplication of
+    # variables in the case of shared layers/variables that appear in
+    # multiple places in the Network?
+    weights = []
+    for layer in six.itervalues(self._layers):
+      weights += layer.weights
+    return weights
+
+  @property
+  def trainable_weights(self):
+    weights = []
+    for layer in six.itervalues(self._layers):
+      weights += layer.trainable_weights
+    return weights
+
+  @property
+  def non_trainable_weights(self):
+    weights = []
+    for layer in six.itervalues(self._layers):
+      weights += layer.non_trainable_weights
+    return weights
+
+  @property
+  def trainable(self):
+    return True
+
+  @trainable.setter
+  def trainable(self, value):
+    if not value:
+      # We believe it better to decide which layers & networks are trainable
+      # at the Trainer level than here. Otherwise you can run into trouble if a
+      # layer/network is shared between two models, but is trainable in one
+      # but not the other (like with adversarial networks).
+      raise AttributeError("cannot mark Network as not trainable")
+
+  @property
+  def layers(self):
+    return self._layers.values()
+
+  def add_variable(self, name, shape, dtype=None, initializer=None,
+                   regularizer=None, trainable=True, constraint=None):
+    raise RuntimeError(
+        "add_variable not supported in Network class yet. Please file an issue "
+        "at https://github.com/tensorflow/tensorflow/issues/new if this is "
+        "important to you")
+
+  def __call__(self, inputs, *args, **kwargs):
+    # TODO(josh11b,ashankar,agarwal): Can we reduce the number of context
+    # managers here and/or move some of the work into the constructor
+    # for performance reasons?
+    with ops.container(self._container):
+      with variable_scope.variable_scope(variable_scope.get_variable_scope(),
+                                         use_resource=True):
+        return super(Network, self).__call__(inputs, *args, **kwargs)
+
+  # TODO(josh11b): Support other Layer methods needed for graph mode, such as for
+  # losses and updates
+
+
+class Sequential(Network):
+  """Represents a linear sequence of Layers.
+
+  The output of each layer is provided as the input to the next.
+  The inputs passed to `__call__` are passed to the inputs of the first
+  Layer, and it returns the outputs of the last Layer.
+
+  Args:
+    layers: An optional sequence of tf.layers.Layer objects.
+    name: An optional string name to use for this Network.
+  """
+
+  def __init__(self, layers=None, name=None):
+    super(Sequential, self).__init__(name=name)
+    if layers:
+      for l in layers:
+        self.add_layer(l)
+
+  def call(self, inputs):
+    """Call each Layer in the order they were added."""
+    # TODO(josh11b): Support "mode" and maybe other arguments
+    for l in self.layers:
+      inputs = l(inputs)
+    return inputs
diff --git a/tensorflow/contrib/eager/python/network_test.py b/tensorflow/contrib/eager/python/network_test.py
new file mode 100644
index 0000000000..f0dcae85ee
--- /dev/null
+++ b/tensorflow/contrib/eager/python/network_test.py
@@ -0,0 +1,107 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.contrib.eager.python import network
+from tensorflow.python.eager import test
+from tensorflow.python.framework import constant_op
+from tensorflow.python.layers import core
+
+
+# pylint: disable=not-callable
+class MyNetwork(network.Network):
+
+  def __init__(self):
+    super(MyNetwork, self).__init__(name="abcd")
+    self.l1 = self.add_layer(core.Dense(1, use_bias=False))
+
+  def call(self, x):
+    return self.l1(x)
+
+
+class NetworkTest(test.TestCase):
+
+  def testTrainableAttribute(self):
+    net = network.Network()
+    self.assertTrue(net.trainable)
+    with self.assertRaises(AttributeError):
+      net.trainable = False
+    self.assertTrue(net.trainable)
+
+  def testNetworkCall(self):
+    net = MyNetwork()
+    net(constant_op.constant([[2.0]]))  # Force variables to be created.
+    self.assertEqual(1, len(net.trainable_variables))
+    net.trainable_variables[0].assign([[17.0]])
+    # TODO(josh11b): Support passing Python values to networks.
+    result = net(constant_op.constant([[2.0]]))
+    self.assertEqual(34.0, result.numpy())
+
+  def testNetworkAsAGraph(self):
+    self.skipTest("TODO(ashankar,josh11b): FIX THIS")
+    # Verify that we're using ResourceVariables
+
+  def testNetworkVariablesDoNotInterfere(self):
+    self.skipTest("TODO: FIX THIS")
+    net1 = MyNetwork()
+    net2 = MyNetwork()
+
+    one = constant_op.constant([[1.]])
+
+    print(type(net1(one)))
+    net2(one)
+
+    net1.trainable_weights[0].assign(constant_op.constant([[1.]]))
+    net2.trainable_weights[0].assign(constant_op.constant([[2.]]))
+
+    print("NET1")
+    print(net1.name)
+    print(net1.variables)
+    print(net1(one))
+
+    print("NET2")
+    print(net2.name)
+    print(net2.variables)
+    print(net2(one))
+
+
+class SequentialTest(test.TestCase):
+
+  def testTwoLayers(self):
+    # Create a sequential network with one layer.
+    net = network.Sequential([core.Dense(1, use_bias=False)])
+
+    # Set that layer's weights so it multiplies by 3
+    l1 = net.get_layer(index=0)
+    net(constant_op.constant([[2.0]]))  # Create l1's variables
+    self.assertEqual(1, len(l1.trainable_variables))
+    l1.trainable_variables[0].assign([[3.0]])
+    self.assertEqual(21.0, net(constant_op.constant([[7.0]])).numpy())
+
+    # Add a second layer to the network.
+    l2 = core.Dense(1, use_bias=False)
+    net.add_layer(l2)
+
+    # Set the second layer's weights so it multiplies by 11
+    net(constant_op.constant([[2.0]]))  # Create l2's variables
+    self.assertEqual(1, len(l2.trainable_variables))
+    l2.trainable_variables[0].assign([[11.0]])
+    self.assertEqual(231.0, net(constant_op.constant([[7.0]])).numpy())
+
+
+if __name__ == "__main__":
+  test.main()
-- 
GitLab


From 10d0ae696c7b5618cae9e3845af8300fe62870a2 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 11 Oct 2017 14:45:28 -0700
Subject: [PATCH 271/909] [XLA:CPU] Adds intra-op parallelism to the
 "sequential" CPU backend (which already has intra-op parallelism for library
 calls). Adds support for parallel task assignment to instructions in entry
 (or embedded) computations. Adds code to emit calls to a new a runtime
 parallel fork/join function for instructions which have been assigned
 parallel tasks. Adds a simple cost model for I/O bound instructions.

*) Translation (deleuze model) wall time (seconds).
             large_model  small_model  small_model_small_attn
sequential:  0.00556      0.00484      0.00155
parallel:    0.00263      0.00163      0.00106

*) Wavenet
sequential: Avg. latency (30 runs): 1026.13ms, min/max: 988/1108ms
parallel:   Avg. latency (30 runs): 800.633ms, min/max: 785/818ms

*) ParallelFusion benchmark.
Benchmark                          Time(ns)        CPU(ns)     Iterations
----------------------------------------------------------
sequential cpu backend (at head)   610584         611467           1000
parallel cpu backend               153241         836097           4528
sequential cpu backend (this CL)   113482         679535           6017

PiperOrigin-RevId: 171877766
---
 tensorflow/compiler/xla/service/cpu/BUILD     |  18 ++
 .../compiler/xla/service/cpu/cpu_compiler.cc  |  15 +-
 .../compiler/xla/service/cpu/cpu_compiler.h   |   2 +-
 .../cpu/cpu_parallelization_preparation.cc    |  20 --
 .../compiler/xla/service/cpu/cpu_runtime.cc   |   3 +
 .../compiler/xla/service/cpu/cpu_runtime.h    |   1 +
 .../compiler/xla/service/cpu/ir_emitter.cc    | 192 +++++++++++++++---
 .../compiler/xla/service/cpu/ir_emitter.h     |  21 +-
 .../service/cpu/parallel_task_assignment.cc   | 148 ++++++++++++--
 .../service/cpu/parallel_task_assignment.h    |  49 +++++
 .../xla/service/cpu/runtime_fork_join.cc      |  93 +++++++++
 .../xla/service/cpu/runtime_fork_join.h       |  33 +++
 .../xla/service/cpu/simple_orc_jit.cc         |   2 +
 tensorflow/compiler/xla/tests/BUILD           |   2 +
 tensorflow/compiler/xla/tests/fusion_test.cc  | 136 +++++++++++--
 15 files changed, 644 insertions(+), 91 deletions(-)
 create mode 100644 tensorflow/compiler/xla/service/cpu/runtime_fork_join.cc
 create mode 100644 tensorflow/compiler/xla/service/cpu/runtime_fork_join.h

diff --git a/tensorflow/compiler/xla/service/cpu/BUILD b/tensorflow/compiler/xla/service/cpu/BUILD
index 8ab358fe17..c71eca0d39 100644
--- a/tensorflow/compiler/xla/service/cpu/BUILD
+++ b/tensorflow/compiler/xla/service/cpu/BUILD
@@ -87,6 +87,7 @@ cc_library(
         ":ir_emitter",
         ":layout_assignment",
         ":parallel_cpu_executable",
+        ":parallel_task_assignment",
         ":simple_orc_jit",
         "//tensorflow/compiler/xla:literal_util",
         "//tensorflow/compiler/xla:protobuf_util",
@@ -155,6 +156,7 @@ cc_library(
         ":disassembler",
         ":external_constant_pool",
         ":runtime_conv2d",
+        ":runtime_fork_join",
         ":runtime_matmul",
         ":runtime_single_threaded_conv2d",
         ":runtime_single_threaded_matmul",
@@ -243,6 +245,7 @@ cc_library(
         ":dot_op_emitter",
         ":external_constant_pool",
         ":ir_emission_utils",
+        ":shape_partition",
         ":simple_orc_jit",
         "//tensorflow/compiler/xla:shape_util",
         "//tensorflow/compiler/xla:status_macros",
@@ -505,6 +508,20 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "runtime_fork_join",
+    srcs = ["runtime_fork_join.cc"],
+    hdrs = ["runtime_fork_join.h"],
+    copts = runtime_copts(),
+    visibility = ["//visibility:public"],
+    deps = [
+        "//tensorflow/compiler/xla:executable_run_options",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:lib_internal",
+        "//third_party/eigen3",
+    ],
+)
+
 tf_cc_test(
     name = "cpu_runtime_test",
     srcs = ["cpu_runtime_test.cc"],
@@ -688,6 +705,7 @@ cc_library(
         ":shape_partition",
         "//tensorflow/compiler/xla/service:hlo",
         "//tensorflow/compiler/xla/service:hlo_cost_analysis",
+        "//tensorflow/compiler/xla/service:hlo_pass",
     ],
 )
 
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
index 386800d221..3272044faa 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
@@ -58,6 +58,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/cpu/ir_emitter.h"
 #include "tensorflow/compiler/xla/service/cpu/layout_assignment.h"
 #include "tensorflow/compiler/xla/service/cpu/parallel_cpu_executable.h"
+#include "tensorflow/compiler/xla/service/cpu/parallel_task_assignment.h"
 #include "tensorflow/compiler/xla/service/cpu/simple_orc_jit.h"
 #include "tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h"
 #include "tensorflow/compiler/xla/service/flatten_call_graph.h"
@@ -248,7 +249,7 @@ class CollectProfileCandidates : public DfsHloVisitorWithDefault {
 };
 }  // namespace
 
-Status CpuCompiler::RunHloPasses(HloModule* module) {
+Status CpuCompiler::RunHloPasses(HloModule* module, bool is_aot_compile) {
   // Optimization pipeline.
   HloPassPipeline pipeline("CPU");
   pipeline.AddInvariantChecker<HloVerifier>(ShapeSizeBytesFunction());
@@ -316,6 +317,14 @@ Status CpuCompiler::RunHloPasses(HloModule* module) {
   if (options::CpuParallelBackendRequested(module->config())) {
     pipeline.AddPass<ParallelizationPreparation>(max_parallelism,
                                                  ShapeSizeBytesFunction());
+  } else if (!is_aot_compile) {
+    // Run ParallelTaskAssigner to assign parallel tasks to HLOs in module.
+    // Note this is not run for AOT because it would bring in thread pool
+    // and thread synchronization dependencies which would likely increase
+    // binary size (and most AOT applications are single-threaded).
+    // TODO(29630486) Support multi-threaded AOT.
+    pipeline.AddPass<ParallelTaskAssigner>(max_parallelism,
+                                           ShapeSizeBytesFunction(), module);
   }
   // Copy insertion should be performed immediately before IR emission to avoid
   // inserting unnecessary copies (later pass adds an instruction which
@@ -450,7 +459,7 @@ StatusOr<std::unique_ptr<Executable>> CpuCompiler::Compile(
   llvm_module->setDataLayout(jit->data_layout());
   llvm_module->setTargetTriple(jit->target_triple().getTriple());
 
-  TF_RETURN_IF_ERROR(RunHloPasses(module.get()));
+  TF_RETURN_IF_ERROR(RunHloPasses(module.get(), /*is_aot_compile=*/false));
 
   HloComputation* computation = module->entry_computation();
   std::unordered_map<const HloInstruction*, size_t> hlo_to_profile_idx;
@@ -749,7 +758,7 @@ CpuCompiler::CompileAheadOfTime(std::vector<std::unique_ptr<HloModule>> modules,
     HloModule* module = modules[i].get();
     VLOG(1) << "Compiling ahead-of-time: " << module->name();
 
-    TF_RETURN_IF_ERROR(RunHloPasses(module));
+    TF_RETURN_IF_ERROR(RunHloPasses(module, /*is_aot_compile=*/true));
 
     TF_ASSIGN_OR_RETURN(
         SequentialHloOrdering::HloModuleSequence module_sequence,
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_compiler.h b/tensorflow/compiler/xla/service/cpu/cpu_compiler.h
index bd3541500d..21dd128619 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_compiler.h
+++ b/tensorflow/compiler/xla/service/cpu/cpu_compiler.h
@@ -131,7 +131,7 @@ class CpuCompiler : public LLVMCompiler {
 
   // Runs the HLO passes which are necessary for both optimizations and
   // correctness.
-  Status RunHloPasses(HloModule* module);
+  Status RunHloPasses(HloModule* module, bool is_aot_compile);
 
   TF_DISALLOW_COPY_AND_ASSIGN(CpuCompiler);
 };
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_parallelization_preparation.cc b/tensorflow/compiler/xla/service/cpu/cpu_parallelization_preparation.cc
index 2cd0aa7880..662ee60923 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_parallelization_preparation.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_parallelization_preparation.cc
@@ -116,26 +116,6 @@ StatusOr<bool> ParallelizationPreparation::RunParallelTaskAssignment(
   // Assign parallel tasks to HLOs in entry computation.
   HloComputation* computation = module->entry_computation();
   for (auto* instruction : computation->instructions()) {
-    // Currently, we do not assign parallel tasks to instructions with at least
-    // one of the following properties:
-    // *) Internal threading (library calls to kConv, kDot, and kCustomCall).
-    // *) Emit custom loops (kSelectAndScatter, FusionKind::kTransposeDot).
-    // *) Tuple-shaped.
-    // TODO(b/27458679) Parallelize instructions which are skipped here.
-    if (instruction->opcode() == HloOpcode::kParameter ||
-        instruction->opcode() == HloOpcode::kConstant ||
-        instruction->opcode() == HloOpcode::kCall ||
-        instruction->opcode() == HloOpcode::kCustomCall ||
-        instruction->opcode() == HloOpcode::kSelectAndScatter ||
-        (instruction->opcode() == HloOpcode::kConvolution &&
-         PotentiallyImplementedAsEigenConvolution(*instruction)) ||
-        PotentiallyImplementedAsEigenDot(*instruction) ||
-        (instruction->opcode() == HloOpcode::kFusion &&
-         instruction->fusion_kind() != HloInstruction::FusionKind::kLoop) ||
-        ShapeUtil::IsTuple(instruction->shape())) {
-      continue;
-    }
-
     // Calculate target parallel task count in [1, max_parallelism_].
     const int64 target_parallel_task_count =
         parallel_task_assignment.GetTargetParallelTaskCount(instruction);
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_runtime.cc b/tensorflow/compiler/xla/service/cpu/cpu_runtime.cc
index c7155b858b..7908dc173d 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_runtime.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_runtime.cc
@@ -51,6 +51,9 @@ extern const char* const kAcquireOutfeedBufferForPopulationSymbolName =
     "__xla_cpu_runtime_AcquireOutfeedBufferForPopulation";
 extern const char* const kReleaseOutfeedBufferAfterPopulationSymbolName =
     "__xla_cpu_runtime_ReleaseOutfeedBufferAfterPopulation";
+extern const char* const kParallelForkJoinSymbolName =
+    "__xla_cpu_runtime_ParallelForkJoin";
+
 extern const char* const kXlaCpuRuntimeSymbolNamePrefix = "__xla_cpu_runtime_";
 }  // namespace runtime
 }  // namespace cpu
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_runtime.h b/tensorflow/compiler/xla/service/cpu/cpu_runtime.h
index 29feb7267f..2ade455b8a 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_runtime.h
+++ b/tensorflow/compiler/xla/service/cpu/cpu_runtime.h
@@ -51,6 +51,7 @@ extern const char* const kAcquireInfeedBufferForDequeueSymbolName;
 extern const char* const kReleaseInfeedBufferAfterDequeueSymbolName;
 extern const char* const kAcquireOutfeedBufferForPopulationSymbolName;
 extern const char* const kReleaseOutfeedBufferAfterPopulationSymbolName;
+extern const char* const kParallelForkJoinSymbolName;
 
 // All symbol names for XLA CPU runtime functions need to start with this
 // prefix.
diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
index 633ad0290c..c38325554f 100644
--- a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
@@ -42,6 +42,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/cpu/dot_op_emitter.h"
 #include "tensorflow/compiler/xla/service/cpu/elemental_ir_emitter.h"
 #include "tensorflow/compiler/xla/service/cpu/ir_emission_utils.h"
+#include "tensorflow/compiler/xla/service/cpu/shape_partition.h"
 #include "tensorflow/compiler/xla/service/cpu/simple_orc_jit.h"
 #include "tensorflow/compiler/xla/service/elemental_ir_emitter.h"
 #include "tensorflow/compiler/xla/service/hlo_opcode.h"
@@ -186,20 +187,9 @@ void IrEmitter::InitializeIrFunction(const string& function_name) {
   // Even though the type of params and temps is void** in the host's view, in
   // LLVM IR this is represented by i8*, similarly to void*. It's up to the code
   // to use GEPs to unravel the indirection layers.
-  llvm::Type* i8_ptr_type = llvm::Type::getInt8PtrTy(module_->getContext());
-  llvm::Type* i8_ptr_ptr_type = i8_ptr_type->getPointerTo();
-  llvm::Type* i64_ptr_type = llvm::Type::getInt64PtrTy(module_->getContext());
-  std::vector<llvm::Type*> compute_function_params(
-      {i8_ptr_type, i8_ptr_type, i8_ptr_ptr_type, i8_ptr_ptr_type});
-  if (IsParallelContext()) {
-    compute_function_params.push_back(i64_ptr_type);
-  }
-  if (hlo_to_profile_idx_) {
-    compute_function_params.push_back(i64_ptr_type);
-  }
   llvm::FunctionType* compute_function_type = llvm::FunctionType::get(
       /*Result=*/llvm::Type::getVoidTy(module_->getContext()),
-      /*Params=*/compute_function_params,
+      /*Params=*/GetComputeFunctionParams(),
       /*isVarArg=*/false);
 
   // Functions with local linkage get an inlining bonus.  Because we know
@@ -221,7 +211,7 @@ void IrEmitter::InitializeIrFunction(const string& function_name) {
   (++arg_iter)->setName("run_options");
   (++arg_iter)->setName("params");
   (++arg_iter)->setName("temps");
-  if (IsParallelContext()) {
+  if (num_dynamic_loop_bounds_ > 0) {
     (++arg_iter)->setName("dynamic_loop_bounds");
   }
   if (hlo_to_profile_idx_) {
@@ -2286,8 +2276,19 @@ Status IrEmitter::HandleCall(HloInstruction* call) {
   }
 
   TF_RETURN_IF_ERROR(EmitTargetAddressForOp(call));
-  EmitArrayFunctionCallInto(call_ir_function, parameter_addresses,
-                            emitted_value_[call], computation->name());
+
+  if (!computation->root_instruction()->outer_dimension_partitions().empty() &&
+      !parallel_cpu_backend_) {
+    // ParallelTaskAssignment assigned partitions, emit call to
+    // ParallelForkJoin.
+    TF_RETURN_IF_ERROR(EmitParallelForkJoin(parameter_addresses,
+                                            emitted_value_[call], computation,
+                                            call_ir_function));
+  } else {
+    EmitArrayFunctionCallInto(call_ir_function, parameter_addresses,
+                              emitted_value_[call], computation->name());
+  }
+
   return Status::OK();
 }
 
@@ -2597,7 +2598,7 @@ Status IrEmitter::FinishVisit(HloInstruction* root) {
   // For the parallel cpu backend, we record the total for each embedded
   // computation callee with its caller kCall HLO.
   HloInstruction* hlo_to_lookup = nullptr;
-  if (IsParallelContext()) {
+  if (parallel_cpu_backend_ && is_top_level_computation_) {
     auto* computation = root->parent();
     auto* entry_computation = computation->parent()->entry_computation();
     if (computation != entry_computation) {
@@ -2755,12 +2756,27 @@ llvm::Type* IrEmitter::IrShapeType(const Shape& shape) {
   return llvm_ir::ShapeToIrType(shape, &ir_builder_);
 }
 
+std::vector<llvm::Type*> IrEmitter::GetComputeFunctionParams() {
+  llvm::Type* i8_ptr_type = llvm::Type::getInt8PtrTy(module_->getContext());
+  llvm::Type* i8_ptr_ptr_type = i8_ptr_type->getPointerTo();
+  llvm::Type* i64_ptr_type = llvm::Type::getInt64PtrTy(module_->getContext());
+  std::vector<llvm::Type*> compute_function_params(
+      {i8_ptr_type, i8_ptr_type, i8_ptr_ptr_type, i8_ptr_ptr_type});
+  if (num_dynamic_loop_bounds_ > 0) {
+    compute_function_params.push_back(i64_ptr_type);
+  }
+  if (hlo_to_profile_idx_) {
+    compute_function_params.push_back(i64_ptr_type);
+  }
+  return compute_function_params;
+}
+
 llvm::Argument* IrEmitter::GetResultArgument() {
   return GetArg(compute_function_, 0);
 }
 
 llvm::Argument* IrEmitter::GetProfileCountersArgument() {
-  const int64 arg_index = IsParallelContext() ? 5 : 4;
+  const int64 arg_index = num_dynamic_loop_bounds_ > 0 ? 5 : 4;
   return hlo_to_profile_idx_ ? GetArg(compute_function_, arg_index) : nullptr;
 }
 
@@ -2843,18 +2859,11 @@ llvm::Value* IrEmitter::EmitElementFunctionCall(
       AsStringRef(tensorflow::strings::StrCat(name, "_return_value")));
 }
 
-// Emits a core function call based on the following pseudo-code.
-//
-//   char** parameter_addresses_buffer =
-//       allocate buffer with a pointer for each parameter to the function
-//   for each parameter index, i.e. for i = 0, ..., #parameters:
-//     parameter_addresses_buffer[i] = parameter_addresses[i]
-//   call function(return_value_buffer,
-//                 parameter_addresses_buffer,
-//                 temps)
-//   return return_value_buffer  -- address of the return value.
-void IrEmitter::EmitArrayFunctionCallInto(
-    llvm::Function* function,
+// Emits code to allocate an array of parameter address pointers, and store
+// each address from 'parameter_addresses'.
+// Returns an array of compute function call arguments (including parameter
+// address buffer).
+std::vector<llvm::Value*> IrEmitter::GetArrayFunctionCallArguments(
     tensorflow::gtl::ArraySlice<llvm::Value*> parameter_addresses,
     llvm::Value* return_value_buffer, tensorflow::StringPiece name) {
   llvm::Value* parameter_addresses_buffer =
@@ -2883,7 +2892,26 @@ void IrEmitter::EmitArrayFunctionCallInto(
   if (auto* profile_counters = GetProfileCountersArgument()) {
     arguments.push_back(profile_counters);
   }
-  ir_builder_.CreateCall(function, arguments);
+  return arguments;
+}
+
+// Emits a core function call based on the following pseudo-code.
+//
+//   char** parameter_addresses_buffer =
+//       allocate buffer with a pointer for each parameter to the function
+//   for each parameter index, i.e. for i = 0, ..., #parameters:
+//     parameter_addresses_buffer[i] = parameter_addresses[i]
+//   call function(return_value_buffer,
+//                 parameter_addresses_buffer,
+//                 temps)
+//   return return_value_buffer  -- address of the return value.
+void IrEmitter::EmitArrayFunctionCallInto(
+    llvm::Function* function,
+    tensorflow::gtl::ArraySlice<llvm::Value*> parameter_addresses,
+    llvm::Value* return_value_buffer, tensorflow::StringPiece name) {
+  ir_builder_.CreateCall(
+      function, GetArrayFunctionCallArguments(parameter_addresses,
+                                              return_value_buffer, name));
 }
 
 llvm::Value* IrEmitter::EmitArrayFunctionCall(
@@ -2903,6 +2931,110 @@ llvm::Value* IrEmitter::EmitArrayFunctionCall(
   return return_value_buffer;
 }
 
+// Emits a call to a runtime fork/join function which dispatches parallel
+// calls to 'parallel_function' (and joins threads before returning).
+Status IrEmitter::EmitParallelForkJoin(
+    tensorflow::gtl::ArraySlice<llvm::Value*> parameter_addresses,
+    llvm::Value* output_address, HloComputation* computation,
+    llvm::Function* parallel_function) {
+  HloInstruction* root = computation->root_instruction();
+
+  // Build ParallelForkJoin function type.
+  std::vector<llvm::Type*> compute_function_params = GetComputeFunctionParams();
+  // Number of parallel compute functions.
+  compute_function_params.push_back(ir_builder_.getInt32Ty());
+  // Array of partitions. There is an array element for each
+  // partition x partition_dim x 2 (for dimension start and limit).
+  compute_function_params.push_back(
+      llvm::Type::getInt64PtrTy(module_->getContext()));
+  // Number of partitioned most-major dimensions in 'root.shape'.
+  compute_function_params.push_back(ir_builder_.getInt32Ty());
+  // Function pointer for compute function to be dispatched in parallel.
+  compute_function_params.push_back(
+      llvm::Type::getInt8PtrTy(module_->getContext()));
+
+  llvm::FunctionType* fork_join_type = llvm::FunctionType::get(
+      /*Result=*/llvm::Type::getVoidTy(module_->getContext()),
+      /*Params=*/compute_function_params,
+      /*isVarArg=*/false);
+
+  llvm::Function* fork_join_func =
+      llvm::cast<llvm::Function>(module_->getOrInsertFunction(
+          runtime::kParallelForkJoinSymbolName, fork_join_type));
+  fork_join_func->setCallingConv(llvm::CallingConv::C);
+  fork_join_func->setDoesNotThrow();
+
+  // Add common compute function arguments.
+  const string name = computation->name();
+  std::vector<llvm::Value*> arguments =
+      GetArrayFunctionCallArguments(parameter_addresses, output_address, name);
+
+  // Create ShapePartitionIterator to generate all partitions of 'root.shape'.
+  ShapePartitionIterator partition_iterator(root->shape(),
+                                            root->outer_dimension_partitions());
+  const int64 num_partitions = partition_iterator.GetTotalPartitionCount();
+  // Add argument specifying the number of parallel partitions.
+  arguments.push_back(ir_builder_.getInt32(num_partitions));
+
+  // The number of partitioned most-major dimensions in 'root.shape'.
+  const int32 num_partitioned_dims = root->outer_dimension_partitions().size();
+  // A dimension partition consists of two elements: [start_index, limit_index).
+  const int32 dim_partition_size = 2;
+  // Calculate array partition stride.
+  const int32 array_partition_stride =
+      num_partitioned_dims * dim_partition_size;
+  // Calculate the total number of elements in the partition array.
+  const int32 partition_array_size =
+      dim_partition_size * num_partitioned_dims * num_partitions;
+
+  // Store dimension partition values as llvm constants in 'partitions'.
+  // See comments in runtime_fork_join.cc for array layout description.
+  std::vector<llvm::Constant*> partitions(partition_array_size);
+  for (int32 i = 0; i < num_partitions; ++i) {
+    std::vector<std::pair<int64, int64>> dim_partitions =
+        partition_iterator.GetPartition(i);
+    CHECK_EQ(num_partitioned_dims, dim_partitions.size());
+    const int32 partition_index = i * array_partition_stride;
+    for (int32 j = 0; j < num_partitioned_dims; ++j) {
+      const std::pair<int64, int64>& dim_partition = dim_partitions[j];
+      const int32 index = partition_index + j * dim_partition_size;
+      // Store partition [dim_start, dim_limit) intervals for each dimension.
+      partitions[index] = ir_builder_.getInt64(dim_partition.first);
+      partitions[index + 1] =
+          ir_builder_.getInt64(dim_partition.first + dim_partition.second);
+    }
+  }
+
+  // Create global variable out of dimension partitions in 'partitions'.
+  llvm::ArrayType* partitions_array_type =
+      llvm::ArrayType::get(ir_builder_.getInt64Ty(), partition_array_size);
+  llvm::Constant* partitions_array =
+      llvm::ConstantArray::get(partitions_array_type, partitions);
+  llvm::GlobalVariable* global_partitions_array = new llvm::GlobalVariable(
+      /*Module=*/*module_,
+      /*Type=*/partitions_array_type,
+      /*isConstant=*/true,
+      /*Linkage=*/llvm::GlobalValue::PrivateLinkage,
+      /*Initializer=*/partitions_array,
+      /*Name=*/
+      AsStringRef(
+          tensorflow::strings::StrCat(name, "_parallel_dimension_partitions")));
+
+  // Add argument specifying parallel dimension partitions.
+  arguments.push_back(ir_builder_.CreateBitCast(
+      global_partitions_array,
+      llvm::Type::getInt64PtrTy(module_->getContext())));
+  // Add argument specifying the number of partitioned most-major dimensions.
+  arguments.push_back(ir_builder_.getInt32(num_partitioned_dims));
+  // Add argument for parallel compute function pointer.
+  arguments.push_back(
+      ir_builder_.CreateBitCast(parallel_function, ir_builder_.getInt8PtrTy()));
+  // Emit call to parallel fork/join.
+  ir_builder_.CreateCall(fork_join_func, arguments);
+
+  return Status::OK();
+}
+
 Status IrEmitter::EmitTargetAddressForOp(const HloInstruction* op) {
   llvm::Value* addr;
   const Shape& target_shape = op->shape();
diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.h b/tensorflow/compiler/xla/service/cpu/ir_emitter.h
index 53c4b6f241..58c185af1e 100644
--- a/tensorflow/compiler/xla/service/cpu/ir_emitter.h
+++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.h
@@ -249,6 +249,9 @@ class IrEmitter : public DfsHloVisitorWithDefault {
   // Convenience function to get the IR type matching the given shape.
   llvm::Type* IrShapeType(const Shape& shape);
 
+  // Returns an array of compute function parameter types.
+  std::vector<llvm::Type*> GetComputeFunctionParams();
+
   // Get the llvm::Value* that represents the "retval" argument of the
   // computation function being emitted by this emitter.
   llvm::Argument* GetResultArgument();
@@ -323,6 +326,18 @@ class IrEmitter : public DfsHloVisitorWithDefault {
       tensorflow::gtl::ArraySlice<llvm::Value*> parameter_addresses,
       tensorflow::StringPiece name);
 
+  // Returns an array of compute function call arguments.
+  std::vector<llvm::Value*> GetArrayFunctionCallArguments(
+      tensorflow::gtl::ArraySlice<llvm::Value*> parameter_addresses,
+      llvm::Value* return_value_buffer, tensorflow::StringPiece name);
+
+  // Emits a call to a runtime fork/join function which dispatches parallel
+  // calls to 'parallel_function' (and joins threads before returning).
+  Status EmitParallelForkJoin(
+      tensorflow::gtl::ArraySlice<llvm::Value*> parameter_addresses,
+      llvm::Value* output_address, HloComputation* computation,
+      llvm::Function* parallel_function);
+
   // Verifies that the element types of all of the given operand instructions
   // match and are of one of the given supported types.
   Status ElementTypesSameAndSupported(
@@ -596,12 +611,6 @@ class IrEmitter : public DfsHloVisitorWithDefault {
   Status EmitXfeedTransfer(XfeedKind kind, const Shape& shape,
                            llvm::Value* program_buffer_address);
 
-  // Returns true if the current function being emitted is called in a
-  // parallel context (returns false otherwise).
-  bool IsParallelContext() {
-    return parallel_cpu_backend_ && is_top_level_computation_;
-  }
-
   const HloModuleConfig& hlo_module_config_;
 
   const bool parallel_cpu_backend_;
diff --git a/tensorflow/compiler/xla/service/cpu/parallel_task_assignment.cc b/tensorflow/compiler/xla/service/cpu/parallel_task_assignment.cc
index d4b5e41f50..7219736b9e 100644
--- a/tensorflow/compiler/xla/service/cpu/parallel_task_assignment.cc
+++ b/tensorflow/compiler/xla/service/cpu/parallel_task_assignment.cc
@@ -48,29 +48,56 @@ class SimpleCostModel : public ParallelCostModel {
 class DefaultCostModel : public ParallelCostModel {
  public:
   DefaultCostModel(const int64 max_parallelism,
+                   const HloCostAnalysis::ShapeSizeFunction& shape_size,
                    std::unique_ptr<HloCostAnalysis> cost_analysis)
       : max_parallelism_(max_parallelism),
+        shape_size_(shape_size),
         cost_analysis_(std::move(cost_analysis)) {}
   ~DefaultCostModel() override {}
 
   int64 GetParallelTaskCount(HloInstruction* instruction) override {
-    // Calculate the instruction cost in cycles.
-    // TODO(29630486) Improve on this linear cost model.
-    // Consider making 'min_cost_per_thread' be a function of the target
-    // bandwidth limit for instructions with low arithmetic complexity.
-    const int64 instruction_cost =
-        1 * cost_analysis_->flop_count(*instruction) +
-        2 * cost_analysis_->transcendental_count(*instruction) +
-        10 * cost_analysis_->bytes_accessed(*instruction);
-    // Minimum per-thread cost is 100us of work on a 2GHz core.
-    const int64 min_cost_per_thread = 100000;
+    // Parameters for parallel task count computation.
+    int64 instruction_cost;
+    int64 min_cost_per_thread;
+    int64 max_parallelism;
+    // Calculate flops-to-bytes-ratio for 'instruction'.
+    const int64 bytes_accessed =
+        std::max(1LL, cost_analysis_->bytes_accessed(*instruction));
+    const float flops_to_bytes_ratio =
+        cost_analysis_->flop_count(*instruction) /
+        static_cast<float>(bytes_accessed);
+    // Check for I/O bound instructions.
+    if (flops_to_bytes_ratio <= 1.0) {
+      // Limit max parallelism for I/O bound instructions by assuming a
+      // sub-linear scaling function (fit based on emperical benchmark results).
+      // TODO(29630486) Develop system bandwidth model.
+      max_parallelism =
+          std::ceil(std::sqrt(tensorflow::port::NumSchedulableCPUs()));
+      // Use shape size instruction cost and L2 cache size min per-thread cost.
+      instruction_cost = shape_size_(instruction->shape());
+      min_cost_per_thread = 256LL << 10;  // 256KB L2 Cache size.
+    } else {
+      // Use max parallelism for compute bound instructions.
+      max_parallelism = max_parallelism_;
+      // Calculate the instruction cost in cycles.
+      // TODO(29630486) Improve on this linear cost model.
+      // Consider making 'min_cost_per_thread' be a function of the target
+      // bandwidth limit for instructions with low arithmetic complexity.
+      instruction_cost =
+          1 * cost_analysis_->flop_count(*instruction) +
+          2 * cost_analysis_->transcendental_count(*instruction) +
+          10 * cost_analysis_->bytes_accessed(*instruction);
+      // Minimum per-thread cost is 100us of work on a 2GHz core.
+      min_cost_per_thread = 100000;
+    }
     // Return target parallel task count in [1, max_parallelism_].
-    return std::min(max_parallelism_,
+    return std::min(max_parallelism,
                     std::max(1LL, instruction_cost / min_cost_per_thread));
   }
 
  private:
   const int64 max_parallelism_;
+  const HloCostAnalysis::ShapeSizeFunction shape_size_;
   const std::unique_ptr<HloCostAnalysis> cost_analysis_;
 };
 
@@ -86,7 +113,7 @@ ParallelTaskAssignment::ParallelTaskAssignment(
   Status status = computation->root_instruction()->Accept(cost_analysis.get());
   if (status.ok()) {
     // Set default cost model based on 'cost_analysis'.
-    cost_model_.reset(new DefaultCostModel(max_parallelism,
+    cost_model_.reset(new DefaultCostModel(max_parallelism, shape_size,
                                            std::move(cost_analysis)));
   } else {
     // Fall back to a simple cost model based on hlo size and L2 cache size.
@@ -121,5 +148,102 @@ int64 ParallelTaskAssignment::GetTargetParallelTaskCount(
   return cost_model_->GetParallelTaskCount(instruction);
 }
 
+StatusOr<bool> ParallelTaskAssigner::Run(HloModule* module) {
+  XLA_VLOG_LINES(2, "ParallelTaskAssigner ENTRY");
+  XLA_VLOG_LINES(3, module->ToString());
+
+  // Compute target parallel task counts for all instructions in 'module'.
+  HloToParallelTasks hlo_to_parallel_tasks;
+  ComputeTargetParallelTasks(module, &hlo_to_parallel_tasks);
+
+  // Assign parallel tasks to target specific instructions in 'module'.
+  // TODO(b/27458679) Support inter-op parallelism.
+  bool changed = AssignParallelTasks(module, hlo_to_parallel_tasks);
+
+  XLA_VLOG_LINES(2, "ParallelTaskAssigner EXIT");
+  XLA_VLOG_LINES(3, module->ToString());
+  return changed;
+}
+
+bool ParallelTaskAssigner::AssignParallelTasks(
+    HloModule* module, const HloToParallelTasks& hlo_to_parallel_tasks) {
+  return AssignParallelTasksHelper(module, module->entry_computation(),
+                                   hlo_to_parallel_tasks);
+}
+
+bool ParallelTaskAssigner::AssignParallelTasksHelper(
+    HloModule* module, HloComputation* computation,
+    const HloToParallelTasks& hlo_to_parallel_tasks) {
+  bool changed = false;
+  // Snapshot set of instructions because outlining modifies the set below.
+  std::vector<HloInstruction*> instructions(computation->instructions().begin(),
+                                            computation->instructions().end());
+  for (auto* instruction : instructions) {
+    // Assign parallel tasks to sub-computations for While and Call HLOs.
+    // TODO(b/27458679) Evaluate alternative intra-op parallelsim placement,
+    // and support other callable computations like reduce.
+    if (instruction->opcode() == HloOpcode::kWhile) {
+      changed |= AssignParallelTasksHelper(module, instruction->while_body(),
+                                           hlo_to_parallel_tasks);
+      continue;
+    } else if (instruction->opcode() == HloOpcode::kCall) {
+      changed |= AssignParallelTasksHelper(module, instruction->to_apply(),
+                                           hlo_to_parallel_tasks);
+      continue;
+    }
+    // Skip if no parallel tasks were computed in first pass.
+    auto it = hlo_to_parallel_tasks.find(instruction);
+    if (it == hlo_to_parallel_tasks.end()) {
+      continue;
+    }
+    // Get target parallel task count computed for 'instruction'.
+    const int64 target_parallel_task_count = (*it).second;
+    // Assign feasible dimension partitions (based on actual dimension sizes).
+    auto dim_partition_counts = ShapePartitionAssigner(instruction->shape())
+                                    .Run(target_parallel_task_count);
+    const int64 total_partition_count =
+        ShapePartitionAssigner::GetTotalPartitionCount(dim_partition_counts);
+    if (total_partition_count <= 1) {
+      // Feasible partition calculation resulting in no partitioning, so skip.
+      continue;
+    }
+
+    // Outline 'instruction' in 'computation' for parallel task assignment.
+    auto* call = module->OutlineExpressionFromComputation(
+        {instruction},
+        tensorflow::strings::StrCat("parallel_", instruction->name()),
+        computation);
+
+    // Set assigned dimension partitioning to 'instruction'.
+    auto* new_root = call->to_apply()->root_instruction();
+    new_root->set_outer_dimension_partitions(dim_partition_counts);
+
+    VLOG(2) << "Assigned parallel task count: " << total_partition_count
+            << " to instruction: " << new_root->name()
+            << " parent: " << new_root->parent()->name();
+    changed = true;
+  }
+  return changed;
+}
+
+void ParallelTaskAssigner::ComputeTargetParallelTasks(
+    HloModule* module, HloToParallelTasks* hlo_to_parallel_tasks) {
+  // Compute parallel task counts for all instructions in 'module'.
+  for (auto* computation : module->computations()) {
+    if (computation->IsFusionComputation()) {
+      continue;
+    }
+    for (auto* instruction : computation->instructions()) {
+      // Query ParallelTaskAssignment for target parallel task count.
+      const int64 target_parallel_task_count =
+          parallel_task_assignment_.GetTargetParallelTaskCount(instruction);
+      if (target_parallel_task_count > 1) {
+        hlo_to_parallel_tasks->insert(
+            {instruction, target_parallel_task_count});
+      }
+    }
+  }
+}
+
 }  // namespace cpu
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/service/cpu/parallel_task_assignment.h b/tensorflow/compiler/xla/service/cpu/parallel_task_assignment.h
index 15f065a3ad..e036da5784 100644
--- a/tensorflow/compiler/xla/service/cpu/parallel_task_assignment.h
+++ b/tensorflow/compiler/xla/service/cpu/parallel_task_assignment.h
@@ -18,6 +18,7 @@ limitations under the License.
 
 #include "tensorflow/compiler/xla/service/hlo_cost_analysis.h"
 #include "tensorflow/compiler/xla/service/hlo_module.h"
+#include "tensorflow/compiler/xla/service/hlo_pass_interface.h"
 
 namespace xla {
 namespace cpu {
@@ -49,6 +50,54 @@ class ParallelTaskAssignment {
   std::unique_ptr<ParallelCostModel> cost_model_;
 };
 
+// ParallelTaskAssigner computes target parallel task counts for all HLOs
+// in the module, then assigns parallel task counts to HLOs in the entry
+// computation, or to HLOs in embedded computations invoked by (potentially
+// nested) kWhile or kCall instructions.
+// Each HLO which is assigned parallel task counts is outlined into its
+// own embedded computation, which is compiled as a parallel compute function,
+// and which is invoked from a kCall instruction that is lowered in codegen to
+// a runtime parallel fork/join call.
+class ParallelTaskAssigner : public HloPassInterface {
+ public:
+  // 'max_parallelism': the maximum parallel task count per instruction.
+  // 'shape_size': shape size function used by HloCostAnalysis during parallel
+  //               task assignment.
+  // 'module': the containing HloModule.
+  ParallelTaskAssigner(const int64 max_parallelism,
+                       const HloCostAnalysis::ShapeSizeFunction& shape_size,
+                       HloModule* module)
+      : parallel_task_assignment_(max_parallelism, shape_size, module) {}
+  ~ParallelTaskAssigner() override {}
+
+  tensorflow::StringPiece name() const override {
+    return "cpu-parallel-task-assigner";
+  }
+
+  // Run parallel task assigner on 'module'.
+  // Returns true if the computation was changed, false otherwise.
+  StatusOr<bool> Run(HloModule* module) override;
+
+ private:
+  using HloToParallelTasks = std::unordered_map<const HloInstruction*, int64>;
+
+  // Assigns target parallel tasks from 'hlo_to_parallel_tasks' to HLOs in
+  // 'module'.
+  // Returns true if the computation was changed, false otherwise.
+  bool AssignParallelTasks(HloModule* module,
+                           const HloToParallelTasks& hlo_to_parallel_tasks);
+  bool AssignParallelTasksHelper(
+      HloModule* module, HloComputation* computation,
+      const HloToParallelTasks& hlo_to_parallel_tasks);
+
+  // Computes target parallel task counts (returned in 'parallel_task_counts')
+  // for parallelizable instructions in 'module'.
+  void ComputeTargetParallelTasks(HloModule* module,
+                                  HloToParallelTasks* hlo_to_parallel_tasks);
+
+  ParallelTaskAssignment parallel_task_assignment_;
+};
+
 }  // namespace cpu
 }  // namespace xla
 
diff --git a/tensorflow/compiler/xla/service/cpu/runtime_fork_join.cc b/tensorflow/compiler/xla/service/cpu/runtime_fork_join.cc
new file mode 100644
index 0000000000..af2f3de6b8
--- /dev/null
+++ b/tensorflow/compiler/xla/service/cpu/runtime_fork_join.cc
@@ -0,0 +1,93 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/service/cpu/runtime_fork_join.h"
+
+#define EIGEN_USE_THREADS
+
+#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "tensorflow/compiler/xla/executable_run_options.h"
+#include "tensorflow/core/lib/core/blocking_counter.h"
+#include "tensorflow/core/platform/logging.h"
+
+using ComputeFunctionType = void (*)(void*, const void*, const void**, void**,
+                                     int64*, uint64*);
+
+// Dispatches 'num_partitions - 1' calls to 'function_ptr' in parallel.
+// Calls 'function_ptr' for first partition inline.
+// Uses blocking counter to synchonize threads after parallel calls complete.
+//
+// The 'partitions' array has a total number of elements equal to
+// 'num_partitions * num_partitioned_dims * 2' (the '2' is necessary to specify
+// dimension start and limit indices).
+//
+// The 'partitions' array layout stores array elements in memory with dimension
+// start limit as the most-minor dimension, followed by dimension, then
+// partition.
+//
+// EX: Layout of 'partitions' array with 'num_partitions = 2', and
+//     'num_partitioned_dims = 3'
+//
+//   [partition0_dim0_start]
+//   [partition0_dim0_limit]
+//   [partition0_dim1_start]
+//   [partition0_dim1_limit]
+//   [partition0_dim2_start]
+//   [partition0_dim2_limit]
+//   [partition1_dim0_start]
+//   [partition1_dim0_limit]
+//   [partition1_dim1_start]
+//   [partition1_dim1_limit]
+//   [partition1_dim2_start]
+//   [partition1_dim2_limit]
+//
+void __xla_cpu_runtime_ParallelForkJoin(
+    void* result_ptr, const void* run_options_ptr, const void** params,
+    void** temps, uint64* prof_counters, tensorflow::int32 num_partitions,
+    tensorflow::int64* partitions, tensorflow::int32 num_partitioned_dims,
+    void* function_ptr) {
+  VLOG(2) << "ParallelForkJoin ENTRY"
+          << " num_partitions: " << num_partitions
+          << " num_partitioned_dims: " << num_partitioned_dims;
+  CHECK_GT(num_partitions, 1);
+  CHECK_GT(num_partitioned_dims, 0);
+  const xla::ExecutableRunOptions* run_options =
+      static_cast<const xla::ExecutableRunOptions*>(run_options_ptr);
+  ComputeFunctionType function =
+      reinterpret_cast<ComputeFunctionType>(function_ptr);
+  // Compute partition stride in 'partitions' array.
+  const int64 stride = 2 * num_partitioned_dims;
+
+  // Dispatch 'num_partitions - 1' compute functions to run in parallel.
+  tensorflow::BlockingCounter bc(num_partitions - 1);
+  for (tensorflow::int32 i = 1; i < num_partitions; ++i) {
+    const int64 offset = i * stride;
+    run_options->intra_op_thread_pool()->enqueue_function(
+        [i, function, result_ptr, run_options_ptr, params, temps, prof_counters,
+         partitions, offset, &bc]() {
+          function(result_ptr, run_options_ptr, params, temps,
+                   &partitions[offset], prof_counters);
+          bc.DecrementCount();
+          VLOG(3) << "ParallelForkJoin partition " << i << " done.";
+        });
+  }
+
+  // Call first compute function inline.
+  function(result_ptr, run_options_ptr, params, temps, &partitions[0],
+           prof_counters);
+  VLOG(3) << "ParallelForkJoin partition 0 done.";
+  bc.Wait();
+  VLOG(2) << "ParallelForkJoin EXIT";
+}
diff --git a/tensorflow/compiler/xla/service/cpu/runtime_fork_join.h b/tensorflow/compiler/xla/service/cpu/runtime_fork_join.h
new file mode 100644
index 0000000000..1ddcaf5274
--- /dev/null
+++ b/tensorflow/compiler/xla/service/cpu/runtime_fork_join.h
@@ -0,0 +1,33 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef THIRD_PARTY_TENSORFLOW_COMPILER_XLA_SERVICE_CPU_RUNTIME_FORK_JOIN_H_
+#define THIRD_PARTY_TENSORFLOW_COMPILER_XLA_SERVICE_CPU_RUNTIME_FORK_JOIN_H_
+
+#include "tensorflow/core/platform/types.h"
+
+extern "C" {
+
+// Dispatches 'num_partitions' parallel calls to 'function_ptr' and joins
+// threads before returning. See comments in runtime_fork_join.cc for details.
+extern void __xla_cpu_runtime_ParallelForkJoin(
+    void* result_ptr, const void* run_options_ptr, const void** params,
+    void** temps, uint64* prof_counters, tensorflow::int32 num_partitions,
+    tensorflow::int64* partitions, tensorflow::int32 num_partitioned_dims,
+    void* function_ptr);
+
+}  // extern "C"
+
+#endif  // THIRD_PARTY_TENSORFLOW_COMPILER_XLA_SERVICE_CPU_RUNTIME_FORK_JOIN_H_
diff --git a/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc b/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc
index c614e334a8..cfffb3fbc3 100644
--- a/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc
+++ b/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc
@@ -32,6 +32,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/cpu/cpu_runtime_neon.h"
 #include "tensorflow/compiler/xla/service/cpu/cpu_runtime_sse4_1.h"
 #include "tensorflow/compiler/xla/service/cpu/runtime_conv2d.h"
+#include "tensorflow/compiler/xla/service/cpu/runtime_fork_join.h"
 #include "tensorflow/compiler/xla/service/cpu/runtime_matmul.h"
 #include "tensorflow/compiler/xla/service/cpu/runtime_single_threaded_conv2d.h"
 #include "tensorflow/compiler/xla/service/cpu/runtime_single_threaded_matmul.h"
@@ -104,6 +105,7 @@ class JITSymbolTable {
     ADD_JIT_SYMBOL_TO_TABLE(EigenSingleThreadedConvF32);
     ADD_JIT_SYMBOL_TO_TABLE(EigenSingleThreadedMatMulF32);
     ADD_JIT_SYMBOL_TO_TABLE(EigenSingleThreadedMatMulF64);
+    ADD_JIT_SYMBOL_TO_TABLE(ParallelForkJoin);
 
 #undef ADD_JIT_SYMBOL_TO_TABLE
   }
diff --git a/tensorflow/compiler/xla/tests/BUILD b/tensorflow/compiler/xla/tests/BUILD
index f37a331a72..256ec71ab5 100644
--- a/tensorflow/compiler/xla/tests/BUILD
+++ b/tensorflow/compiler/xla/tests/BUILD
@@ -1410,8 +1410,10 @@ xla_test(
         "//tensorflow/compiler/xla/tests:hlo_test_base",
         "//tensorflow/compiler/xla/tests:literal_test_util",
         "//tensorflow/compiler/xla/tests:xla_internal_test_main",
+        "//tensorflow/core:core_cpu_internal",
         "//tensorflow/core:lib",
         "//tensorflow/core:test",
+        "//third_party/eigen3",
     ],
 )
 
diff --git a/tensorflow/compiler/xla/tests/fusion_test.cc b/tensorflow/compiler/xla/tests/fusion_test.cc
index 3bf9ccb197..a8f6488996 100644
--- a/tensorflow/compiler/xla/tests/fusion_test.cc
+++ b/tensorflow/compiler/xla/tests/fusion_test.cc
@@ -17,8 +17,12 @@ limitations under the License.
 #include <algorithm>
 #include <memory>
 #include <new>
+#include <random>
 #include <utility>
 
+#define EIGEN_USE_THREADS
+
+#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 #include "tensorflow/compiler/xla/array2d.h"
 #include "tensorflow/compiler/xla/client/client_library.h"
 #include "tensorflow/compiler/xla/client/computation.h"
@@ -37,6 +41,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/tests/literal_test_util.h"
 #include "tensorflow/compiler/xla/tests/test_macros.h"
 #include "tensorflow/compiler/xla/xla_data.pb.h"
+#include "tensorflow/core/common_runtime/eigen_thread_pool.h"
 #include "tensorflow/core/lib/gtl/array_slice.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/protobuf.h"
@@ -250,6 +255,42 @@ XLA_TEST_F(FusionTest, Parameter) {
                               ErrorSpec(1e-4));
 }
 
+XLA_TEST_F(FusionTest, RandomizedParallelPartition) {
+  // Tests parallel partitioning of a fusion instruction.
+  // Create shape with random outer dimension size to generate random parallel
+  // partition counts for each test run.
+  const int seed = tensorflow::testing::RandomSeed();
+  LOG(INFO) << "RandomizedParallelPartition seed: " << seed;
+  std::mt19937 generator(seed);
+  std::uniform_int_distribution<int> distribution(128, 1024);
+  const int64 rand_dim0_size = distribution(generator);
+  const int64 dim1_size = 1024;
+  Shape shape =
+      ShapeUtil::MakeShapeWithLayout(F32, {rand_dim0_size, dim1_size}, {1, 0});
+  // Build simple fusion computation: y = x^2 (elementwise).
+  auto builder = HloComputation::Builder(TestName());
+  auto hlo_module = CreateNewModule();
+
+  auto two = builder.AddInstruction(
+      HloInstruction::CreateConstant(Literal::CreateR0<float>(2.0)));
+  auto x =
+      builder.AddInstruction(HloInstruction::CreateBroadcast(shape, two, {}));
+  auto y = builder.AddInstruction(
+      HloInstruction::CreateBinary(shape, HloOpcode::kMultiply, x, x));
+
+  hlo_module->AddEntryComputation(builder.Build())
+      ->CreateFusionInstruction(/*instructions_to_fuse=*/{y, x, two},
+                                HloInstruction::FusionKind::kLoop);
+  // Compute result.
+  auto result = ExecuteAndTransfer(std::move(hlo_module), {});
+  // Every element of result should be y = x^2 = 4.0.
+  for (int i = 0; i < rand_dim0_size; ++i) {
+    for (int j = 0; j < dim1_size; ++j) {
+      EXPECT_EQ(4.0, result->Get<float>({i, j}));
+    }
+  }
+}
+
 XLA_TEST_F(FusionTest, BroadcastIntoBinaryOp) {
   auto builder = HloComputation::Builder(TestName());
   auto hlo_module = CreateNewModule();
@@ -722,47 +763,104 @@ void BM_ParallelFusion(int num_iters) {
   auto executors = PlatformUtil::GetStreamExecutors(platform).ValueOrDie();
   StreamExecutorMemoryAllocator allocator(platform, executors);
 
-  const int64 intra_op_parallelism_threads = 16;
+  const int64 intra_op_parallelism_threads = 24;
   xla::LocalClientOptions client_options;
   client_options.set_platform(platform);
   client_options.set_intra_op_parallelism_threads(intra_op_parallelism_threads);
   auto client =
       ClientLibrary::GetOrCreateLocalClient(client_options).ValueOrDie();
 
-  const int64 dim_size = 1024;
-  // Create a simple fusable elementwise computation.
+  auto* transfer_manager =
+      TransferManager::GetForPlatform(platform).ValueOrDie();
+  int device_ordinal = client->default_device_ordinal();
+
+  // Computation shape parameters.
+  const int64 param0_dim0 = 1024;
+  const int64 param0_dim1 = 1024;
+  const int64 param1_dim0 = 1024;
+  const int64 param1_dim1 = 1024;
+  const int64 param2_dim0 = 1024;
+  const int64 param2_dim1 = 1024;
+
+  // Create computation.
   ComputationBuilder builder(client, "ParallelFusion");
-  Shape input_shape = ShapeUtil::MakeShape(F32, {dim_size, dim_size});
-  auto input0 = builder.Broadcast(builder.ConstantR0<float>(1.5f),
-                                  AsInt64Slice(input_shape.dimensions()));
-  auto input1 = builder.Broadcast(builder.ConstantR0<float>(2.0f),
-                                  AsInt64Slice(input_shape.dimensions()));
-  auto input2 = builder.Broadcast(builder.ConstantR0<float>(3.0f),
-                                  AsInt64Slice(input_shape.dimensions()));
-  auto x = builder.Mul(input0, input1);
-  auto y = builder.Add(x, input2);
+  Shape shape0 = ShapeUtil::MakeShape(F32, {param0_dim0, param0_dim1});
+  auto param0 = builder.Parameter(0, shape0, "param0");
+  Shape shape1 = ShapeUtil::MakeShape(F32, {param1_dim0, param1_dim1});
+  auto param1 = builder.Parameter(1, shape1, "param1");
+  Shape shape2 = ShapeUtil::MakeShape(F32, {param2_dim0, param2_dim1});
+  auto param2 = builder.Parameter(2, shape2, "param2");
+
+  auto x = builder.Mul(param0, param1);
+  auto y = builder.Add(x, param2);
   auto computation = builder.Build().ConsumeValueOrDie();
 
+  // Transfer literals to device.
+  auto buffer0 =
+      ScopedShapedBuffer::Allocate(shape0, &allocator, /*device_ordinal=*/0)
+          .ConsumeValueOrDie();
+  auto param0_literal =
+      Literal::CreateR2F32Linspace(1.0, 2.0, param0_dim0, param0_dim1);
+  ASSERT_IS_OK(transfer_manager->TransferLiteralToDevice(
+      executors[device_ordinal], *param0_literal, buffer0->mutable_buffer({})));
+
+  auto buffer1 =
+      ScopedShapedBuffer::Allocate(shape1, &allocator, /*device_ordinal=*/0)
+          .ConsumeValueOrDie();
+  auto param1_literal =
+      Literal::CreateR2F32Linspace(1.0, 2.0, param1_dim0, param1_dim1);
+  ASSERT_IS_OK(transfer_manager->TransferLiteralToDevice(
+      executors[device_ordinal], *param1_literal, buffer1->mutable_buffer({})));
+
+  auto buffer2 =
+      ScopedShapedBuffer::Allocate(shape2, &allocator, /*device_ordinal=*/0)
+          .ConsumeValueOrDie();
+  auto param2_literal =
+      Literal::CreateR2F32Linspace(1.0, 2.0, param2_dim0, param2_dim1);
+  ASSERT_IS_OK(transfer_manager->TransferLiteralToDevice(
+      executors[device_ordinal], *param2_literal, buffer2->mutable_buffer({})));
+
+  // Build executable.
   std::unique_ptr<LocalExecutable> executable =
-      client->Compile(computation, {}, ExecutableBuildOptions())
+      client
+          ->Compile(computation,
+                    {&buffer0->shape(), &buffer1->shape(), &buffer2->shape()},
+                    ExecutableBuildOptions())
           .ConsumeValueOrDie();
 
-  // Run some warm-up executions.
+  se::Stream stream(executors[client->default_device_ordinal()]);
+  stream.Init();
+
+  // Initialize thread pool.
+  tensorflow::thread::ThreadPool pool(tensorflow::Env::Default(), "XLAEigen",
+                                      intra_op_parallelism_threads);
+  tensorflow::EigenThreadPoolWrapper tp(&pool);
+  Eigen::ThreadPoolDevice device(&tp, tp.NumThreads());
+
+  // Initialize ExecutableRunOptions.
   ExecutableRunOptions options;
-  options.set_allocator(&allocator);
+  options.set_allocator(&allocator).set_stream(&stream);
+  options.set_intra_op_thread_pool(&device);
+
+  // Run some warm-up executions.
   const int kWarmups = 2;
   for (int i = 0; i < kWarmups; ++i) {
-    auto result = executable->Run({}, options);
+    auto result =
+        executable->Run({buffer0.get(), buffer1.get(), buffer2.get()}, options);
     ASSERT_TRUE(result.ok());
   }
 
   // Run benchmark.
-  tensorflow::testing::BytesProcessed(static_cast<int64>(num_iters) * dim_size *
-                                      dim_size * sizeof(float));
+  const int64 total_bytes = param0_dim0 * param0_dim0 +
+                            param1_dim0 * param1_dim0 +
+                            param2_dim0 * param2_dim0;
+  tensorflow::testing::BytesProcessed(static_cast<int64>(num_iters) *
+                                      total_bytes * sizeof(float));
   tensorflow::testing::UseRealTime();
   tensorflow::testing::StartTiming();
   for (int i = 0; i < num_iters; ++i) {
-    auto result = executable->Run({}, options);
+    auto result =
+        executable->Run({buffer0.get(), buffer1.get(), buffer2.get()}, options);
     ASSERT_TRUE(result.ok());
   }
 }
-- 
GitLab


From ccfa8f4f1492c5cf1a7db35b2dba1f7b5424f0e2 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Wed, 11 Oct 2017 15:18:35 -0700
Subject: [PATCH 272/909] [XLA:CPU] Switch TF gather's HLO implementation to
 use dynamic-update-slice in a "while" loop.

Benchmarks results (times in ms):

  nontrivial_gather.axis0_cpu:     0.110
  nontrivial_gather.axis0_xla_cpu: 0.139
  nontrivial_gather.axis1_cpu:     0.093
  nontrivial_gather.axis1_xla_cpu: 0.142
  nontrivial_gather.axis4_cpu:     1.183
  nontrivial_gather.axis4_xla_cpu: 2.658
  slice_gather.axis0_cpu:          0.00388
  slice_gather.axis0_xla_cpu:      0.00397
  slice_gather.axis1_cpu:          0.00421
  slice_gather.axis1_xla_cpu:      0.00427
  slice_gather.axis4_cpu:          0.252
  slice_gather.axis4_xla_cpu:      0.114

As you can see, the pure-XLA implementation is slower in all the nontrivial
cases and as-fast or faster in the slice-gather cases.

The slice-gather cases are gathers that can be implemented as a single XLA
dynamic-slice, and so the speedup here is likely understated: Once we can
simplify the gather to a single dynamic-slice, we should be able to do many
other optimizations to it, ideally fusing it so it has zero cost.

The nontrivial gathers all gather more than one element, and are implemented
with an XLA while loop.  The most important one is the axis 0 gather --
gathering from an inner dimension is so slow no matter what you do that it's
probably not worth optimizing.

It's possible to make this XLA implementation faster -- one option I've
considered is "unrolling" the gather into a series of dynamic-slice's that are
then concat'ed together.  This would be totally fusable, unlike the
implementation in this CL.  Another option would be adding a notion of
uninitialized memory into XLA -- part of what makes us slow is that we have to
initialize the memset our output to 0 before we overwrite it.

But given that the shape we're benchmarking here is totally arbitrary, and
given that we're getting decent performance, I think this is good enough to
start with.

PiperOrigin-RevId: 171883273
---
 .../compiler/aot/tests/tfcompile_test.cc      |  27 ---
 tensorflow/compiler/aot/tfcompile.bzl         |   3 -
 tensorflow/compiler/tests/BUILD               |   6 +-
 tensorflow/compiler/tests/gather_test.py      |  93 +++++++++-
 tensorflow/compiler/tf2xla/kernels/BUILD      |  30 ----
 .../compiler/tf2xla/kernels/gather_op.cc      | 165 +++---------------
 .../tf2xla/kernels/gather_op_helpers.h        |   4 +-
 .../kernels/gather_op_kernel_float_int32.cc   |  72 --------
 .../kernels/gather_op_kernel_float_int64.cc   |  72 --------
 .../tf2xla/kernels/tensor_array_ops.cc        |   3 +-
 .../compiler/tf2xla/kernels/variable_ops.cc   |   3 +-
 11 files changed, 122 insertions(+), 356 deletions(-)
 delete mode 100644 tensorflow/compiler/tf2xla/kernels/gather_op_kernel_float_int32.cc
 delete mode 100644 tensorflow/compiler/tf2xla/kernels/gather_op_kernel_float_int64.cc

diff --git a/tensorflow/compiler/aot/tests/tfcompile_test.cc b/tensorflow/compiler/aot/tests/tfcompile_test.cc
index cfde5651c6..6b037f276a 100644
--- a/tensorflow/compiler/aot/tests/tfcompile_test.cc
+++ b/tensorflow/compiler/aot/tests/tfcompile_test.cc
@@ -180,33 +180,6 @@ TEST(TFCompileTest, Gather) {
     }
     EXPECT_EQ(gather_const.result0_data(), gather.results()[0]);
   }
-
-  // Bad indices returns an error.
-  {
-    const float params[4] = {1, 2, 3, 4};
-    std::copy(params + 0, params + 4, gather.arg0_data());
-    const int32 indices[2] = {1, 4};
-    std::copy(indices + 0, indices + 2, gather.arg1_data());
-    EXPECT_FALSE(gather.Run());
-    EXPECT_EQ(gather.error_msg(), "Invalid index for gather");
-  }
-
-  // Try a successful gather again, after the error, to ensure the error state
-  // is cleared.
-  {
-    const float params[4] = {1, 2, 3, 4};
-    std::copy(params + 0, params + 4, gather.arg0_data());
-    const int32 indices[2] = {1, 3};
-    std::copy(indices + 0, indices + 2, gather.arg1_data());
-    EXPECT_TRUE(gather.Run());
-    EXPECT_EQ(gather.error_msg(), "");
-    const float results[2] = {2, 4};
-    for (int i = 0; i < 2; ++i) {
-      EXPECT_EQ(gather.result0(i), results[i]);
-      EXPECT_EQ(gather.result0_data()[i], results[i]);
-    }
-    EXPECT_EQ(gather.result0_data(), gather.results()[0]);
-  }
 }
 
 TEST(TFCompileTest, MatMul2) {
diff --git a/tensorflow/compiler/aot/tfcompile.bzl b/tensorflow/compiler/aot/tfcompile.bzl
index 461a9315c5..4888760acd 100644
--- a/tensorflow/compiler/aot/tfcompile.bzl
+++ b/tensorflow/compiler/aot/tfcompile.bzl
@@ -186,8 +186,6 @@ def tf_library(name, graph, config,
           "//tensorflow/compiler/xla:xla_data_proto",
       ] or []) + (include_standard_runtime_deps and [
           # TODO(cwhipkey): only depend on kernel code that the model actually needed.
-          "//tensorflow/compiler/tf2xla/kernels:gather_op_kernel_float_int32",
-          "//tensorflow/compiler/tf2xla/kernels:gather_op_kernel_float_int64",
           "//tensorflow/compiler/tf2xla/kernels:index_ops_kernel_argmax_float_1d",
           "//tensorflow/compiler/tf2xla/kernels:index_ops_kernel_argmax_float_2d",
           "//tensorflow/compiler/xla/service/cpu:cpu_runtime_avx",
@@ -295,7 +293,6 @@ def tf_library(name, graph, config,
         tags=tags,
     )
 
-
 def target_llvm_triple():
   """Returns the target LLVM triple to be used for compiling the target."""
   # TODO(toddw): Add target_triple for other targets.  For details see:
diff --git a/tensorflow/compiler/tests/BUILD b/tensorflow/compiler/tests/BUILD
index eded6dc463..d4fe02854a 100644
--- a/tensorflow/compiler/tests/BUILD
+++ b/tensorflow/compiler/tests/BUILD
@@ -504,12 +504,8 @@ tf_xla_py_test(
 
 tf_xla_py_test(
     name = "gather_test",
-    size = "small",
+    size = "medium",
     srcs = ["gather_test.py"],
-    # Gather needs CustomCall on CPU, which is not available in normal
-    # (not precompiled) TensorFlow. The flag below excludes the CPU
-    # backend.
-    disabled_backends = "cpu",
     deps = [
         ":xla_test",
         "//tensorflow/python:array_ops",
diff --git a/tensorflow/compiler/tests/gather_test.py b/tensorflow/compiler/tests/gather_test.py
index d2a4e4bbd4..4b81c1d7ab 100644
--- a/tensorflow/compiler/tests/gather_test.py
+++ b/tensorflow/compiler/tests/gather_test.py
@@ -24,8 +24,12 @@ from tensorflow.compiler.tests import xla_test
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import variables
+from tensorflow.python.platform import flags
 from tensorflow.python.platform import test
 
+FLAGS = flags.FLAGS
+
 _TEST_TYPES = [dtypes.float32]
 
 
@@ -81,8 +85,31 @@ class GatherTest(xla_test.XLATestCase):
           expected = np.take(params_np, [0, 1, 0, 2], axis=axis)
           self.assertAllEqual(expected, gather_val)
 
+  def testSimpleTwoD32_Int64Indices(self):
+    if np.int64 not in self.int_types:
+      return
+
+    with self.test_session() as session, self.test_scope():
+      data = np.array([[0, 1, 2], [3, 4, 5], [6, 7, 8], [9, 10, 11],
+                       [12, 13, 14]])
+      # The indices must be in bounds for any axis.
+      indices_np = np.array([0, 1, 0, 2])
+      for dtype in _TEST_TYPES:
+        for axis in 0, 1, -1:
+          params_np = self._buildParams(data, dtype)
+          params = array_ops.placeholder(dtype=dtype)
+          indices = array_ops.placeholder(dtype=dtypes.int64)
+          gather_t = array_ops.gather(params, indices, axis=axis)
+          gather_val = session.run(
+              gather_t, feed_dict={
+                  params: params_np,
+                  indices: indices_np
+              })
+          expected = np.take(params_np, [0, 1, 0, 2], axis=axis)
+          self.assertAllEqual(expected, gather_val)
+
   def testHigherRank(self):
-    # Check that scalar and empty indices shapes work as well.
+    """Check that scalar and empty indices shapes work as well."""
     shape = (2, 1, 3, 2)
     for indices_shape in (), (0,), (2, 0), (2, 3):
       for dtype in _TEST_TYPES:
@@ -98,5 +125,67 @@ class GatherTest(xla_test.XLATestCase):
             self.assertAllEqual(gather_np, gather_value)
 
 
-if __name__ == "__main__":
+class GatherBenchmark(test.Benchmark):
+  """Microbenchmarks for the gather op."""
+
+  def _benchmarkGather(self, name, axis, gather_indices, use_xla_jit):
+
+    def BuilderFn():
+      inputs = variables.Variable(
+          array_ops.zeros([100, 100, 10, 100, 50], dtype=dtypes.float32),
+          dtype=dtypes.float32,
+          name='input')
+      indices = variables.Variable(
+          gather_indices, dtype=dtypes.int32, name='indices')
+      gather_t = array_ops.gather(inputs, indices, axis=axis)
+      return '%s.axis%d' % (name, axis), [gather_t]
+
+    xla_test.Benchmark(self, BuilderFn, use_xla_jit=use_xla_jit, device='cpu')
+
+  def _benchmarkSliceGather(self, axis, use_xla_jit):
+    """Benchmarks a gather op that's really a dynamic slice."""
+    self._benchmarkGather('slice_gather', axis, [1], use_xla_jit)
+
+  def _benchmarkNontrivialGather(self, axis, use_xla_jit):
+    self._benchmarkGather('nontrivial_gather', axis, [9, 1, 0, 2] * 4,
+                          use_xla_jit)
+
+  def benchmarkSliceGatherAxis0(self):
+    self._benchmarkSliceGather(axis=0, use_xla_jit=False)
+
+  def benchmarkSliceGatherAxis0XLA(self):
+    self._benchmarkSliceGather(axis=0, use_xla_jit=True)
+
+  def benchmarkSliceGatherAxis1(self):
+    self._benchmarkSliceGather(axis=1, use_xla_jit=False)
+
+  def benchmarkSliceGatherAxis1XLA(self):
+    self._benchmarkSliceGather(axis=1, use_xla_jit=True)
+
+  def benchmarkSliceGatherAxis4(self):
+    self._benchmarkSliceGather(axis=4, use_xla_jit=False)
+
+  def benchmarkSliceGatherAxis4XLA(self):
+    self._benchmarkSliceGather(axis=4, use_xla_jit=True)
+
+  def benchmarkNontrivialGatherAxis0(self):
+    self._benchmarkNontrivialGather(axis=0, use_xla_jit=False)
+
+  def benchmarkNontrivialGatherAxis0XLA(self):
+    self._benchmarkNontrivialGather(axis=0, use_xla_jit=True)
+
+  def benchmarkNontrivialGatherAxis1(self):
+    self._benchmarkNontrivialGather(axis=1, use_xla_jit=False)
+
+  def benchmarkNontrivialGatherAxis1XLA(self):
+    self._benchmarkNontrivialGather(axis=1, use_xla_jit=True)
+
+  def benchmarkNontrivialGatherAxis4(self):
+    self._benchmarkNontrivialGather(axis=4, use_xla_jit=False)
+
+  def benchmarkNontrivialGatherAxis4XLA(self):
+    self._benchmarkNontrivialGather(axis=4, use_xla_jit=True)
+
+
+if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/compiler/tf2xla/kernels/BUILD b/tensorflow/compiler/tf2xla/kernels/BUILD
index 6a0c4fef75..f44d61de68 100644
--- a/tensorflow/compiler/tf2xla/kernels/BUILD
+++ b/tensorflow/compiler/tf2xla/kernels/BUILD
@@ -132,8 +132,6 @@ tf_kernel_library(
     name = "xla_cpu_only_ops",
     srcs = ["index_ops_cpu.cc"],
     deps = [
-        ":gather_op_kernel_float_int32",
-        ":gather_op_kernel_float_int64",
         ":index_ops_kernel_argmax_float_1d",
         ":index_ops_kernel_argmax_float_2d",
         "//tensorflow/compiler/tf2xla:common",
@@ -149,34 +147,6 @@ tf_kernel_library(
     ],
 )
 
-cc_library(
-    name = "gather_op_kernel_float_int32",
-    srcs = ["gather_op_kernel_float_int32.cc"],
-    visibility = ["//visibility:public"],
-    deps = [
-        "//tensorflow/compiler/tf2xla:xla_local_runtime_context",
-        "//tensorflow/core:framework_lite",
-        "//tensorflow/core/kernels:bounds_check",
-        "//tensorflow/core/kernels:gather_functor_hdr",
-        "//third_party/eigen3",
-    ],
-    alwayslink = 1,
-)
-
-cc_library(
-    name = "gather_op_kernel_float_int64",
-    srcs = ["gather_op_kernel_float_int64.cc"],
-    visibility = ["//visibility:public"],
-    deps = [
-        "//tensorflow/compiler/tf2xla:xla_local_runtime_context",
-        "//tensorflow/core:framework_lite",
-        "//tensorflow/core/kernels:bounds_check",
-        "//tensorflow/core/kernels:gather_functor_hdr",
-        "//third_party/eigen3",
-    ],
-    alwayslink = 1,
-)
-
 cc_library(
     name = "index_ops_kernel_argmax_float_1d",
     srcs = ["index_ops_kernel_argmax_float_1d.cc"],
diff --git a/tensorflow/compiler/tf2xla/kernels/gather_op.cc b/tensorflow/compiler/tf2xla/kernels/gather_op.cc
index 2c7d445600..db449ec345 100644
--- a/tensorflow/compiler/tf2xla/kernels/gather_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/gather_op.cc
@@ -30,7 +30,7 @@ xla::ComputationDataHandle XlaComputeGatherDynamicSlice(
     XlaOpKernelContext* context, const xla::ComputationDataHandle& input,
     const TensorShape& input_shape, const xla::ComputationDataHandle& indices,
     const TensorShape& indices_shape, int64 axis, DataType dtype,
-    xla::ComputationBuilder* builder) {
+    DataType index_type, xla::ComputationBuilder* builder) {
   // Although the indices Tensor is flattened into rank 1 during the lookup,
   // and each scalar entry is used as an index into the first dimension of the
   // input, the output is returned with shape:
@@ -80,22 +80,23 @@ xla::ComputationDataHandle XlaComputeGatherDynamicSlice(
   // Specify the shape of the loop-carried Tensor tuple.
   xla::PrimitiveType ptype;
   TF_CHECK_OK(DataTypeToPrimitiveType(dtype, &ptype));
+  xla::PrimitiveType idxtype;
+  TF_CHECK_OK(DataTypeToPrimitiveType(index_type, &idxtype));
   std::vector<xla::Shape> tuple_shapes(
       {// The iteration counter i is a scalar, incremented each iteration.
-       xla::ShapeUtil::MakeShape(xla::S32, {}),
+       xla::ShapeUtil::MakeShape(idxtype, {}),
        // The input array has shape input_shape. Loop invariant.
        xla::ShapeUtil::MakeShape(ptype, input_shape.dim_sizes()),
        // The gather indices are reshaped to rank 1. Loop invariant.
-       xla::ShapeUtil::MakeShape(xla::S32, {num_indices}),
+       xla::ShapeUtil::MakeShape(idxtype, {num_indices}),
        // The output array is rank >= 3, and is updated on each loop iteration.
        xla::ShapeUtil::MakeShape(ptype, loop_out_shape.dim_sizes())});
   xla::Shape tuple_shape = xla::ShapeUtil::MakeTupleShape(tuple_shapes);
 
   // Construct the initial values of the loop-carried Tensors.
-  auto init_i = builder->ConstantR0<int32>(0);
-  auto init_out =
-      builder->Broadcast(builder->ConstantLiteral(xla::Literal::Zero(ptype)),
-                         loop_out_shape.dim_sizes());
+  auto init_i = XlaHelpers::Zero(builder, index_type);
+  auto init_out = builder->Broadcast(XlaHelpers::Zero(builder, dtype),
+                                     loop_out_shape.dim_sizes());
   // Flatten the indices into 1-D for ease of iteration.
   auto indices_1d = builder->Reshape(indices, {num_indices});
   auto init = builder->Tuple({init_i, input, indices_1d, init_out});
@@ -105,7 +106,7 @@ xla::ComputationDataHandle XlaComputeGatherDynamicSlice(
                                 "GatherWhileCond");
   condb.Lt(condb.GetTupleElement(
                condb.Parameter(0, tuple_shape, "GatherWhileTuple"), 0),
-           condb.ConstantR0<int32>(num_indices));
+           XlaHelpers::IntegerLiteral(&condb, index_type, num_indices));
   auto cond_status = condb.Build();
   auto cond = cond_status.ConsumeValueOrDie();
 
@@ -127,7 +128,7 @@ xla::ComputationDataHandle XlaComputeGatherDynamicSlice(
     // Slice from the input array.
     auto index = bodyb.DynamicSlice(indices, bodyb.Reshape(i, {1}), {1});
     auto start_indices = bodyb.Pad(
-        bodyb.Reshape(index, {1}), bodyb.ConstantR0<int32>(0),
+        bodyb.Reshape(index, {1}), XlaHelpers::Zero(&bodyb, index_type),
         xla::MakeEdgePaddingConfig(
             {{input_shape_pre_axis.dims(), input_shape_post_axis.dims()}}));
     auto slice_i = bodyb.Reshape(
@@ -136,7 +137,8 @@ xla::ComputationDataHandle XlaComputeGatherDynamicSlice(
 
     // Construct the index into the R3+ output Tensor 0, ..., <index>, 0, ...
     std::vector<xla::ComputationDataHandle> out_index_vals(
-        loop_out_shape.dims(), bodyb.ConstantR1<int32>({0}));
+        loop_out_shape.dims(),
+        bodyb.Reshape(XlaHelpers::Zero(&bodyb, index_type), {1}));
     out_index_vals[input_shape_pre_axis.dims() + extra_dims] =
         bodyb.Reshape(i, {1});
     auto out_index = bodyb.ConcatInDim(out_index_vals, 0);
@@ -144,8 +146,8 @@ xla::ComputationDataHandle XlaComputeGatherDynamicSlice(
     // Update the output Tensor
     auto updated_output = bodyb.DynamicUpdateSlice(output, slice_i, out_index);
 
-    bodyb.Tuple({bodyb.Add(i, bodyb.ConstantR0<int32>(1)), input, indices,
-                 updated_output});
+    bodyb.Tuple({bodyb.Add(i, XlaHelpers::One(&bodyb, index_type)), input,
+                 indices, updated_output});
   }
   auto body_status = bodyb.Build();
   auto body = body_status.ConsumeValueOrDie();
@@ -156,124 +158,6 @@ xla::ComputationDataHandle XlaComputeGatherDynamicSlice(
   return builder->Reshape(gather_output, out_shape.dim_sizes());
 }
 
-namespace {
-
-class GatherOpCustomCall : public XlaOpKernel {
- public:
-  explicit GatherOpCustomCall(OpKernelConstruction* context)
-      : XlaOpKernel(context) {}
-
-  void Compile(XlaOpKernelContext* context) override {
-    const TensorShape params_shape = context->InputShape(0);
-    const auto params_dims = params_shape.dims();
-    const TensorShape indices_shape = context->InputShape(1);
-    OP_REQUIRES(
-        context, TensorShapeUtils::IsVectorOrHigher(params_shape),
-        errors::InvalidArgument("params must be at least 1 dimensional"));
-
-    DataType index_type = input_type(1);
-    OP_REQUIRES(context, index_type == DT_INT32 || index_type == DT_INT64,
-                errors::InvalidArgument("index must be int32 or int64"));
-
-    // GatherV2 added an axis argument. We support both Gather and GatherV2 in
-    // this kernel by defaulting axis to 0 if there are 2 inputs.
-    int64 axis = 0;
-    if (context->num_inputs() == 3) {
-      const TensorShape axis_shape = context->InputShape(2);
-      OP_REQUIRES(context, TensorShapeUtils::IsScalar(axis_shape),
-                  errors::InvalidArgument("axis must be scalar"));
-      DataType axis_type = input_type(2);
-      OP_REQUIRES(context, axis_type == DT_INT32 || axis_type == DT_INT64,
-                  errors::InvalidArgument("axis must be int32 or int64"));
-
-      xla::Literal literal;
-      OP_REQUIRES_OK(context, context->ConstantInput(2, &literal));
-      int64 axis_input = axis_type == DT_INT32 ? literal.Get<int32>({})
-                                               : literal.Get<int64>({});
-      axis = axis_input < 0 ? axis_input + params_dims : axis_input;
-      OP_REQUIRES(context, 0 <= axis && axis < params_dims,
-                  errors::InvalidArgument("Expected axis in the range [",
-                                          -params_dims, ", ", params_dims,
-                                          "), but got ", axis_input));
-    }
-
-    // Check that we have enough index space.
-    const int64 limit = index_type == DT_INT32
-                            ? std::numeric_limits<int32>::max()
-                            : std::numeric_limits<int64>::max();
-    OP_REQUIRES(context, params_shape.dim_size(axis) <= limit,
-                errors::InvalidArgument(
-                    "params.shape[", axis, "] too large for ",
-                    DataTypeString(index_type),
-                    " indexing: ", params_shape.dim_size(axis), " > ", limit));
-
-    // The result shape is params.shape[0:axis] + indices.shape +
-    // params.shape[axis + 1:].
-    TensorShape result_shape;
-    int64 outer_size = 1;
-    int64 inner_size = 1;
-    for (int i = 0; i < axis; i++) {
-      result_shape.AddDim(params_shape.dim_size(i));
-      outer_size *= params_shape.dim_size(i);
-    }
-    result_shape.AppendShape(indices_shape);
-    for (int i = axis + 1; i < params_dims; i++) {
-      result_shape.AddDim(params_shape.dim_size(i));
-      inner_size *= params_shape.dim_size(i);
-    }
-
-    XlaContext& tc = XlaContext::Get(context);
-    OP_REQUIRES(
-        context, tc.allow_cpu_custom_calls(),
-        errors::InvalidArgument("Gather op requires CustomCall on CPU"));
-
-    xla::ComputationBuilder& b = *context->builder();
-
-    // Call gather_xla_float_kernel (from gather_op_kernel_float.cc).
-    // XLA passes <out> to the function, so it is not included here.
-    std::vector<xla::ComputationDataHandle> args;
-    args.push_back(tc.GetOrCreateRuntimeContextParameter());
-    args.push_back(b.ConstantLiteral(
-        *xla::Literal::CreateR0<int64>(indices_shape.num_elements())));
-    args.push_back(
-        b.ConstantLiteral(*xla::Literal::CreateR0<int64>(outer_size)));
-    args.push_back(b.ConstantLiteral(
-        *xla::Literal::CreateR0<int64>(params_shape.dim_size(axis))));
-    args.push_back(
-        b.ConstantLiteral(*xla::Literal::CreateR0<int64>(inner_size)));
-    args.push_back(context->Input(0));
-    args.push_back(context->Input(1));
-
-    xla::Shape xla_out_shape;
-    OP_REQUIRES_OK(
-        context, TensorShapeToXLAShape(DT_FLOAT, result_shape, &xla_out_shape));
-
-    // Call the custom code with args:
-    xla::ComputationDataHandle output;
-    if (index_type == DT_INT32) {
-      output = b.CustomCall("gather_float_int32_xla_impl", args, xla_out_shape);
-    } else {
-      output = b.CustomCall("gather_float_int64_xla_impl", args, xla_out_shape);
-    }
-
-    context->SetOutput(0, output);
-  }
-
- private:
-  TF_DISALLOW_COPY_AND_ASSIGN(GatherOpCustomCall);
-};
-
-REGISTER_XLA_OP(Name("Gather")
-                    .TypeConstraint("Tparams", DT_FLOAT)
-                    .Device(DEVICE_CPU_XLA_JIT),
-                GatherOpCustomCall);
-REGISTER_XLA_OP(Name("GatherV2")
-                    .TypeConstraint("Tparams", DT_FLOAT)
-                    .Device(DEVICE_CPU_XLA_JIT),
-                GatherOpCustomCall);
-
-}  // namespace
-
 GatherOpDynamicSlice::GatherOpDynamicSlice(OpKernelConstruction* context)
     : XlaOpKernel(context) {}
 
@@ -303,20 +187,17 @@ void GatherOpDynamicSlice::Compile(XlaOpKernelContext* context) {
                                 ", ", params_dims, "), but got ", axis));
   }
 
-  xla::ComputationDataHandle gather =
-      XlaComputeGatherDynamicSlice(context, input, input_shape, indices,
-                                   indices_shape, axis, DT_FLOAT, builder);
+  DataType index_type = input_type(1);
+  OP_REQUIRES(context, index_type == DT_INT32 || index_type == DT_INT64,
+              errors::InvalidArgument("indices must be int32 or int64"));
+
+  xla::ComputationDataHandle gather = XlaComputeGatherDynamicSlice(
+      context, input, input_shape, indices, indices_shape, axis, DT_FLOAT,
+      index_type, builder);
   context->SetOutput(0, gather);
 }
 
-REGISTER_XLA_OP(Name("Gather")
-                    .TypeConstraint("Tparams", DT_FLOAT)
-                    .Device(DEVICE_GPU_XLA_JIT),
-                GatherOpDynamicSlice);
-
-REGISTER_XLA_OP(Name("GatherV2")
-                    .TypeConstraint("Tparams", DT_FLOAT)
-                    .Device(DEVICE_GPU_XLA_JIT),
-                GatherOpDynamicSlice);
+REGISTER_XLA_OP(Name("Gather"), GatherOpDynamicSlice);
+REGISTER_XLA_OP(Name("GatherV2"), GatherOpDynamicSlice);
 
 }  // namespace tensorflow
diff --git a/tensorflow/compiler/tf2xla/kernels/gather_op_helpers.h b/tensorflow/compiler/tf2xla/kernels/gather_op_helpers.h
index 5623c4d1c2..2c80395c56 100644
--- a/tensorflow/compiler/tf2xla/kernels/gather_op_helpers.h
+++ b/tensorflow/compiler/tf2xla/kernels/gather_op_helpers.h
@@ -28,11 +28,13 @@ namespace tensorflow {
 
 // Adds to builder an XLA computation that performs a gather on input (of
 // shape input_shape) keyed on indices (of shape indices_shape).
+//
+// index_type must be must be DT_INT32 or DT_INT64.
 xla::ComputationDataHandle XlaComputeGatherDynamicSlice(
     XlaOpKernelContext* ctx, const xla::ComputationDataHandle& input,
     const TensorShape& input_shape, const xla::ComputationDataHandle& indices,
     const TensorShape& indices_shape, int64 axis, DataType dtype,
-    xla::ComputationBuilder* builder);
+    DataType index_type, xla::ComputationBuilder* builder);
 
 }  // namespace tensorflow
 
diff --git a/tensorflow/compiler/tf2xla/kernels/gather_op_kernel_float_int32.cc b/tensorflow/compiler/tf2xla/kernels/gather_op_kernel_float_int32.cc
deleted file mode 100644
index 33b1b087d0..0000000000
--- a/tensorflow/compiler/tf2xla/kernels/gather_op_kernel_float_int32.cc
+++ /dev/null
@@ -1,72 +0,0 @@
-/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#define EIGEN_USE_THREADS
-
-#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
-#include "tensorflow/compiler/tf2xla/xla_local_runtime_context.h"
-#include "tensorflow/core/framework/tensor_types.h"
-#include "tensorflow/core/kernels/gather_functor.h"
-#include "tensorflow/core/platform/dynamic_annotations.h"
-#include "tensorflow/core/platform/macros.h"
-
-namespace tensorflow {
-
-EIGEN_STRONG_INLINE void gather_float_int32_xla_impl(float* out, void** data) {
-  // data is managed by the JIT code so msan can't tell it's initialized.
-  TF_ANNOTATE_MEMORY_IS_INITIALIZED(data, 7 * sizeof(void*));
-
-  int64 indices_size = *static_cast<int64*>(data[1]);
-  int64 params_x = *static_cast<int64*>(data[2]);
-  int64 params_y = *static_cast<int64*>(data[3]);
-  int64 params_z = *static_cast<int64*>(data[4]);
-
-  float* in = static_cast<float*>(data[5]);
-
-  int32* indices = static_cast<int32*>(data[6]);
-  Eigen::DSizes<Eigen::DenseIndex, 3> in_eig_sizes;
-  in_eig_sizes[0] = params_x;
-  in_eig_sizes[1] = params_y;
-  in_eig_sizes[2] = params_z;
-  tensorflow::TTypes<float, 3>::ConstTensor in_eig(in, in_eig_sizes);
-
-  Eigen::DSizes<Eigen::DenseIndex, 1> indices_eig_sizes;
-  indices_eig_sizes[0] = indices_size;
-  tensorflow::TTypes<int32>::ConstFlat indices_eig(indices, indices_eig_sizes);
-
-  Eigen::DSizes<Eigen::DenseIndex, 3> out_eig_sizes;
-  out_eig_sizes[0] = params_x;
-  out_eig_sizes[1] = indices_size;
-  out_eig_sizes[2] = params_z;
-  tensorflow::TTypes<float, 3>::Tensor out_eig(out, out_eig_sizes);
-
-  tensorflow::functor::GatherFunctorCPU<float, int32> f;
-  const int64 bad_i = f(in_eig, indices_eig, out_eig);
-  if (bad_i != -1) {
-    tensorflow::XlaLocalRuntimeContext* runtime_context =
-        static_cast<tensorflow::XlaLocalRuntimeContext*>(data[0]);
-    runtime_context->error = true;
-    runtime_context->error_msg = "Invalid index for gather";
-    for (int i = 0; i < out_eig.size(); ++i) out[i] = 0;
-  }
-}
-
-}  // namespace tensorflow
-
-// Implements gather on CPU. This is called by an XLA custom call, set up by
-// gather_op.cc.
-extern "C" void TF_EXPORT gather_float_int32_xla_impl(float* out, void** data) {
-  tensorflow::gather_float_int32_xla_impl(out, data);
-}
diff --git a/tensorflow/compiler/tf2xla/kernels/gather_op_kernel_float_int64.cc b/tensorflow/compiler/tf2xla/kernels/gather_op_kernel_float_int64.cc
deleted file mode 100644
index 5e2d872ce0..0000000000
--- a/tensorflow/compiler/tf2xla/kernels/gather_op_kernel_float_int64.cc
+++ /dev/null
@@ -1,72 +0,0 @@
-/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#define EIGEN_USE_THREADS
-
-#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
-#include "tensorflow/compiler/tf2xla/xla_local_runtime_context.h"
-#include "tensorflow/core/framework/tensor_types.h"
-#include "tensorflow/core/kernels/gather_functor.h"
-#include "tensorflow/core/platform/dynamic_annotations.h"
-#include "tensorflow/core/platform/macros.h"
-
-namespace tensorflow {
-
-EIGEN_STRONG_INLINE void gather_float_int64_xla_impl(float* out, void** data) {
-  // data is managed by the JIT code so msan can't tell it's initialized.
-  TF_ANNOTATE_MEMORY_IS_INITIALIZED(data, 7 * sizeof(void*));
-
-  int64 indices_size = *static_cast<int64*>(data[1]);
-  int64 params_x = *static_cast<int64*>(data[2]);
-  int64 params_y = *static_cast<int64*>(data[3]);
-  int64 params_z = *static_cast<int64*>(data[4]);
-
-  float* in = static_cast<float*>(data[5]);
-
-  int64* indices = static_cast<int64*>(data[6]);
-  Eigen::DSizes<Eigen::DenseIndex, 3> in_eig_sizes;
-  in_eig_sizes[0] = params_x;
-  in_eig_sizes[1] = params_y;
-  in_eig_sizes[2] = params_z;
-  tensorflow::TTypes<float, 3>::ConstTensor in_eig(in, in_eig_sizes);
-
-  Eigen::DSizes<Eigen::DenseIndex, 1> indices_eig_sizes;
-  indices_eig_sizes[0] = indices_size;
-  tensorflow::TTypes<int64>::ConstFlat indices_eig(indices, indices_eig_sizes);
-
-  Eigen::DSizes<Eigen::DenseIndex, 3> out_eig_sizes;
-  out_eig_sizes[0] = params_x;
-  out_eig_sizes[1] = indices_size;
-  out_eig_sizes[2] = params_z;
-  tensorflow::TTypes<float, 3>::Tensor out_eig(out, out_eig_sizes);
-
-  tensorflow::functor::GatherFunctorCPU<float, int64> f;
-  const int64 bad_i = f(in_eig, indices_eig, out_eig);
-  if (bad_i != -1) {
-    tensorflow::XlaLocalRuntimeContext* runtime_context =
-        static_cast<tensorflow::XlaLocalRuntimeContext*>(data[0]);
-    runtime_context->error = true;
-    runtime_context->error_msg = "Invalid index for gather";
-    for (int i = 0; i < out_eig.size(); ++i) out[i] = 0;
-  }
-}
-
-}  // namespace tensorflow
-
-// Implements gather on CPU. This is called by an XLA custom call, set up by
-// gather_op.cc.
-extern "C" void TF_EXPORT gather_float_int64_xla_impl(float* out, void** data) {
-  tensorflow::gather_float_int64_xla_impl(out, data);
-}
diff --git a/tensorflow/compiler/tf2xla/kernels/tensor_array_ops.cc b/tensorflow/compiler/tf2xla/kernels/tensor_array_ops.cc
index e2d3d40813..351fda2517 100644
--- a/tensorflow/compiler/tf2xla/kernels/tensor_array_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/tensor_array_ops.cc
@@ -307,11 +307,12 @@ class TensorArrayGatherOp : public XlaOpKernel {
     OP_REQUIRES(ctx, indices_shape.dims() == 1,
                 errors::InvalidArgument("indices must be rank 1"));
     auto indices = ctx->Input(1);
+    DataType index_type = ctx->input_type(1);
 
     xla::ComputationDataHandle ta = resource->value;
 
     xla::ComputationDataHandle gather = XlaComputeGatherDynamicSlice(
-        ctx, ta, ta_shape, indices, indices_shape, 0, dtype_, b);
+        ctx, ta, ta_shape, indices, indices_shape, 0, dtype_, index_type, b);
     ctx->SetOutput(0, gather);
   }
 
diff --git a/tensorflow/compiler/tf2xla/kernels/variable_ops.cc b/tensorflow/compiler/tf2xla/kernels/variable_ops.cc
index 4ae9838547..b19ea22f50 100644
--- a/tensorflow/compiler/tf2xla/kernels/variable_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/variable_ops.cc
@@ -111,9 +111,10 @@ class ResourceGatherOp : public XlaOpKernel {
 
     auto indices = ctx->Input(1);
     auto indices_shape = ctx->InputShape(1);
+    DataType index_type = ctx->input_type(1);
     xla::ComputationDataHandle gather = XlaComputeGatherDynamicSlice(
         ctx, resource_handle, resource_shape, indices, indices_shape, 0,
-        resource_dtype, builder);
+        resource_dtype, index_type, builder);
     ctx->SetOutput(0, gather);
   }
 };
-- 
GitLab


From 0ffc6e1723d2c5b59f59560ec74dbc1eb4e264c5 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 11 Oct 2017 15:25:26 -0700
Subject: [PATCH 273/909] Changed embedding op to use parallel version of
 DynamicStitch.

PiperOrigin-RevId: 171884257
---
 tensorflow/compiler/tf2xla/kernels/dynamic_stitch_op.cc | 1 +
 tensorflow/python/ops/embedding_ops.py                  | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/compiler/tf2xla/kernels/dynamic_stitch_op.cc b/tensorflow/compiler/tf2xla/kernels/dynamic_stitch_op.cc
index dde7898015..7349dcb987 100644
--- a/tensorflow/compiler/tf2xla/kernels/dynamic_stitch_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/dynamic_stitch_op.cc
@@ -199,6 +199,7 @@ class DynamicStitchOp : public XlaOpKernel {
 };
 
 REGISTER_XLA_OP(Name("DynamicStitch"), DynamicStitchOp);
+REGISTER_XLA_OP(Name("ParallelDynamicStitch"), DynamicStitchOp);
 
 }  // namespace
 }  // namespace tensorflow
diff --git a/tensorflow/python/ops/embedding_ops.py b/tensorflow/python/ops/embedding_ops.py
index a845afbf93..8c1ccc6840 100644
--- a/tensorflow/python/ops/embedding_ops.py
+++ b/tensorflow/python/ops/embedding_ops.py
@@ -222,7 +222,7 @@ def _embedding_lookup_and_transform(params,
             result = transform_fn(_clip(result, pids, max_norm))
         partitioned_result.append(result)
       # Stitch these back together
-      ret = data_flow_ops.dynamic_stitch(
+      ret = data_flow_ops.parallel_dynamic_stitch(
           pindices, partitioned_result, name=name)
 
       # Determine the static element shape.
-- 
GitLab


From c801d23a3c449cf7ed44df5320e0bc9746bc3752 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 11 Oct 2017 15:34:18 -0700
Subject: [PATCH 274/909] Fix LSTM tests to use the same parameters for all
 blocks.  Now that the tests are correct, lower the tolerance from 1e-2 to
 1e-6.

PiperOrigin-RevId: 171885525
---
 .../rnn/python/kernel_tests/lstm_ops_test.py  | 367 ++++++++----------
 1 file changed, 161 insertions(+), 206 deletions(-)

diff --git a/tensorflow/contrib/rnn/python/kernel_tests/lstm_ops_test.py b/tensorflow/contrib/rnn/python/kernel_tests/lstm_ops_test.py
index 3f72203594..1980d64cd6 100644
--- a/tensorflow/contrib/rnn/python/kernel_tests/lstm_ops_test.py
+++ b/tensorflow/contrib/rnn/python/kernel_tests/lstm_ops_test.py
@@ -38,6 +38,111 @@ from tensorflow.python.platform import test
 block_lstm = lstm_ops._block_lstm  # pylint: disable=protected-access
 
 
+def blocks_match(sess, use_peephole):
+  batch_size = 2
+  input_size = 3
+  cell_size = 4
+  sequence_length = 4
+
+  inputs = []
+  for _ in range(sequence_length):
+    inp = ops.convert_to_tensor(
+        np.random.randn(batch_size, input_size), dtype=dtypes.float32)
+    inputs.append(inp)
+
+  initializer = init_ops.random_uniform_initializer(-0.01, 0.01, seed=19890212)
+
+  with variable_scope.variable_scope("test", initializer=initializer):
+    # magic naming so that the cells pick up these variables and resuse them
+    if use_peephole:
+      wci = variable_scope.get_variable(
+          "rnn/lstm_cell/w_i_diag", shape=[cell_size], dtype=dtypes.float32)
+      wcf = variable_scope.get_variable(
+          "rnn/lstm_cell/w_f_diag", shape=[cell_size], dtype=dtypes.float32)
+      wco = variable_scope.get_variable(
+          "rnn/lstm_cell/w_o_diag", shape=[cell_size], dtype=dtypes.float32)
+
+    w = variable_scope.get_variable(
+        "rnn/lstm_cell/kernel",
+        shape=[input_size + cell_size, cell_size * 4],
+        dtype=dtypes.float32)
+    b = variable_scope.get_variable(
+        "rnn/lstm_cell/bias",
+        shape=[cell_size * 4],
+        dtype=dtypes.float32,
+        initializer=init_ops.zeros_initializer())
+
+    if use_peephole:
+      wci_block = variable_scope.get_variable(
+          "rnn/lstm_cell/lstm_block_wrapper/w_i_diag",
+          initializer=wci.initialized_value())
+      wcf_block = variable_scope.get_variable(
+          "rnn/lstm_cell/lstm_block_wrapper/w_f_diag",
+          initializer=wcf.initialized_value())
+      wco_block = variable_scope.get_variable(
+          "rnn/lstm_cell/lstm_block_wrapper/w_o_diag",
+          initializer=wco.initialized_value())
+    w_block = variable_scope.get_variable(
+        "rnn/lstm_cell/lstm_block_wrapper/kernel",
+        initializer=w.initialized_value())
+    b_block = variable_scope.get_variable(
+        "rnn/lstm_cell/lstm_block_wrapper/bias",
+        initializer=b.initialized_value())
+
+    basic_cell = rnn_cell.LSTMCell(
+        cell_size, use_peepholes=use_peephole, state_is_tuple=True, reuse=True)
+    basic_outputs_op, basic_state_op = rnn.static_rnn(
+        basic_cell, inputs, dtype=dtypes.float32)
+
+    if use_peephole:
+      _, _, _, _, _, _, block_outputs_op = block_lstm(
+          ops.convert_to_tensor(sequence_length, dtype=dtypes.int64),
+          inputs,
+          w,
+          b,
+          wci=wci,
+          wcf=wcf,
+          wco=wco,
+          cell_clip=0,
+          use_peephole=True)
+    else:
+      _, _, _, _, _, _, block_outputs_op = block_lstm(
+          ops.convert_to_tensor(sequence_length, dtype=dtypes.int64),
+          inputs,
+          w,
+          b,
+          cell_clip=0)
+
+    with variable_scope.variable_scope("rnn/lstm_cell", reuse=True):
+      fused_cell = lstm_ops.LSTMBlockFusedCell(
+          cell_size, cell_clip=0, use_peephole=use_peephole)
+      fused_outputs_op, fused_state_op = fused_cell(
+          inputs, dtype=dtypes.float32)
+
+    sess.run([variables.global_variables_initializer()])
+    basic_outputs, basic_state = sess.run([basic_outputs_op, basic_state_op[0]])
+    basic_grads = sess.run(gradients_impl.gradients(basic_outputs_op, inputs))
+    xs = [w, b]
+    if use_peephole:
+      xs += [wci, wcf, wco]
+    basic_wgrads = sess.run(gradients_impl.gradients(basic_outputs_op, xs))
+
+    block_outputs = sess.run(block_outputs_op)
+    block_grads = sess.run(gradients_impl.gradients(block_outputs_op, inputs))
+    block_wgrads = sess.run(gradients_impl.gradients(block_outputs_op, xs))
+
+    xs = [w_block, b_block]
+    if use_peephole:
+      xs += [wci_block, wcf_block, wco_block]
+    fused_outputs, fused_state = sess.run([fused_outputs_op, fused_state_op[0]])
+    fused_grads = sess.run(gradients_impl.gradients(fused_outputs_op, inputs))
+    fused_wgrads = sess.run(gradients_impl.gradients(fused_outputs_op, xs))
+
+    return (basic_state, fused_state, basic_outputs, block_outputs,
+            fused_outputs, basic_grads, block_grads, fused_grads, basic_wgrads,
+            block_wgrads, fused_wgrads)
+
+
 class LSTMBlockCellTest(test.TestCase):
 
   def testNoneDimsWithDynamicRNN(self):
@@ -227,173 +332,28 @@ class LSTMBlockCellTest(test.TestCase):
 
   def testLSTMBasicToBlock(self):
     with self.test_session(use_gpu=True) as sess:
-      batch_size = 2
-      input_size = 3
-      cell_size = 4
-      sequence_length = 5
-
-      inputs = []
-      for _ in range(sequence_length):
-        inp = ops.convert_to_tensor(
-            np.random.randn(batch_size, input_size), dtype=dtypes.float32)
-        inputs.append(inp)
-
-      initializer = init_ops.random_uniform_initializer(
-          -0.01, 0.01, seed=19890212)
-      with variable_scope.variable_scope("basic", initializer=initializer):
-        cell = rnn_cell.BasicLSTMCell(cell_size, state_is_tuple=True)
-        outputs, state = rnn.static_rnn(cell, inputs, dtype=dtypes.float32)
-
-        sess.run([variables.global_variables_initializer()])
-        basic_outputs, basic_state = sess.run([outputs, state[0]])
-        basic_grads = sess.run(gradients_impl.gradients(outputs, inputs))
-        basic_wgrads = sess.run(
-            gradients_impl.gradients(outputs, variables.trainable_variables()))
-
-      with variable_scope.variable_scope("block", initializer=initializer):
-        w = variable_scope.get_variable(
-            "w",
-            shape=[input_size + cell_size, cell_size * 4],
-            dtype=dtypes.float32)
-        b = variable_scope.get_variable(
-            "b",
-            shape=[cell_size * 4],
-            dtype=dtypes.float32,
-            initializer=init_ops.zeros_initializer())
-
-        _, _, _, _, _, _, outputs = block_lstm(
-            ops.convert_to_tensor(
-                sequence_length, dtype=dtypes.int64),
-            inputs,
-            w,
-            b,
-            cell_clip=0)
-
-        sess.run([variables.global_variables_initializer()])
-        block_outputs = sess.run(outputs)
-        block_grads = sess.run(gradients_impl.gradients(outputs, inputs))
-        block_wgrads = sess.run(gradients_impl.gradients(outputs, [w, b]))
+      (basic_state, fused_state, basic_outputs, block_outputs, fused_outputs,
+       basic_grads, block_grads, fused_grads, basic_wgrads, block_wgrads,
+       fused_wgrads) = blocks_match(
+           sess, use_peephole=False)
 
       self.assertAllClose(basic_outputs, block_outputs)
       self.assertAllClose(basic_grads, block_grads)
       for basic, block in zip(basic_wgrads, block_wgrads):
-        self.assertAllClose(basic, block, rtol=1e-2, atol=1e-2)
-
-      with variable_scope.variable_scope("fused", initializer=initializer):
-        cell = lstm_ops.LSTMBlockFusedCell(
-            cell_size, cell_clip=0, use_peephole=False)
-        outputs, state = cell(inputs, dtype=dtypes.float32)
-
-        sess.run([variables.global_variables_initializer()])
-        fused_outputs, fused_state = sess.run([outputs, state[0]])
-        fused_grads = sess.run(gradients_impl.gradients(outputs, inputs))
-        fused_vars = [
-            v for v in variables.trainable_variables()
-            if v.name.startswith("fused/")
-        ]
-        fused_wgrads = sess.run(gradients_impl.gradients(outputs, fused_vars))
+        self.assertAllClose(basic, block, rtol=1e-6, atol=1e-6)
 
       self.assertAllClose(basic_outputs, fused_outputs)
       self.assertAllClose(basic_state, fused_state)
       self.assertAllClose(basic_grads, fused_grads)
-      for basic, fused in zip(basic_wgrads, fused_wgrads):
-        self.assertAllClose(basic, fused, rtol=1e-2, atol=1e-2)
+      for basic, fused in zip(block_wgrads, fused_wgrads):
+        self.assertAllClose(basic, fused, rtol=1e-6, atol=1e-6)
 
   def testLSTMBasicToBlockPeeping(self):
     with self.test_session(use_gpu=True) as sess:
-      batch_size = 2
-      input_size = 3
-      cell_size = 4
-      sequence_length = 4
-
-      inputs = []
-      for _ in range(sequence_length):
-        inp = ops.convert_to_tensor(
-            np.random.randn(batch_size, input_size), dtype=dtypes.float32)
-        inputs.append(inp)
-
-      initializer = init_ops.random_uniform_initializer(
-          -0.01, 0.01, seed=19890212)
-
-      with variable_scope.variable_scope("test", initializer=initializer):
-        # magic naming so that the cells pick up these variables and resuse them
-        wci = variable_scope.get_variable(
-            "rnn/lstm_cell/w_i_diag", shape=[cell_size], dtype=dtypes.float32)
-        wcf = variable_scope.get_variable(
-            "rnn/lstm_cell/w_f_diag", shape=[cell_size], dtype=dtypes.float32)
-        wco = variable_scope.get_variable(
-            "rnn/lstm_cell/w_o_diag", shape=[cell_size], dtype=dtypes.float32)
-
-        w = variable_scope.get_variable(
-            "rnn/lstm_cell/kernel",
-            shape=[input_size + cell_size, cell_size * 4],
-            dtype=dtypes.float32)
-        b = variable_scope.get_variable(
-            "rnn/lstm_cell/bias",
-            shape=[cell_size * 4],
-            dtype=dtypes.float32,
-            initializer=init_ops.zeros_initializer())
-
-        wci_block = variable_scope.get_variable(
-            "rnn/lstm_cell/lstm_block_wrapper/w_i_diag",
-            initializer=wci.initialized_value())
-        wcf_block = variable_scope.get_variable(
-            "rnn/lstm_cell/lstm_block_wrapper/w_f_diag",
-            initializer=wcf.initialized_value())
-        wco_block = variable_scope.get_variable(
-            "rnn/lstm_cell/lstm_block_wrapper/w_o_diag",
-            initializer=wco.initialized_value())
-        w_block = variable_scope.get_variable(
-            "rnn/lstm_cell/lstm_block_wrapper/kernel",
-            initializer=w.initialized_value())
-        b_block = variable_scope.get_variable(
-            "rnn/lstm_cell/lstm_block_wrapper/bias",
-            initializer=b.initialized_value())
-
-        basic_cell = rnn_cell.LSTMCell(
-            cell_size, use_peepholes=True, state_is_tuple=True, reuse=True)
-        basic_outputs_op, basic_state_op = rnn.static_rnn(
-            basic_cell, inputs, dtype=dtypes.float32)
-
-        _, _, _, _, _, _, block_outputs_op = block_lstm(
-            ops.convert_to_tensor(sequence_length, dtype=dtypes.int64),
-            inputs,
-            w,
-            b,
-            wci=wci,
-            wcf=wcf,
-            wco=wco,
-            cell_clip=0,
-            use_peephole=True)
-
-        with variable_scope.variable_scope("rnn/lstm_cell", reuse=True):
-          fused_cell = lstm_ops.LSTMBlockFusedCell(
-              cell_size, cell_clip=0, use_peephole=True)
-          fused_outputs_op, fused_state_op = fused_cell(
-              inputs, dtype=dtypes.float32)
-
-        sess.run([variables.global_variables_initializer()])
-        basic_outputs, basic_state = sess.run(
-            [basic_outputs_op, basic_state_op[0]])
-        basic_grads = sess.run(
-            gradients_impl.gradients(basic_outputs_op, inputs))
-        basic_wgrads = sess.run(
-            gradients_impl.gradients(basic_outputs_op, [w, b, wci, wcf, wco]))
-
-        block_outputs = sess.run(block_outputs_op)
-        block_grads = sess.run(
-            gradients_impl.gradients(block_outputs_op, inputs))
-        block_wgrads = sess.run(
-            gradients_impl.gradients(block_outputs_op, [w, b, wci, wcf, wco]))
-
-        fused_outputs, fused_state = sess.run(
-            [fused_outputs_op, fused_state_op[0]])
-        fused_grads = sess.run(
-            gradients_impl.gradients(fused_outputs_op, inputs))
-        fused_wgrads = sess.run(
-            gradients_impl.gradients(
-                fused_outputs_op,
-                [w_block, b_block, wci_block, wcf_block, wco_block]))
+      (basic_state, fused_state, basic_outputs, block_outputs, fused_outputs,
+       basic_grads, block_grads, fused_grads, basic_wgrads, block_wgrads,
+       fused_wgrads) = blocks_match(
+           sess, use_peephole=True)
 
       self.assertAllClose(basic_outputs, block_outputs)
       self.assertAllClose(basic_grads, block_grads)
@@ -423,45 +383,38 @@ class LSTMBlockCellTest(test.TestCase):
 
       initializer = init_ops.random_uniform_initializer(
           -0.01, 0.01, seed=19890213)
-      with variable_scope.variable_scope("basic", initializer=initializer):
-        cell = rnn_cell.BasicLSTMCell(cell_size, state_is_tuple=True)
-        outputs, state = rnn.static_rnn(
-            cell, inputs, dtype=dtypes.float32, sequence_length=seq_lengths)
-        sess.run([variables.global_variables_initializer()])
-        basic_outputs, basic_state = sess.run([outputs, state[0]])
-        basic_grads = sess.run(gradients_impl.gradients(outputs, inputs))
-        basic_wgrads = sess.run(
-            gradients_impl.gradients(outputs, variables.trainable_variables()))
 
-      with variable_scope.variable_scope("fused", initializer=initializer):
+      with variable_scope.variable_scope(
+          "lstm_block_wrapper", initializer=initializer):
+        # magic naming so that the cells pick up these variables and resuse them
+        variable_scope.get_variable(
+            "kernel",
+            shape=[input_size + cell_size, cell_size * 4],
+            dtype=dtypes.float32)
+
+        variable_scope.get_variable(
+            "bias",
+            shape=[cell_size * 4],
+            dtype=dtypes.float32,
+            initializer=init_ops.zeros_initializer())
+
+      with variable_scope.variable_scope("", reuse=True):
         cell = lstm_ops.LSTMBlockFusedCell(
             cell_size, cell_clip=0, use_peephole=False)
-        outputs, state = cell(
+
+        fused_outputs_op, fused_state_op = cell(
             inputs, dtype=dtypes.float32, sequence_length=seq_lengths)
 
-        sess.run([variables.global_variables_initializer()])
-        fused_outputs, fused_state = sess.run([outputs, state[0]])
-        fused_grads = sess.run(gradients_impl.gradients(outputs, inputs))
-        fused_vars = [
-            v for v in variables.trainable_variables()
-            if v.name.startswith("fused/")
-        ]
-        fused_wgrads = sess.run(gradients_impl.gradients(outputs, fused_vars))
-
-      self.assertAllClose(basic_outputs, fused_outputs)
-      self.assertAllClose(basic_state, fused_state)
-      self.assertAllClose(basic_grads, fused_grads)
-      for basic, fused in zip(basic_wgrads, fused_wgrads):
-        self.assertAllClose(basic, fused, rtol=1e-2, atol=1e-2)
+      cell_vars = [
+          v for v in variables.trainable_variables()
+          if v.name.endswith("kernel") or v.name.endswith("bias")
+      ]
 
       # Verify that state propagation works if we turn our sequence into
       # tiny (single-time) subsequences, i.e. unfuse the cell
-      with variable_scope.variable_scope(
-          "unfused", initializer=initializer) as vs:
-        cell = lstm_ops.LSTMBlockFusedCell(
-            cell_size, cell_clip=0, use_peephole=False)
-        outputs = []
-        state = None
+      unfused_outputs_op = []
+      state = None
+      with variable_scope.variable_scope("", reuse=True):
         for i, inp in enumerate(inputs):
           lengths = [int(i < l) for l in seq_lengths.eval()]
           output, state = cell(
@@ -469,25 +422,27 @@ class LSTMBlockCellTest(test.TestCase):
               initial_state=state,
               dtype=dtypes.float32,
               sequence_length=lengths)
-          vs.reuse_variables()
-          outputs.append(output[0])
-        outputs = array_ops.stack(outputs)
-
-        sess.run([variables.global_variables_initializer()])
-        unfused_outputs, unfused_state = sess.run([outputs, state[0]])
-        unfused_grads = sess.run(gradients_impl.gradients(outputs, inputs))
-        unfused_vars = [
-            v for v in variables.trainable_variables()
-            if v.name.startswith("unfused/")
-        ]
-        unfused_wgrads = sess.run(
-            gradients_impl.gradients(outputs, unfused_vars))
-
-      self.assertAllClose(basic_outputs, unfused_outputs)
-      self.assertAllClose(basic_state, unfused_state)
-      self.assertAllClose(basic_grads, unfused_grads)
-      for basic, unfused in zip(basic_wgrads, unfused_wgrads):
-        self.assertAllClose(basic, unfused, rtol=1e-2, atol=1e-2)
+          unfused_outputs_op.append(output[0])
+      unfused_outputs_op = array_ops.stack(unfused_outputs_op)
+
+      sess.run([variables.global_variables_initializer()])
+      unfused_outputs, unfused_state = sess.run([unfused_outputs_op, state[0]])
+      unfused_grads = sess.run(
+          gradients_impl.gradients(unfused_outputs_op, inputs))
+      unfused_wgrads = sess.run(
+          gradients_impl.gradients(unfused_outputs_op, cell_vars))
+
+      fused_outputs, fused_state = sess.run(
+          [fused_outputs_op, fused_state_op[0]])
+      fused_grads = sess.run(gradients_impl.gradients(fused_outputs_op, inputs))
+      fused_wgrads = sess.run(
+          gradients_impl.gradients(fused_outputs_op, cell_vars))
+
+      self.assertAllClose(fused_outputs, unfused_outputs)
+      self.assertAllClose(fused_state, unfused_state)
+      self.assertAllClose(fused_grads, unfused_grads)
+      for fused, unfused in zip(fused_wgrads, unfused_wgrads):
+        self.assertAllClose(fused, unfused, rtol=1e-6, atol=1e-6)
 
 #### Benchmarking.
 
-- 
GitLab


From 6285db2546f03296a4f30071ce96217ccd17c452 Mon Sep 17 00:00:00 2001
From: Adam Roberts <adarob@google.com>
Date: Wed, 11 Oct 2017 16:09:14 -0700
Subject: [PATCH 275/909] Link RNN ops/kernels in contrib/BUILD.

PiperOrigin-RevId: 171890081
---
 tensorflow/contrib/BUILD     |  2 ++
 tensorflow/contrib/rnn/BUILD | 22 ++++++++++++++++++----
 2 files changed, 20 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/BUILD b/tensorflow/contrib/BUILD
index 65c966aa03..559e3e60d7 100644
--- a/tensorflow/contrib/BUILD
+++ b/tensorflow/contrib/BUILD
@@ -104,6 +104,7 @@ cc_library(
         "//tensorflow/contrib/layers:sparse_feature_cross_op_kernel",
         "//tensorflow/contrib/nccl:nccl_kernels",
         "//tensorflow/contrib/nearest_neighbor:nearest_neighbor_ops_kernels",
+        "//tensorflow/contrib/rnn:all_kernels",
         "//tensorflow/contrib/seq2seq:beam_search_ops_kernels",
         "//tensorflow/contrib/tensor_forest:model_ops_kernels",
         "//tensorflow/contrib/tensor_forest:stats_ops_kernels",
@@ -125,6 +126,7 @@ cc_library(
         "//tensorflow/contrib/layers:sparse_feature_cross_op_op_lib",
         "//tensorflow/contrib/nccl:nccl_ops_op_lib",
         "//tensorflow/contrib/nearest_neighbor:nearest_neighbor_ops_op_lib",
+        "//tensorflow/contrib/rnn:all_ops",
         "//tensorflow/contrib/seq2seq:beam_search_ops_op_lib",
         "//tensorflow/contrib/tensor_forest:model_ops_op_lib",
         "//tensorflow/contrib/tensor_forest:stats_ops_op_lib",
diff --git a/tensorflow/contrib/rnn/BUILD b/tensorflow/contrib/rnn/BUILD
index 7dc76cf622..37fe6e0163 100644
--- a/tensorflow/contrib/rnn/BUILD
+++ b/tensorflow/contrib/rnn/BUILD
@@ -24,6 +24,22 @@ load(
     "tf_kernel_tests_linkstatic",
 )
 
+cc_library(
+    name = "all_ops",
+    deps = [
+        ":gru_ops_op_lib",
+        ":lstm_ops_op_lib",
+    ],
+)
+
+cc_library(
+    name = "all_kernels",
+    deps = [
+        ":gru_ops_kernels",
+        ":lstm_ops_kernels",
+    ],
+)
+
 tf_custom_op_py_library(
     name = "rnn_py",
     srcs = ["__init__.py"] + glob(["python/ops/*.py"]) + [
@@ -34,10 +50,8 @@ tf_custom_op_py_library(
         ":python/ops/_lstm_ops.so",
     ],
     kernels = [
-        ":gru_ops_kernels",
-        ":lstm_ops_kernels",
-        ":gru_ops_op_lib",
-        ":lstm_ops_op_lib",
+        ":all_ops",
+        ":all_kernels",
     ],
     srcs_version = "PY2AND3",
     visibility = ["//visibility:public"],
-- 
GitLab


From d835d677ade78a41e0e097f67c87b6ab8588a90a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 11 Oct 2017 16:53:45 -0700
Subject: [PATCH 276/909] Extend the transpose ops in TensorFlow to support
 conjugate (a.k.a. Hermitian) transposition. Currently, this can only be
 accomplished by adding extra conjugation ops, which means reading the tensor
 data from memory twice. More importantly, Hermitian transpose is the most
 common transposition operation when using complex arithmetic, so using it in
 new code helps prevent "conjugation bugs" by making the math work for real
 and complex types alike. The alias tf.linalg.adjoint was added to help with
 the latter.

Optimized fused conjugate transpose op for GPU will be added in a followup.

Get rid of some duplication of code among CPU/GPU/SYCL in transpose_functor.

Support accelerating 2D transpose ops using MKL in more cases.

PiperOrigin-RevId: 171895454
---
 tensorflow/core/kernels/BUILD                 |  31 +--
 tensorflow/core/kernels/mkl_transpose_op.cc   |  94 +++++--
 tensorflow/core/kernels/transpose_functor.h   | 106 +++++++-
 .../core/kernels/transpose_functor_cpu.cc     | 207 +++++-----------
 .../core/kernels/transpose_functor_gpu.cu.cc  | 233 ++++++++++--------
 tensorflow/core/kernels/transpose_op.cc       |  72 +++++-
 tensorflow/core/kernels/transpose_op.h        |  53 +++-
 tensorflow/core/ops/array_ops.cc              | 131 +++++-----
 .../python/kernel_tests/array_ops_test.py     |   9 +
 .../python/kernel_tests/linalg_ops_test.py    |  15 ++
 .../python/kernel_tests/transpose_op_test.py  |  24 +-
 tensorflow/python/ops/array_ops.py            |  35 ++-
 tensorflow/python/ops/hidden_ops.txt          |   1 +
 tensorflow/python/ops/linalg/linalg_impl.py   |  25 ++
 .../tools/api/golden/tensorflow.linalg.pbtxt  |   6 +-
 tensorflow/tools/api/golden/tensorflow.pbtxt  |   4 +-
 16 files changed, 662 insertions(+), 384 deletions(-)

diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 3b7d803bea..dbf6449bc2 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -1350,10 +1350,11 @@ tf_kernel_library(
     ],
     visibility = [":friends"],
     deps = [
+        ":conv_ops",
+        ":cwise_op",
         ":ops_util",
         "//tensorflow/core:framework",
         "//tensorflow/core:protos_all_cc",
-        "//tensorflow/core/kernels:conv_ops",
         "//third_party/eigen3",
     ],
     alwayslink = 0,
@@ -2276,13 +2277,15 @@ LINALG_DEPS = [
     "//tensorflow/core:framework",
     "//tensorflow/core:lib",
     "//tensorflow/core:linalg_ops_op_lib",
-]
+] + if_cuda([
+    ":cuda_solvers",
+    ":transpose_functor",
+])
 
 tf_kernel_library(
     name = "cholesky_op",
     prefix = "cholesky_op",
     deps = if_cuda([
-        ":cuda_solvers",
         ":matrix_band_part_op",
     ]) + LINALG_DEPS,
 )
@@ -2297,7 +2300,6 @@ tf_kernel_library(
     name = "determinant_op",
     prefix = "determinant_op",
     deps = if_cuda([
-        ":cuda_solvers",
         ":fill_functor",
     ]) + LINALG_DEPS,
 )
@@ -2314,17 +2316,13 @@ tf_kernel_library(
     deps = LINALG_DEPS + if_cuda([
         ":cast_op",
         ":cwise_op",
-        ":cuda_solvers",
-        ":transpose_functor",
     ]),
 )
 
 tf_kernel_library(
     name = "matrix_inverse_op",
     prefix = "matrix_inverse_op",
-    deps = if_cuda([
-        ":cuda_solvers",
-    ]) + LINALG_DEPS,
+    deps = LINALG_DEPS,
 )
 
 tf_kernel_library(
@@ -2336,10 +2334,7 @@ tf_kernel_library(
 tf_kernel_library(
     name = "matrix_solve_op",
     prefix = "matrix_solve_op",
-    deps = if_cuda([
-        ":cuda_solvers",
-        ":transpose_functor",
-    ]) + LINALG_DEPS,
+    deps = LINALG_DEPS,
 )
 
 tf_kernel_library(
@@ -2354,20 +2349,15 @@ tf_kernel_library(
     name = "qr_op",
     prefix = "qr_op",
     deps = LINALG_DEPS + if_cuda([
-        ":cuda_solvers",
         ":cwise_op",
         ":matrix_band_part_op",
-        ":transpose_functor",
     ]),
 )
 
 tf_kernel_library(
     name = "svd_op",
     prefix = "svd_op",
-    deps = LINALG_DEPS + if_cuda([
-        ":cuda_solvers",
-        ":transpose_functor",
-    ]),
+    deps = LINALG_DEPS,
 )
 
 cc_library(
@@ -2457,7 +2447,6 @@ tf_cc_tests(
 MATH_DEPS = [
     ":bounds_check",
     ":fill_functor",
-    ":transpose_functor",
     "//tensorflow/core:core_cpu",
     "//tensorflow/core:framework",
     "//tensorflow/core:lib",
@@ -2617,7 +2606,7 @@ tf_kernel_library(
     name = "reduction_ops",
     gpu_srcs = ["reduction_gpu_kernels.cu.h"],
     prefix = "reduction_ops",
-    deps = MATH_DEPS + if_cuda(["@cub_archive//:cub"]),
+    deps = MATH_DEPS + [":transpose_functor"] + if_cuda(["@cub_archive//:cub"]),
 )
 
 tf_kernel_library(
diff --git a/tensorflow/core/kernels/mkl_transpose_op.cc b/tensorflow/core/kernels/mkl_transpose_op.cc
index 50d25ac511..89a1d5e8a7 100644
--- a/tensorflow/core/kernels/mkl_transpose_op.cc
+++ b/tensorflow/core/kernels/mkl_transpose_op.cc
@@ -39,28 +39,86 @@ namespace tensorflow {
 // REQUIRES: input.dims() == perm.size().
 // REQUIRES: perm is a permutation.
 
-Status MklTransposeCpuOp::DoTranspose(OpKernelContext* ctx, const Tensor& in,
-                                      gtl::ArraySlice<int32> perm,
-                                      Tensor* out) {
-  if (in.dims() == 2 && in.dtype() == DT_FLOAT) {
-    float* user_o = out->flat<float>().data();
-    const float* user_i = in.flat<float>().data();
-
-    // Documentation here: https://software.intel.com/en-us/node/520863
-    // Parameters: (ordering:row-major, operation:transpose, num_rows, num_cols,
-    //              alpha (for scaling), array, dist_bet_adjacent_cols/rows
-    //              (source), array, dist_bet_adjacent_cols/rows (dest))
-    mkl_somatcopy('R', 'T', in.dim_size(0), in.dim_size(1), 1, user_i,
-                  in.dim_size(1), user_o, in.dim_size(0));
+namespace {
+template <typename T>
+void MKLTranspose2D(const char trans, const Tensor& in, Tensor* out) {}
 
+// Documentation here: https://software.intel.com/en-us/node/520863
+// Parameters: (ordering:row-major, operation:transpose, num_rows, num_cols,
+//              alpha (for scaling), array, dist_bet_adjacent_cols/rows
+//              (source), array, dist_bet_adjacent_cols/rows (dest))
+
+#define INSTANTIATE(T, PREFIX)                                                \
+  template <>                                                                 \
+  Status MKLTranspose2D<T>(const char trans, const Tensor& in, Tensor* out) { \
+    mkl_##PREFIX##omatcopy('R', trans, in.dim_size(0), in.dim_size(1), 1,     \
+                           in.flat<T>().data(), in.dim_size(1),               \
+                           out->flat<T>().data(), in.dim_size(0));            \
     return Status::OK();
   }
 
-  // Fallback to eigen if transpose parameters not supported by MKL
-  typedef Eigen::ThreadPoolDevice CPUDevice;
-  return ::tensorflow::DoTranspose(ctx->eigen_device<CPUDevice>(), in, perm,
-                                   out);
-}  // MklTransposeCpuOp::DoTranspose
+  INSTANTIATE(float, s)
+  INSTANTIATE(double, d)
+  INSTANTIATE(complex64, c)
+  INSTANTIATE(complex128, z)
+#undef INSTANTIATE
+
+  static const char kMKLTranspose = 'T';
+  static const char kMKLConjugateTranspose = 'C';
+
+  }  // namespace tensorflow
+
+  Status MklTransposeCpuOp::DoTranspose(OpKernelContext* ctx, const Tensor& in,
+                                        gtl::ArraySlice<int32> perm,
+                                        Tensor* out) {
+    if (in.dims() == 2) {
+      switch (in.dtype()) {
+        case DT_FLOAT:
+          return MKLTranspose2D<float>(kMKLTranspose, in, out);
+        case DT_DOUBLE:
+          return MKLTranspose2D<double>(kMKLTranspose, in, out);
+        case DT_COMPLEX64:
+          return MKLTranspose2D<complex64>(kMKLTranspose, in, out);
+        case DT_COMPLEX128:
+          return MKLTranspose2D<complex128>(kMKLTranspose, in, out);
+        default:
+          break;
+      }
+    }
+    // Fallback to eigen if transpose parameters not supported by MKL
+    typedef Eigen::ThreadPoolDevice CPUDevice;
+    return ::tensorflow::DoTranspose(ctx->eigen_device<CPUDevice>(), in, perm,
+                                     out);
+  }
+
+  Status MklConjugateTransposeCpuOp::DoTranspose(OpKernelContext* ctx,
+                                                 const Tensor& in,
+                                                 gtl::ArraySlice<int32> perm,
+                                                 Tensor* out) {
+    if (in.dims() == 2) {
+      // TODO(rmlarsen): By setting lda and ldb, we could use the MKL kernels
+      // for any transpose that can be reduced to swapping the last two
+      // dimensions in a rank-3 tensor. We can even run each outer dimension in
+      // a separate thread.
+      switch (in.dtype()) {
+        case DT_FLOAT:
+          return MKLTranspose2D<float>(kMKLTranspose, in, out);
+        case DT_DOUBLE:
+          return MKLTranspose2D<double>(kMKLTranspose, in, out);
+        case DT_COMPLEX64:
+          return MKLTranspose2D<complex64>(kMKLConjugateTranspose, in, out);
+        case DT_COMPLEX128:
+          return MKLTranspose2D<complex128>(kMKLConjugateTranspose, in, out);
+        default:
+          break;
+      }
+    }
+    // Fallback to eigen if transpose parameters not supported by MKL
+    typedef Eigen::ThreadPoolDevice CPUDevice;
+    return ::tensorflow::DoConjugateTranspose(ctx->eigen_device<CPUDevice>(),
+                                              in, perm, out);
+  }
+
 }  // namespace tensorflow
 
 #endif  // INTEL_MKL
diff --git a/tensorflow/core/kernels/transpose_functor.h b/tensorflow/core/kernels/transpose_functor.h
index 498030fdfe..317a534fd6 100644
--- a/tensorflow/core/kernels/transpose_functor.h
+++ b/tensorflow/core/kernels/transpose_functor.h
@@ -32,6 +32,24 @@ template <typename Device>
 Status DoTranspose(const Device& device, const Tensor& in,
                    const gtl::ArraySlice<int32> perm, Tensor* out);
 
+// Conjugate and transpose tensor 'in' into tensor 'out' according to dimension
+// permutation 'perm'.
+//
+// REQUIRES: in.dtype() == out->dtype()
+// REQUIRES: in.dims() == out->dims()
+// REQUIRES: in.dims() == perm.size()
+// REQUIRES: in.dim_size(perm[i]) == out->dim_size(i)
+template <typename Device>
+Status DoConjugateTranspose(const Device& device, const Tensor& in,
+                            const gtl::ArraySlice<int32> perm, Tensor* out);
+
+// Primary device specific functor to be specialized for each device and type.
+template <typename Device, typename T, bool conjugate = false>
+struct Transpose {
+  static void run(const Device& d, const Tensor& in,
+                  const gtl::ArraySlice<int32> perm, Tensor* out);
+};
+
 // Implementation details.
 namespace internal {
 
@@ -111,14 +129,15 @@ inline bool NonSingletonDimensionsAlign(const TensorShape& input_shape,
 }
 
 // Device-specific naive implementation for transpose.
-template <typename Device, typename T>
+template <typename Device, typename T, bool conjugate>
 void TransposeSimple(const Device& d, const Tensor& in,
                      const gtl::ArraySlice<int32> perm, Tensor* out);
 
 // Uses Eigen to transpose.
 template <typename Device, typename T, int NDIMS>
 void TransposeUsingEigen(const Device& d, const Tensor& in,
-                         const gtl::ArraySlice<int32> perm, Tensor* out) {
+                         const gtl::ArraySlice<int32> perm, bool conjugate,
+                         Tensor* out) {
   Eigen::array<int, NDIMS> p;
   for (int i = 0; i < NDIMS; ++i) p[i] = perm[i];
   auto x = typename TTypes<T, NDIMS>::ConstTensor(
@@ -127,24 +146,87 @@ void TransposeUsingEigen(const Device& d, const Tensor& in,
   auto y = typename TTypes<T, NDIMS>::Tensor(
       reinterpret_cast<T*>(const_cast<char*>(out->tensor_data().data())),
       out->shape().AsEigenDSizes<NDIMS>());
-  y.device(d) = x.shuffle(p);
+  if (conjugate) {
+    y.device(d) = x.conjugate().shuffle(p);
+  } else {
+    y.device(d) = x.shuffle(p);
+  }
 }
 
+template <typename Device>
+struct DoTransposeImpl {
+  static Status run(const Device& d, const Tensor& in,
+                    const gtl::ArraySlice<int32> perm, bool conjugate,
+                    Tensor* out) {
+    CHECK_GE(in.dims(), 2);
+    CHECK_EQ(in.dims(), out->dims());
+    CHECK_EQ(in.dims(), perm.size());
+    CHECK_EQ(in.dtype(), out->dtype());
+    switch (in.dtype()) {
+      case DT_BOOL:
+      case DT_INT8:
+      case DT_QINT8:
+      case DT_QUINT8:
+      case DT_UINT8:
+        Transpose<Device, uint8>::run(d, in, perm, out);
+        break;
+
+      case DT_BFLOAT16:
+      case DT_HALF:
+      case DT_INT16:
+      case DT_QINT16:
+      case DT_QUINT16:
+      case DT_UINT16:
+        Transpose<Device, uint16>::run(d, in, perm, out);
+        break;
+
+      case DT_FLOAT:
+      case DT_INT32:
+      case DT_QINT32:
+        Transpose<Device, uint32>::run(d, in, perm, out);
+        break;
+
+      case DT_DOUBLE:
+      case DT_INT64:
+        Transpose<Device, uint64>::run(d, in, perm, out);
+        break;
+
+      case DT_COMPLEX64:
+        if (conjugate) {
+          Transpose<Device, complex64, true>::run(d, in, perm, out);
+        } else {
+          Transpose<Device, complex64, false>::run(d, in, perm, out);
+        }
+        break;
+
+      case DT_COMPLEX128:
+        if (conjugate) {
+          Transpose<Device, complex128, true>::run(d, in, perm, out);
+        } else {
+          Transpose<Device, complex128, false>::run(d, in, perm, out);
+        }
+        break;
+
+      case DT_STRING:
+        Transpose<Device, string>::run(d, in, perm, out);
+        break;
+
+      default:
+        return errors::Unimplemented("Unsupported dtype on CPU: ", in.dtype());
+    }
+    return Status::OK();
+  }
+};
 
 #ifdef TENSORFLOW_USE_SYCL
 // For SYCL lets always go through Eigen
 template <typename Device, typename T>
 void TransposeSYCL(const Device& d, const Tensor& in,
-                   const gtl::ArraySlice<int32> perm, Tensor* out);
-#endif // TENSORFLOW_USE_SYCL
-}  // namespace internal
-
-template <typename Device, typename T>
-struct Transpose {
-  static void run(const Device& d, const Tensor& in,
-                  const gtl::ArraySlice<int32> perm, Tensor* out);
-};
+                   const gtl::ArraySlice<int32> perm, bool conjugate,
+                   Tensor* out);
+#endif  // TENSORFLOW_USE_SYCL
 
+}  // namespace internal
 }  // namespace tensorflow
 
 #endif  // TENSORFLOW_CORE_KERNELS_TRANSPOSE_FUNCTOR_H_
diff --git a/tensorflow/core/kernels/transpose_functor_cpu.cc b/tensorflow/core/kernels/transpose_functor_cpu.cc
index c3f3df722f..b983bf695c 100644
--- a/tensorflow/core/kernels/transpose_functor_cpu.cc
+++ b/tensorflow/core/kernels/transpose_functor_cpu.cc
@@ -19,10 +19,12 @@ limitations under the License.
 #include "tensorflow/core/kernels/ops_util.h"
 #include "tensorflow/core/kernels/transpose_functor.h"
 
+typedef Eigen::ThreadPoolDevice CPUDevice;
+
 namespace tensorflow {
 namespace internal {
 
-template <typename Device, typename T>
+template <typename Device, typename T, bool conjugate>
 void TransposeSimple(const Device& d, const Tensor& in,
                      const gtl::ArraySlice<int32> perm, Tensor* out) {
   const int ndims = in.dims();
@@ -41,122 +43,90 @@ void TransposeSimple(const Device& d, const Tensor& in,
       i_idx += (t / out_strides[i]) * in_strides[perm[i]];
       t = t % out_strides[i];
     }
-    q[o_idx] = p[i_idx];
+    if (conjugate) {
+      q[o_idx] = Eigen::numext::conj(p[i_idx]);
+    } else {
+      q[o_idx] = p[i_idx];
+    }
   }
 }
 
 }  // end namespace internal
 
-typedef Eigen::ThreadPoolDevice CPUDevice;
-
-template <typename T>
-struct Transpose<CPUDevice, T> {
+template <typename T, bool conjugate>
+struct Transpose<CPUDevice, T, conjugate> {
   static void run(const CPUDevice& d, const Tensor& in,
                   const gtl::ArraySlice<int32> perm, Tensor* out) {
     switch (in.dims()) {
       case 2:
-        internal::TransposeUsingEigen<CPUDevice, T, 2>(d, in, perm, out);
+        internal::TransposeUsingEigen<CPUDevice, T, 2>(d, in, perm, conjugate,
+                                                       out);
         break;
       case 3:
-        internal::TransposeUsingEigen<CPUDevice, T, 3>(d, in, perm, out);
+        internal::TransposeUsingEigen<CPUDevice, T, 3>(d, in, perm, conjugate,
+                                                       out);
         break;
       case 4:
-        internal::TransposeUsingEigen<CPUDevice, T, 4>(d, in, perm, out);
+        internal::TransposeUsingEigen<CPUDevice, T, 4>(d, in, perm, conjugate,
+                                                       out);
         break;
       case 5:
-        internal::TransposeUsingEigen<CPUDevice, T, 5>(d, in, perm, out);
+        internal::TransposeUsingEigen<CPUDevice, T, 5>(d, in, perm, conjugate,
+                                                       out);
         break;
       default:
-        internal::TransposeSimple<CPUDevice, T>(d, in, perm, out);
+        internal::TransposeSimple<CPUDevice, T, conjugate>(d, in, perm, out);
         break;
     }
   }
 };
 
-// TODO(yangzihao): Merge this code with its GPU counterpart to reduce code
-// duplication.
 template <>
-Status DoTranspose<CPUDevice>(const CPUDevice& d, const Tensor& in,
-                              const gtl::ArraySlice<int32> perm, Tensor* out) {
-  typedef CPUDevice Device;
-  CHECK_GE(in.dims(), 2);
-  CHECK_EQ(in.dims(), out->dims());
-  CHECK_EQ(in.dims(), perm.size());
-  CHECK_EQ(in.dtype(), out->dtype());
-  switch (in.dtype()) {
-    case DT_BOOL:
-    case DT_INT8:
-    case DT_QINT8:
-    case DT_QUINT8:
-    case DT_UINT8:
-      Transpose<Device, uint8>::run(d, in, perm, out);
-      break;
-
-    case DT_BFLOAT16:
-    case DT_HALF:
-    case DT_INT16:
-    case DT_QINT16:
-    case DT_QUINT16:
-    case DT_UINT16:
-      Transpose<Device, uint16>::run(d, in, perm, out);
-      break;
-
-    case DT_FLOAT:
-    case DT_INT32:
-    case DT_QINT32:
-      Transpose<Device, uint32>::run(d, in, perm, out);
-      break;
-
-    case DT_COMPLEX64:
-    case DT_DOUBLE:
-    case DT_INT64:
-      Transpose<Device, uint64>::run(d, in, perm, out);
-      break;
-
-    case DT_COMPLEX128:
-      Transpose<Device, complex128>::run(d, in, perm, out);
-      break;
-
-    case DT_STRING:
-      Transpose<Device, string>::run(d, in, perm, out);
-      break;
+Status DoTranspose(const CPUDevice& device, const Tensor& in,
+                   const gtl::ArraySlice<int32> perm, Tensor* out) {
+  return internal::DoTransposeImpl<CPUDevice>::run(device, in, perm,
+                                                   false /* conjugate */, out);
+}
 
-    default:
-      return errors::Unimplemented("Unsupported dtype on CPU: ", in.dtype());
-  }
-  return Status::OK();
+template <>
+Status DoConjugateTranspose(const CPUDevice& device, const Tensor& in,
+                            const gtl::ArraySlice<int32> perm, Tensor* out) {
+  return internal::DoTransposeImpl<CPUDevice>::run(device, in, perm,
+                                                   true /* conjugate */, out);
 }
 
 #ifdef TENSORFLOW_USE_SYCL
 typedef Eigen::SyclDevice SYCLDevice;
 
+namespace internal {
 template <typename Device, typename T>
 void TransposeSYCL(const Device& d, const Tensor& in,
-               const gtl::ArraySlice<int32> perm, Tensor* out) {
+                   const gtl::ArraySlice<int32> perm, bool conjugate,
+                   Tensor* out) {
   switch (in.dims()) {
     case 1:
-      internal::TransposeUsingEigen<Device, T, 1>(d, in, perm, out);
+      TransposeUsingEigen<SYCLDevice, T, 1>(d, in, perm, conjugate, out);
       break;
     case 2:
-      internal::TransposeUsingEigen<Device, T, 2>(d, in, perm, out);
+      TransposeUsingEigen<SYCLDevice, T, 2>(d, in, perm, conjugate, out);
       break;
     case 3:
-      internal::TransposeUsingEigen<Device, T, 3>(d, in, perm, out);
+      TransposeUsingEigen<SYCLDevice, T, 3>(d, in, perm, conjugate, out);
       break;
     case 4:
-      internal::TransposeUsingEigen<Device, T, 4>(d, in, perm, out);
+      TransposeUsingEigen<SYCLDevice, T, 4>(d, in, perm, conjugate, out);
       break;
     case 5:
-      internal::TransposeUsingEigen<Device, T, 5>(d, in, perm, out);
+      TransposeUsingEigen<SYCLDevice, T, 5>(d, in, perm, conjugate, out);
       break;
     case 6:
-      internal::TransposeUsingEigen<Device, T, 6>(d, in, perm, out);
+      TransposeUsingEigen<SYCLDevice, T, 6>(d, in, perm, conjugate, out);
       break;
     case 7:
-      internal::TransposeUsingEigen<Device, T, 7>(d, in, perm, out);
+      TransposeUsingEigen<SYCLDevice, T, 7>(d, in, perm, conjugate, out);
       break;
     case 8:
-      internal::TransposeUsingEigen<Device, T, 8>(d, in, perm, out);
+      TransposeUsingEigen<SYCLDevice, T, 8>(d, in, perm, conjugate, out);
       break;
     default:
       LOG(FATAL) << "Unsupported TransposeUsingEigen for: " << in.dims();
@@ -164,87 +134,38 @@ void TransposeSYCL(const Device& d, const Tensor& in,
   }
 }
 
-template <typename T>
-struct Transpose<SYCLDevice, T> {
+}  // namespace internal
+
+template <typename T, bool conjugate>
+struct Transpose<SYCLDevice, T, conjugate> {
   static void run(const SYCLDevice& d, const Tensor& in,
                   const gtl::ArraySlice<int32> perm, Tensor* out) {
-    switch (in.dims()) {
-      case 1:
-        internal::TransposeUsingEigen<SYCLDevice, T, 1>(d, in, perm, out);
-        break;
-      case 2:
-        internal::TransposeUsingEigen<SYCLDevice, T, 2>(d, in, perm, out);
-        break;
-      case 3:
-        internal::TransposeUsingEigen<SYCLDevice, T, 3>(d, in, perm, out);
-        break;
-      case 4:
-        internal::TransposeUsingEigen<SYCLDevice, T, 4>(d, in, perm, out);
-        break;
-      case 5:
-        internal::TransposeUsingEigen<SYCLDevice, T, 5>(d, in, perm, out);
-        break;
-      case 6:
-        internal::TransposeUsingEigen<SYCLDevice, T, 6>(d, in, perm, out);
-        break;
-      case 7:
-        internal::TransposeUsingEigen<SYCLDevice, T, 7>(d, in, perm, out);
-        break;
-      case 8:
-        internal::TransposeUsingEigen<SYCLDevice, T, 8>(d, in, perm, out);
-        break;
-      default:
-        LOG(FATAL) << "Unsupported TransposeUsingEigen for: " << in.dims();
-        break;
-    }
+    internal::TransposeSycl(d, in, perm, conjugate, out);
   }
 };
 
-template <>
-Status DoTranspose<SYCLDevice>(const SYCLDevice& d, const Tensor& in,
-                           const gtl::ArraySlice<int32> perm, Tensor* out) {
-  CHECK_GE(in.dims(), 2);
-  CHECK_EQ(in.dims(), out->dims());
-  CHECK_EQ(in.dims(), perm.size());
-  CHECK_EQ(in.dtype(), out->dtype());
-  switch (in.dtype()) {
-    case DT_BOOL:
-    case DT_INT8:
-    case DT_QINT8:
-    case DT_QUINT8:
-    case DT_UINT8:
-      TransposeSYCL<SYCLDevice, uint8>(d, in, perm, out);
-      break;
-
-    case DT_BFLOAT16:
-    case DT_HALF:
-    case DT_INT16:
-    case DT_QINT16:
-    case DT_QUINT16:
-    case DT_UINT16:
-      TransposeSYCL<SYCLDevice, uint16>(d, in, perm, out);
-      break;
-    case DT_FLOAT:
-    case DT_INT32:
-    case DT_QINT32:
-      TransposeSYCL<SYCLDevice, uint32>(d, in, perm, out);
-      break;
-
-    case DT_COMPLEX64:
-    case DT_DOUBLE:
-    case DT_INT64:
-      TransposeSYCL<SYCLDevice, uint64>(d, in, perm, out);
-      break;
+template <bool conjugate>
+struct Transpose<SYCLDevice, string, conjugate> {
+  static void run(const SYCLDevice& d, const Tensor& in,
+                  const gtl::ArraySlice<int32> perm, Tensor* out) {
+    LOG(FATAL) << "DT_STRING not supported on SYCL device.";
+  }
+};
 
-    case DT_COMPLEX128:
-      TransposeSYCL<SYCLDevice, complex128>(d, in, perm, out);
-      break;
+template <>
+Status DoTranspose(const SYCLDevice& device, const Tensor& in,
+                   const gtl::ArraySlice<int32> perm, Tensor* out) {
+  return internal::DoTransposeImpl<SYCLDevice>::run(device, in, perm,
+                                                    false /* conjugate */, out);
+}
 
-    default:
-      return errors::Unimplemented("Unsupported dtype on SYCL: ", in.dtype());
-  }
-  return Status::OK();
+template <>
+Status DoConjugateTranspose(const SYCLDevice& device, const Tensor& in,
+                            const gtl::ArraySlice<int32> perm, Tensor* out) {
+  return internal::DoTransposeImpl<SYCLDevice>::run(device, in, perm,
+                                                    true /* conjugate */, out);
 }
+
 #endif // TENSORFLOW_USE_SYCL
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/kernels/transpose_functor_gpu.cu.cc b/tensorflow/core/kernels/transpose_functor_gpu.cu.cc
index a118cc80c9..87af1ba0c4 100644
--- a/tensorflow/core/kernels/transpose_functor_gpu.cu.cc
+++ b/tensorflow/core/kernels/transpose_functor_gpu.cu.cc
@@ -18,18 +18,21 @@ limitations under the License.
 #define EIGEN_USE_GPU
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
-#include "tensorflow/core/kernels/transpose_functor.h"
+#include "tensorflow/core/kernels/cwise_ops.h"
 #include "tensorflow/core/kernels/ops_util.h"
+#include "tensorflow/core/kernels/transpose_functor.h"
 #include "tensorflow/core/util/cuda_kernel_helper.h"
 
 // TODO(yangzihao): Remove the dependency of conv_2d.h once we move all
 // GPU util functions and transpose kernels into separate files.
 #include "tensorflow/core/kernels/conv_2d.h"
 
+typedef Eigen::GpuDevice GPUDevice;
+
 namespace tensorflow {
 namespace internal {
 
-template <typename T>
+template <typename T, bool conjugate>
 __global__ void TransposeKernel(int nthreads, const T* src, const int32* buf,
                                 const int32 ndims, T* dst) {
   const int32* in_strides = buf;
@@ -42,11 +45,15 @@ __global__ void TransposeKernel(int nthreads, const T* src, const int32* buf,
       i_idx += (t / out_strides[i]) * in_strides[perm[i]];
       t = t % out_strides[i];
     }
-    dst[o_idx] = ldg(src + i_idx);
+    if (conjugate) {
+      dst[o_idx] = Eigen::numext::conj(ldg(src + i_idx));
+    } else {
+      dst[o_idx] = ldg(src + i_idx);
+    }
   }
 }
 
-template <typename Device, typename T>
+template <typename Device, typename T, bool conjugate>
 void TransposeSimple(const Device& d, const Tensor& in,
                      const gtl::ArraySlice<int32> perm, Tensor* out) {
   // Ensures we can use 32-bit index.
@@ -73,9 +80,10 @@ void TransposeSimple(const Device& d, const Tensor& in,
   const T* p = reinterpret_cast<const T*>(in.tensor_data().data());
   T* q = reinterpret_cast<T*>(const_cast<char*>((out->tensor_data().data())));
   CudaLaunchConfig cfg = GetCudaLaunchConfig(nelem, d);
-  TransposeKernel<<<cfg.block_count, cfg.thread_per_block, 0, d.stream()>>>(
-      cfg.virtual_thread_count, p, reinterpret_cast<const int32*>(dev_buf),
-      ndims, q);
+  TransposeKernel<T, conjugate>
+      <<<cfg.block_count, cfg.thread_per_block, 0, d.stream()>>>(
+          cfg.virtual_thread_count, p, reinterpret_cast<const int32*>(dev_buf),
+          ndims, q);
   // Safe to deallocate immediately after the kernel launch.
   d.deallocate(dev_buf);
 }
@@ -84,133 +92,152 @@ void TransposeSimple(const Device& d, const Tensor& in,
 // then call special kernels to swap either dimension 1 and dimension 2 or
 // dimension 0 and dimension 2. It returns true if the operation is success,
 // false otherwise.
-template <typename T>
-bool TransposeUsingTile(const Eigen::GpuDevice& d, const Tensor& in,
-                        const gtl::ArraySlice<int32> perm, Tensor* out) {
-  // First try to reduce the dimensions of the input tensor.
-  TransposePermsVec new_perm;
-  TransposeDimsVec new_dims;
-  ReduceTransposeDimensions(in.shape(), perm, &new_perm, &new_dims);
-
-  // Only use special GPU kernel when dimension is 2 or 3.
-  int dims = new_dims.size();
-  if (dims < 2 || dims > 3) return false;
-  auto in_data = reinterpret_cast<const T*>(in.tensor_data().data());
-  auto out_data =
-      reinterpret_cast<T*>(const_cast<char*>(out->tensor_data().data()));
-  switch (dims) {
-    case 2:
-      if (new_perm[0] == 1 && new_perm[1] == 0) {
-        // Add the first dimension size as 1.
-        new_dims.insert(new_dims.begin(), 1);
-        tensorflow::functor::SwapDimension1And2InTensor3<Eigen::GpuDevice, T>()(
-            d, in_data, new_dims, out_data);
-        return true;
-      }
-      break;
-    case 3:
-      if (new_perm == TransposePermsVec({0, 2, 1})) {
-        tensorflow::functor::SwapDimension1And2InTensor3<Eigen::GpuDevice, T>()(
-            d, in_data, new_dims, out_data);
-        return true;
-      } else if (new_perm == TransposePermsVec({2, 1, 0})) {
-        tensorflow::functor::SwapDimension0And2InTensor3<Eigen::GpuDevice, T>()(
-            d, in_data, new_dims, out_data);
-        return true;
-      } else {
-        // do not handle other 3D permutations
+template <typename T, bool conjugate = false>
+struct TransposeUsingTile {
+  static bool run(const Eigen::GpuDevice& d, const Tensor& in,
+                  const gtl::ArraySlice<int32> perm, Tensor* out) {
+    // First try to reduce the dimensions of the input tensor.
+    TransposePermsVec new_perm;
+    TransposeDimsVec new_dims;
+    ReduceTransposeDimensions(in.shape(), perm, &new_perm, &new_dims);
+
+    // Only use special GPU kernel when dimension is 2 or 3.
+    int dims = new_dims.size();
+    if (dims < 2 || dims > 3) return false;
+    auto in_data = reinterpret_cast<const T*>(in.tensor_data().data());
+    auto out_data =
+        reinterpret_cast<T*>(const_cast<char*>(out->tensor_data().data()));
+    switch (dims) {
+      case 2:
+        if (new_perm[0] == 1 && new_perm[1] == 0) {
+          // Add the first dimension size as 1.
+          new_dims.insert(new_dims.begin(), 1);
+          tensorflow::functor::SwapDimension1And2InTensor3<GPUDevice, T>()(
+              d, in_data, new_dims, out_data);
+          return true;
+        }
+        break;
+      case 3:
+        if (new_perm == TransposePermsVec({0, 2, 1})) {
+          tensorflow::functor::SwapDimension1And2InTensor3<GPUDevice, T>()(
+              d, in_data, new_dims, out_data);
+          return true;
+        } else if (new_perm == TransposePermsVec({2, 1, 0})) {
+          tensorflow::functor::SwapDimension0And2InTensor3<GPUDevice, T>()(
+              d, in_data, new_dims, out_data);
+          return true;
+        } else {
+          // do not handle other 3D permutations
+          return false;
+        }
+        break;
+      default:
         return false;
-      }
-      break;
-    default:
+    }
+    return false;
+  }
+};
+
+template <bool conjugate>
+struct TransposeUsingTile<complex64, conjugate> {
+  static bool run(const Eigen::GpuDevice& d, const Tensor& in,
+                  const gtl::ArraySlice<int32> perm, Tensor* out) {
+    if (!TransposeUsingTile<uint64>::run(d, in, perm, out)) {
       return false;
+    }
+    if (conjugate) {
+      // TODO(rmlarsen): Get rid of this call and conjugate on-the-fly in the
+      // transposition kernels so we only touch the memory once.
+      functor::UnaryFunctor<GPUDevice, functor::conj<complex64>> conj;
+      conj(d, out->flat<complex64>() /*out*/,
+           const_cast<const Tensor*>(out)->flat<complex64>() /*in*/);
+    }
+    return true;
   }
-  return false;
-}
+};
+
+template <bool conjugate>
+struct TransposeUsingTile<complex128, conjugate> {
+  static bool run(const Eigen::GpuDevice& d, const Tensor& in,
+                  const gtl::ArraySlice<int32> perm, Tensor* out) {
+    if (!TransposeUsingTile<float4>::run(d, in, perm, out)) {
+      return false;
+    }
+    if (conjugate) {
+      // TODO(rmlarsen): Get rid of this call and conjugate on-the-fly in the
+      // transposition kernels so we only touch the memory once.
+      functor::UnaryFunctor<GPUDevice, functor::conj<complex128>> conj;
+      conj(d, out->flat<complex128>() /*out*/,
+           const_cast<const Tensor*>(out)->flat<complex128>() /*in*/);
+    }
+    return true;
+  }
+};
 
 }  // end namespace internal
 
-typedef Eigen::GpuDevice GPUDevice;
+template <>
+Status DoTranspose(const GPUDevice& device, const Tensor& in,
+                   const gtl::ArraySlice<int32> perm, Tensor* out) {
+  return internal::DoTransposeImpl<GPUDevice>::run(device, in, perm,
+                                                   false /* conjugate */, out);
+}
+
+template <>
+Status DoConjugateTranspose(const GPUDevice& device, const Tensor& in,
+                            const gtl::ArraySlice<int32> perm, Tensor* out) {
+  return internal::DoTransposeImpl<GPUDevice>::run(device, in, perm,
+                                                   true /* conjugate */, out);
+}
 
 // Transpose kernel specialized for CPU Device.
-template <typename T>
-struct Transpose<GPUDevice, T> {
+template <typename T, bool conjugate>
+struct Transpose<GPUDevice, T, conjugate> {
   static void run(const GPUDevice& d, const Tensor& in,
                   const gtl::ArraySlice<int32> perm, Tensor* out) {
     switch (in.dims()) {
       case 2:
-        if (!internal::TransposeUsingTile<T>(d, in, perm, out)) {
-          internal::TransposeUsingEigen<GPUDevice, T, 2>(d, in, perm, out);
+        if (!internal::TransposeUsingTile<T, conjugate>::run(d, in, perm,
+                                                             out)) {
+          internal::TransposeUsingEigen<GPUDevice, T, 2>(d, in, perm, conjugate,
+                                                         out);
         }
         break;
       case 3:
-        if (!internal::TransposeUsingTile<T>(d, in, perm, out)) {
-          internal::TransposeUsingEigen<GPUDevice, T, 3>(d, in, perm, out);
+        if (!internal::TransposeUsingTile<T, conjugate>::run(d, in, perm,
+                                                             out)) {
+          internal::TransposeUsingEigen<GPUDevice, T, 3>(d, in, perm, conjugate,
+                                                         out);
         }
         break;
       case 4:
-        if (!internal::TransposeUsingTile<T>(d, in, perm, out)) {
-          internal::TransposeUsingEigen<GPUDevice, T, 4>(d, in, perm, out);
+        if (!internal::TransposeUsingTile<T, conjugate>::run(d, in, perm,
+                                                             out)) {
+          internal::TransposeUsingEigen<GPUDevice, T, 4>(d, in, perm, conjugate,
+                                                         out);
         }
         break;
       case 5:
-        if (!internal::TransposeUsingTile<T>(d, in, perm, out)) {
-          internal::TransposeUsingEigen<GPUDevice, T, 5>(d, in, perm, out);
+        if (!internal::TransposeUsingTile<T, conjugate>::run(d, in, perm,
+                                                             out)) {
+          internal::TransposeUsingEigen<GPUDevice, T, 5>(d, in, perm, conjugate,
+                                                         out);
         }
         break;
       default:
-        internal::TransposeSimple<GPUDevice, T>(d, in, perm, out);
+        internal::TransposeSimple<GPUDevice, T, conjugate>(d, in, perm, out);
         break;
     }
   }
 };
 
 template <>
-Status DoTranspose<GPUDevice>(const GPUDevice& d, const Tensor& in,
-                              const gtl::ArraySlice<int32> perm, Tensor* out) {
-  CHECK_GE(in.dims(), 2);
-  CHECK_EQ(in.dims(), out->dims());
-  CHECK_EQ(in.dims(), perm.size());
-  CHECK_EQ(in.dtype(), out->dtype());
-  switch (in.dtype()) {
-    case DT_BOOL:
-    case DT_INT8:
-    case DT_QINT8:
-    case DT_QUINT8:
-    case DT_UINT8:
-      Transpose<GPUDevice, uint8>::run(d, in, perm, out);
-      break;
-
-    case DT_BFLOAT16:
-    case DT_HALF:
-    case DT_INT16:
-    case DT_QINT16:
-    case DT_QUINT16:
-    case DT_UINT16:
-      Transpose<GPUDevice, uint16>::run(d, in, perm, out);
-      break;
-
-    case DT_FLOAT:
-    case DT_INT32:
-    case DT_QINT32:
-      Transpose<GPUDevice, uint32>::run(d, in, perm, out);
-      break;
-
-    case DT_COMPLEX64:
-    case DT_DOUBLE:
-    case DT_INT64:
-      Transpose<GPUDevice, uint64>::run(d, in, perm, out);
-      break;
-
-    case DT_COMPLEX128:
-      Transpose<GPUDevice, float4>::run(d, in, perm, out);
-      break;
-
-    default:
-      return errors::Unimplemented("Unsupported dtype on GPU: ", in.dtype());
+struct Transpose<GPUDevice, string> {
+  static void run(const GPUDevice& d, const Tensor& in,
+                  const gtl::ArraySlice<int32> perm, Tensor* out) {
+    LOG(FATAL) << "Transpose of DT_STRING tensor not supported on GPU.";
   }
-  return Status::OK();
-}
+};
 
 }  // namespace tensorflow
 #endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/transpose_op.cc b/tensorflow/core/kernels/transpose_op.cc
index d3305fb83a..e151b38d90 100644
--- a/tensorflow/core/kernels/transpose_op.cc
+++ b/tensorflow/core/kernels/transpose_op.cc
@@ -142,17 +142,18 @@ void TransposeOp::Compute(OpKernelContext* ctx) {
     }
   }
   for (int i = 0; i < dims; ++i) {
-    OP_REQUIRES(ctx, bits[i], errors::InvalidArgument(
-                                  i, " is missing from {",
-                                  str_util::Join(permutation, ","), "}."));
+    OP_REQUIRES(
+        ctx, bits[i],
+        errors::InvalidArgument(i, " is missing from {",
+                                str_util::Join(permutation, ","), "}."));
   }
 
   // 0-D, 1-D, and identity transposes do nothing.
-  if (dims <= 1 || is_identity) {
+  if (!IsConjugate() && (dims <= 1 || is_identity)) {
     ctx->set_output(0, input);
     return;
-  } else if (internal::NonSingletonDimensionsAlign(input.shape(),
-                                                   permutation)) {
+  } else if (!IsConjugate() && internal::NonSingletonDimensionsAlign(
+                                   input.shape(), permutation)) {
     Tensor output;
     OP_REQUIRES(ctx, output.CopyFrom(input, shape),
                 errors::Unknown("Error reshaping Tensor."));
@@ -174,6 +175,15 @@ Status TransposeCpuOp::DoTranspose(OpKernelContext* ctx, const Tensor& in,
                                    out);
 }
 
+Status ConjugateTransposeCpuOp::DoTranspose(OpKernelContext* ctx,
+                                            const Tensor& in,
+                                            gtl::ArraySlice<int32> perm,
+                                            Tensor* out) {
+  typedef Eigen::ThreadPoolDevice CPUDevice;
+  return ::tensorflow::DoConjugateTranspose(ctx->eigen_device<CPUDevice>(), in,
+                                            perm, out);
+}
+
 #ifdef INTEL_MKL
 #define REGISTER(T)                                           \
   REGISTER_KERNEL_BUILDER(Name("Transpose")                   \
@@ -181,7 +191,13 @@ Status TransposeCpuOp::DoTranspose(OpKernelContext* ctx, const Tensor& in,
                               .TypeConstraint<T>("T")         \
                               .TypeConstraint<int32>("Tperm") \
                               .HostMemory("perm"),            \
-                          MklTransposeCpuOp);
+                          MklTransposeCpuOp);                 \
+  REGISTER_KERNEL_BUILDER(Name("ConjugateTranspose")          \
+                              .Device(DEVICE_CPU)             \
+                              .TypeConstraint<T>("T")         \
+                              .TypeConstraint<int32>("Tperm") \
+                              .HostMemory("perm"),            \
+                          MklConjugateTransposeCpuOp);
 TF_CALL_ALL_TYPES(REGISTER);
 REGISTER(bfloat16);
 #undef REGISTER
@@ -194,7 +210,13 @@ REGISTER(bfloat16);
                               .TypeConstraint<T>("T")         \
                               .TypeConstraint<int32>("Tperm") \
                               .HostMemory("perm"),            \
-                          TransposeCpuOp);
+                          TransposeCpuOp);                    \
+  REGISTER_KERNEL_BUILDER(Name("ConjugateTranspose")          \
+                              .Device(DEVICE_CPU)             \
+                              .TypeConstraint<T>("T")         \
+                              .TypeConstraint<int32>("Tperm") \
+                              .HostMemory("perm"),            \
+                          ConjugateTransposeCpuOp);
 TF_CALL_ALL_TYPES(REGISTER)
 REGISTER(bfloat16);
 #undef REGISTER
@@ -207,6 +229,14 @@ Status TransposeGpuOp::DoTranspose(OpKernelContext* ctx, const Tensor& in,
   return ::tensorflow::DoTranspose(ctx->eigen_device<GPUDevice>(), in, perm,
                                    out);
 }
+Status ConjugateTransposeGpuOp::DoTranspose(OpKernelContext* ctx,
+                                            const Tensor& in,
+                                            gtl::ArraySlice<int32> perm,
+                                            Tensor* out) {
+  typedef Eigen::GpuDevice GPUDevice;
+  return ::tensorflow::DoConjugateTranspose(ctx->eigen_device<GPUDevice>(), in,
+                                            perm, out);
+}
 
 #define REGISTER(T)                                           \
   REGISTER_KERNEL_BUILDER(Name("Transpose")                   \
@@ -214,25 +244,45 @@ Status TransposeGpuOp::DoTranspose(OpKernelContext* ctx, const Tensor& in,
                               .TypeConstraint<T>("T")         \
                               .TypeConstraint<int32>("Tperm") \
                               .HostMemory("perm"),            \
-                          TransposeGpuOp);
+                          TransposeGpuOp);                    \
+  REGISTER_KERNEL_BUILDER(Name("ConjugateTranspose")          \
+                              .Device(DEVICE_GPU)             \
+                              .TypeConstraint<T>("T")         \
+                              .TypeConstraint<int32>("Tperm") \
+                              .HostMemory("perm"),            \
+                          ConjugateTransposeGpuOp);
 TF_CALL_POD_TYPES(REGISTER);
 #undef REGISTER
 #endif
 
 #ifdef TENSORFLOW_USE_SYCL
 Status TransposeSyclOp::DoTranspose(OpKernelContext* ctx, const Tensor& in,
-                                   gtl::ArraySlice<int32> perm, Tensor* out) {
+                                    gtl::ArraySlice<int32> perm, Tensor* out) {
   typedef Eigen::SyclDevice SYCLDevice;
   return ::tensorflow::DoTranspose(ctx->eigen_device<SYCLDevice>(), in, perm,
                                    out);
 }
+Status ConjugateTransposeSyclOp::DoTranspose(OpKernelContext* ctx,
+                                             const Tensor& in,
+                                             gtl::ArraySlice<int32> perm,
+                                             Tensor* out) {
+  typedef Eigen::SyclDevice SYCLDevice;
+  return ::tensorflow::DoConjugateTranspose(ctx->eigen_device<SYCLDevice>(), in,
+                                            perm, out);
+}
 #define REGISTER(T)                                           \
   REGISTER_KERNEL_BUILDER(Name("Transpose")                   \
                               .Device(DEVICE_SYCL)            \
                               .TypeConstraint<T>("T")         \
                               .TypeConstraint<int32>("Tperm") \
                               .HostMemory("perm"),            \
-                          TransposeSyclOp);
+                          TransposeSyclOp);                   \
+  REGISTER_KERNEL_BUILDER(Name("ConjugateTranspose")          \
+                              .Device(DEVICE_SYCL)            \
+                              .TypeConstraint<T>("T")         \
+                              .TypeConstraint<int32>("Tperm") \
+                              .HostMemory("perm"),            \
+                          ConjugateTransposeSyclOp);
 TF_CALL_POD_TYPES(REGISTER);
 #undef REGISTER
 #endif
diff --git a/tensorflow/core/kernels/transpose_op.h b/tensorflow/core/kernels/transpose_op.h
index a69eecc2f8..ff9cf5d4ff 100644
--- a/tensorflow/core/kernels/transpose_op.h
+++ b/tensorflow/core/kernels/transpose_op.h
@@ -30,6 +30,7 @@ class TransposeOp : public OpKernel {
  protected:
   virtual Status DoTranspose(OpKernelContext* ctx, const Tensor& in,
                              gtl::ArraySlice<int32> perm, Tensor* out) = 0;
+  virtual bool IsConjugate() const { return false; }
 };
 
 class TransposeCpuOp : public TransposeOp {
@@ -70,7 +71,57 @@ class TransposeSyclOp : public TransposeOp {
   Status DoTranspose(OpKernelContext* ctx, const Tensor& in,
                      gtl::ArraySlice<int32> perm, Tensor* out) override;
 };
-#endif // TENSORFLOW_USE_SYCL
+#endif  // TENSORFLOW_USE_SYCL
+
+// Conjugating transpose ops.
+class ConjugateTransposeCpuOp : public TransposeOp {
+ public:
+  explicit ConjugateTransposeCpuOp(OpKernelConstruction* ctx)
+      : TransposeOp(ctx) {}
+
+ protected:
+  Status DoTranspose(OpKernelContext* ctx, const Tensor& in,
+                     gtl::ArraySlice<int32> perm, Tensor* out) override;
+  bool IsConjugate() const override { return true; }
+};
+
+#ifdef INTEL_MKL
+template <bool conjugate = false>
+class MklConjugateTransposeCpuOp : public TransposeOp {
+ public:
+  explicit MklConjugateTransposeCpuOp(OpKernelConstruction* ctx)
+      : TransposeOp(ctx) {}
+
+ protected:
+  Status DoTranspose(OpKernelContext* ctx, const Tensor& in,
+                     gtl::ArraySlice<int32> perm, Tensor* out) override;
+  bool IsConjugate() const override { return true; }
+};
+#endif  // INTEL_MKL
+
+class ConjugateTransposeGpuOp : public TransposeOp {
+ public:
+  explicit ConjugateTransposeGpuOp(OpKernelConstruction* ctx)
+      : TransposeOp(ctx) {}
+
+ protected:
+  Status DoTranspose(OpKernelContext* ctx, const Tensor& in,
+                     gtl::ArraySlice<int32> perm, Tensor* out) override;
+  bool IsConjugate() const override { return true; }
+};
+
+#ifdef TENSORFLOW_USE_SYCL
+class ConjugateTransposeSyclOp : public TransposeOp {
+ public:
+  explicit ConjugateTransposeSyclOp(OpKernelConstruction* ctx)
+      : TransposeOp(ctx) {}
+
+ protected:
+  Status DoTranspose(OpKernelContext* ctx, const Tensor& in,
+                     gtl::ArraySlice<int32> perm, Tensor* out) override;
+  bool IsConjugate() const override { return true; }
+};
+#endif  // TENSORFLOW_USE_SYCL
 
 }  // namespace tensorflow
 
diff --git a/tensorflow/core/ops/array_ops.cc b/tensorflow/core/ops/array_ops.cc
index 108c29ed6e..25a7c9eb39 100644
--- a/tensorflow/core/ops/array_ops.cc
+++ b/tensorflow/core/ops/array_ops.cc
@@ -110,6 +110,64 @@ Status PadShapeFn(InferenceContext* c) {
   }
 }
 
+Status TransposeShapeFn(InferenceContext* c) {
+  ShapeHandle input = c->input(0);
+  ShapeHandle perm_shape = c->input(1);
+  const Tensor* perm = c->input_tensor(1);
+  DimensionHandle perm_elems = c->NumElements(perm_shape);
+  // If we don't have rank information on the input or value information on
+  // perm we can't return any shape information, otherwise we have enough
+  // information to at least find the rank of the output.
+  if (!c->RankKnown(input) && !c->ValueKnown(perm_elems) && perm == nullptr) {
+    c->set_output(0, c->UnknownShape());
+    return Status::OK();
+  }
+
+  // Find our value of the rank.
+  int64 rank;
+  if (c->RankKnown(input)) {
+    rank = c->Rank(input);
+  } else if (c->ValueKnown(perm_elems)) {
+    rank = c->Value(perm_elems);
+  } else {
+    rank = perm->NumElements();
+  }
+  std::vector<DimensionHandle> dims;
+  dims.resize(rank);
+  TF_RETURN_IF_ERROR(c->WithRank(input, rank, &input));
+  // Ensure that perm is a vector and has rank elements.
+  TF_RETURN_IF_ERROR(c->WithRank(perm_shape, 1, &perm_shape));
+  TF_RETURN_IF_ERROR(c->WithValue(perm_elems, rank, &perm_elems));
+
+  // If we know the rank of the input and the value of perm, we can return
+  // all shape informantion, otherwise we can only return rank information,
+  // but no information for the dimensions.
+  if (perm != nullptr) {
+    std::vector<int64> data;
+    if (perm->dtype() == DT_INT32) {
+      data = AsInt64<int32>(perm, rank);
+    } else {
+      data = AsInt64<int64>(perm, rank);
+    }
+
+    for (int32 i = 0; i < rank; ++i) {
+      int64 in_idx = data[i];
+      if (in_idx >= rank) {
+        return errors::InvalidArgument("perm dim ", in_idx,
+                                       " is out of range of input rank ", rank);
+      }
+      dims[i] = c->Dim(input, in_idx);
+    }
+  } else {
+    for (int i = 0; i < rank; ++i) {
+      dims[i] = c->UnknownDim();
+    }
+  }
+
+  c->set_output(0, c->MakeShape(dims));
+  return Status::OK();
+}
+
 Status SetOutputShapeForReshape(InferenceContext* c) {
   ShapeHandle in = c->input(0);
   ShapeHandle out;
@@ -1913,69 +1971,28 @@ REGISTER_OP("Transpose")
     .Output("y: T")
     .Attr("T: type")
     .Attr("Tperm: {int32, int64} = DT_INT32")
-    .SetShapeFn([](InferenceContext* c) {
-      ShapeHandle input = c->input(0);
-      ShapeHandle perm_shape = c->input(1);
-      const Tensor* perm = c->input_tensor(1);
-      DimensionHandle perm_elems = c->NumElements(perm_shape);
-      // If we don't have rank information on the input or value information on
-      // perm we can't return any shape information, otherwise we have enough
-      // information to at least find the rank of the output.
-      if (!c->RankKnown(input) && !c->ValueKnown(perm_elems) &&
-          perm == nullptr) {
-        c->set_output(0, c->UnknownShape());
-        return Status::OK();
-      }
-
-      // Find our value of the rank.
-      int64 rank;
-      if (c->RankKnown(input)) {
-        rank = c->Rank(input);
-      } else if (c->ValueKnown(perm_elems)) {
-        rank = c->Value(perm_elems);
-      } else {
-        rank = perm->NumElements();
-      }
-      std::vector<DimensionHandle> dims;
-      dims.resize(rank);
-      TF_RETURN_IF_ERROR(c->WithRank(input, rank, &input));
-      // Ensure that perm is a vector and has rank elements.
-      TF_RETURN_IF_ERROR(c->WithRank(perm_shape, 1, &perm_shape));
-      TF_RETURN_IF_ERROR(c->WithValue(perm_elems, rank, &perm_elems));
-
-      // If we know the rank of the input and the value of perm, we can return
-      // all shape informantion, otherwise we can only return rank information,
-      // but no information for the dimensions.
-      if (perm != nullptr) {
-        std::vector<int64> data;
-        if (perm->dtype() == DT_INT32) {
-          data = AsInt64<int32>(perm, rank);
-        } else {
-          data = AsInt64<int64>(perm, rank);
-        }
+    .SetShapeFn(TransposeShapeFn)
+    .Doc(R"doc(
+Shuffle dimensions of x according to a permutation.
 
-        for (int32 i = 0; i < rank; ++i) {
-          int64 in_idx = data[i];
-          if (in_idx >= rank) {
-            return errors::InvalidArgument(
-                "perm dim ", in_idx, " is out of range of input rank ", rank);
-          }
-          dims[i] = c->Dim(input, in_idx);
-        }
-      } else {
-        for (int i = 0; i < rank; ++i) {
-          dims[i] = c->UnknownDim();
-        }
-      }
+The output `y` has the same rank as `x`. The shapes of `x` and `y` satisfy:
+  `y.shape[i] == x.shape[perm[i]] for i in [0, 1, ..., rank(x) - 1]`
+)doc");
 
-      c->set_output(0, c->MakeShape(dims));
-      return Status::OK();
-    })
+// --------------------------------------------------------------------------
+REGISTER_OP("ConjugateTranspose")
+    .Input("x: T")
+    .Input("perm: Tperm")
+    .Output("y: T")
+    .Attr("T: type")
+    .Attr("Tperm: {int32, int64} = DT_INT32")
+    .SetShapeFn(TransposeShapeFn)
     .Doc(R"doc(
-Shuffle dimensions of x according to a permutation.
+Shuffle dimensions of x according to a permutation and conjugate the result.
 
 The output `y` has the same rank as `x`. The shapes of `x` and `y` satisfy:
   `y.shape[i] == x.shape[perm[i]] for i in [0, 1, ..., rank(x) - 1]`
+  `y[i,j,k,...,s,t,u] == conj(x[perm[i], perm[j], perm[k],...,perm[s], perm[t], perm[u]])`
 )doc");
 
 // --------------------------------------------------------------------------
diff --git a/tensorflow/python/kernel_tests/array_ops_test.py b/tensorflow/python/kernel_tests/array_ops_test.py
index 1792886417..8f4c94f318 100644
--- a/tensorflow/python/kernel_tests/array_ops_test.py
+++ b/tensorflow/python/kernel_tests/array_ops_test.py
@@ -51,6 +51,15 @@ class BatchMatrixTransposeTest(test_util.TensorFlowTestCase):
       self.assertEqual((3, 2), transposed.get_shape())
       self.assertAllEqual(expected_transposed, transposed.eval())
 
+  def testConjugate(self):
+    m = [[1 + 1j, 2 + 2j, 3 + 3j], [4 + 4j, 5 + 5j, 6 + 6j]]
+    expected_transposed = [[1 - 1j, 4 - 4j], [2 - 2j, 5 - 5j], [3 - 3j, 6 - 6j]]
+    with self.test_session():
+      matrix = ops.convert_to_tensor(m)
+      transposed = array_ops.matrix_transpose(matrix, conjugate=True)
+      self.assertEqual((3, 2), transposed.get_shape())
+      self.assertAllEqual(expected_transposed, transposed.eval())
+
   def testBatchMatrix(self):
     matrix_0 = [[1, 2, 3], [4, 5, 6]]
     matrix_0_t = [[1, 4], [2, 5], [3, 6]]
diff --git a/tensorflow/python/kernel_tests/linalg_ops_test.py b/tensorflow/python/kernel_tests/linalg_ops_test.py
index 8bb583ce1b..2f28d37eff 100644
--- a/tensorflow/python/kernel_tests/linalg_ops_test.py
+++ b/tensorflow/python/kernel_tests/linalg_ops_test.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 import numpy as np
 
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import linalg_ops
 from tensorflow.python.ops import math_ops
@@ -120,6 +121,20 @@ class SlogdetTest(test.TestCase):
         self.assertAllClose(sign_np, sign_tf.eval(), atol=atol)
 
 
+class AdjointTest(test.TestCase):
+
+  def test_compare_to_numpy(self):
+    for dtype in np.float64, np.float64, np.complex64, np.complex128:
+      matrix_np = np.array([[1 + 1j, 2 + 2j, 3 + 3j], [4 + 4j, 5 + 5j,
+                                                       6 + 6j]]).astype(dtype)
+      expected_transposed = np.conj(matrix_np.T)
+      with self.test_session():
+        matrix = ops.convert_to_tensor(matrix_np)
+        transposed = linalg.adjoint(matrix)
+        self.assertEqual((3, 2), transposed.get_shape())
+        self.assertAllEqual(expected_transposed, transposed.eval())
+
+
 class EyeTest(test.TestCase):
   pass  # Will be filled in below
 
diff --git a/tensorflow/python/kernel_tests/transpose_op_test.py b/tensorflow/python/kernel_tests/transpose_op_test.py
index 9e1f83395b..3b352937c8 100644
--- a/tensorflow/python/kernel_tests/transpose_op_test.py
+++ b/tensorflow/python/kernel_tests/transpose_op_test.py
@@ -38,14 +38,16 @@ class TransposeTest(test.TestCase):
     ret = ret.transpose(perm)
     return ret
 
-  def _compareCpu(self, x, p):
+  def _compareCpu(self, x, p, conjugate=False):
     np_ans = self._np_transpose(x, p)
+    if conjugate:
+      np_ans = np.conj(np_ans)
     with self.test_session(use_gpu=False):
       inx = ops.convert_to_tensor(x)
-      y = array_ops.transpose(inx, p)
+      y = array_ops.transpose(inx, p, conjugate=conjugate)
       tf_ans = y.eval()
-      self.assertAllEqual(np_ans, tf_ans)
       self.assertShapeEqual(np_ans, y)
+      self.assertAllEqual(np_ans, tf_ans)
 
       jacob_t = None
       # Gradient check on CPU.
@@ -62,11 +64,13 @@ class TransposeTest(test.TestCase):
 
       return tf_ans, jacob_t
 
-  def _compareGpu(self, x, p):
+  def _compareGpu(self, x, p, conjugate=False):
     np_ans = self._np_transpose(x, p)
+    if conjugate:
+      np_ans = np.conj(np_ans)
     with self.test_session(use_gpu=True):
       inx = ops.convert_to_tensor(x)
-      y = array_ops.transpose(inx, p)
+      y = array_ops.transpose(inx, p, conjugate=conjugate)
       tf_ans = y.eval()
 
       self.assertAllEqual(np_ans, tf_ans)
@@ -92,10 +96,12 @@ class TransposeTest(test.TestCase):
     # generate all permutations of [0, 1, ... n-1] in random order.
     all_perm = np.random.permutation(
         [p for p in itertools.permutations(range(n))]).astype(np.int32)
-    for p in all_perm[:2]:
-      self._compareCpu(x, p)
-      if use_gpu:
-        self._compareGpu(x, p)
+    cs = [False, True] if x.dtype in [np.complex64, np.complex128] else [False]
+    for c in cs:
+      for p in all_perm[:2]:
+        self._compareCpu(x, p, conjugate=c)
+        if use_gpu:
+          self._compareGpu(x, p, conjugate=c)
 
   def _compare_cpu_gpu(self, x):
     n = np.ndim(x)
diff --git a/tensorflow/python/ops/array_ops.py b/tensorflow/python/ops/array_ops.py
index 61405e3f45..dc3aa735da 100644
--- a/tensorflow/python/ops/array_ops.py
+++ b/tensorflow/python/ops/array_ops.py
@@ -1283,13 +1283,15 @@ def split(value, num_or_size_splits, axis=0, num=None, name="split"):
         name=name)
 
 
-def transpose(a, perm=None, name="transpose"):
+def transpose(a, perm=None, name="transpose", conjugate=False):
   """Transposes `a`. Permutes the dimensions according to `perm`.
 
   The returned tensor's dimension i will correspond to the input dimension
   `perm[i]`. If `perm` is not given, it is set to (n-1...0), where n is
   the rank of the input tensor. Hence by default, this operation performs a
-  regular matrix transpose on 2-D input Tensors.
+  regular matrix transpose on 2-D input Tensors. If conjugate is True and
+  `a.dtype` is either `complex64` or `complex128` then the values of `a`
+  are conjugated and transposed.
 
   For example:
 
@@ -1304,6 +1306,13 @@ def transpose(a, perm=None, name="transpose"):
                                 #  [2, 5]
                                 #  [3, 6]]
 
+  # If x is complex, setting conjugate=True gives the conjugate transpose
+  x = tf.constant([[1 + 1j, 2 + 2j, 3 + 3j],
+                   [4 + 4j, 5 + 5j, 6 + 6j]])
+  tf.transpose(x, conjugate=True)  # [[1 - 1j, 4 - 4j],
+                                   #  [2 - 2j, 5 - 5j],
+                                   #  [3 - 3j, 6 - 6j]]
+
   # 'perm' is more useful for n-dimensional tensors, for n > 2
   x = tf.constant([[[ 1,  2,  3],
                     [ 4,  5,  6]],
@@ -1311,6 +1320,7 @@ def transpose(a, perm=None, name="transpose"):
                     [10, 11, 12]]])
 
   # Take the transpose of the matrices in dimension-0
+  # (this common operation has a shorthand `matrix_transpose`)
   tf.transpose(x, perm=[0, 2, 1])  # [[[1,  4],
                                    #   [2,  5],
                                    #   [3,  6]],
@@ -1323,15 +1333,20 @@ def transpose(a, perm=None, name="transpose"):
     a: A `Tensor`.
     perm: A permutation of the dimensions of `a`.
     name: A name for the operation (optional).
+    conjugate: Optional bool. Setting it to `True` is mathematically equivalent
+      to tf.conj(tf.transpose(input)).
 
   Returns:
     A transposed `Tensor`.
   """
   with ops.name_scope(name, "transpose", [a]) as name:
+    transpose_fn = (
+        gen_array_ops._conjugate_transpose
+        if conjugate else gen_array_ops.transpose)
     if perm is None:
       rank = gen_array_ops.rank(a)
       perm = (rank - 1) - gen_math_ops._range(0, rank, 1)
-      ret = gen_array_ops.transpose(a, perm, name=name)
+      ret = transpose_fn(a, perm, name=name)
       # NOTE(mrry): Setting the shape explicitly because
       #   reverse is not handled by the shape function.
       if context.in_graph_mode():
@@ -1339,12 +1354,12 @@ def transpose(a, perm=None, name="transpose"):
         if input_shape is not None:
           ret.set_shape(input_shape[::-1])
     else:
-      ret = gen_array_ops.transpose(a, perm, name=name)
+      ret = transpose_fn(a, perm, name=name)
     return ret
 
 
 # pylint: disable=invalid-name
-def matrix_transpose(a, name="matrix_transpose"):
+def matrix_transpose(a, name="matrix_transpose", conjugate=False):
   """Transposes last two dimensions of tensor `a`.
 
   For example:
@@ -1355,6 +1370,12 @@ def matrix_transpose(a, name="matrix_transpose"):
                           #  [2, 5],
                           #  [3, 6]]
 
+  x = tf.constant([[1 + 1j, 2 + 2j, 3 + 3j],
+                   [4 + 4j, 5 + 5j, 6 + 6j]])
+  tf.matrix_transpose(x, conjugate=True)  # [[1 - 1j, 4 - 4j],
+                                          #  [2 - 2j, 5 - 5j],
+                                          #  [3 - 3j, 6 - 6j]]
+
   # Matrix with two batch dimensions.
   # x.shape is [1, 2, 3, 4]
   # tf.matrix_transpose(x) is shape [1, 2, 4, 3]
@@ -1374,6 +1395,8 @@ def matrix_transpose(a, name="matrix_transpose"):
   Args:
     a: A `Tensor` with `rank >= 2`.
     name: A name for the operation (optional).
+    conjugate: Optional bool. Setting it to `True` is mathematically equivalent
+      to tf.conj(tf.matrix_transpose(input)).
 
   Returns:
     A transposed batch matrix `Tensor`.
@@ -1401,7 +1424,7 @@ def matrix_transpose(a, name="matrix_transpose"):
       perm = concat((gen_math_ops._range(0, a_rank - 2, 1),
                      [a_rank - 1, a_rank - 2]), 0)
 
-    return transpose(a, perm=perm)
+    return transpose(a, perm=perm, conjugate=conjugate)
 
 
 # pylint: enable=invalid-name
diff --git a/tensorflow/python/ops/hidden_ops.txt b/tensorflow/python/ops/hidden_ops.txt
index d27e867583..fcd378e3c0 100644
--- a/tensorflow/python/ops/hidden_ops.txt
+++ b/tensorflow/python/ops/hidden_ops.txt
@@ -5,6 +5,7 @@ BroadcastGradientArgs
 ConcatOffset
 Concat
 ConcatV2
+ConjugateTranspose
 Const
 DebugGradientIdentity
 EditDistance
diff --git a/tensorflow/python/ops/linalg/linalg_impl.py b/tensorflow/python/ops/linalg/linalg_impl.py
index ca57653d14..32d1b31d7d 100644
--- a/tensorflow/python/ops/linalg/linalg_impl.py
+++ b/tensorflow/python/ops/linalg/linalg_impl.py
@@ -54,3 +54,28 @@ def logdet(matrix, name=None):
     return 2.0 * math_ops.reduce_sum(
         math_ops.log(math_ops.real(array_ops.matrix_diag_part(chol))),
         reduction_indices=[-1])
+
+
+def adjoint(matrix, name=None):
+  """Conjugates and transposes the last two dimensions of tensor `matrix`.
+
+  For example:
+
+  ```python
+  x = tf.constant([[1 + 1j, 2 + 2j, 3 + 3j],
+                   [4 + 4j, 5 + 5j, 6 + 6j]])
+  tf.linalg.adjoint(x)  # [[1 - 1j, 4 - 4j],
+                        #  [2 - 2j, 5 - 5j],
+                        #  [3 - 3j, 6 - 6j]]
+
+  Args:
+    matrix:  A `Tensor`. Must be `float32`, `float64`, `complex64`, or
+      `complex128` with shape `[..., M, M]`.
+    name:  A name to give this `Op` (optional).
+
+  Returns:
+    The adjoint (a.k.a. Hermitian transpose a.k.a. conjugate transpose) of
+    matrix.
+  """
+  with ops.name_scope(name, 'adjoint', [matrix]):
+    return array_ops.matrix_transpose(matrix, conjugate=True)
diff --git a/tensorflow/tools/api/golden/tensorflow.linalg.pbtxt b/tensorflow/tools/api/golden/tensorflow.linalg.pbtxt
index 4c94863caa..0d62585ff4 100644
--- a/tensorflow/tools/api/golden/tensorflow.linalg.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.linalg.pbtxt
@@ -32,6 +32,10 @@ tf_module {
     name: "LinearOperatorScaledIdentity"
     mtype: "<class \'abc.ABCMeta\'>"
   }
+  member_method {
+    name: "adjoint"
+    argspec: "args=[\'matrix\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
   member_method {
     name: "band_part"
     argspec: "args=[\'input\', \'num_lower\', \'num_upper\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
@@ -118,7 +122,7 @@ tf_module {
   }
   member_method {
     name: "transpose"
-    argspec: "args=[\'a\', \'name\'], varargs=None, keywords=None, defaults=[\'matrix_transpose\'], "
+    argspec: "args=[\'a\', \'name\', \'conjugate\'], varargs=None, keywords=None, defaults=[\'matrix_transpose\', \'False\'], "
   }
   member_method {
     name: "triangular_solve"
diff --git a/tensorflow/tools/api/golden/tensorflow.pbtxt b/tensorflow/tools/api/golden/tensorflow.pbtxt
index d77f8fd253..d56a59de72 100644
--- a/tensorflow/tools/api/golden/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.pbtxt
@@ -1354,7 +1354,7 @@ tf_module {
   }
   member_method {
     name: "matrix_transpose"
-    argspec: "args=[\'a\', \'name\'], varargs=None, keywords=None, defaults=[\'matrix_transpose\'], "
+    argspec: "args=[\'a\', \'name\', \'conjugate\'], varargs=None, keywords=None, defaults=[\'matrix_transpose\', \'False\'], "
   }
   member_method {
     name: "matrix_triangular_solve"
@@ -1990,7 +1990,7 @@ tf_module {
   }
   member_method {
     name: "transpose"
-    argspec: "args=[\'a\', \'perm\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'transpose\'], "
+    argspec: "args=[\'a\', \'perm\', \'name\', \'conjugate\'], varargs=None, keywords=None, defaults=[\'None\', \'transpose\', \'False\'], "
   }
   member_method {
     name: "truediv"
-- 
GitLab


From dcd007b12c5516ff53615d7f34dab5841f6aa5c5 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 11 Oct 2017 16:55:38 -0700
Subject: [PATCH 277/909] Go: Update generated wrapper functions for TensorFlow
 ops.

PiperOrigin-RevId: 171895671
---
 tensorflow/go/op/wrappers.go | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go
index cf842f3808..0da7d5e199 100644
--- a/tensorflow/go/op/wrappers.go
+++ b/tensorflow/go/op/wrappers.go
@@ -1461,6 +1461,25 @@ func Slice(scope *Scope, input tf.Output, begin tf.Output, size tf.Output) (outp
 	return op.Output(0)
 }
 
+// Shuffle dimensions of x according to a permutation and conjugate the result.
+//
+// The output `y` has the same rank as `x`. The shapes of `x` and `y` satisfy:
+//   `y.shape[i] == x.shape[perm[i]] for i in [0, 1, ..., rank(x) - 1]`
+//   `y[i,j,k,...,s,t,u] == conj(x[perm[i], perm[j], perm[k],...,perm[s], perm[t], perm[u]])`
+func ConjugateTranspose(scope *Scope, x tf.Output, perm tf.Output) (y tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "ConjugateTranspose",
+		Input: []tf.Input{
+			x, perm,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Checks a tensor for NaN and Inf values.
 //
 // When run, reports an `InvalidArgument` error if `tensor` has any values
-- 
GitLab


From c69b9597995ab6510f9a21b615fe765b417d9cbb Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 11 Oct 2017 17:34:56 -0700
Subject: [PATCH 278/909] Update ops-related pbtxt files.

PiperOrigin-RevId: 171900256
---
 .../core/ops/compat/ops_history.v1.pbtxt      | 32 +++++++++++++++++
 tensorflow/core/ops/ops.pbtxt                 | 34 +++++++++++++++++++
 2 files changed, 66 insertions(+)

diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index 2097c587d5..cccd2d6f97 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -6685,6 +6685,38 @@ op {
     }
   }
 }
+op {
+  name: "ConjugateTranspose"
+  input_arg {
+    name: "x"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "perm"
+    type_attr: "Tperm"
+  }
+  output_arg {
+    name: "y"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+  }
+  attr {
+    name: "Tperm"
+    type: "type"
+    default_value {
+      type: DT_INT32
+    }
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+}
 op {
   name: "Const"
   output_arg {
diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index 7579aef259..cfe7504988 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -4880,6 +4880,40 @@ op {
   summary: "Returns the complex conjugate of a complex number."
   description: "Given a tensor `input` of complex numbers, this operation returns a tensor of\ncomplex numbers that are the complex conjugate of each element in `input`. The\ncomplex numbers in `input` must be of the form \\\\(a + bj\\\\), where *a* is the\nreal part and *b* is the imaginary part.\n\nThe complex conjugate returned by this operation is of the form \\\\(a - bj\\\\).\n\nFor example:\n\n```\n# tensor \'input\' is [-2.25 + 4.75j, 3.25 + 5.75j]\ntf.conj(input) ==> [-2.25 - 4.75j, 3.25 - 5.75j]\n```"
 }
+op {
+  name: "ConjugateTranspose"
+  input_arg {
+    name: "x"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "perm"
+    type_attr: "Tperm"
+  }
+  output_arg {
+    name: "y"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+  }
+  attr {
+    name: "Tperm"
+    type: "type"
+    default_value {
+      type: DT_INT32
+    }
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+  summary: "Shuffle dimensions of x according to a permutation and conjugate the result."
+  description: "The output `y` has the same rank as `x`. The shapes of `x` and `y` satisfy:\n  `y.shape[i] == x.shape[perm[i]] for i in [0, 1, ..., rank(x) - 1]`\n  `y[i,j,k,...,s,t,u] == conj(x[perm[i], perm[j], perm[k],...,perm[s], perm[t], perm[u]])`"
+}
 op {
   name: "Const"
   output_arg {
-- 
GitLab


From 9b26ed77dc2740314f47bcc4c991dd7f729b8d23 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 11 Oct 2017 18:15:32 -0700
Subject: [PATCH 279/909] Implement NCHW_VECT_C support for tf.depth_to_space
 on GPU.

PiperOrigin-RevId: 171904046
---
 tensorflow/core/kernels/depthtospace_op.cc    | 68 +++++++++------
 tensorflow/core/kernels/depthtospace_op.h     |  4 +
 .../core/kernels/depthtospace_op_gpu.cu.cc    | 11 +++
 tensorflow/core/ops/array_ops.cc              |  7 +-
 tensorflow/python/kernel_tests/BUILD          |  4 +-
 .../kernel_tests/depthtospace_op_test.py      | 84 +++++++++++--------
 6 files changed, 114 insertions(+), 64 deletions(-)

diff --git a/tensorflow/core/kernels/depthtospace_op.cc b/tensorflow/core/kernels/depthtospace_op.cc
index 4cf7de0df4..39aa3e9eb0 100644
--- a/tensorflow/core/kernels/depthtospace_op.cc
+++ b/tensorflow/core/kernels/depthtospace_op.cc
@@ -49,34 +49,33 @@ class DepthToSpaceOp : public OpKernel {
     OP_REQUIRES(context, FormatFromString(data_format_str, &data_format_),
                 errors::InvalidArgument("Invalid data format"));
 
+    OP_REQUIRES_OK(context, context->GetAttr("block_size", &block_size_));
+    OP_REQUIRES(context, block_size_ > 1,
+                errors::InvalidArgument("Block size should be > 1, but was: ",
+                                        block_size_));
+
     if (std::is_same<Device, CPUDevice>::value) {
       OP_REQUIRES(
           context, data_format_ == FORMAT_NHWC,
           errors::InvalidArgument(
               "Only NHWC data_format supported on CPU. Got ", data_format_str));
     }
-
-    // TODO(pauldonnelly): Implement NCHW_VECT_C kernel for the GPU.
-    OP_REQUIRES(
-        context, data_format_ != FORMAT_NCHW_VECT_C,
-        errors::InvalidArgument("NHWC_VECT_C kernel not yet implemented."));
-
-    OP_REQUIRES_OK(context, context->GetAttr("block_size", &block_size_));
-
-    OP_REQUIRES(
-        context, block_size_ > 1,
-        errors::InvalidArgument("Block size should be > 1: ", block_size_));
   }
 
   void Compute(OpKernelContext* context) override {
     const Tensor& input = context->input(0);
-
-    // Check on the input dimensions first.
-    // The input is presumed to be [batch, height, width, depth]
     const int dims = input.dims();
-    constexpr int kRequiredDims = 4;
-    OP_REQUIRES(context, kRequiredDims == dims,
-                errors::InvalidArgument("Input rank should be: ", kRequiredDims,
+
+    // Assuming qint8 <--> NCHW_VECT_C, OIHW_VECT_I (int8x4) here.
+    constexpr bool is_int8x4 = std::is_same<T, qint8>::value;
+    OP_REQUIRES(context, (is_int8x4 == (data_format_ == FORMAT_NCHW_VECT_C)),
+                errors::InvalidArgument(
+                    "qint8 should be used with data_format NCHW_VECT_C."));
+
+    constexpr int kVect = is_int8x4 ? 4 : 1;
+    constexpr int kDims = is_int8x4 ? 5 : 4;
+    OP_REQUIRES(context, kDims == dims,
+                errors::InvalidArgument("Input rank should be: ", kDims,
                                         " instead of: ", dims));
 
     constexpr int kNumSpatialDims = 2;
@@ -87,7 +86,8 @@ class DepthToSpaceOp : public OpKernel {
     const int input_width =
         input.dim_size(GetTensorDimIndex<kNumSpatialDims>(data_format_, 'W'));
     const int input_depth =
-        input.dim_size(GetTensorDimIndex<kNumSpatialDims>(data_format_, 'C'));
+        input.dim_size(GetTensorDimIndex<kNumSpatialDims>(data_format_, 'C')) *
+        kVect;
 
     const int block_size_sq = block_size_ * block_size_;
 
@@ -109,13 +109,30 @@ class DepthToSpaceOp : public OpKernel {
                        ShapeFromFormat(data_format_, batch_size, output_height,
                                        output_width, output_depth),
                        &outputs_tensor));
-    auto Tinput = input.tensor<T, kRequiredDims>();
-    auto Toutput = outputs_tensor->tensor<T, kRequiredDims>();
+    auto Tinput = input.tensor<T, kDims>();
+    auto Toutput = outputs_tensor->tensor<T, kDims>();
+
+    if (std::is_same<Device, GPUDevice>::value) {
+      if (is_int8x4) {
+        // NCHW_VECT_C with 4 x qint8 can be treated as NCHW int32.
+        auto Tinput_v = input.template reinterpret_last_dimension<int32, 4>();
+        auto Toutput_v = outputs_tensor->reinterpret_last_dimension<int32, 4>();
+        functor::DepthToSpaceOpFunctor<GPUDevice, int32, FORMAT_NCHW> functor;
+        functor(context->eigen_device<GPUDevice>(), Tinput_v, block_size_,
+                Toutput_v);
+        return;
+      } else if (data_format_ == FORMAT_NCHW) {
+        functor::DepthToSpaceOpFunctor<GPUDevice, T, FORMAT_NCHW> functor;
+        functor(context->eigen_device<GPUDevice>(), Tinput, block_size_,
+                Toutput);
+        return;
+      }
+    }
 
-    if (std::is_same<Device, GPUDevice>::value && data_format_ == FORMAT_NCHW) {
-      functor::DepthToSpaceOpFunctor<Device, T, FORMAT_NCHW> functor;
-      functor(context->eigen_device<Device>(), Tinput, block_size_, Toutput);
-    } else {
+    // NOTE: Assumes data_format_ == FORMAT_NHWC here, since we have rejected
+    // (CPU && data_format_ != FORMAT_NHWC) in the constructor.
+
+    if (!is_int8x4) {
       functor::DepthToSpaceOpFunctor<Device, T, FORMAT_NHWC> functor;
       functor(context->eigen_device<Device>(), Tinput, block_size_, Toutput);
     }
@@ -170,6 +187,9 @@ TF_CALL_ALL_TYPES(REGISTER);
 REGISTER_KERNEL_BUILDER(
     Name("DepthToSpace").Device(DEVICE_GPU).TypeConstraint<float>("T"),
     DepthToSpaceOp<GPUDevice, float>);
+REGISTER_KERNEL_BUILDER(
+    Name("DepthToSpace").Device(DEVICE_GPU).TypeConstraint<qint8>("T"),
+    DepthToSpaceOp<GPUDevice, qint8>);
 #endif  // GOOGLE_CUDA
 
 }  // end namespace tensorflow
diff --git a/tensorflow/core/kernels/depthtospace_op.h b/tensorflow/core/kernels/depthtospace_op.h
index fca375f58b..272468b740 100644
--- a/tensorflow/core/kernels/depthtospace_op.h
+++ b/tensorflow/core/kernels/depthtospace_op.h
@@ -44,6 +44,10 @@ template <typename Device, typename T, TensorFormat data_format>
 struct DepthToSpaceOpFunctor {
   void operator()(const Device& d, typename TTypes<T, 4>::ConstTensor input,
                   int block_size, typename TTypes<T, 4>::Tensor output);
+
+  // This 5-D version is to support NCHW_VECT_C.
+  void operator()(const Device& d, typename TTypes<T, 5>::ConstTensor input,
+                  int block_size, typename TTypes<T, 5>::Tensor output);
 };
 
 }  // namespace functor
diff --git a/tensorflow/core/kernels/depthtospace_op_gpu.cu.cc b/tensorflow/core/kernels/depthtospace_op_gpu.cu.cc
index 8f07c809e6..357c1f1be4 100644
--- a/tensorflow/core/kernels/depthtospace_op_gpu.cu.cc
+++ b/tensorflow/core/kernels/depthtospace_op_gpu.cu.cc
@@ -124,6 +124,10 @@ struct DepthToSpaceOpFunctor<GPUDevice, T, FORMAT_NHWC> {
         input_height, input_width, input_depth, output_height, output_width,
         output_depth, output.data());
   }
+  void operator()(const GPUDevice& d, typename TTypes<T, 5>::ConstTensor input,
+                  int block_size, typename TTypes<T, 5>::Tensor output) {
+    LOG(FATAL) << "5-D tensors should not be used with NHWC format";
+  }
 };
 
 template <typename T>
@@ -143,6 +147,10 @@ struct DepthToSpaceOpFunctor<GPUDevice, T, FORMAT_NCHW> {
         config.virtual_thread_count, input.data(), block_size, input_width,
         output_depth * input_height, output.data());
   }
+  void operator()(const GPUDevice& d, typename TTypes<T, 5>::ConstTensor input,
+                  int block_size, typename TTypes<T, 5>::Tensor output) {
+    LOG(FATAL) << "5-D tensors should not be used with NCHW format";
+  }
 };
 }  // end namespace functor
 
@@ -150,6 +158,9 @@ struct DepthToSpaceOpFunctor<GPUDevice, T, FORMAT_NCHW> {
 template struct functor::DepthToSpaceOpFunctor<GPUDevice, float, FORMAT_NCHW>;
 template struct functor::DepthToSpaceOpFunctor<GPUDevice, float, FORMAT_NHWC>;
 
+// NCHW_VECT_C with 4 x qint8 can be treated as NCHW int32.
+template struct functor::DepthToSpaceOpFunctor<GPUDevice, int32, FORMAT_NCHW>;
+
 }  // end namespace tensorflow
 
 #endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/ops/array_ops.cc b/tensorflow/core/ops/array_ops.cc
index 25a7c9eb39..14b87f0edf 100644
--- a/tensorflow/core/ops/array_ops.cc
+++ b/tensorflow/core/ops/array_ops.cc
@@ -4244,13 +4244,16 @@ REGISTER_OP("DepthToSpace")
       TensorFormat data_format;
       FormatFromString(data_format_str, &data_format);
 
+      constexpr int num_spatial_dims = 2;
+      const int dims =
+          GetTensorDimsFromSpatialDims(num_spatial_dims, data_format);
+
       ShapeHandle input;
-      TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 4, &input));
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(0), dims, &input));
 
       int32 block_size;
       TF_RETURN_IF_ERROR(c->GetAttr("block_size", &block_size));
 
-      constexpr int num_spatial_dims = 2;
       DimensionHandle batch_size =
           c->Dim(input, GetTensorDimIndex<num_spatial_dims>(data_format, 'N'));
       DimensionHandle input_height =
diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD
index b8a7444f45..6beebbf48f 100644
--- a/tensorflow/python/kernel_tests/BUILD
+++ b/tensorflow/python/kernel_tests/BUILD
@@ -1330,7 +1330,7 @@ cuda_py_test(
 
 cuda_py_test(
     name = "depthtospace_op_test",
-    size = "small",
+    size = "medium",
     srcs = ["depthtospace_op_test.py"],
     additional_deps = [
         "//third_party/py/numpy",
@@ -1898,7 +1898,7 @@ cuda_py_test(
 
 cuda_py_test(
     name = "spacetodepth_op_test",
-    size = "small",
+    size = "medium",
     srcs = ["spacetodepth_op_test.py"],
     additional_deps = [
         "//third_party/py/numpy",
diff --git a/tensorflow/python/kernel_tests/depthtospace_op_test.py b/tensorflow/python/kernel_tests/depthtospace_op_test.py
index 6d5dc3846b..792806642a 100644
--- a/tensorflow/python/kernel_tests/depthtospace_op_test.py
+++ b/tensorflow/python/kernel_tests/depthtospace_op_test.py
@@ -26,9 +26,11 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import gen_array_ops
 from tensorflow.python.ops import gradient_checker
 from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import test
+from tensorflow.python.platform import tf_logging
 
 
 class DepthToSpaceTest(test.TestCase):
@@ -201,7 +203,8 @@ class DepthToSpaceTest(test.TestCase):
       _ = array_ops.space_to_depth(x_np, block_size)
 
   def testUnknownShape(self):
-    t = array_ops.depth_to_space(array_ops.placeholder(dtypes.float32), block_size=4)
+    t = array_ops.depth_to_space(
+        array_ops.placeholder(dtypes.float32), block_size=4)
     self.assertEqual(4, t.get_shape().ndims)
 
   def depthToSpaceUsingTranspose(self, tensor, block_size, data_format):
@@ -224,49 +227,58 @@ class DepthToSpaceTest(test.TestCase):
       tensor = array_ops.reshape(tensor, [b, oc, oh, ow])
     return tensor
 
-  def compareToTranspose(self, data_format, batch_size, in_height, in_width,
-                         out_channels, block_size, use_gpu):
-    if use_gpu and not test.is_gpu_available():
-      print("gpu not available")
-      return
-
-    dtype = dtypes.float32
+  def compareToTranspose(self, batch_size, in_height, in_width, out_channels,
+                         block_size, data_format, use_gpu):
     in_channels = out_channels * block_size * block_size
-
-    if data_format == "NHWC":
-      input_shape = [batch_size, in_height, in_width, in_channels]
-    elif data_format == "NCHW":
-      input_shape = [batch_size, in_channels, in_height, in_width]
+    nhwc_input_shape = [batch_size, in_height, in_width, in_channels]
+    nchw_input_shape = [batch_size, in_channels, in_height, in_width]
+    total_size = np.prod(nhwc_input_shape)
+
+    if data_format == "NCHW_VECT_C":
+      # Initialize the input tensor with qint8 values that circle -127..127.
+      x = [((f + 128) % 255) - 127 for f in range(total_size)]
+      t = constant_op.constant(x, shape=nhwc_input_shape, dtype=dtypes.float32)
+      expected = self.depthToSpaceUsingTranspose(t, block_size, "NHWC")
+      t = test_util.NHWCToNCHW_VECT_C(t)
+      t, _, _ = gen_array_ops.quantize_v2(t, -128.0, 127.0, dtypes.qint8)
+      t = array_ops.depth_to_space(t, block_size, data_format="NCHW_VECT_C")
+      t = gen_array_ops.dequantize(t, -128, 127)
+      actual = test_util.NCHW_VECT_CToNHWC(t)
     else:
-      assert False, "unsupported format"
-
-    # Initialize the input tensor with ascending whole numbers.
-    total_size = 1
-    for dim_size in input_shape:
-      total_size *= dim_size
-    x = [f for f in range(total_size)]
-    inputs = constant_op.constant(x, shape=input_shape, dtype=dtype)
-
-    expected = self.depthToSpaceUsingTranspose(inputs, block_size, data_format)
-    actual = array_ops.depth_to_space(
-        inputs, block_size, data_format=data_format)
+      # Initialize the input tensor with ascending whole numbers as floats.
+      x = [f * 1.0 for f in range(total_size)]
+      shape = nchw_input_shape if data_format == "NCHW" else nhwc_input_shape
+      t = constant_op.constant(x, shape=shape, dtype=dtypes.float32)
+      expected = self.depthToSpaceUsingTranspose(t, block_size, data_format)
+      actual = array_ops.depth_to_space(t, block_size, data_format=data_format)
 
     with self.test_session(use_gpu=use_gpu) as sess:
       actual_vals, expected_vals = sess.run([actual, expected])
       self.assertTrue(np.array_equal(actual_vals, expected_vals))
 
   def testAgainstTranspose(self):
-    self.compareToTranspose("NHWC", 3, 2, 3, 1, 2, False)
-    self.compareToTranspose("NHWC", 3, 2, 3, 2, 2, False)
-    self.compareToTranspose("NHWC", 3, 2, 3, 1, 2, True)
-    self.compareToTranspose("NHWC", 3, 2, 3, 2, 2, True)
-
-    self.compareToTranspose("NCHW", 3, 2, 3, 1, 2, True)
-    self.compareToTranspose("NCHW", 3, 2, 3, 2, 2, True)
-    self.compareToTranspose("NCHW", 3, 2, 3, 1, 3, True)
-    self.compareToTranspose("NCHW", 3, 2, 3, 2, 3, True)
-    self.compareToTranspose("NCHW", 5, 7, 11, 3, 2, True)
-    self.compareToTranspose("NCHW", 3, 200, 300, 32, 2, True)
+    self.compareToTranspose(3, 2, 3, 1, 2, "NHWC", False)
+    self.compareToTranspose(3, 2, 3, 2, 2, "NHWC", False)
+    self.compareToTranspose(1, 2, 3, 2, 3, "NHWC", False)
+
+    if not test.is_gpu_available():
+      tf_logging.info("skipping gpu tests since gpu not available")
+      return
+
+    self.compareToTranspose(3, 2, 3, 1, 2, "NHWC", True)
+    self.compareToTranspose(3, 2, 3, 2, 2, "NHWC", True)
+    self.compareToTranspose(3, 2, 3, 1, 2, "NCHW", True)
+    self.compareToTranspose(3, 2, 3, 2, 2, "NCHW", True)
+    self.compareToTranspose(3, 2, 3, 1, 3, "NCHW", True)
+    self.compareToTranspose(3, 2, 3, 2, 3, "NCHW", True)
+    self.compareToTranspose(5, 7, 11, 3, 2, "NCHW", True)
+    self.compareToTranspose(3, 200, 300, 32, 2, "NCHW", True)
+
+    self.compareToTranspose(3, 2, 3, 8, 2, "NCHW_VECT_C", True)
+    self.compareToTranspose(3, 2, 3, 4, 3, "NCHW_VECT_C", True)
+    self.compareToTranspose(3, 2, 3, 8, 3, "NCHW_VECT_C", True)
+    self.compareToTranspose(5, 7, 11, 12, 2, "NCHW_VECT_C", True)
+    self.compareToTranspose(3, 200, 300, 32, 2, "NCHW_VECT_C", True)
 
 
 class DepthToSpaceGradientTest(test.TestCase):
-- 
GitLab


From 915e60d59b2c6201c68cf4a2a83a1599934b0702 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 11 Oct 2017 18:21:53 -0700
Subject: [PATCH 280/909] Optimize gradients for Mean

PiperOrigin-RevId: 171904584
---
 tensorflow/python/framework/constant_op.py    |  3 ++-
 tensorflow/python/ops/math_grad.py            | 20 ++++++++-----------
 .../python/profiler/model_analyzer_test.py    |  4 ++--
 3 files changed, 12 insertions(+), 15 deletions(-)

diff --git a/tensorflow/python/framework/constant_op.py b/tensorflow/python/framework/constant_op.py
index 342fcd98c5..686f5aa6db 100644
--- a/tensorflow/python/framework/constant_op.py
+++ b/tensorflow/python/framework/constant_op.py
@@ -42,6 +42,7 @@ from __future__ import division
 from __future__ import print_function
 
 import numpy as np
+import six
 
 from tensorflow.core.framework import attr_value_pb2
 from tensorflow.python.eager import context
@@ -110,7 +111,7 @@ def convert_to_eager_tensor(value, ctx, dtype=None):
     dtype = dtype.as_datatype_enum
   device = ctx.device_name
   handle = ctx._handle  # pylint: disable=protected-access
-  if isinstance(value, (int, float)):
+  if isinstance(value, (float,) + six.integer_types):
     # Use a scalar cache. This will put each scalar of each type only once on
     # each device. Scalars don't use much device memory but copying scalars can
     # trigger memcpys which are slow.
diff --git a/tensorflow/python/ops/math_grad.py b/tensorflow/python/ops/math_grad.py
index d36d66f899..3754e039ed 100644
--- a/tensorflow/python/ops/math_grad.py
+++ b/tensorflow/python/ops/math_grad.py
@@ -96,20 +96,16 @@ def _MinGrad(op, grad):
 def _MeanGrad(op, grad):
   """Gradient for Mean."""
   sum_grad = _SumGrad(op, grad)[0]
-  input_shape = array_ops.shape(op.inputs[0])
-  output_shape = array_ops.shape(op.outputs[0])
-  # TODO(apassos) remove this device hackery as eager copy to device becomes
-  # more seamless.
-  with ops.colocate_with(input_shape):
+  input_size = op.inputs[0].get_shape().num_elements()
+  output_size = op.outputs[0].get_shape().num_elements()
+  if input_size is not None and output_size is not None:
+    factor = input_size // max(output_size, 1)
+    factor = constant_op.constant(factor, dtype=sum_grad.dtype)
+  else:
+    input_shape = array_ops.shape(op.inputs[0])
+    output_shape = array_ops.shape(op.outputs[0])
     factor = _safe_shape_div(
         math_ops.reduce_prod(input_shape), math_ops.reduce_prod(output_shape))
-  if context.in_eager_mode():
-    # Note that we go through numpy here just so we use the eager per-device
-    # scalar cache. We know the factor is a host memory tensor because it's a
-    # shape, and we also know that converting a scalar into a tensor triggers a
-    # per-device cache.
-    factor = factor.numpy()
-    factor = constant_op.constant(factor, dtype=sum_grad.dtype)
   return sum_grad / math_ops.cast(factor, sum_grad.dtype), None
 
 
diff --git a/tensorflow/python/profiler/model_analyzer_test.py b/tensorflow/python/profiler/model_analyzer_test.py
index 943ae0a3a1..2578fc3e87 100644
--- a/tensorflow/python/profiler/model_analyzer_test.py
+++ b/tensorflow/python/profiler/model_analyzer_test.py
@@ -222,12 +222,12 @@ class PrintModelAnalysisTest(test.TestCase):
         with gfile.Open(outfile, 'r') as f:
           lines = f.read().split('\n')
           result = '\n'.join([l[:min(len(l), 80)] for l in lines])
-          self.assertEqual(compat.as_bytes('node name | # parameters | # float_ops\n_TFProfRoot (--/2.84k params, --/168.85k flops)\n  model_analyzer_testlib.py:63:BuildFullModel (0/1.80k params, 0/45.37k flops)\n    model_analyzer_testlib.py:40:BuildSmallModel (0/0 params, 0/0 flops)\n    model_analyzer_testlib.py:44:BuildSmallModel (0/4 params, 0/8 flops)\n    model_analyzer_testlib.py:48:BuildSmallModel (0/648 params, 0/1.30k flops)\n    model_analyzer_testlib.py:49:BuildSmallModel (0/0 params, 0/23.33k flops)\n    model_analyzer_testlib.py:53:BuildSmallModel (0/1.15k params, 0/2.30k flops)\n    model_analyzer_testlib.py:54:BuildSmallModel (0/0 params, 0/18.43k flops)\n  model_analyzer_testlib.py:63:BuildFullModel (gradient) (0/0 params, 0/67.39k f\n    model_analyzer_testlib.py:49:BuildSmallModel (gradient) (0/0 params, 0/46.66\n    model_analyzer_testlib.py:54:BuildSmallModel (gradient) (0/0 params, 0/20.74\n  model_analyzer_testlib.py:67:BuildFullModel (0/1.04k params, 0/18.57k flops)\n  model_analyzer_testlib.py:67:BuildFullModel (gradient) (0/0 params, 0/37.00k f\n  model_analyzer_testlib.py:69:BuildFullModel (0/0 params, 0/0 flops)\n  model_analyzer_testlib.py:70:BuildFullModel (0/0 params, 0/258 flops)\n  model_analyzer_testlib.py:70:BuildFullModel (gradient) (0/0 params, 0/130 flop\n  model_analyzer_testlib.py:72:BuildFullModel (0/0 params, 0/141 flops)\n'),
+          self.assertEqual(compat.as_bytes('node name | # parameters | # float_ops\n_TFProfRoot (--/2.84k params, --/168.85k flops)\n  model_analyzer_testlib.py:63:BuildFullModel (0/1.80k params, 0/45.37k flops)\n    model_analyzer_testlib.py:40:BuildSmallModel (0/0 params, 0/0 flops)\n    model_analyzer_testlib.py:44:BuildSmallModel (0/4 params, 0/8 flops)\n    model_analyzer_testlib.py:48:BuildSmallModel (0/648 params, 0/1.30k flops)\n    model_analyzer_testlib.py:49:BuildSmallModel (0/0 params, 0/23.33k flops)\n    model_analyzer_testlib.py:53:BuildSmallModel (0/1.15k params, 0/2.30k flops)\n    model_analyzer_testlib.py:54:BuildSmallModel (0/0 params, 0/18.43k flops)\n  model_analyzer_testlib.py:63:BuildFullModel (gradient) (0/0 params, 0/67.39k f\n    model_analyzer_testlib.py:49:BuildSmallModel (gradient) (0/0 params, 0/46.66\n    model_analyzer_testlib.py:54:BuildSmallModel (gradient) (0/0 params, 0/20.74\n  model_analyzer_testlib.py:67:BuildFullModel (0/1.04k params, 0/18.57k flops)\n  model_analyzer_testlib.py:67:BuildFullModel (gradient) (0/0 params, 0/37.00k f\n  model_analyzer_testlib.py:69:BuildFullModel (0/0 params, 0/0 flops)\n  model_analyzer_testlib.py:70:BuildFullModel (0/0 params, 0/258 flops)\n  model_analyzer_testlib.py:70:BuildFullModel (gradient) (0/0 params, 0/129 flop\n  model_analyzer_testlib.py:72:BuildFullModel (0/0 params, 0/141 flops)\n'),
                            compat.as_bytes(result))
 
         self.assertLess(0, tfprof_node.total_exec_micros)
         self.assertEqual(2844, tfprof_node.total_parameters)
-        self.assertEqual(168855, tfprof_node.total_float_ops)
+        self.assertEqual(168854, tfprof_node.total_float_ops)
         self.assertEqual(8, len(tfprof_node.children))
         self.assertEqual('_TFProfRoot', tfprof_node.name)
         self.assertEqual(
-- 
GitLab


From 0790dacbe8652c667b6e74986699dc8c1626f00b Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Wed, 11 Oct 2017 20:54:28 -0700
Subject: [PATCH 281/909] Internal change.

PiperOrigin-RevId: 171913954
---
 tensorflow/contrib/timeseries/python/timeseries/BUILD | 1 +
 tensorflow/contrib/training/BUILD                     | 1 +
 2 files changed, 2 insertions(+)

diff --git a/tensorflow/contrib/timeseries/python/timeseries/BUILD b/tensorflow/contrib/timeseries/python/timeseries/BUILD
index da583a2ba0..76e8ccc62a 100644
--- a/tensorflow/contrib/timeseries/python/timeseries/BUILD
+++ b/tensorflow/contrib/timeseries/python/timeseries/BUILD
@@ -371,6 +371,7 @@ py_test(
         "ar_model_test.py",
     ],
     srcs_version = "PY2AND3",
+    tags = ["notsan"],
     deps = [
         ":ar_model",
         ":estimators",
diff --git a/tensorflow/contrib/training/BUILD b/tensorflow/contrib/training/BUILD
index 8e3d869a51..80a5debe99 100644
--- a/tensorflow/contrib/training/BUILD
+++ b/tensorflow/contrib/training/BUILD
@@ -263,6 +263,7 @@ py_test(
     srcs = ["python/training/training_test.py"],
     shard_count = 3,
     srcs_version = "PY2AND3",
+    tags = ["notsan"],
     deps = [
         ":training_py",
         "//tensorflow/contrib/framework:framework_py",
-- 
GitLab


From ec1496d13b4580c4459f024a1d62127fef8cea70 Mon Sep 17 00:00:00 2001
From: Jianwei Xie <xiejw@google.com>
Date: Wed, 11 Oct 2017 21:03:46 -0700
Subject: [PATCH 282/909] Update docstring for tpu-config

PiperOrigin-RevId: 171914551
---
 tensorflow/contrib/tpu/python/tpu/tpu_config.py | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_config.py b/tensorflow/contrib/tpu/python/tpu/tpu_config.py
index 02135bfe40..44069cfb55 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_config.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_config.py
@@ -36,10 +36,16 @@ class TPUConfig(
       global step is increased `iterations_per_loop` times in one `Session.run`.
       It is recommended to be set as number of global steps for next checkpoint.
     num_shards: The number of TPU shards in the system.
-    per_host_input_for_training: If `True`, `input_fn` is invoked per host
-      rather than per shard. Note: This behavior is going to be default as
-      `True` soon, so this flag will be removed after that. Also note that this
-      only works for single-host TPU training now.
+    per_host_input_for_training: If `True`, `input_fn` is invoked Per-Host
+      rather than Per-Core. With Per-Host input pipeline deployment, `input_fn`
+      is invoked once on each host. To be precise, with a global batch size
+      `train_batch_size` in `TPUEstimator` constructor, the batch size for each
+      shard is `train_batch_size` // #hosts. With Per-Core input pipeline
+      deployment, the shard batch size is `train_batch_size` // #cores. Note:
+      This behavior is going to be default as `True` soon, so this flag will be
+      removed after that. Also note that this only works for single-host TPU
+      training now (tracked in b/67051042). For multi-host, please use Per-Core,
+      i.e., `False` for `per_host_input_for_training`.
   """
 
   def __new__(cls,
-- 
GitLab


From 1f1b2bb6c3833a472036da22b7c910f5f2bdf694 Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Wed, 11 Oct 2017 21:14:24 -0700
Subject: [PATCH 283/909] Automated g4 rollback of changelist 171877766

PiperOrigin-RevId: 171915087
---
 tensorflow/compiler/xla/service/cpu/BUILD     |  18 --
 .../compiler/xla/service/cpu/cpu_compiler.cc  |  15 +-
 .../compiler/xla/service/cpu/cpu_compiler.h   |   2 +-
 .../cpu/cpu_parallelization_preparation.cc    |  20 ++
 .../compiler/xla/service/cpu/cpu_runtime.cc   |   3 -
 .../compiler/xla/service/cpu/cpu_runtime.h    |   1 -
 .../compiler/xla/service/cpu/ir_emitter.cc    | 192 +++---------------
 .../compiler/xla/service/cpu/ir_emitter.h     |  21 +-
 .../service/cpu/parallel_task_assignment.cc   | 148 ++------------
 .../service/cpu/parallel_task_assignment.h    |  49 -----
 .../xla/service/cpu/runtime_fork_join.cc      |  93 ---------
 .../xla/service/cpu/runtime_fork_join.h       |  33 ---
 .../xla/service/cpu/simple_orc_jit.cc         |   2 -
 tensorflow/compiler/xla/tests/BUILD           |   2 -
 tensorflow/compiler/xla/tests/fusion_test.cc  | 136 ++-----------
 15 files changed, 91 insertions(+), 644 deletions(-)
 delete mode 100644 tensorflow/compiler/xla/service/cpu/runtime_fork_join.cc
 delete mode 100644 tensorflow/compiler/xla/service/cpu/runtime_fork_join.h

diff --git a/tensorflow/compiler/xla/service/cpu/BUILD b/tensorflow/compiler/xla/service/cpu/BUILD
index c71eca0d39..8ab358fe17 100644
--- a/tensorflow/compiler/xla/service/cpu/BUILD
+++ b/tensorflow/compiler/xla/service/cpu/BUILD
@@ -87,7 +87,6 @@ cc_library(
         ":ir_emitter",
         ":layout_assignment",
         ":parallel_cpu_executable",
-        ":parallel_task_assignment",
         ":simple_orc_jit",
         "//tensorflow/compiler/xla:literal_util",
         "//tensorflow/compiler/xla:protobuf_util",
@@ -156,7 +155,6 @@ cc_library(
         ":disassembler",
         ":external_constant_pool",
         ":runtime_conv2d",
-        ":runtime_fork_join",
         ":runtime_matmul",
         ":runtime_single_threaded_conv2d",
         ":runtime_single_threaded_matmul",
@@ -245,7 +243,6 @@ cc_library(
         ":dot_op_emitter",
         ":external_constant_pool",
         ":ir_emission_utils",
-        ":shape_partition",
         ":simple_orc_jit",
         "//tensorflow/compiler/xla:shape_util",
         "//tensorflow/compiler/xla:status_macros",
@@ -508,20 +505,6 @@ cc_library(
     ],
 )
 
-cc_library(
-    name = "runtime_fork_join",
-    srcs = ["runtime_fork_join.cc"],
-    hdrs = ["runtime_fork_join.h"],
-    copts = runtime_copts(),
-    visibility = ["//visibility:public"],
-    deps = [
-        "//tensorflow/compiler/xla:executable_run_options",
-        "//tensorflow/core:lib",
-        "//tensorflow/core:lib_internal",
-        "//third_party/eigen3",
-    ],
-)
-
 tf_cc_test(
     name = "cpu_runtime_test",
     srcs = ["cpu_runtime_test.cc"],
@@ -705,7 +688,6 @@ cc_library(
         ":shape_partition",
         "//tensorflow/compiler/xla/service:hlo",
         "//tensorflow/compiler/xla/service:hlo_cost_analysis",
-        "//tensorflow/compiler/xla/service:hlo_pass",
     ],
 )
 
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
index 3272044faa..386800d221 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
@@ -58,7 +58,6 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/cpu/ir_emitter.h"
 #include "tensorflow/compiler/xla/service/cpu/layout_assignment.h"
 #include "tensorflow/compiler/xla/service/cpu/parallel_cpu_executable.h"
-#include "tensorflow/compiler/xla/service/cpu/parallel_task_assignment.h"
 #include "tensorflow/compiler/xla/service/cpu/simple_orc_jit.h"
 #include "tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h"
 #include "tensorflow/compiler/xla/service/flatten_call_graph.h"
@@ -249,7 +248,7 @@ class CollectProfileCandidates : public DfsHloVisitorWithDefault {
 };
 }  // namespace
 
-Status CpuCompiler::RunHloPasses(HloModule* module, bool is_aot_compile) {
+Status CpuCompiler::RunHloPasses(HloModule* module) {
   // Optimization pipeline.
   HloPassPipeline pipeline("CPU");
   pipeline.AddInvariantChecker<HloVerifier>(ShapeSizeBytesFunction());
@@ -317,14 +316,6 @@ Status CpuCompiler::RunHloPasses(HloModule* module, bool is_aot_compile) {
   if (options::CpuParallelBackendRequested(module->config())) {
     pipeline.AddPass<ParallelizationPreparation>(max_parallelism,
                                                  ShapeSizeBytesFunction());
-  } else if (!is_aot_compile) {
-    // Run ParallelTaskAssigner to assign parallel tasks to HLOs in module.
-    // Note this is not run for AOT because it would bring in thread pool
-    // and thread synchronization dependencies which would likely increase
-    // binary size (and most AOT applications are single-threaded).
-    // TODO(29630486) Support multi-threaded AOT.
-    pipeline.AddPass<ParallelTaskAssigner>(max_parallelism,
-                                           ShapeSizeBytesFunction(), module);
   }
   // Copy insertion should be performed immediately before IR emission to avoid
   // inserting unnecessary copies (later pass adds an instruction which
@@ -459,7 +450,7 @@ StatusOr<std::unique_ptr<Executable>> CpuCompiler::Compile(
   llvm_module->setDataLayout(jit->data_layout());
   llvm_module->setTargetTriple(jit->target_triple().getTriple());
 
-  TF_RETURN_IF_ERROR(RunHloPasses(module.get(), /*is_aot_compile=*/false));
+  TF_RETURN_IF_ERROR(RunHloPasses(module.get()));
 
   HloComputation* computation = module->entry_computation();
   std::unordered_map<const HloInstruction*, size_t> hlo_to_profile_idx;
@@ -758,7 +749,7 @@ CpuCompiler::CompileAheadOfTime(std::vector<std::unique_ptr<HloModule>> modules,
     HloModule* module = modules[i].get();
     VLOG(1) << "Compiling ahead-of-time: " << module->name();
 
-    TF_RETURN_IF_ERROR(RunHloPasses(module, /*is_aot_compile=*/true));
+    TF_RETURN_IF_ERROR(RunHloPasses(module));
 
     TF_ASSIGN_OR_RETURN(
         SequentialHloOrdering::HloModuleSequence module_sequence,
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_compiler.h b/tensorflow/compiler/xla/service/cpu/cpu_compiler.h
index 21dd128619..bd3541500d 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_compiler.h
+++ b/tensorflow/compiler/xla/service/cpu/cpu_compiler.h
@@ -131,7 +131,7 @@ class CpuCompiler : public LLVMCompiler {
 
   // Runs the HLO passes which are necessary for both optimizations and
   // correctness.
-  Status RunHloPasses(HloModule* module, bool is_aot_compile);
+  Status RunHloPasses(HloModule* module);
 
   TF_DISALLOW_COPY_AND_ASSIGN(CpuCompiler);
 };
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_parallelization_preparation.cc b/tensorflow/compiler/xla/service/cpu/cpu_parallelization_preparation.cc
index 662ee60923..2cd0aa7880 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_parallelization_preparation.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_parallelization_preparation.cc
@@ -116,6 +116,26 @@ StatusOr<bool> ParallelizationPreparation::RunParallelTaskAssignment(
   // Assign parallel tasks to HLOs in entry computation.
   HloComputation* computation = module->entry_computation();
   for (auto* instruction : computation->instructions()) {
+    // Currently, we do not assign parallel tasks to instructions with at least
+    // one of the following properties:
+    // *) Internal threading (library calls to kConv, kDot, and kCustomCall).
+    // *) Emit custom loops (kSelectAndScatter, FusionKind::kTransposeDot).
+    // *) Tuple-shaped.
+    // TODO(b/27458679) Parallelize instructions which are skipped here.
+    if (instruction->opcode() == HloOpcode::kParameter ||
+        instruction->opcode() == HloOpcode::kConstant ||
+        instruction->opcode() == HloOpcode::kCall ||
+        instruction->opcode() == HloOpcode::kCustomCall ||
+        instruction->opcode() == HloOpcode::kSelectAndScatter ||
+        (instruction->opcode() == HloOpcode::kConvolution &&
+         PotentiallyImplementedAsEigenConvolution(*instruction)) ||
+        PotentiallyImplementedAsEigenDot(*instruction) ||
+        (instruction->opcode() == HloOpcode::kFusion &&
+         instruction->fusion_kind() != HloInstruction::FusionKind::kLoop) ||
+        ShapeUtil::IsTuple(instruction->shape())) {
+      continue;
+    }
+
     // Calculate target parallel task count in [1, max_parallelism_].
     const int64 target_parallel_task_count =
         parallel_task_assignment.GetTargetParallelTaskCount(instruction);
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_runtime.cc b/tensorflow/compiler/xla/service/cpu/cpu_runtime.cc
index 7908dc173d..c7155b858b 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_runtime.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_runtime.cc
@@ -51,9 +51,6 @@ extern const char* const kAcquireOutfeedBufferForPopulationSymbolName =
     "__xla_cpu_runtime_AcquireOutfeedBufferForPopulation";
 extern const char* const kReleaseOutfeedBufferAfterPopulationSymbolName =
     "__xla_cpu_runtime_ReleaseOutfeedBufferAfterPopulation";
-extern const char* const kParallelForkJoinSymbolName =
-    "__xla_cpu_runtime_ParallelForkJoin";
-
 extern const char* const kXlaCpuRuntimeSymbolNamePrefix = "__xla_cpu_runtime_";
 }  // namespace runtime
 }  // namespace cpu
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_runtime.h b/tensorflow/compiler/xla/service/cpu/cpu_runtime.h
index 2ade455b8a..29feb7267f 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_runtime.h
+++ b/tensorflow/compiler/xla/service/cpu/cpu_runtime.h
@@ -51,7 +51,6 @@ extern const char* const kAcquireInfeedBufferForDequeueSymbolName;
 extern const char* const kReleaseInfeedBufferAfterDequeueSymbolName;
 extern const char* const kAcquireOutfeedBufferForPopulationSymbolName;
 extern const char* const kReleaseOutfeedBufferAfterPopulationSymbolName;
-extern const char* const kParallelForkJoinSymbolName;
 
 // All symbol names for XLA CPU runtime functions need to start with this
 // prefix.
diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
index c38325554f..633ad0290c 100644
--- a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
@@ -42,7 +42,6 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/cpu/dot_op_emitter.h"
 #include "tensorflow/compiler/xla/service/cpu/elemental_ir_emitter.h"
 #include "tensorflow/compiler/xla/service/cpu/ir_emission_utils.h"
-#include "tensorflow/compiler/xla/service/cpu/shape_partition.h"
 #include "tensorflow/compiler/xla/service/cpu/simple_orc_jit.h"
 #include "tensorflow/compiler/xla/service/elemental_ir_emitter.h"
 #include "tensorflow/compiler/xla/service/hlo_opcode.h"
@@ -187,9 +186,20 @@ void IrEmitter::InitializeIrFunction(const string& function_name) {
   // Even though the type of params and temps is void** in the host's view, in
   // LLVM IR this is represented by i8*, similarly to void*. It's up to the code
   // to use GEPs to unravel the indirection layers.
+  llvm::Type* i8_ptr_type = llvm::Type::getInt8PtrTy(module_->getContext());
+  llvm::Type* i8_ptr_ptr_type = i8_ptr_type->getPointerTo();
+  llvm::Type* i64_ptr_type = llvm::Type::getInt64PtrTy(module_->getContext());
+  std::vector<llvm::Type*> compute_function_params(
+      {i8_ptr_type, i8_ptr_type, i8_ptr_ptr_type, i8_ptr_ptr_type});
+  if (IsParallelContext()) {
+    compute_function_params.push_back(i64_ptr_type);
+  }
+  if (hlo_to_profile_idx_) {
+    compute_function_params.push_back(i64_ptr_type);
+  }
   llvm::FunctionType* compute_function_type = llvm::FunctionType::get(
       /*Result=*/llvm::Type::getVoidTy(module_->getContext()),
-      /*Params=*/GetComputeFunctionParams(),
+      /*Params=*/compute_function_params,
       /*isVarArg=*/false);
 
   // Functions with local linkage get an inlining bonus.  Because we know
@@ -211,7 +221,7 @@ void IrEmitter::InitializeIrFunction(const string& function_name) {
   (++arg_iter)->setName("run_options");
   (++arg_iter)->setName("params");
   (++arg_iter)->setName("temps");
-  if (num_dynamic_loop_bounds_ > 0) {
+  if (IsParallelContext()) {
     (++arg_iter)->setName("dynamic_loop_bounds");
   }
   if (hlo_to_profile_idx_) {
@@ -2276,19 +2286,8 @@ Status IrEmitter::HandleCall(HloInstruction* call) {
   }
 
   TF_RETURN_IF_ERROR(EmitTargetAddressForOp(call));
-
-  if (!computation->root_instruction()->outer_dimension_partitions().empty() &&
-      !parallel_cpu_backend_) {
-    // ParallelTaskAssignment assigned partitions, emit call to
-    // ParallelForkJoin.
-    TF_RETURN_IF_ERROR(EmitParallelForkJoin(parameter_addresses,
-                                            emitted_value_[call], computation,
-                                            call_ir_function));
-  } else {
-    EmitArrayFunctionCallInto(call_ir_function, parameter_addresses,
-                              emitted_value_[call], computation->name());
-  }
-
+  EmitArrayFunctionCallInto(call_ir_function, parameter_addresses,
+                            emitted_value_[call], computation->name());
   return Status::OK();
 }
 
@@ -2598,7 +2597,7 @@ Status IrEmitter::FinishVisit(HloInstruction* root) {
   // For the parallel cpu backend, we record the total for each embedded
   // computation callee with its caller kCall HLO.
   HloInstruction* hlo_to_lookup = nullptr;
-  if (parallel_cpu_backend_ && is_top_level_computation_) {
+  if (IsParallelContext()) {
     auto* computation = root->parent();
     auto* entry_computation = computation->parent()->entry_computation();
     if (computation != entry_computation) {
@@ -2756,27 +2755,12 @@ llvm::Type* IrEmitter::IrShapeType(const Shape& shape) {
   return llvm_ir::ShapeToIrType(shape, &ir_builder_);
 }
 
-std::vector<llvm::Type*> IrEmitter::GetComputeFunctionParams() {
-  llvm::Type* i8_ptr_type = llvm::Type::getInt8PtrTy(module_->getContext());
-  llvm::Type* i8_ptr_ptr_type = i8_ptr_type->getPointerTo();
-  llvm::Type* i64_ptr_type = llvm::Type::getInt64PtrTy(module_->getContext());
-  std::vector<llvm::Type*> compute_function_params(
-      {i8_ptr_type, i8_ptr_type, i8_ptr_ptr_type, i8_ptr_ptr_type});
-  if (num_dynamic_loop_bounds_ > 0) {
-    compute_function_params.push_back(i64_ptr_type);
-  }
-  if (hlo_to_profile_idx_) {
-    compute_function_params.push_back(i64_ptr_type);
-  }
-  return compute_function_params;
-}
-
 llvm::Argument* IrEmitter::GetResultArgument() {
   return GetArg(compute_function_, 0);
 }
 
 llvm::Argument* IrEmitter::GetProfileCountersArgument() {
-  const int64 arg_index = num_dynamic_loop_bounds_ > 0 ? 5 : 4;
+  const int64 arg_index = IsParallelContext() ? 5 : 4;
   return hlo_to_profile_idx_ ? GetArg(compute_function_, arg_index) : nullptr;
 }
 
@@ -2859,11 +2843,18 @@ llvm::Value* IrEmitter::EmitElementFunctionCall(
       AsStringRef(tensorflow::strings::StrCat(name, "_return_value")));
 }
 
-// Emits code to allocate an array of parameter address pointers, and store
-// each address from 'parameter_addresses'.
-// Returns an array of compute function call arguments (including parameter
-// address buffer).
-std::vector<llvm::Value*> IrEmitter::GetArrayFunctionCallArguments(
+// Emits a core function call based on the following pseudo-code.
+//
+//   char** parameter_addresses_buffer =
+//       allocate buffer with a pointer for each parameter to the function
+//   for each parameter index, i.e. for i = 0, ..., #parameters:
+//     parameter_addresses_buffer[i] = parameter_addresses[i]
+//   call function(return_value_buffer,
+//                 parameter_addresses_buffer,
+//                 temps)
+//   return return_value_buffer  -- address of the return value.
+void IrEmitter::EmitArrayFunctionCallInto(
+    llvm::Function* function,
     tensorflow::gtl::ArraySlice<llvm::Value*> parameter_addresses,
     llvm::Value* return_value_buffer, tensorflow::StringPiece name) {
   llvm::Value* parameter_addresses_buffer =
@@ -2892,26 +2883,7 @@ std::vector<llvm::Value*> IrEmitter::GetArrayFunctionCallArguments(
   if (auto* profile_counters = GetProfileCountersArgument()) {
     arguments.push_back(profile_counters);
   }
-  return arguments;
-}
-
-// Emits a core function call based on the following pseudo-code.
-//
-//   char** parameter_addresses_buffer =
-//       allocate buffer with a pointer for each parameter to the function
-//   for each parameter index, i.e. for i = 0, ..., #parameters:
-//     parameter_addresses_buffer[i] = parameter_addresses[i]
-//   call function(return_value_buffer,
-//                 parameter_addresses_buffer,
-//                 temps)
-//   return return_value_buffer  -- address of the return value.
-void IrEmitter::EmitArrayFunctionCallInto(
-    llvm::Function* function,
-    tensorflow::gtl::ArraySlice<llvm::Value*> parameter_addresses,
-    llvm::Value* return_value_buffer, tensorflow::StringPiece name) {
-  ir_builder_.CreateCall(
-      function, GetArrayFunctionCallArguments(parameter_addresses,
-                                              return_value_buffer, name));
+  ir_builder_.CreateCall(function, arguments);
 }
 
 llvm::Value* IrEmitter::EmitArrayFunctionCall(
@@ -2931,110 +2903,6 @@ llvm::Value* IrEmitter::EmitArrayFunctionCall(
   return return_value_buffer;
 }
 
-// Emits a call to a runtime fork/join function which dispatches parallel
-// calls to 'parallel_function' (and joins threads before returning).
-Status IrEmitter::EmitParallelForkJoin(
-    tensorflow::gtl::ArraySlice<llvm::Value*> parameter_addresses,
-    llvm::Value* output_address, HloComputation* computation,
-    llvm::Function* parallel_function) {
-  HloInstruction* root = computation->root_instruction();
-
-  // Build ParallelForkJoin function type.
-  std::vector<llvm::Type*> compute_function_params = GetComputeFunctionParams();
-  // Number of parallel compute functions.
-  compute_function_params.push_back(ir_builder_.getInt32Ty());
-  // Array of partitions. There is an array element for each
-  // partition x partition_dim x 2 (for dimension start and limit).
-  compute_function_params.push_back(
-      llvm::Type::getInt64PtrTy(module_->getContext()));
-  // Number of partitioned most-major dimensions in 'root.shape'.
-  compute_function_params.push_back(ir_builder_.getInt32Ty());
-  // Function pointer for compute function to be dispatched in parallel.
-  compute_function_params.push_back(
-      llvm::Type::getInt8PtrTy(module_->getContext()));
-
-  llvm::FunctionType* fork_join_type = llvm::FunctionType::get(
-      /*Result=*/llvm::Type::getVoidTy(module_->getContext()),
-      /*Params=*/compute_function_params,
-      /*isVarArg=*/false);
-
-  llvm::Function* fork_join_func =
-      llvm::cast<llvm::Function>(module_->getOrInsertFunction(
-          runtime::kParallelForkJoinSymbolName, fork_join_type));
-  fork_join_func->setCallingConv(llvm::CallingConv::C);
-  fork_join_func->setDoesNotThrow();
-
-  // Add common compute function arguments.
-  const string name = computation->name();
-  std::vector<llvm::Value*> arguments =
-      GetArrayFunctionCallArguments(parameter_addresses, output_address, name);
-
-  // Create ShapePartitionIterator to generate all partitions of 'root.shape'.
-  ShapePartitionIterator partition_iterator(root->shape(),
-                                            root->outer_dimension_partitions());
-  const int64 num_partitions = partition_iterator.GetTotalPartitionCount();
-  // Add argument specifying the number of parallel partitions.
-  arguments.push_back(ir_builder_.getInt32(num_partitions));
-
-  // The number of partitioned most-major dimensions in 'root.shape'.
-  const int32 num_partitioned_dims = root->outer_dimension_partitions().size();
-  // A dimension partition consists of two elements: [start_index, limit_index).
-  const int32 dim_partition_size = 2;
-  // Calculate array partition stride.
-  const int32 array_partition_stride =
-      num_partitioned_dims * dim_partition_size;
-  // Calculate the total number of elements in the partition array.
-  const int32 partition_array_size =
-      dim_partition_size * num_partitioned_dims * num_partitions;
-
-  // Store dimension partition values as llvm constants in 'partitions'.
-  // See comments in runtime_fork_join.cc for array layout description.
-  std::vector<llvm::Constant*> partitions(partition_array_size);
-  for (int32 i = 0; i < num_partitions; ++i) {
-    std::vector<std::pair<int64, int64>> dim_partitions =
-        partition_iterator.GetPartition(i);
-    CHECK_EQ(num_partitioned_dims, dim_partitions.size());
-    const int32 partition_index = i * array_partition_stride;
-    for (int32 j = 0; j < num_partitioned_dims; ++j) {
-      const std::pair<int64, int64>& dim_partition = dim_partitions[j];
-      const int32 index = partition_index + j * dim_partition_size;
-      // Store partition [dim_start, dim_limit) intervals for each dimension.
-      partitions[index] = ir_builder_.getInt64(dim_partition.first);
-      partitions[index + 1] =
-          ir_builder_.getInt64(dim_partition.first + dim_partition.second);
-    }
-  }
-
-  // Create global variable out of dimension partitions in 'partitions'.
-  llvm::ArrayType* partitions_array_type =
-      llvm::ArrayType::get(ir_builder_.getInt64Ty(), partition_array_size);
-  llvm::Constant* partitions_array =
-      llvm::ConstantArray::get(partitions_array_type, partitions);
-  llvm::GlobalVariable* global_partitions_array = new llvm::GlobalVariable(
-      /*Module=*/*module_,
-      /*Type=*/partitions_array_type,
-      /*isConstant=*/true,
-      /*Linkage=*/llvm::GlobalValue::PrivateLinkage,
-      /*Initializer=*/partitions_array,
-      /*Name=*/
-      AsStringRef(
-          tensorflow::strings::StrCat(name, "_parallel_dimension_partitions")));
-
-  // Add argument specifying parallel dimension partitions.
-  arguments.push_back(ir_builder_.CreateBitCast(
-      global_partitions_array,
-      llvm::Type::getInt64PtrTy(module_->getContext())));
-  // Add argument specifying the number of partitioned most-major dimensions.
-  arguments.push_back(ir_builder_.getInt32(num_partitioned_dims));
-  // Add argument for parallel compute function pointer.
-  arguments.push_back(
-      ir_builder_.CreateBitCast(parallel_function, ir_builder_.getInt8PtrTy()));
-  // Emit call to parallel fork/join.
-  ir_builder_.CreateCall(fork_join_func, arguments);
-
-  return Status::OK();
-}
-
 Status IrEmitter::EmitTargetAddressForOp(const HloInstruction* op) {
   llvm::Value* addr;
   const Shape& target_shape = op->shape();
diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.h b/tensorflow/compiler/xla/service/cpu/ir_emitter.h
index 58c185af1e..53c4b6f241 100644
--- a/tensorflow/compiler/xla/service/cpu/ir_emitter.h
+++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.h
@@ -249,9 +249,6 @@ class IrEmitter : public DfsHloVisitorWithDefault {
   // Convenience function to get the IR type matching the given shape.
   llvm::Type* IrShapeType(const Shape& shape);
 
-  // Returns an array of compute function parameter types.
-  std::vector<llvm::Type*> GetComputeFunctionParams();
-
   // Get the llvm::Value* that represents the "retval" argument of the
   // computation function being emitted by this emitter.
   llvm::Argument* GetResultArgument();
@@ -326,18 +323,6 @@ class IrEmitter : public DfsHloVisitorWithDefault {
       tensorflow::gtl::ArraySlice<llvm::Value*> parameter_addresses,
       tensorflow::StringPiece name);
 
-  // Returns an array of compute function call arguments.
-  std::vector<llvm::Value*> GetArrayFunctionCallArguments(
-      tensorflow::gtl::ArraySlice<llvm::Value*> parameter_addresses,
-      llvm::Value* return_value_buffer, tensorflow::StringPiece name);
-
-  // Emits a call to a runtime fork/join function which dispatches parallel
-  // calls to 'parallel_function' (and joins threads before returning).
-  Status EmitParallelForkJoin(
-      tensorflow::gtl::ArraySlice<llvm::Value*> parameter_addresses,
-      llvm::Value* output_address, HloComputation* computation,
-      llvm::Function* parallel_function);
-
   // Verifies that the element types of all of the given operand instructions
   // match and are of one of the given supported types.
   Status ElementTypesSameAndSupported(
@@ -611,6 +596,12 @@ class IrEmitter : public DfsHloVisitorWithDefault {
   Status EmitXfeedTransfer(XfeedKind kind, const Shape& shape,
                            llvm::Value* program_buffer_address);
 
+  // Returns true if the current function being emitted is called in a
+  // parallel context (returns false otherwise).
+  bool IsParallelContext() {
+    return parallel_cpu_backend_ && is_top_level_computation_;
+  }
+
   const HloModuleConfig& hlo_module_config_;
 
   const bool parallel_cpu_backend_;
diff --git a/tensorflow/compiler/xla/service/cpu/parallel_task_assignment.cc b/tensorflow/compiler/xla/service/cpu/parallel_task_assignment.cc
index 7219736b9e..d4b5e41f50 100644
--- a/tensorflow/compiler/xla/service/cpu/parallel_task_assignment.cc
+++ b/tensorflow/compiler/xla/service/cpu/parallel_task_assignment.cc
@@ -48,56 +48,29 @@ class SimpleCostModel : public ParallelCostModel {
 class DefaultCostModel : public ParallelCostModel {
  public:
   DefaultCostModel(const int64 max_parallelism,
-                   const HloCostAnalysis::ShapeSizeFunction& shape_size,
                    std::unique_ptr<HloCostAnalysis> cost_analysis)
       : max_parallelism_(max_parallelism),
-        shape_size_(shape_size),
         cost_analysis_(std::move(cost_analysis)) {}
   ~DefaultCostModel() override {}
 
   int64 GetParallelTaskCount(HloInstruction* instruction) override {
-    // Parameters for parallel task count computation.
-    int64 instruction_cost;
-    int64 min_cost_per_thread;
-    int64 max_parallelism;
-    // Calculate flops-to-bytes-ratio for 'instruction'.
-    const int64 bytes_accessed =
-        std::max(1LL, cost_analysis_->bytes_accessed(*instruction));
-    const float flops_to_bytes_ratio =
-        cost_analysis_->flop_count(*instruction) /
-        static_cast<float>(bytes_accessed);
-    // Check for I/O bound instructions.
-    if (flops_to_bytes_ratio <= 1.0) {
-      // Limit max parallelism for I/O bound instructions by assuming a
-      // sub-linear scaling function (fit based on emperical benchmark results).
-      // TODO(29630486) Develop system bandwidth model.
-      max_parallelism =
-          std::ceil(std::sqrt(tensorflow::port::NumSchedulableCPUs()));
-      // Use shape size instruction cost and L2 cache size min per-thread cost.
-      instruction_cost = shape_size_(instruction->shape());
-      min_cost_per_thread = 256LL << 10;  // 256KB L2 Cache size.
-    } else {
-      // Use max parallelism for compute bound instructions.
-      max_parallelism = max_parallelism_;
-      // Calculate the instruction cost in cycles.
-      // TODO(29630486) Improve on this linear cost model.
-      // Consider making 'min_cost_per_thread' be a function of the target
-      // bandwidth limit for instructions with low arithmetic complexity.
-      instruction_cost =
-          1 * cost_analysis_->flop_count(*instruction) +
-          2 * cost_analysis_->transcendental_count(*instruction) +
-          10 * cost_analysis_->bytes_accessed(*instruction);
-      // Minimum per-thread cost is 100us of work on a 2GHz core.
-      min_cost_per_thread = 100000;
-    }
+    // Calculate the instruction cost in cycles.
+    // TODO(29630486) Improve on this linear cost model.
+    // Consider making 'min_cost_per_thread' be a function of the target
+    // bandwidth limit for instructions with low arithmetic complexity.
+    const int64 instruction_cost =
+        1 * cost_analysis_->flop_count(*instruction) +
+        2 * cost_analysis_->transcendental_count(*instruction) +
+        10 * cost_analysis_->bytes_accessed(*instruction);
+    // Minimum per-thread cost is 100us of work on a 2GHz core.
+    const int64 min_cost_per_thread = 100000;
     // Return target parallel task count in [1, max_parallelism_].
-    return std::min(max_parallelism,
+    return std::min(max_parallelism_,
                     std::max(1LL, instruction_cost / min_cost_per_thread));
   }
 
  private:
   const int64 max_parallelism_;
-  const HloCostAnalysis::ShapeSizeFunction shape_size_;
   const std::unique_ptr<HloCostAnalysis> cost_analysis_;
 };
 
@@ -113,7 +86,7 @@ ParallelTaskAssignment::ParallelTaskAssignment(
   Status status = computation->root_instruction()->Accept(cost_analysis.get());
   if (status.ok()) {
     // Set default cost model based on 'cost_analysis'.
-    cost_model_.reset(new DefaultCostModel(max_parallelism, shape_size,
+    cost_model_.reset(new DefaultCostModel(max_parallelism,
                                            std::move(cost_analysis)));
   } else {
     // Fall back to a simple cost model based on hlo size and L2 cache size.
@@ -148,102 +121,5 @@ int64 ParallelTaskAssignment::GetTargetParallelTaskCount(
   return cost_model_->GetParallelTaskCount(instruction);
 }
 
-StatusOr<bool> ParallelTaskAssigner::Run(HloModule* module) {
-  XLA_VLOG_LINES(2, "ParallelTaskAssigner ENTRY");
-  XLA_VLOG_LINES(3, module->ToString());
-
-  // Compute target parallel task counts for all instructions in 'module'.
-  HloToParallelTasks hlo_to_parallel_tasks;
-  ComputeTargetParallelTasks(module, &hlo_to_parallel_tasks);
-
-  // Assign parallel tasks to target specific instructions in 'module'.
-  // TODO(b/27458679) Support inter-op parallelism.
-  bool changed = AssignParallelTasks(module, hlo_to_parallel_tasks);
-
-  XLA_VLOG_LINES(2, "ParallelTaskAssigner EXIT");
-  XLA_VLOG_LINES(3, module->ToString());
-  return changed;
-}
-
-bool ParallelTaskAssigner::AssignParallelTasks(
-    HloModule* module, const HloToParallelTasks& hlo_to_parallel_tasks) {
-  return AssignParallelTasksHelper(module, module->entry_computation(),
-                                   hlo_to_parallel_tasks);
-}
-
-bool ParallelTaskAssigner::AssignParallelTasksHelper(
-    HloModule* module, HloComputation* computation,
-    const HloToParallelTasks& hlo_to_parallel_tasks) {
-  bool changed = false;
-  // Snapshot set of instructions because outlining modifies the set below.
-  std::vector<HloInstruction*> instructions(computation->instructions().begin(),
-                                            computation->instructions().end());
-  for (auto* instruction : instructions) {
-    // Assign parallel tasks to sub-computations for While and Call HLOs.
-    // TODO(b/27458679) Evaluate alternative intra-op parallelsim placement,
-    // and support other callable computations like reduce.
-    if (instruction->opcode() == HloOpcode::kWhile) {
-      changed |= AssignParallelTasksHelper(module, instruction->while_body(),
-                                           hlo_to_parallel_tasks);
-      continue;
-    } else if (instruction->opcode() == HloOpcode::kCall) {
-      changed |= AssignParallelTasksHelper(module, instruction->to_apply(),
-                                           hlo_to_parallel_tasks);
-      continue;
-    }
-    // Skip if no parallel tasks were computed in first pass.
-    auto it = hlo_to_parallel_tasks.find(instruction);
-    if (it == hlo_to_parallel_tasks.end()) {
-      continue;
-    }
-    // Get target parallel task count computed for 'instruction'.
-    const int64 target_parallel_task_count = (*it).second;
-    // Assign feasible dimension partitions (based on actual dimension sizes).
-    auto dim_partition_counts = ShapePartitionAssigner(instruction->shape())
-                                    .Run(target_parallel_task_count);
-    const int64 total_partition_count =
-        ShapePartitionAssigner::GetTotalPartitionCount(dim_partition_counts);
-    if (total_partition_count <= 1) {
-      // Feasible partition calculation resulting in no partitioning, so skip.
-      continue;
-    }
-
-    // Outline 'instruction' in 'computation' for parallel task assignment.
-    auto* call = module->OutlineExpressionFromComputation(
-        {instruction},
-        tensorflow::strings::StrCat("parallel_", instruction->name()),
-        computation);
-
-    // Set assigned dimension partitioning to 'instruction'.
-    auto* new_root = call->to_apply()->root_instruction();
-    new_root->set_outer_dimension_partitions(dim_partition_counts);
-
-    VLOG(2) << "Assigned parallel task count: " << total_partition_count
-            << " to instruction: " << new_root->name()
-            << " parent: " << new_root->parent()->name();
-    changed = true;
-  }
-  return changed;
-}
-
-void ParallelTaskAssigner::ComputeTargetParallelTasks(
-    HloModule* module, HloToParallelTasks* hlo_to_parallel_tasks) {
-  // Compute parallel task counts for all instructions in 'module'.
-  for (auto* computation : module->computations()) {
-    if (computation->IsFusionComputation()) {
-      continue;
-    }
-    for (auto* instruction : computation->instructions()) {
-      // Query ParallelTaskAssignment for target parallel task count.
-      const int64 target_parallel_task_count =
-          parallel_task_assignment_.GetTargetParallelTaskCount(instruction);
-      if (target_parallel_task_count > 1) {
-        hlo_to_parallel_tasks->insert(
-            {instruction, target_parallel_task_count});
-      }
-    }
-  }
-}
-
 }  // namespace cpu
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/service/cpu/parallel_task_assignment.h b/tensorflow/compiler/xla/service/cpu/parallel_task_assignment.h
index e036da5784..15f065a3ad 100644
--- a/tensorflow/compiler/xla/service/cpu/parallel_task_assignment.h
+++ b/tensorflow/compiler/xla/service/cpu/parallel_task_assignment.h
@@ -18,7 +18,6 @@ limitations under the License.
 
 #include "tensorflow/compiler/xla/service/hlo_cost_analysis.h"
 #include "tensorflow/compiler/xla/service/hlo_module.h"
-#include "tensorflow/compiler/xla/service/hlo_pass_interface.h"
 
 namespace xla {
 namespace cpu {
@@ -50,54 +49,6 @@ class ParallelTaskAssignment {
   std::unique_ptr<ParallelCostModel> cost_model_;
 };
 
-// ParallelTaskAssigner computes target parallel task counts for all HLOs
-// in the module, then assigns parallel task counts to HLOs in the entry
-// computation, or to HLOs in embedded computations invoked by (potentially
-// nested) kWhile or kCall instructions.
-// Each HLO which is assigned parallel task counts is outlined into its
-// own embedded computation, which is compiled as a parallel compute function,
-// and which is invoked from a kCall instruction that is lowered in codegen to
-// a runtime parallel fork/join call.
-class ParallelTaskAssigner : public HloPassInterface {
- public:
-  // 'max_parallelism': the maximum parallel task count per instruction.
-  // 'shape_size': shape size function used by HloCostAnalysis during parallel
-  //               task assignment.
-  // 'module': the containing HloModule.
-  ParallelTaskAssigner(const int64 max_parallelism,
-                       const HloCostAnalysis::ShapeSizeFunction& shape_size,
-                       HloModule* module)
-      : parallel_task_assignment_(max_parallelism, shape_size, module) {}
-  ~ParallelTaskAssigner() override {}
-
-  tensorflow::StringPiece name() const override {
-    return "cpu-parallel-task-assigner";
-  }
-
-  // Run parallel task assigner on 'module'.
-  // Returns true if the computation was changed, false otherwise.
-  StatusOr<bool> Run(HloModule* module) override;
-
- private:
-  using HloToParallelTasks = std::unordered_map<const HloInstruction*, int64>;
-
-  // Assigns target parallel tasks from 'hlo_to_parallel_tasks' to HLOs in
-  // 'module'.
-  // Returns true if the computation was changed, false otherwise.
-  bool AssignParallelTasks(HloModule* module,
-                           const HloToParallelTasks& hlo_to_parallel_tasks);
-  bool AssignParallelTasksHelper(
-      HloModule* module, HloComputation* computation,
-      const HloToParallelTasks& hlo_to_parallel_tasks);
-
-  // Computes target parallel task counts (returned in 'parallel_task_counts')
-  // for parallelizable instructions in 'module'.
-  void ComputeTargetParallelTasks(HloModule* module,
-                                  HloToParallelTasks* hlo_to_parallel_tasks);
-
-  ParallelTaskAssignment parallel_task_assignment_;
-};
-
 }  // namespace cpu
 }  // namespace xla
 
diff --git a/tensorflow/compiler/xla/service/cpu/runtime_fork_join.cc b/tensorflow/compiler/xla/service/cpu/runtime_fork_join.cc
deleted file mode 100644
index af2f3de6b8..0000000000
--- a/tensorflow/compiler/xla/service/cpu/runtime_fork_join.cc
+++ /dev/null
@@ -1,93 +0,0 @@
-/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#include "tensorflow/compiler/xla/service/cpu/runtime_fork_join.h"
-
-#define EIGEN_USE_THREADS
-
-#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
-#include "tensorflow/compiler/xla/executable_run_options.h"
-#include "tensorflow/core/lib/core/blocking_counter.h"
-#include "tensorflow/core/platform/logging.h"
-
-using ComputeFunctionType = void (*)(void*, const void*, const void**, void**,
-                                     int64*, uint64*);
-
-// Dispatches 'num_partitions - 1' calls to 'function_ptr' in parallel.
-// Calls 'function_ptr' for first partition inline.
-// Uses blocking counter to synchonize threads after parallel calls complete.
-//
-// The 'partitions' array has a total number of elements equal to
-// 'num_partitions * num_partitioned_dims * 2' (the '2' is necessary to specify
-// dimension start and limit indices).
-//
-// The 'partitions' array layout stores array elements in memory with dimension
-// start limit as the most-minor dimension, followed by dimension, then
-// partition.
-//
-// EX: Layout of 'partitions' array with 'num_partitions = 2', and
-//     'num_partitioned_dims = 3'
-//
-//   [partition0_dim0_start]
-//   [partition0_dim0_limit]
-//   [partition0_dim1_start]
-//   [partition0_dim1_limit]
-//   [partition0_dim2_start]
-//   [partition0_dim2_limit]
-//   [partition1_dim0_start]
-//   [partition1_dim0_limit]
-//   [partition1_dim1_start]
-//   [partition1_dim1_limit]
-//   [partition1_dim2_start]
-//   [partition1_dim2_limit]
-//
-void __xla_cpu_runtime_ParallelForkJoin(
-    void* result_ptr, const void* run_options_ptr, const void** params,
-    void** temps, uint64* prof_counters, tensorflow::int32 num_partitions,
-    tensorflow::int64* partitions, tensorflow::int32 num_partitioned_dims,
-    void* function_ptr) {
-  VLOG(2) << "ParallelForkJoin ENTRY"
-          << " num_partitions: " << num_partitions
-          << " num_partitioned_dims: " << num_partitioned_dims;
-  CHECK_GT(num_partitions, 1);
-  CHECK_GT(num_partitioned_dims, 0);
-  const xla::ExecutableRunOptions* run_options =
-      static_cast<const xla::ExecutableRunOptions*>(run_options_ptr);
-  ComputeFunctionType function =
-      reinterpret_cast<ComputeFunctionType>(function_ptr);
-  // Compute partition stride in 'partitions' array.
-  const int64 stride = 2 * num_partitioned_dims;
-
-  // Dispatch 'num_partitions - 1' compute functions to run in parallel.
-  tensorflow::BlockingCounter bc(num_partitions - 1);
-  for (tensorflow::int32 i = 1; i < num_partitions; ++i) {
-    const int64 offset = i * stride;
-    run_options->intra_op_thread_pool()->enqueue_function(
-        [i, function, result_ptr, run_options_ptr, params, temps, prof_counters,
-         partitions, offset, &bc]() {
-          function(result_ptr, run_options_ptr, params, temps,
-                   &partitions[offset], prof_counters);
-          bc.DecrementCount();
-          VLOG(3) << "ParallelForkJoin partition " << i << " done.";
-        });
-  }
-
-  // Call first compute function inline.
-  function(result_ptr, run_options_ptr, params, temps, &partitions[0],
-           prof_counters);
-  VLOG(3) << "ParallelForkJoin partition 0 done.";
-  bc.Wait();
-  VLOG(2) << "ParallelForkJoin EXIT";
-}
diff --git a/tensorflow/compiler/xla/service/cpu/runtime_fork_join.h b/tensorflow/compiler/xla/service/cpu/runtime_fork_join.h
deleted file mode 100644
index 1ddcaf5274..0000000000
--- a/tensorflow/compiler/xla/service/cpu/runtime_fork_join.h
+++ /dev/null
@@ -1,33 +0,0 @@
-/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#ifndef THIRD_PARTY_TENSORFLOW_COMPILER_XLA_SERVICE_CPU_RUNTIME_FORK_JOIN_H_
-#define THIRD_PARTY_TENSORFLOW_COMPILER_XLA_SERVICE_CPU_RUNTIME_FORK_JOIN_H_
-
-#include "tensorflow/core/platform/types.h"
-
-extern "C" {
-
-// Dispatches 'num_partitions' parallel calls to 'function_ptr' and joins
-// threads before returning. See comments in runtime_fork_join.cc for details.
-extern void __xla_cpu_runtime_ParallelForkJoin(
-    void* result_ptr, const void* run_options_ptr, const void** params,
-    void** temps, uint64* prof_counters, tensorflow::int32 num_partitions,
-    tensorflow::int64* partitions, tensorflow::int32 num_partitioned_dims,
-    void* function_ptr);
-
-}  // extern "C"
-
-#endif  // THIRD_PARTY_TENSORFLOW_COMPILER_XLA_SERVICE_CPU_RUNTIME_FORK_JOIN_H_
diff --git a/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc b/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc
index cfffb3fbc3..c614e334a8 100644
--- a/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc
+++ b/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc
@@ -32,7 +32,6 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/cpu/cpu_runtime_neon.h"
 #include "tensorflow/compiler/xla/service/cpu/cpu_runtime_sse4_1.h"
 #include "tensorflow/compiler/xla/service/cpu/runtime_conv2d.h"
-#include "tensorflow/compiler/xla/service/cpu/runtime_fork_join.h"
 #include "tensorflow/compiler/xla/service/cpu/runtime_matmul.h"
 #include "tensorflow/compiler/xla/service/cpu/runtime_single_threaded_conv2d.h"
 #include "tensorflow/compiler/xla/service/cpu/runtime_single_threaded_matmul.h"
@@ -105,7 +104,6 @@ class JITSymbolTable {
     ADD_JIT_SYMBOL_TO_TABLE(EigenSingleThreadedConvF32);
     ADD_JIT_SYMBOL_TO_TABLE(EigenSingleThreadedMatMulF32);
     ADD_JIT_SYMBOL_TO_TABLE(EigenSingleThreadedMatMulF64);
-    ADD_JIT_SYMBOL_TO_TABLE(ParallelForkJoin);
 
 #undef ADD_JIT_SYMBOL_TO_TABLE
   }
diff --git a/tensorflow/compiler/xla/tests/BUILD b/tensorflow/compiler/xla/tests/BUILD
index 256ec71ab5..f37a331a72 100644
--- a/tensorflow/compiler/xla/tests/BUILD
+++ b/tensorflow/compiler/xla/tests/BUILD
@@ -1410,10 +1410,8 @@ xla_test(
         "//tensorflow/compiler/xla/tests:hlo_test_base",
         "//tensorflow/compiler/xla/tests:literal_test_util",
         "//tensorflow/compiler/xla/tests:xla_internal_test_main",
-        "//tensorflow/core:core_cpu_internal",
         "//tensorflow/core:lib",
         "//tensorflow/core:test",
-        "//third_party/eigen3",
     ],
 )
 
diff --git a/tensorflow/compiler/xla/tests/fusion_test.cc b/tensorflow/compiler/xla/tests/fusion_test.cc
index a8f6488996..3bf9ccb197 100644
--- a/tensorflow/compiler/xla/tests/fusion_test.cc
+++ b/tensorflow/compiler/xla/tests/fusion_test.cc
@@ -17,12 +17,8 @@ limitations under the License.
 #include <algorithm>
 #include <memory>
 #include <new>
-#include <random>
 #include <utility>
 
-#define EIGEN_USE_THREADS
-
-#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 #include "tensorflow/compiler/xla/array2d.h"
 #include "tensorflow/compiler/xla/client/client_library.h"
 #include "tensorflow/compiler/xla/client/computation.h"
@@ -41,7 +37,6 @@ limitations under the License.
 #include "tensorflow/compiler/xla/tests/literal_test_util.h"
 #include "tensorflow/compiler/xla/tests/test_macros.h"
 #include "tensorflow/compiler/xla/xla_data.pb.h"
-#include "tensorflow/core/common_runtime/eigen_thread_pool.h"
 #include "tensorflow/core/lib/gtl/array_slice.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/protobuf.h"
@@ -255,42 +250,6 @@ XLA_TEST_F(FusionTest, Parameter) {
                               ErrorSpec(1e-4));
 }
 
-XLA_TEST_F(FusionTest, RandomizedParallelPartition) {
-  // Tests parallel partitioning of a fusion instruction.
-  // Create shape with random outer dimension size to generate random parallel
-  // partition counts for each test run.
-  const int seed = tensorflow::testing::RandomSeed();
-  LOG(INFO) << "RandomizedParallelPartition seed: " << seed;
-  std::mt19937 generator(seed);
-  std::uniform_int_distribution<int> distribution(128, 1024);
-  const int64 rand_dim0_size = distribution(generator);
-  const int64 dim1_size = 1024;
-  Shape shape =
-      ShapeUtil::MakeShapeWithLayout(F32, {rand_dim0_size, dim1_size}, {1, 0});
-  // Build simple fusion computation: y = x^2 (elementwise).
-  auto builder = HloComputation::Builder(TestName());
-  auto hlo_module = CreateNewModule();
-
-  auto two = builder.AddInstruction(
-      HloInstruction::CreateConstant(Literal::CreateR0<float>(2.0)));
-  auto x =
-      builder.AddInstruction(HloInstruction::CreateBroadcast(shape, two, {}));
-  auto y = builder.AddInstruction(
-      HloInstruction::CreateBinary(shape, HloOpcode::kMultiply, x, x));
-
-  hlo_module->AddEntryComputation(builder.Build())
-      ->CreateFusionInstruction(/*instructions_to_fuse=*/{y, x, two},
-                                HloInstruction::FusionKind::kLoop);
-  // Compute result.
-  auto result = ExecuteAndTransfer(std::move(hlo_module), {});
-  // Every element of result should be y = x^2 = 4.0.
-  for (int i = 0; i < rand_dim0_size; ++i) {
-    for (int j = 0; j < dim1_size; ++j) {
-      EXPECT_EQ(4.0, result->Get<float>({i, j}));
-    }
-  }
-}
-
 XLA_TEST_F(FusionTest, BroadcastIntoBinaryOp) {
   auto builder = HloComputation::Builder(TestName());
   auto hlo_module = CreateNewModule();
@@ -763,104 +722,47 @@ void BM_ParallelFusion(int num_iters) {
   auto executors = PlatformUtil::GetStreamExecutors(platform).ValueOrDie();
   StreamExecutorMemoryAllocator allocator(platform, executors);
 
-  const int64 intra_op_parallelism_threads = 24;
+  const int64 intra_op_parallelism_threads = 16;
   xla::LocalClientOptions client_options;
   client_options.set_platform(platform);
   client_options.set_intra_op_parallelism_threads(intra_op_parallelism_threads);
   auto client =
       ClientLibrary::GetOrCreateLocalClient(client_options).ValueOrDie();
 
-  auto* transfer_manager =
-      TransferManager::GetForPlatform(platform).ValueOrDie();
-  int device_ordinal = client->default_device_ordinal();
-
-  // Computation shape parameters.
-  const int64 param0_dim0 = 1024;
-  const int64 param0_dim1 = 1024;
-  const int64 param1_dim0 = 1024;
-  const int64 param1_dim1 = 1024;
-  const int64 param2_dim0 = 1024;
-  const int64 param2_dim1 = 1024;
-
-  // Create computation.
+  const int64 dim_size = 1024;
+  // Create a simple fusable elementwise computation.
   ComputationBuilder builder(client, "ParallelFusion");
-  Shape shape0 = ShapeUtil::MakeShape(F32, {param0_dim0, param0_dim1});
-  auto param0 = builder.Parameter(0, shape0, "param0");
-  Shape shape1 = ShapeUtil::MakeShape(F32, {param1_dim0, param1_dim1});
-  auto param1 = builder.Parameter(1, shape1, "param1");
-  Shape shape2 = ShapeUtil::MakeShape(F32, {param2_dim0, param2_dim1});
-  auto param2 = builder.Parameter(2, shape2, "param2");
-
-  auto x = builder.Mul(param0, param1);
-  auto y = builder.Add(x, param2);
+  Shape input_shape = ShapeUtil::MakeShape(F32, {dim_size, dim_size});
+  auto input0 = builder.Broadcast(builder.ConstantR0<float>(1.5f),
+                                  AsInt64Slice(input_shape.dimensions()));
+  auto input1 = builder.Broadcast(builder.ConstantR0<float>(2.0f),
+                                  AsInt64Slice(input_shape.dimensions()));
+  auto input2 = builder.Broadcast(builder.ConstantR0<float>(3.0f),
+                                  AsInt64Slice(input_shape.dimensions()));
+  auto x = builder.Mul(input0, input1);
+  auto y = builder.Add(x, input2);
   auto computation = builder.Build().ConsumeValueOrDie();
 
-  // Transfer literals to device.
-  auto buffer0 =
-      ScopedShapedBuffer::Allocate(shape0, &allocator, /*device_ordinal=*/0)
-          .ConsumeValueOrDie();
-  auto param0_literal =
-      Literal::CreateR2F32Linspace(1.0, 2.0, param0_dim0, param0_dim1);
-  ASSERT_IS_OK(transfer_manager->TransferLiteralToDevice(
-      executors[device_ordinal], *param0_literal, buffer0->mutable_buffer({})));
-
-  auto buffer1 =
-      ScopedShapedBuffer::Allocate(shape1, &allocator, /*device_ordinal=*/0)
-          .ConsumeValueOrDie();
-  auto param1_literal =
-      Literal::CreateR2F32Linspace(1.0, 2.0, param1_dim0, param1_dim1);
-  ASSERT_IS_OK(transfer_manager->TransferLiteralToDevice(
-      executors[device_ordinal], *param1_literal, buffer1->mutable_buffer({})));
-
-  auto buffer2 =
-      ScopedShapedBuffer::Allocate(shape2, &allocator, /*device_ordinal=*/0)
-          .ConsumeValueOrDie();
-  auto param2_literal =
-      Literal::CreateR2F32Linspace(1.0, 2.0, param2_dim0, param2_dim1);
-  ASSERT_IS_OK(transfer_manager->TransferLiteralToDevice(
-      executors[device_ordinal], *param2_literal, buffer2->mutable_buffer({})));
-
-  // Build executable.
   std::unique_ptr<LocalExecutable> executable =
-      client
-          ->Compile(computation,
-                    {&buffer0->shape(), &buffer1->shape(), &buffer2->shape()},
-                    ExecutableBuildOptions())
+      client->Compile(computation, {}, ExecutableBuildOptions())
           .ConsumeValueOrDie();
 
-  se::Stream stream(executors[client->default_device_ordinal()]);
-  stream.Init();
-
-  // Initialize thread pool.
-  tensorflow::thread::ThreadPool pool(tensorflow::Env::Default(), "XLAEigen",
-                                      intra_op_parallelism_threads);
-  tensorflow::EigenThreadPoolWrapper tp(&pool);
-  Eigen::ThreadPoolDevice device(&tp, tp.NumThreads());
-
-  // Initialize ExecutableRunOptions.
-  ExecutableRunOptions options;
-  options.set_allocator(&allocator).set_stream(&stream);
-  options.set_intra_op_thread_pool(&device);
-
   // Run some warm-up executions.
+  ExecutableRunOptions options;
+  options.set_allocator(&allocator);
   const int kWarmups = 2;
   for (int i = 0; i < kWarmups; ++i) {
-    auto result =
-        executable->Run({buffer0.get(), buffer1.get(), buffer2.get()}, options);
+    auto result = executable->Run({}, options);
     ASSERT_TRUE(result.ok());
   }
 
   // Run benchmark.
-  const int64 total_bytes = param0_dim0 * param0_dim0 +
-                            param1_dim0 * param1_dim0 +
-                            param2_dim0 * param2_dim0;
-  tensorflow::testing::BytesProcessed(static_cast<int64>(num_iters) *
-                                      total_bytes * sizeof(float));
+  tensorflow::testing::BytesProcessed(static_cast<int64>(num_iters) * dim_size *
+                                      dim_size * sizeof(float));
   tensorflow::testing::UseRealTime();
   tensorflow::testing::StartTiming();
   for (int i = 0; i < num_iters; ++i) {
-    auto result =
-        executable->Run({buffer0.get(), buffer1.get(), buffer2.get()}, options);
+    auto result = executable->Run({}, options);
     ASSERT_TRUE(result.ok());
   }
 }
-- 
GitLab


From e725083ea3269a25acf968300ce5dd05cc626bfe Mon Sep 17 00:00:00 2001
From: Rohan Jain <rohanj@google.com>
Date: Wed, 11 Oct 2017 21:31:36 -0700
Subject: [PATCH 284/909] Disabling a couple of ClusterFLR tests since test
 clusters in GRPC seem to have issues with multiple servers and have
 intermittent failures (https://github.com/grpc/grpc/issues/10142)

PiperOrigin-RevId: 171915902
---
 tensorflow/core/distributed_runtime/BUILD                | 4 ----
 .../cluster_function_library_runtime_test.cc             | 9 +++++++--
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/tensorflow/core/distributed_runtime/BUILD b/tensorflow/core/distributed_runtime/BUILD
index 26e82fbb9a..07e279cb64 100644
--- a/tensorflow/core/distributed_runtime/BUILD
+++ b/tensorflow/core/distributed_runtime/BUILD
@@ -121,10 +121,6 @@ tf_cc_test(
     name = "cluster_function_library_runtime_test",
     srcs = ["cluster_function_library_runtime_test.cc"],
     linkstatic = tf_kernel_tests_linkstatic(),
-    tags = [
-        "no_oss",
-        "nomac",
-    ],
     deps = [
         ":worker_session",
         "//tensorflow/core:framework_internal",
diff --git a/tensorflow/core/distributed_runtime/cluster_function_library_runtime_test.cc b/tensorflow/core/distributed_runtime/cluster_function_library_runtime_test.cc
index 6855313b3b..04587dd8ca 100644
--- a/tensorflow/core/distributed_runtime/cluster_function_library_runtime_test.cc
+++ b/tensorflow/core/distributed_runtime/cluster_function_library_runtime_test.cc
@@ -214,7 +214,11 @@ node {
   TF_EXPECT_GRAPH_EQ(expected, actual);
 }
 
-TEST_F(ClusterFunctionLibraryRuntimeTest, InstantiateAndRun) {
+// Disabling the following two tests since there seem to be some issues with
+// GRPC bringing up multiple processes as sub-processes.
+// More info at: https://github.com/grpc/grpc/issues/10142.
+// TODO(rohanj): Enable tests when the grpc bug is fixed.
+TEST_F(ClusterFunctionLibraryRuntimeTest, DISABLED_InstantiateAndRun) {
   FunctionDefLibrary proto;
   *(proto.add_function()) = test::function::XTimesTwoInt32();
   FunctionLibraryDefinition lib_def(OpRegistry::Global(), proto);
@@ -227,7 +231,8 @@ TEST_F(ClusterFunctionLibraryRuntimeTest, InstantiateAndRun) {
   test::ExpectTensorEqual<int32>(y, test::AsTensor<int32>({2, 4, 6, 8}));
 }
 
-TEST_F(ClusterFunctionLibraryRuntimeTest, InstantiateAndRunAttrSubstitution) {
+TEST_F(ClusterFunctionLibraryRuntimeTest,
+       DISABLED_InstantiateAndRunAttrSubstitution) {
   FunctionDefLibrary proto;
   *(proto.add_function()) = test::function::XTimesTwo();
   FunctionLibraryDefinition lib_def(OpRegistry::Global(), proto);
-- 
GitLab


From fb180d58d16f1b63a6f714593c58bbd8978bb4a3 Mon Sep 17 00:00:00 2001
From: Derek Murray <mrry@google.com>
Date: Wed, 11 Oct 2017 21:32:02 -0700
Subject: [PATCH 285/909] [tf.data] Add `tf.contrib.data.get_single_element()`.

This utility function is designed for using a `tf.data.Dataset` in a serving
context, where it is useful for expressing the stateless transformation from a
fed-in batch into the serving input.

PiperOrigin-RevId: 171915928
---
 .../python/kernel_tests/iterator_ops_test.py  | 25 ++++++++++
 .../contrib/data/python/ops/dataset_ops.py    | 46 +++++++++++++++++++
 tensorflow/core/kernels/iterator_ops.cc       | 40 ++++++++++++++++
 tensorflow/core/ops/dataset_ops.cc            | 30 ++++++++++++
 4 files changed, 141 insertions(+)

diff --git a/tensorflow/contrib/data/python/kernel_tests/iterator_ops_test.py b/tensorflow/contrib/data/python/kernel_tests/iterator_ops_test.py
index 8d8cb574ea..20f6d6ba34 100644
--- a/tensorflow/contrib/data/python/kernel_tests/iterator_ops_test.py
+++ b/tensorflow/contrib/data/python/kernel_tests/iterator_ops_test.py
@@ -584,6 +584,31 @@ class IteratorTest(test.TestCase):
         with self.assertRaises(errors.InvalidArgumentError):
           sess.run(restore_op)
 
+  def testToSingleElement(self):
+    skip_value = array_ops.placeholder(dtypes.int64, shape=[])
+    take_value = array_ops.placeholder_with_default(
+        constant_op.constant(1, dtype=dtypes.int64), shape=[])
+
+    dataset = (dataset_ops.Dataset.range(100)
+               .skip(skip_value)
+               .map(lambda x: x * x)
+               .take(take_value))
+
+    element = dataset_ops.get_single_element(dataset)
+
+    with self.test_session() as sess:
+      self.assertEqual(0, sess.run(element, feed_dict={skip_value: 0}))
+      self.assertEqual(25, sess.run(element, feed_dict={skip_value: 5}))
+      self.assertEqual(100, sess.run(element, feed_dict={skip_value: 10}))
+
+      with self.assertRaisesRegexp(errors.InvalidArgumentError,
+                                   "Dataset was empty."):
+        sess.run(element, feed_dict={skip_value: 100})
+
+      with self.assertRaisesRegexp(errors.InvalidArgumentError,
+                                   "Dataset had more than one element."):
+        sess.run(element, feed_dict={skip_value: 0, take_value: 2})
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/contrib/data/python/ops/dataset_ops.py b/tensorflow/contrib/data/python/ops/dataset_ops.py
index b74dcd3be2..fe1d50db33 100644
--- a/tensorflow/contrib/data/python/ops/dataset_ops.py
+++ b/tensorflow/contrib/data/python/ops/dataset_ops.py
@@ -26,6 +26,7 @@ from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.util import nest
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import tensor_shape
+from tensorflow.python.ops import gen_dataset_ops
 from tensorflow.python.ops import gen_io_ops
 from tensorflow.python.ops import script_ops
 from tensorflow.python.util import deprecation
@@ -760,3 +761,48 @@ class Dataset(dataset_ops.Dataset):
     if not isinstance(dataset, dataset_ops.Dataset):
       raise TypeError("`transformation_func` must return a Dataset.")
     return Dataset(dataset)
+
+
+def get_single_element(dataset):
+  """Returns the single element in `dataset` as a nested structure of tensors.
+
+  This function enables you to use a @{tf.data.Dataset} in a stateless
+  "tensor-in tensor-out" expression, without creating a @{tf.data.Iterator}.
+  This can be useful when your preprocessing transformations are expressed
+  as a `Dataset`, and you want to use the transformation at serving time.
+  For example:
+
+  ```python
+  input_batch = tf.placeholder(tf.string, shape=[BATCH_SIZE])
+
+  def preprocessing_fn(input_str):
+    # ...
+    return image, label
+
+  dataset = (tf.data.Dataset.from_tensor_slices(input_batch)
+             .map(preprocessing_fn, num_parallel_calls=BATCH_SIZE)
+             .batch(BATCH_SIZE))
+
+  image_batch, label_batch = tf.contrib.data.get_single_element(dataset)
+  ```
+
+  Args:
+    dataset: A @{tf.data.Dataset} object containing a single element.
+
+  Returns:
+    A nested structure of @{tf.Tensor} objects, corresponding to the single
+    element of `dataset`.
+
+  Raises:
+    TypeError: if `dataset` is not a `tf.data.Dataset` object.
+    InvalidArgumentError (at runtime): if `dataset` does not contain exactly
+      one element.
+  """
+  if not isinstance(dataset, dataset_ops.Dataset):
+    raise TypeError("`dataset` must be a `tf.data.Dataset` object.")
+  return nest.pack_sequence_as(
+      dataset.output_types,
+      gen_dataset_ops.dataset_to_single_element(
+          dataset._as_variant_tensor(),  # pylint: disable=protected-access
+          output_types=nest.flatten(dataset.output_types),
+          output_shapes=nest.flatten(dataset.output_shapes)))
diff --git a/tensorflow/core/kernels/iterator_ops.cc b/tensorflow/core/kernels/iterator_ops.cc
index 0a59d3c963..df13edc83a 100644
--- a/tensorflow/core/kernels/iterator_ops.cc
+++ b/tensorflow/core/kernels/iterator_ops.cc
@@ -256,6 +256,44 @@ class MakeIteratorOp : public OpKernel {
   }
 };
 
+class ToSingleElementOp : public OpKernel {
+ public:
+  explicit ToSingleElementOp(OpKernelConstruction* ctx) : OpKernel(ctx) {}
+
+  void Compute(OpKernelContext* ctx) override {
+    DatasetBase* dataset;
+    OP_REQUIRES_OK(ctx, GetDatasetFromVariantTensor(ctx->input(0), &dataset));
+    auto iterator = dataset->MakeIterator("SingleElementIterator");
+
+    IteratorContext::Params params;
+    params.env = ctx->env();
+    params.step_id = ctx->step_id();
+    params.resource_manager = ctx->resource_manager();
+    params.runner = *(ctx->runner());
+    IteratorContext iter_ctx(std::move(params));
+
+    std::vector<Tensor> components;
+    components.reserve(dataset->output_dtypes().size());
+    bool end_of_sequence;
+
+    OP_REQUIRES_OK(ctx,
+                   iterator->GetNext(&iter_ctx, &components, &end_of_sequence));
+    OP_REQUIRES(ctx, !end_of_sequence,
+                errors::InvalidArgument("Dataset was empty."));
+
+    for (int i = 0; i < components.size(); ++i) {
+      // TODO(mrry): Check that the shapes match the shape attrs.
+      ctx->set_output(i, components[i]);
+    }
+
+    components.clear();
+    OP_REQUIRES_OK(ctx,
+                   iterator->GetNext(&iter_ctx, &components, &end_of_sequence));
+    OP_REQUIRES(ctx, end_of_sequence,
+                errors::InvalidArgument("Dataset had more than one element."));
+  }
+};
+
 class SaveIteratorOp : public OpKernel {
  public:
   explicit SaveIteratorOp(OpKernelConstruction* ctx) : OpKernel(ctx) {}
@@ -609,6 +647,8 @@ class IteratorFromStringHandleOp : public OpKernel {
 REGISTER_KERNEL_BUILDER(Name("Iterator").Device(DEVICE_CPU), IteratorHandleOp);
 REGISTER_KERNEL_BUILDER(Name("MakeIterator").Device(DEVICE_CPU),
                         MakeIteratorOp);
+REGISTER_KERNEL_BUILDER(Name("DatasetToSingleElement").Device(DEVICE_CPU),
+                        ToSingleElementOp);
 REGISTER_KERNEL_BUILDER(Name("SaveIterator").Device(DEVICE_CPU),
                         SaveIteratorOp);
 REGISTER_KERNEL_BUILDER(Name("RestoreIterator").Device(DEVICE_CPU),
diff --git a/tensorflow/core/ops/dataset_ops.cc b/tensorflow/core/ops/dataset_ops.cc
index ac15a3f71b..fe346b5240 100644
--- a/tensorflow/core/ops/dataset_ops.cc
+++ b/tensorflow/core/ops/dataset_ops.cc
@@ -653,6 +653,36 @@ REGISTER_OP("IteratorGetNext")
 Gets the next output from the given iterator.
 )doc");
 
+REGISTER_OP("DatasetToSingleElement")
+    .Input("dataset: variant")
+    .Output("components: output_types")
+    .Attr("output_types: list(type) >= 1")
+    .Attr("output_shapes: list(shape) >= 1")
+    .SetShapeFn([](shape_inference::InferenceContext* c) {
+      shape_inference::ShapeHandle unused;
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 0, &unused));
+      std::vector<PartialTensorShape> output_shapes;
+      TF_RETURN_IF_ERROR(c->GetAttr("output_shapes", &output_shapes));
+      if (output_shapes.size() != c->num_outputs()) {
+        return errors::InvalidArgument(
+            "`output_shapes` must be the same length as `output_types` (",
+            output_shapes.size(), " vs. ", c->num_outputs());
+      }
+      for (size_t i = 0; i < output_shapes.size(); ++i) {
+        shape_inference::ShapeHandle output_shape_handle;
+        TF_RETURN_IF_ERROR(c->MakeShapeFromPartialTensorShape(
+            output_shapes[i], &output_shape_handle));
+        c->set_output(static_cast<int>(i), output_shape_handle);
+      }
+      return Status::OK();
+    })
+    .Doc(R"doc(
+Outputs the single element from the given dataset.
+
+dataset: A handle to a dataset that contains a single element.
+components: The components of the single element of `input`.
+)doc");
+
 REGISTER_OP("IteratorToStringHandle")
     .Input("resource_handle: resource")
     .Output("string_handle: string")
-- 
GitLab


From 27a7e5cfdb4ef9d5e3b710873c428cb44630622a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 11 Oct 2017 22:06:23 -0700
Subject: [PATCH 286/909] Improves error message when labels is None.

PiperOrigin-RevId: 171917834
---
 tensorflow/contrib/estimator/BUILD            |  1 +
 .../estimator/python/estimator/head.py        |  6 ++
 .../estimator/python/estimator/head_test.py   | 29 +++++++
 tensorflow/python/estimator/canned/head.py    | 10 ++-
 .../python/estimator/canned/head_test.py      | 86 +++++++++++++++++++
 5 files changed, 130 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/estimator/BUILD b/tensorflow/contrib/estimator/BUILD
index 3b61afe45e..ddfedba579 100644
--- a/tensorflow/contrib/estimator/BUILD
+++ b/tensorflow/contrib/estimator/BUILD
@@ -149,6 +149,7 @@ py_test(
         "//tensorflow/python:array_ops",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:constant_op",
+        "//tensorflow/python:control_flow_ops",
         "//tensorflow/python:dtypes",
         "//tensorflow/python:errors",
         "//tensorflow/python:framework_ops",
diff --git a/tensorflow/contrib/estimator/python/estimator/head.py b/tensorflow/contrib/estimator/python/estimator/head.py
index 9b14622ff6..e7fe454fbf 100644
--- a/tensorflow/contrib/estimator/python/estimator/head.py
+++ b/tensorflow/contrib/estimator/python/estimator/head.py
@@ -227,6 +227,12 @@ class _MultiLabelHead(head_lib._Head):  # pylint:disable=protected-access
     return self._n_classes
 
   def _process_labels(self, labels):
+    if labels is None:
+      raise ValueError(
+          'You must provide a labels Tensor. Given: None. '
+          'Suggested troubleshooting steps: Check that your data contain '
+          'your label feature. Check that your input_fn properly parses and '
+          'returns labels.')
     if isinstance(labels, sparse_tensor.SparseTensor):
       if labels.dtype == dtypes.string:
         label_ids_values = lookup_ops.index_table_from_tensor(
diff --git a/tensorflow/contrib/estimator/python/estimator/head_test.py b/tensorflow/contrib/estimator/python/estimator/head_test.py
index 9dd9e43327..dcbe62b497 100644
--- a/tensorflow/contrib/estimator/python/estimator/head_test.py
+++ b/tensorflow/contrib/estimator/python/estimator/head_test.py
@@ -32,6 +32,7 @@ from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import sparse_tensor
 from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import string_ops
 from tensorflow.python.platform import test
 from tensorflow.python.saved_model import signature_constants
@@ -261,6 +262,18 @@ class MultiLabelHead(test.TestCase):
         actual_unweighted_loss.eval(
             {labels_placeholder: np.array([1, 1], dtype=np.int64)})
 
+  def test_eval_labels_none(self):
+    """Tests that error is raised when labels is None."""
+    head = head_lib.multi_label_head(n_classes=2)
+
+    with self.assertRaisesRegexp(
+        ValueError, r'You must provide a labels Tensor\. Given: None\.'):
+      head.create_estimator_spec(
+          features={'x': np.array(((42,),), dtype=np.int32)},
+          mode=model_fn.ModeKeys.EVAL,
+          logits=np.array([[-10., 10.], [-15., 10.]], dtype=np.float32),
+          labels=None)
+
   def _test_eval(self, head, logits, labels, expected_loss, expected_metrics):
     spec = head.create_estimator_spec(
         features={'x': np.array(((42,),), dtype=np.int32)},
@@ -504,6 +517,22 @@ class MultiLabelHead(test.TestCase):
       self.assertAllClose(
           expected_unweighted_loss, actual_unweighted_loss.eval(), atol=1e-4)
 
+  def test_train_labels_none(self):
+    """Tests that error is raised when labels is None."""
+    head = head_lib.multi_label_head(n_classes=2)
+    def _no_op_train_fn(loss):
+      del loss
+      return control_flow_ops.no_op()
+
+    with self.assertRaisesRegexp(
+        ValueError, r'You must provide a labels Tensor\. Given: None\.'):
+      head.create_estimator_spec(
+          features={'x': np.array(((42,),), dtype=np.int32)},
+          mode=model_fn.ModeKeys.TRAIN,
+          logits=np.array([[-10., 10.], [-15., 10.]], dtype=np.float32),
+          labels=None,
+          train_op_fn=_no_op_train_fn)
+
   def _test_train(self, head, logits, labels, expected_loss):
     expected_train_result = 'my_train_op'
     def _train_op_fn(loss):
diff --git a/tensorflow/python/estimator/canned/head.py b/tensorflow/python/estimator/canned/head.py
index e53626fc54..b796a3f954 100644
--- a/tensorflow/python/estimator/canned/head.py
+++ b/tensorflow/python/estimator/canned/head.py
@@ -198,6 +198,12 @@ def _maybe_expand_dim(tensor):
 
 def _check_and_reshape_dense_labels(labels, expected_labels_dimension):
   """Checks dense labels type and shape and reshapes to 2D Tensor."""
+  if labels is None:
+    raise ValueError(
+        'You must provide a labels Tensor. Given: None. '
+        'Suggested troubleshooting steps: Check that your data contain '
+        'your label feature. Check that your input_fn properly parses and '
+        'returns labels.')
   with ops.name_scope(None, 'labels', (labels,)) as scope:
     labels = sparse_tensor.convert_to_tensor_or_sparse_tensor(labels)
     if isinstance(labels, sparse_tensor.SparseTensor):
@@ -829,8 +835,8 @@ class _RegressionHeadWithMeanSquaredErrorLoss(_Head):
   def create_loss(self, features, mode, logits, labels):
     """See `Head`."""
     del mode, features  # Unused for this head.
-    labels = _check_and_reshape_dense_labels(
-        math_ops.to_float(labels), self._logits_dimension)
+    labels = _check_and_reshape_dense_labels(labels, self._logits_dimension)
+    labels = math_ops.to_float(labels)
     return LossAndLabels(
         unweighted_loss=losses.mean_squared_error(
             labels=labels, predictions=logits, reduction=losses.Reduction.NONE),
diff --git a/tensorflow/python/estimator/canned/head_test.py b/tensorflow/python/estimator/canned/head_test.py
index 74460fdd0a..22f27a8d5a 100644
--- a/tensorflow/python/estimator/canned/head_test.py
+++ b/tensorflow/python/estimator/canned/head_test.py
@@ -393,6 +393,19 @@ class MultiClassHeadWithSoftmaxCrossEntropyLoss(test.TestCase):
       self.assertAllClose(
           expected_unreduced_loss, unweighted_loss.eval(), rtol=1e-2, atol=1e-2)
 
+  def test_eval_labels_none(self):
+    """Tests that error is raised when labels is None."""
+    head = head_lib._multi_class_head_with_softmax_cross_entropy_loss(
+        n_classes=3)
+
+    with self.assertRaisesRegexp(
+        ValueError, r'You must provide a labels Tensor\. Given: None\.'):
+      head.create_estimator_spec(
+          features={'x': np.array(((42,),), dtype=np.int32)},
+          mode=model_fn.ModeKeys.EVAL,
+          logits=np.array(((10, 0, 0), (0, 10, 0),), dtype=np.float32),
+          labels=None)
+
   def test_eval(self):
     n_classes = 3
     head = head_lib._multi_class_head_with_softmax_cross_entropy_loss(n_classes)
@@ -582,6 +595,23 @@ class MultiClassHeadWithSoftmaxCrossEntropyLoss(test.TestCase):
       self.assertAllClose(
           expected_unreduced_loss, unweighted_loss.eval(), rtol=1e-2, atol=1e-2)
 
+  def test_train_labels_none(self):
+    """Tests that error is raised when labels is None."""
+    head = head_lib._multi_class_head_with_softmax_cross_entropy_loss(
+        n_classes=3)
+    def _no_op_train_fn(loss):
+      del loss
+      return control_flow_ops.no_op()
+
+    with self.assertRaisesRegexp(
+        ValueError, r'You must provide a labels Tensor\. Given: None\.'):
+      head.create_estimator_spec(
+          features={'x': np.array(((42,),), dtype=np.int32)},
+          mode=model_fn.ModeKeys.TRAIN,
+          logits=np.array(((10, 0, 0), (0, 10, 0),), dtype=np.float32),
+          labels=None,
+          train_op_fn=_no_op_train_fn)
+
   def test_train(self):
     n_classes = 3
     head = head_lib._multi_class_head_with_softmax_cross_entropy_loss(n_classes)
@@ -1053,6 +1083,18 @@ class BinaryLogisticHeadWithSigmoidCrossEntropyLossTest(test.TestCase):
       self.assertAllClose(
           expected_unreduced_loss, unweighted_loss.eval(), rtol=1e-2, atol=1e-2)
 
+  def test_eval_labels_none(self):
+    """Tests that error is raised when labels is None."""
+    head = head_lib._binary_logistic_head_with_sigmoid_cross_entropy_loss()
+
+    with self.assertRaisesRegexp(
+        ValueError, r'You must provide a labels Tensor\. Given: None\.'):
+      head.create_estimator_spec(
+          features={'x': np.array(((42,),), dtype=np.int32)},
+          mode=model_fn.ModeKeys.EVAL,
+          logits=np.array(((45,), (-41,),), dtype=np.float32),
+          labels=None)
+
   def test_eval(self):
     head = head_lib._binary_logistic_head_with_sigmoid_cross_entropy_loss()
     logits = np.array(((45,), (-41,),), dtype=np.float32)
@@ -1257,6 +1299,22 @@ class BinaryLogisticHeadWithSigmoidCrossEntropyLossTest(test.TestCase):
       _initialize_variables(self, monitored_session.Scaffold())
       self.assertAllClose(expected_unreduced_loss, unweighted_loss.eval())
 
+  def test_train_labels_none(self):
+    """Tests that error is raised when labels is None."""
+    head = head_lib._binary_logistic_head_with_sigmoid_cross_entropy_loss()
+    def _no_op_train_fn(loss):
+      del loss
+      return control_flow_ops.no_op()
+
+    with self.assertRaisesRegexp(
+        ValueError, r'You must provide a labels Tensor\. Given: None\.'):
+      head.create_estimator_spec(
+          features={'x': np.array(((42,),), dtype=np.int32)},
+          mode=model_fn.ModeKeys.TRAIN,
+          logits=np.array(((45,), (-41,),), dtype=np.float32),
+          labels=None,
+          train_op_fn=_no_op_train_fn)
+
   def test_train(self):
     head = head_lib._binary_logistic_head_with_sigmoid_cross_entropy_loss()
 
@@ -1841,6 +1899,18 @@ class RegressionHeadWithMeanSquaredErrorLossTest(test.TestCase):
       # loss = [(43-45)^2, (44-41)] = [4, 9]
       self.assertAllClose(np.array(((4.,), (9.,),)), unweighted_loss.eval())
 
+  def test_eval_labels_none(self):
+    """Tests that error is raised when labels is None."""
+    head = head_lib._regression_head_with_mean_squared_error_loss()
+
+    with self.assertRaisesRegexp(
+        ValueError, r'You must provide a labels Tensor\. Given: None\.'):
+      head.create_estimator_spec(
+          features={'x': np.array(((42,),), dtype=np.int32)},
+          mode=model_fn.ModeKeys.EVAL,
+          logits=np.array(((45,), (41,),), dtype=np.float32),
+          labels=None)
+
   def test_eval(self):
     head = head_lib._regression_head_with_mean_squared_error_loss()
     self.assertEqual(1, head.logits_dimension)
@@ -1899,6 +1969,22 @@ class RegressionHeadWithMeanSquaredErrorLossTest(test.TestCase):
       # loss = [(43-45)^2, (44-41)] = [4, 9]
       self.assertAllClose(np.array(((4.,), (9.,),)), unweighted_loss.eval())
 
+  def test_train_labels_none(self):
+    """Tests that error is raised when labels is None."""
+    head = head_lib._regression_head_with_mean_squared_error_loss()
+    def _no_op_train_fn(loss):
+      del loss
+      return control_flow_ops.no_op()
+
+    with self.assertRaisesRegexp(
+        ValueError, r'You must provide a labels Tensor\. Given: None\.'):
+      head.create_estimator_spec(
+          features={'x': np.array(((42,),), dtype=np.int32)},
+          mode=model_fn.ModeKeys.TRAIN,
+          logits=np.array(((45,), (41,),), dtype=np.float32),
+          labels=None,
+          train_op_fn=_no_op_train_fn)
+
   def test_train(self):
     head = head_lib._regression_head_with_mean_squared_error_loss()
     self.assertEqual(1, head.logits_dimension)
-- 
GitLab


From e0e6fee48fbba939cef757b4f0f576de4dc57449 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 11 Oct 2017 22:06:52 -0700
Subject: [PATCH 287/909] Update ops-related pbtxt files.

PiperOrigin-RevId: 171917856
---
 .../core/ops/compat/ops_history.v1.pbtxt      | 23 ++++++++++++++++
 tensorflow/core/ops/ops.pbtxt                 | 26 +++++++++++++++++++
 2 files changed, 49 insertions(+)

diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index cccd2d6f97..2015acb1c4 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -7980,6 +7980,29 @@ op {
     }
   }
 }
+op {
+  name: "DatasetToSingleElement"
+  input_arg {
+    name: "dataset"
+    type: DT_VARIANT
+  }
+  output_arg {
+    name: "components"
+    type_list_attr: "output_types"
+  }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
+  }
+}
 op {
   name: "DebugGradientIdentity"
   input_arg {
diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index cfe7504988..15de2d2155 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -5961,6 +5961,32 @@ op {
   summary: "Compute the cumulative sum of the tensor `x` along `axis`."
   description: "By default, this op performs an inclusive cumsum, which means that the first\nelement of the input is identical to the first element of the output:\n\n```python\ntf.cumsum([a, b, c])  # => [a, a + b, a + b + c]\n```\n\nBy setting the `exclusive` kwarg to `True`, an exclusive cumsum is\nperformed instead:\n\n```python\ntf.cumsum([a, b, c], exclusive=True)  # => [0, a, a + b]\n```\n\nBy setting the `reverse` kwarg to `True`, the cumsum is performed in the\nopposite direction:\n\n```python\ntf.cumsum([a, b, c], reverse=True)  # => [a + b + c, b + c, c]\n```\n\nThis is more efficient than using separate `tf.reverse` ops.\n\nThe `reverse` and `exclusive` kwargs can also be combined:\n\n```python\ntf.cumsum([a, b, c], exclusive=True, reverse=True)  # => [b + c, c, 0]\n```"
 }
+op {
+  name: "DatasetToSingleElement"
+  input_arg {
+    name: "dataset"
+    description: "A handle to a dataset that contains a single element."
+    type: DT_VARIANT
+  }
+  output_arg {
+    name: "components"
+    description: "The components of the single element of `input`."
+    type_list_attr: "output_types"
+  }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
+  }
+  summary: "Outputs the single element from the given dataset."
+}
 op {
   name: "DebugGradientIdentity"
   input_arg {
-- 
GitLab


From e32292c1091d28fe2f8f21ef69b6be606ac08781 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 11 Oct 2017 22:13:11 -0700
Subject: [PATCH 288/909] Go: Update generated wrapper functions for TensorFlow
 ops.

PiperOrigin-RevId: 171918115
---
 tensorflow/go/op/wrappers.go | 33 +++++++++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)

diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go
index 0da7d5e199..8f5ee9c3df 100644
--- a/tensorflow/go/op/wrappers.go
+++ b/tensorflow/go/op/wrappers.go
@@ -5477,6 +5477,39 @@ func IteratorToStringHandle(scope *Scope, resource_handle tf.Output) (string_han
 	return op.Output(0)
 }
 
+// Outputs the single element from the given dataset.
+//
+// Arguments:
+//	dataset: A handle to a dataset that contains a single element.
+//
+//
+//
+// Returns The components of the single element of `input`.
+func DatasetToSingleElement(scope *Scope, dataset tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (components []tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
+	opspec := tf.OpSpec{
+		Type: "DatasetToSingleElement",
+		Input: []tf.Input{
+			dataset,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	if scope.Err() != nil {
+		return
+	}
+	var idx int
+	var err error
+	if components, idx, err = makeOutputList(op, idx, "components"); err != nil {
+		scope.UpdateErr("DatasetToSingleElement", err)
+		return
+	}
+	return components
+}
+
 // Gets the next output from the given iterator.
 func IteratorGetNext(scope *Scope, iterator tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (components []tf.Output) {
 	if scope.Err() != nil {
-- 
GitLab


From d455bd6a851450657c702808d096f39583b949b5 Mon Sep 17 00:00:00 2001
From: Martin Wicke <wicke@google.com>
Date: Wed, 11 Oct 2017 22:34:50 -0700
Subject: [PATCH 289/909] Add more explicit carve-out for experimental proto
 fields.

PiperOrigin-RevId: 171919244
---
 tensorflow/docs_src/programmers_guide/version_compat.md | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tensorflow/docs_src/programmers_guide/version_compat.md b/tensorflow/docs_src/programmers_guide/version_compat.md
index db6d596acf..d3e8e42509 100644
--- a/tensorflow/docs_src/programmers_guide/version_compat.md
+++ b/tensorflow/docs_src/programmers_guide/version_compat.md
@@ -67,7 +67,9 @@ backward incompatible ways between minor releases. These include:
 
 *   **Experimental APIs**: The @{tf.contrib} module and its submodules in Python
     and any functions in the C API or fields in protocol buffers that are
-    explicitly commented as being experimental.
+    explicitly commented as being experimental. In particular, any field in a
+    protocol buffer which is called "experimental" and all its fields and
+    submessages can change at any time.
 
 *   **Other languages**: TensorFlow APIs in languages other than Python and C,
     such as:
-- 
GitLab


From cec93f10dcf5d2e647a41bd6bf95357cf9d9169d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 11 Oct 2017 23:59:11 -0700
Subject: [PATCH 290/909] Optimized C++ and CUDA kernels for matrix_set_diag
 op. The new code is faster and more readable and avoids an issue with using
 the Eigen generator mechanism with GPUs on Windows.

PiperOrigin-RevId: 171924800
---
 tensorflow/core/kernels/cholesky_op.cc        | 15 ++--
 .../core/kernels/matrix_band_part_op.cc       |  5 +-
 .../kernels/matrix_band_part_op_gpu.cu.cc     |  5 +-
 tensorflow/core/kernels/matrix_inverse_op.cc  | 13 ++--
 tensorflow/core/kernels/matrix_set_diag_op.cc | 71 ++++++++-----------
 tensorflow/core/kernels/matrix_set_diag_op.h  | 68 ++----------------
 .../core/kernels/matrix_set_diag_op_gpu.cu.cc | 70 ++++++++++++++++--
 7 files changed, 121 insertions(+), 126 deletions(-)

diff --git a/tensorflow/core/kernels/cholesky_op.cc b/tensorflow/core/kernels/cholesky_op.cc
index 8b401a565b..bcd42dc8d7 100644
--- a/tensorflow/core/kernels/cholesky_op.cc
+++ b/tensorflow/core/kernels/cholesky_op.cc
@@ -112,6 +112,14 @@ class CholeskyOpGpu : public AsyncOpKernel {
                                 input.dim_size(ndims - 2), " != ", n),
         done);
 
+    if (input.NumElements() == 0) {
+      // If X is an empty matrix (0 rows, 0 col), X * X' == X.
+      // Therefore, we return X.
+      context->set_output(0, input);
+      done();
+      return;
+    }
+
     // Allocate output.
     // TODO(rmlarsen): Convert to std::make_unique when available.
     std::unique_ptr<CudaSolver> solver(new CudaSolver(context));
@@ -121,13 +129,6 @@ class CholeskyOpGpu : public AsyncOpKernel {
                              {0}, 0, input.shape(), &output),
                          done);
 
-    if (n == 0) {
-      // If X is an empty matrix (0 rows, 0 col), X * X' == X.
-      // Therefore, we return X.
-      done();
-      return;
-    }
-
     // Copy the lower triangular part of the input matrices to the output and
     // set the strictly upper triangular part to zero. We use a pre-existing
     // kernel MatrixBandPart to do this for all matrices in the batch at once,
diff --git a/tensorflow/core/kernels/matrix_band_part_op.cc b/tensorflow/core/kernels/matrix_band_part_op.cc
index e5f9086dba..d7fff4bb0c 100644
--- a/tensorflow/core/kernels/matrix_band_part_op.cc
+++ b/tensorflow/core/kernels/matrix_band_part_op.cc
@@ -80,8 +80,9 @@ class MatrixBandPartOp : public OpKernel {
                                         input_reshaped.dimension(2),
                                         ") got: ", num_upper));
 
-    if ((num_lower < 0 || num_lower == input_reshaped.dimension(1)) &&
-        (num_upper < 0 || num_upper == input_reshaped.dimension(2))) {
+    if (input.NumElements() == 0 ||
+        ((num_lower < 0 || num_lower == input_reshaped.dimension(1)) &&
+         (num_upper < 0 || num_upper == input_reshaped.dimension(2)))) {
       // This is a no-op.
       context->set_output(0, input);
       return;
diff --git a/tensorflow/core/kernels/matrix_band_part_op_gpu.cu.cc b/tensorflow/core/kernels/matrix_band_part_op_gpu.cu.cc
index 41b2f5c0ef..628d22b458 100644
--- a/tensorflow/core/kernels/matrix_band_part_op_gpu.cu.cc
+++ b/tensorflow/core/kernels/matrix_band_part_op_gpu.cu.cc
@@ -54,17 +54,14 @@ struct MatrixBandPartFunctor<GPUDevice, Scalar> {
                   int num_lower_diags, int num_upper_diags,
                   typename TTypes<Scalar, 3>::ConstTensor input,
                   typename TTypes<Scalar, 3>::Tensor output) {
-    using CudaType = typename CUDAComplexT<Scalar>::type;
     const int batch_size = input.dimension(0);
     const int m = input.dimension(1);
     const int n = input.dimension(2);
-    const CudaType* input_ptr = reinterpret_cast<const CudaType*>(input.data());
-    CudaType* output_ptr = reinterpret_cast<CudaType*>(output.data());
     CudaLaunchConfig config = GetCudaLaunchConfig(batch_size * m * n, device);
     MatrixBandPartKernel<<<config.block_count, config.thread_per_block, 0,
                            device.stream()>>>(
         config.virtual_thread_count, batch_size, m, n, num_lower_diags,
-        num_upper_diags, input_ptr, output_ptr);
+        num_upper_diags, input.data(), output.data());
   }
 };
 
diff --git a/tensorflow/core/kernels/matrix_inverse_op.cc b/tensorflow/core/kernels/matrix_inverse_op.cc
index a152b5cbee..832e508bb7 100644
--- a/tensorflow/core/kernels/matrix_inverse_op.cc
+++ b/tensorflow/core/kernels/matrix_inverse_op.cc
@@ -109,6 +109,13 @@ class MatrixInverseOpGpu : public AsyncOpKernel {
                                 input.dim_size(ndims - 2), " != ", n),
         done);
 
+    // By definition, an empty matrix's inverse is an empty matrix.
+    if (input.NumElements() == 0) {
+      context->set_output(0, input);
+      done();
+      return;
+    }
+
     // Allocate output.
     Tensor* output;
     OP_REQUIRES_OK_ASYNC(context,
@@ -116,12 +123,6 @@ class MatrixInverseOpGpu : public AsyncOpKernel {
                              {0}, 0, input.shape(), &output),
                          done);
 
-    // By definition, an empty matrix's inverse is an empty matrix.
-    if (input.NumElements() == 0) {
-      done();
-      return;
-    }
-
     // TODO(rmlarsen): Convert to std::make_unique when available.
     std::unique_ptr<CudaSolver> solver(new CudaSolver(context));
 
diff --git a/tensorflow/core/kernels/matrix_set_diag_op.cc b/tensorflow/core/kernels/matrix_set_diag_op.cc
index 9573c4f8d1..9dd665392b 100644
--- a/tensorflow/core/kernels/matrix_set_diag_op.cc
+++ b/tensorflow/core/kernels/matrix_set_diag_op.cc
@@ -23,8 +23,6 @@ limitations under the License.
 
 #include "tensorflow/core/kernels/matrix_set_diag_op.h"
 
-#include <memory>
-#include <vector>
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
@@ -32,6 +30,7 @@ limitations under the License.
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/framework/types.h"
+#include "tensorflow/core/lib/core/threadpool.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/macros.h"
 
@@ -73,22 +72,21 @@ class MatrixSetDiagOp : public OpKernel {
                     input_shape.DebugString(), " and diagonal shape: ",
                     diag_shape.DebugString()));
 
+    if (input.NumElements() == 0) {
+      // This is a no-op.
+      context->set_output(0, input);
+      return;
+    }
+
     auto input_reshaped = input.flat_inner_dims<T, 3>();
     auto diag_reshaped = diag.flat_inner_dims<T, 2>();
-
     Tensor* output = nullptr;
     OP_REQUIRES_OK(context, context->forward_input_or_allocate_output(
                                 {0}, 0, input_shape, &output));
     auto output_reshaped = output->flat_inner_dims<T, 3>();
-    Tensor scratch_tensor;
-    OP_REQUIRES_OK(context,
-                   context->allocate_temp(DataTypeToEnum<T>::value,
-                                          TensorShape({}), &scratch_tensor));
-    auto scratch = scratch_tensor.scalar<T>();
-
-    functor::MatrixSetDiag<Device, T>::Compute(context->eigen_device<Device>(),
-                                               input_reshaped, diag_reshaped,
-                                               scratch, output_reshaped);
+    functor::MatrixSetDiag<Device, T>::Compute(
+        context, context->eigen_device<Device>(), input_reshaped, diag_reshaped,
+        output_reshaped);
   }
 
  private:
@@ -116,32 +114,25 @@ namespace functor {
 // Implementation of the functor specialization for CPU.
 template <typename T>
 struct MatrixSetDiag<CPUDevice, T> {
-  static void Compute(const CPUDevice& d,
+  static void Compute(OpKernelContext* context, const CPUDevice& device,
                       typename TTypes<T, 3>::ConstTensor input,
                       typename TTypes<T, 2>::ConstTensor diag,
-                      typename TTypes<T>::Scalar scratch,
                       typename TTypes<T, 3>::Tensor output) {
-    output.device(d) = input;
-    for (int64 r = 0; r < output.dimension(0); ++r) {
-      for (int64 d = 0; d < diag.dimension(1); ++d) {
-        output(r, d, d) = diag(r, d);
-      }
+    if (input.data() != output.data()) {
+      output.device(device) = input;
     }
-  }
-};
-
-template <>
-struct MatrixSetDiag<CPUDevice, bool> {
-  static void Compute(const CPUDevice& d, TTypes<bool, 3>::ConstTensor input,
-                      TTypes<bool, 2>::ConstTensor diag,
-                      TTypes<bool>::Scalar scratch,
-                      TTypes<bool, 3>::Tensor output) {
-    output.device(d) = input;
-    for (int64 r = 0; r < output.dimension(0); ++r) {
-      for (int64 d = 0; d < diag.dimension(1); ++d) {
-        output(r, d, d) = diag(r, d);
+    auto compute_shard = [&output, &diag](int64 begin, int64 end) {
+      for (int64 batch = begin; batch < end; ++batch) {
+        for (int64 col = 0; col < diag.dimension(1); ++col) {
+          output(batch, col, col) = diag(batch, col);
+        }
       }
-    }
+    };
+    auto thread_pool =
+        context->device()->tensorflow_cpu_worker_threads()->workers;
+    int64 cost_per_batch = 10 * output.dimension(1);  // Heuristic.
+    thread_pool->ParallelFor(output.dimension(0), cost_per_batch,
+                             std::move(compute_shard));
   }
 };
 
@@ -151,13 +142,13 @@ struct MatrixSetDiag<CPUDevice, bool> {
 
 // Forward declarations of the functor specializations for GPU.
 namespace functor {
-#define DECLARE_GPU_SPEC(T)                                         \
-  template <>                                                       \
-  void MatrixSetDiag<GPUDevice, T>::Compute(                        \
-      const GPUDevice& d, typename TTypes<T, 3>::ConstTensor input, \
-      typename TTypes<T, 2>::ConstTensor diag,                      \
-      typename TTypes<T>::Scalar scratch,                           \
-      typename TTypes<T, 3>::Tensor output);                        \
+#define DECLARE_GPU_SPEC(T)                         \
+  template <>                                       \
+  void MatrixSetDiag<GPUDevice, T>::Compute(        \
+      OpKernelContext* context, const GPUDevice& d, \
+      typename TTypes<T, 3>::ConstTensor input,     \
+      typename TTypes<T, 2>::ConstTensor diag,      \
+      typename TTypes<T, 3>::Tensor output);        \
   extern template struct MatrixSetDiag<GPUDevice, T>;
 
 TF_CALL_GPU_NUMBER_TYPES(DECLARE_GPU_SPEC);
diff --git a/tensorflow/core/kernels/matrix_set_diag_op.h b/tensorflow/core/kernels/matrix_set_diag_op.h
index 63e5650bf0..aeb144559f 100644
--- a/tensorflow/core/kernels/matrix_set_diag_op.h
+++ b/tensorflow/core/kernels/matrix_set_diag_op.h
@@ -16,80 +16,22 @@ limitations under the License.
 #ifndef TENSORFLOW_KERNELS_MATRIX_SET_DIAG_OP_H_
 #define TENSORFLOW_KERNELS_MATRIX_SET_DIAG_OP_H_
 
-// Generator definition for MatrixSetDiagOp, must be compilable by nvcc.
-
-#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/platform/types.h"
 
 namespace tensorflow {
-
-namespace generator {
-
-template <typename T>
-class OverwriteDiagGenerator {
- public:
-  EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
-  OverwriteDiagGenerator(typename TTypes<T, 2>::ConstTensor diag,
-                         typename TTypes<T, 3>::Tensor output)
-      : diag_(diag), output_(output) {}
-
-  EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE T
-  operator()(const Eigen::array<Eigen::DenseIndex, 2>& coords) const {
-    Eigen::array<Eigen::DenseIndex, 3> diag_from_coords(
-        {coords[0], coords[1], coords[1]});
-
-    // This is the side effect we care about.
-    output_(diag_from_coords) = diag_(coords);
-
-    return T(0);
-  }
-
- private:
-  typename TTypes<T, 2>::ConstTensor diag_;
-  mutable typename TTypes<T, 3>::Tensor output_;
-};
-
-}  // namespace generator
-
 namespace functor {
 
 template <typename Device, typename T>
 struct MatrixSetDiag {
-  EIGEN_ALWAYS_INLINE static void Compute(
-      const Device& d, typename TTypes<T, 3>::ConstTensor input,
-      typename TTypes<T, 2>::ConstTensor diag,
-      typename TTypes<T>::Scalar scratch,
-      typename TTypes<T, 3>::Tensor output) {
-    output.device(d) = input;
-    generator::OverwriteDiagGenerator<T> generator(diag, output);
-    // Use sum() to force the generation to aggregate to the scalar
-    // output scratch.  This in turn forces each element of the
-    // generator to execute.  The side effect of the execution is to
-    // update the diagonal components of output with diag.
-    scratch.device(d) = diag.generate(generator).sum();
-  }
-};
-
-template <typename Device>
-struct MatrixSetDiag<Device, bool> {
-  EIGEN_ALWAYS_INLINE static void Compute(const Device& d,
-                                          TTypes<bool, 3>::ConstTensor input,
-                                          TTypes<bool, 2>::ConstTensor diag,
-                                          TTypes<bool>::Scalar scratch,
-                                          TTypes<bool, 3>::Tensor output) {
-    output.device(d) = input;
-    generator::OverwriteDiagGenerator<bool> generator(diag, output);
-    // Use all() to force the generation to aggregate to the scalar
-    // output scratch.  This in turn forces each element of the
-    // generator to execute.  The side effect of the execution is to
-    // update the diagonal components of output with diag.
-    scratch.device(d) = diag.generate(generator).all();
-  }
+  static void Compute(OpKernelContext* context, const Device& d,
+                      typename TTypes<T, 3>::ConstTensor input,
+                      typename TTypes<T, 2>::ConstTensor diag,
+                      typename TTypes<T, 3>::Tensor output);
 };
 
 }  // namespace functor
-
 }  // namespace tensorflow
 
 #endif  // TENSORFLOW_KERNELS_MATRIX_SET_DIAG_OP_H_
diff --git a/tensorflow/core/kernels/matrix_set_diag_op_gpu.cu.cc b/tensorflow/core/kernels/matrix_set_diag_op_gpu.cu.cc
index 8e41ce5860..35037b8e14 100644
--- a/tensorflow/core/kernels/matrix_set_diag_op_gpu.cu.cc
+++ b/tensorflow/core/kernels/matrix_set_diag_op_gpu.cu.cc
@@ -19,20 +19,82 @@ limitations under the License.
 
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/kernels/matrix_set_diag_op.h"
+#include "tensorflow/core/util/cuda_kernel_helper.h"
 
 namespace tensorflow {
+namespace functor {
 
 typedef Eigen::GpuDevice GPUDevice;
 
-#define DEFINE_GPU_SPEC(T)                             \
-  template class generator::OverwriteDiagGenerator<T>; \
-  template struct functor::MatrixSetDiag<GPUDevice, T>;
+template <typename Scalar>
+__global__ void MatrixSetDiagKernel(const int num_threads, const int m,
+                                    const int n, const int minsize,
+                                    const Scalar* diag_ptr,
+                                    Scalar* output_ptr) {
+  CUDA_1D_KERNEL_LOOP(index, num_threads) {
+    const int batch = index / minsize;
+    const int col = index - batch * minsize;
+    const int out_index = batch * m * n + (n + 1) * col;
+    output_ptr[out_index] = diag_ptr[index];
+  }
+}
+
+template <typename Scalar>
+__global__ void MatrixCopyInputAndSetDiagKernel(
+    const int num_threads, const int m, const int n, const int minsize,
+    const Scalar* input_ptr, const Scalar* diag_ptr, Scalar* output_ptr) {
+  CUDA_1D_KERNEL_LOOP(index, num_threads) {
+    const int global_row = index / n;
+    const int col = index - global_row * n;
+    const int batch = global_row / m;
+    const int row = global_row - batch * m;
+    if (col == row) {
+      // Because col = index % n, and row = (index / n) % m,
+      // we know that col==row => col < minsize, so the following is safe:
+      output_ptr[index] = diag_ptr[batch * minsize + col];
+    } else {
+      output_ptr[index] = input_ptr[index];
+    }
+  }
+}
+
+template <typename Scalar>
+struct MatrixSetDiag<GPUDevice, Scalar> {
+  static void Compute(OpKernelContext* context, const GPUDevice& device,
+                      typename TTypes<Scalar, 3>::ConstTensor input,
+                      typename TTypes<Scalar, 2>::ConstTensor diag,
+                      typename TTypes<Scalar, 3>::Tensor output) {
+    const int batch_size = input.dimension(0);
+    const int m = input.dimension(1);
+    const int n = input.dimension(2);
+    const int minsize = std::min(m, n);
+    CHECK_EQ(diag.dimension(1), minsize);
+    if (batch_size == 0 || minsize == 0) return;
+    if (input.data() == output.data()) {
+      CudaLaunchConfig config =
+          GetCudaLaunchConfig(batch_size * minsize, device);
+      MatrixSetDiagKernel<Scalar>
+          <<<config.block_count, config.thread_per_block, 0, device.stream()>>>(
+              config.virtual_thread_count, m, n, minsize, diag.data(),
+              output.data());
+    } else {
+      CudaLaunchConfig config = GetCudaLaunchConfig(batch_size * m * n, device);
+      MatrixCopyInputAndSetDiagKernel<Scalar>
+          <<<config.block_count, config.thread_per_block, 0, device.stream()>>>(
+              config.virtual_thread_count, m, n, minsize, input.data(),
+              diag.data(), output.data());
+    }
+  }
+};
+
+#define DEFINE_GPU_SPEC(T) template struct MatrixSetDiag<GPUDevice, T>;
 
 TF_CALL_GPU_NUMBER_TYPES(DEFINE_GPU_SPEC);
 TF_CALL_bool(DEFINE_GPU_SPEC);
 TF_CALL_complex64(DEFINE_GPU_SPEC);
 TF_CALL_complex128(DEFINE_GPU_SPEC);
 
-}  // end namespace tensorflow
+}  // namespace functor
+}  // namespace tensorflow
 
 #endif  // GOOGLE_CUDA
-- 
GitLab


From 9148cddacffadafbfa2faeba25c5bc0273f60528 Mon Sep 17 00:00:00 2001
From: Benoit Steiner <bsteiner@google.com>
Date: Thu, 12 Oct 2017 01:26:31 -0700
Subject: [PATCH 291/909] Made sure that the virtual placer correctly handles
 short device names

PiperOrigin-RevId: 171931173
---
 .../core/grappler/costs/virtual_placer.cc     |  7 +++--
 .../grappler/costs/virtual_placer_test.cc     | 27 +++++++++++++++++++
 2 files changed, 32 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/grappler/costs/virtual_placer.cc b/tensorflow/core/grappler/costs/virtual_placer.cc
index 965a2d2517..8f5f16e490 100644
--- a/tensorflow/core/grappler/costs/virtual_placer.cc
+++ b/tensorflow/core/grappler/costs/virtual_placer.cc
@@ -25,6 +25,11 @@ namespace grappler {
 
 VirtualPlacer::VirtualPlacer(const Cluster* cluster) {
   CHECK(cluster);
+
+  // Default job name for canonical device name. Needs to be set before the
+  // first call to to_lfqn_or_empty()
+  default_job_name_lowercase_ = "localhost";
+
   devices_ = cluster->GetDevices();
   lfqn_map_.reserve(devices_.size());
   for (const auto& kv : devices_) {
@@ -83,8 +88,6 @@ VirtualPlacer::VirtualPlacer(const Cluster* cluster) {
     }
   }
 
-  // Default job name for canonical device name.
-  default_job_name_lowercase_ = "localhost";
   // Scan the device names from the cluster, and if there is one job name used,
   // use it for canonical device name.
   std::unordered_set<string> job_names_from_cluster;
diff --git a/tensorflow/core/grappler/costs/virtual_placer_test.cc b/tensorflow/core/grappler/costs/virtual_placer_test.cc
index 1c2e2815a6..d1f9cd2176 100644
--- a/tensorflow/core/grappler/costs/virtual_placer_test.cc
+++ b/tensorflow/core/grappler/costs/virtual_placer_test.cc
@@ -53,6 +53,33 @@ TEST(VirtualPlacerTest, LocalDevices) {
             placer.get_canonical_device_name(node));
 }
 
+TEST(VirtualPlacerTest, ShortNames) {
+  // Create a virtual cluster with a local CPU and a local GPU
+  std::unordered_map<string, DeviceProperties> devices;
+  DeviceProperties cpu_device;
+  cpu_device.set_type("CPU");
+  devices["/CPU:0"] = cpu_device;
+  DeviceProperties gpu_device;
+  gpu_device.set_type("GPU");
+  devices["/GPU:0"] = gpu_device;
+  VirtualCluster cluster(devices);
+  VirtualPlacer placer(&cluster);
+
+  NodeDef node;
+  node.set_op("Conv2D");
+  // node.device() is empty, but GPU is default device if there is.
+  EXPECT_EQ("GPU", placer.get_device(node).type());
+  EXPECT_EQ("/GPU:0", placer.get_canonical_device_name(node));
+
+  node.set_device("CPU");
+  EXPECT_EQ("CPU", placer.get_device(node).type());
+  EXPECT_EQ("/CPU:0", placer.get_canonical_device_name(node));
+
+  node.set_device("GPU:0");
+  EXPECT_EQ("GPU", placer.get_device(node).type());
+  EXPECT_EQ("/GPU:0", placer.get_canonical_device_name(node));
+}
+
 TEST(VirtualPlacerTest, PlacementOnNonDefaultDevice) {
   // Create a virtual cluster with a CPU and a device:TPU
   // Test that placement on TPU works
-- 
GitLab


From f4d591da72033730db3c7d741a837da3174fd5bc Mon Sep 17 00:00:00 2001
From: Atlas7 <johnnychan0302@gmail.com>
Date: Thu, 12 Oct 2017 10:43:31 +0100
Subject: [PATCH 292/909] add code sample on inspect checkpoint variables

---
 .../docs_src/programmers_guide/saved_model.md | 33 +++++++++++++++++++
 1 file changed, 33 insertions(+)

diff --git a/tensorflow/docs_src/programmers_guide/saved_model.md b/tensorflow/docs_src/programmers_guide/saved_model.md
index 9262143ad8..6bc2cbb9e3 100644
--- a/tensorflow/docs_src/programmers_guide/saved_model.md
+++ b/tensorflow/docs_src/programmers_guide/saved_model.md
@@ -158,6 +158,39 @@ Notes:
    optionally choose names for the variables in the checkpoint files.
 
 
+### Inspect variables in a checkpoint
+
+We can quickly inspect variables in a checkpoint with the 
+[`inspect_checkpoint`](https://www.tensorflow.org/code/tensorflow/python/tools/inspect_checkpoint.py) library.
+
+Continuing from the save/restore examples shown earlier:
+
+```python
+# import the inspect_checkpoint library
+from tensorflow.python.tools import inspect_checkpoint as chkp
+
+# print all tensors in checkpoint file
+chkp.print_tensors_in_checkpoint_file("/tmp/model.ckpt", tensor_name='', all_tensors=True)
+
+# tensor_name:  v1
+# [ 1.  1.  1.]
+# tensor_name:  v2
+# [-1. -1. -1. -1. -1.]
+
+# print only tensor v1 in checkpoint file
+chkp.print_tensors_in_checkpoint_file("/tmp/model.ckpt", tensor_name='v1', all_tensors=False)
+
+# tensor_name:  v1
+# [ 1.  1.  1.]
+
+# print only tensor v2 in checkpoint file
+chkp.print_tensors_in_checkpoint_file("/tmp/model.ckpt", tensor_name='v2', all_tensors=False)
+
+# tensor_name:  v2
+# [-1. -1. -1. -1. -1.]
+```
+
+
 <a name="models"></a>
 ## Overview of saving and restoring models
 
-- 
GitLab


From 8623af4c20cfec376f90c5e2f59ed4fe96a60dd4 Mon Sep 17 00:00:00 2001
From: Ryohei Kuroki <ryohei.kuroki@gmail.com>
Date: Thu, 12 Oct 2017 22:00:49 +0900
Subject: [PATCH 293/909] MAINTAINER is deprecated, use LABEL instead

---
 tensorflow/tools/docker/Dockerfile           | 2 +-
 tensorflow/tools/docker/Dockerfile.devel     | 2 +-
 tensorflow/tools/docker/Dockerfile.devel-gpu | 2 +-
 tensorflow/tools/docker/Dockerfile.gpu       | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/tools/docker/Dockerfile b/tensorflow/tools/docker/Dockerfile
index 07a972400d..024cb40eb4 100644
--- a/tensorflow/tools/docker/Dockerfile
+++ b/tensorflow/tools/docker/Dockerfile
@@ -1,6 +1,6 @@
 FROM ubuntu:16.04
 
-MAINTAINER Craig Citro <craigcitro@google.com>
+LABEL maintainer="Craig Citro <craigcitro@google.com>"
 
 # Pick up some TF dependencies
 RUN apt-get update && apt-get install -y --no-install-recommends \
diff --git a/tensorflow/tools/docker/Dockerfile.devel b/tensorflow/tools/docker/Dockerfile.devel
index 60a94504b7..2d4f03fbb7 100644
--- a/tensorflow/tools/docker/Dockerfile.devel
+++ b/tensorflow/tools/docker/Dockerfile.devel
@@ -1,6 +1,6 @@
 FROM ubuntu:16.04
 
-MAINTAINER Craig Citro <craigcitro@google.com>
+LABEL maintainer="Craig Citro <craigcitro@google.com>"
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
         build-essential \
diff --git a/tensorflow/tools/docker/Dockerfile.devel-gpu b/tensorflow/tools/docker/Dockerfile.devel-gpu
index a607e5e27b..1b605587ff 100644
--- a/tensorflow/tools/docker/Dockerfile.devel-gpu
+++ b/tensorflow/tools/docker/Dockerfile.devel-gpu
@@ -1,6 +1,6 @@
 FROM nvidia/cuda:8.0-cudnn6-devel-ubuntu16.04
 
-MAINTAINER Craig Citro <craigcitro@google.com>
+LABEL maintainer="Craig Citro <craigcitro@google.com>"
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
         build-essential \
diff --git a/tensorflow/tools/docker/Dockerfile.gpu b/tensorflow/tools/docker/Dockerfile.gpu
index da83a30058..0571dd7391 100644
--- a/tensorflow/tools/docker/Dockerfile.gpu
+++ b/tensorflow/tools/docker/Dockerfile.gpu
@@ -1,6 +1,6 @@
 FROM nvidia/cuda:8.0-cudnn6-devel-ubuntu16.04
 
-MAINTAINER Craig Citro <craigcitro@google.com>
+LABEL maintainer="Craig Citro <craigcitro@google.com>"
 
 # Pick up some TF dependencies
 RUN apt-get update && apt-get install -y --no-install-recommends \
-- 
GitLab


From 66b1615b6e2783c9ddce27e7b084fcc230c3a594 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 12 Oct 2017 07:20:33 -0700
Subject: [PATCH 294/909] BUILD cleanup in contrib/boosted_trees/...

PiperOrigin-RevId: 171956450
---
 tensorflow/contrib/boosted_trees/BUILD        | 71 +++++++++++++------
 .../boosted_trees/estimator_batch/BUILD       | 41 ++++++++---
 tensorflow/contrib/boosted_trees/lib/BUILD    |  4 --
 3 files changed, 79 insertions(+), 37 deletions(-)

diff --git a/tensorflow/contrib/boosted_trees/BUILD b/tensorflow/contrib/boosted_trees/BUILD
index 726a8f692f..f3ae4e3092 100644
--- a/tensorflow/contrib/boosted_trees/BUILD
+++ b/tensorflow/contrib/boosted_trees/BUILD
@@ -68,6 +68,10 @@ py_library(
     srcs = ["python/utils/losses.py"],
     srcs_version = "PY2AND3",
     deps = [
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:control_flow_ops",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:math_ops",
         "//tensorflow/python:nn",
     ],
 )
@@ -82,7 +86,11 @@ py_test(
     ],
     deps = [
         ":losses",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:constant_op",
+        "//tensorflow/python:dtypes",
         "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:platform_test",
         "//third_party/py/numpy",
     ],
 )
@@ -94,13 +102,30 @@ py_library(
     ],
     srcs_version = "PY2AND3",
     deps = [
+        ":gen_model_ops_py",
         "//tensorflow/contrib/boosted_trees:batch_ops_utils_py",
         "//tensorflow/contrib/boosted_trees:boosted_trees_ops_py",
         "//tensorflow/contrib/boosted_trees/lib:categorical_split_handler",
         "//tensorflow/contrib/boosted_trees/lib:ordinal_split_handler",
         "//tensorflow/contrib/boosted_trees/proto:learner_proto_py",
+        "//tensorflow/contrib/layers:layers_py",
         "//tensorflow/contrib/learn",
         "//tensorflow/contrib/stateless",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:constant_op",
+        "//tensorflow/python:control_flow_ops",
+        "//tensorflow/python:dtypes",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:gradients",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:platform",
+        "//tensorflow/python:sparse_tensor",
+        "//tensorflow/python:summary",
+        "//tensorflow/python:tensor_shape",
+        "//tensorflow/python:training",
+        "//tensorflow/python:variable_scope",
+        "//tensorflow/python:variables",
+        "//tensorflow/python/feature_column",
     ],
 )
 
@@ -116,10 +141,19 @@ py_test(
     deps = [
         ":gbdt_batch",
         ":losses",
+        ":model_ops_py",
         "//tensorflow/contrib/boosted_trees/proto:learner_proto_py",
         "//tensorflow/contrib/boosted_trees/proto:tree_config_proto_py",
+        "//tensorflow/contrib/layers:layers_py",
+        "//tensorflow/contrib/learn",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:dtypes",
         "//tensorflow/python:framework_test_lib",
-        "//third_party/py/numpy",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:platform_test",
+        "//tensorflow/python:resources",
+        "//tensorflow/python:sparse_tensor",
+        "//tensorflow/python:variables",
     ],
 )
 
@@ -138,8 +172,6 @@ py_test(
         ":prediction_ops_py",
         "//tensorflow/contrib/boosted_trees/proto:learner_proto_py",
         "//tensorflow/contrib/boosted_trees/proto:tree_config_proto_py",
-        "//tensorflow/python:array_ops",
-        "//tensorflow/python:dtypes",
         "//tensorflow/python:framework_ops",
         "//tensorflow/python:framework_test_lib",
         "//tensorflow/python:platform_test",
@@ -181,6 +213,9 @@ py_test(
     deps = [
         ":quantile_ops_py",
         "//tensorflow/contrib/boosted_trees/proto:quantiles_proto_py",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:constant_op",
+        "//tensorflow/python:dtypes",
         "//tensorflow/python:framework_ops",
         "//tensorflow/python:framework_test_lib",
         "//tensorflow/python:platform_test",
@@ -233,7 +268,6 @@ py_test(
         "nomac",  # b/63258195
     ],
     deps = [
-        ":boosted_trees_ops_loader",
         ":model_ops_py",
         ":training_ops_py",
         "//tensorflow/contrib/boosted_trees/proto:learner_proto_py",
@@ -269,7 +303,9 @@ tf_custom_op_py_library(
     deps = [
         "//tensorflow/contrib/util:util_py",
         "//tensorflow/core:protos_all_py",
+        "//tensorflow/python:errors",
         "//tensorflow/python:framework_for_generated_wrappers",
+        "//tensorflow/python:platform",
         "//tensorflow/python:resources",
     ],
 )
@@ -394,16 +430,12 @@ tf_custom_op_py_library(
 
 tf_kernel_library(
     name = "split_handler_ops_kernels",
-    srcs = [
-        "kernels/split_handler_ops.cc",
-    ],
+    srcs = ["kernels/split_handler_ops.cc"],
     deps = [
         "//tensorflow/contrib/boosted_trees/lib:feature-column-handlers",
         "//tensorflow/contrib/boosted_trees/proto:split_info_proto_cc",
         "//tensorflow/contrib/boosted_trees/proto:tree_config_proto_cc",
-        "//tensorflow/core:framework",
         "//tensorflow/core:framework_headers_lib",
-        "//third_party/eigen3",
     ],
     alwayslink = 1,
 )
@@ -435,8 +467,6 @@ tf_custom_op_py_library(
     deps = [
         ":boosted_trees_ops_loader",
         ":gen_training_ops_py",
-        "//tensorflow/contrib/util:util_py",
-        "//tensorflow/python:framework_for_generated_wrappers",
     ],
 )
 
@@ -451,7 +481,6 @@ tf_kernel_library(
         "//tensorflow/contrib/boosted_trees/proto:split_info_proto_cc",
         "//tensorflow/contrib/boosted_trees/proto:tree_config_proto_cc",
         "//tensorflow/contrib/boosted_trees/resources:decision_tree_ensemble_resource",
-        "//tensorflow/core:framework",
         "//tensorflow/core:framework_headers_lib",
         "//third_party/eigen3",
     ],
@@ -500,7 +529,6 @@ tf_kernel_library(
         "//tensorflow/contrib/boosted_trees/proto:learner_proto_cc",
         "//tensorflow/contrib/boosted_trees/proto:tree_config_proto_cc",
         "//tensorflow/contrib/boosted_trees/resources:decision_tree_ensemble_resource",
-        "//tensorflow/core:framework",
         "//tensorflow/core:framework_headers_lib",
         "//third_party/eigen3",
     ],
@@ -542,16 +570,13 @@ tf_custom_op_py_library(
 
 tf_kernel_library(
     name = "quantile_ops_kernels",
-    srcs = [
-        "kernels/quantile_ops.cc",
-    ],
+    srcs = ["kernels/quantile_ops.cc"],
     deps = [
         "//tensorflow/contrib/boosted_trees/lib:utils",
         "//tensorflow/contrib/boosted_trees/lib:weighted_quantiles",
         "//tensorflow/contrib/boosted_trees/proto:quantiles_proto_cc",
         "//tensorflow/contrib/boosted_trees/resources:quantile_stream_resource",
         "//tensorflow/core:framework_headers_lib",
-        "//third_party/eigen3",
     ],
     alwayslink = 1,
 )
@@ -581,8 +606,6 @@ tf_custom_op_py_library(
         ":batch_ops_utils_py",
         ":boosted_trees_ops_loader",
         ":gen_stats_accumulator_ops_py_wrap",
-        "//tensorflow/contrib/util:util_py",
-        "//tensorflow/core:protos_all_py",
         "//tensorflow/python:framework_for_generated_wrappers",
         "//tensorflow/python:resources",
         "//tensorflow/python:training",
@@ -591,13 +614,10 @@ tf_custom_op_py_library(
 
 tf_kernel_library(
     name = "stats_accumulator_ops_kernels",
-    srcs = [
-        "kernels/stats_accumulator_ops.cc",
-    ],
+    srcs = ["kernels/stats_accumulator_ops.cc"],
     deps = [
         "//tensorflow/contrib/boosted_trees/lib:utils",
         "//tensorflow/contrib/boosted_trees/resources:stamped_resource",
-        "//tensorflow/core:framework",
         "//tensorflow/core:framework_headers_lib",
     ],
     alwayslink = 1,
@@ -609,7 +629,12 @@ py_library(
     name = "boosted_trees_pip",
     deps = [
         ":init_py",
+        "//tensorflow/contrib/boosted_trees:gbdt_batch",
+        "//tensorflow/contrib/boosted_trees/estimator_batch:custom_export_strategy",
         "//tensorflow/contrib/boosted_trees/estimator_batch:init_py",
+        "//tensorflow/contrib/boosted_trees/estimator_batch:trainer_hooks",
+        "//tensorflow/contrib/boosted_trees/lib:categorical_split_handler",
+        "//tensorflow/contrib/boosted_trees/lib:ordinal_split_handler",
         "//tensorflow/contrib/boosted_trees/proto:learner_proto_py",
         "//tensorflow/contrib/boosted_trees/proto:quantiles_proto_py",
         "//tensorflow/contrib/boosted_trees/proto:split_info_proto_py",
diff --git a/tensorflow/contrib/boosted_trees/estimator_batch/BUILD b/tensorflow/contrib/boosted_trees/estimator_batch/BUILD
index f9e186788f..d0ee1fd60d 100644
--- a/tensorflow/contrib/boosted_trees/estimator_batch/BUILD
+++ b/tensorflow/contrib/boosted_trees/estimator_batch/BUILD
@@ -27,13 +27,6 @@ py_library(
         "__init__.py",
     ],
     srcs_version = "PY2AND3",
-    deps = [
-        "custom_export_strategy",
-        ":custom_loss_head",
-        ":estimator",
-        ":model",
-        ":trainer_hooks",
-    ],
 )
 
 py_library(
@@ -41,7 +34,12 @@ py_library(
     srcs = ["model.py"],
     srcs_version = "PY2AND3",
     deps = [
+        ":trainer_hooks",
         "//tensorflow/contrib/boosted_trees:gbdt_batch",
+        "//tensorflow/contrib/boosted_trees:model_ops_py",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:state_ops",
+        "//tensorflow/python:training",
     ],
 )
 
@@ -51,6 +49,10 @@ py_library(
     srcs_version = "PY2AND3",
     deps = [
         "//tensorflow/contrib/learn",
+        "//tensorflow/core:protos_all_py",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:platform",
+        "//tensorflow/python:training",
     ],
 )
 
@@ -61,6 +63,15 @@ py_test(
     srcs_version = "PY2AND3",
     deps = [
         ":trainer_hooks",
+        "//tensorflow/contrib/framework:framework_py",
+        "//tensorflow/python:constant_op",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:platform_test",
+        "//tensorflow/python:session",
+        "//tensorflow/python:state_ops",
+        "//tensorflow/python:training",
+        "//tensorflow/python:variables",
     ],
 )
 
@@ -69,6 +80,10 @@ py_library(
     srcs = ["custom_loss_head.py"],
     srcs_version = "PY2AND3",
     deps = [
+        "//tensorflow/contrib/learn",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:dtypes",
+        "//tensorflow/python:math_ops",
     ],
 )
 
@@ -82,6 +97,11 @@ py_library(
         "//tensorflow/contrib/decision_trees/proto:generic_tree_model_extensions_py",
         "//tensorflow/contrib/decision_trees/proto:generic_tree_model_py",
         "//tensorflow/contrib/learn",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:platform",
+        "//tensorflow/python:session",
+        "//tensorflow/python/saved_model:loader",
+        "//tensorflow/python/saved_model:tag_constants",
     ],
 )
 
@@ -92,8 +112,9 @@ py_test(
     srcs_version = "PY2AND3",
     deps = [
         ":custom_export_strategy",
-        "//tensorflow/contrib/decision_trees/proto:generic_tree_model_extensions_py",
-        "//tensorflow/contrib/decision_trees/proto:generic_tree_model_py",
+        "//tensorflow/contrib/boosted_trees/proto:tree_config_proto_py",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:platform_test",
     ],
 )
 
@@ -103,6 +124,6 @@ py_library(
     srcs_version = "PY2AND3",
     deps = [
         ":model",
-        ":trainer_hooks",
+        "//tensorflow/contrib/learn",
     ],
 )
diff --git a/tensorflow/contrib/boosted_trees/lib/BUILD b/tensorflow/contrib/boosted_trees/lib/BUILD
index 9b3ffa98e3..70aa0284a6 100644
--- a/tensorflow/contrib/boosted_trees/lib/BUILD
+++ b/tensorflow/contrib/boosted_trees/lib/BUILD
@@ -132,7 +132,6 @@ tf_cc_test(
         ":random_tree_gen",
         "//tensorflow/contrib/boosted_trees/resources:decision_tree_ensemble_resource",
         "//tensorflow/core:framework_headers_lib",
-        "//tensorflow/core:lib",
         "//tensorflow/core:tensor_testutil",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
@@ -149,7 +148,6 @@ cc_library(
     deps = [
         ":utils",
         "//tensorflow/core:framework_headers_lib",
-        "//tensorflow/core:lib",
         "//tensorflow/core:test",
         "//tensorflow/core:testlib",
     ],
@@ -197,7 +195,6 @@ tf_cc_test(
     srcs = ["quantiles/weighted_quantiles_buffer_test.cc"],
     deps = [
         ":weighted_quantiles",
-        "//tensorflow/core",
         "//tensorflow/core:lib",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
@@ -210,7 +207,6 @@ tf_cc_test(
     srcs = ["quantiles/weighted_quantiles_summary_test.cc"],
     deps = [
         ":weighted_quantiles",
-        "//tensorflow/core",
         "//tensorflow/core:lib",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
-- 
GitLab


From 2da50752715f7aea65c0b2e80a14c296d0e4176c Mon Sep 17 00:00:00 2001
From: Peter Hawkins <phawkins@google.com>
Date: Thu, 12 Oct 2017 07:32:58 -0700
Subject: [PATCH 295/909] [TF:TPU] Move the metadata for tpu.replicate() into a
 separate TPUReplicateMetadata graph node, rather than attaching a copy of it
 to every node that is to be replicated.

PiperOrigin-RevId: 171957514
---
 tensorflow/contrib/tpu/ops/replication_ops.cc |  5 +++++
 tensorflow/contrib/tpu/python/tpu/tpu.py      | 22 ++++++++-----------
 2 files changed, 14 insertions(+), 13 deletions(-)

diff --git a/tensorflow/contrib/tpu/ops/replication_ops.cc b/tensorflow/contrib/tpu/ops/replication_ops.cc
index a40e2a7898..b40dac4717 100644
--- a/tensorflow/contrib/tpu/ops/replication_ops.cc
+++ b/tensorflow/contrib/tpu/ops/replication_ops.cc
@@ -22,6 +22,11 @@ namespace tensorflow {
 using shape_inference::InferenceContext;
 using shape_inference::ShapeHandle;
 
+REGISTER_OP("TPUReplicateMetadata")
+    .Attr("num_replicas: int >= 0")
+    .Attr("global_tpu_id: list(int) = []")
+    .SetShapeFn(shape_inference::UnknownShape);
+
 REGISTER_OP("TPUReplicatedInput")
     .Input("inputs: N * T")
     .Output("output: T")
diff --git a/tensorflow/contrib/tpu/python/tpu/tpu.py b/tensorflow/contrib/tpu/python/tpu/tpu.py
index f6800e3e24..fa5760953d 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu.py
@@ -105,9 +105,8 @@ class TPUReplicateContext(control_flow_ops.ControlFlowContext):
   """A ControlFlowContext for nodes inside a TPU computation.
 
   The primary role of TPUReplicateContext is to mark operators inside a
-  tpu.replicate() computation with attributes:
-  * _tpu_replicate=XYZ, where XYZ is a unique name, and
-  * _tpu_num_replicas=k, where k is the number of replicas.
+  tpu.replicate() computation with the attribute "_tpu_replicate=XYZ", where XYZ
+  is a unique name.
 
   We use a ControlFlowContext to perform the annotation since it
   integrates with Tensorflow constructs like ResourceVariables. For example,
@@ -116,11 +115,9 @@ class TPUReplicateContext(control_flow_ops.ControlFlowContext):
   to build the variable's definition outside the replicated computation.
   """
 
-  def __init__(self, name, num_replicas, global_tpu_id=None):
+  def __init__(self, name):
     control_flow_ops.ControlFlowContext.__init__(self)
     self._name = name
-    self._num_replicas = num_replicas
-    self._global_tpu_id = [] if global_tpu_id is None else global_tpu_id
 
   def AddOp(self, op):
     self._AddOpInternal(op)
@@ -135,8 +132,6 @@ class TPUReplicateContext(control_flow_ops.ControlFlowContext):
     if "_tpu_replicate" in op.node_def.attr:
       raise ValueError("TPU computations cannot be nested")
     op.node_def.attr["_tpu_replicate"].s = self._name
-    op.node_def.attr["_tpu_num_replicas"].i = self._num_replicas
-    op.node_def.attr["_tpu_global_id"].list.i.extend(self._global_tpu_id)
     op.graph.prevent_feeding(op)
     op.graph.prevent_fetching(op)
 
@@ -243,14 +238,15 @@ def replicate(computation,
       computation_inputs.append(
           tpu_ops.tpu_replicated_input(replicas, name="input{}".format(i)))
 
-    context = TPUReplicateContext(
-        name=graph.unique_name("cluster"),
-        num_replicas=num_replicas,
-        global_tpu_id=global_tpu_id)
+    context = TPUReplicateContext(name=graph.unique_name("cluster"))
     try:
       context.Enter()
 
-      with tpu_function.tpu_shard_context(num_replicas):
+      metadata = tpu_ops.tpu_replicate_metadata(
+          num_replicas=num_replicas, global_tpu_id=global_tpu_id)
+
+      with tpu_function.tpu_shard_context(
+          num_replicas), ops.control_dependencies([metadata]):
 
         # The EncapsulateTPUComputations rewrite needs to identify the
         # replicated arguments inside each computation. Adds identity operators
-- 
GitLab


From d244ffb69ceb971d34e330cae45ed944d488e9ec Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 12 Oct 2017 08:12:26 -0700
Subject: [PATCH 296/909] Minimal support for running OpsTest on GPU, using
 CUDA unified memory.

PiperOrigin-RevId: 171961190
---
 tensorflow/core/BUILD                         |  3 +
 .../gpu/gpu_managed_allocator.cc              | 39 ++++++++
 .../gpu/gpu_managed_allocator.h               | 36 +++++++
 tensorflow/core/kernels/BUILD                 |  8 +-
 tensorflow/core/kernels/ops_testutil.cc       | 67 +++++++++++++
 tensorflow/core/kernels/ops_testutil.h        | 99 +++++++------------
 6 files changed, 187 insertions(+), 65 deletions(-)
 create mode 100644 tensorflow/core/common_runtime/gpu/gpu_managed_allocator.cc
 create mode 100644 tensorflow/core/common_runtime/gpu/gpu_managed_allocator.h
 create mode 100644 tensorflow/core/kernels/ops_testutil.cc

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 74aecbc1f2..4d9f368bc0 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -889,6 +889,7 @@ cc_library(
         ":test",
         "//tensorflow/cc:scope",
         "//tensorflow/core/kernels:constant_op",
+        "//tensorflow/core/kernels:ops_testutil",
         "//tensorflow/core/kernels:ops_util",
     ],
 )
@@ -2122,6 +2123,7 @@ GPU_RUNTIME_HEADERS = [
     "common_runtime/gpu/gpu_debug_allocator.h",
     "common_runtime/gpu/gpu_device.h",
     "common_runtime/gpu/gpu_init.h",
+    "common_runtime/gpu/gpu_managed_allocator.h",
     "common_runtime/gpu/gpu_stream_util.h",
     "common_runtime/gpu/gpu_util.h",
     "common_runtime/gpu/pool_allocator.h",
@@ -2136,6 +2138,7 @@ tf_cuda_library(
         "common_runtime/gpu/gpu_debug_allocator.cc",
         "common_runtime/gpu/gpu_device.cc",
         "common_runtime/gpu/gpu_device_factory.cc",
+        "common_runtime/gpu/gpu_managed_allocator.cc",
         "common_runtime/gpu/gpu_stream_util.cc",
         "common_runtime/gpu/gpu_util.cc",
         "common_runtime/gpu/gpu_util_platform_specific.cc",
diff --git a/tensorflow/core/common_runtime/gpu/gpu_managed_allocator.cc b/tensorflow/core/common_runtime/gpu/gpu_managed_allocator.cc
new file mode 100644
index 0000000000..613633eb91
--- /dev/null
+++ b/tensorflow/core/common_runtime/gpu/gpu_managed_allocator.cc
@@ -0,0 +1,39 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifdef GOOGLE_CUDA
+#define EIGEN_USE_GPU
+#endif
+
+#include "tensorflow/core/common_runtime/gpu/gpu_managed_allocator.h"
+
+namespace tensorflow {
+
+void* GpuManagedAllocator::AllocateRaw(size_t alignment, size_t num_bytes) {
+  void* ptr = nullptr;
+#ifdef GOOGLE_CUDA
+  CHECK_EQ(cudaMallocManaged(&ptr, num_bytes), cudaSuccess);
+#endif
+  CHECK(!(reinterpret_cast<uintptr_t>(ptr) & (alignment - 1)));
+  return ptr;
+}
+
+void GpuManagedAllocator::DeallocateRaw(void* ptr) {
+#ifdef GOOGLE_CUDA
+  CHECK_EQ(cudaFree(ptr), cudaSuccess);
+#endif
+}
+
+}  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/gpu/gpu_managed_allocator.h b/tensorflow/core/common_runtime/gpu/gpu_managed_allocator.h
new file mode 100644
index 0000000000..006b2ca448
--- /dev/null
+++ b/tensorflow/core/common_runtime/gpu/gpu_managed_allocator.h
@@ -0,0 +1,36 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef THIRD_PARTY_TENSORFLOW_CORE_COMMON_RUNTIME_GPU_GPU_MANAGED_ALLOCATOR_H_
+#define THIRD_PARTY_TENSORFLOW_CORE_COMMON_RUNTIME_GPU_GPU_MANAGED_ALLOCATOR_H_
+
+#include "tensorflow/core/framework/allocator.h"
+
+namespace tensorflow {
+
+// An allocator for CUDA unified memory. Memory allocated with this allocator
+// can be accessed from both host and device. CUDA transparently migrates dirty
+// pages, which can be slow. Therefore, this allocator is intended for
+// convenience in functional tests only.
+class GpuManagedAllocator : public Allocator {
+ public:
+  string Name() override { return "GpuManagedAllocator"; }
+  void* AllocateRaw(size_t alignment, size_t num_bytes) override;
+  void DeallocateRaw(void* ptr) override;
+};
+
+}  // namespace tensorflow
+
+#endif  // THIRD_PARTY_TENSORFLOW_CORE_COMMON_RUNTIME_GPU_GPU_MANAGED_ALLOCATOR_H_
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index dbf6449bc2..0073ba1a96 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -32,6 +32,7 @@ load(
     "tf_cc_tests",
     "tf_cc_binary",
     "tf_copts",
+    "tf_cuda_library",
     "tf_opts_nortti_if_android",
     "tf_kernel_library",
     "tf_mkl_kernel_library",
@@ -225,10 +226,15 @@ cc_library(
     ],
 )
 
-cc_library(
+tf_cuda_library(
     name = "ops_testutil",
     testonly = 1,
+    srcs = ["ops_testutil.cc"],
     hdrs = ["ops_testutil.h"],
+    cuda_deps = [
+        "//tensorflow/core:gpu_lib",
+        "//tensorflow/core:gpu_runtime",
+    ],
     deps = [
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:core_cpu_internal",
diff --git a/tensorflow/core/kernels/ops_testutil.cc b/tensorflow/core/kernels/ops_testutil.cc
new file mode 100644
index 0000000000..cd13d31bbc
--- /dev/null
+++ b/tensorflow/core/kernels/ops_testutil.cc
@@ -0,0 +1,67 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifdef GOOGLE_CUDA
+#define EIGEN_USE_GPU
+#include "tensorflow/core/common_runtime/gpu/gpu_managed_allocator.h"
+#endif
+
+#include "tensorflow/core/kernels/ops_testutil.h"
+
+namespace tensorflow {
+
+void OpsTestBase::SetDevice(const DeviceType& device_type,
+                            std::unique_ptr<Device> device) {
+  CHECK(device_.get()) << "No device provided";
+  device_type_ = device_type;
+  device_ = std::move(device);
+#ifdef GOOGLE_CUDA
+  if (device_type == DEVICE_GPU) {
+    managed_allocator_.reset(new GpuManagedAllocator());
+    allocator_ = managed_allocator_.get();
+  } else {
+    managed_allocator_.reset();
+    allocator_ = device_->GetAllocator(AllocatorAttributes());
+  }
+#else
+  CHECK_NE(device_type, DEVICE_GPU)
+      << "Requesting GPU on binary compiled without GOOGLE_CUDA.";
+#endif
+}
+
+Tensor* OpsTestBase::GetOutput(int output_index) {
+  CHECK_LT(output_index, context_->num_outputs());
+  Tensor* output = context_->mutable_output(output_index);
+#ifdef GOOGLE_CUDA
+  if (device_type_ == DEVICE_GPU) {
+    managed_outputs_.resize(context_->num_outputs());
+    // Copy the output tensor to managed memory if we haven't done so.
+    if (!managed_outputs_[output_index]) {
+      Tensor* managed_output =
+          new Tensor(allocator(), output->dtype(), output->shape());
+      auto src = output->tensor_data();
+      auto dst = managed_output->tensor_data();
+      context_->eigen_gpu_device().memcpy(const_cast<char*>(dst.data()),
+                                          src.data(), src.size());
+      context_->eigen_gpu_device().synchronize();
+      managed_outputs_[output_index] = managed_output;
+    }
+    output = managed_outputs_[output_index];
+  }
+#endif
+  return output;
+}
+
+}  // namespace tensorflow
diff --git a/tensorflow/core/kernels/ops_testutil.h b/tensorflow/core/kernels/ops_testutil.h
index 2a6b9e00bf..2c195beb7f 100644
--- a/tensorflow/core/kernels/ops_testutil.h
+++ b/tensorflow/core/kernels/ops_testutil.h
@@ -47,7 +47,6 @@ limitations under the License.
 #include "tensorflow/core/util/tensor_slice_reader_cache.h"
 
 namespace tensorflow {
-
 namespace test {
 
 inline void SetOutputAttrs(OpKernelContext::Params* params,
@@ -71,25 +70,22 @@ inline void SetOutputAttrs(OpKernelContext::Params* params,
 // to use the BrainClient interface.
 class OpsTestBase : public ::testing::Test {
  public:
-  OpsTestBase() : device_type_(DEVICE_CPU) {
-    device_.reset(
-        DeviceFactory::NewDevice("CPU", {}, "/job:a/replica:0/task:0"));
+  OpsTestBase()
+      : device_(DeviceFactory::NewDevice("CPU", {}, "/job:a/replica:0/task:0")),
+        device_type_(DEVICE_CPU) {
     CHECK(device_.get()) << "Could not create CPU device";
+    allocator_ = device_->GetAllocator(AllocatorAttributes());
   }
 
   ~OpsTestBase() override {
     gtl::STLDeleteElements(&tensors_);
+    gtl::STLDeleteElements(&managed_outputs_);
     context_.reset(nullptr);
     params_.reset(nullptr);
   }
 
   // Allow kernel unit tests to run on GPU
-  void SetDevice(const DeviceType& device_type,
-                 std::unique_ptr<Device> device) {
-    CHECK(device_.get()) << "No device provided";
-    device_type_ = device_type;
-    device_ = std::move(device);
-  }
+  void SetDevice(const DeviceType& device_type, std::unique_ptr<Device> device);
 
   void set_node_def(const NodeDef& node_def) { node_def_.CopyFrom(node_def); }
 
@@ -118,42 +114,14 @@ class OpsTestBase : public ::testing::Test {
   // TODO(vrv): Replace with something like a BrainClient Feed.
   template <typename T>
   void AddInput(const TensorShape& shape, std::function<T(int)> input_mapping) {
-    CHECK_GT(input_types_.size(), inputs_.size())
-        << "Adding more inputs than types; perhaps you need to call MakeOp";
-    bool is_ref = IsRefType(input_types_[inputs_.size()]);
-    Tensor* input = new Tensor(device_->GetAllocator(AllocatorAttributes()),
-                               DataTypeToEnum<T>::v(), shape);
-    test::FillFn(input, input_mapping);
-    tensors_.push_back(input);
-    if (is_ref) {
-      CHECK_EQ(RemoveRefType(input_types_[inputs_.size()]),
-               DataTypeToEnum<T>::v());
-      inputs_.push_back({&lock_for_refs_, input});
-    } else {
-      CHECK_EQ(input_types_[inputs_.size()], DataTypeToEnum<T>::v());
-      inputs_.push_back({nullptr, input});
-    }
+    test::FillFn(AddInput(DataTypeToEnum<T>::v(), shape), input_mapping);
   }
 
   // Like AddInput but takes in an explicit arrayslice of data.
   template <typename T>
   void AddInputFromArray(const TensorShape& shape,
                          const gtl::ArraySlice<T>& data) {
-    CHECK_GT(input_types_.size(), inputs_.size())
-        << "Adding more inputs than types; perhaps you need to call MakeOp";
-    bool is_ref = IsRefType(input_types_[inputs_.size()]);
-    Tensor* input = new Tensor(device_->GetAllocator(AllocatorAttributes()),
-                               DataTypeToEnum<T>::v(), shape);
-    test::FillValues<T>(input, data);
-    tensors_.push_back(input);
-    if (is_ref) {
-      CHECK_EQ(RemoveRefType(input_types_[inputs_.size()]),
-               DataTypeToEnum<T>::v());
-      inputs_.push_back({&lock_for_refs_, input});
-    } else {
-      CHECK_EQ(input_types_[inputs_.size()], DataTypeToEnum<T>::v());
-      inputs_.push_back({nullptr, input});
-    }
+    test::FillValues<T>(AddInput(DataTypeToEnum<T>::v(), shape), data);
   }
 
   // Convenience function to add an input and populate it with the elements from
@@ -161,21 +129,7 @@ class OpsTestBase : public ::testing::Test {
   template <typename T, typename SrcType>
   void AddInputFromList(const TensorShape& shape,
                         std::initializer_list<SrcType> data) {
-    CHECK_GT(input_types_.size(), inputs_.size())
-        << "Adding more inputs than types; perhaps you need to call MakeOp";
-    bool is_ref = IsRefType(input_types_[inputs_.size()]);
-    Tensor* input = new Tensor(device_->GetAllocator(AllocatorAttributes()),
-                               DataTypeToEnum<T>::v(), shape);
-    test::FillValues<T>(input, data);
-    tensors_.push_back(input);
-    if (is_ref) {
-      CHECK_EQ(RemoveRefType(input_types_[inputs_.size()]),
-               DataTypeToEnum<T>::v());
-      inputs_.push_back({&lock_for_refs_, input});
-    } else {
-      CHECK_EQ(input_types_[inputs_.size()], DataTypeToEnum<T>::v());
-      inputs_.push_back({nullptr, input});
-    }
+    test::FillValues<T>(AddInput(DataTypeToEnum<T>::v(), shape), data);
   }
 
   // Adds a Resource type as input. If <container> is empty, uses the default
@@ -197,8 +151,7 @@ class OpsTestBase : public ::testing::Test {
     handle.set_name(name);
     handle.set_hash_code(type_index.hash_code());
     handle.set_maybe_type_name(type_index.name());
-    Tensor* input = new Tensor(device_->GetAllocator(AllocatorAttributes()),
-                               DT_RESOURCE, TensorShape({}));
+    Tensor* input = new Tensor(allocator(), DT_RESOURCE, TensorShape({}));
     input->scalar<ResourceHandle>()() = handle;
     tensors_.push_back(input);
     inputs_.push_back({nullptr, input});
@@ -246,19 +199,33 @@ class OpsTestBase : public ::testing::Test {
   // Returns the tensor output for 'output_index'.
   //
   // REQUIRES: 0 <= output_index < context_->num_outputs()
-  Tensor* GetOutput(int output_index) {
-    CHECK_LT(output_index, context_->num_outputs());
-    return context_->mutable_output(output_index);
-  }
+  Tensor* GetOutput(int output_index);
 
-  Allocator* allocator() {
-    return device_->GetAllocator(AllocatorAttributes());
-  }
+  Allocator* allocator() { return allocator_; }
 
   const DataTypeVector& output_types() const { return kernel_->output_types(); }
 
+ private:
+  Tensor* AddInput(DataType dtype, const TensorShape& shape) {
+    CHECK_GT(input_types_.size(), inputs_.size())
+        << "Adding more inputs than types; perhaps you need to call MakeOp";
+    bool is_ref = IsRefType(input_types_[inputs_.size()]);
+    Tensor* input = new Tensor(allocator(), dtype, shape);
+    tensors_.push_back(input);
+    if (is_ref) {
+      CHECK_EQ(RemoveRefType(input_types_[inputs_.size()]), dtype);
+      inputs_.push_back({&lock_for_refs_, input});
+    } else {
+      CHECK_EQ(input_types_[inputs_.size()], dtype);
+      inputs_.push_back({nullptr, input});
+    }
+    return input;
+  }
+
  protected:
   std::unique_ptr<Device> device_;
+  // The device allocator, or the managed_allocator_ below if running on GPU.
+  Allocator* allocator_;
 
   std::unique_ptr<OpKernel> kernel_;
   std::unique_ptr<ScopedStepContainer> step_container_;
@@ -271,9 +238,13 @@ class OpsTestBase : public ::testing::Test {
   gtl::InlinedVector<TensorValue, 4> inputs_;
   // Owns Tensors.
   std::vector<Tensor*> tensors_;
+  // Copies of the outputs in unified memory (host and device accessible).
+  std::vector<Tensor*> managed_outputs_;
 
   std::unique_ptr<OpKernelContext::Params> params_;
   std::unique_ptr<OpKernelContext> context_;
+  // Unified memory allocator, only used when running on GPU.
+  std::unique_ptr<Allocator> managed_allocator_;
 
  private:
   TF_DISALLOW_COPY_AND_ASSIGN(OpsTestBase);
-- 
GitLab


From c71aea70f84e582854f365665899c3045d1a48f0 Mon Sep 17 00:00:00 2001
From: Yunxing Dai <yunxing@google.com>
Date: Thu, 12 Oct 2017 08:29:45 -0700
Subject: [PATCH 297/909] Use a serialized graph compiler to generate xla
 graph.

- Move away from previous TF graph executor, which contains few features that we need and also introduces indeterminism.
- Unlike previous executor, the new serial graph compiler doesn't recurse into a function and inlines it. Instead, it creates a computation of the function and then creates a `call` op to call into the newly created computation.
- Add a optional comparator in DFS algorithm, which is needed to make the compiler deterministic.

RELNOTES: Use a determinisitc executor to generate xla graph.
PiperOrigin-RevId: 171962775
---
 tensorflow/compiler/tf2xla/BUILD              |   3 +
 tensorflow/compiler/tf2xla/graph_compiler.cc  | 248 ++++++++++++++++++
 tensorflow/compiler/tf2xla/graph_compiler.h   | 111 ++++++++
 .../compiler/tf2xla/kernels/sendrecv_ops.cc   |   1 +
 tensorflow/compiler/tf2xla/xla_compiler.cc    | 100 +++----
 tensorflow/compiler/tf2xla/xla_compiler.h     |  75 +++---
 .../compiler/tf2xla/xla_compiler_test.cc      | 127 ++++++++-
 tensorflow/compiler/xla/service/service.cc    |   5 +-
 tensorflow/core/graph/algorithm.cc            |  64 ++++-
 tensorflow/core/graph/algorithm.h             |  43 ++-
 tensorflow/core/graph/algorithm_test.cc       |  35 +++
 tensorflow/core/graph/graph.h                 |   4 +-
 12 files changed, 693 insertions(+), 123 deletions(-)
 create mode 100644 tensorflow/compiler/tf2xla/graph_compiler.cc
 create mode 100644 tensorflow/compiler/tf2xla/graph_compiler.h

diff --git a/tensorflow/compiler/tf2xla/BUILD b/tensorflow/compiler/tf2xla/BUILD
index 4da2ed722e..7865f16e53 100644
--- a/tensorflow/compiler/tf2xla/BUILD
+++ b/tensorflow/compiler/tf2xla/BUILD
@@ -102,11 +102,13 @@ cc_library(
         "xla_helpers.cc",
         "xla_op_kernel.cc",
         "xla_op_registry.cc",
+        "graph_compiler.cc",
         "xla_cpu_backend.cc",
     ] + if_cuda_is_configured([
         "xla_gpu_backend.cc",
     ]),
     hdrs = [
+        "graph_compiler.h",
         "xla_compilation_device.h",
         "xla_compiler.h",
         "xla_context.h",
@@ -117,6 +119,7 @@ cc_library(
     visibility = [":friends"],
     deps = [
         ":common",
+        ":const_analysis",
         ":dump_graph",
         ":functionalize_control_flow",
         "//tensorflow/compiler/xla:literal_util",
diff --git a/tensorflow/compiler/tf2xla/graph_compiler.cc b/tensorflow/compiler/tf2xla/graph_compiler.cc
new file mode 100644
index 0000000000..d5369e478a
--- /dev/null
+++ b/tensorflow/compiler/tf2xla/graph_compiler.cc
@@ -0,0 +1,248 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/tf2xla/graph_compiler.h"
+
+#include <deque>
+#include <numeric>
+#include <vector>
+#include "tensorflow/compiler/tf2xla/const_analysis.h"
+#include "tensorflow/compiler/tf2xla/dump_graph.h"
+#include "tensorflow/compiler/tf2xla/functionalize_control_flow.h"
+#include "tensorflow/compiler/tf2xla/literal_util.h"
+#include "tensorflow/compiler/tf2xla/shape_util.h"
+#include "tensorflow/compiler/tf2xla/type_util.h"
+#include "tensorflow/compiler/tf2xla/xla_compilation_device.h"
+#include "tensorflow/compiler/tf2xla/xla_compiler.h"
+#include "tensorflow/compiler/tf2xla/xla_context.h"
+#include "tensorflow/compiler/tf2xla/xla_op_kernel.h"
+#include "tensorflow/compiler/xla/client/client_library.h"
+#include "tensorflow/core/common_runtime/device.h"
+#include "tensorflow/core/common_runtime/executor.h"
+#include "tensorflow/core/common_runtime/function.h"
+#include "tensorflow/core/common_runtime/graph_optimizer.h"
+#include "tensorflow/core/framework/attr_value_util.h"
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/graph/algorithm.h"
+#include "tensorflow/core/graph/graph_constructor.h"
+#include "tensorflow/core/graph/node_builder.h"
+#include "tensorflow/core/lib/hash/hash.h"
+#include "tensorflow/core/platform/logging.h"
+#include "tensorflow/core/public/version.h"
+
+namespace tensorflow {
+
+namespace {
+Status PrepareArguments(XlaOpKernelContext* ctx, Graph* graph,
+                        const std::vector<const XlaExpression*>& expressions,
+                        std::vector<XlaCompiler::Argument>* args) {
+  auto builder = ctx->builder();
+  std::vector<bool> compile_time_constant_flags(expressions.size());
+
+  TF_RETURN_IF_ERROR(
+      BackwardsConstAnalysis(*graph, &compile_time_constant_flags));
+
+  args->resize(expressions.size());
+  for (int i = 0; i < args->size(); ++i) {
+    XlaCompiler::Argument& arg = (*args)[i];
+    arg.type = ctx->input_type(i);
+
+    TF_RETURN_IF_ERROR(
+        TensorShapeToXLAShape(arg.type, ctx->InputShape(i), &arg.shape));
+
+    if (arg.type == DT_RESOURCE) {
+      return errors::InvalidArgument(
+          "Resource as function argument is not yet implemented.");
+    } else if (expressions[i]->has_constant_value()) {
+      arg.kind = XlaCompiler::Argument::kConstant;
+      arg.constant_value = expressions[i]->constant_value();
+    } else if (compile_time_constant_flags[i]) {
+      arg.kind = XlaCompiler::Argument::kConstant;
+      TF_RET_CHECK(expressions[i]->resource() == nullptr)
+          << "Input with resource is not yet implemented.";
+      TF_ASSIGN_OR_RETURN(auto literal,
+                          builder->ComputeConstant(expressions[i]->handle()));
+      TF_RETURN_IF_ERROR(
+          LiteralToHostTensor(*literal, arg.type, &arg.constant_value));
+    } else {
+      arg.kind = XlaCompiler::Argument::kParameter;
+    }
+  }
+  return Status::OK();
+}
+}  // namespace
+Status GraphCompiler::Compile() {
+  std::vector<NodeBinding> bindings(graph_->num_node_ids());
+  std::vector<Node*> topo_sorted_nodes;
+  // XLA requires determinism, generate a stable ordering from DFS.
+  GetReversePostOrder(*graph_, &topo_sorted_nodes,
+                      /*stable_comparator=*/NodeComparatorID());
+
+  OpKernelContext::Params params;
+  PartiallySetupParams(&params);
+
+  for (Node* n : topo_sorted_nodes) {
+    // Set up bindings.
+    NodeBinding& binding = bindings[n->id()];
+    binding.node = n;
+    Status s = flib_->CreateKernel(n->def(), &binding.op_kernel);
+    binding.output_attrs.resize(n->num_outputs());
+    if (!s.ok()) {
+      binding.op_kernel = nullptr;
+      s = AttachDef(s, *n);
+      LOG(ERROR) << "Executor failed to create kernel. " << s;
+      return s;
+    }
+  }
+
+  // Bindings are initialized by the size of graph_->num_node_ids. However, the
+  // graph may contain dead nodes that still hold a valid node id. Thus
+  // graph_->num_node_ids could be larger than number of topo sorted nodes.
+  TF_RET_CHECK(bindings.size() >= topo_sorted_nodes.size());
+
+  for (Node* n : topo_sorted_nodes) {
+    TF_RET_CHECK(!n->IsRecv() && !n->IsSend() && !n->IsSwitch())
+        << "Not supported node: " << n->DebugString();
+    NodeBinding& binding = bindings[n->id()];
+    params.op_kernel = binding.op_kernel;
+    params.output_attr_array = binding.output_attrs.data();
+
+    // tensor_inputs_ is a buffer reused across graph traversal. We clean up and
+    // reinitialize the buffer before we visit a new node.
+    tensor_inputs_.clear();
+    tensor_inputs_.resize(n->num_inputs());
+
+    // Set up inputs from outputs of previous nodes.
+    for (auto* e : n->in_edges()) {
+      if (e->IsControlEdge()) continue;
+      Node* src = e->src();
+      tensor_inputs_[e->dst_input()] =
+          bindings[src->id()].tensor_values[e->src_output()];
+    }
+
+    OpKernelContext op_context(&params, n->num_outputs());
+    if (IsFunctional(n)) {
+      TF_RETURN_IF_ERROR(CompileFunctionalNode(n, &op_context));
+    } else {
+      device_->Compute(CHECK_NOTNULL(params.op_kernel), &op_context);
+      Status s = op_context.status();
+      TF_RETURN_IF_ERROR(s);
+    }
+
+    // Set up outputs. Also check if outputs from the previous computation is
+    // valid.
+    for (int o = 0; o < n->num_outputs(); ++o) {
+      const auto tensor_val = op_context.release_output(o);
+      if (*op_context.is_output_dead() || tensor_val.tensor == nullptr) {
+        return errors::Internal("Missing xla_context ", o, "-th output from ",
+                                (*op_context.is_output_dead() ? "(dead)" : ""),
+                                SummarizeNode(*n));
+      }
+      binding.tensor_values.push_back(tensor_val);
+    }
+  }
+
+  // Clean up tensor data and op kernels.
+  for (NodeBinding& binding : bindings) {
+    delete binding.op_kernel;
+    for (auto& t : binding.tensor_values) {
+      if (!t.is_ref()) {
+        delete t.tensor;
+      }
+    }
+  }
+  return Status::OK();
+}
+
+bool GraphCompiler::IsFunctional(Node* n) {
+  return n->type_string() == FunctionLibraryDefinition::kGradientOp ||
+         (flib_->GetFunctionLibraryDefinition()->Find(n->def().op()) !=
+          nullptr);
+}
+
+Status GraphCompiler::CompileFunctionalNode(Node* n,
+                                            OpKernelContext* op_context) {
+  TF_RET_CHECK(IsFunctional(n));
+  // For functional nodes, compile them using compiler from the context and call
+  // into the functions.
+  XlaOpKernelContext xla_op_context(op_context);
+
+  XlaCompiler* compiler = xla_op_context.compiler();
+
+  NameAttrList func;
+  if (flib_->GetFunctionLibraryDefinition()->Find(n->def().op())) {
+    func.set_name(n->def().op());
+  } else {
+    func.set_name(FunctionLibraryDefinition::kGradientOp);
+  }
+  *func.mutable_attr() = n->def().attr();
+
+  std::vector<const XlaExpression*> expressions;
+
+  for (auto tensor : tensor_inputs_) {
+    auto expression =
+        reinterpret_cast<const XlaExpression*>(tensor->tensor_data().data());
+    expressions.push_back(expression);
+  }
+
+  // Prepare the arguments and compile the function.
+  std::vector<XlaCompiler::Argument> arguments;
+  const FunctionBody* fbody;
+  TF_RETURN_IF_ERROR(compiler->FindFunctionBody(func, &fbody));
+
+  auto graph = compiler->GetGraph(fbody);
+
+  TF_RETURN_IF_ERROR(
+      PrepareArguments(&xla_op_context, graph.get(), expressions, &arguments));
+
+  XlaCompiler::CompilationResult result;
+
+  TF_RETURN_IF_ERROR(compiler->CompileFunction(XlaCompiler::CompileOptions(),
+                                               func, arguments, &result));
+
+  TF_RET_CHECK(arguments.size() == expressions.size());
+
+  std::vector<xla::ComputationDataHandle> handles;
+  for (int64 i = 0; i < expressions.size(); ++i) {
+    if (arguments[i].kind == XlaCompiler::Argument::kConstant) {
+      continue;
+    }
+    handles.push_back(expressions[i]->handle());
+  }
+
+  XlaContext& context = XlaContext::Get(op_context);
+  auto* b = context.builder();
+
+  auto output_handle = b->Call(*result.computation, handles);
+  // The output handle of `Call` computation is a tuple type. Unzip it so
+  // that it can fit into future computations.
+  for (int64 i = 0; i < n->num_outputs(); ++i) {
+    if (result.outputs[i].is_constant) {
+      xla_op_context.SetConstantOutput(i, result.outputs[i].constant_value);
+    } else {
+      xla_op_context.SetOutput(i, b->GetTupleElement(output_handle, i));
+    }
+  }
+  return b->first_error();
+}
+
+void GraphCompiler::PartiallySetupParams(OpKernelContext::Params* params) {
+  params->device = device_;
+  params->inputs = &tensor_inputs_;
+  params->step_container = step_container_;
+  params->resource_manager = device_->resource_manager();
+}
+
+}  // namespace tensorflow
diff --git a/tensorflow/compiler/tf2xla/graph_compiler.h b/tensorflow/compiler/tf2xla/graph_compiler.h
new file mode 100644
index 0000000000..ccf9351642
--- /dev/null
+++ b/tensorflow/compiler/tf2xla/graph_compiler.h
@@ -0,0 +1,111 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_COMPILER_TF2XLA_GRAPH_COMPILER_H_
+#define TENSORFLOW_COMPILER_TF2XLA_GRAPH_COMPILER_H_
+
+#include "tensorflow/compiler/tf2xla/xla_compilation_device.h"
+#include "tensorflow/compiler/tf2xla/xla_context.h"
+#include "tensorflow/compiler/xla/client/local_client.h"
+#include "tensorflow/core/common_runtime/device.h"
+#include "tensorflow/core/common_runtime/device_mgr.h"
+#include "tensorflow/core/common_runtime/function.h"
+#include "tensorflow/core/framework/function.h"
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/platform/env.h"
+#include "tensorflow/core/platform/mutex.h"
+#include "tensorflow/core/platform/notification.h"
+#include "tensorflow/core/platform/thread_annotations.h"
+#include "tensorflow/core/public/version.h"
+
+namespace tensorflow {
+
+// GraphCompiler compiles the graph in topological order in the current
+// thread. It also resolves the nondeterminism in the graph by enforcing a
+// total order on all inputs to a node. This abstraction helps us create the
+// same XLA computation given two structurally equivalent TensorFlow graphs.
+// If a function call is visited during the graph traversal, it is then
+// compiled through the xla_context into a computation and a `Call` operation
+// is inserted to call into that computation.
+//
+// Note: GraphCompiler was created to remove our dependency to TF Executor in
+// the history. There are still some todos so that we can completely decouple
+// from Executor.
+//
+// TODO(yunxing): Remove usage of XlaCompilationDevice.
+//
+// TODO(yunxing): Remove the hack that wraps XlaExpression within a tensor now
+// that we don't use TF Executor to pass around a tensor.
+//
+// TODO(yunxing): Make XlaOpkernel not a subclass of OpKernel so that it can
+// handle a XlaExpression directly instead of a Tensor. This may require our own
+// op registration infrastructure instead of FunctionLibraryRuntime.
+class GraphCompiler {
+ public:
+  GraphCompiler(XlaContext* xla_context, XlaCompilationDevice* device,
+                Graph* graph, FunctionLibraryRuntime* flib,
+                ScopedStepContainer* step_container)
+      : xla_context_(xla_context),
+        device_(device),
+        graph_(graph),
+        flib_(flib),
+        step_container_(step_container) {}
+
+  // Compiles the graph. The results are written in `xla_context` that is passed
+  // into the compiler.
+  Status Compile();
+
+ private:
+  // NodeBinding is a wrapper on a `Node` that also contains computed
+  // TensorValue.
+  struct NodeBinding {
+    const Node* node;
+    // Kernel for this node, to be filled by CreateKernel.
+    // TODO(yunxing): Switching this to unique_ptr and understand why it crashes
+    // on GPU devices.
+    OpKernel* op_kernel;
+    // Output values of this node.
+    std::vector<TensorValue> tensor_values;
+    // Attributes of the outputs.
+    gtl::InlinedVector<AllocatorAttributes, 4> output_attrs;
+  };
+
+  // Partially sets params. This partially set params can be reused
+  // across multple nodes visit.
+  void PartiallySetupParams(OpKernelContext::Params* params);
+
+  // Tests if a node is a functional node. A functional node represents a
+  // defined computation and should be compiled using `compiler_`.
+  bool IsFunctional(Node* n);
+
+  // Compiles a functional node and writes result to OpkernelContext. A
+  // functional node represents a defined computation and should be compiled
+  // using `compiler_`.
+  Status CompileFunctionalNode(Node* n, OpKernelContext* op_context);
+
+  XlaContext* xla_context_;
+  XlaCompilationDevice* device_;
+  Graph* graph_;
+  FunctionLibraryRuntime* flib_;
+  ScopedStepContainer* step_container_;
+  // A buffer to hold tensor inputs to a node, this is reused across the graph
+  // traversal.
+  gtl::InlinedVector<TensorValue, 4> tensor_inputs_;
+};
+
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_COMPILER_TF2XLA_GRAPH_COMPILER_H_
diff --git a/tensorflow/compiler/tf2xla/kernels/sendrecv_ops.cc b/tensorflow/compiler/tf2xla/kernels/sendrecv_ops.cc
index ed818c56ed..5172781c0d 100644
--- a/tensorflow/compiler/tf2xla/kernels/sendrecv_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/sendrecv_ops.cc
@@ -14,6 +14,7 @@ limitations under the License.
 ==============================================================================*/
 
 #include "tensorflow/compiler/tf2xla/shape_util.h"
+#include "tensorflow/compiler/tf2xla/xla_compiler.h"
 #include "tensorflow/compiler/tf2xla/xla_helpers.h"
 #include "tensorflow/compiler/tf2xla/xla_op_kernel.h"
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
diff --git a/tensorflow/compiler/tf2xla/xla_compiler.cc b/tensorflow/compiler/tf2xla/xla_compiler.cc
index 1cd96fc4e2..a82ef02e32 100644
--- a/tensorflow/compiler/tf2xla/xla_compiler.cc
+++ b/tensorflow/compiler/tf2xla/xla_compiler.cc
@@ -18,12 +18,15 @@ limitations under the License.
 #include <deque>
 #include <numeric>
 
+#include "tensorflow/compiler/tf2xla/const_analysis.h"
 #include "tensorflow/compiler/tf2xla/dump_graph.h"
 #include "tensorflow/compiler/tf2xla/functionalize_control_flow.h"
+#include "tensorflow/compiler/tf2xla/graph_compiler.h"
 #include "tensorflow/compiler/tf2xla/shape_util.h"
 #include "tensorflow/compiler/tf2xla/type_util.h"
 #include "tensorflow/compiler/tf2xla/xla_compilation_device.h"
 #include "tensorflow/compiler/tf2xla/xla_context.h"
+#include "tensorflow/compiler/tf2xla/xla_op_kernel.h"
 #include "tensorflow/compiler/xla/client/client_library.h"
 #include "tensorflow/core/common_runtime/device.h"
 #include "tensorflow/core/common_runtime/executor.h"
@@ -126,6 +129,37 @@ static Status GetFunctionBody(const NameAttrList& function,
   return Status::OK();
 }
 
+Status XlaCompiler::FindFunctionBody(const NameAttrList& function,
+                                     const FunctionBody** fbody) {
+  // The function may be in either the local_flib_runtime_ or flib_runtime_.
+  // Look up the function in local first and if it is not found then look up the
+  // function in flib_runtime_.
+  auto status = GetFunctionBody(function, local_flib_runtime_, fbody);
+  if (!status.ok()) {
+    if (!errors::IsNotFound(status)) {
+      return status;
+    }
+    TF_RETURN_WITH_CONTEXT_IF_ERROR(
+        GetFunctionBody(function, flib_runtime_, fbody),
+        "Local lookup failed with: ", status.error_message());
+  }
+  return Status::OK();
+}
+
+std::unique_ptr<Graph> XlaCompiler::GetGraph(const FunctionBody* fbody) {
+  std::unique_ptr<Graph> graph(new Graph(options_.flib_def));
+  CopyGraph(*fbody->graph, graph.get());
+  OptimizerOptions opts;
+  opts.set_do_common_subexpression_elimination(true);
+  opts.set_do_function_inlining(true);
+  opts.set_do_constant_folding(true);
+  GraphOptimizer optimizer(opts);
+  optimizer.Optimize(flib_runtime_, flib_runtime_->env(),
+                     /*device=*/nullptr, &graph, /*shape_map=*/nullptr);
+
+  return graph;
+}
+
 Status XlaCompiler::CompileFunction(
     const XlaCompiler::CompileOptions& options, const NameAttrList& function,
     const std::vector<XlaCompiler::Argument>& args,
@@ -141,18 +175,7 @@ Status XlaCompiler::CompileFunction(
   }
 
   const FunctionBody* fbody;
-  // The function may be in either the local_flib_runtime_ or flib_runtime_.
-  // Look up the function in local first and if it is not found then look up the
-  // function in flib_runtime_.
-  auto status = GetFunctionBody(function, local_flib_runtime_, &fbody);
-  if (!status.ok()) {
-    if (!errors::IsNotFound(status)) {
-      return status;
-    }
-    TF_RETURN_WITH_CONTEXT_IF_ERROR(
-        GetFunctionBody(function, flib_runtime_, &fbody),
-        "Local lookup failed with: ", status.error_message());
-  }
+  TF_RETURN_IF_ERROR(FindFunctionBody(function, &fbody));
 
   TF_RETURN_IF_ERROR(CheckSignature(fbody->arg_types, args));
 
@@ -189,7 +212,7 @@ namespace {
 Status ExecuteGraph(XlaContext* xla_context, std::unique_ptr<Graph> graph,
                     XlaCompilationDevice* device, FunctionLibraryRuntime* flib,
                     int64 step_id) {
-  // Resource cleanup is a bit messy. XlaContext is a ref-counted resource; the
+  // Resource cleanup is a bit messy. XlaContext is a ref-countd resource; the
   // resource manager takes ownership via Create, and unrefs via Cleanup.  We
   // explicitly add a reference to ensure the refcount at entry is maintained at
   // all exit points; Create and Cleanup are always called in this function.
@@ -206,55 +229,12 @@ Status ExecuteGraph(XlaContext* xla_context, std::unique_ptr<Graph> graph,
       step_container->name(), XlaContext::kXlaContextResourceName,
       xla_context));
 
-  // Create a LocalExecutor that will own and run the graph.
-  // TODO(b/66947550): migrate away from using an Executor in order to guarantee
-  // determinism and thread-safety.
-  LocalExecutorParams exec_params;
-  exec_params.device = device;
-  exec_params.function_library = flib;
-  exec_params.create_kernel = [flib](const NodeDef& ndef, OpKernel** kernel) {
-    return flib->CreateKernel(ndef, kernel);
-  };
-  exec_params.delete_kernel = [](OpKernel* kernel) { delete kernel; };
-  Executor* exec_ptr = nullptr;
-  TF_RETURN_IF_ERROR(NewLocalExecutor(exec_params, graph.release(), &exec_ptr));
-  std::unique_ptr<Executor> exec(exec_ptr);
-  // At this point ownership of the graph has been transferred to exec.
-
-  // Run the graph symbolically, turning the graph into an XLA computation.
-  Executor::Args exec_args;
-  exec_args.step_id = step_id;
-  exec_args.step_container = step_container.get();
-
-  // Pushes closures to run onto `worklist`. We don't run the closures directly
-  // from 'runner' since that might lead to a stack overflow for large graphs.
-  std::deque<Executor::Args::Closure> worklist;
-  exec_args.runner = [&](Executor::Args::Closure c) {
-    worklist.push_back(std::move(c));
-  };
-
-  // The following code assumes there is only one thread involved and no
-  // concurrency, because we did not provide Executor a threaded runner. Async
-  // ops on the XlaCompilation device must not use threads or concurrency
-  // internally.
-  bool done = false;
-  exec->RunAsync(exec_args, [&](const Status& s) {
-    status = s;
-    done = true;
-  });
-  // Repeatedly run closures from the worklist until `done` is signalled.
-  while (!done) {
-    TF_RET_CHECK(!worklist.empty());
-    Executor::Args::Closure& c = worklist.front();
-    c();
-    worklist.pop_front();
-  }
-  TF_RETURN_WITH_CONTEXT_IF_ERROR(
-      status, "Conversion from TensorFlow graph to XLA computation failed.");
-
+  GraphCompiler graph_compiler(xla_context, device, graph.get(), flib,
+                               step_container.get());
+  TF_RETURN_IF_ERROR(graph_compiler.Compile());
   // Explicitly clean up the step container, to capture the cleanup status.
   step_container.reset();
-  return status;
+  return Status::OK();
 }
 
 // Builds XLA computations for each of the arguments to the computation.
diff --git a/tensorflow/compiler/tf2xla/xla_compiler.h b/tensorflow/compiler/tf2xla/xla_compiler.h
index addea74fc2..a8882a638c 100644
--- a/tensorflow/compiler/tf2xla/xla_compiler.h
+++ b/tensorflow/compiler/tf2xla/xla_compiler.h
@@ -29,7 +29,6 @@ limitations under the License.
 #include "tensorflow/core/public/version.h"
 
 namespace tensorflow {
-
 // The XlaCompiler class is responsible for compilation of a self-contained
 // subgraph of a TensorFlow computation using the XLA linear algebra runtime.
 // It does a symbolic execution of the graph starting from specific input
@@ -136,6 +135,27 @@ class XlaCompiler {
     bool operator==(const Argument& other) const;
   };
 
+  // Options pertaining to an individual call to CompileGraph() or
+  // CompileFunction().
+  struct CompileOptions {
+    // If `use_tuple_arg` is true, a single tuple parameter will be used for all
+    // arguments; if false, each argument gets its own parameter.
+    bool use_tuple_arg = false;
+
+    // If 'return_updated_values_for_all_resources' is true, then updated
+    // values of all resource arguments will be included in the
+    // 'resource_updates' of the computation, even if the resource was not
+    // modified by the computation. Used when compiling loop bodies to ensure
+    // the input and output signatures match.
+    bool return_updated_values_for_all_resources = false;
+
+    // If 'resolve_compile_time_constants' is true, then outputs of a
+    // computation that are known to be compile-time constants will be returned
+    // as Tensors at compile-time, rather than as run-time outputs of the
+    // computation.
+    bool resolve_compile_time_constants = true;
+  };
+
   struct OutputDescription {
     // Type and shape of the output.
     DataType type;
@@ -230,39 +250,9 @@ class XlaCompiler {
   };
 
   explicit XlaCompiler(Options options);
-  ~XlaCompiler();
 
-  // Options pertaining to an individual call to CompileGraph() or
-  // CompileFunction().
-  struct CompileOptions {
-    // If `use_tuple_arg` is true, a single tuple parameter will be used for all
-    // arguments; if false, each argument gets its own parameter.
-    bool use_tuple_arg = false;
-
-    // If 'return_updated_values_for_all_resources' is true, then updated
-    // values of all resource resources arguments will be included in the
-    // 'resource_updates' of the computation, even if the resource was not
-    // modified by the computation. Used when compiling loop bodies to ensure
-    // the input and output signatures match.
-    bool return_updated_values_for_all_resources = false;
-
-    // If 'resolve_compile_time_constants' is true, then outputs of a
-    // computation that are known to be compile-time constants will be returned
-    // as Tensors at compile-time, rather than as run-time outputs of the
-    // computation.
-    bool resolve_compile_time_constants = true;
-  };
+  ~XlaCompiler();
 
-  // Compiles a Tensorflow function `fn_name_attrs` into an XLA computation.
-  // `args` describes the arguments to the function, each of which must either
-  // be a runtime-parameter to the XLA computation, a compile-time constant, or
-  // a resource variable. Writes the compiled output to `result`.
-  //
-  // The generated XLA computation returns a tuple containing only the
-  // non-constant outputs as a function of the input arguments. Constant
-  // arguments are returned as host memory tensors in the output list and are
-  // not included in the XLA computation's outputs. The XLA computation is
-  // null if there are no data-dependent outputs and no side effects.
   Status CompileFunction(const CompileOptions& options,
                          const NameAttrList& fn_name_attrs,
                          const std::vector<Argument>& args,
@@ -276,10 +266,17 @@ class XlaCompiler {
                       const std::vector<Argument>& args,
                       CompilationResult* result);
 
+  Status PrepareArguments(xla::ComputationBuilder* builder, NameAttrList func,
+                          const std::vector<DataType>& types,
+                          const std::vector<TensorShape>& shapes,
+                          const std::vector<const XlaExpression*>& expressions,
+                          std::vector<Argument>* args);
+
   // Retrieves the channel handle associated with `key`. Allocates
   // a new channel handle if none exists.
-  // Channel handles can be used to communicate between different computations.
-  // Computations that communicate should be compiled with the same XlaCompiler.
+  // Channel handles can be used to communicate between different
+  // computations. Computations that communicate should be compiled with the
+  // same XlaCompiler.
   Status GetChannelHandle(const string& key, xla::ChannelHandle* channel);
 
   const Options& options() const { return options_; }
@@ -287,6 +284,16 @@ class XlaCompiler {
   FunctionLibraryRuntime* flib_runtime() const { return flib_runtime_; }
 
  private:
+  // Sets the function body `fbody` to the one registered as `function`.
+  Status FindFunctionBody(const NameAttrList& function,
+                          const FunctionBody** fbody);
+
+  // Returns the optimized graph object in this function body.
+  std::unique_ptr<Graph> GetGraph(const FunctionBody* fbody);
+
+  // Graph compiler needs to know how to get an optimized graph from a function
+  // body.
+  friend class GraphCompiler;
   friend class XlaCompilerTest;
 
   Options options_;
diff --git a/tensorflow/compiler/tf2xla/xla_compiler_test.cc b/tensorflow/compiler/tf2xla/xla_compiler_test.cc
index 9af557e23c..93aae8485d 100644
--- a/tensorflow/compiler/tf2xla/xla_compiler_test.cc
+++ b/tensorflow/compiler/tf2xla/xla_compiler_test.cc
@@ -27,6 +27,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/tests/literal_test_util.h"
 #include "tensorflow/core/common_runtime/function.h"
 #include "tensorflow/core/framework/common_shape_fns.h"
+#include "tensorflow/core/framework/function.h"
 #include "tensorflow/core/framework/function_testlib.h"
 #include "tensorflow/core/framework/resource_mgr.h"
 #include "tensorflow/core/framework/tensor_testutil.h"
@@ -95,6 +96,7 @@ class DummyReadResourceOp : public XlaOpKernel {
     dummy->Unref();
 
     ctx->SetOutput(0, ctx->Input(0));
+    ctx->SetOutput(1, ctx->Input(0));
   }
 };
 
@@ -112,22 +114,25 @@ class DummyReadResourceCC {
     if (!scope.ok()) return;
     scope.UpdateStatus(scope.DoShapeInference(ret));
     if (!scope.ok()) return;
-    this->output_ = Output(ret, 0);
+    this->output1_ = Output(ret, 0);
+    this->output2_ = Output(ret, 1);
   }
-  Node* node() const { return output_.node(); }
 
-  Output output_;
+  Output output1_;
+  Output output2_;
 };
 
 REGISTER_OP("DummyReadResource")
     .Input("input: int32")
-    .Output("output: int32")
+    .Output("output1: int32")
+    .Output("output2: int32")
     .SetShapeFn(shape_inference::UnknownShape)
     .Doc(R"doc(
 A dummy Op.
 
 input: dummy input.
-output: dummy output.
+output1: dummy output.
+output2: dummy output.
 )doc");
 
 REGISTER_XLA_OP(Name("DummyReadResource"), DummyReadResourceOp);
@@ -323,7 +328,8 @@ TEST_F(XlaCompilerTest, ResourceManager) {
   Scope scope = Scope::NewRootScope().ExitOnError();
   auto a = ops::_Arg(scope.WithOpName("A"), DT_INT32, 0);
   auto b = DummyReadResourceCC(scope.WithOpName("B"), a);
-  auto c = ops::_Retval(scope.WithOpName("C"), b.output_, 0);
+  auto c = ops::Add(scope.WithOpName("C"), b.output2_, b.output1_);
+  auto d = ops::_Retval(scope.WithOpName("D"), c, 0);
   std::unique_ptr<Graph> graph(new Graph(OpRegistry::Global()));
   TF_ASSERT_OK(scope.ToGraph(graph.get()));
 
@@ -356,6 +362,58 @@ TEST_F(XlaCompilerTest, ResourceManager) {
   resource->Unref();
 }
 
+// Tests compilation and execution of a graph that adds two tensors.
+TEST_F(XlaCompilerTest, DeterministicCompilation) {
+  // Builds a graph that contains a node with two output edges. The compiler
+  // should always traverse them in the same order.
+  const int64 test_count = 2;
+
+  std::vector<XlaCompiler::CompilationResult> results(test_count);
+
+  for (int64 i = 0; i < test_count; ++i) {
+    Scope scope = Scope::NewRootScope().ExitOnError();
+    auto a = ops::_Arg(scope.WithOpName("A"), DT_INT32, 0);
+    auto b = ops::Neg(scope.WithOpName("B"), a);
+    auto c = ops::Neg(scope.WithOpName("C"), a);
+    auto d = ops::Add(scope.WithOpName("D"), b, c);
+    auto e = ops::_Retval(scope.WithOpName("E"), d, 0);
+    std::unique_ptr<Graph> graph(new Graph(OpRegistry::Global()));
+    TF_ASSERT_OK(scope.ToGraph(graph.get()));
+
+    // Builds a description of the argument.
+    std::vector<XlaCompiler::Argument> args(1);
+    args[0].kind = XlaCompiler::Argument::kParameter;
+    args[0].type = DT_INT32;
+    args[0].shape = xla::ShapeUtil::MakeShape(xla::S32, {2});
+
+    // Compiles the graph.
+    auto options = DefaultOptions();
+    XlaCompiler compiler(options);
+
+    TF_ASSERT_OK(compiler.CompileGraph(XlaCompiler::CompileOptions(), "dummy",
+                                       std::move(graph), args, &results[i]));
+  }
+
+  for (int64 i = 1; i < test_count; ++i) {
+    auto m1 =
+        results[i - 1].computation->Snapshot().ValueOrDie()->entry().requests();
+    auto m2 =
+        results[i].computation->Snapshot().ValueOrDie()->entry().requests();
+    // Check if every entry is the same.
+    for (auto& entry1 : m1) {
+      int64 key = entry1.first;
+      auto value1 = entry1.second;
+      auto entry2 = m2.find(key);
+      auto value2 = entry2->second;
+      EXPECT_TRUE(entry2 != m2.end());
+      string str1, str2;
+      value1.AppendToString(&str1);
+      value2.AppendToString(&str2);
+      EXPECT_EQ(str1, str2);
+    }
+  }
+}
+
 // Tests a computation that receives a TensorArray resource as input and
 // updates it.
 TEST_F(XlaCompilerTest, CanPassTensorArraysToAndFromComputation) {
@@ -512,6 +570,63 @@ TEST_F(XlaCompilerTest, UndefinedFunctionFails) {
       << status.error_message();
 }
 
+FunctionDef FillFn() {
+  return FunctionDefHelper::Define(
+      // Name
+      "FillFn",
+      // Args
+      {"x: T", "dims: int32"},
+      // Return values
+      {"y: T"},
+      // Attr def
+      {"T: {float, double, int32, int64}"},
+      // Nodes
+      {{{"y"}, "Fill", {"dims", "x"}, {{"T", "$T"}}}});
+}
+
+TEST_F(XlaCompilerTest, FunctionCallWithConstants) {
+  // Certain operations in a function, "Fill" for example, requires the
+  // operator's argument to be a compile-time constant instead of a parameter.
+  // This testcase tests if XlaCompiler can handle such operators inside
+  // function calls.
+  XlaCompiler compiler(DefaultOptions());
+
+  FunctionDefLibrary flib;
+  *flib.add_function() = FillFn();
+
+  TF_ASSERT_OK(flib_def_->AddFunctionDef(FillFn()));
+
+  std::unique_ptr<Graph> graph(new Graph(OpRegistry::Global()));
+
+  Scope scope = Scope::NewRootScope().ExitOnError();
+  auto value = ops::Const<int32>(scope.WithOpName("value"), 1, {});
+  auto shape = ops::Const<int32>(scope.WithOpName("shape"), {5}, {1});
+  TF_EXPECT_OK(scope.graph()->AddFunctionLibrary(flib));
+
+  NodeDef def;
+  TF_ASSERT_OK(NodeDefBuilder("fill", "FillFn", flib_def_.get())
+                   .Input(value.name(), 0, DT_INT32)
+                   .Input(shape.name(), 1, DT_INT32)
+                   .Finalize(&def));
+  Status status;
+  Node* fill = scope.graph()->AddNode(def, &status);
+  TF_ASSERT_OK(status);
+  TF_ASSERT_OK(scope.DoShapeInference(fill));
+  scope.graph()->AddEdge(value.node(), 0, fill, 0);
+  scope.graph()->AddEdge(shape.node(), 0, fill, 1);
+
+  auto retval = ops::_Retval(scope.WithOpName("retval"), Output(fill), 0);
+
+  TF_ASSERT_OK(scope.ToGraph(graph.get()));
+
+  // Builds a description of the argument.
+  std::vector<XlaCompiler::Argument> args;
+
+  XlaCompiler::CompilationResult result;
+  TF_ASSERT_OK(compiler.CompileGraph(XlaCompiler::CompileOptions(), "fill",
+                                     std::move(graph), args, &result));
+}
+
 // Tests CompileFunction with a local function lookup failing, fails with
 // informative error about both lookups.
 TEST_F(XlaCompilerTest, LocalFunctionWithWrongArgumentsFail) {
diff --git a/tensorflow/compiler/xla/service/service.cc b/tensorflow/compiler/xla/service/service.cc
index bd7898a41f..d279e1f50f 100644
--- a/tensorflow/compiler/xla/service/service.cc
+++ b/tensorflow/compiler/xla/service/service.cc
@@ -187,8 +187,9 @@ tensorflow::Status Service::Computation(const ComputationRequest* arg,
 
   *result->mutable_computation() =
       computation_tracker_.NewComputation(arg->name());
-  VLOG(1) << Printf("Created new computation %s on service %p",
-                    result->computation().ShortDebugString().c_str(), this);
+  VLOG(1) << Printf("Created new computation %s on service %p, name %s",
+                    result->computation().ShortDebugString().c_str(), this,
+                    arg->name().c_str());
   return tensorflow::Status::OK();
 }
 
diff --git a/tensorflow/core/graph/algorithm.cc b/tensorflow/core/graph/algorithm.cc
index 3bfba3fc4e..6ef51aa7df 100644
--- a/tensorflow/core/graph/algorithm.cc
+++ b/tensorflow/core/graph/algorithm.cc
@@ -24,7 +24,8 @@ limitations under the License.
 namespace tensorflow {
 
 void DFS(const Graph& g, const std::function<void(Node*)>& enter,
-         const std::function<void(Node*)>& leave) {
+         const std::function<void(Node*)>& leave,
+         const NodeComparator& stable_comparator) {
   // Stack of work to do.
   struct Work {
     Node* node;
@@ -51,24 +52,41 @@ void DFS(const Graph& g, const std::function<void(Node*)>& enter,
     // Arrange to call leave(n) when all done with descendants.
     if (leave) stack.push_back(Work{n, true});
 
-    // Arrange to work on descendants.
-    for (Node* out : n->out_nodes()) {
+    gtl::iterator_range<NeighborIter> nodes = n->out_nodes();
+    auto add_work = [&visited, &stack](Node* out) {
       if (!visited[out->id()]) {
         // Note; we must not mark as visited until we actually process it.
         stack.push_back(Work{out, false});
       }
+    };
+
+    if (stable_comparator) {
+      std::vector<Node*> nodes_sorted;
+      for (Node* out : nodes) {
+        nodes_sorted.emplace_back(out);
+      }
+      std::sort(nodes_sorted.begin(), nodes_sorted.end(), stable_comparator);
+      for (Node* out : nodes_sorted) {
+        add_work(out);
+      }
+    } else {
+      for (Node* out : nodes) {
+        add_work(out);
+      }
     }
   }
 }
 
 void ReverseDFS(const Graph& g, const std::function<void(Node*)>& enter,
-                const std::function<void(Node*)>& leave) {
-  ReverseDFSFrom(g, {g.sink_node()}, enter, leave);
+                const std::function<void(Node*)>& leave,
+                const NodeComparator& stable_comparator) {
+  ReverseDFSFrom(g, {g.sink_node()}, enter, leave, stable_comparator);
 }
 
 void ReverseDFSFrom(const Graph& g, gtl::ArraySlice<Node*> start,
                     const std::function<void(Node*)>& enter,
-                    const std::function<void(Node*)>& leave) {
+                    const std::function<void(Node*)>& leave,
+                    const NodeComparator& stable_comparator) {
   // Stack of work to do.
   struct Work {
     Node* node;
@@ -97,23 +115,41 @@ void ReverseDFSFrom(const Graph& g, gtl::ArraySlice<Node*> start,
     // Arrange to call leave(n) when all done with descendants.
     if (leave) stack.push_back(Work{n, true});
 
-    // Arrange to work on parents.
-    for (Node* in : n->in_nodes()) {
-      if (!visited[in->id()]) {
+    gtl::iterator_range<NeighborIter> nodes = n->in_nodes();
+
+    auto add_work = [&visited, &stack](Node* out) {
+      if (!visited[out->id()]) {
         // Note; we must not mark as visited until we actually process it.
-        stack.push_back(Work{in, false});
+        stack.push_back(Work{out, false});
+      }
+    };
+
+    if (stable_comparator) {
+      std::vector<Node*> nodes_sorted;
+      for (Node* in : nodes) {
+        nodes_sorted.emplace_back(in);
+      }
+      std::sort(nodes_sorted.begin(), nodes_sorted.end(), stable_comparator);
+      for (Node* in : nodes_sorted) {
+        add_work(in);
+      }
+    } else {
+      for (Node* in : nodes) {
+        add_work(in);
       }
     }
   }
 }
 
-void GetPostOrder(const Graph& g, std::vector<Node*>* order) {
+void GetPostOrder(const Graph& g, std::vector<Node*>* order,
+                  const NodeComparator& stable_comparator) {
   order->clear();
-  DFS(g, nullptr, [order](Node* n) { order->push_back(n); });
+  DFS(g, nullptr, [order](Node* n) { order->push_back(n); }, stable_comparator);
 }
 
-void GetReversePostOrder(const Graph& g, std::vector<Node*>* order) {
-  GetPostOrder(g, order);
+void GetReversePostOrder(const Graph& g, std::vector<Node*>* order,
+                         const NodeComparator& stable_comparator) {
+  GetPostOrder(g, order, stable_comparator);
   std::reverse(order->begin(), order->end());
 }
 
diff --git a/tensorflow/core/graph/algorithm.h b/tensorflow/core/graph/algorithm.h
index 01d36e0a12..5bb6041d98 100644
--- a/tensorflow/core/graph/algorithm.h
+++ b/tensorflow/core/graph/algorithm.h
@@ -25,24 +25,50 @@ limitations under the License.
 
 namespace tensorflow {
 
+// Comparator for two nodes. This is used in order to get a stable ording.
+using NodeComparator = std::function<bool(const Node*, const Node*)>;
+
+// Compares two node based on their ids.
+struct NodeComparatorID {
+  bool operator()(const Node* n1, const Node* n2) const {
+    return n1->id() < n2->id();
+  }
+};
+
+// Compare two nodes based on their names.
+struct NodeComparatorName {
+  bool operator()(const Node* n1, const Node* n2) const {
+    return n1->name() < n2->name();
+  }
+};
+
 // Perform a depth-first-search on g starting at the source node.
 // If enter is not empty, calls enter(n) before visiting any children of n.
 // If leave is not empty, calls leave(n) after visiting all children of n.
+// If stable_comparator is set, a stable ordering of visit is achieved by
+// sorting a node's neighbors first before visiting them.
 extern void DFS(const Graph& g, const std::function<void(Node*)>& enter,
-                const std::function<void(Node*)>& leave);
+                const std::function<void(Node*)>& leave,
+                const NodeComparator& stable_comparator = {});
 
 // Perform a reverse depth-first-search on g starting at the sink node.
 // If enter is not empty, calls enter(n) before visiting any parents of n.
 // If leave is not empty, calls leave(n) after visiting all parents of n.
+// If stable_comparator is set, a stable ordering of visit is achieved by
+// sorting a node's neighbors first before visiting them.
 extern void ReverseDFS(const Graph& g, const std::function<void(Node*)>& enter,
-                       const std::function<void(Node*)>& leave);
+                       const std::function<void(Node*)>& leave,
+                       const NodeComparator& stable_comparator = {});
 
 // Perform a reverse depth-first-search on g starting at the 'start' nodes.
 // If enter is not empty, calls enter(n) before visiting any parents of n.
 // If leave is not empty, calls leave(n) after visiting all parents of n.
+// If stable_comparator is set, a stable ordering of visit is achieved by
+// sorting a node's neighbors first before visiting them.
 extern void ReverseDFSFrom(const Graph& g, gtl::ArraySlice<Node*> start,
                            const std::function<void(Node*)>& enter,
-                           const std::function<void(Node*)>& leave);
+                           const std::function<void(Node*)>& leave,
+                           const NodeComparator& stable_comparator = {});
 
 // Stores in *order the post-order numbering of all nodes
 // in graph found via a depth first search starting at the source node.
@@ -50,11 +76,18 @@ extern void ReverseDFSFrom(const Graph& g, gtl::ArraySlice<Node*> start,
 // Note that this is equivalent to reverse topological sorting when the
 // graph does not have cycles.
 //
+// If stable_comparator is set, a stable ordering of visit is achieved by
+// sorting a node's neighbors first before visiting them.
+//
 // REQUIRES: order is not NULL.
-void GetPostOrder(const Graph& g, std::vector<Node*>* order);
+void GetPostOrder(const Graph& g, std::vector<Node*>* order,
+                  const NodeComparator& stable_comparator = {});
 
 // Stores in *order the reverse post-order numbering of all nodes
-void GetReversePostOrder(const Graph& g, std::vector<Node*>* order);
+// If stable_comparator is set, a stable ordering of visit is achieved by
+// sorting a node's neighbors first before visiting them.
+void GetReversePostOrder(const Graph& g, std::vector<Node*>* order,
+                         const NodeComparator& stable_comparator = {});
 
 // Prune nodes in "g" that are not in some path from the source node
 // to any node in 'nodes'. Returns true if changes were made to the graph.
diff --git a/tensorflow/core/graph/algorithm_test.cc b/tensorflow/core/graph/algorithm_test.cc
index a529760426..0cdcdb6685 100644
--- a/tensorflow/core/graph/algorithm_test.cc
+++ b/tensorflow/core/graph/algorithm_test.cc
@@ -112,5 +112,40 @@ TEST(AlgorithmTest, ReversePostOrder) {
   EXPECT_FALSE(ExpectBefore(orders, order, &error));
 }
 
+TEST(AlgorithmTest, ReversePostOrderStable) {
+  int64 run_count = 100;
+  using namespace ::tensorflow::ops;  // NOLINT(build/namespaces)
+
+  for (int64 i = 0; i < run_count; ++i) {
+    // One source of nondeterminism comes from unordered set with key of a
+    // pointer type, for example the order of FlatSet<Node*> depends on the
+    // raw pointer value of Node. Stable post order suppose to remove this
+    // nondeterminism by enforcing an ordering based on node ids.
+    GraphDefBuilder b(GraphDefBuilder::kFailImmediately);
+    string error;
+    Node* w1 = SourceOp("TestParams", b.opts().WithName("W1"));
+    Node* input =
+        SourceOp("TestInput", b.opts().WithName("input").WithControlInput(w1));
+    BinaryOp("TestMul", w1, {input, 1}, b.opts().WithName("t2"));
+    // Insert different number of nodes between the allocation of t2 and t3,
+    // this creates enough entropy in the memory distance between t2 and t3 thus
+    // forces them to have randomized ordering had stable DFS was not
+    // implemented correctly.
+    for (int64 j = 0; j < i; ++j) {
+      BinaryOp("TestMul", w1, {input, 1},
+               b.opts().WithName(strings::StrCat("internal", j)));
+    }
+
+    BinaryOp("TestMul", w1, {input, 1}, b.opts().WithName("t3"));
+
+    Graph g(OpRegistry::Global());
+    TF_ASSERT_OK(b.ToGraph(&g));
+    std::vector<Node*> order;
+
+    // Test reverse post order generates expected ordering.
+    GetReversePostOrder(g, &order, /*stable_comparator=*/NodeComparatorID());
+    EXPECT_TRUE(ExpectBefore({{"t3", "t2"}}, order, &error));
+  }
+}
 }  // namespace
 }  // namespace tensorflow
diff --git a/tensorflow/core/graph/graph.h b/tensorflow/core/graph/graph.h
index 5a31a6216b..54076ed1ab 100644
--- a/tensorflow/core/graph/graph.h
+++ b/tensorflow/core/graph/graph.h
@@ -298,12 +298,12 @@ class Edge {
   Node* dst() const { return dst_; }
   int id() const { return id_; }
 
-  // Return the number of the source output that produces the data
+  // Return the index of the source output that produces the data
   // carried by this edge.  The special value kControlSlot is used
   // for control dependencies.
   int src_output() const { return src_output_; }
 
-  // Return the number of the destination input that consumes the data
+  // Return the index of the destination input that consumes the data
   // carried by this edge.  The special value kControlSlot is used
   // for control dependencies.
   int dst_input() const { return dst_input_; }
-- 
GitLab


From 76524d0b4ceb82d8833dfbe15ff2a07df5edddb1 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 12 Oct 2017 09:05:37 -0700
Subject: [PATCH 298/909] Add warm_starting_utils for Estimators.

PiperOrigin-RevId: 171966540
---
 tensorflow/python/estimator/BUILD             |  37 ++
 .../python/estimator/warm_starting_util.py    | 337 ++++++++++
 .../estimator/warm_starting_util_test.py      | 579 ++++++++++++++++++
 .../python/feature_column/feature_column.py   |  51 +-
 .../feature_column/feature_column_test.py     |  49 +-
 .../golden/tensorflow.feature_column.pbtxt    |   4 +-
 6 files changed, 1045 insertions(+), 12 deletions(-)
 create mode 100644 tensorflow/python/estimator/warm_starting_util.py
 create mode 100644 tensorflow/python/estimator/warm_starting_util_test.py

diff --git a/tensorflow/python/estimator/BUILD b/tensorflow/python/estimator/BUILD
index 2040d45cb6..e4b2d95acd 100644
--- a/tensorflow/python/estimator/BUILD
+++ b/tensorflow/python/estimator/BUILD
@@ -36,6 +36,7 @@ py_library(
         ":parsing_utils",
         ":run_config",
         ":training",
+        ":warm_starting_util",
         "//tensorflow/python:util",
     ],
 )
@@ -793,3 +794,39 @@ py_test(
         "//tensorflow/python:training",
     ],
 )
+
+py_library(
+    name = "warm_starting_util",
+    srcs = ["warm_starting_util.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:platform",
+        "//tensorflow/python:state_ops",
+        "//tensorflow/python:training",
+        "//tensorflow/python:variable_scope",
+        "//tensorflow/python:variables",
+        "//tensorflow/python/feature_column",
+    ],
+)
+
+py_test(
+    name = "warm_starting_util_test",
+    size = "small",
+    srcs = ["warm_starting_util_test.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":warm_starting_util",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:dtypes",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:init_ops",
+        "//tensorflow/python:training",
+        "//tensorflow/python:variable_scope",
+        "//tensorflow/python:variables",
+        "//tensorflow/python/feature_column",
+        "//third_party/py/numpy",
+    ],
+)
diff --git a/tensorflow/python/estimator/warm_starting_util.py b/tensorflow/python/estimator/warm_starting_util.py
new file mode 100644
index 0000000000..1ee77d6bbf
--- /dev/null
+++ b/tensorflow/python/estimator/warm_starting_util.py
@@ -0,0 +1,337 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Utilities to warm-start TF.Learn Estimators."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import collections
+import six
+
+from tensorflow.python.feature_column import feature_column
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import resource_variable_ops
+from tensorflow.python.ops import state_ops
+from tensorflow.python.ops import variable_scope
+from tensorflow.python.ops import variables
+from tensorflow.python.platform import tf_logging as logging
+from tensorflow.python.training import checkpoint_ops
+from tensorflow.python.training import checkpoint_utils
+from tensorflow.python.training import saver
+
+
+class _WarmStartSettings(
+    collections.namedtuple("_WarmStartSettings", [
+        "ckpt_to_initialize_from",
+        "col_to_prev_vocab",
+        "col_to_prev_tensor",
+        "exclude_columns",
+    ])):
+  """Settings for warm-starting input layer in models.
+
+  Attributes:
+    ckpt_to_initialize_from: [Required] A string specifying the directory with
+      checkpoint file(s) or path to checkpoint from which to warm-start the
+      model parameters.
+    col_to_prev_vocab: [Optional] Dict of `FeatureColumn` to path of the
+      vocabulary used for the `FeatureColumn` in `ckpt_to_initialize_from`. If
+      not explicitly provided, the vocabularies are assumed to be same between
+      previous and present checkpoints.
+    col_to_prev_tensor: [Optional] Dict of `FeatureColumn` to name of the
+      variable (corresponding to the `FeatureColumn`) in
+      `ckpt_to_initialize_from`. If not explicitly provided, the name of the
+      variable is assumed to be same between previous and present checkpoints.
+    exclude_columns: [Optional] List of `FeatureColumn`s that should not be
+      warm-started from provided checkpoint.
+
+  Example Uses:
+
+  # Feature columns defining transformations on inputs.
+  sc_vocab_file = tf.feature_column.categorical_column_with_vocabulary_file(
+      "sc_vocab_file", "new_vocab.txt", vocab_size=100)
+  sc_vocab_list = tf.feature_column.cateogorical_column_with_vocabulary_list(
+      "sc_vocab_list", vocabulary_list=["a", "b"])
+
+  # Warm-start all weights. The parameters corresponding to "sc_vocab_file" have
+  # the same name and same vocab as current checkpoint. The parameters
+  # corresponding to "sc_vocab_list" have the same name.
+  ws = _WarmStartSettings(ckpt_to_initialize_from="/tmp")
+
+  # Warm-start all weights but the parameters corresponding to "sc_vocab_file"
+  # have a different vocab from the one used in current checkpoint.
+  ws = _WarmStartSettings(ckpt_to_initialize_from="/tmp",
+                          col_to_prev_vocab={sc_vocab_file: "old_vocab.txt"})
+
+  # Warm-start all weights but the parameters corresponding to "sc_vocab_file"
+  # have a different vocab from the one used in current checkpoint and the
+  # parameters corresponding to "sc_vocab_list" have a different name from the
+  # current checkpoint.
+  ws = _WarmStartSettings(ckpt_to_initialize_from="/tmp",
+                          col_to_prev_vocab={sc_vocab_file: "old_vocab.txt"},
+                          col_to_prev_tensor={sc_vocab_list: "old_tensor_name"})
+
+  # Warm-start all weights except those corrresponding to "sc_vocab_file".
+  ws = _WarmStartSettings(ckpt_to_initialize_from="/tmp",
+                          exclude_columns=[sc_vocab_file])
+  """
+
+  def __new__(cls,
+              ckpt_to_initialize_from,
+              col_to_prev_vocab=None,
+              col_to_prev_tensor=None,
+              exclude_columns=None):
+    if not ckpt_to_initialize_from:
+      raise ValueError(
+          "`ckpt_to_initialize_from` MUST be set in _WarmStartSettings")
+    return super(_WarmStartSettings, cls).__new__(
+        cls,
+        ckpt_to_initialize_from,
+        col_to_prev_vocab or {},
+        col_to_prev_tensor or {},
+        exclude_columns or [],)
+
+
+def _is_variable(x):
+  return (isinstance(x, variables.Variable) or
+          isinstance(x, resource_variable_ops.ResourceVariable))
+
+
+def _infer_var_name(var):
+  """Returns name of the `var`.
+
+  Args:
+    var: A list. The list can contain either of the following:
+      (i) A single `Variable`
+      (ii) A single `ResourceVariable`
+      (iii) Multiple `Variable` objects which must be slices of the same larger
+        variable.
+      (iv) A single `PartitionedVariable`
+
+  Returns:
+    Name of the `var`
+  """
+  name_to_var_dict = saver.BaseSaverBuilder.OpListToDict(var)
+  if len(name_to_var_dict.keys()) > 1:
+    raise TypeError("`var` passed as arg violates the constraints.")
+  return list(name_to_var_dict.keys())[0]
+
+
+def _warmstart_var(var, prev_ckpt, prev_tensor_name=None):
+  """Warm-starts given variable from `prev_tensor_name` tensor in `prev_ckpt`.
+
+  Args:
+    var: Current graph's variable that needs to be warm-started (initialized).
+      Can be either of the following:
+      (i) `Variable`
+      (ii) `ResourceVariable`
+      (iii) list of `Variable`: The list must contain slices of the same larger
+        variable.
+      (iv) `PartitionedVariable`
+    prev_ckpt: A string specifying the directory with checkpoint file(s) or path
+      to checkpoint. The given checkpoint must have tensor with name
+      `prev_tensor_name` (if not None) or tensor with name same as given `var`.
+    prev_tensor_name: Name of the tensor to lookup in provided `prev_ckpt`. If
+      None, we lookup tensor with same name as given `var`.
+  """
+  if _is_variable(var):
+    current_var_name = _infer_var_name([var])
+  elif isinstance(var, list) and all(_is_variable(v) for v in var):
+    current_var_name = _infer_var_name(var)
+  elif isinstance(var, variables.PartitionedVariable):
+    current_var_name = _infer_var_name([var])
+    var = var._get_variable_list()  # pylint: disable=protected-access
+  else:
+    raise TypeError(
+        "var MUST be one of the following: a Variable, list of Variable or "
+        "PartitionedVariable, but is {}".format(type(var)))
+  if not prev_tensor_name:
+    # Assume tensor name remains the same.
+    prev_tensor_name = current_var_name
+  checkpoint_utils.init_from_checkpoint(prev_ckpt, {prev_tensor_name: var})
+
+
+# pylint: disable=protected-access
+# Accesses protected members of tf.Variable to reset the variable's internal
+# state.
+def _warmstart_var_with_vocab(var,
+                              current_vocab_path,
+                              current_vocab_size,
+                              prev_ckpt,
+                              prev_vocab_path,
+                              current_oov_buckets=0,
+                              prev_tensor_name=None):
+  """Warm-starts given variable from `prev_tensor_name` tensor in `prev_ckpt`.
+
+  Use this method when the `var` is backed by vocabulary. This method stitches
+  the given `var` such that values corresponding to individual features in the
+  vocabulary remain consistent irrespective of changing order of the features
+  between old and new vocabularies.
+
+  Args:
+    var: Current graph's variable that needs to be warm-started (initialized).
+      Can be either of the following:
+      (i) `Variable`
+      (ii) `ResourceVariable`
+      (iii) list of `Variable`: The list must contain slices of the same larger
+        variable.
+      (iv) `PartitionedVariable`
+    current_vocab_path: Path to the vocab file used for the given `var`.
+    current_vocab_size: An `int` specifying the number of entries in the current
+      vocab.
+    prev_ckpt: A string specifying the directory with checkpoint file(s) or path
+      to checkpoint. The given checkpoint must have tensor with name
+      `prev_tensor_name` (if not None) or tensor with name same as given `var`.
+    prev_vocab_path: Path to the vocab file used for the tensor in `prev_ckpt`.
+    current_oov_buckets: An `int` specifying the number of out-of-vocabulary
+      buckets used for given `var`.
+    prev_tensor_name: Name of the tensor to lookup in provided `prev_ckpt`. If
+      None, we lookup tensor with same name as given `var`.
+
+  Raises:
+    ValueError: If required args are not provided.
+  """
+  if not (current_vocab_path and current_vocab_size and prev_ckpt and
+          prev_vocab_path):
+    raise ValueError("Invalid args: Must provide all of [current_vocab_path, "
+                     "current_vocab_size, prev_ckpt, prev_vocab_path}.")
+  if _is_variable(var):
+    var = [var]
+  elif isinstance(var, list) and all(_is_variable(v) for v in var):
+    var = var
+  elif isinstance(var, variables.PartitionedVariable):
+    var = var._get_variable_list()
+  else:
+    raise TypeError(
+        "var MUST be one of the following: a Variable, list of Variable or "
+        "PartitionedVariable, but is {}".format(type(var)))
+
+  if not prev_tensor_name:
+    # Assume tensor name remains the same.
+    prev_tensor_name = _infer_var_name(var)
+
+  for v in var:
+    v_shape = v.get_shape().as_list()
+    slice_info = v._get_save_slice_info()
+    partition_info = None
+    if slice_info:
+      partition_info = variable_scope._PartitionInfo(
+          full_shape=slice_info.full_shape,
+          var_offset=slice_info.var_offset)
+
+    # TODO(vihanjain): This is brittle. Can we instead infer actual initializer
+    # used originally for the variable or use a fixed initializer?
+    def _missing_ids_init(shape, dtype=None):
+      # pylint: disable=cell-var-from-loop
+      if dtype and dtype.base_dtype != v.dtype.base_dtype:
+        raise ValueError("Trying to initialize missing ids with a different "
+                         "dtype `{}` than variable's dtype `{}`".format(
+                             dtype, v.dtype))
+      return array_ops.slice(v.initial_value, [0, 0], shape)
+
+      # pylint: enable=cell-var-from-loop
+
+    # TODO(vihanjain): Support _WarmstartSettings where class vocabularies need
+    # remapping too.
+    init = checkpoint_ops._load_and_remap_matrix_initializer(
+        ckpt_path=saver.latest_checkpoint(prev_ckpt),
+        old_tensor_name=prev_tensor_name,
+        new_row_vocab_size=current_vocab_size,
+        new_col_vocab_size=v_shape[1],
+        old_row_vocab_file=prev_vocab_path,
+        new_row_vocab_file=current_vocab_path,
+        old_col_vocab_file=None,
+        new_col_vocab_file=None,
+        num_row_oov_buckets=current_oov_buckets,
+        num_col_oov_buckets=0,
+        initializer=_missing_ids_init)
+    new_init_val = ops.convert_to_tensor(
+        init(shape=v_shape, partition_info=partition_info))
+    v._initializer_op = state_ops.assign(v, new_init_val)
+# pylint: enable=protected-access
+
+
+def _warmstart_input_layer(cols_to_vars, warmstart_settings):
+  """Warm-starts input layer of a model using given settings.
+
+  Args:
+    cols_to_vars: Dict of feature columns to corresponding graph variables.
+    warmstart_settings: An object of `_WarmStartSettings`.
+
+    Typical usage example:
+
+    ```python
+    tfcl = tf.contrib.layers
+    # Define features and transformations.
+    sc_vocab_list = tf.feature_column.categorical_column_with_vocabulary_list(
+        "sc_vocab_list", vocabulary_list=["a", "b"])
+    sc_vocab_file = tf.feature_column.categorical_column_with_vocabulary_file(
+        "sc_vocab_file", "new_vocab.txt", vocab_size=100)
+    cross = tf.feature_column.crossed_column(
+      [sc_vocab_list, sc_vocab_file], hash_bucket_size=5000)
+
+    all_cols = set(sc_vocab_list, sc_vocab_file, cross)
+    batch_features = tf.parse_example(
+        serialized=serialized_examples,
+        features=tf.contrib.layers.create_feature_spec_for_parsing(all_cols))
+
+    cols_to_vars = {}
+    tf.feature_column.linear_model(
+        features=batch_features,
+        feature_columns=all_cols,
+        units=1,
+        cols_to_vars=cols_to_vars)
+
+    # Warm-start entire input layer.
+    ws_settings = _WarmStartSettings(
+        "/tmp/prev_model_dir",
+        col_to_prev_vocab={sc_vocab_file: "old_vocab.txt"})
+    _warmstart_input_layer(cols_to_vars, ws_settings)
+    # Warm-start bias too.
+    _warmstart_var(cols_to_vars['bias'], ws_settings.ckpt_to_initialize_from)
+    ```
+
+    The above example effectively warm-starts full linear model.
+  """
+  for col, var in six.iteritems(cols_to_vars):
+    if not isinstance(col, feature_column._FeatureColumn):  # pylint: disable=protected-access
+      raise TypeError(
+          "Keys in dict `cols_to_vars` must be of type FeatureColumn. Found "
+          "key of type: {}".format(type(col)))
+    if col in warmstart_settings.exclude_columns:
+      logging.info("Skipping warm-starting column: {}".format(col.name))
+      continue
+
+    prev_tensor_name = warmstart_settings.col_to_prev_tensor.get(col)
+    if isinstance(col, feature_column._VocabularyFileCategoricalColumn):  # pylint: disable=protected-access
+      prev_vocab_path = warmstart_settings.col_to_prev_vocab.get(
+          col, col.vocabulary_file)
+      logging.info("Warm-starting column: {}; prev_vocab: {}; prev_tensor: {}".
+                   format(col.name, prev_vocab_path, (
+                       prev_tensor_name or "Unchanged")))
+      _warmstart_var_with_vocab(
+          var,
+          current_vocab_path=col.vocabulary_file,
+          current_vocab_size=col.vocabulary_size,
+          prev_ckpt=warmstart_settings.ckpt_to_initialize_from,
+          prev_vocab_path=prev_vocab_path,
+          current_oov_buckets=col.num_oov_buckets,
+          prev_tensor_name=prev_tensor_name)
+    else:
+      logging.info("Warm-starting column: {}; prev_tensor: {}".format(
+          col.name, prev_tensor_name or "Unchanged"))
+      _warmstart_var(var, warmstart_settings.ckpt_to_initialize_from,
+                     prev_tensor_name)
diff --git a/tensorflow/python/estimator/warm_starting_util_test.py b/tensorflow/python/estimator/warm_starting_util_test.py
new file mode 100644
index 0000000000..d4f1e3ac9d
--- /dev/null
+++ b/tensorflow/python/estimator/warm_starting_util_test.py
@@ -0,0 +1,579 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for warm_starting_util."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import os
+import numpy as np
+import six
+
+from tensorflow.python.estimator import warm_starting_util as ws_util
+from tensorflow.python.feature_column import feature_column as fc
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import init_ops
+from tensorflow.python.ops import variable_scope
+from tensorflow.python.ops import variables
+from tensorflow.python.platform import test
+from tensorflow.python.training import saver as saver_lib
+
+ones = init_ops.ones_initializer
+norms = init_ops.truncated_normal_initializer
+rand = init_ops.random_uniform_initializer
+
+
+class WarmStartingUtilTest(test.TestCase):
+
+  def _write_vocab(self, string_values, file_name):
+    vocab_file = os.path.join(self.get_temp_dir(), file_name)
+    with open(vocab_file, "w") as f:
+      f.write("\n".join(string_values))
+    return vocab_file
+
+  def _write_checkpoint(self, sess):
+    sess.run(variables.global_variables_initializer())
+    saver = saver_lib.Saver()
+    ckpt_prefix = os.path.join(self.get_temp_dir(), "model")
+    ckpt_state_name = "checkpoint"
+    saver.save(
+        sess, ckpt_prefix, global_step=0, latest_filename=ckpt_state_name)
+
+  def _create_prev_run_var(self,
+                           var_name,
+                           shape=None,
+                           initializer=None,
+                           partitioner=None):
+    with ops.Graph().as_default() as g:
+      with self.test_session(graph=g) as sess:
+        var = variable_scope.get_variable(
+            var_name,
+            shape=shape,
+            initializer=initializer,
+            partitioner=partitioner)
+        self._write_checkpoint(sess)
+        if partitioner:
+          self.assertTrue(isinstance(var, variables.PartitionedVariable))
+          var = var._get_variable_list()
+        return var, sess.run(var)
+
+  def _create_dummy_inputs(self):
+    return {
+        "sc_int": array_ops.sparse_placeholder(dtypes.int32),
+        "sc_hash": array_ops.sparse_placeholder(dtypes.string),
+        "sc_keys": array_ops.sparse_placeholder(dtypes.string),
+        "sc_vocab": array_ops.sparse_placeholder(dtypes.string),
+        "real": array_ops.placeholder(dtypes.float32)
+    }
+
+  def _create_linear_model(self, feature_cols, partitioner):
+    cols_to_vars = {}
+    with variable_scope.variable_scope("", partitioner=partitioner):
+      # Create the variables.
+      fc.linear_model(
+          features=self._create_dummy_inputs(),
+          feature_columns=feature_cols,
+          units=1,
+          cols_to_vars=cols_to_vars)
+    # Return a dictionary mapping each column to its variable, dropping the
+    # 'bias' key that's also filled.
+    cols_to_vars.pop("bias")
+    return cols_to_vars
+
+  def _assert_cols_to_vars(self, cols_to_vars, cols_to_expected_values, sess):
+    for col, expected_values in six.iteritems(cols_to_expected_values):
+      for i, var in enumerate(cols_to_vars[col]):
+        self.assertAllEqual(expected_values[i], var.eval(sess))
+
+  def testWarmStartVar(self):
+    _, prev_val = self._create_prev_run_var(
+        "fruit_weights", initializer=[[0.5], [1.], [1.5], [2.]])
+
+    with ops.Graph().as_default() as g:
+      with self.test_session(graph=g) as sess:
+        fruit_weights = variable_scope.get_variable(
+            "fruit_weights", initializer=[[0.], [0.], [0.], [0.]])
+        ws_util._warmstart_var(fruit_weights, self.get_temp_dir())
+        sess.run(variables.global_variables_initializer())
+        self.assertAllEqual(prev_val, fruit_weights.eval(sess))
+
+  def testWarmStartVarPrevVarPartitioned(self):
+    _, weights = self._create_prev_run_var(
+        "fruit_weights",
+        shape=[4, 1],
+        initializer=[[0.5], [1.], [1.5], [2.]],
+        partitioner=lambda shape, dtype: [2, 1])
+    prev_val = np.concatenate([weights[0], weights[1]], axis=0)
+
+    with ops.Graph().as_default() as g:
+      with self.test_session(graph=g) as sess:
+        fruit_weights = variable_scope.get_variable(
+            "fruit_weights", initializer=[[0.], [0.], [0.], [0.]])
+        ws_util._warmstart_var(fruit_weights, self.get_temp_dir())
+        sess.run(variables.global_variables_initializer())
+        self.assertAllEqual(prev_val, fruit_weights.eval(sess))
+
+  def testWarmStartVarCurrentVarPartitioned(self):
+    _, prev_val = self._create_prev_run_var(
+        "fruit_weights", initializer=[[0.5], [1.], [1.5], [2.]])
+
+    with ops.Graph().as_default() as g:
+      with self.test_session(graph=g) as sess:
+        fruit_weights = variable_scope.get_variable(
+            "fruit_weights",
+            shape=[4, 1],
+            initializer=[[0.], [0.], [0.], [0.]],
+            partitioner=lambda shape, dtype: [2, 1])
+        self.assertTrue(
+            isinstance(fruit_weights, variables.PartitionedVariable))
+        ws_util._warmstart_var(fruit_weights, self.get_temp_dir())
+        sess.run(variables.global_variables_initializer())
+        fruit_weights = fruit_weights._get_variable_list()
+        new_val = np.concatenate(
+            [fruit_weights[0].eval(sess), fruit_weights[1].eval(sess)], axis=0)
+        self.assertAllEqual(prev_val, new_val)
+
+  def testWarmStartVarBothVarsPartitioned(self):
+    _, weights = self._create_prev_run_var(
+        "old_scope/fruit_weights",
+        shape=[4, 1],
+        initializer=[[0.5], [1.], [1.5], [2.]],
+        partitioner=lambda shape, dtype: [2, 1])
+    prev_val = np.concatenate([weights[0], weights[1]], axis=0)
+    # New session and new graph.
+    with ops.Graph().as_default() as g:
+      with self.test_session(graph=g) as sess:
+        fruit_weights = variable_scope.get_variable(
+            "new_scope/fruit_weights",
+            shape=[4, 1],
+            initializer=[[0.], [0.], [0.], [0.]],
+            partitioner=lambda shape, dtype: [2, 1])
+        self.assertTrue(
+            isinstance(fruit_weights, variables.PartitionedVariable))
+        ws_util._warmstart_var(
+            fruit_weights,
+            self.get_temp_dir(),
+            prev_tensor_name="old_scope/fruit_weights")
+        sess.run(variables.global_variables_initializer())
+        fruit_weights = fruit_weights._get_variable_list()
+        new_val = np.concatenate(
+            [fruit_weights[0].eval(sess), fruit_weights[1].eval(sess)], axis=0)
+        self.assertAllEqual(prev_val, new_val)
+
+  def testWarmStartVarWithVocab(self):
+    prev_vocab_path = self._write_vocab(["apple", "banana", "guava", "orange"],
+                                        "old_vocab")
+    _, _ = self._create_prev_run_var(
+        "fruit_weights", initializer=[[0.5], [1.], [1.5], [2.]])
+
+    # New vocab with elements in reverse order and one new element.
+    new_vocab_path = self._write_vocab(
+        ["orange", "guava", "banana", "apple", "raspberry"], "new_vocab")
+    # New session and new graph.
+    with ops.Graph().as_default() as g:
+      with self.test_session(graph=g) as sess:
+        fruit_weights = variable_scope.get_variable(
+            "fruit_weights", initializer=[[0.], [0.], [0.], [0.], [0.]])
+        ws_util._warmstart_var_with_vocab(fruit_weights, new_vocab_path, 5,
+                                          self.get_temp_dir(), prev_vocab_path)
+        sess.run(variables.global_variables_initializer())
+        self.assertAllEqual([[2.], [1.5], [1.], [0.5], [0.]],
+                            fruit_weights.eval(sess))
+
+  def testWarmStartVarWithVocabPrevVarPartitioned(self):
+    prev_vocab_path = self._write_vocab(["apple", "banana", "guava", "orange"],
+                                        "old_vocab")
+    _, _ = self._create_prev_run_var(
+        "fruit_weights",
+        shape=[4, 1],
+        initializer=[[0.5], [1.], [1.5], [2.]],
+        partitioner=lambda shape, dtype: [2, 1])
+
+    # New vocab with elements in reverse order and one new element.
+    new_vocab_path = self._write_vocab(
+        ["orange", "guava", "banana", "apple", "raspberry"], "new_vocab")
+    # New session and new graph.
+    with ops.Graph().as_default() as g:
+      with self.test_session(graph=g) as sess:
+        fruit_weights = variable_scope.get_variable(
+            "fruit_weights", initializer=[[0.], [0.], [0.], [0.], [0.]])
+        ws_util._warmstart_var_with_vocab(fruit_weights, new_vocab_path, 5,
+                                          self.get_temp_dir(), prev_vocab_path)
+        sess.run(variables.global_variables_initializer())
+        self.assertAllEqual([[2.], [1.5], [1.], [0.5], [0.]],
+                            fruit_weights.eval(sess))
+
+  def testWarmStartVarWithVocabCurrentVarPartitioned(self):
+    prev_vocab_path = self._write_vocab(["apple", "banana", "guava", "orange"],
+                                        "old_vocab")
+    _, _ = self._create_prev_run_var(
+        "fruit_weights", initializer=[[0.5], [1.], [1.5], [2.]])
+
+    # New vocab with elements in reverse order and one new element.
+    new_vocab_path = self._write_vocab(
+        ["orange", "guava", "banana", "apple", "raspberry"], "new_vocab")
+    # New session and new graph.
+    with ops.Graph().as_default() as g:
+      with self.test_session(graph=g) as sess:
+        fruit_weights = variable_scope.get_variable(
+            "fruit_weights",
+            shape=[6, 1],
+            initializer=[[0.], [0.], [0.], [0.], [0.], [0.]],
+            partitioner=lambda shape, dtype: [2, 1])
+        ws_util._warmstart_var_with_vocab(
+            fruit_weights,
+            new_vocab_path,
+            5,
+            self.get_temp_dir(),
+            prev_vocab_path,
+            current_oov_buckets=1)
+        sess.run(variables.global_variables_initializer())
+        self.assertTrue(
+            isinstance(fruit_weights, variables.PartitionedVariable))
+        fruit_weights_vars = fruit_weights._get_variable_list()
+        self.assertAllEqual([[2.], [1.5], [1.]],
+                            fruit_weights_vars[0].eval(sess))
+        self.assertAllEqual([[0.5], [0.], [0.]],
+                            fruit_weights_vars[1].eval(sess))
+
+  def testWarmStartVarWithVocabBothVarsPartitioned(self):
+    prev_vocab_path = self._write_vocab(["apple", "banana", "guava", "orange"],
+                                        "old_vocab")
+    _, _ = self._create_prev_run_var(
+        "fruit_weights",
+        shape=[4, 1],
+        initializer=[[0.5], [1.], [1.5], [2.]],
+        partitioner=lambda shape, dtype: [2, 1])
+
+    # New vocab with elements in reverse order and two new elements.
+    new_vocab_path = self._write_vocab(
+        ["orange", "guava", "banana", "apple", "raspberry",
+         "blueberry"], "new_vocab")
+    # New session and new graph.
+    with ops.Graph().as_default() as g:
+      with self.test_session(graph=g) as sess:
+        fruit_weights = variable_scope.get_variable(
+            "fruit_weights",
+            shape=[6, 1],
+            initializer=[[0.], [0.], [0.], [0.], [0.], [0.]],
+            partitioner=lambda shape, dtype: [2, 1])
+        ws_util._warmstart_var_with_vocab(fruit_weights, new_vocab_path, 6,
+                                          self.get_temp_dir(), prev_vocab_path)
+        sess.run(variables.global_variables_initializer())
+        self.assertTrue(
+            isinstance(fruit_weights, variables.PartitionedVariable))
+        fruit_weights_vars = fruit_weights._get_variable_list()
+        self.assertAllEqual([[2.], [1.5], [1.]],
+                            fruit_weights_vars[0].eval(sess))
+        self.assertAllEqual([[0.5], [0.], [0.]],
+                            fruit_weights_vars[1].eval(sess))
+
+  def testWarmStartInputLayer_SparseColumnIntegerized(self):
+    # Create feature column.
+    sc_int = fc.categorical_column_with_identity("sc_int", num_buckets=10)
+
+    # Save checkpoint from which to warm-start.
+    _, prev_int_val = self._create_prev_run_var(
+        "linear_model/sc_int/weights", shape=[10, 1], initializer=ones())
+    # Verify we initialized the values correctly.
+    self.assertAllEqual(np.ones([10, 1]), prev_int_val)
+
+    partitioner = lambda shape, dtype: [1] * len(shape)
+    # New graph, new session WITHOUT warmstarting.
+    with ops.Graph().as_default() as g:
+      with self.test_session(graph=g) as sess:
+        cols_to_vars = self._create_linear_model([sc_int], partitioner)
+        sess.run(variables.global_variables_initializer())
+        # Without warmstarting, the weights should be initialized using default
+        # initializer (which is init_ops.zeros_initializer).
+        self._assert_cols_to_vars(cols_to_vars, {sc_int: [np.zeros([10, 1])]},
+                                  sess)
+
+    # New graph, new session with warmstarting.
+    with ops.Graph().as_default() as g:
+      with self.test_session(graph=g) as sess:
+        cols_to_vars = self._create_linear_model([sc_int], partitioner)
+        ws_util._warmstart_input_layer(cols_to_vars,
+                                       ws_util._WarmStartSettings(
+                                           self.get_temp_dir()))
+        sess.run(variables.global_variables_initializer())
+        # Verify weights were correctly warmstarted.
+        self._assert_cols_to_vars(cols_to_vars, {sc_int: [prev_int_val]}, sess)
+
+  def testWarmStartInputLayer_SparseColumnHashed(self):
+    # Create feature column.
+    sc_hash = fc.categorical_column_with_hash_bucket(
+        "sc_hash", hash_bucket_size=15)
+
+    # Save checkpoint from which to warm-start.
+    _, prev_hash_val = self._create_prev_run_var(
+        "linear_model/sc_hash/weights", shape=[15, 1], initializer=norms())
+
+    partitioner = lambda shape, dtype: [1] * len(shape)
+    # New graph, new session WITHOUT warmstarting.
+    with ops.Graph().as_default() as g:
+      with self.test_session(graph=g) as sess:
+        cols_to_vars = self._create_linear_model([sc_hash], partitioner)
+        sess.run(variables.global_variables_initializer())
+        # Without warmstarting, the weights should be initialized using default
+        # initializer (which is init_ops.zeros_initializer).
+        self._assert_cols_to_vars(cols_to_vars, {sc_hash: [np.zeros([15, 1])]},
+                                  sess)
+
+    # New graph, new session with warmstarting.
+    with ops.Graph().as_default() as g:
+      with self.test_session(graph=g) as sess:
+        cols_to_vars = self._create_linear_model([sc_hash], partitioner)
+        ws_util._warmstart_input_layer(cols_to_vars,
+                                       ws_util._WarmStartSettings(
+                                           self.get_temp_dir()))
+        sess.run(variables.global_variables_initializer())
+        # Verify weights were correctly warmstarted.
+        self._assert_cols_to_vars(cols_to_vars, {sc_hash: [prev_hash_val]},
+                                  sess)
+
+  def testWarmStartInputLayer_SparseColumnVocabulary(self):
+    # Create vocab for sparse column "sc_vocab".
+    vocab_path = self._write_vocab(["apple", "banana", "guava", "orange"],
+                                   "vocab")
+    # Create feature column.
+    sc_vocab = fc.categorical_column_with_vocabulary_file(
+        "sc_vocab", vocabulary_file=vocab_path, vocabulary_size=4)
+
+    # Save checkpoint from which to warm-start.
+    _, prev_vocab_val = self._create_prev_run_var(
+        "linear_model/sc_vocab/weights", shape=[4, 1], initializer=ones())
+
+    partitioner = lambda shape, dtype: [1] * len(shape)
+    # New graph, new session WITHOUT warmstarting.
+    with ops.Graph().as_default() as g:
+      with self.test_session(graph=g) as sess:
+        cols_to_vars = self._create_linear_model([sc_vocab], partitioner)
+        sess.run(variables.global_variables_initializer())
+        # Without warmstarting, the weights should be initialized using default
+        # initializer (which is init_ops.zeros_initializer).
+        self._assert_cols_to_vars(cols_to_vars, {sc_vocab: [np.zeros([4, 1])]},
+                                  sess)
+
+    # New graph, new session with warmstarting.
+    with ops.Graph().as_default() as g:
+      with self.test_session(graph=g) as sess:
+        cols_to_vars = self._create_linear_model([sc_vocab], partitioner)
+        # Since old vocab is not explicitly set in WarmStartSettings, the old
+        # vocab is assumed to be same as new vocab.
+        ws_util._warmstart_input_layer(cols_to_vars,
+                                       ws_util._WarmStartSettings(
+                                           self.get_temp_dir()))
+        sess.run(variables.global_variables_initializer())
+        # Verify weights were correctly warmstarted.
+        self._assert_cols_to_vars(cols_to_vars, {sc_vocab: [prev_vocab_val]},
+                                  sess)
+
+  def testWarmStartInputLayer_BucketizedColumn(self):
+    # Create feature column.
+    real = fc.numeric_column("real")
+    real_bucket = fc.bucketized_column(real, boundaries=[0., 1., 2., 3.])
+
+    # Save checkpoint from which to warm-start.
+    _, prev_bucket_val = self._create_prev_run_var(
+        "linear_model/real_bucketized/weights",
+        shape=[5, 1],
+        initializer=norms())
+
+    partitioner = lambda shape, dtype: [1] * len(shape)
+    # New graph, new session WITHOUT warmstarting.
+    with ops.Graph().as_default() as g:
+      with self.test_session(graph=g) as sess:
+        cols_to_vars = self._create_linear_model([real_bucket], partitioner)
+        sess.run(variables.global_variables_initializer())
+        # Without warmstarting, the weights should be initialized using default
+        # initializer (which is init_ops.zeros_initializer).
+        self._assert_cols_to_vars(cols_to_vars,
+                                  {real_bucket: [np.zeros([5, 1])]}, sess)
+
+    # New graph, new session with warmstarting.
+    with ops.Graph().as_default() as g:
+      with self.test_session(graph=g) as sess:
+        cols_to_vars = self._create_linear_model([real_bucket], partitioner)
+        ws_util._warmstart_input_layer(cols_to_vars,
+                                       ws_util._WarmStartSettings(
+                                           self.get_temp_dir()))
+        sess.run(variables.global_variables_initializer())
+        # Verify weights were correctly warmstarted.
+        self._assert_cols_to_vars(cols_to_vars,
+                                  {real_bucket: [prev_bucket_val]}, sess)
+
+  def testWarmStartInputLayer_MultipleCols(self):
+    # Create vocab for sparse column "sc_vocab".
+    vocab_path = self._write_vocab(["apple", "banana", "guava", "orange"],
+                                   "vocab")
+
+    # Create feature columns.
+    sc_int = fc.categorical_column_with_identity("sc_int", num_buckets=10)
+    sc_hash = fc.categorical_column_with_hash_bucket(
+        "sc_hash", hash_bucket_size=15)
+    sc_keys = fc.categorical_column_with_vocabulary_list(
+        "sc_keys", vocabulary_list=["a", "b", "c", "e"])
+    sc_vocab = fc.categorical_column_with_vocabulary_file(
+        "sc_vocab", vocabulary_file=vocab_path, vocabulary_size=4)
+    real = fc.numeric_column("real")
+    real_bucket = fc.bucketized_column(real, boundaries=[0., 1., 2., 3.])
+    cross = fc.crossed_column([sc_keys, sc_vocab], hash_bucket_size=20)
+    all_linear_cols = [sc_int, sc_hash, sc_keys, sc_vocab, real_bucket, cross]
+
+    # Save checkpoint from which to warm-start.
+    with ops.Graph().as_default() as g:
+      with self.test_session(graph=g) as sess:
+        sc_int_weights = variable_scope.get_variable(
+            "linear_model/sc_int/weights", shape=[10, 1], initializer=ones())
+        sc_hash_weights = variable_scope.get_variable(
+            "linear_model/sc_hash/weights", shape=[15, 1], initializer=norms())
+        sc_keys_weights = variable_scope.get_variable(
+            "linear_model/sc_keys/weights", shape=[4, 1], initializer=rand())
+        sc_vocab_weights = variable_scope.get_variable(
+            "linear_model/sc_vocab/weights", shape=[4, 1], initializer=ones())
+        real_bucket_weights = variable_scope.get_variable(
+            "linear_model/real_bucketized/weights",
+            shape=[5, 1],
+            initializer=norms())
+        cross_weights = variable_scope.get_variable(
+            "linear_model/sc_keys_X_sc_vocab/weights",
+            shape=[20, 1],
+            initializer=rand())
+        self._write_checkpoint(sess)
+        (prev_int_val, prev_hash_val, prev_keys_val, prev_vocab_val,
+         prev_bucket_val, prev_cross_val) = sess.run([
+             sc_int_weights, sc_hash_weights, sc_keys_weights, sc_vocab_weights,
+             real_bucket_weights, cross_weights
+         ])
+        # Verify we initialized the values correctly.
+        self.assertAllEqual(np.ones([10, 1]), prev_int_val)
+
+    partitioner = lambda shape, dtype: [1] * len(shape)
+    # New graph, new session WITHOUT warmstarting.
+    with ops.Graph().as_default() as g:
+      with self.test_session(graph=g) as sess:
+        cols_to_vars = self._create_linear_model(all_linear_cols, partitioner)
+        sess.run(variables.global_variables_initializer())
+        # Without warmstarting, all weights should be initialized using default
+        # initializer (which is init_ops.zeros_initializer).
+        self._assert_cols_to_vars(cols_to_vars, {
+            sc_int: [np.zeros([10, 1])],
+            sc_hash: [np.zeros([15, 1])],
+            sc_keys: [np.zeros([4, 1])],
+            sc_vocab: [np.zeros([4, 1])],
+            real_bucket: [np.zeros([5, 1])],
+            cross: [np.zeros([20, 1])],
+        }, sess)
+
+    # New graph, new session with warmstarting.
+    with ops.Graph().as_default() as g:
+      with self.test_session(graph=g) as sess:
+        cols_to_vars = self._create_linear_model(all_linear_cols, partitioner)
+        ws_util._warmstart_input_layer(cols_to_vars,
+                                       ws_util._WarmStartSettings(
+                                           self.get_temp_dir()))
+        sess.run(variables.global_variables_initializer())
+        # Verify weights were correctly warmstarted.
+        self._assert_cols_to_vars(cols_to_vars, {
+            sc_int: [prev_int_val],
+            sc_hash: [prev_hash_val],
+            sc_keys: [prev_keys_val],
+            sc_vocab: [prev_vocab_val],
+            real_bucket: [prev_bucket_val],
+            cross: [prev_cross_val],
+        }, sess)
+
+  def testWarmStartInputLayerMoreSettings(self):
+    # Create old and new vocabs for sparse column "sc_vocab".
+    prev_vocab_path = self._write_vocab(["apple", "banana", "guava", "orange"],
+                                        "old_vocab")
+    new_vocab_path = self._write_vocab(
+        ["orange", "guava", "banana", "apple", "raspberry",
+         "blueberry"], "new_vocab")
+    # Create feature columns.
+    sc_hash = fc.categorical_column_with_hash_bucket(
+        "sc_hash", hash_bucket_size=15)
+    sc_keys = fc.categorical_column_with_vocabulary_list(
+        "sc_keys", vocabulary_list=["a", "b", "c", "e"])
+    sc_vocab = fc.categorical_column_with_vocabulary_file(
+        "sc_vocab", vocabulary_file=new_vocab_path, vocabulary_size=6)
+    all_linear_cols = [sc_hash, sc_keys, sc_vocab]
+
+    # Save checkpoint from which to warm-start.
+    with ops.Graph().as_default() as g:
+      with self.test_session(graph=g) as sess:
+        _ = variable_scope.get_variable(
+            "linear_model/sc_hash/weights", shape=[15, 1], initializer=norms())
+        sc_keys_weights = variable_scope.get_variable(
+            "some_other_name", shape=[4, 1], initializer=rand())
+        _ = variable_scope.get_variable(
+            "linear_model/sc_vocab/weights",
+            initializer=[[0.5], [1.], [2.], [3.]])
+        self._write_checkpoint(sess)
+        prev_keys_val = sess.run(sc_keys_weights)
+
+    def _partitioner(shape, dtype):  # pylint:disable=unused-argument
+      # Partition each var into 2 equal slices.
+      partitions = [1] * len(shape)
+      partitions[0] = min(2, shape[0].value)
+      return partitions
+
+    # New graph, new session with warmstarting.
+    with ops.Graph().as_default() as g:
+      with self.test_session(graph=g) as sess:
+        cols_to_vars = self._create_linear_model(all_linear_cols, _partitioner)
+        ws_settings = ws_util._WarmStartSettings(
+            self.get_temp_dir(),
+            col_to_prev_vocab={sc_vocab: prev_vocab_path},
+            col_to_prev_tensor={sc_keys: "some_other_name"},
+            exclude_columns=[sc_hash])
+        ws_util._warmstart_input_layer(cols_to_vars, ws_settings)
+        sess.run(variables.global_variables_initializer())
+        # Verify weights were correctly warmstarted.  Var corresponding to
+        # sc_hash should not be warm-started.  Var corresponding to sc_vocab
+        # should be correctly warmstarted after vocab remapping.
+        self._assert_cols_to_vars(cols_to_vars, {
+            sc_keys:
+                np.split(prev_keys_val, 2),
+            sc_hash: [np.zeros([8, 1]), np.zeros([7, 1])],
+            sc_vocab: [
+                np.array([[3.], [2.], [1.]]),
+                np.array([[0.5], [0.], [0.]])
+            ]
+        }, sess)
+
+  def testErrorConditions(self):
+    self.assertRaises(ValueError, ws_util._WarmStartSettings, None)
+    x = variable_scope.get_variable(
+        "x",
+        shape=[4, 1],
+        initializer=ones(),
+        partitioner=lambda shape, dtype: [2, 1])
+
+    # List of PartitionedVariable is invalid type.
+    self.assertRaises(TypeError, ws_util._warmstart_var, [x], prev_ckpt="/tmp")
+    self.assertRaises(TypeError, ws_util._warmstart_var_with_vocab, [x], "/tmp",
+                      5, "/tmp", "/tmp")
+    # Keys of type other than FeatureColumn.
+    self.assertRaises(TypeError, ws_util._warmstart_input_layer,
+                      {"StringType": x}, ws_util._WarmStartSettings("/tmp"))
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/python/feature_column/feature_column.py b/tensorflow/python/feature_column/feature_column.py
index 5746883284..81f4f45fcb 100644
--- a/tensorflow/python/feature_column/feature_column.py
+++ b/tensorflow/python/feature_column/feature_column.py
@@ -159,7 +159,8 @@ from tensorflow.python.util import nest
 def input_layer(features,
                 feature_columns,
                 weight_collections=None,
-                trainable=True):
+                trainable=True,
+                cols_to_vars=None):
   """Returns a dense `Tensor` as input layer based on given `feature_columns`.
 
   Generally a single example in training data is described with FeatureColumns.
@@ -195,6 +196,14 @@ def input_layer(features,
       `tf.GraphKeys.GLOBAL_VARIABLES` and `ops.GraphKeys.MODEL_VARIABLES`.
     trainable: If `True` also add the variable to the graph collection
       `GraphKeys.TRAINABLE_VARIABLES` (see `tf.Variable`).
+    cols_to_vars: If not `None`, must be a dictionary that will be filled with a
+      mapping from `_FeatureColumn` to associated `Variable` (or list of
+      `Variable`, or `PartitionedVariable`.  For example, after the call, we
+      might have cols_to_vars = {_EmbeddingColumn(
+        categorical_column=_HashedCategoricalColumn(
+          key='sparse_feature', hash_bucket_size=5, dtype=tf.string),
+        dimension=10): [<tf.Variable 'some_variable' shape=(5, 10)]}
+      If a column creates no variables, its value will be an empty list.
 
   Returns:
     A `Tensor` which represents input layer of a model. Its shape
@@ -228,6 +237,12 @@ def input_layer(features,
             builder,
             weight_collections=weight_collections,
             trainable=trainable)
+        if cols_to_vars is not None:
+          # Retrieve any variables created (some _DenseColumn's don't create
+          # variables, in which case an empty list is returned).
+          cols_to_vars[column] = ops.get_collection(
+              ops.GraphKeys.GLOBAL_VARIABLES,
+              scope=variable_scope.get_variable_scope().name)
         num_elements = column._variable_shape.num_elements()  # pylint: disable=protected-access
         batch_size = array_ops.shape(tensor)[0]
         tensor = array_ops.reshape(tensor, shape=(batch_size, num_elements))
@@ -241,7 +256,8 @@ def linear_model(features,
                  units=1,
                  sparse_combiner='sum',
                  weight_collections=None,
-                 trainable=True):
+                 trainable=True,
+                 cols_to_vars=None):
   """Returns a linear prediction `Tensor` based on given `feature_columns`.
 
   This function generates a weighted sum based on output dimension `units`.
@@ -285,6 +301,19 @@ def linear_model(features,
       `tf.GraphKeys.GLOBAL_VARIABLES` and `ops.GraphKeys.MODEL_VARIABLES`.
     trainable: If `True` also add the variable to the graph collection
       `GraphKeys.TRAINABLE_VARIABLES` (see `tf.Variable`).
+    cols_to_vars: If not `None`, must be a dictionary that will be filled with a
+      mapping from `_FeatureColumn` to associated `Variable` (or list of
+      `Variable`, or `PartitionedVariable`.  For example,
+      after the call, we might have cols_to_vars = {
+        _NumericColumn(
+          key='numeric_feature1', shape=(1,):
+        <tf.Variable 'linear_model/price2/weights:0' shape=(1, 1)>,
+        'bias': <tf.Variable 'linear_model/bias_weights:0' shape=(1,)>,
+        _NumericColumn(
+          key='numeric_feature2', shape=(2,)):
+        <tf.Variable 'linear_model/price1/weights:0' shape=(2, 1)>}
+      Note that it will also contain a string key 'bias'.  If a column creates
+      no variables, its value will be an empty list.
 
   Returns:
     A `Tensor` which represents predictions/logits of a linear model. Its shape
@@ -313,12 +342,18 @@ def linear_model(features,
       with variable_scope.variable_scope(None, default_name=column.name):
         ordered_columns.append(column)
         if isinstance(column, _CategoricalColumn):
-          weighted_sums.append(_create_categorical_column_weighted_sum(
+          weighted_sum = _create_categorical_column_weighted_sum(
               column, builder, units, sparse_combiner, weight_collections,
-              trainable))
+              trainable)
         else:
-          weighted_sums.append(_create_dense_column_weighted_sum(
-              column, builder, units, weight_collections, trainable))
+          weighted_sum = _create_dense_column_weighted_sum(
+              column, builder, units, weight_collections, trainable)
+        weighted_sums.append(weighted_sum)
+        if cols_to_vars is not None:
+          # Retrieve the variables created.
+          cols_to_vars[column] = ops.get_collection(
+              ops.GraphKeys.GLOBAL_VARIABLES,
+              scope=variable_scope.get_variable_scope().name)
     _verify_static_batch_size_equality(weighted_sums, ordered_columns)
     predictions_no_bias = math_ops.add_n(
         weighted_sums, name='weighted_sum_no_bias')
@@ -330,7 +365,9 @@ def linear_model(features,
         collections=weight_collections)
     predictions = nn_ops.bias_add(
         predictions_no_bias, bias, name='weighted_sum')
-
+    if cols_to_vars is not None:
+      # Add the bias to cols_to_vars as well.
+      cols_to_vars['bias'] = bias
     return predictions
 
 
diff --git a/tensorflow/python/feature_column/feature_column_test.py b/tensorflow/python/feature_column/feature_column_test.py
index 7eb8e8051d..112600439b 100644
--- a/tensorflow/python/feature_column/feature_column_test.py
+++ b/tensorflow/python/feature_column/feature_column_test.py
@@ -221,12 +221,12 @@ class NumericColumnTest(test.TestCase):
               0,
           ])
 
-  def test_dtype_is_convertable_to_float(self):
+  def test_dtype_is_convertible_to_float(self):
     with self.assertRaisesRegexp(ValueError,
                                  'dtype must be convertible to float'):
       fc.numeric_column('aaa', dtype=dtypes.string)
 
-  def test_scalar_deafult_value_fills_the_shape(self):
+  def test_scalar_default_value_fills_the_shape(self):
     a = fc.numeric_column('aaa', shape=[2, 3], default_value=2.)
     self.assertEqual(((2., 2., 2.), (2., 2., 2.)), a.default_value)
 
@@ -1344,6 +1344,20 @@ class LinearModelTest(test.TestCase):
         sess.run(bias.assign([7.]))
         self.assertAllClose([[3217.], [4657.]], predictions.eval())
 
+  def test_fills_cols_to_vars(self):
+    price1 = fc.numeric_column('price1', shape=2)
+    price2 = fc.numeric_column('price2')
+    with ops.Graph().as_default():
+      features = {'price1': [[1., 2.], [5., 6.]], 'price2': [[3.], [4.]]}
+      cols_to_vars = {}
+      fc.linear_model(features, [price1, price2], cols_to_vars=cols_to_vars)
+      bias = get_linear_model_bias()
+      price1_var = get_linear_model_column_var(price1)
+      price2_var = get_linear_model_column_var(price2)
+      self.assertEqual(cols_to_vars['bias'], bias)
+      self.assertAllEqual(cols_to_vars[price1], [price1_var])
+      self.assertAllEqual(cols_to_vars[price2], [price2_var])
+
   def test_dense_collection(self):
     price = fc.numeric_column('price')
     with ops.Graph().as_default() as g:
@@ -1722,6 +1736,35 @@ class InputLayerTest(test.TestCase):
       with _initialized_session():
         self.assertAllClose([[1., 2., 3.], [5., 6., 4.]], net.eval())
 
+  def test_fills_cols_to_vars(self):
+    # Provide three _DenseColumn's to input_layer: a _NumericColumn, a
+    # _BucketizedColumn, and an _EmbeddingColumn.  Only the _EmbeddingColumn
+    # creates a Variable.
+    price1 = fc.numeric_column('price1')
+    dense_feature = fc.numeric_column('dense_feature')
+    dense_feature_bucketized = fc.bucketized_column(
+        dense_feature, boundaries=[0.])
+    some_sparse_column = fc.categorical_column_with_hash_bucket(
+        'sparse_feature', hash_bucket_size=5)
+    some_embedding_column = fc.embedding_column(
+        some_sparse_column, dimension=10)
+    with ops.Graph().as_default():
+      features = {
+          'price1': [[3.], [4.]],
+          'dense_feature': [[-1.], [4.]],
+          'sparse_feature': [['a'], ['x']],
+      }
+      cols_to_vars = {}
+      all_cols = [price1, dense_feature_bucketized, some_embedding_column]
+      fc.input_layer(features, all_cols, cols_to_vars=cols_to_vars)
+      self.assertItemsEqual(list(cols_to_vars.keys()), all_cols)
+      self.assertEqual(0, len(cols_to_vars[price1]))
+      self.assertEqual(0, len(cols_to_vars[dense_feature_bucketized]))
+      self.assertEqual(1, len(cols_to_vars[some_embedding_column]))
+      for var in cols_to_vars[some_embedding_column]:
+        self.assertIsInstance(var, variables_lib.Variable)
+        self.assertAllEqual(var.shape, [5, 10])
+
   def test_column_order(self):
     price_a = fc.numeric_column('price_a')
     price_b = fc.numeric_column('price_b')
@@ -3224,7 +3267,7 @@ class IndicatorColumnTest(test.TestCase):
   def test_transform_with_weighted_column(self):
     # Github issue 12557
     ids = fc.categorical_column_with_vocabulary_list(
-      key='ids', vocabulary_list=('a', 'b', 'c'))
+        key='ids', vocabulary_list=('a', 'b', 'c'))
     weights = fc.weighted_categorical_column(ids, 'weights')
     indicator = fc.indicator_column(weights)
     features = {
diff --git a/tensorflow/tools/api/golden/tensorflow.feature_column.pbtxt b/tensorflow/tools/api/golden/tensorflow.feature_column.pbtxt
index 2a57a845cd..9eb4cb8ce9 100644
--- a/tensorflow/tools/api/golden/tensorflow.feature_column.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.feature_column.pbtxt
@@ -34,11 +34,11 @@ tf_module {
   }
   member_method {
     name: "input_layer"
-    argspec: "args=[\'features\', \'feature_columns\', \'weight_collections\', \'trainable\'], varargs=None, keywords=None, defaults=[\'None\', \'True\'], "
+    argspec: "args=[\'features\', \'feature_columns\', \'weight_collections\', \'trainable\', \'cols_to_vars\'], varargs=None, keywords=None, defaults=[\'None\', \'True\', \'None\'], "
   }
   member_method {
     name: "linear_model"
-    argspec: "args=[\'features\', \'feature_columns\', \'units\', \'sparse_combiner\', \'weight_collections\', \'trainable\'], varargs=None, keywords=None, defaults=[\'1\', \'sum\', \'None\', \'True\'], "
+    argspec: "args=[\'features\', \'feature_columns\', \'units\', \'sparse_combiner\', \'weight_collections\', \'trainable\', \'cols_to_vars\'], varargs=None, keywords=None, defaults=[\'1\', \'sum\', \'None\', \'True\', \'None\'], "
   }
   member_method {
     name: "make_parse_example_spec"
-- 
GitLab


From fa8c1a1f30eed9ea35e1339eb94515cefc97582f Mon Sep 17 00:00:00 2001
From: Yun Peng <pcloudy@google.com>
Date: Thu, 12 Oct 2017 18:23:44 +0200
Subject: [PATCH 299/909] Add missing default config setting in aws.BUILD
 (#13662)

---
 third_party/aws.BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/third_party/aws.BUILD b/third_party/aws.BUILD
index 38b7e0e543..9d8e7946cd 100644
--- a/third_party/aws.BUILD
+++ b/third_party/aws.BUILD
@@ -18,6 +18,7 @@ cc_library(
         "@%ws%//tensorflow:darwin": glob([
             "aws-cpp-sdk-core/source/platform/linux-shared/*.cpp",
         ]),
+        "//conditions:default": [],
     }) + glob([
         "aws-cpp-sdk-core/include/**/*.h",
         "aws-cpp-sdk-core/source/*.cpp",
-- 
GitLab


From ba8b9ac9ff7e58050376ef22435ad1eed85f5a92 Mon Sep 17 00:00:00 2001
From: Yun Peng <pcloudy@google.com>
Date: Thu, 12 Oct 2017 18:33:15 +0200
Subject: [PATCH 300/909] configure.py: Disable AWS support on Windows by
 default

---
 configure.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/configure.py b/configure.py
index c1afdc1f07..95835e538b 100644
--- a/configure.py
+++ b/configure.py
@@ -976,6 +976,7 @@ def main():
   run_gen_git_source(environ_cp)
 
   if is_windows():
+    environ_cp['TF_NEED_S3'] = '0'
     environ_cp['TF_NEED_GCP'] = '0'
     environ_cp['TF_NEED_HDFS'] = '0'
     environ_cp['TF_NEED_JEMALLOC'] = '0'
-- 
GitLab


From d6b616925657bc44de9bd6a5ddf3437e9c7ba88b Mon Sep 17 00:00:00 2001
From: Eugene Brevdo <ebrevdo@google.com>
Date: Thu, 12 Oct 2017 09:56:28 -0700
Subject: [PATCH 301/909] Wrap grad_ys tensors passed to tf.gradients in the
 tf.gradients name scope.

Fixes #13355.

PiperOrigin-RevId: 171972633
---
 .../kernel_tests/tensor_array_ops_test.py     | 18 ++++++++
 tensorflow/python/ops/gradients_impl.py       | 42 ++++++++++++++-----
 tensorflow/python/ops/gradients_test.py       |  9 ++--
 3 files changed, 55 insertions(+), 14 deletions(-)

diff --git a/tensorflow/python/kernel_tests/tensor_array_ops_test.py b/tensorflow/python/kernel_tests/tensor_array_ops_test.py
index fc4f9b22b9..53e045fe86 100644
--- a/tensorflow/python/kernel_tests/tensor_array_ops_test.py
+++ b/tensorflow/python/kernel_tests/tensor_array_ops_test.py
@@ -1362,6 +1362,24 @@ class TensorArrayTest(test.TestCase):
       self.assertEqual(size0_v, 2)
       self.assertEqual(size1_v, 4)
 
+  def testTensorArrayGradYsInCorrectScope(self):
+    n_time = 1
+    n_dim = 1
+    x = constant_op.constant([[1.42]])
+    dy = constant_op.constant([[2.42]])
+
+    ta = tensor_array_ops.TensorArray(
+        dtypes.float32, size=n_time, element_shape=[n_dim])
+    for t in range(n_time):
+      ta = ta.write(index=t, value=x[t])
+      y = ta.stack()
+      # dy is outside of the gradients name scope; tf.gradients must
+      # wrap it in the correct name scope.
+      dx, = gradients_impl.gradients(ys=[y], xs=[x], grad_ys=[dy])
+      with self.test_session(use_gpu=True) as sess:
+        vdx, vdy = sess.run([dx, dy])
+      self.assertAllClose(vdx, vdy)
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/ops/gradients_impl.py b/tensorflow/python/ops/gradients_impl.py
index d9b14de984..f7b72eb82f 100644
--- a/tensorflow/python/ops/gradients_impl.py
+++ b/tensorflow/python/ops/gradients_impl.py
@@ -223,6 +223,7 @@ def _DefaultGradYs(grad_ys, ys, colocate_gradients_with_ops):
   if len(grad_ys) != len(ys):
     raise ValueError("Passed %d grad_ys for %d ys" % (len(grad_ys), len(ys)))
   grad_ys = ops.convert_n_to_tensor_or_indexed_slices(grad_ys, name="grad_y")
+  new_grad_ys = []
   for i in xrange(len(grad_ys)):
     grad_y = grad_ys[i]
     y = ys[i]
@@ -232,28 +233,49 @@ def _DefaultGradYs(grad_ys, ys, colocate_gradients_with_ops):
             "Gradients of complex tensors must set grad_ys (y.dtype = %r)" %
             y.dtype)
       with _maybe_colocate_with(y.op, colocate_gradients_with_ops):
-        grad_ys[i] = array_ops.fill(
+        new_grad_ys.append(array_ops.fill(
             array_ops.shape(y), constant_op.constant(
-                1, dtype=y.dtype))
+                1, dtype=y.dtype, name="grad_ys_%d" % i)))
       continue
     if y.dtype.is_floating or y.dtype.is_integer:
       if not grad_y.dtype.is_floating and not grad_y.dtype.is_integer:
         raise TypeError("Gradient type %s generated for real or "
-                         "integer-valued tensor %s with type %s must be "
-                         "real or integer" %
-                         (dtypes.as_dtype(grad_y.dtype).name, y,
-                          dtypes.as_dtype(y.dtype).name))
+                        "integer-valued tensor %s with type %s must be "
+                        "real or integer" %
+                        (dtypes.as_dtype(grad_y.dtype).name, y,
+                         dtypes.as_dtype(y.dtype).name))
     elif y.dtype.is_complex:
       if not grad_y.dtype.is_complex:
         raise TypeError("Gradient type %s generated for complex-valued "
-                         "tensor %s with type %s must be real" %
-                         (dtypes.as_dtype(grad_y.dtype).name, y,
-                          dtypes.as_dtype(y.dtype).name))
+                        "tensor %s with type %s must be real" %
+                        (dtypes.as_dtype(grad_y.dtype).name, y,
+                         dtypes.as_dtype(y.dtype).name))
     else:
       raise TypeError("Tensor %s with type %s must be numeric "
                       "to obtain a default gradient" %
                       (y, dtypes.as_dtype(y.dtype).name))
-  return grad_ys
+    # Create a grad_y tensor in the name scope of the gradient.
+    # Required for TensorArrays to identify which gradient call a
+    # grad_y value is coming from.
+    if isinstance(grad_y, ops.IndexedSlices):
+      new_grad_ys.append(
+          ops.IndexedSlices(
+              indices=(array_ops.identity(grad_y.indices,
+                                          name="grad_ys_%d_indices" % i)
+                       if isinstance(grad_y.indices, ops.Tensor)
+                       else grad_y.indices),
+              values=(array_ops.identity(grad_y.values,
+                                         name="grad_ys_%d_values" % i)
+                      if isinstance(grad_y.values, ops.Tensor)
+                      else grad_y.values),
+              dense_shape=(array_ops.identity(grad_y.dense_shape,
+                                              name="grad_ys_%d_shape" % i)
+                           if isinstance(grad_y.dense_shape, ops.Tensor)
+                           else grad_y.dense_shape)))
+    else:
+      new_grad_ys.append(array_ops.identity(grad_y, name="grad_ys_%d" % i))
+
+  return new_grad_ys
 
 
 def _IsTrainable(tensor):
diff --git a/tensorflow/python/ops/gradients_test.py b/tensorflow/python/ops/gradients_test.py
index 7a561d046a..de3dd03486 100644
--- a/tensorflow/python/ops/gradients_test.py
+++ b/tensorflow/python/ops/gradients_test.py
@@ -291,10 +291,11 @@ class GradientsTest(test_util.TensorFlowTestCase):
           array_ops.placeholder(dtypes.float32),
           array_ops.placeholder(dtypes.int32))
       dx, = gradients.gradients(y, x, grad_ys=dy)
-      # The gradient of tf.identity should pass the value through unchanged.
-      # A previous version of the code did this only for tf.Tensor, not
-      # tf.IndexedSlices.
-      self.assertEqual(dx, dy)
+      # The IndexedSlices gradient of tf.identity is the identity map.
+      with self.test_session() as sess:
+        vdx, vdy = sess.run(
+            [dx, dy], feed_dict={x: [1.0], dy.indices: [0], dy.values: [2.0]})
+      self.assertEqual(vdx, vdy)
 
   def testNonDifferentiableSwitchInWhileLoop(self):
     with ops.Graph().as_default():
-- 
GitLab


From aa7532c73e3b099fac0d7960b0ee561e54825452 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 12 Oct 2017 10:01:10 -0700
Subject: [PATCH 302/909] eager: Expose tfe.run_test_in_graph_and_eager_modes
 decorator. Useful when writing tests of libraries.

PiperOrigin-RevId: 171973311
---
 tensorflow/contrib/eager/python/tfe.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/tensorflow/contrib/eager/python/tfe.py b/tensorflow/contrib/eager/python/tfe.py
index fbdc576739..1acb1ba1b8 100644
--- a/tensorflow/contrib/eager/python/tfe.py
+++ b/tensorflow/contrib/eager/python/tfe.py
@@ -50,6 +50,8 @@ To use, at program startup, call `tfe.enable_eager_execution()`.
 
 @@in_eager_mode
 @@in_graph_mode
+
+@@run_test_in_graph_and_eager_modes
 """
 
 from __future__ import absolute_import
@@ -60,12 +62,10 @@ from __future__ import print_function
 # pylint:disable=g-bad-import-order,g-import-not-at-top,unused-import
 #
 from tensorflow.contrib.eager.python.datasets import Iterator
-from tensorflow.contrib.eager.python.saver import Saver
 from tensorflow.contrib.eager.python.saver import restore_variables_on_create
+from tensorflow.contrib.eager.python.saver import Saver
 from tensorflow.contrib.eager.python.summary_writer import SummaryWriter
-from tensorflow.python.util.all_util import remove_undocumented
 from tensorflow.python.eager import backprop
-from tensorflow.python.eager.custom_gradient import custom_gradient
 from tensorflow.python.eager import function
 from tensorflow.python.eager.context import enable_eager_execution
 from tensorflow.python.eager.context import in_eager_mode
@@ -74,13 +74,16 @@ from tensorflow.python.eager.context import list_devices
 from tensorflow.python.eager.context import num_gpus
 from tensorflow.python.eager.context import run
 from tensorflow.python.eager.core import enable_tracing
+from tensorflow.python.eager.custom_gradient import custom_gradient
 from tensorflow.python.eager.execution_callbacks import add_execution_callback
 from tensorflow.python.eager.execution_callbacks import clear_execution_callbacks
 from tensorflow.python.eager.execution_callbacks import inf_callback
 from tensorflow.python.eager.execution_callbacks import inf_nan_callback
 from tensorflow.python.eager.execution_callbacks import nan_callback
 from tensorflow.python.eager.execution_callbacks import seterr
+from tensorflow.python.framework.test_util import run_in_graph_and_eager_modes as run_test_in_graph_and_eager_modes
 from tensorflow.python.ops.resource_variable_ops import ResourceVariable as Variable
+from tensorflow.python.util.all_util import remove_undocumented
 
 defun = function.defun
 implicit_gradients = backprop.implicit_grad
-- 
GitLab


From a87606f623123ecc53f86302b348029e9aeefb10 Mon Sep 17 00:00:00 2001
From: Yifei Feng <yifeif@google.com>
Date: Thu, 12 Oct 2017 11:03:15 -0700
Subject: [PATCH 303/909] Make keras estimator_test less flaky.

PiperOrigin-RevId: 171982493
---
 tensorflow/python/keras/BUILD                 |   6 +-
 .../keras/_impl/keras/estimator_test.py       | 161 +++++++++---------
 2 files changed, 87 insertions(+), 80 deletions(-)

diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index 03bf9d2177..fd8ac392de 100644
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -668,17 +668,15 @@ py_test(
     size = "medium",
     srcs = ["_impl/keras/estimator_test.py"],
     srcs_version = "PY2AND3",
-    tags = [
-        "noasan",
-        "notsan",
-    ],
     deps = [
         ":keras",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:dtypes",
         "//tensorflow/python:framework_ops",
+        "//tensorflow/python:framework_test_lib",
         "//tensorflow/python:platform",
         "//tensorflow/python/estimator:numpy_io",
+        "//tensorflow/python/estimator:run_config",
         "//third_party/py/numpy",
     ],
 )
diff --git a/tensorflow/python/keras/_impl/keras/estimator_test.py b/tensorflow/python/keras/_impl/keras/estimator_test.py
index 7967038e76..1144aa3152 100644
--- a/tensorflow/python/keras/_impl/keras/estimator_test.py
+++ b/tensorflow/python/keras/_impl/keras/estimator_test.py
@@ -23,43 +23,42 @@ import tempfile
 
 import numpy as np
 
+from tensorflow.python.estimator import run_config as run_config_lib
 from tensorflow.python.estimator.inputs import numpy_io
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.keras._impl import keras
 from tensorflow.python.keras._impl.keras import testing_utils
+from tensorflow.python.keras._impl.keras.applications import mobilenet
 from tensorflow.python.platform import gfile
 from tensorflow.python.platform import test
 
+
 try:
   import h5py  # pylint:disable=g-import-not-at-top
 except ImportError:
   h5py = None
 
+_RANDOM_SEED = 1337
+_TRAIN_SIZE = 200
+_INPUT_SIZE = (10,)
+_NUM_CLASS = 2
+
 
 def simple_sequential_model():
   model = keras.models.Sequential()
-  model.add(
-      keras.layers.Conv2D(
-          32, kernel_size=(3, 3), activation='relu', input_shape=(14, 14, 3)))
-  model.add(keras.layers.MaxPooling2D(pool_size=(2, 2)))
-  model.add(keras.layers.Dropout(0.25))
-  model.add(keras.layers.Flatten())
-  model.add(keras.layers.Dense(16, activation='relu'))
-  model.add(keras.layers.Dropout(0.25))
-  model.add(keras.layers.Dense(3, activation='softmax'))
+  model.add(keras.layers.Dense(16, activation='relu', input_shape=_INPUT_SIZE))
+  model.add(keras.layers.Dropout(0.1))
+  model.add(keras.layers.Dense(_NUM_CLASS, activation='softmax'))
   return model
 
 
 def simple_functional_model():
-  a = keras.layers.Input(shape=(14, 14, 3))
-  b = keras.layers.Conv2D(32, kernel_size=(3, 3), activation='relu')(a)
-  b = keras.layers.MaxPooling2D(pool_size=(2, 2))(b)
-  b = keras.layers.Dropout(0.25)(b)
-  b = keras.layers.Flatten()(b)
-  b = keras.layers.Dense(16, activation='relu')(b)
-  b = keras.layers.Dropout(0.25)(b)
-  b = keras.layers.Dense(3, activation='softmax')(b)
+  a = keras.layers.Input(shape=_INPUT_SIZE)
+  b = keras.layers.Dense(16, activation='relu')(a)
+  b = keras.layers.Dropout(0.1)(b)
+  b = keras.layers.Dense(_NUM_CLASS, activation='softmax')(b)
   model = keras.models.Model(inputs=[a], outputs=[b])
   return model
 
@@ -70,13 +69,12 @@ def get_resource_for_simple_model(is_sequential, is_evaluate):
   if is_sequential:
     model.build()
   input_name = model.input_names[0]
-
-  np.random.seed(1337)
+  np.random.seed(_RANDOM_SEED)
   (x_train, y_train), (x_test, y_test) = testing_utils.get_test_data(
-      train_samples=200,
-      test_samples=100,
-      input_shape=(14, 14, 3),
-      num_classes=3)
+      train_samples=_TRAIN_SIZE,
+      test_samples=50,
+      input_shape=_INPUT_SIZE,
+      num_classes=_NUM_CLASS)
   y_train = keras.utils.to_categorical(y_train)
   y_test = keras.utils.to_categorical(y_test)
 
@@ -106,9 +104,9 @@ def get_resource_for_simple_model(is_sequential, is_evaluate):
 
 def multi_inputs_multi_outputs_model():
   # test multi-input layer
-  a = keras.layers.Input(shape=(32,), name='input_a')
-  b = keras.layers.Input(shape=(32,), name='input_b')
-  dense = keras.layers.Dense(16, name='dense_1')
+  a = keras.layers.Input(shape=(16,), name='input_a')
+  b = keras.layers.Input(shape=(16,), name='input_b')
+  dense = keras.layers.Dense(8, name='dense_1')
   a_2 = dense(a)
   b_2 = dense(b)
   merged = keras.layers.concatenate([a_2, b_2], name='merge')
@@ -118,19 +116,24 @@ def multi_inputs_multi_outputs_model():
   model.compile(
       loss='categorical_crossentropy',
       optimizer='rmsprop',
-      metrics={'dense_2': 'accuracy',
-               'dense_3': 'accuracy'})
+      metrics={
+          'dense_2': 'categorical_accuracy',
+          'dense_3': 'categorical_accuracy'
+      })
   return model
 
 
-class TestKerasEstimator(test.TestCase):
+class TestKerasEstimator(test_util.TensorFlowTestCase):
 
   def setUp(self):
     self._base_dir = os.path.join(self.get_temp_dir(), 'keras_estimator_test')
     gfile.MakeDirs(self._base_dir)
+    self._config = run_config_lib.RunConfig(
+        tf_random_seed=_RANDOM_SEED, model_dir=self._base_dir)
 
   def tearDown(self):
-    gfile.DeleteRecursively(self._base_dir)
+    if os.path.isdir(self._base_dir):
+      gfile.DeleteRecursively(self._base_dir)
 
   def test_train(self):
     for is_sequential in [True, False]:
@@ -140,17 +143,17 @@ class TestKerasEstimator(test.TestCase):
       keras_model.compile(
           loss='categorical_crossentropy',
           optimizer='rmsprop',
-          metrics=['accuracy', 'mse', keras.metrics.categorical_accuracy])
+          metrics=['mse', keras.metrics.categorical_accuracy])
 
       with self.test_session():
         est_keras = keras.estimator.model_to_estimator(
-            keras_model=keras_model,
-            model_dir=tempfile.mkdtemp(dir=self._base_dir))
-        est_keras.train(input_fn=train_input_fn, steps=200 * 10 / 16)
-        eval_results = est_keras.evaluate(input_fn=eval_input_fn)
-        self.assertGreater(eval_results['accuracy'], 0.9)
-        self.assertGreater(eval_results['categorical_accuracy'], 0.9)
-        self.assertLess(eval_results['mse'], 0.1)
+            keras_model=keras_model, config=self._config)
+        before_eval_results = est_keras.evaluate(
+            input_fn=eval_input_fn, steps=1)
+        est_keras.train(input_fn=train_input_fn, steps=_TRAIN_SIZE / 16)
+        after_eval_results = est_keras.evaluate(input_fn=eval_input_fn, steps=1)
+        self.assertLess(after_eval_results['loss'], before_eval_results['loss'])
+      gfile.DeleteRecursively(self._config.model_dir)
 
   def test_evaluate(self):
     keras_model, (x_train, y_train), (
@@ -173,15 +176,17 @@ class TestKerasEstimator(test.TestCase):
 
     with self.test_session():
       keras_est = keras.estimator.model_to_estimator(
-          keras_model=keras_model,
-          model_dir=tempfile.mkdtemp(dir=self._base_dir))
+          keras_model=keras_model, config=self._config)
       est_eval = keras_est.evaluate(input_fn=eval_input_fn)
 
     metrics = ['loss'] + metrics
 
     # Check loss and all metrics match between keras and estimator.
     def shift(val):
-      return val / 10**int(log10(abs(val)))
+      if val == 0:
+        return 0
+      else:
+        return val / 10**int(log10(abs(val)))
 
     for i, metric_name in enumerate(metrics):
       self.assertAlmostEqual(
@@ -207,8 +212,7 @@ class TestKerasEstimator(test.TestCase):
 
     with self.test_session():
       keras_est = keras.estimator.model_to_estimator(
-          keras_model=keras_model,
-          model_dir=tempfile.mkdtemp(dir=self._base_dir))
+          keras_model=keras_model, config=self._config)
       est_pred = [
           np.argmax(y[keras_model.output_names[0]])
           for y in keras_est.predict(input_fn=pred_input_fn)
@@ -216,11 +220,18 @@ class TestKerasEstimator(test.TestCase):
     self.assertAllEqual(est_pred, keras_pred)
 
   def test_multi_inputs_multi_outputs(self):
-    np.random.seed(1337)
+    np.random.seed(_RANDOM_SEED)
     (a_train, c_train), (a_test, c_test) = testing_utils.get_test_data(
-        train_samples=200, test_samples=100, input_shape=(32,), num_classes=3)
+        train_samples=_TRAIN_SIZE,
+        test_samples=50,
+        input_shape=(16,),
+        num_classes=3)
+    np.random.seed(_RANDOM_SEED)
     (b_train, d_train), (b_test, d_test) = testing_utils.get_test_data(
-        train_samples=200, test_samples=100, input_shape=(32,), num_classes=2)
+        train_samples=_TRAIN_SIZE,
+        test_samples=50,
+        input_shape=(16,),
+        num_classes=2)
     c_train = keras.utils.to_categorical(c_train)
     c_test = keras.utils.to_categorical(c_test)
     d_train = keras.utils.to_categorical(d_train)
@@ -245,7 +256,7 @@ class TestKerasEstimator(test.TestCase):
       }
       return input_dict, output_dict
 
-    def evaluate_input_fn():
+    def eval_input_fn():
       input_dict = {
           'input_a':
               ops.convert_to_tensor(
@@ -267,11 +278,11 @@ class TestKerasEstimator(test.TestCase):
     with self.test_session():
       model = multi_inputs_multi_outputs_model()
       est_keras = keras.estimator.model_to_estimator(
-          keras_model=model, model_dir=tempfile.mkdtemp(dir=self._base_dir))
-      est_keras.train(input_fn=train_input_fn, steps=200 * 10 / 16)
-      eval_results = est_keras.evaluate(input_fn=evaluate_input_fn, steps=1)
-      self.assertGreater(eval_results['accuracy_dense_2'], 0.5)
-      self.assertGreater(eval_results['accuracy_dense_3'], 0.5)
+          keras_model=model, config=self._config)
+      before_eval_results = est_keras.evaluate(input_fn=eval_input_fn, steps=1)
+      est_keras.train(input_fn=train_input_fn, steps=_TRAIN_SIZE / 16)
+      after_eval_results = est_keras.evaluate(input_fn=eval_input_fn, steps=1)
+      self.assertLess(after_eval_results['loss'], before_eval_results['loss'])
 
   def test_init_from_file(self):
     if h5py is None:
@@ -285,7 +296,7 @@ class TestKerasEstimator(test.TestCase):
       keras_model.compile(
           loss='categorical_crossentropy',
           optimizer='rmsprop',
-          metrics=['accuracy'])
+          metrics=['categorical_accuracy'])
       keras_model.fit(x_train, y_train, epochs=1)
       keras_pred = [np.argmax(y) for y in keras_model.predict(x_test)]
       fname = os.path.join(self._base_dir, 'keras_model.h5')
@@ -293,8 +304,7 @@ class TestKerasEstimator(test.TestCase):
 
     with self.test_session():
       keras_est = keras.estimator.model_to_estimator(
-          keras_model_path=fname,
-          model_dir=tempfile.mkdtemp(dir=self._base_dir))
+          keras_model_path=fname, config=self._config)
       est_pred = [
           np.argmax(y[keras_model.output_names[0]])
           for y in keras_est.predict(input_fn=pred_input_fn)
@@ -324,9 +334,11 @@ class TestKerasEstimator(test.TestCase):
             keras_model_path='gs://bucket/object')
 
   def test_invalid_ionames_error(self):
-    np.random.seed(1337)
     (x_train, y_train), (_, _) = testing_utils.get_test_data(
-        train_samples=200, test_samples=100, input_shape=(10,), num_classes=2)
+        train_samples=_TRAIN_SIZE,
+        test_samples=100,
+        input_shape=(10,),
+        num_classes=2)
     y_train = keras.utils.to_categorical(y_train)
 
     def invald_input_name_input_fn():
@@ -356,7 +368,7 @@ class TestKerasEstimator(test.TestCase):
     model.compile(
         loss='categorical_crossentropy', optimizer='adam', metrics=['acc'])
     est_keras = keras.estimator.model_to_estimator(
-        keras_model=model, model_dir=tempfile.mkdtemp(dir=self._base_dir))
+        keras_model=model, config=self._config)
 
     with self.test_session():
       with self.assertRaises(ValueError):
@@ -366,26 +378,23 @@ class TestKerasEstimator(test.TestCase):
         est_keras.train(input_fn=invald_output_name_input_fn, steps=100)
 
   def test_custom_objects(self):
-    keras_model, (_, _), (
-        _, _), train_input_fn, eval_input_fn = get_resource_for_simple_model(
-            is_sequential=True, is_evaluate=True)
-
-    class CustomOp(keras.optimizers.RMSprop):
-      pass
-
-    def custom_loss(y_true, y_pred):
-      return keras.losses.categorical_crossentropy(y_true, y_pred)
-
-    keras_model.compile(
-        loss=custom_loss, optimizer=CustomOp(), metrics=['accuracy'])
+    keras_mobile = mobilenet.MobileNet(weights=None)
+    keras_mobile.compile(loss='categorical_crossentropy', optimizer='adam')
+    custom_objects = {
+        'relu6': mobilenet.relu6,
+        'DepthwiseConv2D': mobilenet.DepthwiseConv2D
+    }
+    with self.assertRaisesRegexp(ValueError, 'relu6'):
+      with self.test_session():
+        keras.estimator.model_to_estimator(
+            keras_model=keras_mobile,
+            model_dir=tempfile.mkdtemp(dir=self._base_dir))
 
     with self.test_session():
-      est_keras = keras.estimator.model_to_estimator(
-          keras_model=keras_model,
-          model_dir=tempfile.mkdtemp(dir=self._base_dir))
-      est_keras.train(input_fn=train_input_fn, steps=200 * 10 / 16)
-      eval_results = est_keras.evaluate(input_fn=eval_input_fn)
-      self.assertGreater(eval_results['accuracy'], 0.9)
+      keras.estimator.model_to_estimator(
+          keras_model=keras_mobile,
+          model_dir=tempfile.mkdtemp(dir=self._base_dir),
+          custom_objects=custom_objects)
 
 
 if __name__ == '__main__':
-- 
GitLab


From 48eee55b02a542a0a30b918a6a64e8ae7283e01a Mon Sep 17 00:00:00 2001
From: David Majnemer <majnemer@google.com>
Date: Thu, 12 Oct 2017 11:05:25 -0700
Subject: [PATCH 304/909] Automated g4 rollback of changelist 170358888

PiperOrigin-RevId: 171982861
---
 .../compiler/tf2xla/kernels/conv_ops.cc       | 18 +++++++----
 .../xla/client/computation_builder.cc         | 30 ++++++++++++++-----
 .../compiler/xla/client/computation_builder.h |  3 +-
 .../compiler/xla/reference_util_test.cc       | 12 +++++---
 .../xla/service/algebraic_simplifier.cc       |  9 ++++--
 .../xla/service/algebraic_simplifier_test.cc  |  6 ++--
 .../xla/service/cpu/conv_canonicalization.cc  | 25 +++++++++-------
 .../service/cpu/conv_canonicalization_test.cc | 12 +++++---
 .../xla/service/cpu/ir_emission_utils.cc      |  8 +++--
 .../compiler/xla/service/cpu/ir_emitter.cc    | 18 ++++++-----
 .../xla/service/gpu/convolution_folding.cc    | 16 ++++++----
 .../service/gpu/convolution_folding_test.cc   | 18 +++++++----
 .../xla/service/gpu/convolution_thunk.cc      |  8 ++---
 .../service/gpu/instruction_fusion_test.cc    |  6 ++--
 .../xla/service/gpu/layout_assignment.cc      |  8 ++---
 .../compiler/xla/service/hlo_cost_analysis.cc |  2 +-
 .../compiler/xla/service/hlo_evaluator.cc     | 17 ++++++-----
 .../xla/service/hlo_evaluator_test.cc         | 12 +++++---
 .../compiler/xla/service/hlo_instruction.cc   | 13 ++++++--
 .../compiler/xla/service/shape_inference.cc   | 12 ++++----
 .../xla/service/shape_inference_test.cc       | 24 ++++++++++-----
 .../convolution_dimension_numbers_test.cc     | 20 ++++++++-----
 .../compiler/xla/tests/convolution_test.cc    | 18 +++++++----
 .../xla/tests/convolution_variants_test.cc    | 24 ++++++++++-----
 tensorflow/compiler/xla/xla_data.proto        | 16 ++++++----
 25 files changed, 229 insertions(+), 126 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/kernels/conv_ops.cc b/tensorflow/compiler/tf2xla/kernels/conv_ops.cc
index 0091b66d28..885f716afa 100644
--- a/tensorflow/compiler/tf2xla/kernels/conv_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/conv_ops.cc
@@ -179,8 +179,10 @@ class ConvOp : public XlaOpKernel {
 
     xla::ConvolutionDimensionNumbers dims;
     std::vector<int64> window_strides;
-    dims.set_batch_dimension(GetTensorBatchDimIndex(num_dims(), data_format_));
-    dims.set_feature_dimension(feature_dim);
+    dims.set_input_batch_dimension(batch_dim);
+    dims.set_output_batch_dimension(batch_dim);
+    dims.set_input_feature_dimension(feature_dim);
+    dims.set_output_feature_dimension(feature_dim);
     for (int i = 0; i < num_spatial_dims_; ++i) {
       int input_dim = GetTensorSpatialDimIndex(num_dims(), data_format_, i);
       dims.add_spatial_dimensions(input_dim);
@@ -285,8 +287,10 @@ class ConvBackpropInputOp : public XlaOpKernel {
     // comment at the top of conv_grad_ops.h for details.
 
     xla::ConvolutionDimensionNumbers dnums;
-    dnums.set_batch_dimension(batch_dim);
-    dnums.set_feature_dimension(feature_dim);
+    dnums.set_input_batch_dimension(batch_dim);
+    dnums.set_output_batch_dimension(batch_dim);
+    dnums.set_input_feature_dimension(feature_dim);
+    dnums.set_output_feature_dimension(feature_dim);
 
     // TF filter shape is [ H, W, ..., inC, outC ]
     // Transpose the input and output features for computing the gradient.
@@ -419,8 +423,10 @@ class ConvBackpropFilterOp : public XlaOpKernel {
     // Each spatial entry has size in_depth * batch
 
     // Swap n_dim and c_dim in the activations.
-    dnums.set_batch_dimension(c_dim);
-    dnums.set_feature_dimension(n_dim);
+    dnums.set_input_batch_dimension(c_dim);
+    dnums.set_output_batch_dimension(c_dim);
+    dnums.set_input_feature_dimension(n_dim);
+    dnums.set_output_feature_dimension(n_dim);
 
     // The gradients become the RHS of the convolution.
     // The gradients have shape [batch, out_rows, out_cols, ..., out_depth]
diff --git a/tensorflow/compiler/xla/client/computation_builder.cc b/tensorflow/compiler/xla/client/computation_builder.cc
index cbd71dad86..206af290c6 100644
--- a/tensorflow/compiler/xla/client/computation_builder.cc
+++ b/tensorflow/compiler/xla/client/computation_builder.cc
@@ -1762,8 +1762,10 @@ void ComputationBuilder::SetDeviceAssignment(
 /* static */ ConvolutionDimensionNumbers
 ComputationBuilder::CreateDefaultConvDimensionNumbers(int num_spatial_dims) {
   ConvolutionDimensionNumbers dimension_numbers;
-  dimension_numbers.set_batch_dimension(kConvBatchDimension);
-  dimension_numbers.set_feature_dimension(kConvFeatureDimension);
+  dimension_numbers.set_input_batch_dimension(kConvBatchDimension);
+  dimension_numbers.set_input_feature_dimension(kConvFeatureDimension);
+  dimension_numbers.set_output_batch_dimension(kConvBatchDimension);
+  dimension_numbers.set_output_feature_dimension(kConvFeatureDimension);
   dimension_numbers.set_kernel_output_feature_dimension(
       kConvKernelOutputDimension);
   dimension_numbers.set_kernel_input_feature_dimension(
@@ -1777,15 +1779,17 @@ ComputationBuilder::CreateDefaultConvDimensionNumbers(int num_spatial_dims) {
 
 /* static */ StatusOr<ConvolutionDimensionNumbers>
 ComputationBuilder::CreateConvDimensionNumbers(
-    int64 batch, int64 feature, int64 first_spatial, int64 second_spatial,
+    int64 input_batch, int64 input_feature, int64 output_batch,
+    int64 output_feature, int64 first_spatial, int64 second_spatial,
     int64 kernel_output_feature, int64 kernel_input_feature,
     int64 kernel_first_spatial, int64 kernel_second_spatial) {
-  if (std::set<int64>({batch, feature, first_spatial, second_spatial}).size() !=
-      4) {
+  if (std::set<int64>(
+          {input_batch, input_feature, first_spatial, second_spatial})
+          .size() != 4) {
     return FailedPrecondition(
         "dimension numbers for the input are not unique: (%lld, %lld, %lld, "
         "%lld)",
-        batch, feature, first_spatial, second_spatial);
+        input_batch, input_feature, first_spatial, second_spatial);
   }
   if (std::set<int64>({kernel_output_feature, kernel_input_feature,
                        kernel_first_spatial, kernel_second_spatial})
@@ -1796,9 +1800,19 @@ ComputationBuilder::CreateConvDimensionNumbers(
         kernel_output_feature, kernel_input_feature, kernel_first_spatial,
         kernel_second_spatial);
   }
+  if (std::set<int64>(
+          {output_batch, output_feature, first_spatial, second_spatial})
+          .size() != 4) {
+    return FailedPrecondition(
+        "dimension numbers for the output are not unique: (%lld, %lld, %lld, "
+        "%lld)",
+        output_batch, output_feature, first_spatial, second_spatial);
+  }
   ConvolutionDimensionNumbers dimension_numbers;
-  dimension_numbers.set_batch_dimension(batch);
-  dimension_numbers.set_feature_dimension(feature);
+  dimension_numbers.set_input_batch_dimension(input_batch);
+  dimension_numbers.set_input_feature_dimension(input_feature);
+  dimension_numbers.set_output_batch_dimension(output_batch);
+  dimension_numbers.set_output_feature_dimension(output_feature);
   dimension_numbers.add_spatial_dimensions(first_spatial);
   dimension_numbers.add_spatial_dimensions(second_spatial);
   dimension_numbers.set_kernel_output_feature_dimension(kernel_output_feature);
diff --git a/tensorflow/compiler/xla/client/computation_builder.h b/tensorflow/compiler/xla/client/computation_builder.h
index 23769f0afc..94b03502f9 100644
--- a/tensorflow/compiler/xla/client/computation_builder.h
+++ b/tensorflow/compiler/xla/client/computation_builder.h
@@ -354,7 +354,8 @@ class ComputationBuilder {
   // Creates a ConvolutionDimensionNumbers with the given arguments. Returns an
   // error if either the input or the weight dimension numbers have conflicts.
   static StatusOr<ConvolutionDimensionNumbers> CreateConvDimensionNumbers(
-      int64 batch, int64 feature, int64 first_spatial, int64 second_spatial,
+      int64 input_batch, int64 input_feature, int64 output_batch,
+      int64 output_feature, int64 first_spatial, int64 second_spatial,
       int64 kernel_output_feature, int64 kernel_input_feature,
       int64 kernel_first_spatial, int64 kernel_second_spatial);
 
diff --git a/tensorflow/compiler/xla/reference_util_test.cc b/tensorflow/compiler/xla/reference_util_test.cc
index 35b5e8cd52..eb6a71242f 100644
--- a/tensorflow/compiler/xla/reference_util_test.cc
+++ b/tensorflow/compiler/xla/reference_util_test.cc
@@ -322,8 +322,10 @@ TEST_F(ReferenceUtilTest, ConvGeneralDimensionsWithSamePadding) {
 
   // Set the convolution dimension numbers.
   ConvolutionDimensionNumbers dimension_numbers;
-  dimension_numbers.set_batch_dimension(2);
-  dimension_numbers.set_feature_dimension(0);
+  dimension_numbers.set_input_batch_dimension(2);
+  dimension_numbers.set_input_feature_dimension(0);
+  dimension_numbers.set_output_batch_dimension(2);
+  dimension_numbers.set_output_feature_dimension(0);
   dimension_numbers.add_spatial_dimensions(1);
   dimension_numbers.add_spatial_dimensions(3);
   dimension_numbers.set_kernel_output_feature_dimension(0);
@@ -374,8 +376,10 @@ TEST_F(ReferenceUtilTest, ConvGeneralDimensionsWithValidPadding) {
 
   // Set the convolution dimension numbers.
   ConvolutionDimensionNumbers dimension_numbers;
-  dimension_numbers.set_batch_dimension(2);
-  dimension_numbers.set_feature_dimension(0);
+  dimension_numbers.set_input_batch_dimension(2);
+  dimension_numbers.set_input_feature_dimension(0);
+  dimension_numbers.set_output_batch_dimension(2);
+  dimension_numbers.set_output_feature_dimension(0);
   dimension_numbers.add_spatial_dimensions(1);
   dimension_numbers.add_spatial_dimensions(3);
 
diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier.cc b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
index 90ab7700ea..6592caa2a6 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
@@ -1511,7 +1511,10 @@ Status AlgebraicSimplifierVisitor::HandleConvolution(
   // still convert Conv into more efficient Matmul with operand transposition
   // (such as the transposition flags in cuBLAS SGEMM).
   if (!LayoutUtil::Equal(input_shape.layout(), convolution_shape.layout()) ||
-      input_shape.layout().minor_to_major(0) != dnums.feature_dimension() ||
+      input_shape.layout().minor_to_major(0) !=
+          dnums.input_feature_dimension() ||
+      convolution_shape.layout().minor_to_major(0) !=
+          dnums.output_feature_dimension() ||
       // The input feature dimension should come later in the minor-to-major
       // order.
       (PositionInContainer(filter_shape.layout().minor_to_major(),
@@ -1530,14 +1533,14 @@ Status AlgebraicSimplifierVisitor::HandleConvolution(
 
   // Replace it with a dot, with bitcasts around it to get the right shape.
   const int64 input_channels =
-      input_shape.dimensions(dnums.feature_dimension());
+      input_shape.dimensions(dnums.input_feature_dimension());
   const int64 output_channels =
       filter_shape.dimensions(dnums.kernel_output_feature_dimension());
 
   // Computes the product of the non-feature dimensions.
   int64 conv_width = 1;
   for (int i = 0; i < input_shape.dimensions_size(); ++i) {
-    if (i != dnums.feature_dimension()) {
+    if (i != dnums.input_feature_dimension()) {
       conv_width *= input_shape.dimensions(i);
     }
   }
diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
index f45e541b2c..af502206e2 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
@@ -1578,7 +1578,8 @@ TEST_F(AlgebraicSimplifierTest, ConvertConvToMatmul) {
     for (int i = 0; i < strlen(options.dim_order); ++i) {
       char ch = options.dim_order[i];
       if (ch == 'N') {
-        dnums.set_batch_dimension(i);
+        dnums.set_input_batch_dimension(i);
+        dnums.set_output_batch_dimension(i);
         in_dims.push_back(options.in_batch);
       } else if (ch == 'H') {
         dnums.set_spatial_dimensions(0, i);
@@ -1587,7 +1588,8 @@ TEST_F(AlgebraicSimplifierTest, ConvertConvToMatmul) {
         dnums.set_spatial_dimensions(1, i);
         in_dims.push_back(options.in_width);
       } else if (ch == 'C') {
-        dnums.set_feature_dimension(i);
+        dnums.set_input_feature_dimension(i);
+        dnums.set_output_feature_dimension(i);
         in_dims.push_back(options.in_channels);
         in_channel_idx = i;
       }
diff --git a/tensorflow/compiler/xla/service/cpu/conv_canonicalization.cc b/tensorflow/compiler/xla/service/cpu/conv_canonicalization.cc
index 069979c661..44cd2171af 100644
--- a/tensorflow/compiler/xla/service/cpu/conv_canonicalization.cc
+++ b/tensorflow/compiler/xla/service/cpu/conv_canonicalization.cc
@@ -36,8 +36,8 @@ StatusOr<bool> ConvCanonicalization::Run(HloModule* module) {
         !PotentiallyImplementedAsEigenConvolution(*hlo)) {
       const ConvolutionDimensionNumbers& dnums =
           hlo->convolution_dimension_numbers();
-      auto batch_dim = dnums.batch_dimension();
-      auto feature_dim = dnums.feature_dimension();
+      auto input_batch_dim = dnums.input_batch_dimension();
+      auto input_feature_dim = dnums.input_feature_dimension();
       auto kernel_input_feature_dim = dnums.kernel_input_feature_dimension();
       auto kernel_output_feature_dim = dnums.kernel_output_feature_dimension();
 
@@ -59,15 +59,16 @@ StatusOr<bool> ConvCanonicalization::Run(HloModule* module) {
 
       std::vector<int64> new_input_dim_order(num_dims);
       std::vector<int64> new_input_dims(num_dims);
-      new_input_dim_order[0] = batch_dim;
-      new_input_dims[0] = input->shape().dimensions(batch_dim);
+      new_input_dim_order[0] = input_batch_dim;
+      new_input_dims[0] = input->shape().dimensions(input_batch_dim);
       for (int i = 0; i < num_spatial_dims; ++i) {
         new_input_dim_order[i + 1] = dnums.spatial_dimensions(i);
         new_input_dims[i + 1] =
             input->shape().dimensions(dnums.spatial_dimensions(i));
       }
-      new_input_dim_order[num_dims - 1] = feature_dim;
-      new_input_dims[num_dims - 1] = input->shape().dimensions(feature_dim);
+      new_input_dim_order[num_dims - 1] = input_feature_dim;
+      new_input_dims[num_dims - 1] =
+          input->shape().dimensions(input_feature_dim);
 
       Shape new_input_shape =
           ShapeUtil::MakeShape(input->shape().element_type(), new_input_dims);
@@ -98,22 +99,26 @@ StatusOr<bool> ConvCanonicalization::Run(HloModule* module) {
                                           new_kernel_dim_order));
 
       std::vector<int64> new_conv_dims(num_dims);
-      new_conv_dims[0] = hlo->shape().dimensions(batch_dim);
+      auto output_batch_dim = dnums.output_batch_dimension();
+      auto output_feature_dim = dnums.output_feature_dimension();
+      new_conv_dims[0] = hlo->shape().dimensions(output_batch_dim);
       for (int i = 0; i < num_spatial_dims; ++i) {
         new_conv_dims[i + 1] =
             hlo->shape().dimensions(dnums.spatial_dimensions(i));
       }
-      new_conv_dims[num_dims - 1] = hlo->shape().dimensions(feature_dim);
+      new_conv_dims[num_dims - 1] = hlo->shape().dimensions(output_feature_dim);
       Shape new_conv_shape =
           ShapeUtil::MakeShape(hlo->shape().element_type(), new_conv_dims);
 
       ConvolutionDimensionNumbers new_dnums;
-      new_dnums.set_batch_dimension(0);
+      new_dnums.set_input_batch_dimension(0);
+      new_dnums.set_output_batch_dimension(0);
       for (int i = 0; i < num_spatial_dims; ++i) {
         new_dnums.add_spatial_dimensions(i + 1);
         new_dnums.add_kernel_spatial_dimensions(i);
       }
-      new_dnums.set_feature_dimension(num_dims - 1);
+      new_dnums.set_input_feature_dimension(num_dims - 1);
+      new_dnums.set_output_feature_dimension(num_dims - 1);
       new_dnums.set_kernel_input_feature_dimension(num_dims - 2);
       new_dnums.set_kernel_output_feature_dimension(num_dims - 1);
 
diff --git a/tensorflow/compiler/xla/service/cpu/conv_canonicalization_test.cc b/tensorflow/compiler/xla/service/cpu/conv_canonicalization_test.cc
index 9e8b785f30..d593ba26b6 100644
--- a/tensorflow/compiler/xla/service/cpu/conv_canonicalization_test.cc
+++ b/tensorflow/compiler/xla/service/cpu/conv_canonicalization_test.cc
@@ -67,10 +67,12 @@ TEST_F(ConvCanonicalizationTest, NonCanonicalToCanonical) {
           kOutputFeatureCount, kInputFeatureCount, kWindowSize, kWindowSize))));
 
   ConvolutionDimensionNumbers dnums;
-  dnums.set_batch_dimension(1);
+  dnums.set_input_batch_dimension(1);
+  dnums.set_output_batch_dimension(1);
   dnums.add_spatial_dimensions(2);
   dnums.add_spatial_dimensions(3);
-  dnums.set_feature_dimension(0);
+  dnums.set_input_feature_dimension(0);
+  dnums.set_output_feature_dimension(0);
   dnums.add_kernel_spatial_dimensions(2);
   dnums.add_kernel_spatial_dimensions(3);
   dnums.set_kernel_input_feature_dimension(1);
@@ -121,10 +123,12 @@ TEST_F(ConvCanonicalizationTest, CanonicalStaysTheSame) {
           kWindowSize, kWindowSize, kInputFeatureCount, kOutputFeatureCount))));
 
   ConvolutionDimensionNumbers dnums;
-  dnums.set_batch_dimension(0);
+  dnums.set_input_batch_dimension(0);
+  dnums.set_output_batch_dimension(0);
   dnums.add_spatial_dimensions(1);
   dnums.add_spatial_dimensions(2);
-  dnums.set_feature_dimension(3);
+  dnums.set_input_feature_dimension(3);
+  dnums.set_output_feature_dimension(3);
   dnums.add_kernel_spatial_dimensions(0);
   dnums.add_kernel_spatial_dimensions(1);
   dnums.set_kernel_input_feature_dimension(2);
diff --git a/tensorflow/compiler/xla/service/cpu/ir_emission_utils.cc b/tensorflow/compiler/xla/service/cpu/ir_emission_utils.cc
index 91b09f2472..ea5b6ca4eb 100644
--- a/tensorflow/compiler/xla/service/cpu/ir_emission_utils.cc
+++ b/tensorflow/compiler/xla/service/cpu/ir_emission_utils.cc
@@ -55,8 +55,12 @@ bool PotentiallyImplementedAsEigenConvolution(
       std::is_sorted(dnums.kernel_spatial_dimensions().begin(),
                      dnums.kernel_spatial_dimensions().end());
 
-  return dnums.batch_dimension() == 0 &&
-         dnums.feature_dimension() == input_shape.dimensions_size() - 1 &&
+  const Shape& output_shape = convolution.shape();
+  return dnums.input_batch_dimension() == 0 &&
+         dnums.input_feature_dimension() == input_shape.dimensions_size() - 1 &&
+         dnums.output_batch_dimension() == 0 &&
+         dnums.output_feature_dimension() ==
+             output_shape.dimensions_size() - 1 &&
          input_spatial_dims_ascending == kernel_spatial_dims_ascending &&
          dnums.kernel_input_feature_dimension() ==
              kernel_shape.dimensions_size() - 2 &&
diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
index 633ad0290c..3d2d0f1029 100644
--- a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
@@ -960,13 +960,14 @@ Status IrEmitter::HandleConvolution(HloInstruction* convolution,
 
       // Input tensor.
       const Shape& input_shape = convolution->operand(0)->shape();
-      int64 input_batch = input_shape.dimensions(dnums.batch_dimension());
+      int64 input_batch = input_shape.dimensions(dnums.input_batch_dimension());
       int64 input_rows = input_shape.dimensions(dnums.spatial_dimensions(0));
       int64 input_cols =
           one_dim_convolution
               ? 1
               : input_shape.dimensions(dnums.spatial_dimensions(1));
-      int64 input_channels = input_shape.dimensions(dnums.feature_dimension());
+      int64 input_channels =
+          input_shape.dimensions(dnums.input_feature_dimension());
 
       // Kernel tensor.
       const Shape& kernel_shape = convolution->operand(1)->shape();
@@ -1081,8 +1082,8 @@ Status IrEmitter::HandleConvolution(HloInstruction* convolution,
         for (int i = 0; i < num_spatial_dims; ++i) {
           output_spatial[i] = index[dnums.spatial_dimensions(i)];
         }
-        llvm::Value* output_feature = index[dnums.feature_dimension()];
-        llvm::Value* batch = index[dnums.batch_dimension()];
+        llvm::Value* output_feature = index[dnums.output_feature_dimension()];
+        llvm::Value* batch = index[dnums.output_batch_dimension()];
 
         // We will accumulate the products into this sum to calculate
         // the output entry at the given index.
@@ -1106,8 +1107,9 @@ Status IrEmitter::HandleConvolution(HloInstruction* convolution,
         }
         llvm::Value* input_feature =
             loops
-                .AddLoop(0, lhs->shape().dimensions(dnums.feature_dimension()),
-                         "iz")
+                .AddLoop(
+                    0, lhs->shape().dimensions(dnums.input_feature_dimension()),
+                    "iz")
                 ->GetIndVarValue();
 
         SetToFirstInsertPoint(loops.GetInnerLoopBodyBasicBlock(), &ir_builder_);
@@ -1187,8 +1189,8 @@ Status IrEmitter::HandleConvolution(HloInstruction* convolution,
         for (int i = 0; i < num_spatial_dims; ++i) {
           input_index[dnums.spatial_dimensions(i)] = input_spatial[i];
         }
-        input_index[dnums.feature_dimension()] = input_feature;
-        input_index[dnums.batch_dimension()] = batch;
+        input_index[dnums.input_feature_dimension()] = input_feature;
+        input_index[dnums.input_batch_dimension()] = batch;
 
         llvm_ir::IrArray kernel_array(GetIrArrayFor(rhs));
         llvm_ir::IrArray::Index kernel_index(num_dims);
diff --git a/tensorflow/compiler/xla/service/gpu/convolution_folding.cc b/tensorflow/compiler/xla/service/gpu/convolution_folding.cc
index edd04773d1..5aaf072f9d 100644
--- a/tensorflow/compiler/xla/service/gpu/convolution_folding.cc
+++ b/tensorflow/compiler/xla/service/gpu/convolution_folding.cc
@@ -72,8 +72,10 @@ MatchBackwardFilter(HloInstruction* conv) {
   // Step 2: match paddings and dimension numbers of the forward convolution.
   const ConvolutionDimensionNumbers& conv_dnums =
       conv->convolution_dimension_numbers();
-  auto batch_dim = conv_dnums.batch_dimension();
-  auto feature_dim = conv_dnums.feature_dimension();
+  auto input_batch_dim = conv_dnums.input_batch_dimension();
+  auto input_feature_dim = conv_dnums.input_feature_dimension();
+  auto output_batch_dim = conv_dnums.output_batch_dimension();
+  auto output_feature_dim = conv_dnums.output_feature_dimension();
   auto spatial_dims = conv_dnums.spatial_dimensions();
 
   for (const WindowDimension& window_dim : conv->window().dimensions()) {
@@ -183,8 +185,10 @@ MatchBackwardFilter(HloInstruction* conv) {
   // convolution. The two activation dimensions are reversed (batch and
   // feature).
   ConvolutionDimensionNumbers backward_conv_dnums;
-  backward_conv_dnums.set_batch_dimension(feature_dim);
-  backward_conv_dnums.set_feature_dimension(batch_dim);
+  backward_conv_dnums.set_input_batch_dimension(input_feature_dim);
+  backward_conv_dnums.set_input_feature_dimension(input_batch_dim);
+  backward_conv_dnums.set_output_batch_dimension(output_feature_dim);
+  backward_conv_dnums.set_output_feature_dimension(output_batch_dim);
   for (int i = 0; i < spatial_dims.size(); ++i) {
     backward_conv_dnums.add_spatial_dimensions(spatial_dims[i]);
   }
@@ -198,9 +202,9 @@ MatchBackwardFilter(HloInstruction* conv) {
   // the dimension numbering of the weight gradients. This transposition maps
   // dimension i to PositionInContainer(transpose->dimensions(), i).
   backward_conv_dnums.set_kernel_input_feature_dimension(
-      PositionInContainer(transpose->dimensions(), batch_dim));
+      PositionInContainer(transpose->dimensions(), output_batch_dim));
   backward_conv_dnums.set_kernel_output_feature_dimension(
-      PositionInContainer(transpose->dimensions(), feature_dim));
+      PositionInContainer(transpose->dimensions(), output_feature_dim));
   for (int i = 0; i < spatial_dims.size(); ++i) {
     backward_conv_dnums.add_kernel_spatial_dimensions(
         PositionInContainer(transpose->dimensions(), spatial_dims[i]));
diff --git a/tensorflow/compiler/xla/service/gpu/convolution_folding_test.cc b/tensorflow/compiler/xla/service/gpu/convolution_folding_test.cc
index 6699c8f3c4..19b122ba06 100644
--- a/tensorflow/compiler/xla/service/gpu/convolution_folding_test.cc
+++ b/tensorflow/compiler/xla/service/gpu/convolution_folding_test.cc
@@ -45,8 +45,10 @@ class ConvolutionFoldingTest : public HloTestBase {
     // dimension in gradients as the input feature dimension in the filter.
     //
     // TODO(jingyue): Add more tests on NCHW input order which TF also supports.
-    tf_default_dnums_for_backward_filter_.set_batch_dimension(3);
-    tf_default_dnums_for_backward_filter_.set_feature_dimension(0);
+    tf_default_dnums_for_backward_filter_.set_input_batch_dimension(3);
+    tf_default_dnums_for_backward_filter_.set_output_batch_dimension(3);
+    tf_default_dnums_for_backward_filter_.set_input_feature_dimension(0);
+    tf_default_dnums_for_backward_filter_.set_output_feature_dimension(0);
     tf_default_dnums_for_backward_filter_.add_spatial_dimensions(1);
     tf_default_dnums_for_backward_filter_.add_spatial_dimensions(2);
     tf_default_dnums_for_backward_filter_.set_kernel_input_feature_dimension(0);
@@ -55,8 +57,10 @@ class ConvolutionFoldingTest : public HloTestBase {
     tf_default_dnums_for_backward_filter_.add_kernel_spatial_dimensions(1);
     tf_default_dnums_for_backward_filter_.add_kernel_spatial_dimensions(2);
 
-    tf_default_dnums_for_backward_input_.set_batch_dimension(0);
-    tf_default_dnums_for_backward_input_.set_feature_dimension(3);
+    tf_default_dnums_for_backward_input_.set_input_batch_dimension(0);
+    tf_default_dnums_for_backward_input_.set_output_batch_dimension(0);
+    tf_default_dnums_for_backward_input_.set_input_feature_dimension(3);
+    tf_default_dnums_for_backward_input_.set_output_feature_dimension(3);
     tf_default_dnums_for_backward_input_.add_spatial_dimensions(1);
     tf_default_dnums_for_backward_input_.add_spatial_dimensions(2);
     tf_default_dnums_for_backward_input_.set_kernel_input_feature_dimension(3);
@@ -250,8 +254,10 @@ TEST_F(ConvolutionFoldingTest, BackwardInputConvolveEvenPadding) {
     conv_window.mutable_dimensions(i)->set_padding_high(3);
   }
   ConvolutionDimensionNumbers conv_dnums;
-  conv_dnums.set_batch_dimension(0);
-  conv_dnums.set_feature_dimension(1);
+  conv_dnums.set_input_batch_dimension(0);
+  conv_dnums.set_output_batch_dimension(0);
+  conv_dnums.set_input_feature_dimension(1);
+  conv_dnums.set_output_feature_dimension(1);
   conv_dnums.add_spatial_dimensions(2);
   conv_dnums.add_spatial_dimensions(3);
   conv_dnums.set_kernel_input_feature_dimension(0);
diff --git a/tensorflow/compiler/xla/service/gpu/convolution_thunk.cc b/tensorflow/compiler/xla/service/gpu/convolution_thunk.cc
index 7dd242425c..536b96dcf6 100644
--- a/tensorflow/compiler/xla/service/gpu/convolution_thunk.cc
+++ b/tensorflow/compiler/xla/service/gpu/convolution_thunk.cc
@@ -141,8 +141,8 @@ tensorflow::Status ConvolutionThunk::ExecuteOnStream(
   BatchDescriptor input_descriptor(effective_num_dimensions);
   input_descriptor.set_layout(DataLayout::kBatchDepthYX)
       .set_feature_map_count(
-          input_shape_.dimensions(dim_nums_.feature_dimension()))
-      .set_count(input_shape_.dimensions(dim_nums_.batch_dimension()));
+          input_shape_.dimensions(dim_nums_.input_feature_dimension()))
+      .set_count(input_shape_.dimensions(dim_nums_.input_batch_dimension()));
   for (int dim = 0; dim < num_dimensions; ++dim) {
     // Note that the dimensions are reversed. The same holds below.
     input_descriptor.set_spatial_dim(
@@ -176,8 +176,8 @@ tensorflow::Status ConvolutionThunk::ExecuteOnStream(
   BatchDescriptor output_descriptor(effective_num_dimensions);
   output_descriptor.set_layout(DataLayout::kBatchDepthYX)
       .set_feature_map_count(
-          output_shape_.dimensions(dim_nums_.feature_dimension()))
-      .set_count(output_shape_.dimensions(dim_nums_.batch_dimension()));
+          output_shape_.dimensions(dim_nums_.output_feature_dimension()))
+      .set_count(output_shape_.dimensions(dim_nums_.output_batch_dimension()));
   for (int dim = 0; dim < num_dimensions; ++dim) {
     output_descriptor.set_spatial_dim(
         static_cast<se::dnn::DimIndex>(effective_num_dimensions - dim - 1),
diff --git a/tensorflow/compiler/xla/service/gpu/instruction_fusion_test.cc b/tensorflow/compiler/xla/service/gpu/instruction_fusion_test.cc
index 0b94594f1d..9a4bfd0905 100644
--- a/tensorflow/compiler/xla/service/gpu/instruction_fusion_test.cc
+++ b/tensorflow/compiler/xla/service/gpu/instruction_fusion_test.cc
@@ -152,8 +152,10 @@ TEST_F(InstructionFusionTest, PotentialBitcastTransposeOfConvolutionUnfused) {
   conv_window_col->set_padding_high(1);
 
   ConvolutionDimensionNumbers conv_dnums;
-  conv_dnums.set_batch_dimension(0);
-  conv_dnums.set_feature_dimension(1);
+  conv_dnums.set_input_batch_dimension(0);
+  conv_dnums.set_output_batch_dimension(0);
+  conv_dnums.set_input_feature_dimension(1);
+  conv_dnums.set_output_feature_dimension(1);
   conv_dnums.add_spatial_dimensions(2);
   conv_dnums.add_spatial_dimensions(3);
   conv_dnums.set_kernel_output_feature_dimension(0);
diff --git a/tensorflow/compiler/xla/service/gpu/layout_assignment.cc b/tensorflow/compiler/xla/service/gpu/layout_assignment.cc
index b0480e2f47..0bbd63fb7b 100644
--- a/tensorflow/compiler/xla/service/gpu/layout_assignment.cc
+++ b/tensorflow/compiler/xla/service/gpu/layout_assignment.cc
@@ -84,8 +84,8 @@ Status GpuLayoutAssignment::AddBackendConstraints(
            --i) {
         input_layout.push_back(dimension_numbers.spatial_dimensions(i));
       }
-      input_layout.push_back(dimension_numbers.feature_dimension());
-      input_layout.push_back(dimension_numbers.batch_dimension());
+      input_layout.push_back(dimension_numbers.input_feature_dimension());
+      input_layout.push_back(dimension_numbers.input_batch_dimension());
       Shape input_shape(input->shape());
       *input_shape.mutable_layout() = LayoutUtil::MakeLayout(input_layout);
 
@@ -106,8 +106,8 @@ Status GpuLayoutAssignment::AddBackendConstraints(
            --i) {
         output_layout.push_back(dimension_numbers.spatial_dimensions(i));
       }
-      output_layout.push_back(dimension_numbers.feature_dimension());
-      output_layout.push_back(dimension_numbers.batch_dimension());
+      output_layout.push_back(dimension_numbers.output_feature_dimension());
+      output_layout.push_back(dimension_numbers.output_batch_dimension());
       Shape output_shape(output->shape());
       *output_shape.mutable_layout() = LayoutUtil::MakeLayout(output_layout);
 
diff --git a/tensorflow/compiler/xla/service/hlo_cost_analysis.cc b/tensorflow/compiler/xla/service/hlo_cost_analysis.cc
index 65725ca692..84d55d4b5f 100644
--- a/tensorflow/compiler/xla/service/hlo_cost_analysis.cc
+++ b/tensorflow/compiler/xla/service/hlo_cost_analysis.cc
@@ -393,7 +393,7 @@ Status HloCostAnalysis::HandleConvolution(HloInstruction* convolution,
                                           const Window& window) {
   const auto& dnums = convolution->convolution_dimension_numbers();
   const int64 output_features =
-      convolution->shape().dimensions(dnums.feature_dimension());
+      convolution->shape().dimensions(dnums.output_feature_dimension());
 
   // For each output element, we do one fma per element in the kernel at some
   // given output feature index.
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator.cc b/tensorflow/compiler/xla/service/hlo_evaluator.cc
index 53e33c9fd0..b28f9b59ab 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator.cc
+++ b/tensorflow/compiler/xla/service/hlo_evaluator.cc
@@ -480,14 +480,17 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault {
     const Literal& lhs_literal = parent_->GetEvaluatedLiteralFor(lhs);
     const Literal& rhs_literal = parent_->GetEvaluatedLiteralFor(rhs);
 
-    // Dimension number applicable for both input (lhs), and output.
-    const int64 batch_dim = dnums.batch_dimension();
-    const int64 z_dim = dnums.feature_dimension();
+    // Dimension number applicable for input (lhs).
+    const int64 input_batch_dim = dnums.input_batch_dimension();
+    const int64 input_z_dim = dnums.input_feature_dimension();
     // Dimension number applicable for kernel (rhs).
     const int64 kernel_input_z_dim = dnums.kernel_input_feature_dimension();
     const int64 kernel_output_z_dim = dnums.kernel_output_feature_dimension();
+    // Dimension number applicable for output.
+    const int64 output_batch_dim = dnums.output_batch_dimension();
+    const int64 output_z_dim = dnums.output_feature_dimension();
 
-    const int64 z_size = ShapeUtil::GetDimension(lhs_shape, z_dim);
+    const int64 z_size = ShapeUtil::GetDimension(lhs_shape, input_z_dim);
 
     std::vector<int64> window_dimension_sizes;
     for (auto i : dnums.kernel_spatial_dimensions()) {
@@ -508,13 +511,13 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault {
       std::fill(rhs_index.begin(), rhs_index.end(), 0);
       std::fill(rhs_spatial_index.begin(), rhs_spatial_index.end(), 0);
 
-      lhs_index[batch_dim] = out_index[batch_dim];
-      rhs_index[kernel_output_z_dim] = out_index[z_dim];
+      lhs_index[input_batch_dim] = out_index[output_batch_dim];
+      rhs_index[kernel_output_z_dim] = out_index[output_z_dim];
 
       // Convolve input feature with kernel.
       do {
         for (int64 iz = 0; iz < z_size; ++iz) {
-          lhs_index[z_dim] = iz;
+          lhs_index[input_z_dim] = iz;
           rhs_index[kernel_input_z_dim] = iz;
 
           // Find corresponding spatial dimension index for input (lhs).
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator_test.cc b/tensorflow/compiler/xla/service/hlo_evaluator_test.cc
index a8a73e866e..5172739624 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_evaluator_test.cc
@@ -736,8 +736,10 @@ TEST_F(HloEvaluatorTest, SimpleConv1D) {
   *window.add_dimensions() = dim;
 
   ConvolutionDimensionNumbers dnums;
-  dnums.set_batch_dimension(0);
-  dnums.set_feature_dimension(1);
+  dnums.set_input_batch_dimension(0);
+  dnums.set_output_batch_dimension(0);
+  dnums.set_input_feature_dimension(1);
+  dnums.set_output_feature_dimension(1);
   dnums.add_spatial_dimensions(2);
 
   dnums.set_kernel_output_feature_dimension(0);
@@ -868,8 +870,10 @@ TEST_F(HloEvaluatorTest, Conv2DGeneralDimensions) {
   *window.add_dimensions() = dim;
 
   ConvolutionDimensionNumbers dnums;
-  dnums.set_batch_dimension(2);
-  dnums.set_feature_dimension(0);
+  dnums.set_input_batch_dimension(2);
+  dnums.set_output_batch_dimension(2);
+  dnums.set_input_feature_dimension(0);
+  dnums.set_output_feature_dimension(0);
   dnums.add_spatial_dimensions(1);
   dnums.add_spatial_dimensions(3);
 
diff --git a/tensorflow/compiler/xla/service/hlo_instruction.cc b/tensorflow/compiler/xla/service/hlo_instruction.cc
index e3e482cf85..b18280552d 100644
--- a/tensorflow/compiler/xla/service/hlo_instruction.cc
+++ b/tensorflow/compiler/xla/service/hlo_instruction.cc
@@ -2638,8 +2638,8 @@ string HloInstruction::ConvolutionDimensionNumbersToString() const {
   // lhs_dims[i] is the symbol of the logical dimension i for the lhs
   // operand. E.g. if batch has dimension number 2, then lhs_dims[2] == "b".
   std::vector<string> lhs_dims(2 + dnums.spatial_dimensions().size());
-  lhs_dims[dnums.batch_dimension()] = 'b';
-  lhs_dims[dnums.feature_dimension()] = 'f';
+  lhs_dims[dnums.input_batch_dimension()] = 'b';
+  lhs_dims[dnums.input_feature_dimension()] = 'f';
   for (int64 i = 0; i < dnums.spatial_dimensions().size(); ++i) {
     lhs_dims[dnums.spatial_dimensions(i)] = StrCat(i);
   }
@@ -2651,12 +2651,19 @@ string HloInstruction::ConvolutionDimensionNumbersToString() const {
     rhs_dims[dnums.kernel_spatial_dimensions(i)] = StrCat(i);
   }
 
+  std::vector<string> output_dims(2 + dnums.spatial_dimensions().size());
+  output_dims[dnums.output_batch_dimension()] = 'b';
+  output_dims[dnums.output_feature_dimension()] = 'f';
+  for (int64 i = 0; i < dnums.spatial_dimensions().size(); ++i) {
+    output_dims[dnums.spatial_dimensions(i)] = StrCat(i);
+  }
+
   result += "dim_labels=";
   append_dims(lhs_dims, operand(0)->shape());
   result += "_";
   append_dims(rhs_dims, operand(1)->shape());
   result += "->";
-  append_dims(lhs_dims, shape());
+  append_dims(output_dims, shape());
   return result;
 }
 
diff --git a/tensorflow/compiler/xla/service/shape_inference.cc b/tensorflow/compiler/xla/service/shape_inference.cc
index b333d232a7..a9f65331e2 100644
--- a/tensorflow/compiler/xla/service/shape_inference.cc
+++ b/tensorflow/compiler/xla/service/shape_inference.cc
@@ -1406,8 +1406,8 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(
   // Verifies that the input and window dimensions are a permutation of
   // the dimension numbers.
   std::vector<int64> input_dnums(num_dims);
-  input_dnums[0] = dnums.batch_dimension();
-  input_dnums[1] = dnums.feature_dimension();
+  input_dnums[0] = dnums.input_batch_dimension();
+  input_dnums[1] = dnums.input_feature_dimension();
   std::copy(dnums.spatial_dimensions().begin(),
             dnums.spatial_dimensions().end(), input_dnums.begin() + 2);
   std::sort(input_dnums.begin(), input_dnums.end());
@@ -1447,8 +1447,8 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(
   for (int i = 0; i < num_spatial_dims; ++i) {
     input_spatial_dims[i] = lhs.dimensions(dnums.spatial_dimensions(i));
   }
-  const int64 input_features = lhs.dimensions(dnums.feature_dimension());
-  const int64 input_batch = lhs.dimensions(dnums.batch_dimension());
+  const int64 input_features = lhs.dimensions(dnums.input_feature_dimension());
+  const int64 input_batch = lhs.dimensions(dnums.input_batch_dimension());
 
   std::vector<int64> kernel_spatial_dims(num_spatial_dims);
   for (int i = 0; i < num_spatial_dims; ++i) {
@@ -1490,8 +1490,8 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(
                              /*allow_negative_padding=*/true));
 
   std::vector<int64> dimensions(num_dims);
-  dimensions[dnums.batch_dimension()] = input_batch;
-  dimensions[dnums.feature_dimension()] = kernel_output_features;
+  dimensions[dnums.output_batch_dimension()] = input_batch;
+  dimensions[dnums.output_feature_dimension()] = kernel_output_features;
   for (int i = 0; i < num_spatial_dims; ++i) {
     dimensions[dnums.spatial_dimensions(i)] = window_output_shape.dimensions(i);
   }
diff --git a/tensorflow/compiler/xla/service/shape_inference_test.cc b/tensorflow/compiler/xla/service/shape_inference_test.cc
index 7c9c7e8d6a..8df4a73229 100644
--- a/tensorflow/compiler/xla/service/shape_inference_test.cc
+++ b/tensorflow/compiler/xla/service/shape_inference_test.cc
@@ -352,8 +352,10 @@ TEST_F(ShapeInferenceTest, Convolve) {
 
   // Dimension order: batch, feature, x0, x1
   Shape lhs_shape = ShapeUtil::MakeShape(F32, {10, 11, 3, 4});
-  dnums.set_batch_dimension(0);
-  dnums.set_feature_dimension(1);
+  dnums.set_input_batch_dimension(0);
+  dnums.set_output_batch_dimension(0);
+  dnums.set_input_feature_dimension(1);
+  dnums.set_output_feature_dimension(1);
   dnums.add_spatial_dimensions(2);
   dnums.add_spatial_dimensions(3);
 
@@ -392,8 +394,10 @@ TEST_F(ShapeInferenceTest, ConvolveWithWindowDilation) {
 
   // Dimension order: batch, feature, x0, x1
   Shape lhs_shape = ShapeUtil::MakeShape(F32, {10, 11, 103, 4});
-  dnums.set_batch_dimension(0);
-  dnums.set_feature_dimension(1);
+  dnums.set_input_batch_dimension(0);
+  dnums.set_output_batch_dimension(0);
+  dnums.set_input_feature_dimension(1);
+  dnums.set_output_feature_dimension(1);
   dnums.add_spatial_dimensions(2);
   dnums.add_spatial_dimensions(3);
 
@@ -433,8 +437,10 @@ TEST_F(ShapeInferenceTest, ConvolveWithBaseDilation) {
 
   // Dimension order: batch, feature, x0, x1
   Shape lhs_shape = ShapeUtil::MakeShape(F32, {10, 11, 3, 4});
-  dnums.set_batch_dimension(0);
-  dnums.set_feature_dimension(1);
+  dnums.set_input_batch_dimension(0);
+  dnums.set_output_batch_dimension(0);
+  dnums.set_input_feature_dimension(1);
+  dnums.set_output_feature_dimension(1);
   dnums.add_spatial_dimensions(2);
   dnums.add_spatial_dimensions(3);
 
@@ -475,8 +481,10 @@ TEST_F(ShapeInferenceTest, ConvolveDimensionNumbersOverlapError) {
   Shape rhs_shape = ShapeUtil::MakeShape(F32, {12, 11, 3, 2});
 
   ConvolutionDimensionNumbers dnums;
-  dnums.set_batch_dimension(3);
-  dnums.set_feature_dimension(2);
+  dnums.set_input_batch_dimension(3);
+  dnums.set_output_batch_dimension(3);
+  dnums.set_input_feature_dimension(2);
+  dnums.set_output_feature_dimension(2);
   dnums.add_spatial_dimensions(0);
   dnums.add_spatial_dimensions(1);
   dnums.set_kernel_input_feature_dimension(0);  // duplicated with kernel_x0
diff --git a/tensorflow/compiler/xla/tests/convolution_dimension_numbers_test.cc b/tensorflow/compiler/xla/tests/convolution_dimension_numbers_test.cc
index 83882ca75e..b0a63bccbb 100644
--- a/tensorflow/compiler/xla/tests/convolution_dimension_numbers_test.cc
+++ b/tensorflow/compiler/xla/tests/convolution_dimension_numbers_test.cc
@@ -39,7 +39,8 @@ class ConvolutionDimensionNumbersTest : public ClientLibraryTestBase {};
 // Tests the convolution operation with invalid input dimension numbers.
 TEST_F(ConvolutionDimensionNumbersTest, InvalidInputDimensionNumbers) {
   auto dimension_numbers_status =
-      ComputationBuilder::CreateConvDimensionNumbers(0, 2, 2, 3, 0, 1, 2, 3);
+      ComputationBuilder::CreateConvDimensionNumbers(0, 2, 0, 2, 2, 3, 0, 1, 2,
+                                                     3);
   ASSERT_FALSE(dimension_numbers_status.ok());
   ASSERT_THAT(dimension_numbers_status.status().error_message(),
               ::testing::HasSubstr("input are not unique"));
@@ -48,7 +49,8 @@ TEST_F(ConvolutionDimensionNumbersTest, InvalidInputDimensionNumbers) {
 // Tests the convolution operation with invalid weight dimension numbers.
 TEST_F(ConvolutionDimensionNumbersTest, InvalidWeightDimensionNumbers) {
   auto dimension_numbers_status =
-      ComputationBuilder::CreateConvDimensionNumbers(0, 1, 2, 3, 2, 3, 2, 3);
+      ComputationBuilder::CreateConvDimensionNumbers(0, 1, 0, 1, 2, 3, 2, 3, 2,
+                                                     3);
   ASSERT_FALSE(dimension_numbers_status.ok());
   ASSERT_THAT(dimension_numbers_status.status().error_message(),
               ::testing::HasSubstr("weight are not unique"));
@@ -73,14 +75,18 @@ XLA_TEST_F(ConvolutionDimensionNumbersTest,
   ConvolutionDimensionNumbers dim_nums =
       ComputationBuilder::CreateDefaultConvDimensionNumbers();
   // Swap batch_dimension and feature_dimension.
-  int64 tmp = dim_nums.batch_dimension();
-  dim_nums.set_batch_dimension(dim_nums.feature_dimension());
-  dim_nums.set_feature_dimension(tmp);
+  int64 old_input_batch_dim = dim_nums.input_batch_dimension();
+  int64 old_output_batch_dim = dim_nums.output_batch_dimension();
+  dim_nums.set_input_batch_dimension(dim_nums.input_feature_dimension());
+  dim_nums.set_output_batch_dimension(dim_nums.output_feature_dimension());
+  dim_nums.set_input_feature_dimension(old_input_batch_dim);
+  dim_nums.set_output_feature_dimension(old_output_batch_dim);
   // Swap kernel_input_feature_dimension and kernel_output_feature_dimension.
-  tmp = dim_nums.kernel_input_feature_dimension();
+  int64 old_kernel_input_feature_dim =
+      dim_nums.kernel_input_feature_dimension();
   dim_nums.set_kernel_input_feature_dimension(
       dim_nums.kernel_output_feature_dimension());
-  dim_nums.set_kernel_output_feature_dimension(tmp);
+  dim_nums.set_kernel_output_feature_dimension(old_kernel_input_feature_dim);
   builder.ConvWithGeneralDimensions(input, conv1, {1, 1}, Padding::kValid,
                                     dim_nums);
 
diff --git a/tensorflow/compiler/xla/tests/convolution_test.cc b/tensorflow/compiler/xla/tests/convolution_test.cc
index 7d06cce0c8..a7089c2897 100644
--- a/tensorflow/compiler/xla/tests/convolution_test.cc
+++ b/tensorflow/compiler/xla/tests/convolution_test.cc
@@ -418,11 +418,13 @@ XLA_TEST_F(ConvolutionTest, Convolve3D_1x4x2x3x3_2x2x2x3x3_Valid) {
 
     // Tensorflow dimension numbers for 3D convolution.
     ConvolutionDimensionNumbers dnums;
-    dnums.set_batch_dimension(0);
+    dnums.set_input_batch_dimension(0);
+    dnums.set_output_batch_dimension(0);
     dnums.add_spatial_dimensions(1);
     dnums.add_spatial_dimensions(2);
     dnums.add_spatial_dimensions(3);
-    dnums.set_feature_dimension(4);
+    dnums.set_input_feature_dimension(4);
+    dnums.set_output_feature_dimension(4);
     dnums.add_kernel_spatial_dimensions(0);
     dnums.add_kernel_spatial_dimensions(1);
     dnums.add_kernel_spatial_dimensions(2);
@@ -469,10 +471,12 @@ XLA_TEST_F(ConvolutionTest, Convolve2D_1x3x3x5_3x3x5x5_Valid) {
 
     // Tensorflow dimension numbers for 2D convolution.
     ConvolutionDimensionNumbers dnums;
-    dnums.set_batch_dimension(0);
+    dnums.set_input_batch_dimension(0);
+    dnums.set_output_batch_dimension(0);
     dnums.add_spatial_dimensions(1);
     dnums.add_spatial_dimensions(2);
-    dnums.set_feature_dimension(3);
+    dnums.set_input_feature_dimension(3);
+    dnums.set_output_feature_dimension(3);
     dnums.add_kernel_spatial_dimensions(0);
     dnums.add_kernel_spatial_dimensions(1);
     dnums.set_kernel_input_feature_dimension(2);
@@ -520,9 +524,11 @@ XLA_TEST_F(ConvolutionTest, Convolve1D_Valid) {
 
     // Tensorflow dimension numbers for 2D convolution.
     ConvolutionDimensionNumbers dnums;
-    dnums.set_batch_dimension(0);
+    dnums.set_input_batch_dimension(0);
+    dnums.set_output_batch_dimension(0);
     dnums.add_spatial_dimensions(1);
-    dnums.set_feature_dimension(2);
+    dnums.set_input_feature_dimension(2);
+    dnums.set_output_feature_dimension(2);
     dnums.add_kernel_spatial_dimensions(0);
     dnums.set_kernel_input_feature_dimension(1);
     dnums.set_kernel_output_feature_dimension(2);
diff --git a/tensorflow/compiler/xla/tests/convolution_variants_test.cc b/tensorflow/compiler/xla/tests/convolution_variants_test.cc
index 145918db3e..9b36e3722b 100644
--- a/tensorflow/compiler/xla/tests/convolution_variants_test.cc
+++ b/tensorflow/compiler/xla/tests/convolution_variants_test.cc
@@ -974,10 +974,12 @@ XLA_TEST_F(ConvolutionVariantsTest, Filter1x2x1x1Input1x2x3x1GeneralPadding) {
 
   ConvolutionDimensionNumbers dnums;
   // NHWC input format.
-  dnums.set_batch_dimension(0);
+  dnums.set_input_batch_dimension(0);
+  dnums.set_output_batch_dimension(0);
   dnums.add_spatial_dimensions(1);
   dnums.add_spatial_dimensions(2);
-  dnums.set_feature_dimension(3);
+  dnums.set_input_feature_dimension(3);
+  dnums.set_output_feature_dimension(3);
 
   // Tensorflow filter shape: [ H, W, inC, outC ]
   dnums.add_kernel_spatial_dimensions(0);
@@ -1014,10 +1016,12 @@ XLA_TEST_F(ConvolutionVariantsTest, Filter1x1x1x1Input1x2x3x1GeneralPadding) {
 
   ConvolutionDimensionNumbers dnums;
   // NHWC input format.
-  dnums.set_batch_dimension(0);
+  dnums.set_input_batch_dimension(0);
+  dnums.set_output_batch_dimension(0);
   dnums.add_spatial_dimensions(1);
   dnums.add_spatial_dimensions(2);
-  dnums.set_feature_dimension(3);
+  dnums.set_input_feature_dimension(3);
+  dnums.set_output_feature_dimension(3);
 
   // Tensorflow filter shape: [ H, W, inC, outC ]
   dnums.add_kernel_spatial_dimensions(0);
@@ -1054,10 +1058,12 @@ XLA_TEST_F(ConvolutionVariantsTest, Filter1x1x1x1Input1x2x3x1NoPadding) {
 
   ConvolutionDimensionNumbers dnums;
   // NHWC input format.
-  dnums.set_batch_dimension(0);
+  dnums.set_input_batch_dimension(0);
+  dnums.set_output_batch_dimension(0);
   dnums.add_spatial_dimensions(1);
   dnums.add_spatial_dimensions(2);
-  dnums.set_feature_dimension(3);
+  dnums.set_input_feature_dimension(3);
+  dnums.set_output_feature_dimension(3);
 
   // Tensorflow filter shape: [ H, W, inC, outC ]
   dnums.add_kernel_spatial_dimensions(0);
@@ -1091,10 +1097,12 @@ XLA_TEST_F(ConvolutionVariantsTest, Filter1x1x2x3Input1x2x3x2NoPadding) {
 
   ConvolutionDimensionNumbers dnums;
   // NHWC input format.
-  dnums.set_batch_dimension(0);
+  dnums.set_input_batch_dimension(0);
+  dnums.set_output_batch_dimension(0);
   dnums.add_spatial_dimensions(1);
   dnums.add_spatial_dimensions(2);
-  dnums.set_feature_dimension(3);
+  dnums.set_input_feature_dimension(3);
+  dnums.set_output_feature_dimension(3);
 
   // Tensorflow filter shape: [ H, W, inC, outC ]
   dnums.add_kernel_spatial_dimensions(0);
diff --git a/tensorflow/compiler/xla/xla_data.proto b/tensorflow/compiler/xla/xla_data.proto
index 3f26b88809..876b073b3f 100644
--- a/tensorflow/compiler/xla/xla_data.proto
+++ b/tensorflow/compiler/xla/xla_data.proto
@@ -392,13 +392,17 @@ message DynamicUpdateSliceRequest {
 }
 
 message ConvolutionDimensionNumbers {
-  // The number of the dimension that represents batch in the input
-  // (lhs) and output.
-  int64 batch_dimension = 1;
+  // The number of the dimension that represents batch in the input.
+  int64 input_batch_dimension = 7;
 
-  // The number of the dimension that represents features in the input
-  // (lhs) and output.
-  int64 feature_dimension = 2;
+  // The number of the dimension that represents features in the input.
+  int64 input_feature_dimension = 8;
+
+  // The number of the dimension that represents batch in the output.
+  int64 output_batch_dimension = 9;
+
+  // The number of the dimension that represents features in the output.
+  int64 output_feature_dimension = 10;
 
   // The dimension numbers for the spatial dimensions that the window
   // moves through in the input (lhs) and output.
-- 
GitLab


From b0c678da96d3f166e1235f6987d732ee69b4bc20 Mon Sep 17 00:00:00 2001
From: Justine Tunney <jart@google.com>
Date: Thu, 12 Oct 2017 11:10:19 -0700
Subject: [PATCH 305/909] Add veneer for SQLite C API

PiperOrigin-RevId: 171983705
---
 tensorflow/BUILD                              |   1 +
 tensorflow/contrib/makefile/Makefile          |   1 +
 tensorflow/core/kernels/BUILD                 |   1 +
 .../kernels/sql/sqlite_query_connection.cc    | 132 +++------
 .../kernels/sql/sqlite_query_connection.h     |  13 +-
 tensorflow/core/lib/db/BUILD                  |  35 +++
 tensorflow/core/lib/db/sqlite.cc              | 167 +++++++++++
 tensorflow/core/lib/db/sqlite.h               | 278 ++++++++++++++++++
 tensorflow/core/lib/db/sqlite_test.cc         | 198 +++++++++++++
 9 files changed, 733 insertions(+), 93 deletions(-)
 create mode 100644 tensorflow/core/lib/db/BUILD
 create mode 100644 tensorflow/core/lib/db/sqlite.cc
 create mode 100644 tensorflow/core/lib/db/sqlite.h
 create mode 100644 tensorflow/core/lib/db/sqlite_test.cc

diff --git a/tensorflow/BUILD b/tensorflow/BUILD
index 8d9089115d..64758dee0e 100644
--- a/tensorflow/BUILD
+++ b/tensorflow/BUILD
@@ -462,6 +462,7 @@ filegroup(
         "//tensorflow/core/kernels/fuzzing:all_files",
         "//tensorflow/core/kernels/hexagon:all_files",
         "//tensorflow/core/kernels/neon:all_files",
+        "//tensorflow/core/lib/db:all_files",
         "//tensorflow/core/ops/compat:all_files",
         "//tensorflow/core/platform/cloud:all_files",
         "//tensorflow/core/platform/default/build_config:all_files",
diff --git a/tensorflow/contrib/makefile/Makefile b/tensorflow/contrib/makefile/Makefile
index e0cfab0b26..be7c790ee9 100644
--- a/tensorflow/contrib/makefile/Makefile
+++ b/tensorflow/contrib/makefile/Makefile
@@ -484,6 +484,7 @@ $(wildcard tensorflow/core/*/*/*main.cc) \
 $(wildcard tensorflow/core/debug/*.cc) \
 $(wildcard tensorflow/core/framework/op_gen_lib.cc) \
 $(wildcard tensorflow/core/graph/dot.*) \
+$(wildcard tensorflow/core/lib/db/*) \
 $(wildcard tensorflow/core/lib/gif/*) \
 $(wildcard tensorflow/core/lib/io/zlib*) \
 $(wildcard tensorflow/core/lib/io/record*) \
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 0073ba1a96..369feaf49a 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -6029,6 +6029,7 @@ tf_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
+        "//tensorflow/core/lib/db:sqlite",
         "@sqlite_archive//:sqlite",
     ],
 )
diff --git a/tensorflow/core/kernels/sql/sqlite_query_connection.cc b/tensorflow/core/kernels/sql/sqlite_query_connection.cc
index 118c9f5545..a9e6ee0969 100644
--- a/tensorflow/core/kernels/sql/sqlite_query_connection.cc
+++ b/tensorflow/core/kernels/sql/sqlite_query_connection.cc
@@ -19,27 +19,8 @@ namespace tensorflow {
 
 namespace sql {
 
-// Returns a Status with the sqlite error message corresponding to the
-// sqlite error number, `sqlite_err`.
-static Status SqliteErrorToStatus(sqlite3* db, int sqlite_err) {
-  if (sqlite_err == SQLITE_OK) {
-    return Status::OK();
-  } else {
-    const char* err_msg = sqlite3_errmsg(db);
-    // TODO(b/64276468) Be smart about the error code being returned
-    return errors::Unknown(
-        tensorflow::strings::Printf("Sqlite error: %s", err_msg));
-  }
-}
-
-SqliteQueryConnection::SqliteQueryConnection(){};
-
-SqliteQueryConnection::~SqliteQueryConnection() {
-  Status s = Close();
-  if (!s.ok()) {
-    LOG(WARNING) << "Failed to close query connection: " << s;
-  }
-}
+SqliteQueryConnection::SqliteQueryConnection() {}
+SqliteQueryConnection::~SqliteQueryConnection() {}
 
 Status SqliteQueryConnection::Open(const string& data_source_name,
                                    const string& query,
@@ -48,8 +29,7 @@ Status SqliteQueryConnection::Open(const string& data_source_name,
     return errors::FailedPrecondition(
         "Failed to open query connection: Connection already opeend.");
   }
-  int err = sqlite3_open(data_source_name.c_str(), &db_);
-  Status s = SqliteErrorToStatus(db_, err);
+  Status s = db::Sqlite::Open(data_source_name, &db_);
   if (s.ok()) {
     query_ = query;
     output_types_ = output_types;
@@ -58,50 +38,37 @@ Status SqliteQueryConnection::Open(const string& data_source_name,
 }
 
 Status SqliteQueryConnection::Close() {
-  int err = sqlite3_finalize(stmt_);
-  if (err != SQLITE_OK) {
-    return SqliteErrorToStatus(db_, err);
-  }
-  stmt_ = nullptr;
-  err = sqlite3_close(db_);
-  if (err != SQLITE_OK) {
-    return SqliteErrorToStatus(db_, err);
-  }
-  db_ = nullptr;
-  return Status::OK();
+  Status s;
+  s.Update(stmt_->Close());
+  s.Update(db_->Close());
+  return s;
 }
 
 Status SqliteQueryConnection::GetNext(std::vector<Tensor>* out_tensors,
                                       bool* end_of_sequence) {
-  if (stmt_ == nullptr) {
-    Status s = ExecuteQuery();
+  if (!stmt_) {
+    Status s = PrepareQuery();
     if (!s.ok()) {
       return s;
     }
   }
-  int rc = sqlite3_step(stmt_);
-  if (rc == SQLITE_ROW) {
+  Status s = stmt_->Step(end_of_sequence);
+  if (!*end_of_sequence) {
     for (int i = 0; i < column_count_; i++) {
       DataType dt = output_types_[i];
       Tensor tensor(cpu_allocator(), dt, {});
       FillTensorWithResultSetEntry(dt, i, &tensor);
       out_tensors->emplace_back(std::move(tensor));
     }
-    *end_of_sequence = false;
-    return Status::OK();
-  } else if (rc == SQLITE_DONE) {
-    *end_of_sequence = true;
-    return Status::OK();
-  } else {
-    return SqliteErrorToStatus(db_, rc);
   }
+  return s;
 }
 
-Status SqliteQueryConnection::ExecuteQuery() {
-  int err = sqlite3_prepare_v2(db_, query_.c_str(), -1, &stmt_, nullptr);
-  Status s = SqliteErrorToStatus(db_, err);
+Status SqliteQueryConnection::PrepareQuery() {
+  stmt_ = db_->Prepare(query_);
+  Status s = stmt_->status();
   if (s.ok()) {
-    int column_count = sqlite3_column_count(stmt_);
+    int column_count = stmt_->ColumnCount();
     if (column_count != output_types_.size()) {
       return errors::InvalidArgument(tensorflow::strings::Printf(
           "The number of columns in query (%d) must match the number of "
@@ -116,54 +83,43 @@ Status SqliteQueryConnection::ExecuteQuery() {
 void SqliteQueryConnection::FillTensorWithResultSetEntry(
     const DataType& data_type, int column_index, Tensor* tensor) {
   switch (data_type) {
-    case DT_STRING: {
-      const void* bytes = sqlite3_column_blob(stmt_, column_index);
-      int num_bytes = sqlite3_column_bytes(stmt_, column_index);
-      string value(reinterpret_cast<const char*>(bytes), num_bytes);
-      tensor->scalar<string>()() = value;
+    case DT_STRING:
+      tensor->scalar<string>()() = stmt_->ColumnString(column_index);
       break;
-    }
-    case DT_INT8: {
-      int8 value = sqlite3_column_int(stmt_, column_index);
-      tensor->scalar<int8>()() = value;
+    case DT_INT8:
+      tensor->scalar<int8>()() =
+          static_cast<int8>(stmt_->ColumnInt(column_index));
       break;
-    }
-    case DT_INT16: {
-      int16 value = sqlite3_column_int(stmt_, column_index);
-      tensor->scalar<int16>()() = value;
+    case DT_INT16:
+      tensor->scalar<int16>()() =
+          static_cast<int16>(stmt_->ColumnInt(column_index));
       break;
-    }
-    case DT_INT32: {
-      int32 value = sqlite3_column_int(stmt_, column_index);
-      tensor->scalar<int32>()() = value;
+    case DT_INT32:
+      tensor->scalar<int32>()() =
+          static_cast<int32>(stmt_->ColumnInt(column_index));
       break;
-    }
-    case DT_INT64: {
-      int64 value = sqlite3_column_int64(stmt_, column_index);
-      tensor->scalar<int64>()() = value;
+    case DT_INT64:
+      tensor->scalar<int64>()() = stmt_->ColumnInt(column_index);
       break;
-    }
-    case DT_UINT8: {
-      uint8 value = sqlite3_column_int(stmt_, column_index);
-      tensor->scalar<uint8>()() = value;
+    case DT_UINT8:
+      tensor->scalar<uint8>()() =
+          static_cast<uint8>(stmt_->ColumnInt(column_index));
       break;
-    }
-    case DT_UINT16: {
-      uint16 value = sqlite3_column_int(stmt_, column_index);
-      tensor->scalar<uint16>()() = value;
+    case DT_UINT16:
+      tensor->scalar<uint16>()() =
+          static_cast<uint16>(stmt_->ColumnInt(column_index));
       break;
-    }
-    case DT_BOOL: {
-      int value = sqlite3_column_int(stmt_, column_index);
-      tensor->scalar<bool>()() = value ? true : false;
+    case DT_BOOL:
+      tensor->scalar<bool>()() = stmt_->ColumnInt(column_index) != 0;
       break;
-    }
-    case DT_DOUBLE: {
-      double value = sqlite3_column_double(stmt_, column_index);
-      tensor->scalar<double>()() = value;
+    case DT_FLOAT:
+      tensor->scalar<float>()() =
+          static_cast<float>(stmt_->ColumnDouble(column_index));
       break;
-    }
-    // Error preemptively thrown by SqlDatasetOp::MakeDataset in this case.
+    case DT_DOUBLE:
+      tensor->scalar<double>()() = stmt_->ColumnDouble(column_index);
+      break;
+      // Error preemptively thrown by SqlDatasetOp::MakeDataset in this case.
     default: {
       LOG(FATAL)
           << "Use of unsupported TensorFlow data type by 'SqlQueryConnection': "
diff --git a/tensorflow/core/kernels/sql/sqlite_query_connection.h b/tensorflow/core/kernels/sql/sqlite_query_connection.h
index 917df37dc1..b0b4737a1e 100644
--- a/tensorflow/core/kernels/sql/sqlite_query_connection.h
+++ b/tensorflow/core/kernels/sql/sqlite_query_connection.h
@@ -15,8 +15,11 @@ limitations under the License.
 #ifndef THIRD_PARTY_TENSORFLOW_CORE_KERNELS_SQL_SQLITE_QUERY_CONNECTION_H_
 #define THIRD_PARTY_TENSORFLOW_CORE_KERNELS_SQL_SQLITE_QUERY_CONNECTION_H_
 
-#include "sqlite3.h"
+#include <memory>
+
 #include "tensorflow/core/kernels/sql/query_connection.h"
+#include "tensorflow/core/lib/db/sqlite.h"
+#include "tensorflow/core/platform/types.h"
 
 namespace tensorflow {
 
@@ -33,14 +36,14 @@ class SqliteQueryConnection : public QueryConnection {
                  bool* end_of_sequence) override;
 
  private:
-  // Executes the query string `query_`.
-  Status ExecuteQuery();
+  // Prepares the query string `query_`.
+  Status PrepareQuery();
   // Fills `tensor` with the column_index_th element of the current row of
   // `stmt_`.
   void FillTensorWithResultSetEntry(const DataType& data_type, int column_index,
                                     Tensor* tensor);
-  sqlite3* db_ = nullptr;
-  sqlite3_stmt* stmt_ = nullptr;
+  std::unique_ptr<db::Sqlite> db_ = nullptr;
+  std::unique_ptr<db::SqliteStatement> stmt_ = nullptr;
   int column_count_ = 0;
   string query_;
   DataTypeVector output_types_;
diff --git a/tensorflow/core/lib/db/BUILD b/tensorflow/core/lib/db/BUILD
new file mode 100644
index 0000000000..367686c16a
--- /dev/null
+++ b/tensorflow/core/lib/db/BUILD
@@ -0,0 +1,35 @@
+# Description:
+#   Libraries for storing tensors in SQL databases.
+
+package(default_visibility = ["//tensorflow:internal"])
+
+licenses(["notice"])  # Apache 2.0
+
+load("//tensorflow:tensorflow.bzl", "tf_cc_test")
+
+cc_library(
+    name = "sqlite",
+    srcs = ["sqlite.cc"],
+    hdrs = ["sqlite.h"],
+    deps = [
+        "//tensorflow/core:lib",
+        "@sqlite_archive//:sqlite",
+    ],
+)
+
+tf_cc_test(
+    name = "sqlite_test",
+    srcs = ["sqlite_test.cc"],
+    deps = [
+        ":sqlite",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+    ],
+)
+
+filegroup(
+    name = "all_files",
+    srcs = glob(["*"]),
+    visibility = ["//tensorflow:__pkg__"],
+)
diff --git a/tensorflow/core/lib/db/sqlite.cc b/tensorflow/core/lib/db/sqlite.cc
new file mode 100644
index 0000000000..108be452a2
--- /dev/null
+++ b/tensorflow/core/lib/db/sqlite.cc
@@ -0,0 +1,167 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/core/lib/db/sqlite.h"
+
+#include "tensorflow/core/lib/io/record_reader.h"
+#include "tensorflow/core/platform/logging.h"
+
+namespace tensorflow {
+namespace db {
+
+/* static */
+Status Sqlite::Open(const string& uri, std::unique_ptr<Sqlite>* db) {
+  sqlite3* sqlite = nullptr;
+  Status s = MakeStatus(sqlite3_open(uri.c_str(), &sqlite));
+  if (s.ok()) {
+    *db = std::unique_ptr<Sqlite>(new Sqlite(sqlite));
+  }
+  return s;
+}
+
+/* static */ Status Sqlite::MakeStatus(int resultCode) {
+  // See: https://sqlite.org/rescode.html
+  switch (resultCode & 0xff) {
+    case SQLITE_OK:
+    case SQLITE_ROW:   // sqlite3_step() has another row ready
+    case SQLITE_DONE:  // sqlite3_step() has finished executing
+      return Status::OK();
+    case SQLITE_ABORT:  // Callback routine requested an abort
+      return errors::Aborted(sqlite3_errstr(resultCode));
+    case SQLITE_READONLY:  // Attempt to write a readonly database
+    case SQLITE_MISMATCH:  // Data type mismatch
+      return errors::FailedPrecondition(sqlite3_errstr(resultCode));
+    case SQLITE_MISUSE:    // Library used incorrectly
+    case SQLITE_INTERNAL:  // Internal logic error in SQLite
+      return errors::Internal(sqlite3_errstr(resultCode));
+    case SQLITE_RANGE:  // 2nd parameter to sqlite3_bind out of range
+      return errors::OutOfRange(sqlite3_errstr(resultCode));
+    case SQLITE_CANTOPEN:    // Unable to open the database file
+    case SQLITE_CONSTRAINT:  // Abort due to constraint violation
+    case SQLITE_NOTFOUND:    // Unknown opcode or statement parameter name
+    case SQLITE_NOTADB:      // File opened that is not a database file
+      return errors::InvalidArgument(sqlite3_errstr(resultCode));
+    case SQLITE_CORRUPT:  // The database disk image is malformed
+      return errors::DataLoss(sqlite3_errstr(resultCode));
+    case SQLITE_AUTH:  // Authorization denied
+    case SQLITE_PERM:  // Access permission denied
+      return errors::PermissionDenied(sqlite3_errstr(resultCode));
+    case SQLITE_FULL:    // Insertion failed because database is full
+    case SQLITE_TOOBIG:  // String or BLOB exceeds size limit
+    case SQLITE_NOLFS:   // Uses OS features not supported on host
+      return errors::ResourceExhausted(sqlite3_errstr(resultCode));
+    case SQLITE_BUSY:      // The database file is locked
+    case SQLITE_LOCKED:    // A table in the database is locked
+    case SQLITE_PROTOCOL:  // Database lock protocol error
+    case SQLITE_NOMEM:     // A malloc() failed
+      return errors::Unavailable(sqlite3_errstr(resultCode));
+    case SQLITE_INTERRUPT:  // Operation terminated by sqlite3_interrupt
+      return errors::Cancelled(sqlite3_errstr(resultCode));
+    case SQLITE_ERROR:   // SQL error or missing database
+    case SQLITE_IOERR:   // Some kind of disk I/O error occurred
+    case SQLITE_SCHEMA:  // The database schema changed
+    default:
+      return errors::Unknown(sqlite3_errstr(resultCode));
+  }
+}
+
+Sqlite::Sqlite(sqlite3* db) : db_(db) {}
+
+Sqlite::~Sqlite() {
+  // close_v2 doesn't care if a stmt hasn't been GC'd yet
+  int rc = sqlite3_close_v2(db_);
+  if (rc != SQLITE_OK) {
+    LOG(ERROR) << "destruct sqlite3: " << MakeStatus(rc);
+  }
+}
+
+Status Sqlite::Close() {
+  // If Close is explicitly called, ordering must be correct.
+  Status s = MakeStatus(sqlite3_close(db_));
+  if (s.ok()) {
+    db_ = nullptr;
+  }
+  return s;
+}
+
+std::unique_ptr<SqliteStatement> Sqlite::Prepare(const string& sql) {
+  sqlite3_stmt* stmt = nullptr;
+  int rc = sqlite3_prepare_v2(db_, sql.c_str(), sql.size() + 1, &stmt, nullptr);
+  return std::unique_ptr<SqliteStatement>(new SqliteStatement(stmt, rc));
+}
+
+SqliteStatement::SqliteStatement(sqlite3_stmt* stmt, int error)
+    : stmt_(stmt), error_(error) {}
+
+SqliteStatement::~SqliteStatement() {
+  int rc = sqlite3_finalize(stmt_);
+  if (rc != SQLITE_OK) {
+    LOG(ERROR) << "destruct sqlite3_stmt: " << Sqlite::MakeStatus(rc);
+  }
+}
+
+Status SqliteStatement::Close() {
+  int rc = sqlite3_finalize(stmt_);
+  if (rc == SQLITE_OK) {
+    stmt_ = nullptr;
+  }
+  Update(rc);
+  return status();
+}
+
+void SqliteStatement::Reset() {
+  sqlite3_reset(stmt_);
+  sqlite3_clear_bindings(stmt_);
+  error_ = SQLITE_OK;
+}
+
+Status SqliteStatement::Step(bool* isDone) {
+  if (TF_PREDICT_FALSE(error_ != SQLITE_OK)) {
+    *isDone = true;
+    return status();
+  }
+  int rc = sqlite3_step(stmt_);
+  switch (rc) {
+    case SQLITE_ROW:
+      *isDone = false;
+      return Status::OK();
+    case SQLITE_DONE:
+      *isDone = true;
+      return Status::OK();
+    default:
+      *isDone = true;
+      error_ = rc;
+      return status();
+  }
+}
+
+Status SqliteStatement::StepAndReset() {
+  if (TF_PREDICT_FALSE(error_ != SQLITE_OK)) {
+    return status();
+  }
+  Status s;
+  int rc = sqlite3_step(stmt_);
+  if (rc != SQLITE_DONE) {
+    if (rc == SQLITE_ROW) {
+      s.Update(errors::Internal("unexpected sqlite row"));
+    } else {
+      s.Update(Sqlite::MakeStatus(rc));
+    }
+  }
+  Reset();
+  return s;
+}
+
+}  // namespace db
+}  // namespace tensorflow
diff --git a/tensorflow/core/lib/db/sqlite.h b/tensorflow/core/lib/db/sqlite.h
new file mode 100644
index 0000000000..316e938f1b
--- /dev/null
+++ b/tensorflow/core/lib/db/sqlite.h
@@ -0,0 +1,278 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_CORE_LIB_DB_SQLITE_H_
+#define TENSORFLOW_CORE_LIB_DB_SQLITE_H_
+
+#include <stddef.h>
+#include <memory>
+
+#include "sqlite3.h"
+#include "tensorflow/core/lib/core/errors.h"
+#include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/platform/macros.h"
+#include "tensorflow/core/platform/types.h"
+
+namespace tensorflow {
+namespace db {
+
+class SqliteStatement;
+
+/// \brief SQLite connection object.
+///
+/// This class is a thin wrapper around `sqlite3` that makes it easier
+/// and safer to use SQLite in the TensorFlow C++ codebase. It removes
+/// deprecated APIs, improves the safety of others, adds helpers, and
+/// pretends UTF16 doesn't exist.
+///
+/// Instances are thread safe, with the exception of Close().
+class Sqlite {
+ public:
+  /// \brief Opens SQLite database file.
+  ///
+  /// The `uri` parameter can be a filename, or a proper URI like
+  /// `file:/tmp/tf.sqlite?mode=ro&cache=private`. It can also be
+  /// `file::memory:` for testing.
+  ///
+  /// See https://sqlite.org/c3ref/open.html
+  static Status Open(const string& uri, std::unique_ptr<Sqlite>* db);
+
+  /// \brief Makes tensorflow::Status for SQLite result code.
+  ///
+  /// See https://sqlite.org/rescode.html
+  static Status MakeStatus(int resultCode);
+
+  /// \brief Destroys object and frees resources.
+  ///
+  /// This will free the underlying object if Close was not called. If
+  /// an error code is returned then it will be logged.
+  ///
+  /// Note: Unlike Close() this destructor maps to sqlite3_close_v2(),
+  /// which is lax about ordering and GC friendly.
+  ~Sqlite();
+
+  /// \brief Frees underlying SQLite object.
+  ///
+  /// Unlike the destructor, all SqliteStatement objects must be closed
+  /// beforehand.
+  Status Close();
+
+  /// \brief Creates SQLite statement.
+  ///
+  /// Call result.status() to determine whether or not this operation
+  /// failed. It is also possible to punt the error checking to after
+  /// the values have been binded and Step() or ExecuteWriteQuery() is
+  /// called.
+  std::unique_ptr<SqliteStatement> Prepare(const string& sql);
+
+ private:
+  explicit Sqlite(sqlite3* db);
+  sqlite3* db_;
+  TF_DISALLOW_COPY_AND_ASSIGN(Sqlite);
+};
+
+/// \brief SQLite prepared statement cursor object.
+///
+/// This class tracks error state internally, like Status::Update.
+///
+/// Instances of this class are not thread safe.
+class SqliteStatement {
+ public:
+  /// \brief Destroys object and finalizes statement if needed.
+  ~SqliteStatement();
+
+  /// \brief Returns SQLite result code state.
+  ///
+  /// This will be SQLITE_OK unless an error happened. If multiple
+  /// errors happened, only the first error code will be returned.
+  int error() { return error_; }
+
+  /// \brief Returns error() as a tensorflow::Status.
+  Status status() { return Sqlite::MakeStatus(error_); }
+
+  /// \brief Finalize statement object.
+  ///
+  /// Please note that the destructor can also do this.
+  Status Close();
+
+  /// \brief Executes query and/or fetches next row.
+  ///
+  /// `isDone` will always be set to true unless SQLITE_ROW is returned
+  /// by the underlying API. If status() is already in an error state,
+  /// then this method is a no-op and the existing status is returned.
+  Status Step(bool* isDone);
+
+  /// \brief Executes query that returns no data.
+  ///
+  /// This helper calls Step(), ensures SQLITE_DONE was returned, then
+  /// resets the statement and clears the bindings. If status() is
+  /// already in an error state, then this method is a no-op and the
+  /// existing status is returned.
+  Status StepAndReset();
+
+  /// \brief Resets statement so it can be executed again.
+  ///
+  /// - Resets the prepared statement
+  /// - Sets all Bind*() values to NULL
+  ///
+  /// Support for calling sqlite3_reset() and sqlite3_clear_bindings()
+  /// independently may be added in the future if a compelling use case
+  /// can be demonstrated.
+  void Reset();
+
+  /// \brief Binds signed 64-bit integer to 1-indexed query parameter.
+  void BindInt(int parameter, int64 value) {
+    Update(sqlite3_bind_int64(stmt_, parameter, value));
+  }
+  void BindInt(const string& parameter, int64 value) {
+    BindInt(GetParameterIndex(parameter), value);
+  }
+
+  /// \brief Binds double to 1-indexed query parameter.
+  void BindDouble(int parameter, double value) {
+    Update(sqlite3_bind_double(stmt_, parameter, value));
+  }
+  void BindDouble(const string& parameter, double value) {
+    BindDouble(GetParameterIndex(parameter), value);
+  }
+
+  /// \brief Copies UTF-8 text to 1-indexed query parameter.
+  ///
+  /// If NUL characters are present, they will still go in the DB and
+  /// be successfully retrieved by ColumnString(); however, the
+  /// behavior of these values with SQLite functions is undefined.
+  void BindText(int parameter, const string& text) {
+    Update(sqlite3_bind_text64(stmt_, parameter, text.data(), text.size(),
+                               SQLITE_TRANSIENT, SQLITE_UTF8));
+  }
+  void BindText(const string& parameter, const string& text) {
+    BindText(GetParameterIndex(parameter), text);
+  }
+
+  /// \brief Copies binary data to 1-indexed query parameter.
+  void BindBlob(int parameter, const string& blob) {
+    Update(sqlite3_bind_blob64(stmt_, parameter, blob.data(), blob.size(),
+                               SQLITE_TRANSIENT));
+  }
+  void BindBlob(const string& parameter, const string& blob) {
+    BindBlob(GetParameterIndex(parameter), blob);
+  }
+
+  /// \brief Binds UTF-8 text to 1-indexed query parameter.
+  ///
+  /// The contents of `text` must not be changed or freed until Reset()
+  /// or Close() is called.
+  ///
+  /// If NUL characters are present, they will still go in the DB and
+  /// be successfully retrieved by ColumnString(); however, the
+  /// behavior of these values with SQLite functions is undefined.
+  void BindTextUnsafe(int parameter, const string& text) {
+    Update(sqlite3_bind_text64(stmt_, parameter, text.data(), text.size(),
+                               SQLITE_STATIC, SQLITE_UTF8));
+  }
+  void BindTextUnsafe(const string& parameter, const string& text) {
+    BindTextUnsafe(GetParameterIndex(parameter), text);
+  }
+
+  /// \brief Binds binary data to 1-indexed query parameter.
+  ///
+  /// The contents of `blob` must not be changed or freed until Reset()
+  /// or Close() is called.
+  void BindBlobUnsafe(int parameter, const string& blob) {
+    Update(sqlite3_bind_blob64(stmt_, parameter, blob.data(), blob.size(),
+                               SQLITE_STATIC));
+  }
+  void BindBlobUnsafe(const string& parameter, const string& text) {
+    BindBlobUnsafe(GetParameterIndex(parameter), text);
+  }
+
+  /// \brief Returns number of columns in result set.
+  int ColumnCount() TF_MUST_USE_RESULT { return sqlite3_column_count(stmt_); }
+
+  /// \brief Returns type of 0-indexed column value in row data.
+  ///
+  /// Please note that SQLite is dynamically typed and the type of a
+  /// particular column can vary from row to row.
+  int ColumnType(int column) TF_MUST_USE_RESULT {
+    return sqlite3_column_type(stmt_, column);
+  }
+
+  /// \brief Returns 0-indexed column from row result coerced as an integer.
+  int64 ColumnInt(int column) TF_MUST_USE_RESULT {
+    return sqlite3_column_int64(stmt_, column);
+  }
+
+  /// \brief Returns 0-indexed column from row result coerced as a double.
+  double ColumnDouble(int column) TF_MUST_USE_RESULT {
+    return sqlite3_column_double(stmt_, column);
+  }
+
+  /// \brief Copies 0-indexed column from row result coerced as a string.
+  ///
+  /// NULL values are returned as empty string. This method should be
+  /// used for both BLOB and TEXT columns. See also: ColumnType().
+  string ColumnString(int column) TF_MUST_USE_RESULT {
+    auto data = sqlite3_column_blob(stmt_, column);
+    if (data == nullptr) {
+      return "";
+    }
+    return {static_cast<const char*>(data),
+            static_cast<size_t>(ColumnSize(column))};
+  }
+
+  /// \brief Returns pointer to binary data at 0-indexed column.
+  ///
+  /// The returned memory will be mutated or freed the next time
+  /// Step() or Reset() is called. No NUL terminator is added. See
+  /// ColumnSize(). Please note that an empty BLOB is NULL.
+  const char* ColumnStringUnsafe(int column) TF_MUST_USE_RESULT {
+    return static_cast<const char*>(sqlite3_column_blob(stmt_, column));
+  }
+
+  /// \brief Returns number of bytes stored at 0-indexed column.
+  int ColumnSize(int column) TF_MUST_USE_RESULT {
+    return sqlite3_column_bytes(stmt_, column);
+  }
+
+ private:
+  friend Sqlite;
+  SqliteStatement(sqlite3_stmt* stmt, int error);  // takes ownership
+
+  void Update(int rc) {
+    if (TF_PREDICT_FALSE(rc != SQLITE_OK)) {
+      if (error_ == SQLITE_OK) {
+        error_ = rc;
+      }
+    }
+  }
+
+  int GetParameterIndex(const string& parameter) {
+    // Each call to this function requires O(n) strncmp().
+    int index = sqlite3_bind_parameter_index(stmt_, parameter.c_str());
+    if (TF_PREDICT_FALSE(index == 0)) {
+      Update(SQLITE_NOTFOUND);
+    }
+    return index;
+  }
+
+  sqlite3_stmt* stmt_;
+  int error_;
+
+  TF_DISALLOW_COPY_AND_ASSIGN(SqliteStatement);
+};
+
+}  // namespace db
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_CORE_LIB_DB_SQLITE_H_
diff --git a/tensorflow/core/lib/db/sqlite_test.cc b/tensorflow/core/lib/db/sqlite_test.cc
new file mode 100644
index 0000000000..ce22379d97
--- /dev/null
+++ b/tensorflow/core/lib/db/sqlite_test.cc
@@ -0,0 +1,198 @@
+/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/core/lib/db/sqlite.h"
+
+#include <limits.h>
+#include <array>
+
+#include "tensorflow/core/lib/core/status_test_util.h"
+#include "tensorflow/core/lib/core/stringpiece.h"
+#include "tensorflow/core/lib/io/path.h"
+#include "tensorflow/core/lib/strings/strcat.h"
+#include "tensorflow/core/platform/test.h"
+
+namespace tensorflow {
+namespace db {
+namespace {
+
+class SqliteTest : public ::testing::Test {
+ protected:
+  void SetUp() override {
+    TF_ASSERT_OK(Sqlite::Open(":memory:", &db_));
+    auto stmt = db_->Prepare("CREATE TABLE T (a BLOB, b BLOB)");
+    TF_ASSERT_OK(stmt->StepAndReset());
+  }
+  std::unique_ptr<Sqlite> db_;
+  bool is_done_;
+};
+
+TEST_F(SqliteTest, InsertAndSelectInt) {
+  auto stmt = db_->Prepare("INSERT INTO T (a, b) VALUES (?, ?)");
+  stmt->BindInt(1, 3);
+  stmt->BindInt(2, -7);
+  TF_ASSERT_OK(stmt->StepAndReset());
+  stmt->BindInt(1, 123);
+  stmt->BindInt(2, -123);
+  TF_ASSERT_OK(stmt->StepAndReset());
+  stmt = db_->Prepare("SELECT a, b FROM T ORDER BY b");
+  TF_ASSERT_OK(stmt->Step(&is_done_));
+  ASSERT_FALSE(is_done_);
+  EXPECT_EQ(123, stmt->ColumnInt(0));
+  EXPECT_EQ(-123, stmt->ColumnInt(1));
+  TF_ASSERT_OK(stmt->Step(&is_done_));
+  ASSERT_FALSE(is_done_);
+  EXPECT_EQ(3, stmt->ColumnInt(0));
+  EXPECT_EQ(-7, stmt->ColumnInt(1));
+  TF_ASSERT_OK(stmt->Step(&is_done_));
+  ASSERT_TRUE(is_done_);
+}
+
+TEST_F(SqliteTest, InsertAndSelectDouble) {
+  auto stmt = db_->Prepare("INSERT INTO T (a, b) VALUES (?, ?)");
+  stmt->BindDouble(1, 6.28318530);
+  stmt->BindDouble(2, 1.61803399);
+  TF_ASSERT_OK(stmt->StepAndReset());
+  stmt = db_->Prepare("SELECT a, b FROM T");
+  TF_ASSERT_OK(stmt->Step(&is_done_));
+  EXPECT_EQ(6.28318530, stmt->ColumnDouble(0));
+  EXPECT_EQ(1.61803399, stmt->ColumnDouble(1));
+  EXPECT_EQ(6, stmt->ColumnInt(0));
+  EXPECT_EQ(1, stmt->ColumnInt(1));
+}
+
+TEST_F(SqliteTest, NulCharsInString) {
+  string s;  // XXX: Want to write {2, '\0'} but not sure why not.
+  s.append(static_cast<size_t>(2), '\0');
+  auto stmt = db_->Prepare("INSERT INTO T (a, b) VALUES (?, ?)");
+  stmt->BindBlob(1, s);
+  stmt->BindText(2, s);
+  TF_ASSERT_OK(stmt->StepAndReset());
+  stmt = db_->Prepare("SELECT a, b FROM T");
+  TF_ASSERT_OK(stmt->Step(&is_done_));
+  EXPECT_EQ(2, stmt->ColumnSize(0));
+  EXPECT_EQ(2, stmt->ColumnString(0).size());
+  EXPECT_EQ('\0', stmt->ColumnString(0).at(0));
+  EXPECT_EQ('\0', stmt->ColumnString(0).at(1));
+  EXPECT_EQ(2, stmt->ColumnSize(1));
+  EXPECT_EQ(2, stmt->ColumnString(1).size());
+  EXPECT_EQ('\0', stmt->ColumnString(1).at(0));
+  EXPECT_EQ('\0', stmt->ColumnString(1).at(1));
+}
+
+TEST_F(SqliteTest, Unicode) {
+  string s = "要依法治国是赞美那些谁是公义的和惩罚恶人。 - 韩非";
+  auto stmt = db_->Prepare("INSERT INTO T (a, b) VALUES (?, ?)");
+  stmt->BindBlob(1, s);
+  stmt->BindText(2, s);
+  TF_ASSERT_OK(stmt->StepAndReset());
+  stmt = db_->Prepare("SELECT a, b FROM T");
+  TF_ASSERT_OK(stmt->Step(&is_done_));
+  EXPECT_EQ(s, stmt->ColumnString(0));
+  EXPECT_EQ(s, stmt->ColumnString(1));
+}
+
+TEST_F(SqliteTest, StepAndResetClearsBindings) {
+  auto stmt = db_->Prepare("INSERT INTO T (a, b) VALUES (?, ?)");
+  stmt->BindInt(1, 1);
+  stmt->BindInt(2, 123);
+  TF_ASSERT_OK(stmt->StepAndReset());
+  stmt->BindInt(1, 2);
+  TF_ASSERT_OK(stmt->StepAndReset());
+  stmt = db_->Prepare("SELECT b FROM T ORDER BY a");
+  TF_ASSERT_OK(stmt->Step(&is_done_));
+  EXPECT_EQ(123, stmt->ColumnInt(0));
+  TF_ASSERT_OK(stmt->Step(&is_done_));
+  EXPECT_EQ(SQLITE_NULL, stmt->ColumnType(0));
+}
+
+TEST_F(SqliteTest, CloseBeforeFinalizeFails) {
+  auto stmt = db_->Prepare("INSERT INTO T (a, b) VALUES (?, ?)");
+  Status s = db_->Close();
+  EXPECT_FALSE(s.ok());
+}
+
+// Rather than bothering to check the status code of creating a
+// statement and every single bind call afterwards, SqliteStatement
+// is designed to carry the first error state forward to Step().
+TEST_F(SqliteTest, ErrorPuntingDoesNotReportLibraryAbuse) {
+  auto stmt = db_->Prepare("lol cat");
+  EXPECT_FALSE(stmt->status().ok());
+  EXPECT_EQ(SQLITE_ERROR, stmt->error());
+  stmt->BindInt(1, 1);
+  stmt->BindInt(2, 2);
+  Status s = stmt->Step(&is_done_);
+  EXPECT_EQ(SQLITE_ERROR, stmt->error());  // first error of several
+  EXPECT_FALSE(s.ok());
+}
+
+TEST_F(SqliteTest, SafeBind) {
+  string s = "hello";
+  auto stmt = db_->Prepare("INSERT INTO T (a, b) VALUES (?, ?)");
+  stmt->BindBlob(1, s);
+  stmt->BindText(2, s);
+  s.at(0) = 'y';
+  TF_ASSERT_OK(stmt->StepAndReset());
+  stmt = db_->Prepare("SELECT a, b FROM T");
+  TF_ASSERT_OK(stmt->Step(&is_done_));
+  EXPECT_EQ("hello", stmt->ColumnString(0));
+  EXPECT_EQ("hello", stmt->ColumnString(1));
+}
+
+TEST_F(SqliteTest, UnsafeBind) {
+  string s = "hello";
+  auto stmt = db_->Prepare("INSERT INTO T (a, b) VALUES (?, ?)");
+  stmt->BindBlobUnsafe(1, s);
+  stmt->BindTextUnsafe(2, s);
+  s.at(0) = 'y';
+  TF_ASSERT_OK(stmt->StepAndReset());
+  stmt = db_->Prepare("SELECT a, b FROM T");
+  TF_ASSERT_OK(stmt->Step(&is_done_));
+  EXPECT_EQ("yello", stmt->ColumnString(0));
+  EXPECT_EQ("yello", stmt->ColumnString(1));
+}
+
+TEST_F(SqliteTest, UnsafeColumn) {
+  auto stmt = db_->Prepare("INSERT INTO T (a, b) VALUES (?, ?)");
+  stmt->BindInt(1, 1);
+  stmt->BindText(2, "hello");
+  TF_ASSERT_OK(stmt->StepAndReset());
+  stmt->BindInt(1, 2);
+  stmt->BindText(2, "there");
+  TF_ASSERT_OK(stmt->StepAndReset());
+  stmt = db_->Prepare("SELECT b FROM T ORDER BY a");
+  TF_ASSERT_OK(stmt->Step(&is_done_));
+  const char* p = stmt->ColumnStringUnsafe(0);
+  EXPECT_EQ('h', *p);
+  TF_ASSERT_OK(stmt->Step(&is_done_));
+  // This will actually happen, but it's not safe to test this behavior.
+  // EXPECT_EQ('t', *p);
+}
+
+TEST_F(SqliteTest, NamedParameterBind) {
+  auto stmt = db_->Prepare("INSERT INTO T (a) VALUES (:a)");
+  stmt->BindText(":a", "lol");
+  TF_ASSERT_OK(stmt->StepAndReset());
+  stmt = db_->Prepare("SELECT COUNT(*) FROM T");
+  TF_ASSERT_OK(stmt->Step(&is_done_));
+  EXPECT_EQ(1, stmt->ColumnInt(0));
+  stmt = db_->Prepare("SELECT a FROM T");
+  TF_ASSERT_OK(stmt->Step(&is_done_));
+  EXPECT_FALSE(is_done_);
+  EXPECT_EQ("lol", stmt->ColumnString(0));
+}
+
+}  // namespace
+}  // namespace db
+}  // namespace tensorflow
-- 
GitLab


From e975d947929c3f396ec536a086e1fb3756efa2e4 Mon Sep 17 00:00:00 2001
From: Jonathan Hseu <jhseu@google.com>
Date: Thu, 12 Oct 2017 11:10:27 -0700
Subject: [PATCH 306/909] Change the default seed in TPUEstimator's RunConfig
 to be None.

PiperOrigin-RevId: 171983725
---
 tensorflow/contrib/tpu/python/tpu/tpu_config.py | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_config.py b/tensorflow/contrib/tpu/python/tpu/tpu_config.py
index 44069cfb55..ece91180af 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_config.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_config.py
@@ -63,7 +63,18 @@ class RunConfig(run_config_lib.RunConfig):
   """RunConfig with TPU support."""
 
   def __init__(self, tpu_config=None, evaluation_master='', master='',
-               **kwargs):
+               tf_random_seed=None, **kwargs):
+    """Constructs a RunConfig.
+
+    Args:
+      tpu_config: the TPUConfig that specifies TPU-specific configuration.
+      evaluation_master: a string. The address of the master to use for eval.
+      master: a string. The address of the master to use for training.
+      tf_random_seed: an int. Sets the TensorFlow random seed. Defaults to None,
+        which initializes it randomly based on the environment.
+    """
+    # We change the default random seed to None because that's a better default.
+    kwargs['tf_random_seed'] = tf_random_seed
     super(RunConfig, self).__init__(**kwargs)
     self._tpu_config = tpu_config or TPUConfig()
     self._evaluation_master = evaluation_master
-- 
GitLab


From 4241b86dc8da0f8ba23cb832c090469635bf09a9 Mon Sep 17 00:00:00 2001
From: Benoit Steiner <bsteiner@google.com>
Date: Thu, 12 Oct 2017 11:25:52 -0700
Subject: [PATCH 307/909] Updated the virtual scheduler to use legal names when
 inserting Send/Recv nodes in the graph.

PiperOrigin-RevId: 171986401
---
 tensorflow/core/grappler/costs/utils.cc       | 17 +++++++++---
 tensorflow/core/grappler/costs/utils_test.cc  |  8 +++---
 .../core/grappler/costs/virtual_scheduler.cc  | 27 ++++++++++++-------
 .../core/grappler/costs/virtual_scheduler.h   |  1 +
 .../grappler/costs/virtual_scheduler_test.cc  |  8 +++---
 5 files changed, 40 insertions(+), 21 deletions(-)

diff --git a/tensorflow/core/grappler/costs/utils.cc b/tensorflow/core/grappler/costs/utils.cc
index 1504d6b74b..ade0ad53fb 100644
--- a/tensorflow/core/grappler/costs/utils.cc
+++ b/tensorflow/core/grappler/costs/utils.cc
@@ -371,8 +371,19 @@ const int TensorSizeHistogram::Index(const uint64 value) const {
 string GetDeviceClassForNonChannelDevice(const string& device_name) {
   DeviceNameUtils::ParsedName parsed_name;
   bool parsed = DeviceNameUtils::ParseFullName(device_name, &parsed_name);
+  if (!parsed) {
+    string name = str_util::StringReplace(device_name, "/job_", "/job:", true);
+    name = str_util::StringReplace(name, "/replica_", "/replica:", true);
+    name = str_util::StringReplace(name, "/task_", "/task:", true);
+    name = str_util::StringReplace(name, "/device_", "/device:", true);
+    name = str_util::StringReplace(name, "GPU_", "GPU:", true);
+    name = str_util::StringReplace(name, "CPU_", "CPU:", true);
+    name = str_util::StringReplace(name, "gpu_", "gpu:", true);
+    name = str_util::StringReplace(name, "cpu_", "cpu:", true);
+    parsed = DeviceNameUtils::ParseFullName(name, &parsed_name);
+  }
   if (parsed) {
-    const string& jobname = parsed_name.has_job ? parsed_name.job : "";
+    const string jobname = parsed_name.has_job ? parsed_name.job : "";
     return strings::StrCat("/", jobname, "/", parsed_name.type);
   } else {
     return "Unclassified";
@@ -384,8 +395,8 @@ string GetDeviceClass(const string& device_name) {
   // in VirtualScheduler. This should be revised with VirtualScheduler as well
   // as VirtualPlacer in the future.
   if (device_name.find("Channel") != string::npos) {
-    const string from = " from ";
-    const string to = " to ";
+    const string from = "_from_";
+    const string to = "_to_";
     const auto from_loc = device_name.find(from);
     const auto to_loc = device_name.find(to);
     const auto src_device_full = device_name.substr(
diff --git a/tensorflow/core/grappler/costs/utils_test.cc b/tensorflow/core/grappler/costs/utils_test.cc
index bd0af79029..baa654f475 100644
--- a/tensorflow/core/grappler/costs/utils_test.cc
+++ b/tensorflow/core/grappler/costs/utils_test.cc
@@ -264,13 +264,13 @@ TEST(TensorSizeHistogramTest, Merge) {
 TEST(DeviceClassTest, GetDeviceClass) {
   EXPECT_EQ(
       "Channel: /ps/CPU -> /worker/GPU",
-      GetDeviceClass("Channel from /job:ps/replica:0/task:0/device:CPU:0 to "
-                     "/job:worker/replica:7/task:0/device:GPU:7"));
+      GetDeviceClass("Channel_from_/job_ps/replica_0/task_0/device_CPU_0_to_"
+                     "/job_worker/replica_7/task_0/device_GPU_7"));
   EXPECT_EQ(
       "Channel: /worker_train/CPU -> /ps/GPU",
       GetDeviceClass(
-          "Channel from /job:worker_train/replica:0/task:0/device:CPU:0 to "
-          "/job:ps/replica:7/task:0/device:GPU:7"));
+          "Channel_from_/job_worker_train/replica_0/task_0/device_CPU_0_to_"
+          "/job_ps/replica_7/task_0/device_GPU_7"));
 }
 
 TEST(DeviceClassTest, GetDeviceClassForNonChannelDevice) {
diff --git a/tensorflow/core/grappler/costs/virtual_scheduler.cc b/tensorflow/core/grappler/costs/virtual_scheduler.cc
index 1ae6fac8c8..d5625ae58f 100644
--- a/tensorflow/core/grappler/costs/virtual_scheduler.cc
+++ b/tensorflow/core/grappler/costs/virtual_scheduler.cc
@@ -310,11 +310,18 @@ string VirtualScheduler::DeviceName(const NodeDef* node) const {
   return placer_.get_canonical_device_name(*node);
 }
 
+string VirtualScheduler::SanitizedDeviceName(const NodeDef* node) const {
+  // Replace the ":" characters that may be present in the device name with "_".
+  // This makes it possible to then use the resulting string in a node name.
+  return str_util::StringReplace(placer_.get_canonical_device_name(*node), ":",
+                                 "_", true);
+}
+
 string VirtualScheduler::ChannelDeviceName(const NodeDef* from,
                                            const NodeDef* to) const {
   CHECK(!initialized_) << "ChannelDeviceName is called after Init().";
-  return kChannelDevice + ": from " + DeviceName(from) + " to " +
-         DeviceName(to);
+  return kChannelDevice + "_from_" + SanitizedDeviceName(from) + "_to_" +
+         SanitizedDeviceName(to);
 }
 
 std::pair<const NodeDef*, const NodeDef*> VirtualScheduler::CreateSendRecv(
@@ -335,15 +342,15 @@ std::pair<const NodeDef*, const NodeDef*> VirtualScheduler::CreateSendRecv(
   auto input_node_port_num = NodePosition(input_name);
   string src_name;
   if (input_node_port_num >= 0) {
-    src_name = strings::StrCat(from->name(), ":", input_node_port_num);
+    src_name = strings::StrCat(from->name(), "_", input_node_port_num);
   } else {
-    src_name = strings::StrCat(from->name(), ":minus1");
+    src_name = strings::StrCat(from->name(), "_minus1");
   }
 
   // _Send op.
   auto* send = new NodeDef();
-  send->set_name("Send " + src_name + " from " + DeviceName(from) + " to " +
-                 DeviceName(to));
+  send->set_name("Send_" + src_name + "_from_" + SanitizedDeviceName(from) +
+                 "_to_" + SanitizedDeviceName(to));
   send->set_op("_Send");
   send->add_input(from->name());
   send->set_device(ChannelDeviceName(from, to));
@@ -354,7 +361,7 @@ std::pair<const NodeDef*, const NodeDef*> VirtualScheduler::CreateSendRecv(
 
   // _Recv op.
   auto* recv = new NodeDef();
-  recv->set_name("Recv " + src_name + " on " + DeviceName(to));
+  recv->set_name("Recv_" + src_name + "_on_" + SanitizedDeviceName(to));
   recv->set_op("_Recv");
   recv->add_input(send->name());
   recv->set_device(DeviceName(to));
@@ -500,8 +507,8 @@ Costs& VirtualScheduler::FindOrCreateZero(const string& op_name,
 bool VirtualScheduler::MarkCurrNodeExecuted(const Costs& node_costs) {
   // Update graph_costs_ and per-op costs.
   graph_costs_ = CombineCosts(graph_costs_, node_costs);
-  const auto* node = ready_nodes_->GetCurrNode();
-  const auto& op_name = node->op();
+  const NodeDef* node = ready_nodes_->GetCurrNode();
+  const string& op_name = node->op();
 
   // Also keep track of op counts and times per op (with their shapes).
   OpContext op_context = GetCurrNode();
@@ -651,7 +658,7 @@ Costs VirtualScheduler::Summary() const {
             << ", num_nodes = " << state.nodes_executed.size()
             << ", execution_time = " << state.GetCurrTime().count()
             << ", memory usage: "
-            << "persistenst = "
+            << "persistent = "
             << strings::HumanReadableNumBytes(persistent_memory_usage)
             << ", peak = "
             << strings::HumanReadableNumBytes(state.max_memory_usage)
diff --git a/tensorflow/core/grappler/costs/virtual_scheduler.h b/tensorflow/core/grappler/costs/virtual_scheduler.h
index 8741afff7d..c9a032d5f8 100644
--- a/tensorflow/core/grappler/costs/virtual_scheduler.h
+++ b/tensorflow/core/grappler/costs/virtual_scheduler.h
@@ -303,6 +303,7 @@ class VirtualScheduler {
   std::pair<const NodeDef*, const NodeDef*> CreateSendRecv(
       const NodeDef* from, const NodeDef* to, const string& input_name);
   string DeviceName(const NodeDef* node) const;
+  string SanitizedDeviceName(const NodeDef* node) const;
   string ChannelDeviceName(const NodeDef* from, const NodeDef* to) const;
 
   // Helper methods.
diff --git a/tensorflow/core/grappler/costs/virtual_scheduler_test.cc b/tensorflow/core/grappler/costs/virtual_scheduler_test.cc
index 5656aab4b4..d291a04308 100644
--- a/tensorflow/core/grappler/costs/virtual_scheduler_test.cc
+++ b/tensorflow/core/grappler/costs/virtual_scheduler_test.cc
@@ -1471,13 +1471,13 @@ TEST_F(VirtualSchedulerTest, InterDeviceTransfer) {
 
   // Helper lambda to extract port num from _Send and _Recv op name.
   auto get_port_num = [](const string& name) -> int {
-    if (name.find("bn:0") != std::string::npos) {
+    if (name.find("bn_0") != std::string::npos) {
       return 0;
-    } else if (name.find("bn:1") != std::string::npos) {
+    } else if (name.find("bn_1") != std::string::npos) {
       return 1;
-    } else if (name.find("bn:2") != std::string::npos) {
+    } else if (name.find("bn_2") != std::string::npos) {
       return 2;
-    } else if (name.find("bn:minus1") != std::string::npos) {
+    } else if (name.find("bn_minus1") != std::string::npos) {
       return -1;
     }
     return -999;
-- 
GitLab


From 453515d166a7d3a42e7075f2267ba85a9ff25b96 Mon Sep 17 00:00:00 2001
From: Asim Shankar <ashankar@google.com>
Date: Thu, 12 Oct 2017 11:31:21 -0700
Subject: [PATCH 308/909] Java: Tweak to address some Javadoc errors.

PiperOrigin-RevId: 171987329
---
 .../main/java/org/tensorflow/DataType.java    |  9 ++++----
 .../src/main/java/org/tensorflow/Output.java  |  4 ++--
 .../src/main/java/org/tensorflow/Tensor.java  | 22 ++++++++++---------
 .../java/org/tensorflow/op/core/Constant.java |  5 +++--
 .../org/tensorflow/types/package-info.java    | 20 ++++++++---------
 5 files changed, 32 insertions(+), 28 deletions(-)

diff --git a/tensorflow/java/src/main/java/org/tensorflow/DataType.java b/tensorflow/java/src/main/java/org/tensorflow/DataType.java
index e835101d08..7b92be6d38 100644
--- a/tensorflow/java/src/main/java/org/tensorflow/DataType.java
+++ b/tensorflow/java/src/main/java/org/tensorflow/DataType.java
@@ -19,9 +19,7 @@ import java.util.HashMap;
 import java.util.Map;
 import org.tensorflow.types.UInt8;
 
-/**
- * Represents the type of elements in a {@link Tensor} as an enum.
- */
+/** Represents the type of elements in a {@link Tensor} as an enum. */
 public enum DataType {
   /** 32-bit single precision floating point. */
   FLOAT(1),
@@ -59,7 +57,7 @@ public enum DataType {
   int c() {
     return value;
   }
-  
+
   // Cached to avoid copying it
   private static final DataType[] values = values();
 
@@ -77,6 +75,9 @@ public enum DataType {
    * Returns the DataType of a Tensor whose elements have the type specified by class {@code c}.
    *
    * @param c The class describing the TensorFlow type of interest.
+   * @return The {@code DataType} enum corresponding to {@code c}.
+   * @throws IllegalArgumentException if objects of {@code c} do not correspond to a TensorFlow
+   *     datatype.
    */
   public static DataType fromClass(Class<?> c) {
     DataType dtype = typeCodes.get(c);
diff --git a/tensorflow/java/src/main/java/org/tensorflow/Output.java b/tensorflow/java/src/main/java/org/tensorflow/Output.java
index 0e17a722ff..479dc8574c 100644
--- a/tensorflow/java/src/main/java/org/tensorflow/Output.java
+++ b/tensorflow/java/src/main/java/org/tensorflow/Output.java
@@ -20,8 +20,8 @@ import java.util.Objects;
 /**
  * A symbolic handle to a tensor produced by an {@link Operation}.
  *
- * <p>An Output<T> is a symbolic handle to a Tensor<T>. The value of the tensor is computed by
- * executing the {@link Operation} in a {@link Session}.
+ * <p>An {@code Output<T>} is a symbolic handle to a {@code Tensor<T>}. The value of the tensor is
+ * computed by executing the {@link Operation} in a {@link Session}.
  *
  * <p>By implementing the {@link Operand} interface, instances of this class also act as operands to
  * {@link org.tensorflow.op.Op Op} instances.
diff --git a/tensorflow/java/src/main/java/org/tensorflow/Tensor.java b/tensorflow/java/src/main/java/org/tensorflow/Tensor.java
index d4b753628b..24a3775db6 100644
--- a/tensorflow/java/src/main/java/org/tensorflow/Tensor.java
+++ b/tensorflow/java/src/main/java/org/tensorflow/Tensor.java
@@ -47,11 +47,11 @@ public final class Tensor<T> implements AutoCloseable {
   /**
    * Creates a Tensor from a Java object.
    *
-   * <p>A {@code Tensor} is a multi-dimensional array of elements of a limited set of types ({@link
-   * types}), so not all Java objects can be converted to a {@code Tensor}. In particular, the
-   * argument {@code obj} must be either a primitive (float, double, int, long, boolean, byte) or a
-   * multi-dimensional array of one of those primitives. The argument {@code type} specifies how to
-   * interpret the first argument as a TensorFlow type. For example:
+   * <p>A {@code Tensor} is a multi-dimensional array of elements of a limited set of types. Not all
+   * Java objects can be converted to a {@code Tensor}. In particular, the argument {@code obj} must
+   * be either a primitive (float, double, int, long, boolean, byte) or a multi-dimensional array of
+   * one of those primitives. The argument {@code type} specifies how to interpret the first
+   * argument as a TensorFlow type. For example:
    *
    * <pre>{@code
    * // Valid: A 64-bit integer scalar.
@@ -94,9 +94,9 @@ public final class Tensor<T> implements AutoCloseable {
    * Tensor<String> m = Tensor.create(matrix, String.class);
    * }</pre>
    *
-   * @param obj The object to convert to a Tensor<T>. Note that whether it is compatible with the
-   *     type T is not checked by the type system. For type-safe creation of tensors, use {@link
-   *     Tensors}.
+   * @param obj The object to convert to a {@code Tensor<T>}. Note that whether it is compatible
+   *     with the type T is not checked by the type system. For type-safe creation of tensors, use
+   *     {@link Tensors}.
    * @param type The class object representing the type T.
    * @throws IllegalArgumentException if {@code obj} is not compatible with the TensorFlow type
    *     system.
@@ -229,7 +229,8 @@ public final class Tensor<T> implements AutoCloseable {
    *
    * <p>Creates a Tensor with the provided shape of any type where the tensor's data has been
    * encoded into {@code data} as per the specification of the TensorFlow <a
-   * href="https://www.tensorflow.org/code/tensorflow/c/c_api.h">C API</a>.
+   * href="https://www.tensorflow.org/code/tensorflow/c/c_api.h">C
+   * API</a>.
    *
    * @param <T> the tensor element type
    * @param type the tensor element type, represented as a class object.
@@ -249,7 +250,8 @@ public final class Tensor<T> implements AutoCloseable {
    *
    * <p>Creates a Tensor with the provided shape of any type where the tensor's data has been
    * encoded into {@code data} as per the specification of the TensorFlow <a
-   * href="https://www.tensorflow.org/code/tensorflow/c/c_api.h">C API</a>.
+   * href="https://www.tensorflow.org/code/tensorflow/c/c_api.h">C
+   * API</a>.
    *
    * @param <T> The tensor element type
    * @param type the tensor element type, specified as a DataType. This must agree with T.
diff --git a/tensorflow/java/src/main/java/org/tensorflow/op/core/Constant.java b/tensorflow/java/src/main/java/org/tensorflow/op/core/Constant.java
index 725c81765a..de4049f66b 100644
--- a/tensorflow/java/src/main/java/org/tensorflow/op/core/Constant.java
+++ b/tensorflow/java/src/main/java/org/tensorflow/op/core/Constant.java
@@ -134,10 +134,11 @@ public final class Constant<T> extends PrimitiveOp implements Operand<T> {
    *
    * <p>Creates a Constant with the provided shape of any type where the constant data has been
    * encoded into {@code data} as per the specification of the TensorFlow <a
-   * href="https://www.tensorflow.org/code/tensorflow/c/c_api.h">C API</a>.
+   * href="https://www.tensorflow.org/code/tensorflow/c/c_api.h">C
+   * API</a>.
    *
    * @param scope is a scope used to add the underlying operation.
-   * @param dataType the tensor datatype.
+   * @param type the tensor datatype.
    * @param shape the tensor shape.
    * @param data a buffer containing the tensor data.
    * @throws IllegalArgumentException If the tensor datatype or shape is not compatible with the
diff --git a/tensorflow/java/src/main/java/org/tensorflow/types/package-info.java b/tensorflow/java/src/main/java/org/tensorflow/types/package-info.java
index 96018c5366..4042fb1669 100644
--- a/tensorflow/java/src/main/java/org/tensorflow/types/package-info.java
+++ b/tensorflow/java/src/main/java/org/tensorflow/types/package-info.java
@@ -14,16 +14,16 @@ limitations under the License.
 ==============================================================================*/
 
 /**
- * Defines classes that represent TensorFlow data types. For each possible data type
- * that can be used in a tensor, there is a corresponding class that
- * is used to represent it. For example, the TensorFlow int32 type is represented by
- * the type {@link Integer} and by the class object {@code Integer.class}. The former is used to
- * support compile-time checking of tensor element types and the latter is used for
- * run-time checking of element types. Classes appearing in this package, such as
- * UInt8, represent TensorFlow data types for which there is no existing Java equivalent.
+ * Defines classes that represent TensorFlow data types. For each possible data type that can be
+ * used in a tensor, there is a corresponding class that is used to represent it. For example, the
+ * TensorFlow int32 type is represented by the type {@link java.lang.Integer} and by the class
+ * object {@code Integer.class}. The former is used to support compile-time checking of tensor
+ * element types and the latter is used for run-time checking of element types. Classes appearing in
+ * this package, such as UInt8, represent TensorFlow data types for which there is no existing Java
+ * equivalent.
  *
- * <p>TensorFlow element types are also separately represented by the {@link DataType} enum, with
- * one enum value per element type. The enum representation is not usually needed, but
- * can be obtained using {@link DataType.fromClass}.
+ * <p>TensorFlow element types are also separately represented by the {@link
+ * org.tensorflow.DataType} enum, with one enum value per element type. The enum representation is
+ * not usually needed, but can be obtained using {@link org.tensorflow.DataType.fromClass}.
  */
 package org.tensorflow.types;
-- 
GitLab


From 2a9b5312ea0e1ee5090e10288c671ab11efd065b Mon Sep 17 00:00:00 2001
From: AG Ramesh <ag.ramesh@intel.com>
Date: Thu, 12 Oct 2017 12:07:34 -0700
Subject: [PATCH 309/909] Fixing a master regression (#13562)

---
 tensorflow/core/BUILD                         | 15 ++++++++++++++-
 tensorflow/core/graph/mkl_layout_pass.cc      |  2 +-
 tensorflow/core/graph/mkl_layout_pass_test.cc |  2 +-
 3 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index ac6f6ff333..7b9333a75d 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -2695,7 +2695,20 @@ tf_cc_test_mkl(
         "//tensorflow/cc:sendrecv_ops",
         "//tensorflow/core/kernels:ops_util",
         "//third_party/eigen3",
-    ],
+    ] + if_mkl([
+        "//tensorflow/core/kernels:mkl_aggregate_ops",
+        "//tensorflow/core/kernels:mkl_concat_op",
+        "//tensorflow/core/kernels:mkl_conv_op",
+        "//tensorflow/core/kernels:mkl_cwise_ops_common",
+        "//tensorflow/core/kernels:mkl_fused_batch_norm_op",
+        "//tensorflow/core/kernels:mkl_identity_op",
+        "//tensorflow/core/kernels:mkl_input_conversion_op",
+        "//tensorflow/core/kernels:mkl_lrn_op",
+        "//tensorflow/core/kernels:mkl_pooling_ops",
+        "//tensorflow/core/kernels:mkl_relu_op",
+        "//tensorflow/core/kernels:mkl_reshape_op",
+        "//tensorflow/core/kernels:mkl_tfconv_op",
+    ]),
 )
 
 tf_cc_tests_gpu(
diff --git a/tensorflow/core/graph/mkl_layout_pass.cc b/tensorflow/core/graph/mkl_layout_pass.cc
index f87a94a76a..f4c9073dee 100644
--- a/tensorflow/core/graph/mkl_layout_pass.cc
+++ b/tensorflow/core/graph/mkl_layout_pass.cc
@@ -543,7 +543,7 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
     string reason;
 
     // Substring that should be checked for in device name for CPU device.
-    const char* const kCPUDeviceSubStr = "cpu";
+    const char* const kCPUDeviceSubStr = "CPU";
 
     // If Op has been specifically assigned to a non-CPU device, then No.
     if (!n->assigned_device_name().empty() &&
diff --git a/tensorflow/core/graph/mkl_layout_pass_test.cc b/tensorflow/core/graph/mkl_layout_pass_test.cc
index a2b2f6530d..abc63e4f35 100644
--- a/tensorflow/core/graph/mkl_layout_pass_test.cc
+++ b/tensorflow/core/graph/mkl_layout_pass_test.cc
@@ -39,7 +39,7 @@ limitations under the License.
 namespace tensorflow {
 namespace {
 
-const char kCPUDevice[] = "/job:a/replica:0/task:0/cpu:0";
+const char kCPUDevice[] = "/job:a/replica:0/task:0/device:CPU:0";
 const char kGPUDevice[] = "/job:a/replica:0/task:0/device:GPU:0";
 
 static void InitGraph(const string& s, Graph* graph,
-- 
GitLab


From deb72df13e588eda07968adc306c1a87416cf7fc Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 12 Oct 2017 12:41:40 -0700
Subject: [PATCH 310/909] This creates a library under contrib for
 implementations of common metric learning losses, starting with: *
 [Contrastive
 loss](http://yann.lecun.com/exdb/publis/pdf/hadsell-chopra-lecun-06.pdf) *
 [Triplet Loss with semihard negative
 mining](https://arxiv.org/abs/1503.03832) * [Npairs
 loss](http://www.nec-labs.com/uploads/images/Department-Images/MediaAnalytics/papers/nips16_npairmetriclearning.pdf)
 * Npairs loss w/ multilabel support * [Lifted structured
 loss](http://www.cv-foundation.org/openaccess/content_cvpr_2016/papers/Song_Deep_Metric_Learning_CVPR_2016_paper.pdf)
 * [Structured clustering embedding](https://arxiv.org/abs/1612.01213)

PiperOrigin-RevId: 171997156
---
 tensorflow/contrib/BUILD                      |    1 +
 tensorflow/contrib/cmake/tf_python.cmake      |    1 +
 tensorflow/contrib/losses/BUILD               |   44 +
 tensorflow/contrib/losses/__init__.py         |    3 +-
 .../losses/python/metric_learning/__init__.py |   39 +
 .../python/metric_learning/metric_loss_ops.py | 1031 +++++++++++++++++
 .../metric_learning/metric_loss_ops_test.py   |  562 +++++++++
 7 files changed, 1680 insertions(+), 1 deletion(-)
 create mode 100644 tensorflow/contrib/losses/python/metric_learning/__init__.py
 create mode 100644 tensorflow/contrib/losses/python/metric_learning/metric_loss_ops.py
 create mode 100644 tensorflow/contrib/losses/python/metric_learning/metric_loss_ops_test.py

diff --git a/tensorflow/contrib/BUILD b/tensorflow/contrib/BUILD
index 559e3e60d7..3d580fae14 100644
--- a/tensorflow/contrib/BUILD
+++ b/tensorflow/contrib/BUILD
@@ -53,6 +53,7 @@ py_library(
         "//tensorflow/contrib/linear_optimizer:sdca_ops_py",
         "//tensorflow/contrib/lookup:lookup_py",
         "//tensorflow/contrib/losses:losses_py",
+        "//tensorflow/contrib/losses:metric_learning_py",
         "//tensorflow/contrib/memory_stats:memory_stats_py",
         "//tensorflow/contrib/meta_graph_transform",
         "//tensorflow/contrib/metrics:metrics_py",
diff --git a/tensorflow/contrib/cmake/tf_python.cmake b/tensorflow/contrib/cmake/tf_python.cmake
index 883b36b3fb..e83618a94e 100755
--- a/tensorflow/contrib/cmake/tf_python.cmake
+++ b/tensorflow/contrib/cmake/tf_python.cmake
@@ -499,6 +499,7 @@ add_python_module("tensorflow/contrib/lookup")
 add_python_module("tensorflow/contrib/losses")
 add_python_module("tensorflow/contrib/losses/python")
 add_python_module("tensorflow/contrib/losses/python/losses")
+add_python_module("tensorflow/contrib/losses/python/metric_learning")
 add_python_module("tensorflow/contrib/makefile")
 add_python_module("tensorflow/contrib/makefile/test")
 add_python_module("tensorflow/contrib/memory_stats")
diff --git a/tensorflow/contrib/losses/BUILD b/tensorflow/contrib/losses/BUILD
index f75b0aa1b3..33fbbe12d3 100644
--- a/tensorflow/contrib/losses/BUILD
+++ b/tensorflow/contrib/losses/BUILD
@@ -15,6 +15,7 @@ py_library(
         "__init__.py",
         "python/losses/__init__.py",
         "python/losses/loss_ops.py",
+        "python/metric_learning/metric_loss_ops.py",
     ],
     srcs_version = "PY2AND3",
     deps = [
@@ -50,6 +51,49 @@ py_test(
     ],
 )
 
+py_library(
+    name = "metric_learning_py",
+    srcs = [
+        "python/metric_learning/__init__.py",
+        "python/metric_learning/metric_loss_ops.py",
+    ],
+    srcs_version = "PY2AND3",
+    deps = [
+        "//tensorflow/contrib/framework:framework_py",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:framework_for_generated_wrappers",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:nn",
+        "//tensorflow/python:nn_ops",
+        "//tensorflow/python:script_ops",
+        "//tensorflow/python:util",
+    ],
+)
+
+py_test(
+    name = "metric_loss_ops_test",
+    srcs = [
+        "python/metric_learning/metric_loss_ops_test.py",
+    ],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":metric_learning_py",
+        "//tensorflow/contrib/framework:framework_py",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:errors",
+        "//tensorflow/python:framework_for_generated_wrappers",
+        "//tensorflow/python:init_ops",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:random_ops",
+        "//tensorflow/python:random_seed",
+        "//tensorflow/python:training",
+        "//tensorflow/python:variable_scope",
+        "//tensorflow/python:variables",
+        "//third_party/py/numpy",
+    ],
+)
+
 filegroup(
     name = "all_files",
     srcs = glob(
diff --git a/tensorflow/contrib/losses/__init__.py b/tensorflow/contrib/losses/__init__.py
index 790bf61367..db58647d48 100644
--- a/tensorflow/contrib/losses/__init__.py
+++ b/tensorflow/contrib/losses/__init__.py
@@ -22,10 +22,10 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from tensorflow.contrib.losses.python import metric_learning
 # pylint: disable=wildcard-import
 from tensorflow.contrib.losses.python.losses import *
 # pylint: enable=wildcard-import
-
 from tensorflow.python.util.all_util import remove_undocumented
 
 _allowed_symbols = [
@@ -43,5 +43,6 @@ _allowed_symbols = [
     'sigmoid_cross_entropy',
     'softmax_cross_entropy',
     'sparse_softmax_cross_entropy',
+    'metric_learning'
 ]
 remove_undocumented(__name__, _allowed_symbols)
diff --git a/tensorflow/contrib/losses/python/metric_learning/__init__.py b/tensorflow/contrib/losses/python/metric_learning/__init__.py
new file mode 100644
index 0000000000..4e551d6aca
--- /dev/null
+++ b/tensorflow/contrib/losses/python/metric_learning/__init__.py
@@ -0,0 +1,39 @@
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Ops for building neural network losses.
+
+See @{$python/contrib.losses}.
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+# pylint: disable=wildcard-import
+from tensorflow.contrib.losses.python.metric_learning.metric_loss_ops import *
+# pylint: enable=wildcard-import
+from tensorflow.python.util.all_util import remove_undocumented
+
+_allowed_symbols = [
+    'contrastive_loss',
+    'cluster_loss',
+    'lifted_struct_loss',
+    'npairs_loss',
+    'npairs_loss_multilabel',
+    'triplet_semihard_loss',
+]
+remove_undocumented(__name__, _allowed_symbols)
+
+
diff --git a/tensorflow/contrib/losses/python/metric_learning/metric_loss_ops.py b/tensorflow/contrib/losses/python/metric_learning/metric_loss_ops.py
new file mode 100644
index 0000000000..c3a57ba51b
--- /dev/null
+++ b/tensorflow/contrib/losses/python/metric_learning/metric_loss_ops.py
@@ -0,0 +1,1031 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Implements various metric learning losses."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import sparse_tensor
+from tensorflow.python.framework import tensor_shape
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import logging_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import nn
+from tensorflow.python.ops import script_ops
+from tensorflow.python.ops import sparse_ops
+from tensorflow.python.summary import summary
+try:
+  # pylint: disable=g-import-not-at-top
+  from sklearn import metrics
+  HAS_SKLEARN = True
+except ImportError:
+  HAS_SKLEARN = False
+
+
+def pairwise_distance(feature, squared=False):
+  """Computes the pairwise distance matrix with numerical stability.
+
+  output[i, j] = || feature[i, :] - feature[j, :] ||_2
+
+  Args:
+    feature: 2-D Tensor of size [number of data, feature dimension].
+    squared: Boolean, whether or not to square the pairwise distances.
+
+  Returns:
+    pairwise_distances: 2-D Tensor of size [number of data, number of data].
+  """
+  pairwise_distances_squared = math_ops.add(
+      math_ops.reduce_sum(
+          math_ops.square(feature),
+          axis=[1],
+          keep_dims=True),
+      math_ops.reduce_sum(
+          math_ops.square(
+              array_ops.transpose(feature)),
+          axis=[0],
+          keep_dims=True)) - 2.0 * math_ops.matmul(
+              feature, array_ops.transpose(feature))
+
+  # Deal with numerical inaccuracies. Set small negatives to zero.
+  pairwise_distances_squared = math_ops.maximum(pairwise_distances_squared, 0.0)
+  # Get the mask where the zero distances are at.
+  error_mask = math_ops.less_equal(pairwise_distances_squared, 0.0)
+
+  # Optionally take the sqrt.
+  if squared:
+    pairwise_distances = pairwise_distances_squared
+  else:
+    pairwise_distances = math_ops.sqrt(
+        pairwise_distances_squared + math_ops.to_float(error_mask) * 1e-16)
+
+  # Undo conditionally adding 1e-16.
+  pairwise_distances = math_ops.multiply(
+      pairwise_distances, math_ops.to_float(math_ops.logical_not(error_mask)))
+
+  num_data = array_ops.shape(feature)[0]
+  # Explicitly set diagonals to zero.
+  mask_offdiagonals = array_ops.ones_like(pairwise_distances) - array_ops.diag(
+      array_ops.ones([num_data]))
+  pairwise_distances = math_ops.multiply(pairwise_distances, mask_offdiagonals)
+  return pairwise_distances
+
+
+def contrastive_loss(labels, embeddings_anchor, embeddings_positive,
+                     margin=1.0):
+  """Computes the contrastive loss.
+
+  This loss encourages the embedding to be close to each other for
+    the samples of the same label and the embedding to be far apart at least
+    by the margin constant for the samples of different labels.
+  See: http://yann.lecun.com/exdb/publis/pdf/hadsell-chopra-lecun-06.pdf
+
+  Args:
+    labels: 1-D tf.int32 `Tensor` with shape [batch_size] of
+      binary labels indicating positive vs negative pair.
+    embeddings_anchor: 2-D float `Tensor` of embedding vectors for the anchor
+      images. Embeddings should be l2 normalized.
+    embeddings_positive: 2-D float `Tensor` of embedding vectors for the
+      positive images. Embeddings should be l2 normalized.
+    margin: margin term in the loss definition.
+
+  Returns:
+    contrastive_loss: tf.float32 scalar.
+  """
+  # Get per pair distances
+  distances = math_ops.sqrt(
+      math_ops.reduce_sum(
+          math_ops.square(embeddings_anchor - embeddings_positive), 1))
+
+  # Add contrastive loss for the siamese network.
+  #   label here is {0,1} for neg, pos.
+  return math_ops.reduce_mean(
+      math_ops.to_float(labels) * math_ops.square(distances) +
+      (1. - math_ops.to_float(labels)) *
+      math_ops.square(math_ops.maximum(margin - distances, 0.)),
+      name='contrastive_loss')
+
+
+def masked_maximum(data, mask, dim=1):
+  """Computes the axis wise maximum over chosen elements.
+
+  Args:
+    data: 2-D float `Tensor` of size [n, m].
+    mask: 2-D Boolean `Tensor` of size [n, m].
+    dim: The dimension over which to compute the maximum.
+
+  Returns:
+    masked_maximums: N-D `Tensor`.
+      The maximized dimension is of size 1 after the operation.
+  """
+  axis_minimums = math_ops.reduce_min(data, dim, keep_dims=True)
+  masked_maximums = math_ops.reduce_max(
+      math_ops.multiply(
+          data - axis_minimums, mask), dim, keep_dims=True) + axis_minimums
+  return masked_maximums
+
+
+def masked_minimum(data, mask, dim=1):
+  """Computes the axis wise minimum over chosen elements.
+
+  Args:
+    data: 2-D float `Tensor` of size [n, m].
+    mask: 2-D Boolean `Tensor` of size [n, m].
+    dim: The dimension over which to compute the minimum.
+
+  Returns:
+    masked_minimums: N-D `Tensor`.
+      The minimized dimension is of size 1 after the operation.
+  """
+  axis_maximums = math_ops.reduce_max(data, dim, keep_dims=True)
+  masked_minimums = math_ops.reduce_min(
+      math_ops.multiply(
+          data - axis_maximums, mask), dim, keep_dims=True) + axis_maximums
+  return masked_minimums
+
+
+def triplet_semihard_loss(labels, embeddings, margin=1.0):
+  """Computes the triplet loss with semi-hard negative mining.
+
+  The loss encourages the positive distances (between a pair of embeddings with
+  the same labels) to be smaller than the minimum negative distance among
+  which are at least greater than the positive distance plus the margin constant
+  (called semi-hard negative) in the mini-batch. If no such negative exists,
+  uses the largest negative distance instead.
+  See: https://arxiv.org/abs/1503.03832.
+
+  Args:
+    labels: 1-D tf.int32 `Tensor` with shape [batch_size] of
+      multiclass integer labels.
+    embeddings: 2-D float `Tensor` of embedding vectors. Embeddings should
+      be l2 normalized.
+    margin: Float, margin term in the loss definition.
+
+  Returns:
+    triplet_loss: tf.float32 scalar.
+  """
+  # Reshape [batch_size] label tensor to a [batch_size, 1] label tensor.
+  lshape = array_ops.shape(labels)
+  assert lshape.shape == 1
+  labels = array_ops.reshape(labels, [lshape[0], 1])
+
+  # Build pairwise squared distance matrix.
+  pdist_matrix = pairwise_distance(embeddings, squared=True)
+  # Build pairwise binary adjacency matrix.
+  adjacency = math_ops.equal(labels, array_ops.transpose(labels))
+  # Invert so we can select negatives only.
+  adjacency_not = math_ops.logical_not(adjacency)
+
+  batch_size = array_ops.size(labels)
+
+  # Compute the mask.
+  pdist_matrix_tile = array_ops.tile(pdist_matrix, [batch_size, 1])
+  mask = math_ops.logical_and(
+      array_ops.tile(adjacency_not, [batch_size, 1]),
+      math_ops.greater(
+          pdist_matrix_tile, array_ops.reshape(
+              array_ops.transpose(pdist_matrix), [-1, 1])))
+  mask_final = array_ops.reshape(
+      math_ops.greater(
+          math_ops.reduce_sum(
+              math_ops.cast(
+                  mask, dtype=dtypes.float32), 1, keep_dims=True),
+          0.0), [batch_size, batch_size])
+  mask_final = array_ops.transpose(mask_final)
+
+  adjacency_not = math_ops.cast(adjacency_not, dtype=dtypes.float32)
+  mask = math_ops.cast(mask, dtype=dtypes.float32)
+
+  # negatives_outside: smallest D_an where D_an > D_ap.
+  negatives_outside = array_ops.reshape(
+      masked_minimum(pdist_matrix_tile, mask), [batch_size, batch_size])
+  negatives_outside = array_ops.transpose(negatives_outside)
+
+  # negatives_inside: largest D_an.
+  negatives_inside = array_ops.tile(
+      masked_maximum(pdist_matrix, adjacency_not), [1, batch_size])
+  semi_hard_negatives = array_ops.where(
+      mask_final, negatives_outside, negatives_inside)
+
+  loss_mat = math_ops.add(margin, pdist_matrix - semi_hard_negatives)
+
+  mask_positives = math_ops.cast(
+      adjacency, dtype=dtypes.float32) - array_ops.diag(
+          array_ops.ones([batch_size]))
+
+  # In lifted-struct, the authors multiply 0.5 for upper triangular
+  #   in semihard, they take all positive pairs except the diagonal.
+  num_positives = math_ops.reduce_sum(mask_positives)
+
+  triplet_loss = math_ops.truediv(
+      math_ops.reduce_sum(
+          math_ops.maximum(
+              math_ops.multiply(loss_mat, mask_positives), 0.0)),
+      num_positives,
+      name='triplet_semihard_loss')
+
+  return triplet_loss
+
+
+# pylint: disable=line-too-long
+def npairs_loss(labels, embeddings_anchor, embeddings_positive,
+                reg_lambda=0.002, print_losses=False):
+  """Computes the npairs loss.
+
+  Npairs loss expects paired data where a pair is composed of samples from the
+  same labels and each pairs in the minibatch have different labels. The loss
+  has two components. The first component is the L2 regularizer on the
+  embedding vectors. The second component is the sum of cross entropy loss
+  which takes each row of the pair-wise similarity matrix as logits and
+  the remapped one-hot labels as labels.
+
+  See: http://www.nec-labs.com/uploads/images/Department-Images/MediaAnalytics/papers/nips16_npairmetriclearning.pdf
+
+  Args:
+    labels: 1-D tf.int32 `Tensor` of shape [batch_size/2].
+    embeddings_anchor: 2-D Tensor of shape [batch_size/2, embedding_dim] for the
+      embedding vectors for the anchor images. Embeddings should not be
+      l2 normalized.
+    embeddings_positive: 2-D Tensor of shape [batch_size/2, embedding_dim] for the
+      embedding vectors for the positive images. Embeddings should not be
+      l2 normalized.
+    reg_lambda: Float. L2 regularization term on the embedding vectors.
+    print_losses: Boolean. Option to print the xent and l2loss.
+
+  Returns:
+    npairs_loss: tf.float32 scalar.
+  """
+  # pylint: enable=line-too-long
+  # Add the regularizer on the embedding.
+  reg_anchor = math_ops.reduce_mean(
+      math_ops.reduce_sum(math_ops.square(embeddings_anchor), 1))
+  reg_positive = math_ops.reduce_mean(
+      math_ops.reduce_sum(math_ops.square(embeddings_positive), 1))
+  l2loss = math_ops.multiply(
+      0.25 * reg_lambda, reg_anchor + reg_positive, name='l2loss')
+
+  # Get per pair similarities.
+  similarity_matrix = math_ops.matmul(
+      embeddings_anchor, embeddings_positive, transpose_a=False,
+      transpose_b=True)
+
+  # Reshape [batch_size] label tensor to a [batch_size, 1] label tensor.
+  lshape = array_ops.shape(labels)
+  assert lshape.shape == 1
+  labels = array_ops.reshape(labels, [lshape[0], 1])
+
+  labels_remapped = math_ops.to_float(
+      math_ops.equal(labels, array_ops.transpose(labels)))
+  labels_remapped /= math_ops.reduce_sum(labels_remapped, 1, keep_dims=True)
+
+  # Add the softmax loss.
+  xent_loss = nn.softmax_cross_entropy_with_logits(
+      logits=similarity_matrix, labels=labels_remapped)
+  xent_loss = math_ops.reduce_mean(xent_loss, name='xentropy')
+
+  if print_losses:
+    xent_loss = logging_ops.Print(
+        xent_loss, ['cross entropy:', xent_loss, 'l2loss:', l2loss])
+
+  return l2loss + xent_loss
+
+
+def _build_multilabel_adjacency(sparse_labels):
+  """Builds multilabel adjacency matrix.
+
+  As of March 14th, 2017, there's no op for the dot product between
+  two sparse tensors in TF. However, there is `sparse_minimum` op which is
+  equivalent to an AND op between two sparse boolean tensors.
+  This computes the dot product between two sparse boolean inputs.
+
+  Args:
+    sparse_labels: List of 1-D boolean sparse tensors.
+
+  Returns:
+    adjacency_matrix: 2-D dense `Tensor`.
+  """
+  num_pairs = len(sparse_labels)
+  adjacency_matrix = array_ops.zeros([num_pairs, num_pairs])
+  for i in range(num_pairs):
+    for j in range(num_pairs):
+      sparse_dot_product = math_ops.to_float(
+          sparse_ops.sparse_reduce_sum(sparse_ops.sparse_minimum(
+              sparse_labels[i], sparse_labels[j])))
+      sparse_dot_product = array_ops.expand_dims(sparse_dot_product, 0)
+      sparse_dot_product = array_ops.expand_dims(sparse_dot_product, 1)
+      one_hot_matrix = array_ops.pad(sparse_dot_product,
+                                     [[i, num_pairs-i-1],
+                                      [j, num_pairs-j-1]], 'CONSTANT')
+      adjacency_matrix += one_hot_matrix
+
+  return adjacency_matrix
+
+
+def npairs_loss_multilabel(sparse_labels, embeddings_anchor,
+                           embeddings_positive, reg_lambda=0.002,
+                           print_losses=False):
+  r"""Computes the npairs loss with multilabel data.
+
+  Npairs loss expects paired data where a pair is composed of samples from the
+  same labels and each pairs in the minibatch have different labels. The loss
+  has two components. The first component is the L2 regularizer on the
+  embedding vectors. The second component is the sum of cross entropy loss
+  which takes each row of the pair-wise similarity matrix as logits and
+  the remapped one-hot labels as labels. Here, the similarity is defined by the
+  dot product between two embedding vectors. S_{i,j} = f(x_i)^T f(x_j)
+
+  To deal with multilabel inputs, we use the count of label intersection
+  i.e. L_{i,j} = | set_of_labels_for(i) \cap set_of_labels_for(j) |
+  Then we normalize each rows of the count based label matrix so that each row
+  sums to one.
+
+  Args:
+    sparse_labels: List of 1-D Boolean `SparseTensor` of dense_shape
+                   [batch_size/2, num_classes] labels for the anchor-pos pairs.
+    embeddings_anchor: 2-D `Tensor` of shape [batch_size/2, embedding_dim] for
+      the embedding vectors for the anchor images. Embeddings should not be
+      l2 normalized.
+    embeddings_positive: 2-D `Tensor` of shape [batch_size/2, embedding_dim] for
+      the embedding vectors for the positive images. Embeddings should not be
+      l2 normalized.
+    reg_lambda: Float. L2 regularization term on the embedding vectors.
+    print_losses: Boolean. Option to print the xent and l2loss.
+
+  Returns:
+    npairs_loss: tf.float32 scalar.
+  Raises:
+    TypeError: When the specified sparse_labels is not a `SparseTensor`.
+  """
+  if False in [isinstance(
+      l, sparse_tensor.SparseTensor) for l in sparse_labels]:
+    raise TypeError(
+        'sparse_labels must be a list of SparseTensors, but got %s' % str(
+            sparse_labels))
+
+  with ops.name_scope('NpairsLossMultiLabel'):
+    # Add the regularizer on the embedding.
+    reg_anchor = math_ops.reduce_mean(
+        math_ops.reduce_sum(math_ops.square(embeddings_anchor), 1))
+    reg_positive = math_ops.reduce_mean(
+        math_ops.reduce_sum(math_ops.square(embeddings_positive), 1))
+    l2loss = math_ops.multiply(0.25 * reg_lambda,
+                               reg_anchor + reg_positive, name='l2loss')
+
+    # Get per pair similarities.
+    similarity_matrix = math_ops.matmul(
+        embeddings_anchor, embeddings_positive, transpose_a=False,
+        transpose_b=True)
+
+    # TODO(coreylynch): need to check the sparse values
+    # TODO(coreylynch): are composed only of 0's and 1's.
+
+    multilabel_adjacency_matrix = _build_multilabel_adjacency(sparse_labels)
+    labels_remapped = math_ops.to_float(multilabel_adjacency_matrix)
+    labels_remapped /= math_ops.reduce_sum(labels_remapped, 1, keep_dims=True)
+
+    # Add the softmax loss.
+    xent_loss = nn.softmax_cross_entropy_with_logits(
+        logits=similarity_matrix, labels=labels_remapped)
+    xent_loss = math_ops.reduce_mean(xent_loss, name='xentropy')
+
+    if print_losses:
+      xent_loss = logging_ops.Print(
+          xent_loss, ['cross entropy:', xent_loss, 'l2loss:', l2loss])
+
+    return l2loss + xent_loss
+
+
+def lifted_struct_loss(labels, embeddings, margin=1.0):
+  """Computes the lifted structured loss.
+
+  The loss encourages the positive distances (between a pair of embeddings
+  with the same labels) to be smaller than any negative distances (between a
+  pair of embeddings with different labels) in the mini-batch in a way
+  that is differentiable with respect to the embedding vectors.
+  See: https://arxiv.org/abs/1511.06452.
+
+  Args:
+    labels: 1-D tf.int32 `Tensor` with shape [batch_size] of
+      multiclass integer labels.
+    embeddings: 2-D float `Tensor` of embedding vectors. Embeddings should not
+      be l2 normalized.
+    margin: Float, margin term in the loss definition.
+
+  Returns:
+    lifted_loss: tf.float32 scalar.
+  """
+  # Reshape [batch_size] label tensor to a [batch_size, 1] label tensor.
+  lshape = array_ops.shape(labels)
+  assert lshape.shape == 1
+  labels = array_ops.reshape(labels, [lshape[0], 1])
+
+  # Build pairwise squared distance matrix.
+  pairwise_distances = pairwise_distance(embeddings)
+
+  # Build pairwise binary adjacency matrix.
+  adjacency = math_ops.equal(labels, array_ops.transpose(labels))
+  # Invert so we can select negatives only.
+  adjacency_not = math_ops.logical_not(adjacency)
+
+  batch_size = array_ops.size(labels)
+
+  diff = margin - pairwise_distances
+  mask = math_ops.cast(adjacency_not, dtype=dtypes.float32)
+  # Safe maximum: Temporarily shift negative distances
+  #   above zero before taking max.
+  #     this is to take the max only among negatives.
+  row_minimums = math_ops.reduce_min(diff, 1, keep_dims=True)
+  row_negative_maximums = math_ops.reduce_max(
+      math_ops.multiply(
+          diff - row_minimums, mask), 1, keep_dims=True) + row_minimums
+
+  # Compute the loss.
+  # Keep track of matrix of maximums where M_ij = max(m_i, m_j)
+  #   where m_i is the max of alpha - negative D_i's.
+  # This matches the Caffe loss layer implementation at:
+  #   https://github.com/rksltnl/Caffe-Deep-Metric-Learning-CVPR16/blob/0efd7544a9846f58df923c8b992198ba5c355454/src/caffe/layers/lifted_struct_similarity_softmax_layer.cpp  # pylint: disable=line-too-long
+
+  max_elements = math_ops.maximum(
+      row_negative_maximums, array_ops.transpose(row_negative_maximums))
+  diff_tiled = array_ops.tile(diff, [batch_size, 1])
+  mask_tiled = array_ops.tile(mask, [batch_size, 1])
+  max_elements_vect = array_ops.reshape(
+      array_ops.transpose(max_elements), [-1, 1])
+
+  loss_exp_left = array_ops.reshape(
+      math_ops.reduce_sum(math_ops.multiply(
+          math_ops.exp(
+              diff_tiled - max_elements_vect),
+          mask_tiled), 1, keep_dims=True), [batch_size, batch_size])
+
+  loss_mat = max_elements + math_ops.log(
+      loss_exp_left + array_ops.transpose(loss_exp_left))
+  # Add the positive distance.
+  loss_mat += pairwise_distances
+
+  mask_positives = math_ops.cast(
+      adjacency, dtype=dtypes.float32) - array_ops.diag(
+          array_ops.ones([batch_size]))
+
+  # *0.5 for upper triangular, and another *0.5 for 1/2 factor for loss^2.
+  num_positives = math_ops.reduce_sum(mask_positives) / 2.0
+
+  lifted_loss = math_ops.truediv(
+      0.25 * math_ops.reduce_sum(
+          math_ops.square(
+              math_ops.maximum(
+                  math_ops.multiply(loss_mat, mask_positives), 0.0))),
+      num_positives,
+      name='liftedstruct_loss')
+  return lifted_loss
+
+
+def update_1d_tensor(y, index, value):
+  """Updates 1d tensor y so that y[index] = value.
+
+  Args:
+    y: 1-D Tensor.
+    index: index of y to modify.
+    value: new value to write at y[index].
+
+  Returns:
+    y_mod: 1-D Tensor. Tensor y after the update.
+  """
+  value = array_ops.squeeze(value)
+  # modify the 1D tensor x at index with value.
+  # ex) chosen_ids = update_1D_tensor(chosen_ids, cluster_idx, best_medoid)
+  y_before = array_ops.slice(y, [0], [index])
+  y_after = array_ops.slice(y, [index + 1], [-1])
+  y_mod = array_ops.concat([y_before, [value], y_after], 0)
+  return y_mod
+
+
+def get_cluster_assignment(pairwise_distances, centroid_ids):
+  """Assign data points to the neareset centroids.
+
+  Tensorflow has numerical instability and doesn't always choose
+    the data point with theoretically zero distance as it's nearest neighbor.
+    Thus, for each centroid in centroid_ids, explicitly assign
+    the centroid itself as the nearest centroid.
+    This is done through the mask tensor and the constraint_vect tensor.
+
+  Args:
+    pairwise_distances: 2-D Tensor of pairwise distances.
+    centroid_ids: 1-D Tensor of centroid indices.
+
+  Returns:
+    y_fixed: 1-D tensor of cluster assignment.
+  """
+  predictions = math_ops.argmin(
+      array_ops.gather(pairwise_distances, centroid_ids), dimension=0)
+  batch_size = array_ops.shape(pairwise_distances)[0]
+
+  # Deal with numerical instability
+  mask = math_ops.reduce_any(array_ops.one_hot(
+      centroid_ids, batch_size, True, False, axis=-1, dtype=dtypes.bool),
+                             axis=0)
+  constraint_one_hot = math_ops.multiply(
+      array_ops.one_hot(centroid_ids,
+                        batch_size,
+                        array_ops.constant(1, dtype=dtypes.int64),
+                        array_ops.constant(0, dtype=dtypes.int64),
+                        axis=0,
+                        dtype=dtypes.int64),
+      math_ops.to_int64(math_ops.range(array_ops.shape(centroid_ids)[0])))
+  constraint_vect = math_ops.reduce_sum(
+      array_ops.transpose(constraint_one_hot), axis=0)
+
+  y_fixed = array_ops.where(mask, constraint_vect, predictions)
+  return y_fixed
+
+
+def compute_facility_energy(pairwise_distances, centroid_ids):
+  """Compute the average travel distance to the assigned centroid.
+
+  Args:
+    pairwise_distances: 2-D Tensor of pairwise distances.
+    centroid_ids: 1-D Tensor of indices.
+
+  Returns:
+    facility_energy: dtypes.float32 scalar.
+  """
+  return -1.0 * math_ops.reduce_sum(
+      math_ops.reduce_min(
+          array_ops.gather(pairwise_distances, centroid_ids), axis=0))
+
+
+def compute_clustering_score(labels, predictions, margin_type):
+  """Computes the clustering score via sklearn.metrics functions.
+
+  There are various ways to compute the clustering score. Intuitively,
+  we want to measure the agreement of two clustering assignments (labels vs
+  predictions) ignoring the permutations and output a score from zero to one.
+  (where the values close to one indicate significant agreement).
+  This code supports following scoring functions:
+    nmi: normalized mutual information
+    ami: adjusted mutual information
+    ari: adjusted random index
+    vmeasure: v-measure
+    const: indicator checking whether the two clusterings are the same.
+  See http://scikit-learn.org/stable/modules/classes.html#clustering-metrics
+    for the detailed descriptions.
+  Args:
+    labels: 1-D Tensor. ground truth cluster assignment.
+    predictions: 1-D Tensor. predicted cluster assignment.
+    margin_type: Type of structured margin to use. Default is nmi.
+  Returns:
+    clustering_score: dtypes.float32 scalar.
+      The possible valid values are from zero to one.
+      Zero means the worst clustering and one means the perfect clustering.
+  Raises:
+    ValueError: margin_type is not recognized.
+  """
+  margin_type_to_func = {
+      'nmi': _compute_nmi_score,
+      'ami': _compute_ami_score,
+      'ari': _compute_ari_score,
+      'vmeasure': _compute_vmeasure_score,
+      'const': _compute_zeroone_score
+  }
+
+  if margin_type not in margin_type_to_func:
+    raise ValueError('Unrecognized margin_type: %s' % margin_type)
+  clustering_score_fn = margin_type_to_func[margin_type]
+  return array_ops.squeeze(clustering_score_fn(labels, predictions))
+
+
+def _compute_nmi_score(labels, predictions):
+  return math_ops.to_float(
+      script_ops.py_func(
+          metrics.normalized_mutual_info_score, [labels, predictions],
+          [dtypes.float64],
+          name='nmi'))
+
+
+def _compute_ami_score(labels, predictions):
+  ami_score = math_ops.to_float(
+      script_ops.py_func(
+          metrics.adjusted_mutual_info_score, [labels, predictions],
+          [dtypes.float64],
+          name='ami'))
+  return math_ops.maximum(0.0, ami_score)
+
+
+def _compute_ari_score(labels, predictions):
+  ari_score = math_ops.to_float(
+      script_ops.py_func(
+          metrics.adjusted_rand_score, [labels, predictions], [dtypes.float64],
+          name='ari'))
+  # ari score can go below 0
+  # http://scikit-learn.org/stable/modules/clustering.html#adjusted-rand-score
+  return math_ops.maximum(0.0, ari_score)
+
+
+def _compute_vmeasure_score(labels, predictions):
+  vmeasure_score = math_ops.to_float(
+      script_ops.py_func(
+          metrics.v_measure_score, [labels, predictions], [dtypes.float64],
+          name='vmeasure'))
+  return math_ops.maximum(0.0, vmeasure_score)
+
+
+def _compute_zeroone_score(labels, predictions):
+  zeroone_score = math_ops.to_float(
+      math_ops.equal(
+          math_ops.reduce_sum(
+              math_ops.to_int32(math_ops.equal(labels, predictions))),
+          array_ops.shape(labels)[0]))
+  return zeroone_score
+
+
+def _find_loss_augmented_facility_idx(pairwise_distances, labels, chosen_ids,
+                                      candidate_ids, margin_multiplier,
+                                      margin_type):
+  """Find the next centroid that maximizes the loss augmented inference.
+
+  This function is a subroutine called from compute_augmented_facility_locations
+
+  Args:
+    pairwise_distances: 2-D Tensor of pairwise distances.
+    labels: 1-D Tensor of ground truth cluster assignment.
+    chosen_ids: 1-D Tensor of current centroid indices.
+    candidate_ids: 1-D Tensor of candidate indices.
+    margin_multiplier: multiplication constant.
+    margin_type: Type of structured margin to use. Default is nmi.
+
+  Returns:
+    integer index.
+  """
+  num_candidates = array_ops.shape(candidate_ids)[0]
+
+  pairwise_distances_chosen = array_ops.gather(pairwise_distances, chosen_ids)
+  pairwise_distances_candidate = array_ops.gather(
+      pairwise_distances, candidate_ids)
+  pairwise_distances_chosen_tile = array_ops.tile(
+      pairwise_distances_chosen, [1, num_candidates])
+
+  candidate_scores = -1.0 * math_ops.reduce_sum(
+      array_ops.reshape(
+          math_ops.reduce_min(
+              array_ops.concat([
+                  pairwise_distances_chosen_tile,
+                  array_ops.reshape(pairwise_distances_candidate, [1, -1])
+              ], 0),
+              axis=0,
+              keep_dims=True), [num_candidates, -1]),
+      axis=1)
+
+  nmi_scores = array_ops.zeros([num_candidates])
+  iteration = array_ops.constant(0)
+
+  def func_cond(iteration, nmi_scores):
+    del nmi_scores  # Unused in func_cond()
+    return iteration < num_candidates
+
+  def func_body(iteration, nmi_scores):
+    predictions = get_cluster_assignment(
+        pairwise_distances,
+        array_ops.concat([chosen_ids, [candidate_ids[iteration]]], 0))
+    nmi_score_i = compute_clustering_score(labels, predictions, margin_type)
+    pad_before = array_ops.zeros([iteration])
+    pad_after = array_ops.zeros([num_candidates - 1 - iteration])
+    # return 1 - NMI score as the structured loss.
+    #   because NMI is higher the better [0,1].
+    return iteration + 1, nmi_scores + array_ops.concat(
+        [pad_before, [1.0 - nmi_score_i], pad_after], 0)
+
+  _, nmi_scores = control_flow_ops.while_loop(
+      func_cond, func_body, [iteration, nmi_scores])
+
+  candidate_scores = math_ops.add(
+      candidate_scores, margin_multiplier * nmi_scores)
+
+  argmax_index = math_ops.to_int32(
+      math_ops.argmax(candidate_scores, dimension=0))
+
+  return candidate_ids[argmax_index]
+
+
+def compute_augmented_facility_locations(pairwise_distances, labels, all_ids,
+                                         margin_multiplier, margin_type):
+  """Computes the centroid locations.
+
+  Args:
+    pairwise_distances: 2-D Tensor of pairwise distances.
+    labels: 1-D Tensor of ground truth cluster assignment.
+    all_ids: 1-D Tensor of all data indices.
+    margin_multiplier: multiplication constant.
+    margin_type: Type of structured margin to use. Default is nmi.
+
+  Returns:
+    chosen_ids: 1-D Tensor of chosen centroid indices.
+  """
+
+  def func_cond_augmented(iteration, chosen_ids):
+    del chosen_ids  # Unused argument in func_cond_augmented.
+    return iteration < num_classes
+
+  def func_body_augmented(iteration, chosen_ids):
+    # find a new facility location to add
+    #  based on the clustering score and the NMI score
+    candidate_ids = array_ops.setdiff1d(all_ids, chosen_ids)[0]
+    new_chosen_idx = _find_loss_augmented_facility_idx(pairwise_distances,
+                                                       labels, chosen_ids,
+                                                       candidate_ids,
+                                                       margin_multiplier,
+                                                       margin_type)
+    chosen_ids = array_ops.concat([chosen_ids, [new_chosen_idx]], 0)
+    return iteration + 1, chosen_ids
+
+  num_classes = array_ops.size(array_ops.unique(labels)[0])
+  chosen_ids = array_ops.constant(0, dtype=dtypes.int32, shape=[0])
+
+  # num_classes get determined at run time based on the sampled batch.
+  iteration = array_ops.constant(0)
+
+  _, chosen_ids = control_flow_ops.while_loop(
+      func_cond_augmented,
+      func_body_augmented, [iteration, chosen_ids],
+      shape_invariants=[iteration.get_shape(), tensor_shape.TensorShape(
+          [None])])
+  return chosen_ids
+
+
+def update_medoid_per_cluster(pairwise_distances, pairwise_distances_subset,
+                              labels, chosen_ids, cluster_member_ids,
+                              cluster_idx, margin_multiplier, margin_type):
+  """Updates the cluster medoid per cluster.
+
+  Args:
+    pairwise_distances: 2-D Tensor of pairwise distances.
+    pairwise_distances_subset: 2-D Tensor of pairwise distances for one cluster.
+    labels: 1-D Tensor of ground truth cluster assignment.
+    chosen_ids: 1-D Tensor of cluster centroid indices.
+    cluster_member_ids: 1-D Tensor of cluster member indices for one cluster.
+    cluster_idx: Index of this one cluster.
+    margin_multiplier: multiplication constant.
+    margin_type: Type of structured margin to use. Default is nmi.
+
+  Returns:
+    chosen_ids: Updated 1-D Tensor of cluster centroid indices.
+  """
+
+  def func_cond(iteration, scores_margin):
+    del scores_margin  # Unused variable scores_margin.
+    return iteration < num_candidates
+
+  def func_body(iteration, scores_margin):
+    # swap the current medoid with the candidate cluster member
+    candidate_medoid = math_ops.to_int32(cluster_member_ids[iteration])
+    tmp_chosen_ids = update_1d_tensor(chosen_ids, cluster_idx, candidate_medoid)
+    predictions = get_cluster_assignment(pairwise_distances, tmp_chosen_ids)
+    metric_score = compute_clustering_score(labels, predictions, margin_type)
+    pad_before = array_ops.zeros([iteration])
+    pad_after = array_ops.zeros([num_candidates - 1 - iteration])
+    return iteration + 1, scores_margin + array_ops.concat(
+        [pad_before, [1.0 - metric_score], pad_after], 0)
+
+  # pairwise_distances_subset is of size [p, 1, 1, p],
+  #   the intermediate dummy dimensions at
+  #   [1, 2] makes this code work in the edge case where p=1.
+  #   this happens if the cluster size is one.
+  scores_fac = -1.0 * math_ops.reduce_sum(
+      array_ops.squeeze(pairwise_distances_subset, [1, 2]), axis=0)
+
+  iteration = array_ops.constant(0)
+  num_candidates = array_ops.size(cluster_member_ids)
+  scores_margin = array_ops.zeros([num_candidates])
+
+  _, scores_margin = control_flow_ops.while_loop(func_cond, func_body,
+                                                 [iteration, scores_margin])
+  candidate_scores = math_ops.add(scores_fac, margin_multiplier * scores_margin)
+
+  argmax_index = math_ops.to_int32(
+      math_ops.argmax(candidate_scores, dimension=0))
+
+  best_medoid = math_ops.to_int32(cluster_member_ids[argmax_index])
+  chosen_ids = update_1d_tensor(chosen_ids, cluster_idx, best_medoid)
+  return chosen_ids
+
+
+def update_all_medoids(pairwise_distances, predictions, labels, chosen_ids,
+                       margin_multiplier, margin_type):
+  """Updates all cluster medoids a cluster at a time.
+
+  Args:
+    pairwise_distances: 2-D Tensor of pairwise distances.
+    predictions: 1-D Tensor of predicted cluster assignment.
+    labels: 1-D Tensor of ground truth cluster assignment.
+    chosen_ids: 1-D Tensor of cluster centroid indices.
+    margin_multiplier: multiplication constant.
+    margin_type: Type of structured margin to use. Default is nmi.
+
+  Returns:
+    chosen_ids: Updated 1-D Tensor of cluster centroid indices.
+  """
+
+  def func_cond_augmented_pam(iteration, chosen_ids):
+    del chosen_ids  # Unused argument.
+    return iteration < num_classes
+
+  def func_body_augmented_pam(iteration, chosen_ids):
+    """Call the update_medoid_per_cluster subroutine."""
+    mask = math_ops.equal(
+        math_ops.to_int64(predictions), math_ops.to_int64(iteration))
+    this_cluster_ids = array_ops.where(mask)
+
+    pairwise_distances_subset = array_ops.transpose(
+        array_ops.gather(
+            array_ops.transpose(
+                array_ops.gather(pairwise_distances, this_cluster_ids)),
+            this_cluster_ids))
+
+    chosen_ids = update_medoid_per_cluster(pairwise_distances,
+                                           pairwise_distances_subset, labels,
+                                           chosen_ids, this_cluster_ids,
+                                           iteration, margin_multiplier,
+                                           margin_type)
+    return iteration + 1, chosen_ids
+
+  unique_class_ids = array_ops.unique(labels)[0]
+  num_classes = array_ops.size(unique_class_ids)
+  iteration = array_ops.constant(0)
+
+  _, chosen_ids = control_flow_ops.while_loop(
+      func_cond_augmented_pam, func_body_augmented_pam, [iteration, chosen_ids])
+  return chosen_ids
+
+
+def compute_augmented_facility_locations_pam(pairwise_distances,
+                                             labels,
+                                             margin_multiplier,
+                                             margin_type,
+                                             chosen_ids,
+                                             pam_max_iter=5):
+  """Refine the cluster centroids with PAM local search.
+
+  For fixed iterations, alternate between updating the cluster assignment
+    and updating cluster medoids.
+
+  Args:
+    pairwise_distances: 2-D Tensor of pairwise distances.
+    labels: 1-D Tensor of ground truth cluster assignment.
+    margin_multiplier: multiplication constant.
+    margin_type: Type of structured margin to use. Default is nmi.
+    chosen_ids: 1-D Tensor of initial estimate of cluster centroids.
+    pam_max_iter: Number of refinement iterations.
+
+  Returns:
+    chosen_ids: Updated 1-D Tensor of cluster centroid indices.
+  """
+  for _ in range(pam_max_iter):
+    # update the cluster assignment given the chosen_ids (S_pred)
+    predictions = get_cluster_assignment(pairwise_distances, chosen_ids)
+
+    # update the medoids per each cluster
+    chosen_ids = update_all_medoids(pairwise_distances, predictions, labels,
+                                    chosen_ids, margin_multiplier, margin_type)
+
+  return chosen_ids
+
+
+def compute_gt_cluster_score(pairwise_distances, labels):
+  """Compute ground truth facility location score.
+
+  Loop over each unique classes and compute average travel distances.
+
+  Args:
+    pairwise_distances: 2-D Tensor of pairwise distances.
+    labels: 1-D Tensor of ground truth cluster assignment.
+
+  Returns:
+    gt_cluster_score: dtypes.float32 score.
+  """
+  unique_class_ids = array_ops.unique(labels)[0]
+  num_classes = array_ops.size(unique_class_ids)
+  iteration = array_ops.constant(0)
+  gt_cluster_score = array_ops.constant(0.0, dtype=dtypes.float32)
+
+  def func_cond(iteration, gt_cluster_score):
+    del gt_cluster_score  # Unused argument.
+    return iteration < num_classes
+
+  def func_body(iteration, gt_cluster_score):
+    """Per each cluster, compute the average travel distance."""
+    mask = math_ops.equal(labels, unique_class_ids[iteration])
+    this_cluster_ids = array_ops.where(mask)
+    pairwise_distances_subset = array_ops.transpose(
+        array_ops.gather(
+            array_ops.transpose(
+                array_ops.gather(pairwise_distances, this_cluster_ids)),
+            this_cluster_ids))
+    this_cluster_score = -1.0 * math_ops.reduce_min(
+        math_ops.reduce_sum(
+            pairwise_distances_subset, axis=0))
+    return iteration + 1, gt_cluster_score + this_cluster_score
+
+  _, gt_cluster_score = control_flow_ops.while_loop(
+      func_cond, func_body, [iteration, gt_cluster_score])
+  return gt_cluster_score
+
+
+def cluster_loss(labels,
+                 embeddings,
+                 margin_multiplier,
+                 enable_pam_finetuning=True,
+                 margin_type='nmi',
+                 print_losses=False):
+  """Computes the clustering loss.
+
+  The following structured margins are supported:
+    nmi: normalized mutual information
+    ami: adjusted mutual information
+    ari: adjusted random index
+    vmeasure: v-measure
+    const: indicator checking whether the two clusterings are the same.
+
+  Args:
+    labels: 2-D Tensor of labels of shape [batch size, 1]
+    embeddings: 2-D Tensor of embeddings of shape
+      [batch size, embedding dimension]. Embeddings should be l2 normalized.
+    margin_multiplier: float32 scalar. multiplier on the structured margin term
+      See section 3.2 of paper for discussion.
+    enable_pam_finetuning: Boolean, Whether to run local pam refinement.
+      See section 3.4 of paper for discussion.
+    margin_type: Type of structured margin to use. See section 3.2 of
+      paper for discussion. Can be 'nmi', 'ami', 'ari', 'vmeasure', 'const'.
+    print_losses: Boolean. Option to print the loss.
+
+  Paper: https://arxiv.org/abs/1612.01213.
+
+  Returns:
+    clustering_loss: A float32 scalar `Tensor`.
+  Raises:
+    ImportError: If sklearn dependency is not installed.
+  """
+  if not HAS_SKLEARN:
+    raise ImportError('Cluster loss depends on sklearn.')
+  pairwise_distances = pairwise_distance(embeddings)
+  labels = array_ops.squeeze(labels)
+  all_ids = math_ops.range(array_ops.shape(embeddings)[0])
+
+  # Compute the loss augmented inference and get the cluster centroids.
+  chosen_ids = compute_augmented_facility_locations(pairwise_distances, labels,
+                                                    all_ids, margin_multiplier,
+                                                    margin_type)
+  # Given the predicted centroids, compute the clustering score.
+  score_pred = compute_facility_energy(pairwise_distances, chosen_ids)
+
+  # Branch whether to use PAM finetuning.
+  if enable_pam_finetuning:
+    # Initialize with augmented facility solution.
+    chosen_ids = compute_augmented_facility_locations_pam(pairwise_distances,
+                                                          labels,
+                                                          margin_multiplier,
+                                                          margin_type,
+                                                          chosen_ids)
+    score_pred = compute_facility_energy(pairwise_distances, chosen_ids)
+
+  # Given the predicted centroids, compute the cluster assignments.
+  predictions = get_cluster_assignment(pairwise_distances, chosen_ids)
+
+  # Compute the clustering (i.e. NMI) score between the two assignments.
+  clustering_score_pred = compute_clustering_score(labels, predictions,
+                                                   margin_type)
+
+  # Compute the clustering score from labels.
+  score_gt = compute_gt_cluster_score(pairwise_distances, labels)
+
+  # Compute the hinge loss.
+  clustering_loss = math_ops.maximum(
+      score_pred + margin_multiplier * (1.0 - clustering_score_pred) - score_gt,
+      0.0,
+      name='clustering_loss')
+  clustering_loss.set_shape([])
+
+  if print_losses:
+    clustering_loss = logging_ops.Print(
+        clustering_loss,
+        ['clustering_loss: ', clustering_loss, array_ops.shape(
+            clustering_loss)])
+
+  # Clustering specific summary.
+  summary.scalar('losses/score_pred', score_pred)
+  summary.scalar('losses/' + margin_type, clustering_score_pred)
+  summary.scalar('losses/score_gt', score_gt)
+
+  return clustering_loss
diff --git a/tensorflow/contrib/losses/python/metric_learning/metric_loss_ops_test.py b/tensorflow/contrib/losses/python/metric_learning/metric_loss_ops_test.py
new file mode 100644
index 0000000000..4ec539ab42
--- /dev/null
+++ b/tensorflow/contrib/losses/python/metric_learning/metric_loss_ops_test.py
@@ -0,0 +1,562 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for triplet_semihard_loss."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+from tensorflow.contrib.losses.python import metric_learning as metric_loss_ops
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import sparse_tensor
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import nn
+from tensorflow.python.platform import test
+try:
+  # pylint: disable=g-import-not-at-top
+  from sklearn import datasets
+  from sklearn import metrics
+  HAS_SKLEARN = True
+except ImportError:
+  HAS_SKLEARN = False
+
+
+def pairwise_distance_np(feature, squared=False):
+  """Computes the pairwise distance matrix in numpy.
+
+  Args:
+    feature: 2-D numpy array of size [number of data, feature dimension]
+    squared: Boolean. If true, output is the pairwise squared euclidean
+      distance matrix; else, output is the pairwise euclidean distance matrix.
+
+  Returns:
+    pairwise_distances: 2-D numpy array of size
+      [number of data, number of data].
+  """
+  triu = np.triu_indices(feature.shape[0], 1)
+  upper_tri_pdists = np.linalg.norm(feature[triu[1]] - feature[triu[0]], axis=1)
+  if squared:
+    upper_tri_pdists **= 2.
+  num_data = feature.shape[0]
+  pairwise_distances = np.zeros((num_data, num_data))
+  pairwise_distances[np.triu_indices(num_data, 1)] = upper_tri_pdists
+  # Make symmetrical.
+  pairwise_distances = pairwise_distances + pairwise_distances.T - np.diag(
+      pairwise_distances.diagonal())
+  return pairwise_distances
+
+
+class ContrastiveLossTest(test.TestCase):
+
+  def testContrastive(self):
+    with self.test_session():
+      num_data = 10
+      feat_dim = 6
+      margin = 1.0
+
+      embeddings_anchor = np.random.rand(num_data, feat_dim).astype(np.float32)
+      embeddings_positive = np.random.rand(num_data, feat_dim).astype(
+          np.float32)
+      labels = np.random.randint(0, 2, size=(num_data,)).astype(np.float32)
+
+      # Compute the loss in NP
+      dist = np.sqrt(
+          np.sum(np.square(embeddings_anchor - embeddings_positive), axis=1))
+      loss_np = np.mean(
+          labels * np.square(dist) +
+          (1.0 - labels) * np.square(np.maximum(margin - dist, 0.0)))
+      # Compute the loss with TF
+      loss_tf = metric_loss_ops.contrastive_loss(
+          labels=ops.convert_to_tensor(labels),
+          embeddings_anchor=ops.convert_to_tensor(embeddings_anchor),
+          embeddings_positive=ops.convert_to_tensor(embeddings_positive),
+          margin=margin)
+      loss_tf = loss_tf.eval()
+      self.assertAllClose(loss_np, loss_tf)
+
+
+class TripletSemiHardLossTest(test.TestCase):
+
+  def testTripletSemiHard(self):
+    with self.test_session():
+      num_data = 10
+      feat_dim = 6
+      margin = 1.0
+      num_classes = 4
+
+      embedding = np.random.rand(num_data, feat_dim).astype(np.float32)
+      labels = np.random.randint(
+          0, num_classes, size=(num_data)).astype(np.float32)
+
+      # Reshape labels to compute adjacency matrix.
+      labels_reshaped = np.reshape(labels, (labels.shape[0], 1))
+      # Compute the loss in NP.
+      adjacency = np.equal(labels_reshaped, labels_reshaped.T)
+
+      pdist_matrix = pairwise_distance_np(embedding, squared=True)
+      loss_np = 0.0
+      num_positives = 0.0
+      for i in range(num_data):
+        for j in range(num_data):
+          if adjacency[i][j] > 0.0 and i != j:
+            num_positives += 1.0
+
+            pos_distance = pdist_matrix[i][j]
+            neg_distances = []
+
+            for k in range(num_data):
+              if adjacency[i][k] == 0:
+                neg_distances.append(pdist_matrix[i][k])
+
+            # Sort by distance.
+            neg_distances.sort()
+            chosen_neg_distance = neg_distances[0]
+
+            for l in range(len(neg_distances)):
+              chosen_neg_distance = neg_distances[l]
+              if chosen_neg_distance > pos_distance:
+                break
+
+            loss_np += np.maximum(
+                0.0, margin - chosen_neg_distance + pos_distance)
+
+      loss_np /= num_positives
+
+      # Compute the loss in TF.
+      loss_tf = metric_loss_ops.triplet_semihard_loss(
+          labels=ops.convert_to_tensor(labels),
+          embeddings=ops.convert_to_tensor(embedding),
+          margin=margin)
+      loss_tf = loss_tf.eval()
+      self.assertAllClose(loss_np, loss_tf)
+
+
+class LiftedStructLossTest(test.TestCase):
+
+  def testLiftedStruct(self):
+    with self.test_session():
+      num_data = 10
+      feat_dim = 6
+      margin = 1.0
+      num_classes = 4
+
+      embedding = np.random.rand(num_data, feat_dim).astype(np.float32)
+      labels = np.random.randint(
+          0, num_classes, size=(num_data)).astype(np.float32)
+      # Reshape labels to compute adjacency matrix.
+      labels_reshaped = np.reshape(labels, (labels.shape[0], 1))
+
+      # Compute the loss in NP
+      adjacency = np.equal(labels_reshaped, labels_reshaped.T)
+      pdist_matrix = pairwise_distance_np(embedding)
+      loss_np = 0.0
+      num_constraints = 0.0
+      for i in range(num_data):
+        for j in range(num_data):
+          if adjacency[i][j] > 0.0 and i != j:
+            d_pos = pdist_matrix[i][j]
+            negs = []
+            for k in range(num_data):
+              if not adjacency[i][k]:
+                negs.append(margin - pdist_matrix[i][k])
+            for l in range(num_data):
+              if not adjacency[j][l]:
+                negs.append(margin - pdist_matrix[j][l])
+
+            negs = np.array(negs)
+            max_elem = np.max(negs)
+            negs -= max_elem
+            negs = np.exp(negs)
+            soft_maximum = np.log(np.sum(negs)) + max_elem
+
+            num_constraints += 1.0
+            this_loss = max(soft_maximum + d_pos, 0)
+            loss_np += this_loss * this_loss
+
+      loss_np = loss_np / num_constraints / 2.0
+
+      # Compute the loss in TF
+      loss_tf = metric_loss_ops.lifted_struct_loss(
+          labels=ops.convert_to_tensor(labels),
+          embeddings=ops.convert_to_tensor(embedding),
+          margin=margin)
+      loss_tf = loss_tf.eval()
+      self.assertAllClose(loss_np, loss_tf)
+
+
+def convert_to_list_of_sparse_tensor(np_matrix):
+  list_of_sparse_tensors = []
+  nrows, ncols = np_matrix.shape
+  for i in range(nrows):
+    sp_indices = []
+    for j in range(ncols):
+      if np_matrix[i][j] == 1:
+        sp_indices.append([j])
+
+    num_non_zeros = len(sp_indices)
+    list_of_sparse_tensors.append(sparse_tensor.SparseTensor(
+        indices=np.array(sp_indices),
+        values=np.ones((num_non_zeros,)),
+        dense_shape=np.array([ncols,])))
+
+  return list_of_sparse_tensors
+
+
+class NpairsLossTest(test.TestCase):
+
+  def testNpairs(self):
+    with self.test_session():
+      num_data = 15
+      feat_dim = 6
+      num_classes = 5
+      reg_lambda = 0.02
+
+      embeddings_anchor = np.random.rand(num_data, feat_dim).astype(np.float32)
+      embeddings_positive = np.random.rand(num_data, feat_dim).astype(
+          np.float32)
+
+      labels = np.random.randint(
+          0, num_classes, size=(num_data)).astype(np.float32)
+      # Reshape labels to compute adjacency matrix.
+      labels_reshaped = np.reshape(labels, (labels.shape[0], 1))
+
+      # Compute the loss in NP
+      reg_term = np.mean(np.sum(np.square(embeddings_anchor), 1))
+      reg_term += np.mean(np.sum(np.square(embeddings_positive), 1))
+      reg_term *= 0.25 * reg_lambda
+
+      similarity_matrix = np.matmul(embeddings_anchor, embeddings_positive.T)
+
+      labels_remapped = np.equal(
+          labels_reshaped, labels_reshaped.T).astype(np.float32)
+      labels_remapped /= np.sum(labels_remapped, axis=1, keepdims=True)
+
+      xent_loss = math_ops.reduce_mean(nn.softmax_cross_entropy_with_logits(
+          logits=ops.convert_to_tensor(similarity_matrix),
+          labels=ops.convert_to_tensor(labels_remapped))).eval()
+      loss_np = xent_loss + reg_term
+
+      # Compute the loss in TF
+      loss_tf = metric_loss_ops.npairs_loss(
+          labels=ops.convert_to_tensor(labels),
+          embeddings_anchor=ops.convert_to_tensor(embeddings_anchor),
+          embeddings_positive=ops.convert_to_tensor(embeddings_positive),
+          reg_lambda=reg_lambda)
+      loss_tf = loss_tf.eval()
+      self.assertAllClose(loss_np, loss_tf)
+
+
+class NpairsLossMultiLabelTest(test.TestCase):
+
+  def testNpairsMultiLabelLossWithSingleLabelEqualsNpairsLoss(self):
+    with self.test_session():
+      num_data = 15
+      feat_dim = 6
+      reg_lambda = 0.02
+
+      embeddings_anchor = np.random.rand(num_data, feat_dim).astype(np.float32)
+      embeddings_positive = np.random.rand(num_data, feat_dim).astype(
+          np.float32)
+      labels = np.arange(num_data)
+      labels = np.reshape(labels, -1)
+
+      # Compute vanila npairs loss.
+      loss_npairs = metric_loss_ops.npairs_loss(
+          labels=ops.convert_to_tensor(labels),
+          embeddings_anchor=ops.convert_to_tensor(embeddings_anchor),
+          embeddings_positive=ops.convert_to_tensor(embeddings_positive),
+          reg_lambda=reg_lambda).eval()
+
+      # Compute npairs multilabel loss.
+      labels_one_hot = np.identity(num_data)
+      loss_npairs_multilabel = metric_loss_ops.npairs_loss_multilabel(
+          sparse_labels=convert_to_list_of_sparse_tensor(labels_one_hot),
+          embeddings_anchor=ops.convert_to_tensor(embeddings_anchor),
+          embeddings_positive=ops.convert_to_tensor(embeddings_positive),
+          reg_lambda=reg_lambda).eval()
+
+      self.assertAllClose(loss_npairs, loss_npairs_multilabel)
+
+  def testNpairsMultiLabel(self):
+    with self.test_session():
+      num_data = 15
+      feat_dim = 6
+      num_classes = 10
+      reg_lambda = 0.02
+
+      embeddings_anchor = np.random.rand(num_data, feat_dim).astype(np.float32)
+      embeddings_positive = np.random.rand(num_data, feat_dim).astype(
+          np.float32)
+
+      labels = np.random.randint(0, 2, (num_data, num_classes))
+      # set entire column to one so that each row has at least one bit set.
+      labels[:, -1] = 1
+
+      # Compute the loss in NP
+      reg_term = np.mean(np.sum(np.square(embeddings_anchor), 1))
+      reg_term += np.mean(np.sum(np.square(embeddings_positive), 1))
+      reg_term *= 0.25 * reg_lambda
+
+      similarity_matrix = np.matmul(embeddings_anchor, embeddings_positive.T)
+
+      labels_remapped = np.dot(labels, labels.T).astype(np.float)
+      labels_remapped /= np.sum(labels_remapped, 1, keepdims=True)
+
+      xent_loss = math_ops.reduce_mean(nn.softmax_cross_entropy_with_logits(
+          logits=ops.convert_to_tensor(similarity_matrix),
+          labels=ops.convert_to_tensor(labels_remapped))).eval()
+      loss_np = xent_loss + reg_term
+
+      # Compute the loss in TF
+      loss_tf = metric_loss_ops.npairs_loss_multilabel(
+          sparse_labels=convert_to_list_of_sparse_tensor(labels),
+          embeddings_anchor=ops.convert_to_tensor(embeddings_anchor),
+          embeddings_positive=ops.convert_to_tensor(embeddings_positive),
+          reg_lambda=reg_lambda)
+      loss_tf = loss_tf.eval()
+
+      self.assertAllClose(loss_np, loss_tf)
+
+
+def compute_ground_truth_cluster_score(feat, y):
+  y_unique = np.unique(y)
+  score_gt_np = 0.0
+  for c in y_unique:
+    feat_subset = feat[y == c, :]
+    pdist_subset = pairwise_distance_np(feat_subset)
+    score_gt_np += -1.0 * np.min(np.sum(pdist_subset, axis=0))
+  score_gt_np = score_gt_np.astype(np.float32)
+  return score_gt_np
+
+
+def compute_cluster_loss_numpy(feat,
+                               y,
+                               margin_multiplier=1.0,
+                               enable_pam_finetuning=True):
+  if enable_pam_finetuning:
+    facility = ForwardGreedyFacility(
+        n_clusters=np.unique(y).size).pam_augmented_fit(feat, y,
+                                                        margin_multiplier)
+  else:
+    facility = ForwardGreedyFacility(
+        n_clusters=np.unique(y).size).loss_augmented_fit(feat, y,
+                                                         margin_multiplier)
+
+  score_augmented = facility.score_aug_
+  score_gt = compute_ground_truth_cluster_score(feat, y)
+  return np.maximum(np.float32(0.0), score_augmented - score_gt)
+
+
+class ForwardGreedyFacility(object):
+
+  def __init__(self, n_clusters=8):
+    self.n_clusters = n_clusters
+    self.center_ics_ = None
+
+  def _check_init_args(self):
+    # Check n_clusters.
+    if (self.n_clusters is None or self.n_clusters <= 0 or
+        not isinstance(self.n_clusters, int)):
+      raise ValueError('n_clusters has to be nonnegative integer.')
+
+  def loss_augmented_fit(self, feat, y, loss_mult):
+    """Fit K-Medoids to the provided data."""
+    self._check_init_args()
+    # Check that the array is good and attempt to convert it to
+    # Numpy array if possible.
+    feat = self._check_array(feat)
+    # Apply distance metric to get the distance matrix.
+    pdists = pairwise_distance_np(feat)
+
+    num_data = feat.shape[0]
+    candidate_ids = list(range(num_data))
+    candidate_scores = np.zeros(num_data,)
+    subset = []
+
+    k = 0
+    while k < self.n_clusters:
+      candidate_scores = []
+      for i in candidate_ids:
+        # push i to subset.
+        subset.append(i)
+        marginal_cost = -1.0 * np.sum(np.min(pdists[:, subset], axis=1))
+        loss = 1.0 - metrics.normalized_mutual_info_score(
+            y, self._get_cluster_ics(pdists, subset))
+        candidate_scores.append(marginal_cost + loss_mult * loss)
+        # remove i from subset.
+        subset.pop()
+
+      # push i_star to subset.
+      i_star = candidate_ids[np.argmax(candidate_scores)]
+      subset.append(i_star)
+      # remove i_star from candidate indices.
+      candidate_ids.remove(i_star)
+      k += 1
+
+    # Expose labels_ which are the assignments of
+    # the training data to clusters.
+    self.labels_ = self._get_cluster_ics(pdists, subset)
+    # Expose cluster centers, i.e. medoids.
+    self.cluster_centers_ = feat.take(subset, axis=0)
+    # Expose indices of chosen cluster centers.
+    self.center_ics_ = subset
+    # Expose the score = -\sum_{i \in V} min_{j \in S} || x_i - x_j ||
+    self.score_ = np.float32(-1.0) * self._get_facility_distance(pdists, subset)
+    self.score_aug_ = self.score_ + loss_mult * (
+        1.0 - metrics.normalized_mutual_info_score(
+            y, self._get_cluster_ics(pdists, subset)))
+    self.score_aug_ = self.score_aug_.astype(np.float32)
+    # Expose the chosen cluster indices.
+    self.subset_ = subset
+    return self
+
+  def _augmented_update_medoid_ics_in_place(self, pdists, y_gt, cluster_ics,
+                                            medoid_ics, loss_mult):
+    for cluster_idx in range(self.n_clusters):
+      # y_pred = self._get_cluster_ics(D, medoid_ics)
+      # Don't prematurely do the assignment step.
+      # Do this after we've updated all cluster medoids.
+      y_pred = cluster_ics
+
+      if sum(y_pred == cluster_idx) == 0:
+        # Cluster is empty.
+        continue
+
+      curr_score = (
+          -1.0 * np.sum(
+              pdists[medoid_ics[cluster_idx], y_pred == cluster_idx]) +
+          loss_mult * (1.0 - metrics.normalized_mutual_info_score(
+              y_gt, y_pred)))
+
+      pdist_in = pdists[y_pred == cluster_idx, :]
+      pdist_in = pdist_in[:, y_pred == cluster_idx]
+
+      all_scores_fac = np.sum(-1.0 * pdist_in, axis=1)
+      all_scores_loss = []
+      for i in range(y_pred.size):
+        if y_pred[i] != cluster_idx:
+          continue
+        # remove this cluster's current centroid
+        medoid_ics_i = medoid_ics[:cluster_idx] + medoid_ics[cluster_idx + 1:]
+        # add this new candidate to the centroid list
+        medoid_ics_i += [i]
+        y_pred_i = self._get_cluster_ics(pdists, medoid_ics_i)
+        all_scores_loss.append(loss_mult * (
+            1.0 - metrics.normalized_mutual_info_score(y_gt, y_pred_i)))
+
+      all_scores = all_scores_fac + all_scores_loss
+      max_score_idx = np.argmax(all_scores)
+      max_score = all_scores[max_score_idx]
+
+      if max_score > curr_score:
+        medoid_ics[cluster_idx] = np.where(
+            y_pred == cluster_idx)[0][max_score_idx]
+
+  def pam_augmented_fit(self, feat, y, loss_mult):
+    pam_max_iter = 5
+    self._check_init_args()
+    feat = self._check_array(feat)
+    pdists = pairwise_distance_np(feat)
+    self.loss_augmented_fit(feat, y, loss_mult)
+    print('PAM -1 (before PAM): score: %f, score_aug: %f' % (
+        self.score_, self.score_aug_))
+    # Initialize from loss augmented facility location
+    subset = self.center_ics_
+    for iter_ in range(pam_max_iter):
+      # update the cluster assignment
+      cluster_ics = self._get_cluster_ics(pdists, subset)
+      # update the medoid for each clusters
+      self._augmented_update_medoid_ics_in_place(pdists, y, cluster_ics, subset,
+                                                 loss_mult)
+      self.score_ = np.float32(-1.0) * self._get_facility_distance(
+          pdists, subset)
+      self.score_aug_ = self.score_ + loss_mult * (
+          1.0 - metrics.normalized_mutual_info_score(
+              y, self._get_cluster_ics(pdists, subset)))
+      self.score_aug_ = self.score_aug_.astype(np.float32)
+      print('PAM iter: %d, score: %f, score_aug: %f' % (iter_, self.score_,
+                                                        self.score_aug_))
+
+    self.center_ics_ = subset
+    self.labels_ = cluster_ics
+    return self
+
+  def _check_array(self, feat):
+    # Check that the number of clusters is less than or equal to
+    # the number of samples
+    if self.n_clusters > feat.shape[0]:
+      raise ValueError('The number of medoids ' + '({}) '.format(
+          self.n_clusters) + 'must be larger than the number ' +
+                       'of samples ({})'.format(feat.shape[0]))
+    return feat
+
+  def _get_cluster_ics(self, pdists, subset):
+    """Returns cluster indices for pdist and current medoid indices."""
+    # Assign data points to clusters based on
+    # which cluster assignment yields
+    # the smallest distance`
+    cluster_ics = np.argmin(pdists[subset, :], axis=0)
+    return cluster_ics
+
+  def _get_facility_distance(self, pdists, subset):
+    return np.sum(np.min(pdists[subset, :], axis=0))
+
+
+class ClusterLossTest(test.TestCase):
+
+  def _genClusters(self, n_samples, n_clusters):
+    blobs = datasets.make_blobs(
+        n_samples=n_samples, centers=n_clusters)
+    embedding, labels = blobs
+    embedding = (embedding - embedding.mean(axis=0)) / embedding.std(axis=0)
+    embedding = embedding.astype(np.float32)
+    return embedding, labels
+
+  def testClusteringLossPAMOff(self):
+    if not HAS_SKLEARN:
+      return
+    with self.test_session():
+      margin_multiplier = 10.0
+      embeddings, labels = self._genClusters(n_samples=128, n_clusters=64)
+
+      loss_np = compute_cluster_loss_numpy(
+          embeddings, labels, margin_multiplier, enable_pam_finetuning=False)
+      loss_tf = metric_loss_ops.cluster_loss(
+          labels=ops.convert_to_tensor(labels),
+          embeddings=ops.convert_to_tensor(embeddings),
+          margin_multiplier=margin_multiplier,
+          enable_pam_finetuning=False)
+      loss_tf = loss_tf.eval()
+      self.assertAllClose(loss_np, loss_tf)
+
+  def testClusteringLossPAMOn(self):
+    if not HAS_SKLEARN:
+      return
+    with self.test_session():
+      margin_multiplier = 10.0
+      embeddings, labels = self._genClusters(n_samples=128, n_clusters=64)
+
+      loss_np = compute_cluster_loss_numpy(
+          embeddings, labels, margin_multiplier, enable_pam_finetuning=True)
+      loss_tf = metric_loss_ops.cluster_loss(
+          labels=ops.convert_to_tensor(labels),
+          embeddings=ops.convert_to_tensor(embeddings),
+          margin_multiplier=margin_multiplier,
+          enable_pam_finetuning=True)
+      loss_tf = loss_tf.eval()
+      self.assertAllClose(loss_np, loss_tf)
+
+if __name__ == '__main__':
+  test.main()
-- 
GitLab


From 4efb8cf64bb2f0f0727430d38ca6e48a99e572f7 Mon Sep 17 00:00:00 2001
From: Benoit Steiner <bsteiner@google.com>
Date: Thu, 12 Oct 2017 13:01:17 -0700
Subject: [PATCH 311/909] Register a dummy estimation function for _Send ops.
 This ensures that we don't get negative cost estimates for _Send ops.

PiperOrigin-RevId: 171999586
---
 tensorflow/core/grappler/costs/op_level_cost_estimator.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tensorflow/core/grappler/costs/op_level_cost_estimator.cc b/tensorflow/core/grappler/costs/op_level_cost_estimator.cc
index 7a1295c91e..a2fa847df2 100644
--- a/tensorflow/core/grappler/costs/op_level_cost_estimator.cc
+++ b/tensorflow/core/grappler/costs/op_level_cost_estimator.cc
@@ -35,6 +35,7 @@ constexpr char kRefIdentity[] = "RefIdentity";
 constexpr char kNoOp[] = "NoOp";
 constexpr char kReshape[] = "Reshape";
 constexpr char kRecv[] = "_Recv";
+constexpr char kSend[] = "_Send";
 constexpr char kBatchMatMul[] = "BatchMatMul";
 constexpr char kVariable[] = "Variable";
 constexpr char kVariableV2[] = "VariableV2";
@@ -165,6 +166,7 @@ OpLevelCostEstimator::OpLevelCostEstimator() {
       {kNoOp, wrap(&OpLevelCostEstimator::PredictNoOp)},
       {kReshape, wrap(&OpLevelCostEstimator::PredictNoOp)},
       {kRecv, wrap(&OpLevelCostEstimator::PredictNoOp)},
+      {kSend, wrap(&OpLevelCostEstimator::PredictNoOp)},
       {kVariable, wrap(&OpLevelCostEstimator::PredictNoOp)},
       {kVariableV2, wrap(&OpLevelCostEstimator::PredictNoOp)},
       {kBatchMatMul, wrap(&OpLevelCostEstimator::PredictBatchMatMul)},
-- 
GitLab


From 79c821165896b149624a529a6f499ad25960c84d Mon Sep 17 00:00:00 2001
From: Jianwei Xie <xiejw@google.com>
Date: Thu, 12 Oct 2017 13:18:39 -0700
Subject: [PATCH 312/909] Adds explicit type checks for all steps/iterations in
 TPUEstimator.

PiperOrigin-RevId: 172001904
---
 tensorflow/contrib/tpu/BUILD                  |  1 +
 .../contrib/tpu/python/tpu/tpu_config.py      |  9 ++++++
 .../contrib/tpu/python/tpu/tpu_estimator.py   | 13 ++++++--
 tensorflow/contrib/tpu/python/tpu/util.py     | 31 +++++++++++++++++++
 4 files changed, 52 insertions(+), 2 deletions(-)
 create mode 100644 tensorflow/contrib/tpu/python/tpu/util.py

diff --git a/tensorflow/contrib/tpu/BUILD b/tensorflow/contrib/tpu/BUILD
index e753fe7a51..970fc97605 100644
--- a/tensorflow/contrib/tpu/BUILD
+++ b/tensorflow/contrib/tpu/BUILD
@@ -35,6 +35,7 @@ py_library(
     srcs = [
         "python/tpu/tpu_config.py",
         "python/tpu/tpu_estimator.py",
+        "python/tpu/util.py",
     ],
     srcs_version = "PY2AND3",
     deps = [
diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_config.py b/tensorflow/contrib/tpu/python/tpu/tpu_config.py
index ece91180af..b1d3952d1e 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_config.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_config.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 
 import collections
 
+from tensorflow.contrib.tpu.python.tpu import util as util_lib
 from tensorflow.python.estimator import run_config as run_config_lib
 
 
@@ -52,6 +53,14 @@ class TPUConfig(
               iterations_per_loop=2,
               num_shards=2,
               per_host_input_for_training=False):
+
+    # Check iterations_per_loop.
+    util_lib.check_positive_integer(iterations_per_loop,
+                                    'TPUConfig iterations_per_loop')
+
+    # Check num_shards.
+    util_lib.check_positive_integer(num_shards, 'TPUConfig num_shards')
+
     return super(TPUConfig, cls).__new__(
         cls,
         iterations_per_loop=iterations_per_loop,
diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
index b5001d596b..43f9defd54 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
@@ -31,6 +31,7 @@ from tensorflow.contrib.tpu.python.tpu import tpu_config
 from tensorflow.contrib.tpu.python.tpu import tpu_feed
 from tensorflow.contrib.tpu.python.tpu import tpu_function
 from tensorflow.contrib.tpu.python.tpu import training_loop
+from tensorflow.contrib.tpu.python.tpu import util as util_lib
 
 from tensorflow.core.protobuf import config_pb2
 
@@ -1318,6 +1319,12 @@ class TPUEstimator(estimator_lib.Estimator):
           'For TPU training, one of `steps` or `max_steps` must be set. '
           'Cannot be both `None`.')
 
+    # Estimator.train has explicit positiveness check.
+    if steps is not None:
+      util_lib.check_positive_integer(steps, 'Train steps')
+    if max_steps is not None:
+      util_lib.check_positive_integer(max_steps, 'Train max_steps')
+
     return [_TPUStopAtStepHook(self._iterations_per_training_loop,
                                steps, max_steps)]
 
@@ -1328,8 +1335,8 @@ class TPUEstimator(estimator_lib.Estimator):
 
     if steps is None:
       raise ValueError('Evaluate `steps` must be set on TPU. Cannot be `None`.')
-    if steps <= 0:
-      raise ValueError('Must specify steps > 0, given: {}'.format(steps))
+
+    util_lib.check_positive_integer(steps, 'Eval steps')
 
     hooks = []
     hooks.append(evaluation._StopAfterNEvalsHook(  # pylint: disable=protected-access
@@ -1609,3 +1616,5 @@ def _validate_tpu_training_graph():
   if not cross_replica_sum_ops:
     raise ValueError(
         'CrossShardOptimizer must be used for model training on TPUs.')
+
+
diff --git a/tensorflow/contrib/tpu/python/tpu/util.py b/tensorflow/contrib/tpu/python/tpu/util.py
new file mode 100644
index 0000000000..b8ea307d89
--- /dev/null
+++ b/tensorflow/contrib/tpu/python/tpu/util.py
@@ -0,0 +1,31 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ===================================================================
+
+"""Utilities for the functionalities."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import six
+
+
+def check_positive_integer(value, name):
+  """Checks whether `value` is a positive integer."""
+  if not isinstance(value, six.integer_types):
+    raise TypeError('{} must be int, got {}'.format(name, type(value)))
+
+  if value <= 0:
+    raise ValueError('{} must be positive, got {}'.format(name, value))
-- 
GitLab


From 80e3b24f27b94c9efbb57cdc2fc57e76cf3e32f2 Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Thu, 12 Oct 2017 13:40:58 -0700
Subject: [PATCH 313/909] Increase the size of extenders_test to avoid flaky
 timeouts.

PiperOrigin-RevId: 172005195
---
 tensorflow/contrib/estimator/BUILD | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/estimator/BUILD b/tensorflow/contrib/estimator/BUILD
index ddfedba579..4dd9f19ec3 100644
--- a/tensorflow/contrib/estimator/BUILD
+++ b/tensorflow/contrib/estimator/BUILD
@@ -92,7 +92,7 @@ py_library(
 
 py_test(
     name = "extenders_test",
-    size = "small",
+    size = "medium",
     srcs = ["python/estimator/extenders_test.py"],
     srcs_version = "PY2AND3",
     deps = [
-- 
GitLab


From f26057f0bf1b4c9dfbffd6828b11dbd2135639ed Mon Sep 17 00:00:00 2001
From: Jiri Simsa <jsimsa@google.com>
Date: Thu, 12 Oct 2017 14:11:36 -0700
Subject: [PATCH 314/909] Adding fused version of map and batch.

PiperOrigin-RevId: 172009823
---
 .../contrib/data/python/kernel_tests/BUILD    |  29 +-
 .../kernel_tests/batch_dataset_op_test.py     | 106 +++++-
 .../contrib/data/python/ops/batching.py       |  76 ++++
 tensorflow/core/kernels/BUILD                 |  16 +
 tensorflow/core/kernels/batch_dataset_op.cc   |   5 +-
 .../core/kernels/map_and_batch_dataset_op.cc  | 351 ++++++++++++++++++
 tensorflow/core/ops/dataset_ops.cc            |  26 ++
 7 files changed, 575 insertions(+), 34 deletions(-)
 create mode 100644 tensorflow/core/kernels/map_and_batch_dataset_op.cc

diff --git a/tensorflow/contrib/data/python/kernel_tests/BUILD b/tensorflow/contrib/data/python/kernel_tests/BUILD
index faf051203c..b090aac0fc 100644
--- a/tensorflow/contrib/data/python/kernel_tests/BUILD
+++ b/tensorflow/contrib/data/python/kernel_tests/BUILD
@@ -11,6 +11,7 @@ py_test(
     size = "small",
     srcs = ["batch_dataset_op_test.py"],
     srcs_version = "PY2AND3",
+    tags = ["no_windows"],
     deps = [
         "//tensorflow/contrib/data/python/ops:dataset_ops",
         "//tensorflow/contrib/data/python/ops:transformation_ops",
@@ -24,6 +25,7 @@ py_test(
         "//tensorflow/python:string_ops",
         "//tensorflow/python:tensor_shape",
         "//tensorflow/python:util",
+        "//tensorflow/python/data/ops:dataset_ops",
         "//third_party/py/numpy",
     ],
 )
@@ -237,33 +239,6 @@ py_test(
     ],
 )
 
-py_test(
-    name = "scan_dataset_op_test",
-    size = "small",
-    srcs = ["scan_dataset_op_test.py"],
-    srcs_version = "PY2AND3",
-    deps = [
-        "//tensorflow/contrib/data/python/ops:dataset_ops",
-        "//tensorflow/contrib/data/python/ops:transformation_ops",
-        "//tensorflow/python:array_ops",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:constant_op",
-        "//tensorflow/python:data_flow_ops",
-        "//tensorflow/python:dtypes",
-        "//tensorflow/python:errors",
-        "//tensorflow/python:functional_ops",
-        "//tensorflow/python:io_ops",
-        "//tensorflow/python:lookup_ops",
-        "//tensorflow/python:math_ops",
-        "//tensorflow/python:random_ops",
-        "//tensorflow/python:script_ops",
-        "//tensorflow/python:string_ops",
-        "//tensorflow/python:util",
-        "//tensorflow/python:variable_scope",
-        "//third_party/py/numpy",
-    ],
-)
-
 py_test(
     name = "range_dataset_op_test",
     size = "small",
diff --git a/tensorflow/contrib/data/python/kernel_tests/batch_dataset_op_test.py b/tensorflow/contrib/data/python/kernel_tests/batch_dataset_op_test.py
index 91f100e0f0..1e7d448949 100644
--- a/tensorflow/contrib/data/python/kernel_tests/batch_dataset_op_test.py
+++ b/tensorflow/contrib/data/python/kernel_tests/batch_dataset_op_test.py
@@ -22,7 +22,7 @@ import math
 import numpy as np
 
 from tensorflow.contrib.data.python.ops import batching
-from tensorflow.contrib.data.python.ops import dataset_ops
+from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
@@ -256,8 +256,9 @@ class BatchDatasetTest(test.TestCase):
   def testDenseToSparseBatchDatasetWithUnknownShape(self):
     components = np.random.randint(5, size=(40,)).astype(np.int32)
     iterator = (dataset_ops.Dataset.from_tensor_slices(components)
-                .map(lambda x: array_ops.fill([x, x], x)).dense_to_sparse_batch(
-                    4, [5, -1]).make_initializable_iterator())
+                .map(lambda x: array_ops.fill([x, x], x)).apply(
+                    batching.dense_to_sparse_batch(
+                        4, [5, -1])).make_initializable_iterator())
     init_op = iterator.initializer
     get_next = sparse_tensor.SparseTensor(*iterator.get_next())
 
@@ -285,7 +286,8 @@ class BatchDatasetTest(test.TestCase):
   def testDenseToSparseBatchDatasetWithInvalidShape(self):
     input_tensor = array_ops.constant([[1]])
     iterator = (dataset_ops.Dataset.from_tensors(input_tensor)
-                .dense_to_sparse_batch(4, [-2]).make_initializable_iterator())
+                .apply(batching.dense_to_sparse_batch(4, [-2]))
+                .make_initializable_iterator())
     init_op = iterator.initializer
 
     with self.test_session() as sess:
@@ -424,6 +426,102 @@ class BatchDatasetTest(test.TestCase):
     self.assertEqual([None], dataset.output_shapes[1][0].as_list())
     self.assertEqual([None, 30], dataset.output_shapes[1][1].as_list())
 
+  def testBatchAndMapDataset(self):
+    """Test a dataset that maps a TF function across its input elements."""
+    # The pipeline is TensorSliceDataset ->
+    # RepeatDataset(count) -> BatchAndMapDataset(square_3, batch_size).
+    components = (np.arange(7),
+                  np.array([[1, 2, 3]]) * np.arange(7)[:, np.newaxis],
+                  np.array(37.0) * np.arange(7))
+
+    count = array_ops.placeholder(dtypes.int64, shape=[])
+    batch_size = array_ops.placeholder(dtypes.int64, shape=[])
+
+    def _map_fn(x, y, z):
+      return math_ops.square(x), math_ops.square(y), math_ops.square(z)
+
+    iterator = (dataset_ops.Dataset.from_tensor_slices(components).repeat(count)
+                .apply(batching.map_and_batch(_map_fn, batch_size))
+                .make_initializable_iterator())
+    init_op = iterator.initializer
+    get_next = iterator.get_next()
+
+    self.assertEqual([[None] + list(c.shape[1:]) for c in components],
+                     [t.shape.as_list() for t in get_next])
+
+    with self.test_session() as sess:
+      # Batch of a finite input, where the batch_size divides the
+      # total number of elements.
+      sess.run(init_op, feed_dict={count: 28, batch_size: 14})
+      num_batches = (28 * 7) // 14
+      for i in range(num_batches):
+        result = sess.run(get_next)
+        for component, result_component in zip(components, result):
+          for j in range(14):
+            self.assertAllEqual(component[(i*14 + j) % 7]**2,
+                                result_component[j])
+      with self.assertRaises(errors.OutOfRangeError):
+        sess.run(get_next)
+
+      # Batch of a finite input, where the batch_size does not
+      # divide the total number of elements.
+      sess.run(init_op, feed_dict={count: 14, batch_size: 8})
+
+      # We expect (num_batches - 1) full-sized batches.
+      num_batches = int(math.ceil((14 * 7) / 8))
+      for i in range(num_batches - 1):
+        result = sess.run(get_next)
+        for component, result_component in zip(components, result):
+          for j in range(8):
+            self.assertAllEqual(component[(i*8 + j) % 7]**2,
+                                result_component[j])
+      # The last batch should fail with `OutOfRange`.
+      with self.assertRaises(errors.OutOfRangeError):
+        sess.run(get_next)
+
+      # Batch of an empty input should fail straight away.
+      sess.run(init_op, feed_dict={count: 0, batch_size: 8})
+      with self.assertRaises(errors.OutOfRangeError):
+        sess.run(get_next)
+
+      # Empty batch should be an initialization time error.
+      with self.assertRaises(errors.InvalidArgumentError):
+        sess.run(init_op, feed_dict={count: 14, batch_size: 0})
+
+  def testBatchAndMapDatasetFails(self):
+    """Test a dataset that maps a TF function across its input elements."""
+    dataset = dataset_ops.Dataset.from_tensors(
+        array_ops.check_numerics(
+            constant_op.constant(1.0) / constant_op.constant(0.0), "oops"))
+    batch_size = array_ops.placeholder(dtypes.int64, shape=[])
+    iterator = (dataset.apply(batching.map_and_batch(lambda x: x, batch_size))
+                .make_initializable_iterator())
+    init_op = iterator.initializer
+    with self.test_session() as sess:
+      with self.assertRaisesRegexp(errors.InvalidArgumentError, "oops"):
+        sess.run(init_op, feed_dict={batch_size: 14})
+
+  def testBatchAndMapDatasetShapeMismatch(self):
+    """Test a dataset that maps a TF function across its input elements."""
+    def generator():
+      yield [1]
+      yield [2]
+      yield [3]
+      yield [[4, 5, 6]]
+
+    dataset = dataset_ops.Dataset.from_generator(
+        generator, output_types=dtypes.int32)
+    batch_size = 4
+    iterator = (
+        dataset.apply(batching.map_and_batch(lambda x: x, batch_size))
+        .make_initializable_iterator())
+    init_op = iterator.initializer
+    get_next = iterator.get_next()
+    with self.test_session() as sess:
+      sess.run(init_op)
+      with self.assertRaisesRegexp(errors.InvalidArgumentError,
+                                   "number of elements does not match"):
+        sess.run(get_next)
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/contrib/data/python/ops/batching.py b/tensorflow/contrib/data/python/ops/batching.py
index ccfa8747da..abc9212a87 100644
--- a/tensorflow/contrib/data/python/ops/batching.py
+++ b/tensorflow/contrib/data/python/ops/batching.py
@@ -272,3 +272,79 @@ class _RestructuredDataset(dataset_ops.Dataset):
   @property
   def output_shapes(self):
     return self._output_shapes
+
+
+class _MapAndBatchDataset(dataset_ops.MapDataset):
+  """A `Dataset` that maps a function over a batch of elements."""
+
+  def __init__(self, input_dataset, map_func, batch_size, num_parallel_batches):
+    """See `Dataset.map()` for details."""
+    super(_MapAndBatchDataset, self).__init__(input_dataset, map_func)
+
+    self._batch_size = ops.convert_to_tensor(
+        batch_size, dtype=dtypes.int64, name="batch_size")
+    self._num_parallel_batches = ops.convert_to_tensor(
+        num_parallel_batches, dtype=dtypes.int64, name="num_parallel_batches")
+
+  def _as_variant_tensor(self):
+    # pylint: disable=protected-access
+    input_resource = self._input_dataset._as_variant_tensor()
+    return gen_dataset_ops.map_and_batch_dataset(
+        input_resource,
+        self._map_func.captured_inputs,
+        f=self._map_func,
+        batch_size=self._batch_size,
+        num_parallel_batches=self._num_parallel_batches,
+        output_types=nest.flatten(self.output_types),
+        output_shapes=nest.flatten(self.output_shapes))
+    # pylint: enable=protected-access
+
+  @property
+  def output_shapes(self):
+    return nest.pack_sequence_as(self._output_shapes, [
+        tensor_shape.vector(tensor_util.constant_value(
+            self._batch_size)).concatenate(s)
+        for s in nest.flatten(self._output_shapes)
+    ])
+
+  @property
+  def output_types(self):
+    return self._output_types
+
+
+def map_and_batch(map_func, batch_size, num_parallel_batches=1):
+  """Fused implementation of `map` and `batch`.
+
+  Maps `map_func` across `batch_size` consecutive elements of this dataset
+  and then combines them into a batch. Similarly to `batch_and_drop_remainder`,
+  if the batch size does not evenly divide the input dataset size, this
+  transformation will drop the final smaller element.
+
+
+  Functionally, it is equivalent to `map` followed by
+  `batch_and_drop_remainder`. However, by fusing the two transformations
+  together, the implementation can be more efficient. This transformation is a
+  stop gap solution for performance critical workloads. Once automatic input
+  pipeline optimization are implemented, the fusing of map and batch will not
+  need to be exposed at the API level and this method will be removed.
+
+  Args:
+    map_func: A function mapping a nested structure of tensors to another
+      nested structure of tensors.
+    batch_size: A `tf.int64` scalar `tf.Tensor`, representing the number of
+      consecutive elements of this dataset to combine in a single batch.
+    num_parallel_batches: A `tf.int64` scalar `tf.Tensor`, representing the
+      number of batches to create in parallel. On one hand, higher values can
+      help mitigate the effect of stragglers. On the other hand, higher values
+      can increasing contention if CPU is scarce.
+
+  Returns:
+    A `Dataset` transformation function, which can be passed to
+    @{tf.contrib.data.Dataset.apply}.
+  """
+
+  def _apply_fn(dataset):
+    return _MapAndBatchDataset(dataset, map_func, batch_size,
+                               num_parallel_batches)
+
+  return _apply_fn
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 369feaf49a..13dbf38fe6 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -5781,6 +5781,21 @@ tf_kernel_library(
     ],
 )
 
+tf_kernel_library(
+    name = "map_and_batch_dataset_op",
+    srcs = ["map_and_batch_dataset_op.cc"],
+    deps = [
+        ":captured_function",
+        ":dataset",
+        ":inplace_ops",
+        "//tensorflow/core:core_cpu_internal",
+        "//tensorflow/core:dataset_ops_op_lib",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:lib_internal",
+    ],
+)
+
 tf_kernel_library(
     name = "parallel_map_dataset_op",
     srcs = ["parallel_map_dataset_op.cc"],
@@ -6074,6 +6089,7 @@ tf_kernel_library(
         ":ignore_errors_dataset_op",
         ":interleave_dataset_op",
         ":iterator_ops",
+        ":map_and_batch_dataset_op",
         ":map_dataset_op",
         ":padded_batch_dataset_op",
         ":parallel_map_dataset_op",
diff --git a/tensorflow/core/kernels/batch_dataset_op.cc b/tensorflow/core/kernels/batch_dataset_op.cc
index 51e7c33713..631840081f 100644
--- a/tensorflow/core/kernels/batch_dataset_op.cc
+++ b/tensorflow/core/kernels/batch_dataset_op.cc
@@ -91,10 +91,9 @@ class BatchDatasetOp : public UnaryDatasetOpKernel {
           (parent->NumElements() / parent->dim_size(0))) {
         TensorShape chip_shape = parent->shape();
         chip_shape.RemoveDim(0);
-        return errors::Internal(
+        return errors::InvalidArgument(
             "HandleElementToSlice Cannot copy slice: number of elements does "
-            "not "
-            "match.  Shapes are: [element]: ",
+            "not match. Shapes are: [element]: ",
             element.shape().DebugString(),
             ", [parent slice]: ", chip_shape.DebugString());
       }
diff --git a/tensorflow/core/kernels/map_and_batch_dataset_op.cc b/tensorflow/core/kernels/map_and_batch_dataset_op.cc
new file mode 100644
index 0000000000..332a96ae03
--- /dev/null
+++ b/tensorflow/core/kernels/map_and_batch_dataset_op.cc
@@ -0,0 +1,351 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#define EIGEN_USE_THREADS
+
+#include "tensorflow/core/lib/core/blocking_counter.h"
+#include "tensorflow/core/common_runtime/function.h"
+#include "tensorflow/core/framework/partial_tensor_shape.h"
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/kernels/captured_function.h"
+#include "tensorflow/core/kernels/dataset.h"
+#include "tensorflow/core/kernels/inplace_ops_functor.h"
+#include "tensorflow/core/lib/random/random.h"
+#include "tensorflow/core/lib/strings/strcat.h"
+#include "tensorflow/core/platform/tracing.h"
+
+namespace tensorflow {
+
+namespace {
+
+// See documentation in ../ops/dataset_ops.cc for a high-level
+// description of the following op.
+
+class MapAndBatchDatasetOp : public UnaryDatasetOpKernel {
+ public:
+  explicit MapAndBatchDatasetOp(OpKernelConstruction* ctx)
+      : UnaryDatasetOpKernel(ctx),
+        graph_def_version_(ctx->graph_def_version()) {
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("f", &func_));
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("output_types", &output_types_));
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("output_shapes", &output_shapes_));
+  }
+
+ protected:
+  void MakeDataset(OpKernelContext* ctx, DatasetBase* input,
+                   DatasetBase** output) override {
+    OpInputList inputs;
+    OP_REQUIRES_OK(ctx, ctx->input_list("other_arguments", &inputs));
+    std::vector<Tensor> other_arguments;
+    other_arguments.reserve(inputs.size());
+    for (const Tensor& t : inputs) {
+      other_arguments.push_back(t);
+    }
+
+    int64 batch_size;
+    OP_REQUIRES_OK(ctx, ParseScalarArgument(ctx, "batch_size", &batch_size));
+    OP_REQUIRES(
+        ctx, batch_size > 0,
+        errors::InvalidArgument("batch_size must be greater than zero."));
+
+    int64 num_parallel_batches;
+    OP_REQUIRES_OK(ctx, ParseScalarArgument(ctx, "num_parallel_batches",
+                                            &num_parallel_batches));
+    OP_REQUIRES(ctx, num_parallel_batches > 0,
+                errors::InvalidArgument(
+                    "num_parallel_batches must be greater than zero."));
+
+    std::unique_ptr<CapturedFunction> captured_func;
+    OP_REQUIRES_OK(ctx, CapturedFunction::Create(ctx, func_, graph_def_version_,
+                                                 std::move(other_arguments),
+                                                 &captured_func));
+
+    *output = new Dataset(input, batch_size, num_parallel_batches,
+                          output_types_, output_shapes_,
+                          std::move(captured_func), &ctx->eigen_cpu_device());
+  }
+
+ private:
+  class Dataset : public DatasetBase {
+   public:
+    Dataset(const DatasetBase* input, int64 batch_size,
+            int64 num_parallel_batches, const DataTypeVector& output_types,
+            const std::vector<PartialTensorShape>& output_shapes,
+            std::unique_ptr<CapturedFunction> captured_func,
+            const Eigen::ThreadPoolDevice* device)
+        : input_(input),
+          batch_size_(batch_size),
+          num_parallel_batches_(num_parallel_batches),
+          output_types_(output_types),
+          output_shapes_(output_shapes),
+          captured_func_(std::move(captured_func)),
+          device_(device) {
+      input_->Ref();
+    }
+
+    ~Dataset() override { input_->Unref(); }
+
+    std::unique_ptr<IteratorBase> MakeIterator(
+        const string& prefix) const override {
+      return std::unique_ptr<IteratorBase>(
+          new Iterator({this, strings::StrCat(prefix, "::MapAndBatch")}));
+    }
+
+    const DataTypeVector& output_dtypes() const override {
+      return output_types_;
+    }
+
+    const std::vector<PartialTensorShape>& output_shapes() const override {
+      return output_shapes_;
+    }
+
+    string DebugString() override { return "MapAndBatchDatasetOp::Dataset"; }
+
+   private:
+    class Iterator : public DatasetIterator<Dataset> {
+     public:
+      explicit Iterator(const Params& params)
+          : DatasetIterator<Dataset>(params),
+            input_impl_(params.dataset->input_->MakeIterator(params.prefix)),
+            invocation_results_(params.dataset->batch_size_ *
+                                params.dataset->num_parallel_batches_),
+            batch_results_(params.dataset->num_parallel_batches_) {}
+
+      ~Iterator() override {
+        // TODO(mrry): Replace this cancellation logic with a
+        // CancellationManager. The syntax would be more heavyweight,
+        // but it would be possible to thread a cancellation manager
+        // through the IteratorContext to upstream,
+        // potentially-blocking iterators, when we add these.
+        mutex_lock l(mu_);
+        if (current_batch_index_ != -1) {
+          for (size_t batch_index = 0;
+               batch_index < dataset()->num_parallel_batches_; ++batch_index) {
+            WaitForBatch(batch_index).IgnoreError();
+            // Deallocate tensors allocated for the output.
+            batch_results_[batch_index].output.clear();
+          }
+        }
+      }
+
+      // TODO(jsimsa): Implement and profile the following alternative design:
+      //
+      // 0. Set the number of in-flight batches and invocations independently
+      // (though obviously the max number of in-flight invocations must be <
+      // batch_size * num_parallel_batches). Maintain a current producing batch
+      // index and offset.
+      // 1. Issue invocations in order of batch and offset, as you do currently.
+      // 2. When an invocation finishes, increment the current producing batch
+      // and offset. If that invocation would start a new batch and give more
+      // than num_parallel_batches in-flight, block; else start the new
+      // invocation into that location.
+      // 3. When a GetNext() call arrives, block until there's a full batch.
+      // Before returning the batch, if the number of pending invocations is
+      // less than the max, issue that number of invocations.
+      Status GetNextInternal(IteratorContext* ctx,
+                             std::vector<Tensor>* out_tensors,
+                             bool* end_of_sequence) override {
+        mutex_lock l(mu_);
+
+        // One-time initialization.
+        if (current_batch_index_ == -1) {
+          current_batch_index_ = 0;
+          for (size_t i = 0; i < dataset()->num_parallel_batches_; ++i) {
+            StartInvocationBatch(ctx, i);
+          }
+        }
+
+        if (end_of_input_) {
+          *end_of_sequence = true;
+          return Status::OK();
+        }
+
+        Status status = WaitForBatch(current_batch_index_);
+        if (!status.ok()) {
+          // Deallocate tensors allocated for the output.
+          batch_results_[current_batch_index_].output.clear();
+        } else {
+          *out_tensors = std::move(batch_results_[current_batch_index_].output);
+          *end_of_sequence = false;
+        }
+        StartInvocationBatch(ctx, current_batch_index_);
+        current_batch_index_ =
+            (current_batch_index_ + 1) % dataset()->num_parallel_batches_;
+        return status;
+      }
+
+     private:
+      struct BatchResult {
+        mutex mu;
+        bool output_allocated GUARDED_BY(mu);
+        std::vector<Tensor> output;
+        std::unique_ptr<BlockingCounter> counter;
+      };
+
+      struct InvocationResult {
+        Status status;
+        std::vector<Tensor> return_values;
+      };
+
+      int64 ComputeInvocationIndex(int64 batch_index, int64 offset) {
+        return batch_index * dataset()->batch_size_ + offset;
+      }
+
+      void EnsureOutputAllocated(BatchResult* batch_result,
+                                 const std::vector<Tensor>& return_values) {
+        mutex_lock l(batch_result->mu);
+        if (batch_result->output_allocated) {
+          return;
+        }
+        const size_t num_components = return_values.size();
+        for (size_t i = 0; i < num_components; ++i) {
+          TensorShape component_shape({dataset()->batch_size_});
+          component_shape.AppendShape(return_values[i].shape());
+          Tensor component(cpu_allocator(), return_values[i].dtype(),
+                           component_shape);
+          batch_result->output.emplace_back(std::move(component));
+        }
+        batch_result->output_allocated = true;
+      }
+
+      void InvokeFunctionLocked(IteratorContext* ctx, int64 batch_index,
+                                int64 offset) EXCLUSIVE_LOCKS_REQUIRED(mu_) {
+        size_t index = ComputeInvocationIndex(batch_index, offset);
+        InvocationResult* result = &invocation_results_[index];
+        BatchResult* batch_result = &batch_results_[batch_index];
+
+        // Get the next input element.
+        std::vector<Tensor> input_element;
+        result->status =
+            input_impl_->GetNext(ctx, &input_element, &end_of_input_);
+        if (end_of_input_ || !result->status.ok()) {
+          batch_result->counter->DecrementCount();
+          return;
+        }
+
+        // Call `captured_func_(input_element)`, store the result in
+        // `result->return_values`, and notify `batch_result->counter`
+        // to unblock a consumer.
+        FunctionLibraryRuntime::Options opts;
+        opts.step_id = CapturedFunction::generate_step_id();
+        ScopedStepContainer* step_container = new ScopedStepContainer(
+            opts.step_id, [this, ctx](const string& name) {
+              dataset()
+                  ->captured_func_->resource_manager()
+                  ->Cleanup(name)
+                  .IgnoreError();
+            });
+        opts.step_container = step_container;
+        opts.runner = ctx->runner();
+        dataset()->captured_func_->RunAsync(
+            opts, input_element, &result->return_values,
+            [this, result, step_container, batch_result,
+             offset](Status ret_status) {
+              delete step_container;
+              result->status.Update(ret_status);
+              if (ret_status.ok()) {
+                EnsureOutputAllocated(batch_result, result->return_values);
+                const size_t num_components = result->return_values.size();
+                for (size_t i = 0; i < num_components; ++i) {
+                  Tensor tensor = result->return_values[i];
+                  Tensor* batch = &(batch_result->output)[i];
+                  if (tensor.NumElements() !=
+                      (batch->NumElements() / batch->dim_size(0))) {
+                    TensorShape batch_shape = batch->shape();
+                    batch_shape.RemoveDim(0);
+                    result->status.Update(errors::InvalidArgument(
+                        "Cannot add tensor to the batch: number of "
+                        "elements does not match. Shapes are: [tensor]: ",
+                        tensor.shape().DebugString(),
+                        ", [batch]: ", batch_shape.DebugString()));
+                    break;
+                  }
+                  Status copy_status = ::tensorflow::functor::DoParallelConcat(
+                      *dataset()->device_, tensor, offset, batch);
+                  if (!copy_status.ok()) {
+                    result->status.Update(copy_status);
+                    break;
+                  }
+                }
+              }
+              batch_result->counter->DecrementCount();
+            });
+      }
+
+      void StartInvocationBatch(IteratorContext* ctx, int64 batch_index)
+          EXCLUSIVE_LOCKS_REQUIRED(mu_) {
+        port::Tracing::TraceMe activity(strings::StrCat(prefix(), "::Start"));
+        // Initialize batch result.
+        mutex_lock l(batch_results_[batch_index].mu);
+        batch_results_[batch_index].output_allocated = false;
+        batch_results_[batch_index].counter.reset(
+            new BlockingCounter(dataset()->batch_size_));
+        // Initialize invocation results.
+        for (size_t i = 0; i < dataset()->batch_size_; ++i) {
+          size_t index = ComputeInvocationIndex(batch_index, i);
+          InvocationResult* result = &invocation_results_[index];
+          *result = InvocationResult();
+        }
+        // Start individual invocations.
+        for (size_t i = 0; i < dataset()->batch_size_; ++i) {
+          InvokeFunctionLocked(ctx, batch_index, i);
+        }
+      }
+
+      Status WaitForBatch(int64 batch_index) EXCLUSIVE_LOCKS_REQUIRED(mu_) {
+        port::Tracing::TraceMe activity(strings::StrCat(prefix(), "::Wait"));
+        batch_results_[batch_index].counter->Wait();
+        Status status = Status::OK();
+        for (size_t i = 0; i < dataset()->batch_size_; ++i) {
+          size_t index = ComputeInvocationIndex(batch_index, i);
+          InvocationResult* result = &invocation_results_[index];
+          if (!result->status.ok()) {
+            VLOG(3) << "failed to process element[" << i
+                    << "]: " << result->status;
+            status.Update(result->status);
+          }
+        }
+        return status;
+      }
+
+      mutex mu_;
+      int32 current_batch_index_ GUARDED_BY(mu_) = -1;
+      const std::unique_ptr<IteratorBase> input_impl_ GUARDED_BY(mu_);
+      std::vector<InvocationResult> invocation_results_ GUARDED_BY(mu_);
+      std::vector<BatchResult> batch_results_ GUARDED_BY(mu_);
+      bool end_of_input_ GUARDED_BY(mu_) = false;
+    };
+
+    const DatasetBase* const input_;
+    const NameAttrList func_;
+    const int64 batch_size_;
+    const int64 num_parallel_batches_;
+    const DataTypeVector output_types_;
+    const std::vector<PartialTensorShape> output_shapes_;
+    const std::unique_ptr<CapturedFunction> captured_func_;
+    const Eigen::ThreadPoolDevice* device_; // not owned
+  };
+
+  const int graph_def_version_;
+  DataTypeVector output_types_;
+  std::vector<PartialTensorShape> output_shapes_;
+  NameAttrList func_;
+};
+
+REGISTER_KERNEL_BUILDER(Name("MapAndBatchDataset").Device(DEVICE_CPU),
+                        MapAndBatchDatasetOp);
+
+}  // namespace
+
+}  // namespace tensorflow
diff --git a/tensorflow/core/ops/dataset_ops.cc b/tensorflow/core/ops/dataset_ops.cc
index fe346b5240..566049179a 100644
--- a/tensorflow/core/ops/dataset_ops.cc
+++ b/tensorflow/core/ops/dataset_ops.cc
@@ -184,6 +184,32 @@ num_parallel_calls: The number of concurrent invocations of `f` that process
   elements from `input_dataset` in parallel.
 )doc");
 
+REGISTER_OP("MapAndBatchDataset")
+    .Input("input_dataset: variant")
+    .Input("other_arguments: Targuments")
+    .Input("batch_size: int64")
+    .Input("num_parallel_batches: int64")
+    .Output("handle: variant")
+    .Attr("f: func")
+    .Attr("Targuments: list(type) >= 0")
+    .Attr("output_types: list(type) >= 1")
+    .Attr("output_shapes: list(shape) >= 1")
+    .SetShapeFn(shape_inference::ScalarShape)
+    .Doc(R"doc(
+Creates a dataset that applies `f` to the outputs of `input_dataset` and then
+batches `batch_size` of them.
+
+Unlike a "MapDataset", which applies `f` sequentially, this dataset invokes up
+to `batch_size * num_parallel_batches` copies of `f` in parallel.
+
+batch_size: A scalar representing the number of elements to accumulate in a
+  batch. It determines the number of concurrent invocations of `f` that process
+  elements from `input_dataset` in parallel.
+num_parallel_batches: A scalar representing the number of batches to create in
+  parallel. Processing multiple batches in parallel benefits workloads prone to
+  stragglers.
+)doc");
+
 REGISTER_OP("PrefetchDataset")
     .Input("input_dataset: variant")
     .Input("buffer_size: int64")
-- 
GitLab


From 4351f887967a521458f98948e278abcd6ad63292 Mon Sep 17 00:00:00 2001
From: Chris Ying <chrisying@google.com>
Date: Thu, 12 Oct 2017 14:27:43 -0700
Subject: [PATCH 315/909] Multi-axis batch normalization support and changing
 from num_virtual_batches to virtual_batch_size

Key changes:
1) adding support for multi-axis batch norm by allowing the axis to be either an int or a list of ints
2) multi-axis batch norm is handled entirely by TensorFlow ops at the moment (no special kernel) and the performance is heavily dependent on the Tensor format (see reduce_* kernels for reduction rules)
3) Fix ghost batch norm by sharing the same gamma/beta/mean/var parameter across all virtual batches
4) Change ghost batch norm API to virtual_batch_size to be more consistent with its intended use case.

PiperOrigin-RevId: 172012360
---
 tensorflow/python/layers/normalization.py     | 301 +++++++++---------
 .../python/layers/normalization_test.py       | 231 ++++++++++++--
 ...nsorflow.layers.-batch-normalization.pbtxt |   2 +-
 .../tools/api/golden/tensorflow.layers.pbtxt  |   2 +-
 4 files changed, 352 insertions(+), 184 deletions(-)

diff --git a/tensorflow/python/layers/normalization.py b/tensorflow/python/layers/normalization.py
index ebcf397625..4dab87b227 100644
--- a/tensorflow/python/layers/normalization.py
+++ b/tensorflow/python/layers/normalization.py
@@ -49,9 +49,14 @@ class BatchNormalization(base.Layer):
   Sergey Ioffe, Christian Szegedy
 
   Arguments:
-    axis: An `int`, the axis that should be normalized (typically the features
-      axis). For instance, after a `Conv2D` layer with
-      `data_format="channels_first"`, set `axis=1` in `BatchNormalization`.
+    axis: An `int` or list of `int`, the axis or axes that should be
+        normalized, typically the features axis/axes. For instance, after a
+        `Conv2D` layer with `data_format="channels_first"`, set `axis=1`. If a
+        list of axes is provided, each axis in `axis` will be normalized
+        simultaneously. Default is `-1` which takes uses last axis. Note: when
+        using multi-axis batch norm, the `beta`, `gamma`, `moving_mean`, and
+        `moving_variance` variables are the same rank as the input Tensor, with
+        dimension size 1 in all reduced (non-axis) dimensions).
     momentum: Momentum for the moving average.
     epsilon: Small float added to variance to avoid dividing by zero.
     center: If True, add offset of `beta` to normalized tensor. If False, `beta`
@@ -90,11 +95,12 @@ class BatchNormalization(base.Layer):
       If `None`, use the system recommended implementation.
     trainable: Boolean, if `True` also add variables to the graph collection
       `GraphKeys.TRAINABLE_VARIABLES` (see tf.Variable).
-    num_virtual_batches: An `int`, specifies the number of virtual batches to
-      operate over. If not greater than 1, will perform "ghost batch
-      normalization", which creates virtual sub-batches to operate over for
-      batch norm. Default is 1 virtual batch, in which no virtual batching is
-      performed. Must divide the actual batch size during graph execution.
+    virtual_batch_size: An `int`. By default, `virtual_batch_size` is `None`,
+      which means batch normalization is performed across the whole batch. When
+      `virtual_batch_size` is not `None`, instead perform "Ghost Batch
+      Normalization", which creates virtual sub-batches which are each
+      normalized separately (with shared gamma, beta, and moving statistics).
+      Must divide the actual batch size during execution.
     name: A string, the name of the layer.
   """
 
@@ -117,7 +123,7 @@ class BatchNormalization(base.Layer):
                renorm_momentum=0.99,
                fused=None,
                trainable=True,
-               num_virtual_batches=1,
+               virtual_batch_size=None,
                name=None,
                **kwargs):
     super(BatchNormalization, self).__init__(
@@ -136,16 +142,13 @@ class BatchNormalization(base.Layer):
     self.beta_constraint = beta_constraint
     self.gamma_constraint = gamma_constraint
     self.renorm = renorm
+    self.virtual_batch_size = virtual_batch_size
     if fused is None:
       fused = True
 
     self.fused = fused
     self._bessels_correction_test_only = True
 
-    if num_virtual_batches < 1:
-      raise ValueError('num_virtual_batches must be a positive integer')
-    self.num_virtual_batches = num_virtual_batches
-
     if renorm:
       renorm_clipping = renorm_clipping or {}
       keys = ['rmax', 'rmin', 'dmax']
@@ -159,14 +162,36 @@ class BatchNormalization(base.Layer):
     input_shape = tensor_shape.TensorShape(input_shape)
     if not input_shape.ndims:
       raise ValueError('Input has undefined rank:', input_shape)
-    ndim = len(input_shape)
-    if self.axis < 0:
-      axis = ndim + self.axis
-    else:
-      axis = self.axis
-    if axis < 0 or axis >= ndim:
-      raise ValueError('Value of `axis` argument ' + str(self.axis) +
-                       ' is out of range for input with rank ' + str(ndim))
+    ndims = len(input_shape)
+
+    # Convert axis to list and resolve negatives
+    if isinstance(self.axis, int):
+      self.axis = [self.axis]
+
+    if not isinstance(self.axis, list):
+      raise TypeError('axis must be int or list, type given: %s'
+                      % type(self.axis))
+
+    for idx, x in enumerate(self.axis):
+      if x < 0:
+        self.axis[idx] = ndims + x
+
+    # Validate axes
+    for x in self.axis:
+      if x < 0 or x >= ndims:
+        raise ValueError('Invalid axis: %d' % x)
+    if len(self.axis) != len(set(self.axis)):
+      raise ValueError('Duplicate axis: %s' % self.axis)
+
+    if self.virtual_batch_size is not None:
+      if self.virtual_batch_size <= 0:
+        raise ValueError('virtual_batch_size must be a positive integer that '
+                         'divides the true batch size of the input Tensor')
+      # If using virtual batches, the first dimension must be the batch
+      # dimension and cannot be the batch norm axis
+      if 0 in self.axis:
+        raise ValueError('When using virtual_batch_size, the batch dimension '
+                         'must be 0 and thus axis cannot include 0')
 
     if self.fused:
       # Currently fused batch norm doesn't support renorm and beta/gamma
@@ -174,30 +199,51 @@ class BatchNormalization(base.Layer):
       # dimension on axis 1 and 3.
       # TODO(yaozhang): if input is not 4D, reshape it to 4D and reshape the
       # output back to its original shape accordingly.
-      self.fused = not self.renorm and ndim == 4 and axis in [
-          1, 3
-      ] and self.beta_regularizer is None and self.gamma_regularizer is None
+      self.fused = (not self.renorm and
+                    ndims == 4 and
+                    self.axis in [[1], [3]] and
+                    self.beta_regularizer is None and
+                    self.gamma_regularizer is None and
+                    self.virtual_batch_size is None)
+      # TODO(chrisying): fused batch norm is currently not supported for
+      # multi-axis batch norm and by extension virtual batches. In some cases,
+      # it might be possible to use fused batch norm but would require reshaping
+      # the Tensor to 4D with the axis in 1 or 3 (preferred 1) which is
+      # particularly tricky. A compromise might be to just support the most
+      # common use case (turning 5D w/ virtual batch to NCHW)
 
     if self.fused:
-      if axis == 1:
+      if self.axis == [1]:
         self._data_format = 'NCHW'
-      else:
+      elif self.axis == [3]:
         self._data_format = 'NHWC'
-
-    param_dim = input_shape[axis]
-    if not param_dim.value:
-      raise ValueError('Input has undefined `axis` dimension. Input shape: ',
-                       input_shape)
-    self.input_spec = base.InputSpec(ndim=ndim,
-                                     axes={self.axis: param_dim.value})
-
-    if self.num_virtual_batches > 1:
-      # the axis dim is combined with num_virtual_batches
-      param_dim = input_shape[axis] * self.num_virtual_batches
+      else:
+        raise ValueError('Unsupported axis, fused batch norm only supports '
+                         'axis == [1] or axis == [3]')
+
+    axis_to_dim = {x: input_shape[x].value for x in self.axis}
+    for x in axis_to_dim:
+      if axis_to_dim[x] is None:
+        raise ValueError('Input has undefined `axis` dimension. Input shape: ',
+                         input_shape)
+    self.input_spec = base.InputSpec(ndim=ndims, axes=axis_to_dim)
+
+    if len(axis_to_dim) == 1 and self.virtual_batch_size is None:
+      # Single axis batch norm (most common/default use-case)
+      param_shape = (list(axis_to_dim.values())[0],)
+    else:
+      # Parameter shape is the original shape but with 1 in all non-axis dims
+      param_shape = [axis_to_dim[i] if i in axis_to_dim
+                     else 1 for i in range(ndims)]
+      if self.virtual_batch_size is not None:
+        # When using virtual batches, add an extra dim at index 1
+        param_shape.insert(1, 1)
+        for idx, x in enumerate(self.axis):
+          self.axis[idx] = x + 1      # Account for added dimension
 
     if self.scale:
       self.gamma = self.add_variable(name='gamma',
-                                     shape=(param_dim,),
+                                     shape=param_shape,
                                      initializer=self.gamma_initializer,
                                      regularizer=self.gamma_regularizer,
                                      constraint=self.gamma_constraint,
@@ -205,11 +251,11 @@ class BatchNormalization(base.Layer):
     else:
       self.gamma = None
       if self.fused:
-        self._gamma_const = array_ops.constant(1.0, shape=(param_dim,))
+        self._gamma_const = array_ops.constant(1.0, shape=param_shape)
 
     if self.center:
       self.beta = self.add_variable(name='beta',
-                                    shape=(param_dim,),
+                                    shape=param_shape,
                                     initializer=self.beta_initializer,
                                     regularizer=self.beta_regularizer,
                                     constraint=self.beta_constraint,
@@ -217,7 +263,7 @@ class BatchNormalization(base.Layer):
     else:
       self.beta = None
       if self.fused:
-        self._beta_const = array_ops.constant(0.0, shape=(param_dim,))
+        self._beta_const = array_ops.constant(0.0, shape=param_shape)
 
     # Disable variable partitioning when creating the moving mean and variance
     try:
@@ -228,14 +274,16 @@ class BatchNormalization(base.Layer):
         partitioner = None
       self.moving_mean = self.add_variable(
           name='moving_mean',
-          shape=(param_dim,),
+          shape=param_shape,
           initializer=self.moving_mean_initializer,
           trainable=False)
+
       self.moving_variance = self.add_variable(
           name='moving_variance',
-          shape=(param_dim,),
+          shape=param_shape,
           initializer=self.moving_variance_initializer,
           trainable=False)
+
       self._one_minus_decay = 1.0 - self.momentum
       if self.renorm:
         # Create variables to maintain the moving mean and standard deviation.
@@ -256,7 +304,7 @@ class BatchNormalization(base.Layer):
           device = ((lambda _: self.moving_mean.device)
                     if context.in_graph_mode() else self.moving_mean.device)
           with ops.device(device):
-            self.renorm_mean = _renorm_variable('renorm_mean', (param_dim,))
+            self.renorm_mean = _renorm_variable('renorm_mean', param_shape)
             self.renorm_mean_weight = _renorm_variable('renorm_mean_weight', ())
           # We initialize renorm_stddev to 0, and maintain the (0-initialized)
           # renorm_stddev_weight. This allows us to (1) mix the average
@@ -265,7 +313,7 @@ class BatchNormalization(base.Layer):
           device = ((lambda _: self.moving_variance.device)
                     if context.in_graph_mode() else self.moving_variance.device)
           with ops.device(device):
-            self.renorm_stddev = _renorm_variable('renorm_stddev', (param_dim,))
+            self.renorm_stddev = _renorm_variable('renorm_stddev', param_shape)
             self.renorm_stddev_weight = _renorm_variable(
                 'renorm_stddev_weight', ())
     finally:
@@ -315,6 +363,9 @@ class BatchNormalization(base.Layer):
 
     output, mean, variance = utils.smart_cond(
         training, _fused_batch_norm_training, _fused_batch_norm_inference)
+    mean = array_ops.reshape(mean, shape=self.moving_mean.get_shape())
+    variance = array_ops.reshape(variance,
+                                 shape=self.moving_variance.get_shape())
     if not self._bessels_correction_test_only:
       # Remove Bessel's correction to be consistent with non-fused batch norm.
       # Note that the variance computed by fused batch norm is
@@ -326,9 +377,9 @@ class BatchNormalization(base.Layer):
 
     training_value = utils.constant_value(training)
     if training_value is None:
-      one_minus_decay = _smart_select(training,
-                                      lambda: self._one_minus_decay,
-                                      lambda: 0.)
+      one_minus_decay = utils.smart_cond(training,
+                                         lambda: self._one_minus_decay,
+                                         lambda: 0.)
     else:
       one_minus_decay = ops.convert_to_tensor(self._one_minus_decay)
     if training_value or training_value is None:
@@ -371,9 +422,9 @@ class BatchNormalization(base.Layer):
       d = math_ops.maximum(d, -dmax)
       d = math_ops.minimum(d, dmax)
     # When not training, use r=1, d=0, and decay=1 meaning no updates.
-    r = _smart_select(training, lambda: r, lambda: array_ops.ones_like(r))
-    d = _smart_select(training, lambda: d, lambda: array_ops.zeros_like(d))
-    decay = _smart_select(training, lambda: self.renorm_momentum, lambda: 1.)
+    r = utils.smart_cond(training, lambda: r, lambda: array_ops.ones_like(r))
+    d = utils.smart_cond(training, lambda: d, lambda: array_ops.zeros_like(d))
+    decay = utils.smart_cond(training, lambda: self.renorm_momentum, lambda: 1.)
 
     def _update_renorm_variable(var, weight, value):
       """Updates a moving average and weight, returns the unbiased value."""
@@ -406,65 +457,32 @@ class BatchNormalization(base.Layer):
     return (r, d, new_mean, new_variance)
 
   def call(self, inputs, training=False):
-    if self.num_virtual_batches > 1:
-      # Virtual batches (aka ghost batches) can be simulated by using some
-      # reshape/transpose tricks on top of base batch normalization.
+    if self.virtual_batch_size is not None:
+      # Virtual batches (aka ghost batches) can be simulated by reshaping the
+      # Tensor and reusing the existing batch norm implementation
       original_shape = [-1] + inputs.shape.as_list()[1:]
-      expanded_shape = [-1, self.num_virtual_batches] + original_shape[1:]
+      expanded_shape = [self.virtual_batch_size, -1] + original_shape[1:]
 
-      # Will cause errors if num_virtual_batches does not divide the batch size
+      # Will cause errors if virtual_batch_size does not divide the batch size
       inputs = array_ops.reshape(inputs, expanded_shape)
 
-      ndims = len(expanded_shape)
-      if self.axis < 0:
-        axis = ndims + self.axis
-      else:
-        axis = self.axis + 1      # Account for the added dimension
-
-      # Permute the num_virtual_batch dimension (dim 1) to be adjacent to axis
-      # TODO(b/66257056): when multi-axis batch normalization is implemented,
-      # this permutation trick and the combined_dim reshape are no longer
-      # necessary and can be reworked to simply use broadcasting.
-      permutation = ([0] + list(range(2, axis)) + [1, axis] +
-                     list(range(axis + 1, ndims)))
-      inverse_permutation = [x[1] for x in
-                             sorted(zip(permutation, range(ndims)))]
-      inputs = array_ops.transpose(inputs, perm=permutation)
-
-      # Combine the axis and num_virtual_batch dimension in order to take
-      # advantage of fused batch normalization
-      combined_dim = expanded_shape[1] * expanded_shape[axis]
-      perm_shape = [-1] + inputs.shape.as_list()[1:]
-      combined_shape = (perm_shape[:axis - 1] +
-                        [combined_dim] +
-                        perm_shape[axis + 1:])
-      inputs = array_ops.reshape(inputs, combined_shape)
-      # After the above reshape, the batch norm axis is the original self.axis
-
-      # Undoes the reshaping and transposing tricks done above
       def undo_virtual_batching(outputs):
-        outputs = array_ops.reshape(outputs, perm_shape)
-        outputs = array_ops.transpose(outputs, perm=inverse_permutation)
         outputs = array_ops.reshape(outputs, original_shape)
         return outputs
 
     if self.fused:
       outputs = self._fused_batch_norm(inputs, training=training)
-      if self.num_virtual_batches > 1:
+      if self.virtual_batch_size is not None:
+        # Currently never reaches here since fused_batch_norm does not support
+        # virtual batching
         return undo_virtual_batching(outputs)
       return outputs
 
-    # First, compute the axes along which to reduce the mean / variance,
-    # as well as the broadcast shape to be used for all parameters.
+    # Compute the axes along which to reduce the mean / variance
     input_shape = inputs.get_shape()
-    ndim = len(input_shape)
-    reduction_axes = list(range(len(input_shape)))
-    del reduction_axes[self.axis]
-    broadcast_shape = [1] * len(input_shape)
-    broadcast_shape[self.axis] = input_shape[self.axis].value
-
-    # Determines whether broadcasting is needed.
-    needs_broadcasting = (sorted(reduction_axes) != list(range(ndim))[:-1])
+    reduction_axes = [i for i in range(len(input_shape)) if i not in self.axis]
+    if self.virtual_batch_size is not None:
+      del reduction_axes[1]     # Do not reduce along virtual batch dim
 
     scale, offset = self.gamma, self.beta
 
@@ -473,13 +491,18 @@ class BatchNormalization(base.Layer):
     if training_value is not False:
       # Some of the computations here are not necessary when training==False
       # but not a constant. However, this makes the code simpler.
-      mean, variance = nn.moments(inputs, reduction_axes)
-      mean = _smart_select(training,
-                           lambda: mean,
-                           lambda: self.moving_mean)
-      variance = _smart_select(training,
-                               lambda: variance,
-                               lambda: self.moving_variance)
+      keep_dims = self.virtual_batch_size is not None or len(self.axis) > 1
+      mean, variance = nn.moments(inputs, reduction_axes, keep_dims=keep_dims)
+
+      moving_mean = self.moving_mean
+      moving_variance = self.moving_variance
+
+      mean = utils.smart_cond(training,
+                              lambda: mean,
+                              lambda: moving_mean)
+      variance = utils.smart_cond(training,
+                                  lambda: variance,
+                                  lambda: moving_variance)
 
       if self.renorm:
         r, d, new_mean, new_variance = self._renorm_correction_and_moments(
@@ -498,7 +521,18 @@ class BatchNormalization(base.Layer):
         new_mean, new_variance = mean, variance
 
       # Update moving averages when training, and prevent updates otherwise.
-      decay = _smart_select(training, lambda: self.momentum, lambda: 1.)
+      decay = utils.smart_cond(training, lambda: self.momentum, lambda: 1.)
+      if self.virtual_batch_size is not None:
+        # This isn't strictly correct since in ghost batch norm, you are
+        # supposed to sequentially update the moving_mean and moving_variance
+        # with each sub-batch. However, since the moving statistics are only
+        # used during evaluation, it is more efficient to just update in one
+        # step and should not make a significant difference in the result.
+        new_mean = math_ops.reduce_mean(new_mean,
+                                        axis=1, keep_dims=True)
+        new_variance = math_ops.reduce_mean(new_variance,
+                                            axis=1, keep_dims=True)
+
       mean_update = moving_averages.assign_moving_average(
           self.moving_mean, new_mean, decay, zero_debias=False)
       variance_update = moving_averages.assign_moving_average(
@@ -510,9 +544,13 @@ class BatchNormalization(base.Layer):
     else:
       mean, variance = self.moving_mean, self.moving_variance
 
+    # Broadcasting only necessary for single-axis batch norm
+    broadcast_shape = [1] * len(input_shape)
+    broadcast_shape[self.axis[0]] = input_shape[self.axis[0]].value
+    rank = len(inputs.get_shape())
     def _broadcast(v):
-      if needs_broadcasting and v is not None:
-        # In this case we must explicitly broadcast all parameters.
+      if v is not None and len(v.get_shape()) != rank:
+        assert len(self.axis) == 1
         return array_ops.reshape(v, broadcast_shape)
       return v
 
@@ -523,7 +561,7 @@ class BatchNormalization(base.Layer):
                                      _broadcast(scale),
                                      self.epsilon)
 
-    if self.num_virtual_batches > 1:
+    if self.virtual_batch_size is not None:
       return undo_virtual_batching(outputs)
 
     return outputs
@@ -551,7 +589,7 @@ def batch_normalization(inputs,
                         renorm_clipping=None,
                         renorm_momentum=0.99,
                         fused=None,
-                        num_virtual_batches=1):
+                        virtual_batch_size=None):
   """Functional interface for the batch normalization layer.
 
   Reference: http://arxiv.org/abs/1502.03167
@@ -623,11 +661,12 @@ def batch_normalization(inputs,
       to get the means and variances for inference.
     fused: if `True`, use a faster, fused implementation if possible.
       If `None`, use the system recommended implementation.
-    num_virtual_batches: An `int`, specifies the number of virtual batches to
-      operate over. If greater than 1, will perform "ghost batch
-      normalization", which creates virtual sub-batches to operate over for
-      batch norm. Default is 1 virtual batch, in which no virtual batching is
-      performed. Must divide the actual batch size during graph execution.
+    virtual_batch_size: An `int`. By default, `virtual_batch_size` is `None`,
+      which means batch normalization is performed across the whole batch. When
+      `virtual_batch_size` is not `None`, instead perform "Ghost Batch
+      Normalization", which creates virtual sub-batches which are each
+      normalized separately (with shared gamma, beta, and moving statistics).
+      Must divide the actual batch size during execution.
 
   Returns:
     Output tensor.
@@ -651,7 +690,7 @@ def batch_normalization(inputs,
       renorm_momentum=renorm_momentum,
       fused=fused,
       trainable=trainable,
-      num_virtual_batches=num_virtual_batches,
+      virtual_batch_size=virtual_batch_size,
       name=name,
       _reuse=reuse,
       _scope=name)
@@ -663,37 +702,3 @@ def batch_normalization(inputs,
 BatchNorm = BatchNormalization
 batch_norm = batch_normalization
 
-# Helper function
-
-
-def _smart_select(pred, fn_then, fn_else):
-  """Selects fn_then() or fn_else() based on the value of pred.
-
-  The purpose of this function is the same as `utils.smart_cond`. However, at
-  the moment there is a bug (b/36297356) that seems to kick in only when
-  `smart_cond` delegates to `tf.cond`, which sometimes results in the training
-  hanging when using parameter servers. This function will output the result
-  of `fn_then` or `fn_else` if `pred` is known at graph construction time.
-  Otherwise, it will use `tf.where` which will result in some redundant work
-  (both branches will be computed but only one selected). However, the tensors
-  involved will usually be small (means and variances in batchnorm), so the
-  cost will be small and will not be incurred at all if `pred` is a constant.
-
-  Args:
-    pred: A boolean scalar `Tensor`.
-    fn_then: A callable to use when pred==True.
-    fn_else: A callable to use when pred==False.
-
-  Returns:
-    A `Tensor` whose value is fn_then() or fn_else() based on the value of pred.
-  """
-  pred_value = utils.constant_value(pred)
-  if pred_value:
-    return fn_then()
-  elif pred_value is False:
-    return fn_else()
-  t_then = array_ops.expand_dims(fn_then(), 0)
-  t_else = array_ops.expand_dims(fn_else(), 0)
-  pred = array_ops.reshape(pred, [1])
-  result = array_ops.where(pred, t_then, t_else)
-  return array_ops.squeeze(result, [0])
diff --git a/tensorflow/python/layers/normalization_test.py b/tensorflow/python/layers/normalization_test.py
index ccb0662c4e..f8d9d2948c 100644
--- a/tensorflow/python/layers/normalization_test.py
+++ b/tensorflow/python/layers/normalization_test.py
@@ -823,12 +823,20 @@ class BNTest(test.TestCase):
         self.assertAllClose(y_train, yt_val_train, atol=1e-5)
         self.assertAllClose(y_test, yt_val_test, atol=1e-5)
 
-  def testGhostBNVirtualBatch1(self):
+  def testGhostBNNegativeVirtualBatch(self):
+    shape = [6, 5, 4, 3]
+    inp = random_ops.random_uniform(shape, seed=1)
+
+    with self.assertRaises(ValueError):
+      normalization_layers.batch_normalization(
+          inp, virtual_batch_size=-1)
+
+  def testGhostBNVirtualBatchFull(self):
     shape = [6, 5, 4, 3]
     inp = random_ops.random_uniform(shape, seed=1)
     out1 = normalization_layers.batch_normalization(inp)
     out2 = normalization_layers.batch_normalization(
-        inp, num_virtual_batches=1)
+        inp, virtual_batch_size=6)
 
     self.assertListEqual(
         out1.shape.as_list(), out2.shape.as_list())
@@ -841,19 +849,11 @@ class BNTest(test.TestCase):
 
       self.assertAllClose(y1, y2, atol=1e-5)
 
-  def testGhostBNNegativeVirtualBatch(self):
-    shape = [6, 5, 4, 3]
-    inp = random_ops.random_uniform(shape, seed=1)
-
-    with self.assertRaises(ValueError):
-      normalization_layers.batch_normalization(
-          inp, num_virtual_batches=-1)
-
   def testGhostBNInputOutputShapesMatch(self):
     shape = [6, 4, 3]
     inp = random_ops.random_uniform(shape, seed=1)
     out = normalization_layers.batch_normalization(
-        inp, num_virtual_batches=2)
+        inp, virtual_batch_size=3)
     self.assertListEqual(out.shape.as_list(), shape)
 
   def testGhostBNUnknownBatchSize(self):
@@ -861,7 +861,7 @@ class BNTest(test.TestCase):
     tf_shape = [None, 5, 4]
     inp = array_ops.placeholder(dtypes.float32, tf_shape)
     out = normalization_layers.batch_normalization(
-        inp, num_virtual_batches=5)
+        inp, virtual_batch_size=2)
 
     with self.test_session(use_gpu=True) as sess:
       sess.run(variables.global_variables_initializer())
@@ -873,7 +873,7 @@ class BNTest(test.TestCase):
 
   def testGhostBN2Dims(self):
     shape = [6, 2]
-    num_virtual_batches = 2
+    virtual_batch_size = 3
     beta = 2.
     gamma = 3.
     momentum = 0.8
@@ -888,10 +888,11 @@ class BNTest(test.TestCase):
         epsilon=epsilon,
         beta_initializer=init_ops.constant_initializer(beta),
         gamma_initializer=init_ops.constant_initializer(gamma),
-        num_virtual_batches=num_virtual_batches)
+        virtual_batch_size=virtual_batch_size)
     out = bn.apply(inp, training=is_training)
-    ghost_shape = ([shape[0] // num_virtual_batches,
-                    num_virtual_batches, shape[1]])
+    ghost_shape = ([virtual_batch_size,
+                    shape[0] // virtual_batch_size,
+                    shape[1]])
 
     with self.test_session(use_gpu=True) as sess:
       sess.run(variables.global_variables_initializer())
@@ -899,10 +900,14 @@ class BNTest(test.TestCase):
         x = np.random.random(shape)
 
         sub_batched = np.reshape(x, ghost_shape)
-        means = np.mean(sub_batched, axis=0)
-        variances = np.var(sub_batched, axis=0)
-        moving_means = moving_means * momentum + means * (1. - momentum)
-        moving_vars = moving_vars * momentum + variances * (1. - momentum)
+        means = np.mean(sub_batched, axis=0, keepdims=True)
+        variances = np.var(sub_batched, axis=0, keepdims=True)
+
+        avg_means = np.mean(means, axis=1, keepdims=True)
+        avg_variances = np.mean(variances, axis=1, keepdims=True)
+
+        moving_means = moving_means * momentum + avg_means * (1. - momentum)
+        moving_vars = moving_vars * momentum + avg_variances * (1. - momentum)
 
         y_train = ((sub_batched - means) /
                    (variances + epsilon) ** 0.5 * gamma) + beta
@@ -921,13 +926,13 @@ class BNTest(test.TestCase):
 
   def testGhostBN4DimsAxis3(self):
     shape = [6, 10, 10, 3]
-    num_virtual_batches = 3
+    virtual_batch_size = 2
     beta = 2.
     gamma = 3.
     momentum = 0.8
     epsilon = 1e-3
-    moving_means = np.zeros([1, 3, 1, 1, 3], dtype=np.float32)
-    moving_vars = np.ones([1, 3, 1, 1, 3], dtype=np.float32)
+    moving_means = np.zeros([1, 1, 1, 1, 3], dtype=np.float32)
+    moving_vars = np.ones([1, 1, 1, 1, 3], dtype=np.float32)
 
     inp = array_ops.placeholder(dtypes.float32, shape)
     is_training = array_ops.placeholder(dtypes.bool)
@@ -937,9 +942,9 @@ class BNTest(test.TestCase):
         epsilon=epsilon,
         beta_initializer=init_ops.constant_initializer(beta),
         gamma_initializer=init_ops.constant_initializer(gamma),
-        num_virtual_batches=num_virtual_batches)
+        virtual_batch_size=virtual_batch_size)
     out = bn.apply(inp, training=is_training)
-    ghost_shape = ([shape[0] // num_virtual_batches, num_virtual_batches] +
+    ghost_shape = ([virtual_batch_size, shape[0] // virtual_batch_size] +
                    shape[1:])
 
     with self.test_session(use_gpu=True) as sess:
@@ -950,8 +955,12 @@ class BNTest(test.TestCase):
         sub_batched = np.reshape(x, ghost_shape)
         means = np.mean(sub_batched, axis=(0, 2, 3), keepdims=True)
         variances = np.var(sub_batched, axis=(0, 2, 3), keepdims=True)
-        moving_means = moving_means * momentum + means * (1. - momentum)
-        moving_vars = moving_vars * momentum + variances * (1. - momentum)
+
+        avg_means = np.mean(means, axis=1, keepdims=True)
+        avg_variances = np.mean(variances, axis=1, keepdims=True)
+
+        moving_means = moving_means * momentum + avg_means * (1. - momentum)
+        moving_vars = moving_vars * momentum + avg_variances * (1. - momentum)
 
         y_train = ((sub_batched - means) /
                    (variances + epsilon) ** 0.5 * gamma) + beta
@@ -970,13 +979,13 @@ class BNTest(test.TestCase):
 
   def testGhostBN4DimsAxis1(self):
     shape = [6, 3, 10, 10]
-    num_virtual_batches = 3
+    virtual_batch_size = 2
     beta = 2.
     gamma = 3.
     momentum = 0.8
     epsilon = 1e-3
-    moving_means = np.zeros([1, 3, 3, 1, 1], dtype=np.float32)
-    moving_vars = np.ones([1, 3, 3, 1, 1], dtype=np.float32)
+    moving_means = np.zeros([1, 1, 3, 1, 1], dtype=np.float32)
+    moving_vars = np.ones([1, 1, 3, 1, 1], dtype=np.float32)
 
     inp = array_ops.placeholder(dtypes.float32, shape)
     is_training = array_ops.placeholder(dtypes.bool)
@@ -986,10 +995,160 @@ class BNTest(test.TestCase):
         epsilon=epsilon,
         beta_initializer=init_ops.constant_initializer(beta),
         gamma_initializer=init_ops.constant_initializer(gamma),
-        num_virtual_batches=num_virtual_batches,
+        virtual_batch_size=virtual_batch_size,
         fused=False)      # NCHW is unsupported by CPU fused batch norm
     out = bn.apply(inp, training=is_training)
-    ghost_shape = ([shape[0] // num_virtual_batches, num_virtual_batches] +
+    ghost_shape = ([virtual_batch_size, shape[0] // virtual_batch_size] +
+                   shape[1:])
+
+    with self.test_session(use_gpu=True) as sess:
+      sess.run(variables.global_variables_initializer())
+      for _ in range(5):
+        x = np.random.random(shape)
+
+        sub_batched = np.reshape(x, ghost_shape)
+        means = np.mean(sub_batched, axis=(0, 3, 4), keepdims=True)
+        variances = np.var(sub_batched, axis=(0, 3, 4), keepdims=True)
+
+        avg_means = np.mean(means, axis=1, keepdims=True)
+        avg_variances = np.mean(variances, axis=1, keepdims=True)
+
+        moving_means = moving_means * momentum + avg_means * (1. - momentum)
+        moving_vars = moving_vars * momentum + avg_variances * (1. - momentum)
+
+        y_train = ((sub_batched - means) /
+                   (variances + epsilon) ** 0.5 * gamma) + beta
+        y_test = ((sub_batched - moving_means) /
+                  (moving_vars + epsilon) ** 0.5 * gamma) + beta
+
+        y_train = np.reshape(y_train, shape)
+        y_test = np.reshape(y_test, shape)
+
+        y_val_train, _, _ = sess.run([out] + bn.updates,
+                                     feed_dict={inp: x, is_training: True})
+        y_val_test = sess.run(out, feed_dict={inp: x, is_training: False})
+
+        self.assertAllClose(y_train, y_val_train, atol=1e-2)
+        self.assertAllClose(y_test, y_val_test, atol=1e-2)
+
+  def testMultiAxisInvalid(self):
+    shape = [6, 5, 4, 3]
+    inp = random_ops.random_uniform(shape, seed=1)
+
+    with self.assertRaises(ValueError):
+      normalization_layers.batch_normalization(
+          inp, axis=[1, 4])    # out of bounds
+
+    with self.assertRaises(ValueError):
+      normalization_layers.batch_normalization(
+          inp, axis=[-5, 1])   # out of bounds
+
+    with self.assertRaises(ValueError):
+      normalization_layers.batch_normalization(
+          inp, axis=[1, 2, 1])   # duplicate
+
+  def test3DInputMultiAxis12(self):
+    epsilon = 1e-3
+    bn = normalization_layers.BatchNormalization(
+        axis=[1, 2], epsilon=epsilon, momentum=0.9)
+    inputs = variables.Variable(
+        np.random.random((5, 4, 3)) + 100, dtype=dtypes.float32)
+    training = array_ops.placeholder(dtype='bool')
+    outputs = bn.apply(inputs, training=training)
+
+    with self.test_session() as sess:
+      # Test training with placeholder learning phase.
+      sess.run(variables.global_variables_initializer())
+
+      np_gamma, np_beta = sess.run([bn.gamma, bn.beta])
+
+      for _ in range(100):
+        np_output, _, _ = sess.run([outputs] + bn.updates,
+                                   feed_dict={training: True})
+        # Verify that the axis is normalized during training.
+        normed_np_output = ((np_output - epsilon) * np_gamma) + np_beta
+        self.assertAlmostEqual(np.mean(normed_np_output), 0., places=1)
+        self.assertAlmostEqual(np.std(normed_np_output), 1., places=1)
+
+      # Verify that the statistics are updated during training.
+      moving_mean, moving_var = sess.run([bn.moving_mean, bn.moving_variance])
+      np_inputs = sess.run(inputs)
+      mean = np.mean(np_inputs, axis=0, keepdims=True)
+      std = np.std(np_inputs, axis=0, keepdims=True)
+      variance = np.square(std)
+      self.assertAllClose(mean, moving_mean, atol=1e-2)
+      self.assertAllClose(variance, moving_var, atol=1e-2)
+
+      # Test inference with placeholder learning phase.
+      np_output = sess.run(outputs, feed_dict={training: False})
+
+      # Verify that the axis is normalized during inference.
+      normed_np_output = ((np_output - epsilon) * np_gamma) + np_beta
+      self.assertAlmostEqual(np.mean(normed_np_output), 0., places=1)
+      self.assertAlmostEqual(np.std(normed_np_output), 1., places=1)
+
+  def test5DInputMultiAxis123(self):
+    epsilon = 1e-3
+    bn = normalization_layers.BatchNormalization(
+        axis=[1, 2, 3], epsilon=epsilon, momentum=0.9)
+    inputs = variables.Variable(
+        np.random.random((5, 3, 4, 4, 3)) + 100, dtype=dtypes.float32)
+    training = array_ops.placeholder(dtype='bool')
+    outputs = bn.apply(inputs, training=training)
+
+    with self.test_session() as sess:
+      # Test training with placeholder learning phase.
+      sess.run(variables.global_variables_initializer())
+
+      np_gamma, np_beta = sess.run([bn.gamma, bn.beta])
+
+      for _ in range(100):
+        np_output, _, _ = sess.run([outputs] + bn.updates,
+                                   feed_dict={training: True})
+        # Verify that the axis is normalized during training.
+        normed_np_output = ((np_output - epsilon) * np_gamma) + np_beta
+        self.assertAlmostEqual(np.mean(normed_np_output), 0., places=1)
+        self.assertAlmostEqual(np.std(normed_np_output), 1., places=1)
+
+      # Verify that the statistics are updated during training.
+      moving_mean, moving_var = sess.run([bn.moving_mean, bn.moving_variance])
+      np_inputs = sess.run(inputs)
+      mean = np.mean(np_inputs, axis=(0, 4), keepdims=True)
+      std = np.std(np_inputs, axis=(0, 4), keepdims=True)
+      variance = np.square(std)
+      self.assertAllClose(mean, moving_mean, atol=1e-2)
+      self.assertAllClose(variance, moving_var, atol=1e-2)
+
+      # Test inference with placeholder learning phase.
+      np_output = sess.run(outputs, feed_dict={training: False})
+
+      # Verify that the axis is normalized during inference.
+      normed_np_output = ((np_output - epsilon) * np_gamma) + np_beta
+      self.assertAlmostEqual(np.mean(normed_np_output), 0., places=1)
+      self.assertAlmostEqual(np.std(normed_np_output), 1., places=1)
+
+  def testGhostBN5DimsMultiAxis14(self):
+    shape = [6, 3, 10, 10, 4]
+    virtual_batch_size = 3
+    beta = 2.
+    gamma = 3.
+    momentum = 0.8
+    epsilon = 1e-3
+    moving_means = np.zeros([1, 1, 3, 1, 1, 4], dtype=np.float32)
+    moving_vars = np.ones([1, 1, 3, 1, 1, 4], dtype=np.float32)
+
+    inp = array_ops.placeholder(dtypes.float32, shape)
+    is_training = array_ops.placeholder(dtypes.bool)
+    bn = normalization_layers.BatchNormalization(
+        axis=[1, 4],
+        momentum=momentum,
+        epsilon=epsilon,
+        beta_initializer=init_ops.constant_initializer(beta),
+        gamma_initializer=init_ops.constant_initializer(gamma),
+        virtual_batch_size=virtual_batch_size,
+        fused=False)
+    out = bn.apply(inp, training=is_training)
+    ghost_shape = ([virtual_batch_size, shape[0] // virtual_batch_size] +
                    shape[1:])
 
     with self.test_session(use_gpu=True) as sess:
@@ -1000,8 +1159,12 @@ class BNTest(test.TestCase):
         sub_batched = np.reshape(x, ghost_shape)
         means = np.mean(sub_batched, axis=(0, 3, 4), keepdims=True)
         variances = np.var(sub_batched, axis=(0, 3, 4), keepdims=True)
-        moving_means = moving_means * momentum + means * (1. - momentum)
-        moving_vars = moving_vars * momentum + variances * (1. - momentum)
+
+        avg_means = np.mean(means, axis=1, keepdims=True)
+        avg_variances = np.mean(variances, axis=1, keepdims=True)
+
+        moving_means = moving_means * momentum + avg_means * (1. - momentum)
+        moving_vars = moving_vars * momentum + avg_variances * (1. - momentum)
 
         y_train = ((sub_batched - means) /
                    (variances + epsilon) ** 0.5 * gamma) + beta
diff --git a/tensorflow/tools/api/golden/tensorflow.layers.-batch-normalization.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.-batch-normalization.pbtxt
index 1faa22f09b..c66af13850 100644
--- a/tensorflow/tools/api/golden/tensorflow.layers.-batch-normalization.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.layers.-batch-normalization.pbtxt
@@ -81,7 +81,7 @@ tf_class {
   }
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'axis\', \'momentum\', \'epsilon\', \'center\', \'scale\', \'beta_initializer\', \'gamma_initializer\', \'moving_mean_initializer\', \'moving_variance_initializer\', \'beta_regularizer\', \'gamma_regularizer\', \'beta_constraint\', \'gamma_constraint\', \'renorm\', \'renorm_clipping\', \'renorm_momentum\', \'fused\', \'trainable\', \'num_virtual_batches\', \'name\'], varargs=None, keywords=kwargs, defaults=[\'-1\', \'0.99\', \'0.001\', \'True\', \'True\', \'<tensorflow.python.ops.init_ops.Zeros object instance>\', \'<tensorflow.python.ops.init_ops.Ones object instance>\', \'<tensorflow.python.ops.init_ops.Zeros object instance>\', \'<tensorflow.python.ops.init_ops.Ones object instance>\', \'None\', \'None\', \'None\', \'None\', \'False\', \'None\', \'0.99\', \'None\', \'True\', \'1\', \'None\'], "
+    argspec: "args=[\'self\', \'axis\', \'momentum\', \'epsilon\', \'center\', \'scale\', \'beta_initializer\', \'gamma_initializer\', \'moving_mean_initializer\', \'moving_variance_initializer\', \'beta_regularizer\', \'gamma_regularizer\', \'beta_constraint\', \'gamma_constraint\', \'renorm\', \'renorm_clipping\', \'renorm_momentum\', \'fused\', \'trainable\', \'virtual_batch_size\', \'name\'], varargs=None, keywords=kwargs, defaults=[\'-1\', \'0.99\', \'0.001\', \'True\', \'True\', \'<tensorflow.python.ops.init_ops.Zeros object instance>\', \'<tensorflow.python.ops.init_ops.Ones object instance>\', \'<tensorflow.python.ops.init_ops.Zeros object instance>\', \'<tensorflow.python.ops.init_ops.Ones object instance>\', \'None\', \'None\', \'None\', \'None\', \'False\', \'None\', \'0.99\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_loss"
diff --git a/tensorflow/tools/api/golden/tensorflow.layers.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.pbtxt
index a252765bb1..dad514b534 100644
--- a/tensorflow/tools/api/golden/tensorflow.layers.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.layers.pbtxt
@@ -90,7 +90,7 @@ tf_module {
   }
   member_method {
     name: "batch_normalization"
-    argspec: "args=[\'inputs\', \'axis\', \'momentum\', \'epsilon\', \'center\', \'scale\', \'beta_initializer\', \'gamma_initializer\', \'moving_mean_initializer\', \'moving_variance_initializer\', \'beta_regularizer\', \'gamma_regularizer\', \'beta_constraint\', \'gamma_constraint\', \'training\', \'trainable\', \'name\', \'reuse\', \'renorm\', \'renorm_clipping\', \'renorm_momentum\', \'fused\', \'num_virtual_batches\'], varargs=None, keywords=None, defaults=[\'-1\', \'0.99\', \'0.001\', \'True\', \'True\', \'<tensorflow.python.ops.init_ops.Zeros object instance>\', \'<tensorflow.python.ops.init_ops.Ones object instance>\', \'<tensorflow.python.ops.init_ops.Zeros object instance>\', \'<tensorflow.python.ops.init_ops.Ones object instance>\', \'None\', \'None\', \'None\', \'None\', \'False\', \'True\', \'None\', \'None\', \'False\', \'None\', \'0.99\', \'None\', \'1\'], "
+    argspec: "args=[\'inputs\', \'axis\', \'momentum\', \'epsilon\', \'center\', \'scale\', \'beta_initializer\', \'gamma_initializer\', \'moving_mean_initializer\', \'moving_variance_initializer\', \'beta_regularizer\', \'gamma_regularizer\', \'beta_constraint\', \'gamma_constraint\', \'training\', \'trainable\', \'name\', \'reuse\', \'renorm\', \'renorm_clipping\', \'renorm_momentum\', \'fused\', \'virtual_batch_size\'], varargs=None, keywords=None, defaults=[\'-1\', \'0.99\', \'0.001\', \'True\', \'True\', \'<tensorflow.python.ops.init_ops.Zeros object instance>\', \'<tensorflow.python.ops.init_ops.Ones object instance>\', \'<tensorflow.python.ops.init_ops.Zeros object instance>\', \'<tensorflow.python.ops.init_ops.Ones object instance>\', \'None\', \'None\', \'None\', \'None\', \'False\', \'True\', \'None\', \'None\', \'False\', \'None\', \'0.99\', \'None\', \'None\'], "
   }
   member_method {
     name: "conv1d"
-- 
GitLab


From 87c59cd4e92c688e32f4ab9502e1a9e269ab43c8 Mon Sep 17 00:00:00 2001
From: Toby Boyd <tobyboyd@google.com>
Date: Thu, 12 Oct 2017 14:33:22 -0700
Subject: [PATCH 316/909] Internal change.

PiperOrigin-RevId: 172013289
---
 tensorflow/core/util/test_log.proto | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tensorflow/core/util/test_log.proto b/tensorflow/core/util/test_log.proto
index 409d5db211..a5476382f2 100644
--- a/tensorflow/core/util/test_log.proto
+++ b/tensorflow/core/util/test_log.proto
@@ -57,12 +57,15 @@ message BuildConfiguration {
 
 message CommitId {
   oneof kind {
+    // Submitted changelist.
     int64 changelist = 1;
     string hash = 2;
   }
   // Hash of intermediate change between hash/changelist and what was tested.
   // Not used if the build is from a commit without modifications.
   string snapshot = 3;
+  // Changelist tested if the change list is not already submitted.
+  int64 pending_changelist = 4;
 };
 
 message CPUInfo {
-- 
GitLab


From 8c57896d0c4d376687400afc0aefce2978312592 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 12 Oct 2017 14:35:52 -0700
Subject: [PATCH 317/909] Update ops-related pbtxt files.

PiperOrigin-RevId: 172013619
---
 .../core/ops/compat/ops_history.v1.pbtxt      | 44 +++++++++++++++++
 tensorflow/core/ops/ops.pbtxt                 | 48 +++++++++++++++++++
 2 files changed, 92 insertions(+)

diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index 2015acb1c4..d93a4ff933 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -15594,6 +15594,50 @@ op {
   }
   is_stateful: true
 }
+op {
+  name: "MapAndBatchDataset"
+  input_arg {
+    name: "input_dataset"
+    type: DT_VARIANT
+  }
+  input_arg {
+    name: "other_arguments"
+    type_list_attr: "Targuments"
+  }
+  input_arg {
+    name: "batch_size"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "num_parallel_batches"
+    type: DT_INT64
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "f"
+    type: "func"
+  }
+  attr {
+    name: "Targuments"
+    type: "list(type)"
+    has_minimum: true
+  }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
+  }
+}
 op {
   name: "MapClear"
   attr {
diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index 15de2d2155..6403dcf78c 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -12329,6 +12329,54 @@ op {
   description: "This operation may be executed multiple times. Each execution will reset the\niterator in `iterator` to the first element of `dataset`."
   is_stateful: true
 }
+op {
+  name: "MapAndBatchDataset"
+  input_arg {
+    name: "input_dataset"
+    type: DT_VARIANT
+  }
+  input_arg {
+    name: "other_arguments"
+    type_list_attr: "Targuments"
+  }
+  input_arg {
+    name: "batch_size"
+    description: "A scalar representing the number of elements to accumulate in a\nbatch. It determines the number of concurrent invocations of `f` that process\nelements from `input_dataset` in parallel."
+    type: DT_INT64
+  }
+  input_arg {
+    name: "num_parallel_batches"
+    description: "A scalar representing the number of batches to create in\nparallel. Processing multiple batches in parallel benefits workloads prone to\nstragglers."
+    type: DT_INT64
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "f"
+    type: "func"
+  }
+  attr {
+    name: "Targuments"
+    type: "list(type)"
+    has_minimum: true
+  }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
+  }
+  summary: "Creates a dataset that applies `f` to the outputs of `input_dataset` and then"
+  description: "batches `batch_size` of them.\n\nUnlike a \"MapDataset\", which applies `f` sequentially, this dataset invokes up\nto `batch_size * num_parallel_batches` copies of `f` in parallel."
+}
 op {
   name: "MapClear"
   attr {
-- 
GitLab


From 1885344c6a87866e2accb65dd2462d6815419f13 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 12 Oct 2017 14:36:09 -0700
Subject: [PATCH 318/909] Explictly raise an exception if sharding fails
 because the batch size is unknown.

Right now, TPUEstimator will raise an error message complaining that sharding
failed because (None % integer value) is not a well-defined operation. I hope
the new error message will make it easier for people to figure out what's going
on.

PiperOrigin-RevId: 172013663
---
 tensorflow/contrib/tpu/python/tpu/tpu_sharding.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_sharding.py b/tensorflow/contrib/tpu/python/tpu/tpu_sharding.py
index d545a94ca6..f8ba7d45e2 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_sharding.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_sharding.py
@@ -177,6 +177,10 @@ class ShardingPolicy(object):
       raise ValueError("shape %s does not contain shard_dimension %d" %
                        (shape.as_list(), self._shard_dimension))
     dims = shape.as_list()
+    if dims[self._shard_dimension] is None:
+      raise ValueError("shape %s must have a fixed size for dimension %d "
+                       "that is known at graph construction time." %
+                       (shape.as_list(), self._shard_dimension))
     if (dims[self._shard_dimension] % self._number_of_shards) != 0:
       raise ValueError("shape %s cannot be sharded %d ways along dimension %d" %
                        (shape.as_list(), self._number_of_shards,
-- 
GitLab


From f7b322934bbd697fa5488cf1de59d40d654c5e7a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 12 Oct 2017 14:42:04 -0700
Subject: [PATCH 319/909] Use canonical method for not changing the scope, but
 changing scope options.

PiperOrigin-RevId: 172014544
---
 tensorflow/contrib/rnn/python/kernel_tests/lstm_ops_test.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/rnn/python/kernel_tests/lstm_ops_test.py b/tensorflow/contrib/rnn/python/kernel_tests/lstm_ops_test.py
index 1980d64cd6..0a474f7831 100644
--- a/tensorflow/contrib/rnn/python/kernel_tests/lstm_ops_test.py
+++ b/tensorflow/contrib/rnn/python/kernel_tests/lstm_ops_test.py
@@ -398,7 +398,8 @@ class LSTMBlockCellTest(test.TestCase):
             dtype=dtypes.float32,
             initializer=init_ops.zeros_initializer())
 
-      with variable_scope.variable_scope("", reuse=True):
+      with variable_scope.variable_scope(
+          variable_scope.get_variable_scope(), reuse=True):
         cell = lstm_ops.LSTMBlockFusedCell(
             cell_size, cell_clip=0, use_peephole=False)
 
@@ -414,7 +415,8 @@ class LSTMBlockCellTest(test.TestCase):
       # tiny (single-time) subsequences, i.e. unfuse the cell
       unfused_outputs_op = []
       state = None
-      with variable_scope.variable_scope("", reuse=True):
+      with variable_scope.variable_scope(
+          variable_scope.get_variable_scope(), reuse=True):
         for i, inp in enumerate(inputs):
           lengths = [int(i < l) for l in seq_lengths.eval()]
           output, state = cell(
-- 
GitLab


From 7c2993a2358688f2a4e314ee8c15c40ebc4ce9f1 Mon Sep 17 00:00:00 2001
From: Jacques Pienaar <jpienaar@google.com>
Date: Thu, 12 Oct 2017 15:05:29 -0700
Subject: [PATCH 320/909] [XLA] Return status instead of TF_RET_CHECK.

TF_RET_CHECK logs the call stack and HloEvaluator is often called with
parameters and non-constant ops which results in a logging output in cases where the caller expects it could fail.

PiperOrigin-RevId: 172018263
---
 tensorflow/compiler/xla/service/hlo_evaluator.cc | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_evaluator.cc b/tensorflow/compiler/xla/service/hlo_evaluator.cc
index b28f9b59ab..20dba60f4e 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator.cc
+++ b/tensorflow/compiler/xla/service/hlo_evaluator.cc
@@ -1243,8 +1243,14 @@ StatusOr<std::unique_ptr<Literal>> HloEvaluator::Evaluate(
 
 StatusOr<std::unique_ptr<Literal>> HloEvaluator::Evaluate(
     HloInstruction* instruction) {
-  TF_RET_CHECK(hlo_query::AllOperandsAreConstants(*instruction));
-  TF_RET_CHECK(instruction->opcode() != HloOpcode::kParameter);
+  if (instruction->opcode() == HloOpcode::kParameter) {
+    return tensorflow::errors::FailedPrecondition(
+        "Cannot evaluate a parameter.");
+  }
+  if (!hlo_query::AllOperandsAreConstants(*instruction)) {
+    return tensorflow::errors::FailedPrecondition(
+        "Not all operands are constants.");
+  }
   TF_RETURN_IF_ERROR(ShapeUtil::ValidateShape(instruction->shape()));
 
   arg_literals_.clear();
-- 
GitLab


From abb6b8e73cc0c7b566536df52e93a7eb6327b788 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 12 Oct 2017 15:08:11 -0700
Subject: [PATCH 321/909] Add linux_arm64 and linux_armhf config_setting.

PiperOrigin-RevId: 172018709
---
 tensorflow/BUILD                      | 16 ++++++++++++++++
 tensorflow/compiler/aot/tfcompile.bzl |  2 ++
 tensorflow/tensorflow.bzl             |  6 ++++++
 3 files changed, 24 insertions(+)

diff --git a/tensorflow/BUILD b/tensorflow/BUILD
index 64758dee0e..9d07697d01 100644
--- a/tensorflow/BUILD
+++ b/tensorflow/BUILD
@@ -141,6 +141,22 @@ config_setting(
     visibility = ["//visibility:public"],
 )
 
+config_setting(
+    name = "linux_armhf",
+    values = {
+        "cpu": "armeabi-v7a",
+    },
+    visibility = ["//visibility:public"],
+)
+
+config_setting(
+    name = "linux_arm64",
+    values = {
+        "cpu": "arm64-v8a",
+    },
+    visibility = ["//visibility:public"],
+)
+
 config_setting(
     name = "debug",
     values = {
diff --git a/tensorflow/compiler/aot/tfcompile.bzl b/tensorflow/compiler/aot/tfcompile.bzl
index 4888760acd..c900d201d2 100644
--- a/tensorflow/compiler/aot/tfcompile.bzl
+++ b/tensorflow/compiler/aot/tfcompile.bzl
@@ -302,6 +302,8 @@ def target_llvm_triple():
       "//tensorflow:android_arm": "armv7-none-android",
       "//tensorflow:android_arm64": "aarch64-none-android",
       "//tensorflow:android_x86": "i686-none-android",
+      "//tensorflow:linux_armhf": "armv7-none-linux-gnueabihf",
+      "//tensorflow:linux_arm64": "aarch64-none-linux-gnu",
       "//tensorflow:linux_ppc64le": "ppc64le-ibm-linux-gnu",
       "//tensorflow:darwin": "x86_64-none-darwin",
       "//conditions:default": "x86_64-pc-linux",
diff --git a/tensorflow/tensorflow.bzl b/tensorflow/tensorflow.bzl
index 0f074151db..5ec31e492c 100644
--- a/tensorflow/tensorflow.bzl
+++ b/tensorflow/tensorflow.bzl
@@ -112,6 +112,9 @@ def if_ios(a):
 def if_mobile(a):
   return select({
       clean_dep("//tensorflow:android"): a,
+      # Treat arm linux devices as mobile.
+      clean_dep("//tensorflow:linux_arm64"): a,
+      clean_dep("//tensorflow:linux_armhf"): a,
       clean_dep("//tensorflow:ios"): a,
       "//conditions:default": [],
   })
@@ -120,6 +123,9 @@ def if_mobile(a):
 def if_not_mobile(a):
   return select({
       clean_dep("//tensorflow:android"): [],
+      # Treat arm linux devices as mobile.
+      clean_dep("//tensorflow:linux_arm64"): [],
+      clean_dep("//tensorflow:linux_armhf"): [],
       clean_dep("//tensorflow:ios"): [],
       "//conditions:default": a,
   })
-- 
GitLab


From 9d111d9c5137b28782b487972364b75c81937cb1 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 12 Oct 2017 15:11:02 -0700
Subject: [PATCH 322/909] Require that work_element_count be positive in
 GetCudaLaunchConfig() rather than trying to launch kernels with an invalid
 config and leaving the stream in an error state. The latter is much harder to
 debug.

PiperOrigin-RevId: 172019169
---
 tensorflow/core/util/cuda_kernel_helper.h | 15 ++++-----------
 1 file changed, 4 insertions(+), 11 deletions(-)

diff --git a/tensorflow/core/util/cuda_kernel_helper.h b/tensorflow/core/util/cuda_kernel_helper.h
index 9e76e37898..8315f208e7 100644
--- a/tensorflow/core/util/cuda_kernel_helper.h
+++ b/tensorflow/core/util/cuda_kernel_helper.h
@@ -154,15 +154,11 @@ struct CudaLaunchConfig {
 // Calculate the Cuda launch config we should use for a kernel launch.
 // This is assuming the kernel is quite simple and will largely be
 // memory-limited.
+// REQUIRES: work_element_count > 0.
 inline CudaLaunchConfig GetCudaLaunchConfig(int work_element_count,
                                             const GPUDevice& d) {
+  CHECK_GT(work_element_count, 0);
   CudaLaunchConfig config;
-
-  // in case of invalid input, return the default value config, which has all -1
-  if (work_element_count <= 0) {
-    return config;
-  }
-
   const int virtual_thread_count = work_element_count;
   const int physical_thread_count = std::min(
       d.getNumCudaMultiProcessors() * d.maxCudaThreadsPerMultiProcessor(),
@@ -180,17 +176,14 @@ inline CudaLaunchConfig GetCudaLaunchConfig(int work_element_count,
 
 // Calculate the Cuda launch config we should use for a kernel launch. This
 // variant takes the resource limits of func into account to maximize occupancy.
+// REQUIRES: work_element_count > 0.
 template <typename DeviceFunc>
 inline CudaLaunchConfig GetCudaLaunchConfig(int work_element_count,
                                             const GPUDevice& d, DeviceFunc func,
                                             size_t dynamic_shared_memory_size,
                                             int block_size_limit) {
+  CHECK_GT(work_element_count, 0);
   CudaLaunchConfig config;
-
-  if (work_element_count <= 0) {
-    return config;
-  }
-
   int block_count = 0;
   int thread_per_block = 0;
 
-- 
GitLab


From d204addb96d2fa05128e570c20eae92bb2fc043c Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Thu, 12 Oct 2017 15:40:49 -0700
Subject: [PATCH 323/909] Internal change

PiperOrigin-RevId: 172023756
---
 tensorflow/compiler/tests/BUILD     | 4 ++++
 tensorflow/compiler/xla/tests/BUILD | 2 ++
 tensorflow/contrib/rnn/BUILD        | 3 +++
 3 files changed, 9 insertions(+)

diff --git a/tensorflow/compiler/tests/BUILD b/tensorflow/compiler/tests/BUILD
index d4fe02854a..72a0360de2 100644
--- a/tensorflow/compiler/tests/BUILD
+++ b/tensorflow/compiler/tests/BUILD
@@ -97,6 +97,9 @@ tf_xla_py_test(
     size = "small",
     srcs = ["binary_ops_test.py"],
     shard_count = 5,
+    tags = [
+        "optonly",  # Times out frequently in fastbuild mode.
+    ],
     deps = [
         ":xla_test",
         "//tensorflow/python:array_ops",
@@ -179,6 +182,7 @@ tf_xla_py_test(
         "noasan",
         "nomsan",
         "notsan",
+        "optonly",  # Times out frequently in fastbuild mode.
     ],
     deps = [
         ":xla_test",
diff --git a/tensorflow/compiler/xla/tests/BUILD b/tensorflow/compiler/xla/tests/BUILD
index f37a331a72..769f509adc 100644
--- a/tensorflow/compiler/xla/tests/BUILD
+++ b/tensorflow/compiler/xla/tests/BUILD
@@ -389,6 +389,7 @@ xla_test(
     name = "params_test",
     srcs = ["params_test.cc"],
     shard_count = 30,
+    tags = ["optonly"],
     deps = [
         "//tensorflow/compiler/xla:array2d",
         "//tensorflow/compiler/xla:literal_util",
@@ -1477,6 +1478,7 @@ xla_test(
 xla_test(
     name = "local_client_execute_test",
     srcs = ["local_client_execute_test.cc"],
+    tags = ["optonly"],
     deps = [
         "//tensorflow/compiler/xla:literal_util",
         "//tensorflow/compiler/xla:shape_util",
diff --git a/tensorflow/contrib/rnn/BUILD b/tensorflow/contrib/rnn/BUILD
index 37fe6e0163..6395cd8316 100644
--- a/tensorflow/contrib/rnn/BUILD
+++ b/tensorflow/contrib/rnn/BUILD
@@ -140,6 +140,9 @@ cuda_py_tests(
         "//tensorflow/python:variable_scope",
         "//tensorflow/python:variables",
     ],
+    tags = [
+        "optonly",
+    ],
 )
 
 cuda_py_tests(
-- 
GitLab


From 61026adaef0b2cf4318ce90fff3faeedc6cba7d8 Mon Sep 17 00:00:00 2001
From: Jayaram Bobba <jayaram.bobba@intel.com>
Date: Tue, 3 Oct 2017 13:55:45 -0700
Subject: [PATCH 324/909] Set mkl memory allocation upper bound to the total
 physical memory available on the CPU unless explicitly specified by user

---
 tensorflow/core/BUILD                         | 16 +++++
 .../core/common_runtime/mkl_cpu_allocator.h   | 63 +++++++++++++++----
 .../common_runtime/mkl_cpu_allocator_test.cc  | 55 ++++++++++++++++
 3 files changed, 123 insertions(+), 11 deletions(-)
 create mode 100644 tensorflow/core/common_runtime/mkl_cpu_allocator_test.cc

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 3953575e1b..88a1533b69 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -2668,6 +2668,22 @@ tf_cc_tests(
     ],
 )
 
+tf_cc_test_mkl(
+    name = "mkl_runtime_tests",
+    size = "small",
+    srcs = ["common_runtime/mkl_cpu_allocator_test.cc"],
+    linkstatic = 1,
+    deps = [
+        ":core",
+        ":core_cpu",
+        ":framework",
+        ":framework_internal",
+        ":test",
+        ":test_main",
+        ":testlib",
+    ],
+)
+
 tf_cc_test_mkl(
     name = "mkl_related_tests",
     size = "small",
diff --git a/tensorflow/core/common_runtime/mkl_cpu_allocator.h b/tensorflow/core/common_runtime/mkl_cpu_allocator.h
index f16da10d7a..5951b3b6a1 100644
--- a/tensorflow/core/common_runtime/mkl_cpu_allocator.h
+++ b/tensorflow/core/common_runtime/mkl_cpu_allocator.h
@@ -21,9 +21,13 @@ limitations under the License.
 
 #ifdef INTEL_MKL
 
+#include <unistd.h>
+#include <cstdlib>
 #include <string>
 #include "tensorflow/core/common_runtime/bfc_allocator.h"
 #include "tensorflow/core/framework/allocator.h"
+#include "tensorflow/core/lib/strings/numbers.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/mem.h"
 
 #include "i_malloc.h"
@@ -46,10 +50,50 @@ class MklCPUAllocator : public Allocator {
  public:
   // Constructor and other standard functions
 
-  MklCPUAllocator() {
+  /// Environment variable that user can set to upper bound on memory allocation
+  static constexpr const char kMaxLimitStr[] = "TF_MKL_ALLOC_MAX_BYTES";
+
+  /// Default upper limit on allocator size - 64GB
+  static const size_t kDefaultMaxLimit = 64LL << 30;
+
+  MklCPUAllocator() { TF_CHECK_OK(Initialize()); }
+
+  ~MklCPUAllocator() override { delete allocator_; }
+
+  Status Initialize() {
     VLOG(2) << "MklCPUAllocator: In MklCPUAllocator";
-    allocator_ =
-        new BFCAllocator(new MklSubAllocator, kMaxMemSize, kAllowGrowth, kName);
+
+    // Set upper bound on memory allocation to physical RAM available on the
+    // CPU unless explicitly specified by user
+    uint64 max_mem_bytes = kDefaultMaxLimit;
+#if defined(_SC_PHYS_PAGES) && defined(_SC_PAGESIZE)
+    max_mem_bytes =
+        (uint64)sysconf(_SC_PHYS_PAGES) * (uint64)sysconf(_SC_PAGESIZE);
+#endif
+    char* user_mem_bytes = getenv(kMaxLimitStr);
+
+    if (user_mem_bytes != NULL) {
+      uint64 user_val = 0;
+      if (!strings::safe_strtou64(user_mem_bytes, &user_val)) {
+        return errors::InvalidArgument("Invalid memory limit (", user_mem_bytes,
+                                       ") specified for MKL allocator through ",
+                                       kMaxLimitStr);
+      }
+#if defined(_SC_PHYS_PAGES) && defined(_SC_PAGESIZE)
+      if (user_val > max_mem_bytes) {
+        LOG(WARNING) << "The user specifed a memory limit " << kMaxLimitStr
+                     << "=" << user_val
+                     << " greater than available physical memory: "
+                     << max_mem_bytes
+                     << ". This could significantly reduce performance!";
+      }
+#endif
+      max_mem_bytes = user_val;
+    }
+
+    VLOG(1) << "MklCPUAllocator: Setting max_mem_bytes: " << max_mem_bytes;
+    allocator_ = new BFCAllocator(new MklSubAllocator, max_mem_bytes,
+                                  kAllowGrowth, kName);
 
     // For redirecting all allocations from MKL to this allocator
     // From: http://software.intel.com/en-us/node/528565
@@ -57,9 +101,9 @@ class MklCPUAllocator : public Allocator {
     i_calloc = CallocHook;
     i_realloc = ReallocHook;
     i_free = FreeHook;
-  }
 
-  ~MklCPUAllocator() override { delete allocator_; }
+    return Status::OK();
+  }
 
   inline string Name() override { return kName; }
 
@@ -71,6 +115,8 @@ class MklCPUAllocator : public Allocator {
     allocator_->DeallocateRaw(ptr);
   }
 
+  void GetStats(AllocatorStats* stats) { return allocator_->GetStats(stats); }
+
  private:
   // Hooks provided by this allocator for memory allocation routines from MKL
 
@@ -96,16 +142,11 @@ class MklCPUAllocator : public Allocator {
     TF_CHECK_OK(s);  // way to assert with an error message
   }
 
-  // TODO(jbobba): We should ideally move this into CPUOptions in config.proto.
-  /// Memory limit - 64GB
-  static const size_t kMaxMemSize =
-      static_cast<size_t>(64) * 1024 * 1024 * 1024;
-
   /// Do we allow growth in BFC Allocator
   static const bool kAllowGrowth = true;
 
   /// Name
-  static constexpr const char* kName = "mklcpu";
+  static constexpr const char kName[] = "mklcpu";
 
   /// The alignment that we need for the allocations
   static const size_t kAlignment = 64;
diff --git a/tensorflow/core/common_runtime/mkl_cpu_allocator_test.cc b/tensorflow/core/common_runtime/mkl_cpu_allocator_test.cc
new file mode 100644
index 0000000000..cfefaa92e4
--- /dev/null
+++ b/tensorflow/core/common_runtime/mkl_cpu_allocator_test.cc
@@ -0,0 +1,55 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifdef INTEL_MKL
+
+#include "tensorflow/core/common_runtime/mkl_cpu_allocator.h"
+
+#include "tensorflow/core/lib/core/status_test_util.h"
+#include "tensorflow/core/platform/logging.h"
+#include "tensorflow/core/platform/test.h"
+
+namespace tensorflow {
+
+constexpr char MklCPUAllocator::kMaxLimitStr[];
+
+TEST(MKLBFCAllocatorTest, TestMaxLimit) {
+  AllocatorStats stats;
+  setenv(MklCPUAllocator::kMaxLimitStr, "1000", 1);
+  MklCPUAllocator a;
+  TF_EXPECT_OK(a.Initialize());
+  a.GetStats(&stats);
+  EXPECT_EQ(stats.bytes_limit, 1000);
+
+  unsetenv(MklCPUAllocator::kMaxLimitStr);
+  TF_EXPECT_OK(a.Initialize());
+  a.GetStats(&stats);
+  uint64 max_mem_bytes = MklCPUAllocator::kDefaultMaxLimit;
+#if defined(_SC_PHYS_PAGES) && defined(_SC_PAGESIZE)
+  max_mem_bytes =
+      (uint64)sysconf(_SC_PHYS_PAGES) * (uint64)sysconf(_SC_PAGESIZE);
+#endif
+  EXPECT_EQ(stats.bytes_limit, max_mem_bytes);
+
+  setenv(MklCPUAllocator::kMaxLimitStr, "wrong-input", 1);
+  EXPECT_TRUE(errors::IsInvalidArgument(a.Initialize()));
+
+  setenv(MklCPUAllocator::kMaxLimitStr, "-20", 1);
+  EXPECT_TRUE(errors::IsInvalidArgument(a.Initialize()));
+}
+
+}  // namespace tensorflow
+
+#endif  // INTEL_MKL
-- 
GitLab


From d6f817a58fd1b03d2c7ffd01627da5ba5024d15f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 12 Oct 2017 15:55:22 -0700
Subject: [PATCH 325/909] Make CheckNumerics on GPU asynchronous and remove the
 need for a hard device synchronization, which can have a significant
 performance impact.

PiperOrigin-RevId: 172025744
---
 tensorflow/core/kernels/check_numerics_op.cc | 124 +++++++++++--------
 1 file changed, 73 insertions(+), 51 deletions(-)

diff --git a/tensorflow/core/kernels/check_numerics_op.cc b/tensorflow/core/kernels/check_numerics_op.cc
index 1d8874b4df..56cb50d2d1 100644
--- a/tensorflow/core/kernels/check_numerics_op.cc
+++ b/tensorflow/core/kernels/check_numerics_op.cc
@@ -25,7 +25,8 @@ limitations under the License.
 #include "tensorflow/core/framework/types.h"
 
 #if GOOGLE_CUDA
-#include "tensorflow/core/platform/stream_executor.h"
+#include "tensorflow/core/common_runtime/gpu/gpu_event_mgr.h"
+#include "tensorflow/core/platform/cuda.h"
 #endif  // GOOGLE_CUDA
 namespace tensorflow {
 
@@ -99,19 +100,24 @@ class CheckNumericsOp<CPUDevice, T> : public OpKernel {
 #if GOOGLE_CUDA
 // Partial specialization for GPU
 template <typename T>
-class CheckNumericsOp<GPUDevice, T> : public OpKernel {
+class CheckNumericsOp<GPUDevice, T> : public AsyncOpKernel {
  public:
   typedef GPUDevice Device;
 
-  explicit CheckNumericsOp(OpKernelConstruction* context) : OpKernel(context) {
+  explicit CheckNumericsOp(OpKernelConstruction* context)
+      : AsyncOpKernel(context) {
     // message_ is used as the prefix for the assertion error message. For
     // instance, this can be the name of the input op that produced the tensor.
     OP_REQUIRES_OK(context, context->GetAttr("message", &message_));
   }
 
-  void Compute(OpKernelContext* context) override {
+  void ComputeAsync(OpKernelContext* context, DoneCallback done) override {
     // pass along the input to the output
     context->set_output(0, context->input(0));
+    if (context->input(0).NumElements() == 0) {
+      done();
+      return;
+    }
     auto input = context->input(0).flat<T>();
 
     // Allocate and initialize the elements to hold the check results
@@ -122,7 +128,8 @@ class CheckNumericsOp<GPUDevice, T> : public OpKernel {
                                 &abnormal_detected));
 
     auto* stream = context->op_device_context()->stream();
-    OP_REQUIRES(context, stream, errors::Internal("No GPU stream available."));
+    OP_REQUIRES_ASYNC(context, stream != nullptr,
+                      errors::Internal("No GPU stream available."), done);
 
     perftools::gputools::DeviceMemoryBase abnormal_detected_ptr(
         abnormal_detected.flat<int>().data(),
@@ -139,41 +146,59 @@ class CheckNumericsOp<GPUDevice, T> : public OpKernel {
     AllocatorAttributes attr;
     attr.set_on_host(true);
     attr.set_gpu_compatible(true);
-    Tensor abnormal_detected_out;
-    OP_REQUIRES_OK(context, context->allocate_temp(
-                                DT_INT32, TensorShape({abnormal_detected_size}),
-                                &abnormal_detected_out, attr));
-    int* abnormal_detected_host = abnormal_detected_out.flat<int>().data();
-    stream->ThenMemcpy(abnormal_detected_host, abnormal_detected_ptr,
-                       abnormal_detected_size * sizeof(int));
-    stream->BlockHostUntilDone();
-    OP_REQUIRES(context, stream->ok(),
-                errors::Internal("cudaMemcpy from device to host failed"));
-
-    int is_nan = abnormal_detected_host[0];
-    int is_inf = abnormal_detected_host[1];
-    if (is_nan || is_inf) {
-      string status;
-      LOG(ERROR) << "abnormal_detected_host @" << abnormal_detected_host
-                 << " = {" << is_nan << ", " << is_inf << "} " << message_;
-
-      // Results should always be 1 or 0.  If we see anything else then
-      // there has been some GPU memory corruption.
-      CHECK_GE(is_nan, 0);
-      CHECK_GE(is_inf, 0);
-      CHECK_LE(is_nan, 1);
-      CHECK_LE(is_inf, 1);
-
-      if (is_nan && is_inf) {
-        status = "Inf and NaN";
-      } else if (is_nan) {
-        status = "NaN";
-      } else if (is_inf) {
-        status = "Inf";
+    Tensor abnormal_detected_host;
+    OP_REQUIRES_OK_ASYNC(
+        context,
+        context->allocate_temp(DT_INT32, TensorShape({abnormal_detected_size}),
+                               &abnormal_detected_host, attr),
+        done);
+    OP_REQUIRES_ASYNC(
+        context,
+        stream
+            ->ThenMemcpy(abnormal_detected_host.flat<int>().data(),
+                         abnormal_detected_ptr,
+                         abnormal_detected_size * sizeof(int))
+            .ok(),
+        errors::Internal("cudaMemcpy from device to host failed"), done);
+
+    // We have observed crashes on some network stacks when not holding
+    // this tensor reference.
+    TensorReference abnormal_detected_ref(abnormal_detected);
+    auto check_cb = [this, stream, abnormal_detected_ref,
+                     abnormal_detected_host, context, done]() {
+      ::perftools::gputools::cuda::ScopedActivateExecutorContext
+          scoped_activation{stream->parent()};
+
+      auto abnormal_detected_host_flat = abnormal_detected_host.flat<int>();
+      int is_nan = abnormal_detected_host_flat(0);
+      int is_inf = abnormal_detected_host_flat(1);
+      if (is_nan || is_inf) {
+        string status;
+        LOG(ERROR) << "abnormal_detected_host @"
+                   << abnormal_detected_host_flat.data() << " = {" << is_nan
+                   << ", " << is_inf << "} " << message_;
+
+        // Results should always be 1 or 0.  If we see anything else then
+        // there has been some GPU memory corruption.
+        CHECK_GE(is_nan, 0);
+        CHECK_GE(is_inf, 0);
+        CHECK_LE(is_nan, 1);
+        CHECK_LE(is_inf, 1);
+
+        if (is_nan && is_inf) {
+          status = "Inf and NaN";
+        } else if (is_nan) {
+          status = "NaN";
+        } else if (is_inf) {
+          status = "Inf";
+        }
+        context->SetStatus(errors::InvalidArgument(message_, " : Tensor had ",
+                                                   status, " values"));
       }
-      context->SetStatus(errors::InvalidArgument(message_, " : Tensor had ",
-                                                 status, " values"));
-    }
+      done();
+    };
+    context->device()->tensorflow_gpu_device_info()->event_mgr->ThenExecute(
+        stream, std::move(check_cb));
   }
 
  private:
@@ -192,18 +217,15 @@ TF_CALL_float(REGISTER_CPU_KERNEL);
 TF_CALL_double(REGISTER_CPU_KERNEL);
 
 #if GOOGLE_CUDA
-REGISTER_KERNEL_BUILDER(Name("CheckNumerics")
-                            .Device(DEVICE_GPU)
-                            .TypeConstraint<Eigen::half>("T"),
-                        CheckNumericsOp<GPUDevice, Eigen::half>);
-REGISTER_KERNEL_BUILDER(Name("CheckNumerics")
-                            .Device(DEVICE_GPU)
-                            .TypeConstraint<float>("T"),
-                        CheckNumericsOp<GPUDevice, float>);
-REGISTER_KERNEL_BUILDER(Name("CheckNumerics")
-                            .Device(DEVICE_GPU)
-                            .TypeConstraint<double>("T"),
-                        CheckNumericsOp<GPUDevice, double>);
+REGISTER_KERNEL_BUILDER(
+    Name("CheckNumerics").Device(DEVICE_GPU).TypeConstraint<Eigen::half>("T"),
+    CheckNumericsOp<GPUDevice, Eigen::half>);
+REGISTER_KERNEL_BUILDER(
+    Name("CheckNumerics").Device(DEVICE_GPU).TypeConstraint<float>("T"),
+    CheckNumericsOp<GPUDevice, float>);
+REGISTER_KERNEL_BUILDER(
+    Name("CheckNumerics").Device(DEVICE_GPU).TypeConstraint<double>("T"),
+    CheckNumericsOp<GPUDevice, double>);
 #endif  // GOOGLE_CUDA
 
 }  // namespace tensorflow
-- 
GitLab


From bc52fbda2bbe458c9ff5f20ebc48188959ebe026 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 12 Oct 2017 16:10:41 -0700
Subject: [PATCH 326/909] Replace tf.conj(tf.transpose(foo)) and
 tf.transpose(tf.conj(foo)) idioms with tf.linalg.adjoint(foo) or
 tf.transpose(foo, conjugate=True), and clean up a few places that can avoid
 explicit adjoints as inputs to matmul.

PiperOrigin-RevId: 172027859
---
 tensorflow/python/BUILD                       |  1 +
 tensorflow/python/kernel_tests/BUILD          |  2 +
 .../python/kernel_tests/cholesky_op_test.py   |  3 +-
 .../linalg/linear_operator_util_test.py       | 23 ---------
 tensorflow/python/kernel_tests/qr_op_test.py  |  2 +-
 tensorflow/python/ops/linalg/BUILD            | 11 ++++
 tensorflow/python/ops/linalg/linalg_impl.py   |  3 +-
 .../python/ops/linalg/linear_operator.py      |  5 +-
 .../python/ops/linalg/linear_operator_diag.py |  5 +-
 .../ops/linalg/linear_operator_identity.py    |  7 +--
 .../linear_operator_lower_triangular.py       |  3 +-
 .../ops/linalg/linear_operator_test_util.py   | 50 ++++++++++---------
 .../python/ops/linalg/linear_operator_util.py | 47 -----------------
 tensorflow/python/ops/linalg_grad.py          | 13 ++---
 14 files changed, 63 insertions(+), 112 deletions(-)

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index b9b85909a3..ac16ca1830 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -1720,6 +1720,7 @@ py_library(
         ":framework_for_generated_wrappers",
         ":linalg_ops",
         ":math_ops",
+        "//tensorflow/python/ops/linalg:linalg_impl",
     ],
 )
 
diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD
index 6beebbf48f..d6eba3c31a 100644
--- a/tensorflow/python/kernel_tests/BUILD
+++ b/tensorflow/python/kernel_tests/BUILD
@@ -154,6 +154,7 @@ cuda_py_test(
         "//tensorflow/python:linalg_ops",
         "//tensorflow/python:math_ops",
         "//tensorflow/python:platform",
+        "//tensorflow/python/ops/linalg",
     ],
     shard_count = 5,
     tags = ["no_windows_gpu"],
@@ -2642,6 +2643,7 @@ cuda_py_test(
         "//tensorflow/python:gradients",
         "//tensorflow/python:linalg_ops",
         "//tensorflow/python:math_ops",
+        "//tensorflow/python/ops/linalg",
     ],
     shard_count = 20,
 )
diff --git a/tensorflow/python/kernel_tests/cholesky_op_test.py b/tensorflow/python/kernel_tests/cholesky_op_test.py
index 2da7672f55..782e6b5068 100644
--- a/tensorflow/python/kernel_tests/cholesky_op_test.py
+++ b/tensorflow/python/kernel_tests/cholesky_op_test.py
@@ -35,6 +35,7 @@ from tensorflow.python.ops import linalg_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import random_ops
 from tensorflow.python.ops import variables
+from tensorflow.python.ops.linalg import linalg
 from tensorflow.python.platform import test
 from tensorflow.python.platform import tf_logging
 
@@ -72,7 +73,7 @@ def TriAngSolveCompositeGrad(l, grad):
   # we can ommit the conjugate transpose here.
   z_h = math_ops.conj(array_ops.matrix_transpose(l_inverse_middle))
   grad_a = linalg_ops.matrix_triangular_solve(l, z_h, adjoint=True)
-  grad_a += math_ops.conj(array_ops.matrix_transpose(grad_a))
+  grad_a += linalg.adjoint(grad_a)
   return grad_a * 0.5
 
 
diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_util_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_util_test.py
index ca3c8647db..e1edffc3d9 100644
--- a/tensorflow/python/kernel_tests/linalg/linear_operator_util_test.py
+++ b/tensorflow/python/kernel_tests/linalg/linear_operator_util_test.py
@@ -227,29 +227,6 @@ class MatmulWithBroadcastTest(test.TestCase):
       self.assertAllEqual(expected, result)
 
 
-class MatrixAdjointTest(test.TestCase):
-
-  def testNonBatchMatrix(self):
-    a = [[1, 2, 3j], [4, 5, -6j]]  # Shape (2, 3)
-    expected = [[1, 4], [2, 5], [-3j, 6j]]  # Shape (3, 2)
-    with self.test_session():
-      a_adj = linear_operator_util.matrix_adjoint(a)
-      self.assertEqual((3, 2), a_adj.get_shape())
-      self.assertAllClose(expected, a_adj.eval())
-
-  def testBatchMatrix(self):
-    matrix_0 = [[1j, 2, 3], [4, 5, 6]]
-    matrix_0_a = [[-1j, 4], [2, 5], [3, 6]]
-    matrix_1 = [[11, 22, 33], [44, 55, 66j]]
-    matrix_1_a = [[11, 44], [22, 55], [33, -66j]]
-    batch_matrix = [matrix_0, matrix_1]  # Shape (2, 2, 3)
-    expected_adj = [matrix_0_a, matrix_1_a]  # Shape (2, 3, 2)
-    with self.test_session():
-      matrix_adj = linear_operator_util.matrix_adjoint(batch_matrix)
-      self.assertEqual((2, 3, 2), matrix_adj.get_shape())
-      self.assertAllEqual(expected_adj, matrix_adj.eval())
-
-
 class DomainDimensionStubOperator(object):
 
   def __init__(self, domain_dimension):
diff --git a/tensorflow/python/kernel_tests/qr_op_test.py b/tensorflow/python/kernel_tests/qr_op_test.py
index f7de2949a4..b4fd89bd03 100644
--- a/tensorflow/python/kernel_tests/qr_op_test.py
+++ b/tensorflow/python/kernel_tests/qr_op_test.py
@@ -103,7 +103,7 @@ def _GetQrOpTest(dtype_, shape_, full_matrices_, use_static_shape_):
 
   def CheckUnitary(self, x):
     # Tests that x[...,:,:]^H * x[...,:,:] is close to the identity.
-    xx = math_ops.matmul(math_ops.conj(x), x, transpose_a=True)
+    xx = math_ops.matmul(x, x, adjoint_a=True)
     identity = array_ops.matrix_band_part(array_ops.ones_like(xx), 0, 0)
     if is_single:
       tol = 1e-5
diff --git a/tensorflow/python/ops/linalg/BUILD b/tensorflow/python/ops/linalg/BUILD
index a36e0a4be1..b88e72a6f3 100644
--- a/tensorflow/python/ops/linalg/BUILD
+++ b/tensorflow/python/ops/linalg/BUILD
@@ -25,6 +25,17 @@ py_library(
     ],
 )
 
+py_library(
+    name = "linalg_impl",
+    srcs = ["linalg_impl.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:linalg_ops",
+        "//tensorflow/python:math_ops",
+    ],
+)
+
 filegroup(
     name = "all_files",
     srcs = glob(
diff --git a/tensorflow/python/ops/linalg/linalg_impl.py b/tensorflow/python/ops/linalg/linalg_impl.py
index 32d1b31d7d..1fdec2b51b 100644
--- a/tensorflow/python/ops/linalg/linalg_impl.py
+++ b/tensorflow/python/ops/linalg/linalg_impl.py
@@ -57,7 +57,7 @@ def logdet(matrix, name=None):
 
 
 def adjoint(matrix, name=None):
-  """Conjugates and transposes the last two dimensions of tensor `matrix`.
+  """Transposes the last two dimensions of and conjugates tensor `matrix`.
 
   For example:
 
@@ -78,4 +78,5 @@ def adjoint(matrix, name=None):
     matrix.
   """
   with ops.name_scope(name, 'adjoint', [matrix]):
+    matrix = ops.convert_to_tensor(matrix, name='matrix')
     return array_ops.matrix_transpose(matrix, conjugate=True)
diff --git a/tensorflow/python/ops/linalg/linear_operator.py b/tensorflow/python/ops/linalg/linear_operator.py
index 17c338ec75..0d04e29eb3 100644
--- a/tensorflow/python/ops/linalg/linear_operator.py
+++ b/tensorflow/python/ops/linalg/linear_operator.py
@@ -29,6 +29,7 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import check_ops
 from tensorflow.python.ops import linalg_ops
 from tensorflow.python.ops import math_ops
+from tensorflow.python.ops.linalg import linalg_impl as linalg
 from tensorflow.python.ops.linalg import linear_operator_util
 from tensorflow.python.platform import tf_logging as logging
 
@@ -551,7 +552,7 @@ class LinearOperator(object):
         "  Requires conversion to a dense matrix.")
     return check_ops.assert_equal(
         dense,
-        linear_operator_util.matrix_adjoint(dense),
+        linalg.adjoint(dense),
         message="Matrix was not equal to its adjoint.")
 
   def assert_self_adjoint(self, name="assert_self_adjoint"):
@@ -722,7 +723,7 @@ class LinearOperator(object):
     logging.warn(
         "Using (possibly slow) default implementation of solve."
         "  Requires conversion to a dense matrix and O(N^3) operations.")
-    rhs = linear_operator_util.matrix_adjoint(rhs) if adjoint_arg else rhs
+    rhs = linalg.adjoint(rhs) if adjoint_arg else rhs
     if self._can_use_cholesky():
       return linalg_ops.cholesky_solve(self._get_cached_chol(), rhs)
     return linalg_ops.matrix_solve(
diff --git a/tensorflow/python/ops/linalg/linear_operator_diag.py b/tensorflow/python/ops/linalg/linear_operator_diag.py
index e1558a351d..a4724d030f 100644
--- a/tensorflow/python/ops/linalg/linear_operator_diag.py
+++ b/tensorflow/python/ops/linalg/linear_operator_diag.py
@@ -23,6 +23,7 @@ from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import check_ops
 from tensorflow.python.ops import math_ops
+from tensorflow.python.ops.linalg import linalg_impl as linalg
 from tensorflow.python.ops.linalg import linear_operator
 from tensorflow.python.ops.linalg import linear_operator_util
 
@@ -216,7 +217,7 @@ class LinearOperatorDiag(linear_operator.LinearOperator):
 
   def _matmul(self, x, adjoint=False, adjoint_arg=False):
     diag_term = math_ops.conj(self._diag) if adjoint else self._diag
-    x = linear_operator_util.matrix_adjoint(x) if adjoint_arg else x
+    x = linalg.adjoint(x) if adjoint_arg else x
     diag_mat = array_ops.expand_dims(diag_term, -1)
     return diag_mat * x
 
@@ -229,7 +230,7 @@ class LinearOperatorDiag(linear_operator.LinearOperator):
 
   def _solve(self, rhs, adjoint=False, adjoint_arg=False):
     diag_term = math_ops.conj(self._diag) if adjoint else self._diag
-    rhs = linear_operator_util.matrix_adjoint(rhs) if adjoint_arg else rhs
+    rhs = linalg.adjoint(rhs) if adjoint_arg else rhs
     inv_diag_mat = array_ops.expand_dims(1. / diag_term, -1)
     return rhs * inv_diag_mat
 
diff --git a/tensorflow/python/ops/linalg/linear_operator_identity.py b/tensorflow/python/ops/linalg/linear_operator_identity.py
index 18bd2f9f6d..740c6c811f 100644
--- a/tensorflow/python/ops/linalg/linear_operator_identity.py
+++ b/tensorflow/python/ops/linalg/linear_operator_identity.py
@@ -28,6 +28,7 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import check_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import math_ops
+from tensorflow.python.ops.linalg import linalg_impl as linalg
 from tensorflow.python.ops.linalg import linear_operator
 from tensorflow.python.ops.linalg import linear_operator_util
 
@@ -345,7 +346,7 @@ class LinearOperatorIdentity(BaseLinearOperatorIdentity):
 
   def _matmul(self, x, adjoint=False, adjoint_arg=False):
     # Note that adjoint has no effect since this matrix is self-adjoint.
-    x = linear_operator_util.matrix_adjoint(x) if adjoint_arg else x
+    x = linalg.adjoint(x) if adjoint_arg else x
     if self._assert_proper_shapes:
       aps = linear_operator_util.assert_compatible_matrix_dimensions(self, x)
       x = control_flow_ops.with_dependencies([aps], x)
@@ -644,7 +645,7 @@ class LinearOperatorScaledIdentity(BaseLinearOperatorIdentity):
         message="LinearOperator was not self-adjoint")
 
   def _matmul(self, x, adjoint=False, adjoint_arg=False):
-    x = linear_operator_util.matrix_adjoint(x) if adjoint_arg else x
+    x = linalg.adjoint(x) if adjoint_arg else x
     if adjoint:
       matrix = self._multiplier_matrix_conj
     else:
@@ -662,7 +663,7 @@ class LinearOperatorScaledIdentity(BaseLinearOperatorIdentity):
         self._abs_multiplier)
 
   def _solve(self, rhs, adjoint=False, adjoint_arg=False):
-    rhs = linear_operator_util.matrix_adjoint(rhs) if adjoint_arg else rhs
+    rhs = linalg.adjoint(rhs) if adjoint_arg else rhs
     if adjoint:
       matrix = self._multiplier_matrix_conj
     else:
diff --git a/tensorflow/python/ops/linalg/linear_operator_lower_triangular.py b/tensorflow/python/ops/linalg/linear_operator_lower_triangular.py
index 4b074f5cec..6ea55f0367 100644
--- a/tensorflow/python/ops/linalg/linear_operator_lower_triangular.py
+++ b/tensorflow/python/ops/linalg/linear_operator_lower_triangular.py
@@ -23,6 +23,7 @@ from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import linalg_ops
 from tensorflow.python.ops import math_ops
+from tensorflow.python.ops.linalg import linalg_impl as linalg
 from tensorflow.python.ops.linalg import linear_operator
 from tensorflow.python.ops.linalg import linear_operator_util
 
@@ -198,7 +199,7 @@ class LinearOperatorLowerTriangular(linear_operator.LinearOperator):
         math_ops.log(math_ops.abs(self._diag)), reduction_indices=[-1])
 
   def _solve(self, rhs, adjoint=False, adjoint_arg=False):
-    rhs = linear_operator_util.matrix_adjoint(rhs) if adjoint_arg else rhs
+    rhs = linalg.adjoint(rhs) if adjoint_arg else rhs
     return linalg_ops.matrix_triangular_solve(
         self._tril, rhs, lower=True, adjoint=adjoint)
 
diff --git a/tensorflow/python/ops/linalg/linear_operator_test_util.py b/tensorflow/python/ops/linalg/linear_operator_test_util.py
index b86cb6d84d..4a601047b6 100644
--- a/tensorflow/python/ops/linalg/linear_operator_test_util.py
+++ b/tensorflow/python/ops/linalg/linear_operator_test_util.py
@@ -31,7 +31,7 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import linalg_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import random_ops
-from tensorflow.python.ops.linalg import linear_operator_util
+from tensorflow.python.ops.linalg import linalg_impl as linalg
 from tensorflow.python.platform import test
 
 
@@ -196,8 +196,7 @@ class LinearOperatorDerivedClassTest(test.TestCase):
             if not use_placeholder:
               self.assertAllEqual(shape[:-2], op_log_abs_det.get_shape())
             op_log_abs_det_v, mat_log_abs_det_v = sess.run(
-                [op_log_abs_det, mat_log_abs_det],
-                feed_dict=feed_dict)
+                [op_log_abs_det, mat_log_abs_det], feed_dict=feed_dict)
             self.assertAC(op_log_abs_det_v, mat_log_abs_det_v)
 
   def test_matmul(self):
@@ -215,14 +214,15 @@ class LinearOperatorDerivedClassTest(test.TestCase):
                 # If adjoint_arg, compute A X^H^H = A X.
                 if adjoint_arg:
                   op_matmul = operator.matmul(
-                      linear_operator_util.matrix_adjoint(x),
-                      adjoint=adjoint, adjoint_arg=adjoint_arg)
+                      linalg.adjoint(x),
+                      adjoint=adjoint,
+                      adjoint_arg=adjoint_arg)
                 else:
                   op_matmul = operator.matmul(x, adjoint=adjoint)
                 mat_matmul = math_ops.matmul(mat, x, adjoint_a=adjoint)
                 if not use_placeholder:
-                  self.assertAllEqual(
-                      op_matmul.get_shape(), mat_matmul.get_shape())
+                  self.assertAllEqual(op_matmul.get_shape(),
+                                      mat_matmul.get_shape())
                 op_matmul_v, mat_matmul_v = sess.run(
                     [op_matmul, mat_matmul], feed_dict=feed_dict)
                 self.assertAC(op_matmul_v, mat_matmul_v)
@@ -242,17 +242,18 @@ class LinearOperatorDerivedClassTest(test.TestCase):
                 # If adjoint_arg, solve A X = (rhs^H)^H = rhs.
                 if adjoint_arg:
                   op_solve = operator.solve(
-                      linear_operator_util.matrix_adjoint(rhs),
-                      adjoint=adjoint, adjoint_arg=adjoint_arg)
+                      linalg.adjoint(rhs),
+                      adjoint=adjoint,
+                      adjoint_arg=adjoint_arg)
                 else:
                   op_solve = operator.solve(
                       rhs, adjoint=adjoint, adjoint_arg=adjoint_arg)
                 mat_solve = linalg_ops.matrix_solve(mat, rhs, adjoint=adjoint)
                 if not use_placeholder:
-                  self.assertAllEqual(
-                      op_solve.get_shape(), mat_solve.get_shape())
-                op_solve_v, mat_solve_v = sess.run([op_solve, mat_solve],
-                                                   feed_dict=feed_dict)
+                  self.assertAllEqual(op_solve.get_shape(),
+                                      mat_solve.get_shape())
+                op_solve_v, mat_solve_v = sess.run(
+                    [op_solve, mat_solve], feed_dict=feed_dict)
                 self.assertAC(op_solve_v, mat_solve_v)
 
   def test_trace(self):
@@ -268,8 +269,8 @@ class LinearOperatorDerivedClassTest(test.TestCase):
             mat_trace = math_ops.trace(mat)
             if not use_placeholder:
               self.assertAllEqual(op_trace.get_shape(), mat_trace.get_shape())
-            op_trace_v, mat_trace_v = sess.run([op_trace, mat_trace],
-                                               feed_dict=feed_dict)
+            op_trace_v, mat_trace_v = sess.run(
+                [op_trace, mat_trace], feed_dict=feed_dict)
             self.assertAC(op_trace_v, mat_trace_v)
 
   def test_add_to_tensor(self):
@@ -286,8 +287,8 @@ class LinearOperatorDerivedClassTest(test.TestCase):
             if not use_placeholder:
               self.assertAllEqual(shape, op_plus_2mat.get_shape())
 
-            op_plus_2mat_v, mat_v = sess.run([op_plus_2mat, mat],
-                                             feed_dict=feed_dict)
+            op_plus_2mat_v, mat_v = sess.run(
+                [op_plus_2mat, mat], feed_dict=feed_dict)
 
             self.assertAC(op_plus_2mat_v, 3 * mat_v)
 
@@ -304,8 +305,8 @@ class LinearOperatorDerivedClassTest(test.TestCase):
             mat_diag_part = array_ops.matrix_diag_part(mat)
 
             if not use_placeholder:
-              self.assertAllEqual(
-                  mat_diag_part.get_shape(), op_diag_part.get_shape())
+              self.assertAllEqual(mat_diag_part.get_shape(),
+                                  op_diag_part.get_shape())
 
             op_diag_part_, mat_diag_part_ = sess.run(
                 [op_diag_part, mat_diag_part], feed_dict=feed_dict)
@@ -584,13 +585,16 @@ def random_sign_uniform(shape,
     if seed is not None:
       seed += 12
     signs = math_ops.sign(
-        random_ops.random_uniform(
-            shape, minval=-1., maxval=1., seed=seed))
+        random_ops.random_uniform(shape, minval=-1., maxval=1., seed=seed))
     return unsigned_samples * math_ops.cast(signs, unsigned_samples.dtype)
 
 
-def random_normal_correlated_columns(
-    shape, mean=0.0, stddev=1.0, dtype=dtypes.float32, eps=1e-4, seed=None):
+def random_normal_correlated_columns(shape,
+                                     mean=0.0,
+                                     stddev=1.0,
+                                     dtype=dtypes.float32,
+                                     eps=1e-4,
+                                     seed=None):
   """Batch matrix with (possibly complex) Gaussian entries and correlated cols.
 
   Returns random batch matrix `A` with specified element-wise `mean`, `stddev`,
diff --git a/tensorflow/python/ops/linalg/linear_operator_util.py b/tensorflow/python/ops/linalg/linear_operator_util.py
index 2659bd32e9..427bd1e890 100644
--- a/tensorflow/python/ops/linalg/linear_operator_util.py
+++ b/tensorflow/python/ops/linalg/linear_operator_util.py
@@ -289,53 +289,6 @@ def matmul_with_broadcast(a,
         b_is_sparse=b_is_sparse)
 
 
-def matrix_adjoint(a, name="matrix_adjoint"):
-  """Transposes last two dimensions of tensor `a`, and takes complex conjugate.
-
-  If `a` is real valued, the result is equivalent to `matrix_transpose`.
-
-  For example:
-
-  ```python
-  # Matrix with no batch dimension.
-  # 'x' is [[1 2 3j]
-  #         [4 5 -6j]]
-  tf.matrix_adjoint(x) ==> [[1 4]
-                            [2 5]
-                            [-3j 6j]]
-
-  # Matrix with two batch dimensions.
-  # x.shape is [1, 2, 3, 4]
-  # tf.matrix_adjoint(x) is shape [1, 2, 4, 3]
-  ```
-
-  Note that `tf.matmul` provides kwargs allowing for adjoint of arguments.  This
-  is done with minimal cost, and is preferable to using this function. E.g.
-
-  ```
-  # Good!  Adjoint is taken at minimal additional cost.
-  tf.matmul(matrix, b, adjoint_b=True)
-
-  # Inefficient!
-  tf.matmul(matrix, tf.matrix_adjoint(b))
-  ```
-
-  Args:
-    a: A `Tensor` with `rank >= 2`.
-    name: A name for the operation (optional).
-
-  Returns:
-    A batch matrix `Tensor` with same `dtype` as `a`.
-
-  Raises:
-    ValueError:  If `a` is determined statically to have `rank < 2`.
-  """
-  with ops.name_scope(name, values=[a]):
-    a = ops.convert_to_tensor(a, name="a")
-    a_transpose = array_ops.matrix_transpose(a)
-    return math_ops.conj(a_transpose)
-
-
 def shape_tensor(shape, name=None):
   """Convert Tensor using default type, unless empty list or tuple."""
   # Works just like random_ops._ShapeTensor.
diff --git a/tensorflow/python/ops/linalg_grad.py b/tensorflow/python/ops/linalg_grad.py
index 1752164d7a..ec263591e1 100644
--- a/tensorflow/python/ops/linalg_grad.py
+++ b/tensorflow/python/ops/linalg_grad.py
@@ -32,6 +32,7 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import linalg_ops
 from tensorflow.python.ops import math_ops
+from tensorflow.python.ops.linalg import linalg_impl as _linalg
 
 
 @ops.RegisterGradient("MatrixInverse")
@@ -76,7 +77,7 @@ def _CholeskyGrad(op, grad):
   grad_a = math_ops.matmul(
       math_ops.matmul(l_inverse, middle, adjoint_a=True), l_inverse)
 
-  grad_a += math_ops.conj(array_ops.matrix_transpose(grad_a))
+  grad_a += _linalg.adjoint(grad_a)
   return grad_a * 0.5
 
 
@@ -229,8 +230,7 @@ def _SelfAdjointEigV2Grad(op, grad_e, grad_v):
                                    adjoint_b=True))
     # The forward op only depends on the lower triangular part of a, so here we
     # symmetrize and take the lower triangle
-    grad_a = array_ops.matrix_band_part(
-        grad_a + math_ops.conj(array_ops.matrix_transpose(grad_a)), -1, 0)
+    grad_a = array_ops.matrix_band_part(grad_a + _linalg.adjoint(grad_a), -1, 0)
     grad_a = array_ops.matrix_set_diag(grad_a,
                                        0.5 * array_ops.matrix_diag_part(grad_a))
     return grad_a
@@ -240,9 +240,6 @@ def _SelfAdjointEigV2Grad(op, grad_e, grad_v):
 def _SvdGrad(op, grad_s, grad_u, grad_v):
   """Gradient for Svd based on Giles' algorithm. Reference at top of file."""
 
-  def _Adjoint(x):
-    return math_ops.conj(array_ops.matrix_transpose(x))
-
   if op.get_attr("compute_uv") and not op.get_attr("full_matrices"):
     raise NotImplementedError(
         "SVD gradient is not implemented for compute_uv=True and "
@@ -337,8 +334,8 @@ def _SvdGrad(op, grad_s, grad_u, grad_v):
       f_v = f * v_gv[..., :m, :m]
 
     grad_a_nouv = (
-        grad_s_mat + math_ops.matmul(f_u + _Adjoint(f_u), s_mat) +
-        math_ops.matmul(s_mat, f_v + _Adjoint(f_v)))
+        grad_s_mat + math_ops.matmul(f_u + _linalg.adjoint(f_u), s_mat) +
+        math_ops.matmul(s_mat, f_v + _linalg.adjoint(f_v)))
 
     if m != n:
       grad_a_nouv = array_ops.concat(
-- 
GitLab


From 1002f974f58b23c528436e34c06384b8bffb2485 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 12 Oct 2017 16:16:14 -0700
Subject: [PATCH 327/909] Puts the global step read within the global step name
 scope, for clean/organized graph displays in TensorBoard.

PiperOrigin-RevId: 172028555
---
 tensorflow/python/training/training_util.py | 20 +++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/tensorflow/python/training/training_util.py b/tensorflow/python/training/training_util.py
index c5163f9798..bdd4ca734e 100644
--- a/tensorflow/python/training/training_util.py
+++ b/tensorflow/python/training/training_util.py
@@ -212,13 +212,14 @@ def _get_or_create_global_step_read(graph=None):
     return None
   # add 'zero' so that it will create a copy of variable as Tensor.
   with graph.as_default() as g, g.name_scope(None):
-    # using initialized_value to ensure that global_step is initialized before
-    # this run. This is needed for example Estimator makes all model_fn build
-    # under global_step_read_tensor dependency.
-    global_step_value = global_step_tensor.initialized_value() if isinstance(
-        global_step_tensor, variables.Variable) else global_step_tensor
-    global_step_read_tensor = global_step_value + 0
-    ops.add_to_collection(GLOBAL_STEP_READ_KEY, global_step_read_tensor)
+    with g.name_scope(global_step_tensor.op.name + '/'):
+      # using initialized_value to ensure that global_step is initialized before
+      # this run. This is needed for example Estimator makes all model_fn build
+      # under global_step_read_tensor dependency.
+      global_step_value = global_step_tensor.initialized_value() if isinstance(
+          global_step_tensor, variables.Variable) else global_step_tensor
+      global_step_read_tensor = global_step_value + 0
+      ops.add_to_collection(GLOBAL_STEP_READ_KEY, global_step_read_tensor)
   return _get_global_step_read(graph)
 
 
@@ -231,5 +232,6 @@ def _increment_global_step(increment, graph=None):
         'tf.train.get_or_create_global_step before calling increment.')
   global_step_read_tensor = _get_or_create_global_step_read(graph)
   with graph.as_default() as g, g.name_scope(None):
-    with ops.control_dependencies([global_step_read_tensor]):
-      return state_ops.assign_add(global_step_tensor, increment)
+    with g.name_scope(global_step_tensor.op.name + '/'):
+      with ops.control_dependencies([global_step_read_tensor]):
+        return state_ops.assign_add(global_step_tensor, increment)
-- 
GitLab


From b2648acad33de88a820aa82b9e404bfac3f40801 Mon Sep 17 00:00:00 2001
From: gunan <gunan@google.com>
Date: Thu, 12 Oct 2017 16:28:50 -0700
Subject: [PATCH 328/909] Add a new MKL build script for linux. (#13673)

---
 .../tools/ci_build/linux/cpu/run_mkl.sh       | 36 +++++++++++++++++++
 1 file changed, 36 insertions(+)
 create mode 100755 tensorflow/tools/ci_build/linux/cpu/run_mkl.sh

diff --git a/tensorflow/tools/ci_build/linux/cpu/run_mkl.sh b/tensorflow/tools/ci_build/linux/cpu/run_mkl.sh
new file mode 100755
index 0000000000..dbf376be6f
--- /dev/null
+++ b/tensorflow/tools/ci_build/linux/cpu/run_mkl.sh
@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# ==============================================================================
+
+set -e
+set -x
+
+N_JOBS=$(grep -c ^processor /proc/cpuinfo)
+
+echo ""
+echo "Bazel will use ${N_JOBS} concurrent job(s)."
+echo ""
+
+# Run configure.
+export TF_NEED_CUDA=0
+export PYTHON_BIN_PATH=`which python2`
+yes "" | $PYTHON_BIN_PATH configure.py
+
+# Run bazel test command. Double test timeouts to avoid flakes.
+bazel test --test_tag_filters=-no_oss,-oss_serial,-gpu,-benchmark-test --test_lang_filters=py -k \
+    --jobs=${N_JOBS} --test_timeout 300,450,1200,3600 --build_tests_only \
+    --config=mkl --config=opt --test_output=errors -- \
+    //tensorflow/... -//tensorflow/compiler/... -//tensorflow/contrib/...
-- 
GitLab


From 19708cc7d8e34e830a716d3f9896294489d3b535 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 12 Oct 2017 16:39:22 -0700
Subject: [PATCH 329/909] A few profiler improvements. 1. Use a id_to_string
 map to reduce the profile size (2/3 in xception) 2. dedup code view's
 function name with extra file base name. 3. remove code view display
 heuristic that doesn't work in some cases. 4. make the profile_context
 thread-safe.

PiperOrigin-RevId: 172031528
---
 .../core/profiler/internal/tfprof_code.cc     |  62 +++-------
 .../core/profiler/internal/tfprof_code.h      |   3 -
 .../core/profiler/internal/tfprof_node.cc     |   2 +-
 .../core/profiler/internal/tfprof_node.h      |  70 ++++++++++-
 .../core/profiler/internal/tfprof_node_show.h |   8 +-
 .../core/profiler/internal/tfprof_stats.cc    |  19 ++-
 .../core/profiler/internal/tfprof_stats.h     |   2 +
 tensorflow/core/profiler/tfprof_log.proto     |  21 +++-
 tensorflow/python/profiler/profile_context.py | 109 +++++++++---------
 tensorflow/python/profiler/tfprof_logger.py   |  25 +++-
 ...er.-op-log-proto.-id-to-string-entry.pbtxt |  84 ++++++++++++++
 .../tensorflow.profiler.-op-log-proto.pbtxt   |   8 ++
 12 files changed, 287 insertions(+), 126 deletions(-)
 create mode 100644 tensorflow/tools/api/golden/tensorflow.profiler.-op-log-proto.-id-to-string-entry.pbtxt

diff --git a/tensorflow/core/profiler/internal/tfprof_code.cc b/tensorflow/core/profiler/internal/tfprof_code.cc
index c9c0baa908..2c4f52e3ad 100644
--- a/tensorflow/core/profiler/internal/tfprof_code.cc
+++ b/tensorflow/core/profiler/internal/tfprof_code.cc
@@ -36,7 +36,7 @@ namespace {
 const char* const kGradientSuffix = " (gradient)";
 
 // Convert to Trace proto into a short readable string.
-string GetTraceString(const CodeDef::Trace& trace) {
+string GetTraceString(const CallStack::Trace& trace) {
   string ntrace = io::Basename(trace.file()).ToString();
   ntrace += strings::StrCat(":", trace.lineno());
   if (trace.function().length() < 20) {
@@ -112,7 +112,11 @@ class FunctionTable {
     pprof::Function* func_pb = &function_table_[key];
     // function index should start from 1.
     func_pb->set_id(function_table_.size());
-    func_pb->set_name(string_table_->GetIndex(func_name));
+
+    string file_base = io::Basename(file_path).ToString();
+    file_base = file_base.substr(0, file_base.find_last_of("."));
+    func_pb->set_name(
+        string_table_->GetIndex(strings::StrCat(file_base, ":", func_name)));
     func_pb->set_filename(string_table_->GetIndex(file_path));
     func_pb->set_start_line(func_start_line);
     return func_pb->id();
@@ -142,6 +146,7 @@ class LocationTable {
                   uint64 called_func_start_line) {
     auto key = std::tuple<string, string, uint64>(
         file_path, called_function_name, line_number);
+
     auto idx = location_table_.find(key);
     if (idx != location_table_.end()) {
       return idx->second.id();
@@ -376,10 +381,9 @@ class PprofProfileImpl : public PprofProfile {
 }  // namespace
 
 void TFCode::AddNode(TFGraphNode* node) {
-  if (node->code().traces_size() == 0) {
+  if (!node->call_stack() || node->call_stack()->traces().empty()) {
     return;
   }
-
   // We infer the forward operation name from gradient op name. So, we can
   // map gradient op traces to forward op traces.
   // E.g. gradient node of 'inp_1/Conv2D' would be 'gradients/inp_1/Conv2D_grad.
@@ -397,42 +401,26 @@ void TFCode::AddNode(TFGraphNode* node) {
     forward_nodes_[node->name()] = node;
   }
 
-  // Track if this is the first trace (first node). If true, add all
-  // traces to common_traces_. Otherwise, remove uncommon traces from
-  // common traces_.
-  bool first_trace = false;
   if (!root_) {
     graph_root_.reset(new TFMultiGraphNode(kTFProfRoot));
     root_.reset(new CodeNode(graph_root_.get(), nullptr, ""));
-    first_trace = true;
   }
 
   CodeNode* pre_code_node = root_.get();
   // TODO(xpan): Consider to release CodeDef after TFCode is built. It
   // takes a lot of memory.
   std::set<string> traces;
-  for (int i = 0; i < node->code().traces_size(); ++i) {
+  for (int i = 0; i < node->call_stack()->traces().size(); ++i) {
     // Unlike op name, which is globally unique, trace name is only unique
     // w.r.t. it's parent.
-    const string& trace = GetTraceString(node->code().traces(i));
+    const string& trace = GetTraceString(node->call_stack()->traces().at(i));
     traces.insert(trace);
-    pre_code_node =
-        pre_code_node->AddChildren(trace, &node->code().traces(i), "");
-    if (i == node->code().traces_size() - 1) {
+    pre_code_node = pre_code_node->AddChildren(
+        trace, &node->call_stack()->traces().at(i), "");
+    if (i == node->call_stack()->traces().size() - 1) {
       pre_code_node->node->AddGraphNode(node);
     }
   }
-  if (first_trace) {
-    common_traces_.insert(traces.begin(), traces.end());
-  } else {
-    for (auto it = common_traces_.begin(); it != common_traces_.end();) {
-      if (traces.find(*it) == traces.end()) {
-        common_traces_.erase(it++);
-      } else {
-        ++it;
-      }
-    }
-  }
 }
 
 void TFCode::Build() {
@@ -447,12 +435,12 @@ void TFCode::Build() {
     TFGraphNode* fn = forward_it->second;
     CodeNode* leaf = nullptr;
     CodeNode* pre_code_node = root_.get();
-    for (int i = 0; i < fn->code().traces_size(); ++i) {
+    for (int i = 0; i < fn->call_stack()->traces().size(); ++i) {
       const string& trace =
-          GetTraceString(fn->code().traces(i)) + kGradientSuffix;
-      pre_code_node = pre_code_node->AddChildren(trace, &fn->code().traces(i),
-                                                 kGradientSuffix);
-      if (i == fn->code().traces_size() - 1) {
+          GetTraceString(fn->call_stack()->traces().at(i)) + kGradientSuffix;
+      pre_code_node = pre_code_node->AddChildren(
+          trace, &fn->call_stack()->traces().at(i), kGradientSuffix);
+      if (i == fn->call_stack()->traces().size() - 1) {
         leaf = pre_code_node;
       }
     }
@@ -463,17 +451,6 @@ void TFCode::Build() {
   if (unaccounted_nodes > 0) {
     fprintf(stderr, "%lld gradient nodes not accounted\n", unaccounted_nodes);
   }
-
-  // For trace that all traces share, such as "main", "apply_op", people
-  // are unlikely inerested. We track them and hide them from display.
-  if (forward_nodes_.size() > 100) {
-    std::set<string> tmp = common_traces_;
-    for (const string& t : tmp) {
-      common_traces_.insert(t + kGradientSuffix);
-    }
-  } else {
-    common_traces_.clear();
-  }
 }
 
 const ShowMultiNode* TFCode::ShowInternal(const Options& opts,
@@ -590,8 +567,7 @@ std::vector<CodeNode*> TFCode::PrintScope(const std::vector<CodeNode*> roots,
       continue;
     }
     int ident = last_ident;
-    bool show = ShouldShow(node, opts, depth) &&
-                common_traces_.find(node->name()) == common_traces_.end();
+    bool show = ShouldShow(node, opts, depth);
     if (show) ident += 2;
 
     std::vector<CodeNode*> show_cnodes =
diff --git a/tensorflow/core/profiler/internal/tfprof_code.h b/tensorflow/core/profiler/internal/tfprof_code.h
index 82bac8f415..a118752fce 100644
--- a/tensorflow/core/profiler/internal/tfprof_code.h
+++ b/tensorflow/core/profiler/internal/tfprof_code.h
@@ -85,9 +85,6 @@ class TFCode : public TFMultiShow {
   string FormatNode(CodeNode* node, const Options& opts, int64 indent) const;
   string FormatNodeMemory(CodeNode* node, int64 bytes, int64 total_bytes) const;
 
-  // Common traces track the code path that all traces share. Such as
-  // "main()", "create_op", etc.
-  std::set<string> common_traces_;
   std::unique_ptr<CodeNode> root_;
   std::unique_ptr<TFMultiGraphNode> graph_root_;
   std::unique_ptr<PprofProfile> pprof_profile_;
diff --git a/tensorflow/core/profiler/internal/tfprof_node.cc b/tensorflow/core/profiler/internal/tfprof_node.cc
index e2be2cf4cf..f283fafc0f 100644
--- a/tensorflow/core/profiler/internal/tfprof_node.cc
+++ b/tensorflow/core/profiler/internal/tfprof_node.cc
@@ -227,7 +227,7 @@ std::vector<int64> ShapeProtoToVec(const TensorShapeProto& shape_pb) {
   return shape_vec;
 }
 
-TensorShapeProto VecToShapeProto(const std::vector<int64> shape_vec) {
+TensorShapeProto VecToShapeProto(const std::vector<int64>& shape_vec) {
   TensorShapeProto shape_pb;
   if (shape_vec.empty()) {
     shape_pb.set_unknown_rank(true);
diff --git a/tensorflow/core/profiler/internal/tfprof_node.h b/tensorflow/core/profiler/internal/tfprof_node.h
index 95d199e5b9..34bc0a581d 100644
--- a/tensorflow/core/profiler/internal/tfprof_node.h
+++ b/tensorflow/core/profiler/internal/tfprof_node.h
@@ -38,10 +38,52 @@ namespace tensorflow {
 namespace tfprof {
 std::vector<int64> ShapeProtoToVec(const TensorShapeProto& shape_pb);
 
-TensorShapeProto VecToShapeProto(const std::vector<int64> shape_vec);
+TensorShapeProto VecToShapeProto(const std::vector<int64>& shape_vec);
 
 class TFGraphNode;
 
+class CallStack {
+ public:
+  class Trace {
+   public:
+    Trace(const CodeDef::Trace* trace,
+          const std::map<int64, string>* id_to_string)
+        : trace_(trace), id_to_string_(id_to_string) {}
+
+    const int32 lineno() const { return trace_->lineno(); }
+    string file() const {
+      // Backward compatible with old proto files.
+      if (!trace_->file().empty()) return trace_->file();
+      return id_to_string_->at(trace_->file_id());
+    }
+    string function() const {
+      // Backward compatible with old proto files.
+      if (!trace_->function().empty()) return trace_->function();
+      return id_to_string_->at(trace_->function_id());
+    }
+    int32 func_start_line() const { return trace_->func_start_line(); }
+
+   private:
+    const CodeDef::Trace* trace_;
+    const std::map<int64, string>* id_to_string_;
+  };
+
+  CallStack(const CodeDef& def, const std::map<int64, string>* id_to_string)
+      : def_(def) {
+    traces_.reserve(def.traces_size());
+    for (const auto& t : def_.traces()) {
+      traces_.emplace_back(&t, id_to_string);
+    }
+  }
+
+  const CodeDef& code_def() const { return def_; }
+  const std::vector<Trace>& traces() const { return traces_; }
+
+ private:
+  std::vector<Trace> traces_;
+  CodeDef def_;
+};
+
 class ExecStep {
  public:
   ExecStep() {}
@@ -195,8 +237,9 @@ class ExecStep {
 
 class TFGraphNode {
  public:
-  TFGraphNode(const ProfileNode& node, const ProfileProto& profile) {
-    FromProto(node, profile);
+  TFGraphNode(const ProfileNode& node, const ProfileProto& profile,
+              const std::map<int64, string>* id_to_string) {
+    FromProto(node, profile, id_to_string);
   }
 
   TFGraphNode(const NodeDef* node, int64 id) {
@@ -247,7 +290,12 @@ class TFGraphNode {
   void AddFloatOps(int64 float_ops) { node_.set_float_ops(float_ops); }
 
   // TODO(xpan): This could take a lot of memory.
-  void AddCode(const CodeDef& code) { node_.mutable_trace()->MergeFrom(code); }
+  void AddCode(const CodeDef& code,
+               const std::map<int64, string>* id_to_string) {
+    if (!call_stack_) {
+      call_stack_.reset(new CallStack(code, id_to_string));
+    }
+  }
 
   const string& name() const { return node_.name(); }
   int64 id() const { return node_.id(); }
@@ -311,13 +359,21 @@ class TFGraphNode {
       int64 id = nodes_map.at(s.first)->id();
       (*node_.mutable_src_output_index())[id] = s.second;
     }
+
+    if (call_stack_) {
+      node_.clear_trace();
+      node_.mutable_trace()->MergeFrom(call_stack_->code_def());
+    }
     return node_;
   }
 
-  void FromProto(const ProfileNode& node, const ProfileProto& profile) {
+  void FromProto(const ProfileNode& node, const ProfileProto& profile,
+                 const std::map<int64, string>* id_to_string) {
     node_.Clear();
     node_.MergeFrom(node);
 
+    call_stack_.reset(new CallStack(node.trace(), id_to_string));
+
     op_types_.clear();
     op_types_.insert(node_.op_types().begin(), node_.op_types().end());
 
@@ -554,7 +610,7 @@ class TFGraphNode {
     // Otherwise, return dynamic float_ops.
     return node_.float_ops() * run_count(step);
   }
-  const CodeDef& code() { return node_.trace(); }
+  const CallStack* call_stack() { return call_stack_.get(); }
   string canonical_device() const { return node_.canonical_device(); }
   string host_device() const { return node_.host_device(); }
   const std::set<string>& op_types() const { return op_types_; }
@@ -582,6 +638,8 @@ class TFGraphNode {
 
   ProfileNode node_;
 
+  std::unique_ptr<CallStack> call_stack_;
+
   std::vector<int64> shape_;
   // Won't missing input_idx. But some shapes might be empty (unknown).
   std::map<int, std::vector<int64>> input_shapes_;
diff --git a/tensorflow/core/profiler/internal/tfprof_node_show.h b/tensorflow/core/profiler/internal/tfprof_node_show.h
index d3c5ffd7f6..3788bf3e80 100644
--- a/tensorflow/core/profiler/internal/tfprof_node_show.h
+++ b/tensorflow/core/profiler/internal/tfprof_node_show.h
@@ -111,12 +111,12 @@ class ShowMultiNode {
 
 class CodeNode : public ShowMultiNode {
  public:
-  CodeNode(TFMultiGraphNode* node, const CodeDef::Trace* trace,
+  CodeNode(TFMultiGraphNode* node, const CallStack::Trace* trace,
            const string& suffix)
       : ShowMultiNode(node), trace_(trace), suffix_(suffix) {}
   ~CodeNode() override {}
 
-  CodeNode* AddChildren(const string& name, const CodeDef::Trace* trace,
+  CodeNode* AddChildren(const string& name, const CallStack::Trace* trace,
                         const string suffix) {
     auto it = children_.find(name);
     if (it != children_.end()) {
@@ -133,7 +133,7 @@ class CodeNode : public ShowMultiNode {
 
   bool has_trace() const { return trace_ != nullptr; }
   const int32 lineno() const { return trace_->lineno(); }
-  string file() const { return trace_->file() + suffix_; }
+  string file() const { return trace_->file(); }
   string function() const { return trace_->function() + suffix_; }
   int32 func_start_line() const { return trace_->func_start_line(); }
 
@@ -141,7 +141,7 @@ class CodeNode : public ShowMultiNode {
   std::vector<CodeNode*> show_children;
 
  private:
-  const CodeDef::Trace* trace_;
+  const CallStack::Trace* trace_;
   string suffix_;
   std::vector<std::unique_ptr<TFMultiGraphNode>> graph_children_;
   std::map<string, std::unique_ptr<CodeNode>> children_;
diff --git a/tensorflow/core/profiler/internal/tfprof_stats.cc b/tensorflow/core/profiler/internal/tfprof_stats.cc
index eb84bada13..b4b98141f3 100644
--- a/tensorflow/core/profiler/internal/tfprof_stats.cc
+++ b/tensorflow/core/profiler/internal/tfprof_stats.cc
@@ -81,9 +81,12 @@ TFStats::TFStats(const string& filename,
     fprintf(stderr, "Failed to parse profile\n");
     return;
   }
-
+  for (const auto& entry : profile.id_to_string()) {
+    id_to_string_[entry.first] = entry.second;
+  }
   for (const auto& node_pb : profile.nodes()) {
-    std::unique_ptr<TFGraphNode> node(new TFGraphNode(node_pb.second, profile));
+    std::unique_ptr<TFGraphNode> node(
+        new TFGraphNode(node_pb.second, profile, &id_to_string_));
     nodes_map_.insert(std::pair<string, std::unique_ptr<TFGraphNode>>(
         node_pb.second.name(), std::move(node)));
   }
@@ -216,6 +219,11 @@ void TFStats::AddOpLogProto(std::unique_ptr<OpLogProto> op_log) {
   if (!op_log) {
     return;
   }
+  for (const auto& entry : op_log->id_to_string()) {
+    if (id_to_string_.find(entry.first) == id_to_string_.end()) {
+      id_to_string_[entry.first] = entry.second;
+    }
+  }
   for (const OpLogEntry& entry : op_log->log_entries()) {
     auto node = nodes_map_.find(entry.name());
     if (node == nodes_map_.end()) continue;
@@ -227,9 +235,7 @@ void TFStats::AddOpLogProto(std::unique_ptr<OpLogProto> op_log) {
     }
     if (entry.has_code_def()) {
       has_code_traces_ = true;
-      if (node->second->code().traces_size() == 0) {
-        node->second->AddCode(entry.code_def());
-      }
+      node->second->AddCode(entry.code_def(), &id_to_string_);
     }
   }
 }
@@ -269,6 +275,9 @@ void TFStats::AddRunMeta(int64 step, std::unique_ptr<RunMetadata> run_meta) {
 
 void TFStats::WriteProfile(const string& filename) {
   ProfileProto profile;
+  for (const auto& entry : id_to_string_) {
+    (*profile.mutable_id_to_string())[entry.first] = entry.second;
+  }
   for (auto it = nodes_map_.begin(); it != nodes_map_.end(); it++) {
     if (it->second->id() < 0) {
       continue;
diff --git a/tensorflow/core/profiler/internal/tfprof_stats.h b/tensorflow/core/profiler/internal/tfprof_stats.h
index 3b1251152d..bb4baea738 100644
--- a/tensorflow/core/profiler/internal/tfprof_stats.h
+++ b/tensorflow/core/profiler/internal/tfprof_stats.h
@@ -109,6 +109,8 @@ class TFStats {
   std::map<string, std::unique_ptr<TFGraphNode>> nodes_map_;
   GraphNodeProto empty_graph_node_;
   MultiGraphNodeProto empty_multi_graph_node_;
+
+  std::map<int64, string> id_to_string_;
 };
 
 }  // namespace tfprof
diff --git a/tensorflow/core/profiler/tfprof_log.proto b/tensorflow/core/profiler/tfprof_log.proto
index ae571e2540..a1410c7c79 100644
--- a/tensorflow/core/profiler/tfprof_log.proto
+++ b/tensorflow/core/profiler/tfprof_log.proto
@@ -8,10 +8,17 @@ import "tensorflow/core/framework/attr_value.proto";
 message CodeDef {
   repeated Trace traces = 1;
   message Trace {
-    string file = 1;
+    string file = 1 [deprecated = true];  // deprecated by file_id.
+    int64 file_id = 6;
+
     int32 lineno = 2;
-    string function = 3;
-    string line = 4;
+
+    string function = 3 [deprecated = true];  // deprecated by function_id.
+    int64 function_id = 7;
+
+    string line = 4 [deprecated = true];  // deprecated line_id.
+    int64 line_id = 8;
+
     int32 func_start_line = 5;
   }
 }
@@ -32,6 +39,10 @@ message OpLogEntry {
 
 message OpLogProto {
   repeated OpLogEntry log_entries = 1;
+
+  // Maps from id of CodeDef file,function,line to its string
+  // In the future can also map other id of other fields to string.
+  map<int64, string> id_to_string = 2;
 }
 
 // A proto representation of the profiler's profile.
@@ -44,6 +55,10 @@ message ProfileProto {
   bool has_trace = 2;
   // Traced steps.
   repeated int64 steps = 3;
+
+  // Maps from id of CodeDef file,function,line to its string
+  // In the future can also map other id of other fields to string.
+  map<int64, string> id_to_string = 4;
 }
 
 message ProfileNode {
diff --git a/tensorflow/python/profiler/profile_context.py b/tensorflow/python/profiler/profile_context.py
index 1710209ed9..0c31cf8f13 100644
--- a/tensorflow/python/profiler/profile_context.py
+++ b/tensorflow/python/profiler/profile_context.py
@@ -47,56 +47,53 @@ def _profiled_run(self,
   """Overwrites the session.run()."""
   # pylint: disable=protected-access
   # Count the session steps.
-  with self.profile_context._new_step():
+  with self.profile_context._new_step() as step:
     # Fast path if no need for profiling.
-    if self.profile_context._is_fast_path():
-      return self._profiler_run_internal(
-          fetches, feed_dict, options, run_metadata)
-
-    step = self.profile_context._step
-
-    # Maybe trace this step.
-    if self.profile_context._should_trace():
-      # Enable tracing, perform auto profiling or auto dump.
-      if not run_metadata:
-        run_metadata = config_pb2.RunMetadata()
-
-      if not options:
-        options = config_pb2.RunOptions(
-            trace_level=config_pb2.RunOptions.FULL_TRACE)
-        old_trace_level = options.trace_level
+    if not self.profile_context._is_fast_path():
+      # Maybe trace this step.
+      if self.profile_context._should_trace():
+        # Enable tracing, perform auto profiling or auto dump.
+        if not run_metadata:
+          run_metadata = config_pb2.RunMetadata()
+
+        if not options:
+          options = config_pb2.RunOptions(
+              trace_level=config_pb2.RunOptions.FULL_TRACE)
+          old_trace_level = options.trace_level
+        else:
+          old_trace_level = options.trace_level
+          options.trace_level = config_pb2.RunOptions.FULL_TRACE
+
+        ret = self._profiler_run_internal(
+            fetches, feed_dict, options, run_metadata)
+
+        self.profile_context.profiler._graph = self.graph
+        self.profile_context.profiler.add_step(step, run_metadata)
+        options.trace_level = old_trace_level
       else:
-        old_trace_level = options.trace_level
-        options.trace_level = config_pb2.RunOptions.FULL_TRACE
-
-      ret = self._profiler_run_internal(
-          fetches, feed_dict, options, run_metadata)
-
-      self.profile_context.profiler._graph = self.graph
-      self.profile_context.profiler.add_step(step, run_metadata)
-      options.trace_level = old_trace_level
-    else:
-      ret = self._profiler_run_internal(fetches, feed_dict, options)
-
-    # Maybe dump profile.
-    self.profile_context._maybe_dump()
-
-    # Maybe profile:
-    to_profiles = self.profile_context._profile_candidates()
-    for to_prof in to_profiles:
-      cmd, opts, _ = to_prof
-      if cmd == 'graph':
-        self.profile_context.profiler.profile_graph(opts)
-      elif cmd == 'scope':
-        self.profile_context.profiler.profile_name_scope(opts)
-      elif cmd == 'op':
-        self.profile_context.profiler.profile_operations(opts)
-      elif cmd == 'code':
-        self.profile_context.profiler.profile_python(opts)
-      else:
-        raise ValueError('Unknown cmd: %s\n' % cmd)
-
-    return ret
+        ret = self._profiler_run_internal(fetches, feed_dict, options)
+
+      # Maybe dump profile.
+      self.profile_context._maybe_dump()
+
+      # Maybe profile:
+      to_profiles = self.profile_context._profile_candidates()
+      for to_prof in to_profiles:
+        cmd, opts, _ = to_prof
+        if cmd == 'graph':
+          self.profile_context.profiler.profile_graph(opts)
+        elif cmd == 'scope':
+          self.profile_context.profiler.profile_name_scope(opts)
+        elif cmd == 'op':
+          self.profile_context.profiler.profile_operations(opts)
+        elif cmd == 'code':
+          self.profile_context.profiler.profile_python(opts)
+        else:
+          raise ValueError('Unknown cmd: %s\n' % cmd)
+      return ret
+  # Fast no lock path.
+  return self._profiler_run_internal(
+      fetches, feed_dict, options, run_metadata)
   # pylint: enable=protected-access
 
 
@@ -183,10 +180,9 @@ class ProfileContext(object):
   @property
   def profiler(self):
     """Returns the current profiler object."""
-    with self._lock:
-      if not self._profiler:
-        self._profiler = model_analyzer.Profiler(ops.get_default_graph())
-      return self._profiler
+    if not self._profiler:
+      self._profiler = model_analyzer.Profiler(ops.get_default_graph())
+    return self._profiler
 
   def trace_next_step(self):
     """Enables tracing and add traces to profiler at next step."""
@@ -222,10 +218,11 @@ class ProfileContext(object):
 
   @contextlib.contextmanager
   def _new_step(self):
-    yield
-    self._step += 1
-    self._trace_next_step = False
-    self._dump_next_step = False
+    with self._lock:
+      yield self._step
+      self._step += 1
+      self._trace_next_step = False
+      self._dump_next_step = False
 
   def _profile_candidates(self):
     to_profile = []
diff --git a/tensorflow/python/profiler/tfprof_logger.py b/tensorflow/python/profiler/tfprof_logger.py
index 9020f60421..838064a1f0 100644
--- a/tensorflow/python/profiler/tfprof_logger.py
+++ b/tensorflow/python/profiler/tfprof_logger.py
@@ -63,6 +63,15 @@ def _fill_missing_graph_shape(graph, run_meta):
   return graph
 
 
+def _str_id(s, str_to_id):
+  """Maps string to id."""
+  num = str_to_id.get(s, None)
+  if num is None:
+    num = len(str_to_id)
+    str_to_id[s] = num
+  return num
+
+
 def _get_logged_ops(graph, run_meta=None, add_trace=True,
                     add_trainable_var=True):
   """Extract trainable model parameters and FLOPs for ops from a Graph.
@@ -75,12 +84,15 @@ def _get_logged_ops(graph, run_meta=None, add_trace=True,
       '_trainable_variables'.
   Returns:
     logged_ops: dict mapping from op_name to OpLogEntry.
+    string_to_id: dict mapping from string to id.
   """
   if run_meta:
     graph = _fill_missing_graph_shape(graph, run_meta)
 
   op_missing_shape = 0
   logged_ops = {}
+  string_to_id = dict()
+  string_to_id['none'] = len(string_to_id)
   # TODO(xpan): Work with Profiler more efficiently.
   for op in graph.get_operations():
     try:
@@ -101,10 +113,10 @@ def _get_logged_ops(graph, run_meta=None, add_trace=True,
     if add_trace:
       for tb in op.traceback_with_start_lines:
         trace = entry.code_def.traces.add()
-        trace.file = tb[0] if tb[0] else 'none'
+        trace.file_id = _str_id(tb[0], string_to_id) if tb[0] else 0
         trace.lineno = tb[1] if tb[1] else -1
-        trace.function = tb[2] if tb[2] else 'none'
-        trace.line = tb[3] if tb[3] else 'none'
+        trace.function_id = _str_id(tb[2], string_to_id) if tb[2] else 0
+        trace.line_id = _str_id(tb[3], string_to_id) if tb[3] else 0
         trace.func_start_line = tb[4] if tb[4] else -1
       add_entry = True
 
@@ -124,7 +136,7 @@ def _get_logged_ops(graph, run_meta=None, add_trace=True,
   if op_missing_shape > 0 and not run_meta:
     sys.stderr.write('%d ops no flops stats due to incomplete shapes.\n' %
                      op_missing_shape)
-  return logged_ops
+  return logged_ops, string_to_id
 
 
 def _merge_default_with_oplog(graph, op_log=None, run_meta=None,
@@ -142,7 +154,7 @@ def _merge_default_with_oplog(graph, op_log=None, run_meta=None,
     tmp_op_log: Merged OpLogProto proto.
   """
   tmp_op_log = tfprof_log_pb2.OpLogProto()
-  logged_ops = _get_logged_ops(
+  logged_ops, string_to_id = _get_logged_ops(
       graph, run_meta, add_trace=add_trace, add_trainable_var=add_trainable_var)
 
   if not op_log:
@@ -161,6 +173,9 @@ def _merge_default_with_oplog(graph, op_log=None, run_meta=None,
       else:
         all_ops[op_name] = entry
     tmp_op_log.log_entries.extend(all_ops.values())
+
+  for s, i in six.iteritems(string_to_id):
+    tmp_op_log.id_to_string[i] = s
   return tmp_op_log
 
 
diff --git a/tensorflow/tools/api/golden/tensorflow.profiler.-op-log-proto.-id-to-string-entry.pbtxt b/tensorflow/tools/api/golden/tensorflow.profiler.-op-log-proto.-id-to-string-entry.pbtxt
new file mode 100644
index 0000000000..8c4727cf35
--- /dev/null
+++ b/tensorflow/tools/api/golden/tensorflow.profiler.-op-log-proto.-id-to-string-entry.pbtxt
@@ -0,0 +1,84 @@
+path: "tensorflow.profiler.OpLogProto.IdToStringEntry"
+tf_class {
+  is_instance: "<class \'tensorflow.core.profiler.tfprof_log_pb2.IdToStringEntry\'>"
+  is_instance: "<type \'google.protobuf.pyext._message.CMessage\'>"
+  member {
+    name: "DESCRIPTOR"
+    mtype: "<type \'google.protobuf.pyext._message.MessageDescriptor\'>"
+  }
+  member {
+    name: "Extensions"
+    mtype: "<type \'getset_descriptor\'>"
+  }
+  member {
+    name: "KEY_FIELD_NUMBER"
+    mtype: "<type \'int\'>"
+  }
+  member {
+    name: "VALUE_FIELD_NUMBER"
+    mtype: "<type \'int\'>"
+  }
+  member_method {
+    name: "ByteSize"
+  }
+  member_method {
+    name: "Clear"
+  }
+  member_method {
+    name: "ClearExtension"
+  }
+  member_method {
+    name: "ClearField"
+  }
+  member_method {
+    name: "CopyFrom"
+  }
+  member_method {
+    name: "DiscardUnknownFields"
+  }
+  member_method {
+    name: "FindInitializationErrors"
+  }
+  member_method {
+    name: "FromString"
+  }
+  member_method {
+    name: "HasExtension"
+  }
+  member_method {
+    name: "HasField"
+  }
+  member_method {
+    name: "IsInitialized"
+  }
+  member_method {
+    name: "ListFields"
+  }
+  member_method {
+    name: "MergeFrom"
+  }
+  member_method {
+    name: "MergeFromString"
+  }
+  member_method {
+    name: "ParseFromString"
+  }
+  member_method {
+    name: "RegisterExtension"
+  }
+  member_method {
+    name: "SerializePartialToString"
+  }
+  member_method {
+    name: "SerializeToString"
+  }
+  member_method {
+    name: "SetInParent"
+  }
+  member_method {
+    name: "WhichOneof"
+  }
+  member_method {
+    name: "__init__"
+  }
+}
diff --git a/tensorflow/tools/api/golden/tensorflow.profiler.-op-log-proto.pbtxt b/tensorflow/tools/api/golden/tensorflow.profiler.-op-log-proto.pbtxt
index c5f9c78c9e..1071a82b5c 100644
--- a/tensorflow/tools/api/golden/tensorflow.profiler.-op-log-proto.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.profiler.-op-log-proto.pbtxt
@@ -10,6 +10,14 @@ tf_class {
     name: "Extensions"
     mtype: "<type \'getset_descriptor\'>"
   }
+  member {
+    name: "ID_TO_STRING_FIELD_NUMBER"
+    mtype: "<type \'int\'>"
+  }
+  member {
+    name: "IdToStringEntry"
+    mtype: "<class \'google.protobuf.pyext.cpp_message.GeneratedProtocolMessageType\'>"
+  }
   member {
     name: "LOG_ENTRIES_FIELD_NUMBER"
     mtype: "<type \'int\'>"
-- 
GitLab


From 0c563bd8e163dbee028e4a762298ba950684ea91 Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Thu, 12 Oct 2017 16:30:00 -0700
Subject: [PATCH 330/909] Fix cuDNN version string in CUDA9 cuDNN7 docker
 image.

---
 tensorflow/tools/docker/Dockerfile.devel-gpu-cuda9-cudnn7 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/tools/docker/Dockerfile.devel-gpu-cuda9-cudnn7 b/tensorflow/tools/docker/Dockerfile.devel-gpu-cuda9-cudnn7
index ac1a437031..75351ecfba 100644
--- a/tensorflow/tools/docker/Dockerfile.devel-gpu-cuda9-cudnn7
+++ b/tensorflow/tools/docker/Dockerfile.devel-gpu-cuda9-cudnn7
@@ -85,7 +85,7 @@ ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH
 ENV TF_NEED_CUDA 1
 ENV TF_CUDA_COMPUTE_CAPABILITIES 3.0,3.5,5.2,6.0,6.1
 ENV TF_CUDA_VERSION 9.0
-ENV TF_CUDNN_VERSION 7.0
+ENV TF_CUDNN_VERSION 7
 RUN ./configure
 
 RUN LD_LIBRARY_PATH=/usr/local/cuda/lib64/stubs:${LD_LIBRARY_PATH} \
-- 
GitLab


From 6ef542822b8760de4beeb4ad2602fb863888d47f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 12 Oct 2017 17:03:10 -0700
Subject: [PATCH 331/909] BUILD cleanup in contrib/boosted_trees

PiperOrigin-RevId: 172034428
---
 tensorflow/contrib/boosted_trees/BUILD | 29 +++++++-------------------
 1 file changed, 8 insertions(+), 21 deletions(-)

diff --git a/tensorflow/contrib/boosted_trees/BUILD b/tensorflow/contrib/boosted_trees/BUILD
index f3ae4e3092..1b85c260c0 100644
--- a/tensorflow/contrib/boosted_trees/BUILD
+++ b/tensorflow/contrib/boosted_trees/BUILD
@@ -302,11 +302,8 @@ tf_custom_op_py_library(
     srcs_version = "PY2AND3",
     deps = [
         "//tensorflow/contrib/util:util_py",
-        "//tensorflow/core:protos_all_py",
         "//tensorflow/python:errors",
-        "//tensorflow/python:framework_for_generated_wrappers",
         "//tensorflow/python:platform",
-        "//tensorflow/python:resources",
     ],
 )
 
@@ -345,21 +342,17 @@ tf_custom_op_py_library(
     deps = [
         ":boosted_trees_ops_loader",
         ":gen_model_ops_py",
-        "//tensorflow/contrib/util:util_py",
-        "//tensorflow/core:protos_all_py",
         "//tensorflow/python:framework_for_generated_wrappers",
         "//tensorflow/python:resources",
+        "//tensorflow/python:training",
     ],
 )
 
 tf_kernel_library(
     name = "model_ops_kernels",
-    srcs = [
-        "kernels/model_ops.cc",
-    ],
+    srcs = ["kernels/model_ops.cc"],
     deps = [
         "//tensorflow/contrib/boosted_trees/lib:utils",
-        "//tensorflow/contrib/boosted_trees/proto:tree_config_proto_cc",
         "//tensorflow/contrib/boosted_trees/resources:decision_tree_ensemble_resource",
         "//tensorflow/core:framework_headers_lib",
         "//third_party/eigen3",
@@ -423,8 +416,6 @@ tf_custom_op_py_library(
     deps = [
         ":boosted_trees_ops_loader",
         ":gen_split_handler_ops_py",
-        "//tensorflow/contrib/util:util_py",
-        "//tensorflow/python:framework_for_generated_wrappers",
     ],
 )
 
@@ -472,17 +463,16 @@ tf_custom_op_py_library(
 
 tf_kernel_library(
     name = "training_ops_kernels",
-    srcs = [
-        "kernels/training_ops.cc",
-    ],
+    srcs = ["kernels/training_ops.cc"],
     deps = [
         "//tensorflow/contrib/boosted_trees/lib:utils",
+        "//tensorflow/contrib/boosted_trees/lib:weighted_quantiles",
         "//tensorflow/contrib/boosted_trees/proto:learner_proto_cc",
+        "//tensorflow/contrib/boosted_trees/proto:quantiles_proto_cc",
         "//tensorflow/contrib/boosted_trees/proto:split_info_proto_cc",
-        "//tensorflow/contrib/boosted_trees/proto:tree_config_proto_cc",
         "//tensorflow/contrib/boosted_trees/resources:decision_tree_ensemble_resource",
+        "//tensorflow/contrib/boosted_trees/resources:quantile_stream_resource",
         "//tensorflow/core:framework_headers_lib",
-        "//third_party/eigen3",
     ],
     alwayslink = 1,
 )
@@ -519,9 +509,7 @@ tf_custom_op_py_library(
 
 tf_kernel_library(
     name = "prediction_ops_kernels",
-    srcs = [
-        "kernels/prediction_ops.cc",
-    ],
+    srcs = ["kernels/prediction_ops.cc"],
     deps = [
         "//tensorflow/contrib/boosted_trees/lib:example_partitioner",
         "//tensorflow/contrib/boosted_trees/lib:models",
@@ -560,10 +548,9 @@ tf_custom_op_py_library(
         ":batch_ops_utils_py",
         ":boosted_trees_ops_loader",
         ":gen_quantile_ops_py_wrap",
-        "//tensorflow/contrib/util:util_py",
-        "//tensorflow/core:protos_all_py",
         "//tensorflow/python:framework_for_generated_wrappers",
         "//tensorflow/python:resources",
+        "//tensorflow/python:sparse_tensor",
         "//tensorflow/python:training",
     ],
 )
-- 
GitLab


From 4b178957917d95fbe6305381764e39453f6bb8d0 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 12 Oct 2017 17:33:22 -0700
Subject: [PATCH 332/909] typo

PiperOrigin-RevId: 172037998
---
 tensorflow/docs_src/tutorials/image_retraining.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/docs_src/tutorials/image_retraining.md b/tensorflow/docs_src/tutorials/image_retraining.md
index 90652ac405..5708b27278 100644
--- a/tensorflow/docs_src/tutorials/image_retraining.md
+++ b/tensorflow/docs_src/tutorials/image_retraining.md
@@ -272,7 +272,7 @@ them destroys their meaning.
 
 There are several other parameters you can try adjusting to see if they help
 your results. The `--learning_rate` controls the magnitude of the updates to the
-final layer during training. Intuitively if this is smaller than the learning
+final layer during training. Intuitively if this is smaller then the learning
 will take longer, but it can end up helping the overall precision. That's not
 always the case though, so you need to experiment carefully to see what works
 for your case. The `--train_batch_size` controls how many images are examined
-- 
GitLab


From 915a8ac568f0a67d6000ab70a665817deff7888c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 12 Oct 2017 17:41:09 -0700
Subject: [PATCH 333/909] [TF:XLA] Implement BitwiseAnd, BitwiseOr, and Invert
 operators.

PiperOrigin-RevId: 172038787
---
 tensorflow/compiler/tests/BUILD               |   1 +
 tensorflow/compiler/tests/binary_ops_test.py  |  15 ++
 tensorflow/compiler/tests/randomized_tests.cc |  42 +++-
 tensorflow/compiler/tests/unary_ops_test.py   |   9 +
 .../compiler/tf2xla/kernels/binary_ops.cc     |   2 +
 .../compiler/tf2xla/kernels/unary_ops.cc      |   1 +
 .../xla/service/elemental_ir_emitter.cc       |  23 +-
 .../compiler/xla/service/shape_inference.cc   |  15 +-
 .../xla/tests/array_elementwise_ops_test.cc   | 203 +++++++++++++++++-
 .../xla/tests/scalar_computations_test.cc     |  68 +++++-
 10 files changed, 349 insertions(+), 30 deletions(-)

diff --git a/tensorflow/compiler/tests/BUILD b/tensorflow/compiler/tests/BUILD
index 72a0360de2..0eed475140 100644
--- a/tensorflow/compiler/tests/BUILD
+++ b/tensorflow/compiler/tests/BUILD
@@ -103,6 +103,7 @@ tf_xla_py_test(
     deps = [
         ":xla_test",
         "//tensorflow/python:array_ops",
+        "//tensorflow/python:bitwise_ops",
         "//tensorflow/python:framework_for_generated_wrappers",
         "//tensorflow/python:math_ops",
         "//tensorflow/python:math_ops_gen",
diff --git a/tensorflow/compiler/tests/binary_ops_test.py b/tensorflow/compiler/tests/binary_ops_test.py
index 792c01327c..44b32b1668 100644
--- a/tensorflow/compiler/tests/binary_ops_test.py
+++ b/tensorflow/compiler/tests/binary_ops_test.py
@@ -24,6 +24,7 @@ from tensorflow.compiler.tests.xla_test import XLATestCase
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import bitwise_ops
 from tensorflow.python.ops import gen_math_ops
 from tensorflow.python.ops import gen_nn_ops
 from tensorflow.python.ops import math_ops
@@ -45,6 +46,10 @@ class BinaryOpsTest(XLATestCase):
         equality_test = self.assertAllClose
       equality_test(result, expected, rtol=1e-3)
 
+  def _testSymmetricBinary(self, op, a, b, expected, equality_test=None):
+    self._testBinary(op, a, b, expected, equality_test)
+    self._testBinary(op, b, a, expected, equality_test)
+
   def ListsAreClose(self, result, expected, rtol):
     """Tests closeness of two lists of floats."""
     self.assertEqual(len(result), len(expected))
@@ -193,6 +198,16 @@ class BinaryOpsTest(XLATestCase):
           np.array([3, 3, -1, -9, -8], dtype=dtype),
           np.array([2, -2, 7, 2, -4], dtype=dtype),
           expected=np.array([1, -1, 0, -4, 2], dtype=dtype))
+      self._testSymmetricBinary(
+          bitwise_ops.bitwise_and,
+          np.array([0b1, 0b101, 0b1000], dtype=dtype),
+          np.array([0b0, 0b101, 0b1001], dtype=dtype),
+          expected=np.array([0b0, 0b101, 0b1000], dtype=dtype))
+      self._testSymmetricBinary(
+          bitwise_ops.bitwise_or,
+          np.array([0b1, 0b101, 0b1000], dtype=dtype),
+          np.array([0b0, 0b101, 0b1001], dtype=dtype),
+          expected=np.array([0b1, 0b101, 0b1001], dtype=dtype))
 
   def testNumericOps(self):
     for dtype in self.numeric_types:
diff --git a/tensorflow/compiler/tests/randomized_tests.cc b/tensorflow/compiler/tests/randomized_tests.cc
index fef12d9397..56e10a1587 100644
--- a/tensorflow/compiler/tests/randomized_tests.cc
+++ b/tensorflow/compiler/tests/randomized_tests.cc
@@ -1168,6 +1168,28 @@ TEST_F(OpTest, BiasAddV1) {
   });
 }
 
+TEST_F(OpTest, BitwiseAnd) {
+  Repeatedly([this]() {
+    DataType type = DT_INT32;
+    auto dims = BroadcastableDims();
+    return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("BitwiseAnd")
+                                             .RandomInput(type, dims.first)
+                                             .RandomInput(type, dims.second)
+                                             .Attr("T", type));
+  });
+}
+
+TEST_F(OpTest, BitwiseOr) {
+  Repeatedly([this]() {
+    DataType type = DT_INT32;
+    auto dims = BroadcastableDims();
+    return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("BitwiseOr")
+                                             .RandomInput(type, dims.first)
+                                             .RandomInput(type, dims.second)
+                                             .Attr("T", type));
+  });
+}
+
 TEST_F(OpTest, BroadcastArgs) {
   Repeatedly([this]() {
     // TODO(phawkins): only int32 seems to be implemented in Tensorflow.
@@ -1729,6 +1751,14 @@ TEST_F(OpTest, GreaterEqual) {
   });
 }
 
+TEST_F(OpTest, Invert) {
+  Repeatedly([this]() {
+    DataType type = DT_INT32;
+    return ExpectTfAndXlaOutputsAreClose(
+        OpTestBuilder("Invert").RandomInput(type).Attr("T", type));
+  });
+}
+
 TEST_F(OpTest, L2Loss) {
   Repeatedly([this]() {
     DataType type = DT_FLOAT;
@@ -1791,28 +1821,28 @@ TEST_F(OpTest, Log1p) {
   });
 }
 
-TEST_F(OpTest, BooleanAnd) {
+TEST_F(OpTest, LogicalAnd) {
   Repeatedly([this]() {
     auto dims = BroadcastableDims();
     return ExpectTfAndXlaOutputsAreClose(
-        OpTestBuilder("BooleanAnd")
+        OpTestBuilder("LogicalAnd")
             .RandomInput(DT_BOOL, dims.first)
             .RandomInput(DT_BOOL, dims.second));
   });
 }
 
-TEST_F(OpTest, BooleanNot) {
+TEST_F(OpTest, LogicalNot) {
   Repeatedly([this]() {
     return ExpectTfAndXlaOutputsAreClose(
-        OpTestBuilder("BooleanNot").RandomInput(DT_BOOL));
+        OpTestBuilder("LogicalNot").RandomInput(DT_BOOL));
   });
 }
 
-TEST_F(OpTest, BooleanOr) {
+TEST_F(OpTest, LogicalOr) {
   Repeatedly([this]() {
     auto dims = BroadcastableDims();
     return ExpectTfAndXlaOutputsAreClose(
-        OpTestBuilder("BooleanOr")
+        OpTestBuilder("LogicalOr")
             .RandomInput(DT_BOOL, dims.first)
             .RandomInput(DT_BOOL, dims.second));
   });
diff --git a/tensorflow/compiler/tests/unary_ops_test.py b/tensorflow/compiler/tests/unary_ops_test.py
index 6f19834160..71221b284d 100644
--- a/tensorflow/compiler/tests/unary_ops_test.py
+++ b/tensorflow/compiler/tests/unary_ops_test.py
@@ -26,6 +26,7 @@ from six.moves import xrange  # pylint: disable=redefined-builtin
 from tensorflow.compiler.tests.xla_test import XLATestCase
 from tensorflow.python.framework import dtypes
 from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import bitwise_ops
 from tensorflow.python.ops import gen_nn_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import nn_ops
@@ -327,6 +328,13 @@ class UnaryOpsTest(XLATestCase):
           np.array([-1, -0.5, 0, 0.3], dtype=dtype),
           expected=np.array([-1, -64.0 / 127, 0, 38.0 / 127], dtype=dtype))
 
+  def testIntOps(self):
+    for dtype in self.int_types:
+      self._assertOpOutputMatchesExpected(
+          bitwise_ops.invert,
+          np.array([0, -1, 1, 16, 42], dtype=dtype),
+          expected=np.array([-1, 0, -2, -17, -43], dtype=dtype))
+
   def testNumericOps(self):
     for dtype in self.numeric_types:
       self._assertOpOutputMatchesExpected(
@@ -558,5 +566,6 @@ class UnaryOpsTest(XLATestCase):
           log_eps + ten, -log_eps, -log_eps - one, -log_eps + one,
           -log_eps - ten, -log_eps + ten], dtype)
 
+
 if __name__ == "__main__":
   googletest.main()
diff --git a/tensorflow/compiler/tf2xla/kernels/binary_ops.cc b/tensorflow/compiler/tf2xla/kernels/binary_ops.cc
index a180f1e4d9..d635507989 100644
--- a/tensorflow/compiler/tf2xla/kernels/binary_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/binary_ops.cc
@@ -96,6 +96,8 @@ static xla::ComputationDataHandle FloorModImpl(xla::ComputationBuilder* b,
 XLA_MAKE_BINARY(FloorMod,
                 FloorModImpl(b, input_type(0), lhs, rhs, broadcast_helper));
 
+XLA_MAKE_BINARY(BitwiseAnd, b->And(lhs, rhs, extend_dimensions));
+XLA_MAKE_BINARY(BitwiseOr, b->Or(lhs, rhs, extend_dimensions));
 XLA_MAKE_BINARY(LogicalAnd, b->And(lhs, rhs, extend_dimensions));
 XLA_MAKE_BINARY(LogicalOr, b->Or(lhs, rhs, extend_dimensions));
 XLA_MAKE_BINARY(Mod, b->Rem(lhs, rhs, extend_dimensions));
diff --git a/tensorflow/compiler/tf2xla/kernels/unary_ops.cc b/tensorflow/compiler/tf2xla/kernels/unary_ops.cc
index 8f04fc94be..651bbe2b40 100644
--- a/tensorflow/compiler/tf2xla/kernels/unary_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/unary_ops.cc
@@ -87,6 +87,7 @@ XLAJIT_MAKE_UNARY(Log, b->Log(x));
 // TODO(b/34703906): use a more accurate implementation of log1p.
 XLAJIT_MAKE_UNARY(Log1p, b->Log(b->Add(XlaHelpers::One(b, input_type(0)), x)));
 
+XLAJIT_MAKE_UNARY(Invert, b->Not(x));
 XLAJIT_MAKE_UNARY(LogicalNot, b->Not(x));
 XLAJIT_MAKE_UNARY(Neg, b->Neg(x));
 
diff --git a/tensorflow/compiler/xla/service/elemental_ir_emitter.cc b/tensorflow/compiler/xla/service/elemental_ir_emitter.cc
index 3a8f70a8ef..fb4d233d04 100644
--- a/tensorflow/compiler/xla/service/elemental_ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/elemental_ir_emitter.cc
@@ -126,14 +126,21 @@ StatusOr<llvm::Value*> ElementalIrEmitter::EmitIntegerUnaryOp(
     }
     case HloOpcode::kNegate:
       return ir_builder_->CreateNeg(operand_value);
-    case HloOpcode::kNot:
-      // It is not sufficient to just call CreateNot() here because a PRED is
-      // represented as an i8 and the truth value is stored only in the bottom
-      // bit.
-      return ir_builder_->CreateZExt(
-          ir_builder_->CreateNot(ir_builder_->CreateTrunc(
-              operand_value, ir_builder_->getInt1Ty())),
-          llvm_ir::PrimitiveTypeToIrType(PRED, ir_builder_));
+    case HloOpcode::kNot: {
+      auto type = op->shape().element_type();
+      if (type == PRED) {
+        // It is not sufficient to just call CreateNot() here because a PRED
+        // is represented as an i8 and the truth value is stored only in the
+        // bottom bit.
+        return ir_builder_->CreateZExt(
+            ir_builder_->CreateNot(ir_builder_->CreateTrunc(
+                operand_value, ir_builder_->getInt1Ty())),
+            llvm_ir::PrimitiveTypeToIrType(PRED, ir_builder_));
+      } else if (primitive_util::IsIntegralType(type)) {
+        return ir_builder_->CreateNot(operand_value);
+      }
+      return Unimplemented("unary op Not is not defined for type '%d'", type);
+    }
     default:
       return Unimplemented("unary integer op '%s'",
                            HloOpcodeString(op->opcode()).c_str());
diff --git a/tensorflow/compiler/xla/service/shape_inference.cc b/tensorflow/compiler/xla/service/shape_inference.cc
index a9f65331e2..a091a067c1 100644
--- a/tensorflow/compiler/xla/service/shape_inference.cc
+++ b/tensorflow/compiler/xla/service/shape_inference.cc
@@ -323,10 +323,11 @@ StatusOr<Shape> InferWindowOutputShape(const Shape& base_shape,
       return arg;
 
     case UNOP_NOT:
-      if (arg.element_type() != PRED) {
+      if (arg.element_type() != PRED &&
+          !primitive_util::IsIntegralType(arg.element_type())) {
         return InvalidArgument(
-            "expected pred element type in argument to logical-not operation; "
-            "got %s",
+            "expected pred or an integral element type in argument to not "
+            "operation; got %s",
             PrimitiveType_Name(arg.element_type()).c_str());
       }
       return arg;
@@ -752,15 +753,15 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(
 
     case BINOP_AND:
     case BINOP_OR:
-      if (lhs.element_type() != PRED) {
+      if (lhs.element_type() != PRED &&
+          !primitive_util::IsIntegralType(lhs.element_type())) {
         return InvalidArgument(
-            "expected pred element type in argument to logical and/or "
-            "operation; got %s",
+            "expected pred or integral type in argument to and/or operation; "
+            "got %s",
             PrimitiveType_Name(lhs.element_type()).c_str());
       }
       return InferElementwiseBinaryOpShape(operation, lhs, rhs,
                                            broadcast_dimensions);
-
     case BINOP_EQ:
     case BINOP_GE:
     case BINOP_GT:
diff --git a/tensorflow/compiler/xla/tests/array_elementwise_ops_test.cc b/tensorflow/compiler/xla/tests/array_elementwise_ops_test.cc
index 08b39b6379..eb931dcff3 100644
--- a/tensorflow/compiler/xla/tests/array_elementwise_ops_test.cc
+++ b/tensorflow/compiler/xla/tests/array_elementwise_ops_test.cc
@@ -496,7 +496,7 @@ XLA_TEST_F(ArrayElementwiseOpTest, MulTwoConstantU32s) {
   ComputeAndCompareR1<uint32>(&builder, expected, {});
 }
 
-XLA_TEST_F(ArrayElementwiseOpTest, BooleanAnd) {
+XLA_TEST_F(ArrayElementwiseOpTest, AndPredR1) {
   ComputationBuilder builder(client_, TestName());
   auto a = builder.ConstantR1<bool>({false, false, true, true});
   auto b = builder.ConstantR1<bool>({false, true, false, true});
@@ -505,7 +505,17 @@ XLA_TEST_F(ArrayElementwiseOpTest, BooleanAnd) {
   ComputeAndCompareR1<bool>(&builder, {false, false, false, true}, {});
 }
 
-XLA_TEST_F(ArrayElementwiseOpTest, BooleanAndZeroElement) {
+XLA_TEST_F(ArrayElementwiseOpTest, AndPredR2) {
+  ComputationBuilder builder(client_, TestName());
+  auto a = builder.ConstantR2<bool>({{false, false}, {true, true}});
+  auto b = builder.ConstantR2<bool>({{false, true}, {false, true}});
+  auto out = builder.And(a, b);
+
+  Array2D<bool> expected_array({{false, false}, {false, true}});
+  ComputeAndCompareR2<bool>(&builder, expected_array, {});
+}
+
+XLA_TEST_F(ArrayElementwiseOpTest, AndZeroElementPredR1) {
   ComputationBuilder builder(client_, TestName());
   auto a = builder.ConstantR1<bool>({});
   auto b = builder.ConstantR1<bool>({});
@@ -514,7 +524,63 @@ XLA_TEST_F(ArrayElementwiseOpTest, BooleanAndZeroElement) {
   ComputeAndCompareR1<bool>(&builder, {}, {});
 }
 
-XLA_TEST_F(ArrayElementwiseOpTest, BooleanOr) {
+XLA_TEST_F(ArrayElementwiseOpTest, AndS32R1) {
+  ComputationBuilder builder(client_, TestName());
+  auto a = builder.ConstantR1<int32>({0, -1, -8});
+  auto b = builder.ConstantR1<int32>({5, -7, 12});
+  auto out = builder.And(a, b);
+
+  ComputeAndCompareR1<int32>(&builder, {0, -7, 8}, {});
+}
+
+XLA_TEST_F(ArrayElementwiseOpTest, AndS32R2) {
+  ComputationBuilder builder(client_, TestName());
+  auto a = builder.ConstantR2<int32>({{0, -5}, {-1, 5}});
+  auto b = builder.ConstantR2<int32>({{1, -6}, {4, 5}});
+  auto out = builder.And(a, b);
+
+  Array2D<int32> expected_array({{0, -6}, {4, 5}});
+  ComputeAndCompareR2<int32>(&builder, expected_array, {});
+}
+
+XLA_TEST_F(ArrayElementwiseOpTest, AndZeroElementS32R1) {
+  ComputationBuilder builder(client_, TestName());
+  auto a = builder.ConstantR1<int32>({});
+  auto b = builder.ConstantR1<int32>({});
+  auto out = builder.And(a, b);
+
+  ComputeAndCompareR1<int32>(&builder, {}, {});
+}
+
+XLA_TEST_F(ArrayElementwiseOpTest, AndU32R1) {
+  ComputationBuilder builder(client_, TestName());
+  auto a = builder.ConstantR1<int32>({0, 1, 8});
+  auto b = builder.ConstantR1<int32>({5, 7, 12});
+  auto out = builder.And(a, b);
+
+  ComputeAndCompareR1<int32>(&builder, {0, 1, 8}, {});
+}
+
+XLA_TEST_F(ArrayElementwiseOpTest, AndU32R2) {
+  ComputationBuilder builder(client_, TestName());
+  auto a = builder.ConstantR2<uint32>({{0, 1}, {3, 8}});
+  auto b = builder.ConstantR2<uint32>({{1, 0}, {7, 6}});
+  auto out = builder.And(a, b);
+
+  Array2D<uint32> expected_array({{0, 0}, {3, 0}});
+  ComputeAndCompareR2<uint32>(&builder, expected_array, {});
+}
+
+XLA_TEST_F(ArrayElementwiseOpTest, AndZeroElementU32R1) {
+  ComputationBuilder builder(client_, TestName());
+  auto a = builder.ConstantR1<uint32>({});
+  auto b = builder.ConstantR1<uint32>({});
+  auto out = builder.And(a, b);
+
+  ComputeAndCompareR1<uint32>(&builder, {}, {});
+}
+
+XLA_TEST_F(ArrayElementwiseOpTest, OrPredR1) {
   ComputationBuilder builder(client_, TestName());
   auto a = builder.ConstantR1<bool>({false, false, true, true});
   auto b = builder.ConstantR1<bool>({false, true, false, true});
@@ -523,7 +589,17 @@ XLA_TEST_F(ArrayElementwiseOpTest, BooleanOr) {
   ComputeAndCompareR1<bool>(&builder, {false, true, true, true}, {});
 }
 
-XLA_TEST_F(ArrayElementwiseOpTest, BooleanOrZeroElement) {
+XLA_TEST_F(ArrayElementwiseOpTest, OrPredR2) {
+  ComputationBuilder builder(client_, TestName());
+  auto a = builder.ConstantR2<bool>({{false, false}, {true, true}});
+  auto b = builder.ConstantR2<bool>({{false, true}, {false, true}});
+  auto out = builder.Or(a, b);
+
+  Array2D<bool> expected_array({{false, true}, {true, true}});
+  ComputeAndCompareR2<bool>(&builder, expected_array, {});
+}
+
+XLA_TEST_F(ArrayElementwiseOpTest, OrZeroElementPredR1) {
   ComputationBuilder builder(client_, TestName());
   auto a = builder.ConstantR1<bool>({});
   auto b = builder.ConstantR1<bool>({});
@@ -532,7 +608,63 @@ XLA_TEST_F(ArrayElementwiseOpTest, BooleanOrZeroElement) {
   ComputeAndCompareR1<bool>(&builder, {}, {});
 }
 
-XLA_TEST_F(ArrayElementwiseOpTest, BooleanNot) {
+XLA_TEST_F(ArrayElementwiseOpTest, OrS32R1) {
+  ComputationBuilder builder(client_, TestName());
+  auto a = builder.ConstantR1<int32>({0, -1, 8});
+  auto b = builder.ConstantR1<int32>({5, -7, 4});
+  auto out = builder.Or(a, b);
+
+  ComputeAndCompareR1<int32>(&builder, {5, -1, 12}, {});
+}
+
+XLA_TEST_F(ArrayElementwiseOpTest, OrS32R2) {
+  ComputationBuilder builder(client_, TestName());
+  auto a = builder.ConstantR2<int32>({{0, -1}, {8, 8}});
+  auto b = builder.ConstantR2<int32>({{5, -7}, {4, 1}});
+  auto out = builder.Or(a, b);
+
+  Array2D<int32> expected_array({{5, -1}, {12, 9}});
+  ComputeAndCompareR2<int32>(&builder, expected_array, {});
+}
+
+XLA_TEST_F(ArrayElementwiseOpTest, OrZeroElementS32R1) {
+  ComputationBuilder builder(client_, TestName());
+  auto a = builder.ConstantR1<int32>({});
+  auto b = builder.ConstantR1<int32>({});
+  auto out = builder.Or(a, b);
+
+  ComputeAndCompareR1<int32>(&builder, {}, {});
+}
+
+XLA_TEST_F(ArrayElementwiseOpTest, OrU32R1) {
+  ComputationBuilder builder(client_, TestName());
+  auto a = builder.ConstantR1<uint32>({0, 1, 8});
+  auto b = builder.ConstantR1<uint32>({5, 7, 4});
+  auto out = builder.Or(a, b);
+
+  ComputeAndCompareR1<uint32>(&builder, {5, 7, 12}, {});
+}
+
+XLA_TEST_F(ArrayElementwiseOpTest, OrU32R2) {
+  ComputationBuilder builder(client_, TestName());
+  auto a = builder.ConstantR2<uint32>({{0, 1}, {8, 8}});
+  auto b = builder.ConstantR2<uint32>({{5, 7}, {4, 1}});
+  auto out = builder.Or(a, b);
+
+  Array2D<uint32> expected_array({{5, 7}, {12, 9}});
+  ComputeAndCompareR2<uint32>(&builder, expected_array, {});
+}
+
+XLA_TEST_F(ArrayElementwiseOpTest, OrZeroElementU32R1) {
+  ComputationBuilder builder(client_, TestName());
+  auto a = builder.ConstantR1<uint32>({});
+  auto b = builder.ConstantR1<uint32>({});
+  auto out = builder.Or(a, b);
+
+  ComputeAndCompareR1<uint32>(&builder, {}, {});
+}
+
+XLA_TEST_F(ArrayElementwiseOpTest, NotPredR1) {
   ComputationBuilder builder(client_, TestName());
   auto a = builder.ConstantR1<bool>({false, true, true, false});
   auto out = builder.Not(a);
@@ -540,7 +672,16 @@ XLA_TEST_F(ArrayElementwiseOpTest, BooleanNot) {
   ComputeAndCompareR1<bool>(&builder, {true, false, false, true}, {});
 }
 
-XLA_TEST_F(ArrayElementwiseOpTest, BooleanNotZeroElement) {
+XLA_TEST_F(ArrayElementwiseOpTest, NotPredR2) {
+  ComputationBuilder builder(client_, TestName());
+  auto a = builder.ConstantR2<bool>({{false, true}, {true, false}});
+  auto out = builder.Not(a);
+
+  Array2D<bool> expected_array({{true, false}, {false, true}});
+  ComputeAndCompareR2<bool>(&builder, expected_array, {});
+}
+
+XLA_TEST_F(ArrayElementwiseOpTest, NotZeroElementPredR1) {
   ComputationBuilder builder(client_, TestName());
   auto a = builder.ConstantR1<bool>({});
   auto out = builder.Not(a);
@@ -548,6 +689,56 @@ XLA_TEST_F(ArrayElementwiseOpTest, BooleanNotZeroElement) {
   ComputeAndCompareR1<bool>(&builder, {}, {});
 }
 
+XLA_TEST_F(ArrayElementwiseOpTest, NotS32R1) {
+  ComputationBuilder builder(client_, TestName());
+  auto a = builder.ConstantR1<int32>({-1, 0, 1});
+  auto out = builder.Not(a);
+
+  ComputeAndCompareR1<int32>(&builder, {0, -1, -2}, {});
+}
+
+XLA_TEST_F(ArrayElementwiseOpTest, NotS32R2) {
+  ComputationBuilder builder(client_, TestName());
+  auto a = builder.ConstantR2<int32>({{-1, 0}, {1, 8}});
+  auto out = builder.Not(a);
+
+  Array2D<int32> expected_array({{0, -1}, {-2, -9}});
+  ComputeAndCompareR2<int32>(&builder, expected_array, {});
+}
+
+XLA_TEST_F(ArrayElementwiseOpTest, NotZeroElementS32R1) {
+  ComputationBuilder builder(client_, TestName());
+  auto a = builder.ConstantR1<int32>({});
+  auto out = builder.Not(a);
+
+  ComputeAndCompareR1<int32>(&builder, {}, {});
+}
+
+XLA_TEST_F(ArrayElementwiseOpTest, NotU32R1) {
+  ComputationBuilder builder(client_, TestName());
+  auto a = builder.ConstantR1<uint32>({0, 4294967295});
+  auto out = builder.Not(a);
+
+  ComputeAndCompareR1<uint32>(&builder, {4294967295, 0}, {});
+}
+
+XLA_TEST_F(ArrayElementwiseOpTest, NotU32R2) {
+  ComputationBuilder builder(client_, TestName());
+  auto a = builder.ConstantR2<uint32>({{0, 4294967295}, {1, 4294967294}});
+  auto out = builder.Not(a);
+
+  Array2D<uint32> expected_array({{4294967295, 0}, {4294967294, 1}});
+  ComputeAndCompareR2<uint32>(&builder, expected_array, {});
+}
+
+XLA_TEST_F(ArrayElementwiseOpTest, NotZeroElementU32R1) {
+  ComputationBuilder builder(client_, TestName());
+  auto a = builder.ConstantR1<uint32>({});
+  auto out = builder.Not(a);
+
+  ComputeAndCompareR1<uint32>(&builder, {}, {});
+}
+
 XLA_TEST_F(ArrayElementwiseOpTest, CompareEqF32s) {
   SetFastMathDisabled(true);
   ComputationBuilder builder(client_, TestName());
diff --git a/tensorflow/compiler/xla/tests/scalar_computations_test.cc b/tensorflow/compiler/xla/tests/scalar_computations_test.cc
index da84d185ca..b5e7570778 100644
--- a/tensorflow/compiler/xla/tests/scalar_computations_test.cc
+++ b/tensorflow/compiler/xla/tests/scalar_computations_test.cc
@@ -459,7 +459,7 @@ XLA_TEST_F(ScalarComputationsTest, RemTwoScalarsU32) {
   ComputeAndCompareR0<uint32>(&builder, 2, {});
 }
 
-XLA_TEST_F(ScalarComputationsTest, BooleanAnd) {
+XLA_TEST_F(ScalarComputationsTest, AndBool) {
   for (bool x : {false, true}) {
     for (bool y : {false, true}) {
       ComputationBuilder builder(client_, TestName());
@@ -470,7 +470,29 @@ XLA_TEST_F(ScalarComputationsTest, BooleanAnd) {
   }
 }
 
-XLA_TEST_F(ScalarComputationsTest, BooleanOr) {
+XLA_TEST_F(ScalarComputationsTest, AndS32) {
+  for (int32 x : {0, 8}) {
+    for (int32 y : {1, -16}) {
+      ComputationBuilder builder(client_, TestName());
+      builder.And(builder.ConstantR0<int32>(x), builder.ConstantR0<int32>(y));
+
+      ComputeAndCompareR0<int32>(&builder, x & y, {});
+    }
+  }
+}
+
+XLA_TEST_F(ScalarComputationsTest, AndU32) {
+  for (uint32 x : {0, 8}) {
+    for (uint32 y : {1, 16}) {
+      ComputationBuilder builder(client_, TestName());
+      builder.And(builder.ConstantR0<uint32>(x), builder.ConstantR0<uint32>(y));
+
+      ComputeAndCompareR0<uint32>(&builder, x & y, {});
+    }
+  }
+}
+
+XLA_TEST_F(ScalarComputationsTest, OrBool) {
   for (bool x : {false, true}) {
     for (bool y : {false, true}) {
       ComputationBuilder builder(client_, TestName());
@@ -481,7 +503,29 @@ XLA_TEST_F(ScalarComputationsTest, BooleanOr) {
   }
 }
 
-XLA_TEST_F(ScalarComputationsTest, BooleanNot) {
+XLA_TEST_F(ScalarComputationsTest, OrS32) {
+  for (int32 x : {0, 8}) {
+    for (int32 y : {1, -16}) {
+      ComputationBuilder builder(client_, TestName());
+      builder.Or(builder.ConstantR0<int32>(x), builder.ConstantR0<int32>(y));
+
+      ComputeAndCompareR0<int32>(&builder, x | y, {});
+    }
+  }
+}
+
+XLA_TEST_F(ScalarComputationsTest, OrU32) {
+  for (uint32 x : {0, 8}) {
+    for (uint32 y : {1, 16}) {
+      ComputationBuilder builder(client_, TestName());
+      builder.Or(builder.ConstantR0<uint32>(x), builder.ConstantR0<uint32>(y));
+
+      ComputeAndCompareR0<uint32>(&builder, x | y, {});
+    }
+  }
+}
+
+XLA_TEST_F(ScalarComputationsTest, NotBool) {
   for (bool x : {false, true}) {
     ComputationBuilder builder(client_, TestName());
     builder.Not(builder.ConstantR0<bool>(x));
@@ -490,6 +534,24 @@ XLA_TEST_F(ScalarComputationsTest, BooleanNot) {
   }
 }
 
+XLA_TEST_F(ScalarComputationsTest, NotS32) {
+  for (int32 x : {-1, 0, 1}) {
+    ComputationBuilder builder(client_, TestName());
+    builder.Not(builder.ConstantR0<int32>(x));
+
+    ComputeAndCompareR0<int32>(&builder, ~x, {});
+  }
+}
+
+XLA_TEST_F(ScalarComputationsTest, NotU32) {
+  for (uint32 x : {0, 1, 2}) {
+    ComputationBuilder builder(client_, TestName());
+    builder.Not(builder.ConstantR0<uint32>(x));
+
+    ComputeAndCompareR0<uint32>(&builder, ~x, {});
+  }
+}
+
 XLA_TEST_F(ScalarComputationsTest, SelectScalarTrue) {
   ComputationBuilder builder(client_, TestName());
   builder.Select(builder.ConstantR0<bool>(true),     // The predicate.
-- 
GitLab


From dec1c9296f72e146423d5cb2fffed1c65ef4e8d6 Mon Sep 17 00:00:00 2001
From: Ali Yahya <alive@google.com>
Date: Thu, 12 Oct 2017 17:45:50 -0700
Subject: [PATCH 334/909] TFE: Adds a destructor to ResourceVariables in Python
 that destroys the underlying resource. This makes the lifetime of the
 underlying resource match that of its corresponding Python object.

PiperOrigin-RevId: 172039259
---
 tensorflow/python/BUILD                       |  1 +
 .../resource_variable_ops_test.py             | 10 ++
 .../python/ops/resource_variable_ops.py       |  5 +
 tensorflow/python/training/adam_test.py       | 82 ++++++++--------
 tensorflow/python/training/saver_test.py      | 97 ++++++++++---------
 5 files changed, 107 insertions(+), 88 deletions(-)

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index ac16ca1830..9582fda88f 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -3396,6 +3396,7 @@ cuda_py_test(
         ":training",
         ":platform_test",
         ":client_testlib",
+        ":variable_scope",
         "//third_party/py/numpy",
     ],
 )
diff --git a/tensorflow/python/kernel_tests/resource_variable_ops_test.py b/tensorflow/python/kernel_tests/resource_variable_ops_test.py
index 8cf8286ed1..6f2bc2f752 100644
--- a/tensorflow/python/kernel_tests/resource_variable_ops_test.py
+++ b/tensorflow/python/kernel_tests/resource_variable_ops_test.py
@@ -422,6 +422,16 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase):
       self.assertEqual(1, v1.read_value().numpy())
       self.assertEqual(2, v2.read_value().numpy())
 
+  def testDestruction(self):
+    with context.eager_mode():
+      var = resource_variable_ops.ResourceVariable(initial_value=1.0,
+                                                   name="var8")
+      var.__del__()
+      with self.assertRaisesRegexp(errors.NotFoundError,
+                                   r"Resource .*\/var8\/.* does not exist."):
+        resource_variable_ops.destroy_resource_op(var._handle,
+                                                  ignore_lookup_error=False)
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/ops/resource_variable_ops.py b/tensorflow/python/ops/resource_variable_ops.py
index cbfa141256..99ff02873b 100644
--- a/tensorflow/python/ops/resource_variable_ops.py
+++ b/tensorflow/python/ops/resource_variable_ops.py
@@ -427,6 +427,11 @@ class ResourceVariable(variables.Variable):
     self._constraint = None
   # LINT.ThenChange(//tensorflow/python/eager/graph_callable.py)
 
+  def __del__(self):
+    if context.in_eager_mode():
+      gen_resource_variable_ops.destroy_resource_op(self._handle,
+                                                    ignore_lookup_error=False)
+
   @property
   def dtype(self):
     """The dtype of this variable."""
diff --git a/tensorflow/python/training/adam_test.py b/tensorflow/python/training/adam_test.py
index defcf33714..96de9b921b 100644
--- a/tensorflow/python/training/adam_test.py
+++ b/tensorflow/python/training/adam_test.py
@@ -29,6 +29,7 @@ from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import resource_variable_ops
+from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import test
 from tensorflow.python.training import adam
@@ -152,53 +153,54 @@ class AdamOptimizerTest(test.TestCase):
 
   def doTestBasic(self, use_resource=False):
     for i, dtype in enumerate([dtypes.half, dtypes.float32, dtypes.float64]):
-      # Initialize variables for numpy implementation.
-      m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0
-      var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype)
-      grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype)
-      var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype)
-      grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype)
-
-      if use_resource:
-        var0 = resource_variable_ops.ResourceVariable(
-            var0_np, name="var0_%d" % i)
-        var1 = resource_variable_ops.ResourceVariable(
-            var1_np, name="var1_%d" % i)
-      else:
-        var0 = variables.Variable(var0_np)
-        var1 = variables.Variable(var1_np)
-      grads0 = constant_op.constant(grads0_np)
-      grads1 = constant_op.constant(grads1_np)
-
-      opt = adam.AdamOptimizer()
-      update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
+      with variable_scope.variable_scope("%d" % i):
+        # Initialize variables for numpy implementation.
+        m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0
+        var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype)
+        grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype)
+        var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype)
+        grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype)
 
-      if context.in_graph_mode():
-        self.evaluate(variables.global_variables_initializer())
-        # Fetch params to validate initial values
-        self.assertAllClose([1.0, 2.0], self.evaluate(var0))
-        self.assertAllClose([3.0, 4.0], self.evaluate(var1))
+        if use_resource:
+          var0 = resource_variable_ops.ResourceVariable(
+              var0_np, name="var0_%d" % i)
+          var1 = resource_variable_ops.ResourceVariable(
+              var1_np, name="var1_%d" % i)
+        else:
+          var0 = variables.Variable(var0_np)
+          var1 = variables.Variable(var1_np)
+        grads0 = constant_op.constant(grads0_np)
+        grads1 = constant_op.constant(grads1_np)
 
-      beta1_power, beta2_power = opt._get_beta_accumulators()
+        opt = adam.AdamOptimizer()
+        update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
 
-      # Run 3 steps of Adam
-      for t in range(1, 4):
         if context.in_graph_mode():
-          self.evaluate(update)
-        elif t > 1:
-          opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
+          self.evaluate(variables.global_variables_initializer())
+          # Fetch params to validate initial values
+          self.assertAllClose([1.0, 2.0], self.evaluate(var0))
+          self.assertAllClose([3.0, 4.0], self.evaluate(var1))
 
-        self.assertAllCloseAccordingToType(0.9**(t + 1),
-                                           self.evaluate(beta1_power))
-        self.assertAllCloseAccordingToType(0.999**(t + 1),
-                                           self.evaluate(beta2_power))
+        beta1_power, beta2_power = opt._get_beta_accumulators()
 
-        var0_np, m0, v0 = adam_update_numpy(var0_np, grads0_np, t, m0, v0)
-        var1_np, m1, v1 = adam_update_numpy(var1_np, grads1_np, t, m1, v1)
+        # Run 3 steps of Adam
+        for t in range(1, 4):
+          if context.in_graph_mode():
+            self.evaluate(update)
+          elif t > 1:
+            opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
+
+          self.assertAllCloseAccordingToType(0.9**(t + 1),
+                                             self.evaluate(beta1_power))
+          self.assertAllCloseAccordingToType(0.999**(t + 1),
+                                             self.evaluate(beta2_power))
+
+          var0_np, m0, v0 = adam_update_numpy(var0_np, grads0_np, t, m0, v0)
+          var1_np, m1, v1 = adam_update_numpy(var1_np, grads1_np, t, m1, v1)
 
-        # Validate updated params
-        self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0))
-        self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1))
+          # Validate updated params
+          self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0))
+          self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1))
 
   def testBasic(self):
     with self.test_session():
diff --git a/tensorflow/python/training/saver_test.py b/tensorflow/python/training/saver_test.py
index 07cd67a4b9..a8eb8e5fcf 100644
--- a/tensorflow/python/training/saver_test.py
+++ b/tensorflow/python/training/saver_test.py
@@ -110,32 +110,32 @@ class SaverTest(test.TestCase):
     # Start a second session.  In that session the parameter nodes
     # have not been initialized either.
     with self.test_session(graph=ops_lib.Graph()) as sess:
-      v0 = variable_op(-1.0, name="v0")
-      v1 = variable_op(-1.0, name="v1")
-      v2 = saver_test_utils.CheckpointedOp(name="v2")
+      v0_2 = variable_op(-1.0, name="v0")
+      v1_2 = variable_op(-1.0, name="v1")
+      v2_2 = saver_test_utils.CheckpointedOp(name="v2")
 
       # Assert that the variables are not initialized.
       if context.in_graph_mode():
         self.assertEqual(
             len(variables.report_uninitialized_variables().eval()), 2)
-        self.assertEqual(0, len(v2.keys().eval()))
-        self.assertEqual(0, len(v2.values().eval()))
+        self.assertEqual(0, len(v2_2.keys().eval()))
+        self.assertEqual(0, len(v2_2.values().eval()))
       # Restore the saved values in the parameter nodes.
-      save = saver_module.Saver({"v0": v0, "v1": v1, "v2": v2.saveable})
+      save = saver_module.Saver({"v0": v0_2, "v1": v1_2, "v2": v2_2.saveable})
       save.restore(sess, save_path)
       # Check that the parameter nodes have been restored.
-      self.assertEqual(10.0, self.evaluate(v0))
-      self.assertEqual(20.0, self.evaluate(v1))
-      self.assertEqual(b"k1", self.evaluate(v2.keys()))
-      self.assertEqual(30.0, self.evaluate(v2.values()))
+      self.assertEqual(10.0, self.evaluate(v0_2))
+      self.assertEqual(20.0, self.evaluate(v1_2))
+      self.assertEqual(b"k1", self.evaluate(v2_2.keys()))
+      self.assertEqual(30.0, self.evaluate(v2_2.values()))
 
     # Build another graph with 2 nodes, initialized
     # differently, and a Restore node for them.
     with self.test_session(graph=ops_lib.Graph()) as sess:
-      v0_2 = variable_op(1000.0, name="v0")
-      v1_2 = variable_op(2000.0, name="v1")
-      v2_2 = saver_test_utils.CheckpointedOp(name="v2")
-      v2_init = v2_2.insert("k1000", 3000.0)
+      v0_3 = variable_op(1000.0, name="v0")
+      v1_3 = variable_op(2000.0, name="v1")
+      v2_3 = saver_test_utils.CheckpointedOp(name="v2")
+      v2_init = v2_3.insert("k1000", 3000.0)
 
       # Check that the parameter nodes have been initialized.
       if context.in_graph_mode():
@@ -143,19 +143,19 @@ class SaverTest(test.TestCase):
         self.evaluate(init_all_op)
         # TODO(xpan): Why _mutable_hash_table_v2 doesn't create empty
         # table as it claims in eager mode?
-        self.assertEqual(b"k1000", self.evaluate(v2_2.keys()))
-        self.assertEqual(3000.0, self.evaluate(v2_2.values()))
-      self.assertEqual(1000.0, self.evaluate(v0_2))
-      self.assertEqual(2000.0, self.evaluate(v1_2))
+        self.assertEqual(b"k1000", self.evaluate(v2_3.keys()))
+        self.assertEqual(3000.0, self.evaluate(v2_3.values()))
+      self.assertEqual(1000.0, self.evaluate(v0_3))
+      self.assertEqual(2000.0, self.evaluate(v1_3))
 
       # Restore the values saved earlier in the parameter nodes.
-      save2 = saver_module.Saver({"v0": v0_2, "v1": v1_2, "v2": v2_2.saveable})
+      save2 = saver_module.Saver({"v0": v0_3, "v1": v1_3, "v2": v2_3.saveable})
       save2.restore(sess, save_path)
       # Check that the parameter nodes have been restored.
-      self.assertEqual(10.0, self.evaluate(v0_2))
-      self.assertEqual(20.0, self.evaluate(v1_2))
-      self.assertEqual(b"k1", self.evaluate(v2_2.keys()))
-      self.assertEqual(30.0, self.evaluate(v2_2.values()))
+      self.assertEqual(10.0, self.evaluate(v0_3))
+      self.assertEqual(20.0, self.evaluate(v1_3))
+      self.assertEqual(b"k1", self.evaluate(v2_3.keys()))
+      self.assertEqual(30.0, self.evaluate(v2_3.values()))
 
   def testBasic(self):
     self.basicSaveRestore(variables.Variable)
@@ -487,10 +487,10 @@ class SaverTest(test.TestCase):
       val = save.save(sess, save_path)
       self.assertEqual(save_path, val)
     with self.test_session() as sess:
-      var = resource_variable_ops.ResourceVariable(other_value, name=var_name)
-      save = saver_module.Saver({var_name: var})
+      var2 = resource_variable_ops.ResourceVariable(other_value, name=var_name)
+      save = saver_module.Saver({var_name: var2})
       save.restore(sess, save_path)
-      self.assertAllClose(var_value, self.evaluate(var))
+      self.assertAllClose(var_value, self.evaluate(var2))
 
   def testCacheRereadsFile(self):
     save_path = os.path.join(self.get_temp_dir(), "cache_rereads")
@@ -618,28 +618,29 @@ class SaverTest(test.TestCase):
     global_step_int = 5
     # Save and reload one Variable named "var0".
     self._SaveAndLoad("var0", 0.0, 1.0, save_path)
-    for use_tensor in [True, False]:
-      var = resource_variable_ops.ResourceVariable(1.0, name="var0")
-      save = saver_module.Saver(
-          {
-              var._shared_name: var
-          }, pad_step_number=pad_step_number)
-      if context.in_graph_mode():
-        self.evaluate(var.initializer)
-        sess = ops_lib.get_default_session()
-      else:
-        sess = None
-      if use_tensor:
-        global_step = constant_op.constant(global_step_int)
-        val = save.save(sess, save_path, global_step=global_step)
-      else:
-        val = save.save(sess, save_path, global_step=global_step_int)
-      if pad_step_number:
-        expected_save_path = "%s-%s" % (save_path,
-                                        "{:08d}".format(global_step_int))
-      else:
-        expected_save_path = "%s-%d" % (save_path, global_step_int)
-      self.assertEqual(expected_save_path, val)
+    for i, use_tensor in enumerate([True, False]):
+      with variable_scope.variable_scope("%d" % i):
+        var = resource_variable_ops.ResourceVariable(1.0, name="var0")
+        save = saver_module.Saver(
+            {
+                var._shared_name: var
+            }, pad_step_number=pad_step_number)
+        if context.in_graph_mode():
+          self.evaluate(var.initializer)
+          sess = ops_lib.get_default_session()
+        else:
+          sess = None
+        if use_tensor:
+          global_step = constant_op.constant(global_step_int)
+          val = save.save(sess, save_path, global_step=global_step)
+        else:
+          val = save.save(sess, save_path, global_step=global_step_int)
+        if pad_step_number:
+          expected_save_path = "%s-%s" % (save_path,
+                                          "{:08d}".format(global_step_int))
+        else:
+          expected_save_path = "%s-%d" % (save_path, global_step_int)
+        self.assertEqual(expected_save_path, val)
 
   def testSaveWithGlobalStepWithPadding(self):
     self.testSaveWithGlobalStep(pad_step_number=True)
-- 
GitLab


From 96e9e8e8fc8d027021225fc8bd665968859c8d01 Mon Sep 17 00:00:00 2001
From: namrata-ibm <bhavenamrata@gmail.com>
Date: Fri, 13 Oct 2017 06:29:48 +0530
Subject: [PATCH 335/909] Enable vectorization on z13 (#13659)

---
 tensorflow/core/kernels/sparse_matmul_op.h | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/kernels/sparse_matmul_op.h b/tensorflow/core/kernels/sparse_matmul_op.h
index 308b641b54..cca52558ae 100644
--- a/tensorflow/core/kernels/sparse_matmul_op.h
+++ b/tensorflow/core/kernels/sparse_matmul_op.h
@@ -54,8 +54,9 @@ EIGEN_DEVICE_FUNC inline Packet pexpand_bf16_u(const Packet& from) {
 }
 
 // Specialization non-scalar version on non-sse.
+// Enable vectorization on z13 and higher
 #if defined(EIGEN_VECTORIZE_ALTIVEC) || defined(EIGEN_VECTORIZE_VSX) || \
-    defined(EIGEN_VECTORIZE_NEON)
+    defined(EIGEN_VECTORIZE_NEON) || defined(EIGEN_VECTORIZE_ZVECTOR)
 template <typename Packet>
 EIGEN_DEVICE_FUNC inline Packet4f pexpand_bf16_l(const Packet4f& from) {
   float r[4];
@@ -126,8 +127,9 @@ EIGEN_DEVICE_FUNC inline Packet pload2bf16(
 }
 
 // Specialization for pload4bf16 and pload2bf16 for non-sse.
+// Enable vectorization on z13 and higher.
 #if defined(EIGEN_VECTORIZE_ALTIVEC) || defined(EIGEN_VECTORIZE_VSX) || \
-    defined(EIGEN_VECTORIZE_NEON)
+    defined(EIGEN_VECTORIZE_NEON) || defined(EIGEN_VECTORIZE_ZVECTOR)
 template <>
 EIGEN_STRONG_INLINE Packet4f pload4bf16<Packet4f>(const float* from) {
   tensorflow::uint32 p[4];
-- 
GitLab


From cb7dca274970573d018f39cae42a194b3452f799 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 12 Oct 2017 17:57:21 -0700
Subject: [PATCH 336/909] Switch to nest.flatten() in tf.layers.Layer to allow
 dicts and arbitrary nesting in layer inputs & outputs.

PiperOrigin-RevId: 172040243
---
 .../python/keras/_impl/keras/layers/merge.py  |  4 +-
 .../keras/_impl/keras/layers/merge_test.py    |  6 +-
 tensorflow/python/layers/base.py              | 58 +++++++++----------
 tensorflow/python/layers/base_test.py         | 24 ++++++++
 4 files changed, 57 insertions(+), 35 deletions(-)

diff --git a/tensorflow/python/keras/_impl/keras/layers/merge.py b/tensorflow/python/keras/_impl/keras/layers/merge.py
index b6391dba25..84b65d87c2 100644
--- a/tensorflow/python/keras/_impl/keras/layers/merge.py
+++ b/tensorflow/python/keras/_impl/keras/layers/merge.py
@@ -318,9 +318,9 @@ class Concatenate(_Merge):
 
   def build(self, input_shape):
     # Used purely for shape validation.
-    if not isinstance(input_shape, list):
+    if not (isinstance(input_shape, list) and len(input_shape) > 1):
       raise ValueError('`Concatenate` layer should be called '
-                       'on a list of inputs')
+                       'on a list containing at least two inputs')
     if all([shape is None for shape in input_shape]):
       return
     reduced_inputs_shapes = [
diff --git a/tensorflow/python/keras/_impl/keras/layers/merge_test.py b/tensorflow/python/keras/_impl/keras/layers/merge_test.py
index ea76337317..a574658279 100644
--- a/tensorflow/python/keras/_impl/keras/layers/merge_test.py
+++ b/tensorflow/python/keras/_impl/keras/layers/merge_test.py
@@ -141,11 +141,11 @@ class MergeLayersTest(test.TestCase):
   def test_concatenate_errors(self):
     i1 = keras.layers.Input(shape=(4, 5))
     i2 = keras.layers.Input(shape=(3, 5))
-    with self.assertRaises(ValueError):
+    with self.assertRaisesRegexp(ValueError, 'inputs with matching shapes'):
       keras.layers.concatenate([i1, i2], axis=-1)
-    with self.assertRaises(ValueError):
+    with self.assertRaisesRegexp(ValueError, 'called on a list'):
       keras.layers.concatenate(i1, axis=-1)
-    with self.assertRaises(ValueError):
+    with self.assertRaisesRegexp(ValueError, 'called on a list'):
       keras.layers.concatenate([i1], axis=-1)
 
   def test_merge_dot(self):
diff --git a/tensorflow/python/layers/base.py b/tensorflow/python/layers/base.py
index 711ffdfa9c..12c7fd7ef9 100644
--- a/tensorflow/python/layers/base.py
+++ b/tensorflow/python/layers/base.py
@@ -242,7 +242,7 @@ class Layer(object):
       return
     self._updates += updates
     if inputs is not None:
-      inputs = _to_list(inputs)
+      inputs = nest.flatten(inputs)
     if not inputs:
       inputs = None
     if inputs is not None:
@@ -273,7 +273,7 @@ class Layer(object):
     if context.in_eager_mode():
       raise RuntimeError('Layer.get_updates_for not supported in Eager mode.')
     if inputs is not None:
-      inputs = _to_list(inputs)
+      inputs = nest.flatten(inputs)
     if not inputs:
       inputs = None
     if inputs is not None:
@@ -318,7 +318,7 @@ class Layer(object):
       return
     self._losses += losses
     if inputs is not None:
-      inputs = _to_list(inputs)
+      inputs = nest.flatten(inputs)
     if not inputs:
       inputs = None
     if inputs is not None:
@@ -351,7 +351,7 @@ class Layer(object):
     if context.in_eager_mode():
       raise RuntimeError('Layer.get_losses_for not supported in Eager mode.')
     if inputs is not None:
-      inputs = _to_list(inputs)
+      inputs = nest.flatten(inputs)
     if not inputs:
       inputs = None
     if inputs is not None:
@@ -505,13 +505,14 @@ class Layer(object):
       ValueError: if the layer's `call` method returns None (an invalid value).
     """
     self._set_scope(kwargs.pop('scope', None))
+    input_list = nest.flatten(inputs)
 
     in_graph_mode = context.in_graph_mode()
     # Ensure the Layer, if being reused, is working with inputs from
     # the same graph as where it was created.
     if in_graph_mode:
       try:
-        ops._get_graph_from_inputs(nest.flatten(inputs), graph=self.graph)  # pylint: disable=protected-access
+        ops._get_graph_from_inputs(input_list, graph=self.graph)  # pylint: disable=protected-access
       except ValueError as e:
         raise ValueError('Input graph and Layer graph are not the same: %s' % e)
       user_kwargs = copy.copy(kwargs)
@@ -539,7 +540,7 @@ class Layer(object):
                                'Eager mode. Found an activity_regularizer in '
                                '%s(%s).' % (self.__class__.__name__, self))
             # TODO(agarwal): support _keras_history in Eager mode.
-            for x in _to_list(inputs):
+            for x in input_list:
               if hasattr(x, '_keras_history'):
                 raise ValueError('_keras_history currently unsupported in '
                                  'Eager mode. Found _keras_history in %s while '
@@ -548,17 +549,13 @@ class Layer(object):
 
           # Check input assumptions set before layer building, e.g. input rank.
           self._assert_input_compatibility(inputs)
-          input_list = nest.flatten(inputs)
           if input_list and self._dtype is None:
             try:
               self._dtype = input_list[0].dtype.name
             except AttributeError:
               pass
-          input_shapes = [x.get_shape() for x in input_list]
-          if len(input_shapes) == 1:
-            self.build(input_shapes[0])
-          else:
-            self.build(input_shapes)
+          input_shapes = nest.map_structure(lambda x: x.get_shape(), inputs)
+          self.build(input_shapes)
         try:
           # Note: not all sub-classes of Layer call Layer.__init__ (especially
           # the ones under tensorflow/python/keras). Hence we recompute this
@@ -583,7 +580,7 @@ class Layer(object):
           # Note that it should be applied every time the layer creates a new
           # output, since it is output-specific.
           if self._activity_regularizer:
-            output_list = _to_list(outputs)
+            output_list = nest.flatten(outputs)
             for output in output_list:
               with ops.name_scope('ActivityRegularizer'):
                 activity_regularization = self._activity_regularizer(output)
@@ -608,11 +605,10 @@ class Layer(object):
       if _have_all_keras_metadata(inputs):
         # If the layer returns tensors from its inputs, unmodified,
         # we copy them to avoid loss of tensor metadata.
-        output_ls = _to_list(outputs)
-        inputs_ls = _to_list(inputs)
+        output_ls = nest.flatten(outputs)
         output_ls_copy = []
         for x in output_ls:
-          if x in inputs_ls:
+          if x in input_list:
             with ops.name_scope(scope.original_name_scope):
               x = array_ops.identity(x)
           output_ls_copy.append(x)
@@ -683,8 +679,8 @@ class Layer(object):
             `call` method of the layer at the call that created the node.
     """
     assert context.in_graph_mode()
-    input_tensors = _to_list(input_tensors)
-    output_tensors = _to_list(output_tensors)
+    input_tensors = nest.flatten(input_tensors)
+    output_tensors = nest.flatten(output_tensors)
 
     # Collect input tensor(s) coordinates.
     inbound_layers = []
@@ -1011,10 +1007,10 @@ class Layer(object):
     if not self.input_spec:
       return
     if not isinstance(self.input_spec, (list, tuple)):
-      input_spec = _to_list(self.input_spec)
+      input_spec = nest.flatten(self.input_spec)
     else:
       input_spec = self.input_spec
-    inputs = _to_list(inputs)
+    inputs = nest.flatten(inputs)
     if len(inputs) != len(input_spec):
       raise ValueError('Layer ' + self.name + ' expects ' +
                        str(len(input_spec)) + ' inputs, '
@@ -1904,11 +1900,11 @@ class Network(Layer):
         A tensor if there is a single output, or
         a list of tensors if there are more than one outputs.
     """
-    inputs = _to_list(inputs)
+    inputs = nest.flatten(inputs)
     if mask is None:
       masks = [None for _ in range(len(inputs))]
     else:
-      masks = _to_list(mask)
+      masks = nest.flatten(mask)
     # Try to retrieve cached outputs if the layer has already been called
     # on these exact inputs.
     cache_key = _object_list_uid(inputs) + '_' + _object_list_uid(masks)
@@ -2081,9 +2077,10 @@ class Network(Layer):
               if 'mask' in estimator_util.fn_args(layer.call):
                 if 'mask' not in kwargs:
                   kwargs['mask'] = computed_mask
-              output_tensors = _to_list(layer.call(computed_tensor, **kwargs))
+              output_tensors = nest.flatten(
+                  layer.call(computed_tensor, **kwargs))
               if hasattr(layer, 'compute_mask'):
-                output_masks = _to_list(
+                output_masks = nest.flatten(
                     layer.compute_mask(computed_tensor, computed_mask))
               else:
                 output_masks = [None for _ in range(len(output_tensors))]
@@ -2095,9 +2092,10 @@ class Network(Layer):
               if 'mask' in estimator_util.fn_args(layer.call):
                 if 'mask' not in kwargs:
                   kwargs['mask'] = computed_masks
-              output_tensors = _to_list(layer.call(computed_tensors, **kwargs))
+              output_tensors = nest.flatten(
+                  layer.call(computed_tensors, **kwargs))
               if hasattr(layer, 'compute_mask'):
-                output_masks = _to_list(
+                output_masks = nest.flatten(
                     layer.compute_mask(computed_tensors, computed_masks))
               else:
                 output_masks = [None for _ in range(len(output_tensors))]
@@ -2204,8 +2202,8 @@ def _add_elements_to_collection(elements, collection_list):
     raise RuntimeError('Using collections from Layers not supported in Eager '
                        'mode. Tried to add %s to %s' % (elements,
                                                         collection_list))
-  elements = _to_list(elements)
-  collection_list = _to_list(collection_list)
+  elements = nest.flatten(elements)
+  collection_list = nest.flatten(collection_list)
   for name in collection_list:
     collection = ops.get_collection_ref(name)
     collection_set = set(collection)
@@ -2215,7 +2213,7 @@ def _add_elements_to_collection(elements, collection_list):
 
 
 def _object_list_uid(object_list):
-  object_list = _to_list(object_list)
+  object_list = nest.flatten(object_list)
   return ', '.join([str(abs(id(x))) for x in object_list])
 
 
@@ -2261,7 +2259,7 @@ def _collect_previous_mask(input_tensors):
   Returns:
       A mask tensor or list of mask tensors.
   """
-  input_tensors = _to_list(input_tensors)
+  input_tensors = nest.flatten(input_tensors)
   masks = []
   for x in input_tensors:
     if hasattr(x, '_keras_mask'):
diff --git a/tensorflow/python/layers/base_test.py b/tensorflow/python/layers/base_test.py
index 93d2d80850..813a2fe755 100644
--- a/tensorflow/python/layers/base_test.py
+++ b/tensorflow/python/layers/base_test.py
@@ -548,6 +548,30 @@ class BaseLayerTest(test.TestCase):
     with self.assertRaises(ValueError):
       dense.count_params()
 
+  @test_util.run_in_graph_and_eager_modes()
+  def testDictInputOutput(self):
+
+    class DictLayer(base_layers.Layer):
+
+      def call(self, inputs):
+        return {'l' + key: inputs[key] for key in inputs}
+
+    layer = DictLayer()
+    if context.in_graph_mode():
+      i1 = array_ops.placeholder('int32')
+      i2 = array_ops.placeholder('float32')
+      result = layer.apply({'abel': i1, 'ogits': i2})
+      self.assertTrue(isinstance(result, dict))
+      self.assertEqual(set(['label', 'logits']), set(result.keys()))
+    else:
+      i1 = constant_op.constant(3)
+      i2 = constant_op.constant(4.0)
+      result = layer.apply({'abel': i1, 'ogits': i2})
+      self.assertTrue(isinstance(result, dict))
+      self.assertEqual(set(['label', 'logits']), set(result.keys()))
+      self.assertEqual(3, result['label'].numpy())
+      self.assertEqual(4.0, result['logits'].numpy())
+
 
 class NetworkTest(test.TestCase):
 
-- 
GitLab


From 215aca58c6429ba16d8c88fdcf1fc6971e318ccc Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 12 Oct 2017 18:01:34 -0700
Subject: [PATCH 337/909] Removes brittleness from some tests in dnn_test.

PiperOrigin-RevId: 172040631
---
 .../learn/python/learn/estimators/dnn_test.py | 26 ++++++++++++++-----
 1 file changed, 19 insertions(+), 7 deletions(-)

diff --git a/tensorflow/contrib/learn/python/learn/estimators/dnn_test.py b/tensorflow/contrib/learn/python/learn/estimators/dnn_test.py
index 2fec0508a5..12f9bba531 100644
--- a/tensorflow/contrib/learn/python/learn/estimators/dnn_test.py
+++ b/tensorflow/contrib/learn/python/learn/estimators/dnn_test.py
@@ -348,6 +348,12 @@ class DNNClassifierTest(test.TestCase):
     for prediction in predictions:
       self.assertIn(prediction, (0, 1))
 
+  def _assertClassificationPredictions(
+      self, expected_len, n_classes, predictions):
+    self.assertEqual(expected_len, len(predictions))
+    for prediction in predictions:
+      self.assertIn(prediction, range(n_classes))
+
   def _assertProbabilities(self, expected_batch_size, expected_n_classes,
                            probabilities):
     self.assertEqual(expected_batch_size, len(probabilities))
@@ -732,7 +738,7 @@ class DNNClassifierTest(test.TestCase):
     self.assertIn('loss', scores)
     predicted_classes = classifier.predict_classes(
         input_fn=_input_fn, as_iterable=False)
-    self._assertBinaryPredictions(3, predicted_classes)
+    self._assertClassificationPredictions(3, n_classes, predicted_classes)
     predictions = classifier.predict(input_fn=_input_fn, as_iterable=False)
     self.assertAllEqual(predicted_classes, predictions)
     probabilities = classifier.predict_proba(
@@ -765,8 +771,9 @@ class DNNClassifierTest(test.TestCase):
         feature_column.real_valued_column('age')
     ]
 
+    n_classes = 3
     classifier = dnn.DNNClassifier(
-        n_classes=3,
+        n_classes=n_classes,
         feature_columns=feature_columns,
         hidden_units=[3, 3],
         config=run_config.RunConfig(tf_random_seed=1))
@@ -780,7 +787,7 @@ class DNNClassifierTest(test.TestCase):
     predicted_classes = list(
         classifier.predict_classes(
             input_fn=predict_input_fn, as_iterable=True))
-    self.assertListEqual(predicted_classes, [1, 0, 0])
+    self._assertClassificationPredictions(3, n_classes, predicted_classes)
     predictions = list(
         classifier.predict(
             input_fn=predict_input_fn, as_iterable=True))
@@ -788,8 +795,7 @@ class DNNClassifierTest(test.TestCase):
     predicted_proba = list(
         classifier.predict_proba(
             input_fn=predict_input_fn, as_iterable=True))
-    self.assertAllClose(
-        predicted_proba, [[0., 1., 0.], [1., 0., 0.], [1., 0., 0.]], atol=0.3)
+    self._assertProbabilities(3, n_classes, predicted_proba)
 
   def testCustomMetrics(self):
     """Tests custom evaluation metrics."""
@@ -1214,6 +1220,12 @@ class DNNRegressorTest(test.TestCase):
     scores = regressor.evaluate(input_fn=_input_fn_eval, steps=1)
     self.assertIn('loss', scores)
 
+  def _assertRegressionOutputs(
+      self, predictions, expected_shape):
+    predictions_nparray = np.array(predictions)
+    self.assertAllEqual(expected_shape, predictions_nparray.shape)
+    self.assertTrue(np.issubdtype(predictions_nparray.dtype, np.float))
+
   def testPredict_AsIterableFalse(self):
     """Tests predict method with as_iterable=False."""
     labels = [1., 0., 0.2]
@@ -1252,7 +1264,7 @@ class DNNRegressorTest(test.TestCase):
     self.assertIn('loss', scores)
     predicted_scores = regressor.predict_scores(
         input_fn=_input_fn, as_iterable=False)
-    self.assertAllClose(labels, predicted_scores, atol=0.2)
+    self._assertRegressionOutputs(predicted_scores, [3])
     predictions = regressor.predict(input_fn=_input_fn, as_iterable=False)
     self.assertAllClose(predicted_scores, predictions)
 
@@ -1296,7 +1308,7 @@ class DNNRegressorTest(test.TestCase):
     predicted_scores = list(
         regressor.predict_scores(
             input_fn=predict_input_fn, as_iterable=True))
-    self.assertAllClose(labels, predicted_scores, atol=0.2)
+    self._assertRegressionOutputs(predicted_scores, [3])
     predictions = list(
         regressor.predict(input_fn=predict_input_fn, as_iterable=True))
     self.assertAllClose(predicted_scores, predictions)
-- 
GitLab


From 0d5e77eda4b1e0db61e12284943ef4457d3dbff9 Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Thu, 12 Oct 2017 18:05:25 -0700
Subject: [PATCH 338/909] Automated g4 rollback of changelist 172019169

PiperOrigin-RevId: 172041133
---
 tensorflow/core/util/cuda_kernel_helper.h | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/tensorflow/core/util/cuda_kernel_helper.h b/tensorflow/core/util/cuda_kernel_helper.h
index 8315f208e7..9e76e37898 100644
--- a/tensorflow/core/util/cuda_kernel_helper.h
+++ b/tensorflow/core/util/cuda_kernel_helper.h
@@ -154,11 +154,15 @@ struct CudaLaunchConfig {
 // Calculate the Cuda launch config we should use for a kernel launch.
 // This is assuming the kernel is quite simple and will largely be
 // memory-limited.
-// REQUIRES: work_element_count > 0.
 inline CudaLaunchConfig GetCudaLaunchConfig(int work_element_count,
                                             const GPUDevice& d) {
-  CHECK_GT(work_element_count, 0);
   CudaLaunchConfig config;
+
+  // in case of invalid input, return the default value config, which has all -1
+  if (work_element_count <= 0) {
+    return config;
+  }
+
   const int virtual_thread_count = work_element_count;
   const int physical_thread_count = std::min(
       d.getNumCudaMultiProcessors() * d.maxCudaThreadsPerMultiProcessor(),
@@ -176,14 +180,17 @@ inline CudaLaunchConfig GetCudaLaunchConfig(int work_element_count,
 
 // Calculate the Cuda launch config we should use for a kernel launch. This
 // variant takes the resource limits of func into account to maximize occupancy.
-// REQUIRES: work_element_count > 0.
 template <typename DeviceFunc>
 inline CudaLaunchConfig GetCudaLaunchConfig(int work_element_count,
                                             const GPUDevice& d, DeviceFunc func,
                                             size_t dynamic_shared_memory_size,
                                             int block_size_limit) {
-  CHECK_GT(work_element_count, 0);
   CudaLaunchConfig config;
+
+  if (work_element_count <= 0) {
+    return config;
+  }
+
   int block_count = 0;
   int thread_per_block = 0;
 
-- 
GitLab


From d4d8b81209138332a9b4c16ae106d1f01e9e412d Mon Sep 17 00:00:00 2001
From: Derek Murray <mrry@google.com>
Date: Thu, 12 Oct 2017 18:08:29 -0700
Subject: [PATCH 339/909] [tf.data] Actually add
 `tf.contrib.data.get_single_element()` to the public API.

PiperOrigin-RevId: 172041381
---
 tensorflow/contrib/data/__init__.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tensorflow/contrib/data/__init__.py b/tensorflow/contrib/data/__init__.py
index 4c32c72ad4..7ff26e087b 100644
--- a/tensorflow/contrib/data/__init__.py
+++ b/tensorflow/contrib/data/__init__.py
@@ -32,6 +32,7 @@ See the @{$datasets$Importing Data} Programmer's Guide for an overview.
 @@rejection_resample
 @@sloppy_interleave
 
+@@get_single_element
 """
 
 from __future__ import absolute_import
@@ -44,6 +45,7 @@ from tensorflow.contrib.data.python.ops.batching import batch_and_drop_remainder
 from tensorflow.contrib.data.python.ops.batching import dense_to_sparse_batch
 from tensorflow.contrib.data.python.ops.batching import unbatch
 from tensorflow.contrib.data.python.ops.dataset_ops import Dataset
+from tensorflow.contrib.data.python.ops.dataset_ops import get_single_element
 from tensorflow.contrib.data.python.ops.enumerate_ops import enumerate_dataset
 from tensorflow.contrib.data.python.ops.error_ops import ignore_errors
 from tensorflow.contrib.data.python.ops.grouping import group_by_window
-- 
GitLab


From 33fc95f46257e07deed852acf65806055672ce25 Mon Sep 17 00:00:00 2001
From: Benoit Steiner <bsteiner@google.com>
Date: Thu, 12 Oct 2017 18:33:29 -0700
Subject: [PATCH 340/909] Determine peak memory usage from the result of a
 simulation.

PiperOrigin-RevId: 172043591
---
 tensorflow/core/grappler/clusters/cluster.cc  |   4 +
 tensorflow/core/grappler/clusters/cluster.h   |   3 +
 tensorflow/core/grappler/costs/BUILD          |   3 +
 .../core/grappler/costs/cost_estimator.h      |   2 +
 .../core/grappler/costs/graph_memory.cc       | 191 +++++++++++++++---
 tensorflow/core/grappler/costs/graph_memory.h |  44 ++--
 .../core/grappler/costs/graph_memory_test.cc  | 105 ++++++++--
 7 files changed, 301 insertions(+), 51 deletions(-)

diff --git a/tensorflow/core/grappler/clusters/cluster.cc b/tensorflow/core/grappler/clusters/cluster.cc
index 3205d67517..ead44de1e2 100644
--- a/tensorflow/core/grappler/clusters/cluster.cc
+++ b/tensorflow/core/grappler/clusters/cluster.cc
@@ -45,6 +45,10 @@ void Cluster::DisableDetailedStats(bool disable) {
   }
 }
 
+bool Cluster::DetailedStatsEnabled() const {
+  return options_.config.graph_options().build_cost_model() != 0;
+}
+
 void Cluster::DisableOptimizer(bool disable) {
   OptimizerOptions* options =
       options_.config.mutable_graph_options()->mutable_optimizer_options();
diff --git a/tensorflow/core/grappler/clusters/cluster.h b/tensorflow/core/grappler/clusters/cluster.h
index 911bc1e5a2..616ab6ffdc 100644
--- a/tensorflow/core/grappler/clusters/cluster.h
+++ b/tensorflow/core/grappler/clusters/cluster.h
@@ -68,6 +68,9 @@ class Cluster {
   // before Provision().
   void DisableDetailedStats(bool disable);
 
+  // Returns true iff the collection of detailed statistics is enabled.
+  bool DetailedStatsEnabled() const;
+
   // Disable the TensorFlow optimizer. This ensures that the graph that TF
   // executes is similar to the input graph. Must be called before Provision().
   void DisableOptimizer(bool disable);
diff --git a/tensorflow/core/grappler/costs/BUILD b/tensorflow/core/grappler/costs/BUILD
index 1d0bd42372..257e8e8d04 100644
--- a/tensorflow/core/grappler/costs/BUILD
+++ b/tensorflow/core/grappler/costs/BUILD
@@ -83,11 +83,14 @@ cc_library(
     hdrs = ["graph_memory.h"],
     visibility = ["//visibility:public"],
     deps = [
+        ":cost_estimator",
         ":graph_properties",
         "//tensorflow/core:framework",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core/grappler:grappler_item",
+        "//tensorflow/core/grappler:utils",
         "//tensorflow/core/grappler/clusters:cluster",
+        "//tensorflow/core/grappler/clusters:virtual_cluster",
     ],
 )
 
diff --git a/tensorflow/core/grappler/costs/cost_estimator.h b/tensorflow/core/grappler/costs/cost_estimator.h
index 868c4a9733..cf9fa4fdaf 100644
--- a/tensorflow/core/grappler/costs/cost_estimator.h
+++ b/tensorflow/core/grappler/costs/cost_estimator.h
@@ -121,6 +121,8 @@ Costs::Costs() {
 Costs Costs::ZeroCosts() {
   Costs costs;
   costs.execution_time = Duration::zero();
+  costs.compute_time = Duration::zero();
+  costs.memory_time = Duration::zero();
   costs.max_memory = kZeroMemory;
   costs.max_per_op_buffers = kZeroMemory;
   costs.max_per_op_streaming = kZeroMemory;
diff --git a/tensorflow/core/grappler/costs/graph_memory.cc b/tensorflow/core/grappler/costs/graph_memory.cc
index b7827fc1ad..0adec584a8 100644
--- a/tensorflow/core/grappler/costs/graph_memory.cc
+++ b/tensorflow/core/grappler/costs/graph_memory.cc
@@ -14,45 +14,45 @@ limitations under the License.
 ==============================================================================*/
 
 #include "tensorflow/core/grappler/costs/graph_memory.h"
-
+#include <list>
+#include "tensorflow/core/framework/allocation_description.pb.h"
 #include "tensorflow/core/framework/node_def.pb.h"
+#include "tensorflow/core/framework/step_stats.pb.h"
+#include "tensorflow/core/framework/tensor_description.pb.h"
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/tensor_shape.pb.h"
+#include "tensorflow/core/grappler/clusters/virtual_cluster.h"
 #include "tensorflow/core/grappler/costs/graph_properties.h"
+#include "tensorflow/core/grappler/utils.h"
 
 namespace tensorflow {
 namespace grappler {
 
-Status GraphMemory::InferStatically() {
-  GraphProperties properties(item_);
-  TF_RETURN_IF_ERROR(properties.InferStatically());
-  return InferFromGraphProperties(&properties);
+Status GraphMemory::InferStatically(
+    const std::unordered_map<string, DeviceProperties>& devices) {
+  VirtualCluster cluster(devices);
+  TF_RETURN_IF_ERROR(cluster.Provision());
+  return InferDynamically(&cluster);
 }
 
 Status GraphMemory::InferDynamically(Cluster* cluster) {
-  GraphProperties properties(item_);
-  TF_RETURN_IF_ERROR(properties.InferDynamically(cluster));
-  return InferFromGraphProperties(&properties);
+  if (!cluster->DetailedStatsEnabled()) {
+    return errors::Unavailable("Detailed stats collection must be enabled");
+  }
+  TF_RETURN_IF_ERROR(cluster->Initialize(item_));
+  RunMetadata metadata;
+  TF_RETURN_IF_ERROR(
+      cluster->Run(item_.graph, item_.feed, item_.fetch, &metadata));
+  InferFromTrace(metadata.step_stats());
+  return Status::OK();
 }
 
-Status GraphMemory::InferFromGraphProperties(GraphProperties* properties) {
-  // Compute the worst case usage between initialization and normal mode.
-  // TODO(bsteiner): we should consider persistent memory usage separately.
-  int64 worst_case_init_mem_usage;
-  int64 best_case_init_mem_usage;
-  InferMemUsageForNodes(item_.InitOpsFanin(), properties,
-                        &worst_case_init_mem_usage, &best_case_init_mem_usage);
-  int64 worst_case_main_mem_usage;
-  int64 best_case_main_mem_usage;
-  InferMemUsageForNodes(item_.MainOpsFanin(), properties,
-                        &worst_case_main_mem_usage, &best_case_main_mem_usage);
-
-  worst_case_memory_usage_ =
-      std::max(worst_case_init_mem_usage, worst_case_main_mem_usage);
-  best_case_memory_usage_ =
-      std::max(best_case_init_mem_usage, best_case_main_mem_usage);
-
-  return Status::OK();
+int64 GraphMemory::GetWorstCaseMemoryUsage() const {
+  int64 worst_case = -1;
+  for (const auto& peak_usage : peak_usage_) {
+    worst_case = std::max(worst_case, peak_usage.second.used_memory);
+  }
+  return worst_case;
 }
 
 void GraphMemory::InferMemUsageForNodes(
@@ -105,5 +105,144 @@ int64 GraphMemory::InferMemUsageForNeighbors(
   return neighbors_memory_usage;
 }
 
+static GraphMemory::LiveTensor* FindOrCreateLiveTensor(
+    const string& node_name, int output_id,
+    std::unordered_map<string, GraphMemory::LiveTensor*>* live_tensors,
+    std::list<GraphMemory::LiveTensor>* device_tensors) {
+  string name = strings::StrCat(node_name, ":", output_id);
+  GraphMemory::LiveTensor* live;
+  auto it = live_tensors->find(name);
+  if (it == live_tensors->end()) {
+    GraphMemory::LiveTensor temp;
+    temp.node = node_name;
+    temp.output_id = output_id;
+    temp.allocation_time = 0;
+    temp.deallocation_time = 0;
+    device_tensors->push_front(temp);
+    live = &device_tensors->front();
+    (*live_tensors)[name] = live;
+  } else {
+    live = it->second;
+  }
+  return live;
+}
+
+namespace {
+struct Event {
+  int64 timestamp;
+  bool allocated;
+  const GraphMemory::LiveTensor* tensor;
+
+  bool operator<(const Event& other) const {
+    return timestamp < other.timestamp;
+  }
+};
+}  // namespace
+
+void GraphMemory::InferFromTrace(const StepStats& timeline) {
+  std::unordered_map<string, string> node_placement;
+  for (const auto& dev_stats : timeline.dev_stats()) {
+    for (const auto& node_stats : dev_stats.node_stats()) {
+      node_placement[node_stats.node_name()] = dev_stats.device();
+    }
+  }
+
+  std::unordered_map<string, LiveTensor*> live_tensors;
+  std::unordered_map<string, std::list<LiveTensor>> live_tensors_per_device;
+
+  NodeMap node_map(&item_.graph);
+  for (const auto& dev_stats : timeline.dev_stats()) {
+    std::list<LiveTensor>& device_tensors =
+        live_tensors_per_device[dev_stats.device()];
+    for (const auto& node_stats : dev_stats.node_stats()) {
+      for (int i = 0; i < node_stats.output_size(); ++i) {
+        const auto& output = node_stats.output(i);
+
+        LiveTensor* live = FindOrCreateLiveTensor(
+            node_stats.node_name(), i, &live_tensors, &device_tensors);
+        live->memory_used = output.tensor_description()
+                                .allocation_description()
+                                .allocated_bytes();
+        // Allocations typically take place at the very beginning of the op
+        // execution.
+        live->allocation_time =
+            Costs::MicroSeconds(node_stats.all_start_micros());
+        // Add one nanosecond to the completion time of the ops to account for
+        // TF overhead that slightly delays deallocations.
+        live->deallocation_time = std::max<Costs::Duration>(
+            live->deallocation_time,
+            Costs::NanoSeconds(1) +
+                Costs::MicroSeconds(node_stats.all_start_micros() +
+                                    node_stats.op_end_rel_micros()));
+      }
+
+      const NodeDef* node = node_map.GetNode(node_stats.node_name());
+      if (!node) {
+        // Skip nodes inserted by TF since they don't exist in the original
+        // graph (e.g _Send/_Recv nodes).
+        continue;
+      }
+      for (const string& input : node->input()) {
+        int position;
+        string input_node = ParseNodeName(input, &position);
+
+        LiveTensor* live = FindOrCreateLiveTensor(
+            input_node, position, &live_tensors,
+            &live_tensors_per_device[node_placement[input_node]]);
+        live->deallocation_time = std::max<Costs::Duration>(
+            live->deallocation_time,
+            Costs::NanoSeconds(1) +
+                Costs::MicroSeconds(node_stats.all_start_micros() +
+                                    node_stats.op_end_rel_micros()));
+      }
+    }
+  }
+
+  for (const auto& live_per_device : live_tensors_per_device) {
+    std::vector<Event> events;
+    events.reserve(2 * live_per_device.second.size());
+    for (const auto& live : live_per_device.second) {
+      events.push_back(Event{live.allocation_time.count(), true, &live});
+      events.push_back(Event{live.deallocation_time.count(), false, &live});
+    }
+    std::stable_sort(events.begin(), events.end());
+    size_t peak = 0;
+    std::set<const LiveTensor*> live_at_peak;
+    size_t current = 0;
+    std::set<const LiveTensor*> currently_live;
+    for (int i = 0; i < events.size(); ++i) {
+      const auto& event = events[i];
+
+      if (event.allocated) {
+        VLOG(1) << "At time " << event.timestamp << " allocated "
+                << event.tensor->memory_used << " for tensor "
+                << event.tensor->node << ":" << event.tensor->output_id;
+        current += event.tensor->memory_used;
+        currently_live.insert(event.tensor);
+      } else {
+        VLOG(1) << "At time " << event.timestamp << " deallocated "
+                << event.tensor->memory_used << " for tensor "
+                << event.tensor->node << ":" << event.tensor->output_id;
+        current -= event.tensor->memory_used;
+        currently_live.erase(event.tensor);
+      }
+      if (i + 1 == events.size() ||
+          event.timestamp != events[i + 1].timestamp) {
+        if (current > peak) {
+          peak = current;
+          live_at_peak = currently_live;
+        }
+      }
+    }
+    MemoryUsage& peak_mem_usage = peak_usage_[live_per_device.first];
+    peak_mem_usage.used_memory = peak;
+    peak_mem_usage.live_tensors.clear();
+    peak_mem_usage.live_tensors.reserve(live_at_peak.size());
+    for (const auto& live : live_at_peak) {
+      peak_mem_usage.live_tensors.push_back(*live);
+    }
+  }
+}
+
 }  // end namespace grappler
 }  // end namespace tensorflow
diff --git a/tensorflow/core/grappler/costs/graph_memory.h b/tensorflow/core/grappler/costs/graph_memory.h
index a3e152a0e1..859e4c012c 100644
--- a/tensorflow/core/grappler/costs/graph_memory.h
+++ b/tensorflow/core/grappler/costs/graph_memory.h
@@ -18,6 +18,7 @@ limitations under the License.
 
 #include "tensorflow/core/framework/node_def.pb.h"
 #include "tensorflow/core/grappler/clusters/cluster.h"
+#include "tensorflow/core/grappler/costs/cost_estimator.h"
 #include "tensorflow/core/grappler/costs/graph_properties.h"
 #include "tensorflow/core/grappler/grappler_item.h"
 
@@ -27,20 +28,37 @@ namespace grappler {
 // Infer the worst case memory usage for a given grappler item.
 class GraphMemory {
  public:
+  struct LiveTensor {
+    string node;
+    int output_id;
+    size_t memory_used;
+    Costs::Duration allocation_time;
+    Costs::Duration deallocation_time;
+  };
+  struct MemoryUsage {
+    int64 used_memory;
+    std::vector<LiveTensor> live_tensors;
+  };
+
   explicit GraphMemory(const GrapplerItem& item)
-      : item_(item), worst_case_memory_usage_(-1) {}
+      : item_(item), unknown_usage_({-1, {}}) {}
 
-  Status InferStatically();
+  Status InferStatically(
+      const std::unordered_map<string, DeviceProperties>& devices);
   Status InferDynamically(Cluster* cluster);
-  Status InferFromGraphProperties(GraphProperties* properties);
 
-  // Worst case memory usage in bytes, or -1 if the usage is unknown.
-  int64 GetWorstCaseMemoryUsage() const { return worst_case_memory_usage_; }
+  // Worst case memory usage in bytes, or -1 if the usage is unknown. If there
+  // are multiple devices, returns the highest per device memory usage.
+  int64 GetWorstCaseMemoryUsage() const;
 
-  // Best case memory usage in bytes, or -1 if the usage is unknown.
-  // This corresponds to the case where all the data is swapped out excepted
-  // that which is needed for a single node to perform its computations.
-  int64 GetBestCaseMemoryUsage() const { return best_case_memory_usage_; }
+  // Returns the peak memory usage for the specified device.
+  const MemoryUsage& GetPeakMemoryUsage(const string& device) const {
+    auto it = peak_usage_.find(device);
+    if (it == peak_usage_.end()) {
+      return unknown_usage_;
+    }
+    return it->second;
+  }
 
  private:
   void InferMemUsageForNodes(const std::vector<const NodeDef*>& nodes,
@@ -49,10 +67,12 @@ class GraphMemory {
   int64 InferMemUsageForNeighbors(
       const std::vector<OpInfo::TensorProperties>& props) const;
 
-  // Inputs
+  void InferFromTrace(const StepStats& timeline);
+
   GrapplerItem item_;
-  int64 worst_case_memory_usage_;
-  int64 best_case_memory_usage_;
+  std::unordered_map<string, int64> worst_case_memory_usage_;
+  std::unordered_map<string, MemoryUsage> peak_usage_;
+  const MemoryUsage unknown_usage_;
 };
 
 }  // end namespace grappler
diff --git a/tensorflow/core/grappler/costs/graph_memory_test.cc b/tensorflow/core/grappler/costs/graph_memory_test.cc
index 82c86064c6..e4d0cf7813 100644
--- a/tensorflow/core/grappler/costs/graph_memory_test.cc
+++ b/tensorflow/core/grappler/costs/graph_memory_test.cc
@@ -22,36 +22,115 @@ namespace tensorflow {
 namespace grappler {
 namespace {
 
-class GraphMemoryTest : public ::testing::Test {};
+class GraphMemoryTest : public ::testing::Test {
+ protected:
+  std::unordered_map<string, DeviceProperties> devices_;
+
+ public:
+  GraphMemoryTest() {
+    devices_["/CPU:0"].set_type("CPU");
+    devices_["/CPU:0"].set_num_cores(1);
+    devices_["/CPU:0"].set_frequency(1);
+    devices_["/CPU:0"].set_bandwidth(1);
+
+    devices_["/GPU:0"].set_type("GPU");
+    devices_["/GPU:0"].set_num_cores(1);
+    devices_["/GPU:0"].set_frequency(1);
+    devices_["/CPU:0"].set_bandwidth(1);
+    (*devices_["/GPU:0"].mutable_environment())["architecture"] = "3";
+  }
+};
 
 TEST_F(GraphMemoryTest, Basic) {
-  TrivialTestGraphInputYielder fake_input(4, 1, 10, false, {{"CPU:0"}});
+  TrivialTestGraphInputYielder fake_input(4, 1, 10, false, {"/CPU:0"});
   GrapplerItem item;
   CHECK(fake_input.NextItem(&item));
+  item.feed.clear();
 
   GraphMemory memory(item);
-  Status s = memory.InferStatically();
+  Status s = memory.InferStatically(devices_);
   TF_CHECK_OK(s);
-  // 5 AddN + 1 random op each generating 10 values -> 240 bytes
-  // 4 more bytes for the mean of the distribution and 4 more for the stddev.
-  EXPECT_EQ(248, memory.GetWorstCaseMemoryUsage());
-  // If at most one op executes at a time, it needs 10 inputs values and 10
-  // output values, or 8 bytes.
-  EXPECT_EQ(80, memory.GetBestCaseMemoryUsage());
+  const GraphMemory::MemoryUsage& mem_usage =
+      memory.GetPeakMemoryUsage("/CPU:0");
+  EXPECT_EQ(120, mem_usage.used_memory);
+
+  std::set<string> tensors;
+  for (const auto& t : mem_usage.live_tensors) {
+    tensors.insert(strings::StrCat(t.node, ":", t.output_id));
+  }
+  // When the execution of the 'Square' node completes, TF can start executing
+  // 'Square_1' and release the memory used by 'x'. Since we can't be sure of
+  // the order in which this takes place, in the worst case the 3 tensors are in
+  // memory.
+  std::set<string> expected;
+  expected.insert("Square:0");
+  expected.insert("Square_1:0");
+  expected.insert("x:0");
+  EXPECT_EQ(expected, tensors);
 }
 
 TEST_F(GraphMemoryTest, UnknownBatchSize) {
-  TrivialTestGraphInputYielder fake_input(4, 1, -1, false, {{"CPU:0"}});
+  TrivialTestGraphInputYielder fake_input(4, 1, -1, false, {"/CPU:0"});
   GrapplerItem item;
   CHECK(fake_input.NextItem(&item));
+  item.feed.clear();
 
   GraphMemory memory(item);
-  Status s = memory.InferStatically();
+  Status s = memory.InferStatically(devices_);
   TF_CHECK_OK(s);
   // Same maths as before, except that batch size is unknown and therefore
   // assumed to be one.
-  EXPECT_EQ(32, memory.GetWorstCaseMemoryUsage());
-  EXPECT_EQ(12, memory.GetBestCaseMemoryUsage());
+  const GraphMemory::MemoryUsage& mem_usage =
+      memory.GetPeakMemoryUsage("/CPU:0");
+  EXPECT_EQ(16, mem_usage.used_memory);
+
+  std::set<string> tensors;
+  for (const auto& t : mem_usage.live_tensors) {
+    tensors.insert(strings::StrCat(t.node, ":", t.output_id));
+  }
+  std::set<string> expected;
+  expected.insert("Const/Const:0");
+  expected.insert("Square:0");
+  expected.insert("x:0");
+  EXPECT_EQ(expected, tensors);
+}
+
+TEST_F(GraphMemoryTest, MultiDevice) {
+  TrivialTestGraphInputYielder fake_input(4, 2, 1024 * 1024, false,
+                                          {"/CPU:0", "/GPU:0"});
+  GrapplerItem item;
+  CHECK(fake_input.NextItem(&item));
+  item.feed.clear();
+
+  GraphMemory memory(item);
+  Status s = memory.InferStatically(devices_);
+  TF_CHECK_OK(s);
+
+  const GraphMemory::MemoryUsage& cpu_mem = memory.GetPeakMemoryUsage("/CPU:0");
+  EXPECT_EQ(16777216, cpu_mem.used_memory);
+  std::set<string> cpu_tensors;
+  for (const auto& t : cpu_mem.live_tensors) {
+    cpu_tensors.insert(strings::StrCat(t.node, ":", t.output_id));
+  }
+  std::set<string> cpu_expected;
+  cpu_expected.insert("Recv_Square_1_0_on_/CPU_0:0");
+  cpu_expected.insert("Square:0");
+  cpu_expected.insert("x:0");
+  cpu_expected.insert("AddN:0");
+  EXPECT_EQ(cpu_expected, cpu_tensors);
+
+  const GraphMemory::MemoryUsage& gpu_mem = memory.GetPeakMemoryUsage("/GPU:0");
+  EXPECT_EQ(16777216, gpu_mem.used_memory);
+  std::set<string> gpu_tensors;
+  for (const auto& t : gpu_mem.live_tensors) {
+    gpu_tensors.insert(strings::StrCat(t.node, ":", t.output_id));
+  }
+  std::set<string> gpu_expected;
+  gpu_expected.insert("Recv_AddN_0_on_/GPU_0:0");
+  gpu_expected.insert("Square_1:0");
+  gpu_expected.insert("AddN_1:0");
+  gpu_expected.insert("AddN_3:0");
+  EXPECT_EQ(gpu_expected, gpu_tensors);
 }
 
 }  // namespace
-- 
GitLab


From d100f65c58ada9df6124e5c366a8877b7ba03235 Mon Sep 17 00:00:00 2001
From: Eugene Brevdo <ebrevdo@google.com>
Date: Thu, 12 Oct 2017 18:48:54 -0700
Subject: [PATCH 341/909] Temporarily make tf.where emit an old-style NodeDef
 (cast non-bool inputs to bool)

PiperOrigin-RevId: 172044654
---
 tensorflow/python/ops/array_ops.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/ops/array_ops.py b/tensorflow/python/ops/array_ops.py
index dc3aa735da..c00efb16ba 100644
--- a/tensorflow/python/ops/array_ops.py
+++ b/tensorflow/python/ops/array_ops.py
@@ -2466,8 +2466,14 @@ def where(condition, x=None, y=None, name=None):
   """
   if x is None and y is None:
     with ops.name_scope(name, "Where", [condition]) as name:
-      condition = ops.convert_to_tensor(condition, dtype=dtypes.bool)
-      return gen_array_ops.where(input=condition, name=name)
+      # Temporarily create an old style WhereOp nodedef + Operation without the
+      # attribute "T".
+      # TODO(b/67720963): Roll this back when the issue is resolved.
+      condition = gen_math_ops.cast(condition, dtypes.bool)
+      output = gen_array_ops.where(input=condition, name=name)
+      if context.in_graph_mode():
+        output.op._node_def.attr.clear()
+      return output
   elif x is not None and y is not None:
     return gen_math_ops._select(condition=condition, t=x, e=y, name=name)
   else:
-- 
GitLab


From 97dec812b8393e0bf6dde554c82526bfb901f015 Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Thu, 12 Oct 2017 18:55:24 -0700
Subject: [PATCH 342/909] Internal change.

PiperOrigin-RevId: 172045110
---
 tensorflow/contrib/cmake/tf_tests.cmake | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/contrib/cmake/tf_tests.cmake b/tensorflow/contrib/cmake/tf_tests.cmake
index 4cf22a9c47..9b80cda577 100644
--- a/tensorflow/contrib/cmake/tf_tests.cmake
+++ b/tensorflow/contrib/cmake/tf_tests.cmake
@@ -232,6 +232,7 @@ if (tensorflow_BUILD_PYTHON_TESTS)
       "${tensorflow_source_dir}/tensorflow/python/ops/init_ops.py"
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/self_adjoint_eig_op_test.py"
       # misc
+      "${tensorflow_source_dir}/tensorflow/contrib/data/python/kernel_tests/batch_dataset_op_test.py"
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/variable_scope_test.py"
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/reshape_op_test.py"
       "${tensorflow_source_dir}/tensorflow/python/training/evaluation_test.py"
-- 
GitLab


From 71568313f00efc0c3521f2e194ed66dc60b61f90 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 12 Oct 2017 19:45:43 -0700
Subject: [PATCH 343/909] Add streaming_false_{negative,positive}_rate and
 streaming_false_{negative,positive}_rate_at_thresholds.

PiperOrigin-RevId: 172048554
---
 tensorflow/contrib/metrics/__init__.py        |   8 +
 .../contrib/metrics/python/ops/metric_ops.py  | 347 +++++++++
 .../metrics/python/ops/metric_ops_test.py     | 728 ++++++++++++++++++
 3 files changed, 1083 insertions(+)

diff --git a/tensorflow/contrib/metrics/__init__.py b/tensorflow/contrib/metrics/__init__.py
index a9bce65e55..2c48882d0e 100644
--- a/tensorflow/contrib/metrics/__init__.py
+++ b/tensorflow/contrib/metrics/__init__.py
@@ -22,6 +22,10 @@ See the @{$python/contrib.metrics} guide.
 @@streaming_recall_at_thresholds
 @@streaming_precision
 @@streaming_precision_at_thresholds
+@@streaming_false_positive_rate
+@@streaming_false_positive_rate_at_thresholds
+@@streaming_false_negative_rate
+@@streaming_false_negative_rate_at_thresholds
 @@streaming_auc
 @@streaming_curve_points
 @@streaming_recall_at_k
@@ -80,8 +84,12 @@ from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_auc
 from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_concat
 from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_covariance
 from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_curve_points
+from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_false_negative_rate
+from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_false_negative_rate_at_thresholds
 from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_false_negatives
 from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_false_negatives_at_thresholds
+from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_false_positive_rate
+from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_false_positive_rate_at_thresholds
 from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_false_positives
 from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_false_positives_at_thresholds
 from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_mean
diff --git a/tensorflow/contrib/metrics/python/ops/metric_ops.py b/tensorflow/contrib/metrics/python/ops/metric_ops.py
index 76986d0156..85c8e9038a 100644
--- a/tensorflow/contrib/metrics/python/ops/metric_ops.py
+++ b/tensorflow/contrib/metrics/python/ops/metric_ops.py
@@ -565,6 +565,213 @@ def streaming_recall(predictions, labels, weights=None,
       updates_collections=updates_collections, name=name)
 
 
+def _true_negatives(labels, predictions, weights=None,
+                    metrics_collections=None,
+                    updates_collections=None,
+                    name=None):
+  """Sum the weights of true negatives.
+
+  If `weights` is `None`, weights default to 1. Use weights of 0 to mask values.
+
+  Args:
+    labels: The ground truth values, a `Tensor` whose dimensions must match
+      `predictions`. Will be cast to `bool`.
+    predictions: The predicted values, a `Tensor` of arbitrary dimensions. Will
+      be cast to `bool`.
+    weights: Optional `Tensor` whose rank is either 0, or the same rank as
+      `labels`, and must be broadcastable to `labels` (i.e., all dimensions must
+      be either `1`, or the same as the corresponding `labels` dimension).
+    metrics_collections: An optional list of collections that the metric
+      value variable should be added to.
+    updates_collections: An optional list of collections that the metric update
+      ops should be added to.
+    name: An optional variable_scope name.
+
+  Returns:
+    value_tensor: A `Tensor` representing the current value of the metric.
+    update_op: An operation that accumulates the error from a batch of data.
+
+  Raises:
+    ValueError: If `predictions` and `labels` have mismatched shapes, or if
+      `weights` is not `None` and its shape doesn't match `predictions`, or if
+      either `metrics_collections` or `updates_collections` are not a list or
+      tuple.
+  """
+  with variable_scope.variable_scope(
+      name, 'true_negatives', (predictions, labels, weights)):
+
+    predictions, labels, weights = _remove_squeezable_dimensions(
+        predictions=math_ops.cast(predictions, dtype=dtypes.bool),
+        labels=math_ops.cast(labels, dtype=dtypes.bool),
+        weights=weights)
+    is_true_negative = math_ops.logical_and(math_ops.equal(labels, False),
+                                            math_ops.equal(predictions, False))
+    return _count_condition(is_true_negative, weights, metrics_collections,
+                            updates_collections)
+
+
+def streaming_false_positive_rate(predictions, labels, weights=None,
+                                  metrics_collections=None,
+                                  updates_collections=None,
+                                  name=None):
+  """Computes the false positive rate of predictions with respect to labels.
+
+  The `false_positive_rate` function creates two local variables,
+  `false_positives` and `true_negatives`, that are used to compute the
+  false positive rate. This value is ultimately returned as
+  `false_positive_rate`, an idempotent operation that simply divides
+  `false_positives` by the sum of `false_positives` and `true_negatives`.
+
+  For estimation of the metric over a stream of data, the function creates an
+  `update_op` operation that updates these variables and returns the
+  `false_positive_rate`. `update_op` weights each prediction by the
+  corresponding value in `weights`.
+
+  If `weights` is `None`, weights default to 1. Use weights of 0 to mask values.
+
+  Args:
+    predictions: The predicted values, a `Tensor` of arbitrary dimensions. Will
+      be cast to `bool`.
+    labels: The ground truth values, a `Tensor` whose dimensions must match
+      `predictions`. Will be cast to `bool`.
+    weights: Optional `Tensor` whose rank is either 0, or the same rank as
+      `labels`, and must be broadcastable to `labels` (i.e., all dimensions must
+      be either `1`, or the same as the corresponding `labels` dimension).
+    metrics_collections: An optional list of collections that
+     `false_positive_rate` should be added to.
+    updates_collections: An optional list of collections that `update_op` should
+      be added to.
+    name: An optional variable_scope name.
+
+  Returns:
+    false_positive_rate: Scalar float `Tensor` with the value of
+      `false_positives` divided by the sum of `false_positives` and
+      `true_negatives`.
+    update_op: `Operation` that increments `false_positives` and
+      `true_negatives` variables appropriately and whose value matches
+      `false_positive_rate`.
+
+  Raises:
+    ValueError: If `predictions` and `labels` have mismatched shapes, or if
+      `weights` is not `None` and its shape doesn't match `predictions`, or if
+      either `metrics_collections` or `updates_collections` are not a list or
+      tuple.
+  """
+  with variable_scope.variable_scope(
+      name, 'false_positive_rate', (predictions, labels, weights)):
+    predictions, labels, weights = _remove_squeezable_dimensions(
+        predictions=math_ops.cast(predictions, dtype=dtypes.bool),
+        labels=math_ops.cast(labels, dtype=dtypes.bool),
+        weights=weights)
+
+    false_p, false_positives_update_op = metrics.false_positives(
+        labels, predictions, weights, metrics_collections=None,
+        updates_collections=None, name=None)
+    true_n, true_negatives_update_op = _true_negatives(
+        labels, predictions, weights, metrics_collections=None,
+        updates_collections=None, name=None)
+
+    def compute_fpr(fp, tn, name):
+      return array_ops.where(
+          math_ops.greater(fp + tn, 0),
+          math_ops.div(fp, fp + tn),
+          0,
+          name)
+
+    fpr = compute_fpr(false_p, true_n, 'value')
+    update_op = compute_fpr(
+        false_positives_update_op, true_negatives_update_op, 'update_op')
+
+    if metrics_collections:
+      ops.add_to_collections(metrics_collections, fpr)
+
+    if updates_collections:
+      ops.add_to_collections(updates_collections, update_op)
+
+    return fpr, update_op
+
+
+def streaming_false_negative_rate(predictions, labels, weights=None,
+                                  metrics_collections=None,
+                                  updates_collections=None,
+                                  name=None):
+  """Computes the false negative rate of predictions with respect to labels.
+
+  The `false_negative_rate` function creates two local variables,
+  `false_negatives` and `true_positives`, that are used to compute the
+  false positive rate. This value is ultimately returned as
+  `false_negative_rate`, an idempotent operation that simply divides
+  `false_negatives` by the sum of `false_negatives` and `true_positives`.
+
+  For estimation of the metric over a stream of data, the function creates an
+  `update_op` operation that updates these variables and returns the
+  `false_negative_rate`. `update_op` weights each prediction by the
+  corresponding value in `weights`.
+
+  If `weights` is `None`, weights default to 1. Use weights of 0 to mask values.
+
+  Args:
+    predictions: The predicted values, a `Tensor` of arbitrary dimensions. Will
+      be cast to `bool`.
+    labels: The ground truth values, a `Tensor` whose dimensions must match
+      `predictions`. Will be cast to `bool`.
+    weights: Optional `Tensor` whose rank is either 0, or the same rank as
+      `labels`, and must be broadcastable to `labels` (i.e., all dimensions must
+      be either `1`, or the same as the corresponding `labels` dimension).
+    metrics_collections: An optional list of collections that
+      `false_negative_rate` should be added to.
+    updates_collections: An optional list of collections that `update_op` should
+      be added to.
+    name: An optional variable_scope name.
+
+  Returns:
+    false_negative_rate: Scalar float `Tensor` with the value of
+      `false_negatives` divided by the sum of `false_negatives` and
+      `true_positives`.
+    update_op: `Operation` that increments `false_negatives` and
+      `true_positives` variables appropriately and whose value matches
+      `false_negative_rate`.
+
+  Raises:
+    ValueError: If `predictions` and `labels` have mismatched shapes, or if
+      `weights` is not `None` and its shape doesn't match `predictions`, or if
+      either `metrics_collections` or `updates_collections` are not a list or
+      tuple.
+  """
+  with variable_scope.variable_scope(
+      name, 'false_negative_rate', (predictions, labels, weights)):
+    predictions, labels, weights = _remove_squeezable_dimensions(
+        predictions=math_ops.cast(predictions, dtype=dtypes.bool),
+        labels=math_ops.cast(labels, dtype=dtypes.bool),
+        weights=weights)
+
+    false_n, false_negatives_update_op = metrics.false_negatives(
+        labels, predictions, weights, metrics_collections=None,
+        updates_collections=None, name=None)
+    true_p, true_positives_update_op = metrics.true_positives(
+        labels, predictions, weights, metrics_collections=None,
+        updates_collections=None, name=None)
+
+    def compute_fnr(fn, tp, name):
+      return array_ops.where(
+          math_ops.greater(fn + tp, 0),
+          math_ops.div(fn, fn + tp),
+          0,
+          name)
+
+    fnr = compute_fnr(false_n, true_p, 'value')
+    update_op = compute_fnr(
+        false_negatives_update_op, true_positives_update_op, 'update_op')
+
+    if metrics_collections:
+      ops.add_to_collections(metrics_collections, fnr)
+
+    if updates_collections:
+      ops.add_to_collections(updates_collections, update_op)
+
+    return fnr, update_op
+
+
 def _streaming_confusion_matrix_at_thresholds(
     predictions, labels, thresholds, weights=None, includes=None):
   """Computes true_positives, false_negatives, true_negatives, false_positives.
@@ -1114,6 +1321,142 @@ def streaming_recall_at_thresholds(predictions, labels, thresholds,
       updates_collections=updates_collections, name=name)
 
 
+def streaming_false_positive_rate_at_thresholds(
+    predictions, labels, thresholds, weights=None, metrics_collections=None,
+    updates_collections=None, name=None):
+  """Computes various fpr values for different `thresholds` on `predictions`.
+
+  The `streaming_false_positive_rate_at_thresholds` function creates two
+  local variables, `false_positives`, `true_negatives`, for various values of
+  thresholds. `false_positive_rate[i]` is defined as the total weight
+  of values in `predictions` above `thresholds[i]` whose corresponding entry in
+  `labels` is `False`, divided by the total weight of `False` values in `labels`
+  (`false_positives[i] / (false_positives[i] + true_negatives[i])`).
+
+  For estimation of the metric over a stream of data, the function creates an
+  `update_op` operation that updates these variables and returns the
+  `false_positive_rate`.
+
+  If `weights` is `None`, weights default to 1. Use weights of 0 to mask values.
+
+  Args:
+    predictions: A floating point `Tensor` of arbitrary shape and whose values
+      are in the range `[0, 1]`.
+    labels: A `bool` `Tensor` whose shape matches `predictions`.
+    thresholds: A python list or tuple of float thresholds in `[0, 1]`.
+    weights: `Tensor` whose rank is either 0, or the same rank as `labels`, and
+      must be broadcastable to `labels` (i.e., all dimensions must be either
+      `1`, or the same as the corresponding `labels` dimension).
+    metrics_collections: An optional list of collections that
+      `false_positive_rate` should be added to.
+    updates_collections: An optional list of collections that `update_op` should
+      be added to.
+    name: An optional variable_scope name.
+
+  Returns:
+    false_positive_rate: A float `Tensor` of shape `[len(thresholds)]`.
+    update_op: An operation that increments the `false_positives` and
+      `true_negatives` variables that are used in the computation of
+      `false_positive_rate`.
+
+  Raises:
+    ValueError: If `predictions` and `labels` have mismatched shapes, or if
+      `weights` is not `None` and its shape doesn't match `predictions`, or if
+      either `metrics_collections` or `updates_collections` are not a list or
+      tuple.
+  """
+  with variable_scope.variable_scope(
+      name, 'false_positive_rate_at_thresholds',
+      (predictions, labels, weights)):
+    values, update_ops = _streaming_confusion_matrix_at_thresholds(
+        predictions, labels, thresholds, weights, includes=('fp', 'tn'))
+
+    # Avoid division by zero.
+    epsilon = 1e-7
+    def compute_fpr(fp, tn, name):
+      return math_ops.div(fp, epsilon + fp + tn, name='fpr_' + name)
+
+    fpr = compute_fpr(values['fp'], values['tn'], 'value')
+    update_op = compute_fpr(
+        update_ops['fp'], update_ops['tn'], 'update_op')
+
+    if metrics_collections:
+      ops.add_to_collections(metrics_collections, fpr)
+
+    if updates_collections:
+      ops.add_to_collections(updates_collections, update_op)
+
+    return fpr, update_op
+
+
+def streaming_false_negative_rate_at_thresholds(
+    predictions, labels, thresholds, weights=None, metrics_collections=None,
+    updates_collections=None, name=None):
+  """Computes various fnr values for different `thresholds` on `predictions`.
+
+  The `streaming_false_negative_rate_at_thresholds` function creates two
+  local variables, `false_negatives`, `true_positives`, for various values of
+  thresholds. `false_negative_rate[i]` is defined as the total weight
+  of values in `predictions` above `thresholds[i]` whose corresponding entry in
+  `labels` is `False`, divided by the total weight of `True` values in `labels`
+  (`false_negatives[i] / (false_negatives[i] + true_positives[i])`).
+
+  For estimation of the metric over a stream of data, the function creates an
+  `update_op` operation that updates these variables and returns the
+  `false_positive_rate`.
+
+  If `weights` is `None`, weights default to 1. Use weights of 0 to mask values.
+
+  Args:
+    predictions: A floating point `Tensor` of arbitrary shape and whose values
+      are in the range `[0, 1]`.
+    labels: A `bool` `Tensor` whose shape matches `predictions`.
+    thresholds: A python list or tuple of float thresholds in `[0, 1]`.
+    weights: `Tensor` whose rank is either 0, or the same rank as `labels`, and
+      must be broadcastable to `labels` (i.e., all dimensions must be either
+      `1`, or the same as the corresponding `labels` dimension).
+    metrics_collections: An optional list of collections that
+      `false_negative_rate` should be added to.
+    updates_collections: An optional list of collections that `update_op` should
+      be added to.
+    name: An optional variable_scope name.
+
+  Returns:
+    false_negative_rate: A float `Tensor` of shape `[len(thresholds)]`.
+    update_op: An operation that increments the `false_negatives` and
+      `true_positives` variables that are used in the computation of
+      `false_negative_rate`.
+
+  Raises:
+    ValueError: If `predictions` and `labels` have mismatched shapes, or if
+      `weights` is not `None` and its shape doesn't match `predictions`, or if
+      either `metrics_collections` or `updates_collections` are not a list or
+      tuple.
+  """
+  with variable_scope.variable_scope(
+      name, 'false_negative_rate_at_thresholds',
+      (predictions, labels, weights)):
+    values, update_ops = _streaming_confusion_matrix_at_thresholds(
+        predictions, labels, thresholds, weights, includes=('fn', 'tp'))
+
+    # Avoid division by zero.
+    epsilon = 1e-7
+    def compute_fnr(fn, tp, name):
+      return math_ops.div(fn, epsilon + fn + tp, name='fnr_' + name)
+
+    fnr = compute_fnr(values['fn'], values['tp'], 'value')
+    update_op = compute_fnr(
+        update_ops['fn'], update_ops['tp'], 'update_op')
+
+    if metrics_collections:
+      ops.add_to_collections(metrics_collections, fnr)
+
+    if updates_collections:
+      ops.add_to_collections(updates_collections, update_op)
+
+    return fnr, update_op
+
+
 def _at_k_name(name, k=None, class_id=None):
   if k is not None:
     name = '%s_at_%d' % (name, k)
@@ -2479,8 +2822,12 @@ __all__ = [
     'streaming_accuracy',
     'streaming_auc',
     'streaming_curve_points',
+    'streaming_false_negative_rate',
+    'streaming_false_negative_rate_at_thresholds',
     'streaming_false_negatives',
     'streaming_false_negatives_at_thresholds',
+    'streaming_false_positive_rate',
+    'streaming_false_positive_rate_at_thresholds',
     'streaming_false_positives',
     'streaming_false_positives_at_thresholds',
     'streaming_mean',
diff --git a/tensorflow/contrib/metrics/python/ops/metric_ops_test.py b/tensorflow/contrib/metrics/python/ops/metric_ops_test.py
index 9b959b43a9..e2067297cd 100644
--- a/tensorflow/contrib/metrics/python/ops/metric_ops_test.py
+++ b/tensorflow/contrib/metrics/python/ops/metric_ops_test.py
@@ -1355,6 +1355,262 @@ class StreamingRecallTest(test.TestCase):
       self.assertEqual(0, recall.eval())
 
 
+class StreamingFPRTest(test.TestCase):
+
+  def setUp(self):
+    np.random.seed(1)
+    ops.reset_default_graph()
+
+  def testVars(self):
+    metrics.streaming_false_positive_rate(
+        predictions=array_ops.ones((10, 1)), labels=array_ops.ones((10, 1)))
+    _assert_local_variables(self, (
+        'false_positive_rate/false_positives/count:0',
+        'false_positive_rate/true_negatives/count:0'))
+
+  def testMetricsCollection(self):
+    my_collection_name = '__metrics__'
+    mean, _ = metrics.streaming_false_positive_rate(
+        predictions=array_ops.ones((10, 1)),
+        labels=array_ops.ones((10, 1)),
+        metrics_collections=[my_collection_name])
+    self.assertListEqual(ops.get_collection(my_collection_name), [mean])
+
+  def testUpdatesCollection(self):
+    my_collection_name = '__updates__'
+    _, update_op = metrics.streaming_false_positive_rate(
+        predictions=array_ops.ones((10, 1)),
+        labels=array_ops.ones((10, 1)),
+        updates_collections=[my_collection_name])
+    self.assertListEqual(ops.get_collection(my_collection_name), [update_op])
+
+  def testValueTensorIsIdempotent(self):
+    predictions = random_ops.random_uniform(
+        (10, 3), maxval=1, dtype=dtypes_lib.int64, seed=1)
+    labels = random_ops.random_uniform(
+        (10, 3), maxval=1, dtype=dtypes_lib.int64, seed=2)
+    fpr, update_op = metrics.streaming_false_positive_rate(
+        predictions, labels)
+
+    with self.test_session() as sess:
+      sess.run(variables.local_variables_initializer())
+
+      # Run several updates.
+      for _ in range(10):
+        sess.run(update_op)
+
+      # Then verify idempotency.
+      initial_fpr = fpr.eval()
+      for _ in range(10):
+        self.assertEqual(initial_fpr, fpr.eval())
+
+  def testAllCorrect(self):
+    np_inputs = np.random.randint(0, 2, size=(100, 1))
+
+    predictions = constant_op.constant(np_inputs)
+    labels = constant_op.constant(np_inputs)
+    fpr, update_op = metrics.streaming_false_positive_rate(
+        predictions, labels)
+
+    with self.test_session() as sess:
+      sess.run(variables.local_variables_initializer())
+      sess.run(update_op)
+      self.assertEqual(0, fpr.eval())
+
+  def testSomeCorrect(self):
+    predictions = constant_op.constant([1, 0, 1, 0], shape=(1, 4))
+    labels = constant_op.constant([0, 1, 1, 0], shape=(1, 4))
+    fpr, update_op = metrics.streaming_false_positive_rate(
+        predictions, labels)
+
+    with self.test_session() as sess:
+      sess.run(variables.local_variables_initializer())
+      self.assertAlmostEqual(0.5, update_op.eval())
+      self.assertAlmostEqual(0.5, fpr.eval())
+
+  def testWeighted1d(self):
+    predictions = constant_op.constant([[1, 0, 1, 0], [0, 1, 0, 1]])
+    labels = constant_op.constant([[0, 1, 1, 0], [1, 0, 0, 1]])
+    weights = constant_op.constant([[2], [5]])
+    fpr, update_op = metrics.streaming_false_positive_rate(
+        predictions, labels, weights=weights)
+
+    with self.test_session() as sess:
+      sess.run(variables.local_variables_initializer())
+      weighted_fp = 2.0 + 5.0
+      weighted_f = (2.0 + 2.0) + (5.0 + 5.0)
+      expected_fpr = weighted_fp / weighted_f
+      self.assertAlmostEqual(expected_fpr, update_op.eval())
+      self.assertAlmostEqual(expected_fpr, fpr.eval())
+
+  def testWeighted2d(self):
+    predictions = constant_op.constant([[1, 0, 1, 0], [0, 1, 0, 1]])
+    labels = constant_op.constant([[0, 1, 1, 0], [1, 0, 0, 1]])
+    weights = constant_op.constant([[1, 2, 3, 4], [4, 3, 2, 1]])
+    fpr, update_op = metrics.streaming_false_positive_rate(
+        predictions, labels, weights=weights)
+
+    with self.test_session() as sess:
+      sess.run(variables.local_variables_initializer())
+      weighted_fp = 1.0 + 3.0
+      weighted_f = (1.0 + 4.0) + (2.0 + 3.0)
+      expected_fpr = weighted_fp / weighted_f
+      self.assertAlmostEqual(expected_fpr, update_op.eval())
+      self.assertAlmostEqual(expected_fpr, fpr.eval())
+
+  def testAllIncorrect(self):
+    np_inputs = np.random.randint(0, 2, size=(100, 1))
+
+    predictions = constant_op.constant(np_inputs)
+    labels = constant_op.constant(1 - np_inputs)
+    fpr, update_op = metrics.streaming_false_positive_rate(
+        predictions, labels)
+
+    with self.test_session() as sess:
+      sess.run(variables.local_variables_initializer())
+      sess.run(update_op)
+      self.assertEqual(1, fpr.eval())
+
+  def testZeroFalsePositivesAndTrueNegativesGivesZeroFPR(self):
+    predictions = array_ops.ones((1, 4))
+    labels = array_ops.ones((1, 4))
+    fpr, update_op = metrics.streaming_false_positive_rate(
+        predictions, labels)
+
+    with self.test_session() as sess:
+      sess.run(variables.local_variables_initializer())
+      sess.run(update_op)
+      self.assertEqual(0, fpr.eval())
+
+
+class StreamingFNRTest(test.TestCase):
+
+  def setUp(self):
+    np.random.seed(1)
+    ops.reset_default_graph()
+
+  def testVars(self):
+    metrics.streaming_false_negative_rate(
+        predictions=array_ops.ones((10, 1)), labels=array_ops.ones((10, 1)))
+    _assert_local_variables(self, (
+        'false_negative_rate/false_negatives/count:0',
+        'false_negative_rate/true_positives/count:0'))
+
+  def testMetricsCollection(self):
+    my_collection_name = '__metrics__'
+    mean, _ = metrics.streaming_false_negative_rate(
+        predictions=array_ops.ones((10, 1)),
+        labels=array_ops.ones((10, 1)),
+        metrics_collections=[my_collection_name])
+    self.assertListEqual(ops.get_collection(my_collection_name), [mean])
+
+  def testUpdatesCollection(self):
+    my_collection_name = '__updates__'
+    _, update_op = metrics.streaming_false_negative_rate(
+        predictions=array_ops.ones((10, 1)),
+        labels=array_ops.ones((10, 1)),
+        updates_collections=[my_collection_name])
+    self.assertListEqual(ops.get_collection(my_collection_name), [update_op])
+
+  def testValueTensorIsIdempotent(self):
+    predictions = random_ops.random_uniform(
+        (10, 3), maxval=1, dtype=dtypes_lib.int64, seed=1)
+    labels = random_ops.random_uniform(
+        (10, 3), maxval=1, dtype=dtypes_lib.int64, seed=2)
+    fnr, update_op = metrics.streaming_false_negative_rate(
+        predictions, labels)
+
+    with self.test_session() as sess:
+      sess.run(variables.local_variables_initializer())
+
+      # Run several updates.
+      for _ in range(10):
+        sess.run(update_op)
+
+      # Then verify idempotency.
+      initial_fnr = fnr.eval()
+      for _ in range(10):
+        self.assertEqual(initial_fnr, fnr.eval())
+
+  def testAllCorrect(self):
+    np_inputs = np.random.randint(0, 2, size=(100, 1))
+
+    predictions = constant_op.constant(np_inputs)
+    labels = constant_op.constant(np_inputs)
+    fnr, update_op = metrics.streaming_false_negative_rate(
+        predictions, labels)
+
+    with self.test_session() as sess:
+      sess.run(variables.local_variables_initializer())
+      sess.run(update_op)
+      self.assertEqual(0, fnr.eval())
+
+  def testSomeCorrect(self):
+    predictions = constant_op.constant([1, 0, 1, 0], shape=(1, 4))
+    labels = constant_op.constant([0, 1, 1, 0], shape=(1, 4))
+    fnr, update_op = metrics.streaming_false_negative_rate(
+        predictions, labels)
+
+    with self.test_session() as sess:
+      sess.run(variables.local_variables_initializer())
+      self.assertAlmostEqual(0.5, update_op.eval())
+      self.assertAlmostEqual(0.5, fnr.eval())
+
+  def testWeighted1d(self):
+    predictions = constant_op.constant([[1, 0, 1, 0], [0, 1, 0, 1]])
+    labels = constant_op.constant([[0, 1, 1, 0], [1, 0, 0, 1]])
+    weights = constant_op.constant([[2], [5]])
+    fnr, update_op = metrics.streaming_false_negative_rate(
+        predictions, labels, weights=weights)
+
+    with self.test_session() as sess:
+      sess.run(variables.local_variables_initializer())
+      weighted_fn = 2.0 + 5.0
+      weighted_t = (2.0 + 2.0) + (5.0 + 5.0)
+      expected_fnr = weighted_fn / weighted_t
+      self.assertAlmostEqual(expected_fnr, update_op.eval())
+      self.assertAlmostEqual(expected_fnr, fnr.eval())
+
+  def testWeighted2d(self):
+    predictions = constant_op.constant([[1, 0, 1, 0], [0, 1, 0, 1]])
+    labels = constant_op.constant([[0, 1, 1, 0], [1, 0, 0, 1]])
+    weights = constant_op.constant([[1, 2, 3, 4], [4, 3, 2, 1]])
+    fnr, update_op = metrics.streaming_false_negative_rate(
+        predictions, labels, weights=weights)
+
+    with self.test_session() as sess:
+      sess.run(variables.local_variables_initializer())
+      weighted_fn = 2.0 + 4.0
+      weighted_t = (2.0 + 3.0) + (1.0 + 4.0)
+      expected_fnr = weighted_fn / weighted_t
+      self.assertAlmostEqual(expected_fnr, update_op.eval())
+      self.assertAlmostEqual(expected_fnr, fnr.eval())
+
+  def testAllIncorrect(self):
+    np_inputs = np.random.randint(0, 2, size=(100, 1))
+
+    predictions = constant_op.constant(np_inputs)
+    labels = constant_op.constant(1 - np_inputs)
+    fnr, update_op = metrics.streaming_false_negative_rate(
+        predictions, labels)
+
+    with self.test_session() as sess:
+      sess.run(variables.local_variables_initializer())
+      sess.run(update_op)
+      self.assertEqual(1, fnr.eval())
+
+  def testZeroFalseNegativesAndTruePositivesGivesZeroFNR(self):
+    predictions = array_ops.zeros((1, 4))
+    labels = array_ops.zeros((1, 4))
+    fnr, update_op = metrics.streaming_false_negative_rate(
+        predictions, labels)
+
+    with self.test_session() as sess:
+      sess.run(variables.local_variables_initializer())
+      sess.run(update_op)
+      self.assertEqual(0, fnr.eval())
+
+
 class StreamingCurvePointsTest(test.TestCase):
 
   def setUp(self):
@@ -2268,6 +2524,478 @@ class StreamingPrecisionRecallThresholdsTest(test.TestCase):
       self.assertAlmostEqual(expected_rec, rec.eval(), 2)
 
 
+class StreamingFPRThresholdsTest(test.TestCase):
+
+  def setUp(self):
+    np.random.seed(1)
+    ops.reset_default_graph()
+
+  def testVars(self):
+    metrics.streaming_false_positive_rate_at_thresholds(
+        predictions=array_ops.ones((10, 1)),
+        labels=array_ops.ones((10, 1)),
+        thresholds=[0, 0.5, 1.0])
+    _assert_local_variables(self, (
+        'false_positive_rate_at_thresholds/false_positives:0',
+        'false_positive_rate_at_thresholds/true_negatives:0',))
+
+  def testMetricsCollection(self):
+    my_collection_name = '__metrics__'
+    fpr, _ = metrics.streaming_false_positive_rate_at_thresholds(
+        predictions=array_ops.ones((10, 1)),
+        labels=array_ops.ones((10, 1)),
+        thresholds=[0, 0.5, 1.0],
+        metrics_collections=[my_collection_name])
+    self.assertListEqual(ops.get_collection(my_collection_name), [fpr])
+
+  def testUpdatesCollection(self):
+    my_collection_name = '__updates__'
+    _, update_op = metrics.streaming_false_positive_rate_at_thresholds(
+        predictions=array_ops.ones((10, 1)),
+        labels=array_ops.ones((10, 1)),
+        thresholds=[0, 0.5, 1.0],
+        updates_collections=[my_collection_name])
+    self.assertListEqual(
+        ops.get_collection(my_collection_name), [update_op])
+
+  def testValueTensorIsIdempotent(self):
+    predictions = random_ops.random_uniform(
+        (10, 3), maxval=1, dtype=dtypes_lib.float32, seed=1)
+    labels = random_ops.random_uniform(
+        (10, 3), maxval=1, dtype=dtypes_lib.int64, seed=2)
+    thresholds = [0, 0.5, 1.0]
+    fpr, fpr_op = metrics.streaming_false_positive_rate_at_thresholds(
+        predictions, labels, thresholds)
+
+    with self.test_session() as sess:
+      sess.run(variables.local_variables_initializer())
+
+      # Run several updates.
+      for _ in range(10):
+        sess.run(fpr_op)
+
+      # Then verify idempotency.
+      initial_fpr = fpr.eval()
+      for _ in range(10):
+        self.assertAllClose(initial_fpr, fpr.eval())
+
+  def testAllCorrect(self):
+    inputs = np.random.randint(0, 2, size=(100, 1))
+
+    with self.test_session() as sess:
+      predictions = constant_op.constant(inputs, dtype=dtypes_lib.float32)
+      labels = constant_op.constant(inputs)
+      thresholds = [0.5]
+      fpr, fpr_op = metrics.streaming_false_positive_rate_at_thresholds(
+          predictions, labels, thresholds)
+
+      sess.run(variables.local_variables_initializer())
+      sess.run(fpr_op)
+
+      self.assertEqual(0, fpr.eval())
+
+  def testSomeCorrect(self):
+    with self.test_session() as sess:
+      predictions = constant_op.constant(
+          [1, 0, 1, 0], shape=(1, 4), dtype=dtypes_lib.float32)
+      labels = constant_op.constant([0, 1, 1, 0], shape=(1, 4))
+      thresholds = [0.5]
+      fpr, fpr_op = metrics.streaming_false_positive_rate_at_thresholds(
+          predictions, labels, thresholds)
+
+      sess.run(variables.local_variables_initializer())
+      sess.run(fpr_op)
+
+      self.assertAlmostEqual(0.5, fpr.eval())
+
+  def testAllIncorrect(self):
+    inputs = np.random.randint(0, 2, size=(100, 1))
+
+    with self.test_session() as sess:
+      predictions = constant_op.constant(inputs, dtype=dtypes_lib.float32)
+      labels = constant_op.constant(1 - inputs, dtype=dtypes_lib.float32)
+      thresholds = [0.5]
+      fpr, fpr_op = metrics.streaming_false_positive_rate_at_thresholds(
+          predictions, labels, thresholds)
+
+      sess.run(variables.local_variables_initializer())
+      sess.run(fpr_op)
+
+      self.assertAlmostEqual(1, fpr.eval())
+
+  def testWeights1d(self):
+    with self.test_session() as sess:
+      predictions = constant_op.constant(
+          [[1, 0], [1, 0]], shape=(2, 2), dtype=dtypes_lib.float32)
+      labels = constant_op.constant([[0, 1], [1, 0]], shape=(2, 2))
+      weights = constant_op.constant(
+          [[0], [1]], shape=(2, 1), dtype=dtypes_lib.float32)
+      thresholds = [0.5, 1.1]
+      fpr, fpr_op = metrics.streaming_false_positive_rate_at_thresholds(
+          predictions, labels, thresholds, weights=weights)
+
+      [fpr_low, fpr_high] = array_ops.split(
+          value=fpr, num_or_size_splits=2, axis=0)
+      fpr_low = array_ops.reshape(fpr_low, shape=())
+      fpr_high = array_ops.reshape(fpr_high, shape=())
+
+      sess.run(variables.local_variables_initializer())
+      sess.run(fpr_op)
+
+      self.assertAlmostEqual(0.0, fpr_low.eval(), places=5)
+      self.assertAlmostEqual(0.0, fpr_high.eval(), places=5)
+
+  def testWeights2d(self):
+    with self.test_session() as sess:
+      predictions = constant_op.constant(
+          [[1, 0], [1, 0]], shape=(2, 2), dtype=dtypes_lib.float32)
+      labels = constant_op.constant([[0, 1], [1, 0]], shape=(2, 2))
+      weights = constant_op.constant(
+          [[0, 0], [1, 1]], shape=(2, 2), dtype=dtypes_lib.float32)
+      thresholds = [0.5, 1.1]
+      fpr, fpr_op = metrics.streaming_false_positive_rate_at_thresholds(
+          predictions, labels, thresholds, weights=weights)
+
+      [fpr_low, fpr_high] = array_ops.split(
+          value=fpr, num_or_size_splits=2, axis=0)
+      fpr_low = array_ops.reshape(fpr_low, shape=())
+      fpr_high = array_ops.reshape(fpr_high, shape=())
+
+      sess.run(variables.local_variables_initializer())
+      sess.run(fpr_op)
+
+      self.assertAlmostEqual(0.0, fpr_low.eval(), places=5)
+      self.assertAlmostEqual(0.0, fpr_high.eval(), places=5)
+
+  def testExtremeThresholds(self):
+    with self.test_session() as sess:
+      predictions = constant_op.constant(
+          [1, 0, 1, 0], shape=(1, 4), dtype=dtypes_lib.float32)
+      labels = constant_op.constant([0, 1, 1, 1], shape=(1, 4))
+      thresholds = [-1.0, 2.0]  # lower/higher than any values
+      fpr, fpr_op = metrics.streaming_false_positive_rate_at_thresholds(
+          predictions, labels, thresholds)
+
+      [fpr_low, fpr_high] = array_ops.split(
+          value=fpr, num_or_size_splits=2, axis=0)
+
+      sess.run(variables.local_variables_initializer())
+      sess.run(fpr_op)
+
+      self.assertAlmostEqual(1.0, fpr_low.eval(), places=5)
+      self.assertAlmostEqual(0.0, fpr_high.eval(), places=5)
+
+  def testZeroLabelsPredictions(self):
+    with self.test_session() as sess:
+      predictions = array_ops.zeros([4], dtype=dtypes_lib.float32)
+      labels = array_ops.zeros([4])
+      thresholds = [0.5]
+      fpr, fpr_op = metrics.streaming_false_positive_rate_at_thresholds(
+          predictions, labels, thresholds)
+
+      sess.run(variables.local_variables_initializer())
+      sess.run(fpr_op)
+
+      self.assertAlmostEqual(0, fpr.eval(), 6)
+
+  def testWithMultipleUpdates(self):
+    num_samples = 1000
+    batch_size = 10
+    num_batches = int(num_samples / batch_size)
+
+    # Create the labels and data.
+    labels = np.random.randint(0, 2, size=(num_samples, 1))
+    noise = np.random.normal(0.0, scale=0.2, size=(num_samples, 1))
+    predictions = 0.4 + 0.2 * labels + noise
+    predictions[predictions > 1] = 1
+    predictions[predictions < 0] = 0
+    thresholds = [0.3]
+
+    fp = 0
+    tn = 0
+    for i in range(num_samples):
+      if predictions[i] > thresholds[0]:
+        if labels[i] == 0:
+          fp += 1
+      else:
+        if labels[i] == 0:
+          tn += 1
+    epsilon = 1e-7
+    expected_fpr = fp / (epsilon + fp + tn)
+
+    labels = labels.astype(np.float32)
+    predictions = predictions.astype(np.float32)
+
+    with self.test_session() as sess:
+      # Reshape the data so its easy to queue up:
+      predictions_batches = predictions.reshape((batch_size, num_batches))
+      labels_batches = labels.reshape((batch_size, num_batches))
+
+      # Enqueue the data:
+      predictions_queue = data_flow_ops.FIFOQueue(
+          num_batches, dtypes=dtypes_lib.float32, shapes=(batch_size,))
+      labels_queue = data_flow_ops.FIFOQueue(
+          num_batches, dtypes=dtypes_lib.float32, shapes=(batch_size,))
+
+      for i in range(int(num_batches)):
+        tf_prediction = constant_op.constant(predictions_batches[:, i])
+        tf_label = constant_op.constant(labels_batches[:, i])
+        sess.run([
+            predictions_queue.enqueue(tf_prediction),
+            labels_queue.enqueue(tf_label)
+        ])
+
+      tf_predictions = predictions_queue.dequeue()
+      tf_labels = labels_queue.dequeue()
+
+      fpr, fpr_op = metrics.streaming_false_positive_rate_at_thresholds(
+          tf_predictions, tf_labels, thresholds)
+
+      sess.run(variables.local_variables_initializer())
+      for _ in range(int(num_samples / batch_size)):
+        sess.run(fpr_op)
+      # Since this is only approximate, we can't expect a 6 digits match.
+      # Although with higher number of samples/thresholds we should see the
+      # accuracy improving
+      self.assertAlmostEqual(expected_fpr, fpr.eval(), 2)
+
+
+class StreamingFNRThresholdsTest(test.TestCase):
+
+  def setUp(self):
+    np.random.seed(1)
+    ops.reset_default_graph()
+
+  def testVars(self):
+    metrics.streaming_false_negative_rate_at_thresholds(
+        predictions=array_ops.ones((10, 1)),
+        labels=array_ops.ones((10, 1)),
+        thresholds=[0, 0.5, 1.0])
+    _assert_local_variables(self, (
+        'false_negative_rate_at_thresholds/false_negatives:0',
+        'false_negative_rate_at_thresholds/true_positives:0',))
+
+  def testMetricsCollection(self):
+    my_collection_name = '__metrics__'
+    fnr, _ = metrics.streaming_false_negative_rate_at_thresholds(
+        predictions=array_ops.ones((10, 1)),
+        labels=array_ops.ones((10, 1)),
+        thresholds=[0, 0.5, 1.0],
+        metrics_collections=[my_collection_name])
+    self.assertListEqual(ops.get_collection(my_collection_name), [fnr])
+
+  def testUpdatesCollection(self):
+    my_collection_name = '__updates__'
+    _, update_op = metrics.streaming_false_negative_rate_at_thresholds(
+        predictions=array_ops.ones((10, 1)),
+        labels=array_ops.ones((10, 1)),
+        thresholds=[0, 0.5, 1.0],
+        updates_collections=[my_collection_name])
+    self.assertListEqual(
+        ops.get_collection(my_collection_name), [update_op])
+
+  def testValueTensorIsIdempotent(self):
+    predictions = random_ops.random_uniform(
+        (10, 3), maxval=1, dtype=dtypes_lib.float32, seed=1)
+    labels = random_ops.random_uniform(
+        (10, 3), maxval=1, dtype=dtypes_lib.int64, seed=2)
+    thresholds = [0, 0.5, 1.0]
+    fnr, fnr_op = metrics.streaming_false_negative_rate_at_thresholds(
+        predictions, labels, thresholds)
+
+    with self.test_session() as sess:
+      sess.run(variables.local_variables_initializer())
+
+      # Run several updates.
+      for _ in range(10):
+        sess.run(fnr_op)
+
+      # Then verify idempotency.
+      initial_fnr = fnr.eval()
+      for _ in range(10):
+        self.assertAllClose(initial_fnr, fnr.eval())
+
+  def testAllCorrect(self):
+    inputs = np.random.randint(0, 2, size=(100, 1))
+
+    with self.test_session() as sess:
+      predictions = constant_op.constant(inputs, dtype=dtypes_lib.float32)
+      labels = constant_op.constant(inputs)
+      thresholds = [0.5]
+      fnr, fnr_op = metrics.streaming_false_negative_rate_at_thresholds(
+          predictions, labels, thresholds)
+
+      sess.run(variables.local_variables_initializer())
+      sess.run(fnr_op)
+
+      self.assertEqual(0, fnr.eval())
+
+  def testSomeCorrect(self):
+    with self.test_session() as sess:
+      predictions = constant_op.constant(
+          [1, 0, 1, 0], shape=(1, 4), dtype=dtypes_lib.float32)
+      labels = constant_op.constant([0, 1, 1, 0], shape=(1, 4))
+      thresholds = [0.5]
+      fnr, fnr_op = metrics.streaming_false_negative_rate_at_thresholds(
+          predictions, labels, thresholds)
+
+      sess.run(variables.local_variables_initializer())
+      sess.run(fnr_op)
+
+      self.assertAlmostEqual(0.5, fnr.eval())
+
+  def testAllIncorrect(self):
+    inputs = np.random.randint(0, 2, size=(100, 1))
+
+    with self.test_session() as sess:
+      predictions = constant_op.constant(inputs, dtype=dtypes_lib.float32)
+      labels = constant_op.constant(1 - inputs, dtype=dtypes_lib.float32)
+      thresholds = [0.5]
+      fnr, fnr_op = metrics.streaming_false_negative_rate_at_thresholds(
+          predictions, labels, thresholds)
+
+      sess.run(variables.local_variables_initializer())
+      sess.run(fnr_op)
+
+      self.assertAlmostEqual(1, fnr.eval())
+
+  def testWeights1d(self):
+    with self.test_session() as sess:
+      predictions = constant_op.constant(
+          [[1, 0], [1, 0]], shape=(2, 2), dtype=dtypes_lib.float32)
+      labels = constant_op.constant([[0, 1], [1, 0]], shape=(2, 2))
+      weights = constant_op.constant(
+          [[0], [1]], shape=(2, 1), dtype=dtypes_lib.float32)
+      thresholds = [0.5, 1.1]
+      fnr, fnr_op = metrics.streaming_false_negative_rate_at_thresholds(
+          predictions, labels, thresholds, weights=weights)
+
+      [fnr_low, fnr_high] = array_ops.split(
+          value=fnr, num_or_size_splits=2, axis=0)
+      fnr_low = array_ops.reshape(fnr_low, shape=())
+      fnr_high = array_ops.reshape(fnr_high, shape=())
+
+      sess.run(variables.local_variables_initializer())
+      sess.run(fnr_op)
+
+      self.assertAlmostEqual(0.0, fnr_low.eval(), places=5)
+      self.assertAlmostEqual(1.0, fnr_high.eval(), places=5)
+
+  def testWeights2d(self):
+    with self.test_session() as sess:
+      predictions = constant_op.constant(
+          [[1, 0], [1, 0]], shape=(2, 2), dtype=dtypes_lib.float32)
+      labels = constant_op.constant([[0, 1], [1, 0]], shape=(2, 2))
+      weights = constant_op.constant(
+          [[0, 0], [1, 1]], shape=(2, 2), dtype=dtypes_lib.float32)
+      thresholds = [0.5, 1.1]
+      fnr, fnr_op = metrics.streaming_false_negative_rate_at_thresholds(
+          predictions, labels, thresholds, weights=weights)
+
+      [fnr_low, fnr_high] = array_ops.split(
+          value=fnr, num_or_size_splits=2, axis=0)
+      fnr_low = array_ops.reshape(fnr_low, shape=())
+      fnr_high = array_ops.reshape(fnr_high, shape=())
+
+      sess.run(variables.local_variables_initializer())
+      sess.run(fnr_op)
+
+      self.assertAlmostEqual(0.0, fnr_low.eval(), places=5)
+      self.assertAlmostEqual(1.0, fnr_high.eval(), places=5)
+
+  def testExtremeThresholds(self):
+    with self.test_session() as sess:
+      predictions = constant_op.constant(
+          [1, 0, 1, 0], shape=(1, 4), dtype=dtypes_lib.float32)
+      labels = constant_op.constant([0, 1, 1, 1], shape=(1, 4))
+      thresholds = [-1.0, 2.0]  # lower/higher than any values
+      fnr, fnr_op = metrics.streaming_false_negative_rate_at_thresholds(
+          predictions, labels, thresholds)
+
+      [fnr_low, fnr_high] = array_ops.split(
+          value=fnr, num_or_size_splits=2, axis=0)
+
+      sess.run(variables.local_variables_initializer())
+      sess.run(fnr_op)
+
+      self.assertAlmostEqual(0.0, fnr_low.eval())
+      self.assertAlmostEqual(1.0, fnr_high.eval())
+
+  def testZeroLabelsPredictions(self):
+    with self.test_session() as sess:
+      predictions = array_ops.zeros([4], dtype=dtypes_lib.float32)
+      labels = array_ops.zeros([4])
+      thresholds = [0.5]
+      fnr, fnr_op = metrics.streaming_false_negative_rate_at_thresholds(
+          predictions, labels, thresholds)
+
+      sess.run(variables.local_variables_initializer())
+      sess.run(fnr_op)
+
+      self.assertAlmostEqual(0, fnr.eval(), 6)
+
+  def testWithMultipleUpdates(self):
+    num_samples = 1000
+    batch_size = 10
+    num_batches = int(num_samples / batch_size)
+
+    # Create the labels and data.
+    labels = np.random.randint(0, 2, size=(num_samples, 1))
+    noise = np.random.normal(0.0, scale=0.2, size=(num_samples, 1))
+    predictions = 0.4 + 0.2 * labels + noise
+    predictions[predictions > 1] = 1
+    predictions[predictions < 0] = 0
+    thresholds = [0.3]
+
+    fn = 0
+    tp = 0
+    for i in range(num_samples):
+      if predictions[i] > thresholds[0]:
+        if labels[i] == 1:
+          tp += 1
+      else:
+        if labels[i] == 1:
+          fn += 1
+    epsilon = 1e-7
+    expected_fnr = fn / (epsilon + fn + tp)
+
+    labels = labels.astype(np.float32)
+    predictions = predictions.astype(np.float32)
+
+    with self.test_session() as sess:
+      # Reshape the data so its easy to queue up:
+      predictions_batches = predictions.reshape((batch_size, num_batches))
+      labels_batches = labels.reshape((batch_size, num_batches))
+
+      # Enqueue the data:
+      predictions_queue = data_flow_ops.FIFOQueue(
+          num_batches, dtypes=dtypes_lib.float32, shapes=(batch_size,))
+      labels_queue = data_flow_ops.FIFOQueue(
+          num_batches, dtypes=dtypes_lib.float32, shapes=(batch_size,))
+
+      for i in range(int(num_batches)):
+        tf_prediction = constant_op.constant(predictions_batches[:, i])
+        tf_label = constant_op.constant(labels_batches[:, i])
+        sess.run([
+            predictions_queue.enqueue(tf_prediction),
+            labels_queue.enqueue(tf_label)
+        ])
+
+      tf_predictions = predictions_queue.dequeue()
+      tf_labels = labels_queue.dequeue()
+
+      fnr, fnr_op = metrics.streaming_false_negative_rate_at_thresholds(
+          tf_predictions, tf_labels, thresholds)
+
+      sess.run(variables.local_variables_initializer())
+      for _ in range(int(num_samples / batch_size)):
+        sess.run(fnr_op)
+      # Since this is only approximate, we can't expect a 6 digits match.
+      # Although with higher number of samples/thresholds we should see the
+      # accuracy improving
+      self.assertAlmostEqual(expected_fnr, fnr.eval(), 2)
+
+
 # TODO(ptucker): Remove when we remove `streaming_recall_at_k`.
 # This op will be deprecated soon in favor of `streaming_sparse_recall_at_k`.
 # Until then, this test validates that both ops yield the same results.
-- 
GitLab


From 2c5ad1633c47ee4a1a5ccbaa2d4730786708ed92 Mon Sep 17 00:00:00 2001
From: Russell Power <power@google.com>
Date: Thu, 12 Oct 2017 20:15:34 -0700
Subject: [PATCH 344/909] Terminate process when we hit an unexpected exception
 in the infeed queue.

PiperOrigin-RevId: 172050536
---
 tensorflow/contrib/tpu/python/tpu/tpu_estimator.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
index 43f9defd54..f6f89786c5 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 
 import collections
 import copy
+import os
 import threading
 import six
 from six.moves import queue as Queue  # pylint: disable=redefined-builtin
@@ -278,7 +279,15 @@ class _InfeedThreadController(_InfeedOutfeedThreadBaseController):
       iterations = signal
       for i in range(iterations):
         logging.debug('Infeed enqueue for iteration (%d, %d)', count, i)
-        session.run(enqueue_ops)
+        try:
+          session.run(enqueue_ops)
+        except:  # pylint: disable=bare-except
+          # Hard exit from the interpreter.
+          #
+          # TODO(power) -- possibly communicate this to the main thread somehow.
+          logging.fatal('Infeed controller failed to enqueue ops.  Aborting.',
+                        exc_info=True)
+          os._exit(1)  # pylint: disable=protected-access
       count += 1
 
   def join(self):
-- 
GitLab


From 6a1c71803bc9ec89d81c6d4e1317e9a36782f52f Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Thu, 12 Oct 2017 20:24:53 -0700
Subject: [PATCH 345/909] Replace typeof with decltype.

PiperOrigin-RevId: 172051036
---
 tensorflow/core/kernels/where_op_gpu.cu.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/kernels/where_op_gpu.cu.h b/tensorflow/core/kernels/where_op_gpu.cu.h
index ce8e435c95..57f51889de 100644
--- a/tensorflow/core/kernels/where_op_gpu.cu.h
+++ b/tensorflow/core/kernels/where_op_gpu.cu.h
@@ -278,7 +278,7 @@ struct Where<GPUDevice, NDIM, T, TIndex> {
 
     typedef std::decay<T> DT;
     CubDeviceSelectFlaggedCounter<
-        T, TIndex, typeof(output_iterator) /*OutputIterator*/,
+        T, TIndex, decltype(output_iterator) /*OutputIterator*/,
         std::is_convertible<DT, bool>::value /*IsConvertibleToBool*/>
         counter;
     auto first_success = counter(/*temp_storage*/ nullptr, temp_storage_bytes,
-- 
GitLab


From 99dc61dbe520b43fcc1919124d2281d3c4fdfa85 Mon Sep 17 00:00:00 2001
From: Kiril Gorovoy <kgorovoy@google.com>
Date: Thu, 12 Oct 2017 20:32:41 -0700
Subject: [PATCH 346/909] Make SavedModel loading forward compatible by
 removing default attributes from the graph def that were not present in the
 consumer (server).

PiperOrigin-RevId: 172051437
---
 tensorflow/cc/saved_model/BUILD               |    2 +
 tensorflow/cc/saved_model/loader.cc           |   16 +
 tensorflow/cc/saved_model/loader_test.cc      |   24 +-
 .../00000123/assets/foo.txt                   |    1 +
 .../00000123/saved_model.pbtxt                | 2728 +++++++++++++++++
 .../variables/variables.data-00000-of-00001   |  Bin 0 -> 12 bytes
 .../00000123/variables/variables.index        |  Bin 0 -> 151 bytes
 7 files changed, 2769 insertions(+), 2 deletions(-)
 create mode 100644 tensorflow/cc/saved_model/testdata/half_plus_two_forward_compatibility/00000123/assets/foo.txt
 create mode 100755 tensorflow/cc/saved_model/testdata/half_plus_two_forward_compatibility/00000123/saved_model.pbtxt
 create mode 100755 tensorflow/cc/saved_model/testdata/half_plus_two_forward_compatibility/00000123/variables/variables.data-00000-of-00001
 create mode 100755 tensorflow/cc/saved_model/testdata/half_plus_two_forward_compatibility/00000123/variables/variables.index

diff --git a/tensorflow/cc/saved_model/BUILD b/tensorflow/cc/saved_model/BUILD
index 1cc7cf3f20..e43ff91c60 100644
--- a/tensorflow/cc/saved_model/BUILD
+++ b/tensorflow/cc/saved_model/BUILD
@@ -54,6 +54,7 @@ cc_library(
     hdrs = ["loader.h"],
     deps = [
         ":constants",
+        "//tensorflow/core:framework",
     ] + if_not_mobile([
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:lib",
@@ -88,6 +89,7 @@ tf_cc_test(
 filegroup(
     name = "saved_model_half_plus_two",
     srcs = glob([
+        "testdata/half_plus_two_forward_compatibility/**",
         "testdata/half_plus_two_pbtxt/**",
         "testdata/half_plus_two_main_op/**",
         "testdata/half_plus_two/**",
diff --git a/tensorflow/cc/saved_model/loader.cc b/tensorflow/cc/saved_model/loader.cc
index f98abc8a81..462308a48f 100644
--- a/tensorflow/cc/saved_model/loader.cc
+++ b/tensorflow/cc/saved_model/loader.cc
@@ -18,6 +18,7 @@ limitations under the License.
 #include <unordered_set>
 
 #include "tensorflow/cc/saved_model/constants.h"
+#include "tensorflow/core/framework/graph_def_util.h"
 #include "tensorflow/core/lib/io/path.h"
 #include "tensorflow/core/lib/monitoring/counter.h"
 #include "tensorflow/core/lib/strings/strcat.h"
@@ -224,6 +225,18 @@ Status GetAssetFileDefs(const MetaGraphDef& meta_graph_def,
   return Status::OK();
 }
 
+// For forward compatibility, remove new default attributes from the graph def
+// that were not present in the consumer (e.g. If graph was exported using
+// code that's newer than the server and a new default attr was added).
+Status RemoveNewDefaultAttrsFromMetaGraphDef(MetaGraphDef* meta_graph_def) {
+  OpListOpRegistry producer_op_registry(
+      &meta_graph_def->meta_info_def().stripped_op_list());
+  OpRegistry* consumer_op_registry = OpRegistry::Global();
+  return RemoveNewDefaultAttrsFromGraphDef(meta_graph_def->mutable_graph_def(),
+                                           *consumer_op_registry,
+                                           producer_op_registry, nullptr);
+}
+
 Status LoadSavedModelInternal(const SessionOptions& session_options,
                               const RunOptions& run_options,
                               const string& export_dir,
@@ -241,6 +254,9 @@ Status LoadSavedModelInternal(const SessionOptions& session_options,
   TF_RETURN_IF_ERROR(
       FindMetaGraphDefToLoad(saved_model_proto, tags, &bundle->meta_graph_def));
 
+  TF_RETURN_IF_ERROR(
+      RemoveNewDefaultAttrsFromMetaGraphDef(&bundle->meta_graph_def));
+
   TF_RETURN_IF_ERROR(LoadMetaGraphIntoSession(
       bundle->meta_graph_def, session_options, &bundle->session));
 
diff --git a/tensorflow/cc/saved_model/loader_test.cc b/tensorflow/cc/saved_model/loader_test.cc
index 0ad6b33bba..6dd14837b5 100644
--- a/tensorflow/cc/saved_model/loader_test.cc
+++ b/tensorflow/cc/saved_model/loader_test.cc
@@ -29,10 +29,12 @@ limitations under the License.
 namespace tensorflow {
 namespace {
 
-constexpr char kTestDataPbTxt[] =
-    "cc/saved_model/testdata/half_plus_two_pbtxt/00000123";
+constexpr char kTestDataForwardCompatibility[] =
+    "cc/saved_model/testdata/half_plus_two_forward_compatibility/00000123";
 constexpr char kTestDataMainOp[] =
     "cc/saved_model/testdata/half_plus_two_main_op/00000123";
+constexpr char kTestDataPbTxt[] =
+    "cc/saved_model/testdata/half_plus_two_pbtxt/00000123";
 constexpr char kTestDataSharded[] =
     "cc/saved_model/testdata/half_plus_two/00000123";
 
@@ -167,6 +169,24 @@ TEST_F(LoaderTest, PbtxtFormat) {
   CheckSavedModelBundle(export_dir, bundle);
 }
 
+// Forward compatibility graph has a new attr with a default value equal to the
+// value used by the server. If we handle new default attrs correctly, this test
+// will pass. This simulates adding new atts to the training code while server
+// code lags behind.
+TEST_F(LoaderTest, ForwardCompatibility) {
+  SavedModelBundle bundle;
+  SessionOptions session_options;
+  RunOptions run_options;
+
+  // TODO(b/67753689): Add support for regenerating this model in the export
+  // code.
+  const string export_dir =
+      io::JoinPath(testing::TensorFlowSrcRoot(), kTestDataForwardCompatibility);
+  TF_ASSERT_OK(LoadSavedModel(session_options, run_options, export_dir,
+                              {kSavedModelTagServe}, &bundle));
+  CheckSavedModelBundle(export_dir, bundle);
+}
+
 TEST_F(LoaderTest, MainOpFormat) {
   SavedModelBundle bundle;
   SessionOptions session_options;
diff --git a/tensorflow/cc/saved_model/testdata/half_plus_two_forward_compatibility/00000123/assets/foo.txt b/tensorflow/cc/saved_model/testdata/half_plus_two_forward_compatibility/00000123/assets/foo.txt
new file mode 100644
index 0000000000..f9ff036688
--- /dev/null
+++ b/tensorflow/cc/saved_model/testdata/half_plus_two_forward_compatibility/00000123/assets/foo.txt
@@ -0,0 +1 @@
+asset-file-contents
\ No newline at end of file
diff --git a/tensorflow/cc/saved_model/testdata/half_plus_two_forward_compatibility/00000123/saved_model.pbtxt b/tensorflow/cc/saved_model/testdata/half_plus_two_forward_compatibility/00000123/saved_model.pbtxt
new file mode 100755
index 0000000000..e799b3579c
--- /dev/null
+++ b/tensorflow/cc/saved_model/testdata/half_plus_two_forward_compatibility/00000123/saved_model.pbtxt
@@ -0,0 +1,2728 @@
+saved_model_schema_version: 1
+meta_graphs {
+  meta_info_def {
+    stripped_op_list {
+      op {
+        name: "Add"
+        input_arg {
+          name: "x"
+          type_attr: "T"
+        }
+        input_arg {
+          name: "y"
+          type_attr: "T"
+        }
+        output_arg {
+          name: "z"
+          type_attr: "T"
+        }
+        attr {
+          name: "T"
+          type: "type"
+          allowed_values {
+            list {
+              type: DT_HALF
+              type: DT_FLOAT
+              type: DT_DOUBLE
+              type: DT_UINT8
+              type: DT_INT8
+              type: DT_INT16
+              type: DT_INT32
+              type: DT_INT64
+              type: DT_COMPLEX64
+              type: DT_COMPLEX128
+              type: DT_STRING
+            }
+          }
+        }
+      }
+      op {
+        name: "Assign"
+        input_arg {
+          name: "ref"
+          type_attr: "T"
+          is_ref: true
+        }
+        input_arg {
+          name: "value"
+          type_attr: "T"
+        }
+        output_arg {
+          name: "output_ref"
+          type_attr: "T"
+          is_ref: true
+        }
+        attr {
+          name: "T"
+          type: "type"
+        }
+        attr {
+          name: "validate_shape"
+          type: "bool"
+          default_value {
+            b: true
+          }
+        }
+        attr {
+          name: "use_locking"
+          type: "bool"
+          default_value {
+            b: true
+          }
+        }
+        allows_uninitialized_input: true
+      }
+      op {
+        name: "Const"
+        output_arg {
+          name: "output"
+          type_attr: "dtype"
+        }
+        attr {
+          name: "value"
+          type: "tensor"
+        }
+        attr {
+          name: "dtype"
+          type: "type"
+        }
+      }
+      op {
+        name: "Identity"
+        input_arg {
+          name: "input"
+          type_attr: "T"
+        }
+        output_arg {
+          name: "output"
+          type_attr: "T"
+        }
+        attr {
+          name: "T"
+          type: "type"
+        }
+      }
+      op {
+        name: "MergeV2Checkpoints"
+        input_arg {
+          name: "checkpoint_prefixes"
+          type: DT_STRING
+        }
+        input_arg {
+          name: "destination_prefix"
+          type: DT_STRING
+        }
+        attr {
+          name: "delete_old_dirs"
+          type: "bool"
+          default_value {
+            b: true
+          }
+        }
+      }
+      op {
+        name: "Mul"
+        input_arg {
+          name: "x"
+          type_attr: "T"
+        }
+        input_arg {
+          name: "y"
+          type_attr: "T"
+        }
+        output_arg {
+          name: "z"
+          type_attr: "T"
+        }
+        attr {
+          name: "T"
+          type: "type"
+          allowed_values {
+            list {
+              type: DT_HALF
+              type: DT_FLOAT
+              type: DT_DOUBLE
+              type: DT_UINT8
+              type: DT_INT8
+              type: DT_UINT16
+              type: DT_INT16
+              type: DT_INT32
+              type: DT_INT64
+              type: DT_COMPLEX64
+              type: DT_COMPLEX128
+            }
+          }
+        }
+        is_commutative: true
+      }
+      op {
+        name: "NoOp"
+      }
+      op {
+        name: "Pack"
+        input_arg {
+          name: "values"
+          type_attr: "T"
+          number_attr: "N"
+        }
+        output_arg {
+          name: "output"
+          type_attr: "T"
+        }
+        attr {
+          name: "N"
+          type: "int"
+          has_minimum: true
+          minimum: 1
+        }
+        attr {
+          name: "T"
+          type: "type"
+        }
+        attr {
+          name: "axis"
+          type: "int"
+          default_value {
+            i: 0
+          }
+        }
+      }
+      op {
+        name: "ParseExample"
+        input_arg {
+          name: "serialized"
+          type_attr: "TInputs"
+        }
+        input_arg {
+          name: "names"
+          type: DT_STRING
+        }
+        input_arg {
+          name: "sparse_keys"
+          type: DT_STRING
+          number_attr: "Nsparse"
+        }
+        input_arg {
+          name: "dense_keys"
+          type: DT_STRING
+          number_attr: "Ndense"
+        }
+        input_arg {
+          name: "dense_defaults"
+          type_list_attr: "Tdense"
+        }
+        output_arg {
+          name: "sparse_indices"
+          type: DT_INT64
+          number_attr: "Nsparse"
+        }
+        output_arg {
+          name: "sparse_values"
+          type_list_attr: "sparse_types"
+        }
+        output_arg {
+          name: "sparse_shapes"
+          type: DT_INT64
+          number_attr: "Nsparse"
+        }
+        output_arg {
+          name: "dense_values"
+          type_list_attr: "Tdense"
+        }
+        attr {
+          name: "Nsparse"
+          type: "int"
+          has_minimum: true
+        }
+        attr {
+          name: "TInputs"
+          type: "type"
+          default_value {
+            type: DT_STRING
+          }
+          allowed_values {
+            list {
+              type: DT_STRING
+              type: DT_INT64
+            }
+          }
+        }
+        attr {
+          name: "Ndense"
+          type: "int"
+          has_minimum: true
+        }
+        attr {
+          name: "sparse_types"
+          type: "list(type)"
+          has_minimum: true
+          allowed_values {
+            list {
+              type: DT_FLOAT
+              type: DT_INT64
+              type: DT_STRING
+            }
+          }
+        }
+        attr {
+          name: "Tdense"
+          type: "list(type)"
+          has_minimum: true
+          allowed_values {
+            list {
+              type: DT_FLOAT
+              type: DT_INT64
+              type: DT_STRING
+            }
+          }
+        }
+        attr {
+          name: "dense_shapes"
+          type: "list(shape)"
+          has_minimum: true
+        }
+      }
+      op {
+        name: "Placeholder"
+        output_arg {
+          name: "output"
+          type_attr: "dtype"
+        }
+        attr {
+          name: "dtype"
+          type: "type"
+        }
+        attr {
+          name: "shape"
+          type: "shape"
+          default_value {
+            shape {
+              unknown_rank: true
+            }
+          }
+        }
+      }
+      op {
+        name: "Reshape"
+        input_arg {
+          name: "tensor"
+          type_attr: "T"
+        }
+        input_arg {
+          name: "shape"
+          type_attr: "Tshape"
+        }
+        output_arg {
+          name: "output"
+          type_attr: "T"
+        }
+        attr {
+          name: "T"
+          type: "type"
+        }
+        attr {
+          name: "Tshape"
+          type: "type"
+          default_value {
+            type: DT_INT32
+          }
+          allowed_values {
+            list {
+              type: DT_INT32
+              type: DT_INT64
+            }
+          }
+        }
+      }
+      op {
+        name: "RestoreV2"
+        input_arg {
+          name: "prefix"
+          type: DT_STRING
+        }
+        input_arg {
+          name: "tensor_names"
+          type: DT_STRING
+        }
+        input_arg {
+          name: "shape_and_slices"
+          type: DT_STRING
+        }
+        output_arg {
+          name: "tensors"
+          type_list_attr: "dtypes"
+        }
+        attr {
+          name: "dtypes"
+          type: "list(type)"
+          has_minimum: true
+          minimum: 1
+        }
+      }
+      op {
+        name: "SaveV2"
+        input_arg {
+          name: "prefix"
+          type: DT_STRING
+        }
+        input_arg {
+          name: "tensor_names"
+          type: DT_STRING
+        }
+        input_arg {
+          name: "shape_and_slices"
+          type: DT_STRING
+        }
+        input_arg {
+          name: "tensors"
+          type_list_attr: "dtypes"
+        }
+        attr {
+          name: "dtypes"
+          type: "list(type)"
+          has_minimum: true
+          minimum: 1
+        }
+      }
+      op {
+        name: "ShardedFilename"
+        input_arg {
+          name: "basename"
+          type: DT_STRING
+        }
+        input_arg {
+          name: "shard"
+          type: DT_INT32
+        }
+        input_arg {
+          name: "num_shards"
+          type: DT_INT32
+        }
+        output_arg {
+          name: "filename"
+          type: DT_STRING
+        }
+      }
+      op {
+        name: "StringJoin"
+        input_arg {
+          name: "inputs"
+          type: DT_STRING
+          number_attr: "N"
+        }
+        output_arg {
+          name: "output"
+          type: DT_STRING
+        }
+        attr {
+          name: "N"
+          type: "int"
+          has_minimum: true
+          minimum: 1
+        }
+        attr {
+          name: "separator"
+          type: "string"
+          default_value {
+            s: ""
+          }
+        }
+      }
+      op {
+        name: "VariableV2"
+        output_arg {
+          name: "ref"
+          type_attr: "dtype"
+          is_ref: true
+        }
+        attr {
+          name: "shape"
+          type: "shape"
+        }
+        attr {
+          name: "dtype"
+          type: "type"
+        }
+        attr {
+          name: "container"
+          type: "string"
+          default_value {
+            s: ""
+          }
+        }
+        attr {
+          name: "shared_name"
+          type: "string"
+          default_value {
+            s: ""
+          }
+        }
+        is_stateful: true
+      }
+    }
+    tags: "serve"
+    tensorflow_version: "1.1.0-rc2"
+    tensorflow_git_version: "unknown"
+  }
+  graph_def {
+    node {
+      name: "a/initial_value"
+      op: "Const"
+      attr {
+        key: "_output_shapes"
+        value {
+          list {
+            shape {
+            }
+          }
+        }
+      }
+      attr {
+        key: "dtype"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_FLOAT
+            tensor_shape {
+            }
+            float_val: 0.5
+          }
+        }
+      }
+    }
+    node {
+      name: "a"
+      op: "VariableV2"
+      attr {
+        key: "_output_shapes"
+        value {
+          list {
+            shape {
+            }
+          }
+        }
+      }
+      attr {
+        key: "container"
+        value {
+          s: ""
+        }
+      }
+      attr {
+        key: "dtype"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "shape"
+        value {
+          shape {
+          }
+        }
+      }
+      attr {
+        key: "shared_name"
+        value {
+          s: ""
+        }
+      }
+    }
+    node {
+      name: "a/Assign"
+      op: "Assign"
+      input: "a"
+      input: "a/initial_value"
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "_class"
+        value {
+          list {
+            s: "loc:@a"
+          }
+        }
+      }
+      attr {
+        key: "_output_shapes"
+        value {
+          list {
+            shape {
+            }
+          }
+        }
+      }
+      attr {
+        key: "use_locking"
+        value {
+          b: true
+        }
+      }
+      attr {
+        key: "validate_shape"
+        value {
+          b: true
+        }
+      }
+    }
+    node {
+      name: "a/read"
+      op: "Identity"
+      input: "a"
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "_class"
+        value {
+          list {
+            s: "loc:@a"
+          }
+        }
+      }
+      attr {
+        key: "_output_shapes"
+        value {
+          list {
+            shape {
+            }
+          }
+        }
+      }
+    }
+    node {
+      name: "b/initial_value"
+      op: "Const"
+      attr {
+        key: "_output_shapes"
+        value {
+          list {
+            shape {
+            }
+          }
+        }
+      }
+      attr {
+        key: "dtype"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_FLOAT
+            tensor_shape {
+            }
+            float_val: 2.0
+          }
+        }
+      }
+    }
+    node {
+      name: "b"
+      op: "VariableV2"
+      attr {
+        key: "_output_shapes"
+        value {
+          list {
+            shape {
+            }
+          }
+        }
+      }
+      attr {
+        key: "container"
+        value {
+          s: ""
+        }
+      }
+      attr {
+        key: "dtype"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "shape"
+        value {
+          shape {
+          }
+        }
+      }
+      attr {
+        key: "shared_name"
+        value {
+          s: ""
+        }
+      }
+    }
+    node {
+      name: "b/Assign"
+      op: "Assign"
+      input: "b"
+      input: "b/initial_value"
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "_class"
+        value {
+          list {
+            s: "loc:@b"
+          }
+        }
+      }
+      attr {
+        key: "_output_shapes"
+        value {
+          list {
+            shape {
+            }
+          }
+        }
+      }
+      attr {
+        key: "use_locking"
+        value {
+          b: true
+        }
+      }
+      attr {
+        key: "validate_shape"
+        value {
+          b: true
+        }
+      }
+    }
+    node {
+      name: "b/read"
+      op: "Identity"
+      input: "b"
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "_class"
+        value {
+          list {
+            s: "loc:@b"
+          }
+        }
+      }
+      attr {
+        key: "_output_shapes"
+        value {
+          list {
+            shape {
+            }
+          }
+        }
+      }
+    }
+    node {
+      name: "c/initial_value"
+      op: "Const"
+      attr {
+        key: "_output_shapes"
+        value {
+          list {
+            shape {
+            }
+          }
+        }
+      }
+      attr {
+        key: "dtype"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_FLOAT
+            tensor_shape {
+            }
+            float_val: 3.0
+          }
+        }
+      }
+    }
+    node {
+      name: "c"
+      op: "VariableV2"
+      attr {
+        key: "_output_shapes"
+        value {
+          list {
+            shape {
+            }
+          }
+        }
+      }
+      attr {
+        key: "container"
+        value {
+          s: ""
+        }
+      }
+      attr {
+        key: "dtype"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "shape"
+        value {
+          shape {
+          }
+        }
+      }
+      attr {
+        key: "shared_name"
+        value {
+          s: ""
+        }
+      }
+    }
+    node {
+      name: "c/Assign"
+      op: "Assign"
+      input: "c"
+      input: "c/initial_value"
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "_class"
+        value {
+          list {
+            s: "loc:@c"
+          }
+        }
+      }
+      attr {
+        key: "_output_shapes"
+        value {
+          list {
+            shape {
+            }
+          }
+        }
+      }
+      attr {
+        key: "use_locking"
+        value {
+          b: true
+        }
+      }
+      attr {
+        key: "validate_shape"
+        value {
+          b: true
+        }
+      }
+    }
+    node {
+      name: "c/read"
+      op: "Identity"
+      input: "c"
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "_class"
+        value {
+          list {
+            s: "loc:@c"
+          }
+        }
+      }
+      attr {
+        key: "_output_shapes"
+        value {
+          list {
+            shape {
+            }
+          }
+        }
+      }
+    }
+    node {
+      name: "tf_example"
+      op: "Placeholder"
+      attr {
+        key: "_output_shapes"
+        value {
+          list {
+            shape {
+              unknown_rank: true
+            }
+          }
+        }
+      }
+      attr {
+        key: "dtype"
+        value {
+          type: DT_STRING
+        }
+      }
+      attr {
+        key: "shape"
+        value {
+          shape {
+            unknown_rank: true
+          }
+        }
+      }
+    }
+    node {
+      name: "ParseExample/Const"
+      op: "Const"
+      attr {
+        key: "_output_shapes"
+        value {
+          list {
+            shape {
+              dim {
+              }
+            }
+          }
+        }
+      }
+      attr {
+        key: "dtype"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_FLOAT
+            tensor_shape {
+              dim {
+              }
+            }
+          }
+        }
+      }
+    }
+    node {
+      name: "ParseExample/key_x2"
+      op: "Const"
+      attr {
+        key: "_output_shapes"
+        value {
+          list {
+            shape {
+              dim {
+                size: 1
+              }
+            }
+          }
+        }
+      }
+      attr {
+        key: "dtype"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_FLOAT
+            tensor_shape {
+              dim {
+                size: 1
+              }
+            }
+            float_val: 0.0
+          }
+        }
+      }
+    }
+    node {
+      name: "ParseExample/Reshape/shape"
+      op: "Const"
+      attr {
+        key: "_output_shapes"
+        value {
+          list {
+            shape {
+              dim {
+                size: 1
+              }
+            }
+          }
+        }
+      }
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+              dim {
+                size: 1
+              }
+            }
+            int_val: 1
+          }
+        }
+      }
+    }
+    node {
+      name: "ParseExample/Reshape"
+      op: "Reshape"
+      input: "ParseExample/key_x2"
+      input: "ParseExample/Reshape/shape"
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "Tshape"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "_output_shapes"
+        value {
+          list {
+            shape {
+              dim {
+                size: 1
+              }
+            }
+          }
+        }
+      }
+    }
+    node {
+      name: "ParseExample/ParseExample/names"
+      op: "Const"
+      attr {
+        key: "_output_shapes"
+        value {
+          list {
+            shape {
+              dim {
+              }
+            }
+          }
+        }
+      }
+      attr {
+        key: "dtype"
+        value {
+          type: DT_STRING
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_STRING
+            tensor_shape {
+              dim {
+              }
+            }
+          }
+        }
+      }
+    }
+    node {
+      name: "ParseExample/ParseExample/dense_keys_0"
+      op: "Const"
+      attr {
+        key: "_output_shapes"
+        value {
+          list {
+            shape {
+            }
+          }
+        }
+      }
+      attr {
+        key: "dtype"
+        value {
+          type: DT_STRING
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_STRING
+            tensor_shape {
+            }
+            string_val: "x"
+          }
+        }
+      }
+    }
+    node {
+      name: "ParseExample/ParseExample/dense_keys_1"
+      op: "Const"
+      attr {
+        key: "_output_shapes"
+        value {
+          list {
+            shape {
+            }
+          }
+        }
+      }
+      attr {
+        key: "dtype"
+        value {
+          type: DT_STRING
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_STRING
+            tensor_shape {
+            }
+            string_val: "x2"
+          }
+        }
+      }
+    }
+    node {
+      name: "ParseExample/ParseExample"
+      op: "ParseExample"
+      input: "tf_example"
+      input: "ParseExample/ParseExample/names"
+      input: "ParseExample/ParseExample/dense_keys_0"
+      input: "ParseExample/ParseExample/dense_keys_1"
+      input: "ParseExample/Const"
+      input: "ParseExample/Reshape"
+      attr {
+        key: "Ndense"
+        value {
+          i: 2
+        }
+      }
+      attr {
+        key: "TInputs"
+        value {
+          type: DT_STRING
+        }
+      }
+      attr {
+        key: "Nsparse"
+        value {
+          i: 0
+        }
+      }
+      attr {
+        key: "Tdense"
+        value {
+          list {
+            type: DT_FLOAT
+            type: DT_FLOAT
+          }
+        }
+      }
+      attr {
+        key: "_output_shapes"
+        value {
+          list {
+            shape {
+              dim {
+                size: -1
+              }
+              dim {
+                size: 1
+              }
+            }
+            shape {
+              dim {
+                size: -1
+              }
+              dim {
+                size: 1
+              }
+            }
+          }
+        }
+      }
+      attr {
+        key: "dense_shapes"
+        value {
+          list {
+            shape {
+              dim {
+                size: 1
+              }
+            }
+            shape {
+              dim {
+                size: 1
+              }
+            }
+          }
+        }
+      }
+      attr {
+        key: "sparse_types"
+        value {
+          list {
+          }
+        }
+      }
+    }
+    node {
+      name: "x"
+      op: "Identity"
+      input: "ParseExample/ParseExample"
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "_output_shapes"
+        value {
+          list {
+            shape {
+              dim {
+                size: -1
+              }
+              dim {
+                size: 1
+              }
+            }
+          }
+        }
+      }
+    }
+    node {
+      name: "Mul"
+      op: "Mul"
+      input: "a/read"
+      input: "x"
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "_output_shapes"
+        value {
+          list {
+            shape {
+              dim {
+                size: -1
+              }
+              dim {
+                size: 1
+              }
+            }
+          }
+        }
+      }
+    }
+    node {
+      name: "y"
+      op: "Add"
+      input: "Mul"
+      input: "b/read"
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "_output_shapes"
+        value {
+          list {
+            shape {
+              dim {
+                size: -1
+              }
+              dim {
+                size: 1
+              }
+            }
+          }
+        }
+      }
+    }
+    node {
+      name: "Mul_1"
+      op: "Mul"
+      input: "a/read"
+      input: "x"
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "_output_shapes"
+        value {
+          list {
+            shape {
+              dim {
+                size: -1
+              }
+              dim {
+                size: 1
+              }
+            }
+          }
+        }
+      }
+    }
+    node {
+      name: "y2"
+      op: "Add"
+      input: "Mul_1"
+      input: "c/read"
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "_output_shapes"
+        value {
+          list {
+            shape {
+              dim {
+                size: -1
+              }
+              dim {
+                size: 1
+              }
+            }
+          }
+        }
+      }
+    }
+    node {
+      name: "x2"
+      op: "Identity"
+      input: "ParseExample/ParseExample:1"
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "_output_shapes"
+        value {
+          list {
+            shape {
+              dim {
+                size: -1
+              }
+              dim {
+                size: 1
+              }
+            }
+          }
+        }
+      }
+    }
+    node {
+      name: "Mul_2"
+      op: "Mul"
+      input: "a/read"
+      input: "x2"
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "_output_shapes"
+        value {
+          list {
+            shape {
+              dim {
+                size: -1
+              }
+              dim {
+                size: 1
+              }
+            }
+          }
+        }
+      }
+    }
+    node {
+      name: "y3"
+      op: "Add"
+      input: "Mul_2"
+      input: "c/read"
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "_output_shapes"
+        value {
+          list {
+            shape {
+              dim {
+                size: -1
+              }
+              dim {
+                size: 1
+              }
+            }
+          }
+        }
+      }
+    }
+    node {
+      name: "Const"
+      op: "Const"
+      attr {
+        key: "_output_shapes"
+        value {
+          list {
+            shape {
+            }
+          }
+        }
+      }
+      attr {
+        key: "dtype"
+        value {
+          type: DT_STRING
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_STRING
+            tensor_shape {
+            }
+            string_val: "/tmp/original/export/assets/foo.txt"
+          }
+        }
+      }
+    }
+    node {
+      name: "filename_tensor/initial_value"
+      op: "Const"
+      attr {
+        key: "_output_shapes"
+        value {
+          list {
+            shape {
+            }
+          }
+        }
+      }
+      attr {
+        key: "dtype"
+        value {
+          type: DT_STRING
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_STRING
+            tensor_shape {
+            }
+            string_val: "foo.txt"
+          }
+        }
+      }
+    }
+    node {
+      name: "filename_tensor"
+      op: "VariableV2"
+      attr {
+        key: "_output_shapes"
+        value {
+          list {
+            shape {
+            }
+          }
+        }
+      }
+      attr {
+        key: "container"
+        value {
+          s: ""
+        }
+      }
+      attr {
+        key: "dtype"
+        value {
+          type: DT_STRING
+        }
+      }
+      attr {
+        key: "shape"
+        value {
+          shape {
+          }
+        }
+      }
+      attr {
+        key: "shared_name"
+        value {
+          s: ""
+        }
+      }
+    }
+    node {
+      name: "filename_tensor/Assign"
+      op: "Assign"
+      input: "filename_tensor"
+      input: "filename_tensor/initial_value"
+      attr {
+        key: "T"
+        value {
+          type: DT_STRING
+        }
+      }
+      attr {
+        key: "_class"
+        value {
+          list {
+            s: "loc:@filename_tensor"
+          }
+        }
+      }
+      attr {
+        key: "_output_shapes"
+        value {
+          list {
+            shape {
+            }
+          }
+        }
+      }
+      attr {
+        key: "use_locking"
+        value {
+          b: true
+        }
+      }
+      attr {
+        key: "validate_shape"
+        value {
+          b: true
+        }
+      }
+    }
+    node {
+      name: "filename_tensor/read"
+      op: "Identity"
+      input: "filename_tensor"
+      attr {
+        key: "T"
+        value {
+          type: DT_STRING
+        }
+      }
+      attr {
+        key: "_class"
+        value {
+          list {
+            s: "loc:@filename_tensor"
+          }
+        }
+      }
+      attr {
+        key: "_output_shapes"
+        value {
+          list {
+            shape {
+            }
+          }
+        }
+      }
+    }
+    node {
+      name: "Assign/value"
+      op: "Const"
+      attr {
+        key: "_output_shapes"
+        value {
+          list {
+            shape {
+            }
+          }
+        }
+      }
+      attr {
+        key: "dtype"
+        value {
+          type: DT_STRING
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_STRING
+            tensor_shape {
+            }
+            string_val: "foo.txt"
+          }
+        }
+      }
+    }
+    node {
+      name: "Assign"
+      op: "Assign"
+      input: "filename_tensor"
+      input: "Assign/value"
+      attr {
+        key: "T"
+        value {
+          type: DT_STRING
+        }
+      }
+      attr {
+        key: "_class"
+        value {
+          list {
+            s: "loc:@filename_tensor"
+          }
+        }
+      }
+      attr {
+        key: "_output_shapes"
+        value {
+          list {
+            shape {
+            }
+          }
+        }
+      }
+      attr {
+        key: "use_locking"
+        value {
+          b: false
+        }
+      }
+      attr {
+        key: "validate_shape"
+        value {
+          b: true
+        }
+      }
+    }
+    node {
+      name: "init"
+      op: "NoOp"
+      input: "^a/Assign"
+      input: "^b/Assign"
+      input: "^c/Assign"
+    }
+    node {
+      name: "group_deps"
+      op: "NoOp"
+      input: "^Assign"
+    }
+    node {
+      name: "save/Const"
+      op: "Const"
+      attr {
+        key: "_output_shapes"
+        value {
+          list {
+            shape {
+            }
+          }
+        }
+      }
+      attr {
+        key: "dtype"
+        value {
+          type: DT_STRING
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_STRING
+            tensor_shape {
+            }
+            string_val: "model"
+          }
+        }
+      }
+    }
+    node {
+      name: "save/StringJoin/inputs_1"
+      op: "Const"
+      attr {
+        key: "_output_shapes"
+        value {
+          list {
+            shape {
+            }
+          }
+        }
+      }
+      attr {
+        key: "dtype"
+        value {
+          type: DT_STRING
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_STRING
+            tensor_shape {
+            }
+            string_val: "_temp_80e928f1e0c844239d136d1ea966099d/part"
+          }
+        }
+      }
+    }
+    node {
+      name: "save/StringJoin"
+      op: "StringJoin"
+      input: "save/Const"
+      input: "save/StringJoin/inputs_1"
+      attr {
+        key: "N"
+        value {
+          i: 2
+        }
+      }
+      attr {
+        key: "_output_shapes"
+        value {
+          list {
+            shape {
+            }
+          }
+        }
+      }
+      attr {
+        key: "separator"
+        value {
+          s: ""
+        }
+      }
+    }
+    node {
+      name: "save/num_shards"
+      op: "Const"
+      attr {
+        key: "_output_shapes"
+        value {
+          list {
+            shape {
+            }
+          }
+        }
+      }
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+            }
+            int_val: 1
+          }
+        }
+      }
+    }
+    node {
+      name: "save/ShardedFilename/shard"
+      op: "Const"
+      attr {
+        key: "_output_shapes"
+        value {
+          list {
+            shape {
+            }
+          }
+        }
+      }
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+            }
+            int_val: 0
+          }
+        }
+      }
+    }
+    node {
+      name: "save/ShardedFilename"
+      op: "ShardedFilename"
+      input: "save/StringJoin"
+      input: "save/ShardedFilename/shard"
+      input: "save/num_shards"
+      attr {
+        key: "_output_shapes"
+        value {
+          list {
+            shape {
+            }
+          }
+        }
+      }
+    }
+    node {
+      name: "save/SaveV2/tensor_names"
+      op: "Const"
+      attr {
+        key: "_output_shapes"
+        value {
+          list {
+            shape {
+              dim {
+                size: 3
+              }
+            }
+          }
+        }
+      }
+      attr {
+        key: "dtype"
+        value {
+          type: DT_STRING
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_STRING
+            tensor_shape {
+              dim {
+                size: 3
+              }
+            }
+            string_val: "a"
+            string_val: "b"
+            string_val: "c"
+          }
+        }
+      }
+    }
+    node {
+      name: "save/SaveV2/shape_and_slices"
+      op: "Const"
+      attr {
+        key: "_output_shapes"
+        value {
+          list {
+            shape {
+              dim {
+                size: 3
+              }
+            }
+          }
+        }
+      }
+      attr {
+        key: "dtype"
+        value {
+          type: DT_STRING
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_STRING
+            tensor_shape {
+              dim {
+                size: 3
+              }
+            }
+            string_val: ""
+            string_val: ""
+            string_val: ""
+          }
+        }
+      }
+    }
+    node {
+      name: "save/SaveV2"
+      op: "SaveV2"
+      input: "save/ShardedFilename"
+      input: "save/SaveV2/tensor_names"
+      input: "save/SaveV2/shape_and_slices"
+      input: "a"
+      input: "b"
+      input: "c"
+      attr {
+        key: "dtypes"
+        value {
+          list {
+            type: DT_FLOAT
+            type: DT_FLOAT
+            type: DT_FLOAT
+          }
+        }
+      }
+    }
+    node {
+      name: "save/control_dependency"
+      op: "Identity"
+      input: "save/ShardedFilename"
+      input: "^save/SaveV2"
+      attr {
+        key: "T"
+        value {
+          type: DT_STRING
+        }
+      }
+      attr {
+        key: "_class"
+        value {
+          list {
+            s: "loc:@save/ShardedFilename"
+          }
+        }
+      }
+      attr {
+        key: "_output_shapes"
+        value {
+          list {
+            shape {
+            }
+          }
+        }
+      }
+    }
+    node {
+      name: "save/MergeV2Checkpoints/checkpoint_prefixes"
+      op: "Pack"
+      input: "save/ShardedFilename"
+      input: "^save/control_dependency"
+      attr {
+        key: "N"
+        value {
+          i: 1
+        }
+      }
+      attr {
+        key: "T"
+        value {
+          type: DT_STRING
+        }
+      }
+      attr {
+        key: "_output_shapes"
+        value {
+          list {
+            shape {
+              dim {
+                size: 1
+              }
+            }
+          }
+        }
+      }
+      attr {
+        key: "axis"
+        value {
+          i: 0
+        }
+      }
+    }
+    node {
+      name: "save/MergeV2Checkpoints"
+      op: "MergeV2Checkpoints"
+      input: "save/MergeV2Checkpoints/checkpoint_prefixes"
+      input: "save/Const"
+      attr {
+        key: "delete_old_dirs"
+        value {
+          b: true
+        }
+      }
+    }
+    node {
+      name: "save/Identity"
+      op: "Identity"
+      input: "save/Const"
+      input: "^save/control_dependency"
+      input: "^save/MergeV2Checkpoints"
+      attr {
+        key: "T"
+        value {
+          type: DT_STRING
+        }
+      }
+      attr {
+        key: "_output_shapes"
+        value {
+          list {
+            shape {
+            }
+          }
+        }
+      }
+    }
+    node {
+      name: "save/RestoreV2/tensor_names"
+      op: "Const"
+      attr {
+        key: "_output_shapes"
+        value {
+          list {
+            shape {
+              dim {
+                size: 1
+              }
+            }
+          }
+        }
+      }
+      attr {
+        key: "dtype"
+        value {
+          type: DT_STRING
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_STRING
+            tensor_shape {
+              dim {
+                size: 1
+              }
+            }
+            string_val: "a"
+          }
+        }
+      }
+    }
+    node {
+      name: "save/RestoreV2/shape_and_slices"
+      op: "Const"
+      attr {
+        key: "_output_shapes"
+        value {
+          list {
+            shape {
+              dim {
+                size: 1
+              }
+            }
+          }
+        }
+      }
+      attr {
+        key: "dtype"
+        value {
+          type: DT_STRING
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_STRING
+            tensor_shape {
+              dim {
+                size: 1
+              }
+            }
+            string_val: ""
+          }
+        }
+      }
+    }
+    node {
+      name: "save/RestoreV2"
+      op: "RestoreV2"
+      input: "save/Const"
+      input: "save/RestoreV2/tensor_names"
+      input: "save/RestoreV2/shape_and_slices"
+      attr {
+        key: "_output_shapes"
+        value {
+          list {
+            shape {
+              unknown_rank: true
+            }
+          }
+        }
+      }
+      attr {
+        key: "dtypes"
+        value {
+          list {
+            type: DT_FLOAT
+          }
+        }
+      }
+    }
+    node {
+      name: "save/Assign"
+      op: "Assign"
+      input: "a"
+      input: "save/RestoreV2"
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "_class"
+        value {
+          list {
+            s: "loc:@a"
+          }
+        }
+      }
+      attr {
+        key: "_output_shapes"
+        value {
+          list {
+            shape {
+            }
+          }
+        }
+      }
+      attr {
+        key: "use_locking"
+        value {
+          b: true
+        }
+      }
+      attr {
+        key: "validate_shape"
+        value {
+          b: true
+        }
+      }
+    }
+    node {
+      name: "save/RestoreV2_1/tensor_names"
+      op: "Const"
+      attr {
+        key: "_output_shapes"
+        value {
+          list {
+            shape {
+              dim {
+                size: 1
+              }
+            }
+          }
+        }
+      }
+      attr {
+        key: "dtype"
+        value {
+          type: DT_STRING
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_STRING
+            tensor_shape {
+              dim {
+                size: 1
+              }
+            }
+            string_val: "b"
+          }
+        }
+      }
+    }
+    node {
+      name: "save/RestoreV2_1/shape_and_slices"
+      op: "Const"
+      attr {
+        key: "_output_shapes"
+        value {
+          list {
+            shape {
+              dim {
+                size: 1
+              }
+            }
+          }
+        }
+      }
+      attr {
+        key: "dtype"
+        value {
+          type: DT_STRING
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_STRING
+            tensor_shape {
+              dim {
+                size: 1
+              }
+            }
+            string_val: ""
+          }
+        }
+      }
+    }
+    node {
+      name: "save/RestoreV2_1"
+      op: "RestoreV2"
+      input: "save/Const"
+      input: "save/RestoreV2_1/tensor_names"
+      input: "save/RestoreV2_1/shape_and_slices"
+      attr {
+        key: "_output_shapes"
+        value {
+          list {
+            shape {
+              unknown_rank: true
+            }
+          }
+        }
+      }
+      attr {
+        key: "dtypes"
+        value {
+          list {
+            type: DT_FLOAT
+          }
+        }
+      }
+    }
+    node {
+      name: "save/Assign_1"
+      op: "Assign"
+      input: "b"
+      input: "save/RestoreV2_1"
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "_class"
+        value {
+          list {
+            s: "loc:@b"
+          }
+        }
+      }
+      attr {
+        key: "_output_shapes"
+        value {
+          list {
+            shape {
+            }
+          }
+        }
+      }
+      attr {
+        key: "use_locking"
+        value {
+          b: true
+        }
+      }
+      attr {
+        key: "validate_shape"
+        value {
+          b: true
+        }
+      }
+    }
+    node {
+      name: "save/RestoreV2_2/tensor_names"
+      op: "Const"
+      attr {
+        key: "_output_shapes"
+        value {
+          list {
+            shape {
+              dim {
+                size: 1
+              }
+            }
+          }
+        }
+      }
+      attr {
+        key: "dtype"
+        value {
+          type: DT_STRING
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_STRING
+            tensor_shape {
+              dim {
+                size: 1
+              }
+            }
+            string_val: "c"
+          }
+        }
+      }
+    }
+    node {
+      name: "save/RestoreV2_2/shape_and_slices"
+      op: "Const"
+      attr {
+        key: "_output_shapes"
+        value {
+          list {
+            shape {
+              dim {
+                size: 1
+              }
+            }
+          }
+        }
+      }
+      attr {
+        key: "dtype"
+        value {
+          type: DT_STRING
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_STRING
+            tensor_shape {
+              dim {
+                size: 1
+              }
+            }
+            string_val: ""
+          }
+        }
+      }
+    }
+    node {
+      name: "save/RestoreV2_2"
+      op: "RestoreV2"
+      input: "save/Const"
+      input: "save/RestoreV2_2/tensor_names"
+      input: "save/RestoreV2_2/shape_and_slices"
+      attr {
+        key: "_output_shapes"
+        value {
+          list {
+            shape {
+              unknown_rank: true
+            }
+          }
+        }
+      }
+      attr {
+        key: "dtypes"
+        value {
+          list {
+            type: DT_FLOAT
+          }
+        }
+      }
+    }
+    node {
+      name: "save/Assign_2"
+      op: "Assign"
+      input: "c"
+      input: "save/RestoreV2_2"
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "_class"
+        value {
+          list {
+            s: "loc:@c"
+          }
+        }
+      }
+      attr {
+        key: "_output_shapes"
+        value {
+          list {
+            shape {
+            }
+          }
+        }
+      }
+      attr {
+        key: "use_locking"
+        value {
+          b: true
+        }
+      }
+      attr {
+        key: "validate_shape"
+        value {
+          b: true
+        }
+      }
+    }
+    node {
+      name: "save/restore_shard"
+      op: "NoOp"
+      input: "^save/Assign"
+      input: "^save/Assign_1"
+      input: "^save/Assign_2"
+    }
+    node {
+      name: "save/restore_all"
+      op: "NoOp"
+      input: "^save/restore_shard"
+    }
+    versions {
+      producer: 23
+    }
+  }
+  saver_def {
+    filename_tensor_name: "save/Const:0"
+    save_tensor_name: "save/Identity:0"
+    restore_op_name: "save/restore_all"
+    max_to_keep: 5
+    sharded: true
+    keep_checkpoint_every_n_hours: 10000.0
+    version: V2
+  }
+  collection_def {
+    key: "asset_filepaths"
+    value {
+      node_list {
+        value: "Const:0"
+      }
+    }
+  }
+  collection_def {
+    key: "legacy_init_op"
+    value {
+      node_list {
+        value: "group_deps"
+      }
+    }
+  }
+  collection_def {
+    key: "saved_model_assets"
+    value {
+      any_list {
+        value {
+          type_url: "type.googleapis.com/tensorflow.AssetFileDef"
+          value: "\n\t\n\007Const:0\022\007foo.txt"
+        }
+      }
+    }
+  }
+  collection_def {
+    key: "trainable_variables"
+    value {
+      bytes_list {
+        value: "\n\003a:0\022\010a/Assign\032\010a/read:0"
+        value: "\n\003b:0\022\010b/Assign\032\010b/read:0"
+        value: "\n\003c:0\022\010c/Assign\032\010c/read:0"
+      }
+    }
+  }
+  collection_def {
+    key: "variables"
+    value {
+      bytes_list {
+        value: "\n\003a:0\022\010a/Assign\032\010a/read:0"
+        value: "\n\003b:0\022\010b/Assign\032\010b/read:0"
+        value: "\n\003c:0\022\010c/Assign\032\010c/read:0"
+      }
+    }
+  }
+  signature_def {
+    key: "classify_x2_to_y3"
+    value {
+      inputs {
+        key: "inputs"
+        value {
+          name: "x2:0"
+          dtype: DT_FLOAT
+          tensor_shape {
+            dim {
+              size: -1
+            }
+            dim {
+              size: 1
+            }
+          }
+        }
+      }
+      outputs {
+        key: "scores"
+        value {
+          name: "y3:0"
+          dtype: DT_FLOAT
+          tensor_shape {
+            dim {
+              size: -1
+            }
+            dim {
+              size: 1
+            }
+          }
+        }
+      }
+      method_name: "tensorflow/serving/classify"
+    }
+  }
+  signature_def {
+    key: "classify_x_to_y"
+    value {
+      inputs {
+        key: "inputs"
+        value {
+          name: "tf_example:0"
+          dtype: DT_STRING
+          tensor_shape {
+            unknown_rank: true
+          }
+        }
+      }
+      outputs {
+        key: "scores"
+        value {
+          name: "y:0"
+          dtype: DT_FLOAT
+          tensor_shape {
+            dim {
+              size: -1
+            }
+            dim {
+              size: 1
+            }
+          }
+        }
+      }
+      method_name: "tensorflow/serving/classify"
+    }
+  }
+  signature_def {
+    key: "regress_x2_to_y3"
+    value {
+      inputs {
+        key: "inputs"
+        value {
+          name: "x2:0"
+          dtype: DT_FLOAT
+          tensor_shape {
+            dim {
+              size: -1
+            }
+            dim {
+              size: 1
+            }
+          }
+        }
+      }
+      outputs {
+        key: "outputs"
+        value {
+          name: "y3:0"
+          dtype: DT_FLOAT
+          tensor_shape {
+            dim {
+              size: -1
+            }
+            dim {
+              size: 1
+            }
+          }
+        }
+      }
+      method_name: "tensorflow/serving/regress"
+    }
+  }
+  signature_def {
+    key: "regress_x_to_y"
+    value {
+      inputs {
+        key: "inputs"
+        value {
+          name: "tf_example:0"
+          dtype: DT_STRING
+          tensor_shape {
+            unknown_rank: true
+          }
+        }
+      }
+      outputs {
+        key: "outputs"
+        value {
+          name: "y:0"
+          dtype: DT_FLOAT
+          tensor_shape {
+            dim {
+              size: -1
+            }
+            dim {
+              size: 1
+            }
+          }
+        }
+      }
+      method_name: "tensorflow/serving/regress"
+    }
+  }
+  signature_def {
+    key: "regress_x_to_y2"
+    value {
+      inputs {
+        key: "inputs"
+        value {
+          name: "tf_example:0"
+          dtype: DT_STRING
+          tensor_shape {
+            unknown_rank: true
+          }
+        }
+      }
+      outputs {
+        key: "outputs"
+        value {
+          name: "y2:0"
+          dtype: DT_FLOAT
+          tensor_shape {
+            dim {
+              size: -1
+            }
+            dim {
+              size: 1
+            }
+          }
+        }
+      }
+      method_name: "tensorflow/serving/regress"
+    }
+  }
+  signature_def {
+    key: "serving_default"
+    value {
+      inputs {
+        key: "x"
+        value {
+          name: "x:0"
+          dtype: DT_FLOAT
+          tensor_shape {
+            dim {
+              size: -1
+            }
+            dim {
+              size: 1
+            }
+          }
+        }
+      }
+      outputs {
+        key: "y"
+        value {
+          name: "y:0"
+          dtype: DT_FLOAT
+          tensor_shape {
+            dim {
+              size: -1
+            }
+            dim {
+              size: 1
+            }
+          }
+        }
+      }
+      method_name: "tensorflow/serving/predict"
+    }
+  }
+}
diff --git a/tensorflow/cc/saved_model/testdata/half_plus_two_forward_compatibility/00000123/variables/variables.data-00000-of-00001 b/tensorflow/cc/saved_model/testdata/half_plus_two_forward_compatibility/00000123/variables/variables.data-00000-of-00001
new file mode 100755
index 0000000000000000000000000000000000000000..15b75d6ef6bffc336d138d923badb3928b8c4c13
GIT binary patch
literal 12
TcmZQzV6bOkU~phyaBu(s1VaG;

literal 0
HcmV?d00001

diff --git a/tensorflow/cc/saved_model/testdata/half_plus_two_forward_compatibility/00000123/variables/variables.index b/tensorflow/cc/saved_model/testdata/half_plus_two_forward_compatibility/00000123/variables/variables.index
new file mode 100755
index 0000000000000000000000000000000000000000..7ec9fb4fe2dd21d0a6c324aecd7658fc37cf2326
GIT binary patch
literal 151
zcmZQzVB=tvV&Y(AVB}8ZU=(7|U@>L0P?u+5<V^x`6<9P_Oa=Z{_%kr_CW8eyG+0dE
zUu!zdz`(%32qK<{ZrcN*!JGr17H(i*WJ+Ohf(u1#dimX*BZLnmKnREbZs=Aib-xV&
DMWhyT

literal 0
HcmV?d00001

-- 
GitLab


From a06b378194780c30ee695e9fe9a5b77aaf8bf1f4 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 12 Oct 2017 22:21:01 -0700
Subject: [PATCH 347/909] Add "clear_output_shapes" option to FoldConstants
 transformer in tools/graph_transforms.

By setting this option to false, the transformer will not strip off the shape
information stored as attributes.

PiperOrigin-RevId: 172057283
---
 tensorflow/tools/graph_transforms/README.md   |   7 +-
 .../graph_transforms/fold_constants_lib.cc    | 108 +++++++++++++-----
 .../graph_transforms/fold_constants_test.cc   |  48 +++++++-
 3 files changed, 126 insertions(+), 37 deletions(-)

diff --git a/tensorflow/tools/graph_transforms/README.md b/tensorflow/tools/graph_transforms/README.md
index 00297f07b7..c7f7eca257 100644
--- a/tensorflow/tools/graph_transforms/README.md
+++ b/tensorflow/tools/graph_transforms/README.md
@@ -385,7 +385,12 @@ input is collapsed down into a simple constant.
 
 ### fold_constants
 
-Args: None \
+Args:
+
+*   clear_output_shapes: Clears tensor shape information saved as attributes.
+    Some older graphs containes out-of-date information and may cause import
+    errors. Defaults to true.
+
 Prerequisites: None
 
 Looks for any sub-graphs within the model that always evaluate to constant
diff --git a/tensorflow/tools/graph_transforms/fold_constants_lib.cc b/tensorflow/tools/graph_transforms/fold_constants_lib.cc
index 0f5bc2bcdd..30290c7a16 100644
--- a/tensorflow/tools/graph_transforms/fold_constants_lib.cc
+++ b/tensorflow/tools/graph_transforms/fold_constants_lib.cc
@@ -15,6 +15,9 @@ limitations under the License.
 
 #include "tensorflow/tools/graph_transforms/fold_constants_lib.h"
 
+#include <algorithm>
+#include <iterator>
+
 #include "tensorflow/core/common_runtime/constant_folding.h"
 #include "tensorflow/core/common_runtime/shape_refiner.h"
 #include "tensorflow/core/graph/graph_constructor.h"
@@ -194,56 +197,99 @@ Status ShapeForNode(const TransformFuncContext& context,
 Status FoldConstants(const GraphDef& input_graph_def,
                      const TransformFuncContext& context,
                      GraphDef* output_graph_def) {
-  // Some older GraphDefs have saved _output_shapes attributes which are out of
-  // date and cause import errors, so clean them up first.
-  GraphDef cleaned_graph_def;
-  RemoveAttributes(input_graph_def, {"_output_shapes"}, &cleaned_graph_def);
-
-  // Set specified shapes.
-  for (NodeDef& node : *cleaned_graph_def.mutable_node()) {
-    TensorShape shape;
-    bool has_shape_specified;
-    TF_RETURN_IF_ERROR(
-        ShapeForNode(context, node.name(), &shape, &has_shape_specified));
-    if (has_shape_specified) {
-      SetNodeAttr("shape", shape, &node);
-    }
-  }
-
   Graph input_graph(OpRegistry::Global());
+  TF_RETURN_IF_ERROR(input_graph.AddFunctionLibrary(input_graph_def.library()));
+
   ShapeRefiner shape_refiner(input_graph.versions(), input_graph.op_registry());
-  shape_refiner.set_require_shape_inference_fns(true);
+  shape_refiner.set_require_shape_inference_fns(false);
   shape_refiner.set_disable_constant_propagation(false);
-  ImportGraphDefOptions import_opts;
-  TF_RETURN_IF_ERROR(ImportGraphDef(import_opts, cleaned_graph_def,
-                                    &input_graph, &shape_refiner));
-  DeviceAttributes device_attributes;
-  subgraph::RewriteGraphMetadata metadata;
-  TF_RETURN_IF_ERROR(subgraph::RewriteGraphForExecution(
-      &input_graph, context.input_names, context.output_names, {},
-      device_attributes, false /* use_function_convention */, &metadata));
+  shape_refiner.set_function_library_for_shape_inference(
+      &input_graph.flib_def());
 
-  ConstantFoldingOptions cf_opts;
+  bool clear_output_shapes;
+  TF_RETURN_IF_ERROR(context.GetOneBoolParameter("clear_output_shapes", true,
+                                                 &clear_output_shapes));
+  if (clear_output_shapes) {
+    // Some older GraphDefs have saved _output_shapes attributes which are out
+    // of date and cause import errors, so clean them up first.
+    GraphDef cleaned_graph_def;
+    RemoveAttributes(input_graph_def, {"_output_shapes"}, &cleaned_graph_def);
+
+    // Set specified shapes.
+    for (NodeDef& node : *cleaned_graph_def.mutable_node()) {
+      TensorShape shape;
+      bool has_shape_specified;
+      TF_RETURN_IF_ERROR(
+          ShapeForNode(context, node.name(), &shape, &has_shape_specified));
+      if (has_shape_specified) {
+        SetNodeAttr("shape", shape, &node);
+      }
+    }
+
+    TF_RETURN_IF_ERROR(
+        ImportGraphDef({}, cleaned_graph_def, &input_graph, &shape_refiner));
+  } else {
+    TF_RETURN_IF_ERROR(
+        ImportGraphDef({}, input_graph_def, &input_graph, &shape_refiner));
+  }
+
+  // Sorted array of input names as lookup table.
+  std::vector<TensorId> input_names;
+  input_names.reserve(context.input_names.size());
+  std::transform(context.input_names.begin(), context.input_names.end(),
+                 std::back_inserter(input_names),
+                 [](const string& name) { return ParseTensorName(name); });
+
+  const auto compare = [](TensorId lhs, TensorId rhs) {
+    return lhs.first < rhs.first;
+  };
+
+  std::sort(input_names.begin(), input_names.end(), compare);
 
   // Set statically inferred shapes.
   std::unordered_map<string, std::vector<PartialTensorShape>> shape_map;
   for (const Node* const node : input_graph.nodes()) {
     auto ctx = shape_refiner.GetContext(node);
-    if (ctx == nullptr) continue;
+    if (ctx == nullptr) {
+      continue;
+    }
 
-    std::vector<PartialTensorShape>* partial_shapes = &shape_map[node->name()];
+    std::vector<PartialTensorShape>& partial_shapes = shape_map[node->name()];
     if (ctx->num_outputs() <= 0) continue;
-    partial_shapes->resize(ctx->num_outputs());
+    partial_shapes.resize(ctx->num_outputs());
 
     // Check all outputs.
     for (const Edge* out_edge : node->out_edges()) {
       if (out_edge->IsControlEdge()) continue;
 
       const int output_idx = out_edge->src_output();
-      TF_RETURN_IF_ERROR(ShapeHandleToTensorShape(
-          ctx->output(output_idx), ctx, &(*partial_shapes)[output_idx]));
+      TF_RETURN_IF_ERROR(ShapeHandleToTensorShape(ctx->output(output_idx), ctx,
+                                                  &partial_shapes[output_idx]));
+    }
+
+    // RewriteGraphForExecution() will add a Recv node for each input. Shape
+    // refiner does not include shape information of these Recv nodes. Therefore
+    // we add entries for Recv nodes here.
+    const auto pair = std::equal_range(input_names.begin(), input_names.end(),
+                                       TensorId{node->name(), 0}, compare);
+    for (auto it = pair.first; it != pair.second; ++it) {
+      const string recv_name =
+          strings::StrCat("_recv_", it->first, "_", it->second);
+      auto& recv_partial_shapes = shape_map[recv_name];
+      // For whatever reason (for example, name collision) if the map entry was
+      // already there, then do nothing.
+      if (recv_partial_shapes.empty()) {
+        recv_partial_shapes.push_back(partial_shapes[it->second]);
+      }
     }
   }
+
+  subgraph::RewriteGraphMetadata unused_metadata;
+  TF_RETURN_IF_ERROR(subgraph::RewriteGraphForExecution(
+      &input_graph, context.input_names, context.output_names, {}, {},
+      false /* use_function_convention */, &unused_metadata));
+
+  ConstantFoldingOptions cf_opts;
   cf_opts.shape_map = &shape_map;
 
   // Exclude specified nodes from constant folding.
diff --git a/tensorflow/tools/graph_transforms/fold_constants_test.cc b/tensorflow/tools/graph_transforms/fold_constants_test.cc
index d4100a652f..fd4188a6a4 100644
--- a/tensorflow/tools/graph_transforms/fold_constants_test.cc
+++ b/tensorflow/tools/graph_transforms/fold_constants_test.cc
@@ -20,6 +20,8 @@ limitations under the License.
 #include "tensorflow/cc/ops/nn_ops.h"
 #include "tensorflow/cc/ops/sendrecv_ops.h"
 #include "tensorflow/cc/ops/standard_ops.h"
+#include "tensorflow/core/framework/attr_value.pb.h"
+#include "tensorflow/core/framework/tensor_shape.pb.h"
 #include "tensorflow/core/framework/tensor_testutil.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
 #include "tensorflow/core/platform/test.h"
@@ -71,7 +73,7 @@ class ConstantFoldingTest : public ::testing::Test {
     test::FillIota<float>(&placeholder_tensor, 1.0f);
     TestConstantFolding(graph_def,
                         {{"placeholder_expect_remains", placeholder_tensor}},
-                        {}, {"output_expect_remains"});
+                        {}, {"output_expect_remains"}, {});
   }
 
   void TestOpExclusionAdd() {
@@ -105,7 +107,7 @@ class ConstantFoldingTest : public ::testing::Test {
     test::FillIota<float>(&placeholder_tensor, 1.0f);
     TestConstantFolding(graph_def,
                         {{"placeholder_expect_remains", placeholder_tensor}},
-                        {"Add"}, {"output_expect_remains"});
+                        {"Add"}, {"output_expect_remains"}, {});
   }
 
   void TestShapePropagation() {
@@ -129,13 +131,46 @@ class ConstantFoldingTest : public ::testing::Test {
     test::FillIota<float>(&placeholder_tensor, 1.0);
     TestConstantFolding(graph_def,
                         {{"placeholder_expect_remains", placeholder_tensor}},
-                        {}, {"output_expect_remains"});
+                        {}, {"output_expect_remains"}, {});
+  }
+
+  void TestPreserveOutputShapes() {
+    auto root = tensorflow::Scope::NewRootScope();
+    using namespace ::tensorflow::ops;  // NOLINT(build/namespaces)
+
+    tensorflow::AttrValue shape_attr;
+    auto* shape_proto = shape_attr.mutable_list()->add_shape();
+    shape_proto->add_dim()->set_size(1);
+    shape_proto->add_dim()->set_size(1);
+    shape_proto->add_dim()->set_size(3);
+
+    Output placeholder =
+        Placeholder(root.WithOpName("placeholder_expect_remains"), DT_FLOAT);
+    placeholder.node()->AddAttr("_output_shapes", shape_attr);
+
+    Output shape = Shape(root.WithOpName("shape_expect_removed"), placeholder);
+    Output cast = Cast(root.WithOpName("cast_expect_removed"), shape, DT_FLOAT);
+    Output mul =
+        Mul(root.WithOpName("output_expect_remains"), cast, placeholder);
+
+    GraphDef graph_def;
+    TF_ASSERT_OK(root.ToGraphDef(&graph_def));
+
+    Tensor placeholder_tensor(DT_FLOAT, TensorShape({1, 1, 3}));
+    test::FillIota<float>(&placeholder_tensor, 1.0);
+
+    graph_transforms::TransformFuncContext context;
+    context.params["clear_output_shapes"] = {"false"};
+    TestConstantFolding(graph_def,
+                        {{"placeholder_expect_remains", placeholder_tensor}},
+                        {}, {"output_expect_remains"}, context);
   }
 
   void TestConstantFolding(const GraphDef& graph_def,
                            std::vector<std::pair<string, Tensor> > inputs,
                            std::vector<string> excluded_ops,
-                           const std::vector<string>& outputs) {
+                           const std::vector<string>& outputs,
+                           graph_transforms::TransformFuncContext context) {
     std::unique_ptr<tensorflow::Session> unfolded_session(
         tensorflow::NewSession(tensorflow::SessionOptions()));
     TF_ASSERT_OK(unfolded_session->Create(graph_def));
@@ -143,7 +178,6 @@ class ConstantFoldingTest : public ::testing::Test {
     TF_ASSERT_OK(unfolded_session->Run(inputs, outputs, {}, &unfolded_tensors));
 
     GraphDef folded_graph_def;
-    graph_transforms::TransformFuncContext context;
     for (const std::pair<string, Tensor>& input : inputs) {
       context.input_names.push_back(input.first);
     }
@@ -269,6 +303,10 @@ TEST_F(ConstantFoldingTest, TestOpExclusionAdd) { TestOpExclusionAdd(); }
 
 TEST_F(ConstantFoldingTest, TestShapePropagation) { TestShapePropagation(); }
 
+TEST_F(ConstantFoldingTest, TestPreserveOutputShapes) {
+  TestPreserveOutputShapes();
+}
+
 TEST_F(ConstantFoldingTest, TestReplaceSendRecvs) { TestReplaceSendRecvs(); }
 
 TEST_F(ConstantFoldingTest, TestRemoveUnusedNodes) { TestRemoveUnusedNodes(); }
-- 
GitLab


From 5acdad4209cc8ee1a5c9421cbe38b0f2538843eb Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Fri, 13 Oct 2017 00:50:20 -0700
Subject: [PATCH 348/909] Automated g4 rollback of changelist 172048554

PiperOrigin-RevId: 172065800
---
 tensorflow/contrib/metrics/__init__.py        |   8 -
 .../contrib/metrics/python/ops/metric_ops.py  | 347 ---------
 .../metrics/python/ops/metric_ops_test.py     | 728 ------------------
 3 files changed, 1083 deletions(-)

diff --git a/tensorflow/contrib/metrics/__init__.py b/tensorflow/contrib/metrics/__init__.py
index 2c48882d0e..a9bce65e55 100644
--- a/tensorflow/contrib/metrics/__init__.py
+++ b/tensorflow/contrib/metrics/__init__.py
@@ -22,10 +22,6 @@ See the @{$python/contrib.metrics} guide.
 @@streaming_recall_at_thresholds
 @@streaming_precision
 @@streaming_precision_at_thresholds
-@@streaming_false_positive_rate
-@@streaming_false_positive_rate_at_thresholds
-@@streaming_false_negative_rate
-@@streaming_false_negative_rate_at_thresholds
 @@streaming_auc
 @@streaming_curve_points
 @@streaming_recall_at_k
@@ -84,12 +80,8 @@ from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_auc
 from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_concat
 from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_covariance
 from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_curve_points
-from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_false_negative_rate
-from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_false_negative_rate_at_thresholds
 from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_false_negatives
 from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_false_negatives_at_thresholds
-from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_false_positive_rate
-from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_false_positive_rate_at_thresholds
 from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_false_positives
 from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_false_positives_at_thresholds
 from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_mean
diff --git a/tensorflow/contrib/metrics/python/ops/metric_ops.py b/tensorflow/contrib/metrics/python/ops/metric_ops.py
index 85c8e9038a..76986d0156 100644
--- a/tensorflow/contrib/metrics/python/ops/metric_ops.py
+++ b/tensorflow/contrib/metrics/python/ops/metric_ops.py
@@ -565,213 +565,6 @@ def streaming_recall(predictions, labels, weights=None,
       updates_collections=updates_collections, name=name)
 
 
-def _true_negatives(labels, predictions, weights=None,
-                    metrics_collections=None,
-                    updates_collections=None,
-                    name=None):
-  """Sum the weights of true negatives.
-
-  If `weights` is `None`, weights default to 1. Use weights of 0 to mask values.
-
-  Args:
-    labels: The ground truth values, a `Tensor` whose dimensions must match
-      `predictions`. Will be cast to `bool`.
-    predictions: The predicted values, a `Tensor` of arbitrary dimensions. Will
-      be cast to `bool`.
-    weights: Optional `Tensor` whose rank is either 0, or the same rank as
-      `labels`, and must be broadcastable to `labels` (i.e., all dimensions must
-      be either `1`, or the same as the corresponding `labels` dimension).
-    metrics_collections: An optional list of collections that the metric
-      value variable should be added to.
-    updates_collections: An optional list of collections that the metric update
-      ops should be added to.
-    name: An optional variable_scope name.
-
-  Returns:
-    value_tensor: A `Tensor` representing the current value of the metric.
-    update_op: An operation that accumulates the error from a batch of data.
-
-  Raises:
-    ValueError: If `predictions` and `labels` have mismatched shapes, or if
-      `weights` is not `None` and its shape doesn't match `predictions`, or if
-      either `metrics_collections` or `updates_collections` are not a list or
-      tuple.
-  """
-  with variable_scope.variable_scope(
-      name, 'true_negatives', (predictions, labels, weights)):
-
-    predictions, labels, weights = _remove_squeezable_dimensions(
-        predictions=math_ops.cast(predictions, dtype=dtypes.bool),
-        labels=math_ops.cast(labels, dtype=dtypes.bool),
-        weights=weights)
-    is_true_negative = math_ops.logical_and(math_ops.equal(labels, False),
-                                            math_ops.equal(predictions, False))
-    return _count_condition(is_true_negative, weights, metrics_collections,
-                            updates_collections)
-
-
-def streaming_false_positive_rate(predictions, labels, weights=None,
-                                  metrics_collections=None,
-                                  updates_collections=None,
-                                  name=None):
-  """Computes the false positive rate of predictions with respect to labels.
-
-  The `false_positive_rate` function creates two local variables,
-  `false_positives` and `true_negatives`, that are used to compute the
-  false positive rate. This value is ultimately returned as
-  `false_positive_rate`, an idempotent operation that simply divides
-  `false_positives` by the sum of `false_positives` and `true_negatives`.
-
-  For estimation of the metric over a stream of data, the function creates an
-  `update_op` operation that updates these variables and returns the
-  `false_positive_rate`. `update_op` weights each prediction by the
-  corresponding value in `weights`.
-
-  If `weights` is `None`, weights default to 1. Use weights of 0 to mask values.
-
-  Args:
-    predictions: The predicted values, a `Tensor` of arbitrary dimensions. Will
-      be cast to `bool`.
-    labels: The ground truth values, a `Tensor` whose dimensions must match
-      `predictions`. Will be cast to `bool`.
-    weights: Optional `Tensor` whose rank is either 0, or the same rank as
-      `labels`, and must be broadcastable to `labels` (i.e., all dimensions must
-      be either `1`, or the same as the corresponding `labels` dimension).
-    metrics_collections: An optional list of collections that
-     `false_positive_rate` should be added to.
-    updates_collections: An optional list of collections that `update_op` should
-      be added to.
-    name: An optional variable_scope name.
-
-  Returns:
-    false_positive_rate: Scalar float `Tensor` with the value of
-      `false_positives` divided by the sum of `false_positives` and
-      `true_negatives`.
-    update_op: `Operation` that increments `false_positives` and
-      `true_negatives` variables appropriately and whose value matches
-      `false_positive_rate`.
-
-  Raises:
-    ValueError: If `predictions` and `labels` have mismatched shapes, or if
-      `weights` is not `None` and its shape doesn't match `predictions`, or if
-      either `metrics_collections` or `updates_collections` are not a list or
-      tuple.
-  """
-  with variable_scope.variable_scope(
-      name, 'false_positive_rate', (predictions, labels, weights)):
-    predictions, labels, weights = _remove_squeezable_dimensions(
-        predictions=math_ops.cast(predictions, dtype=dtypes.bool),
-        labels=math_ops.cast(labels, dtype=dtypes.bool),
-        weights=weights)
-
-    false_p, false_positives_update_op = metrics.false_positives(
-        labels, predictions, weights, metrics_collections=None,
-        updates_collections=None, name=None)
-    true_n, true_negatives_update_op = _true_negatives(
-        labels, predictions, weights, metrics_collections=None,
-        updates_collections=None, name=None)
-
-    def compute_fpr(fp, tn, name):
-      return array_ops.where(
-          math_ops.greater(fp + tn, 0),
-          math_ops.div(fp, fp + tn),
-          0,
-          name)
-
-    fpr = compute_fpr(false_p, true_n, 'value')
-    update_op = compute_fpr(
-        false_positives_update_op, true_negatives_update_op, 'update_op')
-
-    if metrics_collections:
-      ops.add_to_collections(metrics_collections, fpr)
-
-    if updates_collections:
-      ops.add_to_collections(updates_collections, update_op)
-
-    return fpr, update_op
-
-
-def streaming_false_negative_rate(predictions, labels, weights=None,
-                                  metrics_collections=None,
-                                  updates_collections=None,
-                                  name=None):
-  """Computes the false negative rate of predictions with respect to labels.
-
-  The `false_negative_rate` function creates two local variables,
-  `false_negatives` and `true_positives`, that are used to compute the
-  false positive rate. This value is ultimately returned as
-  `false_negative_rate`, an idempotent operation that simply divides
-  `false_negatives` by the sum of `false_negatives` and `true_positives`.
-
-  For estimation of the metric over a stream of data, the function creates an
-  `update_op` operation that updates these variables and returns the
-  `false_negative_rate`. `update_op` weights each prediction by the
-  corresponding value in `weights`.
-
-  If `weights` is `None`, weights default to 1. Use weights of 0 to mask values.
-
-  Args:
-    predictions: The predicted values, a `Tensor` of arbitrary dimensions. Will
-      be cast to `bool`.
-    labels: The ground truth values, a `Tensor` whose dimensions must match
-      `predictions`. Will be cast to `bool`.
-    weights: Optional `Tensor` whose rank is either 0, or the same rank as
-      `labels`, and must be broadcastable to `labels` (i.e., all dimensions must
-      be either `1`, or the same as the corresponding `labels` dimension).
-    metrics_collections: An optional list of collections that
-      `false_negative_rate` should be added to.
-    updates_collections: An optional list of collections that `update_op` should
-      be added to.
-    name: An optional variable_scope name.
-
-  Returns:
-    false_negative_rate: Scalar float `Tensor` with the value of
-      `false_negatives` divided by the sum of `false_negatives` and
-      `true_positives`.
-    update_op: `Operation` that increments `false_negatives` and
-      `true_positives` variables appropriately and whose value matches
-      `false_negative_rate`.
-
-  Raises:
-    ValueError: If `predictions` and `labels` have mismatched shapes, or if
-      `weights` is not `None` and its shape doesn't match `predictions`, or if
-      either `metrics_collections` or `updates_collections` are not a list or
-      tuple.
-  """
-  with variable_scope.variable_scope(
-      name, 'false_negative_rate', (predictions, labels, weights)):
-    predictions, labels, weights = _remove_squeezable_dimensions(
-        predictions=math_ops.cast(predictions, dtype=dtypes.bool),
-        labels=math_ops.cast(labels, dtype=dtypes.bool),
-        weights=weights)
-
-    false_n, false_negatives_update_op = metrics.false_negatives(
-        labels, predictions, weights, metrics_collections=None,
-        updates_collections=None, name=None)
-    true_p, true_positives_update_op = metrics.true_positives(
-        labels, predictions, weights, metrics_collections=None,
-        updates_collections=None, name=None)
-
-    def compute_fnr(fn, tp, name):
-      return array_ops.where(
-          math_ops.greater(fn + tp, 0),
-          math_ops.div(fn, fn + tp),
-          0,
-          name)
-
-    fnr = compute_fnr(false_n, true_p, 'value')
-    update_op = compute_fnr(
-        false_negatives_update_op, true_positives_update_op, 'update_op')
-
-    if metrics_collections:
-      ops.add_to_collections(metrics_collections, fnr)
-
-    if updates_collections:
-      ops.add_to_collections(updates_collections, update_op)
-
-    return fnr, update_op
-
-
 def _streaming_confusion_matrix_at_thresholds(
     predictions, labels, thresholds, weights=None, includes=None):
   """Computes true_positives, false_negatives, true_negatives, false_positives.
@@ -1321,142 +1114,6 @@ def streaming_recall_at_thresholds(predictions, labels, thresholds,
       updates_collections=updates_collections, name=name)
 
 
-def streaming_false_positive_rate_at_thresholds(
-    predictions, labels, thresholds, weights=None, metrics_collections=None,
-    updates_collections=None, name=None):
-  """Computes various fpr values for different `thresholds` on `predictions`.
-
-  The `streaming_false_positive_rate_at_thresholds` function creates two
-  local variables, `false_positives`, `true_negatives`, for various values of
-  thresholds. `false_positive_rate[i]` is defined as the total weight
-  of values in `predictions` above `thresholds[i]` whose corresponding entry in
-  `labels` is `False`, divided by the total weight of `False` values in `labels`
-  (`false_positives[i] / (false_positives[i] + true_negatives[i])`).
-
-  For estimation of the metric over a stream of data, the function creates an
-  `update_op` operation that updates these variables and returns the
-  `false_positive_rate`.
-
-  If `weights` is `None`, weights default to 1. Use weights of 0 to mask values.
-
-  Args:
-    predictions: A floating point `Tensor` of arbitrary shape and whose values
-      are in the range `[0, 1]`.
-    labels: A `bool` `Tensor` whose shape matches `predictions`.
-    thresholds: A python list or tuple of float thresholds in `[0, 1]`.
-    weights: `Tensor` whose rank is either 0, or the same rank as `labels`, and
-      must be broadcastable to `labels` (i.e., all dimensions must be either
-      `1`, or the same as the corresponding `labels` dimension).
-    metrics_collections: An optional list of collections that
-      `false_positive_rate` should be added to.
-    updates_collections: An optional list of collections that `update_op` should
-      be added to.
-    name: An optional variable_scope name.
-
-  Returns:
-    false_positive_rate: A float `Tensor` of shape `[len(thresholds)]`.
-    update_op: An operation that increments the `false_positives` and
-      `true_negatives` variables that are used in the computation of
-      `false_positive_rate`.
-
-  Raises:
-    ValueError: If `predictions` and `labels` have mismatched shapes, or if
-      `weights` is not `None` and its shape doesn't match `predictions`, or if
-      either `metrics_collections` or `updates_collections` are not a list or
-      tuple.
-  """
-  with variable_scope.variable_scope(
-      name, 'false_positive_rate_at_thresholds',
-      (predictions, labels, weights)):
-    values, update_ops = _streaming_confusion_matrix_at_thresholds(
-        predictions, labels, thresholds, weights, includes=('fp', 'tn'))
-
-    # Avoid division by zero.
-    epsilon = 1e-7
-    def compute_fpr(fp, tn, name):
-      return math_ops.div(fp, epsilon + fp + tn, name='fpr_' + name)
-
-    fpr = compute_fpr(values['fp'], values['tn'], 'value')
-    update_op = compute_fpr(
-        update_ops['fp'], update_ops['tn'], 'update_op')
-
-    if metrics_collections:
-      ops.add_to_collections(metrics_collections, fpr)
-
-    if updates_collections:
-      ops.add_to_collections(updates_collections, update_op)
-
-    return fpr, update_op
-
-
-def streaming_false_negative_rate_at_thresholds(
-    predictions, labels, thresholds, weights=None, metrics_collections=None,
-    updates_collections=None, name=None):
-  """Computes various fnr values for different `thresholds` on `predictions`.
-
-  The `streaming_false_negative_rate_at_thresholds` function creates two
-  local variables, `false_negatives`, `true_positives`, for various values of
-  thresholds. `false_negative_rate[i]` is defined as the total weight
-  of values in `predictions` above `thresholds[i]` whose corresponding entry in
-  `labels` is `False`, divided by the total weight of `True` values in `labels`
-  (`false_negatives[i] / (false_negatives[i] + true_positives[i])`).
-
-  For estimation of the metric over a stream of data, the function creates an
-  `update_op` operation that updates these variables and returns the
-  `false_positive_rate`.
-
-  If `weights` is `None`, weights default to 1. Use weights of 0 to mask values.
-
-  Args:
-    predictions: A floating point `Tensor` of arbitrary shape and whose values
-      are in the range `[0, 1]`.
-    labels: A `bool` `Tensor` whose shape matches `predictions`.
-    thresholds: A python list or tuple of float thresholds in `[0, 1]`.
-    weights: `Tensor` whose rank is either 0, or the same rank as `labels`, and
-      must be broadcastable to `labels` (i.e., all dimensions must be either
-      `1`, or the same as the corresponding `labels` dimension).
-    metrics_collections: An optional list of collections that
-      `false_negative_rate` should be added to.
-    updates_collections: An optional list of collections that `update_op` should
-      be added to.
-    name: An optional variable_scope name.
-
-  Returns:
-    false_negative_rate: A float `Tensor` of shape `[len(thresholds)]`.
-    update_op: An operation that increments the `false_negatives` and
-      `true_positives` variables that are used in the computation of
-      `false_negative_rate`.
-
-  Raises:
-    ValueError: If `predictions` and `labels` have mismatched shapes, or if
-      `weights` is not `None` and its shape doesn't match `predictions`, or if
-      either `metrics_collections` or `updates_collections` are not a list or
-      tuple.
-  """
-  with variable_scope.variable_scope(
-      name, 'false_negative_rate_at_thresholds',
-      (predictions, labels, weights)):
-    values, update_ops = _streaming_confusion_matrix_at_thresholds(
-        predictions, labels, thresholds, weights, includes=('fn', 'tp'))
-
-    # Avoid division by zero.
-    epsilon = 1e-7
-    def compute_fnr(fn, tp, name):
-      return math_ops.div(fn, epsilon + fn + tp, name='fnr_' + name)
-
-    fnr = compute_fnr(values['fn'], values['tp'], 'value')
-    update_op = compute_fnr(
-        update_ops['fn'], update_ops['tp'], 'update_op')
-
-    if metrics_collections:
-      ops.add_to_collections(metrics_collections, fnr)
-
-    if updates_collections:
-      ops.add_to_collections(updates_collections, update_op)
-
-    return fnr, update_op
-
-
 def _at_k_name(name, k=None, class_id=None):
   if k is not None:
     name = '%s_at_%d' % (name, k)
@@ -2822,12 +2479,8 @@ __all__ = [
     'streaming_accuracy',
     'streaming_auc',
     'streaming_curve_points',
-    'streaming_false_negative_rate',
-    'streaming_false_negative_rate_at_thresholds',
     'streaming_false_negatives',
     'streaming_false_negatives_at_thresholds',
-    'streaming_false_positive_rate',
-    'streaming_false_positive_rate_at_thresholds',
     'streaming_false_positives',
     'streaming_false_positives_at_thresholds',
     'streaming_mean',
diff --git a/tensorflow/contrib/metrics/python/ops/metric_ops_test.py b/tensorflow/contrib/metrics/python/ops/metric_ops_test.py
index e2067297cd..9b959b43a9 100644
--- a/tensorflow/contrib/metrics/python/ops/metric_ops_test.py
+++ b/tensorflow/contrib/metrics/python/ops/metric_ops_test.py
@@ -1355,262 +1355,6 @@ class StreamingRecallTest(test.TestCase):
       self.assertEqual(0, recall.eval())
 
 
-class StreamingFPRTest(test.TestCase):
-
-  def setUp(self):
-    np.random.seed(1)
-    ops.reset_default_graph()
-
-  def testVars(self):
-    metrics.streaming_false_positive_rate(
-        predictions=array_ops.ones((10, 1)), labels=array_ops.ones((10, 1)))
-    _assert_local_variables(self, (
-        'false_positive_rate/false_positives/count:0',
-        'false_positive_rate/true_negatives/count:0'))
-
-  def testMetricsCollection(self):
-    my_collection_name = '__metrics__'
-    mean, _ = metrics.streaming_false_positive_rate(
-        predictions=array_ops.ones((10, 1)),
-        labels=array_ops.ones((10, 1)),
-        metrics_collections=[my_collection_name])
-    self.assertListEqual(ops.get_collection(my_collection_name), [mean])
-
-  def testUpdatesCollection(self):
-    my_collection_name = '__updates__'
-    _, update_op = metrics.streaming_false_positive_rate(
-        predictions=array_ops.ones((10, 1)),
-        labels=array_ops.ones((10, 1)),
-        updates_collections=[my_collection_name])
-    self.assertListEqual(ops.get_collection(my_collection_name), [update_op])
-
-  def testValueTensorIsIdempotent(self):
-    predictions = random_ops.random_uniform(
-        (10, 3), maxval=1, dtype=dtypes_lib.int64, seed=1)
-    labels = random_ops.random_uniform(
-        (10, 3), maxval=1, dtype=dtypes_lib.int64, seed=2)
-    fpr, update_op = metrics.streaming_false_positive_rate(
-        predictions, labels)
-
-    with self.test_session() as sess:
-      sess.run(variables.local_variables_initializer())
-
-      # Run several updates.
-      for _ in range(10):
-        sess.run(update_op)
-
-      # Then verify idempotency.
-      initial_fpr = fpr.eval()
-      for _ in range(10):
-        self.assertEqual(initial_fpr, fpr.eval())
-
-  def testAllCorrect(self):
-    np_inputs = np.random.randint(0, 2, size=(100, 1))
-
-    predictions = constant_op.constant(np_inputs)
-    labels = constant_op.constant(np_inputs)
-    fpr, update_op = metrics.streaming_false_positive_rate(
-        predictions, labels)
-
-    with self.test_session() as sess:
-      sess.run(variables.local_variables_initializer())
-      sess.run(update_op)
-      self.assertEqual(0, fpr.eval())
-
-  def testSomeCorrect(self):
-    predictions = constant_op.constant([1, 0, 1, 0], shape=(1, 4))
-    labels = constant_op.constant([0, 1, 1, 0], shape=(1, 4))
-    fpr, update_op = metrics.streaming_false_positive_rate(
-        predictions, labels)
-
-    with self.test_session() as sess:
-      sess.run(variables.local_variables_initializer())
-      self.assertAlmostEqual(0.5, update_op.eval())
-      self.assertAlmostEqual(0.5, fpr.eval())
-
-  def testWeighted1d(self):
-    predictions = constant_op.constant([[1, 0, 1, 0], [0, 1, 0, 1]])
-    labels = constant_op.constant([[0, 1, 1, 0], [1, 0, 0, 1]])
-    weights = constant_op.constant([[2], [5]])
-    fpr, update_op = metrics.streaming_false_positive_rate(
-        predictions, labels, weights=weights)
-
-    with self.test_session() as sess:
-      sess.run(variables.local_variables_initializer())
-      weighted_fp = 2.0 + 5.0
-      weighted_f = (2.0 + 2.0) + (5.0 + 5.0)
-      expected_fpr = weighted_fp / weighted_f
-      self.assertAlmostEqual(expected_fpr, update_op.eval())
-      self.assertAlmostEqual(expected_fpr, fpr.eval())
-
-  def testWeighted2d(self):
-    predictions = constant_op.constant([[1, 0, 1, 0], [0, 1, 0, 1]])
-    labels = constant_op.constant([[0, 1, 1, 0], [1, 0, 0, 1]])
-    weights = constant_op.constant([[1, 2, 3, 4], [4, 3, 2, 1]])
-    fpr, update_op = metrics.streaming_false_positive_rate(
-        predictions, labels, weights=weights)
-
-    with self.test_session() as sess:
-      sess.run(variables.local_variables_initializer())
-      weighted_fp = 1.0 + 3.0
-      weighted_f = (1.0 + 4.0) + (2.0 + 3.0)
-      expected_fpr = weighted_fp / weighted_f
-      self.assertAlmostEqual(expected_fpr, update_op.eval())
-      self.assertAlmostEqual(expected_fpr, fpr.eval())
-
-  def testAllIncorrect(self):
-    np_inputs = np.random.randint(0, 2, size=(100, 1))
-
-    predictions = constant_op.constant(np_inputs)
-    labels = constant_op.constant(1 - np_inputs)
-    fpr, update_op = metrics.streaming_false_positive_rate(
-        predictions, labels)
-
-    with self.test_session() as sess:
-      sess.run(variables.local_variables_initializer())
-      sess.run(update_op)
-      self.assertEqual(1, fpr.eval())
-
-  def testZeroFalsePositivesAndTrueNegativesGivesZeroFPR(self):
-    predictions = array_ops.ones((1, 4))
-    labels = array_ops.ones((1, 4))
-    fpr, update_op = metrics.streaming_false_positive_rate(
-        predictions, labels)
-
-    with self.test_session() as sess:
-      sess.run(variables.local_variables_initializer())
-      sess.run(update_op)
-      self.assertEqual(0, fpr.eval())
-
-
-class StreamingFNRTest(test.TestCase):
-
-  def setUp(self):
-    np.random.seed(1)
-    ops.reset_default_graph()
-
-  def testVars(self):
-    metrics.streaming_false_negative_rate(
-        predictions=array_ops.ones((10, 1)), labels=array_ops.ones((10, 1)))
-    _assert_local_variables(self, (
-        'false_negative_rate/false_negatives/count:0',
-        'false_negative_rate/true_positives/count:0'))
-
-  def testMetricsCollection(self):
-    my_collection_name = '__metrics__'
-    mean, _ = metrics.streaming_false_negative_rate(
-        predictions=array_ops.ones((10, 1)),
-        labels=array_ops.ones((10, 1)),
-        metrics_collections=[my_collection_name])
-    self.assertListEqual(ops.get_collection(my_collection_name), [mean])
-
-  def testUpdatesCollection(self):
-    my_collection_name = '__updates__'
-    _, update_op = metrics.streaming_false_negative_rate(
-        predictions=array_ops.ones((10, 1)),
-        labels=array_ops.ones((10, 1)),
-        updates_collections=[my_collection_name])
-    self.assertListEqual(ops.get_collection(my_collection_name), [update_op])
-
-  def testValueTensorIsIdempotent(self):
-    predictions = random_ops.random_uniform(
-        (10, 3), maxval=1, dtype=dtypes_lib.int64, seed=1)
-    labels = random_ops.random_uniform(
-        (10, 3), maxval=1, dtype=dtypes_lib.int64, seed=2)
-    fnr, update_op = metrics.streaming_false_negative_rate(
-        predictions, labels)
-
-    with self.test_session() as sess:
-      sess.run(variables.local_variables_initializer())
-
-      # Run several updates.
-      for _ in range(10):
-        sess.run(update_op)
-
-      # Then verify idempotency.
-      initial_fnr = fnr.eval()
-      for _ in range(10):
-        self.assertEqual(initial_fnr, fnr.eval())
-
-  def testAllCorrect(self):
-    np_inputs = np.random.randint(0, 2, size=(100, 1))
-
-    predictions = constant_op.constant(np_inputs)
-    labels = constant_op.constant(np_inputs)
-    fnr, update_op = metrics.streaming_false_negative_rate(
-        predictions, labels)
-
-    with self.test_session() as sess:
-      sess.run(variables.local_variables_initializer())
-      sess.run(update_op)
-      self.assertEqual(0, fnr.eval())
-
-  def testSomeCorrect(self):
-    predictions = constant_op.constant([1, 0, 1, 0], shape=(1, 4))
-    labels = constant_op.constant([0, 1, 1, 0], shape=(1, 4))
-    fnr, update_op = metrics.streaming_false_negative_rate(
-        predictions, labels)
-
-    with self.test_session() as sess:
-      sess.run(variables.local_variables_initializer())
-      self.assertAlmostEqual(0.5, update_op.eval())
-      self.assertAlmostEqual(0.5, fnr.eval())
-
-  def testWeighted1d(self):
-    predictions = constant_op.constant([[1, 0, 1, 0], [0, 1, 0, 1]])
-    labels = constant_op.constant([[0, 1, 1, 0], [1, 0, 0, 1]])
-    weights = constant_op.constant([[2], [5]])
-    fnr, update_op = metrics.streaming_false_negative_rate(
-        predictions, labels, weights=weights)
-
-    with self.test_session() as sess:
-      sess.run(variables.local_variables_initializer())
-      weighted_fn = 2.0 + 5.0
-      weighted_t = (2.0 + 2.0) + (5.0 + 5.0)
-      expected_fnr = weighted_fn / weighted_t
-      self.assertAlmostEqual(expected_fnr, update_op.eval())
-      self.assertAlmostEqual(expected_fnr, fnr.eval())
-
-  def testWeighted2d(self):
-    predictions = constant_op.constant([[1, 0, 1, 0], [0, 1, 0, 1]])
-    labels = constant_op.constant([[0, 1, 1, 0], [1, 0, 0, 1]])
-    weights = constant_op.constant([[1, 2, 3, 4], [4, 3, 2, 1]])
-    fnr, update_op = metrics.streaming_false_negative_rate(
-        predictions, labels, weights=weights)
-
-    with self.test_session() as sess:
-      sess.run(variables.local_variables_initializer())
-      weighted_fn = 2.0 + 4.0
-      weighted_t = (2.0 + 3.0) + (1.0 + 4.0)
-      expected_fnr = weighted_fn / weighted_t
-      self.assertAlmostEqual(expected_fnr, update_op.eval())
-      self.assertAlmostEqual(expected_fnr, fnr.eval())
-
-  def testAllIncorrect(self):
-    np_inputs = np.random.randint(0, 2, size=(100, 1))
-
-    predictions = constant_op.constant(np_inputs)
-    labels = constant_op.constant(1 - np_inputs)
-    fnr, update_op = metrics.streaming_false_negative_rate(
-        predictions, labels)
-
-    with self.test_session() as sess:
-      sess.run(variables.local_variables_initializer())
-      sess.run(update_op)
-      self.assertEqual(1, fnr.eval())
-
-  def testZeroFalseNegativesAndTruePositivesGivesZeroFNR(self):
-    predictions = array_ops.zeros((1, 4))
-    labels = array_ops.zeros((1, 4))
-    fnr, update_op = metrics.streaming_false_negative_rate(
-        predictions, labels)
-
-    with self.test_session() as sess:
-      sess.run(variables.local_variables_initializer())
-      sess.run(update_op)
-      self.assertEqual(0, fnr.eval())
-
-
 class StreamingCurvePointsTest(test.TestCase):
 
   def setUp(self):
@@ -2524,478 +2268,6 @@ class StreamingPrecisionRecallThresholdsTest(test.TestCase):
       self.assertAlmostEqual(expected_rec, rec.eval(), 2)
 
 
-class StreamingFPRThresholdsTest(test.TestCase):
-
-  def setUp(self):
-    np.random.seed(1)
-    ops.reset_default_graph()
-
-  def testVars(self):
-    metrics.streaming_false_positive_rate_at_thresholds(
-        predictions=array_ops.ones((10, 1)),
-        labels=array_ops.ones((10, 1)),
-        thresholds=[0, 0.5, 1.0])
-    _assert_local_variables(self, (
-        'false_positive_rate_at_thresholds/false_positives:0',
-        'false_positive_rate_at_thresholds/true_negatives:0',))
-
-  def testMetricsCollection(self):
-    my_collection_name = '__metrics__'
-    fpr, _ = metrics.streaming_false_positive_rate_at_thresholds(
-        predictions=array_ops.ones((10, 1)),
-        labels=array_ops.ones((10, 1)),
-        thresholds=[0, 0.5, 1.0],
-        metrics_collections=[my_collection_name])
-    self.assertListEqual(ops.get_collection(my_collection_name), [fpr])
-
-  def testUpdatesCollection(self):
-    my_collection_name = '__updates__'
-    _, update_op = metrics.streaming_false_positive_rate_at_thresholds(
-        predictions=array_ops.ones((10, 1)),
-        labels=array_ops.ones((10, 1)),
-        thresholds=[0, 0.5, 1.0],
-        updates_collections=[my_collection_name])
-    self.assertListEqual(
-        ops.get_collection(my_collection_name), [update_op])
-
-  def testValueTensorIsIdempotent(self):
-    predictions = random_ops.random_uniform(
-        (10, 3), maxval=1, dtype=dtypes_lib.float32, seed=1)
-    labels = random_ops.random_uniform(
-        (10, 3), maxval=1, dtype=dtypes_lib.int64, seed=2)
-    thresholds = [0, 0.5, 1.0]
-    fpr, fpr_op = metrics.streaming_false_positive_rate_at_thresholds(
-        predictions, labels, thresholds)
-
-    with self.test_session() as sess:
-      sess.run(variables.local_variables_initializer())
-
-      # Run several updates.
-      for _ in range(10):
-        sess.run(fpr_op)
-
-      # Then verify idempotency.
-      initial_fpr = fpr.eval()
-      for _ in range(10):
-        self.assertAllClose(initial_fpr, fpr.eval())
-
-  def testAllCorrect(self):
-    inputs = np.random.randint(0, 2, size=(100, 1))
-
-    with self.test_session() as sess:
-      predictions = constant_op.constant(inputs, dtype=dtypes_lib.float32)
-      labels = constant_op.constant(inputs)
-      thresholds = [0.5]
-      fpr, fpr_op = metrics.streaming_false_positive_rate_at_thresholds(
-          predictions, labels, thresholds)
-
-      sess.run(variables.local_variables_initializer())
-      sess.run(fpr_op)
-
-      self.assertEqual(0, fpr.eval())
-
-  def testSomeCorrect(self):
-    with self.test_session() as sess:
-      predictions = constant_op.constant(
-          [1, 0, 1, 0], shape=(1, 4), dtype=dtypes_lib.float32)
-      labels = constant_op.constant([0, 1, 1, 0], shape=(1, 4))
-      thresholds = [0.5]
-      fpr, fpr_op = metrics.streaming_false_positive_rate_at_thresholds(
-          predictions, labels, thresholds)
-
-      sess.run(variables.local_variables_initializer())
-      sess.run(fpr_op)
-
-      self.assertAlmostEqual(0.5, fpr.eval())
-
-  def testAllIncorrect(self):
-    inputs = np.random.randint(0, 2, size=(100, 1))
-
-    with self.test_session() as sess:
-      predictions = constant_op.constant(inputs, dtype=dtypes_lib.float32)
-      labels = constant_op.constant(1 - inputs, dtype=dtypes_lib.float32)
-      thresholds = [0.5]
-      fpr, fpr_op = metrics.streaming_false_positive_rate_at_thresholds(
-          predictions, labels, thresholds)
-
-      sess.run(variables.local_variables_initializer())
-      sess.run(fpr_op)
-
-      self.assertAlmostEqual(1, fpr.eval())
-
-  def testWeights1d(self):
-    with self.test_session() as sess:
-      predictions = constant_op.constant(
-          [[1, 0], [1, 0]], shape=(2, 2), dtype=dtypes_lib.float32)
-      labels = constant_op.constant([[0, 1], [1, 0]], shape=(2, 2))
-      weights = constant_op.constant(
-          [[0], [1]], shape=(2, 1), dtype=dtypes_lib.float32)
-      thresholds = [0.5, 1.1]
-      fpr, fpr_op = metrics.streaming_false_positive_rate_at_thresholds(
-          predictions, labels, thresholds, weights=weights)
-
-      [fpr_low, fpr_high] = array_ops.split(
-          value=fpr, num_or_size_splits=2, axis=0)
-      fpr_low = array_ops.reshape(fpr_low, shape=())
-      fpr_high = array_ops.reshape(fpr_high, shape=())
-
-      sess.run(variables.local_variables_initializer())
-      sess.run(fpr_op)
-
-      self.assertAlmostEqual(0.0, fpr_low.eval(), places=5)
-      self.assertAlmostEqual(0.0, fpr_high.eval(), places=5)
-
-  def testWeights2d(self):
-    with self.test_session() as sess:
-      predictions = constant_op.constant(
-          [[1, 0], [1, 0]], shape=(2, 2), dtype=dtypes_lib.float32)
-      labels = constant_op.constant([[0, 1], [1, 0]], shape=(2, 2))
-      weights = constant_op.constant(
-          [[0, 0], [1, 1]], shape=(2, 2), dtype=dtypes_lib.float32)
-      thresholds = [0.5, 1.1]
-      fpr, fpr_op = metrics.streaming_false_positive_rate_at_thresholds(
-          predictions, labels, thresholds, weights=weights)
-
-      [fpr_low, fpr_high] = array_ops.split(
-          value=fpr, num_or_size_splits=2, axis=0)
-      fpr_low = array_ops.reshape(fpr_low, shape=())
-      fpr_high = array_ops.reshape(fpr_high, shape=())
-
-      sess.run(variables.local_variables_initializer())
-      sess.run(fpr_op)
-
-      self.assertAlmostEqual(0.0, fpr_low.eval(), places=5)
-      self.assertAlmostEqual(0.0, fpr_high.eval(), places=5)
-
-  def testExtremeThresholds(self):
-    with self.test_session() as sess:
-      predictions = constant_op.constant(
-          [1, 0, 1, 0], shape=(1, 4), dtype=dtypes_lib.float32)
-      labels = constant_op.constant([0, 1, 1, 1], shape=(1, 4))
-      thresholds = [-1.0, 2.0]  # lower/higher than any values
-      fpr, fpr_op = metrics.streaming_false_positive_rate_at_thresholds(
-          predictions, labels, thresholds)
-
-      [fpr_low, fpr_high] = array_ops.split(
-          value=fpr, num_or_size_splits=2, axis=0)
-
-      sess.run(variables.local_variables_initializer())
-      sess.run(fpr_op)
-
-      self.assertAlmostEqual(1.0, fpr_low.eval(), places=5)
-      self.assertAlmostEqual(0.0, fpr_high.eval(), places=5)
-
-  def testZeroLabelsPredictions(self):
-    with self.test_session() as sess:
-      predictions = array_ops.zeros([4], dtype=dtypes_lib.float32)
-      labels = array_ops.zeros([4])
-      thresholds = [0.5]
-      fpr, fpr_op = metrics.streaming_false_positive_rate_at_thresholds(
-          predictions, labels, thresholds)
-
-      sess.run(variables.local_variables_initializer())
-      sess.run(fpr_op)
-
-      self.assertAlmostEqual(0, fpr.eval(), 6)
-
-  def testWithMultipleUpdates(self):
-    num_samples = 1000
-    batch_size = 10
-    num_batches = int(num_samples / batch_size)
-
-    # Create the labels and data.
-    labels = np.random.randint(0, 2, size=(num_samples, 1))
-    noise = np.random.normal(0.0, scale=0.2, size=(num_samples, 1))
-    predictions = 0.4 + 0.2 * labels + noise
-    predictions[predictions > 1] = 1
-    predictions[predictions < 0] = 0
-    thresholds = [0.3]
-
-    fp = 0
-    tn = 0
-    for i in range(num_samples):
-      if predictions[i] > thresholds[0]:
-        if labels[i] == 0:
-          fp += 1
-      else:
-        if labels[i] == 0:
-          tn += 1
-    epsilon = 1e-7
-    expected_fpr = fp / (epsilon + fp + tn)
-
-    labels = labels.astype(np.float32)
-    predictions = predictions.astype(np.float32)
-
-    with self.test_session() as sess:
-      # Reshape the data so its easy to queue up:
-      predictions_batches = predictions.reshape((batch_size, num_batches))
-      labels_batches = labels.reshape((batch_size, num_batches))
-
-      # Enqueue the data:
-      predictions_queue = data_flow_ops.FIFOQueue(
-          num_batches, dtypes=dtypes_lib.float32, shapes=(batch_size,))
-      labels_queue = data_flow_ops.FIFOQueue(
-          num_batches, dtypes=dtypes_lib.float32, shapes=(batch_size,))
-
-      for i in range(int(num_batches)):
-        tf_prediction = constant_op.constant(predictions_batches[:, i])
-        tf_label = constant_op.constant(labels_batches[:, i])
-        sess.run([
-            predictions_queue.enqueue(tf_prediction),
-            labels_queue.enqueue(tf_label)
-        ])
-
-      tf_predictions = predictions_queue.dequeue()
-      tf_labels = labels_queue.dequeue()
-
-      fpr, fpr_op = metrics.streaming_false_positive_rate_at_thresholds(
-          tf_predictions, tf_labels, thresholds)
-
-      sess.run(variables.local_variables_initializer())
-      for _ in range(int(num_samples / batch_size)):
-        sess.run(fpr_op)
-      # Since this is only approximate, we can't expect a 6 digits match.
-      # Although with higher number of samples/thresholds we should see the
-      # accuracy improving
-      self.assertAlmostEqual(expected_fpr, fpr.eval(), 2)
-
-
-class StreamingFNRThresholdsTest(test.TestCase):
-
-  def setUp(self):
-    np.random.seed(1)
-    ops.reset_default_graph()
-
-  def testVars(self):
-    metrics.streaming_false_negative_rate_at_thresholds(
-        predictions=array_ops.ones((10, 1)),
-        labels=array_ops.ones((10, 1)),
-        thresholds=[0, 0.5, 1.0])
-    _assert_local_variables(self, (
-        'false_negative_rate_at_thresholds/false_negatives:0',
-        'false_negative_rate_at_thresholds/true_positives:0',))
-
-  def testMetricsCollection(self):
-    my_collection_name = '__metrics__'
-    fnr, _ = metrics.streaming_false_negative_rate_at_thresholds(
-        predictions=array_ops.ones((10, 1)),
-        labels=array_ops.ones((10, 1)),
-        thresholds=[0, 0.5, 1.0],
-        metrics_collections=[my_collection_name])
-    self.assertListEqual(ops.get_collection(my_collection_name), [fnr])
-
-  def testUpdatesCollection(self):
-    my_collection_name = '__updates__'
-    _, update_op = metrics.streaming_false_negative_rate_at_thresholds(
-        predictions=array_ops.ones((10, 1)),
-        labels=array_ops.ones((10, 1)),
-        thresholds=[0, 0.5, 1.0],
-        updates_collections=[my_collection_name])
-    self.assertListEqual(
-        ops.get_collection(my_collection_name), [update_op])
-
-  def testValueTensorIsIdempotent(self):
-    predictions = random_ops.random_uniform(
-        (10, 3), maxval=1, dtype=dtypes_lib.float32, seed=1)
-    labels = random_ops.random_uniform(
-        (10, 3), maxval=1, dtype=dtypes_lib.int64, seed=2)
-    thresholds = [0, 0.5, 1.0]
-    fnr, fnr_op = metrics.streaming_false_negative_rate_at_thresholds(
-        predictions, labels, thresholds)
-
-    with self.test_session() as sess:
-      sess.run(variables.local_variables_initializer())
-
-      # Run several updates.
-      for _ in range(10):
-        sess.run(fnr_op)
-
-      # Then verify idempotency.
-      initial_fnr = fnr.eval()
-      for _ in range(10):
-        self.assertAllClose(initial_fnr, fnr.eval())
-
-  def testAllCorrect(self):
-    inputs = np.random.randint(0, 2, size=(100, 1))
-
-    with self.test_session() as sess:
-      predictions = constant_op.constant(inputs, dtype=dtypes_lib.float32)
-      labels = constant_op.constant(inputs)
-      thresholds = [0.5]
-      fnr, fnr_op = metrics.streaming_false_negative_rate_at_thresholds(
-          predictions, labels, thresholds)
-
-      sess.run(variables.local_variables_initializer())
-      sess.run(fnr_op)
-
-      self.assertEqual(0, fnr.eval())
-
-  def testSomeCorrect(self):
-    with self.test_session() as sess:
-      predictions = constant_op.constant(
-          [1, 0, 1, 0], shape=(1, 4), dtype=dtypes_lib.float32)
-      labels = constant_op.constant([0, 1, 1, 0], shape=(1, 4))
-      thresholds = [0.5]
-      fnr, fnr_op = metrics.streaming_false_negative_rate_at_thresholds(
-          predictions, labels, thresholds)
-
-      sess.run(variables.local_variables_initializer())
-      sess.run(fnr_op)
-
-      self.assertAlmostEqual(0.5, fnr.eval())
-
-  def testAllIncorrect(self):
-    inputs = np.random.randint(0, 2, size=(100, 1))
-
-    with self.test_session() as sess:
-      predictions = constant_op.constant(inputs, dtype=dtypes_lib.float32)
-      labels = constant_op.constant(1 - inputs, dtype=dtypes_lib.float32)
-      thresholds = [0.5]
-      fnr, fnr_op = metrics.streaming_false_negative_rate_at_thresholds(
-          predictions, labels, thresholds)
-
-      sess.run(variables.local_variables_initializer())
-      sess.run(fnr_op)
-
-      self.assertAlmostEqual(1, fnr.eval())
-
-  def testWeights1d(self):
-    with self.test_session() as sess:
-      predictions = constant_op.constant(
-          [[1, 0], [1, 0]], shape=(2, 2), dtype=dtypes_lib.float32)
-      labels = constant_op.constant([[0, 1], [1, 0]], shape=(2, 2))
-      weights = constant_op.constant(
-          [[0], [1]], shape=(2, 1), dtype=dtypes_lib.float32)
-      thresholds = [0.5, 1.1]
-      fnr, fnr_op = metrics.streaming_false_negative_rate_at_thresholds(
-          predictions, labels, thresholds, weights=weights)
-
-      [fnr_low, fnr_high] = array_ops.split(
-          value=fnr, num_or_size_splits=2, axis=0)
-      fnr_low = array_ops.reshape(fnr_low, shape=())
-      fnr_high = array_ops.reshape(fnr_high, shape=())
-
-      sess.run(variables.local_variables_initializer())
-      sess.run(fnr_op)
-
-      self.assertAlmostEqual(0.0, fnr_low.eval(), places=5)
-      self.assertAlmostEqual(1.0, fnr_high.eval(), places=5)
-
-  def testWeights2d(self):
-    with self.test_session() as sess:
-      predictions = constant_op.constant(
-          [[1, 0], [1, 0]], shape=(2, 2), dtype=dtypes_lib.float32)
-      labels = constant_op.constant([[0, 1], [1, 0]], shape=(2, 2))
-      weights = constant_op.constant(
-          [[0, 0], [1, 1]], shape=(2, 2), dtype=dtypes_lib.float32)
-      thresholds = [0.5, 1.1]
-      fnr, fnr_op = metrics.streaming_false_negative_rate_at_thresholds(
-          predictions, labels, thresholds, weights=weights)
-
-      [fnr_low, fnr_high] = array_ops.split(
-          value=fnr, num_or_size_splits=2, axis=0)
-      fnr_low = array_ops.reshape(fnr_low, shape=())
-      fnr_high = array_ops.reshape(fnr_high, shape=())
-
-      sess.run(variables.local_variables_initializer())
-      sess.run(fnr_op)
-
-      self.assertAlmostEqual(0.0, fnr_low.eval(), places=5)
-      self.assertAlmostEqual(1.0, fnr_high.eval(), places=5)
-
-  def testExtremeThresholds(self):
-    with self.test_session() as sess:
-      predictions = constant_op.constant(
-          [1, 0, 1, 0], shape=(1, 4), dtype=dtypes_lib.float32)
-      labels = constant_op.constant([0, 1, 1, 1], shape=(1, 4))
-      thresholds = [-1.0, 2.0]  # lower/higher than any values
-      fnr, fnr_op = metrics.streaming_false_negative_rate_at_thresholds(
-          predictions, labels, thresholds)
-
-      [fnr_low, fnr_high] = array_ops.split(
-          value=fnr, num_or_size_splits=2, axis=0)
-
-      sess.run(variables.local_variables_initializer())
-      sess.run(fnr_op)
-
-      self.assertAlmostEqual(0.0, fnr_low.eval())
-      self.assertAlmostEqual(1.0, fnr_high.eval())
-
-  def testZeroLabelsPredictions(self):
-    with self.test_session() as sess:
-      predictions = array_ops.zeros([4], dtype=dtypes_lib.float32)
-      labels = array_ops.zeros([4])
-      thresholds = [0.5]
-      fnr, fnr_op = metrics.streaming_false_negative_rate_at_thresholds(
-          predictions, labels, thresholds)
-
-      sess.run(variables.local_variables_initializer())
-      sess.run(fnr_op)
-
-      self.assertAlmostEqual(0, fnr.eval(), 6)
-
-  def testWithMultipleUpdates(self):
-    num_samples = 1000
-    batch_size = 10
-    num_batches = int(num_samples / batch_size)
-
-    # Create the labels and data.
-    labels = np.random.randint(0, 2, size=(num_samples, 1))
-    noise = np.random.normal(0.0, scale=0.2, size=(num_samples, 1))
-    predictions = 0.4 + 0.2 * labels + noise
-    predictions[predictions > 1] = 1
-    predictions[predictions < 0] = 0
-    thresholds = [0.3]
-
-    fn = 0
-    tp = 0
-    for i in range(num_samples):
-      if predictions[i] > thresholds[0]:
-        if labels[i] == 1:
-          tp += 1
-      else:
-        if labels[i] == 1:
-          fn += 1
-    epsilon = 1e-7
-    expected_fnr = fn / (epsilon + fn + tp)
-
-    labels = labels.astype(np.float32)
-    predictions = predictions.astype(np.float32)
-
-    with self.test_session() as sess:
-      # Reshape the data so its easy to queue up:
-      predictions_batches = predictions.reshape((batch_size, num_batches))
-      labels_batches = labels.reshape((batch_size, num_batches))
-
-      # Enqueue the data:
-      predictions_queue = data_flow_ops.FIFOQueue(
-          num_batches, dtypes=dtypes_lib.float32, shapes=(batch_size,))
-      labels_queue = data_flow_ops.FIFOQueue(
-          num_batches, dtypes=dtypes_lib.float32, shapes=(batch_size,))
-
-      for i in range(int(num_batches)):
-        tf_prediction = constant_op.constant(predictions_batches[:, i])
-        tf_label = constant_op.constant(labels_batches[:, i])
-        sess.run([
-            predictions_queue.enqueue(tf_prediction),
-            labels_queue.enqueue(tf_label)
-        ])
-
-      tf_predictions = predictions_queue.dequeue()
-      tf_labels = labels_queue.dequeue()
-
-      fnr, fnr_op = metrics.streaming_false_negative_rate_at_thresholds(
-          tf_predictions, tf_labels, thresholds)
-
-      sess.run(variables.local_variables_initializer())
-      for _ in range(int(num_samples / batch_size)):
-        sess.run(fnr_op)
-      # Since this is only approximate, we can't expect a 6 digits match.
-      # Although with higher number of samples/thresholds we should see the
-      # accuracy improving
-      self.assertAlmostEqual(expected_fnr, fnr.eval(), 2)
-
-
 # TODO(ptucker): Remove when we remove `streaming_recall_at_k`.
 # This op will be deprecated soon in favor of `streaming_sparse_recall_at_k`.
 # Until then, this test validates that both ops yield the same results.
-- 
GitLab


From 2645045db26914e179be2e161134be7e9cd9002b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 13 Oct 2017 02:10:57 -0700
Subject: [PATCH 349/909] Extend the ExecuteParallel service interface to allow
 multiple devices per computation.

PiperOrigin-RevId: 172071664
---
 tensorflow/compiler/xla/client/client.cc      |  3 -
 tensorflow/compiler/xla/client/client.h       | 11 +++-
 tensorflow/compiler/xla/service/compiler.h    |  3 +-
 .../compiler/xla/service/cpu/cpu_compiler.cc  |  2 +-
 .../compiler/xla/service/cpu/cpu_compiler.h   |  3 +-
 .../compiler/xla/service/gpu/gpu_compiler.cc  |  2 +-
 .../compiler/xla/service/gpu/gpu_compiler.h   |  3 +-
 .../xla/service/interpreter/compiler.cc       |  2 +-
 .../xla/service/interpreter/compiler.h        |  3 +-
 tensorflow/compiler/xla/service/service.cc    | 57 ++++++++++++++-----
 tensorflow/compiler/xla/service/service.h     |  2 +-
 tensorflow/compiler/xla/xla.proto             | 11 ++--
 12 files changed, 68 insertions(+), 34 deletions(-)

diff --git a/tensorflow/compiler/xla/client/client.cc b/tensorflow/compiler/xla/client/client.cc
index 7db2ea79fb..92cd8e729d 100644
--- a/tensorflow/compiler/xla/client/client.cc
+++ b/tensorflow/compiler/xla/client/client.cc
@@ -242,9 +242,6 @@ StatusOr<std::vector<std::unique_ptr<GlobalData>>> Client::ExecuteParallel(
     for (GlobalData* argument : computation.arguments) {
       *single_request.add_arguments() = argument->handle();
     }
-    if (computation.device_handle != nullptr) {
-      *single_request.mutable_device_handle() = *computation.device_handle;
-    }
     *single_request.mutable_execution_options() = computation.execution_options;
     *request.add_requests() = single_request;
   }
diff --git a/tensorflow/compiler/xla/client/client.h b/tensorflow/compiler/xla/client/client.h
index e72816a621..a716159f9e 100644
--- a/tensorflow/compiler/xla/client/client.h
+++ b/tensorflow/compiler/xla/client/client.h
@@ -45,6 +45,10 @@ class Client {
   // * If execution_options is not nullptr, these options are passed to the
   //   service to affect how it compiles our computation.  (The pointer does not
   //   need to live beyond this call.)
+  // * If execution_options.device_handles is not empty, the computation is
+  //   executed on the devices associated with the handles by partitioning the
+  //   computation based on the attached sharding attributes. Otherwise, a
+  //   device is chosen by the service.
   // * If execution_profile is not nullptr then the pointed-to ExecutionProfile
   //   will be filled with profile data from the execution.
   StatusOr<std::unique_ptr<GlobalData>> Execute(
@@ -54,12 +58,13 @@ class Client {
       ExecutionProfile* execution_profile = nullptr);
 
   // A struct to represent a computation instance to be executed.
-  // * If device_handle is not nullptr, the computation is executed on a device
-  //   associated with the handle. Otherwise, a device is chosen by the service.
+  // * If execution_options.device_handles is not empty, the computation is
+  //   executed on the devices associated with the handles by partitioning the
+  //   computation based on the attached sharding attributes. Otherwise, a
+  //   device is chosen by the service.
   struct ComputationInstance {
     const Computation& computation;
     std::vector<GlobalData*> arguments;
-    const DeviceHandle* device_handle;
     ExecutionOptions execution_options;
     ExecutionProfile* execution_profile;
   };
diff --git a/tensorflow/compiler/xla/service/compiler.h b/tensorflow/compiler/xla/service/compiler.h
index d5bd9214be..4c2d9600d9 100644
--- a/tensorflow/compiler/xla/service/compiler.h
+++ b/tensorflow/compiler/xla/service/compiler.h
@@ -114,7 +114,8 @@ class Compiler {
   // sequence of executable objects.
   virtual StatusOr<std::vector<std::unique_ptr<Executable>>> Compile(
       std::vector<std::unique_ptr<HloModule>> modules,
-      std::vector<perftools::gputools::StreamExecutor*> stream_exec) = 0;
+      std::vector<std::vector<perftools::gputools::StreamExecutor*>>
+          stream_exec) = 0;
 
   // Compiles the HLO module for ahead-of-time execution.  This is intended for
   // use in static compilation.
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
index 386800d221..1437fb4cf9 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
@@ -653,7 +653,7 @@ StatusOr<std::unique_ptr<Executable>> CpuCompiler::Compile(
 
 StatusOr<std::vector<std::unique_ptr<Executable>>> CpuCompiler::Compile(
     std::vector<std::unique_ptr<HloModule>> modules,
-    std::vector<se::StreamExecutor*> stream_execs) {
+    std::vector<std::vector<se::StreamExecutor*>> stream_execs) {
   return Unimplemented(
       "Compilation of multiple HLO modules is not yet supported on CPU.");
 }
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_compiler.h b/tensorflow/compiler/xla/service/cpu/cpu_compiler.h
index bd3541500d..a301d04337 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_compiler.h
+++ b/tensorflow/compiler/xla/service/cpu/cpu_compiler.h
@@ -115,7 +115,8 @@ class CpuCompiler : public LLVMCompiler {
 
   StatusOr<std::vector<std::unique_ptr<Executable>>> Compile(
       std::vector<std::unique_ptr<HloModule>> modules,
-      std::vector<perftools::gputools::StreamExecutor*> stream_exec) override;
+      std::vector<std::vector<perftools::gputools::StreamExecutor*>>
+          stream_execs) override;
 
   StatusOr<std::vector<std::unique_ptr<AotCompilationResult>>>
   CompileAheadOfTime(std::vector<std::unique_ptr<HloModule>> modules,
diff --git a/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc b/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc
index 0bcdf8a61d..57f11db11f 100644
--- a/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc
+++ b/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc
@@ -408,7 +408,7 @@ StatusOr<std::unique_ptr<Executable>> GpuCompiler::Compile(
 
 StatusOr<std::vector<std::unique_ptr<Executable>>> GpuCompiler::Compile(
     std::vector<std::unique_ptr<HloModule>> modules,
-    std::vector<se::StreamExecutor*> stream_execs) {
+    std::vector<std::vector<se::StreamExecutor*>> stream_execs) {
   return Unimplemented(
       "Compilation of multiple HLO modules is not yet supported on GPU.");
 }
diff --git a/tensorflow/compiler/xla/service/gpu/gpu_compiler.h b/tensorflow/compiler/xla/service/gpu/gpu_compiler.h
index b5ffeef44f..58e835e5ee 100644
--- a/tensorflow/compiler/xla/service/gpu/gpu_compiler.h
+++ b/tensorflow/compiler/xla/service/gpu/gpu_compiler.h
@@ -46,7 +46,8 @@ class GpuCompiler : public LLVMCompiler {
 
   StatusOr<std::vector<std::unique_ptr<Executable>>> Compile(
       std::vector<std::unique_ptr<HloModule>> modules,
-      std::vector<perftools::gputools::StreamExecutor*> stream_exec) override;
+      std::vector<std::vector<perftools::gputools::StreamExecutor*>>
+          stream_execs) override;
 
   StatusOr<std::vector<std::unique_ptr<AotCompilationResult>>>
   CompileAheadOfTime(std::vector<std::unique_ptr<HloModule>> module,
diff --git a/tensorflow/compiler/xla/service/interpreter/compiler.cc b/tensorflow/compiler/xla/service/interpreter/compiler.cc
index c8d02834f4..93ea2f7367 100644
--- a/tensorflow/compiler/xla/service/interpreter/compiler.cc
+++ b/tensorflow/compiler/xla/service/interpreter/compiler.cc
@@ -88,7 +88,7 @@ StatusOr<std::unique_ptr<Executable>> InterpreterCompiler::Compile(
 
 StatusOr<std::vector<std::unique_ptr<Executable>>> InterpreterCompiler::Compile(
     std::vector<std::unique_ptr<HloModule>> /*hlo_modules*/,
-    std::vector<se::StreamExecutor*> /*stream_execs*/) {
+    std::vector<std::vector<se::StreamExecutor*>> /*stream_execs*/) {
   return tensorflow::errors::Unimplemented(
       "Compilation of multiple HLO modules is not supported on Interpreter.");
 }
diff --git a/tensorflow/compiler/xla/service/interpreter/compiler.h b/tensorflow/compiler/xla/service/interpreter/compiler.h
index 13db38ab60..cfdc9b6256 100644
--- a/tensorflow/compiler/xla/service/interpreter/compiler.h
+++ b/tensorflow/compiler/xla/service/interpreter/compiler.h
@@ -49,7 +49,8 @@ class InterpreterCompiler : public Compiler {
 
   StatusOr<std::vector<std::unique_ptr<Executable>>> Compile(
       std::vector<std::unique_ptr<HloModule>> hlo_modules,
-      std::vector<perftools::gputools::StreamExecutor*> stream_exec) override;
+      std::vector<std::vector<perftools::gputools::StreamExecutor*>>
+          stream_exec) override;
 
   StatusOr<std::vector<std::unique_ptr<AotCompilationResult>>>
   CompileAheadOfTime(std::vector<std::unique_ptr<HloModule>> hlo_modules,
diff --git a/tensorflow/compiler/xla/service/service.cc b/tensorflow/compiler/xla/service/service.cc
index d279e1f50f..0fbc2f2fec 100644
--- a/tensorflow/compiler/xla/service/service.cc
+++ b/tensorflow/compiler/xla/service/service.cc
@@ -338,7 +338,7 @@ StatusOr<std::vector<std::unique_ptr<Executable>>> Service::BuildExecutables(
     std::vector<VersionedComputationHandle> versioned_handles,
     std::vector<std::unique_ptr<HloModuleConfig>> module_configs,
     Backend* backend,
-    std::vector<perftools::gputools::StreamExecutor*> executors) {
+    std::vector<std::vector<perftools::gputools::StreamExecutor*>> executors) {
   VLOG(1) << Printf("BuildExecutable on service %p", this);
 
   // Dump computation proto state if flag is set.
@@ -615,31 +615,41 @@ tensorflow::Status Service::ExecuteParallel(const ExecuteParallelRequest* arg,
   VLOG(1) << "running execute-parallel request: " << arg->ShortDebugString();
 
   std::vector<std::vector<se::DeviceMemoryBase>> all_arguments;
-  std::vector<perftools::gputools::StreamExecutor*> executors;
+  std::vector<std::vector<perftools::gputools::StreamExecutor*>> all_executors;
   std::vector<VersionedComputationHandle> versioned_handles;
   std::vector<std::unique_ptr<HloModuleConfig>> module_configs;
   std::vector<string> computation_names;
   std::vector<DeviceHandle> device_handles;
 
-  if (arg->requests_size() * options_.number_of_replicas() >
+  int num_requested_devices =
+      std::accumulate(arg->requests().begin(), arg->requests().end(), 0,
+                      [](int a, const ExecuteRequest& r) -> int {
+                        return a + r.execution_options().device_handles_size();
+                      });
+  if (num_requested_devices * options_.number_of_replicas() >
       execute_backend_->device_count()) {
     return FailedPrecondition(
         "there are not enough stream executors to execute %d computations",
-        arg->requests_size());
+        num_requested_devices);
   }
 
   for (int64 i = 0; i < arg->requests_size(); ++i) {
     // Get the stream executor for the i'th computation. This stream executor
     // is one of the executors to run the replicated computation.
-    if (!arg->requests(i).has_device_handle()) {
+    const ExecutionOptions& execution_options =
+        arg->requests(i).execution_options();
+    if (execution_options.device_handles().empty()) {
       return FailedPrecondition(
           "device handles must be given to execute parallel computations");
     }
-    TF_ASSIGN_OR_RETURN(
-        auto replicas,
-        Replicas(*execute_backend_, arg->requests(i).device_handle()));
-    se::StreamExecutor* executor = replicas[0];
-    CHECK(executor != nullptr);
+    std::vector<perftools::gputools::StreamExecutor*> executors;
+    for (const auto& device_handle : execution_options.device_handles()) {
+      TF_ASSIGN_OR_RETURN(auto replicas,
+                          Replicas(*execute_backend_, device_handle));
+      se::StreamExecutor* executor = replicas[0];
+      CHECK(executor != nullptr);
+      executors.push_back(executor);
+    }
 
     // Resolve the UserComputation object associated with the requested
     // computation and compute the program shape.
@@ -658,10 +668,12 @@ tensorflow::Status Service::ExecuteParallel(const ExecuteParallelRequest* arg,
 
     // Resolve the allocations for the arguments of the computation, and create
     // a vector of device memory offsets for the arguments from the allocations.
+    // In the case of partitioned computations, assume all arguments go on the
+    // zeroth core.
     TF_ASSIGN_OR_RETURN(
         std::vector<const Allocation*> arg_allocations,
         ResolveAndValidateArguments(request.arguments(), execute_backend_.get(),
-                                    executor->device_ordinal()));
+                                    executors[0]->device_ordinal()));
     std::vector<se::DeviceMemoryBase> arguments;
     arguments.reserve(arg_allocations.size());
     for (const Allocation* allocation : arg_allocations) {
@@ -678,11 +690,15 @@ tensorflow::Status Service::ExecuteParallel(const ExecuteParallelRequest* arg,
 
     // Adds to the vectors to build and execute the computations after the loop.
     all_arguments.push_back(arguments);
+    all_arguments.insert(all_arguments.end(), executors.size() - 1, {});
     versioned_handles.push_back(versioned_handle);
     module_configs.push_back(std::move(module_config));
-    computation_names.push_back(user_computation->name());
-    executors.push_back(executor);
-    device_handles.push_back(arg->requests(i).device_handle());
+    computation_names.insert(computation_names.end(), executors.size(),
+                             user_computation->name());
+    all_executors.push_back(executors);
+    device_handles.insert(device_handles.end(),
+                          execution_options.device_handles().begin(),
+                          execution_options.device_handles().end());
   }
 
   // Build the user computations into HloModules and compile to generate the
@@ -690,7 +706,7 @@ tensorflow::Status Service::ExecuteParallel(const ExecuteParallelRequest* arg,
   TF_ASSIGN_OR_RETURN(
       std::vector<std::unique_ptr<Executable>> executables,
       BuildExecutables(versioned_handles, std::move(module_configs),
-                       execute_backend_.get(), executors));
+                       execute_backend_.get(), all_executors));
   std::vector<Executable*> executable_ptrs;
   executable_ptrs.reserve(executables.size());
   for (const auto& executable : executables) {
@@ -752,6 +768,17 @@ tensorflow::Status Service::Execute(const ExecuteRequest* arg,
     return InvalidArgument("computations may not be empty");
   }
 
+  // If we received multiple device handles, we must partition the module.
+  if (arg->execution_options().device_handles_size() > 1) {
+    ExecuteParallelRequest parallel_arg;
+    *parallel_arg.add_requests() = *arg;
+    ExecuteParallelResponse parallel_result;
+    TF_RETURN_IF_ERROR(ExecuteParallel(&parallel_arg, &parallel_result));
+    TF_RET_CHECK(parallel_result.responses_size() > 0);
+    *result = parallel_result.responses(0);
+    return Status::OK();
+  }
+
   TF_ASSIGN_OR_RETURN(
       std::shared_ptr<const ProgramShape> program_shape,
       user_computation->ComputeProgramShape(versioned_handle.version));
diff --git a/tensorflow/compiler/xla/service/service.h b/tensorflow/compiler/xla/service/service.h
index f96f18f072..2452259f73 100644
--- a/tensorflow/compiler/xla/service/service.h
+++ b/tensorflow/compiler/xla/service/service.h
@@ -293,7 +293,7 @@ class Service : public ServiceInterface {
       std::vector<VersionedComputationHandle> versioned_handles,
       std::vector<std::unique_ptr<HloModuleConfig>> module_configs,
       Backend* backend,
-      std::vector<perftools::gputools::StreamExecutor*> executors);
+      std::vector<std::vector<perftools::gputools::StreamExecutor*>> executors);
 
   // Similar to BuildExecutable, but look in the compilation cache for the
   // executable first. If the executable is not in the cache, it is built and
diff --git a/tensorflow/compiler/xla/xla.proto b/tensorflow/compiler/xla/xla.proto
index 4840ddb881..7f4bd26d1b 100644
--- a/tensorflow/compiler/xla/xla.proto
+++ b/tensorflow/compiler/xla/xla.proto
@@ -191,6 +191,11 @@ message ExecutionOptions {
   uint64 seed = 3;
 
   DebugOptions debug_options = 4;
+
+  // This optional field specifies a particular set of devices to run the
+  // computation on. The computation will be partitioned across these devices.
+  // If not provided, the default device will be chosen.
+  repeated DeviceHandle device_handles = 5;
 }
 
 message SnapshotComputationRequest {
@@ -312,12 +317,8 @@ message ExecuteRequest {
   ComputationHandle computation = 1;
   repeated GlobalDataHandle arguments = 2;
 
-  // This optional field specifies a particular device to run the computation.
-  // If not provided, the default device will be chosen.
-  DeviceHandle device_handle = 5;
-
   // Options that affect how XLA compiles and runs code to service this request.
-  ExecutionOptions execution_options = 6;
+  ExecutionOptions execution_options = 5;
 }
 
 message ExecuteParallelRequest {
-- 
GitLab


From 7e31a198c8a6a6a618ca959a69941bcdd82cb140 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 13 Oct 2017 02:39:17 -0700
Subject: [PATCH 350/909] Small improvement of verification heuristics.

PiperOrigin-RevId: 172073518
---
 .../learn/python/learn/utils/saved_model_export_utils.py    | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/learn/python/learn/utils/saved_model_export_utils.py b/tensorflow/contrib/learn/python/learn/utils/saved_model_export_utils.py
index 5975103f4f..a7a1411b60 100644
--- a/tensorflow/contrib/learn/python/learn/utils/saved_model_export_utils.py
+++ b/tensorflow/contrib/learn/python/learn/utils/saved_model_export_utils.py
@@ -109,7 +109,11 @@ def build_standardized_signature_def(input_tensors, output_tensors,
     classes = _get_classification_classes(output_tensors)
     scores = _get_classification_scores(output_tensors)
     if classes is None and scores is None:
-      (_, classes), = output_tensors.items()
+      items = output_tensors.items()
+      if items[0][1].dtype == dtypes.string:
+        (_, classes), = items
+      else:
+        (_, scores), = items
     return signature_def_utils.classification_signature_def(
         examples, classes, scores)
   elif _is_regression_problem(problem_type, input_tensors, output_tensors):
-- 
GitLab


From d5e99136e47f1add4df3f577983ec7b02d20d4c3 Mon Sep 17 00:00:00 2001
From: Simon Perkins <simon.perkins@gmail.com>
Date: Fri, 13 Oct 2017 12:10:59 +0200
Subject: [PATCH 351/909] Fix possible deadlocks in Staging Areas.

Previously, `notify_one` was used to notify inserters and removers
waiting to insert and remove elements into the Staging Areas. This could
result in deadlock when  many removers where waiting for
different keys in the case of the MapStagingArea, or were waiting on
either peeks or get operations in the StagingArea.

For example, if two removers were waiting for keys 2 and 3 in a
MapStaging Area respectively, and 2 was inserted but only 3's remover was
notified, it is possible that 2's remover would never be notified resulting
in deadlock. Thus, both should be notified.

Similarly in the case of the StagingArea with a remover and a peeker
wanting to remove the last element and peek at a specific element
respectively, it is not clear which one should be notified due to
an insert. Thus, both should be notified.

Additionally, all inserters are now notified when an element is removed.
Consider the case where two inserters are waiting to small elements into
the Staging Area and a remover removes a single large element. As there
may be space for both insertion elements, both inserters should be
notified.
---
 tensorflow/core/kernels/map_stage_op.cc | 12 +++++++++---
 tensorflow/core/kernels/stage_op.cc     | 14 ++++++++++----
 2 files changed, 19 insertions(+), 7 deletions(-)

diff --git a/tensorflow/core/kernels/map_stage_op.cc b/tensorflow/core/kernels/map_stage_op.cc
index 0168b57d35..7b5a464b72 100644
--- a/tensorflow/core/kernels/map_stage_op.cc
+++ b/tensorflow/core/kernels/map_stage_op.cc
@@ -111,15 +111,21 @@ class StagingMap : public ResourceBase {
   void notify_inserters_if_bounded(std::unique_lock<std::mutex>* lock) {
     if (has_capacity() || has_memory_limit()) {
       lock->unlock();
-      full_.notify_one();
+      // Notify all inserters. The removal of an element
+      // may make memory available for many inserters
+      // to insert new elements
+      full_.notify_all();
     }
   }
 
-  // Notify any removers waiting to extract values
+  // Notify all removers waiting to extract values
   // that data is now available
   void notify_removers(std::unique_lock<std::mutex>* lock) {
     lock->unlock();
-    not_empty_.notify_one();
+    // Notify all removers. This is because they are
+    // waiting for specific keys to appear in the map
+    // so we don't know which one to wake up.
+    not_empty_.notify_all();
   }
 
   bool has_capacity() const { return capacity_ > 0; }
diff --git a/tensorflow/core/kernels/stage_op.cc b/tensorflow/core/kernels/stage_op.cc
index 1717428adf..0fae46dea6 100644
--- a/tensorflow/core/kernels/stage_op.cc
+++ b/tensorflow/core/kernels/stage_op.cc
@@ -53,7 +53,10 @@ class Buffer : public ResourceBase {
   void notify_inserters_if_bounded(std::unique_lock<std::mutex>* lock) {
     if (IsBounded()) {
       lock->unlock();
-      full_cond_var_.notify_one();
+      // Notify all inserters. The removal of an element
+      // may make memory available for many inserters
+      // to insert new elements
+      full_cond_var_.notify_all();
     }
   }
 
@@ -115,9 +118,12 @@ class Buffer : public ResourceBase {
     buf_.push_back(std::move(*tuple));
 
     lock.unlock();
-    // maybe possible to optimize by reducing
-    // how often this signal is sent
-    non_empty_cond_var_.notify_one();
+    // Notify all removers. Removers
+    // may be peeking at a specific element or waiting
+    // for the element at the front of the deque.
+    // As we don't know the appropriate one to wake up
+    // we should wake them all.
+    non_empty_cond_var_.notify_all();
 
     return Status::OK();
   }
-- 
GitLab


From a3b2d6f395ef3f66c9ccd8578e94243e49f76576 Mon Sep 17 00:00:00 2001
From: Peter Hawkins <phawkins@google.com>
Date: Fri, 13 Oct 2017 06:55:45 -0700
Subject: [PATCH 352/909] [TF:XLA] Add bound to ArgMax in randomized tests.

PiperOrigin-RevId: 172091245
---
 tensorflow/compiler/tests/randomized_tests.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/compiler/tests/randomized_tests.cc b/tensorflow/compiler/tests/randomized_tests.cc
index 56e10a1587..5129171cd4 100644
--- a/tensorflow/compiler/tests/randomized_tests.cc
+++ b/tensorflow/compiler/tests/randomized_tests.cc
@@ -899,7 +899,7 @@ TEST_F(OpTest, ApproximateEqual) {
 
 TEST_F(OpTest, ArgMax) {
   Repeatedly([this]() {
-    std::vector<int64> dims = RandomDims(1, 5);
+    std::vector<int64> dims = RandomDims(1, 5, 1);
     int num_dims = dims.size();
     int reduce_dim =
         std::uniform_int_distribution<int32>(-num_dims, num_dims)(generator());
-- 
GitLab


From 1c241e5ba7fa7068f9cf8f925638b170db57c438 Mon Sep 17 00:00:00 2001
From: Peter Hawkins <phawkins@google.com>
Date: Fri, 13 Oct 2017 07:00:42 -0700
Subject: [PATCH 353/909] [XLA] Add ShiftLeft, ShiftRightArithmetic, and
 ShiftRightLogical operators.

PiperOrigin-RevId: 172091595
---
 .../xla/client/computation_builder.cc         | 18 ++++
 .../compiler/xla/client/computation_builder.h | 10 +++
 .../compiler/xla/service/dfs_hlo_visitor.h    | 15 ++++
 .../xla/service/elemental_ir_emitter.cc       |  9 ++
 .../compiler/xla/service/hlo_evaluator.cc     | 87 +++++++++++++++++++
 .../compiler/xla/service/hlo_graph_dumper.cc  |  3 +
 .../compiler/xla/service/hlo_instruction.cc   | 22 +++++
 .../compiler/xla/service/hlo_matchers.h       |  3 +
 tensorflow/compiler/xla/service/hlo_opcode.cc |  6 ++
 tensorflow/compiler/xla/service/hlo_opcode.h  |  3 +
 .../xla/service/instruction_fusion.cc         |  3 +
 .../compiler/xla/service/shape_inference.cc   |  9 ++
 .../compiler/xla/service/user_computation.cc  |  6 ++
 .../xla/tests/array_elementwise_ops_test.cc   | 66 ++++++++++++++
 tensorflow/compiler/xla/xla_data.proto        |  4 +
 15 files changed, 264 insertions(+)

diff --git a/tensorflow/compiler/xla/client/computation_builder.cc b/tensorflow/compiler/xla/client/computation_builder.cc
index 206af290c6..dcbdb3525e 100644
--- a/tensorflow/compiler/xla/client/computation_builder.cc
+++ b/tensorflow/compiler/xla/client/computation_builder.cc
@@ -972,6 +972,24 @@ ComputationDataHandle ComputationBuilder::Not(
   return UnaryOp(UNOP_NOT, operand);
 }
 
+ComputationDataHandle ComputationBuilder::ShiftLeft(
+    const ComputationDataHandle& lhs, const ComputationDataHandle& rhs,
+    tensorflow::gtl::ArraySlice<int64> broadcast_dimensions) {
+  return BinaryOp(BINOP_SHIFT_LEFT, lhs, rhs, broadcast_dimensions);
+}
+
+ComputationDataHandle ComputationBuilder::ShiftRightArithmetic(
+    const ComputationDataHandle& lhs, const ComputationDataHandle& rhs,
+    tensorflow::gtl::ArraySlice<int64> broadcast_dimensions) {
+  return BinaryOp(BINOP_SHIFT_RIGHT_ARITHMETIC, lhs, rhs, broadcast_dimensions);
+}
+
+ComputationDataHandle ComputationBuilder::ShiftRightLogical(
+    const ComputationDataHandle& lhs, const ComputationDataHandle& rhs,
+    tensorflow::gtl::ArraySlice<int64> broadcast_dimensions) {
+  return BinaryOp(BINOP_SHIFT_RIGHT_LOGICAL, lhs, rhs, broadcast_dimensions);
+}
+
 ComputationDataHandle ComputationBuilder::Abs(
     const ComputationDataHandle& operand) {
   return UnaryOp(UNOP_ABS, operand);
diff --git a/tensorflow/compiler/xla/client/computation_builder.h b/tensorflow/compiler/xla/client/computation_builder.h
index 94b03502f9..cdd9c8847f 100644
--- a/tensorflow/compiler/xla/client/computation_builder.h
+++ b/tensorflow/compiler/xla/client/computation_builder.h
@@ -472,6 +472,16 @@ class ComputationBuilder {
 
   ComputationDataHandle Not(const ComputationDataHandle& operand);
 
+  ComputationDataHandle ShiftLeft(
+      const ComputationDataHandle& lhs, const ComputationDataHandle& rhs,
+      tensorflow::gtl::ArraySlice<int64> broadcast_dimensions = {});
+  ComputationDataHandle ShiftRightArithmetic(
+      const ComputationDataHandle& lhs, const ComputationDataHandle& rhs,
+      tensorflow::gtl::ArraySlice<int64> broadcast_dimensions = {});
+  ComputationDataHandle ShiftRightLogical(
+      const ComputationDataHandle& lhs, const ComputationDataHandle& rhs,
+      tensorflow::gtl::ArraySlice<int64> broadcast_dimensions = {});
+
   // Reduces an array among the provided dimensions, given "computation" as a
   // reduction operator.
   ComputationDataHandle Reduce(
diff --git a/tensorflow/compiler/xla/service/dfs_hlo_visitor.h b/tensorflow/compiler/xla/service/dfs_hlo_visitor.h
index 8c864f3d07..5b1dbf439c 100644
--- a/tensorflow/compiler/xla/service/dfs_hlo_visitor.h
+++ b/tensorflow/compiler/xla/service/dfs_hlo_visitor.h
@@ -167,6 +167,21 @@ class DfsHloVisitor {
                           HloInstruction* rhs) {
     return HandleElementwiseBinary(or_);
   }
+  virtual Status HandleShiftLeft(HloInstruction* shift_left,
+                                 HloInstruction* lhs, HloInstruction* rhs) {
+    return HandleElementwiseBinary(shift_left);
+  }
+  virtual Status HandleShiftRightArithmetic(
+      HloInstruction* shift_right_arithmetic, HloInstruction* lhs,
+      HloInstruction* rhs) {
+    return HandleElementwiseBinary(shift_right_arithmetic);
+  }
+  virtual Status HandleShiftRightLogical(HloInstruction* shift_right_logical,
+                                         HloInstruction* lhs,
+                                         HloInstruction* rhs) {
+    return HandleElementwiseBinary(shift_right_logical);
+  }
+
   virtual Status HandleReducePrecision(HloInstruction* reduce_precision) {
     return HandleElementwiseUnary(reduce_precision);
   }
diff --git a/tensorflow/compiler/xla/service/elemental_ir_emitter.cc b/tensorflow/compiler/xla/service/elemental_ir_emitter.cc
index fb4d233d04..44f709bede 100644
--- a/tensorflow/compiler/xla/service/elemental_ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/elemental_ir_emitter.cc
@@ -568,6 +568,12 @@ StatusOr<llvm::Value*> ElementalIrEmitter::EmitIntegerBinaryOp(
       return ir_builder_->CreateAnd(lhs_value, rhs_value);
     case HloOpcode::kOr:
       return ir_builder_->CreateOr(lhs_value, rhs_value);
+    case HloOpcode::kShiftLeft:
+      return ir_builder_->CreateShl(lhs_value, rhs_value);
+    case HloOpcode::kShiftRightArithmetic:
+      return ir_builder_->CreateAShr(lhs_value, rhs_value);
+    case HloOpcode::kShiftRightLogical:
+      return ir_builder_->CreateLShr(lhs_value, rhs_value);
     default:
       return Unimplemented("binary integer op '%s'",
                            HloOpcodeString(op->opcode()).c_str());
@@ -830,6 +836,9 @@ llvm_ir::ElementGenerator ElementalIrEmitter::MakeElementGenerator(
     case HloOpcode::kSubtract:
     case HloOpcode::kAnd:
     case HloOpcode::kOr:
+    case HloOpcode::kShiftLeft:
+    case HloOpcode::kShiftRightArithmetic:
+    case HloOpcode::kShiftRightLogical:
       return [this, hlo, &operand_to_generator](
                  const IrArray::Index& index) -> StatusOr<llvm::Value*> {
         const HloInstruction* lhs = hlo->operand(0);
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator.cc b/tensorflow/compiler/xla/service/hlo_evaluator.cc
index 20dba60f4e..5fd891835d 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator.cc
+++ b/tensorflow/compiler/xla/service/hlo_evaluator.cc
@@ -387,6 +387,93 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault {
     return Status::OK();
   };
 
+  template <typename NativeT,
+            typename std::enable_if<
+                std::is_integral<NativeT>::value &&
+                !std::is_same<NativeT, bool>::value>::type* = nullptr>
+  Status HandleShiftLeft(HloInstruction* shl, HloInstruction* lhs,
+                         HloInstruction* rhs) {
+    TF_ASSIGN_OR_RETURN(
+        parent_->evaluated_[shl],
+        ElementWiseBinaryOp(shl, [](NativeT lhs_elem, NativeT rhs_elem) {
+          return lhs_elem << rhs_elem;
+        }));
+    return Status::OK();
+  }
+
+  template <typename NativeT,
+            typename std::enable_if<!std::is_integral<NativeT>::value ||
+                                    std::is_same<NativeT, bool>::value>::type* =
+                nullptr>
+  Status HandleShiftLeft(HloInstruction* shl, HloInstruction* lhs,
+                         HloInstruction* rhs) {
+    return InvalidArgument("Unsupported type for ShiftLeft");
+  }
+
+  Status HandleShiftLeft(HloInstruction* shl, HloInstruction* lhs,
+                         HloInstruction* rhs) override {
+    return HandleShiftLeft<ReturnT>(shl, lhs, rhs);
+  }
+  template <typename NativeT,
+            typename std::enable_if<
+                std::is_integral<NativeT>::value &&
+                !std::is_same<NativeT, bool>::value>::type* = nullptr>
+  Status HandleShiftRightArithmetic(HloInstruction* shr, HloInstruction* lhs,
+                                    HloInstruction* rhs) {
+    typedef typename std::make_signed<NativeT>::type SignedT;
+    TF_ASSIGN_OR_RETURN(
+        parent_->evaluated_[shr],
+        ElementWiseBinaryOp(shr, [](NativeT lhs_elem, NativeT rhs_elem) {
+          return static_cast<NativeT>(static_cast<SignedT>(lhs_elem) >>
+                                      rhs_elem);
+        }));
+    return Status::OK();
+  }
+
+  template <typename NativeT,
+            typename std::enable_if<!std::is_integral<NativeT>::value ||
+                                    std::is_same<NativeT, bool>::value>::type* =
+                nullptr>
+  Status HandleShiftRightArithmetic(HloInstruction* shr, HloInstruction* lhs,
+                                    HloInstruction* rhs) {
+    return InvalidArgument("Unsupported type for ShiftRightArithmetic");
+  }
+
+  Status HandleShiftRightArithmetic(HloInstruction* shra, HloInstruction* lhs,
+                                    HloInstruction* rhs) override {
+    return HandleShiftRightArithmetic<ReturnT>(shra, lhs, rhs);
+  }
+
+  template <typename NativeT,
+            typename std::enable_if<
+                std::is_integral<NativeT>::value &&
+                !std::is_same<NativeT, bool>::value>::type* = nullptr>
+  Status HandleShiftRightLogical(HloInstruction* shr, HloInstruction* lhs,
+                                 HloInstruction* rhs) {
+    typedef typename std::make_unsigned<NativeT>::type UnsignedT;
+    TF_ASSIGN_OR_RETURN(
+        parent_->evaluated_[shr],
+        ElementWiseBinaryOp(shr, [](NativeT lhs_elem, NativeT rhs_elem) {
+          return static_cast<NativeT>(static_cast<UnsignedT>(lhs_elem) >>
+                                      rhs_elem);
+        }));
+    return Status::OK();
+  }
+
+  template <typename NativeT,
+            typename std::enable_if<!std::is_integral<NativeT>::value ||
+                                    std::is_same<NativeT, bool>::value>::type* =
+                nullptr>
+  Status HandleShiftRightLogical(HloInstruction* shr, HloInstruction* lhs,
+                                 HloInstruction* rhs) {
+    return InvalidArgument("Unsupported type for ShiftRightLogical");
+  }
+
+  Status HandleShiftRightLogical(HloInstruction* shrl, HloInstruction* lhs,
+                                 HloInstruction* rhs) override {
+    return HandleShiftRightLogical<ReturnT>(shrl, lhs, rhs);
+  }
+
   Status HandleClamp(HloInstruction* clamp, HloInstruction* min,
                      HloInstruction* arg, HloInstruction* max) override {
     std::function<ReturnT(ReturnT, ReturnT, ReturnT)> clamp_op =
diff --git a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
index 20fc85c0e9..24e390529e 100644
--- a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
+++ b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
@@ -789,6 +789,9 @@ ColorScheme HloDotDumper::GetInstructionColor(const HloInstruction* instr) {
     case HloOpcode::kPower:
     case HloOpcode::kRemainder:
     case HloOpcode::kSelect:
+    case HloOpcode::kShiftLeft:
+    case HloOpcode::kShiftRightArithmetic:
+    case HloOpcode::kShiftRightLogical:
     case HloOpcode::kSign:
     case HloOpcode::kSin:
     case HloOpcode::kSlice:
diff --git a/tensorflow/compiler/xla/service/hlo_instruction.cc b/tensorflow/compiler/xla/service/hlo_instruction.cc
index b18280552d..72f4d0715d 100644
--- a/tensorflow/compiler/xla/service/hlo_instruction.cc
+++ b/tensorflow/compiler/xla/service/hlo_instruction.cc
@@ -163,6 +163,9 @@ HloInstruction::CreateGetTupleElement(const Shape& shape,
     case (HloOpcode::kSubtract):
     case (HloOpcode::kAnd):
     case (HloOpcode::kOr):
+    case (HloOpcode::kShiftLeft):
+    case (HloOpcode::kShiftRightArithmetic):
+    case (HloOpcode::kShiftRightLogical):
       break;
     default:
       LOG(FATAL) << "Invalid binary instruction opcode "
@@ -905,6 +908,9 @@ std::unique_ptr<HloInstruction> HloInstruction::CloneWithNewOperands(
     case HloOpcode::kRemainder:
     case HloOpcode::kAnd:
     case HloOpcode::kOr:
+    case HloOpcode::kShiftLeft:
+    case HloOpcode::kShiftRightArithmetic:
+    case HloOpcode::kShiftRightLogical:
       CHECK_EQ(new_operands.size(), 2);
       return CreateBinary(shape, opcode_, new_operands[0], new_operands[1]);
     // Ternary ops.
@@ -1293,6 +1299,9 @@ bool HloInstruction::IdenticalSlowPath(
     case HloOpcode::kPower:
     case HloOpcode::kRemainder:
     case HloOpcode::kSelect:
+    case HloOpcode::kShiftLeft:
+    case HloOpcode::kShiftRightArithmetic:
+    case HloOpcode::kShiftRightLogical:
     case HloOpcode::kSign:
     case HloOpcode::kSin:
     case HloOpcode::kSubtract:
@@ -1984,6 +1993,13 @@ Status HloInstruction::Visit(DfsHloVisitor* visitor) {
       return visitor->HandleAnd(this, operands_[0], operands_[1]);
     case HloOpcode::kOr:
       return visitor->HandleOr(this, operands_[0], operands_[1]);
+    case HloOpcode::kShiftLeft:
+      return visitor->HandleShiftLeft(this, operands_[0], operands_[1]);
+    case HloOpcode::kShiftRightArithmetic:
+      return visitor->HandleShiftRightArithmetic(this, operands_[0],
+                                                 operands_[1]);
+    case HloOpcode::kShiftRightLogical:
+      return visitor->HandleShiftRightLogical(this, operands_[0], operands_[1]);
     case HloOpcode::kConcatenate:
       return visitor->HandleConcatenate(this, operands_);
     case HloOpcode::kConvert:
@@ -2344,6 +2360,9 @@ bool HloInstruction::IsElementwiseBinary() const {
     case HloOpcode::kSubtract:
     case HloOpcode::kAnd:
     case HloOpcode::kOr:
+    case HloOpcode::kShiftLeft:
+    case HloOpcode::kShiftRightArithmetic:
+    case HloOpcode::kShiftRightLogical:
       return true;
     default:
       return false;
@@ -2393,6 +2412,9 @@ bool HloInstruction::IsElementwise() const {
     case HloOpcode::kSubtract:
     case HloOpcode::kAnd:
     case HloOpcode::kOr:
+    case HloOpcode::kShiftLeft:
+    case HloOpcode::kShiftRightArithmetic:
+    case HloOpcode::kShiftRightLogical:
       return true;
 
     // Ternary elementwise operations.
diff --git a/tensorflow/compiler/xla/service/hlo_matchers.h b/tensorflow/compiler/xla/service/hlo_matchers.h
index ab5e5463fa..d1ae5f776d 100644
--- a/tensorflow/compiler/xla/service/hlo_matchers.h
+++ b/tensorflow/compiler/xla/service/hlo_matchers.h
@@ -104,6 +104,9 @@ HLO_MATCHER(Rng);
 HLO_MATCHER(Select);
 HLO_MATCHER(SelectAndScatter);
 HLO_MATCHER(Send);
+HLO_MATCHER(ShiftLeft);
+HLO_MATCHER(ShiftRightLogical);
+HLO_MATCHER(ShiftRightArithmetic);
 HLO_MATCHER(Sign);
 HLO_MATCHER(Slice);
 HLO_MATCHER(Sort);
diff --git a/tensorflow/compiler/xla/service/hlo_opcode.cc b/tensorflow/compiler/xla/service/hlo_opcode.cc
index d3d78f4a99..e98012ec0c 100644
--- a/tensorflow/compiler/xla/service/hlo_opcode.cc
+++ b/tensorflow/compiler/xla/service/hlo_opcode.cc
@@ -147,6 +147,12 @@ string HloOpcodeString(HloOpcode opcode) {
       return "select";
     case HloOpcode::kSend:
       return "send";
+    case HloOpcode::kShiftLeft:
+      return "shift-left";
+    case HloOpcode::kShiftRightArithmetic:
+      return "shift-right-arithmetic";
+    case HloOpcode::kShiftRightLogical:
+      return "shift-right-logical";
     case HloOpcode::kSign:
       return "sign";
     case HloOpcode::kSin:
diff --git a/tensorflow/compiler/xla/service/hlo_opcode.h b/tensorflow/compiler/xla/service/hlo_opcode.h
index 9c26f360fb..057d4f6ea7 100644
--- a/tensorflow/compiler/xla/service/hlo_opcode.h
+++ b/tensorflow/compiler/xla/service/hlo_opcode.h
@@ -88,6 +88,9 @@ enum class HloOpcode {
   kSelect,
   kSelectAndScatter,
   kSend,
+  kShiftLeft,
+  kShiftRightArithmetic,
+  kShiftRightLogical,
   kSign,
   kSin,
   kSlice,
diff --git a/tensorflow/compiler/xla/service/instruction_fusion.cc b/tensorflow/compiler/xla/service/instruction_fusion.cc
index e08e4e4d69..7e46d79ba4 100644
--- a/tensorflow/compiler/xla/service/instruction_fusion.cc
+++ b/tensorflow/compiler/xla/service/instruction_fusion.cc
@@ -69,6 +69,9 @@ namespace xla {
     case HloOpcode::kReverse:
     case HloOpcode::kRoundNearestAfz:
     case HloOpcode::kSelect:
+    case HloOpcode::kShiftLeft:
+    case HloOpcode::kShiftRightArithmetic:
+    case HloOpcode::kShiftRightLogical:
     case HloOpcode::kSign:
     case HloOpcode::kSin:
     case HloOpcode::kSlice:
diff --git a/tensorflow/compiler/xla/service/shape_inference.cc b/tensorflow/compiler/xla/service/shape_inference.cc
index a091a067c1..f3c8e3aff3 100644
--- a/tensorflow/compiler/xla/service/shape_inference.cc
+++ b/tensorflow/compiler/xla/service/shape_inference.cc
@@ -117,6 +117,12 @@ BinaryOperation OpcodeToBinaryOperation(HloOpcode opcode) {
       return BINOP_OR;
     case HloOpcode::kAnd:
       return BINOP_AND;
+    case HloOpcode::kShiftLeft:
+      return BINOP_SHIFT_LEFT;
+    case HloOpcode::kShiftRightArithmetic:
+      return BINOP_SHIFT_RIGHT_ARITHMETIC;
+    case HloOpcode::kShiftRightLogical:
+      return BINOP_SHIFT_RIGHT_LOGICAL;
     default:
       LOG(FATAL) << "unhandled opcode " << opcode;
   }
@@ -748,6 +754,9 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(
     case BINOP_DIV:
     case BINOP_REM:
     case BINOP_MUL:
+    case BINOP_SHIFT_LEFT:
+    case BINOP_SHIFT_RIGHT_ARITHMETIC:
+    case BINOP_SHIFT_RIGHT_LOGICAL:
       return InferElementwiseBinaryOpShape(operation, lhs, rhs,
                                            broadcast_dimensions);
 
diff --git a/tensorflow/compiler/xla/service/user_computation.cc b/tensorflow/compiler/xla/service/user_computation.cc
index 317817d022..b3506b72bf 100644
--- a/tensorflow/compiler/xla/service/user_computation.cc
+++ b/tensorflow/compiler/xla/service/user_computation.cc
@@ -115,6 +115,12 @@ HloOpcode BinaryOperationToHloOpcode(BinaryOperation binop) {
       return HloOpcode::kOr;
     case BINOP_AND:
       return HloOpcode::kAnd;
+    case BINOP_SHIFT_LEFT:
+      return HloOpcode::kShiftLeft;
+    case BINOP_SHIFT_RIGHT_ARITHMETIC:
+      return HloOpcode::kShiftRightArithmetic;
+    case BINOP_SHIFT_RIGHT_LOGICAL:
+      return HloOpcode::kShiftRightLogical;
     default:
       LOG(FATAL) << "unhandled operation " << binop;
   }
diff --git a/tensorflow/compiler/xla/tests/array_elementwise_ops_test.cc b/tensorflow/compiler/xla/tests/array_elementwise_ops_test.cc
index eb931dcff3..a62b13e04f 100644
--- a/tensorflow/compiler/xla/tests/array_elementwise_ops_test.cc
+++ b/tensorflow/compiler/xla/tests/array_elementwise_ops_test.cc
@@ -739,6 +739,72 @@ XLA_TEST_F(ArrayElementwiseOpTest, NotZeroElementU32R1) {
   ComputeAndCompareR1<uint32>(&builder, {}, {});
 }
 
+XLA_TEST_F(ArrayElementwiseOpTest, ShiftLeftS32) {
+  ComputationBuilder builder(client_, TestName());
+  auto a =
+      builder.ConstantR1<int32>({static_cast<int32>(0x12345678),
+                                 static_cast<int32>(0xF0001000), 1, 3, 77});
+  auto b = builder.ConstantR1<int32>({4, 8, 2, 7, 15});
+  auto out = builder.ShiftLeft(a, b);
+
+  ComputeAndCompareR1<int32>(
+      &builder,
+      {static_cast<int32>(0x23456780), 0x00100000, 0x4, 0x180, 2523136}, {});
+}
+
+XLA_TEST_F(ArrayElementwiseOpTest, ShiftRightArithmeticS32) {
+  ComputationBuilder builder(client_, TestName());
+  auto a =
+      builder.ConstantR1<int32>({static_cast<int32>(0x92345678),
+                                 static_cast<int32>(0x10001000), 1, 3, 77});
+  auto b = builder.ConstantR1<int32>({4, 8, 2, 7, 2});
+  auto out = builder.ShiftRightArithmetic(a, b);
+
+  ComputeAndCompareR1<int32>(&builder,
+                             {static_cast<int32>(0xF9234567),
+                              static_cast<int32>(0x00100010), 0, 0, 19},
+                             {});
+}
+
+XLA_TEST_F(ArrayElementwiseOpTest, ShiftRightLogicalS32) {
+  ComputationBuilder builder(client_, TestName());
+  auto a =
+      builder.ConstantR1<int32>({static_cast<int32>(0x92345678),
+                                 static_cast<int32>(0x10001000), 1, 3, 77});
+  auto b = builder.ConstantR1<int32>({4, 8, 2, 7, 5});
+  auto out = builder.ShiftRightLogical(a, b);
+
+  ComputeAndCompareR1<int32>(&builder, {0x09234567, 0x00100010, 0, 0, 2}, {});
+}
+
+XLA_TEST_F(ArrayElementwiseOpTest, ShiftLeftU32) {
+  ComputationBuilder builder(client_, TestName());
+  auto a = builder.ConstantR1<uint32>({0x12345678, 0xF0001000, 1, 3, 77});
+  auto b = builder.ConstantR1<uint32>({4, 8, 2, 7, 15});
+  auto out = builder.ShiftLeft(a, b);
+
+  ComputeAndCompareR1<uint32>(
+      &builder, {0x23456780, 0x00100000, 0x4, 0x180, 2523136}, {});
+}
+
+XLA_TEST_F(ArrayElementwiseOpTest, ShiftRightArithmeticU32) {
+  ComputationBuilder builder(client_, TestName());
+  auto a = builder.ConstantR1<uint32>({0x92345678, 0x10001000, 1, 3, 77});
+  auto b = builder.ConstantR1<uint32>({4, 8, 2, 7, 2});
+  auto out = builder.ShiftRightArithmetic(a, b);
+
+  ComputeAndCompareR1<uint32>(&builder, {0xF9234567, 0x00100010, 0, 0, 19}, {});
+}
+
+XLA_TEST_F(ArrayElementwiseOpTest, ShiftRightLogicalU32) {
+  ComputationBuilder builder(client_, TestName());
+  auto a = builder.ConstantR1<uint32>({0x92345678, 0x10001000, 1, 3, 77});
+  auto b = builder.ConstantR1<uint32>({4, 8, 2, 7, 5});
+  auto out = builder.ShiftRightLogical(a, b);
+
+  ComputeAndCompareR1<uint32>(&builder, {0x09234567, 0x00100010, 0, 0, 2}, {});
+}
+
 XLA_TEST_F(ArrayElementwiseOpTest, CompareEqF32s) {
   SetFastMathDisabled(true);
   ComputationBuilder builder(client_, TestName());
diff --git a/tensorflow/compiler/xla/xla_data.proto b/tensorflow/compiler/xla/xla_data.proto
index 876b073b3f..0d7e583bed 100644
--- a/tensorflow/compiler/xla/xla_data.proto
+++ b/tensorflow/compiler/xla/xla_data.proto
@@ -713,6 +713,10 @@ enum BinaryOperation {
   // Logical operators
   BINOP_AND = 18;
   BINOP_OR = 19;
+
+  BINOP_SHIFT_LEFT = 20;
+  BINOP_SHIFT_RIGHT_ARITHMETIC = 21;
+  BINOP_SHIFT_RIGHT_LOGICAL = 22;
 }
 
 message BinaryOpRequest {
-- 
GitLab


From 4503f464628d4ba6f01e0e5b2aa9ff829763982b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 13 Oct 2017 08:43:51 -0700
Subject: [PATCH 354/909] Avoid cache thrashing in CTC beam search

Change the logic that identifies topK choices with a cache friendly alternative.

PiperOrigin-RevId: 172101068
---
 tensorflow/core/util/ctc/ctc_beam_search.h | 83 +++++++++++++++-------
 1 file changed, 59 insertions(+), 24 deletions(-)

diff --git a/tensorflow/core/util/ctc/ctc_beam_search.h b/tensorflow/core/util/ctc/ctc_beam_search.h
index f1773bcd95..372f25a143 100644
--- a/tensorflow/core/util/ctc/ctc_beam_search.h
+++ b/tensorflow/core/util/ctc/ctc_beam_search.h
@@ -102,6 +102,11 @@ class CTCBeamSearchDecoder : public CTCDecoder {
   template <typename Vector>
   void Step(const Vector& log_input_t);
 
+  template <typename Vector>
+  float GetTopK(const int K, const Vector& input,
+                std::vector<float>* top_k_logits,
+                std::vector<int>* top_k_indices);
+
   // Retrieve the beam scorer instance used during decoding.
   BaseBeamScorer<CTCBeamState>* GetBeamScorer() const { return beam_scorer_; }
 
@@ -202,31 +207,59 @@ Status CTCBeamSearchDecoder<CTCBeamState, CTCBeamComparer>::Decode(
   return Status::OK();
 }
 
+template <typename CTCBeamState, typename CTCBeamComparer>
+template <typename Vector>
+float CTCBeamSearchDecoder<CTCBeamState, CTCBeamComparer>::GetTopK(
+    const int K, const Vector& input, std::vector<float>* top_k_logits,
+    std::vector<int>* top_k_indices) {
+  // Find Top K choices, complexity nk in worst case. The array input is read
+  // just once.
+  CHECK_EQ(num_classes_, input.size());
+  top_k_logits->clear();
+  top_k_indices->clear();
+  top_k_logits->resize(K, -INFINITY);
+  top_k_indices->resize(K, -1);
+  for (int j = 0; j < num_classes_ - 1; ++j) {
+    const float logit = input(j);
+    if (logit > (*top_k_logits)[K - 1]) {
+      int k = K - 1;
+      while (k > 0 && logit > (*top_k_logits)[k - 1]) {
+        (*top_k_logits)[k] = (*top_k_logits)[k - 1];
+        (*top_k_indices)[k] = (*top_k_indices)[k - 1];
+        k--;
+      }
+      (*top_k_logits)[k] = logit;
+      (*top_k_indices)[k] = j;
+    }
+  }
+  // Return max value which is in 0th index or blank character logit
+  return std::max((*top_k_logits)[0], input(num_classes_ - 1));
+}
+
 template <typename CTCBeamState, typename CTCBeamComparer>
 template <typename Vector>
 void CTCBeamSearchDecoder<CTCBeamState, CTCBeamComparer>::Step(
     const Vector& raw_input) {
-  Eigen::ArrayXf input = raw_input;
-  // Remove the max for stability when performing log-prob calculations.
-  input -= input.maxCoeff();
-
-  // Minimum allowed input value for label selection:
-  float label_selection_input_min = -std::numeric_limits<float>::infinity();
-  if (label_selection_size_ > 0 && label_selection_size_ < input.size()) {
-    std::vector<float> input_copy(input.data(), input.data() + input.size());
-    std::nth_element(input_copy.begin(),
-                     input_copy.begin() + label_selection_size_ - 1,
-                     input_copy.end(), [](float a, float b) { return a > b; });
-    label_selection_input_min = input_copy[label_selection_size_ - 1];
-  }
-  if (label_selection_margin_ >= 0) {
-    // max element is 0, per normalization above
-    label_selection_input_min =
-        std::max(label_selection_input_min, -label_selection_margin_);
+  std::vector<float> top_k_logits;
+  std::vector<int> top_k_indices;
+  const bool top_k =
+      (label_selection_size_ > 0 && label_selection_size_ < raw_input.size());
+  // Number of character classes to consider in each step.
+  const int max_classes = top_k ? label_selection_size_ : (num_classes_ - 1);
+  // Get max coefficient and remove it from raw_input later.
+  float max_coeff;
+  if (top_k) {
+    max_coeff = GetTopK(label_selection_size_, raw_input, &top_k_logits,
+                        &top_k_indices);
+  } else {
+    max_coeff = raw_input.maxCoeff();
   }
+  const float label_selection_input_min =
+      (label_selection_margin_ >= 0) ? (max_coeff - label_selection_margin_)
+                                     : -std::numeric_limits<float>::infinity();
 
   // Extract the beams sorted in decreasing new probability
-  CHECK_EQ(num_classes_, input.size());
+  CHECK_EQ(num_classes_, raw_input.size());
 
   std::unique_ptr<std::vector<BeamEntry*>> branches(leaves_.Extract());
   leaves_.Reset();
@@ -252,10 +285,10 @@ void CTCBeamSearchDecoder<CTCBeamState, CTCBeamComparer>::Step(
                       beam_scorer_->GetStateExpansionScore(b->state, previous));
       }
       // Plabel(l=abc @ t=6) *= P(c @ 6)
-      b->newp.label += input(b->label);
+      b->newp.label += raw_input(b->label) - max_coeff;
     }
     // Pblank(l=abc @ t=6) = P(l=abc @ t=5) * P(- @ 6)
-    b->newp.blank = b->oldp.total + input(blank_index_);
+    b->newp.blank = b->oldp.total + raw_input(blank_index_) - max_coeff;
     // P(l=abc @ t=6) = Plabel(l=abc @ t=6) + Pblank(l=abc @ t=6)
     b->newp.total = LogSumExp(b->newp.blank, b->newp.label);
 
@@ -285,13 +318,15 @@ void CTCBeamSearchDecoder<CTCBeamState, CTCBeamComparer>::Step(
       continue;
     }
 
-    for (int ind = 0; ind < num_classes_ - 1; ind++) {
+    for (int ind = 0; ind < max_classes; ind++) {
+      const int label = top_k ? top_k_indices[ind] : ind;
+      const float logit = top_k ? top_k_logits[ind] : raw_input(ind);
       // Perform label selection: if input for this label looks very
       // unpromising, never evaluate it with a scorer.
-      if (input(ind) < label_selection_input_min) {
+      if (logit < label_selection_input_min) {
         continue;
       }
-      BeamEntry& c = b->GetChild(ind);
+      BeamEntry& c = b->GetChild(label);
       if (!c.Active()) {
         //   Pblank(l=abcd @ t=6) = 0
         c.newp.blank = kLogZero;
@@ -301,7 +336,7 @@ void CTCBeamSearchDecoder<CTCBeamState, CTCBeamComparer>::Step(
         //   Plabel(l=abcd @ t=6) = P(l=abc @ t=5) * P(d @ 6)
         beam_scorer_->ExpandState(b->state, b->label, &c.state, c.label);
         float previous = (c.label == b->label) ? b->oldp.blank : b->oldp.total;
-        c.newp.label = input(c.label) +
+        c.newp.label = logit - max_coeff +
                        beam_scorer_->GetStateExpansionScore(c.state, previous);
         // P(l=abcd @ t=6) = Plabel(l=abcd @ t=6)
         c.newp.total = c.newp.label;
-- 
GitLab


From 7061d2ad54e22fc6922d070379df22890f3b4e14 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 13 Oct 2017 09:22:01 -0700
Subject: [PATCH 355/909] Internal change.

PiperOrigin-RevId: 172105420
---
 tensorflow/BUILD   | 2 +-
 tensorflow/c/BUILD | 5 ++++-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/tensorflow/BUILD b/tensorflow/BUILD
index 9d07697d01..3868a1814b 100644
--- a/tensorflow/BUILD
+++ b/tensorflow/BUILD
@@ -307,8 +307,8 @@ config_setting(
 package_group(
     name = "internal",
     packages = [
-        "//learning/protonn/llgtm/...",
         "//tensorflow/...",
+        "//tensorflow_fold/llgtm/...",
     ],
 )
 
diff --git a/tensorflow/c/BUILD b/tensorflow/c/BUILD
index 6919dfe711..ef7eb5a4d1 100644
--- a/tensorflow/c/BUILD
+++ b/tensorflow/c/BUILD
@@ -30,7 +30,10 @@ tf_cuda_library(
     name = "c_api_internal",
     srcs = ["c_api.h"],
     hdrs = ["c_api_internal.h"],
-    visibility = ["//tensorflow/c:__subpackages__"],
+    visibility = [
+        "//tensorflow:internal",
+        "//tensorflow/c:__subpackages__",
+    ],
     deps = select({
         "//tensorflow:android": [
             "//tensorflow/core:android_tensorflow_lib_lite",
-- 
GitLab


From a0f70954f17756fd11c8032d14769353a253bb0d Mon Sep 17 00:00:00 2001
From: Jiri Simsa <jsimsa@google.com>
Date: Fri, 13 Oct 2017 09:45:50 -0700
Subject: [PATCH 356/909] adding test for heterogeneous dataset from generator

PiperOrigin-RevId: 172107872
---
 .../dataset_constructor_op_test.py             | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/tensorflow/python/kernel_tests/dataset_constructor_op_test.py b/tensorflow/python/kernel_tests/dataset_constructor_op_test.py
index 7d850cfb98..0dcce727a3 100644
--- a/tensorflow/python/kernel_tests/dataset_constructor_op_test.py
+++ b/tensorflow/python/kernel_tests/dataset_constructor_op_test.py
@@ -503,6 +503,24 @@ class DatasetConstructorTest(test.TestCase):
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
+  def testFromGeneratorHeterogeneous(self):
+    def generator():
+      yield 1
+      yield [2, 3]
+
+    iterator = (
+        dataset_ops.Dataset.from_generator(
+            generator, output_types=dtypes.int64).make_initializable_iterator())
+    init_op = iterator.initializer
+    get_next = iterator.get_next()
+
+    with self.test_session() as sess:
+      sess.run(init_op)
+      self.assertAllEqual(1, sess.run(get_next))
+      self.assertAllEqual([2, 3], sess.run(get_next))
+      with self.assertRaises(errors.OutOfRangeError):
+        sess.run(get_next)
+
   def testSplitPipelineFailsWithPlacementError(self):
     with session.Session(
         target="",
-- 
GitLab


From aea2a316fc63a3bf922ed48b844c4f254a53449c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 13 Oct 2017 09:48:51 -0700
Subject: [PATCH 357/909] Direct users of the old KMeans estimator to use the
 new one that's built from the core Estimator API.

PiperOrigin-RevId: 172108321
---
 .../learn/python/learn/estimators/kmeans.py   | 20 +++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/learn/python/learn/estimators/kmeans.py b/tensorflow/contrib/learn/python/learn/estimators/kmeans.py
index a92302420f..992b804f59 100644
--- a/tensorflow/contrib/learn/python/learn/estimators/kmeans.py
+++ b/tensorflow/contrib/learn/python/learn/estimators/kmeans.py
@@ -12,7 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Implementation of k-means clustering on top of `Estimator` API."""
+"""Implementation of k-means clustering on top of `Estimator` API.
+
+This module is deprecated. Please use
+@{tf.contrib.factorization.KMeansClustering} instead of
+@{tf.contrib.learn.KMeansClustering}. It has a similar interface, but uses the
+@{tf.estimator.Estimator} API instead of @{tf.contrib.learn.Estimator}.
+"""
 
 from __future__ import absolute_import
 from __future__ import division
@@ -29,12 +35,17 @@ from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import state_ops
-from tensorflow.python.summary import summary
 from tensorflow.python.ops.control_flow_ops import with_dependencies
 from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.summary import summary
 from tensorflow.python.training import session_run_hook
 from tensorflow.python.training.session_run_hook import SessionRunArgs
+from tensorflow.python.util.deprecation import deprecated
+
+_USE_TF_CONTRIB_FACTORIZATION = (
+    'Please use tf.contrib.factorization.KMeansClustering instead of'
+    ' tf.contrib.learn.KMeansClustering. It has a similar interface, but uses'
+    ' the tf.estimator.Estimator API instead of tf.contrib.learn.Estimator.')
 
 
 class _LossRelativeChangeHook(session_run_hook.SessionRunHook):
@@ -153,6 +164,7 @@ class KMeansClustering(estimator.Estimator):
   ALL_SCORES = 'all_scores'
   LOSS_OP_NAME = 'kmeans_loss'
 
+  @deprecated(None, _USE_TF_CONTRIB_FACTORIZATION)
   def __init__(self,
                num_clusters,
                model_dir=None,
@@ -204,6 +216,7 @@ class KMeansClustering(estimator.Estimator):
         model_dir=model_dir,
         config=config)
 
+  @deprecated(None, _USE_TF_CONTRIB_FACTORIZATION)
   def predict_cluster_idx(self, input_fn=None):
     """Yields predicted cluster indices."""
     key = KMeansClustering.CLUSTER_IDX
@@ -212,6 +225,7 @@ class KMeansClustering(estimator.Estimator):
     for result in results:
       yield result[key]
 
+  @deprecated(None, _USE_TF_CONTRIB_FACTORIZATION)
   def score(self, input_fn=None, steps=None):
     """Predict total sum of distances to nearest clusters.
 
@@ -229,6 +243,7 @@ class KMeansClustering(estimator.Estimator):
         self.evaluate(
             input_fn=input_fn, steps=steps)[KMeansClustering.SCORES])
 
+  @deprecated(None, _USE_TF_CONTRIB_FACTORIZATION)
   def transform(self, input_fn=None, as_iterable=False):
     """Transforms each element to distances to cluster centers.
 
@@ -255,6 +270,7 @@ class KMeansClustering(estimator.Estimator):
     else:
       return results
 
+  @deprecated(None, _USE_TF_CONTRIB_FACTORIZATION)
   def clusters(self):
     """Returns cluster centers."""
     return super(KMeansClustering, self).get_variable_value(self.CLUSTERS)
-- 
GitLab


From 23cce3c457bb85f69d8a013a6d75bb41a5ea01e5 Mon Sep 17 00:00:00 2001
From: David Norman <DavidNorman@users.noreply.github.com>
Date: Fri, 13 Oct 2017 18:08:05 +0100
Subject: [PATCH 358/909] [XLA] Reorder the parameters in a map inline
 operation according to the parameter number (#13579)

* Reorder the parameters according to the actual parameter number

* Adding test for inliner changes
---
 tensorflow/compiler/xla/service/inliner.cc    |  6 ++-
 .../compiler/xla/service/inliner_test.cc      | 39 +++++++++++++++++++
 2 files changed, 44 insertions(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xla/service/inliner.cc b/tensorflow/compiler/xla/service/inliner.cc
index 0682434bfb..6ea0f127d5 100644
--- a/tensorflow/compiler/xla/service/inliner.cc
+++ b/tensorflow/compiler/xla/service/inliner.cc
@@ -90,8 +90,12 @@ Status InlinerVisitor::HandleMap(
     // different than the map shape. Hence, a broadcast is needed, else the
     // cloned operand with new shape and operands work.
     if (root.opcode() != HloOpcode::kConstant) {
+      std::vector<HloInstruction*> params;
+      for (int64 o = 0; o < root.operands().size(); o++) {
+        params.push_back(operands[root.operand(o)->parameter_number()]);
+      }
       HloInstruction* placed_instruction = computation_->AddInstruction(
-          root.CloneWithNewOperands(map->shape(), operands));
+          root.CloneWithNewOperands(map->shape(), params));
       TF_RETURN_IF_ERROR(
           computation_->ReplaceInstruction(map, placed_instruction));
     } else {
diff --git a/tensorflow/compiler/xla/service/inliner_test.cc b/tensorflow/compiler/xla/service/inliner_test.cc
index 9d845c5545..7aa1c7c835 100644
--- a/tensorflow/compiler/xla/service/inliner_test.cc
+++ b/tensorflow/compiler/xla/service/inliner_test.cc
@@ -108,5 +108,44 @@ TEST_F(InlinerTest, MapConstant) {
   LiteralTestUtil::ExpectEqual(*result, *expected);
 }
 
+TEST_F(InlinerTest, MapSubtractOppositeOrder) {
+  Shape r0f32 = ShapeUtil::MakeShape(F32, {});
+
+  // Note that the parameter ordinals are in the opposite order to their
+  // position as operands
+  auto max_builder = HloComputation::Builder(TestName());
+  auto param1 = max_builder.AddInstruction(
+          HloInstruction::CreateParameter(1, r0f32, "x"));
+  auto param2 = max_builder.AddInstruction(
+          HloInstruction::CreateParameter(0, r0f32, "y"));
+  max_builder.AddInstruction(HloInstruction::CreateBinary(
+          param1->shape(), HloOpcode::kSubtract, param1, param2));
+  auto max_f32 = max_builder.Build();
+
+  auto builder = HloComputation::Builder("MapSubFunction");
+  auto lhs = builder.AddInstruction(
+    HloInstruction::CreateConstant(Literal::CreateR1<float>({1, 2, 3, 4})));
+  auto rhs = builder.AddInstruction(
+    HloInstruction::CreateConstant(Literal::CreateR1<float>({4, 3, 2, 1})));
+  builder.AddInstruction(
+    HloInstruction::CreateMap(lhs->shape(), {lhs, rhs}, max_f32.get()));
+
+  auto computation = builder.Build();
+  auto hlo_module = CreateNewModule();
+  hlo_module->AddEmbeddedComputation(std::move(max_f32));
+  hlo_module->AddEntryComputation(std::move(computation));
+
+  Inliner inliner;
+  EXPECT_TRUE(inliner.Run(hlo_module.get()).ValueOrDie());
+  EXPECT_THAT(hlo_module->entry_computation()->root_instruction(),
+          op::Subtract(rhs, lhs));
+
+  // Verify execution on CPU.
+  auto result = ExecuteAndTransfer(std::move(hlo_module), {});
+  auto expected = Literal::CreateR1<float>({3, 1, -1, -3});
+  LiteralTestUtil::ExpectEqual(*result, *expected);
+}
+
+
 }  // namespace
 }  // namespace xla
-- 
GitLab


From 6025f064bca43be5a08f23338635b4beb588670b Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Fri, 13 Oct 2017 10:08:55 -0700
Subject: [PATCH 359/909] Improve shape inference for `tf.slice` (#13561)

* Improve shape inference for `tf.slice`

This fix is an effort to address the issue raised by 4590 where
improvement of shape inference for `tf.slice` is needed.

When one of the size element is unknwon, the output shape is completely
unknwon (with right rank):
```
>>> z = tf.zeros((1, 2, 3))
>>> z.get_shape().as_list()
[1, 2, 3]
>>> m = tf.slice(z, [0, 0, 0], [tf.constant(1) + 0, 2, -1])
>>> m.get_shape().as_list()
[None, None, None]
```

This fix improves the shape inference so that:
```python
>>> import tensorflow as tf
>>> z = tf.zeros((1, 2, 3))
>>> m = tf.slice(z, [0, 0, 0], [tf.constant(1) + 0, 2, -1])
>>> m.get_shape().as_list()
[None, 2, None]
```

Note: this fix does not handle the case where one of the size element is
`-1` and one of the size element is unknown. However, it is an improvement
nevertheless.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Add test cases for partial shape inference for `tf.slice`

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Address review feedback

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Inline `SliceHelper` for simplication

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/core/ops/array_ops.cc              | 32 ++++++++++++++++---
 .../python/kernel_tests/slice_op_test.py      | 11 +++++++
 2 files changed, 39 insertions(+), 4 deletions(-)

diff --git a/tensorflow/core/ops/array_ops.cc b/tensorflow/core/ops/array_ops.cc
index a17e56b9a0..15b09c2c16 100644
--- a/tensorflow/core/ops/array_ops.cc
+++ b/tensorflow/core/ops/array_ops.cc
@@ -2283,6 +2283,8 @@ size(t) ==> 12
 
 namespace {
 
+// This SliceHelper processes the output shape of the `slice`
+// when the tensor of `sizes` is available.
 template <typename T>
 Status SliceHelper(InferenceContext* c, ShapeHandle begin_value,
                    const Tensor* sizes_value,
@@ -2308,7 +2310,6 @@ Status SliceHelper(InferenceContext* c, ShapeHandle begin_value,
 
   return Status::OK();
 }
-
 }  // namespace
 
 // --------------------------------------------------------------------------
@@ -2339,9 +2340,10 @@ REGISTER_OP("Slice")
       ShapeHandle begin_value;
       TF_RETURN_IF_ERROR(c->MakeShapeFromShapeTensor(1, &begin_value));
 
-      // NOTE(mrry): We can't use `MakeShapeFromShapeTensor` for `sizes` because
-      // it might contain -1, which can't be represented (-1 in the ShapeHandle
-      // would mean "unknown".
+      // We check the tensor value here and will only use
+      // `MakeShapeFromShapeTensor` when `sizes_value` is null.
+      // The reason is that `sizes`might contain -1, which can't
+      // be represented (-1 in the ShapeHandle would mean "unknown".
       const Tensor* sizes_value = c->input_tensor(2);
 
       if (sizes_value != nullptr) {
@@ -2361,6 +2363,28 @@ REGISTER_OP("Slice")
         c->set_output(0, c->MakeShape(dims));
         return Status::OK();
       } else {
+        // In case `sizes` is not available (`sizes_value` is null),
+        // we could try to use `MakeShapeFromShapeTensor` here.
+        // If sizes contain -1, we will simply consider it as `Unknown`.
+        // This is less than ideal but still an improvement of shape inference.
+        // The following is an example that returns [None, 1, None] with this
+        // code path:
+        //   z = tf.zeros((1, 2, 3))
+        //   m = tf.slice(z, [0, 0, 0], [tf.constant(1) + 0, 1, -1])
+        //   m.get_shape().as_list()
+        ShapeHandle sizes_value;
+        TF_RETURN_IF_ERROR(c->MakeShapeFromShapeTensor(2, &sizes_value));
+        if (c->RankKnown(sizes_value)) {
+          TF_RETURN_IF_ERROR(
+              c->WithRank(begin_value, c->Rank(sizes_value), &begin_value));
+          std::vector<DimensionHandle> dims;
+          for (int i = 0; i < c->Rank(sizes_value); ++i) {
+            dims.emplace_back(c->Dim(sizes_value, i));
+          }
+          c->set_output(0, c->MakeShape(dims));
+          return Status::OK();
+        }
+
         // We might know the rank of the input.
         if (c->RankKnown(input)) {
           c->set_output(0, c->UnknownShapeOfRank(c->Rank(input)));
diff --git a/tensorflow/python/kernel_tests/slice_op_test.py b/tensorflow/python/kernel_tests/slice_op_test.py
index f6997e9c61..f415d9e70d 100644
--- a/tensorflow/python/kernel_tests/slice_op_test.py
+++ b/tensorflow/python/kernel_tests/slice_op_test.py
@@ -217,6 +217,17 @@ class SliceTest(test.TestCase):
     self.assertEqual(expected_val.shape, slice_t.get_shape())
     self.assertEqual(expected_val.shape, slice2_t.get_shape())
 
+  def testPartialShapeInference(self):
+    z = array_ops.zeros((1, 2, 3))
+    self.assertAllEqual(z.get_shape().as_list(), [1, 2, 3])
+
+    m1 = array_ops.slice(z, [0, 0, 0], [-1, -1, -1])
+    self.assertAllEqual(m1.get_shape().as_list(), [1, 2, 3])
+
+    m2 = array_ops.slice(z, [0, 0, 0], [constant_op.constant(1) + 0, 2, -1])
+    self.assertAllEqual(m2.get_shape().as_list(), [None, 2, None])
+
+
   def _testGradientSlice(self, input_shape, slice_begin, slice_size):
     with self.test_session(use_gpu=True):
       num_inputs = np.prod(input_shape)
-- 
GitLab


From bf842104c998e598a9843b425ecebef14b2f67b6 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 13 Oct 2017 10:29:22 -0700
Subject: [PATCH 360/909] Removing custom CROSSTOOL and BUILD file for TF cpu
 builds as they are redundant now that bazel-toolchains repo is live

PiperOrigin-RevId: 172113861
---
 third_party/toolchains/cpus/BUILD     |  82 ---
 third_party/toolchains/cpus/CROSSTOOL | 918 --------------------------
 2 files changed, 1000 deletions(-)
 delete mode 100644 third_party/toolchains/cpus/BUILD
 delete mode 100644 third_party/toolchains/cpus/CROSSTOOL

diff --git a/third_party/toolchains/cpus/BUILD b/third_party/toolchains/cpus/BUILD
deleted file mode 100644
index 45ec9f8c87..0000000000
--- a/third_party/toolchains/cpus/BUILD
+++ /dev/null
@@ -1,82 +0,0 @@
-# A build file to configure cc toolchain for CPU build used with Bazel remote
-# execution service
-# DO NOT EDIT: automatically generated BUILD file
-
-licenses(["restricted"])
-
-package(default_visibility = ["//visibility:public"])
-
-cc_library(
-    name = "malloc",
-)
-
-cc_library(
-    name = "stl",
-)
-
-filegroup(
-    name = "empty",
-    srcs = [],
-)
-
-filegroup(
-    name = "cc_wrapper",
-    srcs = ["cc_wrapper.sh"],
-)
-
-# This is the entry point for --crosstool_top.  Toolchains are found
-# by lopping off the name of --crosstool_top and searching for
-# the "${CPU}" entry in the toolchains attribute.
-cc_toolchain_suite(
-    name = "toolchain",
-    toolchains = {
-        "k8|clang": ":cc-compiler-k8",
-        "armeabi-v7a|compiler": ":cc-compiler-armeabi-v7a",
-        "ios_x86_64|compiler": ":cc-compiler-ios_x86_64",
-    },
-)
-
-cc_toolchain(
-    name = "cc-compiler-k8",
-    all_files = ":empty",
-    compiler_files = ":empty",
-    cpu = "k8",
-    dwp_files = ":empty",
-    dynamic_runtime_libs = [":empty"],
-    linker_files = ":empty",
-    objcopy_files = ":empty",
-    static_runtime_libs = [":empty"],
-    strip_files = ":empty",
-    supports_param_files = 1,
-)
-
-# Android tooling requires a default toolchain for the armeabi-v7a cpu.
-cc_toolchain(
-    name = "cc-compiler-armeabi-v7a",
-    all_files = ":empty",
-    compiler_files = ":empty",
-    cpu = "local",
-    dwp_files = ":empty",
-    dynamic_runtime_libs = [":empty"],
-    linker_files = ":empty",
-    objcopy_files = ":empty",
-    static_runtime_libs = [":empty"],
-    strip_files = ":empty",
-    supports_param_files = 1,
-)
-
-# ios crosstool configuration requires a default toolchain for the
-# ios_x86_64 cpu.
-cc_toolchain(
-    name = "cc-compiler-ios_x86_64",
-    all_files = ":empty",
-    compiler_files = ":empty",
-    cpu = "local",
-    dwp_files = ":empty",
-    dynamic_runtime_libs = [":empty"],
-    linker_files = ":empty",
-    objcopy_files = ":empty",
-    static_runtime_libs = [":empty"],
-    strip_files = ":empty",
-    supports_param_files = 1,
-)
diff --git a/third_party/toolchains/cpus/CROSSTOOL b/third_party/toolchains/cpus/CROSSTOOL
deleted file mode 100644
index 66039c2135..0000000000
--- a/third_party/toolchains/cpus/CROSSTOOL
+++ /dev/null
@@ -1,918 +0,0 @@
-# A crosstool configuration for CPU build used with Bazel remote
-# execution service
-# DO NOT EDIT: automatically generated file
-
-major_version: "local"
-minor_version: ""
-default_target_cpu: "same_as_host"
-
-default_toolchain {
-  cpu: "k8"
-  toolchain_identifier: "linux_gnu_x86"
-}
-
-default_toolchain {
-  cpu: "armeabi-v7a"
-  toolchain_identifier: "stub_armeabi-v7a"
-}
-
-default_toolchain {
-  cpu: "x64_windows_msvc"
-  toolchain_identifier: "msvc_x64"
-}
-
-default_toolchain {
-  cpu: "x64_windows_msys"
-  toolchain_identifier: "msys_x64"
-}
-
-default_toolchain {
-  cpu: "s390x"
-  toolchain_identifier: "linux_gnu_x86"
-}
-
-default_toolchain {
-  cpu: "ios_x86_64"
-  toolchain_identifier: "ios_x86_64"
-}
-
-# Android tooling requires a default toolchain for the armeabi-v7a cpu.
-toolchain {
-  abi_version: "armeabi-v7a"
-  abi_libc_version: "armeabi-v7a"
-  builtin_sysroot: ""
-  compiler: "compiler"
-  host_system_name: "armeabi-v7a"
-  needsPic: true
-  supports_gold_linker: false
-  supports_incremental_linker: false
-  supports_fission: false
-  supports_interface_shared_objects: false
-  supports_normalizing_ar: false
-  supports_start_end_lib: false
-  target_libc: "armeabi-v7a"
-  target_cpu: "armeabi-v7a"
-  target_system_name: "armeabi-v7a"
-  toolchain_identifier: "stub_armeabi-v7a"
-
-  tool_path { name: "ar" path: "/bin/false" }
-  tool_path { name: "compat-ld" path: "/bin/false" }
-  tool_path { name: "cpp" path: "/bin/false" }
-  tool_path { name: "dwp" path: "/bin/false" }
-  tool_path { name: "gcc" path: "/bin/false" }
-  tool_path { name: "gcov" path: "/bin/false" }
-  tool_path { name: "ld" path: "/bin/false" }
-
-  tool_path { name: "nm" path: "/bin/false" }
-  tool_path { name: "objcopy" path: "/bin/false" }
-  tool_path { name: "objdump" path: "/bin/false" }
-  tool_path { name: "strip" path: "/bin/false" }
-  linking_mode_flags { mode: DYNAMIC }
-}
-
-toolchain {
-  toolchain_identifier: "ios_x86_64"
-  host_system_name: "x86_64-apple-macosx"
-  target_system_name: "x86_64-apple-ios"
-  target_cpu: "ios_x86_64"
-  target_libc: "ios"
-  compiler: "compiler"
-  abi_version: "local"
-  abi_libc_version: "local"
-  supports_gold_linker: false
-  supports_incremental_linker: false
-  supports_fission: false
-  supports_interface_shared_objects: false
-  supports_normalizing_ar: false
-  supports_start_end_lib: false
-
-  tool_path { name: "ar" path: "/bin/false" }
-  tool_path { name: "compat-ld" path: "/bin/false" }
-  tool_path { name: "cpp" path: "/bin/false" }
-  tool_path { name: "dwp" path: "/bin/false" }
-  tool_path { name: "gcc" path: "/bin/false" }
-  tool_path { name: "gcov" path: "/bin/false" }
-  tool_path { name: "ld" path: "/bin/false" }
-
-  tool_path { name: "nm" path: "/bin/false" }
-  tool_path { name: "objcopy" path: "/bin/false" }
-  tool_path { name: "objdump" path: "/bin/false" }
-  tool_path { name: "strip" path: "/bin/false" }
-  linking_mode_flags { mode: DYNAMIC }
-}
-
-toolchain {
-  toolchain_identifier: "linux_gnu_x86"
-  abi_version: "clang"
-  abi_libc_version: "glibc_2.19"
-  builtin_sysroot: ""
-  compiler: "clang"
-  host_system_name: "i686-unknown-linux-gnu"
-  needsPic: true
-  supports_gold_linker: true
-  supports_incremental_linker: false
-  supports_fission: false
-  supports_interface_shared_objects: false
-  supports_normalizing_ar: false
-  supports_start_end_lib: true
-  target_libc: "glibc_2.19"
-  target_cpu: "k8"
-  target_system_name: "x86_64-unknown-linux-gnu"
-  cxx_flag: "-std=c++0x"
-  linker_flag: "-lstdc++"
-  linker_flag: "-lm"
-  linker_flag: "-fuse-ld=gold"
-  linker_flag: "-B/usr/local/bin"
-  linker_flag: "-B/usr/bin"
-  cxx_builtin_include_directory: "/usr/include/c++/4.9"
-  cxx_builtin_include_directory: "/usr/include/x86_64-linux-gnu/c++/4.9"
-  cxx_builtin_include_directory: "/usr/include/c++/4.9/backward"
-  cxx_builtin_include_directory: "/usr/local/include"
-  cxx_builtin_include_directory: "/usr/local/lib/clang/5.0.0/include"
-  cxx_builtin_include_directory: "/usr/include/x86_64-linux-gnu"
-  cxx_builtin_include_directory: "/usr/include"
-  objcopy_embed_flag: "-I"
-  objcopy_embed_flag: "binary"
-  unfiltered_cxx_flag: "-Wno-builtin-macro-redefined"
-  unfiltered_cxx_flag: "-D__DATE__=\"redacted\""
-  unfiltered_cxx_flag: "-D__TIMESTAMP__=\"redacted\""
-  unfiltered_cxx_flag: "-D__TIME__=\"redacted\""
-  compiler_flag: "-U_FORTIFY_SOURCE"
-  compiler_flag: "-fstack-protector"
-  compiler_flag: "-Wall"
-  compiler_flag: "-B/usr/local/bin"
-  compiler_flag: "-B/usr/bin"
-  compiler_flag: "-fcolor-diagnostics"
-  compiler_flag: "-fno-omit-frame-pointer"
-  tool_path {name: "ld" path: "/usr/bin/ld" }
-  tool_path {name: "cpp" path: "/usr/bin/cpp" }
-  tool_path {name: "dwp" path: "/usr/bin/dwp" }
-  tool_path {name: "gcov" path: "/usr/bin/gcov" }
-  tool_path {name: "nm" path: "/usr/bin/nm" }
-  tool_path {name: "objcopy" path: "/usr/bin/objcopy" }
-  tool_path {name: "objdump" path: "/usr/bin/objdump" }
-  tool_path {name: "strip" path: "/usr/bin/strip" }
-  tool_path {name: "gcc" path: "/usr/local/bin/clang" }
-  tool_path {name: "ar" path: "/usr/bin/ar" }
-
-  compilation_mode_flags {
-    mode: DBG
-    compiler_flag: "-g"
-  }
-  compilation_mode_flags {
-    mode: OPT
-    compiler_flag: "-g0"
-    compiler_flag: "-O2"
-    compiler_flag: "-D_FORTIFY_SOURCE=1"
-    compiler_flag: "-DNDEBUG"
-    compiler_flag: "-ffunction-sections"
-    compiler_flag: "-fdata-sections"
-    linker_flag: "-Wl,--gc-sections"
-  }
-  linking_mode_flags { mode: DYNAMIC }
-
-
-    feature {
-      name: 'coverage'
-      provides: 'profile'
-      flag_set {
-        action: 'preprocess-assemble'
-        action: 'c-compile'
-        action: 'c++-compile'
-        action: 'c++-header-parsing'
-        action: 'c++-header-preprocessing'
-        action: 'c++-module-compile'
-        flag_group {
-          flag: '-fprofile-arcs'
-          flag: '-ftest-coverage'
-      }
-
-
-
-      }
-      flag_set {
-        action: 'c++-link-interface-dynamic-library'
-        action: 'c++-link-dynamic-library'
-        action: 'c++-link-executable'
-        flag_group {
-          flag: '-lgcov'
-      }
-      }
-    }
-}
-
-toolchain {
-  toolchain_identifier: "msvc_x64"
-  host_system_name: "local"
-  target_system_name: "local"
-
-  abi_version: "local"
-  abi_libc_version: "local"
-  target_cpu: "x64_windows"
-  compiler: "cl"
-  target_libc: "msvcrt140"
-  default_python_version: "python2.7"
-
-
-
-  tool_path {
-    name: "ar"
-    path: ""
-  }
-  tool_path {
-    name: "cpp"
-    path: ""
-  }
-  tool_path {
-    name: "gcc"
-    path: ""
-  }
-  tool_path {
-    name: "gcov"
-    path: "wrapper/bin/msvc_nop.bat"
-  }
-  tool_path {
-    name: "ld"
-    path: ""
-  }
-  tool_path {
-    name: "nm"
-    path: "wrapper/bin/msvc_nop.bat"
-  }
-  tool_path {
-    name: "objcopy"
-    path: "wrapper/bin/msvc_nop.bat"
-  }
-  tool_path {
-    name: "objdump"
-    path: "wrapper/bin/msvc_nop.bat"
-  }
-  tool_path {
-    name: "strip"
-    path: "wrapper/bin/msvc_nop.bat"
-  }
-  supports_gold_linker: false
-  supports_start_end_lib: false
-  supports_interface_shared_objects: false
-  supports_incremental_linker: false
-  supports_normalizing_ar: true
-  needsPic: false
-
-  # TODO(pcloudy): Review those flags below, they should be defined by cl.exe
-  compiler_flag: "/DOS_WINDOWS=OS_WINDOWS"
-  compiler_flag: "/DCOMPILER_MSVC"
-
-  # Don't pollute with GDI macros in windows.h.
-  compiler_flag: "/DNOGDI"
-  # Don't define min/max macros in windows.h.
-  compiler_flag: "/DNOMINMAX"
-  compiler_flag: "/DPRAGMA_SUPPORTED"
-  # Platform defines.
-  compiler_flag: "/D_WIN32_WINNT=0x0600"
-  # Turn off warning messages.
-  compiler_flag: "/D_CRT_SECURE_NO_DEPRECATE"
-  compiler_flag: "/D_CRT_SECURE_NO_WARNINGS"
-  compiler_flag: "/D_SILENCE_STDEXT_HASH_DEPRECATION_WARNINGS"
-  # Use math constants (M_PI, etc.) from the math library
-  compiler_flag: "/D_USE_MATH_DEFINES"
-
-  # Useful options to have on for compilation.
-  # Increase the capacity of object files to 2^32 sections.
-  compiler_flag: "/bigobj"
-  # Allocate 500MB for precomputed headers.
-  compiler_flag: "/Zm500"
-  # Use unsigned char by default.
-  compiler_flag: "/J"
-  # Use function level linking.
-  compiler_flag: "/Gy"
-  # Use string pooling.
-  compiler_flag: "/GF"
-  # Catch both asynchronous (structured) and synchronous (C++) exceptions.
-  compiler_flag: "/EHsc"
-
-  # Globally disabled warnings.
-  # Don't warn about elements of array being be default initialized.
-  compiler_flag: "/wd4351"
-  # Don't warn about no matching delete found.
-  compiler_flag: "/wd4291"
-  # Don't warn about diamond inheritance patterns.
-  compiler_flag: "/wd4250"
-  # Don't warn about insecure functions (e.g. non _s functions).
-  compiler_flag: "/wd4996"
-
-  linker_flag: "/MACHINE:X64"
-
-  linker_flag: "/SUBSYSTEM:CONSOLE"
-
-  feature {
-    name: "no_legacy_features"
-  }
-
-  # Suppress startup banner.
-  feature {
-    name: "nologo"
-    flag_set {
-      action: "c-compile"
-      action: "c++-compile"
-      action: "c++-module-compile"
-      action: "c++-module-codegen"
-      action: "c++-header-parsing"
-      action: "c++-header-preprocessing"
-      action: "assemble"
-      action: "preprocess-assemble"
-      action: "c++-link-executable"
-      action: "c++-link-dynamic-library"
-      action: "c++-link-static-library"
-      action: "c++-link-alwayslink-static-library"
-      action: "c++-link-pic-static-library"
-      action: "c++-link-alwayslink-pic-static-library"
-      flag_group {
-        flag: "/nologo"
-      }
-    }
-  }
-
-  feature {
-    name: "msvc_env"
-    env_set {
-      action: "c-compile"
-      action: "c++-compile"
-      action: "c++-module-compile"
-      action: "c++-module-codegen"
-      action: "c++-header-parsing"
-      action: "c++-header-preprocessing"
-      action: "assemble"
-      action: "preprocess-assemble"
-      action: "c++-link-executable"
-      action: "c++-link-dynamic-library"
-      action: "c++-link-static-library"
-      action: "c++-link-alwayslink-static-library"
-      action: "c++-link-pic-static-library"
-      action: "c++-link-alwayslink-pic-static-library"
-      env_entry {
-        key: "PATH"
-        value: ""
-      }
-      env_entry {
-        key: "INCLUDE"
-        value: ""
-      }
-      env_entry {
-        key: "LIB"
-        value: ""
-      }
-      env_entry {
-        key: "TMP"
-        value: ""
-      }
-    }
-  }
-
-  feature {
-    name: "use_linker"
-    env_set {
-      action: "c++-link-executable"
-      action: "c++-link-dynamic-library"
-      env_entry {
-        key: "USE_LINKER"
-        value: "1"
-      }
-    }
-  }
-
-  feature {
-    name: 'include_paths'
-    flag_set {
-      action: 'preprocess-assemble'
-      action: 'c-compile'
-      action: 'c++-compile'
-      action: 'c++-header-parsing'
-      action: 'c++-header-preprocessing'
-      action: 'c++-module-compile'
-      flag_group {
-        iterate_over: 'quote_include_paths'
-        flag: '/I%{quote_include_paths}'
-      }
-      flag_group {
-        iterate_over: 'include_paths'
-        flag: '/I%{include_paths}'
-      }
-      flag_group {
-        iterate_over: 'system_include_paths'
-        flag: '/I%{system_include_paths}'
-      }
-    }
-  }
-
-  # Stop adding any flag for dotD file, Bazel knows how to parse the output of /showIncludes option
-  # TODO(bazel-team): Remove this empty feature. https://github.com/bazelbuild/bazel/issues/2868
-  feature {
-    name: 'dependency_file'
-  }
-
-  # Tell Bazel to parse the output of /showIncludes
-  feature {
-    name: 'parse_showincludes'
-    flag_set {
-      action: 'assemble'
-      action: 'preprocess-assemble'
-      action: 'c-compile'
-      action: 'c++-compile'
-      action: 'c++-module-compile'
-      action: 'c++-header-preprocessing'
-      action: 'c++-header-parsing'
-      flag_group {
-        flag: "/showIncludes"
-      }
-    }
-  }
-
-  # Stop passing -frandom-seed option
-  feature {
-    name: 'random_seed'
-  }
-
-  # This feature is just for enabling flag_set in action_config for -c and -o options during the transitional period
-  feature {
-    name: 'compile_action_flags_in_flag_set'
-  }
-
-  action_config {
-    config_name: 'c-compile'
-    action_name: 'c-compile'
-    tool {
-      tool_path: ''
-    }
-    flag_set {
-      flag_group {
-        flag: '/c'
-        flag: '%{source_file}'
-      }
-    }
-    flag_set {
-      expand_if_all_available: 'output_object_file'
-      flag_group {
-        flag: '/Fo%{output_object_file}'
-      }
-    }
-    flag_set {
-      expand_if_all_available: 'output_assembly_file'
-      flag_group {
-        flag: '/Fa%{output_assembly_file}'
-      }
-    }
-    flag_set {
-      expand_if_all_available: 'output_preprocess_file'
-      flag_group {
-        flag: '/P'
-        flag: '/Fi%{output_preprocess_file}'
-      }
-    }
-    implies: 'nologo'
-    implies: 'msvc_env'
-    implies: 'parse_showincludes'
-  }
-
-  action_config {
-    config_name: 'c++-compile'
-    action_name: 'c++-compile'
-    tool {
-      tool_path: ''
-    }
-    flag_set {
-      flag_group {
-        flag: '/c'
-        flag: '%{source_file}'
-      }
-    }
-    flag_set {
-      expand_if_all_available: 'output_object_file'
-      flag_group {
-        flag: '/Fo%{output_object_file}'
-      }
-    }
-    flag_set {
-      expand_if_all_available: 'output_assembly_file'
-      flag_group {
-        flag: '/Fa%{output_assembly_file}'
-      }
-    }
-    flag_set {
-      expand_if_all_available: 'output_preprocess_file'
-      flag_group {
-        flag: '/P'
-        flag: '/Fi%{output_preprocess_file}'
-      }
-    }
-    implies: 'nologo'
-    implies: 'msvc_env'
-    implies: 'parse_showincludes'
-  }
-
-  action_config {
-    config_name: 'c++-link-executable'
-    action_name: 'c++-link-executable'
-    tool {
-      tool_path: ''
-    }
-    implies: 'nologo'
-    implies: 'linkstamps'
-    implies: 'output_execpath_flags'
-    implies: 'input_param_flags'
-    implies: 'legacy_link_flags'
-    implies: 'linker_param_file'
-    implies: 'msvc_env'
-    implies: 'use_linker'
-  }
-
-  action_config {
-    config_name: 'c++-link-dynamic-library'
-    action_name: 'c++-link-dynamic-library'
-    tool {
-      tool_path: ''
-    }
-    implies: 'nologo'
-    implies: 'shared_flag'
-    implies: 'linkstamps'
-    implies: 'output_execpath_flags'
-    implies: 'input_param_flags'
-    implies: 'has_configured_linker_path'
-    implies: 'legacy_link_flags'
-    implies: 'linker_param_file'
-    implies: 'msvc_env'
-    implies: 'use_linker'
-  }
-
-  action_config {
-    config_name: 'c++-link-static-library'
-    action_name: 'c++-link-static-library'
-    tool {
-      tool_path: ''
-    }
-    implies: 'nologo'
-    implies: 'archiver_flags'
-    implies: 'input_param_flags'
-    implies: 'linker_param_file'
-    implies: 'msvc_env'
-  }
-
-  action_config {
-    config_name: 'c++-link-alwayslink-static-library'
-    action_name: 'c++-link-alwayslink-static-library'
-    tool {
-      tool_path: ''
-    }
-    implies: 'nologo'
-    implies: 'archiver_flags'
-    implies: 'input_param_flags'
-    implies: 'linker_param_file'
-    implies: 'msvc_env'
-  }
-
-  # TODO(pcloudy): The following action_config is listed in MANDATORY_LINK_TARGET_TYPES.
-  # But do we really need them on Windows?
-  action_config {
-    config_name: 'c++-link-pic-static-library'
-    action_name: 'c++-link-pic-static-library'
-    tool {
-      tool_path: ''
-    }
-    implies: 'nologo'
-    implies: 'archiver_flags'
-    implies: 'input_param_flags'
-    implies: 'linker_param_file'
-    implies: 'msvc_env'
-  }
-
-  action_config {
-    config_name: 'c++-link-alwayslink-pic-static-library'
-    action_name: 'c++-link-alwayslink-pic-static-library'
-    tool {
-      tool_path: ''
-    }
-    implies: 'nologo'
-    implies: 'archiver_flags'
-    implies: 'input_param_flags'
-    implies: 'linker_param_file'
-    implies: 'msvc_env'
-  }
-
-  action_config {
-    config_name: 'c++-link-interface-dynamic-library'
-    action_name: 'c++-link-interface-dynamic-library'
-    tool {
-      tool_path: ''
-    }
-    implies: 'nologo'
-    implies: 'linker_param_file'
-    implies: 'msvc_env'
-  }
-
-  feature {
-    name: 'generate_pdb_file'
-    requires: {
-      feature: 'dbg'
-    }
-    requires: {
-      feature: 'fastbuild'
-    }
-  }
-
-  feature {
-    name: 'has_configured_linker_path'
-  }
-
-  feature {
-    name: 'shared_flag'
-    flag_set {
-      action: 'c++-link-dynamic-library'
-      flag_group {
-        flag: '/DLL'
-      }
-    }
-  }
-
-  feature {
-    name: 'linkstamps'
-    flag_set {
-      action: 'c++-link-executable'
-      action: 'c++-link-dynamic-library'
-      expand_if_all_available: 'linkstamp_paths'
-      flag_group {
-        iterate_over: 'linkstamp_paths'
-        flag: '%{linkstamp_paths}'
-      }
-    }
-  }
-
-  feature {
-    name: 'output_execpath_flags'
-    flag_set {
-      expand_if_all_available: 'output_execpath'
-      action: 'c++-link-executable'
-      action: 'c++-link-dynamic-library'
-      flag_group {
-        flag: '/OUT:%{output_execpath}'
-      }
-    }
-  }
-
-  feature {
-    name: 'archiver_flags'
-    flag_set {
-      expand_if_all_available: 'output_execpath'
-      action: 'c++-link-static-library'
-      action: 'c++-link-alwayslink-static-library'
-      action: 'c++-link-pic-static-library'
-      action: 'c++-link-alwayslink-pic-static-library'
-      flag_group {
-        flag: '/OUT:%{output_execpath}'
-      }
-    }
-  }
-
-  feature {
-    name: 'input_param_flags'
-    flag_set {
-      expand_if_all_available: 'library_search_directories'
-      action: 'c++-link-executable'
-      action: 'c++-link-dynamic-library'
-      flag_group {
-        iterate_over: 'library_search_directories'
-        flag: "-L%{library_search_directories}"
-      }
-    }
-    flag_set {
-      expand_if_all_available: 'libopts'
-      action: 'c++-link-executable'
-      action: 'c++-link-dynamic-library'
-      flag_group {
-        iterate_over: 'libopts'
-        flag: '%{libopts}'
-      }
-    }
-    flag_set {
-      expand_if_all_available: 'libraries_to_link'
-      action: 'c++-link-executable'
-      action: 'c++-link-dynamic-library'
-      action: 'c++-link-static-library'
-      action: 'c++-link-alwayslink-static-library'
-      action: 'c++-link-pic-static-library'
-      action: 'c++-link-alwayslink-pic-static-library'
-      flag_group {
-        iterate_over: 'libraries_to_link'
-        flag_group {
-          expand_if_equal: {
-            variable: 'libraries_to_link.type'
-            value: 'object_file_group'
-          }
-          iterate_over: 'libraries_to_link.object_files'
-          flag_group {
-            flag: '%{libraries_to_link.object_files}'
-          }
-        }
-        flag_group {
-          expand_if_equal: {
-            variable: 'libraries_to_link.type'
-            value: 'object_file'
-          }
-          flag_group {
-            flag: '%{libraries_to_link.name}'
-          }
-        }
-        flag_group {
-          expand_if_equal: {
-            variable: 'libraries_to_link.type'
-            value: 'interface_library'
-          }
-          flag_group {
-            expand_if_false: 'libraries_to_link.is_whole_archive'
-            flag: '%{libraries_to_link.name}'
-          }
-          flag_group {
-            expand_if_true: 'libraries_to_link.is_whole_archive'
-            flag: '/WHOLEARCHIVE:%{libraries_to_link.name}'
-          }
-        }
-        flag_group {
-          expand_if_equal: {
-            variable: 'libraries_to_link.type'
-            value: 'static_library'
-          }
-          flag_group {
-            expand_if_false: 'libraries_to_link.is_whole_archive'
-            flag: '%{libraries_to_link.name}'
-          }
-          flag_group {
-            expand_if_true: 'libraries_to_link.is_whole_archive'
-            flag: '/WHOLEARCHIVE:%{libraries_to_link.name}'
-          }
-        }
-        flag_group {
-          expand_if_equal: {
-            variable: 'libraries_to_link.type'
-            value: 'dynamic_library'
-          }
-          flag_group {
-            expand_if_false: 'libraries_to_link.is_whole_archive'
-            flag: '%{libraries_to_link.name}'
-          }
-          flag_group {
-            expand_if_true: 'libraries_to_link.is_whole_archive'
-            flag: '/WHOLEARCHIVE:%{libraries_to_link.name}'
-          }
-        }
-        flag_group {
-          expand_if_equal: {
-            variable: 'libraries_to_link.type'
-            value: 'versioned_dynamic_library'
-          }
-          flag_group {
-            expand_if_false: 'libraries_to_link.is_whole_archive'
-            flag: '%{libraries_to_link.name}'
-          }
-          flag_group {
-            expand_if_true: 'libraries_to_link.is_whole_archive'
-            flag: '/WHOLEARCHIVE:%{libraries_to_link.name}'
-          }
-        }
-      }
-    }
-  }
-
-  feature {
-    name: 'legacy_link_flags'
-    flag_set {
-      expand_if_all_available: 'legacy_link_flags'
-      action: 'c++-link-executable'
-      action: 'c++-link-dynamic-library'
-      flag_group {
-        iterate_over: 'legacy_link_flags'
-        flag: '%{legacy_link_flags}'
-      }
-    }
-  }
-
-  feature {
-    name: 'linker_param_file'
-    flag_set {
-      expand_if_all_available: 'linker_param_file'
-      action: 'c++-link-executable'
-      action: 'c++-link-dynamic-library'
-      action: 'c++-link-static-library'
-      action: 'c++-link-alwayslink-static-library'
-      action: 'c++-link-pic-static-library'
-      action: 'c++-link-alwayslink-pic-static-library'
-      flag_group {
-        flag: '@%{linker_param_file}'
-      }
-    }
-  }
-
-  feature {
-    name: 'link_crt_library'
-    flag_set {
-      action: 'c-compile'
-      action: 'c++-compile'
-      flag_group {
-        # The flag is filled by cc_configure.
-        # The default option is /MT, set USE_DYNAMIC_CRT=1 to change it to /MD
-        flag: ""
-      }
-    }
-    flag_set {
-      action: 'c++-link-executable'
-      action: 'c++-link-dynamic-library'
-      flag_group {
-      # The flag is filled by cc_configure.
-        # The default value is libcmt.lib, set USE_DYNAMIC_CRT=1 to change it to msvcrt.lib
-        flag: "/DEFAULTLIB:"
-      }
-    }
-  }
-
-  feature {
-    name: 'link_crt_debug_library'
-    flag_set {
-      action: 'c-compile'
-      action: 'c++-compile'
-      flag_group {
-        # The flag is filled by cc_configure.
-        # The default option is /MTd, set USE_DYNAMIC_CRT=1 to change it to /MDd
-        flag: ""
-      }
-    }
-    flag_set {
-      action: 'c++-link-executable'
-      action: 'c++-link-dynamic-library'
-      flag_group {
-        # The flag is filled by cc_configure.
-        # The default value is libcmtd.lib, set USE_DYNAMIC_CRT=1 to change it to msvcrtd.lib
-        flag: "/DEFAULTLIB:"
-      }
-    }
-  }
-
-  feature {
-    name: 'dbg'
-    flag_set {
-      action: 'c-compile'
-      action: 'c++-compile'
-      flag_group {
-        flag: "/Od"
-        flag: "/Z7"
-        # This will signal the wrapper that we are doing a debug build, which sets
-        # some internal state of the toolchain wrapper. It is intentionally a "-"
-        # flag to make this very obvious.
-        flag: "-g"
-      }
-    }
-    flag_set {
-      action: 'c++-link-executable'
-      action: 'c++-link-dynamic-library'
-      flag_group {
-        flag: "/DEBUG:FULL"
-        flag: "/INCREMENTAL:NO"
-      }
-    }
-    implies: 'link_crt_debug_library'
-    implies: 'generate_pdb_file'
-  }
-
-  feature {
-    name: 'fastbuild'
-    flag_set {
-      action: 'c-compile'
-      action: 'c++-compile'
-      flag_group {
-        flag: "/Od"
-        flag: "/Z7"
-      }
-    }
-    flag_set {
-      action: 'c++-link-executable'
-      action: 'c++-link-dynamic-library'
-      flag_group {
-        flag: "/DEBUG:FASTLINK"
-        flag: "/INCREMENTAL:NO"
-      }
-    }
-    implies: 'link_crt_library'
-    implies: 'generate_pdb_file'
-  }
-
-  feature {
-    name: 'opt'
-    flag_set {
-      action: 'c-compile'
-      action: 'c++-compile'
-      flag_group {
-        flag: "/O2"
-      }
-    }
-    implies: 'link_crt_library'
-  }
-
-
-
-}
-- 
GitLab


From 943feb0d3be870481f4537da53ae2b3c92b30fc0 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 13 Oct 2017 10:36:50 -0700
Subject: [PATCH 361/909] Fix a dtype

PiperOrigin-RevId: 172114960
---
 .../contrib/cudnn_rnn/python/layers/cudnn_rnn.py   | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/tensorflow/contrib/cudnn_rnn/python/layers/cudnn_rnn.py b/tensorflow/contrib/cudnn_rnn/python/layers/cudnn_rnn.py
index 810fb6450c..f6c206022c 100644
--- a/tensorflow/contrib/cudnn_rnn/python/layers/cudnn_rnn.py
+++ b/tensorflow/contrib/cudnn_rnn/python/layers/cudnn_rnn.py
@@ -191,12 +191,16 @@ class _CudnnRNN(base_layer.Layer):
         invoking __call__().
 
     Raises:
-      ValueError: if direction is invalid.
+      ValueError: if direction is invalid. Or dtype is not supported.
     """
     super(_CudnnRNN, self).__init__(dtype=dtype, name=name)
     cudnn_rnn_ops.check_direction(direction)
     cudnn_rnn_ops.check_input_mode(input_mode)
 
+    if dtype not in [dtypes.float32, dtypes.float64]:
+      raise ValueError("Only support float32, float64, provided %s" % dtype)
+    # Layer self.dtype is type name, the original DType object is kept here.
+    self._plain_dtype = dtype
     self._num_layers = num_layers
     self._num_units = num_units
     self._input_mode = input_mode
@@ -329,17 +333,17 @@ class _CudnnRNN(base_layer.Layer):
         custom_getter=self._update_trainable_weights):
       if self._kernel_initializer is None:
         self._kernel_initializer = init_ops.glorot_uniform_initializer(
-            seed=self._seed, dtype=self.dtype)
+            seed=self._seed, dtype=self._plain_dtype)
       if self._bias_initializer is None:
         self._bias_initializer = init_ops.constant_initializer(
-            0.0, dtype=self.dtype)
+            0.0, dtype=self._plain_dtype)
 
       weights = [
-          self._kernel_initializer(sp, dtype=self.dtype)
+          self._kernel_initializer(sp, dtype=self._plain_dtype)
           for sp in self.canonical_weight_shapes
       ]
       biases = [
-          self._bias_initializer(sp, dtype=self.dtype)
+          self._bias_initializer(sp, dtype=self._plain_dtype)
           for sp in self.canonical_bias_shapes
       ]
       opaque_params_t = self._canonical_to_opaque(weights, biases)
-- 
GitLab


From ba9e61255cf66355d6e75f283cb1e7c1f30ecb96 Mon Sep 17 00:00:00 2001
From: Jianwei Xie <xiejw@google.com>
Date: Fri, 13 Oct 2017 10:51:01 -0700
Subject: [PATCH 362/909] Make Per-Host input_fn deployment as default (instead
 of Per-Core)

PiperOrigin-RevId: 172117003
---
 tensorflow/contrib/tpu/python/tpu/tpu_config.py | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_config.py b/tensorflow/contrib/tpu/python/tpu/tpu_config.py
index b1d3952d1e..0a3be8503a 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_config.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_config.py
@@ -42,17 +42,16 @@ class TPUConfig(
       is invoked once on each host. To be precise, with a global batch size
       `train_batch_size` in `TPUEstimator` constructor, the batch size for each
       shard is `train_batch_size` // #hosts. With Per-Core input pipeline
-      deployment, the shard batch size is `train_batch_size` // #cores. Note:
-      This behavior is going to be default as `True` soon, so this flag will be
-      removed after that. Also note that this only works for single-host TPU
-      training now (tracked in b/67051042). For multi-host, please use Per-Core,
-      i.e., `False` for `per_host_input_for_training`.
+      deployment, the shard batch size is `train_batch_size` // #cores.  Note
+      that this only works for single-host TPU training now (tracked in
+      b/67051042). For multi-host, please use Per-Core, i.e., `False` for
+      `per_host_input_for_training`.
   """
 
   def __new__(cls,
               iterations_per_loop=2,
               num_shards=2,
-              per_host_input_for_training=False):
+              per_host_input_for_training=True):
 
     # Check iterations_per_loop.
     util_lib.check_positive_integer(iterations_per_loop,
-- 
GitLab


From aa99ddb85e54664a16d824b96418c20a9b841692 Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Fri, 13 Oct 2017 11:01:00 -0700
Subject: [PATCH 363/909] Fix the python 3 breakage caused by 172073518

PiperOrigin-RevId: 172118528
---
 .../learn/python/learn/utils/saved_model_export_utils.py        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/learn/python/learn/utils/saved_model_export_utils.py b/tensorflow/contrib/learn/python/learn/utils/saved_model_export_utils.py
index a7a1411b60..49413092a6 100644
--- a/tensorflow/contrib/learn/python/learn/utils/saved_model_export_utils.py
+++ b/tensorflow/contrib/learn/python/learn/utils/saved_model_export_utils.py
@@ -109,7 +109,7 @@ def build_standardized_signature_def(input_tensors, output_tensors,
     classes = _get_classification_classes(output_tensors)
     scores = _get_classification_scores(output_tensors)
     if classes is None and scores is None:
-      items = output_tensors.items()
+      items = list(output_tensors.items())
       if items[0][1].dtype == dtypes.string:
         (_, classes), = items
       else:
-- 
GitLab


From 2611ba3c15b61c982a47558b06ee8384f916d197 Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Fri, 13 Oct 2017 11:25:03 -0700
Subject: [PATCH 364/909] Caching for variable scopes in Layers

PiperOrigin-RevId: 172122586
---
 tensorflow/python/layers/base.py        |  20 ++++-
 tensorflow/python/ops/variable_scope.py | 102 +++++++++++++-----------
 2 files changed, 74 insertions(+), 48 deletions(-)

diff --git a/tensorflow/python/layers/base.py b/tensorflow/python/layers/base.py
index 12c7fd7ef9..99a30657ef 100644
--- a/tensorflow/python/layers/base.py
+++ b/tensorflow/python/layers/base.py
@@ -529,8 +529,22 @@ class Layer(object):
         # to __call__, hence we set previous_mask as the default value.
         kwargs['mask'] = previous_mask
 
-    with vs.variable_scope(
-        self._scope, reuse=(self.built or self._reuse)) as scope:
+    if self.built:
+      try:
+        # Some classes which inherit from Layer do not use its constructor, so
+        # rather than initializing to None we check for an AttributeError.
+        scope_context_manager = self._always_reuse_variable_scope
+      except AttributeError:
+        # From this point we will always set reuse=True, so create a "final"
+        # variable scope with this setting. We avoid re-creating variable scopes
+        # after this point as an optimization.
+        self._always_reuse_variable_scope = vs.variable_scope(
+            self._scope, reuse=True)
+        scope_context_manager = self._always_reuse_variable_scope
+    else:
+      scope_context_manager = vs.variable_scope(
+          self._scope, reuse=self._reuse)
+    with scope_context_manager as scope:
       with ops.name_scope(scope.original_name_scope):
         if not self.built:
           if not in_graph_mode:
@@ -636,7 +650,7 @@ class Layer(object):
 
   def __deepcopy__(self, memo):
     no_copy = set(['_graph'])
-    shallow_copy = set(['_scope'])
+    shallow_copy = set(['_scope', '_always_reuse_variable_scope'])
     cls = self.__class__
     result = cls.__new__(cls)
     memo[id(self)] = result
diff --git a/tensorflow/python/ops/variable_scope.py b/tensorflow/python/ops/variable_scope.py
index d0ebfdb85e..87805b5171 100644
--- a/tensorflow/python/ops/variable_scope.py
+++ b/tensorflow/python/ops/variable_scope.py
@@ -1460,37 +1460,18 @@ class _pure_variable_scope(object):  # pylint: disable=invalid-name
     self._dtype = dtype
     self._use_resource = use_resource
     self._constraint = constraint
-
-  def __enter__(self):
-    """Begins the scope block.
-
-    Returns:
-      A VariableScope.
-    Raises:
-      ValueError: when trying to reuse within a create scope, or create within
-        a reuse scope, or if reuse is not `None` or `True`.
-      TypeError: when the types of some arguments are not appropriate.
-    """
     get_variable_scope()  # Ensure that a default exists, then get a pointer.
     # Get the reference to the collection as we want to modify it in place.
     self._default_varscope = ops.get_collection_ref(_VARSCOPE_KEY)
-    self._old = self._default_varscope[0]
     self._var_store = _get_default_variable_store()
     if isinstance(self._name_or_scope, VariableScope):
       self._new_name = self._name_or_scope.name
-    else:
-      self._new_name = (
-          self._old.name + "/" + self._name_or_scope if self._old.name
-          else self._name_or_scope)
-    self._var_store.open_variable_scope(self._new_name)
-    if isinstance(self._name_or_scope, VariableScope):
-      self._old_subscopes = copy.copy(self._var_store.variable_scopes_count)
       name_scope = self._name_or_scope._name_scope  # pylint: disable=protected-access
       # Handler for the case when we jump to a shared scope.  We create a new
-      #   VariableScope (self._default_varscope[0]) that contains a copy of the
+      #   VariableScope (self._var_scope_object) that contains a copy of the
       #   provided shared scope, possibly with changed reuse and initializer, if
       #   the user requested this.
-      self._default_varscope[0] = VariableScope(
+      variable_scope_object = VariableScope(
           self._name_or_scope.reuse if not self._reuse else self._reuse,
           name=self._new_name,
           initializer=self._name_or_scope.initializer,
@@ -1503,29 +1484,48 @@ class _pure_variable_scope(object):  # pylint: disable=invalid-name
           use_resource=self._name_or_scope.use_resource,
           constraint=self._constraint)
       if self._initializer is not None:
-        self._default_varscope[0].set_initializer(self._initializer)
+        variable_scope_object.set_initializer(self._initializer)
       if self._regularizer is not None:
-        self._default_varscope[0].set_regularizer(self._regularizer)
+        variable_scope_object.set_regularizer(self._regularizer)
       if self._caching_device is not None:
-        self._default_varscope[0].set_caching_device(self._caching_device)
+        variable_scope_object.set_caching_device(self._caching_device)
       if self._partitioner is not None:
-        self._default_varscope[0].set_partitioner(self._partitioner)
+        variable_scope_object.set_partitioner(self._partitioner)
       if self._custom_getter is not None:
-        self._default_varscope[0].set_custom_getter(
+        variable_scope_object.set_custom_getter(
             _maybe_wrap_custom_getter(
                 self._custom_getter, self._name_or_scope.custom_getter))
       if self._dtype is not None:
-        self._default_varscope[0].set_dtype(self._dtype)
+        variable_scope_object.set_dtype(self._dtype)
       if self._use_resource is not None:
-        self._default_varscope[0].set_use_resource(self._use_resource)
-      return self._default_varscope[0]
+        variable_scope_object.set_use_resource(self._use_resource)
+      self._cached_variable_scope_object = variable_scope_object
+
+  def __enter__(self):
+    """Begins the scope block.
+
+    Returns:
+      A VariableScope.
+    Raises:
+      ValueError: when trying to reuse within a create scope, or create within
+        a reuse scope, or if reuse is not `None` or `True`.
+      TypeError: when the types of some arguments are not appropriate.
+    """
+    self._old = self._default_varscope[0]
+    if isinstance(self._name_or_scope, VariableScope):
+      self._var_store.open_variable_scope(self._new_name)
+      self._old_subscopes = copy.copy(self._var_store.variable_scopes_count)
+      variable_scope_object = self._cached_variable_scope_object
     else:
       # Handler for the case when we just prolong current variable scope.
       #   VariableScope with name extended by the provided one, and inherited
       #   reuse and initializer (except if the user provided values to set).
+      self._new_name = (
+          self._old.name + "/" + self._name_or_scope if self._old.name
+          else self._name_or_scope)
       self._reuse = (self._reuse
                      or self._old.reuse)  # Re-using is inherited by sub-scopes.
-      self._default_varscope[0] = VariableScope(
+      variable_scope_object = VariableScope(
           self._reuse,
           name=self._new_name,
           initializer=self._old.initializer,
@@ -1538,22 +1538,24 @@ class _pure_variable_scope(object):  # pylint: disable=invalid-name
           name_scope=self._old_name_scope or self._name_or_scope,
           constraint=self._constraint)
       if self._initializer is not None:
-        self._default_varscope[0].set_initializer(self._initializer)
+        variable_scope_object.set_initializer(self._initializer)
       if self._regularizer is not None:
-        self._default_varscope[0].set_regularizer(self._regularizer)
+        variable_scope_object.set_regularizer(self._regularizer)
       if self._caching_device is not None:
-        self._default_varscope[0].set_caching_device(self._caching_device)
+        variable_scope_object.set_caching_device(self._caching_device)
       if self._partitioner is not None:
-        self._default_varscope[0].set_partitioner(self._partitioner)
+        variable_scope_object.set_partitioner(self._partitioner)
       if self._custom_getter is not None:
-        self._default_varscope[0].set_custom_getter(
+        variable_scope_object.set_custom_getter(
             _maybe_wrap_custom_getter(self._custom_getter,
                                       self._old.custom_getter))
       if self._dtype is not None:
-        self._default_varscope[0].set_dtype(self._dtype)
+        variable_scope_object.set_dtype(self._dtype)
       if self._use_resource is not None:
-        self._default_varscope[0].set_use_resource(self._use_resource)
-      return self._default_varscope[0]
+        variable_scope_object.set_use_resource(self._use_resource)
+      self._var_store.open_variable_scope(self._new_name)
+    self._default_varscope[0] = variable_scope_object
+    return variable_scope_object
 
   def __exit__(self, type_arg, value_arg, traceback_arg):
     # If jumping out from a non-prolonged scope, restore counts.
@@ -1769,11 +1771,20 @@ class variable_scope(object):  # pylint: disable=invalid-name
     self._in_graph_mode = not context.in_eager_mode()
     if self._in_graph_mode:
       self._graph = ops._get_graph_from_inputs(self._values)  # pylint: disable=protected-access
+    self._cached_pure_variable_scope = None
+    self._current_name_scope = None
 
   def __enter__(self):
     if self._in_graph_mode:
       self._graph_context_manager = self._graph.as_default()
       self._graph_context_manager.__enter__()
+    if self._cached_pure_variable_scope is not None:
+      # Fast path for re-entering variable_scopes. We've held on to the pure
+      # variable scope from a previous __enter__, so we avoid some overhead by
+      # re-using that object.
+      if self._current_name_scope is not None:
+        self._current_name_scope.__enter__()
+      return self._cached_pure_variable_scope.__enter__()
     if self._name_or_scope is not None:
       if not isinstance(self._name_or_scope,
                         (VariableScope,) + six.string_types):
@@ -1790,7 +1801,7 @@ class variable_scope(object):  # pylint: disable=invalid-name
           old_name_scope = current_name_scope_name
         else:
           old_name_scope = self._name_or_scope.original_name_scope
-        self._pure_variable_scope = _pure_variable_scope(
+        self._cached_pure_variable_scope = _pure_variable_scope(
             self._name_or_scope,
             reuse=self._reuse,
             initializer=self._initializer,
@@ -1802,11 +1813,11 @@ class variable_scope(object):  # pylint: disable=invalid-name
             dtype=self._dtype,
             use_resource=self._use_resource,
             constraint=self._constraint)
-        return self._pure_variable_scope.__enter__()
+        return self._cached_pure_variable_scope.__enter__()
       else:
         self._current_name_scope = None
         # This can only happen if someone is entering the root variable scope.
-        self._pure_variable_scope = _pure_variable_scope(
+        self._cached_pure_variable_scope = _pure_variable_scope(
             self._name_or_scope,
             reuse=self._reuse,
             initializer=self._initializer,
@@ -1817,7 +1828,7 @@ class variable_scope(object):  # pylint: disable=invalid-name
             dtype=self._dtype,
             use_resource=self._use_resource,
             constraint=self._constraint)
-        return self._pure_variable_scope.__enter__()
+        return self._cached_pure_variable_scope.__enter__()
 
     else:  # Here name_or_scope is None. Using default name, but made unique.
       if self._reuse:
@@ -1825,7 +1836,7 @@ class variable_scope(object):  # pylint: disable=invalid-name
       self._current_name_scope = ops.name_scope(self._default_name)
       current_name_scope_name = self._current_name_scope.__enter__()
       unique_default_name = _get_unique_variable_scope(self._default_name)
-      self._pure_variable_scope = _pure_variable_scope(
+      self._cached_pure_variable_scope = _pure_variable_scope(
           unique_default_name,
           initializer=self._initializer,
           regularizer=self._regularizer,
@@ -1836,10 +1847,11 @@ class variable_scope(object):  # pylint: disable=invalid-name
           dtype=self._dtype,
           use_resource=self._use_resource,
           constraint=self._constraint)
-      return self._pure_variable_scope.__enter__()
+      return self._cached_pure_variable_scope.__enter__()
 
   def __exit__(self, type_arg, value_arg, traceback_arg):
-    self._pure_variable_scope.__exit__(type_arg, value_arg, traceback_arg)
+    self._cached_pure_variable_scope.__exit__(
+        type_arg, value_arg, traceback_arg)
     if self._current_name_scope:
       self._current_name_scope.__exit__(type_arg, value_arg, traceback_arg)
     if self._in_graph_mode:
-- 
GitLab


From b05fd283a73908eb3cefa194e2771ca4d11f6864 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 13 Oct 2017 11:38:16 -0700
Subject: [PATCH 365/909] [XLA] The shape of the clone of a tuple should be the
 same as the shape of the original.

This is true even if the layout of the tuple is weird - e.g. the subshapes of the output don't match the shape of the operands.

PiperOrigin-RevId: 172124743
---
 .../compiler/xla/service/hlo_instruction.cc     |  7 +++++--
 .../xla/service/hlo_instruction_test.cc         | 17 +++++++++++++++++
 2 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_instruction.cc b/tensorflow/compiler/xla/service/hlo_instruction.cc
index 72f4d0715d..202e7c54b1 100644
--- a/tensorflow/compiler/xla/service/hlo_instruction.cc
+++ b/tensorflow/compiler/xla/service/hlo_instruction.cc
@@ -987,8 +987,11 @@ std::unique_ptr<HloInstruction> HloInstruction::CloneWithNewOperands(
     case HloOpcode::kTranspose:
       CHECK_EQ(new_operands.size(), 1);
       return CreateTranspose(shape, new_operands[0], dimensions_);
-    case HloOpcode::kTuple:
-      return CreateTuple(new_operands);
+    case HloOpcode::kTuple: {
+      auto new_tuple = CreateTuple(new_operands);
+      *new_tuple->mutable_shape() = shape;
+      return new_tuple;
+    }
     case HloOpcode::kWhile:
       CHECK_EQ(new_operands.size(), 1);
       return CreateWhile(shape, while_condition(), while_body(),
diff --git a/tensorflow/compiler/xla/service/hlo_instruction_test.cc b/tensorflow/compiler/xla/service/hlo_instruction_test.cc
index 3601d5cdbe..45f9128eab 100644
--- a/tensorflow/compiler/xla/service/hlo_instruction_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_instruction_test.cc
@@ -729,6 +729,23 @@ TEST_F(HloInstructionTest, PreserveOutfeedShapeThroughClone) {
   EXPECT_TRUE(ShapeUtil::Equal(clone10->outfeed_shape(), shape10));
 }
 
+TEST_F(HloInstructionTest, PreserveTupleShapeThroughClone) {
+  HloComputation::Builder builder(TestName());
+  auto* constant = builder.AddInstruction(
+      HloInstruction::CreateConstant(Literal::CreateR2<float>({
+          {1, 2},
+          {3, 4},
+      })));
+  auto* tuple =
+      builder.AddInstruction(HloInstruction::CreateTuple({constant, constant}));
+  *ShapeUtil::GetMutableSubshape(tuple->mutable_shape(), {0})
+       ->mutable_layout() = LayoutUtil::MakeLayout({0, 1});
+  *ShapeUtil::GetMutableSubshape(tuple->mutable_shape(), {1})
+       ->mutable_layout() = LayoutUtil::MakeLayout({1, 0});
+  auto tuple_clone = tuple->Clone();
+  EXPECT_TRUE(ShapeUtil::Equal(tuple_clone->shape(), tuple->shape()));
+}
+
 TEST_F(HloInstructionTest, FusionOpWithCalledComputations) {
   // Create a fusion instruction containing a single unary operation.
   const Shape scalar_shape = ShapeUtil::MakeShape(F32, {});
-- 
GitLab


From 6a04ffd44758dea92120ac3c0f3bdf473c5b77d6 Mon Sep 17 00:00:00 2001
From: Kiril Gorovoy <kgorovoy@google.com>
Date: Fri, 13 Oct 2017 12:00:54 -0700
Subject: [PATCH 366/909] Fix build issue when tf/core:framework on an Android
 build.

PiperOrigin-RevId: 172127789
---
 tensorflow/cc/saved_model/BUILD | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/cc/saved_model/BUILD b/tensorflow/cc/saved_model/BUILD
index e43ff91c60..67b2e4b81a 100644
--- a/tensorflow/cc/saved_model/BUILD
+++ b/tensorflow/cc/saved_model/BUILD
@@ -54,9 +54,9 @@ cc_library(
     hdrs = ["loader.h"],
     deps = [
         ":constants",
-        "//tensorflow/core:framework",
     ] + if_not_mobile([
         "//tensorflow/core:core_cpu",
+        "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
         "//tensorflow/core:protos_all_cc",
-- 
GitLab


From 26cc81e405e0defe00ba879929ec837fa93854d3 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 13 Oct 2017 12:09:58 -0700
Subject: [PATCH 367/909] Automated g4 rollback of changelist 172041133

PiperOrigin-RevId: 172129075
---
 tensorflow/core/util/cuda_kernel_helper.h     | 15 ++--
 .../core/util/cuda_kernel_helper_test.cu.cc   | 69 -------------------
 tensorflow/tensorflow.bzl                     |  2 +-
 3 files changed, 5 insertions(+), 81 deletions(-)

diff --git a/tensorflow/core/util/cuda_kernel_helper.h b/tensorflow/core/util/cuda_kernel_helper.h
index 9e76e37898..8315f208e7 100644
--- a/tensorflow/core/util/cuda_kernel_helper.h
+++ b/tensorflow/core/util/cuda_kernel_helper.h
@@ -154,15 +154,11 @@ struct CudaLaunchConfig {
 // Calculate the Cuda launch config we should use for a kernel launch.
 // This is assuming the kernel is quite simple and will largely be
 // memory-limited.
+// REQUIRES: work_element_count > 0.
 inline CudaLaunchConfig GetCudaLaunchConfig(int work_element_count,
                                             const GPUDevice& d) {
+  CHECK_GT(work_element_count, 0);
   CudaLaunchConfig config;
-
-  // in case of invalid input, return the default value config, which has all -1
-  if (work_element_count <= 0) {
-    return config;
-  }
-
   const int virtual_thread_count = work_element_count;
   const int physical_thread_count = std::min(
       d.getNumCudaMultiProcessors() * d.maxCudaThreadsPerMultiProcessor(),
@@ -180,17 +176,14 @@ inline CudaLaunchConfig GetCudaLaunchConfig(int work_element_count,
 
 // Calculate the Cuda launch config we should use for a kernel launch. This
 // variant takes the resource limits of func into account to maximize occupancy.
+// REQUIRES: work_element_count > 0.
 template <typename DeviceFunc>
 inline CudaLaunchConfig GetCudaLaunchConfig(int work_element_count,
                                             const GPUDevice& d, DeviceFunc func,
                                             size_t dynamic_shared_memory_size,
                                             int block_size_limit) {
+  CHECK_GT(work_element_count, 0);
   CudaLaunchConfig config;
-
-  if (work_element_count <= 0) {
-    return config;
-  }
-
   int block_count = 0;
   int thread_per_block = 0;
 
diff --git a/tensorflow/core/util/cuda_kernel_helper_test.cu.cc b/tensorflow/core/util/cuda_kernel_helper_test.cu.cc
index 623f7bab90..6991554eff 100644
--- a/tensorflow/core/util/cuda_kernel_helper_test.cu.cc
+++ b/tensorflow/core/util/cuda_kernel_helper_test.cu.cc
@@ -111,28 +111,6 @@ class CudaLaunchConfigTest : public ::testing::Test {
 TEST_F(CudaLaunchConfigTest, GetCudaLaunchConfig) {
   CudaLaunchConfig cfg;
 
-  // test invalid inputs
-  CudaLaunchConfig default_value;
-  cfg = GetCudaLaunchConfig(0, d);
-  EXPECT_EQ(default_value.virtual_thread_count, cfg.virtual_thread_count);
-  EXPECT_EQ(default_value.block_count, cfg.block_count);
-  EXPECT_EQ(default_value.thread_per_block, cfg.thread_per_block);
-
-  cfg = GetCudaLaunchConfig(-1, d);
-  EXPECT_EQ(default_value.virtual_thread_count, cfg.virtual_thread_count);
-  EXPECT_EQ(default_value.block_count, cfg.block_count);
-  EXPECT_EQ(default_value.thread_per_block, cfg.thread_per_block);
-
-  cfg = GetCudaLaunchConfig(0, d, Count1D, 0, 0);
-  EXPECT_EQ(default_value.virtual_thread_count, cfg.virtual_thread_count);
-  EXPECT_EQ(default_value.block_count, cfg.block_count);
-  EXPECT_EQ(default_value.thread_per_block, cfg.thread_per_block);
-
-  cfg = GetCudaLaunchConfig(-1, d, Count1D, 0, 0);
-  EXPECT_EQ(default_value.virtual_thread_count, cfg.virtual_thread_count);
-  EXPECT_EQ(default_value.block_count, cfg.block_count);
-  EXPECT_EQ(default_value.thread_per_block, cfg.thread_per_block);
-
   // test valid inputs
   #define TEST_LAUNCH_PARAMETER(work_element_count)                             \
     cfg = GetCudaLaunchConfig(bufsize, d);                                      \
@@ -184,34 +162,6 @@ TEST_F(CudaLaunchConfigTest, GetCuda2DLaunchConfig) {
   Cuda2DLaunchConfig cfg;
   CudaLaunchConfig cfg1d;
 
-  // test invalid inputs
-  Cuda2DLaunchConfig default_value;
-  cfg = GetCuda2DLaunchConfig(1, 0, d);
-  EXPECT_EQ(default_value, cfg);
-  cfg = GetCuda2DLaunchConfig(1, -1, d);
-  EXPECT_EQ(default_value, cfg);
-  cfg = GetCuda2DLaunchConfig(-1, 1, d);
-  EXPECT_EQ(default_value, cfg);
-  cfg = GetCuda2DLaunchConfig(-1, 1, d);
-  EXPECT_EQ(default_value, cfg);
-  cfg = GetCuda2DLaunchConfig(0, -1, d);
-  EXPECT_EQ(default_value, cfg);
-  cfg = GetCuda2DLaunchConfig(0, 0, d);
-  EXPECT_EQ(default_value, cfg);
-
-  cfg = GetCuda2DLaunchConfig(1, 0, d, Count2D, 0, 0);
-  EXPECT_EQ(default_value, cfg);
-  cfg = GetCuda2DLaunchConfig(1, -1, d, Count2D, 0, 0);
-  EXPECT_EQ(default_value, cfg);
-  cfg = GetCuda2DLaunchConfig(-1, 1, d, Count2D, 0, 0);
-  EXPECT_EQ(default_value, cfg);
-  cfg = GetCuda2DLaunchConfig(-1, 1, d, Count2D, 0, 0);
-  EXPECT_EQ(default_value, cfg);
-  cfg = GetCuda2DLaunchConfig(0, -1, d, Count2D, 0, 0);
-  EXPECT_EQ(default_value, cfg);
-  cfg = GetCuda2DLaunchConfig(0, 0, d, Count2D, 0, 0);
-  EXPECT_EQ(default_value, cfg);
-
   // test valid inputs
   #define TEST_LAUNCH_PARAMETER(dimx, dimy)                                     \
     cfg1d = GetCudaLaunchConfig(bufsize, d);                                    \
@@ -252,25 +202,6 @@ TEST_F(CudaLaunchConfigTest, GetCuda3DLaunchConfig) {
   Cuda3DLaunchConfig cfg;
   CudaLaunchConfig cfg1d;
 
-  // test invalid inputs
-  Cuda3DLaunchConfig default_value;
-  cfg = GetCuda3DLaunchConfig(0, 1, 1, d, Count3D, 0, 0);
-  EXPECT_EQ(default_value, cfg);
-  cfg = GetCuda3DLaunchConfig(-1, 1, 1, d, Count3D, 0, 0);
-  EXPECT_EQ(default_value, cfg);
-  cfg = GetCuda3DLaunchConfig(1, 0, 1, d, Count3D, 0, 0);
-  EXPECT_EQ(default_value, cfg);
-  cfg = GetCuda3DLaunchConfig(1, -1, 1, d, Count3D, 0, 0);
-  EXPECT_EQ(default_value, cfg);
-  cfg = GetCuda3DLaunchConfig(1, 1, 0, d, Count3D, 0, 0);
-  EXPECT_EQ(default_value, cfg);
-  cfg = GetCuda3DLaunchConfig(1, 1, -1, d, Count3D, 0, 0);
-  EXPECT_EQ(default_value, cfg);
-  cfg = GetCuda3DLaunchConfig(0, 0, 0, d, Count3D, 0, 0);
-  EXPECT_EQ(default_value, cfg);
-  cfg = GetCuda3DLaunchConfig(-1, -1, -1, d, Count3D, 0, 0);
-  EXPECT_EQ(default_value, cfg);
-
   // test valid inputs
   #define TEST_LAUNCH_PARAMETER(dimx, dimy, dimz)                               \
     cfg1d = GetCudaLaunchConfig(bufsize, d, SetOutbufZero, 0, 0);               \
diff --git a/tensorflow/tensorflow.bzl b/tensorflow/tensorflow.bzl
index 5ec31e492c..3dd716f106 100644
--- a/tensorflow/tensorflow.bzl
+++ b/tensorflow/tensorflow.bzl
@@ -647,7 +647,7 @@ def tf_cuda_only_cc_test(name,
           clean_dep("//tensorflow:darwin"): 1,
           "//conditions:default": 0,
       }),
-      tags=tags)
+      tags=tags + tf_cuda_tests_tags())
 
 # Create a cc_test for each of the tensorflow tests listed in "tests"
 def tf_cc_tests(srcs,
-- 
GitLab


From d8168396f11ad34939819b8e866668ad375998c1 Mon Sep 17 00:00:00 2001
From: Jiri Simsa <jsimsa@google.com>
Date: Fri, 13 Oct 2017 12:18:46 -0700
Subject: [PATCH 368/909] Excluding test failing on windows with cmake.

PiperOrigin-RevId: 172130104
---
 tensorflow/contrib/cmake/tf_tests.cmake                       | 4 ++--
 tensorflow/contrib/data/python/kernel_tests/BUILD             | 2 --
 .../contrib/data/python/kernel_tests/batch_dataset_op_test.py | 2 +-
 3 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/tensorflow/contrib/cmake/tf_tests.cmake b/tensorflow/contrib/cmake/tf_tests.cmake
index 9b80cda577..a560807fb6 100644
--- a/tensorflow/contrib/cmake/tf_tests.cmake
+++ b/tensorflow/contrib/cmake/tf_tests.cmake
@@ -251,12 +251,12 @@ if (tensorflow_BUILD_PYTHON_TESTS)
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/concat_op_test.py"  # numerical issues
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/linalg_grad_test.py"  # cudaSolver handle creation fails.
 
-      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/array_ops_test.py"  # depends on python/framework/test_ops     
+      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/array_ops_test.py"  # depends on python/framework/test_ops
       # Dataset tests
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/dataset_constructor_op_test.py"
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/iterator_ops_cluster_test.py"
+      "${tensorflow_source_dir}/tensorflow/contrib/data/python/kernel_tests/batch_dataset_op_test.py" # b/67743142
       # Broken tensorboard test due to cmake issues.
-      "${tensorflow_source_dir}/tensorflow/contrib/data/python/kernel_tests/dataset_constructor_op_test.py"
       "${tensorflow_source_dir}/tensorflow/contrib/data/python/kernel_tests/iterator_ops_cluster_test.py"  # Needs portpicker
       "${tensorflow_source_dir}/tensorflow/contrib/data/python/kernel_tests/sloppy_transformation_dataset_op_test.py"  # b/65430561
       # tensor_forest tests (also note that we exclude the hybrid tests for now)
diff --git a/tensorflow/contrib/data/python/kernel_tests/BUILD b/tensorflow/contrib/data/python/kernel_tests/BUILD
index b090aac0fc..c34c9dad9b 100644
--- a/tensorflow/contrib/data/python/kernel_tests/BUILD
+++ b/tensorflow/contrib/data/python/kernel_tests/BUILD
@@ -11,7 +11,6 @@ py_test(
     size = "small",
     srcs = ["batch_dataset_op_test.py"],
     srcs_version = "PY2AND3",
-    tags = ["no_windows"],
     deps = [
         "//tensorflow/contrib/data/python/ops:dataset_ops",
         "//tensorflow/contrib/data/python/ops:transformation_ops",
@@ -25,7 +24,6 @@ py_test(
         "//tensorflow/python:string_ops",
         "//tensorflow/python:tensor_shape",
         "//tensorflow/python:util",
-        "//tensorflow/python/data/ops:dataset_ops",
         "//third_party/py/numpy",
     ],
 )
diff --git a/tensorflow/contrib/data/python/kernel_tests/batch_dataset_op_test.py b/tensorflow/contrib/data/python/kernel_tests/batch_dataset_op_test.py
index 1e7d448949..add17ff8bc 100644
--- a/tensorflow/contrib/data/python/kernel_tests/batch_dataset_op_test.py
+++ b/tensorflow/contrib/data/python/kernel_tests/batch_dataset_op_test.py
@@ -22,7 +22,7 @@ import math
 import numpy as np
 
 from tensorflow.contrib.data.python.ops import batching
-from tensorflow.python.data.ops import dataset_ops
+from tensorflow.contrib.data.python.ops import dataset_ops
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
-- 
GitLab


From 45b115151e8b1fd88c8f525fb131e7980e6c726a Mon Sep 17 00:00:00 2001
From: Chris Ying <chrisying@google.com>
Date: Fri, 13 Oct 2017 12:19:39 -0700
Subject: [PATCH 369/909] Remove unnecessary reshape and get_shape from
 fused_batch_norm.

PiperOrigin-RevId: 172130212
---
 tensorflow/python/layers/normalization.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/tensorflow/python/layers/normalization.py b/tensorflow/python/layers/normalization.py
index 4dab87b227..d82946382f 100644
--- a/tensorflow/python/layers/normalization.py
+++ b/tensorflow/python/layers/normalization.py
@@ -363,9 +363,6 @@ class BatchNormalization(base.Layer):
 
     output, mean, variance = utils.smart_cond(
         training, _fused_batch_norm_training, _fused_batch_norm_inference)
-    mean = array_ops.reshape(mean, shape=self.moving_mean.get_shape())
-    variance = array_ops.reshape(variance,
-                                 shape=self.moving_variance.get_shape())
     if not self._bessels_correction_test_only:
       # Remove Bessel's correction to be consistent with non-fused batch norm.
       # Note that the variance computed by fused batch norm is
-- 
GitLab


From 61b521679b9468b4b4ffa792a1ebc6a5ecd28317 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 13 Oct 2017 12:26:18 -0700
Subject: [PATCH 370/909] Collapse backprop implementation of LSTMBlockCell
 into one CUDA kernel. Add benchmarks for backprop. Speed difference is minor
 - will need to move everything out of the graph for large speedups, I think.

Also template the fprop kernel on use_peephole.

Original change by @duckworthd

PiperOrigin-RevId: 172131001
---
 tensorflow/contrib/rnn/kernels/lstm_ops.cc    |  56 +++++++
 tensorflow/contrib/rnn/kernels/lstm_ops.h     |  58 -------
 .../contrib/rnn/kernels/lstm_ops_gpu.cu.cc    | 156 ++++++++++++++++--
 .../rnn/python/kernel_tests/lstm_ops_test.py  |  77 ++++++++-
 4 files changed, 265 insertions(+), 82 deletions(-)

diff --git a/tensorflow/contrib/rnn/kernels/lstm_ops.cc b/tensorflow/contrib/rnn/kernels/lstm_ops.cc
index 2b56c6f95a..941a457fd3 100644
--- a/tensorflow/contrib/rnn/kernels/lstm_ops.cc
+++ b/tensorflow/contrib/rnn/kernels/lstm_ops.cc
@@ -121,6 +121,62 @@ void LSTMBlockCellFpropWithEigen(
   h.device(d) = o * co;
 }
 
+template <typename Device, typename T, bool USE_CUBLAS>
+void LSTMBlockCellBpropWithEigen(
+    const LSTMBlockCell& cell, OpKernelContext* ctx, const Device& d,
+    bool use_peephole, typename TTypes<T>::ConstMatrix x,
+    typename TTypes<T>::ConstMatrix cs_prev,
+    typename TTypes<T>::ConstMatrix h_prev, typename TTypes<T>::ConstMatrix w,
+    typename TTypes<T>::ConstVec wci, typename TTypes<T>::ConstVec wcf,
+    typename TTypes<T>::ConstVec wco, typename TTypes<T>::ConstVec b,
+    typename TTypes<T>::ConstMatrix i, typename TTypes<T>::ConstMatrix cs,
+    typename TTypes<T>::ConstMatrix f, typename TTypes<T>::ConstMatrix o,
+    typename TTypes<T>::ConstMatrix ci, typename TTypes<T>::ConstMatrix co,
+    typename TTypes<T>::ConstMatrix cs_grad,
+    typename TTypes<T>::ConstMatrix h_grad, typename TTypes<T>::Matrix do_,
+    typename TTypes<T>::Matrix dcs, typename TTypes<T>::Matrix dci,
+    typename TTypes<T>::Matrix df, typename TTypes<T>::Matrix di,
+    typename TTypes<T>::Matrix dicfo, typename TTypes<T>::Matrix cs_prev_grad,
+    typename TTypes<T>::Vec wci_grad, typename TTypes<T>::Vec wcf_grad,
+    typename TTypes<T>::Vec wco_grad) {
+  // do[t] = sigm'(o[t]) .* dh[t] .* co[t]
+  do_.device(d) = o * (o.constant(T(1)) - o) * h_grad * co;
+
+  // dcs[t] += tanh'(cs[t]) .* dh[t] .* o[t] + dcs[t + 1] .* f[t + 1]
+  dcs.device(d) = (co.constant(T(1)) - co * co) * h_grad * o + cs_grad;
+
+  Eigen::array<Eigen::DenseIndex, 2> p_shape({1, cell.cell_size()});
+  Eigen::array<Eigen::DenseIndex, 2> p_broadcast_shape({cell.batch_size(), 1});
+  if (use_peephole) {
+    dcs.device(d) =
+        dcs + do_ * wco.reshape(p_shape).broadcast(p_broadcast_shape);
+  }
+
+  // dci[t] = tanh'(ci[t]) dcs[t] i[t]
+  dci.device(d) = (ci.constant(T(1)) - ci * ci) * dcs * i;
+
+  // df[t] = sigm'(f[t]) dcs[t] cs[t - 1]
+  df.device(d) = f * (f.constant(T(1)) - f) * dcs * cs_prev;
+
+  // di[t] = sigm'(i[t]) dcs[t] ci[t]
+  di.device(d) = i * (i.constant(T(1)) - i) * dcs * ci;
+
+  dicfo.slice(cell.icfo_i_offsets(), cell.cell_extents()).device(d) = di;
+  dicfo.slice(cell.icfo_c_offsets(), cell.cell_extents()).device(d) = dci;
+  dicfo.slice(cell.icfo_f_offsets(), cell.cell_extents()).device(d) = df;
+  dicfo.slice(cell.icfo_o_offsets(), cell.cell_extents()).device(d) = do_;
+
+  cs_prev_grad.device(d) = dcs * f;
+  if (use_peephole) {
+    cs_prev_grad.device(d) =
+        cs_prev_grad + di * wci.reshape(p_shape).broadcast(p_broadcast_shape) +
+        df * wcf.reshape(p_shape).broadcast(p_broadcast_shape);
+    wci_grad.device(d) = (di * cs_prev).sum(Eigen::array<int, 1>({0}));
+    wcf_grad.device(d) = (df * cs_prev).sum(Eigen::array<int, 1>({0}));
+    wco_grad.device(d) = (do_ * cs).sum(Eigen::array<int, 1>({0}));
+  }
+}
+
 #define DEFINE_CPU_SPECS(T)                                                    \
   template <>                                                                  \
   void LSTMBlockCellFprop<CPUDevice, T, false /* USE_CUBLAS */>::operator()(   \
diff --git a/tensorflow/contrib/rnn/kernels/lstm_ops.h b/tensorflow/contrib/rnn/kernels/lstm_ops.h
index 53641ff47e..1906581b16 100644
--- a/tensorflow/contrib/rnn/kernels/lstm_ops.h
+++ b/tensorflow/contrib/rnn/kernels/lstm_ops.h
@@ -196,64 +196,6 @@ struct LSTMBlockCellBprop : public LSTMBlockCell {
       typename TTypes<T>::Vec wco_grad);
 };
 
-// TODO(b/63339763): Once GPUDevice implementation no longer relies on Eigen,
-// move into lstm_ops.cc.
-template <typename Device, typename T, bool USE_CUBLAS>
-void LSTMBlockCellBpropWithEigen(
-    const LSTMBlockCell& cell, OpKernelContext* ctx, const Device& d,
-    bool use_peephole, typename TTypes<T>::ConstMatrix x,
-    typename TTypes<T>::ConstMatrix cs_prev,
-    typename TTypes<T>::ConstMatrix h_prev, typename TTypes<T>::ConstMatrix w,
-    typename TTypes<T>::ConstVec wci, typename TTypes<T>::ConstVec wcf,
-    typename TTypes<T>::ConstVec wco, typename TTypes<T>::ConstVec b,
-    typename TTypes<T>::ConstMatrix i, typename TTypes<T>::ConstMatrix cs,
-    typename TTypes<T>::ConstMatrix f, typename TTypes<T>::ConstMatrix o,
-    typename TTypes<T>::ConstMatrix ci, typename TTypes<T>::ConstMatrix co,
-    typename TTypes<T>::ConstMatrix cs_grad,
-    typename TTypes<T>::ConstMatrix h_grad, typename TTypes<T>::Matrix do_,
-    typename TTypes<T>::Matrix dcs, typename TTypes<T>::Matrix dci,
-    typename TTypes<T>::Matrix df, typename TTypes<T>::Matrix di,
-    typename TTypes<T>::Matrix dicfo, typename TTypes<T>::Matrix cs_prev_grad,
-    typename TTypes<T>::Vec wci_grad, typename TTypes<T>::Vec wcf_grad,
-    typename TTypes<T>::Vec wco_grad) {
-  // do[t] = sigm'(o[t]) .* dh[t] .* co[t]
-  do_.device(d) = o * (o.constant(T(1)) - o) * h_grad * co;
-
-  // dcs[t] += tanh'(cs[t]) .* dh[t] .* o[t] + dcs[t + 1] .* f[t + 1]
-  dcs.device(d) = (co.constant(T(1)) - co * co) * h_grad * o + cs_grad;
-
-  Eigen::array<Eigen::DenseIndex, 2> p_shape({1, cell.cell_size()});
-  Eigen::array<Eigen::DenseIndex, 2> p_broadcast_shape({cell.batch_size(), 1});
-  if (use_peephole) {
-    dcs.device(d) =
-        dcs + do_ * wco.reshape(p_shape).broadcast(p_broadcast_shape);
-  }
-
-  // dci[t] = tanh'(ci[t]) dcs[t] i[t]
-  dci.device(d) = (ci.constant(T(1)) - ci * ci) * dcs * i;
-
-  // df[t] = sigm'(f[t]) dcs[t] cs[t - 1]
-  df.device(d) = f * (f.constant(T(1)) - f) * dcs * cs_prev;
-
-  // di[t] = sigm'(i[t]) dcs[t] ci[t]
-  di.device(d) = i * (i.constant(T(1)) - i) * dcs * ci;
-
-  dicfo.slice(cell.icfo_i_offsets(), cell.cell_extents()).device(d) = di;
-  dicfo.slice(cell.icfo_c_offsets(), cell.cell_extents()).device(d) = dci;
-  dicfo.slice(cell.icfo_f_offsets(), cell.cell_extents()).device(d) = df;
-  dicfo.slice(cell.icfo_o_offsets(), cell.cell_extents()).device(d) = do_;
-
-  cs_prev_grad.device(d) = dcs * f;
-  if (use_peephole) {
-    cs_prev_grad.device(d) =
-        cs_prev_grad + di * wci.reshape(p_shape).broadcast(p_broadcast_shape) +
-        df * wcf.reshape(p_shape).broadcast(p_broadcast_shape);
-    wci_grad.device(d) = (di * cs_prev).sum(Eigen::array<int, 1>({0}));
-    wcf_grad.device(d) = (df * cs_prev).sum(Eigen::array<int, 1>({0}));
-    wco_grad.device(d) = (do_ * cs).sum(Eigen::array<int, 1>({0}));
-  }
-}
-
 template <typename Device, typename T, bool USE_CUBLAS>
 struct BlockLSTMBprop : public LSTMBlockCell {
   BlockLSTMBprop(const int batch_size, const int input_size,
diff --git a/tensorflow/contrib/rnn/kernels/lstm_ops_gpu.cu.cc b/tensorflow/contrib/rnn/kernels/lstm_ops_gpu.cu.cc
index 90990fe452..d82676ff7e 100644
--- a/tensorflow/contrib/rnn/kernels/lstm_ops_gpu.cu.cc
+++ b/tensorflow/contrib/rnn/kernels/lstm_ops_gpu.cu.cc
@@ -39,12 +39,12 @@ namespace {
 // Launch with blocks of (batch x 32)
 //
 // TODO(b/67600500): Try making 'use_peephole' a template parameter.
-template <typename T>
+template <typename T, bool use_peephole>
 __global__ void lstm_gates(const T* icfo, const T* b, const T* cs_prev,
                            const T* wci, const T* wcf, const T* wco, T* o, T* h,
                            T* ci, T* cs, T* co, T* i, T* f, const T forget_bias,
-                           const T cell_clip, const bool use_peephole,
-                           const int batch_size, const int cell_size) {
+                           const T cell_clip, const int batch_size,
+                           const int cell_size) {
   const int batch_id = blockIdx.x * blockDim.x + threadIdx.x;
   const int act_id = blockIdx.y * blockDim.y + threadIdx.y;
 
@@ -108,7 +108,8 @@ __global__ void lstm_gates(const T* icfo, const T* b, const T* cs_prev,
   }
   i[cid] = i_local;
 
-  T ci_local = tanh_op(icfo[1 * cell_size + gid] + b[1 * cell_size + act_id]);
+  const T ci_local =
+      tanh_op(icfo[1 * cell_size + gid] + b[1 * cell_size + act_id]);
   ci[cid] = ci_local;
 
   T f_local;
@@ -127,7 +128,7 @@ __global__ void lstm_gates(const T* icfo, const T* b, const T* cs_prev,
   }
   cs[cid] = cs_local;
 
-  T co_local = tanh_op(cs_local);
+  const T co_local = tanh_op(cs_local);
   co[cid] = co_local;
 
   T o_local;
@@ -212,16 +213,141 @@ void LSTMBlockCellFpropWithCUDA(
   dim3 grid_dim_2d(Eigen::divup(batch_size, static_cast<int>(block_dim_2d.x)),
                    Eigen::divup(cell_size, static_cast<int>(block_dim_2d.y)));
 
-  lstm_gates<<<grid_dim_2d, block_dim_2d, 0, cu_stream>>>(
-      icfo.data(), b.data(), cs_prev.data(), wci.data(), wcf.data(), wco.data(),
-      o.data(), h.data(), ci.data(), cs.data(), co.data(), i.data(), f.data(),
-      forget_bias, cell_clip, use_peephole, batch_size, cell_size);
+  if (use_peephole) {
+    lstm_gates<T, true><<<grid_dim_2d, block_dim_2d, 0, cu_stream>>>(
+        icfo.data(), b.data(), cs_prev.data(), wci.data(), wcf.data(),
+        wco.data(), o.data(), h.data(), ci.data(), cs.data(), co.data(),
+        i.data(), f.data(), forget_bias, cell_clip, batch_size, cell_size);
+  } else {
+    lstm_gates<T, false><<<grid_dim_2d, block_dim_2d, 0, cu_stream>>>(
+        icfo.data(), b.data(), cs_prev.data(), wci.data(), wcf.data(),
+        wco.data(), o.data(), h.data(), ci.data(), cs.data(), co.data(),
+        i.data(), f.data(), forget_bias, cell_clip, batch_size, cell_size);
+  }
+}
+
+template <typename T>
+__global__ void lstm_gates_bprop(
+    const T* cs_prev,  // [batch_size, cell_size]
+    const T* h_prev,   // [batch_size, cell_size]
+    const T* w,        // [input_size + cell_size, 4 * cell_size]
+    const T* wci,      // [cell_size]
+    const T* wcf,      // [cell_size]
+    const T* wco,      // [cell_size]
+    const T* b,        // [4 * cell_size]
+    const T* i,        // [batch_size, cell_size]
+    const T* cs,       // [batch_size, cell_size]
+    const T* f,        // [batch_size, cell_size]
+    const T* o,        // [batch_size, cell_size]
+    const T* ci,       // [batch_size, cell_size]
+    const T* co,       // [batch_size, cell_size]
+    const T* cs_grad,  // [batch_size, cell_size]
+    const T* h_grad,   // [batch_size, cell_size]
+    T* do_,            // [batch_size, cell_size]
+    T* dcs,            // [batch_size, cell_size]
+    T* dci,            // [batch_size, cell_size]
+    T* df,             // [batch_size, cell_size]
+    T* di,             // [batch_size, cell_size]
+    T* dicfo,          // [input_size + cell_size, 4 * cell_size]
+    T* cs_prev_grad,   // [batch_size, cell_size]
+    const int batch_size, const int cell_size, const bool use_peephole) {
+  const int batch_id = blockIdx.x * blockDim.x + threadIdx.x;
+  const int act_id = blockIdx.y * blockDim.y + threadIdx.y;
+
+  if (batch_id >= batch_size || act_id >= cell_size) return;
+
+  const int gid = batch_id * cell_size * 4 + act_id;
+  const int cid = batch_id * cell_size + act_id;
+
+  const T one = static_cast<T>(1.0f);
+
+  // do[t] = sigm'(o[t]) .* dh[t] .* co[t]
+  const T o_local = o[cid];
+  const T h_grad_local = h_grad[cid];
+  const T co_local = co[cid];
+  const T ci_local = ci[cid];
+  const T do_local = o_local * (one - o_local) * h_grad_local * co_local;
+  const T i_local = i[cid];
+  const T f_local = f[cid];
+
+  do_[cid] = do_local;
+
+  // dcs[t] += tanh'(cs[t]) .* dh[t] .* o[t] + dcs[t + 1] .* f[t + 1]
+  T dcs_local =
+      (one - co_local * co_local) * h_grad_local * o_local + cs_grad[cid];
+  if (use_peephole) {
+    dcs_local += do_local * wco[act_id];
+  }
+  dcs[cid] = dcs_local;
+
+  // dci[t] = tanh'(ci[t]) dcs[t] i[t]
+  const T dci_local = (one - ci_local * ci_local) * dcs_local * i_local;
+  dci[cid] = dci_local;
+
+  // df[t] = sigm'(f[t]) dcs[t] cs[t - 1]
+  const T df_local = f_local * (one - f_local) * dcs_local * cs_prev[cid];
+  df[cid] = df_local;
+
+  // di[t] = sigm'(i[t]) dcs[t] ci[t]
+  const T di_local = i_local * (one - i_local) * dcs_local * ci_local;
+  di[cid] = di_local;
+
+  dicfo[gid + 0 * cell_size] = di_local;
+  dicfo[gid + 1 * cell_size] = dci_local;
+  dicfo[gid + 2 * cell_size] = df_local;
+  dicfo[gid + 3 * cell_size] = do_local;
+
+  cs_prev_grad[cid] = dcs_local * f_local;
+  if (use_peephole) {
+    cs_prev_grad[cid] += di_local * wci[act_id] + df_local * wcf[act_id];
+  }
+}
+
+template <typename T>
+void LSTMBlockCellBpropWithCUDA(
+    OpKernelContext* ctx, const GPUDevice& d, typename TTypes<T>::ConstMatrix x,
+    typename TTypes<T>::ConstMatrix cs_prev,
+    typename TTypes<T>::ConstMatrix h_prev, typename TTypes<T>::ConstMatrix w,
+    typename TTypes<T>::ConstVec wci, typename TTypes<T>::ConstVec wcf,
+    typename TTypes<T>::ConstVec wco, typename TTypes<T>::ConstVec b,
+    typename TTypes<T>::ConstMatrix i, typename TTypes<T>::ConstMatrix cs,
+    typename TTypes<T>::ConstMatrix f, typename TTypes<T>::ConstMatrix o,
+    typename TTypes<T>::ConstMatrix ci, typename TTypes<T>::ConstMatrix co,
+    typename TTypes<T>::ConstMatrix cs_grad,
+    typename TTypes<T>::ConstMatrix h_grad, typename TTypes<T>::Matrix do_,
+    typename TTypes<T>::Matrix dcs, typename TTypes<T>::Matrix dci,
+    typename TTypes<T>::Matrix df, typename TTypes<T>::Matrix di,
+    typename TTypes<T>::Matrix dicfo, typename TTypes<T>::Matrix cs_prev_grad,
+    typename TTypes<T>::Vec wci_grad, typename TTypes<T>::Vec wcf_grad,
+    typename TTypes<T>::Vec wco_grad, const int batch_size, const int cell_size,
+    const bool use_peephole) {
+  const cudaStream_t& cu_stream = GetCudaStream(ctx);
+
+  dim3 block_dim_2d(min(batch_size, 8), 32);
+  dim3 grid_dim_2d(Eigen::divup(batch_size, static_cast<int>(block_dim_2d.x)),
+                   Eigen::divup(cell_size, static_cast<int>(block_dim_2d.y)));
+
+  lstm_gates_bprop<<<grid_dim_2d, block_dim_2d, 0, cu_stream>>>(
+      cs_prev.data(), h_prev.data(), w.data(), wci.data(), wcf.data(),
+      wco.data(), b.data(), i.data(), cs.data(), f.data(), o.data(), ci.data(),
+      co.data(), cs_grad.data(), h_grad.data(), do_.data(), dcs.data(),
+      dci.data(), df.data(), di.data(), dicfo.data(), cs_prev_grad.data(),
+      batch_size, cell_size, use_peephole);
+
+  if (use_peephole) {
+    Eigen::array<Eigen::DenseIndex, 2> p_shape({1, cell_size});
+    Eigen::array<Eigen::DenseIndex, 2> p_broadcast_shape({batch_size, 1});
+    cs_prev_grad.device(d) =
+        cs_prev_grad + di * wci.reshape(p_shape).broadcast(p_broadcast_shape) +
+        df * wcf.reshape(p_shape).broadcast(p_broadcast_shape);
+    wci_grad.device(d) = (di * cs_prev).sum(Eigen::array<int, 1>({0}));
+    wcf_grad.device(d) = (df * cs_prev).sum(Eigen::array<int, 1>({0}));
+    wco_grad.device(d) = (do_ * cs).sum(Eigen::array<int, 1>({0}));
+  }
 }
 
 }  // namespace
 
-// TODO(b/63339763): Provide an alternative implementation for
-// LSTMBlockCellBprop that doesn't rely on Eigen.
 #define DEFINE_GPU_SPECS(T)                                                    \
   template struct TensorZero<GPUDevice, T>;                                    \
   template struct TensorUnalignedZero<GPUDevice, T>;                           \
@@ -267,10 +393,10 @@ void LSTMBlockCellFpropWithCUDA(
       typename TTypes<T>::Matrix cs_prev_grad,                                 \
       typename TTypes<T>::Vec wci_grad, typename TTypes<T>::Vec wcf_grad,      \
       typename TTypes<T>::Vec wco_grad) {                                      \
-    LSTMBlockCellBpropWithEigen<GPUDevice, T, true /* USE_CUBLAS */>(          \
-        *this, ctx, d, use_peephole, x, cs_prev, h_prev, w, wci, wcf, wco, b,  \
-        i, cs, f, o, ci, co, cs_grad, h_grad, do_, dcs, dci, df, di, dicfo,    \
-        cs_prev_grad, wci_grad, wcf_grad, wco_grad);                           \
+    LSTMBlockCellBpropWithCUDA<T>(                                             \
+        ctx, d, x, cs_prev, h_prev, w, wci, wcf, wco, b, i, cs, f, o, ci, co,  \
+        cs_grad, h_grad, do_, dcs, dci, df, di, dicfo, cs_prev_grad, wci_grad, \
+        wcf_grad, wco_grad, batch_size_, cell_size_, use_peephole);            \
   }                                                                            \
   template struct LSTMBlockCellFprop<GPUDevice, T, true /* USE_CUBLAS */>;     \
   template struct LSTMBlockCellBprop<GPUDevice, T, true /* USE_CUBLAS */>;     \
diff --git a/tensorflow/contrib/rnn/python/kernel_tests/lstm_ops_test.py b/tensorflow/contrib/rnn/python/kernel_tests/lstm_ops_test.py
index 0a474f7831..a288072ae5 100644
--- a/tensorflow/contrib/rnn/python/kernel_tests/lstm_ops_test.py
+++ b/tensorflow/contrib/rnn/python/kernel_tests/lstm_ops_test.py
@@ -458,17 +458,16 @@ class BenchmarkLSTMBlock(test.Benchmark):
     print("batch_size,cell_size,input_size,time_steps,use_gpu,wall_time")
     iters = 10
     for config in benchmarking.dict_product({
-        "batch_size": [1, 32, 128],
-        "cell_size": [32, 128, 512],
-        "input_size": [128, 512],
-        "time_steps": [10, 25, 100],
+        "batch_size": [1, 8, 13, 32, 67, 128],
+        "cell_size": [128, 250, 512, 650, 1024, 1350],
+        "time_steps": [40],
         "use_gpu": [True, False]
     }):
       with ops.Graph().as_default():
         with benchmarking.device(use_gpu=config["use_gpu"]):
-          inputs = variable_scope.get_variable("x", [
-              config["time_steps"], config["batch_size"], config["input_size"]
-          ])
+          inputs = variable_scope.get_variable(
+              "x",
+              [config["time_steps"], config["batch_size"], config["cell_size"]])
           cell = lstm_ops.LSTMBlockCell(config["cell_size"])
           outputs = rnn.dynamic_rnn(
               cell, inputs, time_major=True, dtype=dtypes.float32)
@@ -482,12 +481,72 @@ class BenchmarkLSTMBlock(test.Benchmark):
         # is set, this will produce a copy-paste-able CSV file.
         print(",".join(
             map(str, [
-                config["batch_size"], config["cell_size"], config["input_size"],
+                config["batch_size"], config["cell_size"], config["cell_size"],
                 config["time_steps"], config["use_gpu"], wall_time
             ])))
         benchmark_name_template = "_".join([
             "LSTMBlockCell_fprop", "BS%(batch_size)i", "CS%(cell_size)i",
-            "IS%(input_size)i", "TS%(time_steps)i", "gpu_%(use_gpu)s"
+            "IS%(cell_size)i", "TS%(time_steps)i", "gpu_%(use_gpu)s"
+        ])
+
+        self.report_benchmark(
+            name=benchmark_name_template % config,
+            iters=iters,
+            wall_time=wall_time,
+            extras=config)
+
+  def benchmarkLSTMBlockCellBpropWithDynamicRNN(self):
+    print("BlockLSTMCell backward propagation via dynamic_rnn().")
+    print("--------------------------------------------------------------")
+    print("LSTMBlockCell Seconds per inference.")
+    print("batch_size,cell_size,input_size,time_steps,use_gpu,wall_time")
+    iters = 10
+    for config in benchmarking.dict_product({
+        "batch_size": [1, 8, 13, 32, 67, 128],
+        "cell_size": [128, 250, 512, 650, 1024, 1350],
+        "time_steps": [40],
+        "use_gpu": [True, False]
+    }):
+      with ops.Graph().as_default():
+        with benchmarking.device(use_gpu=config["use_gpu"]):
+          time_steps = config["time_steps"]
+          batch_size = config["batch_size"]
+          cell_size = input_size = config["cell_size"]
+          inputs = variable_scope.get_variable(
+              "x", [time_steps, batch_size, cell_size],
+              trainable=False,
+              dtype=dtypes.float32)
+          with variable_scope.variable_scope(
+              "rnn", reuse=variable_scope.AUTO_REUSE):
+            w = variable_scope.get_variable(
+                "rnn/lstm_cell/kernel",
+                shape=[input_size + cell_size, cell_size * 4],
+                dtype=dtypes.float32)
+            b = variable_scope.get_variable(
+                "rnn/lstm_cell/bias",
+                shape=[cell_size * 4],
+                dtype=dtypes.float32,
+                initializer=init_ops.zeros_initializer())
+            cell = lstm_ops.LSTMBlockCell(cell_size)
+            outputs = rnn.dynamic_rnn(
+                cell, inputs, time_major=True, dtype=dtypes.float32)
+          grads = gradients_impl.gradients(outputs, [inputs, w, b])
+          init_op = variables.global_variables_initializer()
+
+        with session.Session() as sess:
+          sess.run(init_op)
+          wall_time = benchmarking.seconds_per_run(grads, sess, iters)
+
+        # Print to stdout. If the TEST_REPORT_FILE_PREFIX environment variable
+        # is set, this will produce a copy-paste-able CSV file.
+        print(",".join(
+            map(str, [
+                batch_size, cell_size, cell_size, time_steps, config["use_gpu"],
+                wall_time
+            ])))
+        benchmark_name_template = "_".join([
+            "LSTMBlockCell_bprop", "BS%(batch_size)i", "CS%(cell_size)i",
+            "IS%(cell_size)i", "TS%(time_steps)i", "gpu_%(use_gpu)s"
         ])
 
         self.report_benchmark(
-- 
GitLab


From 23afdede4726a8fcb1886c619186b863c04ffc1e Mon Sep 17 00:00:00 2001
From: Michael Case <mikecase@chromium.org>
Date: Fri, 13 Oct 2017 12:33:39 -0700
Subject: [PATCH 371/909] Merge changes from 1.4-rc0 back into master (#13670)

* Update RELEASE NOTES for TensorFlow 1.4

* Update the version strings for TF 1.4-rc0.

* Update version strings in POM files missed by update script.

* Pin TensorBoard 0.4 to TensorFlow 1.4

* Fixing the name of the disabled test. (#13592)

* Revert "Implementing ghost batch norm as defined in https://arxiv.org/pdf/1705.08741."

This reverts commit 125f7afa4a483855dc75791445d2dea64587876a.

* Disable iterator_ops_test on Windows for 1.4 release (#13609)

* Disable failing Windows tests for r1.4 release.

testRemoteIteratorUsingRemoteCallOpDirectSessionGPUCPU test is failing
with "TypeError: only integer scalar arrays can be converted to a scalar
index" on the Windows GPU Release bot. Disabling test.

* Fix typo.

* Also disalbe iterator_ops_test from contrib/.

* Add contributing authors to 1.4 Release notes.

Thanks!

* Fixes to authors.

Removed duplicate and removed googler from contributing author list.

* Fixes and additions to release notes.

Added line about Keras moving into core.
Added line about CUDA/cuDNN versions.
Added line about custom ops.

* Back out cherry-picking batch norm revert into master.
---
 RELEASE.md                                    | 81 ++++++++++++++++++-
 tensorflow/contrib/cmake/tf_tests.cmake       |  4 +
 tensorflow/core/public/version.h              |  4 +-
 tensorflow/docs_src/install/install_c.md      |  2 +-
 tensorflow/docs_src/install/install_go.md     |  2 +-
 tensorflow/docs_src/install/install_java.md   | 18 ++---
 .../docs_src/install/install_sources.md       | 10 +--
 tensorflow/java/maven/libtensorflow/pom.xml   |  2 +-
 .../java/maven/libtensorflow_jni/pom.xml      |  2 +-
 tensorflow/java/maven/pom.xml                 |  2 +-
 tensorflow/java/maven/proto/pom.xml           |  2 +-
 tensorflow/java/maven/tensorflow/pom.xml      |  2 +-
 tensorflow/tools/docker/Dockerfile.devel      |  2 +-
 tensorflow/tools/docker/Dockerfile.devel-gpu  |  2 +-
 tensorflow/tools/pip_package/setup.py         |  2 +-
 15 files changed, 109 insertions(+), 28 deletions(-)

diff --git a/RELEASE.md b/RELEASE.md
index c5f1e8b309..d30ee69f40 100644
--- a/RELEASE.md
+++ b/RELEASE.md
@@ -1,19 +1,50 @@
 # Release 1.4.0
 
 ## Major Features And Improvements
+* `tf.keras` is now part of the core TensorFlow API.
 * `tf.data` is now part of the core TensorFlow API.
   * The API is now subject to backwards compatibility guarantees.
   * For a guide to migrating from the `tf.contrib.data` API, see the
-    [README] (https://github.com/tensorflow/tensorflow/blob/r1.4/tensorflow/contrib/data/README.md).
+    [README](https://github.com/tensorflow/tensorflow/blob/r1.4/tensorflow/contrib/data/README.md).
   * Major new features include `Dataset.from_generator()` (for building an input
     pipeline from a Python generator), and the `Dataset.apply()` method for
     applying custom transformation functions.
   * Several custom transformation functions have been added, including
     `tf.contrib.data.batch_and_drop_remainder()` and
     `tf.contrib.data.sloppy_interleave()`.
+* Add `train_and_evaluate` for simple distributed `Estimator` training.
+* Add `tf.spectral.dct` for computing the DCT-II.
+* Add Mel-Frequency Cepstral Coefficient support to `tf.contrib.signal`
+  (with GPU and gradient support).
+* Add a self-check on `import tensorflow` for Windows DLL issues.
+* Add NCHW support to `tf.depth_to_space` on GPU.
+* SinhArcsinh (scalar) distribution added to `contrib.distributions`.
+* Make `GANEstimator` opensource.
+* `Estimator.export_savedmodel()` now includes all valid serving signatures
+  that can be constructed from the Serving Input Receiver and all available
+  ExportOutputs. For instance, a classifier may provide regression- and
+  prediction-flavored outputs, in addition to the classification-flavored one.
+  Building signatures from these allows TF Serving to honor requests using the
+  different APIs (Classify, Regress, and Predict). Furthermore,
+  `serving_input_receiver_fn()` may now specify alternative subsets of nodes
+  that may act as inputs. This allows, for instance, producing a prediction
+  signature for a classifier that accepts raw `Tensors` instead of a serialized
+  `tf.Example`.
+* Add `tf.contrib.bayesflow.hmc`.
+* Add `tf.contrib.distributions.MixtureSameFamily`.
+* Make `Dataset.shuffle()` always reshuffles after each iteration by default.
+* Add `tf.contrib.bayesflow.metropolis_hastings`.
+* Add `log_rate` parameter to `tf.contrib.distributions.Poisson`.
+* Extend `tf.contrib.distributions.bijector` API to handle some non-injective
+  transforms.
 * Java:
-  * Generics (e.g., `Tensor<Integer>`) for improved type-safety (courtesy @andrewcmyers).
+  * Generics (e.g., `Tensor<Integer>`) for improved type-safety
+    (courtesy @andrewcmyers).
   * Support for multi-dimensional string tensors.
+  * Support loading of custom operations (e.g. many in `tf.contrib`) on Linux
+    and OS X
+* All our prebuilt binaries have been built with CUDA 8 and cuDNN 6.
+  We anticipate releasing TensorFlow 1.5 with CUDA 9 and cuDNN 7.
 
 ## Bug Fixes and Other Changes
 * `tf.nn.rnn_cell.DropoutWrapper` is now more careful about dropping out LSTM
@@ -25,11 +56,57 @@
 * Removed `tf.contrib.training.python_input`.  The same behavior, in a more
   flexible and reproducible package, is available via the new
   `tf.contrib.data.Dataset.from_generator` method!
+* Fix `tf.contrib.distributions.Affine` incorrectly computing log-det-jacobian.
+* Fix `tf.random_gamma` incorrectly handling non-batch, scalar draws.
+* Resolved a race condition in TensorForest TreePredictionsV4Op.
+* Google Cloud Storage file system and Hadoop file system support are now
+  default build options.
+* Custom op libraries must link against libtensorflow_framework.so
+  (installed at `tf.sysconfig.get_lib()`).
 
 ## Breaking Changes to the API
 * The signature of the `tf.contrib.data.rejection_resample()` function has been
   changed. It now returns a function that can be used as an argument to
   `Dataset.apply()`.
+* Remove `tf.contrib.data.Iterator.from_dataset()` method. Use
+  `Dataset.make_initializable_iterator()` instead.
+* Remove seldom used and unnecessary `tf.contrib.data.Iterator.dispose_op()`.
+* Reorder some TFGAN loss functions in a non-backwards compatible way.
+
+## Thanks to our Contributors
+
+This release contains contributions from many people at Google, as well as:
+
+4d55397500, Abdullah Alrasheed, abenmao, Adam Salvail, Aditya Dhulipala, Ag Ramesh,
+Akimasa Kimura, Alan Du, Alan Yee, Alexander, Amit Kushwaha, Amy, Andrei Costinescu,
+Andrei Nigmatulin, Andrew Erlichson, Andrew Myers, Andrew Stepanov, Androbin, AngryPowman,
+Anish Shah, Anton Daitche, Artsiom Chapialiou, asdf2014, Aseem Raj Baranwal, Ash Hall,
+Bart Kiers, Batchu Venkat Vishal, ben, Ben Barsdell, Bill Piel, Carl Thomé, Catalin Voss,
+Changming Sun, Chengzhi Chen, Chi Zeng, Chris Antaki, Chris Donahue, Chris Oelmueller,
+Chris Tava, Clayne Robison, Codrut, Courtial Florian, Dalmo Cirne, Dan J, Darren Garvey,
+David Kristoffersson, David Norman, David RöThlisberger, DavidNorman, Dhruv, DimanNe,
+Dorokhov, Duncan Mac-Vicar P, EdwardDixon, EMCP, error.d, FAIJUL, Fan Xia,
+Francois Xavier, Fred Reiss, Freedom" Koan-Sin Tan, Fritz Obermeyer, Gao, Xiang,
+Guenther Schmuelling, Guo Yejun (郭叶军), Hans Gaiser, HectorSVC, Hyungsuk Yoon,
+James Pruegsanusak, Jay Young, Jean Wanka, Jeff Carpenter, Jeremy Rutman, Jeroen BéDorf,
+Jett Jones, Jimmy Jia, jinghuangintel, jinze1994, JKurland, Joel Hestness, joetoth,
+John B Nelson, John Impallomeni, John Lawson, Jonas, Jonathan Dekhtiar, joshkyh, Jun Luan,
+Jun Mei, Kai Sasaki, Karl Lessard, karl@kubx.ca, Kb Sriram, Kenichi Ueno, Kevin Slagle,
+Kongsea, Lakshay Garg, lhlmgr, Lin Min, liu.guangcong, Loki Der Quaeler, Louie Helm,
+lucasmoura, Luke Iwanski, Lyndon White, Mahmoud Abuzaina, Marcel Puyat, Mark Aaron Shirley,
+Michele Colombo, MtDersvan, Namrata-Ibm, Nathan Luehr, Naurril, Nayana Thorat, Nicolas Lopez,
+Niranjan Hasabnis, Nolan Liu, Nouce, Oliver Hennigh, osdamv, Patrik Erdes,
+Patryk Chrabaszcz, Pavel Christof, Penghao Cen, postBG, Qingqing Cao, Qingying Chen, qjivy,
+Raphael, Rasmi, raymondxyang, Renze Yu, resec, Roffel, Ruben Vereecken, Ryohei Kuroki,
+sandipmgiri, Santiago Castro, Scott Kirkland, Sean Vig, Sebastian Raschka, Sebastian Weiss,
+Sergey Kolesnikov, Sergii Khomenko, Shahid, Shivam Kotwalia, Stuart Berg, Sumit Gouthaman,
+superzerg, Sven Mayer, tetris, Ti Zhou, Tiago Freitas Pereira, Tian Jin, Tomoaki Oiki,
+Vaibhav Sood, vfdev, Vivek Rane, Vladimir Moskva, wangqr, Weber Xie, Will Frey,
+Yan Facai (颜发才), yanivbl6, Yaroslav Bulatov, Yixing Lao, Yong Tang, youkaichao,
+Yuan (Terry) Tang, Yue Zhang, Yuxin Wu, Ziming Dong, ZxYuan, 黄璞
+
+We are also grateful to all who filed issues or helped resolve them, asked and
+answered questions, and were part of inspiring discussions.
 
 # Release 1.3.0
 
diff --git a/tensorflow/contrib/cmake/tf_tests.cmake b/tensorflow/contrib/cmake/tf_tests.cmake
index c8e61747ed..530fcee774 100644
--- a/tensorflow/contrib/cmake/tf_tests.cmake
+++ b/tensorflow/contrib/cmake/tf_tests.cmake
@@ -230,6 +230,8 @@ if (tensorflow_BUILD_PYTHON_TESTS)
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/diag_op_test.py"
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/linalg_ops_test.py"
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/init_ops_test.py"
+      # Type error in testRemoteIteratorUsingRemoteCallOpDirectSessionGPUCPU.
+      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/iterator_ops_test.py"
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/self_adjoint_eig_op_test.py"
       # misc
       "${tensorflow_source_dir}/tensorflow/contrib/data/python/kernel_tests/batch_dataset_op_test.py"
@@ -259,6 +261,8 @@ if (tensorflow_BUILD_PYTHON_TESTS)
       "${tensorflow_source_dir}/tensorflow/contrib/data/python/kernel_tests/dataset_constructor_op_test.py"
       "${tensorflow_source_dir}/tensorflow/contrib/data/python/kernel_tests/iterator_ops_cluster_test.py"  # Needs portpicker
       "${tensorflow_source_dir}/tensorflow/contrib/data/python/kernel_tests/sloppy_transformation_dataset_op_test.py"  # b/65430561
+      # Type error in testRemoteIteratorUsingRemoteCallOpDirectSessionGPUCPU.
+      "${tensorflow_source_dir}/tensorflow/contrib/data/python/kernel_tests/iterator_ops_test.py"
       # tensor_forest tests (also note that we exclude the hybrid tests for now)
       "${tensorflow_source_dir}/tensorflow/contrib/tensor_forest/python/kernel_tests/count_extremely_random_stats_op_test.py"  # Results in wrong order.
       "${tensorflow_source_dir}/tensorflow/contrib/tensor_forest/python/kernel_tests/sample_inputs_op_test.py"  # Results in wrong order.
diff --git a/tensorflow/core/public/version.h b/tensorflow/core/public/version.h
index ccb861c93a..5d2298f7b7 100644
--- a/tensorflow/core/public/version.h
+++ b/tensorflow/core/public/version.h
@@ -19,12 +19,12 @@ limitations under the License.
 // TensorFlow uses semantic versioning, see http://semver.org/.
 
 #define TF_MAJOR_VERSION 1
-#define TF_MINOR_VERSION 3
+#define TF_MINOR_VERSION 4
 #define TF_PATCH_VERSION 0
 
 // TF_VERSION_SUFFIX is non-empty for pre-releases (e.g. "-alpha", "-alpha.1",
 // "-beta", "-rc", "-rc.1")
-#define TF_VERSION_SUFFIX ""
+#define TF_VERSION_SUFFIX "-rc0"
 
 #define TF_STR_HELPER(x) #x
 #define TF_STR(x) TF_STR_HELPER(x)
diff --git a/tensorflow/docs_src/install/install_c.md b/tensorflow/docs_src/install/install_c.md
index 7ebf5c4a2c..586bb6dead 100644
--- a/tensorflow/docs_src/install/install_c.md
+++ b/tensorflow/docs_src/install/install_c.md
@@ -35,7 +35,7 @@ enable TensorFlow for C:
          OS="linux" # Change to "darwin" for Mac OS
          TARGET_DIRECTORY="/usr/local"
          curl -L \
-           "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-${OS}-x86_64-1.3.0.tar.gz" |
+           "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-${OS}-x86_64-1.4.0-rc0.tar.gz" |
            sudo tar -C $TARGET_DIRECTORY -xz
 
      The `tar` command extracts the TensorFlow C library into the `lib`
diff --git a/tensorflow/docs_src/install/install_go.md b/tensorflow/docs_src/install/install_go.md
index b991fd0f93..1d00661d83 100644
--- a/tensorflow/docs_src/install/install_go.md
+++ b/tensorflow/docs_src/install/install_go.md
@@ -35,7 +35,7 @@ steps to install this library and enable TensorFlow for Go:
          TF_TYPE="cpu" # Change to "gpu" for GPU support
          TARGET_DIRECTORY='/usr/local'
          curl -L \
-           "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-$(go env GOOS)-x86_64-1.3.0.tar.gz" |
+           "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-$(go env GOOS)-x86_64-1.4.0-rc0.tar.gz" |
          sudo tar -C $TARGET_DIRECTORY -xz
 
      The `tar` command extracts the TensorFlow C library into the `lib`
diff --git a/tensorflow/docs_src/install/install_java.md b/tensorflow/docs_src/install/install_java.md
index 2adcd4da73..3b3acfdcb3 100644
--- a/tensorflow/docs_src/install/install_java.md
+++ b/tensorflow/docs_src/install/install_java.md
@@ -34,7 +34,7 @@ following to the project's `pom.xml` to use the TensorFlow Java APIs:
 <dependency>
   <groupId>org.tensorflow</groupId>
   <artifactId>tensorflow</artifactId>
-  <version>1.3.0</version>
+  <version>1.4.0-rc0</version>
 </dependency>
 ```
 
@@ -63,7 +63,7 @@ As an example, these steps will create a Maven project that uses TensorFlow:
                <dependency>
                  <groupId>org.tensorflow</groupId>
                  <artifactId>tensorflow</artifactId>
-                 <version>1.3.0</version>
+                 <version>1.4.0-rc0</version>
                </dependency>
              </dependencies>
          </project>
@@ -122,7 +122,7 @@ refer to the simpler instructions above instead.
 Take the following steps to install TensorFlow for Java on Linux or Mac OS:
 
   1. Download
-     [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.3.0.jar),
+     [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.4.0-rc0.jar),
      which is the TensorFlow Java Archive (JAR).
 
   2. Decide whether you will run TensorFlow for Java on CPU(s) only or with
@@ -141,7 +141,7 @@ Take the following steps to install TensorFlow for Java on Linux or Mac OS:
          OS=$(uname -s | tr '[:upper:]' '[:lower:]')
          mkdir -p ./jni
          curl -L \
-           "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-${TF_TYPE}-${OS}-x86_64-1.3.0.tar.gz" |
+           "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-${TF_TYPE}-${OS}-x86_64-1.4.0-rc0.tar.gz" |
            tar -xz -C ./jni
 
 ### Install on Windows
@@ -149,10 +149,10 @@ Take the following steps to install TensorFlow for Java on Linux or Mac OS:
 Take the following steps to install TensorFlow for Java on Windows:
 
   1. Download
-     [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.3.0.jar),
+     [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.4.0-rc0.jar),
      which is the TensorFlow Java Archive (JAR).
   2. Download the following Java Native Interface (JNI) file appropriate for
-     [TensorFlow for Java on Windows](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-cpu-windows-x86_64-1.3.0.zip).
+     [TensorFlow for Java on Windows](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-cpu-windows-x86_64-1.4.0-rc0.zip).
   3. Extract this .zip file.
 
 
@@ -200,7 +200,7 @@ must be part of your `classpath`. For example, you can include the
 downloaded `.jar` in your `classpath` by using the `-cp` compilation flag
 as follows:
 
-<pre><b>javac -cp libtensorflow-1.3.0.jar HelloTF.java</b></pre>
+<pre><b>javac -cp libtensorflow-1.4.0-rc0.jar HelloTF.java</b></pre>
 
 
 ### Running
@@ -214,11 +214,11 @@ two files are available to the JVM:
 For example, the following command line executes the `HelloTF` program on Linux
 and Mac OS X:
 
-<pre><b>java -cp libtensorflow-1.3.0.jar:. -Djava.library.path=./jni HelloTF</b></pre>
+<pre><b>java -cp libtensorflow-1.4.0-rc0.jar:. -Djava.library.path=./jni HelloTF</b></pre>
 
 And the following command line executes the `HelloTF` program on Windows:
 
-<pre><b>java -cp libtensorflow-1.3.0.jar;. -Djava.library.path=jni HelloTF</b></pre>
+<pre><b>java -cp libtensorflow-1.4.0-rc0.jar;. -Djava.library.path=jni HelloTF</b></pre>
 
 If the program prints <tt>Hello from <i>version</i></tt>, you've successfully
 installed TensorFlow for Java and are ready to use the API.  If the program
diff --git a/tensorflow/docs_src/install/install_sources.md b/tensorflow/docs_src/install/install_sources.md
index 3d143506f0..6114496cd5 100644
--- a/tensorflow/docs_src/install/install_sources.md
+++ b/tensorflow/docs_src/install/install_sources.md
@@ -441,8 +441,8 @@ Stack Overflow and specify the `tensorflow` tag.
 **Linux**
 <table>
 <tr><th>Version:</th><th>CPU/GPU:</th><th>Python Version:</th><th>Compiler:</th><th>Build Tools:</th><th>cuDNN:</th><th>CUDA:</th></tr>
-<tr><td>tensorflow-1.3.0</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>GCC 4.8</td><td>Bazel 0.4.5</td><td>N/A</td><td>N/A</td></tr>
-<tr><td>tensorflow_gpu-1.3.0</td><td>GPU</td><td>2.7, 3.3-3.6</td><td>GCC 4.8</td><td>Bazel 0.4.5</td><td>6</td><td>8</td></tr>
+<tr><td>tensorflow-1.4.0rc0</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>GCC 4.8</td><td>Bazel 0.4.5</td><td>N/A</td><td>N/A</td></tr>
+<tr><td>tensorflow_gpu-1.4.0rc0</td><td>GPU</td><td>2.7, 3.3-3.6</td><td>GCC 4.8</td><td>Bazel 0.4.5</td><td>6</td><td>8</td></tr>
 <tr><td>tensorflow-1.2.0</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>GCC 4.8</td><td>Bazel 0.4.5</td><td>N/A</td><td>N/A</td></tr>
 <tr><td>tensorflow_gpu-1.2.0</td><td>GPU</td><td>2.7, 3.3-3.6</td><td>GCC 4.8</td><td>Bazel 0.4.5</td><td>5.1</td><td>8</td></tr>
 <tr><td>tensorflow-1.1.0</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>GCC 4.8</td><td>Bazel 0.4.2</td><td>N/A</td><td>N/A</td></tr>
@@ -454,7 +454,7 @@ Stack Overflow and specify the `tensorflow` tag.
 **Mac**
 <table>
 <tr><th>Version:</th><th>CPU/GPU:</th><th>Python Version:</th><th>Compiler:</th><th>Build Tools:</th><th>cuDNN:</th><th>CUDA:</th></tr>
-<tr><td>tensorflow-1.3.0</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>Clang from xcode</td><td>Bazel 0.4.5</td><td>N/A</td><td>N/A</td></tr>
+<tr><td>tensorflow-1.4.0rc0</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>Clang from xcode</td><td>Bazel 0.4.5</td><td>N/A</td><td>N/A</td></tr>
 <tr><td>ttensorflow-1.2.0</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>Clang from xcode</td><td>Bazel 0.4.5</td><td>N/A</td><td>N/A</td></tr>
 <tr><td>ttensorflow-1.1.0</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>Clang from xcode</td><td>Bazel 0.4.2</td><td>N/A</td><td>N/A</td></tr>
 <tr><td>ttensorflow_gpu-1.1.0</td><td>GPU</td><td>2.7, 3.3-3.6</td><td>Clang from xcode</td><td>Bazel 0.4.2</td><td>5.1</td><td>8</td></tr>
@@ -465,8 +465,8 @@ Stack Overflow and specify the `tensorflow` tag.
 **Windows**
 <table>
 <tr><th>Version:</th><th>CPU/GPU:</th><th>Python Version:</th><th>Compiler:</th><th>Build Tools:</th><th>cuDNN:</th><th>CUDA:</th></tr>
-<tr><td>tensorflow-1.3.0</td><td>CPU</td><td>3.5-3.6</td><td>MSVC 2015 update 3</td><td>Cmake v3.6.3</td><td>N/A</td><td>N/A</td></tr>
-<tr><td>tensorflow_gpu-1.3.0</td><td>GPU</td><td>3.5-3.6</td><td>MSVC 2015 update 3</td><td>Cmake v3.6.3</td><td>6</td><td>8</td></tr>
+<tr><td>tensorflow-1.4.0rc0</td><td>CPU</td><td>3.5-3.6</td><td>MSVC 2015 update 3</td><td>Cmake v3.6.3</td><td>N/A</td><td>N/A</td></tr>
+<tr><td>tensorflow_gpu-1.4.0rc0</td><td>GPU</td><td>3.5-3.6</td><td>MSVC 2015 update 3</td><td>Cmake v3.6.3</td><td>6</td><td>8</td></tr>
 <tr><td>tensorflow-1.2.0</td><td>CPU</td><td>3.5-3.6</td><td>MSVC 2015 update 3</td><td>Cmake v3.6.3</td><td>N/A</td><td>N/A</td></tr>
 <tr><td>tensorflow_gpu-1.2.0</td><td>GPU</td><td>3.5-3.6</td><td>MSVC 2015 update 3</td><td>Cmake v3.6.3</td><td>5.1</td><td>8</td></tr>
 <tr><td>tensorflow-1.1.0</td><td>CPU</td><td>3.5</td><td>MSVC 2015 update 3</td><td>Cmake v3.6.3</td><td>N/A</td><td>N/A</td></tr>
diff --git a/tensorflow/java/maven/libtensorflow/pom.xml b/tensorflow/java/maven/libtensorflow/pom.xml
index 6cc1102930..a5b05132cf 100644
--- a/tensorflow/java/maven/libtensorflow/pom.xml
+++ b/tensorflow/java/maven/libtensorflow/pom.xml
@@ -6,7 +6,7 @@
   <parent>
     <groupId>org.tensorflow</groupId>
     <artifactId>parentpom</artifactId>
-    <version>1.3.0</version>
+    <version>1.4.0-rc0</version>
     <relativePath>../</relativePath>
   </parent>
   <artifactId>libtensorflow</artifactId>
diff --git a/tensorflow/java/maven/libtensorflow_jni/pom.xml b/tensorflow/java/maven/libtensorflow_jni/pom.xml
index 0b22844898..d863f03e3c 100644
--- a/tensorflow/java/maven/libtensorflow_jni/pom.xml
+++ b/tensorflow/java/maven/libtensorflow_jni/pom.xml
@@ -6,7 +6,7 @@
   <parent>
     <groupId>org.tensorflow</groupId>
     <artifactId>parentpom</artifactId>
-    <version>1.3.0</version>
+    <version>1.4.0-rc0</version>
     <relativePath>../</relativePath>
   </parent>
   <artifactId>libtensorflow_jni</artifactId>
diff --git a/tensorflow/java/maven/pom.xml b/tensorflow/java/maven/pom.xml
index 0a3552d756..1f5b056961 100644
--- a/tensorflow/java/maven/pom.xml
+++ b/tensorflow/java/maven/pom.xml
@@ -6,7 +6,7 @@
   <modelVersion>4.0.0</modelVersion>
   <groupId>org.tensorflow</groupId>
   <artifactId>parentpom</artifactId>
-  <version>1.3.0</version>
+  <version>1.4.0-rc0</version>
   <packaging>pom</packaging>
 
   <url>https://www.tensorflow.org</url>
diff --git a/tensorflow/java/maven/proto/pom.xml b/tensorflow/java/maven/proto/pom.xml
index b76b28aa15..51d53f6aba 100644
--- a/tensorflow/java/maven/proto/pom.xml
+++ b/tensorflow/java/maven/proto/pom.xml
@@ -6,7 +6,7 @@
   <parent>
     <groupId>org.tensorflow</groupId>
     <artifactId>parentpom</artifactId>
-    <version>1.3.0</version>
+    <version>1.4.0-rc0</version>
     <relativePath>../</relativePath>
   </parent>
   <artifactId>proto</artifactId>
diff --git a/tensorflow/java/maven/tensorflow/pom.xml b/tensorflow/java/maven/tensorflow/pom.xml
index c2af55f5ce..8cc3f113e3 100644
--- a/tensorflow/java/maven/tensorflow/pom.xml
+++ b/tensorflow/java/maven/tensorflow/pom.xml
@@ -6,7 +6,7 @@
   <parent>
     <groupId>org.tensorflow</groupId>
     <artifactId>parentpom</artifactId>
-    <version>1.3.0</version>
+    <version>1.4.0-rc0</version>
     <relativePath>../</relativePath>
   </parent>
   <artifactId>tensorflow</artifactId>
diff --git a/tensorflow/tools/docker/Dockerfile.devel b/tensorflow/tools/docker/Dockerfile.devel
index 2d4f03fbb7..20e1dcd085 100644
--- a/tensorflow/tools/docker/Dockerfile.devel
+++ b/tensorflow/tools/docker/Dockerfile.devel
@@ -72,7 +72,7 @@ RUN mkdir /bazel && \
 
 RUN git clone https://github.com/tensorflow/tensorflow.git && \
     cd tensorflow && \
-    git checkout r1.3
+    git checkout r1.4
 WORKDIR /tensorflow
 
 # TODO(craigcitro): Don't install the pip package, since it makes it
diff --git a/tensorflow/tools/docker/Dockerfile.devel-gpu b/tensorflow/tools/docker/Dockerfile.devel-gpu
index 1b605587ff..21a44ee404 100644
--- a/tensorflow/tools/docker/Dockerfile.devel-gpu
+++ b/tensorflow/tools/docker/Dockerfile.devel-gpu
@@ -73,7 +73,7 @@ RUN mkdir /bazel && \
 
 RUN git clone https://github.com/tensorflow/tensorflow.git && \
     cd tensorflow && \
-    git checkout r1.3
+    git checkout r1.4
 WORKDIR /tensorflow
 
 # Configure the build for our CUDA configuration.
diff --git a/tensorflow/tools/pip_package/setup.py b/tensorflow/tools/pip_package/setup.py
index f476fe766f..2ffaf7b1aa 100644
--- a/tensorflow/tools/pip_package/setup.py
+++ b/tensorflow/tools/pip_package/setup.py
@@ -29,7 +29,7 @@ from setuptools.dist import Distribution
 # This version string is semver compatible, but incompatible with pip.
 # For pip, we will remove all '-' characters from this string, and use the
 # result for pip.
-_VERSION = '1.3.0'
+_VERSION = '1.4.0-rc0'
 
 REQUIRED_PACKAGES = [
     'enum34 >= 1.1.6',
-- 
GitLab


From cbb4d188897e9aea23a5582ff0bc44bb9d60db01 Mon Sep 17 00:00:00 2001
From: Akshay Agrawal <akshayka@google.com>
Date: Fri, 13 Oct 2017 12:27:48 -0700
Subject: [PATCH 372/909] Watch trainable variables in graph_callable decorated
 functions

PiperOrigin-RevId: 172131167
---
 tensorflow/python/eager/BUILD                  |  1 +
 tensorflow/python/eager/graph_callable.py      |  3 +++
 tensorflow/python/eager/graph_callable_test.py | 12 ++++++++++++
 3 files changed, 16 insertions(+)

diff --git a/tensorflow/python/eager/BUILD b/tensorflow/python/eager/BUILD
index 963eaf0742..3586311c92 100644
--- a/tensorflow/python/eager/BUILD
+++ b/tensorflow/python/eager/BUILD
@@ -322,6 +322,7 @@ py_test(
     srcs = ["graph_callable_test.py"],
     srcs_version = "PY2AND3",
     deps = [
+        ":backprop",
         ":graph_callable",
         "//tensorflow/python:dtypes",
         "//tensorflow/python:function",
diff --git a/tensorflow/python/eager/graph_callable.py b/tensorflow/python/eager/graph_callable.py
index a1bdba6e4e..3aba164630 100644
--- a/tensorflow/python/eager/graph_callable.py
+++ b/tensorflow/python/eager/graph_callable.py
@@ -264,6 +264,9 @@ class _InitializingFunctionObject(object):
     initialized = [resource_variable_ops.var_is_initialized_op(
         v.handle).numpy() for v in self._call_fn.variables]
     if all(x for x in initialized):
+      for v in self._call_fn.variables:
+        if v._trainable:  # pylint: disable=protected-access
+          tape.watch_variable(v)
       return self._call_fn(*args)
     elif all(not x for x in initialized):
       return self._init_fn(*args)
diff --git a/tensorflow/python/eager/graph_callable_test.py b/tensorflow/python/eager/graph_callable_test.py
index 57e1a062e1..e77a33981d 100644
--- a/tensorflow/python/eager/graph_callable_test.py
+++ b/tensorflow/python/eager/graph_callable_test.py
@@ -17,6 +17,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from tensorflow.python.eager import backprop
 from tensorflow.python.eager import graph_callable
 from tensorflow.python.eager import test
 from tensorflow.python.framework import constant_op
@@ -233,6 +234,17 @@ class GraphCallableTest(test.TestCase):
     self.assertTrue(([1, 2, 3] == my_function(
         constant_op.constant([1, 2, 3], dtype=dtypes.float32)).numpy()).all())
 
+  def testGradients(self):
+    @graph_callable.graph_callable([])
+    def my_function():
+      v = variable_scope.get_variable(
+          "v", initializer=init_ops.constant_initializer(3.), shape=())
+      return v * v
+
+    grad_fn = backprop.implicit_grad(my_function)
+    grads_and_vars = list(zip(*grad_fn()))
+    self.assertEqual(6., grads_and_vars[0][0].numpy())
+
 
 if __name__ == "__main__":
   test.main()
-- 
GitLab


From 3f82236798b02ddb0b00b25c23e7443b6dbf5c01 Mon Sep 17 00:00:00 2001
From: Peter Hawkins <phawkins@google.com>
Date: Fri, 13 Oct 2017 12:41:43 -0700
Subject: [PATCH 373/909] [TPU] Register gradient for CrossReplicaSum. Happily,
 the gradient for CrossReplicaSum is just a CrossReplicaSum.

PiperOrigin-RevId: 172132628
---
 tensorflow/contrib/tpu/python/ops/tpu_ops.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/tensorflow/contrib/tpu/python/ops/tpu_ops.py b/tensorflow/contrib/tpu/python/ops/tpu_ops.py
index 8d3344fac3..33e47f674d 100644
--- a/tensorflow/contrib/tpu/python/ops/tpu_ops.py
+++ b/tensorflow/contrib/tpu/python/ops/tpu_ops.py
@@ -21,9 +21,11 @@ from __future__ import print_function
 
 import platform
 
+from tensorflow.python.framework import ops
 
 if platform.system() != "Windows":
   # pylint: disable=wildcard-import,unused-import,g-import-not-at-top
+  from tensorflow.contrib.tpu.ops import gen_tpu_ops
   from tensorflow.contrib.tpu.ops.gen_tpu_ops import *
 
   from tensorflow.contrib.util import loader
@@ -32,6 +34,12 @@ if platform.system() != "Windows":
 
   _tpu_ops = loader.load_op_library(
       resource_loader.get_path_to_datafile("_tpu_ops.so"))
+
+  @ops.RegisterGradient("CrossReplicaSum")
+  def _cross_replica_sum_grad(op, grad):
+    del op  # Unused
+    # The gradient of a cross replica sum is also a cross-replica sum.
+    return gen_tpu_ops.cross_replica_sum(grad)
 else:
   # We have already built the appropriate libraries into the binary via CMake
   # if we have built contrib, so we don't need this
-- 
GitLab


From d871fdce70acc165e652c66638943b40ffcda7a3 Mon Sep 17 00:00:00 2001
From: Jingyue Wu <jingyue@google.com>
Date: Fri, 13 Oct 2017 12:54:33 -0700
Subject: [PATCH 374/909] [Grappler] Remove reshapes whose source shape and
 destination shape are equal.

Also makes ArithmeticOptimizer::Optimize run shape inference at the beginning,
and clear _output_shapes at the end.

PiperOrigin-RevId: 172133948
---
 tensorflow/core/grappler/optimizers/BUILD     |  1 +
 .../optimizers/arithmetic_optimizer.cc        | 72 ++++++++++++++-
 .../optimizers/arithmetic_optimizer.h         |  7 +-
 .../optimizers/arithmetic_optimizer_test.cc   | 87 +++++++++++++++++++
 .../grappler/optimizers/meta_optimizer.cc     |  7 +-
 .../core/protobuf/rewriter_config.proto       |  4 +
 6 files changed, 173 insertions(+), 5 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/BUILD b/tensorflow/core/grappler/optimizers/BUILD
index 06a62f2a00..74030908fe 100644
--- a/tensorflow/core/grappler/optimizers/BUILD
+++ b/tensorflow/core/grappler/optimizers/BUILD
@@ -169,6 +169,7 @@ cc_library(
         "//tensorflow/core/grappler:grappler_item",
         "//tensorflow/core/grappler:op_types",
         "//tensorflow/core/grappler:utils",
+        "//tensorflow/core/grappler/costs:graph_properties",
     ],
 )
 
diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
index 971163eadf..8ef3383aa3 100644
--- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
@@ -20,6 +20,7 @@ limitations under the License.
 #include "tensorflow/core/framework/node_def.pb.h"
 #include "tensorflow/core/framework/op.h"
 #include "tensorflow/core/framework/tensor_shape.pb.h"
+#include "tensorflow/core/grappler/costs/graph_properties.h"
 #include "tensorflow/core/grappler/grappler_item.h"
 #include "tensorflow/core/grappler/op_types.h"
 #include "tensorflow/core/lib/strings/strcat.h"
@@ -333,6 +334,54 @@ static bool IsNumberType(DataType dtype) {
          number_types.end();
 }
 
+const char kOutputShapesAttr[] = "_output_shapes";
+
+// Returns whether `reshape` is an identity op. The tensor that `reshape`
+// reshapes is the `output_pos`-th output of node `input`.
+static bool ReshapeIsIdentity(const NodeDef& reshape, const NodeDef& input,
+                              const int output_pos) {
+  if (!reshape.attr().count(kOutputShapesAttr) ||
+      !input.attr().count(kOutputShapesAttr)) {
+    return false;
+  }
+
+  PartialTensorShape src_shape(
+      input.attr().at(kOutputShapesAttr).list().shape(output_pos));
+  PartialTensorShape dst_shape(
+      reshape.attr().at(kOutputShapesAttr).list().shape(0));
+  if (src_shape.unknown_rank() || dst_shape.unknown_rank()) {
+    return false;
+  }
+
+  if (!dst_shape.IsCompatibleWith(src_shape)) {
+    return false;
+  }
+
+  // Returns false when src_shape or dst_shape has >=2 dimensions with unknown
+  // sizes.
+  auto num_unknown_dim_sizes = [](const PartialTensorShape& partial_shape) {
+    auto dim_sizes = partial_shape.dim_sizes();
+    return std::count(dim_sizes.begin(), dim_sizes.end(), -1);
+  };
+  int src_num_unknown_dim_sizes = num_unknown_dim_sizes(src_shape);
+  int dst_num_unknown_dim_sizes = num_unknown_dim_sizes(dst_shape);
+  if (src_num_unknown_dim_sizes > 1 || dst_num_unknown_dim_sizes > 1) {
+    return false;
+  }
+
+  // Now, src_shape and dst_shape have at most one dimension with unknown
+  // sizes, and are compatible. Therefore, the reshape is a no-op when
+  //
+  // 1. at least one of them is fully-defined, or
+  // 2. both are partially defined and the -1 appears on the same dimension,
+  //    i.e., IsIdenticalTo returns true.
+  if (src_num_unknown_dim_sizes == 1 && dst_num_unknown_dim_sizes == 1) {
+    return dst_shape.IsIdenticalTo(src_shape);
+  }
+
+  return true;
+}
+
 string ArithmeticOptimizer::TrySimplifyAndReplaceUses(
     const NodeDef* node, GraphDef* graph_def, NodeMap* node_map,
     std::vector<const NodeDef*>* new_nodes) const {
@@ -370,13 +419,25 @@ string ArithmeticOptimizer::TrySimplifyAndReplaceUses(
     //      |      |
     //    input ---+
     NodeDef* reshape = node_map->GetNode(node->name());
-    const NodeDef* input = node_map->GetNode(node->input(0));
+    int output_pos = 0;
+    string input_node_name = ParseNodeName(node->input(0), &output_pos);
+    const NodeDef* input = node_map->GetNode(input_node_name);
     if (input->op() == "Reshape") {
       reshape->set_input(0, input->input(0));
       node_map->UpdateInput(reshape->name(), input->name(), input->input(0));
       new_nodes->push_back(reshape);
       return reshape->name();
     }
+
+    // If the reshape is a no-op, forward its input to its consumers. This is
+    // considered aggressive and turned off by default, because users may state
+    // that the placeholder outputs tensors of shape [M, N] while feeding it
+    // with tensors of shape [M*N] (or worse). The reshape nodes are then
+    // necessary to update the tensor metadata to the required shape.
+    if (opt_level_ == RewriterConfig::AGGRESSIVE &&
+        ReshapeIsIdentity(*reshape, *input, output_pos)) {
+      return reshape->input(0);
+    }
   }
 
   if (node->op() == "Transpose") {
@@ -652,9 +713,18 @@ Status ArithmeticOptimizer::Optimize(Cluster* /*cluster*/,
   *optimized_graph = item.graph;
   nodes_to_preserve_ = item.NodesToPreserve();
 
+  GraphProperties graph_properties(item);
+  TF_RETURN_IF_ERROR(graph_properties.InferStatically());
+  TF_RETURN_IF_ERROR(graph_properties.AnnotateOutputShapes(optimized_graph));
+
   DedupComputations(optimized_graph);
   SimplifyArithmeticOps(optimized_graph);
 
+  // Clear output shapes.
+  for (int i = 0; i < optimized_graph->node_size(); ++i) {
+    optimized_graph->mutable_node(i)->mutable_attr()->erase(kOutputShapesAttr);
+  }
+
   return Status::OK();
 }
 
diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.h b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.h
index fc381ec907..53cec11ff6 100644
--- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.h
+++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.h
@@ -19,6 +19,7 @@ limitations under the License.
 #include <unordered_set>
 #include "tensorflow/core/grappler/optimizers/graph_optimizer.h"
 #include "tensorflow/core/grappler/utils.h"
+#include "tensorflow/core/protobuf/rewriter_config.pb.h"
 
 namespace tensorflow {
 namespace grappler {
@@ -27,7 +28,9 @@ namespace grappler {
 // run a model.
 class ArithmeticOptimizer : public GraphOptimizer {
  public:
-  ArithmeticOptimizer() {}
+  ArithmeticOptimizer() : opt_level_(RewriterConfig::ON) {}
+  explicit ArithmeticOptimizer(RewriterConfig::Toggle opt_level)
+      : opt_level_(opt_level) {}
   ~ArithmeticOptimizer() override {}
 
   string name() const override { return "arithmetic_optimizer"; };
@@ -64,6 +67,8 @@ class ArithmeticOptimizer : public GraphOptimizer {
       std::vector<const NodeDef*>* new_nodes) const;
 
   std::unordered_set<string> nodes_to_preserve_;
+
+  RewriterConfig::Toggle opt_level_;
 };
 
 }  // end namespace grappler
diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc b/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc
index 39b4999808..a4de838a65 100644
--- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc
+++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc
@@ -77,6 +77,93 @@ TEST_F(ArithmeticOptimizerTest, OpDedupping) {
   EXPECT_EQ("c1", new_add.input(1));
 }
 
+TEST_F(ArithmeticOptimizerTest, IdentityReshape) {
+  tensorflow::Scope s = tensorflow::Scope::NewRootScope();
+  Output inputs =
+      ops::Placeholder(s, DT_FLOAT, ops::Placeholder::Shape({-1, 3, 28, 28}));
+  Output inputs_shape = ops::Shape(s, inputs);
+  // The target shape of the reshape is the concatenation of `batch_size` and
+  // [3,28,28].
+  Output batch_size = ops::Slice(s, inputs_shape, ops::Const(s, {0}, {1}),
+                                 ops::Const(s, {1}, {1}));
+  Output target_shape = ops::Concat(
+      s.WithOpName("target_shape"),
+      {batch_size, ops::Const(s, {3, 28, 28}, {3})}, ops::Const(s, {0}, {}));
+  Output reshape = ops::Reshape(s, inputs, target_shape);
+  Output outputs = ops::Identity(s.WithOpName("outputs"), reshape);
+
+  GrapplerItem item;
+  item.fetch = {"outputs"};
+  TF_CHECK_OK(s.ToGraphDef(&item.graph));
+
+  GraphDef output;
+  TF_EXPECT_OK(ArithmeticOptimizer(RewriterConfig::AGGRESSIVE)
+                   .Optimize(nullptr, item, &output));
+
+  item.graph = output;
+  TF_EXPECT_OK(ModelPruner().Optimize(nullptr, item, &output));
+
+  for (const auto& node : output.node()) {
+    LOG(INFO) << node.DebugString();
+  }
+
+  EXPECT_EQ(0, std::count_if(
+                   output.node().begin(), output.node().end(),
+                   [](const NodeDef& node) { return node.op() == "Reshape"; }));
+}
+
+TEST_F(ArithmeticOptimizerTest, NotIdentityReshape) {
+  // Reshape from [-1,3,28,28] to [8,-1,28,28] is not identity, because it can
+  // be from [4,3,28,28] to [8,6,28,28].
+  tensorflow::Scope s = tensorflow::Scope::NewRootScope();
+  Output inputs =
+      ops::Placeholder(s, DT_FLOAT, ops::Placeholder::Shape({-1, 3, 28, 28}));
+  Output reshape = ops::Reshape(s, inputs, ops::Const(s, {8, -1, 28, 28}, {4}));
+  Output outputs = ops::Identity(s.WithOpName("outputs"), reshape);
+
+  GrapplerItem item;
+  item.fetch = {"outputs"};
+  TF_CHECK_OK(s.ToGraphDef(&item.graph));
+
+  GraphDef output;
+  TF_EXPECT_OK(ArithmeticOptimizer(RewriterConfig::AGGRESSIVE)
+                   .Optimize(nullptr, item, &output));
+
+  item.graph = output;
+  TF_EXPECT_OK(ModelPruner().Optimize(nullptr, item, &output));
+
+  for (const auto& node : output.node()) {
+    LOG(INFO) << node.DebugString();
+  }
+
+  EXPECT_EQ(1, std::count_if(
+                   output.node().begin(), output.node().end(),
+                   [](const NodeDef& node) { return node.op() == "Reshape"; }));
+}
+
+TEST_F(ArithmeticOptimizerTest, NotIdentityReshapeTooManyUnknownDimSizes) {
+  tensorflow::Scope s = tensorflow::Scope::NewRootScope();
+  Output inputs =
+      ops::Placeholder(s, DT_FLOAT, ops::Placeholder::Shape({4, 3}));
+  Output reshape = ops::Reshape(s, inputs, ops::Const(s, {-1, -1}, {2}));
+  Output outputs = ops::Identity(s.WithOpName("outputs"), reshape);
+
+  GrapplerItem item;
+  item.fetch = {"outputs"};
+  TF_CHECK_OK(s.ToGraphDef(&item.graph));
+
+  GraphDef output;
+  TF_EXPECT_OK(ArithmeticOptimizer(RewriterConfig::AGGRESSIVE)
+                   .Optimize(nullptr, item, &output));
+
+  item.graph = output;
+  TF_EXPECT_OK(ModelPruner().Optimize(nullptr, item, &output));
+
+  EXPECT_EQ(1, std::count_if(
+                   output.node().begin(), output.node().end(),
+                   [](const NodeDef& node) { return node.op() == "Reshape"; }));
+}
+
 TEST_F(ArithmeticOptimizerTest, CombineReshapes) {
   // Converts an NCHW_VECT_C tensor to NHWC and then flattens it to 2D. The two
   // reshapes should be combined.
diff --git a/tensorflow/core/grappler/optimizers/meta_optimizer.cc b/tensorflow/core/grappler/optimizers/meta_optimizer.cc
index 6718d2d739..1174a390f3 100644
--- a/tensorflow/core/grappler/optimizers/meta_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/meta_optimizer.cc
@@ -46,7 +46,8 @@ std::unique_ptr<GraphOptimizer> MetaOptimizer::NewOptimizer(
     graph_optimizer.reset(new MemoryOptimizer(RewriterConfig::MANUAL));
   }
   if (optimizer == "arithmetic") {
-    graph_optimizer.reset(new ArithmeticOptimizer());
+    graph_optimizer.reset(
+        new ArithmeticOptimizer(cfg_.arithmetic_optimization()));
   }
   if (optimizer == "autoparallel") {
     graph_optimizer.reset(
@@ -67,8 +68,8 @@ Status MetaOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item,
           std::unique_ptr<GraphOptimizer>(new ConstantFolding(cpu_device_)));
     }
     if (cfg_.arithmetic_optimization() != RewriterConfig::OFF) {
-      optimizers.push_back(
-          std::unique_ptr<GraphOptimizer>(new ArithmeticOptimizer()));
+      optimizers.push_back(std::unique_ptr<GraphOptimizer>(
+          new ArithmeticOptimizer(cfg_.arithmetic_optimization())));
     }
     if (cfg_.optimize_tensor_layout()) {
       optimizers.push_back(
diff --git a/tensorflow/core/protobuf/rewriter_config.proto b/tensorflow/core/protobuf/rewriter_config.proto
index 8a8dd3c7d5..d67088311b 100644
--- a/tensorflow/core/protobuf/rewriter_config.proto
+++ b/tensorflow/core/protobuf/rewriter_config.proto
@@ -23,6 +23,10 @@ message RewriterConfig {
     DEFAULT = 0;
     ON = 1;
     OFF = 2;
+    // Enable some aggressive optimizations that use assumptions that TF graphs
+    // may break. For example, assume the shape of a placeholder matches its
+    // actual feed.
+    AGGRESSIVE = 3;
   }
 
   // Optimize tensor layouts
-- 
GitLab


From f49f1021a8375375d4ebb819544318c68fdec34d Mon Sep 17 00:00:00 2001
From: Yunxing Dai <yunxing@google.com>
Date: Fri, 13 Oct 2017 13:01:19 -0700
Subject: [PATCH 375/909] [TF/XLA] Change GraphCompiler's to use a comparator
 based on node's name.

We realized that sorting the graph by id of a node is not always deterministic as ids themselves are randomly ordered by tensorflow.

RELNOTES: n/a
PiperOrigin-RevId: 172134671
---
 tensorflow/compiler/tf2xla/graph_compiler.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/compiler/tf2xla/graph_compiler.cc b/tensorflow/compiler/tf2xla/graph_compiler.cc
index d5369e478a..6f2f59d98f 100644
--- a/tensorflow/compiler/tf2xla/graph_compiler.cc
+++ b/tensorflow/compiler/tf2xla/graph_compiler.cc
@@ -88,7 +88,7 @@ Status GraphCompiler::Compile() {
   std::vector<Node*> topo_sorted_nodes;
   // XLA requires determinism, generate a stable ordering from DFS.
   GetReversePostOrder(*graph_, &topo_sorted_nodes,
-                      /*stable_comparator=*/NodeComparatorID());
+                      /*stable_comparator=*/NodeComparatorName());
 
   OpKernelContext::Params params;
   PartiallySetupParams(&params);
-- 
GitLab


From 7f30cbf0e21ecc3dd302c754a7b9ddecf474b115 Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Fri, 13 Oct 2017 13:02:47 -0700
Subject: [PATCH 376/909] Internal change.

PiperOrigin-RevId: 172134904
---
 tensorflow/contrib/gan/BUILD  | 1 +
 tensorflow/python/keras/BUILD | 1 +
 2 files changed, 2 insertions(+)

diff --git a/tensorflow/contrib/gan/BUILD b/tensorflow/contrib/gan/BUILD
index 64bff7cecf..27a5d6ec31 100644
--- a/tensorflow/contrib/gan/BUILD
+++ b/tensorflow/contrib/gan/BUILD
@@ -441,6 +441,7 @@ py_test(
     srcs = ["python/estimator/python/gan_estimator_test.py"],
     shard_count = 1,
     srcs_version = "PY2AND3",
+    tags = ["notsan"],
     deps = [
         ":gan_estimator",
         ":namedtuples",
diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index fd8ac392de..d61733dff6 100644
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -668,6 +668,7 @@ py_test(
     size = "medium",
     srcs = ["_impl/keras/estimator_test.py"],
     srcs_version = "PY2AND3",
+    tags = ["notsan"],
     deps = [
         ":keras",
         "//tensorflow/python:client_testlib",
-- 
GitLab


From eaec44d1d8ed124cde40f767cd73bbb6ffdad7c2 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Fri, 13 Oct 2017 13:08:05 -0700
Subject: [PATCH 377/909] [XLA] DCE dead computations.

We get a dead computation when e.g. we delete a reduction or remove a
while loop.

PiperOrigin-RevId: 172135511
---
 tensorflow/compiler/xla/service/hlo_dce.cc    | 23 +++++
 .../compiler/xla/service/hlo_dce_test.cc      | 88 +++++++++++++++++++
 2 files changed, 111 insertions(+)

diff --git a/tensorflow/compiler/xla/service/hlo_dce.cc b/tensorflow/compiler/xla/service/hlo_dce.cc
index 71321e5e9a..a4921232f5 100644
--- a/tensorflow/compiler/xla/service/hlo_dce.cc
+++ b/tensorflow/compiler/xla/service/hlo_dce.cc
@@ -64,6 +64,29 @@ StatusOr<bool> HloDCE::Run(HloModule* module) {
     }
   }
 
+  // Now DCE HloComputations.  First, collect the computations that are
+  // referenced by some remaining instruction.
+  std::unordered_set<HloComputation*> live_computations;
+  if (HloComputation* entry_computation = module->entry_computation()) {
+    live_computations.insert(entry_computation);
+  }
+  for (auto* computation : module->MakeComputationPostOrder()) {
+    for (auto* instruction : computation->instructions()) {
+      for (auto* subcomp : instruction->called_computations()) {
+        live_computations.insert(subcomp);
+      }
+    }
+  }
+
+  // Remove dead computations.
+  std::list<HloComputation*> computations = module->MakeComputationPostOrder();
+  for (auto* computation : computations) {
+    if (live_computations.count(computation) == 0) {
+      TF_RETURN_IF_ERROR(module->RemoveEmbeddedComputation(computation));
+      changed = true;
+    }
+  }
+
   return changed;
 }
 
diff --git a/tensorflow/compiler/xla/service/hlo_dce_test.cc b/tensorflow/compiler/xla/service/hlo_dce_test.cc
index fa0ab98649..d54b9a2708 100644
--- a/tensorflow/compiler/xla/service/hlo_dce_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_dce_test.cc
@@ -299,5 +299,93 @@ TEST_F(HloDceTest, CalledComputationWithNestedSideEffect) {
   EXPECT_TRUE(HasInstruction(*computation, live_call));
 }
 
+TEST_F(HloDceTest, RemoveDeadSubcomputation) {
+  auto module = CreateNewModule();
+  HloComputation::Builder builder(TestName());
+
+  HloComputation::Builder subcomp_builder("reduction_subcomp");
+  {
+    auto* param0 =
+        subcomp_builder.AddInstruction(HloInstruction::CreateParameter(
+            /*parameter_number=*/0, ShapeUtil::MakeShape(F32, {}), "param0"));
+    auto* param1 =
+        subcomp_builder.AddInstruction(HloInstruction::CreateParameter(
+            /*parameter_number=*/1, ShapeUtil::MakeShape(F32, {}), "param1"));
+    subcomp_builder.AddInstruction(HloInstruction::CreateBinary(
+        ShapeUtil::MakeShape(F32, {}), HloOpcode::kAdd, param0, param1));
+  }
+  auto reduce_subcomp = module->AddEmbeddedComputation(subcomp_builder.Build());
+
+  // Create a dead reduce instruction.
+  builder.AddInstruction(HloInstruction::CreateReduce(
+      ShapeUtil::MakeShape(F32, {1}),
+      builder.AddInstruction(HloInstruction::CreateParameter(
+          /*parameter_number=*/0, ShapeUtil::MakeShape(F32, {100}), "param0")),
+      builder.AddInstruction(
+          HloInstruction::CreateConstant(Literal::CreateR0<float>(0))),
+      /*dimensions_to_reduce=*/{0}, reduce_subcomp));
+
+  // Add another instruction as the root of the computation.
+  builder.AddInstruction(
+      HloInstruction::CreateConstant(Literal::CreateR0<float>(0)));
+
+  module->AddEntryComputation(builder.Build());
+  EXPECT_EQ(module->MakeComputationPostOrder().size(), 2);
+
+  HloDCE dce;
+  EXPECT_TRUE(dce.Run(module.get()).ValueOrDie());
+
+  // We should have DCE'ed the reduction computation along with the reduction
+  // instruction.
+  EXPECT_EQ(module->MakeComputationPostOrder().size(), 1);
+}
+
+TEST_F(HloDceTest, KeepUsedSubcomputation) {
+  auto module = CreateNewModule();
+  HloComputation::Builder builder(TestName());
+
+  HloComputation::Builder subcomp_builder("reduction_subcomp");
+  {
+    auto* param0 =
+        subcomp_builder.AddInstruction(HloInstruction::CreateParameter(
+            /*parameter_number=*/0, ShapeUtil::MakeShape(F32, {}), "param0"));
+    auto* param1 =
+        subcomp_builder.AddInstruction(HloInstruction::CreateParameter(
+            /*parameter_number=*/1, ShapeUtil::MakeShape(F32, {}), "param1"));
+    subcomp_builder.AddInstruction(HloInstruction::CreateBinary(
+        ShapeUtil::MakeShape(F32, {}), HloOpcode::kAdd, param0, param1));
+  }
+  auto reduce_subcomp = module->AddEmbeddedComputation(subcomp_builder.Build());
+
+  // Create a dead reduce instruction.
+  builder.AddInstruction(HloInstruction::CreateReduce(
+      ShapeUtil::MakeShape(F32, {1}),
+      builder.AddInstruction(HloInstruction::CreateParameter(
+          /*parameter_number=*/0, ShapeUtil::MakeShape(F32, {100}), "param0")),
+      builder.AddInstruction(
+          HloInstruction::CreateConstant(Literal::CreateR0<float>(0))),
+      /*dimensions_to_reduce=*/{0}, reduce_subcomp));
+
+  // Add another instruction as the root of the computation that also uses
+  // reduce_subcomp.
+  builder.AddInstruction(HloInstruction::CreateReduce(
+      ShapeUtil::MakeShape(F32, {1}),
+      builder.AddInstruction(HloInstruction::CreateParameter(
+          /*parameter_number=*/1, ShapeUtil::MakeShape(F32, {100}), "param1")),
+      builder.AddInstruction(
+          HloInstruction::CreateConstant(Literal::CreateR0<float>(0))),
+      /*dimensions_to_reduce=*/{0}, reduce_subcomp));
+
+  module->AddEntryComputation(builder.Build());
+  EXPECT_EQ(module->MakeComputationPostOrder().size(), 2);
+
+  HloDCE dce;
+  EXPECT_TRUE(dce.Run(module.get()).ValueOrDie());
+
+  // We shouldn't have DCE'ed reduce_subcomp, even though we removed one of
+  // its users.
+  EXPECT_EQ(module->MakeComputationPostOrder().size(), 2);
+}
+
 }  // namespace
 }  // namespace xla
-- 
GitLab


From b57187b3a4ca94869e224d0e1a0020b6fee1abd1 Mon Sep 17 00:00:00 2001
From: Andrew Harp <andrewharp@google.com>
Date: Fri, 13 Oct 2017 13:19:07 -0700
Subject: [PATCH 378/909] Automated g4 rollback of changelist 172018709

PiperOrigin-RevId: 172136820
---
 tensorflow/BUILD                      | 16 ----------------
 tensorflow/compiler/aot/tfcompile.bzl |  2 --
 tensorflow/tensorflow.bzl             |  6 ------
 3 files changed, 24 deletions(-)

diff --git a/tensorflow/BUILD b/tensorflow/BUILD
index 3868a1814b..a563e3b383 100644
--- a/tensorflow/BUILD
+++ b/tensorflow/BUILD
@@ -141,22 +141,6 @@ config_setting(
     visibility = ["//visibility:public"],
 )
 
-config_setting(
-    name = "linux_armhf",
-    values = {
-        "cpu": "armeabi-v7a",
-    },
-    visibility = ["//visibility:public"],
-)
-
-config_setting(
-    name = "linux_arm64",
-    values = {
-        "cpu": "arm64-v8a",
-    },
-    visibility = ["//visibility:public"],
-)
-
 config_setting(
     name = "debug",
     values = {
diff --git a/tensorflow/compiler/aot/tfcompile.bzl b/tensorflow/compiler/aot/tfcompile.bzl
index c900d201d2..4888760acd 100644
--- a/tensorflow/compiler/aot/tfcompile.bzl
+++ b/tensorflow/compiler/aot/tfcompile.bzl
@@ -302,8 +302,6 @@ def target_llvm_triple():
       "//tensorflow:android_arm": "armv7-none-android",
       "//tensorflow:android_arm64": "aarch64-none-android",
       "//tensorflow:android_x86": "i686-none-android",
-      "//tensorflow:linux_armhf": "armv7-none-linux-gnueabihf",
-      "//tensorflow:linux_arm64": "aarch64-none-linux-gnu",
       "//tensorflow:linux_ppc64le": "ppc64le-ibm-linux-gnu",
       "//tensorflow:darwin": "x86_64-none-darwin",
       "//conditions:default": "x86_64-pc-linux",
diff --git a/tensorflow/tensorflow.bzl b/tensorflow/tensorflow.bzl
index 3dd716f106..3001a37473 100644
--- a/tensorflow/tensorflow.bzl
+++ b/tensorflow/tensorflow.bzl
@@ -112,9 +112,6 @@ def if_ios(a):
 def if_mobile(a):
   return select({
       clean_dep("//tensorflow:android"): a,
-      # Treat arm linux devices as mobile.
-      clean_dep("//tensorflow:linux_arm64"): a,
-      clean_dep("//tensorflow:linux_armhf"): a,
       clean_dep("//tensorflow:ios"): a,
       "//conditions:default": [],
   })
@@ -123,9 +120,6 @@ def if_mobile(a):
 def if_not_mobile(a):
   return select({
       clean_dep("//tensorflow:android"): [],
-      # Treat arm linux devices as mobile.
-      clean_dep("//tensorflow:linux_arm64"): [],
-      clean_dep("//tensorflow:linux_armhf"): [],
       clean_dep("//tensorflow:ios"): [],
       "//conditions:default": a,
   })
-- 
GitLab


From 4ad0692230a1e50073a3c471be84e3d7178b9d37 Mon Sep 17 00:00:00 2001
From: Russell Power <power@google.com>
Date: Fri, 13 Oct 2017 13:39:06 -0700
Subject: [PATCH 379/909] Automated g4 rollback of changelist 172050536

PiperOrigin-RevId: 172139466
---
 tensorflow/contrib/tpu/python/tpu/tpu_estimator.py | 11 +----------
 1 file changed, 1 insertion(+), 10 deletions(-)

diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
index f6f89786c5..43f9defd54 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
@@ -21,7 +21,6 @@ from __future__ import print_function
 
 import collections
 import copy
-import os
 import threading
 import six
 from six.moves import queue as Queue  # pylint: disable=redefined-builtin
@@ -279,15 +278,7 @@ class _InfeedThreadController(_InfeedOutfeedThreadBaseController):
       iterations = signal
       for i in range(iterations):
         logging.debug('Infeed enqueue for iteration (%d, %d)', count, i)
-        try:
-          session.run(enqueue_ops)
-        except:  # pylint: disable=bare-except
-          # Hard exit from the interpreter.
-          #
-          # TODO(power) -- possibly communicate this to the main thread somehow.
-          logging.fatal('Infeed controller failed to enqueue ops.  Aborting.',
-                        exc_info=True)
-          os._exit(1)  # pylint: disable=protected-access
+        session.run(enqueue_ops)
       count += 1
 
   def join(self):
-- 
GitLab


From f688c35681623f38acdd9ba3a4db73fd092e13f3 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 13 Oct 2017 13:41:50 -0700
Subject: [PATCH 380/909] Define Evaluator class.

PiperOrigin-RevId: 172139804
---
 tensorflow/contrib/eager/python/BUILD         |  26 +++
 tensorflow/contrib/eager/python/evaluator.py  | 217 ++++++++++++++++++
 .../contrib/eager/python/evaluator_test.py    | 124 ++++++++++
 .../contrib/eager/python/metrics_impl.py      |  14 +-
 .../contrib/eager/python/metrics_test.py      |   2 +
 5 files changed, 377 insertions(+), 6 deletions(-)
 create mode 100644 tensorflow/contrib/eager/python/evaluator.py
 create mode 100644 tensorflow/contrib/eager/python/evaluator_test.py

diff --git a/tensorflow/contrib/eager/python/BUILD b/tensorflow/contrib/eager/python/BUILD
index 94f21808a3..0c61630aa8 100644
--- a/tensorflow/contrib/eager/python/BUILD
+++ b/tensorflow/contrib/eager/python/BUILD
@@ -12,6 +12,7 @@ py_library(
     visibility = ["//visibility:public"],
     deps = [
         ":datasets",
+        ":evaluator",
         ":metrics",
         ":network",
         ":saver",
@@ -149,6 +150,31 @@ py_test(
     ],
 )
 
+py_library(
+    name = "evaluator",
+    srcs = [
+        "evaluator.py",
+    ],
+    srcs_version = "PY2AND3",
+    visibility = ["//tensorflow:internal"],
+    deps = [
+        ":datasets",
+        ":metrics",
+    ],
+)
+
+py_test(
+    name = "evaluator_test",
+    srcs = ["evaluator_test.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":evaluator",
+        ":metrics",
+        "//tensorflow/python/data/ops:dataset_ops",
+        "//tensorflow/python/eager:test",
+    ],
+)
+
 py_library(
     name = "network",
     srcs = ["network.py"],
diff --git a/tensorflow/contrib/eager/python/evaluator.py b/tensorflow/contrib/eager/python/evaluator.py
new file mode 100644
index 0000000000..d757e976ee
--- /dev/null
+++ b/tensorflow/contrib/eager/python/evaluator.py
@@ -0,0 +1,217 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Class Evaluator holds Metrics for the duration of an evaluation run."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import six
+
+from tensorflow.contrib.eager.python import datasets
+from tensorflow.contrib.eager.python import metrics
+
+
+class Evaluator(object):
+  """This holds and updates Metrics for the duration of a single eval run.
+
+  Usage:
+    evaluator = my_model.evaluator() # or MyEvaluator(my_model)
+    for example_batch in ...:
+      evaluator(example_batch)
+    results = evaluator.all_metric_results(optional_summary_writer)
+
+  Or, if you are getting your examples from a tf.data.Dataset, you can use
+  the evaluate_on_dataset() method.
+
+  Implementers of Evaluators should
+  (a) Call `add_metric()` and/or `add_evaluator()` in __init__().
+  (b) Override the `call()` method. It will be passed the output of the
+      model's `eval_data()` method, and should call its contained metrics
+      (treating them as callables) and any child Evaluators (using their
+      call() method to avoid calling eval_data() again).
+
+  Args:
+    model: A `Model` object with an `eval_data()` method.
+  """
+
+  def __init__(self, model):
+    self._model = model
+    self._metrics = {}
+    self._evaluators = {}
+
+  # ---- API for users ----
+  def __call__(self, *args, **kwargs):
+    """Update metrics with a minibatch of input examples."""
+    return self.call(self._model.eval_data(*args, **kwargs))
+
+  def all_metric_results(self):  # TODO(josh11b): Add optional summary_writer.
+    """Returns dict mapping metric name -> value."""
+    results = {}
+    for name, metric in six.iteritems(self._metrics):
+      results[name] = metric.result()
+    for prefix, evaluator in six.iteritems(self._evaluators):
+      for name, metric in six.iteritems(evaluator._metrics):  # pylint: disable=protected-access
+        results[prefix + "/" + name] = metric.result()
+    return results
+
+  def evaluate_on_dataset(self, dataset, *args, **kwargs):
+    """Convenience method for performing an eval on a Dataset."""
+    for example in datasets.Iterator(dataset):
+      self.__call__(example, *args, **kwargs)
+    # TODO(josh11b): Add optional summary_writer.
+    return self.all_metric_results()
+
+  # ---- To be implemented by descendants ---
+  def call(self, eval_data):
+    """Update metrics using the output of self.model."""
+    raise NotImplementedError("Evaluators must define a call member function.")
+
+  # ---- For use by descendants ---
+  @property
+  def model(self):
+    return self._model
+
+  def add_metric(self, metric):
+    """Add a Metric to be tracked.
+
+    Rule: metrics can only be in one `Evaluator`.
+
+    Args:
+      metric: A `Metric` object.
+
+    Returns:
+      The `metric` passed into this function.
+
+    Raises:
+      RuntimeError: If called before __init__.
+      TypeError: If `metric` is not of the correct type.
+      ValueError: If there is a name collision between Metrics.
+    """
+    if not hasattr(self, "_metrics"):
+      raise RuntimeError(
+          "Need to call Evaluator.__init__ before adding metrics")
+    if not isinstance(metric, metrics.Metric):
+      raise TypeError(
+          "Evaluator.add_metric() passed type %s, not a tfe.metrics.Metric" %
+          (type(metric),))
+    if metric.name in self._metrics:
+      if metric is self._metrics[metric.name]:
+        return metric
+      raise ValueError(
+          "Attempt to add two Metrics with the name '%s' to the same Evaluator "
+          "'%s'" % (metric.name, self.name))
+    self._metrics[metric.name] = metric
+    return metric
+
+  def add_evaluator(self, prefix, evaluator):
+    """Add a contained `Evaluator`.
+
+    This is for delegating to another `Evaluator`, e.g. for when you have a
+    model with multiple heads. Users should manually invoke the child
+    `Evaluator`'s `call` method from their `call` method.
+
+    Args:
+      prefix: A string. Metrics from `evaluator` are exported with this
+        prefix and a '/'.
+      evaluator: An `Evaluator` object.
+
+    Returns:
+      The value of `evaluator` passed into this function.
+
+    Raises:
+      RuntimeError: If called before __init__.
+      TypeError: If `evaluator` is not of the correct type.
+      ValueError: If an `Evaluator` has already been added with that `prefix`.
+    """
+    if not hasattr(self, "_evaluators"):
+      raise RuntimeError(
+          "Need to call Evaluator.__init__ before adding evaluators")
+    if not isinstance(evaluator, Evaluator):
+      raise TypeError(
+          "Evaluator.add_evaluator() passed type %s, not a tfe.Evaluator." %
+          (type(evaluator),))
+    if prefix in self._evaluators:
+      if evaluator is self._evaluators[prefix]:
+        return evaluator
+      raise RuntimeError(
+          "Attempt to add two Evaluators with the same prefix '%s'." % prefix)
+    self._evaluators[prefix] = evaluator
+    return evaluator
+
+  @property
+  def metric_variables(self):
+    v = []
+    for metric in six.itervalues(self._metrics):
+      v += metric.variables
+    for evaluator in six.itervalues(self._evaluators):
+      v += evaluator.metric_variables
+    return v
+
+  @property
+  def metrics(self):
+    m = []
+    for metric in six.itervalues(self._metrics):
+      m.append(metric)
+    for evaluator in six.itervalues(self._evaluators):
+      m += evaluator.metrics
+    return m
+
+
+class SparseSoftmaxEvaluator(Evaluator):
+  """Evaluator for a sparse softmax model.
+
+  Computes a standard set of metrics for single-label, multi-class
+  models.
+
+  Args:
+    model: A `SparseSoftmaxModel` object or a `Model` whose `eval_data()`
+      method produces a `dict` containing values for the loss, true
+      label, predicted class, and optional weights.
+    loss_key: Optional key for looking up the value of the loss in the
+      `eval_data()` dict. Defaults to "loss".
+    label_key: Optional key for looking up the value of the label in the
+      `eval_data()` dict. Defaults to "label".
+    predicted_class_key: Optional key for looking up the value of the
+      predicted class in the `eval_data()` dict. Defaults to "predicted_class".
+    weights_key: Optional key for looking up the value of the weights
+      in the `eval_data()` dict. Defaults to "weights". Note that weights
+      are optional, and default to 1 if not present in `eval_data`.
+  """
+
+  def __init__(self, model, loss_key="loss", label_key="label",
+               predicted_class_key="predicted_class", weights_key="weights"):
+    super(SparseSoftmaxEvaluator, self).__init__(model)
+    # TODO(josh11b): Expand this to include everything from the standard
+    # SparseSoftmax Head.
+    self.avg_loss = self.add_metric(metrics.Mean("Avg_Loss"))
+    self.accuracy = self.add_metric(metrics.Accuracy())
+    self.loss_key = loss_key
+    self.label_key = label_key
+    self.predicted_class_key = predicted_class_key
+    self.weights_key = weights_key
+
+  def call(self, eval_data):
+    """Update metrics for `eval_data` dict (described above)."""
+    weights = eval_data.get(self.weights_key, None)
+    if weights is None:
+      self.avg_loss(eval_data[self.loss_key])
+      self.accuracy(eval_data[self.label_key],
+                    eval_data[self.predicted_class_key])
+    else:
+      self.avg_loss(eval_data[self.loss_key], weights=weights)
+      self.accuracy(eval_data[self.label_key],
+                    eval_data[self.predicted_class_key],
+                    weights=weights)
diff --git a/tensorflow/contrib/eager/python/evaluator_test.py b/tensorflow/contrib/eager/python/evaluator_test.py
new file mode 100644
index 0000000000..099e10e230
--- /dev/null
+++ b/tensorflow/contrib/eager/python/evaluator_test.py
@@ -0,0 +1,124 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for class Evaluator."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.contrib.eager.python import evaluator
+from tensorflow.contrib.eager.python import metrics
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.eager import test
+
+
+class IdentityModel(object):
+
+  def eval_data(self, d):
+    return d
+
+
+class PrefixLModel(object):
+
+  def eval_data(self, d):
+    return {"l_" + key: d[key] for key in d}
+
+
+class SimpleEvaluator(evaluator.Evaluator):
+
+  def __init__(self, model):
+    super(SimpleEvaluator, self).__init__(model)
+    self.mean = self.add_metric(metrics.Mean("mean"))
+
+  def call(self, eval_data):
+    self.mean(eval_data)
+
+
+class DelegatingEvaluator(evaluator.Evaluator):
+
+  def __init__(self, model):
+    super(DelegatingEvaluator, self).__init__(model)
+    self.sub = self.add_evaluator("inner", SimpleEvaluator(model))
+    self.mean = self.add_metric(metrics.Mean("outer-mean"))
+
+  def call(self, eval_data):
+    # Keys here come from PrefixLModel, which adds "l_".
+    self.mean(eval_data["l_outer"])
+    self.sub.call(eval_data["l_inner"])
+
+
+# pylint: disable=not-callable
+class EvaluatorTest(test.TestCase):
+
+  def testSimple(self):
+    e = SimpleEvaluator(IdentityModel())
+    e(3.0)
+    e([5.0, 7.0, 9.0])
+    results = e.all_metric_results()
+    self.assertEqual(set(["mean"]), set(results.keys()))
+    self.assertEqual(6.0, results["mean"].numpy())
+
+  def testComposition(self):
+    e = DelegatingEvaluator(PrefixLModel())
+    e({"inner": 2.0, "outer": 100.0})
+    e({"inner": 4.0, "outer": 1000.0})
+    results = e.all_metric_results()
+    self.assertEqual(set(["inner/mean", "outer-mean"]), set(results.keys()))
+    self.assertEqual(3.0, results["inner/mean"].numpy())
+    self.assertEqual(550.0, results["outer-mean"].numpy())
+
+  def testMetricVariables(self):
+    e = DelegatingEvaluator(PrefixLModel())
+    e({"inner": 2.0, "outer": 100.0})
+    prefix_count = {}
+    for v in e.metric_variables:
+      p = v.name.split("/")[0]
+      prefix_count[p] = prefix_count.get(p, 0) + 1
+    self.assertEqual({"outer-mean": 2, "mean": 2}, prefix_count)
+
+  def testDataset(self):
+    e = SimpleEvaluator(IdentityModel())
+    ds = dataset_ops.Dataset.from_tensor_slices([3.0, 5.0, 7.0, 9.0])
+    results = e.evaluate_on_dataset(ds)
+    self.assertEqual(set(["mean"]), set(results.keys()))
+    self.assertEqual(6.0, results["mean"].numpy())
+
+  def testModelProperty(self):
+    m = IdentityModel()
+    e = SimpleEvaluator(m)
+    self.assertIs(m, e.model)
+
+  def testMetricsProperty(self):
+    e = DelegatingEvaluator(PrefixLModel())
+    names = set([m.name for m in e.metrics])
+    self.assertEqual(set(["outer-mean", "mean"]), names)
+
+
+class SparseSoftmaxEvaluatorTest(test.TestCase):
+
+  def testSimple(self):
+    e = evaluator.SparseSoftmaxEvaluator(IdentityModel())
+    e({e.loss_key: 1.0, e.label_key: 5, e.predicted_class_key: 5})
+    e({e.loss_key: [0.0, 3.0, 4.0],
+       e.label_key: [1, 2, 3],
+       e.predicted_class_key: [1, 1, 3]})
+    results = e.all_metric_results()
+    self.assertEqual(set(["Avg_Loss", "Accuracy"]), set(results.keys()))
+    self.assertEqual(2.0, results["Avg_Loss"].numpy())
+    self.assertEqual(0.75, results["Accuracy"].numpy())
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/contrib/eager/python/metrics_impl.py b/tensorflow/contrib/eager/python/metrics_impl.py
index 6bc0ce6dce..63a0f8d9a4 100644
--- a/tensorflow/contrib/eager/python/metrics_impl.py
+++ b/tensorflow/contrib/eager/python/metrics_impl.py
@@ -28,9 +28,9 @@ from tensorflow.python.ops import variable_scope
 class Metric(object):
   """A metric holds state for aggregating statistics over an evaluation run.
 
-  Users will use Network.add_metric() to add Metric objects to their
-  evaluation network, call them in each step, and then use
-  Network.all_metric_results() at the end.
+  Users will use Evaluator.add_metric() to add Metric objects to their
+  evaluation, call them in each step, and then use
+  Evaluator.all_metric_results() at the end.
 
   Descendants will implement:
   * call(): Should follow this pattern:
@@ -54,7 +54,7 @@ class Metric(object):
     # a name/variable scope?
     # TODO(josh11b): self._in_graph_mode = context.in_graph_mode()
 
-  # ---- API for users ---
+  # ---- API for users ----
   def __call__(self, *args, **kwargs):
     # TODO(josh11b): If self._in_graph_mode is true, make self.call() into a
     # graph callable here, so that variable updates happen without requiring
@@ -65,6 +65,8 @@ class Metric(object):
     if not self.built:
       # TODO(ashankar): Set up container isolation so there is no chance
       # distinct metrics objects accidentally share variables.
+      # TODO(josh11b): Replace things like spaces in self._name to create
+      # a valid scope name.
       with variable_scope.variable_scope(
           self._name, use_resource=True, reuse=False):
         ret = self.call(*args, **kwargs)
@@ -96,7 +98,7 @@ class Metric(object):
   #   for the use case where they want to record the metric's state
   #   for each example and then later decide which examples they want
   #   to aggregate over. (Recommended -- not too much harder and adds
-  #   flexibilty over previous option.)
+  #   flexibility over previous option.)
   # I'm going with the second strategy since we can define a default
   # implementation of aggregate() that will work for most descendants.
   def aggregate(self, metrics):
@@ -121,7 +123,7 @@ class Metric(object):
       self._vars[i].assign_add(math_ops.add_n([m._vars[i] for m in metrics]))
     # pylint: enable=protected-access
 
-  def result(self):
+  def result(self):  # TODO(josh11b): Add an optional summary_writer parameter.
     """Computes and returns a final value for the metric."""
     raise NotImplementedError("Metrics must define a result() member function")
 
diff --git a/tensorflow/contrib/eager/python/metrics_test.py b/tensorflow/contrib/eager/python/metrics_test.py
index 8c2d8081ba..089bad5a0e 100644
--- a/tensorflow/contrib/eager/python/metrics_test.py
+++ b/tensorflow/contrib/eager/python/metrics_test.py
@@ -12,6 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
+"""Tests for Metrics."""
+
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
-- 
GitLab


From 2f3812501d3ac10165acfcae228e52f6f3358f7e Mon Sep 17 00:00:00 2001
From: Derek Murray <derek.murray@gmail.com>
Date: Fri, 13 Oct 2017 13:59:23 -0700
Subject: [PATCH 381/909] Add `cudnn_rnn_ops` to the Windows build

Fixes #13696.
---
 tensorflow/contrib/cmake/tf_core_kernels.cmake | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tensorflow/contrib/cmake/tf_core_kernels.cmake b/tensorflow/contrib/cmake/tf_core_kernels.cmake
index 46c680aad5..3a2fe35a3e 100644
--- a/tensorflow/contrib/cmake/tf_core_kernels.cmake
+++ b/tensorflow/contrib/cmake/tf_core_kernels.cmake
@@ -65,6 +65,8 @@ if(tensorflow_BUILD_CONTRIB_KERNELS)
       "${tensorflow_source_dir}/tensorflow/contrib/boosted_trees/ops/split_handler_ops.cc"
       "${tensorflow_source_dir}/tensorflow/contrib/boosted_trees/ops/stats_accumulator_ops.cc"
       "${tensorflow_source_dir}/tensorflow/contrib/boosted_trees/ops/training_ops.cc"
+      "${tensorflow_source_dir}/tensorflow/contrib/cudnn_rnn/kernels/cudnn_rnn_ops.cc"
+      "${tensorflow_source_dir}/tensorflow/contrib/cudnn_rnn/ops/cudnn_rnn_ops.cc"
       "${tensorflow_source_dir}/tensorflow/contrib/factorization/kernels/clustering_ops.cc"
       "${tensorflow_source_dir}/tensorflow/contrib/factorization/kernels/masked_matmul_ops.cc"
       "${tensorflow_source_dir}/tensorflow/contrib/factorization/kernels/wals_solver_ops.cc"
-- 
GitLab


From 0c2a50e951bb840e84b0bc643b85f104c59a10ef Mon Sep 17 00:00:00 2001
From: Asim Shankar <ashankar@google.com>
Date: Fri, 13 Oct 2017 14:04:21 -0700
Subject: [PATCH 382/909] eager: Fix issue with custom_gradients and
 implicit_gradients.

While at it, clean up some dead code/comments in tape.py

PiperOrigin-RevId: 172143125
---
 tensorflow/python/BUILD                    |  1 -
 tensorflow/python/eager/BUILD              | 10 ++++-----
 tensorflow/python/eager/backprop_test.py   | 26 ++++++++++++++++++++++
 tensorflow/python/eager/custom_gradient.py |  9 ++++++--
 tensorflow/python/eager/tape.py            | 17 +-------------
 5 files changed, 38 insertions(+), 25 deletions(-)

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 9582fda88f..2738022584 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -1838,7 +1838,6 @@ py_library(
         ":variables",
         "//tensorflow/core:protos_all_py",
         "//tensorflow/python/eager:context",
-        "//tensorflow/python/eager:custom_gradient",
         "//tensorflow/python/eager:tape",
     ],
 )
diff --git a/tensorflow/python/eager/BUILD b/tensorflow/python/eager/BUILD
index 3586311c92..d34ef3d55b 100644
--- a/tensorflow/python/eager/BUILD
+++ b/tensorflow/python/eager/BUILD
@@ -76,10 +76,6 @@ py_library(
     srcs = ["tape.py"],
     srcs_version = "PY2AND3",
     visibility = ["//tensorflow:internal"],
-    deps = [
-        "//tensorflow/python:dtypes",
-        "//tensorflow/python:util",
-    ],
 )
 
 py_library(
@@ -106,6 +102,7 @@ cuda_py_test(
     additional_deps = [
         ":backprop",
         ":context",
+        ":custom_gradient",
         ":test",
         "//tensorflow/python:embedding_ops",
         "//tensorflow/python:array_ops",
@@ -113,7 +110,6 @@ cuda_py_test(
         "//tensorflow/python:nn_ops",
         "//tensorflow/python:random_ops",
         "//tensorflow/python:nn_grad",
-        "//tensorflow/python:resource_variable_ops",
         "//tensorflow/python:training",
     ],
 )
@@ -233,9 +229,11 @@ py_library(
     srcs_version = "PY2AND3",
     visibility = ["//tensorflow:internal"],
     deps = [
-        ":core",
+        ":context",
         ":tape",
+        "//tensorflow/python:array_ops",
         "//tensorflow/python:framework_ops",
+        "//tensorflow/python:resource_variable_ops",
         "//tensorflow/python:util",
     ],
 )
diff --git a/tensorflow/python/eager/backprop_test.py b/tensorflow/python/eager/backprop_test.py
index 3b72974fc7..2409a7b198 100644
--- a/tensorflow/python/eager/backprop_test.py
+++ b/tensorflow/python/eager/backprop_test.py
@@ -21,6 +21,7 @@ import numpy as np
 from tensorflow.python import pywrap_tensorflow
 from tensorflow.python.eager import backprop
 from tensorflow.python.eager import context
+from tensorflow.python.eager import custom_gradient
 from tensorflow.python.eager import tape
 from tensorflow.python.eager import test
 from tensorflow.python.framework import constant_op
@@ -449,6 +450,31 @@ class BackpropTest(test.TestCase):
     # pylint: enable=protected-access
     context.context().clear_post_execution_callbacks()
 
+  def testImplicitGradientsCustomGradientAndCachedVariableValue(self):
+
+    @custom_gradient.custom_gradient
+    def my_square(x):
+      result = math_ops.square(x)
+
+      def grad(dr):
+        return 2 * dr * x + 1
+
+      return result, grad
+
+    x = resource_variable_ops.ResourceVariable(
+        initial_value=3, name='X.' + self.id())
+
+    def f():
+      return my_square(x)
+
+    g = backprop.implicit_grad(f)
+
+    grads_and_vars = g()
+    self.assertEqual(1, len(grads_and_vars))
+    grad, var = grads_and_vars[0]
+    self.assertEqual(7, grad.numpy())
+    self.assertEqual(x, var)
+
 
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/python/eager/custom_gradient.py b/tensorflow/python/eager/custom_gradient.py
index 87348e87b1..df116dd819 100644
--- a/tensorflow/python/eager/custom_gradient.py
+++ b/tensorflow/python/eager/custom_gradient.py
@@ -22,6 +22,7 @@ from tensorflow.python.eager import context
 from tensorflow.python.eager import tape
 from tensorflow.python.framework import ops as tf_ops
 from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.util import nest
 from tensorflow.python.util import tf_decorator
 
@@ -68,8 +69,12 @@ def custom_gradient(f):
       return nest.pack_sequence_as(
           structure=result, flat_sequence=all_tensors[:len(flat_result)])
 
-    input_tensors = [x for x in args
-                     if isinstance(x, tf_ops.Tensor)]
+    input_tensors = []
+    for x in args:
+      if isinstance(x, tf_ops.Tensor):
+        input_tensors.append(x)
+      if isinstance(x, resource_variable_ops.ResourceVariable):
+        input_tensors.append(x.read_value())
 
     with tape.stop_recording():
       result, grad_fn = f(*args, **kwargs)
diff --git a/tensorflow/python/eager/tape.py b/tensorflow/python/eager/tape.py
index 4578a7190d..76c6fa5ad8 100644
--- a/tensorflow/python/eager/tape.py
+++ b/tensorflow/python/eager/tape.py
@@ -22,8 +22,6 @@ import collections
 import contextlib
 import threading
 
-from tensorflow.python.util import tf_contextlib
-
 
 def tid(tensor):
   return tensor._id  # pylint: disable=protected-access
@@ -154,13 +152,6 @@ class _TapeStack(threading.local):
   def stack(self):
     return self._stack
 
-  @tf_contextlib.contextmanager
-  def replace_stack(self, new_stack):
-    old = self._stack
-    self._stack = new_stack
-    yield
-    self._stack = old
-
 
 # The global tape stack.
 _tape_stack = _TapeStack()
@@ -176,9 +167,6 @@ def watch(tensor):
 
   Args:
     tensor: tensor to be watched.
-
-  Returns:
-    The tensor, potentially wrapped by all tapes in the stack.
   """
   for t in _tape_stack.stack:
     t.watch(tensor)
@@ -189,9 +177,6 @@ def watch_variable(variable):
 
   Args:
     variable: variable to be watched.
-
-  Returns:
-    The tensor, potentially wrapped by all tapes in the stack.
   """
   for t in _tape_stack.stack:
     t.watch_variable(variable)
@@ -215,7 +200,7 @@ def stop_recording():
 
 
 def should_record(tensors):
-  """Returns true if any tape in the stach watches any of these tensors."""
+  """Returns true if any tape in the stack watches any of these tensors."""
   if not _tape_stack.stack:
     return False
   return any(x.should_record(tensors) for x in _tape_stack.stack)
-- 
GitLab


From ba3b33884b5b45096930b3849a2a0c4177f8a42b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 13 Oct 2017 14:58:26 -0700
Subject: [PATCH 383/909] Improve numerical accuracy of KL computation

PiperOrigin-RevId: 172150350
---
 .../gan/python/eval/python/classifier_metrics_impl.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/gan/python/eval/python/classifier_metrics_impl.py b/tensorflow/contrib/gan/python/eval/python/classifier_metrics_impl.py
index 3a6456f038..6074694f8b 100644
--- a/tensorflow/contrib/gan/python/eval/python/classifier_metrics_impl.py
+++ b/tensorflow/contrib/gan/python/eval/python/classifier_metrics_impl.py
@@ -317,13 +317,22 @@ def classifier_score(images, classifier_fn, num_batches=1):
       name='RunClassifier')
   logits = array_ops.concat(array_ops.unstack(logits), 0)
   logits.shape.assert_has_rank(2)
+
+  # Use maximum precision for best results.
+  logits_dtype = logits.dtype
+  if logits_dtype != dtypes.float64:
+    logits = math_ops.cast(logits, dtypes.float64)
+
   p = nn_ops.softmax(logits)
   q = math_ops.reduce_mean(p, axis=0)
   kl = _kl_divergence(p, logits, q)
   kl.shape.assert_has_rank(1)
   log_score = math_ops.reduce_mean(kl)
+  final_score = math_ops.exp(log_score)
 
-  return math_ops.exp(log_score)
+  if logits_dtype != dtypes.float64:
+    final_score = math_ops.cast(final_score, dtypes.float64)
+  return final_score
 
 
 inception_score = functools.partial(
-- 
GitLab


From 0bbdeaf45e07e1f5fb5e15961104e348e3ad8777 Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Fri, 13 Oct 2017 15:03:45 -0700
Subject: [PATCH 384/909] Ports the eager gradient tape to C.

The tape stack is still in python as is the backprop code.

PiperOrigin-RevId: 172151189
---
 tensorflow/c/eager/BUILD                   |  11 ++
 tensorflow/c/eager/tape.cc                 | 102 ++++++++++
 tensorflow/c/eager/tape.h                  |  96 +++++++++
 tensorflow/contrib/cmake/tf_c.cmake        |   2 +
 tensorflow/python/eager/BUILD              |   1 +
 tensorflow/python/eager/imperative_grad.py |   6 +-
 tensorflow/python/eager/pywrap_tfe.h       |  18 ++
 tensorflow/python/eager/pywrap_tfe_src.cc  | 215 +++++++++++++++++++++
 tensorflow/python/eager/tape.py            |  62 ++----
 tensorflow/python/pywrap_tfe.i             |   6 +
 10 files changed, 469 insertions(+), 50 deletions(-)
 create mode 100644 tensorflow/c/eager/tape.cc
 create mode 100644 tensorflow/c/eager/tape.h

diff --git a/tensorflow/c/eager/BUILD b/tensorflow/c/eager/BUILD
index d39f229b42..96f3c3e195 100644
--- a/tensorflow/c/eager/BUILD
+++ b/tensorflow/c/eager/BUILD
@@ -101,3 +101,14 @@ tf_cc_test(
         "//tensorflow/core:test_main",
     ],
 )
+
+cc_library(
+    name = "tape",
+    srcs = ["tape.cc"],
+    hdrs = ["tape.h"],
+    visibility = ["//tensorflow:internal"],
+    deps = [
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+    ],
+)
diff --git a/tensorflow/c/eager/tape.cc b/tensorflow/c/eager/tape.cc
new file mode 100644
index 0000000000..464612a81e
--- /dev/null
+++ b/tensorflow/c/eager/tape.cc
@@ -0,0 +1,102 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/c/eager/tape.h"
+
+namespace tensorflow {
+namespace eager {
+
+bool GradientTape::ShouldRecord(gtl::ArraySlice<int64> tensor_ids) {
+  for (int64 i : tensor_ids) {
+    if (tensor_tape_.find(i) != tensor_tape_.end()) {
+      return true;
+    }
+  }
+  return false;
+}
+
+void GradientTape::Watch(int64 tensor_id) {
+  tensor_tape_.emplace(tensor_id, -1);
+}
+
+void GradientTape::RecordOperation(
+    const string& op_type, gtl::ArraySlice<TapeTensor> output_tensors,
+    gtl::ArraySlice<int64> input_tensor_id, void* backward_function,
+    const std::function<void()>& backward_function_deleter) {
+  if (!ShouldRecord(input_tensor_id)) {
+    backward_function_deleter();
+    return;
+  }
+  std::vector<int64> ids;
+  ids.reserve(input_tensor_id.size());
+  for (int64 i : input_tensor_id) {
+    tensor_usage_[i]++;
+    ids.push_back(i);
+  }
+  const int64 op_id = next_op_id_++;
+  std::vector<TapeTensor> tensors;
+  tensors.reserve(output_tensors.size());
+  for (const TapeTensor& o : output_tensors) {
+    // Note: the tensor can have already been watched and hence be in the tape,
+    // so we cannot check that we're inserting it here.
+    tensor_tape_[o.id] = op_id;
+    tensor_usage_[o.id] = 1;
+    tensors.push_back(o);
+  }
+  op_tape_[op_id] = OpTapeEntry{op_type, tensors, ids, backward_function,
+                                backward_function_deleter};
+}
+
+void GradientTape::DeleteTrace(int64 tensor_id) {
+  auto it = tensor_usage_.find(tensor_id);
+  if (it == tensor_usage_.end()) {
+    return;
+  }
+  it->second--;
+  if (it->second != 0) {
+    return;
+  }
+  tensor_usage_.erase(it);
+  auto tensor_op_it = tensor_tape_.find(tensor_id);
+  if (tensor_op_it == tensor_tape_.end()) {
+    return;
+  }
+  const int64 op_id = tensor_op_it->second;
+  if (op_id == -1) {
+    // Do not delete watched tensors.
+    return;
+  }
+  tensor_tape_.erase(tensor_op_it);
+  auto op_it = op_tape_.find(op_id);
+  CHECK(op_it != op_tape_.end());
+  for (const auto& output : op_it->second.output_tensor_info) {
+    if (tensor_usage_.find(output.id) != tensor_usage_.end()) {
+      // Found a usage for an output, so cannot delete the op.
+      return;
+    }
+  }
+  for (int64 id : op_it->second.input_tensor_id) {
+    DeleteTrace(id);
+  }
+  op_it->second.backward_function_deleter();
+  op_tape_.erase(op_it);
+}
+
+std::pair<TensorTape, OpTape> GradientTape::Export() {
+  return {std::move(tensor_tape_), std::move(op_tape_)};
+}
+
+}  // namespace eager
+}  // namespace tensorflow
diff --git a/tensorflow/c/eager/tape.h b/tensorflow/c/eager/tape.h
new file mode 100644
index 0000000000..df51f300eb
--- /dev/null
+++ b/tensorflow/c/eager/tape.h
@@ -0,0 +1,96 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_C_EAGER_TAPE_H_
+#define TENSORFLOW_C_EAGER_TAPE_H_
+
+// Language-agnostic gradient tape. Does not perform backpropagation, just
+// maintains the data structures required to do so.
+
+#include <unordered_map>
+#include <vector>
+#include "tensorflow/core/framework/tensor_shape.h"
+#include "tensorflow/core/framework/types.h"
+#include "tensorflow/core/lib/gtl/array_slice.h"
+#include "tensorflow/core/platform/types.h"
+
+namespace tensorflow {
+namespace eager {
+
+// Information about a tensor.
+struct TapeTensor {
+  int64 id;  // Expected to be unique in the lifetime of this process.
+  DataType dtype;
+  TensorShape shape;
+};
+
+// Represents an entry in the tape.
+struct OpTapeEntry {
+  string op_type;
+  std::vector<TapeTensor> output_tensor_info;
+  std::vector<int64> input_tensor_id;
+
+  // TODO(apassos) consider narrowing down this interface.
+  void* backward_function;
+
+  // Should be called before deleting the backward function. TODO(apassos) use
+  // unique_ptrs to ensure this happens.
+  std::function<void()> backward_function_deleter;
+};
+
+// Map from tensor_id to internally-defined operation-id of the operation which
+// produced this tensor. A value of -1 means that the tensor was directly
+// watched and not the result of any operation in the tape.
+using TensorTape = std::unordered_map<int64, int64>;
+
+// Map from operation-id to tape entry.
+using OpTape = std::unordered_map<int64, OpTapeEntry>;
+
+// Traces the execution of operations, doing eager garbage collection, and
+// exporting a full trace so other code can do backpropagation. Not thread-safe.
+class GradientTape {
+ public:
+  GradientTape() {}
+
+  bool ShouldRecord(gtl::ArraySlice<int64> tensor_ids);
+
+  void Watch(int64 tensor_id);
+
+  void RecordOperation(const string& op_type,
+                       gtl::ArraySlice<TapeTensor> output_tensors,
+                       gtl::ArraySlice<int64> input_tensor_id,
+                       void* backward_function,
+                       const std::function<void()>& backward_function_deleter);
+
+  void DeleteTrace(int64 tensor_id);
+
+  // Note: it is only valid to call Export once per tape, and after calling
+  // export the tape is no longer valid (i.e. calls to ShouldRecord, Watch,
+  // Record, and Delete have undefined behavior).
+  std::pair<TensorTape, OpTape> Export();
+
+ private:
+  TensorTape tensor_tape_;
+  OpTape op_tape_;
+  int64 next_op_id_{0};
+
+  // Map from tensor id to number of remaining usages (i.e. how many entries in
+  // the tape refer to it); to aid in tape garbage collection.
+  std::unordered_map<int64, int64> tensor_usage_;
+};
+
+}  // namespace eager
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_C_EAGER_TAPE_H_
diff --git a/tensorflow/contrib/cmake/tf_c.cmake b/tensorflow/contrib/cmake/tf_c.cmake
index c5a1018127..f3882e8cf7 100644
--- a/tensorflow/contrib/cmake/tf_c.cmake
+++ b/tensorflow/contrib/cmake/tf_c.cmake
@@ -21,6 +21,8 @@ set(tf_c_srcs
     "${tensorflow_source_dir}/tensorflow/c/c_api_function.cc"
     "${tensorflow_source_dir}/tensorflow/c/eager/c_api.cc"
     "${tensorflow_source_dir}/tensorflow/c/eager/c_api.h"
+    "${tensorflow_source_dir}/tensorflow/c/eager/tape.cc"
+    "${tensorflow_source_dir}/tensorflow/c/eager/tape.h"
     "${tensorflow_source_dir}/tensorflow/c/eager/runtime.cc"
     "${tensorflow_source_dir}/tensorflow/c/eager/runtime.h"
     "${tensorflow_source_dir}/tensorflow/c/checkpoint_reader.cc"
diff --git a/tensorflow/python/eager/BUILD b/tensorflow/python/eager/BUILD
index d34ef3d55b..1d20a0782f 100644
--- a/tensorflow/python/eager/BUILD
+++ b/tensorflow/python/eager/BUILD
@@ -15,6 +15,7 @@ cc_library(
     deps = [
         "//tensorflow/c:c_api",
         "//tensorflow/c/eager:c_api",
+        "//tensorflow/c/eager:tape",
         "//tensorflow/core:lib",
         "//tensorflow/python:ndarray_tensor",
         "//tensorflow/python:ndarray_tensor_bridge",
diff --git a/tensorflow/python/eager/imperative_grad.py b/tensorflow/python/eager/imperative_grad.py
index f388d0a148..dd9d691d26 100644
--- a/tensorflow/python/eager/imperative_grad.py
+++ b/tensorflow/python/eager/imperative_grad.py
@@ -73,10 +73,10 @@ def _prepare_backprop(vspace, target, tensor_to_op, op_to_entry, id_sources):
   while tensor_stack:
     t = tensor_stack.pop()
     op = tensor_to_op.get(t, None)
-    # op is None if the tensor is a source (i.e. was watched directly)
-    if op is None or op in o_to_e:
+    # op is None or -1 if the tensor is a source (i.e. was watched directly)
+    if op is None or op == -1 or op in o_to_e:
       continue
-    op_trace = op_to_entry[op]
+    op_trace = tape.TapeEntry(*op_to_entry[op])
     o_to_e[op] = op_trace
     for it in op_trace.input_ids:
       if it in tensor_usage_counts:
diff --git a/tensorflow/python/eager/pywrap_tfe.h b/tensorflow/python/eager/pywrap_tfe.h
index 5a72f422cf..9834095c87 100644
--- a/tensorflow/python/eager/pywrap_tfe.h
+++ b/tensorflow/python/eager/pywrap_tfe.h
@@ -87,4 +87,22 @@ TFE_TensorHandle* EagerTensorHandle(const PyObject* o);
 // newly created type, or nullptr on error.
 PyObject* TFE_Py_InitEagerTensor(PyObject* base_class);
 
+PyObject* TFE_Py_NewTape();
+PyObject* TFE_Py_TapeShouldRecord(PyObject* py_tape, PyObject* tensors);
+void TFE_Py_TapeWatch(PyObject* tape, tensorflow::int64 tensor_id);
+void TFE_Py_TapeDeleteTrace(PyObject* tape, tensorflow::int64 tensor_id);
+
+// Records an operation in the gradient tape. `tape` should point to an object
+// returned by TFE_Py_NewTape. op_type is a string for the operation type, used
+// in the backprop code. output_tensors should be a list of python ops.Tensor
+// objects. input_tensor_ids should be a list of python integers with the ids of
+// the input tensors of the recorded operation. backward_function should be the
+// function to be called during backprop to, given the gradients of the output
+// tensors, produce the gradients of the input tensors.
+void TFE_Py_TapeRecordOperation(PyObject* tape, PyObject* op_type,
+                                PyObject* output_tensors,
+                                PyObject* input_tensor_ids,
+                                PyObject* backward_function);
+PyObject* TFE_Py_TapeExport(PyObject* tape);
+
 #endif  // TENSORFLOW_PYTHON_EAGER_PYWRAP_TFE_H_
diff --git a/tensorflow/python/eager/pywrap_tfe_src.cc b/tensorflow/python/eager/pywrap_tfe_src.cc
index 3d64c875ec..402b84d7c6 100644
--- a/tensorflow/python/eager/pywrap_tfe_src.cc
+++ b/tensorflow/python/eager/pywrap_tfe_src.cc
@@ -16,6 +16,7 @@ limitations under the License.
 #include "tensorflow/python/eager/pywrap_tfe.h"
 
 #include "tensorflow/c/c_api.h"
+#include "tensorflow/c/eager/tape.h"
 #include "tensorflow/core/lib/strings/strcat.h"
 #include "tensorflow/core/platform/mutex.h"
 #include "tensorflow/core/platform/types.h"
@@ -438,3 +439,217 @@ void TFE_DeleteContextCapsule(PyObject* context) {
   TFE_DeleteContext(ctx, status);
   TF_DeleteStatus(status);
 }
+
+typedef struct {
+  PyObject_HEAD
+      /* Type-specific fields go here. */
+      tensorflow::eager::GradientTape* tape;
+} TFE_Py_Tape;
+
+static void TFE_Py_Tape_Delete(PyObject* tape) {
+  delete reinterpret_cast<TFE_Py_Tape*>(tape)->tape;
+  Py_TYPE(tape)->tp_free(tape);
+}
+
+static PyTypeObject TFE_Py_Tape_Type = {
+    PyVarObject_HEAD_INIT(NULL, 0) "tfe.Tape", /* tp_name */
+    sizeof(TFE_Py_Tape),                       /* tp_basicsize */
+    0,                                         /* tp_itemsize */
+    &TFE_Py_Tape_Delete,                       /* tp_dealloc */
+    0,                                         /* tp_print */
+    0,                                         /* tp_getattr */
+    0,                                         /* tp_setattr */
+    0,                                         /* tp_reserved */
+    0,                                         /* tp_repr */
+    0,                                         /* tp_as_number */
+    0,                                         /* tp_as_sequence */
+    0,                                         /* tp_as_mapping */
+    0,                                         /* tp_hash  */
+    0,                                         /* tp_call */
+    0,                                         /* tp_str */
+    0,                                         /* tp_getattro */
+    0,                                         /* tp_setattro */
+    0,                                         /* tp_as_buffer */
+    Py_TPFLAGS_DEFAULT,                        /* tp_flags */
+    "TFE_Py_Tape objects",                     /* tp_doc */
+};
+
+PyObject* TFE_Py_NewTape() {
+  TFE_Py_Tape_Type.tp_new = PyType_GenericNew;
+  if (PyType_Ready(&TFE_Py_Tape_Type) < 0) return nullptr;
+  TFE_Py_Tape* tape = PyObject_NEW(TFE_Py_Tape, &TFE_Py_Tape_Type);
+  tape->tape = new tensorflow::eager::GradientTape();
+  return reinterpret_cast<PyObject*>(tape);
+}
+
+static tensorflow::int64 MakeInt(PyObject* integer) {
+#if PY_MAJOR_VERSION >= 3
+  return PyLong_AsLong(integer);
+#else
+  return PyInt_AsLong(integer);
+#endif
+}
+
+static std::vector<tensorflow::int64> MakeIntList(PyObject* list) {
+  if (list == Py_None) {
+    return {};
+  }
+  PyObject* seq = PySequence_Fast(list, "expected a sequence");
+  if (seq == nullptr) {
+    return {};
+  }
+  int len = PySequence_Size(list);
+  std::vector<tensorflow::int64> tensor_ids;
+  tensor_ids.reserve(len);
+  for (int i = 0; i < len; ++i) {
+    PyObject* item = PySequence_Fast_GET_ITEM(seq, i);
+    if (PyLong_Check(item)) {
+      tensorflow::int64 id = MakeInt(item);
+      tensor_ids.push_back(id);
+    } else {
+      tensor_ids.push_back(-1);
+    }
+  }
+  Py_DECREF(seq);
+  return tensor_ids;
+}
+
+PyObject* TFE_Py_TapeShouldRecord(PyObject* py_tape, PyObject* tensors) {
+  TFE_Py_Tape* tape = reinterpret_cast<TFE_Py_Tape*>(py_tape);
+  return PyBool_FromLong(tape->tape->ShouldRecord(MakeIntList(tensors)));
+}
+
+void TFE_Py_TapeWatch(PyObject* tape, tensorflow::int64 tensor_id) {
+  reinterpret_cast<TFE_Py_Tape*>(tape)->tape->Watch(tensor_id);
+}
+
+// TODO(apassos) have a fast path for eager tensors here which gets information
+// from the handle instead of from the python object, and use this only for the
+// case of graph tensors.
+static tensorflow::eager::TapeTensor TapeTensorFromTensor(PyObject* tensor) {
+  PyObject* id_field = PyObject_GetAttrString(tensor, "_id");
+  tensorflow::int64 id = MakeInt(id_field);
+  Py_DECREF(id_field);
+  if (PyErr_Occurred() != nullptr) {
+    return tensorflow::eager::TapeTensor{
+        id, static_cast<tensorflow::DataType>(0), tensorflow::TensorShape({})};
+  }
+  PyObject* dtype_object = PyObject_GetAttrString(tensor, "dtype");
+  PyObject* dtype_enum = PyObject_GetAttrString(dtype_object, "_type_enum");
+  Py_DECREF(dtype_object);
+  tensorflow::DataType dtype =
+      static_cast<tensorflow::DataType>(MakeInt(dtype_enum));
+  Py_DECREF(dtype_enum);
+  if (PyErr_Occurred() != nullptr) {
+    return tensorflow::eager::TapeTensor{id, dtype,
+                                         tensorflow::TensorShape({})};
+  }
+  static char _shape_tuple[] = "_shape_tuple";
+  PyObject* shape_tuple = PyObject_CallMethod(tensor, _shape_tuple, nullptr);
+  if (PyErr_Occurred() != nullptr) {
+    return tensorflow::eager::TapeTensor{id, dtype,
+                                         tensorflow::TensorShape({})};
+  }
+  auto l = MakeIntList(shape_tuple);
+  Py_DECREF(shape_tuple);
+  // Replace -1, which represents accidental Nones which can occur in graph mode
+  // and can cause errors in shape cosntruction with 0s.
+  for (auto& c : l) {
+    if (c < 0) {
+      c = 0;
+    }
+  }
+  tensorflow::TensorShape shape(l);
+  return tensorflow::eager::TapeTensor{id, dtype, shape};
+}
+
+void TFE_Py_TapeRecordOperation(PyObject* tape, PyObject* op_type,
+                                PyObject* output_tensors,
+                                PyObject* input_tensor_ids,
+                                PyObject* backward_function) {
+  std::vector<tensorflow::int64> input_ids = MakeIntList(input_tensor_ids);
+  std::vector<tensorflow::eager::TapeTensor> output_info;
+  PyObject* seq = PySequence_Fast(output_tensors,
+                                  "expected a sequence of integer tensor ids");
+  int len = PySequence_Size(output_tensors);
+  output_info.reserve(len);
+  for (int i = 0; i < len; ++i) {
+    output_info.push_back(
+        TapeTensorFromTensor(PySequence_Fast_GET_ITEM(seq, i)));
+    if (PyErr_Occurred() != nullptr) {
+      Py_DECREF(seq);
+      return;
+    }
+  }
+  Py_DECREF(seq);
+  Py_INCREF(backward_function);
+  reinterpret_cast<TFE_Py_Tape*>(tape)->tape->RecordOperation(
+      PyBytes_AsString(op_type), output_info, input_ids, backward_function,
+      [backward_function]() { Py_DECREF(backward_function); });
+}
+
+void TFE_Py_TapeDeleteTrace(PyObject* tape, tensorflow::int64 tensor_id) {
+  reinterpret_cast<TFE_Py_Tape*>(tape)->tape->DeleteTrace(tensor_id);
+}
+
+// TODO(apassos) when backprop.py moves to C most of this exporting logic can
+// disappear.
+PyObject* TFE_Py_TapeExport(PyObject* tape) {
+  std::pair<tensorflow::eager::TensorTape, tensorflow::eager::OpTape> exported =
+      reinterpret_cast<TFE_Py_Tape*>(tape)->tape->Export();
+  PyObject* tensor_tape = PyDict_New();
+  for (const auto& pair : exported.first) {
+    PyObject* tid = PyLong_FromLong(pair.first);
+    PyObject* opid = PyLong_FromLong(pair.second);
+    PyDict_SetItem(tensor_tape, tid, opid);
+    Py_DECREF(tid);
+    Py_DECREF(opid);
+  }
+
+  PyObject* op_tape = PyDict_New();
+  for (const auto& pair : exported.second) {
+    PyObject* opid = PyLong_FromLong(pair.first);
+    const auto& entry = pair.second;
+    PyObject* op_type = PyBytes_FromString(entry.op_type.c_str());
+    PyObject* output_ids = PyList_New(entry.output_tensor_info.size());
+    for (int i = 0; i < entry.output_tensor_info.size(); ++i) {
+      PyObject* tid = PyLong_FromLong(entry.output_tensor_info[i].id);
+      PyList_SET_ITEM(output_ids, i, tid);
+    }
+    PyObject* input_ids = PyList_New(entry.input_tensor_id.size());
+    for (int i = 0; i < entry.input_tensor_id.size(); ++i) {
+      PyObject* tid = PyLong_FromLong(entry.input_tensor_id[i]);
+      PyList_SET_ITEM(input_ids, i, tid);
+    }
+    PyObject* backward_function =
+        reinterpret_cast<PyObject*>(entry.backward_function);
+    PyObject* output_shape_and_dtype =
+        PyList_New(entry.output_tensor_info.size());
+    for (int i = 0; i < entry.output_tensor_info.size(); ++i) {
+      const tensorflow::TensorShape& shape = entry.output_tensor_info[i].shape;
+      PyObject* shape_list = PyList_New(shape.dims());
+      for (int j = 0; j < shape.dims(); ++j) {
+        PyList_SET_ITEM(shape_list, j, PyLong_FromLong(shape.dim_size(j)));
+      }
+      PyObject* type_enum = PyLong_FromLong(entry.output_tensor_info[i].dtype);
+      PyObject* tuple = PyTuple_Pack(2, shape_list, type_enum);
+      Py_DECREF(shape_list);
+      Py_DECREF(type_enum);
+      PyList_SET_ITEM(output_shape_and_dtype, i, tuple);
+    }
+    PyObject* opinfo = PyTuple_Pack(5, op_type, output_ids, input_ids,
+                                    backward_function, output_shape_and_dtype);
+    Py_DECREF(op_type);
+    Py_DECREF(output_ids);
+    Py_DECREF(input_ids);
+    Py_DECREF(backward_function);
+    Py_DECREF(output_shape_and_dtype);
+    PyDict_SetItem(op_tape, opid, opinfo);
+    Py_DECREF(opid);
+    Py_DECREF(opinfo);
+  }
+  PyObject* retval = PyTuple_Pack(2, tensor_tape, op_tape);
+  Py_DECREF(tensor_tape);
+  Py_DECREF(op_tape);
+  return retval;
+}
diff --git a/tensorflow/python/eager/tape.py b/tensorflow/python/eager/tape.py
index 76c6fa5ad8..c16aa8c2f7 100644
--- a/tensorflow/python/eager/tape.py
+++ b/tensorflow/python/eager/tape.py
@@ -22,6 +22,9 @@ import collections
 import contextlib
 import threading
 
+from tensorflow.python import pywrap_tensorflow
+from tensorflow.python.util import compat
+
 
 def tid(tensor):
   return tensor._id  # pylint: disable=protected-access
@@ -56,16 +59,7 @@ class Tape(object):
   """Represents a gradient propagation trace."""
 
   def __init__(self):
-    # _tensor_tape maps from tensor IDs to their operation IDs
-    self._tensor_tape = {}
-    # maps from tensor ID to usage count. Triggers garbage collection when this
-    # goes to zero.
-    self._tensor_usage = {}
-    # maps from operation ID to TapeEntry
-    self._op_tape = {}
-    # next operation ID
-    self._next_op_id = 0
-    # Set of directly watched variables
+    self._tape = pywrap_tensorflow.TFE_Py_NewTape()
     self._watched_variables = set()
 
   def should_record(self, tensors):
@@ -77,14 +71,12 @@ class Tape(object):
     Returns:
       True if any of the tensors is in the tape.
     """
-    return any(x._id in self._tensor_tape for x in tensors)  # pylint: disable=protected-access
+    return pywrap_tensorflow.TFE_Py_TapeShouldRecord(
+        self._tape, [x._id  for x in tensors])  # pylint: disable=protected-access
 
   def watch(self, tensor):
     """Adds a tensor to the tape."""
-    i = tid(tensor)
-    if i not in self._tensor_tape:
-      self._tensor_tape[i] = None
-      self._tensor_usage[i] = 1
+    pywrap_tensorflow.TFE_Py_TapeWatch(self._tape, tid(tensor))
 
   def watch_variable(self, v):
     self._watched_variables.add(v)
@@ -93,39 +85,15 @@ class Tape(object):
   def record_operation(self, op_type, output_tensors, input_tensors,
                        backward_function):
     """Records an operation in the tape."""
-    if not self.should_record(input_tensors):
-      return output_tensors
-    for t in output_tensors:
-      i = tid(t)
-      self._tensor_tape[i] = self._next_op_id
-      self._tensor_usage[i] = 1
-    for t in input_tensors:
-      i = tid(t)
-      self._tensor_usage[i] = self._tensor_usage.get(i, 0) + 1
-    self._op_tape[self._next_op_id] = TapeEntry(
-        op_type,
-        [tid(t) for t in output_tensors],
-        [tid(t) for t in input_tensors],
-        backward_function,
-        [(_tensor_shape(t), t.dtype) for t in output_tensors])
-    self._next_op_id += 1
+    pywrap_tensorflow.TFE_Py_TapeRecordOperation(
+        self._tape,
+        compat.as_bytes(op_type),
+        output_tensors,
+        [x._id for x in input_tensors],  # pylint: disable=protected-access
+        backward_function)
 
   def _delete_tensor_id(self, i):
-    if i in self._tensor_usage:
-      self._tensor_usage[i] -= 1
-      if self._tensor_usage[i] == 0:
-        del self._tensor_usage[i]
-        op_id = self._tensor_tape.pop(i, None)
-        if op_id is None:
-          return
-        op = self._op_tape[op_id]
-        if not any(tensor_id in self._tensor_usage
-                   for tensor_id in op.output_ids):
-          del self._op_tape[op_id]
-          for tensor_id in op.input_ids:
-            # TODO(apassos) this recursion might come to bite us. Consider
-            # adding an explicit stack if this ever gets out of hand
-            self._delete_tensor_id(tensor_id)
+    pywrap_tensorflow.TFE_Py_TapeDeleteTrace(self._tape, i)
 
   def delete_trace(self, tensor_id):
     """Deletes any trace we have for this tensor."""
@@ -139,7 +107,7 @@ class Tape(object):
        responsible for generating that tensor.
       op_tape: a map from <identifier for op> to TapeEntry for that op.
     """
-    return self._tensor_tape, self._op_tape
+    return pywrap_tensorflow.TFE_Py_TapeExport(self._tape)
 
 
 class _TapeStack(threading.local):
diff --git a/tensorflow/python/pywrap_tfe.i b/tensorflow/python/pywrap_tfe.i
index d5b7294c82..5c624a9c12 100644
--- a/tensorflow/python/pywrap_tfe.i
+++ b/tensorflow/python/pywrap_tfe.i
@@ -24,6 +24,12 @@ limitations under the License.
 %rename("%s") TFE_Py_RegisterExceptionClass;
 %rename("%s") TFE_Py_Execute;
 %rename("%s") TFE_Py_UID;
+%rename("%s") TFE_Py_NewTape;
+%rename("%s") TFE_Py_TapeShouldRecord;
+%rename("%s") TFE_Py_TapeWatch;
+%rename("%s") TFE_Py_TapeDeleteTrace;
+%rename("%s") TFE_Py_TapeRecordOperation;
+%rename("%s") TFE_Py_TapeExport;
 
 
 %{
-- 
GitLab


From 7679a2ec746bec36191087feaf9ec8371180669c Mon Sep 17 00:00:00 2001
From: Reed Wanderman-Milne <reedwm@google.com>
Date: Fri, 13 Oct 2017 15:22:19 -0700
Subject: [PATCH 385/909] Fix crash if tf.nn.conv2d_backprop_filter or
 tf.nn.conv2d_backprop_input is run with empty filter or input respectively.
 Resolves #13643.

PiperOrigin-RevId: 172153646
---
 .../core/kernels/conv_grad_filter_ops.cc      | 15 ++++++++++
 .../core/kernels/conv_grad_input_ops.cc       | 15 ++++++++++
 .../python/kernel_tests/conv_ops_test.py      | 29 +++++++++++++++++++
 3 files changed, 59 insertions(+)

diff --git a/tensorflow/core/kernels/conv_grad_filter_ops.cc b/tensorflow/core/kernels/conv_grad_filter_ops.cc
index 5e09963d2d..3d2bb57aff 100644
--- a/tensorflow/core/kernels/conv_grad_filter_ops.cc
+++ b/tensorflow/core/kernels/conv_grad_filter_ops.cc
@@ -221,6 +221,11 @@ class Conv2DFastBackpropFilterOp : public OpKernel {
     OP_REQUIRES_OK(context,
                    context->allocate_output(0, filter_shape, &filter_backprop));
 
+    // If there is nothing to compute, return.
+    if (filter_shape.num_elements() == 0) {
+      return;
+    }
+
 #if defined TENSORFLOW_USE_LIBXSMM && defined TENSORFLOW_USE_LIBXSMM_BACKWARD
     int64 pad_top, pad_bottom;
     int64 pad_left, pad_right;
@@ -313,6 +318,11 @@ class Conv2DCustomBackpropFilterOp : public OpKernel {
     OP_REQUIRES_OK(context,
                    context->allocate_output(0, filter_shape, &filter_backprop));
 
+    // If there is nothing to compute, return.
+    if (filter_shape.num_elements() == 0) {
+      return;
+    }
+
     int64 pad_top, pad_bottom;
     int64 pad_left, pad_right;
     OP_REQUIRES_OK(
@@ -527,6 +537,11 @@ class Conv2DSlowBackpropFilterOp : public OpKernel {
     OP_REQUIRES_OK(context,
                    context->allocate_output(0, filter_shape, &filter_backprop));
 
+    // If there is nothing to compute, return.
+    if (filter_shape.num_elements() == 0) {
+      return;
+    }
+
     // For now we take the stride from the second and third dimensions only (we
     // do not support striding on the batch or depth dimension).
     const int stride_rows = GetTensorDim(strides_, data_format_, 'H');
diff --git a/tensorflow/core/kernels/conv_grad_input_ops.cc b/tensorflow/core/kernels/conv_grad_input_ops.cc
index 0b2d01afa9..d28f6b4d10 100644
--- a/tensorflow/core/kernels/conv_grad_input_ops.cc
+++ b/tensorflow/core/kernels/conv_grad_input_ops.cc
@@ -225,6 +225,11 @@ class Conv2DFastBackpropInputOp : public OpKernel {
     OP_REQUIRES_OK(context,
                    context->allocate_output(0, input_shape, &in_backprop));
 
+    // If there is nothing to compute, return.
+    if (input_shape.num_elements() == 0) {
+      return;
+    }
+
 #if defined TENSORFLOW_USE_LIBXSMM && defined TENSORFLOW_USE_LIBXSMM_BACKWARD
     int64 pad_top, pad_bottom;
     int64 pad_left, pad_right;
@@ -318,6 +323,11 @@ class Conv2DCustomBackpropInputOp : public OpKernel {
     OP_REQUIRES_OK(context,
                    context->allocate_output(0, input_shape, &in_backprop));
 
+    // If there is nothing to compute, return.
+    if (input_shape.num_elements() == 0) {
+      return;
+    }
+
 // TODO(andydavis) Consider moving code shared with
 // Conv2DCustomBackpropFilterOp into a shared helper function.
 #if defined TENSORFLOW_USE_LIBXSMM && defined TENSORFLOW_USE_LIBXSMM_BACKWARD
@@ -603,6 +613,11 @@ class Conv2DSlowBackpropInputOp : public OpKernel {
     OP_REQUIRES_OK(context,
                    context->allocate_output(0, input_shape, &in_backprop));
 
+    // If there is nothing to compute, return.
+    if (input_shape.num_elements() == 0) {
+      return;
+    }
+
     // For now we take the stride from the second and third dimensions only (we
     // do not support striding on the batch or depth dimension).
     const int stride_rows = GetTensorDim(strides_, data_format_, 'H');
diff --git a/tensorflow/python/kernel_tests/conv_ops_test.py b/tensorflow/python/kernel_tests/conv_ops_test.py
index 4c5b72671c..22e5400c37 100644
--- a/tensorflow/python/kernel_tests/conv_ops_test.py
+++ b/tensorflow/python/kernel_tests/conv_ops_test.py
@@ -480,6 +480,21 @@ class Conv2DTest(test.TestCase):
           use_gpu=use_gpu,
           err=1e-5)
 
+  @test_util.run_in_graph_and_eager_modes()
+  def testConv2DEmptyBackpropInput(self):
+    expected_output = []
+    for (data_format, use_gpu) in GetTestConfigs():
+      self._RunAndVerifyBackpropInput(
+          input_sizes=[0, 2, 3, 1],
+          filter_sizes=[2, 2, 1, 1],
+          output_sizes=[0, 1, 2, 1],
+          strides=[1, 1],
+          padding="VALID",
+          expected=expected_output,
+          data_format=data_format,
+          use_gpu=use_gpu,
+          err=1e-5)
+
   @test_util.run_in_graph_and_eager_modes()
   def testConv2D2x2Depth3ValidBackpropInput(self):
     expected_output = [
@@ -634,6 +649,20 @@ class Conv2DTest(test.TestCase):
           data_format=data_format,
           use_gpu=use_gpu)
 
+  @test_util.run_in_graph_and_eager_modes()
+  def testConv2DEmptyBackpropFilter(self):
+    expected = []
+    for (data_format, use_gpu) in GetTestConfigs():
+      self._RunAndVerifyBackpropFilter(
+          input_sizes=[1, 2, 3, 1],
+          filter_sizes=[2, 2, 1, 0],
+          output_sizes=[1, 1, 2, 0],
+          strides=[1, 1],
+          padding="VALID",
+          expected=expected,
+          data_format=data_format,
+          use_gpu=use_gpu)
+
   @test_util.run_in_graph_and_eager_modes()
   def testConv2D2x2Depth3ValidBackpropFilter(self):
     expected = [
-- 
GitLab


From 40d5bf33829249404f935441bac0fa1615a58c13 Mon Sep 17 00:00:00 2001
From: Skye Wanderman-Milne <skyewm@google.com>
Date: Fri, 13 Oct 2017 16:01:24 -0700
Subject: [PATCH 386/909] Enable Operation._add_control_inputs() with the C API
 and related improvements

This change:
 - Implements the C API logic for Operation._add_control_inputs()
 - Adds type-checking to Operation._add_control_input()
 - Makes Graph::AddControlEdge() update the node def if necessary
 - Makes Graph::AddControlEdge() a no-op if the control edge already exists

The AddControlEdge() changes may have a performance impact if anything
is sensitive to AddControlEdge(), but nothing is to my knowledge. I'm
not sure what benchmarks would confirm this.

PiperOrigin-RevId: 172158589
---
 .../tf2xla/functionalize_control_flow.cc      |  8 ++-
 tensorflow/core/graph/graph.cc                | 29 ++++++++
 tensorflow/core/graph/graph.h                 | 34 ++++++----
 tensorflow/core/graph/graph_partition.cc      |  5 +-
 tensorflow/core/graph/graph_test.cc           | 66 +++++++++++++++++++
 tensorflow/python/framework/ops.py            | 18 +++--
 tensorflow/python/framework/ops_test.py       | 25 ++++++-
 7 files changed, 160 insertions(+), 25 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/functionalize_control_flow.cc b/tensorflow/compiler/tf2xla/functionalize_control_flow.cc
index b9b2b4be27..40bc164c50 100644
--- a/tensorflow/compiler/tf2xla/functionalize_control_flow.cc
+++ b/tensorflow/compiler/tf2xla/functionalize_control_flow.cc
@@ -475,9 +475,11 @@ Status FunctionalizeLoop(Graph* graph, Frame* frame,
           int dst_input = edge->dst_input();
           graph->RemoveEdge(edge);
 
-          int src_output =
-              dst_input == Graph::kControlSlot ? Graph::kControlSlot : i;
-          graph->AddEdge(while_node, src_output, dst, dst_input);
+          if (dst_input == Graph::kControlSlot) {
+            graph->AddControlEdge(while_node, dst);
+          } else {
+            graph->AddEdge(while_node, i, dst, dst_input);
+          }
         }
       }
     }
diff --git a/tensorflow/core/graph/graph.cc b/tensorflow/core/graph/graph.cc
index daefb6b1fb..87c41186d5 100644
--- a/tensorflow/core/graph/graph.cc
+++ b/tensorflow/core/graph/graph.cc
@@ -424,6 +424,35 @@ void Graph::RemoveEdge(const Edge* e) {
   --num_edges_;
 }
 
+const Edge* Graph::AddControlEdge(Node* source, Node* dest,
+                                  bool allow_duplicates) {
+  if (!allow_duplicates) {
+    for (const Edge* edge : dest->in_edges()) {
+      if (edge->IsControlEdge() && edge->src() == source) {
+        // The requested edge already exists.
+        return nullptr;
+      }
+    }
+  }
+  // Modify dest's NodeDef if necessary.
+  if (!source->IsSource() && !dest->IsSink() && !allow_duplicates) {
+    // Check if this input is already in dest's NodeDef.
+    const string new_input = strings::StrCat("^", source->name());
+    bool input_exists = false;
+    for (const string& input : dest->props_->node_def.input()) {
+      if (input == new_input) {
+        input_exists = true;
+        break;
+      }
+    }
+    if (!input_exists) {
+      dest->MaybeCopyOnWrite();
+      dest->props_->node_def.add_input(new_input);
+    }
+  }
+  return AddEdge(source, kControlSlot, dest, kControlSlot);
+}
+
 Status Graph::UpdateEdge(Node* new_src, int new_src_index, Node* dst,
                          int dst_index) {
   TF_RETURN_IF_ERROR(IsValidOutputTensor(new_src, new_src_index));
diff --git a/tensorflow/core/graph/graph.h b/tensorflow/core/graph/graph.h
index 54076ed1ab..7c7f641265 100644
--- a/tensorflow/core/graph/graph.h
+++ b/tensorflow/core/graph/graph.h
@@ -424,28 +424,36 @@ class Graph {
   // returned instance.
   Node* CopyNode(Node* node);
 
-  // Remove a node from this graph, including all edges from or to it.
+  // Removes a node from this graph, including all edges from or to it.
   // *node should not be accessed after calling this function.
   // REQUIRES: node->IsOp()
   void RemoveNode(Node* node);
 
-  // Add an edge that connects the xth output of "source" to the yth input
-  // of "dest".
+  // Adds an edge that connects the xth output of `source` to the yth input of
+  // `dest` and returns it. Does not update dest's NodeDef.
   const Edge* AddEdge(Node* source, int x, Node* dest, int y);
 
-  // Add a control-edge (no data flows along this edge) that
-  // connects "source" to "dest".
-  const Edge* AddControlEdge(Node* source, Node* dest) {
-    return AddEdge(source, kControlSlot, dest, kControlSlot);
-  }
-
-  // Removes edge from the graph.
+  // Adds a control edge (no data flows along this edge) that connects `source`
+  // to `dest`. If `dest`s NodeDef is missing the corresponding control input,
+  // adds the control input.
+  //
+  // If such a control edge already exists and `allow_duplicates` is false, no
+  // edge is added and the function returns nullptr. Otherwise the edge is
+  // unconditionally created and returned. The NodeDef is not updated if
+  // `allow_duplicates` is true.
+  // TODO(skyewm): // TODO(skyewm): allow_duplicates is needed only by
+  // graph_partition.cc. Figure out if we can do away with it.
+  const Edge* AddControlEdge(Node* source, Node* dest,
+                             bool allow_duplicates = false);
+
+  // Removes edge from the graph. Does not update the destination node's
+  // NodeDef.
   // REQUIRES: The edge must exist.
   void RemoveEdge(const Edge* edge);
 
-  // Updates the input to a node.  The existing edge to `dst` is removed
-  // and an edge from `new_src` to `dst` is created. The NodeDef associated with
-  // `dst` is also updated.
+  // Updates the input to a node.  The existing edge to `dst` is removed and an
+  // edge from `new_src` to `dst` is created. The NodeDef associated with `dst`
+  // is also updated.
   Status UpdateEdge(Node* new_src, int new_src_index, Node* dst, int dst_index);
 
   // Adds the function and gradient definitions in `fdef_lib` to this graph's op
diff --git a/tensorflow/core/graph/graph_partition.cc b/tensorflow/core/graph/graph_partition.cc
index 71d8cdd6ab..b9e3cba035 100644
--- a/tensorflow/core/graph/graph_partition.cc
+++ b/tensorflow/core/graph/graph_partition.cc
@@ -728,7 +728,10 @@ Status AddControlFlow(const PartitionOptions& opts, Graph* g,
             strings::StrCat(dst_frame_name, "$$", dst_device);
         ControlLoop loop = control_loops[cl_key];
         DCHECK(loop.enter != nullptr);
-        g->AddControlEdge(loop.merge, dst);
+        // Note that we'll create multiple duplicate edges if dst has multiple
+        // cross-device inputs. This is expected by the logic in Partition(), so
+        // it can add control edges to the recv nodes once they're created.
+        g->AddControlEdge(loop.merge, dst, /*allow_duplicates=*/true);
       }
     }
   }
diff --git a/tensorflow/core/graph/graph_test.cc b/tensorflow/core/graph/graph_test.cc
index 85eba0e166..e5d57facaa 100644
--- a/tensorflow/core/graph/graph_test.cc
+++ b/tensorflow/core/graph/graph_test.cc
@@ -104,6 +104,20 @@ class GraphTest : public ::testing::Test {
     return node;
   }
 
+  void FromGraphDef(const string& gdef_ascii) {
+    GraphDef gdef;
+    CHECK(protobuf::TextFormat::ParseFromString(gdef_ascii, &gdef));
+    GraphConstructorOptions opts;
+    TF_CHECK_OK(ConvertGraphDefToGraph(opts, gdef, &graph_));
+  }
+
+  Node* FindNode(const string& name) {
+    for (Node* node : graph_.nodes()) {
+      if (node->name() == name) return node;
+    }
+    LOG(FATAL) << name;
+  }
+
   Graph graph_;
 
  private:
@@ -411,6 +425,58 @@ TEST_F(GraphTest, IsValidNode) {
             s.error_message());
 }
 
+TEST_F(GraphTest, AddControlEdge) {
+  FromGraphDef(
+      "node { name: 'A' op: 'OneOutput' }"
+      "node { name: 'B' op: 'OneInputTwoOutputs' input: [ 'A:0' ] }"
+      "node { name: 'C' op: 'NoOp' } ");
+  Node* a = FindNode("A");
+  Node* b = FindNode("B");
+  Node* c = FindNode("C");
+
+  // Add a control edge.
+  const Edge* edge = graph_.AddControlEdge(c, a);
+  ASSERT_TRUE(edge != nullptr);
+  // Check newly-created edge.
+  EXPECT_EQ(edge->src(), c);
+  EXPECT_EQ(edge->src_output(), Graph::kControlSlot);
+  EXPECT_EQ(edge->dst(), a);
+  EXPECT_EQ(edge->dst_input(), Graph::kControlSlot);
+  // Check A's NodeDef.
+  ASSERT_EQ(a->def().input_size(), 1);
+  EXPECT_EQ(a->def().input(0), "^C");
+
+  // Can add control edge redundant with data edge.
+  edge = graph_.AddControlEdge(a, b);
+  EXPECT_TRUE(edge != nullptr);
+  ASSERT_EQ(b->def().input_size(), 2);
+  EXPECT_EQ(b->def().input(0), "A:0");
+  EXPECT_EQ(b->def().input(1), "^A");
+
+  // Doesn't add edge redundant with control edge.
+  edge = graph_.AddControlEdge(a, b);
+  EXPECT_TRUE(edge == nullptr);
+  EXPECT_EQ(b->def().input_size(), 2);
+
+  // Can add redundant control edge with create_duplicate.
+  edge = graph_.AddControlEdge(a, b, /*create_duplicate=*/true);
+  EXPECT_TRUE(edge != nullptr);
+  // create_duplicate causes the NodeDef not to be updated.
+  ASSERT_EQ(b->def().input_size(), 2);
+  EXPECT_EQ(b->def().input(0), "A:0");
+  EXPECT_EQ(b->def().input(1), "^A");
+
+  // Add control edge from source.
+  edge = graph_.AddControlEdge(graph_.source_node(), b);
+  EXPECT_TRUE(edge != nullptr);
+  // Check that we don't include source input in the NodeDef.
+  EXPECT_EQ(b->def().input_size(), 2);
+  // Doesn't add redundant edge.
+  edge = graph_.AddControlEdge(graph_.source_node(), b);
+  EXPECT_TRUE(edge == nullptr);
+  EXPECT_EQ(b->def().input_size(), 2);
+}
+
 TEST_F(GraphTest, UpdateEdge) {
   // Build a little graph
   Node* a = FromNodeDef("A", "OneOutput", 0);
diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index 7f5f60e599..6077d602c4 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -1798,15 +1798,19 @@ class Operation(object):
       TypeError: if ops is not a list of Operations.
       ValueError: if any op in ops is from a different graph.
     """
-    assert not self._c_op, (
-        "Operation._add_control_inputs doesn't work with C API")
-    if ops:
+    if self._c_op:
       for op in ops:
         if not isinstance(op, Operation):
           raise TypeError("op must be an Operation: %s" % op)
-        _assert_same_graph(self, op)
-        self._control_inputs.append(op)
-      self._recompute_node_def()
+        c_api.AddControlInput(self._graph._c_graph, self._c_op, op._c_op)  # pylint: disable=protected-access
+    else:
+      if ops:
+        for op in ops:
+          if not isinstance(op, Operation):
+            raise TypeError("op must be an Operation: %s" % op)
+          _assert_same_graph(self, op)
+          self._control_inputs.append(op)
+        self._recompute_node_def()
 
   def _add_control_input(self, op):
     """Add a new control input to this operation.
@@ -1819,6 +1823,8 @@ class Operation(object):
       ValueError: if op is from a different graph.
     """
     if self._c_op:
+      if not isinstance(op, Operation):
+        raise TypeError("op must be an Operation: %s" % op)
       c_api.AddControlInput(self._graph._c_graph, self._c_op, op._c_op)  # pylint: disable=protected-access
     else:
       self._add_control_inputs([op])
diff --git a/tensorflow/python/framework/ops_test.py b/tensorflow/python/framework/ops_test.py
index 9ef7f59529..f20c808cde 100644
--- a/tensorflow/python/framework/ops_test.py
+++ b/tensorflow/python/framework/ops_test.py
@@ -390,11 +390,32 @@ class OperationTest(test_util.TensorFlowTestCase):
 
   # TODO(nolivia): test all error cases
   def testAddControlInput(self):
+    # The C API dedups redundant control edges, pure Python does not
+    if ops._USE_C_API: return
+    with ops.Graph().as_default():
+      x = constant_op.constant(1).op
+      y = constant_op.constant(2).op
+      z = constant_op.constant(3).op
+    z._add_control_input(x)  # pylint: disable=protected-access
+    self.assertEqual(z.control_inputs, [x])
+    z._add_control_input(x)  # pylint: disable=protected-access
+    self.assertEqual(z.control_inputs, [x, x])
+    z._add_control_inputs([x, y, y])  # pylint: disable=protected-access
+    self.assertEqual(z.control_inputs, [x, x, x, y, y])
+
+  def testAddControlInputC(self):
+    # The C API dedups redundant control edges, pure Python does not
+    if not ops._USE_C_API: return
     with ops.Graph().as_default():
       x = constant_op.constant(1).op
       y = constant_op.constant(2).op
-    y._add_control_input(x)  # pylint: disable=protected-access
-    self.assertEqual(y.control_inputs, [x])
+      z = constant_op.constant(3).op
+    z._add_control_input(x)  # pylint: disable=protected-access
+    self.assertEqual(z.control_inputs, [x])
+    z._add_control_input(x)  # pylint: disable=protected-access
+    self.assertEqual(z.control_inputs, [x])
+    z._add_control_inputs([x, y, y])  # pylint: disable=protected-access
+    self.assertEqual(z.control_inputs, [x, y])
 
   def testControlInputCycle(self):
     # Non-C API path has a different error message
-- 
GitLab


From d426d3029727785676d1a7fbb7973a3a6ceb4842 Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Fri, 13 Oct 2017 16:11:26 -0700
Subject: [PATCH 387/909] Internal change.

PiperOrigin-RevId: 172159815
---
 tensorflow/core/util/test_log.proto | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tensorflow/core/util/test_log.proto b/tensorflow/core/util/test_log.proto
index a5476382f2..8ea59e1068 100644
--- a/tensorflow/core/util/test_log.proto
+++ b/tensorflow/core/util/test_log.proto
@@ -188,5 +188,9 @@ message TestResults {
   BenchmarkType benchmark_type = 10;
 
   // Used for differentiating between continuous and debug builds.
+  // Must be one of:
+  // * cbuild: results from continuous build.
+  // * presubmit: results from oneshot requests.
+  // * culprit: results from culprit finder rerun.
   string run_mode = 11;
 };
-- 
GitLab


From 5dd569cf026bae92330a194c8f2895d0f48149d9 Mon Sep 17 00:00:00 2001
From: Mark Heffernan <meheff@google.com>
Date: Fri, 13 Oct 2017 16:24:56 -0700
Subject: [PATCH 388/909] Make the HLO proto representation (hlo.proto) full
 fidelity. Hlo modules can be serialized to HLO protos and deserialized
 without any information loss.

As part of this change, a bug is fixed in NameUniquer. Previously, passing names with numeric suffixes could result in name collisions.

PiperOrigin-RevId: 172161360
---
 tensorflow/compiler/xla/protobuf_util.cc      |  30 ++-
 tensorflow/compiler/xla/protobuf_util.h       |  10 +-
 tensorflow/compiler/xla/service/BUILD         |  13 ++
 tensorflow/compiler/xla/service/hlo.proto     |  67 ++++++
 .../compiler/xla/service/hlo_computation.cc   |  29 +++
 .../compiler/xla/service/hlo_computation.h    |  16 ++
 .../compiler/xla/service/hlo_instruction.cc   | 199 +++++++++++++++---
 .../compiler/xla/service/hlo_instruction.h    |  29 ++-
 tensorflow/compiler/xla/service/hlo_module.cc |  33 +++
 tensorflow/compiler/xla/service/hlo_module.h  |   6 +
 tensorflow/compiler/xla/service/hlo_opcode.cc |  85 ++++++++
 tensorflow/compiler/xla/service/hlo_opcode.h  |   4 +
 .../compiler/xla/service/name_uniquer.cc      |  22 +-
 .../compiler/xla/service/name_uniquer.h       |   2 +-
 .../compiler/xla/service/name_uniquer_test.cc |  72 +++++++
 15 files changed, 569 insertions(+), 48 deletions(-)
 create mode 100644 tensorflow/compiler/xla/service/name_uniquer_test.cc

diff --git a/tensorflow/compiler/xla/protobuf_util.cc b/tensorflow/compiler/xla/protobuf_util.cc
index cdc4139cd6..c032cb8dc5 100644
--- a/tensorflow/compiler/xla/protobuf_util.cc
+++ b/tensorflow/compiler/xla/protobuf_util.cc
@@ -51,21 +51,39 @@ StatusOr<string> ToJson(const tensorflow::protobuf::Message& message) {
   return json_output;
 }
 
-Status DumpJsonToDirectory(const tensorflow::protobuf::Message& message,
-                           const string& directory, const string& file_name) {
-  TF_ASSIGN_OR_RETURN(const string json_output, ToJson(message));
+namespace {
 
-  tensorflow::Env* env = tensorflow::Env::Default();
-  TF_RETURN_IF_ERROR(env->RecursivelyCreateDir(directory));
-  string safe_file_name = file_name + ".json";
+string SanitizeFilename(const string& file_name) {
+  string safe_file_name = file_name;
   for (char& c : safe_file_name) {
     if (c == '/' || c == '\\') {
       c = '_';
     }
   }
+  return safe_file_name;
+}
+
+}  // namespace
+
+Status DumpJsonToDirectory(const tensorflow::protobuf::Message& message,
+                           const string& directory, const string& file_name) {
+  TF_ASSIGN_OR_RETURN(const string json_output, ToJson(message));
+
+  tensorflow::Env* env = tensorflow::Env::Default();
+  TF_RETURN_IF_ERROR(env->RecursivelyCreateDir(directory));
+  string safe_file_name = SanitizeFileName(file_name) + ".json";
   const string path = tensorflow::io::JoinPath(directory, safe_file_name);
   return tensorflow::WriteStringToFile(env, path, json_output);
 }
 
+Status DumpProtoToDirectory(const tensorflow::protobuf::Message& message,
+                            const string& directory, const string& file_name) {
+  tensorflow::Env* env = tensorflow::Env::Default();
+  TF_RETURN_IF_ERROR(env->RecursivelyCreateDir(directory));
+  string safe_file_name = SanitizeFileName(file_name) + ".pb";
+  const string path = tensorflow::io::JoinPath(directory, safe_file_name);
+  return tensorflow::WriteBinaryProto(env, path, message);
+}
+
 }  // namespace protobuf_util
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/protobuf_util.h b/tensorflow/compiler/xla/protobuf_util.h
index 1a895c3585..7accb22e0c 100644
--- a/tensorflow/compiler/xla/protobuf_util.h
+++ b/tensorflow/compiler/xla/protobuf_util.h
@@ -35,10 +35,12 @@ extern bool ProtobufEquals(const tensorflow::protobuf::Message& m1,
 // Returns 'message' as a JSON string.
 StatusOr<string> ToJson(const tensorflow::protobuf::Message& message);
 
-// Converts 'message' to JSON, and dumps it to the path formed by joining
-// 'directory/file_name.json'. The 'directory' is recursively created if it
-// doesn't already exist, and the 'file_name' is sanitized by replacing illegal
-// characters with underscore '_'.
+// Writes the given message in binary proto or JSON format to the path formed by
+// joining 'directory/file_name.pb' (or file_name.json). The 'directory' is
+// recursively created if it doesn't already exist, and the 'file_name' is
+// sanitized by replacing illegal characters with underscore '_'.
+Status DumpProtoToDirectory(const tensorflow::protobuf::Message& message,
+                            const string& directory, const string& file_name);
 Status DumpJsonToDirectory(const tensorflow::protobuf::Message& message,
                            const string& directory, const string& file_name);
 
diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD
index c1bb7107b6..3e85c796f2 100644
--- a/tensorflow/compiler/xla/service/BUILD
+++ b/tensorflow/compiler/xla/service/BUILD
@@ -433,6 +433,7 @@ cc_library(
         ":hlo_evaluator",
         ":hlo_execution_profile",
         ":hlo_module_config",
+        ":hlo_verifier",
         ":platform_util",
         ":session_proto",
         ":transfer_manager",
@@ -719,6 +720,18 @@ cc_library(
     ],
 )
 
+tf_cc_test(
+    name = "name_uniquer_test",
+    srcs = ["name_uniquer_test.cc"],
+    deps = [
+        ":name_uniquer",
+        "//tensorflow/compiler/xla:test",
+        "//tensorflow/compiler/xla:test_helpers",
+        "//tensorflow/compiler/xla/tests:xla_internal_test_main",
+        "//tensorflow/core:test",
+    ],
+)
+
 cc_library(
     name = "liveness_util",
     srcs = ["liveness_util.cc"],
diff --git a/tensorflow/compiler/xla/service/hlo.proto b/tensorflow/compiler/xla/service/hlo.proto
index af853385d6..79493c4112 100644
--- a/tensorflow/compiler/xla/service/hlo.proto
+++ b/tensorflow/compiler/xla/service/hlo.proto
@@ -39,6 +39,8 @@ message HloInstructionProto {
   string name = 1;
   string opcode = 2;
   xla.Shape shape = 3;
+
+  // TODO(b/67782397): Replace instruction names with HloInstruction ids.
   repeated string operand_names = 4;
   repeated string control_predecessor_names = 5;
   repeated string called_computation_names = 6;
@@ -58,6 +60,64 @@ message HloInstructionProto {
 
   // Index for kGetTupleElement.
   int64 tuple_index = 13;
+
+  // Dimensions present for some operations that require reshaping or
+  // broadcasting, including Reshape, Reduce, ReduceWindow, and Reverse.
+  repeated int64 dimensions = 14;
+
+  // Describes the window in a windowed operation such as convolution.
+  xla.Window window = 15;
+
+  // Describes the dimension numbers used for a convolution.
+  xla.ConvolutionDimensionNumbers convolution_dimension_numbers = 16;
+
+  // Describes the [begin, end) index range and stride for slices.
+  message SliceDimensions {
+    int64 start = 1;
+    int64 limit = 2;
+    int64 stride = 3;
+  }
+  repeated SliceDimensions slice_dimensions = 17;
+
+  // The bit sizes for a reduce-precision operation.
+  int32 exponent_bits = 18;
+  int32 mantissa_bits = 19;
+
+  // Describes the [start, start + size) range size for a dynamic slice
+  // ('start' is specified dynamically in the second operand of the operation).
+  repeated int64 dynamic_slice_sizes = 20;
+
+  // The padding configuration that describes the edge padding and interior
+  // padding of this pad instruction. Only set for pad instructions.
+  xla.PaddingConfig padding_config = 21;
+
+  // Outfeed configuration information, only present for kOutfeed.
+  bytes outfeed_config = 22;
+
+  // The distribution requested for random number generation.
+  // Only present for kRng.
+  xla.RandomDistribution distribution = 23;
+
+  // A small float number added to the variance to avoid divide-by-zero error.
+  // Only present for kBatchNormTraining.
+  float epsilon = 24;
+
+  // An integer value representing the index of the feature dimension.
+  // Only present for kBatchNormTraining.
+  int64 feature_index = 25;
+
+  // Represents a unique identifier for each Send/Recv instruction pair.
+  // Only present for kSend or kRecv.
+  int64 channel_id = 26;
+
+  // The string representation of the infeed configuration.
+  bytes infeed_config = 27;
+
+  // Name of a global symbol to call, only present for kCustomCall.
+  string custom_call_target = 28;
+
+  // Shape of outfeed request.
+  xla.Shape outfeed_shape = 29;
 }
 
 // Serialization of HloComputation.
@@ -67,6 +127,9 @@ message HloComputationProto {
   // The array of instructions is always in a valid dependency order, where
   // operands appear before their users.
   repeated HloInstructionProto instructions = 2;
+
+  // The name of the root of the computation.
+  string root_name = 3;
 }
 
 // Serialization of HloModule.
@@ -187,3 +250,7 @@ message HloProto {
   HloOrderingProto hlo_ordering = 2;
   BufferAssignmentProto buffer_assignment = 3;
 }
+
+message HloProtos {
+  repeated HloProto hlo_protos = 1;
+}
diff --git a/tensorflow/compiler/xla/service/hlo_computation.cc b/tensorflow/compiler/xla/service/hlo_computation.cc
index 444104d88f..9b3104eaac 100644
--- a/tensorflow/compiler/xla/service/hlo_computation.cc
+++ b/tensorflow/compiler/xla/service/hlo_computation.cc
@@ -400,9 +400,38 @@ HloComputationProto HloComputation::ToProto() const {
     HloInstructionProto instruction_proto = instruction->ToProto();
     proto.add_instructions()->Swap(&instruction_proto);
   }
+  proto.set_root_name(root_instruction()->name());
   return proto;
 }
 
+/* static */ StatusOr<std::unique_ptr<HloComputation>>
+HloComputation::CreateFromProto(
+    HloModule* module, const HloComputationProto& proto,
+    tensorflow::gtl::FlatMap<string, HloComputation*>* computation_map,
+    HloInstruction* fusion_instruction) {
+  std::vector<std::unique_ptr<HloInstruction>> instructions;
+  tensorflow::gtl::FlatMap<string, HloInstruction*> instruction_map;
+  int64 parameter_count = 0;
+  for (const HloInstructionProto& instruction_proto : proto.instructions()) {
+    TF_ASSIGN_OR_RETURN(
+        std::unique_ptr<HloInstruction> instruction,
+        HloInstruction::CreateFromProto(module, instruction_proto,
+                                        instruction_map, computation_map));
+    if (instruction->opcode() == HloOpcode::kParameter) {
+      parameter_count++;
+    }
+    TF_RET_CHECK(!ContainsKey(instruction_map, instruction->name()));
+    instruction_map[instruction->name()] = instruction.get();
+    instructions.push_back(std::move(instruction));
+  }
+
+  TF_RET_CHECK(!proto.root_name().empty());
+  TF_RET_CHECK(ContainsKey(instruction_map, proto.root_name()));
+  HloInstruction* root = instruction_map.at(proto.root_name());
+  return WrapUnique(new HloComputation(
+      proto.name(), parameter_count, &instructions, root, fusion_instruction));
+}
+
 void HloComputation::FuseInstructionsInto(
     tensorflow::gtl::ArraySlice<HloInstruction*> instructions_to_fuse,
     HloInstruction* fusion_instruction) {
diff --git a/tensorflow/compiler/xla/service/hlo_computation.h b/tensorflow/compiler/xla/service/hlo_computation.h
index b929b41bad..3515a6b5df 100644
--- a/tensorflow/compiler/xla/service/hlo_computation.h
+++ b/tensorflow/compiler/xla/service/hlo_computation.h
@@ -143,6 +143,22 @@ class HloComputation {
   // Returns a serialized representation of this computation.
   HloComputationProto ToProto() const;
 
+  // Creates a computation from the given proto. Arguments:
+  //
+  //   module: the module which will contain the computation. The newly created
+  //     computation is *not* added to the module, however.
+  //   proto: the proto to convert from.
+  //   computation_map: a map from computation name to HloComputation*. This map
+  //     must contain all computations which the newly constructed computation
+  //     calls.
+  //  fusion_instruction: if non-null then the newly created computation will be
+  //     constructed as a fused computation with this instruction as its fusion
+  //     parent.
+  static StatusOr<std::unique_ptr<HloComputation>> CreateFromProto(
+      HloModule* module, const HloComputationProto& proto,
+      tensorflow::gtl::FlatMap<string, HloComputation*>* computation_map,
+      HloInstruction* fusion_instruction = nullptr);
+
   // Gets the instructions in this computation.
   //
   // The returned type is a range of HloInstruction*s, so you can iterate over
diff --git a/tensorflow/compiler/xla/service/hlo_instruction.cc b/tensorflow/compiler/xla/service/hlo_instruction.cc
index 202e7c54b1..021e5881c8 100644
--- a/tensorflow/compiler/xla/service/hlo_instruction.cc
+++ b/tensorflow/compiler/xla/service/hlo_instruction.cc
@@ -47,6 +47,101 @@ using ::tensorflow::str_util::Join;
 using ::tensorflow::strings::StrAppend;
 using ::tensorflow::strings::StrCat;
 
+/* static */
+StatusOr<std::unique_ptr<HloInstruction>> HloInstruction::CreateFromProto(
+    HloModule* module, const HloInstructionProto& proto,
+    const tensorflow::gtl::FlatMap<string, HloInstruction*>& instruction_map,
+    tensorflow::gtl::FlatMap<string, HloComputation*>* computation_map) {
+  TF_RET_CHECK(!proto.opcode().empty());
+  TF_ASSIGN_OR_RETURN(HloOpcode opcode, StringToHloOpcode(proto.opcode()));
+  TF_RET_CHECK(proto.has_shape());
+
+  auto instruction = WrapUnique(new HloInstruction(opcode, proto.shape()));
+  for (const string& operand_name : proto.operand_names()) {
+    TF_RET_CHECK(ContainsKey(instruction_map, operand_name))
+        << "No instruction named " << operand_name;
+    instruction->AppendOperand(instruction_map.at(operand_name));
+  }
+  for (const string& predecessor_name : proto.control_predecessor_names()) {
+    TF_RET_CHECK(ContainsKey(instruction_map, predecessor_name))
+        << "No instruction named " << predecessor_name;
+    TF_RETURN_IF_ERROR(instruction_map.at(predecessor_name)
+                           ->AddControlDependencyTo(instruction.get()));
+  }
+
+  // In the proto, fused computations are held exclusively within the
+  // HloInstructionProto and do not appear as an HloComputationProto within the
+  // HloModuleProto.
+  if (instruction->opcode() == HloOpcode::kFusion) {
+    TF_RET_CHECK(proto.has_fused_instructions_computation());
+    TF_RET_CHECK(!proto.fusion_kind().empty());
+    TF_ASSIGN_OR_RETURN(instruction->fusion_kind_,
+                        StringToFusionKind(proto.fusion_kind()));
+    TF_ASSIGN_OR_RETURN(
+        std::unique_ptr<HloComputation> fused_computation,
+        HloComputation::CreateFromProto(
+            module, proto.fused_instructions_computation(), computation_map,
+            /*fusion_instruction=*/instruction.get()));
+    instruction->called_computations_.push_back(
+        module->AddEmbeddedComputation(std::move(fused_computation)));
+  } else {
+    for (const string& computation_name : proto.called_computation_names()) {
+      TF_RET_CHECK(ContainsKey(*computation_map, computation_name))
+          << "No computation named " << computation_name;
+      instruction->called_computations_.push_back(
+          computation_map->at(computation_name));
+    }
+  }
+
+  TF_RET_CHECK(!proto.name().empty());
+  instruction->name_ = proto.name();
+
+  instruction->metadata_ = proto.metadata();
+  if (proto.has_literal()) {
+    instruction->literal_ = MakeUnique<Literal>(proto.literal());
+  }
+  instruction->parameter_number_ = proto.parameter_number();
+  instruction->parameter_name_ = proto.parameter_name();
+
+  instruction->tuple_index_ = proto.tuple_index();
+  for (int64 dimension : proto.dimensions()) {
+    instruction->dimensions_.push_back(dimension);
+  }
+  if (proto.has_window()) {
+    instruction->window_ = MakeUnique<Window>(proto.window());
+  }
+  if (proto.has_convolution_dimension_numbers()) {
+    instruction->convolution_dimension_numbers_ =
+        MakeUnique<ConvolutionDimensionNumbers>(
+            proto.convolution_dimension_numbers());
+  }
+  for (const HloInstructionProto::SliceDimensions& slice_dimensions :
+       proto.slice_dimensions()) {
+    instruction->slice_starts_.push_back(slice_dimensions.start());
+    instruction->slice_limits_.push_back(slice_dimensions.limit());
+    instruction->slice_strides_.push_back(slice_dimensions.stride());
+  }
+  instruction->exponent_bits_ = proto.exponent_bits();
+  instruction->mantissa_bits_ = proto.mantissa_bits();
+  for (int64 dynamic_slice_size : proto.dynamic_slice_sizes()) {
+    instruction->dynamic_slice_sizes_.push_back(dynamic_slice_size);
+  }
+  if (proto.has_padding_config()) {
+    instruction->padding_config_ =
+        MakeUnique<PaddingConfig>(proto.padding_config());
+  }
+  instruction->outfeed_config_ = proto.outfeed_config();
+  instruction->distribution_ = proto.distribution();
+  instruction->epsilon_ = proto.epsilon();
+  instruction->feature_index_ = proto.feature_index();
+  instruction->channel_id_ = proto.channel_id();
+  instruction->infeed_config_ = proto.infeed_config();
+  instruction->custom_call_target_ = proto.custom_call_target();
+  instruction->outfeed_shape_ = proto.outfeed_shape();
+
+  return std::move(instruction);
+}
+
 /* static */ std::unique_ptr<HloInstruction> HloInstruction::CreateParameter(
     int64 parameter_number, const Shape& shape, const string& name) {
   auto instruction =
@@ -1774,37 +1869,59 @@ HloInstructionProto HloInstruction::ToProto() const {
   for (const HloInstruction* control : control_predecessors_) {
     *proto.add_control_predecessor_names() = control->name();
   }
-  for (const HloComputation* computation : called_computations_) {
-    *proto.add_called_computation_names() = computation->name();
-  }
+
   *proto.mutable_metadata() = metadata_;
-  switch (opcode_) {
-    case HloOpcode::kConstant:
-      *proto.mutable_literal() = literal_->ToProto();
-      break;
-    case HloOpcode::kParameter:
-      proto.set_parameter_number(parameter_number_);
-      proto.set_parameter_name(parameter_name_);
-      break;
-    case HloOpcode::kFusion: {
-      HloComputationProto* proto_fused_computation =
-          proto.mutable_fused_instructions_computation();
-      proto_fused_computation->set_name(name());
-
-      // Fill in fused instructions in post order.
-      auto fused_instructions =
-          fused_instructions_computation()->MakeInstructionPostOrder();
-      for (auto fused_instruction : fused_instructions) {
-        HloInstructionProto fused_proto = fused_instruction->ToProto();
-        proto_fused_computation->add_instructions()->Swap(&fused_proto);
-      }
-      break;
+  if (literal_ != nullptr) {
+    *proto.mutable_literal() = literal_->ToProto();
+  }
+  proto.set_parameter_number(parameter_number_);
+  proto.set_parameter_name(parameter_name_);
+  if (opcode() == HloOpcode::kFusion) {
+    proto.set_fusion_kind(xla::ToString(fusion_kind()));
+    *proto.mutable_fused_instructions_computation() =
+        fused_instructions_computation()->ToProto();
+  } else {
+    for (const HloComputation* computation : called_computations_) {
+      *proto.add_called_computation_names() = computation->name();
     }
-    case HloOpcode::kGetTupleElement:
-      proto.set_tuple_index(tuple_index_);
-      break;
-    default: {}  // Nothing to do
   }
+
+  proto.set_tuple_index(tuple_index_);
+  for (int64 dimension : dimensions_) {
+    proto.add_dimensions(dimension);
+  }
+  if (window_ != nullptr) {
+    *proto.mutable_window() = *window_;
+  }
+  if (convolution_dimension_numbers_ != nullptr) {
+    *proto.mutable_convolution_dimension_numbers() =
+        *convolution_dimension_numbers_;
+  }
+  for (int i = 0; i < slice_starts_.size(); ++i) {
+    auto* slice_dimension = proto.add_slice_dimensions();
+    slice_dimension->set_start(slice_starts_[i]);
+    slice_dimension->set_limit(slice_limits_[i]);
+    slice_dimension->set_stride(slice_strides_[i]);
+  }
+  proto.set_exponent_bits(exponent_bits_);
+  proto.set_mantissa_bits(mantissa_bits_);
+  for (int64 slice_size : dynamic_slice_sizes_) {
+    proto.add_dynamic_slice_sizes(slice_size);
+  }
+  if (padding_config_ != nullptr) {
+    *proto.mutable_padding_config() = *padding_config_;
+  }
+  proto.set_outfeed_config(outfeed_config_);
+  if (opcode() == HloOpcode::kRng) {
+    proto.set_distribution(distribution_);
+  }
+  proto.set_epsilon(epsilon_);
+  proto.set_feature_index(feature_index_);
+  proto.set_channel_id(channel_id_);
+  proto.set_infeed_config(infeed_config_);
+  proto.set_custom_call_target(custom_call_target_);
+  *proto.mutable_outfeed_shape() = outfeed_shape_;
+
   return proto;
 }
 
@@ -2636,6 +2753,32 @@ string ToString(HloInstruction::FusionKind kind) {
   }
 }
 
+StatusOr<HloInstruction::FusionKind> StringToFusionKind(
+    const string& kind_name) {
+  if (kind_name == "kLoop") {
+    return HloInstruction::FusionKind::kLoop;
+  }
+  if (kind_name == "kInput") {
+    return HloInstruction::FusionKind::kInput;
+  }
+  if (kind_name == "kOutput") {
+    return HloInstruction::FusionKind::kOutput;
+  }
+  if (kind_name == "kTransposeDot") {
+    return HloInstruction::FusionKind::kTransposeDot;
+  }
+  if (kind_name == "kConvBackwardFilter") {
+    return HloInstruction::FusionKind::kConvBackwardFilter;
+  }
+  if (kind_name == "kConvBackwardInput") {
+    return HloInstruction::FusionKind::kConvBackwardInput;
+  }
+  if (kind_name == "kCustom") {
+    return HloInstruction::FusionKind::kCustom;
+  }
+  return InvalidArgument("Unknown fusion kind: %s", kind_name.c_str());
+}
+
 std::ostream& operator<<(std::ostream& os, HloInstruction::FusionKind kind) {
   return os << ToString(kind);
 }
diff --git a/tensorflow/compiler/xla/service/hlo_instruction.h b/tensorflow/compiler/xla/service/hlo_instruction.h
index 011cc8f742..d2a15b0f96 100644
--- a/tensorflow/compiler/xla/service/hlo_instruction.h
+++ b/tensorflow/compiler/xla/service/hlo_instruction.h
@@ -72,6 +72,23 @@ class HloInstruction {
   };
 
   ~HloInstruction();
+
+  // Creates an instruction from the given proto. Arguments:
+  //
+  //   module: the module which will contain the instruction. The newly created
+  //     instruction is *not* added to the module or any computation, however.
+  //   proto: the proto to convert from.
+  //   instruction_map: a map from instruction name to HloInstruction*. This map
+  //     must contain all operands of the newly constructed instruction.
+  //   computation_map: a map from computation name to HloComputation*. This map
+  //     must contain all computations which the newly constructed instruction
+  //     calls. If the instruction is a fusion instruction, then the fusion
+  //     computation is added to this map and the module.
+  static StatusOr<std::unique_ptr<HloInstruction>> CreateFromProto(
+      HloModule* module, const HloInstructionProto& proto,
+      const tensorflow::gtl::FlatMap<string, HloInstruction*>& instruction_map,
+      tensorflow::gtl::FlatMap<string, HloComputation*>* computation_map);
+
   // Creates a parameter-retrieving instruction.
   static std::unique_ptr<HloInstruction> CreateParameter(int64 parameter_number,
                                                          const Shape& shape,
@@ -1075,7 +1092,7 @@ class HloInstruction {
   std::unique_ptr<Literal> literal_;
 
   // Constant index, only present for kGetTupleElement.
-  int64 tuple_index_ = 0;
+  int64 tuple_index_ = -1;
 
   // Dimensions present for some operations that require reshaping or
   // broadcasting, including Reshape, Reduce, ReduceWindow, and Reverse.
@@ -1093,8 +1110,8 @@ class HloInstruction {
   std::vector<int64> slice_strides_;
 
   // The bit sizes for a reduce-precision operation.
-  int32 exponent_bits_;
-  int32 mantissa_bits_;
+  int32 exponent_bits_ = 0;
+  int32 mantissa_bits_ = 0;
 
   // Describes the [start, start + size) range size for a dynamic slice
   // ('start' is specified dynamically in the second operand of the operation).
@@ -1144,11 +1161,11 @@ class HloInstruction {
 
   // A small float number added to the variance to avoid divide-by-zero error.
   // Only present for kBatchNormTraining.
-  float epsilon_;
+  float epsilon_ = 0.0f;
 
   // An integer value representing the index of the feature dimension.
   // Only present for kBatchNormTraining.
-  int64 feature_index_;
+  int64 feature_index_ = -1;
 
   // Represents a unique identifier for each Send/Recv instruction pair.
   // Only present for kSend or kRecv.
@@ -1174,6 +1191,8 @@ class HloInstruction {
 };
 
 string ToString(HloInstruction::FusionKind kind);
+StatusOr<HloInstruction::FusionKind> StringToFusionKind(
+    const string& kind_name);
 
 std::ostream& operator<<(std::ostream& os, HloInstruction::FusionKind kind);
 
diff --git a/tensorflow/compiler/xla/service/hlo_module.cc b/tensorflow/compiler/xla/service/hlo_module.cc
index 14590112a1..5bc7a36439 100644
--- a/tensorflow/compiler/xla/service/hlo_module.cc
+++ b/tensorflow/compiler/xla/service/hlo_module.cc
@@ -167,12 +167,45 @@ HloModuleProto HloModule::ToProto() const {
   proto.set_name(name_);
   proto.set_entry_computation_name(entry_computation_->name());
   for (const HloComputation* computation : MakeComputationPostOrder()) {
+    // Fusion computations are added when the fusion instructions are created by
+    // HloInstruction::CreateFromProto.
+    if (computation->IsFusionComputation()) {
+      continue;
+    }
     HloComputationProto computation_proto = computation->ToProto();
     proto.add_computations()->Swap(&computation_proto);
   }
   return proto;
 }
 
+/* static */
+StatusOr<std::unique_ptr<HloModule>> HloModule::CreateFromProto(
+    const HloModuleProto& proto,
+    const VersionedComputationHandle& entry_computation_handle,
+    const HloModuleConfig& config) {
+  auto module =
+      MakeUnique<HloModule>(proto.name(), entry_computation_handle, config);
+  tensorflow::gtl::FlatMap<string, HloComputation*> computation_map;
+  for (const HloComputationProto& computation_proto : proto.computations()) {
+    TF_ASSIGN_OR_RETURN(std::unique_ptr<HloComputation> computation,
+                        HloComputation::CreateFromProto(
+                            module.get(), computation_proto, &computation_map));
+    CHECK_NE(computation.get(), nullptr);
+    TF_RET_CHECK(!ContainsKey(computation_map, computation->name()));
+    string computation_name = computation->name();
+    if (proto.entry_computation_name() == computation_name) {
+      computation_map[computation_name] =
+          module->AddEntryComputation(std::move(computation));
+    } else {
+      computation_map[computation_name] =
+          module->AddEmbeddedComputation(std::move(computation));
+    }
+  }
+  TF_RET_CHECK(module->entry_computation_ != nullptr);
+
+  return std::move(module);
+}
+
 namespace {
 // Returns whether `hlo` is used outside the given subcomputation.
 // `instructions_in_subcomputation` is the instruction set of the given
diff --git a/tensorflow/compiler/xla/service/hlo_module.h b/tensorflow/compiler/xla/service/hlo_module.h
index 3546f4b3f7..96c17d6297 100644
--- a/tensorflow/compiler/xla/service/hlo_module.h
+++ b/tensorflow/compiler/xla/service/hlo_module.h
@@ -140,7 +140,13 @@ class HloModule {
   const HloModuleConfig& config() const { return config_; }
 
   string ToString() const;
+
+  // Convert an HloModule to or from a proto.
   HloModuleProto ToProto() const;
+  static StatusOr<std::unique_ptr<HloModule>> CreateFromProto(
+      const HloModuleProto& proto,
+      const VersionedComputationHandle& entry_computation_handle,
+      const HloModuleConfig& config);
 
   // Outlines the given expression from the given computation.
   // instructions_to_outline contains the instructions that form the expression.
diff --git a/tensorflow/compiler/xla/service/hlo_opcode.cc b/tensorflow/compiler/xla/service/hlo_opcode.cc
index e98012ec0c..db3abeab22 100644
--- a/tensorflow/compiler/xla/service/hlo_opcode.cc
+++ b/tensorflow/compiler/xla/service/hlo_opcode.cc
@@ -15,6 +15,8 @@ limitations under the License.
 
 #include "tensorflow/compiler/xla/service/hlo_opcode.h"
 #include "tensorflow/compiler/xla/types.h"
+#include "tensorflow/compiler/xla/util.h"
+#include "tensorflow/core/lib/gtl/flatmap.h"
 
 namespace xla {
 
@@ -178,6 +180,89 @@ string HloOpcodeString(HloOpcode opcode) {
   }
 }
 
+StatusOr<HloOpcode> StringToHloOpcode(const string& opcode_name) {
+  static auto* opcode_map = new tensorflow::gtl::FlatMap<string, HloOpcode>(
+      {{"abs", HloOpcode::kAbs},
+       {"add", HloOpcode::kAdd},
+       {"batch-norm-training", HloOpcode::kBatchNormTraining},
+       {"batch-norm-inference", HloOpcode::kBatchNormInference},
+       {"batch-norm-grad", HloOpcode::kBatchNormGrad},
+       {"bitcast", HloOpcode::kBitcast},
+       {"broadcast", HloOpcode::kBroadcast},
+       {"call", HloOpcode::kCall},
+       {"clamp", HloOpcode::kClamp},
+       {"concatenate", HloOpcode::kConcatenate},
+       {"constant", HloOpcode::kConstant},
+       {"convert", HloOpcode::kConvert},
+       {"convolution", HloOpcode::kConvolution},
+       {"cosine", HloOpcode::kCos},
+       {"cross-replica-sum", HloOpcode::kCrossReplicaSum},
+       {"custom-call", HloOpcode::kCustomCall},
+       {"copy", HloOpcode::kCopy},
+       {"divide", HloOpcode::kDivide},
+       {"dot", HloOpcode::kDot},
+       {"dynamic-slice", HloOpcode::kDynamicSlice},
+       {"dynamic-update-slice", HloOpcode::kDynamicUpdateSlice},
+       {"equal-to", HloOpcode::kEq},
+       {"exponential", HloOpcode::kExp},
+       {"floor", HloOpcode::kFloor},
+       {"ceil", HloOpcode::kCeil},
+       {"fusion", HloOpcode::kFusion},
+       {"greater-than-or-equal-to", HloOpcode::kGe},
+       {"get-tuple-element", HloOpcode::kGetTupleElement},
+       {"greater-than", HloOpcode::kGt},
+       {"index", HloOpcode::kIndex},
+       {"infeed", HloOpcode::kInfeed},
+       {"is-finite", HloOpcode::kIsFinite},
+       {"less-than-or-equal-to", HloOpcode::kLe},
+       {"log", HloOpcode::kLog},
+       {"and", HloOpcode::kAnd},
+       {"or", HloOpcode::kOr},
+       {"not", HloOpcode::kNot},
+       {"less-than", HloOpcode::kLt},
+       {"map", HloOpcode::kMap},
+       {"maximum", HloOpcode::kMaximum},
+       {"minimum", HloOpcode::kMinimum},
+       {"multiply", HloOpcode::kMultiply},
+       {"not-equal-to", HloOpcode::kNe},
+       {"negate", HloOpcode::kNegate},
+       {"outfeed", HloOpcode::kOutfeed},
+       {"pad", HloOpcode::kPad},
+       {"parameter", HloOpcode::kParameter},
+       {"power", HloOpcode::kPower},
+       {"recv", HloOpcode::kRecv},
+       {"reduce", HloOpcode::kReduce},
+       {"reduce-precision", HloOpcode::kReducePrecision},
+       {"reduce-window", HloOpcode::kReduceWindow},
+       {"remainder", HloOpcode::kRemainder},
+       {"reshape", HloOpcode::kReshape},
+       {"reverse", HloOpcode::kReverse},
+       {"rng", HloOpcode::kRng},
+       {"round-nearest-afz", HloOpcode::kRoundNearestAfz},
+       {"select-and-scatter", HloOpcode::kSelectAndScatter},
+       {"select", HloOpcode::kSelect},
+       {"send", HloOpcode::kSend},
+       {"shift-left", HloOpcode::kShiftLeft},
+       {"shift-right-arithmetic", HloOpcode::kShiftRightArithmetic},
+       {"shift-right-logical", HloOpcode::kShiftRightLogical},
+       {"sign", HloOpcode::kSign},
+       {"sine", HloOpcode::kSin},
+       {"slice", HloOpcode::kSlice},
+       {"sort", HloOpcode::kSort},
+       {"subtract", HloOpcode::kSubtract},
+       {"tanh", HloOpcode::kTanh},
+       {"trace", HloOpcode::kTrace},
+       {"transpose", HloOpcode::kTranspose},
+       {"tuple", HloOpcode::kTuple},
+       {"update", HloOpcode::kUpdate},
+       {"while", HloOpcode::kWhile}});
+  auto it = opcode_map->find(opcode_name);
+  if (it == opcode_map->end()) {
+    return InvalidArgument("Unknown opcode: %s", opcode_name.c_str());
+  }
+  return it->second;
+}
+
 bool HloOpcodeIsComparison(HloOpcode opcode) {
   switch (opcode) {
     case HloOpcode::kGe:
diff --git a/tensorflow/compiler/xla/service/hlo_opcode.h b/tensorflow/compiler/xla/service/hlo_opcode.h
index 057d4f6ea7..4593df671e 100644
--- a/tensorflow/compiler/xla/service/hlo_opcode.h
+++ b/tensorflow/compiler/xla/service/hlo_opcode.h
@@ -18,6 +18,7 @@ limitations under the License.
 
 #include <iosfwd>
 #include <string>
+#include "tensorflow/compiler/xla/statusor.h"
 #include "tensorflow/compiler/xla/types.h"
 
 namespace xla {
@@ -107,6 +108,9 @@ enum class HloOpcode {
 // Returns a string representation of the opcode.
 string HloOpcodeString(HloOpcode opcode);
 
+// Returns a string representation of the opcode.
+StatusOr<HloOpcode> StringToHloOpcode(const string& opcode_name);
+
 inline std::ostream& operator<<(std::ostream& os, HloOpcode opcode) {
   return os << HloOpcodeString(opcode);
 }
diff --git a/tensorflow/compiler/xla/service/name_uniquer.cc b/tensorflow/compiler/xla/service/name_uniquer.cc
index 069f85af72..a0d08c288d 100644
--- a/tensorflow/compiler/xla/service/name_uniquer.cc
+++ b/tensorflow/compiler/xla/service/name_uniquer.cc
@@ -23,7 +23,24 @@ namespace xla {
 
 string NameUniquer::GetUniqueName(tensorflow::StringPiece prefix) {
   string root = prefix.empty() ? "name" : prefix.ToString();
-  int* count = &(generated_names_[root]);
+
+  // Strip away numeric suffix (if any). Only recognize separator if it is in
+  // the middle of the name.
+  size_t separator_index = root.rfind(separator_);
+  if (separator_index != string::npos && (separator_index > 0) &&
+      (separator_index < root.size() - 1)) {
+    string after_suffix = root.substr(separator_index + 1);
+    int64 numeric_suffix;
+    if (tensorflow::strings::safe_strto64(after_suffix, &numeric_suffix)) {
+      // Remove numeric suffix from root.
+      root = root.substr(0, separator_index);
+      // Update count to at least the numeric suffix value to avoid future
+      // colisions with this name.
+      generated_names_[root] = std::max(generated_names_[root], numeric_suffix);
+    }
+  }
+
+  int64* count = &(generated_names_[root]);
   if (*count == 0) {
     *count = 1;
     return root;
@@ -31,9 +48,6 @@ string NameUniquer::GetUniqueName(tensorflow::StringPiece prefix) {
     tensorflow::strings::StrAppend(&root, separator_, *count);
     // Increment lookup under old 'root' name.
     (*count)++;
-    // Initialize count under new 'root' name.
-    count = &(generated_names_[root]);
-    *count = 1;
     return root;
   }
 }
diff --git a/tensorflow/compiler/xla/service/name_uniquer.h b/tensorflow/compiler/xla/service/name_uniquer.h
index b0944adbc1..ed379b5225 100644
--- a/tensorflow/compiler/xla/service/name_uniquer.h
+++ b/tensorflow/compiler/xla/service/name_uniquer.h
@@ -43,7 +43,7 @@ class NameUniquer {
 
   // Map from name prefix to the number of names generated using that prefix
   // so far.
-  std::unordered_map<string, int> generated_names_;
+  std::unordered_map<string, int64> generated_names_;
 
   TF_DISALLOW_COPY_AND_ASSIGN(NameUniquer);
 };
diff --git a/tensorflow/compiler/xla/service/name_uniquer_test.cc b/tensorflow/compiler/xla/service/name_uniquer_test.cc
new file mode 100644
index 0000000000..9f0747a6e2
--- /dev/null
+++ b/tensorflow/compiler/xla/service/name_uniquer_test.cc
@@ -0,0 +1,72 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/service/name_uniquer.h"
+
+#include <memory>
+#include <utility>
+#include <vector>
+
+#include "tensorflow/core/platform/test.h"
+
+namespace xla {
+namespace {
+
+class NameUniquerTest : public ::testing::Test {};
+
+TEST_F(NameUniquerTest, SimpleUniquer) {
+  NameUniquer uniquer;
+
+  EXPECT_EQ("foo", uniquer.GetUniqueName("foo"));
+  EXPECT_EQ("foo__1", uniquer.GetUniqueName("foo"));
+  EXPECT_EQ("foo__2", uniquer.GetUniqueName("foo"));
+  EXPECT_EQ("bar", uniquer.GetUniqueName("bar"));
+  EXPECT_EQ("foo__3", uniquer.GetUniqueName("foo"));
+  EXPECT_EQ("bar__1", uniquer.GetUniqueName("bar"));
+  EXPECT_EQ("qux", uniquer.GetUniqueName("qux"));
+}
+
+TEST_F(NameUniquerTest, DifferentSeparator) {
+  NameUniquer uniquer(".");
+
+  EXPECT_EQ("foo", uniquer.GetUniqueName("foo"));
+  EXPECT_EQ("foo.1", uniquer.GetUniqueName("foo"));
+  EXPECT_EQ("foo.2", uniquer.GetUniqueName("foo"));
+  EXPECT_EQ("bar", uniquer.GetUniqueName("bar"));
+  EXPECT_EQ("foo.3", uniquer.GetUniqueName("foo"));
+  EXPECT_EQ("bar.1", uniquer.GetUniqueName("bar"));
+}
+
+TEST_F(NameUniquerTest, NumericSuffixes) {
+  NameUniquer uniquer(".");
+
+  EXPECT_EQ("foo", uniquer.GetUniqueName("foo"));
+  EXPECT_EQ("foo.54", uniquer.GetUniqueName("foo.54"));
+  EXPECT_EQ("foo.55", uniquer.GetUniqueName("foo"));
+  EXPECT_EQ("foo.55.1", uniquer.GetUniqueName("foo.55.1"));
+  EXPECT_EQ("foo.55.2", uniquer.GetUniqueName("foo.55.1"));
+  EXPECT_EQ("bar", uniquer.GetUniqueName("bar.-1000"));
+  EXPECT_EQ("bar.1", uniquer.GetUniqueName("bar.-2000"));
+  EXPECT_EQ("bar.2", uniquer.GetUniqueName("bar.1"));
+
+  // Separator is only recognized in the middle of the prefix.
+  EXPECT_EQ(".10", uniquer.GetUniqueName(".10"));
+  EXPECT_EQ(".10.1", uniquer.GetUniqueName(".10"));
+  EXPECT_EQ("foobar.", uniquer.GetUniqueName("foobar."));
+  EXPECT_EQ("foobar..1", uniquer.GetUniqueName("foobar."));
+}
+
+}  // namespace
+}  // namespace xla
-- 
GitLab


From 84580227a398e68001c2114fae966a62ac918045 Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Fri, 13 Oct 2017 16:30:53 -0700
Subject: [PATCH 389/909] imperative_grad takes the tape instead of popping it.

PiperOrigin-RevId: 172162006
---
 tensorflow/python/eager/BUILD              |  1 -
 tensorflow/python/eager/backprop.py        |  3 ++-
 tensorflow/python/eager/imperative_grad.py | 11 +++++------
 3 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/tensorflow/python/eager/BUILD b/tensorflow/python/eager/BUILD
index 1d20a0782f..69b96df87c 100644
--- a/tensorflow/python/eager/BUILD
+++ b/tensorflow/python/eager/BUILD
@@ -431,5 +431,4 @@ py_library(
     name = "imperative_grad",
     srcs = ["imperative_grad.py"],
     srcs_version = "PY2AND3",
-    deps = [":tape"],
 )
diff --git a/tensorflow/python/eager/backprop.py b/tensorflow/python/eager/backprop.py
index 554b9a818c..0060dd0c1c 100644
--- a/tensorflow/python/eager/backprop.py
+++ b/tensorflow/python/eager/backprop.py
@@ -338,6 +338,7 @@ def implicit_val_and_grad(f):
     variables = tape.top_tape_watched_variables()
     sources = [x.handle for x in variables]
     grad = imperative_grad.imperative_grad(_default_vspace,
+                                           tape.pop_tape(),
                                            nest.flatten(end_node),
                                            sources)
     return end_node, list(zip(grad, variables))
@@ -574,7 +575,7 @@ def val_and_grad_function(f, params=None):
       tape.watch(args[i])
     result = f(*args)
     return result, imperative_grad.imperative_grad(
-        _default_vspace, nest.flatten(result), sources,
+        _default_vspace, tape.pop_tape(), nest.flatten(result), sources,
         output_gradients=nest.flatten(dy) if dy is not None else None)
 
   return decorated
diff --git a/tensorflow/python/eager/imperative_grad.py b/tensorflow/python/eager/imperative_grad.py
index dd9d691d26..d30d124040 100644
--- a/tensorflow/python/eager/imperative_grad.py
+++ b/tensorflow/python/eager/imperative_grad.py
@@ -20,7 +20,7 @@ from __future__ import print_function
 
 import collections
 
-from tensorflow.python.eager import tape
+from tensorflow.python.eager import tape as tape_module
 
 
 # Terminology:
@@ -76,7 +76,7 @@ def _prepare_backprop(vspace, target, tensor_to_op, op_to_entry, id_sources):
     # op is None or -1 if the tensor is a source (i.e. was watched directly)
     if op is None or op == -1 or op in o_to_e:
       continue
-    op_trace = tape.TapeEntry(*op_to_entry[op])
+    op_trace = tape_module.TapeEntry(*op_to_entry[op])
     o_to_e[op] = op_trace
     for it in op_trace.input_ids:
       if it in tensor_usage_counts:
@@ -125,6 +125,7 @@ VSpace = collections.namedtuple(
 
 def imperative_grad(
     vspace,
+    tape,
     target,
     sources,
     output_gradients=None):
@@ -136,6 +137,7 @@ def imperative_grad(
 
   Args:
    vspace: the vector space in which to differentiate.
+   tape: the gradient tape which stores the trace.
    target: either a Tensor or list of Tensors to be differentiated.
    sources: list of Tensors for which we want gradients
    output_gradients: if not None, a list of gradient provided for each Target,
@@ -152,10 +154,7 @@ def imperative_grad(
      or if only non-differentiable functions of the source were used in the
      computation of target.
   """
-  if not tape._tape_stack.stack:  # pylint: disable=protected-access
-    raise RuntimeError("Computing a gradient with no tape present")
-  bp_tape = tape.pop_tape()
-  tensor_to_op, op_to_entry = bp_tape.export()
+  tensor_to_op, op_to_entry = tape.export()
   # This overwrites the op_to_entry variable, which will release all memory used
   # to keep traces that are irrelevant to the gradient computation we're doing
   # here.
-- 
GitLab


From 8fe6ea5f32c4ad5a5feb7f54d0fba8ddab4927ca Mon Sep 17 00:00:00 2001
From: Benoit Steiner <bsteiner@google.com>
Date: Fri, 13 Oct 2017 17:25:08 -0700
Subject: [PATCH 390/909] Python wrapper to access the predicted peak memory
 usage

PiperOrigin-RevId: 172167437
---
 tensorflow/python/BUILD                    |  1 +
 tensorflow/python/grappler/cluster.i       | 51 ++++++++++++++++++++++
 tensorflow/python/grappler/cluster.py      | 13 ++++++
 tensorflow/python/grappler/cluster_test.py | 20 +++++++++
 4 files changed, 85 insertions(+)

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 2738022584..1885caf695 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -3004,6 +3004,7 @@ tf_py_wrap_cc(
         "//tensorflow/core/grappler:grappler_item",
         "//tensorflow/core/grappler:grappler_item_builder",
         "//tensorflow/core/grappler/clusters:single_machine",
+        "//tensorflow/core/grappler/costs:graph_memory",
         "//tensorflow/core/grappler/optimizers:meta_optimizer",
         "//tensorflow/core:lib",
         "//tensorflow/core:reader_base",
diff --git a/tensorflow/python/grappler/cluster.i b/tensorflow/python/grappler/cluster.i
index d38eb73ad2..3df9431282 100644
--- a/tensorflow/python/grappler/cluster.i
+++ b/tensorflow/python/grappler/cluster.i
@@ -43,6 +43,7 @@ limitations under the License.
 %{
 #include "tensorflow/core/grappler/devices.h"
 #include "tensorflow/core/grappler/clusters/single_machine.h"
+#include "tensorflow/core/grappler/costs/graph_memory.h"
 #include "tensorflow/core/grappler/costs/op_performance_data.pb.h"
 #include "tensorflow/core/grappler/costs/measuring_cost_estimator.h"
 #include "tensorflow/core/grappler/costs/utils.h"
@@ -143,6 +144,53 @@ static PyObject* TF_MeasureCosts(
   return ret;
 }
 
+
+static PyObject* TF_DeterminePeakMemoryUsage(
+    const tensorflow::grappler::GrapplerItem* item, tensorflow::grappler::Cluster* cluster,
+    TF_Status* out_status) {
+  if (!item || !cluster) {
+    tensorflow::Status status(tensorflow::error::Code::INTERNAL,
+                              "You need both a cluster and an item to determine peak memory usage");
+    tensorflow::Set_TF_Status_from_Status(out_status, status);
+    Py_RETURN_NONE;
+  }
+  tensorflow::grappler::GraphMemory memory(*item);
+
+  tensorflow::Status status;
+  if (cluster->DetailedStatsEnabled()) {
+    status = memory.InferDynamically(cluster);
+  } else {
+    status = memory.InferStatically(cluster->GetDevices());
+  }
+  if (!status.ok()) {
+    tensorflow::Set_TF_Status_from_Status(out_status, status);
+    Py_RETURN_NONE;
+  }
+
+  PyObject* result = PyDict_New();
+  for (const auto& device : cluster->GetDevices()) {
+    const tensorflow::grappler::GraphMemory::MemoryUsage& usage =
+        memory.GetPeakMemoryUsage(device.first);
+    PyObject* per_device = PyList_New(usage.live_tensors.size());
+    for (int i = 0; i < usage.live_tensors.size(); ++i) {
+      const auto& live_tensor = usage.live_tensors[i];
+      PyObject* live = PyTuple_New(5);
+      PyTuple_SetItem(live, 0, PyString_FromString(live_tensor.node.c_str()));
+      PyTuple_SetItem(live, 1, PyInt_FromLong(live_tensor.output_id));
+      PyTuple_SetItem(live, 2, PyLong_FromLong(live_tensor.memory_used));
+      PyTuple_SetItem(live, 3, PyLong_FromLong(live_tensor.allocation_time.count()));
+      PyTuple_SetItem(live, 4, PyLong_FromLong(live_tensor.deallocation_time.count()));
+      PyList_SetItem(per_device, i, live);
+
+    }
+    PyObject* ret = PyTuple_New(2);
+    PyTuple_SetItem(ret, 0, PyLong_FromLong(usage.used_memory));
+    PyTuple_SetItem(ret, 1, per_device);
+    PyDict_SetItem(result, PyString_FromString(device.first.c_str()), ret);
+  }
+  return result;
+}
+
 %}
 
 // Wrap these functions.
@@ -153,3 +201,6 @@ static void TF_DeleteCluster(tensorflow::grappler::Cluster* cluster);
 static PyObject* TF_MeasureCosts(
     const tensorflow::grappler::GrapplerItem* item, tensorflow::grappler::Cluster* cluster,
     bool generate_timeline, TF_Status* out_status);
+static PyObject* TF_DeterminePeakMemoryUsage(
+    const tensorflow::grappler::GrapplerItem* item, tensorflow::grappler::Cluster* cluster,
+    TF_Status* out_status);
diff --git a/tensorflow/python/grappler/cluster.py b/tensorflow/python/grappler/cluster.py
index ce6d5c111b..baac604f41 100644
--- a/tensorflow/python/grappler/cluster.py
+++ b/tensorflow/python/grappler/cluster.py
@@ -72,3 +72,16 @@ class Cluster(object):
           op_performance_data_pb2.OpPerformance.FromString(op_perf_bytes))
     return (op_perfs, run_time,
             step_stats_pb2.StepStats.FromString(step_stats_bytes))
+
+  def DeterminePeakMemoryUsage(self, item):
+    """Returns a snapshot of the peak memory usage.
+
+    Args:
+      item: the item for which to measure the costs.
+    Returns: a hashtable indexed by device name.
+    """
+    with errors.raise_exception_on_not_ok_status() as status:
+      ret_from_swig = tf_cluster.TF_DeterminePeakMemoryUsage(
+          item.tf_item, self._tf_cluster, status)
+
+    return ret_from_swig
diff --git a/tensorflow/python/grappler/cluster_test.py b/tensorflow/python/grappler/cluster_test.py
index e49ca69419..de4ded571f 100644
--- a/tensorflow/python/grappler/cluster_test.py
+++ b/tensorflow/python/grappler/cluster_test.py
@@ -62,6 +62,26 @@ class ClusterTest(test.TestCase):
       self.assertEqual(len(op_perfs), 0)
       self.assertEqual(len(step_stats.dev_stats), 0)
 
+  def testMemoryEstimates(self):
+    with ops.Graph().as_default() as g:
+      with ops.device('/job:localhost/replica:0/task:0/cpu:0'):
+        a = random_ops.random_uniform(shape=())
+        b = random_ops.random_uniform(shape=())
+        c = a + b
+        train_op = ops.get_collection_ref(ops.GraphKeys.TRAIN_OP)
+        train_op.append(c)
+        mg = meta_graph.create_meta_graph_def(graph=g)
+        grappler_item = item.Item(mg)
+        grappler_cluster = cluster.Cluster(
+            disable_detailed_stats=True, disable_timeline=True)
+        peak_mem = grappler_cluster.DeterminePeakMemoryUsage(grappler_item)
+        self.assertLessEqual(1, len(peak_mem))
+        snapshot = peak_mem['/job:localhost/replica:0/task:0/cpu:0']
+        peak_usage = snapshot[0]
+        self.assertEqual(52, peak_usage)
+        live_tensors = snapshot[1]
+        self.assertEqual(15, len(live_tensors))
+
 
 if __name__ == '__main__':
   test.main()
-- 
GitLab


From 5a8c47079f664b280bb28eb34ce2c93534305cda Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 13 Oct 2017 17:26:07 -0700
Subject: [PATCH 391/909] Optimized C++ and CUDA kernels for transposition.  *
 Shard fallback CPU implementation.  * Optimize index calculations by trading
 1 mod for 1 subtraction and 1 multiply (which have much lower combined
 latency).  * Add optimized GPU kernels for on-the-fly conjugate
 transposition.

PiperOrigin-RevId: 172167514
---
 tensorflow/core/kernels/BUILD                 |   2 +-
 tensorflow/core/kernels/conv_2d.h             |   4 +-
 tensorflow/core/kernels/conv_ops_gpu_3.cu.cc  | 138 +++++++++++++-----
 tensorflow/core/kernels/transpose_functor.h   |   3 +
 .../core/kernels/transpose_functor_cpu.cc     |  49 ++++---
 .../core/kernels/transpose_functor_gpu.cu.cc  |  45 +++---
 6 files changed, 155 insertions(+), 86 deletions(-)

diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 13dbf38fe6..2c02571346 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -1357,9 +1357,9 @@ tf_kernel_library(
     visibility = [":friends"],
     deps = [
         ":conv_ops",
-        ":cwise_op",
         ":ops_util",
         "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
         "//tensorflow/core:protos_all_cc",
         "//third_party/eigen3",
     ],
diff --git a/tensorflow/core/kernels/conv_2d.h b/tensorflow/core/kernels/conv_2d.h
index 8de8f1b265..f78a162a8e 100644
--- a/tensorflow/core/kernels/conv_2d.h
+++ b/tensorflow/core/kernels/conv_2d.h
@@ -260,7 +260,7 @@ struct NCHWToNHWC {
 //   [dim0, dim1, dim2]
 // to:
 //   [dim0, dim2, dim1]
-template <typename Device, typename T>
+template <typename Device, typename T, bool conjugate = false>
 struct SwapDimension1And2InTensor3 {
   void operator()(const Device& d, const T* in,
                   const gtl::ArraySlice<int64>& input_dims, T* out);
@@ -270,7 +270,7 @@ struct SwapDimension1And2InTensor3 {
 //   [dim0, dim1, dim2]
 // to:
 //   [dim2, dim1, dim0]
-template <typename Device, typename T>
+template <typename Device, typename T, bool conjugate = false>
 struct SwapDimension0And2InTensor3 {
   void operator()(const Device& d, const T* in,
                   const gtl::ArraySlice<int64>& input_dims, T* out);
diff --git a/tensorflow/core/kernels/conv_ops_gpu_3.cu.cc b/tensorflow/core/kernels/conv_ops_gpu_3.cu.cc
index 9083626fbf..6e10b53cf7 100644
--- a/tensorflow/core/kernels/conv_ops_gpu_3.cu.cc
+++ b/tensorflow/core/kernels/conv_ops_gpu_3.cu.cc
@@ -31,6 +31,48 @@ namespace tensorflow {
 typedef Eigen::GpuDevice GPUDevice;
 
 namespace functor {
+namespace {
+template <typename T, bool conjugate>
+struct maybe_conj {
+  __device__ static __inline__ T run(T x) {
+    if (conjugate) {
+      return Eigen::numext::conj(x);
+    } else {
+      return x;
+    }
+  }
+};
+
+// Partial specializations for Cuda types used to store complex numbers.
+template <bool conjugate>
+struct maybe_conj<float2, conjugate> {
+  __device__ static __inline__ float2 run(float2 c) {
+    if (conjugate) {
+      float2 c_conj;
+      c_conj.x = c.x;
+      c_conj.y = -c.y;
+      return c_conj;
+    } else {
+      return c;
+    }
+  }
+};
+
+template <bool conjugate>
+struct maybe_conj<double2, conjugate> {
+  __device__ static __inline__ double2 run(double2 c) {
+    if (conjugate) {
+      double2 c_conj;
+      c_conj.x = c.x;
+      c_conj.y = -c.y;
+      return c_conj;
+    } else {
+      return c;
+    }
+  }
+};
+
+}  // namespace
 
 // TODO(mjanusz): Move this to a shared util file.
 // A simple array that contains data that can be passed between CPU and GPU.
@@ -118,14 +160,15 @@ EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index<IndexCount> FlatToTensorIndex(
     int index, const Dimension<IndexCount>& dims) {
   Index<IndexCount> tensor_index;
   for (int i = IndexCount - 1; i >= 0; i--) {
-    tensor_index[i] = index % dims[i];
-    index /= dims[i];
+    int new_index = index / dims[i];
+    tensor_index[i] = index - dims[i] * new_index;
+    index = new_index;
   }
   return tensor_index;
 }
 
 // A Cuda custom kernel that swaps dimension-0 and dimension-2 of a 3D tensor.
-template <typename T>
+template <typename T, bool conjugate = false>
 __global__ void SwapDimension0And2InTensor3Simple(int nthreads, const T* input,
                                                   Dimension<3> input_dims,
                                                   T* output) {
@@ -146,12 +189,13 @@ __global__ void SwapDimension0And2InTensor3Simple(int nthreads, const T* input,
 
     int input_index = TensorIndexToFlat(input_tensor_index, input_dims);
 
-    output[output_index] = ldg(input + input_index);
+    output[output_index] =
+        maybe_conj<T, conjugate>::run(ldg(input + input_index));
   }
 }
 
 // A Cuda custom kernel that swaps dimension-1 and dimension-2 of a 3D tensor.
-template <typename T>
+template <typename T, bool conjugate = false>
 __global__ void SwapDimension1And2InTensor3Simple(int nthreads, const T* input,
                                                   Dimension<3> input_dims,
                                                   T* output) {
@@ -171,7 +215,8 @@ __global__ void SwapDimension1And2InTensor3Simple(int nthreads, const T* input,
 
     int input_index = TensorIndexToFlat(input_tensor_index, input_dims);
 
-    output[output_index] = ldg(input + input_index);
+    output[output_index] =
+        maybe_conj<T, conjugate>::run(ldg(input + input_index));
   }
 }
 
@@ -185,7 +230,7 @@ __global__ void SwapDimension1And2InTensor3Simple(int nthreads, const T* input,
 // For best performance, you should probably set TileSize equal to the number of
 // threads in a warp (32 in nvidia GPUs).  With a TileSize of 32, NumSubTiles ==
 // 4 or 8 seems to get the best performance on K40 GPUs.
-template <typename T, int TileSize, int NumSubTiles>
+template <typename T, int TileSize, int NumSubTiles, bool conjugate = false>
 __global__ void SwapDimension1And2InTensor3UsingTiles(const T* input,
                                                       Dimension<3> input_dims,
                                                       T* output) {
@@ -207,11 +252,14 @@ __global__ void SwapDimension1And2InTensor3UsingTiles(const T* input,
   int x = threadIdx.x;
 
   Dimension<3> output_dims = {
-      input_dims[0], input_dims[2], input_dims[1],
+      input_dims[0],
+      input_dims[2],
+      input_dims[1],
   };
 
   Dimension<3> input_dims_in_tiles = {
-      input_dims[0], (input_dims[1] + TileSize - 1) / TileSize,
+      input_dims[0],
+      (input_dims[1] + TileSize - 1) / TileSize,
       (input_dims[2] + TileSize - 1) / TileSize,
   };
 
@@ -219,7 +267,8 @@ __global__ void SwapDimension1And2InTensor3UsingTiles(const T* input,
       FlatToTensorIndex(blockIdx.x, input_dims_in_tiles);
 
   Index<3> input_tile_origin = {
-      input_tile_index[0], input_tile_index[1] * TileSize,
+      input_tile_index[0],
+      input_tile_index[1] * TileSize,
       input_tile_index[2] * TileSize,
   };
 
@@ -243,18 +292,22 @@ __global__ void SwapDimension1And2InTensor3UsingTiles(const T* input,
   if (x < tile_width) {
     int y_end = min(y_start + kSubTileSize, tile_height);
     for (int y = y_start; y < y_end; y++) {
-      shared_memory_tile[y][x] = input[input_flat_index + y * input_dims[2]];
+      shared_memory_tile[y][x] = maybe_conj<T, conjugate>::run(
+          input[input_flat_index + y * input_dims[2]]);
     }
   }
 
   __syncthreads();
 
   Index<3> output_tile_index = {
-      input_tile_index[0], input_tile_index[2], input_tile_index[1],
+      input_tile_index[0],
+      input_tile_index[2],
+      input_tile_index[1],
   };
 
   Index<3> output_tile_origin = {
-      output_tile_index[0], output_tile_index[1] * TileSize,
+      output_tile_index[0],
+      output_tile_index[1] * TileSize,
       output_tile_index[2] * TileSize,
   };
 
@@ -285,7 +338,7 @@ __global__ void SwapDimension1And2InTensor3UsingTiles(const T* input,
 // kTileLength (we currently set it to 64) and its height is small_dim,
 // We set the thread block's X dimension to be tile_num_per_block, and its Y
 // and Z to be one.
-template <typename T, int ShmemSize, bool SmallDim2>
+template <typename T, int ShmemSize, bool SmallDim2, bool conjugate = false>
 __global__ void SwapDimension1And2InTensor3SmallDim(const T* input,
                                                     int batch_per_block,
                                                     Dimension<3> input_dims,
@@ -328,9 +381,9 @@ __global__ void SwapDimension1And2InTensor3SmallDim(const T* input,
       for (int y = 0; y < small_dim; y++) {
         int shmem_index =
             SmallDim2 ? (x + y * tile_height) : (x * small_dim + y);
-        shared_memory_tile[shmem_index] =
+        shared_memory_tile[shmem_index] = maybe_conj<T, conjugate>::run(
             ldg(input + thread_origin_idx +
-                y * (SmallDim2 ? tile_height : large_dim));
+                y * (SmallDim2 ? tile_height : large_dim)));
       }
     }
 
@@ -480,15 +533,15 @@ struct PadInput<GPUDevice, T, int, NDIMS> {
     const Dimension<NDIMS - 2> padding_left_dim(padding_left);
 
     if (format == FORMAT_NHWC) {
-      PadInputCustomKernelNHWC<T, NDIMS><<<
-          config.block_count, config.thread_per_block, 0, d.stream()>>>(
-          config.virtual_thread_count, in.data(), input_dims, out.data(),
-          output_dims, padding_left_dim);
+      PadInputCustomKernelNHWC<T, NDIMS>
+          <<<config.block_count, config.thread_per_block, 0, d.stream()>>>(
+              config.virtual_thread_count, in.data(), input_dims, out.data(),
+              output_dims, padding_left_dim);
     } else if (format == FORMAT_NCHW) {
-      PadInputCustomKernelNCHW<T, NDIMS><<<
-          config.block_count, config.thread_per_block, 0, d.stream()>>>(
-          config.virtual_thread_count, in.data(), input_dims, out.data(),
-          output_dims, padding_left_dim);
+      PadInputCustomKernelNCHW<T, NDIMS>
+          <<<config.block_count, config.thread_per_block, 0, d.stream()>>>(
+              config.virtual_thread_count, in.data(), input_dims, out.data(),
+              output_dims, padding_left_dim);
     } else {
       LOG(FATAL) << "Invalid data format: " << format;
     }
@@ -498,7 +551,7 @@ struct PadInput<GPUDevice, T, int, NDIMS> {
 // Launch the GPU kernel that would swap dimension-1 and dimension-2 in a
 // 3D tensor. It looks at the shape of the incoming data, and decides the best
 // strategy to launch.
-template <typename T>
+template <typename T, bool conjugate = false>
 void RunSwapDimension1And2InTensor3(const GPUDevice& d, const T* input,
                                     const Dimension<3>& input_dims, T* output) {
   // If both dimensions are not trivial, use tiles for the actual swapping.
@@ -516,7 +569,8 @@ void RunSwapDimension1And2InTensor3(const GPUDevice& d, const T* input,
   if (use_tiles) {
     static const int TileSize = 32;
     Dimension<3> input_dims_in_tiles = {
-        input_dims[0], (input_dims[1] + TileSize - 1) / TileSize,
+        input_dims[0],
+        (input_dims[1] + TileSize - 1) / TileSize,
         (input_dims[2] + TileSize - 1) / TileSize,
     };
     int total_tiles_count = input_dims_in_tiles[0] * input_dims_in_tiles[1] *
@@ -524,9 +578,9 @@ void RunSwapDimension1And2InTensor3(const GPUDevice& d, const T* input,
     // We get best performance when TileSize is the number of threads in a warp
     // (32 on our GPUs) and NumSubTiles is 8, so our block size is 8 * 32 = 256
     // threads.
-    SwapDimension1And2InTensor3UsingTiles<T, TileSize, NumSubTiles><<<
-        total_tiles_count, dim3(TileSize, NumSubTiles), 0, d.stream()>>>(
-        input, input_dims, output);
+    SwapDimension1And2InTensor3UsingTiles<T, TileSize, NumSubTiles, conjugate>
+        <<<total_tiles_count, dim3(TileSize, NumSubTiles), 0, d.stream()>>>(
+            input, input_dims, output);
   } else if (use_small_dim) {
     // When only one of the dimensions is smaller than kMinDimensionToUseTiles,
     // we use one block to process a rectangle region with the size of
@@ -549,19 +603,19 @@ void RunSwapDimension1And2InTensor3(const GPUDevice& d, const T* input,
     int batch_per_block = (input_dims[0] + grid_dim_y - 1) / grid_dim_y;
     if (input_dims[2] < input_dims[1]) {
       SwapDimension1And2InTensor3SmallDim<
-          T, kTileLength * kMinDimensionToUseTiles, true>
+          T, kTileLength * kMinDimensionToUseTiles, true, conjugate>
           <<<dim3(tile_num_per_block, grid_dim_y), kTileLength, 0,
              d.stream()>>>(input, batch_per_block, input_dims, output);
     } else {
       SwapDimension1And2InTensor3SmallDim<
-          T, kTileLength * kMinDimensionToUseTiles, false>
+          T, kTileLength * kMinDimensionToUseTiles, false, conjugate>
           <<<dim3(tile_num_per_block, grid_dim_y), kTileLength, 0,
              d.stream()>>>(input, batch_per_block, input_dims, output);
     }
   } else {
     int total_element_count = input_dims[0] * input_dims[1] * input_dims[2];
     CudaLaunchConfig config = GetCudaLaunchConfig(total_element_count, d);
-    SwapDimension1And2InTensor3Simple<T>
+    SwapDimension1And2InTensor3Simple<T, conjugate>
         <<<config.block_count, config.thread_per_block, 0, d.stream()>>>(
             config.virtual_thread_count, input, input_dims, output);
   }
@@ -569,22 +623,22 @@ void RunSwapDimension1And2InTensor3(const GPUDevice& d, const T* input,
 
 // A GPU helper functor that does general dimension 1 and 2 switch for 3D
 // tensor.
-template <typename T>
-struct SwapDimension1And2InTensor3<GPUDevice, T> {
+template <typename T, bool conjugate>
+struct SwapDimension1And2InTensor3<GPUDevice, T, conjugate> {
   typedef GPUDevice Device;
   void operator()(const Device& d, const T* in,
                   const gtl::ArraySlice<int64>& combined_dims, T* out) {
     Dimension<3> input_dims = {static_cast<int>(combined_dims[0]),
                                static_cast<int>(combined_dims[1]),
                                static_cast<int>(combined_dims[2])};
-    RunSwapDimension1And2InTensor3(d, in, input_dims, out);
+    RunSwapDimension1And2InTensor3<T, conjugate>(d, in, input_dims, out);
   }
 };
 
 // A GPU helper functor that does general dimension 0 and 2 switch for 3D
 // tensor.
-template <typename T>
-struct SwapDimension0And2InTensor3<GPUDevice, T> {
+template <typename T, bool conjugate>
+struct SwapDimension0And2InTensor3<GPUDevice, T, conjugate> {
   typedef GPUDevice Device;
   void operator()(const Device& d, const T* in,
                   const gtl::ArraySlice<int64>& combined_dims, T* out) {
@@ -593,7 +647,7 @@ struct SwapDimension0And2InTensor3<GPUDevice, T> {
                                static_cast<int>(combined_dims[2])};
     size_t total_size = combined_dims[0] * combined_dims[1] * combined_dims[2];
     CudaLaunchConfig config = GetCudaLaunchConfig(total_size, d);
-    SwapDimension0And2InTensor3Simple<T>
+    SwapDimension0And2InTensor3Simple<T, conjugate>
         <<<config.block_count, config.thread_per_block, 0, d.stream()>>>(
             config.virtual_thread_count, in, input_dims, out);
   }
@@ -653,12 +707,20 @@ template struct functor::SwapDimension1And2InTensor3<GPUDevice, uint16>;
 template struct functor::SwapDimension1And2InTensor3<GPUDevice, uint32>;
 template struct functor::SwapDimension1And2InTensor3<GPUDevice, uint64>;
 template struct functor::SwapDimension1And2InTensor3<GPUDevice, float4>;
+template struct functor::SwapDimension1And2InTensor3<GPUDevice, float2,
+                                                     /*conjugate=*/true>;
+template struct functor::SwapDimension1And2InTensor3<GPUDevice, double2,
+                                                     /*conjugate=*/true>;
 
 template struct functor::SwapDimension0And2InTensor3<GPUDevice, uint8>;
 template struct functor::SwapDimension0And2InTensor3<GPUDevice, uint16>;
 template struct functor::SwapDimension0And2InTensor3<GPUDevice, uint32>;
 template struct functor::SwapDimension0And2InTensor3<GPUDevice, uint64>;
 template struct functor::SwapDimension0And2InTensor3<GPUDevice, float4>;
+template struct functor::SwapDimension0And2InTensor3<GPUDevice, float2,
+                                                     /*conjugate=*/true>;
+template struct functor::SwapDimension0And2InTensor3<GPUDevice, double2,
+                                                     /*conjugate=*/true>;
 
 // For 2d ops.
 template struct functor::TransformFilter<GPUDevice, float, int, 4>;
diff --git a/tensorflow/core/kernels/transpose_functor.h b/tensorflow/core/kernels/transpose_functor.h
index 317a534fd6..87569f0275 100644
--- a/tensorflow/core/kernels/transpose_functor.h
+++ b/tensorflow/core/kernels/transpose_functor.h
@@ -16,8 +16,11 @@ limitations under the License.
 #ifndef TENSORFLOW_CORE_KERNELS_TRANSPOSE_FUNCTOR_H_
 #define TENSORFLOW_CORE_KERNELS_TRANSPOSE_FUNCTOR_H_
 
+#include <string>
+#include <vector>
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_types.h"
+#include "tensorflow/core/platform/logging.h"
 
 namespace tensorflow {
 
diff --git a/tensorflow/core/kernels/transpose_functor_cpu.cc b/tensorflow/core/kernels/transpose_functor_cpu.cc
index b983bf695c..b2de012be1 100644
--- a/tensorflow/core/kernels/transpose_functor_cpu.cc
+++ b/tensorflow/core/kernels/transpose_functor_cpu.cc
@@ -15,9 +15,16 @@ limitations under the License.
 
 #define EIGEN_USE_THREADS
 
+#include <complex>
+
+#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 #include "tensorflow/core/framework/attr_value.pb.h"
+#include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/kernels/ops_util.h"
 #include "tensorflow/core/kernels/transpose_functor.h"
+#include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/lib/gtl/array_slice.h"
+#include "tensorflow/core/lib/gtl/inlined_vector.h"
 
 typedef Eigen::ThreadPoolDevice CPUDevice;
 
@@ -25,30 +32,36 @@ namespace tensorflow {
 namespace internal {
 
 template <typename Device, typename T, bool conjugate>
-void TransposeSimple(const Device& d, const Tensor& in,
+void TransposeSimple(const Device& device, const Tensor& in,
                      const gtl::ArraySlice<int32> perm, Tensor* out) {
   const int ndims = in.dims();
   gtl::InlinedVector<int64, 8> in_strides = ComputeStride<int64>(in.shape());
   gtl::InlinedVector<int64, 8> out_strides = ComputeStride<int64>(out->shape());
-  const int64 nelem = in.NumElements();
   const T* p = reinterpret_cast<const T*>(in.tensor_data().data());
   T* q = reinterpret_cast<T*>(const_cast<char*>((out->tensor_data().data())));
-
-  // TODO(zhifengc): Shard by range.
-  // TODO(zhifengc): Avoids the division.
-  for (int64 o_idx = 0; o_idx < nelem; ++o_idx) {
-    int64 i_idx = 0;
-    int64 t = o_idx;
-    for (int i = 0; i < ndims; ++i) {
-      i_idx += (t / out_strides[i]) * in_strides[perm[i]];
-      t = t % out_strides[i];
+  auto transpose_fn = [=](int64 begin, int64 end) {
+    for (int64 o_idx = begin; o_idx < end; ++o_idx) {
+      int64 i_idx = 0;
+      int64 t = o_idx;
+      for (int i = 0; i < ndims; ++i) {
+        const int64 ratio = t / out_strides[i];
+        t -= ratio * out_strides[i];
+        i_idx += ratio * in_strides[perm[i]];
+      }
+      if (conjugate) {
+        q[o_idx] = Eigen::numext::conj(p[i_idx]);
+      } else {
+        q[o_idx] = p[i_idx];
+      }
     }
-    if (conjugate) {
-      q[o_idx] = Eigen::numext::conj(p[i_idx]);
-    } else {
-      q[o_idx] = p[i_idx];
-    }
-  }
+  };
+  double cycles_per_element =
+      (conjugate ? 1 : 0) + ndims * (Eigen::TensorOpCost::DivCost<int64>() +
+                                     2 * Eigen::TensorOpCost::MulCost<int64>() +
+                                     2 * Eigen::TensorOpCost::AddCost<int64>());
+  Eigen::TensorOpCost cost(/*bytes_loaded=*/sizeof(T),
+                           /*bytes_stored=*/sizeof(T), cycles_per_element);
+  device.parallelFor(in.NumElements(), cost, std::move(transpose_fn));
 }
 
 }  // end namespace internal
@@ -166,6 +179,6 @@ Status DoConjugateTranspose(const SYCLDevice& device, const Tensor& in,
                                                     true /* conjugate */, out);
 }
 
-#endif // TENSORFLOW_USE_SYCL
+#endif  // TENSORFLOW_USE_SYCL
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/kernels/transpose_functor_gpu.cu.cc b/tensorflow/core/kernels/transpose_functor_gpu.cu.cc
index 87af1ba0c4..364baf9a51 100644
--- a/tensorflow/core/kernels/transpose_functor_gpu.cu.cc
+++ b/tensorflow/core/kernels/transpose_functor_gpu.cu.cc
@@ -18,7 +18,6 @@ limitations under the License.
 #define EIGEN_USE_GPU
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
-#include "tensorflow/core/kernels/cwise_ops.h"
 #include "tensorflow/core/kernels/ops_util.h"
 #include "tensorflow/core/kernels/transpose_functor.h"
 #include "tensorflow/core/util/cuda_kernel_helper.h"
@@ -41,9 +40,10 @@ __global__ void TransposeKernel(int nthreads, const T* src, const int32* buf,
   CUDA_1D_KERNEL_LOOP(o_idx, nthreads) {
     int32 i_idx = 0;
     int32 t = o_idx;
-    for (int i = 0; i < ndims; ++i) {
-      i_idx += (t / out_strides[i]) * in_strides[perm[i]];
-      t = t % out_strides[i];
+    for (int32 i = 0; i < ndims; ++i) {
+      const int32 ratio = t / out_strides[i];
+      t -= ratio * out_strides[i];
+      i_idx += ratio * in_strides[perm[i]];
     }
     if (conjugate) {
       dst[o_idx] = Eigen::numext::conj(ldg(src + i_idx));
@@ -112,18 +112,21 @@ struct TransposeUsingTile {
         if (new_perm[0] == 1 && new_perm[1] == 0) {
           // Add the first dimension size as 1.
           new_dims.insert(new_dims.begin(), 1);
-          tensorflow::functor::SwapDimension1And2InTensor3<GPUDevice, T>()(
+          tensorflow::functor::SwapDimension1And2InTensor3<GPUDevice, T,
+                                                           conjugate>()(
               d, in_data, new_dims, out_data);
           return true;
         }
         break;
       case 3:
         if (new_perm == TransposePermsVec({0, 2, 1})) {
-          tensorflow::functor::SwapDimension1And2InTensor3<GPUDevice, T>()(
+          tensorflow::functor::SwapDimension1And2InTensor3<GPUDevice, T,
+                                                           conjugate>()(
               d, in_data, new_dims, out_data);
           return true;
         } else if (new_perm == TransposePermsVec({2, 1, 0})) {
-          tensorflow::functor::SwapDimension0And2InTensor3<GPUDevice, T>()(
+          tensorflow::functor::SwapDimension0And2InTensor3<GPUDevice, T,
+                                                           conjugate>()(
               d, in_data, new_dims, out_data);
           return true;
         } else {
@@ -142,17 +145,11 @@ template <bool conjugate>
 struct TransposeUsingTile<complex64, conjugate> {
   static bool run(const Eigen::GpuDevice& d, const Tensor& in,
                   const gtl::ArraySlice<int32> perm, Tensor* out) {
-    if (!TransposeUsingTile<uint64>::run(d, in, perm, out)) {
-      return false;
-    }
-    if (conjugate) {
-      // TODO(rmlarsen): Get rid of this call and conjugate on-the-fly in the
-      // transposition kernels so we only touch the memory once.
-      functor::UnaryFunctor<GPUDevice, functor::conj<complex64>> conj;
-      conj(d, out->flat<complex64>() /*out*/,
-           const_cast<const Tensor*>(out)->flat<complex64>() /*in*/);
+    if (!conjugate) {
+      return TransposeUsingTile<uint64>::run(d, in, perm, out);
+    } else {
+      return TransposeUsingTile<float2, true>::run(d, in, perm, out);
     }
-    return true;
   }
 };
 
@@ -160,17 +157,11 @@ template <bool conjugate>
 struct TransposeUsingTile<complex128, conjugate> {
   static bool run(const Eigen::GpuDevice& d, const Tensor& in,
                   const gtl::ArraySlice<int32> perm, Tensor* out) {
-    if (!TransposeUsingTile<float4>::run(d, in, perm, out)) {
-      return false;
-    }
-    if (conjugate) {
-      // TODO(rmlarsen): Get rid of this call and conjugate on-the-fly in the
-      // transposition kernels so we only touch the memory once.
-      functor::UnaryFunctor<GPUDevice, functor::conj<complex128>> conj;
-      conj(d, out->flat<complex128>() /*out*/,
-           const_cast<const Tensor*>(out)->flat<complex128>() /*in*/);
+    if (!conjugate) {
+      return TransposeUsingTile<float4>::run(d, in, perm, out);
+    } else {
+      return TransposeUsingTile<double2, true>::run(d, in, perm, out);
     }
-    return true;
   }
 };
 
-- 
GitLab


From 860f8c50753bcbfca8243c585033b3d44c4b7c7f Mon Sep 17 00:00:00 2001
From: Chris Ying <chrisying@google.com>
Date: Fri, 13 Oct 2017 18:00:04 -0700
Subject: [PATCH 392/909] Fix case where broadcasting is not necessary.

PiperOrigin-RevId: 172169909
---
 tensorflow/python/layers/normalization.py | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/tensorflow/python/layers/normalization.py b/tensorflow/python/layers/normalization.py
index d82946382f..df2b97f03e 100644
--- a/tensorflow/python/layers/normalization.py
+++ b/tensorflow/python/layers/normalization.py
@@ -477,7 +477,8 @@ class BatchNormalization(base.Layer):
 
     # Compute the axes along which to reduce the mean / variance
     input_shape = inputs.get_shape()
-    reduction_axes = [i for i in range(len(input_shape)) if i not in self.axis]
+    ndims = len(input_shape)
+    reduction_axes = [i for i in range(ndims) if i not in self.axis]
     if self.virtual_batch_size is not None:
       del reduction_axes[1]     # Do not reduce along virtual batch dim
 
@@ -541,13 +542,15 @@ class BatchNormalization(base.Layer):
     else:
       mean, variance = self.moving_mean, self.moving_variance
 
-    # Broadcasting only necessary for single-axis batch norm
-    broadcast_shape = [1] * len(input_shape)
+    # Broadcasting only necessary for single-axis batch norm where the axis is
+    # not the last dimension
+    broadcast_shape = [1] * ndims
     broadcast_shape[self.axis[0]] = input_shape[self.axis[0]].value
     rank = len(inputs.get_shape())
     def _broadcast(v):
-      if v is not None and len(v.get_shape()) != rank:
-        assert len(self.axis) == 1
+      if (v is not None and
+          len(v.get_shape()) != rank and
+          reduction_axes != list(range(ndims))[:-1]):
         return array_ops.reshape(v, broadcast_shape)
       return v
 
-- 
GitLab


From 7cdd26f606b39e3e487ec15dfa6eb5c6cf63ef84 Mon Sep 17 00:00:00 2001
From: gunan <gunan@google.com>
Date: Fri, 13 Oct 2017 22:24:27 -0700
Subject: [PATCH 393/909] Reenable tests that use matrix_set_diag op. (#13708)

---
 tensorflow/contrib/cmake/tf_tests.cmake | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/tensorflow/contrib/cmake/tf_tests.cmake b/tensorflow/contrib/cmake/tf_tests.cmake
index 530fcee774..9108586e17 100644
--- a/tensorflow/contrib/cmake/tf_tests.cmake
+++ b/tensorflow/contrib/cmake/tf_tests.cmake
@@ -199,7 +199,7 @@ if (tensorflow_BUILD_PYTHON_TESTS)
     "${tensorflow_source_dir}/tensorflow/contrib/keras/python/keras/preprocessing/*_test.py"
     "${tensorflow_source_dir}/tensorflow/contrib/tfprof/python/tools/tfprof/pprof_profiler_test.py"
     # flaky tests
-    "${tensorflow_source_dir}/tensorflow/python/kernel_tests/cwise_ops_test.py"
+    "${tensorflow_source_dir}/tensorflow/python/kernel_tests/cwise_ops_test.py"  # takes very long to run
     "${tensorflow_source_dir}/tensorflow/contrib/tfprof/python/tools/tfprof/internal/run_metadata_test.py"
     # Loading resources in contrib doesn't seem to work on Windows
     "${tensorflow_source_dir}/tensorflow/contrib/tensor_forest/client/random_forest_test.py"
@@ -225,11 +225,6 @@ if (tensorflow_BUILD_PYTHON_TESTS)
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/string_to_number_op_test.py"
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/clip_ops_test.py"
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/tensor_array_ops_test.py"  # Needs portpicker.
-      # Matrix_set_diag failing on GPU on windows.
-      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/cholesky_op_test.py"
-      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/diag_op_test.py"
-      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/linalg_ops_test.py"
-      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/init_ops_test.py"
       # Type error in testRemoteIteratorUsingRemoteCallOpDirectSessionGPUCPU.
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/iterator_ops_test.py"
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/self_adjoint_eig_op_test.py"
@@ -251,7 +246,6 @@ if (tensorflow_BUILD_PYTHON_TESTS)
       "${tensorflow_source_dir}/tensorflow/python/training/server_lib_test.py"  # Test occasionally deadlocks.
       "${tensorflow_source_dir}/tensorflow/python/debug/lib/session_debug_multi_gpu_test.py"
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/concat_op_test.py"  # numerical issues
-      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/linalg_grad_test.py"  # cudaSolver handle creation fails.
 
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/array_ops_test.py"  # depends on python/framework/test_ops     
       # Dataset tests
-- 
GitLab


From a3667c483ebf839653d71cf42a0a71196a513dc9 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Sat, 14 Oct 2017 01:54:54 -0700
Subject: [PATCH 394/909] Add streaming_false_{negative,positive}_rate and
 streaming_false_{negative,positive}_rate_at_thresholds.

PiperOrigin-RevId: 172191462
---
 tensorflow/contrib/metrics/__init__.py        |   8 +
 .../contrib/metrics/python/ops/metric_ops.py  | 347 +++++++++
 .../metrics/python/ops/metric_ops_test.py     | 720 ++++++++++++++++++
 3 files changed, 1075 insertions(+)

diff --git a/tensorflow/contrib/metrics/__init__.py b/tensorflow/contrib/metrics/__init__.py
index a9bce65e55..2c48882d0e 100644
--- a/tensorflow/contrib/metrics/__init__.py
+++ b/tensorflow/contrib/metrics/__init__.py
@@ -22,6 +22,10 @@ See the @{$python/contrib.metrics} guide.
 @@streaming_recall_at_thresholds
 @@streaming_precision
 @@streaming_precision_at_thresholds
+@@streaming_false_positive_rate
+@@streaming_false_positive_rate_at_thresholds
+@@streaming_false_negative_rate
+@@streaming_false_negative_rate_at_thresholds
 @@streaming_auc
 @@streaming_curve_points
 @@streaming_recall_at_k
@@ -80,8 +84,12 @@ from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_auc
 from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_concat
 from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_covariance
 from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_curve_points
+from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_false_negative_rate
+from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_false_negative_rate_at_thresholds
 from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_false_negatives
 from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_false_negatives_at_thresholds
+from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_false_positive_rate
+from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_false_positive_rate_at_thresholds
 from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_false_positives
 from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_false_positives_at_thresholds
 from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_mean
diff --git a/tensorflow/contrib/metrics/python/ops/metric_ops.py b/tensorflow/contrib/metrics/python/ops/metric_ops.py
index 76986d0156..85c8e9038a 100644
--- a/tensorflow/contrib/metrics/python/ops/metric_ops.py
+++ b/tensorflow/contrib/metrics/python/ops/metric_ops.py
@@ -565,6 +565,213 @@ def streaming_recall(predictions, labels, weights=None,
       updates_collections=updates_collections, name=name)
 
 
+def _true_negatives(labels, predictions, weights=None,
+                    metrics_collections=None,
+                    updates_collections=None,
+                    name=None):
+  """Sum the weights of true negatives.
+
+  If `weights` is `None`, weights default to 1. Use weights of 0 to mask values.
+
+  Args:
+    labels: The ground truth values, a `Tensor` whose dimensions must match
+      `predictions`. Will be cast to `bool`.
+    predictions: The predicted values, a `Tensor` of arbitrary dimensions. Will
+      be cast to `bool`.
+    weights: Optional `Tensor` whose rank is either 0, or the same rank as
+      `labels`, and must be broadcastable to `labels` (i.e., all dimensions must
+      be either `1`, or the same as the corresponding `labels` dimension).
+    metrics_collections: An optional list of collections that the metric
+      value variable should be added to.
+    updates_collections: An optional list of collections that the metric update
+      ops should be added to.
+    name: An optional variable_scope name.
+
+  Returns:
+    value_tensor: A `Tensor` representing the current value of the metric.
+    update_op: An operation that accumulates the error from a batch of data.
+
+  Raises:
+    ValueError: If `predictions` and `labels` have mismatched shapes, or if
+      `weights` is not `None` and its shape doesn't match `predictions`, or if
+      either `metrics_collections` or `updates_collections` are not a list or
+      tuple.
+  """
+  with variable_scope.variable_scope(
+      name, 'true_negatives', (predictions, labels, weights)):
+
+    predictions, labels, weights = _remove_squeezable_dimensions(
+        predictions=math_ops.cast(predictions, dtype=dtypes.bool),
+        labels=math_ops.cast(labels, dtype=dtypes.bool),
+        weights=weights)
+    is_true_negative = math_ops.logical_and(math_ops.equal(labels, False),
+                                            math_ops.equal(predictions, False))
+    return _count_condition(is_true_negative, weights, metrics_collections,
+                            updates_collections)
+
+
+def streaming_false_positive_rate(predictions, labels, weights=None,
+                                  metrics_collections=None,
+                                  updates_collections=None,
+                                  name=None):
+  """Computes the false positive rate of predictions with respect to labels.
+
+  The `false_positive_rate` function creates two local variables,
+  `false_positives` and `true_negatives`, that are used to compute the
+  false positive rate. This value is ultimately returned as
+  `false_positive_rate`, an idempotent operation that simply divides
+  `false_positives` by the sum of `false_positives` and `true_negatives`.
+
+  For estimation of the metric over a stream of data, the function creates an
+  `update_op` operation that updates these variables and returns the
+  `false_positive_rate`. `update_op` weights each prediction by the
+  corresponding value in `weights`.
+
+  If `weights` is `None`, weights default to 1. Use weights of 0 to mask values.
+
+  Args:
+    predictions: The predicted values, a `Tensor` of arbitrary dimensions. Will
+      be cast to `bool`.
+    labels: The ground truth values, a `Tensor` whose dimensions must match
+      `predictions`. Will be cast to `bool`.
+    weights: Optional `Tensor` whose rank is either 0, or the same rank as
+      `labels`, and must be broadcastable to `labels` (i.e., all dimensions must
+      be either `1`, or the same as the corresponding `labels` dimension).
+    metrics_collections: An optional list of collections that
+     `false_positive_rate` should be added to.
+    updates_collections: An optional list of collections that `update_op` should
+      be added to.
+    name: An optional variable_scope name.
+
+  Returns:
+    false_positive_rate: Scalar float `Tensor` with the value of
+      `false_positives` divided by the sum of `false_positives` and
+      `true_negatives`.
+    update_op: `Operation` that increments `false_positives` and
+      `true_negatives` variables appropriately and whose value matches
+      `false_positive_rate`.
+
+  Raises:
+    ValueError: If `predictions` and `labels` have mismatched shapes, or if
+      `weights` is not `None` and its shape doesn't match `predictions`, or if
+      either `metrics_collections` or `updates_collections` are not a list or
+      tuple.
+  """
+  with variable_scope.variable_scope(
+      name, 'false_positive_rate', (predictions, labels, weights)):
+    predictions, labels, weights = _remove_squeezable_dimensions(
+        predictions=math_ops.cast(predictions, dtype=dtypes.bool),
+        labels=math_ops.cast(labels, dtype=dtypes.bool),
+        weights=weights)
+
+    false_p, false_positives_update_op = metrics.false_positives(
+        labels, predictions, weights, metrics_collections=None,
+        updates_collections=None, name=None)
+    true_n, true_negatives_update_op = _true_negatives(
+        labels, predictions, weights, metrics_collections=None,
+        updates_collections=None, name=None)
+
+    def compute_fpr(fp, tn, name):
+      return array_ops.where(
+          math_ops.greater(fp + tn, 0),
+          math_ops.div(fp, fp + tn),
+          0,
+          name)
+
+    fpr = compute_fpr(false_p, true_n, 'value')
+    update_op = compute_fpr(
+        false_positives_update_op, true_negatives_update_op, 'update_op')
+
+    if metrics_collections:
+      ops.add_to_collections(metrics_collections, fpr)
+
+    if updates_collections:
+      ops.add_to_collections(updates_collections, update_op)
+
+    return fpr, update_op
+
+
+def streaming_false_negative_rate(predictions, labels, weights=None,
+                                  metrics_collections=None,
+                                  updates_collections=None,
+                                  name=None):
+  """Computes the false negative rate of predictions with respect to labels.
+
+  The `false_negative_rate` function creates two local variables,
+  `false_negatives` and `true_positives`, that are used to compute the
+  false positive rate. This value is ultimately returned as
+  `false_negative_rate`, an idempotent operation that simply divides
+  `false_negatives` by the sum of `false_negatives` and `true_positives`.
+
+  For estimation of the metric over a stream of data, the function creates an
+  `update_op` operation that updates these variables and returns the
+  `false_negative_rate`. `update_op` weights each prediction by the
+  corresponding value in `weights`.
+
+  If `weights` is `None`, weights default to 1. Use weights of 0 to mask values.
+
+  Args:
+    predictions: The predicted values, a `Tensor` of arbitrary dimensions. Will
+      be cast to `bool`.
+    labels: The ground truth values, a `Tensor` whose dimensions must match
+      `predictions`. Will be cast to `bool`.
+    weights: Optional `Tensor` whose rank is either 0, or the same rank as
+      `labels`, and must be broadcastable to `labels` (i.e., all dimensions must
+      be either `1`, or the same as the corresponding `labels` dimension).
+    metrics_collections: An optional list of collections that
+      `false_negative_rate` should be added to.
+    updates_collections: An optional list of collections that `update_op` should
+      be added to.
+    name: An optional variable_scope name.
+
+  Returns:
+    false_negative_rate: Scalar float `Tensor` with the value of
+      `false_negatives` divided by the sum of `false_negatives` and
+      `true_positives`.
+    update_op: `Operation` that increments `false_negatives` and
+      `true_positives` variables appropriately and whose value matches
+      `false_negative_rate`.
+
+  Raises:
+    ValueError: If `predictions` and `labels` have mismatched shapes, or if
+      `weights` is not `None` and its shape doesn't match `predictions`, or if
+      either `metrics_collections` or `updates_collections` are not a list or
+      tuple.
+  """
+  with variable_scope.variable_scope(
+      name, 'false_negative_rate', (predictions, labels, weights)):
+    predictions, labels, weights = _remove_squeezable_dimensions(
+        predictions=math_ops.cast(predictions, dtype=dtypes.bool),
+        labels=math_ops.cast(labels, dtype=dtypes.bool),
+        weights=weights)
+
+    false_n, false_negatives_update_op = metrics.false_negatives(
+        labels, predictions, weights, metrics_collections=None,
+        updates_collections=None, name=None)
+    true_p, true_positives_update_op = metrics.true_positives(
+        labels, predictions, weights, metrics_collections=None,
+        updates_collections=None, name=None)
+
+    def compute_fnr(fn, tp, name):
+      return array_ops.where(
+          math_ops.greater(fn + tp, 0),
+          math_ops.div(fn, fn + tp),
+          0,
+          name)
+
+    fnr = compute_fnr(false_n, true_p, 'value')
+    update_op = compute_fnr(
+        false_negatives_update_op, true_positives_update_op, 'update_op')
+
+    if metrics_collections:
+      ops.add_to_collections(metrics_collections, fnr)
+
+    if updates_collections:
+      ops.add_to_collections(updates_collections, update_op)
+
+    return fnr, update_op
+
+
 def _streaming_confusion_matrix_at_thresholds(
     predictions, labels, thresholds, weights=None, includes=None):
   """Computes true_positives, false_negatives, true_negatives, false_positives.
@@ -1114,6 +1321,142 @@ def streaming_recall_at_thresholds(predictions, labels, thresholds,
       updates_collections=updates_collections, name=name)
 
 
+def streaming_false_positive_rate_at_thresholds(
+    predictions, labels, thresholds, weights=None, metrics_collections=None,
+    updates_collections=None, name=None):
+  """Computes various fpr values for different `thresholds` on `predictions`.
+
+  The `streaming_false_positive_rate_at_thresholds` function creates two
+  local variables, `false_positives`, `true_negatives`, for various values of
+  thresholds. `false_positive_rate[i]` is defined as the total weight
+  of values in `predictions` above `thresholds[i]` whose corresponding entry in
+  `labels` is `False`, divided by the total weight of `False` values in `labels`
+  (`false_positives[i] / (false_positives[i] + true_negatives[i])`).
+
+  For estimation of the metric over a stream of data, the function creates an
+  `update_op` operation that updates these variables and returns the
+  `false_positive_rate`.
+
+  If `weights` is `None`, weights default to 1. Use weights of 0 to mask values.
+
+  Args:
+    predictions: A floating point `Tensor` of arbitrary shape and whose values
+      are in the range `[0, 1]`.
+    labels: A `bool` `Tensor` whose shape matches `predictions`.
+    thresholds: A python list or tuple of float thresholds in `[0, 1]`.
+    weights: `Tensor` whose rank is either 0, or the same rank as `labels`, and
+      must be broadcastable to `labels` (i.e., all dimensions must be either
+      `1`, or the same as the corresponding `labels` dimension).
+    metrics_collections: An optional list of collections that
+      `false_positive_rate` should be added to.
+    updates_collections: An optional list of collections that `update_op` should
+      be added to.
+    name: An optional variable_scope name.
+
+  Returns:
+    false_positive_rate: A float `Tensor` of shape `[len(thresholds)]`.
+    update_op: An operation that increments the `false_positives` and
+      `true_negatives` variables that are used in the computation of
+      `false_positive_rate`.
+
+  Raises:
+    ValueError: If `predictions` and `labels` have mismatched shapes, or if
+      `weights` is not `None` and its shape doesn't match `predictions`, or if
+      either `metrics_collections` or `updates_collections` are not a list or
+      tuple.
+  """
+  with variable_scope.variable_scope(
+      name, 'false_positive_rate_at_thresholds',
+      (predictions, labels, weights)):
+    values, update_ops = _streaming_confusion_matrix_at_thresholds(
+        predictions, labels, thresholds, weights, includes=('fp', 'tn'))
+
+    # Avoid division by zero.
+    epsilon = 1e-7
+    def compute_fpr(fp, tn, name):
+      return math_ops.div(fp, epsilon + fp + tn, name='fpr_' + name)
+
+    fpr = compute_fpr(values['fp'], values['tn'], 'value')
+    update_op = compute_fpr(
+        update_ops['fp'], update_ops['tn'], 'update_op')
+
+    if metrics_collections:
+      ops.add_to_collections(metrics_collections, fpr)
+
+    if updates_collections:
+      ops.add_to_collections(updates_collections, update_op)
+
+    return fpr, update_op
+
+
+def streaming_false_negative_rate_at_thresholds(
+    predictions, labels, thresholds, weights=None, metrics_collections=None,
+    updates_collections=None, name=None):
+  """Computes various fnr values for different `thresholds` on `predictions`.
+
+  The `streaming_false_negative_rate_at_thresholds` function creates two
+  local variables, `false_negatives`, `true_positives`, for various values of
+  thresholds. `false_negative_rate[i]` is defined as the total weight
+  of values in `predictions` above `thresholds[i]` whose corresponding entry in
+  `labels` is `False`, divided by the total weight of `True` values in `labels`
+  (`false_negatives[i] / (false_negatives[i] + true_positives[i])`).
+
+  For estimation of the metric over a stream of data, the function creates an
+  `update_op` operation that updates these variables and returns the
+  `false_positive_rate`.
+
+  If `weights` is `None`, weights default to 1. Use weights of 0 to mask values.
+
+  Args:
+    predictions: A floating point `Tensor` of arbitrary shape and whose values
+      are in the range `[0, 1]`.
+    labels: A `bool` `Tensor` whose shape matches `predictions`.
+    thresholds: A python list or tuple of float thresholds in `[0, 1]`.
+    weights: `Tensor` whose rank is either 0, or the same rank as `labels`, and
+      must be broadcastable to `labels` (i.e., all dimensions must be either
+      `1`, or the same as the corresponding `labels` dimension).
+    metrics_collections: An optional list of collections that
+      `false_negative_rate` should be added to.
+    updates_collections: An optional list of collections that `update_op` should
+      be added to.
+    name: An optional variable_scope name.
+
+  Returns:
+    false_negative_rate: A float `Tensor` of shape `[len(thresholds)]`.
+    update_op: An operation that increments the `false_negatives` and
+      `true_positives` variables that are used in the computation of
+      `false_negative_rate`.
+
+  Raises:
+    ValueError: If `predictions` and `labels` have mismatched shapes, or if
+      `weights` is not `None` and its shape doesn't match `predictions`, or if
+      either `metrics_collections` or `updates_collections` are not a list or
+      tuple.
+  """
+  with variable_scope.variable_scope(
+      name, 'false_negative_rate_at_thresholds',
+      (predictions, labels, weights)):
+    values, update_ops = _streaming_confusion_matrix_at_thresholds(
+        predictions, labels, thresholds, weights, includes=('fn', 'tp'))
+
+    # Avoid division by zero.
+    epsilon = 1e-7
+    def compute_fnr(fn, tp, name):
+      return math_ops.div(fn, epsilon + fn + tp, name='fnr_' + name)
+
+    fnr = compute_fnr(values['fn'], values['tp'], 'value')
+    update_op = compute_fnr(
+        update_ops['fn'], update_ops['tp'], 'update_op')
+
+    if metrics_collections:
+      ops.add_to_collections(metrics_collections, fnr)
+
+    if updates_collections:
+      ops.add_to_collections(updates_collections, update_op)
+
+    return fnr, update_op
+
+
 def _at_k_name(name, k=None, class_id=None):
   if k is not None:
     name = '%s_at_%d' % (name, k)
@@ -2479,8 +2822,12 @@ __all__ = [
     'streaming_accuracy',
     'streaming_auc',
     'streaming_curve_points',
+    'streaming_false_negative_rate',
+    'streaming_false_negative_rate_at_thresholds',
     'streaming_false_negatives',
     'streaming_false_negatives_at_thresholds',
+    'streaming_false_positive_rate',
+    'streaming_false_positive_rate_at_thresholds',
     'streaming_false_positives',
     'streaming_false_positives_at_thresholds',
     'streaming_mean',
diff --git a/tensorflow/contrib/metrics/python/ops/metric_ops_test.py b/tensorflow/contrib/metrics/python/ops/metric_ops_test.py
index 9b959b43a9..cc0ad155fa 100644
--- a/tensorflow/contrib/metrics/python/ops/metric_ops_test.py
+++ b/tensorflow/contrib/metrics/python/ops/metric_ops_test.py
@@ -1355,6 +1355,262 @@ class StreamingRecallTest(test.TestCase):
       self.assertEqual(0, recall.eval())
 
 
+class StreamingFPRTest(test.TestCase):
+
+  def setUp(self):
+    np.random.seed(1)
+    ops.reset_default_graph()
+
+  def testVars(self):
+    metrics.streaming_false_positive_rate(
+        predictions=array_ops.ones((10, 1)), labels=array_ops.ones((10, 1)))
+    _assert_local_variables(self, (
+        'false_positive_rate/false_positives/count:0',
+        'false_positive_rate/true_negatives/count:0'))
+
+  def testMetricsCollection(self):
+    my_collection_name = '__metrics__'
+    mean, _ = metrics.streaming_false_positive_rate(
+        predictions=array_ops.ones((10, 1)),
+        labels=array_ops.ones((10, 1)),
+        metrics_collections=[my_collection_name])
+    self.assertListEqual(ops.get_collection(my_collection_name), [mean])
+
+  def testUpdatesCollection(self):
+    my_collection_name = '__updates__'
+    _, update_op = metrics.streaming_false_positive_rate(
+        predictions=array_ops.ones((10, 1)),
+        labels=array_ops.ones((10, 1)),
+        updates_collections=[my_collection_name])
+    self.assertListEqual(ops.get_collection(my_collection_name), [update_op])
+
+  def testValueTensorIsIdempotent(self):
+    predictions = random_ops.random_uniform(
+        (10, 3), maxval=1, dtype=dtypes_lib.int64, seed=1)
+    labels = random_ops.random_uniform(
+        (10, 3), maxval=1, dtype=dtypes_lib.int64, seed=2)
+    fpr, update_op = metrics.streaming_false_positive_rate(
+        predictions, labels)
+
+    with self.test_session() as sess:
+      sess.run(variables.local_variables_initializer())
+
+      # Run several updates.
+      for _ in range(10):
+        sess.run(update_op)
+
+      # Then verify idempotency.
+      initial_fpr = fpr.eval()
+      for _ in range(10):
+        self.assertEqual(initial_fpr, fpr.eval())
+
+  def testAllCorrect(self):
+    np_inputs = np.random.randint(0, 2, size=(100, 1))
+
+    predictions = constant_op.constant(np_inputs)
+    labels = constant_op.constant(np_inputs)
+    fpr, update_op = metrics.streaming_false_positive_rate(
+        predictions, labels)
+
+    with self.test_session() as sess:
+      sess.run(variables.local_variables_initializer())
+      sess.run(update_op)
+      self.assertEqual(0, fpr.eval())
+
+  def testSomeCorrect(self):
+    predictions = constant_op.constant([1, 0, 1, 0], shape=(1, 4))
+    labels = constant_op.constant([0, 1, 1, 0], shape=(1, 4))
+    fpr, update_op = metrics.streaming_false_positive_rate(
+        predictions, labels)
+
+    with self.test_session() as sess:
+      sess.run(variables.local_variables_initializer())
+      self.assertAlmostEqual(0.5, update_op.eval())
+      self.assertAlmostEqual(0.5, fpr.eval())
+
+  def testWeighted1d(self):
+    predictions = constant_op.constant([[1, 0, 1, 0], [0, 1, 0, 1]])
+    labels = constant_op.constant([[0, 1, 1, 0], [1, 0, 0, 1]])
+    weights = constant_op.constant([[2], [5]])
+    fpr, update_op = metrics.streaming_false_positive_rate(
+        predictions, labels, weights=weights)
+
+    with self.test_session() as sess:
+      sess.run(variables.local_variables_initializer())
+      weighted_fp = 2.0 + 5.0
+      weighted_f = (2.0 + 2.0) + (5.0 + 5.0)
+      expected_fpr = weighted_fp / weighted_f
+      self.assertAlmostEqual(expected_fpr, update_op.eval())
+      self.assertAlmostEqual(expected_fpr, fpr.eval())
+
+  def testWeighted2d(self):
+    predictions = constant_op.constant([[1, 0, 1, 0], [0, 1, 0, 1]])
+    labels = constant_op.constant([[0, 1, 1, 0], [1, 0, 0, 1]])
+    weights = constant_op.constant([[1, 2, 3, 4], [4, 3, 2, 1]])
+    fpr, update_op = metrics.streaming_false_positive_rate(
+        predictions, labels, weights=weights)
+
+    with self.test_session() as sess:
+      sess.run(variables.local_variables_initializer())
+      weighted_fp = 1.0 + 3.0
+      weighted_f = (1.0 + 4.0) + (2.0 + 3.0)
+      expected_fpr = weighted_fp / weighted_f
+      self.assertAlmostEqual(expected_fpr, update_op.eval())
+      self.assertAlmostEqual(expected_fpr, fpr.eval())
+
+  def testAllIncorrect(self):
+    np_inputs = np.random.randint(0, 2, size=(100, 1))
+
+    predictions = constant_op.constant(np_inputs)
+    labels = constant_op.constant(1 - np_inputs)
+    fpr, update_op = metrics.streaming_false_positive_rate(
+        predictions, labels)
+
+    with self.test_session() as sess:
+      sess.run(variables.local_variables_initializer())
+      sess.run(update_op)
+      self.assertEqual(1, fpr.eval())
+
+  def testZeroFalsePositivesAndTrueNegativesGivesZeroFPR(self):
+    predictions = array_ops.ones((1, 4))
+    labels = array_ops.ones((1, 4))
+    fpr, update_op = metrics.streaming_false_positive_rate(
+        predictions, labels)
+
+    with self.test_session() as sess:
+      sess.run(variables.local_variables_initializer())
+      sess.run(update_op)
+      self.assertEqual(0, fpr.eval())
+
+
+class StreamingFNRTest(test.TestCase):
+
+  def setUp(self):
+    np.random.seed(1)
+    ops.reset_default_graph()
+
+  def testVars(self):
+    metrics.streaming_false_negative_rate(
+        predictions=array_ops.ones((10, 1)), labels=array_ops.ones((10, 1)))
+    _assert_local_variables(self, (
+        'false_negative_rate/false_negatives/count:0',
+        'false_negative_rate/true_positives/count:0'))
+
+  def testMetricsCollection(self):
+    my_collection_name = '__metrics__'
+    mean, _ = metrics.streaming_false_negative_rate(
+        predictions=array_ops.ones((10, 1)),
+        labels=array_ops.ones((10, 1)),
+        metrics_collections=[my_collection_name])
+    self.assertListEqual(ops.get_collection(my_collection_name), [mean])
+
+  def testUpdatesCollection(self):
+    my_collection_name = '__updates__'
+    _, update_op = metrics.streaming_false_negative_rate(
+        predictions=array_ops.ones((10, 1)),
+        labels=array_ops.ones((10, 1)),
+        updates_collections=[my_collection_name])
+    self.assertListEqual(ops.get_collection(my_collection_name), [update_op])
+
+  def testValueTensorIsIdempotent(self):
+    predictions = random_ops.random_uniform(
+        (10, 3), maxval=1, dtype=dtypes_lib.int64, seed=1)
+    labels = random_ops.random_uniform(
+        (10, 3), maxval=1, dtype=dtypes_lib.int64, seed=2)
+    fnr, update_op = metrics.streaming_false_negative_rate(
+        predictions, labels)
+
+    with self.test_session() as sess:
+      sess.run(variables.local_variables_initializer())
+
+      # Run several updates.
+      for _ in range(10):
+        sess.run(update_op)
+
+      # Then verify idempotency.
+      initial_fnr = fnr.eval()
+      for _ in range(10):
+        self.assertEqual(initial_fnr, fnr.eval())
+
+  def testAllCorrect(self):
+    np_inputs = np.random.randint(0, 2, size=(100, 1))
+
+    predictions = constant_op.constant(np_inputs)
+    labels = constant_op.constant(np_inputs)
+    fnr, update_op = metrics.streaming_false_negative_rate(
+        predictions, labels)
+
+    with self.test_session() as sess:
+      sess.run(variables.local_variables_initializer())
+      sess.run(update_op)
+      self.assertEqual(0, fnr.eval())
+
+  def testSomeCorrect(self):
+    predictions = constant_op.constant([1, 0, 1, 0], shape=(1, 4))
+    labels = constant_op.constant([0, 1, 1, 0], shape=(1, 4))
+    fnr, update_op = metrics.streaming_false_negative_rate(
+        predictions, labels)
+
+    with self.test_session() as sess:
+      sess.run(variables.local_variables_initializer())
+      self.assertAlmostEqual(0.5, update_op.eval())
+      self.assertAlmostEqual(0.5, fnr.eval())
+
+  def testWeighted1d(self):
+    predictions = constant_op.constant([[1, 0, 1, 0], [0, 1, 0, 1]])
+    labels = constant_op.constant([[0, 1, 1, 0], [1, 0, 0, 1]])
+    weights = constant_op.constant([[2], [5]])
+    fnr, update_op = metrics.streaming_false_negative_rate(
+        predictions, labels, weights=weights)
+
+    with self.test_session() as sess:
+      sess.run(variables.local_variables_initializer())
+      weighted_fn = 2.0 + 5.0
+      weighted_t = (2.0 + 2.0) + (5.0 + 5.0)
+      expected_fnr = weighted_fn / weighted_t
+      self.assertAlmostEqual(expected_fnr, update_op.eval())
+      self.assertAlmostEqual(expected_fnr, fnr.eval())
+
+  def testWeighted2d(self):
+    predictions = constant_op.constant([[1, 0, 1, 0], [0, 1, 0, 1]])
+    labels = constant_op.constant([[0, 1, 1, 0], [1, 0, 0, 1]])
+    weights = constant_op.constant([[1, 2, 3, 4], [4, 3, 2, 1]])
+    fnr, update_op = metrics.streaming_false_negative_rate(
+        predictions, labels, weights=weights)
+
+    with self.test_session() as sess:
+      sess.run(variables.local_variables_initializer())
+      weighted_fn = 2.0 + 4.0
+      weighted_t = (2.0 + 3.0) + (1.0 + 4.0)
+      expected_fnr = weighted_fn / weighted_t
+      self.assertAlmostEqual(expected_fnr, update_op.eval())
+      self.assertAlmostEqual(expected_fnr, fnr.eval())
+
+  def testAllIncorrect(self):
+    np_inputs = np.random.randint(0, 2, size=(100, 1))
+
+    predictions = constant_op.constant(np_inputs)
+    labels = constant_op.constant(1 - np_inputs)
+    fnr, update_op = metrics.streaming_false_negative_rate(
+        predictions, labels)
+
+    with self.test_session() as sess:
+      sess.run(variables.local_variables_initializer())
+      sess.run(update_op)
+      self.assertEqual(1, fnr.eval())
+
+  def testZeroFalseNegativesAndTruePositivesGivesZeroFNR(self):
+    predictions = array_ops.zeros((1, 4))
+    labels = array_ops.zeros((1, 4))
+    fnr, update_op = metrics.streaming_false_negative_rate(
+        predictions, labels)
+
+    with self.test_session() as sess:
+      sess.run(variables.local_variables_initializer())
+      sess.run(update_op)
+      self.assertEqual(0, fnr.eval())
+
+
 class StreamingCurvePointsTest(test.TestCase):
 
   def setUp(self):
@@ -2268,6 +2524,470 @@ class StreamingPrecisionRecallThresholdsTest(test.TestCase):
       self.assertAlmostEqual(expected_rec, rec.eval(), 2)
 
 
+class StreamingFPRThresholdsTest(test.TestCase):
+
+  def setUp(self):
+    np.random.seed(1)
+    ops.reset_default_graph()
+
+  def testVars(self):
+    metrics.streaming_false_positive_rate_at_thresholds(
+        predictions=array_ops.ones((10, 1)),
+        labels=array_ops.ones((10, 1)),
+        thresholds=[0, 0.5, 1.0])
+    _assert_local_variables(self, (
+        'false_positive_rate_at_thresholds/false_positives:0',
+        'false_positive_rate_at_thresholds/true_negatives:0',))
+
+  def testMetricsCollection(self):
+    my_collection_name = '__metrics__'
+    fpr, _ = metrics.streaming_false_positive_rate_at_thresholds(
+        predictions=array_ops.ones((10, 1)),
+        labels=array_ops.ones((10, 1)),
+        thresholds=[0, 0.5, 1.0],
+        metrics_collections=[my_collection_name])
+    self.assertListEqual(ops.get_collection(my_collection_name), [fpr])
+
+  def testUpdatesCollection(self):
+    my_collection_name = '__updates__'
+    _, update_op = metrics.streaming_false_positive_rate_at_thresholds(
+        predictions=array_ops.ones((10, 1)),
+        labels=array_ops.ones((10, 1)),
+        thresholds=[0, 0.5, 1.0],
+        updates_collections=[my_collection_name])
+    self.assertListEqual(
+        ops.get_collection(my_collection_name), [update_op])
+
+  def testValueTensorIsIdempotent(self):
+    predictions = random_ops.random_uniform(
+        (10, 3), maxval=1, dtype=dtypes_lib.float32, seed=1)
+    labels = random_ops.random_uniform(
+        (10, 3), maxval=1, dtype=dtypes_lib.int64, seed=2)
+    thresholds = [0, 0.5, 1.0]
+    fpr, fpr_op = metrics.streaming_false_positive_rate_at_thresholds(
+        predictions, labels, thresholds)
+
+    with self.test_session() as sess:
+      sess.run(variables.local_variables_initializer())
+
+      # Run several updates.
+      for _ in range(10):
+        sess.run(fpr_op)
+
+      # Then verify idempotency.
+      initial_fpr = fpr.eval()
+      for _ in range(10):
+        self.assertAllClose(initial_fpr, fpr.eval())
+
+  def testAllCorrect(self):
+    inputs = np.random.randint(0, 2, size=(100, 1))
+
+    with self.test_session() as sess:
+      predictions = constant_op.constant(inputs, dtype=dtypes_lib.float32)
+      labels = constant_op.constant(inputs)
+      thresholds = [0.5]
+      fpr, fpr_op = metrics.streaming_false_positive_rate_at_thresholds(
+          predictions, labels, thresholds)
+
+      sess.run(variables.local_variables_initializer())
+      sess.run(fpr_op)
+
+      self.assertEqual(0, fpr.eval())
+
+  def testSomeCorrect(self):
+    with self.test_session() as sess:
+      predictions = constant_op.constant(
+          [1, 0, 1, 0], shape=(1, 4), dtype=dtypes_lib.float32)
+      labels = constant_op.constant([0, 1, 1, 0], shape=(1, 4))
+      thresholds = [0.5]
+      fpr, fpr_op = metrics.streaming_false_positive_rate_at_thresholds(
+          predictions, labels, thresholds)
+
+      sess.run(variables.local_variables_initializer())
+      sess.run(fpr_op)
+
+      self.assertAlmostEqual(0.5, fpr.eval())
+
+  def testAllIncorrect(self):
+    inputs = np.random.randint(0, 2, size=(100, 1))
+
+    with self.test_session() as sess:
+      predictions = constant_op.constant(inputs, dtype=dtypes_lib.float32)
+      labels = constant_op.constant(1 - inputs, dtype=dtypes_lib.float32)
+      thresholds = [0.5]
+      fpr, fpr_op = metrics.streaming_false_positive_rate_at_thresholds(
+          predictions, labels, thresholds)
+
+      sess.run(variables.local_variables_initializer())
+      sess.run(fpr_op)
+
+      self.assertAlmostEqual(1, fpr.eval())
+
+  def testWeights1d(self):
+    with self.test_session() as sess:
+      predictions = constant_op.constant(
+          [[1, 0], [1, 0]], shape=(2, 2), dtype=dtypes_lib.float32)
+      labels = constant_op.constant([[0, 1], [1, 0]], shape=(2, 2))
+      weights = constant_op.constant(
+          [[0], [1]], shape=(2, 1), dtype=dtypes_lib.float32)
+      thresholds = [0.5, 1.1]
+      fpr, fpr_op = metrics.streaming_false_positive_rate_at_thresholds(
+          predictions, labels, thresholds, weights=weights)
+
+      fpr_low = fpr[0]
+      fpr_high = fpr[1]
+
+      sess.run(variables.local_variables_initializer())
+      sess.run(fpr_op)
+
+      self.assertAlmostEqual(0.0, fpr_low.eval(), places=5)
+      self.assertAlmostEqual(0.0, fpr_high.eval(), places=5)
+
+  def testWeights2d(self):
+    with self.test_session() as sess:
+      predictions = constant_op.constant(
+          [[1, 0], [1, 0]], shape=(2, 2), dtype=dtypes_lib.float32)
+      labels = constant_op.constant([[0, 1], [1, 0]], shape=(2, 2))
+      weights = constant_op.constant(
+          [[0, 0], [1, 1]], shape=(2, 2), dtype=dtypes_lib.float32)
+      thresholds = [0.5, 1.1]
+      fpr, fpr_op = metrics.streaming_false_positive_rate_at_thresholds(
+          predictions, labels, thresholds, weights=weights)
+
+      fpr_low = fpr[0]
+      fpr_high = fpr[1]
+
+      sess.run(variables.local_variables_initializer())
+      sess.run(fpr_op)
+
+      self.assertAlmostEqual(0.0, fpr_low.eval(), places=5)
+      self.assertAlmostEqual(0.0, fpr_high.eval(), places=5)
+
+  def testExtremeThresholds(self):
+    with self.test_session() as sess:
+      predictions = constant_op.constant(
+          [1, 0, 1, 0], shape=(1, 4), dtype=dtypes_lib.float32)
+      labels = constant_op.constant([0, 1, 1, 1], shape=(1, 4))
+      thresholds = [-1.0, 2.0]  # lower/higher than any values
+      fpr, fpr_op = metrics.streaming_false_positive_rate_at_thresholds(
+          predictions, labels, thresholds)
+
+      fpr_low = fpr[0]
+      fpr_high = fpr[1]
+
+      sess.run(variables.local_variables_initializer())
+      sess.run(fpr_op)
+
+      self.assertAlmostEqual(1.0, fpr_low.eval(), places=5)
+      self.assertAlmostEqual(0.0, fpr_high.eval(), places=5)
+
+  def testZeroLabelsPredictions(self):
+    with self.test_session() as sess:
+      predictions = array_ops.zeros([4], dtype=dtypes_lib.float32)
+      labels = array_ops.zeros([4])
+      thresholds = [0.5]
+      fpr, fpr_op = metrics.streaming_false_positive_rate_at_thresholds(
+          predictions, labels, thresholds)
+
+      sess.run(variables.local_variables_initializer())
+      sess.run(fpr_op)
+
+      self.assertAlmostEqual(0, fpr.eval(), 6)
+
+  def testWithMultipleUpdates(self):
+    num_samples = 1000
+    batch_size = 10
+    num_batches = int(num_samples / batch_size)
+
+    # Create the labels and data.
+    labels = np.random.randint(0, 2, size=(num_samples, 1))
+    noise = np.random.normal(0.0, scale=0.2, size=(num_samples, 1))
+    predictions = 0.4 + 0.2 * labels + noise
+    predictions[predictions > 1] = 1
+    predictions[predictions < 0] = 0
+    thresholds = [0.3]
+
+    fp = 0
+    tn = 0
+    for i in range(num_samples):
+      if predictions[i] > thresholds[0]:
+        if labels[i] == 0:
+          fp += 1
+      else:
+        if labels[i] == 0:
+          tn += 1
+    epsilon = 1e-7
+    expected_fpr = fp / (epsilon + fp + tn)
+
+    labels = labels.astype(np.float32)
+    predictions = predictions.astype(np.float32)
+
+    with self.test_session() as sess:
+      # Reshape the data so its easy to queue up:
+      predictions_batches = predictions.reshape((batch_size, num_batches))
+      labels_batches = labels.reshape((batch_size, num_batches))
+
+      # Enqueue the data:
+      predictions_queue = data_flow_ops.FIFOQueue(
+          num_batches, dtypes=dtypes_lib.float32, shapes=(batch_size,))
+      labels_queue = data_flow_ops.FIFOQueue(
+          num_batches, dtypes=dtypes_lib.float32, shapes=(batch_size,))
+
+      for i in range(int(num_batches)):
+        tf_prediction = constant_op.constant(predictions_batches[:, i])
+        tf_label = constant_op.constant(labels_batches[:, i])
+        sess.run([
+            predictions_queue.enqueue(tf_prediction),
+            labels_queue.enqueue(tf_label)
+        ])
+
+      tf_predictions = predictions_queue.dequeue()
+      tf_labels = labels_queue.dequeue()
+
+      fpr, fpr_op = metrics.streaming_false_positive_rate_at_thresholds(
+          tf_predictions, tf_labels, thresholds)
+
+      sess.run(variables.local_variables_initializer())
+      for _ in range(int(num_samples / batch_size)):
+        sess.run(fpr_op)
+      # Since this is only approximate, we can't expect a 6 digits match.
+      # Although with higher number of samples/thresholds we should see the
+      # accuracy improving
+      self.assertAlmostEqual(expected_fpr, fpr.eval(), 2)
+
+
+class StreamingFNRThresholdsTest(test.TestCase):
+
+  def setUp(self):
+    np.random.seed(1)
+    ops.reset_default_graph()
+
+  def testVars(self):
+    metrics.streaming_false_negative_rate_at_thresholds(
+        predictions=array_ops.ones((10, 1)),
+        labels=array_ops.ones((10, 1)),
+        thresholds=[0, 0.5, 1.0])
+    _assert_local_variables(self, (
+        'false_negative_rate_at_thresholds/false_negatives:0',
+        'false_negative_rate_at_thresholds/true_positives:0',))
+
+  def testMetricsCollection(self):
+    my_collection_name = '__metrics__'
+    fnr, _ = metrics.streaming_false_negative_rate_at_thresholds(
+        predictions=array_ops.ones((10, 1)),
+        labels=array_ops.ones((10, 1)),
+        thresholds=[0, 0.5, 1.0],
+        metrics_collections=[my_collection_name])
+    self.assertListEqual(ops.get_collection(my_collection_name), [fnr])
+
+  def testUpdatesCollection(self):
+    my_collection_name = '__updates__'
+    _, update_op = metrics.streaming_false_negative_rate_at_thresholds(
+        predictions=array_ops.ones((10, 1)),
+        labels=array_ops.ones((10, 1)),
+        thresholds=[0, 0.5, 1.0],
+        updates_collections=[my_collection_name])
+    self.assertListEqual(
+        ops.get_collection(my_collection_name), [update_op])
+
+  def testValueTensorIsIdempotent(self):
+    predictions = random_ops.random_uniform(
+        (10, 3), maxval=1, dtype=dtypes_lib.float32, seed=1)
+    labels = random_ops.random_uniform(
+        (10, 3), maxval=1, dtype=dtypes_lib.int64, seed=2)
+    thresholds = [0, 0.5, 1.0]
+    fnr, fnr_op = metrics.streaming_false_negative_rate_at_thresholds(
+        predictions, labels, thresholds)
+
+    with self.test_session() as sess:
+      sess.run(variables.local_variables_initializer())
+
+      # Run several updates.
+      for _ in range(10):
+        sess.run(fnr_op)
+
+      # Then verify idempotency.
+      initial_fnr = fnr.eval()
+      for _ in range(10):
+        self.assertAllClose(initial_fnr, fnr.eval())
+
+  def testAllCorrect(self):
+    inputs = np.random.randint(0, 2, size=(100, 1))
+
+    with self.test_session() as sess:
+      predictions = constant_op.constant(inputs, dtype=dtypes_lib.float32)
+      labels = constant_op.constant(inputs)
+      thresholds = [0.5]
+      fnr, fnr_op = metrics.streaming_false_negative_rate_at_thresholds(
+          predictions, labels, thresholds)
+
+      sess.run(variables.local_variables_initializer())
+      sess.run(fnr_op)
+
+      self.assertEqual(0, fnr.eval())
+
+  def testSomeCorrect(self):
+    with self.test_session() as sess:
+      predictions = constant_op.constant(
+          [1, 0, 1, 0], shape=(1, 4), dtype=dtypes_lib.float32)
+      labels = constant_op.constant([0, 1, 1, 0], shape=(1, 4))
+      thresholds = [0.5]
+      fnr, fnr_op = metrics.streaming_false_negative_rate_at_thresholds(
+          predictions, labels, thresholds)
+
+      sess.run(variables.local_variables_initializer())
+      sess.run(fnr_op)
+
+      self.assertAlmostEqual(0.5, fnr.eval())
+
+  def testAllIncorrect(self):
+    inputs = np.random.randint(0, 2, size=(100, 1))
+
+    with self.test_session() as sess:
+      predictions = constant_op.constant(inputs, dtype=dtypes_lib.float32)
+      labels = constant_op.constant(1 - inputs, dtype=dtypes_lib.float32)
+      thresholds = [0.5]
+      fnr, fnr_op = metrics.streaming_false_negative_rate_at_thresholds(
+          predictions, labels, thresholds)
+
+      sess.run(variables.local_variables_initializer())
+      sess.run(fnr_op)
+
+      self.assertAlmostEqual(1, fnr.eval())
+
+  def testWeights1d(self):
+    with self.test_session() as sess:
+      predictions = constant_op.constant(
+          [[1, 0], [1, 0]], shape=(2, 2), dtype=dtypes_lib.float32)
+      labels = constant_op.constant([[0, 1], [1, 0]], shape=(2, 2))
+      weights = constant_op.constant(
+          [[0], [1]], shape=(2, 1), dtype=dtypes_lib.float32)
+      thresholds = [0.5, 1.1]
+      fnr, fnr_op = metrics.streaming_false_negative_rate_at_thresholds(
+          predictions, labels, thresholds, weights=weights)
+
+      fnr_low = fnr[0]
+      fnr_high = fnr[1]
+
+      sess.run(variables.local_variables_initializer())
+      sess.run(fnr_op)
+
+      self.assertAlmostEqual(0.0, fnr_low.eval(), places=5)
+      self.assertAlmostEqual(1.0, fnr_high.eval(), places=5)
+
+  def testWeights2d(self):
+    with self.test_session() as sess:
+      predictions = constant_op.constant(
+          [[1, 0], [1, 0]], shape=(2, 2), dtype=dtypes_lib.float32)
+      labels = constant_op.constant([[0, 1], [1, 0]], shape=(2, 2))
+      weights = constant_op.constant(
+          [[0, 0], [1, 1]], shape=(2, 2), dtype=dtypes_lib.float32)
+      thresholds = [0.5, 1.1]
+      fnr, fnr_op = metrics.streaming_false_negative_rate_at_thresholds(
+          predictions, labels, thresholds, weights=weights)
+
+      fnr_low = fnr[0]
+      fnr_high = fnr[1]
+
+      sess.run(variables.local_variables_initializer())
+      sess.run(fnr_op)
+
+      self.assertAlmostEqual(0.0, fnr_low.eval(), places=5)
+      self.assertAlmostEqual(1.0, fnr_high.eval(), places=5)
+
+  def testExtremeThresholds(self):
+    with self.test_session() as sess:
+      predictions = constant_op.constant(
+          [1, 0, 1, 0], shape=(1, 4), dtype=dtypes_lib.float32)
+      labels = constant_op.constant([0, 1, 1, 1], shape=(1, 4))
+      thresholds = [-1.0, 2.0]  # lower/higher than any values
+      fnr, fnr_op = metrics.streaming_false_negative_rate_at_thresholds(
+          predictions, labels, thresholds)
+
+      fnr_low = fnr[0]
+      fnr_high = fnr[1]
+
+      sess.run(variables.local_variables_initializer())
+      sess.run(fnr_op)
+
+      self.assertAlmostEqual(0.0, fnr_low.eval())
+      self.assertAlmostEqual(1.0, fnr_high.eval())
+
+  def testZeroLabelsPredictions(self):
+    with self.test_session() as sess:
+      predictions = array_ops.zeros([4], dtype=dtypes_lib.float32)
+      labels = array_ops.zeros([4])
+      thresholds = [0.5]
+      fnr, fnr_op = metrics.streaming_false_negative_rate_at_thresholds(
+          predictions, labels, thresholds)
+
+      sess.run(variables.local_variables_initializer())
+      sess.run(fnr_op)
+
+      self.assertAlmostEqual(0, fnr.eval(), 6)
+
+  def testWithMultipleUpdates(self):
+    num_samples = 1000
+    batch_size = 10
+    num_batches = int(num_samples / batch_size)
+
+    # Create the labels and data.
+    labels = np.random.randint(0, 2, size=(num_samples, 1))
+    noise = np.random.normal(0.0, scale=0.2, size=(num_samples, 1))
+    predictions = 0.4 + 0.2 * labels + noise
+    predictions[predictions > 1] = 1
+    predictions[predictions < 0] = 0
+    thresholds = [0.3]
+
+    fn = 0
+    tp = 0
+    for i in range(num_samples):
+      if predictions[i] > thresholds[0]:
+        if labels[i] == 1:
+          tp += 1
+      else:
+        if labels[i] == 1:
+          fn += 1
+    epsilon = 1e-7
+    expected_fnr = fn / (epsilon + fn + tp)
+
+    labels = labels.astype(np.float32)
+    predictions = predictions.astype(np.float32)
+
+    with self.test_session() as sess:
+      # Reshape the data so its easy to queue up:
+      predictions_batches = predictions.reshape((batch_size, num_batches))
+      labels_batches = labels.reshape((batch_size, num_batches))
+
+      # Enqueue the data:
+      predictions_queue = data_flow_ops.FIFOQueue(
+          num_batches, dtypes=dtypes_lib.float32, shapes=(batch_size,))
+      labels_queue = data_flow_ops.FIFOQueue(
+          num_batches, dtypes=dtypes_lib.float32, shapes=(batch_size,))
+
+      for i in range(int(num_batches)):
+        tf_prediction = constant_op.constant(predictions_batches[:, i])
+        tf_label = constant_op.constant(labels_batches[:, i])
+        sess.run([
+            predictions_queue.enqueue(tf_prediction),
+            labels_queue.enqueue(tf_label)
+        ])
+
+      tf_predictions = predictions_queue.dequeue()
+      tf_labels = labels_queue.dequeue()
+
+      fnr, fnr_op = metrics.streaming_false_negative_rate_at_thresholds(
+          tf_predictions, tf_labels, thresholds)
+
+      sess.run(variables.local_variables_initializer())
+      for _ in range(int(num_samples / batch_size)):
+        sess.run(fnr_op)
+      # Since this is only approximate, we can't expect a 6 digits match.
+      # Although with higher number of samples/thresholds we should see the
+      # accuracy improving
+      self.assertAlmostEqual(expected_fnr, fnr.eval(), 2)
+
+
 # TODO(ptucker): Remove when we remove `streaming_recall_at_k`.
 # This op will be deprecated soon in favor of `streaming_sparse_recall_at_k`.
 # Until then, this test validates that both ops yield the same results.
-- 
GitLab


From 6720027d27c96e1d8b9792d060dc2a9aeda60cea Mon Sep 17 00:00:00 2001
From: Chris Tava <chris1tava@gmail.com>
Date: Sat, 14 Oct 2017 10:34:32 -0400
Subject: [PATCH 395/909] Updating install_golang.sh - bumping to 1.9.1

---
 tensorflow/tools/ci_build/install/install_golang.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/tools/ci_build/install/install_golang.sh b/tensorflow/tools/ci_build/install/install_golang.sh
index 596265b069..55c1674495 100755
--- a/tensorflow/tools/ci_build/install/install_golang.sh
+++ b/tensorflow/tools/ci_build/install/install_golang.sh
@@ -16,7 +16,7 @@
 
 set -ex
 
-GOLANG_URL="https://storage.googleapis.com/golang/go1.9.linux-amd64.tar.gz"
+GOLANG_URL="https://storage.googleapis.com/golang/go1.9.1.linux-amd64.tar.gz"
 
 sudo mkdir -p /usr/local
 wget -q -O - "${GOLANG_URL}" | sudo tar -C /usr/local -xz
-- 
GitLab


From d76ab0431f30ed7ca283b2e343fefb7b4cb9e9f5 Mon Sep 17 00:00:00 2001
From: Bill Prin <waprin@gmail.com>
Date: Sat, 14 Oct 2017 11:19:52 -0700
Subject: [PATCH 396/909] Fix typos in datasets guide (#13701)

* Fix typos in datasets guide

* mrry review
---
 tensorflow/docs_src/programmers_guide/datasets.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/docs_src/programmers_guide/datasets.md b/tensorflow/docs_src/programmers_guide/datasets.md
index fd1c927539..bec1bb1bf0 100644
--- a/tensorflow/docs_src/programmers_guide/datasets.md
+++ b/tensorflow/docs_src/programmers_guide/datasets.md
@@ -44,7 +44,7 @@ To start an input pipeline, you must define a *source*. For example, to
 construct a `Dataset` from some tensors in memory, you can use
 `tf.data.Dataset.from_tensors()` or
 `tf.data.Dataset.from_tensor_slices()`. Alternatively, if your input
-data are on disk in the recommend TFRecord format, you can construct a
+data are on disk in the recommended TFRecord format, you can construct a
 `tf.data.TFRecordDataset`.
 
 Once you have a `Dataset` object, you can *transform* it into a new `Dataset` by
-- 
GitLab


From 81bc3b4e746a35906c8b7ab01f27a4c9ada4fd66 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Sat, 14 Oct 2017 19:14:12 +0000
Subject: [PATCH 397/909] Fix cmake build with
 `Dtensorflow_BUILD_ALL_KERNELS=OFF` error

This fix tries to address the issue raised in #11975 where cmake
with `Dtensorflow_BUILD_ALL_KERNELS=OFF` will throw out an error:
```
[ 93%] Building CXX object CMakeFiles/tf_tools_transform_graph_lib.dir/workspace/tensorflow/tools/graph_transforms/freeze_requantization_ranges.cc.o
CMakeFiles/tf_core_cpu.dir/workspace/tensorflow/core/grappler/costs/measuring_cost_estimator.cc.o: In function `tensorflow::grappler::MeasuringCostEstimator::MeasuringCostEstimator(tensorflow::grappler::Cluster*, int, int)':
measuring_cost_estimator.cc:(.text+0x18c): undefined reference to `tensorflow::SanitizeThreadSuffix(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >)'
collect2: error: ld returned 1 exit status
make[2]: *** [benchmark_model] Error 1
CMakeFiles/benchmark_model.dir/build.make:949: recipe for target 'benchmark_model' failed
make[1]: *** [CMakeFiles/benchmark_model.dir/all] Error 2
CMakeFiles/Makefile2:6884: recipe for target 'CMakeFiles/benchmark_model.dir/all' failed
make[1]: *** Waiting for unfinished jobs....
```

The issue is casued by `ops_util.cc` which is needed even if all kernels are OFF.

This fix fixes the issue with cmake file update.

This fix fixes 11975.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/contrib/cmake/tf_core_kernels.cmake | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tensorflow/contrib/cmake/tf_core_kernels.cmake b/tensorflow/contrib/cmake/tf_core_kernels.cmake
index 3a2fe35a3e..65565aad7e 100644
--- a/tensorflow/contrib/cmake/tf_core_kernels.cmake
+++ b/tensorflow/contrib/cmake/tf_core_kernels.cmake
@@ -33,6 +33,8 @@ else(tensorflow_BUILD_ALL_KERNELS)
      "${tensorflow_source_dir}/tensorflow/core/kernels/matmul_op.cc"
      "${tensorflow_source_dir}/tensorflow/core/kernels/no_op.h"
      "${tensorflow_source_dir}/tensorflow/core/kernels/no_op.cc"
+     "${tensorflow_source_dir}/tensorflow/core/kernels/ops_util.h"
+     "${tensorflow_source_dir}/tensorflow/core/kernels/ops_util.cc"
      "${tensorflow_source_dir}/tensorflow/core/kernels/sendrecv_ops.h"
      "${tensorflow_source_dir}/tensorflow/core/kernels/sendrecv_ops.cc"
   )
-- 
GitLab


From d3cd82071ff3cd7afef7b726c3e01f6953618013 Mon Sep 17 00:00:00 2001
From: Chris Leary <leary@google.com>
Date: Sat, 14 Oct 2017 19:34:46 -0700
Subject: [PATCH 398/909] [XLA] Avoid unnecessary spaces in identifiers.

PiperOrigin-RevId: 172224302
---
 tensorflow/compiler/xla/service/algebraic_simplifier.cc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier.cc b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
index 6592caa2a6..39e8430ed3 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
@@ -98,11 +98,11 @@ bool ReshapeIsBitcast(
 HloComputation* CreateScalarBinaryComputation(HloModule* module,
                                               PrimitiveType primitive_type,
                                               HloOpcode opcode) {
-  HloComputation::Builder b("scalar computation");
+  HloComputation::Builder b("scalar_computation");
   auto scalar_lhs = b.AddInstruction(HloInstruction::CreateParameter(
-      0, ShapeUtil::MakeShape(F32, {}), "scalar lhs"));
+      0, ShapeUtil::MakeShape(F32, {}), "scalar_lhs"));
   auto scalar_rhs = b.AddInstruction(HloInstruction::CreateParameter(
-      1, ShapeUtil::MakeShape(F32, {}), "scalar rhs"));
+      1, ShapeUtil::MakeShape(F32, {}), "scalar_rhs"));
   auto scalar_op = b.AddInstruction(
       HloInstruction::CreateBinary(ShapeUtil::MakeShape(primitive_type, {}),
                                    opcode, scalar_lhs, scalar_rhs));
-- 
GitLab


From e49839df71904f6f7f22245f9eaaeb6487c9d196 Mon Sep 17 00:00:00 2001
From: CQY <qychen@pku.edu.cn>
Date: Sun, 15 Oct 2017 11:04:45 +0800
Subject: [PATCH 399/909] Support reversing bool sequence.

---
 tensorflow/core/kernels/reverse_sequence_op.cc        | 3 +++
 tensorflow/core/kernels/reverse_sequence_op_gpu.cu.cc | 1 +
 2 files changed, 4 insertions(+)

diff --git a/tensorflow/core/kernels/reverse_sequence_op.cc b/tensorflow/core/kernels/reverse_sequence_op.cc
index 505c512cc4..d1980d4b65 100644
--- a/tensorflow/core/kernels/reverse_sequence_op.cc
+++ b/tensorflow/core/kernels/reverse_sequence_op.cc
@@ -175,6 +175,7 @@ class ReverseSequenceOp : public OpKernel {
   REGISTER_REVERSE_SEQUENCE(type, int64);
 
 TF_CALL_NUMBER_TYPES(REGISTER_REVERSE_SEQUENCE_LEN);
+TF_CALL_bool(REGISTER_REVERSE_SEQUENCE_LEN);
 
 #if GOOGLE_CUDA
 
@@ -200,6 +201,7 @@ namespace functor {
   DECLARE_GPU_SPEC_LEN(T, 5);
 
 TF_CALL_GPU_NUMBER_TYPES(DECLARE_GPU_SPECS);
+TF_CALL_bool(DECLARE_GPU_SPECS);
 
 }  // namespace functor
 
@@ -215,6 +217,7 @@ TF_CALL_GPU_NUMBER_TYPES(DECLARE_GPU_SPECS);
   REGISTER_REVERSE_SEQUENCE_GPU(type, int64);
 
 TF_CALL_GPU_NUMBER_TYPES(REGISTER_REVERSE_SEQUENCE_GPU_LEN);
+TF_CALL_bool(REGISTER_REVERSE_SEQUENCE_GPU_LEN);
 
 #undef REGISTER_REVERSE_SEQUENCE_GPU
 
diff --git a/tensorflow/core/kernels/reverse_sequence_op_gpu.cu.cc b/tensorflow/core/kernels/reverse_sequence_op_gpu.cu.cc
index 373fd60687..cb49f14525 100644
--- a/tensorflow/core/kernels/reverse_sequence_op_gpu.cu.cc
+++ b/tensorflow/core/kernels/reverse_sequence_op_gpu.cu.cc
@@ -39,6 +39,7 @@ typedef Eigen::GpuDevice GPUDevice;
   DEFINE_GPU_SPEC_LEN(T, 5);
 
 TF_CALL_GPU_NUMBER_TYPES(DEFINE_GPU_SPECS);
+TF_CALL_bool(DEFINE_GPU_SPECS);
 
 }  // end namespace tensorflow
 
-- 
GitLab


From ac6442ee5227693a8a0f23cb52b821e54e777b54 Mon Sep 17 00:00:00 2001
From: gunan <gunan@google.com>
Date: Sat, 14 Oct 2017 21:13:44 -0700
Subject: [PATCH 400/909] Tidy up tf_tests.cmake and reenable tests that are
 not failing anymore. (#13709)

---
 tensorflow/contrib/cmake/tf_tests.cmake | 80 +++++++------------------
 1 file changed, 20 insertions(+), 60 deletions(-)

diff --git a/tensorflow/contrib/cmake/tf_tests.cmake b/tensorflow/contrib/cmake/tf_tests.cmake
index 9108586e17..3d2e299ebb 100644
--- a/tensorflow/contrib/cmake/tf_tests.cmake
+++ b/tensorflow/contrib/cmake/tf_tests.cmake
@@ -179,6 +179,9 @@ if (tensorflow_BUILD_PYTHON_TESTS)
 
   # exclude the ones we don't want
   set(tf_test_src_py_exclude
+    # generally excluded
+    "${tensorflow_source_dir}/tensorflow/python/kernel_tests/__init__.py"
+
     # Python source line inspection tests are flaky on Windows (b/36375074).
     "${tensorflow_source_dir}/tensorflow/python/debug/cli/analyzer_cli_test.py"
     "${tensorflow_source_dir}/tensorflow/python/debug/cli/profile_analyzer_cli_test.py"
@@ -188,16 +191,10 @@ if (tensorflow_BUILD_PYTHON_TESTS)
     "${tensorflow_source_dir}/tensorflow/python/debug/lib/dist_session_debug_grpc_test.py"
     "${tensorflow_source_dir}/tensorflow/python/debug/lib/session_debug_grpc_test.py"
     # generally not working
-    "${tensorflow_source_dir}/tensorflow/python/kernel_tests/__init__.py"
-    "${tensorflow_source_dir}/tensorflow/python/kernel_tests/benchmark_test.py"
-    "${tensorflow_source_dir}/tensorflow/python/kernel_tests/resource_variable_ops_test.py"
     "${tensorflow_source_dir}/tensorflow/python/profiler/pprof_profiler_test.py"
     # flaky test
     "${tensorflow_source_dir}/tensorflow/python/profiler/internal/run_metadata_test.py"
     "${tensorflow_source_dir}/tensorflow/python/saved_model/saved_model_test.py"
-    # requires scipy
-    "${tensorflow_source_dir}/tensorflow/contrib/keras/python/keras/preprocessing/*_test.py"
-    "${tensorflow_source_dir}/tensorflow/contrib/tfprof/python/tools/tfprof/pprof_profiler_test.py"
     # flaky tests
     "${tensorflow_source_dir}/tensorflow/python/kernel_tests/cwise_ops_test.py"  # takes very long to run
     "${tensorflow_source_dir}/tensorflow/contrib/tfprof/python/tools/tfprof/internal/run_metadata_test.py"
@@ -213,58 +210,47 @@ if (tensorflow_BUILD_PYTHON_TESTS)
   if (WIN32)
     set(tf_test_src_py_exclude
       ${tf_test_src_py_exclude}
-      # generally excluded
-      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/__init__.py"
-
       # TODO: failing tests.
       # Nothing critical in here but should get this list down to []
       # The failing list is grouped by failure source
+
       # stl on windows handles overflows different
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/as_string_op_test.py"
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/cast_op_test.py"
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/string_to_number_op_test.py"
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/clip_ops_test.py"
-      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/tensor_array_ops_test.py"  # Needs portpicker.
+      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/concat_op_test.py"  # numerical issues
+      # Float division by zero
+      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/benchmark_test.py"
       # Type error in testRemoteIteratorUsingRemoteCallOpDirectSessionGPUCPU.
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/iterator_ops_test.py"
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/self_adjoint_eig_op_test.py"
-      # misc
+      "${tensorflow_source_dir}/tensorflow/contrib/data/python/kernel_tests/iterator_ops_test.py"
+      # IteratorGetMax OutOfRangeError
       "${tensorflow_source_dir}/tensorflow/contrib/data/python/kernel_tests/batch_dataset_op_test.py"
-      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/variable_scope_test.py"
-      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/reshape_op_test.py"
-      "${tensorflow_source_dir}/tensorflow/python/training/evaluation_test.py"
-      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/neon_depthwise_conv_op_test.py"  # Depends on gemmlowp -> pthread.
+      # Depends on gemmlowp -> pthread
+      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/neon_depthwise_conv_op_test.py"
       # int32/int64 mixup
+      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/variable_scope_test.py"
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/functional_ops_test.py"
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/py_func_test.py"
+      # Windows file management related issues.
+      "${tensorflow_source_dir}/tensorflow/python/training/evaluation_test.py"
       # training tests
       "${tensorflow_source_dir}/tensorflow/python/training/basic_session_run_hooks_test.py"  # Needs tf.contrib fix.
-      "${tensorflow_source_dir}/tensorflow/python/training/localhost_cluster_performance_test.py"  # Needs portpicker.
       "${tensorflow_source_dir}/tensorflow/python/training/quantize_training_test.py"  # Needs quantization ops to be included in windows.
       "${tensorflow_source_dir}/tensorflow/python/training/supervisor_test.py"  # Flaky I/O error on rename.
-      "${tensorflow_source_dir}/tensorflow/python/training/sync_replicas_optimizer_test.py"  # Needs portpicker.
       "${tensorflow_source_dir}/tensorflow/python/training/server_lib_test.py"  # Test occasionally deadlocks.
-      "${tensorflow_source_dir}/tensorflow/python/debug/lib/session_debug_multi_gpu_test.py"
-      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/concat_op_test.py"  # numerical issues
-
-      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/array_ops_test.py"  # depends on python/framework/test_ops     
+      "${tensorflow_source_dir}/tensorflow/python/debug/lib/session_debug_multi_gpu_test.py"  # Fails on multiple GPUs.
       # Dataset tests
-      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/dataset_constructor_op_test.py"
-      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/iterator_ops_cluster_test.py"
-      # Broken tensorboard test due to cmake issues.
-      "${tensorflow_source_dir}/tensorflow/contrib/data/python/kernel_tests/dataset_constructor_op_test.py"
-      "${tensorflow_source_dir}/tensorflow/contrib/data/python/kernel_tests/iterator_ops_cluster_test.py"  # Needs portpicker
-      "${tensorflow_source_dir}/tensorflow/contrib/data/python/kernel_tests/sloppy_transformation_dataset_op_test.py"  # b/65430561
-      # Type error in testRemoteIteratorUsingRemoteCallOpDirectSessionGPUCPU.
-      "${tensorflow_source_dir}/tensorflow/contrib/data/python/kernel_tests/iterator_ops_test.py"
+      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/dataset_constructor_op_test.py"  # Segfaults on windows
+      "${tensorflow_source_dir}/tensorflow/contrib/data/python/kernel_tests/dataset_constructor_op_test.py"  # Segfaults on Windows.
       # tensor_forest tests (also note that we exclude the hybrid tests for now)
       "${tensorflow_source_dir}/tensorflow/contrib/tensor_forest/python/kernel_tests/count_extremely_random_stats_op_test.py"  # Results in wrong order.
       "${tensorflow_source_dir}/tensorflow/contrib/tensor_forest/python/kernel_tests/sample_inputs_op_test.py"  # Results in wrong order.
       "${tensorflow_source_dir}/tensorflow/contrib/tensor_forest/python/kernel_tests/scatter_add_ndim_op_test.py"  # Bad placement.
       "${tensorflow_source_dir}/tensorflow/contrib/tensor_forest/python/topn_test.py"  # Results inaccurate
       "${tensorflow_source_dir}/tensorflow/python/ops/cloud/bigquery_reader_ops_test.py"  # No libcurl support
-      # Newly running on Windows since TensorBoard backend move. Fail on Windows and need debug.
-      "${tensorflow_source_dir}/tensorflow/contrib/data/python/kernel_tests/dataset_constructor_op_test.py"  # Segfaults on Windows.
       # Dask.Dataframe bugs on Window Build
       "${tensorflow_source_dir}/tensorflow/contrib/learn/python/learn/tests/dataframe/tensorflow_dataframe_test.py"
       "${tensorflow_source_dir}/tensorflow/contrib/learn/python/learn/learn_io/data_feeder_test.py"
@@ -273,37 +259,11 @@ if (tensorflow_BUILD_PYTHON_TESTS)
       # Need extra build
       "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/conditional_distribution_test.py"
       "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/conditional_transformed_distribution_test.py"
+      "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/estimator_test.py"
+      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/array_ops_test.py"  # depends on python/framework/test_ops     
       # Windows Path
       "${tensorflow_source_dir}/tensorflow/contrib/framework/python/ops/checkpoint_ops_test.py" #TODO: Fix path
-      "${tensorflow_source_dir}/tensorflow/contrib/keras/python/keras/models_test.py"
-      # Related to Windows Multiprocessing https://github.com/fchollet/keras/issues/5071
-      "${tensorflow_source_dir}/tensorflow/contrib/keras/python/keras/engine/training_test.py"
-      "${tensorflow_source_dir}/tensorflow/contrib/keras/python/keras/utils/data_utils_test.py"
-      "${tensorflow_source_dir}/tensorflow/contrib/keras/python/keras/callbacks_test.py"
-      # Scipy needed
-      "${tensorflow_source_dir}/tensorflow/contrib/keras/python/keras/preprocessing/image_test.py"
-      "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/bijectors/sigmoid_test.py"
-      "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/binomial_test.py"
-      "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/chi2_test.py"
-      "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/geometric_test.py"
-      "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/inverse_gamma_test.py"
-      "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/logistic_test.py"
-      "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/mixture_test.py"
-      "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/mvn_diag_test.py"
-      "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/mvn_full_covariance_test.py"
-      "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/mvn_tril_test.py"
-      "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/negative_binomial_test.py"
-      "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/poisson_test.py"
-      "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/quantized_distribution_test.py"
-      "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/relaxed_bernoulli_test.py"
-      "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/relaxed_onehot_categorical_test.py"
-      "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/transformed_distribution_test.py"
-      "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/vector_student_t_test.py"
-      "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/wishart_test.py"
-      "${tensorflow_source_dir}/tensorflow/contrib/factorization/python/ops/kmeans_test.py"
-      "${tensorflow_source_dir}/tensorflow/contrib/learn/python/learn/estimators/kmeans_test.py"
-      # Failing with TF 1.3 (TODO)
-      "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/estimator_test.py"
+      # Numpy upgrade needed?
       "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/bijectors/sinh_arcsinh_test.py"
       # Test should only be run manually
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/reduction_ops_test_big.py"
-- 
GitLab


From 4f3ce3d1bc424af25036b178782d83cea14ca084 Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Sat, 14 Oct 2017 21:23:47 -0700
Subject: [PATCH 401/909] Disable the newly failing windows GPU tests.

---
 tensorflow/contrib/cmake/tf_tests.cmake | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/cmake/tf_tests.cmake b/tensorflow/contrib/cmake/tf_tests.cmake
index 3d2e299ebb..9dc0262844 100644
--- a/tensorflow/contrib/cmake/tf_tests.cmake
+++ b/tensorflow/contrib/cmake/tf_tests.cmake
@@ -219,7 +219,9 @@ if (tensorflow_BUILD_PYTHON_TESTS)
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/cast_op_test.py"
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/string_to_number_op_test.py"
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/clip_ops_test.py"
-      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/concat_op_test.py"  # numerical issues
+      # Numerical issues, calculations off.
+      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/concat_op_test.py"
+      "${tensorflow_source_dir}/tensorflow/contrib/factorization/python/ops/wals_test.py"  
       # Float division by zero
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/benchmark_test.py"
       # Type error in testRemoteIteratorUsingRemoteCallOpDirectSessionGPUCPU.
@@ -260,7 +262,9 @@ if (tensorflow_BUILD_PYTHON_TESTS)
       "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/conditional_distribution_test.py"
       "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/conditional_transformed_distribution_test.py"
       "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/estimator_test.py"
-      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/array_ops_test.py"  # depends on python/framework/test_ops     
+      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/array_ops_test.py"  # depends on python/framework/test_ops
+      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/depthtospace_op_test.py"  # QuantizeV2
+      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/spacetodepth_op_test.py"  # QuantizeV2
       # Windows Path
       "${tensorflow_source_dir}/tensorflow/contrib/framework/python/ops/checkpoint_ops_test.py" #TODO: Fix path
       # Numpy upgrade needed?
-- 
GitLab


From 3c29973a6b973e4a857251ba1c42b5ff26609057 Mon Sep 17 00:00:00 2001
From: "Yuan (Terry) Tang" <terrytangyuan@users.noreply.github.com>
Date: Sun, 15 Oct 2017 02:09:36 -0400
Subject: [PATCH 402/909] Fixed incorrect `hooks` doc in `EvalSpec` (#13724)

---
 tensorflow/python/estimator/training.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/estimator/training.py b/tensorflow/python/estimator/training.py
index 45bff233ea..1131995b3e 100644
--- a/tensorflow/python/estimator/training.py
+++ b/tensorflow/python/estimator/training.py
@@ -199,7 +199,7 @@ class EvalSpec(
         evaluations on different data sets. Metrics for different evaluations
         are saved in separate folders, and appear separately in tensorboard.
       hooks: Iterable of `tf.train.SessionRunHook` objects to run
-        on all workers (including chief) during training.
+        during evaluation.
       exporters: Iterable of `Exporter`s, or a single one, or `None`.
         `exporters` will be invoked after each evaluation.
       start_delay_secs: Int. Start evaluating after waiting for this many
-- 
GitLab


From 2c0a0d7a8b1f4c2911e17f9629e9e47682bfffe5 Mon Sep 17 00:00:00 2001
From: Tian Jin <tjingrant@gmail.com>
Date: Sun, 15 Oct 2017 02:10:19 -0400
Subject: [PATCH 403/909] Boring ssl update. (#13638)

* update boringssl

* disabling the patch for now

* enable the patch file
---
 tensorflow/workspace.bzl                      |   7 +-
 .../boringssl/add_boringssl_s390x.patch       | 122 +-----------------
 2 files changed, 4 insertions(+), 125 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 6151dc6241..47e99903d1 100644
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -593,11 +593,10 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   patched_http_archive(
       name = "boringssl",
       urls = [
-          "http://mirror.bazel.build/github.com/google/boringssl/archive/e3860009a091cd1bd2bc189cdbc3c6d095abde84.tar.gz",
-          # "https://github.com/google/boringssl/archive/e3860009a091cd1bd2bc189cdbc3c6d095abde84.tar.gz",  # 2017-07-07
+          "https://github.com/google/boringssl/archive/72cfd9f49ec5fbc2db368b76398c196dafe6a4bc.tar.gz",
       ],
-      sha256 = "02f5950f93c4fd3691771c07c9d04cf2999ab01383ff99da345249e93b0fcfb2",
-      strip_prefix = "boringssl-e3860009a091cd1bd2bc189cdbc3c6d095abde84",
+      sha256 = "5e6f7b72c74adeb902581271925ddb979e77b96327abd76604ce894d80680e51",
+      strip_prefix = "boringssl-72cfd9f49ec5fbc2db368b76398c196dafe6a4bc",
       # Add patch to boringssl code to support s390x
       patch_file = str(Label("//third_party/boringssl:add_boringssl_s390x.patch")),
   )
diff --git a/third_party/boringssl/add_boringssl_s390x.patch b/third_party/boringssl/add_boringssl_s390x.patch
index 8b42d10e68..7e95224f30 100644
--- a/third_party/boringssl/add_boringssl_s390x.patch
+++ b/third_party/boringssl/add_boringssl_s390x.patch
@@ -10,124 +10,4 @@ index 7a3adfb..88012ad 100644
 +#define OPENSSL_64_BIT
  #else
  #error "Unknown target CPU"
- #endif
-diff --git a/BUILD b/BUILD
-index 6b645e61..c90b7beb 100644
---- a/BUILD
-+++ b/BUILD
-@@ -40,29 +40,46 @@ config_setting(
-     values = {"cpu": "darwin"},
- )
- 
--boringssl_copts = [
--    # Assembler option --noexecstack adds .note.GNU-stack to each object to
--    # ensure that binaries can be built with non-executable stack.
--    "-Wa,--noexecstack",
--
--    # This is needed on Linux systems (at least) to get rwlock in pthread.
--    "-D_XOPEN_SOURCE=700",
--
--    # This list of warnings should match those in the top-level CMakeLists.txt.
--    "-Wall",
--    "-Werror",
--    "-Wformat=2",
--    "-Wsign-compare",
--    "-Wmissing-field-initializers",
--    "-Wwrite-strings",
--    "-Wshadow",
--    "-fno-common",
--
--    # Modern build environments should be able to set this to use atomic
--    # operations for reference counting rather than locks. However, it's
--    # known not to work on some Android builds.
--    # "-DOPENSSL_C11_ATOMIC",
--] + select({
-+config_setting(
-+    name = "windows",
-+    values = {"cpu": "x64_windows"},
-+    visibility = ["//visibility:public"],
-+)
-+
-+config_setting(
-+    name = "windows_msvc",
-+    values = {"cpu": "x64_windows_msvc"},
-+    visibility = ["//visibility:public"],
-+)
-+
-+boringssl_copts = select({
-+    ":windows": [
-+        "-DWIN32_LEAN_AND_MEAN",
-+    ],
-+    "//conditions:default": [
-+        # Assembler option --noexecstack adds .note.GNU-stack to each object to
-+        # ensure that binaries can be built with non-executable stack.
-+        "-Wa,--noexecstack",
-+
-+        # This is needed on Linux systems (at least) to get rwlock in pthread.
-+        "-D_XOPEN_SOURCE=700",
-+
-+        # This list of warnings should match those in the top-level CMakeLists.txt.
-+        "-Wall",
-+        "-Werror",
-+        "-Wformat=2",
-+        "-Wsign-compare",
-+        "-Wmissing-field-initializers",
-+        "-Wwrite-strings",
-+        "-Wshadow",
-+        "-fno-common",
-+
-+        # Modern build environments should be able to set this to use atomic
-+        # operations for reference counting rather than locks. However, it's
-+        # known not to work on some Android builds.
-+        # "-DOPENSSL_C11_ATOMIC",
-+    ],
-+}) + select({
-     ":linux_x86_64": [],
-     ":mac_x86_64": [],
-     "//conditions:default": ["-DOPENSSL_NO_ASM"],
-@@ -75,18 +92,26 @@ crypto_sources_asm = select({
- })
- 
- # For C targets only (not C++), compile with C11 support.
--boringssl_copts_c11 = boringssl_copts + [
--    "-std=c11",
--    "-Wmissing-prototypes",
--    "-Wold-style-definition",
--    "-Wstrict-prototypes",
--]
-+boringssl_copts_c11 = boringssl_copts + select({
-+    ":windows": [],
-+    ":windows_msvc": [],
-+    "//conditions:default": [
-+        "-std=c11",
-+        "-Wmissing-prototypes",
-+        "-Wold-style-definition",
-+        "-Wstrict-prototypes",
-+    ],
-+})
- 
- # For C targets only (not C++), compile with C11 support.
--boringssl_copts_cxx = boringssl_copts + [
--    "-std=c++11",
--    "-Wmissing-declarations",
--]
-+boringssl_copts_cxx = boringssl_copts + select({
-+    ":windows": [],
-+    ":windows_msvc": [],
-+    "//conditions:default": [
-+        "-std=c++11",
-+        "-Wmissing-declarations",
-+    ],
-+})
- 
- cc_library(
-     name = "crypto",
-@@ -96,6 +121,8 @@ cc_library(
-     includes = ["src/include"],
-     linkopts = select({
-         ":mac_x86_64": [],
-+        ":windows": [],
-+        ":windows_msvc": [],
-         "//conditions:default": ["-lpthread"],
-     }),
-     visibility = ["//visibility:public"],
+ #endif
\ No newline at end of file
-- 
GitLab


From b75a5c20a5479fe465868ec2054c3086fc612fbc Mon Sep 17 00:00:00 2001
From: KB Sriram <kbsriram@gmail.com>
Date: Sat, 14 Oct 2017 23:14:05 -0700
Subject: [PATCH 404/909] Skeleton code for annotation processor. (#12828)

Add build changes and skeleton code (with test harness) for
the Operator annotation processor. This change focuses on build
and test-related changes, and generates an empty Ops class.

Please see #7149 for the master tracking issue.
---
 tensorflow/java/BUILD                         |  39 +++++
 .../processor/OperatorProcessor.java          | 149 ++++++++++++++++++
 .../javax.annotation.processing.Processor     |   1 +
 .../tensorflow/op/annotation/Operator.java    |   2 +-
 .../processor/OperatorProcessorTest.java      |  51 ++++++
 .../processor/operator/bad/BasicBad.java      |   7 +
 .../processor/operator/good/BasicGood.java    |   6 +
 .../tools/ci_build/builds/libtensorflow.sh    |  45 +++++-
 tensorflow/workspace.bzl                      |  25 +++
 9 files changed, 323 insertions(+), 2 deletions(-)
 create mode 100644 tensorflow/java/src/gen/java/org/tensorflow/processor/OperatorProcessor.java
 create mode 100644 tensorflow/java/src/gen/resources/META-INF/services/javax.annotation.processing.Processor
 create mode 100644 tensorflow/java/src/test/java/org/tensorflow/processor/OperatorProcessorTest.java
 create mode 100644 tensorflow/java/src/test/resources/org/tensorflow/processor/operator/bad/BasicBad.java
 create mode 100644 tensorflow/java/src/test/resources/org/tensorflow/processor/operator/good/BasicGood.java

diff --git a/tensorflow/java/BUILD b/tensorflow/java/BUILD
index a380bc2c71..ea6c5a494a 100644
--- a/tensorflow/java/BUILD
+++ b/tensorflow/java/BUILD
@@ -24,6 +24,7 @@ java_library(
     ],
     data = [":libtensorflow_jni"],
     javacopts = JAVACOPTS,
+    plugins = [":processor"],
     visibility = ["//visibility:public"],
 )
 
@@ -41,6 +42,21 @@ filegroup(
     ],
 )
 
+java_plugin(
+    name = "processor",
+    generates_api = True,
+    processor_class = "org.tensorflow.processor.OperatorProcessor",
+    visibility = ["//visibility:public"],
+    deps = [":processor_library"],
+)
+
+java_library(
+    name = "processor_library",
+    srcs = glob(["src/gen/java/org/tensorflow/processor/**/*.java"]),
+    javacopts = JAVACOPTS,
+    resources = glob(["src/gen/resources/META-INF/services/javax.annotation.processing.Processor"]),
+)
+
 filegroup(
     name = "java_op_sources",
     srcs = glob(["src/main/java/org/tensorflow/op/**/*.java"]) + [
@@ -264,6 +280,29 @@ tf_java_test(
     ],
 )
 
+java_test(
+    name = "OperatorProcessorTest",
+    size = "small",
+    srcs = ["src/test/java/org/tensorflow/processor/OperatorProcessorTest.java"],
+    javacopts = JAVACOPTS,
+    resources = [":processor_test_resources"],
+    test_class = "org.tensorflow.processor.OperatorProcessorTest",
+    deps = [
+        ":processor_library",
+        "@com_google_testing_compile",
+        "@com_google_truth",
+        "@junit",
+    ],
+)
+
+filegroup(
+    name = "processor_test_resources",
+    srcs = glob([
+        "src/test/resources/org/tensorflow/**/*.java",
+        "src/main/java/org/tensorflow/op/annotation/Operator.java",
+    ]),
+)
+
 filegroup(
     name = "libtensorflow_jni",
     srcs = select({
diff --git a/tensorflow/java/src/gen/java/org/tensorflow/processor/OperatorProcessor.java b/tensorflow/java/src/gen/java/org/tensorflow/processor/OperatorProcessor.java
new file mode 100644
index 0000000000..19b4f8ddda
--- /dev/null
+++ b/tensorflow/java/src/gen/java/org/tensorflow/processor/OperatorProcessor.java
@@ -0,0 +1,149 @@
+package org.tensorflow.processor;
+
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+import javax.annotation.processing.AbstractProcessor;
+import javax.annotation.processing.Filer;
+import javax.annotation.processing.Messager;
+import javax.annotation.processing.ProcessingEnvironment;
+import javax.annotation.processing.RoundEnvironment;
+import javax.lang.model.SourceVersion;
+import javax.lang.model.element.Element;
+import javax.lang.model.element.TypeElement;
+import javax.tools.Diagnostic.Kind;
+
+/**
+ * A compile-time Processor that aggregates classes annotated with {@link
+ * org.tensorflow.op.annotation.Operator} and generates the {@code Ops} convenience API. Please
+ * refer to the {@link org.tensorflow.op.annotation.Operator} annotation for details about the API
+ * generated for each annotated class.
+ *
+ * <p>Note that this processor can only be invoked once, in a single compilation run that includes
+ * all the {@code Operator} annotated source classes. The reason is that the {@code Ops} API is an
+ * "aggregating" API, and annotation processing does not permit modifying an already generated
+ * class.
+ *
+ * @see org.tensorflow.op.annotation.Operator
+ */
+public final class OperatorProcessor extends AbstractProcessor {
+
+  @Override
+  public SourceVersion getSupportedSourceVersion() {
+    return SourceVersion.latestSupported();
+  }
+
+  @Override
+  public synchronized void init(ProcessingEnvironment processingEnv) {
+    super.init(processingEnv);
+    messager = processingEnv.getMessager();
+    filer = processingEnv.getFiler();
+  }
+
+  @Override
+  public boolean process(Set<? extends TypeElement> annotations, RoundEnvironment roundEnv) {
+    // Nothing needs to be done at the end of all rounds.
+    if (roundEnv.processingOver()) {
+      return false;
+    }
+
+    // Nothing to look at in this round.
+    if (annotations.size() == 0) {
+      return false;
+    }
+
+    // We expect to be registered for exactly one annotation.
+    if (annotations.size() != 1) {
+      throw new IllegalStateException(
+          "Unexpected - multiple annotations registered: " + annotations);
+    }
+    TypeElement annotation = annotations.iterator().next();
+    Set<? extends Element> annotated = roundEnv.getElementsAnnotatedWith(annotation);
+
+    // If there are no annotated elements, claim the annotion but do nothing.
+    if (annotated.size() == 0) {
+      return true;
+    }
+
+    // This processor has to aggregate all op classes in one round, as it generates a single Ops
+    // API class which cannot be modified once generated. If we find an annotation after we've
+    // generated our code, flag the location of each such class.
+    if (hasRun) {
+      for (Element e : annotated) {
+        error(
+            e,
+            "The Operator processor has already processed @Operator annotated sources\n"
+                + "and written out an Ops API. It cannot process additional @Operator sources.\n"
+                + "One reason this can happen is if other annotation processors generate\n"
+                + "new @Operator source files.");
+      }
+      return true;
+    }
+
+    // Collect all classes tagged with our annotation.
+    Set<TypeElement> opClasses = new HashSet<TypeElement>();
+    if (!collectOpClasses(roundEnv, opClasses, annotation)) {
+      return true;
+    }
+
+    // Nothing to do when there are no tagged classes.
+    if (opClasses.isEmpty()) {
+      return true;
+    }
+
+    // TODO:(kbsriram) validate operator classes and generate Op API.
+    writeApi();
+    hasRun = true;
+    return true;
+  }
+
+  @Override
+  public Set<String> getSupportedAnnotationTypes() {
+    return Collections.singleton(String.format("%s.annotation.Operator", OP_PACKAGE));
+  }
+
+  private void writeApi() {
+    // Generate an empty class for now and get the build working correctly. This will be changed to
+    // generate the actual API once we've done with build-related changes.
+    // TODO:(kbsriram)
+    try (PrintWriter writer =
+        new PrintWriter(filer.createSourceFile(String.format("%s.Ops", OP_PACKAGE)).openWriter())) {
+      writer.println(String.format("package %s;", OP_PACKAGE));
+      writer.println("public class Ops{}");
+    } catch (IOException e) {
+      error(null, "Unexpected failure generating API: %s", e.getMessage());
+    }
+  }
+
+  private boolean collectOpClasses(
+      RoundEnvironment roundEnv, Set<TypeElement> opClasses, TypeElement annotation) {
+    boolean result = true;
+    for (Element e : roundEnv.getElementsAnnotatedWith(annotation)) {
+      // @Operator can only apply to types, so e must be a TypeElement.
+      if (!(e instanceof TypeElement)) {
+        error(
+            e,
+            "@Operator can only be applied to classes, but this is a %s",
+            e.getKind().toString());
+        result = false;
+        continue;
+      }
+      opClasses.add((TypeElement) e);
+    }
+    return result;
+  }
+
+  private void error(Element e, String message, Object... args) {
+    if (args != null && args.length > 0) {
+      message = String.format(message, args);
+    }
+    messager.printMessage(Kind.ERROR, message, e);
+  }
+
+  private Filer filer;
+  private Messager messager;
+  private boolean hasRun = false;
+  private static final String OP_PACKAGE = "org.tensorflow.op";
+}
diff --git a/tensorflow/java/src/gen/resources/META-INF/services/javax.annotation.processing.Processor b/tensorflow/java/src/gen/resources/META-INF/services/javax.annotation.processing.Processor
new file mode 100644
index 0000000000..9a4fc98a89
--- /dev/null
+++ b/tensorflow/java/src/gen/resources/META-INF/services/javax.annotation.processing.Processor
@@ -0,0 +1 @@
+org.tensorflow.processor.OperatorProcessor
diff --git a/tensorflow/java/src/main/java/org/tensorflow/op/annotation/Operator.java b/tensorflow/java/src/main/java/org/tensorflow/op/annotation/Operator.java
index 59476fb43d..3782240edb 100644
--- a/tensorflow/java/src/main/java/org/tensorflow/op/annotation/Operator.java
+++ b/tensorflow/java/src/main/java/org/tensorflow/op/annotation/Operator.java
@@ -54,7 +54,7 @@ import java.lang.annotation.Target;
  */
 @Documented
 @Target(ElementType.TYPE)
-@Retention(RetentionPolicy.CLASS)
+@Retention(RetentionPolicy.SOURCE)
 public @interface Operator {
   /**
    * Specify an optional group within the {@code Ops} class.
diff --git a/tensorflow/java/src/test/java/org/tensorflow/processor/OperatorProcessorTest.java b/tensorflow/java/src/test/java/org/tensorflow/processor/OperatorProcessorTest.java
new file mode 100644
index 0000000000..9fa1bad20d
--- /dev/null
+++ b/tensorflow/java/src/test/java/org/tensorflow/processor/OperatorProcessorTest.java
@@ -0,0 +1,51 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+package org.tensorflow.processor;
+
+import static com.google.testing.compile.CompilationSubject.assertThat;
+
+import com.google.testing.compile.Compilation;
+import com.google.testing.compile.Compiler;
+import com.google.testing.compile.JavaFileObjects;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Basic tests for {@link org.tensorflow.processor.operator.OperatorProcessor}. */
+@RunWith(JUnit4.class)
+public final class OperatorProcessorTest {
+
+  @Test
+  public void basicGood() {
+    Compilation compile = compile("org/tensorflow/processor/operator/good/BasicGood.java");
+    assertThat(compile).succeededWithoutWarnings();
+    assertThat(compile).generatedSourceFile("org.tensorflow.op.Ops");
+  }
+
+  @Test
+  public void basicBad() {
+    assertThat(compile("org/tensorflow/processor/operator/bad/BasicBad.java")).failed();
+  }
+
+  // Create a compilation unit that includes the @Operator annotation and processor.
+  private static Compilation compile(String path) {
+    return Compiler.javac()
+        .withProcessors(new OperatorProcessor())
+        .compile(
+            JavaFileObjects.forResource("src/main/java/org/tensorflow/op/annotation/Operator.java"),
+            JavaFileObjects.forResource(path));
+  }
+}
diff --git a/tensorflow/java/src/test/resources/org/tensorflow/processor/operator/bad/BasicBad.java b/tensorflow/java/src/test/resources/org/tensorflow/processor/operator/bad/BasicBad.java
new file mode 100644
index 0000000000..5ad3242637
--- /dev/null
+++ b/tensorflow/java/src/test/resources/org/tensorflow/processor/operator/bad/BasicBad.java
@@ -0,0 +1,7 @@
+package org.tensorflow.processor.operator.bad;
+
+import org.tensorflow.op.annotation.Operator;
+
+public class BasicBad {
+  @Operator int foo;
+}
diff --git a/tensorflow/java/src/test/resources/org/tensorflow/processor/operator/good/BasicGood.java b/tensorflow/java/src/test/resources/org/tensorflow/processor/operator/good/BasicGood.java
new file mode 100644
index 0000000000..fb69e83939
--- /dev/null
+++ b/tensorflow/java/src/test/resources/org/tensorflow/processor/operator/good/BasicGood.java
@@ -0,0 +1,6 @@
+package org.tensorflow.processor.operator.good;
+
+import org.tensorflow.op.annotation.Operator;
+
+@Operator
+public class BasicGood {}
diff --git a/tensorflow/tools/ci_build/builds/libtensorflow.sh b/tensorflow/tools/ci_build/builds/libtensorflow.sh
index 5052d3626c..26713dded8 100755
--- a/tensorflow/tools/ci_build/builds/libtensorflow.sh
+++ b/tensorflow/tools/ci_build/builds/libtensorflow.sh
@@ -78,9 +78,52 @@ function build_libtensorflow_tarball() {
     //tensorflow/tools/lib_package:libtensorflow_proto.zip
 
   mkdir -p ${DIR}
+
   cp bazel-bin/tensorflow/tools/lib_package/libtensorflow.tar.gz ${DIR}/libtensorflow${TARBALL_SUFFIX}.tar.gz
   cp bazel-bin/tensorflow/tools/lib_package/libtensorflow_jni.tar.gz ${DIR}/libtensorflow_jni${TARBALL_SUFFIX}.tar.gz
-  cp bazel-bin/tensorflow/java/libtensorflow.jar bazel-bin/tensorflow/java/libtensorflow-src.jar ${DIR}
+  cp bazel-bin/tensorflow/java/libtensorflow.jar ${DIR}
+  cp_normalized_srcjar bazel-bin/tensorflow/java/libtensorflow-src.jar ${DIR}/libtensorflow-src.jar
   cp bazel-genfiles/tensorflow/tools/lib_package/libtensorflow_proto.zip ${DIR}
   chmod -x ${DIR}/*
 }
+
+# Helper function to copy a srcjar after moving any source files
+# directly under the root to the "maven-style" src/main/java layout
+#
+# Source files generated by annotation processors appear directly
+# under the root of srcjars jars created by bazel, rather than under
+# the maven-style src/main/java subdirectory.
+#
+# Bazel manages annotation generated source as follows: First, it
+# calls javac with options that create generated files under a
+# bazel-out directory. Next, it archives the generated source files
+# into a srcjar directly under the root. There doesn't appear to be a
+# simple way to parameterize this from bazel, hence this helper to
+# "normalize" the srcjar layout.
+#
+# Arguments:
+#   src_jar - path to the original srcjar
+#   dest_jar - path to the destination
+# Returns:
+#   None
+function cp_normalized_srcjar() {
+  local src_jar="$1"
+  local dest_jar="$2"
+  if [[ -z "${src_jar}" || -z "${dest_jar}" ]]; then
+    echo "Unexpected: missing arguments" >&2
+    exit 2
+  fi
+  local tmp_dir
+  tmp_dir=$(mktemp -d)
+  cp "${src_jar}" "${tmp_dir}/orig.jar"
+  pushd "${tmp_dir}"
+  # Extract any src/ files
+  jar -xf "${tmp_dir}/orig.jar" src/
+  # Extract any org/ files under src/main/java
+  (mkdir -p src/main/java && cd src/main/java && jar -xf "${tmp_dir}/orig.jar" org/)
+  # Repackage src/
+  jar -cMf "${tmp_dir}/new.jar" src
+  popd
+  cp "${tmp_dir}/new.jar" "${dest_jar}"
+  rm -rf "${tmp_dir}"
+}
diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 47e99903d1..41d6a6c671 100644
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -699,6 +699,31 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       repository = tf_repo_name,
   )
 
+  java_import_external(
+      name = "com_google_testing_compile",
+      jar_sha256 = "edc180fdcd9f740240da1a7a45673f46f59c5578d8cd3fbc912161f74b5aebb8",
+      jar_urls = [
+          "http://mirror.bazel.build/repo1.maven.org/maven2/com/google/testing/compile/compile-testing/0.11/compile-testing-0.11.jar",
+          "http://repo1.maven.org/maven2/com/google/testing/compile/compile-testing/0.11/compile-testing-0.11.jar",
+          "http://maven.ibiblio.org/maven2/com/google/testing/compile/compile-testing/0.11/compile-testing-0.11.jar",
+      ],
+      licenses = ["notice"],  # New BSD License
+      testonly_ = True,
+      deps = ["@com_google_guava", "@com_google_truth"],
+  )
+
+  java_import_external(
+      name = "com_google_truth",
+      jar_sha256 = "032eddc69652b0a1f8d458f999b4a9534965c646b8b5de0eba48ee69407051df",
+      jar_urls = [
+          "http://mirror.bazel.build/repo1.maven.org/maven2/com/google/truth/truth/0.32/truth-0.32.jar",
+          "http://repo1.maven.org/maven2/com/google/truth/truth/0.32/truth-0.32.jar",
+      ],
+      licenses = ["notice"],  # Apache 2.0
+      testonly_ = True,
+      deps = ["@com_google_guava"],
+  )
+
   native.new_http_archive(
       name = "com_google_pprof",
       urls = [
-- 
GitLab


From dc65f63c3d5c22eea00833abe72f9b5cd8a662ba Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Sun, 15 Oct 2017 12:01:47 -0700
Subject: [PATCH 405/909] Update documentation for uint16 support in
 `tf.resize_...` ops (#13721)

* Update documentation for uint16 support in `tf.resize_...` ops

This fix tries to address the different between the registered
kernels and the documentation differnet for uint16 support of:
`tf.resize_area`
`tf.resize_bicubic`
`tf.resize_bilinear`
`tf.resize_nearest_neighbor`

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Add test cases for uint16 of `tf.resize_...`

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Add test case of float16 for `tf.resize...`.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/core/ops/image_ops.cc        | 8 ++++----
 tensorflow/python/ops/image_ops_test.py | 4 ++--
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/tensorflow/core/ops/image_ops.cc b/tensorflow/core/ops/image_ops.cc
index a44bac60bf..66765a3333 100644
--- a/tensorflow/core/ops/image_ops.cc
+++ b/tensorflow/core/ops/image_ops.cc
@@ -151,7 +151,7 @@ REGISTER_OP("ResizeArea")
     .Input("images: T")
     .Input("size: int32")
     .Output("resized_images: float")
-    .Attr("T: {uint8, int8, int16, int32, int64, half, float, double}")
+    .Attr("T: {int8, uint8, int16, uint16, int32, int64, half, float, double}")
     .Attr("align_corners: bool = false")
     .SetShapeFn(ResizeShapeFn)
     .Doc(R"doc(
@@ -179,7 +179,7 @@ REGISTER_OP("ResizeBicubic")
     .Input("images: T")
     .Input("size: int32")
     .Output("resized_images: float")
-    .Attr("T: {uint8, int8, int16, int32, int64, half, float, double}")
+    .Attr("T: {int8, uint8, int16, uint16, int32, int64, half, float, double}")
     .Attr("align_corners: bool = false")
     .SetShapeFn(ResizeShapeFn)
     .Doc(R"doc(
@@ -227,7 +227,7 @@ REGISTER_OP("ResizeBilinear")
     .Input("images: T")
     .Input("size: int32")
     .Output("resized_images: float")
-    .Attr("T: {uint8, int8, int16, int32, int64, half, float, double}")
+    .Attr("T: {int8, uint8, int16, uint16, int32, int64, half, float, double}")
     .Attr("align_corners: bool = false")
     .SetShapeFn(ResizeShapeFn)
     .Doc(R"doc(
@@ -311,7 +311,7 @@ REGISTER_OP("ResizeNearestNeighbor")
     .Input("images: T")
     .Input("size: int32")
     .Output("resized_images: T")
-    .Attr("T: {uint8, int8, int16, int32, int64, half, float, double}")
+    .Attr("T: {int8, uint8, int16, uint16, int32, int64, half, float, double}")
     .Attr("align_corners: bool = false")
     .SetShapeFn(ResizeShapeFn)
     .Doc(R"doc(
diff --git a/tensorflow/python/ops/image_ops_test.py b/tensorflow/python/ops/image_ops_test.py
index ebbf581204..348c005ff3 100644
--- a/tensorflow/python/ops/image_ops_test.py
+++ b/tensorflow/python/ops/image_ops_test.py
@@ -1672,8 +1672,8 @@ class ResizeImagesTest(test_util.TensorFlowTestCase):
              image_ops.ResizeMethod.BICUBIC,
              image_ops.ResizeMethod.AREA]
 
-  TYPES = [np.uint8, np.int8, np.int16, np.int32, np.int64,
-           np.float32, np.float64]
+  TYPES = [np.uint8, np.int8, np.uint16, np.int16, np.int32, np.int64,
+           np.float16, np.float32, np.float64]
 
   def _assertShapeInference(self, pre_shape, size, post_shape):
     # Try single image resize
-- 
GitLab


From d8b4b00de8d022e8ccd24bebc1173c2c4244dbf9 Mon Sep 17 00:00:00 2001
From: Mark Daoust <markdaoust@google.com>
Date: Sun, 15 Oct 2017 14:37:24 -0700
Subject: [PATCH 406/909] Add note pointing to master version of adding_an_op.

Fixes #13607

PiperOrigin-RevId: 172262174
---
 tensorflow/docs_src/extend/adding_an_op.md | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/tensorflow/docs_src/extend/adding_an_op.md b/tensorflow/docs_src/extend/adding_an_op.md
index 7d71fb5f4a..15d6d77f5e 100644
--- a/tensorflow/docs_src/extend/adding_an_op.md
+++ b/tensorflow/docs_src/extend/adding_an_op.md
@@ -1,5 +1,12 @@
 # Adding a New Op
 
+Note: By default [tensorflow.org](http://tensorflow.org) shows docs for the
+most recent stable version. The instructions in this doc require building from
+source. You will probably want to build from the `master` version of tensorflow.
+You should, as a result, be sure you are following the
+[`master` version of this doc](https://www.tensorflow.org/versions/master/extend/adding_an_op),
+in case there have been any changes.
+
 If you'd like to create an op that isn't covered by the existing TensorFlow
 library, we recommend that you first try writing the op in Python as
 a composition of existing Python ops or functions. If that isn't possible, you
-- 
GitLab


From 1cf9f7ab2fd019531629b266e7d0863b0d6417ce Mon Sep 17 00:00:00 2001
From: Shanqing Cai <cais@google.com>
Date: Sun, 15 Oct 2017 18:15:53 -0700
Subject: [PATCH 407/909] tfdbg: add persistent config

* Add two persistent UI configurations backed by a file at ~/.tfdbg_config by default.
  * graph_recursion_depth, which controls the recursive output of li/lo commands.
  * mouse_mode, which controls the mouse state of the CursesUI.
* Add `config` command to set and inspect the persistent configuration. E.g.,
  * config show
  * config set graph_recursion_depth 3
  * config set mouse_mode False

Fixes: #13449
PiperOrigin-RevId: 172270804
---
 .../docs_src/programmers_guide/debugger.md    |   3 +
 tensorflow/python/debug/BUILD                 |  24 +++
 tensorflow/python/debug/cli/analyzer_cli.py   |  36 +++-
 .../python/debug/cli/analyzer_cli_test.py     |  26 ++-
 tensorflow/python/debug/cli/base_ui.py        |  39 ++++-
 tensorflow/python/debug/cli/cli_config.py     | 160 ++++++++++++++++++
 .../python/debug/cli/cli_config_test.py       | 137 +++++++++++++++
 tensorflow/python/debug/cli/curses_ui.py      |  11 +-
 tensorflow/python/debug/cli/curses_ui_test.py |   4 +-
 .../python/debug/cli/profile_analyzer_cli.py  |   5 +-
 tensorflow/python/debug/cli/readline_ui.py    |   4 +-
 .../python/debug/cli/readline_ui_test.py      |  17 +-
 tensorflow/python/debug/cli/ui_factory.py     |  11 +-
 .../debug/wrappers/local_cli_wrapper.py       |   3 +-
 14 files changed, 451 insertions(+), 29 deletions(-)
 create mode 100644 tensorflow/python/debug/cli/cli_config.py
 create mode 100644 tensorflow/python/debug/cli/cli_config_test.py

diff --git a/tensorflow/docs_src/programmers_guide/debugger.md b/tensorflow/docs_src/programmers_guide/debugger.md
index 510c4df940..3ede42e8f7 100644
--- a/tensorflow/docs_src/programmers_guide/debugger.md
+++ b/tensorflow/docs_src/programmers_guide/debugger.md
@@ -186,6 +186,9 @@ Try the following commands at the `tfdbg>` prompt (referencing the code at
 | | `--tensor_dtype_filter <pattern>` | Execute the next `Session.run`, dumping only Tensors with data types (`dtype`s) matching the given regular-expression pattern. | `run --tensor_dtype_filter int.*` |
 | | `-p` | Execute the next `Session.run` call in profiling mode. | `run -p` |
 | **`ri`** | | **Display information about the run the current run, including fetches and feeds.** | `ri` |
+| **`config`** | | **Set or show persistent TFDBG UI configuration.** | |
+| | `set` | Set the value of a config item: {`graph_recursion_depth`, `mouse_mode`}. | `config set graph_recursion_depth 3` |
+| | `show` | Show current persistent UI configuration. | `config show` |
 | **`help`** | | **Print general help information** | `help` |
 | | `help <command>` | Print help for given command. | `help lt` |
 
diff --git a/tensorflow/python/debug/BUILD b/tensorflow/python/debug/BUILD
index ee53469cc7..b68b6e05b6 100644
--- a/tensorflow/python/debug/BUILD
+++ b/tensorflow/python/debug/BUILD
@@ -150,6 +150,13 @@ py_library(
     ],
 )
 
+py_library(
+    name = "cli_config",
+    srcs = ["cli/cli_config.py"],
+    srcs_version = "PY2AND3",
+    deps = [":debugger_cli_common"],
+)
+
 py_library(
     name = "command_parser",
     srcs = ["cli/command_parser.py"],
@@ -197,6 +204,7 @@ py_library(
     srcs = ["cli/analyzer_cli.py"],
     srcs_version = "PY2AND3",
     deps = [
+        ":cli_config",
         ":cli_shared",
         ":command_parser",
         ":debug_graphs",
@@ -249,6 +257,7 @@ py_library(
     srcs = ["cli/base_ui.py"],
     srcs_version = "PY2AND3",
     deps = [
+        ":cli_config",
         ":command_parser",
         ":debugger_cli_common",
     ],
@@ -583,6 +592,7 @@ py_test(
     srcs = ["cli/readline_ui_test.py"],
     srcs_version = "PY2AND3",
     deps = [
+        ":cli_config",
         ":debugger_cli_common",
         ":readline_ui",
         ":ui_factory",
@@ -724,6 +734,19 @@ py_test(
     ],
 )
 
+py_test(
+    name = "cli_config_test",
+    size = "small",
+    srcs = ["cli/cli_config_test.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":cli_config",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:platform",
+        "//tensorflow/python:platform_test",
+    ],
+)
+
 py_test(
     name = "command_parser_test",
     size = "small",
@@ -791,6 +814,7 @@ cuda_py_test(
     srcs = ["cli/analyzer_cli_test.py"],
     additional_deps = [
         ":analyzer_cli",
+        ":cli_config",
         ":command_parser",
         ":debug_data",
         ":debug_utils",
diff --git a/tensorflow/python/debug/cli/analyzer_cli.py b/tensorflow/python/debug/cli/analyzer_cli.py
index 22d1b4b543..afa3363d99 100644
--- a/tensorflow/python/debug/cli/analyzer_cli.py
+++ b/tensorflow/python/debug/cli/analyzer_cli.py
@@ -29,6 +29,7 @@ import re
 
 from six.moves import xrange  # pylint: disable=redefined-builtin
 
+from tensorflow.python.debug.cli import cli_config
 from tensorflow.python.debug.cli import cli_shared
 from tensorflow.python.debug.cli import command_parser
 from tensorflow.python.debug.cli import debugger_cli_common
@@ -140,11 +141,13 @@ class DebugAnalyzer(object):
   _GRAPH_STRUCT_OP_TYPE_BLACKLIST = (
       "_Send", "_Recv", "_HostSend", "_HostRecv", "_Retval")
 
-  def __init__(self, debug_dump):
+  def __init__(self, debug_dump, config):
     """DebugAnalyzer constructor.
 
     Args:
       debug_dump: A DebugDumpDir object.
+      config: A `cli_config.CLIConfig` object that carries user-facing
+        configurations.
     """
 
     self._debug_dump = debug_dump
@@ -153,6 +156,21 @@ class DebugAnalyzer(object):
     # Initialize tensor filters state.
     self._tensor_filters = {}
 
+    self._build_argument_parsers(config)
+    config.set_callback("graph_recursion_depth",
+                        self._build_argument_parsers)
+
+    # TODO(cais): Implement list_nodes.
+
+  def _build_argument_parsers(self, config):
+    """Build argument parsers for DebugAnalayzer.
+
+    Args:
+      config: A `cli_config.CLIConfig` object.
+
+    Returns:
+      A dict mapping command handler name to `ArgumentParser` instance.
+    """
     # Argument parsers for command handlers.
     self._arg_parsers = {}
 
@@ -242,7 +260,7 @@ class DebugAnalyzer(object):
         "--depth",
         dest="depth",
         type=int,
-        default=20,
+        default=config.get("graph_recursion_depth"),
         help="Maximum depth of recursion used when showing the input tree.")
     ap.add_argument(
         "-r",
@@ -273,7 +291,7 @@ class DebugAnalyzer(object):
         "--depth",
         dest="depth",
         type=int,
-        default=20,
+        default=config.get("graph_recursion_depth"),
         help="Maximum depth of recursion used when showing the output tree.")
     ap.add_argument(
         "-r",
@@ -386,8 +404,6 @@ class DebugAnalyzer(object):
         "(may be slow for large results).")
     self._arg_parsers["eval"] = ap
 
-    # TODO(cais): Implement list_nodes.
-
   def add_tensor_filter(self, filter_name, filter_callable):
     """Add a tensor filter.
 
@@ -1540,7 +1556,8 @@ class DebugAnalyzer(object):
 def create_analyzer_ui(debug_dump,
                        tensor_filters=None,
                        ui_type="curses",
-                       on_ui_exit=None):
+                       on_ui_exit=None,
+                       config=None):
   """Create an instance of CursesUI based on a DebugDumpDir object.
 
   Args:
@@ -1549,19 +1566,22 @@ def create_analyzer_ui(debug_dump,
       filter (Callable).
     ui_type: (str) requested UI type, e.g., "curses", "readline".
     on_ui_exit: (`Callable`) the callback to be called when the UI exits.
+    config: A `cli_config.CLIConfig` object.
 
   Returns:
     (base_ui.BaseUI) A BaseUI subtype object with a set of standard analyzer
       commands and tab-completions registered.
   """
+  if config is None:
+    config = cli_config.CLIConfig()
 
-  analyzer = DebugAnalyzer(debug_dump)
+  analyzer = DebugAnalyzer(debug_dump, config=config)
   if tensor_filters:
     for tensor_filter_name in tensor_filters:
       analyzer.add_tensor_filter(
           tensor_filter_name, tensor_filters[tensor_filter_name])
 
-  cli = ui_factory.get_ui(ui_type, on_ui_exit=on_ui_exit)
+  cli = ui_factory.get_ui(ui_type, on_ui_exit=on_ui_exit, config=config)
   cli.register_command_handler(
       "list_tensors",
       analyzer.list_tensors,
diff --git a/tensorflow/python/debug/cli/analyzer_cli_test.py b/tensorflow/python/debug/cli/analyzer_cli_test.py
index 8fcdcc777e..a7c1d35399 100644
--- a/tensorflow/python/debug/cli/analyzer_cli_test.py
+++ b/tensorflow/python/debug/cli/analyzer_cli_test.py
@@ -28,6 +28,7 @@ from tensorflow.core.protobuf import config_pb2
 from tensorflow.core.protobuf import rewriter_config_pb2
 from tensorflow.python.client import session
 from tensorflow.python.debug.cli import analyzer_cli
+from tensorflow.python.debug.cli import cli_config
 from tensorflow.python.debug.cli import cli_shared
 from tensorflow.python.debug.cli import command_parser
 from tensorflow.python.debug.cli import debugger_cli_common
@@ -45,6 +46,11 @@ from tensorflow.python.platform import test
 from tensorflow.python.util import tf_inspect
 
 
+def _cli_config_from_temp_file():
+  return cli_config.CLIConfig(
+      config_file_path=os.path.join(tempfile.mkdtemp(), ".tfdbg_config"))
+
+
 def no_rewrite_session_config():
   rewriter_config = rewriter_config_pb2.RewriterConfig(
       disable_model_pruning=True,
@@ -512,7 +518,7 @@ def create_analyzer_cli(dump):
        and has the common tfdbg commands, e.g., lt, ni, li, lo, registered.
   """
   # Construct the analyzer.
-  analyzer = analyzer_cli.DebugAnalyzer(dump)
+  analyzer = analyzer_cli.DebugAnalyzer(dump, _cli_config_from_temp_file())
 
   # Construct the handler registry.
   registry = debugger_cli_common.CommandHandlerRegistry()
@@ -1216,12 +1222,14 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase):
         "       [-14.,   4.]])"], out.lines)
 
   def testAddGetTensorFilterLambda(self):
-    analyzer = analyzer_cli.DebugAnalyzer(self._debug_dump)
+    analyzer = analyzer_cli.DebugAnalyzer(self._debug_dump,
+                                          _cli_config_from_temp_file())
     analyzer.add_tensor_filter("foo_filter", lambda x, y: True)
     self.assertTrue(analyzer.get_tensor_filter("foo_filter")(None, None))
 
   def testAddGetTensorFilterNestedFunction(self):
-    analyzer = analyzer_cli.DebugAnalyzer(self._debug_dump)
+    analyzer = analyzer_cli.DebugAnalyzer(self._debug_dump,
+                                          _cli_config_from_temp_file())
 
     def foo_filter(unused_arg_0, unused_arg_1):
       return True
@@ -1230,14 +1238,16 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase):
     self.assertTrue(analyzer.get_tensor_filter("foo_filter")(None, None))
 
   def testAddTensorFilterEmptyName(self):
-    analyzer = analyzer_cli.DebugAnalyzer(self._debug_dump)
+    analyzer = analyzer_cli.DebugAnalyzer(self._debug_dump,
+                                          _cli_config_from_temp_file())
 
     with self.assertRaisesRegexp(ValueError,
                                  "Input argument filter_name cannot be empty."):
       analyzer.add_tensor_filter("", lambda datum, tensor: True)
 
   def testAddTensorFilterNonStrName(self):
-    analyzer = analyzer_cli.DebugAnalyzer(self._debug_dump)
+    analyzer = analyzer_cli.DebugAnalyzer(self._debug_dump,
+                                          _cli_config_from_temp_file())
 
     with self.assertRaisesRegexp(
         TypeError,
@@ -1245,7 +1255,8 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase):
       analyzer.add_tensor_filter(1, lambda datum, tensor: True)
 
   def testAddGetTensorFilterNonCallable(self):
-    analyzer = analyzer_cli.DebugAnalyzer(self._debug_dump)
+    analyzer = analyzer_cli.DebugAnalyzer(self._debug_dump,
+                                          _cli_config_from_temp_file())
 
     with self.assertRaisesRegexp(
         TypeError, "Input argument filter_callable is expected to be callable, "
@@ -1253,7 +1264,8 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase):
       analyzer.add_tensor_filter("foo_filter", "bar")
 
   def testGetNonexistentTensorFilter(self):
-    analyzer = analyzer_cli.DebugAnalyzer(self._debug_dump)
+    analyzer = analyzer_cli.DebugAnalyzer(self._debug_dump,
+                                          _cli_config_from_temp_file())
 
     analyzer.add_tensor_filter("foo_filter", lambda datum, tensor: True)
     with self.assertRaisesRegexp(ValueError,
diff --git a/tensorflow/python/debug/cli/base_ui.py b/tensorflow/python/debug/cli/base_ui.py
index eed2b03a1a..464fefbf41 100644
--- a/tensorflow/python/debug/cli/base_ui.py
+++ b/tensorflow/python/debug/cli/base_ui.py
@@ -17,6 +17,9 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import argparse
+
+from tensorflow.python.debug.cli import cli_config
 from tensorflow.python.debug.cli import command_parser
 from tensorflow.python.debug.cli import debugger_cli_common
 
@@ -29,11 +32,13 @@ class BaseUI(object):
   ERROR_MESSAGE_PREFIX = "ERROR: "
   INFO_MESSAGE_PREFIX = "INFO: "
 
-  def __init__(self, on_ui_exit=None):
+  def __init__(self, on_ui_exit=None, config=None):
     """Constructor of the base class.
 
     Args:
       on_ui_exit: (`Callable`) the callback to be called when the UI exits.
+      config: An instance of `cli_config.CLIConfig()` carrying user-facing
+        configurations.
     """
 
     self._on_ui_exit = on_ui_exit
@@ -50,6 +55,20 @@ class BaseUI(object):
         [debugger_cli_common.CommandHandlerRegistry.HELP_COMMAND] +
         debugger_cli_common.CommandHandlerRegistry.HELP_COMMAND_ALIASES)
 
+    self._config = config or cli_config.CLIConfig()
+    self._config_argparser = argparse.ArgumentParser(
+        description="config command", usage=argparse.SUPPRESS)
+    subparsers = self._config_argparser.add_subparsers()
+    set_parser = subparsers.add_parser("set")
+    set_parser.add_argument("property_name", type=str)
+    set_parser.add_argument("property_value", type=str)
+    set_parser = subparsers.add_parser("show")
+    self.register_command_handler(
+        "config",
+        self._config_command_handler,
+        self._config_argparser.format_help(),
+        prefix_aliases=["cfg"])
+
   def set_help_intro(self, help_intro):
     """Set an introductory message to the help output of the command registry.
 
@@ -176,3 +195,21 @@ class BaseUI(object):
         except_last_word = " ".join(items[:-1]) + " "
 
     return context, prefix, except_last_word
+
+  @property
+  def config(self):
+    """Obtain the CLIConfig of this `BaseUI` instance."""
+    return self._config
+
+  def _config_command_handler(self, args, screen_info=None):
+    """Command handler for the "config" command."""
+    del screen_info  # Currently unused.
+
+    parsed = self._config_argparser.parse_args(args)
+    if hasattr(parsed, "property_name") and hasattr(parsed, "property_value"):
+      # set.
+      self._config.set(parsed.property_name, parsed.property_value)
+      return self._config.summarize(highlight=parsed.property_name)
+    else:
+      # show.
+      return self._config.summarize()
diff --git a/tensorflow/python/debug/cli/cli_config.py b/tensorflow/python/debug/cli/cli_config.py
new file mode 100644
index 0000000000..beed4f0a82
--- /dev/null
+++ b/tensorflow/python/debug/cli/cli_config.py
@@ -0,0 +1,160 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Configurations for TensorFlow Debugger (TFDBG) command-line interfaces."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import collections
+import json
+import os
+
+from tensorflow.python.debug.cli import debugger_cli_common
+from tensorflow.python.platform import gfile
+
+RL = debugger_cli_common.RichLine
+
+
+class CLIConfig(object):
+  """Client-facing configurations for TFDBG command-line interfaces."""
+
+  _CONFIG_FILE_NAME = ".tfdbg_config"
+
+  _DEFAULT_CONFIG = [
+      ("graph_recursion_depth", 20),
+      ("mouse_mode", True),
+  ]
+
+  def __init__(self, config_file_path=None):
+    self._config_file_path = (config_file_path or
+                              self._default_config_file_path())
+    self._config = collections.OrderedDict(self._DEFAULT_CONFIG)
+    if gfile.Exists(self._config_file_path):
+      config = self._load_from_file()
+      for key, value in config.items():
+        self._config[key] = value
+    self._save_to_file()
+
+    self._set_callbacks = dict()
+
+  def get(self, property_name):
+    if property_name not in self._config:
+      raise KeyError("%s is not a valid property name." % property_name)
+    return self._config[property_name]
+
+  def set(self, property_name, property_val):
+    """Set the value of a property.
+
+    Supports limitd property value types: `bool`, `int` and `str`.
+
+    Args:
+      property_name: Name of the property.
+      property_val: Value of the property. If the property has `bool` type and
+        this argument has `str` type, the `str` value will be parsed as a `bool`
+
+    Raises:
+      ValueError: if a `str` property_value fails to be parsed as a `bool`.
+      KeyError: if `property_name` is an invalid property name.
+    """
+    if property_name not in self._config:
+      raise KeyError("%s is not a valid property name." % property_name)
+
+    orig_val = self._config[property_name]
+    if isinstance(orig_val, bool):
+      if isinstance(property_val, str):
+        if property_val.lower() in ("1", "true", "t", "yes", "y", "on"):
+          property_val = True
+        elif property_val.lower() in ("0", "false", "f", "no", "n", "off"):
+          property_val = False
+        else:
+          raise ValueError(
+              "Invalid string value for bool type: %s" % property_val)
+      else:
+        property_val = bool(property_val)
+    elif isinstance(orig_val, int):
+      property_val = int(property_val)
+    elif isinstance(orig_val, str):
+      property_val = str(property_val)
+    else:
+      raise TypeError("Unsupported property type: %s" % type(orig_val))
+    self._config[property_name] = property_val
+    self._save_to_file()
+
+    # Invoke set-callback.
+    if property_name in self._set_callbacks:
+      self._set_callbacks[property_name](self._config)
+
+  def set_callback(self, property_name, callback):
+    """Set a set-callback for given property.
+
+    Args:
+      property_name: Name of the property.
+      callback: The callback as a `callable` of signature:
+          def cbk(config):
+        where config is the config after it is set to the new value.
+        The callback is invoked each time the set() method is called with the
+        matching property_name.
+
+    Raises:
+      KeyError: If property_name does not exist.
+      TypeError: If `callback` is not callable.
+    """
+    if property_name not in self._config:
+      raise KeyError("%s is not a valid property name." % property_name)
+    if not callable(callback):
+      raise TypeError("The callback object provided is not callable.")
+    self._set_callbacks[property_name] = callback
+
+  def _default_config_file_path(self):
+    return os.path.join(os.path.expanduser("~"), self._CONFIG_FILE_NAME)
+
+  def _save_to_file(self):
+    try:
+      with gfile.Open(self._config_file_path, "w") as config_file:
+        json.dump(self._config, config_file)
+    except IOError:
+      pass
+
+  def summarize(self, highlight=None):
+    """Get a text summary of the config.
+
+    Args:
+      highlight: A property name to highlight in the output.
+
+    Returns:
+      A `RichTextLines` output.
+    """
+    lines = [RL("Command-line configuration:", "bold"), RL("")]
+    for name, val in self._config.items():
+      highlight_attr = "bold" if name == highlight else None
+      line = RL("  ")
+      line += RL(name, ["underline", highlight_attr])
+      line += RL(": ")
+      line += RL(str(val), font_attr=highlight_attr)
+      lines.append(line)
+    return debugger_cli_common.rich_text_lines_from_rich_line_list(lines)
+
+  def _load_from_file(self):
+    try:
+      with gfile.Open(self._config_file_path, "r") as config_file:
+        config_dict = json.load(config_file)
+        config = collections.OrderedDict()
+        for key in sorted(config_dict.keys()):
+          config[key] = config_dict[key]
+        return config
+    except (IOError, ValueError):
+      # The reading of the config file may fail due to IO issues or file
+      # corruption. We do not want tfdbg to error out just because of that.
+      return dict()
diff --git a/tensorflow/python/debug/cli/cli_config_test.py b/tensorflow/python/debug/cli/cli_config_test.py
new file mode 100644
index 0000000000..f2b44aa637
--- /dev/null
+++ b/tensorflow/python/debug/cli/cli_config_test.py
@@ -0,0 +1,137 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for cli_config."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import json
+import os
+import shutil
+import tempfile
+
+from tensorflow.python.debug.cli import cli_config
+from tensorflow.python.framework import test_util
+from tensorflow.python.platform import gfile
+from tensorflow.python.platform import googletest
+
+
+class CLIConfigTest(test_util.TensorFlowTestCase):
+
+  def setUp(self):
+    self._tmp_dir = tempfile.mkdtemp()
+    self._tmp_config_path = os.path.join(self._tmp_dir, ".tfdbg_config")
+    self.assertFalse(gfile.Exists(self._tmp_config_path))
+    super(CLIConfigTest, self).setUp()
+
+  def tearDown(self):
+    shutil.rmtree(self._tmp_dir)
+    super(CLIConfigTest, self).tearDown()
+
+  def testConstructCLIConfigWithoutFile(self):
+    config = cli_config.CLIConfig(config_file_path=self._tmp_config_path)
+    self.assertEqual(20, config.get("graph_recursion_depth"))
+    self.assertEqual(True, config.get("mouse_mode"))
+    with self.assertRaises(KeyError):
+      config.get("property_that_should_not_exist")
+    self.assertTrue(gfile.Exists(self._tmp_config_path))
+
+  def testCLIConfigForwardCompatibilityTest(self):
+    config = cli_config.CLIConfig(config_file_path=self._tmp_config_path)
+    with open(self._tmp_config_path, "rt") as f:
+      config_json = json.load(f)
+    # Remove a field to simulate forward compatibility test.
+    del config_json["graph_recursion_depth"]
+    with open(self._tmp_config_path, "wt") as f:
+      json.dump(config_json, f)
+
+    config = cli_config.CLIConfig(config_file_path=self._tmp_config_path)
+    self.assertEqual(20, config.get("graph_recursion_depth"))
+
+  def testModifyConfigValue(self):
+    config = cli_config.CLIConfig(config_file_path=self._tmp_config_path)
+    config.set("graph_recursion_depth", 9)
+    config.set("mouse_mode", False)
+    self.assertEqual(9, config.get("graph_recursion_depth"))
+    self.assertEqual(False, config.get("mouse_mode"))
+
+  def testModifyConfigValueWithTypeCasting(self):
+    config = cli_config.CLIConfig(config_file_path=self._tmp_config_path)
+    config.set("graph_recursion_depth", "18")
+    config.set("mouse_mode", "false")
+    self.assertEqual(18, config.get("graph_recursion_depth"))
+    self.assertEqual(False, config.get("mouse_mode"))
+
+  def testModifyConfigValueWithTypeCastingFailure(self):
+    config = cli_config.CLIConfig(config_file_path=self._tmp_config_path)
+    with self.assertRaises(ValueError):
+      config.set("mouse_mode", "maybe")
+
+  def testLoadFromModifiedConfigFile(self):
+    config = cli_config.CLIConfig(config_file_path=self._tmp_config_path)
+    config.set("graph_recursion_depth", 9)
+    config.set("mouse_mode", False)
+    config2 = cli_config.CLIConfig(config_file_path=self._tmp_config_path)
+    self.assertEqual(9, config2.get("graph_recursion_depth"))
+    self.assertEqual(False, config2.get("mouse_mode"))
+
+  def testSummarizeFromConfig(self):
+    config = cli_config.CLIConfig(config_file_path=self._tmp_config_path)
+    output = config.summarize()
+    self.assertEqual(
+        ["Command-line configuration:",
+         "",
+         "  graph_recursion_depth: %d" % config.get("graph_recursion_depth"),
+         "  mouse_mode: %s" % config.get("mouse_mode")], output.lines)
+
+  def testSummarizeFromConfigWithHighlight(self):
+    config = cli_config.CLIConfig(config_file_path=self._tmp_config_path)
+    output = config.summarize(highlight="mouse_mode")
+    self.assertEqual(
+        ["Command-line configuration:",
+         "",
+         "  graph_recursion_depth: %d" % config.get("graph_recursion_depth"),
+         "  mouse_mode: %s" % config.get("mouse_mode")], output.lines)
+    self.assertEqual((2, 12, ["underline", "bold"]),
+                     output.font_attr_segs[3][0])
+    self.assertEqual((14, 18, "bold"), output.font_attr_segs[3][1])
+
+  def testSetCallback(self):
+    config = cli_config.CLIConfig(config_file_path=self._tmp_config_path)
+
+    test_value = {"graph_recursion_depth": -1}
+    def callback(config):
+      test_value["graph_recursion_depth"] = config.get("graph_recursion_depth")
+    config.set_callback("graph_recursion_depth", callback)
+
+    config.set("graph_recursion_depth", config.get("graph_recursion_depth") - 1)
+    self.assertEqual(test_value["graph_recursion_depth"],
+                     config.get("graph_recursion_depth"))
+
+  def testSetCallbackInvalidPropertyName(self):
+    config = cli_config.CLIConfig(config_file_path=self._tmp_config_path)
+
+    with self.assertRaises(KeyError):
+      config.set_callback("nonexistent_property_name", print)
+
+  def testSetCallbackNotCallable(self):
+    config = cli_config.CLIConfig(config_file_path=self._tmp_config_path)
+
+    with self.assertRaises(TypeError):
+      config.set_callback("graph_recursion_depth", 1)
+
+
+if __name__ == "__main__":
+  googletest.main()
diff --git a/tensorflow/python/debug/cli/curses_ui.py b/tensorflow/python/debug/cli/curses_ui.py
index 498e346393..bb52f90512 100644
--- a/tensorflow/python/debug/cli/curses_ui.py
+++ b/tensorflow/python/debug/cli/curses_ui.py
@@ -273,14 +273,16 @@ class CursesUI(base_ui.BaseUI):
 
   _single_instance_lock = threading.Lock()
 
-  def __init__(self, on_ui_exit=None):
+  def __init__(self, on_ui_exit=None, config=None):
     """Constructor of CursesUI.
 
     Args:
       on_ui_exit: (Callable) Callback invoked when the UI exits.
+      config: An instance of `cli_config.CLIConfig()` carrying user-facing
+        configurations.
     """
 
-    base_ui.BaseUI.__init__(self, on_ui_exit=on_ui_exit)
+    base_ui.BaseUI.__init__(self, on_ui_exit=on_ui_exit, config=config)
 
     self._screen_init()
     self._screen_refresh_size()
@@ -445,8 +447,11 @@ class CursesUI(base_ui.BaseUI):
     curses.cbreak()
     self._stdscr.keypad(1)
 
-    self._mouse_enabled = enable_mouse_on_start
+    self._mouse_enabled = self.config.get("mouse_mode")
     self._screen_set_mousemask()
+    self.config.set_callback(
+        "mouse_mode",
+        lambda cfg: self._set_mouse_enabled(cfg.get("mouse_mode")))
 
     self._screen_create_command_window()
 
diff --git a/tensorflow/python/debug/cli/curses_ui_test.py b/tensorflow/python/debug/cli/curses_ui_test.py
index 15e1356d29..4ca11e7e41 100644
--- a/tensorflow/python/debug/cli/curses_ui_test.py
+++ b/tensorflow/python/debug/cli/curses_ui_test.py
@@ -704,8 +704,8 @@ class CursesTest(test_util.TensorFlowTestCase):
     # The manually registered command, along with the automatically registered
     # exit commands should appear in the candidates.
     self.assertEqual(
-        [["a", "babble", "exit", "h", "help", "m", "mouse", "quit"]],
-        ui.candidates_lists)
+        [["a", "babble", "cfg", "config", "exit", "h", "help", "m", "mouse",
+          "quit"]], ui.candidates_lists)
 
     # The two candidates have no common prefix. So no command should have been
     # issued.
diff --git a/tensorflow/python/debug/cli/profile_analyzer_cli.py b/tensorflow/python/debug/cli/profile_analyzer_cli.py
index 3304194b1c..a384d255ba 100644
--- a/tensorflow/python/debug/cli/profile_analyzer_cli.py
+++ b/tensorflow/python/debug/cli/profile_analyzer_cli.py
@@ -768,7 +768,8 @@ class ProfileAnalyzer(object):
 def create_profiler_ui(graph,
                        run_metadata,
                        ui_type="curses",
-                       on_ui_exit=None):
+                       on_ui_exit=None,
+                       config=None):
   """Create an instance of CursesUI based on a `tf.Graph` and `RunMetadata`.
 
   Args:
@@ -776,11 +777,13 @@ def create_profiler_ui(graph,
     run_metadata: A `RunMetadata` protobuf object.
     ui_type: (str) requested UI type, e.g., "curses", "readline".
     on_ui_exit: (`Callable`) the callback to be called when the UI exits.
+    config: An instance of `cli_config.CLIConfig`.
 
   Returns:
     (base_ui.BaseUI) A BaseUI subtype object with a set of standard analyzer
       commands and tab-completions registered.
   """
+  del config  # Currently unused.
 
   analyzer = ProfileAnalyzer(graph, run_metadata)
 
diff --git a/tensorflow/python/debug/cli/readline_ui.py b/tensorflow/python/debug/cli/readline_ui.py
index 4317d14461..151638789f 100644
--- a/tensorflow/python/debug/cli/readline_ui.py
+++ b/tensorflow/python/debug/cli/readline_ui.py
@@ -26,8 +26,8 @@ from tensorflow.python.debug.cli import debugger_cli_common
 class ReadlineUI(base_ui.BaseUI):
   """Readline-based Command-line UI."""
 
-  def __init__(self, on_ui_exit=None):
-    base_ui.BaseUI.__init__(self, on_ui_exit=on_ui_exit)
+  def __init__(self, on_ui_exit=None, config=None):
+    base_ui.BaseUI.__init__(self, on_ui_exit=on_ui_exit, config=config)
     self._init_input()
 
   def _init_input(self):
diff --git a/tensorflow/python/debug/cli/readline_ui_test.py b/tensorflow/python/debug/cli/readline_ui_test.py
index f3ea02df13..c38d3b5b19 100644
--- a/tensorflow/python/debug/cli/readline_ui_test.py
+++ b/tensorflow/python/debug/cli/readline_ui_test.py
@@ -20,6 +20,7 @@ from __future__ import print_function
 import argparse
 import tempfile
 
+from tensorflow.python.debug.cli import cli_config
 from tensorflow.python.debug.cli import debugger_cli_common
 from tensorflow.python.debug.cli import readline_ui
 from tensorflow.python.debug.cli import ui_factory
@@ -32,7 +33,9 @@ class MockReadlineUI(readline_ui.ReadlineUI):
   """Test subclass of ReadlineUI that bypasses terminal manipulations."""
 
   def __init__(self, on_ui_exit=None, command_sequence=None):
-    readline_ui.ReadlineUI.__init__(self, on_ui_exit=on_ui_exit)
+    readline_ui.ReadlineUI.__init__(
+        self, on_ui_exit=on_ui_exit,
+        config=cli_config.CLIConfig(config_file_path=tempfile.mktemp()))
 
     self._command_sequence = command_sequence
     self._command_counter = 0
@@ -161,6 +164,18 @@ class CursesTest(test_util.TensorFlowTestCase):
     with gfile.Open(output_path, "r") as f:
       self.assertEqual("bar\nbar\n", f.read())
 
+  def testConfigSetAndShow(self):
+    """Run UI with an initial command specified."""
+
+    ui = MockReadlineUI(command_sequence=[
+        "config set graph_recursion_depth 5", "config show", "exit"])
+    ui.run_ui()
+    outputs = ui.observers["screen_outputs"]
+    self.assertEqual(
+        ["Command-line configuration:",
+         "",
+         "  graph_recursion_depth: 5"], outputs[1].lines[:3])
+
 
 if __name__ == "__main__":
   googletest.main()
diff --git a/tensorflow/python/debug/cli/ui_factory.py b/tensorflow/python/debug/cli/ui_factory.py
index 06b2d28bde..a07d165ef9 100644
--- a/tensorflow/python/debug/cli/ui_factory.py
+++ b/tensorflow/python/debug/cli/ui_factory.py
@@ -23,7 +23,10 @@ import copy
 SUPPORTED_UI_TYPES = ["curses", "readline"]
 
 
-def get_ui(ui_type, on_ui_exit=None, available_ui_types=None):
+def get_ui(ui_type,
+           on_ui_exit=None,
+           available_ui_types=None,
+           config=None):
   """Create a `base_ui.BaseUI` subtype.
 
   This factory method attempts to fallback to other available ui_types on
@@ -36,6 +39,8 @@ def get_ui(ui_type, on_ui_exit=None, available_ui_types=None):
     on_ui_exit: (`Callable`) the callback to be called when the UI exits.
     available_ui_types: (`None` or `list` of `str`) Manually-set available
       ui_types.
+    config: An instance of `cli_config.CLIConfig()` carrying user-facing
+      configurations.
 
   Returns:
     A `base_ui.BaseUI` subtype object.
@@ -53,10 +58,10 @@ def get_ui(ui_type, on_ui_exit=None, available_ui_types=None):
     # pylint: disable=g-import-not-at-top
     if not ui_type or ui_type == "curses":
       from tensorflow.python.debug.cli import curses_ui
-      return curses_ui.CursesUI(on_ui_exit=on_ui_exit)
+      return curses_ui.CursesUI(on_ui_exit=on_ui_exit, config=config)
     elif ui_type == "readline":
       from tensorflow.python.debug.cli import readline_ui
-      return readline_ui.ReadlineUI(on_ui_exit=on_ui_exit)
+      return readline_ui.ReadlineUI(on_ui_exit=on_ui_exit, config=config)
     # pylint: enable=g-import-not-at-top
   except ImportError:
     available_ui_types.remove(ui_type)
diff --git a/tensorflow/python/debug/wrappers/local_cli_wrapper.py b/tensorflow/python/debug/wrappers/local_cli_wrapper.py
index e06267ff5a..5bf6d9d1f4 100644
--- a/tensorflow/python/debug/wrappers/local_cli_wrapper.py
+++ b/tensorflow/python/debug/wrappers/local_cli_wrapper.py
@@ -414,7 +414,8 @@ class LocalCLIDebugWrapperSession(framework.BaseDebugWrapperSession):
   def _prep_profile_cli_for_run_end(self, py_graph, run_metadata):
     self._init_command = "lp"
     self._run_cli = profile_analyzer_cli.create_profiler_ui(
-        py_graph, run_metadata, ui_type=self._ui_type)
+        py_graph, run_metadata, ui_type=self._ui_type,
+        config=self._run_cli.config)
     self._title = "run-end (profiler mode): " + self._run_description
 
   def _launch_cli(self):
-- 
GitLab


From 1b0ca6d1dfea0f328737a026135669783f840c1e Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Sun, 15 Oct 2017 19:00:25 -0700
Subject: [PATCH 408/909] Fix build error with boringssl (#13734)

* Fix build error with boringssl

This fix tries to fix build error with boringssl on `Ubuntu 16.06`, `gcc version 5.4.0 20160609 (Ubuntu 5.4.0-6ubuntu1~16.04.4)`, and `bazel` `0.6.1`.

```sh
  /usr/bin/gcc -U_FORTIFY_SOURCE -fstack-protector -Wall -B/usr/bin -B/usr/bin -Wunused-but-set-parameter -Wno-free-nonheap-object -fno-omit-frame-pointer -g0 -O2 '-D_FORTIFY_SOURCE=1' -DNDEBUG -ffunction-sections -fdata-sections '-march=native' -MD -MF bazel-out/local-opt/bin/external/curl/_objs/curl/external/curl/lib/curl_multibyte.pic.d -fPIC -iquote external/curl -iquote bazel-out/local-opt/genfiles/external/curl -iquote external/zlib_archive -iquote bazel-out/local-opt/genfiles/external/zlib_archive -iquote external/bazel_tools -iquote bazel-out/local-opt/genfiles/external/bazel_tools -iquote external/boringssl -iquote bazel-out/local-opt/genfiles/external/boringssl -isystem external/curl/include -isystem bazel-out/local-opt/genfiles/external/curl/include -isystem external/zlib_archive -isystem bazel-out/local-opt/genfiles/external/zlib_archive -isystem external/bazel_tools/tools/cpp/gcc3 -isystem external/boringssl/src/include -isystem bazel-out/local-opt/genfiles/external/boringssl/src/include -Iexternal/curl/lib -D_GNU_SOURCE -DHAVE_CONFIG_H -DCURL_DISABLE_FTP -DCURL_DISABLE_NTLM -DHAVE_LIBZ -DHAVE_ZLIB_H -Wno-string-plus-int '-DCURL_MAX_WRITE_SIZE=65536' -fno-canonical-system-headers -Wno-builtin-macro-redefined '-D__DATE__="redacted"' '-D__TIMESTAMP__="redacted"' '-D__TIME__="redacted"' -c external/curl/lib/curl_multibyte.c -o bazel-out/local-opt/bin/external/curl/_objs/curl/external/curl/lib/curl_multibyte.pic.o)
ERROR: /home/ubuntu/.cache/bazel/_bazel_ubuntu/ad1e09741bb4109fbc70ef8216b59ee2/external/boringssl/BUILD:128:1: C++ compilation of rule '@boringssl//:ssl' failed (Exit 1)
external/boringssl/src/ssl/t1_lib.cc: In function 'int bssl::ssl_ext_key_share_parse_clienthello(bssl::SSL_HANDSHAKE*, bool*, bssl::Array<unsigned char>*, uint8_t*, CBS*)':
external/boringssl/src/ssl/t1_lib.cc:2189:7: error: 'peer_key.cbs_st::len' may be used uninitialized in this function [-Werror=maybe-uninitialized]
   CBS peer_key;
           ^
external/boringssl/src/ssl/t1_lib.cc:2189:7: error: 'peer_key.cbs_st::data' may be used uninitialized in this function [-Werror=maybe-uninitialized]
cc1plus: all warnings being treated as errors
Target //tensorflow/tools/pip_package:build_pip_package failed to build
Use --verbose_failures to see the command lines of failed build steps.
INFO: Elapsed time: 107.966s, Critical Path: 18.12s
FAILED: Build did NOT complete successfully
```

This fix is related to PR 13638

This fix fixes 13733.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Fix build error with clang

Change `-Wno-maybe-uninitialized` -> `-Wno-uninitialized`

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 .../boringssl/add_boringssl_s390x.patch       | 24 +++++++++++++------
 1 file changed, 17 insertions(+), 7 deletions(-)

diff --git a/third_party/boringssl/add_boringssl_s390x.patch b/third_party/boringssl/add_boringssl_s390x.patch
index 7e95224f30..b684dc6df7 100644
--- a/third_party/boringssl/add_boringssl_s390x.patch
+++ b/third_party/boringssl/add_boringssl_s390x.patch
@@ -1,13 +1,23 @@
-diff --git a/src/include/openssl/base.h b/src/include/openssl/base.h
-index 7a3adfb..88012ad 100644
---- a/src/include/openssl/base.h
-+++ b/src/include/openssl/base.h
-@@ -94,6 +94,8 @@ extern "C" {
+diff -ur a/BUILD b/BUILD
+--- a/BUILD	2017-10-10 15:50:34.000000000 +0000
++++ b/BUILD	2017-10-15 21:19:02.057606476 +0000
+@@ -63,6 +63,7 @@
+     "-Wwrite-strings",
+     "-Wshadow",
+     "-fno-common",
++    "-Wno-uninitialized",
+ 
+     # Modern build environments should be able to set this to use atomic
+     # operations for reference counting rather than locks. However, it's
+diff -ur a/src/include/openssl/base.h b/src/include/openssl/base.h
+--- a/src/include/openssl/base.h	2017-10-10 15:50:34.000000000 +0000
++++ b/src/include/openssl/base.h	2017-10-15 19:49:38.182154627 +0000
+@@ -106,6 +106,8 @@
  #define OPENSSL_PNACL
  #elif defined(__myriad2__)
  #define OPENSSL_32_BIT
 +#elif defined(__s390x__)
 +#define OPENSSL_64_BIT
  #else
- #error "Unknown target CPU"
- #endif
\ No newline at end of file
+ // Note BoringSSL only supports standard 32-bit and 64-bit two's-complement,
+ // little-endian architectures. Functions will not produce the correct answer
-- 
GitLab


From 78e01f465c8f474b79015c1dc1851c609e3dc9f4 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Sun, 15 Oct 2017 19:17:57 -0700
Subject: [PATCH 409/909] Update CUB to 1.7.4 (#13732)

* Update CUB to 1.7.4

This fix updates CUB to 1.7.4. It consists of bug fixes in radix sort,
compared with the old version (1.7.3).

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Update CMAKE file for CUB 1.7.4

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/contrib/cmake/external/cub.cmake | 4 ++--
 tensorflow/workspace.bzl                    | 8 ++++----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/tensorflow/contrib/cmake/external/cub.cmake b/tensorflow/contrib/cmake/external/cub.cmake
index d98579d207..7b263806d7 100644
--- a/tensorflow/contrib/cmake/external/cub.cmake
+++ b/tensorflow/contrib/cmake/external/cub.cmake
@@ -14,8 +14,8 @@
 # ==============================================================================
 include (ExternalProject)
 
-set(cub_URL http://mirror.bazel.build/github.com/NVlabs/cub/archive/1.7.3.zip)
-set(cub_HASH SHA256=b7ead9e291d34ffa8074243541c1380d63be63f88de23de8ee548db573b72ebe)
+set(cub_URL https://github.com/NVlabs/cub/archive/1.7.4.zip)
+set(cub_HASH SHA256=20a1a39fd97e5da7f40f5f2e7fd73fd2ea59f9dc4bb8a6c5f228aa543e727e31)
 set(cub_BUILD ${CMAKE_CURRENT_BINARY_DIR}/cub/src/cub)
 set(cub_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/cub/src/cub)
 set(cub_ARCHIVE_DIR ${CMAKE_CURRENT_BINARY_DIR}/external/cub_archive)
diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 41d6a6c671..b5618a06ca 100644
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -738,11 +738,11 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   native.new_http_archive(
       name = "cub_archive",
       urls = [
-          "http://mirror.bazel.build/github.com/NVlabs/cub/archive/1.7.3.zip",
-          # "https://github.com/NVlabs/cub/archive/1.7.3.zip",
+          "http://mirror.bazel.build/github.com/NVlabs/cub/archive/1.7.4.zip",
+          "https://github.com/NVlabs/cub/archive/1.7.4.zip",
       ],
-      sha256 = "b7ead9e291d34ffa8074243541c1380d63be63f88de23de8ee548db573b72ebe",
-      strip_prefix = "cub-1.7.3",
+      sha256 = "20a1a39fd97e5da7f40f5f2e7fd73fd2ea59f9dc4bb8a6c5f228aa543e727e31",
+      strip_prefix = "cub-1.7.4",
       build_file = str(Label("//third_party:cub.BUILD")),
   )
 
-- 
GitLab


From e30246c49b353b9136f69caef23e7ba0e9df0f0e Mon Sep 17 00:00:00 2001
From: Chris Leary <leary@google.com>
Date: Sun, 15 Oct 2017 20:17:11 -0700
Subject: [PATCH 410/909] [XLA] Make pad shape inference error more
 informative.

PiperOrigin-RevId: 172276292
---
 tensorflow/compiler/xla/service/shape_inference.cc | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xla/service/shape_inference.cc b/tensorflow/compiler/xla/service/shape_inference.cc
index f3c8e3aff3..6be6b77e85 100644
--- a/tensorflow/compiler/xla/service/shape_inference.cc
+++ b/tensorflow/compiler/xla/service/shape_inference.cc
@@ -464,7 +464,10 @@ StatusOr<Shape> InferWindowOutputShape(const Shape& base_shape,
   }
   if (ShapeUtil::Rank(operand_shape) != padding_config.dimensions_size()) {
     return InvalidArgument(
-        "the rank of the operand and the padding configuration do not match.");
+        "The rank of the operand and the padding configuration do not match: "
+        "%s vs %s",
+        ShapeUtil::HumanString(operand_shape).c_str(),
+        padding_config.ShortDebugString().c_str());
   }
   if (operand_shape.element_type() != padding_value_shape.element_type()) {
     return InvalidArgument(
-- 
GitLab


From ca8af1d0dbb605087a4f8ae076188f2b9a26b1ba Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Sun, 15 Oct 2017 21:36:48 -0700
Subject: [PATCH 411/909] Replace NcclReduce/Broadcast ops during graph
 optimization so that we can generate gradients for Reduce/Broadcast. Changing
 _NcclBroadcastRecv shape input to int32 so that the corresponding Const op is
 outputting to HostMem.

PiperOrigin-RevId: 172279684
---
 tensorflow/contrib/nccl/BUILD                 |   2 +
 tensorflow/contrib/nccl/kernels/nccl_ops.cc   |  28 +-
 .../contrib/nccl/kernels/nccl_rewrite.cc      | 271 ++++++++++++++++++
 tensorflow/contrib/nccl/ops/nccl_ops.cc       |  84 ++++--
 .../contrib/nccl/python/ops/nccl_ops.py       | 138 ++++-----
 .../contrib/nccl/python/ops/nccl_ops_test.py  |  87 +++---
 6 files changed, 483 insertions(+), 127 deletions(-)
 create mode 100644 tensorflow/contrib/nccl/kernels/nccl_rewrite.cc

diff --git a/tensorflow/contrib/nccl/BUILD b/tensorflow/contrib/nccl/BUILD
index d6508362b8..5e7263ff62 100644
--- a/tensorflow/contrib/nccl/BUILD
+++ b/tensorflow/contrib/nccl/BUILD
@@ -71,10 +71,12 @@ tf_kernel_library(
         "kernels/nccl_manager.cc",
         "kernels/nccl_manager.h",
         "kernels/nccl_ops.cc",
+        "kernels/nccl_rewrite.cc",
     ],
     deps = [
         "//tensorflow/core:framework",
         "//tensorflow/core:gpu_headers_lib",
+        "//tensorflow/core:proto_text",
         "@nccl_archive//:nccl",
     ],
     alwayslink = 1,
diff --git a/tensorflow/contrib/nccl/kernels/nccl_ops.cc b/tensorflow/contrib/nccl/kernels/nccl_ops.cc
index 4eb52492db..266d4f6f0d 100644
--- a/tensorflow/contrib/nccl/kernels/nccl_ops.cc
+++ b/tensorflow/contrib/nccl/kernels/nccl_ops.cc
@@ -15,8 +15,6 @@ limitations under the License.
 
 #if GOOGLE_CUDA
 
-#include <memory>
-#include <unordered_map>
 #include <vector>
 
 #include "src/nccl.h"
@@ -24,6 +22,7 @@ limitations under the License.
 #include "tensorflow/core/framework/op_kernel.h"
 
 namespace tensorflow {
+namespace {
 
 // Base class for all communicator ops that use nccl.
 //
@@ -134,7 +133,7 @@ class NcclReduceSendKernel : public NcclReduceOpBase {
         compute_stream, &c->input(0), std::move(actual_done));
   }
 };
-REGISTER_KERNEL_BUILDER(Name("NcclReduceSend").Device(DEVICE_GPU),
+REGISTER_KERNEL_BUILDER(Name("_NcclReduceSend").Device(DEVICE_GPU),
                         NcclReduceSendKernel);
 
 // To execute a single reduce, this kernel is called once for one devices, and
@@ -166,7 +165,7 @@ class NcclReduceRecvKernel : public NcclReduceOpBase {
  private:
   ncclRedOp_t reduction_op_;
 };
-REGISTER_KERNEL_BUILDER(Name("NcclReduceRecv").Device(DEVICE_GPU),
+REGISTER_KERNEL_BUILDER(Name("_NcclReduceRecv").Device(DEVICE_GPU),
                         NcclReduceRecvKernel);
 
 // To execute a single broadcast, this kernel is called once for one device, and
@@ -191,7 +190,7 @@ class NcclBroadcastSendKernel : public NcclAsyncOpBase {
         std::move(actual_done));
   }
 };
-REGISTER_KERNEL_BUILDER(Name("NcclBroadcastSend").Device(DEVICE_GPU),
+REGISTER_KERNEL_BUILDER(Name("_NcclBroadcastSend").Device(DEVICE_GPU),
                         NcclBroadcastSendKernel);
 
 // To execute a single broadcast, this kernel is called once for all but one of
@@ -206,7 +205,7 @@ class NcclBroadcastRecvKernel : public NcclAsyncOpBase {
     const Tensor& shape_t = c->input(0);
     TensorShape shape;
     OP_REQUIRES_OK_ASYNC(
-        c, TensorShapeUtils::MakeShape(shape_t.vec<int64>(), &shape), done);
+        c, TensorShapeUtils::MakeShape(shape_t.vec<int32>(), &shape), done);
     Tensor* out_t;
     OP_REQUIRES_OK_ASYNC(c, c->allocate_output(0, shape, &out_t), done);
 
@@ -224,9 +223,24 @@ class NcclBroadcastRecvKernel : public NcclAsyncOpBase {
   }
 };
 REGISTER_KERNEL_BUILDER(
-    Name("NcclBroadcastRecv").Device(DEVICE_GPU).HostMemory("shape"),
+    Name("_NcclBroadcastRecv").Device(DEVICE_GPU).HostMemory("shape"),
     NcclBroadcastRecvKernel);
 
+// Define stub kernels for the ops that get replaced post placement.
+class NcclStubKernel : public AsyncOpKernel {
+ public:
+  explicit NcclStubKernel(OpKernelConstruction* c) : AsyncOpKernel(c) {}
+  void ComputeAsync(OpKernelContext* c, DoneCallback done) override {
+    c->SetStatus(errors::Unimplemented(
+        "This op should be replaced during graph optimization."));
+    done();
+  }
+};
+REGISTER_KERNEL_BUILDER(Name("NcclBroadcast").Device(DEVICE_GPU),
+                        NcclStubKernel);
+REGISTER_KERNEL_BUILDER(Name("NcclReduce").Device(DEVICE_GPU), NcclStubKernel);
+
+}  // namespace
 }  // namespace tensorflow
 
 #endif  // GOOGLE_CUDA
diff --git a/tensorflow/contrib/nccl/kernels/nccl_rewrite.cc b/tensorflow/contrib/nccl/kernels/nccl_rewrite.cc
new file mode 100644
index 0000000000..94a77c59da
--- /dev/null
+++ b/tensorflow/contrib/nccl/kernels/nccl_rewrite.cc
@@ -0,0 +1,271 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+
+#include <forward_list>
+#include <vector>
+
+#include "tensorflow/core/common_runtime/optimization_registry.h"
+#include "tensorflow/core/graph/node_builder.h"
+
+namespace tensorflow {
+namespace {
+
+// Replaces NcclReduce node with _NcclReduceRecv reusing one input of same
+// device, adds one _NcclReduceSend for each other input.
+Status ReplaceReduce(Graph* graph, Node* node) {
+  string reduction;
+  TF_RETURN_IF_ERROR(GetNodeAttr(node->attrs(), "reduction", &reduction));
+  DataType dtype;
+  TF_RETURN_IF_ERROR(GetNodeAttr(node->attrs(), "T", &dtype));
+  int num_devices = node->num_inputs();
+  string shared_name = node->name();
+  auto make_builder = [&](StringPiece op_name, StringPiece suffix) {
+    return NodeBuilder(strings::StrCat(shared_name, suffix), op_name)
+        .Attr("reduction", reduction)
+        .Attr("num_devices", num_devices)
+        .Attr("shared_name", shared_name)
+        .Attr("T", dtype);
+  };
+  std::vector<Node*> control_inputs;
+  for (const auto& edge : node->in_edges()) {
+    if (edge->IsControlEdge()) {
+      control_inputs.push_back(edge->src());
+    }
+  }
+  std::vector<NodeBuilder::NodeOut> out_nodes;
+  for (const auto& edge : node->out_edges()) {
+    out_nodes.emplace_back(edge->dst(), edge->dst_input());
+  }
+  int recv_dev = node->assigned_device_name_index();
+  NodeBuilder recv_builder =
+      make_builder("_NcclReduceRecv", "Recv").ControlInputs(control_inputs);
+  bool recv_input_set = false;
+  int send_counter = 0;
+  for (const auto& edge : node->in_edges()) {
+    Node* src_node = edge->src();
+    if (edge->IsControlEdge()) {
+      continue;
+    }
+    int send_dev = src_node->assigned_device_name_index();
+    if (!recv_input_set && send_dev == recv_dev) {
+      recv_builder.Input(src_node);
+      recv_input_set = true;
+      continue;
+    }
+    auto send_builder = make_builder("_NcclReduceSend",
+                                     strings::StrCat("Send_", ++send_counter))
+                            .Input(src_node)
+                            .ControlInputs(control_inputs);
+    Node* send_node = nullptr;
+    TF_RETURN_IF_ERROR(send_builder.Finalize(graph, &send_node));
+    send_node->set_assigned_device_name_index(send_dev);
+    // Send nodes don't have any outputs and therefore have no data dependencies
+    // to the outputs of the graph. We add a control dependency to the receive
+    // node so that those 'dangling' nodes are run.
+    // TODO(b/67027412): Avoid these cross-device control edges.
+    for (const auto& out_node : out_nodes) {
+      graph->AddControlEdge(send_node, out_node.node);
+    }
+  }
+  if (!recv_input_set) {
+    return errors::InvalidArgument(
+        "No input tensor uses the same device as the NcclReduce op");
+  }
+  Node* recv_node = nullptr;
+  TF_RETURN_IF_ERROR(recv_builder.Finalize(graph, &recv_node));
+  recv_node->set_assigned_device_name_index(recv_dev);
+  graph->RemoveNode(node);
+  for (const auto& out_node : out_nodes) {
+    if (out_node.index == Graph::kControlSlot) {
+      graph->AddControlEdge(recv_node, out_node.node);
+    } else {
+      graph->AddEdge(recv_node, 0, out_node.node, out_node.index);
+    }
+  }
+  return Status::OK();
+}
+
+TensorProto TensorFromShape(const TensorShapeProto& shape) {
+  TensorProto result;
+  result.set_dtype(DT_INT32);
+  for (const auto& dim : shape.dim()) {
+    result.add_int_val(dim.size());
+  }
+  result.mutable_tensor_shape()->add_dim()->set_size(shape.dim_size());
+  return result;
+}
+
+// Replaces NcclBroadcast node with _NcclBroadcastSend, connects the input to
+// all outputs of same device, adds one _NcclBroadcastRecv for each other output
+// device.
+Status ReplaceBroadcast(Graph* graph, Node* node) {
+  DataType dtype;
+  TF_RETURN_IF_ERROR(GetNodeAttr(node->attrs(), "T", &dtype));
+  int send_dev = node->assigned_device_name_index();
+  int num_devices = 0;  // Number of distinct devices, incremented below.
+
+  // Map device name index to nodes that take the broadcast as input.
+  std::vector<std::forward_list<NodeBuilder::NodeOut>> out_nodes_map;
+  for (const auto& edge : node->out_edges()) {
+    int dst_dev = edge->IsControlEdge()
+                      ? send_dev
+                      : edge->dst()->assigned_device_name_index();
+    if (out_nodes_map.size() <= dst_dev) {
+      out_nodes_map.resize(dst_dev + 1);
+    }
+    auto it = out_nodes_map.begin() + dst_dev;
+    if (it->empty()) {
+      ++num_devices;
+    }
+    it->emplace_front(NodeBuilder::NodeOut(edge->dst(), edge->dst_input()));
+  }
+
+  if (num_devices <= 1) {
+    // Only one participating device, skip NCCL op.
+    const Edge* in_edge = nullptr;
+    TF_RETURN_IF_ERROR(node->input_edge(0, &in_edge));
+    Node* in_node = in_edge->src();
+    int in_index = in_edge->src_output();
+    graph->RemoveNode(node);
+    for (const auto& out_nodes : out_nodes_map) {
+      for (const auto& out_node : out_nodes) {
+        if (out_node.index == Graph::kControlSlot) {
+          graph->AddControlEdge(in_node, out_node.node);
+        } else {
+          graph->AddEdge(in_node, in_index, out_node.node, out_node.index);
+        }
+      }
+    }
+    return Status::OK();
+  }
+
+  string shared_name = node->name();
+  auto make_builder = [&](StringPiece op_name, StringPiece suffix) {
+    return NodeBuilder(strings::StrCat(shared_name, suffix), op_name)
+        .Attr("num_devices", num_devices)
+        .Attr("shared_name", shared_name)
+        .Attr("T", dtype);
+  };
+
+  // Create broadcast send node and replace the original broadcast node.
+  NodeBuilder::NodeOut in_node;
+  NodeBuilder send_builder = make_builder("_NcclBroadcastSend", "Send");
+  for (const auto& edge : node->in_edges()) {
+    if (edge->IsControlEdge()) {
+      send_builder.ControlInput(edge->src());
+    } else {
+      in_node = NodeBuilder::NodeOut(edge->src(), edge->src_output());
+      send_builder.Input(in_node);
+    }
+  }
+  Node* send_node = nullptr;
+  TF_RETURN_IF_ERROR(send_builder.Finalize(graph, &send_node));
+  send_node->set_assigned_device_name_index(send_dev);
+
+  TensorShapeProto shape_proto;
+  TF_RETURN_IF_ERROR(GetNodeAttr(node->attrs(), "shape", &shape_proto));
+
+  // Delete the original node before reconnecting to outputs.
+  graph->RemoveNode(node);
+
+  // Connect all outputs on the device of broadcast send.
+  for (const auto& out_node : out_nodes_map[send_dev]) {
+    if (out_node.index == Graph::kControlSlot) {
+      graph->AddControlEdge(send_node, out_node.node);
+    } else {
+      graph->AddEdge(in_node.node, in_node.index, out_node.node,
+                     out_node.index);
+      // Add control edge so send node is run.
+      graph->AddControlEdge(send_node, out_node.node);
+    }
+  }
+  out_nodes_map[send_dev].clear();
+
+  TensorProto tensor_proto = TensorFromShape(shape_proto);
+  bool is_fully_defined = TensorShape(shape_proto).IsFullyDefined();
+  string shape_name = strings::StrCat(in_node.node->name(), "/Shape");
+  Node* shape_node = nullptr;
+  if (!is_fully_defined) {
+    NodeBuilder shape_builder(shape_name, "Shape");
+    shape_builder.Input(in_node).Attr("out_type", DT_INT32).Attr("T", dtype);
+    TF_RETURN_IF_ERROR(shape_builder.Finalize(graph, &shape_node));
+    shape_node->set_assigned_device_name_index(send_dev);
+  }
+
+  // For all other devices, create a broadcast receive and connect outputs.
+  for (int recv_dev = 0; recv_dev < out_nodes_map.size(); ++recv_dev) {
+    if (out_nodes_map[recv_dev].empty()) {
+      continue;
+    }
+    if (is_fully_defined) {
+      // If the shape is fully defined, define one const node per device.
+      NodeBuilder shape_builder(strings::StrCat(shape_name, recv_dev), "Const");
+      shape_builder.Attr("value", tensor_proto).Attr("dtype", DT_INT32);
+      TF_RETURN_IF_ERROR(shape_builder.Finalize(graph, &shape_node));
+      shape_node->set_assigned_device_name_index(recv_dev);
+    }
+    Node* recv_node;
+    TF_RETURN_IF_ERROR(
+        make_builder("_NcclBroadcastRecv", strings::StrCat("Recv_", recv_dev))
+            .Input(shape_node)
+            .Finalize(graph, &recv_node));
+    recv_node->set_assigned_device_name_index(recv_dev);
+    for (const auto& out_node : out_nodes_map[recv_dev]) {
+      graph->AddEdge(recv_node, 0, out_node.node, out_node.index);
+    }
+  }
+
+  return Status::OK();
+}
+
+// Replaces occurrences of Nccl{Reduce, Broadcast}Input/Output with their
+// _Nccl...Send/Recv counterparts and removes data dependencies between them.
+class NcclReplacePass : public GraphOptimizationPass {
+ public:
+  Status Run(const GraphOptimizationPassOptions& options) override {
+    if (options.graph == nullptr) {
+      return Status::OK();
+    }
+    Graph* graph = options.graph->get();
+    if (graph == nullptr) {
+      return errors::Internal(
+          "NCCL replacement should happen before partitioning and a "
+          "graph should be available.");
+    }
+    // Find reduction and broadcast ops and replace them with Send/Recv ops.
+    for (Node* node : graph->op_nodes()) {
+      StringPiece type = node->type_string();
+      if (!type.starts_with("Nccl")) {
+        continue;
+      }
+      if (type == "NcclReduce") {
+        TF_RETURN_IF_ERROR(ReplaceReduce(graph, node));
+      }
+      if (type == "NcclBroadcast") {
+        TF_RETURN_IF_ERROR(ReplaceBroadcast(graph, node));
+      }
+    }
+    return Status::OK();
+  }
+};
+REGISTER_OPTIMIZATION(OptimizationPassRegistry::POST_PLACEMENT, 0,
+                      NcclReplacePass);
+
+}  // namespace
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/contrib/nccl/ops/nccl_ops.cc b/tensorflow/contrib/nccl/ops/nccl_ops.cc
index 532c79c24c..8eb804c2e9 100644
--- a/tensorflow/contrib/nccl/ops/nccl_ops.cc
+++ b/tensorflow/contrib/nccl/ops/nccl_ops.cc
@@ -45,7 +45,28 @@ num_devices: The number of devices participating in this reduction.
 shared_name: Identifier that shared between ops of the same reduction.
 )doc");
 
-REGISTER_OP("NcclReduceSend")
+// Note: This op has no kernel implementation, but is replaced by
+// _NcclReduceSend and _NcclReduceRecv during graph optimization stage.
+REGISTER_OP("NcclReduce")
+    .Input("input: num_devices * T")
+    .Output("data: T")
+    .Attr("reduction: {'min', 'max', 'prod', 'sum'}")
+    .Attr("T: {float, float64, int32, int64}")
+    .Attr("num_devices: int")
+    .SetIsStateful()
+    .SetShapeFn(shape_inference::UnchangedShape)
+    .Doc(R"doc(
+Reduces `input` from `num_devices` using `reduction` to a single device.
+
+The graph should be constructed so that all inputs have a valid device
+assignment, and the op itself is assigned one of these devices.
+
+input: The input to the reduction.
+data: the value of the reduction across all `num_devices` devices.
+reduction: the reduction operation to perform.
+    )doc");
+
+REGISTER_OP("_NcclReduceSend")
     .Input("input: T")
     .Attr("reduction: {'min', 'max', 'prod', 'sum'}")
     .Attr("T: {float, float64, int32, int64}")
@@ -54,19 +75,20 @@ REGISTER_OP("NcclReduceSend")
     .SetIsStateful()
     .SetShapeFn(shape_inference::NoOutputs)
     .Doc(R"doc(
-Reduces `input` to the NcclReduceRecv op registered in the same `shared_name`.
+Replacement node for NcclReduce.
 
+Reduces `input` to the NcclReduceRecv op registered in the same `shared_name`.
 The graph should be constructed so that 'num_devices-1' devices run
-`NcclReduceSend` and one device runs NcclReduceRecv op with shared_name value
+`_NcclReduceSend` and one device runs _NcclReduceRecv op with shared_name value
 `c`. Failure to do so will cause the graph execution to fail to complete.
 
-input: The input to the reduction
+input: The input to the reduction.
 reduction: the reduction operation to perform.
 num_devices: The number of devices participating in this reduction.
 shared_name: Identifier that is shared between ops of the same reduce.
     )doc");
 
-REGISTER_OP("NcclReduceRecv")
+REGISTER_OP("_NcclReduceRecv")
     .Input("input: T")
     .Output("data: T")
     .Attr("reduction: {'min', 'max', 'prod', 'sum'}")
@@ -76,21 +98,42 @@ REGISTER_OP("NcclReduceRecv")
     .SetIsStateful()
     .SetShapeFn(shape_inference::UnchangedShape)
     .Doc(R"doc(
+Replacement node for NcclReduce.
+
 Reduces 'input' from this op and the NcclReduceSend ops registered in the same
 `shared_name`.
-
 The graph should be constructed so that 'num_devices-1' devices run
-`NcclReduceSend` and one device runs NcclReduceRecv op with shared_name value
+`_NcclReduceSend` and one device runs _NcclReduceRecv op with shared_name value
 `c`. Failure to do so will cause the graph execution to fail to complete.
 
-input: The input to the reduction
+input: The input to the reduction.
 data: The reduced data received from this op and the NcclReduceSend op.
 reduction: the reduction operation to perform.
 num_devices: The number of devices participating in this reduction.
 shared_name: Identifier that is shared between ops of the same reduce.
     )doc");
 
-REGISTER_OP("NcclBroadcastSend")
+// Note: This op has no kernel implementation, but is replaced by
+// _NcclBroadcastSend and _NcclBroadcastRecv during graph optimization stage.
+REGISTER_OP("NcclBroadcast")
+    .Input("input: T")
+    .Output("output: T")
+    .Attr("T: {float, float64, int32, int64}")
+    .Attr("shape: shape")
+    .SetIsStateful()
+    .SetShapeFn(shape_inference::UnchangedShape)
+    .Doc(R"doc(
+Sends `input` to all devices that are connected to the output.
+
+The graph should be constructed so that all ops connected to the output have a
+valid device assignment, and the op itself is assigned one of these devices.
+
+input: The input to the broadcast.
+output: The same as input.
+shape: The shape of the input tensor.
+    )doc");
+
+REGISTER_OP("_NcclBroadcastSend")
     .Input("input: T")
     .Attr("T: {float, float64, int32, int64}")
     .Attr("num_devices: int")
@@ -98,19 +141,21 @@ REGISTER_OP("NcclBroadcastSend")
     .SetIsStateful()
     .SetShapeFn(shape_inference::NoOutputs)
     .Doc(R"doc(
-Sends `input` to the NcclBroadcastRecv ops registered in the same `shared_name`.
+Replacement node for NcclBroadcast.
 
-The graph should be constructed so that one device runs `NcclBroadcastSend` and
-`num_devices-1` devices run NcclBroadcastRecv ops with shared_name value `c`.
+Sends `input` to the _NcclBroadcastRecv ops registered in the same
+`shared_name`.
+The graph should be constructed so that one device runs `_NcclBroadcastSend` and
+`num_devices-1` devices run _NcclBroadcastRecv ops with shared_name value `c`.
 Failure to do so will cause the graph execution to fail to complete.
 
-input: The input to the broadcast
+input: The input to the broadcast.
 num_devices: The number of devices participating in this reduction.
 shared_name: Identifier that is shared between ops of the same broadcast.
     )doc");
 
-REGISTER_OP("NcclBroadcastRecv")
-    .Input("shape: int64")
+REGISTER_OP("_NcclBroadcastRecv")
+    .Input("shape: int32")
     .Output("output: T")
     .Attr("T: {float, float64, int32, int64}")
     .Attr("num_devices: int")
@@ -123,11 +168,12 @@ REGISTER_OP("NcclBroadcastRecv")
       return Status::OK();
     })
     .Doc(R"doc(
-Sends data of shape `shape` from the NcclBroadcastSend op registered in the
-same `shared_name`.
+Replacement node for NcclBroadcast.
 
-The graph should be constructed so that one device runs `NcclBroadcastSend` and
-`num_devices-1` devices run NcclBroadcastRecv ops with shared_name value `c`.
+Sends data of shape `shape` from the _NcclBroadcastSend op registered in the
+same `shared_name`.
+The graph should be constructed so that one device runs `_NcclBroadcastSend` and
+`num_devices-1` devices run _NcclBroadcastRecv ops with shared_name value `c`.
 Failure to do so will cause the graph execution to fail to complete.
 
 shape: The shape of the output.
diff --git a/tensorflow/contrib/nccl/python/ops/nccl_ops.py b/tensorflow/contrib/nccl/python/ops/nccl_ops.py
index 906d9f948a..8dc038b9ac 100644
--- a/tensorflow/contrib/nccl/python/ops/nccl_ops.py
+++ b/tensorflow/contrib/nccl/python/ops/nccl_ops.py
@@ -23,9 +23,7 @@ from tensorflow.contrib.nccl.ops import gen_nccl_ops
 from tensorflow.contrib.util import loader
 from tensorflow.python.eager import context
 from tensorflow.python.framework import device
-from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
-from tensorflow.python.ops import array_ops
 from tensorflow.python.platform import resource_loader
 
 _nccl_ops_so = loader.load_op_library(
@@ -64,13 +62,13 @@ def _all_sum_grad(op, grad):
     LookupError: If `reduction` is not `sum`.
   """
   if op.get_attr('reduction') != 'sum':
-    raise LookupError('No gradient defined for NcclAllReduce except all_sum.')
+    raise LookupError('No gradient defined for NcclAllReduce except sum.')
 
-  _check_device_assignment(grad)
+  _check_device(grad, expected=op.device)
   num_devices = op.get_attr('num_devices')
   shared_name = op.get_attr('shared_name') + '_grad'
 
-  with ops.device(grad.device):
+  with ops.device(op.device):
     return gen_nccl_ops.nccl_all_reduce(
         input=grad,
         reduction='sum',
@@ -129,7 +127,7 @@ def all_max(tensors):
   return _apply_all_reduce('max', tensors)
 
 
-def reduce_sum(tensors, dst_device):
+def reduce_sum(tensors):
   """Returns a tensor with the reduce sum across `tensors`.
 
   The computation is done with a reduce operation, so only one tensor is
@@ -138,54 +136,76 @@ def reduce_sum(tensors, dst_device):
   Args:
     tensors: The input tensors across which to sum; must be assigned
       to GPU devices.
-    dst_device: The device of the returned tensor.
 
   Returns:
-    A tensor containing the sum of the input tensors, with the device of the
-    tensor being `dst_device`.
+    A tensor containing the sum of the input tensors.
+
+  Raises:
+    LookupError: If context is not currently using a GPU device.
+  """
+  return _apply_reduce('sum', tensors)
+
+
+@ops.RegisterGradient('NcclReduce')
+def _reduce_sum_grad(op, grad):
+  """The gradients for input `Operation` of `reduce_sum`.
+
+  Args:
+    op: The `sum send` `Operation` that we are differentiating.
+    grad: Gradient with respect to the output of the `reduce_sum` op.
+
+  Returns:
+    The gradient with respect to the input of `reduce_sum` op.
+
+  Raises:
+    LookupError: If the reduction attribute of op is not `sum`.
   """
-  return _apply_reduce('sum', tensors, dst_device)
+  if op.get_attr('reduction') != 'sum':
+    raise LookupError('No gradient defined for NcclReduce except sum.')
+  _check_device(grad, expected=op.device)
 
+  with ops.device(op.device):
+    result = gen_nccl_ops.nccl_broadcast(input=grad, shape=grad.shape)
 
-def broadcast(src_tensor, dst_devices):
-  """Returns a list of tensors on `dst_devices`, each with value `tensor`.
+  return [result] * len(op.inputs)
 
-  The computation is done with a broadcast nccl operation, so if only some of
-  the returned tensors and src_tensor are evaluated then the computation will
-  hang.
+
+def broadcast(tensor):
+  """Returns a tensor that can be efficiently transferred to other devices.
 
   Args:
-    src_tensor: The tensor to send; must be assigned to a GPU device.
-    dst_devices: The GPU devices to receive the sent tensor.
+    tensor: The tensor to send; must be assigned to a GPU device.
 
   Returns:
-    An `Operation` to send the `src_tensor`, and a list of tensors, each with
-    the value of `src_tensor`, where the device of tensor i is `dst_devices[i]`.
+    A tensor with the value of `src_tensor`, which can be used as input to
+    ops on other GPU devices.
   """
-  if not dst_devices:
-    raise ValueError('Must pass >0 dst_devices to broadcast')
   _check_graph_mode()
-  _check_device_assignment(src_tensor)
+  _check_device(tensor)
 
-  shape = array_ops.shape(src_tensor, out_type=dtypes.int64)
-  num_devices = len(dst_devices) + 1
-  shared_name = _get_shared_name()
+  with ops.device(tensor.device):
+    return gen_nccl_ops.nccl_broadcast(input=tensor, shape=tensor.shape)
 
-  with ops.device(src_tensor.device):
-    send = gen_nccl_ops.nccl_broadcast_send(
-        input=src_tensor, num_devices=num_devices, shared_name=shared_name)
-
-  recvs = []
-  for d in dst_devices:
-    with ops.device(d):
-      recvs.append(
-          gen_nccl_ops.nccl_broadcast_recv(
-              shape=shape,
-              T=src_tensor.dtype,
-              num_devices=num_devices,
-              shared_name=shared_name))
 
-  return send, recvs
+@ops.RegisterGradient('NcclBroadcast')
+def _broadcast_grad(op, accumulated_grad):
+  """The gradients for input `Operation` of `broadcast`.
+
+  Args:
+    op: The `broadcast send` `Operation` that we are differentiating.
+    accumulated_grad: Accumulated gradients with respect to the output of the
+      `broadcast` op.
+
+  Returns:
+    Gradients with respect to the input of `broadcast`.
+  """
+  # Grab inputs of accumulated_grad and replace accumulation with reduce_sum.
+  grads = [t for t in accumulated_grad.op.inputs]
+  for t in grads:
+    _check_device(t)
+
+  with ops.device(op.device):
+    return gen_nccl_ops.nccl_reduce(input=grads, reduction='sum')
 
 
 def _apply_all_reduce(reduction, tensors):
@@ -198,7 +218,7 @@ def _apply_all_reduce(reduction, tensors):
   res = []
 
   for t in tensors:
-    _check_device_assignment(t)
+    _check_device(t)
     with ops.device(t.device):
       res.append(
           gen_nccl_ops.nccl_all_reduce(
@@ -210,40 +230,20 @@ def _apply_all_reduce(reduction, tensors):
   return res
 
 
-def _apply_reduce(reduction, tensors, dst_device):
+def _apply_reduce(reduction, tensors):
   """Helper function for reduce_* functions."""
   if not tensors:
     raise ValueError('Must pass >0 tensors to reduce operations')
-  if not dst_device:
-    raise ValueError('Must pass dst_device to reduce operations')
   _check_graph_mode()
 
+  for t in tensors:
+    _check_device(t)
+  result = gen_nccl_ops.nccl_reduce(input=tensors, reduction=reduction)
   try:
-    recv_index = next(i for i, t in enumerate(tensors)
-                      if t.device == dst_device)
+    next(t for t in tensors if t.device == result.device)
   except StopIteration:
-    raise ValueError('One of the tensors must be assigned to dst_device')
-  shared_name = _get_shared_name()
-
-  sends = []
-  for t in tensors[:recv_index] + tensors[recv_index + 1:]:
-    _check_device_assignment(t)
-    with ops.device(t.device):
-      sends.append(
-          gen_nccl_ops.nccl_reduce_send(
-              input=t,
-              reduction=reduction,
-              num_devices=len(tensors),
-              shared_name=shared_name))
-
-  with ops.device(dst_device):
-    recv = gen_nccl_ops.nccl_reduce_recv(
-        input=tensors[recv_index],
-        reduction=reduction,
-        num_devices=len(tensors),
-        shared_name=shared_name)
-
-  return recv, sends
+    raise ValueError('One input tensor must be assigned to current device')
+  return result
 
 
 _lock = threading.Lock()
@@ -259,9 +259,11 @@ def _get_shared_name():
   return 'c%s' % val
 
 
-def _check_device_assignment(tensor):
+def _check_device(tensor, expected=None):
   if not device.canonical_name(tensor.device):
     raise ValueError('Device assignment required for nccl collective ops')
+  if expected and expected != tensor.device:
+    raise ValueError('Expected device %s, got %s' % (expected, tensor.device))
 
 
 def _check_graph_mode():
diff --git a/tensorflow/contrib/nccl/python/ops/nccl_ops_test.py b/tensorflow/contrib/nccl/python/ops/nccl_ops_test.py
index 96d67723a0..255409303a 100644
--- a/tensorflow/contrib/nccl/python/ops/nccl_ops_test.py
+++ b/tensorflow/contrib/nccl/python/ops/nccl_ops_test.py
@@ -22,8 +22,10 @@ from functools import partial
 import numpy as np
 
 from tensorflow.contrib import nccl
+from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import gradients
 from tensorflow.python.platform import test
 
 
@@ -36,27 +38,30 @@ def _DeviceTensors(tensors, devices):
 
 
 def _NcclAllReduce(nccl_fun, tensors, devices):
-  return nccl_fun(_DeviceTensors(tensors, devices)), []
+  return nccl_fun(_DeviceTensors(tensors, devices))
 
 
 def _NcclReduce(nccl_fun, tensors, devices):
-  d_tensors = _DeviceTensors(tensors, devices)
   receiver = np.random.randint(0, len(devices))
-  received_tensor, send_ops = nccl_fun(d_tensors, devices[receiver])
-  return [received_tensor], send_ops
+  with ops.device(devices[receiver]):
+    return [nccl_fun(_DeviceTensors(tensors, devices))]
 
 
 def _NcclBroadcast(tensors, devices):
   sender = np.random.randint(0, len(devices))
-  d_tensor = _DeviceTensors(tensors[0:1], devices[sender:sender + 1])[0]
-  other_devices = devices[:sender] + devices[sender + 1:]
-  send_op, received_tensors = nccl.broadcast(d_tensor, other_devices)
-  return received_tensors, [send_op]
+  with ops.device(devices[sender]):
+    tensor = array_ops.identity(tensors[0])
+    broadcast = nccl.broadcast(tensor)
+  return _DeviceTensors([broadcast] * len(devices), devices)
 
 
 class NcclTestCase(test.TestCase):
 
-  def _Test(self, nccl_reduce, numpy_fn):
+  def _Test(self,
+            nccl_reduce,
+            numpy_fn,
+            device_sets=(['/device:GPU:1', '/device:GPU:2', '/device:GPU:0'],
+                         ['/device:GPU:1', '/device:GPU:0'])):
     """Tests that nccl_reduce does the same as reduction with numpy_fn.
 
     Args:
@@ -65,6 +70,7 @@ class NcclTestCase(test.TestCase):
           reduction.
       numpy_fn: A function taking two tensors and returning the reduction of the
           two.
+      device_sets: Tuple of virtual devices to run test on.
     """
     if not test.is_gpu_available():
       return  # Test requires access to a GPU
@@ -74,26 +80,28 @@ class NcclTestCase(test.TestCase):
       # same communicator across multiple sessions.
       with self.test_session(use_gpu=True) as sess:
 
-        for devices in [['/device:GPU:1', '/device:GPU:2', '/device:GPU:0'],
-                        ['/device:GPU:1', '/device:GPU:0']]:
+        for devices in device_sets:
           shape = (3, 4)
           random = (np.random.random_sample(shape) - .5) * 1024
-          tensors = [random.astype(dtype)] * len(devices)
+          tensors = []
+          for _ in devices:
+            tensors.append(random.astype(dtype))
           np_ans = tensors[0]
           for t in tensors[1:]:
             np_ans = numpy_fn(np_ans, t)
 
-          reduce_tensors, reduce_ops = nccl_reduce(tensors, devices)
+          reduce_tensors = nccl_reduce(tensors, devices)
           self.assertNotEmpty(reduce_tensors)
 
           # Test shape inference.
           for r in reduce_tensors:
             self.assertEqual(shape, r.get_shape())
 
+          result_tensors = [array_ops.identity(t) for t in reduce_tensors]
+
           # Test execution and results.
-          nccl_results = sess.run(reduce_tensors + reduce_ops)
-          for r in nccl_results[:len(reduce_tensors)]:
-            self.assertAllClose(r, np_ans)
+          for t in sess.run(result_tensors):
+            self.assertAllClose(t, np_ans)
 
   def _TestGradient(self, nccl_reduce, numpy_fn):
     """Tests the gradient of nccl_reduce.
@@ -106,14 +114,11 @@ class NcclTestCase(test.TestCase):
           reduction of the two.
     """
     def _Gradient(tensors, devices):
-      reduce_tensors, _ = nccl_reduce(tensors, devices)
-      tensor_ops = [t.op for t in reduce_tensors]
-      d_tensors = _DeviceTensors(tensors, devices)
-      grad_tensors = [
-          ops.get_gradient_function(op)(op, loss)
-          for op, loss in zip(tensor_ops, d_tensors)
-      ]
-      return grad_tensors, []
+      inputs = [array_ops.placeholder(t.dtype, t.shape) for t in tensors]
+      reduce_tensors = nccl_reduce(inputs, devices)
+      losses = _DeviceTensors(tensors, [t.device for t in reduce_tensors])
+      grads = gradients.gradients(reduce_tensors, inputs, losses)
+      return [g for g in grads if g is not None]
 
     self._Test(_Gradient, numpy_fn)
 
@@ -142,27 +147,43 @@ class SingleReduceTest(NcclTestCase):
   def testSum(self):
     self._Test(partial(_NcclReduce, nccl.reduce_sum), lambda x, y: x + y)
 
+  def testSumGrad(self):
+    self._TestGradient(partial(_NcclReduce, nccl.reduce_sum), lambda x, y: x)
+
 
 class BroadcastTest(NcclTestCase):
 
   def testBroadcast(self):
     self._Test(_NcclBroadcast, lambda x, y: x)
 
+  def testBroadcastSingleDevice(self):
+    # Broadcasts on a single device are removed completely during rewrite.
+    self._Test(_NcclBroadcast, lambda x, y: x,
+               (['/device:GPU:0', '/device:GPU:0']))
+
+  def testBroadcastToCpuError(self):
+    # Broadcasts to CPU is not supported.
+    with self.assertRaisesRegexp(
+        errors.NotFoundError,
+        "No registered '_NcclBroadcastRecv' OpKernel for CPU devices"):
+      self._Test(_NcclBroadcast, lambda x, y: x,
+                 (['/device:GPU:0', '/device:CPU:0']))
+
+  def testBroadcastGrad(self):
+    self._TestGradient(_NcclBroadcast, lambda x, y: x + y)
+
 
 class CombinedTest(NcclTestCase):
   """Test all-reduce vs. single-reduce plus broadcast in one session.run."""
 
-  def _combined(self, tensors, devices):
-    all_reduce_tensors = _NcclAllReduce(nccl.all_sum, tensors, devices)[0]
-    single_reduce_tensors, single_reduce_ops = _NcclReduce(
-        nccl.reduce_sum, tensors, devices)
-    broadcast_tensors, broadcast_ops = _NcclBroadcast(single_reduce_tensors,
-                                                      devices)
-    all_tensors = all_reduce_tensors + single_reduce_tensors + broadcast_tensors
-    return all_tensors, single_reduce_ops + broadcast_ops
+  def _Combined(self, tensors, devices):
+    all_reduce_tensors = _NcclAllReduce(nccl.all_sum, tensors, devices)
+    single_reduce_tensors = _NcclReduce(nccl.reduce_sum, tensors, devices)
+    broadcast_tensors = _NcclBroadcast(single_reduce_tensors, devices)
+    return all_reduce_tensors + broadcast_tensors
 
   def testCombined(self):
-    self._Test(self._combined, lambda x, y: x + y)
+    self._Test(self._Combined, lambda x, y: x + y)
 
 
 if __name__ == '__main__':
-- 
GitLab


From 1324d40ad7996e4f2afd8f5c5b5f68416c9872e1 Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Sun, 15 Oct 2017 22:46:47 -0700
Subject: [PATCH 412/909] Internal change.

PiperOrigin-RevId: 172282778
---
 tensorflow/python/estimator/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/python/estimator/BUILD b/tensorflow/python/estimator/BUILD
index e4b2d95acd..9670827e41 100644
--- a/tensorflow/python/estimator/BUILD
+++ b/tensorflow/python/estimator/BUILD
@@ -146,6 +146,7 @@ py_test(
     srcs = ["training_test.py"],
     shard_count = 4,
     srcs_version = "PY2AND3",
+    tags = ["notsan"],
     deps = [
         ":dnn",
         ":estimator",
-- 
GitLab


From 48c4b45ab68f09317415ef2b03c74e349319ce8b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 16 Oct 2017 05:51:00 -0700
Subject: [PATCH 413/909] Remove unused BUILD dependencies

PiperOrigin-RevId: 172314225
---
 tensorflow/compiler/xla/service/BUILD         |  1 -
 tensorflow/compiler/xla/service/llvm_ir/BUILD |  1 -
 tensorflow/contrib/cloud/kernels/BUILD        | 21 +++++++------------
 tensorflow/core/kernels/BUILD                 |  1 -
 tensorflow/core/platform/cloud/BUILD          | 17 ++++++---------
 5 files changed, 13 insertions(+), 28 deletions(-)

diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD
index 3e85c796f2..d1335e20e0 100644
--- a/tensorflow/compiler/xla/service/BUILD
+++ b/tensorflow/compiler/xla/service/BUILD
@@ -433,7 +433,6 @@ cc_library(
         ":hlo_evaluator",
         ":hlo_execution_profile",
         ":hlo_module_config",
-        ":hlo_verifier",
         ":platform_util",
         ":session_proto",
         ":transfer_manager",
diff --git a/tensorflow/compiler/xla/service/llvm_ir/BUILD b/tensorflow/compiler/xla/service/llvm_ir/BUILD
index 70579e3273..075d4a1ab5 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/BUILD
+++ b/tensorflow/compiler/xla/service/llvm_ir/BUILD
@@ -137,7 +137,6 @@ cc_library(
         "//tensorflow/compiler/xla/service:hlo",
         "//tensorflow/compiler/xla/service/gpu:parallel_loop_emitter",
         "//tensorflow/compiler/xla/service/gpu:partition_assignment",
-        "@llvm//:core",
     ],
 )
 
diff --git a/tensorflow/contrib/cloud/kernels/BUILD b/tensorflow/contrib/cloud/kernels/BUILD
index 09ec7e42c7..56f930a9a8 100644
--- a/tensorflow/contrib/cloud/kernels/BUILD
+++ b/tensorflow/contrib/cloud/kernels/BUILD
@@ -23,7 +23,9 @@ load(
 filegroup(
     name = "all_files",
     srcs = glob(
-        ["**/*"],
+        include = [
+            "**/*",
+        ],
         exclude = [
             "**/METADATA",
             "**/OWNERS",
@@ -34,9 +36,7 @@ filegroup(
 
 tf_kernel_library(
     name = "bigquery_reader_ops",
-    srcs = [
-        "bigquery_reader_ops.cc",
-    ],
+    srcs = ["bigquery_reader_ops.cc"],
     visibility = ["//visibility:public"],
     deps = [
         ":bigquery_table_accessor",
@@ -50,12 +50,8 @@ tf_kernel_library(
 
 cc_library(
     name = "bigquery_table_accessor",
-    srcs = [
-        "bigquery_table_accessor.cc",
-    ],
-    hdrs = [
-        "bigquery_table_accessor.h",
-    ],
+    srcs = ["bigquery_table_accessor.cc"],
+    hdrs = ["bigquery_table_accessor.h"],
     copts = tf_copts(),
     linkstatic = 1,
     deps = [
@@ -64,7 +60,6 @@ cc_library(
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core/platform/cloud:curl_http_request",
         "//tensorflow/core/platform/cloud:google_auth_provider",
-        "//tensorflow/core/platform/cloud:http_request",
     ],
     alwayslink = 1,
 )
@@ -88,8 +83,6 @@ tf_cc_test(
 
 tf_proto_library(
     name = "bigquery_table_partition_proto",
-    srcs = [
-        "bigquery_table_partition.proto",
-    ],
+    srcs = ["bigquery_table_partition.proto"],
     cc_api_version = 2,
 )
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 2c02571346..d1a2362e5e 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -5671,7 +5671,6 @@ cc_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
-        "//tensorflow/core/util/tensor_bundle",
     ],
 )
 
diff --git a/tensorflow/core/platform/cloud/BUILD b/tensorflow/core/platform/cloud/BUILD
index c937fea049..901fb79d6a 100644
--- a/tensorflow/core/platform/cloud/BUILD
+++ b/tensorflow/core/platform/cloud/BUILD
@@ -15,7 +15,9 @@ load(
 filegroup(
     name = "all_files",
     srcs = glob(
-        ["**/*"],
+        include = [
+            "**/*",
+        ],
         exclude = [
             "**/METADATA",
             "**/OWNERS",
@@ -41,12 +43,8 @@ cc_library(
 
 cc_library(
     name = "gcs_file_system",
-    srcs = [
-        "gcs_file_system.cc",
-    ],
-    hdrs = [
-        "gcs_file_system.h",
-    ],
+    srcs = ["gcs_file_system.cc"],
+    hdrs = ["gcs_file_system.h"],
     linkstatic = 1,  # Needed since alwayslink is broken in bazel b/27630669
     visibility = ["//visibility:public"],
     deps = [
@@ -106,9 +104,7 @@ cc_library(
 
 cc_library(
     name = "google_auth_provider",
-    srcs = [
-        "google_auth_provider.cc",
-    ],
+    srcs = ["google_auth_provider.cc"],
     hdrs = [
         "auth_provider.h",
         "google_auth_provider.h",
@@ -116,7 +112,6 @@ cc_library(
     visibility = ["//tensorflow:__subpackages__"],
     deps = [
         ":curl_http_request",
-        ":http_request",
         ":oauth_client",
         ":retrying_utils",
         "//tensorflow/core:lib",
-- 
GitLab


From c0f8a1fad0c91aa04b69e17717b75a43b3ca7ecd Mon Sep 17 00:00:00 2001
From: Taehoon Lee <me@taehoonlee.com>
Date: Mon, 16 Oct 2017 21:57:10 +0900
Subject: [PATCH 414/909] Fix typos

---
 tensorflow/compiler/xla/service/hlo_computation_test.cc | 2 +-
 tensorflow/contrib/all_reduce/python/all_reduce.py      | 2 +-
 tensorflow/contrib/kfac/python/ops/loss_functions.py    | 6 +++---
 tensorflow/contrib/kfac/python/ops/op_queue.py          | 2 +-
 tensorflow/core/kernels/conv_ops_gpu_3.cu.cc            | 2 +-
 tensorflow/core/kernels/dataset.h                       | 2 +-
 6 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_computation_test.cc b/tensorflow/compiler/xla/service/hlo_computation_test.cc
index ccab7bf348..7b7588f4ba 100644
--- a/tensorflow/compiler/xla/service/hlo_computation_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_computation_test.cc
@@ -310,7 +310,7 @@ TEST_F(HloComputationTest, DeepCopyArrayAtIndices) {
 }
 
 TEST_F(HloComputationTest, DeepCopyTupleAtIndices) {
-  // Test that DeepCopyInstruction properly copies elements of a a tuple as
+  // Test that DeepCopyInstruction properly copies elements of a tuple as
   // specified by the given indices.
   auto builder = HloComputation::Builder(TestName());
   auto constant1 = builder.AddInstruction(HloInstruction::CreateConstant(
diff --git a/tensorflow/contrib/all_reduce/python/all_reduce.py b/tensorflow/contrib/all_reduce/python/all_reduce.py
index 22d7633ce2..a5057da9fd 100644
--- a/tensorflow/contrib/all_reduce/python/all_reduce.py
+++ b/tensorflow/contrib/all_reduce/python/all_reduce.py
@@ -191,7 +191,7 @@ def _ragged_split(tensor, pieces):
 
 
 def _ring_permutations(num_workers, num_subchunks, gpu_perm):
-  """"Generate an array of device index arrays, one for for each subchunk.
+  """"Generate an array of device index arrays, one for each subchunk.
 
   In the basic ring reduction algorithm there are size(T)/num_devices
   data chunks and each device process one chunk per tick, i.e. sending
diff --git a/tensorflow/contrib/kfac/python/ops/loss_functions.py b/tensorflow/contrib/kfac/python/ops/loss_functions.py
index d80382b9cf..979a4fd1de 100644
--- a/tensorflow/contrib/kfac/python/ops/loss_functions.py
+++ b/tensorflow/contrib/kfac/python/ops/loss_functions.py
@@ -104,7 +104,7 @@ class LossFunction(object):
 
   @abc.abstractmethod
   def multiply_hessian_factor_transpose(self, vector):
-    """Right-multiply a vector by the tranpose of a factor B of the Hessian.
+    """Right-multiply a vector by the transpose of a factor B of the Hessian.
 
     Here the 'Hessian' is the Hessian matrix (i.e. matrix of 2nd-derivatives)
     of the loss function with respect to its inputs.  Typically this will be
@@ -218,7 +218,7 @@ class NegativeLogProbLoss(LossFunction):
 
   @abc.abstractmethod
   def multiply_fisher_factor_transpose(self, vector):
-    """Right-multiply a vector by the tranpose of a factor B of the Fisher.
+    """Right-multiply a vector by the transpose of a factor B of the Fisher.
 
     Here the 'Fisher' is the Fisher information matrix (i.e. expected outer-
     product of gradients) with respect to the parameters of the underlying
@@ -397,7 +397,7 @@ class NormalMeanVarianceNegativeLogProbLoss(DistributionNegativeLogProbLoss):
 
   This class parameterizes a multivariate normal distribution with n independent
   dimensions. Unlike `NormalMeanNegativeLogProbLoss`, this class does not
-  assume the variance is held constant. The Fisher Information for for n = 1
+  assume the variance is held constant. The Fisher Information for n = 1
   is given by,
 
   F = [[1 / variance,                0],
diff --git a/tensorflow/contrib/kfac/python/ops/op_queue.py b/tensorflow/contrib/kfac/python/ops/op_queue.py
index 0617c5be4d..831870fca4 100644
--- a/tensorflow/contrib/kfac/python/ops/op_queue.py
+++ b/tensorflow/contrib/kfac/python/ops/op_queue.py
@@ -61,7 +61,7 @@ class OpQueue(object):
       sess: tf.Session.
 
     Returns:
-      Next Op chosen from from 'ops'.
+      Next Op chosen from 'ops'.
     """
     # In Python 3, type(next_op_name) == bytes. Calling bytes.decode('ascii')
     # returns a str.
diff --git a/tensorflow/core/kernels/conv_ops_gpu_3.cu.cc b/tensorflow/core/kernels/conv_ops_gpu_3.cu.cc
index 6e10b53cf7..9a00a091bd 100644
--- a/tensorflow/core/kernels/conv_ops_gpu_3.cu.cc
+++ b/tensorflow/core/kernels/conv_ops_gpu_3.cu.cc
@@ -394,7 +394,7 @@ __global__ void SwapDimension1And2InTensor3SmallDim(const T* input,
     int output_block_idx = SmallDim2 ? block_offset : block_offset * small_dim;
     int output_block_origin_idx = output_block_offset + output_block_idx;
 
-    // Store the tranposed memory region in shared memory to device.
+    // Store the transposed memory region in shared memory to device.
     if (x < tile_height) {
       for (int y = 0; y < small_dim; y++) {
         int output_idx = output_block_origin_idx + x +
diff --git a/tensorflow/core/kernels/dataset.h b/tensorflow/core/kernels/dataset.h
index f9ffc4e065..9486f478b6 100644
--- a/tensorflow/core/kernels/dataset.h
+++ b/tensorflow/core/kernels/dataset.h
@@ -471,7 +471,7 @@ class DatasetIterator : public IteratorBase {
     // Owns one reference on the shared dataset resource.
     const DatasetType* dataset;
 
-    // Identifies the sequence of iterators leading up to to this iterator.
+    // Identifies the sequence of iterators leading up to this iterator.
     const string prefix;
   };
 
-- 
GitLab


From 610e1185da37ce8415db64ae60150f253b3fadc9 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 16 Oct 2017 06:07:14 -0700
Subject: [PATCH 415/909] - Modified Jacobian computations in
 CurvatureMatrixVectorProductComputer to use true partial derivatives.  This
 is done using the newly introduced stop_gradients argument to tf.gradients.

PiperOrigin-RevId: 172315620
---
 .../ops/curvature_matrix_vector_products.py   | 26 +++++++++----------
 tensorflow/contrib/kfac/python/ops/utils.py   |  5 ++--
 2 files changed, 16 insertions(+), 15 deletions(-)

diff --git a/tensorflow/contrib/kfac/python/ops/curvature_matrix_vector_products.py b/tensorflow/contrib/kfac/python/ops/curvature_matrix_vector_products.py
index bf59a92fa6..21b5cde9b9 100644
--- a/tensorflow/contrib/kfac/python/ops/curvature_matrix_vector_products.py
+++ b/tensorflow/contrib/kfac/python/ops/curvature_matrix_vector_products.py
@@ -36,13 +36,13 @@ class CurvatureMatrixVectorProductComputer(object):
   For example, the Fisher associated with a log-prob loss w.r.t. the
   parameters.
 
-  The vecs argument to each method are lists of tensors that must be the
+  The 'vecs' argument to each method are lists of tensors that must be the
   size as the corresponding ones from "wrt_tensors".  They represent
   the vector being multiplied.
 
   "factors" of the matrix M are defined as matrices B such that B*B^T = M.
-  Methods that multiply by the factor B take a "loss_inner_vecs" argument
-  instead of vecs, which must be a list of tensors with shapes given by the
+  Methods that multiply by the factor B take a 'loss_inner_vecs' argument
+  instead of 'vecs', which must be a list of tensors with shapes given by the
   corresponding XXX_inner_shapes property.
 
   Note that matrix-vector products are not normalized by the batch size, nor
@@ -61,7 +61,8 @@ class CurvatureMatrixVectorProductComputer(object):
     Args:
       losses: A list of LossFunction instances whose sum defines the total loss.
       wrt_tensors: A list of Tensors to compute the differential quantities
-        defining the matrices with respect to (see class description).
+        (defining the matrices) with respect to.  See class description for more
+        info.
     """
     self._losses = losses
     self._inputs_to_losses = list(loss.inputs for loss in losses)
@@ -73,24 +74,23 @@ class CurvatureMatrixVectorProductComputer(object):
     return math_ops.add_n(tuple(loss.evaluate() for loss in self._losses))
 
   # Jacobian multiplication functions:
-  # NOTE: These implementations use tf.gradients and thus aren't actually
-  # computing partial derivatives, but total derivatives instead (despite what
-  # the documentation for tf.gradients says).  Because we require partial
-  # derivatives for Jacobians this implementation will only be correct if the
-  # partial derivatives are equal to the full derivatives.  This happens as long
-  # as the elements of wrt_tensors don't depend on each other in the graph.  If
-  # these tensors are standard neural network parameters this will be true.
   def _multiply_jacobian(self, vecs):
     """Multiply vecs by the Jacobian of losses."""
+    # We stop gradients at wrt_tensors to produce partial derivatives (which is
+    # what we want for Jacobians).
     jacobian_vecs_flat = utils.fwd_gradients(
-        self._inputs_to_losses_flat, self._wrt_tensors, grad_xs=vecs)
+        self._inputs_to_losses_flat, self._wrt_tensors, grad_xs=vecs,
+        stop_gradients=self._wrt_tensors)
     return nest.pack_sequence_as(self._inputs_to_losses, jacobian_vecs_flat)
 
   def _multiply_jacobian_transpose(self, loss_vecs):
     """Multiply vecs by the transpose Jacobian of losses."""
     loss_vecs_flat = nest.flatten(loss_vecs)
+    # We stop gradients at wrt_tensors to produce partial derivatives (which is
+    # what we want for Jacobians).
     return gradients_impl.gradients(
-        self._inputs_to_losses_flat, self._wrt_tensors, grad_ys=loss_vecs_flat)
+        self._inputs_to_losses_flat, self._wrt_tensors, grad_ys=loss_vecs_flat,
+        stop_gradients=self._wrt_tensors)
 
   # Losses Fisher/Hessian multiplication functions:
   def _multiply_loss_fisher(self, loss_vecs):
diff --git a/tensorflow/contrib/kfac/python/ops/utils.py b/tensorflow/contrib/kfac/python/ops/utils.py
index b34b4e10ad..a7473481e4 100644
--- a/tensorflow/contrib/kfac/python/ops/utils.py
+++ b/tensorflow/contrib/kfac/python/ops/utils.py
@@ -250,7 +250,7 @@ def generate_random_signs(shape, dtype=dtypes.float32):
   return 2 * math_ops.cast(ints, dtype=dtype) - 1
 
 
-def fwd_gradients(ys, xs, grad_xs=None):
+def fwd_gradients(ys, xs, grad_xs=None, stop_gradients=None):
   """Compute forward-mode gradients."""
   # See b/37888268.
 
@@ -260,7 +260,8 @@ def fwd_gradients(ys, xs, grad_xs=None):
   # generated by the first gradients_impl.gradients call.
 
   us = [array_ops.zeros_like(y) + float("nan") for y in ys]
-  dydxs = gradients_impl.gradients(ys, xs, grad_ys=us)
+  dydxs = gradients_impl.gradients(ys, xs, grad_ys=us,
+                                   stop_gradients=stop_gradients)
 
   # Deal with strange types that gradients_impl.gradients returns but can't
   # deal with.
-- 
GitLab


From 28f0b54ad0009210a3734486f88804134f34abd0 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 16 Oct 2017 07:13:21 -0700
Subject: [PATCH 416/909] PiperOrigin-RevId: 172320984

---
 tensorflow/core/distributed_runtime/scheduler.cc | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tensorflow/core/distributed_runtime/scheduler.cc b/tensorflow/core/distributed_runtime/scheduler.cc
index 844a0643e6..4766f4c33b 100644
--- a/tensorflow/core/distributed_runtime/scheduler.cc
+++ b/tensorflow/core/distributed_runtime/scheduler.cc
@@ -226,7 +226,6 @@ Microseconds GreedyScheduler::ComputeSchedule(
   while (!event_queue.empty()) {
     Event event = event_queue.top();
     event_queue.pop();
-    Microseconds curr_time;
     if (event.is_completion) {
       Sim* sim = device_states_[event.node->assigned_device_name()];
       --sim->num_running;
-- 
GitLab


From bba2bc1c6c6a47c2b9c0889d4aa2628a0cdf6c92 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 16 Oct 2017 07:52:10 -0700
Subject: [PATCH 417/909] Fixing comment mismatch.

PiperOrigin-RevId: 172324333
---
 tensorflow/contrib/boosted_trees/examples/binary_mnist.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/boosted_trees/examples/binary_mnist.py b/tensorflow/contrib/boosted_trees/examples/binary_mnist.py
index 9be362f5c8..c003b1de66 100644
--- a/tensorflow/contrib/boosted_trees/examples/binary_mnist.py
+++ b/tensorflow/contrib/boosted_trees/examples/binary_mnist.py
@@ -52,7 +52,7 @@ def get_input_fn(data,
   ids = np.where((data.labels == 4) | (data.labels == 9))
   images = data.images[ids]
   labels = data.labels[ids]
-  # Make digit 4 label 0, 9 is 1.
+  # Make digit 4 label 1, 9 is 0.
   labels = labels == 4
 
   def _input_fn():
-- 
GitLab


From a799ade213cecb3c1c1d19eca6a0bfa3fddf0113 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 16 Oct 2017 08:04:40 -0700
Subject: [PATCH 418/909] Automated g4 rollback of changelist 171877766

PiperOrigin-RevId: 172325692
---
 tensorflow/compiler/xla/service/cpu/BUILD     |  18 ++
 .../compiler/xla/service/cpu/cpu_compiler.cc  |  15 +-
 .../compiler/xla/service/cpu/cpu_compiler.h   |   2 +-
 .../cpu/cpu_parallelization_preparation.cc    |  20 --
 .../compiler/xla/service/cpu/cpu_runtime.cc   |   3 +
 .../compiler/xla/service/cpu/cpu_runtime.h    |   1 +
 .../compiler/xla/service/cpu/ir_emitter.cc    | 192 +++++++++++++++---
 .../compiler/xla/service/cpu/ir_emitter.h     |  21 +-
 .../service/cpu/parallel_task_assignment.cc   | 148 ++++++++++++--
 .../service/cpu/parallel_task_assignment.h    |  49 +++++
 .../xla/service/cpu/runtime_fork_join.cc      |  97 +++++++++
 .../xla/service/cpu/runtime_fork_join.h       |  33 +++
 .../xla/service/cpu/simple_orc_jit.cc         |   2 +
 tensorflow/compiler/xla/tests/BUILD           |   2 +
 tensorflow/compiler/xla/tests/fusion_test.cc  | 136 +++++++++++--
 15 files changed, 648 insertions(+), 91 deletions(-)
 create mode 100644 tensorflow/compiler/xla/service/cpu/runtime_fork_join.cc
 create mode 100644 tensorflow/compiler/xla/service/cpu/runtime_fork_join.h

diff --git a/tensorflow/compiler/xla/service/cpu/BUILD b/tensorflow/compiler/xla/service/cpu/BUILD
index 8ab358fe17..c71eca0d39 100644
--- a/tensorflow/compiler/xla/service/cpu/BUILD
+++ b/tensorflow/compiler/xla/service/cpu/BUILD
@@ -87,6 +87,7 @@ cc_library(
         ":ir_emitter",
         ":layout_assignment",
         ":parallel_cpu_executable",
+        ":parallel_task_assignment",
         ":simple_orc_jit",
         "//tensorflow/compiler/xla:literal_util",
         "//tensorflow/compiler/xla:protobuf_util",
@@ -155,6 +156,7 @@ cc_library(
         ":disassembler",
         ":external_constant_pool",
         ":runtime_conv2d",
+        ":runtime_fork_join",
         ":runtime_matmul",
         ":runtime_single_threaded_conv2d",
         ":runtime_single_threaded_matmul",
@@ -243,6 +245,7 @@ cc_library(
         ":dot_op_emitter",
         ":external_constant_pool",
         ":ir_emission_utils",
+        ":shape_partition",
         ":simple_orc_jit",
         "//tensorflow/compiler/xla:shape_util",
         "//tensorflow/compiler/xla:status_macros",
@@ -505,6 +508,20 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "runtime_fork_join",
+    srcs = ["runtime_fork_join.cc"],
+    hdrs = ["runtime_fork_join.h"],
+    copts = runtime_copts(),
+    visibility = ["//visibility:public"],
+    deps = [
+        "//tensorflow/compiler/xla:executable_run_options",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:lib_internal",
+        "//third_party/eigen3",
+    ],
+)
+
 tf_cc_test(
     name = "cpu_runtime_test",
     srcs = ["cpu_runtime_test.cc"],
@@ -688,6 +705,7 @@ cc_library(
         ":shape_partition",
         "//tensorflow/compiler/xla/service:hlo",
         "//tensorflow/compiler/xla/service:hlo_cost_analysis",
+        "//tensorflow/compiler/xla/service:hlo_pass",
     ],
 )
 
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
index 1437fb4cf9..ce4d109214 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
@@ -58,6 +58,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/cpu/ir_emitter.h"
 #include "tensorflow/compiler/xla/service/cpu/layout_assignment.h"
 #include "tensorflow/compiler/xla/service/cpu/parallel_cpu_executable.h"
+#include "tensorflow/compiler/xla/service/cpu/parallel_task_assignment.h"
 #include "tensorflow/compiler/xla/service/cpu/simple_orc_jit.h"
 #include "tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h"
 #include "tensorflow/compiler/xla/service/flatten_call_graph.h"
@@ -248,7 +249,7 @@ class CollectProfileCandidates : public DfsHloVisitorWithDefault {
 };
 }  // namespace
 
-Status CpuCompiler::RunHloPasses(HloModule* module) {
+Status CpuCompiler::RunHloPasses(HloModule* module, bool is_aot_compile) {
   // Optimization pipeline.
   HloPassPipeline pipeline("CPU");
   pipeline.AddInvariantChecker<HloVerifier>(ShapeSizeBytesFunction());
@@ -316,6 +317,14 @@ Status CpuCompiler::RunHloPasses(HloModule* module) {
   if (options::CpuParallelBackendRequested(module->config())) {
     pipeline.AddPass<ParallelizationPreparation>(max_parallelism,
                                                  ShapeSizeBytesFunction());
+  } else if (!is_aot_compile) {
+    // Run ParallelTaskAssigner to assign parallel tasks to HLOs in module.
+    // Note this is not run for AOT because it would bring in thread pool
+    // and thread synchronization dependencies which would likely increase
+    // binary size (and most AOT applications are single-threaded).
+    // TODO(29630486) Support multi-threaded AOT.
+    pipeline.AddPass<ParallelTaskAssigner>(max_parallelism,
+                                           ShapeSizeBytesFunction(), module);
   }
   // Copy insertion should be performed immediately before IR emission to avoid
   // inserting unnecessary copies (later pass adds an instruction which
@@ -450,7 +459,7 @@ StatusOr<std::unique_ptr<Executable>> CpuCompiler::Compile(
   llvm_module->setDataLayout(jit->data_layout());
   llvm_module->setTargetTriple(jit->target_triple().getTriple());
 
-  TF_RETURN_IF_ERROR(RunHloPasses(module.get()));
+  TF_RETURN_IF_ERROR(RunHloPasses(module.get(), /*is_aot_compile=*/false));
 
   HloComputation* computation = module->entry_computation();
   std::unordered_map<const HloInstruction*, size_t> hlo_to_profile_idx;
@@ -749,7 +758,7 @@ CpuCompiler::CompileAheadOfTime(std::vector<std::unique_ptr<HloModule>> modules,
     HloModule* module = modules[i].get();
     VLOG(1) << "Compiling ahead-of-time: " << module->name();
 
-    TF_RETURN_IF_ERROR(RunHloPasses(module));
+    TF_RETURN_IF_ERROR(RunHloPasses(module, /*is_aot_compile=*/true));
 
     TF_ASSIGN_OR_RETURN(
         SequentialHloOrdering::HloModuleSequence module_sequence,
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_compiler.h b/tensorflow/compiler/xla/service/cpu/cpu_compiler.h
index a301d04337..d091302474 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_compiler.h
+++ b/tensorflow/compiler/xla/service/cpu/cpu_compiler.h
@@ -132,7 +132,7 @@ class CpuCompiler : public LLVMCompiler {
 
   // Runs the HLO passes which are necessary for both optimizations and
   // correctness.
-  Status RunHloPasses(HloModule* module);
+  Status RunHloPasses(HloModule* module, bool is_aot_compile);
 
   TF_DISALLOW_COPY_AND_ASSIGN(CpuCompiler);
 };
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_parallelization_preparation.cc b/tensorflow/compiler/xla/service/cpu/cpu_parallelization_preparation.cc
index 2cd0aa7880..662ee60923 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_parallelization_preparation.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_parallelization_preparation.cc
@@ -116,26 +116,6 @@ StatusOr<bool> ParallelizationPreparation::RunParallelTaskAssignment(
   // Assign parallel tasks to HLOs in entry computation.
   HloComputation* computation = module->entry_computation();
   for (auto* instruction : computation->instructions()) {
-    // Currently, we do not assign parallel tasks to instructions with at least
-    // one of the following properties:
-    // *) Internal threading (library calls to kConv, kDot, and kCustomCall).
-    // *) Emit custom loops (kSelectAndScatter, FusionKind::kTransposeDot).
-    // *) Tuple-shaped.
-    // TODO(b/27458679) Parallelize instructions which are skipped here.
-    if (instruction->opcode() == HloOpcode::kParameter ||
-        instruction->opcode() == HloOpcode::kConstant ||
-        instruction->opcode() == HloOpcode::kCall ||
-        instruction->opcode() == HloOpcode::kCustomCall ||
-        instruction->opcode() == HloOpcode::kSelectAndScatter ||
-        (instruction->opcode() == HloOpcode::kConvolution &&
-         PotentiallyImplementedAsEigenConvolution(*instruction)) ||
-        PotentiallyImplementedAsEigenDot(*instruction) ||
-        (instruction->opcode() == HloOpcode::kFusion &&
-         instruction->fusion_kind() != HloInstruction::FusionKind::kLoop) ||
-        ShapeUtil::IsTuple(instruction->shape())) {
-      continue;
-    }
-
     // Calculate target parallel task count in [1, max_parallelism_].
     const int64 target_parallel_task_count =
         parallel_task_assignment.GetTargetParallelTaskCount(instruction);
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_runtime.cc b/tensorflow/compiler/xla/service/cpu/cpu_runtime.cc
index c7155b858b..7908dc173d 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_runtime.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_runtime.cc
@@ -51,6 +51,9 @@ extern const char* const kAcquireOutfeedBufferForPopulationSymbolName =
     "__xla_cpu_runtime_AcquireOutfeedBufferForPopulation";
 extern const char* const kReleaseOutfeedBufferAfterPopulationSymbolName =
     "__xla_cpu_runtime_ReleaseOutfeedBufferAfterPopulation";
+extern const char* const kParallelForkJoinSymbolName =
+    "__xla_cpu_runtime_ParallelForkJoin";
+
 extern const char* const kXlaCpuRuntimeSymbolNamePrefix = "__xla_cpu_runtime_";
 }  // namespace runtime
 }  // namespace cpu
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_runtime.h b/tensorflow/compiler/xla/service/cpu/cpu_runtime.h
index 29feb7267f..2ade455b8a 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_runtime.h
+++ b/tensorflow/compiler/xla/service/cpu/cpu_runtime.h
@@ -51,6 +51,7 @@ extern const char* const kAcquireInfeedBufferForDequeueSymbolName;
 extern const char* const kReleaseInfeedBufferAfterDequeueSymbolName;
 extern const char* const kAcquireOutfeedBufferForPopulationSymbolName;
 extern const char* const kReleaseOutfeedBufferAfterPopulationSymbolName;
+extern const char* const kParallelForkJoinSymbolName;
 
 // All symbol names for XLA CPU runtime functions need to start with this
 // prefix.
diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
index 3d2d0f1029..52085d1376 100644
--- a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
@@ -42,6 +42,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/cpu/dot_op_emitter.h"
 #include "tensorflow/compiler/xla/service/cpu/elemental_ir_emitter.h"
 #include "tensorflow/compiler/xla/service/cpu/ir_emission_utils.h"
+#include "tensorflow/compiler/xla/service/cpu/shape_partition.h"
 #include "tensorflow/compiler/xla/service/cpu/simple_orc_jit.h"
 #include "tensorflow/compiler/xla/service/elemental_ir_emitter.h"
 #include "tensorflow/compiler/xla/service/hlo_opcode.h"
@@ -186,20 +187,9 @@ void IrEmitter::InitializeIrFunction(const string& function_name) {
   // Even though the type of params and temps is void** in the host's view, in
   // LLVM IR this is represented by i8*, similarly to void*. It's up to the code
   // to use GEPs to unravel the indirection layers.
-  llvm::Type* i8_ptr_type = llvm::Type::getInt8PtrTy(module_->getContext());
-  llvm::Type* i8_ptr_ptr_type = i8_ptr_type->getPointerTo();
-  llvm::Type* i64_ptr_type = llvm::Type::getInt64PtrTy(module_->getContext());
-  std::vector<llvm::Type*> compute_function_params(
-      {i8_ptr_type, i8_ptr_type, i8_ptr_ptr_type, i8_ptr_ptr_type});
-  if (IsParallelContext()) {
-    compute_function_params.push_back(i64_ptr_type);
-  }
-  if (hlo_to_profile_idx_) {
-    compute_function_params.push_back(i64_ptr_type);
-  }
   llvm::FunctionType* compute_function_type = llvm::FunctionType::get(
       /*Result=*/llvm::Type::getVoidTy(module_->getContext()),
-      /*Params=*/compute_function_params,
+      /*Params=*/GetComputeFunctionParams(),
       /*isVarArg=*/false);
 
   // Functions with local linkage get an inlining bonus.  Because we know
@@ -221,7 +211,7 @@ void IrEmitter::InitializeIrFunction(const string& function_name) {
   (++arg_iter)->setName("run_options");
   (++arg_iter)->setName("params");
   (++arg_iter)->setName("temps");
-  if (IsParallelContext()) {
+  if (num_dynamic_loop_bounds_ > 0) {
     (++arg_iter)->setName("dynamic_loop_bounds");
   }
   if (hlo_to_profile_idx_) {
@@ -2288,8 +2278,19 @@ Status IrEmitter::HandleCall(HloInstruction* call) {
   }
 
   TF_RETURN_IF_ERROR(EmitTargetAddressForOp(call));
-  EmitArrayFunctionCallInto(call_ir_function, parameter_addresses,
-                            emitted_value_[call], computation->name());
+
+  if (!computation->root_instruction()->outer_dimension_partitions().empty() &&
+      !parallel_cpu_backend_) {
+    // ParallelTaskAssignment assigned partitions, emit call to
+    // ParallelForkJoin.
+    TF_RETURN_IF_ERROR(EmitParallelForkJoin(parameter_addresses,
+                                            emitted_value_[call], computation,
+                                            call_ir_function));
+  } else {
+    EmitArrayFunctionCallInto(call_ir_function, parameter_addresses,
+                              emitted_value_[call], computation->name());
+  }
+
   return Status::OK();
 }
 
@@ -2599,7 +2600,7 @@ Status IrEmitter::FinishVisit(HloInstruction* root) {
   // For the parallel cpu backend, we record the total for each embedded
   // computation callee with its caller kCall HLO.
   HloInstruction* hlo_to_lookup = nullptr;
-  if (IsParallelContext()) {
+  if (parallel_cpu_backend_ && is_top_level_computation_) {
     auto* computation = root->parent();
     auto* entry_computation = computation->parent()->entry_computation();
     if (computation != entry_computation) {
@@ -2757,12 +2758,27 @@ llvm::Type* IrEmitter::IrShapeType(const Shape& shape) {
   return llvm_ir::ShapeToIrType(shape, &ir_builder_);
 }
 
+std::vector<llvm::Type*> IrEmitter::GetComputeFunctionParams() {
+  llvm::Type* i8_ptr_type = llvm::Type::getInt8PtrTy(module_->getContext());
+  llvm::Type* i8_ptr_ptr_type = i8_ptr_type->getPointerTo();
+  llvm::Type* i64_ptr_type = llvm::Type::getInt64PtrTy(module_->getContext());
+  std::vector<llvm::Type*> compute_function_params(
+      {i8_ptr_type, i8_ptr_type, i8_ptr_ptr_type, i8_ptr_ptr_type});
+  if (num_dynamic_loop_bounds_ > 0) {
+    compute_function_params.push_back(i64_ptr_type);
+  }
+  if (hlo_to_profile_idx_) {
+    compute_function_params.push_back(i64_ptr_type);
+  }
+  return compute_function_params;
+}
+
 llvm::Argument* IrEmitter::GetResultArgument() {
   return GetArg(compute_function_, 0);
 }
 
 llvm::Argument* IrEmitter::GetProfileCountersArgument() {
-  const int64 arg_index = IsParallelContext() ? 5 : 4;
+  const int64 arg_index = num_dynamic_loop_bounds_ > 0 ? 5 : 4;
   return hlo_to_profile_idx_ ? GetArg(compute_function_, arg_index) : nullptr;
 }
 
@@ -2845,18 +2861,11 @@ llvm::Value* IrEmitter::EmitElementFunctionCall(
       AsStringRef(tensorflow::strings::StrCat(name, "_return_value")));
 }
 
-// Emits a core function call based on the following pseudo-code.
-//
-//   char** parameter_addresses_buffer =
-//       allocate buffer with a pointer for each parameter to the function
-//   for each parameter index, i.e. for i = 0, ..., #parameters:
-//     parameter_addresses_buffer[i] = parameter_addresses[i]
-//   call function(return_value_buffer,
-//                 parameter_addresses_buffer,
-//                 temps)
-//   return return_value_buffer  -- address of the return value.
-void IrEmitter::EmitArrayFunctionCallInto(
-    llvm::Function* function,
+// Emits code to allocate an array of parameter address pointers, and store
+// each address from 'parameter_addresses'.
+// Returns an array of compute function call arguments (including parameter
+// address buffer).
+std::vector<llvm::Value*> IrEmitter::GetArrayFunctionCallArguments(
     tensorflow::gtl::ArraySlice<llvm::Value*> parameter_addresses,
     llvm::Value* return_value_buffer, tensorflow::StringPiece name) {
   llvm::Value* parameter_addresses_buffer =
@@ -2885,7 +2894,26 @@ void IrEmitter::EmitArrayFunctionCallInto(
   if (auto* profile_counters = GetProfileCountersArgument()) {
     arguments.push_back(profile_counters);
   }
-  ir_builder_.CreateCall(function, arguments);
+  return arguments;
+}
+
+// Emits a core function call based on the following pseudo-code.
+//
+//   char** parameter_addresses_buffer =
+//       allocate buffer with a pointer for each parameter to the function
+//   for each parameter index, i.e. for i = 0, ..., #parameters:
+//     parameter_addresses_buffer[i] = parameter_addresses[i]
+//   call function(return_value_buffer,
+//                 parameter_addresses_buffer,
+//                 temps)
+//   return return_value_buffer  -- address of the return value.
+void IrEmitter::EmitArrayFunctionCallInto(
+    llvm::Function* function,
+    tensorflow::gtl::ArraySlice<llvm::Value*> parameter_addresses,
+    llvm::Value* return_value_buffer, tensorflow::StringPiece name) {
+  ir_builder_.CreateCall(
+      function, GetArrayFunctionCallArguments(parameter_addresses,
+                                              return_value_buffer, name));
 }
 
 llvm::Value* IrEmitter::EmitArrayFunctionCall(
@@ -2905,6 +2933,110 @@ llvm::Value* IrEmitter::EmitArrayFunctionCall(
   return return_value_buffer;
 }
 
+// Emits a call to a runtime fork/join function which dispatches parallel
+// calls to 'parallel_function' (and joins threads before returning).
+Status IrEmitter::EmitParallelForkJoin(
+    tensorflow::gtl::ArraySlice<llvm::Value*> parameter_addresses,
+    llvm::Value* output_address, HloComputation* computation,
+    llvm::Function* parallel_function) {
+  HloInstruction* root = computation->root_instruction();
+
+  // Build ParallelForkJoin function type.
+  std::vector<llvm::Type*> compute_function_params = GetComputeFunctionParams();
+  // Number of parallel compute functions.
+  compute_function_params.push_back(ir_builder_.getInt32Ty());
+  // Array of partitions. There is an array element for each
+  // partition x partition_dim x 2 (for dimension start and limit).
+  compute_function_params.push_back(
+      llvm::Type::getInt64PtrTy(module_->getContext()));
+  // Number of partitioned most-major dimensions in 'root.shape'.
+  compute_function_params.push_back(ir_builder_.getInt32Ty());
+  // Function pointer for compute function to be dispatched in parallel.
+  compute_function_params.push_back(
+      llvm::Type::getInt8PtrTy(module_->getContext()));
+
+  llvm::FunctionType* fork_join_type = llvm::FunctionType::get(
+      /*Result=*/llvm::Type::getVoidTy(module_->getContext()),
+      /*Params=*/compute_function_params,
+      /*isVarArg=*/false);
+
+  llvm::Function* fork_join_func =
+      llvm::cast<llvm::Function>(module_->getOrInsertFunction(
+          runtime::kParallelForkJoinSymbolName, fork_join_type));
+  fork_join_func->setCallingConv(llvm::CallingConv::C);
+  fork_join_func->setDoesNotThrow();
+
+  // Add common compute function arguments.
+  const string name = computation->name();
+  std::vector<llvm::Value*> arguments =
+      GetArrayFunctionCallArguments(parameter_addresses, output_address, name);
+
+  // Create ShapePartitionIterator to generate all partitions of 'root.shape'.
+  ShapePartitionIterator partition_iterator(root->shape(),
+                                            root->outer_dimension_partitions());
+  const int64 num_partitions = partition_iterator.GetTotalPartitionCount();
+  // Add argument specifying the number of parallel partitions.
+  arguments.push_back(ir_builder_.getInt32(num_partitions));
+
+  // The number of partitioned most-major dimensions in 'root.shape'.
+  const int32 num_partitioned_dims = root->outer_dimension_partitions().size();
+  // A dimension partition consists of two elements: [start_index, limit_index).
+  const int32 dim_partition_size = 2;
+  // Calculate array partition stride.
+  const int32 array_partition_stride =
+      num_partitioned_dims * dim_partition_size;
+  // Calculate the total number of elements in the partition array.
+  const int32 partition_array_size =
+      dim_partition_size * num_partitioned_dims * num_partitions;
+
+  // Store dimension partition values as llvm constants in 'partitions'.
+  // See comments in runtime_fork_join.cc for array layout description.
+  std::vector<llvm::Constant*> partitions(partition_array_size);
+  for (int32 i = 0; i < num_partitions; ++i) {
+    std::vector<std::pair<int64, int64>> dim_partitions =
+        partition_iterator.GetPartition(i);
+    CHECK_EQ(num_partitioned_dims, dim_partitions.size());
+    const int32 partition_index = i * array_partition_stride;
+    for (int32 j = 0; j < num_partitioned_dims; ++j) {
+      const std::pair<int64, int64>& dim_partition = dim_partitions[j];
+      const int32 index = partition_index + j * dim_partition_size;
+      // Store partition [dim_start, dim_limit) intervals for each dimension.
+      partitions[index] = ir_builder_.getInt64(dim_partition.first);
+      partitions[index + 1] =
+          ir_builder_.getInt64(dim_partition.first + dim_partition.second);
+    }
+  }
+
+  // Create global variable out of dimension partitions in 'partitions'.
+  llvm::ArrayType* partitions_array_type =
+      llvm::ArrayType::get(ir_builder_.getInt64Ty(), partition_array_size);
+  llvm::Constant* partitions_array =
+      llvm::ConstantArray::get(partitions_array_type, partitions);
+  llvm::GlobalVariable* global_partitions_array = new llvm::GlobalVariable(
+      /*Module=*/*module_,
+      /*Type=*/partitions_array_type,
+      /*isConstant=*/true,
+      /*Linkage=*/llvm::GlobalValue::PrivateLinkage,
+      /*Initializer=*/partitions_array,
+      /*Name=*/
+      AsStringRef(
+          tensorflow::strings::StrCat(name, "_parallel_dimension_partitions")));
+
+  // Add argument specifying parallel dimension partitions.
+  arguments.push_back(ir_builder_.CreateBitCast(
+      global_partitions_array,
+      llvm::Type::getInt64PtrTy(module_->getContext())));
+  // Add argument specifying the number of partitioned most-major dimensions.
+  arguments.push_back(ir_builder_.getInt32(num_partitioned_dims));
+  // Add argument for parallel compute function pointer.
+  arguments.push_back(
+      ir_builder_.CreateBitCast(parallel_function, ir_builder_.getInt8PtrTy()));
+  // Emit call to parallel fork/join.
+  ir_builder_.CreateCall(fork_join_func, arguments);
+
+  return Status::OK();
+}
+
 Status IrEmitter::EmitTargetAddressForOp(const HloInstruction* op) {
   llvm::Value* addr;
   const Shape& target_shape = op->shape();
diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.h b/tensorflow/compiler/xla/service/cpu/ir_emitter.h
index 53c4b6f241..58c185af1e 100644
--- a/tensorflow/compiler/xla/service/cpu/ir_emitter.h
+++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.h
@@ -249,6 +249,9 @@ class IrEmitter : public DfsHloVisitorWithDefault {
   // Convenience function to get the IR type matching the given shape.
   llvm::Type* IrShapeType(const Shape& shape);
 
+  // Returns an array of compute function parameter types.
+  std::vector<llvm::Type*> GetComputeFunctionParams();
+
   // Get the llvm::Value* that represents the "retval" argument of the
   // computation function being emitted by this emitter.
   llvm::Argument* GetResultArgument();
@@ -323,6 +326,18 @@ class IrEmitter : public DfsHloVisitorWithDefault {
       tensorflow::gtl::ArraySlice<llvm::Value*> parameter_addresses,
       tensorflow::StringPiece name);
 
+  // Returns an array of compute function call arguments.
+  std::vector<llvm::Value*> GetArrayFunctionCallArguments(
+      tensorflow::gtl::ArraySlice<llvm::Value*> parameter_addresses,
+      llvm::Value* return_value_buffer, tensorflow::StringPiece name);
+
+  // Emits a call to a runtime fork/join function which dispatches parallel
+  // calls to 'parallel_function' (and joins threads before returning).
+  Status EmitParallelForkJoin(
+      tensorflow::gtl::ArraySlice<llvm::Value*> parameter_addresses,
+      llvm::Value* output_address, HloComputation* computation,
+      llvm::Function* parallel_function);
+
   // Verifies that the element types of all of the given operand instructions
   // match and are of one of the given supported types.
   Status ElementTypesSameAndSupported(
@@ -596,12 +611,6 @@ class IrEmitter : public DfsHloVisitorWithDefault {
   Status EmitXfeedTransfer(XfeedKind kind, const Shape& shape,
                            llvm::Value* program_buffer_address);
 
-  // Returns true if the current function being emitted is called in a
-  // parallel context (returns false otherwise).
-  bool IsParallelContext() {
-    return parallel_cpu_backend_ && is_top_level_computation_;
-  }
-
   const HloModuleConfig& hlo_module_config_;
 
   const bool parallel_cpu_backend_;
diff --git a/tensorflow/compiler/xla/service/cpu/parallel_task_assignment.cc b/tensorflow/compiler/xla/service/cpu/parallel_task_assignment.cc
index d4b5e41f50..5afb2e67ff 100644
--- a/tensorflow/compiler/xla/service/cpu/parallel_task_assignment.cc
+++ b/tensorflow/compiler/xla/service/cpu/parallel_task_assignment.cc
@@ -48,29 +48,56 @@ class SimpleCostModel : public ParallelCostModel {
 class DefaultCostModel : public ParallelCostModel {
  public:
   DefaultCostModel(const int64 max_parallelism,
+                   const HloCostAnalysis::ShapeSizeFunction& shape_size,
                    std::unique_ptr<HloCostAnalysis> cost_analysis)
       : max_parallelism_(max_parallelism),
+        shape_size_(shape_size),
         cost_analysis_(std::move(cost_analysis)) {}
   ~DefaultCostModel() override {}
 
   int64 GetParallelTaskCount(HloInstruction* instruction) override {
-    // Calculate the instruction cost in cycles.
-    // TODO(29630486) Improve on this linear cost model.
-    // Consider making 'min_cost_per_thread' be a function of the target
-    // bandwidth limit for instructions with low arithmetic complexity.
-    const int64 instruction_cost =
-        1 * cost_analysis_->flop_count(*instruction) +
-        2 * cost_analysis_->transcendental_count(*instruction) +
-        10 * cost_analysis_->bytes_accessed(*instruction);
-    // Minimum per-thread cost is 100us of work on a 2GHz core.
-    const int64 min_cost_per_thread = 100000;
+    // Parameters for parallel task count computation.
+    int64 instruction_cost;
+    int64 min_cost_per_thread;
+    int64 max_parallelism;
+    // Calculate flops-to-bytes-ratio for 'instruction'.
+    const int64 bytes_accessed =
+        std::max(1LL, cost_analysis_->bytes_accessed(*instruction));
+    const float flops_to_bytes_ratio =
+        cost_analysis_->flop_count(*instruction) /
+        static_cast<float>(bytes_accessed);
+    // Check for I/O bound instructions.
+    if (flops_to_bytes_ratio <= 1.0) {
+      // Limit max parallelism for I/O bound instructions by assuming a
+      // sub-linear scaling function (fit based on empirical benchmark results).
+      // TODO(29630486) Develop system bandwidth model.
+      max_parallelism =
+          std::ceil(std::sqrt(tensorflow::port::NumSchedulableCPUs()));
+      // Use shape size instruction cost and L2 cache size min per-thread cost.
+      instruction_cost = shape_size_(instruction->shape());
+      min_cost_per_thread = 256LL << 10;  // 256KB L2 Cache size.
+    } else {
+      // Use max parallelism for compute bound instructions.
+      max_parallelism = max_parallelism_;
+      // Calculate the instruction cost in cycles.
+      // TODO(29630486) Improve on this linear cost model.
+      // Consider making 'min_cost_per_thread' be a function of the target
+      // bandwidth limit for instructions with low arithmetic complexity.
+      instruction_cost =
+          1 * cost_analysis_->flop_count(*instruction) +
+          2 * cost_analysis_->transcendental_count(*instruction) +
+          10 * cost_analysis_->bytes_accessed(*instruction);
+      // Minimum per-thread cost is 100us of work on a 2GHz core.
+      min_cost_per_thread = 100000;
+    }
     // Return target parallel task count in [1, max_parallelism_].
-    return std::min(max_parallelism_,
+    return std::min(max_parallelism,
                     std::max(1LL, instruction_cost / min_cost_per_thread));
   }
 
  private:
   const int64 max_parallelism_;
+  const HloCostAnalysis::ShapeSizeFunction shape_size_;
   const std::unique_ptr<HloCostAnalysis> cost_analysis_;
 };
 
@@ -86,7 +113,7 @@ ParallelTaskAssignment::ParallelTaskAssignment(
   Status status = computation->root_instruction()->Accept(cost_analysis.get());
   if (status.ok()) {
     // Set default cost model based on 'cost_analysis'.
-    cost_model_.reset(new DefaultCostModel(max_parallelism,
+    cost_model_.reset(new DefaultCostModel(max_parallelism, shape_size,
                                            std::move(cost_analysis)));
   } else {
     // Fall back to a simple cost model based on hlo size and L2 cache size.
@@ -121,5 +148,102 @@ int64 ParallelTaskAssignment::GetTargetParallelTaskCount(
   return cost_model_->GetParallelTaskCount(instruction);
 }
 
+StatusOr<bool> ParallelTaskAssigner::Run(HloModule* module) {
+  XLA_VLOG_LINES(2, "ParallelTaskAssigner ENTRY");
+  XLA_VLOG_LINES(3, module->ToString());
+
+  // Compute target parallel task counts for all instructions in 'module'.
+  HloToParallelTasks hlo_to_parallel_tasks;
+  ComputeTargetParallelTasks(module, &hlo_to_parallel_tasks);
+
+  // Assign parallel tasks to target specific instructions in 'module'.
+  // TODO(b/27458679) Support inter-op parallelism.
+  bool changed = AssignParallelTasks(module, hlo_to_parallel_tasks);
+
+  XLA_VLOG_LINES(2, "ParallelTaskAssigner EXIT");
+  XLA_VLOG_LINES(3, module->ToString());
+  return changed;
+}
+
+bool ParallelTaskAssigner::AssignParallelTasks(
+    HloModule* module, const HloToParallelTasks& hlo_to_parallel_tasks) {
+  return AssignParallelTasksHelper(module, module->entry_computation(),
+                                   hlo_to_parallel_tasks);
+}
+
+bool ParallelTaskAssigner::AssignParallelTasksHelper(
+    HloModule* module, HloComputation* computation,
+    const HloToParallelTasks& hlo_to_parallel_tasks) {
+  bool changed = false;
+  // Snapshot set of instructions because outlining modifies the set below.
+  std::vector<HloInstruction*> instructions(computation->instructions().begin(),
+                                            computation->instructions().end());
+  for (auto* instruction : instructions) {
+    // Assign parallel tasks to sub-computations for While and Call HLOs.
+    // TODO(b/27458679) Evaluate alternative intra-op parallelsim placement,
+    // and support other callable computations like reduce.
+    if (instruction->opcode() == HloOpcode::kWhile) {
+      changed |= AssignParallelTasksHelper(module, instruction->while_body(),
+                                           hlo_to_parallel_tasks);
+      continue;
+    } else if (instruction->opcode() == HloOpcode::kCall) {
+      changed |= AssignParallelTasksHelper(module, instruction->to_apply(),
+                                           hlo_to_parallel_tasks);
+      continue;
+    }
+    // Skip if no parallel tasks were computed in first pass.
+    auto it = hlo_to_parallel_tasks.find(instruction);
+    if (it == hlo_to_parallel_tasks.end()) {
+      continue;
+    }
+    // Get target parallel task count computed for 'instruction'.
+    const int64 target_parallel_task_count = (*it).second;
+    // Assign feasible dimension partitions (based on actual dimension sizes).
+    auto dim_partition_counts = ShapePartitionAssigner(instruction->shape())
+                                    .Run(target_parallel_task_count);
+    const int64 total_partition_count =
+        ShapePartitionAssigner::GetTotalPartitionCount(dim_partition_counts);
+    if (total_partition_count <= 1) {
+      // Feasible partition calculation resulting in no partitioning, so skip.
+      continue;
+    }
+
+    // Outline 'instruction' in 'computation' for parallel task assignment.
+    auto* call = module->OutlineExpressionFromComputation(
+        {instruction},
+        tensorflow::strings::StrCat("parallel_", instruction->name()),
+        computation);
+
+    // Set assigned dimension partitioning to 'instruction'.
+    auto* new_root = call->to_apply()->root_instruction();
+    new_root->set_outer_dimension_partitions(dim_partition_counts);
+
+    VLOG(2) << "Assigned parallel task count: " << total_partition_count
+            << " to instruction: " << new_root->name()
+            << " parent: " << new_root->parent()->name();
+    changed = true;
+  }
+  return changed;
+}
+
+void ParallelTaskAssigner::ComputeTargetParallelTasks(
+    HloModule* module, HloToParallelTasks* hlo_to_parallel_tasks) {
+  // Compute parallel task counts for all instructions in 'module'.
+  for (auto* computation : module->computations()) {
+    if (computation->IsFusionComputation()) {
+      continue;
+    }
+    for (auto* instruction : computation->instructions()) {
+      // Query ParallelTaskAssignment for target parallel task count.
+      const int64 target_parallel_task_count =
+          parallel_task_assignment_.GetTargetParallelTaskCount(instruction);
+      if (target_parallel_task_count > 1) {
+        hlo_to_parallel_tasks->insert(
+            {instruction, target_parallel_task_count});
+      }
+    }
+  }
+}
+
 }  // namespace cpu
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/service/cpu/parallel_task_assignment.h b/tensorflow/compiler/xla/service/cpu/parallel_task_assignment.h
index 15f065a3ad..e036da5784 100644
--- a/tensorflow/compiler/xla/service/cpu/parallel_task_assignment.h
+++ b/tensorflow/compiler/xla/service/cpu/parallel_task_assignment.h
@@ -18,6 +18,7 @@ limitations under the License.
 
 #include "tensorflow/compiler/xla/service/hlo_cost_analysis.h"
 #include "tensorflow/compiler/xla/service/hlo_module.h"
+#include "tensorflow/compiler/xla/service/hlo_pass_interface.h"
 
 namespace xla {
 namespace cpu {
@@ -49,6 +50,54 @@ class ParallelTaskAssignment {
   std::unique_ptr<ParallelCostModel> cost_model_;
 };
 
+// ParallelTaskAssigner computes target parallel task counts for all HLOs
+// in the module, then assigns parallel task counts to HLOs in the entry
+// computation, or to HLOs in embedded computations invoked by (potentially
+// nested) kWhile or kCall instructions.
+// Each HLO which is assigned parallel task counts is outlined into its
+// own embedded computation, which is compiled as a parallel compute function,
+// and which is invoked from a kCall instruction that is lowered in codegen to
+// a runtime parallel fork/join call.
+class ParallelTaskAssigner : public HloPassInterface {
+ public:
+  // 'max_parallelism': the maximum parallel task count per instruction.
+  // 'shape_size': shape size function used by HloCostAnalysis during parallel
+  //               task assignment.
+  // 'module': the containing HloModule.
+  ParallelTaskAssigner(const int64 max_parallelism,
+                       const HloCostAnalysis::ShapeSizeFunction& shape_size,
+                       HloModule* module)
+      : parallel_task_assignment_(max_parallelism, shape_size, module) {}
+  ~ParallelTaskAssigner() override {}
+
+  tensorflow::StringPiece name() const override {
+    return "cpu-parallel-task-assigner";
+  }
+
+  // Run parallel task assigner on 'module'.
+  // Returns true if the computation was changed, false otherwise.
+  StatusOr<bool> Run(HloModule* module) override;
+
+ private:
+  using HloToParallelTasks = std::unordered_map<const HloInstruction*, int64>;
+
+  // Assigns target parallel tasks from 'hlo_to_parallel_tasks' to HLOs in
+  // 'module'.
+  // Returns true if the computation was changed, false otherwise.
+  bool AssignParallelTasks(HloModule* module,
+                           const HloToParallelTasks& hlo_to_parallel_tasks);
+  bool AssignParallelTasksHelper(
+      HloModule* module, HloComputation* computation,
+      const HloToParallelTasks& hlo_to_parallel_tasks);
+
+  // Computes target parallel task counts (returned in 'parallel_task_counts')
+  // for parallelizable instructions in 'module'.
+  void ComputeTargetParallelTasks(HloModule* module,
+                                  HloToParallelTasks* hlo_to_parallel_tasks);
+
+  ParallelTaskAssignment parallel_task_assignment_;
+};
+
 }  // namespace cpu
 }  // namespace xla
 
diff --git a/tensorflow/compiler/xla/service/cpu/runtime_fork_join.cc b/tensorflow/compiler/xla/service/cpu/runtime_fork_join.cc
new file mode 100644
index 0000000000..d03da46575
--- /dev/null
+++ b/tensorflow/compiler/xla/service/cpu/runtime_fork_join.cc
@@ -0,0 +1,97 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/service/cpu/runtime_fork_join.h"
+
+#define EIGEN_USE_THREADS
+
+#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "tensorflow/compiler/xla/executable_run_options.h"
+#include "tensorflow/core/lib/core/blocking_counter.h"
+#include "tensorflow/core/platform/logging.h"
+#include "tensorflow/core/platform/types.h"
+
+using tensorflow::int32;
+using tensorflow::int64;
+using tensorflow::uint64;
+
+using ComputeFunctionType = void (*)(void*, const void*, const void**, void**,
+                                     int64*, uint64*);
+
+// Dispatches 'num_partitions - 1' calls to 'function_ptr' in parallel.
+// Calls 'function_ptr' for first partition inline.
+// Uses blocking counter to synchonize threads after parallel calls complete.
+//
+// The 'partitions' array has a total number of elements equal to
+// 'num_partitions * num_partitioned_dims * 2' (the '2' is necessary to specify
+// dimension start and limit indices).
+//
+// The 'partitions' array layout stores array elements in memory with dimension
+// start limit as the most-minor dimension, followed by dimension, then
+// partition.
+//
+// EX: Layout of 'partitions' array with 'num_partitions = 2', and
+//     'num_partitioned_dims = 3'
+//
+//   [partition0_dim0_start]
+//   [partition0_dim0_limit]
+//   [partition0_dim1_start]
+//   [partition0_dim1_limit]
+//   [partition0_dim2_start]
+//   [partition0_dim2_limit]
+//   [partition1_dim0_start]
+//   [partition1_dim0_limit]
+//   [partition1_dim1_start]
+//   [partition1_dim1_limit]
+//   [partition1_dim2_start]
+//   [partition1_dim2_limit]
+//
+void __xla_cpu_runtime_ParallelForkJoin(
+    void* result_ptr, const void* run_options_ptr, const void** params,
+    void** temps, uint64* prof_counters, int32 num_partitions,
+    int64* partitions, int32 num_partitioned_dims, void* function_ptr) {
+  VLOG(2) << "ParallelForkJoin ENTRY"
+          << " num_partitions: " << num_partitions
+          << " num_partitioned_dims: " << num_partitioned_dims;
+  CHECK_GT(num_partitions, 1);
+  CHECK_GT(num_partitioned_dims, 0);
+  const xla::ExecutableRunOptions* run_options =
+      static_cast<const xla::ExecutableRunOptions*>(run_options_ptr);
+  ComputeFunctionType function =
+      reinterpret_cast<ComputeFunctionType>(function_ptr);
+  // Compute partition stride in 'partitions' array.
+  const int64 stride = 2 * num_partitioned_dims;
+
+  // Dispatch 'num_partitions - 1' compute functions to run in parallel.
+  tensorflow::BlockingCounter bc(num_partitions - 1);
+  for (int32 i = 1; i < num_partitions; ++i) {
+    const int64 offset = i * stride;
+    run_options->intra_op_thread_pool()->enqueueNoNotification(
+        [i, function, result_ptr, run_options_ptr, params, temps, prof_counters,
+         partitions, offset, &bc]() {
+          function(result_ptr, run_options_ptr, params, temps,
+                   &partitions[offset], prof_counters);
+          bc.DecrementCount();
+          VLOG(3) << "ParallelForkJoin partition " << i << " done.";
+        });
+  }
+
+  // Call first compute function inline.
+  function(result_ptr, run_options_ptr, params, temps, &partitions[0],
+           prof_counters);
+  VLOG(3) << "ParallelForkJoin partition 0 done.";
+  bc.Wait();
+  VLOG(2) << "ParallelForkJoin EXIT";
+}
diff --git a/tensorflow/compiler/xla/service/cpu/runtime_fork_join.h b/tensorflow/compiler/xla/service/cpu/runtime_fork_join.h
new file mode 100644
index 0000000000..fcf1cc6207
--- /dev/null
+++ b/tensorflow/compiler/xla/service/cpu/runtime_fork_join.h
@@ -0,0 +1,33 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef THIRD_PARTY_TENSORFLOW_COMPILER_XLA_SERVICE_CPU_RUNTIME_FORK_JOIN_H_
+#define THIRD_PARTY_TENSORFLOW_COMPILER_XLA_SERVICE_CPU_RUNTIME_FORK_JOIN_H_
+
+#include "tensorflow/core/platform/types.h"
+
+extern "C" {
+
+// Dispatches 'num_partitions' parallel calls to 'function_ptr' and joins
+// threads before returning. See comments in runtime_fork_join.cc for details.
+extern void __xla_cpu_runtime_ParallelForkJoin(
+    void* result_ptr, const void* run_options_ptr, const void** params,
+    void** temps, tensorflow::uint64* prof_counters,
+    tensorflow::int32 num_partitions, tensorflow::int64* partitions,
+    tensorflow::int32 num_partitioned_dims, void* function_ptr);
+
+}  // extern "C"
+
+#endif  // THIRD_PARTY_TENSORFLOW_COMPILER_XLA_SERVICE_CPU_RUNTIME_FORK_JOIN_H_
diff --git a/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc b/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc
index c614e334a8..cfffb3fbc3 100644
--- a/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc
+++ b/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc
@@ -32,6 +32,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/cpu/cpu_runtime_neon.h"
 #include "tensorflow/compiler/xla/service/cpu/cpu_runtime_sse4_1.h"
 #include "tensorflow/compiler/xla/service/cpu/runtime_conv2d.h"
+#include "tensorflow/compiler/xla/service/cpu/runtime_fork_join.h"
 #include "tensorflow/compiler/xla/service/cpu/runtime_matmul.h"
 #include "tensorflow/compiler/xla/service/cpu/runtime_single_threaded_conv2d.h"
 #include "tensorflow/compiler/xla/service/cpu/runtime_single_threaded_matmul.h"
@@ -104,6 +105,7 @@ class JITSymbolTable {
     ADD_JIT_SYMBOL_TO_TABLE(EigenSingleThreadedConvF32);
     ADD_JIT_SYMBOL_TO_TABLE(EigenSingleThreadedMatMulF32);
     ADD_JIT_SYMBOL_TO_TABLE(EigenSingleThreadedMatMulF64);
+    ADD_JIT_SYMBOL_TO_TABLE(ParallelForkJoin);
 
 #undef ADD_JIT_SYMBOL_TO_TABLE
   }
diff --git a/tensorflow/compiler/xla/tests/BUILD b/tensorflow/compiler/xla/tests/BUILD
index 769f509adc..b02d906d93 100644
--- a/tensorflow/compiler/xla/tests/BUILD
+++ b/tensorflow/compiler/xla/tests/BUILD
@@ -1411,8 +1411,10 @@ xla_test(
         "//tensorflow/compiler/xla/tests:hlo_test_base",
         "//tensorflow/compiler/xla/tests:literal_test_util",
         "//tensorflow/compiler/xla/tests:xla_internal_test_main",
+        "//tensorflow/core:core_cpu_internal",
         "//tensorflow/core:lib",
         "//tensorflow/core:test",
+        "//third_party/eigen3",
     ],
 )
 
diff --git a/tensorflow/compiler/xla/tests/fusion_test.cc b/tensorflow/compiler/xla/tests/fusion_test.cc
index 3bf9ccb197..a8f6488996 100644
--- a/tensorflow/compiler/xla/tests/fusion_test.cc
+++ b/tensorflow/compiler/xla/tests/fusion_test.cc
@@ -17,8 +17,12 @@ limitations under the License.
 #include <algorithm>
 #include <memory>
 #include <new>
+#include <random>
 #include <utility>
 
+#define EIGEN_USE_THREADS
+
+#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 #include "tensorflow/compiler/xla/array2d.h"
 #include "tensorflow/compiler/xla/client/client_library.h"
 #include "tensorflow/compiler/xla/client/computation.h"
@@ -37,6 +41,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/tests/literal_test_util.h"
 #include "tensorflow/compiler/xla/tests/test_macros.h"
 #include "tensorflow/compiler/xla/xla_data.pb.h"
+#include "tensorflow/core/common_runtime/eigen_thread_pool.h"
 #include "tensorflow/core/lib/gtl/array_slice.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/protobuf.h"
@@ -250,6 +255,42 @@ XLA_TEST_F(FusionTest, Parameter) {
                               ErrorSpec(1e-4));
 }
 
+XLA_TEST_F(FusionTest, RandomizedParallelPartition) {
+  // Tests parallel partitioning of a fusion instruction.
+  // Create shape with random outer dimension size to generate random parallel
+  // partition counts for each test run.
+  const int seed = tensorflow::testing::RandomSeed();
+  LOG(INFO) << "RandomizedParallelPartition seed: " << seed;
+  std::mt19937 generator(seed);
+  std::uniform_int_distribution<int> distribution(128, 1024);
+  const int64 rand_dim0_size = distribution(generator);
+  const int64 dim1_size = 1024;
+  Shape shape =
+      ShapeUtil::MakeShapeWithLayout(F32, {rand_dim0_size, dim1_size}, {1, 0});
+  // Build simple fusion computation: y = x^2 (elementwise).
+  auto builder = HloComputation::Builder(TestName());
+  auto hlo_module = CreateNewModule();
+
+  auto two = builder.AddInstruction(
+      HloInstruction::CreateConstant(Literal::CreateR0<float>(2.0)));
+  auto x =
+      builder.AddInstruction(HloInstruction::CreateBroadcast(shape, two, {}));
+  auto y = builder.AddInstruction(
+      HloInstruction::CreateBinary(shape, HloOpcode::kMultiply, x, x));
+
+  hlo_module->AddEntryComputation(builder.Build())
+      ->CreateFusionInstruction(/*instructions_to_fuse=*/{y, x, two},
+                                HloInstruction::FusionKind::kLoop);
+  // Compute result.
+  auto result = ExecuteAndTransfer(std::move(hlo_module), {});
+  // Every element of result should be y = x^2 = 4.0.
+  for (int i = 0; i < rand_dim0_size; ++i) {
+    for (int j = 0; j < dim1_size; ++j) {
+      EXPECT_EQ(4.0, result->Get<float>({i, j}));
+    }
+  }
+}
+
 XLA_TEST_F(FusionTest, BroadcastIntoBinaryOp) {
   auto builder = HloComputation::Builder(TestName());
   auto hlo_module = CreateNewModule();
@@ -722,47 +763,104 @@ void BM_ParallelFusion(int num_iters) {
   auto executors = PlatformUtil::GetStreamExecutors(platform).ValueOrDie();
   StreamExecutorMemoryAllocator allocator(platform, executors);
 
-  const int64 intra_op_parallelism_threads = 16;
+  const int64 intra_op_parallelism_threads = 24;
   xla::LocalClientOptions client_options;
   client_options.set_platform(platform);
   client_options.set_intra_op_parallelism_threads(intra_op_parallelism_threads);
   auto client =
       ClientLibrary::GetOrCreateLocalClient(client_options).ValueOrDie();
 
-  const int64 dim_size = 1024;
-  // Create a simple fusable elementwise computation.
+  auto* transfer_manager =
+      TransferManager::GetForPlatform(platform).ValueOrDie();
+  int device_ordinal = client->default_device_ordinal();
+
+  // Computation shape parameters.
+  const int64 param0_dim0 = 1024;
+  const int64 param0_dim1 = 1024;
+  const int64 param1_dim0 = 1024;
+  const int64 param1_dim1 = 1024;
+  const int64 param2_dim0 = 1024;
+  const int64 param2_dim1 = 1024;
+
+  // Create computation.
   ComputationBuilder builder(client, "ParallelFusion");
-  Shape input_shape = ShapeUtil::MakeShape(F32, {dim_size, dim_size});
-  auto input0 = builder.Broadcast(builder.ConstantR0<float>(1.5f),
-                                  AsInt64Slice(input_shape.dimensions()));
-  auto input1 = builder.Broadcast(builder.ConstantR0<float>(2.0f),
-                                  AsInt64Slice(input_shape.dimensions()));
-  auto input2 = builder.Broadcast(builder.ConstantR0<float>(3.0f),
-                                  AsInt64Slice(input_shape.dimensions()));
-  auto x = builder.Mul(input0, input1);
-  auto y = builder.Add(x, input2);
+  Shape shape0 = ShapeUtil::MakeShape(F32, {param0_dim0, param0_dim1});
+  auto param0 = builder.Parameter(0, shape0, "param0");
+  Shape shape1 = ShapeUtil::MakeShape(F32, {param1_dim0, param1_dim1});
+  auto param1 = builder.Parameter(1, shape1, "param1");
+  Shape shape2 = ShapeUtil::MakeShape(F32, {param2_dim0, param2_dim1});
+  auto param2 = builder.Parameter(2, shape2, "param2");
+
+  auto x = builder.Mul(param0, param1);
+  auto y = builder.Add(x, param2);
   auto computation = builder.Build().ConsumeValueOrDie();
 
+  // Transfer literals to device.
+  auto buffer0 =
+      ScopedShapedBuffer::Allocate(shape0, &allocator, /*device_ordinal=*/0)
+          .ConsumeValueOrDie();
+  auto param0_literal =
+      Literal::CreateR2F32Linspace(1.0, 2.0, param0_dim0, param0_dim1);
+  ASSERT_IS_OK(transfer_manager->TransferLiteralToDevice(
+      executors[device_ordinal], *param0_literal, buffer0->mutable_buffer({})));
+
+  auto buffer1 =
+      ScopedShapedBuffer::Allocate(shape1, &allocator, /*device_ordinal=*/0)
+          .ConsumeValueOrDie();
+  auto param1_literal =
+      Literal::CreateR2F32Linspace(1.0, 2.0, param1_dim0, param1_dim1);
+  ASSERT_IS_OK(transfer_manager->TransferLiteralToDevice(
+      executors[device_ordinal], *param1_literal, buffer1->mutable_buffer({})));
+
+  auto buffer2 =
+      ScopedShapedBuffer::Allocate(shape2, &allocator, /*device_ordinal=*/0)
+          .ConsumeValueOrDie();
+  auto param2_literal =
+      Literal::CreateR2F32Linspace(1.0, 2.0, param2_dim0, param2_dim1);
+  ASSERT_IS_OK(transfer_manager->TransferLiteralToDevice(
+      executors[device_ordinal], *param2_literal, buffer2->mutable_buffer({})));
+
+  // Build executable.
   std::unique_ptr<LocalExecutable> executable =
-      client->Compile(computation, {}, ExecutableBuildOptions())
+      client
+          ->Compile(computation,
+                    {&buffer0->shape(), &buffer1->shape(), &buffer2->shape()},
+                    ExecutableBuildOptions())
           .ConsumeValueOrDie();
 
-  // Run some warm-up executions.
+  se::Stream stream(executors[client->default_device_ordinal()]);
+  stream.Init();
+
+  // Initialize thread pool.
+  tensorflow::thread::ThreadPool pool(tensorflow::Env::Default(), "XLAEigen",
+                                      intra_op_parallelism_threads);
+  tensorflow::EigenThreadPoolWrapper tp(&pool);
+  Eigen::ThreadPoolDevice device(&tp, tp.NumThreads());
+
+  // Initialize ExecutableRunOptions.
   ExecutableRunOptions options;
-  options.set_allocator(&allocator);
+  options.set_allocator(&allocator).set_stream(&stream);
+  options.set_intra_op_thread_pool(&device);
+
+  // Run some warm-up executions.
   const int kWarmups = 2;
   for (int i = 0; i < kWarmups; ++i) {
-    auto result = executable->Run({}, options);
+    auto result =
+        executable->Run({buffer0.get(), buffer1.get(), buffer2.get()}, options);
     ASSERT_TRUE(result.ok());
   }
 
   // Run benchmark.
-  tensorflow::testing::BytesProcessed(static_cast<int64>(num_iters) * dim_size *
-                                      dim_size * sizeof(float));
+  const int64 total_bytes = param0_dim0 * param0_dim0 +
+                            param1_dim0 * param1_dim0 +
+                            param2_dim0 * param2_dim0;
+  tensorflow::testing::BytesProcessed(static_cast<int64>(num_iters) *
+                                      total_bytes * sizeof(float));
   tensorflow::testing::UseRealTime();
   tensorflow::testing::StartTiming();
   for (int i = 0; i < num_iters; ++i) {
-    auto result = executable->Run({}, options);
+    auto result =
+        executable->Run({buffer0.get(), buffer1.get(), buffer2.get()}, options);
     ASSERT_TRUE(result.ok());
   }
 }
-- 
GitLab


From 05aebd4c342c3ab432250fa3ef17bf212061f931 Mon Sep 17 00:00:00 2001
From: Shanqing Cai <cais@google.com>
Date: Mon, 16 Oct 2017 08:11:06 -0700
Subject: [PATCH 419/909] tfdbg doc: Fix minor typo

PiperOrigin-RevId: 172326303
---
 tensorflow/docs_src/programmers_guide/debugger.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/docs_src/programmers_guide/debugger.md b/tensorflow/docs_src/programmers_guide/debugger.md
index 3ede42e8f7..58154d19e7 100644
--- a/tensorflow/docs_src/programmers_guide/debugger.md
+++ b/tensorflow/docs_src/programmers_guide/debugger.md
@@ -141,7 +141,7 @@ Try the following commands at the `tfdbg>` prompt (referencing the code at
 | **`lt`** | | **List dumped tensors.** | `lt` |
 | | `-n <name_pattern>` | List dumped tensors with names matching given regular-expression pattern. | `lt -n Softmax.*` |
 | | `-t <op_pattern>` | List dumped tensors with op types matching given regular-expression pattern. | `lt -t MatMul` |
-| | `s <sort_key>` | Sort the output by given `sort_key`, whose possible values are `timestamp` (default), `dump_size`, `op_type` and `tensor_name`. | `lt -s dump_size` |
+| | `-s <sort_key>` | Sort the output by given `sort_key`, whose possible values are `timestamp` (default), `dump_size`, `op_type` and `tensor_name`. | `lt -s dump_size` |
 | | `-r` | Sort in reverse order. | `lt -r -s dump_size` |
 | **`pt`** | | **Print value of a dumped tensor.** | |
 | | `pt <tensor>` | Print tensor value. | `pt hidden/Relu:0` |
-- 
GitLab


From 51c115b33fecd9e96aa12c264c2c717afe8bfea8 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 16 Oct 2017 09:03:28 -0700
Subject: [PATCH 420/909] Fix typo (undefined variable `mean_absolute_error`,
 should refer to `error` previously defined).

PiperOrigin-RevId: 172331504
---
 tensorflow/docs_src/api_guides/python/contrib.metrics.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/docs_src/api_guides/python/contrib.metrics.md b/tensorflow/docs_src/api_guides/python/contrib.metrics.md
index b502826e6a..1eb9cf417a 100644
--- a/tensorflow/docs_src/api_guides/python/contrib.metrics.md
+++ b/tensorflow/docs_src/api_guides/python/contrib.metrics.md
@@ -64,7 +64,7 @@ sess.run(tf.local_variables_initializer())
 for batch in range(num_batches):
   sess.run([update_op_acc, update_op_error])
 
-accuracy, mean_absolute_error = sess.run([accuracy, mean_absolute_error])
+accuracy, error = sess.run([accuracy, error])
 ```
 
 Note that when evaluating the same metric multiple times on different inputs,
-- 
GitLab


From 14a66fd59c75f1b75c3e32a7e243778d3e83d221 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 16 Oct 2017 09:21:09 -0700
Subject: [PATCH 421/909] [TF:XLA] Update xla_data comments for And, Or, and
 Not.

PiperOrigin-RevId: 172333451
---
 tensorflow/compiler/xla/xla_data.proto | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/compiler/xla/xla_data.proto b/tensorflow/compiler/xla/xla_data.proto
index 0d7e583bed..eae284afb7 100644
--- a/tensorflow/compiler/xla/xla_data.proto
+++ b/tensorflow/compiler/xla/xla_data.proto
@@ -621,7 +621,7 @@ message WhileRequest {
 enum UnaryOperation {
   UNOP_INVALID = 0;
 
-  // Elementwise, logical negation
+  // Elementwise, logical negation on booleans and bitwise negation on ints.
   UNOP_NOT = 1;
 
   // Elementwise, computes e^x.
@@ -710,7 +710,7 @@ enum BinaryOperation {
   // Remainder operation.
   BINOP_REM = 17;
 
-  // Logical operators
+  // Element-wise, logical operators on booleans and bitwise operators on ints.
   BINOP_AND = 18;
   BINOP_OR = 19;
 
-- 
GitLab


From 0a572887ffa9879d6a303aafd6de9e288776fc8f Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Mon, 16 Oct 2017 09:43:42 -0700
Subject: [PATCH 422/909] Automated g4 rollback of changelist 172039259

PiperOrigin-RevId: 172336111
---
 tensorflow/python/BUILD                       |  1 -
 .../resource_variable_ops_test.py             | 10 --
 .../python/ops/resource_variable_ops.py       |  5 -
 tensorflow/python/training/adam_test.py       | 82 ++++++++--------
 tensorflow/python/training/saver_test.py      | 97 +++++++++----------
 5 files changed, 88 insertions(+), 107 deletions(-)

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 1885caf695..48436fe8cf 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -3396,7 +3396,6 @@ cuda_py_test(
         ":training",
         ":platform_test",
         ":client_testlib",
-        ":variable_scope",
         "//third_party/py/numpy",
     ],
 )
diff --git a/tensorflow/python/kernel_tests/resource_variable_ops_test.py b/tensorflow/python/kernel_tests/resource_variable_ops_test.py
index 6f2bc2f752..8cf8286ed1 100644
--- a/tensorflow/python/kernel_tests/resource_variable_ops_test.py
+++ b/tensorflow/python/kernel_tests/resource_variable_ops_test.py
@@ -422,16 +422,6 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase):
       self.assertEqual(1, v1.read_value().numpy())
       self.assertEqual(2, v2.read_value().numpy())
 
-  def testDestruction(self):
-    with context.eager_mode():
-      var = resource_variable_ops.ResourceVariable(initial_value=1.0,
-                                                   name="var8")
-      var.__del__()
-      with self.assertRaisesRegexp(errors.NotFoundError,
-                                   r"Resource .*\/var8\/.* does not exist."):
-        resource_variable_ops.destroy_resource_op(var._handle,
-                                                  ignore_lookup_error=False)
-
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/ops/resource_variable_ops.py b/tensorflow/python/ops/resource_variable_ops.py
index 99ff02873b..cbfa141256 100644
--- a/tensorflow/python/ops/resource_variable_ops.py
+++ b/tensorflow/python/ops/resource_variable_ops.py
@@ -427,11 +427,6 @@ class ResourceVariable(variables.Variable):
     self._constraint = None
   # LINT.ThenChange(//tensorflow/python/eager/graph_callable.py)
 
-  def __del__(self):
-    if context.in_eager_mode():
-      gen_resource_variable_ops.destroy_resource_op(self._handle,
-                                                    ignore_lookup_error=False)
-
   @property
   def dtype(self):
     """The dtype of this variable."""
diff --git a/tensorflow/python/training/adam_test.py b/tensorflow/python/training/adam_test.py
index 96de9b921b..defcf33714 100644
--- a/tensorflow/python/training/adam_test.py
+++ b/tensorflow/python/training/adam_test.py
@@ -29,7 +29,6 @@ from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import resource_variable_ops
-from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import test
 from tensorflow.python.training import adam
@@ -153,54 +152,53 @@ class AdamOptimizerTest(test.TestCase):
 
   def doTestBasic(self, use_resource=False):
     for i, dtype in enumerate([dtypes.half, dtypes.float32, dtypes.float64]):
-      with variable_scope.variable_scope("%d" % i):
-        # Initialize variables for numpy implementation.
-        m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0
-        var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype)
-        grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype)
-        var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype)
-        grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype)
-
-        if use_resource:
-          var0 = resource_variable_ops.ResourceVariable(
-              var0_np, name="var0_%d" % i)
-          var1 = resource_variable_ops.ResourceVariable(
-              var1_np, name="var1_%d" % i)
-        else:
-          var0 = variables.Variable(var0_np)
-          var1 = variables.Variable(var1_np)
-        grads0 = constant_op.constant(grads0_np)
-        grads1 = constant_op.constant(grads1_np)
+      # Initialize variables for numpy implementation.
+      m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0
+      var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype)
+      grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype)
+      var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype)
+      grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype)
+
+      if use_resource:
+        var0 = resource_variable_ops.ResourceVariable(
+            var0_np, name="var0_%d" % i)
+        var1 = resource_variable_ops.ResourceVariable(
+            var1_np, name="var1_%d" % i)
+      else:
+        var0 = variables.Variable(var0_np)
+        var1 = variables.Variable(var1_np)
+      grads0 = constant_op.constant(grads0_np)
+      grads1 = constant_op.constant(grads1_np)
 
-        opt = adam.AdamOptimizer()
-        update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
+      opt = adam.AdamOptimizer()
+      update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
 
-        if context.in_graph_mode():
-          self.evaluate(variables.global_variables_initializer())
-          # Fetch params to validate initial values
-          self.assertAllClose([1.0, 2.0], self.evaluate(var0))
-          self.assertAllClose([3.0, 4.0], self.evaluate(var1))
+      if context.in_graph_mode():
+        self.evaluate(variables.global_variables_initializer())
+        # Fetch params to validate initial values
+        self.assertAllClose([1.0, 2.0], self.evaluate(var0))
+        self.assertAllClose([3.0, 4.0], self.evaluate(var1))
 
-        beta1_power, beta2_power = opt._get_beta_accumulators()
+      beta1_power, beta2_power = opt._get_beta_accumulators()
 
-        # Run 3 steps of Adam
-        for t in range(1, 4):
-          if context.in_graph_mode():
-            self.evaluate(update)
-          elif t > 1:
-            opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
+      # Run 3 steps of Adam
+      for t in range(1, 4):
+        if context.in_graph_mode():
+          self.evaluate(update)
+        elif t > 1:
+          opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
 
-          self.assertAllCloseAccordingToType(0.9**(t + 1),
-                                             self.evaluate(beta1_power))
-          self.assertAllCloseAccordingToType(0.999**(t + 1),
-                                             self.evaluate(beta2_power))
+        self.assertAllCloseAccordingToType(0.9**(t + 1),
+                                           self.evaluate(beta1_power))
+        self.assertAllCloseAccordingToType(0.999**(t + 1),
+                                           self.evaluate(beta2_power))
 
-          var0_np, m0, v0 = adam_update_numpy(var0_np, grads0_np, t, m0, v0)
-          var1_np, m1, v1 = adam_update_numpy(var1_np, grads1_np, t, m1, v1)
+        var0_np, m0, v0 = adam_update_numpy(var0_np, grads0_np, t, m0, v0)
+        var1_np, m1, v1 = adam_update_numpy(var1_np, grads1_np, t, m1, v1)
 
-          # Validate updated params
-          self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0))
-          self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1))
+        # Validate updated params
+        self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0))
+        self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1))
 
   def testBasic(self):
     with self.test_session():
diff --git a/tensorflow/python/training/saver_test.py b/tensorflow/python/training/saver_test.py
index a8eb8e5fcf..07cd67a4b9 100644
--- a/tensorflow/python/training/saver_test.py
+++ b/tensorflow/python/training/saver_test.py
@@ -110,32 +110,32 @@ class SaverTest(test.TestCase):
     # Start a second session.  In that session the parameter nodes
     # have not been initialized either.
     with self.test_session(graph=ops_lib.Graph()) as sess:
-      v0_2 = variable_op(-1.0, name="v0")
-      v1_2 = variable_op(-1.0, name="v1")
-      v2_2 = saver_test_utils.CheckpointedOp(name="v2")
+      v0 = variable_op(-1.0, name="v0")
+      v1 = variable_op(-1.0, name="v1")
+      v2 = saver_test_utils.CheckpointedOp(name="v2")
 
       # Assert that the variables are not initialized.
       if context.in_graph_mode():
         self.assertEqual(
             len(variables.report_uninitialized_variables().eval()), 2)
-        self.assertEqual(0, len(v2_2.keys().eval()))
-        self.assertEqual(0, len(v2_2.values().eval()))
+        self.assertEqual(0, len(v2.keys().eval()))
+        self.assertEqual(0, len(v2.values().eval()))
       # Restore the saved values in the parameter nodes.
-      save = saver_module.Saver({"v0": v0_2, "v1": v1_2, "v2": v2_2.saveable})
+      save = saver_module.Saver({"v0": v0, "v1": v1, "v2": v2.saveable})
       save.restore(sess, save_path)
       # Check that the parameter nodes have been restored.
-      self.assertEqual(10.0, self.evaluate(v0_2))
-      self.assertEqual(20.0, self.evaluate(v1_2))
-      self.assertEqual(b"k1", self.evaluate(v2_2.keys()))
-      self.assertEqual(30.0, self.evaluate(v2_2.values()))
+      self.assertEqual(10.0, self.evaluate(v0))
+      self.assertEqual(20.0, self.evaluate(v1))
+      self.assertEqual(b"k1", self.evaluate(v2.keys()))
+      self.assertEqual(30.0, self.evaluate(v2.values()))
 
     # Build another graph with 2 nodes, initialized
     # differently, and a Restore node for them.
     with self.test_session(graph=ops_lib.Graph()) as sess:
-      v0_3 = variable_op(1000.0, name="v0")
-      v1_3 = variable_op(2000.0, name="v1")
-      v2_3 = saver_test_utils.CheckpointedOp(name="v2")
-      v2_init = v2_3.insert("k1000", 3000.0)
+      v0_2 = variable_op(1000.0, name="v0")
+      v1_2 = variable_op(2000.0, name="v1")
+      v2_2 = saver_test_utils.CheckpointedOp(name="v2")
+      v2_init = v2_2.insert("k1000", 3000.0)
 
       # Check that the parameter nodes have been initialized.
       if context.in_graph_mode():
@@ -143,19 +143,19 @@ class SaverTest(test.TestCase):
         self.evaluate(init_all_op)
         # TODO(xpan): Why _mutable_hash_table_v2 doesn't create empty
         # table as it claims in eager mode?
-        self.assertEqual(b"k1000", self.evaluate(v2_3.keys()))
-        self.assertEqual(3000.0, self.evaluate(v2_3.values()))
-      self.assertEqual(1000.0, self.evaluate(v0_3))
-      self.assertEqual(2000.0, self.evaluate(v1_3))
+        self.assertEqual(b"k1000", self.evaluate(v2_2.keys()))
+        self.assertEqual(3000.0, self.evaluate(v2_2.values()))
+      self.assertEqual(1000.0, self.evaluate(v0_2))
+      self.assertEqual(2000.0, self.evaluate(v1_2))
 
       # Restore the values saved earlier in the parameter nodes.
-      save2 = saver_module.Saver({"v0": v0_3, "v1": v1_3, "v2": v2_3.saveable})
+      save2 = saver_module.Saver({"v0": v0_2, "v1": v1_2, "v2": v2_2.saveable})
       save2.restore(sess, save_path)
       # Check that the parameter nodes have been restored.
-      self.assertEqual(10.0, self.evaluate(v0_3))
-      self.assertEqual(20.0, self.evaluate(v1_3))
-      self.assertEqual(b"k1", self.evaluate(v2_3.keys()))
-      self.assertEqual(30.0, self.evaluate(v2_3.values()))
+      self.assertEqual(10.0, self.evaluate(v0_2))
+      self.assertEqual(20.0, self.evaluate(v1_2))
+      self.assertEqual(b"k1", self.evaluate(v2_2.keys()))
+      self.assertEqual(30.0, self.evaluate(v2_2.values()))
 
   def testBasic(self):
     self.basicSaveRestore(variables.Variable)
@@ -487,10 +487,10 @@ class SaverTest(test.TestCase):
       val = save.save(sess, save_path)
       self.assertEqual(save_path, val)
     with self.test_session() as sess:
-      var2 = resource_variable_ops.ResourceVariable(other_value, name=var_name)
-      save = saver_module.Saver({var_name: var2})
+      var = resource_variable_ops.ResourceVariable(other_value, name=var_name)
+      save = saver_module.Saver({var_name: var})
       save.restore(sess, save_path)
-      self.assertAllClose(var_value, self.evaluate(var2))
+      self.assertAllClose(var_value, self.evaluate(var))
 
   def testCacheRereadsFile(self):
     save_path = os.path.join(self.get_temp_dir(), "cache_rereads")
@@ -618,29 +618,28 @@ class SaverTest(test.TestCase):
     global_step_int = 5
     # Save and reload one Variable named "var0".
     self._SaveAndLoad("var0", 0.0, 1.0, save_path)
-    for i, use_tensor in enumerate([True, False]):
-      with variable_scope.variable_scope("%d" % i):
-        var = resource_variable_ops.ResourceVariable(1.0, name="var0")
-        save = saver_module.Saver(
-            {
-                var._shared_name: var
-            }, pad_step_number=pad_step_number)
-        if context.in_graph_mode():
-          self.evaluate(var.initializer)
-          sess = ops_lib.get_default_session()
-        else:
-          sess = None
-        if use_tensor:
-          global_step = constant_op.constant(global_step_int)
-          val = save.save(sess, save_path, global_step=global_step)
-        else:
-          val = save.save(sess, save_path, global_step=global_step_int)
-        if pad_step_number:
-          expected_save_path = "%s-%s" % (save_path,
-                                          "{:08d}".format(global_step_int))
-        else:
-          expected_save_path = "%s-%d" % (save_path, global_step_int)
-        self.assertEqual(expected_save_path, val)
+    for use_tensor in [True, False]:
+      var = resource_variable_ops.ResourceVariable(1.0, name="var0")
+      save = saver_module.Saver(
+          {
+              var._shared_name: var
+          }, pad_step_number=pad_step_number)
+      if context.in_graph_mode():
+        self.evaluate(var.initializer)
+        sess = ops_lib.get_default_session()
+      else:
+        sess = None
+      if use_tensor:
+        global_step = constant_op.constant(global_step_int)
+        val = save.save(sess, save_path, global_step=global_step)
+      else:
+        val = save.save(sess, save_path, global_step=global_step_int)
+      if pad_step_number:
+        expected_save_path = "%s-%s" % (save_path,
+                                        "{:08d}".format(global_step_int))
+      else:
+        expected_save_path = "%s-%d" % (save_path, global_step_int)
+      self.assertEqual(expected_save_path, val)
 
   def testSaveWithGlobalStepWithPadding(self):
     self.testSaveWithGlobalStep(pad_step_number=True)
-- 
GitLab


From 07c6faf039ff1a49a540f8caafd7a30692fcf14d Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Mon, 16 Oct 2017 09:53:11 -0700
Subject: [PATCH 423/909] Adds a host-memory GPU kernel for DestroyResourceOp.

PiperOrigin-RevId: 172337312
---
 tensorflow/core/kernels/resource_variable_ops.cc | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tensorflow/core/kernels/resource_variable_ops.cc b/tensorflow/core/kernels/resource_variable_ops.cc
index 3cca493972..90db0c2b7b 100644
--- a/tensorflow/core/kernels/resource_variable_ops.cc
+++ b/tensorflow/core/kernels/resource_variable_ops.cc
@@ -200,6 +200,9 @@ class DestroyResourceOp : public OpKernel {
 
 REGISTER_KERNEL_BUILDER(Name("DestroyResourceOp").Device(DEVICE_CPU),
                         DestroyResourceOp);
+REGISTER_KERNEL_BUILDER(
+    Name("DestroyResourceOp").Device(DEVICE_GPU).HostMemory("resource"),
+    DestroyResourceOp);
 
 template <typename Device, typename T>
 class AssignVariableOp : public OpKernel {
-- 
GitLab


From 19fd294eae4e8e22f6ab46b21cf41323750a1c69 Mon Sep 17 00:00:00 2001
From: Brennan Saeta <saeta@google.com>
Date: Mon, 16 Oct 2017 10:09:55 -0700
Subject: [PATCH 424/909] Support ClusterSpec propagation with XLA Devices

Currently, you cannot use ClusterSpec propagation in conjunction with XLA devices, as the RenamedDevice wraps the underlying device and breaks the dynamic cast.

PiperOrigin-RevId: 172339725
---
 tensorflow/compiler/jit/xla_device.cc           | 4 +++-
 tensorflow/core/common_runtime/renamed_device.h | 7 +++++++
 tensorflow/core/framework/device_base.h         | 3 +++
 3 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/tensorflow/compiler/jit/xla_device.cc b/tensorflow/compiler/jit/xla_device.cc
index a2c91511ec..7ccea58f6e 100644
--- a/tensorflow/compiler/jit/xla_device.cc
+++ b/tensorflow/compiler/jit/xla_device.cc
@@ -28,6 +28,7 @@ limitations under the License.
 #include "tensorflow/core/common_runtime/device_factory.h"
 #include "tensorflow/core/common_runtime/dma_helper.h"
 #include "tensorflow/core/common_runtime/function.h"
+#include "tensorflow/core/common_runtime/renamed_device.h"
 #include "tensorflow/core/framework/allocator.h"
 #include "tensorflow/core/framework/device_base.h"
 #include "tensorflow/core/framework/function.h"
@@ -161,7 +162,8 @@ const DeviceType& XlaDevice::Metadata::jit_device_type() const {
 
 /* static */ Status XlaDevice::GetMetadata(OpKernelContext* ctx,
                                            const Metadata** metadata) {
-  XlaDevice* xla_device = dynamic_cast<XlaDevice*>(ctx->device());
+  XlaDevice* xla_device =
+      dynamic_cast<XlaDevice*>(ctx->device()->UnderlyingDevice());
   if (xla_device == nullptr) {
     return errors::Internal(
         "Cannot get XLA metadata from non-XLA device \"", ctx->device()->name(),
diff --git a/tensorflow/core/common_runtime/renamed_device.h b/tensorflow/core/common_runtime/renamed_device.h
index 0158e18ced..22a70fbdfa 100644
--- a/tensorflow/core/common_runtime/renamed_device.h
+++ b/tensorflow/core/common_runtime/renamed_device.h
@@ -37,6 +37,13 @@ class RenamedDevice : public Device {
     return underlying_->RequiresRecordingAccessedTensors();
   }
 
+  const DeviceBase* UnderlyingDevice() const override {
+    return underlying_->UnderlyingDevice();
+  }
+  DeviceBase* UnderlyingDevice() override {
+    return underlying_->UnderlyingDevice();
+  }
+
   const CpuWorkerThreads* tensorflow_cpu_worker_threads() const override {
     return underlying_->tensorflow_cpu_worker_threads();
   }
diff --git a/tensorflow/core/framework/device_base.h b/tensorflow/core/framework/device_base.h
index 14a96c57b5..33bd5d250c 100644
--- a/tensorflow/core/framework/device_base.h
+++ b/tensorflow/core/framework/device_base.h
@@ -188,6 +188,9 @@ class DeviceBase {
   // by GPU devices to return a derived type.
   virtual PerOpGpuDevice* MakeGpuDevice() { return nullptr; }
 
+  virtual DeviceBase* UnderlyingDevice() { return this; }
+  virtual const DeviceBase* UnderlyingDevice() const { return this; }
+
   // This is overridden by GPU devices to reinitialize the derived
   // type returned by MakeGpuDevice.
   virtual void ReinitializeGpuDevice(OpKernelContext* /*context*/,
-- 
GitLab


From 3b595a805bbcf4be24a2e01abe1b8031d82dc57b Mon Sep 17 00:00:00 2001
From: Brennan Saeta <saeta@google.com>
Date: Mon, 16 Oct 2017 10:13:58 -0700
Subject: [PATCH 425/909] Support a configurable TPU job name

PiperOrigin-RevId: 172340173
---
 .../contrib/tpu/python/tpu/tpu_config.py      | 16 +++++--
 .../contrib/tpu/python/tpu/tpu_estimator.py   | 45 ++++++++++++++++++-
 2 files changed, 56 insertions(+), 5 deletions(-)

diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_config.py b/tensorflow/contrib/tpu/python/tpu/tpu_config.py
index 0a3be8503a..79fd8b839b 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_config.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_config.py
@@ -27,7 +27,10 @@ from tensorflow.python.estimator import run_config as run_config_lib
 
 class TPUConfig(
     collections.namedtuple('TPUConfig', [
-        'iterations_per_loop', 'num_shards', 'per_host_input_for_training'
+        'iterations_per_loop',
+        'num_shards',
+        'per_host_input_for_training',
+        'tpu_job_name',
     ])):
   """TPU related configuration required by `TPUEstimator`.
 
@@ -46,12 +49,17 @@ class TPUConfig(
       that this only works for single-host TPU training now (tracked in
       b/67051042). For multi-host, please use Per-Core, i.e., `False` for
       `per_host_input_for_training`.
+    tpu_job_name: The name of the TPU job. Typically, this name is auto-inferred
+      within TPUEstimator, however when using ClusterSpec propagation in more
+      esoteric cluster configurations, you may need to specify the job name as a
+      string.
   """
 
   def __new__(cls,
               iterations_per_loop=2,
               num_shards=2,
-              per_host_input_for_training=True):
+              per_host_input_for_training=True,
+              tpu_job_name=None):
 
     # Check iterations_per_loop.
     util_lib.check_positive_integer(iterations_per_loop,
@@ -59,12 +67,12 @@ class TPUConfig(
 
     # Check num_shards.
     util_lib.check_positive_integer(num_shards, 'TPUConfig num_shards')
-
     return super(TPUConfig, cls).__new__(
         cls,
         iterations_per_loop=iterations_per_loop,
         num_shards=num_shards,
-        per_host_input_for_training=per_host_input_for_training)
+        per_host_input_for_training=per_host_input_for_training,
+        tpu_job_name=tpu_job_name)
 
 
 class RunConfig(run_config_lib.RunConfig):
diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
index 43f9defd54..de6c8140c6 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
@@ -122,12 +122,55 @@ def _increase_eval_step_op(iterations_per_loop):
       use_locking=True)
 
 
+_DEFAULT_JOB_NAME = 'tpu_worker'
+_DEFAULT_COORDINATOR_JOB_NAME = 'coordinator'
+_LOCAL_MASTERS = ('', 'local')
+
+
 def _tpu_job(run_config, mode):
+  """Returns the job name to use to place TPU computations on.
+
+  Args:
+    run_config: The tpu_config.RunConfig used for this custom estimator.
+    mode: A model_fn_lib.ModeKeys value.
+
+  Returns:
+    A string containing the job name, or None if no job should be specified.
+
+  Raises:
+    ValueError: If the user needs to specify a tpu_job_name, because we are
+      unable to infer the job name automatically, or if the user-specified job
+      names are inappropriate.
+  """
+  # If the user specifies the tpu_job_name, use that.
+  if run_config.tpu_config.tpu_job_name:
+    return run_config.tpu_config.tpu_job_name
+
   # The tpu job is determined by the run_config. Right now, this method is
   # required as tpu_config is not part of the RunConfig.
   master = (run_config.evaluation_master if mode == model_fn_lib.ModeKeys.EVAL
             else run_config.master)
-  return None if master in ['', 'local'] else 'tpu_worker'
+  if master in _LOCAL_MASTERS:
+    return None
+
+  if (not run_config.session_config or
+      not run_config.session_config.cluster_def.job):
+    return _DEFAULT_JOB_NAME
+  cluster_def = run_config.session_config.cluster_def
+  job_names = set([job.name for job in cluster_def.job])
+  if _DEFAULT_JOB_NAME in job_names:
+    # b/37868888 tracks allowing ClusterSpec propagation to reuse job names.
+    raise ValueError('Currently, tpu_worker is not an allowed job name.')
+  if len(job_names) == 1:
+    return cluster_def.job[0].name
+  if len(job_names) == 2:
+    if _DEFAULT_COORDINATOR_JOB_NAME in job_names:
+      job_names.remove(_DEFAULT_COORDINATOR_JOB_NAME)
+      return job_names.pop()
+    # TODO(b/67716447): Include more sophisticated heuristics.
+  raise ValueError(
+      'Could not infer TPU job name. Please specify a tpu_job_name as part of '
+      'your TPUConfig.')
 
 
 def _is_running_on_cpu(use_tpu, mode, eval_batch_size):
-- 
GitLab


From 2b8ddee235cf5b18a56e3434a72150daaf107169 Mon Sep 17 00:00:00 2001
From: Jayaram Bobba <jayaram.bobba@intel.com>
Date: Mon, 16 Oct 2017 10:31:29 -0700
Subject: [PATCH 426/909] Use char* for mkl allocator strings instead of char[]
 to workaround build issues (#13697)

---
 tensorflow/core/common_runtime/mkl_cpu_allocator.h       | 4 ++--
 tensorflow/core/common_runtime/mkl_cpu_allocator_test.cc | 2 --
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/tensorflow/core/common_runtime/mkl_cpu_allocator.h b/tensorflow/core/common_runtime/mkl_cpu_allocator.h
index 5951b3b6a1..53e80b1ee3 100644
--- a/tensorflow/core/common_runtime/mkl_cpu_allocator.h
+++ b/tensorflow/core/common_runtime/mkl_cpu_allocator.h
@@ -51,7 +51,7 @@ class MklCPUAllocator : public Allocator {
   // Constructor and other standard functions
 
   /// Environment variable that user can set to upper bound on memory allocation
-  static constexpr const char kMaxLimitStr[] = "TF_MKL_ALLOC_MAX_BYTES";
+  static constexpr const char* kMaxLimitStr = "TF_MKL_ALLOC_MAX_BYTES";
 
   /// Default upper limit on allocator size - 64GB
   static const size_t kDefaultMaxLimit = 64LL << 30;
@@ -146,7 +146,7 @@ class MklCPUAllocator : public Allocator {
   static const bool kAllowGrowth = true;
 
   /// Name
-  static constexpr const char kName[] = "mklcpu";
+  static constexpr const char* kName = "mklcpu";
 
   /// The alignment that we need for the allocations
   static const size_t kAlignment = 64;
diff --git a/tensorflow/core/common_runtime/mkl_cpu_allocator_test.cc b/tensorflow/core/common_runtime/mkl_cpu_allocator_test.cc
index cfefaa92e4..a67411cd2e 100644
--- a/tensorflow/core/common_runtime/mkl_cpu_allocator_test.cc
+++ b/tensorflow/core/common_runtime/mkl_cpu_allocator_test.cc
@@ -23,8 +23,6 @@ limitations under the License.
 
 namespace tensorflow {
 
-constexpr char MklCPUAllocator::kMaxLimitStr[];
-
 TEST(MKLBFCAllocatorTest, TestMaxLimit) {
   AllocatorStats stats;
   setenv(MklCPUAllocator::kMaxLimitStr, "1000", 1);
-- 
GitLab


From 2ebbce8e82d8d07bee6b8be14a3961ebdef977a0 Mon Sep 17 00:00:00 2001
From: Se-won Kim <30789814+wonsekim@users.noreply.github.com>
Date: Tue, 17 Oct 2017 02:31:58 +0900
Subject: [PATCH 427/909] fixed type error (API r1.3 document, tf.truncatediv)
 (#13712)

It was just type error
from '-7 / 5 = 1'  to '-7 / 5 = -1'
---
 tensorflow/core/ops/math_ops.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/ops/math_ops.cc b/tensorflow/core/ops/math_ops.cc
index 74af6f7f4a..7b971a9fd5 100644
--- a/tensorflow/core/ops/math_ops.cc
+++ b/tensorflow/core/ops/math_ops.cc
@@ -623,7 +623,7 @@ REGISTER_OP("TruncateDiv")
 Returns x / y element-wise for integer types.
 
 Truncation designates that negative numbers will round fractional quantities
-toward zero. I.e. -7 / 5 = 1. This matches C semantics but it is different
+toward zero. I.e. -7 / 5 = -1. This matches C semantics but it is different
 than Python semantics. See `FloorDiv` for a division function that matches
 Python Semantics.
 
-- 
GitLab


From 5f62ef255eecc1a1e28a9ad91de63ea29cd97ef5 Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Mon, 16 Oct 2017 10:32:58 -0700
Subject: [PATCH 428/909] Proper use of convert_to_tensor in custom_gradient

PiperOrigin-RevId: 172342933
---
 tensorflow/python/eager/backprop_test.py   | 25 ++++++++++++++++++++++
 tensorflow/python/eager/custom_gradient.py | 11 +---------
 2 files changed, 26 insertions(+), 10 deletions(-)

diff --git a/tensorflow/python/eager/backprop_test.py b/tensorflow/python/eager/backprop_test.py
index 2409a7b198..d53c69afcc 100644
--- a/tensorflow/python/eager/backprop_test.py
+++ b/tensorflow/python/eager/backprop_test.py
@@ -475,6 +475,31 @@ class BackpropTest(test.TestCase):
     self.assertEqual(7, grad.numpy())
     self.assertEqual(x, var)
 
+  def testCustomGradient(self):
+
+    @custom_gradient.custom_gradient
+    def my_mul(x, y):
+      result = x*y
+
+      def grad(dr):
+        return [dr*y, dr*x]
+      return result, grad
+
+    lr = 0.25
+    x = resource_variable_ops.ResourceVariable(2., name='x')
+
+    def loss(x):
+      return my_mul(2., x.read_value())
+
+    loss_grads_fn = backprop.implicit_val_and_grad(loss)
+
+    losses = []
+    for _ in range(5):
+      loss, grads_and_vars = loss_grads_fn(x)
+      losses.append(loss.numpy())
+      for (grad, var) in grads_and_vars:
+        var.assign_sub(lr*grad)
+    self.assertAllEqual(losses, [4.0, 3., 2., 1., 0.])
 
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/python/eager/custom_gradient.py b/tensorflow/python/eager/custom_gradient.py
index df116dd819..4ac30075b2 100644
--- a/tensorflow/python/eager/custom_gradient.py
+++ b/tensorflow/python/eager/custom_gradient.py
@@ -22,7 +22,6 @@ from tensorflow.python.eager import context
 from tensorflow.python.eager import tape
 from tensorflow.python.framework import ops as tf_ops
 from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.util import nest
 from tensorflow.python.util import tf_decorator
 
@@ -69,19 +68,11 @@ def custom_gradient(f):
       return nest.pack_sequence_as(
           structure=result, flat_sequence=all_tensors[:len(flat_result)])
 
-    input_tensors = []
-    for x in args:
-      if isinstance(x, tf_ops.Tensor):
-        input_tensors.append(x)
-      if isinstance(x, resource_variable_ops.ResourceVariable):
-        input_tensors.append(x.read_value())
+    input_tensors = [tf_ops.convert_to_tensor(x) for x in args]
 
     with tape.stop_recording():
       result, grad_fn = f(*args, **kwargs)
 
-    # TODO(apassos): naive uses of custom_gradient will not get the correct
-    # second derivative this way if they capture any output tensors. Change the
-    # signature of custom_gradient.
     def actual_grad_fn(*outputs):
       return nest.flatten(grad_fn(*outputs))
 
-- 
GitLab


From 2f916fcf2f49f8a30d8460c1c8d7812d637ab0e4 Mon Sep 17 00:00:00 2001
From: Yifei Feng <fengyifei2026@gmail.com>
Date: Mon, 16 Oct 2017 10:45:33 -0700
Subject: [PATCH 429/909] Add freeze_graph to CONSOLE_SCRIPTS. (#13739)

Make freeze_graph accessible from command line from pip package.
---
 tensorflow/tools/pip_package/setup.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/tools/pip_package/setup.py b/tensorflow/tools/pip_package/setup.py
index 2ffaf7b1aa..02723f3e79 100644
--- a/tensorflow/tools/pip_package/setup.py
+++ b/tensorflow/tools/pip_package/setup.py
@@ -67,6 +67,7 @@ if sys.version_info < (3, 4):
 
 # pylint: disable=line-too-long
 CONSOLE_SCRIPTS = [
+    'freeze_graph = tensorflow.python.tools.freeze_graph:main',
     'saved_model_cli = tensorflow.python.tools.saved_model_cli:main',
     # We need to keep the TensorBoard command, even though the console script
     # is now declared by the tensorboard pip package. If we remove the
-- 
GitLab


From 531b66789bd25291266aba1fc7f7d33ece7089ce Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 16 Oct 2017 10:53:09 -0700
Subject: [PATCH 430/909] Added a cleaner mechanism to set the global constants
 in fisher_blocks.py and fisher_factors.py in the form of a function
 "set_global_constants".

The old way of just manually setting these constants by importing the specific modules and accessing them directly should still work, but this new method is preferred.

PiperOrigin-RevId: 172345996
---
 .../contrib/kfac/python/ops/fisher_blocks.py  |  8 +++++++
 .../kfac/python/ops/fisher_blocks_lib.py      |  3 ++-
 .../contrib/kfac/python/ops/fisher_factors.py | 22 ++++++++++++++++---
 .../kfac/python/ops/fisher_factors_lib.py     |  1 +
 4 files changed, 30 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/kfac/python/ops/fisher_blocks.py b/tensorflow/contrib/kfac/python/ops/fisher_blocks.py
index 3bae45b324..9d8bb8c8ce 100644
--- a/tensorflow/contrib/kfac/python/ops/fisher_blocks.py
+++ b/tensorflow/contrib/kfac/python/ops/fisher_blocks.py
@@ -34,6 +34,14 @@ from tensorflow.python.ops import math_ops
 NORMALIZE_DAMPING_POWER = 1.0
 
 
+def set_global_constants(normalize_damping_power=None):
+  """Sets various global constants used by the classes in this module."""
+  global NORMALIZE_DAMPING_POWER
+
+  if normalize_damping_power is not None:
+    NORMALIZE_DAMPING_POWER = normalize_damping_power
+
+
 @six.add_metaclass(abc.ABCMeta)
 class FisherBlock(object):
   """Abstract base class for objects modeling approximate Fisher matrix blocks.
diff --git a/tensorflow/contrib/kfac/python/ops/fisher_blocks_lib.py b/tensorflow/contrib/kfac/python/ops/fisher_blocks_lib.py
index c6cc169b37..59389f8d38 100644
--- a/tensorflow/contrib/kfac/python/ops/fisher_blocks_lib.py
+++ b/tensorflow/contrib/kfac/python/ops/fisher_blocks_lib.py
@@ -31,7 +31,8 @@ _allowed_symbols = [
     'KroneckerProductFB',
     'FullyConnectedKFACBasicFB',
     'ConvKFCBasicFB',
-    'ConvDiagonalFB'
+    'ConvDiagonalFB',
+    'set_global_constants',
 ]
 
 remove_undocumented(__name__, allowed_exception_list=_allowed_symbols)
diff --git a/tensorflow/contrib/kfac/python/ops/fisher_factors.py b/tensorflow/contrib/kfac/python/ops/fisher_factors.py
index eacd9f53b1..d3c783ee2f 100644
--- a/tensorflow/contrib/kfac/python/ops/fisher_factors.py
+++ b/tensorflow/contrib/kfac/python/ops/fisher_factors.py
@@ -33,9 +33,6 @@ from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops import variables
 from tensorflow.python.training import moving_averages
 
-# TODO(someone): come up with a better mechanism to set these constants
-# externally. See b/67084987
-
 # Whether to initialize covariance estimators at a zero matrix (or the identity
 # matrix).
 INIT_COVARIANCES_AT_ZERO = False
@@ -53,6 +50,25 @@ EIGENVALUE_DECOMPOSITION_THRESHOLD = 2
 EIGENVALUE_CLIPPING_THRESHOLD = 0.0
 
 
+def set_global_constants(init_covariances_at_zero=None, zero_debias=None,
+                         eigenvalue_decomposition_threshold=None,
+                         eigenvalue_clipping_threshold=None):
+  """Sets various global constants used by the classes in this module."""
+  global INIT_COVARIANCES_AT_ZERO
+  global ZERO_DEBIAS
+  global EIGENVALUE_DECOMPOSITION_THRESHOLD
+  global EIGENVALUE_CLIPPING_THRESHOLD
+
+  if init_covariances_at_zero is not None:
+    INIT_COVARIANCES_AT_ZERO = init_covariances_at_zero
+  if zero_debias is not None:
+    ZERO_DEBIAS = zero_debias
+  if eigenvalue_decomposition_threshold is not None:
+    EIGENVALUE_DECOMPOSITION_THRESHOLD = eigenvalue_decomposition_threshold
+  if eigenvalue_clipping_threshold is not None:
+    EIGENVALUE_CLIPPING_THRESHOLD = eigenvalue_clipping_threshold
+
+
 def inverse_initializer(shape, dtype, partition_info=None):  # pylint: disable=unused-argument
   return array_ops.diag(array_ops.ones(shape[0], dtype))
 
diff --git a/tensorflow/contrib/kfac/python/ops/fisher_factors_lib.py b/tensorflow/contrib/kfac/python/ops/fisher_factors_lib.py
index 49a07b1598..23ee93cd40 100644
--- a/tensorflow/contrib/kfac/python/ops/fisher_factors_lib.py
+++ b/tensorflow/contrib/kfac/python/ops/fisher_factors_lib.py
@@ -40,6 +40,7 @@ _allowed_symbols = [
     "ConvInputKroneckerFactor",
     "ConvOutputKroneckerFactor",
     "ConvDiagonalFactor",
+    "set_global_constants",
 ]
 
 remove_undocumented(__name__, allowed_exception_list=_allowed_symbols)
-- 
GitLab


From f8471a8012e823795f5d29f728f36e3e02dbb353 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 16 Oct 2017 10:55:01 -0700
Subject: [PATCH 431/909] Fix xla_jit_compiled_cpu_function deps to pull in
 cpu_plugin.

The intention was always for the user to only depend on
xla_jit_compiled_cpu_function, and not need dependencies on internal targets.

PiperOrigin-RevId: 172346257
---
 tensorflow/compiler/tf2xla/BUILD | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/compiler/tf2xla/BUILD b/tensorflow/compiler/tf2xla/BUILD
index 7865f16e53..3c94bcafc1 100644
--- a/tensorflow/compiler/tf2xla/BUILD
+++ b/tensorflow/compiler/tf2xla/BUILD
@@ -87,6 +87,7 @@ cc_library(
         "//tensorflow/compiler/xla:xla_data_proto",
         "//tensorflow/compiler/xla/client:client_library",
         "//tensorflow/compiler/xla/client:local_client",
+        "//tensorflow/compiler/xla/service:cpu_plugin",
         "//tensorflow/compiler/xla/service/cpu:cpu_executable",
         "//tensorflow/core:lib",
         "//tensorflow/core:protos_all_cc",
@@ -227,7 +228,6 @@ tf_cc_test(
         "//tensorflow/compiler/xla:statusor",
         "//tensorflow/compiler/xla:xla_data_proto",
         "//tensorflow/compiler/xla/client:local_client",
-        "//tensorflow/compiler/xla/service:cpu_plugin",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:protos_all_cc",
-- 
GitLab


From cc5268be7d251e5116229f83aacab80ae6dd917f Mon Sep 17 00:00:00 2001
From: Derek Murray <mrry@google.com>
Date: Mon, 16 Oct 2017 10:57:04 -0700
Subject: [PATCH 432/909] [tf.data] Fix broken implementation of
 `Dataset.from_generator()` on Windows.

Due to a mix-up between NumPy's default array element type for a Python `int` on Windows and Linux, a tf.py_func() in `Dataset.from_generator()` would appear to return the wrong type on Windows (np.int32 instead of np.int64).

All code using `Dataset.from_generator()` on Windows was previously broken. This change fixes both `tf.data.Dataset.from_generator()` and `tf.contrib.data.Dataset.from_generator()`. It also enables test coverage for this method on Windows, which should prevent future breakage.

PiperOrigin-RevId: 172346533
---
 tensorflow/contrib/cmake/tf_tests.cmake       |   1 -
 .../contrib/data/python/ops/dataset_ops.py    | 123 +-------
 tensorflow/python/data/ops/dataset_ops.py     |   5 +-
 tensorflow/python/kernel_tests/BUILD          |  21 ++
 .../dataset_constructor_op_test.py            | 265 +---------------
 .../dataset_from_generator_op_test.py         | 286 ++++++++++++++++++
 6 files changed, 321 insertions(+), 380 deletions(-)
 create mode 100644 tensorflow/python/kernel_tests/dataset_from_generator_op_test.py

diff --git a/tensorflow/contrib/cmake/tf_tests.cmake b/tensorflow/contrib/cmake/tf_tests.cmake
index a560807fb6..1d58b1d416 100644
--- a/tensorflow/contrib/cmake/tf_tests.cmake
+++ b/tensorflow/contrib/cmake/tf_tests.cmake
@@ -255,7 +255,6 @@ if (tensorflow_BUILD_PYTHON_TESTS)
       # Dataset tests
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/dataset_constructor_op_test.py"
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/iterator_ops_cluster_test.py"
-      "${tensorflow_source_dir}/tensorflow/contrib/data/python/kernel_tests/batch_dataset_op_test.py" # b/67743142
       # Broken tensorboard test due to cmake issues.
       "${tensorflow_source_dir}/tensorflow/contrib/data/python/kernel_tests/iterator_ops_cluster_test.py"  # Needs portpicker
       "${tensorflow_source_dir}/tensorflow/contrib/data/python/kernel_tests/sloppy_transformation_dataset_op_test.py"  # b/65430561
diff --git a/tensorflow/contrib/data/python/ops/dataset_ops.py b/tensorflow/contrib/data/python/ops/dataset_ops.py
index fe1d50db33..45d6dbe743 100644
--- a/tensorflow/contrib/data/python/ops/dataset_ops.py
+++ b/tensorflow/contrib/data/python/ops/dataset_ops.py
@@ -24,11 +24,8 @@ from tensorflow.contrib.data.python.ops import grouping
 
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.util import nest
-from tensorflow.python.framework import dtypes
-from tensorflow.python.framework import tensor_shape
 from tensorflow.python.ops import gen_dataset_ops
 from tensorflow.python.ops import gen_io_ops
-from tensorflow.python.ops import script_ops
 from tensorflow.python.util import deprecation
 
 
@@ -139,124 +136,8 @@ class Dataset(dataset_ops.Dataset):
     Returns:
       A `Dataset`.
     """
-    if not callable(generator):
-      raise TypeError("`generator` must be callable.")
-    if output_shapes is None:
-      output_shapes = nest.map_structure(
-          lambda _: tensor_shape.TensorShape(None), output_types)
-    else:
-      output_shapes = nest.map_structure_up_to(
-          output_types, tensor_shape.as_shape, output_shapes)
-
-    flattened_types = nest.flatten(output_types)
-    flattened_shapes = nest.flatten(output_shapes)
-
-    generator_state = dataset_ops.Dataset._GeneratorState(generator)
-
-    def get_iterator_id_map_fn(unused_dummy):
-      """Creates a unique `iterator_id` for each pass over the dataset.
-
-      The "iterator_id" disambiguates between multiple concurrently
-      existing iterators.
-
-      Args:
-        unused_dummy: Ignored value.
-
-      Returns:
-        A `tf.int64` tensor whose value uniquely identifies an iterator in
-        `generator_state`.
-      """
-      return script_ops.py_func(
-          generator_state.get_next_id, [], dtypes.int64, stateful=True)
-
-    def generator_map_fn(iterator_id_t):
-      """Generates the next element from iterator with ID `iterator_id_t`.
-
-      We map this function across an infinite repetition of the
-      `iterator_id_t`, and raise `StopIteration` to terminate the iteration.
-
-      Args:
-        iterator_id_t: A `tf.int64` tensor whose value uniquely identifies
-          the iterator in `generator_state` from which to generate an element.
-
-      Returns:
-        A nested structure of tensors representing an element from the iterator.
-      """
-
-      def generator_py_func(iterator_id):
-        """A `py_func` that will be called to invoke the iterator."""
-        try:
-          values = next(generator_state.get_iterator(iterator_id))
-        except StopIteration:
-          generator_state.iterator_completed(iterator_id)
-          raise StopIteration("Iteration finished.")
-
-        # Use the same _convert function from the py_func() implementation to
-        # convert the returned values to arrays early, so that we can inspect
-        # their values.
-        # pylint: disable=protected-access
-        ret_arrays = [
-            script_ops.FuncRegistry._convert(ret, dtype=dtype.as_numpy_dtype)
-            for ret, dtype in zip(nest.flatten_up_to(output_types, values),
-                                  flattened_types)
-        ]
-        # pylint: enable=protected-access
-
-        # Additional type and shape checking to ensure that the components
-        # of the generated element match the `output_types` and `output_shapes`
-        # arguments.
-        for (ret_array, expected_dtype, expected_shape) in zip(
-            ret_arrays, flattened_types, flattened_shapes):
-          if ret_array.dtype != expected_dtype.as_numpy_dtype:
-            raise TypeError(
-                "`generator` yielded an element of type %s where an element "
-                "of type %s was expected." % (ret_array.dtype,
-                                              expected_dtype.as_numpy_dtype))
-          if not expected_shape.is_compatible_with(ret_array.shape):
-            raise ValueError(
-                "`generator` yielded an element of shape %s where an element "
-                "of shape %s was expected." % (ret_array.shape, expected_shape))
-
-        return ret_arrays
-
-      flat_values = script_ops.py_func(
-          generator_py_func, [iterator_id_t], flattened_types, stateful=True)
-
-      # The `py_func()` op drops the inferred shapes, so we add them back in
-      # here.
-      if output_shapes is not None:
-        for ret_t, shape in zip(flat_values, flattened_shapes):
-          ret_t.set_shape(shape)
-
-      return nest.pack_sequence_as(output_types, flat_values)
-
-    # This function associates each traversal of `generator` with a unique
-    # iterator ID.
-    def flat_map_fn(iterator_id_t):
-      # First, generate an infinite dataset containing the iterator ID repeated
-      # forever.
-      repeated_id = Dataset.from_tensors(iterator_id_t).repeat(None)
-
-      # The `generator_map_fn` gets the next element from the iterator with the
-      # relevant ID, and raises StopIteration when that iterator contains no
-      # more elements.
-      return repeated_id.map(generator_map_fn)
-
-    # A single-element dataset that, each time it is evaluated, contains a
-    # freshly-generated and unique (for the returned dataset) int64
-    # ID that will be used to identify the appropriate Python state, which
-    # is encapsulated in `generator_state`, and captured in
-    # `get_iterator_id_map_fn`.
-    dummy = 0
-    id_dataset = Dataset.from_tensors(dummy).map(get_iterator_id_map_fn)
-
-    # A dataset that contains all of the elements generated by a
-    # single iterator created from `generator`, identified by the
-    # iterator ID contained in `id_dataset`. Lifting the iteration
-    # into a flat_map here enables multiple repetitions and/or nested
-    # versions of the returned dataset to be created, because it forces
-    # the generation of a new ID for each version.
-    return id_dataset.flat_map(flat_map_fn)
+    return Dataset(dataset_ops.Dataset.from_generator(
+        generator, output_types, output_shapes))
 
   @staticmethod
   @deprecation.deprecated(None, "Use `tf.data.Dataset.range()`.")
diff --git a/tensorflow/python/data/ops/dataset_ops.py b/tensorflow/python/data/ops/dataset_ops.py
index 9ea6a2cf8e..5f2e6296a8 100644
--- a/tensorflow/python/data/ops/dataset_ops.py
+++ b/tensorflow/python/data/ops/dataset_ops.py
@@ -201,7 +201,10 @@ class Dataset(object):
       with self._lock:
         ret = self._next_id
         self._next_id += 1
-      return ret
+      # NOTE(mrry): Explicitly create an array of `np.int64` because implicit
+      # casting in `py_func()` will create an array of `np.int32` on Windows,
+      # leading to a runtime error.
+      return np.array(ret, dtype=np.int64)
 
     def get_iterator(self, iterator_id):
       return self._iterators[iterator_id]
diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD
index d6eba3c31a..1380ef5b6a 100644
--- a/tensorflow/python/kernel_tests/BUILD
+++ b/tensorflow/python/kernel_tests/BUILD
@@ -2890,6 +2890,27 @@ tf_py_test(
     ],
 )
 
+tf_py_test(
+    name = "dataset_from_generator_op_test",
+    size = "small",
+    srcs = ["dataset_from_generator_op_test.py"],
+    additional_deps = [
+        "//third_party/py/numpy",
+        "//tensorflow/core:protos_all_py",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:dtypes",
+        "//tensorflow/python:errors",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:resource_variable_ops",
+        "//tensorflow/python:session",
+        "//tensorflow/python:sparse_tensor",
+        "//tensorflow/python/data/ops:dataset_ops",
+        "//tensorflow/python/data/util:nest",
+    ],
+)
+
 tf_py_test(
     name = "filter_dataset_op_test",
     size = "small",
diff --git a/tensorflow/python/kernel_tests/dataset_constructor_op_test.py b/tensorflow/python/kernel_tests/dataset_constructor_op_test.py
index 0dcce727a3..b51d483b5b 100644
--- a/tensorflow/python/kernel_tests/dataset_constructor_op_test.py
+++ b/tensorflow/python/kernel_tests/dataset_constructor_op_test.py
@@ -17,8 +17,6 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import threading
-
 import numpy as np
 
 from tensorflow.core.protobuf import config_pb2
@@ -153,8 +151,9 @@ class DatasetConstructorTest(test.TestCase):
 
   # pylint: disable=g-long-lambda,unnecessary-lambda
   def testNestedStructure(self):
-    components = (np.array([1, 2, 3]), (np.array([4., 5.]), np.array([6., 7.])),
-                  np.array([8, 9, 10]))
+    components = (np.array([1, 2, 3], dtype=np.int64),
+                  (np.array([4., 5.]), np.array([6., 7.])),
+                  np.array([8, 9, 10], dtype=np.int64))
 
     dataset = dataset_ops.Dataset.from_tensors(components)
     self.assertEquals((dtypes.int64, (dtypes.float64, dtypes.float64),
@@ -227,8 +226,10 @@ class DatasetConstructorTest(test.TestCase):
 
     # Define a separate set of components with matching leading
     # dimension for the from-slices constructor.
-    components_for_slices = (np.array([1, 2, 3]), (np.array(
-        [4., 5., 6.]), np.array([7., 8., 9.])), np.array([10, 11, 12]))
+    components_for_slices = (np.array([1, 2, 3], dtype=np.int64),
+                             (np.array([4., 5., 6.]),
+                              np.array([7., 8., 9.])),
+                             np.array([10, 11, 12], dtype=np.int64))
 
     dataset = dataset_ops.Dataset.from_tensor_slices(components_for_slices)
     self.assertEquals((dtypes.int64, (dtypes.float64, dtypes.float64),
@@ -246,7 +247,7 @@ class DatasetConstructorTest(test.TestCase):
     self.assertEquals([3], dataset.output_shapes["b"])
 
   def testNonSequenceNestedStructure(self):
-    components = np.array([1, 2, 3])
+    components = np.array([1, 2, 3], dtype=np.int64)
 
     dataset = dataset_ops.Dataset.from_tensors(components)
     self.assertEquals(dtypes.int64, dataset.output_types)
@@ -271,256 +272,6 @@ class DatasetConstructorTest(test.TestCase):
     self.assertEquals(dtypes.int64, get_next.dtype)
     self.assertEquals([3], get_next.shape)
 
-  def _testFromGenerator(self, generator, elem_sequence, num_repeats):
-    iterator = (
-        dataset_ops.Dataset.from_generator(generator, output_types=dtypes.int64)
-        .repeat(num_repeats)
-        .prefetch(5)
-        .make_initializable_iterator())
-    init_op = iterator.initializer
-    get_next = iterator.get_next()
-
-    with self.test_session() as sess:
-      for _ in range(2):  # Run twice to test reinitialization.
-        sess.run(init_op)
-        for _ in range(num_repeats):
-          for elem in elem_sequence:
-            self.assertAllEqual(elem, sess.run(get_next))
-        with self.assertRaises(errors.OutOfRangeError):
-          sess.run(get_next)
-
-  def _testFromGeneratorOneShot(self, generator, elem_sequence, num_repeats):
-    iterator = (
-        dataset_ops.Dataset.from_generator(generator, output_types=dtypes.int64)
-        .repeat(num_repeats)
-        .prefetch(5)
-        .make_one_shot_iterator())
-    get_next = iterator.get_next()
-
-    with self.test_session() as sess:
-      for _ in range(num_repeats):
-        for elem in elem_sequence:
-          self.assertAllEqual(elem, sess.run(get_next))
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
-
-  def testFromGeneratorUsingFunction(self):
-    def generator():
-      for i in range(1, 100):
-        yield [i] * i
-    elem_sequence = list(generator())
-    self._testFromGenerator(generator, elem_sequence, 1)
-    self._testFromGenerator(generator, elem_sequence, 5)
-    self._testFromGeneratorOneShot(generator, elem_sequence, 1)
-    self._testFromGeneratorOneShot(generator, elem_sequence, 5)
-
-  def testFromGeneratorUsingList(self):
-    generator = lambda: [[i] * i for i in range(1, 100)]
-    elem_sequence = list(generator())
-    self._testFromGenerator(generator, elem_sequence, 1)
-    self._testFromGenerator(generator, elem_sequence, 5)
-
-  def testFromGeneratorUsingNdarray(self):
-    generator = lambda: np.arange(100, dtype=np.int64)
-    elem_sequence = list(generator())
-    self._testFromGenerator(generator, elem_sequence, 1)
-    self._testFromGenerator(generator, elem_sequence, 5)
-
-  def testFromGeneratorUsingGeneratorExpression(self):
-    # NOTE(mrry): Generator *expressions* are not repeatable (or in
-    # general reusable), because they eagerly evaluate the `for`
-    # expression as `iter(range(1, 100))` and discard the means of
-    # reconstructing `range(1, 100)`. Wrapping the generator
-    # expression in a `lambda` makes it repeatable.
-    generator = lambda: ([i] * i for i in range(1, 100))
-    elem_sequence = list(generator())
-    self._testFromGenerator(generator, elem_sequence, 1)
-    self._testFromGenerator(generator, elem_sequence, 5)
-
-  def testFromMultipleConcurrentGenerators(self):
-    num_inner_repeats = 5
-    num_outer_repeats = 100
-
-    def generator():
-      for i in range(1, 10):
-        yield ([i] * i, [i, i ** 2, i ** 3])
-    input_list = list(generator())
-
-    # The interleave transformation is essentially a flat map that
-    # draws from multiple input datasets concurrently (in a cyclic
-    # fashion). By placing `Datsaet.from_generator()` inside an
-    # interleave, we test its behavior when multiple iterators are
-    # active at the same time; by additionally prefetching inside the
-    # interleave, we create the possibility of parallel (modulo GIL)
-    # invocations to several iterators created by the same dataset.
-    def interleave_fn(_):
-      return (dataset_ops.Dataset.from_generator(
-          generator, output_types=(dtypes.int64, dtypes.int64),
-          output_shapes=([None], [3]))
-              .repeat(num_inner_repeats).prefetch(5))
-
-    iterator = (
-        dataset_ops.Dataset.range(num_outer_repeats)
-        .interleave(interleave_fn, cycle_length=10,
-                    block_length=len(input_list))
-        .make_initializable_iterator())
-    init_op = iterator.initializer
-    get_next = iterator.get_next()
-
-    with self.test_session() as sess:
-      sess.run(init_op)
-      for _ in range(num_inner_repeats * num_outer_repeats):
-        for elem in input_list:
-          val0, val1 = sess.run(get_next)
-          self.assertAllEqual(elem[0], val0)
-          self.assertAllEqual(elem[1], val1)
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
-
-  def testFromGeneratorsRunningInParallel(self):
-    num_parallel_iterators = 3
-
-    # Define shared state that multiple iterator instances will access to
-    # demonstrate their concurrent activity.
-    lock = threading.Lock()
-    condition = threading.Condition(lock)
-    next_ticket = [0]  # GUARDED_BY(lock)
-
-    def generator():
-      # NOTE(mrry): We yield one element before the barrier, because
-      # the current implementation of `Dataset.interleave()` must
-      # fetch one element from each incoming dataset to start the
-      # prefetching.
-      yield 0
-
-      # Define a barrier that `num_parallel_iterators` iterators must enter
-      # before any can proceed. Demonstrates that multiple iterators may be
-      # active at the same time.
-      condition.acquire()
-      ticket = next_ticket[0]
-      next_ticket[0] += 1
-      if ticket == num_parallel_iterators - 1:
-        # The last iterator to join the barrier notifies the others.
-        condition.notify_all()
-      else:
-        # Wait until the last iterator enters the barrier.
-        while next_ticket[0] < num_parallel_iterators:
-          condition.wait()
-      condition.release()
-
-      yield 1
-
-    # As in `testFromMultipleConcurrentGenerators()`, we use a combination of
-    # `Dataset.interleave()` and `Dataset.prefetch()` to cause multiple
-    # iterators to be active concurrently.
-    def interleave_fn(_):
-      return dataset_ops.Dataset.from_generator(
-          generator, output_types=dtypes.int64, output_shapes=[]).prefetch(2)
-
-    iterator = (
-        dataset_ops.Dataset.range(num_parallel_iterators)
-        .interleave(
-            interleave_fn, cycle_length=num_parallel_iterators, block_length=1)
-        .make_initializable_iterator())
-    init_op = iterator.initializer
-    get_next = iterator.get_next()
-
-    with self.test_session() as sess:
-      sess.run(init_op)
-      for elem in [0, 1]:
-        for _ in range(num_parallel_iterators):
-          self.assertAllEqual(elem, sess.run(get_next))
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
-
-  def testFromGeneratorImplicitConversion(self):
-    def generator():
-      yield [1]
-      yield [2]
-      yield [3]
-
-    for dtype in [dtypes.int8, dtypes.int32, dtypes.int64]:
-      iterator = (dataset_ops.Dataset.from_generator(
-          generator, output_types=dtype, output_shapes=[1])
-                  .make_initializable_iterator())
-      init_op = iterator.initializer
-      get_next = iterator.get_next()
-
-      self.assertEqual(dtype, get_next.dtype)
-
-      with self.test_session() as sess:
-        sess.run(init_op)
-        for expected in [[1], [2], [3]]:
-          next_val = sess.run(get_next)
-          self.assertEqual(dtype.as_numpy_dtype, next_val.dtype)
-          self.assertAllEqual(expected, next_val)
-        with self.assertRaises(errors.OutOfRangeError):
-          sess.run(get_next)
-
-  def testFromGeneratorTypeError(self):
-    def generator():
-      yield np.array([1, 2, 3], dtype=np.int64)
-      yield np.array([4, 5, 6], dtype=np.int64)
-      yield "ERROR"
-      yield np.array([7, 8, 9], dtype=np.int64)
-
-    iterator = (dataset_ops.Dataset.from_generator(
-        generator, output_types=dtypes.int64, output_shapes=[3])
-                .make_initializable_iterator())
-    init_op = iterator.initializer
-    get_next = iterator.get_next()
-
-    with self.test_session() as sess:
-      sess.run(init_op)
-      self.assertAllEqual([1, 2, 3], sess.run(get_next))
-      self.assertAllEqual([4, 5, 6], sess.run(get_next))
-      with self.assertRaisesOpError(r"invalid literal for long\(\)"):
-        sess.run(get_next)
-      self.assertAllEqual([7, 8, 9], sess.run(get_next))
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
-
-  def testFromGeneratorShapeError(self):
-    def generator():
-      yield np.array([1, 2, 3], dtype=np.int64)
-      yield np.array([4, 5, 6], dtype=np.int64)
-      yield np.array([7, 8, 9, 10], dtype=np.int64)
-      yield np.array([11, 12, 13], dtype=np.int64)
-
-    iterator = (dataset_ops.Dataset.from_generator(
-        generator, output_types=dtypes.int64, output_shapes=[3])
-                .make_initializable_iterator())
-    init_op = iterator.initializer
-    get_next = iterator.get_next()
-
-    with self.test_session() as sess:
-      sess.run(init_op)
-      self.assertAllEqual([1, 2, 3], sess.run(get_next))
-      self.assertAllEqual([4, 5, 6], sess.run(get_next))
-      with self.assertRaisesOpError(r"element of shape \(3,\) was expected"):
-        sess.run(get_next)
-      self.assertAllEqual([11, 12, 13], sess.run(get_next))
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
-
-  def testFromGeneratorHeterogeneous(self):
-    def generator():
-      yield 1
-      yield [2, 3]
-
-    iterator = (
-        dataset_ops.Dataset.from_generator(
-            generator, output_types=dtypes.int64).make_initializable_iterator())
-    init_op = iterator.initializer
-    get_next = iterator.get_next()
-
-    with self.test_session() as sess:
-      sess.run(init_op)
-      self.assertAllEqual(1, sess.run(get_next))
-      self.assertAllEqual([2, 3], sess.run(get_next))
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
-
   def testSplitPipelineFailsWithPlacementError(self):
     with session.Session(
         target="",
diff --git a/tensorflow/python/kernel_tests/dataset_from_generator_op_test.py b/tensorflow/python/kernel_tests/dataset_from_generator_op_test.py
new file mode 100644
index 0000000000..e774256695
--- /dev/null
+++ b/tensorflow/python/kernel_tests/dataset_from_generator_op_test.py
@@ -0,0 +1,286 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for the experimental input pipeline ops."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import threading
+
+import numpy as np
+
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import errors
+from tensorflow.python.platform import test
+
+
+class DatasetConstructorTest(test.TestCase):
+
+  def _testFromGenerator(self, generator, elem_sequence, num_repeats):
+    iterator = (
+        dataset_ops.Dataset.from_generator(generator, output_types=dtypes.int64)
+        .repeat(num_repeats)
+        .prefetch(5)
+        .make_initializable_iterator())
+    init_op = iterator.initializer
+    get_next = iterator.get_next()
+
+    with self.test_session() as sess:
+      for _ in range(2):  # Run twice to test reinitialization.
+        sess.run(init_op)
+        for _ in range(num_repeats):
+          for elem in elem_sequence:
+            self.assertAllEqual(elem, sess.run(get_next))
+        with self.assertRaises(errors.OutOfRangeError):
+          sess.run(get_next)
+
+  def _testFromGeneratorOneShot(self, generator, elem_sequence, num_repeats):
+    iterator = (
+        dataset_ops.Dataset.from_generator(generator, output_types=dtypes.int64)
+        .repeat(num_repeats)
+        .prefetch(5)
+        .make_one_shot_iterator())
+    get_next = iterator.get_next()
+
+    with self.test_session() as sess:
+      for _ in range(num_repeats):
+        for elem in elem_sequence:
+          self.assertAllEqual(elem, sess.run(get_next))
+      with self.assertRaises(errors.OutOfRangeError):
+        sess.run(get_next)
+
+  def testFromGeneratorUsingFunction(self):
+    def generator():
+      for i in range(1, 100):
+        yield [i] * i
+    elem_sequence = list(generator())
+    self._testFromGenerator(generator, elem_sequence, 1)
+    self._testFromGenerator(generator, elem_sequence, 5)
+    self._testFromGeneratorOneShot(generator, elem_sequence, 1)
+    self._testFromGeneratorOneShot(generator, elem_sequence, 5)
+
+  def testFromGeneratorUsingList(self):
+    generator = lambda: [[i] * i for i in range(1, 100)]
+    elem_sequence = list(generator())
+    self._testFromGenerator(generator, elem_sequence, 1)
+    self._testFromGenerator(generator, elem_sequence, 5)
+
+  def testFromGeneratorUsingNdarray(self):
+    generator = lambda: np.arange(100, dtype=np.int64)
+    elem_sequence = list(generator())
+    self._testFromGenerator(generator, elem_sequence, 1)
+    self._testFromGenerator(generator, elem_sequence, 5)
+
+  def testFromGeneratorUsingGeneratorExpression(self):
+    # NOTE(mrry): Generator *expressions* are not repeatable (or in
+    # general reusable), because they eagerly evaluate the `for`
+    # expression as `iter(range(1, 100))` and discard the means of
+    # reconstructing `range(1, 100)`. Wrapping the generator
+    # expression in a `lambda` makes it repeatable.
+    generator = lambda: ([i] * i for i in range(1, 100))
+    elem_sequence = list(generator())
+    self._testFromGenerator(generator, elem_sequence, 1)
+    self._testFromGenerator(generator, elem_sequence, 5)
+
+  def testFromMultipleConcurrentGenerators(self):
+    num_inner_repeats = 5
+    num_outer_repeats = 100
+
+    def generator():
+      for i in range(1, 10):
+        yield ([i] * i, [i, i ** 2, i ** 3])
+    input_list = list(generator())
+
+    # The interleave transformation is essentially a flat map that
+    # draws from multiple input datasets concurrently (in a cyclic
+    # fashion). By placing `Datsaet.from_generator()` inside an
+    # interleave, we test its behavior when multiple iterators are
+    # active at the same time; by additionally prefetching inside the
+    # interleave, we create the possibility of parallel (modulo GIL)
+    # invocations to several iterators created by the same dataset.
+    def interleave_fn(_):
+      return (dataset_ops.Dataset.from_generator(
+          generator, output_types=(dtypes.int64, dtypes.int64),
+          output_shapes=([None], [3]))
+              .repeat(num_inner_repeats).prefetch(5))
+
+    iterator = (
+        dataset_ops.Dataset.range(num_outer_repeats)
+        .interleave(interleave_fn, cycle_length=10,
+                    block_length=len(input_list))
+        .make_initializable_iterator())
+    init_op = iterator.initializer
+    get_next = iterator.get_next()
+
+    with self.test_session() as sess:
+      sess.run(init_op)
+      for _ in range(num_inner_repeats * num_outer_repeats):
+        for elem in input_list:
+          val0, val1 = sess.run(get_next)
+          self.assertAllEqual(elem[0], val0)
+          self.assertAllEqual(elem[1], val1)
+      with self.assertRaises(errors.OutOfRangeError):
+        sess.run(get_next)
+
+  def testFromGeneratorsRunningInParallel(self):
+    num_parallel_iterators = 3
+
+    # Define shared state that multiple iterator instances will access to
+    # demonstrate their concurrent activity.
+    lock = threading.Lock()
+    condition = threading.Condition(lock)
+    next_ticket = [0]  # GUARDED_BY(lock)
+
+    def generator():
+      # NOTE(mrry): We yield one element before the barrier, because
+      # the current implementation of `Dataset.interleave()` must
+      # fetch one element from each incoming dataset to start the
+      # prefetching.
+      yield 0
+
+      # Define a barrier that `num_parallel_iterators` iterators must enter
+      # before any can proceed. Demonstrates that multiple iterators may be
+      # active at the same time.
+      condition.acquire()
+      ticket = next_ticket[0]
+      next_ticket[0] += 1
+      if ticket == num_parallel_iterators - 1:
+        # The last iterator to join the barrier notifies the others.
+        condition.notify_all()
+      else:
+        # Wait until the last iterator enters the barrier.
+        while next_ticket[0] < num_parallel_iterators:
+          condition.wait()
+      condition.release()
+
+      yield 1
+
+    # As in `testFromMultipleConcurrentGenerators()`, we use a combination of
+    # `Dataset.interleave()` and `Dataset.prefetch()` to cause multiple
+    # iterators to be active concurrently.
+    def interleave_fn(_):
+      return dataset_ops.Dataset.from_generator(
+          generator, output_types=dtypes.int64, output_shapes=[]).prefetch(2)
+
+    iterator = (
+        dataset_ops.Dataset.range(num_parallel_iterators)
+        .interleave(
+            interleave_fn, cycle_length=num_parallel_iterators, block_length=1)
+        .make_initializable_iterator())
+    init_op = iterator.initializer
+    get_next = iterator.get_next()
+
+    with self.test_session() as sess:
+      sess.run(init_op)
+      for elem in [0, 1]:
+        for _ in range(num_parallel_iterators):
+          self.assertAllEqual(elem, sess.run(get_next))
+      with self.assertRaises(errors.OutOfRangeError):
+        sess.run(get_next)
+
+  def testFromGeneratorImplicitConversion(self):
+    def generator():
+      yield [1]
+      yield [2]
+      yield [3]
+
+    for dtype in [dtypes.int8, dtypes.int32, dtypes.int64]:
+      iterator = (dataset_ops.Dataset.from_generator(
+          generator, output_types=dtype, output_shapes=[1])
+                  .make_initializable_iterator())
+      init_op = iterator.initializer
+      get_next = iterator.get_next()
+
+      self.assertEqual(dtype, get_next.dtype)
+
+      with self.test_session() as sess:
+        sess.run(init_op)
+        for expected in [[1], [2], [3]]:
+          next_val = sess.run(get_next)
+          self.assertEqual(dtype.as_numpy_dtype, next_val.dtype)
+          self.assertAllEqual(expected, next_val)
+        with self.assertRaises(errors.OutOfRangeError):
+          sess.run(get_next)
+
+  def testFromGeneratorTypeError(self):
+    def generator():
+      yield np.array([1, 2, 3], dtype=np.int64)
+      yield np.array([4, 5, 6], dtype=np.int64)
+      yield "ERROR"
+      yield np.array([7, 8, 9], dtype=np.int64)
+
+    iterator = (dataset_ops.Dataset.from_generator(
+        generator, output_types=dtypes.int64, output_shapes=[3])
+                .make_initializable_iterator())
+    init_op = iterator.initializer
+    get_next = iterator.get_next()
+
+    with self.test_session() as sess:
+      sess.run(init_op)
+      self.assertAllEqual([1, 2, 3], sess.run(get_next))
+      self.assertAllEqual([4, 5, 6], sess.run(get_next))
+      # NOTE(mrry): Type name in message differs between Python 2 (`long`) and
+      # 3 (`int`).
+      with self.assertRaisesOpError(r"invalid literal for"):
+        sess.run(get_next)
+      self.assertAllEqual([7, 8, 9], sess.run(get_next))
+      with self.assertRaises(errors.OutOfRangeError):
+        sess.run(get_next)
+
+  def testFromGeneratorShapeError(self):
+    def generator():
+      yield np.array([1, 2, 3], dtype=np.int64)
+      yield np.array([4, 5, 6], dtype=np.int64)
+      yield np.array([7, 8, 9, 10], dtype=np.int64)
+      yield np.array([11, 12, 13], dtype=np.int64)
+
+    iterator = (dataset_ops.Dataset.from_generator(
+        generator, output_types=dtypes.int64, output_shapes=[3])
+                .make_initializable_iterator())
+    init_op = iterator.initializer
+    get_next = iterator.get_next()
+
+    with self.test_session() as sess:
+      sess.run(init_op)
+      self.assertAllEqual([1, 2, 3], sess.run(get_next))
+      self.assertAllEqual([4, 5, 6], sess.run(get_next))
+      with self.assertRaisesOpError(r"element of shape \(3,\) was expected"):
+        sess.run(get_next)
+      self.assertAllEqual([11, 12, 13], sess.run(get_next))
+      with self.assertRaises(errors.OutOfRangeError):
+        sess.run(get_next)
+
+  def testFromGeneratorHeterogeneous(self):
+    def generator():
+      yield 1
+      yield [2, 3]
+
+    iterator = (
+        dataset_ops.Dataset.from_generator(
+            generator, output_types=dtypes.int64).make_initializable_iterator())
+    init_op = iterator.initializer
+    get_next = iterator.get_next()
+
+    with self.test_session() as sess:
+      sess.run(init_op)
+      self.assertAllEqual(1, sess.run(get_next))
+      self.assertAllEqual([2, 3], sess.run(get_next))
+      with self.assertRaises(errors.OutOfRangeError):
+        sess.run(get_next)
+
+
+if __name__ == "__main__":
+  test.main()
-- 
GitLab


From f0e3edf8b1c8de49672d78abe73dcd0b1f02620c Mon Sep 17 00:00:00 2001
From: Jacques Pienaar <jpienaar@google.com>
Date: Mon, 16 Oct 2017 11:09:36 -0700
Subject: [PATCH 433/909] [TF2XLA] Keep Switch and Merge nodes in own clusters.

* Keep Switch and Merge nodes in separate clusters to avoid creating irreducible graphs;
* Merge Switch nodes with common predicates;
* Add support for if-then structure;
* Squash trivial Switch->Merge groups;
* Merge newly Merge free nodes with Switch & Merge free inputs;
* Check to see if it is a Merge node before merging to common merge node;
* Return an error if all Switches have not been replaced;
* Add test fir tf,case;

PiperOrigin-RevId: 172348729
---
 .../tf2xla/functionalize_control_flow.cc      | 233 ++++++++++++++----
 1 file changed, 181 insertions(+), 52 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/functionalize_control_flow.cc b/tensorflow/compiler/tf2xla/functionalize_control_flow.cc
index 40bc164c50..abfc856904 100644
--- a/tensorflow/compiler/tf2xla/functionalize_control_flow.cc
+++ b/tensorflow/compiler/tf2xla/functionalize_control_flow.cc
@@ -74,6 +74,18 @@ struct Frame {
   std::unordered_set<Node*> nodes;
 };
 
+// Returns a textual representation of the names of the nodes in the input.
+template <typename T>
+string NodesToString(const T& nodes) {
+  return strings::StrCat("{",
+                         str_util::Join(nodes, ",",
+                                        [](string* output, const Node* node) {
+                                          strings::StrAppend(output,
+                                                             node->name());
+                                        }),
+                         "}");
+}
+
 // Copies a subgraph from `graph` to `output` by performing a reverse DFS
 // starting at nodes in vector `stack`.
 // `node_map` is a vector indexed by source node ID to dest nodes.
@@ -93,12 +105,13 @@ Status CopySubgraph(const Graph& graph, const Frame* frame,
                     std::vector<Node*> stack,
                     const std::vector<bool>& squash_src_outputs,
                     std::vector<Node*>* node_map, Graph* output) {
+  VLOG(3) << "Stack: " << NodesToString(stack);
   std::vector<bool> visited(graph.num_node_ids(), false);
   while (!stack.empty()) {
     Node* n = stack.back();
     stack.pop_back();
 
-    VLOG(3) << "Copying node " << n->name();
+    VLOG(5) << "Copying node " << n->name();
 
     if (visited[n->id()]) continue;
     visited[n->id()] = true;
@@ -577,7 +590,7 @@ class FunctionalizeCond {
   // id in the original graph.
   struct CondArgs {
     struct CondCmp {
-      bool operator()(const Node* a, const Node* b) {
+      bool operator()(const Node* a, const Node* b) const {
         return a->id() < b->id();
       }
     };
@@ -613,7 +626,10 @@ class FunctionalizeCond {
 
   // If `from` and `to` correspond to different clusters, then merge the nodes
   // in the clustered graph corresponding to `from` and `to`.
-  void ContractEdge(Cluster* from, Cluster* to);
+  //
+  // If `remove_from_graph` is specified then the `from` node is also removed
+  // from the clustered graph post contracting the edge.
+  void ContractEdge(Cluster* from, Cluster* to, bool remove_from_graph = false);
 
   // Converts a Merge node to a XlaIf. This encapsulates the process of
   // extracting the bodies needed for the then and else branch, creates a XlaIf
@@ -621,6 +637,10 @@ class FunctionalizeCond {
   // merge node with a XlaIf.
   Status ConvertMergeToXlaIf(Cluster* merge_cluster);
 
+  // Removes a Switch cluster feeding directly into a Merge cluster by removing
+  // the Switch and Merge nodes and collapsing into a single cluster.
+  Status RemoveTrivialMerge(Cluster* merge_cluster);
+
   // Returns the switch cluster corresponding to the merge node. This function
   // only returns the switch cluster in the simple case where we have a switch
   // node is the entry of a diamond corresponding to a conditional:
@@ -629,7 +649,10 @@ class FunctionalizeCond {
   //          /      \
   //     Branch      Branch
   //          \      /
-  //           merge_cluster
+  //        merge_cluster
+  //
+  // Note: either of the branches may be empty. The case where both branches are
+  // empty is handled by RemoveTrivialMerge.
   gtl::optional<Cluster*> GetSwitchCluster(const Cluster& merge_cluster);
 
   // Determines the arguments needed as input to the Merge cluster originating
@@ -661,8 +684,8 @@ class FunctionalizeCond {
   template <class T>
   void RemoveUnusedArgs(const T& args);
 
-  // Removes all Merge nodes that are unused.
-  void RemoveUnusedMergeNodes(Cluster* merge_cluster);
+  // Removes all Merge nodes in merge_cluster.
+  void RemoveMergeNodes(Cluster* merge_cluster);
 
   // Returns the representative member of the corresponding cluster.
   ClusterHandle Representative(const Node* node) {
@@ -713,6 +736,24 @@ string DebugString(const Graph& graph,
   return strings::StrCat(ret, "}");
 }
 
+string DebugString(const FunctionalizeCond::ClusteredGraph& clustered_graph) {
+  string ret = "digraph {\ncompound=true;labeljust=\"r\";\n";
+  auto name = [](const FunctionalizeCond::Cluster& cluster) {
+    return cluster.representative.ToString();
+  };
+  for (auto kv : clustered_graph) {
+    strings::StrAppend(&ret, kv.first.ToString(), " [label=\"", name(kv.second),
+                       " (", kv.second.switch_nodes.size(), ", ",
+                       kv.second.merge_nodes.size(), ")\"];\n");
+  }
+  for (auto kv : clustered_graph) {
+    for (auto in : kv.second.in_nodes) {
+      strings::StrAppend(&ret, name(*in), " -> ", name(kv.second), ";\n");
+    }
+  }
+  return strings::StrCat(ret, "}");
+}
+
 bool IsDeadSwitch(const Node* node) {
   for (const Edge* e : node->out_edges()) {
     const Node* dst = e->dst();
@@ -754,21 +795,22 @@ void FunctionalizeCond::CreateClusters() {
   // conservatively assuming all merge nodes become XlaIf nodes.
   clusters_.resize(clusters_.size() + merge_nodes_.size());
 
-  // Merge a cluster with its input, unless the input is a Switch node or the
-  // node is a Merge node.
+  // Merge a cluster with its input, unless the input is a Switch node or
+  // the node is a Merge node.
   for (const Node* node : graph_->nodes()) {
-    if (IsMerge(node) || !node->IsOp()) {
+    if (IsMerge(node) || IsSwitch(node) || !node->IsOp()) {
       continue;
     }
     for (const Node* in : node->in_nodes()) {
-      if (!IsSwitch(in) && in->IsOp()) {
+      if (in->IsOp() && !IsSwitch(in) && !IsMerge(in)) {
         clusters_.at(node).Merge(&clusters_.at(in));
       }
     }
   }
 }
 
-void FunctionalizeCond::ContractEdge(Cluster* from, Cluster* to) {
+void FunctionalizeCond::ContractEdge(Cluster* from, Cluster* to,
+                                     bool remove_from_graph) {
   VLOG(3) << "ContractEdge from = " << from->representative
           << " to = " << to->representative;
   if (from->representative == to->representative) {
@@ -801,6 +843,10 @@ void FunctionalizeCond::ContractEdge(Cluster* from, Cluster* to) {
   to->out_nodes.erase(from);
   clusters_.at(to->representative).Merge(&clusters_.at(from->representative));
   from->visited = true;
+
+  if (remove_from_graph) {
+    clustered_graph_.erase(from->representative);
+  }
 }
 
 void FunctionalizeCond::CreateClusteredGraph() {
@@ -839,6 +885,22 @@ void FunctionalizeCond::CreateClusteredGraph() {
     update_cluster_for_node(node).merge_nodes.insert(node);
   }
 
+  // Merge Switch nodes with common predicate.
+  std::unordered_map<Node*, std::vector<Node*>> predicate_to_switch;
+  for (Node* node : switch_nodes_) {
+    Node* tmp;
+    TF_CHECK_OK(node->input_node(1, &tmp));
+    predicate_to_switch[tmp].push_back(node);
+  }
+  for (auto kv : predicate_to_switch) {
+    Cluster& first = clustered_graph_.at(Representative(kv.second.front()));
+    for (Node* switch_node : kv.second) {
+      ClusterHandle handle = Representative(switch_node);
+      Cluster& cluster = clustered_graph_.at(handle);
+      ContractEdge(&cluster, &first, /*remove_from_graph=*/true);
+    }
+  }
+
   // Merge Merge nodes with common input together.
   for (Node* node : merge_nodes_) {
     Cluster& cluster = clustered_graph_.at(Representative(node));
@@ -847,35 +909,47 @@ void FunctionalizeCond::CreateClusteredGraph() {
         continue;
       }
       Cluster& cluster_node_in = clustered_graph_.at(Representative(in));
+      // ContractEdge can modify out_nodes of cluster_node_in, so traverse
+      // over out_nodes assuming it does.
       for (auto it = cluster_node_in.out_nodes.begin();
            it != cluster_node_in.out_nodes.end();) {
-        ContractEdge(*it++, &cluster);
+        if (!(*it)->merge_nodes.empty()) {
+          ContractEdge(*it++, &cluster, /*remove_from_graph=*/true);
+        } else {
+          ++it;
+        }
       }
     }
   }
 
-  VLOG(3) << "ClusteredGraph: " << DebugString(*graph_, &clusters_);
+  VLOG(3) << "Graph with clusters: " << DebugString(*graph_, &clusters_);
+  VLOG(3) << "ClusteredGraph: " << DebugString(clustered_graph_);
 }
 
 gtl::optional<FunctionalizeCond::Cluster*> FunctionalizeCond::GetSwitchCluster(
     const Cluster& merge_cluster) {
   VLOG(3) << "GetSwitchCluster for " << merge_cluster.representative;
   gtl::optional<Cluster*> switch_cluster;
-  if (merge_cluster.in_nodes.size() != 2) {
+  if (merge_cluster.in_nodes.size() > 2) {
     return gtl::nullopt;
   }
-  for (const Cluster* in : merge_cluster.in_nodes) {
-    if (in->in_nodes.size() != 1) {
+  for (Cluster* in : merge_cluster.in_nodes) {
+    Cluster* cluster = in;
+    if (in->switch_nodes.empty()) {
+      if (in->in_nodes.size() != 1) {
+        return gtl::nullopt;
+      }
+      // There is only a single `in` cluster.
+      cluster = *in->in_nodes.begin();
+    }
+    if (cluster->switch_nodes.empty()) {
       return gtl::nullopt;
     }
-    for (auto inin : in->in_nodes) {
-      if (switch_cluster.has_value()) {
-        if (*switch_cluster != inin) {
-          return gtl::nullopt;
-        }
-      } else {
-        switch_cluster = inin;
-      }
+
+    if (switch_cluster.has_value() && *switch_cluster != cluster) {
+      return gtl::nullopt;
+    } else {
+      switch_cluster = cluster;
     }
   }
   return switch_cluster;
@@ -889,6 +963,9 @@ xla::StatusOr<FunctionalizeCond::CondArgs> FunctionalizeCond::DetermineCondArgs(
   auto feeds_into_branch_cluster = [&](Node* switch_cluster) {
     for (Node* out : switch_cluster->out_nodes()) {
       ClusterHandle repr = Representative(out);
+      if (repr == merge_cluster.representative) {
+        return true;
+      }
       for (Cluster* in : merge_cluster.in_nodes) {
         if (repr == in->representative) {
           return true;
@@ -919,12 +996,9 @@ xla::StatusOr<FunctionalizeCond::CondArgs> FunctionalizeCond::DetermineCondArgs(
 xla::StatusOr<Node*> FunctionalizeCond::BuildAndAddXlaIfOp(
     const CondArgs& cond_args, const Cluster& merge_cluster,
     const std::vector<Node*>& outputs) {
-  VLOG(2) << "Build if op for {"
-          << str_util::Join(merge_cluster.merge_nodes, ", ",
-                            [](string* out, const Node* node) {
-                              strings::StrAppend(out, node->name());
-                            })
-          << "}";
+  VLOG(2) << "Build if op for " << NodesToString(merge_cluster.merge_nodes)
+          << " with input " << NodesToString(cond_args.args);
+
   NodeDef if_def;
   // Create a new If node using the name of the merge node.
   NodeDefBuilder builder(
@@ -941,6 +1015,7 @@ xla::StatusOr<Node*> FunctionalizeCond::BuildAndAddXlaIfOp(
     auto body = xla::MakeUnique<Graph>(graph_->op_registry());
     TF_RETURN_IF_ERROR(
         ExtractBody(cond_args, merge_cluster, outputs, i, body.get()));
+    VLOG(3) << "Body " << branch[i] << ": " << DebugString(body.get());
     FunctionDef body_fdef;
     TF_RETURN_IF_ERROR(GraphToFunctionDef(*body, body_name.name(), &body_fdef));
     TF_RETURN_IF_ERROR(library_->AddFunctionDef(body_fdef));
@@ -1001,10 +1076,7 @@ void FunctionalizeCond::RemoveClusterNodes(Cluster* cluster) {
 
 template <class T>
 void FunctionalizeCond::RemoveUnusedArgs(const T& args) {
-  VLOG(2) << "RemoveUnusedArgs among: "
-          << str_util::Join(args, ", ", [](string* output, const Node* node) {
-               strings::StrAppend(output, node->name());
-             });
+  VLOG(2) << "RemoveUnusedArgs among: " << NodesToString(args);
 
   std::deque<Node*> to_delete;
   for (Node* arg : args) {
@@ -1029,7 +1101,8 @@ Status FunctionalizeCond::ExtractBody(const CondArgs& cond_args,
                                       const Cluster& merge_cluster,
                                       const std::vector<Node*>& outputs,
                                       int input_edge, Graph* body) {
-  VLOG(2) << "ExtractBody for " << merge_cluster.representative;
+  VLOG(2) << "ExtractBody for " << merge_cluster.representative
+          << " along edge " << input_edge;
   std::vector<bool> squash_src_outputs(graph_->num_node_ids(), false);
   std::vector<Node*> node_map(graph_->num_node_ids(), nullptr);
   int arg_count = 0;
@@ -1053,12 +1126,21 @@ Status FunctionalizeCond::ExtractBody(const CondArgs& cond_args,
     TF_ASSIGN_OR_RETURN(node_map.at(node->id()),
                         BuildRetvalNode(body, node->output_type(0),
                                         /*index=*/j));
-    Node* in;
-    TF_RETURN_IF_ERROR(node->input_node(input_edge, &in));
+    const Edge* in_edge;
+    TF_RETURN_IF_ERROR(node->input_edge(input_edge, &in_edge));
+    Node* in = in_edge->src();
     if (node_map.at(in->id()) == nullptr) {
       node_map.at(in->id()) = body->CopyNode(in);
     }
-    body->AddEdge(node_map.at(in->id()), j, node_map.at(node->id()), 0);
+
+    if (cond_args.args.find(in) == cond_args.args.end()) {
+      body->AddEdge(node_map.at(in->id()), in_edge->src_output(),
+                    node_map.at(node->id()), 0);
+    } else {
+      body->AddEdge(node_map.at(in->id()), 0, node_map.at(node->id()), 0);
+      // Don't include input nodes that are already just returned in stack.
+      continue;
+    }
     stack.push_back(in);
   }
 
@@ -1108,17 +1190,46 @@ Status FunctionalizeCond::AddOutputEdges(const std::vector<Node*>& outputs,
   return Status::OK();
 }
 
-void FunctionalizeCond::RemoveUnusedMergeNodes(Cluster* merge_cluster) {
-  VLOG(3) << "RemoveUnusedMergeNodes for " << merge_cluster->representative;
+void FunctionalizeCond::RemoveMergeNodes(Cluster* merge_cluster) {
+  VLOG(3) << "RemoveMergeNodes for " << merge_cluster->representative;
   // Remove all merge nodes now dead post extraction of If.
   for (auto it = merge_cluster->merge_nodes.begin();
        it != merge_cluster->merge_nodes.end();) {
     Node* node = *it;
-    if (node->out_edges().empty()) {
-      graph_->RemoveNode(node);
-      merge_cluster->merge_nodes.erase(*it++);
+    graph_->RemoveNode(node);
+    merge_cluster->merge_nodes.erase(*it++);
+  }
+}
+
+Status FunctionalizeCond::RemoveTrivialMerge(Cluster* merge_cluster) {
+  Cluster* switch_cluster = *merge_cluster->in_nodes.begin();
+  if (switch_cluster->switch_nodes.empty()) {
+    return errors::FailedPrecondition(
+        "Not a trivial merge: no Switch node feeding into Merge node");
+  }
+
+  for (auto it = merge_cluster->merge_nodes.begin();
+       it != merge_cluster->merge_nodes.end();) {
+    // We have the following structure:
+    //   Op -> Switch -> Merge -> Consumer
+    // and we want to transform it to:
+    //   Op -> Consumer
+    Node* merge_node = *it;
+    Node* switch_node;
+    const Edge* in = nullptr;
+    TF_RETURN_IF_ERROR(merge_node->input_node(0, &switch_node));
+    TF_RETURN_IF_ERROR(switch_node->input_edge(0, &in));
+    for (auto out : merge_node->out_edges()) {
+      int src_output = out->dst_input() == Graph::kControlSlot
+                           ? Graph::kControlSlot
+                           : in->src_output();
+      graph_->AddEdge(in->src(), src_output, out->dst(), out->dst_input());
     }
+    graph_->RemoveNode(*it++);
   }
+  RemoveUnusedArgs(switch_cluster->switch_nodes);
+
+  return Status::OK();
 }
 
 Status FunctionalizeCond::ConvertMergeToXlaIf(Cluster* merge_cluster) {
@@ -1127,12 +1238,8 @@ Status FunctionalizeCond::ConvertMergeToXlaIf(Cluster* merge_cluster) {
   if (!switch_cluster.has_value()) {
     return errors::FailedPrecondition(
         "Merge cluster was not part of a simple conditional in the clustered "
-        "graph. Graph nodes in merge cluster {",
-        str_util::Join(merge_cluster->merge_nodes, ", ",
-                       [](string* output, Node* node) {
-                         strings::StrAppend(output, node->name());
-                       }),
-        "}");
+        "graph. Graph nodes in merge cluster ",
+        NodesToString(merge_cluster->merge_nodes));
   }
   TF_ASSIGN_OR_RETURN(auto cond_args,
                       DetermineCondArgs(*merge_cluster, **switch_cluster));
@@ -1153,15 +1260,17 @@ Status FunctionalizeCond::ConvertMergeToXlaIf(Cluster* merge_cluster) {
   // Remove the old nodes from the graph_ and contract the edges of the
   // clustered graph.
   for (auto in : merge_cluster->in_nodes) {
-    RemoveClusterNodes(in);
+    if (in != *switch_cluster) {
+      RemoveClusterNodes(in);
+    }
   }
+  RemoveMergeNodes(merge_cluster);
   RemoveUnusedArgs(cond_args.args);
   auto in_nodes = merge_cluster->in_nodes;
   for (auto it = in_nodes.begin(); it != in_nodes.end();) {
     ContractEdge(*it++, merge_cluster);
   }
   ContractEdge(*switch_cluster, merge_cluster);
-  RemoveUnusedMergeNodes(merge_cluster);
   clusters_[if_node].Get() = ClusterHandle(merge_cluster->representative);
 
   return Status::OK();
@@ -1230,7 +1339,27 @@ Status FunctionalizeCond::Functionalize(Graph* graph,
   for (auto it = queue.begin(); it != queue.end();) {
     Cluster* merge_cluster = (*it).second;
     ++it;
-    TF_RETURN_IF_ERROR(fc.ConvertMergeToXlaIf(merge_cluster));
+    if (merge_cluster->in_nodes.size() == 1) {
+      TF_RETURN_IF_ERROR(fc.RemoveTrivialMerge(merge_cluster));
+    } else {
+      TF_RETURN_IF_ERROR(fc.ConvertMergeToXlaIf(merge_cluster));
+    }
+
+    // Contract newly Merge free merge_cluster with incoming nodes without
+    // Switch or Merge nodes.
+    std::vector<Cluster*> in_nodes(merge_cluster->in_nodes.begin(),
+                                   merge_cluster->in_nodes.end());
+    for (auto in : in_nodes) {
+      if (in->merge_nodes.empty() && in->switch_nodes.empty()) {
+        fc.ContractEdge(in, merge_cluster);
+      }
+    }
+  }
+
+  if (!fc.switch_nodes_.empty()) {
+    return errors::Internal(
+        "Failed to functionalize control flow with Switch nodes remaining: ",
+        NodesToString(fc.switch_nodes_));
   }
   return Status::OK();
 }
-- 
GitLab


From 2487732ff111daedaf489672700ccfbf2088c3de Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 16 Oct 2017 11:17:08 -0700
Subject: [PATCH 434/909] Add tf.contrib.distributions.bijectors.Gumbel.

PiperOrigin-RevId: 172350038
---
 tensorflow/contrib/distributions/BUILD        |  19 +++
 .../kernel_tests/bijectors/gumbel_test.py     |  70 ++++++++++
 .../python/ops/bijectors/__init__.py          |   2 +
 .../python/ops/bijectors/gumbel.py            |  29 ++++
 .../python/ops/bijectors/gumbel_impl.py       | 124 ++++++++++++++++++
 5 files changed, 244 insertions(+)
 create mode 100644 tensorflow/contrib/distributions/python/kernel_tests/bijectors/gumbel_test.py
 create mode 100644 tensorflow/contrib/distributions/python/ops/bijectors/gumbel.py
 create mode 100644 tensorflow/contrib/distributions/python/ops/bijectors/gumbel_impl.py

diff --git a/tensorflow/contrib/distributions/BUILD b/tensorflow/contrib/distributions/BUILD
index 93770c37de..825ec652d0 100644
--- a/tensorflow/contrib/distributions/BUILD
+++ b/tensorflow/contrib/distributions/BUILD
@@ -797,6 +797,25 @@ cuda_py_test(
     ],
 )
 
+cuda_py_test(
+    name = "gumbel_test",
+    size = "small",
+    srcs = ["python/kernel_tests/bijectors/gumbel_test.py"],
+    additional_deps = [
+        ":bijectors_py",
+        ":distributions_py",
+        "//third_party/py/numpy",
+        "@six_archive//:six",
+        "//tensorflow/contrib/linalg:linalg_py",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework_for_generated_wrappers",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:platform_test",
+    ],
+)
+
 cuda_py_test(
     name = "inline_test",
     size = "small",
diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/gumbel_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/gumbel_test.py
new file mode 100644
index 0000000000..9a905980c7
--- /dev/null
+++ b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/gumbel_test.py
@@ -0,0 +1,70 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for Bijector."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+from scipy import stats
+
+from tensorflow.contrib.distributions.python.ops.bijectors.gumbel import Gumbel
+from tensorflow.python.ops.distributions.bijector_test_util import assert_bijective_and_finite
+from tensorflow.python.ops.distributions.bijector_test_util import assert_scalar_congruency
+from tensorflow.python.platform import test
+
+
+class GumbelBijectorTest(test.TestCase):
+  """Tests correctness of the Gumbel bijector."""
+
+  def testBijector(self):
+    with self.test_session():
+      loc = 0.3
+      scale = 5.
+      bijector = Gumbel(loc=loc, scale=scale, event_ndims=1, validate_args=True)
+      self.assertEqual("gumbel", bijector.name)
+      x = np.array([[[-3.], [0.], [0.5], [4.2], [12.]]], dtype=np.float32)
+      # Gumbel distribution
+      gumbel_dist = stats.gumbel_r(loc=loc, scale=scale)
+      y = gumbel_dist.cdf(x).astype(np.float32)
+      self.assertAllClose(y, bijector.forward(x).eval())
+      self.assertAllClose(x, bijector.inverse(y).eval())
+      self.assertAllClose(
+          # We should lose a dimension from calculating the determinant of the
+          # jacobian.
+          np.squeeze(gumbel_dist.logpdf(x), axis=2),
+          bijector.forward_log_det_jacobian(x).eval())
+      self.assertAllClose(
+          -bijector.inverse_log_det_jacobian(y).eval(),
+          bijector.forward_log_det_jacobian(x).eval(),
+          rtol=1e-4,
+          atol=0.)
+
+  def testScalarCongruency(self):
+    with self.test_session():
+      assert_scalar_congruency(
+          Gumbel(loc=0.3, scale=20.), lower_x=1., upper_x=100., rtol=0.02)
+
+  def testBijectiveAndFinite(self):
+    with self.test_session():
+      bijector = Gumbel(loc=0., scale=3.0, event_ndims=0, validate_args=True)
+      x = np.linspace(-10., 10., num=10).astype(np.float32)
+      y = np.linspace(0.01, 0.99, num=10).astype(np.float32)
+      assert_bijective_and_finite(bijector, x, y, rtol=1e-3)
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/__init__.py b/tensorflow/contrib/distributions/python/ops/bijectors/__init__.py
index c9ed546a34..e62f900bbf 100644
--- a/tensorflow/contrib/distributions/python/ops/bijectors/__init__.py
+++ b/tensorflow/contrib/distributions/python/ops/bijectors/__init__.py
@@ -22,6 +22,7 @@
 @@CholeskyOuterProduct
 @@ConditionalBijector
 @@Exp
+@@Gumbel
 @@Identity
 @@Inline
 @@Invert
@@ -48,6 +49,7 @@ from tensorflow.contrib.distributions.python.ops.bijectors.chain import *
 from tensorflow.contrib.distributions.python.ops.bijectors.cholesky_outer_product import *
 from tensorflow.contrib.distributions.python.ops.bijectors.conditional_bijector import *
 from tensorflow.contrib.distributions.python.ops.bijectors.exp import *
+from tensorflow.contrib.distributions.python.ops.bijectors.gumbel import *
 from tensorflow.contrib.distributions.python.ops.bijectors.inline import *
 from tensorflow.contrib.distributions.python.ops.bijectors.invert import *
 from tensorflow.contrib.distributions.python.ops.bijectors.permute import *
diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/gumbel.py b/tensorflow/contrib/distributions/python/ops/bijectors/gumbel.py
new file mode 100644
index 0000000000..cf37aa5111
--- /dev/null
+++ b/tensorflow/contrib/distributions/python/ops/bijectors/gumbel.py
@@ -0,0 +1,29 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Gumbel bijector."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+# go/tf-wildcard-import
+# pylint: disable=wildcard-import
+from tensorflow.contrib.distributions.python.ops.bijectors.gumbel_impl import *
+# pylint: enable=wildcard-import
+from tensorflow.python.util.all_util import remove_undocumented
+
+_allowed_symbols = ["Gumbel"]
+
+remove_undocumented(__name__, _allowed_symbols)
diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/gumbel_impl.py b/tensorflow/contrib/distributions/python/ops/bijectors/gumbel_impl.py
new file mode 100644
index 0000000000..67f3978556
--- /dev/null
+++ b/tensorflow/contrib/distributions/python/ops/bijectors/gumbel_impl.py
@@ -0,0 +1,124 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Gumbel bijector."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import check_ops
+from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops.distributions import bijector
+
+__all__ = [
+    "Gumbel",
+]
+
+
+class Gumbel(bijector.Bijector):
+  """Compute `Y = g(X) = exp(-exp(-(X - loc) / scale))`.
+
+  This bijector maps inputs from `[-inf, inf]` to [0, 1]`. The inverse of the
+  bijector applied to a uniform random variable `X ~ U(0, 1) gives back a
+  random variable with the
+  [Gumbel distribution](https://en.wikipedia.org/wiki/Gumbel_distribution):
+
+  ```none
+  Y ~ Gumbel(loc, scale)
+  pdf(y; loc, scale) = exp(
+    -( (y - loc) / scale + exp(- (y - loc) / scale) ) ) / scale
+  ```
+  """
+
+  def __init__(self,
+               loc=0.,
+               scale=1.,
+               event_ndims=0,
+               validate_args=False,
+               name="gumbel"):
+    """Instantiates the `Gumbel` bijector.
+
+    Args:
+      loc: Float-like `Tensor` that is the same dtype and is
+        broadcastable with `scale`.
+        This is `loc` in `Y = g(X) = exp(-exp(-(X - loc) / scale))`.
+      scale: Positive Float-like `Tensor` that is the same dtype and is
+        broadcastable with `loc`.
+        This is `scale` in `Y = g(X) = exp(-exp(-(X - loc) / scale))`.
+      event_ndims: Python scalar indicating the number of dimensions associated
+        with a particular draw from the distribution.
+      validate_args: Python `bool` indicating whether arguments should be
+        checked for correctness.
+      name: Python `str` name given to ops managed by this object.
+    """
+    self._graph_parents = []
+    self._name = name
+    self._validate_args = validate_args
+    with self._name_scope("init", values=[loc, scale]):
+      self._loc = ops.convert_to_tensor(loc, name="loc")
+      self._scale = ops.convert_to_tensor(scale, name="scale")
+      check_ops.assert_same_float_dtype([self._loc, self._scale])
+      if validate_args:
+        self._scale = control_flow_ops.with_dependencies([
+            check_ops.assert_positive(
+                self._scale, message="Argument scale was not positive")
+        ], self._scale)
+
+    super(Gumbel, self).__init__(
+        event_ndims=event_ndims, validate_args=validate_args, name=name)
+
+  @property
+  def loc(self):
+    """The `loc` in `Y = g(X) = exp(-exp(-(X - loc) / scale))`."""
+    return self._loc
+
+  @property
+  def scale(self):
+    """This is `scale` in `Y = g(X) = exp(-exp(-(X - loc) / scale))`."""
+    return self._scale
+
+  def _forward(self, x):
+    z = (x - self.loc) / self.scale
+    return math_ops.exp(-math_ops.exp(-z))
+
+  def _inverse(self, y):
+    y = self._maybe_assert_valid_y(y)
+    return self.loc - self.scale * math_ops.log(-math_ops.log(y))
+
+  def _inverse_log_det_jacobian(self, y):
+    y = self._maybe_assert_valid_y(y)
+    event_dims = self._event_dims_tensor(y)
+    return math_ops.reduce_sum(
+        math_ops.log(self.scale / (-math_ops.log(y) * y)), axis=event_dims)
+
+  def _forward_log_det_jacobian(self, x):
+    event_dims = self._event_dims_tensor(x)
+    z = (x - self.loc) / self.scale
+    return math_ops.reduce_sum(
+        -z - math_ops.exp(-z) - math_ops.log(self.scale), axis=event_dims)
+
+  def _maybe_assert_valid_y(self, y):
+    if not self.validate_args:
+      return y
+    is_positive = check_ops.assert_non_negative(
+        y, message="Inverse transformation input must be greater than 0.")
+    less_than_one = check_ops.assert_less_equal(
+        y,
+        constant_op.constant(1., y.dtype),
+        message="Inverse transformation input must be less than or equal to 1.")
+    return control_flow_ops.with_dependencies([is_positive, less_than_one], y)
-- 
GitLab


From 74bd8ff717eaf08bf64f4b16c0bca40173b19614 Mon Sep 17 00:00:00 2001
From: Eugene Brevdo <ebrevdo@google.com>
Date: Mon, 16 Oct 2017 11:32:49 -0700
Subject: [PATCH 435/909] [tf.contrib.seq2seq] Some light cleanup in beam
 search decoder code.

PiperOrigin-RevId: 172352767
---
 .../kernel_tests/beam_search_decoder_test.py  |  3 +-
 .../seq2seq/python/ops/beam_search_decoder.py | 71 ++++++++++---------
 2 files changed, 39 insertions(+), 35 deletions(-)

diff --git a/tensorflow/contrib/seq2seq/python/kernel_tests/beam_search_decoder_test.py b/tensorflow/contrib/seq2seq/python/kernel_tests/beam_search_decoder_test.py
index 2caeb9eb61..8d4ec4b4db 100644
--- a/tensorflow/contrib/seq2seq/python/kernel_tests/beam_search_decoder_test.py
+++ b/tensorflow/contrib/seq2seq/python/kernel_tests/beam_search_decoder_test.py
@@ -80,8 +80,7 @@ class TestEosMasking(test.TestCase):
     ])
 
     eos_token = 0
-    previously_finished = constant_op.constant(
-        [[0, 1, 0], [0, 1, 1]], dtype=dtypes.float32)
+    previously_finished = np.array([[0, 1, 0], [0, 1, 1]], dtype=bool)
     masked = beam_search_decoder._mask_probs(probs, eos_token,
                                              previously_finished)
 
diff --git a/tensorflow/contrib/seq2seq/python/ops/beam_search_decoder.py b/tensorflow/contrib/seq2seq/python/ops/beam_search_decoder.py
index e22912ac5c..112ac57a1b 100644
--- a/tensorflow/contrib/seq2seq/python/ops/beam_search_decoder.py
+++ b/tensorflow/contrib/seq2seq/python/ops/beam_search_decoder.py
@@ -20,9 +20,10 @@ from __future__ import print_function
 
 import collections
 
+import numpy as np
+
 from tensorflow.contrib.seq2seq.python.ops import beam_search_ops
 from tensorflow.contrib.seq2seq.python.ops import decoder
-from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
@@ -390,17 +391,17 @@ class BeamSearchDecoder(decoder.Decoder):
     We do this so that we can use nest and not run into problems with shapes.
 
     Args:
-      t: Tensor of dimension [batch_size*beam_width, s]
-      s: Tensor, Python int, or TensorShape.
+      t: `Tensor`, either scalar or shaped `[batch_size * beam_width] + s`.
+      s: `Tensor`, Python int, or `TensorShape`.
 
     Returns:
-      Either a reshaped version of t with dimension
-      [batch_size, beam_width, s] if t's first dimension is of size
-      batch_size*beam_width or t if not.
+      If `t` is a matrix or higher order tensor, then the return value is
+      `t` reshaped to `[batch_size, beam_width] + s`.  Otherwise `t` is
+      returned unchanged.
 
     Raises:
-      TypeError: If t is an instance of TensorArray.
-      ValueError: If the rank of t is not statically known.
+      TypeError: If `t` is an instance of `TensorArray`.
+      ValueError: If the rank of `t` is not statically known.
     """
     _check_maybe(t)
     if t.shape.ndims >= 1:
@@ -411,19 +412,19 @@ class BeamSearchDecoder(decoder.Decoder):
   def _maybe_merge_batch_beams(self, t, s):
     """Splits the tensor from a batch by beams into a batch of beams.
 
-    More exactly, t is a tensor of dimension [batch_size*beam_width, s]. We
-    reshape this into [batch_size, beam_width, s]
+    More exactly, `t` is a tensor of dimension `[batch_size * beam_width] + s`,
+    then we reshape it to `[batch_size, beam_width] + s`.
 
     Args:
-      t: Tensor of dimension [batch_size*beam_width, s]
-      s: Tensor, Python int, or TensorShape.
+      t: `Tensor` of dimension `[batch_size * beam_width] + s`.
+      s: `Tensor`, Python int, or `TensorShape`.
 
     Returns:
-      A reshaped version of t with dimension [batch_size, beam_width, s].
+      A reshaped version of t with shape `[batch_size, beam_width] + s`.
 
     Raises:
-      TypeError: If t is an instance of TensorArray.
-      ValueError:  If the rank of t is not statically known.
+      TypeError: If `t` is an instance of `TensorArray`.
+      ValueError:  If the rank of `t` is not statically known.
     """
     _check_maybe(t)
     if t.shape.ndims >= 2:
@@ -521,14 +522,12 @@ def _beam_search_step(time, logits, next_cell_state, beam_state, batch_size,
   # Calculate the continuation lengths by adding to all continuing beams.
   vocab_size = logits.shape[-1].value or array_ops.shape(logits)[-1]
   lengths_to_add = array_ops.one_hot(
-      indices=array_ops.tile(
-          array_ops.reshape(end_token, [1, 1]), [batch_size, beam_width]),
+      indices=array_ops.fill([batch_size, beam_width], end_token),
       depth=vocab_size,
-      on_value=constant_op.constant(0, dtype=dtypes.int64),
-      off_value=constant_op.constant(1, dtype=dtypes.int64),
+      on_value=np.int64(0), off_value=np.int64(1),
       dtype=dtypes.int64)
-  add_mask = (1 - math_ops.to_int64(previously_finished))
-  lengths_to_add = array_ops.expand_dims(add_mask, 2) * lengths_to_add
+  add_mask = math_ops.to_int64(math_ops.logical_not(previously_finished))
+  lengths_to_add *= array_ops.expand_dims(add_mask, 2)
   new_prediction_lengths = (
       lengths_to_add + array_ops.expand_dims(prediction_lengths, 2))
 
@@ -592,9 +591,7 @@ def _beam_search_step(time, logits, next_cell_state, beam_state, batch_size,
   # 1. Finished beams remain unchanged
   # 2. Beams that are now finished (EOS predicted) remain unchanged
   # 3. Beams that are not yet finished have their length increased by 1
-  lengths_to_add = math_ops.to_int64(
-      math_ops.not_equal(next_word_ids, end_token))
-  lengths_to_add = (1 - math_ops.to_int64(next_finished)) * lengths_to_add
+  lengths_to_add = math_ops.to_int64(math_ops.logical_not(next_finished))
   next_prediction_len = _tensor_gather_helper(
       gather_indices=next_beam_ids,
       gather_from=beam_state.lengths,
@@ -652,13 +649,20 @@ def _get_scores(log_probs, sequence_lengths, length_penalty_weight):
 def _length_penalty(sequence_lengths, penalty_factor):
   """Calculates the length penalty. See https://arxiv.org/abs/1609.08144.
 
+  Returns the length penalty tensor:
+  ```
+  [(5+sequence_lengths)/6]**penalty_factor
+  ```
+  where all operations are performed element-wise.
+
   Args:
-    sequence_lengths: The sequence length of all hypotheses, a tensor
-      of shape [beam_size, vocab_size].
+    sequence_lengths: `Tensor`, the sequence lengths of each hypotheses.
     penalty_factor: A scalar that weights the length penalty.
 
   Returns:
-    The length penalty factor, a tensor fo shape [beam_size].
+    If the penalty is `0`, returns the scalar `1.0`.  Otherwise returns
+    the length penalty factor, a tensor with the same shape as
+    `sequence_lengths`.
   """
   penalty_factor = ops.convert_to_tensor(penalty_factor, name="penalty_factor")
   penalty_factor.set_shape(())  # penalty should be a scalar.
@@ -680,8 +684,7 @@ def _mask_probs(probs, eos_token, finished):
     eos_token: An int32 id corresponding to the EOS token to allocate
       probability to.
     finished: A boolean tensor of shape `[batch_size, beam_width]` that
-      specifies which
-      elements in the beam are finished already.
+      specifies which elements in the beam are finished already.
 
   Returns:
     A tensor of shape `[batch_size, beam_width, vocab_size]`, where unfinished
@@ -689,10 +692,12 @@ def _mask_probs(probs, eos_token, finished):
     probability on the EOS token.
   """
   vocab_size = array_ops.shape(probs)[2]
-  finished_mask = array_ops.expand_dims(
-      math_ops.to_float(1. - math_ops.to_float(finished)), 2)
+  finished_mask = math_ops.cast(array_ops.expand_dims(finished, 2), probs.dtype)
+  not_finished_mask = math_ops.cast(
+      array_ops.expand_dims(math_ops.logical_not(finished), 2),
+      probs.dtype)
   # These examples are not finished and we leave them
-  non_finished_examples = finished_mask * probs
+  non_finished_examples = not_finished_mask * probs
   # All finished examples are replaced with a vector that has all
   # probability on EOS
   finished_row = array_ops.one_hot(
@@ -701,7 +706,7 @@ def _mask_probs(probs, eos_token, finished):
       dtype=probs.dtype,
       on_value=0.,
       off_value=probs.dtype.min)
-  finished_examples = (1. - finished_mask) * finished_row
+  finished_examples = finished_mask * finished_row
   return finished_examples + non_finished_examples
 
 
-- 
GitLab


From b6d14a05cc51a6c32086cea8e6950ed45372fa7f Mon Sep 17 00:00:00 2001
From: Jiri Simsa <jsimsa@google.com>
Date: Mon, 16 Oct 2017 11:37:02 -0700
Subject: [PATCH 436/909] Fix divergence between core.data and contrib.data
 Python tests.

PiperOrigin-RevId: 172353443
---
 .../kernel_tests/iterator_ops_cluster_test.py | 54 +++++++++----------
 1 file changed, 26 insertions(+), 28 deletions(-)

diff --git a/tensorflow/python/kernel_tests/iterator_ops_cluster_test.py b/tensorflow/python/kernel_tests/iterator_ops_cluster_test.py
index d7315a2526..45dfa13720 100644
--- a/tensorflow/python/kernel_tests/iterator_ops_cluster_test.py
+++ b/tensorflow/python/kernel_tests/iterator_ops_cluster_test.py
@@ -53,13 +53,8 @@ class IteratorClusterTest(test.TestCase):
       with self.assertRaises(errors.InvalidArgumentError):
         sess.run(get_next_op)
 
-  def testRemoteIteratorUsingRemoteCallOp(self):
-    worker_config = config_pb2.ConfigProto()
-    worker_config.device_count["CPU"] = 2
-    worker, _ = test_util.create_local_cluster(
-        1, 1, worker_config=worker_config)
-
-    with ops.device("/job:worker/replica:0/task:0/cpu:1"):
+  def _testRemoteIteratorHelper(self, device0, device1, target):
+    with ops.device(device1):
       dataset_3 = dataset_ops.Dataset.from_tensor_slices([1, 2, 3])
       iterator_3 = dataset_3.make_one_shot_iterator()
       iterator_3_handle = iterator_3.string_handle()
@@ -70,7 +65,7 @@ class IteratorClusterTest(test.TestCase):
           h, dataset_3.output_types, dataset_3.output_shapes)
       return remote_iterator.get_next()
 
-    with ops.device("/job:worker/replica:0/task:0/cpu:0"):
+    with ops.device(device0):
       target_placeholder = array_ops.placeholder(dtypes.string, shape=[])
       remote_op = functional_ops.remote_call(
           args=[iterator_3_handle],
@@ -78,32 +73,35 @@ class IteratorClusterTest(test.TestCase):
           f=_remote_fn,
           target=target_placeholder)
 
-    with session.Session(worker[0].target) as sess:
-      elem = sess.run(
-          remote_op,
-          feed_dict={target_placeholder: "/job:worker/replica:0/task:0/cpu:1"})
+    with session.Session(target) as sess:
+      elem = sess.run(remote_op, feed_dict={target_placeholder: device1})
       self.assertEqual(elem, [1])
       # Fails when target is cpu:0 where the resource is not located.
       with self.assertRaises(errors.InvalidArgumentError):
-        sess.run(
-            remote_op,
-            feed_dict={
-                target_placeholder: "/job:worker/replica:0/task:0/cpu:0"
-            })
-      elem = sess.run(
-          remote_op,
-          feed_dict={target_placeholder: "/job:worker/replica:0/task:0/cpu:1"})
+        sess.run(remote_op, feed_dict={target_placeholder: device0})
+      elem = sess.run(iterator_3.get_next())
       self.assertEqual(elem, [2])
-      elem = sess.run(
-          remote_op,
-          feed_dict={target_placeholder: "/job:worker/replica:0/task:0/cpu:1"})
+      elem = sess.run(remote_op, feed_dict={target_placeholder: device1})
       self.assertEqual(elem, [3])
       with self.assertRaises(errors.OutOfRangeError):
-        sess.run(
-            remote_op,
-            feed_dict={
-                target_placeholder: "/job:worker/replica:0/task:0/cpu:1"
-            })
+        sess.run(remote_op, feed_dict={target_placeholder: device1})
+
+  def testRemoteIteratorUsingRemoteCallOp(self):
+    worker_config = config_pb2.ConfigProto()
+    worker_config.device_count["CPU"] = 2
+    worker, _ = test_util.create_local_cluster(
+        1, 1, worker_config=worker_config)
+
+    self._testRemoteIteratorHelper("/job:worker/replica:0/task:0/cpu:0",
+                                   "/job:worker/replica:0/task:0/cpu:1",
+                                   worker[0].target)
+
+  def testRemoteIteratorUsingRemoteCallOpCrossProcess(self):
+    workers, _ = test_util.create_local_cluster(2, 1)
+
+    self._testRemoteIteratorHelper("/job:worker/replica:0/task:0/cpu:0",
+                                   "/job:worker/replica:0/task:1/cpu:0",
+                                   workers[0].target)
 
 
 if __name__ == "__main__":
-- 
GitLab


From 2ed69577b08e5e8845619748249ecd41dc0f7c87 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Mon, 16 Oct 2017 12:37:40 -0700
Subject: [PATCH 437/909] Add fast_tensor_util.cpp to .gitignore (#13749)

While working on building TensorFlow I noticed that a file
`fast_tensor_util.cpp` is generated:
```sh
ubuntu@ubuntu:~/tensorflow$ git status
On branch master
Your branch is up-to-date with 'origin/master'.
Untracked files:
  (use "git add <file>..." to include in what will be committed)

        tensorflow/python/framework/fast_tensor_util.cpp

nothing added to commit but untracked files present (use "git add" to track)
ubuntu@ubuntu:~/tensorflow$
```

This fix adds `fast_tensor_util.cpp` to .gitignore so that
it will not be added inadvertently when adding commit with `git add -A`.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index 09734fe497..9572a3e97c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -17,3 +17,4 @@ cmake_build/
 .idea/**
 /build/
 /tensorflow/core/util/version_info.cc
+/tensorflow/python/framework/fast_tensor_util.cpp
-- 
GitLab


From 8de821fc169fb9bad8be681801e8551171f8e44a Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Mon, 16 Oct 2017 12:42:35 -0700
Subject: [PATCH 438/909] make_vjp in eager

PiperOrigin-RevId: 172363016
---
 tensorflow/python/eager/backprop.py      | 56 ++++++++++++++++++++++++
 tensorflow/python/eager/backprop_test.py | 10 +++++
 2 files changed, 66 insertions(+)

diff --git a/tensorflow/python/eager/backprop.py b/tensorflow/python/eager/backprop.py
index 0060dd0c1c..7f1a770513 100644
--- a/tensorflow/python/eager/backprop.py
+++ b/tensorflow/python/eager/backprop.py
@@ -581,6 +581,62 @@ def val_and_grad_function(f, params=None):
   return decorated
 
 
+def make_vjp(f, params=None):
+  """Returns a function that computes f and is vjp w.r.t. params.
+
+  The term "vjp" here is an abbreviation for vector-jacobian product.
+
+  Args:
+    f: the function to be differentiated.
+    params: the parameters (numbers or names) to differentiate with respect to.
+       A value of None will differentiate with respect to all parameters.
+
+  Returns:
+    A function, which when called, returns a tuple (value, vjp), where:
+    - value is the result of calling f.
+    - vjp is a function, which takes a vector as an argument and
+      returns the product of that vector with the Jacobian of f.
+      Providing no argument to vjp is equivalent to providing a
+      vector of ones.
+
+    For example,
+    ```python
+    def f(x):
+      return x * x
+
+    wrapped_fn = tfe.make_vjp(f)
+    result, vjp = wrapped_fn(tf.constant(3.0))
+    # result is 9.0
+    vjp()  # the vjp function rturns 6.0
+
+  """
+
+  parameter_positions = _get_arg_spec(f, params)
+
+  def decorated(*args, **kwds):
+    """Computes the value and gradient of the decorated function."""
+    assert not kwds, "The gradient function can't take keyword arguments."
+    tape.push_new_tape()
+    sources = []
+    args = [
+        ops.convert_to_tensor(args[i]) if i in parameter_positions else args[i]
+        for i in range(len(args))
+    ]
+    args = _ensure_unique_tensor_objects(parameter_positions, args)
+    for i in parameter_positions:
+      sources.append(args[i])
+      tape.watch(args[i])
+    result = f(*args)
+    t = tape.pop_tape()
+    def vjp(dy=None):
+      return imperative_grad.imperative_grad(
+          _default_vspace, t, nest.flatten(result), sources,
+          output_gradients=nest.flatten(dy) if dy is not None else None)
+    return result, vjp
+
+  return decorated
+
+
 def _aggregate_grads(gradients):
   """Aggregate gradients from multiple sources.
 
diff --git a/tensorflow/python/eager/backprop_test.py b/tensorflow/python/eager/backprop_test.py
index d53c69afcc..9083e3a712 100644
--- a/tensorflow/python/eager/backprop_test.py
+++ b/tensorflow/python/eager/backprop_test.py
@@ -168,6 +168,16 @@ class BackpropTest(test.TestCase):
     grad = backprop.gradients_function(second, [0])(f)[0]
     self.assertAllEqual([[0.0]], grad.numpy())
 
+  def testMakeVJP(self):
+
+    def f(x):
+      return x * x
+
+    wrapped_fn = backprop.make_vjp(f)
+    result, vjp = wrapped_fn(constant_op.constant(3.0))
+    self.assertEqual(result.numpy(), 9.0)
+    self.assertEqual(vjp(2.0)[0].numpy(), 12.0)
+
   def testGradGrad(self):
 
     def sq(x):
-- 
GitLab


From d4efe4dd39d6894779bb09462b2af8161b3dedad Mon Sep 17 00:00:00 2001
From: Akshay Agrawal <akshayka@google.com>
Date: Mon, 16 Oct 2017 12:45:10 -0700
Subject: [PATCH 439/909] Implement set_shape for EagerTensors for compatibiity
 with ops that call it

Checks if shape is not compatible with the Eager tensor's shape,
raises an error if it is not.

PiperOrigin-RevId: 172363347
---
 tensorflow/python/eager/BUILD       |  1 +
 tensorflow/python/eager/ops_test.py | 11 +++++++++++
 tensorflow/python/framework/ops.py  |  9 ++++++---
 3 files changed, 18 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/eager/BUILD b/tensorflow/python/eager/BUILD
index 69b96df87c..5a2592287c 100644
--- a/tensorflow/python/eager/BUILD
+++ b/tensorflow/python/eager/BUILD
@@ -408,6 +408,7 @@ py_test(
         "//tensorflow/python:math_ops",
         "//tensorflow/python:random_ops",
         "//tensorflow/python:sparse_ops",
+        "//tensorflow/python:tensor_shape",
         "//third_party/py/numpy",
     ],
 )
diff --git a/tensorflow/python/eager/ops_test.py b/tensorflow/python/eager/ops_test.py
index 7d54b8d2d8..78423468ea 100644
--- a/tensorflow/python/eager/ops_test.py
+++ b/tensorflow/python/eager/ops_test.py
@@ -26,6 +26,7 @@ from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import tensor_shape
 from tensorflow.python.framework import test_util
 from tensorflow.python.layers import core
 from tensorflow.python.ops import array_ops
@@ -302,6 +303,16 @@ class OpsTest(test_util.TensorFlowTestCase):
   def testIdentity(self):
     self.assertEqual(2, array_ops.identity(2).numpy())
 
+  def testIncompatibleSetShape(self):
+    x = constant_op.constant(1)
+    with self.assertRaises(ValueError):
+      x.set_shape((1, 2))
+
+  def testCompatibleSetShape(self):
+    x = constant_op.constant([[1, 2]])
+    x.set_shape(tensor_shape.TensorShape([None, 2]))
+    self.assertEqual(x.get_shape(), (1, 2))
+
 
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index 6077d602c4..a52a0cfc2d 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -727,6 +727,12 @@ class _EagerTensorBase(Tensor):
   def __nonzero__(self):
     return self.__bool__()
 
+  def set_shape(self, shape):
+    if not self.shape.is_compatible_with(shape):
+      raise ValueError(
+          "EagerTensor's shape %s is not compatible with supplied shape %s" %
+          (self.shape, shape))
+
   # Methods not supported / implemented for Eager Tensors.
   @property
   def op(self):
@@ -740,9 +746,6 @@ class _EagerTensorBase(Tensor):
   def name(self):
     raise NotImplementedError("name not supported for Eager Tensors.")
 
-  def set_shape(self, shape):
-    raise NotImplementedError("set_shape not supported for Eager Tensors.")
-
   @property
   def value_index(self):
     raise NotImplementedError("value_index not supported for Eager Tensors.")
-- 
GitLab


From 58eee58840094782b22dbb8981b513df3797eac0 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 16 Oct 2017 13:03:47 -0700
Subject: [PATCH 440/909] Remove broken link.

PiperOrigin-RevId: 172366027
---
 tensorflow/contrib/gan/README.md | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/gan/README.md b/tensorflow/contrib/gan/README.md
index 10458a2458..5d74df3ef7 100644
--- a/tensorflow/contrib/gan/README.md
+++ b/tensorflow/contrib/gan/README.md
@@ -51,9 +51,10 @@ network to evaluate your unconditional generative model. You can also also use
 your own pretrained classifier for more specific performance numbers, or use
 other methods for evaluating conditional generative models.
 
-* [examples](https://github.com/tensorflow/models/tree/master/gan/):
+* examples (coming soon):
 See examples of how to use TFGAN to make GAN training easier, or use the more complicated examples to jumpstart your
-own project.
+own project. These include unconditional and conditional GANs, InfoGANs,
+adversarial losses on existing networks, and image-to-image translation.
 
 ## Training a GAN model
 
-- 
GitLab


From b1128a402d473cc6a43c99a081446c1b45305dd9 Mon Sep 17 00:00:00 2001
From: Mustafa Ispir <ispir@google.com>
Date: Mon, 16 Oct 2017 13:10:20 -0700
Subject: [PATCH 441/909] Move global_step_read dependency to model_fn instead
 of input_fn.

PiperOrigin-RevId: 172366972
---
 tensorflow/python/estimator/estimator.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/python/estimator/estimator.py b/tensorflow/python/estimator/estimator.py
index 4dfc53aadf..00a57f11dc 100644
--- a/tensorflow/python/estimator/estimator.py
+++ b/tensorflow/python/estimator/estimator.py
@@ -708,11 +708,11 @@ class Estimator(object):
       random_seed.set_random_seed(self._config.tf_random_seed)
       global_step_tensor = self._create_and_assert_global_step(g)
       global_step_read_tensor = training_util._get_or_create_global_step_read()  # pylint: disable=protected-access
+      features, labels = self._get_features_and_labels_from_input_fn(
+          input_fn, model_fn_lib.ModeKeys.TRAIN)
       with ops.control_dependencies([global_step_read_tensor]):
-        features, labels = self._get_features_and_labels_from_input_fn(
-            input_fn, model_fn_lib.ModeKeys.TRAIN)
-      estimator_spec = self._call_model_fn(
-          features, labels, model_fn_lib.ModeKeys.TRAIN, self.config)
+        estimator_spec = self._call_model_fn(
+            features, labels, model_fn_lib.ModeKeys.TRAIN, self.config)
       # Check if the user created a loss summary, and add one if they didn't.
       # We assume here that the summary is called 'loss'. If it is not, we will
       # make another one with the name 'loss' to ensure it shows up in the right
-- 
GitLab


From 940455b04c843333f1a359fcb2412ebba1780a7f Mon Sep 17 00:00:00 2001
From: Amit Patankar <amitpatankar@google.com>
Date: Mon, 16 Oct 2017 13:33:55 -0700
Subject: [PATCH 442/909] Creating a patch for the wrong links that still point
 to dev. (#13752)

---
 tensorflow/docs_src/install/install_linux.md  | 22 +++++++++----------
 tensorflow/docs_src/install/install_mac.md    | 10 ++++-----
 .../docs_src/install/install_sources.md       |  4 ++--
 3 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/tensorflow/docs_src/install/install_linux.md b/tensorflow/docs_src/install/install_linux.md
index 2b488cc4f5..9d204cc246 100644
--- a/tensorflow/docs_src/install/install_linux.md
+++ b/tensorflow/docs_src/install/install_linux.md
@@ -184,7 +184,7 @@ Take the following steps to install TensorFlow with Virtualenv:
      virtualenv environment:
 
      <pre>(tensorflow)$ <b>pip3 install --upgrade \
-     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0dev-cp34-cp34m-linux_x86_64.whl</b></pre>
+     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0rc0-cp34-cp34m-linux_x86_64.whl</b></pre>
 
 If you encounter installation problems, see
 [Common Installation Problems](#common_installation_problems).
@@ -289,7 +289,7 @@ take the following steps:
 
      <pre>
      $ <b>sudo pip3 install --upgrade \
-     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0dev-cp34-cp34m-linux_x86_64.whl</b>
+     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0rc0-cp34-cp34m-linux_x86_64.whl</b>
      </pre>
 
      If this step fails, see
@@ -476,7 +476,7 @@ Take the following steps to install TensorFlow in an Anaconda environment:
 
      <pre>
      (tensorflow)$ <b>pip install --ignore-installed --upgrade \
-     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0dev-cp34-cp34m-linux_x86_64.whl</b></pre>
+     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0rc0-cp34-cp34m-linux_x86_64.whl</b></pre>
 
 
 <a name="ValidateYourInstallation"></a>
@@ -644,14 +644,14 @@ This section documents the relevant values for Linux installations.
 CPU only:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0dev-cp27-none-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0rc0-cp27-none-linux_x86_64.whl
 </pre>
 
 
 GPU support:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.4.0dev-cp27-none-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.4.0rc0-cp27-none-linux_x86_64.whl
 </pre>
 
 Note that GPU support requires the NVIDIA hardware and software described in
@@ -663,14 +663,14 @@ Note that GPU support requires the NVIDIA hardware and software described in
 CPU only:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0dev-cp34-cp34m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0rc0-cp34-cp34m-linux_x86_64.whl
 </pre>
 
 
 GPU support:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.4.0dev-cp34-cp34m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.4.0rc0-cp34-cp34m-linux_x86_64.whl
 </pre>
 
 Note that GPU support requires the NVIDIA hardware and software described in
@@ -682,14 +682,14 @@ Note that GPU support requires the NVIDIA hardware and software described in
 CPU only:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0dev-cp35-cp35m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0rc0-cp35-cp35m-linux_x86_64.whl
 </pre>
 
 
 GPU support:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.4.0dev-cp35-cp35m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.4.0rc0-cp35-cp35m-linux_x86_64.whl
 </pre>
 
 
@@ -701,14 +701,14 @@ Note that GPU support requires the NVIDIA hardware and software described in
 CPU only:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0dev-cp36-cp36m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0rc0-cp36-cp36m-linux_x86_64.whl
 </pre>
 
 
 GPU support:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.4.0dev-cp36-cp36m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.4.0rc0-cp36-cp36m-linux_x86_64.whl
 </pre>
 
 
diff --git a/tensorflow/docs_src/install/install_mac.md b/tensorflow/docs_src/install/install_mac.md
index efd977089b..6da22784bf 100644
--- a/tensorflow/docs_src/install/install_mac.md
+++ b/tensorflow/docs_src/install/install_mac.md
@@ -109,7 +109,7 @@ Take the following steps to install TensorFlow with Virtualenv:
      TensorFlow in the active Virtualenv is as follows:
 
      <pre> $ <b>pip3 install --upgrade \
-     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.4.0dev-py2-none-any.whl</b></pre>
+     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.4.0rc0-py2-none-any.whl</b></pre>
 
 If you encounter installation problems, see
 [Common Installation Problems](#common-installation-problems).
@@ -230,7 +230,7 @@ take the following steps:
      issue the following command:
 
      <pre> $ <b>sudo pip3 install --upgrade \
-     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.4.0dev-py2-none-any.whl</b> </pre>
+     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.4.0rc0-py2-none-any.whl</b> </pre>
 
      If the preceding command fails, see
      [installation problems](#common-installation-problems).
@@ -339,7 +339,7 @@ Take the following steps to install TensorFlow in an Anaconda environment:
      TensorFlow for Python 2.7:
 
      <pre> (tensorflow)$ <b>pip install --ignore-installed --upgrade \
-     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.4.0dev-py2-none-any.whl</b></pre>
+     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.4.0rc0-py2-none-any.whl</b></pre>
 
 
 <a name="ValidateYourInstallation"></a>
@@ -512,7 +512,7 @@ This section documents the relevant values for Mac OS installations.
 
 
 <pre>
-https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.4.0dev-py2-none-any.whl
+https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.4.0rc0-py2-none-any.whl
 </pre>
 
 
@@ -520,7 +520,7 @@ https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.4.0dev-py2-none-a
 
 
 <pre>
-https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.4.0dev-py3-none-any.whl
+https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.4.0rc0-py3-none-any.whl
 </pre>
 
 
diff --git a/tensorflow/docs_src/install/install_sources.md b/tensorflow/docs_src/install/install_sources.md
index 6114496cd5..b853d87816 100644
--- a/tensorflow/docs_src/install/install_sources.md
+++ b/tensorflow/docs_src/install/install_sources.md
@@ -349,10 +349,10 @@ Invoke `pip install` to install that pip package.
 The filename of the `.whl` file depends on your platform.
 For example, the following command will install the pip package
 
-for TensorFlow 1.4.0dev on Linux:
+for TensorFlow 1.4.0rc0 on Linux:
 
 <pre>
-$ <b>sudo pip install /tmp/tensorflow_pkg/tensorflow-1.4.0dev-py2-none-any.whl</b>
+$ <b>sudo pip install /tmp/tensorflow_pkg/tensorflow-1.4.0rc0-py2-none-any.whl</b>
 </pre>
 
 ## Validate your installation
-- 
GitLab


From 24f9c6e0dbd449624aa1db543550ec412975492e Mon Sep 17 00:00:00 2001
From: Saurabh Saxena <srbs@google.com>
Date: Mon, 16 Oct 2017 13:40:08 -0700
Subject: [PATCH 443/909] Add support for saving DT_VARIANT tensors in
 TensorBundle. Add support for reading Varint64 to InputBuffer.

PiperOrigin-RevId: 172371104
---
 tensorflow/core/lib/core/coding.h             |   3 +
 tensorflow/core/lib/io/inputbuffer.cc         |  26 +++-
 tensorflow/core/lib/io/inputbuffer.h          |  26 ++++
 tensorflow/core/lib/io/inputbuffer_test.cc    |  39 ++++++
 tensorflow/core/util/tensor_bundle/BUILD      |   1 +
 .../core/util/tensor_bundle/tensor_bundle.cc  | 128 +++++++++++++++++-
 .../util/tensor_bundle/tensor_bundle_test.cc  |  75 ++++++++++
 7 files changed, 289 insertions(+), 9 deletions(-)

diff --git a/tensorflow/core/lib/core/coding.h b/tensorflow/core/lib/core/coding.h
index 77d52a909b..8265aec870 100644
--- a/tensorflow/core/lib/core/coding.h
+++ b/tensorflow/core/lib/core/coding.h
@@ -31,6 +31,9 @@ namespace core {
 // Maximum number of bytes occupied by a varint32.
 static const int kMaxVarint32Bytes = 5;
 
+// Maximum number of bytes occupied by a varint64.
+static const int kMaxVarint64Bytes = 10;
+
 // Lower-level versions of Put... that write directly into a character buffer
 // REQUIRES: dst has enough space for the value being written
 extern void EncodeFixed16(char* dst, uint16 value);
diff --git a/tensorflow/core/lib/io/inputbuffer.cc b/tensorflow/core/lib/io/inputbuffer.cc
index 7efe2dc543..4d35af49b2 100644
--- a/tensorflow/core/lib/io/inputbuffer.cc
+++ b/tensorflow/core/lib/io/inputbuffer.cc
@@ -116,17 +116,35 @@ Status InputBuffer::ReadNBytes(int64 bytes_to_read, char* result,
 }
 
 Status InputBuffer::ReadVarint32Fallback(uint32* result) {
+  Status s = ReadVarintFallback(result, core::kMaxVarint32Bytes);
+  if (errors::IsDataLoss(s)) {
+    return errors::DataLoss("Stored data is too large to be a varint32.");
+  }
+  return s;
+}
+
+Status InputBuffer::ReadVarint64Fallback(uint64* result) {
+  Status s = ReadVarintFallback(result, core::kMaxVarint64Bytes);
+  if (errors::IsDataLoss(s)) {
+    return errors::DataLoss("Stored data is too large to be a varint64.");
+  }
+  return s;
+}
+
+template <typename T>
+Status InputBuffer::ReadVarintFallback(T* result, int max_bytes) {
   uint8 scratch = 0;
-  char* p = reinterpret_cast<char*>(&scratch);
+  auto* p = reinterpret_cast<char*>(&scratch);
   size_t unused_bytes_read = 0;
 
   *result = 0;
-  for (int shift = 0; shift <= 28; shift += 7) {
+  for (int index = 0; index < max_bytes; index++) {
+    int shift = 7 * index;
     TF_RETURN_IF_ERROR(ReadNBytes(1, p, &unused_bytes_read));
-    *result |= (scratch & 127) << shift;
+    *result |= (static_cast<T>(scratch) & 127) << shift;
     if (!(scratch & 128)) return Status::OK();
   }
-  return errors::DataLoss("Stored data is too large to be a varint32.");
+  return errors::DataLoss("Stored data longer than ", max_bytes, " bytes.");
 }
 
 Status InputBuffer::SkipNBytes(int64 bytes_to_skip) {
diff --git a/tensorflow/core/lib/io/inputbuffer.h b/tensorflow/core/lib/io/inputbuffer.h
index 94a8cfd39b..b3740f396c 100644
--- a/tensorflow/core/lib/io/inputbuffer.h
+++ b/tensorflow/core/lib/io/inputbuffer.h
@@ -60,6 +60,9 @@ class InputBuffer {
   // Reads a single varint32.
   Status ReadVarint32(uint32* result);
 
+  // Reads a single varint64.
+  Status ReadVarint64(uint64* result);
+
   // Like ReadNBytes() without returning the bytes read.
   Status SkipNBytes(int64 bytes_to_skip);
 
@@ -82,6 +85,15 @@ class InputBuffer {
   // Internal slow-path routine used by ReadVarint32().
   Status ReadVarint32Fallback(uint32* result);
 
+  // Internal slow-path routine used by ReadVarint64().
+  Status ReadVarint64Fallback(uint64* result);
+
+  // Helper method for reading a varint which can span at max `max_bytes`.
+  // If the varint is longer, a DataLoss error status is returned.
+  // If end of file is reached while reading, OutOfRange error is returned.
+  template <typename T>
+  Status ReadVarintFallback(T* result, int max_bytes);
+
   RandomAccessFile* file_;  // Not owned
   int64 file_pos_;          // Next position to read from in "file_"
   size_t size_;             // Size of "buf_"
@@ -109,6 +121,20 @@ inline Status InputBuffer::ReadVarint32(uint32* result) {
   }
 }
 
+// Inlined for performance.
+inline Status InputBuffer::ReadVarint64(uint64* result) {
+  if (pos_ + core::kMaxVarint64Bytes <= limit_) {
+    // Fast path: directly parse from buffered data.
+    // Reads strictly from the range [pos_, limit_).
+    const char* offset = core::GetVarint64Ptr(pos_, limit_, result);
+    if (offset == nullptr) return errors::OutOfRange("Parsed past limit.");
+    pos_ = const_cast<char*>(offset);
+    return Status::OK();
+  } else {
+    return ReadVarint64Fallback(result);
+  }
+}
+
 }  // namespace io
 }  // namespace tensorflow
 
diff --git a/tensorflow/core/lib/io/inputbuffer_test.cc b/tensorflow/core/lib/io/inputbuffer_test.cc
index 6771697a16..6be1f819c2 100644
--- a/tensorflow/core/lib/io/inputbuffer_test.cc
+++ b/tensorflow/core/lib/io/inputbuffer_test.cc
@@ -329,5 +329,44 @@ TEST(InputBuffer, ReadVarint32) {
   }
 }
 
+TEST(InputBuffer, ReadVarint64) {
+  Env* env = Env::Default();
+  string fname = testing::TmpDir() + "/inputbuffer_test";
+
+  // Generates data.
+  std::vector<uint64> data;
+  uint64 i = 0;
+  for (; i < (1U << 10); i += 1) data.push_back(i);
+  for (; i < (1U << 15); i += 5) data.push_back(i);
+  for (; i < (1U << 31); i += 164817) data.push_back(i);
+  for (; i < (1ULL << 63); i += 16481797854795663UL) data.push_back(i);
+  data.push_back(std::numeric_limits<uint64>::max());
+
+  // Writes the varints.
+  {
+    std::unique_ptr<WritableFile> file;
+    TF_CHECK_OK(env->NewWritableFile(fname, &file));
+    string varint;
+    for (uint64 number : data) {
+      varint.clear();
+      core::PutVarint64(&varint, number);
+      TF_CHECK_OK(file->Append(StringPiece(varint)));
+    }
+  }
+
+  for (auto buf_size : BufferSizes()) {
+    std::unique_ptr<RandomAccessFile> file;
+    TF_CHECK_OK(env->NewRandomAccessFile(fname, &file));
+    io::InputBuffer in(file.get(), buf_size);
+    uint64 result = 0;
+
+    for (uint64 expected : data) {
+      TF_ASSERT_OK(in.ReadVarint64(&result));
+      EXPECT_EQ(expected, result);
+    }
+    EXPECT_TRUE(errors::IsOutOfRange(in.ReadVarint64(&result)));
+  }
+}
+
 }  // namespace
 }  // namespace tensorflow
diff --git a/tensorflow/core/util/tensor_bundle/BUILD b/tensorflow/core/util/tensor_bundle/BUILD
index 4e957ec3df..166bd0f659 100644
--- a/tensorflow/core/util/tensor_bundle/BUILD
+++ b/tensorflow/core/util/tensor_bundle/BUILD
@@ -66,6 +66,7 @@ tf_cc_test(
     srcs = ["tensor_bundle_test.cc"],
     deps = [
         ":tensor_bundle",
+        "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
         "//tensorflow/core:protos_all_cc",
diff --git a/tensorflow/core/util/tensor_bundle/tensor_bundle.cc b/tensorflow/core/util/tensor_bundle/tensor_bundle.cc
index 02eb042a0b..d0e54b7e47 100644
--- a/tensorflow/core/util/tensor_bundle/tensor_bundle.cc
+++ b/tensorflow/core/util/tensor_bundle/tensor_bundle.cc
@@ -22,10 +22,14 @@ limitations under the License.
 #include <utility>
 
 #include "tensorflow/core/framework/register_types.h"
+#include "tensorflow/core/framework/tensor.pb.h"
 #include "tensorflow/core/framework/tensor_shape.pb_text.h"
 #include "tensorflow/core/framework/tensor_shape.pb.h"
 #include "tensorflow/core/framework/types.h"
 #include "tensorflow/core/framework/types.pb_text.h"
+#include "tensorflow/core/framework/variant.h"
+#include "tensorflow/core/framework/variant_op_registry.h"
+#include "tensorflow/core/framework/variant_tensor_data.h"
 #include "tensorflow/core/framework/versions.h"
 #include "tensorflow/core/framework/versions.pb.h"
 #include "tensorflow/core/lib/core/coding.h"
@@ -109,6 +113,64 @@ Status ReadStringTensor(io::InputBuffer* buffered_file, size_t num_elements,
   return Status::OK();
 }
 
+Status ReadVariantTensor(io::InputBuffer* buffered_file, Tensor* ret,
+                         size_t offset, size_t size, uint32* actual_crc32c) {
+  // On-disk format:
+  //   [varint64 len1][bytes variant1][4 byte checksum]
+  //   ..
+  //   [varint64 lenN][bytes variantN][4 byte checksum]
+  // Var "crc32c" checksums all the lens, variant bytes, individual variant
+  // checksums (as uint32, not varint32 bytes).
+  if (size == 0) return Status::OK();
+  size_t num_elements = ret->NumElements();
+
+  // Reads the actual string bytes.
+  TF_RETURN_IF_ERROR(buffered_file->Seek(offset));
+  for (size_t i = 0; i < num_elements; ++i) {
+    // Read the serialized variant length.
+    uint64 string_length = 0;
+    TF_RETURN_IF_ERROR(buffered_file->ReadVarint64(&string_length));
+    *actual_crc32c = crc32c::Extend(
+        *actual_crc32c, reinterpret_cast<const char*>(&string_length),
+        sizeof(uint64));
+    // Read the actual serialized variant.
+    string buffer;
+    buffer.resize(string_length);
+    size_t bytes_read = 0;
+    TF_RETURN_IF_ERROR(
+        buffered_file->ReadNBytes(string_length, &buffer[0], &bytes_read));
+    *actual_crc32c = crc32c::Extend(*actual_crc32c, buffer.data(), bytes_read);
+    VariantTensorDataProto proto;
+    proto.ParseFromString(buffer);
+    Variant v = proto;
+    if (!DecodeUnaryVariant(&v)) {
+      return errors::Internal("Could not decode variant with type_name: \"",
+                              v.TypeName(), "\".  Perhaps you forgot to ",
+                              "register a decoder via ",
+                              "REGISTER_UNARY_VARIANT_DECODE_FUNCTION?");
+    }
+
+    // Read the checksum.
+    uint32 checksum = 0;
+    size_t unused_bytes_read = 0;
+    TF_RETURN_IF_ERROR(buffered_file->ReadNBytes(
+        sizeof(uint32), reinterpret_cast<char*>(&checksum),
+        &unused_bytes_read));
+    if (crc32c::Unmask(checksum) != *actual_crc32c) {
+      return errors::DataLoss(
+          "The checksum after Variant ", i, " does not match.",
+          " Expected: ", strings::Printf("%08u", crc32c::Unmask(checksum)),
+          " Actual: ", strings::Printf("%08u", *actual_crc32c));
+    }
+    *actual_crc32c = crc32c::Extend(
+        *actual_crc32c, reinterpret_cast<char*>(&checksum), sizeof(uint32));
+
+    ret->flat<Variant>()(i) = std::move(v);
+  }
+
+  return Status::OK();
+}
+
 char* GetBackingBuffer(const Tensor& val) {
   CHECK(DataTypeCanUseMemcpy(val.dtype())) << val.dtype();
   return const_cast<char*>(val.tensor_data().data());
@@ -134,6 +196,7 @@ Status ParseEntryProto(StringPiece key, StringPiece value,
 Status WriteTensor(const Tensor& val, FileOutputBuffer* out,
                    size_t* bytes_written) {
   DCHECK_NE(val.dtype(), DT_STRING);
+  DCHECK_NE(val.dtype(), DT_VARIANT);
   *bytes_written = val.TotalBytes();
   char* buf = GetBackingBuffer(val);
   VLOG(1) << "Appending " << *bytes_written << " bytes to file";
@@ -188,6 +251,54 @@ Status WriteStringTensor(const Tensor& val, FileOutputBuffer* out,
   return Status::OK();
 }
 
+Status WriteVariantTensor(const Tensor& val, FileOutputBuffer* out,
+                          size_t* bytes_written, uint32* crc32c) {
+  // On-disk format:
+  //   [varint64 len1][bytes variant1][4 byte checksum]
+  //   ..
+  //   [varint64 lenN][bytes variantN][4 byte checksum]
+  // Var "crc32c" checksums all the lens, variant bytes, individual variant
+  // checksums (as uint32, not varint32 bytes).
+  DCHECK_EQ(val.dtype(), DT_VARIANT);
+
+  *crc32c = 0;
+  *bytes_written = 0;
+  for (int64 i = 0; i < val.NumElements(); ++i) {
+    VariantTensorData data;
+    val.flat<Variant>()(i).Encode(&data);
+    VariantTensorDataProto proto;
+    data.ToProto(&proto);
+    string elem;
+    proto.SerializeToString(&elem);
+
+    // Write the length of the serialized variant.
+    DCHECK_EQ(elem.size(), static_cast<uint64>(elem.size()));
+    const auto elem_size = static_cast<uint64>(elem.size());
+    string len;
+    core::PutVarint64(&len, elem_size);
+    TF_RETURN_IF_ERROR(out->Append(len));
+    *crc32c = crc32c::Extend(*crc32c, reinterpret_cast<const char*>(&elem_size),
+                             sizeof(uint64));
+    *bytes_written += sizeof(uint64);
+
+    // Write the serialized variant.
+    TF_RETURN_IF_ERROR(out->Append(elem));
+    *crc32c = crc32c::Extend(*crc32c, elem.data(), elem.size());
+    *bytes_written += elem.size();
+
+    // Write the checksum.
+    const uint32 length_checksum = crc32c::Mask(*crc32c);
+    TF_RETURN_IF_ERROR(out->Append(StringPiece(
+        reinterpret_cast<const char*>(&length_checksum), sizeof(uint32))));
+    *crc32c =
+        crc32c::Extend(*crc32c, reinterpret_cast<const char*>(&length_checksum),
+                       sizeof(uint32));
+    *bytes_written += sizeof(uint32);
+  }
+
+  return Status::OK();
+}
+
 // Reads file[offset:offset+size) into destination[0:size).  Each Read() copies
 // at most "buffer_size" bytes.
 //
@@ -312,11 +423,13 @@ Status BundleWriter::Add(StringPiece key, const Tensor& val) {
   size_t data_bytes_written = 0;
   uint32 crc32c = 0;
   out_->clear_crc32c();
-  if (val.dtype() != DT_STRING) {
+  if (val.dtype() == DT_STRING) {
+    status_ = WriteStringTensor(val, out_.get(), &data_bytes_written, &crc32c);
+  } else if (val.dtype() == DT_VARIANT) {
+    status_ = WriteVariantTensor(val, out_.get(), &data_bytes_written, &crc32c);
+  } else {
     status_ = WriteTensor(val, out_.get(), &data_bytes_written);
     crc32c = out_->crc32c();
-  } else {
-    status_ = WriteStringTensor(val, out_.get(), &data_bytes_written, &crc32c);
   }
 
   if (status_.ok()) {
@@ -707,13 +820,13 @@ Status BundleReader::GetValue(const BundleEntryProto& entry, Tensor* val) {
   }
 
   // Validates the "size" field.
-  if (entry.dtype() != DT_STRING) {
+  if (entry.dtype() != DT_STRING && entry.dtype() != DT_VARIANT) {
     if (entry.size() != ret->TotalBytes()) {
       return errors::DataLoss("Invalid size in bundle entry: key ", key(),
                               "; stored size ", entry.size(),
                               "; expected size ", ret->TotalBytes());
     }
-  } else {
+  } else if (entry.dtype() == DT_STRING) {
     // Relaxes the check for string tensors as follows:
     //   entry.size() == bytes(varint lengths) + bytes(data)
     //                >= NumElems + bytes(data), since size bytes(varint) >= 1.
@@ -752,6 +865,11 @@ Status BundleReader::GetValue(const BundleEntryProto& entry, Tensor* val) {
                                         entry.size(), 8 << 20 /* 8MB buffer */,
                                         backing_buffer));
     actual_crc32c = crc32c::Value(backing_buffer, entry.size());
+  } else if (entry.dtype() == DT_VARIANT) {
+    // Relies on io::InputBuffer's buffering, because we issue many neighboring
+    // reads for a single string tensor.
+    TF_RETURN_IF_ERROR(ReadVariantTensor(buffered_file, ret, entry.offset(),
+                                         entry.size(), &actual_crc32c));
   } else {
     // Relies on io::InputBuffer's buffering, because we issue many neighboring
     // reads for a single string tensor.
diff --git a/tensorflow/core/util/tensor_bundle/tensor_bundle_test.cc b/tensorflow/core/util/tensor_bundle/tensor_bundle_test.cc
index 4ee1578510..341aae36f4 100644
--- a/tensorflow/core/util/tensor_bundle/tensor_bundle_test.cc
+++ b/tensorflow/core/util/tensor_bundle/tensor_bundle_test.cc
@@ -20,6 +20,8 @@ limitations under the License.
 
 #include "tensorflow/core/framework/tensor_testutil.h"
 #include "tensorflow/core/framework/types.pb.h"
+#include "tensorflow/core/framework/variant.h"
+#include "tensorflow/core/framework/variant_op_registry.h"
 #include "tensorflow/core/framework/versions.pb.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
 #include "tensorflow/core/lib/io/path.h"
@@ -64,6 +66,30 @@ void Expect(BundleReader* reader, const string& key,
   test::ExpectTensorEqual<T>(val, expected_val);
 }
 
+template <class T>
+void ExpectVariant(BundleReader* reader, const string& key,
+                   const Tensor& expected_t) {
+  // Tests for Contains().
+  EXPECT_TRUE(reader->Contains(key));
+  // Tests for LookupDtypeAndShape().
+  DataType dtype;
+  TensorShape shape;
+  TF_ASSERT_OK(reader->LookupDtypeAndShape(key, &dtype, &shape));
+  // Tests for Lookup(), checking tensor contents.
+  EXPECT_EQ(expected_t.dtype(), dtype);
+  EXPECT_EQ(expected_t.shape(), shape);
+  Tensor actual_t(dtype, shape);
+  TF_ASSERT_OK(reader->Lookup(key, &actual_t));
+  for (int i = 0; i < expected_t.NumElements(); i++) {
+    Variant actual_var = actual_t.flat<Variant>()(i);
+    Variant expected_var = expected_t.flat<Variant>()(i);
+    EXPECT_EQ(actual_var.TypeName(), expected_var.TypeName());
+    auto* actual_val = actual_var.get<T>();
+    auto* expected_val = expected_var.get<T>();
+    EXPECT_EQ(*expected_val, *actual_val);
+  }
+}
+
 template <typename T>
 void ExpectNext(BundleReader* reader, const Tensor& expected_val) {
   EXPECT_TRUE(reader->Valid());
@@ -460,6 +486,55 @@ TEST(TensorBundleTest, StringTensors) {
   }
 }
 
+class VariantObject {
+ public:
+  VariantObject() {}
+  VariantObject(const string& metadata, int64 value)
+      : metadata_(metadata), value_(value) {}
+
+  string TypeName() const { return "TEST VariantObject"; }
+  void Encode(VariantTensorData* data) const {
+    data->set_type_name(TypeName());
+    data->set_metadata(metadata_);
+    Tensor val_t = Tensor(DT_INT64, TensorShape({}));
+    val_t.scalar<int64>()() = value_;
+    *(data->add_tensors()) = val_t;
+  }
+  bool Decode(const VariantTensorData& data) {
+    EXPECT_EQ(data.type_name(), TypeName());
+    data.get_metadata(&metadata_);
+    EXPECT_EQ(data.tensors_size(), 1);
+    value_ = data.tensors(0).scalar<int64>()();
+    return true;
+  }
+  bool operator==(const VariantObject other) const {
+    return metadata_ == other.metadata_ && value_ == other.value_;
+  }
+  string metadata_;
+  int64 value_;
+};
+
+REGISTER_UNARY_VARIANT_DECODE_FUNCTION(VariantObject, "TEST VariantObject");
+
+TEST(TensorBundleTest, VariantTensors) {
+  {
+    BundleWriter writer(Env::Default(), Prefix("foo"));
+    TF_EXPECT_OK(
+        writer.Add("variant_tensor",
+                   test::AsTensor<Variant>({VariantObject("test", 10),
+                                            VariantObject("test1", 20)})));
+    TF_ASSERT_OK(writer.Finish());
+  }
+  {
+    BundleReader reader(Env::Default(), Prefix("foo"));
+    TF_ASSERT_OK(reader.status());
+    ExpectVariant<VariantObject>(
+        &reader, "variant_tensor",
+        test::AsTensor<Variant>(
+            {VariantObject("test", 10), VariantObject("test1", 20)}));
+  }
+}
+
 TEST(TensorBundleTest, DirectoryStructure) {
   Env* env = Env::Default();
   // Writes two bundles.
-- 
GitLab


From b46b741b7a8f04049b451be5299c61a373ec7612 Mon Sep 17 00:00:00 2001
From: Russell Power <power@google.com>
Date: Mon, 16 Oct 2017 13:46:18 -0700
Subject: [PATCH 444/909] Close session on infeed error.  This should fix most
 of the cases where the client process hangs waiting for the main training
 loop to exit.

PiperOrigin-RevId: 172371951
---
 .../contrib/tpu/python/tpu/tpu_estimator.py   | 30 ++++++++++++-------
 1 file changed, 19 insertions(+), 11 deletions(-)

diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
index de6c8140c6..04e0719a1b 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
@@ -312,17 +312,25 @@ class _InfeedThreadController(_InfeedOutfeedThreadBaseController):
 
   def _input_thread_fn_for_loading(self, session, enqueue_ops):
     count = 0
-    while True:
-      signal = self._signal_queue.get()
-      if signal == _SIGNAL.STOP:
-        logging.info('Stop Infeed input thread.')
-        return
-
-      iterations = signal
-      for i in range(iterations):
-        logging.debug('Infeed enqueue for iteration (%d, %d)', count, i)
-        session.run(enqueue_ops)
-      count += 1
+    try:
+      while True:
+        signal = self._signal_queue.get()
+        if signal == _SIGNAL.STOP:
+          logging.info('Stop Infeed input thread.')
+          return
+
+        iterations = signal
+        for i in range(iterations):
+          logging.debug('Infeed enqueue for iteration (%d, %d)', count, i)
+          session.run(enqueue_ops)
+        count += 1
+    except Exception:  # pylint: disable=broad-except
+      logging.error(
+          'Failed running infeed, closing session.\n'
+          'You may see an exception from your main session after this.',
+          exc_info=1
+      )
+      session.close()
 
   def join(self):
     logging.info('Waiting for Infeed Thread to exit.')
-- 
GitLab


From a36ff5499df443d768fd2f4ff810f9daba30d35a Mon Sep 17 00:00:00 2001
From: Mark Daoust <markdaoust@google.com>
Date: Mon, 16 Oct 2017 13:50:36 -0700
Subject: [PATCH 445/909] Respect __array__ and __array_interface__ for string
 types

__array__ fixes use-cases like:

  import tensorflow as tf
  import pandas as pd
  series = pd.Series(['a','b','c'])
  tf.constant(series)
  df = pd.DataFrame({'a':[1,2,3],'b':['a','b','c']})
  tf.data.Dataset.from_tensor_slices(dict(df))

PiperOrigin-RevId: 172372593
---
 tensorflow/python/framework/tensor_util.py    | 11 ++++--
 .../python/framework/tensor_util_test.py      | 39 +++++++++++++++++++
 2 files changed, 47 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/framework/tensor_util.py b/tensorflow/python/framework/tensor_util.py
index 414c61e930..63324e5977 100644
--- a/tensorflow/python/framework/tensor_util.py
+++ b/tensorflow/python/framework/tensor_util.py
@@ -362,10 +362,15 @@ def make_tensor_proto(values, dtype=None, shape=None, verify_shape=False):
       nparray = values.astype(dtype.as_numpy_dtype)
     else:
       nparray = values
-  elif callable(getattr(values, "__array__", None)):
-    # If a class has the __array__ method, then it is possible to convert
-    # to numpy array.
+  elif callable(getattr(values, "__array__", None)) or isinstance(
+      getattr(values, "__array_interface__", None), dict):
+    # If a class has the __array__ method, or __array_interface__ dict, then it
+    # is possible to convert to numpy array.
     nparray = np.asarray(values, dtype=dtype)
+
+    # This is the preferred way to create an array from the object, so replace
+    # the `values` with the array so that _FlattenToStrings is not run.
+    values = nparray
   else:
     if values is None:
       raise ValueError("None values not supported.")
diff --git a/tensorflow/python/framework/tensor_util_test.py b/tensorflow/python/framework/tensor_util_test.py
index c4937de936..dda72fc0c8 100644
--- a/tensorflow/python/framework/tensor_util_test.py
+++ b/tensorflow/python/framework/tensor_util_test.py
@@ -492,6 +492,45 @@ class TensorUtilTest(test.TestCase):
     self.assertEquals(np.object, a.dtype)
     self.assertAllEqual(np.array([[b"a", b"ab"], [b"abc", b"abcd"]]), a)
 
+  def testArrayMethod(self):
+
+    class Wrapper(object):
+
+      def __array__(self):
+        return np.array([b"foo", b"bar", b"baz"])
+
+    t = tensor_util.make_tensor_proto(Wrapper(), shape=[1, 3])
+    self.assertProtoEquals("""
+      dtype: DT_STRING
+      tensor_shape { dim { size: 1 } dim { size: 3 } }
+      string_val: "foo"
+      string_val: "bar"
+      string_val: "baz"
+      """, t)
+    a = tensor_util.MakeNdarray(t)
+    self.assertEquals(np.object, a.dtype)
+    self.assertAllEqual(np.array([[b"foo", b"bar", b"baz"]]), a)
+
+  def testArrayInterface(self):
+
+    class Wrapper(object):
+
+      @property
+      def __array_interface__(self):
+        return np.array([b"foo", b"bar", b"baz"]).__array_interface__
+
+    t = tensor_util.make_tensor_proto(Wrapper(), shape=[1, 3])
+    self.assertProtoEquals("""
+      dtype: DT_STRING
+      tensor_shape { dim { size: 1 } dim { size: 3 } }
+      string_val: "foo"
+      string_val: "bar"
+      string_val: "baz"
+      """, t)
+    a = tensor_util.MakeNdarray(t)
+    self.assertEquals(np.object, a.dtype)
+    self.assertAllEqual(np.array([[b"foo", b"bar", b"baz"]]), a)
+
   def testStringTuple(self):
     t = tensor_util.make_tensor_proto((b"a", b"ab", b"abc", b"abcd"))
     self.assertProtoEquals("""
-- 
GitLab


From 36ff23d81b271aaf4a4e106c042f76e80484d769 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 16 Oct 2017 14:01:43 -0700
Subject: [PATCH 446/909] Batch norm folding immediately fails if
 FusedBatchNorm ops are present.

PiperOrigin-RevId: 172374244
---
 .../quantize/python/fold_batch_norms.py       |  4 +++
 .../quantize/python/fold_batch_norms_test.py  | 28 +++++++++++++++++++
 2 files changed, 32 insertions(+)

diff --git a/tensorflow/contrib/quantize/python/fold_batch_norms.py b/tensorflow/contrib/quantize/python/fold_batch_norms.py
index c9d16fb329..c416689510 100644
--- a/tensorflow/contrib/quantize/python/fold_batch_norms.py
+++ b/tensorflow/contrib/quantize/python/fold_batch_norms.py
@@ -40,6 +40,10 @@ def FoldBatchNorms(graph):
   Raises:
     ValueError: When batch norm folding fails.
   """
+  # Fail immediately when the graph contains unsupported fused batch norm ops.
+  if any(op for op in graph.get_operations() if op.type == 'FusedBatchNorm'):
+    raise ValueError('Fused batch norm is not supported')
+
   input_to_ops_map = input_to_ops.InputToOps(graph)
 
   for bn in common.BatchNormGroups(graph):
diff --git a/tensorflow/contrib/quantize/python/fold_batch_norms_test.py b/tensorflow/contrib/quantize/python/fold_batch_norms_test.py
index 4f11188a55..ddedb0a2c0 100644
--- a/tensorflow/contrib/quantize/python/fold_batch_norms_test.py
+++ b/tensorflow/contrib/quantize/python/fold_batch_norms_test.py
@@ -57,6 +57,34 @@ class FoldBatchNormsTest(test_util.TensorFlowTestCase):
     for parameters in parameters_list:
       test_fn(parameters[0], parameters[1], parameters[2])
 
+  def testFailsWithFusedBatchNorm(self):
+    self._RunTestOverParameters(self._TestFailsWithFusedBatchNorm)
+
+  def _TestFailsWithFusedBatchNorm(self, relu, relu_op_name, with_bypass):
+    """Tests that batch norm fails when fused batch norm ops are present."""
+    g = ops.Graph()
+    with g.as_default():
+      batch_size, height, width = 5, 128, 128
+      inputs = array_ops.zeros((batch_size, height, width, 3))
+      out_depth = 3 if with_bypass else 32
+      stride = 1 if with_bypass else 2
+      activation_fn = None if with_bypass else relu
+      batch_norm_params = _DEFAULT_BATCH_NORM_PARAMS.copy()
+      batch_norm_params['fused'] = True
+      scope = 'test/test2' if with_bypass else 'test'
+      node = conv2d(inputs, out_depth, [5, 5], stride=stride, padding='SAME',
+                    weights_initializer=self._WeightInit(0.09),
+                    activation_fn=activation_fn,
+                    normalizer_fn=batch_norm,
+                    normalizer_params=batch_norm_params,
+                    scope=scope)
+      if with_bypass:
+        node = math_ops.add(inputs, node, name='test/Add')
+        relu(node, name='test/' + relu_op_name)
+
+      with self.assertRaises(ValueError):
+        fold_batch_norms.FoldBatchNorms(g)
+
   def _TestFoldConv2d(self, relu, relu_op_name, with_bypass):
     """Tests folding cases: inputs -> Conv2d with batch norm -> Relu*.
 
-- 
GitLab


From 7b3ea8e6176319467cb1a49a1a662d868a205b91 Mon Sep 17 00:00:00 2001
From: Jacques Pienaar <jpienaar@google.com>
Date: Mon, 16 Oct 2017 14:18:10 -0700
Subject: [PATCH 447/909] [TF2XLA] Expand comparator and use consistently in
 sorting arguments.

PiperOrigin-RevId: 172376836
---
 .../tf2xla/functionalize_control_flow.cc      | 32 ++++++++++---------
 .../tf2xla/functionalize_control_flow_test.cc | 20 ++++++------
 2 files changed, 27 insertions(+), 25 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/functionalize_control_flow.cc b/tensorflow/compiler/tf2xla/functionalize_control_flow.cc
index abfc856904..35b6960a98 100644
--- a/tensorflow/compiler/tf2xla/functionalize_control_flow.cc
+++ b/tensorflow/compiler/tf2xla/functionalize_control_flow.cc
@@ -590,8 +590,13 @@ class FunctionalizeCond {
   // id in the original graph.
   struct CondArgs {
     struct CondCmp {
-      bool operator()(const Node* a, const Node* b) const {
-        return a->id() < b->id();
+      bool operator()(const Node* lhs, const Node* rhs) const {
+        bool lhs_is_resource =
+            lhs->num_inputs() > 0 ? (lhs->input_type(0) == DT_RESOURCE) : false;
+        bool rhs_is_resource =
+            rhs->num_inputs() > 0 ? (rhs->input_type(0) == DT_RESOURCE) : false;
+        return std::tie(lhs_is_resource, lhs->name()) <
+               std::tie(rhs_is_resource, rhs->name());
       }
     };
     Node* conditional = nullptr;
@@ -710,7 +715,7 @@ std::ostream& operator<<(std::ostream& os,
 // between the nodes and the nodes in each cluster.
 string DebugString(const Graph& graph,
                    FunctionalizeCond::ClusterHandle::Vector* clusters) {
-  string ret = "digraph {\ncompound=true;labeljust=\"r\";\n";
+  string ret = "digraph {\ncompound=true;labeljust=\"r\";ranksep=0.24\n";
   std::map<FunctionalizeCond::ClusterHandle, string> subgraphs;
   for (Node* n : graph.nodes()) {
     if (n->IsOp()) {
@@ -720,8 +725,8 @@ string DebugString(const Graph& graph,
   }
   for (auto kv : subgraphs) {
     strings::StrAppend(&ret, "subgraph cluster_", kv.first.ToString(), " {\n",
-                       "label = \"", kv.first.ToString(), "\";\n", kv.second,
-                       "}\n");
+                       "style=filled; color=lightgrey;", "label = \"",
+                       kv.first.ToString(), "\";\n", kv.second, "}\n");
   }
   for (Node* n : graph.nodes()) {
     if (!n->IsOp()) {
@@ -1110,11 +1115,6 @@ Status FunctionalizeCond::ExtractBody(const CondArgs& cond_args,
     DataType dtype = arg->input_type(0);
     TF_ASSIGN_OR_RETURN(Node * arg_node,
                         BuildArgNode(body, dtype, arg_count++));
-    if (dtype == DT_RESOURCE) {
-      bool constant;
-      TF_RETURN_IF_ERROR(GetNodeAttr(arg->attrs(), "is_constant", &constant));
-      TF_RET_CHECK(constant);
-    }
     node_map.at(arg->id()) = arg_node;
     squash_src_outputs.at(arg->id()) = true;
   }
@@ -1247,9 +1247,7 @@ Status FunctionalizeCond::ConvertMergeToXlaIf(Cluster* merge_cluster) {
   // Sort the outputs by ID to produce more stable output.
   std::vector<Node*> outputs(merge_cluster->merge_nodes.begin(),
                              merge_cluster->merge_nodes.end());
-  std::sort(
-      outputs.begin(), outputs.end(),
-      [](const Node* lhs, const Node* rhs) { return lhs->id() < rhs->id(); });
+  std::sort(outputs.begin(), outputs.end(), CondArgs::CondCmp());
 
   // Extract bodies and builds a If operator.
   TF_ASSIGN_OR_RETURN(Node * if_node,
@@ -1370,7 +1368,7 @@ Status FunctionalizeCond::Functionalize(Graph* graph,
 // functional equivalents.
 Status FunctionalizeControlFlow(Graph* graph,
                                 FunctionLibraryDefinition* library) {
-  VLOG(2) << "FunctionalizeControlFlow: "
+  VLOG(2) << "FunctionalizeControlFlow (initial): "
           << dump_graph::DumpGraphToFile("functionalize_initial", *graph);
   // Note: BuildControlFlowInfo() requires that the graph's source node is
   // connected to all source nodes in the graph. Many graphs violate this
@@ -1448,7 +1446,11 @@ Status FunctionalizeControlFlow(Graph* graph,
   // FunctionalizeControlFlow is invoked for every function, so the loops's
   // bodies and conditionals that were extracted into functions will be handled
   // in successive invocations.
-  return FunctionalizeCond::Functionalize(graph, library);
+  TF_RETURN_IF_ERROR(FunctionalizeCond::Functionalize(graph, library));
+
+  VLOG(2) << "FunctionalizeControlFlow (final): "
+          << dump_graph::DumpGraphToFile("functionalize_final", *graph);
+  return Status::OK();
 }
 
 }  // namespace tensorflow
diff --git a/tensorflow/compiler/tf2xla/functionalize_control_flow_test.cc b/tensorflow/compiler/tf2xla/functionalize_control_flow_test.cc
index 4acdf1a26d..01d2b28275 100644
--- a/tensorflow/compiler/tf2xla/functionalize_control_flow_test.cc
+++ b/tensorflow/compiler/tf2xla/functionalize_control_flow_test.cc
@@ -110,7 +110,7 @@ TEST(FunctionalizeControlFlow, Conditional) {
     auto x = ops::Placeholder(scope.WithOpName("x"), DT_INT32);
     auto less = ops::Less(scope.WithOpName("cond/Less"), y, x);
     auto if_op = ops::XlaIf(scope.WithOpName("cond/Merge_If"), less,
-                            std::initializer_list<Input>{x, y, less}, then_fn,
+                            std::initializer_list<Input>{less, y, x}, then_fn,
                             else_fn, {DT_INT32});
     GraphDef expected;
     TF_EXPECT_OK(scope.ToGraphDef(&expected));
@@ -120,10 +120,10 @@ TEST(FunctionalizeControlFlow, Conditional) {
   // then body.
   {
     Scope scope = Scope::NewRootScope().ExitOnError();
-    auto arg_0 = ops::_Arg(scope.WithOpName("_arg0"), DT_INT32, 0);
+    auto arg_0 = ops::_Arg(scope.WithOpName("_arg0"), DT_BOOL, 0);
     auto arg_1 = ops::_Arg(scope.WithOpName("_arg1"), DT_INT32, 1);
-    auto arg_2 = ops::_Arg(scope.WithOpName("_arg2"), DT_BOOL, 2);
-    auto identity = ops::Identity(scope.WithOpName("cond/Identity"), arg_2);
+    auto arg_2 = ops::_Arg(scope.WithOpName("_arg2"), DT_INT32, 2);
+    auto identity = ops::Identity(scope.WithOpName("cond/Identity"), arg_0);
     auto cond = ops::Const(
         scope.WithOpName("cond").WithControlDependencies(identity), 17);
     auto mul = ops::Mul(scope.WithOpName("cond/Mul"), arg_1, cond);
@@ -136,20 +136,20 @@ TEST(FunctionalizeControlFlow, Conditional) {
     TF_EXPECT_OK(InstantiateFunctionForTest(then_fn.name(), library, &result));
 
     EXPECT_EQ(DataTypeVector{DT_INT32}, result.ret_types);
-    EXPECT_EQ((DataTypeVector{DT_INT32, DT_INT32, DT_BOOL}), result.arg_types);
+    EXPECT_EQ((DataTypeVector{DT_BOOL, DT_INT32, DT_INT32}), result.arg_types);
     TF_EXPECT_GRAPH_EQ(expected, result.gdef);
   }
 
   // else body.
   {
     Scope scope = Scope::NewRootScope().ExitOnError();
-    auto arg_0 = ops::_Arg(scope.WithOpName("_arg0"), DT_INT32, 0);
+    auto arg_0 = ops::_Arg(scope.WithOpName("_arg0"), DT_BOOL, 0);
     auto arg_1 = ops::_Arg(scope.WithOpName("_arg1"), DT_INT32, 1);
-    auto arg_2 = ops::_Arg(scope.WithOpName("_arg2"), DT_BOOL, 2);
-    auto identity = ops::Identity(scope.WithOpName("cond/Identity_1"), arg_2);
+    auto arg_2 = ops::_Arg(scope.WithOpName("_arg2"), DT_INT32, 2);
+    auto identity = ops::Identity(scope.WithOpName("cond/Identity_1"), arg_0);
     auto cond_1 = ops::Const(
         scope.WithOpName("cond_1").WithControlDependencies(identity), 23);
-    auto add = ops::Add(scope.WithOpName("cond/false/add"), arg_0, cond_1);
+    auto add = ops::Add(scope.WithOpName("cond/false/add"), arg_2, cond_1);
     auto retval0 = ops::_Retval(scope.WithOpName("_retval0_RetVal"), add, 0);
 
     GraphDef expected;
@@ -159,7 +159,7 @@ TEST(FunctionalizeControlFlow, Conditional) {
     TF_EXPECT_OK(InstantiateFunctionForTest(else_fn.name(), library, &result));
 
     EXPECT_EQ(DataTypeVector{DT_INT32}, result.ret_types);
-    EXPECT_EQ((DataTypeVector{DT_INT32, DT_INT32, DT_BOOL}), result.arg_types);
+    EXPECT_EQ((DataTypeVector{DT_BOOL, DT_INT32, DT_INT32}), result.arg_types);
     TF_EXPECT_GRAPH_EQ(expected, result.gdef);
   }
 }
-- 
GitLab


From 7fd47e4d2de009eba0698e7b5f65ae899f6f6624 Mon Sep 17 00:00:00 2001
From: Skye Wanderman-Milne <skyewm@google.com>
Date: Mon, 16 Oct 2017 14:34:38 -0700
Subject: [PATCH 448/909] Enable C API for gradients_test.py

PiperOrigin-RevId: 172379338
---
 tensorflow/python/ops/gradients_test.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/tensorflow/python/ops/gradients_test.py b/tensorflow/python/ops/gradients_test.py
index de3dd03486..f0cffbab30 100644
--- a/tensorflow/python/ops/gradients_test.py
+++ b/tensorflow/python/ops/gradients_test.py
@@ -78,6 +78,7 @@ def _OpsBetween(graph, to_ops, from_ops):
   return between_ops
 
 
+@test_util.with_c_api
 class GradientsTest(test_util.TensorFlowTestCase):
 
   def _OpNames(self, op_list):
@@ -264,6 +265,10 @@ class GradientsTest(test_util.TensorFlowTestCase):
       self.assertEqual(10.0, grads[1].eval())
 
   def testNoGradientForStringOutputs(self):
+    # This test can't be run twice because the TestStringOutput gradient can
+    # only be registered once. Just run with the C API enabled.
+    if not ops._USE_C_API: return
+
     with ops.Graph().as_default():
 
       def _TestOpGrad(_, float_grad, string_grad):
@@ -409,6 +414,7 @@ class GradientsTest(test_util.TensorFlowTestCase):
         np.testing.assert_allclose(a, b)
 
 
+@test_util.with_c_api
 class FunctionGradientsTest(test_util.TensorFlowTestCase):
 
   @classmethod
@@ -498,6 +504,7 @@ class FunctionGradientsTest(test_util.TensorFlowTestCase):
         f.add_to_graph(ops.Graph())
 
 
+@test_util.with_c_api
 class StopGradientTest(test_util.TensorFlowTestCase):
 
   def testStopGradient(self):
@@ -508,6 +515,7 @@ class StopGradientTest(test_util.TensorFlowTestCase):
     assert igrad is None
 
 
+@test_util.with_c_api
 class PreventGradientTest(test_util.TensorFlowTestCase):
 
   def testPreventGradient(self):
@@ -518,6 +526,7 @@ class PreventGradientTest(test_util.TensorFlowTestCase):
         _ = gradients.gradients(out, inp)
 
 
+@test_util.with_c_api
 class HessianVectorProductTest(test_util.TensorFlowTestCase):
 
   def testHessianVectorProduct(self):
@@ -546,6 +555,7 @@ class HessianVectorProductTest(test_util.TensorFlowTestCase):
       self.assertAllClose(hess_v_value, hess_v_actual)
 
 
+@test_util.with_c_api
 class HessianTest(test_util.TensorFlowTestCase):
 
   def testHessian1D(self):
@@ -594,6 +604,7 @@ class HessianTest(test_util.TensorFlowTestCase):
           gradients.hessians(x, x)
 
 
+@test_util.with_c_api
 class IndexedSlicesToTensorTest(test_util.TensorFlowTestCase):
 
   def testIndexedSlicesToTensor(self):
@@ -651,6 +662,9 @@ class IndexedSlicesToTensorTest(test_util.TensorFlowTestCase):
     c_sparse = ops.IndexedSlices(
         array_ops.placeholder(dtypes.float32),
         array_ops.placeholder(dtypes.int32), constant([100, 100, 100, 100]))
+    # "always" filter prevents the warning from being suppressed if it was
+    # already triggered in a different test.
+    warnings.simplefilter("always")
     with warnings.catch_warnings(record=True) as w:
       math_ops.multiply(c_sparse, 1.0)
     self.assertEqual(1, len(w))
@@ -671,6 +685,7 @@ class IndexedSlicesToTensorTest(test_util.TensorFlowTestCase):
         str(w[0].message))
 
 
+@test_util.with_c_api
 class OnlyRealGradientsTest(test_util.TensorFlowTestCase):
 
   def testRealOnly(self):
-- 
GitLab


From afaacfdb4bbece8acc03b0456cfe2819db01f5c8 Mon Sep 17 00:00:00 2001
From: Akshay Agrawal <akshayka@google.com>
Date: Mon, 16 Oct 2017 14:43:07 -0700
Subject: [PATCH 449/909] Default to procuring ResourceVariables in
 variable_scope.variable when use_resource is not set and Eager mode is
 enabled.

PiperOrigin-RevId: 172380659
---
 tensorflow/python/ops/variable_scope.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/ops/variable_scope.py b/tensorflow/python/ops/variable_scope.py
index 87805b5171..4614110ba6 100644
--- a/tensorflow/python/ops/variable_scope.py
+++ b/tensorflow/python/ops/variable_scope.py
@@ -1927,11 +1927,17 @@ def variable(initial_value=None,
              caching_device=None,
              name=None,
              dtype=None):
-  if get_variable_scope().use_resource:
+  use_resource = get_variable_scope().use_resource
+  if use_resource or (use_resource is None and context.in_eager_mode()):
     return resource_variable_ops.ResourceVariable(
         initial_value=initial_value, trainable=trainable,
         collections=collections, validate_shape=validate_shape,
         caching_device=caching_device, name=name, dtype=dtype)
+  elif not use_resource and context.in_eager_mode():
+    raise RuntimeError(
+        "VariableScope should use resource variable in Eager mode, but "
+        "use_resource is False."
+    )
   else:
     return variables.Variable(
         initial_value=initial_value, trainable=trainable,
-- 
GitLab


From 4e1d0f4e32b7f7a463930543dc773997bdb9d545 Mon Sep 17 00:00:00 2001
From: Justine Tunney <jart@google.com>
Date: Mon, 16 Oct 2017 14:57:15 -0700
Subject: [PATCH 450/909] Fix broken link in debugger doc (#13757)

---
 tensorflow/docs_src/programmers_guide/debugger.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/docs_src/programmers_guide/debugger.md b/tensorflow/docs_src/programmers_guide/debugger.md
index 3ede42e8f7..3f9f155457 100644
--- a/tensorflow/docs_src/programmers_guide/debugger.md
+++ b/tensorflow/docs_src/programmers_guide/debugger.md
@@ -440,7 +440,7 @@ accuracy_score = classifier.evaluate(x=test_set.data,
 
 
 [debug_tflearn_iris.py](https://www.tensorflow.org/code/tensorflow/python/debug/examples/debug_tflearn_iris.py),
-based on {$tflearn$tf-learn's iris tutorial}, contains a full example of how to
+based on @{$tflearn$tf-learn's iris tutorial}, contains a full example of how to
 use the tfdbg with `Estimator`s. To run this example, do:
 
 ```none
-- 
GitLab


From 7b6eec7e1175624458a48945bba3f6400e754d33 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Mon, 16 Oct 2017 15:25:22 -0700
Subject: [PATCH 451/909] Add cc file with definition of
 tensorflow::gtl::nullopt.

If you ODR-use nullopt, you currently get a linker error.  Oops.

PiperOrigin-RevId: 172387553
---
 tensorflow/core/lib/gtl/optional.cc | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)
 create mode 100644 tensorflow/core/lib/gtl/optional.cc

diff --git a/tensorflow/core/lib/gtl/optional.cc b/tensorflow/core/lib/gtl/optional.cc
new file mode 100644
index 0000000000..8dea073788
--- /dev/null
+++ b/tensorflow/core/lib/gtl/optional.cc
@@ -0,0 +1,25 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/lib/gtl/optional.h"
+
+namespace tensorflow {
+namespace gtl {
+
+nullopt_t::init_t nullopt_t::init;
+extern const nullopt_t nullopt{nullopt_t::init};
+
+}  // namespace gtl
+}  // namespace tensorflow
-- 
GitLab


From dc442f4ce2d3b11b56721337fe2b9e2282be93be Mon Sep 17 00:00:00 2001
From: Skye Wanderman-Milne <skyewm@google.com>
Date: Mon, 16 Oct 2017 15:29:59 -0700
Subject: [PATCH 452/909] Add return_nodes option to ImportGraphDef

The is similar to the return_tensors option. return_tensors cannot be
used to fetch nodes with no outputs, so return_nodes is necessary.

In addition, this change also refactors the ImportGraphDef signature
to return all optional return values in a single struct. This is to
keep the ImportGraphDef signature from getting too long, and also
makes the call sites simpler.

PiperOrigin-RevId: 172388270
---
 tensorflow/c/c_api.cc                         |  18 +-
 tensorflow/c/while_loop_test.cc               |   6 +-
 tensorflow/core/graph/graph_constructor.cc    |  73 +++++--
 tensorflow/core/graph/graph_constructor.h     |  60 ++++--
 .../core/graph/graph_constructor_test.cc      | 191 ++++++++++++------
 5 files changed, 240 insertions(+), 108 deletions(-)

diff --git a/tensorflow/c/c_api.cc b/tensorflow/c/c_api.cc
index 334f867e47..79fbd8c90c 100644
--- a/tensorflow/c/c_api.cc
+++ b/tensorflow/c/c_api.cc
@@ -1854,18 +1854,18 @@ static void GraphImportGraphDefLocked(TF_Graph* graph, const GraphDef& def,
     return;
   }
   const int last_node_id = graph->graph.num_node_ids();
-  std::vector<std::pair<Node*, int>> return_outputs_vec;
-  status->status = tensorflow::ImportGraphDef(
-      opts->opts, def, &graph->graph, &graph->refiner, &return_outputs_vec);
+  tensorflow::ImportGraphDefResults results;
+  status->status = tensorflow::ImportGraphDef(opts->opts, def, &graph->graph,
+                                              &graph->refiner, &results);
   if (!status->status.ok()) return;
   for (int i = last_node_id; i < graph->graph.num_node_ids(); ++i) {
     auto* node = graph->graph.FindNodeId(i);
     if (node != nullptr) graph->name_map[node->name()] = node;
   }
-  DCHECK_EQ(return_outputs_vec.size(), num_return_outputs);
+  DCHECK_EQ(results.return_tensors.size(), num_return_outputs);
   for (int i = 0; i < num_return_outputs; ++i) {
-    return_outputs[i].oper = ToOperation(return_outputs_vec[i].first);
-    return_outputs[i].index = return_outputs_vec[i].second;
+    return_outputs[i].oper = ToOperation(results.return_tensors[i].first);
+    return_outputs[i].index = results.return_tensors[i].second;
   }
 }
 
@@ -1945,11 +1945,11 @@ Status CopyGraph(Graph* src_graph, Graph* dst_graph,
   }
 
   // TOOD(skyewm): change to OutputTensor
-  std::vector<std::pair<Node*, int>> return_tensors;
+  tensorflow::ImportGraphDefResults results;
   TF_RETURN_IF_ERROR(
-      ImportGraphDef(opts, gdef, dst_graph, dst_refiner, &return_tensors));
+      ImportGraphDef(opts, gdef, dst_graph, dst_refiner, &results));
 
-  for (const auto& pair : return_tensors) {
+  for (const auto& pair : results.return_tensors) {
     return_nodes->emplace_back(pair.first, pair.second);
   }
   return Status::OK();
diff --git a/tensorflow/c/while_loop_test.cc b/tensorflow/c/while_loop_test.cc
index 2423d83dda..d2d887f32c 100644
--- a/tensorflow/c/while_loop_test.cc
+++ b/tensorflow/c/while_loop_test.cc
@@ -318,7 +318,7 @@ TEST_F(CApiWhileLoopTest, InvalidCondOutputNode) {
   // TODO(skyewm): this error message could be more informative. Add explicit
   // checks for this case in the while loop implementation?
   ExpectError(TF_INVALID_ARGUMENT,
-              "Requested return node 'p0' not found in graph def");
+              "Requested return tensor 'p0:0' not found in graph def");
 }
 
 TEST_F(CApiWhileLoopTest, InvalidCondOutputIndex) {
@@ -358,7 +358,7 @@ TEST_F(CApiWhileLoopTest, InvalidBodyOutputNode) {
   // TODO(skyewm): this error message could be more informative. Add explicit
   // checks for this case in the while loop implementation?
   ExpectError(TF_INVALID_ARGUMENT,
-              "Requested return node 'p0' not found in graph def");
+              "Requested return tensor 'p0:0' not found in graph def");
 }
 
 // TODO(skyewm): enable this when it works (currently segfaults!)
@@ -389,7 +389,7 @@ TEST_F(CApiWhileLoopTest, WrongGraph) {
   params_->body_outputs[0] = inputs_[0];
   // TODO(skyewm): improve error message
   ExpectError(TF_INVALID_ARGUMENT,
-              "Requested return node 'p0' not found in graph def");
+              "Requested return tensor 'p0:0' not found in graph def");
 }
 
 TEST_F(CApiWhileLoopTest, BadTypes) {
diff --git a/tensorflow/core/graph/graph_constructor.cc b/tensorflow/core/graph/graph_constructor.cc
index 15f7b9fe8c..92b4843221 100644
--- a/tensorflow/core/graph/graph_constructor.cc
+++ b/tensorflow/core/graph/graph_constructor.cc
@@ -79,6 +79,7 @@ class GraphConstructor {
           skip_mapped_nodes(in.skip_mapped_nodes),
           control_dependencies(in.control_dependencies),
           return_tensors(in.return_tensors),
+          return_nodes(in.return_nodes),
           importing(true) {}
 
     bool allow_internal_ops;
@@ -89,6 +90,7 @@ class GraphConstructor {
     bool skip_mapped_nodes;
     std::vector<string> control_dependencies;
     std::vector<TensorId> return_tensors;
+    std::vector<StringPiece> return_nodes;
 
     // TODO(ashankar): This bool exists to separate out functionality required
     // to make ImportGraphDef a close equivalent of Python's import_graph_def
@@ -109,6 +111,7 @@ class GraphConstructor {
                           const FunctionDefLibrary* library, Graph* g,
                           ShapeRefiner* refiner,
                           std::vector<std::pair<Node*, int>>* return_tensors,
+                          std::vector<Node*>* return_nodes,
                           std::vector<TensorId>* unused_input_map_keys) {
     if (versions) {
       TF_RETURN_IF_ERROR(CheckVersions(*versions, TF_GRAPH_DEF_VERSION,
@@ -116,7 +119,7 @@ class GraphConstructor {
                                        "GraphDef", "graph"));
     }
     GraphConstructor c(opts, node_defs, versions, library, g, refiner,
-                       return_tensors, unused_input_map_keys);
+                       return_tensors, return_nodes, unused_input_map_keys);
     const Status s = c.TryImport();
     if (!s.ok()) c.Undo();
     return s;
@@ -128,6 +131,7 @@ class GraphConstructor {
                    const FunctionDefLibrary* library, Graph* g,
                    ShapeRefiner* refiner,
                    std::vector<std::pair<Node*, int>>* return_tensors,
+                   std::vector<Node*>* return_nodes,
                    std::vector<TensorId>* unused_input_map_keys)
       : opts_(opts),
         node_defs_(node_defs),
@@ -137,6 +141,7 @@ class GraphConstructor {
         original_versions_(g->versions()),
         refiner_(refiner),
         return_tensors_(return_tensors),
+        return_nodes_(return_nodes),
         unused_input_map_keys_(unused_input_map_keys) {}
 
   Status TryImport() {
@@ -148,6 +153,7 @@ class GraphConstructor {
     TF_RETURN_IF_ERROR(AddBackEdges());
     TF_RETURN_IF_ERROR(UpdateVersionDef());
     TF_RETURN_IF_ERROR(PopulateReturnTensors());
+    TF_RETURN_IF_ERROR(PopulateReturnNodes());
     FixupSourceAndSinkEdges(g_);
     return Status::OK();
   }
@@ -160,6 +166,7 @@ class GraphConstructor {
   Status AddBackEdges();
   Status UpdateVersionDef();
   Status PopulateReturnTensors();
+  Status PopulateReturnNodes();
 
   void Undo();
 
@@ -196,6 +203,9 @@ class GraphConstructor {
   // May be null. Not owned.
   std::vector<std::pair<Node*, int>>* return_tensors_;
 
+  // May be null. Not owned.
+  std::vector<Node*>* return_nodes_;
+
   // May be null. Not owned.
   std::vector<TensorId>* unused_input_map_keys_;
 
@@ -913,7 +923,8 @@ Status GraphConstructor::PopulateReturnTensors() {
       // Locate id in imported nodes
       auto iter = gdef_nodes_.find(id.first);
       if (iter == gdef_nodes_.end()) {
-        return errors::InvalidArgument("Requested return node '", id.first,
+        return errors::InvalidArgument("Requested return tensor '",
+                                       id.ToString(),
                                        "' not found in graph def");
       }
       int num_outputs = iter->second.node->num_outputs();
@@ -935,6 +946,19 @@ Status GraphConstructor::PopulateReturnTensors() {
   return Status::OK();
 }
 
+Status GraphConstructor::PopulateReturnNodes() {
+  if (opts_.return_nodes.empty()) return Status::OK();
+  for (StringPiece name : opts_.return_nodes) {
+    auto iter = gdef_nodes_.find(name);
+    if (iter == gdef_nodes_.end()) {
+      return errors::InvalidArgument("Requested return node '", name,
+                                     "' not found in graph def");
+    }
+    return_nodes_->push_back(iter->second.node);
+  }
+  return Status::OK();
+}
+
 void GraphConstructor::Undo() {
   for (const auto& iter : gdef_nodes_) {
     if (iter.second.node != nullptr) {
@@ -965,7 +989,8 @@ Status ConvertGraphDefToGraph(const GraphConstructorOptions& opts,
   ShapeRefiner refiner(gdef.versions().producer(), g->op_registry());
   return GraphConstructor::Construct(
       opts, gdef.node(), &gdef.versions(), &gdef.library(), g, &refiner,
-      /*return_tensors=*/nullptr, /*unused_input_map_keys=*/nullptr);
+      /*return_tensors=*/nullptr, /*return_nodes=*/nullptr,
+      /*unused_input_map_keys=*/nullptr);
 }
 
 Status ConvertNodeDefsToGraph(const GraphConstructorOptions& opts,
@@ -978,31 +1003,40 @@ Status ConvertNodeDefsToGraph(const GraphConstructorOptions& opts,
   }
   return GraphConstructor::Construct(opts, node_defs, nullptr, nullptr, g,
                                      &refiner, /*return_tensors=*/nullptr,
+                                     /*return_nodes=*/nullptr,
                                      /*unused_input_map_keys=*/nullptr);
 }
 
 Status ImportGraphDef(const ImportGraphDefOptions& opts, const GraphDef& gdef,
                       Graph* g, ShapeRefiner* refiner,
-                      std::vector<std::pair<Node*, int>>* return_tensors,
-                      std::vector<TensorId>* unused_input_map_keys) {
+                      ImportGraphDefResults* results) {
   if (!opts.return_tensors.empty()) {
-    if (return_tensors == nullptr) {
+    if (results == nullptr) {
       return errors::InvalidArgument(
-          "return_tensors argument to ImportGraphDef() must be non-null if "
+          "results argument to ImportGraphDef() must be non-null if "
           "opts.return_tensors is non-empty");
     }
-    if (!return_tensors->empty()) {
+  }
+
+  if (!opts.return_nodes.empty()) {
+    if (opts.skip_mapped_nodes) {
+      return errors::InvalidArgument(
+          "Requesting return_nodes with skip_mapped_nodes set is not currently "
+          "supported");
+    }
+    if (results == nullptr) {
       return errors::InvalidArgument(
-          "return_tensors argument to ImportGraphDef() should be empty (has "
-          "size ",
-          return_tensors->size(), ")");
+          "results argument to ImportGraphDef() must be non-null if "
+          "opts.return_nodes is non-empty");
     }
   }
-  if (unused_input_map_keys != nullptr && !unused_input_map_keys->empty()) {
-    return errors::InvalidArgument(
-        "If non-null, unused_input_map_keys argument to ImportGraphDef() should"
-        " be empty (has size ",
-        unused_input_map_keys->size(), ")");
+
+  if (results != nullptr) {
+    if (!results->return_tensors.empty() || !results->return_nodes.empty() ||
+        !results->unused_input_map_keys.empty()) {
+      return errors::InvalidArgument(
+          "All fields in results argument to ImportGraphDef() must be empty.");
+    }
   }
 
   ShapeRefiner default_refiner(gdef.versions().producer(), g->op_registry());
@@ -1034,9 +1068,10 @@ Status ImportGraphDef(const ImportGraphDefOptions& opts, const GraphDef& gdef,
   refiner->set_graph_def_version(
       std::min(refiner->graph_def_version(), gdef.versions().producer()));
 
-  return GraphConstructor::Construct(opts, gdef.node(), &gdef.versions(),
-                                     &gdef.library(), g, refiner,
-                                     return_tensors, unused_input_map_keys);
+  return GraphConstructor::Construct(
+      opts, gdef.node(), &gdef.versions(), &gdef.library(), g, refiner,
+      &results->return_tensors, &results->return_nodes,
+      &results->unused_input_map_keys);
 }
 
 void CopyGraph(const Graph& src, Graph* dest) {
diff --git a/tensorflow/core/graph/graph_constructor.h b/tensorflow/core/graph/graph_constructor.h
index a8f9f2b245..6cd9347d96 100644
--- a/tensorflow/core/graph/graph_constructor.h
+++ b/tensorflow/core/graph/graph_constructor.h
@@ -72,8 +72,6 @@ struct ImportGraphDefOptions {
   // used to create the existing nodes referenced in `input_map`.
   // TODO(skyewm): can we remove this requirement? How do we access the original
   // shape refiner?
-  //
-  // TODO(skyewm): add functionality to retrieve unused `input_map` keys
   std::map<TensorId, TensorId> input_map;
 
   // If true, nodes that will have all output edges removed because of
@@ -88,10 +86,10 @@ struct ImportGraphDefOptions {
   // other nodes in `gdef`.
   std::vector<string> control_dependencies;
 
-  // Tensors in `gdef` that will be returned via the `return_tensors` output
-  // parameter of `ImportGraphDef()`. If this list is non-empty, the caller must
-  // pass an empty vector to `ImportGraphDef()`. The vector will be populated
-  // with the imported nodes in `g`.
+  // Tensors in `gdef` that will be returned via the ImportGraphDefResults
+  // output parameter of `ImportGraphDef()`. If this list is non-empty, the
+  // caller must pass a results object to `ImportGraphDef()`. The
+  // `return_tensors` field will be populated with the imported nodes in `g`.
   //
   // Entries should not include `prefix`, i.e., each TensorId's name should be
   // the name as it originally appears in `gdef`.
@@ -100,12 +98,43 @@ struct ImportGraphDefOptions {
   // corresponding existing tensor in `g` will be returned.
   std::vector<TensorId> return_tensors;
 
+  // The names of nodes in `gdef` that will be returned via the
+  // ImportGraphDefResults output parameter of `ImportGraphDef()`. If this list
+  // is non-empty, the caller must pass a results object to
+  // `ImportGraphDef()`. The `return_nodes` field will be populated with the
+  // imported nodes in `g`.
+  //
+  // Entries should not include `prefix`, i.e., each node's name should be the
+  // name as it originally appears in `gdef`.
+  //
+  // Unlike `return_tensors`, `input_map` has no effect on the nodes
+  // returned. `return_nodes` must be empty if `skip_mapped_nodes` is true.
+  // TODO(skyewm): make this work with `skip_mapped_nodes` if there's a need.
+  std::vector<StringPiece> return_nodes;
+
   // TODO(ashankar): Enable handling of GraphDefs produced by newer binaries
   // with ops that are not defined in the binary calling ImportGraphDef.
   // Similar to the producer_op_list argument to import_graph_def in the
   // python API.
 };
 
+// Optional results that may be returned by ImportGraphDef.
+struct ImportGraphDefResults {
+  // The requested tensors associated with
+  // ImportGraphDefOptions::return_tensors. Note that the index may be different
+  // than the requested index if the returned tensor has been remapped according
+  // to `input_map`.
+  typedef int Index;
+  std::vector<std::pair<Node*, Index>> return_tensors;
+
+  // The requested nodes associated with ImportGraphDefOptions::return_nodes.
+  std::vector<Node*> return_nodes;
+
+  // Keys in ImportGraphDefOptions::input_map that weren't used as an input to
+  // any node in`gdef`.
+  std::vector<TensorId> unused_input_map_keys;
+};
+
 // Adds the graph in GraphDef `gdef` into an existing Graph `*g`.
 //
 // On error, returns non-OK and leaves `*g` unmodified.
@@ -115,21 +144,16 @@ struct ImportGraphDefOptions {
 // allows the caller to validate shapes of those nodes (since
 // ShapeRefiner::AddNode must be called in topological order).
 //
-// Each `return_tensors` entry is the requested node and output index. The index
-// is included in case the returned tensor has been remapped according to
-// `input_map`.
-//
-// If `unused_input_map_keys` is non-null, it should be empty and will be
-// populated with any keys in `opts.input_map` that aren't used as an input to
-// any node in `gdef`.
+// `results` must be non-null if `opts.return_tensors` or `opts.result_nodes` is
+// non-empty. It can also be set to fetch the unused input map keys. If it's
+// non-null, all the vector fields must be empty.
 //
 // TODO(ashankar): Push this mechanism and get rid of Session::Extend()
 // as a means of enhancing an existing Graph.
-extern Status ImportGraphDef(
-    const ImportGraphDefOptions& opts, const GraphDef& gdef, Graph* g,
-    ShapeRefiner* refiner,
-    std::vector<std::pair<Node*, int>>* return_tensors = nullptr,
-    std::vector<TensorId>* unused_input_map_keys = nullptr);
+extern Status ImportGraphDef(const ImportGraphDefOptions& opts,
+                             const GraphDef& gdef, Graph* g,
+                             ShapeRefiner* refiner,
+                             ImportGraphDefResults* results = nullptr);
 
 // Make a copy of "src" into "*dest".
 //
diff --git a/tensorflow/core/graph/graph_constructor_test.cc b/tensorflow/core/graph/graph_constructor_test.cc
index f88d707ec5..5242c56ce6 100644
--- a/tensorflow/core/graph/graph_constructor_test.cc
+++ b/tensorflow/core/graph/graph_constructor_test.cc
@@ -71,14 +71,12 @@ class GraphConstructorTest : public ::testing::Test {
   void ExpectError(const string& gdef_ascii, const ImportGraphDefOptions& opts,
                    const std::vector<string>& expected_error_strs,
                    ShapeRefiner* refiner = nullptr,
-                   std::vector<std::pair<Node*, int>>* return_tensors = nullptr,
-                   std::vector<TensorId>* unused_input_map_keys = nullptr) {
+                   ImportGraphDefResults* results = nullptr) {
     // Used to verify that errors don't change graph
     const string original_graph_description = GraphDebugString();
 
     Convert(gdef_ascii);
-    Status status = ImportGraphDef(opts, gdef_, &graph_, refiner,
-                                   return_tensors, unused_input_map_keys);
+    Status status = ImportGraphDef(opts, gdef_, &graph_, refiner, results);
     EXPECT_FALSE(status.ok());
 
     for (const string& error : expected_error_strs) {
@@ -97,11 +95,9 @@ class GraphConstructorTest : public ::testing::Test {
 
   void ExpectOK(const string& gdef_ascii, const ImportGraphDefOptions& opts,
                 ShapeRefiner* refiner = nullptr,
-                std::vector<std::pair<Node*, int>>* return_tensors = nullptr,
-                std::vector<TensorId>* unused_input_map_keys = nullptr) {
+                ImportGraphDefResults* results = nullptr) {
     Convert(gdef_ascii);
-    Status s = ImportGraphDef(opts, gdef_, &graph_, refiner, return_tensors,
-                              unused_input_map_keys);
+    Status s = ImportGraphDef(opts, gdef_, &graph_, refiner, results);
     EXPECT_EQ(Status::OK(), s) << s;
   }
 
@@ -1440,26 +1436,25 @@ TEST_F(GraphConstructorTest, ImportGraphDef_InputMapDuplicateNodeNames) {
 TEST_F(GraphConstructorTest, ImportGraphDef_InputMapUnusedKeys) {
   ShapeRefiner refiner(TF_GRAPH_DEF_VERSION, graph_.op_registry());
 
-  std::vector<TensorId> unused_input_map_keys;
-
   // No input map
   ImportGraphDefOptions opts;
+  ImportGraphDefResults results;
   ExpectOK(
       "node { name: 'W1' op: 'TestParams' }"
       "node { name: 'input' op: 'TestInput' }",
-      opts, &refiner, nullptr, &unused_input_map_keys);
-  EXPECT_TRUE(unused_input_map_keys.empty());
+      opts, &refiner, &results);
+  EXPECT_TRUE(results.unused_input_map_keys.empty());
 
   // Non-empty unused_input_map_keys
-  unused_input_map_keys.push_back(TensorId());
-  ExpectError("node { name: 'W2' op: 'TestParams' }", opts,
-              {"If non-null, unused_input_map_keys argument to ImportGraphDef()"
-               " should be empty (has size 1)"},
-              &refiner, nullptr, &unused_input_map_keys);
+  results.unused_input_map_keys.push_back(TensorId());
+  ExpectError(
+      "node { name: 'W2' op: 'TestParams' }", opts,
+      {"All fields in results argument to ImportGraphDef() must be empty."},
+      &refiner, &results);
 
   // Input map with some used, some unused keys
   const int kControlSlot = Graph::kControlSlot;
-  unused_input_map_keys.clear();
+  results.unused_input_map_keys.clear();
   opts.input_map[TensorId("W2", kControlSlot)] = TensorId("W1", kControlSlot);
   opts.input_map[TensorId("new_input", 0)] = TensorId("input", 0);
   opts.input_map[TensorId("new_input", 1)] = TensorId("input", 0);
@@ -1473,11 +1468,11 @@ TEST_F(GraphConstructorTest, ImportGraphDef_InputMapUnusedKeys) {
       node { name: 't1' op: 'TestMul' input: [ 'new_input:0', 'new_input:1' ] }
       node { name: 't2' op: 'TestMul' input: [ 't1:0', 't1:0' ] }
       )EOF",
-      opts, &refiner, nullptr, &unused_input_map_keys);
+      opts, &refiner, &results);
 
   std::vector<TensorId> expected_unused_keys = {
       TensorId("new_input", kControlSlot), TensorId("t1", 1)};
-  EXPECT_EQ(unused_input_map_keys, expected_unused_keys);
+  EXPECT_EQ(results.unused_input_map_keys, expected_unused_keys);
 }
 
 TEST_F(GraphConstructorTest, ImportGraphDef_SkipMappedNodes_FullyMapped) {
@@ -1567,11 +1562,11 @@ TEST_F(GraphConstructorTest, ImportGraphDef_ReturnTensors) {
   opts.return_tensors.push_back({"input", 1});
   opts.return_tensors.push_back({"t1", 0});
   opts.return_tensors.push_back({"input", 0});
-  std::vector<std::pair<Node*, int>> return_tensors;
+  ImportGraphDefResults results;
   ExpectOK(
       "node { name: 'input' op: 'TestInput' }"
       "node { name: 't1' op: 'TestMul' input: ['input:0', 'input:1'] }",
-      opts, &refiner, &return_tensors);
+      opts, &refiner, &results);
 
   // Sanity checks
   EXPECT_TRUE(HasNode("input"));
@@ -1580,74 +1575,70 @@ TEST_F(GraphConstructorTest, ImportGraphDef_ReturnTensors) {
   EXPECT_TRUE(HasEdge("input", 1, "t1", 1));
 
   // Check return tensors
-  ASSERT_EQ(return_tensors.size(), 3);
-  EXPECT_EQ(return_tensors[0].first->name(), "input");
-  EXPECT_EQ(return_tensors[0].second, 1);
-  EXPECT_EQ(return_tensors[1].first->name(), "t1");
-  EXPECT_EQ(return_tensors[1].second, 0);
-  EXPECT_EQ(return_tensors[2].first->name(), "input");
-  EXPECT_EQ(return_tensors[2].second, 0);
+  ASSERT_EQ(results.return_tensors.size(), 3);
+  EXPECT_EQ(results.return_tensors[0].first->name(), "input");
+  EXPECT_EQ(results.return_tensors[0].second, 1);
+  EXPECT_EQ(results.return_tensors[1].first->name(), "t1");
+  EXPECT_EQ(results.return_tensors[1].second, 0);
+  EXPECT_EQ(results.return_tensors[2].first->name(), "input");
+  EXPECT_EQ(results.return_tensors[2].second, 0);
 
   // Test using prefix and returning element from input_map
   opts.return_tensors.clear();
-  return_tensors.clear();
+  results = ImportGraphDefResults();
   opts.prefix = "import";
   opts.input_map[{"new_input", 1}] = {"input", 0};
   opts.return_tensors.push_back({"new_input", 0});
   opts.return_tensors.push_back({"new_input", 1});
   ExpectOK("node { name: 'new_input' op: 'TestInput' }", opts, &refiner,
-           &return_tensors);
+           &results);
 
   EXPECT_TRUE(HasNode("import/new_input"));
 
-  ASSERT_EQ(return_tensors.size(), 2);
-  EXPECT_EQ(return_tensors[0].first->name(), "import/new_input");
-  EXPECT_EQ(return_tensors[0].second, 0);
-  EXPECT_EQ(return_tensors[1].first->name(), "input");
-  EXPECT_EQ(return_tensors[1].second, 0);
+  ASSERT_EQ(results.return_tensors.size(), 2);
+  EXPECT_EQ(results.return_tensors[0].first->name(), "import/new_input");
+  EXPECT_EQ(results.return_tensors[0].second, 0);
+  EXPECT_EQ(results.return_tensors[1].first->name(), "input");
+  EXPECT_EQ(results.return_tensors[1].second, 0);
 
   // Test returning node remapped to source node
   opts.prefix.clear();
   opts.input_map.clear();
   opts.return_tensors.clear();
-  return_tensors.clear();
+  results = ImportGraphDefResults();
   opts.input_map[{"new_input", 0}] = {"_SOURCE", 0};
   opts.return_tensors.push_back({"new_input", 0});
   ExpectOK("node { name: 'new_input' op: 'TestInput' }", opts, &refiner,
-           &return_tensors);
+           &results);
 
   EXPECT_TRUE(HasNode("new_input"));
 
-  ASSERT_EQ(return_tensors.size(), 1);
-  EXPECT_EQ(return_tensors[0].first->name(), "_SOURCE");
-  EXPECT_EQ(return_tensors[0].second, 0);
+  ASSERT_EQ(results.return_tensors.size(), 1);
+  EXPECT_EQ(results.return_tensors[0].first->name(), "_SOURCE");
+  EXPECT_EQ(results.return_tensors[0].second, 0);
 }
 
 TEST_F(GraphConstructorTest, ImportGraphDef_ReturnTensorsErrors) {
-  // Passing in return_tensors with empty opts.return_tensors is OK
+  // Null results with non-empty opts.return_tensors
   ImportGraphDefOptions opts;
-  std::vector<std::pair<Node*, int>> return_tensors;
-  ExpectOK("node { name: 'input' op: 'TestInput' }", opts, nullptr,
-           &return_tensors);
-
-  // Null return_tensors with non-empty opts.return_tensors
   opts.return_tensors.push_back({"new_input", 0});
   ExpectError("node { name: 'new_input' op: 'TestInput' }", opts,
-              {"return_tensors argument to ImportGraphDef() must be non-null "
-               "if opts.return_tensors is non-empty"});
+              {"results argument to ImportGraphDef() must be non-null if "
+               "opts.return_tensors is non-empty"});
 
-  // Non-empty return_tensors
-  return_tensors.push_back({nullptr, 0});
-  ExpectError("node { name: 'new_input' op: 'TestInput' }", opts,
-              {"return_tensors argument to ImportGraphDef() should be empty "
-               "(has size 1)"},
-              nullptr, &return_tensors);
+  // Non-empty results.return_tensors
+  ImportGraphDefResults results;
+  results.return_tensors.push_back({nullptr, 0});
+  ExpectError(
+      "node { name: 'new_input' op: 'TestInput' }", opts,
+      {"All fields in results argument to ImportGraphDef() must be empty."},
+      nullptr, &results);
 
   // Requesting tensor that isn't in graph def
-  return_tensors.clear();
+  results.return_tensors.clear();
   ExpectError("node { name: 'W1' op: 'TestParams' }", opts,
-              {"Requested return node 'new_input' not found in graph def"},
-              nullptr, &return_tensors);
+              {"Requested return tensor 'new_input:0' not found in graph def"},
+              nullptr, &results);
 
   // Requesting invalid node index
   opts.return_tensors.clear();
@@ -1655,7 +1646,89 @@ TEST_F(GraphConstructorTest, ImportGraphDef_ReturnTensorsErrors) {
   ExpectError("node { name: 'new_input' op: 'TestInput' }", opts,
               {"Invalid return output 2 of node 'new_input', which has 2 "
                "output(s)"},
-              nullptr, &return_tensors);
+              nullptr, &results);
+}
+
+TEST_F(GraphConstructorTest, ImportGraphDef_ReturnNodes) {
+  ShapeRefiner refiner(TF_GRAPH_DEF_VERSION, graph_.op_registry());
+
+  ImportGraphDefOptions opts;
+  opts.return_nodes.push_back("input");
+  opts.return_nodes.push_back("t1");
+  ImportGraphDefResults results;
+  ExpectOK(
+      "node { name: 'input' op: 'TestInput' }"
+      "node { name: 'input2' op: 'TestInput' }"
+      "node { name: 't1' op: 'TestMul' input: ['input:0', 'input2:1'] }",
+      opts, &refiner, &results);
+
+  // Sanity checks
+  EXPECT_TRUE(HasNode("input"));
+  EXPECT_TRUE(HasNode("input2"));
+  EXPECT_TRUE(HasNode("t1"));
+  EXPECT_TRUE(HasEdge("input", 0, "t1", 0));
+  EXPECT_TRUE(HasEdge("input2", 1, "t1", 1));
+
+  // Check return tensors
+  ASSERT_EQ(results.return_nodes.size(), 2);
+  EXPECT_EQ(results.return_tensors.size(), 0);
+  EXPECT_EQ(results.unused_input_map_keys.size(), 0);
+  EXPECT_EQ(results.return_nodes[0]->name(), "input");
+  EXPECT_EQ(results.return_nodes[1]->name(), "t1");
+
+  // Test using prefix
+  opts = ImportGraphDefOptions();
+  results = ImportGraphDefResults();
+  opts.prefix = "import";
+  opts.return_nodes.push_back("input");
+  ExpectOK("node { name: 'input' op: 'TestInput' }", opts, &refiner, &results);
+
+  EXPECT_TRUE(HasNode("import/input"));
+
+  ASSERT_EQ(results.return_nodes.size(), 1);
+  EXPECT_EQ(results.return_nodes[0]->name(), "import/input");
+
+  // Test that input_map has no effect
+  opts = ImportGraphDefOptions();
+  results = ImportGraphDefResults();
+  opts.input_map[{"new_input", 0}] = {"input", 0};
+  opts.return_nodes.push_back("new_input");
+  ExpectOK("node { name: 'new_input' op: 'TestInput' }", opts, &refiner,
+           &results);
+
+  EXPECT_TRUE(HasNode("new_input"));
+
+  ASSERT_EQ(results.return_nodes.size(), 1);
+  EXPECT_EQ(results.return_nodes[0]->name(), "new_input");
+}
+
+TEST_F(GraphConstructorTest, ImportGraphDef_ReturnNodesErrors) {
+  // Null results with non-empty opts.return_nodes
+  ImportGraphDefOptions opts;
+  opts.return_nodes.push_back("new_input");
+  ExpectError("node { name: 'new_input' op: 'TestInput' }", opts,
+              {"results argument to ImportGraphDef() must be non-null if "
+               "opts.return_nodes is non-empty"});
+
+  // Non-empty results.return_nodes
+  ImportGraphDefResults results;
+  results.return_nodes.push_back(nullptr);
+  ExpectError(
+      "node { name: 'new_input' op: 'TestInput' }", opts,
+      {"All fields in results argument to ImportGraphDef() must be empty."},
+      nullptr, &results);
+
+  // Requesting node that isn't in graph def
+  results.return_nodes.clear();
+  ExpectError("node { name: 'W1' op: 'TestParams' }", opts,
+              {"Requested return node 'new_input' not found in graph def"},
+              nullptr, &results);
+
+  // Requesting return_nodes with skip_mapped_nodes not yet implemented
+  opts.skip_mapped_nodes = true;
+  ExpectError("node { name: 'new_input' op: 'TestInput' }", opts,
+              {"Requesting return_nodes with skip_mapped_nodes set is not "
+               "currently supported"});
 }
 
 TEST_F(GraphConstructorTest, ImportGraphDef_WithCycle) {
-- 
GitLab


From 01c76110eb3cb1c378c9d7a14ca9f838bad6c7d1 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 16 Oct 2017 15:37:37 -0700
Subject: [PATCH 453/909] Uses head.name in name_scope. This improves the graph
 naming for MultiHead.

PiperOrigin-RevId: 172389494
---
 .../contrib/estimator/python/estimator/head.py       | 10 +++++-----
 tensorflow/python/estimator/canned/head.py           | 12 ++++++------
 2 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/tensorflow/contrib/estimator/python/estimator/head.py b/tensorflow/contrib/estimator/python/estimator/head.py
index e7fe454fbf..f8648fe5bf 100644
--- a/tensorflow/contrib/estimator/python/estimator/head.py
+++ b/tensorflow/contrib/estimator/python/estimator/head.py
@@ -59,7 +59,7 @@ def multi_class_head(n_classes,
       `label_vocabulary`. Also there will be errors if vocabulary is not
       provided and labels are string.
     name: name of the head. If provided, summary and metrics keys will be
-      suffixed by `"/" + name`.
+      suffixed by `"/" + name`. Also used as `name_scope` when creating ops.
 
   Returns:
     An instance of `_Head` for multi class classification.
@@ -98,7 +98,7 @@ def binary_classification_head(
       `label_vocabulary`. Also there will be errors if vocabulary is not
       provided and labels are string.
     name: name of the head. If provided, summary and metrics keys will be
-      suffixed by `"/" + name`.
+      suffixed by `"/" + name`. Also used as `name_scope` when creating ops.
 
   Returns:
     An instance of `_Head` for binary classification.
@@ -129,7 +129,7 @@ def regression_head(weight_column=None,
       of the last dimension of the labels `Tensor` (typically, this has shape
       `[batch_size, label_dimension]`).
     name: name of the head. If provided, summary and metrics keys will be
-      suffixed by `"/" + name`.
+      suffixed by `"/" + name`. Also used as `name_scope` when creating ops.
 
   Returns:
     An instance of `_Head` for linear regression.
@@ -172,7 +172,7 @@ def multi_label_head(n_classes,
       string type and have any value in `label_vocabulary`. Also there will be
       errors if vocabulary is not provided and labels are string.
     name: name of the head. If provided, summary and metrics keys will be
-      suffixed by `"/" + name`.
+      suffixed by `"/" + name`. Also used as `name_scope` when creating ops.
 
   Returns:
     An instance of `_Head` for multi-label classification.
@@ -272,7 +272,7 @@ class _MultiLabelHead(head_lib._Head):  # pylint:disable=protected-access
   def create_estimator_spec(
       self, features, mode, logits, labels=None, train_op_fn=None):
     """See `Head`."""
-    with ops.name_scope('head'):
+    with ops.name_scope(self._name, 'head'):
       logits = head_lib._check_logits(logits, self.logits_dimension)  # pylint:disable=protected-access
 
       # Predict.
diff --git a/tensorflow/python/estimator/canned/head.py b/tensorflow/python/estimator/canned/head.py
index b796a3f954..beafe0d5c4 100644
--- a/tensorflow/python/estimator/canned/head.py
+++ b/tensorflow/python/estimator/canned/head.py
@@ -361,7 +361,7 @@ def _multi_class_head_with_softmax_cross_entropy_loss(n_classes,
       `label_vocabulary`. Also there will be errors if vocabulary is not
       provided and labels are string.
     name: name of the head. If provided, summary and metrics keys will be
-      suffixed by `"/" + name`.
+      suffixed by `"/" + name`. Also used as `name_scope` when creating ops.
 
   Returns:
     An instance of `_Head` for multi class classification.
@@ -453,7 +453,7 @@ class _MultiClassHeadWithSoftmaxCrossEntropyLoss(_Head):
   def create_estimator_spec(
       self, features, mode, logits, labels=None, train_op_fn=None):
     """See `Head`."""
-    with ops.name_scope('head'):
+    with ops.name_scope(self._name, 'head'):
       logits = _check_logits(logits, self.logits_dimension)
 
       # Predict.
@@ -562,7 +562,7 @@ def _binary_logistic_head_with_sigmoid_cross_entropy_loss(
       `label_vocabulary`. Also there will be errors if vocabulary is not
       provided and labels are string.
     name: name of the head. If provided, summary and metrics keys will be
-      suffixed by `"/" + name`.
+      suffixed by `"/" + name`. Also used as `name_scope` when creating ops.
 
   Returns:
     An instance of `Head` for binary classification.
@@ -702,7 +702,7 @@ class _BinaryLogisticHeadWithSigmoidCrossEntropyLoss(_Head):
       self, features, mode, logits, labels=None, train_op_fn=None):
     """See `Head`."""
     # Predict.
-    with ops.name_scope('head'):
+    with ops.name_scope(self._name, 'head'):
       with ops.name_scope(None, 'predictions', (logits,)):
         pred_keys = prediction_keys.PredictionKeys
         logits = _check_logits(logits, self.logits_dimension)
@@ -802,7 +802,7 @@ def _regression_head_with_mean_squared_error_loss(weight_column=None,
       of the last dimension of the labels `Tensor` (typically, this has shape
       `[batch_size, label_dimension]`).
     name: name of the head. If provided, summary and metrics keys will be
-      suffixed by `"/" + name`.
+      suffixed by `"/" + name`. Also used as `name_scope` when creating ops.
 
   Returns:
     An instance of `_Head` for linear regression.
@@ -846,7 +846,7 @@ class _RegressionHeadWithMeanSquaredErrorLoss(_Head):
       self, features, mode, logits, labels=None, train_op_fn=None):
     """See `Head`."""
     # Predict.
-    with ops.name_scope('head'):
+    with ops.name_scope(self._name, 'head'):
       logits = _check_logits(logits, self._logits_dimension)
       predictions = {prediction_keys.PredictionKeys.PREDICTIONS: logits}
       if mode == model_fn.ModeKeys.PREDICT:
-- 
GitLab


From 51e5b692a5f8f6942cb43291ba9faab39e4b6104 Mon Sep 17 00:00:00 2001
From: Michael Case <mikecase@chromium.org>
Date: Mon, 16 Oct 2017 15:46:36 -0700
Subject: [PATCH 454/909] Fix ambiguous type comparison in s3_crypto.cc
 (#13758)

tensorflow/contrib/s3/s3_crypto.cc(74): error C2666:
'std::fpos<_Mbstatet>::operator ==': 3 overloads have similar conversions
could be 'bool std::fpos<_Mbstatet>::operator ==(std::streamoff) const'
or 'bool std::fpos<_Mbstatet>::operator ==(const std::fpos<_Mbstatet> &)
We were seeing this compilation error on Windows builds.
---
 tensorflow/contrib/s3/s3_crypto.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/s3/s3_crypto.cc b/tensorflow/contrib/s3/s3_crypto.cc
index 1450384dc0..bbd66371e4 100644
--- a/tensorflow/contrib/s3/s3_crypto.cc
+++ b/tensorflow/contrib/s3/s3_crypto.cc
@@ -71,7 +71,7 @@ class S3Sha256OpenSSLImpl : public Aws::Utils::Crypto::Hash {
     SHA256_Init(&sha256);
 
     auto currentPos = stream.tellg();
-    if (currentPos == -1) {
+    if (currentPos == std::streampos(std::streamoff(-1))) {
       currentPos = 0;
       stream.clear();
     }
-- 
GitLab


From a072aa0d2c8412160748995bdea0cc15f121fd95 Mon Sep 17 00:00:00 2001
From: Vijay Vasudevan <vrv@google.com>
Date: Mon, 16 Oct 2017 15:50:36 -0700
Subject: [PATCH 455/909] Revert "Fix broken link in debugger doc" (#13760)

* Revert "Fix broken link in debugger doc (#13757)"

This reverts commit 4e1d0f4e32b7f7a463930543dc773997bdb9d545.
---
 tensorflow/docs_src/programmers_guide/debugger.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/docs_src/programmers_guide/debugger.md b/tensorflow/docs_src/programmers_guide/debugger.md
index 3f9f155457..3ede42e8f7 100644
--- a/tensorflow/docs_src/programmers_guide/debugger.md
+++ b/tensorflow/docs_src/programmers_guide/debugger.md
@@ -440,7 +440,7 @@ accuracy_score = classifier.evaluate(x=test_set.data,
 
 
 [debug_tflearn_iris.py](https://www.tensorflow.org/code/tensorflow/python/debug/examples/debug_tflearn_iris.py),
-based on @{$tflearn$tf-learn's iris tutorial}, contains a full example of how to
+based on {$tflearn$tf-learn's iris tutorial}, contains a full example of how to
 use the tfdbg with `Estimator`s. To run this example, do:
 
 ```none
-- 
GitLab


From e3b8d3cc2a0099fecdc103f8422b34eda1eaee1f Mon Sep 17 00:00:00 2001
From: michelleirvine <michelleirvine@google.com>
Date: Mon, 16 Oct 2017 16:00:55 -0700
Subject: [PATCH 456/909] Update README.md (#13688)

* Update README.md

Update information about local builds and TensorFlow's CI system.

* Update README.md

* Update README.md

* Update README.md
---
 tensorflow/tools/ci_build/README.md | 143 ++++++++++------------------
 1 file changed, 52 insertions(+), 91 deletions(-)

diff --git a/tensorflow/tools/ci_build/README.md b/tensorflow/tools/ci_build/README.md
index ad83669950..acef833909 100644
--- a/tensorflow/tools/ci_build/README.md
+++ b/tensorflow/tools/ci_build/README.md
@@ -1,115 +1,76 @@
 # TensorFlow Builds
 
-This directory contains all the files and setup instructions to run all
-the important builds and tests. **You can trivially run it yourself!** It also
-run continuous integration [ci.tensorflow.org](https://ci.tensorflow.org).
-
-
+This directory contains all the files and setup instructions to run all the
+important builds and tests. You can run it yourself!
 
 ## Run It Yourself
 
-1. Install [Docker](http://www.docker.com/). Follow instructions
-   [on the Docker site](https://docs.docker.com/installation/).
-
-   You can run all the jobs **without docker** if you are on mac or on linux
-   and you just don't want docker. Just install all the dependencies from
-   [Installing TensorFlow](https://www.tensorflow.org/install/).
-   Then run any of the one liners below without the
-   `tensorflow/tools/ci_build/ci_build.sh` in them.
-
-2. Clone tensorflow repository.
-
-   ```bash
-   git clone https://github.com/tensorflow/tensorflow.git
-   ```
-
-3. Go to tensorflow directory
-
-   ```bash
-   cd tensorflow
-   ```
-
-4. Build what you want, for example
-
-   ```bash
-   tensorflow/tools/ci_build/ci_build.sh CPU bazel test //tensorflow/...
-   ```
-   If you are using the Docker image on Windows or OS X, the Docker VM's default
-   memory limit may be too low to build TensorFlow. This can result in
-   strange-looking errors, e.g. the compilation may fail with `gcc: internal
-   compiler error: Killed (program cc1plus)`. Try increasing the memory limit in
-   the Docker preferences.
-
-
-## Jobs
-
-The jobs run by [ci.tensorflow.org](https://ci.tensorflow.org) include following:
-
-```bash
-# Note: You can run the following one-liners yourself if you have Docker. Run
-# without `tensorflow/tools/ci_build/ci_build.sh` on mac or linux without Docker.
-
-# build and run cpu tests
-tensorflow/tools/ci_build/ci_build.sh CPU bazel test //tensorflow/...
+You have two options when running TensorFlow tests locally on your
+machine. First, using docker, you can run our Continuous Integration
+(CI) scripts on tensorflow devel images. The other option is to install
+all TensorFlow dependencies on your machine and run the scripts
+natively on your system.
 
-# build and run gpu tests (note if you get unstable results you may be running
-# out of gpu memory - if so add "--jobs=1" argument)
-tensorflow/tools/ci_build/ci_build.sh GPU bazel test -c opt --config=cuda //tensorflow/...
+### Run TensorFlow CI Scripts using Docker
 
-# build pip with gpu support
-tensorflow/tools/ci_build/ci_build.sh GPU tensorflow/tools/ci_build/builds/pip.sh GPU -c opt --config=cuda
+1.  Install Docker following the [instructions on the docker website](https://docs.docker.com/engine/installation/).
 
-# build and run gpu tests using python 3
-CI_DOCKER_EXTRA_PARAMS="-e CI_BUILD_PYTHON=python3" tensorflow/tools/ci_build/ci_build.sh GPU tensorflow/tools/ci_build/builds/pip.sh GPU -c opt --config=cuda
+2.  Start a container with one of the devel images here:
+    https://hub.docker.com/r/tensorflow/tensorflow/tags/.
 
-# build android example app
-tensorflow/tools/ci_build/ci_build.sh ANDROID tensorflow/tools/ci_build/builds/android.sh
+3.  Based on your choice of the image, pick one of the scripts under
+    https://github.com/tensorflow/tensorflow/tree/master/tensorflow/tools/ci_build/linux
+    and run them from the TensorFlow repository root.
 
-# cmake cpu build and test
-tensorflow/tools/ci_build/ci_build.sh CPU tensorflow/tools/ci_build/builds/cmake.sh
+### Run TensorFlow CI Scripts Natively on your Machine
 
-# run bash inside the container
-CI_DOCKER_EXTRA_PARAMS='-it --rm' tensorflow/tools/ci_build/ci_build.sh CPU /bin/bash
-```
+1.  Follow the instructions at https://www.tensorflow.org/install/install_sources,
+    but stop when you get to the section "Configure the installation". You do not
+    need to configure the installation to run the CI scripts.
 
-**Note**: The set of jobs and how they are triggered is still evolving.
-There are builds for master branch on cpu, gpu and android. There is a build
-for incoming gerrit changes. Gpu tests and benchmark are coming soon. Check
-[ci.tensorflow.org](https://ci.tensorflow.org) for current jobs.
+2.  Pick the appropriate OS and python version you have installed,
+    and run the script under tensorflow/tools/ci_build/<OS>.
 
+## TensorFlow Continuous Integration
 
+To verify that new changes don’t break TensorFlow, we run builds and
+tests on either [Jenkins](https://jenkins-ci.org/) or a CI system
+internal to Google.
 
-## How Does TensorFlow Continuous Integration Work
+We can trigger builds and tests on updates to master or on each pull
+request. Contact one of the repository maintainers to trigger builds
+on your pull request.
 
-We use [jenkins](https://jenkins-ci.org/) as our continuous integration.
-It is running at [ci.tensorflow.org](https://ci.tensorflow.org).
-All the jobs are run within [docker](http://www.docker.com/) containers.
+### View CI Results
 
-Builds can be triggered by push to master, push a change set or manually.
-The build started in jenkins will first pull the git tree. Then jenkins builds
-a docker container (using one of those Dockerfile.* files in this directory).
-The build itself is run within the container itself.
+The Pull Request will show if the change passed or failed the checks.
 
-Source tree lives in jenkins job workspace. Docker container for jenkins
-are transient - deleted after the build. Containers build very fast thanks
-to docker caching. Individual builds are fast thanks to bazel caching.
+From the pull request, click **Show all checks** to see the list of builds
+and tests. Click on **Details** to see the results from Jenkins or the internal
+CI system.
 
+Results from Jenkins are displayed in the Jenkins UI. For more information,
+see the [Jenkns documentation](https://jenkins.io/doc/).
 
+Results from the internal CI system are displayed in the Build Status UI. In
+this UI, to see the logs for a failed build:
 
-## Implementation Details
+*   Click on the **INVOCATION LOG** tab to see the invocation log.
 
-* The ci_build.sh script create and run docker container with all dependencies.
-  The builds/with_the_same_user together with ci_build.sh creates an environment
-  which is the same inside the container as it is outside. The same user, group,
-  path, so that docker symlinks work inside and outside the container. You can
-  use it for your development. Edit files in your git clone directory. If you
-  run the ci_build.sh it gets this directory mapped inside the container and
-  build your tree.
+*   Click on the **ARTIFACTS** tab to see a list of all artifacts, including logs.
 
-* The unusual `bazel-ci_build-cache` directory is mapped to docker container
-  performing the build using docker's --volume parameter. This way we cache
-  bazel output between builds.
+*   Individual test logs may be available. To see these logs, from the **TARGETS**
+    tab, click on the failed target. Then, click on the **TARGET LOG** tab to see
+    its test log.
 
-* The `builds` directory within this folder contains shell scripts to run within
-  the container. They essentially contains workarounds for current limitations
-  of bazel.
+    If you’re looking at target that is sharded or a test that is flaky, then
+    the build tool divided the target into multiple shards or ran the test
+    multiple times. Each test log is specific to the shard, run, and attempt.
+    To see a specific log:
+    
+    1.  Click on the log icon that is on the right next to the shard, run,
+        and attempt number.
+        
+    2.  In the grid that appears on the right, click on the specific shard,
+        run, and attempt to view its log. You can also type the desired shard,
+        run, or attempt number in the field above its grid.
-- 
GitLab


From 99dffc958a1cfa4e5a2f81e8f4085277a0c34bd9 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 16 Oct 2017 16:34:35 -0700
Subject: [PATCH 457/909] Better error message for eager-specific APIs

PiperOrigin-RevId: 172397124
---
 tensorflow/contrib/eager/python/saver.py | 10 ++++++++++
 tensorflow/python/eager/backprop.py      |  4 ++++
 tensorflow/python/eager/function_test.py | 11 +++++++++++
 3 files changed, 25 insertions(+)

diff --git a/tensorflow/contrib/eager/python/saver.py b/tensorflow/contrib/eager/python/saver.py
index d289b83f53..2bf11d3f20 100644
--- a/tensorflow/contrib/eager/python/saver.py
+++ b/tensorflow/contrib/eager/python/saver.py
@@ -19,6 +19,7 @@ from __future__ import print_function
 
 import contextlib
 
+from tensorflow.python.eager import context
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import resource_variable_ops
@@ -64,7 +65,12 @@ def restore_variables_on_create(save_path):
 
   Raises:
     NotFoundError: If the variable is not found in checkpoint.
+    ValueError: If not used in eager mode.
   """
+  if context.in_graph_mode():
+    raise ValueError(
+        "Currently, restore_variables_on_create can only be used with "
+        "eager execution enabled.")
   if save_path:
     ckpt_var_cache = dict()
     reader = checkpoint_utils.load_checkpoint(save_path)
@@ -102,6 +108,10 @@ class Saver(object):
   """
 
   def __init__(self, var_list):
+    if context.in_graph_mode():
+      raise ValueError("Currently, tfe.Saver can only be used when eager "
+                       "execution is enabled. Use tf.train.Saver when "
+                       "building graphs.")
     self._saver = _saver.Saver(var_list=var_list)
 
   def save(self, save_path, global_step=None):
diff --git a/tensorflow/python/eager/backprop.py b/tensorflow/python/eager/backprop.py
index 7f1a770513..1819fba4cb 100644
--- a/tensorflow/python/eager/backprop.py
+++ b/tensorflow/python/eager/backprop.py
@@ -337,6 +337,10 @@ def implicit_val_and_grad(f):
     end_node = f(*args)
     variables = tape.top_tape_watched_variables()
     sources = [x.handle for x in variables]
+
+    if not sources:
+      raise ValueError("no trainable variables were accessed while the "
+                       "function was being computed.")
     grad = imperative_grad.imperative_grad(_default_vspace,
                                            tape.pop_tape(),
                                            nest.flatten(end_node),
diff --git a/tensorflow/python/eager/function_test.py b/tensorflow/python/eager/function_test.py
index 899b6d59b7..e27f9ebc27 100644
--- a/tensorflow/python/eager/function_test.py
+++ b/tensorflow/python/eager/function_test.py
@@ -31,6 +31,7 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import clip_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import resource_variable_ops
+from tensorflow.python.ops import variable_scope
 
 
 class FunctionTest(test.TestCase):
@@ -99,6 +100,16 @@ class FunctionTest(test.TestCase):
 
     self.assertAllEqual(2, g(constant_op.constant(2)).numpy())
 
+  def testGraphModeEagerGradError(self):
+    with context.graph_mode():
+      def f():
+        x = variable_scope.get_variable(
+            'v', initializer=constant_op.constant(1.0))
+        return x * constant_op.constant(2.0)
+      with self.assertRaisesRegexp(ValueError,
+                                   'no trainable variables were accessed'):
+        backprop.implicit_val_and_grad(f)()
+
   def testDefunCallBackpropUsingSameObjectForMultipleArguments(self):
 
     @function.defun
-- 
GitLab


From 528457ea3cbe4edfbd3eb90c303b2a1408fe8d65 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Mon, 16 Oct 2017 17:19:26 -0700
Subject: [PATCH 458/909] Add GPU and CPU implementation of
 `tf.histogram_fixed_width`. (#13731)

* Add GPU and CPU implementation of `tf.histogram_fixed_width`.

This fix adds the GPU and CPU implementation of `tf.histogram_fixed_width`.
The previous implementation was done in python. This fix adds
C++ kernel for GPU and CPU>

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Update python ops for `tf.histogram_fixed_width`

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Update test cases to invoke GPU test for `tf.histogram_fixed_width`

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Disable int64 output on GPU for now as atomicAdd is not supported yet.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Address review feedback and use a stable version of summation.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Maintain backward compatibility of the API

Keep `dtype` and make sure `nbins = 100` is in attr

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/core/BUILD                         |   1 +
 tensorflow/core/kernels/BUILD                 |  12 ++
 tensorflow/core/kernels/histogram_op.cc       | 147 ++++++++++++++++++
 tensorflow/core/kernels/histogram_op.h        |  38 +++++
 .../core/kernels/histogram_op_gpu.cu.cc       | 125 +++++++++++++++
 tensorflow/core/ops/math_ops.cc               |  38 +++++
 tensorflow/python/ops/histogram_ops.py        |  31 +---
 tensorflow/python/ops/histogram_ops_test.py   |   8 +-
 8 files changed, 369 insertions(+), 31 deletions(-)
 create mode 100644 tensorflow/core/kernels/histogram_op.cc
 create mode 100644 tensorflow/core/kernels/histogram_op.h
 create mode 100644 tensorflow/core/kernels/histogram_op_gpu.cu.cc

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index f60c0d76cf..a0c8fae69a 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -781,6 +781,7 @@ cc_library(
         "//tensorflow/core/kernels:dataset_ops",
         "//tensorflow/core/kernels:fake_quant_ops",
         "//tensorflow/core/kernels:function_ops",
+        "//tensorflow/core/kernels:histogram_op",
         "//tensorflow/core/kernels:image",
         "//tensorflow/core/kernels:io",
         "//tensorflow/core/kernels:linalg",
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 2c02571346..ca5356b6e7 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -2499,6 +2499,7 @@ cc_library(
         ":cross_op",
         ":cwise_op",
         ":fft_ops",
+        ":histogram_op",
         ":matmul_op",
         ":population_count_op",
         ":reduction_ops",
@@ -3096,6 +3097,17 @@ tf_kernel_library(
     ],
 )
 
+tf_kernel_library(
+    name = "histogram_op",
+    prefix = "histogram_op",
+    deps = [
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:lib_internal",
+        "//third_party/eigen3",
+    ] + if_cuda(["@cub_archive//:cub"]),
+)
+
 tf_kernel_library(
     name = "l2loss_op",
     prefix = "l2loss_op",
diff --git a/tensorflow/core/kernels/histogram_op.cc b/tensorflow/core/kernels/histogram_op.cc
new file mode 100644
index 0000000000..c170f172e4
--- /dev/null
+++ b/tensorflow/core/kernels/histogram_op.cc
@@ -0,0 +1,147 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+// See docs in ../ops/math_ops.cc.
+
+#define EIGEN_USE_THREADS
+
+#include "tensorflow/core/kernels/histogram_op.h"
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/framework/register_types.h"
+#include "tensorflow/core/framework/types.h"
+#include "tensorflow/core/lib/core/threadpool.h"
+#include "tensorflow/core/platform/types.h"
+
+namespace tensorflow {
+
+typedef Eigen::ThreadPoolDevice CPUDevice;
+typedef Eigen::GpuDevice GPUDevice;
+
+namespace functor {
+
+template <typename T, typename Tout>
+struct HistogramFixedWidthFunctor<CPUDevice, T, Tout> {
+  static Status Compute(OpKernelContext* context,
+                        const typename TTypes<T, 1>::ConstTensor& values,
+                        const typename TTypes<T, 1>::ConstTensor& value_range,
+                        int32 nbins, typename TTypes<Tout, 1>::Tensor& out) {
+    const CPUDevice& d = context->eigen_device<CPUDevice>();
+
+    Tensor index_to_bin_tensor;
+
+    TF_RETURN_IF_ERROR(context->forward_input_or_allocate_temp(
+        {0}, DataTypeToEnum<int32>::value, TensorShape({values.size()}),
+        &index_to_bin_tensor));
+    auto index_to_bin = index_to_bin_tensor.flat<int32>();
+
+    const double step = static_cast<double>(value_range(1) - value_range(0)) /
+                        static_cast<double>(nbins);
+
+    // The calculation is done by finding the slot of each value in `values`.
+    // With [a, b]:
+    //   step = (b - a) / nbins
+    //   (x - a) / step
+    // , then the entries are mapped to output.
+    index_to_bin.device(d) =
+        ((values.cwiseMax(value_range(0)) - values.constant(value_range(0)))
+             .template cast<double>() /
+         step)
+            .template cast<int32>()
+            .cwiseMin(nbins - 1);
+
+    out.setZero();
+    for (int32 i = 0; i < index_to_bin.size(); i++) {
+      out(index_to_bin(i)) += Tout(1);
+    }
+    return Status::OK();
+  }
+};
+
+}  // namespace functor
+
+template <typename Device, typename T, typename Tout>
+class HistogramFixedWidthOp : public OpKernel {
+ public:
+  explicit HistogramFixedWidthOp(OpKernelConstruction* ctx) : OpKernel(ctx) {
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("nbins", &nbins_));
+    OP_REQUIRES(
+        ctx, (nbins_ > 0),
+        errors::InvalidArgument("nbins should be a positive number, but got '",
+                                nbins_, "'"));
+  }
+
+  void Compute(OpKernelContext* ctx) override {
+    const Tensor& values_tensor = ctx->input(0);
+    const Tensor& value_range_tensor = ctx->input(1);
+
+    OP_REQUIRES(ctx, TensorShapeUtils::IsVector(value_range_tensor.shape()),
+                errors::InvalidArgument("value_range should be a vector."));
+    OP_REQUIRES(ctx, (value_range_tensor.shape().num_elements() == 2),
+                errors::InvalidArgument(
+                    "value_range should be a vector of 2 elements."));
+
+    const auto values = values_tensor.flat<T>();
+    const auto value_range = value_range_tensor.flat<T>();
+
+    OP_REQUIRES(
+        ctx, (value_range(0) < value_range(1)),
+        errors::InvalidArgument("value_range should satisfy value_range[0] < "
+                                "value_range[1], but got '[",
+                                value_range(0), ", ", value_range(1), "]'"));
+
+    Tensor* out_tensor;
+    OP_REQUIRES_OK(ctx,
+                   ctx->allocate_output(0, TensorShape({nbins_}), &out_tensor));
+    auto out = out_tensor->flat<Tout>();
+
+    OP_REQUIRES_OK(
+        ctx, functor::HistogramFixedWidthFunctor<Device, T, Tout>::Compute(
+                 ctx, values, value_range, nbins_, out));
+  }
+
+ private:
+  int nbins_;
+};
+
+#define REGISTER_KERNELS(type)                                           \
+  REGISTER_KERNEL_BUILDER(Name("HistogramFixedWidth")                    \
+                              .Device(DEVICE_CPU)                        \
+                              .TypeConstraint<type>("T")                 \
+                              .TypeConstraint<int32>("dtype"),           \
+                          HistogramFixedWidthOp<CPUDevice, type, int32>) \
+  REGISTER_KERNEL_BUILDER(Name("HistogramFixedWidth")                    \
+                              .Device(DEVICE_CPU)                        \
+                              .TypeConstraint<type>("T")                 \
+                              .TypeConstraint<int64>("dtype"),           \
+                          HistogramFixedWidthOp<CPUDevice, type, int64>)
+
+TF_CALL_REAL_NUMBER_TYPES(REGISTER_KERNELS);
+#undef REGISTER_KERNELS
+
+#if GOOGLE_CUDA
+#define REGISTER_KERNELS(type)                                 \
+  REGISTER_KERNEL_BUILDER(Name("HistogramFixedWidth")          \
+                              .Device(DEVICE_GPU)              \
+                              .HostMemory("value_range")       \
+                              .TypeConstraint<type>("T")       \
+                              .TypeConstraint<int32>("dtype"), \
+                          HistogramFixedWidthOp<GPUDevice, type, int32>)
+
+TF_CALL_GPU_NUMBER_TYPES(REGISTER_KERNELS);
+#undef REGISTER_KERNELS
+
+#endif  // GOOGLE_CUDA
+
+}  // end namespace tensorflow
diff --git a/tensorflow/core/kernels/histogram_op.h b/tensorflow/core/kernels/histogram_op.h
new file mode 100644
index 0000000000..1b253f7fed
--- /dev/null
+++ b/tensorflow/core/kernels/histogram_op.h
@@ -0,0 +1,38 @@
+/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_HISTOGRAM_OP_H_
+#define TENSORFLOW_HISTOGRAM_OP_H_
+
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/framework/tensor_types.h"
+#include "tensorflow/core/framework/types.h"
+#include "tensorflow/core/lib/core/errors.h"
+
+namespace tensorflow {
+namespace functor {
+
+template <typename Device, typename T, typename Tout>
+struct HistogramFixedWidthFunctor {
+  static Status Compute(OpKernelContext* context,
+                        const typename TTypes<T, 1>::ConstTensor& values,
+                        const typename TTypes<T, 1>::ConstTensor& value_range,
+                        int32 nbins, typename TTypes<Tout, 1>::Tensor& out);
+};
+
+}  // end namespace functor
+}  // end namespace tensorflow
+
+#endif  // TENSORFLOW_HISTOGRAM_OP_H_
diff --git a/tensorflow/core/kernels/histogram_op_gpu.cu.cc b/tensorflow/core/kernels/histogram_op_gpu.cu.cc
new file mode 100644
index 0000000000..c2bb958be8
--- /dev/null
+++ b/tensorflow/core/kernels/histogram_op_gpu.cu.cc
@@ -0,0 +1,125 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+
+#define EIGEN_USE_GPU
+
+#include "tensorflow/core/kernels/histogram_op.h"
+#include "external/cub_archive/cub/device/device_histogram.cuh"
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/framework/register_types.h"
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/framework/tensor_shape.h"
+#include "tensorflow/core/platform/logging.h"
+#include "tensorflow/core/platform/types.h"
+#include "tensorflow/core/util/cuda_kernel_helper.h"
+#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+
+namespace tensorflow {
+
+typedef Eigen::GpuDevice GPUDevice;
+
+namespace functor {
+
+// TODO(yongtang) int64 of atomicAdd is not supported yet.
+template <typename T, typename Tout>
+struct HistogramFixedWidthFunctor<GPUDevice, T, Tout> {
+  static Status Compute(OpKernelContext* context,
+                        const typename TTypes<T, 1>::ConstTensor& values,
+                        const typename TTypes<T, 1>::ConstTensor& value_range,
+                        int32 nbins, typename TTypes<Tout, 1>::Tensor& out) {
+    tensorflow::AllocatorAttributes pinned_allocator;
+    pinned_allocator.set_on_host(true);
+    pinned_allocator.set_gpu_compatible(true);
+
+    Tensor levels_tensor;
+    TF_RETURN_IF_ERROR(context->allocate_temp(
+        DataTypeToEnum<T>::value, TensorShape({nbins + 1}), &levels_tensor,
+        pinned_allocator));
+    auto levels = levels_tensor.flat<T>();
+
+    const double step = static_cast<double>(value_range(1) - value_range(0)) /
+                        static_cast<double>(nbins);
+    levels(0) = std::numeric_limits<T>::lowest();
+    for (int i = 1; i < nbins; i++) {
+      levels(i) =
+          static_cast<T>(static_cast<double>(value_range(0)) + step * i);
+    }
+    levels(nbins) = std::numeric_limits<T>::max();
+
+    size_t temp_storage_bytes = 0;
+    const T* d_samples = values.data();
+    Tout* d_histogram = out.data();
+    int num_levels = levels.size();
+    T* d_levels = levels.data();
+    int num_samples = values.size();
+    const cudaStream_t& stream = GetCudaStream(context);
+
+    // The first HistogramRange is to obtain the temp storage size required
+    // with d_temp_storage = NULL passed to the call.
+    auto err = cub::DeviceHistogram::HistogramRange(
+        /* d_temp_storage */ NULL,
+        /* temp_storage_bytes */ temp_storage_bytes,
+        /* d_samples */ d_samples,
+        /* d_histogram */ d_histogram,
+        /* num_levels */ num_levels,
+        /* d_levels */ d_levels,
+        /* num_samples */ num_samples,
+        /* stream */ stream);
+    if (err != cudaSuccess) {
+      return errors::Internal(
+          "Could not launch HistogramRange to get temp storage: ",
+          cudaGetErrorString(err), ".");
+    }
+
+    Tensor temp_storage;
+    TF_RETURN_IF_ERROR(context->allocate_temp(
+        DataTypeToEnum<int8>::value,
+        TensorShape({static_cast<int64>(temp_storage_bytes)}), &temp_storage));
+
+    void* d_temp_storage = temp_storage.flat<int8>().data();
+
+    // The second HistogramRange is to actual run with d_temp_storage
+    // allocated with temp_storage_bytes.
+    err = cub::DeviceHistogram::HistogramRange(
+        /* d_temp_storage */ d_temp_storage,
+        /* temp_storage_bytes */ temp_storage_bytes,
+        /* d_samples */ d_samples,
+        /* d_histogram */ d_histogram,
+        /* num_levels */ num_levels,
+        /* d_levels */ d_levels,
+        /* num_samples */ num_samples,
+        /* stream */ stream);
+    if (err != cudaSuccess) {
+      return errors::Internal("Could not launch HistogramRange: ",
+                              cudaGetErrorString(err), ".");
+    }
+
+    return Status::OK();
+  }
+};
+
+}  // end namespace functor
+
+#define REGISTER_GPU_SPEC(type) \
+  template struct functor::HistogramFixedWidthFunctor<GPUDevice, type, int32>;
+
+TF_CALL_GPU_NUMBER_TYPES(REGISTER_GPU_SPEC);
+#undef REGISTER_GPU_SPEC
+
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/ops/math_ops.cc b/tensorflow/core/ops/math_ops.cc
index 7b971a9fd5..a1c608ee54 100644
--- a/tensorflow/core/ops/math_ops.cc
+++ b/tensorflow/core/ops/math_ops.cc
@@ -2250,6 +2250,44 @@ product: Pairwise cross product of the vectors in `a` and `b`.
 
 // --------------------------------------------------------------------------
 
+REGISTER_OP("HistogramFixedWidth")
+    .Input("values: T")
+    .Input("value_range: T")
+    .Output("out: dtype")
+    .Attr("nbins: int = 100")
+    .Attr("T: {int32, int64, float32, float64}")
+    .Attr("dtype: {int32, int64} = DT_INT32")
+    .SetShapeFn([](InferenceContext* c) {
+      c->set_output(0, c->UnknownShapeOfRank(1));
+      return Status::OK();
+    })
+    .Doc(R"doc(
+Return histogram of values.
+
+Given the tensor `values`, this operation returns a rank 1 histogram counting
+the number of entries in `values` that fall into every bin.  The bins are
+equal width and determined by the arguments `value_range` and `nbins`.
+
+```python
+# Bins will be:  (-inf, 1), [1, 2), [2, 3), [3, 4), [4, inf)
+nbins = 5
+value_range = [0.0, 5.0]
+new_values = [-1.0, 0.0, 1.5, 2.0, 5.0, 15]
+
+with tf.get_default_session() as sess:
+  hist = tf.histogram_fixed_width(new_values, value_range, nbins=5)
+  variables.global_variables_initializer().run()
+  sess.run(hist) => [2, 1, 1, 0, 2]
+```
+
+values:  Numeric `Tensor`.
+value_range:  Shape [2] `Tensor` of same `dtype` as `values`.
+  values <= value_range[0] will be mapped to hist[0],
+  values >= value_range[1] will be mapped to hist[-1].
+nbins:  Scalar `int32 Tensor`.  Number of histogram bins.
+out: A 1-D `Tensor` holding histogram of values.
+)doc");
+
 REGISTER_OP("Bincount")
     .Input("arr: int32")
     .Input("size: int32")
diff --git a/tensorflow/python/ops/histogram_ops.py b/tensorflow/python/ops/histogram_ops.py
index c2077d51af..040c3a5ae8 100644
--- a/tensorflow/python/ops/histogram_ops.py
+++ b/tensorflow/python/ops/histogram_ops.py
@@ -28,6 +28,7 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import clip_ops
+from tensorflow.python.ops import gen_math_ops
 from tensorflow.python.ops import math_ops
 
 
@@ -69,30 +70,6 @@ def histogram_fixed_width(values,
   ```
   """
   with ops.name_scope(name, 'histogram_fixed_width',
-                      [values, value_range, nbins]) as scope:
-    values = ops.convert_to_tensor(values, name='values')
-    values = array_ops.reshape(values, [-1])
-    value_range = ops.convert_to_tensor(value_range, name='value_range')
-    nbins = ops.convert_to_tensor(nbins, dtype=dtypes.int32, name='nbins')
-    nbins_float = math_ops.cast(nbins, values.dtype)
-
-    # Map tensor values that fall within value_range to [0, 1].
-    scaled_values = math_ops.truediv(values - value_range[0],
-                                     value_range[1] - value_range[0],
-                                     name='scaled_values')
-
-    # map tensor values within the open interval value_range to {0,.., nbins-1},
-    # values outside the open interval will be zero or less, or nbins or more.
-    indices = math_ops.floor(nbins_float * scaled_values, name='indices')
-
-    # Clip edge cases (e.g. value = value_range[1]) or "outliers."
-    indices = math_ops.cast(
-        clip_ops.clip_by_value(indices, 0, nbins_float - 1), dtypes.int32)
-
-    # TODO(langmore) This creates an array of ones to add up and place in the
-    # bins.  This is inefficient, so replace when a better Op is available.
-    return math_ops.unsorted_segment_sum(
-        array_ops.ones_like(indices, dtype=dtype),
-        indices,
-        nbins,
-        name=scope)
+                      [values, value_range, nbins]) as name:
+    return gen_math_ops.histogram_fixed_width(values, value_range, nbins,
+                                              dtype=dtype, name=name)
diff --git a/tensorflow/python/ops/histogram_ops_test.py b/tensorflow/python/ops/histogram_ops_test.py
index e819e0234d..bf6e0296f6 100644
--- a/tensorflow/python/ops/histogram_ops_test.py
+++ b/tensorflow/python/ops/histogram_ops_test.py
@@ -36,7 +36,7 @@ class HistogramFixedWidthTest(test.TestCase):
     value_range = [0.0, 5.0]
     values = []
     expected_bin_counts = [0, 0, 0, 0, 0]
-    with self.test_session():
+    with self.test_session(use_gpu=True):
       hist = histogram_ops.histogram_fixed_width(values, value_range, nbins=5)
       self.assertEqual(dtypes.int32, hist.dtype)
       self.assertAllClose(expected_bin_counts, hist.eval())
@@ -47,7 +47,7 @@ class HistogramFixedWidthTest(test.TestCase):
     value_range = [0.0, 5.0]
     values = [-1.0, 0.0, 1.5, 2.0, 5.0, 15]
     expected_bin_counts = [2, 1, 1, 0, 2]
-    with self.test_session():
+    with self.test_session(use_gpu=True):
       hist = histogram_ops.histogram_fixed_width(
           values, value_range, nbins=5, dtype=dtypes.int64)
       self.assertEqual(dtypes.int64, hist.dtype)
@@ -59,7 +59,7 @@ class HistogramFixedWidthTest(test.TestCase):
     value_range = np.float64([0.0, 5.0])
     values = np.float64([-1.0, 0.0, 1.5, 2.0, 5.0, 15])
     expected_bin_counts = [2, 1, 1, 0, 2]
-    with self.test_session():
+    with self.test_session(use_gpu=True):
       hist = histogram_ops.histogram_fixed_width(values, value_range, nbins=5)
       self.assertEqual(dtypes.int32, hist.dtype)
       self.assertAllClose(expected_bin_counts, hist.eval())
@@ -70,7 +70,7 @@ class HistogramFixedWidthTest(test.TestCase):
     value_range = [0.0, 5.0]
     values = [[-1.0, 0.0, 1.5], [2.0, 5.0, 15]]
     expected_bin_counts = [2, 1, 1, 0, 2]
-    with self.test_session():
+    with self.test_session(use_gpu=True):
       hist = histogram_ops.histogram_fixed_width(values, value_range, nbins=5)
       self.assertEqual(dtypes.int32, hist.dtype)
       self.assertAllClose(expected_bin_counts, hist.eval())
-- 
GitLab


From 684f88fa7e61721c3264dc70abeed2b3e6fa7717 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Mon, 16 Oct 2017 17:22:26 -0700
Subject: [PATCH 459/909] [XLA:GPU] Don't crash with --vmodule=gpu_compiler=2
 if we can't run ptxas.

At --vmodule=gpu_compiler=2, we run ptxas over our generated PTX, to
validate it, and also to dump out stats like the number of registers
used.

But previously, this would fail if your GPU was anything other than
sm_35 (i.e. K20/40/80), because we didn't pass down cc_major/cc_minor to
ptxas.  And moreover, if ptxas failed to compile your program, we'd
LOG(FATAL), which is probably no what you want.

This change fixes both those issues.  Tested on my local GTX1080.

PiperOrigin-RevId: 172403304
---
 .../compiler/xla/service/gpu/gpu_compiler.cc  | 20 +++++++++++++------
 1 file changed, 14 insertions(+), 6 deletions(-)

diff --git a/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc b/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc
index 57f11db11f..3e16e4e3c4 100644
--- a/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc
+++ b/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc
@@ -67,6 +67,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/util.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/io/path.h"
+#include "tensorflow/core/lib/strings/strcat.h"
 #include "tensorflow/core/platform/cuda_libdevice_path.h"
 #include "tensorflow/core/platform/env.h"
 #include "tensorflow/core/platform/logging.h"
@@ -84,6 +85,8 @@ namespace gpu {
 
 namespace {
 
+using tensorflow::strings::StrCat;
+
 // Any address of a variable residing in global memory or returned by one of the
 // memory allocation routines from the driver or runtime API is always aligned
 // to at least 256 bytes.
@@ -223,7 +226,7 @@ tensorflow::Status PrepareHloModuleForIrEmitting(
 }
 
 // Invokes the ptxas tool on the given PTX string, and dumps its output.
-void DumpPtxasInfo(const string& ptx) {
+void DumpPtxasInfo(const string& ptx, int cc_major, int cc_minor) {
   const string ptxas_path =
       tensorflow::io::JoinPath(tensorflow::CudaRoot(), "bin/ptxas");
   // Do not log PTX stats if ptxas is not found at the given path.
@@ -245,17 +248,22 @@ void DumpPtxasInfo(const string& ptx) {
 
   // Invoke ptxas and collect its output.
   tensorflow::SubProcess ptxas_info_dumper;
-  ptxas_info_dumper.SetProgram(ptxas_path, {ptxas_path, ptx_path, "-o",
-                                            "/dev/null", "-v", "-arch=sm_35"});
+  ptxas_info_dumper.SetProgram(ptxas_path,
+                               {ptxas_path, ptx_path, "-o", "/dev/null", "-v",
+                                StrCat("-arch=sm_", cc_major, cc_minor)});
   ptxas_info_dumper.SetChannelAction(tensorflow::CHAN_STDERR,
                                      tensorflow::ACTION_PIPE);
-  CHECK(ptxas_info_dumper.Start());
+  if (!ptxas_info_dumper.Start()) {
+    LOG(ERROR) << "Failed to launch ptxas.";
+    return;
+  }
   string stderr_output;
   int exit_status = ptxas_info_dumper.Communicate(
       /*stdin_input=*/nullptr, /*stdout_output=*/nullptr, &stderr_output);
   XLA_LOG_LINES(tensorflow::INFO, stderr_output);
   if (exit_status != 0) {
-    LOG(FATAL) << "Invalid PTX. See the error message above for reasons.";
+    LOG(ERROR) << "ptxas exited with non-zero error code " << exit_status
+               << ".";
   }
 }
 
@@ -387,7 +395,7 @@ StatusOr<std::unique_ptr<Executable>> GpuCompiler::Compile(
   VLOG(2) << "PTX:";
   XLA_VLOG_LINES(2, *ptx);
   if (VLOG_IS_ON(2)) {
-    DumpPtxasInfo(*ptx);
+    DumpPtxasInfo(*ptx, cc_major, cc_minor);
   }
 
   auto thunk_schedule = MakeUnique<ThunkSchedule>(
-- 
GitLab


From 5c5dc8d5641b7c915f681109921dfb2b3e082a9b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 16 Oct 2017 17:59:11 -0700
Subject: [PATCH 460/909] Adding an ItemHandler that does lookups.  This allows
 decoding of tf.Examples where IDs are not materialized (e.g.
 'image/object/class/text' present but 'image/object/class/label' not).

PiperOrigin-RevId: 172406978
---
 .../python/slim/data/tfexample_decoder.py     | 36 +++++++++++++++++++
 .../slim/data/tfexample_decoder_test.py       | 31 ++++++++++++++++
 2 files changed, 67 insertions(+)

diff --git a/tensorflow/contrib/slim/python/slim/data/tfexample_decoder.py b/tensorflow/contrib/slim/python/slim/data/tfexample_decoder.py
index 094568389c..7a56df9e97 100644
--- a/tensorflow/contrib/slim/python/slim/data/tfexample_decoder.py
+++ b/tensorflow/contrib/slim/python/slim/data/tfexample_decoder.py
@@ -207,6 +207,42 @@ class Tensor(ItemHandler):
     return tensor
 
 
+class LookupTensor(Tensor):
+  """An ItemHandler that returns a parsed Tensor, the result of a lookup."""
+
+  def __init__(self,
+               tensor_key,
+               table,
+               shape_keys=None,
+               shape=None,
+               default_value=''):
+    """Initializes the LookupTensor handler.
+
+    See Tensor.  Simply calls a vocabulary (most often, a label mapping) lookup.
+
+    Args:
+      tensor_key: the name of the `TFExample` feature to read the tensor from.
+      table: A tf.lookup table.
+      shape_keys: Optional name or list of names of the TF-Example feature in
+        which the tensor shape is stored. If a list, then each corresponds to
+        one dimension of the shape.
+      shape: Optional output shape of the `Tensor`. If provided, the `Tensor` is
+        reshaped accordingly.
+      default_value: The value used when the `tensor_key` is not found in a
+        particular `TFExample`.
+
+    Raises:
+      ValueError: if both `shape_keys` and `shape` are specified.
+    """
+    self._table = table
+    super(LookupTensor, self).__init__(tensor_key, shape_keys, shape,
+                                       default_value)
+
+  def tensors_to_item(self, keys_to_tensors):
+    unmapped_tensor = super(LookupTensor, self).tensors_to_item(keys_to_tensors)
+    return self._table.lookup(unmapped_tensor)
+
+
 class SparseTensor(ItemHandler):
   """An ItemHandler for SparseTensors."""
 
diff --git a/tensorflow/contrib/slim/python/slim/data/tfexample_decoder_test.py b/tensorflow/contrib/slim/python/slim/data/tfexample_decoder_test.py
index 60d1eba07f..9c5a14d006 100644
--- a/tensorflow/contrib/slim/python/slim/data/tfexample_decoder_test.py
+++ b/tensorflow/contrib/slim/python/slim/data/tfexample_decoder_test.py
@@ -28,6 +28,7 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import image_ops
+from tensorflow.python.ops import lookup_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import parsing_ops
 from tensorflow.python.platform import test
@@ -811,6 +812,36 @@ class TFExampleDecoderTest(test.TestCase):
       self.assertAllEqual(np.squeeze(output_image[0, :, :, :]), image)
       self.assertAllEqual(np.squeeze(output_image[1, :, :, :]), image)
 
+  def testDecodeExampleWithLookup(self):
+
+    example = example_pb2.Example(features=feature_pb2.Features(feature={
+        'image/object/class/text': self._BytesFeature(
+            np.array(['cat', 'dog', 'guinea pig'])),
+    }))
+    serialized_example = example.SerializeToString()
+    # 'dog' -> 0, 'guinea pig' -> 1, 'cat' -> 2
+    table = lookup_ops.index_table_from_tensor(
+        constant_op.constant(['dog', 'guinea pig', 'cat']))
+
+    with self.test_session() as sess:
+      sess.run(lookup_ops.tables_initializer())
+
+      serialized_example = array_ops.reshape(serialized_example, shape=[])
+
+      keys_to_features = {
+          'image/object/class/text': parsing_ops.VarLenFeature(dtypes.string),
+      }
+
+      items_to_handlers = {
+          'labels':
+              tfexample_decoder.LookupTensor('image/object/class/text', table),
+      }
+
+      decoder = tfexample_decoder.TFExampleDecoder(keys_to_features,
+                                                   items_to_handlers)
+      obtained_class_ids = decoder.decode(serialized_example)[0].eval()
+
+    self.assertAllClose([2, 0, 1], obtained_class_ids)
 
 if __name__ == '__main__':
   test.main()
-- 
GitLab


From ecaa2eee832bd5b4286377f0f853c961c6ac2ab2 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 16 Oct 2017 18:06:20 -0700
Subject: [PATCH 461/909] math_grad: Fast path for when broadcasting is not
 needed.

PiperOrigin-RevId: 172407754
---
 tensorflow/contrib/compiler/jit_test.py       | 28 ++++++------
 .../graph_editor/tests/transform_test.py      |  4 +-
 .../layers/python/layers/optimizers_test.py   |  4 +-
 .../keras/_impl/keras/optimizers_test.py      |  5 ++-
 tensorflow/python/ops/math_grad.py            | 22 +++++++++-
 tensorflow/python/ops/rnn_cell_impl.py        | 44 ++++++++++++++-----
 6 files changed, 77 insertions(+), 30 deletions(-)

diff --git a/tensorflow/contrib/compiler/jit_test.py b/tensorflow/contrib/compiler/jit_test.py
index 94aff13a49..2108e42bce 100644
--- a/tensorflow/contrib/compiler/jit_test.py
+++ b/tensorflow/contrib/compiler/jit_test.py
@@ -173,12 +173,12 @@ class CompilationEnabledInGradientTest(test.TestCase):
 
   def testCompilationInGradient(self):
     with self.test_session():
-      x = constant_op.constant(3)
-      y_nc = math_ops.add(x, x, name="not_compiled")
+      x = constant_op.constant([[3]])
+      y_nc = math_ops.matmul(x, x, name="not_compiled")
       with jit.experimental_jit_scope():
-        y_c = math_ops.add(y_nc, y_nc, name="compiled")
+        y_c = math_ops.matmul(y_nc, y_nc, name="compiled")
       x_grads = gradients.gradients([y_c], [x])[0]
-      operations = x_grads.graph.get_operations()
+      operations = x.graph.get_operations()
       c_grad_ops = [
           op for op in operations if "gradients/compiled" in op.name]
       nc_grad_ops = [
@@ -191,19 +191,19 @@ class CompilationEnabledInGradientTest(test.TestCase):
         with self.assertRaisesRegexp(ValueError, "No attr named"):
           ncg.get_attr("_XlaCompile")
 
-      # d/dx (4 * x)
-      self.assertAllClose(4, x_grads.eval())
+      # d/dx (x ** 4) = 4 * (x ** 3)
+      self.assertAllClose([[108]], x_grads.eval())
 
   def testCompilationGradientScopeNames(self):
     with self.test_session(graph=ops.Graph()):
       with jit.experimental_jit_scope():
         # XlaScope 0
-        a1 = constant_op.constant(1)
-        a1t = a1 + a1
+        a1 = constant_op.constant([[1]])
+        a1t = math_ops.matmul(a1, a1)
       with jit.experimental_jit_scope():
         # XlaScope 1
-        a2 = constant_op.constant(1)
-        a2t = a2 + a2
+        a2 = constant_op.constant([[1]])
+        a2t = math_ops.matmul(a2, a2)
 
       self.assertEqual(b"jit_scope_0", a1.op.get_attr("_XlaScope"))
       self.assertEqual(b"jit_scope_1", a2.op.get_attr("_XlaScope"))
@@ -220,12 +220,12 @@ class CompilationEnabledInGradientTest(test.TestCase):
     with self.test_session(graph=ops.Graph()):
       with jit.experimental_jit_scope(True, separate_compiled_gradients=True):
         # XlaScope 0
-        a1 = constant_op.constant(1)
-        a1t = a1 + a1
+        a1 = constant_op.constant([[1]])
+        a1t = math_ops.matmul(a1, a1)
       with jit.experimental_jit_scope(True, separate_compiled_gradients=True):
         # XlaScope 1
-        a2 = constant_op.constant(1)
-        a2t = a2 + a2
+        a2 = constant_op.constant([[1]])
+        a2t = math_ops.matmul(a2, a2)
 
       self.assertEqual(b"jit_scope_0", a1.op.get_attr("_XlaScope"))
       self.assertEqual(b"jit_scope_1", a2.op.get_attr("_XlaScope"))
diff --git a/tensorflow/contrib/graph_editor/tests/transform_test.py b/tensorflow/contrib/graph_editor/tests/transform_test.py
index ab5776b9dd..ca00394388 100644
--- a/tensorflow/contrib/graph_editor/tests/transform_test.py
+++ b/tensorflow/contrib/graph_editor/tests/transform_test.py
@@ -191,14 +191,14 @@ class TransformTest(test.TestCase):
     # Extract the operations.
     replacement_ts = {w.value(): g}
     original_mul1_grad = (ops.get_default_graph().
-                          get_operation_by_name("grad/mul1_grad/mul_1"))
+                          get_operation_by_name("grad/mul1_grad/Mul_1"))
 
     # Should not raise exception.
     res = ge.graph_replace(g, replacement_ts, dst_scope="res")
 
     # Extract the operations after graph_replace.
     result_mul1_grad = (ops.get_default_graph().
-                        get_operation_by_name("res/grad/mul1_grad/mul_1"))
+                        get_operation_by_name("res/grad/mul1_grad/Mul_1"))
 
     # Make sure _original_ops are as expected.
     self.assertEquals(original_mul1_grad._original_op.name, u"mul1")
diff --git a/tensorflow/contrib/layers/python/layers/optimizers_test.py b/tensorflow/contrib/layers/python/layers/optimizers_test.py
index 8813a99f19..1ea25bd1a5 100644
--- a/tensorflow/contrib/layers/python/layers/optimizers_test.py
+++ b/tensorflow/contrib/layers/python/layers/optimizers_test.py
@@ -176,7 +176,7 @@ class OptimizersTest(test.TestCase):
       session.run(train, feed_dict={x: 5})
       var_value, global_step_value = session.run([var, global_step])
       # Due to randomness the following number may change if graph is different.
-      self.assertAlmostEqual(var_value, 8.5591021, 4)
+      self.assertAlmostEqual(var_value, 9.86912, 4)
       self.assertEqual(global_step_value, 1)
 
   def testGradientNoiseWithClipping(self):
@@ -193,7 +193,7 @@ class OptimizersTest(test.TestCase):
       variables.global_variables_initializer().run()
       session.run(train, feed_dict={x: 5})
       var_value, global_step_value = session.run([var, global_step])
-      self.assertAlmostEqual(var_value, 9.0, 4)
+      self.assertAlmostEqual(var_value, 9.86912, 4)
       self.assertEqual(global_step_value, 1)
 
   def testGradientClip(self):
diff --git a/tensorflow/python/keras/_impl/keras/optimizers_test.py b/tensorflow/python/keras/_impl/keras/optimizers_test.py
index b63d82f6a0..6e9e4e6c99 100644
--- a/tensorflow/python/keras/_impl/keras/optimizers_test.py
+++ b/tensorflow/python/keras/_impl/keras/optimizers_test.py
@@ -93,7 +93,10 @@ class KerasOptimizersTest(test.TestCase):
   def test_adadelta(self):
     with self.test_session():
       _test_optimizer(keras.optimizers.Adadelta(), target=0.6)
-      _test_optimizer(keras.optimizers.Adadelta(decay=1e-3), target=0.6)
+      # Accuracy seems dependent on the initialization. Even adding tf.Print
+      # nodes in the graph seemed to affect the initialization seed, and hence
+      # the accuracy.
+      _test_optimizer(keras.optimizers.Adadelta(decay=1e-3), target=0.4)
 
   def test_adam(self):
     with self.test_session():
diff --git a/tensorflow/python/ops/math_grad.py b/tensorflow/python/ops/math_grad.py
index 3754e039ed..38fe093ba7 100644
--- a/tensorflow/python/ops/math_grad.py
+++ b/tensorflow/python/ops/math_grad.py
@@ -700,10 +700,26 @@ def _AddNGrad(op, grad):
   return [grad] * len(op.inputs)
 
 
+def _ShapesFullySpecifiedAndEqual(x, y, grad):
+  # pylint: disable=protected-access
+  x_shape = x._shape_tuple()
+  y_shape = y._shape_tuple()
+  grad_shape = grad._shape_tuple()
+  # pylint: enable=protected-access
+  return (x_shape == y_shape and
+          x_shape == grad_shape and
+          x_shape is not None and
+          None not in x_shape)
+
+
 @ops.RegisterGradient("Add")
 def _AddGrad(op, grad):
+  """Gradient for Add."""
   x = op.inputs[0]
   y = op.inputs[1]
+  if (isinstance(grad, ops.Tensor) and
+      _ShapesFullySpecifiedAndEqual(x, y, grad)):
+    return grad, grad
   sx = array_ops.shape(x)
   sy = array_ops.shape(y)
   # pylint: disable=protected-access
@@ -731,10 +747,14 @@ def _MulGrad(op, grad):
   """The gradient of scalar multiplication."""
   x = op.inputs[0]
   y = op.inputs[1]
+  # pylint: disable=protected-access
+  if (isinstance(grad, ops.Tensor) and
+      _ShapesFullySpecifiedAndEqual(x, y, grad) and
+      grad.dtype in (dtypes.int32, dtypes.float32)):
+    return gen_math_ops._mul(grad, y), gen_math_ops._mul(grad, x)
   assert x.dtype.base_dtype == y.dtype.base_dtype, (x.dtype, " vs. ", y.dtype)
   sx = array_ops.shape(x)
   sy = array_ops.shape(y)
-  # pylint: disable=protected-access
   rx, ry = gen_array_ops._broadcast_gradient_args(sx, sy)
   # pylint: enable=protected-access
   x = math_ops.conj(x)
diff --git a/tensorflow/python/ops/rnn_cell_impl.py b/tensorflow/python/ops/rnn_cell_impl.py
index 4056eade81..fb7b6d11a5 100644
--- a/tensorflow/python/ops/rnn_cell_impl.py
+++ b/tensorflow/python/ops/rnn_cell_impl.py
@@ -178,8 +178,13 @@ class RNNCell(base_layer.Layer):
                              custom_getter=self._rnn_get_variable) as scope:
         return super(RNNCell, self).__call__(inputs, state, scope=scope)
     else:
-      with vs.variable_scope(vs.get_variable_scope(),
-                             custom_getter=self._rnn_get_variable):
+      scope_attrname = "rnncell_scope"
+      scope = getattr(self, scope_attrname, None)
+      if scope is None:
+        scope = vs.variable_scope(vs.get_variable_scope(),
+                                  custom_getter=self._rnn_get_variable)
+        setattr(self, scope_attrname, scope)
+      with scope:
         return super(RNNCell, self).__call__(inputs, state)
 
   def _rnn_get_variable(self, getter, *args, **kwargs):
@@ -230,9 +235,20 @@ class RNNCell(base_layer.Layer):
       a nested list or tuple (of the same structure) of `2-D` tensors with
       the shapes `[batch_size x s]` for each s in `state_size`.
     """
+    # Try to use the last cached zero_state. This is done to avoid recreating
+    # zeros, especially when eager execution is enabled.
+    state_size = self.state_size
+    if hasattr(self, "_last_zero_state"):
+      (last_state_size, last_batch_size, last_dtype,
+       last_output) = getattr(self, "_last_zero_state")
+      if (last_batch_size == batch_size and
+          last_dtype == dtype and
+          last_state_size == state_size):
+        return last_output
     with ops.name_scope(type(self).__name__ + "ZeroState", values=[batch_size]):
-      state_size = self.state_size
-      return _zero_state_tensors(state_size, batch_size, dtype)
+      output = _zero_state_tensors(state_size, batch_size, dtype)
+    self._last_zero_state = (state_size, batch_size, dtype, output)
+    return output
 
 
 class BasicRNNCell(RNNCell):
@@ -428,21 +444,27 @@ class BasicLSTMCell(RNNCell):
         `state_is_tuple`).
     """
     sigmoid = math_ops.sigmoid
+    one = constant_op.constant(1, dtype=dtypes.int32)
     # Parameters of gates are concatenated into one multiply for efficiency.
     if self._state_is_tuple:
       c, h = state
     else:
-      c, h = array_ops.split(value=state, num_or_size_splits=2, axis=1)
+      c, h = array_ops.split(value=state, num_or_size_splits=2, axis=one)
 
     if self._linear is None:
       self._linear = _Linear([inputs, h], 4 * self._num_units, True)
     # i = input_gate, j = new_input, f = forget_gate, o = output_gate
     i, j, f, o = array_ops.split(
-        value=self._linear([inputs, h]), num_or_size_splits=4, axis=1)
+        value=self._linear([inputs, h]), num_or_size_splits=4, axis=one)
 
-    new_c = (
-        c * sigmoid(f + self._forget_bias) + sigmoid(i) * self._activation(j))
-    new_h = self._activation(new_c) * sigmoid(o)
+    forget_bias_tensor = constant_op.constant(self._forget_bias, dtype=f.dtype)
+    # Note that using `add` and `multiply` instead of `+` and `*` gives a
+    # performance improvement. So using those at the cost of readability.
+    add = math_ops.add
+    multiply = math_ops.multiply
+    new_c = add(multiply(c, sigmoid(add(f, forget_bias_tensor))),
+                multiply(sigmoid(i), self._activation(j)))
+    new_h = multiply(self._activation(new_c), sigmoid(o))
 
     if self._state_is_tuple:
       new_state = LSTMStateTuple(new_c, new_h)
@@ -1186,7 +1208,9 @@ class _Linear(object):
     if len(args) == 1:
       res = math_ops.matmul(args[0], self._weights)
     else:
-      res = math_ops.matmul(array_ops.concat(args, 1), self._weights)
+      # Explicitly creating a one for a minor performance improvement.
+      one = constant_op.constant(1, dtype=dtypes.int32)
+      res = math_ops.matmul(array_ops.concat(args, one), self._weights)
     if self._build_bias:
       res = nn_ops.bias_add(res, self._biases)
     return res
-- 
GitLab


From a9da1baf65b42de9751959cccf6d899c69c0156b Mon Sep 17 00:00:00 2001
From: Vijay Vasudevan <vrv@google.com>
Date: Mon, 16 Oct 2017 18:19:58 -0700
Subject: [PATCH 462/909] Disable probable timeout flake on Ubuntu machines.

PiperOrigin-RevId: 172408922
---
 .../python/kernel_tests/dataset_from_generator_op_test.py      | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/kernel_tests/dataset_from_generator_op_test.py b/tensorflow/python/kernel_tests/dataset_from_generator_op_test.py
index e774256695..cd2bec8432 100644
--- a/tensorflow/python/kernel_tests/dataset_from_generator_op_test.py
+++ b/tensorflow/python/kernel_tests/dataset_from_generator_op_test.py
@@ -135,7 +135,8 @@ class DatasetConstructorTest(test.TestCase):
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
-  def testFromGeneratorsRunningInParallel(self):
+  # TODO(b/67868766): Reenable this when the source of flakiness is discovered.
+  def _testFromGeneratorsRunningInParallel(self):
     num_parallel_iterators = 3
 
     # Define shared state that multiple iterator instances will access to
-- 
GitLab


From 0a092298823d73d1e4cc76e81e0825a8789cd6a5 Mon Sep 17 00:00:00 2001
From: Justine Tunney <jart@google.com>
Date: Mon, 16 Oct 2017 19:03:32 -0700
Subject: [PATCH 463/909] Make Snappy header available

This is going to be useful for the tensor database I'm working on.

PiperOrigin-RevId: 172412142
---
 tensorflow/core/BUILD | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 4d9f368bc0..94ddd0840d 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -1398,6 +1398,7 @@ LIB_INTERNAL_PUBLIC_HEADERS = tf_additional_lib_hdrs() + [
     "platform/platform.h",
     "platform/protobuf_internal.h",
     "platform/setround.h",
+    "platform/snappy.h",
     "platform/tensor_coding.h",
     "platform/tracing.h",
 ]
@@ -2257,7 +2258,6 @@ cc_library(
         "lib/io/block_builder.h",
         "lib/io/format.h",
         "lib/random/philox_random_test_utils.h",
-        "platform/snappy.h",
     ],
     deps = [
         ":lib",
-- 
GitLab


From ba5a5bfc23065086990ec3057caa2ded0c8a8dbf Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 16 Oct 2017 21:59:25 -0700
Subject: [PATCH 464/909] Add the op->IsExpensive() argument to tracing calls.

PiperOrigin-RevId: 172422580
---
 tensorflow/compiler/jit/xla_device.cc         |  6 +++--
 .../core/common_runtime/gpu/gpu_device.cc     |  3 ++-
 .../core/common_runtime/threadpool_device.cc  |  3 ++-
 .../core/platform/default/gpu_tracer.cc       |  5 +++-
 tensorflow/core/platform/tracing.h            | 25 ++++++++++++-------
 5 files changed, 28 insertions(+), 14 deletions(-)

diff --git a/tensorflow/compiler/jit/xla_device.cc b/tensorflow/compiler/jit/xla_device.cc
index 7ccea58f6e..d4d8fe1c1d 100644
--- a/tensorflow/compiler/jit/xla_device.cc
+++ b/tensorflow/compiler/jit/xla_device.cc
@@ -241,7 +241,8 @@ void XlaDevice::Compute(OpKernel* op_kernel, OpKernelContext* context) {
   // When TraceMe profiling is off (which is the default), the
   // following TraceMe constructor is simply a conditional test of
   // false value. Measurements show that its overhead is negligible.
-  port::Tracing::TraceMe trace_me(op_kernel->name(), op_kernel->type_string());
+  port::Tracing::TraceMe trace_me(op_kernel->name(), op_kernel->type_string(),
+                                  op_kernel->IsExpensive());
   op_kernel->Compute(context);
 }
 
@@ -249,7 +250,8 @@ void XlaDevice::ComputeAsync(AsyncOpKernel* op_kernel, OpKernelContext* context,
                              AsyncOpKernel::DoneCallback done) {
   VLOG(1) << "XlaDevice::ComputeAsync " << op_kernel->name() << ":"
           << op_kernel->type_string();
-  port::Tracing::TraceMe trace_me(op_kernel->name(), op_kernel->type_string());
+  port::Tracing::TraceMe trace_me(op_kernel->name(), op_kernel->type_string(),
+                                  op_kernel->IsExpensive());
   op_kernel->ComputeAsync(context, done);
 }
 
diff --git a/tensorflow/core/common_runtime/gpu/gpu_device.cc b/tensorflow/core/common_runtime/gpu/gpu_device.cc
index 3324e833ff..12d44cc6b7 100644
--- a/tensorflow/core/common_runtime/gpu/gpu_device.cc
+++ b/tensorflow/core/common_runtime/gpu/gpu_device.cc
@@ -475,7 +475,8 @@ void BaseGPUDevice::ComputeAsync(AsyncOpKernel* op_kernel,
   // When TraceMe profiling is off (which is the default), the
   // following TraceMe constructor is simply a conditional test of
   // false value. Measurements show that its overhead is negligible.
-  port::Tracing::TraceMe activity(op_kernel->name(), op_kernel->type_string());
+  port::Tracing::TraceMe activity(op_kernel->name(), op_kernel->type_string(),
+                                  op_kernel->IsExpensive());
   gpu::cuda::ScopedActivateExecutorContext scoped_activation{stream->parent()};
   op_kernel->ComputeAsync(context, done);
 }
diff --git a/tensorflow/core/common_runtime/threadpool_device.cc b/tensorflow/core/common_runtime/threadpool_device.cc
index 23ccca1c94..5aa01376ab 100644
--- a/tensorflow/core/common_runtime/threadpool_device.cc
+++ b/tensorflow/core/common_runtime/threadpool_device.cc
@@ -48,7 +48,8 @@ void ThreadPoolDevice::Compute(OpKernel* op_kernel, OpKernelContext* context) {
   // When TraceMe profiling is off (which is the default), the
   // following TraceMe constructor is simply a conditional test of
   // false value. Measurements show that its overhead is negligible.
-  port::Tracing::TraceMe trace_me(op_kernel->name(), op_kernel->type_string());
+  port::Tracing::TraceMe trace_me(op_kernel->name(), op_kernel->type_string(),
+                                  op_kernel->IsExpensive());
   if (port::Tracing::IsActive()) {
     // TODO(pbar) We really need a useful identifier of the graph node.
     const uint64 id = Hash64(op_kernel->name());
diff --git a/tensorflow/core/platform/default/gpu_tracer.cc b/tensorflow/core/platform/default/gpu_tracer.cc
index 3f85546127..e52e37ad71 100644
--- a/tensorflow/core/platform/default/gpu_tracer.cc
+++ b/tensorflow/core/platform/default/gpu_tracer.cc
@@ -315,10 +315,13 @@ class GPUTracerImpl : public GPUTracer,
     };
     return new Impl(name);
   }
-  Tracer *StartTracing(StringPiece label) override {
+  Tracer *StartTracing(StringPiece label, bool is_expensive) override {
     // We don't do anything with 'TraceMe' regions yet.
     return nullptr;
   }
+  Tracer *StartTracing(StringPiece label) {
+    return StartTracing(label, /*is_expensive=*/true);
+  }
 
  protected:
   // This callback is used exclusively by CUPTIManager.
diff --git a/tensorflow/core/platform/tracing.h b/tensorflow/core/platform/tracing.h
index b7724bbeae..bb8e902efc 100644
--- a/tensorflow/core/platform/tracing.h
+++ b/tensorflow/core/platform/tracing.h
@@ -169,10 +169,10 @@ class Tracing::Engine {
   // Start tracing under the specified label. Caller should delete the result
   // to stop tracing.
   // May return nullptr if tracing is not supported.
-  virtual Tracer* StartTracing(StringPiece label) = 0;
+  virtual Tracer* StartTracing(StringPiece label, bool is_expensive) = 0;
   // Same as above, but implementations can avoid copying the string.
-  virtual Tracer* StartTracing(string&& label) {
-    return StartTracing(StringPiece(label));
+  virtual Tracer* StartTracing(string&& label, bool is_expensive) {
+    return StartTracing(StringPiece(label), is_expensive);
   }
 };
 
@@ -218,12 +218,14 @@ class Tracing::ScopedAnnotation {
 class Tracing::TraceMe {
  public:
   explicit TraceMe(StringPiece name);
+  TraceMe(StringPiece name, bool is_expensive);
 
   // If tracing is enabled, set up a traceMe with a label of
   // "<name_part1>:<name_part2>".  This can be cheaper than the
   // single-argument constructor because the concatenation of the
   // label string is only done if tracing is enabled.
   TraceMe(StringPiece name_part1, StringPiece name_part2);
+  TraceMe(StringPiece name_part1, StringPiece name_part2, bool is_expensive);
 
  private:
   std::unique_ptr<Engine::Tracer> tracer_;
@@ -245,19 +247,24 @@ inline Tracing::ScopedAnnotation::ScopedAnnotation(StringPiece name_part1,
   }
 }
 
-inline Tracing::TraceMe::TraceMe(StringPiece name) {
+inline Tracing::TraceMe::TraceMe(StringPiece name) : TraceMe(name, true) {}
+
+inline Tracing::TraceMe::TraceMe(StringPiece name, bool is_expensive) {
   auto e = Tracing::engine();
   if (e && e->IsEnabled()) {
-    tracer_.reset(e->StartTracing(name));
+    tracer_.reset(e->StartTracing(name, is_expensive));
   }
 }
 
-inline Tracing::TraceMe::TraceMe(StringPiece name_part1,
-                                 StringPiece name_part2) {
+inline Tracing::TraceMe::TraceMe(StringPiece name_part1, StringPiece name_part2)
+    : TraceMe(name_part1, name_part2, true) {}
+
+inline Tracing::TraceMe::TraceMe(StringPiece name_part1, StringPiece name_part2,
+                                 bool is_expensive) {
   auto e = Tracing::engine();
   if (e && e->IsEnabled()) {
-    tracer_.reset(
-        e->StartTracing(strings::StrCat(name_part1, ":", name_part2)));
+    tracer_.reset(e->StartTracing(strings::StrCat(name_part1, ":", name_part2),
+                                  is_expensive));
   }
 }
 
-- 
GitLab


From a1ba9f3bf16cb53b8468b93021611311a9be55b4 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 17 Oct 2017 08:46:12 -0700
Subject: [PATCH 465/909] HParams: deprecates function set_from_map in favor of
 new function override_from_dict

Reasons to prefer new function name:
 - `set` sounds like it might return the builtin set.
 - There is no datatype `map` in python - it's a builtin, making the implied
   API a little confusing.

PiperOrigin-RevId: 172471191
---
 .../contrib/training/python/training/hparam.py | 18 ++++++++++++------
 .../training/python/training/hparam_test.py    |  4 ++--
 2 files changed, 14 insertions(+), 8 deletions(-)

diff --git a/tensorflow/contrib/training/python/training/hparam.py b/tensorflow/contrib/training/python/training/hparam.py
index c95a73ce44..1b52d23c61 100644
--- a/tensorflow/contrib/training/python/training/hparam.py
+++ b/tensorflow/contrib/training/python/training/hparam.py
@@ -25,6 +25,7 @@ import six
 from tensorflow.contrib.training.python.training import hparam_pb2
 from tensorflow.python.framework import ops
 from tensorflow.python.util import compat
+from tensorflow.python.util import deprecation
 
 # Define the regular expression for parsing a single clause of the input
 # (delimited by commas).  A legal clause looks like:
@@ -470,24 +471,29 @@ class HParams(object):
       type_map[name] = param_type
 
     values_map = parse_values(values, type_map)
-    return self.set_from_map(values_map)
+    return self.override_from_dict(values_map)
 
-  def set_from_map(self, values_map):
+  def override_from_dict(self, values_dict):
     """Override hyperparameter values, parsing new values from a dictionary.
 
     Args:
-      values_map: Dictionary of name:value pairs.
+      values_dict: Dictionary of name:value pairs.
 
     Returns:
       The `HParams` instance.
 
     Raises:
-      ValueError: If `values_map` cannot be parsed.
+      ValueError: If `values_dict` cannot be parsed.
     """
-    for name, value in values_map.items():
+    for name, value in values_dict.items():
       self.set_hparam(name, value)
     return self
 
+  @deprecation.deprecated(None, 'Use `override_from_dict`.')
+  def set_from_map(self, values_map):
+    """DEPRECATED. Use override_from_dict."""
+    return self.override_from_dict(values_dict=values_map)
+
   def set_model_structure(self, model_structure):
     self._model_structure = model_structure
 
@@ -515,7 +521,7 @@ class HParams(object):
       ValueError: If `values_json` cannot be parsed.
     """
     values_map = json.loads(values_json)
-    return self.set_from_map(values_map)
+    return self.override_from_dict(values_map)
 
   def values(self):
     """Return the hyperparameter values as a Python dictionary.
diff --git a/tensorflow/contrib/training/python/training/hparam_test.py b/tensorflow/contrib/training/python/training/hparam_test.py
index b01116a213..a947bf6eda 100644
--- a/tensorflow/contrib/training/python/training/hparam_test.py
+++ b/tensorflow/contrib/training/python/training/hparam_test.py
@@ -93,11 +93,11 @@ class HParamsTest(test.TestCase):
 
   def testSetFromMap(self):
     hparams = hparam.HParams(a=1, b=2.0, c='tanh')
-    hparams.set_from_map({'a': -2, 'c': 'identity'})
+    hparams.override_from_dict({'a': -2, 'c': 'identity'})
     self.assertDictEqual({'a': -2, 'c': 'identity', 'b': 2.0}, hparams.values())
 
     hparams = hparam.HParams(x=1, b=2.0, d=[0.5])
-    hparams.set_from_map({'d': [0.1, 0.2, 0.3]})
+    hparams.override_from_dict({'d': [0.1, 0.2, 0.3]})
     self.assertDictEqual({'d': [0.1, 0.2, 0.3], 'x': 1, 'b': 2.0},
                          hparams.values())
 
-- 
GitLab


From 18f89c81d288f191abd56501ec6f86fe29265bdd Mon Sep 17 00:00:00 2001
From: Eugene Brevdo <ebrevdo@google.com>
Date: Tue, 17 Oct 2017 08:48:29 -0700
Subject: [PATCH 466/909] [tf.contrib.seq2seq] Bugfixes to BeamSearchDecoder
 and GatherTree.

1. Begin the gather tree at the maximum sequence length across all beams (within the batch).
2. Take a second pass starting from t=0 and mask out any beam ids past the *first* beam occurrence of end_token.
3. Update the final sequence lengths to include the first <eos> token in the beam.
4. Update dynamic_decode to allow the BeamSearchDecoder to keep track of its own "finished" states, as the shuffling in the decoder confused the tracking mechanism in dynamic_decode.  This fixes a bug where beam search decoding stops early.
5. Cap sequence length used in GatherTree to min(max_time, max_seq_len(b)) to avoid accessing memory outside the dimensions of input matrices.

Bugs caught by @bdaskalov on github and Pavel Sountsov.  Proper solution and analysis thanks to Rui Zhao.  Thanks all!

Fixes #13536.

PiperOrigin-RevId: 172471462
---
 .../seq2seq/kernels/beam_search_ops.cc        | 104 ++++++++-------
 .../contrib/seq2seq/kernels/beam_search_ops.h |   4 +-
 .../seq2seq/kernels/beam_search_ops_gpu.cu.cc |  32 +++--
 .../contrib/seq2seq/ops/beam_search_ops.cc    |  25 ++--
 .../kernel_tests/beam_search_decoder_test.py  |   9 +-
 .../kernel_tests/beam_search_ops_test.py      | 118 ++++++++----------
 .../seq2seq/python/ops/beam_search_decoder.py |  39 ++++--
 .../contrib/seq2seq/python/ops/decoder.py     |  33 ++++-
 8 files changed, 217 insertions(+), 147 deletions(-)

diff --git a/tensorflow/contrib/seq2seq/kernels/beam_search_ops.cc b/tensorflow/contrib/seq2seq/kernels/beam_search_ops.cc
index aab0f3f494..95273e2b33 100644
--- a/tensorflow/contrib/seq2seq/kernels/beam_search_ops.cc
+++ b/tensorflow/contrib/seq2seq/kernels/beam_search_ops.cc
@@ -49,40 +49,46 @@ class GatherTreeOp : public OpKernel {
     const Device& device = ctx->eigen_device<Device>();
     const Tensor& step_ids = ctx->input(0);
     const Tensor& parent_ids = ctx->input(1);
-    const Tensor& sequence_length = ctx->input(2);
+    const Tensor& max_sequence_lengths = ctx->input(2);
+    const Tensor& end_token = ctx->input(3);
     const TensorShape& step_ids_shape = step_ids.shape();
     OP_REQUIRES(
         ctx, step_ids_shape.dims() == 3,
         errors::InvalidArgument("step_ids must be a 3-tensor, saw shape: ",
                                 step_ids_shape.DebugString()));
-    OP_REQUIRES(
-        ctx, TensorShapeUtils::IsMatrix(sequence_length.shape()),
-        errors::InvalidArgument("sequence_length must be a matrix, saw shape: ",
-                                sequence_length.shape().DebugString()));
-    OP_REQUIRES(ctx, sequence_length.dim_size(0) == step_ids_shape.dim_size(1),
-                errors::InvalidArgument(
-                    "Inconsistent batch sizes: sequence_length.shape[0] (",
-                    sequence_length.dim_size(0), ") != ", "step_ids.shape[1] (",
-                    step_ids_shape.dim_size(1), ")"));
-    OP_REQUIRES(ctx, sequence_length.dim_size(1) == step_ids_shape.dim_size(2),
+    OP_REQUIRES(ctx, TensorShapeUtils::IsVector(max_sequence_lengths.shape()),
                 errors::InvalidArgument(
-                    "Inconsistent batch sizes: sequence_length.shape[1] (",
-                    sequence_length.dim_size(1), ") != ", "step_ids.shape[2] (",
-                    step_ids_shape.dim_size(2), ")"));
+                    "max_sequence_lengths must be a vector, saw shape: ",
+                    max_sequence_lengths.shape().DebugString()));
+    OP_REQUIRES(
+        ctx, TensorShapeUtils::IsScalar(end_token.shape()),
+        errors::InvalidArgument("end_token must be a scalar, saw shape: ",
+                                end_token.shape().DebugString()));
     OP_REQUIRES(
         ctx, step_ids_shape == parent_ids.shape(),
         errors::InvalidArgument(
             "step_ids.shape must match parent_ids.shape.  but shapes are: ",
             step_ids_shape.DebugString(), " and ",
             parent_ids.shape().DebugString()));
+    OP_REQUIRES(
+        ctx,
+        step_ids_shape.dim_size(1) == max_sequence_lengths.shape().dim_size(0),
+        errors::InvalidArgument("batch size dimensions step_ids.shape[1] and "
+                                "max_seqeuence_lengths.shape[0] must match.  "
+                                "but shapes are: ",
+                                step_ids_shape.DebugString(), " and ",
+                                max_sequence_lengths.shape().DebugString()));
     Tensor* beams;
     OP_REQUIRES_OK(ctx, ctx->allocate_output(0, step_ids_shape, &beams));
     typename TTypes<T, 3>::ConstTensor step_ids_t = step_ids.tensor<T, 3>();
     typename TTypes<T, 3>::ConstTensor parent_ids_t = parent_ids.tensor<T, 3>();
-    typename TTypes<T>::ConstMatrix seq_len_t = sequence_length.matrix<T>();
+    typename TTypes<int32>::ConstVec max_seq_lens_t =
+        max_sequence_lengths.vec<int32>();
+    typename TTypes<T>::ConstScalar end_token_t = end_token.scalar<T>();
     typename TTypes<T, 3>::Tensor beams_t = beams->tensor<T, 3>();
+    const T end_token_value = end_token_t();
     functor::GatherTree<Device, T>()(ctx, device, step_ids_t, parent_ids_t,
-                                     seq_len_t, beams_t);
+                                     max_seq_lens_t, end_token_value, beams_t);
   }
 };
 
@@ -99,27 +105,29 @@ namespace functor {
 template <>
 struct GatherTree<CPUDevice, int32> {
   void operator()(OpKernelContext* ctx, const CPUDevice& d,
-                  typename TTypes<int32, 3>::ConstTensor step_ids,
-                  typename TTypes<int32, 3>::ConstTensor parent_ids,
-                  typename TTypes<int32>::ConstMatrix sequence_length,
-                  typename TTypes<int32, 3>::Tensor beams) {
-    const int64 max_time = parent_ids.dimension(0);
-    const int64 batch_size = parent_ids.dimension(1);
-    const int64 beam_width = parent_ids.dimension(2);
+                  TTypes<int32, 3>::ConstTensor step_ids,
+                  TTypes<int32, 3>::ConstTensor parent_ids,
+                  TTypes<int32>::ConstVec max_sequence_lengths,
+                  const int32 end_token, TTypes<int32, 3>::Tensor beams) {
+    const int32 max_time = parent_ids.dimension(0);
+    const int32 batch_size = parent_ids.dimension(1);
+    const int32 beam_width = parent_ids.dimension(2);
     beams.setConstant(-1);
 
-    auto DoWork = [&, ctx](int start_batch_beam, int limit_batch_beam) {
+    auto DoWork = [&, ctx, end_token](int start_batch_beam,
+                                      int limit_batch_beam) {
       for (int32 i = start_batch_beam; i < limit_batch_beam; ++i) {
         const int32 batch = i / beam_width;
         const int32 beam = i % beam_width;
-        int32 seq_len_b = sequence_length(batch, beam);
-        if (seq_len_b <= 0) {
+        const int32 max_seq_len_b =
+            Eigen::numext::mini(max_time, max_sequence_lengths(batch));
+        if (max_seq_len_b <= 0) {
           continue;
         }
-        beams(seq_len_b - 1, batch, beam) =
-            step_ids(seq_len_b - 1, batch, beam);
-        int32 parent = parent_ids(seq_len_b - 1, batch, beam);
-        for (int32 level = seq_len_b - 2; level >= 0; --level) {
+        beams(max_seq_len_b - 1, batch, beam) =
+            step_ids(max_seq_len_b - 1, batch, beam);
+        int32 parent = parent_ids(max_seq_len_b - 1, batch, beam);
+        for (int32 level = max_seq_len_b - 2; level >= 0; --level) {
           if (parent < 0 || parent > beam_width) {
             ctx->SetStatus(
                 errors::InvalidArgument("Saw invalid parent id ", parent,
@@ -130,6 +138,14 @@ struct GatherTree<CPUDevice, int32> {
           beams(level, batch, beam) = step_ids(level, batch, parent);
           parent = parent_ids(level, batch, parent);
         }
+        bool finished = false;
+        for (int32 time = 0; time < max_seq_len_b; ++time) {
+          if (finished) {
+            beams(time, batch, beam) = -1;
+          } else if (beams(time, batch, beam) == end_token) {
+            finished = true;
+          }
+        }
       }
     };
     // Guesstimate of cost; ~5 lookup/store/compare per inner beam
@@ -137,7 +153,7 @@ struct GatherTree<CPUDevice, int32> {
     const int64 batch_beam_cost =
         Eigen::TensorOpCost::DivCost<int32>() +
         6 * Eigen::TensorOpCost::AddCost<int32>() +
-        max_time * (5 * Eigen::TensorOpCost::AddCost<int32>());
+        2 * max_time * (5 * Eigen::TensorOpCost::AddCost<int32>());
     auto worker_threads = *(ctx->device()->tensorflow_cpu_worker_threads());
     Shard(worker_threads.num_threads, worker_threads.workers,
           batch_size * beam_width, batch_beam_cost, DoWork);
@@ -148,24 +164,26 @@ struct GatherTree<CPUDevice, int32> {
 
 #if GOOGLE_CUDA
 namespace functor {
-#define DECLARE_GPU_SPEC(T)                            \
-  template <>                                          \
-  void GatherTree<GPUDevice, T>::operator()(           \
-      OpKernelContext* ctx, const GPUDevice& d,        \
-      typename TTypes<T, 3>::ConstTensor step_ids,     \
-      typename TTypes<T, 3>::ConstTensor parent_ids,   \
-      typename TTypes<T>::ConstMatrix sequence_length, \
-      typename TTypes<T, 3>::Tensor beams);            \
+#define DECLARE_GPU_SPEC(T)                                            \
+  template <>                                                          \
+  void GatherTree<GPUDevice, T>::operator()(                           \
+      OpKernelContext* ctx, const GPUDevice& d,                        \
+      typename TTypes<T, 3>::ConstTensor step_ids,                     \
+      typename TTypes<T, 3>::ConstTensor parent_ids,                   \
+      TTypes<int32>::ConstVec max_sequence_lengths, const T end_token, \
+      typename TTypes<T, 3>::Tensor beams);                            \
   extern template struct GatherTree<GPUDevice, T>;
 
 DECLARE_GPU_SPEC(int32);
 #undef DECLARE_GPU_SPEC
 }  // end namespace functor
 
-#define REGISTER_GPU_KERNEL(T)                                      \
-  REGISTER_KERNEL_BUILDER(                                          \
-      Name("GatherTree").Device(DEVICE_GPU).TypeConstraint<T>("T"), \
-      GatherTreeOp<GPUDevice, T>);
+#define REGISTER_GPU_KERNEL(T)                          \
+  REGISTER_KERNEL_BUILDER(Name("GatherTree")            \
+                              .Device(DEVICE_GPU)       \
+                              .TypeConstraint<T>("T")   \
+                              .HostMemory("end_token"), \
+                          GatherTreeOp<GPUDevice, T>);
 
 REGISTER_GPU_KERNEL(int32);
 #undef REGISTER_GPU_KERNEL
diff --git a/tensorflow/contrib/seq2seq/kernels/beam_search_ops.h b/tensorflow/contrib/seq2seq/kernels/beam_search_ops.h
index 124d07264e..693b02dc43 100644
--- a/tensorflow/contrib/seq2seq/kernels/beam_search_ops.h
+++ b/tensorflow/contrib/seq2seq/kernels/beam_search_ops.h
@@ -31,8 +31,8 @@ struct GatherTree {
   void operator()(OpKernelContext* ctx, const Device& d,
                   typename TTypes<T, 3>::ConstTensor step_ids,
                   typename TTypes<T, 3>::ConstTensor parent_ids,
-                  typename TTypes<T>::ConstMatrix sequence_length,
-                  typename TTypes<T, 3>::Tensor beams);
+                  TTypes<int32>::ConstVec max_sequence_lengths,
+                  const T end_token, typename TTypes<T, 3>::Tensor beams);
 };
 
 }  // namespace functor
diff --git a/tensorflow/contrib/seq2seq/kernels/beam_search_ops_gpu.cu.cc b/tensorflow/contrib/seq2seq/kernels/beam_search_ops_gpu.cu.cc
index ee68b55d20..e71efc48ce 100644
--- a/tensorflow/contrib/seq2seq/kernels/beam_search_ops_gpu.cu.cc
+++ b/tensorflow/contrib/seq2seq/kernels/beam_search_ops_gpu.cu.cc
@@ -29,20 +29,24 @@ template <typename T>
 __global__ void GatherTreeOpKernel(const int32 batch_size, const int32 max_time,
                                    const int32 beam_width, const T* step_ids,
                                    const T* parent_ids,
-                                   const T* sequence_length, T* beams) {
+                                   const int32* max_sequence_lengths,
+                                   const T end_token, T* beams) {
   CUDA_1D_KERNEL_LOOP(i, batch_size * beam_width) {
     const int32 batch = i / beam_width;
     const int32 beam = i % beam_width;
 
-    const int32 seq_len_b = ldg(sequence_length + batch * beam_width + beam);
-    if (seq_len_b <= 0) continue;
+    const int32 max_seq_len_b =
+        Eigen::numext::mini(max_time, ldg(max_sequence_lengths + batch));
+    if (max_seq_len_b <= 0) {
+      continue;
+    }
 
 #define GET_IX(time_ix, beam_ix) \
   (batch_size * beam_width * (time_ix) + beam_width * batch + (beam_ix))
-    const int32 initial_beam_ix = GET_IX(seq_len_b - 1, beam);
+    const int32 initial_beam_ix = GET_IX(max_seq_len_b - 1, beam);
     beams[initial_beam_ix] = ldg(step_ids + initial_beam_ix);
     int32 parent = ldg(parent_ids + initial_beam_ix);
-    for (int32 level = seq_len_b - 2; level >= 0; --level) {
+    for (int32 level = max_seq_len_b - 2; level >= 0; --level) {
       const int32 level_beam_ix = GET_IX(level, beam);
       const int32 level_parent_ix = GET_IX(level, parent);
       if (parent < 0 || parent > beam_width) {
@@ -53,6 +57,15 @@ __global__ void GatherTreeOpKernel(const int32 batch_size, const int32 max_time,
         parent = ldg(parent_ids + level_parent_ix);
       }
     }
+    bool finished = false;
+    for (int32 time = 0; time < max_seq_len_b; ++time) {
+      const int32 level_beam_ix = GET_IX(time, beam);
+      if (finished) {
+        beams[level_beam_ix] = -1;
+      } else if (beams[level_beam_ix] == end_token) {
+        finished = true;
+      }
+    }
 #undef GET_IX
   }
 }
@@ -62,8 +75,8 @@ struct GatherTree<GPUDevice, T> {
   void operator()(OpKernelContext* ctx, const GPUDevice& d,
                   typename TTypes<T, 3>::ConstTensor step_ids,
                   typename TTypes<T, 3>::ConstTensor parent_ids,
-                  typename TTypes<T>::ConstMatrix sequence_length,
-                  typename TTypes<T, 3>::Tensor beams) {
+                  TTypes<int32>::ConstVec max_sequence_length,
+                  const T end_token, typename TTypes<T, 3>::Tensor beams) {
     const int32 max_time = parent_ids.dimension(0);
     const int32 batch_size = parent_ids.dimension(1);
     const int32 beam_width = parent_ids.dimension(2);
@@ -75,7 +88,10 @@ struct GatherTree<GPUDevice, T> {
     GatherTreeOpKernel<T>
         <<<config.block_count, config.thread_per_block, 0, d.stream()>>>(
             batch_size, max_time, beam_width,
-            step_ids.data(), parent_ids.data(), sequence_length.data(),
+            step_ids.data(),
+            parent_ids.data(),
+            max_sequence_length.data(),
+            end_token,
             beams.data());
     // clang-format on
   }
diff --git a/tensorflow/contrib/seq2seq/ops/beam_search_ops.cc b/tensorflow/contrib/seq2seq/ops/beam_search_ops.cc
index 6c445cd460..231504bfbb 100644
--- a/tensorflow/contrib/seq2seq/ops/beam_search_ops.cc
+++ b/tensorflow/contrib/seq2seq/ops/beam_search_ops.cc
@@ -25,27 +25,27 @@ using shape_inference::ShapeHandle;
 REGISTER_OP("GatherTree")
     .Input("step_ids: T")
     .Input("parent_ids: T")
-    .Input("sequence_length: T")
+    .Input("max_sequence_lengths: int32")
+    .Input("end_token: T")
     .Output("beams: T")
     .Attr("T: {int32}")
     .SetShapeFn([](InferenceContext* c) {
-      ShapeHandle step_ids, parent_ids, sequence_length;
+      ShapeHandle step_ids, parent_ids, max_sequence_lengths, end_token;
 
       // step_ids, parent_ids, and output are all shaped:
       //   [max_time, batch_size, beam_width].
-      // sequence_length is shaped [batch_size, beam_width].
+      // max_sequence_length is shaped [batch_size] and end_token is a scalar.
       TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 3, &step_ids));
       TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 3, &parent_ids));
-      TF_RETURN_IF_ERROR(c->WithRank(c->input(2), 2, &sequence_length));
-
-      DimensionHandle batch_size = c->Dim(step_ids, 1);
-      DimensionHandle beam_width = c->Dim(step_ids, 2);
-
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(2), 1, &max_sequence_lengths));
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(3), 0, &end_token));
       TF_RETURN_IF_ERROR(c->Merge(step_ids, parent_ids, &step_ids));
+      DimensionHandle batch_size = c->Dim(step_ids, 1);
       TF_RETURN_IF_ERROR(
-          c->Merge(batch_size, c->Dim(sequence_length, 0), &batch_size));
-      TF_RETURN_IF_ERROR(
-          c->Merge(beam_width, c->Dim(sequence_length, 1), &beam_width));
+          c->Merge(batch_size, c->Dim(max_sequence_lengths, 0), &batch_size));
+      ShapeHandle step_ids_prefix = c->Matrix(c->Dim(step_ids, 0), batch_size);
+      TF_RETURN_IF_ERROR(c->MergePrefix(step_ids, step_ids_prefix, &step_ids,
+                                        &step_ids_prefix));
 
       c->set_output(0, step_ids);
       return tensorflow::Status::OK();
@@ -61,7 +61,8 @@ TODO(ebrevdo): fill in
 
 step_ids: `[max_time, batch_size, beam_width]`.
 parent_ids: `[max_time, batch_size, beam_width]`.
-sequence_length: `[batch_size, beam_width]`.
+max_sequence_lengths: `[batch_size]`.
+end_token: `[]`.
 beams: `[max_time, batch_size, beam_width]`.
 )doc");
 
diff --git a/tensorflow/contrib/seq2seq/python/kernel_tests/beam_search_decoder_test.py b/tensorflow/contrib/seq2seq/python/kernel_tests/beam_search_decoder_test.py
index 8d4ec4b4db..d2beac5f31 100644
--- a/tensorflow/contrib/seq2seq/python/kernel_tests/beam_search_decoder_test.py
+++ b/tensorflow/contrib/seq2seq/python/kernel_tests/beam_search_decoder_test.py
@@ -54,15 +54,18 @@ class TestGatherTree(test.TestCase):
          [[0, 0, 0], [1, 2, 0], [2, 1, 1]]],
         dtype=np.int32).transpose([1, 0, 2])
 
-    # sequence_lengths is shaped (batch_size = 2, beam_width = 3)
-    sequence_lengths = [[3, 3, 3], [3, 3, 3]]
+    # sequence_lengths is shaped (batch_size = 3)
+    max_sequence_lengths = [3, 3]
 
     expected_result = np.array(
         [[[2, 2, 2], [6, 5, 6], [7, 8, 9]],
          [[2, 4, 4], [7, 6, 6], [8, 9, 10]]]).transpose([1, 0, 2])
 
     res = beam_search_ops.gather_tree(
-        predicted_ids, parent_ids, sequence_lengths)
+        predicted_ids,
+        parent_ids,
+        max_sequence_lengths=max_sequence_lengths,
+        end_token=11)
 
     with self.test_session() as sess:
       res_ = sess.run(res)
diff --git a/tensorflow/contrib/seq2seq/python/kernel_tests/beam_search_ops_test.py b/tensorflow/contrib/seq2seq/python/kernel_tests/beam_search_ops_test.py
index 50cccf392f..f301314872 100644
--- a/tensorflow/contrib/seq2seq/python/kernel_tests/beam_search_ops_test.py
+++ b/tensorflow/contrib/seq2seq/python/kernel_tests/beam_search_ops_test.py
@@ -19,6 +19,8 @@ from __future__ import division
 from __future__ import print_function
 # pylint: enable=unused-import
 
+import itertools
+
 import numpy as np
 
 from tensorflow.contrib.seq2seq.python.ops import beam_search_ops
@@ -38,12 +40,14 @@ class GatherTreeTest(test.TestCase):
         [[[1, 2, 3], [4, 5, 6], [7, 8, 9], [-1, -1, -1]]])
     parent_ids = _transpose_batch_time(
         [[[0, 0, 0], [0, 1, 1], [2, 1, 2], [-1, -1, -1]]])
-    sequence_length = [[3, 3, 3]]
+    max_sequence_lengths = [3]
     expected_result = _transpose_batch_time(
         [[[2, 2, 2], [6, 5, 6], [7, 8, 9], [-1, -1, -1]]])
     beams = beam_search_ops.gather_tree(
-        step_ids=step_ids, parent_ids=parent_ids,
-        sequence_length=sequence_length)
+        step_ids=step_ids,
+        parent_ids=parent_ids,
+        max_sequence_lengths=max_sequence_lengths,
+        end_token=10)
     with self.test_session(use_gpu=True):
       self.assertAllEqual(expected_result, beams.eval())
 
@@ -54,11 +58,13 @@ class GatherTreeTest(test.TestCase):
         [[[1, 2, 3], [4, 5, 6], [7, 8, 9], [-1, -1, -1]]])
     parent_ids = _transpose_batch_time(
         [[[0, 0, 0], [0, -1, 1], [2, 1, 2], [-1, -1, -1]]])
-    sequence_length = [[3, 3, 3]]
+    max_sequence_lengths = [3]
     with ops.device("/cpu:0"):
       beams = beam_search_ops.gather_tree(
-          step_ids=step_ids, parent_ids=parent_ids,
-          sequence_length=sequence_length)
+          step_ids=step_ids,
+          parent_ids=parent_ids,
+          max_sequence_lengths=max_sequence_lengths,
+          end_token=10)
     with self.test_session():
       with self.assertRaisesOpError(
           r"parent id -1 at \(batch, time, beam\) == \(0, 0, 1\)"):
@@ -75,78 +81,58 @@ class GatherTreeTest(test.TestCase):
         [[[1, 2, 3], [4, 5, 6], [7, 8, 9], [-1, -1, -1]]])
     parent_ids = _transpose_batch_time(
         [[[0, 0, 0], [0, -1, 1], [2, 1, 2], [-1, -1, -1]]])
-    sequence_length = [[3, 3, 3]]
+    max_sequence_lengths = [3]
     expected_result = _transpose_batch_time(
         [[[2, -1, 2], [6, 5, 6], [7, 8, 9], [-1, -1, -1]]])
     with ops.device("/device:GPU:0"):
       beams = beam_search_ops.gather_tree(
-          step_ids=step_ids, parent_ids=parent_ids,
-          sequence_length=sequence_length)
+          step_ids=step_ids,
+          parent_ids=parent_ids,
+          max_sequence_lengths=max_sequence_lengths,
+          end_token=10)
     with self.test_session(use_gpu=True):
       self.assertAllEqual(expected_result, beams.eval())
 
   def testGatherTreeBatch(self):
-    # sequence_length is [batch_size, beam_width] = [4, 5]
-    sequence_length = [[0] * 5, [1] * 5, [2] * 5, [3] * 5]
+    batch_size = 10
+    beam_width = 15
+    max_time = 8
+    max_sequence_lengths = [0, 1, 2, 4, 7, 8, 9, 10, 11, 0]
+    end_token = 5
 
     with self.test_session(use_gpu=True):
-      # (max_time = 4, batch_size = 4, beam_width = 5)
-      step_ids = _transpose_batch_time(
-          [[[3, 4, 0, 4, 0],
-            [4, 2, 0, 3, 1],
-            [1, 1, 3, 2, 2],
-            [3, 1, 2, 3, 4]],
-           [[3, 4, 0, 4, 0],
-            [4, 2, 0, 3, 1],
-            [1, 1, 3, 2, 2],
-            [3, 1, 2, 3, 4]],
-           [[1, 2, 3, 4, 2],
-            [2, 1, 1, 3, 2],
-            [3, 0, 1, 0, 0],
-            [3, 4, 0, 2, 4]],
-           [[0, 2, 2, 3, 1],
-            [3, 2, 2, 2, 3],
-            [3, 4, 3, 0, 3],
-            [1, 2, 2, 2, 4]]])
-      parent_ids = _transpose_batch_time(
-          [[[4, 2, 4, 3, 4],
-            [3, 4, 0, 2, 0],
-            [3, 1, 3, 2, 2],
-            [0, 2, 1, 4, 2]],
-           [[4, 2, 4, 3, 4],
-            [3, 4, 0, 2, 0],
-            [3, 1, 3, 2, 2],
-            [0, 2, 1, 4, 2]],
-           [[3, 0, 0, 4, 0],
-            [1, 2, 4, 2, 2],
-            [4, 4, 0, 3, 0],
-            [2, 4, 4, 3, 0]],
-           [[3, 1, 4, 1, 3],
-            [3, 2, 4, 0, 4],
-            [1, 0, 1, 4, 2],
-            [0, 3, 2, 0, 1]]])
-      expected_beams = _transpose_batch_time(
-          [[[-1, -1, -1, -1, -1],
-            [-1, -1, -1, -1, -1],
-            [-1, -1, -1, -1, -1],
-            [-1, -1, -1, -1, -1]],
-           [[3, 4, 0, 4, 0],
-            [-1, -1, -1, -1, -1],
-            [-1, -1, -1, -1, -1],
-            [-1, -1, -1, -1, -1]],
-           [[2, 3, 2, 3, 3],
-            [2, 1, 1, 3, 2],
-            [-1, -1, -1, -1, -1],
-            [-1, -1, -1, -1, -1]],
-           [[2, 3, 2, 1, 1],
-            [2, 3, 2, 3, 2],
-            [3, 4, 3, 0, 3],
-            [-1, -1, -1, -1, -1]]])
+      step_ids = np.random.randint(
+          0, high=end_token + 1, size=(max_time, batch_size, beam_width))
+      parent_ids = np.random.randint(
+          0, high=beam_width - 1, size=(max_time, batch_size, beam_width))
 
       beams = beam_search_ops.gather_tree(
-          step_ids=step_ids, parent_ids=parent_ids,
-          sequence_length=sequence_length)
-      self.assertAllEqual(expected_beams, beams.eval())
+          step_ids=step_ids.astype(np.int32),
+          parent_ids=parent_ids.astype(np.int32),
+          max_sequence_lengths=max_sequence_lengths,
+          end_token=end_token)
+
+      self.assertEqual((max_time, batch_size, beam_width), beams.shape)
+      beams_value = beams.eval()
+      for b in range(batch_size):
+        # Past max_sequence_lengths[b], we emit all -1s.
+        b_value = beams_value[max_sequence_lengths[b]:, b, :]
+        self.assertAllClose(b_value, -1. * np.ones_like(b_value))
+      for batch, beam in itertools.product(
+          range(batch_size), range(beam_width)):
+        v = np.squeeze(beams_value[:, batch, beam])
+        if end_token in v:
+          found = np.where(v == end_token)[0]
+          # Should be up to 1 instance of end_token per beam.
+          self.assertEqual(len(found), 1)
+          found = found[0]
+          # If an end_token is found, everything before it should be a
+          # valid id and everything after it should be -1.
+          if found > 0:
+            self.assertAllEqual(
+                v[:found - 1] >= 0, np.ones_like(v[:found - 1], dtype=bool))
+          self.assertAllClose(
+              v[found + 1:], -1 * np.ones_like(v[found + 1:]))
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/contrib/seq2seq/python/ops/beam_search_decoder.py b/tensorflow/contrib/seq2seq/python/ops/beam_search_decoder.py
index 112ac57a1b..a88d4f5b8b 100644
--- a/tensorflow/contrib/seq2seq/python/ops/beam_search_decoder.py
+++ b/tensorflow/contrib/seq2seq/python/ops/beam_search_decoder.py
@@ -253,6 +253,20 @@ class BeamSearchDecoder(decoder.Decoder):
           output_shape_with_unknown_batch)
       return nest.map_structure(lambda s: s[1:], layer_output_shape)
 
+  @property
+  def tracks_own_finished(self):
+    """The BeamSearchDecoder shuffles its beams and their finished state.
+
+    For this reason, it conflicts with the `dynamic_decode` function's
+    tracking of finished states.  Setting this property to true avoids
+    early stopping of decoding due to mismanagement of the finished state
+    in `dynamic_decode`.
+
+    Returns:
+      `True`.
+    """
+    return True
+
   @property
   def output_size(self):
     # Return the cell output and the id
@@ -303,15 +317,23 @@ class BeamSearchDecoder(decoder.Decoder):
         output.
       sequence_lengths: An `int64` tensor shaped `[batch_size, beam_width]`.
         The sequence lengths determined for each beam during decode.
+        **NOTE** These are ignored; the updated sequence lengths are stored in
+        `final_state.lengths`.
 
     Returns:
-      outputs: An instance of FinalBeamSearchDecoderOutput where the
+      outputs: An instance of `FinalBeamSearchDecoderOutput` where the
         predicted_ids are the result of calling _gather_tree.
-      final_state: The same input instance of BeamSearchDecoderState.
+      final_state: The same input instance of `BeamSearchDecoderState`.
     """
+    del sequence_lengths
+    # Get max_sequence_length across all beams for each batch.
+    max_sequence_lengths = math_ops.to_int32(
+        math_ops.reduce_max(final_state.lengths, axis=1))
     predicted_ids = beam_search_ops.gather_tree(
-        outputs.predicted_ids, outputs.parent_ids,
-        sequence_length=sequence_lengths)
+        outputs.predicted_ids,
+        outputs.parent_ids,
+        max_sequence_lengths=max_sequence_lengths,
+        end_token=self._end_token)
     outputs = FinalBeamSearchDecoderOutput(
         beam_search_decoder_output=outputs, predicted_ids=predicted_ids)
     return outputs, final_state
@@ -588,10 +610,11 @@ def _beam_search_step(time, logits, next_cell_state, beam_state, batch_size,
                                       name="next_beam_finished")
 
   # Calculate the length of the next predictions.
-  # 1. Finished beams remain unchanged
-  # 2. Beams that are now finished (EOS predicted) remain unchanged
-  # 3. Beams that are not yet finished have their length increased by 1
-  lengths_to_add = math_ops.to_int64(math_ops.logical_not(next_finished))
+  # 1. Finished beams remain unchanged.
+  # 2. Beams that are now finished (EOS predicted) have their length
+  #    increased by 1.
+  # 3. Beams that are not yet finished have their length increased by 1.
+  lengths_to_add = math_ops.to_int64(math_ops.logical_not(previously_finished))
   next_prediction_len = _tensor_gather_helper(
       gather_indices=next_beam_ids,
       gather_from=beam_state.lengths,
diff --git a/tensorflow/contrib/seq2seq/python/ops/decoder.py b/tensorflow/contrib/seq2seq/python/ops/decoder.py
index fbe53fc60a..f14974b9d5 100644
--- a/tensorflow/contrib/seq2seq/python/ops/decoder.py
+++ b/tensorflow/contrib/seq2seq/python/ops/decoder.py
@@ -100,16 +100,36 @@ class Decoder(object):
 
     Returns:
       `(outputs, next_state, next_inputs, finished)`: `outputs` is an object
-      containing the decoder output, `next_state` is a (structure of) state tensors
-      and TensorArrays, `next_inputs` is the tensor that should be used as input for
-      the next step, `finished` is a boolean tensor telling whether the sequence
-      is complete, for each sequence in the batch.
+      containing the decoder output, `next_state` is a (structure of) state
+      tensors and TensorArrays, `next_inputs` is the tensor that should be used
+      as input for the next step, `finished` is a boolean tensor telling whether
+      the sequence is complete, for each sequence in the batch.
     """
     raise NotImplementedError
 
   def finalize(self, outputs, final_state, sequence_lengths):
     raise NotImplementedError
 
+  @property
+  def tracks_own_finished(self):
+    """Describes whether the Decoder keeps track of finished states.
+
+    Most decoders will emit a true/false `finished` value independently
+    at each time step.  In this case, the `dynamic_decode` function keeps track
+    of which batch entries are already finished, and performs a logical OR to
+    insert new batches to the finished set.
+
+    Some decoders, however, shuffle batches / beams between time steps and
+    `dynamic_decode` will mix up the finished state across these entries because
+    it does not track the reshuffle across time steps.  In this case, it is
+    up to the decoder to declare that it will keep track of its own finished
+    state by setting this property to `True`.
+
+    Returns:
+      Python bool.
+    """
+    return False
+
 
 def _create_zero_outputs(size, dtype, batch_size):
   """Create a zero outputs Tensor structure."""
@@ -232,7 +252,10 @@ def dynamic_decode(decoder,
       """
       (next_outputs, decoder_state, next_inputs,
        decoder_finished) = decoder.step(time, inputs, state)
-      next_finished = math_ops.logical_or(decoder_finished, finished)
+      if decoder.tracks_own_finished:
+        next_finished = decoder_finished
+      else:
+        next_finished = math_ops.logical_or(decoder_finished, finished)
       if maximum_iterations is not None:
         next_finished = math_ops.logical_or(
             next_finished, time + 1 >= maximum_iterations)
-- 
GitLab


From a86a589c8b329176bfbb64552405644cb641d99e Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Tue, 17 Oct 2017 09:39:50 -0700
Subject: [PATCH 467/909] Disable flaky cluster tests in opensource.

PiperOrigin-RevId: 172477381
---
 tensorflow/contrib/opt/BUILD         | 3 +++
 tensorflow/python/BUILD              | 6 +++++-
 tensorflow/python/debug/BUILD        | 2 +-
 tensorflow/python/kernel_tests/BUILD | 5 ++++-
 4 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/tensorflow/contrib/opt/BUILD b/tensorflow/contrib/opt/BUILD
index b5a67206f3..8b2b31d5bc 100644
--- a/tensorflow/contrib/opt/BUILD
+++ b/tensorflow/contrib/opt/BUILD
@@ -145,6 +145,9 @@ tf_py_test(
         "//tensorflow/python:training",
         "//tensorflow/python:variables",
     ],
+    tags = [
+        "no_oss",  # Flaky due to port collisions
+    ],
 )
 
 filegroup(
diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 48436fe8cf..f4106ac68c 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -3197,7 +3197,10 @@ cuda_py_test(
         ":variables",
         "//third_party/py/numpy",
     ],
-    tags = ["oss_serial"],
+    tags = [
+        "no_oss",  # Test flaky due to port collisions.
+        "oss_serial",
+    ],
 )
 
 tf_py_test(
@@ -3213,6 +3216,7 @@ tf_py_test(
         ":variables",
     ],
     tags = [
+        "no_oss",  # Test flaky due to port collisions.
         "notsan",  # data race due to b/62910646
         "oss_serial",
     ],
diff --git a/tensorflow/python/debug/BUILD b/tensorflow/python/debug/BUILD
index b68b6e05b6..68b97ddbe3 100644
--- a/tensorflow/python/debug/BUILD
+++ b/tensorflow/python/debug/BUILD
@@ -897,8 +897,8 @@ cuda_py_test(
         "//tensorflow/python:variables",
     ],
     tags = [
+        "no_oss",  # Test flaky due to port collisions.
         "no_windows",
-        "nomac",  # TODO(cais): Install of futures and grpcio on all macs.
         "notsan",
         "oss_serial",
     ],
diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD
index 1380ef5b6a..f6ecd1f0b8 100644
--- a/tensorflow/python/kernel_tests/BUILD
+++ b/tensorflow/python/kernel_tests/BUILD
@@ -3152,7 +3152,10 @@ tf_py_test(
         "//tensorflow/python/data/ops:dataset_ops",
         "//tensorflow/python/data/ops:iterator_ops",
     ],
-    tags = ["no_windows"],
+    tags = [
+        "no_oss",  # Test flaky due to port collisions.
+        "no_windows",
+    ],
 )
 
 filegroup(
-- 
GitLab


From f8b3ced20f7063b3c8efb0e691f28bef845a05f6 Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Tue, 17 Oct 2017 09:43:46 -0700
Subject: [PATCH 468/909] Reworks the imperative_grad interface.

PiperOrigin-RevId: 172477878
---
 tensorflow/python/eager/backprop.py        | 47 ++++------------------
 tensorflow/python/eager/backprop_test.py   | 15 +++----
 tensorflow/python/eager/imperative_grad.py | 19 ++++++++-
 3 files changed, 33 insertions(+), 48 deletions(-)

diff --git a/tensorflow/python/eager/backprop.py b/tensorflow/python/eager/backprop.py
index 1819fba4cb..61c905f31e 100644
--- a/tensorflow/python/eager/backprop.py
+++ b/tensorflow/python/eager/backprop.py
@@ -681,48 +681,17 @@ def _aggregate_grads(gradients):
     return ops.IndexedSlices(values, indices, dense_shape)
 
 
-# If over MIN_AGGREGATE_COUNT gradients are accumulated and the total
-# memory consumption is over MIN_AGGREGATE_BYTES, do an early aggregation
-# so as to release the gradient tensor to save memory.
-_MIN_AGGREGATE_COUNT = 4
-_MIN_AGGREGATE_BYTES = 128 * 1024 * 1024
-
-
-def _add_new_grads(gradients, gradients_size, tid, grad):
-  """Adds a new gradient and maybe aggregate the gradients.
-
-  Args:
-    gradients: A dict map from tensor id to list of gradients.
-    gradients_size: A dict map from tensor id to its total units. Might
-       not be initialized.
-    tid: Tensor id.
-    grad: New gradient for the `tid`, either a Tensor or IndexedSlices.
-
-  Raises:
-    ValueError: if `grad` is neight Tensor nor IndexedSlices.
-  """
-  tensor_grads = gradients[tid]
-  tensor_grads.append(grad)
-  if len(tensor_grads) < _MIN_AGGREGATE_COUNT:
-    return
-  elif tid not in gradients_size:
-    if isinstance(grad, ops.Tensor):
-      size = functools.reduce(operator.mul, grad._shape_tuple(), 1)  # pylint: disable=protected-access
-    elif isinstance(grad, ops.IndexedSlices):
-      size = functools.reduce(operator.mul, grad.values._shape_tuple(), 1)  # pylint: disable=protected-access
-    else:
-      raise ValueError("Unexpected gradient type: %s" % type(grad))
-    gradients_size[tid] = size
-  else:
-    size = gradients_size[tid]
-
-  # For simplicity, assume each element to be 4 bytes now.
-  if len(tensor_grads) * size * 4 > _MIN_AGGREGATE_BYTES:
-    gradients[tid] = [_aggregate_grads(tensor_grads)]
+def _num_elements(grad):
+  """The number of elements in the `grad` tensor."""
+  if isinstance(grad, ops.Tensor):
+    return functools.reduce(operator.mul, grad._shape_tuple(), 1)  # pylint: disable=protected-access
+  if isinstance(grad, ops.IndexedSlices):
+    return functools.reduce(operator.mul, grad.values._shape_tuple(), 1)  # pylint: disable=protected-access
+  raise ValueError("`grad` not a Tensor or IndexedSlices.")
 
 
 _default_vspace = imperative_grad.VSpace(
-    add_new_grads_fn=_add_new_grads,
+    num_elements_fn=_num_elements,
     aggregate_fn=_aggregate_grads,
     tensor_id=ops.tensor_id,
     zeros=array_ops.zeros,
diff --git a/tensorflow/python/eager/backprop_test.py b/tensorflow/python/eager/backprop_test.py
index 9083e3a712..2645d542c0 100644
--- a/tensorflow/python/eager/backprop_test.py
+++ b/tensorflow/python/eager/backprop_test.py
@@ -22,6 +22,7 @@ from tensorflow.python import pywrap_tensorflow
 from tensorflow.python.eager import backprop
 from tensorflow.python.eager import context
 from tensorflow.python.eager import custom_gradient
+from tensorflow.python.eager import imperative_grad
 from tensorflow.python.eager import tape
 from tensorflow.python.eager import test
 from tensorflow.python.framework import constant_op
@@ -442,21 +443,21 @@ class BackpropTest(test.TestCase):
     # Reduce the aggregation limit, cause the backprop to do some
     # early aggregation.
     # pylint: disable=protected-access
-    old_cnt = backprop._MIN_AGGREGATE_COUNT
-    old_bytes = backprop._MIN_AGGREGATE_BYTES
-    backprop._MIN_AGGREGATE_COUNT = 10
-    backprop._MIN_AGGREGATE_BYTES = 1
+    old_cnt = imperative_grad._MIN_AGGREGATE_COUNT
+    old_bytes = imperative_grad._MIN_AGGREGATE_BYTES
+    imperative_grad._MIN_AGGREGATE_COUNT = 10
+    imperative_grad._MIN_AGGREGATE_BYTES = 1
     _ = backprop.implicit_grad(fn)()
     self.assertEqual(len(add_n), 6)
     del add_n[:]
 
     # Aggregation is also limited by the memory.
-    backprop._MIN_AGGREGATE_BYTES = 10000
+    imperative_grad._MIN_AGGREGATE_BYTES = 10000
     _ = backprop.implicit_grad(fn)()
     self.assertEqual(len(add_n), 2)
 
-    backprop._MIN_AGGREGATE_COUNT = old_cnt
-    backprop._MIN_AGGREGATE_BYTES = old_bytes
+    imperative_grad._MIN_AGGREGATE_COUNT = old_cnt
+    imperative_grad._MIN_AGGREGATE_BYTES = old_bytes
     # pylint: enable=protected-access
     context.context().clear_post_execution_callbacks()
 
diff --git a/tensorflow/python/eager/imperative_grad.py b/tensorflow/python/eager/imperative_grad.py
index d30d124040..ce58e661d7 100644
--- a/tensorflow/python/eager/imperative_grad.py
+++ b/tensorflow/python/eager/imperative_grad.py
@@ -120,7 +120,14 @@ def _initial_gradients(vspace, target, output_gradients, tensor_usage_counts):
 
 VSpace = collections.namedtuple(
     "VSpace",
-    ["add_new_grads_fn", "aggregate_fn", "tensor_id", "zeros", "ones_like"])
+    ["aggregate_fn", "num_elements_fn", "tensor_id", "zeros", "ones_like"])
+
+
+# If over MIN_AGGREGATE_COUNT gradients are accumulated and the total
+# memory consumption is over MIN_AGGREGATE_BYTES, do an early aggregation
+# so as to release the gradient tensor to save memory.
+_MIN_AGGREGATE_COUNT = 4
+_MIN_AGGREGATE_BYTES = 128 * 1024 * 1024
 
 
 def imperative_grad(
@@ -193,7 +200,15 @@ def imperative_grad(
     in_gradients = op_trace.backward_function(*(out_gradients))
     for i, t in enumerate(op_trace.input_ids):
       if in_gradients[i] is not None:
-        vspace.add_new_grads_fn(gradients, gradients_size, t, in_gradients[i])
+        t_grads = gradients.setdefault(t, [])
+        t_grads.append(in_gradients[i])
+        if len(t_grads) >= _MIN_AGGREGATE_COUNT:
+          if t not in gradients_size:
+            gradients_size[t] = vspace.num_elements_fn(t_grads[-1])
+          size = gradients_size[t]
+
+          if len(t_grads) * size * 4 > _MIN_AGGREGATE_BYTES:
+            t_grads[:] = [vspace.aggregate_fn(t_grads)]
       if tensor_usage_counts.get(t, 0) > 0:
         tensor_usage_counts[t] -= 1
         if (t in tensor_to_op
-- 
GitLab


From 4cc3cfe15f8ed29323c669c4cfff7c754fb59136 Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Tue, 17 Oct 2017 10:08:16 -0700
Subject: [PATCH 469/909] Create a macos py3 test script.

PiperOrigin-RevId: 172480793
---
 .../tools/ci_build/osx/cpu/run_py3_cc_core.sh | 37 +++++++++++++++++++
 1 file changed, 37 insertions(+)
 create mode 100755 tensorflow/tools/ci_build/osx/cpu/run_py3_cc_core.sh

diff --git a/tensorflow/tools/ci_build/osx/cpu/run_py3_cc_core.sh b/tensorflow/tools/ci_build/osx/cpu/run_py3_cc_core.sh
new file mode 100755
index 0000000000..8f839ca110
--- /dev/null
+++ b/tensorflow/tools/ci_build/osx/cpu/run_py3_cc_core.sh
@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# ==============================================================================
+
+set -e
+set -x
+
+N_JOBS=$(sysctl -n hw.ncpu)
+N_JOBS=$((N_JOBS+1))
+
+echo ""
+echo "Bazel will use ${N_JOBS} concurrent job(s)."
+echo ""
+
+# Run configure.
+export TF_NEED_CUDA=0
+export PYTHON_BIN_PATH=$(which python3)
+yes "" | $PYTHON_BIN_PATH configure.py
+which bazel
+bazel test --test_tag_filters=-no_oss,-gpu,-benchmark-test,-nomac \
+    --test_timeout 300,450,1200,3600 \
+    --test_size_filters=small,medium \
+    --jobs=${N_JOBS} --build_tests_only --test_output=errors -k -- \
+    //tensorflow/... -//tensorflow/compiler/... -//tensorflow/contrib/...
-- 
GitLab


From de2a766ba6314ca82635d8bfa00bf1075cd68d64 Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Tue, 17 Oct 2017 10:13:31 -0700
Subject: [PATCH 470/909] Disable flaky gru_ops_test in opensource.

PiperOrigin-RevId: 172481341
---
 tensorflow/contrib/rnn/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/contrib/rnn/BUILD b/tensorflow/contrib/rnn/BUILD
index 6395cd8316..571d299ad9 100644
--- a/tensorflow/contrib/rnn/BUILD
+++ b/tensorflow/contrib/rnn/BUILD
@@ -277,6 +277,7 @@ cuda_py_tests(
         "//tensorflow/python:variable_scope",
         "//tensorflow/python:variables",
     ],
+    tags = ["no_oss"],
 )
 
 tf_cc_test(
-- 
GitLab


From c4fd2362bc73b48add49d7af0ddd9ea7b6409323 Mon Sep 17 00:00:00 2001
From: Kiril Gorovoy <kgorovoy@google.com>
Date: Tue, 17 Oct 2017 11:18:59 -0700
Subject: [PATCH 471/909] Fix TF workspace issue that prevents submodules from
 using aws build targets correctly. Fixes
 https://github.com/tensorflow/serving/issues/615

PiperOrigin-RevId: 172489253
---
 tensorflow/workspace.bzl | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 6151dc6241..54559edbea 100644
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -661,6 +661,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       sha256 = "f599b57aec4f03ad696044dd430b2d201864113937353adc346f53ad47991319",
       strip_prefix = "aws-sdk-cpp-1.0.90",
       build_file = str(Label("//third_party:aws.BUILD")),
+      repository = tf_repo_name
   )
 
   java_import_external(
-- 
GitLab


From 476ef197b86f1ab42a73ac4ab50080953578a161 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 17 Oct 2017 11:53:00 -0700
Subject: [PATCH 472/909] Automated g4 rollback of changelist 172051437

PiperOrigin-RevId: 172493077
---
 tensorflow/cc/saved_model/BUILD               |    1 -
 tensorflow/cc/saved_model/loader.cc           |   16 -
 tensorflow/cc/saved_model/loader_test.cc      |   24 +-
 .../00000123/assets/foo.txt                   |    1 -
 .../00000123/saved_model.pbtxt                | 2728 -----------------
 .../variables/variables.data-00000-of-00001   |  Bin 12 -> 0 bytes
 .../00000123/variables/variables.index        |  Bin 151 -> 0 bytes
 7 files changed, 2 insertions(+), 2768 deletions(-)
 delete mode 100644 tensorflow/cc/saved_model/testdata/half_plus_two_forward_compatibility/00000123/assets/foo.txt
 delete mode 100755 tensorflow/cc/saved_model/testdata/half_plus_two_forward_compatibility/00000123/saved_model.pbtxt
 delete mode 100755 tensorflow/cc/saved_model/testdata/half_plus_two_forward_compatibility/00000123/variables/variables.data-00000-of-00001
 delete mode 100755 tensorflow/cc/saved_model/testdata/half_plus_two_forward_compatibility/00000123/variables/variables.index

diff --git a/tensorflow/cc/saved_model/BUILD b/tensorflow/cc/saved_model/BUILD
index 67b2e4b81a..d29ad3ebcb 100644
--- a/tensorflow/cc/saved_model/BUILD
+++ b/tensorflow/cc/saved_model/BUILD
@@ -89,7 +89,6 @@ tf_cc_test(
 filegroup(
     name = "saved_model_half_plus_two",
     srcs = glob([
-        "testdata/half_plus_two_forward_compatibility/**",
         "testdata/half_plus_two_pbtxt/**",
         "testdata/half_plus_two_main_op/**",
         "testdata/half_plus_two/**",
diff --git a/tensorflow/cc/saved_model/loader.cc b/tensorflow/cc/saved_model/loader.cc
index 462308a48f..f98abc8a81 100644
--- a/tensorflow/cc/saved_model/loader.cc
+++ b/tensorflow/cc/saved_model/loader.cc
@@ -18,7 +18,6 @@ limitations under the License.
 #include <unordered_set>
 
 #include "tensorflow/cc/saved_model/constants.h"
-#include "tensorflow/core/framework/graph_def_util.h"
 #include "tensorflow/core/lib/io/path.h"
 #include "tensorflow/core/lib/monitoring/counter.h"
 #include "tensorflow/core/lib/strings/strcat.h"
@@ -225,18 +224,6 @@ Status GetAssetFileDefs(const MetaGraphDef& meta_graph_def,
   return Status::OK();
 }
 
-// For forward compatibility, remove new default attributes from the graph def
-// that were not present in the consumer (e.g. If graph was exported using
-// code that's newer than the server and a new default attr was added).
-Status RemoveNewDefaultAttrsFromMetaGraphDef(MetaGraphDef* meta_graph_def) {
-  OpListOpRegistry producer_op_registry(
-      &meta_graph_def->meta_info_def().stripped_op_list());
-  OpRegistry* consumer_op_registry = OpRegistry::Global();
-  return RemoveNewDefaultAttrsFromGraphDef(meta_graph_def->mutable_graph_def(),
-                                           *consumer_op_registry,
-                                           producer_op_registry, nullptr);
-}
-
 Status LoadSavedModelInternal(const SessionOptions& session_options,
                               const RunOptions& run_options,
                               const string& export_dir,
@@ -254,9 +241,6 @@ Status LoadSavedModelInternal(const SessionOptions& session_options,
   TF_RETURN_IF_ERROR(
       FindMetaGraphDefToLoad(saved_model_proto, tags, &bundle->meta_graph_def));
 
-  TF_RETURN_IF_ERROR(
-      RemoveNewDefaultAttrsFromMetaGraphDef(&bundle->meta_graph_def));
-
   TF_RETURN_IF_ERROR(LoadMetaGraphIntoSession(
       bundle->meta_graph_def, session_options, &bundle->session));
 
diff --git a/tensorflow/cc/saved_model/loader_test.cc b/tensorflow/cc/saved_model/loader_test.cc
index 6dd14837b5..0ad6b33bba 100644
--- a/tensorflow/cc/saved_model/loader_test.cc
+++ b/tensorflow/cc/saved_model/loader_test.cc
@@ -29,12 +29,10 @@ limitations under the License.
 namespace tensorflow {
 namespace {
 
-constexpr char kTestDataForwardCompatibility[] =
-    "cc/saved_model/testdata/half_plus_two_forward_compatibility/00000123";
-constexpr char kTestDataMainOp[] =
-    "cc/saved_model/testdata/half_plus_two_main_op/00000123";
 constexpr char kTestDataPbTxt[] =
     "cc/saved_model/testdata/half_plus_two_pbtxt/00000123";
+constexpr char kTestDataMainOp[] =
+    "cc/saved_model/testdata/half_plus_two_main_op/00000123";
 constexpr char kTestDataSharded[] =
     "cc/saved_model/testdata/half_plus_two/00000123";
 
@@ -169,24 +167,6 @@ TEST_F(LoaderTest, PbtxtFormat) {
   CheckSavedModelBundle(export_dir, bundle);
 }
 
-// Forward compatibility graph has a new attr with a default value equal to the
-// value used by the server. If we handle new default attrs correctly, this test
-// will pass. This simulates adding new atts to the training code while server
-// code lags behind.
-TEST_F(LoaderTest, ForwardCompatibility) {
-  SavedModelBundle bundle;
-  SessionOptions session_options;
-  RunOptions run_options;
-
-  // TODO(b/67753689): Add support for regenerating this model in the export
-  // code.
-  const string export_dir =
-      io::JoinPath(testing::TensorFlowSrcRoot(), kTestDataForwardCompatibility);
-  TF_ASSERT_OK(LoadSavedModel(session_options, run_options, export_dir,
-                              {kSavedModelTagServe}, &bundle));
-  CheckSavedModelBundle(export_dir, bundle);
-}
-
 TEST_F(LoaderTest, MainOpFormat) {
   SavedModelBundle bundle;
   SessionOptions session_options;
diff --git a/tensorflow/cc/saved_model/testdata/half_plus_two_forward_compatibility/00000123/assets/foo.txt b/tensorflow/cc/saved_model/testdata/half_plus_two_forward_compatibility/00000123/assets/foo.txt
deleted file mode 100644
index f9ff036688..0000000000
--- a/tensorflow/cc/saved_model/testdata/half_plus_two_forward_compatibility/00000123/assets/foo.txt
+++ /dev/null
@@ -1 +0,0 @@
-asset-file-contents
\ No newline at end of file
diff --git a/tensorflow/cc/saved_model/testdata/half_plus_two_forward_compatibility/00000123/saved_model.pbtxt b/tensorflow/cc/saved_model/testdata/half_plus_two_forward_compatibility/00000123/saved_model.pbtxt
deleted file mode 100755
index e799b3579c..0000000000
--- a/tensorflow/cc/saved_model/testdata/half_plus_two_forward_compatibility/00000123/saved_model.pbtxt
+++ /dev/null
@@ -1,2728 +0,0 @@
-saved_model_schema_version: 1
-meta_graphs {
-  meta_info_def {
-    stripped_op_list {
-      op {
-        name: "Add"
-        input_arg {
-          name: "x"
-          type_attr: "T"
-        }
-        input_arg {
-          name: "y"
-          type_attr: "T"
-        }
-        output_arg {
-          name: "z"
-          type_attr: "T"
-        }
-        attr {
-          name: "T"
-          type: "type"
-          allowed_values {
-            list {
-              type: DT_HALF
-              type: DT_FLOAT
-              type: DT_DOUBLE
-              type: DT_UINT8
-              type: DT_INT8
-              type: DT_INT16
-              type: DT_INT32
-              type: DT_INT64
-              type: DT_COMPLEX64
-              type: DT_COMPLEX128
-              type: DT_STRING
-            }
-          }
-        }
-      }
-      op {
-        name: "Assign"
-        input_arg {
-          name: "ref"
-          type_attr: "T"
-          is_ref: true
-        }
-        input_arg {
-          name: "value"
-          type_attr: "T"
-        }
-        output_arg {
-          name: "output_ref"
-          type_attr: "T"
-          is_ref: true
-        }
-        attr {
-          name: "T"
-          type: "type"
-        }
-        attr {
-          name: "validate_shape"
-          type: "bool"
-          default_value {
-            b: true
-          }
-        }
-        attr {
-          name: "use_locking"
-          type: "bool"
-          default_value {
-            b: true
-          }
-        }
-        allows_uninitialized_input: true
-      }
-      op {
-        name: "Const"
-        output_arg {
-          name: "output"
-          type_attr: "dtype"
-        }
-        attr {
-          name: "value"
-          type: "tensor"
-        }
-        attr {
-          name: "dtype"
-          type: "type"
-        }
-      }
-      op {
-        name: "Identity"
-        input_arg {
-          name: "input"
-          type_attr: "T"
-        }
-        output_arg {
-          name: "output"
-          type_attr: "T"
-        }
-        attr {
-          name: "T"
-          type: "type"
-        }
-      }
-      op {
-        name: "MergeV2Checkpoints"
-        input_arg {
-          name: "checkpoint_prefixes"
-          type: DT_STRING
-        }
-        input_arg {
-          name: "destination_prefix"
-          type: DT_STRING
-        }
-        attr {
-          name: "delete_old_dirs"
-          type: "bool"
-          default_value {
-            b: true
-          }
-        }
-      }
-      op {
-        name: "Mul"
-        input_arg {
-          name: "x"
-          type_attr: "T"
-        }
-        input_arg {
-          name: "y"
-          type_attr: "T"
-        }
-        output_arg {
-          name: "z"
-          type_attr: "T"
-        }
-        attr {
-          name: "T"
-          type: "type"
-          allowed_values {
-            list {
-              type: DT_HALF
-              type: DT_FLOAT
-              type: DT_DOUBLE
-              type: DT_UINT8
-              type: DT_INT8
-              type: DT_UINT16
-              type: DT_INT16
-              type: DT_INT32
-              type: DT_INT64
-              type: DT_COMPLEX64
-              type: DT_COMPLEX128
-            }
-          }
-        }
-        is_commutative: true
-      }
-      op {
-        name: "NoOp"
-      }
-      op {
-        name: "Pack"
-        input_arg {
-          name: "values"
-          type_attr: "T"
-          number_attr: "N"
-        }
-        output_arg {
-          name: "output"
-          type_attr: "T"
-        }
-        attr {
-          name: "N"
-          type: "int"
-          has_minimum: true
-          minimum: 1
-        }
-        attr {
-          name: "T"
-          type: "type"
-        }
-        attr {
-          name: "axis"
-          type: "int"
-          default_value {
-            i: 0
-          }
-        }
-      }
-      op {
-        name: "ParseExample"
-        input_arg {
-          name: "serialized"
-          type_attr: "TInputs"
-        }
-        input_arg {
-          name: "names"
-          type: DT_STRING
-        }
-        input_arg {
-          name: "sparse_keys"
-          type: DT_STRING
-          number_attr: "Nsparse"
-        }
-        input_arg {
-          name: "dense_keys"
-          type: DT_STRING
-          number_attr: "Ndense"
-        }
-        input_arg {
-          name: "dense_defaults"
-          type_list_attr: "Tdense"
-        }
-        output_arg {
-          name: "sparse_indices"
-          type: DT_INT64
-          number_attr: "Nsparse"
-        }
-        output_arg {
-          name: "sparse_values"
-          type_list_attr: "sparse_types"
-        }
-        output_arg {
-          name: "sparse_shapes"
-          type: DT_INT64
-          number_attr: "Nsparse"
-        }
-        output_arg {
-          name: "dense_values"
-          type_list_attr: "Tdense"
-        }
-        attr {
-          name: "Nsparse"
-          type: "int"
-          has_minimum: true
-        }
-        attr {
-          name: "TInputs"
-          type: "type"
-          default_value {
-            type: DT_STRING
-          }
-          allowed_values {
-            list {
-              type: DT_STRING
-              type: DT_INT64
-            }
-          }
-        }
-        attr {
-          name: "Ndense"
-          type: "int"
-          has_minimum: true
-        }
-        attr {
-          name: "sparse_types"
-          type: "list(type)"
-          has_minimum: true
-          allowed_values {
-            list {
-              type: DT_FLOAT
-              type: DT_INT64
-              type: DT_STRING
-            }
-          }
-        }
-        attr {
-          name: "Tdense"
-          type: "list(type)"
-          has_minimum: true
-          allowed_values {
-            list {
-              type: DT_FLOAT
-              type: DT_INT64
-              type: DT_STRING
-            }
-          }
-        }
-        attr {
-          name: "dense_shapes"
-          type: "list(shape)"
-          has_minimum: true
-        }
-      }
-      op {
-        name: "Placeholder"
-        output_arg {
-          name: "output"
-          type_attr: "dtype"
-        }
-        attr {
-          name: "dtype"
-          type: "type"
-        }
-        attr {
-          name: "shape"
-          type: "shape"
-          default_value {
-            shape {
-              unknown_rank: true
-            }
-          }
-        }
-      }
-      op {
-        name: "Reshape"
-        input_arg {
-          name: "tensor"
-          type_attr: "T"
-        }
-        input_arg {
-          name: "shape"
-          type_attr: "Tshape"
-        }
-        output_arg {
-          name: "output"
-          type_attr: "T"
-        }
-        attr {
-          name: "T"
-          type: "type"
-        }
-        attr {
-          name: "Tshape"
-          type: "type"
-          default_value {
-            type: DT_INT32
-          }
-          allowed_values {
-            list {
-              type: DT_INT32
-              type: DT_INT64
-            }
-          }
-        }
-      }
-      op {
-        name: "RestoreV2"
-        input_arg {
-          name: "prefix"
-          type: DT_STRING
-        }
-        input_arg {
-          name: "tensor_names"
-          type: DT_STRING
-        }
-        input_arg {
-          name: "shape_and_slices"
-          type: DT_STRING
-        }
-        output_arg {
-          name: "tensors"
-          type_list_attr: "dtypes"
-        }
-        attr {
-          name: "dtypes"
-          type: "list(type)"
-          has_minimum: true
-          minimum: 1
-        }
-      }
-      op {
-        name: "SaveV2"
-        input_arg {
-          name: "prefix"
-          type: DT_STRING
-        }
-        input_arg {
-          name: "tensor_names"
-          type: DT_STRING
-        }
-        input_arg {
-          name: "shape_and_slices"
-          type: DT_STRING
-        }
-        input_arg {
-          name: "tensors"
-          type_list_attr: "dtypes"
-        }
-        attr {
-          name: "dtypes"
-          type: "list(type)"
-          has_minimum: true
-          minimum: 1
-        }
-      }
-      op {
-        name: "ShardedFilename"
-        input_arg {
-          name: "basename"
-          type: DT_STRING
-        }
-        input_arg {
-          name: "shard"
-          type: DT_INT32
-        }
-        input_arg {
-          name: "num_shards"
-          type: DT_INT32
-        }
-        output_arg {
-          name: "filename"
-          type: DT_STRING
-        }
-      }
-      op {
-        name: "StringJoin"
-        input_arg {
-          name: "inputs"
-          type: DT_STRING
-          number_attr: "N"
-        }
-        output_arg {
-          name: "output"
-          type: DT_STRING
-        }
-        attr {
-          name: "N"
-          type: "int"
-          has_minimum: true
-          minimum: 1
-        }
-        attr {
-          name: "separator"
-          type: "string"
-          default_value {
-            s: ""
-          }
-        }
-      }
-      op {
-        name: "VariableV2"
-        output_arg {
-          name: "ref"
-          type_attr: "dtype"
-          is_ref: true
-        }
-        attr {
-          name: "shape"
-          type: "shape"
-        }
-        attr {
-          name: "dtype"
-          type: "type"
-        }
-        attr {
-          name: "container"
-          type: "string"
-          default_value {
-            s: ""
-          }
-        }
-        attr {
-          name: "shared_name"
-          type: "string"
-          default_value {
-            s: ""
-          }
-        }
-        is_stateful: true
-      }
-    }
-    tags: "serve"
-    tensorflow_version: "1.1.0-rc2"
-    tensorflow_git_version: "unknown"
-  }
-  graph_def {
-    node {
-      name: "a/initial_value"
-      op: "Const"
-      attr {
-        key: "_output_shapes"
-        value {
-          list {
-            shape {
-            }
-          }
-        }
-      }
-      attr {
-        key: "dtype"
-        value {
-          type: DT_FLOAT
-        }
-      }
-      attr {
-        key: "value"
-        value {
-          tensor {
-            dtype: DT_FLOAT
-            tensor_shape {
-            }
-            float_val: 0.5
-          }
-        }
-      }
-    }
-    node {
-      name: "a"
-      op: "VariableV2"
-      attr {
-        key: "_output_shapes"
-        value {
-          list {
-            shape {
-            }
-          }
-        }
-      }
-      attr {
-        key: "container"
-        value {
-          s: ""
-        }
-      }
-      attr {
-        key: "dtype"
-        value {
-          type: DT_FLOAT
-        }
-      }
-      attr {
-        key: "shape"
-        value {
-          shape {
-          }
-        }
-      }
-      attr {
-        key: "shared_name"
-        value {
-          s: ""
-        }
-      }
-    }
-    node {
-      name: "a/Assign"
-      op: "Assign"
-      input: "a"
-      input: "a/initial_value"
-      attr {
-        key: "T"
-        value {
-          type: DT_FLOAT
-        }
-      }
-      attr {
-        key: "_class"
-        value {
-          list {
-            s: "loc:@a"
-          }
-        }
-      }
-      attr {
-        key: "_output_shapes"
-        value {
-          list {
-            shape {
-            }
-          }
-        }
-      }
-      attr {
-        key: "use_locking"
-        value {
-          b: true
-        }
-      }
-      attr {
-        key: "validate_shape"
-        value {
-          b: true
-        }
-      }
-    }
-    node {
-      name: "a/read"
-      op: "Identity"
-      input: "a"
-      attr {
-        key: "T"
-        value {
-          type: DT_FLOAT
-        }
-      }
-      attr {
-        key: "_class"
-        value {
-          list {
-            s: "loc:@a"
-          }
-        }
-      }
-      attr {
-        key: "_output_shapes"
-        value {
-          list {
-            shape {
-            }
-          }
-        }
-      }
-    }
-    node {
-      name: "b/initial_value"
-      op: "Const"
-      attr {
-        key: "_output_shapes"
-        value {
-          list {
-            shape {
-            }
-          }
-        }
-      }
-      attr {
-        key: "dtype"
-        value {
-          type: DT_FLOAT
-        }
-      }
-      attr {
-        key: "value"
-        value {
-          tensor {
-            dtype: DT_FLOAT
-            tensor_shape {
-            }
-            float_val: 2.0
-          }
-        }
-      }
-    }
-    node {
-      name: "b"
-      op: "VariableV2"
-      attr {
-        key: "_output_shapes"
-        value {
-          list {
-            shape {
-            }
-          }
-        }
-      }
-      attr {
-        key: "container"
-        value {
-          s: ""
-        }
-      }
-      attr {
-        key: "dtype"
-        value {
-          type: DT_FLOAT
-        }
-      }
-      attr {
-        key: "shape"
-        value {
-          shape {
-          }
-        }
-      }
-      attr {
-        key: "shared_name"
-        value {
-          s: ""
-        }
-      }
-    }
-    node {
-      name: "b/Assign"
-      op: "Assign"
-      input: "b"
-      input: "b/initial_value"
-      attr {
-        key: "T"
-        value {
-          type: DT_FLOAT
-        }
-      }
-      attr {
-        key: "_class"
-        value {
-          list {
-            s: "loc:@b"
-          }
-        }
-      }
-      attr {
-        key: "_output_shapes"
-        value {
-          list {
-            shape {
-            }
-          }
-        }
-      }
-      attr {
-        key: "use_locking"
-        value {
-          b: true
-        }
-      }
-      attr {
-        key: "validate_shape"
-        value {
-          b: true
-        }
-      }
-    }
-    node {
-      name: "b/read"
-      op: "Identity"
-      input: "b"
-      attr {
-        key: "T"
-        value {
-          type: DT_FLOAT
-        }
-      }
-      attr {
-        key: "_class"
-        value {
-          list {
-            s: "loc:@b"
-          }
-        }
-      }
-      attr {
-        key: "_output_shapes"
-        value {
-          list {
-            shape {
-            }
-          }
-        }
-      }
-    }
-    node {
-      name: "c/initial_value"
-      op: "Const"
-      attr {
-        key: "_output_shapes"
-        value {
-          list {
-            shape {
-            }
-          }
-        }
-      }
-      attr {
-        key: "dtype"
-        value {
-          type: DT_FLOAT
-        }
-      }
-      attr {
-        key: "value"
-        value {
-          tensor {
-            dtype: DT_FLOAT
-            tensor_shape {
-            }
-            float_val: 3.0
-          }
-        }
-      }
-    }
-    node {
-      name: "c"
-      op: "VariableV2"
-      attr {
-        key: "_output_shapes"
-        value {
-          list {
-            shape {
-            }
-          }
-        }
-      }
-      attr {
-        key: "container"
-        value {
-          s: ""
-        }
-      }
-      attr {
-        key: "dtype"
-        value {
-          type: DT_FLOAT
-        }
-      }
-      attr {
-        key: "shape"
-        value {
-          shape {
-          }
-        }
-      }
-      attr {
-        key: "shared_name"
-        value {
-          s: ""
-        }
-      }
-    }
-    node {
-      name: "c/Assign"
-      op: "Assign"
-      input: "c"
-      input: "c/initial_value"
-      attr {
-        key: "T"
-        value {
-          type: DT_FLOAT
-        }
-      }
-      attr {
-        key: "_class"
-        value {
-          list {
-            s: "loc:@c"
-          }
-        }
-      }
-      attr {
-        key: "_output_shapes"
-        value {
-          list {
-            shape {
-            }
-          }
-        }
-      }
-      attr {
-        key: "use_locking"
-        value {
-          b: true
-        }
-      }
-      attr {
-        key: "validate_shape"
-        value {
-          b: true
-        }
-      }
-    }
-    node {
-      name: "c/read"
-      op: "Identity"
-      input: "c"
-      attr {
-        key: "T"
-        value {
-          type: DT_FLOAT
-        }
-      }
-      attr {
-        key: "_class"
-        value {
-          list {
-            s: "loc:@c"
-          }
-        }
-      }
-      attr {
-        key: "_output_shapes"
-        value {
-          list {
-            shape {
-            }
-          }
-        }
-      }
-    }
-    node {
-      name: "tf_example"
-      op: "Placeholder"
-      attr {
-        key: "_output_shapes"
-        value {
-          list {
-            shape {
-              unknown_rank: true
-            }
-          }
-        }
-      }
-      attr {
-        key: "dtype"
-        value {
-          type: DT_STRING
-        }
-      }
-      attr {
-        key: "shape"
-        value {
-          shape {
-            unknown_rank: true
-          }
-        }
-      }
-    }
-    node {
-      name: "ParseExample/Const"
-      op: "Const"
-      attr {
-        key: "_output_shapes"
-        value {
-          list {
-            shape {
-              dim {
-              }
-            }
-          }
-        }
-      }
-      attr {
-        key: "dtype"
-        value {
-          type: DT_FLOAT
-        }
-      }
-      attr {
-        key: "value"
-        value {
-          tensor {
-            dtype: DT_FLOAT
-            tensor_shape {
-              dim {
-              }
-            }
-          }
-        }
-      }
-    }
-    node {
-      name: "ParseExample/key_x2"
-      op: "Const"
-      attr {
-        key: "_output_shapes"
-        value {
-          list {
-            shape {
-              dim {
-                size: 1
-              }
-            }
-          }
-        }
-      }
-      attr {
-        key: "dtype"
-        value {
-          type: DT_FLOAT
-        }
-      }
-      attr {
-        key: "value"
-        value {
-          tensor {
-            dtype: DT_FLOAT
-            tensor_shape {
-              dim {
-                size: 1
-              }
-            }
-            float_val: 0.0
-          }
-        }
-      }
-    }
-    node {
-      name: "ParseExample/Reshape/shape"
-      op: "Const"
-      attr {
-        key: "_output_shapes"
-        value {
-          list {
-            shape {
-              dim {
-                size: 1
-              }
-            }
-          }
-        }
-      }
-      attr {
-        key: "dtype"
-        value {
-          type: DT_INT32
-        }
-      }
-      attr {
-        key: "value"
-        value {
-          tensor {
-            dtype: DT_INT32
-            tensor_shape {
-              dim {
-                size: 1
-              }
-            }
-            int_val: 1
-          }
-        }
-      }
-    }
-    node {
-      name: "ParseExample/Reshape"
-      op: "Reshape"
-      input: "ParseExample/key_x2"
-      input: "ParseExample/Reshape/shape"
-      attr {
-        key: "T"
-        value {
-          type: DT_FLOAT
-        }
-      }
-      attr {
-        key: "Tshape"
-        value {
-          type: DT_INT32
-        }
-      }
-      attr {
-        key: "_output_shapes"
-        value {
-          list {
-            shape {
-              dim {
-                size: 1
-              }
-            }
-          }
-        }
-      }
-    }
-    node {
-      name: "ParseExample/ParseExample/names"
-      op: "Const"
-      attr {
-        key: "_output_shapes"
-        value {
-          list {
-            shape {
-              dim {
-              }
-            }
-          }
-        }
-      }
-      attr {
-        key: "dtype"
-        value {
-          type: DT_STRING
-        }
-      }
-      attr {
-        key: "value"
-        value {
-          tensor {
-            dtype: DT_STRING
-            tensor_shape {
-              dim {
-              }
-            }
-          }
-        }
-      }
-    }
-    node {
-      name: "ParseExample/ParseExample/dense_keys_0"
-      op: "Const"
-      attr {
-        key: "_output_shapes"
-        value {
-          list {
-            shape {
-            }
-          }
-        }
-      }
-      attr {
-        key: "dtype"
-        value {
-          type: DT_STRING
-        }
-      }
-      attr {
-        key: "value"
-        value {
-          tensor {
-            dtype: DT_STRING
-            tensor_shape {
-            }
-            string_val: "x"
-          }
-        }
-      }
-    }
-    node {
-      name: "ParseExample/ParseExample/dense_keys_1"
-      op: "Const"
-      attr {
-        key: "_output_shapes"
-        value {
-          list {
-            shape {
-            }
-          }
-        }
-      }
-      attr {
-        key: "dtype"
-        value {
-          type: DT_STRING
-        }
-      }
-      attr {
-        key: "value"
-        value {
-          tensor {
-            dtype: DT_STRING
-            tensor_shape {
-            }
-            string_val: "x2"
-          }
-        }
-      }
-    }
-    node {
-      name: "ParseExample/ParseExample"
-      op: "ParseExample"
-      input: "tf_example"
-      input: "ParseExample/ParseExample/names"
-      input: "ParseExample/ParseExample/dense_keys_0"
-      input: "ParseExample/ParseExample/dense_keys_1"
-      input: "ParseExample/Const"
-      input: "ParseExample/Reshape"
-      attr {
-        key: "Ndense"
-        value {
-          i: 2
-        }
-      }
-      attr {
-        key: "TInputs"
-        value {
-          type: DT_STRING
-        }
-      }
-      attr {
-        key: "Nsparse"
-        value {
-          i: 0
-        }
-      }
-      attr {
-        key: "Tdense"
-        value {
-          list {
-            type: DT_FLOAT
-            type: DT_FLOAT
-          }
-        }
-      }
-      attr {
-        key: "_output_shapes"
-        value {
-          list {
-            shape {
-              dim {
-                size: -1
-              }
-              dim {
-                size: 1
-              }
-            }
-            shape {
-              dim {
-                size: -1
-              }
-              dim {
-                size: 1
-              }
-            }
-          }
-        }
-      }
-      attr {
-        key: "dense_shapes"
-        value {
-          list {
-            shape {
-              dim {
-                size: 1
-              }
-            }
-            shape {
-              dim {
-                size: 1
-              }
-            }
-          }
-        }
-      }
-      attr {
-        key: "sparse_types"
-        value {
-          list {
-          }
-        }
-      }
-    }
-    node {
-      name: "x"
-      op: "Identity"
-      input: "ParseExample/ParseExample"
-      attr {
-        key: "T"
-        value {
-          type: DT_FLOAT
-        }
-      }
-      attr {
-        key: "_output_shapes"
-        value {
-          list {
-            shape {
-              dim {
-                size: -1
-              }
-              dim {
-                size: 1
-              }
-            }
-          }
-        }
-      }
-    }
-    node {
-      name: "Mul"
-      op: "Mul"
-      input: "a/read"
-      input: "x"
-      attr {
-        key: "T"
-        value {
-          type: DT_FLOAT
-        }
-      }
-      attr {
-        key: "_output_shapes"
-        value {
-          list {
-            shape {
-              dim {
-                size: -1
-              }
-              dim {
-                size: 1
-              }
-            }
-          }
-        }
-      }
-    }
-    node {
-      name: "y"
-      op: "Add"
-      input: "Mul"
-      input: "b/read"
-      attr {
-        key: "T"
-        value {
-          type: DT_FLOAT
-        }
-      }
-      attr {
-        key: "_output_shapes"
-        value {
-          list {
-            shape {
-              dim {
-                size: -1
-              }
-              dim {
-                size: 1
-              }
-            }
-          }
-        }
-      }
-    }
-    node {
-      name: "Mul_1"
-      op: "Mul"
-      input: "a/read"
-      input: "x"
-      attr {
-        key: "T"
-        value {
-          type: DT_FLOAT
-        }
-      }
-      attr {
-        key: "_output_shapes"
-        value {
-          list {
-            shape {
-              dim {
-                size: -1
-              }
-              dim {
-                size: 1
-              }
-            }
-          }
-        }
-      }
-    }
-    node {
-      name: "y2"
-      op: "Add"
-      input: "Mul_1"
-      input: "c/read"
-      attr {
-        key: "T"
-        value {
-          type: DT_FLOAT
-        }
-      }
-      attr {
-        key: "_output_shapes"
-        value {
-          list {
-            shape {
-              dim {
-                size: -1
-              }
-              dim {
-                size: 1
-              }
-            }
-          }
-        }
-      }
-    }
-    node {
-      name: "x2"
-      op: "Identity"
-      input: "ParseExample/ParseExample:1"
-      attr {
-        key: "T"
-        value {
-          type: DT_FLOAT
-        }
-      }
-      attr {
-        key: "_output_shapes"
-        value {
-          list {
-            shape {
-              dim {
-                size: -1
-              }
-              dim {
-                size: 1
-              }
-            }
-          }
-        }
-      }
-    }
-    node {
-      name: "Mul_2"
-      op: "Mul"
-      input: "a/read"
-      input: "x2"
-      attr {
-        key: "T"
-        value {
-          type: DT_FLOAT
-        }
-      }
-      attr {
-        key: "_output_shapes"
-        value {
-          list {
-            shape {
-              dim {
-                size: -1
-              }
-              dim {
-                size: 1
-              }
-            }
-          }
-        }
-      }
-    }
-    node {
-      name: "y3"
-      op: "Add"
-      input: "Mul_2"
-      input: "c/read"
-      attr {
-        key: "T"
-        value {
-          type: DT_FLOAT
-        }
-      }
-      attr {
-        key: "_output_shapes"
-        value {
-          list {
-            shape {
-              dim {
-                size: -1
-              }
-              dim {
-                size: 1
-              }
-            }
-          }
-        }
-      }
-    }
-    node {
-      name: "Const"
-      op: "Const"
-      attr {
-        key: "_output_shapes"
-        value {
-          list {
-            shape {
-            }
-          }
-        }
-      }
-      attr {
-        key: "dtype"
-        value {
-          type: DT_STRING
-        }
-      }
-      attr {
-        key: "value"
-        value {
-          tensor {
-            dtype: DT_STRING
-            tensor_shape {
-            }
-            string_val: "/tmp/original/export/assets/foo.txt"
-          }
-        }
-      }
-    }
-    node {
-      name: "filename_tensor/initial_value"
-      op: "Const"
-      attr {
-        key: "_output_shapes"
-        value {
-          list {
-            shape {
-            }
-          }
-        }
-      }
-      attr {
-        key: "dtype"
-        value {
-          type: DT_STRING
-        }
-      }
-      attr {
-        key: "value"
-        value {
-          tensor {
-            dtype: DT_STRING
-            tensor_shape {
-            }
-            string_val: "foo.txt"
-          }
-        }
-      }
-    }
-    node {
-      name: "filename_tensor"
-      op: "VariableV2"
-      attr {
-        key: "_output_shapes"
-        value {
-          list {
-            shape {
-            }
-          }
-        }
-      }
-      attr {
-        key: "container"
-        value {
-          s: ""
-        }
-      }
-      attr {
-        key: "dtype"
-        value {
-          type: DT_STRING
-        }
-      }
-      attr {
-        key: "shape"
-        value {
-          shape {
-          }
-        }
-      }
-      attr {
-        key: "shared_name"
-        value {
-          s: ""
-        }
-      }
-    }
-    node {
-      name: "filename_tensor/Assign"
-      op: "Assign"
-      input: "filename_tensor"
-      input: "filename_tensor/initial_value"
-      attr {
-        key: "T"
-        value {
-          type: DT_STRING
-        }
-      }
-      attr {
-        key: "_class"
-        value {
-          list {
-            s: "loc:@filename_tensor"
-          }
-        }
-      }
-      attr {
-        key: "_output_shapes"
-        value {
-          list {
-            shape {
-            }
-          }
-        }
-      }
-      attr {
-        key: "use_locking"
-        value {
-          b: true
-        }
-      }
-      attr {
-        key: "validate_shape"
-        value {
-          b: true
-        }
-      }
-    }
-    node {
-      name: "filename_tensor/read"
-      op: "Identity"
-      input: "filename_tensor"
-      attr {
-        key: "T"
-        value {
-          type: DT_STRING
-        }
-      }
-      attr {
-        key: "_class"
-        value {
-          list {
-            s: "loc:@filename_tensor"
-          }
-        }
-      }
-      attr {
-        key: "_output_shapes"
-        value {
-          list {
-            shape {
-            }
-          }
-        }
-      }
-    }
-    node {
-      name: "Assign/value"
-      op: "Const"
-      attr {
-        key: "_output_shapes"
-        value {
-          list {
-            shape {
-            }
-          }
-        }
-      }
-      attr {
-        key: "dtype"
-        value {
-          type: DT_STRING
-        }
-      }
-      attr {
-        key: "value"
-        value {
-          tensor {
-            dtype: DT_STRING
-            tensor_shape {
-            }
-            string_val: "foo.txt"
-          }
-        }
-      }
-    }
-    node {
-      name: "Assign"
-      op: "Assign"
-      input: "filename_tensor"
-      input: "Assign/value"
-      attr {
-        key: "T"
-        value {
-          type: DT_STRING
-        }
-      }
-      attr {
-        key: "_class"
-        value {
-          list {
-            s: "loc:@filename_tensor"
-          }
-        }
-      }
-      attr {
-        key: "_output_shapes"
-        value {
-          list {
-            shape {
-            }
-          }
-        }
-      }
-      attr {
-        key: "use_locking"
-        value {
-          b: false
-        }
-      }
-      attr {
-        key: "validate_shape"
-        value {
-          b: true
-        }
-      }
-    }
-    node {
-      name: "init"
-      op: "NoOp"
-      input: "^a/Assign"
-      input: "^b/Assign"
-      input: "^c/Assign"
-    }
-    node {
-      name: "group_deps"
-      op: "NoOp"
-      input: "^Assign"
-    }
-    node {
-      name: "save/Const"
-      op: "Const"
-      attr {
-        key: "_output_shapes"
-        value {
-          list {
-            shape {
-            }
-          }
-        }
-      }
-      attr {
-        key: "dtype"
-        value {
-          type: DT_STRING
-        }
-      }
-      attr {
-        key: "value"
-        value {
-          tensor {
-            dtype: DT_STRING
-            tensor_shape {
-            }
-            string_val: "model"
-          }
-        }
-      }
-    }
-    node {
-      name: "save/StringJoin/inputs_1"
-      op: "Const"
-      attr {
-        key: "_output_shapes"
-        value {
-          list {
-            shape {
-            }
-          }
-        }
-      }
-      attr {
-        key: "dtype"
-        value {
-          type: DT_STRING
-        }
-      }
-      attr {
-        key: "value"
-        value {
-          tensor {
-            dtype: DT_STRING
-            tensor_shape {
-            }
-            string_val: "_temp_80e928f1e0c844239d136d1ea966099d/part"
-          }
-        }
-      }
-    }
-    node {
-      name: "save/StringJoin"
-      op: "StringJoin"
-      input: "save/Const"
-      input: "save/StringJoin/inputs_1"
-      attr {
-        key: "N"
-        value {
-          i: 2
-        }
-      }
-      attr {
-        key: "_output_shapes"
-        value {
-          list {
-            shape {
-            }
-          }
-        }
-      }
-      attr {
-        key: "separator"
-        value {
-          s: ""
-        }
-      }
-    }
-    node {
-      name: "save/num_shards"
-      op: "Const"
-      attr {
-        key: "_output_shapes"
-        value {
-          list {
-            shape {
-            }
-          }
-        }
-      }
-      attr {
-        key: "dtype"
-        value {
-          type: DT_INT32
-        }
-      }
-      attr {
-        key: "value"
-        value {
-          tensor {
-            dtype: DT_INT32
-            tensor_shape {
-            }
-            int_val: 1
-          }
-        }
-      }
-    }
-    node {
-      name: "save/ShardedFilename/shard"
-      op: "Const"
-      attr {
-        key: "_output_shapes"
-        value {
-          list {
-            shape {
-            }
-          }
-        }
-      }
-      attr {
-        key: "dtype"
-        value {
-          type: DT_INT32
-        }
-      }
-      attr {
-        key: "value"
-        value {
-          tensor {
-            dtype: DT_INT32
-            tensor_shape {
-            }
-            int_val: 0
-          }
-        }
-      }
-    }
-    node {
-      name: "save/ShardedFilename"
-      op: "ShardedFilename"
-      input: "save/StringJoin"
-      input: "save/ShardedFilename/shard"
-      input: "save/num_shards"
-      attr {
-        key: "_output_shapes"
-        value {
-          list {
-            shape {
-            }
-          }
-        }
-      }
-    }
-    node {
-      name: "save/SaveV2/tensor_names"
-      op: "Const"
-      attr {
-        key: "_output_shapes"
-        value {
-          list {
-            shape {
-              dim {
-                size: 3
-              }
-            }
-          }
-        }
-      }
-      attr {
-        key: "dtype"
-        value {
-          type: DT_STRING
-        }
-      }
-      attr {
-        key: "value"
-        value {
-          tensor {
-            dtype: DT_STRING
-            tensor_shape {
-              dim {
-                size: 3
-              }
-            }
-            string_val: "a"
-            string_val: "b"
-            string_val: "c"
-          }
-        }
-      }
-    }
-    node {
-      name: "save/SaveV2/shape_and_slices"
-      op: "Const"
-      attr {
-        key: "_output_shapes"
-        value {
-          list {
-            shape {
-              dim {
-                size: 3
-              }
-            }
-          }
-        }
-      }
-      attr {
-        key: "dtype"
-        value {
-          type: DT_STRING
-        }
-      }
-      attr {
-        key: "value"
-        value {
-          tensor {
-            dtype: DT_STRING
-            tensor_shape {
-              dim {
-                size: 3
-              }
-            }
-            string_val: ""
-            string_val: ""
-            string_val: ""
-          }
-        }
-      }
-    }
-    node {
-      name: "save/SaveV2"
-      op: "SaveV2"
-      input: "save/ShardedFilename"
-      input: "save/SaveV2/tensor_names"
-      input: "save/SaveV2/shape_and_slices"
-      input: "a"
-      input: "b"
-      input: "c"
-      attr {
-        key: "dtypes"
-        value {
-          list {
-            type: DT_FLOAT
-            type: DT_FLOAT
-            type: DT_FLOAT
-          }
-        }
-      }
-    }
-    node {
-      name: "save/control_dependency"
-      op: "Identity"
-      input: "save/ShardedFilename"
-      input: "^save/SaveV2"
-      attr {
-        key: "T"
-        value {
-          type: DT_STRING
-        }
-      }
-      attr {
-        key: "_class"
-        value {
-          list {
-            s: "loc:@save/ShardedFilename"
-          }
-        }
-      }
-      attr {
-        key: "_output_shapes"
-        value {
-          list {
-            shape {
-            }
-          }
-        }
-      }
-    }
-    node {
-      name: "save/MergeV2Checkpoints/checkpoint_prefixes"
-      op: "Pack"
-      input: "save/ShardedFilename"
-      input: "^save/control_dependency"
-      attr {
-        key: "N"
-        value {
-          i: 1
-        }
-      }
-      attr {
-        key: "T"
-        value {
-          type: DT_STRING
-        }
-      }
-      attr {
-        key: "_output_shapes"
-        value {
-          list {
-            shape {
-              dim {
-                size: 1
-              }
-            }
-          }
-        }
-      }
-      attr {
-        key: "axis"
-        value {
-          i: 0
-        }
-      }
-    }
-    node {
-      name: "save/MergeV2Checkpoints"
-      op: "MergeV2Checkpoints"
-      input: "save/MergeV2Checkpoints/checkpoint_prefixes"
-      input: "save/Const"
-      attr {
-        key: "delete_old_dirs"
-        value {
-          b: true
-        }
-      }
-    }
-    node {
-      name: "save/Identity"
-      op: "Identity"
-      input: "save/Const"
-      input: "^save/control_dependency"
-      input: "^save/MergeV2Checkpoints"
-      attr {
-        key: "T"
-        value {
-          type: DT_STRING
-        }
-      }
-      attr {
-        key: "_output_shapes"
-        value {
-          list {
-            shape {
-            }
-          }
-        }
-      }
-    }
-    node {
-      name: "save/RestoreV2/tensor_names"
-      op: "Const"
-      attr {
-        key: "_output_shapes"
-        value {
-          list {
-            shape {
-              dim {
-                size: 1
-              }
-            }
-          }
-        }
-      }
-      attr {
-        key: "dtype"
-        value {
-          type: DT_STRING
-        }
-      }
-      attr {
-        key: "value"
-        value {
-          tensor {
-            dtype: DT_STRING
-            tensor_shape {
-              dim {
-                size: 1
-              }
-            }
-            string_val: "a"
-          }
-        }
-      }
-    }
-    node {
-      name: "save/RestoreV2/shape_and_slices"
-      op: "Const"
-      attr {
-        key: "_output_shapes"
-        value {
-          list {
-            shape {
-              dim {
-                size: 1
-              }
-            }
-          }
-        }
-      }
-      attr {
-        key: "dtype"
-        value {
-          type: DT_STRING
-        }
-      }
-      attr {
-        key: "value"
-        value {
-          tensor {
-            dtype: DT_STRING
-            tensor_shape {
-              dim {
-                size: 1
-              }
-            }
-            string_val: ""
-          }
-        }
-      }
-    }
-    node {
-      name: "save/RestoreV2"
-      op: "RestoreV2"
-      input: "save/Const"
-      input: "save/RestoreV2/tensor_names"
-      input: "save/RestoreV2/shape_and_slices"
-      attr {
-        key: "_output_shapes"
-        value {
-          list {
-            shape {
-              unknown_rank: true
-            }
-          }
-        }
-      }
-      attr {
-        key: "dtypes"
-        value {
-          list {
-            type: DT_FLOAT
-          }
-        }
-      }
-    }
-    node {
-      name: "save/Assign"
-      op: "Assign"
-      input: "a"
-      input: "save/RestoreV2"
-      attr {
-        key: "T"
-        value {
-          type: DT_FLOAT
-        }
-      }
-      attr {
-        key: "_class"
-        value {
-          list {
-            s: "loc:@a"
-          }
-        }
-      }
-      attr {
-        key: "_output_shapes"
-        value {
-          list {
-            shape {
-            }
-          }
-        }
-      }
-      attr {
-        key: "use_locking"
-        value {
-          b: true
-        }
-      }
-      attr {
-        key: "validate_shape"
-        value {
-          b: true
-        }
-      }
-    }
-    node {
-      name: "save/RestoreV2_1/tensor_names"
-      op: "Const"
-      attr {
-        key: "_output_shapes"
-        value {
-          list {
-            shape {
-              dim {
-                size: 1
-              }
-            }
-          }
-        }
-      }
-      attr {
-        key: "dtype"
-        value {
-          type: DT_STRING
-        }
-      }
-      attr {
-        key: "value"
-        value {
-          tensor {
-            dtype: DT_STRING
-            tensor_shape {
-              dim {
-                size: 1
-              }
-            }
-            string_val: "b"
-          }
-        }
-      }
-    }
-    node {
-      name: "save/RestoreV2_1/shape_and_slices"
-      op: "Const"
-      attr {
-        key: "_output_shapes"
-        value {
-          list {
-            shape {
-              dim {
-                size: 1
-              }
-            }
-          }
-        }
-      }
-      attr {
-        key: "dtype"
-        value {
-          type: DT_STRING
-        }
-      }
-      attr {
-        key: "value"
-        value {
-          tensor {
-            dtype: DT_STRING
-            tensor_shape {
-              dim {
-                size: 1
-              }
-            }
-            string_val: ""
-          }
-        }
-      }
-    }
-    node {
-      name: "save/RestoreV2_1"
-      op: "RestoreV2"
-      input: "save/Const"
-      input: "save/RestoreV2_1/tensor_names"
-      input: "save/RestoreV2_1/shape_and_slices"
-      attr {
-        key: "_output_shapes"
-        value {
-          list {
-            shape {
-              unknown_rank: true
-            }
-          }
-        }
-      }
-      attr {
-        key: "dtypes"
-        value {
-          list {
-            type: DT_FLOAT
-          }
-        }
-      }
-    }
-    node {
-      name: "save/Assign_1"
-      op: "Assign"
-      input: "b"
-      input: "save/RestoreV2_1"
-      attr {
-        key: "T"
-        value {
-          type: DT_FLOAT
-        }
-      }
-      attr {
-        key: "_class"
-        value {
-          list {
-            s: "loc:@b"
-          }
-        }
-      }
-      attr {
-        key: "_output_shapes"
-        value {
-          list {
-            shape {
-            }
-          }
-        }
-      }
-      attr {
-        key: "use_locking"
-        value {
-          b: true
-        }
-      }
-      attr {
-        key: "validate_shape"
-        value {
-          b: true
-        }
-      }
-    }
-    node {
-      name: "save/RestoreV2_2/tensor_names"
-      op: "Const"
-      attr {
-        key: "_output_shapes"
-        value {
-          list {
-            shape {
-              dim {
-                size: 1
-              }
-            }
-          }
-        }
-      }
-      attr {
-        key: "dtype"
-        value {
-          type: DT_STRING
-        }
-      }
-      attr {
-        key: "value"
-        value {
-          tensor {
-            dtype: DT_STRING
-            tensor_shape {
-              dim {
-                size: 1
-              }
-            }
-            string_val: "c"
-          }
-        }
-      }
-    }
-    node {
-      name: "save/RestoreV2_2/shape_and_slices"
-      op: "Const"
-      attr {
-        key: "_output_shapes"
-        value {
-          list {
-            shape {
-              dim {
-                size: 1
-              }
-            }
-          }
-        }
-      }
-      attr {
-        key: "dtype"
-        value {
-          type: DT_STRING
-        }
-      }
-      attr {
-        key: "value"
-        value {
-          tensor {
-            dtype: DT_STRING
-            tensor_shape {
-              dim {
-                size: 1
-              }
-            }
-            string_val: ""
-          }
-        }
-      }
-    }
-    node {
-      name: "save/RestoreV2_2"
-      op: "RestoreV2"
-      input: "save/Const"
-      input: "save/RestoreV2_2/tensor_names"
-      input: "save/RestoreV2_2/shape_and_slices"
-      attr {
-        key: "_output_shapes"
-        value {
-          list {
-            shape {
-              unknown_rank: true
-            }
-          }
-        }
-      }
-      attr {
-        key: "dtypes"
-        value {
-          list {
-            type: DT_FLOAT
-          }
-        }
-      }
-    }
-    node {
-      name: "save/Assign_2"
-      op: "Assign"
-      input: "c"
-      input: "save/RestoreV2_2"
-      attr {
-        key: "T"
-        value {
-          type: DT_FLOAT
-        }
-      }
-      attr {
-        key: "_class"
-        value {
-          list {
-            s: "loc:@c"
-          }
-        }
-      }
-      attr {
-        key: "_output_shapes"
-        value {
-          list {
-            shape {
-            }
-          }
-        }
-      }
-      attr {
-        key: "use_locking"
-        value {
-          b: true
-        }
-      }
-      attr {
-        key: "validate_shape"
-        value {
-          b: true
-        }
-      }
-    }
-    node {
-      name: "save/restore_shard"
-      op: "NoOp"
-      input: "^save/Assign"
-      input: "^save/Assign_1"
-      input: "^save/Assign_2"
-    }
-    node {
-      name: "save/restore_all"
-      op: "NoOp"
-      input: "^save/restore_shard"
-    }
-    versions {
-      producer: 23
-    }
-  }
-  saver_def {
-    filename_tensor_name: "save/Const:0"
-    save_tensor_name: "save/Identity:0"
-    restore_op_name: "save/restore_all"
-    max_to_keep: 5
-    sharded: true
-    keep_checkpoint_every_n_hours: 10000.0
-    version: V2
-  }
-  collection_def {
-    key: "asset_filepaths"
-    value {
-      node_list {
-        value: "Const:0"
-      }
-    }
-  }
-  collection_def {
-    key: "legacy_init_op"
-    value {
-      node_list {
-        value: "group_deps"
-      }
-    }
-  }
-  collection_def {
-    key: "saved_model_assets"
-    value {
-      any_list {
-        value {
-          type_url: "type.googleapis.com/tensorflow.AssetFileDef"
-          value: "\n\t\n\007Const:0\022\007foo.txt"
-        }
-      }
-    }
-  }
-  collection_def {
-    key: "trainable_variables"
-    value {
-      bytes_list {
-        value: "\n\003a:0\022\010a/Assign\032\010a/read:0"
-        value: "\n\003b:0\022\010b/Assign\032\010b/read:0"
-        value: "\n\003c:0\022\010c/Assign\032\010c/read:0"
-      }
-    }
-  }
-  collection_def {
-    key: "variables"
-    value {
-      bytes_list {
-        value: "\n\003a:0\022\010a/Assign\032\010a/read:0"
-        value: "\n\003b:0\022\010b/Assign\032\010b/read:0"
-        value: "\n\003c:0\022\010c/Assign\032\010c/read:0"
-      }
-    }
-  }
-  signature_def {
-    key: "classify_x2_to_y3"
-    value {
-      inputs {
-        key: "inputs"
-        value {
-          name: "x2:0"
-          dtype: DT_FLOAT
-          tensor_shape {
-            dim {
-              size: -1
-            }
-            dim {
-              size: 1
-            }
-          }
-        }
-      }
-      outputs {
-        key: "scores"
-        value {
-          name: "y3:0"
-          dtype: DT_FLOAT
-          tensor_shape {
-            dim {
-              size: -1
-            }
-            dim {
-              size: 1
-            }
-          }
-        }
-      }
-      method_name: "tensorflow/serving/classify"
-    }
-  }
-  signature_def {
-    key: "classify_x_to_y"
-    value {
-      inputs {
-        key: "inputs"
-        value {
-          name: "tf_example:0"
-          dtype: DT_STRING
-          tensor_shape {
-            unknown_rank: true
-          }
-        }
-      }
-      outputs {
-        key: "scores"
-        value {
-          name: "y:0"
-          dtype: DT_FLOAT
-          tensor_shape {
-            dim {
-              size: -1
-            }
-            dim {
-              size: 1
-            }
-          }
-        }
-      }
-      method_name: "tensorflow/serving/classify"
-    }
-  }
-  signature_def {
-    key: "regress_x2_to_y3"
-    value {
-      inputs {
-        key: "inputs"
-        value {
-          name: "x2:0"
-          dtype: DT_FLOAT
-          tensor_shape {
-            dim {
-              size: -1
-            }
-            dim {
-              size: 1
-            }
-          }
-        }
-      }
-      outputs {
-        key: "outputs"
-        value {
-          name: "y3:0"
-          dtype: DT_FLOAT
-          tensor_shape {
-            dim {
-              size: -1
-            }
-            dim {
-              size: 1
-            }
-          }
-        }
-      }
-      method_name: "tensorflow/serving/regress"
-    }
-  }
-  signature_def {
-    key: "regress_x_to_y"
-    value {
-      inputs {
-        key: "inputs"
-        value {
-          name: "tf_example:0"
-          dtype: DT_STRING
-          tensor_shape {
-            unknown_rank: true
-          }
-        }
-      }
-      outputs {
-        key: "outputs"
-        value {
-          name: "y:0"
-          dtype: DT_FLOAT
-          tensor_shape {
-            dim {
-              size: -1
-            }
-            dim {
-              size: 1
-            }
-          }
-        }
-      }
-      method_name: "tensorflow/serving/regress"
-    }
-  }
-  signature_def {
-    key: "regress_x_to_y2"
-    value {
-      inputs {
-        key: "inputs"
-        value {
-          name: "tf_example:0"
-          dtype: DT_STRING
-          tensor_shape {
-            unknown_rank: true
-          }
-        }
-      }
-      outputs {
-        key: "outputs"
-        value {
-          name: "y2:0"
-          dtype: DT_FLOAT
-          tensor_shape {
-            dim {
-              size: -1
-            }
-            dim {
-              size: 1
-            }
-          }
-        }
-      }
-      method_name: "tensorflow/serving/regress"
-    }
-  }
-  signature_def {
-    key: "serving_default"
-    value {
-      inputs {
-        key: "x"
-        value {
-          name: "x:0"
-          dtype: DT_FLOAT
-          tensor_shape {
-            dim {
-              size: -1
-            }
-            dim {
-              size: 1
-            }
-          }
-        }
-      }
-      outputs {
-        key: "y"
-        value {
-          name: "y:0"
-          dtype: DT_FLOAT
-          tensor_shape {
-            dim {
-              size: -1
-            }
-            dim {
-              size: 1
-            }
-          }
-        }
-      }
-      method_name: "tensorflow/serving/predict"
-    }
-  }
-}
diff --git a/tensorflow/cc/saved_model/testdata/half_plus_two_forward_compatibility/00000123/variables/variables.data-00000-of-00001 b/tensorflow/cc/saved_model/testdata/half_plus_two_forward_compatibility/00000123/variables/variables.data-00000-of-00001
deleted file mode 100755
index 15b75d6ef6bffc336d138d923badb3928b8c4c13..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 12
TcmZQzV6bOkU~phyaBu(s1VaG;

diff --git a/tensorflow/cc/saved_model/testdata/half_plus_two_forward_compatibility/00000123/variables/variables.index b/tensorflow/cc/saved_model/testdata/half_plus_two_forward_compatibility/00000123/variables/variables.index
deleted file mode 100755
index 7ec9fb4fe2dd21d0a6c324aecd7658fc37cf2326..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 151
zcmZQzVB=tvV&Y(AVB}8ZU=(7|U@>L0P?u+5<V^x`6<9P_Oa=Z{_%kr_CW8eyG+0dE
zUu!zdz`(%32qK<{ZrcN*!JGr17H(i*WJ+Ohf(u1#dimX*BZLnmKnREbZs=Aib-xV&
DMWhyT

-- 
GitLab


From 962ed613cf1087637848d3e2b23f5b01d93c7eda Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Tue, 17 Oct 2017 12:22:20 -0700
Subject: [PATCH 473/909] Fix #13731 by adding HistogramdFixedWidth in
 hidden_ops.txt and create the python wrapper (#13781)

* Fix 13731 by adding HistogramdFixedWidth in hidden_ops.txt and create the python wrapper

so that both api compatibility and test utility code in contrib could be
preserved. See https://github.com/tensorflow/tensorflow/pull/13731#issuecomment-337186002
for reference.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Add underscore (`_histogram_fixed_width`) in calling gen_math_ops.py

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* clang-format -i histogram_op.cc

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/core/kernels/histogram_op.cc | 24 ++++++++++++------------
 tensorflow/core/ops/math_ops.cc         |  2 +-
 tensorflow/python/ops/hidden_ops.txt    |  1 +
 tensorflow/python/ops/histogram_ops.py  |  4 ++--
 4 files changed, 16 insertions(+), 15 deletions(-)

diff --git a/tensorflow/core/kernels/histogram_op.cc b/tensorflow/core/kernels/histogram_op.cc
index c170f172e4..4e035286f6 100644
--- a/tensorflow/core/kernels/histogram_op.cc
+++ b/tensorflow/core/kernels/histogram_op.cc
@@ -74,45 +74,44 @@ struct HistogramFixedWidthFunctor<CPUDevice, T, Tout> {
 template <typename Device, typename T, typename Tout>
 class HistogramFixedWidthOp : public OpKernel {
  public:
-  explicit HistogramFixedWidthOp(OpKernelConstruction* ctx) : OpKernel(ctx) {
-    OP_REQUIRES_OK(ctx, ctx->GetAttr("nbins", &nbins_));
-    OP_REQUIRES(
-        ctx, (nbins_ > 0),
-        errors::InvalidArgument("nbins should be a positive number, but got '",
-                                nbins_, "'"));
-  }
+  explicit HistogramFixedWidthOp(OpKernelConstruction* ctx) : OpKernel(ctx) {}
 
   void Compute(OpKernelContext* ctx) override {
     const Tensor& values_tensor = ctx->input(0);
     const Tensor& value_range_tensor = ctx->input(1);
+    const Tensor& nbins_tensor = ctx->input(2);
 
     OP_REQUIRES(ctx, TensorShapeUtils::IsVector(value_range_tensor.shape()),
                 errors::InvalidArgument("value_range should be a vector."));
     OP_REQUIRES(ctx, (value_range_tensor.shape().num_elements() == 2),
                 errors::InvalidArgument(
                     "value_range should be a vector of 2 elements."));
+    OP_REQUIRES(ctx, TensorShapeUtils::IsScalar(nbins_tensor.shape()),
+                errors::InvalidArgument("nbins should be a scalar."));
 
     const auto values = values_tensor.flat<T>();
     const auto value_range = value_range_tensor.flat<T>();
+    const auto nbins = nbins_tensor.scalar<int32>()();
 
     OP_REQUIRES(
         ctx, (value_range(0) < value_range(1)),
         errors::InvalidArgument("value_range should satisfy value_range[0] < "
                                 "value_range[1], but got '[",
                                 value_range(0), ", ", value_range(1), "]'"));
+    OP_REQUIRES(
+        ctx, (nbins > 0),
+        errors::InvalidArgument("nbins should be a positive number, but got '",
+                                nbins, "'"));
 
     Tensor* out_tensor;
     OP_REQUIRES_OK(ctx,
-                   ctx->allocate_output(0, TensorShape({nbins_}), &out_tensor));
+                   ctx->allocate_output(0, TensorShape({nbins}), &out_tensor));
     auto out = out_tensor->flat<Tout>();
 
     OP_REQUIRES_OK(
         ctx, functor::HistogramFixedWidthFunctor<Device, T, Tout>::Compute(
-                 ctx, values, value_range, nbins_, out));
+                 ctx, values, value_range, nbins, out));
   }
-
- private:
-  int nbins_;
 };
 
 #define REGISTER_KERNELS(type)                                           \
@@ -135,6 +134,7 @@ TF_CALL_REAL_NUMBER_TYPES(REGISTER_KERNELS);
   REGISTER_KERNEL_BUILDER(Name("HistogramFixedWidth")          \
                               .Device(DEVICE_GPU)              \
                               .HostMemory("value_range")       \
+                              .HostMemory("nbins")             \
                               .TypeConstraint<type>("T")       \
                               .TypeConstraint<int32>("dtype"), \
                           HistogramFixedWidthOp<GPUDevice, type, int32>)
diff --git a/tensorflow/core/ops/math_ops.cc b/tensorflow/core/ops/math_ops.cc
index a1c608ee54..61db896c51 100644
--- a/tensorflow/core/ops/math_ops.cc
+++ b/tensorflow/core/ops/math_ops.cc
@@ -2253,8 +2253,8 @@ product: Pairwise cross product of the vectors in `a` and `b`.
 REGISTER_OP("HistogramFixedWidth")
     .Input("values: T")
     .Input("value_range: T")
+    .Input("nbins: int32")
     .Output("out: dtype")
-    .Attr("nbins: int = 100")
     .Attr("T: {int32, int64, float32, float64}")
     .Attr("dtype: {int32, int64} = DT_INT32")
     .SetShapeFn([](InferenceContext* c) {
diff --git a/tensorflow/python/ops/hidden_ops.txt b/tensorflow/python/ops/hidden_ops.txt
index 04dfb5b65d..86bc038e86 100644
--- a/tensorflow/python/ops/hidden_ops.txt
+++ b/tensorflow/python/ops/hidden_ops.txt
@@ -259,6 +259,7 @@ ComplexAbs
 Conj
 FloorDiv
 FloorMod
+HistogramFixedWidth
 Max
 Mean
 Min
diff --git a/tensorflow/python/ops/histogram_ops.py b/tensorflow/python/ops/histogram_ops.py
index 040c3a5ae8..51e4be9343 100644
--- a/tensorflow/python/ops/histogram_ops.py
+++ b/tensorflow/python/ops/histogram_ops.py
@@ -71,5 +71,5 @@ def histogram_fixed_width(values,
   """
   with ops.name_scope(name, 'histogram_fixed_width',
                       [values, value_range, nbins]) as name:
-    return gen_math_ops.histogram_fixed_width(values, value_range, nbins,
-                                              dtype=dtype, name=name)
+    return gen_math_ops._histogram_fixed_width(values, value_range, nbins,
+                                               dtype=dtype, name=name)
-- 
GitLab


From 568127ac3b8e501bb230ee287ec9a46129fad349 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Tue, 17 Oct 2017 12:23:29 -0700
Subject: [PATCH 474/909] Improve shape inference with `DecodeAndCropJpeg`
 (#13750)

* Improve shape inference with `DecodeAndCropJpeg`

While working on improving shape inference for several
other ops in 13561 and 13193, I noticed that `DecodeAndCropJpeg`
does not inference shape even though crop size might have already
be provided. In that case the shape will be `[h, w, channel]`
and `h`, `w` is part of the `crop_window`.

This fix updates the shape function in `DecodeAndCropJpeg`
for improving shape inference.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Add test cases to cover shape inference for `DecodeAndCropJpeg`

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Address failed unit tests

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/core/ops/image_ops.cc        | 31 ++++++++++++++++++++++++-
 tensorflow/core/ops/image_ops_test.cc   |  6 ++---
 tensorflow/python/ops/image_ops_test.py |  6 ++++-
 3 files changed, 38 insertions(+), 5 deletions(-)

diff --git a/tensorflow/core/ops/image_ops.cc b/tensorflow/core/ops/image_ops.cc
index 66765a3333..89c9da81c5 100644
--- a/tensorflow/core/ops/image_ops.cc
+++ b/tensorflow/core/ops/image_ops.cc
@@ -453,7 +453,36 @@ REGISTER_OP("DecodeAndCropJpeg")
     .Attr("acceptable_fraction: float = 1.0")
     .Attr("dct_method: string = ''")
     .Output("image: uint8")
-    .SetShapeFn(DecodeImageShapeFn)
+    .SetShapeFn([](InferenceContext* c) {
+      ShapeHandle unused;
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 0, &unused));
+      DimensionHandle channels_dim = c->UnknownDim();
+      DimensionHandle h = c->UnknownDim();
+      DimensionHandle w = c->UnknownDim();
+
+      int32 channels;
+      TF_RETURN_IF_ERROR(c->GetAttr("channels", &channels));
+      if (channels != 0) {
+        if (channels < 0) {
+          return errors::InvalidArgument("channels must be non-negative, got ",
+                                         channels);
+        }
+        channels_dim = c->MakeDim(channels);
+      }
+
+      DimensionHandle unused_dim;
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 1, &unused));
+      TF_RETURN_IF_ERROR(c->WithValue(c->Dim(unused, 0), 4, &unused_dim));
+
+      const Tensor* crop_window = c->input_tensor(1);
+      if (crop_window != nullptr) {
+        auto crop_window_vec = crop_window->vec<int32>();
+        h = c->MakeDim(crop_window_vec(2));
+        w = c->MakeDim(crop_window_vec(3));
+      }
+      c->set_output(0, c->MakeShape({h, w, channels_dim}));
+      return Status::OK();
+    })
     .Doc(strings::StrCat(R"doc(
 Decode and Crop a JPEG-encoded image to a uint8 tensor.
 )doc",
diff --git a/tensorflow/core/ops/image_ops_test.cc b/tensorflow/core/ops/image_ops_test.cc
index c34b11a15e..5f0b391b0d 100644
--- a/tensorflow/core/ops/image_ops_test.cc
+++ b/tensorflow/core/ops/image_ops_test.cc
@@ -105,7 +105,7 @@ TEST(ImageOpsTest, DecodeAndCropJpeg_ShapeFn) {
                    .Input({"img", 0, DT_STRING})
                    .Input({"crop_window", 1, DT_INT32})
                    .Finalize(&op.node_def));
-  INFER_OK(op, "[];[]", "[?,?,?]");
+  INFER_OK(op, "[];[?]", "[?,?,?]");
 
   // Set the channel, so that part of output shape is known.
   TF_ASSERT_OK(NodeDefBuilder("test", op_name)
@@ -113,7 +113,7 @@ TEST(ImageOpsTest, DecodeAndCropJpeg_ShapeFn) {
                    .Input({"crop_window", 1, DT_INT32})
                    .Attr("channels", 4)
                    .Finalize(&op.node_def));
-  INFER_OK(op, "[];[]", "[?,?,4]");
+  INFER_OK(op, "[];[?]", "[?,?,4]");
 
   // Negative channel value is rejected.
   TF_ASSERT_OK(NodeDefBuilder("test", op_name)
@@ -139,7 +139,7 @@ TEST(ImageOpsTest, DecodeAndCropJpeg_InvalidCropWindow) {
                    .Input({"img", 0, DT_STRING})
                    .Input({"crop_window", 1, DT_INT32})
                    .Finalize(&op.node_def));
-  INFER_OK(op, "[];[]", "[?,?,?]");
+  INFER_OK(op, "[];[?]", "[?,?,?]");
 }
 
 TEST(ImageOpsTest, EncodeImage_ShapeFn) {
diff --git a/tensorflow/python/ops/image_ops_test.py b/tensorflow/python/ops/image_ops_test.py
index 348c005ff3..b13b73edbb 100644
--- a/tensorflow/python/ops/image_ops_test.py
+++ b/tensorflow/python/ops/image_ops_test.py
@@ -2434,9 +2434,13 @@ class JpegTest(test_util.TensorFlowTestCase):
         y, x, h, w = crop_window
         image1_crop = image_ops.crop_to_bounding_box(image1, y, x, h, w)
 
-        # Combined crop+decode.
+        # Combined decode+crop.
         image2 = image_ops.decode_and_crop_jpeg(jpeg0, crop_window)
 
+        # Combined decode+crop should have the same shape inference
+        self.assertAllEqual(image1_crop.get_shape().as_list(),
+                            image2.get_shape().as_list())
+
         # CropAndDecode should be equal to DecodeJpeg+Crop.
         image1_crop, image2 = sess.run([image1_crop, image2])
         self.assertAllEqual(image1_crop, image2)
-- 
GitLab


From 7f778806153598093af01081650fe36a16a2ff56 Mon Sep 17 00:00:00 2001
From: Shanqing Cai <cais@google.com>
Date: Tue, 17 Oct 2017 13:47:23 -0700
Subject: [PATCH 475/909] tfdbg: add missing -f flag of the pt command to the
 command table

PiperOrigin-RevId: 172508350
---
 tensorflow/docs_src/programmers_guide/debugger.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/docs_src/programmers_guide/debugger.md b/tensorflow/docs_src/programmers_guide/debugger.md
index 58154d19e7..36a016e880 100644
--- a/tensorflow/docs_src/programmers_guide/debugger.md
+++ b/tensorflow/docs_src/programmers_guide/debugger.md
@@ -141,6 +141,7 @@ Try the following commands at the `tfdbg>` prompt (referencing the code at
 | **`lt`** | | **List dumped tensors.** | `lt` |
 | | `-n <name_pattern>` | List dumped tensors with names matching given regular-expression pattern. | `lt -n Softmax.*` |
 | | `-t <op_pattern>` | List dumped tensors with op types matching given regular-expression pattern. | `lt -t MatMul` |
+| | `-f <filter_name>` | List only the tensors that pass a registered tensor filter. | `lt -f has_inf_or_nan` |
 | | `-s <sort_key>` | Sort the output by given `sort_key`, whose possible values are `timestamp` (default), `dump_size`, `op_type` and `tensor_name`. | `lt -s dump_size` |
 | | `-r` | Sort in reverse order. | `lt -r -s dump_size` |
 | **`pt`** | | **Print value of a dumped tensor.** | |
-- 
GitLab


From cfb13fa789bcf1cdbbf0fd38cf7568b7098ab99b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 17 Oct 2017 13:58:00 -0700
Subject: [PATCH 476/909] Added an additional check on the length of the values
 and boundaries lists.

PiperOrigin-RevId: 172510229
---
 tensorflow/python/training/learning_rate_decay.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/training/learning_rate_decay.py b/tensorflow/python/training/learning_rate_decay.py
index bb9e26d8b4..e4a7964aaf 100644
--- a/tensorflow/python/training/learning_rate_decay.py
+++ b/tensorflow/python/training/learning_rate_decay.py
@@ -130,8 +130,12 @@ def piecewise_constant(x, boundaries, values, name=None):
 
   Raises:
     ValueError: if types of `x` and `boundaries` do not match, or types of all
-        `values` do not match.
+        `values` do not match or
+        the number of elements in the lists does not match.
   """
+  if len(boundaries) != len(values) - 1:
+    raise ValueError(
+        "The length of boundaries should be 1 less than the length of values")
   with ops.name_scope(name, "PiecewiseConstant",
                       [x, boundaries, values, name]) as name:
     x = ops.convert_to_tensor(x)
@@ -158,7 +162,6 @@ def piecewise_constant(x, boundaries, values, name=None):
         raise ValueError(
             "Values must have elements all with the same dtype (%s vs %s)." % (
                 values[0].dtype.base_dtype, v.dtype.base_dtype))
-
     pred_fn_pairs = {}
     pred_fn_pairs[x <= boundaries[0]] = lambda: values[0]
     pred_fn_pairs[x > boundaries[-1]] = lambda: values[-1]
-- 
GitLab


From a760b9945511e64172bc4073d3c0efa0888dc2a1 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 17 Oct 2017 14:04:50 -0700
Subject: [PATCH 477/909] Internal change.

PiperOrigin-RevId: 172511553
---
 tensorflow/contrib/timeseries/python/timeseries/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/contrib/timeseries/python/timeseries/BUILD b/tensorflow/contrib/timeseries/python/timeseries/BUILD
index 76e8ccc62a..7491b1b2d2 100644
--- a/tensorflow/contrib/timeseries/python/timeseries/BUILD
+++ b/tensorflow/contrib/timeseries/python/timeseries/BUILD
@@ -105,6 +105,7 @@ py_test(
     tags = [
         "no_pip_gpu",  # b/63391119
         "nomsan",  # Takes too long to run.
+        "notsan",  # b/67865658
     ],
     deps = [
         ":ar_model",
-- 
GitLab


From bb7869aad43e329e9fafa027a23ee96d4ccca124 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 17 Oct 2017 14:11:25 -0700
Subject: [PATCH 478/909] Remove remaining nomac tags for TFBT as the memory
 corruption issue is fixed.

PiperOrigin-RevId: 172512636
---
 tensorflow/contrib/boosted_trees/BUILD | 19 -------------------
 1 file changed, 19 deletions(-)

diff --git a/tensorflow/contrib/boosted_trees/BUILD b/tensorflow/contrib/boosted_trees/BUILD
index 1b85c260c0..66a04d42e9 100644
--- a/tensorflow/contrib/boosted_trees/BUILD
+++ b/tensorflow/contrib/boosted_trees/BUILD
@@ -81,9 +81,6 @@ py_test(
     size = "small",
     srcs = ["python/utils/losses_test.py"],
     srcs_version = "PY2AND3",
-    tags = [
-        "nomac",  # b/63258195
-    ],
     deps = [
         ":losses",
         "//tensorflow/python:array_ops",
@@ -135,7 +132,6 @@ py_test(
     srcs = ["python/training/functions/gbdt_batch_test.py"],
     srcs_version = "PY2AND3",
     tags = [
-        "nomac",  # b/63258195
         "notsan",  # b/62863147
     ],
     deps = [
@@ -164,9 +160,6 @@ py_test(
     size = "small",
     srcs = ["python/kernel_tests/model_ops_test.py"],
     srcs_version = "PY2AND3",
-    tags = [
-        "nomac",  # b/63258195
-    ],
     deps = [
         ":model_ops_py",
         ":prediction_ops_py",
@@ -187,9 +180,6 @@ py_test(
     size = "small",
     srcs = ["python/kernel_tests/prediction_ops_test.py"],
     srcs_version = "PY2AND3",
-    tags = [
-        "nomac",  # b/63258195
-    ],
     deps = [
         ":model_ops_py",
         ":prediction_ops_py",
@@ -207,9 +197,6 @@ py_test(
     size = "small",
     srcs = ["python/kernel_tests/quantile_ops_test.py"],
     srcs_version = "PY2AND3",
-    tags = [
-        "nomac",  # b/63258195
-    ],
     deps = [
         ":quantile_ops_py",
         "//tensorflow/contrib/boosted_trees/proto:quantiles_proto_py",
@@ -247,9 +234,6 @@ py_test(
     size = "small",
     srcs = ["python/kernel_tests/stats_accumulator_ops_test.py"],
     srcs_version = "PY2AND3",
-    tags = [
-        "nomac",  # b/63258195
-    ],
     deps = [
         ":stats_accumulator_ops_py",
         "//tensorflow/python:framework_ops",
@@ -264,9 +248,6 @@ py_test(
     size = "small",
     srcs = ["python/kernel_tests/training_ops_test.py"],
     srcs_version = "PY2AND3",
-    tags = [
-        "nomac",  # b/63258195
-    ],
     deps = [
         ":model_ops_py",
         ":training_ops_py",
-- 
GitLab


From d69948fa09a403106a4b4c68ed00f4a8f0c10ef1 Mon Sep 17 00:00:00 2001
From: Akshay Agrawal <akshayka@google.com>
Date: Tue, 17 Oct 2017 14:25:33 -0700
Subject: [PATCH 479/909] Make tensor_util.constant_value_as_shape compatible
 with Eager mode

PiperOrigin-RevId: 172514789
---
 tensorflow/python/BUILD                         | 1 +
 tensorflow/python/framework/tensor_util.py      | 5 +++++
 tensorflow/python/framework/tensor_util_test.py | 9 +++++++++
 3 files changed, 15 insertions(+)

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index f4106ac68c..e4e284dcdf 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -1141,6 +1141,7 @@ py_test(
         ":client_testlib",
         ":framework",
         ":framework_for_generated_wrappers",
+        ":framework_test_lib",
         ":math_ops",
         ":state_ops_gen",
         "//third_party/py/numpy",
diff --git a/tensorflow/python/framework/tensor_util.py b/tensorflow/python/framework/tensor_util.py
index 63324e5977..53eba8b747 100644
--- a/tensorflow/python/framework/tensor_util.py
+++ b/tensorflow/python/framework/tensor_util.py
@@ -23,6 +23,7 @@ import six
 
 from tensorflow.core.framework import tensor_pb2
 from tensorflow.core.framework import tensor_shape_pb2
+from tensorflow.python.eager import context
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
 from tensorflow.python.util import compat
@@ -769,6 +770,10 @@ def constant_value_as_shape(tensor):  # pylint: disable=invalid-name
   Returns:
     A `TensorShape` based on the constant value of the given `tensor`.
   """
+  if context.in_eager_mode():
+    return tensor_shape.as_shape(
+        [dim if dim != -1 else None for dim in tensor.numpy()])
+
   shape = tensor.get_shape().with_rank(1)
   if tensor.get_shape() == [0]:
     return tensor_shape.scalar()
diff --git a/tensorflow/python/framework/tensor_util_test.py b/tensorflow/python/framework/tensor_util_test.py
index dda72fc0c8..b4f28cfce0 100644
--- a/tensorflow/python/framework/tensor_util_test.py
+++ b/tensorflow/python/framework/tensor_util_test.py
@@ -26,6 +26,7 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
 from tensorflow.python.framework import tensor_util
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gen_state_ops
 from tensorflow.python.ops import math_ops
@@ -901,6 +902,7 @@ class ConstantValueTest(test.TestCase):
 
 class ConstantValueAsShapeTest(test.TestCase):
 
+  @test_util.run_in_graph_and_eager_modes()
   def testConstant(self):
     np_val = np.random.rand(3).astype(np.int32)
     tf_val = constant_op.constant(np_val)
@@ -913,11 +915,18 @@ class ConstantValueAsShapeTest(test.TestCase):
         tensor_shape.TensorShape([]),
         tensor_util.constant_value_as_shape(tf_val))
 
+  @test_util.run_in_graph_and_eager_modes()
   def testShape(self):
     tf_val = array_ops.shape(constant_op.constant(0.0, shape=[1, 2, 3]))
     c_val = tensor_util.constant_value_as_shape(tf_val)
     self.assertEqual(tensor_shape.TensorShape([1, 2, 3]), c_val)
 
+  @test_util.run_in_graph_and_eager_modes()
+  def testMinusOneBecomesNone(self):
+    tf_val = constant_op.constant([-1, 1, -1], shape=[3])
+    c_val = tensor_util.constant_value_as_shape(tf_val)
+    self.assertEqual([None, 1, None], c_val.as_list())
+
   def testPack(self):
     tf_val = array_ops.stack(
         [constant_op.constant(16), 37, array_ops.placeholder(dtypes.int32)])
-- 
GitLab


From cbb705f10149a11b8d17182343ef12ab2dbfd7a8 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Tue, 17 Oct 2017 14:35:51 -0700
Subject: [PATCH 480/909] Fix crash when `tf.pad` is used with int64 paddings.
 (#13517)

* Add int64 bounds support for `tf.image.pad_to_bounding_box`

This fix tries to fix the issue raised in 13506 where int64 data types
for bounds in `tf.image.pad_to_bounding_box` crashes.

The reason of the crash is caused by the fact that int64 was directly
converted into int32 without passing through kernel registeration.

This fix fixes the issue by adding `typename Tpadding` to the template.
---
 tensorflow/core/kernels/pad_op.cc             | 144 ++++++++++++++----
 tensorflow/core/kernels/pad_op.h              |  10 +-
 tensorflow/core/kernels/pad_op_gpu.cu.cc      |  20 ++-
 tensorflow/python/kernel_tests/pad_op_test.py |   9 ++
 tensorflow/python/ops/image_ops_test.py       |  19 +++
 5 files changed, 158 insertions(+), 44 deletions(-)

diff --git a/tensorflow/core/kernels/pad_op.cc b/tensorflow/core/kernels/pad_op.cc
index 6196c5ed93..eff3e4d92c 100644
--- a/tensorflow/core/kernels/pad_op.cc
+++ b/tensorflow/core/kernels/pad_op.cc
@@ -40,9 +40,9 @@ typedef Eigen::ThreadPoolDevice CPUDevice;
 typedef Eigen::GpuDevice GPUDevice;
 #ifdef TENSORFLOW_USE_SYCL
 typedef Eigen::SyclDevice SYCLDevice;
-#endif // TENSORFLOW_USE_SYCL
+#endif  // TENSORFLOW_USE_SYCL
 
-template <typename Device, typename T>
+template <typename Device, typename T, typename Tpadding>
 class PadOp : public OpKernel {
  public:
   explicit PadOp(OpKernelConstruction* context) : OpKernel(context) {}
@@ -82,10 +82,11 @@ class PadOp : public OpKernel {
 
     // Compute the shape of the output tensor, and allocate it.
     TensorShape output_shape;
-    TTypes<int32>::ConstMatrix paddings = in1.matrix<int32>();
+    typename TTypes<Tpadding>::ConstMatrix paddings = in1.matrix<Tpadding>();
     for (int d = 0; d < fixed_dims; ++d) {
-      const int32 before_d = paddings(d, 0);  // Pad before existing elements.
-      const int32 after_d = paddings(d, 1);   // Pad after existing elements.
+      const Tpadding before_d =
+          paddings(d, 0);                       // Pad before existing elements.
+      const Tpadding after_d = paddings(d, 1);  // Pad after existing elements.
       OP_REQUIRES(context, before_d >= 0 && after_d >= 0,
                   errors::InvalidArgument("Paddings must be non-negative: ",
                                           before_d, " ", after_d));
@@ -142,32 +143,47 @@ class PadOp : public OpKernel {
   template <int Dims>
   void Operate(OpKernelContext* context,
                typename TTypes<T, Dims>::ConstTensor input,
-               TTypes<int32>::ConstMatrix paddings, T pad_value,
+               typename TTypes<Tpadding>::ConstMatrix paddings, T pad_value,
                Tensor* output) {
     CHECK_EQ(Dims, paddings.dimension(0));
     CHECK_EQ(2, paddings.dimension(1));
-    Eigen::array<Eigen::IndexPair<int32>, Dims> paddings_array;
+    Eigen::array<Eigen::IndexPair<Tpadding>, Dims> paddings_array;
     for (int i = 0; i < Dims; ++i) {
       paddings_array[i] = {paddings(i, 0), paddings(i, 1)};
     }
-    functor::Pad<Device, T, Dims> functor;
+    functor::Pad<Device, T, Tpadding, Dims> functor;
     functor(context->eigen_device<Device>(), output->tensor<T, Dims>(), input,
             paddings_array, pad_value);
   }
 };
 
-#define REGISTER_KERNEL(type)                                 \
-  REGISTER_KERNEL_BUILDER(Name("Pad")                         \
-                              .Device(DEVICE_CPU)             \
-                              .TypeConstraint<type>("T")      \
-                              .HostMemory("paddings"),        \
-                          PadOp<CPUDevice, type>);            \
-  REGISTER_KERNEL_BUILDER(Name("PadV2")                       \
-                              .Device(DEVICE_CPU)             \
-                              .TypeConstraint<type>("T")      \
-                              .HostMemory("paddings")         \
-                              .HostMemory("constant_values"), \
-                          PadOp<CPUDevice, type>);
+#define REGISTER_KERNEL(type)                                     \
+  REGISTER_KERNEL_BUILDER(Name("Pad")                             \
+                              .Device(DEVICE_CPU)                 \
+                              .TypeConstraint<type>("T")          \
+                              .TypeConstraint<int32>("Tpaddings") \
+                              .HostMemory("paddings"),            \
+                          PadOp<CPUDevice, type, int32>);         \
+  REGISTER_KERNEL_BUILDER(Name("Pad")                             \
+                              .Device(DEVICE_CPU)                 \
+                              .TypeConstraint<type>("T")          \
+                              .TypeConstraint<int64>("Tpaddings") \
+                              .HostMemory("paddings"),            \
+                          PadOp<CPUDevice, type, int64>);         \
+  REGISTER_KERNEL_BUILDER(Name("PadV2")                           \
+                              .Device(DEVICE_CPU)                 \
+                              .TypeConstraint<type>("T")          \
+                              .TypeConstraint<int32>("Tpaddings") \
+                              .HostMemory("paddings")             \
+                              .HostMemory("constant_values"),     \
+                          PadOp<CPUDevice, type, int32>);         \
+  REGISTER_KERNEL_BUILDER(Name("PadV2")                           \
+                              .Device(DEVICE_CPU)                 \
+                              .TypeConstraint<type>("T")          \
+                              .TypeConstraint<int64>("Tpaddings") \
+                              .HostMemory("paddings")             \
+                              .HostMemory("constant_values"),     \
+                          PadOp<CPUDevice, type, int64>);
 
 TF_CALL_POD_TYPES(REGISTER_KERNEL);
 #undef REGISTER_KERNEL
@@ -177,11 +193,17 @@ TF_CALL_POD_TYPES(REGISTER_KERNEL);
 namespace functor {
 #define DECLARE_GPU_SPEC(T, Dims)                                         \
   template <>                                                             \
-  void Pad<GPUDevice, T, Dims>::operator()(                               \
+  void Pad<GPUDevice, T, int32, Dims>::operator()(                        \
       const GPUDevice& d, typename TTypes<T, Dims>::Tensor output,        \
       typename TTypes<T, Dims>::ConstTensor input,                        \
       Eigen::array<Eigen::IndexPair<int32>, Dims> paddings, T pad_value); \
-  extern template struct Pad<GPUDevice, T, Dims>;
+  extern template struct Pad<GPUDevice, T, int32, Dims>;                  \
+  template <>                                                             \
+  void Pad<GPUDevice, T, int64, Dims>::operator()(                        \
+      const GPUDevice& d, typename TTypes<T, Dims>::Tensor output,        \
+      typename TTypes<T, Dims>::ConstTensor input,                        \
+      Eigen::array<Eigen::IndexPair<int64>, Dims> paddings, T pad_value); \
+  extern template struct Pad<GPUDevice, T, int64, Dims>;
 
 #define DECLARE_GPU_SPECS(T) \
   DECLARE_GPU_SPEC(T, 0);    \
@@ -202,14 +224,27 @@ TF_CALL_GPU_NUMBER_TYPES(DECLARE_GPU_SPECS);
                               .TypeConstraint<T>("T")             \
                               .TypeConstraint<int32>("Tpaddings") \
                               .HostMemory("paddings"),            \
-                          PadOp<GPUDevice, T>);                   \
+                          PadOp<GPUDevice, T, int32>);            \
+  REGISTER_KERNEL_BUILDER(Name("Pad")                             \
+                              .Device(DEVICE_GPU)                 \
+                              .TypeConstraint<T>("T")             \
+                              .TypeConstraint<int64>("Tpaddings") \
+                              .HostMemory("paddings"),            \
+                          PadOp<GPUDevice, T, int64>);            \
   REGISTER_KERNEL_BUILDER(Name("PadV2")                           \
                               .Device(DEVICE_GPU)                 \
                               .TypeConstraint<T>("T")             \
                               .TypeConstraint<int32>("Tpaddings") \
                               .HostMemory("paddings")             \
                               .HostMemory("constant_values"),     \
-                          PadOp<GPUDevice, T>)
+                          PadOp<GPUDevice, T, int32>)             \
+  REGISTER_KERNEL_BUILDER(Name("PadV2")                           \
+                              .Device(DEVICE_GPU)                 \
+                              .TypeConstraint<T>("T")             \
+                              .TypeConstraint<int64>("Tpaddings") \
+                              .HostMemory("paddings")             \
+                              .HostMemory("constant_values"),     \
+                          PadOp<GPUDevice, T, int64>)
 
 TF_CALL_GPU_NUMBER_TYPES(REGISTER_GPU_KERNEL);
 
@@ -223,7 +258,15 @@ REGISTER_KERNEL_BUILDER(Name("Pad")
                             .HostMemory("input")
                             .HostMemory("paddings")
                             .HostMemory("output"),
-                        PadOp<CPUDevice, int32>);
+                        PadOp<CPUDevice, int32, int32>);
+REGISTER_KERNEL_BUILDER(Name("Pad")
+                            .Device(DEVICE_GPU)
+                            .TypeConstraint<int32>("T")
+                            .TypeConstraint<int64>("Tpaddings")
+                            .HostMemory("input")
+                            .HostMemory("paddings")
+                            .HostMemory("output"),
+                        PadOp<CPUDevice, int32, int64>);
 REGISTER_KERNEL_BUILDER(Name("PadV2")
                             .Device(DEVICE_GPU)
                             .TypeConstraint<int32>("T")
@@ -232,7 +275,16 @@ REGISTER_KERNEL_BUILDER(Name("PadV2")
                             .HostMemory("paddings")
                             .HostMemory("constant_values")
                             .HostMemory("output"),
-                        PadOp<CPUDevice, int32>);
+                        PadOp<CPUDevice, int32, int32>);
+REGISTER_KERNEL_BUILDER(Name("PadV2")
+                            .Device(DEVICE_GPU)
+                            .TypeConstraint<int32>("T")
+                            .TypeConstraint<int64>("Tpaddings")
+                            .HostMemory("input")
+                            .HostMemory("paddings")
+                            .HostMemory("constant_values")
+                            .HostMemory("output"),
+                        PadOp<CPUDevice, int32, int64>);
 #endif
 
 #ifdef TENSORFLOW_USE_SYCL
@@ -243,14 +295,27 @@ REGISTER_KERNEL_BUILDER(Name("PadV2")
                               .TypeConstraint<T>("T")             \
                               .TypeConstraint<int32>("Tpaddings") \
                               .HostMemory("paddings"),            \
-                          PadOp<SYCLDevice, T>);                  \
+                          PadOp<SYCLDevice, T, int32>);           \
+  REGISTER_KERNEL_BUILDER(Name("Pad")                             \
+                              .Device(DEVICE_SYCL)                \
+                              .TypeConstraint<T>("T")             \
+                              .TypeConstraint<int64>("Tpaddings") \
+                              .HostMemory("paddings"),            \
+                          PadOp<SYCLDevice, T, int64>);           \
   REGISTER_KERNEL_BUILDER(Name("PadV2")                           \
                               .Device(DEVICE_SYCL)                \
                               .TypeConstraint<T>("T")             \
                               .TypeConstraint<int32>("Tpaddings") \
                               .HostMemory("paddings")             \
                               .HostMemory("constant_values"),     \
-                          PadOp<SYCLDevice, T>)
+                          PadOp<SYCLDevice, T, int32>)            \
+  REGISTER_KERNEL_BUILDER(Name("PadV2")                           \
+                              .Device(DEVICE_SYCL)                \
+                              .TypeConstraint<T>("T")             \
+                              .TypeConstraint<int64>("Tpaddings") \
+                              .HostMemory("paddings")             \
+                              .HostMemory("constant_values"),     \
+                          PadOp<SYCLDevice, T, int64>)
 
 TF_CALL_GPU_NUMBER_TYPES_NO_HALF(REGISTER_SYCL_KERNEL);
 REGISTER_KERNEL_BUILDER(Name("Pad")
@@ -260,7 +325,15 @@ REGISTER_KERNEL_BUILDER(Name("Pad")
                             .HostMemory("input")
                             .HostMemory("paddings")
                             .HostMemory("output"),
-                        PadOp<CPUDevice, int32>);
+                        PadOp<CPUDevice, int32, int32>);
+REGISTER_KERNEL_BUILDER(Name("Pad")
+                            .Device(DEVICE_SYCL)
+                            .TypeConstraint<int32>("T")
+                            .TypeConstraint<int64>("Tpaddings")
+                            .HostMemory("input")
+                            .HostMemory("paddings")
+                            .HostMemory("output"),
+                        PadOp<CPUDevice, int32, int64>);
 REGISTER_KERNEL_BUILDER(Name("PadV2")
                             .Device(DEVICE_SYCL)
                             .TypeConstraint<int32>("T")
@@ -269,8 +342,17 @@ REGISTER_KERNEL_BUILDER(Name("PadV2")
                             .HostMemory("paddings")
                             .HostMemory("constant_values")
                             .HostMemory("output"),
-                        PadOp<CPUDevice, int32>);
+                        PadOp<CPUDevice, int32, int32>);
+REGISTER_KERNEL_BUILDER(Name("PadV2")
+                            .Device(DEVICE_SYCL)
+                            .TypeConstraint<int32>("T")
+                            .TypeConstraint<int64>("Tpaddings")
+                            .HostMemory("input")
+                            .HostMemory("paddings")
+                            .HostMemory("constant_values")
+                            .HostMemory("output"),
+                        PadOp<CPUDevice, int32, int64>);
 #undef REGISTER_SYCL_KERNEL
-#endif // TENSORFLOW_USE_SYCL
+#endif  // TENSORFLOW_USE_SYCL
 
 }  // end namespace tensorflow
diff --git a/tensorflow/core/kernels/pad_op.h b/tensorflow/core/kernels/pad_op.h
index 95a7c9a3ae..ee9e0f0330 100644
--- a/tensorflow/core/kernels/pad_op.h
+++ b/tensorflow/core/kernels/pad_op.h
@@ -25,13 +25,13 @@ namespace tensorflow {
 namespace functor {
 
 // Functor used by PadOp to do the computations.
-template <typename Device, typename T, int Dims>
+template <typename Device, typename T, typename Tpadding, int Dims>
 struct Pad {
   // Pad "input" into "output", as specified by "paddings" and "pad_value".
   // See pad_op.cc for details.
   void operator()(const Device& d, typename TTypes<T, Dims>::Tensor output,
                   typename TTypes<T, Dims>::ConstTensor input,
-                  Eigen::array<Eigen::IndexPair<int32>, Dims> paddings,
+                  Eigen::array<Eigen::IndexPair<Tpadding>, Dims> paddings,
                   T pad_value) {
     if (Eigen::internal::is_same<Device, Eigen::GpuDevice>::value &&
         (output.size() <= std::numeric_limits<int32>::max())) {
@@ -42,12 +42,12 @@ struct Pad {
   }
 };
 
-template <typename Device, typename T>
-struct Pad<Device, T, 0> {
+template <typename Device, typename T, typename Tpadding>
+struct Pad<Device, T, Tpadding, 0> {
   // In the scalar case we simply copy the input.
   void operator()(const Device& d, typename TTypes<T, 0>::Tensor output,
                   typename TTypes<T, 0>::ConstTensor input,
-                  Eigen::array<Eigen::IndexPair<int32>, 0>, T) {
+                  Eigen::array<Eigen::IndexPair<Tpadding>, 0>, T) {
     output.device(d) = input;
   }
 };
diff --git a/tensorflow/core/kernels/pad_op_gpu.cu.cc b/tensorflow/core/kernels/pad_op_gpu.cu.cc
index f98631df17..613ad62825 100644
--- a/tensorflow/core/kernels/pad_op_gpu.cu.cc
+++ b/tensorflow/core/kernels/pad_op_gpu.cu.cc
@@ -26,14 +26,18 @@ namespace tensorflow {
 typedef Eigen::GpuDevice GPUDevice;
 
 // Definition of the GPU implementations declared in pad_op.cc.
-#define DEFINE_GPU_SPECS(T)                      \
-  template struct functor::Pad<GPUDevice, T, 0>; \
-  template struct functor::Pad<GPUDevice, T, 1>; \
-  template struct functor::Pad<GPUDevice, T, 2>; \
-  template struct functor::Pad<GPUDevice, T, 3>; \
-  template struct functor::Pad<GPUDevice, T, 4>; \
-  template struct functor::Pad<GPUDevice, T, 5>; \
-  template struct functor::Pad<GPUDevice, T, 6>;
+#define DEFINE_GPU_PAD_SPECS(T, Tpadding)                  \
+  template struct functor::Pad<GPUDevice, T, Tpadding, 0>; \
+  template struct functor::Pad<GPUDevice, T, Tpadding, 1>; \
+  template struct functor::Pad<GPUDevice, T, Tpadding, 2>; \
+  template struct functor::Pad<GPUDevice, T, Tpadding, 3>; \
+  template struct functor::Pad<GPUDevice, T, Tpadding, 4>; \
+  template struct functor::Pad<GPUDevice, T, Tpadding, 5>; \
+  template struct functor::Pad<GPUDevice, T, Tpadding, 6>;
+
+#define DEFINE_GPU_SPECS(T)      \
+  DEFINE_GPU_PAD_SPECS(T, int32) \
+  DEFINE_GPU_PAD_SPECS(T, int64)
 
 TF_CALL_GPU_NUMBER_TYPES(DEFINE_GPU_SPECS);
 
diff --git a/tensorflow/python/kernel_tests/pad_op_test.py b/tensorflow/python/kernel_tests/pad_op_test.py
index ca1f3f878f..1af43e6067 100644
--- a/tensorflow/python/kernel_tests/pad_op_test.py
+++ b/tensorflow/python/kernel_tests/pad_op_test.py
@@ -284,6 +284,15 @@ class PadOpTest(test.TestCase):
     self.assertAllEqual(inp, out)
     self.assertShapeEqual(inp, tf_val)
 
+  def testPadTypes(self):
+    for dtype in [dtypes.int32, dtypes.int64]:
+      paddings = np.zeros((0, 2))
+      inp = np.asarray(7)
+      with self.test_session(use_gpu=True):
+        tf_val = array_ops.pad(inp, constant_op.constant(paddings, dtype=dtype))
+        out = tf_val.eval()
+      self.assertAllEqual(inp, out)
+      self.assertShapeEqual(inp, tf_val)
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/ops/image_ops_test.py b/tensorflow/python/ops/image_ops_test.py
index b13b73edbb..d1554b399f 100644
--- a/tensorflow/python/ops/image_ops_test.py
+++ b/tensorflow/python/ops/image_ops_test.py
@@ -1374,6 +1374,25 @@ class PadToBoundingBoxTest(test_util.TensorFlowTestCase):
     y = image_ops.pad_to_bounding_box(image, 0, 0, height, width)
     self.assertEqual(y.get_shape().as_list(), post_shape)
 
+  def testInt64(self):
+    x = [1, 2, 3,
+         4, 5, 6,
+         7, 8, 9]
+    x_shape = [3, 3, 1]
+
+    y = [0, 0, 0,
+         1, 2, 3,
+         4, 5, 6,
+         7, 8, 9]
+    y_shape = [4, 3, 1]
+    x = np.array(x).reshape(x_shape)
+    y = np.array(y).reshape(y_shape)
+
+    i = constant_op.constant([1, 0, 4, 3], dtype=dtypes.int64)
+    y_tf = image_ops.pad_to_bounding_box(x, i[0], i[1], i[2], i[3])
+    with self.test_session(use_gpu=True):
+      self.assertAllClose(y, y_tf.eval())
+
   def testNoOp(self):
     x_shape = [10, 10, 10]
     x = np.random.uniform(size=x_shape)
-- 
GitLab


From b190a9ebfe102de586313565ecb75f03972d172f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 17 Oct 2017 14:43:08 -0700
Subject: [PATCH 481/909] Preserve metadata_ when using
 HloInstruction::CloneWithNewOperands.

PiperOrigin-RevId: 172517300
---
 .../compiler/xla/service/hlo_instruction.cc   | 146 +++++++++++-------
 .../xla/service/hlo_instruction_test.cc       |   3 +
 2 files changed, 93 insertions(+), 56 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_instruction.cc b/tensorflow/compiler/xla/service/hlo_instruction.cc
index 021e5881c8..f24953051a 100644
--- a/tensorflow/compiler/xla/service/hlo_instruction.cc
+++ b/tensorflow/compiler/xla/service/hlo_instruction.cc
@@ -962,6 +962,8 @@ std::unique_ptr<HloInstruction> HloInstruction::CloneWithNewOperands(
     VLOG(3) << "    " << new_operand->name();
   }
 
+  std::unique_ptr<HloInstruction> clone;
+
   // Explicitly call the factory for the instruction type. This is more robust
   // in the face of code changes than copying fields explicitly. This also
   // properly sets the user fields of the operands.
@@ -984,7 +986,8 @@ std::unique_ptr<HloInstruction> HloInstruction::CloneWithNewOperands(
     case HloOpcode::kSort:
     case HloOpcode::kTanh:
       CHECK_EQ(new_operands.size(), 1);
-      return CreateUnary(shape, opcode_, new_operands[0]);
+      clone = CreateUnary(shape, opcode_, new_operands[0]);
+      break;
     // Binary ops.
     case HloOpcode::kAdd:
     case HloOpcode::kDivide:
@@ -1007,118 +1010,148 @@ std::unique_ptr<HloInstruction> HloInstruction::CloneWithNewOperands(
     case HloOpcode::kShiftRightArithmetic:
     case HloOpcode::kShiftRightLogical:
       CHECK_EQ(new_operands.size(), 2);
-      return CreateBinary(shape, opcode_, new_operands[0], new_operands[1]);
+      clone = CreateBinary(shape, opcode_, new_operands[0], new_operands[1]);
+      break;
     // Ternary ops.
     case HloOpcode::kClamp:
     case HloOpcode::kSelect:
       CHECK_EQ(new_operands.size(), 3);
-      return CreateTernary(shape, opcode_, new_operands[0], new_operands[1],
-                           new_operands[2]);
+      clone = CreateTernary(shape, opcode_, new_operands[0], new_operands[1],
+                            new_operands[2]);
+      break;
     // Other supported ops.
     case HloOpcode::kBroadcast:
       CHECK_EQ(new_operands.size(), 1);
-      return CreateBroadcast(shape, new_operands[0], dimensions_);
+      clone = CreateBroadcast(shape, new_operands[0], dimensions_);
+      break;
     case HloOpcode::kCall:
-      return CreateCall(shape, new_operands, to_apply());
+      clone = CreateCall(shape, new_operands, to_apply());
+      break;
     case HloOpcode::kCustomCall:
-      return CreateCustomCall(shape, new_operands, custom_call_target_);
+      clone = CreateCustomCall(shape, new_operands, custom_call_target_);
+      break;
     case HloOpcode::kConcatenate:
-      return CreateConcatenate(shape, new_operands, dimensions(0));
+      clone = CreateConcatenate(shape, new_operands, dimensions(0));
+      break;
     case HloOpcode::kConvert:
       CHECK_EQ(new_operands.size(), 1);
-      return CreateConvert(shape, new_operands[0]);
+      clone = CreateConvert(shape, new_operands[0]);
+      break;
     case HloOpcode::kReducePrecision:
       CHECK_EQ(new_operands.size(), 1);
-      return CreateReducePrecision(shape, new_operands[0], exponent_bits_,
-                                   mantissa_bits_);
+      clone = CreateReducePrecision(shape, new_operands[0], exponent_bits_,
+                                    mantissa_bits_);
+      break;
     case HloOpcode::kConvolution:
       CHECK_EQ(new_operands.size(), 2);
-      return CreateConvolve(shape, new_operands[0], new_operands[1], *window_,
-                            *convolution_dimension_numbers_);
+      clone = CreateConvolve(shape, new_operands[0], new_operands[1], *window_,
+                             *convolution_dimension_numbers_);
+      break;
     case HloOpcode::kCrossReplicaSum:
       CHECK_EQ(new_operands.size(), 1);
-      return CreateCrossReplicaSum(shape, new_operands[0]);
+      clone = CreateCrossReplicaSum(shape, new_operands[0]);
+      break;
     case HloOpcode::kGetTupleElement:
       CHECK_EQ(new_operands.size(), 1);
-      return CreateGetTupleElement(shape, new_operands[0], tuple_index());
+      clone = CreateGetTupleElement(shape, new_operands[0], tuple_index());
+      break;
     case HloOpcode::kMap:
-      return CreateMap(shape, new_operands, to_apply());
+      clone = CreateMap(shape, new_operands, to_apply());
+      break;
     case HloOpcode::kPad:
       CHECK_EQ(new_operands.size(), 2);
-      return CreatePad(shape, new_operands[0], new_operands[1],
-                       *padding_config_);
+      clone =
+          CreatePad(shape, new_operands[0], new_operands[1], *padding_config_);
+      break;
     case HloOpcode::kReduce:
       CHECK_EQ(new_operands.size(), 2);
-      return CreateReduce(shape, new_operands[0], new_operands[1], dimensions_,
-                          to_apply());
+      clone = CreateReduce(shape, new_operands[0], new_operands[1], dimensions_,
+                           to_apply());
+      break;
     case HloOpcode::kReduceWindow:
       CHECK_EQ(new_operands.size(), 2);
-      return CreateReduceWindow(shape, new_operands[0], new_operands[1],
-                                *window_, to_apply());
+      clone = CreateReduceWindow(shape, new_operands[0], new_operands[1],
+                                 *window_, to_apply());
+      break;
     case HloOpcode::kSelectAndScatter:
       CHECK_EQ(new_operands.size(), 3);
-      return CreateSelectAndScatter(shape, new_operands[0], select(), *window_,
-                                    new_operands[1], new_operands[2],
-                                    scatter());
+      clone =
+          CreateSelectAndScatter(shape, new_operands[0], select(), *window_,
+                                 new_operands[1], new_operands[2], scatter());
+      break;
     case HloOpcode::kReverse:
       CHECK_EQ(new_operands.size(), 1);
-      return CreateReverse(shape, new_operands[0], dimensions_);
+      clone = CreateReverse(shape, new_operands[0], dimensions_);
+      break;
     case HloOpcode::kRng:
-      return CreateRng(shape, distribution_, new_operands);
+      clone = CreateRng(shape, distribution_, new_operands);
+      break;
     case HloOpcode::kReshape:
       CHECK_EQ(new_operands.size(), 1);
-      return CreateReshape(shape, new_operands[0]);
+      clone = CreateReshape(shape, new_operands[0]);
+      break;
     case HloOpcode::kSlice:
       CHECK_EQ(new_operands.size(), 1);
-      return CreateSlice(shape, new_operands[0], slice_starts_, slice_limits_,
-                         slice_strides_);
+      clone = CreateSlice(shape, new_operands[0], slice_starts_, slice_limits_,
+                          slice_strides_);
+      break;
     case HloOpcode::kDynamicSlice:
-      return CreateDynamicSlice(shape, new_operands[0], new_operands[1],
-                                dynamic_slice_sizes_);
+      clone = CreateDynamicSlice(shape, new_operands[0], new_operands[1],
+                                 dynamic_slice_sizes_);
+      break;
     case HloOpcode::kDynamicUpdateSlice:
       CHECK_EQ(new_operands.size(), 3);
-      return CreateDynamicUpdateSlice(shape, new_operands[0], new_operands[1],
-                                      new_operands[2]);
+      clone = CreateDynamicUpdateSlice(shape, new_operands[0], new_operands[1],
+                                       new_operands[2]);
+      break;
     case HloOpcode::kTranspose:
       CHECK_EQ(new_operands.size(), 1);
-      return CreateTranspose(shape, new_operands[0], dimensions_);
-    case HloOpcode::kTuple: {
-      auto new_tuple = CreateTuple(new_operands);
-      *new_tuple->mutable_shape() = shape;
-      return new_tuple;
-    }
+      clone = CreateTranspose(shape, new_operands[0], dimensions_);
+      break;
+    case HloOpcode::kTuple:
+      clone = CreateTuple(new_operands);
+      *clone->mutable_shape() = shape;
+      break;
     case HloOpcode::kWhile:
       CHECK_EQ(new_operands.size(), 1);
-      return CreateWhile(shape, while_condition(), while_body(),
-                         new_operands[0]);
+      clone =
+          CreateWhile(shape, while_condition(), while_body(), new_operands[0]);
+      break;
     case HloOpcode::kConstant:
-      return CreateConstant(literal_->CloneToUnique());
+      clone = CreateConstant(literal_->CloneToUnique());
+      break;
     case HloOpcode::kFusion:
-      return CloneFusionWithNewOperands(shape, new_operands);
+      clone = CloneFusionWithNewOperands(shape, new_operands);
+      break;
     case HloOpcode::kParameter:
-      return CreateParameter(parameter_number_, shape, parameter_name_);
+      clone = CreateParameter(parameter_number_, shape, parameter_name_);
+      break;
     case HloOpcode::kBatchNormTraining:
       CHECK_EQ(new_operands.size(), 3);
-      return CreateBatchNormTraining(shape, new_operands[0], new_operands[1],
-                                     new_operands[2], epsilon(),
-                                     feature_index());
-
+      clone =
+          CreateBatchNormTraining(shape, new_operands[0], new_operands[1],
+                                  new_operands[2], epsilon(), feature_index());
+      break;
     case HloOpcode::kBatchNormInference:
       CHECK_EQ(new_operands.size(), 5);
-      return CreateBatchNormInference(
+      clone = CreateBatchNormInference(
           shape, new_operands[0], new_operands[1], new_operands[2],
           new_operands[3], new_operands[4], epsilon(), feature_index());
+      break;
     case HloOpcode::kInfeed:
       CHECK_EQ(new_operands.size(), 0);
-      return CreateInfeed(shape, infeed_config());
+      clone = CreateInfeed(shape, infeed_config());
+      break;
     case HloOpcode::kOutfeed:
       CHECK_EQ(new_operands.size(), 1);
-      return CreateOutfeed(outfeed_shape_, new_operands[0], outfeed_config());
+      clone = CreateOutfeed(outfeed_shape_, new_operands[0], outfeed_config());
+      break;
     case HloOpcode::kBatchNormGrad:
       CHECK_EQ(new_operands.size(), 5);
-      return CreateBatchNormGrad(shape, new_operands[0], new_operands[1],
-                                 new_operands[2], new_operands[3],
-                                 new_operands[4], epsilon(), feature_index());
+      clone = CreateBatchNormGrad(shape, new_operands[0], new_operands[1],
+                                  new_operands[2], new_operands[3],
+                                  new_operands[4], epsilon(), feature_index());
+      break;
     case HloOpcode::kRecv:
     case HloOpcode::kSend:
     case HloOpcode::kUpdate:
@@ -1126,6 +1159,8 @@ std::unique_ptr<HloInstruction> HloInstruction::CloneWithNewOperands(
     case HloOpcode::kTrace:
       LOG(FATAL) << "Not yet implemented, clone: " << HloOpcodeString(opcode_);
   }
+  clone->set_metadata(metadata_);
+  return clone;
 }
 
 HloInstruction::~HloInstruction() {}
@@ -1168,7 +1203,6 @@ std::unique_ptr<HloInstruction> HloInstruction::Clone(
     }
   }
   clone->set_parent(parent_);
-  clone->set_metadata(metadata_);
   return clone;
 }
 
diff --git a/tensorflow/compiler/xla/service/hlo_instruction_test.cc b/tensorflow/compiler/xla/service/hlo_instruction_test.cc
index 45f9128eab..cdafc05d8c 100644
--- a/tensorflow/compiler/xla/service/hlo_instruction_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_instruction_test.cc
@@ -706,6 +706,9 @@ TEST_F(HloInstructionTest, PreserveMetadataInFusionAndClone) {
       metadata, fusion->fused_expression_root()->metadata()));
   EXPECT_TRUE(protobuf_util::ProtobufEquals(
       metadata, fusion->fused_expression_root()->operand(0)->metadata()));
+
+  auto cloned = fusion->CloneWithNewOperands(fusion->shape(), {});
+  EXPECT_TRUE(protobuf_util::ProtobufEquals(metadata, fusion->metadata()));
 }
 
 TEST_F(HloInstructionTest, PreserveOutfeedShapeThroughClone) {
-- 
GitLab


From e6623e3167867037fb486102deebea22e04318bd Mon Sep 17 00:00:00 2001
From: Mark Daoust <markdaoust@google.com>
Date: Tue, 17 Oct 2017 14:44:47 -0700
Subject: [PATCH 482/909] Link to Datasets doc in release notes

PiperOrigin-RevId: 172517507
---
 RELEASE.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/RELEASE.md b/RELEASE.md
index c5f1e8b309..2c6535c15d 100644
--- a/RELEASE.md
+++ b/RELEASE.md
@@ -1,7 +1,8 @@
 # Release 1.4.0
 
 ## Major Features And Improvements
-* `tf.data` is now part of the core TensorFlow API.
+* [`tf.data`](http://tensorflow.org/programmers_guide/datasets) is now part of
+  the core TensorFlow API.
   * The API is now subject to backwards compatibility guarantees.
   * For a guide to migrating from the `tf.contrib.data` API, see the
     [README] (https://github.com/tensorflow/tensorflow/blob/r1.4/tensorflow/contrib/data/README.md).
-- 
GitLab


From effa8692aebd738c50d2270f5a371528f0eca748 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 17 Oct 2017 14:51:07 -0700
Subject: [PATCH 483/909] A new flag `ignore_live_threads` is available on
 train. If set to True, it will ignore threads that remain running when
 tearing down infrastructure after successfully completing training, instead
 of throwing a RuntimeError.

PiperOrigin-RevId: 172518466
---
 tensorflow/contrib/slim/python/slim/learning.py    | 11 +++++++++--
 tensorflow/python/training/supervisor.py           | 14 +++++++++++---
 .../api/golden/tensorflow.train.-supervisor.pbtxt  |  4 ++--
 3 files changed, 22 insertions(+), 7 deletions(-)

diff --git a/tensorflow/contrib/slim/python/slim/learning.py b/tensorflow/contrib/slim/python/slim/learning.py
index 5ee014a1f1..def00b7618 100644
--- a/tensorflow/contrib/slim/python/slim/learning.py
+++ b/tensorflow/contrib/slim/python/slim/learning.py
@@ -552,7 +552,8 @@ def train(train_op,
           sync_optimizer=None,
           session_config=None,
           session_wrapper=None,
-          trace_every_n_steps=None):
+          trace_every_n_steps=None,
+          ignore_live_threads=False):
   """Runs a training loop using a TensorFlow supervisor.
 
   When the sync_optimizer is supplied, gradient updates are applied
@@ -615,6 +616,9 @@ def train(train_op,
     trace_every_n_steps: produce and save a `Timeline` in Chrome trace format
       and add it to the summaries every `trace_every_n_steps`. If None, no trace
       information will be produced or saved.
+    ignore_live_threads: If `True` ignores threads that remain running after
+      a grace period when stopping the supervisor, instead of raising a
+      RuntimeError.
 
   Returns:
     the value of the loss function after training.
@@ -772,7 +776,10 @@ def train(train_op,
         if logdir and sv.is_chief:
           logging.info('Finished training! Saving model to disk.')
           sv.saver.save(sess, sv.save_path, global_step=sv.global_step)
-          sv.stop(threads, close_summary_writer=True)
+          sv.stop(
+              threads,
+              close_summary_writer=True,
+              ignore_live_threads=ignore_live_threads)
 
     except errors.AbortedError:
       # Always re-run on AbortedError as it indicates a restart of one of the
diff --git a/tensorflow/python/training/supervisor.py b/tensorflow/python/training/supervisor.py
index cfdd03dc15..41dbf6b497 100644
--- a/tensorflow/python/training/supervisor.py
+++ b/tensorflow/python/training/supervisor.py
@@ -768,7 +768,10 @@ class Supervisor(object):
     looper.start()
     return looper
 
-  def stop(self, threads=None, close_summary_writer=True):
+  def stop(self,
+           threads=None,
+           close_summary_writer=True,
+           ignore_live_threads=False):
     """Stop the services and the coordinator.
 
     This does not close the session.
@@ -782,14 +785,19 @@ class Supervisor(object):
       close_summary_writer: Whether to close the `summary_writer`.  Defaults to
         `True` if the summary writer was created by the supervisor, `False`
         otherwise.
+      ignore_live_threads: If `True` ignores threads that remain running after
+        a grace period when joining threads via the coordinator, instead of
+        raising a RuntimeError.
     """
     self._coord.request_stop()
     try:
       # coord.join() re-raises the first reported exception; the "finally"
       # block ensures that we clean up whether or not an exception was
       # reported.
-      self._coord.join(threads,
-                       stop_grace_period_secs=self._stop_grace_secs)
+      self._coord.join(
+          threads,
+          stop_grace_period_secs=self._stop_grace_secs,
+          ignore_live_threads=ignore_live_threads)
     finally:
       # Close the writer last, in case one of the running threads was using it.
       if close_summary_writer and self._summary_writer:
diff --git a/tensorflow/tools/api/golden/tensorflow.train.-supervisor.pbtxt b/tensorflow/tools/api/golden/tensorflow.train.-supervisor.pbtxt
index cc9bd5c136..1f0e59a1ac 100644
--- a/tensorflow/tools/api/golden/tensorflow.train.-supervisor.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.train.-supervisor.pbtxt
@@ -88,7 +88,7 @@ tf_class {
   }
   member_method {
     name: "Stop"
-    argspec: "args=[\'self\', \'threads\', \'close_summary_writer\'], varargs=None, keywords=None, defaults=[\'None\', \'True\'], "
+    argspec: "args=[\'self\', \'threads\', \'close_summary_writer\', \'ignore_live_threads\'], varargs=None, keywords=None, defaults=[\'None\', \'True\', \'False\'], "
   }
   member_method {
     name: "StopOnException"
@@ -136,7 +136,7 @@ tf_class {
   }
   member_method {
     name: "stop"
-    argspec: "args=[\'self\', \'threads\', \'close_summary_writer\'], varargs=None, keywords=None, defaults=[\'None\', \'True\'], "
+    argspec: "args=[\'self\', \'threads\', \'close_summary_writer\', \'ignore_live_threads\'], varargs=None, keywords=None, defaults=[\'None\', \'True\', \'False\'], "
   }
   member_method {
     name: "stop_on_exception"
-- 
GitLab


From 2fb1f1d837d8f86f3ad753ea235a1b3a22ba195f Mon Sep 17 00:00:00 2001
From: Mustafa Ispir <ispir@google.com>
Date: Tue, 17 Oct 2017 14:57:04 -0700
Subject: [PATCH 484/909] Set estimator run_config default random seed to None.
 This will make it aligned with other parts of the TF. Many users are not
 aware of impact of non-random seed. For example it may lead to train only on
 a small fraction of training data due to preemptions. We're changing default
 behavior since we consider it as a bug fix.

PiperOrigin-RevId: 172519268
---
 tensorflow/python/estimator/run_config.py                       | 2 +-
 tensorflow/python/estimator/run_config_test.py                  | 2 +-
 .../tools/api/golden/tensorflow.estimator.-run-config.pbtxt     | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/estimator/run_config.py b/tensorflow/python/estimator/run_config.py
index 1820b2b2d4..372f01dc82 100644
--- a/tensorflow/python/estimator/run_config.py
+++ b/tensorflow/python/estimator/run_config.py
@@ -210,7 +210,7 @@ class RunConfig(object):
 
   def __init__(self,
                model_dir=None,
-               tf_random_seed=1,
+               tf_random_seed=None,
                save_summary_steps=100,
                save_checkpoints_steps=_USE_DEFAULT,
                save_checkpoints_secs=_USE_DEFAULT,
diff --git a/tensorflow/python/estimator/run_config_test.py b/tensorflow/python/estimator/run_config_test.py
index b3c917649f..ecc850d540 100644
--- a/tensorflow/python/estimator/run_config_test.py
+++ b/tensorflow/python/estimator/run_config_test.py
@@ -70,7 +70,7 @@ class RunConfigTest(test.TestCase):
     config = run_config_lib.RunConfig()
     self.assertIsNone(config.model_dir)
     self.assertIsNone(config.session_config)
-    self.assertEqual(1, config.tf_random_seed)
+    self.assertIsNone(config.tf_random_seed)
     self.assertEqual(100, config.save_summary_steps)
     self.assertEqual(600, config.save_checkpoints_secs)
     self.assertIsNone(config.save_checkpoints_steps)
diff --git a/tensorflow/tools/api/golden/tensorflow.estimator.-run-config.pbtxt b/tensorflow/tools/api/golden/tensorflow.estimator.-run-config.pbtxt
index 7ab094c999..d006ecb254 100644
--- a/tensorflow/tools/api/golden/tensorflow.estimator.-run-config.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.estimator.-run-config.pbtxt
@@ -76,7 +76,7 @@ tf_class {
   }
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'model_dir\', \'tf_random_seed\', \'save_summary_steps\', \'save_checkpoints_steps\', \'save_checkpoints_secs\', \'session_config\', \'keep_checkpoint_max\', \'keep_checkpoint_every_n_hours\', \'log_step_count_steps\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'100\', \'<object object instance>\', \'<object object instance>\', \'None\', \'5\', \'10000\', \'100\'], "
+    argspec: "args=[\'self\', \'model_dir\', \'tf_random_seed\', \'save_summary_steps\', \'save_checkpoints_steps\', \'save_checkpoints_secs\', \'session_config\', \'keep_checkpoint_max\', \'keep_checkpoint_every_n_hours\', \'log_step_count_steps\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'100\', \'<object object instance>\', \'<object object instance>\', \'None\', \'5\', \'10000\', \'100\'], "
   }
   member_method {
     name: "replace"
-- 
GitLab


From a925c8dcaf57506c0f7ad167aad6794a88878ca3 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 17 Oct 2017 15:03:01 -0700
Subject: [PATCH 485/909] Improve performance of tf.space_to_depth and
 tf.depth_to_space for typical block sizes of NCHW data layout on GPU with a
 loop kernel.

PiperOrigin-RevId: 172520132
---
 .../core/kernels/depthtospace_op_gpu.cu.cc    | 79 +++++++++++++++++-
 .../core/kernels/spacetodepth_op_gpu.cu.cc    | 83 +++++++++++++++++--
 2 files changed, 151 insertions(+), 11 deletions(-)

diff --git a/tensorflow/core/kernels/depthtospace_op_gpu.cu.cc b/tensorflow/core/kernels/depthtospace_op_gpu.cu.cc
index 357c1f1be4..7a66285383 100644
--- a/tensorflow/core/kernels/depthtospace_op_gpu.cu.cc
+++ b/tensorflow/core/kernels/depthtospace_op_gpu.cu.cc
@@ -67,7 +67,6 @@ __global__ void D2S_NCHW(const int32 nthreads,
                          const int block_size, const int input_width,
                          const int output_depth_by_input_height,
                          dtype* __restrict__ output_ptr) {
-  // TODO(pauldonnelly): Implement more optimized kernels.
   CUDA_1D_KERNEL_LOOP(input_idx, nthreads) {
     // We will be converting the image from ordering:
     // n, bY, bX, oC, iY, iX    (== input_idx)   to
@@ -99,6 +98,47 @@ __global__ void D2S_NCHW(const int32 nthreads,
   }
 }
 
+template <typename dtype, int block_size>
+__global__ void D2S_NCHW_LOOP(const int32 nthreads,
+                              const dtype* __restrict__ input,
+                              const int input_width, const int output_width,
+                              const int output_depth_by_input_area,
+                              const int input_depth_by_input_area,
+                              dtype* __restrict__ output) {
+  CUDA_1D_KERNEL_LOOP(thread_idx, nthreads) {
+    // We will be converting the image from ordering:
+    // n, bY, bX, oC, iY, iX   to
+    // n, oC, iY, bY, iX, bX
+
+    // We assume thread_idx encodes n_oC_iY_iX, and use an unrolled loop over
+    // bY and bX coordinates within the block. This kernel is significantly
+    // more performant than the D2S_NCHW kernel.
+    //   A likely explanation of the improvement is that although both kernels
+    // get input coalescing, this one would write the output data more densely
+    // per warp, so would benefit assuming delayed cache writeback is used.
+
+    const int n_oC_iY = thread_idx / input_width;
+    const int iX = thread_idx - n_oC_iY * input_width;
+
+    const int n = thread_idx / output_depth_by_input_area;
+    const int oC_iY_iX = thread_idx - n * output_depth_by_input_area;
+
+    // Recombine the components and apply to the input and output pointers.
+    auto input_ptr = input + n * input_depth_by_input_area + oC_iY_iX;
+    auto output_ptr = output + (n_oC_iY * output_width + iX) * block_size;
+
+#pragma unroll
+    // Copy a patch of data to the output batch image.
+    for (int bY = 0; bY < block_size; ++bY) {
+#pragma unroll
+      for (int bX = 0; bX < block_size; ++bX) {
+        output_ptr[bY * output_width + bX] = ldg(
+            input_ptr + (bY * block_size + bX) * output_depth_by_input_area);
+      }
+    }
+  }
+}
+
 }  // namespace
 
 // Specialization of DepthToSpaceOpFunctor for a GPUDevice.
@@ -139,10 +179,41 @@ struct DepthToSpaceOpFunctor<GPUDevice, T, FORMAT_NCHW> {
     const int input_height = input.dimension(2);
     const int input_width = input.dimension(3);
     const int output_depth = output.dimension(1);
-    const int total_count =
-        batch_size * input_height * input_width * input_depth;
+    const int input_area = input_width * input_height;
+    const int input_depth_by_input_area = input_depth * input_area;
+
+    // We improve performance by generating instantiations of the loop kernel
+    // for the most common block sizes.
+    if (block_size <= 4) {
+      const int output_width = output.dimension(3);
+      const int output_depth_by_input_area = output_depth * input_area;
+      const int total_count = batch_size * output_depth_by_input_area;
+      CudaLaunchConfig config = GetCudaLaunchConfig(total_count, d);
+      switch (block_size) {
+        case 2:
+          return D2S_NCHW_LOOP<T, 2>
+              <<<config.block_count, config.thread_per_block, 0, d.stream()>>>(
+                  total_count, input.data(), input_width, output_width,
+                  output_depth_by_input_area, input_depth_by_input_area,
+                  output.data());
+        case 3:
+          return D2S_NCHW_LOOP<T, 3>
+              <<<config.block_count, config.thread_per_block, 0, d.stream()>>>(
+                  total_count, input.data(), input_width, output_width,
+                  output_depth_by_input_area, input_depth_by_input_area,
+                  output.data());
+        case 4:
+          return D2S_NCHW_LOOP<T, 4>
+              <<<config.block_count, config.thread_per_block, 0, d.stream()>>>(
+                  total_count, input.data(), input_width, output_width,
+                  output_depth_by_input_area, input_depth_by_input_area,
+                  output.data());
+      }
+    }
+
+    // Other block sizes are processed by the generic kernel.
+    const int total_count = batch_size * input_depth_by_input_area;
     auto config = GetCudaLaunchConfig(total_count, d);
-
     D2S_NCHW<<<config.block_count, config.thread_per_block, 0, d.stream()>>>(
         config.virtual_thread_count, input.data(), block_size, input_width,
         output_depth * input_height, output.data());
diff --git a/tensorflow/core/kernels/spacetodepth_op_gpu.cu.cc b/tensorflow/core/kernels/spacetodepth_op_gpu.cu.cc
index 94c7a0a3f6..a1a01e8813 100644
--- a/tensorflow/core/kernels/spacetodepth_op_gpu.cu.cc
+++ b/tensorflow/core/kernels/spacetodepth_op_gpu.cu.cc
@@ -66,10 +66,6 @@ __global__ void S2D_NCHW(const int32 nthreads,
                          const int block_size, const int output_width,
                          const int input_depth_by_output_height,
                          dtype* __restrict__ output_ptr) {
-  // TODO(pauldonnelly): This kernel gets input coalescing, but not output
-  // coalescing. We could use shared memory to get both. It may also help
-  // to amortize the address calculations via an inner loop over block_size.
-  // A template parameter for the block_size is another potential optimization.
   CUDA_1D_KERNEL_LOOP(input_idx, nthreads) {
     // We assume both the input and output are packed NCHW tensors.
     // input_idx represents an index within the flattened input tensor.
@@ -100,6 +96,48 @@ __global__ void S2D_NCHW(const int32 nthreads,
   }
 }
 
+// Space2Depth kernel for FORMAT_NCHW using a loop over block area.
+// See 'spacetodepth_op.h' for functional specification.
+template <typename dtype, int block_size>
+__global__ void S2D_NCHW_LOOP(const int32 nthreads,
+                              const dtype* __restrict__ input,
+                              const int output_width, const int input_width,
+                              const int input_depth_by_output_area,
+                              const int output_depth_by_output_area,
+                              dtype* __restrict__ output) {
+  CUDA_1D_KERNEL_LOOP(thread_idx, nthreads) {
+    // We will be converting the image from ordering:
+    // n, iC, oY, bY, oX, bX   (== input index) to
+    // n, bY, bX, iC, oY, oX   (== output index)
+
+    // We assume thread_idx encodes n_iC_oY_oX, and use an unrolled loop over
+    // bY and bX coordinates within the block. This kernel gets a small
+    // performance improvement compared with S2D_NCHW due to a denser access
+    // pattern on the input side. (Note: the equivalent D2S kernel gets a larger
+    // improvement as a denser pattern on the output side makes more
+    // difference).
+
+    const int n_iC_oY = thread_idx / output_width;
+    const int oX = thread_idx - n_iC_oY * output_width;
+    const int n = thread_idx / input_depth_by_output_area;
+    const int iC_oY_oX = thread_idx - n * input_depth_by_output_area;
+
+    // Recombine the components and apply to the input and output pointers.
+    auto input_ptr = input + (n_iC_oY * input_width + oX) * block_size;
+    auto output_ptr = output + n * output_depth_by_output_area + iC_oY_oX;
+
+#pragma unroll
+    // Copy a patch of data to the output batch image.
+    for (int bY = 0; bY < block_size; ++bY) {
+#pragma unroll
+      for (int bX = 0; bX < block_size; ++bX) {
+        output_ptr[(bY * block_size + bX) * input_depth_by_output_area] =
+            ldg(input_ptr + bY * input_width + bX);
+      }
+    }
+  }
+}
+
 // Specialization of SpaceToDepthOpFunctor for a CPUDevice.
 namespace functor {
 template <typename T>
@@ -137,9 +175,40 @@ struct SpaceToDepthOpFunctor<GPUDevice, T, FORMAT_NCHW> {
     const int output_depth = output.dimension(1);
     const int output_height = output.dimension(2);
     const int output_width = output.dimension(3);
-
-    const int total_count =
-        batch_size * output_height * output_width * output_depth;
+    const int output_area = output_width * output_height;
+    const int output_depth_by_output_area = output_depth * output_area;
+
+    // We improve performance by generating instantiations of the loop kernel
+    // for the most common block sizes.
+    if (block_size <= 4) {
+      const int input_width = input.dimension(3);
+      const int input_depth_by_output_area = input_depth * output_area;
+      const int total_count = batch_size * input_depth_by_output_area;
+      CudaLaunchConfig config = GetCudaLaunchConfig(total_count, d);
+      switch (block_size) {
+        case 2:
+          return S2D_NCHW_LOOP<T, 2>
+              <<<config.block_count, config.thread_per_block, 0, d.stream()>>>(
+                  total_count, input.data(), output_width, input_width,
+                  input_depth_by_output_area, output_depth_by_output_area,
+                  output.data());
+        case 3:
+          return S2D_NCHW_LOOP<T, 3>
+              <<<config.block_count, config.thread_per_block, 0, d.stream()>>>(
+                  total_count, input.data(), output_width, input_width,
+                  input_depth_by_output_area, output_depth_by_output_area,
+                  output.data());
+        case 4:
+          return S2D_NCHW_LOOP<T, 4>
+              <<<config.block_count, config.thread_per_block, 0, d.stream()>>>(
+                  total_count, input.data(), output_width, input_width,
+                  input_depth_by_output_area, output_depth_by_output_area,
+                  output.data());
+      }
+    }
+
+    // Other block sizes are processed by the generic kernel.
+    const int total_count = batch_size * output_depth_by_output_area;
     CudaLaunchConfig config = GetCudaLaunchConfig(total_count, d);
     S2D_NCHW<<<config.block_count, config.thread_per_block, 0, d.stream()>>>(
         config.virtual_thread_count, input.data(), block_size, output_width,
-- 
GitLab


From 110dfa8953e7e9b625681744fde51f30ace2aecd Mon Sep 17 00:00:00 2001
From: "Joshua V. Dillon" <jvdillon@google.com>
Date: Tue, 17 Oct 2017 15:11:27 -0700
Subject: [PATCH 486/909] Bugfix: add `fill_triangular`,
 `reduce_weighted_logsumexp`, `tridiag` to
 `tf.contrib.distributions.__init__`.

PiperOrigin-RevId: 172521590
---
 tensorflow/contrib/distributions/__init__.py           | 10 +++++++---
 .../contrib/distributions/python/ops/mvn_tril.py       |  8 ++++++++
 2 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/tensorflow/contrib/distributions/__init__.py b/tensorflow/contrib/distributions/__init__.py
index f33cc1de0a..16f6533e57 100644
--- a/tensorflow/contrib/distributions/__init__.py
+++ b/tensorflow/contrib/distributions/__init__.py
@@ -28,8 +28,11 @@ from tensorflow.contrib.distributions.python.ops.chi2 import *
 from tensorflow.contrib.distributions.python.ops.conditional_distribution import *
 from tensorflow.contrib.distributions.python.ops.conditional_transformed_distribution import *
 from tensorflow.contrib.distributions.python.ops.deterministic import *
+from tensorflow.contrib.distributions.python.ops.distribution_util import fill_triangular
 from tensorflow.contrib.distributions.python.ops.distribution_util import matrix_diag_transform
+from tensorflow.contrib.distributions.python.ops.distribution_util import reduce_weighted_logsumexp
 from tensorflow.contrib.distributions.python.ops.distribution_util import softplus_inverse
+from tensorflow.contrib.distributions.python.ops.distribution_util import tridiag
 from tensorflow.contrib.distributions.python.ops.estimator import *
 from tensorflow.contrib.distributions.python.ops.geometric import *
 from tensorflow.contrib.distributions.python.ops.independent import *
@@ -140,13 +143,14 @@ _allowed_symbols = [
     'RelaxedOneHotCategorical',
     'kl_divergence',
     'RegisterKL',
-    'matrix_diag_transform',
     'fill_triangular',
+    'matrix_diag_transform',
+    'reduce_weighted_logsumexp',
+    'softplus_inverse',
+    'tridiag',
     'normal_conjugates_known_scale_posterior',
     'normal_conjugates_known_scale_predictive',
-    'softplus_inverse',
     'percentile',
-    'reduce_weighted_logsumexp',
     'assign_moving_mean_variance',
     'assign_log_moving_mean_exp',
     'moving_mean_variance',
diff --git a/tensorflow/contrib/distributions/python/ops/mvn_tril.py b/tensorflow/contrib/distributions/python/ops/mvn_tril.py
index e3d68f6b4c..260dcc18f5 100644
--- a/tensorflow/contrib/distributions/python/ops/mvn_tril.py
+++ b/tensorflow/contrib/distributions/python/ops/mvn_tril.py
@@ -121,6 +121,14 @@ class MultivariateNormalTriL(
        [-10, 0, 9]]     # shape: [2, 3]
   mvn.prob(x).eval()    # shape: [2]
 
+  # Instantiate a "learnable" MVN.
+  dims = 4
+  with tf.variable_scope("model"):
+    mvn = ds.MultivariateNormalTriL(
+        loc=tf.get_variable(shape=[dims], dtype=tf.float32, name="mu"),
+        scale_tril=ds.fill_triangular(
+            tf.get_variable(shape=[dims * (dims + 1) / 2],
+                            dtype=tf.float32, name="chol_Sigma")))
   ```
 
   """
-- 
GitLab


From 84edca032719d2aa1da465e26c51d32c205df3bc Mon Sep 17 00:00:00 2001
From: Guangda Lai <laigd@google.com>
Date: Tue, 17 Oct 2017 15:18:58 -0700
Subject: [PATCH 487/909] Fix a bug in BFCAllocator::Log2FloorNonZero() under
 non gnuc non windows mode, where it returns the bit index of the most
 significant '1' bit, starting from 1, but the expected result should start
 from 0 (i.e. it should return floor(log2(n)) ).

PiperOrigin-RevId: 172522745
---
 .../core/common_runtime/bfc_allocator.h       | 20 ++++++++-----
 .../gpu/gpu_bfc_allocator_test.cc             | 28 +++++++++++++++----
 2 files changed, 35 insertions(+), 13 deletions(-)

diff --git a/tensorflow/core/common_runtime/bfc_allocator.h b/tensorflow/core/common_runtime/bfc_allocator.h
index 326e0ffe40..20fa05f0d2 100644
--- a/tensorflow/core/common_runtime/bfc_allocator.h
+++ b/tensorflow/core/common_runtime/bfc_allocator.h
@@ -362,6 +362,17 @@ class BFCAllocator : public VisitableAllocator {
 
   // Structures immutable after construction
   size_t memory_limit_ = 0;
+
+  inline int Log2FloorNonZeroSlow(uint64 n) {
+    int r = 0;
+    while (n > 0) {
+      r++;
+      n >>= 1;
+    }
+    return r - 1;
+  }
+
+  // Returns floor(log2(n)).
   inline int Log2FloorNonZero(uint64 n) {
 #if defined(__GNUC__)
     return 63 ^ __builtin_clzll(n);
@@ -370,12 +381,7 @@ class BFCAllocator : public VisitableAllocator {
     _BitScanReverse64(&index, n);
     return index;
 #else
-    int r = 0;
-    while (n > 0) {
-      r++;
-      n >>= 1;
-    }
-    return r;
+    return Log2FloorNonZeroSlow(n);
 #endif
   }
 
@@ -425,7 +431,7 @@ class BFCAllocator : public VisitableAllocator {
   // Stats.
   AllocatorStats stats_ GUARDED_BY(lock_);
 
-  friend class GPUBFCAllocatorBinDebugInfoTest;
+  friend class GPUBFCAllocatorPrivateMethodsTest;
   TF_DISALLOW_COPY_AND_ASSIGN(BFCAllocator);
 };
 
diff --git a/tensorflow/core/common_runtime/gpu/gpu_bfc_allocator_test.cc b/tensorflow/core/common_runtime/gpu/gpu_bfc_allocator_test.cc
index b7554e5b82..00ef130d34 100644
--- a/tensorflow/core/common_runtime/gpu/gpu_bfc_allocator_test.cc
+++ b/tensorflow/core/common_runtime/gpu/gpu_bfc_allocator_test.cc
@@ -354,12 +354,13 @@ BENCHMARK(BM_AllocationDelayed)->Arg(1)->Arg(10)->Arg(100)->Arg(1000);
 
 }  // namespace
 
-class GPUBFCAllocatorBinDebugInfoTest : public ::testing::Test {
+class GPUBFCAllocatorPrivateMethodsTest : public ::testing::Test {
  protected:
-  // This test method is called from a test. The reason for this is that this
-  // class is a friend class to BFCAllocator, but tests are not, so only this
-  // method can access the type BFCAllocator::BinDebugInfo.
-  void testBinDebugInfo() {
+  // The following test methods are called from tests. The reason for this is
+  // that this class is a friend class to BFCAllocator, but tests are not, so
+  // only methods inside this class can access private members of BFCAllocator.
+
+  void TestBinDebugInfo() {
     GPUBFCAllocator a(0, 1 << 30);
 
     std::vector<void*> initial_ptrs;
@@ -436,9 +437,24 @@ class GPUBFCAllocatorBinDebugInfoTest : public ::testing::Test {
       }
     }
   }
+
+  void TestLog2FloorNonZeroSlow() {
+    GPUBFCAllocator a(0 /* device_id */, 1 /* total_memory */);
+    EXPECT_EQ(-1, a.Log2FloorNonZeroSlow(0));
+    EXPECT_EQ(0, a.Log2FloorNonZeroSlow(1));
+    EXPECT_EQ(1, a.Log2FloorNonZeroSlow(2));
+    EXPECT_EQ(1, a.Log2FloorNonZeroSlow(3));
+    EXPECT_EQ(9, a.Log2FloorNonZeroSlow(1023));
+    EXPECT_EQ(10, a.Log2FloorNonZeroSlow(1024));
+    EXPECT_EQ(10, a.Log2FloorNonZeroSlow(1025));
+  }
 };
 
-TEST_F(GPUBFCAllocatorBinDebugInfoTest, BinDebugInfo) { testBinDebugInfo(); }
+TEST_F(GPUBFCAllocatorPrivateMethodsTest, BinDebugInfo) { TestBinDebugInfo(); }
+
+TEST_F(GPUBFCAllocatorPrivateMethodsTest, Log2FloorNonZeroSlow) {
+  TestLog2FloorNonZeroSlow();
+}
 
 }  // namespace tensorflow
 
-- 
GitLab


From 109133639706026d2bd944dd0a0c8fcae0d4fac6 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 17 Oct 2017 15:47:49 -0700
Subject: [PATCH 488/909] Add environment variable to enable setting of CUDA
 context flags.

PiperOrigin-RevId: 172527804
---
 .../stream_executor/cuda/cuda_platform.cc     | 41 ++++++++++++++++++-
 1 file changed, 39 insertions(+), 2 deletions(-)

diff --git a/tensorflow/stream_executor/cuda/cuda_platform.cc b/tensorflow/stream_executor/cuda/cuda_platform.cc
index ee9df5b7de..d69953f557 100644
--- a/tensorflow/stream_executor/cuda/cuda_platform.cc
+++ b/tensorflow/stream_executor/cuda/cuda_platform.cc
@@ -27,6 +27,43 @@ limitations under the License.
 namespace perftools {
 namespace gputools {
 namespace cuda {
+namespace {
+
+// Synchronize with spinlocks.
+const char kScheduleSpinString[] = "spin";
+// Synchronize with spinlocks that also call CPU yield instructions.
+const char kScheduleYieldString[] = "yield";
+// Synchronize with a "synchronization primitive" (e.g. mutex).
+const char kScheduleBlockingSyncString[] = "blocking_sync";
+
+const DeviceOptions GetDeviceOptionsFromEnv() {
+  const char* gpu_schedule_string =
+      std::getenv("TF_CUDA_PLATFORM_GPU_DEVICE_SCHEDULE");
+
+  if (gpu_schedule_string == nullptr) {
+    return perftools::gputools::DeviceOptions::Default();
+  }
+
+  unsigned device_flags = 0;
+  if (strcasecmp(kScheduleSpinString, gpu_schedule_string) == 0) {
+    device_flags = perftools::gputools::DeviceOptions::kScheduleSpin;
+  } else if (strcasecmp(kScheduleYieldString, gpu_schedule_string) == 0) {
+    device_flags = perftools::gputools::DeviceOptions::kScheduleYield;
+  } else if (strcasecmp(kScheduleBlockingSyncString, gpu_schedule_string) ==
+             0) {
+    device_flags = perftools::gputools::DeviceOptions::kScheduleBlockingSync;
+  } else {
+    LOG(QFATAL) << "Unknown option for environment variable "
+                   "TF_CUDA_PLATFORM_GPU_DEVICE_SCHEDULE "
+                << gpu_schedule_string << " should be one of {"
+                << kScheduleBlockingSyncString << ", " << kScheduleSpinString
+                << ", " << kScheduleYieldString << "}";
+  }
+
+  return perftools::gputools::DeviceOptions(device_flags);
+}
+
+}  // namespace
 
 CudaPlatform::CudaPlatform()
     : name_("CUDA"), min_numa_node_(0), limit_numa_node_(0) {}
@@ -112,7 +149,7 @@ port::StatusOr<StreamExecutor*> CudaPlatform::ExecutorForDevice(int ordinal) {
   StreamExecutorConfig config;
   config.ordinal = ordinal;
   config.plugin_config = PluginConfig();
-  config.device_options = DeviceOptions::Default();
+  config.device_options = GetDeviceOptionsFromEnv();
   return GetExecutor(config);
 }
 
@@ -121,7 +158,7 @@ port::StatusOr<StreamExecutor*> CudaPlatform::ExecutorForDeviceWithPluginConfig(
   StreamExecutorConfig config;
   config.ordinal = device_ordinal;
   config.plugin_config = plugin_config;
-  config.device_options = DeviceOptions::Default();
+  config.device_options = GetDeviceOptionsFromEnv();
   return GetExecutor(config);
 }
 
-- 
GitLab


From 4c7e02c082fdf3b4b04e42f1880cf6e0ff4fc409 Mon Sep 17 00:00:00 2001
From: Peter Hawkins <phawkins@google.com>
Date: Tue, 17 Oct 2017 16:05:14 -0700
Subject: [PATCH 489/909] Add bitwise LeftShift (aka. tf.bitwise.left_shift)
 and RightShift (tf.bitwise_right_shift) operators to Tensorflow. Their
 semantics are intended to mirror numpy.left_shift and numpy.right_shift.

Fix a couple of missing cases of missing uint32/uint64 support exposed when adding tests for bitshifts.

PiperOrigin-RevId: 172530375
---
 tensorflow/compiler/tests/binary_ops_test.py  | 16 +++++++
 .../compiler/tf2xla/kernels/binary_ops.cc     |  7 +++
 tensorflow/contrib/makefile/tf_op_files.txt   |  2 +
 tensorflow/core/framework/types.cc            | 12 +++++
 tensorflow/core/framework/types.h             |  3 ++
 tensorflow/core/kernels/BUILD                 |  2 +
 .../kernels/cwise_op_gpu_left_shift.cu.cc     | 27 +++++++++++
 .../kernels/cwise_op_gpu_right_shift.cu.cc    | 27 +++++++++++
 .../core/kernels/cwise_op_left_shift.cc       | 44 ++++++++++++++++++
 .../core/kernels/cwise_op_right_shift.cc      | 44 ++++++++++++++++++
 tensorflow/core/kernels/cwise_ops.h           | 46 +++++++++++++++++++
 tensorflow/core/ops/bitwise_ops.cc            | 33 +++++++++----
 .../python/framework/fast_tensor_util.pyx     | 12 +++++
 tensorflow/python/framework/tensor_util.py    | 16 +++++--
 tensorflow/python/ops/bitwise_ops.py          |  4 ++
 tensorflow/python/ops/bitwise_ops_test.py     | 42 +++++++++++++++++
 .../tools/api/golden/tensorflow.bitwise.pbtxt |  8 ++++
 17 files changed, 334 insertions(+), 11 deletions(-)
 create mode 100644 tensorflow/core/kernels/cwise_op_gpu_left_shift.cu.cc
 create mode 100644 tensorflow/core/kernels/cwise_op_gpu_right_shift.cu.cc
 create mode 100644 tensorflow/core/kernels/cwise_op_left_shift.cc
 create mode 100644 tensorflow/core/kernels/cwise_op_right_shift.cc

diff --git a/tensorflow/compiler/tests/binary_ops_test.py b/tensorflow/compiler/tests/binary_ops_test.py
index 44b32b1668..b387467246 100644
--- a/tensorflow/compiler/tests/binary_ops_test.py
+++ b/tensorflow/compiler/tests/binary_ops_test.py
@@ -209,6 +209,22 @@ class BinaryOpsTest(XLATestCase):
           np.array([0b0, 0b101, 0b1001], dtype=dtype),
           expected=np.array([0b1, 0b101, 0b1001], dtype=dtype))
 
+      lhs = np.array([0, 5, 3, 14], dtype=dtype)
+      rhs = np.array([5, 0, 7, 11], dtype=dtype)
+      self._testBinary(
+          bitwise_ops.left_shift, lhs, rhs,
+          expected=np.left_shift(lhs, rhs))
+      self._testBinary(
+          bitwise_ops.right_shift, lhs, rhs,
+          expected=np.right_shift(lhs, rhs))
+
+      if dtype in [np.int8, np.int16, np.int32, np.int64]:
+        lhs = np.array([-1, -5, -3, -14], dtype=dtype)
+        rhs = np.array([5, 0, 1, 11], dtype=dtype)
+        self._testBinary(
+            bitwise_ops.right_shift, lhs, rhs,
+            expected=np.right_shift(lhs, rhs))
+
   def testNumericOps(self):
     for dtype in self.numeric_types:
       self._testBinary(
diff --git a/tensorflow/compiler/tf2xla/kernels/binary_ops.cc b/tensorflow/compiler/tf2xla/kernels/binary_ops.cc
index d635507989..4673bbda14 100644
--- a/tensorflow/compiler/tf2xla/kernels/binary_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/binary_ops.cc
@@ -98,6 +98,13 @@ XLA_MAKE_BINARY(FloorMod,
 
 XLA_MAKE_BINARY(BitwiseAnd, b->And(lhs, rhs, extend_dimensions));
 XLA_MAKE_BINARY(BitwiseOr, b->Or(lhs, rhs, extend_dimensions));
+
+XLA_MAKE_BINARY(LeftShift, b->ShiftLeft(lhs, rhs, extend_dimensions));
+XLA_MAKE_BINARY(RightShift,
+                (DataTypeIsUnsigned(ctx->input_type(0))
+                     ? b->ShiftRightLogical(lhs, rhs, extend_dimensions)
+                     : b->ShiftRightArithmetic(lhs, rhs, extend_dimensions)));
+
 XLA_MAKE_BINARY(LogicalAnd, b->And(lhs, rhs, extend_dimensions));
 XLA_MAKE_BINARY(LogicalOr, b->Or(lhs, rhs, extend_dimensions));
 XLA_MAKE_BINARY(Mod, b->Rem(lhs, rhs, extend_dimensions));
diff --git a/tensorflow/contrib/makefile/tf_op_files.txt b/tensorflow/contrib/makefile/tf_op_files.txt
index 1fda907074..a8690a04ad 100644
--- a/tensorflow/contrib/makefile/tf_op_files.txt
+++ b/tensorflow/contrib/makefile/tf_op_files.txt
@@ -170,6 +170,8 @@ tensorflow/core/kernels/cwise_op_div.cc
 tensorflow/core/kernels/cwise_op_bitwise_xor.cc
 tensorflow/core/kernels/cwise_op_bitwise_or.cc
 tensorflow/core/kernels/cwise_op_bitwise_and.cc
+tensorflow/core/kernels/cwise_op_left_shift.cc
+tensorflow/core/kernels/cwise_op_right_shift.cc
 tensorflow/core/kernels/cwise_op_add_2.cc
 tensorflow/core/kernels/cwise_op_add_1.cc
 tensorflow/core/kernels/cwise_op_abs.cc
diff --git a/tensorflow/core/framework/types.cc b/tensorflow/core/framework/types.cc
index cc86871cae..faae19585d 100644
--- a/tensorflow/core/framework/types.cc
+++ b/tensorflow/core/framework/types.cc
@@ -335,6 +335,18 @@ bool DataTypeIsInteger(DataType dt) {
   }
 }
 
+bool DataTypeIsUnsigned(DataType dt) {
+  switch (dt) {
+    case DT_UINT8:
+    case DT_UINT16:
+    case DT_UINT32:
+    case DT_UINT64:
+      return true;
+    default:
+      return false;
+  }
+}
+
 int DataTypeSize(DataType dt) {
 #define CASE(T)                  \
   case DataTypeToEnum<T>::value: \
diff --git a/tensorflow/core/framework/types.h b/tensorflow/core/framework/types.h
index 300a57e948..dc53ed4178 100644
--- a/tensorflow/core/framework/types.h
+++ b/tensorflow/core/framework/types.h
@@ -227,6 +227,9 @@ bool DataTypeIsQuantized(DataType dt);
 // Is the dtype nonquantized integral?
 bool DataTypeIsInteger(DataType dt);
 
+// Is the dtype an unsigned integral type?
+bool DataTypeIsUnsigned(DataType dt);
+
 // Returns a 0 on failure
 int DataTypeSize(DataType dt);
 
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index d1a2362e5e..d931f12f6d 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -4512,6 +4512,7 @@ filegroup(
         "cwise_op_greater_equal.cc",
         "cwise_op_invert.cc",
         "cwise_op_isfinite.cc",
+        "cwise_op_left_shift.cc",
         "cwise_op_less.cc",
         "cwise_op_less_equal.cc",
         "cwise_op_log.cc",
@@ -4525,6 +4526,7 @@ filegroup(
         "cwise_op_neg.cc",
         "cwise_op_pow.cc",
         "cwise_op_reciprocal.cc",
+        "cwise_op_right_shift.cc",
         "cwise_op_round.cc",
         "cwise_op_rsqrt.cc",
         "cwise_op_select.cc",
diff --git a/tensorflow/core/kernels/cwise_op_gpu_left_shift.cu.cc b/tensorflow/core/kernels/cwise_op_gpu_left_shift.cu.cc
new file mode 100644
index 0000000000..740048795a
--- /dev/null
+++ b/tensorflow/core/kernels/cwise_op_gpu_left_shift.cu.cc
@@ -0,0 +1,27 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+
+#include "tensorflow/core/kernels/cwise_ops_gpu_common.cu.h"
+
+namespace tensorflow {
+namespace functor {
+DEFINE_BINARY8(left_shift, int8, int16, int32, int64, uint8, uint16, uint32,
+               uint64);
+}  // namespace functor
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/cwise_op_gpu_right_shift.cu.cc b/tensorflow/core/kernels/cwise_op_gpu_right_shift.cu.cc
new file mode 100644
index 0000000000..bb6772772c
--- /dev/null
+++ b/tensorflow/core/kernels/cwise_op_gpu_right_shift.cu.cc
@@ -0,0 +1,27 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+
+#include "tensorflow/core/kernels/cwise_ops_gpu_common.cu.h"
+
+namespace tensorflow {
+namespace functor {
+DEFINE_BINARY8(right_shift, int8, int16, int32, int64, uint8, uint16, uint32,
+               uint64);
+}  // namespace functor
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/cwise_op_left_shift.cc b/tensorflow/core/kernels/cwise_op_left_shift.cc
new file mode 100644
index 0000000000..ccb68139de
--- /dev/null
+++ b/tensorflow/core/kernels/cwise_op_left_shift.cc
@@ -0,0 +1,44 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/kernels/cwise_ops_common.h"
+
+namespace tensorflow {
+REGISTER8(BinaryOp, CPU, "LeftShift", functor::left_shift, int8, int16, int32,
+          int64, uint8, uint16, uint32, uint64);
+
+#if TENSORFLOW_USE_SYCL
+#define REGISTER_SYCL_KERNEL(TYPE)                                     \
+  REGISTER_KERNEL_BUILDER(                                             \
+      Name("LeftShift").Device(DEVICE_SYCL).TypeConstraint<TYPE>("T"), \
+      BinaryOp<SYCLDevice, functor::left_shift<TYPE>>);
+REGISTER_SYCL_KERNEL(int8);
+REGISTER_SYCL_KERNEL(int16);
+REGISTER_SYCL_KERNEL(int32);
+REGISTER_SYCL_KERNEL(int64);
+REGISTER_SYCL_KERNEL(uint8);
+REGISTER_SYCL_KERNEL(uint16);
+REGISTER_SYCL_KERNEL(uint32);
+REGISTER_SYCL_KERNEL(uint64);
+#undef REGISTER_SYCL_KERNEL
+
+#endif  // TENSORFLOW_USE_SYCL
+
+#if GOOGLE_CUDA
+REGISTER8(BinaryOp, GPU, "LeftShift", functor::left_shift, int8, int16, int32,
+          int64, uint8, uint16, uint32, uint64);
+#endif  // GOOGLE_CUDA
+
+}  // namespace tensorflow
diff --git a/tensorflow/core/kernels/cwise_op_right_shift.cc b/tensorflow/core/kernels/cwise_op_right_shift.cc
new file mode 100644
index 0000000000..6dc6b97e35
--- /dev/null
+++ b/tensorflow/core/kernels/cwise_op_right_shift.cc
@@ -0,0 +1,44 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/kernels/cwise_ops_common.h"
+
+namespace tensorflow {
+REGISTER8(BinaryOp, CPU, "RightShift", functor::right_shift, int8, int16, int32,
+          int64, uint8, uint16, uint32, uint64);
+
+#if TENSORFLOW_USE_SYCL
+#define REGISTER_SYCL_KERNEL(TYPE)                                      \
+  REGISTER_KERNEL_BUILDER(                                              \
+      Name("RightShift").Device(DEVICE_SYCL).TypeConstraint<TYPE>("T"), \
+      BinaryOp<SYCLDevice, functor::right_shift<TYPE>>);
+REGISTER_SYCL_KERNEL(int8);
+REGISTER_SYCL_KERNEL(int16);
+REGISTER_SYCL_KERNEL(int32);
+REGISTER_SYCL_KERNEL(int64);
+REGISTER_SYCL_KERNEL(uint8);
+REGISTER_SYCL_KERNEL(uint16);
+REGISTER_SYCL_KERNEL(uint32);
+REGISTER_SYCL_KERNEL(uint64);
+#undef REGISTER_SYCL_KERNEL
+
+#endif  // TENSORFLOW_USE_SYCL
+
+#if GOOGLE_CUDA
+REGISTER8(BinaryOp, GPU, "RightShift", functor::right_shift, int8, int16, int32,
+          int64, uint8, uint16, uint32, uint64);
+#endif  // GOOGLE_CUDA
+
+}  // namespace tensorflow
diff --git a/tensorflow/core/kernels/cwise_ops.h b/tensorflow/core/kernels/cwise_ops.h
index d935331904..89487419ee 100644
--- a/tensorflow/core/kernels/cwise_ops.h
+++ b/tensorflow/core/kernels/cwise_ops.h
@@ -18,6 +18,8 @@ limitations under the License.
 
 #include <cmath>
 #include <functional>
+#include <type_traits>
+
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 #include "tensorflow/core/framework/numeric_types.h"
 #include "tensorflow/core/framework/tensor_types.h"
@@ -810,6 +812,50 @@ struct bitwise_or : base<T, bitwise_or_op<T>> {};
 template <typename T>
 struct bitwise_xor : base<T, Eigen::internal::bitwise_xor_op<T>> {};
 
+template <typename T>
+struct left_shift_op {
+  EIGEN_EMPTY_STRUCT_CTOR(left_shift_op)
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const T operator()(const T& x,
+                                                           const T& y) const {
+    // Avoids UB: don't shift by larger than the bitwidth of T, and
+    // performs left shifts as unsigned shifts.
+    T y_clamped = y;
+    if (y_clamped < 0) {
+      y_clamped = 0;
+    } else if (y_clamped > sizeof(T) * CHAR_BIT - 1) {
+      y_clamped = sizeof(T) * CHAR_BIT - 1;
+    }
+    using U = typename std::make_unsigned<T>::type;
+    return static_cast<T>(static_cast<U>(x) << static_cast<U>(y_clamped));
+  }
+};
+
+template <typename T>
+struct right_shift_op {
+  EIGEN_EMPTY_STRUCT_CTOR(right_shift_op)
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const T operator()(const T& x,
+                                                           const T& y) const {
+    // Avoids UB: don't shift by larger than the bitwidth of T.
+    T y_clamped = y;
+    if (y_clamped < 0) {
+      y_clamped = 0;
+    } else if (y_clamped > sizeof(T) * CHAR_BIT - 1) {
+      y_clamped = sizeof(T) * CHAR_BIT - 1;
+    }
+    // Technically right shifts of signed integers are not necessarily
+    // arithmetic shifts according to the C++ standard. However in practice most
+    // implementations are arithmetic shifts. If this proves to be a problem in
+    // practice, we may need to use an alternative implementation.
+    return x >> y_clamped;
+  }
+};
+
+template <typename T>
+struct left_shift : base<T, left_shift_op<T>> {};
+
+template <typename T>
+struct right_shift : base<T, right_shift_op<T>> {};
+
 template <typename T>
 struct make_complex_func {
   typedef std::complex<T> result_type;
diff --git a/tensorflow/core/ops/bitwise_ops.cc b/tensorflow/core/ops/bitwise_ops.cc
index 3ffc4ab74a..3156162b78 100644
--- a/tensorflow/core/ops/bitwise_ops.cc
+++ b/tensorflow/core/ops/bitwise_ops.cc
@@ -23,7 +23,7 @@ namespace tensorflow {
 REGISTER_OP("Invert")
     .Input("x: T")
     .Output("y: T")
-    .Attr("T: {int8, int16, int32, int64, uint8, uint16}")
+    .Attr("T: {int8, int16, int32, int64, uint8, uint16, uint32, uint64}")
     .SetShapeFn(shape_inference::UnchangedShape)
     .Doc(R"doc(
 Flips all bits elementwise.
@@ -32,18 +32,18 @@ The result will have exactly those bits set, that are not set in `x`. The
 computation is performed on the underlying representation of x.
 )doc");
 
-#define BINARY_BITWISE()                                     \
-  Input("x: T")                                              \
-      .Input("y: T")                                         \
-      .Output("z: T")                                        \
-      .SetIsCommutative()                                    \
-      .Attr("T: {int8, int16, int32, int64, uint8, uint16}") \
+#define BINARY_BITWISE()                                                     \
+  Input("x: T")                                                              \
+      .Input("y: T")                                                         \
+      .Output("z: T")                                                        \
+      .SetIsCommutative()                                                    \
+      .Attr("T: {int8, int16, int32, int64, uint8, uint16, uint32, uint64}") \
       .SetShapeFn(shape_inference::UnchangedShape)
 
 REGISTER_OP("PopulationCount")
     .Input("x: T")
     .Output("y: uint8")
-    .Attr("T: {int8, int16, int32, int64, uint8, uint16}")
+    .Attr("T: {int8, int16, int32, int64, uint8, uint16, uint32, uint64}")
     .SetShapeFn(shape_inference::UnchangedShape)
     .Doc(R"doc(
 Computes element-wise population count (a.k.a. popcount, bitsum, bitcount).
@@ -77,4 +77,21 @@ The result will have those bits set, that are different in `x` and `y`. The
 computation is performed on the underlying representations of `x` and `y`.
 )doc");
 
+REGISTER_OP("LeftShift").BINARY_BITWISE().Doc(R"doc(
+Elementwise computes the bitwise left-shift of `x` and `y`.
+
+If `y` is negative, or greater than or equal to the width of `x` in bits the
+result is implementation defined.
+)doc");
+
+REGISTER_OP("RightShift").BINARY_BITWISE().Doc(R"doc(
+Elementwise computes the bitwise right-shift of `x` and `y`.
+
+Performs a logical shift for unsigned integer types, and an arithmetic shift
+for signed integer types.
+
+If `y` is negative, or greater than or equal to than the width of `x` in bits
+the result is implementation defined.
+)doc");
+
 }  // namespace tensorflow
diff --git a/tensorflow/python/framework/fast_tensor_util.pyx b/tensorflow/python/framework/fast_tensor_util.pyx
index b43ddb4ad3..19928314ef 100644
--- a/tensorflow/python/framework/fast_tensor_util.pyx
+++ b/tensorflow/python/framework/fast_tensor_util.pyx
@@ -30,6 +30,12 @@ def AppendInt32ArrayToTensorProto(
   for i in range(n):
     tensor_proto.int_val.append(nparray[i])
 
+def AppendUInt32ArrayToTensorProto(
+    tensor_proto, np.ndarray[np.uint32_t, ndim=1] nparray):
+  cdef long i, n
+  n = nparray.size
+  for i in range(n):
+    tensor_proto.uint32_val.append(nparray[i])
 
 def AppendInt64ArrayToTensorProto(
     tensor_proto, np.ndarray[np.int64_t, ndim=1] nparray):
@@ -38,6 +44,12 @@ def AppendInt64ArrayToTensorProto(
   for i in range(n):
     tensor_proto.int64_val.append(nparray[i])
 
+def AppendUInt64ArrayToTensorProto(
+    tensor_proto, np.ndarray[np.uint64_t, ndim=1] nparray):
+  cdef long i, n
+  n = nparray.size
+  for i in range(n):
+    tensor_proto.uint64_val.append(nparray[i])
 
 def AppendUInt8ArrayToTensorProto(
     tensor_proto, np.ndarray[np.uint8_t, ndim=1] nparray):
diff --git a/tensorflow/python/framework/tensor_util.py b/tensorflow/python/framework/tensor_util.py
index 53eba8b747..7e74c19124 100644
--- a/tensorflow/python/framework/tensor_util.py
+++ b/tensorflow/python/framework/tensor_util.py
@@ -61,6 +61,8 @@ if _FAST_TENSOR_UTIL_AVAILABLE:
       np.int64: fast_tensor_util.AppendInt64ArrayToTensorProto,
       np.uint8: fast_tensor_util.AppendUInt8ArrayToTensorProto,
       np.uint16: fast_tensor_util.AppendUInt16ArrayToTensorProto,
+      np.uint32: fast_tensor_util.AppendUInt32ArrayToTensorProto,
+      np.uint64: fast_tensor_util.AppendUInt64ArrayToTensorProto,
       np.int8: fast_tensor_util.AppendInt8ArrayToTensorProto,
       np.int16: fast_tensor_util.AppendInt16ArrayToTensorProto,
       np.complex64: fast_tensor_util.AppendComplex64ArrayToTensorProto,
@@ -90,11 +92,17 @@ else:
   def SlowAppendIntArrayToTensorProto(tensor_proto, proto_values):
     tensor_proto.int_val.extend([np.asscalar(x) for x in proto_values])
 
+  def SlowAppendInt64ArrayToTensorProto(tensor_proto, proto_values):
+    tensor_proto.int64_val.extend([np.asscalar(x) for x in proto_values])
+
   def SlowAppendQIntArrayToTensorProto(tensor_proto, proto_values):
     tensor_proto.int_val.extend([np.asscalar(x[0]) for x in proto_values])
 
-  def SlowAppendInt64ArrayToTensorProto(tensor_proto, proto_values):
-    tensor_proto.int64_val.extend([np.asscalar(x) for x in proto_values])
+  def SlowAppendUInt32ArrayToTensorProto(tensor_proto, proto_values):
+    tensor_proto.uint32_val.extend([np.asscalar(x) for x in proto_values])
+
+  def SlowAppendUInt64ArrayToTensorProto(tensor_proto, proto_values):
+    tensor_proto.uint64_val.extend([np.asscalar(x) for x in proto_values])
 
   def SlowAppendComplex64ArrayToTensorProto(tensor_proto, proto_values):
     tensor_proto.scomplex_val.extend([np.asscalar(v)
@@ -120,6 +128,8 @@ else:
       np.int64: SlowAppendInt64ArrayToTensorProto,
       np.uint8: SlowAppendIntArrayToTensorProto,
       np.uint16: SlowAppendIntArrayToTensorProto,
+      np.uint32: SlowAppendUInt32ArrayToTensorProto,
+      np.uint64: SlowAppendUInt64ArrayToTensorProto,
       np.int8: SlowAppendIntArrayToTensorProto,
       np.int16: SlowAppendIntArrayToTensorProto,
       np.complex64: SlowAppendComplex64ArrayToTensorProto,
@@ -190,7 +200,7 @@ def _FlattenToStrings(nested_strings):
 _TENSOR_CONTENT_TYPES = frozenset([
     dtypes.float32, dtypes.float64, dtypes.int32, dtypes.uint8, dtypes.int16,
     dtypes.int8, dtypes.int64, dtypes.qint8, dtypes.quint8, dtypes.qint16,
-    dtypes.quint16, dtypes.qint32,
+    dtypes.quint16, dtypes.qint32, dtypes.uint32, dtypes.uint64
 ])
 
 
diff --git a/tensorflow/python/ops/bitwise_ops.py b/tensorflow/python/ops/bitwise_ops.py
index 44daf13537..e8e187e68f 100644
--- a/tensorflow/python/ops/bitwise_ops.py
+++ b/tensorflow/python/ops/bitwise_ops.py
@@ -19,6 +19,8 @@
 @@bitwise_or
 @@bitwise_xor
 @@invert
+@@left_shift
+@@right_shift
 """
 
 from __future__ import absolute_import
@@ -37,5 +39,7 @@ ops.NotDifferentiable("BitwiseOr")
 ops.NotDifferentiable("BitwiseXor")
 ops.NotDifferentiable("Invert")
 ops.NotDifferentiable("PopulationCount")
+ops.NotDifferentiable("LeftShift")
+ops.NotDifferentiable("RightShift")
 
 remove_undocumented(__name__)
diff --git a/tensorflow/python/ops/bitwise_ops_test.py b/tensorflow/python/ops/bitwise_ops_test.py
index 1d08c8f82d..fa1b219b17 100644
--- a/tensorflow/python/ops/bitwise_ops_test.py
+++ b/tensorflow/python/ops/bitwise_ops_test.py
@@ -93,5 +93,47 @@ class BitwiseOpTest(test_util.TensorFlowTestCase):
           expected = [dtype.max - x for x in inputs]
           self.assertAllEqual(inverted, expected)
 
+  def testShiftsWithPositiveLHS(self):
+    dtype_list = [np.int8, np.int16, np.int32, np.int64,
+                  np.uint8, np.uint16, np.uint32, np.uint64]
+
+    with self.test_session(use_gpu=True) as sess:
+      for dtype in dtype_list:
+        lhs = np.array([0, 5, 3, 14], dtype=dtype)
+        rhs = np.array([5, 0, 7, 3], dtype=dtype)
+        left_shift_result, right_shift_result = sess.run(
+            [bitwise_ops.left_shift(lhs, rhs),
+             bitwise_ops.right_shift(lhs, rhs)])
+        self.assertAllEqual(left_shift_result, np.left_shift(lhs, rhs))
+        self.assertAllEqual(right_shift_result, np.right_shift(lhs, rhs))
+
+  def testShiftsWithNegativeLHS(self):
+    dtype_list = [np.int8, np.int16, np.int32, np.int64]
+
+    with self.test_session(use_gpu=True) as sess:
+      for dtype in dtype_list:
+        lhs = np.array([-1, -5, -3, -14], dtype=dtype)
+        rhs = np.array([5, 0, 7, 11], dtype=dtype)
+        left_shift_result, right_shift_result = sess.run(
+            [bitwise_ops.left_shift(lhs, rhs),
+             bitwise_ops.right_shift(lhs, rhs)])
+        self.assertAllEqual(left_shift_result, np.left_shift(lhs, rhs))
+        self.assertAllEqual(right_shift_result, np.right_shift(lhs, rhs))
+
+  def testImplementationDefinedShiftsDoNotCrash(self):
+    dtype_list = [np.int8, np.int16, np.int32, np.int64]
+
+    with self.test_session(use_gpu=True) as sess:
+      for dtype in dtype_list:
+        lhs = np.array([-1, -5, -3, -14], dtype=dtype)
+        rhs = np.array([-2, 64, 101, 32], dtype=dtype)
+        # We intentionally do not test for specific values here since the exact
+        # outputs are implementation-defined. However, we should not crash or
+        # trigger an undefined-behavior error from tools such as
+        # AddressSanitizer.
+        sess.run([bitwise_ops.left_shift(lhs, rhs),
+                  bitwise_ops.right_shift(lhs, rhs)])
+
+
 if __name__ == "__main__":
   googletest.main()
diff --git a/tensorflow/tools/api/golden/tensorflow.bitwise.pbtxt b/tensorflow/tools/api/golden/tensorflow.bitwise.pbtxt
index 1e4d333cc0..01cbd55c5d 100644
--- a/tensorflow/tools/api/golden/tensorflow.bitwise.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.bitwise.pbtxt
@@ -16,4 +16,12 @@ tf_module {
     name: "invert"
     argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
   }
+  member_method {
+    name: "left_shift"
+    argspec: "args=[\'x\', \'y\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "right_shift"
+    argspec: "args=[\'x\', \'y\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
 }
-- 
GitLab


From 246e66c01cbf515174673c766f81705380ab69f6 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 17 Oct 2017 16:08:07 -0700
Subject: [PATCH 490/909] Update ops-related pbtxt files.

PiperOrigin-RevId: 172530775
---
 .../core/ops/compat/ops_history.v1.pbtxt      | 214 ++++++++++++++++++
 tensorflow/core/ops/ops.pbtxt                 |  78 +++++++
 2 files changed, 292 insertions(+)

diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index d93a4ff933..6772024263 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -5733,6 +5733,68 @@ op {
   }
   is_commutative: true
 }
+op {
+  name: "BitwiseAnd"
+  input_arg {
+    name: "x"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "y"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "z"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT8
+        type: DT_INT16
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+  is_commutative: true
+}
+op {
+  name: "BitwiseOr"
+  input_arg {
+    name: "x"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "y"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "z"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT8
+        type: DT_INT16
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_UINT8
+        type: DT_UINT16
+      }
+    }
+  }
+  is_commutative: true
+}
 op {
   name: "BitwiseOr"
   input_arg {
@@ -5758,6 +5820,8 @@ op {
         type: DT_INT64
         type: DT_UINT8
         type: DT_UINT16
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -5793,6 +5857,38 @@ op {
   }
   is_commutative: true
 }
+op {
+  name: "BitwiseXor"
+  input_arg {
+    name: "x"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "y"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "z"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT8
+        type: DT_INT16
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+  is_commutative: true
+}
 op {
   name: "BroadcastArgs"
   input_arg {
@@ -14315,6 +14411,33 @@ op {
     }
   }
 }
+op {
+  name: "Invert"
+  input_arg {
+    name: "x"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "y"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT8
+        type: DT_INT16
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+}
 op {
   name: "InvertPermutation"
   input_arg {
@@ -14818,6 +14941,38 @@ op {
   }
   is_stateful: true
 }
+op {
+  name: "LeftShift"
+  input_arg {
+    name: "x"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "y"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "z"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT8
+        type: DT_INT16
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+  is_commutative: true
+}
 op {
   name: "Less"
   input_arg {
@@ -20883,6 +21038,33 @@ op {
     }
   }
 }
+op {
+  name: "PopulationCount"
+  input_arg {
+    name: "x"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "y"
+    type: DT_UINT8
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT8
+        type: DT_INT16
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+}
 op {
   name: "Pow"
   input_arg {
@@ -28983,6 +29165,38 @@ op {
     }
   }
 }
+op {
+  name: "RightShift"
+  input_arg {
+    name: "x"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "y"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "z"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT8
+        type: DT_INT16
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+  is_commutative: true
+}
 op {
   name: "Rint"
   input_arg {
diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index 6403dcf78c..623f5457bb 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -4008,6 +4008,8 @@ op {
         type: DT_INT64
         type: DT_UINT8
         type: DT_UINT16
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -4040,6 +4042,8 @@ op {
         type: DT_INT64
         type: DT_UINT8
         type: DT_UINT16
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -4072,6 +4076,8 @@ op {
         type: DT_INT64
         type: DT_UINT8
         type: DT_UINT16
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -11083,6 +11089,8 @@ op {
         type: DT_INT64
         type: DT_UINT8
         type: DT_UINT16
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -11564,6 +11572,40 @@ op {
   description: "See explanations of candidate sampling and the data formats at\ngo/candidate-sampling.\n\nFor each batch, this op picks a single set of sampled candidate labels.\n\nThe advantages of sampling candidates per-batch are simplicity and the\npossibility of efficient dense matrix multiplication. The disadvantage is that\nthe sampled candidates must be chosen independently of the context and of the\ntrue labels."
   is_stateful: true
 }
+op {
+  name: "LeftShift"
+  input_arg {
+    name: "x"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "y"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "z"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT8
+        type: DT_INT16
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+  summary: "Elementwise computes the bitwise left-shift of `x` and `y`."
+  description: "If `y` is negative, or greater than or equal to the width of `x` in bits the\nresult is implementation defined."
+  is_commutative: true
+}
 op {
   name: "Less"
   input_arg {
@@ -16522,6 +16564,8 @@ op {
         type: DT_INT64
         type: DT_UINT8
         type: DT_UINT16
+        type: DT_UINT32
+        type: DT_UINT64
       }
     }
   }
@@ -23211,6 +23255,40 @@ op {
   summary: "Reverses specific dimensions of a tensor."
   description: "NOTE `tf.reverse` has now changed behavior in preparation for 1.0.\n`tf.reverse_v2` is currently an alias that will be deprecated before TF 1.0.\n\nGiven a `tensor`, and a `int32` tensor `axis` representing the set of\ndimensions of `tensor` to reverse. This operation reverses each dimension\n`i` for which there exists `j` s.t. `axis[j] == i`.\n\n`tensor` can have up to 8 dimensions. The number of dimensions specified\nin `axis` may be 0 or more entries. If an index is specified more than\nonce, a InvalidArgument error is raised.\n\nFor example:\n\n```\n# tensor \'t\' is [[[[ 0,  1,  2,  3],\n#                  [ 4,  5,  6,  7],\n#                  [ 8,  9, 10, 11]],\n#                 [[12, 13, 14, 15],\n#                  [16, 17, 18, 19],\n#                  [20, 21, 22, 23]]]]\n# tensor \'t\' shape is [1, 2, 3, 4]\n\n# \'dims\' is [3] or \'dims\' is -1\nreverse(t, dims) ==> [[[[ 3,  2,  1,  0],\n                        [ 7,  6,  5,  4],\n                        [ 11, 10, 9, 8]],\n                       [[15, 14, 13, 12],\n                        [19, 18, 17, 16],\n                        [23, 22, 21, 20]]]]\n\n# \'dims\' is \'[1]\' (or \'dims\' is \'[-3]\')\nreverse(t, dims) ==> [[[[12, 13, 14, 15],\n                        [16, 17, 18, 19],\n                        [20, 21, 22, 23]\n                       [[ 0,  1,  2,  3],\n                        [ 4,  5,  6,  7],\n                        [ 8,  9, 10, 11]]]]\n\n# \'dims\' is \'[2]\' (or \'dims\' is \'[-2]\')\nreverse(t, dims) ==> [[[[8, 9, 10, 11],\n                        [4, 5, 6, 7],\n                        [0, 1, 2, 3]]\n                       [[20, 21, 22, 23],\n                        [16, 17, 18, 19],\n                        [12, 13, 14, 15]]]]\n```"
 }
+op {
+  name: "RightShift"
+  input_arg {
+    name: "x"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "y"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "z"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT8
+        type: DT_INT16
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+  summary: "Elementwise computes the bitwise right-shift of `x` and `y`."
+  description: "Performs a logical shift for unsigned integer types, and an arithmetic shift\nfor signed integer types.\n\nIf `y` is negative, or greater than or equal to than the width of `x` in bits\nthe result is implementation defined."
+  is_commutative: true
+}
 op {
   name: "Rint"
   input_arg {
-- 
GitLab


From 95c7f5344f8da74a839c459c6415855bffe4f004 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 17 Oct 2017 16:15:29 -0700
Subject: [PATCH 491/909] Go: Update generated wrapper functions for TensorFlow
 ops.

PiperOrigin-RevId: 172531834
---
 tensorflow/go/op/wrappers.go | 39 ++++++++++++++++++++++++++++++++++++
 1 file changed, 39 insertions(+)

diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go
index 8f5ee9c3df..c117711c81 100644
--- a/tensorflow/go/op/wrappers.go
+++ b/tensorflow/go/op/wrappers.go
@@ -2324,6 +2324,45 @@ func DecodeWav(scope *Scope, contents tf.Output, optional ...DecodeWavAttr) (aud
 	return op.Output(0), op.Output(1)
 }
 
+// Elementwise computes the bitwise right-shift of `x` and `y`.
+//
+// Performs a logical shift for unsigned integer types, and an arithmetic shift
+// for signed integer types.
+//
+// If `y` is negative, or greater than or equal to than the width of `x` in bits
+// the result is implementation defined.
+func RightShift(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "RightShift",
+		Input: []tf.Input{
+			x, y,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Elementwise computes the bitwise left-shift of `x` and `y`.
+//
+// If `y` is negative, or greater than or equal to the width of `x` in bits the
+// result is implementation defined.
+func LeftShift(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "LeftShift",
+		Input: []tf.Input{
+			x, y,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Elementwise computes the bitwise AND of `x` and `y`.
 //
 // The result will have those bits set, that are set in both `x` and `y`. The
-- 
GitLab


From 47e4d4b6b5742350233a8fd83cd81269792ed286 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 17 Oct 2017 16:19:08 -0700
Subject: [PATCH 492/909] Use optimized functor for conjugate transpose in
 MatrixInverseOp. Introduce convenience functions DoMatrixTranspose and
 DoConjugateMatrixTranspose. Misc. minor cleanup of templates in
 transpose_functor*.

PiperOrigin-RevId: 172532252
---
 tensorflow/core/kernels/cuda_solvers.h        |   8 -
 .../core/kernels/cuda_solvers_gpu.cu.cc       |  18 ---
 tensorflow/core/kernels/matrix_inverse_op.cc  |  12 +-
 tensorflow/core/kernels/matrix_solve_op.cc    |   9 +-
 tensorflow/core/kernels/qr_op_impl.h          |   9 +-
 .../kernels/self_adjoint_eig_v2_op_gpu.cc     |   5 +-
 tensorflow/core/kernels/svd_op_gpu.cu.cc      |  25 ++-
 tensorflow/core/kernels/transpose_functor.h   | 150 ++++++++++--------
 .../core/kernels/transpose_functor_cpu.cc     |  72 +++++----
 .../core/kernels/transpose_functor_gpu.cu.cc  |  52 +++---
 10 files changed, 174 insertions(+), 186 deletions(-)

diff --git a/tensorflow/core/kernels/cuda_solvers.h b/tensorflow/core/kernels/cuda_solvers.h
index 60c4a0bfb4..eb720b191f 100644
--- a/tensorflow/core/kernels/cuda_solvers.h
+++ b/tensorflow/core/kernels/cuda_solvers.h
@@ -409,14 +409,6 @@ class DeviceLapackInfo : public ScratchSpace<int> {
 };
 
 namespace functor {
-// Helper functor to transpose and conjugate all matrices in a flattened batch.
-template <typename Device, typename Scalar>
-struct AdjointBatchFunctor {
-  // We assume that the tensor sizes are correct.
-  void operator()(const Device& device,
-                  typename TTypes<Scalar, 3>::ConstTensor input,
-                  typename TTypes<Scalar, 3>::Tensor output);
-};
 
 // Helper functor to compute the product of diagonal elements in all matrices
 // in a flattened batch.
diff --git a/tensorflow/core/kernels/cuda_solvers_gpu.cu.cc b/tensorflow/core/kernels/cuda_solvers_gpu.cu.cc
index 79961c01ca..4171f9d68e 100644
--- a/tensorflow/core/kernels/cuda_solvers_gpu.cu.cc
+++ b/tensorflow/core/kernels/cuda_solvers_gpu.cu.cc
@@ -29,24 +29,6 @@ namespace functor {
 
 typedef Eigen::GpuDevice GPUDevice;
 
-// TODO(rmlarsen): Add a faster custom kernel similar to
-// SwapDimension1And2InTensor3 in tensorflow/core/kernels/conv_ops_gpu_3.cu.cc
-template <typename Scalar>
-struct AdjointBatchFunctor<GPUDevice, Scalar> {
-  void operator()(const GPUDevice& device,
-                  typename TTypes<Scalar, 3>::ConstTensor input,
-                  typename TTypes<Scalar, 3>::Tensor output) {
-    const Eigen::array<int, 3> perm({0, 2, 1});
-    To32Bit(output).device(device) = To32Bit(input).shuffle(perm).conjugate();
-  }
-};
-
-// Instantiate implementations for the 4 numeric types.
-template struct AdjointBatchFunctor<GPUDevice, float>;
-template struct AdjointBatchFunctor<GPUDevice, double>;
-template struct AdjointBatchFunctor<GPUDevice, complex64>;
-template struct AdjointBatchFunctor<GPUDevice, complex128>;
-
 namespace {
 
 // Hacks around missing support for complex arithmetic in nvcc.
diff --git a/tensorflow/core/kernels/matrix_inverse_op.cc b/tensorflow/core/kernels/matrix_inverse_op.cc
index 832e508bb7..64edfe470d 100644
--- a/tensorflow/core/kernels/matrix_inverse_op.cc
+++ b/tensorflow/core/kernels/matrix_inverse_op.cc
@@ -33,6 +33,7 @@ limitations under the License.
 #if GOOGLE_CUDA
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 #include "tensorflow/core/kernels/cuda_solvers.h"
+#include "tensorflow/core/kernels/transpose_functor.h"
 #endif
 
 namespace tensorflow {
@@ -135,15 +136,15 @@ class MatrixInverseOpGpu : public AsyncOpKernel {
                                        input.shape(), &input_copy),
         done);
     auto input_copy_reshaped = input_copy.template flat_inner_dims<Scalar, 3>();
-    auto input_reshaped = input.template flat_inner_dims<Scalar, 3>();
     const GPUDevice& device = context->eigen_device<GPUDevice>();
     if (!adjoint_) {
       device.memcpy(input_copy.flat<Scalar>().data(),
                     input.flat<Scalar>().data(),
                     input.NumElements() * sizeof(Scalar));
     } else {
-      functor::AdjointBatchFunctor<GPUDevice, Scalar> functor;
-      functor(device, input_reshaped, input_copy_reshaped);
+      OP_REQUIRES_OK_ASYNC(
+          context, DoConjugateMatrixTranspose(device, input, &input_copy),
+          done);
     }
     const int64 batch_size = input_copy_reshaped.dimension(0);
 
@@ -238,10 +239,7 @@ class MatrixInverseOpGpu : public AsyncOpKernel {
             done);
       }
     }
-    // Callback for checking info after kernels finish. Also capture the
-    // temporary Tensors/ScratchSpace so they don't get deallocated before the
-    // kernels run. TODO(rmlarsen): Use move capture once C++14 becomes
-    // available.
+    // Callback for checking info after kernels finish.
     auto info_checker = [context, done](
                             const Status& status,
                             const std::vector<HostLapackInfo>& host_infos) {
diff --git a/tensorflow/core/kernels/matrix_solve_op.cc b/tensorflow/core/kernels/matrix_solve_op.cc
index 862033e9fa..2e4098dfab 100644
--- a/tensorflow/core/kernels/matrix_solve_op.cc
+++ b/tensorflow/core/kernels/matrix_solve_op.cc
@@ -181,9 +181,6 @@ class MatrixSolveOpGpu : public AsyncOpKernel {
     // false, try to reuse the input buffer if this op owns it exclusively.
     Tensor input_copy;
     const GPUDevice& device = context->eigen_device<GPUDevice>();
-    std::vector<int> perm(ndims);
-    std::iota(perm.begin(), perm.end(), 0);
-    std::swap(perm[ndims - 2], perm[ndims - 1]);
     if (adjoint_) {
       // For the adjoint case, it is simpler to always make a transposed copy up
       // front.
@@ -193,7 +190,7 @@ class MatrixSolveOpGpu : public AsyncOpKernel {
                                          input.shape(), &input_copy),
           done);
       OP_REQUIRES_OK_ASYNC(context,
-                           DoTranspose(device, input, perm, &input_copy), done);
+                           DoMatrixTranspose(device, input, &input_copy), done);
     } else {
       OP_REQUIRES_OK_ASYNC(
           context,
@@ -267,7 +264,7 @@ class MatrixSolveOpGpu : public AsyncOpKernel {
         done);
     if (nrhs > 1) {
       OP_REQUIRES_OK_ASYNC(
-          context, DoTranspose(device, rhs, perm, &transposed_rhs), done);
+          context, DoMatrixTranspose(device, rhs, &transposed_rhs), done);
     } else {
       device.memcpy(transposed_rhs.flat<Scalar>().data(),
                     rhs.flat<Scalar>().data(),
@@ -327,7 +324,7 @@ class MatrixSolveOpGpu : public AsyncOpKernel {
     // 4. Transpose X to get the final result in row-major form.
     if (nrhs > 1) {
       OP_REQUIRES_OK_ASYNC(
-          context, DoTranspose(device, transposed_rhs, perm, output), done);
+          context, DoMatrixTranspose(device, transposed_rhs, output), done);
     } else {
       device.memcpy(output->flat<Scalar>().data(),
                     transposed_rhs.flat<Scalar>().data(),
diff --git a/tensorflow/core/kernels/qr_op_impl.h b/tensorflow/core/kernels/qr_op_impl.h
index e263eb22f1..c51d601437 100644
--- a/tensorflow/core/kernels/qr_op_impl.h
+++ b/tensorflow/core/kernels/qr_op_impl.h
@@ -190,12 +190,9 @@ class QrOpGpu : public AsyncOpKernel {
 
     // Transpose input, since cuSolver uses column-major, while TensorFlow uses
     // row-major storage.
-    std::vector<int> perm(ndims);
-    std::iota(perm.begin(), perm.end(), 0);
-    std::swap(perm[ndims - 2], perm[ndims - 1]);
     const GPUDevice& device = context->eigen_device<GPUDevice>();
     OP_REQUIRES_OK_ASYNC(
-        context, DoTranspose(device, input, perm, &input_transposed), done);
+        context, DoMatrixTranspose(device, input, &input_transposed), done);
 
     // Compute QR decomposition in-place in input_transposed.
     std::vector<DeviceLapackInfo> dev_info;
@@ -218,7 +215,7 @@ class QrOpGpu : public AsyncOpKernel {
     // and copy it to the output buffer.
     if (full_matrices_ || m == n) {
       OP_REQUIRES_OK_ASYNC(
-          context, DoTranspose(device, input_transposed, perm, r), done);
+          context, DoMatrixTranspose(device, input_transposed, r), done);
     } else {
       const Scalar alpha(1);
       const Scalar beta(0);
@@ -280,7 +277,7 @@ class QrOpGpu : public AsyncOpKernel {
             done);
       }
       OP_REQUIRES_OK_ASYNC(
-          context, DoTranspose(device, input_transposed, perm, q), done);
+          context, DoMatrixTranspose(device, input_transposed, q), done);
     }
 
     // Asynchronously check return status from cuSolver kernels.
diff --git a/tensorflow/core/kernels/self_adjoint_eig_v2_op_gpu.cc b/tensorflow/core/kernels/self_adjoint_eig_v2_op_gpu.cc
index b0b4f89a27..3a84df07a9 100644
--- a/tensorflow/core/kernels/self_adjoint_eig_v2_op_gpu.cc
+++ b/tensorflow/core/kernels/self_adjoint_eig_v2_op_gpu.cc
@@ -148,11 +148,8 @@ class SelfAdjointEigV2OpGpu : public AsyncOpKernel {
     if (compute_v_) {
       // Transpose eigenvectors now stored in input_copy in column-major form to
       // output in row-major form.
-      std::vector<int> perm(ndims);
-      std::iota(perm.begin(), perm.end(), 0);
-      std::swap(perm[ndims - 2], perm[ndims - 1]);
       OP_REQUIRES_OK_ASYNC(
-          context, DoTranspose(device, input_copy, perm, eigenvectors), done);
+          context, DoMatrixTranspose(device, input_copy, eigenvectors), done);
     }
 
     // Asynchronously check return status from cuSolver kernels.
diff --git a/tensorflow/core/kernels/svd_op_gpu.cu.cc b/tensorflow/core/kernels/svd_op_gpu.cu.cc
index 1603a8aeda..dedc2da60b 100644
--- a/tensorflow/core/kernels/svd_op_gpu.cu.cc
+++ b/tensorflow/core/kernels/svd_op_gpu.cu.cc
@@ -190,8 +190,8 @@ class SvdOpGpu : public AsyncOpKernel {
   // TODO: can the two cases (MgeqN and MlessN) be simplified,
   //   common boilerplate be reduced, or even combined in one method?
   void PerformSVD_MgeqN(OpKernelContext* context, DoneCallback done, int64 m,
-                        int64 n, int64 p, const gtl::ArraySlice<int32>& perm,
-                        const Tensor& M, Tensor* S, Tensor* U, Tensor* V) {
+                        int64 n, int64 p, const Tensor& M, Tensor* S, Tensor* U,
+                        Tensor* V) {
     TensorShape shapeRaw = M.shape();
     shapeRaw.RemoveLastDims(2);
 
@@ -207,7 +207,7 @@ class SvdOpGpu : public AsyncOpKernel {
         solver->allocate_scoped_tensor(M.dtype(), input_shape, &input_copy),
         done);
     auto device = context->eigen_device<GPUDevice>();
-    OP_REQUIRES_OK_ASYNC(context, DoTranspose(device, M, perm, &input_copy),
+    OP_REQUIRES_OK_ASYNC(context, DoMatrixTranspose(device, M, &input_copy),
                          done);
 
     // I need to transpose U at the end
@@ -250,7 +250,7 @@ class SvdOpGpu : public AsyncOpKernel {
 
     // Transpose U
     if (compute_uv_) {
-      OP_REQUIRES_OK_ASYNC(context, DoTranspose(device, u_copy, perm, U), done);
+      OP_REQUIRES_OK_ASYNC(context, DoMatrixTranspose(device, u_copy, U), done);
     }
 
     // now check if the SVD operation succeeded or not
@@ -259,8 +259,8 @@ class SvdOpGpu : public AsyncOpKernel {
 
   // The SVD if m < n
   void PerformSVD_MlessN(OpKernelContext* context, DoneCallback done, int64 m,
-                         int64 n, int64 p, const gtl::ArraySlice<int32>& perm,
-                         const Tensor& M, Tensor* S, Tensor* U, Tensor* V) {
+                         int64 n, int64 p, const Tensor& M, Tensor* S,
+                         Tensor* U, Tensor* V) {
     // Perform the SVD on M'
 
     // Reuse the input buffer or make a copy for the SVD depending on whether
@@ -325,7 +325,7 @@ class SvdOpGpu : public AsyncOpKernel {
     // Transpose V
     if (compute_uv_) {
       auto device = context->eigen_device<GPUDevice>();
-      OP_REQUIRES_OK_ASYNC(context, DoTranspose(device, v_copy, perm, V), done);
+      OP_REQUIRES_OK_ASYNC(context, DoMatrixTranspose(device, v_copy, V), done);
     }
 
     // now check if the SVD operation succeeded or not
@@ -389,19 +389,12 @@ class SvdOpGpu : public AsyncOpKernel {
       return;
     }
 
-    // Prepare permutation
-    std::vector<int32> perm;
-    for (size_t i = 0; i < ndims - 2; ++i) perm.push_back(i);
-    perm.push_back(ndims - 1);  // transpose last two dimensions
-    perm.push_back(ndims - 2);
-    gtl::ArraySlice<int32> permAS(perm);
-
     // call implementations
     if (m >= n) {
-      PerformSVD_MgeqN(context, done, m, n, p, permAS, input, outputS, outputU,
+      PerformSVD_MgeqN(context, done, m, n, p, input, outputS, outputU,
                        outputV);
     } else {
-      PerformSVD_MlessN(context, done, m, n, p, permAS, input, outputS, outputU,
+      PerformSVD_MlessN(context, done, m, n, p, input, outputS, outputU,
                         outputV);
     }
   }
diff --git a/tensorflow/core/kernels/transpose_functor.h b/tensorflow/core/kernels/transpose_functor.h
index 87569f0275..a2eb0263e8 100644
--- a/tensorflow/core/kernels/transpose_functor.h
+++ b/tensorflow/core/kernels/transpose_functor.h
@@ -23,7 +23,6 @@ limitations under the License.
 #include "tensorflow/core/platform/logging.h"
 
 namespace tensorflow {
-
 // Transpose tensor 'in' into tensor 'out' according to dimension
 // permutation 'perm'.
 //
@@ -46,6 +45,17 @@ template <typename Device>
 Status DoConjugateTranspose(const Device& device, const Tensor& in,
                             const gtl::ArraySlice<int32> perm, Tensor* out);
 
+// Convenience versions of DoTranspose that only swap the last (inner) two
+// dimensions.
+template <typename Device>
+Status DoMatrixTranspose(const Device& device, const Tensor& in, Tensor* out);
+
+// Convenience versions of DoConjugateTranspose that only swap the last (inner)
+// two dimensions.
+template <typename Device>
+Status DoConjugateMatrixTranspose(const Device& device, const Tensor& in,
+                                  Tensor* out);
+
 // Primary device specific functor to be specialized for each device and type.
 template <typename Device, typename T, bool conjugate = false>
 struct Transpose {
@@ -131,11 +141,6 @@ inline bool NonSingletonDimensionsAlign(const TensorShape& input_shape,
   return true;
 }
 
-// Device-specific naive implementation for transpose.
-template <typename Device, typename T, bool conjugate>
-void TransposeSimple(const Device& d, const Tensor& in,
-                     const gtl::ArraySlice<int32> perm, Tensor* out);
-
 // Uses Eigen to transpose.
 template <typename Device, typename T, int NDIMS>
 void TransposeUsingEigen(const Device& d, const Tensor& in,
@@ -157,69 +162,78 @@ void TransposeUsingEigen(const Device& d, const Tensor& in,
 }
 
 template <typename Device>
-struct DoTransposeImpl {
-  static Status run(const Device& d, const Tensor& in,
-                    const gtl::ArraySlice<int32> perm, bool conjugate,
-                    Tensor* out) {
-    CHECK_GE(in.dims(), 2);
-    CHECK_EQ(in.dims(), out->dims());
-    CHECK_EQ(in.dims(), perm.size());
-    CHECK_EQ(in.dtype(), out->dtype());
-    switch (in.dtype()) {
-      case DT_BOOL:
-      case DT_INT8:
-      case DT_QINT8:
-      case DT_QUINT8:
-      case DT_UINT8:
-        Transpose<Device, uint8>::run(d, in, perm, out);
-        break;
-
-      case DT_BFLOAT16:
-      case DT_HALF:
-      case DT_INT16:
-      case DT_QINT16:
-      case DT_QUINT16:
-      case DT_UINT16:
-        Transpose<Device, uint16>::run(d, in, perm, out);
-        break;
-
-      case DT_FLOAT:
-      case DT_INT32:
-      case DT_QINT32:
-        Transpose<Device, uint32>::run(d, in, perm, out);
-        break;
-
-      case DT_DOUBLE:
-      case DT_INT64:
-        Transpose<Device, uint64>::run(d, in, perm, out);
-        break;
-
-      case DT_COMPLEX64:
-        if (conjugate) {
-          Transpose<Device, complex64, true>::run(d, in, perm, out);
-        } else {
-          Transpose<Device, complex64, false>::run(d, in, perm, out);
-        }
-        break;
-
-      case DT_COMPLEX128:
-        if (conjugate) {
-          Transpose<Device, complex128, true>::run(d, in, perm, out);
-        } else {
-          Transpose<Device, complex128, false>::run(d, in, perm, out);
-        }
-        break;
-
-      case DT_STRING:
-        Transpose<Device, string>::run(d, in, perm, out);
-        break;
-
-      default:
-        return errors::Unimplemented("Unsupported dtype on CPU: ", in.dtype());
-    }
-    return Status::OK();
+Status DoTransposeImpl(const Device& d, const Tensor& in,
+                       const gtl::ArraySlice<int32> perm, bool conjugate,
+                       Tensor* out) {
+  CHECK_GE(in.dims(), 2);
+  CHECK_EQ(in.dims(), out->dims());
+  CHECK_EQ(in.dims(), perm.size());
+  CHECK_EQ(in.dtype(), out->dtype());
+  switch (in.dtype()) {
+    case DT_BOOL:
+    case DT_INT8:
+    case DT_QINT8:
+    case DT_QUINT8:
+    case DT_UINT8:
+      Transpose<Device, uint8>::run(d, in, perm, out);
+      break;
+
+    case DT_BFLOAT16:
+    case DT_HALF:
+    case DT_INT16:
+    case DT_QINT16:
+    case DT_QUINT16:
+    case DT_UINT16:
+      Transpose<Device, uint16>::run(d, in, perm, out);
+      break;
+
+    case DT_FLOAT:
+    case DT_INT32:
+    case DT_QINT32:
+      Transpose<Device, uint32>::run(d, in, perm, out);
+      break;
+
+    case DT_DOUBLE:
+    case DT_INT64:
+      Transpose<Device, uint64>::run(d, in, perm, out);
+      break;
+
+    case DT_COMPLEX64:
+      if (conjugate) {
+        Transpose<Device, complex64, true>::run(d, in, perm, out);
+      } else {
+        Transpose<Device, complex64, false>::run(d, in, perm, out);
+      }
+      break;
+
+    case DT_COMPLEX128:
+      if (conjugate) {
+        Transpose<Device, complex128, true>::run(d, in, perm, out);
+      } else {
+        Transpose<Device, complex128, false>::run(d, in, perm, out);
+      }
+      break;
+
+    case DT_STRING:
+      Transpose<Device, string>::run(d, in, perm, out);
+      break;
+
+    default:
+      return errors::Unimplemented("Unsupported dtype on CPU: ", in.dtype());
   }
-};
+  return Status::OK();
+}
+
+template <typename Device>
+inline Status DoMatrixTransposeImpl(const Device& device, const Tensor& in,
+                                    bool conjugate, Tensor* out) {
+  const int ndims = in.dims();
+  if (ndims == 0) return Status::OK();
+  TransposePermsVec perm(ndims);
+  std::iota(perm.begin(), perm.end(), 0);
+  std::swap(perm[ndims - 2], perm[ndims - 1]);
+  return DoTransposeImpl(device, in, perm, conjugate, out);
+}
 
 #ifdef TENSORFLOW_USE_SYCL
 // For SYCL lets always go through Eigen
diff --git a/tensorflow/core/kernels/transpose_functor_cpu.cc b/tensorflow/core/kernels/transpose_functor_cpu.cc
index b2de012be1..41b73fdaf4 100644
--- a/tensorflow/core/kernels/transpose_functor_cpu.cc
+++ b/tensorflow/core/kernels/transpose_functor_cpu.cc
@@ -29,17 +29,18 @@ limitations under the License.
 typedef Eigen::ThreadPoolDevice CPUDevice;
 
 namespace tensorflow {
-namespace internal {
+namespace {
 
-template <typename Device, typename T, bool conjugate>
-void TransposeSimple(const Device& device, const Tensor& in,
+template <typename T, bool conjugate>
+void TransposeSimple(const CPUDevice& device, const Tensor& in,
                      const gtl::ArraySlice<int32> perm, Tensor* out) {
   const int ndims = in.dims();
   gtl::InlinedVector<int64, 8> in_strides = ComputeStride<int64>(in.shape());
   gtl::InlinedVector<int64, 8> out_strides = ComputeStride<int64>(out->shape());
   const T* p = reinterpret_cast<const T*>(in.tensor_data().data());
   T* q = reinterpret_cast<T*>(const_cast<char*>((out->tensor_data().data())));
-  auto transpose_fn = [=](int64 begin, int64 end) {
+  auto transpose_fn = [=, &in_strides, &out_strides, &perm](int64 begin,
+                                                            int64 end) {
     for (int64 o_idx = begin; o_idx < end; ++o_idx) {
       int64 i_idx = 0;
       int64 t = o_idx;
@@ -64,7 +65,7 @@ void TransposeSimple(const Device& device, const Tensor& in,
   device.parallelFor(in.NumElements(), cost, std::move(transpose_fn));
 }
 
-}  // end namespace internal
+}  // namespace
 
 template <typename T, bool conjugate>
 struct Transpose<CPUDevice, T, conjugate> {
@@ -88,32 +89,47 @@ struct Transpose<CPUDevice, T, conjugate> {
                                                        out);
         break;
       default:
-        internal::TransposeSimple<CPUDevice, T, conjugate>(d, in, perm, out);
+        TransposeSimple<T, conjugate>(d, in, perm, out);
         break;
     }
   }
 };
 
-template <>
-Status DoTranspose(const CPUDevice& device, const Tensor& in,
-                   const gtl::ArraySlice<int32> perm, Tensor* out) {
-  return internal::DoTransposeImpl<CPUDevice>::run(device, in, perm,
-                                                   false /* conjugate */, out);
-}
+#define INSTANTIATE(DEVICE)                                                 \
+  template <>                                                               \
+  Status DoTranspose(const DEVICE& device, const Tensor& in,                \
+                     const gtl::ArraySlice<int32> perm, Tensor* out) {      \
+    return internal::DoTransposeImpl(device, in, perm, /*conjugate=*/false, \
+                                     out);                                  \
+  }                                                                         \
+  template <>                                                               \
+  Status DoConjugateTranspose(const DEVICE& device, const Tensor& in,       \
+                              const gtl::ArraySlice<int32> perm,            \
+                              Tensor* out) {                                \
+    return internal::DoTransposeImpl(device, in, perm, /*conjugate=*/true,  \
+                                     out);                                  \
+  }                                                                         \
+  template <>                                                               \
+  Status DoMatrixTranspose(const DEVICE& device, const Tensor& in,          \
+                           Tensor* out) {                                   \
+    return internal::DoMatrixTransposeImpl(device, in, /*conjugate=*/false, \
+                                           out);                            \
+  }                                                                         \
+  template <>                                                               \
+  Status DoConjugateMatrixTranspose(const DEVICE& device, const Tensor& in, \
+                                    Tensor* out) {                          \
+    return internal::DoMatrixTransposeImpl(device, in, /*conjugate=*/true,  \
+                                           out);                            \
+  }
 
-template <>
-Status DoConjugateTranspose(const CPUDevice& device, const Tensor& in,
-                            const gtl::ArraySlice<int32> perm, Tensor* out) {
-  return internal::DoTransposeImpl<CPUDevice>::run(device, in, perm,
-                                                   true /* conjugate */, out);
-}
+INSTANTIATE(CPUDevice)
 
 #ifdef TENSORFLOW_USE_SYCL
 typedef Eigen::SyclDevice SYCLDevice;
 
 namespace internal {
-template <typename Device, typename T>
-void TransposeSYCL(const Device& d, const Tensor& in,
+template <typename T>
+void TransposeSYCL(const SYCLDevice& d, const Tensor& in,
                    const gtl::ArraySlice<int32> perm, bool conjugate,
                    Tensor* out) {
   switch (in.dims()) {
@@ -165,19 +181,11 @@ struct Transpose<SYCLDevice, string, conjugate> {
   }
 };
 
-template <>
-Status DoTranspose(const SYCLDevice& device, const Tensor& in,
-                   const gtl::ArraySlice<int32> perm, Tensor* out) {
-  return internal::DoTransposeImpl<SYCLDevice>::run(device, in, perm,
-                                                    false /* conjugate */, out);
-}
+// Explicit instantiation.
+template struct Transpose<SYCLDevice, string, false>;
 
-template <>
-Status DoConjugateTranspose(const SYCLDevice& device, const Tensor& in,
-                            const gtl::ArraySlice<int32> perm, Tensor* out) {
-  return internal::DoTransposeImpl<SYCLDevice>::run(device, in, perm,
-                                                    true /* conjugate */, out);
-}
+INSTANTIATE(SYCLDevice)
+#undef INSTANTIATE
 
 #endif  // TENSORFLOW_USE_SYCL
 
diff --git a/tensorflow/core/kernels/transpose_functor_gpu.cu.cc b/tensorflow/core/kernels/transpose_functor_gpu.cu.cc
index 364baf9a51..493dac9a7c 100644
--- a/tensorflow/core/kernels/transpose_functor_gpu.cu.cc
+++ b/tensorflow/core/kernels/transpose_functor_gpu.cu.cc
@@ -53,8 +53,8 @@ __global__ void TransposeKernel(int nthreads, const T* src, const int32* buf,
   }
 }
 
-template <typename Device, typename T, bool conjugate>
-void TransposeSimple(const Device& d, const Tensor& in,
+template <typename T, bool conjugate>
+void TransposeSimple(const GPUDevice& d, const Tensor& in,
                      const gtl::ArraySlice<int32> perm, Tensor* out) {
   // Ensures we can use 32-bit index.
   const int64 nelem = in.NumElements();
@@ -165,23 +165,9 @@ struct TransposeUsingTile<complex128, conjugate> {
   }
 };
 
-}  // end namespace internal
-
-template <>
-Status DoTranspose(const GPUDevice& device, const Tensor& in,
-                   const gtl::ArraySlice<int32> perm, Tensor* out) {
-  return internal::DoTransposeImpl<GPUDevice>::run(device, in, perm,
-                                                   false /* conjugate */, out);
-}
-
-template <>
-Status DoConjugateTranspose(const GPUDevice& device, const Tensor& in,
-                            const gtl::ArraySlice<int32> perm, Tensor* out) {
-  return internal::DoTransposeImpl<GPUDevice>::run(device, in, perm,
-                                                   true /* conjugate */, out);
-}
+}  // namespace internal
 
-// Transpose kernel specialized for CPU Device.
+// Transpose kernel specialized for GPU Device.
 template <typename T, bool conjugate>
 struct Transpose<GPUDevice, T, conjugate> {
   static void run(const GPUDevice& d, const Tensor& in,
@@ -216,19 +202,43 @@ struct Transpose<GPUDevice, T, conjugate> {
         }
         break;
       default:
-        internal::TransposeSimple<GPUDevice, T, conjugate>(d, in, perm, out);
+        internal::TransposeSimple<T, conjugate>(d, in, perm, out);
         break;
     }
   }
 };
 
-template <>
-struct Transpose<GPUDevice, string> {
+template <bool conjugate>
+struct Transpose<GPUDevice, string, conjugate> {
   static void run(const GPUDevice& d, const Tensor& in,
                   const gtl::ArraySlice<int32> perm, Tensor* out) {
     LOG(FATAL) << "Transpose of DT_STRING tensor not supported on GPU.";
   }
 };
 
+// Explicit instantiation.
+template struct Transpose<GPUDevice, string, false>;
+
+template <>
+Status DoTranspose(const GPUDevice& device, const Tensor& in,
+                   const gtl::ArraySlice<int32> perm, Tensor* out) {
+  return internal::DoTransposeImpl(device, in, perm, /*conjugate=*/false, out);
+}
+template <>
+Status DoConjugateTranspose(const GPUDevice& device, const Tensor& in,
+                            const gtl::ArraySlice<int32> perm, Tensor* out) {
+  return internal::DoTransposeImpl(device, in, perm, /*conjugate=*/true, out);
+}
+template <>
+Status DoMatrixTranspose(const GPUDevice& device, const Tensor& in,
+                         Tensor* out) {
+  return internal::DoMatrixTransposeImpl(device, in, /*conjugate=*/false, out);
+}
+template <>
+Status DoConjugateMatrixTranspose(const GPUDevice& device, const Tensor& in,
+                                  Tensor* out) {
+  return internal::DoMatrixTransposeImpl(device, in, /*conjugate=*/true, out);
+}
+
 }  // namespace tensorflow
 #endif  // GOOGLE_CUDA
-- 
GitLab


From 34a4b21f8f9dea64d3e99a97f639396f2d5556d3 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 17 Oct 2017 16:19:30 -0700
Subject: [PATCH 493/909] Change GBDTClassifer to internally use twice
 differntiable implementation of multiclass cross entropy loss.

PiperOrigin-RevId: 172532288
---
 .../estimator_batch/estimator.py              | 15 +++++++-
 .../contrib/boosted_trees/examples/mnist.py   | 34 ++-----------------
 .../boosted_trees/python/utils/losses.py      |  5 ++-
 3 files changed, 21 insertions(+), 33 deletions(-)

diff --git a/tensorflow/contrib/boosted_trees/estimator_batch/estimator.py b/tensorflow/contrib/boosted_trees/estimator_batch/estimator.py
index f8028acbdb..01752416b3 100644
--- a/tensorflow/contrib/boosted_trees/estimator_batch/estimator.py
+++ b/tensorflow/contrib/boosted_trees/estimator_batch/estimator.py
@@ -19,8 +19,10 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.contrib.boosted_trees.estimator_batch import model
+from tensorflow.contrib.boosted_trees.python.utils import losses
 from tensorflow.contrib.learn.python.learn.estimators import estimator
 from tensorflow.contrib.learn.python.learn.estimators import head as head_lib
+from tensorflow.python.ops import math_ops
 
 
 class GradientBoostedDecisionTreeClassifier(estimator.Estimator):
@@ -65,10 +67,21 @@ class GradientBoostedDecisionTreeClassifier(estimator.Estimator):
     Raises:
       ValueError: If learner_config is not valid.
     """
+    if n_classes > 2:
+      # For multi-class classification, use our loss implementation that
+      # supports second order derivative.
+      def loss_fn(labels, logits, weights=None):
+        result = losses.per_example_maxent_loss(
+            labels=labels, logits=logits, weights=weights,
+            num_classes=n_classes)
+        return math_ops.reduce_mean(result[0])
+    else:
+      loss_fn = None
     head = head_lib.multi_class_head(
         n_classes=n_classes,
         weight_column_name=weight_column_name,
-        enable_centered_bias=False)
+        enable_centered_bias=False,
+        loss_fn=loss_fn)
     if learner_config.num_classes == 0:
       learner_config.num_classes = n_classes
     elif learner_config.num_classes != n_classes:
diff --git a/tensorflow/contrib/boosted_trees/examples/mnist.py b/tensorflow/contrib/boosted_trees/examples/mnist.py
index a3b1cb5154..0539d77720 100644
--- a/tensorflow/contrib/boosted_trees/examples/mnist.py
+++ b/tensorflow/contrib/boosted_trees/examples/mnist.py
@@ -35,18 +35,13 @@ from __future__ import division
 from __future__ import print_function
 
 import argparse
-import functools
 import sys
 
 import numpy as np
 import tensorflow as tf
-from tensorflow.contrib import metrics as metrics_lib
-from tensorflow.contrib.boosted_trees.estimator_batch import custom_loss_head
-from tensorflow.contrib.boosted_trees.estimator_batch.estimator import GradientBoostedDecisionTreeEstimator
+from tensorflow.contrib.boosted_trees.estimator_batch.estimator import GradientBoostedDecisionTreeClassifier
 from tensorflow.contrib.boosted_trees.proto import learner_pb2
-from tensorflow.contrib.boosted_trees.python.utils import losses
 from tensorflow.contrib.learn import learn_runner
-from tensorflow.python.ops import math_ops
 
 
 def get_input_fn(dataset_split,
@@ -88,36 +83,13 @@ def _get_tfbt(output_dir):
   learner_config.growing_mode = growing_mode
   run_config = tf.contrib.learn.RunConfig(save_checkpoints_secs=300)
 
-  # Use Cross Entropy loss (the impl in losses is twice differentiable).
-  loss_fn = functools.partial(
-      losses.per_example_maxent_loss, num_classes=num_classes)
-  logit_dim = num_classes
   learner_config.multi_class_strategy = (
       learner_pb2.LearnerConfig.DIAGONAL_HESSIAN)
 
-  # Since we use custom head, we need to tell how accuracy is calculated.
-  def _multiclass_metrics(predictions, labels, weights):
-    """Prepares eval metrics for multiclass eval."""
-    metrics = dict()
-    logits = predictions["scores"]
-    classes = math_ops.argmax(logits, 1)
-    metrics["accuracy"] = metrics_lib.streaming_accuracy(
-        classes, labels, weights)
-    return metrics
-
-  metrics_fn = _multiclass_metrics
-  # Use custom loss head so we can provide our loss (cross entropy for
-  # multiclass).
-  head = custom_loss_head.CustomLossHead(
-      loss_fn=loss_fn,
-      link_fn=tf.identity,
-      logit_dimension=logit_dim,
-      metrics_fn=metrics_fn)
-
   # Create a TF Boosted trees estimator that can take in custom loss.
-  estimator = GradientBoostedDecisionTreeEstimator(
+  estimator = GradientBoostedDecisionTreeClassifier(
       learner_config=learner_config,
-      head=head,
+      n_classes=num_classes,
       examples_per_layer=FLAGS.examples_per_layer,
       model_dir=output_dir,
       num_trees=FLAGS.num_trees,
diff --git a/tensorflow/contrib/boosted_trees/python/utils/losses.py b/tensorflow/contrib/boosted_trees/python/utils/losses.py
index 4f128b2301..1e8b3ac08a 100644
--- a/tensorflow/contrib/boosted_trees/python/utils/losses.py
+++ b/tensorflow/contrib/boosted_trees/python/utils/losses.py
@@ -101,7 +101,10 @@ def per_example_maxent_loss(labels, weights, logits, num_classes, eps=1e-15):
 
   unweighted_loss = array_ops.expand_dims(-math_ops.log(probs_for_real_class),
                                           1)
-  return unweighted_loss * weights, control_flow_ops.no_op()
+  if weights is None:
+    return unweighted_loss, control_flow_ops.no_op()
+  else:
+    return unweighted_loss * weights, control_flow_ops.no_op()
 
 
 def per_example_squared_loss(labels, weights, predictions):
-- 
GitLab


From 0bf77c23c04394c25380fd09027d729388ebfba4 Mon Sep 17 00:00:00 2001
From: Akshay Agrawal <akshayka@google.com>
Date: Tue, 17 Oct 2017 16:28:53 -0700
Subject: [PATCH 494/909] Pass context handle, device name to EagerTensor
 constructor in convert_to_mixed_eager_tensors

PiperOrigin-RevId: 172533449
---
 tensorflow/python/eager/BUILD                      |  5 +++--
 tensorflow/python/eager/core_test.py               | 12 ++++++++++++
 tensorflow/python/eager/execute.py                 |  7 +++++--
 tensorflow/python/kernel_tests/BUILD               |  1 +
 tensorflow/python/kernel_tests/logging_ops_test.py |  2 ++
 5 files changed, 23 insertions(+), 4 deletions(-)

diff --git a/tensorflow/python/eager/BUILD b/tensorflow/python/eager/BUILD
index 5a2592287c..9e9a7f4c59 100644
--- a/tensorflow/python/eager/BUILD
+++ b/tensorflow/python/eager/BUILD
@@ -93,7 +93,6 @@ cuda_py_test(
         ":test",
         "//tensorflow/python:errors",
         "//tensorflow/python:framework_ops",
-        "//third_party/py/numpy",
     ],
 )
 
@@ -123,10 +122,12 @@ cuda_py_test(
         ":core",
         ":execute",
         ":test",
+        "//third_party/py/numpy",
         "//tensorflow/python:dtypes",
         "//tensorflow/python:errors",
-        "//tensorflow/python:pywrap_tensorflow",
+        "//tensorflow/python:framework_ops",
         "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:pywrap_tensorflow",
     ],
 )
 
diff --git a/tensorflow/python/eager/core_test.py b/tensorflow/python/eager/core_test.py
index 041d388fad..1de72240e3 100644
--- a/tensorflow/python/eager/core_test.py
+++ b/tensorflow/python/eager/core_test.py
@@ -20,6 +20,8 @@ from __future__ import print_function
 
 import threading
 
+import numpy as np
+
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.python import pywrap_tensorflow
 from tensorflow.python.eager import context
@@ -29,6 +31,7 @@ from tensorflow.python.eager import test
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
+from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util
 
 
@@ -460,6 +463,15 @@ class TFETest(test_util.TensorFlowTestCase):
       with context.device('pu:0'):
         _ = constant_op.constant(1)
 
+  def testConvertMixedEagerTensors(self):
+    array = np.zeros((), dtype=np.float32)
+    tensor = constant_op.constant(0., dtype=dtypes.float32)
+    types, tensors = execute_lib.convert_to_mixed_eager_tensors(
+        [array, tensor], context.context())
+    for typ, t in zip(types, tensors):
+      self.assertEquals(typ, dtypes.float32)
+      self.assertIsInstance(t, ops.EagerTensor)
+
 
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/python/eager/execute.py b/tensorflow/python/eager/execute.py
index 04634daba4..983c1ea73e 100644
--- a/tensorflow/python/eager/execute.py
+++ b/tensorflow/python/eager/execute.py
@@ -198,8 +198,11 @@ def args_to_matching_eager(l, ctx, default_dtype=None):
 
 
 def convert_to_mixed_eager_tensors(values, ctx):
-  v = [t if isinstance(t, ops.EagerTensor) else ops.EagerTensor(t, ctx)
-       for t in values]
+  v = [
+      t if isinstance(t, ops.EagerTensor) else ops.EagerTensor(
+          t, context=ctx._handle, device=ctx.device_name)  # pylint: disable=protected-access
+      for t in values
+  ]
   types = [t.dtype for t in v]
   return types, v
 
diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD
index f6ecd1f0b8..847c078971 100644
--- a/tensorflow/python/kernel_tests/BUILD
+++ b/tensorflow/python/kernel_tests/BUILD
@@ -462,6 +462,7 @@ tf_py_test(
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:control_flow_ops",
         "//tensorflow/python:framework_for_generated_wrappers",
+        "//tensorflow/python:framework_test_lib",
         "//tensorflow/python:gradients",
         "//tensorflow/python:logging_ops",
         "//tensorflow/python:math_ops",
diff --git a/tensorflow/python/kernel_tests/logging_ops_test.py b/tensorflow/python/kernel_tests/logging_ops_test.py
index 7fe65c57cc..28c85fa13a 100644
--- a/tensorflow/python/kernel_tests/logging_ops_test.py
+++ b/tensorflow/python/kernel_tests/logging_ops_test.py
@@ -20,6 +20,7 @@ from __future__ import print_function
 
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import gradients_impl
 from tensorflow.python.ops import logging_ops
@@ -58,6 +59,7 @@ class LoggingOpsTest(test.TestCase):
 
 class PrintGradientTest(test.TestCase):
 
+  @test_util.run_in_graph_and_eager_modes()
   def testPrintShape(self):
     inp = constant_op.constant(2.0, shape=[100, 32])
     inp_printed = logging_ops.Print(inp, [inp])
-- 
GitLab


From 33ce1d06393f773c5317bb38ab996c2a7b8aa429 Mon Sep 17 00:00:00 2001
From: Saurabh Saxena <srbs@google.com>
Date: Tue, 17 Oct 2017 16:58:02 -0700
Subject: [PATCH 495/909] Remove unused variable `i_` batch dataset ops.

PiperOrigin-RevId: 172536770
---
 tensorflow/core/kernels/batch_dataset_op.cc                 | 1 -
 tensorflow/core/kernels/dense_to_sparse_batch_dataset_op.cc | 1 -
 tensorflow/core/kernels/padded_batch_dataset_op.cc          | 1 -
 3 files changed, 3 deletions(-)

diff --git a/tensorflow/core/kernels/batch_dataset_op.cc b/tensorflow/core/kernels/batch_dataset_op.cc
index 631840081f..04a41451ea 100644
--- a/tensorflow/core/kernels/batch_dataset_op.cc
+++ b/tensorflow/core/kernels/batch_dataset_op.cc
@@ -181,7 +181,6 @@ class BatchDatasetOp : public UnaryDatasetOpKernel {
 
      private:
       mutex mu_;
-      int64 i_ GUARDED_BY(mu_);
       std::unique_ptr<IteratorBase> input_impl_ GUARDED_BY(mu_);
     };
 
diff --git a/tensorflow/core/kernels/dense_to_sparse_batch_dataset_op.cc b/tensorflow/core/kernels/dense_to_sparse_batch_dataset_op.cc
index 0174c8dfc8..e80d11eaea 100644
--- a/tensorflow/core/kernels/dense_to_sparse_batch_dataset_op.cc
+++ b/tensorflow/core/kernels/dense_to_sparse_batch_dataset_op.cc
@@ -245,7 +245,6 @@ class DenseToSparseBatchDatasetOp : public UnaryDatasetOpKernel {
 
      private:
       mutex mu_;
-      int64 i_ GUARDED_BY(mu_);
       std::unique_ptr<IteratorBase> input_impl_ GUARDED_BY(mu_);
     };
 
diff --git a/tensorflow/core/kernels/padded_batch_dataset_op.cc b/tensorflow/core/kernels/padded_batch_dataset_op.cc
index 7737f57b68..cfc77690b5 100644
--- a/tensorflow/core/kernels/padded_batch_dataset_op.cc
+++ b/tensorflow/core/kernels/padded_batch_dataset_op.cc
@@ -349,7 +349,6 @@ class PaddedBatchDatasetOp : public UnaryDatasetOpKernel {
 
      private:
       mutex mu_;
-      int64 i_ GUARDED_BY(mu_);
       std::unique_ptr<IteratorBase> input_impl_ GUARDED_BY(mu_);
     };
 
-- 
GitLab


From 5f865f703621fed07925b3828f4a731066d98fd6 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 17 Oct 2017 17:47:40 -0700
Subject: [PATCH 496/909] The new class will be used as the base class for the
 existing 2-4 dimensional array classes to share code as well as for creating
 higher dimensional arrays.

The API of the new class is kept compatible with the previous API to
limit the scope of this change.

PiperOrigin-RevId: 172543319
---
 tensorflow/compiler/xla/BUILD         |  23 ++
 tensorflow/compiler/xla/array.h       | 324 ++++++++++++++++++++++++++
 tensorflow/compiler/xla/array2d.h     | 131 ++---------
 tensorflow/compiler/xla/array3d.h     |  94 +-------
 tensorflow/compiler/xla/array4d.h     | 185 ++-------------
 tensorflow/compiler/xla/array_test.cc | 145 ++++++++++++
 6 files changed, 528 insertions(+), 374 deletions(-)
 create mode 100644 tensorflow/compiler/xla/array.h
 create mode 100644 tensorflow/compiler/xla/array_test.cc

diff --git a/tensorflow/compiler/xla/BUILD b/tensorflow/compiler/xla/BUILD
index 6c4c970ce8..be87506d3c 100644
--- a/tensorflow/compiler/xla/BUILD
+++ b/tensorflow/compiler/xla/BUILD
@@ -335,12 +335,33 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "array",
+    hdrs = ["array.h"],
+    deps = [
+        ":types",
+        ":util",
+        "//tensorflow/core:lib",
+    ],
+)
+
+tf_cc_test(
+    name = "array_test",
+    srcs = ["array_test.cc"],
+    deps = [
+        ":array",
+        ":test",
+        "//tensorflow/core:test_main",
+    ],
+)
+
 cc_library(
     name = "array2d",
     srcs = ["array2d.cc"],
     hdrs = ["array2d.h"],
     visibility = ["//visibility:public"],
     deps = [
+        ":array",
         ":types",
         ":util",
         "//tensorflow/core:lib",
@@ -362,6 +383,7 @@ cc_library(
     hdrs = ["array3d.h"],
     visibility = [":friends"],
     deps = [
+        ":array",
         ":types",
         "//tensorflow/core:lib",
     ],
@@ -383,6 +405,7 @@ cc_library(
     hdrs = ["array4d.h"],
     visibility = [":friends"],
     deps = [
+        ":array",
         ":array2d",
         ":types",
         "//tensorflow/core:lib",
diff --git a/tensorflow/compiler/xla/array.h b/tensorflow/compiler/xla/array.h
new file mode 100644
index 0000000000..3be7060a83
--- /dev/null
+++ b/tensorflow/compiler/xla/array.h
@@ -0,0 +1,324 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_COMPILER_XLA_ARRAY_H_
+#define TENSORFLOW_COMPILER_XLA_ARRAY_H_
+
+#include <algorithm>
+#include <array>
+#include <functional>
+#include <initializer_list>
+#include <iterator>
+#include <memory>
+#include <random>
+#include <vector>
+
+#include "tensorflow/compiler/xla/types.h"
+#include "tensorflow/core/lib/core/bits.h"
+#include "tensorflow/core/lib/strings/str_util.h"
+#include "tensorflow/core/lib/strings/strcat.h"
+#include "tensorflow/core/platform/logging.h"
+#include "tensorflow/core/platform/macros.h"
+#include "tensorflow/core/platform/types.h"
+
+namespace xla {
+
+// General N dimensional array class with arbitrary value type.
+template <typename T>
+class Array {
+ public:
+  // Creates a new array with the specified dimensions.
+  explicit Array(const std::vector<int64>& sizes) : Array(sizes, T()) {}
+
+  // Creates a new array with the specified dimensions and specified value for
+  // every cell.
+  Array(const std::vector<int64>& sizes, T value)
+      : sizes_(sizes), values_(new T[num_elements()]) {
+    Fill(value);
+  }
+
+  // Creates a 2D array from the given nested initializer list. The outer
+  // initializer list is the first dimension, the inner is the second dimension.
+  // For example, {{1, 2, 3}, {4, 5, 6}} results in an array with n1=2 and n2=3.
+  Array(std::initializer_list<std::initializer_list<T>> values)
+      : Array(ToInt64Vector({values.size(), values.begin()->size()})) {
+    int64 idx = 0;
+    for (const auto& it1 : values) {
+      for (const auto& it2 : it1) {
+        values_[idx] = it2;
+        ++idx;
+      }
+    }
+    CHECK(idx == num_elements());
+  }
+
+  // Creates a 3D array from the given nested initializer list. The outer
+  // initializer list is the first dimension, and so on.
+  Array(std::initializer_list<std::initializer_list<std::initializer_list<T>>>
+            values)
+      : Array(ToInt64Vector({values.size(), values.begin()->size(),
+                             values.begin()->begin()->size()})) {
+    int64 idx = 0;
+    for (const auto& it1 : values) {
+      for (const auto& it2 : it1) {
+        for (const auto& it3 : it2) {
+          values_[idx] = it3;
+          ++idx;
+        }
+      }
+    }
+    CHECK(idx == num_elements());
+  }
+
+  // Creates a 4D array from the given nested initializer list. The outer
+  // initializer list is the first dimension, and so on.
+  Array(std::initializer_list<
+        std::initializer_list<std::initializer_list<std::initializer_list<T>>>>
+            values)
+      : Array(ToInt64Vector({values.size(), values.begin()->size(),
+                             values.begin()->begin()->size(),
+                             values.begin()->begin()->begin()->size()})) {
+    int64 idx = 0;
+    for (const auto& it1 : values) {
+      for (const auto& it2 : it1) {
+        for (const auto& it3 : it2) {
+          for (const auto& it4 : it3) {
+            values_[idx] = it4;
+            ++idx;
+          }
+        }
+      }
+    }
+    CHECK(idx == num_elements());
+  }
+
+  Array(const Array<T>& other)
+      : sizes_(other.sizes_), values_(new T[num_elements()]) {
+    std::copy(&other.values_[0], &other.values_[0] + num_elements(),
+              &values_[0]);
+  }
+
+  Array<T>& operator=(const Array<T>& other) {
+    sizes_ = other.sizes_;
+    values_.reset(new T[num_elements()]);
+    std::copy(&other.values_[0], &other.values_[0] + num_elements(),
+              &values_[0]);
+    return *this;
+  }
+
+  // Fills the array with the specified value.
+  void Fill(const T& value) {
+    std::fill(&values_[0], &values_[0] + num_elements(), value);
+  }
+
+  // Fills the array with sequentially increasing values.
+  void FillIota(const T& value) {
+    std::iota(&values_[0], &values_[0] + num_elements(), value);
+  }
+
+  // Fills the array with the sequence i*multiplier for i=0,1,...
+  void FillWithMultiples(const T& multiplier) {
+    for (int64 i = 0; i < num_elements(); ++i) {
+      values_[i] = i * multiplier;
+    }
+  }
+
+  // Fills the array with random normal variables with the specified mean.
+  void FillRandom(const T& value, const double mean = 0.0,
+                  const int seed = 12345) {
+    std::mt19937 g(seed);
+    std::normal_distribution<double> distribution(mean,
+                                                  static_cast<double>(value));
+    for (int64 i = 0; i < num_elements(); ++i) {
+      values_[i] = static_cast<T>(distribution(g));
+    }
+  }
+
+  // Sets all the values in the array to values specified in the container.
+  template <typename Container = std::initializer_list<T>>
+  void SetValues(const Container& container) {
+    CHECK_EQ(std::distance(std::begin(container), std::end(container)),
+             num_elements());
+    std::copy(std::begin(container), std::end(container), &values_[0]);
+  }
+
+  // Invokes a callback with the (indices, value_ptr) for each cell in the
+  // array.
+  void Each(std::function<void(tensorflow::gtl::ArraySlice<int64>, T*)> f) {
+    std::vector<int64> index(sizes_.size());
+    for (int64 i = 0; i < num_elements(); ++i, next_index(&index)) {
+      f(index, &values_[i]);
+    }
+  }
+
+  // Invokes a callback with the (indices, value) for each cell in the array.
+  void Each(
+      std::function<void(tensorflow::gtl::ArraySlice<int64>, T)> f) const {
+    std::vector<int64> index(sizes_.size());
+    for (int64 i = 0; i < num_elements(); ++i, next_index(&index)) {
+      f(index, values_[i]);
+    }
+  }
+
+  // Returns the value at the cell specified by the indexes. The number of
+  // arguments have to match with the number of dimensions for the array.
+  template <typename... Dims>
+  const T& operator()(Dims... dims) const {
+    // We are using a std::array to avoid having to allocate memory in this
+    // function for performance reasons.
+    std::array<int64, sizeof...(dims)> indexes{{static_cast<int64>(dims)...}};
+    return values_[calculate_index(indexes)];
+  }
+
+  // Returns the value at the cell specified by the indexes. The number of
+  // arguments have to match with the number of dimensions for the array.
+  template <typename... Dims>
+  T& operator()(Dims... dims) {
+    // We are using a std::array to avoid having to allocate memory in this
+    // function for performance reasons.
+    std::array<int64, sizeof...(dims)> indexes{{static_cast<int64>(dims)...}};
+    return values_[calculate_index(indexes)];
+  }
+
+  // Low-level accessor for stuff like memcmp, handle with care. Returns pointer
+  // to the underlying storage of the array (similarly to std::vector::data()).
+  T* data() const {
+    // TODO(tberghammer): Get rid of the const_cast. Currently it is needed
+    // because the Eigen backend needs a non-const pointers even for reading
+    // from the array.
+    return const_cast<Array*>(this)->values_.get();
+  }
+
+  // Returns the size of the dimension at the given index.
+  int64 dim(int64 n) const {
+    CHECK(n < sizes_.size());
+    return sizes_[n];
+  }
+
+  // Returns a vector containing the dimensions of the array.
+  const std::vector<int64>& dimensions() const { return sizes_; }
+
+  int64 num_dimensions() const { return sizes_.size(); }
+
+  // Returns the total number of elements in the array.
+  int64 num_elements() const {
+    return std::accumulate(sizes_.begin(), sizes_.end(), 1,
+                           std::multiplies<int64>());
+  }
+
+  bool operator==(const Array<T>& other) const {
+    if (sizes_.size() != other.sizes_.size()) {
+      return false;
+    }
+    for (int64 i = 0; i < sizes_.size(); ++i) {
+      if (sizes_[i] != other.sizes_[i]) {
+        return false;
+      }
+    }
+    for (int64 i = 0; i < num_elements(); ++i) {
+      if (values_[i] != other.values_[i]) {
+        return false;
+      }
+    }
+    return true;
+  }
+
+  bool operator!=(const Array<T>& other) const { return !(*this == other); }
+
+  // Returns a string representation of the array suitable for debugging.
+  string ToString() const {
+    std::vector<string> pieces;
+    std::vector<int64> index(sizes_.size());
+    do {
+      // Emit leading spaces and opening square brackets
+      if (index.back() == 0) {
+        for (int64 i = sizes_.size() - 1; i >= 0; --i) {
+          if (i == 0 || index[i - 1] != 0) {
+            for (int64 j = 0; j < sizes_.size(); ++j) {
+              pieces.push_back(j < i ? " " : "[");
+            }
+            break;
+          }
+        }
+      }
+
+      pieces.push_back(
+          tensorflow::strings::AlphaNum(values_[calculate_index(index)])
+              .data());
+
+      // Emit comma if it isn't the last element
+      if (index.back() != sizes_.back() - 1) {
+        pieces.push_back(", ");
+      }
+
+      // Emit closing square brackets
+      for (int64 i = sizes_.size() - 1; i >= 0; --i) {
+        if (index[i] != sizes_[i] - 1) {
+          break;
+        }
+        pieces.push_back("]");
+        if (i != 0 && index[i - 1] != sizes_[i - 1] - 1) {
+          pieces.push_back(",\n");
+        }
+      }
+    } while (next_index(&index));
+    return tensorflow::str_util::Join(pieces, "");
+  }
+
+ private:
+  // Converts an initializer_list of type U to a vector of type int64. Used by
+  // the initializer list based constructors to convert the size type into int64
+  // to be passed to the size based constructor.
+  template <typename U>
+  static std::vector<int64> ToInt64Vector(
+      const std::initializer_list<U>& data) {
+    return std::vector<int64>(data.begin(), data.end());
+  }
+
+  // Returns the linear index from the list of per-dimension indexes. Function
+  // is templated so can be used with an std::array from operator() to avoid
+  // memory allocation.
+  template <typename U>
+  int64 calculate_index(const U& indexes) const {
+    CHECK_EQ(sizes_.size(), indexes.size());
+    int64 index = 0;
+    for (int64 i = 0; i < sizes_.size(); ++i) {
+      index *= sizes_[i];
+      index += indexes[i];
+    }
+    return index;
+  }
+
+  // Advances the specified set of indexes and returns true if we haven't
+  // wrapped around (i.e. result isnt {0, 0, ...}).
+  bool next_index(std::vector<int64>* index) const {
+    CHECK_EQ(index->size(), sizes_.size());
+    for (int64 i = sizes_.size() - 1; i >= 0; --i) {
+      (*index)[i]++;
+      if ((*index)[i] < sizes_[i]) {
+        return true;
+      }
+      (*index)[i] = 0;
+    }
+    return false;
+  }
+
+  std::vector<int64> sizes_;
+  std::unique_ptr<T[]> values_;
+};
+
+}  // namespace xla
+
+#endif  // TENSORFLOW_COMPILER_XLA_ARRAY_H_
diff --git a/tensorflow/compiler/xla/array2d.h b/tensorflow/compiler/xla/array2d.h
index 2737764cbd..bb85fbee9b 100644
--- a/tensorflow/compiler/xla/array2d.h
+++ b/tensorflow/compiler/xla/array2d.h
@@ -24,6 +24,7 @@ limitations under the License.
 #include <random>
 #include <vector>
 
+#include "tensorflow/compiler/xla/array.h"
 #include "tensorflow/compiler/xla/types.h"
 #include "tensorflow/core/lib/core/bits.h"
 #include "tensorflow/core/lib/strings/str_util.h"
@@ -34,93 +35,30 @@ limitations under the License.
 
 namespace xla {
 
-// Simple 2D array structure.
-//
-// The data layout in major-to-minor order is: n1, n2.
 template <typename T>
-class Array2D {
+class Array2D : public Array<T> {
  public:
-  // Creates an empty array.
-  Array2D() : n1_(0), n2_(0) {}
+  Array2D() : Array<T>(std::vector<int64>{0, 0}) {}
 
-  // Creates an array of dimensions n1 x n2, uninitialized values.
   Array2D(const int64 n1, const int64 n2)
-      : n1_(n1), n2_(n2), values_(new T[n1 * n2]()) {
-    Fill(T());
-  }
+      : Array<T>(std::vector<int64>{n1, n2}) {}
 
-  // Creates an array of dimensions n1 x n2, initialized to value.
   Array2D(const int64 n1, const int64 n2, const T value)
-      : n1_(n1), n2_(n2), values_(new T[n1 * n2]()) {
-    Fill(value);
-  }
+      : Array<T>({n1, n2}, value) {}
 
   // Creates an array from the given nested initializer list. The outer
   // initializer list is the first dimension; the inner is the second dimension.
   // For example, {{1, 2, 3}, {4, 5, 6}} results in an array with n1=2 and n2=3.
   Array2D(std::initializer_list<std::initializer_list<T>> values)
-      : Array2D(values.size(), values.begin()->size()) {
-    int64 n1 = 0;
-    for (auto n1_it = values.begin(); n1_it != values.end(); ++n1_it, ++n1) {
-      int64 n2 = 0;
-      for (auto n2_it = n1_it->begin(); n2_it != n1_it->end(); ++n2_it, ++n2) {
-        (*this)(n1, n2) = *n2_it;
-      }
-    }
-  }
+      : Array<T>(values) {}
 
-  Array2D(const Array2D<T>& other) : Array2D(other.n1(), other.n2()) {
-    std::copy(&other.values_[0], &other.values_[0] + num_elements(),
-              &values_[0]);
-  }
-
-  Array2D<T>& operator=(const Array2D<T>& other) {
-    n1_ = other.n1();
-    n2_ = other.n2();
-    values_.reset(new T[num_elements()]);
-    std::copy(&other.values_[0], &other.values_[0] + num_elements(),
-              &values_[0]);
-    return *this;
-  }
+  Array2D(const Array2D<T>& other) : Array<T>(other) {}
 
-  T& operator()(const int64 i1, const int64 i2) {
-    CHECK_LT(i1, n1_);
-    CHECK_LT(i2, n2_);
-    return values_[i1 * n2_ + i2];
-  }
+  int64 n1() const { return this->dim(0); }
+  int64 n2() const { return this->dim(1); }
 
-  const T& operator()(const int64 i1, const int64 i2) const {
-    CHECK_LT(i1, n1_);
-    CHECK_LT(i2, n2_);
-    return values_[i1 * n2_ + i2];
-  }
-
-  // Access to the array's dimensions. height() and width() provide the
-  // canonical interpretation of the array n1 x n2 having n1 rows of n2 columns
-  // each (height is number of rows; width is number of columns).
-  int64 n1() const { return n1_; }
-  int64 n2() const { return n2_; }
-  int64 height() const { return n1_; }
-  int64 width() const { return n2_; }
-  int64 num_elements() const { return n1_ * n2_; }
-
-  // Low-level accessor for stuff like memcmp, handle with care. Returns pointer
-  // to the underlying storage of the array (similarly to std::vector::data()).
-  T* data() const { return const_cast<Array2D*>(this)->values_.get(); }
-
-  // Fills the array with the given value.
-  void Fill(const T& value) {
-    std::fill(&values_[0], &values_[0] + num_elements(), value);
-  }
-
-  // Applies f to all cells in this array, in row-major order.
-  void Each(std::function<void(int64, int64, T*)> f) {
-    for (int64 i0 = 0; i0 < n1(); ++i0) {
-      for (int64 i1 = 0; i1 < n2(); ++i1) {
-        f(i0, i1, &(*this)(i0, i1));
-      }
-    }
-  }
+  int64 height() const { return this->dim(0); }
+  int64 width() const { return this->dim(1); }
 
   // Fills the array with a pattern of values of the form:
   //
@@ -136,55 +74,14 @@ class Array2D {
     }
   }
 
-  // Fills the array with random normal variables of deviation value.
-  void FillRandom(const T& value, const double mean = 0.0,
-                  const int seed = 12345) {
-    std::mt19937 g(seed);
-    std::normal_distribution<double> distribution(mean,
-                                                  static_cast<double>(value));
-    for (int64 i = 0; i < num_elements(); ++i) {
-      values_[i] = static_cast<T>(distribution(g));
-    }
-  }
-
-  // Returns a readable string representation of the array.
-  string ToString() const {
-    std::vector<string> pieces = {"["};
-    for (int64 row = 0; row < height(); ++row) {
-      pieces.push_back("[");
-      for (int64 col = 0; col < width(); ++col) {
-        pieces.push_back(tensorflow::strings::StrCat((*this)(row, col)));
-        pieces.push_back(", ");
-      }
-      pieces.pop_back();
-      pieces.push_back("]");
-      pieces.push_back(",\n ");
-    }
-    pieces.pop_back();
-    pieces.push_back("]");
-    return tensorflow::str_util::Join(pieces, "");
-  }
-
-  bool operator==(const Array2D<T>& other) const {
-    if (n1() != other.n1() || n2() != other.n2()) {
-      return false;
-    }
+  // Applies f to all cells in this array, in row-major order.
+  void Each(std::function<void(int64, int64, T*)> f) {
     for (int64 i0 = 0; i0 < n1(); ++i0) {
       for (int64 i1 = 0; i1 < n2(); ++i1) {
-        if ((*this)(i0, i1) != other(i0, i1)) {
-          return false;
-        }
+        f(i0, i1, &(*this)(i0, i1));
       }
     }
-    return true;
   }
-
-  bool operator!=(const Array2D<T>& other) const { return !(*this == other); }
-
- private:
-  int64 n1_;
-  int64 n2_;
-  std::unique_ptr<T[]> values_;
 };
 
 // Returns a linspace-populated Array2D in the range [from, to] (inclusive)
diff --git a/tensorflow/compiler/xla/array3d.h b/tensorflow/compiler/xla/array3d.h
index 124ccd1975..e9449f01ad 100644
--- a/tensorflow/compiler/xla/array3d.h
+++ b/tensorflow/compiler/xla/array3d.h
@@ -24,6 +24,7 @@ limitations under the License.
 #include <numeric>
 #include <random>
 
+#include "tensorflow/compiler/xla/array.h"
 #include "tensorflow/compiler/xla/types.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/macros.h"
@@ -32,22 +33,16 @@ limitations under the License.
 namespace xla {
 
 // Simple 3D array structure.
-//
-// The data layout in major-to-minor order is: n1, n2, n3.
 template <typename T>
-class Array3D {
+class Array3D : public Array<T> {
  public:
   // Creates an array of dimensions n1 x n2 x n3, uninitialized values.
   Array3D(const int64 n1, const int64 n2, const int64 n3)
-      : n1_(n1), n2_(n2), n3_(n3), values_(new T[n1 * n2 * n3]) {
-    Fill(T());
-  }
+      : Array<T>(std::vector<int64>{n1, n2, n3}) {}
 
   // Creates an array of dimensions n1 x n2 x n3, initialized to value.
   Array3D(const int64 n1, const int64 n2, const int64 n3, const T value)
-      : n1_(n1), n2_(n2), n3_(n3), values_(new T[n1 * n2 * n3]) {
-    Fill(value);
-  }
+      : Array<T>(std::vector<int64>{n1, n2, n3}, value) {}
 
   // Creates an array from the given nested initializer list. The outer
   // initializer list is the first dimension, and so on.
@@ -58,84 +53,11 @@ class Array3D {
   // results in an array with n1=3, n2=4, n3=2.
   Array3D(std::initializer_list<std::initializer_list<std::initializer_list<T>>>
               values)
-      : Array3D(values.size(), values.begin()->size(),
-                values.begin()->begin()->size()) {
-    int64 n1 = 0;
-    for (auto n1_it = values.begin(); n1_it != values.end(); ++n1_it, ++n1) {
-      int64 n2 = 0;
-      for (auto n2_it = n1_it->begin(); n2_it != n1_it->end(); ++n2_it, ++n2) {
-        int64 n3 = 0;
-        for (auto n3_it = n2_it->begin(); n3_it != n2_it->end();
-             ++n3_it, ++n3) {
-          (*this)(n1, n2, n3) = *n3_it;
-        }
-      }
-    }
-  }
+      : Array<T>(values) {}
 
-  Array3D(const Array3D<T>& other)
-      : Array3D(other.n1(), other.n2(), other.n3()) {
-    std::copy(&other.values_[0], &other.values_[0] + num_elements(),
-              &values_[0]);
-  }
-
-  Array3D<T>& operator=(const Array3D<T>& other) {
-    n1_ = other.n1();
-    n2_ = other.n2();
-    n3_ = other.n3();
-    values_.reset(new T[num_elements()]);
-    std::copy(&other.values_[0], &other.values_[0] + num_elements(),
-              &values_[0]);
-    return *this;
-  }
-
-  T& operator()(const int64 i1, const int64 i2, const int64 i3) {
-    CHECK_LT(i1, n1_);
-    CHECK_LT(i2, n2_);
-    CHECK_LT(i3, n3_);
-    return values_[i1 * n2_ * n3_ + i2 * n3_ + i3];
-  }
-
-  const T& operator()(const int64 i1, const int64 i2, const int64 i3) const {
-    CHECK_LT(i1, n1_);
-    CHECK_LT(i2, n2_);
-    CHECK_LT(i3, n3_);
-    return values_[i1 * n2_ * n3_ + i2 * n3_ + i3];
-  }
-
-  // Access to the array's dimensions.
-  int64 n1() const { return n1_; }
-  int64 n2() const { return n2_; }
-  int64 n3() const { return n3_; }
-  int64 num_elements() const { return n1_ * n2_ * n3_; }
-
-  // Fills the array with the given value.
-  void Fill(const T& value) {
-    std::fill(&values_[0], &values_[0] + num_elements(), value);
-  }
-
-  // Fills the array with sequentially increasing values.
-  void FillIota(const T& value) {
-    std::iota(&values_[0], &values_[0] + num_elements(), value);
-  }
-
-  // Fills the array with random normal values with a mean of 0 and standard
-  // deviation of value.
-  void FillRandom(const T& value, const double mean = 0.0,
-                  const int seed = 12345) {
-    std::mt19937 g(seed);
-    std::normal_distribution<double> distribution(mean,
-                                                  static_cast<double>(value));
-    for (int64 i = 0; i < num_elements(); ++i) {
-      values_[i] = static_cast<T>(distribution(g));
-    }
-  }
-
- private:
-  int64 n1_;
-  int64 n2_;
-  int64 n3_;
-  std::unique_ptr<T[]> values_;
+  int64 n1() const { return this->dim(0); }
+  int64 n2() const { return this->dim(1); }
+  int64 n3() const { return this->dim(2); }
 };
 
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/array4d.h b/tensorflow/compiler/xla/array4d.h
index 4c7fce1aaf..f8b2b2afe5 100644
--- a/tensorflow/compiler/xla/array4d.h
+++ b/tensorflow/compiler/xla/array4d.h
@@ -26,6 +26,7 @@ limitations under the License.
 #include <string>
 #include <vector>
 
+#include "tensorflow/compiler/xla/array.h"
 #include "tensorflow/compiler/xla/array2d.h"
 #include "tensorflow/compiler/xla/types.h"
 #include "tensorflow/core/lib/gtl/array_slice.h"
@@ -53,23 +54,15 @@ namespace xla {
 // more than one name is given above. See operator() for the exact
 // calculation of 1d indices from 4d indices.
 template <typename T>
-class Array4D {
+class Array4D : public Array<T> {
  public:
   // Creates a 4D array, uninitialized values.
   Array4D(int64 planes, int64 depth, int64 height, int64 width)
-      : planes_(planes),
-        depth_(depth),
-        height_(height),
-        width_(width),
-        values_(new T[planes * depth * height * width]) {
-    Fill(T());
-  }
+      : Array<T>(std::vector<int64>{planes, depth, height, width}) {}
 
   // Creates a 4D array, initialized to value.
   Array4D(int64 planes, int64 depth, int64 height, int64 width, T value)
-      : Array4D(planes, depth, height, width) {
-    Fill(value);
-  }
+      : Array<T>(std::vector<int64>{planes, depth, height, width}, value) {}
 
   // Creates a 4D array, filled with values.
   //
@@ -80,144 +73,26 @@ class Array4D {
   Array4D(int64 planes, int64 depth, int64 height, int64 width,
           const Container& values)
       : Array4D(planes, depth, height, width) {
-    SetValues(values);
+    this->SetValues(values);
   }
 
   // Construct an Array4D with the given nested initializer list.
   Array4D(std::initializer_list<std::initializer_list<
               std::initializer_list<std::initializer_list<T>>>>
               values)
-      : Array4D(values.size(), values.begin()->size(),
-                values.begin()->begin()->size(),
-                values.begin()->begin()->begin()->size()) {
-    int64 plane = 0;
-    for (const auto values_in_plane : values) {
-      DCHECK_EQ(values_in_plane.size(), depth_);
-      int64 depth = 0;
-      for (const auto values_in_depth : values_in_plane) {
-        DCHECK_EQ(values_in_depth.size(), height_);
-        int64 height = 0;
-        for (const auto values_in_height : values_in_depth) {
-          DCHECK_EQ(values_in_height.size(), width_);
-          int64 width = 0;
-          for (const auto element_value : values_in_height) {
-            (*this)(plane, depth, height, width) = element_value;
-            ++width;
-          }
-          ++height;
-        }
-        ++depth;
-      }
-      ++plane;
-    }
-  }
-
-  Array4D(const Array4D<T>& other)
-      : Array4D(other.planes(), other.depth(), other.height(), other.width()) {
-    std::copy(&other.values_[0], &other.values_[0] + num_elements(),
-              &values_[0]);
-  }
-
-  Array4D<T>& operator=(const Array4D<T>& other) {
-    planes_ = other.planes();
-    depth_ = other.depth();
-    height_ = other.height();
-    width_ = other.width();
-    values_.reset(new T[num_elements()]);
-    std::copy(&other.values_[0], &other.values_[0] + num_elements(),
-              &values_[0]);
-    return *this;
-  }
-
-  T& operator()(int64 plane, int64 depth, int64 height, int64 width) {
-    CHECK_LT(plane, planes_);
-    CHECK_LT(depth, depth_);
-    CHECK_LT(height, height_);
-    CHECK_LT(width, width_);
-    return values_[plane * (depth_ * height_ * width_) +
-                   depth * (height_ * width_) + height * (width_) + width];
-  }
-  const T& operator()(int64 plane, int64 depth, int64 height,
-                      int64 width) const {
-    return const_cast<Array4D*>(this)->operator()(plane, depth, height, width);
-  }
-
-  int64 width() const { return width_; }
-  int64 height() const { return height_; }
-  int64 depth() const { return depth_; }
-  int64 planes() const { return planes_; }
+      : Array<T>(values) {}
 
   // Numerically-named aliases for the various dimensions. This matches the
   // dimension names used in array3d.
-  int64 n4() const { return width_; }
-  int64 n3() const { return height_; }
-  int64 n2() const { return depth_; }
-  int64 n1() const { return planes_; }
-  int64 num_elements() const { return width_ * height_ * depth_ * planes_; }
-
-  // Sets all the values in the array to values.
-  template <typename Container = std::initializer_list<T>>
-  void SetValues(const Container& container) {
-    CHECK_EQ(std::distance(std::begin(container), std::end(container)),
-             num_elements());
-    std::copy(std::begin(container), std::end(container), &values_[0]);
-  }
-
-  // Fills the array with the given value.
-  void Fill(const T& value) {
-    std::fill(&values_[0], &values_[0] + num_elements(), value);
-  }
+  int64 n4() const { return this->dim(3); }
+  int64 n3() const { return this->dim(2); }
+  int64 n2() const { return this->dim(1); }
+  int64 n1() const { return this->dim(0); }
 
-  // Fills the array with iota.
-  void FillIota(const T& value) {
-    std::iota(&values_[0], &values_[0] + num_elements(), value);
-  }
-
-  // Fills the array with random variable with a deviation of value and a mean
-  // of mean.
-  void FillRandom(const T& value, const double mean = 0.0,
-                  const int seed = 12345) {
-    std::mt19937 g(seed);
-    std::normal_distribution<double> distribution(mean,
-                                                  static_cast<double>(value));
-    for (int64 i = 0; i < num_elements(); ++i) {
-      values_[i] = static_cast<T>(distribution(g));
-    }
-  }
-
-  // Fills values with the sequence i*multiplier for i=0,1,...
-  void FillWithMultiples(float multiplier) {
-    for (int64 i = 0; i < num_elements(); ++i) {
-      values_[i] = i * multiplier;
-    }
-  }
-
-  // Invokes a callback with the (indices, value_ptr) for each cell in the 4D
-  // array.
-  void Each(std::function<void(tensorflow::gtl::ArraySlice<int64>, T*)> f) {
-    for (int64 plane = 0; plane < planes(); ++plane) {
-      for (int64 depth = 0; depth < this->depth(); ++depth) {
-        for (int64 height = 0; height < this->height(); ++height) {
-          for (int64 width = 0; width < this->width(); ++width) {
-            auto& value = (*this)(plane, depth, height, width);
-            f({plane, depth, height, width}, &value);
-          }
-        }
-      }
-    }
-  }
-
-  // Invokes a callback with the (indices, value) for each cell in the 4D array.
-  void Each(
-      std::function<void(tensorflow::gtl::ArraySlice<int64>, T)> f) const {
-    // We const_cast to be able to use the common non-const implementation,
-    // but prevent modification of the data by passing it by-value to the
-    // caller.
-    const_cast<Array4D*>(this)->Each(
-        [&f](tensorflow::gtl::ArraySlice<int64> indices, T* value) {
-          f(indices, *value);
-        });
-  }
+  int64 width() const { return this->dim(3); }
+  int64 height() const { return this->dim(2); }
+  int64 depth() const { return this->dim(1); }
+  int64 planes() const { return this->dim(0); }
 
   // Fills all of the {p,z} with the array provided, which specifies {y,x}.
   void FillWithYX(const Array2D<T>& value) {
@@ -267,38 +142,6 @@ class Array4D {
       }
     }
   }
-
-  // Returns a string representation of the 4D array suitable for debugging.
-  string ToString() const {
-    std::vector<string> pieces = {
-        tensorflow::strings::Printf("p=%lld,z=%lld,y=%lld,x=%lld {\n", planes(),
-                                    depth(), height(), width())};
-    for (int64 plane = 0; plane < planes_; ++plane) {
-      pieces.push_back("  {\n");
-      for (int64 depth = 0; depth < depth_; ++depth) {
-        pieces.push_back("    {\n");
-        for (int64 height = 0; height < height_; ++height) {
-          pieces.push_back("      {");
-          for (int64 width = 0; width < width_; ++width) {
-            pieces.push_back(tensorflow::strings::StrCat(
-                (*this)(plane, depth, height, width), ", "));
-          }
-          pieces.push_back("},\n");
-        }
-        pieces.push_back("    },\n");
-      }
-      pieces.push_back("  },\n");
-    }
-    pieces.push_back("}");
-    return tensorflow::str_util::Join(pieces, "");
-  }
-
- private:
-  int64 planes_;
-  int64 depth_;
-  int64 height_;
-  int64 width_;
-  std::unique_ptr<T[]> values_;
 };
 
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/array_test.cc b/tensorflow/compiler/xla/array_test.cc
new file mode 100644
index 0000000000..093784f541
--- /dev/null
+++ b/tensorflow/compiler/xla/array_test.cc
@@ -0,0 +1,145 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/array.h"
+
+#include <initializer_list>
+
+#include "tensorflow/compiler/xla/test.h"
+
+namespace xla {
+namespace {
+
+TEST(ArrayTest, UninitializedDimsCtor) {
+  Array<int> uninit({2, 3});
+  EXPECT_EQ(uninit.num_dimensions(), 2);
+  EXPECT_EQ(uninit.dim(0), 2);
+  EXPECT_EQ(uninit.dim(1), 3);
+  EXPECT_EQ(uninit.num_elements(), 6);
+}
+
+TEST(ArrayTest, FillCtor) {
+  Array<int> fullof7({1, 2, 3}, 7);
+
+  EXPECT_EQ(fullof7.dim(0), 1);
+  EXPECT_EQ(fullof7.dim(1), 2);
+  EXPECT_EQ(fullof7.dim(2), 3);
+
+  for (int64 n0 = 0; n0 < fullof7.dim(0); ++n0) {
+    for (int64 n1 = 0; n1 < fullof7.dim(1); ++n1) {
+      for (int64 n2 = 0; n2 < fullof7.dim(2); ++n2) {
+        EXPECT_EQ(fullof7(n0, n1, n2), 7);
+      }
+    }
+  }
+}
+
+TEST(ArrayTest, InitializerListCtor) {
+  Array<int> arr({{1, 2, 3}, {4, 5, 6}});
+
+  EXPECT_EQ(arr.dim(0), 2);
+  EXPECT_EQ(arr.dim(1), 3);
+
+  EXPECT_EQ(arr(0, 0), 1);
+  EXPECT_EQ(arr(0, 1), 2);
+  EXPECT_EQ(arr(0, 2), 3);
+  EXPECT_EQ(arr(1, 0), 4);
+  EXPECT_EQ(arr(1, 1), 5);
+  EXPECT_EQ(arr(1, 2), 6);
+}
+
+TEST(ArrayTest, IndexingReadWrite) {
+  Array<int> arr({2, 3});
+
+  EXPECT_EQ(arr(1, 1), 0);
+  EXPECT_EQ(arr(1, 2), 0);
+  arr(1, 1) = 51;
+  arr(1, 2) = 61;
+  EXPECT_EQ(arr(1, 1), 51);
+  EXPECT_EQ(arr(1, 2), 61);
+}
+
+TEST(ArrayTest, IndexingReadWriteBool) {
+  Array<bool> arr{{false, true, false}, {false, true, false}};
+
+  EXPECT_EQ(arr(0, 1), true);
+  EXPECT_EQ(arr(0, 2), false);
+  arr(0, 1) = false;
+  arr(0, 2) = true;
+  EXPECT_EQ(arr(0, 1), false);
+  EXPECT_EQ(arr(0, 2), true);
+}
+
+TEST(ArrayTest, Fill) {
+  Array<int> fullof7({2, 3}, 7);
+  for (int64 n1 = 0; n1 < fullof7.dim(0); ++n1) {
+    for (int64 n2 = 0; n2 < fullof7.dim(1); ++n2) {
+      EXPECT_EQ(fullof7(n1, n2), 7);
+    }
+  }
+
+  fullof7.Fill(11);
+  for (int64 n1 = 0; n1 < fullof7.dim(0); ++n1) {
+    for (int64 n2 = 0; n2 < fullof7.dim(1); ++n2) {
+      EXPECT_EQ(fullof7(n1, n2), 11);
+    }
+  }
+}
+
+TEST(ArrayTest, DataPointer) {
+  Array<int> arr{{1, 2, 3}, {4, 5, 6}};
+  EXPECT_EQ(arr.data()[0], 1);
+}
+
+TEST(ArrayTest, Stringification1D) {
+  Array<int64> arr({2}, 1);
+  const string expected = R"([1, 1])";
+  EXPECT_EQ(expected, arr.ToString());
+}
+
+TEST(ArrayTest, Stringification2D) {
+  Array<int64> arr({2, 3}, 7);
+  const string expected = "[[7, 7, 7],\n [7, 7, 7]]";
+  EXPECT_EQ(expected, arr.ToString());
+}
+
+TEST(ArrayTest, Stringification3D) {
+  Array<int64> arr({2, 3, 4}, 5);
+  const string expected = R"([[[5, 5, 5, 5],
+  [5, 5, 5, 5],
+  [5, 5, 5, 5]],
+ [[5, 5, 5, 5],
+  [5, 5, 5, 5],
+  [5, 5, 5, 5]]])";
+  EXPECT_EQ(expected, arr.ToString());
+}
+
+TEST(ArrayTest, Each) {
+  Array<int64> arr({2, 3, 4});
+  arr.FillWithMultiples(1);
+
+  int64 each_count = 0, each_sum = 0;
+  arr.Each([&](tensorflow::gtl::ArraySlice<int64> idx, int cell) {
+    int64 lin_idx = idx[0] * 12 + idx[1] * 4 + idx[2];
+    EXPECT_EQ(lin_idx, cell);
+    each_count++;
+    each_sum += cell;
+  });
+  EXPECT_EQ(arr.num_elements(), each_count);
+  EXPECT_EQ(arr.num_elements() * (arr.num_elements() - 1) / 2, each_sum);
+}
+
+}  // namespace
+}  // namespace xla
-- 
GitLab


From 8c8bf69563a0d5b7f52d6153f09580946296d1f7 Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Tue, 17 Oct 2017 18:31:51 -0700
Subject: [PATCH 497/909] should_use_result eager-safe.
 ResourceVariable.numpy()

PiperOrigin-RevId: 172547480
---
 .../python/kernel_tests/resource_variable_ops_test.py       | 6 ++++++
 tensorflow/python/ops/resource_variable_ops.py              | 6 ++++++
 tensorflow/python/util/tf_should_use.py                     | 5 +++++
 3 files changed, 17 insertions(+)

diff --git a/tensorflow/python/kernel_tests/resource_variable_ops_test.py b/tensorflow/python/kernel_tests/resource_variable_ops_test.py
index 8cf8286ed1..ec9192b1a0 100644
--- a/tensorflow/python/kernel_tests/resource_variable_ops_test.py
+++ b/tensorflow/python/kernel_tests/resource_variable_ops_test.py
@@ -62,6 +62,12 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase):
                                    "Expected float got int32."):
         _ = resource_variable_ops.read_variable_op(handle, dtype=dtypes.float32)
 
+  def testEagerInitializedValue(self):
+    with context.eager_mode():
+      variable = resource_variable_ops.ResourceVariable(1.0, name="eager-init")
+      self.assertAllEqual(variable.numpy(), 1.0)
+      self.assertAllEqual(variable.initialized_value().numpy(), 1.0)
+
   def testAssignVariableDtypeMismatchEager(self):
     with context.eager_mode():
       handle = resource_variable_ops.var_handle_op(
diff --git a/tensorflow/python/ops/resource_variable_ops.py b/tensorflow/python/ops/resource_variable_ops.py
index cbfa141256..2c9a3ff19a 100644
--- a/tensorflow/python/ops/resource_variable_ops.py
+++ b/tensorflow/python/ops/resource_variable_ops.py
@@ -513,6 +513,12 @@ class ResourceVariable(variables.Variable):
       raise RuntimeError("Trying to eval in EAGER mode")
     return self._graph_element.eval(session=session)
 
+  def numpy(self):
+    if context.in_graph_mode():
+      raise NotImplementedError(
+          "numpy() is only available when eager execution is enabled.")
+    return self.read_value().numpy()
+
   def _set_save_slice_info(self, save_slice_info):
     """Sets the slice info for this `ResourceVariable`.
 
diff --git a/tensorflow/python/util/tf_should_use.py b/tensorflow/python/util/tf_should_use.py
index d9b2e6fcd7..99081cb294 100644
--- a/tensorflow/python/util/tf_should_use.py
+++ b/tensorflow/python/util/tf_should_use.py
@@ -44,6 +44,11 @@ def _add_should_use_warning(x, fatal_error=False):
   if x is None:  # special corner case where x is None
     return x
 
+  # TODO(apassos) we don't have an easier way to check because importing context
+  # or ops here would create a BUILD dependency cycle.
+  if type(x).__name__ == 'EagerTensor':
+    return x
+
   def override_method(method):
     def fn(self, *args, **kwargs):
       return method(self, *args, **kwargs)
-- 
GitLab


From a308914e24613a49e7f5d2550a19802e4de1283c Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Tue, 17 Oct 2017 18:32:09 -0700
Subject: [PATCH 498/909] EagerTensor.__array__

Also rewrites most eager tests to remove now-unnecessary calls to .numpy()

PiperOrigin-RevId: 172547504
---
 tensorflow/python/eager/backprop_test.py      |  54 ++++----
 tensorflow/python/eager/core_test.py          |  28 ++--
 tensorflow/python/eager/function_test.py      |  36 ++---
 .../python/eager/graph_callable_test.py       |  20 +--
 tensorflow/python/eager/ops_test.py           | 124 +++++++++---------
 tensorflow/python/eager/tape_test.py          |  22 ++--
 tensorflow/python/eager/tensor_test.py        |  14 +-
 tensorflow/python/framework/ops.py            |   4 +
 8 files changed, 153 insertions(+), 149 deletions(-)

diff --git a/tensorflow/python/eager/backprop_test.py b/tensorflow/python/eager/backprop_test.py
index 2645d542c0..002be95d0f 100644
--- a/tensorflow/python/eager/backprop_test.py
+++ b/tensorflow/python/eager/backprop_test.py
@@ -90,8 +90,8 @@ class BackpropTest(test.TestCase):
       return math_ops.add(c, constant_op.constant(3.0))
 
     grads_and_vars = backprop.implicit_grad(fn)()
-    self.assertEqual(grads_and_vars[0][0].numpy(), 1.0)
-    self.assertEqual(id(grads_and_vars[0][1]), id(x))
+    self.assertAllEqual(grads_and_vars[0][0], 1.0)
+    self.assertAllEqual(id(grads_and_vars[0][1]), id(x))
 
   def testDy(self):
 
@@ -99,7 +99,7 @@ class BackpropTest(test.TestCase):
       return x
 
     grad_fn = backprop.gradients_function(f)
-    self.assertAllEqual(2., grad_fn(1., dy=2.)[0].numpy())
+    self.assertAllEqual(2., grad_fn(1., dy=2.)[0])
 
   def testImplicitGradOverEmbeddingLookup(self):
     batch_size = 8
@@ -131,13 +131,13 @@ class BackpropTest(test.TestCase):
       tf_opt = training.GradientDescentOptimizer(0.1)
       tf_embedding.initializer.run()
 
-      self.assertAllClose(tf_grad.indices.eval(), grad.indices.numpy())
-      self.assertAllClose(tf_grad.values.eval(), grad.values.numpy())
+      self.assertAllClose(tf_grad.indices.eval(), grad.indices)
+      self.assertAllClose(tf_grad.values.eval(), grad.values)
 
       tf_opt.apply_gradients([(tf_grad, tf_embedding)]).run()
       expected = tf_embedding.eval()
     opt.apply_gradients([(grad, embedding)])
-    self.assertAllClose(expected, embedding.read_value().numpy())
+    self.assertAllClose(expected, embedding.read_value())
 
   def testGradientNone(self):
 
@@ -167,7 +167,7 @@ class BackpropTest(test.TestCase):
 
     f = constant_op.constant([[0.1]])
     grad = backprop.gradients_function(second, [0])(f)[0]
-    self.assertAllEqual([[0.0]], grad.numpy())
+    self.assertAllEqual([[0.0]], grad)
 
   def testMakeVJP(self):
 
@@ -176,8 +176,8 @@ class BackpropTest(test.TestCase):
 
     wrapped_fn = backprop.make_vjp(f)
     result, vjp = wrapped_fn(constant_op.constant(3.0))
-    self.assertEqual(result.numpy(), 9.0)
-    self.assertEqual(vjp(2.0)[0].numpy(), 12.0)
+    self.assertAllEqual(result, 9.0)
+    self.assertAllEqual(vjp(2.0)[0], 12.0)
 
   def testGradGrad(self):
 
@@ -190,7 +190,7 @@ class BackpropTest(test.TestCase):
 
     gradgrad = backprop.gradients_function(grad, [0])
 
-    self.assertAllEqual(gradgrad(constant_op.constant(3.0))[0].numpy(), 2.0)
+    self.assertAllEqual(gradgrad(constant_op.constant(3.0))[0], 2.0)
 
   def testGradGradExp(self):
 
@@ -200,7 +200,7 @@ class BackpropTest(test.TestCase):
 
     gradgrad = backprop.gradients_function(grad, [0])
 
-    self.assertAllEqual(gradgrad(constant_op.constant(0.0))[0].numpy(), 1.0)
+    self.assertAllEqual(gradgrad(constant_op.constant(0.0))[0], 1.0)
 
   def testGPU(self):
     if not context.context().num_gpus():
@@ -215,7 +215,7 @@ class BackpropTest(test.TestCase):
         return math_ops.add(c, constant_op.constant(3.0)).as_cpu_tensor()
 
     grad = backprop.gradients_function(fn, [0])(constant_op.constant(1.0))[0]
-    self.assertEqual(grad.numpy(), 1.0)
+    self.assertAllEqual(grad, 1.0)
 
   def testGPUImplicitGrad(self):
     if not context.context().num_gpus():
@@ -240,7 +240,7 @@ class BackpropTest(test.TestCase):
       return math_ops.add(c, constant_op.constant(3.0))
 
     grad = backprop.gradients_function(fn, [0])(constant_op.constant(1.0))[0]
-    self.assertEqual(grad.numpy(), 1.0)
+    self.assertAllEqual(grad, 1.0)
 
   def testTensorCopyGPU2CPU2GPU(self):
     if not context.context().num_gpus():
@@ -254,7 +254,7 @@ class BackpropTest(test.TestCase):
       b = constant_op.constant(2.0)
 
     grad = backprop.gradients_function(f, [0])(a, b)[0]
-    self.assertEqual(grad.numpy(), 1.0)
+    self.assertAllEqual(grad, 1.0)
 
   def testEmptyParams(self):
 
@@ -264,8 +264,8 @@ class BackpropTest(test.TestCase):
     x = constant_op.constant(1.0)
     y = constant_op.constant(2.0)
     dx, dy = backprop.gradients_function(fn)(x, y)
-    self.assertAllEqual(dx.numpy(), y.numpy())
-    self.assertAllEqual(dy.numpy(), x.numpy())
+    self.assertAllEqual(dx, y.numpy())
+    self.assertAllEqual(dy, x.numpy())
 
   def testUnconnectedNone(self):
     v = resource_variable_ops.ResourceVariable(
@@ -285,9 +285,9 @@ class BackpropTest(test.TestCase):
     x = 2.0
     y = 3.0
     val, (dx, dy) = val_and_grads_fn(x, y)
-    self.assertAllClose(val.numpy(), x * y)
-    self.assertAllEqual(dx.numpy(), y)
-    self.assertAllEqual(dy.numpy(), x)
+    self.assertAllClose(val, x * y)
+    self.assertAllEqual(dx, y)
+    self.assertAllEqual(dy, x)
 
   def testNonEmptyParamsForValueAndGradFunction(self):
     def fn(a, b):
@@ -297,9 +297,9 @@ class BackpropTest(test.TestCase):
     x = 2.0
     y = 3.0
     val, grads = val_and_grad_fn(x, y)
-    self.assertAllClose(val.numpy(), x * y)
+    self.assertAllClose(val, x * y)
     self.assertEqual(1, len(grads))
-    self.assertAllEqual(grads[0].numpy(), x)
+    self.assertAllEqual(grads[0], x)
 
   def testTensorCopyCPU2GPU2CPU(self):
     if not context.context().num_gpus():
@@ -317,7 +317,7 @@ class BackpropTest(test.TestCase):
       b = constant_op.constant(2.0)
 
     grad = backprop.gradients_function(f, [0])(a, b)[0]
-    self.assertEqual(grad.numpy(), 1.0)
+    self.assertAllEqual(grad, 1.0)
 
   def testGetAttrType(self):
     typ = backprop.op_attr_type('Add', 'T')
@@ -372,7 +372,7 @@ class BackpropTest(test.TestCase):
       return math_ops.reduce_mean(b)
 
     grad = backprop.implicit_grad(fn)()[0][0]
-    self.assertAllEqual([1.0], grad.numpy())
+    self.assertAllEqual([1.0], grad)
 
   def testOutput(self):
 
@@ -382,7 +382,7 @@ class BackpropTest(test.TestCase):
     x = constant_op.constant([0.0, 1.0, 2.0])
 
     grad = backprop.gradients_function(multiout)(x)[0]
-    self.assertAllEqual([1.0, 3.0, 5.0], grad.numpy())
+    self.assertAllEqual([1.0, 3.0, 5.0], grad)
 
   def testMultiValuePreservesIfNotDiffedAgainst(self):
 
@@ -394,7 +394,7 @@ class BackpropTest(test.TestCase):
     s = [1, 1, 1, 1]
 
     grad = backprop.gradients_function(tfe_conv2d, params=(0,))(i, k, s)[0]
-    self.assertAllEqual([[[[2.0]]]], grad.numpy())
+    self.assertAllEqual([[[[2.0]]]], grad)
 
   def testSameObjectForMultipleArguments(self):
 
@@ -483,8 +483,8 @@ class BackpropTest(test.TestCase):
     grads_and_vars = g()
     self.assertEqual(1, len(grads_and_vars))
     grad, var = grads_and_vars[0]
-    self.assertEqual(7, grad.numpy())
-    self.assertEqual(x, var)
+    self.assertAllEqual(7, grad)
+    self.assertAllEqual(x, var)
 
   def testCustomGradient(self):
 
diff --git a/tensorflow/python/eager/core_test.py b/tensorflow/python/eager/core_test.py
index 1de72240e3..54a0be6dd9 100644
--- a/tensorflow/python/eager/core_test.py
+++ b/tensorflow/python/eager/core_test.py
@@ -151,14 +151,14 @@ class TFETest(test_util.TensorFlowTestCase):
 
     cpu = constant_op.constant([[1., 2.], [3., 4.]])
     c2g = cpu.as_gpu_tensor()
-    self.assertAllEqual(c2g.numpy(), cpu.numpy())
+    self.assertAllEqual(c2g, cpu.numpy())
 
   def testCopyFromCPUToCPU(self):
     ta = constant_op.constant([[1, 2], [3, 4]])
     tb = ta.as_cpu_tensor()
 
     self.assertNotEqual(id(ta), id(tb))
-    self.assertAllEqual(ta.numpy(), tb.numpy())
+    self.assertAllEqual(ta, tb.numpy())
 
   def testRegisterExceptionClass(self):
     with self.assertRaises(TypeError):
@@ -174,7 +174,7 @@ class TFETest(test_util.TensorFlowTestCase):
         num_outputs=1,
         inputs=[three, five],
         attrs=('T', three.dtype.as_datatype_enum))[0]
-    self.assertEqual(15, product.numpy())
+    self.assertAllEqual(15, product)
 
   def testExecuteTooManyNumOutputs(self):
     # num_outputs provided is 50, but only one output is produced.
@@ -184,7 +184,7 @@ class TFETest(test_util.TensorFlowTestCase):
         num_outputs=50,
         inputs=[constant_op.constant(3), constant_op.constant(5)],
         attrs=('T', dtypes.int32.as_datatype_enum))[0]
-    self.assertEqual(15, product.numpy())
+    self.assertAllEqual(15, product)
 
   def testMatMulGPU(self):
     if not context.context().num_gpus():
@@ -197,7 +197,7 @@ class TFETest(test_util.TensorFlowTestCase):
         inputs=[three, five],
         attrs=('transpose_a', False, 'transpose_b', False, 'T',
                three.dtype.as_datatype_enum))[0]
-    self.assertEqual([[15.0]], product.numpy())
+    self.assertAllEqual([[15.0]], product)
 
   def testExecuteStringAttr(self):
     checked_three = execute(
@@ -222,7 +222,7 @@ class TFETest(test_util.TensorFlowTestCase):
         num_outputs=1,
         inputs=[constant_op.constant(3.0), constant_op.constant(2.9)],
         attrs=('tolerance', 0.3, 'T', dtypes.float32.as_datatype_enum))[0]
-    self.assertTrue(almost_equal.numpy())
+    self.assertTrue(almost_equal)
 
   def testExecuteFloatAttrBadValue(self):
     with self.assertRaises(errors.InvalidArgumentError):
@@ -238,7 +238,7 @@ class TFETest(test_util.TensorFlowTestCase):
         num_outputs=1,
         inputs=[constant_op.constant(3), constant_op.constant(4)],
         attrs=('T', dtypes.int32.as_datatype_enum, 'N', 2))[0]
-    self.assertEqual(7, total.numpy())
+    self.assertAllEqual(7, total)
 
   def testExecuteIntAttrBadValue(self):
     with self.assertRaises(errors.InvalidArgumentError):
@@ -257,7 +257,7 @@ class TFETest(test_util.TensorFlowTestCase):
                 constant_op.constant([[5]])],
         attrs=('transpose_a', True, 'transpose_b', False, 'T',
                dtypes.int32.as_datatype_enum))[0]
-    self.assertEqual([[15]], product.numpy())
+    self.assertAllEqual([[15]], product)
 
   def testExecuteShapeAttr(self):
     execute(
@@ -310,7 +310,7 @@ class TFETest(test_util.TensorFlowTestCase):
         inputs=[constant_op.constant([3.0, 5.0, 7.0])],
         attrs=('T', dtypes.float32.as_datatype_enum, 'boundaries', [4.0,
                                                                     6.0]))[0]
-    self.assertAllEqual([0, 1, 2], b.numpy())
+    self.assertAllEqual([0, 1, 2], b)
 
   def testExecuteListFloatAttrBadValue(self):
     with self.assertRaises(errors.InvalidArgumentError):
@@ -335,7 +335,7 @@ class TFETest(test_util.TensorFlowTestCase):
         num_outputs=1,
         inputs=[constant_op.constant([[[3.0]]])],
         attrs=('T', dtypes.float32.as_datatype_enum, 'squeeze_dims', [0, 2]))[0]
-    self.assertAllEqual([3], b.numpy())
+    self.assertAllEqual([3], b)
 
   def testExecuteListIntAttrBadValue(self):
     with self.assertRaises(errors.InvalidArgumentError):
@@ -407,9 +407,9 @@ class TFETest(test_util.TensorFlowTestCase):
         inputs=[constant_op.constant(split_dim),
                 constant_op.constant(value)],
         attrs=('num_split', 3, 'T', dtypes.int32.as_datatype_enum))
-    self.assertAllEqual([[0], [3]], x1.numpy())
-    self.assertAllEqual([[1], [4]], x2.numpy())
-    self.assertAllEqual([[2], [5]], x3.numpy())
+    self.assertAllEqual([[0], [3]], x1)
+    self.assertAllEqual([[1], [4]], x2)
+    self.assertAllEqual([[2], [5]], x3)
 
   def testExecuteBadNumOutputsArgument(self):
     with self.assertRaises(TypeError):
@@ -442,7 +442,7 @@ class TFETest(test_util.TensorFlowTestCase):
     x = constant_op.constant(1)
     three_x = add(add(x, x), x)
     self.assertEquals(dtypes.int32, three_x.dtype)
-    self.assertEquals(3, three_x.numpy())
+    self.assertAllEqual(3, three_x)
 
   def testOperationWithNoInputsRunsOnDevice(self):
     if not context.context().num_gpus():
diff --git a/tensorflow/python/eager/function_test.py b/tensorflow/python/eager/function_test.py
index e27f9ebc27..e9e396b49b 100644
--- a/tensorflow/python/eager/function_test.py
+++ b/tensorflow/python/eager/function_test.py
@@ -53,7 +53,7 @@ class FunctionTest(test.TestCase):
 
     t = constant_op.constant([[1.0, 2.0], [3.0, 4.0]])
     out = sq(t)
-    self.assertAllEqual(out.numpy(), math_ops.matmul(t, t).numpy())
+    self.assertAllEqual(out, math_ops.matmul(t, t).numpy())
 
   def testGraphModeWithGradients(self):
     v = resource_variable_ops.ResourceVariable(1.0)
@@ -66,7 +66,7 @@ class FunctionTest(test.TestCase):
 
       return backprop.implicit_grad(inner)()[0][0]
 
-    self.assertAllEqual(step().numpy(), 2.0)
+    self.assertAllEqual(step(), 2.0)
 
   def testTensorConversionWithDefun(self):
 
@@ -74,7 +74,7 @@ class FunctionTest(test.TestCase):
     def f(x):
       return math_ops.add(x, constant_op.constant(3))
 
-    self.assertAllEqual(5, f(constant_op.constant(2)).numpy())
+    self.assertAllEqual(5, f(constant_op.constant(2)))
 
   def testTensorConversionCall(self):
 
@@ -86,7 +86,7 @@ class FunctionTest(test.TestCase):
     def g(x):
       return f(f(x))
 
-    self.assertAllEqual(8, g(constant_op.constant(2)).numpy())
+    self.assertAllEqual(8, g(constant_op.constant(2)))
 
   def testDefunCallBackprop(self):
 
@@ -98,7 +98,7 @@ class FunctionTest(test.TestCase):
     def g(x):
       return backprop.gradients_function(f, [0])(x)[0]
 
-    self.assertAllEqual(2, g(constant_op.constant(2)).numpy())
+    self.assertAllEqual(2, g(constant_op.constant(2)))
 
   def testGraphModeEagerGradError(self):
     with context.graph_mode():
@@ -149,7 +149,7 @@ class FunctionTest(test.TestCase):
       return f(x)
 
     g = backprop.implicit_grad(g)(constant_op.constant(1.0))[0][0]
-    self.assertEqual(g.numpy(), 1.0)
+    self.assertAllEqual(g, 1.0)
 
   def testGradient(self):
     matmul = function.defun(math_ops.matmul)
@@ -159,7 +159,7 @@ class FunctionTest(test.TestCase):
 
     t = constant_op.constant([[1.0, 2.0], [3.0, 4.0]])
     grad_t, = backprop.gradients_function(sq, [0])(t)
-    self.assertAllEqual(grad_t.numpy(), [[6, 6], [14, 14]])
+    self.assertAllEqual(grad_t, [[6, 6], [14, 14]])
 
   def testGradientInFunction(self):
 
@@ -167,7 +167,7 @@ class FunctionTest(test.TestCase):
     def f(x):
       return backprop.gradients_function(lambda y: y * y, [0])(x)[0]
 
-    self.assertEqual(f(constant_op.constant(1.0)).numpy(), 2.0)
+    self.assertAllEqual(f(constant_op.constant(1.0)), 2.0)
 
   def testFunctionOnDevice(self):
     if not context.context().num_gpus():
@@ -176,7 +176,7 @@ class FunctionTest(test.TestCase):
     x = constant_op.constant([1.]).as_gpu_tensor()
     f = function.defun(math_ops.add)
     y = f(x, x).as_cpu_tensor()
-    self.assertAllEqual(y.numpy(), [2.])
+    self.assertAllEqual(y, [2.])
 
   def testFunctionHandlesInputsOnDifferentDevices(self):
     if not context.context().num_gpus():
@@ -187,7 +187,7 @@ class FunctionTest(test.TestCase):
     value = constant_op.constant([1., 2.]).as_gpu_tensor()
     shape = constant_op.constant([2, 1])
     reshaped = reshape(value, shape).as_cpu_tensor()
-    self.assertAllEqual(reshaped.numpy(), [[1], [2]])
+    self.assertAllEqual(reshaped, [[1], [2]])
 
   def testFunctionHandlesInputsPlacedOnTheWrongDeviceGracefully(self):
     if not context.context().num_gpus():
@@ -210,7 +210,7 @@ class FunctionTest(test.TestCase):
       return my_function(x)[0]
 
     g = backprop.gradients_function(wrapper, [0])(constant_op.constant(0.0))
-    self.assertAllEqual(g[0].numpy(), 1.)
+    self.assertAllEqual(g[0], 1.)
 
   def testNoneOutput(self):
 
@@ -231,7 +231,7 @@ class FunctionTest(test.TestCase):
     def add_one(x):
       return add(x, 1)
 
-    self.assertAllEqual(3, add_one(constant_op.constant(2)).numpy())
+    self.assertAllEqual(3, add_one(constant_op.constant(2)))
 
   def testSequenceInputs(self):
     clip_by_global_norm = function.defun(clip_ops.clip_by_global_norm)
@@ -258,13 +258,13 @@ class FunctionTest(test.TestCase):
         constant_op.constant(5)
     ])
     self.assertEqual(len(ret), 2)
-    self.assertEqual(ret[0][0].numpy(), 2)
-    self.assertEqual(ret[0][1][0][0].numpy(), 8)
-    self.assertEqual(ret[0][1][0][1].numpy(), 4)
+    self.assertAllEqual(ret[0][0], 2)
+    self.assertAllEqual(ret[0][1][0][0], 8)
+    self.assertAllEqual(ret[0][1][0][1], 4)
     self.assertTrue(isinstance(ret[0][1][0], tuple))
-    self.assertEqual(ret[0][1][1].numpy(), 6)
-    self.assertEqual(ret[0][2].numpy(), 10)
-    self.assertEqual(ret[1].numpy(), 15)
+    self.assertAllEqual(ret[0][1][1], 6)
+    self.assertAllEqual(ret[0][2], 10)
+    self.assertAllEqual(ret[1], 15)
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/python/eager/graph_callable_test.py b/tensorflow/python/eager/graph_callable_test.py
index e77a33981d..548e16a909 100644
--- a/tensorflow/python/eager/graph_callable_test.py
+++ b/tensorflow/python/eager/graph_callable_test.py
@@ -57,7 +57,7 @@ class GraphCallableTest(test.TestCase):
       v.assign(x)
 
     my_function(constant_op.constant(4, dtype=dtypes.float32))
-    self.assertEqual(4, my_function.variables[0].read_value().numpy())
+    self.assertAllEqual(4, my_function.variables[0].read_value())
 
   def testFunctionWithoutReturnValueAndArgs(self):
 
@@ -68,7 +68,7 @@ class GraphCallableTest(test.TestCase):
       v.assign(4)
 
     my_function()
-    self.assertEqual(4, my_function.variables[0].read_value().numpy())
+    self.assertAllEqual(4, my_function.variables[0].read_value())
 
   def testVariableAPI(self):
 
@@ -113,7 +113,7 @@ class GraphCallableTest(test.TestCase):
       v.assign(v * x)
       return v.read_value()
 
-    self.assertEqual(my_function(constant_op.constant(2.0)).numpy(), 6.0)
+    self.assertAllEqual(my_function(constant_op.constant(2.0)), 6.0)
 
   def testEmptyInitializer(self):
 
@@ -149,7 +149,7 @@ class GraphCallableTest(test.TestCase):
     def f(x):
       return math_ops.add(x, constant_op.constant(3))
 
-    self.assertAllEqual(5, f(constant_op.constant(2)).numpy())
+    self.assertAllEqual(5, f(constant_op.constant(2)))
 
   def testNestedFunction(self):
 
@@ -165,7 +165,7 @@ class GraphCallableTest(test.TestCase):
     def add_one(x):
       return add(x, 1)
 
-    self.assertAllEqual(3, add_one(constant_op.constant(2)).numpy())
+    self.assertAllEqual(3, add_one(constant_op.constant(2)))
 
   # TODO(ashankar): Make this work.
   # The problem is that the two graph_callables (for add_one and add_two)
@@ -187,8 +187,8 @@ class GraphCallableTest(test.TestCase):
       return add(x, 2)
 
     two = constant_op.constant(2)
-    self.assertAllEqual(3, add_one(two).numpy())
-    self.assertAllEqual(4, add_two(two).numpy())
+    self.assertAllEqual(3, add_one(two))
+    self.assertAllEqual(4, add_two(two))
 
   def testNestedSequenceInputs(self):
     sd = graph_callable.ShapeAndDtype(shape=(), dtype=dtypes.float32)
@@ -205,11 +205,11 @@ class GraphCallableTest(test.TestCase):
               constant_op.constant(4.)]
     ret = my_op(inputs)
     self.assertEqual(len(ret), 2.)
-    self.assertEqual(ret[1].numpy(), 10.)
+    self.assertAllEqual(ret[1], 10.)
 
     my_op.variables[0].assign(1.)
     ret = my_op(inputs)
-    self.assertEqual(ret[1].numpy(), 11.)
+    self.assertAllEqual(ret[1], 11.)
 
   def testVariableShapeIsTensorShape(self):
     @graph_callable.graph_callable([])
@@ -243,7 +243,7 @@ class GraphCallableTest(test.TestCase):
 
     grad_fn = backprop.implicit_grad(my_function)
     grads_and_vars = list(zip(*grad_fn()))
-    self.assertEqual(6., grads_and_vars[0][0].numpy())
+    self.assertAllEqual(6., grads_and_vars[0][0])
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/eager/ops_test.py b/tensorflow/python/eager/ops_test.py
index 78423468ea..6d1a5fe264 100644
--- a/tensorflow/python/eager/ops_test.py
+++ b/tensorflow/python/eager/ops_test.py
@@ -41,7 +41,7 @@ class OpsTest(test_util.TensorFlowTestCase):
     three = constant_op.constant(3)
     five = constant_op.constant(5)
     product = three * five
-    self.assertEqual(15, product.numpy())
+    self.assertAllEqual(15, product)
 
   def testMatMulGPU(self):
     if not context.context().num_gpus():
@@ -49,7 +49,7 @@ class OpsTest(test_util.TensorFlowTestCase):
     three = constant_op.constant([[3.]]).as_gpu_tensor()
     five = constant_op.constant([[5.]]).as_gpu_tensor()
     product = math_ops.matmul(three, five)
-    self.assertEqual([[15.0]], product.numpy())
+    self.assertEqual([[15.0]], product)
 
   def testExecuteStringAttr(self):
     three = constant_op.constant(3.0)
@@ -62,27 +62,27 @@ class OpsTest(test_util.TensorFlowTestCase):
     almost_three = constant_op.constant(2.8)
     almost_equal = math_ops.approximate_equal(
         three, almost_three, tolerance=0.3)
-    self.assertTrue(almost_equal.numpy())
+    self.assertTrue(almost_equal)
 
   def testExecuteIntAttr(self):
     three = constant_op.constant(3)
     four = constant_op.constant(4)
     total = math_ops.add_n([three, four])
-    self.assertEqual(7, total.numpy())
+    self.assertAllEqual(7, total)
 
   def testExecuteBoolAttr(self):
     three = constant_op.constant([[3]])
     five = constant_op.constant([[5]])
     product = math_ops.matmul(three, five, transpose_a=True)
-    self.assertEqual([[15]], product.numpy())
+    self.assertAllEqual([[15]], product)
 
   def testExecuteOneListOutput(self):
     split_dim = constant_op.constant(1)
     value = constant_op.constant([[0, 1, 2], [3, 4, 5]])
     x1, x2, x3 = array_ops.split(value, 3, axis=split_dim)
-    self.assertAllEqual([[0], [3]], x1.numpy())
-    self.assertAllEqual([[1], [4]], x2.numpy())
-    self.assertAllEqual([[2], [5]], x3.numpy())
+    self.assertAllEqual([[0], [3]], x1)
+    self.assertAllEqual([[1], [4]], x2)
+    self.assertAllEqual([[2], [5]], x3)
 
   def testGraphMode(self):
     graph = ops.Graph()
@@ -97,7 +97,7 @@ class OpsTest(test_util.TensorFlowTestCase):
       self.skipTest('No GPUs found')
     with context.device('/gpu:0'):
       r = constant_op.constant(1) + constant_op.constant(2)
-    self.assertEqual(r.numpy(), 3)
+    self.assertEqual(r, 3)
 
   def testExecuteListOutputLen1(self):
     split_dim = constant_op.constant(1)
@@ -105,7 +105,7 @@ class OpsTest(test_util.TensorFlowTestCase):
     result = array_ops.split(value, 1, axis=split_dim)
     self.assertTrue(isinstance(result, list))
     self.assertEqual(1, len(result))
-    self.assertAllEqual([[0, 1, 2], [3, 4, 5]], result[0].numpy())
+    self.assertAllEqual([[0, 1, 2], [3, 4, 5]], result[0])
 
   def testExecuteListOutputLen0(self):
     empty = constant_op.constant([], dtype=dtypes.int32)
@@ -120,8 +120,8 @@ class OpsTest(test_util.TensorFlowTestCase):
     out, idx = result
     self.assertTrue(out is result.out)
     self.assertTrue(idx is result.idx)
-    self.assertAllEqual([2, 4, 6], out.numpy())
-    self.assertAllEqual([1, 3, 5], idx.numpy())
+    self.assertAllEqual([2, 4, 6], out)
+    self.assertAllEqual([1, 3, 5], idx)
 
   def testExecuteMultipleListOutput(self):
     split_dim = constant_op.constant(1, dtype=dtypes.int64)
@@ -138,12 +138,12 @@ class OpsTest(test_util.TensorFlowTestCase):
     self.assertEqual(output_indices, result.output_indices)
     self.assertEqual(output_values, result.output_values)
     self.assertEqual(output_shape, result.output_shape)
-    self.assertAllEqual([[0, 2], [1, 0], [1, 1]], output_indices[0].numpy())
-    self.assertAllEqual([[0, 0], [0, 1]], output_indices[1].numpy())
-    self.assertAllEqual([2, 7, 11], output_values[0].numpy())
-    self.assertAllEqual([3, 5], output_values[1].numpy())
-    self.assertAllEqual([2, 4], output_shape[0].numpy())
-    self.assertAllEqual([2, 3], output_shape[1].numpy())
+    self.assertAllEqual([[0, 2], [1, 0], [1, 1]], output_indices[0])
+    self.assertAllEqual([[0, 0], [0, 1]], output_indices[1])
+    self.assertAllEqual([2, 7, 11], output_values[0])
+    self.assertAllEqual([3, 5], output_values[1])
+    self.assertAllEqual([2, 4], output_shape[0])
+    self.assertAllEqual([2, 3], output_shape[1])
 
   # TODO(josh11b): Test an op that has multiple outputs, some but not
   # all of which are lists. Examples: barrier_take_many (currently
@@ -154,84 +154,84 @@ class OpsTest(test_util.TensorFlowTestCase):
     x = constant_op.constant(1, dtype=dtypes.int32)
     three_x = x + x + x
     self.assertEquals(dtypes.int32, three_x.dtype)
-    self.assertEquals(3, three_x.numpy())
+    self.assertAllEqual(3, three_x)
 
   def testOperatorOverrides(self):
     # TODO(henrytan): test with negative number.
     a = constant_op.constant([1])
     b = constant_op.constant([2])
 
-    self.assertAllEqual((-a).numpy(), [-1])
-    self.assertAllEqual(abs(b).numpy(), [2])
+    self.assertAllEqual((-a), [-1])
+    self.assertAllEqual(abs(b), [2])
 
-    self.assertAllEqual((a + b).numpy(), [3])
-    self.assertAllEqual((a - b).numpy(), [-1])
-    self.assertAllEqual((a * b).numpy(), [2])
-    self.assertAllEqual((a * a).numpy(), [1])
+    self.assertAllEqual((a + b), [3])
+    self.assertAllEqual((a - b), [-1])
+    self.assertAllEqual((a * b), [2])
+    self.assertAllEqual((a * a), [1])
 
-    self.assertAllEqual((a**b).numpy(), [1])
-    self.assertAllEqual((a / b).numpy(), [1 / 2])
-    self.assertAllEqual((a / a).numpy(), [1])
-    self.assertAllEqual((a % b).numpy(), [1])
+    self.assertAllEqual((a**b), [1])
+    self.assertAllEqual((a / b), [1 / 2])
+    self.assertAllEqual((a / a), [1])
+    self.assertAllEqual((a % b), [1])
 
-    self.assertAllEqual((a < b).numpy(), [True])
-    self.assertAllEqual((a <= b).numpy(), [True])
-    self.assertAllEqual((a > b).numpy(), [False])
-    self.assertAllEqual((a >= b).numpy(), [False])
+    self.assertAllEqual((a < b), [True])
+    self.assertAllEqual((a <= b), [True])
+    self.assertAllEqual((a > b), [False])
+    self.assertAllEqual((a >= b), [False])
     self.assertAllEqual((a == b), False)
     self.assertAllEqual((a != b), True)
 
-    self.assertEqual(1, a[constant_op.constant(0)].numpy())
+    self.assertAllEqual(1, a[constant_op.constant(0)])
 
   def test_basic_slice(self):
     npt = np.arange(1, 19, dtype=np.float32).reshape(3, 2, 3)
     t = constant_op.constant(npt)
 
-    self.assertAllEqual(npt[:, :, :], t[:, :, :].numpy())
-    self.assertAllEqual(npt[::, ::, ::], t[::, ::, ::].numpy())
-    self.assertAllEqual(npt[::1, ::1, ::1], t[::1, ::1, ::1].numpy())
-    self.assertAllEqual(npt[::1, ::5, ::2], t[::1, ::5, ::2].numpy())
-    self.assertAllEqual(npt[::-1, :, :], t[::-1, :, :].numpy())
-    self.assertAllEqual(npt[:, ::-1, :], t[:, ::-1, :].numpy())
-    self.assertAllEqual(npt[:, :, ::-1], t[:, :, ::-1].numpy())
-    self.assertAllEqual(npt[-2::-1, :, ::1], t[-2::-1, :, ::1].numpy())
-    self.assertAllEqual(npt[-2::-1, :, ::2], t[-2::-1, :, ::2].numpy())
+    self.assertAllEqual(npt[:, :, :], t[:, :, :])
+    self.assertAllEqual(npt[::, ::, ::], t[::, ::, ::])
+    self.assertAllEqual(npt[::1, ::1, ::1], t[::1, ::1, ::1])
+    self.assertAllEqual(npt[::1, ::5, ::2], t[::1, ::5, ::2])
+    self.assertAllEqual(npt[::-1, :, :], t[::-1, :, :])
+    self.assertAllEqual(npt[:, ::-1, :], t[:, ::-1, :])
+    self.assertAllEqual(npt[:, :, ::-1], t[:, :, ::-1])
+    self.assertAllEqual(npt[-2::-1, :, ::1], t[-2::-1, :, ::1])
+    self.assertAllEqual(npt[-2::-1, :, ::2], t[-2::-1, :, ::2])
 
   def testDegenerateSlices(self):
     npt = np.arange(1, 19, dtype=np.float32).reshape(3, 2, 3)
     t = constant_op.constant(npt)
     # degenerate by offering a forward interval with a negative stride
-    self.assertAllEqual(npt[0:-1:-1, :, :], t[0:-1:-1, :, :].numpy())
+    self.assertAllEqual(npt[0:-1:-1, :, :], t[0:-1:-1, :, :])
     # degenerate with a reverse interval with a positive stride
-    self.assertAllEqual(npt[-1:0, :, :], t[-1:0, :, :].numpy())
+    self.assertAllEqual(npt[-1:0, :, :], t[-1:0, :, :])
     # empty interval in every dimension
-    self.assertAllEqual(npt[-1:0, 2:2, 2:3:-1], t[-1:0, 2:2, 2:3:-1].numpy())
+    self.assertAllEqual(npt[-1:0, 2:2, 2:3:-1], t[-1:0, 2:2, 2:3:-1])
 
   def testEllipsis(self):
     npt = np.array(
         [[[[[1, 2], [3, 4], [5, 6]]], [[[7, 8], [9, 10], [11, 12]]]]])
     t = constant_op.constant(npt)
 
-    self.assertAllEqual(npt[0:], t[0:].numpy())
+    self.assertAllEqual(npt[0:], t[0:])
     # implicit ellipsis
-    self.assertAllEqual(npt[0:, ...], t[0:, ...].numpy())
+    self.assertAllEqual(npt[0:, ...], t[0:, ...])
     # ellipsis alone
-    self.assertAllEqual(npt[...], t[...].numpy())
+    self.assertAllEqual(npt[...], t[...])
     # ellipsis at end
-    self.assertAllEqual(npt[0:1, ...], t[0:1, ...].numpy())
+    self.assertAllEqual(npt[0:1, ...], t[0:1, ...])
     # ellipsis at begin
-    self.assertAllEqual(npt[..., 0:1], t[..., 0:1].numpy())
+    self.assertAllEqual(npt[..., 0:1], t[..., 0:1])
     # ellipsis at middle
-    self.assertAllEqual(npt[0:1, ..., 0:1], t[0:1, ..., 0:1].numpy())
+    self.assertAllEqual(npt[0:1, ..., 0:1], t[0:1, ..., 0:1])
 
   def testShrink(self):
     npt = np.array([[[[[1, 2, 4, 5], [5, 6, 7, 8], [9, 10, 11, 12]]],
                      [[[13, 14, 15, 16], [17, 18, 19, 20], [21, 22, 23, 24]]]]])
     t = constant_op.constant(npt)
-    self.assertAllEqual(npt[:, :, :, :, 3], t[:, :, :, :, 3].numpy())
-    self.assertAllEqual(npt[..., 3], t[..., 3].numpy())
-    self.assertAllEqual(npt[:, 0], t[:, 0].numpy())
-    self.assertAllEqual(npt[:, :, 0], t[:, :, 0].numpy())
+    self.assertAllEqual(npt[:, :, :, :, 3], t[:, :, :, :, 3])
+    self.assertAllEqual(npt[..., 3], t[..., 3])
+    self.assertAllEqual(npt[:, 0], t[:, 0])
+    self.assertAllEqual(npt[:, :, 0], t[:, :, 0])
 
   def testOpWithInputsOnDifferentDevices(self):
     if not context.context().num_gpus():
@@ -242,7 +242,7 @@ class OpsTest(test_util.TensorFlowTestCase):
     value = constant_op.constant([1., 2.]).as_gpu_tensor()
     shape = constant_op.constant([2, 1])
     reshaped = array_ops.reshape(value, shape)
-    self.assertAllEqual([[1], [2]], reshaped.as_cpu_tensor().numpy())
+    self.assertAllEqual([[1], [2]], reshaped.as_cpu_tensor())
 
     # And if the shape is in device memory, it should complain
     # TODO(ashankar): Revisit this - perhaps instead of complaining,
@@ -264,7 +264,7 @@ class OpsTest(test_util.TensorFlowTestCase):
     # The Shape op kernel on GPU places the output in host memory.
     value = constant_op.constant([1.]).as_gpu_tensor()
     shape = array_ops.shape(value)
-    self.assertEquals([1], shape.numpy())
+    self.assertEquals([1], shape)
 
   def testRandomUniform(self):
     scalar_shape = constant_op.constant([], dtype=dtypes.int32)
@@ -276,8 +276,8 @@ class OpsTest(test_util.TensorFlowTestCase):
     x = random_ops.random_uniform(
         scalar_shape, minval=constant_op.constant(5.),
         maxval=constant_op.constant(6.))
-    self.assertLess(x.numpy(), 6)
-    self.assertGreaterEqual(x.numpy(), 5)
+    self.assertLess(x, 6)
+    self.assertGreaterEqual(x, 5)
 
   def testArgsToMatchingEagerDefault(self):
     # Uses default
@@ -298,10 +298,10 @@ class OpsTest(test_util.TensorFlowTestCase):
     flatten_layer = core.Flatten()
     x = constant_op.constant([[[-10, -20], [-30, -40]], [[10, 20], [30, 40]]])
     y = flatten_layer(x)
-    self.assertAllEqual([[-10, -20, -30, -40], [10, 20, 30, 40]], y.numpy())
+    self.assertAllEqual([[-10, -20, -30, -40], [10, 20, 30, 40]], y)
 
   def testIdentity(self):
-    self.assertEqual(2, array_ops.identity(2).numpy())
+    self.assertAllEqual(2, array_ops.identity(2))
 
   def testIncompatibleSetShape(self):
     x = constant_op.constant(1)
diff --git a/tensorflow/python/eager/tape_test.py b/tensorflow/python/eager/tape_test.py
index c34f5cffe3..c97cb62125 100644
--- a/tensorflow/python/eager/tape_test.py
+++ b/tensorflow/python/eager/tape_test.py
@@ -81,8 +81,8 @@ class TapeTest(test.TestCase):
       tf_e = tf_d + tf_f
       tf_da, tf_db = gradients_impl.gradients(tf_e, [tf_a, tf_b])
 
-      self.assertAllEqual(da.numpy(), tf_da.eval())
-      self.assertAllEqual(db.numpy(), tf_db.eval())
+      self.assertAllEqual(da, tf_da.eval())
+      self.assertAllEqual(db, tf_db.eval())
 
   def testBasicFunctional(self):
 
@@ -93,7 +93,7 @@ class TapeTest(test.TestCase):
     aa = constant_op.constant([[1., 0.], [0., 1.]])
     bb = constant_op.constant([[1., 2.], [3., 4.]])
     da, = backprop.gradients_function(forward, ['a'])(aa, bb)
-    self.assertAllEqual(da.numpy(),
+    self.assertAllEqual(da,
                         math_ops.matmul(
                             array_ops.ones_like(aa),
                             array_ops.transpose(bb)).numpy())
@@ -107,7 +107,7 @@ class TapeTest(test.TestCase):
     aa = constant_op.constant([[1., 0.], [0., 1.]])
     bb = constant_op.constant([[1., 2.], [3., 4.]])
     da, = backprop.gradients_function(forward, [0])(aa, bb)
-    self.assertAllEqual(da.numpy(),
+    self.assertAllEqual(da,
                         math_ops.matmul(
                             array_ops.ones_like(aa),
                             array_ops.transpose(bb)).numpy())
@@ -121,11 +121,11 @@ class TapeTest(test.TestCase):
     aa = constant_op.constant([[1., 0.], [0., 1.]])
     bb = constant_op.constant([[1., 2.], [3., 4.]])
     val, (da,) = backprop.val_and_grad_function(forward, ['a'])(aa, bb)
-    self.assertAllEqual(da.numpy(),
+    self.assertAllEqual(da,
                         math_ops.matmul(
                             array_ops.ones_like(aa),
-                            array_ops.transpose(bb)).numpy())
-    self.assertAllEqual(val.numpy(), forward(aa, bb).numpy())
+                            array_ops.transpose(bb)))
+    self.assertAllEqual(val, forward(aa, bb))
 
   def testTwoOutputs(self):
 
@@ -143,8 +143,8 @@ class TapeTest(test.TestCase):
       tf_rr = 2 * math_ops.reduce_sum(tf_mm)
       tf_da, tf_db = gradients_impl.gradients(tf_rr, [tf_a, tf_b])
 
-      self.assertAllEqual(da.numpy(), tf_da.eval())
-      self.assertAllEqual(db.numpy(), tf_db.eval())
+      self.assertAllEqual(da, tf_da.eval())
+      self.assertAllEqual(db, tf_db.eval())
 
   def testGcTwoOutputs(self):
 
@@ -155,7 +155,7 @@ class TapeTest(test.TestCase):
     labels = constant_op.constant([0])
     logits = constant_op.constant([[0.0]])
     grad, = backprop.gradients_function(fn, [0])(logits, labels)
-    self.assertAllEqual(grad.numpy(), [[0.0]])
+    self.assertAllEqual(grad, [[0.0]])
 
   def testTfTensor(self):
 
@@ -164,7 +164,7 @@ class TapeTest(test.TestCase):
 
     t = constant_op.constant(1.0)
     g, = backprop.gradients_function(fn, [0])(t)
-    self.assertEqual(g.numpy(), 1.0)
+    self.assertAllEqual(g, 1.0)
 
   def testTapeGC(self):
     # TODO(apassos) figure out how to test this without using tape internal
diff --git a/tensorflow/python/eager/tensor_test.py b/tensorflow/python/eager/tensor_test.py
index 953807fc2a..e31c03c08d 100644
--- a/tensorflow/python/eager/tensor_test.py
+++ b/tensorflow/python/eager/tensor_test.py
@@ -45,7 +45,7 @@ class TFETensorTest(test_util.TensorFlowTestCase):
 
   def testScalarTensor(self):
     t = _create_tensor(3, dtype=dtypes.int32)
-    self.assertEqual(t.numpy(), _create_tensor(np.array(3)).numpy())
+    self.assertAllEqual(t, _create_tensor(np.array(3)))
     self.assertEqual(dtypes.int32, t.dtype)
     self.assertEqual(0, t.shape.ndims)
     self.assertAllEqual([], t.shape.as_list())
@@ -85,12 +85,12 @@ class TFETensorTest(test_util.TensorFlowTestCase):
   def testNumpyValue(self):
     values = np.array([3.0])
     t = _create_tensor(values)
-    self.assertAllEqual(values, t.numpy())
+    self.assertAllEqual(values, t)
 
   def testNumpyValueWithCast(self):
     values = np.array([3.0], dtype=np.float32)
     t = _create_tensor(values, dtype=dtypes.float64)
-    self.assertAllEqual(values, t.numpy())
+    self.assertAllEqual(values, t)
     ctx = context.context()
     # Bad dtype value.
     with self.assertRaisesRegexp(TypeError, "Invalid dtype argument value"):
@@ -100,13 +100,13 @@ class TFETensorTest(test_util.TensorFlowTestCase):
   def testNumpyOrderHandling(self):
     n = np.array([[1, 2], [3, 4]], order="F")
     t = _create_tensor(n)
-    self.assertAllEqual([[1, 2], [3, 4]], t.numpy())
+    self.assertAllEqual([[1, 2], [3, 4]], t)
 
   def testTensorAndNumpyMatrix(self):
     expected = np.array([[1.0, 2.0], [3.0, 4.0]], np.float32)
     actual = _create_tensor([[1.0, 2.0], [3.0, 4.0]])
-    self.assertAllEqual(expected, actual.numpy())
-    self.assertEqual(np.float32, actual.numpy().dtype)
+    self.assertAllEqual(expected, actual)
+    self.assertEqual(np.float32, actual.dtype)
     self.assertEqual(dtypes.float32, actual.dtype)
     self.assertAllEqual([2, 2], actual.shape.as_list())
 
@@ -140,7 +140,7 @@ class TFETensorTest(test_util.TensorFlowTestCase):
     t = _create_tensor(np.eye(3))
     tensor_str = str(t)
     self.assertIn("shape=%s, dtype=%s" % (t.shape, t.dtype.name), tensor_str)
-    self.assertIn(str(t.numpy()), tensor_str)
+    self.assertIn(str(t), tensor_str)
 
   def testMultiLineTensorRepr(self):
     t = _create_tensor(np.eye(3))
diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index a52a0cfc2d..3ac8a0cb6a 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -25,6 +25,7 @@ import re
 import sys
 import threading
 
+import numpy as np
 import six
 from six.moves import xrange  # pylint: disable=redefined-builtin
 
@@ -618,6 +619,9 @@ class _EagerTensorBase(Tensor):
     """
     return self.as_cpu_tensor()._numpy()  # pylint: disable=protected-access
 
+  def __array__(self):
+    return np.array(self.numpy())
+
   def _numpy(self):
     raise NotImplementedError()
 
-- 
GitLab


From dc27aecb5b3a259bd35d928c643ee1a548279c3a Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Tue, 17 Oct 2017 19:06:38 -0700
Subject: [PATCH 499/909] Remove thread ordering with sleep statements from
 coordinator_test.

PiperOrigin-RevId: 172550053
---
 .../python/training/coordinator_test.py       | 90 +++++++++++++------
 1 file changed, 64 insertions(+), 26 deletions(-)

diff --git a/tensorflow/python/training/coordinator_test.py b/tensorflow/python/training/coordinator_test.py
index 8f4cae6f06..149d3eed41 100644
--- a/tensorflow/python/training/coordinator_test.py
+++ b/tensorflow/python/training/coordinator_test.py
@@ -33,21 +33,26 @@ def StopOnEvent(coord, wait_for_stop, set_when_stopped):
   set_when_stopped.set()
 
 
-def RaiseInN(coord, n_secs, ex, report_exception):
+def RaiseOnEvent(coord, wait_for_stop, set_when_stopped, ex, report_exception):
   try:
-    time.sleep(n_secs)
+    wait_for_stop.wait()
     raise ex
   except RuntimeError as e:
     if report_exception:
       coord.request_stop(e)
     else:
       coord.request_stop(sys.exc_info())
+  finally:
+    if set_when_stopped:
+      set_when_stopped.set()
 
 
-def RaiseInNUsingContextHandler(coord, n_secs, ex):
+def RaiseOnEventUsingContextHandler(coord, wait_for_stop, set_when_stopped, ex):
   with coord.stop_on_exception():
-    time.sleep(n_secs)
+    wait_for_stop.wait()
     raise ex
+  if set_when_stopped:
+    set_when_stopped.set()
 
 
 def SleepABit(n_secs, coord=None):
@@ -167,80 +172,113 @@ class CoordinatorTest(test.TestCase):
 
   def testJoinRaiseReportExcInfo(self):
     coord = coordinator.Coordinator()
+    ev_1 = threading.Event()
+    ev_2 = threading.Event()
     threads = [
-        threading.Thread(target=RaiseInN,
-                         args=(coord, 0.01, RuntimeError("First"), False)),
-        threading.Thread(target=RaiseInN,
-                         args=(coord, 0.05, RuntimeError("Too late"), False))]
+        threading.Thread(
+            target=RaiseOnEvent,
+            args=(coord, ev_1, ev_2, RuntimeError("First"), False)),
+        threading.Thread(
+            target=RaiseOnEvent,
+            args=(coord, ev_2, None, RuntimeError("Too late"), False))]
     for t in threads:
       t.start()
+
+    ev_1.set()
+
     with self.assertRaisesRegexp(RuntimeError, "First"):
       coord.join(threads)
 
   def testJoinRaiseReportException(self):
     coord = coordinator.Coordinator()
+    ev_1 = threading.Event()
+    ev_2 = threading.Event()
     threads = [
-        threading.Thread(target=RaiseInN,
-                         args=(coord, 0.01, RuntimeError("First"), True)),
-        threading.Thread(target=RaiseInN,
-                         args=(coord, 0.05, RuntimeError("Too late"), True))]
+        threading.Thread(
+            target=RaiseOnEvent,
+            args=(coord, ev_1, ev_2, RuntimeError("First"), True)),
+        threading.Thread(
+            target=RaiseOnEvent,
+            args=(coord, ev_2, None, RuntimeError("Too late"), True))]
     for t in threads:
       t.start()
+
+    ev_1.set()
     with self.assertRaisesRegexp(RuntimeError, "First"):
       coord.join(threads)
 
   def testJoinIgnoresOutOfRange(self):
     coord = coordinator.Coordinator()
+    ev_1 = threading.Event()
     threads = [
-        threading.Thread(target=RaiseInN,
-                         args=(coord, 0.01,
-                               errors_impl.OutOfRangeError(None, None, "First"),
-                               True))
+        threading.Thread(
+            target=RaiseOnEvent,
+            args=(coord, ev_1, None,
+                  errors_impl.OutOfRangeError(None, None, "First"),
+                  True))
         ]
     for t in threads:
       t.start()
+
+    ev_1.set()
     coord.join(threads)
 
   def testJoinIgnoresMyExceptionType(self):
     coord = coordinator.Coordinator(clean_stop_exception_types=(ValueError,))
+    ev_1 = threading.Event()
     threads = [
-        threading.Thread(target=RaiseInN,
-                         args=(coord, 0.01, ValueError("Clean stop"), True))
+        threading.Thread(
+            target=RaiseOnEvent,
+            args=(coord, ev_1, None, ValueError("Clean stop"), True))
         ]
     for t in threads:
       t.start()
+
+    ev_1.set()
     coord.join(threads)
 
   def testJoinRaiseReportExceptionUsingHandler(self):
     coord = coordinator.Coordinator()
+    ev_1 = threading.Event()
+    ev_2 = threading.Event()
     threads = [
-        threading.Thread(target=RaiseInNUsingContextHandler,
-                         args=(coord, 0.01, RuntimeError("First"))),
-        threading.Thread(target=RaiseInNUsingContextHandler,
-                         args=(coord, 0.05, RuntimeError("Too late")))]
+        threading.Thread(
+            target=RaiseOnEventUsingContextHandler,
+            args=(coord, ev_1, ev_2, RuntimeError("First"))),
+        threading.Thread(
+            target=RaiseOnEventUsingContextHandler,
+            args=(coord, ev_2, None, RuntimeError("Too late")))]
     for t in threads:
       t.start()
+
+    ev_1.set()
     with self.assertRaisesRegexp(RuntimeError, "First"):
       coord.join(threads)
 
   def testClearStopClearsExceptionToo(self):
     coord = coordinator.Coordinator()
+    ev_1 = threading.Event()
     threads = [
-        threading.Thread(target=RaiseInN,
-                         args=(coord, 0.01, RuntimeError("First"), True)),
+        threading.Thread(
+            target=RaiseOnEvent,
+            args=(coord, ev_1, None, RuntimeError("First"), True)),
         ]
     for t in threads:
       t.start()
+
     with self.assertRaisesRegexp(RuntimeError, "First"):
+      ev_1.set()
       coord.join(threads)
     coord.clear_stop()
     threads = [
-        threading.Thread(target=RaiseInN,
-                         args=(coord, 0.01, RuntimeError("Second"), True)),
+        threading.Thread(
+            target=RaiseOnEvent,
+            args=(coord, ev_1, None, RuntimeError("Second"), True)),
         ]
     for t in threads:
       t.start()
     with self.assertRaisesRegexp(RuntimeError, "Second"):
+      ev_1.set()
       coord.join(threads)
 
   def testRequestStopRaisesIfJoined(self):
-- 
GitLab


From c696dcf24438fdb29394e776f1c865e0167cd368 Mon Sep 17 00:00:00 2001
From: Martin Wicke <wicke@google.com>
Date: Tue, 17 Oct 2017 19:30:00 -0700
Subject: [PATCH 500/909] Fix link in audio tutorial (no https for direct
 download.tensorflow.org links).

PiperOrigin-RevId: 172551595
---
 tensorflow/docs_src/tutorials/audio_recognition.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/docs_src/tutorials/audio_recognition.md b/tensorflow/docs_src/tutorials/audio_recognition.md
index 670e480b12..336f4d9c18 100644
--- a/tensorflow/docs_src/tutorials/audio_recognition.md
+++ b/tensorflow/docs_src/tutorials/audio_recognition.md
@@ -25,7 +25,7 @@ python tensorflow/examples/speech_commands/train.py
 ```
 
 The script will start off by downloading the [Speech Commands
-dataset](https://download.tensorflow.org/data/speech_commands_v0.01.tar.gz),
+dataset](https://storage.cloud.google.com/download.tensorflow.org/data/speech_commands_v0.01.tar.gz),
 which consists of 65,000 WAVE audio files of people saying thirty different
 words. This data was collected by Google and released under a CC BY license, and
 you can help improve it by [contributing five minutes of your own
-- 
GitLab


From a8076b9450ca7873592c115841bdebf5f3febf52 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 17 Oct 2017 20:37:27 -0700
Subject: [PATCH 501/909] [XLA] Try to pass layouts through reshapes.

For reshapes where the operand and the output have the same rank, try to pass the layout through the reshape. The layout that's already present was presumably assigned for some reason, so it has a good chance of being good.

PiperOrigin-RevId: 172555906
---
 .../compiler/xla/service/layout_assignment.cc | 20 +++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

diff --git a/tensorflow/compiler/xla/service/layout_assignment.cc b/tensorflow/compiler/xla/service/layout_assignment.cc
index 2058706f11..7eda7c2284 100644
--- a/tensorflow/compiler/xla/service/layout_assignment.cc
+++ b/tensorflow/compiler/xla/service/layout_assignment.cc
@@ -732,7 +732,8 @@ std::unique_ptr<Layout> LayoutAssignment::ChooseOperandLayoutFromOutputLayout(
     // dimension bound is 1 in the operand shape, there may be several such
     // layouts. So if 'output_layout' is the default layout, try if the
     // reshape is a bitcast when using the same layout. This may avoid copy
-    // operations.
+    // operations. For similar reasons, if the operand and output have the same
+    // rank, try to match the operand's layout to the output.
     if (ShapeUtil::TrueRank(operand->shape()) == 1 &&
         ShapeUtil::Rank(instruction->shape()) == 1) {
       // Don't assign a layout in case of R1 -> effective R1 reshape.
@@ -748,6 +749,13 @@ std::unique_ptr<Layout> LayoutAssignment::ChooseOperandLayoutFromOutputLayout(
     if (ShapeUtil::ReshapeIsBitcast(operand_shape, output_shape_with_layout)) {
       return MakeUnique<Layout>(operand_shape.layout());
     }
+    if (ShapeUtil::Rank(operand_shape) == ShapeUtil::Rank(output_shape)) {
+      *operand_shape.mutable_layout() = output_layout;
+      if (ShapeUtil::ReshapeIsBitcast(operand_shape,
+                                      output_shape_with_layout)) {
+        return MakeUnique<Layout>(output_layout);
+      }
+    }
     auto aligned_operand_shape =
         ShapeUtil::AlignLayouts(output_shape_with_layout, operand_shape);
     if (aligned_operand_shape) {
@@ -796,7 +804,8 @@ std::unique_ptr<Layout> LayoutAssignment::ChooseOutputLayoutFromOperandLayout(
     // dimension bound is 1 in the user shape, there may be several such
     // layouts. So if 'operand_layout' is the default layout, try if the
     // reshape is a bitcast when using the same layout. This may avoid copy
-    // operations.
+    // operations. For similar reasons, if the operand and output have the same
+    // rank, try to match the outputs's layout to the operand.
     if (ShapeUtil::Rank(operand->shape()) == 1 &&
         ShapeUtil::TrueRank(user->shape()) == 1) {
       // Don't assign a layout in case of R1 -> effective R1 reshape.
@@ -812,6 +821,13 @@ std::unique_ptr<Layout> LayoutAssignment::ChooseOutputLayoutFromOperandLayout(
     if (ShapeUtil::ReshapeIsBitcast(output_shape, operand_shape_with_layout)) {
       return MakeUnique<Layout>(output_shape.layout());
     }
+    if (ShapeUtil::Rank(operand->shape()) == ShapeUtil::Rank(output_shape)) {
+      *output_shape.mutable_layout() = operand_layout;
+      if (ShapeUtil::ReshapeIsBitcast(output_shape,
+                                      operand_shape_with_layout)) {
+        return MakeUnique<Layout>(operand_layout);
+      }
+    }
     auto aligned_user_shape =
         ShapeUtil::AlignLayouts(operand_shape_with_layout, output_shape);
     if (aligned_user_shape) {
-- 
GitLab


From 6cf5b86c552093114344bde67ca737fd84b8f57e Mon Sep 17 00:00:00 2001
From: Derek Murray <mrry@google.com>
Date: Tue, 17 Oct 2017 20:39:11 -0700
Subject: [PATCH 502/909] Include <numeric> in transpose_functor.h to fix
 Windows build.

PiperOrigin-RevId: 172556044
---
 tensorflow/core/kernels/transpose_functor.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/core/kernels/transpose_functor.h b/tensorflow/core/kernels/transpose_functor.h
index a2eb0263e8..9781fe3b61 100644
--- a/tensorflow/core/kernels/transpose_functor.h
+++ b/tensorflow/core/kernels/transpose_functor.h
@@ -16,6 +16,7 @@ limitations under the License.
 #ifndef TENSORFLOW_CORE_KERNELS_TRANSPOSE_FUNCTOR_H_
 #define TENSORFLOW_CORE_KERNELS_TRANSPOSE_FUNCTOR_H_
 
+#include <numeric>
 #include <string>
 #include <vector>
 #include "tensorflow/core/framework/tensor.h"
-- 
GitLab


From 35debbdff9e61edc7266bd0ea0636dd5c328114a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 17 Oct 2017 22:40:07 -0700
Subject: [PATCH 503/909] Update inception score to match the openAI version
 from https://github.com/openai/improved-gan/tree/master/inception_score.

PiperOrigin-RevId: 172562573
---
 .../eval/python/classifier_metrics_impl.py    | 81 +++++++++----------
 .../eval/python/classifier_metrics_test.py    | 10 +--
 2 files changed, 43 insertions(+), 48 deletions(-)

diff --git a/tensorflow/contrib/gan/python/eval/python/classifier_metrics_impl.py b/tensorflow/contrib/gan/python/eval/python/classifier_metrics_impl.py
index 6074694f8b..4af87b8b47 100644
--- a/tensorflow/contrib/gan/python/eval/python/classifier_metrics_impl.py
+++ b/tensorflow/contrib/gan/python/eval/python/classifier_metrics_impl.py
@@ -16,6 +16,11 @@
 
 These methods come from https://arxiv.org/abs/1606.03498 and
 https://arxiv.org/abs/1706.08500.
+
+NOTE: This implementation uses the same weights as in
+https://github.com/openai/improved-gan/blob/master/inception_score/model.py,
+but is more numerically stable and is an unbiased estimator of the true
+Inception score even when splitting the inputs into batches.
 """
 
 from __future__ import absolute_import
@@ -54,17 +59,16 @@ __all__ = [
     'classifier_score',
     'frechet_inception_distance',
     'frechet_classifier_distance',
+    'INCEPTION_DEFAULT_IMAGE_SIZE',
 ]
 
 
-INCEPTION_URL = 'http://download.tensorflow.org/models/frozen_inception_v3_2017_09_13.tar.gz'
-INCEPTION_FROZEN_GRAPH = 'frozen_inception_v3.pb'
-INCEPTION_V3_INPUT = 'input'
-INCEPTION_V3_OUTPUT = 'InceptionV3/Logits/SpatialSqueeze:0'
-INCEPTION_V3_FINAL_POOL = 'InceptionV3/Logits/AvgPool_1a_8x8/AvgPool:0'
-_INCEPTION_V3_NUM_CLASSES = 1001
-_INCEPTION_V3_FINAL_POOL_SIZE = 2048
-INCEPTION_V3_DEFAULT_IMG_SIZE = 299
+INCEPTION_URL = 'http://download.tensorflow.org/models/frozen_inception_v1_2015_12_05.tar.gz'
+INCEPTION_FROZEN_GRAPH = 'inceptionv1_for_inception_score.pb'
+INCEPTION_INPUT = 'Mul:0'
+INCEPTION_OUTPUT = 'logits:0'
+INCEPTION_FINAL_POOL = 'pool_3:0'
+INCEPTION_DEFAULT_IMAGE_SIZE = 299
 
 
 def _validate_images(images, image_size):
@@ -106,42 +110,33 @@ def _symmetric_matrix_square_root(mat, eps=1e-10):
 # NOTE: Floating-point inputs are expected to be in [0, 1].
 # Copied from /tensorflow_models/slim/preprocessing/inception_preprocessing.py.
 def preprocess_image(
-    image, height=INCEPTION_V3_DEFAULT_IMG_SIZE,
-    width=INCEPTION_V3_DEFAULT_IMG_SIZE, central_fraction=0.875, scope=None):
-  """Prepare one image for evaluation.
-
-  If height and width are specified it would output an image with that size by
-  applying resize_bilinear.
+    images, height=INCEPTION_DEFAULT_IMAGE_SIZE,
+    width=INCEPTION_DEFAULT_IMAGE_SIZE, scope=None):
+  """Prepare a batch of images for evaluation.
 
-  If central_fraction is specified it would crop the central fraction of the
-  input image.
+  This is the preprocessing portion of the graph from
+  http://download.tensorflow.org/models/image/imagenet/inception-2015-12-05.tgz.
 
   Args:
-    image: 3-D Tensor of image. If dtype is tf.float32 then the range should be
-      [0, 1], otherwise it would converted to tf.float32 assuming that the range
-      is [0, MAX], where MAX is largest positive representable number for
-      int(8/16/32) data type (see `tf.image.convert_image_dtype` for details).
-    height: integer
-    width: integer
-    central_fraction: Optional Float, fraction of the image to crop.
+    images: 3-D or 4-D Tensor of images. Values are in [0, 255].
+    height: Integer. Height of resized output image.
+    width: Integer. Width of resized output image.
     scope: Optional scope for name_scope.
+
   Returns:
-    3-D float Tensor of prepared image.
+    3-D or 4-D float Tensor of prepared image(s). Values are in [-1, 1].
   """
-  with ops.name_scope(scope, 'eval_image', [image, height, width]):
-    if image.dtype != dtypes.float32:
-      image = image_ops.convert_image_dtype(image, dtype=dtypes.float32)
-    # Crop the central region of the image with an area containing 87.5% of
-    # the original image.
-    image = image_ops.central_crop(image, central_fraction=central_fraction)
-
-    # Resize the image to the specified height and width.
-    image = array_ops.expand_dims(image, 0)
-    image = image_ops.resize_bilinear(image, [height, width],
-                                      align_corners=False)
-    image = array_ops.squeeze(image, [0])
-    image = (image - 0.5) * 2.0
-    return image
+  is_single = images.shape.ndims == 3
+  with ops.name_scope(scope, 'preprocess', [images, height, width]):
+    if not images.dtype.is_floating:
+      images = math_ops.to_float(images)
+    images = (images - 128.0) / 128.0
+    if is_single:
+      images = array_ops.expand_dims(images, axis=0)
+    resized = image_ops.resize_bilinear(images, [height, width])
+    if is_single:
+      resized = array_ops.squeeze(resized, axis=0)
+    return resized
 
 
 def _kl_divergence(p, p_logits, q):
@@ -211,9 +206,9 @@ def _default_graph_def_fn():
 def run_inception(images,
                   graph_def=None,
                   default_graph_def_fn=_default_graph_def_fn,
-                  image_size=INCEPTION_V3_DEFAULT_IMG_SIZE,
-                  input_tensor=INCEPTION_V3_INPUT,
-                  output_tensor=INCEPTION_V3_OUTPUT):
+                  image_size=INCEPTION_DEFAULT_IMAGE_SIZE,
+                  input_tensor=INCEPTION_INPUT,
+                  output_tensor=INCEPTION_OUTPUT):
   """Run images through a pretrained Inception classifier.
 
   Args:
@@ -338,7 +333,7 @@ def classifier_score(images, classifier_fn, num_batches=1):
 inception_score = functools.partial(
     classifier_score,
     classifier_fn=functools.partial(
-        run_inception, output_tensor=INCEPTION_V3_OUTPUT))
+        run_inception, output_tensor=INCEPTION_OUTPUT))
 
 
 def trace_sqrt_product(sigma, sigma_v):
@@ -479,4 +474,4 @@ def frechet_classifier_distance(real_images,
 frechet_inception_distance = functools.partial(
     frechet_classifier_distance,
     classifier_fn=functools.partial(
-        run_inception, output_tensor=INCEPTION_V3_FINAL_POOL))
+        run_inception, output_tensor=INCEPTION_FINAL_POOL))
diff --git a/tensorflow/contrib/gan/python/eval/python/classifier_metrics_test.py b/tensorflow/contrib/gan/python/eval/python/classifier_metrics_test.py
index 30285964a5..81fa2fc0f1 100644
--- a/tensorflow/contrib/gan/python/eval/python/classifier_metrics_test.py
+++ b/tensorflow/contrib/gan/python/eval/python/classifier_metrics_test.py
@@ -68,7 +68,7 @@ def _expected_trace_sqrt_product(sigma, sigma_v):
 # A dummy GraphDef string with the minimum number of Ops.
 graphdef_string = """
 node {
-  name: "input"
+  name: "Mul"
   op: "Placeholder"
   attr {
     key: "dtype"
@@ -97,7 +97,7 @@ node {
   }
 }
 node {
-  name: "InceptionV3/Logits/SpatialSqueeze"
+  name: "logits"
   op: "Placeholder"
   attr {
     key: "dtype"
@@ -120,7 +120,7 @@ node {
   }
 }
 node {
-  name: "InceptionV3/Logits/AvgPool_1a_8x8/AvgPool"
+  name: "pool_3"
   op: "Placeholder"
   attr {
     key: "dtype"
@@ -182,7 +182,7 @@ class ClassifierMetricsTest(test.TestCase):
     img = array_ops.ones([batch_size, 299, 299, 3])
     pool = _run_with_mock(
         classifier_metrics.run_inception, img,
-        output_tensor=classifier_metrics.INCEPTION_V3_FINAL_POOL)
+        output_tensor=classifier_metrics.INCEPTION_FINAL_POOL)
 
     self.assertTrue(isinstance(pool, ops.Tensor))
     pool.shape.assert_is_compatible_with([batch_size, 2048])
@@ -306,7 +306,7 @@ class ClassifierMetricsTest(test.TestCase):
     """Test `preprocess_image` graph construction."""
     incorrectly_sized_image = array_ops.zeros([520, 240, 3])
     correct_image = classifier_metrics.preprocess_image(
-        image=incorrectly_sized_image)
+        images=incorrectly_sized_image)
     _run_with_mock(classifier_metrics.run_inception,
                    array_ops.expand_dims(correct_image, 0))
 
-- 
GitLab


From bedfe8ac14bddbf21c5acf80d55abff9df4a7967 Mon Sep 17 00:00:00 2001
From: Sergii Khomenko <x-sam@brainscode.com>
Date: Wed, 18 Oct 2017 08:46:24 +0300
Subject: [PATCH 504/909] Fix minor typos in TF Boosted Trees (#13792)

* Fix a typo in readme of a module of boosted trees

* Remove trailing backslashes
---
 tensorflow/contrib/boosted_trees/README.md                | 2 +-
 tensorflow/contrib/boosted_trees/examples/binary_mnist.py | 2 +-
 tensorflow/contrib/boosted_trees/examples/mnist.py        | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/contrib/boosted_trees/README.md b/tensorflow/contrib/boosted_trees/README.md
index 9ce700f1a1..7d30032e53 100644
--- a/tensorflow/contrib/boosted_trees/README.md
+++ b/tensorflow/contrib/boosted_trees/README.md
@@ -1,7 +1,7 @@
 # TF Boosted Trees (TFBT)
 
 TF Boosted trees is an implementation of a gradient boosting algorithm with
-trees used as week learners.
+trees used as weak learners.
 
 ## Examples
 Folder "examples" demonstrates how TFBT estimators can be used for various
diff --git a/tensorflow/contrib/boosted_trees/examples/binary_mnist.py b/tensorflow/contrib/boosted_trees/examples/binary_mnist.py
index c003b1de66..47ee3d816f 100644
--- a/tensorflow/contrib/boosted_trees/examples/binary_mnist.py
+++ b/tensorflow/contrib/boosted_trees/examples/binary_mnist.py
@@ -21,7 +21,7 @@ r"""Demonstrates multiclass MNIST TF Boosted trees example.
   python tensorflow/contrib/boosted_trees/examples/binary_mnist.py \
   --output_dir="/tmp/binary_mnist" --depth=4 --learning_rate=0.3 \
   --batch_size=10761 --examples_per_layer=10761 --eval_batch_size=1030 \
-  --num_eval_steps=1 --num_trees=10 --l2=1 --vmodule=training_ops=1 \
+  --num_eval_steps=1 --num_trees=10 --l2=1 --vmodule=training_ops=1
 
   When training is done, accuracy on eval data is reported. Point tensorboard
   to the directory for the run to see how the training progresses:
diff --git a/tensorflow/contrib/boosted_trees/examples/mnist.py b/tensorflow/contrib/boosted_trees/examples/mnist.py
index 0539d77720..817c6eb3e1 100644
--- a/tensorflow/contrib/boosted_trees/examples/mnist.py
+++ b/tensorflow/contrib/boosted_trees/examples/mnist.py
@@ -22,7 +22,7 @@ r"""Demonstrates multiclass MNIST TF Boosted trees example.
   python tensorflow/contrib/boosted_trees/examples/mnist.py \
   --output_dir="/tmp/mnist" --depth=4 --learning_rate=0.3 --batch_size=60000  \
   --examples_per_layer=60000 --eval_batch_size=10000 --num_eval_steps=1 \
-  --num_trees=10 --l2=1 --vmodule=training_ops=1 \
+  --num_trees=10 --l2=1 --vmodule=training_ops=1
 
   When training is done, accuracy on eval data is reported. Point tensorboard
   to the directory for the run to see how the training progresses:
-- 
GitLab


From 130ec39dae7bc7e71b739520ea65689f63e292d6 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 18 Oct 2017 05:04:57 -0700
Subject: [PATCH 505/909] Stub support for retrieving LossFunction by name.

PiperOrigin-RevId: 172588516
---
 .../kernel_tests/layer_collection_test.py     | 67 +++++++++++++++++++
 .../python/kernel_tests/optimizer_test.py     |  6 +-
 .../kfac/python/ops/layer_collection.py       | 58 +++++++++++++---
 3 files changed, 119 insertions(+), 12 deletions(-)

diff --git a/tensorflow/contrib/kfac/python/kernel_tests/layer_collection_test.py b/tensorflow/contrib/kfac/python/kernel_tests/layer_collection_test.py
index 633104ace0..13c69d261c 100644
--- a/tensorflow/contrib/kfac/python/kernel_tests/layer_collection_test.py
+++ b/tensorflow/contrib/kfac/python/kernel_tests/layer_collection_test.py
@@ -30,6 +30,43 @@ from tensorflow.python.ops import variable_scope
 from tensorflow.python.platform import test
 
 
+class LayerParametersDictTest(test.TestCase):
+
+  def testSetItem(self):
+    """Ensure insertion, contains, retrieval works for supported key types."""
+    with ops.Graph().as_default():
+      lp_dict = layer_collection.LayerParametersDict()
+
+      x = array_ops.constant(0)
+      y0 = array_ops.constant(0)
+      y1 = array_ops.constant(0)
+      z0 = array_ops.constant(0)
+      z1 = array_ops.constant(0)
+      keys = [x, (y0, y1), [z0, z1]]
+      for key in keys:
+        lp_dict[key] = key
+
+      for key in keys:
+        self.assertTrue(key in lp_dict)
+        self.assertEqual(lp_dict[key], key)
+
+  def testSetItemOverlap(self):
+    """Ensure insertion fails if key overlaps with existing key."""
+    with ops.Graph().as_default():
+      lp_dict = layer_collection.LayerParametersDict()
+
+      x = array_ops.constant(0)
+      y = array_ops.constant(0)
+      lp_dict[x] = 'value'
+
+      with self.assertRaises(ValueError):
+        lp_dict[(x, y)] = 'value'
+
+      # Ensure 'y' wasn't inserted.
+      self.assertTrue(x in lp_dict)
+      self.assertFalse(y in lp_dict)
+
+
 class LayerCollectionTest(test.TestCase):
 
   def testLayerCollectionInit(self):
@@ -157,6 +194,36 @@ class LayerCollectionTest(test.TestCase):
       double_loss = sess.run(lc2.total_sampled_loss())
       self.assertAlmostEqual(2 * single_loss, double_loss)
 
+  def testLossFunctionByName(self):
+    """Ensure loss functions can be identified by name."""
+    with ops.Graph().as_default():
+      logits = linalg_ops.eye(2)
+      lc = layer_collection.LayerCollection()
+
+      # Create a new loss function by name.
+      lc.register_categorical_predictive_distribution(logits, name='loss1')
+      self.assertEqual(1, len(lc.losses))
+
+      # Add logits to same loss function.
+      with self.assertRaises(NotImplementedError):
+        lc.register_categorical_predictive_distribution(logits, name='loss1')
+      self.assertEqual(1, len(lc.losses))
+
+      # Add another new loss function.
+      lc.register_categorical_predictive_distribution(logits, name='loss2')
+      self.assertEqual(2, len(lc.losses))
+
+  def testLossFunctionWithoutName(self):
+    """Ensure loss functions get unique names if 'name' not specified."""
+    with ops.Graph().as_default():
+      logits = linalg_ops.eye(2)
+      lc = layer_collection.LayerCollection()
+
+      # Create a new loss function by name.
+      lc.register_categorical_predictive_distribution(logits)
+      lc.register_categorical_predictive_distribution(logits)
+      self.assertEqual(2, len(lc.losses))
+
   def testRegisterCategoricalPredictiveDistributionBatchSize1(self):
     with ops.Graph().as_default():
       random_seed.set_random_seed(200)
diff --git a/tensorflow/contrib/kfac/python/kernel_tests/optimizer_test.py b/tensorflow/contrib/kfac/python/kernel_tests/optimizer_test.py
index 5f28f57f6a..9325aa1b73 100644
--- a/tensorflow/contrib/kfac/python/kernel_tests/optimizer_test.py
+++ b/tensorflow/contrib/kfac/python/kernel_tests/optimizer_test.py
@@ -21,7 +21,6 @@ from __future__ import print_function
 import numpy as np
 
 from tensorflow.contrib.kfac.python.ops import layer_collection as lc
-from tensorflow.contrib.kfac.python.ops import loss_functions as lf
 from tensorflow.contrib.kfac.python.ops import optimizer
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
@@ -124,9 +123,8 @@ class OptimizerTest(test.TestCase):
   def testUpdateVelocities(self):
     with ops.Graph().as_default(), self.test_session() as sess:
       layers = lc.LayerCollection()
-      layers.losses = [
-          lf.CategoricalLogitsNegativeLogProbLoss(array_ops.constant([1.0]))
-      ]
+      layers.register_categorical_predictive_distribution(
+          array_ops.constant([1.0]))
       opt = optimizer.KfacOptimizer(
           0.1, 0.2, 0.3, layers, momentum=0.5, momentum_type='regular')
       x = variable_scope.get_variable('x', initializer=array_ops.ones((2, 2)))
diff --git a/tensorflow/contrib/kfac/python/ops/layer_collection.py b/tensorflow/contrib/kfac/python/ops/layer_collection.py
index 1b77f5d3ba..0cb55894ad 100644
--- a/tensorflow/contrib/kfac/python/ops/layer_collection.py
+++ b/tensorflow/contrib/kfac/python/ops/layer_collection.py
@@ -55,6 +55,7 @@ class LayerParametersDict(OrderedDict):
     super(LayerParametersDict, self).__init__(*args, **kwargs)
 
   def __setitem__(self, key, value):
+    key = self._canonicalize_key(key)
     tensors = key if isinstance(key, (tuple, list)) else (key,)
     key_collisions = self._tensors.intersection(tensors)
     if key_collisions:
@@ -63,9 +64,23 @@ class LayerParametersDict(OrderedDict):
     super(LayerParametersDict, self).__setitem__(key, value)
 
   def __delitem__(self, key):
+    key = self._canonicalize_key(key)
     self._tensors.remove(key)
     super(LayerParametersDict, self).__delitem__(key)
 
+  def __getitem__(self, key):
+    key = self._canonicalize_key(key)
+    return super(LayerParametersDict, self).__getitem__(key)
+
+  def __contains__(self, key):
+    key = self._canonicalize_key(key)
+    return super(LayerParametersDict, self).__contains__(key)
+
+  def _canonicalize_key(self, key):
+    if isinstance(key, (list, tuple)):
+      return tuple(key)
+    return key
+
 
 # TODO(duckworthd): add capability for LayerCollection to be "finalized"
 # and do this when it gets used by FisherEstimator / KfacOptimizer
@@ -94,13 +109,16 @@ class LayerCollection(object):
     self.fisher_factors = OrderedDict()
     self._generic_registrations = set()
     self._graph = graph or ops.get_default_graph()
-    self.losses = []
+    self._loss_dict = {}  # {str: LossFunction}
     self._subgraph = None
 
     with variable_scope.variable_scope(None, default_name=name) as scope:
       self._var_scope = scope.name
 
-  reset_internals = __init__
+  @property
+  def losses(self):
+    """LossFunctions registered with this LayerCollection."""
+    return list(self._loss_dict.values())
 
   def register_block(self, layer_key, fisher_block):
     """Validates and registers the layer_key associated with the fisher_block.
@@ -277,7 +295,8 @@ class LayerCollection(object):
   def register_categorical_predictive_distribution(self,
                                                    logits,
                                                    seed=None,
-                                                   targets=None):
+                                                   targets=None,
+                                                   name=None):
     """Registers a categorical predictive distribution.
 
     Args:
@@ -288,16 +307,24 @@ class LayerCollection(object):
         total_loss() is required, for example, to estimate the
         "empirical Fisher" (instead of the true Fisher).
         (Default: None)
+      name: (OPTIONAL) str or None. Unique name for this loss function. If None,
+        a new name is generated. (Default: None)
     """
+    name = name or self._graph.unique_name(
+        "register_categorical_predictive_distribution")
+    if name in self._loss_dict:
+      raise NotImplementedError(
+          "Adding logits to an existing LossFunction not yet supported.")
     loss = lf.CategoricalLogitsNegativeLogProbLoss(
         logits, targets=targets, seed=seed)
-    self.losses.append(loss)
+    self._loss_dict[name] = loss
 
   def register_normal_predictive_distribution(self,
                                               mean,
                                               var=0.5,
                                               seed=None,
-                                              targets=None):
+                                              targets=None,
+                                              name=None):
     """Registers a normal predictive distribution.
 
     Args:
@@ -312,15 +339,23 @@ class LayerCollection(object):
         total_loss() is required, for example, to estimate the
         "empirical Fisher" (instead of the true Fisher).
         (Default: None)
+      name: (OPTIONAL) str or None. Unique name for this loss function. If None,
+        a new name is generated. (Default: None)
     """
+    name = name or self._graph.unique_name(
+        "register_normal_predictive_distribution")
+    if name in self._loss_dict:
+      raise NotImplementedError(
+          "Adding logits to an existing LossFunction not yet supported.")
     loss = lf.NormalMeanNegativeLogProbLoss(
         mean, var, targets=targets, seed=seed)
-    self.losses.append(loss)
+    self._loss_dict[name] = loss
 
   def register_multi_bernoulli_predictive_distribution(self,
                                                        logits,
                                                        seed=None,
-                                                       targets=None):
+                                                       targets=None,
+                                                       name=None):
     """Registers a multi-Bernoulli predictive distribution.
 
     Args:
@@ -331,10 +366,17 @@ class LayerCollection(object):
         total_loss() is required, for example, to estimate the
         "empirical Fisher" (instead of the true Fisher).
         (Default: None)
+      name: (OPTIONAL) str or None. Unique name for this loss function. If None,
+        a new name is generated. (Default: None)
     """
+    name = name or self._graph.unique_name(
+        "register_multi_bernoulli_predictive_distribution")
+    if name in self._loss_dict:
+      raise NotImplementedError(
+          "Adding logits to an existing LossFunction not yet supported.")
     loss = lf.MultiBernoulliNegativeLogProbLoss(
         logits, targets=targets, seed=seed)
-    self.losses.append(loss)
+    self._loss_dict[name] = loss
 
   def make_or_get_factor(self, cls, args):
     with variable_scope.variable_scope(self._var_scope):
-- 
GitLab


From 139e1e0771faeaa614e3c6672a5c203866ba0176 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Wed, 18 Oct 2017 09:18:00 -0700
Subject: [PATCH 506/909] Add `int64` axis support for `tf.cumsum` and
 `tf.cumprod` (#13791)

* Add `int64` axis support for `tf.cumsum` and `tf.cumprod`

This fix adds `int64` axis support for `tf.cumsum` and `tf.cumprod`.
Though `int64` is the registered data type for `axis` (`Tidx`), no
kernel is available.

The issue could be described as:
```
>>> import tensorflow as tf
>>> v = tf.cumsum([1, 2, 3], tf.constant(0, tf.int64))
>>> tf.Session().run(v)
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.py", line 889, in run
    run_metadata_ptr)
  File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.py", line 1120, in _run
    feed_dict_tensor, options, run_metadata)
  File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.py", line 1317, in _do_run
    options, run_metadata)
  File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.py", line 1336, in _do_call
    raise type(e)(node_def, op, message)
tensorflow.python.framework.errors_impl.InvalidArgumentError: No OpKernel was registered to support Op 'Cumsum' with these attrs.  Registered devices: [CPU], Registered kernels:
  device='CPU'; T in [DT_COMPLEX128]; Tidx in [DT_INT32]
  device='CPU'; T in [DT_COMPLEX64]; Tidx in [DT_INT32]
  device='CPU'; T in [DT_DOUBLE]; Tidx in [DT_INT32]
  device='CPU'; T in [DT_FLOAT]; Tidx in [DT_INT32]
  device='CPU'; T in [DT_HALF]; Tidx in [DT_INT32]
  device='CPU'; T in [DT_INT8]; Tidx in [DT_INT32]
  device='CPU'; T in [DT_UINT8]; Tidx in [DT_INT32]
  device='CPU'; T in [DT_INT16]; Tidx in [DT_INT32]
  device='CPU'; T in [DT_UINT16]; Tidx in [DT_INT32]
  device='CPU'; T in [DT_INT32]; Tidx in [DT_INT32]
  device='CPU'; T in [DT_INT64]; Tidx in [DT_INT32]

	 [[Node: Cumsum = Cumsum[T=DT_INT32, Tidx=DT_INT64, exclusive=false, reverse=false](Cumsum/x, Const)]]

Caused by op u'Cumsum', defined at:
  File "<stdin>", line 1, in <module>
  File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/math_ops.py", line 2246, in cumsum
    x, axis, exclusive=exclusive, reverse=reverse, name=name)
  File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/gen_math_ops.py", line 1370, in cumsum
    name=name)
  File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/op_def_library.py", line 787, in _apply_op_helper
    op_def=op_def)
  File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/ops.py", line 2966, in create_op
    op_def=op_def)
  File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/ops.py", line 1473, in __init__
    self._traceback = self._graph._extract_stack()  # pylint: disable=protected-access

InvalidArgumentError (see above for traceback): No OpKernel was registered to support Op 'Cumsum' with these attrs.  Registered devices: [CPU], Registered kernels:
  device='CPU'; T in [DT_COMPLEX128]; Tidx in [DT_INT32]
  device='CPU'; T in [DT_COMPLEX64]; Tidx in [DT_INT32]
  device='CPU'; T in [DT_DOUBLE]; Tidx in [DT_INT32]
  device='CPU'; T in [DT_FLOAT]; Tidx in [DT_INT32]
  device='CPU'; T in [DT_HALF]; Tidx in [DT_INT32]
  device='CPU'; T in [DT_INT8]; Tidx in [DT_INT32]
  device='CPU'; T in [DT_UINT8]; Tidx in [DT_INT32]
  device='CPU'; T in [DT_INT16]; Tidx in [DT_INT32]
  device='CPU'; T in [DT_UINT16]; Tidx in [DT_INT32]
  device='CPU'; T in [DT_INT32]; Tidx in [DT_INT32]
  device='CPU'; T in [DT_INT64]; Tidx in [DT_INT32]

	 [[Node: Cumsum = Cumsum[T=DT_INT32, Tidx=DT_INT64, exclusive=false, reverse=false](Cumsum/x, Const)]]

>>>
```

This fix adds the missing kernel.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Add test cases for `int64` axis support of `tf.cumsum` and `tf.cumprod`

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Reformat scan_ops.cc with `clang-format -i`

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/core/kernels/scan_ops.cc           | 98 ++++++++++++-------
 .../python/kernel_tests/scan_ops_test.py      | 18 ++++
 2 files changed, 80 insertions(+), 36 deletions(-)

diff --git a/tensorflow/core/kernels/scan_ops.cc b/tensorflow/core/kernels/scan_ops.cc
index cc434ab0ae..0a6848361a 100644
--- a/tensorflow/core/kernels/scan_ops.cc
+++ b/tensorflow/core/kernels/scan_ops.cc
@@ -35,7 +35,7 @@ namespace tensorflow {
 typedef Eigen::ThreadPoolDevice CPUDevice;
 typedef Eigen::GpuDevice GPUDevice;
 
-template <typename Device, class T, typename Reducer>
+template <typename Device, class T, typename Reducer, typename Tidx>
 class ScanOp : public OpKernel {
  public:
   explicit ScanOp(OpKernelConstruction* ctx) : OpKernel(ctx) {
@@ -51,8 +51,9 @@ class ScanOp : public OpKernel {
                 errors::InvalidArgument("ScanOp: axis must be a scalar, not ",
                                         tensor_axis.shape().DebugString()));
 
-    const int axis_arg = internal::SubtleMustCopy(tensor_axis.scalar<int>()());
-    const int axis = (axis_arg < 0) ? input.dims() + axis_arg : axis_arg;
+    const Tidx axis_arg =
+        internal::SubtleMustCopy(tensor_axis.scalar<Tidx>()());
+    const Tidx axis = (axis_arg < 0) ? input.dims() + axis_arg : axis_arg;
     OP_REQUIRES(ctx, FastBoundsCheck(axis, input.dims()),
                 errors::InvalidArgument(
                     "ScanOp: Expected scan axis in the range [", -input.dims(),
@@ -70,11 +71,11 @@ class ScanOp : public OpKernel {
 
     // Dim reduction.
     int64 reduced_shape[3] = {1, 1, 1};
-    for (int i = 0; i < axis; ++i) {
+    for (Tidx i = 0; i < axis; ++i) {
       reduced_shape[0] *= input.dim_size(i);
     }
     reduced_shape[1] = input.dim_size(axis);
-    for (int i = axis + 1; i < input.dims(); ++i) {
+    for (Tidx i = axis + 1; i < input.dims(); ++i) {
       reduced_shape[2] *= input.dim_size(i);
     }
 
@@ -112,51 +113,76 @@ TF_CALL_GPU_NUMBER_TYPES(DECLARE_FOR_ALL_REDUCERS);
 }  // namespace functor
 #endif  // GOOGLE_CUDA
 
-
 // Register Cumsum kernels
-#define REGISTER_CPU_KERNELS(type)        \
-  REGISTER_KERNEL_BUILDER(                \
-      Name("Cumsum")                      \
-          .Device(DEVICE_CPU)             \
-          .TypeConstraint<type>("T")      \
-          .TypeConstraint<int32>("Tidx"), \
-      ScanOp<CPUDevice, type, Eigen::internal::SumReducer<type>>)
+#define REGISTER_CPU_KERNELS(type)                                       \
+  REGISTER_KERNEL_BUILDER(                                               \
+      Name("Cumsum")                                                     \
+          .Device(DEVICE_CPU)                                            \
+          .TypeConstraint<type>("T")                                     \
+          .TypeConstraint<int32>("Tidx"),                                \
+      ScanOp<CPUDevice, type, Eigen::internal::SumReducer<type>, int32>) \
+  REGISTER_KERNEL_BUILDER(                                               \
+      Name("Cumsum")                                                     \
+          .Device(DEVICE_CPU)                                            \
+          .TypeConstraint<type>("T")                                     \
+          .TypeConstraint<int64>("Tidx"),                                \
+      ScanOp<CPUDevice, type, Eigen::internal::SumReducer<type>, int64>)
 TF_CALL_NUMBER_TYPES(REGISTER_CPU_KERNELS);
 #undef REGISTER_CPU_KERNELS
 
 #if GOOGLE_CUDA
-#define REGISTER_GPU_KERNELS(type)       \
-  REGISTER_KERNEL_BUILDER(               \
-      Name("Cumsum")                     \
-          .Device(DEVICE_GPU)            \
-          .TypeConstraint<type>("T")     \
-          .TypeConstraint<int32>("Tidx") \
-          .HostMemory("axis"),           \
-      ScanOp<GPUDevice, type, Eigen::internal::SumReducer<type>>)
+#define REGISTER_GPU_KERNELS(type)                                       \
+  REGISTER_KERNEL_BUILDER(                                               \
+      Name("Cumsum")                                                     \
+          .Device(DEVICE_GPU)                                            \
+          .TypeConstraint<type>("T")                                     \
+          .TypeConstraint<int32>("Tidx")                                 \
+          .HostMemory("axis"),                                           \
+      ScanOp<GPUDevice, type, Eigen::internal::SumReducer<type>, int32>) \
+  REGISTER_KERNEL_BUILDER(                                               \
+      Name("Cumsum")                                                     \
+          .Device(DEVICE_GPU)                                            \
+          .TypeConstraint<type>("T")                                     \
+          .TypeConstraint<int64>("Tidx")                                 \
+          .HostMemory("axis"),                                           \
+      ScanOp<GPUDevice, type, Eigen::internal::SumReducer<type>, int64>)
 TF_CALL_GPU_NUMBER_TYPES(REGISTER_GPU_KERNELS)
 #undef REGISTER_GPU_KERNELS
 #endif  // GOOGLE_CUDA
 
 // Register Cumprod kernels
-#define REGISTER_CPU_KERNELS(type)        \
-  REGISTER_KERNEL_BUILDER(                \
-      Name("Cumprod")                     \
-          .Device(DEVICE_CPU)             \
-          .TypeConstraint<type>("T")      \
-          .TypeConstraint<int32>("Tidx"), \
-      ScanOp<CPUDevice, type, Eigen::internal::ProdReducer<type>>)
+#define REGISTER_CPU_KERNELS(type)                                        \
+  REGISTER_KERNEL_BUILDER(                                                \
+      Name("Cumprod")                                                     \
+          .Device(DEVICE_CPU)                                             \
+          .TypeConstraint<type>("T")                                      \
+          .TypeConstraint<int32>("Tidx"),                                 \
+      ScanOp<CPUDevice, type, Eigen::internal::ProdReducer<type>, int32>) \
+  REGISTER_KERNEL_BUILDER(                                                \
+      Name("Cumprod")                                                     \
+          .Device(DEVICE_CPU)                                             \
+          .TypeConstraint<type>("T")                                      \
+          .TypeConstraint<int64>("Tidx"),                                 \
+      ScanOp<CPUDevice, type, Eigen::internal::ProdReducer<type>, int64>)
 TF_CALL_NUMBER_TYPES(REGISTER_CPU_KERNELS);
 #undef REGISTER_CPU_KERNELS
 
 #if GOOGLE_CUDA
-#define REGISTER_GPU_KERNELS(type)       \
-  REGISTER_KERNEL_BUILDER(               \
-      Name("Cumprod")                    \
-          .Device(DEVICE_GPU)            \
-          .TypeConstraint<type>("T")     \
-          .TypeConstraint<int32>("Tidx") \
-          .HostMemory("axis"),           \
-      ScanOp<GPUDevice, type, Eigen::internal::ProdReducer<type>>)
+#define REGISTER_GPU_KERNELS(type)                                        \
+  REGISTER_KERNEL_BUILDER(                                                \
+      Name("Cumprod")                                                     \
+          .Device(DEVICE_GPU)                                             \
+          .TypeConstraint<type>("T")                                      \
+          .TypeConstraint<int32>("Tidx")                                  \
+          .HostMemory("axis"),                                            \
+      ScanOp<GPUDevice, type, Eigen::internal::ProdReducer<type>, int32>) \
+  REGISTER_KERNEL_BUILDER(                                                \
+      Name("Cumprod")                                                     \
+          .Device(DEVICE_GPU)                                             \
+          .TypeConstraint<type>("T")                                      \
+          .TypeConstraint<int64>("Tidx")                                  \
+          .HostMemory("axis"),                                            \
+      ScanOp<GPUDevice, type, Eigen::internal::ProdReducer<type>, int64>)
 TF_CALL_GPU_NUMBER_TYPES(REGISTER_GPU_KERNELS)
 #undef REGISTER_GPU_KERNELS
 #endif  // GOOGLE_CUDA
diff --git a/tensorflow/python/kernel_tests/scan_ops_test.py b/tensorflow/python/kernel_tests/scan_ops_test.py
index 6b2b589a06..08b4a2aaae 100644
--- a/tensorflow/python/kernel_tests/scan_ops_test.py
+++ b/tensorflow/python/kernel_tests/scan_ops_test.py
@@ -20,6 +20,8 @@ from __future__ import print_function
 
 import numpy as np
 
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors_impl
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import gradient_checker
@@ -92,6 +94,14 @@ class CumsumTest(test.TestCase):
       for axis in (-1, 0):
         self._compareAll(x, axis)
 
+  def testAxisType(self):
+    for dtype in self.valid_dtypes:
+      x = np.arange(1, 6).reshape([5]).astype(dtype)
+      for axis_dtype in [dtypes.int64, dtypes.int32]:
+        with self.test_session(use_gpu=True):
+          axis = constant_op.constant(0, axis_dtype)
+          tf_out = math_ops.cumsum(x, axis).eval()
+
   def test1D(self):
     for dtype in self.valid_dtypes:
       x = np.arange(1, 6).reshape([5]).astype(dtype)
@@ -190,6 +200,14 @@ class CumprodTest(test.TestCase):
       for axis in (-1, 0):
         self._compareAll(x, axis)
 
+  def testAxisType(self):
+    for dtype in self.valid_dtypes:
+      x = np.arange(1, 6).reshape([5]).astype(dtype)
+      for axis_dtype in [dtypes.int64, dtypes.int32]:
+        with self.test_session(use_gpu=True):
+          axis = constant_op.constant(0, axis_dtype)
+          tf_out = math_ops.cumprod(x, axis).eval()
+
   def test1D(self):
     for dtype in self.valid_dtypes:
       x = np.arange(1, 6).reshape([5]).astype(dtype)
-- 
GitLab


From 75da6f62494e1fd6dd7c197e7b5b79a1a451fb3d Mon Sep 17 00:00:00 2001
From: Christian Grail <cgrail@users.noreply.github.com>
Date: Wed, 18 Oct 2017 18:22:45 +0200
Subject: [PATCH 507/909] gitignore: ignore build files relevant for iOS sample
 apps (#13809)

* gitignore: ignore build files relavant for ios sample apps

* Add extra linespace at the end
---
 .gitignore | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.gitignore b/.gitignore
index 9572a3e97c..9ae0d9c96f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -18,3 +18,7 @@ cmake_build/
 /build/
 /tensorflow/core/util/version_info.cc
 /tensorflow/python/framework/fast_tensor_util.cpp
+Pods
+Podfile.lock
+*.pbxproj
+*.xcworkspacedata
-- 
GitLab


From 3c31886537a8b5fb5ab62b4b925f8ef044960ca3 Mon Sep 17 00:00:00 2001
From: Mark Heffernan <meheff@google.com>
Date: Wed, 18 Oct 2017 09:38:03 -0700
Subject: [PATCH 508/909] Don't emit fusion computations separately in
 HloModule::ToString. These computations are emitted with their fusion
 instruction and therefore don't need to be emitted as a separate comptutation
 in the module.

PiperOrigin-RevId: 172612725
---
 tensorflow/compiler/xla/service/hlo_module.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_module.cc b/tensorflow/compiler/xla/service/hlo_module.cc
index 5bc7a36439..9d4a994838 100644
--- a/tensorflow/compiler/xla/service/hlo_module.cc
+++ b/tensorflow/compiler/xla/service/hlo_module.cc
@@ -154,8 +154,8 @@ string HloModule::ToString() const {
   std::ostringstream s;
   s << "HloModule " << name() << ":\n\n";
   s << "ENTRY " << entry_computation()->ToString() << "\n\n";
-  for (const std::unique_ptr<HloComputation>& computation : computations_) {
-    if (computation.get() != entry_computation()) {
+  for (const HloComputation* computation : MakeNonfusionComputations()) {
+    if (computation != entry_computation()) {
       s << computation->ToString() << "\n\n";
     }
   }
-- 
GitLab


From c9d3377fb4f973e1592ebc71862e02dacf5f3a4f Mon Sep 17 00:00:00 2001
From: "Joshua V. Dillon" <jvdillon@google.com>
Date: Wed, 18 Oct 2017 10:49:20 -0700
Subject: [PATCH 509/909] Make `tf.contrib.distributions` quadrature family
 parameterized by `quadrature_grid_and_prob` vs `quadrature_degree`. Enables
 support of quadrature methods other than Gauss-Hermite.

PiperOrigin-RevId: 172622919
---
 .../kernel_tests/poisson_lognormal_test.py    | 20 +++++--
 .../python/ops/poisson_lognormal.py           | 54 +++++++++++-------
 .../python/ops/vector_diffeomixture.py        | 55 +++++++++++--------
 3 files changed, 79 insertions(+), 50 deletions(-)

diff --git a/tensorflow/contrib/distributions/python/kernel_tests/poisson_lognormal_test.py b/tensorflow/contrib/distributions/python/kernel_tests/poisson_lognormal_test.py
index 7cb46bb236..3ded4159d8 100644
--- a/tensorflow/contrib/distributions/python/kernel_tests/poisson_lognormal_test.py
+++ b/tensorflow/contrib/distributions/python/kernel_tests/poisson_lognormal_test.py
@@ -18,6 +18,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import numpy as np
+
 from tensorflow.contrib.distributions.python.ops import poisson_lognormal
 from tensorflow.contrib.distributions.python.ops import test_util
 from tensorflow.python.platform import test
@@ -32,7 +34,8 @@ class PoissonLogNormalQuadratureCompoundTest(
       pln = poisson_lognormal.PoissonLogNormalQuadratureCompound(
           loc=-2.,
           scale=1.1,
-          quadrature_polynomial_degree=10,
+          quadrature_grid_and_probs=(
+              np.polynomial.hermite.hermgauss(deg=10)),
           validate_args=True)
       self.run_test_sample_consistent_log_prob(
           sess, pln, rtol=0.1)
@@ -42,7 +45,8 @@ class PoissonLogNormalQuadratureCompoundTest(
       pln = poisson_lognormal.PoissonLogNormalQuadratureCompound(
           loc=0.,
           scale=1.,
-          quadrature_polynomial_degree=10,
+          quadrature_grid_and_probs=(
+              np.polynomial.hermite.hermgauss(deg=10)),
           validate_args=True)
       self.run_test_sample_consistent_mean_variance(
           sess, pln, rtol=0.02)
@@ -52,7 +56,8 @@ class PoissonLogNormalQuadratureCompoundTest(
       pln = poisson_lognormal.PoissonLogNormalQuadratureCompound(
           loc=[0., -0.5],
           scale=1.,
-          quadrature_polynomial_degree=10,
+          quadrature_grid_and_probs=(
+              np.polynomial.hermite.hermgauss(deg=10)),
           validate_args=True)
       self.run_test_sample_consistent_log_prob(
           sess, pln, rtol=0.1, atol=0.01)
@@ -62,7 +67,8 @@ class PoissonLogNormalQuadratureCompoundTest(
       pln = poisson_lognormal.PoissonLogNormalQuadratureCompound(
           loc=[0., -0.5],
           scale=1.,
-          quadrature_polynomial_degree=10,
+          quadrature_grid_and_probs=(
+              np.polynomial.hermite.hermgauss(deg=10)),
           validate_args=True)
       self.run_test_sample_consistent_mean_variance(
           sess, pln, rtol=0.1, atol=0.01)
@@ -72,7 +78,8 @@ class PoissonLogNormalQuadratureCompoundTest(
       pln = poisson_lognormal.PoissonLogNormalQuadratureCompound(
           loc=[[0.], [-0.5]],
           scale=[[1., 0.9]],
-          quadrature_polynomial_degree=10,
+          quadrature_grid_and_probs=(
+              np.polynomial.hermite.hermgauss(deg=10)),
           validate_args=True)
       self.run_test_sample_consistent_log_prob(
           sess, pln, rtol=0.1, atol=0.08)
@@ -82,7 +89,8 @@ class PoissonLogNormalQuadratureCompoundTest(
       pln = poisson_lognormal.PoissonLogNormalQuadratureCompound(
           loc=[[0.], [-0.5]],
           scale=[[1., 0.9]],
-          quadrature_polynomial_degree=10,
+          quadrature_grid_and_probs=(
+              np.polynomial.hermite.hermgauss(deg=10)),
           validate_args=True)
       self.run_test_sample_consistent_mean_variance(
           sess, pln, rtol=0.1, atol=0.01)
diff --git a/tensorflow/contrib/distributions/python/ops/poisson_lognormal.py b/tensorflow/contrib/distributions/python/ops/poisson_lognormal.py
index 65ee3a16d6..80d4e2dc5e 100644
--- a/tensorflow/contrib/distributions/python/ops/poisson_lognormal.py
+++ b/tensorflow/contrib/distributions/python/ops/poisson_lognormal.py
@@ -93,7 +93,7 @@ class PoissonLogNormalQuadratureCompound(distribution_lib.Distribution):
           : d=0, ..., deg-1 }
   ```
 
-  where, [`grid, w = numpy.polynomial.hermite.hermgauss(deg)`](
+  where, [e.g., `grid, w = numpy.polynomial.hermite.hermgauss(deg)`](
   https://docs.scipy.org/doc/numpy-1.10.0/reference/generated/numpy.polynomial.hermite.hermgauss.html)
   and `prob = w / sqrt(pi)`.
 
@@ -106,14 +106,15 @@ class PoissonLogNormalQuadratureCompound(distribution_lib.Distribution):
   pln = ds.PoissonLogNormalQuadratureCompound(
       loc=[0., -0.5],
       scale=1.,
-      quadrature_polynomial_degree=10,
+      quadrature_grid_and_probs=(
+        np.polynomial.hermite.hermgauss(deg=10)),
       validate_args=True)
   """
 
   def __init__(self,
                loc,
                scale,
-               quadrature_polynomial_degree=8,
+               quadrature_grid_and_probs=None,
                validate_args=False,
                allow_nan_stats=True,
                name="PoissonLogNormalQuadratureCompound"):
@@ -124,8 +125,9 @@ class PoissonLogNormalQuadratureCompound(distribution_lib.Distribution):
         the LogNormal prior.
       scale: `float`-like (batch of) scalar `Tensor`; the scale parameter of
         the LogNormal prior.
-      quadrature_polynomial_degree: Python `int`-like scalar.
-        Default value: 8.
+      quadrature_grid_and_probs: Python pair of `list`-like objects representing
+        the sample points and the corresponding (possibly normalized) weight.
+        When `None`, defaults to: `np.polynomial.hermite.hermgauss(deg=8)`.
       validate_args: Python `bool`, default `False`. When `True` distribution
         parameters are checked for validity despite possibly degrading runtime
         performance. When `False` invalid inputs may silently render incorrect
@@ -138,6 +140,8 @@ class PoissonLogNormalQuadratureCompound(distribution_lib.Distribution):
 
     Raises:
       TypeError: if `loc.dtype != scale[0].dtype`.
+      ValueError: if `quadrature_grid_and_probs is not None` and
+        `len(quadrature_grid_and_probs[0]) != len(quadrature_grid_and_probs[1])`
     """
     parameters = locals()
     with ops.name_scope(name, values=[loc, scale]):
@@ -153,18 +157,21 @@ class PoissonLogNormalQuadratureCompound(distribution_lib.Distribution):
             "loc.dtype(\"{}\") does not match scale.dtype(\"{}\")".format(
                 loc.dtype.name, scale.dtype.name))
 
-      self._degree = quadrature_polynomial_degree
-
-      grid, prob = np.polynomial.hermite.hermgauss(
-          deg=quadrature_polynomial_degree)
-
-      # It should be that `sum(prob) == sqrt(pi)`, but self-normalization is
-      # more numerically stable.
-      prob = prob.astype(dtype.as_numpy_dtype)
-      prob /= np.linalg.norm(prob, ord=1)
+      if quadrature_grid_and_probs is None:
+        grid, probs = np.polynomial.hermite.hermgauss(deg=8)
+      else:
+        grid, probs = tuple(quadrature_grid_and_probs)
+        if len(grid) != len(probs):
+          raise ValueError("`quadrature_grid_and_probs` must be a `tuple` of "
+                           "same-length list-like objects")
+      grid = grid.astype(dtype.as_numpy_dtype)
+      probs = probs.astype(dtype.as_numpy_dtype)
+      probs /= np.linalg.norm(probs, ord=1)
+      self._quadrature_grid = grid
+      self._quadrature_probs = probs
 
       self._mixture_distribution = categorical_lib.Categorical(
-          logits=np.log(prob),
+          logits=np.log(probs),
           validate_args=validate_args,
           allow_nan_stats=allow_nan_stats)
 
@@ -210,9 +217,14 @@ class PoissonLogNormalQuadratureCompound(distribution_lib.Distribution):
     return self._scale
 
   @property
-  def quadrature_polynomial_degree(self):
-    """Polynomial largest exponent used for Gauss-Hermite quadrature."""
-    return self._degree
+  def quadrature_grid(self):
+    """Quadrature grid points."""
+    return self._quadrature_grid
+
+  @property
+  def quadrature_probs(self):
+    """Quadrature normalized weights."""
+    return self._quadrature_probs
 
   def _batch_shape_tensor(self):
     return array_ops.broadcast_dynamic_shape(
@@ -242,10 +254,10 @@ class PoissonLogNormalQuadratureCompound(distribution_lib.Distribution):
                 [batch_size])),
         seed=distribution_util.gen_new_seed(
             seed, "poisson_lognormal_quadrature_compound"))
-    # Stride `quadrature_polynomial_degree` for `batch_size` number of times.
+    # Stride `quadrature_degree` for `batch_size` number of times.
     offset = math_ops.range(start=0,
-                            limit=batch_size * self._degree,
-                            delta=self._degree,
+                            limit=batch_size * len(self.quadrature_probs),
+                            delta=len(self.quadrature_probs),
                             dtype=ids.dtype)
     ids += offset
     rate = array_ops.gather(
diff --git a/tensorflow/contrib/distributions/python/ops/vector_diffeomixture.py b/tensorflow/contrib/distributions/python/ops/vector_diffeomixture.py
index 438d628da4..33dad811a9 100644
--- a/tensorflow/contrib/distributions/python/ops/vector_diffeomixture.py
+++ b/tensorflow/contrib/distributions/python/ops/vector_diffeomixture.py
@@ -141,7 +141,7 @@ class VectorDiffeomixture(distribution_lib.Distribution):
   and,
 
   ```none
-  grid, weight = np.polynomial.hermite.hermgauss(quadrature_polynomial_degree)
+  grid, weight = np.polynomial.hermite.hermgauss(quadrature_degree)
   prob[k]   = weight[k] / sqrt(pi)
   lambda[k; i] = sigmoid(mix_loc[k] + sqrt(2) mix_scale[k] grid[i])
   ```
@@ -219,7 +219,7 @@ class VectorDiffeomixture(distribution_lib.Distribution):
                distribution,
                loc=None,
                scale=None,
-               quadrature_polynomial_degree=8,
+               quadrature_grid_and_probs=None,
                validate_args=False,
                allow_nan_stats=True,
                name="VectorDiffeomixture"):
@@ -248,7 +248,9 @@ class VectorDiffeomixture(distribution_lib.Distribution):
         `k`-th element represents the `scale` used for the `k`-th affine
         transformation. `LinearOperator`s must have shape `[B1, ..., Bb, d, d]`,
         `b >= 0`, i.e., characterizes `b`-batches of `d x d` matrices
-      quadrature_polynomial_degree: Python `int`-like scalar.
+      quadrature_grid_and_probs: Python pair of `list`-like objects representing
+        the sample points and the corresponding (possibly normalized) weight.
+        When `None`, defaults to: `np.polynomial.hermite.hermgauss(deg=8)`.
       validate_args: Python `bool`, default `False`. When `True` distribution
         parameters are checked for validity despite possibly degrading runtime
         performance. When `False` invalid inputs may silently render incorrect
@@ -262,7 +264,8 @@ class VectorDiffeomixture(distribution_lib.Distribution):
     Raises:
       ValueError: if `not scale or len(scale) < 2`.
       ValueError: if `len(loc) != len(scale)`
-      ValueError: if `quadrature_polynomial_degree < 1`.
+      ValueError: if `quadrature_grid_and_probs is not None` and
+        `len(quadrature_grid_and_probs[0]) != len(quadrature_grid_and_probs[1])`
       ValueError: if `validate_args` and any not scale.is_positive_definite.
       TypeError: if any scale.dtype != scale[0].dtype.
       TypeError: if any loc.dtype != scale[0].dtype.
@@ -307,12 +310,6 @@ class VectorDiffeomixture(distribution_lib.Distribution):
                                name="endpoint_affine_{}".format(k))
           for k, (loc_, scale_) in enumerate(zip(loc, scale))]
 
-      if quadrature_polynomial_degree < 1:
-        raise ValueError("quadrature_polynomial_degree={} "
-                         "is not at least 1".format(
-                             quadrature_polynomial_degree))
-      self._degree = quadrature_polynomial_degree
-
       # TODO(jvdillon): Remove once we support k-mixtures.
       # We make this assertion here because otherwise `grid` would need to be a
       # vector not a scalar.
@@ -320,17 +317,24 @@ class VectorDiffeomixture(distribution_lib.Distribution):
         raise NotImplementedError("Currently only bimixtures are supported; "
                                   "len(scale)={} is not 2.".format(len(scale)))
 
-      grid, prob = np.polynomial.hermite.hermgauss(
-          deg=quadrature_polynomial_degree)
+      if quadrature_grid_and_probs is None:
+        grid, probs = np.polynomial.hermite.hermgauss(deg=8)
+      else:
+        grid, probs = tuple(quadrature_grid_and_probs)
+        if len(grid) != len(probs):
+          raise ValueError("`quadrature_grid_and_probs` must be a `tuple` of "
+                           "same-length list-like objects")
       grid = grid.astype(dtype.as_numpy_dtype)
-      prob = prob.astype(dtype.as_numpy_dtype)
-      prob /= np.linalg.norm(prob, ord=1)
+      probs = probs.astype(dtype.as_numpy_dtype)
+      probs /= np.linalg.norm(probs, ord=1)
+      self._quadrature_grid = grid
+      self._quadrature_probs = probs
 
       # Note: by creating the logits as `log(prob)` we ensure that
       # `self.mixture_distribution.logits` is equivalent to
       # `math_ops.log(self.mixture_distribution.probs)`.
       self._mixture_distribution = categorical_lib.Categorical(
-          logits=np.log(prob),
+          logits=np.log(probs),
           validate_args=validate_args,
           allow_nan_stats=allow_nan_stats)
 
@@ -357,10 +361,10 @@ class VectorDiffeomixture(distribution_lib.Distribution):
                                validate_args=validate_args,
                                name="interpolated_affine_{}".format(k))
           for k, (loc_, scale_) in enumerate(zip(
-              interpolate_loc(quadrature_polynomial_degree,
+              interpolate_loc(len(self._quadrature_grid),
                               self._interpolate_weight,
                               loc),
-              interpolate_scale(quadrature_polynomial_degree,
+              interpolate_scale(len(self._quadrature_grid),
                                 self._interpolate_weight,
                                 scale)))]
 
@@ -416,9 +420,14 @@ class VectorDiffeomixture(distribution_lib.Distribution):
     return self._interpolated_affine
 
   @property
-  def quadrature_polynomial_degree(self):
-    """Polynomial largest exponent used for Gauss-Hermite quadrature."""
-    return self._degree
+  def quadrature_grid(self):
+    """Quadrature grid points."""
+    return self._quadrature_grid
+
+  @property
+  def quadrature_probs(self):
+    """Quadrature normalized weights."""
+    return self._quadrature_probs
 
   def _batch_shape_tensor(self):
     return self._batch_shape_
@@ -454,10 +463,10 @@ class VectorDiffeomixture(distribution_lib.Distribution):
         seed=distribution_util.gen_new_seed(
             seed, "vector_diffeomixture"))
 
-    # Stride `self._degree` for `batch_size` number of times.
+    # Stride `quadrature_degree` for `batch_size` number of times.
     offset = math_ops.range(start=0,
-                            limit=batch_size * self._degree,
-                            delta=self._degree,
+                            limit=batch_size * len(self.quadrature_probs),
+                            delta=len(self.quadrature_probs),
                             dtype=ids.dtype)
 
     weight = array_ops.gather(
-- 
GitLab


From bc0822675598385d8068bf114b453dac52512caf Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Wed, 18 Oct 2017 11:09:27 -0700
Subject: [PATCH 510/909] Fixes test breakage.

PiperOrigin-RevId: 172626499
---
 tensorflow/python/eager/BUILD       | 7 +++----
 tensorflow/python/eager/ops_test.py | 6 +++---
 2 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/tensorflow/python/eager/BUILD b/tensorflow/python/eager/BUILD
index 9e9a7f4c59..ef04f933c5 100644
--- a/tensorflow/python/eager/BUILD
+++ b/tensorflow/python/eager/BUILD
@@ -391,14 +391,14 @@ py_test(
     ],
 )
 
-py_test(
+cuda_py_test(
     name = "ops_test",
     srcs = ["ops_test.py"],
-    srcs_version = "PY2AND3",
-    deps = [
+    additional_deps = [
         ":context",
         ":execute",
         ":test",
+        "//third_party/py/numpy",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:control_flow_ops",
         "//tensorflow/python:dtypes",
@@ -410,7 +410,6 @@ py_test(
         "//tensorflow/python:random_ops",
         "//tensorflow/python:sparse_ops",
         "//tensorflow/python:tensor_shape",
-        "//third_party/py/numpy",
     ],
 )
 
diff --git a/tensorflow/python/eager/ops_test.py b/tensorflow/python/eager/ops_test.py
index 6d1a5fe264..f737bfbc15 100644
--- a/tensorflow/python/eager/ops_test.py
+++ b/tensorflow/python/eager/ops_test.py
@@ -49,7 +49,7 @@ class OpsTest(test_util.TensorFlowTestCase):
     three = constant_op.constant([[3.]]).as_gpu_tensor()
     five = constant_op.constant([[5.]]).as_gpu_tensor()
     product = math_ops.matmul(three, five)
-    self.assertEqual([[15.0]], product)
+    self.assertEqual([[15.0]], product.numpy())
 
   def testExecuteStringAttr(self):
     three = constant_op.constant(3.0)
@@ -97,7 +97,7 @@ class OpsTest(test_util.TensorFlowTestCase):
       self.skipTest('No GPUs found')
     with context.device('/gpu:0'):
       r = constant_op.constant(1) + constant_op.constant(2)
-    self.assertEqual(r, 3)
+    self.assertAllEqual(r, 3)
 
   def testExecuteListOutputLen1(self):
     split_dim = constant_op.constant(1)
@@ -264,7 +264,7 @@ class OpsTest(test_util.TensorFlowTestCase):
     # The Shape op kernel on GPU places the output in host memory.
     value = constant_op.constant([1.]).as_gpu_tensor()
     shape = array_ops.shape(value)
-    self.assertEquals([1], shape)
+    self.assertEqual([1], shape.numpy())
 
   def testRandomUniform(self):
     scalar_shape = constant_op.constant([], dtype=dtypes.int32)
-- 
GitLab


From 9aadd980cd0202a338e058da05c9970ccb160bc5 Mon Sep 17 00:00:00 2001
From: Jinze Bai <baijinze1994@163.com>
Date: Thu, 19 Oct 2017 02:22:24 +0800
Subject: [PATCH 511/909] Add nth_element op (#13720)

* add nth_element op

* change op order in buildifier

* remove the symbol in ops/nn.py

* add nth_element symbol in contrib.nn

* change nth_element symbol to ops.nn_ops
---
 tensorflow/contrib/nn/__init__.py             |   2 +
 tensorflow/core/kernels/BUILD                 |   7 +
 tensorflow/core/kernels/nth_element_op.cc     | 139 ++++++++++++++
 tensorflow/core/kernels/nth_element_op.h      |  39 ++++
 tensorflow/core/ops/nn_ops.cc                 |  50 +++++
 tensorflow/core/ops/nn_ops_test.cc            |  24 +++
 tensorflow/python/kernel_tests/BUILD          |  15 ++
 .../kernel_tests/nth_element_op_test.py       | 174 ++++++++++++++++++
 tensorflow/python/ops/nn_grad.py              |  29 +++
 tensorflow/python/ops/nn_ops.py               |  28 +++
 10 files changed, 507 insertions(+)
 create mode 100644 tensorflow/core/kernels/nth_element_op.cc
 create mode 100644 tensorflow/core/kernels/nth_element_op.h
 create mode 100644 tensorflow/python/kernel_tests/nth_element_op_test.py

diff --git a/tensorflow/contrib/nn/__init__.py b/tensorflow/contrib/nn/__init__.py
index 7007e26bac..3bf795d19a 100644
--- a/tensorflow/contrib/nn/__init__.py
+++ b/tensorflow/contrib/nn/__init__.py
@@ -18,6 +18,7 @@
 @@deprecated_flipped_softmax_cross_entropy_with_logits
 @@deprecated_flipped_sparse_softmax_cross_entropy_with_logits
 @@deprecated_flipped_sigmoid_cross_entropy_with_logits
+@@nth_element
 @@rank_sampled_softmax_loss
 @@scaled_softplus
 """
@@ -31,6 +32,7 @@ from tensorflow.contrib.nn.python.ops.alpha_dropout import *
 from tensorflow.contrib.nn.python.ops.cross_entropy import *
 from tensorflow.contrib.nn.python.ops.sampling_ops import *
 from tensorflow.contrib.nn.python.ops.scaled_softplus import *
+from tensorflow.python.ops.nn_ops import nth_element
 # pylint: enable=unused-import,wildcard-import
 
 from tensorflow.python.util.all_util import remove_undocumented
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index b883be5d02..aba11c4c29 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -2994,6 +2994,7 @@ cc_library(
         ":in_topk_op",
         ":l2loss_op",
         ":lrn_op",
+        ":nth_element_op",
         ":relu_op",
         ":softmax_op",
         ":softplus_op",
@@ -3080,6 +3081,12 @@ tf_kernel_library(
     deps = NN_DEPS + if_cuda(["@cub_archive//:cub"]),
 )
 
+tf_kernel_library(
+    name = "nth_element_op",
+    prefix = "nth_element_op",
+    deps = NN_DEPS,
+)
+
 tf_kernel_library(
     name = "xent_op",
     prefix = "xent_op",
diff --git a/tensorflow/core/kernels/nth_element_op.cc b/tensorflow/core/kernels/nth_element_op.cc
new file mode 100644
index 0000000000..da825e408c
--- /dev/null
+++ b/tensorflow/core/kernels/nth_element_op.cc
@@ -0,0 +1,139 @@
+/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+// See docs in ../ops/nn_ops.cc.
+#include "tensorflow/core/kernels/nth_element_op.h"
+
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/framework/register_types.h"
+#include "tensorflow/core/framework/types.h"
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/platform/logging.h"
+#include "tensorflow/core/util/work_sharder.h"
+#include <vector>
+#include <algorithm>
+#include <iostream>
+
+namespace tensorflow {
+
+typedef Eigen::ThreadPoolDevice CPUDevice;
+
+template <typename Device, typename T>
+class NthElementOp : public OpKernel {
+ public:
+  explicit NthElementOp(OpKernelConstruction* context) : OpKernel(context) {
+    OP_REQUIRES_OK(context, context->GetAttr("reverse", &reverse_));
+  }
+
+  void Compute(OpKernelContext* context) override {
+    // The second args is N, which must be a positive scalar.
+    const auto& n_in = context->input(1);
+    OP_REQUIRES(context, TensorShapeUtils::IsScalar(n_in.shape()),
+                errors::InvalidArgument("N must be scalar, got shape ",
+                                        n_in.shape().DebugString()));
+    int n = n_in.scalar<int32>()();
+    OP_REQUIRES(context, n >= 0,
+                errors::InvalidArgument("Need n >= 0, got ", n));
+
+    // The first args is input tensor, which must have 1 dimension at least.
+    const Tensor& input_in = context->input(0);
+    const int num_dims = input_in.dims();
+    OP_REQUIRES(context, num_dims >= 1,
+                errors::InvalidArgument("Input must be >= 1-D, got shape ",
+                                        input_in.shape().DebugString()));
+    // The last dimension of input tensor must be greater than N.
+    OP_REQUIRES(context, input_in.dim_size(num_dims-1) > n,
+                errors::InvalidArgument("Input must have at least n+1 columns"));
+
+    // std::nth_element only support the nth-smallest selection.
+    if (reverse_) {
+      n = input_in.dim_size(num_dims - 1) - n - 1;
+    }
+
+    // Assume input_shape is [d1,d2,...dk], and output_shape is [d1,d2...dk-1].
+    TensorShape out_shape;
+    for (int i = 0; i < num_dims-1; ++i) {
+      out_shape.AddDim(input_in.dim_size(i));
+    }
+    Tensor* output_tensor = nullptr;
+    OP_REQUIRES_OK(context,
+                   context->allocate_output(0, out_shape, &output_tensor));
+
+    functor::NthElementFunctor<Device, T> nthElementFunc;
+    nthElementFunc(context, input_in, *output_tensor, n, reverse_);
+  }
+
+ private:
+  bool reverse_;
+};
+
+namespace functor {
+
+template <typename T>
+struct NthElementFunctor<CPUDevice, T> {
+  void operator() (OpKernelContext* context,
+                   const Tensor& input_tensor,
+                   Tensor& output_tensor,
+                   int n,
+                   bool reverse) {
+    const T* input = input_tensor.flat<T>().data();
+    T* output = output_tensor.flat<T>().data();
+
+    // Assume input_shape is [d1,d2,...dk], and output_shape is [d1,d2...dk-1],
+    // then num_rows = d1*d2...dk-1, last_dim = dk.
+    const int num_rows = output_tensor.NumElements();
+    const int last_dim = input_tensor.dim_size(input_tensor.dims()-1);
+
+    // Allocate each row to different shard.
+    auto SubNthElement = [&, input, output, last_dim, n](int start,
+                                                         int limit) {
+      // std::nth_element would rearrange the array, so we need a new buffer.
+      std::vector<T> buf(last_dim);
+
+      for (int b = start; b < limit; ++b) {
+        // Copy from one row of elements to buffer
+        const T* input_start = input + b * last_dim;
+        const T* input_end = input + (b+1) * last_dim;
+        std::copy(input_start, input_end, buf.begin());
+
+        std::nth_element(buf.begin(), buf.begin()+n, buf.end());
+        // The element placed in the nth position is exactly the element that
+        // would occur in this position if the range was fully sorted.
+        output[b] = buf[n];
+      }
+    };
+
+    auto worker_threads = *(context->device()->tensorflow_cpu_worker_threads());
+    // The average time complexity of partition-based nth_element (BFPRT) is O(n),
+    // althought the worst time complexity could be O(n^2).
+    // Here, 20 is a empirical factor of cost_per_unit.
+    Shard(worker_threads.num_threads, worker_threads.workers, num_rows,
+          20 * last_dim, SubNthElement);
+  }
+};
+
+}  // namespace functor
+
+
+#define REGISTER_NTHOP(T)                                           \
+  REGISTER_KERNEL_BUILDER(                                          \
+      Name("NthElement").Device(DEVICE_CPU).TypeConstraint<T>("T"), \
+      NthElementOp<CPUDevice, T>)
+
+TF_CALL_REAL_NUMBER_TYPES(REGISTER_NTHOP);
+#undef REGISTER_NTHOP
+
+}  // end namespace tensorflow
+
diff --git a/tensorflow/core/kernels/nth_element_op.h b/tensorflow/core/kernels/nth_element_op.h
new file mode 100644
index 0000000000..11a6c996b0
--- /dev/null
+++ b/tensorflow/core/kernels/nth_element_op.h
@@ -0,0 +1,39 @@
+/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_NTH_ELEMENT_OP_H_
+#define TENSORFLOW_NTH_ELEMENT_OP_H_
+
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/framework/tensor_types.h"
+#include "tensorflow/core/platform/types.h"
+
+namespace tensorflow {
+
+namespace functor {
+
+template <typename Device, typename T>
+struct NthElementFunctor {
+  void operator() (OpKernelContext* context,
+                   const Tensor& input_tensor,
+                   Tensor& output_tensor,
+                   int n);
+};
+
+}  // namespace functor
+
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_NTH_ELEMENT_OP_H_
diff --git a/tensorflow/core/ops/nn_ops.cc b/tensorflow/core/ops/nn_ops.cc
index 5efa55b496..1d26660a4b 100644
--- a/tensorflow/core/ops/nn_ops.cc
+++ b/tensorflow/core/ops/nn_ops.cc
@@ -2260,6 +2260,56 @@ indices: The indices of `values` within the last dimension of `input`.
 
 // --------------------------------------------------------------------------
 
+REGISTER_OP("NthElement")
+    .Input("input: T")
+    .Input("n: int32")
+    .Output("values: T")
+    .Attr("reverse: bool = false")
+    .Attr("T: realnumbertype")
+    .SetShapeFn([](InferenceContext* c) {
+      ShapeHandle input;
+      TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(0), 1, &input));
+
+      // Get the n value from input tensor, and make sure which is a scalar.
+      DimensionHandle n_dim;
+      TF_RETURN_IF_ERROR(c->MakeDimForScalarInput(1, &n_dim));
+
+      // The last dimension of input tensor must be greater than N.
+      DimensionHandle last_dim = c->Dim(input, -1);
+      if (c->ValueKnown(last_dim) && c->ValueKnown(n_dim) &&
+          c->Value(last_dim) <= c->Value(n_dim)) {
+        return errors::InvalidArgument(
+            "Input must have last dimension > n = ", c->Value(n_dim), " but is ",
+            c->Value(last_dim));
+      }
+
+      // Reduce last_dim for output tensor
+      ShapeHandle s;
+      TF_RETURN_IF_ERROR(c->Subshape(input, 0, -1, &s));
+      c->set_output(0, s);
+      return Status::OK();
+    })
+    .Doc(R"doc(
+Finds values of the `n`-th order statistic for the last dmension.
+
+If the input is a vector (rank-1), finds the entries which is the nth-smallest
+value in the vector and outputs their values as scalar tensor.
+
+For matrices (resp. higher rank input), computes the entries which is the
+nth-smallest value in each row (resp. vector along the last dimension). Thus,
+
+    values.shape = input.shape[:-1]
+
+input: 1-D or higher with last dimension at least `n+1`.
+n: 0-D. Position of sorted vector to select along the last dimension (along
+  each row for matrices). Valid range of n is `[0, input.shape[:-1])`
+reverse: When set to True, find the nth-largest value in the vector and vice
+  versa.
+values: The `n`-th order statistic along each last dimensional slice.
+)doc");
+
+// --------------------------------------------------------------------------
+
 REGISTER_OP("FractionalMaxPool")
     .Input("value: T")
     .Output("output: T")
diff --git a/tensorflow/core/ops/nn_ops_test.cc b/tensorflow/core/ops/nn_ops_test.cc
index 4628b725f8..94ecf4d5db 100644
--- a/tensorflow/core/ops/nn_ops_test.cc
+++ b/tensorflow/core/ops/nn_ops_test.cc
@@ -81,6 +81,30 @@ TEST(NNOpsTest, TopKV2_ShapeFn) {
       op, "[1,2,3,4];[]");
 }
 
+TEST(NNOpsTest, NthElement_ShapeFn) {
+  ShapeInferenceTestOp op("NthElement");
+  op.input_tensors.resize(2);
+
+  Tensor n_t;
+  op.input_tensors[1] = &n_t;
+  n_t = test::AsScalar<int32>(20);
+
+  INFER_OK(op, "?;[]", "?");
+  INFER_OK(op, "[21];[]", "[]");
+  INFER_OK(op, "[2,?,?];[]", "[d0_0,d0_1]");
+  INFER_OK(op, "[?,3,?,21];[]", "[d0_0,d0_1,d0_2]");
+
+  INFER_ERROR("Shape must be at least rank 1 but is rank 0", op, "[];[]");
+  INFER_ERROR("Input must have last dimension > n = 20 but is 1", op,
+              "[1];[]");
+  INFER_ERROR("Input must have last dimension > n = 20 but is 20", op,
+              "[1,2,3,20];[]");
+  n_t = test::AsScalar<int32>(-1);
+  INFER_ERROR(
+     "Dimension size, given by scalar input 1, must be non-negative but is -1",
+     op, "[1,2,3,4];[]");
+}
+
 TEST(NNOpsTest, BatchNormWithGlobalNormalization_ShapeFn) {
   ShapeInferenceTestOp op("BatchNormWithGlobalNormalization");
 
diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD
index 847c078971..127845e7d8 100644
--- a/tensorflow/python/kernel_tests/BUILD
+++ b/tensorflow/python/kernel_tests/BUILD
@@ -918,6 +918,21 @@ cuda_py_test(
     ],
 )
 
+cuda_py_test(
+    name = "nth_element_op_test",
+    size = "small",
+    srcs = ["nth_element_op_test.py"],
+    additional_deps = [
+        "//third_party/py/numpy",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework_for_generated_wrappers",
+        "//tensorflow/python:gradients",
+        "//tensorflow/python:nn_grad",
+        "//tensorflow/python:nn_ops",
+    ],
+)
+
 tf_py_test(
     name = "unique_op_test",
     size = "small",
diff --git a/tensorflow/python/kernel_tests/nth_element_op_test.py b/tensorflow/python/kernel_tests/nth_element_op_test.py
new file mode 100644
index 0000000000..58cd46d2d5
--- /dev/null
+++ b/tensorflow/python/kernel_tests/nth_element_op_test.py
@@ -0,0 +1,174 @@
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+import tensorflow.python.ops.nn_grad  # pylint: disable=unused-import
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import nn_ops
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import gradients_impl
+from tensorflow.python.platform import test
+
+
+class NthElementTest(test.TestCase):
+
+  def _validateNthElement(self, inputs, dtype, n, reverse, expected_values):
+    np_expected_values = np.array(expected_values)
+    with self.test_session(use_gpu=False) as sess:
+      inputs_op = ops.convert_to_tensor(inputs, dtype=dtype)
+      values_op = nn_ops.nth_element(inputs_op, n, reverse=reverse)
+      values = sess.run(values_op)
+
+      self.assertShapeEqual(np_expected_values, values_op)
+      self.assertAllClose(np_expected_values, values)
+
+  def testExample1(self):
+    inputs = [2.2, 4.4, 1.1, 5.5, 3.3]
+    self._validateNthElement(inputs, dtypes.float32, 1, False, 2.2)
+    self._validateNthElement(inputs, dtypes.float32, 1, True, 4.4)
+
+  def testExample2(self):
+    inputs = [[2.2, 4.4, 1.1], [5.5, 3.3, 6.6]]
+    self._validateNthElement(inputs, dtypes.float64, 2, False, [4.4, 6.6])
+    self._validateNthElement(inputs, dtypes.float64, 2, True, [1.1, 3.3])
+
+  def testExample3(self):
+    inputs = [[[2, 4, 1], [5, -3, 6]],
+              [[7, 9, -8], [9, 0, 4]]]
+    self._validateNthElement(inputs, dtypes.int32, 0, False,
+                             [[1, -3], [-8, 0]])
+    self._validateNthElement(inputs, dtypes.int64, 0, True,
+                             [[4, 6], [9, 9]])
+
+  def _testFloatLargeInput(self, input_shape):
+    inputs = np.random.random_sample(input_shape)
+    n = np.random.randint(input_shape[-1])
+    sort_inputs = np.sort(inputs)
+    expected_values = sort_inputs[..., n]
+    self._validateNthElement(
+        inputs, dtypes.float32, n, False, expected_values)
+    expected_values = sort_inputs[..., ::-1][..., n]
+    self._validateNthElement(
+        inputs, dtypes.float64, n, True, expected_values)
+
+  def _testIntLargeInput(self, input_shape):
+    inputs = np.random.randint(-1e3, 1e3, input_shape)
+    n = np.random.randint(input_shape[-1])
+    sort_inputs = np.sort(inputs)
+    expected_values = sort_inputs[..., n]
+    self._validateNthElement(
+        inputs, dtypes.int32, n, False, expected_values)
+    expected_values = sort_inputs[..., ::-1][..., n]
+    self._validateNthElement(
+        inputs, dtypes.int64, n, True, expected_values)
+
+  def _testLargeInput(self, input_shape):
+    self._testFloatLargeInput(input_shape)
+    self._testIntLargeInput(input_shape)
+
+  def testLargeInput(self):
+    self._testLargeInput([1])
+    self._testLargeInput([10])
+    self._testLargeInput([5, 10])
+    self._testLargeInput([50, 100])
+    self._testLargeInput([50, 10000])
+    self._testLargeInput([50, 10, 100])
+    self._testLargeInput([50, 10, 10, 100])
+
+  def _testEnumerateN(self, input_shape):
+    inputs = np.random.random_sample(input_shape)
+    sort_inputs = np.sort(inputs)
+    for n in range(input_shape[-1]):
+      expected_values = sort_inputs[..., n]
+      self._validateNthElement(
+          inputs, dtypes.float32, n, False, expected_values)
+      expected_values = sort_inputs[..., ::-1][..., n]
+      self._validateNthElement(
+          inputs, dtypes.float64, n, True, expected_values)
+
+  def testEnumerateN(self):
+    self._testEnumerateN([1])
+    self._testEnumerateN([10])
+    self._testEnumerateN([10, 10])
+    self._testEnumerateN([10, 10, 10])
+    self._testEnumerateN([10, 10, 10, 10])
+
+  def testInvalidInput(self):
+    with self.assertRaisesRegexp(ValueError,
+                                 "at least rank 1 but is rank 0"):
+      nn_ops.nth_element(5, 0)
+
+  def testInvalidInputAtEval(self):
+    with self.test_session(use_gpu=False):
+      v = array_ops.placeholder(dtype=dtypes.float32)
+      with self.assertRaisesOpError("Input must be >= 1-D"):
+        nn_ops.nth_element(v, 0).eval(feed_dict={v: 5.0})
+
+  def testInvalidN(self):
+    with self.assertRaisesRegexp(ValueError,
+                                 "non-negative but is -1"):
+      nn_ops.nth_element([5], -1)
+    with self.assertRaisesRegexp(ValueError,
+                                 "scalar but has rank 1"):
+      nn_ops.nth_element([5, 6, 3], [1])
+
+  def testInvalidNAtEval(self):
+    inputs = [[0.1, 0.2], [0.3, 0.4]]
+    with self.test_session(use_gpu=False):
+      n = array_ops.placeholder(dtypes.int32)
+      values = nn_ops.nth_element(inputs, n)
+      with self.assertRaisesOpError("Need n >= 0, got -7"):
+        values.eval(feed_dict={n: -7})
+
+  def testNTooLarge(self):
+    inputs = [[0.1, 0.2], [0.3, 0.4]]
+    with self.assertRaisesRegexp(ValueError,
+                                 "must have last dimension > n = 2"):
+      nn_ops.nth_element(inputs, 2)
+
+  def testNTooLargeAtEval(self):
+    inputs = [[0.1, 0.2], [0.3, 0.4]]
+    with self.test_session(use_gpu=False):
+      n = array_ops.placeholder(dtypes.int32)
+      values = nn_ops.nth_element(inputs, n)
+      with self.assertRaisesOpError(r"Input must have at least n\+1 columns"):
+        values.eval(feed_dict={n: 2})
+
+  def testGradients(self):
+    with self.test_session(use_gpu=False) as sess:
+      inputs = array_ops.placeholder(dtypes.int32, shape=[3, 5])
+      values = nn_ops.nth_element(inputs, 3)
+      grad = sess.run(
+          gradients_impl.gradients(
+              values, inputs, grad_ys=[[-1., 2., 5.]]),
+          feed_dict={inputs: [[2, -1, 1000, 3, 1000],
+                              [1, 5, 2, 4, 3],
+                              [2, 2, 2, 2, 2],
+                             ]})
+    self.assertAllClose(grad[0], [[0, 0, -0.5, 0, -0.5],
+                                  [0, 0, 0, 2, 0],
+                                  [1, 1, 1, 1, 1],
+                                 ])
+
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/python/ops/nn_grad.py b/tensorflow/python/ops/nn_grad.py
index af610d8fdb..c7c745142b 100644
--- a/tensorflow/python/ops/nn_grad.py
+++ b/tensorflow/python/ops/nn_grad.py
@@ -934,3 +934,32 @@ def _TopKGrad(op, grad, _):
                                  validate_indices=False),
       in_shape), array_ops.zeros(
           [], dtype=dtypes.int32)]
+
+
+@ops.RegisterGradient("NthElement")
+def _NthElementGrad(op, grad):
+  """Return the gradients for NthElement.
+
+  Args:
+    op: The NthElementOp for which we need to generate gradients.
+    grad: Tensor. The gradients passed to the NthElementOp
+
+  Returns:
+    A list of two tensors, the first being the gradient w.r.t. the input,
+    the second being the gradient w.r.t. the N (None).
+  """
+  input = op.inputs[0]
+  output = op.outputs[0]
+
+  # Compute the number of elements which equal to output in each reduction
+  # dimension. If there are multiple elements then the gradient will be
+  # divided between them.
+  indicators = math_ops.cast(
+      math_ops.equal(array_ops.expand_dims(output, -1), input),
+      grad.dtype)
+
+  grad = array_ops.expand_dims(grad, -1)
+  num_selected = array_ops.expand_dims(
+      math_ops.reduce_sum(indicators, -1), -1)
+
+  return [math_ops.div(indicators, num_selected) * grad, None]
diff --git a/tensorflow/python/ops/nn_ops.py b/tensorflow/python/ops/nn_ops.py
index 8876591e53..8741b37c6f 100644
--- a/tensorflow/python/ops/nn_ops.py
+++ b/tensorflow/python/ops/nn_ops.py
@@ -2140,6 +2140,34 @@ def top_k(input, k=1, sorted=True, name=None):
   return gen_nn_ops._top_kv2(input, k=k, sorted=sorted, name=name)
 
 
+def nth_element(input, n, reverse=False, name=None):
+  r"""Finds values of the `n`-th order statistic for the last dmension.
+
+  If the input is a vector (rank-1), finds the entries which is the nth-smallest
+  value in the vector and outputs their values as scalar tensor.
+
+  For matrices (resp. higher rank input), computes the entries which is the
+  nth-smallest value in each row (resp. vector along the last dimension). Thus,
+
+      values.shape = input.shape[:-1]
+
+  Args:
+    input: 1-D or higher `Tensor` with last dimension at least `n+1`.
+    n: A `Tensor` of type `int32`.
+      0-D. Position of sorted vector to select along the last dimension (along
+      each row for matrices). Valid range of n is `[0, input.shape[:-1])`
+    reverse: An optional `bool`. Defaults to `False`.
+      When set to True, find the nth-largest value in the vector and vice
+      versa.
+    name: A name for the operation (optional).
+
+  Returns:
+    A `Tensor`. Has the same type as `input`.
+    The `n`-th order statistic along each last dimensional slice.
+  """
+  return gen_nn_ops.nth_element(input, n, reverse=reverse, name=name)
+
+
 def conv1d(value, filters, stride, padding,
            use_cudnn_on_gpu=None, data_format=None,
            name=None):
-- 
GitLab


From c27a2ecb57c6059235d4ee5e14697f95d1c21235 Mon Sep 17 00:00:00 2001
From: codrut3 <grosu.codrut@gmail.com>
Date: Wed, 18 Oct 2017 21:23:02 +0300
Subject: [PATCH 512/909] Allow num parameter in tf.linspace to be int64.
 (#13755)

* Allow num parameter in tf.linspace to be int64.

Currently tf.linspace is defined for num int32 or int64.
However the kernel only allows int32, even though the op in
core/ops/math_ops permits int64 too.
I slightly changed the kernel to allow int64 too.
I also added tests for RangeOp and LinSpaceOp.

* Change variable names.
---
 tensorflow/core/kernels/BUILD                |  18 +++
 tensorflow/core/kernels/sequence_ops.cc      |  48 +++---
 tensorflow/core/kernels/sequence_ops_test.cc | 148 +++++++++++++++++++
 3 files changed, 196 insertions(+), 18 deletions(-)
 create mode 100644 tensorflow/core/kernels/sequence_ops_test.cc

diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index aba11c4c29..3a06189d72 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -2636,6 +2636,24 @@ tf_kernel_library(
     deps = MATH_DEPS,
 )
 
+tf_cc_test(
+    name = "sequence_ops_test",
+    size = "small",
+    srcs = ["sequence_ops_test.cc"],
+    deps = [
+        ":ops_testutil",
+        ":ops_util",
+        ":sequence_ops",
+        "//tensorflow/core:core_cpu",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+        "//tensorflow/core:testlib",
+    ],
+)
+
 tf_cuda_cc_test(
     name = "cast_op_test",
     size = "small",
diff --git a/tensorflow/core/kernels/sequence_ops.cc b/tensorflow/core/kernels/sequence_ops.cc
index c8ea923020..e2e3758d87 100644
--- a/tensorflow/core/kernels/sequence_ops.cc
+++ b/tensorflow/core/kernels/sequence_ops.cc
@@ -96,7 +96,7 @@ TF_CALL_double(REGISTER_SYCL_KERNEL);
 TF_CALL_int32(REGISTER_SYCL_KERNEL);
 TF_CALL_int64(REGISTER_SYCL_KERNEL);
 #undef REGISTER_SYCL_KERNEL
-#endif // TENSORFLOW_USE_SYCL
+#endif  // TENSORFLOW_USE_SYCL
 
 TF_CALL_float(REGISTER_CPU_KERNEL);
 TF_CALL_double(REGISTER_CPU_KERNEL);
@@ -116,7 +116,7 @@ TF_CALL_int64(REGISTER_GPU_KERNEL);
 #undef REGISTER_CPU_KERNEL
 #undef REGISTER_GPU_KERNEL
 
-template <typename T>
+template <typename T, typename Tnum>
 class LinSpaceOp : public OpKernel {
  public:
   explicit LinSpaceOp(OpKernelConstruction* context) : OpKernel(context) {}
@@ -136,7 +136,7 @@ class LinSpaceOp : public OpKernel {
                                         num_in.shape().DebugString()));
     const T start = start_in.scalar<T>()();
     const T stop = stop_in.scalar<T>()();
-    const int32 num = num_in.scalar<int32>()();
+    const Tnum num = num_in.scalar<Tnum>()();
     OP_REQUIRES(context, num > 0,
                 errors::InvalidArgument("Requires num > 0: ", num));
     Tensor* out = nullptr;
@@ -147,34 +147,46 @@ class LinSpaceOp : public OpKernel {
       flat(0) = start;
     } else {
       const T step = (stop - start) / (num - 1);
-      for (int32 i = 0; i < num; ++i) flat(i) = start + step * i;
+      for (Tnum i = 0; i < num; ++i) flat(i) = start + step * i;
     }
   }
 };
 
-#define REGISTER_KERNEL(DEV, T)                              \
-  REGISTER_KERNEL_BUILDER(Name("LinSpace")                   \
-                              .Device(DEV)                   \
-                              .TypeConstraint<T>("T")        \
-                              .TypeConstraint<int32>("Tidx") \
-                              .HostMemory("start")           \
-                              .HostMemory("stop")            \
-                              .HostMemory("num")             \
-                              .HostMemory("output"),         \
-                          LinSpaceOp<T>);
-#define REGISTER_CPU_KERNEL(T) REGISTER_KERNEL(DEVICE_CPU, T)
+#define REGISTER_KERNEL(DEV, T, Tidx)                       \
+  REGISTER_KERNEL_BUILDER(Name("LinSpace")                  \
+                              .Device(DEV)                  \
+                              .TypeConstraint<T>("T")       \
+                              .TypeConstraint<Tidx>("Tidx") \
+                              .HostMemory("start")          \
+                              .HostMemory("stop")           \
+                              .HostMemory("num")            \
+                              .HostMemory("output"),        \
+                          LinSpaceOp<T, Tidx>);
+
+#define REGISTER_KERNEL_ALL_NUMS(dev, T) \
+  REGISTER_KERNEL(dev, T, int32);        \
+  REGISTER_KERNEL(dev, T, int64)
+
+#define REGISTER_CPU_KERNEL(T) REGISTER_KERNEL_ALL_NUMS(DEVICE_CPU, T)
 TF_CALL_float(REGISTER_CPU_KERNEL);
 TF_CALL_double(REGISTER_CPU_KERNEL);
 
 // NOTE(touts): We register the op on GPU but it still runs on CPU
 // because its inputs and outputs are tagged as HostMemory.
-#define REGISTER_GPU_KERNEL(T) REGISTER_KERNEL(DEVICE_GPU, T)
+#define REGISTER_GPU_KERNEL(T) REGISTER_KERNEL_ALL_NUMS(DEVICE_GPU, T)
 TF_CALL_float(REGISTER_GPU_KERNEL);
 TF_CALL_double(REGISTER_GPU_KERNEL);
+#undef REGISTER_GPU_KERNEL
 
 #ifdef TENSORFLOW_USE_SYCL
-#define REGISTER_SYCL_KERNEL(T) REGISTER_KERNEL(DEVICE_SYCL, T)
+#define REGISTER_SYCL_KERNEL(T) REGISTER_KERNEL_ALL_NUMS(DEVICE_SYCL, T)
 TF_CALL_float(REGISTER_SYCL_KERNEL);
 TF_CALL_double(REGISTER_SYCL_KERNEL);
-#endif // TENSORFLOW_USE_SYCL
+#undef REGISTER_SYCL_KERNEL
+#endif  // TENSORFLOW_USE_SYCL
+
+#undef REGISTER_CPU_KERNEL
+#undef REGISTER_KERNEL_ALL_NUMS
+#undef REGISTER_KERNEL
+
 }  // namespace tensorflow
diff --git a/tensorflow/core/kernels/sequence_ops_test.cc b/tensorflow/core/kernels/sequence_ops_test.cc
new file mode 100644
index 0000000000..5f0e0a69a8
--- /dev/null
+++ b/tensorflow/core/kernels/sequence_ops_test.cc
@@ -0,0 +1,148 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/framework/allocator.h"
+#include "tensorflow/core/framework/fake_input.h"
+#include "tensorflow/core/framework/node_def_builder.h"
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/framework/tensor_testutil.h"
+#include "tensorflow/core/framework/types.h"
+#include "tensorflow/core/framework/types.pb.h"
+#include "tensorflow/core/kernels/ops_testutil.h"
+#include "tensorflow/core/kernels/ops_util.h"
+#include "tensorflow/core/platform/test.h"
+
+namespace tensorflow {
+namespace {
+
+class RangeOpTest : public OpsTestBase {
+ protected:
+  void MakeOp(DataType input_type) {
+    TF_ASSERT_OK(NodeDefBuilder("myop", "Range")
+                     .Input(FakeInput(input_type))
+                     .Input(FakeInput(input_type))
+                     .Input(FakeInput(input_type))
+                     .Finalize(node_def()));
+    TF_ASSERT_OK(InitOp());
+  }
+};
+
+class LinSpaceOpTest : public OpsTestBase {
+ protected:
+  void MakeOp(DataType input_type, DataType index_type) {
+    TF_ASSERT_OK(NodeDefBuilder("myop", "LinSpace")
+                     .Input(FakeInput(input_type))
+                     .Input(FakeInput(input_type))
+                     .Input(FakeInput(index_type))
+                     .Finalize(node_def()));
+    TF_ASSERT_OK(InitOp());
+  }
+};
+
+TEST_F(RangeOpTest, Simple_D32) {
+  MakeOp(DT_INT32);
+
+  // Feed and run
+  AddInputFromArray<int32>(TensorShape({}), {0});
+  AddInputFromArray<int32>(TensorShape({}), {10});
+  AddInputFromArray<int32>(TensorShape({}), {2});
+  TF_ASSERT_OK(RunOpKernel());
+
+  // Check the output
+  Tensor expected(allocator(), DT_INT32, TensorShape({5}));
+  test::FillValues<int32>(&expected, {0, 2, 4, 6, 8});
+  test::ExpectTensorEqual<int32>(expected, *GetOutput(0));
+}
+
+TEST_F(RangeOpTest, Simple_Float) {
+  MakeOp(DT_FLOAT);
+
+  // Feed and run
+  AddInputFromArray<float>(TensorShape({}), {0.5});
+  AddInputFromArray<float>(TensorShape({}), {2});
+  AddInputFromArray<float>(TensorShape({}), {0.3});
+  TF_ASSERT_OK(RunOpKernel());
+
+  // Check the output
+  Tensor expected(allocator(), DT_FLOAT, TensorShape({5}));
+  test::FillValues<float>(&expected, {0.5, 0.8, 1.1, 1.4, 1.7});
+  test::ExpectTensorEqual<float>(expected, *GetOutput(0));
+}
+
+TEST_F(RangeOpTest, Large_Double) {
+  MakeOp(DT_DOUBLE);
+
+  // Feed and run
+  AddInputFromArray<double>(TensorShape({}), {0.0});
+  AddInputFromArray<double>(TensorShape({}), {10000});
+  AddInputFromArray<double>(TensorShape({}), {0.5});
+  TF_ASSERT_OK(RunOpKernel());
+
+  // Check the output
+  Tensor expected(allocator(), DT_DOUBLE, TensorShape({20000}));
+  std::vector<double> result;
+  for (int32 i = 0; i < 20000; ++i) result.push_back(i * 0.5);
+  test::FillValues<double>(&expected, gtl::ArraySlice<double>(result));
+  test::ExpectTensorEqual<double>(expected, *GetOutput(0));
+}
+
+TEST_F(LinSpaceOpTest, Simple_D32) {
+  MakeOp(DT_FLOAT, DT_INT32);
+
+  // Feed and run
+  AddInputFromArray<float>(TensorShape({}), {3.0});
+  AddInputFromArray<float>(TensorShape({}), {7.0});
+  AddInputFromArray<int32>(TensorShape({}), {3});
+  TF_ASSERT_OK(RunOpKernel());
+
+  // Check the output
+  Tensor expected(allocator(), DT_FLOAT, TensorShape({3}));
+  test::FillValues<float>(&expected, {3.0, 5.0, 7.0});
+  test::ExpectTensorEqual<float>(expected, *GetOutput(0));
+}
+
+TEST_F(LinSpaceOpTest, Single_D64) {
+  MakeOp(DT_FLOAT, DT_INT64);
+
+  // Feed and run
+  AddInputFromArray<float>(TensorShape({}), {9.0});
+  AddInputFromArray<float>(TensorShape({}), {100.0});
+  AddInputFromArray<int64>(TensorShape({}), {1});
+  TF_ASSERT_OK(RunOpKernel());
+
+  // Check the output
+  Tensor expected(allocator(), DT_FLOAT, TensorShape({1}));
+  test::FillValues<float>(&expected, {9.0});
+  test::ExpectTensorEqual<float>(expected, *GetOutput(0));
+}
+
+TEST_F(LinSpaceOpTest, Simple_Double) {
+  MakeOp(DT_DOUBLE, DT_INT32);
+
+  // Feed and run
+  AddInputFromArray<double>(TensorShape({}), {5.0});
+  AddInputFromArray<double>(TensorShape({}), {6.0});
+  AddInputFromArray<int32>(TensorShape({}), {6});
+  TF_ASSERT_OK(RunOpKernel());
+
+  // Check the output
+  Tensor expected(allocator(), DT_DOUBLE, TensorShape({6}));
+  test::FillValues<double>(&expected, {5.0, 5.2, 5.4, 5.6, 5.8, 6.0});
+  test::ExpectTensorEqual<double>(expected, *GetOutput(0));
+}
+
+}  // namespace
+}  // namespace tensorflow
-- 
GitLab


From 21f68a85f30ddea7e4a66dbe70d18069a2a1d0a1 Mon Sep 17 00:00:00 2001
From: Mustafa Ispir <ispir@google.com>
Date: Wed, 18 Oct 2017 11:27:38 -0700
Subject: [PATCH 513/909] Remove global step read dependency from model_fn.
 Estimator behavior still will be deterministic since the step checking logic
 in session_run_hooks was changed as follows: * assume stale step * before
 using the step, check for the current value by session.run

PiperOrigin-RevId: 172629797
---
 .../contrib/learn/python/learn/estimators/estimator.py     | 5 ++---
 tensorflow/python/estimator/estimator.py                   | 7 +++----
 2 files changed, 5 insertions(+), 7 deletions(-)

diff --git a/tensorflow/contrib/learn/python/learn/estimators/estimator.py b/tensorflow/contrib/learn/python/learn/estimators/estimator.py
index 8bb1c83a45..788d2d0b1a 100644
--- a/tensorflow/contrib/learn/python/learn/estimators/estimator.py
+++ b/tensorflow/contrib/learn/python/learn/estimators/estimator.py
@@ -981,9 +981,8 @@ class BaseEstimator(
       global_step = training_util.create_global_step(g)
       features, labels = input_fn()
       self._check_inputs(features, labels)
-      global_step_read_tensor = training_util._get_or_create_global_step_read()  # pylint: disable=protected-access
-      with ops.control_dependencies([global_step_read_tensor]):
-        model_fn_ops = self._get_train_ops(features, labels)
+      training_util._get_or_create_global_step_read()  # pylint: disable=protected-access
+      model_fn_ops = self._get_train_ops(features, labels)
       ops.add_to_collection(ops.GraphKeys.LOSSES, model_fn_ops.loss)
       all_hooks.extend(hooks)
       all_hooks.extend([
diff --git a/tensorflow/python/estimator/estimator.py b/tensorflow/python/estimator/estimator.py
index 00a57f11dc..2a4d77b1a6 100644
--- a/tensorflow/python/estimator/estimator.py
+++ b/tensorflow/python/estimator/estimator.py
@@ -707,12 +707,11 @@ class Estimator(object):
     with ops.Graph().as_default() as g, g.device(self._device_fn):
       random_seed.set_random_seed(self._config.tf_random_seed)
       global_step_tensor = self._create_and_assert_global_step(g)
-      global_step_read_tensor = training_util._get_or_create_global_step_read()  # pylint: disable=protected-access
+      training_util._get_or_create_global_step_read()  # pylint: disable=protected-access
       features, labels = self._get_features_and_labels_from_input_fn(
           input_fn, model_fn_lib.ModeKeys.TRAIN)
-      with ops.control_dependencies([global_step_read_tensor]):
-        estimator_spec = self._call_model_fn(
-            features, labels, model_fn_lib.ModeKeys.TRAIN, self.config)
+      estimator_spec = self._call_model_fn(
+          features, labels, model_fn_lib.ModeKeys.TRAIN, self.config)
       # Check if the user created a loss summary, and add one if they didn't.
       # We assume here that the summary is called 'loss'. If it is not, we will
       # make another one with the name 'loss' to ensure it shows up in the right
-- 
GitLab


From 1a325f1330cd6e1204d47f1fcf64caf402788477 Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Wed, 18 Oct 2017 11:35:12 -0700
Subject: [PATCH 514/909] More changs to avoid flakes in
 random_shuffle_queue_test

PiperOrigin-RevId: 172630989
---
 .../kernel_tests/random_shuffle_queue_test.py | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/tensorflow/python/kernel_tests/random_shuffle_queue_test.py b/tensorflow/python/kernel_tests/random_shuffle_queue_test.py
index 1b84af6823..c4e16ff628 100644
--- a/tensorflow/python/kernel_tests/random_shuffle_queue_test.py
+++ b/tensorflow/python/kernel_tests/random_shuffle_queue_test.py
@@ -654,7 +654,8 @@ class RandomShuffleQueueTest(test.TestCase):
 
   def testBlockingDequeueFromClosedQueue(self):
     with self.test_session() as sess:
-      q = data_flow_ops.RandomShuffleQueue(10, 2, dtypes_lib.float32)
+      min_size = 2
+      q = data_flow_ops.RandomShuffleQueue(10, min_size, dtypes_lib.float32)
       elems = [10.0, 20.0, 30.0, 40.0]
       enqueue_op = q.enqueue_many((elems,))
       close_op = q.close()
@@ -664,20 +665,24 @@ class RandomShuffleQueueTest(test.TestCase):
 
       results = []
 
-      def dequeue():
-        for _ in elems:
-          results.append(sess.run(dequeued_t))
+      # Manually dequeue until we hit min_size.
+      results.append(sess.run(dequeued_t))
+      results.append(sess.run(dequeued_t))
+
+      def blocking_dequeue():
+        results.append(sess.run(dequeued_t))
+        results.append(sess.run(dequeued_t))
+
         self.assertItemsEqual(elems, results)
         # Expect the operation to fail due to the queue being closed.
         with self.assertRaisesRegexp(errors_impl.OutOfRangeError,
                                      "is closed and has insufficient"):
           sess.run(dequeued_t)
 
-      dequeue_thread = self.checkedThread(target=dequeue)
+      dequeue_thread = self.checkedThread(target=blocking_dequeue)
       dequeue_thread.start()
-      # The close_op should run after the dequeue_thread has blocked.
-      # TODO(mrry): Figure out how to do this without sleeping.
       time.sleep(0.1)
+
       # The dequeue thread blocked when it hit the min_size requirement.
       self.assertEqual(len(results), 2)
       close_op.run()
-- 
GitLab


From 6a725f6d0974dc71fe4ac311fc8dd16db4257452 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 18 Oct 2017 11:57:04 -0700
Subject: [PATCH 515/909] Add expected keys to predictor exception if
 unexpected key detected.

PiperOrigin-RevId: 172634275
---
 tensorflow/contrib/predictor/predictor.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/contrib/predictor/predictor.py b/tensorflow/contrib/predictor/predictor.py
index dbc0028259..28fa815684 100644
--- a/tensorflow/contrib/predictor/predictor.py
+++ b/tensorflow/contrib/predictor/predictor.py
@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-
 """Abstract base class for all predictors."""
 
 from __future__ import absolute_import
@@ -66,8 +65,9 @@ class Predictor(object):
     expected_keys = set(self.feed_tensors.keys())
     unexpected_keys = input_keys - expected_keys
     if unexpected_keys:
-      raise ValueError('Got unexpected keys in input_dict: {}'.format(
-          unexpected_keys))
+      raise ValueError(
+          'Got unexpected keys in input_dict: {}\nexpected: {}'.format(
+              unexpected_keys, expected_keys))
 
     feed_dict = {}
     for key in self.feed_tensors.keys():
-- 
GitLab


From f5d3bf42b892ecfbde2ce9eb45f00b76473c824a Mon Sep 17 00:00:00 2001
From: Skye Wanderman-Milne <skyewm@google.com>
Date: Wed, 18 Oct 2017 11:58:01 -0700
Subject: [PATCH 516/909] Add TF_GraphGetOpDef() to C API and use in
 Operation.op_def()

Note that this creates a small change in behavior with the C API
enabled, since previously not all Python Operations had an OpDef
(op_def() returns None). With the C API enabled, op_def() always
returns an OpDef.

PiperOrigin-RevId: 172634411
---
 tensorflow/c/c_api.cc                   | 11 +++++++++
 tensorflow/c/c_api.h                    |  7 ++++++
 tensorflow/c/c_api_function_test.cc     | 21 +++++++++++++++++
 tensorflow/c/c_api_test.cc              | 31 +++++++++++++++++++++++++
 tensorflow/python/framework/ops.py      | 15 +++++++++++-
 tensorflow/python/framework/ops_test.py | 15 ++++++++++++
 6 files changed, 99 insertions(+), 1 deletion(-)

diff --git a/tensorflow/c/c_api.cc b/tensorflow/c/c_api.cc
index 79fbd8c90c..cd98393e0a 100644
--- a/tensorflow/c/c_api.cc
+++ b/tensorflow/c/c_api.cc
@@ -1799,6 +1799,17 @@ void TF_GraphToGraphDef(TF_Graph* graph, TF_Buffer* output_graph_def,
   status->status = MessageToBuffer(def, output_graph_def);
 }
 
+void TF_GraphGetOpDef(TF_Graph* graph, const char* op_name,
+                      TF_Buffer* output_op_def, TF_Status* status) {
+  const OpDef* op_def;
+  {
+    mutex_lock l(graph->mu);
+    status->status = graph->graph.op_registry()->LookUpOpDef(op_name, &op_def);
+    if (!status->status.ok()) return;
+  }
+  status->status = MessageToBuffer(*op_def, output_op_def);
+}
+
 TF_ImportGraphDefOptions* TF_NewImportGraphDefOptions() {
   return new TF_ImportGraphDefOptions;
 }
diff --git a/tensorflow/c/c_api.h b/tensorflow/c/c_api.h
index 68a758498d..0c6bb53d01 100644
--- a/tensorflow/c/c_api.h
+++ b/tensorflow/c/c_api.h
@@ -864,6 +864,13 @@ TF_CAPI_EXPORT extern void TF_GraphToGraphDef(TF_Graph* graph,
                                               TF_Buffer* output_graph_def,
                                               TF_Status* status);
 
+// Returns the serialized OpDef proto with name `op_name`, or a bad status if no
+// such op exists. This can return OpDefs of functions copied into the graph.
+TF_CAPI_EXPORT extern void TF_GraphGetOpDef(TF_Graph* graph,
+                                            const char* op_name,
+                                            TF_Buffer* output_op_def,
+                                            TF_Status* status);
+
 // TF_ImportGraphDefOptions holds options that can be passed to
 // TF_GraphImportGraphDef.
 typedef struct TF_ImportGraphDefOptions TF_ImportGraphDefOptions;
diff --git a/tensorflow/c/c_api_function_test.cc b/tensorflow/c/c_api_function_test.cc
index 4db9a90fdc..d5580b6589 100644
--- a/tensorflow/c/c_api_function_test.cc
+++ b/tensorflow/c/c_api_function_test.cc
@@ -1465,5 +1465,26 @@ TEST_F(CApiFunctionTest, AppendHash) {
   ASSERT_EQ(string("func_name_base_qaJ8jA8UmGY"), fdef.signature().name());
 }
 
+TEST_F(CApiFunctionTest, GetOpDef) {
+  DefineFunction(func_name_, &func_);
+  TF_GraphCopyFunction(host_graph_, func_, nullptr, s_);
+  ASSERT_EQ(TF_OK, TF_GetCode(s_)) << TF_Message(s_);
+
+  // Test we can retrieve function OpDef from graph
+  TF_Buffer* buffer = TF_NewBuffer();
+  TF_GraphGetOpDef(host_graph_, func_name_, buffer, s_);
+  ASSERT_EQ(TF_OK, TF_GetCode(s_)) << TF_Message(s_);
+
+  // Sanity check returned OpDef
+  string data(static_cast<const char*>(buffer->data), buffer->length);
+  OpDef op_def;
+  op_def.ParseFromString(data);
+  EXPECT_EQ(op_def.name(), func_name_);
+  EXPECT_EQ(op_def.input_arg_size(), 1);
+  EXPECT_EQ(op_def.output_arg_size(), 1);
+
+  TF_DeleteBuffer(buffer);
+}
+
 }  // namespace
 }  // namespace tensorflow
diff --git a/tensorflow/c/c_api_test.cc b/tensorflow/c/c_api_test.cc
index c442029009..d220bc5e95 100644
--- a/tensorflow/c/c_api_test.cc
+++ b/tensorflow/c/c_api_test.cc
@@ -37,6 +37,7 @@ limitations under the License.
 #include "tensorflow/core/framework/types.pb.h"
 #include "tensorflow/core/graph/tensor_id.h"
 #include "tensorflow/core/lib/core/error_codes.pb.h"
+#include "tensorflow/core/lib/core/status_test_util.h"
 #include "tensorflow/core/lib/io/path.h"
 #include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/lib/strings/strcat.h"
@@ -50,6 +51,11 @@ Status TF_TensorToTensor(const TF_Tensor* src, Tensor* dst);
 
 namespace {
 
+static void ExpectHasSubstr(StringPiece s, StringPiece expected) {
+  EXPECT_TRUE(StringPiece(s).contains(expected))
+      << "'" << s << "' does not contain '" << expected << "'";
+}
+
 TEST(CAPI, Version) { EXPECT_STRNE("", TF_Version()); }
 
 TEST(CAPI, Status) {
@@ -837,6 +843,31 @@ TEST(CAPI, ShapeInferenceError) {
   TF_DeleteStatus(status);
 }
 
+TEST(CAPI, GetOpDef) {
+  TF_Status* status = TF_NewStatus();
+  TF_Graph* graph = TF_NewGraph();
+  TF_Buffer* buffer = TF_NewBuffer();
+
+  TF_GraphGetOpDef(graph, "Add", buffer, status);
+  ASSERT_EQ(TF_OK, TF_GetCode(status));
+  const OpDef* expected_op_def;
+  TF_ASSERT_OK(OpRegistry::Global()->LookUpOpDef("Add", &expected_op_def));
+  string expected_serialized;
+  expected_op_def->SerializeToString(&expected_serialized);
+  string actual_string(reinterpret_cast<const char*>(buffer->data),
+                       buffer->length);
+  EXPECT_EQ(expected_serialized, actual_string);
+
+  TF_GraphGetOpDef(graph, "MyFakeOp", buffer, status);
+  EXPECT_EQ(TF_NOT_FOUND, TF_GetCode(status));
+  ExpectHasSubstr(TF_Message(status),
+                  "Op type not registered 'MyFakeOp' in binary");
+
+  TF_DeleteBuffer(buffer);
+  TF_DeleteGraph(graph);
+  TF_DeleteStatus(status);
+}
+
 void StringVectorToArrays(const std::vector<string>& v,
                           std::unique_ptr<const void* []>* ptrs,
                           std::unique_ptr<size_t[]>* lens) {
diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index 3ac8a0cb6a..75750ecd5a 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -33,6 +33,7 @@ from tensorflow.core.framework import attr_value_pb2
 from tensorflow.core.framework import function_pb2
 from tensorflow.core.framework import graph_pb2
 from tensorflow.core.framework import node_def_pb2
+from tensorflow.core.framework import op_def_pb2
 from tensorflow.core.framework import versions_pb2
 from tensorflow.python import pywrap_tensorflow as c_api
 from tensorflow.python.eager import context
@@ -1985,7 +1986,19 @@ class Operation(object):
       protocol buffer.
     """
     # pylint: enable=line-too-long
-    return self._op_def
+    if self._c_op:
+      with errors.raise_exception_on_not_ok_status() as status:
+        with c_api_util.tf_buffer() as buf:
+          # pylint: disable=protected-access
+          c_api.TF_GraphGetOpDef(self._graph._c_graph,
+                                 compat.as_bytes(self.type), buf, status)
+          # pylint: enable=protected-access
+          data = c_api.TF_GetBuffer(buf)
+      op_def = op_def_pb2.OpDef()
+      op_def.ParseFromString(compat.as_bytes(data))
+      return op_def
+    else:
+      return self._op_def
 
   @property
   def traceback(self):
diff --git a/tensorflow/python/framework/ops_test.py b/tensorflow/python/framework/ops_test.py
index f20c808cde..59c0288457 100644
--- a/tensorflow/python/framework/ops_test.py
+++ b/tensorflow/python/framework/ops_test.py
@@ -504,6 +504,21 @@ class OperationTest(test_util.TensorFlowTestCase):
                                  r"num of inputs: 0\) does not have input 1"):
       x.op._update_input(1, x)  # pylint: disable=protected-access
 
+  def testOpDef(self):
+    x = constant_op.constant(0)
+    y = constant_op.constant(1)
+    z = x + y
+
+    # Pure Python mode doesn't create OpDefs for constants
+    if ops._USE_C_API:
+      self.assertEqual(x.op.op_def.name, "Const")
+      self.assertEqual(len(x.op.op_def.input_arg), 0)
+      self.assertEqual(len(x.op.op_def.output_arg), 1)
+
+    self.assertEqual(z.op.op_def.name, "Add")
+    self.assertEqual(len(z.op.op_def.input_arg), 2)
+    self.assertEqual(len(z.op.op_def.output_arg), 1)
+
 
 @test_util.with_c_api
 class CreateOpTest(test_util.TensorFlowTestCase):
-- 
GitLab


From f5ea388e48a38b935ebd36442f756c8974b7ce3f Mon Sep 17 00:00:00 2001
From: Saurabh Saxena <srbs@google.com>
Date: Wed, 18 Oct 2017 11:58:24 -0700
Subject: [PATCH 517/909] Implement ZlibInputStream::Tell() by keeping track of
 the number of bytes consumed by the reader.

PiperOrigin-RevId: 172634455
---
 tensorflow/core/lib/io/zlib_buffers_test.cc | 172 +++++++++++++++++---
 tensorflow/core/lib/io/zlib_inputstream.cc  |   8 +-
 tensorflow/core/lib/io/zlib_inputstream.h   |   3 +
 3 files changed, 156 insertions(+), 27 deletions(-)

diff --git a/tensorflow/core/lib/io/zlib_buffers_test.cc b/tensorflow/core/lib/io/zlib_buffers_test.cc
index 66ee68a916..156c712db8 100644
--- a/tensorflow/core/lib/io/zlib_buffers_test.cc
+++ b/tensorflow/core/lib/io/zlib_buffers_test.cc
@@ -68,25 +68,25 @@ void TestAllCombinations(CompressionOptions input_options,
     for (auto input_buf_size : InputBufferSizes()) {
       for (auto output_buf_size : OutputBufferSizes()) {
         std::unique_ptr<WritableFile> file_writer;
-        TF_CHECK_OK(env->NewWritableFile(fname, &file_writer));
+        TF_ASSERT_OK(env->NewWritableFile(fname, &file_writer));
         string result;
 
         ZlibOutputBuffer out(file_writer.get(), input_buf_size, output_buf_size,
                              output_options);
-        TF_CHECK_OK(out.Init());
+        TF_ASSERT_OK(out.Init());
 
-        TF_CHECK_OK(out.Append(StringPiece(data)));
-        TF_CHECK_OK(out.Close());
-        TF_CHECK_OK(file_writer->Flush());
-        TF_CHECK_OK(file_writer->Close());
+        TF_ASSERT_OK(out.Append(StringPiece(data)));
+        TF_ASSERT_OK(out.Close());
+        TF_ASSERT_OK(file_writer->Flush());
+        TF_ASSERT_OK(file_writer->Close());
 
         std::unique_ptr<RandomAccessFile> file_reader;
-        TF_CHECK_OK(env->NewRandomAccessFile(fname, &file_reader));
+        TF_ASSERT_OK(env->NewRandomAccessFile(fname, &file_reader));
         std::unique_ptr<RandomAccessInputStream> input_stream(
             new RandomAccessInputStream(file_reader.get()));
         ZlibInputStream in(input_stream.get(), input_buf_size, output_buf_size,
                            input_options);
-        TF_EXPECT_OK(in.ReadNBytes(data.size(), &result));
+        TF_ASSERT_OK(in.ReadNBytes(data.size(), &result));
         EXPECT_EQ(result, data);
       }
     }
@@ -118,24 +118,24 @@ void TestMultipleWrites(uint8 input_buf_size, uint8 output_buf_size,
   string actual_result;
   string expected_result;
 
-  TF_CHECK_OK(env->NewWritableFile(fname, &file_writer));
+  TF_ASSERT_OK(env->NewWritableFile(fname, &file_writer));
   ZlibOutputBuffer out(file_writer.get(), input_buf_size, output_buf_size,
                        output_options);
-  TF_CHECK_OK(out.Init());
+  TF_ASSERT_OK(out.Init());
 
   for (int i = 0; i < num_writes; i++) {
-    TF_CHECK_OK(out.Append(StringPiece(data)));
+    TF_ASSERT_OK(out.Append(StringPiece(data)));
     if (with_flush) {
-      TF_CHECK_OK(out.Flush());
+      TF_ASSERT_OK(out.Flush());
     }
     strings::StrAppend(&expected_result, data);
   }
-  TF_CHECK_OK(out.Close());
-  TF_CHECK_OK(file_writer->Flush());
-  TF_CHECK_OK(file_writer->Close());
+  TF_ASSERT_OK(out.Close());
+  TF_ASSERT_OK(file_writer->Flush());
+  TF_ASSERT_OK(file_writer->Close());
 
   std::unique_ptr<RandomAccessFile> file_reader;
-  TF_CHECK_OK(env->NewRandomAccessFile(fname, &file_reader));
+  TF_ASSERT_OK(env->NewRandomAccessFile(fname, &file_reader));
   std::unique_ptr<RandomAccessInputStream> input_stream(
       new RandomAccessInputStream(file_reader.get()));
   ZlibInputStream in(input_stream.get(), input_buf_size, output_buf_size,
@@ -143,7 +143,7 @@ void TestMultipleWrites(uint8 input_buf_size, uint8 output_buf_size,
 
   for (int i = 0; i < num_writes; i++) {
     string decompressed_output;
-    TF_EXPECT_OK(in.ReadNBytes(data.size(), &decompressed_output));
+    TF_ASSERT_OK(in.ReadNBytes(data.size(), &decompressed_output));
     strings::StrAppend(&actual_result, decompressed_output);
   }
 
@@ -170,19 +170,19 @@ TEST(ZlibInputStream, FailsToReadIfWindowBitsAreIncompatible) {
 
   string data = GenTestString(10);
   std::unique_ptr<WritableFile> file_writer;
-  TF_CHECK_OK(env->NewWritableFile(fname, &file_writer));
+  TF_ASSERT_OK(env->NewWritableFile(fname, &file_writer));
   string result;
   ZlibOutputBuffer out(file_writer.get(), input_buf_size, output_buf_size,
                        output_options);
-  TF_CHECK_OK(out.Init());
+  TF_ASSERT_OK(out.Init());
 
-  TF_CHECK_OK(out.Append(StringPiece(data)));
-  TF_CHECK_OK(out.Close());
-  TF_CHECK_OK(file_writer->Flush());
-  TF_CHECK_OK(file_writer->Close());
+  TF_ASSERT_OK(out.Append(StringPiece(data)));
+  TF_ASSERT_OK(out.Close());
+  TF_ASSERT_OK(file_writer->Flush());
+  TF_ASSERT_OK(file_writer->Close());
 
   std::unique_ptr<RandomAccessFile> file_reader;
-  TF_CHECK_OK(env->NewRandomAccessFile(fname, &file_reader));
+  TF_ASSERT_OK(env->NewRandomAccessFile(fname, &file_reader));
   std::unique_ptr<RandomAccessInputStream> input_stream(
       new RandomAccessInputStream(file_reader.get()));
   ZlibInputStream in(input_stream.get(), input_buf_size, output_buf_size,
@@ -192,5 +192,129 @@ TEST(ZlibInputStream, FailsToReadIfWindowBitsAreIncompatible) {
   CHECK(read_status.error_message().find("inflate() failed") != string::npos);
 }
 
+void WriteCompressedFile(Env* env, const string& fname, int input_buf_size,
+                         int output_buf_size,
+                         const CompressionOptions& output_options,
+                         const string& data) {
+  std::unique_ptr<WritableFile> file_writer;
+  TF_ASSERT_OK(env->NewWritableFile(fname, &file_writer));
+
+  ZlibOutputBuffer out(file_writer.get(), input_buf_size, output_buf_size,
+                       output_options);
+  TF_ASSERT_OK(out.Init());
+
+  TF_ASSERT_OK(out.Append(StringPiece(data)));
+  TF_ASSERT_OK(out.Close());
+  TF_ASSERT_OK(file_writer->Flush());
+  TF_ASSERT_OK(file_writer->Close());
+}
+
+void TestTell(CompressionOptions input_options,
+              CompressionOptions output_options) {
+  Env* env = Env::Default();
+  string fname = testing::TmpDir() + "/zlib_buffers_test";
+  for (auto file_size : NumCopies()) {
+    string data = GenTestString(file_size);
+    for (auto input_buf_size : InputBufferSizes()) {
+      for (auto output_buf_size : OutputBufferSizes()) {
+        // Write the compressed file.
+        WriteCompressedFile(env, fname, input_buf_size, output_buf_size,
+                            output_options, data);
+
+        // Boiler-plate to set up ZlibInputStream.
+        std::unique_ptr<RandomAccessFile> file_reader;
+        TF_ASSERT_OK(env->NewRandomAccessFile(fname, &file_reader));
+        std::unique_ptr<RandomAccessInputStream> input_stream(
+            new RandomAccessInputStream(file_reader.get()));
+        ZlibInputStream in(input_stream.get(), input_buf_size, output_buf_size,
+                           input_options);
+
+        string first_half(data, 0, data.size() / 2);
+        string bytes_read;
+
+        // Read the first half of the uncompressed file and expect that Tell()
+        // returns half the uncompressed length of the file.
+        TF_ASSERT_OK(in.ReadNBytes(first_half.size(), &bytes_read));
+        EXPECT_EQ(in.Tell(), first_half.size());
+        EXPECT_EQ(bytes_read, first_half);
+
+        // Read the remaining half of the uncompressed file and expect that
+        // Tell() points past the end of file.
+        string second_half;
+        TF_ASSERT_OK(
+            in.ReadNBytes(data.size() - first_half.size(), &second_half));
+        EXPECT_EQ(in.Tell(), data.size());
+        bytes_read.append(second_half);
+
+        // Expect that the file is correctly read.
+        EXPECT_EQ(bytes_read, data);
+      }
+    }
+  }
+}
+
+void TestSkipNBytes(CompressionOptions input_options,
+                    CompressionOptions output_options) {
+  Env* env = Env::Default();
+  string fname = testing::TmpDir() + "/zlib_buffers_test";
+  for (auto file_size : NumCopies()) {
+    string data = GenTestString(file_size);
+    for (auto input_buf_size : InputBufferSizes()) {
+      for (auto output_buf_size : OutputBufferSizes()) {
+        // Write the compressed file.
+        WriteCompressedFile(env, fname, input_buf_size, output_buf_size,
+                            output_options, data);
+
+        // Boiler-plate to set up ZlibInputStream.
+        std::unique_ptr<RandomAccessFile> file_reader;
+        TF_ASSERT_OK(env->NewRandomAccessFile(fname, &file_reader));
+        std::unique_ptr<RandomAccessInputStream> input_stream(
+            new RandomAccessInputStream(file_reader.get()));
+        ZlibInputStream in(input_stream.get(), input_buf_size, output_buf_size,
+                           input_options);
+
+        size_t data_half_size = data.size() / 2;
+        string second_half(data, data_half_size, data.size() - data_half_size);
+
+        // Skip past the first half of the file and expect Tell() returns
+        // correctly.
+        TF_ASSERT_OK(in.SkipNBytes(data_half_size));
+        EXPECT_EQ(in.Tell(), data_half_size);
+
+        // Expect that second half is read correctly and Tell() returns past
+        // end of file after reading complete file.
+        string bytes_read;
+        TF_ASSERT_OK(in.ReadNBytes(second_half.size(), &bytes_read));
+        EXPECT_EQ(bytes_read, second_half);
+        EXPECT_EQ(in.Tell(), data.size());
+      }
+    }
+  }
+}
+
+TEST(ZlibInputStream, TellDefaultOptions) {
+  TestTell(CompressionOptions::DEFAULT(), CompressionOptions::DEFAULT());
+}
+
+TEST(ZlibInputStream, TellRawDeflate) {
+  TestTell(CompressionOptions::RAW(), CompressionOptions::RAW());
+}
+
+TEST(ZlibInputStream, TellGzip) {
+  TestTell(CompressionOptions::GZIP(), CompressionOptions::GZIP());
+}
+
+TEST(ZlibInputStream, SkipNBytesDefaultOptions) {
+  TestSkipNBytes(CompressionOptions::DEFAULT(), CompressionOptions::DEFAULT());
+}
+
+TEST(ZlibInputStream, SkipNBytesRawDeflate) {
+  TestSkipNBytes(CompressionOptions::RAW(), CompressionOptions::RAW());
+}
+
+TEST(ZlibInputStream, SkipNBytesGzip) {
+  TestSkipNBytes(CompressionOptions::GZIP(), CompressionOptions::GZIP());
+}
+
 }  // namespace io
 }  // namespace tensorflow
diff --git a/tensorflow/core/lib/io/zlib_inputstream.cc b/tensorflow/core/lib/io/zlib_inputstream.cc
index 4999d5cc90..984fbc2810 100644
--- a/tensorflow/core/lib/io/zlib_inputstream.cc
+++ b/tensorflow/core/lib/io/zlib_inputstream.cc
@@ -32,7 +32,8 @@ ZlibInputStream::ZlibInputStream(
       z_stream_input_(new Bytef[input_buffer_capacity_]),
       z_stream_output_(new Bytef[output_buffer_capacity_]),
       zlib_options_(zlib_options),
-      z_stream_(new z_stream) {
+      z_stream_(new z_stream),
+      bytes_read_(0) {
   InitZlibBuffer();
 }
 
@@ -45,6 +46,7 @@ ZlibInputStream::~ZlibInputStream() {
 Status ZlibInputStream::Reset() {
   TF_RETURN_IF_ERROR(input_stream_->Reset());
   InitZlibBuffer();
+  bytes_read_ = 0;
   return Status::OK();
 }
 
@@ -127,6 +129,7 @@ size_t ZlibInputStream::ReadBytesFromCache(size_t bytes_to_read,
     result->append(next_unread_byte_, can_read_bytes);
     next_unread_byte_ += can_read_bytes;
   }
+  bytes_read_ += can_read_bytes;
   return can_read_bytes;
 }
 
@@ -170,8 +173,7 @@ Status ZlibInputStream::ReadNBytes(int64 bytes_to_read, string* result) {
   return Status::OK();
 }
 
-// TODO(srbs): Implement this.
-int64 ZlibInputStream::Tell() const { return -1; }
+int64 ZlibInputStream::Tell() const { return bytes_read_; }
 
 Status ZlibInputStream::Inflate() {
   int error = inflate(z_stream_.get(), zlib_options_.flush_mode);
diff --git a/tensorflow/core/lib/io/zlib_inputstream.h b/tensorflow/core/lib/io/zlib_inputstream.h
index 8faa7dcb8f..9c7e14441c 100644
--- a/tensorflow/core/lib/io/zlib_inputstream.h
+++ b/tensorflow/core/lib/io/zlib_inputstream.h
@@ -132,6 +132,9 @@ class ZlibInputStream : public InputStreamInterface {
   // Returns the size of [next_unread_byte_, z_stream_->next_out)
   size_t NumUnreadBytes() const;
 
+  // Number of *uncompressed* bytes that have been read from this stream.
+  int64 bytes_read_;
+
   TF_DISALLOW_COPY_AND_ASSIGN(ZlibInputStream);
 };
 
-- 
GitLab


From d19ec7126735ca98a632ebd69ad64973fd454e6e Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Wed, 18 Oct 2017 12:10:56 -0700
Subject: [PATCH 518/909] Update boringssl to
 a0fb951d2a26a8ee746b52f3ba81ab011a0af778 (#13798)

* Update boringssl to a0fb951d2a26a8ee746b52f3ba81ab011a0af778

This fix update boringssl to a0fb951d2a26a8ee746b52f3ba81ab011a0af778,
which contains bug fix related to 13733.

See https://github.com/tensorflow/tensorflow/pull/13734#issuecomment-337440239
for update details.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Remove unneeded patch part for boringssl, as 13733 has been fixed.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Remove s390x patch for boringssl

Per discussion:
https://github.com/tensorflow/tensorflow/pull/13798#discussion_r145459255

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/workspace.bzl                      | 10 ++++----
 .../boringssl/add_boringssl_s390x.patch       | 23 -------------------
 2 files changed, 4 insertions(+), 29 deletions(-)
 delete mode 100644 third_party/boringssl/add_boringssl_s390x.patch

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index ea50d0f296..0997fffc8a 100644
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -590,15 +590,13 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       actual = "@jsoncpp_git//:jsoncpp",
   )
 
-  patched_http_archive(
+  native.http_archive(
       name = "boringssl",
       urls = [
-          "https://github.com/google/boringssl/archive/72cfd9f49ec5fbc2db368b76398c196dafe6a4bc.tar.gz",
+          "https://github.com/google/boringssl/archive/a0fb951d2a26a8ee746b52f3ba81ab011a0af778.tar.gz",
       ],
-      sha256 = "5e6f7b72c74adeb902581271925ddb979e77b96327abd76604ce894d80680e51",
-      strip_prefix = "boringssl-72cfd9f49ec5fbc2db368b76398c196dafe6a4bc",
-      # Add patch to boringssl code to support s390x
-      patch_file = str(Label("//third_party/boringssl:add_boringssl_s390x.patch")),
+      sha256 = "524ba98a56300149696481b4cb9ddebd0c7b7ac9b9f6edee81da2d2d7e5d2bb3",
+      strip_prefix = "boringssl-a0fb951d2a26a8ee746b52f3ba81ab011a0af778",
   )
 
   native.new_http_archive(
diff --git a/third_party/boringssl/add_boringssl_s390x.patch b/third_party/boringssl/add_boringssl_s390x.patch
deleted file mode 100644
index b684dc6df7..0000000000
--- a/third_party/boringssl/add_boringssl_s390x.patch
+++ /dev/null
@@ -1,23 +0,0 @@
-diff -ur a/BUILD b/BUILD
---- a/BUILD	2017-10-10 15:50:34.000000000 +0000
-+++ b/BUILD	2017-10-15 21:19:02.057606476 +0000
-@@ -63,6 +63,7 @@
-     "-Wwrite-strings",
-     "-Wshadow",
-     "-fno-common",
-+    "-Wno-uninitialized",
- 
-     # Modern build environments should be able to set this to use atomic
-     # operations for reference counting rather than locks. However, it's
-diff -ur a/src/include/openssl/base.h b/src/include/openssl/base.h
---- a/src/include/openssl/base.h	2017-10-10 15:50:34.000000000 +0000
-+++ b/src/include/openssl/base.h	2017-10-15 19:49:38.182154627 +0000
-@@ -106,6 +106,8 @@
- #define OPENSSL_PNACL
- #elif defined(__myriad2__)
- #define OPENSSL_32_BIT
-+#elif defined(__s390x__)
-+#define OPENSSL_64_BIT
- #else
- // Note BoringSSL only supports standard 32-bit and 64-bit two's-complement,
- // little-endian architectures. Functions will not produce the correct answer
-- 
GitLab


From ef060d923acef07cd3a4b1134218abc84fcb3a7b Mon Sep 17 00:00:00 2001
From: Shanqing Cai <cais@google.com>
Date: Wed, 18 Oct 2017 12:06:25 -0700
Subject: [PATCH 519/909] Upgrade tensorflow pip dependency version to 3.4.0+

PiperOrigin-RevId: 172635727
---
 tensorflow/tools/pip_package/setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/tools/pip_package/setup.py b/tensorflow/tools/pip_package/setup.py
index a7a0706d0b..c05d39e942 100644
--- a/tensorflow/tools/pip_package/setup.py
+++ b/tensorflow/tools/pip_package/setup.py
@@ -35,7 +35,7 @@ REQUIRED_PACKAGES = [
     'enum34 >= 1.1.6',
     'numpy >= 1.12.1',
     'six >= 1.10.0',
-    'protobuf >= 3.3.0',
+    'protobuf >= 3.4.0',
     'tensorflow-tensorboard >= 0.1.0, < 0.2.0',
 ]
 
-- 
GitLab


From f1603b7893f922dfe64244c6bae9b93d7d594437 Mon Sep 17 00:00:00 2001
From: Yunxing Dai <yunxing@google.com>
Date: Wed, 18 Oct 2017 12:16:38 -0700
Subject: [PATCH 520/909] [XLA] Deterministically dump an executable.

Previously, dumping a executable is nondeterministic as a map in protobuf is serialized in random order.

This CL enables "Deterministic dump" mode of protobuf, which sorts the map first before dumping them. This is helpful in comparing if two dumps are the same in XLA determinism test.

PiperOrigin-RevId: 172637100
---
 tensorflow/compiler/xla/BUILD                 | 1 +
 tensorflow/compiler/xla/service/BUILD         | 2 ++
 tensorflow/compiler/xla/service/executable.cc | 8 +++++++-
 tensorflow/compiler/xla/util.h                | 1 +
 4 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xla/BUILD b/tensorflow/compiler/xla/BUILD
index be87506d3c..e51bbffcd0 100644
--- a/tensorflow/compiler/xla/BUILD
+++ b/tensorflow/compiler/xla/BUILD
@@ -171,6 +171,7 @@ cc_library(
     visibility = ["//visibility:public"],
     deps = [
         ":status",
+        ":status_macros",
         ":types",
         ":xla_data_proto",
         "//tensorflow/core:lib",
diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD
index d1335e20e0..fed7bd01f6 100644
--- a/tensorflow/compiler/xla/service/BUILD
+++ b/tensorflow/compiler/xla/service/BUILD
@@ -581,12 +581,14 @@ cc_library(
         ":shaped_buffer",
         ":versioned_computation_handle",
         "//tensorflow/compiler/xla:executable_run_options",
+        "//tensorflow/compiler/xla:status",
         "//tensorflow/compiler/xla:status_macros",
         "//tensorflow/compiler/xla:statusor",
         "//tensorflow/compiler/xla:util",
         "//tensorflow/compiler/xla:xla_data_proto",
         "//tensorflow/compiler/xla/legacy_flags:debug_options_flags",
         "//tensorflow/core:lib",
+        "//tensorflow/core:lib_internal",
         "//tensorflow/core:stream_executor_no_cuda",
         "//tensorflow/stream_executor",
     ],
diff --git a/tensorflow/compiler/xla/service/executable.cc b/tensorflow/compiler/xla/service/executable.cc
index 62b8fa6a2b..9c96d9eb30 100644
--- a/tensorflow/compiler/xla/service/executable.cc
+++ b/tensorflow/compiler/xla/service/executable.cc
@@ -17,7 +17,9 @@ limitations under the License.
 
 #include "tensorflow/compiler/xla/legacy_flags/debug_options_flags.h"
 #include "tensorflow/compiler/xla/service/hlo_graph_dumper.h"
+#include "tensorflow/compiler/xla/status.h"
 #include "tensorflow/compiler/xla/status_macros.h"
+#include "tensorflow/core/lib/hash/hash.h"
 #include "tensorflow/core/lib/io/path.h"
 #include "tensorflow/core/lib/strings/stringprintf.h"
 #include "tensorflow/core/platform/env.h"
@@ -82,7 +84,11 @@ Status Executable::DumpSessionModule() {
   }
   filename = SanitizeFileName(std::move(filename));
   string file_path = tensorflow::io::JoinPath(directory_path, filename);
-  return tensorflow::WriteBinaryProto(env, file_path, session_module);
+  string result;
+  TF_RET_CHECK(
+      tensorflow::SerializeToStringDeterministic(session_module, &result));
+  return tensorflow::WriteStringToFile(tensorflow::Env::Default(), file_path,
+                                       result);
 }
 
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/util.h b/tensorflow/compiler/xla/util.h
index f6c0bd1563..f58f57b443 100644
--- a/tensorflow/compiler/xla/util.h
+++ b/tensorflow/compiler/xla/util.h
@@ -24,6 +24,7 @@ limitations under the License.
 #include <vector>
 
 #include "tensorflow/compiler/xla/status.h"
+#include "tensorflow/compiler/xla/status_macros.h"
 #include "tensorflow/compiler/xla/types.h"
 #include "tensorflow/compiler/xla/xla_data.pb.h"
 #include "tensorflow/core/lib/core/status.h"
-- 
GitLab


From 192f1c24ec6692342391c03bb620f5de1af9de3b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 18 Oct 2017 12:22:04 -0700
Subject: [PATCH 521/909] Fixed work size computation in Split and SplitV ops
 to avoid integer overflow.

PiperOrigin-RevId: 172637818
---
 tensorflow/core/kernels/split_op.cc   | 8 ++++----
 tensorflow/core/kernels/split_v_op.cc | 8 ++++----
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/tensorflow/core/kernels/split_op.cc b/tensorflow/core/kernels/split_op.cc
index 4d2100c59c..58e1a73be6 100644
--- a/tensorflow/core/kernels/split_op.cc
+++ b/tensorflow/core/kernels/split_op.cc
@@ -167,11 +167,11 @@ class SplitOpCPU : public SplitOpBase<CPUDevice, T> {
     const auto num_threads =
         context->device()->tensorflow_cpu_worker_threads()->num_threads;
     // TODO(jewillco): Tune heuristic further.
+    const auto input_element_count = input_shape.num_elements();
     const bool use_parallelism_between_outputs =
         (num_split >= 4 &&
-         input_shape.num_elements() >=
-             std::max(num_threads, num_split) * 4096 &&
-         input_shape.num_elements() < num_split * 180 * 1024);
+         input_element_count >= std::max(num_threads, num_split) * 4096 &&
+         input_element_count < num_split * 180 * 1024);
 
     auto range_output_func = [&indices, context, &output_shape, prefix_dim_size,
                               split_dim_output_size, suffix_dim_size, &sizes,
@@ -209,7 +209,7 @@ class SplitOpCPU : public SplitOpBase<CPUDevice, T> {
       // Run in parallel, disabling parallelism in functor.
       Shard(num_split,
             context->device()->tensorflow_cpu_worker_threads()->workers,
-            num_split, kint64max, range_output_func);
+            num_split, input_element_count / num_split, range_output_func);
     } else {
       // Run sequentially, but allow internal parallelism in functor.
       range_output_func(0, num_split);
diff --git a/tensorflow/core/kernels/split_v_op.cc b/tensorflow/core/kernels/split_v_op.cc
index e2dd66da1e..3316e5fcc9 100644
--- a/tensorflow/core/kernels/split_v_op.cc
+++ b/tensorflow/core/kernels/split_v_op.cc
@@ -225,11 +225,11 @@ class SplitVOpCPU : public SplitVOpBase<CPUDevice, T, Tlen> {
     const auto num_threads =
         context->device()->tensorflow_cpu_worker_threads()->num_threads;
     // TODO(jewillco): Tune heuristic further.
+    const auto input_element_count = input_shape.num_elements();
     const bool use_parallelism_between_outputs =
         (num_split >= 4 &&
-         input_shape.num_elements() >=
-             std::max(num_threads, num_split) * 4096 &&
-         input_shape.num_elements() < num_split * 180 * 1024);
+         input_element_count >= std::max(num_threads, num_split) * 4096 &&
+         input_element_count < num_split * 180 * 1024);
 
     auto range_output_func = [&indices, context, &input_shape, prefix_dim_size,
                               split_dim, &split_sizes_vec, &split_start_points,
@@ -267,7 +267,7 @@ class SplitVOpCPU : public SplitVOpBase<CPUDevice, T, Tlen> {
       // Run in parallel, disabling parallelism in functor.
       Shard(num_split,
             context->device()->tensorflow_cpu_worker_threads()->workers,
-            num_split, kint64max, range_output_func);
+            num_split, input_element_count / num_split, range_output_func);
     } else {
       // Run sequentially, but allow internal parallelism in functor.
       range_output_func(0, num_split);
-- 
GitLab


From 7321905ff14c47211c95e430625f8b29986c1df1 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Wed, 18 Oct 2017 12:48:13 -0700
Subject: [PATCH 522/909] Add missing `uint16` type registration for ops
 `CropAndResize`/`CropAndResizeGradBoxes` (#13812)

* Add missing `uint16` type registration for ops `CropAndResize`/`CropAndResizeGradBoxes`

This fix adds missing `uint16` type registration in `image_ops.cc`
for `CropAndResize` and `CropAndResizeGradBoxes`.

The kernel of `uint16` is available for `CropAndResize` and `CropAndResizeGradBoxes`
though it is missing in `image_ops.cc`. This fix addresses this issue.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Add incomplete test cases for `CropAndResize`

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/core/kernels/crop_and_resize_op_test.cc | 6 +++++-
 tensorflow/core/ops/image_ops.cc                   | 4 ++--
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/tensorflow/core/kernels/crop_and_resize_op_test.cc b/tensorflow/core/kernels/crop_and_resize_op_test.cc
index 22c659b587..a35e1b0788 100644
--- a/tensorflow/core/kernels/crop_and_resize_op_test.cc
+++ b/tensorflow/core/kernels/crop_and_resize_op_test.cc
@@ -61,8 +61,12 @@ class CropAndResizeOpTest : public OpsTestBase {
 
 REGISTER_TEST(float)
 REGISTER_TEST(double)
-REGISTER_TEST(int8)
 REGISTER_TEST(uint8)
+REGISTER_TEST(uint16)
+REGISTER_TEST(int8)
+REGISTER_TEST(int16)
+REGISTER_TEST(int32)
+REGISTER_TEST(int64)
 
 #undef REGISTER_TEST
 
diff --git a/tensorflow/core/ops/image_ops.cc b/tensorflow/core/ops/image_ops.cc
index 89c9da81c5..e9bf29d172 100644
--- a/tensorflow/core/ops/image_ops.cc
+++ b/tensorflow/core/ops/image_ops.cc
@@ -1097,7 +1097,7 @@ REGISTER_OP("CropAndResize")
     .Input("box_ind: int32")
     .Input("crop_size: int32")
     .Output("crops: float")
-    .Attr("T: {uint8, int8, int16, int32, int64, half, float, double}")
+    .Attr("T: {uint8, uint16, int8, int16, int32, int64, half, float, double}")
     .Attr("method: {'bilinear'} = 'bilinear'")
     .Attr("extrapolation_value: float = 0")
     .SetShapeFn([](InferenceContext* c) {
@@ -1204,7 +1204,7 @@ REGISTER_OP("CropAndResizeGradBoxes")
     .Input("boxes: float")
     .Input("box_ind: int32")
     .Output("output: float")
-    .Attr("T: {uint8, int8, int16, int32, int64, half, float, double}")
+    .Attr("T: {uint8, uint16, int8, int16, int32, int64, half, float, double}")
     .Attr("method: {'bilinear'} = 'bilinear'")
     .SetShapeFn([](InferenceContext* c) {
       c->set_output(0, c->input(2));
-- 
GitLab


From 09ff3f7296a66c39535e097ecb6b82e3fc42ba30 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 18 Oct 2017 12:51:28 -0700
Subject: [PATCH 523/909] Internal change.

PiperOrigin-RevId: 172641543
---
 tensorflow/python/BUILD | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index e4e284dcdf..cbeb0b46cb 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -3685,7 +3685,10 @@ py_test(
     size = "medium",
     srcs = ["training/monitored_session_test.py"],
     srcs_version = "PY2AND3",
-    tags = ["no_windows"],
+    tags = [
+        "no_windows",
+        "notsan",  # b/67945581
+    ],
     deps = [
         ":array_ops",
         ":client_testlib",
-- 
GitLab


From 38bcb3c02fbc5185d6c1fb7e8327a070284b66e4 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 18 Oct 2017 12:53:54 -0700
Subject: [PATCH 524/909] Bug fixes for fold_constants_lib.

1. Tensor names in TF may be in the form of "a:0", "a:1", or "a" as a shorthand
notation of "a:0". FoldConstant library always expected the shorthand notation,
and did not handle the cases where explicit notation was passed to input or
output list. This means that this library could not handle the case when input
or output were not the first output of a node.

2. To match the input nodes in the original graph and the added Recv nodes in
rewritten graph, FoldConstant library used prefix matching. Unfortunately, this
means that when a input name is a prefix of another input name, there is
possibility that wrong Recv node gets matched. For example, if input names were
"placeholder" and "placeholder_1", then it did not handle the case very well.

3. RemoveUnusedNodes() in FoldConstants lib could remove nodes which output
depended on. This happened when an input name points to a node with multiple
outputs and not all outputs of that node were included in the input names.

4. ReplaceSendRecvs() in FoldConstants lib assumed that all input nodes are
removed during rewriting the graph. This assumption is not necessarily true,
and it could add a duplicate node in the graph.

PiperOrigin-RevId: 172641947
---
 .../graph_transforms/fold_constants_lib.cc    | 202 ++++++++----------
 .../graph_transforms/fold_constants_test.cc   |  85 +++++++-
 2 files changed, 175 insertions(+), 112 deletions(-)

diff --git a/tensorflow/tools/graph_transforms/fold_constants_lib.cc b/tensorflow/tools/graph_transforms/fold_constants_lib.cc
index 30290c7a16..f2934a79bd 100644
--- a/tensorflow/tools/graph_transforms/fold_constants_lib.cc
+++ b/tensorflow/tools/graph_transforms/fold_constants_lib.cc
@@ -17,12 +17,20 @@ limitations under the License.
 
 #include <algorithm>
 #include <iterator>
+#include <map>
+#include <string>
+#include <unordered_map>
+#include <unordered_set>
+#include <utility>
+#include <vector>
 
 #include "tensorflow/core/common_runtime/constant_folding.h"
 #include "tensorflow/core/common_runtime/shape_refiner.h"
 #include "tensorflow/core/graph/graph_constructor.h"
 #include "tensorflow/core/graph/node_builder.h"
 #include "tensorflow/core/graph/subgraph.h"
+#include "tensorflow/core/lib/core/stringpiece.h"
+#include "tensorflow/core/lib/strings/numbers.h"
 #include "tensorflow/core/platform/init_main.h"
 #include "tensorflow/core/public/session.h"
 #include "tensorflow/core/util/command_line_flags.h"
@@ -30,33 +38,38 @@ limitations under the License.
 
 namespace tensorflow {
 namespace graph_transforms {
+namespace {
+using StringPieceSet = std::unordered_set<StringPiece, StringPiece::Hasher>;
+template <typename T>
+using StringPieceMap = std::unordered_map<StringPiece, T, StringPiece::Hasher>;
+}  // namespace
 
 Status ReplaceSendRecvs(const GraphDef& original_graph_def,
                         const GraphDef& rewritten_graph_def,
                         const std::vector<string>& inputs,
                         const std::vector<string>& outputs,
                         GraphDef* output_graph_def) {
-  std::map<string, const NodeDef*> original_map;
-  MapNamesToNodes(original_graph_def, &original_map);
-  std::map<string, string> new_node_names;
-  for (const NodeDef& node : rewritten_graph_def.node()) {
-    // If the op isn't a Recv, or it was in the original, nothing to do.
-    if ((node.op() != "_Recv") || (original_map.count(node.name()) == 1)) {
-      continue;
-    }
-    // See if it matches an input from the original.
-    for (const string& input : inputs) {
-      // Here we rely on the naming convention for the Recv nodes that
-      // RewriteGraphForExecution adds in the place of the feed inputs.
-      string input_prefix = "_recv_" + input + "_";
-      if (StringPiece(node.name()).starts_with(input_prefix)) {
-        // If it does, prepare to rename any inputs that refer to it.
-        new_node_names[node.name()] = input;
-      }
-    }
+  // recv_node_names serves as a string storage for recv node names.
+  std::vector<string> recv_node_names(inputs.size());
+  StringPieceMap<TensorId> recv_node_map;
+  StringPieceSet input_nodes;
+  for (int i = 0; i < inputs.size(); ++i) {
+    // RewriteGraphForExecution adds a recv node for each input edge. We assume
+    // here that adding such recv node did not fail. For example, the original
+    // graph did not already have a node with the name for the new added recv
+    // node.
+    TensorId id = ParseTensorName(inputs[i]);
+    input_nodes.insert(id.first);
+    string& recv_node_name = recv_node_names[i];
+    recv_node_name = strings::StrCat("_recv_", id.first, "_", id.second);
+    recv_node_map.emplace(recv_node_name, id);
+  }
+
+  StringPieceMap<const NodeDef*> original_map;
+  for (const NodeDef& node : original_graph_def.node()) {
+    original_map.emplace(node.name(), &node);
   }
 
-  std::vector<NodeDef> nodes_to_add;
   for (const NodeDef& node : rewritten_graph_def.node()) {
     if ((node.op() == "_Send") || (node.op() == "_Recv")) {
       // If the op is a Send or Recv that wasn't in the original, skip it.
@@ -64,55 +77,68 @@ Status ReplaceSendRecvs(const GraphDef& original_graph_def,
         continue;
       }
     }
-    NodeDef new_node;
-    new_node = node;
-    new_node.mutable_input()->Clear();
-    for (const string& old_input : node.input()) {
-      string input_prefix;
-      string input_node_name;
-      string input_suffix;
-      NodeNamePartsFromInput(old_input, &input_prefix, &input_node_name,
-                             &input_suffix);
-      string new_input;
-      if (new_node_names.count(input_node_name) > 0) {
-        new_input =
-            input_prefix + new_node_names[input_node_name] + input_suffix;
-      } else {
-        new_input = old_input;
+
+    NodeDef* new_node = output_graph_def->add_node();
+    new_node->MergeFrom(node);
+    for (int i = 0; i < new_node->input_size(); ++i) {
+      string& input = *new_node->mutable_input(i);
+      TensorId id = ParseTensorName(input);
+      const auto iter = recv_node_map.find(id.first);
+      if (iter != recv_node_map.end()) {
+        // The node being substituted is a Recv node, and it has only one
+        // output. If this input is not a control input, then replace the input
+        // with the mapped value. Otherwise, replace the node name only.
+        if (id.second != Graph::kControlSlot) {
+          CHECK_EQ(id.second, 0);
+          input = iter->second.ToString();
+        } else {
+          id.first = iter->second.first;
+          input = id.ToString();
+        }
       }
-      *(new_node.mutable_input()->Add()) = new_input;
     }
-    nodes_to_add.push_back(new_node);
-  }
-  for (std::pair<string, string> entry : new_node_names) {
-    string removed_node_name = entry.second;
-    const NodeDef* removed_node = original_map[removed_node_name];
-    NodeDef new_node;
-    new_node = *removed_node;
-    nodes_to_add.push_back(new_node);
+
+    // RewriteGraphForExecution() did not remove this input node. Remove this
+    // node name from input_nodes so that a duplicate does not get added to the
+    // output_graph_def.
+    auto iter = input_nodes.find(new_node->name());
+    if (iter != input_nodes.end()) {
+      input_nodes.erase(iter);
+    }
   }
 
-  for (const NodeDef& node : nodes_to_add) {
-    *output_graph_def->mutable_node()->Add() = node;
+  // Some input nodes are removed in rewrite_graph_def. Add those nodes to
+  // output_graph_def.
+  for (StringPiece name : input_nodes) {
+    const NodeDef& removed_node = *CHECK_NOTNULL(original_map[name]);
+    output_graph_def->add_node()->MergeFrom(removed_node);
   }
+
   return Status::OK();
 }
 
 Status RemoveUnusedNodes(const GraphDef& input_graph_def,
                          const TransformFuncContext& context,
                          GraphDef* output_graph_def) {
-  std::map<string, const NodeDef*> node_map;
-  MapNamesToNodes(input_graph_def, &node_map);
+  StringPieceMap<const NodeDef*> node_map;
+  for (const NodeDef& node : input_graph_def.node()) {
+    node_map.emplace(node.name(), &node);
+  }
 
-  std::set<string> used_nodes;
+  std::unordered_set<TensorId, TensorId::Hasher> input_names;
   for (const string& input : context.input_names) {
-    used_nodes.insert(input);
+    input_names.insert(ParseTensorName(input));
+  }
+  StringPieceSet used_nodes;
+  StringPieceSet current_nodes;
+  for (const string& name : context.output_names) {
+    TensorId id = ParseTensorName(name);
+    used_nodes.insert(id.first);
+    current_nodes.insert(id.first);
   }
-  std::vector<string> current_nodes = context.output_names;
   while (!current_nodes.empty()) {
-    std::set<string> next_nodes;
-    for (const string& node_name : current_nodes) {
-      used_nodes.insert(node_name);
+    StringPieceSet next_nodes;
+    for (StringPiece node_name : current_nodes) {
       if (node_map.count(node_name) == 0) {
         LOG(ERROR) << "Bad graph structure, no node named '" << node_name
                    << "' found for input lookup";
@@ -120,14 +146,20 @@ Status RemoveUnusedNodes(const GraphDef& input_graph_def,
                                        node_name, "' found for input lookup");
       }
       const NodeDef& node = *(node_map[node_name]);
-      for (const string& input_name : node.input()) {
-        const string& input_node_name = NodeNameFromInput(input_name);
-        if (used_nodes.count(input_node_name) == 0) {
-          next_nodes.insert(input_node_name);
+      for (const string& input : node.input()) {
+        TensorId id = ParseTensorName(input);
+        if (input_names.count(id) > 0) {
+          continue;
+        }
+        if (used_nodes.insert(id.first).second) {
+          next_nodes.insert(id.first);
         }
       }
     }
-    current_nodes = std::vector<string>(next_nodes.begin(), next_nodes.end());
+    current_nodes.swap(next_nodes);
+  }
+  for (const TensorId& id : input_names) {
+    used_nodes.insert(id.first);
   }
   FilterGraphDef(
       input_graph_def,
@@ -141,7 +173,7 @@ Status RemoveUnusedNodes(const GraphDef& input_graph_def,
 Status ShapeHandleToTensorShape(const shape_inference::ShapeHandle& handle,
                                 shape_inference::InferenceContext* context,
                                 PartialTensorShape* shape) {
-  // The default is already unknown
+  // The default is already unknown.
   if (!context->RankKnown(handle)) return Status::OK();
 
   std::vector<int64> dims(context->Rank(handle));
@@ -151,47 +183,6 @@ Status ShapeHandleToTensorShape(const shape_inference::ShapeHandle& handle,
   return PartialTensorShape::MakePartialShape(dims.data(), dims.size(), shape);
 }
 
-Status ShapeForNode(const TransformFuncContext& context,
-                    const string& node_name, TensorShape* result,
-                    bool* has_shape_specified) {
-  *has_shape_specified = false;
-
-  // Check to see if we have been given a default for all placeholders.
-  if (context.params.count("type")) {
-    if (context.params.at("shape").size() != 1) {
-      return errors::InvalidArgument(
-          "You must pass no more than one default 'shape' to "
-          "fold_constants");
-    }
-    const string& shape_string = context.params.at("shape")[0];
-    TF_RETURN_IF_ERROR(TensorShapeFromString(shape_string, result));
-    *has_shape_specified = true;
-  }
-
-  // See if there's a particular type specified for this placeholder.
-  if (context.params.count("name") || context.params.count("type_for_name")) {
-    if (!context.params.count("name") ||
-        !context.params.count("type_for_name") ||
-        (context.params.at("type_for_name").size() !=
-         context.params.at("name").size())) {
-      return errors::InvalidArgument(
-          "You must pass a 'shape_for_name' arg for every 'name', e.g. "
-          "fold_constants(name=foo, shape_for_name=\"2,2,1\", name=bar, "
-          "shape_for_name=\"1\"");
-    }
-    const int name_count = context.params.at("name").size();
-    for (int i = 0; i < name_count; ++i) {
-      if (context.params.at("name")[i] == node_name) {
-        const string& shape_string = context.params.at("shape_for_name")[i];
-        TF_RETURN_IF_ERROR(TensorShapeFromString(shape_string, result));
-        *has_shape_specified = true;
-      }
-    }
-  }
-
-  return Status::OK();
-}
-
 // Converts any sub-graphs that can be resolved into constant expressions into
 // single Const ops.
 Status FoldConstants(const GraphDef& input_graph_def,
@@ -215,17 +206,6 @@ Status FoldConstants(const GraphDef& input_graph_def,
     GraphDef cleaned_graph_def;
     RemoveAttributes(input_graph_def, {"_output_shapes"}, &cleaned_graph_def);
 
-    // Set specified shapes.
-    for (NodeDef& node : *cleaned_graph_def.mutable_node()) {
-      TensorShape shape;
-      bool has_shape_specified;
-      TF_RETURN_IF_ERROR(
-          ShapeForNode(context, node.name(), &shape, &has_shape_specified));
-      if (has_shape_specified) {
-        SetNodeAttr("shape", shape, &node);
-      }
-    }
-
     TF_RETURN_IF_ERROR(
         ImportGraphDef({}, cleaned_graph_def, &input_graph, &shape_refiner));
   } else {
diff --git a/tensorflow/tools/graph_transforms/fold_constants_test.cc b/tensorflow/tools/graph_transforms/fold_constants_test.cc
index fd4188a6a4..41106de008 100644
--- a/tensorflow/tools/graph_transforms/fold_constants_test.cc
+++ b/tensorflow/tools/graph_transforms/fold_constants_test.cc
@@ -74,6 +74,9 @@ class ConstantFoldingTest : public ::testing::Test {
     TestConstantFolding(graph_def,
                         {{"placeholder_expect_remains", placeholder_tensor}},
                         {}, {"output_expect_remains"}, {});
+    TestConstantFolding(graph_def,
+                        {{"placeholder_expect_remains:0", placeholder_tensor}},
+                        {}, {"output_expect_remains:0"}, {});
   }
 
   void TestOpExclusionAdd() {
@@ -256,10 +259,40 @@ class ConstantFoldingTest : public ::testing::Test {
     EXPECT_EQ(0, node_map.count("new_send"));
   }
 
+  void TestReplaceSendRecvsPrefixNames() {
+    using namespace ::tensorflow::ops;  // NOLINT(build/namespaces)
+
+    auto o_root = tensorflow::Scope::NewRootScope();
+    auto a = Placeholder(o_root.WithOpName("placeholder"), DT_FLOAT);
+    auto b = Placeholder(o_root.WithOpName("placeholder_1"), DT_FLOAT);
+    auto add_o = Add(o_root.WithOpName("add"), a, b);
+    GraphDef o_graph_def;
+    TF_ASSERT_OK(o_root.ToGraphDef(&o_graph_def));
+
+    auto n_root = tensorflow::Scope::NewRootScope();
+    auto c = _Recv(n_root.WithOpName("_recv_placeholder_0"), DT_FLOAT, "", "",
+                   0, "");
+    auto d = _Recv(n_root.WithOpName("_recv_placeholder_1_0"), DT_FLOAT, "", "",
+                   0, "");
+    auto add_n = Add(n_root.WithOpName("add"), c, d);
+    GraphDef n_graph_def;
+    TF_ASSERT_OK(n_root.ToGraphDef(&n_graph_def));
+
+    GraphDef result_graph_def;
+    TF_ASSERT_OK(graph_transforms::ReplaceSendRecvs(
+        o_graph_def, n_graph_def, {"placeholder", "placeholder_1"}, {"add"},
+        &result_graph_def));
+
+    std::map<string, const NodeDef*> node_map;
+    graph_transforms::MapNamesToNodes(result_graph_def, &node_map);
+    EXPECT_EQ(1, node_map.count("placeholder"));
+    EXPECT_EQ(1, node_map.count("placeholder_1"));
+    EXPECT_EQ(1, node_map.count("add"));
+  }
+
   void TestRemoveUnusedNodes() {
     using namespace ::tensorflow::ops;  // NOLINT(build/namespaces)
     auto root = tensorflow::Scope::NewRootScope();
-    using namespace ::tensorflow::ops;  // NOLINT(build/namespaces)
 
     const int width = 100;
 
@@ -295,6 +328,48 @@ class ConstantFoldingTest : public ::testing::Test {
     EXPECT_EQ(1, node_map.count("output"));
     EXPECT_EQ(0, node_map.count("unused"));
   }
+
+  void TestRemoveUnusedNodesMultipleOutputs() {
+    using namespace ::tensorflow::ops;  // NOLINT(build/namespaces)
+    auto root = tensorflow::Scope::NewRootScope();
+
+    //    a    b
+    //     \  /
+    //    shape_n
+    //     \  /
+    //       c
+    auto a = Placeholder(root.WithOpName("a"), DT_FLOAT);
+    auto b = Placeholder(root.WithOpName("b"), DT_FLOAT);
+    auto shape_n = ShapeN(root.WithOpName("shape_n"), {Output(a), Output(b)});
+    auto c = Add(root.WithOpName("c"), shape_n[0], shape_n[1]);
+
+    GraphDef graph_def;
+    TF_ASSERT_OK(root.ToGraphDef(&graph_def));
+    GraphDef result_graph_def;
+    TF_ASSERT_OK(graph_transforms::RemoveUnusedNodes(
+        graph_def, {{shape_n[0].name()}, {"c"}}, &result_graph_def));
+
+    // Only one output of shape_n node is fed input. Hence the graph search
+    // should propagate to inputs of shape_n. Nothing to remove here.
+    std::map<string, const NodeDef*> node_map;
+    graph_transforms::MapNamesToNodes(result_graph_def, &node_map);
+    EXPECT_EQ(1, node_map.count("a"));
+    EXPECT_EQ(1, node_map.count("b"));
+    EXPECT_EQ(1, node_map.count("c"));
+
+    result_graph_def.Clear();
+    TF_ASSERT_OK(graph_transforms::RemoveUnusedNodes(
+        graph_def, {{shape_n[0].name(), shape_n[1].name()}, {"c"}},
+        &result_graph_def));
+
+    // Both outputs of shape_n node are fed inputs. shape_n does not function
+    // and inputs to shape_n should be removed.
+    node_map.clear();
+    graph_transforms::MapNamesToNodes(result_graph_def, &node_map);
+    EXPECT_EQ(0, node_map.count("a"));
+    EXPECT_EQ(0, node_map.count("b"));
+    EXPECT_EQ(1, node_map.count("c"));
+  }
 };
 
 TEST_F(ConstantFoldingTest, TestSimpleAdd) { TestSimpleAdd(); }
@@ -309,7 +384,15 @@ TEST_F(ConstantFoldingTest, TestPreserveOutputShapes) {
 
 TEST_F(ConstantFoldingTest, TestReplaceSendRecvs) { TestReplaceSendRecvs(); }
 
+TEST_F(ConstantFoldingTest, TestReplaceSendRecvsPrefixNames) {
+  TestReplaceSendRecvsPrefixNames();
+}
+
 TEST_F(ConstantFoldingTest, TestRemoveUnusedNodes) { TestRemoveUnusedNodes(); }
 
+TEST_F(ConstantFoldingTest, TestRemoveUnusedNodesMultipleOutputs) {
+  TestRemoveUnusedNodesMultipleOutputs();
+}
+
 }  // namespace graph_transforms
 }  // namespace tensorflow
-- 
GitLab


From b7e85339b286f34f215cca3dcb700dbd8f276de3 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 18 Oct 2017 13:02:53 -0700
Subject: [PATCH 525/909] Adds visibility to sgdr_learning_rate_decay.

Currently SGD with warm restarts is siloed in
tensorflow/contrib/training/python/training/sgdr_learning_rate_decay.py, since
it is not listed in the 'training_py' build filegroup. This change simply adds
sgdr_learning_rate_decay to this filegroup so that other projects can use warm
restarts during optimization.

PiperOrigin-RevId: 172643218
---
 tensorflow/contrib/training/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/contrib/training/BUILD b/tensorflow/contrib/training/BUILD
index 80a5debe99..0df5ff50c0 100644
--- a/tensorflow/contrib/training/BUILD
+++ b/tensorflow/contrib/training/BUILD
@@ -26,6 +26,7 @@ py_library(
         "python/training/resample.py",
         "python/training/sampling_ops.py",
         "python/training/sequence_queueing_state_saver.py",
+        "python/training/sgdr_learning_rate_decay.py",
         "python/training/training.py",
         "python/training/tuner.py",
     ],
-- 
GitLab


From f6968a25c9dfc962851806d094dc98f5b502a4f9 Mon Sep 17 00:00:00 2001
From: Jianwei Xie <xiejw@google.com>
Date: Wed, 18 Oct 2017 13:07:01 -0700
Subject: [PATCH 526/909] Add logging verbosity to mnist.py

PiperOrigin-RevId: 172643922
---
 tensorflow/examples/learn/mnist.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tensorflow/examples/learn/mnist.py b/tensorflow/examples/learn/mnist.py
index 5344526b52..88425ea0d0 100644
--- a/tensorflow/examples/learn/mnist.py
+++ b/tensorflow/examples/learn/mnist.py
@@ -97,6 +97,8 @@ def conv_model(features, labels, mode):
 
 
 def main(unused_args):
+  tf.logging.set_verbosity(tf.logging.INFO)
+
   ### Download and load MNIST dataset.
   mnist = tf.contrib.learn.datasets.DATASETS['mnist']('/tmp/mnist')
   train_input_fn = tf.estimator.inputs.numpy_input_fn(
@@ -115,6 +117,7 @@ def main(unused_args):
   feature_columns = [
       tf.feature_column.numeric_column(
           X_FEATURE, shape=mnist.train.images.shape[1:])]
+
   classifier = tf.estimator.LinearClassifier(
       feature_columns=feature_columns, n_classes=N_DIGITS)
   classifier.train(input_fn=train_input_fn, steps=200)
-- 
GitLab


From 08aeb0f960329efa7f477fd184d2e676a96da415 Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Wed, 18 Oct 2017 13:22:01 -0700
Subject: [PATCH 527/909] Automated g4 rollback of changelist 172336111

PiperOrigin-RevId: 172645893
---
 tensorflow/python/framework/ops.py            | 11 ++-
 .../resource_variable_ops_test.py             | 10 +++
 .../python/ops/resource_variable_ops.py       | 33 ++++++++
 tensorflow/python/training/adam_test.py       | 81 ++++++++++---------
 tensorflow/python/training/saver_test.py      | 47 +++++------
 5 files changed, 117 insertions(+), 65 deletions(-)

diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index 75750ecd5a..85b875aa3a 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -2517,7 +2517,14 @@ class Graph(object):
     # A map from tensor handle to its delete op.
     self._handle_deleters = {}
     # Resource container.
-    self._container = ""
+    if context.in_graph_mode():
+      self._container_prefix = ""
+    else:
+      # In Eager mode, isolate resources (particularly ResourceVariables) in
+      # Graphs by default. This prevents unintended variable sharing. Graph mode
+      # gets this kind of isolation from Sessions.
+      self._container_prefix = "eager-execution-%d/" % (uid(),)
+    self._container = self._container_prefix
     self._registered_ops = op_def_registry.get_registered_ops()
 
     # TODO(skyewm): fold as much of the above as possible into the C
@@ -3829,7 +3836,7 @@ class Graph(object):
     """
     original_container = self._container
     try:
-      self._container = container_name
+      self._container = self._container_prefix + container_name
       yield self._container
     finally:
       self._container = original_container
diff --git a/tensorflow/python/kernel_tests/resource_variable_ops_test.py b/tensorflow/python/kernel_tests/resource_variable_ops_test.py
index ec9192b1a0..23676223dc 100644
--- a/tensorflow/python/kernel_tests/resource_variable_ops_test.py
+++ b/tensorflow/python/kernel_tests/resource_variable_ops_test.py
@@ -428,6 +428,16 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase):
       self.assertEqual(1, v1.read_value().numpy())
       self.assertEqual(2, v2.read_value().numpy())
 
+  def testDestruction(self):
+    with context.eager_mode():
+      var = resource_variable_ops.ResourceVariable(initial_value=1.0,
+                                                   name="var8")
+      var.__del__()
+      with self.assertRaisesRegexp(errors.NotFoundError,
+                                   r"Resource .*\/var8\/.* does not exist."):
+        resource_variable_ops.destroy_resource_op(var._handle,
+                                                  ignore_lookup_error=False)
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/ops/resource_variable_ops.py b/tensorflow/python/ops/resource_variable_ops.py
index 2c9a3ff19a..dd3f167145 100644
--- a/tensorflow/python/ops/resource_variable_ops.py
+++ b/tensorflow/python/ops/resource_variable_ops.py
@@ -427,6 +427,39 @@ class ResourceVariable(variables.Variable):
     self._constraint = None
   # LINT.ThenChange(//tensorflow/python/eager/graph_callable.py)
 
+  def __del__(self):
+    if not self._in_graph_mode:
+      # There is only one ResourceVariable object for each underlying resource
+      # (cached in the Graph's VariableStore when created with get_variable), so
+      # it is safe to delete the resource we have a handle to. Each Graph has a
+      # unique container name in Eager, which prevents resource sharing.
+      #
+      # The Graph's VariableStore contains strong references to ResourceVariable
+      # objects created with get_variable, so this destructor will only be
+      # callled once the Graph is garbage collected for those objects. However,
+      # explicitly created ResourceVariables (e.g. through tfe.Variable) may be
+      # collected earlier.
+      try:
+        # We have checked that this ResourceVariable was created in Eager
+        # mode. However, this destructor may be running in graph mode
+        # (especially during unit tests). To clean up successfully, we switch
+        # back into Eager temporarily.
+        with context.eager_mode():
+          with ops.device(self._handle_device):
+            gen_resource_variable_ops.destroy_resource_op(
+                self._handle, ignore_lookup_error=True)
+      except TypeError:
+        # Suppress some exceptions, mainly for the case when we're running on
+        # module deletion. Things that can go wrong include the context module
+        # already being unloaded, self._handle._handle_data no longer being
+        # valid, and so on. Printing warnings in these cases is silly
+        # (exceptions raised from __del__ are printed as warnings to stderr).
+        pass  # 'NoneType' object is not callable when the handle has been
+              # partially unloaded.
+      except AttributeError:
+        pass  # 'NoneType' object has no attribute 'eager_mode' when context has
+              # been unloaded. Will catch other module unloads as well.
+
   @property
   def dtype(self):
     """The dtype of this variable."""
diff --git a/tensorflow/python/training/adam_test.py b/tensorflow/python/training/adam_test.py
index defcf33714..176d20bd60 100644
--- a/tensorflow/python/training/adam_test.py
+++ b/tensorflow/python/training/adam_test.py
@@ -152,53 +152,54 @@ class AdamOptimizerTest(test.TestCase):
 
   def doTestBasic(self, use_resource=False):
     for i, dtype in enumerate([dtypes.half, dtypes.float32, dtypes.float64]):
-      # Initialize variables for numpy implementation.
-      m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0
-      var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype)
-      grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype)
-      var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype)
-      grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype)
-
-      if use_resource:
-        var0 = resource_variable_ops.ResourceVariable(
-            var0_np, name="var0_%d" % i)
-        var1 = resource_variable_ops.ResourceVariable(
-            var1_np, name="var1_%d" % i)
-      else:
-        var0 = variables.Variable(var0_np)
-        var1 = variables.Variable(var1_np)
-      grads0 = constant_op.constant(grads0_np)
-      grads1 = constant_op.constant(grads1_np)
-
-      opt = adam.AdamOptimizer()
-      update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
+      with self.test_session(graph=ops.Graph()):
+        # Initialize variables for numpy implementation.
+        m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0
+        var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype)
+        grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype)
+        var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype)
+        grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype)
 
-      if context.in_graph_mode():
-        self.evaluate(variables.global_variables_initializer())
-        # Fetch params to validate initial values
-        self.assertAllClose([1.0, 2.0], self.evaluate(var0))
-        self.assertAllClose([3.0, 4.0], self.evaluate(var1))
+        if use_resource:
+          var0 = resource_variable_ops.ResourceVariable(
+              var0_np, name="var0_%d" % i)
+          var1 = resource_variable_ops.ResourceVariable(
+              var1_np, name="var1_%d" % i)
+        else:
+          var0 = variables.Variable(var0_np)
+          var1 = variables.Variable(var1_np)
+        grads0 = constant_op.constant(grads0_np)
+        grads1 = constant_op.constant(grads1_np)
 
-      beta1_power, beta2_power = opt._get_beta_accumulators()
+        opt = adam.AdamOptimizer()
+        update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
 
-      # Run 3 steps of Adam
-      for t in range(1, 4):
         if context.in_graph_mode():
-          self.evaluate(update)
-        elif t > 1:
-          opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
+          self.evaluate(variables.global_variables_initializer())
+          # Fetch params to validate initial values
+          self.assertAllClose([1.0, 2.0], self.evaluate(var0))
+          self.assertAllClose([3.0, 4.0], self.evaluate(var1))
 
-        self.assertAllCloseAccordingToType(0.9**(t + 1),
-                                           self.evaluate(beta1_power))
-        self.assertAllCloseAccordingToType(0.999**(t + 1),
-                                           self.evaluate(beta2_power))
+        beta1_power, beta2_power = opt._get_beta_accumulators()
 
-        var0_np, m0, v0 = adam_update_numpy(var0_np, grads0_np, t, m0, v0)
-        var1_np, m1, v1 = adam_update_numpy(var1_np, grads1_np, t, m1, v1)
+        # Run 3 steps of Adam
+        for t in range(1, 4):
+          if context.in_graph_mode():
+            self.evaluate(update)
+          elif t > 1:
+            opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
+
+          self.assertAllCloseAccordingToType(0.9**(t + 1),
+                                             self.evaluate(beta1_power))
+          self.assertAllCloseAccordingToType(0.999**(t + 1),
+                                             self.evaluate(beta2_power))
+
+          var0_np, m0, v0 = adam_update_numpy(var0_np, grads0_np, t, m0, v0)
+          var1_np, m1, v1 = adam_update_numpy(var1_np, grads1_np, t, m1, v1)
 
-        # Validate updated params
-        self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0))
-        self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1))
+          # Validate updated params
+          self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0))
+          self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1))
 
   def testBasic(self):
     with self.test_session():
diff --git a/tensorflow/python/training/saver_test.py b/tensorflow/python/training/saver_test.py
index 07cd67a4b9..aeb8eaffe8 100644
--- a/tensorflow/python/training/saver_test.py
+++ b/tensorflow/python/training/saver_test.py
@@ -479,14 +479,14 @@ class SaverTest(test.TestCase):
       self.assertEqual(30.0, v2_2.values().eval())
 
   def _SaveAndLoad(self, var_name, var_value, other_value, save_path):
-    with self.test_session() as sess:
+    with self.test_session(graph=ops_lib.Graph()) as sess:
       var = resource_variable_ops.ResourceVariable(var_value, name=var_name)
       save = saver_module.Saver({var_name: var})
       if context.in_graph_mode():
         self.evaluate(var.initializer)
       val = save.save(sess, save_path)
       self.assertEqual(save_path, val)
-    with self.test_session() as sess:
+    with self.test_session(graph=ops_lib.Graph()) as sess:
       var = resource_variable_ops.ResourceVariable(other_value, name=var_name)
       save = saver_module.Saver({var_name: var})
       save.restore(sess, save_path)
@@ -619,27 +619,28 @@ class SaverTest(test.TestCase):
     # Save and reload one Variable named "var0".
     self._SaveAndLoad("var0", 0.0, 1.0, save_path)
     for use_tensor in [True, False]:
-      var = resource_variable_ops.ResourceVariable(1.0, name="var0")
-      save = saver_module.Saver(
-          {
-              var._shared_name: var
-          }, pad_step_number=pad_step_number)
-      if context.in_graph_mode():
-        self.evaluate(var.initializer)
-        sess = ops_lib.get_default_session()
-      else:
-        sess = None
-      if use_tensor:
-        global_step = constant_op.constant(global_step_int)
-        val = save.save(sess, save_path, global_step=global_step)
-      else:
-        val = save.save(sess, save_path, global_step=global_step_int)
-      if pad_step_number:
-        expected_save_path = "%s-%s" % (save_path,
-                                        "{:08d}".format(global_step_int))
-      else:
-        expected_save_path = "%s-%d" % (save_path, global_step_int)
-      self.assertEqual(expected_save_path, val)
+      with self.test_session(graph=ops_lib.Graph()):
+        var = resource_variable_ops.ResourceVariable(1.0, name="var0")
+        save = saver_module.Saver(
+            {
+                var._shared_name: var
+            }, pad_step_number=pad_step_number)
+        if context.in_graph_mode():
+          self.evaluate(var.initializer)
+          sess = ops_lib.get_default_session()
+        else:
+          sess = None
+        if use_tensor:
+          global_step = constant_op.constant(global_step_int)
+          val = save.save(sess, save_path, global_step=global_step)
+        else:
+          val = save.save(sess, save_path, global_step=global_step_int)
+        if pad_step_number:
+          expected_save_path = "%s-%s" % (save_path,
+                                          "{:08d}".format(global_step_int))
+        else:
+          expected_save_path = "%s-%d" % (save_path, global_step_int)
+        self.assertEqual(expected_save_path, val)
 
   def testSaveWithGlobalStepWithPadding(self):
     self.testSaveWithGlobalStep(pad_step_number=True)
-- 
GitLab


From d65f7b9077b7191d3aa3f1183b6b119c480faa05 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 18 Oct 2017 13:26:24 -0700
Subject: [PATCH 528/909] Correct the docstring to reflect that the values of
 cols_to_vars are always lists of Variable's (never single Variable's or
 PartitionedVariables), and make this true for bias.

PiperOrigin-RevId: 172646456
---
 tensorflow/python/feature_column/BUILD        |  1 +
 .../python/feature_column/feature_column.py   | 33 +++++-----
 .../feature_column/feature_column_test.py     | 61 +++++++++++++++++--
 3 files changed, 77 insertions(+), 18 deletions(-)

diff --git a/tensorflow/python/feature_column/BUILD b/tensorflow/python/feature_column/BUILD
index 27062adb61..b1c81dd58c 100644
--- a/tensorflow/python/feature_column/BUILD
+++ b/tensorflow/python/feature_column/BUILD
@@ -86,6 +86,7 @@ py_test(
         "//tensorflow/python:framework_ops",
         "//tensorflow/python:lookup_ops",
         "//tensorflow/python:parsing_ops",
+        "//tensorflow/python:partitioned_variables",
         "//tensorflow/python:session",
         "//tensorflow/python:sparse_tensor",
         "//tensorflow/python:training",
diff --git a/tensorflow/python/feature_column/feature_column.py b/tensorflow/python/feature_column/feature_column.py
index 81f4f45fcb..190a25d4d7 100644
--- a/tensorflow/python/feature_column/feature_column.py
+++ b/tensorflow/python/feature_column/feature_column.py
@@ -197,12 +197,13 @@ def input_layer(features,
     trainable: If `True` also add the variable to the graph collection
       `GraphKeys.TRAINABLE_VARIABLES` (see `tf.Variable`).
     cols_to_vars: If not `None`, must be a dictionary that will be filled with a
-      mapping from `_FeatureColumn` to associated `Variable` (or list of
-      `Variable`, or `PartitionedVariable`.  For example, after the call, we
-      might have cols_to_vars = {_EmbeddingColumn(
+      mapping from `_FeatureColumn` to list of `Variable`s.  For example, after
+      the call, we might have cols_to_vars =
+      {_EmbeddingColumn(
         categorical_column=_HashedCategoricalColumn(
           key='sparse_feature', hash_bucket_size=5, dtype=tf.string),
-        dimension=10): [<tf.Variable 'some_variable' shape=(5, 10)]}
+        dimension=10): [<tf.Variable 'some_variable:0' shape=(5, 10),
+                        <tf.Variable 'some_variable:1' shape=(5, 10)]}
       If a column creates no variables, its value will be an empty list.
 
   Returns:
@@ -302,18 +303,18 @@ def linear_model(features,
     trainable: If `True` also add the variable to the graph collection
       `GraphKeys.TRAINABLE_VARIABLES` (see `tf.Variable`).
     cols_to_vars: If not `None`, must be a dictionary that will be filled with a
-      mapping from `_FeatureColumn` to associated `Variable` (or list of
-      `Variable`, or `PartitionedVariable`.  For example,
-      after the call, we might have cols_to_vars = {
+      mapping from `_FeatureColumn` to associated list of `Variable`s.  For
+      example, after the call, we might have cols_to_vars = {
         _NumericColumn(
           key='numeric_feature1', shape=(1,):
-        <tf.Variable 'linear_model/price2/weights:0' shape=(1, 1)>,
-        'bias': <tf.Variable 'linear_model/bias_weights:0' shape=(1,)>,
+        [<tf.Variable 'linear_model/price2/weights:0' shape=(1, 1)>],
+        'bias': [<tf.Variable 'linear_model/bias_weights:0' shape=(1,)>],
         _NumericColumn(
           key='numeric_feature2', shape=(2,)):
-        <tf.Variable 'linear_model/price1/weights:0' shape=(2, 1)>}
-      Note that it will also contain a string key 'bias'.  If a column creates
-      no variables, its value will be an empty list.
+        [<tf.Variable 'linear_model/price1/weights:0' shape=(2, 1)>]}
+      If a column creates no variables, its value will be an empty list. Note
+      that cols_to_vars will also contain a string key 'bias' that maps to a
+      list of Variables.
 
   Returns:
     A `Tensor` which represents predictions/logits of a linear model. Its shape
@@ -366,8 +367,12 @@ def linear_model(features,
     predictions = nn_ops.bias_add(
         predictions_no_bias, bias, name='weighted_sum')
     if cols_to_vars is not None:
-      # Add the bias to cols_to_vars as well.
-      cols_to_vars['bias'] = bias
+      # Add the bias to cols_to_vars as well, converting the Variable or
+      # PartitionedVariable to a list of Variable's.
+      if isinstance(bias, variables.Variable):
+        cols_to_vars['bias'] = [bias]
+      else:  # Must be a PartitionedVariable.
+        cols_to_vars['bias'] = list(bias)
     return predictions
 
 
diff --git a/tensorflow/python/feature_column/feature_column_test.py b/tensorflow/python/feature_column/feature_column_test.py
index 112600439b..e57e9a9836 100644
--- a/tensorflow/python/feature_column/feature_column_test.py
+++ b/tensorflow/python/feature_column/feature_column_test.py
@@ -41,6 +41,7 @@ from tensorflow.python.framework import sparse_tensor
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import lookup_ops
 from tensorflow.python.ops import parsing_ops
+from tensorflow.python.ops import partitioned_variables
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops import variables as variables_lib
 from tensorflow.python.platform import test
@@ -1354,10 +1355,33 @@ class LinearModelTest(test.TestCase):
       bias = get_linear_model_bias()
       price1_var = get_linear_model_column_var(price1)
       price2_var = get_linear_model_column_var(price2)
-      self.assertEqual(cols_to_vars['bias'], bias)
+      self.assertAllEqual(cols_to_vars['bias'], [bias])
       self.assertAllEqual(cols_to_vars[price1], [price1_var])
       self.assertAllEqual(cols_to_vars[price2], [price2_var])
 
+  def test_fills_cols_to_vars_partitioned_variables(self):
+    price1 = fc.numeric_column('price1', shape=2)
+    price2 = fc.numeric_column('price2', shape=3)
+    with ops.Graph().as_default():
+      features = {
+          'price1': [[1., 2.], [6., 7.]],
+          'price2': [[3., 4., 5.], [8., 9., 10.]]
+      }
+      cols_to_vars = {}
+      with variable_scope.variable_scope(
+          'linear',
+          partitioner=partitioned_variables.fixed_size_partitioner(2, axis=0)):
+        fc.linear_model(features, [price1, price2], cols_to_vars=cols_to_vars)
+      with _initialized_session():
+        self.assertEqual([0.], cols_to_vars['bias'][0].eval())
+        # Partitioning shards the [2, 1] price1 var into 2 [1, 1] Variables.
+        self.assertAllEqual([[0.]], cols_to_vars[price1][0].eval())
+        self.assertAllEqual([[0.]], cols_to_vars[price1][1].eval())
+        # Partitioning shards the [3, 1] price2 var into a [2, 1] Variable and
+        # a [1, 1] Variable.
+        self.assertAllEqual([[0.], [0.]], cols_to_vars[price2][0].eval())
+        self.assertAllEqual([[0.]], cols_to_vars[price2][1].eval())
+
   def test_dense_collection(self):
     price = fc.numeric_column('price')
     with ops.Graph().as_default() as g:
@@ -1761,9 +1785,38 @@ class InputLayerTest(test.TestCase):
       self.assertEqual(0, len(cols_to_vars[price1]))
       self.assertEqual(0, len(cols_to_vars[dense_feature_bucketized]))
       self.assertEqual(1, len(cols_to_vars[some_embedding_column]))
-      for var in cols_to_vars[some_embedding_column]:
-        self.assertIsInstance(var, variables_lib.Variable)
-        self.assertAllEqual(var.shape, [5, 10])
+      self.assertIsInstance(cols_to_vars[some_embedding_column][0],
+                            variables_lib.Variable)
+      self.assertAllEqual(cols_to_vars[some_embedding_column][0].shape, [5, 10])
+
+  def test_fills_cols_to_vars_partitioned_variables(self):
+    price1 = fc.numeric_column('price1')
+    dense_feature = fc.numeric_column('dense_feature')
+    dense_feature_bucketized = fc.bucketized_column(
+        dense_feature, boundaries=[0.])
+    some_sparse_column = fc.categorical_column_with_hash_bucket(
+        'sparse_feature', hash_bucket_size=5)
+    some_embedding_column = fc.embedding_column(
+        some_sparse_column, dimension=10)
+    with ops.Graph().as_default():
+      features = {
+          'price1': [[3.], [4.]],
+          'dense_feature': [[-1.], [4.]],
+          'sparse_feature': [['a'], ['x']],
+      }
+      cols_to_vars = {}
+      all_cols = [price1, dense_feature_bucketized, some_embedding_column]
+      with variable_scope.variable_scope(
+          'input_from_feature_columns',
+          partitioner=partitioned_variables.fixed_size_partitioner(3, axis=0)):
+        fc.input_layer(features, all_cols, cols_to_vars=cols_to_vars)
+      self.assertItemsEqual(list(cols_to_vars.keys()), all_cols)
+      self.assertEqual(0, len(cols_to_vars[price1]))
+      self.assertEqual(0, len(cols_to_vars[dense_feature_bucketized]))
+      self.assertEqual(3, len(cols_to_vars[some_embedding_column]))
+      self.assertAllEqual(cols_to_vars[some_embedding_column][0].shape, [2, 10])
+      self.assertAllEqual(cols_to_vars[some_embedding_column][1].shape, [2, 10])
+      self.assertAllEqual(cols_to_vars[some_embedding_column][2].shape, [1, 10])
 
   def test_column_order(self):
     price_a = fc.numeric_column('price_a')
-- 
GitLab


From 5565aac7876a1eacdae29ac24d95c0e94c9062c6 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 18 Oct 2017 13:33:29 -0700
Subject: [PATCH 529/909] Changes MultiLabelHead.create_loss to return a Tensor
 of size [batch_size, 1], to be consistent with other heads.

PiperOrigin-RevId: 172647355
---
 .../estimator/python/estimator/head.py        | 16 +++++-----
 .../estimator/python/estimator/head_test.py   | 32 +++++++++----------
 2 files changed, 24 insertions(+), 24 deletions(-)

diff --git a/tensorflow/contrib/estimator/python/estimator/head.py b/tensorflow/contrib/estimator/python/estimator/head.py
index f8648fe5bf..ebf91e8bb4 100644
--- a/tensorflow/contrib/estimator/python/estimator/head.py
+++ b/tensorflow/contrib/estimator/python/estimator/head.py
@@ -265,6 +265,9 @@ class _MultiLabelHead(head_lib._Head):  # pylint:disable=protected-access
     unweighted_loss = losses.sigmoid_cross_entropy(
         multi_class_labels=processed_labels, logits=logits,
         reduction=losses.Reduction.NONE)
+    # Averages loss over classes.
+    unweighted_loss = math_ops.reduce_mean(
+        unweighted_loss, axis=-1, keep_dims=True)
     return head_lib.LossAndLabels(
         unweighted_loss=unweighted_loss,
         processed_labels=processed_labels)
@@ -294,12 +297,9 @@ class _MultiLabelHead(head_lib._Head):  # pylint:disable=protected-access
       # Eval.
       unweighted_loss, processed_labels = self.create_loss(
           features=features, mode=mode, logits=logits, labels=labels)
-      # Averages loss over classes.
-      per_example_loss = math_ops.reduce_mean(
-          unweighted_loss, axis=-1, keep_dims=True)
       weights = head_lib._weights(features, self._weight_column)  # pylint:disable=protected-access
       training_loss = losses.compute_weighted_loss(
-          per_example_loss, weights=weights, reduction=losses.Reduction.SUM)
+          unweighted_loss, weights=weights, reduction=losses.Reduction.SUM)
       if mode == model_fn.ModeKeys.EVAL:
         return model_fn.EstimatorSpec(
             mode=model_fn.ModeKeys.EVAL,
@@ -309,7 +309,7 @@ class _MultiLabelHead(head_lib._Head):  # pylint:disable=protected-access
                 labels=processed_labels,
                 probabilities=probabilities,
                 weights=weights,
-                per_example_loss=per_example_loss))
+                unweighted_loss=unweighted_loss))
 
       # Train.
       if train_op_fn is None:
@@ -330,16 +330,16 @@ class _MultiLabelHead(head_lib._Head):  # pylint:disable=protected-access
         loss=training_loss,
         train_op=train_op_fn(training_loss))
 
-  def _eval_metric_ops(self, labels, probabilities, weights, per_example_loss):
+  def _eval_metric_ops(self, labels, probabilities, weights, unweighted_loss):
     """Returns a dict of metrics for eval_metric_ops."""
     with ops.name_scope(
-        None, 'metrics', [labels, probabilities, weights, per_example_loss]):
+        None, 'metrics', [labels, probabilities, weights, unweighted_loss]):
       keys = metric_keys.MetricKeys
       metric_ops = {
           # Estimator already adds a metric for loss.
           head_lib._summary_key(self._name, keys.LOSS_MEAN):  # pylint:disable=protected-access
               metrics_lib.mean(
-                  per_example_loss, weights=weights, name=keys.LOSS_MEAN),
+                  unweighted_loss, weights=weights, name=keys.LOSS_MEAN),
           head_lib._summary_key(self._name, keys.AUC):  # pylint:disable=protected-access
               metrics_lib.auc(
                   labels=labels, predictions=probabilities, weights=weights,
diff --git a/tensorflow/contrib/estimator/python/estimator/head_test.py b/tensorflow/contrib/estimator/python/estimator/head_test.py
index dcbe62b497..ec1386af34 100644
--- a/tensorflow/contrib/estimator/python/estimator/head_test.py
+++ b/tensorflow/contrib/estimator/python/estimator/head_test.py
@@ -80,9 +80,13 @@ def _sigmoid(logits):
 
 
 def _sigmoid_cross_entropy(labels, logits):
+  """Returns sigmoid cross entropy averaged over classes."""
   sigmoid_logits = _sigmoid(logits)
-  return (-labels * np.log(sigmoid_logits)
-          -(1 - labels) * np.log(1 - sigmoid_logits))
+  unreduced_result = (
+      -labels * np.log(sigmoid_logits)
+      -(1 - labels) * np.log(1 - sigmoid_logits))
+  # Mean over classes
+  return np.mean(unreduced_result, axis=-1, keepdims=True)
 
 
 class MultiLabelHead(test.TestCase):
@@ -226,7 +230,7 @@ class MultiLabelHead(test.TestCase):
     # loss = labels * (logits < 0) * (-logits) +
     #        (1 - labels) * (logits > 0) * logits
     expected_unweighted_loss = np.array(
-        [[10., 10.], [15., 0.]], dtype=np.float32)
+        [[(10. + 10.) / 2.], [(15. + 0.) / 2.]], dtype=np.float32)
     actual_unweighted_loss, _ = head.create_loss(
         features={'x': np.array(((42,),), dtype=np.int32)},
         mode=model_fn.ModeKeys.EVAL,
@@ -311,10 +315,8 @@ class MultiLabelHead(test.TestCase):
     labels = np.array([[1, 0], [1, 1]], dtype=np.int64)
     # loss = labels * -log(sigmoid(logits)) +
     #        (1 - labels) * -log(1 - sigmoid(logits))
-    # Average over classes, and sum over examples.
-    expected_loss = (
-        np.sum(_sigmoid_cross_entropy(labels=labels, logits=logits)) / n_classes
-    )
+    # Sum over examples.
+    expected_loss = np.sum(_sigmoid_cross_entropy(labels=labels, logits=logits))
     keys = metric_keys.MetricKeys
     expected_metrics = {
         # Average loss over examples.
@@ -343,10 +345,9 @@ class MultiLabelHead(test.TestCase):
     labels_multi_hot = np.array([[1, 0], [1, 1]], dtype=np.int64)
     # loss = labels * -log(sigmoid(logits)) +
     #        (1 - labels) * -log(1 - sigmoid(logits))
-    # Average over classes, and sum over examples.
+    # Sum over examples.
     expected_loss = (
-        np.sum(_sigmoid_cross_entropy(labels=labels_multi_hot, logits=logits)) /
-        n_classes
+        np.sum(_sigmoid_cross_entropy(labels=labels_multi_hot, logits=logits))
     )
     keys = metric_keys.MetricKeys
     expected_metrics = {
@@ -377,10 +378,9 @@ class MultiLabelHead(test.TestCase):
     labels_multi_hot = np.array([[1, 0], [1, 1]], dtype=np.int64)
     # loss = labels * -log(sigmoid(logits)) +
     #        (1 - labels) * -log(1 - sigmoid(logits))
-    # Average over classes, and sum over examples.
+    # Sum over examples.
     expected_loss = (
-        np.sum(_sigmoid_cross_entropy(labels=labels_multi_hot, logits=logits)) /
-        n_classes
+        np.sum(_sigmoid_cross_entropy(labels=labels_multi_hot, logits=logits))
     )
     keys = metric_keys.MetricKeys
     expected_metrics = {
@@ -407,9 +407,9 @@ class MultiLabelHead(test.TestCase):
     labels = np.array([[1, 0], [1, 1]], dtype=np.int64)
     # loss = labels * -log(sigmoid(logits)) +
     #        (1 - labels) * -log(1 - sigmoid(logits))
-    # Average over classes, and sum over examples.
+    # Sum over examples.
     expected_loss = (
-        np.sum(_sigmoid_cross_entropy(labels=labels, logits=logits)) / n_classes
+        np.sum(_sigmoid_cross_entropy(labels=labels, logits=logits))
     )
 
     keys = metric_keys.MetricKeys
@@ -506,7 +506,7 @@ class MultiLabelHead(test.TestCase):
     # loss = labels * (logits < 0) * (-logits) +
     #        (1 - labels) * (logits > 0) * logits
     expected_unweighted_loss = np.array(
-        [[10., 10.], [15., 0.]], dtype=np.float32)
+        [[(10. + 10.) / 2.], [(15. + 0.) / 2.]], dtype=np.float32)
     actual_unweighted_loss, _ = head.create_loss(
         features={'x': np.array(((42,),), dtype=np.int32)},
         mode=model_fn.ModeKeys.TRAIN,
-- 
GitLab


From cadcda216ec7d6f5f3e36dfc7863634f4f03f71f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 18 Oct 2017 14:03:07 -0700
Subject: [PATCH 530/909] Implementation of the Swish activation function.

PiperOrigin-RevId: 172651500
---
 tensorflow/python/ops/nn.py                   |  1 +
 tensorflow/python/ops/nn_impl.py              | 42 +++++++++++++++++++
 tensorflow/python/ops/nn_test.py              | 27 ++++++++++++
 .../tools/api/golden/tensorflow.nn.pbtxt      |  4 ++
 4 files changed, 74 insertions(+)

diff --git a/tensorflow/python/ops/nn.py b/tensorflow/python/ops/nn.py
index a80662c8b5..79af3ac117 100644
--- a/tensorflow/python/ops/nn.py
+++ b/tensorflow/python/ops/nn.py
@@ -21,6 +21,7 @@ See the @{$python/nn} guide.
 @@relu
 @@relu6
 @@crelu
+@@swish
 @@elu
 @@leaky_relu
 @@selu
diff --git a/tensorflow/python/ops/nn_impl.py b/tensorflow/python/ops/nn_impl.py
index db8e92831e..2c83e4e29f 100644
--- a/tensorflow/python/ops/nn_impl.py
+++ b/tensorflow/python/ops/nn_impl.py
@@ -22,6 +22,7 @@ import math
 
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import function
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import candidate_sampling_ops
@@ -269,6 +270,47 @@ def relu_layer(x, weights, biases, name=None):
     return nn_ops.relu(xw_plus_b, name=name)
 
 
+def _swish_shape(op):
+  """Shape helper function for swish and _swish_grad function below."""
+  return [op.inputs[0].shape]
+
+
+# Set noinline=True so that sigmoid(features) is re-computed during
+# backprop, and we can free the sigmoid(features) expression immediately
+# after use during the forward pass.
+@function.Defun(shape_func=_swish_shape, func_name="swish_grad", noinline=True)
+def _swish_grad(features, grad):
+  """Gradient of Swish function defined below."""
+  sigmoid_features = math_ops.sigmoid(features)
+  activation_grad = (
+      sigmoid_features * (1.0 + features * (1.0 - sigmoid_features)))
+  return grad * activation_grad
+
+
+@function.Defun(
+    grad_func=_swish_grad,
+    shape_func=_swish_shape,
+    func_name="swish",
+    noinline=True)
+def swish(features):
+  # pylint: disable=g-doc-args
+  """Computes the Swish activation function: `x * sigmoid(x)`.
+
+  Source: "Swish: a Self-Gated Activation Function" (Ramachandran et al. 2017)
+  https://arxiv.org/abs/1710.05941
+
+  Args:
+    features: A `Tensor` representing preactivation values.
+    name: A name for the operation (optional).
+
+  Returns:
+    The activation value.
+  """
+  # pylint: enable=g-doc-args
+  features = ops.convert_to_tensor(features, name="features")
+  return features * math_ops.sigmoid(features)
+
+
 def l2_normalize(x, dim, epsilon=1e-12, name=None):
   """Normalizes along dimension `dim` using an L2 norm.
 
diff --git a/tensorflow/python/ops/nn_test.py b/tensorflow/python/ops/nn_test.py
index 3528b60ca7..3b918e4f74 100644
--- a/tensorflow/python/ops/nn_test.py
+++ b/tensorflow/python/ops/nn_test.py
@@ -29,6 +29,7 @@ from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gradient_checker
+from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import nn_impl
 from tensorflow.python.ops import nn_ops
 from tensorflow.python.ops import partitioned_variables
@@ -860,6 +861,32 @@ class LeakyReluTest(test_lib.TestCase):
     self.assertAllClose(outputs, [-0.2, 0.0, 0.5, 1.0, 2.0])
 
 
+class SwishTest(test_lib.TestCase):
+
+  def testValues(self):
+    np_values = np.array(
+        [np.linspace(-10.0, 0.0, 100),
+         np.linspace(0.0, 10.0, 100)],
+        dtype=np.float32)
+    tf_values = constant_op.constant(np_values)
+    actual_tf_outputs = nn_impl.swish(tf_values)
+    expected_tf_outputs = tf_values * math_ops.sigmoid(tf_values)
+    with self.test_session() as sess:
+      actual_outputs, expected_outputs = sess.run(
+          [actual_tf_outputs, expected_tf_outputs])
+    self.assertAllClose(actual_outputs, expected_outputs)
+
+  def testGradients(self):
+    shape = [5, 3, 4]
+    sigma = 5
+    input_values = np.random.randn(*shape) * sigma
+    x_tf = constant_op.constant(input_values)
+    y_tf = nn_impl.swish(x_tf)
+    with self.test_session():
+      err = gradient_checker.compute_gradient_error(x_tf, shape, y_tf, shape)
+    self.assertLess(err, 1e-4)
+
+
 class MomentsTest(test_lib.TestCase):
 
   def doOutputTest(self, input_shape, moments_axes, tol=1e-4,
diff --git a/tensorflow/tools/api/golden/tensorflow.nn.pbtxt b/tensorflow/tools/api/golden/tensorflow.nn.pbtxt
index f10299377b..11637814a6 100644
--- a/tensorflow/tools/api/golden/tensorflow.nn.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.nn.pbtxt
@@ -4,6 +4,10 @@ tf_module {
     name: "rnn_cell"
     mtype: "<type \'module\'>"
   }
+  member {
+    name: "swish"
+    mtype: "<class \'tensorflow.python.framework.function._OverloadedFunction\'>"
+  }
   member_method {
     name: "all_candidate_sampler"
     argspec: "args=[\'true_classes\', \'num_true\', \'num_sampled\', \'unique\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
-- 
GitLab


From 71cea5ba4eafabb4a5515025bd1b6106faa0c958 Mon Sep 17 00:00:00 2001
From: Jianwei Xie <xiejw@google.com>
Date: Wed, 18 Oct 2017 14:06:53 -0700
Subject: [PATCH 531/909] Modify the learn examples wide_and_deep to use
 tf.estimator.train_and_evaluate.

PiperOrigin-RevId: 172652065
---
 .../examples/learn/wide_n_deep_tutorial.py    | 55 +++++++++++--------
 1 file changed, 32 insertions(+), 23 deletions(-)

diff --git a/tensorflow/examples/learn/wide_n_deep_tutorial.py b/tensorflow/examples/learn/wide_n_deep_tutorial.py
index 7b9381311c..e447b3e24e 100644
--- a/tensorflow/examples/learn/wide_n_deep_tutorial.py
+++ b/tensorflow/examples/learn/wide_n_deep_tutorial.py
@@ -107,6 +107,9 @@ deep_columns = [
 ]
 
 
+FLAGS = None
+
+
 def maybe_download(train_data, test_data):
   """Maybe downloads training data and returns train and test file names."""
   if train_data:
@@ -154,7 +157,14 @@ def build_estimator(model_dir, model_type):
 
 
 def input_fn(data_file, num_epochs, shuffle):
-  """Input builder function."""
+  """Returns an `input_fn` required by Estimator train/evaluate.
+
+  Args:
+    data_file: The file path to the dataset.
+    num_epochs: Number of epochs to iterate over data. If `None`, `input_fn`
+      will generate infinite stream of data.
+    shuffle: bool, whether to read the data in random order.
+  """
   df_data = pd.read_csv(
       tf.gfile.Open(data_file),
       names=CSV_COLUMNS,
@@ -164,43 +174,42 @@ def input_fn(data_file, num_epochs, shuffle):
   # remove NaN elements
   df_data = df_data.dropna(how="any", axis=0)
   labels = df_data["income_bracket"].apply(lambda x: ">50K" in x).astype(int)
+
   return tf.estimator.inputs.pandas_input_fn(
       x=df_data,
       y=labels,
       batch_size=100,
       num_epochs=num_epochs,
       shuffle=shuffle,
-      num_threads=5)
+      num_threads=1)
 
 
-def train_and_eval(model_dir, model_type, train_steps, train_data, test_data):
-  """Train and evaluate the model."""
-  train_file_name, test_file_name = maybe_download(train_data, test_data)
+def main(_):
+  tf.logging.set_verbosity(tf.logging.INFO)
+
+  train_file_name, test_file_name = maybe_download(FLAGS.train_data,
+                                                   FLAGS.test_data)
+
   # Specify file path below if want to find the output easily
-  model_dir = tempfile.mkdtemp() if not model_dir else model_dir
+  model_dir = FLAGS.model_dir if FLAGS.model_dir else tempfile.mkdtemp()
 
-  m = build_estimator(model_dir, model_type)
-  # set num_epochs to None to get infinite stream of data.
-  m.train(
+  estimator = build_estimator(model_dir, FLAGS.model_type)
+
+  # `tf.estimator.TrainSpec`, `tf.estimator.EvalSpec`, and
+  # `tf.estimator.train_and_evaluate` API are available in TF 1.4.
+  train_spec = tf.estimator.TrainSpec(
       input_fn=input_fn(train_file_name, num_epochs=None, shuffle=True),
-      steps=train_steps)
-  # set steps to None to run evaluation until all data consumed.
-  results = m.evaluate(
+      max_steps=FLAGS.train_steps)
+
+  eval_spec = tf.estimator.EvalSpec(
       input_fn=input_fn(test_file_name, num_epochs=1, shuffle=False),
+      # set steps to None to run evaluation until all data consumed.
       steps=None)
-  print("model directory = %s" % model_dir)
-  for key in sorted(results):
-    print("%s: %s" % (key, results[key]))
-  # Manual cleanup
-  shutil.rmtree(model_dir)
-
-
-FLAGS = None
 
+  tf.estimator.train_and_evaluate(estimator, train_spec, eval_spec)
 
-def main(_):
-  train_and_eval(FLAGS.model_dir, FLAGS.model_type, FLAGS.train_steps,
-                 FLAGS.train_data, FLAGS.test_data)
+  # Manual cleanup
+  shutil.rmtree(model_dir)
 
 
 if __name__ == "__main__":
-- 
GitLab


From 8548e18647d4e574dbf697a81f844b2f0d89bacb Mon Sep 17 00:00:00 2001
From: Akshay Agrawal <akshayka@google.com>
Date: Wed, 18 Oct 2017 14:15:45 -0700
Subject: [PATCH 532/909] Fix typo in error message for set_caching_device

PiperOrigin-RevId: 172653499
---
 tensorflow/python/ops/variable_scope.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/ops/variable_scope.py b/tensorflow/python/ops/variable_scope.py
index 4614110ba6..22048a0cef 100644
--- a/tensorflow/python/ops/variable_scope.py
+++ b/tensorflow/python/ops/variable_scope.py
@@ -999,7 +999,7 @@ class VariableScope(object):
   def set_caching_device(self, caching_device):
     """Set caching_device for this scope."""
     if context.in_eager_mode():
-      raise NotImplementedError("Partitioned variables are not yet supported "
+      raise NotImplementedError("Caching devices are not yet supported "
                                 "in Eager mode.")
     self._caching_device = caching_device
 
-- 
GitLab


From fae8ee3ae5f758e3f6eec33ec01b933084c5d080 Mon Sep 17 00:00:00 2001
From: Akshay Agrawal <akshayka@google.com>
Date: Wed, 18 Oct 2017 14:47:59 -0700
Subject: [PATCH 533/909] Modifies unsupported properties of EagerTensor to
 raise AttributeErrors instead of NotImplementedErrors.

PiperOrigin-RevId: 172658409
---
 tensorflow/python/framework/ops.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index 85b875aa3a..ef0ed8fc53 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -741,19 +741,19 @@ class _EagerTensorBase(Tensor):
   # Methods not supported / implemented for Eager Tensors.
   @property
   def op(self):
-    raise NotImplementedError("op not supported for Eager Tensors.")
+    raise AttributeError("op not supported for Eager Tensors.")
 
   @property
   def graph(self):
-    raise NotImplementedError("graph not supported for Eager Tensors.")
+    raise AttributeError("graph not supported for Eager Tensors.")
 
   @property
   def name(self):
-    raise NotImplementedError("name not supported for Eager Tensors.")
+    raise AttributeError("name not supported for Eager Tensors.")
 
   @property
   def value_index(self):
-    raise NotImplementedError("value_index not supported for Eager Tensors.")
+    raise AttributeError("value_index not supported for Eager Tensors.")
 
   def consumers(self):
     raise NotImplementedError("consumers not supported for Eager Tensors.")
-- 
GitLab


From b2f5acd2c3fbcccb580d6393c0ce77a32ad01279 Mon Sep 17 00:00:00 2001
From: Yunxing Dai <yunxing@google.com>
Date: Wed, 18 Oct 2017 15:06:19 -0700
Subject: [PATCH 534/909] Fix a memory leak in graph compiler.

- Fix a memory leak in graph compiler by using better memory management.
- Simplify the code as I know understand more assumptions of this part of the stack.

PiperOrigin-RevId: 172661754
---
 tensorflow/compiler/tf2xla/graph_compiler.cc  | 46 +++++++------------
 tensorflow/compiler/tf2xla/graph_compiler.h   | 25 +++++-----
 .../xla_jit_compiled_cpu_function_test.cc     | 14 ++++++
 3 files changed, 44 insertions(+), 41 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/graph_compiler.cc b/tensorflow/compiler/tf2xla/graph_compiler.cc
index 6f2f59d98f..9893afa7a0 100644
--- a/tensorflow/compiler/tf2xla/graph_compiler.cc
+++ b/tensorflow/compiler/tf2xla/graph_compiler.cc
@@ -84,7 +84,7 @@ Status PrepareArguments(XlaOpKernelContext* ctx, Graph* graph,
 }
 }  // namespace
 Status GraphCompiler::Compile() {
-  std::vector<NodeBinding> bindings(graph_->num_node_ids());
+  OutputRegistry output_registry(graph_->num_node_ids());
   std::vector<Node*> topo_sorted_nodes;
   // XLA requires determinism, generate a stable ordering from DFS.
   GetReversePostOrder(*graph_, &topo_sorted_nodes,
@@ -94,30 +94,23 @@ Status GraphCompiler::Compile() {
   PartiallySetupParams(&params);
 
   for (Node* n : topo_sorted_nodes) {
-    // Set up bindings.
-    NodeBinding& binding = bindings[n->id()];
-    binding.node = n;
-    Status s = flib_->CreateKernel(n->def(), &binding.op_kernel);
-    binding.output_attrs.resize(n->num_outputs());
+    NodeOutputs node_outputs;
+    OpKernel* op_kernel_raw = nullptr;
+    Status s = flib_->CreateKernel(n->def(), &op_kernel_raw);
+    // Transfer ownership of the kernel to a local smart pointer.
+    std::unique_ptr<OpKernel> op_kernel(op_kernel_raw);
+
     if (!s.ok()) {
-      binding.op_kernel = nullptr;
       s = AttachDef(s, *n);
       LOG(ERROR) << "Executor failed to create kernel. " << s;
       return s;
     }
-  }
-
-  // Bindings are initialized by the size of graph_->num_node_ids. However, the
-  // graph may contain dead nodes that still hold a valid node id. Thus
-  // graph_->num_node_ids could be larger than number of topo sorted nodes.
-  TF_RET_CHECK(bindings.size() >= topo_sorted_nodes.size());
 
-  for (Node* n : topo_sorted_nodes) {
     TF_RET_CHECK(!n->IsRecv() && !n->IsSend() && !n->IsSwitch())
         << "Not supported node: " << n->DebugString();
-    NodeBinding& binding = bindings[n->id()];
-    params.op_kernel = binding.op_kernel;
-    params.output_attr_array = binding.output_attrs.data();
+    params.op_kernel = op_kernel.get();
+    gtl::InlinedVector<AllocatorAttributes, 4> output_attr(n->num_outputs());
+    params.output_attr_array = output_attr.data();
 
     // tensor_inputs_ is a buffer reused across graph traversal. We clean up and
     // reinitialize the buffer before we visit a new node.
@@ -128,8 +121,10 @@ Status GraphCompiler::Compile() {
     for (auto* e : n->in_edges()) {
       if (e->IsControlEdge()) continue;
       Node* src = e->src();
-      tensor_inputs_[e->dst_input()] =
-          bindings[src->id()].tensor_values[e->src_output()];
+      TF_RET_CHECK(src->id() < output_registry.size());
+      const NodeOutputs& outputs = output_registry[src->id()];
+
+      tensor_inputs_[e->dst_input()] = outputs.values[e->src_output()];
     }
 
     OpKernelContext op_context(&params, n->num_outputs());
@@ -150,17 +145,8 @@ Status GraphCompiler::Compile() {
                                 (*op_context.is_output_dead() ? "(dead)" : ""),
                                 SummarizeNode(*n));
       }
-      binding.tensor_values.push_back(tensor_val);
-    }
-  }
-
-  // Clean up tensor data and op kernels.
-  for (NodeBinding& binding : bindings) {
-    delete binding.op_kernel;
-    for (auto& t : binding.tensor_values) {
-      if (!t.is_ref()) {
-        delete t.tensor;
-      }
+      // Set up outputs
+      output_registry[n->id()].values.push_back(tensor_val);
     }
   }
   return Status::OK();
diff --git a/tensorflow/compiler/tf2xla/graph_compiler.h b/tensorflow/compiler/tf2xla/graph_compiler.h
index ccf9351642..33781d2c21 100644
--- a/tensorflow/compiler/tf2xla/graph_compiler.h
+++ b/tensorflow/compiler/tf2xla/graph_compiler.h
@@ -69,20 +69,23 @@ class GraphCompiler {
   Status Compile();
 
  private:
-  // NodeBinding is a wrapper on a `Node` that also contains computed
-  // TensorValue.
-  struct NodeBinding {
-    const Node* node;
-    // Kernel for this node, to be filled by CreateKernel.
-    // TODO(yunxing): Switching this to unique_ptr and understand why it crashes
-    // on GPU devices.
-    OpKernel* op_kernel;
+  // NodeOutputs is a wrapper over TensorValues that represents outputs of a
+  // node.
+  struct NodeOutputs {
+    ~NodeOutputs() {
+      for (auto& v : values) {
+        CHECK(!v.is_ref());
+        delete v.tensor;
+      }
+    }
+
     // Output values of this node.
-    std::vector<TensorValue> tensor_values;
-    // Attributes of the outputs.
-    gtl::InlinedVector<AllocatorAttributes, 4> output_attrs;
+    std::vector<TensorValue> values;
   };
 
+  // A mapping from node id to node output.
+  using OutputRegistry = std::vector<NodeOutputs>;
+
   // Partially sets params. This partially set params can be reused
   // across multple nodes visit.
   void PartiallySetupParams(OpKernelContext::Params* params);
diff --git a/tensorflow/compiler/tf2xla/xla_jit_compiled_cpu_function_test.cc b/tensorflow/compiler/tf2xla/xla_jit_compiled_cpu_function_test.cc
index 5bee68eefc..6d49298a6f 100644
--- a/tensorflow/compiler/tf2xla/xla_jit_compiled_cpu_function_test.cc
+++ b/tensorflow/compiler/tf2xla/xla_jit_compiled_cpu_function_test.cc
@@ -129,5 +129,19 @@ TEST(XlaJitCompiledCpuFunction, Sum) {
   EXPECT_TRUE(ShapeUtil::Compatible(result0, s32));
 }
 
+// Test when a graph compilation terminates early, resources are properly
+// reclaimed.
+TEST(XlaJitCompiledCpuFunction, SumWithJunkAttr) {
+  GraphDef graph_def = SumGraph();
+
+  (*graph_def.mutable_node(2)->mutable_attr())["junk"] =
+      TypeAttrValue(DT_INT32);
+
+  tf2xla::Config config = SumConfig();
+  EXPECT_FALSE(XlaJitCompiledCpuFunction::Compile(graph_def, config,
+                                                  xla::ExecutableBuildOptions())
+                   .ok());
+}
+
 }  // namespace
 }  // namespace tensorflow
-- 
GitLab


From 818644d0937d1e4b097b15d8c823835baba9fbc7 Mon Sep 17 00:00:00 2001
From: "Joshua V. Dillon" <jvdillon@google.com>
Date: Wed, 18 Oct 2017 15:07:43 -0700
Subject: [PATCH 535/909] Add
 tf.contrib.distributions.bijectors.MaskedAutoregressiveFlow.

PiperOrigin-RevId: 172662078
---
 tensorflow/contrib/distributions/BUILD        |  16 +
 .../bijectors/masked_autoregressive_test.py   | 153 ++++++
 .../python/ops/bijectors/__init__.py          |   5 +
 .../ops/bijectors/masked_autoregressive.py    |  33 ++
 .../bijectors/masked_autoregressive_impl.py   | 473 ++++++++++++++++++
 .../distributions/python/ops/test_util.py     |  39 +-
 6 files changed, 703 insertions(+), 16 deletions(-)
 create mode 100644 tensorflow/contrib/distributions/python/kernel_tests/bijectors/masked_autoregressive_test.py
 create mode 100644 tensorflow/contrib/distributions/python/ops/bijectors/masked_autoregressive.py
 create mode 100644 tensorflow/contrib/distributions/python/ops/bijectors/masked_autoregressive_impl.py

diff --git a/tensorflow/contrib/distributions/BUILD b/tensorflow/contrib/distributions/BUILD
index 825ec652d0..1305c28012 100644
--- a/tensorflow/contrib/distributions/BUILD
+++ b/tensorflow/contrib/distributions/BUILD
@@ -854,6 +854,22 @@ cuda_py_test(
     ],
 )
 
+cuda_py_test(
+    name = "masked_autoregressive_test",
+    size = "small",
+    srcs = ["python/kernel_tests/bijectors/masked_autoregressive_test.py"],
+    additional_deps = [
+        ":bijectors_py",
+        ":distributions_py",
+        "//third_party/py/numpy",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework_for_generated_wrappers",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:platform_test",
+    ],
+)
+
 cuda_py_test(
     name = "permute_test",
     size = "small",
diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/masked_autoregressive_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/masked_autoregressive_test.py
new file mode 100644
index 0000000000..98c09545ac
--- /dev/null
+++ b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/masked_autoregressive_test.py
@@ -0,0 +1,153 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for MaskedAutoregressiveFlow."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.contrib.distributions.python.ops import test_util
+from tensorflow.contrib.distributions.python.ops.bijectors.invert import Invert
+from tensorflow.contrib.distributions.python.ops.bijectors.masked_autoregressive import masked_autoregressive_default_template
+from tensorflow.contrib.distributions.python.ops.bijectors.masked_autoregressive import MaskedAutoregressiveFlow
+from tensorflow.contrib.distributions.python.ops.bijectors.masked_autoregressive_impl import _gen_mask
+from tensorflow.python.framework import constant_op
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import variables
+from tensorflow.python.ops.distributions import normal as normal_lib
+from tensorflow.python.ops.distributions import transformed_distribution as transformed_distribution_lib
+from tensorflow.python.platform import test
+
+
+class GenMaskTest(test.TestCase):
+
+  def test346Exclusive(self):
+    expected_mask = np.array(
+        [[0, 0, 0, 0],
+         [0, 0, 0, 0],
+         [1, 0, 0, 0],
+         [1, 0, 0, 0],
+         [1, 1, 0, 0],
+         [1, 1, 0, 0]])
+    mask = _gen_mask(num_blocks=3, n_in=4, n_out=6, mask_type="exclusive")
+    self.assertAllEqual(expected_mask, mask)
+
+  def test346Inclusive(self):
+    expected_mask = np.array(
+        [[1, 0, 0, 0],
+         [1, 0, 0, 0],
+         [1, 1, 0, 0],
+         [1, 1, 0, 0],
+         [1, 1, 1, 0],
+         [1, 1, 1, 0]])
+    mask = _gen_mask(num_blocks=3, n_in=4, n_out=6, mask_type="inclusive")
+    self.assertAllEqual(expected_mask, mask)
+
+
+class MaskedAutoregressiveFlowTest(test_util.VectorDistributionTestHelpers,
+                                   test.TestCase):
+
+  @property
+  def _autoregressive_flow_kwargs(self):
+    return {
+        "shift_and_log_scale_fn": masked_autoregressive_default_template(
+            hidden_layers=[2], shift_only=False),
+        "is_constant_jacobian": False,
+    }
+
+  def testBijector(self):
+    x_ = np.arange(3 * 4 * 2).astype(np.float32).reshape(3, 4, 2)
+    with self.test_session() as sess:
+      ma = MaskedAutoregressiveFlow(
+          validate_args=True,
+          **self._autoregressive_flow_kwargs)
+      x = constant_op.constant(x_)
+      forward_x = ma.forward(x)
+      # Use identity to invalidate cache.
+      inverse_y = ma.inverse(array_ops.identity(forward_x))
+      fldj = ma.forward_log_det_jacobian(x)
+      # Use identity to invalidate cache.
+      ildj = ma.inverse_log_det_jacobian(array_ops.identity(forward_x))
+      variables.global_variables_initializer().run()
+      [
+          forward_x_,
+          inverse_y_,
+          ildj_,
+          fldj_,
+      ] = sess.run([
+          forward_x,
+          inverse_y,
+          ildj,
+          fldj,
+      ])
+      self.assertEqual("masked_autoregressive_flow", ma.name)
+      self.assertAllClose(forward_x_, forward_x_, rtol=1e-6, atol=0.)
+      self.assertAllClose(x_, inverse_y_, rtol=1e-5, atol=0.)
+      self.assertAllClose(ildj_, -fldj_, rtol=1e-6, atol=0.)
+
+  def testMutuallyConsistent(self):
+    dims = 4
+    with self.test_session() as sess:
+      ma = MaskedAutoregressiveFlow(
+          validate_args=True,
+          **self._autoregressive_flow_kwargs)
+      dist = transformed_distribution_lib.TransformedDistribution(
+          distribution=normal_lib.Normal(loc=0., scale=1.),
+          bijector=ma,
+          event_shape=[dims],
+          validate_args=True)
+      self.run_test_sample_consistent_log_prob(
+          sess=sess,
+          dist=dist,
+          num_samples=int(1e5),
+          radius=1.,
+          center=0.,
+          rtol=0.02)
+
+  def testInvertMutuallyConsistent(self):
+    dims = 4
+    with self.test_session() as sess:
+      ma = Invert(MaskedAutoregressiveFlow(
+          validate_args=True,
+          **self._autoregressive_flow_kwargs))
+      dist = transformed_distribution_lib.TransformedDistribution(
+          distribution=normal_lib.Normal(loc=0., scale=1.),
+          bijector=ma,
+          event_shape=[dims],
+          validate_args=True)
+      self.run_test_sample_consistent_log_prob(
+          sess=sess,
+          dist=dist,
+          num_samples=int(1e5),
+          radius=1.,
+          center=0.,
+          rtol=0.02)
+
+
+class MaskedAutoregressiveFlowShiftOnlyTest(MaskedAutoregressiveFlowTest):
+
+  @property
+  def _autoregressive_flow_kwargs(self):
+    return {
+        "shift_and_log_scale_fn": masked_autoregressive_default_template(
+            hidden_layers=[2], shift_only=True),
+        "is_constant_jacobian": True,
+    }
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/__init__.py b/tensorflow/contrib/distributions/python/ops/bijectors/__init__.py
index e62f900bbf..fd6c509446 100644
--- a/tensorflow/contrib/distributions/python/ops/bijectors/__init__.py
+++ b/tensorflow/contrib/distributions/python/ops/bijectors/__init__.py
@@ -26,6 +26,7 @@
 @@Identity
 @@Inline
 @@Invert
+@@MaskedAutoregressiveFlow
 @@Permute
 @@PowerTransform
 @@Sigmoid
@@ -34,6 +35,9 @@
 @@SoftmaxCentered
 @@Softplus
 @@Weibull
+
+@@masked_autoregressive_default_template
+@@masked_dense
 """
 
 from __future__ import absolute_import
@@ -52,6 +56,7 @@ from tensorflow.contrib.distributions.python.ops.bijectors.exp import *
 from tensorflow.contrib.distributions.python.ops.bijectors.gumbel import *
 from tensorflow.contrib.distributions.python.ops.bijectors.inline import *
 from tensorflow.contrib.distributions.python.ops.bijectors.invert import *
+from tensorflow.contrib.distributions.python.ops.bijectors.masked_autoregressive import *
 from tensorflow.contrib.distributions.python.ops.bijectors.permute import *
 from tensorflow.contrib.distributions.python.ops.bijectors.power_transform import *
 from tensorflow.contrib.distributions.python.ops.bijectors.sigmoid import *
diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/masked_autoregressive.py b/tensorflow/contrib/distributions/python/ops/bijectors/masked_autoregressive.py
new file mode 100644
index 0000000000..132dc570f9
--- /dev/null
+++ b/tensorflow/contrib/distributions/python/ops/bijectors/masked_autoregressive.py
@@ -0,0 +1,33 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""MaskedAutoregressiveFlow bijector."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+# go/tf-wildcard-import
+# pylint: disable=wildcard-import
+from tensorflow.contrib.distributions.python.ops.bijectors.masked_autoregressive_impl import *
+# pylint: enable=wildcard-import
+from tensorflow.python.util.all_util import remove_undocumented
+
+_allowed_symbols = [
+    "MaskedAutoregressiveFlow",
+    "masked_dense",
+    "masked_autoregressive_default_template",
+]
+
+remove_undocumented(__name__, _allowed_symbols)
diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/masked_autoregressive_impl.py b/tensorflow/contrib/distributions/python/ops/bijectors/masked_autoregressive_impl.py
new file mode 100644
index 0000000000..ae14288393
--- /dev/null
+++ b/tensorflow/contrib/distributions/python/ops/bijectors/masked_autoregressive_impl.py
@@ -0,0 +1,473 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""MaskedAutoregressiveFlow bijector."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.layers import core as layers
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import clip_ops
+from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import init_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import nn_ops
+from tensorflow.python.ops import template as template_ops
+from tensorflow.python.ops import variable_scope as variable_scope_lib
+from tensorflow.python.ops.distributions import bijector as bijector_lib
+
+
+__all__ = [
+    "MaskedAutoregressiveFlow",
+    "masked_autoregressive_default_template",
+    "masked_dense",
+]
+
+
+class MaskedAutoregressiveFlow(bijector_lib.Bijector):
+  """Affine MaskedAutoregressiveFlow bijector for vector-valued events.
+
+  The affine autoregressive flow [1] provides a relatively simple framework for
+  user-specified (deep) architectures to learn a distribution over vector-valued
+  events. Regarding terminology,
+
+    "Autoregressive models decompose the joint density as a product of
+    conditionals, and model each conditional in turn. Normalizing flows
+    transform a base density (e.g. a standard Gaussian) into the target density
+    by an invertible transformation with tractable Jacobian." [1]
+
+  In other words, the "autoregressive property" is equivalent to the
+  decomposition, `p(x) = prod{ p(x[i] | x[0:i]) : i=0, ..., d }`. The provided
+  `shift_and_log_scale_fn`, `masked_autoregressive_default_template`, achieves
+  this property by zeroing out weights in its `masked_dense` layers.
+
+  In the `tf.distributions` framework, a "normalizing flow" is implemented as a
+  `tf.distributions.bijectors.Bijector`. The `forward` "autoregression"
+  is implemented using a `tf.while_loop` and a deep neural network (DNN) with
+  masked weights such that the autoregressive property is automatically met in
+  the `inverse`.
+
+  A `TransformedDistribution` using `MaskedAutoregressiveFlow(...)` uses the
+  (expensive) forward-mode calculation to draw samples and the (cheap)
+  reverse-mode calculation to compute log-probabilities. Conversely, a
+  `TransformedDistribution` using `Invert(MaskedAutoregressiveFlow(...))` uses
+  the (expensive) forward-mode calculation to compute log-probabilities and the
+  (cheap) reverse-mode calculation to compute samples.  See "Example Use"
+  [below] for more details.
+
+  Given a `shift_and_log_scale_fn`, the forward and inverse transformations are
+  (a sequence of) affine transformations. A "valid" `shift_and_log_scale_fn`
+  must compute each `shift` (aka `loc` or "mu" [2]) and `log(scale)` (aka
+  "alpha" [2]) such that each are broadcastable with the arguments to `forward`
+  and `inverse`, i.e., such that the calculations in `forward`, `inverse`
+  [below] are possible.
+
+  For convenience, `masked_autoregressive_default_template` is offered as a
+  possible `shift_and_log_scale_fn` function. It implements the MADE
+  architecture [2]. MADE is a feed-forward network that computes a `shift` and
+  `log(scale)` using `masked_dense` layers in a deep neural network. Weights are
+  masked to ensure the autoregressive property. It is possible that this
+  architecture is suboptimal for your task. To build alternative networks,
+  either change the arguments to `masked_autoregressive_default_template`, use
+  the `masked_dense` function to roll-out your own, or use some other
+  architecture, e.g., using `tf.layers`.
+
+  Warning: no attempt is made to validate that the `shift_and_log_scale_fn`
+  enforces the "autoregressive property".
+
+  Assuming `shift_and_log_scale_fn` has valid shape and autoregressive
+  semantics, the forward transformation is,
+
+  ```python
+  def forward(x):
+    y = zeros_like(x)
+    event_size = x.shape[-1]
+    for _ in range(event_size):
+      shift, log_scale = shift_and_log_scale_fn(y)
+      y = x * math_ops.exp(log_scale) + shift
+    return y
+  ```
+
+  and the inverse transformation is,
+
+  ```python
+  def inverse(y):
+    shift, log_scale = shift_and_log_scale_fn(y)
+    return (y - shift) / math_ops.exp(log_scale)
+  ```
+
+  Notice that the `inverse` does not need a for-loop. This is because in the
+  forward pass each calculation of `shift` and `log_scale` is based on the `y`
+  calculated so far (not `x`). In the `inverse`, the `y` is fully known, thus is
+  equivalent to the scaling used in `forward` after `event_size` passes, i.e.,
+  the "last" `y` used to compute `shift`, `log_scale`. (Roughly speaking, this
+  also proves the transform is bijective.)
+
+  #### Example Use
+
+  ```python
+  ds = tf.contrib.distributions
+  bs = tf.contrib.distributions.bijectors
+
+  dims = 5
+
+  # A common choice for a normalizing flow is to use a Gaussian for the base
+  # distribution. (However, any continuous distribution would work.) E.g.,
+  maf = ds.TransformedDistribution(
+      distribution=ds.Normal(loc=0., scale=1.),
+      bijector=bs.MaskedAutoregressiveFlow(
+          shift_and_log_scale_fn=bs.masked_autoregressive_default_template(
+              hidden_layers=[512, 512])),
+      event_shape=[dims])
+
+  x = maf.sample()  # Expensive; uses `tf.while_loop`, no Bijector caching.
+  maf.log_prob(x)   # Almost free; uses Bijector caching.
+  maf.log_prob(0.)  # Cheap; no `tf.while_loop` despite no Bijector caching.
+
+  # [1] also describes an "Inverse Autoregressive Flow", e.g.,
+  iaf = ds.TransformedDistribution(
+      distribution=ds.Normal(loc=0., scale=1.),
+      bijector=bs.Invert(bs.MaskedAutoregressiveFlow(
+          shift_and_log_scale_fn=bs.masked_autoregressive_default_template(
+              hidden_layers=[512, 512]))),
+      event_shape=[dims])
+
+  x = iaf.sample()  # Cheap; no `tf.while_loop` despite no Bijector caching.
+  iaf.log_prob(x)   # Almost free; uses Bijector caching.
+  iaf.log_prob(0.)  # Expensive; uses `tf.while_loop`, no Bijector caching.
+
+  # In many (if not most) cases the default `shift_and_log_scale_fn` will be a
+  # poor choice. Here's an example of using a "shift only" version and with a
+  # different number/depth of hidden layers.
+  shift_only = True
+  maf_no_scale_hidden2 = ds.TransformedDistribution(
+      distribution=ds.Normal(loc=0., scale=1.),
+      bijector=bs.MaskedAutoregressiveFlow(
+          bs.masked_autoregressive_default_template(
+              hidden_layers=[32],
+              shift_only=shift_only),
+          is_constant_jacobian=shift_only),
+      event_shape=[dims])
+  ```
+
+  [1]: "Masked Autoregressive Flow for Density Estimation."
+       George Papamakarios, Theo Pavlakou, Iain Murray. Arxiv. 2017.
+       https://arxiv.org/abs/1705.07057
+
+  [2]: "MADE: Masked Autoencoder for Distribution Estimation."
+       Mathieu Germain, Karol Gregor, Iain Murray, Hugo Larochelle. ICML. 2015.
+       https://arxiv.org/abs/1502.03509
+
+  """
+
+  def __init__(self,
+               shift_and_log_scale_fn,
+               is_constant_jacobian=False,
+               validate_args=False,
+               name=None):
+    """Creates the MaskedAutoregressiveFlow bijector.
+
+    Args:
+      shift_and_log_scale_fn: Python `callable` which computes `shift` and
+        `log_scale` from both the forward domain (`x`) and the inverse domain
+        (`y`). Calculation must respect the "autoregressive property" (see class
+        docstring). Suggested default
+        `masked_autoregressive_default_template(hidden_layers=...)`.
+        Typically the function contains `tf.Variables` and is wrapped using
+        `tf.make_template`. Returning `None` for either (both) `shift`,
+        `log_scale` is equivalent to (but more efficient than) returning zero.
+      is_constant_jacobian: Python `bool`. Default: `False`. When `True` the
+        implementation assumes `log_scale` does not depend on the forward domain
+        (`x`) or inverse domain (`y`) values. (No validation is made;
+        `is_constant_jacobian=False` is always safe but possibly computationally
+        inefficient.)
+      validate_args: Python `bool` indicating whether arguments should be
+        checked for correctness.
+      name: Python `str`, name given to ops managed by this object.
+    """
+    name = name or "masked_autoregressive_flow"
+    self._shift_and_log_scale_fn = shift_and_log_scale_fn
+    super(MaskedAutoregressiveFlow, self).__init__(
+        is_constant_jacobian=is_constant_jacobian,
+        validate_args=validate_args,
+        name=name)
+
+  def _forward(self, x):
+    event_size = array_ops.shape(x)[-1]
+    def _loop_body(index, y0):
+      """While-loop body for autoregression calculation."""
+      # Set caching device to avoid re-getting the tf.Variable for every while
+      # loop iteration.
+      with variable_scope_lib.variable_scope(
+          variable_scope_lib.get_variable_scope()) as vs:
+        if vs.caching_device is None:
+          vs.set_caching_device(lambda op: op.device)
+        shift, log_scale = self._shift_and_log_scale_fn(y0)
+      y = x
+      if log_scale is not None:
+        y *= math_ops.exp(log_scale)
+      if shift is not None:
+        y += shift
+      return index + 1, y
+    _, y = control_flow_ops.while_loop(
+        cond=lambda index, _: index < event_size,
+        body=_loop_body,
+        loop_vars=[0, array_ops.zeros_like(x, name="y0")])
+    return y
+
+  def _inverse(self, y):
+    shift, log_scale = self._shift_and_log_scale_fn(y)
+    x = y
+    if shift is not None:
+      x -= shift
+    if log_scale is not None:
+      x *= math_ops.exp(-log_scale)
+    return x
+
+  def _inverse_log_det_jacobian(self, y):
+    _, log_scale = self._shift_and_log_scale_fn(y)
+    if log_scale is None:
+      return constant_op.constant(0., dtype=y.dtype, name="ildj")
+    return -math_ops.reduce_sum(log_scale, axis=-1)
+
+
+MASK_INCLUSIVE = "inclusive"
+MASK_EXCLUSIVE = "exclusive"
+
+
+def _gen_slices(num_blocks, n_in, n_out, mask_type=MASK_EXCLUSIVE):
+  """Generate the slices for building an autoregressive mask."""
+  # TODO(b/67594795): Better support of dynamic shape.
+  slices = []
+  col = 0
+  d_in = n_in // num_blocks
+  d_out = n_out // num_blocks
+  row = d_out if mask_type == MASK_EXCLUSIVE else 0
+  for _ in range(num_blocks):
+    row_slice = slice(row, None)
+    col_slice = slice(col, col + d_in)
+    slices.append([row_slice, col_slice])
+    col += d_in
+    row += d_out
+  return slices
+
+
+def _gen_mask(num_blocks,
+              n_in,
+              n_out,
+              mask_type=MASK_EXCLUSIVE,
+              dtype=dtypes.float32):
+  """Generate the mask for building an autoregressive dense layer."""
+  # TODO(b/67594795): Better support of dynamic shape.
+  mask = np.zeros([n_out, n_in], dtype=dtype.as_numpy_dtype())
+  slices = _gen_slices(num_blocks, n_in, n_out, mask_type=mask_type)
+  for [row_slice, col_slice] in slices:
+    mask[row_slice, col_slice] = 1
+  return mask
+
+
+def masked_dense(inputs,
+                 units,
+                 num_blocks=None,
+                 exclusive=False,
+                 kernel_initializer=None,
+                 reuse=None,
+                 name=None,
+                 *args,
+                 **kwargs):
+  """A autoregressively masked dense layer. Analogous to `tf.layers.dense`.
+
+  See [1] for detailed explanation.
+
+  [1]: "MADE: Masked Autoencoder for Distribution Estimation."
+       Mathieu Germain, Karol Gregor, Iain Murray, Hugo Larochelle. ICML. 2015.
+       https://arxiv.org/abs/1502.03509
+
+  Arguments:
+    inputs: Tensor input.
+    units: Python `int` scalar representing the dimensionality of the output
+      space.
+    num_blocks: Python `int` scalar representing the number of blocks for the
+      MADE masks.
+    exclusive: Python `bool` scalar representing whether to zero the diagonal of
+      the mask, used for the first layer of a MADE.
+    kernel_initializer: Initializer function for the weight matrix.
+      If `None` (default), weights are initialized using the
+      `tf.glorot_random_initializer`.
+    reuse: Python `bool` scalar representing whether to reuse the weights of a
+      previous layer by the same name.
+    name: Python `str` used to describe ops managed by this function.
+    *args: `tf.layers.dense` arguments.
+    **kwargs: `tf.layers.dense` keyword arguments.
+
+  Returns:
+    Output tensor.
+
+  Raises:
+    NotImplementedError: if rightmost dimension of `inputs` is unknown prior to
+      graph execution.
+  """
+  # TODO(b/67594795): Better support of dynamic shape.
+  input_depth = inputs.shape.with_rank_at_least(1)[-1].value
+  if input_depth is None:
+    raise NotImplementedError(
+        "Rightmost dimension must be known prior to graph execution.")
+
+  mask = _gen_mask(num_blocks, input_depth, units,
+                   MASK_EXCLUSIVE if exclusive else MASK_INCLUSIVE).T
+
+  if kernel_initializer is None:
+    kernel_initializer = init_ops.glorot_normal_initializer()
+
+  def masked_initializer(shape, dtype=None, partition_info=None):
+    return mask * kernel_initializer(shape, dtype, partition_info)
+
+  with ops.name_scope(name, "masked_dense", [inputs, units, num_blocks]):
+    layer = layers.Dense(
+        units,
+        kernel_initializer=masked_initializer,
+        kernel_constraint=lambda x: mask * x,
+        name=name,
+        dtype=inputs.dtype.base_dtype,
+        _scope=name,
+        _reuse=reuse,
+        *args,
+        **kwargs)
+    return layer.apply(inputs)
+
+
+def masked_autoregressive_default_template(
+    hidden_layers,
+    shift_only=False,
+    activation=nn_ops.relu,
+    log_scale_min_clip=-5.,
+    log_scale_max_clip=3.,
+    log_scale_clip_gradient=False,
+    name=None,
+    *args,
+    **kwargs):
+  """Build the MADE Model [1].
+
+  This will be wrapped in a make_template to ensure the variables are only
+  created once. It takes the input and returns the `loc` ("mu" [1]) and
+  `log_scale` ("alpha" [1]) from the MADE network.
+
+  Warning: This function uses `masked_dense` to create randomly initialized
+  `tf.Variables`. It is presumed that these will be fit, just as you would any
+  other neural architecture which uses `tf.layers.dense`.
+
+  #### About Hidden Layers:
+
+  Each element of `hidden_layers` should be greater than the `input_depth`
+  (i.e., `input_depth = tf.shape(input)[-1]` where `input` is the input to the
+  neural network). This is necessary to ensure the autoregressivity property.
+
+  #### About Clipping:
+
+  This function also optionally clips the `log_scale` (but possibly not its
+  gradient). This is useful because if `log_scale` is too small/large it might
+  underflow/overflow making it impossible for the `MaskedAutoregressiveFlow`
+  bijector to implement a bijection. Additionally, the `log_scale_clip_gradient`
+  `bool` indicates whether the gradient should also be clipped. The default does
+  not clip the gradient; this is useful because it still provides gradient
+  information (for fitting) yet solves the numerical stability problem. I.e.,
+  `log_scale_clip_gradient = False` means
+  `grad[exp(clip(x))] = grad[x] exp(clip(x))` rather than the usual
+  `grad[clip(x)] exp(clip(x))`.
+
+  [1]: "MADE: Masked Autoencoder for Distribution Estimation."
+       Mathieu Germain, Karol Gregor, Iain Murray, Hugo Larochelle. ICML. 2015.
+       https://arxiv.org/abs/1502.03509
+
+  Arguments:
+    hidden_layers: Python `list`-like of non-negative integer, scalars
+      indicating the number of units in each hidden layer. Default: `[512, 512].
+    shift_only: Python `bool` indicating if only the `shift` term shall be
+      computed. Default: `False`.
+    activation: Activation function (callable). Explicitly setting to `None`
+      implies a linear activation.
+    log_scale_min_clip: `float`-like scalar `Tensor`, or a `Tensor` with the
+      same shape as `log_scale`. The minimum value to clip by. Default: -5.
+    log_scale_max_clip: `float`-like scalar `Tensor`, or a `Tensor` with the
+      same shape as `log_scale`. The maximum value to clip by. Default: 3.
+    log_scale_clip_gradient: Python `bool` indicating that the gradient of
+      `tf.clip_by_value` should be preserved. Default: `False`.
+    name: A name for ops managed by this function. Default:
+      "masked_autoregressive_default_template".
+    *args: `tf.layers.dense` arguments.
+    **kwargs: `tf.layers.dense` keyword arguments.
+
+  Returns:
+    shift: `Float`-like `Tensor` of shift terms (the "mu" in [2]).
+    log_scale: `Float`-like `Tensor` of log(scale) terms (the "alpha" in [2]).
+
+  Raises:
+    NotImplementedError: if rightmost dimension of `inputs` is unknown prior to
+      graph execution.
+  """
+
+  with ops.name_scope(name, "masked_autoregressive_default_template",
+                      values=[log_scale_min_clip, log_scale_max_clip]):
+    def _fn(x):
+      """MADE parameterized via `masked_autoregressive_default_template`."""
+      # TODO(b/67594795): Better support of dynamic shape.
+      input_depth = x.shape.with_rank_at_least(1)[-1].value
+      if input_depth is None:
+        raise NotImplementedError(
+            "Rightmost dimension must be known prior to graph execution.")
+      input_shape = (np.int32(x.shape.as_list()) if x.shape.is_fully_defined()
+                     else array_ops.shape(x))
+      for i, units in enumerate(hidden_layers):
+        x = masked_dense(
+            inputs=x,
+            units=units,
+            num_blocks=input_depth,
+            exclusive=True if i == 0 else False,
+            activation=activation,
+            *args,
+            **kwargs)
+      x = masked_dense(
+          inputs=x,
+          units=(1 if shift_only else 2) * input_depth,
+          num_blocks=input_depth,
+          activation=None,
+          *args,
+          **kwargs)
+      if shift_only:
+        x = array_ops.reshape(x, shape=input_shape)
+        return x, None
+      x = array_ops.reshape(
+          x, shape=array_ops.concat([input_shape, [2]], axis=0))
+      shift, log_scale = array_ops.unstack(x, num=2, axis=-1)
+      which_clip = (math_ops.clip_by_value if log_scale_clip_gradient
+                    else _clip_by_value_preserve_grad)
+      log_scale = which_clip(log_scale, log_scale_min_clip, log_scale_max_clip)
+      return shift, log_scale
+    return template_ops.make_template(
+        "masked_autoregressive_default_template", _fn)
+
+
+def _clip_by_value_preserve_grad(x, clip_value_min, clip_value_max, name=None):
+  """Clips input while leaving gradient unaltered."""
+  with ops.name_scope(name, "clip_by_value_preserve_grad",
+                      [x, clip_value_min, clip_value_max]):
+    clip_x = clip_ops.clip_by_value(x, clip_value_min, clip_value_max)
+    return x + array_ops.stop_gradient(clip_x - x)
diff --git a/tensorflow/contrib/distributions/python/ops/test_util.py b/tensorflow/contrib/distributions/python/ops/test_util.py
index da7d3907ac..631ffc1bac 100644
--- a/tensorflow/contrib/distributions/python/ops/test_util.py
+++ b/tensorflow/contrib/distributions/python/ops/test_util.py
@@ -25,6 +25,7 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import histogram_ops
 from tensorflow.python.ops import linalg_ops
 from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import variables as variables_ops
 
 
 __all__ = [
@@ -279,26 +280,32 @@ class VectorDistributionTestHelpers(object):
     def monte_carlo_hypersphere_volume(dist, num_samples, radius, center):
       # https://en.wikipedia.org/wiki/Importance_sampling
       x = dist.sample(num_samples, seed=seed)
+      x = array_ops.identity(x)  # Invalidate bijector cacheing.
       return math_ops.reduce_mean(
           math_ops.exp(-dist.log_prob(x)) * is_in_ball(x, radius, center),
           axis=0)
 
-    [
-        batch_shape_,
-        actual_volume_,
-        sample_volume_,
-    ] = sess.run([
-        dist.batch_shape_tensor(),
-        actual_hypersphere_volume(
-            dims=dist.event_shape_tensor()[0],
-            radius=radius),
-        monte_carlo_hypersphere_volume(
-            dist,
-            num_samples=num_samples,
-            radius=radius,
-            center=center),
-    ])
-
+    # Build graph.
+    with ops.name_scope(
+        "run_test_sample_consistent_log_prob",
+        values=[num_samples, radius, center] + dist._graph_parents):  # pylint: disable=protected-access
+      batch_shape = dist.batch_shape_tensor()
+      actual_volume = actual_hypersphere_volume(
+          dims=dist.event_shape_tensor()[0],
+          radius=radius)
+      sample_volume = monte_carlo_hypersphere_volume(
+          dist,
+          num_samples=num_samples,
+          radius=radius,
+          center=center)
+      init_op = variables_ops.global_variables_initializer()
+
+    # Execute graph.
+    sess.run(init_op)
+    [batch_shape_, actual_volume_, sample_volume_] = sess.run([
+        batch_shape, actual_volume, sample_volume])
+
+    # Check results.
     self.assertAllClose(np.tile(actual_volume_, reps=batch_shape_),
                         sample_volume_,
                         rtol=rtol, atol=atol)
-- 
GitLab


From 251070eb2d7dc8376c868b8a86342c3332e706f0 Mon Sep 17 00:00:00 2001
From: Igor Ganichev <iga@google.com>
Date: Wed, 18 Oct 2017 15:17:14 -0700
Subject: [PATCH 536/909] Implement nest.flatten in C++

This function is used extensively in imperative_grad. Implementing
it in C++ reduces SPINN training time by over 8%.

PiperOrigin-RevId: 172663591
---
 tensorflow/contrib/cmake/tf_python.cmake |   2 +
 tensorflow/python/BUILD                  |  13 +++
 tensorflow/python/tensorflow.i           |   2 +
 tensorflow/python/util/nest.py           |  41 ++------
 tensorflow/python/util/util.cc           | 127 +++++++++++++++++++++++
 tensorflow/python/util/util.h            |  74 +++++++++++++
 tensorflow/python/util/util.i            |  42 ++++++++
 7 files changed, 269 insertions(+), 32 deletions(-)
 create mode 100644 tensorflow/python/util/util.cc
 create mode 100644 tensorflow/python/util/util.h
 create mode 100644 tensorflow/python/util/util.i

diff --git a/tensorflow/contrib/cmake/tf_python.cmake b/tensorflow/contrib/cmake/tf_python.cmake
index e83618a94e..8ddfb59595 100755
--- a/tensorflow/contrib/cmake/tf_python.cmake
+++ b/tensorflow/contrib/cmake/tf_python.cmake
@@ -874,6 +874,8 @@ set (pywrap_tensorflow_internal_src
     "${tensorflow_source_dir}/tensorflow/python/lib/io/py_record_writer.cc"
     "${tensorflow_source_dir}/tensorflow/python/util/kernel_registry.h"
     "${tensorflow_source_dir}/tensorflow/python/util/kernel_registry.cc"
+    "${tensorflow_source_dir}/tensorflow/python/util/util.h"
+    "${tensorflow_source_dir}/tensorflow/python/util/util.cc"
     "${tensorflow_source_dir}/tensorflow/cc/framework/ops.cc"
     "${tensorflow_source_dir}/tensorflow/cc/framework/scope.cc"
     "${CMAKE_CURRENT_BINARY_DIR}/pywrap_tensorflow_internal.cc"
diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index cbeb0b46cb..21cdaec477 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -245,6 +245,17 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "cpp_python_util",
+    srcs = ["util/util.cc"],
+    hdrs = ["util/util.h"],
+    deps = [
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//util/python:python_headers",
+    ],
+)
+
 cc_library(
     name = "py_func_lib",
     srcs = ["lib/core/py_func.cc"],
@@ -2982,10 +2993,12 @@ tf_py_wrap_cc(
         "util/stat_summarizer.i",
         "util/tfprof.i",
         "util/transform_graph.i",
+        "util/util.i",
     ],
     deps = [
         ":cost_analyzer_lib",
         ":model_analyzer_lib",
+        ":cpp_python_util",
         ":cpp_shape_inference",
         ":kernel_registry",
         ":numpy_lib",
diff --git a/tensorflow/python/tensorflow.i b/tensorflow/python/tensorflow.i
index 9cef765bf3..d221dd523b 100644
--- a/tensorflow/python/tensorflow.i
+++ b/tensorflow/python/tensorflow.i
@@ -44,6 +44,8 @@ limitations under the License.
 
 %include "tensorflow/python/util/transform_graph.i"
 
+%include "tensorflow/python/util/util.i"
+
 %include "tensorflow/python/grappler/cluster.i"
 %include "tensorflow/python/grappler/item.i"
 %include "tensorflow/python/grappler/tf_optimizer.i"
diff --git a/tensorflow/python/util/nest.py b/tensorflow/python/util/nest.py
index d57140da75..dd6acee3c7 100644
--- a/tensorflow/python/util/nest.py
+++ b/tensorflow/python/util/nest.py
@@ -35,7 +35,7 @@ import collections as _collections
 
 import six as _six
 
-from tensorflow.python.platform import tf_logging as _tf_logging
+from tensorflow.python import pywrap_tensorflow as _pywrap_tensorflow
 from tensorflow.python.util.all_util import remove_undocumented
 
 
@@ -91,26 +91,6 @@ def _yield_value(iterable):
       yield value
 
 
-def _yield_flat_nest(nest):
-  for n in _yield_value(nest):
-    if is_sequence(n):
-      for ni in _yield_flat_nest(n):
-        yield ni
-    else:
-      yield n
-
-
-# Used by `_warn_once` to remember which warning messages have been given.
-_ALREADY_WARNED = {}
-
-
-def _warn_once(message):
-  """Logs a warning message, once per unique string."""
-  if message not in _ALREADY_WARNED:
-    _ALREADY_WARNED[message] = True
-    _tf_logging.warning(message)
-
-
 def is_sequence(seq):
   """Returns a true if its input is a collections.Sequence (except strings).
 
@@ -121,13 +101,7 @@ def is_sequence(seq):
     True if the sequence is a not a string and is a collections.Sequence or a
     dict.
   """
-  if isinstance(seq, dict):
-    return True
-  if isinstance(seq, set):
-    _warn_once("Sets are not currently considered sequences, but this may "
-               "change in the future, so consider avoiding using them.")
-  return (isinstance(seq, _collections.Sequence)
-          and not isinstance(seq, _six.string_types))
+  return _pywrap_tensorflow.IsSequence(seq)
 
 
 def flatten(nest):
@@ -145,6 +119,9 @@ def flatten(nest):
   a correponding plain dict, or vice-versa.
   Dictionaries with non-sortable keys cannot be flattened.
 
+  Users must not modify any collections used in `nest` while this function is
+  running.
+
   Args:
     nest: an arbitrarily nested structure or a scalar object. Note, numpy
         arrays are considered scalars.
@@ -155,10 +132,7 @@ def flatten(nest):
   Raises:
     TypeError: The nest is or contains a dict with non-sortable keys.
   """
-  if is_sequence(nest):
-    return list(_yield_flat_nest(nest))
-  else:
-    return [nest]
+  return _pywrap_tensorflow.Flatten(nest)
 
 
 def _recursive_assert_same_structure(nest1, nest2, check_types):
@@ -692,6 +666,9 @@ def get_traverse_shallow_structure(traverse_fn, structure):
   return _sequence_like(structure, level_traverse)
 
 
+_pywrap_tensorflow.RegisterSequenceClass(_collections.Sequence)
+
+
 _allowed_symbols = [
     "assert_same_structure",
     "is_sequence",
diff --git a/tensorflow/python/util/util.cc b/tensorflow/python/util/util.cc
new file mode 100644
index 0000000000..c3d7611ad4
--- /dev/null
+++ b/tensorflow/python/util/util.cc
@@ -0,0 +1,127 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/python/util/util.h"
+
+#include "tensorflow/core/lib/strings/strcat.h"
+#include "tensorflow/core/platform/logging.h"
+
+namespace tensorflow {
+namespace swig {
+
+namespace {
+
+// Type object for collections.Sequence. This is set by RegisterSequenceClass.
+PyObject* CollectionsSequenceType = nullptr;
+
+bool WarnedThatSetIsNotSequence = false;
+
+// Returns 1 if `o` is considered a sequence for the purposes of Flatten().
+// Returns 0 otherwise.
+// Returns -1 if an error occured.
+int IsSequenceHelper(PyObject* o) {
+  if (PyDict_Check(o)) return true;
+  if (PySet_Check(o) && !WarnedThatSetIsNotSequence) {
+    LOG(WARNING) << "Sets are not currently considered sequences, "
+                    "but this may change in the future, "
+                    "so consider avoiding using them.";
+    WarnedThatSetIsNotSequence = true;
+  }
+  if (CollectionsSequenceType == nullptr) {
+    PyErr_SetString(
+        PyExc_RuntimeError,
+        tensorflow::strings::StrCat(
+            "collections.Sequence type has not been set. "
+            "Please call RegisterSequenceClass before using this module")
+            .c_str());
+    return -1;
+  }
+  int is_instance = PyObject_IsInstance(o, CollectionsSequenceType);
+  if (is_instance == -1) return -1;
+  return static_cast<int>(is_instance != 0 && !PyBytes_Check(o) &&
+#if PY_MAJOR_VERSION < 3
+                          !PyString_Check(o) &&
+#endif
+                          !PyUnicode_Check(o));
+}
+
+bool FlattenHelper(PyObject* nested, PyObject* list) {
+  // if nested is not a sequence, append itself and exit
+  int is_seq = IsSequenceHelper(nested);
+  if (is_seq == -1) return false;
+  if (!is_seq) {
+    return PyList_Append(list, nested) != -1;
+  }
+
+  // if nested if dictionary, sort it by key and recurse on each value
+  if (PyDict_Check(nested)) {
+    PyObject* keys = PyDict_Keys(nested);
+    if (PyList_Sort(keys) == -1) return false;
+    Py_ssize_t size = PyList_Size(keys);
+    for (Py_ssize_t i = 0; i < size; ++i) {
+      // We know that key and val will not be deleted because nested owns
+      // a reference to them and callers of flatten must not modify nested
+      // while the method is running.
+      PyObject* key = PyList_GET_ITEM(keys, i);
+      PyObject* val = PyDict_GetItem(nested, key);
+      if (Py_EnterRecursiveCall(" in Flatten")) {
+        Py_DECREF(keys);
+        return false;
+      }
+      FlattenHelper(val, list);
+      Py_LeaveRecursiveCall();
+    }
+    Py_DECREF(keys);
+    return true;
+  }
+
+  // iterate and recurse
+  PyObject* item;
+  PyObject* iterator = PyObject_GetIter(nested);
+  while ((item = PyIter_Next(iterator)) != nullptr) {
+    FlattenHelper(item, list);
+    Py_DECREF(item);
+  }
+  Py_DECREF(iterator);
+  return true;
+}
+
+}  // anonymous namespace
+
+void RegisterSequenceClass(PyObject* sequence_class) {
+  if (!PyType_Check(sequence_class)) {
+    PyErr_SetString(
+        PyExc_TypeError,
+        tensorflow::strings::StrCat(
+            "Expecting a class definition for `collections.Sequence`. Got ",
+            Py_TYPE(sequence_class)->tp_name)
+            .c_str());
+    return;
+  }
+  CollectionsSequenceType = sequence_class;
+}
+
+bool IsSequence(PyObject* o) { return IsSequenceHelper(o) == 1; }
+
+PyObject* Flatten(PyObject* nested) {
+  PyObject* list = PyList_New(0);
+  if (FlattenHelper(nested, list)) {
+    return list;
+  } else {
+    Py_DECREF(list);
+    return nullptr;
+  }
+}
+}  // namespace swig
+}  // namespace tensorflow
diff --git a/tensorflow/python/util/util.h b/tensorflow/python/util/util.h
new file mode 100644
index 0000000000..493d26b497
--- /dev/null
+++ b/tensorflow/python/util/util.h
@@ -0,0 +1,74 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+// Functions for getting information about kernels registered in the binary.
+#ifndef THIRD_PARTY_TENSORFLOW_PYTHON_UTIL_UTIL_H_
+#define THIRD_PARTY_TENSORFLOW_PYTHON_UTIL_UTIL_H_
+
+#include <Python.h>
+
+namespace tensorflow {
+namespace swig {
+
+// Implements the same interface as tensorflow.util.nest.is_sequence
+// Returns a true if its input is a collections.Sequence (except strings).
+//
+// Args:
+//   seq: an input sequence.
+//
+// Returns:
+//   True if the sequence is a not a string and is a collections.Sequence or a
+//   dict.
+bool IsSequence(PyObject* o);
+
+// Implements the same interface as tensorflow.util.nest.flatten
+//
+// Returns a flat list from a given nested structure.
+//
+// If `nest` is not a sequence, tuple, or dict, then returns a single-element
+// list: `[nest]`.
+//
+// In the case of dict instances, the sequence consists of the values, sorted by
+// key to ensure deterministic behavior. This is true also for `OrderedDict`
+// instances: their sequence order is ignored, the sorting order of keys is
+// used instead. The same convention is followed in `pack_sequence_as`. This
+// correctly repacks dicts and `OrderedDict`s after they have been flattened,
+// and also allows flattening an `OrderedDict` and then repacking it back using
+// a correponding plain dict, or vice-versa.
+// Dictionaries with non-sortable keys cannot be flattened.
+//
+// Args:
+//   nest: an arbitrarily nested structure or a scalar object. Note, numpy
+//       arrays are considered scalars.
+//
+// Returns:
+//   A Python list, the flattened version of the input.
+//   On error, returns nullptr
+//
+// Raises:
+//   TypeError: The nest is or contains a dict with non-sortable keys.
+PyObject* Flatten(PyObject* nested);
+
+// RegisterSequenceClass is used to pass PyTypeObject for collections.Sequence
+// (which is defined in python) into the C++ world.
+// Alternative approach could be to import the collections modules and retrieve
+// the type from the module. This approach also requires some trigger from
+// Python so that we know that Python interpreter had been initialzied.
+void RegisterSequenceClass(PyObject* sequence_class);
+
+}  // namespace swig
+}  // namespace tensorflow
+
+#endif  // THIRD_PARTY_TENSORFLOW_PYTHON_UTIL_UTIL_H_
diff --git a/tensorflow/python/util/util.i b/tensorflow/python/util/util.i
new file mode 100644
index 0000000000..d69084fc00
--- /dev/null
+++ b/tensorflow/python/util/util.i
@@ -0,0 +1,42 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+%include "tensorflow/python/platform/base.i"
+
+%{
+#include "tensorflow/python/util/util.h"
+%}
+
+%ignoreall
+
+%unignore tensorflow;
+%unignore tensorflow::swig;
+// The %exception block defined in tf_session.i releases the Python GIL for
+// the length of each wrapped method. This file is included in tensorflow.i
+// after tf_session.i and inherits this definition. We disable this behavior
+// for functions in this module because they use python methods that need GIL.
+// TODO(iga): Find a way not to leak such definitions across files.
+
+%unignore tensorflow::swig::RegisterSequenceClass;
+%noexception tensorflow::swig::RegisterSequenceClass;
+
+%unignore tensorflow::swig::IsSequence;
+%noexception tensorflow::swig::IsSequence;
+
+%unignore tensorflow::swig::Flatten;
+%noexception tensorflow::swig::Flatten;
+
+%include "tensorflow/python/util/util.h"
+
+%unignoreall
-- 
GitLab


From 9b3233d25e0dd7078667712f128657f3fbb7dbd4 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 18 Oct 2017 15:53:02 -0700
Subject: [PATCH 537/909] Use Windows compatible string comparisons for setting
 cuda device flags.

PiperOrigin-RevId: 172669121
---
 tensorflow/stream_executor/cuda/cuda_platform.cc | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/tensorflow/stream_executor/cuda/cuda_platform.cc b/tensorflow/stream_executor/cuda/cuda_platform.cc
index d69953f557..874ac1ab65 100644
--- a/tensorflow/stream_executor/cuda/cuda_platform.cc
+++ b/tensorflow/stream_executor/cuda/cuda_platform.cc
@@ -45,12 +45,11 @@ const DeviceOptions GetDeviceOptionsFromEnv() {
   }
 
   unsigned device_flags = 0;
-  if (strcasecmp(kScheduleSpinString, gpu_schedule_string) == 0) {
+  if (strcmp(kScheduleSpinString, gpu_schedule_string) == 0) {
     device_flags = perftools::gputools::DeviceOptions::kScheduleSpin;
-  } else if (strcasecmp(kScheduleYieldString, gpu_schedule_string) == 0) {
+  } else if (strcmp(kScheduleYieldString, gpu_schedule_string) == 0) {
     device_flags = perftools::gputools::DeviceOptions::kScheduleYield;
-  } else if (strcasecmp(kScheduleBlockingSyncString, gpu_schedule_string) ==
-             0) {
+  } else if (strcmp(kScheduleBlockingSyncString, gpu_schedule_string) == 0) {
     device_flags = perftools::gputools::DeviceOptions::kScheduleBlockingSync;
   } else {
     LOG(QFATAL) << "Unknown option for environment variable "
-- 
GitLab


From d94db4be7ebc02c2de169274f9f863611e68f98d Mon Sep 17 00:00:00 2001
From: Amit Patankar <amitpatankar@google.com>
Date: Wed, 18 Oct 2017 16:10:42 -0700
Subject: [PATCH 538/909] Branch 172647355 (#13819)

* Update inception score to match the openAI version from https://github.com/openai/improved-gan/tree/master/inception_score.

PiperOrigin-RevId: 172562573

* Stub support for retrieving LossFunction by name.

PiperOrigin-RevId: 172588516

* Don't emit fusion computations separately in HloModule::ToString. These computations are emitted with their fusion instruction and therefore don't need to be emitted as a separate comptutation in the module.

PiperOrigin-RevId: 172612725

* Make `tf.contrib.distributions` quadrature family parameterized by
`quadrature_grid_and_prob` vs `quadrature_degree`. Enables support of
quadrature methods other than Gauss-Hermite.

PiperOrigin-RevId: 172622919

* Fixes test breakage.

PiperOrigin-RevId: 172626499

* Remove global step read dependency from model_fn. Estimator behavior still will be deterministic since the step checking logic in session_run_hooks was changed as follows:
* assume stale step
* before using the step, check for the current value by session.run

PiperOrigin-RevId: 172629797

* More changs to avoid flakes in random_shuffle_queue_test

PiperOrigin-RevId: 172630989

* Add expected keys to predictor exception if unexpected key detected.

PiperOrigin-RevId: 172634275

* Add TF_GraphGetOpDef() to C API and use in Operation.op_def()

Note that this creates a small change in behavior with the C API
enabled, since previously not all Python Operations had an OpDef
(op_def() returns None). With the C API enabled, op_def() always
returns an OpDef.

PiperOrigin-RevId: 172634411

* Implement ZlibInputStream::Tell() by keeping track of the number of bytes
consumed by the reader.

PiperOrigin-RevId: 172634455

* Upgrade tensorflow pip dependency version to 3.4.0+

PiperOrigin-RevId: 172635727

* [XLA] Deterministically dump an executable.

Previously, dumping a executable is nondeterministic as a map in protobuf is serialized in random order.

This CL enables "Deterministic dump" mode of protobuf, which sorts the map first before dumping them. This is helpful in comparing if two dumps are the same in XLA determinism test.

PiperOrigin-RevId: 172637100

* Fixed work size computation in Split and SplitV ops to avoid integer overflow.

PiperOrigin-RevId: 172637818

* Internal change.

PiperOrigin-RevId: 172641543

* Bug fixes for fold_constants_lib.

1. Tensor names in TF may be in the form of "a:0", "a:1", or "a" as a shorthand
notation of "a:0". FoldConstant library always expected the shorthand notation,
and did not handle the cases where explicit notation was passed to input or
output list. This means that this library could not handle the case when input
or output were not the first output of a node.

2. To match the input nodes in the original graph and the added Recv nodes in
rewritten graph, FoldConstant library used prefix matching. Unfortunately, this
means that when a input name is a prefix of another input name, there is
possibility that wrong Recv node gets matched. For example, if input names were
"placeholder" and "placeholder_1", then it did not handle the case very well.

3. RemoveUnusedNodes() in FoldConstants lib could remove nodes which output
depended on. This happened when an input name points to a node with multiple
outputs and not all outputs of that node were included in the input names.

4. ReplaceSendRecvs() in FoldConstants lib assumed that all input nodes are
removed during rewriting the graph. This assumption is not necessarily true,
and it could add a duplicate node in the graph.

PiperOrigin-RevId: 172641947

* Adds visibility to sgdr_learning_rate_decay.

Currently SGD with warm restarts is siloed in
tensorflow/contrib/training/python/training/sgdr_learning_rate_decay.py, since
it is not listed in the 'training_py' build filegroup. This change simply adds
sgdr_learning_rate_decay to this filegroup so that other projects can use warm
restarts during optimization.

PiperOrigin-RevId: 172643218

* Add logging verbosity to mnist.py

PiperOrigin-RevId: 172643922

* Automated g4 rollback of changelist 172336111

PiperOrigin-RevId: 172645893

* Correct the docstring to reflect that the values of cols_to_vars are always lists of Variable's (never single Variable's or PartitionedVariables), and make this true for bias.

PiperOrigin-RevId: 172646456

* Changes MultiLabelHead.create_loss to return a Tensor of size [batch_size, 1], to be consistent with other heads.

PiperOrigin-RevId: 172647355

* Disabling failing contrib tests.
---
 tensorflow/c/c_api.cc                         |  11 +
 tensorflow/c/c_api.h                          |   7 +
 tensorflow/c/c_api_function_test.cc           |  21 ++
 tensorflow/c/c_api_test.cc                    |  31 +++
 tensorflow/compiler/xla/BUILD                 |   1 +
 tensorflow/compiler/xla/service/BUILD         |   2 +
 tensorflow/compiler/xla/service/executable.cc |   8 +-
 tensorflow/compiler/xla/service/hlo_module.cc |   4 +-
 tensorflow/compiler/xla/util.h                |   1 +
 .../contrib/data/python/kernel_tests/BUILD    |   6 +
 .../kernel_tests/poisson_lognormal_test.py    |  20 +-
 .../python/ops/poisson_lognormal.py           |  54 +++--
 .../python/ops/vector_diffeomixture.py        |  55 +++--
 .../estimator/python/estimator/head.py        |  16 +-
 .../estimator/python/estimator/head_test.py   |  32 +--
 .../eval/python/classifier_metrics_impl.py    |  81 ++++---
 .../eval/python/classifier_metrics_test.py    |  10 +-
 .../kernel_tests/layer_collection_test.py     |  67 ++++++
 .../python/kernel_tests/optimizer_test.py     |   6 +-
 .../kfac/python/ops/layer_collection.py       |  58 ++++-
 .../python/learn/estimators/estimator.py      |   5 +-
 tensorflow/contrib/predictor/predictor.py     |   6 +-
 tensorflow/contrib/training/BUILD             |   1 +
 tensorflow/core/kernels/split_op.cc           |   8 +-
 tensorflow/core/kernels/split_v_op.cc         |   8 +-
 tensorflow/core/lib/io/zlib_buffers_test.cc   | 172 ++++++++++++---
 tensorflow/core/lib/io/zlib_inputstream.cc    |   8 +-
 tensorflow/core/lib/io/zlib_inputstream.h     |   3 +
 tensorflow/examples/learn/mnist.py            |   3 +
 tensorflow/python/BUILD                       |   5 +-
 tensorflow/python/eager/BUILD                 |   7 +-
 tensorflow/python/eager/ops_test.py           |   6 +-
 tensorflow/python/estimator/estimator.py      |   7 +-
 tensorflow/python/feature_column/BUILD        |   1 +
 .../python/feature_column/feature_column.py   |  33 +--
 .../feature_column/feature_column_test.py     |  61 +++++-
 tensorflow/python/framework/ops.py            |  26 ++-
 tensorflow/python/framework/ops_test.py       |  15 ++
 .../kernel_tests/random_shuffle_queue_test.py |  19 +-
 .../resource_variable_ops_test.py             |  10 +
 .../python/ops/resource_variable_ops.py       |  33 +++
 tensorflow/python/training/adam_test.py       |  81 +++----
 tensorflow/python/training/saver_test.py      |  47 ++--
 .../graph_transforms/fold_constants_lib.cc    | 202 ++++++++----------
 .../graph_transforms/fold_constants_test.cc   |  85 +++++++-
 tensorflow/tools/pip_package/setup.py         |   2 +-
 46 files changed, 951 insertions(+), 394 deletions(-)

diff --git a/tensorflow/c/c_api.cc b/tensorflow/c/c_api.cc
index 79fbd8c90c..cd98393e0a 100644
--- a/tensorflow/c/c_api.cc
+++ b/tensorflow/c/c_api.cc
@@ -1799,6 +1799,17 @@ void TF_GraphToGraphDef(TF_Graph* graph, TF_Buffer* output_graph_def,
   status->status = MessageToBuffer(def, output_graph_def);
 }
 
+void TF_GraphGetOpDef(TF_Graph* graph, const char* op_name,
+                      TF_Buffer* output_op_def, TF_Status* status) {
+  const OpDef* op_def;
+  {
+    mutex_lock l(graph->mu);
+    status->status = graph->graph.op_registry()->LookUpOpDef(op_name, &op_def);
+    if (!status->status.ok()) return;
+  }
+  status->status = MessageToBuffer(*op_def, output_op_def);
+}
+
 TF_ImportGraphDefOptions* TF_NewImportGraphDefOptions() {
   return new TF_ImportGraphDefOptions;
 }
diff --git a/tensorflow/c/c_api.h b/tensorflow/c/c_api.h
index 76cfcd5e0d..1e8bfdc7b0 100644
--- a/tensorflow/c/c_api.h
+++ b/tensorflow/c/c_api.h
@@ -864,6 +864,13 @@ TF_CAPI_EXPORT extern void TF_GraphToGraphDef(TF_Graph* graph,
                                               TF_Buffer* output_graph_def,
                                               TF_Status* status);
 
+// Returns the serialized OpDef proto with name `op_name`, or a bad status if no
+// such op exists. This can return OpDefs of functions copied into the graph.
+TF_CAPI_EXPORT extern void TF_GraphGetOpDef(TF_Graph* graph,
+                                            const char* op_name,
+                                            TF_Buffer* output_op_def,
+                                            TF_Status* status);
+
 // TF_ImportGraphDefOptions holds options that can be passed to
 // TF_GraphImportGraphDef.
 typedef struct TF_ImportGraphDefOptions TF_ImportGraphDefOptions;
diff --git a/tensorflow/c/c_api_function_test.cc b/tensorflow/c/c_api_function_test.cc
index 4db9a90fdc..d5580b6589 100644
--- a/tensorflow/c/c_api_function_test.cc
+++ b/tensorflow/c/c_api_function_test.cc
@@ -1465,5 +1465,26 @@ TEST_F(CApiFunctionTest, AppendHash) {
   ASSERT_EQ(string("func_name_base_qaJ8jA8UmGY"), fdef.signature().name());
 }
 
+TEST_F(CApiFunctionTest, GetOpDef) {
+  DefineFunction(func_name_, &func_);
+  TF_GraphCopyFunction(host_graph_, func_, nullptr, s_);
+  ASSERT_EQ(TF_OK, TF_GetCode(s_)) << TF_Message(s_);
+
+  // Test we can retrieve function OpDef from graph
+  TF_Buffer* buffer = TF_NewBuffer();
+  TF_GraphGetOpDef(host_graph_, func_name_, buffer, s_);
+  ASSERT_EQ(TF_OK, TF_GetCode(s_)) << TF_Message(s_);
+
+  // Sanity check returned OpDef
+  string data(static_cast<const char*>(buffer->data), buffer->length);
+  OpDef op_def;
+  op_def.ParseFromString(data);
+  EXPECT_EQ(op_def.name(), func_name_);
+  EXPECT_EQ(op_def.input_arg_size(), 1);
+  EXPECT_EQ(op_def.output_arg_size(), 1);
+
+  TF_DeleteBuffer(buffer);
+}
+
 }  // namespace
 }  // namespace tensorflow
diff --git a/tensorflow/c/c_api_test.cc b/tensorflow/c/c_api_test.cc
index c442029009..d220bc5e95 100644
--- a/tensorflow/c/c_api_test.cc
+++ b/tensorflow/c/c_api_test.cc
@@ -37,6 +37,7 @@ limitations under the License.
 #include "tensorflow/core/framework/types.pb.h"
 #include "tensorflow/core/graph/tensor_id.h"
 #include "tensorflow/core/lib/core/error_codes.pb.h"
+#include "tensorflow/core/lib/core/status_test_util.h"
 #include "tensorflow/core/lib/io/path.h"
 #include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/lib/strings/strcat.h"
@@ -50,6 +51,11 @@ Status TF_TensorToTensor(const TF_Tensor* src, Tensor* dst);
 
 namespace {
 
+static void ExpectHasSubstr(StringPiece s, StringPiece expected) {
+  EXPECT_TRUE(StringPiece(s).contains(expected))
+      << "'" << s << "' does not contain '" << expected << "'";
+}
+
 TEST(CAPI, Version) { EXPECT_STRNE("", TF_Version()); }
 
 TEST(CAPI, Status) {
@@ -837,6 +843,31 @@ TEST(CAPI, ShapeInferenceError) {
   TF_DeleteStatus(status);
 }
 
+TEST(CAPI, GetOpDef) {
+  TF_Status* status = TF_NewStatus();
+  TF_Graph* graph = TF_NewGraph();
+  TF_Buffer* buffer = TF_NewBuffer();
+
+  TF_GraphGetOpDef(graph, "Add", buffer, status);
+  ASSERT_EQ(TF_OK, TF_GetCode(status));
+  const OpDef* expected_op_def;
+  TF_ASSERT_OK(OpRegistry::Global()->LookUpOpDef("Add", &expected_op_def));
+  string expected_serialized;
+  expected_op_def->SerializeToString(&expected_serialized);
+  string actual_string(reinterpret_cast<const char*>(buffer->data),
+                       buffer->length);
+  EXPECT_EQ(expected_serialized, actual_string);
+
+  TF_GraphGetOpDef(graph, "MyFakeOp", buffer, status);
+  EXPECT_EQ(TF_NOT_FOUND, TF_GetCode(status));
+  ExpectHasSubstr(TF_Message(status),
+                  "Op type not registered 'MyFakeOp' in binary");
+
+  TF_DeleteBuffer(buffer);
+  TF_DeleteGraph(graph);
+  TF_DeleteStatus(status);
+}
+
 void StringVectorToArrays(const std::vector<string>& v,
                           std::unique_ptr<const void* []>* ptrs,
                           std::unique_ptr<size_t[]>* lens) {
diff --git a/tensorflow/compiler/xla/BUILD b/tensorflow/compiler/xla/BUILD
index be87506d3c..e51bbffcd0 100644
--- a/tensorflow/compiler/xla/BUILD
+++ b/tensorflow/compiler/xla/BUILD
@@ -171,6 +171,7 @@ cc_library(
     visibility = ["//visibility:public"],
     deps = [
         ":status",
+        ":status_macros",
         ":types",
         ":xla_data_proto",
         "//tensorflow/core:lib",
diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD
index ed42358e7e..b3fbb0c513 100644
--- a/tensorflow/compiler/xla/service/BUILD
+++ b/tensorflow/compiler/xla/service/BUILD
@@ -579,12 +579,14 @@ cc_library(
         ":shaped_buffer",
         ":versioned_computation_handle",
         "//tensorflow/compiler/xla:executable_run_options",
+        "//tensorflow/compiler/xla:status",
         "//tensorflow/compiler/xla:status_macros",
         "//tensorflow/compiler/xla:statusor",
         "//tensorflow/compiler/xla:util",
         "//tensorflow/compiler/xla:xla_data_proto",
         "//tensorflow/compiler/xla/legacy_flags:debug_options_flags",
         "//tensorflow/core:lib",
+        "//tensorflow/core:lib_internal",
         "//tensorflow/core:stream_executor_no_cuda",
         "//tensorflow/stream_executor",
     ],
diff --git a/tensorflow/compiler/xla/service/executable.cc b/tensorflow/compiler/xla/service/executable.cc
index 62b8fa6a2b..9c96d9eb30 100644
--- a/tensorflow/compiler/xla/service/executable.cc
+++ b/tensorflow/compiler/xla/service/executable.cc
@@ -17,7 +17,9 @@ limitations under the License.
 
 #include "tensorflow/compiler/xla/legacy_flags/debug_options_flags.h"
 #include "tensorflow/compiler/xla/service/hlo_graph_dumper.h"
+#include "tensorflow/compiler/xla/status.h"
 #include "tensorflow/compiler/xla/status_macros.h"
+#include "tensorflow/core/lib/hash/hash.h"
 #include "tensorflow/core/lib/io/path.h"
 #include "tensorflow/core/lib/strings/stringprintf.h"
 #include "tensorflow/core/platform/env.h"
@@ -82,7 +84,11 @@ Status Executable::DumpSessionModule() {
   }
   filename = SanitizeFileName(std::move(filename));
   string file_path = tensorflow::io::JoinPath(directory_path, filename);
-  return tensorflow::WriteBinaryProto(env, file_path, session_module);
+  string result;
+  TF_RET_CHECK(
+      tensorflow::SerializeToStringDeterministic(session_module, &result));
+  return tensorflow::WriteStringToFile(tensorflow::Env::Default(), file_path,
+                                       result);
 }
 
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/service/hlo_module.cc b/tensorflow/compiler/xla/service/hlo_module.cc
index 5bc7a36439..9d4a994838 100644
--- a/tensorflow/compiler/xla/service/hlo_module.cc
+++ b/tensorflow/compiler/xla/service/hlo_module.cc
@@ -154,8 +154,8 @@ string HloModule::ToString() const {
   std::ostringstream s;
   s << "HloModule " << name() << ":\n\n";
   s << "ENTRY " << entry_computation()->ToString() << "\n\n";
-  for (const std::unique_ptr<HloComputation>& computation : computations_) {
-    if (computation.get() != entry_computation()) {
+  for (const HloComputation* computation : MakeNonfusionComputations()) {
+    if (computation != entry_computation()) {
       s << computation->ToString() << "\n\n";
     }
   }
diff --git a/tensorflow/compiler/xla/util.h b/tensorflow/compiler/xla/util.h
index f6c0bd1563..f58f57b443 100644
--- a/tensorflow/compiler/xla/util.h
+++ b/tensorflow/compiler/xla/util.h
@@ -24,6 +24,7 @@ limitations under the License.
 #include <vector>
 
 #include "tensorflow/compiler/xla/status.h"
+#include "tensorflow/compiler/xla/status_macros.h"
 #include "tensorflow/compiler/xla/types.h"
 #include "tensorflow/compiler/xla/xla_data.pb.h"
 #include "tensorflow/core/lib/core/status.h"
diff --git a/tensorflow/contrib/data/python/kernel_tests/BUILD b/tensorflow/contrib/data/python/kernel_tests/BUILD
index c34c9dad9b..7ec049d29b 100644
--- a/tensorflow/contrib/data/python/kernel_tests/BUILD
+++ b/tensorflow/contrib/data/python/kernel_tests/BUILD
@@ -11,6 +11,9 @@ py_test(
     size = "small",
     srcs = ["batch_dataset_op_test.py"],
     srcs_version = "PY2AND3",
+    tags = [
+        "manual",  # b/67958604
+    ],
     deps = [
         "//tensorflow/contrib/data/python/ops:dataset_ops",
         "//tensorflow/contrib/data/python/ops:transformation_ops",
@@ -352,6 +355,9 @@ py_test(
     size = "small",
     srcs = ["sloppy_transformation_dataset_op_test.py"],
     srcs_version = "PY2AND3",
+    tags = [
+        "manual",  # b/67958761
+    ],
     deps = [
         "//tensorflow/contrib/data/python/ops:dataset_ops",
         "//tensorflow/contrib/data/python/ops:transformation_ops",
diff --git a/tensorflow/contrib/distributions/python/kernel_tests/poisson_lognormal_test.py b/tensorflow/contrib/distributions/python/kernel_tests/poisson_lognormal_test.py
index 7cb46bb236..3ded4159d8 100644
--- a/tensorflow/contrib/distributions/python/kernel_tests/poisson_lognormal_test.py
+++ b/tensorflow/contrib/distributions/python/kernel_tests/poisson_lognormal_test.py
@@ -18,6 +18,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import numpy as np
+
 from tensorflow.contrib.distributions.python.ops import poisson_lognormal
 from tensorflow.contrib.distributions.python.ops import test_util
 from tensorflow.python.platform import test
@@ -32,7 +34,8 @@ class PoissonLogNormalQuadratureCompoundTest(
       pln = poisson_lognormal.PoissonLogNormalQuadratureCompound(
           loc=-2.,
           scale=1.1,
-          quadrature_polynomial_degree=10,
+          quadrature_grid_and_probs=(
+              np.polynomial.hermite.hermgauss(deg=10)),
           validate_args=True)
       self.run_test_sample_consistent_log_prob(
           sess, pln, rtol=0.1)
@@ -42,7 +45,8 @@ class PoissonLogNormalQuadratureCompoundTest(
       pln = poisson_lognormal.PoissonLogNormalQuadratureCompound(
           loc=0.,
           scale=1.,
-          quadrature_polynomial_degree=10,
+          quadrature_grid_and_probs=(
+              np.polynomial.hermite.hermgauss(deg=10)),
           validate_args=True)
       self.run_test_sample_consistent_mean_variance(
           sess, pln, rtol=0.02)
@@ -52,7 +56,8 @@ class PoissonLogNormalQuadratureCompoundTest(
       pln = poisson_lognormal.PoissonLogNormalQuadratureCompound(
           loc=[0., -0.5],
           scale=1.,
-          quadrature_polynomial_degree=10,
+          quadrature_grid_and_probs=(
+              np.polynomial.hermite.hermgauss(deg=10)),
           validate_args=True)
       self.run_test_sample_consistent_log_prob(
           sess, pln, rtol=0.1, atol=0.01)
@@ -62,7 +67,8 @@ class PoissonLogNormalQuadratureCompoundTest(
       pln = poisson_lognormal.PoissonLogNormalQuadratureCompound(
           loc=[0., -0.5],
           scale=1.,
-          quadrature_polynomial_degree=10,
+          quadrature_grid_and_probs=(
+              np.polynomial.hermite.hermgauss(deg=10)),
           validate_args=True)
       self.run_test_sample_consistent_mean_variance(
           sess, pln, rtol=0.1, atol=0.01)
@@ -72,7 +78,8 @@ class PoissonLogNormalQuadratureCompoundTest(
       pln = poisson_lognormal.PoissonLogNormalQuadratureCompound(
           loc=[[0.], [-0.5]],
           scale=[[1., 0.9]],
-          quadrature_polynomial_degree=10,
+          quadrature_grid_and_probs=(
+              np.polynomial.hermite.hermgauss(deg=10)),
           validate_args=True)
       self.run_test_sample_consistent_log_prob(
           sess, pln, rtol=0.1, atol=0.08)
@@ -82,7 +89,8 @@ class PoissonLogNormalQuadratureCompoundTest(
       pln = poisson_lognormal.PoissonLogNormalQuadratureCompound(
           loc=[[0.], [-0.5]],
           scale=[[1., 0.9]],
-          quadrature_polynomial_degree=10,
+          quadrature_grid_and_probs=(
+              np.polynomial.hermite.hermgauss(deg=10)),
           validate_args=True)
       self.run_test_sample_consistent_mean_variance(
           sess, pln, rtol=0.1, atol=0.01)
diff --git a/tensorflow/contrib/distributions/python/ops/poisson_lognormal.py b/tensorflow/contrib/distributions/python/ops/poisson_lognormal.py
index 65ee3a16d6..80d4e2dc5e 100644
--- a/tensorflow/contrib/distributions/python/ops/poisson_lognormal.py
+++ b/tensorflow/contrib/distributions/python/ops/poisson_lognormal.py
@@ -93,7 +93,7 @@ class PoissonLogNormalQuadratureCompound(distribution_lib.Distribution):
           : d=0, ..., deg-1 }
   ```
 
-  where, [`grid, w = numpy.polynomial.hermite.hermgauss(deg)`](
+  where, [e.g., `grid, w = numpy.polynomial.hermite.hermgauss(deg)`](
   https://docs.scipy.org/doc/numpy-1.10.0/reference/generated/numpy.polynomial.hermite.hermgauss.html)
   and `prob = w / sqrt(pi)`.
 
@@ -106,14 +106,15 @@ class PoissonLogNormalQuadratureCompound(distribution_lib.Distribution):
   pln = ds.PoissonLogNormalQuadratureCompound(
       loc=[0., -0.5],
       scale=1.,
-      quadrature_polynomial_degree=10,
+      quadrature_grid_and_probs=(
+        np.polynomial.hermite.hermgauss(deg=10)),
       validate_args=True)
   """
 
   def __init__(self,
                loc,
                scale,
-               quadrature_polynomial_degree=8,
+               quadrature_grid_and_probs=None,
                validate_args=False,
                allow_nan_stats=True,
                name="PoissonLogNormalQuadratureCompound"):
@@ -124,8 +125,9 @@ class PoissonLogNormalQuadratureCompound(distribution_lib.Distribution):
         the LogNormal prior.
       scale: `float`-like (batch of) scalar `Tensor`; the scale parameter of
         the LogNormal prior.
-      quadrature_polynomial_degree: Python `int`-like scalar.
-        Default value: 8.
+      quadrature_grid_and_probs: Python pair of `list`-like objects representing
+        the sample points and the corresponding (possibly normalized) weight.
+        When `None`, defaults to: `np.polynomial.hermite.hermgauss(deg=8)`.
       validate_args: Python `bool`, default `False`. When `True` distribution
         parameters are checked for validity despite possibly degrading runtime
         performance. When `False` invalid inputs may silently render incorrect
@@ -138,6 +140,8 @@ class PoissonLogNormalQuadratureCompound(distribution_lib.Distribution):
 
     Raises:
       TypeError: if `loc.dtype != scale[0].dtype`.
+      ValueError: if `quadrature_grid_and_probs is not None` and
+        `len(quadrature_grid_and_probs[0]) != len(quadrature_grid_and_probs[1])`
     """
     parameters = locals()
     with ops.name_scope(name, values=[loc, scale]):
@@ -153,18 +157,21 @@ class PoissonLogNormalQuadratureCompound(distribution_lib.Distribution):
             "loc.dtype(\"{}\") does not match scale.dtype(\"{}\")".format(
                 loc.dtype.name, scale.dtype.name))
 
-      self._degree = quadrature_polynomial_degree
-
-      grid, prob = np.polynomial.hermite.hermgauss(
-          deg=quadrature_polynomial_degree)
-
-      # It should be that `sum(prob) == sqrt(pi)`, but self-normalization is
-      # more numerically stable.
-      prob = prob.astype(dtype.as_numpy_dtype)
-      prob /= np.linalg.norm(prob, ord=1)
+      if quadrature_grid_and_probs is None:
+        grid, probs = np.polynomial.hermite.hermgauss(deg=8)
+      else:
+        grid, probs = tuple(quadrature_grid_and_probs)
+        if len(grid) != len(probs):
+          raise ValueError("`quadrature_grid_and_probs` must be a `tuple` of "
+                           "same-length list-like objects")
+      grid = grid.astype(dtype.as_numpy_dtype)
+      probs = probs.astype(dtype.as_numpy_dtype)
+      probs /= np.linalg.norm(probs, ord=1)
+      self._quadrature_grid = grid
+      self._quadrature_probs = probs
 
       self._mixture_distribution = categorical_lib.Categorical(
-          logits=np.log(prob),
+          logits=np.log(probs),
           validate_args=validate_args,
           allow_nan_stats=allow_nan_stats)
 
@@ -210,9 +217,14 @@ class PoissonLogNormalQuadratureCompound(distribution_lib.Distribution):
     return self._scale
 
   @property
-  def quadrature_polynomial_degree(self):
-    """Polynomial largest exponent used for Gauss-Hermite quadrature."""
-    return self._degree
+  def quadrature_grid(self):
+    """Quadrature grid points."""
+    return self._quadrature_grid
+
+  @property
+  def quadrature_probs(self):
+    """Quadrature normalized weights."""
+    return self._quadrature_probs
 
   def _batch_shape_tensor(self):
     return array_ops.broadcast_dynamic_shape(
@@ -242,10 +254,10 @@ class PoissonLogNormalQuadratureCompound(distribution_lib.Distribution):
                 [batch_size])),
         seed=distribution_util.gen_new_seed(
             seed, "poisson_lognormal_quadrature_compound"))
-    # Stride `quadrature_polynomial_degree` for `batch_size` number of times.
+    # Stride `quadrature_degree` for `batch_size` number of times.
     offset = math_ops.range(start=0,
-                            limit=batch_size * self._degree,
-                            delta=self._degree,
+                            limit=batch_size * len(self.quadrature_probs),
+                            delta=len(self.quadrature_probs),
                             dtype=ids.dtype)
     ids += offset
     rate = array_ops.gather(
diff --git a/tensorflow/contrib/distributions/python/ops/vector_diffeomixture.py b/tensorflow/contrib/distributions/python/ops/vector_diffeomixture.py
index 438d628da4..33dad811a9 100644
--- a/tensorflow/contrib/distributions/python/ops/vector_diffeomixture.py
+++ b/tensorflow/contrib/distributions/python/ops/vector_diffeomixture.py
@@ -141,7 +141,7 @@ class VectorDiffeomixture(distribution_lib.Distribution):
   and,
 
   ```none
-  grid, weight = np.polynomial.hermite.hermgauss(quadrature_polynomial_degree)
+  grid, weight = np.polynomial.hermite.hermgauss(quadrature_degree)
   prob[k]   = weight[k] / sqrt(pi)
   lambda[k; i] = sigmoid(mix_loc[k] + sqrt(2) mix_scale[k] grid[i])
   ```
@@ -219,7 +219,7 @@ class VectorDiffeomixture(distribution_lib.Distribution):
                distribution,
                loc=None,
                scale=None,
-               quadrature_polynomial_degree=8,
+               quadrature_grid_and_probs=None,
                validate_args=False,
                allow_nan_stats=True,
                name="VectorDiffeomixture"):
@@ -248,7 +248,9 @@ class VectorDiffeomixture(distribution_lib.Distribution):
         `k`-th element represents the `scale` used for the `k`-th affine
         transformation. `LinearOperator`s must have shape `[B1, ..., Bb, d, d]`,
         `b >= 0`, i.e., characterizes `b`-batches of `d x d` matrices
-      quadrature_polynomial_degree: Python `int`-like scalar.
+      quadrature_grid_and_probs: Python pair of `list`-like objects representing
+        the sample points and the corresponding (possibly normalized) weight.
+        When `None`, defaults to: `np.polynomial.hermite.hermgauss(deg=8)`.
       validate_args: Python `bool`, default `False`. When `True` distribution
         parameters are checked for validity despite possibly degrading runtime
         performance. When `False` invalid inputs may silently render incorrect
@@ -262,7 +264,8 @@ class VectorDiffeomixture(distribution_lib.Distribution):
     Raises:
       ValueError: if `not scale or len(scale) < 2`.
       ValueError: if `len(loc) != len(scale)`
-      ValueError: if `quadrature_polynomial_degree < 1`.
+      ValueError: if `quadrature_grid_and_probs is not None` and
+        `len(quadrature_grid_and_probs[0]) != len(quadrature_grid_and_probs[1])`
       ValueError: if `validate_args` and any not scale.is_positive_definite.
       TypeError: if any scale.dtype != scale[0].dtype.
       TypeError: if any loc.dtype != scale[0].dtype.
@@ -307,12 +310,6 @@ class VectorDiffeomixture(distribution_lib.Distribution):
                                name="endpoint_affine_{}".format(k))
           for k, (loc_, scale_) in enumerate(zip(loc, scale))]
 
-      if quadrature_polynomial_degree < 1:
-        raise ValueError("quadrature_polynomial_degree={} "
-                         "is not at least 1".format(
-                             quadrature_polynomial_degree))
-      self._degree = quadrature_polynomial_degree
-
       # TODO(jvdillon): Remove once we support k-mixtures.
       # We make this assertion here because otherwise `grid` would need to be a
       # vector not a scalar.
@@ -320,17 +317,24 @@ class VectorDiffeomixture(distribution_lib.Distribution):
         raise NotImplementedError("Currently only bimixtures are supported; "
                                   "len(scale)={} is not 2.".format(len(scale)))
 
-      grid, prob = np.polynomial.hermite.hermgauss(
-          deg=quadrature_polynomial_degree)
+      if quadrature_grid_and_probs is None:
+        grid, probs = np.polynomial.hermite.hermgauss(deg=8)
+      else:
+        grid, probs = tuple(quadrature_grid_and_probs)
+        if len(grid) != len(probs):
+          raise ValueError("`quadrature_grid_and_probs` must be a `tuple` of "
+                           "same-length list-like objects")
       grid = grid.astype(dtype.as_numpy_dtype)
-      prob = prob.astype(dtype.as_numpy_dtype)
-      prob /= np.linalg.norm(prob, ord=1)
+      probs = probs.astype(dtype.as_numpy_dtype)
+      probs /= np.linalg.norm(probs, ord=1)
+      self._quadrature_grid = grid
+      self._quadrature_probs = probs
 
       # Note: by creating the logits as `log(prob)` we ensure that
       # `self.mixture_distribution.logits` is equivalent to
       # `math_ops.log(self.mixture_distribution.probs)`.
       self._mixture_distribution = categorical_lib.Categorical(
-          logits=np.log(prob),
+          logits=np.log(probs),
           validate_args=validate_args,
           allow_nan_stats=allow_nan_stats)
 
@@ -357,10 +361,10 @@ class VectorDiffeomixture(distribution_lib.Distribution):
                                validate_args=validate_args,
                                name="interpolated_affine_{}".format(k))
           for k, (loc_, scale_) in enumerate(zip(
-              interpolate_loc(quadrature_polynomial_degree,
+              interpolate_loc(len(self._quadrature_grid),
                               self._interpolate_weight,
                               loc),
-              interpolate_scale(quadrature_polynomial_degree,
+              interpolate_scale(len(self._quadrature_grid),
                                 self._interpolate_weight,
                                 scale)))]
 
@@ -416,9 +420,14 @@ class VectorDiffeomixture(distribution_lib.Distribution):
     return self._interpolated_affine
 
   @property
-  def quadrature_polynomial_degree(self):
-    """Polynomial largest exponent used for Gauss-Hermite quadrature."""
-    return self._degree
+  def quadrature_grid(self):
+    """Quadrature grid points."""
+    return self._quadrature_grid
+
+  @property
+  def quadrature_probs(self):
+    """Quadrature normalized weights."""
+    return self._quadrature_probs
 
   def _batch_shape_tensor(self):
     return self._batch_shape_
@@ -454,10 +463,10 @@ class VectorDiffeomixture(distribution_lib.Distribution):
         seed=distribution_util.gen_new_seed(
             seed, "vector_diffeomixture"))
 
-    # Stride `self._degree` for `batch_size` number of times.
+    # Stride `quadrature_degree` for `batch_size` number of times.
     offset = math_ops.range(start=0,
-                            limit=batch_size * self._degree,
-                            delta=self._degree,
+                            limit=batch_size * len(self.quadrature_probs),
+                            delta=len(self.quadrature_probs),
                             dtype=ids.dtype)
 
     weight = array_ops.gather(
diff --git a/tensorflow/contrib/estimator/python/estimator/head.py b/tensorflow/contrib/estimator/python/estimator/head.py
index f8648fe5bf..ebf91e8bb4 100644
--- a/tensorflow/contrib/estimator/python/estimator/head.py
+++ b/tensorflow/contrib/estimator/python/estimator/head.py
@@ -265,6 +265,9 @@ class _MultiLabelHead(head_lib._Head):  # pylint:disable=protected-access
     unweighted_loss = losses.sigmoid_cross_entropy(
         multi_class_labels=processed_labels, logits=logits,
         reduction=losses.Reduction.NONE)
+    # Averages loss over classes.
+    unweighted_loss = math_ops.reduce_mean(
+        unweighted_loss, axis=-1, keep_dims=True)
     return head_lib.LossAndLabels(
         unweighted_loss=unweighted_loss,
         processed_labels=processed_labels)
@@ -294,12 +297,9 @@ class _MultiLabelHead(head_lib._Head):  # pylint:disable=protected-access
       # Eval.
       unweighted_loss, processed_labels = self.create_loss(
           features=features, mode=mode, logits=logits, labels=labels)
-      # Averages loss over classes.
-      per_example_loss = math_ops.reduce_mean(
-          unweighted_loss, axis=-1, keep_dims=True)
       weights = head_lib._weights(features, self._weight_column)  # pylint:disable=protected-access
       training_loss = losses.compute_weighted_loss(
-          per_example_loss, weights=weights, reduction=losses.Reduction.SUM)
+          unweighted_loss, weights=weights, reduction=losses.Reduction.SUM)
       if mode == model_fn.ModeKeys.EVAL:
         return model_fn.EstimatorSpec(
             mode=model_fn.ModeKeys.EVAL,
@@ -309,7 +309,7 @@ class _MultiLabelHead(head_lib._Head):  # pylint:disable=protected-access
                 labels=processed_labels,
                 probabilities=probabilities,
                 weights=weights,
-                per_example_loss=per_example_loss))
+                unweighted_loss=unweighted_loss))
 
       # Train.
       if train_op_fn is None:
@@ -330,16 +330,16 @@ class _MultiLabelHead(head_lib._Head):  # pylint:disable=protected-access
         loss=training_loss,
         train_op=train_op_fn(training_loss))
 
-  def _eval_metric_ops(self, labels, probabilities, weights, per_example_loss):
+  def _eval_metric_ops(self, labels, probabilities, weights, unweighted_loss):
     """Returns a dict of metrics for eval_metric_ops."""
     with ops.name_scope(
-        None, 'metrics', [labels, probabilities, weights, per_example_loss]):
+        None, 'metrics', [labels, probabilities, weights, unweighted_loss]):
       keys = metric_keys.MetricKeys
       metric_ops = {
           # Estimator already adds a metric for loss.
           head_lib._summary_key(self._name, keys.LOSS_MEAN):  # pylint:disable=protected-access
               metrics_lib.mean(
-                  per_example_loss, weights=weights, name=keys.LOSS_MEAN),
+                  unweighted_loss, weights=weights, name=keys.LOSS_MEAN),
           head_lib._summary_key(self._name, keys.AUC):  # pylint:disable=protected-access
               metrics_lib.auc(
                   labels=labels, predictions=probabilities, weights=weights,
diff --git a/tensorflow/contrib/estimator/python/estimator/head_test.py b/tensorflow/contrib/estimator/python/estimator/head_test.py
index dcbe62b497..ec1386af34 100644
--- a/tensorflow/contrib/estimator/python/estimator/head_test.py
+++ b/tensorflow/contrib/estimator/python/estimator/head_test.py
@@ -80,9 +80,13 @@ def _sigmoid(logits):
 
 
 def _sigmoid_cross_entropy(labels, logits):
+  """Returns sigmoid cross entropy averaged over classes."""
   sigmoid_logits = _sigmoid(logits)
-  return (-labels * np.log(sigmoid_logits)
-          -(1 - labels) * np.log(1 - sigmoid_logits))
+  unreduced_result = (
+      -labels * np.log(sigmoid_logits)
+      -(1 - labels) * np.log(1 - sigmoid_logits))
+  # Mean over classes
+  return np.mean(unreduced_result, axis=-1, keepdims=True)
 
 
 class MultiLabelHead(test.TestCase):
@@ -226,7 +230,7 @@ class MultiLabelHead(test.TestCase):
     # loss = labels * (logits < 0) * (-logits) +
     #        (1 - labels) * (logits > 0) * logits
     expected_unweighted_loss = np.array(
-        [[10., 10.], [15., 0.]], dtype=np.float32)
+        [[(10. + 10.) / 2.], [(15. + 0.) / 2.]], dtype=np.float32)
     actual_unweighted_loss, _ = head.create_loss(
         features={'x': np.array(((42,),), dtype=np.int32)},
         mode=model_fn.ModeKeys.EVAL,
@@ -311,10 +315,8 @@ class MultiLabelHead(test.TestCase):
     labels = np.array([[1, 0], [1, 1]], dtype=np.int64)
     # loss = labels * -log(sigmoid(logits)) +
     #        (1 - labels) * -log(1 - sigmoid(logits))
-    # Average over classes, and sum over examples.
-    expected_loss = (
-        np.sum(_sigmoid_cross_entropy(labels=labels, logits=logits)) / n_classes
-    )
+    # Sum over examples.
+    expected_loss = np.sum(_sigmoid_cross_entropy(labels=labels, logits=logits))
     keys = metric_keys.MetricKeys
     expected_metrics = {
         # Average loss over examples.
@@ -343,10 +345,9 @@ class MultiLabelHead(test.TestCase):
     labels_multi_hot = np.array([[1, 0], [1, 1]], dtype=np.int64)
     # loss = labels * -log(sigmoid(logits)) +
     #        (1 - labels) * -log(1 - sigmoid(logits))
-    # Average over classes, and sum over examples.
+    # Sum over examples.
     expected_loss = (
-        np.sum(_sigmoid_cross_entropy(labels=labels_multi_hot, logits=logits)) /
-        n_classes
+        np.sum(_sigmoid_cross_entropy(labels=labels_multi_hot, logits=logits))
     )
     keys = metric_keys.MetricKeys
     expected_metrics = {
@@ -377,10 +378,9 @@ class MultiLabelHead(test.TestCase):
     labels_multi_hot = np.array([[1, 0], [1, 1]], dtype=np.int64)
     # loss = labels * -log(sigmoid(logits)) +
     #        (1 - labels) * -log(1 - sigmoid(logits))
-    # Average over classes, and sum over examples.
+    # Sum over examples.
     expected_loss = (
-        np.sum(_sigmoid_cross_entropy(labels=labels_multi_hot, logits=logits)) /
-        n_classes
+        np.sum(_sigmoid_cross_entropy(labels=labels_multi_hot, logits=logits))
     )
     keys = metric_keys.MetricKeys
     expected_metrics = {
@@ -407,9 +407,9 @@ class MultiLabelHead(test.TestCase):
     labels = np.array([[1, 0], [1, 1]], dtype=np.int64)
     # loss = labels * -log(sigmoid(logits)) +
     #        (1 - labels) * -log(1 - sigmoid(logits))
-    # Average over classes, and sum over examples.
+    # Sum over examples.
     expected_loss = (
-        np.sum(_sigmoid_cross_entropy(labels=labels, logits=logits)) / n_classes
+        np.sum(_sigmoid_cross_entropy(labels=labels, logits=logits))
     )
 
     keys = metric_keys.MetricKeys
@@ -506,7 +506,7 @@ class MultiLabelHead(test.TestCase):
     # loss = labels * (logits < 0) * (-logits) +
     #        (1 - labels) * (logits > 0) * logits
     expected_unweighted_loss = np.array(
-        [[10., 10.], [15., 0.]], dtype=np.float32)
+        [[(10. + 10.) / 2.], [(15. + 0.) / 2.]], dtype=np.float32)
     actual_unweighted_loss, _ = head.create_loss(
         features={'x': np.array(((42,),), dtype=np.int32)},
         mode=model_fn.ModeKeys.TRAIN,
diff --git a/tensorflow/contrib/gan/python/eval/python/classifier_metrics_impl.py b/tensorflow/contrib/gan/python/eval/python/classifier_metrics_impl.py
index 6074694f8b..4af87b8b47 100644
--- a/tensorflow/contrib/gan/python/eval/python/classifier_metrics_impl.py
+++ b/tensorflow/contrib/gan/python/eval/python/classifier_metrics_impl.py
@@ -16,6 +16,11 @@
 
 These methods come from https://arxiv.org/abs/1606.03498 and
 https://arxiv.org/abs/1706.08500.
+
+NOTE: This implementation uses the same weights as in
+https://github.com/openai/improved-gan/blob/master/inception_score/model.py,
+but is more numerically stable and is an unbiased estimator of the true
+Inception score even when splitting the inputs into batches.
 """
 
 from __future__ import absolute_import
@@ -54,17 +59,16 @@ __all__ = [
     'classifier_score',
     'frechet_inception_distance',
     'frechet_classifier_distance',
+    'INCEPTION_DEFAULT_IMAGE_SIZE',
 ]
 
 
-INCEPTION_URL = 'http://download.tensorflow.org/models/frozen_inception_v3_2017_09_13.tar.gz'
-INCEPTION_FROZEN_GRAPH = 'frozen_inception_v3.pb'
-INCEPTION_V3_INPUT = 'input'
-INCEPTION_V3_OUTPUT = 'InceptionV3/Logits/SpatialSqueeze:0'
-INCEPTION_V3_FINAL_POOL = 'InceptionV3/Logits/AvgPool_1a_8x8/AvgPool:0'
-_INCEPTION_V3_NUM_CLASSES = 1001
-_INCEPTION_V3_FINAL_POOL_SIZE = 2048
-INCEPTION_V3_DEFAULT_IMG_SIZE = 299
+INCEPTION_URL = 'http://download.tensorflow.org/models/frozen_inception_v1_2015_12_05.tar.gz'
+INCEPTION_FROZEN_GRAPH = 'inceptionv1_for_inception_score.pb'
+INCEPTION_INPUT = 'Mul:0'
+INCEPTION_OUTPUT = 'logits:0'
+INCEPTION_FINAL_POOL = 'pool_3:0'
+INCEPTION_DEFAULT_IMAGE_SIZE = 299
 
 
 def _validate_images(images, image_size):
@@ -106,42 +110,33 @@ def _symmetric_matrix_square_root(mat, eps=1e-10):
 # NOTE: Floating-point inputs are expected to be in [0, 1].
 # Copied from /tensorflow_models/slim/preprocessing/inception_preprocessing.py.
 def preprocess_image(
-    image, height=INCEPTION_V3_DEFAULT_IMG_SIZE,
-    width=INCEPTION_V3_DEFAULT_IMG_SIZE, central_fraction=0.875, scope=None):
-  """Prepare one image for evaluation.
-
-  If height and width are specified it would output an image with that size by
-  applying resize_bilinear.
+    images, height=INCEPTION_DEFAULT_IMAGE_SIZE,
+    width=INCEPTION_DEFAULT_IMAGE_SIZE, scope=None):
+  """Prepare a batch of images for evaluation.
 
-  If central_fraction is specified it would crop the central fraction of the
-  input image.
+  This is the preprocessing portion of the graph from
+  http://download.tensorflow.org/models/image/imagenet/inception-2015-12-05.tgz.
 
   Args:
-    image: 3-D Tensor of image. If dtype is tf.float32 then the range should be
-      [0, 1], otherwise it would converted to tf.float32 assuming that the range
-      is [0, MAX], where MAX is largest positive representable number for
-      int(8/16/32) data type (see `tf.image.convert_image_dtype` for details).
-    height: integer
-    width: integer
-    central_fraction: Optional Float, fraction of the image to crop.
+    images: 3-D or 4-D Tensor of images. Values are in [0, 255].
+    height: Integer. Height of resized output image.
+    width: Integer. Width of resized output image.
     scope: Optional scope for name_scope.
+
   Returns:
-    3-D float Tensor of prepared image.
+    3-D or 4-D float Tensor of prepared image(s). Values are in [-1, 1].
   """
-  with ops.name_scope(scope, 'eval_image', [image, height, width]):
-    if image.dtype != dtypes.float32:
-      image = image_ops.convert_image_dtype(image, dtype=dtypes.float32)
-    # Crop the central region of the image with an area containing 87.5% of
-    # the original image.
-    image = image_ops.central_crop(image, central_fraction=central_fraction)
-
-    # Resize the image to the specified height and width.
-    image = array_ops.expand_dims(image, 0)
-    image = image_ops.resize_bilinear(image, [height, width],
-                                      align_corners=False)
-    image = array_ops.squeeze(image, [0])
-    image = (image - 0.5) * 2.0
-    return image
+  is_single = images.shape.ndims == 3
+  with ops.name_scope(scope, 'preprocess', [images, height, width]):
+    if not images.dtype.is_floating:
+      images = math_ops.to_float(images)
+    images = (images - 128.0) / 128.0
+    if is_single:
+      images = array_ops.expand_dims(images, axis=0)
+    resized = image_ops.resize_bilinear(images, [height, width])
+    if is_single:
+      resized = array_ops.squeeze(resized, axis=0)
+    return resized
 
 
 def _kl_divergence(p, p_logits, q):
@@ -211,9 +206,9 @@ def _default_graph_def_fn():
 def run_inception(images,
                   graph_def=None,
                   default_graph_def_fn=_default_graph_def_fn,
-                  image_size=INCEPTION_V3_DEFAULT_IMG_SIZE,
-                  input_tensor=INCEPTION_V3_INPUT,
-                  output_tensor=INCEPTION_V3_OUTPUT):
+                  image_size=INCEPTION_DEFAULT_IMAGE_SIZE,
+                  input_tensor=INCEPTION_INPUT,
+                  output_tensor=INCEPTION_OUTPUT):
   """Run images through a pretrained Inception classifier.
 
   Args:
@@ -338,7 +333,7 @@ def classifier_score(images, classifier_fn, num_batches=1):
 inception_score = functools.partial(
     classifier_score,
     classifier_fn=functools.partial(
-        run_inception, output_tensor=INCEPTION_V3_OUTPUT))
+        run_inception, output_tensor=INCEPTION_OUTPUT))
 
 
 def trace_sqrt_product(sigma, sigma_v):
@@ -479,4 +474,4 @@ def frechet_classifier_distance(real_images,
 frechet_inception_distance = functools.partial(
     frechet_classifier_distance,
     classifier_fn=functools.partial(
-        run_inception, output_tensor=INCEPTION_V3_FINAL_POOL))
+        run_inception, output_tensor=INCEPTION_FINAL_POOL))
diff --git a/tensorflow/contrib/gan/python/eval/python/classifier_metrics_test.py b/tensorflow/contrib/gan/python/eval/python/classifier_metrics_test.py
index 30285964a5..81fa2fc0f1 100644
--- a/tensorflow/contrib/gan/python/eval/python/classifier_metrics_test.py
+++ b/tensorflow/contrib/gan/python/eval/python/classifier_metrics_test.py
@@ -68,7 +68,7 @@ def _expected_trace_sqrt_product(sigma, sigma_v):
 # A dummy GraphDef string with the minimum number of Ops.
 graphdef_string = """
 node {
-  name: "input"
+  name: "Mul"
   op: "Placeholder"
   attr {
     key: "dtype"
@@ -97,7 +97,7 @@ node {
   }
 }
 node {
-  name: "InceptionV3/Logits/SpatialSqueeze"
+  name: "logits"
   op: "Placeholder"
   attr {
     key: "dtype"
@@ -120,7 +120,7 @@ node {
   }
 }
 node {
-  name: "InceptionV3/Logits/AvgPool_1a_8x8/AvgPool"
+  name: "pool_3"
   op: "Placeholder"
   attr {
     key: "dtype"
@@ -182,7 +182,7 @@ class ClassifierMetricsTest(test.TestCase):
     img = array_ops.ones([batch_size, 299, 299, 3])
     pool = _run_with_mock(
         classifier_metrics.run_inception, img,
-        output_tensor=classifier_metrics.INCEPTION_V3_FINAL_POOL)
+        output_tensor=classifier_metrics.INCEPTION_FINAL_POOL)
 
     self.assertTrue(isinstance(pool, ops.Tensor))
     pool.shape.assert_is_compatible_with([batch_size, 2048])
@@ -306,7 +306,7 @@ class ClassifierMetricsTest(test.TestCase):
     """Test `preprocess_image` graph construction."""
     incorrectly_sized_image = array_ops.zeros([520, 240, 3])
     correct_image = classifier_metrics.preprocess_image(
-        image=incorrectly_sized_image)
+        images=incorrectly_sized_image)
     _run_with_mock(classifier_metrics.run_inception,
                    array_ops.expand_dims(correct_image, 0))
 
diff --git a/tensorflow/contrib/kfac/python/kernel_tests/layer_collection_test.py b/tensorflow/contrib/kfac/python/kernel_tests/layer_collection_test.py
index 633104ace0..13c69d261c 100644
--- a/tensorflow/contrib/kfac/python/kernel_tests/layer_collection_test.py
+++ b/tensorflow/contrib/kfac/python/kernel_tests/layer_collection_test.py
@@ -30,6 +30,43 @@ from tensorflow.python.ops import variable_scope
 from tensorflow.python.platform import test
 
 
+class LayerParametersDictTest(test.TestCase):
+
+  def testSetItem(self):
+    """Ensure insertion, contains, retrieval works for supported key types."""
+    with ops.Graph().as_default():
+      lp_dict = layer_collection.LayerParametersDict()
+
+      x = array_ops.constant(0)
+      y0 = array_ops.constant(0)
+      y1 = array_ops.constant(0)
+      z0 = array_ops.constant(0)
+      z1 = array_ops.constant(0)
+      keys = [x, (y0, y1), [z0, z1]]
+      for key in keys:
+        lp_dict[key] = key
+
+      for key in keys:
+        self.assertTrue(key in lp_dict)
+        self.assertEqual(lp_dict[key], key)
+
+  def testSetItemOverlap(self):
+    """Ensure insertion fails if key overlaps with existing key."""
+    with ops.Graph().as_default():
+      lp_dict = layer_collection.LayerParametersDict()
+
+      x = array_ops.constant(0)
+      y = array_ops.constant(0)
+      lp_dict[x] = 'value'
+
+      with self.assertRaises(ValueError):
+        lp_dict[(x, y)] = 'value'
+
+      # Ensure 'y' wasn't inserted.
+      self.assertTrue(x in lp_dict)
+      self.assertFalse(y in lp_dict)
+
+
 class LayerCollectionTest(test.TestCase):
 
   def testLayerCollectionInit(self):
@@ -157,6 +194,36 @@ class LayerCollectionTest(test.TestCase):
       double_loss = sess.run(lc2.total_sampled_loss())
       self.assertAlmostEqual(2 * single_loss, double_loss)
 
+  def testLossFunctionByName(self):
+    """Ensure loss functions can be identified by name."""
+    with ops.Graph().as_default():
+      logits = linalg_ops.eye(2)
+      lc = layer_collection.LayerCollection()
+
+      # Create a new loss function by name.
+      lc.register_categorical_predictive_distribution(logits, name='loss1')
+      self.assertEqual(1, len(lc.losses))
+
+      # Add logits to same loss function.
+      with self.assertRaises(NotImplementedError):
+        lc.register_categorical_predictive_distribution(logits, name='loss1')
+      self.assertEqual(1, len(lc.losses))
+
+      # Add another new loss function.
+      lc.register_categorical_predictive_distribution(logits, name='loss2')
+      self.assertEqual(2, len(lc.losses))
+
+  def testLossFunctionWithoutName(self):
+    """Ensure loss functions get unique names if 'name' not specified."""
+    with ops.Graph().as_default():
+      logits = linalg_ops.eye(2)
+      lc = layer_collection.LayerCollection()
+
+      # Create a new loss function by name.
+      lc.register_categorical_predictive_distribution(logits)
+      lc.register_categorical_predictive_distribution(logits)
+      self.assertEqual(2, len(lc.losses))
+
   def testRegisterCategoricalPredictiveDistributionBatchSize1(self):
     with ops.Graph().as_default():
       random_seed.set_random_seed(200)
diff --git a/tensorflow/contrib/kfac/python/kernel_tests/optimizer_test.py b/tensorflow/contrib/kfac/python/kernel_tests/optimizer_test.py
index 5f28f57f6a..9325aa1b73 100644
--- a/tensorflow/contrib/kfac/python/kernel_tests/optimizer_test.py
+++ b/tensorflow/contrib/kfac/python/kernel_tests/optimizer_test.py
@@ -21,7 +21,6 @@ from __future__ import print_function
 import numpy as np
 
 from tensorflow.contrib.kfac.python.ops import layer_collection as lc
-from tensorflow.contrib.kfac.python.ops import loss_functions as lf
 from tensorflow.contrib.kfac.python.ops import optimizer
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
@@ -124,9 +123,8 @@ class OptimizerTest(test.TestCase):
   def testUpdateVelocities(self):
     with ops.Graph().as_default(), self.test_session() as sess:
       layers = lc.LayerCollection()
-      layers.losses = [
-          lf.CategoricalLogitsNegativeLogProbLoss(array_ops.constant([1.0]))
-      ]
+      layers.register_categorical_predictive_distribution(
+          array_ops.constant([1.0]))
       opt = optimizer.KfacOptimizer(
           0.1, 0.2, 0.3, layers, momentum=0.5, momentum_type='regular')
       x = variable_scope.get_variable('x', initializer=array_ops.ones((2, 2)))
diff --git a/tensorflow/contrib/kfac/python/ops/layer_collection.py b/tensorflow/contrib/kfac/python/ops/layer_collection.py
index 1b77f5d3ba..0cb55894ad 100644
--- a/tensorflow/contrib/kfac/python/ops/layer_collection.py
+++ b/tensorflow/contrib/kfac/python/ops/layer_collection.py
@@ -55,6 +55,7 @@ class LayerParametersDict(OrderedDict):
     super(LayerParametersDict, self).__init__(*args, **kwargs)
 
   def __setitem__(self, key, value):
+    key = self._canonicalize_key(key)
     tensors = key if isinstance(key, (tuple, list)) else (key,)
     key_collisions = self._tensors.intersection(tensors)
     if key_collisions:
@@ -63,9 +64,23 @@ class LayerParametersDict(OrderedDict):
     super(LayerParametersDict, self).__setitem__(key, value)
 
   def __delitem__(self, key):
+    key = self._canonicalize_key(key)
     self._tensors.remove(key)
     super(LayerParametersDict, self).__delitem__(key)
 
+  def __getitem__(self, key):
+    key = self._canonicalize_key(key)
+    return super(LayerParametersDict, self).__getitem__(key)
+
+  def __contains__(self, key):
+    key = self._canonicalize_key(key)
+    return super(LayerParametersDict, self).__contains__(key)
+
+  def _canonicalize_key(self, key):
+    if isinstance(key, (list, tuple)):
+      return tuple(key)
+    return key
+
 
 # TODO(duckworthd): add capability for LayerCollection to be "finalized"
 # and do this when it gets used by FisherEstimator / KfacOptimizer
@@ -94,13 +109,16 @@ class LayerCollection(object):
     self.fisher_factors = OrderedDict()
     self._generic_registrations = set()
     self._graph = graph or ops.get_default_graph()
-    self.losses = []
+    self._loss_dict = {}  # {str: LossFunction}
     self._subgraph = None
 
     with variable_scope.variable_scope(None, default_name=name) as scope:
       self._var_scope = scope.name
 
-  reset_internals = __init__
+  @property
+  def losses(self):
+    """LossFunctions registered with this LayerCollection."""
+    return list(self._loss_dict.values())
 
   def register_block(self, layer_key, fisher_block):
     """Validates and registers the layer_key associated with the fisher_block.
@@ -277,7 +295,8 @@ class LayerCollection(object):
   def register_categorical_predictive_distribution(self,
                                                    logits,
                                                    seed=None,
-                                                   targets=None):
+                                                   targets=None,
+                                                   name=None):
     """Registers a categorical predictive distribution.
 
     Args:
@@ -288,16 +307,24 @@ class LayerCollection(object):
         total_loss() is required, for example, to estimate the
         "empirical Fisher" (instead of the true Fisher).
         (Default: None)
+      name: (OPTIONAL) str or None. Unique name for this loss function. If None,
+        a new name is generated. (Default: None)
     """
+    name = name or self._graph.unique_name(
+        "register_categorical_predictive_distribution")
+    if name in self._loss_dict:
+      raise NotImplementedError(
+          "Adding logits to an existing LossFunction not yet supported.")
     loss = lf.CategoricalLogitsNegativeLogProbLoss(
         logits, targets=targets, seed=seed)
-    self.losses.append(loss)
+    self._loss_dict[name] = loss
 
   def register_normal_predictive_distribution(self,
                                               mean,
                                               var=0.5,
                                               seed=None,
-                                              targets=None):
+                                              targets=None,
+                                              name=None):
     """Registers a normal predictive distribution.
 
     Args:
@@ -312,15 +339,23 @@ class LayerCollection(object):
         total_loss() is required, for example, to estimate the
         "empirical Fisher" (instead of the true Fisher).
         (Default: None)
+      name: (OPTIONAL) str or None. Unique name for this loss function. If None,
+        a new name is generated. (Default: None)
     """
+    name = name or self._graph.unique_name(
+        "register_normal_predictive_distribution")
+    if name in self._loss_dict:
+      raise NotImplementedError(
+          "Adding logits to an existing LossFunction not yet supported.")
     loss = lf.NormalMeanNegativeLogProbLoss(
         mean, var, targets=targets, seed=seed)
-    self.losses.append(loss)
+    self._loss_dict[name] = loss
 
   def register_multi_bernoulli_predictive_distribution(self,
                                                        logits,
                                                        seed=None,
-                                                       targets=None):
+                                                       targets=None,
+                                                       name=None):
     """Registers a multi-Bernoulli predictive distribution.
 
     Args:
@@ -331,10 +366,17 @@ class LayerCollection(object):
         total_loss() is required, for example, to estimate the
         "empirical Fisher" (instead of the true Fisher).
         (Default: None)
+      name: (OPTIONAL) str or None. Unique name for this loss function. If None,
+        a new name is generated. (Default: None)
     """
+    name = name or self._graph.unique_name(
+        "register_multi_bernoulli_predictive_distribution")
+    if name in self._loss_dict:
+      raise NotImplementedError(
+          "Adding logits to an existing LossFunction not yet supported.")
     loss = lf.MultiBernoulliNegativeLogProbLoss(
         logits, targets=targets, seed=seed)
-    self.losses.append(loss)
+    self._loss_dict[name] = loss
 
   def make_or_get_factor(self, cls, args):
     with variable_scope.variable_scope(self._var_scope):
diff --git a/tensorflow/contrib/learn/python/learn/estimators/estimator.py b/tensorflow/contrib/learn/python/learn/estimators/estimator.py
index 8bb1c83a45..788d2d0b1a 100644
--- a/tensorflow/contrib/learn/python/learn/estimators/estimator.py
+++ b/tensorflow/contrib/learn/python/learn/estimators/estimator.py
@@ -981,9 +981,8 @@ class BaseEstimator(
       global_step = training_util.create_global_step(g)
       features, labels = input_fn()
       self._check_inputs(features, labels)
-      global_step_read_tensor = training_util._get_or_create_global_step_read()  # pylint: disable=protected-access
-      with ops.control_dependencies([global_step_read_tensor]):
-        model_fn_ops = self._get_train_ops(features, labels)
+      training_util._get_or_create_global_step_read()  # pylint: disable=protected-access
+      model_fn_ops = self._get_train_ops(features, labels)
       ops.add_to_collection(ops.GraphKeys.LOSSES, model_fn_ops.loss)
       all_hooks.extend(hooks)
       all_hooks.extend([
diff --git a/tensorflow/contrib/predictor/predictor.py b/tensorflow/contrib/predictor/predictor.py
index dbc0028259..28fa815684 100644
--- a/tensorflow/contrib/predictor/predictor.py
+++ b/tensorflow/contrib/predictor/predictor.py
@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-
 """Abstract base class for all predictors."""
 
 from __future__ import absolute_import
@@ -66,8 +65,9 @@ class Predictor(object):
     expected_keys = set(self.feed_tensors.keys())
     unexpected_keys = input_keys - expected_keys
     if unexpected_keys:
-      raise ValueError('Got unexpected keys in input_dict: {}'.format(
-          unexpected_keys))
+      raise ValueError(
+          'Got unexpected keys in input_dict: {}\nexpected: {}'.format(
+              unexpected_keys, expected_keys))
 
     feed_dict = {}
     for key in self.feed_tensors.keys():
diff --git a/tensorflow/contrib/training/BUILD b/tensorflow/contrib/training/BUILD
index 80a5debe99..0df5ff50c0 100644
--- a/tensorflow/contrib/training/BUILD
+++ b/tensorflow/contrib/training/BUILD
@@ -26,6 +26,7 @@ py_library(
         "python/training/resample.py",
         "python/training/sampling_ops.py",
         "python/training/sequence_queueing_state_saver.py",
+        "python/training/sgdr_learning_rate_decay.py",
         "python/training/training.py",
         "python/training/tuner.py",
     ],
diff --git a/tensorflow/core/kernels/split_op.cc b/tensorflow/core/kernels/split_op.cc
index 4d2100c59c..58e1a73be6 100644
--- a/tensorflow/core/kernels/split_op.cc
+++ b/tensorflow/core/kernels/split_op.cc
@@ -167,11 +167,11 @@ class SplitOpCPU : public SplitOpBase<CPUDevice, T> {
     const auto num_threads =
         context->device()->tensorflow_cpu_worker_threads()->num_threads;
     // TODO(jewillco): Tune heuristic further.
+    const auto input_element_count = input_shape.num_elements();
     const bool use_parallelism_between_outputs =
         (num_split >= 4 &&
-         input_shape.num_elements() >=
-             std::max(num_threads, num_split) * 4096 &&
-         input_shape.num_elements() < num_split * 180 * 1024);
+         input_element_count >= std::max(num_threads, num_split) * 4096 &&
+         input_element_count < num_split * 180 * 1024);
 
     auto range_output_func = [&indices, context, &output_shape, prefix_dim_size,
                               split_dim_output_size, suffix_dim_size, &sizes,
@@ -209,7 +209,7 @@ class SplitOpCPU : public SplitOpBase<CPUDevice, T> {
       // Run in parallel, disabling parallelism in functor.
       Shard(num_split,
             context->device()->tensorflow_cpu_worker_threads()->workers,
-            num_split, kint64max, range_output_func);
+            num_split, input_element_count / num_split, range_output_func);
     } else {
       // Run sequentially, but allow internal parallelism in functor.
       range_output_func(0, num_split);
diff --git a/tensorflow/core/kernels/split_v_op.cc b/tensorflow/core/kernels/split_v_op.cc
index e2dd66da1e..3316e5fcc9 100644
--- a/tensorflow/core/kernels/split_v_op.cc
+++ b/tensorflow/core/kernels/split_v_op.cc
@@ -225,11 +225,11 @@ class SplitVOpCPU : public SplitVOpBase<CPUDevice, T, Tlen> {
     const auto num_threads =
         context->device()->tensorflow_cpu_worker_threads()->num_threads;
     // TODO(jewillco): Tune heuristic further.
+    const auto input_element_count = input_shape.num_elements();
     const bool use_parallelism_between_outputs =
         (num_split >= 4 &&
-         input_shape.num_elements() >=
-             std::max(num_threads, num_split) * 4096 &&
-         input_shape.num_elements() < num_split * 180 * 1024);
+         input_element_count >= std::max(num_threads, num_split) * 4096 &&
+         input_element_count < num_split * 180 * 1024);
 
     auto range_output_func = [&indices, context, &input_shape, prefix_dim_size,
                               split_dim, &split_sizes_vec, &split_start_points,
@@ -267,7 +267,7 @@ class SplitVOpCPU : public SplitVOpBase<CPUDevice, T, Tlen> {
       // Run in parallel, disabling parallelism in functor.
       Shard(num_split,
             context->device()->tensorflow_cpu_worker_threads()->workers,
-            num_split, kint64max, range_output_func);
+            num_split, input_element_count / num_split, range_output_func);
     } else {
       // Run sequentially, but allow internal parallelism in functor.
       range_output_func(0, num_split);
diff --git a/tensorflow/core/lib/io/zlib_buffers_test.cc b/tensorflow/core/lib/io/zlib_buffers_test.cc
index 66ee68a916..156c712db8 100644
--- a/tensorflow/core/lib/io/zlib_buffers_test.cc
+++ b/tensorflow/core/lib/io/zlib_buffers_test.cc
@@ -68,25 +68,25 @@ void TestAllCombinations(CompressionOptions input_options,
     for (auto input_buf_size : InputBufferSizes()) {
       for (auto output_buf_size : OutputBufferSizes()) {
         std::unique_ptr<WritableFile> file_writer;
-        TF_CHECK_OK(env->NewWritableFile(fname, &file_writer));
+        TF_ASSERT_OK(env->NewWritableFile(fname, &file_writer));
         string result;
 
         ZlibOutputBuffer out(file_writer.get(), input_buf_size, output_buf_size,
                              output_options);
-        TF_CHECK_OK(out.Init());
+        TF_ASSERT_OK(out.Init());
 
-        TF_CHECK_OK(out.Append(StringPiece(data)));
-        TF_CHECK_OK(out.Close());
-        TF_CHECK_OK(file_writer->Flush());
-        TF_CHECK_OK(file_writer->Close());
+        TF_ASSERT_OK(out.Append(StringPiece(data)));
+        TF_ASSERT_OK(out.Close());
+        TF_ASSERT_OK(file_writer->Flush());
+        TF_ASSERT_OK(file_writer->Close());
 
         std::unique_ptr<RandomAccessFile> file_reader;
-        TF_CHECK_OK(env->NewRandomAccessFile(fname, &file_reader));
+        TF_ASSERT_OK(env->NewRandomAccessFile(fname, &file_reader));
         std::unique_ptr<RandomAccessInputStream> input_stream(
             new RandomAccessInputStream(file_reader.get()));
         ZlibInputStream in(input_stream.get(), input_buf_size, output_buf_size,
                            input_options);
-        TF_EXPECT_OK(in.ReadNBytes(data.size(), &result));
+        TF_ASSERT_OK(in.ReadNBytes(data.size(), &result));
         EXPECT_EQ(result, data);
       }
     }
@@ -118,24 +118,24 @@ void TestMultipleWrites(uint8 input_buf_size, uint8 output_buf_size,
   string actual_result;
   string expected_result;
 
-  TF_CHECK_OK(env->NewWritableFile(fname, &file_writer));
+  TF_ASSERT_OK(env->NewWritableFile(fname, &file_writer));
   ZlibOutputBuffer out(file_writer.get(), input_buf_size, output_buf_size,
                        output_options);
-  TF_CHECK_OK(out.Init());
+  TF_ASSERT_OK(out.Init());
 
   for (int i = 0; i < num_writes; i++) {
-    TF_CHECK_OK(out.Append(StringPiece(data)));
+    TF_ASSERT_OK(out.Append(StringPiece(data)));
     if (with_flush) {
-      TF_CHECK_OK(out.Flush());
+      TF_ASSERT_OK(out.Flush());
     }
     strings::StrAppend(&expected_result, data);
   }
-  TF_CHECK_OK(out.Close());
-  TF_CHECK_OK(file_writer->Flush());
-  TF_CHECK_OK(file_writer->Close());
+  TF_ASSERT_OK(out.Close());
+  TF_ASSERT_OK(file_writer->Flush());
+  TF_ASSERT_OK(file_writer->Close());
 
   std::unique_ptr<RandomAccessFile> file_reader;
-  TF_CHECK_OK(env->NewRandomAccessFile(fname, &file_reader));
+  TF_ASSERT_OK(env->NewRandomAccessFile(fname, &file_reader));
   std::unique_ptr<RandomAccessInputStream> input_stream(
       new RandomAccessInputStream(file_reader.get()));
   ZlibInputStream in(input_stream.get(), input_buf_size, output_buf_size,
@@ -143,7 +143,7 @@ void TestMultipleWrites(uint8 input_buf_size, uint8 output_buf_size,
 
   for (int i = 0; i < num_writes; i++) {
     string decompressed_output;
-    TF_EXPECT_OK(in.ReadNBytes(data.size(), &decompressed_output));
+    TF_ASSERT_OK(in.ReadNBytes(data.size(), &decompressed_output));
     strings::StrAppend(&actual_result, decompressed_output);
   }
 
@@ -170,19 +170,19 @@ TEST(ZlibInputStream, FailsToReadIfWindowBitsAreIncompatible) {
 
   string data = GenTestString(10);
   std::unique_ptr<WritableFile> file_writer;
-  TF_CHECK_OK(env->NewWritableFile(fname, &file_writer));
+  TF_ASSERT_OK(env->NewWritableFile(fname, &file_writer));
   string result;
   ZlibOutputBuffer out(file_writer.get(), input_buf_size, output_buf_size,
                        output_options);
-  TF_CHECK_OK(out.Init());
+  TF_ASSERT_OK(out.Init());
 
-  TF_CHECK_OK(out.Append(StringPiece(data)));
-  TF_CHECK_OK(out.Close());
-  TF_CHECK_OK(file_writer->Flush());
-  TF_CHECK_OK(file_writer->Close());
+  TF_ASSERT_OK(out.Append(StringPiece(data)));
+  TF_ASSERT_OK(out.Close());
+  TF_ASSERT_OK(file_writer->Flush());
+  TF_ASSERT_OK(file_writer->Close());
 
   std::unique_ptr<RandomAccessFile> file_reader;
-  TF_CHECK_OK(env->NewRandomAccessFile(fname, &file_reader));
+  TF_ASSERT_OK(env->NewRandomAccessFile(fname, &file_reader));
   std::unique_ptr<RandomAccessInputStream> input_stream(
       new RandomAccessInputStream(file_reader.get()));
   ZlibInputStream in(input_stream.get(), input_buf_size, output_buf_size,
@@ -192,5 +192,129 @@ TEST(ZlibInputStream, FailsToReadIfWindowBitsAreIncompatible) {
   CHECK(read_status.error_message().find("inflate() failed") != string::npos);
 }
 
+void WriteCompressedFile(Env* env, const string& fname, int input_buf_size,
+                         int output_buf_size,
+                         const CompressionOptions& output_options,
+                         const string& data) {
+  std::unique_ptr<WritableFile> file_writer;
+  TF_ASSERT_OK(env->NewWritableFile(fname, &file_writer));
+
+  ZlibOutputBuffer out(file_writer.get(), input_buf_size, output_buf_size,
+                       output_options);
+  TF_ASSERT_OK(out.Init());
+
+  TF_ASSERT_OK(out.Append(StringPiece(data)));
+  TF_ASSERT_OK(out.Close());
+  TF_ASSERT_OK(file_writer->Flush());
+  TF_ASSERT_OK(file_writer->Close());
+}
+
+void TestTell(CompressionOptions input_options,
+              CompressionOptions output_options) {
+  Env* env = Env::Default();
+  string fname = testing::TmpDir() + "/zlib_buffers_test";
+  for (auto file_size : NumCopies()) {
+    string data = GenTestString(file_size);
+    for (auto input_buf_size : InputBufferSizes()) {
+      for (auto output_buf_size : OutputBufferSizes()) {
+        // Write the compressed file.
+        WriteCompressedFile(env, fname, input_buf_size, output_buf_size,
+                            output_options, data);
+
+        // Boiler-plate to set up ZlibInputStream.
+        std::unique_ptr<RandomAccessFile> file_reader;
+        TF_ASSERT_OK(env->NewRandomAccessFile(fname, &file_reader));
+        std::unique_ptr<RandomAccessInputStream> input_stream(
+            new RandomAccessInputStream(file_reader.get()));
+        ZlibInputStream in(input_stream.get(), input_buf_size, output_buf_size,
+                           input_options);
+
+        string first_half(data, 0, data.size() / 2);
+        string bytes_read;
+
+        // Read the first half of the uncompressed file and expect that Tell()
+        // returns half the uncompressed length of the file.
+        TF_ASSERT_OK(in.ReadNBytes(first_half.size(), &bytes_read));
+        EXPECT_EQ(in.Tell(), first_half.size());
+        EXPECT_EQ(bytes_read, first_half);
+
+        // Read the remaining half of the uncompressed file and expect that
+        // Tell() points past the end of file.
+        string second_half;
+        TF_ASSERT_OK(
+            in.ReadNBytes(data.size() - first_half.size(), &second_half));
+        EXPECT_EQ(in.Tell(), data.size());
+        bytes_read.append(second_half);
+
+        // Expect that the file is correctly read.
+        EXPECT_EQ(bytes_read, data);
+      }
+    }
+  }
+}
+
+void TestSkipNBytes(CompressionOptions input_options,
+                    CompressionOptions output_options) {
+  Env* env = Env::Default();
+  string fname = testing::TmpDir() + "/zlib_buffers_test";
+  for (auto file_size : NumCopies()) {
+    string data = GenTestString(file_size);
+    for (auto input_buf_size : InputBufferSizes()) {
+      for (auto output_buf_size : OutputBufferSizes()) {
+        // Write the compressed file.
+        WriteCompressedFile(env, fname, input_buf_size, output_buf_size,
+                            output_options, data);
+
+        // Boiler-plate to set up ZlibInputStream.
+        std::unique_ptr<RandomAccessFile> file_reader;
+        TF_ASSERT_OK(env->NewRandomAccessFile(fname, &file_reader));
+        std::unique_ptr<RandomAccessInputStream> input_stream(
+            new RandomAccessInputStream(file_reader.get()));
+        ZlibInputStream in(input_stream.get(), input_buf_size, output_buf_size,
+                           input_options);
+
+        size_t data_half_size = data.size() / 2;
+        string second_half(data, data_half_size, data.size() - data_half_size);
+
+        // Skip past the first half of the file and expect Tell() returns
+        // correctly.
+        TF_ASSERT_OK(in.SkipNBytes(data_half_size));
+        EXPECT_EQ(in.Tell(), data_half_size);
+
+        // Expect that second half is read correctly and Tell() returns past
+        // end of file after reading complete file.
+        string bytes_read;
+        TF_ASSERT_OK(in.ReadNBytes(second_half.size(), &bytes_read));
+        EXPECT_EQ(bytes_read, second_half);
+        EXPECT_EQ(in.Tell(), data.size());
+      }
+    }
+  }
+}
+
+TEST(ZlibInputStream, TellDefaultOptions) {
+  TestTell(CompressionOptions::DEFAULT(), CompressionOptions::DEFAULT());
+}
+
+TEST(ZlibInputStream, TellRawDeflate) {
+  TestTell(CompressionOptions::RAW(), CompressionOptions::RAW());
+}
+
+TEST(ZlibInputStream, TellGzip) {
+  TestTell(CompressionOptions::GZIP(), CompressionOptions::GZIP());
+}
+
+TEST(ZlibInputStream, SkipNBytesDefaultOptions) {
+  TestSkipNBytes(CompressionOptions::DEFAULT(), CompressionOptions::DEFAULT());
+}
+
+TEST(ZlibInputStream, SkipNBytesRawDeflate) {
+  TestSkipNBytes(CompressionOptions::RAW(), CompressionOptions::RAW());
+}
+
+TEST(ZlibInputStream, SkipNBytesGzip) {
+  TestSkipNBytes(CompressionOptions::GZIP(), CompressionOptions::GZIP());
+}
+
 }  // namespace io
 }  // namespace tensorflow
diff --git a/tensorflow/core/lib/io/zlib_inputstream.cc b/tensorflow/core/lib/io/zlib_inputstream.cc
index 4999d5cc90..984fbc2810 100644
--- a/tensorflow/core/lib/io/zlib_inputstream.cc
+++ b/tensorflow/core/lib/io/zlib_inputstream.cc
@@ -32,7 +32,8 @@ ZlibInputStream::ZlibInputStream(
       z_stream_input_(new Bytef[input_buffer_capacity_]),
       z_stream_output_(new Bytef[output_buffer_capacity_]),
       zlib_options_(zlib_options),
-      z_stream_(new z_stream) {
+      z_stream_(new z_stream),
+      bytes_read_(0) {
   InitZlibBuffer();
 }
 
@@ -45,6 +46,7 @@ ZlibInputStream::~ZlibInputStream() {
 Status ZlibInputStream::Reset() {
   TF_RETURN_IF_ERROR(input_stream_->Reset());
   InitZlibBuffer();
+  bytes_read_ = 0;
   return Status::OK();
 }
 
@@ -127,6 +129,7 @@ size_t ZlibInputStream::ReadBytesFromCache(size_t bytes_to_read,
     result->append(next_unread_byte_, can_read_bytes);
     next_unread_byte_ += can_read_bytes;
   }
+  bytes_read_ += can_read_bytes;
   return can_read_bytes;
 }
 
@@ -170,8 +173,7 @@ Status ZlibInputStream::ReadNBytes(int64 bytes_to_read, string* result) {
   return Status::OK();
 }
 
-// TODO(srbs): Implement this.
-int64 ZlibInputStream::Tell() const { return -1; }
+int64 ZlibInputStream::Tell() const { return bytes_read_; }
 
 Status ZlibInputStream::Inflate() {
   int error = inflate(z_stream_.get(), zlib_options_.flush_mode);
diff --git a/tensorflow/core/lib/io/zlib_inputstream.h b/tensorflow/core/lib/io/zlib_inputstream.h
index 8faa7dcb8f..9c7e14441c 100644
--- a/tensorflow/core/lib/io/zlib_inputstream.h
+++ b/tensorflow/core/lib/io/zlib_inputstream.h
@@ -132,6 +132,9 @@ class ZlibInputStream : public InputStreamInterface {
   // Returns the size of [next_unread_byte_, z_stream_->next_out)
   size_t NumUnreadBytes() const;
 
+  // Number of *uncompressed* bytes that have been read from this stream.
+  int64 bytes_read_;
+
   TF_DISALLOW_COPY_AND_ASSIGN(ZlibInputStream);
 };
 
diff --git a/tensorflow/examples/learn/mnist.py b/tensorflow/examples/learn/mnist.py
index 5344526b52..88425ea0d0 100644
--- a/tensorflow/examples/learn/mnist.py
+++ b/tensorflow/examples/learn/mnist.py
@@ -97,6 +97,8 @@ def conv_model(features, labels, mode):
 
 
 def main(unused_args):
+  tf.logging.set_verbosity(tf.logging.INFO)
+
   ### Download and load MNIST dataset.
   mnist = tf.contrib.learn.datasets.DATASETS['mnist']('/tmp/mnist')
   train_input_fn = tf.estimator.inputs.numpy_input_fn(
@@ -115,6 +117,7 @@ def main(unused_args):
   feature_columns = [
       tf.feature_column.numeric_column(
           X_FEATURE, shape=mnist.train.images.shape[1:])]
+
   classifier = tf.estimator.LinearClassifier(
       feature_columns=feature_columns, n_classes=N_DIGITS)
   classifier.train(input_fn=train_input_fn, steps=200)
diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index e4e284dcdf..cbeb0b46cb 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -3685,7 +3685,10 @@ py_test(
     size = "medium",
     srcs = ["training/monitored_session_test.py"],
     srcs_version = "PY2AND3",
-    tags = ["no_windows"],
+    tags = [
+        "no_windows",
+        "notsan",  # b/67945581
+    ],
     deps = [
         ":array_ops",
         ":client_testlib",
diff --git a/tensorflow/python/eager/BUILD b/tensorflow/python/eager/BUILD
index 9e9a7f4c59..ef04f933c5 100644
--- a/tensorflow/python/eager/BUILD
+++ b/tensorflow/python/eager/BUILD
@@ -391,14 +391,14 @@ py_test(
     ],
 )
 
-py_test(
+cuda_py_test(
     name = "ops_test",
     srcs = ["ops_test.py"],
-    srcs_version = "PY2AND3",
-    deps = [
+    additional_deps = [
         ":context",
         ":execute",
         ":test",
+        "//third_party/py/numpy",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:control_flow_ops",
         "//tensorflow/python:dtypes",
@@ -410,7 +410,6 @@ py_test(
         "//tensorflow/python:random_ops",
         "//tensorflow/python:sparse_ops",
         "//tensorflow/python:tensor_shape",
-        "//third_party/py/numpy",
     ],
 )
 
diff --git a/tensorflow/python/eager/ops_test.py b/tensorflow/python/eager/ops_test.py
index 6d1a5fe264..f737bfbc15 100644
--- a/tensorflow/python/eager/ops_test.py
+++ b/tensorflow/python/eager/ops_test.py
@@ -49,7 +49,7 @@ class OpsTest(test_util.TensorFlowTestCase):
     three = constant_op.constant([[3.]]).as_gpu_tensor()
     five = constant_op.constant([[5.]]).as_gpu_tensor()
     product = math_ops.matmul(three, five)
-    self.assertEqual([[15.0]], product)
+    self.assertEqual([[15.0]], product.numpy())
 
   def testExecuteStringAttr(self):
     three = constant_op.constant(3.0)
@@ -97,7 +97,7 @@ class OpsTest(test_util.TensorFlowTestCase):
       self.skipTest('No GPUs found')
     with context.device('/gpu:0'):
       r = constant_op.constant(1) + constant_op.constant(2)
-    self.assertEqual(r, 3)
+    self.assertAllEqual(r, 3)
 
   def testExecuteListOutputLen1(self):
     split_dim = constant_op.constant(1)
@@ -264,7 +264,7 @@ class OpsTest(test_util.TensorFlowTestCase):
     # The Shape op kernel on GPU places the output in host memory.
     value = constant_op.constant([1.]).as_gpu_tensor()
     shape = array_ops.shape(value)
-    self.assertEquals([1], shape)
+    self.assertEqual([1], shape.numpy())
 
   def testRandomUniform(self):
     scalar_shape = constant_op.constant([], dtype=dtypes.int32)
diff --git a/tensorflow/python/estimator/estimator.py b/tensorflow/python/estimator/estimator.py
index 00a57f11dc..2a4d77b1a6 100644
--- a/tensorflow/python/estimator/estimator.py
+++ b/tensorflow/python/estimator/estimator.py
@@ -707,12 +707,11 @@ class Estimator(object):
     with ops.Graph().as_default() as g, g.device(self._device_fn):
       random_seed.set_random_seed(self._config.tf_random_seed)
       global_step_tensor = self._create_and_assert_global_step(g)
-      global_step_read_tensor = training_util._get_or_create_global_step_read()  # pylint: disable=protected-access
+      training_util._get_or_create_global_step_read()  # pylint: disable=protected-access
       features, labels = self._get_features_and_labels_from_input_fn(
           input_fn, model_fn_lib.ModeKeys.TRAIN)
-      with ops.control_dependencies([global_step_read_tensor]):
-        estimator_spec = self._call_model_fn(
-            features, labels, model_fn_lib.ModeKeys.TRAIN, self.config)
+      estimator_spec = self._call_model_fn(
+          features, labels, model_fn_lib.ModeKeys.TRAIN, self.config)
       # Check if the user created a loss summary, and add one if they didn't.
       # We assume here that the summary is called 'loss'. If it is not, we will
       # make another one with the name 'loss' to ensure it shows up in the right
diff --git a/tensorflow/python/feature_column/BUILD b/tensorflow/python/feature_column/BUILD
index 27062adb61..b1c81dd58c 100644
--- a/tensorflow/python/feature_column/BUILD
+++ b/tensorflow/python/feature_column/BUILD
@@ -86,6 +86,7 @@ py_test(
         "//tensorflow/python:framework_ops",
         "//tensorflow/python:lookup_ops",
         "//tensorflow/python:parsing_ops",
+        "//tensorflow/python:partitioned_variables",
         "//tensorflow/python:session",
         "//tensorflow/python:sparse_tensor",
         "//tensorflow/python:training",
diff --git a/tensorflow/python/feature_column/feature_column.py b/tensorflow/python/feature_column/feature_column.py
index 81f4f45fcb..190a25d4d7 100644
--- a/tensorflow/python/feature_column/feature_column.py
+++ b/tensorflow/python/feature_column/feature_column.py
@@ -197,12 +197,13 @@ def input_layer(features,
     trainable: If `True` also add the variable to the graph collection
       `GraphKeys.TRAINABLE_VARIABLES` (see `tf.Variable`).
     cols_to_vars: If not `None`, must be a dictionary that will be filled with a
-      mapping from `_FeatureColumn` to associated `Variable` (or list of
-      `Variable`, or `PartitionedVariable`.  For example, after the call, we
-      might have cols_to_vars = {_EmbeddingColumn(
+      mapping from `_FeatureColumn` to list of `Variable`s.  For example, after
+      the call, we might have cols_to_vars =
+      {_EmbeddingColumn(
         categorical_column=_HashedCategoricalColumn(
           key='sparse_feature', hash_bucket_size=5, dtype=tf.string),
-        dimension=10): [<tf.Variable 'some_variable' shape=(5, 10)]}
+        dimension=10): [<tf.Variable 'some_variable:0' shape=(5, 10),
+                        <tf.Variable 'some_variable:1' shape=(5, 10)]}
       If a column creates no variables, its value will be an empty list.
 
   Returns:
@@ -302,18 +303,18 @@ def linear_model(features,
     trainable: If `True` also add the variable to the graph collection
       `GraphKeys.TRAINABLE_VARIABLES` (see `tf.Variable`).
     cols_to_vars: If not `None`, must be a dictionary that will be filled with a
-      mapping from `_FeatureColumn` to associated `Variable` (or list of
-      `Variable`, or `PartitionedVariable`.  For example,
-      after the call, we might have cols_to_vars = {
+      mapping from `_FeatureColumn` to associated list of `Variable`s.  For
+      example, after the call, we might have cols_to_vars = {
         _NumericColumn(
           key='numeric_feature1', shape=(1,):
-        <tf.Variable 'linear_model/price2/weights:0' shape=(1, 1)>,
-        'bias': <tf.Variable 'linear_model/bias_weights:0' shape=(1,)>,
+        [<tf.Variable 'linear_model/price2/weights:0' shape=(1, 1)>],
+        'bias': [<tf.Variable 'linear_model/bias_weights:0' shape=(1,)>],
         _NumericColumn(
           key='numeric_feature2', shape=(2,)):
-        <tf.Variable 'linear_model/price1/weights:0' shape=(2, 1)>}
-      Note that it will also contain a string key 'bias'.  If a column creates
-      no variables, its value will be an empty list.
+        [<tf.Variable 'linear_model/price1/weights:0' shape=(2, 1)>]}
+      If a column creates no variables, its value will be an empty list. Note
+      that cols_to_vars will also contain a string key 'bias' that maps to a
+      list of Variables.
 
   Returns:
     A `Tensor` which represents predictions/logits of a linear model. Its shape
@@ -366,8 +367,12 @@ def linear_model(features,
     predictions = nn_ops.bias_add(
         predictions_no_bias, bias, name='weighted_sum')
     if cols_to_vars is not None:
-      # Add the bias to cols_to_vars as well.
-      cols_to_vars['bias'] = bias
+      # Add the bias to cols_to_vars as well, converting the Variable or
+      # PartitionedVariable to a list of Variable's.
+      if isinstance(bias, variables.Variable):
+        cols_to_vars['bias'] = [bias]
+      else:  # Must be a PartitionedVariable.
+        cols_to_vars['bias'] = list(bias)
     return predictions
 
 
diff --git a/tensorflow/python/feature_column/feature_column_test.py b/tensorflow/python/feature_column/feature_column_test.py
index 112600439b..e57e9a9836 100644
--- a/tensorflow/python/feature_column/feature_column_test.py
+++ b/tensorflow/python/feature_column/feature_column_test.py
@@ -41,6 +41,7 @@ from tensorflow.python.framework import sparse_tensor
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import lookup_ops
 from tensorflow.python.ops import parsing_ops
+from tensorflow.python.ops import partitioned_variables
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops import variables as variables_lib
 from tensorflow.python.platform import test
@@ -1354,10 +1355,33 @@ class LinearModelTest(test.TestCase):
       bias = get_linear_model_bias()
       price1_var = get_linear_model_column_var(price1)
       price2_var = get_linear_model_column_var(price2)
-      self.assertEqual(cols_to_vars['bias'], bias)
+      self.assertAllEqual(cols_to_vars['bias'], [bias])
       self.assertAllEqual(cols_to_vars[price1], [price1_var])
       self.assertAllEqual(cols_to_vars[price2], [price2_var])
 
+  def test_fills_cols_to_vars_partitioned_variables(self):
+    price1 = fc.numeric_column('price1', shape=2)
+    price2 = fc.numeric_column('price2', shape=3)
+    with ops.Graph().as_default():
+      features = {
+          'price1': [[1., 2.], [6., 7.]],
+          'price2': [[3., 4., 5.], [8., 9., 10.]]
+      }
+      cols_to_vars = {}
+      with variable_scope.variable_scope(
+          'linear',
+          partitioner=partitioned_variables.fixed_size_partitioner(2, axis=0)):
+        fc.linear_model(features, [price1, price2], cols_to_vars=cols_to_vars)
+      with _initialized_session():
+        self.assertEqual([0.], cols_to_vars['bias'][0].eval())
+        # Partitioning shards the [2, 1] price1 var into 2 [1, 1] Variables.
+        self.assertAllEqual([[0.]], cols_to_vars[price1][0].eval())
+        self.assertAllEqual([[0.]], cols_to_vars[price1][1].eval())
+        # Partitioning shards the [3, 1] price2 var into a [2, 1] Variable and
+        # a [1, 1] Variable.
+        self.assertAllEqual([[0.], [0.]], cols_to_vars[price2][0].eval())
+        self.assertAllEqual([[0.]], cols_to_vars[price2][1].eval())
+
   def test_dense_collection(self):
     price = fc.numeric_column('price')
     with ops.Graph().as_default() as g:
@@ -1761,9 +1785,38 @@ class InputLayerTest(test.TestCase):
       self.assertEqual(0, len(cols_to_vars[price1]))
       self.assertEqual(0, len(cols_to_vars[dense_feature_bucketized]))
       self.assertEqual(1, len(cols_to_vars[some_embedding_column]))
-      for var in cols_to_vars[some_embedding_column]:
-        self.assertIsInstance(var, variables_lib.Variable)
-        self.assertAllEqual(var.shape, [5, 10])
+      self.assertIsInstance(cols_to_vars[some_embedding_column][0],
+                            variables_lib.Variable)
+      self.assertAllEqual(cols_to_vars[some_embedding_column][0].shape, [5, 10])
+
+  def test_fills_cols_to_vars_partitioned_variables(self):
+    price1 = fc.numeric_column('price1')
+    dense_feature = fc.numeric_column('dense_feature')
+    dense_feature_bucketized = fc.bucketized_column(
+        dense_feature, boundaries=[0.])
+    some_sparse_column = fc.categorical_column_with_hash_bucket(
+        'sparse_feature', hash_bucket_size=5)
+    some_embedding_column = fc.embedding_column(
+        some_sparse_column, dimension=10)
+    with ops.Graph().as_default():
+      features = {
+          'price1': [[3.], [4.]],
+          'dense_feature': [[-1.], [4.]],
+          'sparse_feature': [['a'], ['x']],
+      }
+      cols_to_vars = {}
+      all_cols = [price1, dense_feature_bucketized, some_embedding_column]
+      with variable_scope.variable_scope(
+          'input_from_feature_columns',
+          partitioner=partitioned_variables.fixed_size_partitioner(3, axis=0)):
+        fc.input_layer(features, all_cols, cols_to_vars=cols_to_vars)
+      self.assertItemsEqual(list(cols_to_vars.keys()), all_cols)
+      self.assertEqual(0, len(cols_to_vars[price1]))
+      self.assertEqual(0, len(cols_to_vars[dense_feature_bucketized]))
+      self.assertEqual(3, len(cols_to_vars[some_embedding_column]))
+      self.assertAllEqual(cols_to_vars[some_embedding_column][0].shape, [2, 10])
+      self.assertAllEqual(cols_to_vars[some_embedding_column][1].shape, [2, 10])
+      self.assertAllEqual(cols_to_vars[some_embedding_column][2].shape, [1, 10])
 
   def test_column_order(self):
     price_a = fc.numeric_column('price_a')
diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index 3ac8a0cb6a..85b875aa3a 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -33,6 +33,7 @@ from tensorflow.core.framework import attr_value_pb2
 from tensorflow.core.framework import function_pb2
 from tensorflow.core.framework import graph_pb2
 from tensorflow.core.framework import node_def_pb2
+from tensorflow.core.framework import op_def_pb2
 from tensorflow.core.framework import versions_pb2
 from tensorflow.python import pywrap_tensorflow as c_api
 from tensorflow.python.eager import context
@@ -1985,7 +1986,19 @@ class Operation(object):
       protocol buffer.
     """
     # pylint: enable=line-too-long
-    return self._op_def
+    if self._c_op:
+      with errors.raise_exception_on_not_ok_status() as status:
+        with c_api_util.tf_buffer() as buf:
+          # pylint: disable=protected-access
+          c_api.TF_GraphGetOpDef(self._graph._c_graph,
+                                 compat.as_bytes(self.type), buf, status)
+          # pylint: enable=protected-access
+          data = c_api.TF_GetBuffer(buf)
+      op_def = op_def_pb2.OpDef()
+      op_def.ParseFromString(compat.as_bytes(data))
+      return op_def
+    else:
+      return self._op_def
 
   @property
   def traceback(self):
@@ -2504,7 +2517,14 @@ class Graph(object):
     # A map from tensor handle to its delete op.
     self._handle_deleters = {}
     # Resource container.
-    self._container = ""
+    if context.in_graph_mode():
+      self._container_prefix = ""
+    else:
+      # In Eager mode, isolate resources (particularly ResourceVariables) in
+      # Graphs by default. This prevents unintended variable sharing. Graph mode
+      # gets this kind of isolation from Sessions.
+      self._container_prefix = "eager-execution-%d/" % (uid(),)
+    self._container = self._container_prefix
     self._registered_ops = op_def_registry.get_registered_ops()
 
     # TODO(skyewm): fold as much of the above as possible into the C
@@ -3816,7 +3836,7 @@ class Graph(object):
     """
     original_container = self._container
     try:
-      self._container = container_name
+      self._container = self._container_prefix + container_name
       yield self._container
     finally:
       self._container = original_container
diff --git a/tensorflow/python/framework/ops_test.py b/tensorflow/python/framework/ops_test.py
index f20c808cde..59c0288457 100644
--- a/tensorflow/python/framework/ops_test.py
+++ b/tensorflow/python/framework/ops_test.py
@@ -504,6 +504,21 @@ class OperationTest(test_util.TensorFlowTestCase):
                                  r"num of inputs: 0\) does not have input 1"):
       x.op._update_input(1, x)  # pylint: disable=protected-access
 
+  def testOpDef(self):
+    x = constant_op.constant(0)
+    y = constant_op.constant(1)
+    z = x + y
+
+    # Pure Python mode doesn't create OpDefs for constants
+    if ops._USE_C_API:
+      self.assertEqual(x.op.op_def.name, "Const")
+      self.assertEqual(len(x.op.op_def.input_arg), 0)
+      self.assertEqual(len(x.op.op_def.output_arg), 1)
+
+    self.assertEqual(z.op.op_def.name, "Add")
+    self.assertEqual(len(z.op.op_def.input_arg), 2)
+    self.assertEqual(len(z.op.op_def.output_arg), 1)
+
 
 @test_util.with_c_api
 class CreateOpTest(test_util.TensorFlowTestCase):
diff --git a/tensorflow/python/kernel_tests/random_shuffle_queue_test.py b/tensorflow/python/kernel_tests/random_shuffle_queue_test.py
index 1b84af6823..c4e16ff628 100644
--- a/tensorflow/python/kernel_tests/random_shuffle_queue_test.py
+++ b/tensorflow/python/kernel_tests/random_shuffle_queue_test.py
@@ -654,7 +654,8 @@ class RandomShuffleQueueTest(test.TestCase):
 
   def testBlockingDequeueFromClosedQueue(self):
     with self.test_session() as sess:
-      q = data_flow_ops.RandomShuffleQueue(10, 2, dtypes_lib.float32)
+      min_size = 2
+      q = data_flow_ops.RandomShuffleQueue(10, min_size, dtypes_lib.float32)
       elems = [10.0, 20.0, 30.0, 40.0]
       enqueue_op = q.enqueue_many((elems,))
       close_op = q.close()
@@ -664,20 +665,24 @@ class RandomShuffleQueueTest(test.TestCase):
 
       results = []
 
-      def dequeue():
-        for _ in elems:
-          results.append(sess.run(dequeued_t))
+      # Manually dequeue until we hit min_size.
+      results.append(sess.run(dequeued_t))
+      results.append(sess.run(dequeued_t))
+
+      def blocking_dequeue():
+        results.append(sess.run(dequeued_t))
+        results.append(sess.run(dequeued_t))
+
         self.assertItemsEqual(elems, results)
         # Expect the operation to fail due to the queue being closed.
         with self.assertRaisesRegexp(errors_impl.OutOfRangeError,
                                      "is closed and has insufficient"):
           sess.run(dequeued_t)
 
-      dequeue_thread = self.checkedThread(target=dequeue)
+      dequeue_thread = self.checkedThread(target=blocking_dequeue)
       dequeue_thread.start()
-      # The close_op should run after the dequeue_thread has blocked.
-      # TODO(mrry): Figure out how to do this without sleeping.
       time.sleep(0.1)
+
       # The dequeue thread blocked when it hit the min_size requirement.
       self.assertEqual(len(results), 2)
       close_op.run()
diff --git a/tensorflow/python/kernel_tests/resource_variable_ops_test.py b/tensorflow/python/kernel_tests/resource_variable_ops_test.py
index ec9192b1a0..23676223dc 100644
--- a/tensorflow/python/kernel_tests/resource_variable_ops_test.py
+++ b/tensorflow/python/kernel_tests/resource_variable_ops_test.py
@@ -428,6 +428,16 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase):
       self.assertEqual(1, v1.read_value().numpy())
       self.assertEqual(2, v2.read_value().numpy())
 
+  def testDestruction(self):
+    with context.eager_mode():
+      var = resource_variable_ops.ResourceVariable(initial_value=1.0,
+                                                   name="var8")
+      var.__del__()
+      with self.assertRaisesRegexp(errors.NotFoundError,
+                                   r"Resource .*\/var8\/.* does not exist."):
+        resource_variable_ops.destroy_resource_op(var._handle,
+                                                  ignore_lookup_error=False)
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/ops/resource_variable_ops.py b/tensorflow/python/ops/resource_variable_ops.py
index 2c9a3ff19a..dd3f167145 100644
--- a/tensorflow/python/ops/resource_variable_ops.py
+++ b/tensorflow/python/ops/resource_variable_ops.py
@@ -427,6 +427,39 @@ class ResourceVariable(variables.Variable):
     self._constraint = None
   # LINT.ThenChange(//tensorflow/python/eager/graph_callable.py)
 
+  def __del__(self):
+    if not self._in_graph_mode:
+      # There is only one ResourceVariable object for each underlying resource
+      # (cached in the Graph's VariableStore when created with get_variable), so
+      # it is safe to delete the resource we have a handle to. Each Graph has a
+      # unique container name in Eager, which prevents resource sharing.
+      #
+      # The Graph's VariableStore contains strong references to ResourceVariable
+      # objects created with get_variable, so this destructor will only be
+      # callled once the Graph is garbage collected for those objects. However,
+      # explicitly created ResourceVariables (e.g. through tfe.Variable) may be
+      # collected earlier.
+      try:
+        # We have checked that this ResourceVariable was created in Eager
+        # mode. However, this destructor may be running in graph mode
+        # (especially during unit tests). To clean up successfully, we switch
+        # back into Eager temporarily.
+        with context.eager_mode():
+          with ops.device(self._handle_device):
+            gen_resource_variable_ops.destroy_resource_op(
+                self._handle, ignore_lookup_error=True)
+      except TypeError:
+        # Suppress some exceptions, mainly for the case when we're running on
+        # module deletion. Things that can go wrong include the context module
+        # already being unloaded, self._handle._handle_data no longer being
+        # valid, and so on. Printing warnings in these cases is silly
+        # (exceptions raised from __del__ are printed as warnings to stderr).
+        pass  # 'NoneType' object is not callable when the handle has been
+              # partially unloaded.
+      except AttributeError:
+        pass  # 'NoneType' object has no attribute 'eager_mode' when context has
+              # been unloaded. Will catch other module unloads as well.
+
   @property
   def dtype(self):
     """The dtype of this variable."""
diff --git a/tensorflow/python/training/adam_test.py b/tensorflow/python/training/adam_test.py
index defcf33714..176d20bd60 100644
--- a/tensorflow/python/training/adam_test.py
+++ b/tensorflow/python/training/adam_test.py
@@ -152,53 +152,54 @@ class AdamOptimizerTest(test.TestCase):
 
   def doTestBasic(self, use_resource=False):
     for i, dtype in enumerate([dtypes.half, dtypes.float32, dtypes.float64]):
-      # Initialize variables for numpy implementation.
-      m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0
-      var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype)
-      grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype)
-      var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype)
-      grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype)
-
-      if use_resource:
-        var0 = resource_variable_ops.ResourceVariable(
-            var0_np, name="var0_%d" % i)
-        var1 = resource_variable_ops.ResourceVariable(
-            var1_np, name="var1_%d" % i)
-      else:
-        var0 = variables.Variable(var0_np)
-        var1 = variables.Variable(var1_np)
-      grads0 = constant_op.constant(grads0_np)
-      grads1 = constant_op.constant(grads1_np)
-
-      opt = adam.AdamOptimizer()
-      update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
+      with self.test_session(graph=ops.Graph()):
+        # Initialize variables for numpy implementation.
+        m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0
+        var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype)
+        grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype)
+        var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype)
+        grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype)
 
-      if context.in_graph_mode():
-        self.evaluate(variables.global_variables_initializer())
-        # Fetch params to validate initial values
-        self.assertAllClose([1.0, 2.0], self.evaluate(var0))
-        self.assertAllClose([3.0, 4.0], self.evaluate(var1))
+        if use_resource:
+          var0 = resource_variable_ops.ResourceVariable(
+              var0_np, name="var0_%d" % i)
+          var1 = resource_variable_ops.ResourceVariable(
+              var1_np, name="var1_%d" % i)
+        else:
+          var0 = variables.Variable(var0_np)
+          var1 = variables.Variable(var1_np)
+        grads0 = constant_op.constant(grads0_np)
+        grads1 = constant_op.constant(grads1_np)
 
-      beta1_power, beta2_power = opt._get_beta_accumulators()
+        opt = adam.AdamOptimizer()
+        update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
 
-      # Run 3 steps of Adam
-      for t in range(1, 4):
         if context.in_graph_mode():
-          self.evaluate(update)
-        elif t > 1:
-          opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
+          self.evaluate(variables.global_variables_initializer())
+          # Fetch params to validate initial values
+          self.assertAllClose([1.0, 2.0], self.evaluate(var0))
+          self.assertAllClose([3.0, 4.0], self.evaluate(var1))
 
-        self.assertAllCloseAccordingToType(0.9**(t + 1),
-                                           self.evaluate(beta1_power))
-        self.assertAllCloseAccordingToType(0.999**(t + 1),
-                                           self.evaluate(beta2_power))
+        beta1_power, beta2_power = opt._get_beta_accumulators()
 
-        var0_np, m0, v0 = adam_update_numpy(var0_np, grads0_np, t, m0, v0)
-        var1_np, m1, v1 = adam_update_numpy(var1_np, grads1_np, t, m1, v1)
+        # Run 3 steps of Adam
+        for t in range(1, 4):
+          if context.in_graph_mode():
+            self.evaluate(update)
+          elif t > 1:
+            opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
+
+          self.assertAllCloseAccordingToType(0.9**(t + 1),
+                                             self.evaluate(beta1_power))
+          self.assertAllCloseAccordingToType(0.999**(t + 1),
+                                             self.evaluate(beta2_power))
+
+          var0_np, m0, v0 = adam_update_numpy(var0_np, grads0_np, t, m0, v0)
+          var1_np, m1, v1 = adam_update_numpy(var1_np, grads1_np, t, m1, v1)
 
-        # Validate updated params
-        self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0))
-        self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1))
+          # Validate updated params
+          self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0))
+          self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1))
 
   def testBasic(self):
     with self.test_session():
diff --git a/tensorflow/python/training/saver_test.py b/tensorflow/python/training/saver_test.py
index 07cd67a4b9..aeb8eaffe8 100644
--- a/tensorflow/python/training/saver_test.py
+++ b/tensorflow/python/training/saver_test.py
@@ -479,14 +479,14 @@ class SaverTest(test.TestCase):
       self.assertEqual(30.0, v2_2.values().eval())
 
   def _SaveAndLoad(self, var_name, var_value, other_value, save_path):
-    with self.test_session() as sess:
+    with self.test_session(graph=ops_lib.Graph()) as sess:
       var = resource_variable_ops.ResourceVariable(var_value, name=var_name)
       save = saver_module.Saver({var_name: var})
       if context.in_graph_mode():
         self.evaluate(var.initializer)
       val = save.save(sess, save_path)
       self.assertEqual(save_path, val)
-    with self.test_session() as sess:
+    with self.test_session(graph=ops_lib.Graph()) as sess:
       var = resource_variable_ops.ResourceVariable(other_value, name=var_name)
       save = saver_module.Saver({var_name: var})
       save.restore(sess, save_path)
@@ -619,27 +619,28 @@ class SaverTest(test.TestCase):
     # Save and reload one Variable named "var0".
     self._SaveAndLoad("var0", 0.0, 1.0, save_path)
     for use_tensor in [True, False]:
-      var = resource_variable_ops.ResourceVariable(1.0, name="var0")
-      save = saver_module.Saver(
-          {
-              var._shared_name: var
-          }, pad_step_number=pad_step_number)
-      if context.in_graph_mode():
-        self.evaluate(var.initializer)
-        sess = ops_lib.get_default_session()
-      else:
-        sess = None
-      if use_tensor:
-        global_step = constant_op.constant(global_step_int)
-        val = save.save(sess, save_path, global_step=global_step)
-      else:
-        val = save.save(sess, save_path, global_step=global_step_int)
-      if pad_step_number:
-        expected_save_path = "%s-%s" % (save_path,
-                                        "{:08d}".format(global_step_int))
-      else:
-        expected_save_path = "%s-%d" % (save_path, global_step_int)
-      self.assertEqual(expected_save_path, val)
+      with self.test_session(graph=ops_lib.Graph()):
+        var = resource_variable_ops.ResourceVariable(1.0, name="var0")
+        save = saver_module.Saver(
+            {
+                var._shared_name: var
+            }, pad_step_number=pad_step_number)
+        if context.in_graph_mode():
+          self.evaluate(var.initializer)
+          sess = ops_lib.get_default_session()
+        else:
+          sess = None
+        if use_tensor:
+          global_step = constant_op.constant(global_step_int)
+          val = save.save(sess, save_path, global_step=global_step)
+        else:
+          val = save.save(sess, save_path, global_step=global_step_int)
+        if pad_step_number:
+          expected_save_path = "%s-%s" % (save_path,
+                                          "{:08d}".format(global_step_int))
+        else:
+          expected_save_path = "%s-%d" % (save_path, global_step_int)
+        self.assertEqual(expected_save_path, val)
 
   def testSaveWithGlobalStepWithPadding(self):
     self.testSaveWithGlobalStep(pad_step_number=True)
diff --git a/tensorflow/tools/graph_transforms/fold_constants_lib.cc b/tensorflow/tools/graph_transforms/fold_constants_lib.cc
index 30290c7a16..f2934a79bd 100644
--- a/tensorflow/tools/graph_transforms/fold_constants_lib.cc
+++ b/tensorflow/tools/graph_transforms/fold_constants_lib.cc
@@ -17,12 +17,20 @@ limitations under the License.
 
 #include <algorithm>
 #include <iterator>
+#include <map>
+#include <string>
+#include <unordered_map>
+#include <unordered_set>
+#include <utility>
+#include <vector>
 
 #include "tensorflow/core/common_runtime/constant_folding.h"
 #include "tensorflow/core/common_runtime/shape_refiner.h"
 #include "tensorflow/core/graph/graph_constructor.h"
 #include "tensorflow/core/graph/node_builder.h"
 #include "tensorflow/core/graph/subgraph.h"
+#include "tensorflow/core/lib/core/stringpiece.h"
+#include "tensorflow/core/lib/strings/numbers.h"
 #include "tensorflow/core/platform/init_main.h"
 #include "tensorflow/core/public/session.h"
 #include "tensorflow/core/util/command_line_flags.h"
@@ -30,33 +38,38 @@ limitations under the License.
 
 namespace tensorflow {
 namespace graph_transforms {
+namespace {
+using StringPieceSet = std::unordered_set<StringPiece, StringPiece::Hasher>;
+template <typename T>
+using StringPieceMap = std::unordered_map<StringPiece, T, StringPiece::Hasher>;
+}  // namespace
 
 Status ReplaceSendRecvs(const GraphDef& original_graph_def,
                         const GraphDef& rewritten_graph_def,
                         const std::vector<string>& inputs,
                         const std::vector<string>& outputs,
                         GraphDef* output_graph_def) {
-  std::map<string, const NodeDef*> original_map;
-  MapNamesToNodes(original_graph_def, &original_map);
-  std::map<string, string> new_node_names;
-  for (const NodeDef& node : rewritten_graph_def.node()) {
-    // If the op isn't a Recv, or it was in the original, nothing to do.
-    if ((node.op() != "_Recv") || (original_map.count(node.name()) == 1)) {
-      continue;
-    }
-    // See if it matches an input from the original.
-    for (const string& input : inputs) {
-      // Here we rely on the naming convention for the Recv nodes that
-      // RewriteGraphForExecution adds in the place of the feed inputs.
-      string input_prefix = "_recv_" + input + "_";
-      if (StringPiece(node.name()).starts_with(input_prefix)) {
-        // If it does, prepare to rename any inputs that refer to it.
-        new_node_names[node.name()] = input;
-      }
-    }
+  // recv_node_names serves as a string storage for recv node names.
+  std::vector<string> recv_node_names(inputs.size());
+  StringPieceMap<TensorId> recv_node_map;
+  StringPieceSet input_nodes;
+  for (int i = 0; i < inputs.size(); ++i) {
+    // RewriteGraphForExecution adds a recv node for each input edge. We assume
+    // here that adding such recv node did not fail. For example, the original
+    // graph did not already have a node with the name for the new added recv
+    // node.
+    TensorId id = ParseTensorName(inputs[i]);
+    input_nodes.insert(id.first);
+    string& recv_node_name = recv_node_names[i];
+    recv_node_name = strings::StrCat("_recv_", id.first, "_", id.second);
+    recv_node_map.emplace(recv_node_name, id);
+  }
+
+  StringPieceMap<const NodeDef*> original_map;
+  for (const NodeDef& node : original_graph_def.node()) {
+    original_map.emplace(node.name(), &node);
   }
 
-  std::vector<NodeDef> nodes_to_add;
   for (const NodeDef& node : rewritten_graph_def.node()) {
     if ((node.op() == "_Send") || (node.op() == "_Recv")) {
       // If the op is a Send or Recv that wasn't in the original, skip it.
@@ -64,55 +77,68 @@ Status ReplaceSendRecvs(const GraphDef& original_graph_def,
         continue;
       }
     }
-    NodeDef new_node;
-    new_node = node;
-    new_node.mutable_input()->Clear();
-    for (const string& old_input : node.input()) {
-      string input_prefix;
-      string input_node_name;
-      string input_suffix;
-      NodeNamePartsFromInput(old_input, &input_prefix, &input_node_name,
-                             &input_suffix);
-      string new_input;
-      if (new_node_names.count(input_node_name) > 0) {
-        new_input =
-            input_prefix + new_node_names[input_node_name] + input_suffix;
-      } else {
-        new_input = old_input;
+
+    NodeDef* new_node = output_graph_def->add_node();
+    new_node->MergeFrom(node);
+    for (int i = 0; i < new_node->input_size(); ++i) {
+      string& input = *new_node->mutable_input(i);
+      TensorId id = ParseTensorName(input);
+      const auto iter = recv_node_map.find(id.first);
+      if (iter != recv_node_map.end()) {
+        // The node being substituted is a Recv node, and it has only one
+        // output. If this input is not a control input, then replace the input
+        // with the mapped value. Otherwise, replace the node name only.
+        if (id.second != Graph::kControlSlot) {
+          CHECK_EQ(id.second, 0);
+          input = iter->second.ToString();
+        } else {
+          id.first = iter->second.first;
+          input = id.ToString();
+        }
       }
-      *(new_node.mutable_input()->Add()) = new_input;
     }
-    nodes_to_add.push_back(new_node);
-  }
-  for (std::pair<string, string> entry : new_node_names) {
-    string removed_node_name = entry.second;
-    const NodeDef* removed_node = original_map[removed_node_name];
-    NodeDef new_node;
-    new_node = *removed_node;
-    nodes_to_add.push_back(new_node);
+
+    // RewriteGraphForExecution() did not remove this input node. Remove this
+    // node name from input_nodes so that a duplicate does not get added to the
+    // output_graph_def.
+    auto iter = input_nodes.find(new_node->name());
+    if (iter != input_nodes.end()) {
+      input_nodes.erase(iter);
+    }
   }
 
-  for (const NodeDef& node : nodes_to_add) {
-    *output_graph_def->mutable_node()->Add() = node;
+  // Some input nodes are removed in rewrite_graph_def. Add those nodes to
+  // output_graph_def.
+  for (StringPiece name : input_nodes) {
+    const NodeDef& removed_node = *CHECK_NOTNULL(original_map[name]);
+    output_graph_def->add_node()->MergeFrom(removed_node);
   }
+
   return Status::OK();
 }
 
 Status RemoveUnusedNodes(const GraphDef& input_graph_def,
                          const TransformFuncContext& context,
                          GraphDef* output_graph_def) {
-  std::map<string, const NodeDef*> node_map;
-  MapNamesToNodes(input_graph_def, &node_map);
+  StringPieceMap<const NodeDef*> node_map;
+  for (const NodeDef& node : input_graph_def.node()) {
+    node_map.emplace(node.name(), &node);
+  }
 
-  std::set<string> used_nodes;
+  std::unordered_set<TensorId, TensorId::Hasher> input_names;
   for (const string& input : context.input_names) {
-    used_nodes.insert(input);
+    input_names.insert(ParseTensorName(input));
+  }
+  StringPieceSet used_nodes;
+  StringPieceSet current_nodes;
+  for (const string& name : context.output_names) {
+    TensorId id = ParseTensorName(name);
+    used_nodes.insert(id.first);
+    current_nodes.insert(id.first);
   }
-  std::vector<string> current_nodes = context.output_names;
   while (!current_nodes.empty()) {
-    std::set<string> next_nodes;
-    for (const string& node_name : current_nodes) {
-      used_nodes.insert(node_name);
+    StringPieceSet next_nodes;
+    for (StringPiece node_name : current_nodes) {
       if (node_map.count(node_name) == 0) {
         LOG(ERROR) << "Bad graph structure, no node named '" << node_name
                    << "' found for input lookup";
@@ -120,14 +146,20 @@ Status RemoveUnusedNodes(const GraphDef& input_graph_def,
                                        node_name, "' found for input lookup");
       }
       const NodeDef& node = *(node_map[node_name]);
-      for (const string& input_name : node.input()) {
-        const string& input_node_name = NodeNameFromInput(input_name);
-        if (used_nodes.count(input_node_name) == 0) {
-          next_nodes.insert(input_node_name);
+      for (const string& input : node.input()) {
+        TensorId id = ParseTensorName(input);
+        if (input_names.count(id) > 0) {
+          continue;
+        }
+        if (used_nodes.insert(id.first).second) {
+          next_nodes.insert(id.first);
         }
       }
     }
-    current_nodes = std::vector<string>(next_nodes.begin(), next_nodes.end());
+    current_nodes.swap(next_nodes);
+  }
+  for (const TensorId& id : input_names) {
+    used_nodes.insert(id.first);
   }
   FilterGraphDef(
       input_graph_def,
@@ -141,7 +173,7 @@ Status RemoveUnusedNodes(const GraphDef& input_graph_def,
 Status ShapeHandleToTensorShape(const shape_inference::ShapeHandle& handle,
                                 shape_inference::InferenceContext* context,
                                 PartialTensorShape* shape) {
-  // The default is already unknown
+  // The default is already unknown.
   if (!context->RankKnown(handle)) return Status::OK();
 
   std::vector<int64> dims(context->Rank(handle));
@@ -151,47 +183,6 @@ Status ShapeHandleToTensorShape(const shape_inference::ShapeHandle& handle,
   return PartialTensorShape::MakePartialShape(dims.data(), dims.size(), shape);
 }
 
-Status ShapeForNode(const TransformFuncContext& context,
-                    const string& node_name, TensorShape* result,
-                    bool* has_shape_specified) {
-  *has_shape_specified = false;
-
-  // Check to see if we have been given a default for all placeholders.
-  if (context.params.count("type")) {
-    if (context.params.at("shape").size() != 1) {
-      return errors::InvalidArgument(
-          "You must pass no more than one default 'shape' to "
-          "fold_constants");
-    }
-    const string& shape_string = context.params.at("shape")[0];
-    TF_RETURN_IF_ERROR(TensorShapeFromString(shape_string, result));
-    *has_shape_specified = true;
-  }
-
-  // See if there's a particular type specified for this placeholder.
-  if (context.params.count("name") || context.params.count("type_for_name")) {
-    if (!context.params.count("name") ||
-        !context.params.count("type_for_name") ||
-        (context.params.at("type_for_name").size() !=
-         context.params.at("name").size())) {
-      return errors::InvalidArgument(
-          "You must pass a 'shape_for_name' arg for every 'name', e.g. "
-          "fold_constants(name=foo, shape_for_name=\"2,2,1\", name=bar, "
-          "shape_for_name=\"1\"");
-    }
-    const int name_count = context.params.at("name").size();
-    for (int i = 0; i < name_count; ++i) {
-      if (context.params.at("name")[i] == node_name) {
-        const string& shape_string = context.params.at("shape_for_name")[i];
-        TF_RETURN_IF_ERROR(TensorShapeFromString(shape_string, result));
-        *has_shape_specified = true;
-      }
-    }
-  }
-
-  return Status::OK();
-}
-
 // Converts any sub-graphs that can be resolved into constant expressions into
 // single Const ops.
 Status FoldConstants(const GraphDef& input_graph_def,
@@ -215,17 +206,6 @@ Status FoldConstants(const GraphDef& input_graph_def,
     GraphDef cleaned_graph_def;
     RemoveAttributes(input_graph_def, {"_output_shapes"}, &cleaned_graph_def);
 
-    // Set specified shapes.
-    for (NodeDef& node : *cleaned_graph_def.mutable_node()) {
-      TensorShape shape;
-      bool has_shape_specified;
-      TF_RETURN_IF_ERROR(
-          ShapeForNode(context, node.name(), &shape, &has_shape_specified));
-      if (has_shape_specified) {
-        SetNodeAttr("shape", shape, &node);
-      }
-    }
-
     TF_RETURN_IF_ERROR(
         ImportGraphDef({}, cleaned_graph_def, &input_graph, &shape_refiner));
   } else {
diff --git a/tensorflow/tools/graph_transforms/fold_constants_test.cc b/tensorflow/tools/graph_transforms/fold_constants_test.cc
index fd4188a6a4..41106de008 100644
--- a/tensorflow/tools/graph_transforms/fold_constants_test.cc
+++ b/tensorflow/tools/graph_transforms/fold_constants_test.cc
@@ -74,6 +74,9 @@ class ConstantFoldingTest : public ::testing::Test {
     TestConstantFolding(graph_def,
                         {{"placeholder_expect_remains", placeholder_tensor}},
                         {}, {"output_expect_remains"}, {});
+    TestConstantFolding(graph_def,
+                        {{"placeholder_expect_remains:0", placeholder_tensor}},
+                        {}, {"output_expect_remains:0"}, {});
   }
 
   void TestOpExclusionAdd() {
@@ -256,10 +259,40 @@ class ConstantFoldingTest : public ::testing::Test {
     EXPECT_EQ(0, node_map.count("new_send"));
   }
 
+  void TestReplaceSendRecvsPrefixNames() {
+    using namespace ::tensorflow::ops;  // NOLINT(build/namespaces)
+
+    auto o_root = tensorflow::Scope::NewRootScope();
+    auto a = Placeholder(o_root.WithOpName("placeholder"), DT_FLOAT);
+    auto b = Placeholder(o_root.WithOpName("placeholder_1"), DT_FLOAT);
+    auto add_o = Add(o_root.WithOpName("add"), a, b);
+    GraphDef o_graph_def;
+    TF_ASSERT_OK(o_root.ToGraphDef(&o_graph_def));
+
+    auto n_root = tensorflow::Scope::NewRootScope();
+    auto c = _Recv(n_root.WithOpName("_recv_placeholder_0"), DT_FLOAT, "", "",
+                   0, "");
+    auto d = _Recv(n_root.WithOpName("_recv_placeholder_1_0"), DT_FLOAT, "", "",
+                   0, "");
+    auto add_n = Add(n_root.WithOpName("add"), c, d);
+    GraphDef n_graph_def;
+    TF_ASSERT_OK(n_root.ToGraphDef(&n_graph_def));
+
+    GraphDef result_graph_def;
+    TF_ASSERT_OK(graph_transforms::ReplaceSendRecvs(
+        o_graph_def, n_graph_def, {"placeholder", "placeholder_1"}, {"add"},
+        &result_graph_def));
+
+    std::map<string, const NodeDef*> node_map;
+    graph_transforms::MapNamesToNodes(result_graph_def, &node_map);
+    EXPECT_EQ(1, node_map.count("placeholder"));
+    EXPECT_EQ(1, node_map.count("placeholder_1"));
+    EXPECT_EQ(1, node_map.count("add"));
+  }
+
   void TestRemoveUnusedNodes() {
     using namespace ::tensorflow::ops;  // NOLINT(build/namespaces)
     auto root = tensorflow::Scope::NewRootScope();
-    using namespace ::tensorflow::ops;  // NOLINT(build/namespaces)
 
     const int width = 100;
 
@@ -295,6 +328,48 @@ class ConstantFoldingTest : public ::testing::Test {
     EXPECT_EQ(1, node_map.count("output"));
     EXPECT_EQ(0, node_map.count("unused"));
   }
+
+  void TestRemoveUnusedNodesMultipleOutputs() {
+    using namespace ::tensorflow::ops;  // NOLINT(build/namespaces)
+    auto root = tensorflow::Scope::NewRootScope();
+
+    //    a    b
+    //     \  /
+    //    shape_n
+    //     \  /
+    //       c
+    auto a = Placeholder(root.WithOpName("a"), DT_FLOAT);
+    auto b = Placeholder(root.WithOpName("b"), DT_FLOAT);
+    auto shape_n = ShapeN(root.WithOpName("shape_n"), {Output(a), Output(b)});
+    auto c = Add(root.WithOpName("c"), shape_n[0], shape_n[1]);
+
+    GraphDef graph_def;
+    TF_ASSERT_OK(root.ToGraphDef(&graph_def));
+    GraphDef result_graph_def;
+    TF_ASSERT_OK(graph_transforms::RemoveUnusedNodes(
+        graph_def, {{shape_n[0].name()}, {"c"}}, &result_graph_def));
+
+    // Only one output of shape_n node is fed input. Hence the graph search
+    // should propagate to inputs of shape_n. Nothing to remove here.
+    std::map<string, const NodeDef*> node_map;
+    graph_transforms::MapNamesToNodes(result_graph_def, &node_map);
+    EXPECT_EQ(1, node_map.count("a"));
+    EXPECT_EQ(1, node_map.count("b"));
+    EXPECT_EQ(1, node_map.count("c"));
+
+    result_graph_def.Clear();
+    TF_ASSERT_OK(graph_transforms::RemoveUnusedNodes(
+        graph_def, {{shape_n[0].name(), shape_n[1].name()}, {"c"}},
+        &result_graph_def));
+
+    // Both outputs of shape_n node are fed inputs. shape_n does not function
+    // and inputs to shape_n should be removed.
+    node_map.clear();
+    graph_transforms::MapNamesToNodes(result_graph_def, &node_map);
+    EXPECT_EQ(0, node_map.count("a"));
+    EXPECT_EQ(0, node_map.count("b"));
+    EXPECT_EQ(1, node_map.count("c"));
+  }
 };
 
 TEST_F(ConstantFoldingTest, TestSimpleAdd) { TestSimpleAdd(); }
@@ -309,7 +384,15 @@ TEST_F(ConstantFoldingTest, TestPreserveOutputShapes) {
 
 TEST_F(ConstantFoldingTest, TestReplaceSendRecvs) { TestReplaceSendRecvs(); }
 
+TEST_F(ConstantFoldingTest, TestReplaceSendRecvsPrefixNames) {
+  TestReplaceSendRecvsPrefixNames();
+}
+
 TEST_F(ConstantFoldingTest, TestRemoveUnusedNodes) { TestRemoveUnusedNodes(); }
 
+TEST_F(ConstantFoldingTest, TestRemoveUnusedNodesMultipleOutputs) {
+  TestRemoveUnusedNodesMultipleOutputs();
+}
+
 }  // namespace graph_transforms
 }  // namespace tensorflow
diff --git a/tensorflow/tools/pip_package/setup.py b/tensorflow/tools/pip_package/setup.py
index 02723f3e79..4f0de8f768 100644
--- a/tensorflow/tools/pip_package/setup.py
+++ b/tensorflow/tools/pip_package/setup.py
@@ -35,7 +35,7 @@ REQUIRED_PACKAGES = [
     'enum34 >= 1.1.6',
     'numpy >= 1.12.1',
     'six >= 1.10.0',
-    'protobuf >= 3.3.0',
+    'protobuf >= 3.4.0',
     'tensorflow-tensorboard >= 0.4.0rc1, < 0.5.0',
 ]
 
-- 
GitLab


From 2b91b812ef50384cd0526ea513f1cf585adb6ef7 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 18 Oct 2017 16:03:56 -0700
Subject: [PATCH 539/909] Move random tests to a separate subdirectory.

This in preparation for refactoring the tests to use a common library.

PiperOrigin-RevId: 172670730
---
 tensorflow/BUILD                              |   1 +
 tensorflow/python/kernel_tests/BUILD          | 112 +--------------
 tensorflow/python/kernel_tests/random/BUILD   | 135 ++++++++++++++++++
 .../{ => random}/multinomial_op_big_test.py   |   0
 .../{ => random}/multinomial_op_test.py       |   0
 .../{ => random}/random_crop_test.py          |   0
 .../{ => random}/random_gamma_test.py         |   0
 .../{ => random}/random_ops_test.py           |   0
 .../{ => random}/random_poisson_test.py       |   0
 .../{ => random}/random_shuffle_queue_test.py |   0
 10 files changed, 140 insertions(+), 108 deletions(-)
 create mode 100644 tensorflow/python/kernel_tests/random/BUILD
 rename tensorflow/python/kernel_tests/{ => random}/multinomial_op_big_test.py (100%)
 rename tensorflow/python/kernel_tests/{ => random}/multinomial_op_test.py (100%)
 rename tensorflow/python/kernel_tests/{ => random}/random_crop_test.py (100%)
 rename tensorflow/python/kernel_tests/{ => random}/random_gamma_test.py (100%)
 rename tensorflow/python/kernel_tests/{ => random}/random_ops_test.py (100%)
 rename tensorflow/python/kernel_tests/{ => random}/random_poisson_test.py (100%)
 rename tensorflow/python/kernel_tests/{ => random}/random_shuffle_queue_test.py (100%)

diff --git a/tensorflow/BUILD b/tensorflow/BUILD
index a563e3b383..93646ad650 100644
--- a/tensorflow/BUILD
+++ b/tensorflow/BUILD
@@ -501,6 +501,7 @@ filegroup(
         "//tensorflow/python/kernel_tests:all_files",
         "//tensorflow/python/kernel_tests/distributions:all_files",
         "//tensorflow/python/kernel_tests/linalg:all_files",
+        "//tensorflow/python/kernel_tests/random:all_files",
         "//tensorflow/python/ops/distributions:all_files",
         "//tensorflow/python/ops/linalg:all_files",
         "//tensorflow/python/profiler:all_files",
diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD
index 847c078971..dece290f83 100644
--- a/tensorflow/python/kernel_tests/BUILD
+++ b/tensorflow/python/kernel_tests/BUILD
@@ -622,21 +622,6 @@ tf_py_test(
     ],
 )
 
-tf_py_test(
-    name = "random_shuffle_queue_test",
-    size = "small",
-    srcs = ["random_shuffle_queue_test.py"],
-    additional_deps = [
-        "//third_party/py/numpy",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:data_flow_ops",
-        "//tensorflow/python:errors",
-        "//tensorflow/python:framework",
-        "//tensorflow/python:framework_for_generated_wrappers",
-        "//tensorflow/python:platform",
-    ],
-)
-
 cuda_py_test(
     name = "resource_variable_ops_test",
     size = "small",
@@ -1538,43 +1523,6 @@ cuda_py_test(
     ],
 )
 
-cuda_py_test(
-    name = "multinomial_op_test",
-    size = "small",
-    srcs = ["multinomial_op_test.py"],
-    additional_deps = [
-        "//third_party/py/numpy",
-        "//tensorflow/core:protos_all_py",
-        "//tensorflow/python:array_ops",
-        "//tensorflow/python:client",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:control_flow_ops",
-        "//tensorflow/python:framework",
-        "//tensorflow/python:framework_for_generated_wrappers",
-        "//tensorflow/python:math_ops",
-        "//tensorflow/python:random_ops",
-    ],
-)
-
-cuda_py_test(
-    name = "multinomial_op_big_test",
-    size = "medium",
-    srcs = ["multinomial_op_big_test.py"],
-    additional_deps = [
-        "//third_party/py/numpy",
-        "//tensorflow/core:protos_all_py",
-        "//tensorflow/python:array_ops",
-        "//tensorflow/python:client",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:control_flow_ops",
-        "//tensorflow/python:framework",
-        "//tensorflow/python:framework_for_generated_wrappers",
-        "//tensorflow/python:math_ops",
-        "//tensorflow/python:random_ops",
-    ],
-    shard_count = 3,
-)
-
 cuda_py_test(
     name = "numerics_test",
     size = "small",
@@ -1659,30 +1607,6 @@ cuda_py_test(
     tags = ["no_windows"],
 )
 
-cuda_py_test(
-    name = "random_crop_test",
-    size = "small",
-    srcs = ["random_crop_test.py"],
-    additional_deps = [
-        "//third_party/py/numpy",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:random_ops",
-    ],
-)
-
-cuda_py_test(
-    name = "random_ops_test",
-    size = "medium",
-    srcs = ["random_ops_test.py"],
-    additional_deps = [
-        "//third_party/py/numpy",
-        "//tensorflow/python:array_ops",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:framework_for_generated_wrappers",
-        "//tensorflow/python:random_ops",
-    ],
-)
-
 cuda_py_test(
     name = "reduce_join_op_test",
     size = "small",
@@ -2359,37 +2283,6 @@ cuda_py_test(
     shard_count = 4,
 )
 
-cuda_py_test(
-    name = "random_gamma_test",
-    size = "medium",
-    srcs = ["random_gamma_test.py"],
-    additional_deps = [
-        "//third_party/py/numpy",
-        "//tensorflow/python:array_ops",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:framework",
-        "//tensorflow/python:framework_for_generated_wrappers",
-        "//tensorflow/python:platform",
-        "//tensorflow/python:random_ops",
-    ],
-    shard_count = 4,
-)
-
-cuda_py_test(
-    name = "random_poisson_test",
-    size = "medium",
-    srcs = ["random_poisson_test.py"],
-    additional_deps = [
-        "//third_party/py/numpy",
-        "//tensorflow/python:array_ops",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:framework",
-        "//tensorflow/python:framework_for_generated_wrappers",
-        "//tensorflow/python:platform",
-        "//tensorflow/python:random_ops",
-    ],
-)
-
 cuda_py_test(
     name = "rnn_test",
     size = "medium",
@@ -2805,7 +2698,10 @@ tf_py_test(
         "//tensorflow/python:variables",
     ],
     shard_count = 3,
-    tags = ["no_windows_gpu"],
+    tags = [
+        "no_windows_gpu",
+        "nozapfhahn",
+    ],
 )
 
 tf_py_test(
diff --git a/tensorflow/python/kernel_tests/random/BUILD b/tensorflow/python/kernel_tests/random/BUILD
new file mode 100644
index 0000000000..88a4ddf7f2
--- /dev/null
+++ b/tensorflow/python/kernel_tests/random/BUILD
@@ -0,0 +1,135 @@
+# Tests of TensorFlow kernels written using the Python API.
+
+package(
+    default_visibility = ["//tensorflow:internal"],
+)
+
+licenses(["notice"])  # Apache 2.0
+
+load("//tensorflow:tensorflow.bzl", "tf_py_test")
+load("//tensorflow:tensorflow.bzl", "cuda_py_test")
+load("//tensorflow:tensorflow.bzl", "sycl_py_test")
+
+# CPU only tests should use tf_py_test, GPU tests use cuda_py_test
+# Please avoid the py_tests and cuda_py_tests (plural) while we
+# fix the shared/overbroad dependencies.
+
+tf_py_test(
+    name = "random_shuffle_queue_test",
+    size = "small",
+    srcs = ["random_shuffle_queue_test.py"],
+    additional_deps = [
+        "//third_party/py/numpy",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:data_flow_ops",
+        "//tensorflow/python:errors",
+        "//tensorflow/python:framework",
+        "//tensorflow/python:framework_for_generated_wrappers",
+        "//tensorflow/python:platform",
+    ],
+)
+
+cuda_py_test(
+    name = "multinomial_op_test",
+    size = "small",
+    srcs = ["multinomial_op_test.py"],
+    additional_deps = [
+        "//third_party/py/numpy",
+        "//tensorflow/core:protos_all_py",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:client",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:control_flow_ops",
+        "//tensorflow/python:framework",
+        "//tensorflow/python:framework_for_generated_wrappers",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:random_ops",
+    ],
+)
+
+cuda_py_test(
+    name = "multinomial_op_big_test",
+    size = "medium",
+    srcs = ["multinomial_op_big_test.py"],
+    additional_deps = [
+        "//third_party/py/numpy",
+        "//tensorflow/core:protos_all_py",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:client",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:control_flow_ops",
+        "//tensorflow/python:framework",
+        "//tensorflow/python:framework_for_generated_wrappers",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:random_ops",
+    ],
+    shard_count = 3,
+)
+
+cuda_py_test(
+    name = "random_crop_test",
+    size = "small",
+    srcs = ["random_crop_test.py"],
+    additional_deps = [
+        "//third_party/py/numpy",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:random_ops",
+    ],
+)
+
+cuda_py_test(
+    name = "random_ops_test",
+    size = "medium",
+    srcs = ["random_ops_test.py"],
+    additional_deps = [
+        "//third_party/py/numpy",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework_for_generated_wrappers",
+        "//tensorflow/python:random_ops",
+    ],
+)
+
+cuda_py_test(
+    name = "random_gamma_test",
+    size = "medium",
+    srcs = ["random_gamma_test.py"],
+    additional_deps = [
+        "//third_party/py/numpy",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework",
+        "//tensorflow/python:framework_for_generated_wrappers",
+        "//tensorflow/python:platform",
+        "//tensorflow/python:random_ops",
+    ],
+    shard_count = 4,
+    tags = ["nozapfhahn"],
+)
+
+cuda_py_test(
+    name = "random_poisson_test",
+    size = "medium",
+    srcs = ["random_poisson_test.py"],
+    additional_deps = [
+        "//third_party/py/numpy",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework",
+        "//tensorflow/python:framework_for_generated_wrappers",
+        "//tensorflow/python:platform",
+        "//tensorflow/python:random_ops",
+    ],
+)
+
+filegroup(
+    name = "all_files",
+    srcs = glob(
+        ["**/*"],
+        exclude = [
+            "**/METADATA",
+            "**/OWNERS",
+        ],
+    ),
+    visibility = ["//tensorflow:__subpackages__"],
+)
diff --git a/tensorflow/python/kernel_tests/multinomial_op_big_test.py b/tensorflow/python/kernel_tests/random/multinomial_op_big_test.py
similarity index 100%
rename from tensorflow/python/kernel_tests/multinomial_op_big_test.py
rename to tensorflow/python/kernel_tests/random/multinomial_op_big_test.py
diff --git a/tensorflow/python/kernel_tests/multinomial_op_test.py b/tensorflow/python/kernel_tests/random/multinomial_op_test.py
similarity index 100%
rename from tensorflow/python/kernel_tests/multinomial_op_test.py
rename to tensorflow/python/kernel_tests/random/multinomial_op_test.py
diff --git a/tensorflow/python/kernel_tests/random_crop_test.py b/tensorflow/python/kernel_tests/random/random_crop_test.py
similarity index 100%
rename from tensorflow/python/kernel_tests/random_crop_test.py
rename to tensorflow/python/kernel_tests/random/random_crop_test.py
diff --git a/tensorflow/python/kernel_tests/random_gamma_test.py b/tensorflow/python/kernel_tests/random/random_gamma_test.py
similarity index 100%
rename from tensorflow/python/kernel_tests/random_gamma_test.py
rename to tensorflow/python/kernel_tests/random/random_gamma_test.py
diff --git a/tensorflow/python/kernel_tests/random_ops_test.py b/tensorflow/python/kernel_tests/random/random_ops_test.py
similarity index 100%
rename from tensorflow/python/kernel_tests/random_ops_test.py
rename to tensorflow/python/kernel_tests/random/random_ops_test.py
diff --git a/tensorflow/python/kernel_tests/random_poisson_test.py b/tensorflow/python/kernel_tests/random/random_poisson_test.py
similarity index 100%
rename from tensorflow/python/kernel_tests/random_poisson_test.py
rename to tensorflow/python/kernel_tests/random/random_poisson_test.py
diff --git a/tensorflow/python/kernel_tests/random_shuffle_queue_test.py b/tensorflow/python/kernel_tests/random/random_shuffle_queue_test.py
similarity index 100%
rename from tensorflow/python/kernel_tests/random_shuffle_queue_test.py
rename to tensorflow/python/kernel_tests/random/random_shuffle_queue_test.py
-- 
GitLab


From bba3957467ad8ba9351b829036120412d5d006cb Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 18 Oct 2017 16:25:56 -0700
Subject: [PATCH 540/909] Replace as_gpu_tensor and as_cpu_tensor to gpu and
 cpu

PiperOrigin-RevId: 172673720
---
 .../contrib/eager/python/summary_writer.py    | 18 +++++++-------
 tensorflow/contrib/eager/python/tfe_test.py   |  4 ++--
 tensorflow/python/eager/backprop_test.py      | 14 +++++------
 tensorflow/python/eager/benchmarks_test.py    |  4 ++--
 tensorflow/python/eager/core_test.py          | 24 +++++++++----------
 tensorflow/python/eager/function_test.py      | 12 +++++-----
 tensorflow/python/eager/ops_test.py           | 12 +++++-----
 tensorflow/python/framework/constant_op.py    |  2 +-
 tensorflow/python/framework/ops.py            | 11 ++++-----
 tensorflow/python/ops/array_grad.py           |  2 +-
 10 files changed, 50 insertions(+), 53 deletions(-)

diff --git a/tensorflow/contrib/eager/python/summary_writer.py b/tensorflow/contrib/eager/python/summary_writer.py
index 39993558e3..5a698b92c6 100644
--- a/tensorflow/contrib/eager/python/summary_writer.py
+++ b/tensorflow/contrib/eager/python/summary_writer.py
@@ -32,9 +32,9 @@ from tensorflow.python.ops import summary_op_util
 from tensorflow.python.ops import variable_scope
 
 
-def _maybe_as_cpu_tensor(v):
+def _maybe_cpu(v):
   if isinstance(v, (ops.EagerTensor, ops.Tensor)):
-    return v.as_cpu_tensor()
+    return v.cpu()
   else:
     return v
 
@@ -161,9 +161,9 @@ class SummaryWriter(object):
         gen_summary_ops.write_summary(
             self._resource,
             self._update_global_step_tensor(),
-            _maybe_as_cpu_tensor(tensor),
+            _maybe_cpu(tensor),
             tag,
-            _maybe_as_cpu_tensor(metadata),
+            _maybe_cpu(metadata),
             name=scope)
 
   def scalar(self, name, tensor, family=None):
@@ -185,7 +185,7 @@ class SummaryWriter(object):
           name, family, values=[tensor]) as (tag, scope):
         gen_summary_ops.write_scalar_summary(
             self._resource, self._update_global_step_tensor(),
-            tag, _maybe_as_cpu_tensor(tensor), name=scope)
+            tag, _maybe_cpu(tensor), name=scope)
 
   def histogram(self, name, tensor, family=None):
     """Write a histogram summary.
@@ -203,7 +203,7 @@ class SummaryWriter(object):
           name, family, values=[tensor]) as (tag, scope):
         gen_summary_ops.write_histogram_summary(
             self._resource, self._update_global_step_tensor(),
-            tag, _maybe_as_cpu_tensor(tensor), name=scope)
+            tag, _maybe_cpu(tensor), name=scope)
 
   def image(self, name, tensor, bad_color=None, max_images=3, family=None):
     """Write an image summary."""
@@ -214,7 +214,7 @@ class SummaryWriter(object):
           name, family, values=[tensor]) as (tag, scope):
         gen_summary_ops.write_image_summary(
             self._resource, self._update_global_step_tensor(),
-            tag, _maybe_as_cpu_tensor(tensor), bad_color_, max_images,
+            tag, _maybe_cpu(tensor), bad_color_, max_images,
             name=scope)
 
   def audio(self, name, tensor, sample_rate, max_outputs, family=None):
@@ -238,7 +238,7 @@ class SummaryWriter(object):
         gen_summary_ops.write_audio_summary(
             self._resource, self._update_global_step_tensor(),
             tag,
-            _maybe_as_cpu_tensor(tensor),
-            sample_rate=_maybe_as_cpu_tensor(sample_rate),
+            _maybe_cpu(tensor),
+            sample_rate=_maybe_cpu(sample_rate),
             max_outputs=max_outputs,
             name=scope)
diff --git a/tensorflow/contrib/eager/python/tfe_test.py b/tensorflow/contrib/eager/python/tfe_test.py
index 3d57a98a2e..eab8958f23 100644
--- a/tensorflow/contrib/eager/python/tfe_test.py
+++ b/tensorflow/contrib/eager/python/tfe_test.py
@@ -75,7 +75,7 @@ class TFETest(test_util.TensorFlowTestCase):
       self.skipTest('No GPUs available')
 
     # tf.Tensor.as_gpu_device() moves a tensor to GPU.
-    x = constant_op.constant([[1., 2.], [3., 4.]]).as_gpu_tensor()
+    x = constant_op.constant([[1., 2.], [3., 4.]]).gpu()
     # Alternatively, tf.device() as a context manager places tensors and
     # operations.
     with ops.device('gpu:0'):
@@ -85,7 +85,7 @@ class TFETest(test_util.TensorFlowTestCase):
     reduction_indices = range(x.shape.ndims)
     m = math_ops.reduce_mean(x, reduction_indices)
     # m is on GPU, bring it back to CPU and compare.
-    self.assertEqual(3.5, m.as_cpu_tensor().numpy())
+    self.assertEqual(3.5, m.cpu().numpy())
 
   def testListDevices(self):
     # Expect at least one device.
diff --git a/tensorflow/python/eager/backprop_test.py b/tensorflow/python/eager/backprop_test.py
index 002be95d0f..5161095683 100644
--- a/tensorflow/python/eager/backprop_test.py
+++ b/tensorflow/python/eager/backprop_test.py
@@ -209,10 +209,10 @@ class BackpropTest(test.TestCase):
     def fn(x):
       with context.device('/gpu:0'):
         b = constant_op.constant(2.0)
-        c = math_ops.add(x.as_gpu_tensor(), b)
-        # TODO(apassos): remove as_cpu_tensor below by making TensorVSPace aware
+        c = math_ops.add(x.gpu(), b)
+        # TODO(apassos): remove cpu below by making TensorVSPace aware
         # of devices.
-        return math_ops.add(c, constant_op.constant(3.0)).as_cpu_tensor()
+        return math_ops.add(c, constant_op.constant(3.0)).cpu()
 
     grad = backprop.gradients_function(fn, [0])(constant_op.constant(1.0))[0]
     self.assertAllEqual(grad, 1.0)
@@ -230,7 +230,7 @@ class BackpropTest(test.TestCase):
         return v.read_value()
 
     self.assertEqual(
-        backprop.implicit_grad(f)()[0][0].as_cpu_tensor().numpy(), 1.0)
+        backprop.implicit_grad(f)()[0][0].cpu().numpy(), 1.0)
 
   def testCPU(self):
 
@@ -247,7 +247,7 @@ class BackpropTest(test.TestCase):
       self.skipTest('No GPUs found')
 
     def f(a, b):
-      return a.as_cpu_tensor() + b.as_cpu_tensor()
+      return a.cpu() + b.cpu()
 
     with context.device('/gpu:0'):
       a = constant_op.constant(1.0)
@@ -309,8 +309,8 @@ class BackpropTest(test.TestCase):
     # back: e (cpu) -> add (cpu) -> c (cpu->gpu) -> add (gpu) -> grad (gpu->cpu)
     def f(a, b):
       with context.device('/gpu:0'):
-        c = math_ops.add(a.as_gpu_tensor(0), b.as_gpu_tensor(0))
-      return math_ops.add(c.as_cpu_tensor(), constant_op.constant(3.0))
+        c = math_ops.add(a.gpu(0), b.gpu(0))
+      return math_ops.add(c.cpu(), constant_op.constant(3.0))
 
     with context.device('/cpu:0'):
       a = constant_op.constant(1.0)
diff --git a/tensorflow/python/eager/benchmarks_test.py b/tensorflow/python/eager/benchmarks_test.py
index 407d1e979c..ebc9e346c0 100644
--- a/tensorflow/python/eager/benchmarks_test.py
+++ b/tensorflow/python/eager/benchmarks_test.py
@@ -104,7 +104,7 @@ def benchmark_matmul(shape, n, use_gpu=False):
   transpose_b = (shape[0] != shape[1])
   m = random_ops.random_uniform(shape)
   if use_gpu:
-    m = m.as_gpu_tensor()
+    m = m.gpu()
     # Warm up the GPU - the very first kernel invocation
     # seems to require a bunch of setup.
     math_ops.matmul(m, m, transpose_b=transpose_b)
@@ -113,7 +113,7 @@ def benchmark_matmul(shape, n, use_gpu=False):
     return "MatMul {}: {:30s}".format(shape, s)
 
   if not use_gpu:
-    a = m.as_cpu_tensor().numpy()
+    a = m.cpu().numpy()
     b = a.T if transpose_b else a
     with timer(label("np.dot"), iters=n) as iters:
       for _ in iters:
diff --git a/tensorflow/python/eager/core_test.py b/tensorflow/python/eager/core_test.py
index 54a0be6dd9..2449162dca 100644
--- a/tensorflow/python/eager/core_test.py
+++ b/tensorflow/python/eager/core_test.py
@@ -122,13 +122,13 @@ class TFETest(test_util.TensorFlowTestCase):
     if not context.context().num_gpus():
       self.skipTest('No GPUs found')
 
-    x = constant_op.constant(1.).as_gpu_tensor()
+    x = constant_op.constant(1.).gpu()
     with context.device('gpu:0'):
       y = constant_op.constant(2.)
     # Add would fail if t2 were not on GPU
     result = execute(
         b'Add', 1, inputs=[x, y],
-        attrs=('T', x.dtype.as_datatype_enum))[0].as_cpu_tensor().numpy()
+        attrs=('T', x.dtype.as_datatype_enum))[0].cpu().numpy()
     self.assertEqual(3, result)
 
   def testCopyBetweenDevices(self):
@@ -136,26 +136,26 @@ class TFETest(test_util.TensorFlowTestCase):
       self.skipTest('No GPUs found')
 
     x = constant_op.constant([[1., 2.], [3., 4.]])
-    x = x.as_cpu_tensor()
-    x = x.as_gpu_tensor()
-    x = x.as_gpu_tensor()
-    x = x.as_cpu_tensor()
+    x = x.cpu()
+    x = x.gpu()
+    x = x.gpu()
+    x = x.cpu()
 
     # Invalid device
     with self.assertRaises(RuntimeError):
-      x.as_gpu_tensor(context.context().num_gpus() + 1)
+      x.gpu(context.context().num_gpus() + 1)
 
   def testNumpyForceCPU(self):
     if not context.context().num_gpus():
       self.skipTest('No GPUs found')
 
     cpu = constant_op.constant([[1., 2.], [3., 4.]])
-    c2g = cpu.as_gpu_tensor()
+    c2g = cpu.gpu()
     self.assertAllEqual(c2g, cpu.numpy())
 
   def testCopyFromCPUToCPU(self):
     ta = constant_op.constant([[1, 2], [3, 4]])
-    tb = ta.as_cpu_tensor()
+    tb = ta.cpu()
 
     self.assertNotEqual(id(ta), id(tb))
     self.assertAllEqual(ta, tb.numpy())
@@ -189,8 +189,8 @@ class TFETest(test_util.TensorFlowTestCase):
   def testMatMulGPU(self):
     if not context.context().num_gpus():
       self.skipTest('No GPUs found')
-    three = constant_op.constant([[3.]]).as_gpu_tensor()
-    five = constant_op.constant([[5.]]).as_gpu_tensor()
+    three = constant_op.constant([[3.]]).gpu()
+    five = constant_op.constant([[5.]]).gpu()
     product = execute(
         b'MatMul',
         num_outputs=1,
@@ -450,7 +450,7 @@ class TFETest(test_util.TensorFlowTestCase):
     shape = constant_op.constant([], dtype=dtypes.int32)
 
     # x: Run the "TruncatedNormal" op CPU and copy result to GPU.
-    x = truncated_normal(shape).as_gpu_tensor()
+    x = truncated_normal(shape).gpu()
     # y: Explicitly run the "TruncatedNormal" op on GPU.
     with context.device('gpu:0'):
       y = truncated_normal(shape)
diff --git a/tensorflow/python/eager/function_test.py b/tensorflow/python/eager/function_test.py
index e9e396b49b..fb647f5c21 100644
--- a/tensorflow/python/eager/function_test.py
+++ b/tensorflow/python/eager/function_test.py
@@ -173,9 +173,9 @@ class FunctionTest(test.TestCase):
     if not context.context().num_gpus():
       self.skipTest('No GPUs found')
 
-    x = constant_op.constant([1.]).as_gpu_tensor()
+    x = constant_op.constant([1.]).gpu()
     f = function.defun(math_ops.add)
-    y = f(x, x).as_cpu_tensor()
+    y = f(x, x).cpu()
     self.assertAllEqual(y, [2.])
 
   def testFunctionHandlesInputsOnDifferentDevices(self):
@@ -184,9 +184,9 @@ class FunctionTest(test.TestCase):
 
     # The Reshape op requires the shape tensor to be placed in host memory.
     reshape = function.defun(array_ops.reshape)
-    value = constant_op.constant([1., 2.]).as_gpu_tensor()
+    value = constant_op.constant([1., 2.]).gpu()
     shape = constant_op.constant([2, 1])
-    reshaped = reshape(value, shape).as_cpu_tensor()
+    reshaped = reshape(value, shape).cpu()
     self.assertAllEqual(reshaped, [[1], [2]])
 
   def testFunctionHandlesInputsPlacedOnTheWrongDeviceGracefully(self):
@@ -195,8 +195,8 @@ class FunctionTest(test.TestCase):
 
     # The Reshape op requires the shape tensor to be placed in host memory.
     reshape = function.defun(array_ops.reshape)
-    value = constant_op.constant([1., 2.]).as_gpu_tensor()
-    shape = constant_op.constant([2, 1]).as_gpu_tensor()
+    value = constant_op.constant([1., 2.]).gpu()
+    shape = constant_op.constant([2, 1]).gpu()
     with self.assertRaises(errors.InvalidArgumentError):
       reshape(value, shape)
 
diff --git a/tensorflow/python/eager/ops_test.py b/tensorflow/python/eager/ops_test.py
index f737bfbc15..2ebb625f9f 100644
--- a/tensorflow/python/eager/ops_test.py
+++ b/tensorflow/python/eager/ops_test.py
@@ -46,8 +46,8 @@ class OpsTest(test_util.TensorFlowTestCase):
   def testMatMulGPU(self):
     if not context.context().num_gpus():
       self.skipTest('No GPUs found')
-    three = constant_op.constant([[3.]]).as_gpu_tensor()
-    five = constant_op.constant([[5.]]).as_gpu_tensor()
+    three = constant_op.constant([[3.]]).gpu()
+    five = constant_op.constant([[5.]]).gpu()
     product = math_ops.matmul(three, five)
     self.assertEqual([[15.0]], product.numpy())
 
@@ -239,10 +239,10 @@ class OpsTest(test_util.TensorFlowTestCase):
 
     # The GPU kernel for the Reshape op requires that the
     # shape input be on CPU.
-    value = constant_op.constant([1., 2.]).as_gpu_tensor()
+    value = constant_op.constant([1., 2.]).gpu()
     shape = constant_op.constant([2, 1])
     reshaped = array_ops.reshape(value, shape)
-    self.assertAllEqual([[1], [2]], reshaped.as_cpu_tensor())
+    self.assertAllEqual([[1], [2]], reshaped.cpu())
 
     # And if the shape is in device memory, it should complain
     # TODO(ashankar): Revisit this - perhaps instead of complaining,
@@ -250,7 +250,7 @@ class OpsTest(test_util.TensorFlowTestCase):
     with self.assertRaisesRegexp(
         errors.InvalidArgumentError,
         'cannot compute Reshape as input #1 was expected to be on'):
-      reshaped = array_ops.reshape(value, shape.as_gpu_tensor())
+      reshaped = array_ops.reshape(value, shape.gpu())
 
   def testInvalidInputDataType(self):
     # Fill requires the first input to be an int32 tensor.
@@ -262,7 +262,7 @@ class OpsTest(test_util.TensorFlowTestCase):
     if not context.context().num_gpus():
       self.skipTest('No GPUs found')
     # The Shape op kernel on GPU places the output in host memory.
-    value = constant_op.constant([1.]).as_gpu_tensor()
+    value = constant_op.constant([1.]).gpu()
     shape = array_ops.shape(value)
     self.assertEqual([1], shape.numpy())
 
diff --git a/tensorflow/python/framework/constant_op.py b/tensorflow/python/framework/constant_op.py
index 686f5aa6db..34848af53b 100644
--- a/tensorflow/python/framework/constant_op.py
+++ b/tensorflow/python/framework/constant_op.py
@@ -195,7 +195,7 @@ def constant(value, dtype=None, shape=None, name="Const", verify_shape=False):
         # We don't have a Fill kernel for bool dtype on GPU. So we first run
         # Fill on CPU and then copy to GPU if needed.
         with ops.device("/device:CPU:0"):
-          x = _eager_fill(shape.as_list(), t.as_cpu_tensor(), ctx)
+          x = _eager_fill(shape.as_list(), t.cpu(), ctx)
         return _eager_identity(x, ctx)
       else:
         return _eager_fill(shape.as_list(), t, ctx)
diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index ef0ed8fc53..e7e36941e5 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -607,9 +607,6 @@ class _EagerTensorBase(Tensor):
   def numpy(self):
     """Returns a numpy array with the same contents as the Tensor.
 
-    The contents of the Tensor must be backed by host memory. The
-    as_cpu_tensor() method can be used ensure that this is true.
-
     TODO(ashankar,agarwal): Perhaps this should NOT reference the underlying
     buffer but instead always explicitly copy? Note that currently it may or may
     not copy based on whether the numpy data is properly aligned or not.
@@ -618,7 +615,7 @@ class _EagerTensorBase(Tensor):
       A numpy array that may share memory with the Tensor object. Any changes
       to one may be reflected in the other.
     """
-    return self.as_cpu_tensor()._numpy()  # pylint: disable=protected-access
+    return self.cpu()._numpy()  # pylint: disable=protected-access
 
   def __array__(self):
     return np.array(self.numpy())
@@ -703,11 +700,11 @@ class _EagerTensorBase(Tensor):
     """The shape of the tensor as a list."""
     return list(self._shape_tuple())
 
-  def as_cpu_tensor(self):
+  def cpu(self):
     """A copy of this Tensor with contents backed by host memory."""
     return self._copy(context.context(), "CPU:0")
 
-  def as_gpu_tensor(self, gpu_index=0):
+  def gpu(self, gpu_index=0):
     """A copy of this Tensor with contents backed by memory on the GPU.
 
     Arguments:
@@ -727,7 +724,7 @@ class _EagerTensorBase(Tensor):
     if self.dtype != dtypes.bool:
       raise ValueError(
           "Non-boolean tensor %s cannot be converted to boolean." % repr(self))
-    return bool(self.as_cpu_tensor().numpy())
+    return bool(self.cpu().numpy())
 
   def __nonzero__(self):
     return self.__bool__()
diff --git a/tensorflow/python/ops/array_grad.py b/tensorflow/python/ops/array_grad.py
index 9f8acb2ae3..2ee298ad44 100644
--- a/tensorflow/python/ops/array_grad.py
+++ b/tensorflow/python/ops/array_grad.py
@@ -397,7 +397,7 @@ def _GatherV2Grad(op, grad):
   # For axis 0 gathers, build an appropriately shaped IndexedSlices.
   if axis_static == 0:
     if context.in_eager_mode():
-      params_tail_shape = params_shape.as_cpu_tensor()[1:]
+      params_tail_shape = params_shape.cpu()[1:]
     else:
       params_tail_shape = params_shape[1:]
     values_shape = array_ops.concat([indices_size, params_tail_shape], 0)
-- 
GitLab


From b1cf67b0f6b9c600a03bbdc3eec4fd7b2b6d2deb Mon Sep 17 00:00:00 2001
From: Igor Saprykin <isaprykin@google.com>
Date: Wed, 18 Oct 2017 17:26:17 -0700
Subject: [PATCH 541/909] Migrate text_classification.py from .contrib utils to
 .core.

Some usages are left untouched: datasets, VocabularyProcessor.  A tracking bug is filed for embed_sequence.

Tested by re-running and the loss numbers look similar to the ones before the change.

PiperOrigin-RevId: 172681096
---
 tensorflow/examples/learn/text_classification.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tensorflow/examples/learn/text_classification.py b/tensorflow/examples/learn/text_classification.py
index 26e6e086b3..ba89c532be 100644
--- a/tensorflow/examples/learn/text_classification.py
+++ b/tensorflow/examples/learn/text_classification.py
@@ -91,11 +91,11 @@ def rnn_model(features, labels, mode):
   word_list = tf.unstack(word_vectors, axis=1)
 
   # Create a Gated Recurrent Unit cell with hidden size of EMBEDDING_SIZE.
-  cell = tf.contrib.rnn.GRUCell(EMBEDDING_SIZE)
+  cell = tf.nn.rnn_cell.GRUCell(EMBEDDING_SIZE)
 
   # Create an unrolled Recurrent Neural Networks to length of
   # MAX_DOCUMENT_LENGTH and passes word_list as inputs for each unit.
-  _, encoding = tf.contrib.rnn.static_rnn(cell, word_list, dtype=tf.float32)
+  _, encoding = tf.nn.static_rnn(cell, word_list, dtype=tf.float32)
 
   # Given encoding of RNN, take encoding of last step (e.g hidden size of the
   # neural network of last step) and pass it as features for softmax
@@ -107,6 +107,8 @@ def rnn_model(features, labels, mode):
 
 def main(unused_argv):
   global n_words
+  tf.logging.set_verbosity(tf.logging.INFO)
+
   # Prepare training and testing data
   dbpedia = tf.contrib.learn.datasets.load_dataset(
       'dbpedia', test_with_fake_data=FLAGS.test_with_fake_data)
-- 
GitLab


From cdd3b14e47e9d5143569730485ad963936256d45 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 18 Oct 2017 17:50:10 -0700
Subject: [PATCH 542/909] Fix MKL build broken by stray line left behind in
 transpose_op.h.

PiperOrigin-RevId: 172683597
---
 tensorflow/core/kernels/transpose_op.h | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tensorflow/core/kernels/transpose_op.h b/tensorflow/core/kernels/transpose_op.h
index ff9cf5d4ff..ae67592d04 100644
--- a/tensorflow/core/kernels/transpose_op.h
+++ b/tensorflow/core/kernels/transpose_op.h
@@ -86,7 +86,6 @@ class ConjugateTransposeCpuOp : public TransposeOp {
 };
 
 #ifdef INTEL_MKL
-template <bool conjugate = false>
 class MklConjugateTransposeCpuOp : public TransposeOp {
  public:
   explicit MklConjugateTransposeCpuOp(OpKernelConstruction* ctx)
-- 
GitLab


From 4e5b3696c6747af8e824fbb47c91e980821d24f3 Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Wed, 18 Oct 2017 17:53:41 -0700
Subject: [PATCH 543/909] Adding ApiDefMap class to op_gen_lib to read ApiDef
 proto files.

PiperOrigin-RevId: 172683926
---
 tensorflow/core/BUILD                        |   2 +
 tensorflow/core/framework/op_gen_lib.cc      | 202 ++++++++++++-
 tensorflow/core/framework/op_gen_lib.h       |  44 +++
 tensorflow/core/framework/op_gen_lib_test.cc | 284 +++++++++++++++++++
 4 files changed, 531 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 94ddd0840d..5ab84fec5b 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -510,6 +510,7 @@ cc_library(
     visibility = ["//visibility:public"],
     deps = [
         ":lib",
+        ":lib_internal",
         ":op_gen_overrides_proto_cc",
         ":protos_all_cc",
     ],
@@ -2504,6 +2505,7 @@ tf_cc_test(
     srcs = ["framework/op_gen_lib_test.cc"],
     deps = [
         ":op_gen_lib",
+        ":protos_all_cc",
         ":test",
         ":test_main",
     ],
diff --git a/tensorflow/core/framework/op_gen_lib.cc b/tensorflow/core/framework/op_gen_lib.cc
index 143da996a1..cfaca897ba 100644
--- a/tensorflow/core/framework/op_gen_lib.cc
+++ b/tensorflow/core/framework/op_gen_lib.cc
@@ -17,11 +17,12 @@ limitations under the License.
 
 #include <vector>
 #include "tensorflow/core/framework/attr_value.pb.h"
-#include "tensorflow/core/framework/op_def.pb.h"
 #include "tensorflow/core/framework/op_gen_overrides.pb.h"
 #include "tensorflow/core/lib/core/errors.h"
+#include "tensorflow/core/lib/gtl/map_util.h"
 #include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/lib/strings/strcat.h"
+#include "tensorflow/core/platform/protobuf.h"
 
 namespace tensorflow {
 
@@ -393,4 +394,203 @@ const OpGenOverride* OpGenOverrideMap::ApplyOverride(OpDef* op_def) const {
   return &proto;
 }
 
+namespace {
+
+// Initializes given ApiDef with data in OpDef.
+void InitApiDefFromOpDef(const OpDef& op_def, ApiDef* api_def) {
+  api_def->set_graph_op_name(op_def.name());
+  api_def->set_visibility(ApiDef::VISIBLE);
+
+  auto* endpoint = api_def->add_endpoint();
+  endpoint->set_name(op_def.name());
+  if (op_def.has_deprecation()) {
+    endpoint->set_deprecation_version(op_def.deprecation().version());
+  }
+
+  for (const auto& op_in_arg : op_def.input_arg()) {
+    auto* api_in_arg = api_def->add_in_arg();
+    api_in_arg->set_name(op_in_arg.name());
+    api_in_arg->set_rename_to(op_in_arg.name());
+    api_in_arg->set_description(op_in_arg.description());
+  }
+  for (const auto& op_out_arg : op_def.output_arg()) {
+    auto* api_out_arg = api_def->add_out_arg();
+    api_out_arg->set_name(op_out_arg.name());
+    api_out_arg->set_rename_to(op_out_arg.name());
+    api_out_arg->set_description(op_out_arg.description());
+  }
+  for (const auto& op_attr : op_def.attr()) {
+    auto* api_attr = api_def->add_attr();
+    api_attr->set_name(op_attr.name());
+    api_attr->set_rename_to(op_attr.name());
+    if (op_attr.has_default_value()) {
+      *api_attr->mutable_default_value() = op_attr.default_value();
+    }
+    api_attr->set_description(op_attr.description());
+  }
+  api_def->set_summary(op_def.summary());
+  api_def->set_description(op_def.description());
+}
+
+// Updates base_arg based on overrides in new_arg.
+void MergeArg(ApiDef::Arg* base_arg, const ApiDef::Arg& new_arg) {
+  if (!new_arg.rename_to().empty()) {
+    base_arg->set_rename_to(new_arg.rename_to());
+  }
+  if (!new_arg.description().empty()) {
+    base_arg->set_description(new_arg.description());
+  }
+}
+
+// Updates base_attr based on overrides in new_attr.
+void MergeAttr(ApiDef::Attr* base_attr, const ApiDef::Attr& new_attr) {
+  if (!new_attr.rename_to().empty()) {
+    base_attr->set_rename_to(new_attr.rename_to());
+  }
+  if (new_attr.has_default_value()) {
+    *base_attr->mutable_default_value() = new_attr.default_value();
+  }
+  if (!new_attr.description().empty()) {
+    base_attr->set_description(new_attr.description());
+  }
+}
+
+// Updates base_api_def based on overrides in new_api_def.
+Status MergeApiDefs(ApiDef* base_api_def, const ApiDef& new_api_def) {
+  // Merge visibility
+  if (new_api_def.visibility() != ApiDef::DEFAULT_VISIBILITY) {
+    base_api_def->set_visibility(new_api_def.visibility());
+  }
+  // Merge endpoints
+  if (new_api_def.endpoint_size() > 0) {
+    base_api_def->clear_endpoint();
+    std::copy(
+        new_api_def.endpoint().begin(), new_api_def.endpoint().end(),
+        protobuf::RepeatedFieldBackInserter(base_api_def->mutable_endpoint()));
+  }
+  // Merge args
+  for (const auto& new_arg : new_api_def.in_arg()) {
+    bool found_base_arg = false;
+    for (int i = 0; i < base_api_def->in_arg_size(); ++i) {
+      auto* base_arg = base_api_def->mutable_in_arg(i);
+      if (base_arg->name() == new_arg.name()) {
+        MergeArg(base_arg, new_arg);
+        found_base_arg = true;
+        break;
+      }
+    }
+    if (!found_base_arg) {
+      return errors::FailedPrecondition("Argument ", new_arg.name(),
+                                        " not defined in base api for ",
+                                        base_api_def->graph_op_name());
+    }
+  }
+  for (const auto& new_arg : new_api_def.out_arg()) {
+    bool found_base_arg = false;
+    for (int i = 0; i < base_api_def->out_arg_size(); ++i) {
+      auto* base_arg = base_api_def->mutable_out_arg(i);
+      if (base_arg->name() == new_arg.name()) {
+        MergeArg(base_arg, new_arg);
+        found_base_arg = true;
+        break;
+      }
+    }
+    if (!found_base_arg) {
+      return errors::FailedPrecondition("Argument ", new_arg.name(),
+                                        " not defined in base api for ",
+                                        base_api_def->graph_op_name());
+    }
+  }
+  // Merge arg order
+  if (new_api_def.arg_order_size() > 0) {
+    base_api_def->clear_arg_order();
+    std::copy(
+        new_api_def.arg_order().begin(), new_api_def.arg_order().end(),
+        protobuf::RepeatedFieldBackInserter(base_api_def->mutable_arg_order()));
+  }
+  // Merge attributes
+  for (const auto& new_attr : new_api_def.attr()) {
+    bool found_base_attr = false;
+    for (int i = 0; i < base_api_def->attr_size(); ++i) {
+      auto* base_attr = base_api_def->mutable_attr(i);
+      if (base_attr->name() == new_attr.name()) {
+        MergeAttr(base_attr, new_attr);
+        found_base_attr = true;
+        break;
+      }
+    }
+    if (!found_base_attr) {
+      return errors::FailedPrecondition("Attribute ", new_attr.name(),
+                                        " not defined in base api for ",
+                                        base_api_def->graph_op_name());
+    }
+  }
+  // Merge summary
+  if (!new_api_def.summary().empty()) {
+    base_api_def->set_summary(new_api_def.summary());
+  }
+  // Merge description
+  auto description = new_api_def.description().empty()
+                         ? base_api_def->description()
+                         : new_api_def.description();
+
+  if (!new_api_def.description_prefix().empty()) {
+    description =
+        strings::StrCat(new_api_def.description_prefix(), "\n", description);
+  }
+  if (!new_api_def.description_suffix().empty()) {
+    description =
+        strings::StrCat(description, "\n", new_api_def.description_suffix());
+  }
+  base_api_def->set_description(description);
+  return Status::OK();
+}
+}  // namespace
+
+ApiDefMap::ApiDefMap(const OpList& op_list) {
+  for (const auto& op : op_list.op()) {
+    ApiDef api_def;
+    InitApiDefFromOpDef(op, &api_def);
+    map_[op.name()] = api_def;
+  }
+}
+
+ApiDefMap::~ApiDefMap() {}
+
+Status ApiDefMap::LoadFileList(Env* env, const std::vector<string>& filenames) {
+  for (const auto& filename : filenames) {
+    TF_RETURN_IF_ERROR(LoadFile(env, filename));
+  }
+  return Status::OK();
+}
+
+Status ApiDefMap::LoadFile(Env* env, const string& filename) {
+  if (filename.empty()) return Status::OK();
+  string contents;
+  TF_RETURN_IF_ERROR(ReadFileToString(env, filename, &contents));
+  TF_RETURN_IF_ERROR(LoadApiDef(contents));
+  return Status::OK();
+}
+
+Status ApiDefMap::LoadApiDef(const string& api_def_file_contents) {
+  const string contents = PBTxtFromMultiline(api_def_file_contents);
+  ApiDefs api_defs;
+  protobuf::TextFormat::ParseFromString(contents, &api_defs);
+  for (const auto& api_def : api_defs.op()) {
+    // Check if the op definition is already loaded.
+    if (map_.find(api_def.graph_op_name()) != map_.end()) {
+      // Overwrite current api def with data in api_def.
+      TF_RETURN_IF_ERROR(MergeApiDefs(&map_[api_def.graph_op_name()], api_def));
+    } else {
+      return errors::FailedPrecondition(
+          "Unexpected ApiDef override: ", api_def.graph_op_name(),
+          " is not defined in base ApiDef.");
+    }
+  }
+  return Status::OK();
+}
+
+const tensorflow::ApiDef* ApiDefMap::GetApiDef(const string& name) const {
+  return gtl::FindOrNull(map_, name);
+}
 }  // namespace tensorflow
diff --git a/tensorflow/core/framework/op_gen_lib.h b/tensorflow/core/framework/op_gen_lib.h
index dbe0a8e190..efb287477b 100644
--- a/tensorflow/core/framework/op_gen_lib.h
+++ b/tensorflow/core/framework/op_gen_lib.h
@@ -18,6 +18,8 @@ limitations under the License.
 
 #include <string>
 #include <unordered_map>
+#include "tensorflow/core/framework/api_def.pb.h"
+#include "tensorflow/core/framework/op_def.pb.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/core/stringpiece.h"
 #include "tensorflow/core/platform/env.h"
@@ -74,6 +76,48 @@ class OpGenOverrideMap {
   std::unordered_map<string, std::unique_ptr<OpGenOverride>> map_;
 };
 
+// Takes a list of files with ApiDefs text protos, and allows you to
+// look up the specific ApiDef for any given op.
+class ApiDefMap {
+ public:
+  // OpList must be a superset of ops of any subsequently loaded
+  // ApiDef.
+  explicit ApiDefMap(const OpList& op_list);
+  ~ApiDefMap();
+
+  // You can call this method multiple times to load multiple
+  // sets of files. Api definitions are merged if the same
+  // op definition is loaded multiple times. Later-loaded
+  // definitions take precedense.
+  // ApiDefs loaded from files must contain a subset of ops defined
+  // in the OpList passed to the constructor.
+  Status LoadFileList(Env* env, const std::vector<string>& filenames);
+
+  // Load a single file. Api definitions are merged if the same
+  // op definition is loaded multiple times. Later-loaded
+  // definitions take precedense.
+  // ApiDefs loaded from file must contain a subset of ops defined
+  // in the OpList passed to the constructor.
+  Status LoadFile(Env* env, const string& filename);
+
+  // Load ApiDefs from string containing ApiDefs text proto.
+  // api_def_file_contents is expected to be in "multiline format".
+  // ApiDefs must contain a subset of ops defined in OpsList
+  // passed to the constructor.
+  Status LoadApiDef(const string& api_def_file_contents);
+
+  // Look up ApiDef proto based on the given graph op name.
+  // If graph op name is not in this ApiDefMap, returns nullptr.
+  //
+  // Note: Returned ApiDef pointer should stay valid even after calling
+  // Load* functions defined above. Subsequent calls to Load* might modify
+  // returned ApiDef contents, but should never remove the ApiDef itself.
+  const ApiDef* GetApiDef(const string& name) const;
+
+ private:
+  std::unordered_map<string, ApiDef> map_;
+};
+
 }  // namespace tensorflow
 
 #endif  // TENSORFLOW_FRAMEWORK_OP_GEN_LIB_H_
diff --git a/tensorflow/core/framework/op_gen_lib_test.cc b/tensorflow/core/framework/op_gen_lib_test.cc
index cc1d117f38..b7ee6db991 100644
--- a/tensorflow/core/framework/op_gen_lib_test.cc
+++ b/tensorflow/core/framework/op_gen_lib_test.cc
@@ -15,11 +15,60 @@ limitations under the License.
 
 #include "tensorflow/core/framework/op_gen_lib.h"
 
+#include "tensorflow/core/framework/op_def.pb.h"
 #include "tensorflow/core/platform/test.h"
 
 namespace tensorflow {
 namespace {
 
+constexpr char kTestOpList[] = R"(op {
+  name: "testop"
+  input_arg {
+    name: "arg_a"
+  }
+  input_arg {
+    name: "arg_b"
+  }
+  output_arg {
+    name: "arg_c"
+  }
+  attr {
+    name: "attr_a"
+  }
+  deprecation {
+    version: 123
+    explanation: "foo"
+  }
+)";
+
+constexpr char kTestApiDef[] = R"(op {
+  graph_op_name: "testop"
+  visibility: VISIBLE
+  endpoint {
+    name: "testop1"
+  }
+  in_arg {
+    name: "arg_a"
+  }
+  in_arg {
+    name: "arg_b"
+  }
+  out_arg {
+    name: "arg_c"
+  }
+  attr {
+    name: "attr_a"
+  }
+  summary: "Mock op for testing."
+  description: <<END
+Description for the
+testop.
+END
+  arg_order: "arg_a"
+  arg_order: "arg_b"
+}
+)";
+
 TEST(OpGenLibTest, MultilinePBTxt) {
   // Non-multiline pbtxt
   const string pbtxt = R"(foo: "abc"
@@ -127,5 +176,240 @@ END  # Comment 2
   EXPECT_EQ(pbtxt, PBTxtFromMultiline(ml));
 }
 
+TEST(OpGenLibTest, ApiDefAccessInvalidName) {
+  OpList op_list;
+  protobuf::TextFormat::ParseFromString(kTestOpList, &op_list);  // NOLINT
+
+  ApiDefMap api_map(op_list);
+  ASSERT_EQ(nullptr, api_map.GetApiDef("testop5"));
+}
+
+TEST(OpGenLibTest, ApiDefInitializedFromOpDef) {
+  const string expected_api_def = R"(graph_op_name: "testop"
+visibility: VISIBLE
+endpoint {
+  name: "testop"
+  deprecation_version: 123
+}
+in_arg {
+  name: "arg_a"
+  rename_to: "arg_a"
+}
+in_arg {
+  name: "arg_b"
+  rename_to: "arg_b"
+}
+out_arg {
+  name: "arg_c"
+  rename_to: "arg_c"
+}
+attr {
+  name: "attr_a"
+  rename_to: "attr_a"
+}
+)";
+  OpList op_list;
+  protobuf::TextFormat::ParseFromString(kTestOpList, &op_list);  // NOLINT
+
+  ApiDefMap api_map(op_list);
+  const auto* api_def = api_map.GetApiDef("testop");
+  ASSERT_EQ(expected_api_def, api_def->DebugString());
+}
+
+TEST(OpGenLibTest, ApiDefLoadSingleApiDef) {
+  const string expected_api_def = R"(op {
+  graph_op_name: "testop"
+  visibility: VISIBLE
+  endpoint {
+    name: "testop1"
+  }
+  in_arg {
+    name: "arg_a"
+    rename_to: "arg_a"
+  }
+  in_arg {
+    name: "arg_b"
+    rename_to: "arg_b"
+  }
+  out_arg {
+    name: "arg_c"
+    rename_to: "arg_c"
+  }
+  attr {
+    name: "attr_a"
+    rename_to: "attr_a"
+  }
+  summary: "Mock op for testing."
+  description: "Description for the\ntestop."
+  arg_order: "arg_a"
+  arg_order: "arg_b"
+}
+)";
+  OpList op_list;
+  protobuf::TextFormat::ParseFromString(kTestOpList, &op_list);  // NOLINT
+
+  ApiDefMap api_map(op_list);
+  TF_CHECK_OK(api_map.LoadApiDef(kTestApiDef));
+  const auto* api_def = api_map.GetApiDef("testop");
+  EXPECT_EQ(1, api_def->endpoint_size());
+  EXPECT_EQ("testop1", api_def->endpoint(0).name());
+
+  ApiDefs api_defs;
+  *api_defs.add_op() = *api_def;
+  EXPECT_EQ(expected_api_def, api_defs.DebugString());
+}
+
+TEST(OpGenLibTest, ApiDefOverrideVisibility) {
+  const string api_def1 = R"(
+op {
+  graph_op_name: "testop"
+  endpoint {
+    name: "testop2"
+  }
+}
+)";
+  const string api_def2 = R"(
+op {
+  graph_op_name: "testop"
+  visibility: HIDDEN
+  endpoint {
+    name: "testop2"
+  }
+}
+)";
+  OpList op_list;
+  protobuf::TextFormat::ParseFromString(kTestOpList, &op_list);  // NOLINT
+
+  ApiDefMap api_map(op_list);
+  TF_CHECK_OK(api_map.LoadApiDef(kTestApiDef));
+  auto* api_def = api_map.GetApiDef("testop");
+  EXPECT_EQ(ApiDef::VISIBLE, api_def->visibility());
+
+  // Loading ApiDef with default visibility should
+  // keep current visibility.
+  TF_CHECK_OK(api_map.LoadApiDef(api_def1));
+  EXPECT_EQ(ApiDef::VISIBLE, api_def->visibility());
+
+  // Loading ApiDef with non-default visibility,
+  // should update visibility.
+  TF_CHECK_OK(api_map.LoadApiDef(api_def2));
+  EXPECT_EQ(ApiDef::HIDDEN, api_def->visibility());
+}
+
+TEST(OpGenLibTest, ApiDefOverrideEndpoints) {
+  const string api_def1 = R"(
+op {
+  graph_op_name: "testop"
+  endpoint {
+    name: "testop2"
+  }
+}
+)";
+  OpList op_list;
+  protobuf::TextFormat::ParseFromString(kTestOpList, &op_list);  // NOLINT
+
+  ApiDefMap api_map(op_list);
+  TF_CHECK_OK(api_map.LoadApiDef(kTestApiDef));
+  auto* api_def = api_map.GetApiDef("testop");
+  ASSERT_EQ(1, api_def->endpoint_size());
+  EXPECT_EQ("testop1", api_def->endpoint(0).name());
+
+  TF_CHECK_OK(api_map.LoadApiDef(api_def1));
+  ASSERT_EQ(1, api_def->endpoint_size());
+  EXPECT_EQ("testop2", api_def->endpoint(0).name());
+}
+
+TEST(OpGenLibTest, ApiDefOverrideArgs) {
+  const string api_def1 = R"(
+op {
+  graph_op_name: "testop"
+  in_arg {
+    name: "arg_a"
+    rename_to: "arg_aa"
+  }
+  out_arg {
+    name: "arg_c"
+    rename_to: "arg_cc"
+  }
+  arg_order: "arg_aa"
+  arg_order: "arg_b"
+}
+)";
+  OpList op_list;
+  protobuf::TextFormat::ParseFromString(kTestOpList, &op_list);  // NOLINT
+
+  ApiDefMap api_map(op_list);
+  TF_CHECK_OK(api_map.LoadApiDef(kTestApiDef));
+  TF_CHECK_OK(api_map.LoadApiDef(api_def1));
+  const auto* api_def = api_map.GetApiDef("testop");
+  ASSERT_EQ(2, api_def->in_arg_size());
+  EXPECT_EQ("arg_aa", api_def->in_arg(0).rename_to());
+  // 2nd in_arg is not renamed
+  EXPECT_EQ("arg_b", api_def->in_arg(1).rename_to());
+
+  ASSERT_EQ(1, api_def->out_arg_size());
+  EXPECT_EQ("arg_cc", api_def->out_arg(0).rename_to());
+
+  ASSERT_EQ(2, api_def->arg_order_size());
+  EXPECT_EQ("arg_aa", api_def->arg_order(0));
+  EXPECT_EQ("arg_b", api_def->arg_order(1));
+}
+
+TEST(OpGenLibTest, ApiDefOverrideDescriptions) {
+  const string api_def1 = R"(
+op {
+  graph_op_name: "testop"
+  summary: "New summary"
+  description: <<END
+New description
+END
+  description_prefix: "A"
+  description_suffix: "Z"
+}
+)";
+
+  const string api_def2 = R"(
+op {
+  graph_op_name: "testop"
+  description_prefix: "B"
+  description_suffix: "Y"
+}
+)";
+
+  OpList op_list;
+  protobuf::TextFormat::ParseFromString(kTestOpList, &op_list);  // NOLINT
+
+  ApiDefMap api_map(op_list);
+  TF_CHECK_OK(api_map.LoadApiDef(kTestApiDef));
+  TF_CHECK_OK(api_map.LoadApiDef(api_def1));
+  const auto* api_def = api_map.GetApiDef("testop");
+  EXPECT_EQ("New summary", api_def->summary());
+  EXPECT_EQ("A\nNew description\nZ", api_def->description());
+  EXPECT_EQ("", api_def->description_prefix());
+  EXPECT_EQ("", api_def->description_suffix());
+
+  TF_CHECK_OK(api_map.LoadApiDef(api_def2));
+  EXPECT_EQ("B\nA\nNew description\nZ\nY", api_def->description());
+  EXPECT_EQ("", api_def->description_prefix());
+  EXPECT_EQ("", api_def->description_suffix());
+}
+
+TEST(OpGenLibTest, ApiDefInvalidOpInOverride) {
+  const string api_def1 = R"(
+op {
+  graph_op_name: "different_testop"
+  endpoint {
+    name: "testop2"
+  }
+}
+)";
+  OpList op_list;
+  protobuf::TextFormat::ParseFromString(kTestOpList, &op_list);  // NOLINT
+
+  ApiDefMap api_map(op_list);
+  TF_CHECK_OK(api_map.LoadApiDef(kTestApiDef));
+  auto status = api_map.LoadApiDef(api_def1);
+  ASSERT_EQ(tensorflow::error::FAILED_PRECONDITION, status.code());
+}
 }  // namespace
 }  // namespace tensorflow
-- 
GitLab


From 274786ab9b825b18dc9a7dd5eb3f312ec6cb92f9 Mon Sep 17 00:00:00 2001
From: Kay Zhu <kayzhu@google.com>
Date: Wed, 18 Oct 2017 17:58:50 -0700
Subject: [PATCH 544/909] [XLA] Run HLO verifier for hlo_evaluator_test and fix
 one shape mismatch in DoesConcateSimple test case.

PiperOrigin-RevId: 172684383
---
 tensorflow/compiler/xla/service/BUILD         |   2 +-
 .../xla/service/hlo_evaluator_test.cc         | 302 ++++++++----------
 2 files changed, 129 insertions(+), 175 deletions(-)

diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD
index fed7bd01f6..1ef329365e 100644
--- a/tensorflow/compiler/xla/service/BUILD
+++ b/tensorflow/compiler/xla/service/BUILD
@@ -117,7 +117,7 @@ tf_cc_test(
         "//tensorflow/compiler/xla:util",
         "//tensorflow/compiler/xla:xla_data_proto",
         "//tensorflow/compiler/xla/client:computation_builder",
-        "//tensorflow/compiler/xla/tests:hlo_test_base",
+        "//tensorflow/compiler/xla/tests:hlo_verified_test_base",
         "//tensorflow/compiler/xla/tests:literal_test_util",
         "//tensorflow/compiler/xla/tests:xla_internal_test_main",  # fixdeps: keep
         "//tensorflow/core:lib",
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator_test.cc b/tensorflow/compiler/xla/service/hlo_evaluator_test.cc
index 5172739624..85477af6fe 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_evaluator_test.cc
@@ -30,7 +30,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/status_macros.h"
 #include "tensorflow/compiler/xla/statusor.h"
 #include "tensorflow/compiler/xla/test.h"
-#include "tensorflow/compiler/xla/tests/hlo_test_base.h"
+#include "tensorflow/compiler/xla/tests/hlo_verified_test_base.h"
 #include "tensorflow/compiler/xla/tests/literal_test_util.h"
 #include "tensorflow/compiler/xla/types.h"
 #include "tensorflow/compiler/xla/util.h"
@@ -41,7 +41,7 @@ limitations under the License.
 namespace xla {
 namespace {
 
-class HloEvaluatorTest : public HloTestBase {
+class HloEvaluatorTest : public HloVerifiedTestBase {
  protected:
   HloEvaluatorTest() { evaluator_ = MakeUnique<HloEvaluator>(); }
 
@@ -62,8 +62,7 @@ TEST_F(HloEvaluatorTest, DoesClamp) {
   auto c3 = b.AddInstruction(HloInstruction::CreateConstant(std::move(value)));
   auto instruction = b.AddInstruction(
       HloInstruction::CreateTernary(shape, HloOpcode::kClamp, c1, c2, c3));
-  HloModule module(TestName());
-  module.AddEntryComputation(b.Build());
+  module().AddEntryComputation(b.Build());
 
   std::unique_ptr<Literal> result =
       evaluator_->Evaluate(instruction, {}).ConsumeValueOrDie();
@@ -89,8 +88,7 @@ TEST_F(HloEvaluatorTest, DoesSelect) {
       b.AddInstruction(HloInstruction::CreateConstant(std::move(on_false)));
   auto instruction = b.AddInstruction(
       HloInstruction::CreateTernary(shape, HloOpcode::kSelect, c1, c2, c3));
-  HloModule module(TestName());
-  module.AddEntryComputation(b.Build());
+  module().AddEntryComputation(b.Build());
 
   std::unique_ptr<Literal> result =
       evaluator_->Evaluate(instruction, {}).ConsumeValueOrDie();
@@ -112,8 +110,7 @@ TEST_F(HloEvaluatorTest, DoesAdd) {
   auto c2 = b.AddInstruction(HloInstruction::CreateConstant(std::move(rhs)));
   auto instruction = b.AddInstruction(
       HloInstruction::CreateBinary(shape, HloOpcode::kAdd, c1, c2));
-  HloModule module(TestName());
-  module.AddEntryComputation(b.Build());
+  module().AddEntryComputation(b.Build());
 
   std::unique_ptr<Literal> result =
       evaluator_->Evaluate(instruction, {}).ConsumeValueOrDie();
@@ -125,111 +122,100 @@ TEST_F(HloEvaluatorTest, DoesAdd) {
 
 // Verifies that HloEvaluator evaluates a HLO instruction that performs
 // element-wise divide with 2 operands.
-TEST_F(HloEvaluatorTest, DoesDivide) {
-  {
-    auto lhs_s64 = Literal::CreateR2<int64>({{1, 0}, {-100, 4}});
-    auto rhs_s64 = Literal::CreateR2<int64>({{2, 4}, {4, 4}});
-
-    Shape shape_s64 = ShapeUtil::MakeShape(S64, {2, 2});
-    HloComputation::Builder b(TestName());
-    auto c1_s64 =
-        b.AddInstruction(HloInstruction::CreateConstant(std::move(lhs_s64)));
-    auto c2_s64 =
-        b.AddInstruction(HloInstruction::CreateConstant(std::move(rhs_s64)));
-    auto instruction = b.AddInstruction(HloInstruction::CreateBinary(
-        shape_s64, HloOpcode::kDivide, c1_s64, c2_s64));
-    HloModule module(TestName());
-    module.AddEntryComputation(b.Build());
-
-    std::unique_ptr<Literal> result =
-        evaluator_->Evaluate(instruction, {}).ConsumeValueOrDie();
-
-    auto expected = Literal::CreateR2<int64>({{0, 0}, {-25, 1}});
-
-    LiteralTestUtil::ExpectEqual(*expected, *result);
-  }
-  {
-    auto lhs_f64 = Literal::CreateR2<double>({{1.0, 0.0}, {-100.0, 4.0}});
-    auto rhs_f64 = Literal::CreateR2<double>({{2.2, 4.0}, {4.0, 4.0}});
-
-    Shape shape_f64 = ShapeUtil::MakeShape(F64, {2, 2});
-    HloComputation::Builder b(TestName());
-    auto c1_f64 =
-        b.AddInstruction(HloInstruction::CreateConstant(std::move(lhs_f64)));
-    auto c2_f64 =
-        b.AddInstruction(HloInstruction::CreateConstant(std::move(rhs_f64)));
-    auto instruction = b.AddInstruction(HloInstruction::CreateBinary(
-        shape_f64, HloOpcode::kDivide, c1_f64, c2_f64));
-    HloModule module(TestName());
-    module.AddEntryComputation(b.Build());
-
-    auto result = evaluator_->Evaluate(instruction, {}).ConsumeValueOrDie();
-
-    auto expected =
-        Literal::CreateR2<double>({{0.45454545454545453, 0}, {-25, 1}});
-
-    LiteralTestUtil::ExpectEqual(*expected, *result);
-  }
+TEST_F(HloEvaluatorTest, DoesDivideInt64) {
+  auto lhs_s64 = Literal::CreateR2<int64>({{1, 0}, {-100, 4}});
+  auto rhs_s64 = Literal::CreateR2<int64>({{2, 4}, {4, 4}});
+
+  Shape shape_s64 = ShapeUtil::MakeShape(S64, {2, 2});
+  HloComputation::Builder b(TestName());
+  auto c1_s64 =
+      b.AddInstruction(HloInstruction::CreateConstant(std::move(lhs_s64)));
+  auto c2_s64 =
+      b.AddInstruction(HloInstruction::CreateConstant(std::move(rhs_s64)));
+  auto instruction = b.AddInstruction(HloInstruction::CreateBinary(
+      shape_s64, HloOpcode::kDivide, c1_s64, c2_s64));
+  module().AddEntryComputation(b.Build());
+
+  std::unique_ptr<Literal> result =
+      evaluator_->Evaluate(instruction, {}).ConsumeValueOrDie();
+
+  auto expected = Literal::CreateR2<int64>({{0, 0}, {-25, 1}});
+
+  LiteralTestUtil::ExpectEqual(*expected, *result);
+}
+TEST_F(HloEvaluatorTest, DoesDivideDouble) {
+  auto lhs_f64 = Literal::CreateR2<double>({{1.0, 0.0}, {-100.0, 4.0}});
+  auto rhs_f64 = Literal::CreateR2<double>({{2.2, 4.0}, {4.0, 4.0}});
+
+  Shape shape_f64 = ShapeUtil::MakeShape(F64, {2, 2});
+  HloComputation::Builder b(TestName());
+  auto c1_f64 =
+      b.AddInstruction(HloInstruction::CreateConstant(std::move(lhs_f64)));
+  auto c2_f64 =
+      b.AddInstruction(HloInstruction::CreateConstant(std::move(rhs_f64)));
+  auto instruction = b.AddInstruction(HloInstruction::CreateBinary(
+      shape_f64, HloOpcode::kDivide, c1_f64, c2_f64));
+  module().AddEntryComputation(b.Build());
+
+  auto result = evaluator_->Evaluate(instruction, {}).ConsumeValueOrDie();
+
+  auto expected =
+      Literal::CreateR2<double>({{0.45454545454545453, 0}, {-25, 1}});
+
+  LiteralTestUtil::ExpectEqual(*expected, *result);
 }
 
 // Verifies that HloEvaluator evaluates a HLO instruction that performs
 // element-wise abs op with 1 operand.
-TEST_F(HloEvaluatorTest, DoesAbs) {
-  {
-    auto operand = Literal::CreateR2<int64>({{1, -20}, {-100, 4}});
-    const Shape& shape = ShapeUtil::MakeShape(S64, {2, 2});
-    HloComputation::Builder b(TestName());
-    auto c1 =
-        b.AddInstruction(HloInstruction::CreateConstant(std::move(operand)));
-    auto instruction = b.AddInstruction(
-        HloInstruction::CreateUnary(shape, HloOpcode::kAbs, c1));
-    HloModule module(TestName());
-    module.AddEntryComputation(b.Build());
-
-    std::unique_ptr<Literal> result =
-        evaluator_->Evaluate(instruction, {}).ConsumeValueOrDie();
-
-    auto expected = Literal::CreateR2<int64>({{1, 20}, {100, 4}});
-
-    LiteralTestUtil::ExpectEqual(*expected, *result);
-  }
+TEST_F(HloEvaluatorTest, DoesAbsR2) {
+  auto operand = Literal::CreateR2<int64>({{1, -20}, {-100, 4}});
+  const Shape& shape = ShapeUtil::MakeShape(S64, {2, 2});
+  HloComputation::Builder b(TestName());
+  auto c1 =
+      b.AddInstruction(HloInstruction::CreateConstant(std::move(operand)));
+  auto instruction =
+      b.AddInstruction(HloInstruction::CreateUnary(shape, HloOpcode::kAbs, c1));
+  module().AddEntryComputation(b.Build());
 
+  std::unique_ptr<Literal> result =
+      evaluator_->Evaluate(instruction, {}).ConsumeValueOrDie();
+
+  auto expected = Literal::CreateR2<int64>({{1, 20}, {100, 4}});
+
+  LiteralTestUtil::ExpectEqual(*expected, *result);
+}
+TEST_F(HloEvaluatorTest, DoesAbsR0) {
   // For R0 literal.
-  {
-    const Shape& r0 = ShapeUtil::MakeShape(F32, {});
-    auto operand = Literal::CreateR0<float>(-1.0f);
-    HloComputation::Builder b(TestName());
-    auto c1 =
-        b.AddInstruction(HloInstruction::CreateConstant(std::move(operand)));
-    auto instruction =
-        b.AddInstruction(HloInstruction::CreateUnary(r0, HloOpcode::kAbs, c1));
-    HloModule module(TestName());
-    module.AddEntryComputation(b.Build());
-
-    auto result = evaluator_->Evaluate(instruction).ConsumeValueOrDie();
-    auto expected = Literal::CreateR0<float>(1.0f);
-
-    LiteralTestUtil::ExpectEqual(*expected, *result);
-  }
+  const Shape& r0 = ShapeUtil::MakeShape(F32, {});
+  auto operand = Literal::CreateR0<float>(-1.0f);
+  HloComputation::Builder b(TestName());
+  auto c1 =
+      b.AddInstruction(HloInstruction::CreateConstant(std::move(operand)));
+  auto instruction =
+      b.AddInstruction(HloInstruction::CreateUnary(r0, HloOpcode::kAbs, c1));
+  module().AddEntryComputation(b.Build());
+
+  auto result = evaluator_->Evaluate(instruction).ConsumeValueOrDie();
+  auto expected = Literal::CreateR0<float>(1.0f);
 
+  LiteralTestUtil::ExpectEqual(*expected, *result);
+}
+TEST_F(HloEvaluatorTest, DoesAbsR1WithZeroSize) {
   // For R1 literal with dimension of size 0.
-  {
-    Shape empty_r1 = ShapeUtil::MakeShape(F32, {0});
-    auto operand = Literal::CreateR1<float>({});
-    HloComputation::Builder b(TestName());
-    auto c1 =
-        b.AddInstruction(HloInstruction::CreateConstant(std::move(operand)));
-    auto instruction = b.AddInstruction(
-        HloInstruction::CreateUnary(empty_r1, HloOpcode::kAbs, c1));
-    HloModule module(TestName());
-    module.AddEntryComputation(b.Build());
-
-    auto result = evaluator_->Evaluate(instruction).ConsumeValueOrDie();
-    auto expected = Literal::CreateR1<float>({});
-
-    LiteralTestUtil::ExpectEqual(*expected, *result);
-  }
-}  // namespace
+  Shape empty_r1 = ShapeUtil::MakeShape(F32, {0});
+  auto operand = Literal::CreateR1<float>({});
+  HloComputation::Builder b(TestName());
+  auto c1 =
+      b.AddInstruction(HloInstruction::CreateConstant(std::move(operand)));
+  auto instruction = b.AddInstruction(
+      HloInstruction::CreateUnary(empty_r1, HloOpcode::kAbs, c1));
+  module().AddEntryComputation(b.Build());
+
+  auto result = evaluator_->Evaluate(instruction).ConsumeValueOrDie();
+  auto expected = Literal::CreateR1<float>({});
+
+  LiteralTestUtil::ExpectEqual(*expected, *result);
+}
 
 // Verifies that HloEvaluator evaluates a HLO Computation with non-parameter nor
 // constant operands.
@@ -253,8 +239,7 @@ TEST_F(HloEvaluatorTest, DoesTraverseInstructions) {
       b.AddInstruction(HloInstruction::CreateParameter(2, shape, "rhs2"));
   b.AddInstruction(HloInstruction::CreateBinary(shape, HloOpcode::kAdd,
                                                 lhs_instruction, param_rhs2));
-  HloModule module(TestName());
-  auto computation = module.AddEntryComputation(b.Build());
+  auto computation = module().AddEntryComputation(b.Build());
 
   std::unique_ptr<Literal> result =
       evaluator_->Evaluate(*computation, args).ConsumeValueOrDie();
@@ -279,8 +264,7 @@ TEST_F(HloEvaluatorTest, DoesReshape) {
   const int64 permutation[] = {1, 2, 0, 4, 3};
   b.AddInstruction(
       HloInstruction::CreateTranspose(shape, literal_instruction, permutation));
-  HloModule module(TestName());
-  auto computation = module.AddEntryComputation(b.Build());
+  auto computation = module().AddEntryComputation(b.Build());
 
   std::unique_ptr<Literal> result =
       evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie();
@@ -303,8 +287,7 @@ TEST_F(HloEvaluatorTest, DoesBroadcast) {
       HloInstruction::CreateConstant(std::move(input_literal)));
   b.AddInstruction(HloInstruction::CreateBroadcast(
       output_literal->shape(), literal_instruction, {1, 2}));
-  HloModule module(TestName());
-  auto computation = module.AddEntryComputation(b.Build());
+  auto computation = module().AddEntryComputation(b.Build());
 
   std::unique_ptr<Literal> result =
       evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie();
@@ -324,8 +307,7 @@ TEST_F(HloEvaluatorTest, DoesBroadcastScalar) {
   b.AddInstruction(HloInstruction::CreateBroadcast(
       output_literal->shape(), literal_instruction,
       /*broadcast_dimensions=*/{}));
-  HloModule module(TestName());
-  auto computation = module.AddEntryComputation(b.Build());
+  auto computation = module().AddEntryComputation(b.Build());
 
   std::unique_ptr<Literal> result =
       evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie();
@@ -343,11 +325,10 @@ TEST_F(HloEvaluatorTest, DoesConcatenateSimple) {
 
   std::vector<HloInstruction*> operands = {operand1, operand2};
 
-  Shape shape = ShapeUtil::MakeShape(S64, {2, 2});
+  Shape shape = ShapeUtil::MakeShape(S64, {4, 2});
   b.AddInstruction(HloInstruction::CreateConcatenate(shape, operands, 0));
 
-  HloModule module(TestName());
-  auto computation = module.AddEntryComputation(b.Build());
+  auto computation = module().AddEntryComputation(b.Build());
 
   std::unique_ptr<Literal> result =
       evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie();
@@ -370,8 +351,7 @@ TEST_F(HloEvaluatorTest, ConcatenateHandlesShapeWithZeroElement) {
   Shape shape = ShapeUtil::MakeShape(S64, {2});
   b.AddInstruction(HloInstruction::CreateConcatenate(shape, operands, 0));
 
-  HloModule module(TestName());
-  auto computation = module.AddEntryComputation(b.Build());
+  auto computation = module().AddEntryComputation(b.Build());
 
   std::unique_ptr<Literal> result =
       evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie();
@@ -392,8 +372,7 @@ TEST_F(HloEvaluatorTest, ConvertWithSameLayout) {
   HloInstruction* constant = b.AddInstruction(
       HloInstruction::CreateConstant(std::move(input_literal)));
   b.AddInstruction(HloInstruction::CreateConvert(expected->shape(), constant));
-  HloModule module(TestName());
-  auto computation = module.AddEntryComputation(b.Build());
+  auto computation = module().AddEntryComputation(b.Build());
 
   std::unique_ptr<Literal> result =
       evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie();
@@ -414,8 +393,7 @@ TEST_F(HloEvaluatorTest, ConvertWithDifferentLayout) {
   HloInstruction* constant = b.AddInstruction(
       HloInstruction::CreateConstant(std::move(input_literal)));
   b.AddInstruction(HloInstruction::CreateConvert(expected->shape(), constant));
-  HloModule module(TestName());
-  auto computation = module.AddEntryComputation(b.Build());
+  auto computation = module().AddEntryComputation(b.Build());
 
   std::unique_ptr<Literal> result =
       evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie();
@@ -451,8 +429,7 @@ TEST_F(HloEvaluatorTest, Pad2DIntegerArrayWithZeroDimension) {
   Shape shape = ShapeUtil::MakeShape(S32, {5, 2});
   auto pad_instruction = b.AddInstruction(HloInstruction::CreatePad(
       shape, operand_instruction, padding_value_instruction, padding_config));
-  HloModule module(TestName());
-  module.AddEntryComputation(b.Build());
+  module().AddEntryComputation(b.Build());
 
   auto result = evaluator_->Evaluate(pad_instruction).ConsumeValueOrDie();
 
@@ -479,8 +456,7 @@ TEST_F(HloEvaluatorTest, Pad4DFloatArrayWithInteriorPadding) {
       CreatePaddingConfig({{{1, 0, 2}}, {{0, 2, 1}}, {{0, 0, 0}}, {{0, 0, 0}}});
   b.AddInstruction(HloInstruction::CreatePad(
       shape, input_instruction, pad_instruction, r4_padding_on_dim0_dim1));
-  HloModule module(TestName());
-  auto computation = module.AddEntryComputation(b.Build());
+  auto computation = module().AddEntryComputation(b.Build());
 
   std::unique_ptr<Literal> result =
       evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie();
@@ -525,8 +501,7 @@ TEST_F(HloEvaluatorTest, NegativePadding2D) {
                                              pad_value_instruction,
                                              r2_padding_on_dim0_dim1));
 
-  HloModule module(TestName());
-  auto computation = module.AddEntryComputation(b.Build());
+  auto computation = module().AddEntryComputation(b.Build());
 
   std::unique_ptr<Literal> result =
       evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie();
@@ -572,8 +547,7 @@ TEST_F(HloEvaluatorTest, NegativeAndInteriorPadding2D) {
                                              pad_value_instruction,
                                              r2_padding_on_dim0_dim1));
 
-  HloModule module(TestName());
-  auto computation = module.AddEntryComputation(b.Build());
+  auto computation = module().AddEntryComputation(b.Build());
 
   std::unique_ptr<Literal> result =
       evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie();
@@ -609,8 +583,7 @@ TEST_F(HloEvaluatorTest, DotRank2AndRank1) {
   Shape shape = ShapeUtil::MakeShape(F32, {4, 2});
   b.AddInstruction(HloInstruction::CreateBinary(
       shape, HloOpcode::kDot, lhs_instruction, rhs_instruction));
-  HloModule module(TestName());
-  auto computation = module.AddEntryComputation(b.Build());
+  auto computation = module().AddEntryComputation(b.Build());
 
   std::unique_ptr<Literal> result =
       evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie();
@@ -653,8 +626,7 @@ TEST_F(HloEvaluatorTest, DotRank1AndRank2) {
   Shape shape = ShapeUtil::MakeShape(F32, {2});
   b.AddInstruction(HloInstruction::CreateBinary(
       shape, HloOpcode::kDot, lhs_instruction, rhs_instruction));
-  HloModule module(TestName());
-  auto computation = module.AddEntryComputation(b.Build());
+  auto computation = module().AddEntryComputation(b.Build());
 
   std::unique_ptr<Literal> result =
       evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie();
@@ -695,8 +667,7 @@ TEST_F(HloEvaluatorTest, DotRank2AndRank2) {
   Shape shape = ShapeUtil::MakeShape(F32, {4, 2});
   b.AddInstruction(HloInstruction::CreateBinary(
       shape, HloOpcode::kDot, lhs_instruction, rhs_instruction));
-  HloModule module(TestName());
-  auto computation = module.AddEntryComputation(b.Build());
+  auto computation = module().AddEntryComputation(b.Build());
 
   std::unique_ptr<Literal> result =
       evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie();
@@ -749,8 +720,7 @@ TEST_F(HloEvaluatorTest, SimpleConv1D) {
   const Shape& shape = ShapeUtil::MakeShape(F32, {1, 1, 3});
   b.AddInstruction(HloInstruction::CreateConvolve(
       shape, lhs_instruction, rhs_instruction, window, dnums));
-  HloModule module(TestName());
-  auto computation = module.AddEntryComputation(b.Build());
+  auto computation = module().AddEntryComputation(b.Build());
 
   std::unique_ptr<Literal> result =
       evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie();
@@ -805,8 +775,7 @@ TEST_F(HloEvaluatorTest, Simple4x4Conv2DWith2x2Kernel) {
   const Shape& shape = ShapeUtil::MakeShape(F32, {1, 1, 4, 4});
   b.AddInstruction(HloInstruction::CreateConvolve(
       shape, lhs_instruction, rhs_instruction, window, dnums));
-  HloModule module(TestName());
-  auto computation = module.AddEntryComputation(b.Build());
+  auto computation = module().AddEntryComputation(b.Build());
 
   std::unique_ptr<Literal> result =
       evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie();
@@ -885,8 +854,7 @@ TEST_F(HloEvaluatorTest, Conv2DGeneralDimensions) {
   const Shape& shape = ShapeUtil::MakeShape(F32, {1, 1, 1, 2});
   b.AddInstruction(HloInstruction::CreateConvolve(
       shape, lhs_instruction, rhs_instruction, window, dnums));
-  HloModule module(TestName());
-  auto computation = module.AddEntryComputation(b.Build());
+  auto computation = module().AddEntryComputation(b.Build());
 
   std::unique_ptr<Literal> result =
       evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie();
@@ -944,8 +912,7 @@ TEST_F(HloEvaluatorTest, DilatedBaseConv2DWithHighPadding) {
   const Shape& shape = ShapeUtil::MakeShape(F32, {1, 1, 7, 7});
   b.AddInstruction(HloInstruction::CreateConvolve(
       shape, lhs_instruction, rhs_instruction, window, dnums));
-  HloModule module(TestName());
-  auto computation = module.AddEntryComputation(b.Build());
+  auto computation = module().AddEntryComputation(b.Build());
 
   std::unique_ptr<Literal> result =
       evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie();
@@ -1009,8 +976,7 @@ TEST_F(HloEvaluatorTest, DilatedBaseConv2DWithLowAndHighPadding) {
   const Shape& shape = ShapeUtil::MakeShape(F32, {1, 1, 8, 8});
   b.AddInstruction(HloInstruction::CreateConvolve(
       shape, lhs_instruction, rhs_instruction, window, dnums));
-  HloModule module(TestName());
-  auto computation = module.AddEntryComputation(b.Build());
+  auto computation = module().AddEntryComputation(b.Build());
 
   std::unique_ptr<Literal> result =
       evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie();
@@ -1082,8 +1048,7 @@ TEST_F(HloEvaluatorTest,
   const Shape& shape = ShapeUtil::MakeShape(F32, {1, 1, 9, 3});
   b.AddInstruction(HloInstruction::CreateConvolve(
       shape, lhs_instruction, rhs_instruction, window, dnums));
-  HloModule module(TestName());
-  auto computation = module.AddEntryComputation(b.Build());
+  auto computation = module().AddEntryComputation(b.Build());
 
   std::unique_ptr<Literal> result =
       evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie();
@@ -1131,15 +1096,14 @@ TEST_F(HloEvaluatorTest, ReduceAdd) {
       HloInstruction::CreateParameter(1, scalar_shape, "rhs"));
   add_computation.AddInstruction(HloInstruction::CreateBinary(
       scalar_shape, HloOpcode::kAdd, param_lhs, param_rhs));
-  HloModule module(TestName());
-  auto add_func = module.AddEmbeddedComputation(add_computation.Build());
+  auto add_func = module().AddEmbeddedComputation(add_computation.Build());
 
   Shape shape = ShapeUtil::MakeShape(F32, {2});
   b.AddInstruction(
       HloInstruction::CreateReduce(shape, arg_instruction, init_value,
                                    /*dimensions_to_reduce=*/{1}, add_func));
 
-  auto computation = module.AddEntryComputation(b.Build());
+  auto computation = module().AddEntryComputation(b.Build());
 
   std::unique_ptr<Literal> result =
       evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie();
@@ -1175,8 +1139,7 @@ TEST_F(HloEvaluatorTest, ReduceWindowMax) {
       HloInstruction::CreateParameter(1, scalar_shape, "rhs"));
   max_computation.AddInstruction(HloInstruction::CreateBinary(
       scalar_shape, HloOpcode::kMaximum, param_lhs, param_rhs));
-  HloModule module(TestName());
-  auto max_func = module.AddEmbeddedComputation(max_computation.Build());
+  auto max_func = module().AddEmbeddedComputation(max_computation.Build());
 
   Window window;
   WindowDimension dim;
@@ -1193,7 +1156,7 @@ TEST_F(HloEvaluatorTest, ReduceWindowMax) {
   b.AddInstruction(HloInstruction::CreateReduceWindow(
       shape, arg_instruction, init_value, window, max_func));
 
-  auto computation = module.AddEntryComputation(b.Build());
+  auto computation = module().AddEntryComputation(b.Build());
   std::unique_ptr<Literal> result =
       evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie();
 
@@ -1227,8 +1190,7 @@ TEST_F(HloEvaluatorTest, ReduceWindowAdd) {
       HloInstruction::CreateParameter(1, scalar_shape, "rhs"));
   add_computation.AddInstruction(HloInstruction::CreateBinary(
       scalar_shape, HloOpcode::kAdd, param_lhs, param_rhs));
-  HloModule module(TestName());
-  auto add_func = module.AddEmbeddedComputation(add_computation.Build());
+  auto add_func = module().AddEmbeddedComputation(add_computation.Build());
 
   Window window;
   WindowDimension dim;
@@ -1251,7 +1213,7 @@ TEST_F(HloEvaluatorTest, ReduceWindowAdd) {
   b.AddInstruction(HloInstruction::CreateReduceWindow(
       shape, arg_instruction, init_value, window, add_func));
 
-  auto computation = module.AddEntryComputation(b.Build());
+  auto computation = module().AddEntryComputation(b.Build());
   std::unique_ptr<Literal> result =
       evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie();
 
@@ -1281,8 +1243,7 @@ TEST_F(HloEvaluatorTest, ReduceWindowAdd6D) {
       HloInstruction::CreateParameter(1, scalar_shape, "rhs"));
   add_computation.AddInstruction(HloInstruction::CreateBinary(
       scalar_shape, HloOpcode::kAdd, param_lhs, param_rhs));
-  HloModule module(TestName());
-  auto add_func = module.AddEmbeddedComputation(add_computation.Build());
+  auto add_func = module().AddEmbeddedComputation(add_computation.Build());
 
   Window window;
 
@@ -1313,7 +1274,7 @@ TEST_F(HloEvaluatorTest, ReduceWindowAdd6D) {
   b.AddInstruction(HloInstruction::CreateReduceWindow(
       shape, arg_instruction, init_value, window, add_func));
 
-  auto computation = module.AddEntryComputation(b.Build());
+  auto computation = module().AddEntryComputation(b.Build());
   std::unique_ptr<Literal> result =
       evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie();
 
@@ -1344,8 +1305,7 @@ TEST_F(HloEvaluatorTest, StridedSlice) {
                                                /*start_indices=*/{0, 2},
                                                /*limit_indices=*/{3, 5},
                                                /*strides=*/{2, 3}));
-  HloModule module(TestName());
-  auto computation = module.AddEntryComputation(b.Build());
+  auto computation = module().AddEntryComputation(b.Build());
 
   std::unique_ptr<Literal> result =
       evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie();
@@ -1379,8 +1339,7 @@ TEST_F(HloEvaluatorTest, DynamicSlice) {
   Shape shape = ShapeUtil::MakeShape(F32, {2, 3});
   b.AddInstruction(HloInstruction::CreateDynamicSlice(shape, operand,
                                                       start_indices, {2, 3}));
-  HloModule module(TestName());
-  auto computation = module.AddEntryComputation(b.Build());
+  auto computation = module().AddEntryComputation(b.Build());
 
   std::unique_ptr<Literal> result =
       evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie();
@@ -1416,8 +1375,7 @@ TEST_F(HloEvaluatorTest, DynamicSliceModSlice) {
   Shape shape = ShapeUtil::MakeShape(F32, {2, 3});
   b.AddInstruction(HloInstruction::CreateDynamicSlice(shape, operand,
                                                       start_indices, {2, 3}));
-  HloModule module(TestName());
-  auto computation = module.AddEntryComputation(b.Build());
+  auto computation = module().AddEntryComputation(b.Build());
 
   std::unique_ptr<Literal> result =
       evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie();
@@ -1454,8 +1412,7 @@ TEST_F(HloEvaluatorTest, DynamicSliceUpdate) {
   Shape shape = ShapeUtil::MakeShape(F64, {2, 3});
   b.AddInstruction(HloInstruction::CreateDynamicUpdateSlice(
       shape, operand, update, start_indices));
-  HloModule module(TestName());
-  auto computation = module.AddEntryComputation(b.Build());
+  auto computation = module().AddEntryComputation(b.Build());
 
   std::unique_ptr<Literal> result =
       evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie();
@@ -1491,8 +1448,7 @@ TEST_F(HloEvaluatorTest, SetAndGetTuples) {
   Shape shape = ShapeUtil::MakeShape(F64, {2, 3});
   b.AddInstruction(HloInstruction::CreateGetTupleElement(shape, tuple, 1));
 
-  HloModule module(TestName());
-  auto computation = module.AddEntryComputation(b.Build());
+  auto computation = module().AddEntryComputation(b.Build());
   std::unique_ptr<Literal> result =
       evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie();
 
@@ -1531,8 +1487,7 @@ TEST_F(HloEvaluatorTest, SetAndGetNestedTuples) {
   b.AddInstruction(
       HloInstruction::CreateGetTupleElement(tuple2->shape(), outer_tuple, 1));
 
-  HloModule module(TestName());
-  auto computation = module.AddEntryComputation(b.Build());
+  auto computation = module().AddEntryComputation(b.Build());
   std::unique_ptr<Literal> result =
       evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie();
 
@@ -1572,8 +1527,7 @@ TEST_F(HloEvaluatorTest, Reverse) {
 
   const Shape shape = ShapeUtil::MakeShape(F32, {4, 3, 2, 1});
   b.AddInstruction(HloInstruction::CreateReverse(shape, operand, {0, 1}));
-  HloModule module(TestName());
-  auto computation = module.AddEntryComputation(b.Build());
+  auto computation = module().AddEntryComputation(b.Build());
 
   std::unique_ptr<Literal> result =
       evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie();
-- 
GitLab


From 6c297fa9d5a0add0e38aceaceb57b0c6d83e0aca Mon Sep 17 00:00:00 2001
From: Benoit Steiner <bsteiner@google.com>
Date: Wed, 18 Oct 2017 17:59:33 -0700
Subject: [PATCH 545/909] Disable constant folding when propagating shapes
 through functions if requested.

PiperOrigin-RevId: 172684434
---
 tensorflow/core/common_runtime/shape_refiner.cc | 1 +
 tensorflow/core/common_runtime/shape_refiner.h  | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/common_runtime/shape_refiner.cc b/tensorflow/core/common_runtime/shape_refiner.cc
index 2a0bdc9a7b..1ed5eb3f22 100644
--- a/tensorflow/core/common_runtime/shape_refiner.cc
+++ b/tensorflow/core/common_runtime/shape_refiner.cc
@@ -148,6 +148,7 @@ Status ShapeRefiner::InferShapesForFunction(
   }
 
   ShapeRefiner refiner(graph.versions().producer(), &function_library);
+  refiner.set_disable_constant_propagation(disable_constant_propagation_);
   refiner.set_function_library_for_shape_inference(&function_library);
   if (keep_nested_shapes) refiner.set_keep_nested_shape_inferences();
 
diff --git a/tensorflow/core/common_runtime/shape_refiner.h b/tensorflow/core/common_runtime/shape_refiner.h
index bf4c6d8891..d1288d671e 100644
--- a/tensorflow/core/common_runtime/shape_refiner.h
+++ b/tensorflow/core/common_runtime/shape_refiner.h
@@ -206,7 +206,7 @@ class ShapeRefiner {
   // - outer_context will contain output shapes inferred from input shapes
   // - outer_context will contain nested inferences collection, iff
   //   keep_nested_shapes is true
-  static Status InferShapesForFunction(
+  Status InferShapesForFunction(
       const tensorflow::FunctionLibraryDefinition& function_library,
       const tensorflow::FunctionDef& function_def, bool keep_nested_shapes,
       ExtendedInferenceContext* outer_context);
-- 
GitLab


From b8f8a3d3660c75a4034bbe69a766d481638a6a4e Mon Sep 17 00:00:00 2001
From: Igor Saprykin <isaprykin@google.com>
Date: Wed, 18 Oct 2017 20:28:27 -0700
Subject: [PATCH 546/909] Fix the build file and a memory issue for
 text_classification_character_cnn.py.

PiperOrigin-RevId: 172695522
---
 tensorflow/BUILD                                      |  2 +-
 .../learn/text_classification_character_cnn.py        | 11 ++++-------
 2 files changed, 5 insertions(+), 8 deletions(-)

diff --git a/tensorflow/BUILD b/tensorflow/BUILD
index 93646ad650..e351037abb 100644
--- a/tensorflow/BUILD
+++ b/tensorflow/BUILD
@@ -123,7 +123,7 @@ config_setting(
 config_setting(
     name = "ios_x86_64",
     values = {
-        "cc_target_os": "apple",
+        "crosstool_top": "//tools/osx/crosstool:crosstool",
         "cpu": "ios_x86_64",
     },
     visibility = ["//visibility:public"],
diff --git a/tensorflow/examples/learn/text_classification_character_cnn.py b/tensorflow/examples/learn/text_classification_character_cnn.py
index 5f7c8e7371..363ff00362 100644
--- a/tensorflow/examples/learn/text_classification_character_cnn.py
+++ b/tensorflow/examples/learn/text_classification_character_cnn.py
@@ -30,7 +30,6 @@ import sys
 
 import numpy as np
 import pandas
-from sklearn import metrics
 import tensorflow as tf
 
 FLAGS = None
@@ -106,6 +105,8 @@ def char_cnn_model(features, labels, mode):
 
 
 def main(unused_argv):
+  tf.logging.set_verbosity(tf.logging.INFO)
+
   # Prepare training and testing data
   dbpedia = tf.contrib.learn.datasets.load_dataset(
       'dbpedia', test_with_fake_data=FLAGS.test_with_fake_data, size='large')
@@ -130,7 +131,7 @@ def main(unused_argv):
   train_input_fn = tf.estimator.inputs.numpy_input_fn(
       x={CHARS_FEATURE: x_train},
       y=y_train,
-      batch_size=len(x_train),
+      batch_size=128,
       num_epochs=None,
       shuffle=True)
   classifier.train(input_fn=train_input_fn, steps=100)
@@ -145,13 +146,9 @@ def main(unused_argv):
   y_predicted = np.array(list(p['class'] for p in predictions))
   y_predicted = y_predicted.reshape(np.array(y_test).shape)
 
-  # Score with sklearn.
-  score = metrics.accuracy_score(y_test, y_predicted)
-  print('Accuracy (sklearn): {0:f}'.format(score))
-
   # Score with tensorflow.
   scores = classifier.evaluate(input_fn=test_input_fn)
-  print('Accuracy (tensorflow): {0:f}'.format(scores['accuracy']))
+  print('Accuracy: {0:f}'.format(scores['accuracy']))
 
 
 if __name__ == '__main__':
-- 
GitLab


From 76ee352ccdb6927ca961c100b21b31f4431134b0 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 18 Oct 2017 20:30:20 -0700
Subject: [PATCH 547/909] Multi-minibatch support for
 tf.contrib.kfac.fisher_blocks.FullyConnectedDiagonalFB.

PiperOrigin-RevId: 172695625
---
 .../python/kernel_tests/fisher_blocks_test.py | 146 +++++++++++++++++
 .../kernel_tests/layer_collection_test.py     |   7 +-
 .../contrib/kfac/python/ops/fisher_blocks.py  | 150 +++++++++++++++---
 .../contrib/kfac/python/ops/fisher_factors.py |  19 ++-
 .../kfac/python/ops/layer_collection.py       |   6 +-
 5 files changed, 298 insertions(+), 30 deletions(-)

diff --git a/tensorflow/contrib/kfac/python/kernel_tests/fisher_blocks_test.py b/tensorflow/contrib/kfac/python/kernel_tests/fisher_blocks_test.py
index f48d1980ba..9b13756e62 100644
--- a/tensorflow/contrib/kfac/python/kernel_tests/fisher_blocks_test.py
+++ b/tensorflow/contrib/kfac/python/kernel_tests/fisher_blocks_test.py
@@ -209,6 +209,152 @@ class NaiveDiagonalFBTest(test.TestCase):
       self.assertAllClose(output_flat, explicit)
 
 
+class FullyConnectedDiagonalFB(test.TestCase):
+
+  def setUp(self):
+    super(FullyConnectedDiagonalFB, self).setUp()
+
+    self.batch_size = 4
+    self.input_size = 6
+    self.output_size = 3
+
+    self.inputs = np.random.randn(self.batch_size, self.input_size).astype(
+        np.float32)
+    self.outputs = np.zeros([self.batch_size, self.output_size]).astype(
+        np.float32)
+    self.output_grads = np.random.randn(self.batch_size,
+                                        self.output_size).astype(np.float32)
+    self.w = np.random.randn(self.input_size, self.output_size).astype(
+        np.float32)
+    self.b = np.random.randn(self.output_size).astype(np.float32)
+
+  def fisherApprox(self, has_bias=False):
+    """Fisher approximation using default inputs."""
+    if has_bias:
+      inputs = np.concatenate(
+          [self.inputs, np.ones([self.batch_size, 1])], axis=1)
+    else:
+      inputs = self.inputs
+    return self.buildDiagonalFisherApproximation(inputs, self.output_grads)
+
+  def buildDiagonalFisherApproximation(self, inputs, output_grads):
+    """Builds explicit diagonal Fisher approximation.
+
+    Fisher's diagonal is (d loss / d w)'s elements squared for
+      d/dw = E[outer(input, output_grad)]
+
+    where the expectation is taken over examples.
+
+    Args:
+      inputs: np.array of shape [batch_size, input_size].
+      output_grads: np.array of shape [batch_size, output_size].
+
+    Returns:
+      Diagonal np.array of shape [num_params, num_params] for num_params =
+      input_size * output_size.
+    """
+    batch_size = inputs.shape[0]
+    assert output_grads.shape[0] == batch_size
+    input_size = inputs.shape[1]
+    output_size = output_grads.shape[1]
+    fisher_diag = np.zeros((input_size, output_size))
+    for i in range(batch_size):
+      fisher_diag += np.square(np.outer(inputs[i], output_grads[i]))
+    return np.diag(fisher_diag.flatten()) / batch_size
+
+  def testMultiply(self):
+    result, _ = self.runFisherBlockOps(self.w, [self.inputs], [self.outputs],
+                                       [self.output_grads])
+
+    # Construct Fisher-vector product.
+    expected_result = self.fisherApprox().dot(self.w.flatten())
+    expected_result = expected_result.reshape(
+        [self.input_size, self.output_size])
+
+    self.assertAllClose(expected_result, result)
+
+  def testMultiplyInverse(self):
+    _, result = self.runFisherBlockOps(self.w, [self.inputs], [self.outputs],
+                                       [self.output_grads])
+
+    # Construct inverse Fisher-vector product.
+    expected_result = np.linalg.inv(self.fisherApprox()).dot(self.w.flatten())
+    expected_result = expected_result.reshape(
+        [self.input_size, self.output_size])
+
+    self.assertAllClose(expected_result, result)
+
+  def testRegisterAdditionalMinibatch(self):
+    """Ensure 1 big minibatch and 2 small minibatches are equivalent."""
+    multiply_result_big, multiply_inverse_result_big = self.runFisherBlockOps(
+        self.w, [self.inputs], [self.outputs], [self.output_grads])
+    multiply_result_small, multiply_inverse_result_small = (
+        self.runFisherBlockOps(self.w,
+                               np.split(self.inputs, 2),
+                               np.split(self.outputs, 2),
+                               np.split(self.output_grads, 2)))
+
+    self.assertAllClose(multiply_result_big, multiply_result_small)
+    self.assertAllClose(multiply_inverse_result_big,
+                        multiply_inverse_result_small)
+
+  def testMultiplyHasBias(self):
+    result, _ = self.runFisherBlockOps((self.w, self.b), [self.inputs],
+                                       [self.outputs], [self.output_grads])
+    expected_result = self.fisherApprox(True).dot(
+        np.concatenate([self.w.flatten(), self.b.flatten()]))
+    expected_result = expected_result.reshape(
+        [self.input_size + 1, self.output_size])
+    expected_result = (expected_result[:-1], expected_result[-1])
+
+    self.assertEqual(len(result), 2)
+    self.assertAllClose(expected_result[0], result[0])
+    self.assertAllClose(expected_result[1], result[1])
+
+  def runFisherBlockOps(self, params, inputs, outputs, output_grads):
+    """Run Ops guaranteed by FisherBlock interface.
+
+    Args:
+      params: Tensor or 2-tuple of Tensors. Represents weights or weights and
+        bias of this layer.
+      inputs: list of Tensors of shape [batch_size, input_size]. Inputs to
+        layer.
+      outputs: list of Tensors of shape [batch_size, output_size].
+        Preactivations produced by layer.
+      output_grads: list of Tensors of shape [batch_size, output_size].
+        Gradient of loss with respect to 'outputs'.
+
+    Returns:
+      multiply_result: Result of FisherBlock.multiply(params)
+      multiply_inverse_result: Result of FisherBlock.multiply_inverse(params)
+    """
+
+    def _as_tensors(tensor_or_tuple):
+      if isinstance(tensor_or_tuple, (tuple, list)):
+        return tuple(ops.convert_to_tensor(t) for t in tensor_or_tuple)
+      return ops.convert_to_tensor(tensor_or_tuple)
+
+    with ops.Graph().as_default(), self.test_session() as sess:
+      inputs = [_as_tensors(i) for i in inputs]
+      outputs = [_as_tensors(o) for o in outputs]
+      output_grads = [_as_tensors(og) for og in output_grads]
+      params = _as_tensors(params)
+
+      block = fb.FullyConnectedDiagonalFB(
+          lc.LayerCollection(), has_bias=isinstance(params, (tuple, list)))
+      for (i, o) in zip(inputs, outputs):
+        block.register_additional_minibatch(i, o)
+
+      block.instantiate_factors((output_grads,), damping=0.0)
+
+      sess.run(tf_variables.global_variables_initializer())
+      sess.run(block._factor.make_covariance_update_op(0.0))
+      multiply_result = sess.run(block.multiply(params))
+      multiply_inverse_result = sess.run(block.multiply_inverse(params))
+
+    return multiply_result, multiply_inverse_result
+
+
 class FullyConnectedKFACBasicFBTest(test.TestCase):
 
   def testFullyConnectedKFACBasicFBInit(self):
diff --git a/tensorflow/contrib/kfac/python/kernel_tests/layer_collection_test.py b/tensorflow/contrib/kfac/python/kernel_tests/layer_collection_test.py
index 13c69d261c..53d40da586 100644
--- a/tensorflow/contrib/kfac/python/kernel_tests/layer_collection_test.py
+++ b/tensorflow/contrib/kfac/python/kernel_tests/layer_collection_test.py
@@ -81,6 +81,11 @@ class LayerCollectionTest(test.TestCase):
       lc = layer_collection.LayerCollection()
       lc.register_fully_connected(
           array_ops.constant(1), array_ops.constant(2), array_ops.constant(3))
+      lc.register_fully_connected(
+          array_ops.constant(1),
+          array_ops.constant(2),
+          array_ops.constant(3),
+          approx=layer_collection.APPROX_DIAGONAL_NAME)
       lc.register_conv2d(
           array_ops.constant(4), [1, 1, 1, 1], 'SAME',
           array_ops.ones((1, 1, 1, 1)), array_ops.constant(3))
@@ -91,7 +96,7 @@ class LayerCollectionTest(test.TestCase):
           16,
           approx=layer_collection.APPROX_DIAGONAL_NAME)
 
-      self.assertEqual(4, len(lc.get_blocks()))
+      self.assertEqual(5, len(lc.get_blocks()))
 
   def testRegisterBlocksMultipleRegistrations(self):
     with ops.Graph().as_default():
diff --git a/tensorflow/contrib/kfac/python/ops/fisher_blocks.py b/tensorflow/contrib/kfac/python/ops/fisher_blocks.py
index 9d8bb8c8ce..6cca2272d7 100644
--- a/tensorflow/contrib/kfac/python/ops/fisher_blocks.py
+++ b/tensorflow/contrib/kfac/python/ops/fisher_blocks.py
@@ -12,7 +12,26 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""FisherBlock definitions."""
+"""FisherBlock definitions.
+
+This library contains classes for estimating blocks in a model's Fisher
+Information matrix. Suppose one has a model that parameterizes a posterior
+distribution over 'y' given 'x' with parameters 'params', p(y | x, params). Its
+Fisher Information matrix is given by,
+
+  F(params) = E[ v(x, y, params) v(x, y, params)^T ]
+
+where,
+
+  v(x, y, params) = (d / d params) log p(y | x, params)
+
+and the expectation is taken with respect to the data's distribution for 'x' and
+the model's posterior distribution for 'y',
+
+  x ~ p(x)
+  y ~ p(y | x, params)
+
+"""
 
 from __future__ import absolute_import
 from __future__ import division
@@ -133,8 +152,9 @@ class FullFB(FisherBlock):
 
   def multiply(self, vector):
     vector_flat = utils.tensors_to_column(vector)
-    out_flat = (math_ops.matmul(self._factor.get_cov(), vector_flat) +
-                self._damping * vector_flat)
+    out_flat = (
+        math_ops.matmul(self._factor.get_cov(), vector_flat) +
+        self._damping * vector_flat)
     return utils.column_to_tensors(vector, out_flat)
 
   def full_fisher_block(self):
@@ -193,54 +213,105 @@ class NaiveDiagonalFB(FisherBlock):
 class FullyConnectedDiagonalFB(FisherBlock):
   """FisherBlock for fully-connected (dense) layers using a diagonal approx.
 
-  Unlike NaiveDiagonalFB this uses the low-variance "sum of squares" estimator
-  that is computed using the well-known trick.
-  """
+  Estimates the Fisher Information matrix's diagonal entries for a fully
+  connected layer. Unlike NaiveDiagonalFB this uses the low-variance "sum of
+  squares" estimator.
 
-  # TODO(jamesmartens): add units tests for this class
+  Let 'params' be a vector parameterizing a model and 'i' an arbitrary index
+  into it. We are interested in Fisher(params)[i, i]. This is,
 
-  def __init__(self, layer_collection, inputs, outputs, has_bias=False):
+    Fisher(params)[i, i] = E[ v(x, y, params) v(x, y, params)^T ][i, i]
+                         = E[ v(x, y, params)[i] ^ 2 ]
+
+  Consider fully connected layer in this model with (unshared) weight matrix
+  'w'. For an example 'x' that produces layer inputs 'a' and output
+  preactivations 's',
+
+    v(x, y, w) = vec( x (d loss / d s)^T )
+
+  This FisherBlock tracks Fisher(params)[i, i] for all indices 'i' corresponding
+  to the layer's parameters 'w'.
+  """
+
+  def __init__(self, layer_collection, has_bias=False):
     """Creates a FullyConnectedDiagonalFB block.
 
     Args:
       layer_collection: The collection of all layers in the K-FAC approximate
           Fisher information matrix to which this FisherBlock belongs.
-      inputs: The Tensor of input activations to this layer.
-      outputs: The Tensor of output pre-activations from this layer.
       has_bias: Whether the component Kronecker factors have an additive bias.
           (Default: False)
     """
-    self._inputs = inputs
-    self._outputs = outputs
+    self._inputs = []
+    self._outputs = []
     self._has_bias = has_bias
 
     super(FullyConnectedDiagonalFB, self).__init__(layer_collection)
 
   def instantiate_factors(self, grads_list, damping):
+    inputs = _concat_along_batch_dim(self._inputs)
+    grads_list = tuple(_concat_along_batch_dim(grads) for grads in grads_list)
+
     self._damping = damping
     self._factor = self._layer_collection.make_or_get_factor(
-        fisher_factors.FullyConnectedDiagonalFactor, (self._inputs, grads_list,
-                                                      self._has_bias))
+        fisher_factors.FullyConnectedDiagonalFactor,
+        (inputs, grads_list, self._has_bias))
 
   def multiply_inverse(self, vector):
+    """Approximate damped inverse Fisher-vector product.
+
+    Args:
+      vector: Tensor or 2-tuple of Tensors. if self._has_bias, Tensor of shape
+        [input_size, output_size] corresponding to layer's weights. If not, a
+        2-tuple of the former and a Tensor of shape [output_size] corresponding
+        to the layer's bias.
+
+    Returns:
+      Tensor of the same shape, corresponding to the inverse Fisher-vector
+      product.
+    """
     reshaped_vect = utils.layer_params_to_mat2d(vector)
     reshaped_out = reshaped_vect / (self._factor.get_cov() + self._damping)
     return utils.mat2d_to_layer_params(vector, reshaped_out)
 
   def multiply(self, vector):
+    """Approximate damped Fisher-vector product.
+
+    Args:
+      vector: Tensor or 2-tuple of Tensors. if self._has_bias, Tensor of shape
+        [input_size, output_size] corresponding to layer's weights. If not, a
+        2-tuple of the former and a Tensor of shape [output_size] corresponding
+        to the layer's bias.
+
+    Returns:
+      Tensor of the same shape, corresponding to the Fisher-vector product.
+    """
     reshaped_vect = utils.layer_params_to_mat2d(vector)
     reshaped_out = reshaped_vect * (self._factor.get_cov() + self._damping)
     return utils.mat2d_to_layer_params(vector, reshaped_out)
 
   def tensors_to_compute_grads(self):
+    """Tensors to compute derivative of loss with respect to."""
     return self._outputs
 
+  def register_additional_minibatch(self, inputs, outputs):
+    """Registers an additional minibatch to the FisherBlock.
+
+    Args:
+      inputs: Tensor of shape [batch_size, input_size]. Inputs to the
+        matrix-multiply.
+      outputs: Tensor of shape [batch_size, output_size]. Layer preactivations.
+    """
+    self._inputs.append(inputs)
+    self._outputs.append(outputs)
+
 
 class ConvDiagonalFB(FisherBlock):
   """FisherBlock for convolutional layers using a diagonal approx.
 
   Unlike NaiveDiagonalFB this uses the low-variance "sum of squares" estimator.
   """
+
   # TODO(jamesmartens): add units tests for this class
 
   def __init__(self, layer_collection, params, inputs, outputs, strides,
@@ -271,14 +342,14 @@ class ConvDiagonalFB(FisherBlock):
     self._filter_shape = tuple(fltr.shape.as_list())
 
     input_shape = tuple(inputs.shape.as_list())
-    self._num_locations = (input_shape[1] * input_shape[2]
-                           // (strides[1] * strides[2]))
+    self._num_locations = (
+        input_shape[1] * input_shape[2] // (strides[1] * strides[2]))
 
     super(ConvDiagonalFB, self).__init__(layer_collection)
 
   def instantiate_factors(self, grads_list, damping):
     if NORMALIZE_DAMPING_POWER:
-      damping /= self._num_locations ** NORMALIZE_DAMPING_POWER
+      damping /= self._num_locations**NORMALIZE_DAMPING_POWER
     self._damping = damping
 
     self._factor = self._layer_collection.make_or_get_factor(
@@ -345,10 +416,12 @@ class KroneckerProductFB(FisherBlock):
     left_factor = self._input_factor.get_cov()
     right_factor = self._output_factor.get_cov()
     reshaped_vector = utils.layer_params_to_mat2d(vector)
-    reshaped_out = (math_ops.matmul(reshaped_vector, right_factor) +
-                    self._output_damping * reshaped_vector)
-    reshaped_out = (math_ops.matmul(left_factor, reshaped_out) +
-                    self._input_damping * reshaped_out)
+    reshaped_out = (
+        math_ops.matmul(reshaped_vector, right_factor) +
+        self._output_damping * reshaped_vector)
+    reshaped_out = (
+        math_ops.matmul(left_factor, reshaped_out) +
+        self._input_damping * reshaped_out)
     if self._renorm_coeff != 1.0:
       reshaped_out *= math_ops.cast(
           self._renorm_coeff, dtype=reshaped_out.dtype)
@@ -394,8 +467,8 @@ class FullyConnectedKFACBasicFB(KroneckerProductFB):
 
   def instantiate_factors(self, grads_list, damping):
     self._input_factor = self._layer_collection.make_or_get_factor(
-        fisher_factors.FullyConnectedKroneckerFactor, ((self._inputs,),
-                                                       self._has_bias))
+        fisher_factors.FullyConnectedKroneckerFactor,
+        ((self._inputs,), self._has_bias))
     self._output_factor = self._layer_collection.make_or_get_factor(
         fisher_factors.FullyConnectedKroneckerFactor, (grads_list,))
     self._register_damped_input_and_output_inverses(damping)
@@ -438,8 +511,8 @@ class ConvKFCBasicFB(KroneckerProductFB):
     self._filter_shape = tuple(fltr.shape.as_list())
 
     input_shape = tuple(inputs.shape.as_list())
-    self._num_locations = (input_shape[1] * input_shape[2] //
-                           (strides[1] * strides[2]))
+    self._num_locations = (
+        input_shape[1] * input_shape[2] // (strides[1] * strides[2]))
 
     super(ConvKFCBasicFB, self).__init__(layer_collection)
 
@@ -461,3 +534,30 @@ class ConvKFCBasicFB(KroneckerProductFB):
 
   def tensors_to_compute_grads(self):
     return self._outputs
+
+
+def _concat_along_batch_dim(tensor_list):
+  """Concatenate tensors along batch (first) dimension.
+
+  Args:
+    tensor_list: list of Tensors or list of tuples of Tensors.
+
+  Returns:
+    Tensor or tuple of Tensors.
+
+  Raises:
+    ValueError: If 'tensor_list' is empty.
+
+  """
+  if not tensor_list:
+    raise ValueError(
+        "Cannot concatenate Tensors if there are no Tensors to concatenate.")
+
+  if isinstance(tensor_list[0], (tuple, list)):
+    # [(tensor1a, tensor1b),
+    #  (tensor2a, tensor2b), ...] --> (tensor_a, tensor_b)
+    return tuple(
+        array_ops.concat(tensors, axis=0) for tensors in zip(*tensor_list))
+  else:
+    # [tensor1, tensor2] --> tensor
+    return array_ops.concat(tensor_list, axis=0)
diff --git a/tensorflow/contrib/kfac/python/ops/fisher_factors.py b/tensorflow/contrib/kfac/python/ops/fisher_factors.py
index d3c783ee2f..86a1782fcf 100644
--- a/tensorflow/contrib/kfac/python/ops/fisher_factors.py
+++ b/tensorflow/contrib/kfac/python/ops/fisher_factors.py
@@ -428,11 +428,28 @@ class NaiveDiagonalFactor(DiagonalFactor):
 
 
 class FullyConnectedDiagonalFactor(DiagonalFactor):
-  """FisherFactor for a diagonal approx of a fully-connected layer's Fisher."""
+  r"""FisherFactor for a diagonal approx of a fully-connected layer's Fisher.
+
+  Given in = [batch_size, input_size] and out_grad = [batch_size, output_size],
+  approximates the covariance as,
+
+    Cov(in, out) = (1/batch_size) \sum_{i} outer(in[i], out_grad[i]) ** 2.0
+
+  where the square is taken element-wise.
+  """
 
   # TODO(jamesmartens): add units tests for this class
 
   def __init__(self, inputs, outputs_grads, has_bias=False):
+    """Instantiate FullyConnectedDiagonalFactor.
+
+    Args:
+      inputs: Tensor of shape [batch_size, input_size]. Inputs to fully
+        connected layer.
+      outputs_grads: List of Tensors of shape [batch_size, output_size].
+        Gradient of loss with respect to layer's preactivations.
+      has_bias: bool. If True, append '1' to each input.
+    """
     self._outputs_grads = outputs_grads
     self._batch_size = array_ops.shape(inputs)[0]
     self._orig_tensors_name = scope_string_from_params((inputs,) +
diff --git a/tensorflow/contrib/kfac/python/ops/layer_collection.py b/tensorflow/contrib/kfac/python/ops/layer_collection.py
index 0cb55894ad..beb8ef136e 100644
--- a/tensorflow/contrib/kfac/python/ops/layer_collection.py
+++ b/tensorflow/contrib/kfac/python/ops/layer_collection.py
@@ -259,9 +259,9 @@ class LayerCollection(object):
                           fb.FullyConnectedKFACBasicFB(self, inputs, outputs,
                                                        has_bias))
     elif approx == APPROX_DIAGONAL_NAME:
-      self.register_block(params,
-                          fb.FullyConnectedDiagonalFB(self, inputs, outputs,
-                                                      has_bias))
+      block = fb.FullyConnectedDiagonalFB(self, has_bias)
+      block.register_additional_minibatch(inputs, outputs)
+      self.register_block(params, block)
     else:
       raise ValueError("Bad value {} for approx.".format(approx))
 
-- 
GitLab


From f4db3b27430479cccb51518952102d63a3ebc916 Mon Sep 17 00:00:00 2001
From: gunan <gunan@google.com>
Date: Wed, 18 Oct 2017 20:47:56 -0700
Subject: [PATCH 548/909] Disable flaky tests in cmake build. (#13816)

---
 tensorflow/contrib/cmake/tf_tests.cmake | 21 ++++++++++++++-------
 1 file changed, 14 insertions(+), 7 deletions(-)

diff --git a/tensorflow/contrib/cmake/tf_tests.cmake b/tensorflow/contrib/cmake/tf_tests.cmake
index 24f21afdfc..3cc874e4f2 100644
--- a/tensorflow/contrib/cmake/tf_tests.cmake
+++ b/tensorflow/contrib/cmake/tf_tests.cmake
@@ -192,12 +192,10 @@ if (tensorflow_BUILD_PYTHON_TESTS)
     "${tensorflow_source_dir}/tensorflow/python/debug/lib/session_debug_grpc_test.py"
     # generally not working
     "${tensorflow_source_dir}/tensorflow/python/profiler/pprof_profiler_test.py"
-    # flaky test
-    "${tensorflow_source_dir}/tensorflow/python/profiler/internal/run_metadata_test.py"
+    # Fails because uses data dependencies with bazel
     "${tensorflow_source_dir}/tensorflow/python/saved_model/saved_model_test.py"
-    # flaky tests
-    "${tensorflow_source_dir}/tensorflow/python/kernel_tests/cwise_ops_test.py"  # takes very long to run
-    "${tensorflow_source_dir}/tensorflow/contrib/tfprof/python/tools/tfprof/internal/run_metadata_test.py"
+    # Takes very long to run without sharding (defined in bazel build file).
+    "${tensorflow_source_dir}/tensorflow/python/kernel_tests/cwise_ops_test.py"
     # Loading resources in contrib doesn't seem to work on Windows
     "${tensorflow_source_dir}/tensorflow/contrib/tensor_forest/client/random_forest_test.py"
     "${tensorflow_source_dir}/tensorflow/contrib/tensor_forest/python/tensor_forest_test.py"
@@ -216,14 +214,22 @@ if (tensorflow_BUILD_PYTHON_TESTS)
 
       # stl on windows handles overflows different
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/as_string_op_test.py"
-      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/cast_op_test.py"
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/string_to_number_op_test.py"
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/clip_ops_test.py"
       # Numerical issues, calculations off.
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/concat_op_test.py"
-      "${tensorflow_source_dir}/tensorflow/contrib/factorization/python/ops/wals_test.py"  
+      "${tensorflow_source_dir}/tensorflow/contrib/factorization/python/ops/wals_test.py"
       # Float division by zero
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/benchmark_test.py"
+      # Flaky, for unknown reasons. Cannot reproduce in terminal. Revisit once we can get stack traces.
+      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/batch_matmul_op_test.py"
+      # Flaky because of local cluster creation.
+      "${tensorflow_source_dir}/tensorflow/python/training/sync_replicas_optimizer_test.py"
+      "${tensorflow_source_dir}/tensorflow/python/debug/lib/session_debug_grpc_test.py"
+      "${tensorflow_source_dir}tensorflow/python/training/localhost_cluster_performance_test.py"
+      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/iterator_ops_cluster_test.py"
+      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/functional_ops_test.py"
+      "${tensorflow_source_dir}/tensorflow/contrib/data/python/kernel_tests/iterator_ops_cluster_test.py"
       # Type error in testRemoteIteratorUsingRemoteCallOpDirectSessionGPUCPU.
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/iterator_ops_test.py"
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/self_adjoint_eig_op_test.py"
@@ -233,6 +239,7 @@ if (tensorflow_BUILD_PYTHON_TESTS)
       # Depends on gemmlowp -> pthread
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/neon_depthwise_conv_op_test.py"
       # int32/int64 mixup
+      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/cast_op_test.py"
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/variable_scope_test.py"
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/functional_ops_test.py"
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/py_func_test.py"
-- 
GitLab


From 52e9b19b4fdf7ac966858f906c397e5ba2a1bf85 Mon Sep 17 00:00:00 2001
From: Tian Jin <tian.jin1@ibm.com>
Date: Thu, 19 Oct 2017 10:27:40 -0400
Subject: [PATCH 549/909] fix aws build file for linux ppc

---
 third_party/aws.BUILD | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/third_party/aws.BUILD b/third_party/aws.BUILD
index 9d8e7946cd..bc6a2fd8cc 100644
--- a/third_party/aws.BUILD
+++ b/third_party/aws.BUILD
@@ -18,6 +18,9 @@ cc_library(
         "@%ws%//tensorflow:darwin": glob([
             "aws-cpp-sdk-core/source/platform/linux-shared/*.cpp",
         ]),
+        "@%ws%//tensorflow:linux_ppc64le": glob([
+            "aws-cpp-sdk-core/source/platform/linux-shared/*.cpp",
+        ]),
         "//conditions:default": [],
     }) + glob([
         "aws-cpp-sdk-core/include/**/*.h",
@@ -57,6 +60,11 @@ cc_library(
             "ENABLE_CURL_CLIENT",
             "ENABLE_NO_ENCRYPTION",
         ],
+        "@%ws%//tensorflow:linux_ppc64le": [
+            "PLATFORM_LINUX",
+            "ENABLE_CURL_CLIENT",
+            "ENABLE_NO_ENCRYPTION",
+        ],
         "//conditions:default": [],
     }),
     includes = [
-- 
GitLab


From 502340916822f8cafade906c1c42acd842ddb7ed Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Thu, 19 Oct 2017 09:43:01 -0700
Subject: [PATCH 550/909] tfe.enable_eager_execution fails only if graphs have
 been created. It is otherwise idempotent.

PiperOrigin-RevId: 172757241
---
 tensorflow/contrib/eager/python/tfe.py      |  4 +-
 tensorflow/contrib/eager/python/tfe_test.py |  8 ---
 tensorflow/python/eager/BUILD               |  2 +-
 tensorflow/python/eager/context.py          | 64 ---------------------
 tensorflow/python/eager/test.py             |  4 +-
 tensorflow/python/framework/ops.py          | 58 +++++++++++++++++++
 6 files changed, 63 insertions(+), 77 deletions(-)

diff --git a/tensorflow/contrib/eager/python/tfe.py b/tensorflow/contrib/eager/python/tfe.py
index 1acb1ba1b8..c519df8b5c 100644
--- a/tensorflow/contrib/eager/python/tfe.py
+++ b/tensorflow/contrib/eager/python/tfe.py
@@ -67,12 +67,10 @@ from tensorflow.contrib.eager.python.saver import Saver
 from tensorflow.contrib.eager.python.summary_writer import SummaryWriter
 from tensorflow.python.eager import backprop
 from tensorflow.python.eager import function
-from tensorflow.python.eager.context import enable_eager_execution
 from tensorflow.python.eager.context import in_eager_mode
 from tensorflow.python.eager.context import in_graph_mode
 from tensorflow.python.eager.context import list_devices
 from tensorflow.python.eager.context import num_gpus
-from tensorflow.python.eager.context import run
 from tensorflow.python.eager.core import enable_tracing
 from tensorflow.python.eager.custom_gradient import custom_gradient
 from tensorflow.python.eager.execution_callbacks import add_execution_callback
@@ -81,6 +79,8 @@ from tensorflow.python.eager.execution_callbacks import inf_callback
 from tensorflow.python.eager.execution_callbacks import inf_nan_callback
 from tensorflow.python.eager.execution_callbacks import nan_callback
 from tensorflow.python.eager.execution_callbacks import seterr
+from tensorflow.python.framework.ops import enable_eager_execution
+from tensorflow.python.framework.ops import eager_run as run
 from tensorflow.python.framework.test_util import run_in_graph_and_eager_modes as run_test_in_graph_and_eager_modes
 from tensorflow.python.ops.resource_variable_ops import ResourceVariable as Variable
 from tensorflow.python.util.all_util import remove_undocumented
diff --git a/tensorflow/contrib/eager/python/tfe_test.py b/tensorflow/contrib/eager/python/tfe_test.py
index eab8958f23..6b5053125b 100644
--- a/tensorflow/contrib/eager/python/tfe_test.py
+++ b/tensorflow/contrib/eager/python/tfe_test.py
@@ -95,14 +95,6 @@ class TFETest(test_util.TensorFlowTestCase):
     devices = tfe.list_devices()
     self.assertEqual(len(devices) - 1, tfe.num_gpus())
 
-  def testCallingEnableEagerExecutionMoreThanOnce(self):
-    # Note that eager.test.main() has already invoked enable_eager_exceution().
-    with self.assertRaisesRegexp(
-        ValueError, r'Do not call tfe\.%s more than once in the same process' %
-        tfe.enable_eager_execution.__name__):
-      tfe.enable_eager_execution()
-
-
 if __name__ == '__main__':
   tfe.enable_eager_execution()
   test.main()
diff --git a/tensorflow/python/eager/BUILD b/tensorflow/python/eager/BUILD
index ef04f933c5..f5b946ec26 100644
--- a/tensorflow/python/eager/BUILD
+++ b/tensorflow/python/eager/BUILD
@@ -151,9 +151,9 @@ py_library(
     srcs_version = "PY2AND3",
     visibility = ["//tensorflow:internal"],
     deps = [
-        ":context",
         ":core",
         "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework_ops",
     ],
 )
 
diff --git a/tensorflow/python/eager/context.py b/tensorflow/python/eager/context.py
index 996748a870..aa7cba56de 100644
--- a/tensorflow/python/eager/context.py
+++ b/tensorflow/python/eager/context.py
@@ -26,10 +26,8 @@ import threading
 from tensorflow.python import pywrap_tensorflow
 from tensorflow.python.framework import device as pydev
 from tensorflow.python.framework import errors
-from tensorflow.python.platform import app
 from tensorflow.python.util import compat
 from tensorflow.python.util import tf_contextlib
-from tensorflow.python.util import tf_inspect
 
 GRAPH_MODE = 0
 EAGER_MODE = 1
@@ -423,68 +421,6 @@ def device(name):
   return context().device(name)
 
 
-def run(main=None, argv=None):
-  """Runs the program with an optional main function and argv list.
-
-  The program will run with eager execution enabled.
-
-  Example:
-  ```python
-  import tensorflow as tf
-  # Import subject to future changes:
-  from tensorflow.contrib.eager.python import tfe
-
-  def main(_):
-    u = tf.constant(6.0)
-    v = tf.constant(7.0)
-    print(u * v)
-
-  if __name__ == "__main__":
-    tfe.run()
-  ```
-
-  Args:
-    main: the main function to run.
-    argv: the arguments to pass to it.
-  """
-  enable_eager_execution()
-  app.run(main, argv)
-
-
-# TODO(apassos): This should not be a part of the public API.
-def enable_eager_execution():
-  """Enables, for the rest of the lifetime of this program, eager execution.
-
-  If not called immediately on startup risks creating breakage and bugs. Calling
-  this method more than once in the same process will lead to an exception.
-
-  Example:
-  ```python
-  # Before eager execution is enabled, `Tensor`s are symbolic and do not hold
-  # concrete values (they are to be executed in a `tf.Session`).
-  assert not hasattr(tf.multiply(6, 7), "numpy")
-
-  tfe.enable_eager_execution()
-
-  # After eager execution is enabled, operations are executed as they are
-  # defined and `Tensor`s hold concrete values, which can be accessed as
-  # `numpy.ndarray`s through the `numpy()` method.
-  assert tf.multiply(6, 7).numpy() == 42
-  ```
-
-  Raises:
-    ValueError: If this method has already been invoked in the current process.
-  """
-  global _default_mode
-  if _default_mode == EAGER_MODE:
-    func_name = (
-        "tfe." + tf_inspect.getframeinfo(tf_inspect.currentframe()).function)
-    raise ValueError(
-        "Do not call %s more than once in the same process. Note eager-mode "
-        "methods such as tfe.run() also call %s." % (func_name, func_name))
-  _default_mode = EAGER_MODE
-
-
 def list_devices():
   """List the names of the available devices.
 
diff --git a/tensorflow/python/eager/test.py b/tensorflow/python/eager/test.py
index 3d8af7e056..f6a46e7eb3 100644
--- a/tensorflow/python/eager/test.py
+++ b/tensorflow/python/eager/test.py
@@ -18,11 +18,11 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.python.eager import context as _context
+from tensorflow.python.framework import ops as _ops
 from tensorflow.python.platform import test as _test
 from tensorflow.python.platform.test import *  # pylint: disable=wildcard-import
 
 
 def main(argv=None):
-  _context.enable_eager_execution()
+  _ops.enable_eager_execution()
   _test.main(argv)
diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index e7e36941e5..ef708a4703 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -47,6 +47,7 @@ from tensorflow.python.framework import op_def_registry
 from tensorflow.python.framework import registry
 from tensorflow.python.framework import tensor_shape
 from tensorflow.python.framework import versions
+from tensorflow.python.platform import app
 from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.util import compat
 from tensorflow.python.util import decorator_utils
@@ -4558,6 +4559,63 @@ class _DefaultGraphStack(_DefaultStack):  # pylint: disable=protected-access
 _default_graph_stack = _DefaultGraphStack()
 
 
+def enable_eager_execution():
+  """Enables, for the rest of the lifetime of this program, eager execution.
+
+  If not called immediately on startup risks creating breakage and bugs.
+
+  Example:
+  ```python
+  tfe.enable_eager_execution()
+
+  # After eager execution is enabled, operations are executed as they are
+  # defined and `Tensor`s hold concrete values, which can be accessed as
+  # `numpy.ndarray`s through the `numpy()` method.
+  assert tf.multiply(6, 7).numpy() == 42
+  ```
+
+  Raises:
+    ValueError: If this method has already been invoked in the current process.
+  """
+  # pylint: disable=protected-access
+  if context._default_mode == context.GRAPH_MODE:
+    graph_mode_has_been_used = (
+        _default_session_stack.stack or
+        _default_graph_stack._global_default_graph is not None)
+    if graph_mode_has_been_used:
+      raise ValueError(
+          "tfe.enable_eager_execution has to be called at program startup.")
+  context._default_mode = context.EAGER_MODE
+
+
+def eager_run(main=None, argv=None):
+  """Runs the program with an optional main function and argv list.
+
+  The program will run with eager execution enabled.
+
+  Example:
+  ```python
+  import tensorflow as tf
+  # Import subject to future changes:
+  from tensorflow.contrib.eager.python import tfe
+
+  def main(_):
+    u = tf.constant(6.0)
+    v = tf.constant(7.0)
+    print(u * v)
+
+  if __name__ == "__main__":
+    tfe.run()
+  ```
+
+  Args:
+    main: the main function to run.
+    argv: the arguments to pass to it.
+  """
+  enable_eager_execution()
+  app.run(main, argv)
+
+
 def reset_default_graph():
   """Clears the default graph stack and resets the global default graph.
 
-- 
GitLab


From df22cf83a21b62838ecf6f3a1c8a9c30ab20d482 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 19 Oct 2017 09:43:08 -0700
Subject: [PATCH 551/909] Makes export_outputs in multi_label_head consistent
 with multi_class_head and simplifies multi_head export_outputs.

PiperOrigin-RevId: 172757258
---
 tensorflow/contrib/estimator/BUILD            |  1 +
 .../estimator/python/estimator/head.py        | 11 +++-
 .../estimator/python/estimator/head_test.py   | 27 +++++++-
 .../estimator/python/estimator/multi_head.py  |  5 +-
 .../python/estimator/multi_head_test.py       |  4 +-
 tensorflow/python/estimator/canned/head.py    | 63 +++++++++----------
 6 files changed, 72 insertions(+), 39 deletions(-)

diff --git a/tensorflow/contrib/estimator/BUILD b/tensorflow/contrib/estimator/BUILD
index 4dd9f19ec3..89c26d1d2f 100644
--- a/tensorflow/contrib/estimator/BUILD
+++ b/tensorflow/contrib/estimator/BUILD
@@ -135,6 +135,7 @@ py_library(
         "//tensorflow/python/estimator:model_fn",
         "//tensorflow/python/estimator:prediction_keys",
         "//tensorflow/python/ops/losses",
+        "//tensorflow/python/saved_model:signature_constants",
     ],
 )
 
diff --git a/tensorflow/contrib/estimator/python/estimator/head.py b/tensorflow/contrib/estimator/python/estimator/head.py
index ebf91e8bb4..d01b30d7f9 100644
--- a/tensorflow/contrib/estimator/python/estimator/head.py
+++ b/tensorflow/contrib/estimator/python/estimator/head.py
@@ -33,8 +33,11 @@ from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import metrics as metrics_lib
 from tensorflow.python.ops import sparse_ops
 from tensorflow.python.ops.losses import losses
+from tensorflow.python.saved_model import signature_constants
 from tensorflow.python.summary import summary
 
+_DEFAULT_SERVING_KEY = signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY
+
 
 def multi_class_head(n_classes,
                      weight_column=None,
@@ -287,11 +290,17 @@ class _MultiLabelHead(head_lib._Head):  # pylint:disable=protected-access
             pred_keys.PROBABILITIES: probabilities,
         }
       if mode == model_fn.ModeKeys.PREDICT:
+        classifier_output = head_lib._classification_output(  # pylint:disable=protected-access
+            scores=probabilities, n_classes=self._n_classes,
+            label_vocabulary=self._label_vocabulary)
         return model_fn.EstimatorSpec(
             mode=model_fn.ModeKeys.PREDICT,
             predictions=predictions,
             export_outputs={
-                '': export_output.ClassificationOutput(scores=probabilities)
+                _DEFAULT_SERVING_KEY: classifier_output,
+                head_lib._CLASSIFY_SERVING_KEY: classifier_output,  # pylint:disable=protected-access
+                head_lib._PREDICT_SERVING_KEY: (  # pylint:disable=protected-access
+                    export_output.PredictOutput(predictions))
             })
 
       # Eval.
diff --git a/tensorflow/contrib/estimator/python/estimator/head_test.py b/tensorflow/contrib/estimator/python/estimator/head_test.py
index ec1386af34..b7252f93ee 100644
--- a/tensorflow/contrib/estimator/python/estimator/head_test.py
+++ b/tensorflow/contrib/estimator/python/estimator/head_test.py
@@ -143,6 +143,7 @@ class MultiLabelHead(test.TestCase):
     logits = np.array(
         [[0., 1., 2., -1.], [-1., -2., -3., 1.]], dtype=np.float32)
     expected_probabilities = _sigmoid(logits)
+    expected_export_classes = [[b'0', b'1', b'2', b'3']] * 2
 
     spec = head.create_estimator_spec(
         features={'x': np.array(((42,),), dtype=np.int32)},
@@ -150,7 +151,8 @@ class MultiLabelHead(test.TestCase):
         logits=logits)
 
     self.assertItemsEqual(
-        ('', _DEFAULT_SERVING_KEY), spec.export_outputs.keys())
+        (_DEFAULT_SERVING_KEY, 'predict', 'classification'),
+        spec.export_outputs.keys())
 
     # Assert predictions and export_outputs.
     with self.test_session() as sess:
@@ -166,6 +168,29 @@ class MultiLabelHead(test.TestCase):
       self.assertAllClose(
           expected_probabilities,
           sess.run(spec.export_outputs[_DEFAULT_SERVING_KEY].scores))
+      self.assertAllEqual(
+          expected_export_classes,
+          sess.run(spec.export_outputs[_DEFAULT_SERVING_KEY].classes))
+
+  def test_predict_with_label_vocabulary(self):
+    n_classes = 4
+    head = head_lib.multi_label_head(
+        n_classes, label_vocabulary=['foo', 'bar', 'foobar', 'barfoo'])
+
+    logits = np.array(
+        [[0., 1., 2., -1.], [-1., -2., -3., 1.]], dtype=np.float32)
+    expected_export_classes = [[b'foo', b'bar', b'foobar', b'barfoo']] * 2
+
+    spec = head.create_estimator_spec(
+        features={'x': np.array(((42,),), dtype=np.int32)},
+        mode=model_fn.ModeKeys.PREDICT,
+        logits=logits)
+
+    with self.test_session() as sess:
+      _initialize_variables(self, spec.scaffold)
+      self.assertAllEqual(
+          expected_export_classes,
+          sess.run(spec.export_outputs[_DEFAULT_SERVING_KEY].classes))
 
   def test_weight_should_not_impact_prediction(self):
     n_classes = 4
diff --git a/tensorflow/contrib/estimator/python/estimator/multi_head.py b/tensorflow/contrib/estimator/python/estimator/multi_head.py
index e6340424f7..64b2a9dee8 100644
--- a/tensorflow/contrib/estimator/python/estimator/multi_head.py
+++ b/tensorflow/contrib/estimator/python/estimator/multi_head.py
@@ -236,7 +236,10 @@ class _MultiHead(head_lib._Head):  # pylint:disable=protected-access
     for head, spec in zip(self._heads, all_estimator_spec):
       head_name = head.name
       for k, v in six.iteritems(spec.export_outputs):
-        key = '%s/%s' % (k, head_name) if k else head_name
+        if k == _DEFAULT_SERVING_KEY:
+          key = head_name
+        else:
+          key = '%s/%s' % (k, head_name)
         export_outputs[key] = v
       for k, v in six.iteritems(spec.predictions):
         predictions[(head_name, k)] = v
diff --git a/tensorflow/contrib/estimator/python/estimator/multi_head_test.py b/tensorflow/contrib/estimator/python/estimator/multi_head_test.py
index e86cb2b96f..48027035ce 100644
--- a/tensorflow/contrib/estimator/python/estimator/multi_head_test.py
+++ b/tensorflow/contrib/estimator/python/estimator/multi_head_test.py
@@ -126,8 +126,8 @@ class MultiHeadTest(test.TestCase):
         logits=logits)
 
     self.assertItemsEqual(
-        (_DEFAULT_SERVING_KEY, _DEFAULT_SERVING_KEY + '/head1', 'head1',
-         _DEFAULT_SERVING_KEY + '/head2', 'head2'),
+        (_DEFAULT_SERVING_KEY, 'head1', 'classification/head1', 'predict/head1',
+         'head2', 'classification/head2', 'predict/head2'),
         spec.export_outputs.keys())
 
     # Assert predictions and export_outputs.
diff --git a/tensorflow/python/estimator/canned/head.py b/tensorflow/python/estimator/canned/head.py
index beafe0d5c4..1cc82c5055 100644
--- a/tensorflow/python/estimator/canned/head.py
+++ b/tensorflow/python/estimator/canned/head.py
@@ -269,6 +269,21 @@ def _indicator_labels_mean(labels, weights=None, name=None):
     return metrics_lib.mean(labels, weights=weights, name=scope)
 
 
+def _classification_output(scores, n_classes, label_vocabulary=None):
+  batch_size = array_ops.shape(scores)[0]
+  if label_vocabulary:
+    export_class_list = label_vocabulary
+  else:
+    export_class_list = string_ops.as_string(math_ops.range(n_classes))
+  export_output_classes = array_ops.tile(
+      input=array_ops.expand_dims(input=export_class_list, axis=0),
+      multiples=[batch_size, 1])
+  return export_output.ClassificationOutput(
+      scores=scores,
+      # `ClassificationOutput` requires string classes.
+      classes=export_output_classes)
+
+
 def _accuracy_baseline(labels_mean):
   """Return accuracy baseline based on labels mean.
 
@@ -401,12 +416,11 @@ class _MultiClassHeadWithSoftmaxCrossEntropyLoss(_Head):
   def logits_dimension(self):
     return self._n_classes
 
-  def _eval_metric_ops(self, labels, probabilities, logits,
-                       class_ids, weights, unweighted_loss):
+  def _eval_metric_ops(self, labels, class_ids, weights, unweighted_loss):
     """Returns the Eval metric ops."""
     with ops.name_scope(
         None, 'metrics',
-        (labels, probabilities, logits, class_ids, weights, unweighted_loss)):
+        (labels, class_ids, weights, unweighted_loss)):
       keys = metric_keys.MetricKeys
       metric_ops = {
           # Estimator already adds a metric for loss.
@@ -479,18 +493,9 @@ class _MultiClassHeadWithSoftmaxCrossEntropyLoss(_Head):
             pred_keys.CLASSES: classes,
         }
       if mode == model_fn.ModeKeys.PREDICT:
-        batch_size = array_ops.shape(probabilities)[0]
-        export_class_list = self._label_vocabulary
-        if not export_class_list:
-          export_class_list = string_ops.as_string(
-              math_ops.range(self._n_classes))
-        export_output_classes = array_ops.tile(
-            input=array_ops.expand_dims(input=export_class_list, axis=0),
-            multiples=[batch_size, 1])
-        classifier_output = export_output.ClassificationOutput(
-            scores=probabilities,
-            # `ClassificationOutput` requires string classes.
-            classes=export_output_classes)
+        classifier_output = _classification_output(
+            scores=probabilities, n_classes=self._n_classes,
+            label_vocabulary=self._label_vocabulary)
         return model_fn.EstimatorSpec(
             mode=model_fn.ModeKeys.PREDICT,
             predictions=predictions,
@@ -513,8 +518,6 @@ class _MultiClassHeadWithSoftmaxCrossEntropyLoss(_Head):
             loss=training_loss,
             eval_metric_ops=self._eval_metric_ops(
                 labels=label_ids,
-                probabilities=probabilities,
-                logits=logits,
                 class_ids=class_ids,
                 unweighted_loss=unweighted_loss,
                 weights=weights))
@@ -611,12 +614,12 @@ class _BinaryLogisticHeadWithSigmoidCrossEntropyLoss(_Head):
                        labels,
                        logits,
                        logistic,
-                       scores,
                        class_ids,
                        unweighted_loss,
                        weights=None):
-    with ops.name_scope(None, 'metrics', (labels, logits, logistic, scores,
-                                          class_ids, unweighted_loss, weights)):
+    with ops.name_scope(
+        None, 'metrics',
+        (labels, logits, logistic, class_ids, unweighted_loss, weights)):
       keys = metric_keys.MetricKeys
       labels_mean = _indicator_labels_mean(
           labels=labels, weights=weights, name=keys.LABEL_MEAN)
@@ -709,7 +712,8 @@ class _BinaryLogisticHeadWithSigmoidCrossEntropyLoss(_Head):
         logistic = math_ops.sigmoid(logits, name=pred_keys.LOGISTIC)
         two_class_logits = array_ops.concat(
             (array_ops.zeros_like(logits), logits), 1, name='two_class_logits')
-        scores = nn.softmax(two_class_logits, name=pred_keys.PROBABILITIES)
+        probabilities = nn.softmax(
+            two_class_logits, name=pred_keys.PROBABILITIES)
         class_ids = array_ops.reshape(
             math_ops.argmax(two_class_logits, axis=1), (-1, 1), name='classes')
         if self._label_vocabulary:
@@ -722,22 +726,14 @@ class _BinaryLogisticHeadWithSigmoidCrossEntropyLoss(_Head):
         predictions = {
             pred_keys.LOGITS: logits,
             pred_keys.LOGISTIC: logistic,
-            pred_keys.PROBABILITIES: scores,
+            pred_keys.PROBABILITIES: probabilities,
             pred_keys.CLASS_IDS: class_ids,
             pred_keys.CLASSES: classes,
         }
       if mode == model_fn.ModeKeys.PREDICT:
-        batch_size = array_ops.shape(logistic)[0]
-        export_class_list = self._label_vocabulary
-        if not export_class_list:
-          export_class_list = string_ops.as_string([0, 1])
-        export_output_classes = array_ops.tile(
-            input=array_ops.expand_dims(input=export_class_list, axis=0),
-            multiples=[batch_size, 1])
-        classifier_output = export_output.ClassificationOutput(
-            scores=scores,
-            # `ClassificationOutput` requires string classes.
-            classes=export_output_classes)
+        classifier_output = _classification_output(
+            scores=probabilities, n_classes=2,
+            label_vocabulary=self._label_vocabulary)
         return model_fn.EstimatorSpec(
             mode=model_fn.ModeKeys.PREDICT,
             predictions=predictions,
@@ -764,7 +760,6 @@ class _BinaryLogisticHeadWithSigmoidCrossEntropyLoss(_Head):
                 labels=processed_labels,
                 logits=logits,
                 logistic=logistic,
-                scores=scores,
                 class_ids=class_ids,
                 unweighted_loss=unweighted_loss,
                 weights=weights))
-- 
GitLab


From fa058053e59efc0d60147b40e265392a858bc3cd Mon Sep 17 00:00:00 2001
From: "Joshua V. Dillon" <jvdillon@google.com>
Date: Thu, 19 Oct 2017 09:47:56 -0700
Subject: [PATCH 552/909] Add `quantile` to
 `tf.distributions.TransformedDistribution`.

PiperOrigin-RevId: 172757818
---
 .../transformed_distribution_test.py             | 12 ++++++++++++
 .../ops/conditional_transformed_distribution.py  | 16 ++++++++++++++++
 .../distributions/transformed_distribution.py    | 13 +++++++++++++
 3 files changed, 41 insertions(+)

diff --git a/tensorflow/contrib/distributions/python/kernel_tests/transformed_distribution_test.py b/tensorflow/contrib/distributions/python/kernel_tests/transformed_distribution_test.py
index 4001530f66..103d8e1862 100644
--- a/tensorflow/contrib/distributions/python/kernel_tests/transformed_distribution_test.py
+++ b/tensorflow/contrib/distributions/python/kernel_tests/transformed_distribution_test.py
@@ -116,6 +116,18 @@ class TransformedDistributionTest(test.TestCase):
             np.log(sp_normal.pdf(2.13) + sp_normal.pdf(-2.13)),
             abs_normal.log_prob(2.13).eval())
 
+  def testQuantile(self):
+    with self.test_session() as sess:
+      logit_normal = self._cls()(
+          distribution=ds.Normal(loc=0., scale=1.),
+          bijector=bs.Sigmoid(),
+          validate_args=True)
+      grid = [0., 0.25, 0.5, 0.75, 1.]
+      q = logit_normal.quantile(grid)
+      cdf = logit_normal.cdf(q)
+      cdf_ = sess.run(cdf)
+      self.assertAllClose(grid, cdf_, rtol=1e-6, atol=0.)
+
   def testCachedSamples(self):
     exp_forward_only = bs.Exp(event_ndims=0)
     exp_forward_only._inverse = self._make_unimplemented(
diff --git a/tensorflow/contrib/distributions/python/ops/conditional_transformed_distribution.py b/tensorflow/contrib/distributions/python/ops/conditional_transformed_distribution.py
index f1b7bf468e..599c855cda 100644
--- a/tensorflow/contrib/distributions/python/ops/conditional_transformed_distribution.py
+++ b/tensorflow/contrib/distributions/python/ops/conditional_transformed_distribution.py
@@ -198,3 +198,19 @@ class ConditionalTransformedDistribution(
     distribution_kwargs = distribution_kwargs or {}
     x = self.bijector.inverse(y, **bijector_kwargs)
     return self.distribution.survival_function(x, **distribution_kwargs)
+
+  @distribution_util.AppendDocstring(kwargs_dict=_condition_kwargs_dict)
+  def _quantile(self, value, bijector_kwargs=None, distribution_kwargs=None):
+    if self._is_maybe_event_override:
+      raise NotImplementedError("quantile is not implemented when overriding "
+                                "event_shape")
+    if not self.bijector._is_injective:  # pylint: disable=protected-access
+      raise NotImplementedError("quantile is not implemented when "
+                                "bijector is not injective.")
+    bijector_kwargs = bijector_kwargs or {}
+    distribution_kwargs = distribution_kwargs or {}
+    # x_q is the "qth quantile" of X iff q = P[X <= x_q].  Now, since X =
+    # g^{-1}(Y), q = P[X <= x_q] = P[g^{-1}(Y) <= x_q] = P[Y <= g(x_q)],
+    # implies the qth quantile of Y is g(x_q).
+    inv_cdf = self.distribution.quantile(value, **distribution_kwargs)
+    return self.bijector.forward(inv_cdf, **bijector_kwargs)
diff --git a/tensorflow/python/ops/distributions/transformed_distribution.py b/tensorflow/python/ops/distributions/transformed_distribution.py
index 15a1125f82..ba25b2c348 100644
--- a/tensorflow/python/ops/distributions/transformed_distribution.py
+++ b/tensorflow/python/ops/distributions/transformed_distribution.py
@@ -503,6 +503,19 @@ class TransformedDistribution(distribution_lib.Distribution):
     x = self.bijector.inverse(y)
     return self.distribution.survival_function(x)
 
+  def _quantile(self, value):
+    if self._is_maybe_event_override:
+      raise NotImplementedError("quantile is not implemented when overriding "
+                                "event_shape")
+    if not self.bijector._is_injective:  # pylint: disable=protected-access
+      raise NotImplementedError("quantile is not implemented when "
+                                "bijector is not injective.")
+    # x_q is the "qth quantile" of X iff q = P[X <= x_q].  Now, since X =
+    # g^{-1}(Y), q = P[X <= x_q] = P[g^{-1}(Y) <= x_q] = P[Y <= g(x_q)],
+    # implies the qth quantile of Y is g(x_q).
+    inv_cdf = self.distribution.quantile(value)
+    return self.bijector.forward(inv_cdf)
+
   def _entropy(self):
     if not self.bijector.is_constant_jacobian:
       raise NotImplementedError("entropy is not implemented")
-- 
GitLab


From f477d48ce7b89d1ded6c823dcf9518fccd837643 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Thu, 19 Oct 2017 10:11:57 -0700
Subject: [PATCH 553/909] [XLA:GPU] Use cuMemcpy for device-to-device data
 copies

PiperOrigin-RevId: 172760878
---
 .../compiler/xla/service/gpu/copy_thunk.cc    | 28 +++++++++++--
 .../compiler/xla/service/gpu/copy_thunk.h     | 39 ++++++++++++++----
 .../compiler/xla/service/gpu/ir_emitter.h     |  8 +++-
 .../xla/service/gpu/ir_emitter_unnested.cc    | 41 ++++++++++++++++---
 4 files changed, 97 insertions(+), 19 deletions(-)

diff --git a/tensorflow/compiler/xla/service/gpu/copy_thunk.cc b/tensorflow/compiler/xla/service/gpu/copy_thunk.cc
index 87858e9409..f4498663b1 100644
--- a/tensorflow/compiler/xla/service/gpu/copy_thunk.cc
+++ b/tensorflow/compiler/xla/service/gpu/copy_thunk.cc
@@ -20,15 +20,16 @@ limitations under the License.
 namespace xla {
 namespace gpu {
 
-CopyThunk::CopyThunk(const void* source_address,
-                     const BufferAllocation::Slice& destination_buffer,
-                     uint64 mem_size, const HloInstruction* hlo_instruction)
+HostToDeviceCopyThunk::HostToDeviceCopyThunk(
+    const void* source_address,
+    const BufferAllocation::Slice& destination_buffer, uint64 mem_size,
+    const HloInstruction* hlo_instruction)
     : Thunk(Kind::kCopy, hlo_instruction),
       source_address_(source_address),
       destination_buffer_(destination_buffer),
       mem_size_(mem_size) {}
 
-tensorflow::Status CopyThunk::ExecuteOnStream(
+tensorflow::Status HostToDeviceCopyThunk::ExecuteOnStream(
     const BufferAllocations& buffer_allocations,
     perftools::gputools::Stream* stream) {
   perftools::gputools::DeviceMemoryBase destination_data =
@@ -37,5 +38,24 @@ tensorflow::Status CopyThunk::ExecuteOnStream(
   return tensorflow::Status::OK();
 }
 
+DeviceToDeviceCopyThunk::DeviceToDeviceCopyThunk(
+    const BufferAllocation::Slice& source_buffer,
+    const BufferAllocation::Slice& destination_buffer, uint64 mem_size,
+    const HloInstruction* hlo_instruction)
+    : Thunk(Kind::kCopy, hlo_instruction),
+      source_buffer_(source_buffer),
+      destination_buffer_(destination_buffer),
+      mem_size_(mem_size) {}
+
+tensorflow::Status DeviceToDeviceCopyThunk::ExecuteOnStream(
+    const BufferAllocations& buffer_allocations,
+    perftools::gputools::Stream* stream) {
+  perftools::gputools::DeviceMemoryBase destination_data =
+      buffer_allocations.GetDeviceAddress(destination_buffer_);
+  perftools::gputools::DeviceMemoryBase source_data =
+      buffer_allocations.GetDeviceAddress(source_buffer_);
+  stream->ThenMemcpy(&destination_data, source_data, mem_size_);
+  return tensorflow::Status::OK();
+}
 }  // namespace gpu
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/service/gpu/copy_thunk.h b/tensorflow/compiler/xla/service/gpu/copy_thunk.h
index 6b8c432715..e2783fd255 100644
--- a/tensorflow/compiler/xla/service/gpu/copy_thunk.h
+++ b/tensorflow/compiler/xla/service/gpu/copy_thunk.h
@@ -26,19 +26,18 @@ limitations under the License.
 namespace xla {
 namespace gpu {
 
-// A thunk that copies data. For now, it copies data only from host to device.
-// But it can be extended to perform device-to-host or intra-device copying.
-class CopyThunk : public Thunk {
+// A thunk that copies data from a host buffer to a device buffer.
+class HostToDeviceCopyThunk : public Thunk {
  public:
   // Constructs a CopyThunk that copies host data from `source_address` to the
   // device buffer `destination_buffer`. `mem_size` is the size of the data in
   // bytes.
-  CopyThunk(const void* source_address,
-            const BufferAllocation::Slice& destination_buffer, uint64 mem_size,
-            const HloInstruction* hlo_instruction);
+  HostToDeviceCopyThunk(const void* source_address,
+                        const BufferAllocation::Slice& destination_buffer,
+                        uint64 mem_size, const HloInstruction* hlo_instruction);
 
-  CopyThunk(const CopyThunk&) = delete;
-  CopyThunk& operator=(const CopyThunk&) = delete;
+  HostToDeviceCopyThunk(const HostToDeviceCopyThunk&) = delete;
+  HostToDeviceCopyThunk& operator=(const HostToDeviceCopyThunk&) = delete;
 
   tensorflow::Status ExecuteOnStream(
       const BufferAllocations& buffer_allocations,
@@ -50,6 +49,30 @@ class CopyThunk : public Thunk {
   const uint64 mem_size_;
 };
 
+// A thunk that copies data from a device buffer to another device buffer.
+class DeviceToDeviceCopyThunk : public Thunk {
+ public:
+  // Constructs a CopyThunk that copies host data from `source_buffer` to the
+  // device buffer `destination_buffer`. `mem_size` is the size of the data in
+  // bytes.
+  DeviceToDeviceCopyThunk(const BufferAllocation::Slice& source_buffer,
+                          const BufferAllocation::Slice& destination_buffer,
+                          uint64 mem_size,
+                          const HloInstruction* hlo_instruction);
+
+  DeviceToDeviceCopyThunk(const DeviceToDeviceCopyThunk&) = delete;
+  DeviceToDeviceCopyThunk& operator=(const DeviceToDeviceCopyThunk&) = delete;
+
+  tensorflow::Status ExecuteOnStream(
+      const BufferAllocations& buffer_allocations,
+      perftools::gputools::Stream* stream) override;
+
+ private:
+  const BufferAllocation::Slice source_buffer_;
+  const BufferAllocation::Slice destination_buffer_;
+  const uint64 mem_size_;
+};
+
 }  // namespace gpu
 }  // namespace xla
 
diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter.h b/tensorflow/compiler/xla/service/gpu/ir_emitter.h
index 5e3f3bfdf1..29c3761dc5 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emitter.h
+++ b/tensorflow/compiler/xla/service/gpu/ir_emitter.h
@@ -339,8 +339,12 @@ class IrEmitterUnnested : public IrEmitter {
   // to make sure `inst` outlives the lifetime of the returned Thunk object.
   std::unique_ptr<Thunk> BuildGemmThunk(const HloInstruction* inst);
 
-  // Returns a CopyThunk that calls host-to-device cuMemcpy to implement `inst`.
-  std::unique_ptr<Thunk> BuildCopyThunk(const HloInstruction* inst);
+  // Returns a thunk that calls host-to-device cuMemcpy to implement `inst`.
+  std::unique_ptr<Thunk> BuildHostToDeviceCopyThunk(const HloInstruction* inst);
+
+  // Returns a thunk that calls device-to-device cuMemcpy to implement `inst`.
+  std::unique_ptr<Thunk> BuildDeviceToDeviceCopyThunk(
+      const HloInstruction* inst);
 
   // Returns an InfeedThunk that performs device-to-device memcpy to implement
   // `inst`.
diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
index 120d50ed25..734c793c15 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
+++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
@@ -146,7 +146,7 @@ Status IrEmitterUnnested::Postprocess(HloInstruction* hlo) {
 }
 
 namespace {
-bool ImplementedAsMemcpy(const HloInstruction& hlo) {
+bool ImplementedAsHostToDeviceMemcpy(const HloInstruction& hlo) {
   // `hlo` needs to satisfy three conditions to be implemented as a
   // host-to-device cuMemcpy.
   //
@@ -157,6 +157,20 @@ bool ImplementedAsMemcpy(const HloInstruction& hlo) {
          hlo.operand(0)->opcode() == HloOpcode::kConstant &&
          ShapeUtil::Equal(hlo.operand(0)->shape(), hlo.shape());
 }
+
+bool ImplementedAsDeviceToDeviceMemcpy(
+    const BufferAssignment& buffer_assignment, const HloInstruction& hlo) {
+  // `hlo` needs to satisfy three conditions to be implemented as a
+  // device-to-device cuMemcpy.
+  //
+  // 1. `hlo` is a kCopy instruction.
+  // 2. `hlo` and its operand have the same shape (thus the same layout too).
+  // 3. The operand to `hlo` has a buffer assignment (constants do not, for
+  //    instance) which means the source buffer also resides on the device.
+  return hlo.opcode() == HloOpcode::kCopy &&
+         ShapeUtil::Equal(hlo.operand(0)->shape(), hlo.shape()) &&
+         buffer_assignment.HasTopLevelAllocation(hlo.operand(0));
+}
 }  // namespace
 
 llvm::Function* IrEmitterUnnested::BuildKernelPrototype(
@@ -664,8 +678,13 @@ int64 EmitTranspose021Tiled(llvm_ir::IrArray input, llvm_ir::IrArray output,
 }  // namespace
 
 Status IrEmitterUnnested::HandleCopy(HloInstruction* copy) {
-  if (ImplementedAsMemcpy(*copy)) {
-    thunk_sequence_->emplace_back(BuildCopyThunk(copy));
+  if (ImplementedAsHostToDeviceMemcpy(*copy)) {
+    thunk_sequence_->emplace_back(BuildHostToDeviceCopyThunk(copy));
+    return Status::OK();
+  }
+  if (ImplementedAsDeviceToDeviceMemcpy(
+          ir_emitter_context_->buffer_assignment(), *copy)) {
+    thunk_sequence_->emplace_back(BuildDeviceToDeviceCopyThunk(copy));
     return Status::OK();
   }
   bool is_transpose_021;
@@ -1579,11 +1598,11 @@ std::unique_ptr<Thunk> IrEmitterUnnested::BuildKernelThunk(
                                  llvm_ir::AsString(kernel->getName()), inst);
 }
 
-std::unique_ptr<Thunk> IrEmitterUnnested::BuildCopyThunk(
+std::unique_ptr<Thunk> IrEmitterUnnested::BuildHostToDeviceCopyThunk(
     const HloInstruction* inst) {
   const HloInstruction* operand = inst->operand(0);
   CHECK_EQ(HloOpcode::kConstant, operand->opcode());
-  return MakeUnique<CopyThunk>(
+  return MakeUnique<HostToDeviceCopyThunk>(
       /*source_address=*/operand->literal().InternalData(),
       /*destination_buffer=*/GetAllocationSlice(*inst),
       /*mem_size=*/
@@ -1592,6 +1611,18 @@ std::unique_ptr<Thunk> IrEmitterUnnested::BuildCopyThunk(
       inst);
 }
 
+std::unique_ptr<Thunk> IrEmitterUnnested::BuildDeviceToDeviceCopyThunk(
+    const HloInstruction* inst) {
+  const HloInstruction* operand = inst->operand(0);
+  return MakeUnique<DeviceToDeviceCopyThunk>(
+      /*source_address=*/GetAllocationSlice(*operand),
+      /*destination_buffer=*/GetAllocationSlice(*inst),
+      /*mem_size=*/
+      llvm_ir::ByteSizeOf(operand->shape(),
+                          ir_emitter_context_->llvm_module()->getDataLayout()),
+      inst);
+}
+
 std::unique_ptr<Thunk> IrEmitterUnnested::BuildInfeedThunk(
     const HloInstruction* inst) {
   CHECK_EQ(HloOpcode::kInfeed, inst->opcode());
-- 
GitLab


From fb7892e6d0d749251415fe308c618667058c1c7c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 19 Oct 2017 10:44:04 -0700
Subject: [PATCH 554/909] Note that tfgan.eval.preprocess_image now expects
 inputs from [0, 255] range.

PiperOrigin-RevId: 172765720
---
 .../contrib/gan/python/eval/python/classifier_metrics_impl.py    | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tensorflow/contrib/gan/python/eval/python/classifier_metrics_impl.py b/tensorflow/contrib/gan/python/eval/python/classifier_metrics_impl.py
index 4af87b8b47..f7c70db0e0 100644
--- a/tensorflow/contrib/gan/python/eval/python/classifier_metrics_impl.py
+++ b/tensorflow/contrib/gan/python/eval/python/classifier_metrics_impl.py
@@ -107,7 +107,6 @@ def _symmetric_matrix_square_root(mat, eps=1e-10):
 
 
 # Convenience preprocessing function, with fixed defaults.
-# NOTE: Floating-point inputs are expected to be in [0, 1].
 # Copied from /tensorflow_models/slim/preprocessing/inception_preprocessing.py.
 def preprocess_image(
     images, height=INCEPTION_DEFAULT_IMAGE_SIZE,
-- 
GitLab


From 89f15928293f62b63404ffbdabbb6a3b07274ff8 Mon Sep 17 00:00:00 2001
From: chi-hung <chi-hung.weng@gmx.de>
Date: Fri, 20 Oct 2017 02:10:26 +0800
Subject: [PATCH 555/909] fix the issue: "libcuda.so.1 not found" and add some
 minor changes. (#13811)

* fix the issue: "libcuda.so.1 not found" and add some minor changes.

* Install wheel instead of python-wheel. Also, unnecessary lines are removed.
---
 .../docker/Dockerfile.devel-gpu-cuda9-cudnn7  | 33 ++++++++++++-------
 1 file changed, 22 insertions(+), 11 deletions(-)

diff --git a/tensorflow/tools/docker/Dockerfile.devel-gpu-cuda9-cudnn7 b/tensorflow/tools/docker/Dockerfile.devel-gpu-cuda9-cudnn7
index 75351ecfba..4558bc5293 100644
--- a/tensorflow/tools/docker/Dockerfile.devel-gpu-cuda9-cudnn7
+++ b/tensorflow/tools/docker/Dockerfile.devel-gpu-cuda9-cudnn7
@@ -42,6 +42,7 @@ RUN pip --no-cache-dir install \
         scipy \
         sklearn \
         pandas \
+        wheel \
         && \
     python -m ipykernel.kernelspec
 
@@ -80,22 +81,32 @@ RUN git clone https://github.com/tensorflow/tensorflow.git && \
 WORKDIR /tensorflow
 
 # Configure the build for our CUDA configuration.
-ENV CI_BUILD_PYTHON python
-ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH
-ENV TF_NEED_CUDA 1
-ENV TF_CUDA_COMPUTE_CAPABILITIES 3.0,3.5,5.2,6.0,6.1
-ENV TF_CUDA_VERSION 9.0
-ENV TF_CUDNN_VERSION 7
+ENV CI_BUILD_PYTHON=python \
+    LD_LIBRARY_PATH=/usr/local/cuda/extras/CUPTI/lib64:${LD_LIBRARY_PATH} \
+    CUDNN_INSTALL_PATH=/usr/lib/x86_64-linux-gnu \
+    PYTHON_BIN_PATH=/usr/bin/python \
+    PYTHON_LIB_PATH=/usr/local/lib/python2.7/dist-packages \
+    TF_NEED_CUDA=1 \
+    TF_CUDA_VERSION=9.0 \
+    TF_CUDA_COMPUTE_CAPABILITIES=3.0,3.5,5.2,6.0,6.1,7.0 \
+    TF_CUDNN_VERSION=7
 RUN ./configure
 
-RUN LD_LIBRARY_PATH=/usr/local/cuda/lib64/stubs:${LD_LIBRARY_PATH} \
-    bazel build -c opt --config=cuda --cxxopt="-D_GLIBCXX_USE_CXX11_ABI=0" \
-        --jobs=${TF_AVAILABLE_CPUS} \
-        tensorflow/tools/pip_package:build_pip_package && \
-    mkdir -p /pip_pkg && \
+# Build and Install TensorFlow.
+RUN ln -s /usr/local/cuda/lib64/stubs/libcuda.so /usr/local/cuda/lib64/stubs/libcuda.so.1 && \
+    LD_LIBRARY_PATH=/usr/local/cuda/lib64/stubs:${LD_LIBRARY_PATH} \
+    bazel build -c opt \
+                --config=cuda \
+                --cxxopt="-D_GLIBCXX_USE_CXX11_ABI=0" \
+                --jobs=${TF_AVAILABLE_CPUS} \
+                tensorflow/tools/pip_package:build_pip_package && \
+    mkdir /pip_pkg && \
     bazel-bin/tensorflow/tools/pip_package/build_pip_package /pip_pkg
 
+# Clean up pip wheel and Bazel cache when done.
 RUN pip --no-cache-dir install --upgrade /pip_pkg/tensorflow-*.whl && \
+    rm -rf /pip_pkg && \
+    rm -rf /root/.cache
 
 WORKDIR /root
 
-- 
GitLab


From 31587244e4821fbb4eebcf7847281a1df3da6a2a Mon Sep 17 00:00:00 2001
From: Igor Saprykin <isaprykin@google.com>
Date: Thu, 19 Oct 2017 11:22:37 -0700
Subject: [PATCH 556/909] Remove sklearn from text_classification_cnn.py.

PiperOrigin-RevId: 172772457
---
 .../examples/learn/text_classification_cnn.py       | 13 ++-----------
 1 file changed, 2 insertions(+), 11 deletions(-)

diff --git a/tensorflow/examples/learn/text_classification_cnn.py b/tensorflow/examples/learn/text_classification_cnn.py
index 0ee2405c8b..be262285a3 100644
--- a/tensorflow/examples/learn/text_classification_cnn.py
+++ b/tensorflow/examples/learn/text_classification_cnn.py
@@ -22,7 +22,6 @@ import sys
 
 import numpy as np
 import pandas
-from sklearn import metrics
 import tensorflow as tf
 
 FLAGS = None
@@ -134,23 +133,15 @@ def main(unused_argv):
       shuffle=True)
   classifier.train(input_fn=train_input_fn, steps=100)
 
-  # Predict.
+  # Evaluate.
   test_input_fn = tf.estimator.inputs.numpy_input_fn(
       x={WORDS_FEATURE: x_test},
       y=y_test,
       num_epochs=1,
       shuffle=False)
-  predictions = classifier.predict(input_fn=test_input_fn)
-  y_predicted = np.array(list(p['class'] for p in predictions))
-  y_predicted = y_predicted.reshape(np.array(y_test).shape)
 
-  # Score with sklearn.
-  score = metrics.accuracy_score(y_test, y_predicted)
-  print('Accuracy (sklearn): {0:f}'.format(score))
-
-  # Score with tensorflow.
   scores = classifier.evaluate(input_fn=test_input_fn)
-  print('Accuracy (tensorflow): {0:f}'.format(scores['accuracy']))
+  print('Accuracy: {0:f}'.format(scores['accuracy']))
 
 
 if __name__ == '__main__':
-- 
GitLab


From 1c878e69cbb6276ee38364e42fde4db2239e15a6 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 19 Oct 2017 11:39:07 -0700
Subject: [PATCH 557/909] Fixed small typo in a docstring.

PiperOrigin-RevId: 172775242
---
 tensorflow/python/ops/nn_ops.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/ops/nn_ops.py b/tensorflow/python/ops/nn_ops.py
index 8876591e53..5f82323bfc 100644
--- a/tensorflow/python/ops/nn_ops.py
+++ b/tensorflow/python/ops/nn_ops.py
@@ -2114,7 +2114,7 @@ def dropout(x, keep_prob, noise_shape=None, seed=None, name=None):  # pylint: di
 def top_k(input, k=1, sorted=True, name=None):
   """Finds values and indices of the `k` largest entries for the last dimension.
 
-  If the input is a vector (rank-1), finds the `k` largest entries in the vector
+  If the input is a vector (rank=1), finds the `k` largest entries in the vector
   and outputs their values and indices as vectors.  Thus `values[j]` is the
   `j`-th largest entry in `input`, and its index is `indices[j]`.
 
-- 
GitLab


From e6e21eb8203183c32dfc156566b58338d5ba204b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 19 Oct 2017 12:20:03 -0700
Subject: [PATCH 558/909] internal change.

PiperOrigin-RevId: 172780953
---
 .../boosted_trees/lib/utils/batch_features.h     | 16 ++++++++++++++++
 .../resources/decision_tree_ensemble_resource.h  |  3 ++-
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/boosted_trees/lib/utils/batch_features.h b/tensorflow/contrib/boosted_trees/lib/utils/batch_features.h
index bb11dc9a07..7a550d6f73 100644
--- a/tensorflow/contrib/boosted_trees/lib/utils/batch_features.h
+++ b/tensorflow/contrib/boosted_trees/lib/utils/batch_features.h
@@ -45,6 +45,22 @@ class BatchFeatures {
                     std::vector<Tensor> sparse_int_feature_values_list,
                     std::vector<Tensor> sparse_int_feature_shapes_list);
 
+  Status GetFeatureColumnSizes(int64* const num_dense_float_features,
+                               int64* const num_sparse_float_features,
+                               int64* const num_sparse_int_features) const {
+    QCHECK_NE(num_dense_float_features, nullptr);
+    QCHECK_NE(num_sparse_float_features, nullptr);
+    QCHECK_NE(num_sparse_int_features, nullptr);
+    *num_dense_float_features = dense_float_feature_columns_.size();
+    *num_sparse_float_features = sparse_float_feature_columns_.size();
+    *num_sparse_int_features = sparse_int_feature_columns_.size();
+    if (*num_dense_float_features == 0 && *num_sparse_float_features == 0 &&
+        *num_sparse_int_features == 0) {
+      return errors::FailedPrecondition("Not intialized yet.");
+    }
+    return Status::OK();
+  }
+
   // Creates an example iterable for the requested slice.
   ExamplesIterable examples_iterable(int64 example_start,
                                      int64 example_end) const {
diff --git a/tensorflow/contrib/boosted_trees/resources/decision_tree_ensemble_resource.h b/tensorflow/contrib/boosted_trees/resources/decision_tree_ensemble_resource.h
index 77e6ecb443..284ad5cdb9 100644
--- a/tensorflow/contrib/boosted_trees/resources/decision_tree_ensemble_resource.h
+++ b/tensorflow/contrib/boosted_trees/resources/decision_tree_ensemble_resource.h
@@ -47,6 +47,7 @@ class DecisionTreeEnsembleResource : public StampedResource {
   int32 num_trees() const { return decision_tree_ensemble_->trees_size(); }
 
   bool InitFromSerialized(const string& serialized, const int64 stamp_token) {
+    CHECK_EQ(stamp(), -1) << "Must Reset before Init.";
     if (ParseProtoUnlimited(decision_tree_ensemble_, serialized)) {
       set_stamp(stamp_token);
       return true;
@@ -126,7 +127,7 @@ class DecisionTreeEnsembleResource : public StampedResource {
 
   // Resets the resource and frees the protos in arena.
   // Caller needs to hold the mutex lock while calling this.
-  void Reset() {
+  virtual void Reset() {
     // Reset stamp.
     set_stamp(-1);
 
-- 
GitLab


From 618e28759b0647f7ef98c70f872f4d6efa8e2002 Mon Sep 17 00:00:00 2001
From: Asim Shankar <ashankar@google.com>
Date: Thu, 19 Oct 2017 12:28:58 -0700
Subject: [PATCH 559/909] eager: Expose the Network class.

PiperOrigin-RevId: 172782028
---
 tensorflow/contrib/eager/python/tfe.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tensorflow/contrib/eager/python/tfe.py b/tensorflow/contrib/eager/python/tfe.py
index c519df8b5c..353b9d2bda 100644
--- a/tensorflow/contrib/eager/python/tfe.py
+++ b/tensorflow/contrib/eager/python/tfe.py
@@ -43,6 +43,7 @@ To use, at program startup, call `tfe.enable_eager_execution()`.
 @@seterr
 
 @@Iterator
+@@Network
 @@Saver
 @@SummaryWriter
 @@restore_variables_on_create
@@ -62,6 +63,7 @@ from __future__ import print_function
 # pylint:disable=g-bad-import-order,g-import-not-at-top,unused-import
 #
 from tensorflow.contrib.eager.python.datasets import Iterator
+from tensorflow.contrib.eager.python.network import Network
 from tensorflow.contrib.eager.python.saver import restore_variables_on_create
 from tensorflow.contrib.eager.python.saver import Saver
 from tensorflow.contrib.eager.python.summary_writer import SummaryWriter
-- 
GitLab


From a23abe0c226ec1b99f8bddcb2fba65a46365684b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 19 Oct 2017 12:20:03 -0700
Subject: [PATCH 560/909] internal change.

PiperOrigin-RevId: 172780953
---
 tensorflow/contrib/eager/python/tfe.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/tensorflow/contrib/eager/python/tfe.py b/tensorflow/contrib/eager/python/tfe.py
index 353b9d2bda..c519df8b5c 100644
--- a/tensorflow/contrib/eager/python/tfe.py
+++ b/tensorflow/contrib/eager/python/tfe.py
@@ -43,7 +43,6 @@ To use, at program startup, call `tfe.enable_eager_execution()`.
 @@seterr
 
 @@Iterator
-@@Network
 @@Saver
 @@SummaryWriter
 @@restore_variables_on_create
@@ -63,7 +62,6 @@ from __future__ import print_function
 # pylint:disable=g-bad-import-order,g-import-not-at-top,unused-import
 #
 from tensorflow.contrib.eager.python.datasets import Iterator
-from tensorflow.contrib.eager.python.network import Network
 from tensorflow.contrib.eager.python.saver import restore_variables_on_create
 from tensorflow.contrib.eager.python.saver import Saver
 from tensorflow.contrib.eager.python.summary_writer import SummaryWriter
-- 
GitLab


From 435995bc4ec221ba31a60bbebc20921a8dbd98bd Mon Sep 17 00:00:00 2001
From: Asim Shankar <ashankar@google.com>
Date: Thu, 19 Oct 2017 12:28:58 -0700
Subject: [PATCH 561/909] eager: Expose the Network class.

PiperOrigin-RevId: 172782028
---
 tensorflow/contrib/eager/python/tfe.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tensorflow/contrib/eager/python/tfe.py b/tensorflow/contrib/eager/python/tfe.py
index c519df8b5c..353b9d2bda 100644
--- a/tensorflow/contrib/eager/python/tfe.py
+++ b/tensorflow/contrib/eager/python/tfe.py
@@ -43,6 +43,7 @@ To use, at program startup, call `tfe.enable_eager_execution()`.
 @@seterr
 
 @@Iterator
+@@Network
 @@Saver
 @@SummaryWriter
 @@restore_variables_on_create
@@ -62,6 +63,7 @@ from __future__ import print_function
 # pylint:disable=g-bad-import-order,g-import-not-at-top,unused-import
 #
 from tensorflow.contrib.eager.python.datasets import Iterator
+from tensorflow.contrib.eager.python.network import Network
 from tensorflow.contrib.eager.python.saver import restore_variables_on_create
 from tensorflow.contrib.eager.python.saver import Saver
 from tensorflow.contrib.eager.python.summary_writer import SummaryWriter
-- 
GitLab


From 3f063271c5a862e8c5deeb5a3738ca20db7771d5 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 19 Oct 2017 12:33:22 -0700
Subject: [PATCH 562/909] Improve inception score documentation.

PiperOrigin-RevId: 172782572
---
 .../gan/python/eval/python/classifier_metrics_impl.py        | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/gan/python/eval/python/classifier_metrics_impl.py b/tensorflow/contrib/gan/python/eval/python/classifier_metrics_impl.py
index f7c70db0e0..d4c080cab3 100644
--- a/tensorflow/contrib/gan/python/eval/python/classifier_metrics_impl.py
+++ b/tensorflow/contrib/gan/python/eval/python/classifier_metrics_impl.py
@@ -106,8 +106,6 @@ def _symmetric_matrix_square_root(mat, eps=1e-10):
       math_ops.matmul(u, array_ops.diag(si)), v, transpose_b=True)
 
 
-# Convenience preprocessing function, with fixed defaults.
-# Copied from /tensorflow_models/slim/preprocessing/inception_preprocessing.py.
 def preprocess_image(
     images, height=INCEPTION_DEFAULT_IMAGE_SIZE,
     width=INCEPTION_DEFAULT_IMAGE_SIZE, scope=None):
@@ -116,6 +114,9 @@ def preprocess_image(
   This is the preprocessing portion of the graph from
   http://download.tensorflow.org/models/image/imagenet/inception-2015-12-05.tgz.
 
+  Note that it expects Tensors in [0, 255]. This function maps pixel values to
+  [-1, 1] and resizes to match the InceptionV1 network.
+
   Args:
     images: 3-D or 4-D Tensor of images. Values are in [0, 255].
     height: Integer. Height of resized output image.
-- 
GitLab


From 013125dc926bafab43db507d18f255cd25503f11 Mon Sep 17 00:00:00 2001
From: Kay Zhu <kayzhu@google.com>
Date: Thu, 19 Oct 2017 12:46:01 -0700
Subject: [PATCH 563/909] [XLA] Documentation: use more correct term for
 fractionally strided convolutions / LHS dilation.

PiperOrigin-RevId: 172784169
---
 tensorflow/docs_src/performance/xla/operation_semantics.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/docs_src/performance/xla/operation_semantics.md b/tensorflow/docs_src/performance/xla/operation_semantics.md
index 52258cbae7..91c0d5b8c6 100644
--- a/tensorflow/docs_src/performance/xla/operation_semantics.md
+++ b/tensorflow/docs_src/performance/xla/operation_semantics.md
@@ -328,9 +328,9 @@ placed between each of the entries in that dimension, increasing the size of the
 array. The holes are filled with a no-op value, which for convolution means
 zeroes.
 
-Dilation of the rhs is also called atrous convolution. For more details, see the
-@{tf.nn.atrous_conv2d}. Dilation of the lhs is
-also called deconvolution.
+Dilation of the rhs is also called atrous convolution. For more details, see
+@{tf.nn.atrous_conv2d}. Dilation of the lhs is also called transposed
+convolution. For more details, see @{tf.nn.conv2d_transpose}.
 
 The output shape has these dimensions, in this order:
 
-- 
GitLab


From f843da7cb24501f609d3ce46e7279ddd8c34e338 Mon Sep 17 00:00:00 2001
From: David Soergel <soergel@google.com>
Date: Thu, 19 Oct 2017 13:11:53 -0700
Subject: [PATCH 564/909] Update serving_input_fn argument name to
 serving_input_receiver_fn

PiperOrigin-RevId: 172787460
---
 tensorflow/python/estimator/exporter.py       | 28 ++++++++++---------
 tensorflow/python/estimator/exporter_test.py  | 20 ++++++-------
 tensorflow/python/estimator/training_test.py  |  2 +-
 ...tensorflow.estimator.-final-exporter.pbtxt |  2 +-
 ...ensorflow.estimator.-latest-exporter.pbtxt |  2 +-
 5 files changed, 28 insertions(+), 26 deletions(-)

diff --git a/tensorflow/python/estimator/exporter.py b/tensorflow/python/estimator/exporter.py
index 56400ab935..c6f20d4a9e 100644
--- a/tensorflow/python/estimator/exporter.py
+++ b/tensorflow/python/estimator/exporter.py
@@ -71,7 +71,7 @@ class _SavedModelExporter(Exporter):
 
   def __init__(self,
                name,
-               serving_input_fn,
+               serving_input_receiver_fn,
                assets_extra=None,
                as_text=False):
     """Create an `Exporter` to use with `tf.estimator.EvalSpec`.
@@ -79,8 +79,8 @@ class _SavedModelExporter(Exporter):
     Args:
       name: unique name of this `Exporter` that is going to be used in the
         export path.
-      serving_input_fn: a function that takes no arguments and returns an
-        `ServingInputReceiver`.
+      serving_input_receiver_fn: a function that takes no arguments and returns
+        a `ServingInputReceiver`.
       assets_extra: An optional dict specifying how to populate the assets.extra
         directory within the exported SavedModel.  Each key should give the
         destination path (including the filename) relative to the assets.extra
@@ -95,7 +95,7 @@ class _SavedModelExporter(Exporter):
       ValueError: if any arguments is invalid.
     """
     self._name = name
-    self._serving_input_fn = serving_input_fn
+    self._serving_input_receiver_fn = serving_input_receiver_fn
     self._assets_extra = assets_extra
     self._as_text = as_text
 
@@ -109,7 +109,7 @@ class _SavedModelExporter(Exporter):
 
     export_result = estimator.export_savedmodel(
         export_path,
-        self._serving_input_fn,
+        self._serving_input_receiver_fn,
         assets_extra=self._assets_extra,
         as_text=self._as_text,
         checkpoint_path=checkpoint_path)
@@ -125,7 +125,7 @@ class FinalExporter(Exporter):
 
   def __init__(self,
                name,
-               serving_input_fn,
+               serving_input_receiver_fn,
                assets_extra=None,
                as_text=False):
     """Create an `Exporter` to use with `tf.estimator.EvalSpec`.
@@ -133,8 +133,8 @@ class FinalExporter(Exporter):
     Args:
       name: unique name of this `Exporter` that is going to be used in the
         export path.
-      serving_input_fn: a function that takes no arguments and returns an
-        `ServingInputReceiver`.
+      serving_input_receiver_fn: a function that takes no arguments and returns
+        a `ServingInputReceiver`.
       assets_extra: An optional dict specifying how to populate the assets.extra
         directory within the exported SavedModel.  Each key should give the
         destination path (including the filename) relative to the assets.extra
@@ -148,7 +148,8 @@ class FinalExporter(Exporter):
     Raises:
       ValueError: if any arguments is invalid.
     """
-    self._saved_model_exporter = _SavedModelExporter(name, serving_input_fn,
+    self._saved_model_exporter = _SavedModelExporter(name,
+                                                     serving_input_receiver_fn,
                                                      assets_extra, as_text)
 
   @property
@@ -175,7 +176,7 @@ class LatestExporter(Exporter):
 
   def __init__(self,
                name,
-               serving_input_fn,
+               serving_input_receiver_fn,
                assets_extra=None,
                as_text=False,
                exports_to_keep=5):
@@ -184,8 +185,8 @@ class LatestExporter(Exporter):
     Args:
       name: unique name of this `Exporter` that is going to be used in the
         export path.
-      serving_input_fn: a function that takes no arguments and returns an
-        `ServingInputReceiver`.
+      serving_input_receiver_fn: a function that takes no arguments and returns
+        a `ServingInputReceiver`.
       assets_extra: An optional dict specifying how to populate the assets.extra
         directory within the exported SavedModel.  Each key should give the
         destination path (including the filename) relative to the assets.extra
@@ -202,7 +203,8 @@ class LatestExporter(Exporter):
     Raises:
       ValueError: if any arguments is invalid.
     """
-    self._saved_model_exporter = _SavedModelExporter(name, serving_input_fn,
+    self._saved_model_exporter = _SavedModelExporter(name,
+                                                     serving_input_receiver_fn,
                                                      assets_extra, as_text)
     self._exports_to_keep = exports_to_keep
     if exports_to_keep is not None and exports_to_keep <= 0:
diff --git a/tensorflow/python/estimator/exporter_test.py b/tensorflow/python/estimator/exporter_test.py
index f90c35dce7..8e0f66cece 100644
--- a/tensorflow/python/estimator/exporter_test.py
+++ b/tensorflow/python/estimator/exporter_test.py
@@ -33,19 +33,19 @@ from tensorflow.python.util import compat
 class LatestExporterTest(test.TestCase):
 
   def test_error_out_if_exports_to_keep_is_zero(self):
-    def _serving_input_fn():
+    def _serving_input_receiver_fn():
       pass
 
     with self.assertRaisesRegexp(ValueError, "positive number"):
       exporter = exporter_lib.LatestExporter(
           name="latest_exporter",
-          serving_input_fn=_serving_input_fn,
+          serving_input_receiver_fn=_serving_input_receiver_fn,
           exports_to_keep=0)
       self.assertEqual("latest_exporter", exporter.name)
 
   def test_latest_exporter(self):
 
-    def _serving_input_fn():
+    def _serving_input_receiver_fn():
       pass
 
     export_dir_base = tempfile.mkdtemp() + "export/"
@@ -53,7 +53,7 @@ class LatestExporterTest(test.TestCase):
 
     exporter = exporter_lib.LatestExporter(
         name="latest_exporter",
-        serving_input_fn=_serving_input_fn,
+        serving_input_receiver_fn=_serving_input_receiver_fn,
         assets_extra={"from/path": "to/path"},
         as_text=False,
         exports_to_keep=5)
@@ -66,14 +66,14 @@ class LatestExporterTest(test.TestCase):
     self.assertEqual("export_result_path", export_result)
     estimator.export_savedmodel.assert_called_with(
         export_dir_base,
-        _serving_input_fn,
+        _serving_input_receiver_fn,
         assets_extra={"from/path": "to/path"},
         as_text=False,
         checkpoint_path="checkpoint_path")
 
   def test_only_the_last_export_is_saved(self):
 
-    def _serving_input_fn():
+    def _serving_input_receiver_fn():
       pass
 
     export_dir_base = tempfile.mkdtemp() + "export/"
@@ -81,7 +81,7 @@ class LatestExporterTest(test.TestCase):
 
     exporter = exporter_lib.FinalExporter(
         name="latest_exporter",
-        serving_input_fn=_serving_input_fn,
+        serving_input_receiver_fn=_serving_input_receiver_fn,
         assets_extra={"from/path": "to/path"},
         as_text=False)
     estimator = test.mock.Mock(spec=estimator_lib.Estimator)
@@ -99,7 +99,7 @@ class LatestExporterTest(test.TestCase):
     self.assertEqual("export_result_path", export_result)
     estimator.export_savedmodel.assert_called_with(
         export_dir_base,
-        _serving_input_fn,
+        _serving_input_receiver_fn,
         assets_extra={"from/path": "to/path"},
         as_text=False,
         checkpoint_path="checkpoint_path")
@@ -117,12 +117,12 @@ class LatestExporterTest(test.TestCase):
     self.assertTrue(gfile.Exists(export_dir_3))
     self.assertTrue(gfile.Exists(export_dir_4))
 
-    def _serving_input_fn():
+    def _serving_input_receiver_fn():
       return array_ops.constant([1]), None
 
     exporter = exporter_lib.LatestExporter(
         name="latest_exporter",
-        serving_input_fn=_serving_input_fn,
+        serving_input_receiver_fn=_serving_input_receiver_fn,
         exports_to_keep=2)
     estimator = test.mock.Mock(spec=estimator_lib.Estimator)
     # Garbage collect all but the most recent 2 exports,
diff --git a/tensorflow/python/estimator/training_test.py b/tensorflow/python/estimator/training_test.py
index d88ca2c925..1862e325e2 100644
--- a/tensorflow/python/estimator/training_test.py
+++ b/tensorflow/python/estimator/training_test.py
@@ -1569,7 +1569,7 @@ class TrainAndEvaluateIntegrationTest(test.TestCase):
     serving_input_receiver_fn = (
         export_lib.build_parsing_serving_input_receiver_fn(feature_spec))
     return exporter_lib.LatestExporter(
-        name, serving_input_fn=serving_input_receiver_fn)
+        name, serving_input_receiver_fn=serving_input_receiver_fn)
 
   def _extract_loss_and_global_step(self, event_folder):
     """Returns the loss and global step in last event."""
diff --git a/tensorflow/tools/api/golden/tensorflow.estimator.-final-exporter.pbtxt b/tensorflow/tools/api/golden/tensorflow.estimator.-final-exporter.pbtxt
index 4c2dbc4d37..ee37b1fa21 100644
--- a/tensorflow/tools/api/golden/tensorflow.estimator.-final-exporter.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.estimator.-final-exporter.pbtxt
@@ -9,7 +9,7 @@ tf_class {
   }
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'name\', \'serving_input_fn\', \'assets_extra\', \'as_text\'], varargs=None, keywords=None, defaults=[\'None\', \'False\'], "
+    argspec: "args=[\'self\', \'name\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\'], varargs=None, keywords=None, defaults=[\'None\', \'False\'], "
   }
   member_method {
     name: "export"
diff --git a/tensorflow/tools/api/golden/tensorflow.estimator.-latest-exporter.pbtxt b/tensorflow/tools/api/golden/tensorflow.estimator.-latest-exporter.pbtxt
index ae1483bf3f..2a9d029029 100644
--- a/tensorflow/tools/api/golden/tensorflow.estimator.-latest-exporter.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.estimator.-latest-exporter.pbtxt
@@ -9,7 +9,7 @@ tf_class {
   }
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'name\', \'serving_input_fn\', \'assets_extra\', \'as_text\', \'exports_to_keep\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'5\'], "
+    argspec: "args=[\'self\', \'name\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'exports_to_keep\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'5\'], "
   }
   member_method {
     name: "export"
-- 
GitLab


From 3472340ecfba0b960e948fcbaed42d1c08d87b6b Mon Sep 17 00:00:00 2001
From: Jonathan Hseu <jhseu@google.com>
Date: Thu, 19 Oct 2017 13:39:41 -0700
Subject: [PATCH 565/909] Set evaluation_master to master if not set. The
 current default confuses a lot of users.

Delete the tf_random_seed default since it was updated to None in core in
cl/172519268.

PiperOrigin-RevId: 172791174
---
 tensorflow/contrib/tpu/python/tpu/tpu_config.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_config.py b/tensorflow/contrib/tpu/python/tpu/tpu_config.py
index 79fd8b839b..3965c087a1 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_config.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_config.py
@@ -78,22 +78,24 @@ class TPUConfig(
 class RunConfig(run_config_lib.RunConfig):
   """RunConfig with TPU support."""
 
-  def __init__(self, tpu_config=None, evaluation_master='', master='',
-               tf_random_seed=None, **kwargs):
+  def __init__(self, tpu_config=None, evaluation_master=None, master='',
+               **kwargs):
     """Constructs a RunConfig.
 
     Args:
       tpu_config: the TPUConfig that specifies TPU-specific configuration.
       evaluation_master: a string. The address of the master to use for eval.
+        Defaults to master if not set.
       master: a string. The address of the master to use for training.
       tf_random_seed: an int. Sets the TensorFlow random seed. Defaults to None,
         which initializes it randomly based on the environment.
     """
-    # We change the default random seed to None because that's a better default.
-    kwargs['tf_random_seed'] = tf_random_seed
     super(RunConfig, self).__init__(**kwargs)
     self._tpu_config = tpu_config or TPUConfig()
-    self._evaluation_master = evaluation_master
+    if evaluation_master is None:
+      self._evaluation_master = master
+    else:
+      self._evaluation_master = evaluation_master
     self._master = master
 
   @property
-- 
GitLab


From f2aa6c0777369700a9dd79a0c22d7f3f7dcb0835 Mon Sep 17 00:00:00 2001
From: Andrew Harp <andrewharp@google.com>
Date: Thu, 19 Oct 2017 13:49:21 -0700
Subject: [PATCH 566/909] Log initialization and warmup time to proto results
 in benchmark tool.

PiperOrigin-RevId: 172792563
---
 tensorflow/tools/benchmark/benchmark_model.cc | 60 ++++++++++++++-----
 1 file changed, 44 insertions(+), 16 deletions(-)

diff --git a/tensorflow/tools/benchmark/benchmark_model.cc b/tensorflow/tools/benchmark/benchmark_model.cc
index f84ae5c7ce..2d59299da4 100644
--- a/tensorflow/tools/benchmark/benchmark_model.cc
+++ b/tensorflow/tools/benchmark/benchmark_model.cc
@@ -230,6 +230,23 @@ Status CalculateFlops(const GraphDef& graph,
   return Status::OK();
 }
 
+void RecordBenchmarkEntry(const string& output_prefix,
+                          const string& benchmark_name, const string& postfix,
+                          int num_runs, double total_time_s,
+                          double throughput = -1.0) {
+  std::stringstream stream;
+  stream << benchmark_name;
+  if (!postfix.empty()) {
+    stream << "_" << postfix;
+  }
+
+  TestReporter node_reporter(output_prefix, stream.str());
+  TF_QCHECK_OK(node_reporter.Initialize());
+  TF_QCHECK_OK(
+      node_reporter.Benchmark(num_runs, -1.0, total_time_s, throughput));
+  TF_QCHECK_OK(node_reporter.Close());
+}
+
 Status RunBenchmark(const std::vector<InputLayerInfo>& inputs,
                     const std::vector<string>& outputs, Session* session,
                     StatSummarizer* stats, int64* inference_time_us) {
@@ -350,7 +367,7 @@ int Main(int argc, char** argv) {
   bool show_type = true;
   bool show_summary = true;
   bool show_flops = false;
-  int warmup_runs = 2;
+  int warmup_runs = 1;
 
   std::vector<Flag> flag_list = {
       Flag("graph", &graph, "graph file name"),
@@ -441,8 +458,14 @@ int Main(int argc, char** argv) {
   std::unique_ptr<Session> session;
   std::unique_ptr<StatSummarizer> stats;
   std::unique_ptr<GraphDef> graph_def;
+
+  int64 initialization_start_us = Env::Default()->NowMicros();
   Status initialize_status =
       InitializeSession(num_threads, graph, &session, &graph_def);
+  int64 initialization_end_us = Env::Default()->NowMicros();
+  double initialization_time_s =
+      (initialization_end_us - initialization_start_us) / 1000000.0;
+  LOG(INFO) << "Initialized session in " << initialization_time_s << "s";
   if (!initialize_status.ok()) {
     return -1;
   }
@@ -587,11 +610,23 @@ int Main(int argc, char** argv) {
         static_cast<double>(no_stat_wall_time) / (1024 * 1024);
 
     // Report the stats.
-    TestReporter reporter(output_prefix, benchmark_name);
-    TF_QCHECK_OK(reporter.Initialize());
-    TF_QCHECK_OK(reporter.Benchmark(no_stat_num_runs, -1.0, no_stat_wall_time,
-                                    throughput));
-    TF_QCHECK_OK(reporter.Close());
+    RecordBenchmarkEntry(output_prefix, benchmark_name, "", no_stat_num_runs,
+                         no_stat_wall_time, throughput);
+
+    // Session initialization time.
+    RecordBenchmarkEntry(output_prefix, benchmark_name, "meta-init", 1,
+                         initialization_time_s);
+
+    // First inference time. Note: if warmup_runs is > 1 this will actually be
+    // an average of all the warmup runs.
+    RecordBenchmarkEntry(output_prefix, benchmark_name, "meta-first-inference",
+                         warmup_runs, warmup_time_us / 1000000.0);
+
+    // Time from starting to intialize TF to getting the first result back.
+    // This also assumes that only one warmup run is performed.
+    RecordBenchmarkEntry(
+        output_prefix, benchmark_name, "meta-init-plus-first-inference", 1,
+        initialization_time_s + (warmup_time_us / 1000000.0) / warmup_runs);
 
     std::map<string, int64> node_type_map_count;
     std::map<string, int64> node_type_map_time;
@@ -603,17 +638,10 @@ int Main(int argc, char** argv) {
                               &node_type_map_memory,
                               &node_type_map_times_called, &accumulated_us);
     for (const auto& time : node_type_map_time) {
-      std::stringstream stream;
-      stream << benchmark_name << "_" << time.first;
-      TestReporter node_reporter(output_prefix, stream.str());
-
       LOG(INFO) << "Outputting: [" << time.first << "]";
-
-      TF_QCHECK_OK(node_reporter.Initialize());
-      TF_QCHECK_OK(node_reporter.Benchmark(
-          stat_num_runs, -1.0, (time.second * stat_num_runs) / 1000000.0f,
-          -1.0));
-      TF_QCHECK_OK(node_reporter.Close());
+      RecordBenchmarkEntry(output_prefix, benchmark_name, time.first,
+                           stat_num_runs,
+                           (time.second * stat_num_runs) / 1000000.0f);
     }
   }
 
-- 
GitLab


From 355ec38d80b14353e52b8de9f9db276e18f53e13 Mon Sep 17 00:00:00 2001
From: Jonathan Hseu <jhseu@google.com>
Date: Thu, 19 Oct 2017 13:39:41 -0700
Subject: [PATCH 567/909] Set evaluation_master to master if not set. The
 current default confuses a lot of users.

Delete the tf_random_seed default since it was updated to None in core in
cl/172519268.

PiperOrigin-RevId: 172791174
---
 tensorflow/tools/benchmark/benchmark_model.cc | 60 +++++--------------
 1 file changed, 16 insertions(+), 44 deletions(-)

diff --git a/tensorflow/tools/benchmark/benchmark_model.cc b/tensorflow/tools/benchmark/benchmark_model.cc
index 2d59299da4..f84ae5c7ce 100644
--- a/tensorflow/tools/benchmark/benchmark_model.cc
+++ b/tensorflow/tools/benchmark/benchmark_model.cc
@@ -230,23 +230,6 @@ Status CalculateFlops(const GraphDef& graph,
   return Status::OK();
 }
 
-void RecordBenchmarkEntry(const string& output_prefix,
-                          const string& benchmark_name, const string& postfix,
-                          int num_runs, double total_time_s,
-                          double throughput = -1.0) {
-  std::stringstream stream;
-  stream << benchmark_name;
-  if (!postfix.empty()) {
-    stream << "_" << postfix;
-  }
-
-  TestReporter node_reporter(output_prefix, stream.str());
-  TF_QCHECK_OK(node_reporter.Initialize());
-  TF_QCHECK_OK(
-      node_reporter.Benchmark(num_runs, -1.0, total_time_s, throughput));
-  TF_QCHECK_OK(node_reporter.Close());
-}
-
 Status RunBenchmark(const std::vector<InputLayerInfo>& inputs,
                     const std::vector<string>& outputs, Session* session,
                     StatSummarizer* stats, int64* inference_time_us) {
@@ -367,7 +350,7 @@ int Main(int argc, char** argv) {
   bool show_type = true;
   bool show_summary = true;
   bool show_flops = false;
-  int warmup_runs = 1;
+  int warmup_runs = 2;
 
   std::vector<Flag> flag_list = {
       Flag("graph", &graph, "graph file name"),
@@ -458,14 +441,8 @@ int Main(int argc, char** argv) {
   std::unique_ptr<Session> session;
   std::unique_ptr<StatSummarizer> stats;
   std::unique_ptr<GraphDef> graph_def;
-
-  int64 initialization_start_us = Env::Default()->NowMicros();
   Status initialize_status =
       InitializeSession(num_threads, graph, &session, &graph_def);
-  int64 initialization_end_us = Env::Default()->NowMicros();
-  double initialization_time_s =
-      (initialization_end_us - initialization_start_us) / 1000000.0;
-  LOG(INFO) << "Initialized session in " << initialization_time_s << "s";
   if (!initialize_status.ok()) {
     return -1;
   }
@@ -610,23 +587,11 @@ int Main(int argc, char** argv) {
         static_cast<double>(no_stat_wall_time) / (1024 * 1024);
 
     // Report the stats.
-    RecordBenchmarkEntry(output_prefix, benchmark_name, "", no_stat_num_runs,
-                         no_stat_wall_time, throughput);
-
-    // Session initialization time.
-    RecordBenchmarkEntry(output_prefix, benchmark_name, "meta-init", 1,
-                         initialization_time_s);
-
-    // First inference time. Note: if warmup_runs is > 1 this will actually be
-    // an average of all the warmup runs.
-    RecordBenchmarkEntry(output_prefix, benchmark_name, "meta-first-inference",
-                         warmup_runs, warmup_time_us / 1000000.0);
-
-    // Time from starting to intialize TF to getting the first result back.
-    // This also assumes that only one warmup run is performed.
-    RecordBenchmarkEntry(
-        output_prefix, benchmark_name, "meta-init-plus-first-inference", 1,
-        initialization_time_s + (warmup_time_us / 1000000.0) / warmup_runs);
+    TestReporter reporter(output_prefix, benchmark_name);
+    TF_QCHECK_OK(reporter.Initialize());
+    TF_QCHECK_OK(reporter.Benchmark(no_stat_num_runs, -1.0, no_stat_wall_time,
+                                    throughput));
+    TF_QCHECK_OK(reporter.Close());
 
     std::map<string, int64> node_type_map_count;
     std::map<string, int64> node_type_map_time;
@@ -638,10 +603,17 @@ int Main(int argc, char** argv) {
                               &node_type_map_memory,
                               &node_type_map_times_called, &accumulated_us);
     for (const auto& time : node_type_map_time) {
+      std::stringstream stream;
+      stream << benchmark_name << "_" << time.first;
+      TestReporter node_reporter(output_prefix, stream.str());
+
       LOG(INFO) << "Outputting: [" << time.first << "]";
-      RecordBenchmarkEntry(output_prefix, benchmark_name, time.first,
-                           stat_num_runs,
-                           (time.second * stat_num_runs) / 1000000.0f);
+
+      TF_QCHECK_OK(node_reporter.Initialize());
+      TF_QCHECK_OK(node_reporter.Benchmark(
+          stat_num_runs, -1.0, (time.second * stat_num_runs) / 1000000.0f,
+          -1.0));
+      TF_QCHECK_OK(node_reporter.Close());
     }
   }
 
-- 
GitLab


From 70b8b08ae3923cdf3f238c9636a4166e71b0102e Mon Sep 17 00:00:00 2001
From: Andrew Harp <andrewharp@google.com>
Date: Thu, 19 Oct 2017 13:49:21 -0700
Subject: [PATCH 568/909] Log initialization and warmup time to proto results
 in benchmark tool.

PiperOrigin-RevId: 172792563
---
 tensorflow/tools/benchmark/benchmark_model.cc | 60 ++++++++++++++-----
 1 file changed, 44 insertions(+), 16 deletions(-)

diff --git a/tensorflow/tools/benchmark/benchmark_model.cc b/tensorflow/tools/benchmark/benchmark_model.cc
index f84ae5c7ce..2d59299da4 100644
--- a/tensorflow/tools/benchmark/benchmark_model.cc
+++ b/tensorflow/tools/benchmark/benchmark_model.cc
@@ -230,6 +230,23 @@ Status CalculateFlops(const GraphDef& graph,
   return Status::OK();
 }
 
+void RecordBenchmarkEntry(const string& output_prefix,
+                          const string& benchmark_name, const string& postfix,
+                          int num_runs, double total_time_s,
+                          double throughput = -1.0) {
+  std::stringstream stream;
+  stream << benchmark_name;
+  if (!postfix.empty()) {
+    stream << "_" << postfix;
+  }
+
+  TestReporter node_reporter(output_prefix, stream.str());
+  TF_QCHECK_OK(node_reporter.Initialize());
+  TF_QCHECK_OK(
+      node_reporter.Benchmark(num_runs, -1.0, total_time_s, throughput));
+  TF_QCHECK_OK(node_reporter.Close());
+}
+
 Status RunBenchmark(const std::vector<InputLayerInfo>& inputs,
                     const std::vector<string>& outputs, Session* session,
                     StatSummarizer* stats, int64* inference_time_us) {
@@ -350,7 +367,7 @@ int Main(int argc, char** argv) {
   bool show_type = true;
   bool show_summary = true;
   bool show_flops = false;
-  int warmup_runs = 2;
+  int warmup_runs = 1;
 
   std::vector<Flag> flag_list = {
       Flag("graph", &graph, "graph file name"),
@@ -441,8 +458,14 @@ int Main(int argc, char** argv) {
   std::unique_ptr<Session> session;
   std::unique_ptr<StatSummarizer> stats;
   std::unique_ptr<GraphDef> graph_def;
+
+  int64 initialization_start_us = Env::Default()->NowMicros();
   Status initialize_status =
       InitializeSession(num_threads, graph, &session, &graph_def);
+  int64 initialization_end_us = Env::Default()->NowMicros();
+  double initialization_time_s =
+      (initialization_end_us - initialization_start_us) / 1000000.0;
+  LOG(INFO) << "Initialized session in " << initialization_time_s << "s";
   if (!initialize_status.ok()) {
     return -1;
   }
@@ -587,11 +610,23 @@ int Main(int argc, char** argv) {
         static_cast<double>(no_stat_wall_time) / (1024 * 1024);
 
     // Report the stats.
-    TestReporter reporter(output_prefix, benchmark_name);
-    TF_QCHECK_OK(reporter.Initialize());
-    TF_QCHECK_OK(reporter.Benchmark(no_stat_num_runs, -1.0, no_stat_wall_time,
-                                    throughput));
-    TF_QCHECK_OK(reporter.Close());
+    RecordBenchmarkEntry(output_prefix, benchmark_name, "", no_stat_num_runs,
+                         no_stat_wall_time, throughput);
+
+    // Session initialization time.
+    RecordBenchmarkEntry(output_prefix, benchmark_name, "meta-init", 1,
+                         initialization_time_s);
+
+    // First inference time. Note: if warmup_runs is > 1 this will actually be
+    // an average of all the warmup runs.
+    RecordBenchmarkEntry(output_prefix, benchmark_name, "meta-first-inference",
+                         warmup_runs, warmup_time_us / 1000000.0);
+
+    // Time from starting to intialize TF to getting the first result back.
+    // This also assumes that only one warmup run is performed.
+    RecordBenchmarkEntry(
+        output_prefix, benchmark_name, "meta-init-plus-first-inference", 1,
+        initialization_time_s + (warmup_time_us / 1000000.0) / warmup_runs);
 
     std::map<string, int64> node_type_map_count;
     std::map<string, int64> node_type_map_time;
@@ -603,17 +638,10 @@ int Main(int argc, char** argv) {
                               &node_type_map_memory,
                               &node_type_map_times_called, &accumulated_us);
     for (const auto& time : node_type_map_time) {
-      std::stringstream stream;
-      stream << benchmark_name << "_" << time.first;
-      TestReporter node_reporter(output_prefix, stream.str());
-
       LOG(INFO) << "Outputting: [" << time.first << "]";
-
-      TF_QCHECK_OK(node_reporter.Initialize());
-      TF_QCHECK_OK(node_reporter.Benchmark(
-          stat_num_runs, -1.0, (time.second * stat_num_runs) / 1000000.0f,
-          -1.0));
-      TF_QCHECK_OK(node_reporter.Close());
+      RecordBenchmarkEntry(output_prefix, benchmark_name, time.first,
+                           stat_num_runs,
+                           (time.second * stat_num_runs) / 1000000.0f);
     }
   }
 
-- 
GitLab


From 3238fab55f4e9daf5a06fc44e78082da42fad8a0 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 19 Oct 2017 13:50:28 -0700
Subject: [PATCH 569/909] Fix potential Tensor memory leak in GraphCompiler.

PiperOrigin-RevId: 172792724
---
 tensorflow/compiler/tf2xla/graph_compiler.cc | 29 ++++++++++++++------
 tensorflow/compiler/tf2xla/graph_compiler.h  | 17 ------------
 2 files changed, 20 insertions(+), 26 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/graph_compiler.cc b/tensorflow/compiler/tf2xla/graph_compiler.cc
index 9893afa7a0..8062f0c03c 100644
--- a/tensorflow/compiler/tf2xla/graph_compiler.cc
+++ b/tensorflow/compiler/tf2xla/graph_compiler.cc
@@ -38,6 +38,7 @@ limitations under the License.
 #include "tensorflow/core/graph/algorithm.h"
 #include "tensorflow/core/graph/graph_constructor.h"
 #include "tensorflow/core/graph/node_builder.h"
+#include "tensorflow/core/lib/gtl/cleanup.h"
 #include "tensorflow/core/lib/hash/hash.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/public/version.h"
@@ -84,9 +85,20 @@ Status PrepareArguments(XlaOpKernelContext* ctx, Graph* graph,
 }
 }  // namespace
 Status GraphCompiler::Compile() {
-  OutputRegistry output_registry(graph_->num_node_ids());
-  std::vector<Node*> topo_sorted_nodes;
+  // Maintain a mapping from node id to node outputs.
+  using NodeOutputs = std::vector<TensorValue>;
+  std::vector<NodeOutputs> output_registry(graph_->num_node_ids());
+  auto output_registry_cleanup = gtl::MakeCleanup([&output_registry] {
+    for (const NodeOutputs& outputs : output_registry) {
+      for (const TensorValue& value : outputs) {
+        CHECK(!value.is_ref());
+        delete value.tensor;
+      }
+    }
+  });
+
   // XLA requires determinism, generate a stable ordering from DFS.
+  std::vector<Node*> topo_sorted_nodes;
   GetReversePostOrder(*graph_, &topo_sorted_nodes,
                       /*stable_comparator=*/NodeComparatorName());
 
@@ -94,7 +106,6 @@ Status GraphCompiler::Compile() {
   PartiallySetupParams(&params);
 
   for (Node* n : topo_sorted_nodes) {
-    NodeOutputs node_outputs;
     OpKernel* op_kernel_raw = nullptr;
     Status s = flib_->CreateKernel(n->def(), &op_kernel_raw);
     // Transfer ownership of the kernel to a local smart pointer.
@@ -122,9 +133,9 @@ Status GraphCompiler::Compile() {
       if (e->IsControlEdge()) continue;
       Node* src = e->src();
       TF_RET_CHECK(src->id() < output_registry.size());
-      const NodeOutputs& outputs = output_registry[src->id()];
+      const NodeOutputs& src_outputs = output_registry[src->id()];
 
-      tensor_inputs_[e->dst_input()] = outputs.values[e->src_output()];
+      tensor_inputs_[e->dst_input()] = src_outputs[e->src_output()];
     }
 
     OpKernelContext op_context(&params, n->num_outputs());
@@ -138,15 +149,15 @@ Status GraphCompiler::Compile() {
 
     // Set up outputs. Also check if outputs from the previous computation is
     // valid.
+    NodeOutputs& outputs = output_registry[n->id()];
+    outputs.resize(n->num_outputs());
     for (int o = 0; o < n->num_outputs(); ++o) {
-      const auto tensor_val = op_context.release_output(o);
-      if (*op_context.is_output_dead() || tensor_val.tensor == nullptr) {
+      outputs[o] = op_context.release_output(o);
+      if (*op_context.is_output_dead() || outputs[o].tensor == nullptr) {
         return errors::Internal("Missing xla_context ", o, "-th output from ",
                                 (*op_context.is_output_dead() ? "(dead)" : ""),
                                 SummarizeNode(*n));
       }
-      // Set up outputs
-      output_registry[n->id()].values.push_back(tensor_val);
     }
   }
   return Status::OK();
diff --git a/tensorflow/compiler/tf2xla/graph_compiler.h b/tensorflow/compiler/tf2xla/graph_compiler.h
index 33781d2c21..ba00160b6d 100644
--- a/tensorflow/compiler/tf2xla/graph_compiler.h
+++ b/tensorflow/compiler/tf2xla/graph_compiler.h
@@ -69,23 +69,6 @@ class GraphCompiler {
   Status Compile();
 
  private:
-  // NodeOutputs is a wrapper over TensorValues that represents outputs of a
-  // node.
-  struct NodeOutputs {
-    ~NodeOutputs() {
-      for (auto& v : values) {
-        CHECK(!v.is_ref());
-        delete v.tensor;
-      }
-    }
-
-    // Output values of this node.
-    std::vector<TensorValue> values;
-  };
-
-  // A mapping from node id to node output.
-  using OutputRegistry = std::vector<NodeOutputs>;
-
   // Partially sets params. This partially set params can be reused
   // across multple nodes visit.
   void PartiallySetupParams(OpKernelContext::Params* params);
-- 
GitLab


From 3f56b1402409ad4efb8dd931d5b1b7bdc713597e Mon Sep 17 00:00:00 2001
From: Andrew Harp <andrewharp@google.com>
Date: Thu, 19 Oct 2017 13:49:21 -0700
Subject: [PATCH 570/909] Log initialization and warmup time to proto results
 in benchmark tool.

PiperOrigin-RevId: 172792563
---
 tensorflow/compiler/tf2xla/graph_compiler.cc | 29 ++++++--------------
 tensorflow/compiler/tf2xla/graph_compiler.h  | 17 ++++++++++++
 2 files changed, 26 insertions(+), 20 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/graph_compiler.cc b/tensorflow/compiler/tf2xla/graph_compiler.cc
index 8062f0c03c..9893afa7a0 100644
--- a/tensorflow/compiler/tf2xla/graph_compiler.cc
+++ b/tensorflow/compiler/tf2xla/graph_compiler.cc
@@ -38,7 +38,6 @@ limitations under the License.
 #include "tensorflow/core/graph/algorithm.h"
 #include "tensorflow/core/graph/graph_constructor.h"
 #include "tensorflow/core/graph/node_builder.h"
-#include "tensorflow/core/lib/gtl/cleanup.h"
 #include "tensorflow/core/lib/hash/hash.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/public/version.h"
@@ -85,20 +84,9 @@ Status PrepareArguments(XlaOpKernelContext* ctx, Graph* graph,
 }
 }  // namespace
 Status GraphCompiler::Compile() {
-  // Maintain a mapping from node id to node outputs.
-  using NodeOutputs = std::vector<TensorValue>;
-  std::vector<NodeOutputs> output_registry(graph_->num_node_ids());
-  auto output_registry_cleanup = gtl::MakeCleanup([&output_registry] {
-    for (const NodeOutputs& outputs : output_registry) {
-      for (const TensorValue& value : outputs) {
-        CHECK(!value.is_ref());
-        delete value.tensor;
-      }
-    }
-  });
-
-  // XLA requires determinism, generate a stable ordering from DFS.
+  OutputRegistry output_registry(graph_->num_node_ids());
   std::vector<Node*> topo_sorted_nodes;
+  // XLA requires determinism, generate a stable ordering from DFS.
   GetReversePostOrder(*graph_, &topo_sorted_nodes,
                       /*stable_comparator=*/NodeComparatorName());
 
@@ -106,6 +94,7 @@ Status GraphCompiler::Compile() {
   PartiallySetupParams(&params);
 
   for (Node* n : topo_sorted_nodes) {
+    NodeOutputs node_outputs;
     OpKernel* op_kernel_raw = nullptr;
     Status s = flib_->CreateKernel(n->def(), &op_kernel_raw);
     // Transfer ownership of the kernel to a local smart pointer.
@@ -133,9 +122,9 @@ Status GraphCompiler::Compile() {
       if (e->IsControlEdge()) continue;
       Node* src = e->src();
       TF_RET_CHECK(src->id() < output_registry.size());
-      const NodeOutputs& src_outputs = output_registry[src->id()];
+      const NodeOutputs& outputs = output_registry[src->id()];
 
-      tensor_inputs_[e->dst_input()] = src_outputs[e->src_output()];
+      tensor_inputs_[e->dst_input()] = outputs.values[e->src_output()];
     }
 
     OpKernelContext op_context(&params, n->num_outputs());
@@ -149,15 +138,15 @@ Status GraphCompiler::Compile() {
 
     // Set up outputs. Also check if outputs from the previous computation is
     // valid.
-    NodeOutputs& outputs = output_registry[n->id()];
-    outputs.resize(n->num_outputs());
     for (int o = 0; o < n->num_outputs(); ++o) {
-      outputs[o] = op_context.release_output(o);
-      if (*op_context.is_output_dead() || outputs[o].tensor == nullptr) {
+      const auto tensor_val = op_context.release_output(o);
+      if (*op_context.is_output_dead() || tensor_val.tensor == nullptr) {
         return errors::Internal("Missing xla_context ", o, "-th output from ",
                                 (*op_context.is_output_dead() ? "(dead)" : ""),
                                 SummarizeNode(*n));
       }
+      // Set up outputs
+      output_registry[n->id()].values.push_back(tensor_val);
     }
   }
   return Status::OK();
diff --git a/tensorflow/compiler/tf2xla/graph_compiler.h b/tensorflow/compiler/tf2xla/graph_compiler.h
index ba00160b6d..33781d2c21 100644
--- a/tensorflow/compiler/tf2xla/graph_compiler.h
+++ b/tensorflow/compiler/tf2xla/graph_compiler.h
@@ -69,6 +69,23 @@ class GraphCompiler {
   Status Compile();
 
  private:
+  // NodeOutputs is a wrapper over TensorValues that represents outputs of a
+  // node.
+  struct NodeOutputs {
+    ~NodeOutputs() {
+      for (auto& v : values) {
+        CHECK(!v.is_ref());
+        delete v.tensor;
+      }
+    }
+
+    // Output values of this node.
+    std::vector<TensorValue> values;
+  };
+
+  // A mapping from node id to node output.
+  using OutputRegistry = std::vector<NodeOutputs>;
+
   // Partially sets params. This partially set params can be reused
   // across multple nodes visit.
   void PartiallySetupParams(OpKernelContext::Params* params);
-- 
GitLab


From e858968dc720f47dd15ed8c2d7a5c3910a7e29b1 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 19 Oct 2017 13:50:28 -0700
Subject: [PATCH 571/909] Fix potential Tensor memory leak in GraphCompiler.

PiperOrigin-RevId: 172792724
---
 tensorflow/compiler/tf2xla/graph_compiler.cc | 29 ++++++++++++++------
 tensorflow/compiler/tf2xla/graph_compiler.h  | 17 ------------
 2 files changed, 20 insertions(+), 26 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/graph_compiler.cc b/tensorflow/compiler/tf2xla/graph_compiler.cc
index 9893afa7a0..8062f0c03c 100644
--- a/tensorflow/compiler/tf2xla/graph_compiler.cc
+++ b/tensorflow/compiler/tf2xla/graph_compiler.cc
@@ -38,6 +38,7 @@ limitations under the License.
 #include "tensorflow/core/graph/algorithm.h"
 #include "tensorflow/core/graph/graph_constructor.h"
 #include "tensorflow/core/graph/node_builder.h"
+#include "tensorflow/core/lib/gtl/cleanup.h"
 #include "tensorflow/core/lib/hash/hash.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/public/version.h"
@@ -84,9 +85,20 @@ Status PrepareArguments(XlaOpKernelContext* ctx, Graph* graph,
 }
 }  // namespace
 Status GraphCompiler::Compile() {
-  OutputRegistry output_registry(graph_->num_node_ids());
-  std::vector<Node*> topo_sorted_nodes;
+  // Maintain a mapping from node id to node outputs.
+  using NodeOutputs = std::vector<TensorValue>;
+  std::vector<NodeOutputs> output_registry(graph_->num_node_ids());
+  auto output_registry_cleanup = gtl::MakeCleanup([&output_registry] {
+    for (const NodeOutputs& outputs : output_registry) {
+      for (const TensorValue& value : outputs) {
+        CHECK(!value.is_ref());
+        delete value.tensor;
+      }
+    }
+  });
+
   // XLA requires determinism, generate a stable ordering from DFS.
+  std::vector<Node*> topo_sorted_nodes;
   GetReversePostOrder(*graph_, &topo_sorted_nodes,
                       /*stable_comparator=*/NodeComparatorName());
 
@@ -94,7 +106,6 @@ Status GraphCompiler::Compile() {
   PartiallySetupParams(&params);
 
   for (Node* n : topo_sorted_nodes) {
-    NodeOutputs node_outputs;
     OpKernel* op_kernel_raw = nullptr;
     Status s = flib_->CreateKernel(n->def(), &op_kernel_raw);
     // Transfer ownership of the kernel to a local smart pointer.
@@ -122,9 +133,9 @@ Status GraphCompiler::Compile() {
       if (e->IsControlEdge()) continue;
       Node* src = e->src();
       TF_RET_CHECK(src->id() < output_registry.size());
-      const NodeOutputs& outputs = output_registry[src->id()];
+      const NodeOutputs& src_outputs = output_registry[src->id()];
 
-      tensor_inputs_[e->dst_input()] = outputs.values[e->src_output()];
+      tensor_inputs_[e->dst_input()] = src_outputs[e->src_output()];
     }
 
     OpKernelContext op_context(&params, n->num_outputs());
@@ -138,15 +149,15 @@ Status GraphCompiler::Compile() {
 
     // Set up outputs. Also check if outputs from the previous computation is
     // valid.
+    NodeOutputs& outputs = output_registry[n->id()];
+    outputs.resize(n->num_outputs());
     for (int o = 0; o < n->num_outputs(); ++o) {
-      const auto tensor_val = op_context.release_output(o);
-      if (*op_context.is_output_dead() || tensor_val.tensor == nullptr) {
+      outputs[o] = op_context.release_output(o);
+      if (*op_context.is_output_dead() || outputs[o].tensor == nullptr) {
         return errors::Internal("Missing xla_context ", o, "-th output from ",
                                 (*op_context.is_output_dead() ? "(dead)" : ""),
                                 SummarizeNode(*n));
       }
-      // Set up outputs
-      output_registry[n->id()].values.push_back(tensor_val);
     }
   }
   return Status::OK();
diff --git a/tensorflow/compiler/tf2xla/graph_compiler.h b/tensorflow/compiler/tf2xla/graph_compiler.h
index 33781d2c21..ba00160b6d 100644
--- a/tensorflow/compiler/tf2xla/graph_compiler.h
+++ b/tensorflow/compiler/tf2xla/graph_compiler.h
@@ -69,23 +69,6 @@ class GraphCompiler {
   Status Compile();
 
  private:
-  // NodeOutputs is a wrapper over TensorValues that represents outputs of a
-  // node.
-  struct NodeOutputs {
-    ~NodeOutputs() {
-      for (auto& v : values) {
-        CHECK(!v.is_ref());
-        delete v.tensor;
-      }
-    }
-
-    // Output values of this node.
-    std::vector<TensorValue> values;
-  };
-
-  // A mapping from node id to node output.
-  using OutputRegistry = std::vector<NodeOutputs>;
-
   // Partially sets params. This partially set params can be reused
   // across multple nodes visit.
   void PartiallySetupParams(OpKernelContext::Params* params);
-- 
GitLab


From 7fe3744373751ee6a79bb23c6c20343a91d07b28 Mon Sep 17 00:00:00 2001
From: Mark Heffernan <meheff@google.com>
Date: Thu, 19 Oct 2017 14:01:53 -0700
Subject: [PATCH 572/909] Add local client execution test which uses infeed and
 outfeed.

PiperOrigin-RevId: 172794367
---
 .../xla/tests/local_client_execute_test.cc    | 26 +++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/tensorflow/compiler/xla/tests/local_client_execute_test.cc b/tensorflow/compiler/xla/tests/local_client_execute_test.cc
index c74213f7f9..329b53012f 100644
--- a/tensorflow/compiler/xla/tests/local_client_execute_test.cc
+++ b/tensorflow/compiler/xla/tests/local_client_execute_test.cc
@@ -37,6 +37,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/tests/test_macros.h"
 #include "tensorflow/compiler/xla/tests/test_utils.h"
 #include "tensorflow/compiler/xla/xla_data.pb.h"
+#include "tensorflow/core/platform/env.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/stream_executor_no_cuda.h"
 #include "tensorflow/core/platform/test.h"
@@ -859,6 +860,31 @@ XLA_TEST_F(LocalClientExecuteTest, ShapeBufferToLiteralConversion64bit) {
                            Literal::CreateR0<int64>(123456789000LL).get()}));
 }
 
+// TODO(b/34359662): Support infeed/outfeed on GPU and CPU parallel.
+// 2017-10-18.
+XLA_TEST_F(LocalClientExecuteTest,
+           DISABLED_ON_GPU(DISABLED_ON_CPU_PARALLEL(InfeedOutfeedTest))) {
+  ComputationBuilder builder(local_client_, TestName());
+  const Shape shape = ShapeUtil::MakeShape(F32, {3});
+  auto in = builder.Infeed(shape);
+  auto constant = builder.ConstantR1<float>({1.0f, 2.0f, 3.0f});
+  auto sum = builder.Add(in, constant);
+  builder.Outfeed(sum, shape, /*outfeed_config=*/"");
+
+  std::unique_ptr<tensorflow::Thread> thread(
+      tensorflow::Env::Default()->StartThread(
+          tensorflow::ThreadOptions(), "execute_thread",
+          [&] { ExecuteLocallyOrDie(builder.Build().ValueOrDie(), {}); }));
+
+  ASSERT_IS_OK(local_client_->TransferToInfeed(
+      *Literal::CreateR1<float>({-5.0, 123.0, 42.0})));
+
+  TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<Literal> result,
+                          local_client_->TransferFromOutfeed(&shape));
+
+  LiteralTestUtil::ExpectR1Equal<float>({-4.0, 125.0, 45.0}, *result);
+}
+
 // Benchmark that measures the overhead of the LocalClient API when running a
 // trivial computation
 void BM_LocalClientOverhead(int num_iters) {
-- 
GitLab


From eb978292e0ac46dd16c820b9989ad1776295517a Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Thu, 19 Oct 2017 14:16:16 -0700
Subject: [PATCH 573/909] Context-manager-based gradient API

PiperOrigin-RevId: 172796719
---
 tensorflow/contrib/eager/python/tfe.py     |  2 +
 tensorflow/python/eager/backprop.py        | 91 ++++++++++++++++++++++
 tensorflow/python/eager/backprop_test.py   | 21 +++++
 tensorflow/python/eager/imperative_grad.py |  2 +-
 4 files changed, 115 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/eager/python/tfe.py b/tensorflow/contrib/eager/python/tfe.py
index 353b9d2bda..25942aadfb 100644
--- a/tensorflow/contrib/eager/python/tfe.py
+++ b/tensorflow/contrib/eager/python/tfe.py
@@ -26,6 +26,7 @@ To use, at program startup, call `tfe.enable_eager_execution()`.
 @@implicit_value_and_gradients
 @@gradients_function
 @@value_and_gradients_function
+@@GradientTape
 
 @@enable_tracing
 @@flush_trace
@@ -92,5 +93,6 @@ implicit_gradients = backprop.implicit_grad
 implicit_value_and_gradients = backprop.implicit_val_and_grad
 gradients_function = backprop.gradients_function
 value_and_gradients_function = backprop.val_and_grad_function
+GradientTape = backprop.GradientTape  # pylint: disable=invalid-name
 
 remove_undocumented(__name__)
diff --git a/tensorflow/python/eager/backprop.py b/tensorflow/python/eager/backprop.py
index 61c905f31e..da17be05b7 100644
--- a/tensorflow/python/eager/backprop.py
+++ b/tensorflow/python/eager/backprop.py
@@ -36,6 +36,7 @@ from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.util import nest
 from tensorflow.python.util import tf_inspect
 
@@ -696,3 +697,93 @@ _default_vspace = imperative_grad.VSpace(
     tensor_id=ops.tensor_id,
     zeros=array_ops.zeros,
     ones_like=array_ops.ones_like)
+
+
+class GradientTape(object):
+  """Records operations to use to compute gradients.
+
+  Operations are recorded if:
+    - they happen in code marked by this context manager
+    - at least one of their inputs is being watched
+
+  Outputs of recorded operations are watched. Variables are automatically
+  watched and tensors can be manually watched by calling the watch method on the
+  context manager.
+
+  Example usage:
+
+  ```python
+  with tfe.GradientTape() as g:
+    x = tf.constant(3.0)
+    g.watch(x)
+    y = x * x
+  grad = g.gradient(y, [x])[0]
+  assert grad.numpy() == 6.0
+  ```
+
+  It is possible to use GradientTapes to compute higher-order derivatives as
+  follows:
+
+  ```python
+  with tfe.GradientTape() as g:
+    x = tf.constant(3.0)
+    g.watch(x)
+    y = x * x
+    with tfe.GradientTape() as gg:
+      gg.watch(y)
+      z = 2 * y
+    inner_grad = gg.gradient(z, [y])[0]
+    assert inner_grad.numpy() == 2
+    y = y + inner_grad
+  grad = g.gradient(y, [x])[0]
+  assert grad.numpy() == 6.0
+  ```
+  """
+
+  def __init__(self):
+    self._tape = None
+
+  def __enter__(self):
+    tape.push_new_tape()
+    return self
+
+  def __exit__(self, typ, value, traceback):
+    self._tape = tape.pop_tape()
+
+  def watch(self, tensor):
+    """Ensures that `tensor` is being traced by this tape.
+
+    Args:
+      tensor: a Tensor or Variable a list of Tensors or Variables.
+    """
+    for t in nest.flatten(tensor):
+      if isinstance(t, resource_variable_ops.ResourceVariable):
+        t = t.handle
+      tape.watch(t)
+
+  def gradient(self, target, sources):
+    """Computes the gradient using information traced by the tape.
+
+    Args:
+      target: the tensor to be differentiated.
+      sources: a list of Tensors or Variables, the target will be
+       differentiated with respect to the sources.
+
+    Returns:
+      a list of Tensors (or IndexedSlices, or None), one for each element in
+      `sources`.
+
+    Raises:
+      RuntimeError: if called inside the context of the tape, or if called more
+       than once.
+    """
+    if self._tape is None:
+      raise RuntimeError("GradientTape.gradient can only be called once, and "
+                         "only when the context manager has exited.")
+    sources = [x.handle if isinstance(x, resource_variable_ops.ResourceVariable)
+               else x
+               for x in sources]
+    grad = imperative_grad.imperative_grad(
+        _default_vspace, self._tape, [target], sources)
+    self.tape = None
+    return grad
diff --git a/tensorflow/python/eager/backprop_test.py b/tensorflow/python/eager/backprop_test.py
index 5161095683..95d5f0adcb 100644
--- a/tensorflow/python/eager/backprop_test.py
+++ b/tensorflow/python/eager/backprop_test.py
@@ -277,6 +277,27 @@ class BackpropTest(test.TestCase):
 
     self.assertEqual(backprop.implicit_grad(f)()[0][0], None)
 
+  def testGradientTape(self):
+    with backprop.GradientTape() as g:
+      x = constant_op.constant(3.0)
+      g.watch(x)
+      y = x * x
+      with backprop.GradientTape() as gg:
+        gg.watch(y)
+        z = 2 * y
+      inner_grad = gg.gradient(z, [y])[0]
+      self.assertEqual(inner_grad.numpy(), 2.0)
+      y += inner_grad
+    grad = g.gradient(y, [x])[0]
+    self.assertEqual(grad.numpy(), 6.0)
+
+  def testGradientTapeVariable(self):
+    v = resource_variable_ops.ResourceVariable(1.0)
+    with backprop.GradientTape() as g:
+      y = v * v
+    grad = g.gradient(y, [v])[0]
+    self.assertAllEqual(grad, 2.0)
+
   def testEmptyParamsForValueAndGradFunction(self):
     def fn(a, b):
       return a * b
diff --git a/tensorflow/python/eager/imperative_grad.py b/tensorflow/python/eager/imperative_grad.py
index ce58e661d7..c87719f84a 100644
--- a/tensorflow/python/eager/imperative_grad.py
+++ b/tensorflow/python/eager/imperative_grad.py
@@ -215,7 +215,7 @@ def imperative_grad(
             and tensor_usage_counts[t] == 0
             and t not in id_sources):
           in_op = tensor_to_op[t]
-          if in_op is None:
+          if in_op is None or in_op == -1:
             continue
           if op_missing_tensor.get(in_op, 0) > 0:
             op_missing_tensor[in_op] -= 1
-- 
GitLab


From ee7f8d973b9ea05495d799a728ab5ad9c654d125 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 19 Oct 2017 14:22:29 -0700
Subject: [PATCH 574/909] Use "nullptr" instead of other null pointer constants

PiperOrigin-RevId: 172797910
---
 tensorflow/python/eager/pywrap_tfe_src.cc | 40 +++++++++++------------
 1 file changed, 20 insertions(+), 20 deletions(-)

diff --git a/tensorflow/python/eager/pywrap_tfe_src.cc b/tensorflow/python/eager/pywrap_tfe_src.cc
index 402b84d7c6..7456eb10f8 100644
--- a/tensorflow/python/eager/pywrap_tfe_src.cc
+++ b/tensorflow/python/eager/pywrap_tfe_src.cc
@@ -452,26 +452,26 @@ static void TFE_Py_Tape_Delete(PyObject* tape) {
 }
 
 static PyTypeObject TFE_Py_Tape_Type = {
-    PyVarObject_HEAD_INIT(NULL, 0) "tfe.Tape", /* tp_name */
-    sizeof(TFE_Py_Tape),                       /* tp_basicsize */
-    0,                                         /* tp_itemsize */
-    &TFE_Py_Tape_Delete,                       /* tp_dealloc */
-    0,                                         /* tp_print */
-    0,                                         /* tp_getattr */
-    0,                                         /* tp_setattr */
-    0,                                         /* tp_reserved */
-    0,                                         /* tp_repr */
-    0,                                         /* tp_as_number */
-    0,                                         /* tp_as_sequence */
-    0,                                         /* tp_as_mapping */
-    0,                                         /* tp_hash  */
-    0,                                         /* tp_call */
-    0,                                         /* tp_str */
-    0,                                         /* tp_getattro */
-    0,                                         /* tp_setattro */
-    0,                                         /* tp_as_buffer */
-    Py_TPFLAGS_DEFAULT,                        /* tp_flags */
-    "TFE_Py_Tape objects",                     /* tp_doc */
+    PyVarObject_HEAD_INIT(nullptr, 0) "tfe.Tape", /* tp_name */
+    sizeof(TFE_Py_Tape),                          /* tp_basicsize */
+    0,                                            /* tp_itemsize */
+    &TFE_Py_Tape_Delete,                          /* tp_dealloc */
+    nullptr,                                      /* tp_print */
+    nullptr,                                      /* tp_getattr */
+    nullptr,                                      /* tp_setattr */
+    nullptr,                                      /* tp_reserved */
+    nullptr,                                      /* tp_repr */
+    nullptr,                                      /* tp_as_number */
+    nullptr,                                      /* tp_as_sequence */
+    nullptr,                                      /* tp_as_mapping */
+    nullptr,                                      /* tp_hash  */
+    nullptr,                                      /* tp_call */
+    nullptr,                                      /* tp_str */
+    nullptr,                                      /* tp_getattro */
+    nullptr,                                      /* tp_setattro */
+    nullptr,                                      /* tp_as_buffer */
+    Py_TPFLAGS_DEFAULT,                           /* tp_flags */
+    "TFE_Py_Tape objects",                        /* tp_doc */
 };
 
 PyObject* TFE_Py_NewTape() {
-- 
GitLab


From eeabcaffd502a5e9fb3664eaa89134c855a86148 Mon Sep 17 00:00:00 2001
From: Derek Murray <mrry@google.com>
Date: Thu, 19 Oct 2017 14:23:03 -0700
Subject: [PATCH 575/909] `tf.py_func`: Handle NumPy arrays of np.object that
 hold unicode strings.

This also fixes a bug affecting `tf.data.Dataset.from_generator()` on Python 3,
where the generator yields Unicode (i.e. default) strings.

PiperOrigin-RevId: 172798007
---
 .../dataset_from_generator_op_test.py         | 20 +++++++++++++++++++
 .../python/kernel_tests/py_func_test.py       | 15 ++++++++++++++
 tensorflow/python/lib/core/py_func.cc         | 11 ++++++++--
 3 files changed, 44 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/kernel_tests/dataset_from_generator_op_test.py b/tensorflow/python/kernel_tests/dataset_from_generator_op_test.py
index cd2bec8432..f129d07b57 100644
--- a/tensorflow/python/kernel_tests/dataset_from_generator_op_test.py
+++ b/tensorflow/python/kernel_tests/dataset_from_generator_op_test.py
@@ -216,6 +216,26 @@ class DatasetConstructorTest(test.TestCase):
         with self.assertRaises(errors.OutOfRangeError):
           sess.run(get_next)
 
+  def testFromGeneratorString(self):
+    def generator():
+      yield "foo"
+      yield b"bar"
+      yield u"baz"
+
+    iterator = (dataset_ops.Dataset.from_generator(
+        generator, output_types=dtypes.string, output_shapes=[])
+                .make_initializable_iterator())
+    init_op = iterator.initializer
+    get_next = iterator.get_next()
+
+    with self.test_session() as sess:
+      sess.run(init_op)
+      for expected in [b"foo", b"bar", b"baz"]:
+        next_val = sess.run(get_next)
+        self.assertAllEqual(expected, next_val)
+      with self.assertRaises(errors.OutOfRangeError):
+        sess.run(get_next)
+
   def testFromGeneratorTypeError(self):
     def generator():
       yield np.array([1, 2, 3], dtype=np.int64)
diff --git a/tensorflow/python/kernel_tests/py_func_test.py b/tensorflow/python/kernel_tests/py_func_test.py
index 4bd5b79797..7ed99c1be9 100644
--- a/tensorflow/python/kernel_tests/py_func_test.py
+++ b/tensorflow/python/kernel_tests/py_func_test.py
@@ -148,6 +148,21 @@ class PyOpTest(test.TestCase):
       z, = script_ops.py_func(read_and_return_strings, [x, y], [dtypes.string])
       self.assertListEqual(list(z.eval()), [b"hello there", b"hi there"])
 
+  def testObjectArraysAreConvertedToBytes(self):
+
+    def read_object_array():
+      return np.array([b" there", u" ya"], dtype=np.object)
+
+    def read_and_return_strings(x, y):
+      return x + y
+
+    with self.test_session():
+      x = constant_op.constant(["hello", "hi"], dtypes.string)
+      y, = script_ops.py_func(read_object_array, [],
+                              [dtypes.string])
+      z, = script_ops.py_func(read_and_return_strings, [x, y], [dtypes.string])
+      self.assertListEqual(list(z.eval()), [b"hello there", b"hi ya"])
+
   def testStringPadding(self):
     correct = [b"this", b"is", b"a", b"test"]
     with self.test_session():
diff --git a/tensorflow/python/lib/core/py_func.cc b/tensorflow/python/lib/core/py_func.cc
index 84cb4885f6..a62847614c 100644
--- a/tensorflow/python/lib/core/py_func.cc
+++ b/tensorflow/python/lib/core/py_func.cc
@@ -297,8 +297,15 @@ Status ConvertNdarrayToTensor(PyObject* obj, Tensor* ret) {
         char* el;
         Py_ssize_t el_size;
         if (PyBytes_AsStringAndSize(input_data[i], &el, &el_size) == -1) {
-          return errors::Unimplemented("Unsupported object type ",
-                                       input_data[i]->ob_type->tp_name);
+#if PY_MAJOR_VERSION >= 3
+          el = PyUnicode_AsUTF8AndSize(input_data[i], &el_size);
+          if (!el) {
+#endif
+            return errors::Unimplemented("Unsupported object type ",
+                                         input_data[i]->ob_type->tp_name);
+#if PY_MAJOR_VERSION >= 3
+          }
+#endif
         }
         tflat(i) = string(el, el_size);
       }
-- 
GitLab


From fe82a3165d1be801df64bd7dc3009ba8773ed4a9 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 19 Oct 2017 14:22:29 -0700
Subject: [PATCH 576/909] Use "nullptr" instead of other null pointer constants

PiperOrigin-RevId: 172797910
---
 .../dataset_from_generator_op_test.py         | 20 -------------------
 .../python/kernel_tests/py_func_test.py       | 15 --------------
 tensorflow/python/lib/core/py_func.cc         | 11 ++--------
 3 files changed, 2 insertions(+), 44 deletions(-)

diff --git a/tensorflow/python/kernel_tests/dataset_from_generator_op_test.py b/tensorflow/python/kernel_tests/dataset_from_generator_op_test.py
index f129d07b57..cd2bec8432 100644
--- a/tensorflow/python/kernel_tests/dataset_from_generator_op_test.py
+++ b/tensorflow/python/kernel_tests/dataset_from_generator_op_test.py
@@ -216,26 +216,6 @@ class DatasetConstructorTest(test.TestCase):
         with self.assertRaises(errors.OutOfRangeError):
           sess.run(get_next)
 
-  def testFromGeneratorString(self):
-    def generator():
-      yield "foo"
-      yield b"bar"
-      yield u"baz"
-
-    iterator = (dataset_ops.Dataset.from_generator(
-        generator, output_types=dtypes.string, output_shapes=[])
-                .make_initializable_iterator())
-    init_op = iterator.initializer
-    get_next = iterator.get_next()
-
-    with self.test_session() as sess:
-      sess.run(init_op)
-      for expected in [b"foo", b"bar", b"baz"]:
-        next_val = sess.run(get_next)
-        self.assertAllEqual(expected, next_val)
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
-
   def testFromGeneratorTypeError(self):
     def generator():
       yield np.array([1, 2, 3], dtype=np.int64)
diff --git a/tensorflow/python/kernel_tests/py_func_test.py b/tensorflow/python/kernel_tests/py_func_test.py
index 7ed99c1be9..4bd5b79797 100644
--- a/tensorflow/python/kernel_tests/py_func_test.py
+++ b/tensorflow/python/kernel_tests/py_func_test.py
@@ -148,21 +148,6 @@ class PyOpTest(test.TestCase):
       z, = script_ops.py_func(read_and_return_strings, [x, y], [dtypes.string])
       self.assertListEqual(list(z.eval()), [b"hello there", b"hi there"])
 
-  def testObjectArraysAreConvertedToBytes(self):
-
-    def read_object_array():
-      return np.array([b" there", u" ya"], dtype=np.object)
-
-    def read_and_return_strings(x, y):
-      return x + y
-
-    with self.test_session():
-      x = constant_op.constant(["hello", "hi"], dtypes.string)
-      y, = script_ops.py_func(read_object_array, [],
-                              [dtypes.string])
-      z, = script_ops.py_func(read_and_return_strings, [x, y], [dtypes.string])
-      self.assertListEqual(list(z.eval()), [b"hello there", b"hi ya"])
-
   def testStringPadding(self):
     correct = [b"this", b"is", b"a", b"test"]
     with self.test_session():
diff --git a/tensorflow/python/lib/core/py_func.cc b/tensorflow/python/lib/core/py_func.cc
index a62847614c..84cb4885f6 100644
--- a/tensorflow/python/lib/core/py_func.cc
+++ b/tensorflow/python/lib/core/py_func.cc
@@ -297,15 +297,8 @@ Status ConvertNdarrayToTensor(PyObject* obj, Tensor* ret) {
         char* el;
         Py_ssize_t el_size;
         if (PyBytes_AsStringAndSize(input_data[i], &el, &el_size) == -1) {
-#if PY_MAJOR_VERSION >= 3
-          el = PyUnicode_AsUTF8AndSize(input_data[i], &el_size);
-          if (!el) {
-#endif
-            return errors::Unimplemented("Unsupported object type ",
-                                         input_data[i]->ob_type->tp_name);
-#if PY_MAJOR_VERSION >= 3
-          }
-#endif
+          return errors::Unimplemented("Unsupported object type ",
+                                       input_data[i]->ob_type->tp_name);
         }
         tflat(i) = string(el, el_size);
       }
-- 
GitLab


From 17ba3a69f4c3509711a3da5eff3cb6be99e0936d Mon Sep 17 00:00:00 2001
From: Derek Murray <mrry@google.com>
Date: Thu, 19 Oct 2017 14:23:03 -0700
Subject: [PATCH 577/909] `tf.py_func`: Handle NumPy arrays of np.object that
 hold unicode strings.

This also fixes a bug affecting `tf.data.Dataset.from_generator()` on Python 3,
where the generator yields Unicode (i.e. default) strings.

PiperOrigin-RevId: 172798007
---
 .../dataset_from_generator_op_test.py         | 20 +++++++++++++++++++
 .../python/kernel_tests/py_func_test.py       | 15 ++++++++++++++
 tensorflow/python/lib/core/py_func.cc         | 11 ++++++++--
 3 files changed, 44 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/kernel_tests/dataset_from_generator_op_test.py b/tensorflow/python/kernel_tests/dataset_from_generator_op_test.py
index cd2bec8432..f129d07b57 100644
--- a/tensorflow/python/kernel_tests/dataset_from_generator_op_test.py
+++ b/tensorflow/python/kernel_tests/dataset_from_generator_op_test.py
@@ -216,6 +216,26 @@ class DatasetConstructorTest(test.TestCase):
         with self.assertRaises(errors.OutOfRangeError):
           sess.run(get_next)
 
+  def testFromGeneratorString(self):
+    def generator():
+      yield "foo"
+      yield b"bar"
+      yield u"baz"
+
+    iterator = (dataset_ops.Dataset.from_generator(
+        generator, output_types=dtypes.string, output_shapes=[])
+                .make_initializable_iterator())
+    init_op = iterator.initializer
+    get_next = iterator.get_next()
+
+    with self.test_session() as sess:
+      sess.run(init_op)
+      for expected in [b"foo", b"bar", b"baz"]:
+        next_val = sess.run(get_next)
+        self.assertAllEqual(expected, next_val)
+      with self.assertRaises(errors.OutOfRangeError):
+        sess.run(get_next)
+
   def testFromGeneratorTypeError(self):
     def generator():
       yield np.array([1, 2, 3], dtype=np.int64)
diff --git a/tensorflow/python/kernel_tests/py_func_test.py b/tensorflow/python/kernel_tests/py_func_test.py
index 4bd5b79797..7ed99c1be9 100644
--- a/tensorflow/python/kernel_tests/py_func_test.py
+++ b/tensorflow/python/kernel_tests/py_func_test.py
@@ -148,6 +148,21 @@ class PyOpTest(test.TestCase):
       z, = script_ops.py_func(read_and_return_strings, [x, y], [dtypes.string])
       self.assertListEqual(list(z.eval()), [b"hello there", b"hi there"])
 
+  def testObjectArraysAreConvertedToBytes(self):
+
+    def read_object_array():
+      return np.array([b" there", u" ya"], dtype=np.object)
+
+    def read_and_return_strings(x, y):
+      return x + y
+
+    with self.test_session():
+      x = constant_op.constant(["hello", "hi"], dtypes.string)
+      y, = script_ops.py_func(read_object_array, [],
+                              [dtypes.string])
+      z, = script_ops.py_func(read_and_return_strings, [x, y], [dtypes.string])
+      self.assertListEqual(list(z.eval()), [b"hello there", b"hi ya"])
+
   def testStringPadding(self):
     correct = [b"this", b"is", b"a", b"test"]
     with self.test_session():
diff --git a/tensorflow/python/lib/core/py_func.cc b/tensorflow/python/lib/core/py_func.cc
index 84cb4885f6..a62847614c 100644
--- a/tensorflow/python/lib/core/py_func.cc
+++ b/tensorflow/python/lib/core/py_func.cc
@@ -297,8 +297,15 @@ Status ConvertNdarrayToTensor(PyObject* obj, Tensor* ret) {
         char* el;
         Py_ssize_t el_size;
         if (PyBytes_AsStringAndSize(input_data[i], &el, &el_size) == -1) {
-          return errors::Unimplemented("Unsupported object type ",
-                                       input_data[i]->ob_type->tp_name);
+#if PY_MAJOR_VERSION >= 3
+          el = PyUnicode_AsUTF8AndSize(input_data[i], &el_size);
+          if (!el) {
+#endif
+            return errors::Unimplemented("Unsupported object type ",
+                                         input_data[i]->ob_type->tp_name);
+#if PY_MAJOR_VERSION >= 3
+          }
+#endif
         }
         tflat(i) = string(el, el_size);
       }
-- 
GitLab


From 21d2de1c8d34d5094472dd828394c239d6111e0d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 19 Oct 2017 15:02:01 -0700
Subject: [PATCH 578/909] Add a recursive descent parser for the HloModule
 string. It constructs an HloModule object from a string printed by
 HloModule::ToString().

This is a initial stage. It currently supports:
- unary, binary, ternary ops, and other ops that don't have extra attributes.
- module with entry computation only.
- simple cases for constant instruction.

To make the parser simpler, this cl removes a whitespace and adds a '%' before the computation name in HloComputation::ToString().

Further steps will enable parsing subcomputations, more cases of constants, tuple, and ops that require extra attributes (e.g., broadcast dimensions, subcomputation).

PiperOrigin-RevId: 172804214
---
 tensorflow/BUILD                              |   1 +
 .../compiler/xla/service/hlo_computation.cc   |   4 +-
 tensorflow/compiler/xla/shape_util.cc         |  45 +-
 tensorflow/compiler/xla/tools/parser/BUILD    |  84 +++
 .../compiler/xla/tools/parser/README.md       |  69 +++
 .../compiler/xla/tools/parser/hlo_lexer.cc    | 270 ++++++++++
 .../compiler/xla/tools/parser/hlo_lexer.h     | 108 ++++
 .../compiler/xla/tools/parser/hlo_parser.cc   | 502 ++++++++++++++++++
 .../compiler/xla/tools/parser/hlo_parser.h    |  37 ++
 .../xla/tools/parser/hlo_parser_test.cc       | 240 +++++++++
 .../compiler/xla/tools/parser/hlo_token.h     |  58 ++
 11 files changed, 1402 insertions(+), 16 deletions(-)
 create mode 100644 tensorflow/compiler/xla/tools/parser/BUILD
 create mode 100644 tensorflow/compiler/xla/tools/parser/README.md
 create mode 100644 tensorflow/compiler/xla/tools/parser/hlo_lexer.cc
 create mode 100644 tensorflow/compiler/xla/tools/parser/hlo_lexer.h
 create mode 100644 tensorflow/compiler/xla/tools/parser/hlo_parser.cc
 create mode 100644 tensorflow/compiler/xla/tools/parser/hlo_parser.h
 create mode 100644 tensorflow/compiler/xla/tools/parser/hlo_parser_test.cc
 create mode 100644 tensorflow/compiler/xla/tools/parser/hlo_token.h

diff --git a/tensorflow/BUILD b/tensorflow/BUILD
index e351037abb..d5c56cdc18 100644
--- a/tensorflow/BUILD
+++ b/tensorflow/BUILD
@@ -340,6 +340,7 @@ filegroup(
         "//tensorflow/compiler/xla/service/llvm_ir:all_files",
         "//tensorflow/compiler/xla/tests:all_files",
         "//tensorflow/compiler/xla/tools:all_files",
+        "//tensorflow/compiler/xla/tools/parser:all_files",
         "//tensorflow/contrib:all_files",
         "//tensorflow/contrib/all_reduce:all_files",
         "//tensorflow/contrib/android:all_files",
diff --git a/tensorflow/compiler/xla/service/hlo_computation.cc b/tensorflow/compiler/xla/service/hlo_computation.cc
index 9b3104eaac..51ead753f0 100644
--- a/tensorflow/compiler/xla/service/hlo_computation.cc
+++ b/tensorflow/compiler/xla/service/hlo_computation.cc
@@ -373,8 +373,8 @@ string HloComputation::ToString(int nested_level) const {
   for (int i = 0; i < nested_level; i++) {
     s << "    ";
   }
-  s << name() << " " << ShapeUtil::HumanString(ComputeProgramShape())
-    << " { \n";
+  s << "%" << name() << " " << ShapeUtil::HumanString(ComputeProgramShape())
+    << " {\n";
   for (const HloInstruction* instruction : MakeInstructionPostOrder()) {
     for (int i = 0; i < nested_level; i++) {
       s << "    ";
diff --git a/tensorflow/compiler/xla/shape_util.cc b/tensorflow/compiler/xla/shape_util.cc
index 8e16056b23..af583bed62 100644
--- a/tensorflow/compiler/xla/shape_util.cc
+++ b/tensorflow/compiler/xla/shape_util.cc
@@ -102,6 +102,32 @@ bool CompareShapes(const Shape& lhs, const Shape& rhs, bool compare_layouts) {
   return true;
 }
 
+// Constructs and returns the new shape with the given minor_to_major order in
+// its Layout.
+StatusOr<Shape> MakeShapeWithLayoutInternal(
+    PrimitiveType element_type, tensorflow::gtl::ArraySlice<int64> dimensions,
+    tensorflow::gtl::ArraySlice<int64> minor_to_major) {
+  if (dimensions.size() != minor_to_major.size()) {
+    return InvalidArgument("Dimensions size is %ld, but layout size is %ld.",
+                           dimensions.size(), minor_to_major.size());
+  }
+  if (element_type == OPAQUE || element_type == TUPLE) {
+    return InvalidArgument("Unsupported element type: %s",
+                           PrimitiveType_Name(element_type).c_str());
+  }
+  Shape shape = ShapeUtil::MakeShape(element_type, dimensions);
+  auto min2maj = shape.mutable_layout()->mutable_minor_to_major();
+  min2maj->Clear();
+  for (int64 value : minor_to_major) {
+    min2maj->Add(value);
+  }
+  if (!shape.has_layout()) {
+    return InvalidArgument("Shape has no layout.");
+  }
+  TF_RETURN_IF_ERROR(ShapeUtil::ValidateShape(shape));
+  return shape;
+}
+
 }  // namespace
 
 /* static */ bool ShapeUtil::Equal(const Shape& lhs, const Shape& rhs) {
@@ -152,16 +178,8 @@ bool CompareShapes(const Shape& lhs, const Shape& rhs, bool compare_layouts) {
 /* static */ Shape ShapeUtil::MakeShapeWithLayout(
     PrimitiveType element_type, tensorflow::gtl::ArraySlice<int64> dimensions,
     tensorflow::gtl::ArraySlice<int64> minor_to_major) {
-  CHECK_EQ(dimensions.size(), minor_to_major.size());
-  Shape shape = MakeShape(element_type, dimensions);
-  auto min2maj = shape.mutable_layout()->mutable_minor_to_major();
-  min2maj->Clear();
-  for (int64 value : minor_to_major) {
-    min2maj->Add(value);
-  }
-  DCHECK(shape.has_layout());
-  TF_DCHECK_OK(ValidateShape(shape));
-  return shape;
+  return MakeShapeWithLayoutInternal(element_type, dimensions, minor_to_major)
+      .ValueOrDie();
 }
 
 /* static */ Shape ShapeUtil::MakeShapeWithMonotonicDim0MajorLayout(
@@ -499,11 +517,10 @@ StatusOr<Shape> ParseShapeStringInternal(tensorflow::StringPiece* s) {
       // Extract the layout minor-to-major and set it.
       TF_ASSIGN_OR_RETURN(std::vector<int64> min2maj,
                           comma_list_to_int64s(layout_string));
-      TF_RET_CHECK(dimensions.size() == min2maj.size());
-      result =
-          ShapeUtil::MakeShapeWithLayout(primitive_type, dimensions, min2maj);
+      TF_ASSIGN_OR_RETURN(result, MakeShapeWithLayoutInternal(
+                                      primitive_type, dimensions, min2maj));
     }
-    TF_DCHECK_OK(ShapeUtil::ValidateShape(result));
+    TF_RETURN_IF_ERROR(ShapeUtil::ValidateShape(result));
     return std::move(result);
   }
 
diff --git a/tensorflow/compiler/xla/tools/parser/BUILD b/tensorflow/compiler/xla/tools/parser/BUILD
new file mode 100644
index 0000000000..c84ca9fc83
--- /dev/null
+++ b/tensorflow/compiler/xla/tools/parser/BUILD
@@ -0,0 +1,84 @@
+# Build file for the Hlo parser.
+
+licenses(["notice"])  # Apache 2.0
+
+package(
+    default_visibility = [":friends"],
+)
+
+package_group(
+    name = "friends",
+    includes = [
+        "//tensorflow/compiler/xla:friends",
+    ],
+)
+
+# Filegroup used to collect source files for dependency checking.
+filegroup(
+    name = "c_srcs",
+    data = glob([
+        "**/*.cc",
+        "**/*.h",
+    ]),
+)
+
+load("//tensorflow:tensorflow.bzl", "tf_cc_test")
+
+cc_library(
+    name = "hlo_lexer",
+    srcs = ["hlo_lexer.cc"],
+    hdrs = [
+        "hlo_lexer.h",
+        "hlo_token.h",
+    ],
+    deps = [
+        "//tensorflow/compiler/xla:shape_util",
+        "//tensorflow/compiler/xla:statusor",
+        "//tensorflow/compiler/xla:util",
+        "//tensorflow/compiler/xla:xla_data_proto",
+        "//tensorflow/compiler/xla/service:hlo",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:regexp_internal",
+    ],
+)
+
+cc_library(
+    name = "hlo_parser",
+    srcs = ["hlo_parser.cc"],
+    hdrs = ["hlo_parser.h"],
+    deps = [
+        ":hlo_lexer",
+        "//tensorflow/compiler/xla:shape_util",
+        "//tensorflow/compiler/xla:statusor",
+        "//tensorflow/compiler/xla:util",
+        "//tensorflow/compiler/xla:xla_data_proto",
+        "//tensorflow/compiler/xla/service:hlo",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:lib_internal",
+    ],
+)
+
+tf_cc_test(
+    name = "hlo_parser_test",
+    size = "small",
+    srcs = ["hlo_parser_test.cc"],
+    deps = [
+        ":hlo_parser",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+    ],
+)
+
+# -----------------------------------------------------------------------------
+
+filegroup(
+    name = "all_files",
+    srcs = glob(
+        ["**/*"],
+        exclude = [
+            "**/METADATA",
+            "**/OWNERS",
+        ],
+    ),
+    visibility = ["//tensorflow:__subpackages__"],
+)
diff --git a/tensorflow/compiler/xla/tools/parser/README.md b/tensorflow/compiler/xla/tools/parser/README.md
new file mode 100644
index 0000000000..a334bc2b29
--- /dev/null
+++ b/tensorflow/compiler/xla/tools/parser/README.md
@@ -0,0 +1,69 @@
+# HloModule string syntax
+
+TODO: Support subcomputations (for fusion, reduce, while, ...).
+
+TODO: Support ops that require extra attributes, e.g. dimensions, strides.
+
+```yacc
+hlo_module
+  : 'HloModule' name computation
+  ;
+
+computation
+  : 'ENTRY' name param_list '->' shape instruction_list
+  ;
+
+instruction_list
+  : '{' instruction_list1 '}'
+  ;
+instruction_list1
+  : instruction
+  | instruction_list1 instruction
+  ;
+instruction
+  : name '=' shape opcode operands
+  ;
+
+operands
+  : '(' operands1 ')'
+  ;
+operands1
+  : /*empty*/
+  | operand
+  | operands1 ',' operand
+  ;
+operand
+  : shape name
+  ;
+
+param_list
+  : '(' param_list1 ')'
+  ;
+param_list1
+  : /*empty*/
+  | param
+  | param_list1 ',' param
+  ;
+param
+  : name shape
+  ;
+
+shape
+  : shape_val_
+  | '(' tuple_elements ')'
+  ;
+tuple_elements
+  : /*empty*/
+  | shape (',' shape)*
+  ;
+
+name
+  : identifier ':'
+  | '%' identifier
+  ;
+
+identifier
+  : [a-zA-Z_][a-zA-Z0-9_.-]*
+  ;
+
+```
diff --git a/tensorflow/compiler/xla/tools/parser/hlo_lexer.cc b/tensorflow/compiler/xla/tools/parser/hlo_lexer.cc
new file mode 100644
index 0000000000..3e84ffcbd2
--- /dev/null
+++ b/tensorflow/compiler/xla/tools/parser/hlo_lexer.cc
@@ -0,0 +1,270 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/tools/parser/hlo_lexer.h"
+
+#include <unordered_map>
+
+#include "tensorflow/compiler/xla/shape_util.h"
+#include "tensorflow/compiler/xla/statusor.h"
+#include "tensorflow/compiler/xla/util.h"
+#include "tensorflow/core/lib/gtl/optional.h"
+#include "tensorflow/core/lib/strings/numbers.h"
+#include "tensorflow/core/platform/regexp.h"
+
+namespace xla {
+namespace tools {
+
+using tensorflow::StringPiece;
+
+namespace {
+
+constexpr int kEOF = -1;
+constexpr int kError = -2;
+
+// [a-zA-Z0-9_.-]
+bool IsIdentifierChar(char c) {
+  return isalnum(static_cast<unsigned char>(c)) || c == '-' || c == '.' ||
+         c == '_';
+}
+
+}  // namespace
+
+int HloLexer::GetNextChar() {
+  int current_char = PeekCurrentChar();
+  if (current_char != kEOF && current_char != kError) {
+    current_ptr_++;
+  }
+  return current_char;
+}
+
+int HloLexer::PeekCurrentChar() const {
+  if (current_ptr_ == buf_.end()) {
+    return kEOF;
+  }
+  char current_char = *current_ptr_;
+  if (current_char == 0) {
+    // '\0' should not appear in the middle of the string.
+    return kError;
+  }
+  return static_cast<unsigned char>(current_char);
+}
+
+bool HloLexer::CanDereference(const char* ptr) const {
+  return ptr < buf_.end() && ptr >= buf_.begin();
+}
+
+StringPiece HloLexer::StringPieceFromPointers(const char* begin,
+                                              const char* end) const {
+  CHECK(begin <= end);
+  CHECK(begin == buf_.end() || CanDereference(begin));
+  CHECK(end == buf_.end() || CanDereference(end));
+  return StringPiece(begin, end - begin);
+}
+
+tensorflow::RegexpStringPiece HloLexer::RegexpStringPieceFromPointers(
+    const char* begin, const char* end) const {
+  CHECK(begin <= end);
+  CHECK(begin == buf_.end() || CanDereference(begin));
+  CHECK(end == buf_.end() || CanDereference(end));
+  return tensorflow::RegexpStringPiece(begin, end - begin);
+}
+
+TokKind HloLexer::LexToken() {
+  while (true) {
+    token_start_ = current_ptr_;
+
+    int current_char = GetNextChar();
+    switch (current_char) {
+      default:
+        // [a-zA-Z_]
+        if (isalpha(static_cast<unsigned char>(current_char)) ||
+            current_char == '_') {
+          return LexIdentifier();
+        }
+        return TokKind::kError;
+      case kEOF:
+        // Hit the end of the input buffer.
+        return TokKind::kEof;
+      case kError:
+        // Hit an invalid character in the input buffer.
+        return TokKind::kError;
+      case ' ':
+      case '\t':
+      case '\n':
+      case '\r':
+        // Ignore whitespace.
+        continue;
+      case '0':
+      case '1':
+      case '2':
+      case '3':
+      case '4':
+      case '5':
+      case '6':
+      case '7':
+      case '8':
+      case '9':
+      case '-':
+        if (current_char == '-' && PeekCurrentChar() == '>') {
+          current_ptr_++;
+          return TokKind::kArrow;
+        }
+        return LexDigitOrNegative();
+      case '=':
+        return TokKind::kEqual;
+      case ',':
+        return TokKind::kComma;
+      case '%':
+        return LexPercent();
+      case ':':
+        return TokKind::kColon;
+      case '[':
+        return TokKind::kLsquare;
+      case ']':
+        return TokKind::kRsquare;
+      case '{':
+        return TokKind::kLbrace;
+      case '}':
+        return TokKind::kRbrace;
+      case '(':
+        return TokKind::kLparen;
+      case ')':
+        return TokKind::kRparen;
+    }
+  }
+}
+
+// Lex a shape, name, keyword, or opcode.
+// shape    ::= ([a-zA-Z0-9_]*[0-9]*)\[([0-9,]*)\](?:\s*{([0-9,]*)})?
+// name     ::= [a-zA-Z_][a-zA-Z0-9_.-]*:
+// keyword  ::= HloModule, ENTRY, ...
+// opcode   ::= add, greater-than, ...
+TokKind HloLexer::LexIdentifier() {
+  {
+    auto consumable = RegexpStringPieceFromPointers(token_start_, buf_.end());
+    // 'consumable' will be advanced iff its prefix matches the pattern.
+    static LazyRE2 shape_pattern = {
+        R"(^(\w*\d*)\[([\d,]*)\](?:\s*{([\d,]*)})?)"};
+    if (RE2::Consume(&consumable, *shape_pattern)) {
+      auto status_or_shape = ShapeUtil::ParseShapeString(
+          StringPieceFromPointers(token_start_, consumable.begin()));
+      if (status_or_shape.ok()) {
+        // This is a shape string.
+        shape_val_ = status_or_shape.ValueOrDie();
+        current_ptr_ = consumable.begin();
+        return TokKind::kShape;
+      }
+    }
+  }
+
+  while (IsIdentifierChar(PeekCurrentChar())) {
+    current_ptr_++;
+  }
+
+  // If followed by ':', it's a name.
+  if (PeekCurrentChar() == ':') {
+    str_val_.assign(token_start_, current_ptr_);
+    current_ptr_++;  // skip ':'
+    return TokKind::kName;
+  }
+
+  StringPiece identifier = StringPieceFromPointers(token_start_, current_ptr_);
+
+  // See if this is a keyword.
+#define KEYWORD(STR)            \
+  do {                          \
+    if (identifier == #STR) {   \
+      return TokKind::kw_##STR; \
+    }                           \
+  } while (false)
+
+  KEYWORD(true);
+  KEYWORD(false);
+  KEYWORD(HloModule);
+  KEYWORD(ENTRY);
+
+#undef KEYWORD
+
+  // See if this is an opcode.
+  auto opcode = StringToHloOpcode(identifier.ToString());
+  if (opcode.ok()) {
+    opcode_val_ = opcode.ValueOrDie();
+    return TokKind::kOpcode;
+  }
+
+  current_ptr_ = token_start_ + 1;
+  return TokKind::kError;
+}
+
+// Lex names after a % character.
+// name ::= [a-zA-Z_][a-zA-Z0-9_.-]*
+TokKind HloLexer::LexPercent() {
+  const char* name_start = current_ptr_;
+  if (isalpha(static_cast<unsigned char>(PeekCurrentChar())) ||
+      PeekCurrentChar() == '_') {
+    current_ptr_++;
+    while (IsIdentifierChar(PeekCurrentChar())) {
+      current_ptr_++;
+    }
+    str_val_.assign(name_start, current_ptr_);
+    return TokKind::kName;
+  }
+  return TokKind::kError;
+}
+
+// Lex integer and floating-point values.
+// int             [-]?[0-9]+
+// fp with exp     [-]?([0-9]+|[0-9]+[.][0-9]*|[0-9]*[.][0-9]+)([eE][+-]?[0-9]+)
+// fp without exp  [-]?([0-9]+[.][0-9]*|[0-9]*[.][0-9]+)
+TokKind HloLexer::LexDigitOrNegative() {
+  auto consumable = RegexpStringPieceFromPointers(token_start_, buf_.end());
+  static LazyRE2 float_pattern = {
+      R"([-]?((\d+|\d+[.]\d*|\d*[.]\d+)([eE][+-]?\d+))|(\d+[.]\d*|\d*[.]\d+))"};
+  if (RE2::Consume(&consumable, *float_pattern)) {
+    current_ptr_ = consumable.begin();
+    tensorflow::strings::safe_strtod(string(token_start_, current_ptr_).c_str(),
+                                     &decimal_val_);
+    return TokKind::kDecimal;
+  }
+
+  static LazyRE2 int_pattern = {R"([-]?\d+)"};
+  if (RE2::Consume(&consumable, *int_pattern)) {
+    current_ptr_ = consumable.begin();
+    tensorflow::strings::safe_strto64(
+        StringPieceFromPointers(token_start_, current_ptr_), &int64_val_);
+    return TokKind::kInt;
+  }
+
+  return TokKind::kError;
+}
+
+StringPiece HloLexer::GetCurrentLine() const {
+  const char* start = token_start_;
+  const char* end = current_ptr_;
+  if (!CanDereference(start) || !CanDereference(end)) {
+    return "LINE OUT OF RANGE";
+  }
+  while (start > buf_.begin() && *start != '\n') {
+    start--;
+  }
+  while (end < buf_.end() && *end != '\n') {
+    end++;
+  }
+  return StringPieceFromPointers(start, end);
+}
+
+}  // namespace tools
+}  // namespace xla
diff --git a/tensorflow/compiler/xla/tools/parser/hlo_lexer.h b/tensorflow/compiler/xla/tools/parser/hlo_lexer.h
new file mode 100644
index 0000000000..20278fd6cd
--- /dev/null
+++ b/tensorflow/compiler/xla/tools/parser/hlo_lexer.h
@@ -0,0 +1,108 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_COMPILER_XLA_TOOLS_PARSER_HLO_LEXER_H_
+#define TENSORFLOW_COMPILER_XLA_TOOLS_PARSER_HLO_LEXER_H_
+
+#include <string>
+
+#include "tensorflow/compiler/xla/service/hlo_opcode.h"
+#include "tensorflow/compiler/xla/tools/parser/hlo_token.h"
+#include "tensorflow/compiler/xla/xla_data.pb.h"
+#include "tensorflow/core/lib/core/stringpiece.h"
+#include "tensorflow/core/platform/logging.h"
+#include "tensorflow/core/platform/regexp.h"
+#include "tensorflow/core/platform/types.h"
+
+namespace xla {
+namespace tools {
+
+// Lexer for the HloModule::ToString() format text.
+class HloLexer {
+ public:
+  explicit HloLexer(tensorflow::StringPiece buf) : buf_(buf) {
+    current_ptr_ = buf_.begin();
+  }
+
+  TokKind Lex() { return current_kind_ = LexToken(); }
+  TokKind GetKind() const { return current_kind_; }
+  string GetStrVal() const {
+    CHECK(GetKind() == TokKind::kName);
+    return str_val_;
+  }
+  Shape GetShapeVal() const {
+    CHECK(GetKind() == TokKind::kShape);
+    return shape_val_;
+  }
+  HloOpcode GetOpcodeVal() const {
+    CHECK(GetKind() == TokKind::kOpcode);
+    return opcode_val_;
+  }
+  int64 GetInt64Val() const {
+    CHECK(GetKind() == TokKind::kInt);
+    return int64_val_;
+  }
+  double GetDecimalVal() const {
+    CHECK(GetKind() == TokKind::kDecimal);
+    return decimal_val_;
+  }
+
+  // Returns the line of text that is currently being lexed.
+  tensorflow::StringPiece GetCurrentLine() const;
+
+ private:
+  // Returns the current character. If it's neither the end of input buffer nor
+  // an invalid character, moves the pointer forward.
+  int GetNextChar();
+
+  // Returns the current character.
+  int PeekCurrentChar() const;
+
+  // Creates StringPiece with the given begin and end. Exits if the begin > end,
+  // or it's out of the range of the current buffer.
+  tensorflow::StringPiece StringPieceFromPointers(const char* begin,
+                                                  const char* end) const;
+  tensorflow::RegexpStringPiece RegexpStringPieceFromPointers(
+      const char* begin, const char* end) const;
+
+  // Returns true if the given ptr is dereferenceable within the range of the
+  // current buffer.
+  bool CanDereference(const char* ptr) const;
+
+  TokKind LexToken();
+
+  TokKind LexIdentifier();
+  TokKind LexPercent();
+  TokKind LexShape();
+  TokKind LexConstant();
+  TokKind LexDigitOrNegative();
+
+  const tensorflow::StringPiece buf_;
+  const char* current_ptr_;
+
+  // Information about the current token.
+  const char* token_start_;
+  TokKind current_kind_;
+  string str_val_;
+  Shape shape_val_;
+  HloOpcode opcode_val_;
+  int64 int64_val_;
+  double decimal_val_;
+};
+
+}  // namespace tools
+}  // namespace xla
+
+#endif  // TENSORFLOW_COMPILER_XLA_TOOLS_PARSER_HLO_LEXER_H_
diff --git a/tensorflow/compiler/xla/tools/parser/hlo_parser.cc b/tensorflow/compiler/xla/tools/parser/hlo_parser.cc
new file mode 100644
index 0000000000..57700493e6
--- /dev/null
+++ b/tensorflow/compiler/xla/tools/parser/hlo_parser.cc
@@ -0,0 +1,502 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/tools/parser/hlo_parser.h"
+
+#include "tensorflow/compiler/xla/shape_util.h"
+#include "tensorflow/core/lib/gtl/map_util.h"
+#include "tensorflow/core/lib/strings/strcat.h"
+
+namespace xla {
+namespace tools {
+
+namespace {
+
+using tensorflow::StringPiece;
+using tensorflow::strings::StrCat;
+
+// Parser for the HloModule::ToString() format text.
+class HloParser {
+ public:
+  explicit HloParser(StringPiece str) : lexer_(str) {}
+
+  // Runs the parser. Returns false if an error occurred.
+  bool Run();
+
+  // Returns the parsed HloModule.
+  std::unique_ptr<HloModule> ConsumeHloModule() { return std::move(module_); }
+
+  // Returns the error information.
+  string GetError() const { return tensorflow::str_util::Join(error_, "\n"); }
+
+ private:
+  // ParseXXX returns false if an error occurred.
+  bool ParseHloModule();
+  bool ParseComputation();
+  bool ParseInstructionList(HloComputation::Builder* builder);
+  bool ParseInstruction(HloComputation::Builder* builder);
+  bool ParseLiteral(std::unique_ptr<Literal>* literal, const Shape& shape);
+  bool ParseOperands(std::vector<HloInstruction*>* operands,
+                     const int expected_size);
+  bool ParseParamList();
+  bool ParseName(string* result);
+  bool ParseShape(Shape* result);
+  bool ParseOpcode(HloOpcode* result);
+  bool ParseInt64(int64* result);
+  bool ParseDecimal(double* result);
+  bool ParseBool(bool* result);
+  bool ParseToken(TokKind kind, const string& msg);
+
+  // Logs the current parsing line and the given message. Always returns false.
+  bool TokenError(StringPiece msg);
+
+  // If the current token is 'kind', eats it (i.e. lexes the next token) and
+  // returns true.
+  bool EatIfPresent(TokKind kind);
+
+  // Adds the instruction to the pool. Returns false and emits an error if the
+  // instruction already exists.
+  bool AddInstruction(const string& name, HloInstruction* instruction);
+
+  // The map from the instruction name to the instruction. This does not own the
+  // instructions.
+  std::unordered_map<string, HloInstruction*> instruction_pool_;
+
+  HloLexer lexer_;
+  std::unique_ptr<HloModule> module_;
+  std::vector<string> error_;
+};
+
+bool HloParser::TokenError(StringPiece msg) {
+  error_.push_back(
+      StrCat("was parsing \"", lexer_.GetCurrentLine(), "\"; ", msg));
+  return false;
+}
+
+bool HloParser::Run() {
+  lexer_.Lex();
+  return ParseHloModule();
+}
+
+// ::= 'HloModule' name computation
+bool HloParser::ParseHloModule() {
+  if (lexer_.GetKind() != TokKind::kw_HloModule) {
+    return TokenError("expects HloModule");
+  }
+  // Eat 'HloModule'
+  lexer_.Lex();
+
+  string name;
+  if (!ParseName(&name)) {
+    return false;
+  }
+
+  module_ = MakeUnique<HloModule>(name);
+
+  return ParseComputation();
+}
+
+// computation ::= 'ENTRY' name param_list '->' shape instruction_list
+bool HloParser::ParseComputation() {
+  string name;
+  if (!ParseToken(TokKind::kw_ENTRY, "expects 'ENTRY'") || !ParseName(&name)) {
+    return false;
+  }
+  auto builder = MakeUnique<HloComputation::Builder>(name);
+
+  Shape shape;
+  if (!ParseParamList() || !ParseToken(TokKind::kArrow, "expects '->'") ||
+      !ParseShape(&shape) || !ParseInstructionList(builder.get())) {
+    return false;
+  }
+  module_->AddEntryComputation(builder->Build());
+  return true;
+}
+
+// instruction_list ::= '{' instruction_list1 '}'
+// instruction_list1 ::= (instruction)+
+bool HloParser::ParseInstructionList(HloComputation::Builder* builder) {
+  if (!ParseToken(TokKind::kLbrace,
+                  "expects '{' at the beginning of instruction list.")) {
+    return false;
+  }
+  do {
+    if (!ParseInstruction(builder)) {
+      return false;
+    }
+  } while (lexer_.GetKind() != TokKind::kRbrace);
+  return ParseToken(TokKind::kRbrace,
+                    "expects '}' at the end of instruction list.");
+}
+
+// instruction ::= name '=' shape opcode operands
+bool HloParser::ParseInstruction(HloComputation::Builder* builder) {
+  string name;
+  Shape shape;
+  HloOpcode opcode;
+  std::vector<HloInstruction*> operands;
+  if (!ParseName(&name) ||
+      !ParseToken(TokKind::kEqual, "expects '=' in instruction") ||
+      !ParseShape(&shape) || !ParseOpcode(&opcode)) {
+    return false;
+  }
+  switch (opcode) {
+    case HloOpcode::kParameter: {
+      int64 parameter_number;
+      return ParseToken(TokKind::kLparen,
+                        "expects '(' before parameter number") &&
+             ParseInt64(&parameter_number) &&
+             ParseToken(TokKind::kRparen,
+                        "expects ')' after parameter number") &&
+             AddInstruction(
+                 name, builder->AddInstruction(HloInstruction::CreateParameter(
+                           parameter_number, shape, name)));
+    }
+    case HloOpcode::kConstant: {
+      std::unique_ptr<Literal> literal;
+      return ParseToken(TokKind::kLparen,
+                        "expects '(' before parameter number") &&
+             ParseLiteral(&literal, shape) &&
+             ParseToken(TokKind::kRparen,
+                        "expects ')' after parameter number") &&
+             AddInstruction(
+                 name, builder->AddInstruction(
+                           HloInstruction::CreateConstant(std::move(literal))));
+    }
+    // Unary ops.
+    case HloOpcode::kAbs:
+    case HloOpcode::kRoundNearestAfz:
+    case HloOpcode::kBitcast:
+    case HloOpcode::kCeil:
+    case HloOpcode::kCopy:
+    case HloOpcode::kCos:
+    case HloOpcode::kExp:
+    case HloOpcode::kIsFinite:
+    case HloOpcode::kFloor:
+    case HloOpcode::kLog:
+    case HloOpcode::kNot:
+    case HloOpcode::kNegate:
+    case HloOpcode::kSign:
+    case HloOpcode::kSin:
+    case HloOpcode::kSort:
+    case HloOpcode::kTanh: {
+      return ParseOperands(&operands, /*expected_size=*/1) &&
+             AddInstruction(name,
+                            builder->AddInstruction(HloInstruction::CreateUnary(
+                                shape, opcode, operands[0])));
+    }
+    // Binary ops.
+    case HloOpcode::kAdd:
+    case HloOpcode::kDivide:
+    case HloOpcode::kMultiply:
+    case HloOpcode::kSubtract:
+    case HloOpcode::kEq:
+    case HloOpcode::kGe:
+    case HloOpcode::kGt:
+    case HloOpcode::kLe:
+    case HloOpcode::kLt:
+    case HloOpcode::kNe:
+    case HloOpcode::kDot:
+    case HloOpcode::kMaximum:
+    case HloOpcode::kMinimum:
+    case HloOpcode::kPower:
+    case HloOpcode::kRemainder:
+    case HloOpcode::kAnd:
+    case HloOpcode::kOr:
+    case HloOpcode::kShiftLeft:
+    case HloOpcode::kShiftRightArithmetic:
+    case HloOpcode::kShiftRightLogical: {
+      return ParseOperands(&operands, /*expected_size=*/2) &&
+             AddInstruction(
+                 name, builder->AddInstruction(HloInstruction::CreateBinary(
+                           shape, opcode, operands[0], operands[1])));
+    }
+    // Ternary ops.
+    case HloOpcode::kClamp:
+    case HloOpcode::kSelect: {
+      return ParseOperands(&operands, /*expected_size=*/3) &&
+             AddInstruction(
+                 name,
+                 builder->AddInstruction(HloInstruction::CreateTernary(
+                     shape, opcode, operands[0], operands[1], operands[2])));
+    }
+    // Other supported ops.
+    case HloOpcode::kConvert: {
+      return ParseOperands(&operands, /*expected_size=*/1) &&
+             AddInstruction(
+                 name, builder->AddInstruction(
+                           HloInstruction::CreateConvert(shape, operands[0])));
+    }
+    case HloOpcode::kCrossReplicaSum: {
+      return ParseOperands(&operands, /*expected_size=*/1) &&
+             AddInstruction(name, builder->AddInstruction(
+                                      HloInstruction::CreateCrossReplicaSum(
+                                          shape, operands[0])));
+    }
+    case HloOpcode::kReshape: {
+      return ParseOperands(&operands, /*expected_size=*/1) &&
+             AddInstruction(
+                 name, builder->AddInstruction(
+                           HloInstruction::CreateReshape(shape, operands[0])));
+    }
+    case HloOpcode::kBroadcast:
+    case HloOpcode::kCall:
+    case HloOpcode::kCustomCall:
+    case HloOpcode::kConcatenate:
+    case HloOpcode::kReducePrecision:
+    case HloOpcode::kConvolution:
+    case HloOpcode::kGetTupleElement:
+    case HloOpcode::kMap:
+    case HloOpcode::kPad:
+    case HloOpcode::kReduce:
+    case HloOpcode::kReduceWindow:
+    case HloOpcode::kSelectAndScatter:
+    case HloOpcode::kReverse:
+    case HloOpcode::kRng:
+    case HloOpcode::kSlice:
+    case HloOpcode::kDynamicSlice:
+    case HloOpcode::kDynamicUpdateSlice:
+    case HloOpcode::kTranspose:
+    case HloOpcode::kTuple:
+    case HloOpcode::kWhile:
+    case HloOpcode::kFusion:
+    case HloOpcode::kBatchNormTraining:
+    case HloOpcode::kBatchNormInference:
+    case HloOpcode::kInfeed:
+    case HloOpcode::kOutfeed:
+    case HloOpcode::kBatchNormGrad:
+    case HloOpcode::kRecv:
+    case HloOpcode::kSend:
+    case HloOpcode::kUpdate:
+    case HloOpcode::kIndex:
+    case HloOpcode::kTrace:
+      return TokenError(StrCat("parsing not yet implemented for op: ",
+                               HloOpcodeString(opcode)));
+  }
+}
+
+bool HloParser::ParseLiteral(std::unique_ptr<Literal>* literal,
+                             const Shape& shape) {
+  switch (shape.element_type()) {
+    case PRED:
+      bool b;
+      if (!ParseBool(&b)) {
+        return false;
+      }
+      *literal = Literal::CreateR0<bool>(b);
+      return true;
+    case S32:
+      int64 i;
+      if (!ParseInt64(&i)) {
+        return false;
+      }
+      *literal = Literal::CreateR0<int32>(i);
+      return true;
+    case F32:
+      double d;
+      if (!ParseDecimal(&d)) {
+        return false;
+      }
+      *literal = Literal::CreateR0<float>(d);
+      return true;
+    default:
+      return TokenError(StrCat("unsupported constant in shape: ",
+                               ShapeUtil::HumanString(shape)));
+  }
+}
+
+// operands ::= '(' operands1 ')'
+// operands1
+//   ::= /*empty*/
+//   ::= operand (, operand)*
+// operand ::= shape name
+bool HloParser::ParseOperands(std::vector<HloInstruction*>* operands,
+                              const int expected_size) {
+  if (!ParseToken(TokKind::kLparen,
+                  "expects '(' at the beginning of operands")) {
+    return false;
+  }
+  if (lexer_.GetKind() == TokKind::kRparen) {
+    // empty
+  } else {
+    do {
+      Shape shape;
+      string name;
+      if (!ParseShape(&shape) || !ParseName(&name)) {
+        return false;
+      }
+      HloInstruction* instruction =
+          tensorflow::gtl::FindPtrOrNull(instruction_pool_, name);
+      if (!instruction) {
+        return TokenError(StrCat("instruction does not exist: ", name));
+      }
+      operands->push_back(instruction);
+    } while (EatIfPresent(TokKind::kComma));
+  }
+  if (expected_size != operands->size()) {
+    return TokenError(StrCat("expects ", expected_size, " operands, but has ",
+                             operands->size(), " operands"));
+  }
+  return ParseToken(TokKind::kRparen, "expects ')' at the end of operands");
+}
+
+// param_list ::= '(' param_list1 ')'
+// param_list1
+//   ::= /*empty*/
+//   ::= param (',' param)*
+// param ::= name shape
+bool HloParser::ParseParamList() {
+  if (!ParseToken(TokKind::kLparen,
+                  "expects '(' at the beginning of param list")) {
+    return false;
+  }
+
+  if (lexer_.GetKind() == TokKind::kRparen) {
+    // empty
+  } else {
+    do {
+      Shape shape;
+      if (!ParseToken(TokKind::kName, "expects name in parameter") ||
+          !ParseShape(&shape)) {
+        return false;
+      }
+    } while (EatIfPresent(TokKind::kComma));
+  }
+  return ParseToken(TokKind::kRparen, "expects ')' at the end of param list");
+}
+
+// shape ::= shape_val_
+// shape ::= '(' tuple_elements ')'
+// tuple_elements
+//   ::= /*empty*/
+//   ::= shape (',' shape)*
+bool HloParser::ParseShape(Shape* result) {
+  if (EatIfPresent(TokKind::kLparen)) {  // Tuple
+    std::vector<Shape> shapes;
+    if (lexer_.GetKind() == TokKind::kRparen) {
+      /*empty*/
+    } else {
+      // shape (',' shape)*
+      do {
+        shapes.emplace_back();
+        if (!ParseShape(&shapes.back())) {
+          return false;
+        }
+      } while (EatIfPresent(TokKind::kComma));
+    }
+    *result = ShapeUtil::MakeTupleShape(shapes);
+    return ParseToken(TokKind::kRparen, "expects ')' at the end of tuple.");
+  }
+
+  if (lexer_.GetKind() != TokKind::kShape) {
+    return TokenError("expects shape");
+  }
+  *result = lexer_.GetShapeVal();
+  lexer_.Lex();
+  return true;
+}
+
+bool HloParser::ParseName(string* result) {
+  VLOG(1) << "ParseName";
+  if (lexer_.GetKind() != TokKind::kName) {
+    return TokenError("expects name");
+  }
+  *result = lexer_.GetStrVal();
+  lexer_.Lex();
+  return true;
+}
+
+bool HloParser::ParseOpcode(HloOpcode* result) {
+  VLOG(1) << "ParseOpcode";
+  if (lexer_.GetKind() != TokKind::kOpcode) {
+    return TokenError("expects opcode");
+  }
+  *result = lexer_.GetOpcodeVal();
+  lexer_.Lex();
+  return true;
+}
+
+bool HloParser::ParseInt64(int64* result) {
+  VLOG(1) << "ParseInt64";
+  if (lexer_.GetKind() != TokKind::kInt) {
+    return TokenError("expects integer");
+  }
+  *result = lexer_.GetInt64Val();
+  lexer_.Lex();
+  return true;
+}
+
+bool HloParser::ParseDecimal(double* result) {
+  switch (lexer_.GetKind()) {
+    case TokKind::kDecimal:
+      *result = lexer_.GetDecimalVal();
+      break;
+    case TokKind::kInt:
+      *result = static_cast<double>(lexer_.GetInt64Val());
+      break;
+    default:
+      return TokenError("expects decimal or integer");
+  }
+  lexer_.Lex();
+  return true;
+}
+
+bool HloParser::ParseBool(bool* result) {
+  if (lexer_.GetKind() != TokKind::kw_true &&
+      lexer_.GetKind() != TokKind::kw_false) {
+    return TokenError("expects true or false");
+  }
+  *result = lexer_.GetKind() == TokKind::kw_true;
+  lexer_.Lex();
+  return true;
+}
+
+bool HloParser::ParseToken(TokKind kind, const string& msg) {
+  if (lexer_.GetKind() != kind) {
+    return TokenError(msg);
+  }
+  lexer_.Lex();
+  return true;
+}
+
+bool HloParser::EatIfPresent(TokKind kind) {
+  if (lexer_.GetKind() != kind) {
+    return false;
+  }
+  lexer_.Lex();
+  return true;
+}
+
+bool HloParser::AddInstruction(const string& name,
+                               HloInstruction* instruction) {
+  auto result = instruction_pool_.insert({name, instruction});
+  if (!result.second) {
+    return TokenError(StrCat("instruction already exists: ", name));
+  }
+  return true;
+}
+
+}  // namespace
+
+StatusOr<std::unique_ptr<HloModule>> Parse(StringPiece str) {
+  HloParser parser(str);
+  if (!parser.Run()) {
+    return InvalidArgument("Syntax error: %s", parser.GetError().c_str());
+  }
+  return parser.ConsumeHloModule();
+}
+
+}  // namespace tools
+}  // namespace xla
diff --git a/tensorflow/compiler/xla/tools/parser/hlo_parser.h b/tensorflow/compiler/xla/tools/parser/hlo_parser.h
new file mode 100644
index 0000000000..9aaf18ef20
--- /dev/null
+++ b/tensorflow/compiler/xla/tools/parser/hlo_parser.h
@@ -0,0 +1,37 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_COMPILER_XLA_TOOLS_PARSER_HLO_PARSER_H_
+#define TENSORFLOW_COMPILER_XLA_TOOLS_PARSER_HLO_PARSER_H_
+
+#include "tensorflow/compiler/xla/ptr_util.h"
+#include "tensorflow/compiler/xla/service/hlo_computation.h"
+#include "tensorflow/compiler/xla/service/hlo_instruction.h"
+#include "tensorflow/compiler/xla/service/hlo_module.h"
+#include "tensorflow/compiler/xla/statusor.h"
+#include "tensorflow/compiler/xla/tools/parser/hlo_lexer.h"
+#include "tensorflow/compiler/xla/xla_data.pb.h"
+
+namespace xla {
+namespace tools {
+
+// The api of the hlo parser. Given a string in the HloModule::ToString()
+// format, returns the parsed HloModule.
+StatusOr<std::unique_ptr<HloModule>> Parse(tensorflow::StringPiece str);
+
+}  // namespace tools
+}  // namespace xla
+
+#endif  // TENSORFLOW_COMPILER_XLA_TOOLS_PARSER_HLO_PARSER_H_
diff --git a/tensorflow/compiler/xla/tools/parser/hlo_parser_test.cc b/tensorflow/compiler/xla/tools/parser/hlo_parser_test.cc
new file mode 100644
index 0000000000..4ecece3eac
--- /dev/null
+++ b/tensorflow/compiler/xla/tools/parser/hlo_parser_test.cc
@@ -0,0 +1,240 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/tools/parser/hlo_parser.h"
+
+#include <string>
+#include "tensorflow/core/lib/core/status_test_util.h"
+#include "tensorflow/core/platform/test.h"
+
+namespace xla {
+namespace tools {
+namespace {
+
+struct TestData {
+  string test_name;
+  string module_string;
+};
+
+string TestDataToString(const ::testing::TestParamInfo<TestData>& data) {
+  return data.param.test_name;
+}
+
+std::vector<TestData> CreateTestCases() {
+  // clang-format off
+  return std::vector<TestData>({
+// ax + y
+{
+"AxpyParam",
+R"(HloModule axpy_module:
+
+ENTRY %axpy.v5 (alpha: f32[2,4], x: f32[2,4], y: f32[2,4]) -> f32[2,4] {
+  %alpha = f32[2,4]{1,0} parameter(0)
+  %x = f32[2,4]{1,0} parameter(1)
+  %multiply = f32[2,4]{1,0} multiply(f32[2,4]{1,0} %alpha, f32[2,4]{1,0} %x)
+  %y = f32[2,4]{1,0} parameter(2)
+  %add = f32[2,4]{1,0} add(f32[2,4]{1,0} %multiply, f32[2,4]{1,0} %y)
+}
+
+)"
+},
+// pred constant
+{
+"ConstantPred",
+R"(HloModule constant_pred_module:
+
+ENTRY %constant_pred () -> pred[] {
+  %constant = pred[] constant(true)
+}
+
+)"
+},
+// s32 constant
+{
+"ConstantS32",
+R"(HloModule constant_s32_module:
+
+ENTRY %constant_s32 () -> s32[] {
+  %constant = s32[] constant(-42)
+}
+
+)"
+},
+// f32 constant, but the value is not a decimal
+{
+"ConstantF32", R"(HloModule ConstantF32_module:
+
+ENTRY %ConstantF32.v4 () -> f32[] {
+  %constant = f32[] constant(42)
+}
+
+)"
+},
+// constant + constant
+{
+"AddConstants",
+R"(HloModule add_constants_module:
+
+ENTRY %add_constants () -> f32[] {
+  %constant = f32[] constant(3.14)
+  %add = f32[] add(f32[] %constant, f32[] %constant)
+}
+
+)"
+},
+// v1 > v2 ? v1 : v2
+{
+"SelectR1F32",
+R"(HloModule SelectR1F32WithCmpR1F32sFromParamsSmall_module:
+
+ENTRY %SelectR1F32WithCmpR1F32sFromParamsSmall.v4 (v1: f32[4], v2: f32[4]) -> f32[4] {
+  %v1 = f32[4]{0} parameter(0)
+  %v2 = f32[4]{0} parameter(1)
+  %greater-than = pred[4]{0} greater-than(f32[4]{0} %v1, f32[4]{0} %v2)
+  %select = f32[4]{0} select(pred[4]{0} %greater-than, f32[4]{0} %v1, f32[4]{0} %v2)
+}
+
+)"
+}
+  });
+  // clang-format on
+}
+
+class HloParserTest : public ::testing::Test,
+                      public ::testing::WithParamInterface<TestData> {
+ protected:
+  void ExpectSuccess() {
+    const string& original = GetParam().module_string;
+    auto result = Parse(original);
+    TF_EXPECT_OK(result.status());
+    EXPECT_EQ(original, result.ValueOrDie()->ToString());
+  }
+};
+
+TEST_P(HloParserTest, Run) { ExpectSuccess(); }
+
+INSTANTIATE_TEST_CASE_P(HloParserTestSuccessInstantiation, HloParserTest,
+                        ::testing::ValuesIn(CreateTestCases()),
+                        TestDataToString);
+
+TEST_F(HloParserTest, Empty) {
+  const string original = "";
+  auto result = Parse(original);
+  EXPECT_NE(tensorflow::Status::OK(), result.status());
+}
+
+TEST_F(HloParserTest, Garbage) {
+  const string original = "HloModule thi$ str1ng makes# N0 sen$e @all!*&^%$";
+  auto result = Parse(original);
+  EXPECT_NE(tensorflow::Status::OK(), result.status());
+}
+
+TEST_F(HloParserTest, WrongOpcode) {
+  const string original = R"(HloModule wrong_opcode:
+
+ENTRY %blabla (x: f32[], y: f32[]) -> f32[] {
+  %x = f32[]{} parameter(0)
+  %y = f32[]{} parameter(1)
+  %le = pred[]{} le(f32[]{} %x, f32[]{} %y)
+}
+
+)";
+  auto result = Parse(original);
+  EXPECT_NE(tensorflow::Status::OK(), result.status());
+}
+
+TEST_F(HloParserTest, WrongShape) {
+  const string original = R"(HloModule wrong_opcode:
+
+ENTRY %blabla (x: g32[]) -> g32[] {
+  %x = g32[]{} parameter(0)
+}
+
+)";
+  auto result = Parse(original);
+  EXPECT_NE(tensorflow::Status::OK(), result.status());
+}
+
+TEST_F(HloParserTest, WrongOperandsSize) {
+  const string original = R"(HloModule wrong_opcode:
+
+ENTRY %blabla (x: f32[]) -> pred[] {
+  %x = f32[]{} parameter(0)
+  %eq = pred[]{} equal-to(f32[]{} %x)
+}
+
+)";
+  auto result = Parse(original);
+  EXPECT_NE(tensorflow::Status::OK(), result.status());
+}
+
+TEST_F(HloParserTest, OperandNotFound) {
+  const string original = R"(HloModule operand_not_found:
+ENTRY %blabla (x: f32[]) -> pred[] {
+  %x = f32[]{} parameter(0)
+  %eq = pred[]{} equal-to(f32[]{} %x, f32[]{} %y)
+}
+)";
+  auto result = Parse(original);
+  EXPECT_NE(tensorflow::Status::OK(), result.status());
+}
+
+TEST_F(HloParserTest, MoreConstants) {
+  const string original = R"(HloModule SelectScalarS32True_module:
+
+ENTRY %SelectScalarS32True.v4 () -> s32[] {
+  %constant.2 = pred[] constant(true)
+  %constant.1 = s32[] constant(-42)
+  %constant = s32[] constant(42)
+  %select = s32[] select(pred[] %constant.2, s32[] %constant.1, s32[] %constant)
+}
+
+)";
+  auto result = Parse(original);
+  TF_EXPECT_OK(result.status());
+  // Constant instructions have no name. The string will be parsed successfully
+  // but the constant names will not be exactly the same.
+}
+
+TEST_F(HloParserTest, ConstantWithExp) {
+  const string original = R"(HloModule ConstantWithExp_module:
+
+ENTRY %ConstantWithExp.v4 () -> f32[] {
+  %constant.1 = f32[] constant(3e+2)
+}
+
+)";
+  auto result = Parse(original);
+  TF_EXPECT_OK(result.status());
+  // The string will be parsed successfully but the output strings are not
+  // exactly the same, because "3e2" is parsed into value 300 and will be
+  // printed as "300".
+}
+
+TEST_F(HloParserTest, Tuple) {
+  const string original = R"(HloModule EmptyTupleCreate_module:
+
+ENTRY %EmptyTupleCreate.v1 () -> () {
+  %tuple = () tuple()
+}
+
+)";
+  auto result = Parse(original);
+  EXPECT_NE(tensorflow::Status::OK(), result.status());
+}
+
+}  // namespace
+}  // namespace tools
+}  // namespace xla
diff --git a/tensorflow/compiler/xla/tools/parser/hlo_token.h b/tensorflow/compiler/xla/tools/parser/hlo_token.h
new file mode 100644
index 0000000000..1f75e17c7f
--- /dev/null
+++ b/tensorflow/compiler/xla/tools/parser/hlo_token.h
@@ -0,0 +1,58 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_COMPILER_XLA_TOOLS_PARSER_HLO_TOKEN_H_
+#define TENSORFLOW_COMPILER_XLA_TOOLS_PARSER_HLO_TOKEN_H_
+
+namespace xla {
+namespace tools {
+
+// Defines different kinds of tokens in a hlo module string.
+enum class TokKind {
+  // Markers
+  kEof,
+  kError,
+
+  // Tokens with no info.
+  kEqual,  // =
+  kComma,  // ,
+  kColon,  // :
+  kLsquare,
+  kRsquare,  // [  ]
+  kLbrace,
+  kRbrace,  // {  }
+  kLparen,
+  kRparen,  // (  )
+
+  kArrow,  // ->
+
+  // Keywords
+  kw_HloModule,
+  kw_ENTRY,
+  kw_true,
+  kw_false,
+
+  // Typed tokens.
+  kName,     // %foo
+  kShape,    // f32[2,3]{1,0}
+  kOpcode,   // add
+  kInt,      // 42
+  kDecimal,  // 4.2
+};
+
+}  // namespace tools
+}  // namespace xla
+
+#endif  // TENSORFLOW_COMPILER_XLA_TOOLS_PARSER_HLO_TOKEN_H_
-- 
GitLab


From 47e92cfd08a230034268a1eeca625fd1e9908616 Mon Sep 17 00:00:00 2001
From: Derek Murray <mrry@google.com>
Date: Thu, 19 Oct 2017 14:23:03 -0700
Subject: [PATCH 579/909] `tf.py_func`: Handle NumPy arrays of np.object that
 hold unicode strings.

This also fixes a bug affecting `tf.data.Dataset.from_generator()` on Python 3,
where the generator yields Unicode (i.e. default) strings.

PiperOrigin-RevId: 172798007
---
 tensorflow/BUILD                              |   1 -
 .../compiler/xla/service/hlo_computation.cc   |   4 +-
 tensorflow/compiler/xla/shape_util.cc         |  45 +-
 tensorflow/compiler/xla/tools/parser/BUILD    |  84 ---
 .../compiler/xla/tools/parser/README.md       |  69 ---
 .../compiler/xla/tools/parser/hlo_lexer.cc    | 270 ----------
 .../compiler/xla/tools/parser/hlo_lexer.h     | 108 ----
 .../compiler/xla/tools/parser/hlo_parser.cc   | 502 ------------------
 .../compiler/xla/tools/parser/hlo_parser.h    |  37 --
 .../xla/tools/parser/hlo_parser_test.cc       | 240 ---------
 .../compiler/xla/tools/parser/hlo_token.h     |  58 --
 11 files changed, 16 insertions(+), 1402 deletions(-)
 delete mode 100644 tensorflow/compiler/xla/tools/parser/BUILD
 delete mode 100644 tensorflow/compiler/xla/tools/parser/README.md
 delete mode 100644 tensorflow/compiler/xla/tools/parser/hlo_lexer.cc
 delete mode 100644 tensorflow/compiler/xla/tools/parser/hlo_lexer.h
 delete mode 100644 tensorflow/compiler/xla/tools/parser/hlo_parser.cc
 delete mode 100644 tensorflow/compiler/xla/tools/parser/hlo_parser.h
 delete mode 100644 tensorflow/compiler/xla/tools/parser/hlo_parser_test.cc
 delete mode 100644 tensorflow/compiler/xla/tools/parser/hlo_token.h

diff --git a/tensorflow/BUILD b/tensorflow/BUILD
index d5c56cdc18..e351037abb 100644
--- a/tensorflow/BUILD
+++ b/tensorflow/BUILD
@@ -340,7 +340,6 @@ filegroup(
         "//tensorflow/compiler/xla/service/llvm_ir:all_files",
         "//tensorflow/compiler/xla/tests:all_files",
         "//tensorflow/compiler/xla/tools:all_files",
-        "//tensorflow/compiler/xla/tools/parser:all_files",
         "//tensorflow/contrib:all_files",
         "//tensorflow/contrib/all_reduce:all_files",
         "//tensorflow/contrib/android:all_files",
diff --git a/tensorflow/compiler/xla/service/hlo_computation.cc b/tensorflow/compiler/xla/service/hlo_computation.cc
index 51ead753f0..9b3104eaac 100644
--- a/tensorflow/compiler/xla/service/hlo_computation.cc
+++ b/tensorflow/compiler/xla/service/hlo_computation.cc
@@ -373,8 +373,8 @@ string HloComputation::ToString(int nested_level) const {
   for (int i = 0; i < nested_level; i++) {
     s << "    ";
   }
-  s << "%" << name() << " " << ShapeUtil::HumanString(ComputeProgramShape())
-    << " {\n";
+  s << name() << " " << ShapeUtil::HumanString(ComputeProgramShape())
+    << " { \n";
   for (const HloInstruction* instruction : MakeInstructionPostOrder()) {
     for (int i = 0; i < nested_level; i++) {
       s << "    ";
diff --git a/tensorflow/compiler/xla/shape_util.cc b/tensorflow/compiler/xla/shape_util.cc
index af583bed62..8e16056b23 100644
--- a/tensorflow/compiler/xla/shape_util.cc
+++ b/tensorflow/compiler/xla/shape_util.cc
@@ -102,32 +102,6 @@ bool CompareShapes(const Shape& lhs, const Shape& rhs, bool compare_layouts) {
   return true;
 }
 
-// Constructs and returns the new shape with the given minor_to_major order in
-// its Layout.
-StatusOr<Shape> MakeShapeWithLayoutInternal(
-    PrimitiveType element_type, tensorflow::gtl::ArraySlice<int64> dimensions,
-    tensorflow::gtl::ArraySlice<int64> minor_to_major) {
-  if (dimensions.size() != minor_to_major.size()) {
-    return InvalidArgument("Dimensions size is %ld, but layout size is %ld.",
-                           dimensions.size(), minor_to_major.size());
-  }
-  if (element_type == OPAQUE || element_type == TUPLE) {
-    return InvalidArgument("Unsupported element type: %s",
-                           PrimitiveType_Name(element_type).c_str());
-  }
-  Shape shape = ShapeUtil::MakeShape(element_type, dimensions);
-  auto min2maj = shape.mutable_layout()->mutable_minor_to_major();
-  min2maj->Clear();
-  for (int64 value : minor_to_major) {
-    min2maj->Add(value);
-  }
-  if (!shape.has_layout()) {
-    return InvalidArgument("Shape has no layout.");
-  }
-  TF_RETURN_IF_ERROR(ShapeUtil::ValidateShape(shape));
-  return shape;
-}
-
 }  // namespace
 
 /* static */ bool ShapeUtil::Equal(const Shape& lhs, const Shape& rhs) {
@@ -178,8 +152,16 @@ StatusOr<Shape> MakeShapeWithLayoutInternal(
 /* static */ Shape ShapeUtil::MakeShapeWithLayout(
     PrimitiveType element_type, tensorflow::gtl::ArraySlice<int64> dimensions,
     tensorflow::gtl::ArraySlice<int64> minor_to_major) {
-  return MakeShapeWithLayoutInternal(element_type, dimensions, minor_to_major)
-      .ValueOrDie();
+  CHECK_EQ(dimensions.size(), minor_to_major.size());
+  Shape shape = MakeShape(element_type, dimensions);
+  auto min2maj = shape.mutable_layout()->mutable_minor_to_major();
+  min2maj->Clear();
+  for (int64 value : minor_to_major) {
+    min2maj->Add(value);
+  }
+  DCHECK(shape.has_layout());
+  TF_DCHECK_OK(ValidateShape(shape));
+  return shape;
 }
 
 /* static */ Shape ShapeUtil::MakeShapeWithMonotonicDim0MajorLayout(
@@ -517,10 +499,11 @@ StatusOr<Shape> ParseShapeStringInternal(tensorflow::StringPiece* s) {
       // Extract the layout minor-to-major and set it.
       TF_ASSIGN_OR_RETURN(std::vector<int64> min2maj,
                           comma_list_to_int64s(layout_string));
-      TF_ASSIGN_OR_RETURN(result, MakeShapeWithLayoutInternal(
-                                      primitive_type, dimensions, min2maj));
+      TF_RET_CHECK(dimensions.size() == min2maj.size());
+      result =
+          ShapeUtil::MakeShapeWithLayout(primitive_type, dimensions, min2maj);
     }
-    TF_RETURN_IF_ERROR(ShapeUtil::ValidateShape(result));
+    TF_DCHECK_OK(ShapeUtil::ValidateShape(result));
     return std::move(result);
   }
 
diff --git a/tensorflow/compiler/xla/tools/parser/BUILD b/tensorflow/compiler/xla/tools/parser/BUILD
deleted file mode 100644
index c84ca9fc83..0000000000
--- a/tensorflow/compiler/xla/tools/parser/BUILD
+++ /dev/null
@@ -1,84 +0,0 @@
-# Build file for the Hlo parser.
-
-licenses(["notice"])  # Apache 2.0
-
-package(
-    default_visibility = [":friends"],
-)
-
-package_group(
-    name = "friends",
-    includes = [
-        "//tensorflow/compiler/xla:friends",
-    ],
-)
-
-# Filegroup used to collect source files for dependency checking.
-filegroup(
-    name = "c_srcs",
-    data = glob([
-        "**/*.cc",
-        "**/*.h",
-    ]),
-)
-
-load("//tensorflow:tensorflow.bzl", "tf_cc_test")
-
-cc_library(
-    name = "hlo_lexer",
-    srcs = ["hlo_lexer.cc"],
-    hdrs = [
-        "hlo_lexer.h",
-        "hlo_token.h",
-    ],
-    deps = [
-        "//tensorflow/compiler/xla:shape_util",
-        "//tensorflow/compiler/xla:statusor",
-        "//tensorflow/compiler/xla:util",
-        "//tensorflow/compiler/xla:xla_data_proto",
-        "//tensorflow/compiler/xla/service:hlo",
-        "//tensorflow/core:lib",
-        "//tensorflow/core:regexp_internal",
-    ],
-)
-
-cc_library(
-    name = "hlo_parser",
-    srcs = ["hlo_parser.cc"],
-    hdrs = ["hlo_parser.h"],
-    deps = [
-        ":hlo_lexer",
-        "//tensorflow/compiler/xla:shape_util",
-        "//tensorflow/compiler/xla:statusor",
-        "//tensorflow/compiler/xla:util",
-        "//tensorflow/compiler/xla:xla_data_proto",
-        "//tensorflow/compiler/xla/service:hlo",
-        "//tensorflow/core:lib",
-        "//tensorflow/core:lib_internal",
-    ],
-)
-
-tf_cc_test(
-    name = "hlo_parser_test",
-    size = "small",
-    srcs = ["hlo_parser_test.cc"],
-    deps = [
-        ":hlo_parser",
-        "//tensorflow/core:test",
-        "//tensorflow/core:test_main",
-    ],
-)
-
-# -----------------------------------------------------------------------------
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/compiler/xla/tools/parser/README.md b/tensorflow/compiler/xla/tools/parser/README.md
deleted file mode 100644
index a334bc2b29..0000000000
--- a/tensorflow/compiler/xla/tools/parser/README.md
+++ /dev/null
@@ -1,69 +0,0 @@
-# HloModule string syntax
-
-TODO: Support subcomputations (for fusion, reduce, while, ...).
-
-TODO: Support ops that require extra attributes, e.g. dimensions, strides.
-
-```yacc
-hlo_module
-  : 'HloModule' name computation
-  ;
-
-computation
-  : 'ENTRY' name param_list '->' shape instruction_list
-  ;
-
-instruction_list
-  : '{' instruction_list1 '}'
-  ;
-instruction_list1
-  : instruction
-  | instruction_list1 instruction
-  ;
-instruction
-  : name '=' shape opcode operands
-  ;
-
-operands
-  : '(' operands1 ')'
-  ;
-operands1
-  : /*empty*/
-  | operand
-  | operands1 ',' operand
-  ;
-operand
-  : shape name
-  ;
-
-param_list
-  : '(' param_list1 ')'
-  ;
-param_list1
-  : /*empty*/
-  | param
-  | param_list1 ',' param
-  ;
-param
-  : name shape
-  ;
-
-shape
-  : shape_val_
-  | '(' tuple_elements ')'
-  ;
-tuple_elements
-  : /*empty*/
-  | shape (',' shape)*
-  ;
-
-name
-  : identifier ':'
-  | '%' identifier
-  ;
-
-identifier
-  : [a-zA-Z_][a-zA-Z0-9_.-]*
-  ;
-
-```
diff --git a/tensorflow/compiler/xla/tools/parser/hlo_lexer.cc b/tensorflow/compiler/xla/tools/parser/hlo_lexer.cc
deleted file mode 100644
index 3e84ffcbd2..0000000000
--- a/tensorflow/compiler/xla/tools/parser/hlo_lexer.cc
+++ /dev/null
@@ -1,270 +0,0 @@
-/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#include "tensorflow/compiler/xla/tools/parser/hlo_lexer.h"
-
-#include <unordered_map>
-
-#include "tensorflow/compiler/xla/shape_util.h"
-#include "tensorflow/compiler/xla/statusor.h"
-#include "tensorflow/compiler/xla/util.h"
-#include "tensorflow/core/lib/gtl/optional.h"
-#include "tensorflow/core/lib/strings/numbers.h"
-#include "tensorflow/core/platform/regexp.h"
-
-namespace xla {
-namespace tools {
-
-using tensorflow::StringPiece;
-
-namespace {
-
-constexpr int kEOF = -1;
-constexpr int kError = -2;
-
-// [a-zA-Z0-9_.-]
-bool IsIdentifierChar(char c) {
-  return isalnum(static_cast<unsigned char>(c)) || c == '-' || c == '.' ||
-         c == '_';
-}
-
-}  // namespace
-
-int HloLexer::GetNextChar() {
-  int current_char = PeekCurrentChar();
-  if (current_char != kEOF && current_char != kError) {
-    current_ptr_++;
-  }
-  return current_char;
-}
-
-int HloLexer::PeekCurrentChar() const {
-  if (current_ptr_ == buf_.end()) {
-    return kEOF;
-  }
-  char current_char = *current_ptr_;
-  if (current_char == 0) {
-    // '\0' should not appear in the middle of the string.
-    return kError;
-  }
-  return static_cast<unsigned char>(current_char);
-}
-
-bool HloLexer::CanDereference(const char* ptr) const {
-  return ptr < buf_.end() && ptr >= buf_.begin();
-}
-
-StringPiece HloLexer::StringPieceFromPointers(const char* begin,
-                                              const char* end) const {
-  CHECK(begin <= end);
-  CHECK(begin == buf_.end() || CanDereference(begin));
-  CHECK(end == buf_.end() || CanDereference(end));
-  return StringPiece(begin, end - begin);
-}
-
-tensorflow::RegexpStringPiece HloLexer::RegexpStringPieceFromPointers(
-    const char* begin, const char* end) const {
-  CHECK(begin <= end);
-  CHECK(begin == buf_.end() || CanDereference(begin));
-  CHECK(end == buf_.end() || CanDereference(end));
-  return tensorflow::RegexpStringPiece(begin, end - begin);
-}
-
-TokKind HloLexer::LexToken() {
-  while (true) {
-    token_start_ = current_ptr_;
-
-    int current_char = GetNextChar();
-    switch (current_char) {
-      default:
-        // [a-zA-Z_]
-        if (isalpha(static_cast<unsigned char>(current_char)) ||
-            current_char == '_') {
-          return LexIdentifier();
-        }
-        return TokKind::kError;
-      case kEOF:
-        // Hit the end of the input buffer.
-        return TokKind::kEof;
-      case kError:
-        // Hit an invalid character in the input buffer.
-        return TokKind::kError;
-      case ' ':
-      case '\t':
-      case '\n':
-      case '\r':
-        // Ignore whitespace.
-        continue;
-      case '0':
-      case '1':
-      case '2':
-      case '3':
-      case '4':
-      case '5':
-      case '6':
-      case '7':
-      case '8':
-      case '9':
-      case '-':
-        if (current_char == '-' && PeekCurrentChar() == '>') {
-          current_ptr_++;
-          return TokKind::kArrow;
-        }
-        return LexDigitOrNegative();
-      case '=':
-        return TokKind::kEqual;
-      case ',':
-        return TokKind::kComma;
-      case '%':
-        return LexPercent();
-      case ':':
-        return TokKind::kColon;
-      case '[':
-        return TokKind::kLsquare;
-      case ']':
-        return TokKind::kRsquare;
-      case '{':
-        return TokKind::kLbrace;
-      case '}':
-        return TokKind::kRbrace;
-      case '(':
-        return TokKind::kLparen;
-      case ')':
-        return TokKind::kRparen;
-    }
-  }
-}
-
-// Lex a shape, name, keyword, or opcode.
-// shape    ::= ([a-zA-Z0-9_]*[0-9]*)\[([0-9,]*)\](?:\s*{([0-9,]*)})?
-// name     ::= [a-zA-Z_][a-zA-Z0-9_.-]*:
-// keyword  ::= HloModule, ENTRY, ...
-// opcode   ::= add, greater-than, ...
-TokKind HloLexer::LexIdentifier() {
-  {
-    auto consumable = RegexpStringPieceFromPointers(token_start_, buf_.end());
-    // 'consumable' will be advanced iff its prefix matches the pattern.
-    static LazyRE2 shape_pattern = {
-        R"(^(\w*\d*)\[([\d,]*)\](?:\s*{([\d,]*)})?)"};
-    if (RE2::Consume(&consumable, *shape_pattern)) {
-      auto status_or_shape = ShapeUtil::ParseShapeString(
-          StringPieceFromPointers(token_start_, consumable.begin()));
-      if (status_or_shape.ok()) {
-        // This is a shape string.
-        shape_val_ = status_or_shape.ValueOrDie();
-        current_ptr_ = consumable.begin();
-        return TokKind::kShape;
-      }
-    }
-  }
-
-  while (IsIdentifierChar(PeekCurrentChar())) {
-    current_ptr_++;
-  }
-
-  // If followed by ':', it's a name.
-  if (PeekCurrentChar() == ':') {
-    str_val_.assign(token_start_, current_ptr_);
-    current_ptr_++;  // skip ':'
-    return TokKind::kName;
-  }
-
-  StringPiece identifier = StringPieceFromPointers(token_start_, current_ptr_);
-
-  // See if this is a keyword.
-#define KEYWORD(STR)            \
-  do {                          \
-    if (identifier == #STR) {   \
-      return TokKind::kw_##STR; \
-    }                           \
-  } while (false)
-
-  KEYWORD(true);
-  KEYWORD(false);
-  KEYWORD(HloModule);
-  KEYWORD(ENTRY);
-
-#undef KEYWORD
-
-  // See if this is an opcode.
-  auto opcode = StringToHloOpcode(identifier.ToString());
-  if (opcode.ok()) {
-    opcode_val_ = opcode.ValueOrDie();
-    return TokKind::kOpcode;
-  }
-
-  current_ptr_ = token_start_ + 1;
-  return TokKind::kError;
-}
-
-// Lex names after a % character.
-// name ::= [a-zA-Z_][a-zA-Z0-9_.-]*
-TokKind HloLexer::LexPercent() {
-  const char* name_start = current_ptr_;
-  if (isalpha(static_cast<unsigned char>(PeekCurrentChar())) ||
-      PeekCurrentChar() == '_') {
-    current_ptr_++;
-    while (IsIdentifierChar(PeekCurrentChar())) {
-      current_ptr_++;
-    }
-    str_val_.assign(name_start, current_ptr_);
-    return TokKind::kName;
-  }
-  return TokKind::kError;
-}
-
-// Lex integer and floating-point values.
-// int             [-]?[0-9]+
-// fp with exp     [-]?([0-9]+|[0-9]+[.][0-9]*|[0-9]*[.][0-9]+)([eE][+-]?[0-9]+)
-// fp without exp  [-]?([0-9]+[.][0-9]*|[0-9]*[.][0-9]+)
-TokKind HloLexer::LexDigitOrNegative() {
-  auto consumable = RegexpStringPieceFromPointers(token_start_, buf_.end());
-  static LazyRE2 float_pattern = {
-      R"([-]?((\d+|\d+[.]\d*|\d*[.]\d+)([eE][+-]?\d+))|(\d+[.]\d*|\d*[.]\d+))"};
-  if (RE2::Consume(&consumable, *float_pattern)) {
-    current_ptr_ = consumable.begin();
-    tensorflow::strings::safe_strtod(string(token_start_, current_ptr_).c_str(),
-                                     &decimal_val_);
-    return TokKind::kDecimal;
-  }
-
-  static LazyRE2 int_pattern = {R"([-]?\d+)"};
-  if (RE2::Consume(&consumable, *int_pattern)) {
-    current_ptr_ = consumable.begin();
-    tensorflow::strings::safe_strto64(
-        StringPieceFromPointers(token_start_, current_ptr_), &int64_val_);
-    return TokKind::kInt;
-  }
-
-  return TokKind::kError;
-}
-
-StringPiece HloLexer::GetCurrentLine() const {
-  const char* start = token_start_;
-  const char* end = current_ptr_;
-  if (!CanDereference(start) || !CanDereference(end)) {
-    return "LINE OUT OF RANGE";
-  }
-  while (start > buf_.begin() && *start != '\n') {
-    start--;
-  }
-  while (end < buf_.end() && *end != '\n') {
-    end++;
-  }
-  return StringPieceFromPointers(start, end);
-}
-
-}  // namespace tools
-}  // namespace xla
diff --git a/tensorflow/compiler/xla/tools/parser/hlo_lexer.h b/tensorflow/compiler/xla/tools/parser/hlo_lexer.h
deleted file mode 100644
index 20278fd6cd..0000000000
--- a/tensorflow/compiler/xla/tools/parser/hlo_lexer.h
+++ /dev/null
@@ -1,108 +0,0 @@
-/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#ifndef TENSORFLOW_COMPILER_XLA_TOOLS_PARSER_HLO_LEXER_H_
-#define TENSORFLOW_COMPILER_XLA_TOOLS_PARSER_HLO_LEXER_H_
-
-#include <string>
-
-#include "tensorflow/compiler/xla/service/hlo_opcode.h"
-#include "tensorflow/compiler/xla/tools/parser/hlo_token.h"
-#include "tensorflow/compiler/xla/xla_data.pb.h"
-#include "tensorflow/core/lib/core/stringpiece.h"
-#include "tensorflow/core/platform/logging.h"
-#include "tensorflow/core/platform/regexp.h"
-#include "tensorflow/core/platform/types.h"
-
-namespace xla {
-namespace tools {
-
-// Lexer for the HloModule::ToString() format text.
-class HloLexer {
- public:
-  explicit HloLexer(tensorflow::StringPiece buf) : buf_(buf) {
-    current_ptr_ = buf_.begin();
-  }
-
-  TokKind Lex() { return current_kind_ = LexToken(); }
-  TokKind GetKind() const { return current_kind_; }
-  string GetStrVal() const {
-    CHECK(GetKind() == TokKind::kName);
-    return str_val_;
-  }
-  Shape GetShapeVal() const {
-    CHECK(GetKind() == TokKind::kShape);
-    return shape_val_;
-  }
-  HloOpcode GetOpcodeVal() const {
-    CHECK(GetKind() == TokKind::kOpcode);
-    return opcode_val_;
-  }
-  int64 GetInt64Val() const {
-    CHECK(GetKind() == TokKind::kInt);
-    return int64_val_;
-  }
-  double GetDecimalVal() const {
-    CHECK(GetKind() == TokKind::kDecimal);
-    return decimal_val_;
-  }
-
-  // Returns the line of text that is currently being lexed.
-  tensorflow::StringPiece GetCurrentLine() const;
-
- private:
-  // Returns the current character. If it's neither the end of input buffer nor
-  // an invalid character, moves the pointer forward.
-  int GetNextChar();
-
-  // Returns the current character.
-  int PeekCurrentChar() const;
-
-  // Creates StringPiece with the given begin and end. Exits if the begin > end,
-  // or it's out of the range of the current buffer.
-  tensorflow::StringPiece StringPieceFromPointers(const char* begin,
-                                                  const char* end) const;
-  tensorflow::RegexpStringPiece RegexpStringPieceFromPointers(
-      const char* begin, const char* end) const;
-
-  // Returns true if the given ptr is dereferenceable within the range of the
-  // current buffer.
-  bool CanDereference(const char* ptr) const;
-
-  TokKind LexToken();
-
-  TokKind LexIdentifier();
-  TokKind LexPercent();
-  TokKind LexShape();
-  TokKind LexConstant();
-  TokKind LexDigitOrNegative();
-
-  const tensorflow::StringPiece buf_;
-  const char* current_ptr_;
-
-  // Information about the current token.
-  const char* token_start_;
-  TokKind current_kind_;
-  string str_val_;
-  Shape shape_val_;
-  HloOpcode opcode_val_;
-  int64 int64_val_;
-  double decimal_val_;
-};
-
-}  // namespace tools
-}  // namespace xla
-
-#endif  // TENSORFLOW_COMPILER_XLA_TOOLS_PARSER_HLO_LEXER_H_
diff --git a/tensorflow/compiler/xla/tools/parser/hlo_parser.cc b/tensorflow/compiler/xla/tools/parser/hlo_parser.cc
deleted file mode 100644
index 57700493e6..0000000000
--- a/tensorflow/compiler/xla/tools/parser/hlo_parser.cc
+++ /dev/null
@@ -1,502 +0,0 @@
-/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#include "tensorflow/compiler/xla/tools/parser/hlo_parser.h"
-
-#include "tensorflow/compiler/xla/shape_util.h"
-#include "tensorflow/core/lib/gtl/map_util.h"
-#include "tensorflow/core/lib/strings/strcat.h"
-
-namespace xla {
-namespace tools {
-
-namespace {
-
-using tensorflow::StringPiece;
-using tensorflow::strings::StrCat;
-
-// Parser for the HloModule::ToString() format text.
-class HloParser {
- public:
-  explicit HloParser(StringPiece str) : lexer_(str) {}
-
-  // Runs the parser. Returns false if an error occurred.
-  bool Run();
-
-  // Returns the parsed HloModule.
-  std::unique_ptr<HloModule> ConsumeHloModule() { return std::move(module_); }
-
-  // Returns the error information.
-  string GetError() const { return tensorflow::str_util::Join(error_, "\n"); }
-
- private:
-  // ParseXXX returns false if an error occurred.
-  bool ParseHloModule();
-  bool ParseComputation();
-  bool ParseInstructionList(HloComputation::Builder* builder);
-  bool ParseInstruction(HloComputation::Builder* builder);
-  bool ParseLiteral(std::unique_ptr<Literal>* literal, const Shape& shape);
-  bool ParseOperands(std::vector<HloInstruction*>* operands,
-                     const int expected_size);
-  bool ParseParamList();
-  bool ParseName(string* result);
-  bool ParseShape(Shape* result);
-  bool ParseOpcode(HloOpcode* result);
-  bool ParseInt64(int64* result);
-  bool ParseDecimal(double* result);
-  bool ParseBool(bool* result);
-  bool ParseToken(TokKind kind, const string& msg);
-
-  // Logs the current parsing line and the given message. Always returns false.
-  bool TokenError(StringPiece msg);
-
-  // If the current token is 'kind', eats it (i.e. lexes the next token) and
-  // returns true.
-  bool EatIfPresent(TokKind kind);
-
-  // Adds the instruction to the pool. Returns false and emits an error if the
-  // instruction already exists.
-  bool AddInstruction(const string& name, HloInstruction* instruction);
-
-  // The map from the instruction name to the instruction. This does not own the
-  // instructions.
-  std::unordered_map<string, HloInstruction*> instruction_pool_;
-
-  HloLexer lexer_;
-  std::unique_ptr<HloModule> module_;
-  std::vector<string> error_;
-};
-
-bool HloParser::TokenError(StringPiece msg) {
-  error_.push_back(
-      StrCat("was parsing \"", lexer_.GetCurrentLine(), "\"; ", msg));
-  return false;
-}
-
-bool HloParser::Run() {
-  lexer_.Lex();
-  return ParseHloModule();
-}
-
-// ::= 'HloModule' name computation
-bool HloParser::ParseHloModule() {
-  if (lexer_.GetKind() != TokKind::kw_HloModule) {
-    return TokenError("expects HloModule");
-  }
-  // Eat 'HloModule'
-  lexer_.Lex();
-
-  string name;
-  if (!ParseName(&name)) {
-    return false;
-  }
-
-  module_ = MakeUnique<HloModule>(name);
-
-  return ParseComputation();
-}
-
-// computation ::= 'ENTRY' name param_list '->' shape instruction_list
-bool HloParser::ParseComputation() {
-  string name;
-  if (!ParseToken(TokKind::kw_ENTRY, "expects 'ENTRY'") || !ParseName(&name)) {
-    return false;
-  }
-  auto builder = MakeUnique<HloComputation::Builder>(name);
-
-  Shape shape;
-  if (!ParseParamList() || !ParseToken(TokKind::kArrow, "expects '->'") ||
-      !ParseShape(&shape) || !ParseInstructionList(builder.get())) {
-    return false;
-  }
-  module_->AddEntryComputation(builder->Build());
-  return true;
-}
-
-// instruction_list ::= '{' instruction_list1 '}'
-// instruction_list1 ::= (instruction)+
-bool HloParser::ParseInstructionList(HloComputation::Builder* builder) {
-  if (!ParseToken(TokKind::kLbrace,
-                  "expects '{' at the beginning of instruction list.")) {
-    return false;
-  }
-  do {
-    if (!ParseInstruction(builder)) {
-      return false;
-    }
-  } while (lexer_.GetKind() != TokKind::kRbrace);
-  return ParseToken(TokKind::kRbrace,
-                    "expects '}' at the end of instruction list.");
-}
-
-// instruction ::= name '=' shape opcode operands
-bool HloParser::ParseInstruction(HloComputation::Builder* builder) {
-  string name;
-  Shape shape;
-  HloOpcode opcode;
-  std::vector<HloInstruction*> operands;
-  if (!ParseName(&name) ||
-      !ParseToken(TokKind::kEqual, "expects '=' in instruction") ||
-      !ParseShape(&shape) || !ParseOpcode(&opcode)) {
-    return false;
-  }
-  switch (opcode) {
-    case HloOpcode::kParameter: {
-      int64 parameter_number;
-      return ParseToken(TokKind::kLparen,
-                        "expects '(' before parameter number") &&
-             ParseInt64(&parameter_number) &&
-             ParseToken(TokKind::kRparen,
-                        "expects ')' after parameter number") &&
-             AddInstruction(
-                 name, builder->AddInstruction(HloInstruction::CreateParameter(
-                           parameter_number, shape, name)));
-    }
-    case HloOpcode::kConstant: {
-      std::unique_ptr<Literal> literal;
-      return ParseToken(TokKind::kLparen,
-                        "expects '(' before parameter number") &&
-             ParseLiteral(&literal, shape) &&
-             ParseToken(TokKind::kRparen,
-                        "expects ')' after parameter number") &&
-             AddInstruction(
-                 name, builder->AddInstruction(
-                           HloInstruction::CreateConstant(std::move(literal))));
-    }
-    // Unary ops.
-    case HloOpcode::kAbs:
-    case HloOpcode::kRoundNearestAfz:
-    case HloOpcode::kBitcast:
-    case HloOpcode::kCeil:
-    case HloOpcode::kCopy:
-    case HloOpcode::kCos:
-    case HloOpcode::kExp:
-    case HloOpcode::kIsFinite:
-    case HloOpcode::kFloor:
-    case HloOpcode::kLog:
-    case HloOpcode::kNot:
-    case HloOpcode::kNegate:
-    case HloOpcode::kSign:
-    case HloOpcode::kSin:
-    case HloOpcode::kSort:
-    case HloOpcode::kTanh: {
-      return ParseOperands(&operands, /*expected_size=*/1) &&
-             AddInstruction(name,
-                            builder->AddInstruction(HloInstruction::CreateUnary(
-                                shape, opcode, operands[0])));
-    }
-    // Binary ops.
-    case HloOpcode::kAdd:
-    case HloOpcode::kDivide:
-    case HloOpcode::kMultiply:
-    case HloOpcode::kSubtract:
-    case HloOpcode::kEq:
-    case HloOpcode::kGe:
-    case HloOpcode::kGt:
-    case HloOpcode::kLe:
-    case HloOpcode::kLt:
-    case HloOpcode::kNe:
-    case HloOpcode::kDot:
-    case HloOpcode::kMaximum:
-    case HloOpcode::kMinimum:
-    case HloOpcode::kPower:
-    case HloOpcode::kRemainder:
-    case HloOpcode::kAnd:
-    case HloOpcode::kOr:
-    case HloOpcode::kShiftLeft:
-    case HloOpcode::kShiftRightArithmetic:
-    case HloOpcode::kShiftRightLogical: {
-      return ParseOperands(&operands, /*expected_size=*/2) &&
-             AddInstruction(
-                 name, builder->AddInstruction(HloInstruction::CreateBinary(
-                           shape, opcode, operands[0], operands[1])));
-    }
-    // Ternary ops.
-    case HloOpcode::kClamp:
-    case HloOpcode::kSelect: {
-      return ParseOperands(&operands, /*expected_size=*/3) &&
-             AddInstruction(
-                 name,
-                 builder->AddInstruction(HloInstruction::CreateTernary(
-                     shape, opcode, operands[0], operands[1], operands[2])));
-    }
-    // Other supported ops.
-    case HloOpcode::kConvert: {
-      return ParseOperands(&operands, /*expected_size=*/1) &&
-             AddInstruction(
-                 name, builder->AddInstruction(
-                           HloInstruction::CreateConvert(shape, operands[0])));
-    }
-    case HloOpcode::kCrossReplicaSum: {
-      return ParseOperands(&operands, /*expected_size=*/1) &&
-             AddInstruction(name, builder->AddInstruction(
-                                      HloInstruction::CreateCrossReplicaSum(
-                                          shape, operands[0])));
-    }
-    case HloOpcode::kReshape: {
-      return ParseOperands(&operands, /*expected_size=*/1) &&
-             AddInstruction(
-                 name, builder->AddInstruction(
-                           HloInstruction::CreateReshape(shape, operands[0])));
-    }
-    case HloOpcode::kBroadcast:
-    case HloOpcode::kCall:
-    case HloOpcode::kCustomCall:
-    case HloOpcode::kConcatenate:
-    case HloOpcode::kReducePrecision:
-    case HloOpcode::kConvolution:
-    case HloOpcode::kGetTupleElement:
-    case HloOpcode::kMap:
-    case HloOpcode::kPad:
-    case HloOpcode::kReduce:
-    case HloOpcode::kReduceWindow:
-    case HloOpcode::kSelectAndScatter:
-    case HloOpcode::kReverse:
-    case HloOpcode::kRng:
-    case HloOpcode::kSlice:
-    case HloOpcode::kDynamicSlice:
-    case HloOpcode::kDynamicUpdateSlice:
-    case HloOpcode::kTranspose:
-    case HloOpcode::kTuple:
-    case HloOpcode::kWhile:
-    case HloOpcode::kFusion:
-    case HloOpcode::kBatchNormTraining:
-    case HloOpcode::kBatchNormInference:
-    case HloOpcode::kInfeed:
-    case HloOpcode::kOutfeed:
-    case HloOpcode::kBatchNormGrad:
-    case HloOpcode::kRecv:
-    case HloOpcode::kSend:
-    case HloOpcode::kUpdate:
-    case HloOpcode::kIndex:
-    case HloOpcode::kTrace:
-      return TokenError(StrCat("parsing not yet implemented for op: ",
-                               HloOpcodeString(opcode)));
-  }
-}
-
-bool HloParser::ParseLiteral(std::unique_ptr<Literal>* literal,
-                             const Shape& shape) {
-  switch (shape.element_type()) {
-    case PRED:
-      bool b;
-      if (!ParseBool(&b)) {
-        return false;
-      }
-      *literal = Literal::CreateR0<bool>(b);
-      return true;
-    case S32:
-      int64 i;
-      if (!ParseInt64(&i)) {
-        return false;
-      }
-      *literal = Literal::CreateR0<int32>(i);
-      return true;
-    case F32:
-      double d;
-      if (!ParseDecimal(&d)) {
-        return false;
-      }
-      *literal = Literal::CreateR0<float>(d);
-      return true;
-    default:
-      return TokenError(StrCat("unsupported constant in shape: ",
-                               ShapeUtil::HumanString(shape)));
-  }
-}
-
-// operands ::= '(' operands1 ')'
-// operands1
-//   ::= /*empty*/
-//   ::= operand (, operand)*
-// operand ::= shape name
-bool HloParser::ParseOperands(std::vector<HloInstruction*>* operands,
-                              const int expected_size) {
-  if (!ParseToken(TokKind::kLparen,
-                  "expects '(' at the beginning of operands")) {
-    return false;
-  }
-  if (lexer_.GetKind() == TokKind::kRparen) {
-    // empty
-  } else {
-    do {
-      Shape shape;
-      string name;
-      if (!ParseShape(&shape) || !ParseName(&name)) {
-        return false;
-      }
-      HloInstruction* instruction =
-          tensorflow::gtl::FindPtrOrNull(instruction_pool_, name);
-      if (!instruction) {
-        return TokenError(StrCat("instruction does not exist: ", name));
-      }
-      operands->push_back(instruction);
-    } while (EatIfPresent(TokKind::kComma));
-  }
-  if (expected_size != operands->size()) {
-    return TokenError(StrCat("expects ", expected_size, " operands, but has ",
-                             operands->size(), " operands"));
-  }
-  return ParseToken(TokKind::kRparen, "expects ')' at the end of operands");
-}
-
-// param_list ::= '(' param_list1 ')'
-// param_list1
-//   ::= /*empty*/
-//   ::= param (',' param)*
-// param ::= name shape
-bool HloParser::ParseParamList() {
-  if (!ParseToken(TokKind::kLparen,
-                  "expects '(' at the beginning of param list")) {
-    return false;
-  }
-
-  if (lexer_.GetKind() == TokKind::kRparen) {
-    // empty
-  } else {
-    do {
-      Shape shape;
-      if (!ParseToken(TokKind::kName, "expects name in parameter") ||
-          !ParseShape(&shape)) {
-        return false;
-      }
-    } while (EatIfPresent(TokKind::kComma));
-  }
-  return ParseToken(TokKind::kRparen, "expects ')' at the end of param list");
-}
-
-// shape ::= shape_val_
-// shape ::= '(' tuple_elements ')'
-// tuple_elements
-//   ::= /*empty*/
-//   ::= shape (',' shape)*
-bool HloParser::ParseShape(Shape* result) {
-  if (EatIfPresent(TokKind::kLparen)) {  // Tuple
-    std::vector<Shape> shapes;
-    if (lexer_.GetKind() == TokKind::kRparen) {
-      /*empty*/
-    } else {
-      // shape (',' shape)*
-      do {
-        shapes.emplace_back();
-        if (!ParseShape(&shapes.back())) {
-          return false;
-        }
-      } while (EatIfPresent(TokKind::kComma));
-    }
-    *result = ShapeUtil::MakeTupleShape(shapes);
-    return ParseToken(TokKind::kRparen, "expects ')' at the end of tuple.");
-  }
-
-  if (lexer_.GetKind() != TokKind::kShape) {
-    return TokenError("expects shape");
-  }
-  *result = lexer_.GetShapeVal();
-  lexer_.Lex();
-  return true;
-}
-
-bool HloParser::ParseName(string* result) {
-  VLOG(1) << "ParseName";
-  if (lexer_.GetKind() != TokKind::kName) {
-    return TokenError("expects name");
-  }
-  *result = lexer_.GetStrVal();
-  lexer_.Lex();
-  return true;
-}
-
-bool HloParser::ParseOpcode(HloOpcode* result) {
-  VLOG(1) << "ParseOpcode";
-  if (lexer_.GetKind() != TokKind::kOpcode) {
-    return TokenError("expects opcode");
-  }
-  *result = lexer_.GetOpcodeVal();
-  lexer_.Lex();
-  return true;
-}
-
-bool HloParser::ParseInt64(int64* result) {
-  VLOG(1) << "ParseInt64";
-  if (lexer_.GetKind() != TokKind::kInt) {
-    return TokenError("expects integer");
-  }
-  *result = lexer_.GetInt64Val();
-  lexer_.Lex();
-  return true;
-}
-
-bool HloParser::ParseDecimal(double* result) {
-  switch (lexer_.GetKind()) {
-    case TokKind::kDecimal:
-      *result = lexer_.GetDecimalVal();
-      break;
-    case TokKind::kInt:
-      *result = static_cast<double>(lexer_.GetInt64Val());
-      break;
-    default:
-      return TokenError("expects decimal or integer");
-  }
-  lexer_.Lex();
-  return true;
-}
-
-bool HloParser::ParseBool(bool* result) {
-  if (lexer_.GetKind() != TokKind::kw_true &&
-      lexer_.GetKind() != TokKind::kw_false) {
-    return TokenError("expects true or false");
-  }
-  *result = lexer_.GetKind() == TokKind::kw_true;
-  lexer_.Lex();
-  return true;
-}
-
-bool HloParser::ParseToken(TokKind kind, const string& msg) {
-  if (lexer_.GetKind() != kind) {
-    return TokenError(msg);
-  }
-  lexer_.Lex();
-  return true;
-}
-
-bool HloParser::EatIfPresent(TokKind kind) {
-  if (lexer_.GetKind() != kind) {
-    return false;
-  }
-  lexer_.Lex();
-  return true;
-}
-
-bool HloParser::AddInstruction(const string& name,
-                               HloInstruction* instruction) {
-  auto result = instruction_pool_.insert({name, instruction});
-  if (!result.second) {
-    return TokenError(StrCat("instruction already exists: ", name));
-  }
-  return true;
-}
-
-}  // namespace
-
-StatusOr<std::unique_ptr<HloModule>> Parse(StringPiece str) {
-  HloParser parser(str);
-  if (!parser.Run()) {
-    return InvalidArgument("Syntax error: %s", parser.GetError().c_str());
-  }
-  return parser.ConsumeHloModule();
-}
-
-}  // namespace tools
-}  // namespace xla
diff --git a/tensorflow/compiler/xla/tools/parser/hlo_parser.h b/tensorflow/compiler/xla/tools/parser/hlo_parser.h
deleted file mode 100644
index 9aaf18ef20..0000000000
--- a/tensorflow/compiler/xla/tools/parser/hlo_parser.h
+++ /dev/null
@@ -1,37 +0,0 @@
-/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#ifndef TENSORFLOW_COMPILER_XLA_TOOLS_PARSER_HLO_PARSER_H_
-#define TENSORFLOW_COMPILER_XLA_TOOLS_PARSER_HLO_PARSER_H_
-
-#include "tensorflow/compiler/xla/ptr_util.h"
-#include "tensorflow/compiler/xla/service/hlo_computation.h"
-#include "tensorflow/compiler/xla/service/hlo_instruction.h"
-#include "tensorflow/compiler/xla/service/hlo_module.h"
-#include "tensorflow/compiler/xla/statusor.h"
-#include "tensorflow/compiler/xla/tools/parser/hlo_lexer.h"
-#include "tensorflow/compiler/xla/xla_data.pb.h"
-
-namespace xla {
-namespace tools {
-
-// The api of the hlo parser. Given a string in the HloModule::ToString()
-// format, returns the parsed HloModule.
-StatusOr<std::unique_ptr<HloModule>> Parse(tensorflow::StringPiece str);
-
-}  // namespace tools
-}  // namespace xla
-
-#endif  // TENSORFLOW_COMPILER_XLA_TOOLS_PARSER_HLO_PARSER_H_
diff --git a/tensorflow/compiler/xla/tools/parser/hlo_parser_test.cc b/tensorflow/compiler/xla/tools/parser/hlo_parser_test.cc
deleted file mode 100644
index 4ecece3eac..0000000000
--- a/tensorflow/compiler/xla/tools/parser/hlo_parser_test.cc
+++ /dev/null
@@ -1,240 +0,0 @@
-/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#include "tensorflow/compiler/xla/tools/parser/hlo_parser.h"
-
-#include <string>
-#include "tensorflow/core/lib/core/status_test_util.h"
-#include "tensorflow/core/platform/test.h"
-
-namespace xla {
-namespace tools {
-namespace {
-
-struct TestData {
-  string test_name;
-  string module_string;
-};
-
-string TestDataToString(const ::testing::TestParamInfo<TestData>& data) {
-  return data.param.test_name;
-}
-
-std::vector<TestData> CreateTestCases() {
-  // clang-format off
-  return std::vector<TestData>({
-// ax + y
-{
-"AxpyParam",
-R"(HloModule axpy_module:
-
-ENTRY %axpy.v5 (alpha: f32[2,4], x: f32[2,4], y: f32[2,4]) -> f32[2,4] {
-  %alpha = f32[2,4]{1,0} parameter(0)
-  %x = f32[2,4]{1,0} parameter(1)
-  %multiply = f32[2,4]{1,0} multiply(f32[2,4]{1,0} %alpha, f32[2,4]{1,0} %x)
-  %y = f32[2,4]{1,0} parameter(2)
-  %add = f32[2,4]{1,0} add(f32[2,4]{1,0} %multiply, f32[2,4]{1,0} %y)
-}
-
-)"
-},
-// pred constant
-{
-"ConstantPred",
-R"(HloModule constant_pred_module:
-
-ENTRY %constant_pred () -> pred[] {
-  %constant = pred[] constant(true)
-}
-
-)"
-},
-// s32 constant
-{
-"ConstantS32",
-R"(HloModule constant_s32_module:
-
-ENTRY %constant_s32 () -> s32[] {
-  %constant = s32[] constant(-42)
-}
-
-)"
-},
-// f32 constant, but the value is not a decimal
-{
-"ConstantF32", R"(HloModule ConstantF32_module:
-
-ENTRY %ConstantF32.v4 () -> f32[] {
-  %constant = f32[] constant(42)
-}
-
-)"
-},
-// constant + constant
-{
-"AddConstants",
-R"(HloModule add_constants_module:
-
-ENTRY %add_constants () -> f32[] {
-  %constant = f32[] constant(3.14)
-  %add = f32[] add(f32[] %constant, f32[] %constant)
-}
-
-)"
-},
-// v1 > v2 ? v1 : v2
-{
-"SelectR1F32",
-R"(HloModule SelectR1F32WithCmpR1F32sFromParamsSmall_module:
-
-ENTRY %SelectR1F32WithCmpR1F32sFromParamsSmall.v4 (v1: f32[4], v2: f32[4]) -> f32[4] {
-  %v1 = f32[4]{0} parameter(0)
-  %v2 = f32[4]{0} parameter(1)
-  %greater-than = pred[4]{0} greater-than(f32[4]{0} %v1, f32[4]{0} %v2)
-  %select = f32[4]{0} select(pred[4]{0} %greater-than, f32[4]{0} %v1, f32[4]{0} %v2)
-}
-
-)"
-}
-  });
-  // clang-format on
-}
-
-class HloParserTest : public ::testing::Test,
-                      public ::testing::WithParamInterface<TestData> {
- protected:
-  void ExpectSuccess() {
-    const string& original = GetParam().module_string;
-    auto result = Parse(original);
-    TF_EXPECT_OK(result.status());
-    EXPECT_EQ(original, result.ValueOrDie()->ToString());
-  }
-};
-
-TEST_P(HloParserTest, Run) { ExpectSuccess(); }
-
-INSTANTIATE_TEST_CASE_P(HloParserTestSuccessInstantiation, HloParserTest,
-                        ::testing::ValuesIn(CreateTestCases()),
-                        TestDataToString);
-
-TEST_F(HloParserTest, Empty) {
-  const string original = "";
-  auto result = Parse(original);
-  EXPECT_NE(tensorflow::Status::OK(), result.status());
-}
-
-TEST_F(HloParserTest, Garbage) {
-  const string original = "HloModule thi$ str1ng makes# N0 sen$e @all!*&^%$";
-  auto result = Parse(original);
-  EXPECT_NE(tensorflow::Status::OK(), result.status());
-}
-
-TEST_F(HloParserTest, WrongOpcode) {
-  const string original = R"(HloModule wrong_opcode:
-
-ENTRY %blabla (x: f32[], y: f32[]) -> f32[] {
-  %x = f32[]{} parameter(0)
-  %y = f32[]{} parameter(1)
-  %le = pred[]{} le(f32[]{} %x, f32[]{} %y)
-}
-
-)";
-  auto result = Parse(original);
-  EXPECT_NE(tensorflow::Status::OK(), result.status());
-}
-
-TEST_F(HloParserTest, WrongShape) {
-  const string original = R"(HloModule wrong_opcode:
-
-ENTRY %blabla (x: g32[]) -> g32[] {
-  %x = g32[]{} parameter(0)
-}
-
-)";
-  auto result = Parse(original);
-  EXPECT_NE(tensorflow::Status::OK(), result.status());
-}
-
-TEST_F(HloParserTest, WrongOperandsSize) {
-  const string original = R"(HloModule wrong_opcode:
-
-ENTRY %blabla (x: f32[]) -> pred[] {
-  %x = f32[]{} parameter(0)
-  %eq = pred[]{} equal-to(f32[]{} %x)
-}
-
-)";
-  auto result = Parse(original);
-  EXPECT_NE(tensorflow::Status::OK(), result.status());
-}
-
-TEST_F(HloParserTest, OperandNotFound) {
-  const string original = R"(HloModule operand_not_found:
-ENTRY %blabla (x: f32[]) -> pred[] {
-  %x = f32[]{} parameter(0)
-  %eq = pred[]{} equal-to(f32[]{} %x, f32[]{} %y)
-}
-)";
-  auto result = Parse(original);
-  EXPECT_NE(tensorflow::Status::OK(), result.status());
-}
-
-TEST_F(HloParserTest, MoreConstants) {
-  const string original = R"(HloModule SelectScalarS32True_module:
-
-ENTRY %SelectScalarS32True.v4 () -> s32[] {
-  %constant.2 = pred[] constant(true)
-  %constant.1 = s32[] constant(-42)
-  %constant = s32[] constant(42)
-  %select = s32[] select(pred[] %constant.2, s32[] %constant.1, s32[] %constant)
-}
-
-)";
-  auto result = Parse(original);
-  TF_EXPECT_OK(result.status());
-  // Constant instructions have no name. The string will be parsed successfully
-  // but the constant names will not be exactly the same.
-}
-
-TEST_F(HloParserTest, ConstantWithExp) {
-  const string original = R"(HloModule ConstantWithExp_module:
-
-ENTRY %ConstantWithExp.v4 () -> f32[] {
-  %constant.1 = f32[] constant(3e+2)
-}
-
-)";
-  auto result = Parse(original);
-  TF_EXPECT_OK(result.status());
-  // The string will be parsed successfully but the output strings are not
-  // exactly the same, because "3e2" is parsed into value 300 and will be
-  // printed as "300".
-}
-
-TEST_F(HloParserTest, Tuple) {
-  const string original = R"(HloModule EmptyTupleCreate_module:
-
-ENTRY %EmptyTupleCreate.v1 () -> () {
-  %tuple = () tuple()
-}
-
-)";
-  auto result = Parse(original);
-  EXPECT_NE(tensorflow::Status::OK(), result.status());
-}
-
-}  // namespace
-}  // namespace tools
-}  // namespace xla
diff --git a/tensorflow/compiler/xla/tools/parser/hlo_token.h b/tensorflow/compiler/xla/tools/parser/hlo_token.h
deleted file mode 100644
index 1f75e17c7f..0000000000
--- a/tensorflow/compiler/xla/tools/parser/hlo_token.h
+++ /dev/null
@@ -1,58 +0,0 @@
-/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#ifndef TENSORFLOW_COMPILER_XLA_TOOLS_PARSER_HLO_TOKEN_H_
-#define TENSORFLOW_COMPILER_XLA_TOOLS_PARSER_HLO_TOKEN_H_
-
-namespace xla {
-namespace tools {
-
-// Defines different kinds of tokens in a hlo module string.
-enum class TokKind {
-  // Markers
-  kEof,
-  kError,
-
-  // Tokens with no info.
-  kEqual,  // =
-  kComma,  // ,
-  kColon,  // :
-  kLsquare,
-  kRsquare,  // [  ]
-  kLbrace,
-  kRbrace,  // {  }
-  kLparen,
-  kRparen,  // (  )
-
-  kArrow,  // ->
-
-  // Keywords
-  kw_HloModule,
-  kw_ENTRY,
-  kw_true,
-  kw_false,
-
-  // Typed tokens.
-  kName,     // %foo
-  kShape,    // f32[2,3]{1,0}
-  kOpcode,   // add
-  kInt,      // 42
-  kDecimal,  // 4.2
-};
-
-}  // namespace tools
-}  // namespace xla
-
-#endif  // TENSORFLOW_COMPILER_XLA_TOOLS_PARSER_HLO_TOKEN_H_
-- 
GitLab


From 6c074971ab80362954bea07ff2896cb91636b787 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 19 Oct 2017 15:02:01 -0700
Subject: [PATCH 580/909] Add a recursive descent parser for the HloModule
 string. It constructs an HloModule object from a string printed by
 HloModule::ToString().

This is a initial stage. It currently supports:
- unary, binary, ternary ops, and other ops that don't have extra attributes.
- module with entry computation only.
- simple cases for constant instruction.

To make the parser simpler, this cl removes a whitespace and adds a '%' before the computation name in HloComputation::ToString().

Further steps will enable parsing subcomputations, more cases of constants, tuple, and ops that require extra attributes (e.g., broadcast dimensions, subcomputation).

PiperOrigin-RevId: 172804214
---
 tensorflow/BUILD                              |   1 +
 .../compiler/xla/service/hlo_computation.cc   |   4 +-
 tensorflow/compiler/xla/shape_util.cc         |  45 +-
 tensorflow/compiler/xla/tools/parser/BUILD    |  84 +++
 .../compiler/xla/tools/parser/README.md       |  69 +++
 .../compiler/xla/tools/parser/hlo_lexer.cc    | 270 ++++++++++
 .../compiler/xla/tools/parser/hlo_lexer.h     | 108 ++++
 .../compiler/xla/tools/parser/hlo_parser.cc   | 502 ++++++++++++++++++
 .../compiler/xla/tools/parser/hlo_parser.h    |  37 ++
 .../xla/tools/parser/hlo_parser_test.cc       | 240 +++++++++
 .../compiler/xla/tools/parser/hlo_token.h     |  58 ++
 11 files changed, 1402 insertions(+), 16 deletions(-)
 create mode 100644 tensorflow/compiler/xla/tools/parser/BUILD
 create mode 100644 tensorflow/compiler/xla/tools/parser/README.md
 create mode 100644 tensorflow/compiler/xla/tools/parser/hlo_lexer.cc
 create mode 100644 tensorflow/compiler/xla/tools/parser/hlo_lexer.h
 create mode 100644 tensorflow/compiler/xla/tools/parser/hlo_parser.cc
 create mode 100644 tensorflow/compiler/xla/tools/parser/hlo_parser.h
 create mode 100644 tensorflow/compiler/xla/tools/parser/hlo_parser_test.cc
 create mode 100644 tensorflow/compiler/xla/tools/parser/hlo_token.h

diff --git a/tensorflow/BUILD b/tensorflow/BUILD
index e351037abb..d5c56cdc18 100644
--- a/tensorflow/BUILD
+++ b/tensorflow/BUILD
@@ -340,6 +340,7 @@ filegroup(
         "//tensorflow/compiler/xla/service/llvm_ir:all_files",
         "//tensorflow/compiler/xla/tests:all_files",
         "//tensorflow/compiler/xla/tools:all_files",
+        "//tensorflow/compiler/xla/tools/parser:all_files",
         "//tensorflow/contrib:all_files",
         "//tensorflow/contrib/all_reduce:all_files",
         "//tensorflow/contrib/android:all_files",
diff --git a/tensorflow/compiler/xla/service/hlo_computation.cc b/tensorflow/compiler/xla/service/hlo_computation.cc
index 9b3104eaac..51ead753f0 100644
--- a/tensorflow/compiler/xla/service/hlo_computation.cc
+++ b/tensorflow/compiler/xla/service/hlo_computation.cc
@@ -373,8 +373,8 @@ string HloComputation::ToString(int nested_level) const {
   for (int i = 0; i < nested_level; i++) {
     s << "    ";
   }
-  s << name() << " " << ShapeUtil::HumanString(ComputeProgramShape())
-    << " { \n";
+  s << "%" << name() << " " << ShapeUtil::HumanString(ComputeProgramShape())
+    << " {\n";
   for (const HloInstruction* instruction : MakeInstructionPostOrder()) {
     for (int i = 0; i < nested_level; i++) {
       s << "    ";
diff --git a/tensorflow/compiler/xla/shape_util.cc b/tensorflow/compiler/xla/shape_util.cc
index 8e16056b23..af583bed62 100644
--- a/tensorflow/compiler/xla/shape_util.cc
+++ b/tensorflow/compiler/xla/shape_util.cc
@@ -102,6 +102,32 @@ bool CompareShapes(const Shape& lhs, const Shape& rhs, bool compare_layouts) {
   return true;
 }
 
+// Constructs and returns the new shape with the given minor_to_major order in
+// its Layout.
+StatusOr<Shape> MakeShapeWithLayoutInternal(
+    PrimitiveType element_type, tensorflow::gtl::ArraySlice<int64> dimensions,
+    tensorflow::gtl::ArraySlice<int64> minor_to_major) {
+  if (dimensions.size() != minor_to_major.size()) {
+    return InvalidArgument("Dimensions size is %ld, but layout size is %ld.",
+                           dimensions.size(), minor_to_major.size());
+  }
+  if (element_type == OPAQUE || element_type == TUPLE) {
+    return InvalidArgument("Unsupported element type: %s",
+                           PrimitiveType_Name(element_type).c_str());
+  }
+  Shape shape = ShapeUtil::MakeShape(element_type, dimensions);
+  auto min2maj = shape.mutable_layout()->mutable_minor_to_major();
+  min2maj->Clear();
+  for (int64 value : minor_to_major) {
+    min2maj->Add(value);
+  }
+  if (!shape.has_layout()) {
+    return InvalidArgument("Shape has no layout.");
+  }
+  TF_RETURN_IF_ERROR(ShapeUtil::ValidateShape(shape));
+  return shape;
+}
+
 }  // namespace
 
 /* static */ bool ShapeUtil::Equal(const Shape& lhs, const Shape& rhs) {
@@ -152,16 +178,8 @@ bool CompareShapes(const Shape& lhs, const Shape& rhs, bool compare_layouts) {
 /* static */ Shape ShapeUtil::MakeShapeWithLayout(
     PrimitiveType element_type, tensorflow::gtl::ArraySlice<int64> dimensions,
     tensorflow::gtl::ArraySlice<int64> minor_to_major) {
-  CHECK_EQ(dimensions.size(), minor_to_major.size());
-  Shape shape = MakeShape(element_type, dimensions);
-  auto min2maj = shape.mutable_layout()->mutable_minor_to_major();
-  min2maj->Clear();
-  for (int64 value : minor_to_major) {
-    min2maj->Add(value);
-  }
-  DCHECK(shape.has_layout());
-  TF_DCHECK_OK(ValidateShape(shape));
-  return shape;
+  return MakeShapeWithLayoutInternal(element_type, dimensions, minor_to_major)
+      .ValueOrDie();
 }
 
 /* static */ Shape ShapeUtil::MakeShapeWithMonotonicDim0MajorLayout(
@@ -499,11 +517,10 @@ StatusOr<Shape> ParseShapeStringInternal(tensorflow::StringPiece* s) {
       // Extract the layout minor-to-major and set it.
       TF_ASSIGN_OR_RETURN(std::vector<int64> min2maj,
                           comma_list_to_int64s(layout_string));
-      TF_RET_CHECK(dimensions.size() == min2maj.size());
-      result =
-          ShapeUtil::MakeShapeWithLayout(primitive_type, dimensions, min2maj);
+      TF_ASSIGN_OR_RETURN(result, MakeShapeWithLayoutInternal(
+                                      primitive_type, dimensions, min2maj));
     }
-    TF_DCHECK_OK(ShapeUtil::ValidateShape(result));
+    TF_RETURN_IF_ERROR(ShapeUtil::ValidateShape(result));
     return std::move(result);
   }
 
diff --git a/tensorflow/compiler/xla/tools/parser/BUILD b/tensorflow/compiler/xla/tools/parser/BUILD
new file mode 100644
index 0000000000..c84ca9fc83
--- /dev/null
+++ b/tensorflow/compiler/xla/tools/parser/BUILD
@@ -0,0 +1,84 @@
+# Build file for the Hlo parser.
+
+licenses(["notice"])  # Apache 2.0
+
+package(
+    default_visibility = [":friends"],
+)
+
+package_group(
+    name = "friends",
+    includes = [
+        "//tensorflow/compiler/xla:friends",
+    ],
+)
+
+# Filegroup used to collect source files for dependency checking.
+filegroup(
+    name = "c_srcs",
+    data = glob([
+        "**/*.cc",
+        "**/*.h",
+    ]),
+)
+
+load("//tensorflow:tensorflow.bzl", "tf_cc_test")
+
+cc_library(
+    name = "hlo_lexer",
+    srcs = ["hlo_lexer.cc"],
+    hdrs = [
+        "hlo_lexer.h",
+        "hlo_token.h",
+    ],
+    deps = [
+        "//tensorflow/compiler/xla:shape_util",
+        "//tensorflow/compiler/xla:statusor",
+        "//tensorflow/compiler/xla:util",
+        "//tensorflow/compiler/xla:xla_data_proto",
+        "//tensorflow/compiler/xla/service:hlo",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:regexp_internal",
+    ],
+)
+
+cc_library(
+    name = "hlo_parser",
+    srcs = ["hlo_parser.cc"],
+    hdrs = ["hlo_parser.h"],
+    deps = [
+        ":hlo_lexer",
+        "//tensorflow/compiler/xla:shape_util",
+        "//tensorflow/compiler/xla:statusor",
+        "//tensorflow/compiler/xla:util",
+        "//tensorflow/compiler/xla:xla_data_proto",
+        "//tensorflow/compiler/xla/service:hlo",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:lib_internal",
+    ],
+)
+
+tf_cc_test(
+    name = "hlo_parser_test",
+    size = "small",
+    srcs = ["hlo_parser_test.cc"],
+    deps = [
+        ":hlo_parser",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+    ],
+)
+
+# -----------------------------------------------------------------------------
+
+filegroup(
+    name = "all_files",
+    srcs = glob(
+        ["**/*"],
+        exclude = [
+            "**/METADATA",
+            "**/OWNERS",
+        ],
+    ),
+    visibility = ["//tensorflow:__subpackages__"],
+)
diff --git a/tensorflow/compiler/xla/tools/parser/README.md b/tensorflow/compiler/xla/tools/parser/README.md
new file mode 100644
index 0000000000..a334bc2b29
--- /dev/null
+++ b/tensorflow/compiler/xla/tools/parser/README.md
@@ -0,0 +1,69 @@
+# HloModule string syntax
+
+TODO: Support subcomputations (for fusion, reduce, while, ...).
+
+TODO: Support ops that require extra attributes, e.g. dimensions, strides.
+
+```yacc
+hlo_module
+  : 'HloModule' name computation
+  ;
+
+computation
+  : 'ENTRY' name param_list '->' shape instruction_list
+  ;
+
+instruction_list
+  : '{' instruction_list1 '}'
+  ;
+instruction_list1
+  : instruction
+  | instruction_list1 instruction
+  ;
+instruction
+  : name '=' shape opcode operands
+  ;
+
+operands
+  : '(' operands1 ')'
+  ;
+operands1
+  : /*empty*/
+  | operand
+  | operands1 ',' operand
+  ;
+operand
+  : shape name
+  ;
+
+param_list
+  : '(' param_list1 ')'
+  ;
+param_list1
+  : /*empty*/
+  | param
+  | param_list1 ',' param
+  ;
+param
+  : name shape
+  ;
+
+shape
+  : shape_val_
+  | '(' tuple_elements ')'
+  ;
+tuple_elements
+  : /*empty*/
+  | shape (',' shape)*
+  ;
+
+name
+  : identifier ':'
+  | '%' identifier
+  ;
+
+identifier
+  : [a-zA-Z_][a-zA-Z0-9_.-]*
+  ;
+
+```
diff --git a/tensorflow/compiler/xla/tools/parser/hlo_lexer.cc b/tensorflow/compiler/xla/tools/parser/hlo_lexer.cc
new file mode 100644
index 0000000000..3e84ffcbd2
--- /dev/null
+++ b/tensorflow/compiler/xla/tools/parser/hlo_lexer.cc
@@ -0,0 +1,270 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/tools/parser/hlo_lexer.h"
+
+#include <unordered_map>
+
+#include "tensorflow/compiler/xla/shape_util.h"
+#include "tensorflow/compiler/xla/statusor.h"
+#include "tensorflow/compiler/xla/util.h"
+#include "tensorflow/core/lib/gtl/optional.h"
+#include "tensorflow/core/lib/strings/numbers.h"
+#include "tensorflow/core/platform/regexp.h"
+
+namespace xla {
+namespace tools {
+
+using tensorflow::StringPiece;
+
+namespace {
+
+constexpr int kEOF = -1;
+constexpr int kError = -2;
+
+// [a-zA-Z0-9_.-]
+bool IsIdentifierChar(char c) {
+  return isalnum(static_cast<unsigned char>(c)) || c == '-' || c == '.' ||
+         c == '_';
+}
+
+}  // namespace
+
+int HloLexer::GetNextChar() {
+  int current_char = PeekCurrentChar();
+  if (current_char != kEOF && current_char != kError) {
+    current_ptr_++;
+  }
+  return current_char;
+}
+
+int HloLexer::PeekCurrentChar() const {
+  if (current_ptr_ == buf_.end()) {
+    return kEOF;
+  }
+  char current_char = *current_ptr_;
+  if (current_char == 0) {
+    // '\0' should not appear in the middle of the string.
+    return kError;
+  }
+  return static_cast<unsigned char>(current_char);
+}
+
+bool HloLexer::CanDereference(const char* ptr) const {
+  return ptr < buf_.end() && ptr >= buf_.begin();
+}
+
+StringPiece HloLexer::StringPieceFromPointers(const char* begin,
+                                              const char* end) const {
+  CHECK(begin <= end);
+  CHECK(begin == buf_.end() || CanDereference(begin));
+  CHECK(end == buf_.end() || CanDereference(end));
+  return StringPiece(begin, end - begin);
+}
+
+tensorflow::RegexpStringPiece HloLexer::RegexpStringPieceFromPointers(
+    const char* begin, const char* end) const {
+  CHECK(begin <= end);
+  CHECK(begin == buf_.end() || CanDereference(begin));
+  CHECK(end == buf_.end() || CanDereference(end));
+  return tensorflow::RegexpStringPiece(begin, end - begin);
+}
+
+TokKind HloLexer::LexToken() {
+  while (true) {
+    token_start_ = current_ptr_;
+
+    int current_char = GetNextChar();
+    switch (current_char) {
+      default:
+        // [a-zA-Z_]
+        if (isalpha(static_cast<unsigned char>(current_char)) ||
+            current_char == '_') {
+          return LexIdentifier();
+        }
+        return TokKind::kError;
+      case kEOF:
+        // Hit the end of the input buffer.
+        return TokKind::kEof;
+      case kError:
+        // Hit an invalid character in the input buffer.
+        return TokKind::kError;
+      case ' ':
+      case '\t':
+      case '\n':
+      case '\r':
+        // Ignore whitespace.
+        continue;
+      case '0':
+      case '1':
+      case '2':
+      case '3':
+      case '4':
+      case '5':
+      case '6':
+      case '7':
+      case '8':
+      case '9':
+      case '-':
+        if (current_char == '-' && PeekCurrentChar() == '>') {
+          current_ptr_++;
+          return TokKind::kArrow;
+        }
+        return LexDigitOrNegative();
+      case '=':
+        return TokKind::kEqual;
+      case ',':
+        return TokKind::kComma;
+      case '%':
+        return LexPercent();
+      case ':':
+        return TokKind::kColon;
+      case '[':
+        return TokKind::kLsquare;
+      case ']':
+        return TokKind::kRsquare;
+      case '{':
+        return TokKind::kLbrace;
+      case '}':
+        return TokKind::kRbrace;
+      case '(':
+        return TokKind::kLparen;
+      case ')':
+        return TokKind::kRparen;
+    }
+  }
+}
+
+// Lex a shape, name, keyword, or opcode.
+// shape    ::= ([a-zA-Z0-9_]*[0-9]*)\[([0-9,]*)\](?:\s*{([0-9,]*)})?
+// name     ::= [a-zA-Z_][a-zA-Z0-9_.-]*:
+// keyword  ::= HloModule, ENTRY, ...
+// opcode   ::= add, greater-than, ...
+TokKind HloLexer::LexIdentifier() {
+  {
+    auto consumable = RegexpStringPieceFromPointers(token_start_, buf_.end());
+    // 'consumable' will be advanced iff its prefix matches the pattern.
+    static LazyRE2 shape_pattern = {
+        R"(^(\w*\d*)\[([\d,]*)\](?:\s*{([\d,]*)})?)"};
+    if (RE2::Consume(&consumable, *shape_pattern)) {
+      auto status_or_shape = ShapeUtil::ParseShapeString(
+          StringPieceFromPointers(token_start_, consumable.begin()));
+      if (status_or_shape.ok()) {
+        // This is a shape string.
+        shape_val_ = status_or_shape.ValueOrDie();
+        current_ptr_ = consumable.begin();
+        return TokKind::kShape;
+      }
+    }
+  }
+
+  while (IsIdentifierChar(PeekCurrentChar())) {
+    current_ptr_++;
+  }
+
+  // If followed by ':', it's a name.
+  if (PeekCurrentChar() == ':') {
+    str_val_.assign(token_start_, current_ptr_);
+    current_ptr_++;  // skip ':'
+    return TokKind::kName;
+  }
+
+  StringPiece identifier = StringPieceFromPointers(token_start_, current_ptr_);
+
+  // See if this is a keyword.
+#define KEYWORD(STR)            \
+  do {                          \
+    if (identifier == #STR) {   \
+      return TokKind::kw_##STR; \
+    }                           \
+  } while (false)
+
+  KEYWORD(true);
+  KEYWORD(false);
+  KEYWORD(HloModule);
+  KEYWORD(ENTRY);
+
+#undef KEYWORD
+
+  // See if this is an opcode.
+  auto opcode = StringToHloOpcode(identifier.ToString());
+  if (opcode.ok()) {
+    opcode_val_ = opcode.ValueOrDie();
+    return TokKind::kOpcode;
+  }
+
+  current_ptr_ = token_start_ + 1;
+  return TokKind::kError;
+}
+
+// Lex names after a % character.
+// name ::= [a-zA-Z_][a-zA-Z0-9_.-]*
+TokKind HloLexer::LexPercent() {
+  const char* name_start = current_ptr_;
+  if (isalpha(static_cast<unsigned char>(PeekCurrentChar())) ||
+      PeekCurrentChar() == '_') {
+    current_ptr_++;
+    while (IsIdentifierChar(PeekCurrentChar())) {
+      current_ptr_++;
+    }
+    str_val_.assign(name_start, current_ptr_);
+    return TokKind::kName;
+  }
+  return TokKind::kError;
+}
+
+// Lex integer and floating-point values.
+// int             [-]?[0-9]+
+// fp with exp     [-]?([0-9]+|[0-9]+[.][0-9]*|[0-9]*[.][0-9]+)([eE][+-]?[0-9]+)
+// fp without exp  [-]?([0-9]+[.][0-9]*|[0-9]*[.][0-9]+)
+TokKind HloLexer::LexDigitOrNegative() {
+  auto consumable = RegexpStringPieceFromPointers(token_start_, buf_.end());
+  static LazyRE2 float_pattern = {
+      R"([-]?((\d+|\d+[.]\d*|\d*[.]\d+)([eE][+-]?\d+))|(\d+[.]\d*|\d*[.]\d+))"};
+  if (RE2::Consume(&consumable, *float_pattern)) {
+    current_ptr_ = consumable.begin();
+    tensorflow::strings::safe_strtod(string(token_start_, current_ptr_).c_str(),
+                                     &decimal_val_);
+    return TokKind::kDecimal;
+  }
+
+  static LazyRE2 int_pattern = {R"([-]?\d+)"};
+  if (RE2::Consume(&consumable, *int_pattern)) {
+    current_ptr_ = consumable.begin();
+    tensorflow::strings::safe_strto64(
+        StringPieceFromPointers(token_start_, current_ptr_), &int64_val_);
+    return TokKind::kInt;
+  }
+
+  return TokKind::kError;
+}
+
+StringPiece HloLexer::GetCurrentLine() const {
+  const char* start = token_start_;
+  const char* end = current_ptr_;
+  if (!CanDereference(start) || !CanDereference(end)) {
+    return "LINE OUT OF RANGE";
+  }
+  while (start > buf_.begin() && *start != '\n') {
+    start--;
+  }
+  while (end < buf_.end() && *end != '\n') {
+    end++;
+  }
+  return StringPieceFromPointers(start, end);
+}
+
+}  // namespace tools
+}  // namespace xla
diff --git a/tensorflow/compiler/xla/tools/parser/hlo_lexer.h b/tensorflow/compiler/xla/tools/parser/hlo_lexer.h
new file mode 100644
index 0000000000..20278fd6cd
--- /dev/null
+++ b/tensorflow/compiler/xla/tools/parser/hlo_lexer.h
@@ -0,0 +1,108 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_COMPILER_XLA_TOOLS_PARSER_HLO_LEXER_H_
+#define TENSORFLOW_COMPILER_XLA_TOOLS_PARSER_HLO_LEXER_H_
+
+#include <string>
+
+#include "tensorflow/compiler/xla/service/hlo_opcode.h"
+#include "tensorflow/compiler/xla/tools/parser/hlo_token.h"
+#include "tensorflow/compiler/xla/xla_data.pb.h"
+#include "tensorflow/core/lib/core/stringpiece.h"
+#include "tensorflow/core/platform/logging.h"
+#include "tensorflow/core/platform/regexp.h"
+#include "tensorflow/core/platform/types.h"
+
+namespace xla {
+namespace tools {
+
+// Lexer for the HloModule::ToString() format text.
+class HloLexer {
+ public:
+  explicit HloLexer(tensorflow::StringPiece buf) : buf_(buf) {
+    current_ptr_ = buf_.begin();
+  }
+
+  TokKind Lex() { return current_kind_ = LexToken(); }
+  TokKind GetKind() const { return current_kind_; }
+  string GetStrVal() const {
+    CHECK(GetKind() == TokKind::kName);
+    return str_val_;
+  }
+  Shape GetShapeVal() const {
+    CHECK(GetKind() == TokKind::kShape);
+    return shape_val_;
+  }
+  HloOpcode GetOpcodeVal() const {
+    CHECK(GetKind() == TokKind::kOpcode);
+    return opcode_val_;
+  }
+  int64 GetInt64Val() const {
+    CHECK(GetKind() == TokKind::kInt);
+    return int64_val_;
+  }
+  double GetDecimalVal() const {
+    CHECK(GetKind() == TokKind::kDecimal);
+    return decimal_val_;
+  }
+
+  // Returns the line of text that is currently being lexed.
+  tensorflow::StringPiece GetCurrentLine() const;
+
+ private:
+  // Returns the current character. If it's neither the end of input buffer nor
+  // an invalid character, moves the pointer forward.
+  int GetNextChar();
+
+  // Returns the current character.
+  int PeekCurrentChar() const;
+
+  // Creates StringPiece with the given begin and end. Exits if the begin > end,
+  // or it's out of the range of the current buffer.
+  tensorflow::StringPiece StringPieceFromPointers(const char* begin,
+                                                  const char* end) const;
+  tensorflow::RegexpStringPiece RegexpStringPieceFromPointers(
+      const char* begin, const char* end) const;
+
+  // Returns true if the given ptr is dereferenceable within the range of the
+  // current buffer.
+  bool CanDereference(const char* ptr) const;
+
+  TokKind LexToken();
+
+  TokKind LexIdentifier();
+  TokKind LexPercent();
+  TokKind LexShape();
+  TokKind LexConstant();
+  TokKind LexDigitOrNegative();
+
+  const tensorflow::StringPiece buf_;
+  const char* current_ptr_;
+
+  // Information about the current token.
+  const char* token_start_;
+  TokKind current_kind_;
+  string str_val_;
+  Shape shape_val_;
+  HloOpcode opcode_val_;
+  int64 int64_val_;
+  double decimal_val_;
+};
+
+}  // namespace tools
+}  // namespace xla
+
+#endif  // TENSORFLOW_COMPILER_XLA_TOOLS_PARSER_HLO_LEXER_H_
diff --git a/tensorflow/compiler/xla/tools/parser/hlo_parser.cc b/tensorflow/compiler/xla/tools/parser/hlo_parser.cc
new file mode 100644
index 0000000000..57700493e6
--- /dev/null
+++ b/tensorflow/compiler/xla/tools/parser/hlo_parser.cc
@@ -0,0 +1,502 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/tools/parser/hlo_parser.h"
+
+#include "tensorflow/compiler/xla/shape_util.h"
+#include "tensorflow/core/lib/gtl/map_util.h"
+#include "tensorflow/core/lib/strings/strcat.h"
+
+namespace xla {
+namespace tools {
+
+namespace {
+
+using tensorflow::StringPiece;
+using tensorflow::strings::StrCat;
+
+// Parser for the HloModule::ToString() format text.
+class HloParser {
+ public:
+  explicit HloParser(StringPiece str) : lexer_(str) {}
+
+  // Runs the parser. Returns false if an error occurred.
+  bool Run();
+
+  // Returns the parsed HloModule.
+  std::unique_ptr<HloModule> ConsumeHloModule() { return std::move(module_); }
+
+  // Returns the error information.
+  string GetError() const { return tensorflow::str_util::Join(error_, "\n"); }
+
+ private:
+  // ParseXXX returns false if an error occurred.
+  bool ParseHloModule();
+  bool ParseComputation();
+  bool ParseInstructionList(HloComputation::Builder* builder);
+  bool ParseInstruction(HloComputation::Builder* builder);
+  bool ParseLiteral(std::unique_ptr<Literal>* literal, const Shape& shape);
+  bool ParseOperands(std::vector<HloInstruction*>* operands,
+                     const int expected_size);
+  bool ParseParamList();
+  bool ParseName(string* result);
+  bool ParseShape(Shape* result);
+  bool ParseOpcode(HloOpcode* result);
+  bool ParseInt64(int64* result);
+  bool ParseDecimal(double* result);
+  bool ParseBool(bool* result);
+  bool ParseToken(TokKind kind, const string& msg);
+
+  // Logs the current parsing line and the given message. Always returns false.
+  bool TokenError(StringPiece msg);
+
+  // If the current token is 'kind', eats it (i.e. lexes the next token) and
+  // returns true.
+  bool EatIfPresent(TokKind kind);
+
+  // Adds the instruction to the pool. Returns false and emits an error if the
+  // instruction already exists.
+  bool AddInstruction(const string& name, HloInstruction* instruction);
+
+  // The map from the instruction name to the instruction. This does not own the
+  // instructions.
+  std::unordered_map<string, HloInstruction*> instruction_pool_;
+
+  HloLexer lexer_;
+  std::unique_ptr<HloModule> module_;
+  std::vector<string> error_;
+};
+
+bool HloParser::TokenError(StringPiece msg) {
+  error_.push_back(
+      StrCat("was parsing \"", lexer_.GetCurrentLine(), "\"; ", msg));
+  return false;
+}
+
+bool HloParser::Run() {
+  lexer_.Lex();
+  return ParseHloModule();
+}
+
+// ::= 'HloModule' name computation
+bool HloParser::ParseHloModule() {
+  if (lexer_.GetKind() != TokKind::kw_HloModule) {
+    return TokenError("expects HloModule");
+  }
+  // Eat 'HloModule'
+  lexer_.Lex();
+
+  string name;
+  if (!ParseName(&name)) {
+    return false;
+  }
+
+  module_ = MakeUnique<HloModule>(name);
+
+  return ParseComputation();
+}
+
+// computation ::= 'ENTRY' name param_list '->' shape instruction_list
+bool HloParser::ParseComputation() {
+  string name;
+  if (!ParseToken(TokKind::kw_ENTRY, "expects 'ENTRY'") || !ParseName(&name)) {
+    return false;
+  }
+  auto builder = MakeUnique<HloComputation::Builder>(name);
+
+  Shape shape;
+  if (!ParseParamList() || !ParseToken(TokKind::kArrow, "expects '->'") ||
+      !ParseShape(&shape) || !ParseInstructionList(builder.get())) {
+    return false;
+  }
+  module_->AddEntryComputation(builder->Build());
+  return true;
+}
+
+// instruction_list ::= '{' instruction_list1 '}'
+// instruction_list1 ::= (instruction)+
+bool HloParser::ParseInstructionList(HloComputation::Builder* builder) {
+  if (!ParseToken(TokKind::kLbrace,
+                  "expects '{' at the beginning of instruction list.")) {
+    return false;
+  }
+  do {
+    if (!ParseInstruction(builder)) {
+      return false;
+    }
+  } while (lexer_.GetKind() != TokKind::kRbrace);
+  return ParseToken(TokKind::kRbrace,
+                    "expects '}' at the end of instruction list.");
+}
+
+// instruction ::= name '=' shape opcode operands
+bool HloParser::ParseInstruction(HloComputation::Builder* builder) {
+  string name;
+  Shape shape;
+  HloOpcode opcode;
+  std::vector<HloInstruction*> operands;
+  if (!ParseName(&name) ||
+      !ParseToken(TokKind::kEqual, "expects '=' in instruction") ||
+      !ParseShape(&shape) || !ParseOpcode(&opcode)) {
+    return false;
+  }
+  switch (opcode) {
+    case HloOpcode::kParameter: {
+      int64 parameter_number;
+      return ParseToken(TokKind::kLparen,
+                        "expects '(' before parameter number") &&
+             ParseInt64(&parameter_number) &&
+             ParseToken(TokKind::kRparen,
+                        "expects ')' after parameter number") &&
+             AddInstruction(
+                 name, builder->AddInstruction(HloInstruction::CreateParameter(
+                           parameter_number, shape, name)));
+    }
+    case HloOpcode::kConstant: {
+      std::unique_ptr<Literal> literal;
+      return ParseToken(TokKind::kLparen,
+                        "expects '(' before parameter number") &&
+             ParseLiteral(&literal, shape) &&
+             ParseToken(TokKind::kRparen,
+                        "expects ')' after parameter number") &&
+             AddInstruction(
+                 name, builder->AddInstruction(
+                           HloInstruction::CreateConstant(std::move(literal))));
+    }
+    // Unary ops.
+    case HloOpcode::kAbs:
+    case HloOpcode::kRoundNearestAfz:
+    case HloOpcode::kBitcast:
+    case HloOpcode::kCeil:
+    case HloOpcode::kCopy:
+    case HloOpcode::kCos:
+    case HloOpcode::kExp:
+    case HloOpcode::kIsFinite:
+    case HloOpcode::kFloor:
+    case HloOpcode::kLog:
+    case HloOpcode::kNot:
+    case HloOpcode::kNegate:
+    case HloOpcode::kSign:
+    case HloOpcode::kSin:
+    case HloOpcode::kSort:
+    case HloOpcode::kTanh: {
+      return ParseOperands(&operands, /*expected_size=*/1) &&
+             AddInstruction(name,
+                            builder->AddInstruction(HloInstruction::CreateUnary(
+                                shape, opcode, operands[0])));
+    }
+    // Binary ops.
+    case HloOpcode::kAdd:
+    case HloOpcode::kDivide:
+    case HloOpcode::kMultiply:
+    case HloOpcode::kSubtract:
+    case HloOpcode::kEq:
+    case HloOpcode::kGe:
+    case HloOpcode::kGt:
+    case HloOpcode::kLe:
+    case HloOpcode::kLt:
+    case HloOpcode::kNe:
+    case HloOpcode::kDot:
+    case HloOpcode::kMaximum:
+    case HloOpcode::kMinimum:
+    case HloOpcode::kPower:
+    case HloOpcode::kRemainder:
+    case HloOpcode::kAnd:
+    case HloOpcode::kOr:
+    case HloOpcode::kShiftLeft:
+    case HloOpcode::kShiftRightArithmetic:
+    case HloOpcode::kShiftRightLogical: {
+      return ParseOperands(&operands, /*expected_size=*/2) &&
+             AddInstruction(
+                 name, builder->AddInstruction(HloInstruction::CreateBinary(
+                           shape, opcode, operands[0], operands[1])));
+    }
+    // Ternary ops.
+    case HloOpcode::kClamp:
+    case HloOpcode::kSelect: {
+      return ParseOperands(&operands, /*expected_size=*/3) &&
+             AddInstruction(
+                 name,
+                 builder->AddInstruction(HloInstruction::CreateTernary(
+                     shape, opcode, operands[0], operands[1], operands[2])));
+    }
+    // Other supported ops.
+    case HloOpcode::kConvert: {
+      return ParseOperands(&operands, /*expected_size=*/1) &&
+             AddInstruction(
+                 name, builder->AddInstruction(
+                           HloInstruction::CreateConvert(shape, operands[0])));
+    }
+    case HloOpcode::kCrossReplicaSum: {
+      return ParseOperands(&operands, /*expected_size=*/1) &&
+             AddInstruction(name, builder->AddInstruction(
+                                      HloInstruction::CreateCrossReplicaSum(
+                                          shape, operands[0])));
+    }
+    case HloOpcode::kReshape: {
+      return ParseOperands(&operands, /*expected_size=*/1) &&
+             AddInstruction(
+                 name, builder->AddInstruction(
+                           HloInstruction::CreateReshape(shape, operands[0])));
+    }
+    case HloOpcode::kBroadcast:
+    case HloOpcode::kCall:
+    case HloOpcode::kCustomCall:
+    case HloOpcode::kConcatenate:
+    case HloOpcode::kReducePrecision:
+    case HloOpcode::kConvolution:
+    case HloOpcode::kGetTupleElement:
+    case HloOpcode::kMap:
+    case HloOpcode::kPad:
+    case HloOpcode::kReduce:
+    case HloOpcode::kReduceWindow:
+    case HloOpcode::kSelectAndScatter:
+    case HloOpcode::kReverse:
+    case HloOpcode::kRng:
+    case HloOpcode::kSlice:
+    case HloOpcode::kDynamicSlice:
+    case HloOpcode::kDynamicUpdateSlice:
+    case HloOpcode::kTranspose:
+    case HloOpcode::kTuple:
+    case HloOpcode::kWhile:
+    case HloOpcode::kFusion:
+    case HloOpcode::kBatchNormTraining:
+    case HloOpcode::kBatchNormInference:
+    case HloOpcode::kInfeed:
+    case HloOpcode::kOutfeed:
+    case HloOpcode::kBatchNormGrad:
+    case HloOpcode::kRecv:
+    case HloOpcode::kSend:
+    case HloOpcode::kUpdate:
+    case HloOpcode::kIndex:
+    case HloOpcode::kTrace:
+      return TokenError(StrCat("parsing not yet implemented for op: ",
+                               HloOpcodeString(opcode)));
+  }
+}
+
+bool HloParser::ParseLiteral(std::unique_ptr<Literal>* literal,
+                             const Shape& shape) {
+  switch (shape.element_type()) {
+    case PRED:
+      bool b;
+      if (!ParseBool(&b)) {
+        return false;
+      }
+      *literal = Literal::CreateR0<bool>(b);
+      return true;
+    case S32:
+      int64 i;
+      if (!ParseInt64(&i)) {
+        return false;
+      }
+      *literal = Literal::CreateR0<int32>(i);
+      return true;
+    case F32:
+      double d;
+      if (!ParseDecimal(&d)) {
+        return false;
+      }
+      *literal = Literal::CreateR0<float>(d);
+      return true;
+    default:
+      return TokenError(StrCat("unsupported constant in shape: ",
+                               ShapeUtil::HumanString(shape)));
+  }
+}
+
+// operands ::= '(' operands1 ')'
+// operands1
+//   ::= /*empty*/
+//   ::= operand (, operand)*
+// operand ::= shape name
+bool HloParser::ParseOperands(std::vector<HloInstruction*>* operands,
+                              const int expected_size) {
+  if (!ParseToken(TokKind::kLparen,
+                  "expects '(' at the beginning of operands")) {
+    return false;
+  }
+  if (lexer_.GetKind() == TokKind::kRparen) {
+    // empty
+  } else {
+    do {
+      Shape shape;
+      string name;
+      if (!ParseShape(&shape) || !ParseName(&name)) {
+        return false;
+      }
+      HloInstruction* instruction =
+          tensorflow::gtl::FindPtrOrNull(instruction_pool_, name);
+      if (!instruction) {
+        return TokenError(StrCat("instruction does not exist: ", name));
+      }
+      operands->push_back(instruction);
+    } while (EatIfPresent(TokKind::kComma));
+  }
+  if (expected_size != operands->size()) {
+    return TokenError(StrCat("expects ", expected_size, " operands, but has ",
+                             operands->size(), " operands"));
+  }
+  return ParseToken(TokKind::kRparen, "expects ')' at the end of operands");
+}
+
+// param_list ::= '(' param_list1 ')'
+// param_list1
+//   ::= /*empty*/
+//   ::= param (',' param)*
+// param ::= name shape
+bool HloParser::ParseParamList() {
+  if (!ParseToken(TokKind::kLparen,
+                  "expects '(' at the beginning of param list")) {
+    return false;
+  }
+
+  if (lexer_.GetKind() == TokKind::kRparen) {
+    // empty
+  } else {
+    do {
+      Shape shape;
+      if (!ParseToken(TokKind::kName, "expects name in parameter") ||
+          !ParseShape(&shape)) {
+        return false;
+      }
+    } while (EatIfPresent(TokKind::kComma));
+  }
+  return ParseToken(TokKind::kRparen, "expects ')' at the end of param list");
+}
+
+// shape ::= shape_val_
+// shape ::= '(' tuple_elements ')'
+// tuple_elements
+//   ::= /*empty*/
+//   ::= shape (',' shape)*
+bool HloParser::ParseShape(Shape* result) {
+  if (EatIfPresent(TokKind::kLparen)) {  // Tuple
+    std::vector<Shape> shapes;
+    if (lexer_.GetKind() == TokKind::kRparen) {
+      /*empty*/
+    } else {
+      // shape (',' shape)*
+      do {
+        shapes.emplace_back();
+        if (!ParseShape(&shapes.back())) {
+          return false;
+        }
+      } while (EatIfPresent(TokKind::kComma));
+    }
+    *result = ShapeUtil::MakeTupleShape(shapes);
+    return ParseToken(TokKind::kRparen, "expects ')' at the end of tuple.");
+  }
+
+  if (lexer_.GetKind() != TokKind::kShape) {
+    return TokenError("expects shape");
+  }
+  *result = lexer_.GetShapeVal();
+  lexer_.Lex();
+  return true;
+}
+
+bool HloParser::ParseName(string* result) {
+  VLOG(1) << "ParseName";
+  if (lexer_.GetKind() != TokKind::kName) {
+    return TokenError("expects name");
+  }
+  *result = lexer_.GetStrVal();
+  lexer_.Lex();
+  return true;
+}
+
+bool HloParser::ParseOpcode(HloOpcode* result) {
+  VLOG(1) << "ParseOpcode";
+  if (lexer_.GetKind() != TokKind::kOpcode) {
+    return TokenError("expects opcode");
+  }
+  *result = lexer_.GetOpcodeVal();
+  lexer_.Lex();
+  return true;
+}
+
+bool HloParser::ParseInt64(int64* result) {
+  VLOG(1) << "ParseInt64";
+  if (lexer_.GetKind() != TokKind::kInt) {
+    return TokenError("expects integer");
+  }
+  *result = lexer_.GetInt64Val();
+  lexer_.Lex();
+  return true;
+}
+
+bool HloParser::ParseDecimal(double* result) {
+  switch (lexer_.GetKind()) {
+    case TokKind::kDecimal:
+      *result = lexer_.GetDecimalVal();
+      break;
+    case TokKind::kInt:
+      *result = static_cast<double>(lexer_.GetInt64Val());
+      break;
+    default:
+      return TokenError("expects decimal or integer");
+  }
+  lexer_.Lex();
+  return true;
+}
+
+bool HloParser::ParseBool(bool* result) {
+  if (lexer_.GetKind() != TokKind::kw_true &&
+      lexer_.GetKind() != TokKind::kw_false) {
+    return TokenError("expects true or false");
+  }
+  *result = lexer_.GetKind() == TokKind::kw_true;
+  lexer_.Lex();
+  return true;
+}
+
+bool HloParser::ParseToken(TokKind kind, const string& msg) {
+  if (lexer_.GetKind() != kind) {
+    return TokenError(msg);
+  }
+  lexer_.Lex();
+  return true;
+}
+
+bool HloParser::EatIfPresent(TokKind kind) {
+  if (lexer_.GetKind() != kind) {
+    return false;
+  }
+  lexer_.Lex();
+  return true;
+}
+
+bool HloParser::AddInstruction(const string& name,
+                               HloInstruction* instruction) {
+  auto result = instruction_pool_.insert({name, instruction});
+  if (!result.second) {
+    return TokenError(StrCat("instruction already exists: ", name));
+  }
+  return true;
+}
+
+}  // namespace
+
+StatusOr<std::unique_ptr<HloModule>> Parse(StringPiece str) {
+  HloParser parser(str);
+  if (!parser.Run()) {
+    return InvalidArgument("Syntax error: %s", parser.GetError().c_str());
+  }
+  return parser.ConsumeHloModule();
+}
+
+}  // namespace tools
+}  // namespace xla
diff --git a/tensorflow/compiler/xla/tools/parser/hlo_parser.h b/tensorflow/compiler/xla/tools/parser/hlo_parser.h
new file mode 100644
index 0000000000..9aaf18ef20
--- /dev/null
+++ b/tensorflow/compiler/xla/tools/parser/hlo_parser.h
@@ -0,0 +1,37 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_COMPILER_XLA_TOOLS_PARSER_HLO_PARSER_H_
+#define TENSORFLOW_COMPILER_XLA_TOOLS_PARSER_HLO_PARSER_H_
+
+#include "tensorflow/compiler/xla/ptr_util.h"
+#include "tensorflow/compiler/xla/service/hlo_computation.h"
+#include "tensorflow/compiler/xla/service/hlo_instruction.h"
+#include "tensorflow/compiler/xla/service/hlo_module.h"
+#include "tensorflow/compiler/xla/statusor.h"
+#include "tensorflow/compiler/xla/tools/parser/hlo_lexer.h"
+#include "tensorflow/compiler/xla/xla_data.pb.h"
+
+namespace xla {
+namespace tools {
+
+// The api of the hlo parser. Given a string in the HloModule::ToString()
+// format, returns the parsed HloModule.
+StatusOr<std::unique_ptr<HloModule>> Parse(tensorflow::StringPiece str);
+
+}  // namespace tools
+}  // namespace xla
+
+#endif  // TENSORFLOW_COMPILER_XLA_TOOLS_PARSER_HLO_PARSER_H_
diff --git a/tensorflow/compiler/xla/tools/parser/hlo_parser_test.cc b/tensorflow/compiler/xla/tools/parser/hlo_parser_test.cc
new file mode 100644
index 0000000000..4ecece3eac
--- /dev/null
+++ b/tensorflow/compiler/xla/tools/parser/hlo_parser_test.cc
@@ -0,0 +1,240 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/tools/parser/hlo_parser.h"
+
+#include <string>
+#include "tensorflow/core/lib/core/status_test_util.h"
+#include "tensorflow/core/platform/test.h"
+
+namespace xla {
+namespace tools {
+namespace {
+
+struct TestData {
+  string test_name;
+  string module_string;
+};
+
+string TestDataToString(const ::testing::TestParamInfo<TestData>& data) {
+  return data.param.test_name;
+}
+
+std::vector<TestData> CreateTestCases() {
+  // clang-format off
+  return std::vector<TestData>({
+// ax + y
+{
+"AxpyParam",
+R"(HloModule axpy_module:
+
+ENTRY %axpy.v5 (alpha: f32[2,4], x: f32[2,4], y: f32[2,4]) -> f32[2,4] {
+  %alpha = f32[2,4]{1,0} parameter(0)
+  %x = f32[2,4]{1,0} parameter(1)
+  %multiply = f32[2,4]{1,0} multiply(f32[2,4]{1,0} %alpha, f32[2,4]{1,0} %x)
+  %y = f32[2,4]{1,0} parameter(2)
+  %add = f32[2,4]{1,0} add(f32[2,4]{1,0} %multiply, f32[2,4]{1,0} %y)
+}
+
+)"
+},
+// pred constant
+{
+"ConstantPred",
+R"(HloModule constant_pred_module:
+
+ENTRY %constant_pred () -> pred[] {
+  %constant = pred[] constant(true)
+}
+
+)"
+},
+// s32 constant
+{
+"ConstantS32",
+R"(HloModule constant_s32_module:
+
+ENTRY %constant_s32 () -> s32[] {
+  %constant = s32[] constant(-42)
+}
+
+)"
+},
+// f32 constant, but the value is not a decimal
+{
+"ConstantF32", R"(HloModule ConstantF32_module:
+
+ENTRY %ConstantF32.v4 () -> f32[] {
+  %constant = f32[] constant(42)
+}
+
+)"
+},
+// constant + constant
+{
+"AddConstants",
+R"(HloModule add_constants_module:
+
+ENTRY %add_constants () -> f32[] {
+  %constant = f32[] constant(3.14)
+  %add = f32[] add(f32[] %constant, f32[] %constant)
+}
+
+)"
+},
+// v1 > v2 ? v1 : v2
+{
+"SelectR1F32",
+R"(HloModule SelectR1F32WithCmpR1F32sFromParamsSmall_module:
+
+ENTRY %SelectR1F32WithCmpR1F32sFromParamsSmall.v4 (v1: f32[4], v2: f32[4]) -> f32[4] {
+  %v1 = f32[4]{0} parameter(0)
+  %v2 = f32[4]{0} parameter(1)
+  %greater-than = pred[4]{0} greater-than(f32[4]{0} %v1, f32[4]{0} %v2)
+  %select = f32[4]{0} select(pred[4]{0} %greater-than, f32[4]{0} %v1, f32[4]{0} %v2)
+}
+
+)"
+}
+  });
+  // clang-format on
+}
+
+class HloParserTest : public ::testing::Test,
+                      public ::testing::WithParamInterface<TestData> {
+ protected:
+  void ExpectSuccess() {
+    const string& original = GetParam().module_string;
+    auto result = Parse(original);
+    TF_EXPECT_OK(result.status());
+    EXPECT_EQ(original, result.ValueOrDie()->ToString());
+  }
+};
+
+TEST_P(HloParserTest, Run) { ExpectSuccess(); }
+
+INSTANTIATE_TEST_CASE_P(HloParserTestSuccessInstantiation, HloParserTest,
+                        ::testing::ValuesIn(CreateTestCases()),
+                        TestDataToString);
+
+TEST_F(HloParserTest, Empty) {
+  const string original = "";
+  auto result = Parse(original);
+  EXPECT_NE(tensorflow::Status::OK(), result.status());
+}
+
+TEST_F(HloParserTest, Garbage) {
+  const string original = "HloModule thi$ str1ng makes# N0 sen$e @all!*&^%$";
+  auto result = Parse(original);
+  EXPECT_NE(tensorflow::Status::OK(), result.status());
+}
+
+TEST_F(HloParserTest, WrongOpcode) {
+  const string original = R"(HloModule wrong_opcode:
+
+ENTRY %blabla (x: f32[], y: f32[]) -> f32[] {
+  %x = f32[]{} parameter(0)
+  %y = f32[]{} parameter(1)
+  %le = pred[]{} le(f32[]{} %x, f32[]{} %y)
+}
+
+)";
+  auto result = Parse(original);
+  EXPECT_NE(tensorflow::Status::OK(), result.status());
+}
+
+TEST_F(HloParserTest, WrongShape) {
+  const string original = R"(HloModule wrong_opcode:
+
+ENTRY %blabla (x: g32[]) -> g32[] {
+  %x = g32[]{} parameter(0)
+}
+
+)";
+  auto result = Parse(original);
+  EXPECT_NE(tensorflow::Status::OK(), result.status());
+}
+
+TEST_F(HloParserTest, WrongOperandsSize) {
+  const string original = R"(HloModule wrong_opcode:
+
+ENTRY %blabla (x: f32[]) -> pred[] {
+  %x = f32[]{} parameter(0)
+  %eq = pred[]{} equal-to(f32[]{} %x)
+}
+
+)";
+  auto result = Parse(original);
+  EXPECT_NE(tensorflow::Status::OK(), result.status());
+}
+
+TEST_F(HloParserTest, OperandNotFound) {
+  const string original = R"(HloModule operand_not_found:
+ENTRY %blabla (x: f32[]) -> pred[] {
+  %x = f32[]{} parameter(0)
+  %eq = pred[]{} equal-to(f32[]{} %x, f32[]{} %y)
+}
+)";
+  auto result = Parse(original);
+  EXPECT_NE(tensorflow::Status::OK(), result.status());
+}
+
+TEST_F(HloParserTest, MoreConstants) {
+  const string original = R"(HloModule SelectScalarS32True_module:
+
+ENTRY %SelectScalarS32True.v4 () -> s32[] {
+  %constant.2 = pred[] constant(true)
+  %constant.1 = s32[] constant(-42)
+  %constant = s32[] constant(42)
+  %select = s32[] select(pred[] %constant.2, s32[] %constant.1, s32[] %constant)
+}
+
+)";
+  auto result = Parse(original);
+  TF_EXPECT_OK(result.status());
+  // Constant instructions have no name. The string will be parsed successfully
+  // but the constant names will not be exactly the same.
+}
+
+TEST_F(HloParserTest, ConstantWithExp) {
+  const string original = R"(HloModule ConstantWithExp_module:
+
+ENTRY %ConstantWithExp.v4 () -> f32[] {
+  %constant.1 = f32[] constant(3e+2)
+}
+
+)";
+  auto result = Parse(original);
+  TF_EXPECT_OK(result.status());
+  // The string will be parsed successfully but the output strings are not
+  // exactly the same, because "3e2" is parsed into value 300 and will be
+  // printed as "300".
+}
+
+TEST_F(HloParserTest, Tuple) {
+  const string original = R"(HloModule EmptyTupleCreate_module:
+
+ENTRY %EmptyTupleCreate.v1 () -> () {
+  %tuple = () tuple()
+}
+
+)";
+  auto result = Parse(original);
+  EXPECT_NE(tensorflow::Status::OK(), result.status());
+}
+
+}  // namespace
+}  // namespace tools
+}  // namespace xla
diff --git a/tensorflow/compiler/xla/tools/parser/hlo_token.h b/tensorflow/compiler/xla/tools/parser/hlo_token.h
new file mode 100644
index 0000000000..1f75e17c7f
--- /dev/null
+++ b/tensorflow/compiler/xla/tools/parser/hlo_token.h
@@ -0,0 +1,58 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_COMPILER_XLA_TOOLS_PARSER_HLO_TOKEN_H_
+#define TENSORFLOW_COMPILER_XLA_TOOLS_PARSER_HLO_TOKEN_H_
+
+namespace xla {
+namespace tools {
+
+// Defines different kinds of tokens in a hlo module string.
+enum class TokKind {
+  // Markers
+  kEof,
+  kError,
+
+  // Tokens with no info.
+  kEqual,  // =
+  kComma,  // ,
+  kColon,  // :
+  kLsquare,
+  kRsquare,  // [  ]
+  kLbrace,
+  kRbrace,  // {  }
+  kLparen,
+  kRparen,  // (  )
+
+  kArrow,  // ->
+
+  // Keywords
+  kw_HloModule,
+  kw_ENTRY,
+  kw_true,
+  kw_false,
+
+  // Typed tokens.
+  kName,     // %foo
+  kShape,    // f32[2,3]{1,0}
+  kOpcode,   // add
+  kInt,      // 42
+  kDecimal,  // 4.2
+};
+
+}  // namespace tools
+}  // namespace xla
+
+#endif  // TENSORFLOW_COMPILER_XLA_TOOLS_PARSER_HLO_TOKEN_H_
-- 
GitLab


From 2cd178ef5a4e5cac27b55729f0203c4864540063 Mon Sep 17 00:00:00 2001
From: David Majnemer <majnemer@google.com>
Date: Thu, 19 Oct 2017 15:22:08 -0700
Subject: [PATCH 581/909] [XLA] Teach transpose folding how to transpose the
 LHS of convolutions

This is now possible now that we have added the required fields to
ConvolutionDimensionNumbers.

PiperOrigin-RevId: 172807540
---
 .../compiler/xla/service/transpose_folding.cc | 105 ++++++++++++------
 .../xla/service/transpose_folding_test.cc     |  28 +++--
 2 files changed, 89 insertions(+), 44 deletions(-)

diff --git a/tensorflow/compiler/xla/service/transpose_folding.cc b/tensorflow/compiler/xla/service/transpose_folding.cc
index 816c8a7485..8c2640adf5 100644
--- a/tensorflow/compiler/xla/service/transpose_folding.cc
+++ b/tensorflow/compiler/xla/service/transpose_folding.cc
@@ -58,14 +58,32 @@ TransposeFolding::OperandIndices CanFoldOperandsIntoConvolution(
     return {};
   }
 
-  // We only support folding the RHS.
-  const int64 kRhsOperandIndex = 1;
-  auto& operand = *convolution.operand(kRhsOperandIndex);
-  if (operand.opcode() == HloOpcode::kTranspose && operand.user_count() == 1) {
-    return transposable_conv_operands(convolution, {kRhsOperandIndex});
+  const ConvolutionDimensionNumbers& dnums =
+      convolution.convolution_dimension_numbers();
+
+  TransposeFolding::OperandIndices operand_set;
+  for (int64 i = 0; i < convolution.operand_count(); ++i) {
+    auto& operand = *convolution.operand(i);
+    if (operand.opcode() == HloOpcode::kTranspose &&
+        operand.user_count() == 1) {
+      const auto& transpose_dimensions = operand.dimensions();
+      // We can transpose the LHS so long as it doesn't move around spatial
+      // dimensions because ConvolutionDimensionNumbers doesn't have different
+      // fields for input and output spatial dimensions.
+      if (i == 0 &&
+          std::any_of(dnums.spatial_dimensions().begin(),
+                      dnums.spatial_dimensions().end(),
+                      [&](const int64 spatial_dimension) {
+                        return transpose_dimensions[spatial_dimension] !=
+                               spatial_dimension;
+                      })) {
+        continue;
+      }
+      operand_set.push_back(i);
+    }
   }
 
-  return {};
+  return transposable_conv_operands(convolution, operand_set);
 }
 
 using InstructionOperandsPair =
@@ -98,40 +116,61 @@ bool FoldTransposeIntoDot(InstructionOperandsPair pair) {
 // Returns whether the module is changed.
 bool FoldTransposeIntoConvolution(InstructionOperandsPair pair) {
   auto& convolution = *pair.first;
-
-  // We only support fusing the RHS transpose into convolution.
-  //
-  // ConvolutionDimensionNumbers doesn't make enough of a distinction between
-  // the output and the activations.
-  //
-  // TODO(b/37125184): Support transposing the LHS too.
-  if (pair.second.size() != 1 || pair.second.front() != 1) {
-    return false;
-  }
+  auto& operand_indices = pair.second;
 
   const ConvolutionDimensionNumbers& dnums =
       convolution.convolution_dimension_numbers();
-  HloInstruction& transpose = *convolution.mutable_operand(1);
-  CHECK_EQ(transpose.opcode(), HloOpcode::kTranspose);
-  const auto& transpose_dimensions = transpose.dimensions();
-  HloInstruction& transpose_operand = *transpose.mutable_operand(0);
-
-  // Everything remains the same except for the kernel dimension numbers. We
-  // need to apply the transpose permutation to the original shape to figure out
-  // what the new logical dimensions are.
   ConvolutionDimensionNumbers new_dnums = dnums;
-  new_dnums.set_kernel_input_feature_dimension(
-      transpose_dimensions[dnums.kernel_input_feature_dimension()]);
-  new_dnums.set_kernel_output_feature_dimension(
-      transpose_dimensions[dnums.kernel_output_feature_dimension()]);
-  for (auto& kernel_spatial_dimension :
-       *new_dnums.mutable_kernel_spatial_dimensions()) {
-    kernel_spatial_dimension = transpose_dimensions[kernel_spatial_dimension];
+
+  HloInstruction* new_lhs;
+  const int64 kLhsIdx = 0;
+  if (std::find(operand_indices.begin(), operand_indices.end(), kLhsIdx) !=
+      operand_indices.end()) {
+    HloInstruction& transpose = *convolution.mutable_operand(kLhsIdx);
+    const auto& transpose_dimensions = transpose.dimensions();
+    HloInstruction& transpose_operand = *transpose.mutable_operand(0);
+
+    // Everything remains the same except for the input/output dimension
+    // numbers. We need to apply the transpose permutation to the original shape
+    // to figure out what the new logical dimensions are.
+    new_dnums.set_input_batch_dimension(
+        transpose_dimensions[dnums.input_batch_dimension()]);
+    new_dnums.set_input_feature_dimension(
+        transpose_dimensions[dnums.input_feature_dimension()]);
+    for (const auto& spatial_dimension : dnums.spatial_dimensions()) {
+      CHECK_EQ(spatial_dimension, transpose_dimensions[spatial_dimension]);
+    }
+    new_lhs = &transpose_operand;
+  } else {
+    new_lhs = convolution.mutable_operand(kLhsIdx);
+  }
+
+  HloInstruction* new_rhs;
+  const int64 kRhsIdx = 1;
+  if (std::find(operand_indices.begin(), operand_indices.end(), kRhsIdx) !=
+      operand_indices.end()) {
+    HloInstruction& transpose = *convolution.mutable_operand(kRhsIdx);
+    const auto& transpose_dimensions = transpose.dimensions();
+    HloInstruction& transpose_operand = *transpose.mutable_operand(0);
+
+    // Everything remains the same except for the kernel dimension numbers. We
+    // need to apply the transpose permutation to the original shape to figure
+    // out what the new logical dimensions are.
+    new_dnums.set_kernel_input_feature_dimension(
+        transpose_dimensions[dnums.kernel_input_feature_dimension()]);
+    new_dnums.set_kernel_output_feature_dimension(
+        transpose_dimensions[dnums.kernel_output_feature_dimension()]);
+    for (auto& kernel_spatial_dimension :
+         *new_dnums.mutable_kernel_spatial_dimensions()) {
+      kernel_spatial_dimension = transpose_dimensions[kernel_spatial_dimension];
+    }
+    new_rhs = &transpose_operand;
+  } else {
+    new_rhs = convolution.mutable_operand(kRhsIdx);
   }
 
   auto new_conv = HloInstruction::CreateConvolve(
-      convolution.shape(), convolution.mutable_operand(0), &transpose_operand,
-      convolution.window(), new_dnums);
+      convolution.shape(), new_lhs, new_rhs, convolution.window(), new_dnums);
   TF_CHECK_OK(convolution.parent()->ReplaceWithNewInstruction(
       &convolution, std::move(new_conv)));
 
diff --git a/tensorflow/compiler/xla/service/transpose_folding_test.cc b/tensorflow/compiler/xla/service/transpose_folding_test.cc
index a6161b4646..00462f9be1 100644
--- a/tensorflow/compiler/xla/service/transpose_folding_test.cc
+++ b/tensorflow/compiler/xla/service/transpose_folding_test.cc
@@ -313,8 +313,7 @@ TEST_F(TransposeFoldingTest, FoldConvComplexTransposeRhs) {
       new_conv->convolution_dimension_numbers().kernel_spatial_dimensions(1));
 }
 
-// Test that a transpose of the activations does not get folded into
-// convolution.
+// Test that a transpose of the activations gets folded into convolution.
 TEST_F(TransposeFoldingTest, FoldConvTransposeLhs) {
   auto builder = HloComputation::Builder("entry_computation");
   HloInstruction* x = builder.AddInstruction(HloInstruction::CreateParameter(
@@ -348,18 +347,25 @@ TEST_F(TransposeFoldingTest, FoldConvTransposeLhs) {
       module.AddEntryComputation(builder.Build(conv));
   FoldTranspose(&module);
 
-  // Instructions after folding: transpose_x, y, and the convolution.
+  // Instructions after folding: x, y, and the convolution.
   std::unordered_set<HloInstruction*> instruction_set(
       entry_computation->instructions().begin(),
       entry_computation->instructions().end());
-  CHECK_EQ(1, instruction_set.erase(x)) << "x is not in entry_computation.";
-  CHECK_EQ(1, instruction_set.erase(y)) << "y is not in entry_computation.";
-  CHECK_EQ(1, instruction_set.erase(transpose_x))
-      << "transpose_x is not in entry_computation.";
-  CHECK_EQ(1, instruction_set.erase(conv))
-      << "transpose_x is not in entry_computation.";
-  CHECK_EQ(0, instruction_set.size())
-      << "entry_computation should contain exactly 4 instructions.";
+  EXPECT_EQ(1, instruction_set.erase(x)) << "x is not in entry_computation.";
+  EXPECT_EQ(1, instruction_set.erase(y)) << "y is not in entry_computation.";
+  EXPECT_EQ(1, instruction_set.size())
+      << "entry_computation should contain exactly 3 instructions.";
+  HloInstruction* new_conv = *instruction_set.begin();
+  EXPECT_EQ(HloOpcode::kConvolution, new_conv->opcode());
+  EXPECT_EQ(dnums.input_feature_dimension(),
+            new_conv->convolution_dimension_numbers().input_batch_dimension());
+  EXPECT_EQ(
+      dnums.input_batch_dimension(),
+      new_conv->convolution_dimension_numbers().input_feature_dimension());
+  EXPECT_EQ(dnums.spatial_dimensions(0),
+            new_conv->convolution_dimension_numbers().spatial_dimensions(0));
+  EXPECT_EQ(dnums.spatial_dimensions(1),
+            new_conv->convolution_dimension_numbers().spatial_dimensions(1));
 }
 
 }  // namespace
-- 
GitLab


From f080052284a4a39113051fb1178d91365e9872a8 Mon Sep 17 00:00:00 2001
From: Igor Saprykin <isaprykin@google.com>
Date: Thu, 19 Oct 2017 15:27:52 -0700
Subject: [PATCH 582/909] Move text_classification_character_rnn from .contrib
 utils to .core utils.

Also removes sklearn comparison.

PiperOrigin-RevId: 172808535
---
 .../text_classification_character_rnn.py      | 19 +++++--------------
 1 file changed, 5 insertions(+), 14 deletions(-)

diff --git a/tensorflow/examples/learn/text_classification_character_rnn.py b/tensorflow/examples/learn/text_classification_character_rnn.py
index 1fc9388a1a..86adc056ad 100644
--- a/tensorflow/examples/learn/text_classification_character_rnn.py
+++ b/tensorflow/examples/learn/text_classification_character_rnn.py
@@ -30,7 +30,6 @@ import sys
 
 import numpy as np
 import pandas
-from sklearn import metrics
 import tensorflow as tf
 
 FLAGS = None
@@ -46,8 +45,8 @@ def char_rnn_model(features, labels, mode):
   byte_vectors = tf.one_hot(features[CHARS_FEATURE], 256, 1., 0.)
   byte_list = tf.unstack(byte_vectors, axis=1)
 
-  cell = tf.contrib.rnn.GRUCell(HIDDEN_SIZE)
-  _, encoding = tf.contrib.rnn.static_rnn(cell, byte_list, dtype=tf.float32)
+  cell = tf.nn.rnn_cell.GRUCell(HIDDEN_SIZE)
+  _, encoding = tf.nn.static_rnn(cell, byte_list, dtype=tf.float32)
 
   logits = tf.layers.dense(encoding, MAX_LABEL, activation=None)
 
@@ -98,28 +97,20 @@ def main(unused_argv):
   train_input_fn = tf.estimator.inputs.numpy_input_fn(
       x={CHARS_FEATURE: x_train},
       y=y_train,
-      batch_size=len(x_train),
+      batch_size=128,
       num_epochs=None,
       shuffle=True)
   classifier.train(input_fn=train_input_fn, steps=100)
 
-  # Predict.
+  # Eval.
   test_input_fn = tf.estimator.inputs.numpy_input_fn(
       x={CHARS_FEATURE: x_test},
       y=y_test,
       num_epochs=1,
       shuffle=False)
-  predictions = classifier.predict(input_fn=test_input_fn)
-  y_predicted = np.array(list(p['class'] for p in predictions))
-  y_predicted = y_predicted.reshape(np.array(y_test).shape)
 
-  # Score with sklearn.
-  score = metrics.accuracy_score(y_test, y_predicted)
-  print('Accuracy (sklearn): {0:f}'.format(score))
-
-  # Score with tensorflow.
   scores = classifier.evaluate(input_fn=test_input_fn)
-  print('Accuracy (tensorflow): {0:f}'.format(scores['accuracy']))
+  print('Accuracy: {0:f}'.format(scores['accuracy']))
 
 
 if __name__ == '__main__':
-- 
GitLab


From bc93dcbd9f7b445c5f6f0d1c8f597324d412a76a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 19 Oct 2017 16:00:31 -0700
Subject: [PATCH 583/909] Fix precision/recall test.

Precision and Recall have as the numerator TP: true positives.
The labels generated in the test were only negative, and hence the test passed before because all updates were 0.

PiperOrigin-RevId: 172812994
---
 .../metrics/python/ops/metric_ops_test.py     | 58 +++++++++----------
 1 file changed, 26 insertions(+), 32 deletions(-)

diff --git a/tensorflow/contrib/metrics/python/ops/metric_ops_test.py b/tensorflow/contrib/metrics/python/ops/metric_ops_test.py
index cc0ad155fa..f288fceef6 100644
--- a/tensorflow/contrib/metrics/python/ops/metric_ops_test.py
+++ b/tensorflow/contrib/metrics/python/ops/metric_ops_test.py
@@ -1101,7 +1101,7 @@ class StreamingPrecisionTest(test.TestCase):
     predictions = random_ops.random_uniform(
         (10, 3), maxval=1, dtype=dtypes_lib.int64, seed=1)
     labels = random_ops.random_uniform(
-        (10, 3), maxval=1, dtype=dtypes_lib.int64, seed=2)
+        (10, 3), maxval=2, dtype=dtypes_lib.int64, seed=2)
     precision, update_op = metrics.streaming_precision(predictions, labels)
 
     with self.test_session() as sess:
@@ -1265,7 +1265,7 @@ class StreamingRecallTest(test.TestCase):
     predictions = random_ops.random_uniform(
         (10, 3), maxval=1, dtype=dtypes_lib.int64, seed=1)
     labels = random_ops.random_uniform(
-        (10, 3), maxval=1, dtype=dtypes_lib.int64, seed=2)
+        (10, 3), maxval=2, dtype=dtypes_lib.int64, seed=2)
     recall, update_op = metrics.streaming_recall(predictions, labels)
 
     with self.test_session() as sess:
@@ -1388,7 +1388,7 @@ class StreamingFPRTest(test.TestCase):
     predictions = random_ops.random_uniform(
         (10, 3), maxval=1, dtype=dtypes_lib.int64, seed=1)
     labels = random_ops.random_uniform(
-        (10, 3), maxval=1, dtype=dtypes_lib.int64, seed=2)
+        (10, 3), maxval=2, dtype=dtypes_lib.int64, seed=2)
     fpr, update_op = metrics.streaming_false_positive_rate(
         predictions, labels)
 
@@ -1516,7 +1516,7 @@ class StreamingFNRTest(test.TestCase):
     predictions = random_ops.random_uniform(
         (10, 3), maxval=1, dtype=dtypes_lib.int64, seed=1)
     labels = random_ops.random_uniform(
-        (10, 3), maxval=1, dtype=dtypes_lib.int64, seed=2)
+        (10, 3), maxval=2, dtype=dtypes_lib.int64, seed=2)
     fnr, update_op = metrics.streaming_false_negative_rate(
         predictions, labels)
 
@@ -1737,7 +1737,7 @@ class StreamingAUCTest(test.TestCase):
     predictions = random_ops.random_uniform(
         (10, 3), maxval=1, dtype=dtypes_lib.float32, seed=1)
     labels = random_ops.random_uniform(
-        (10, 3), maxval=1, dtype=dtypes_lib.int64, seed=2)
+        (10, 3), maxval=2, dtype=dtypes_lib.int64, seed=2)
     auc, update_op = metrics.streaming_auc(predictions, labels)
 
     with self.test_session() as sess:
@@ -2009,7 +2009,7 @@ class StreamingSpecificityAtSensitivityTest(test.TestCase):
     predictions = random_ops.random_uniform(
         (10, 3), maxval=1, dtype=dtypes_lib.float32, seed=1)
     labels = random_ops.random_uniform(
-        (10, 3), maxval=1, dtype=dtypes_lib.int64, seed=2)
+        (10, 3), maxval=2, dtype=dtypes_lib.int64, seed=2)
     specificity, update_op = metrics.streaming_specificity_at_sensitivity(
         predictions, labels, sensitivity=0.7)
 
@@ -2271,7 +2271,7 @@ class StreamingPrecisionRecallThresholdsTest(test.TestCase):
     predictions = random_ops.random_uniform(
         (10, 3), maxval=1, dtype=dtypes_lib.float32, seed=1)
     labels = random_ops.random_uniform(
-        (10, 3), maxval=1, dtype=dtypes_lib.int64, seed=2)
+        (10, 3), maxval=2, dtype=dtypes_lib.int64, seed=2)
     thresholds = [0, 0.5, 1.0]
     prec, prec_op = metrics.streaming_precision_at_thresholds(predictions,
                                                               labels,
@@ -2282,12 +2282,14 @@ class StreamingPrecisionRecallThresholdsTest(test.TestCase):
     with self.test_session() as sess:
       sess.run(variables.local_variables_initializer())
 
-      # Run several updates, then verify idempotency.
-      sess.run([prec_op, rec_op])
+      # Run several updates.
+      for _ in range(10):
+        sess.run([prec_op, rec_op])
+
+      # Then verify idempotency.
       initial_prec = prec.eval()
       initial_rec = rec.eval()
       for _ in range(10):
-        sess.run([prec_op, rec_op])
         self.assertAllClose(initial_prec, prec.eval())
         self.assertAllClose(initial_rec, rec.eval())
 
@@ -2361,14 +2363,10 @@ class StreamingPrecisionRecallThresholdsTest(test.TestCase):
       rec, rec_op = metrics.streaming_recall_at_thresholds(
           predictions, labels, thresholds, weights=weights)
 
-      [prec_low, prec_high] = array_ops.split(
-          value=prec, num_or_size_splits=2, axis=0)
-      prec_low = array_ops.reshape(prec_low, shape=())
-      prec_high = array_ops.reshape(prec_high, shape=())
-      [rec_low, rec_high] = array_ops.split(
-          value=rec, num_or_size_splits=2, axis=0)
-      rec_low = array_ops.reshape(rec_low, shape=())
-      rec_high = array_ops.reshape(rec_high, shape=())
+      prec_low = prec[0]
+      prec_high = prec[1]
+      rec_low = rec[0]
+      rec_high = rec[1]
 
       sess.run(variables.local_variables_initializer())
       sess.run([prec_op, rec_op])
@@ -2391,14 +2389,10 @@ class StreamingPrecisionRecallThresholdsTest(test.TestCase):
       rec, rec_op = metrics.streaming_recall_at_thresholds(
           predictions, labels, thresholds, weights=weights)
 
-      [prec_low, prec_high] = array_ops.split(
-          value=prec, num_or_size_splits=2, axis=0)
-      prec_low = array_ops.reshape(prec_low, shape=())
-      prec_high = array_ops.reshape(prec_high, shape=())
-      [rec_low, rec_high] = array_ops.split(
-          value=rec, num_or_size_splits=2, axis=0)
-      rec_low = array_ops.reshape(rec_low, shape=())
-      rec_high = array_ops.reshape(rec_high, shape=())
+      prec_low = prec[0]
+      prec_high = prec[1]
+      rec_low = rec[0]
+      rec_high = rec[1]
 
       sess.run(variables.local_variables_initializer())
       sess.run([prec_op, rec_op])
@@ -2420,10 +2414,10 @@ class StreamingPrecisionRecallThresholdsTest(test.TestCase):
       rec, rec_op = metrics.streaming_recall_at_thresholds(predictions, labels,
                                                            thresholds)
 
-      [prec_low, prec_high] = array_ops.split(
-          value=prec, num_or_size_splits=2, axis=0)
-      [rec_low, rec_high] = array_ops.split(
-          value=rec, num_or_size_splits=2, axis=0)
+      prec_low = prec[0]
+      prec_high = prec[1]
+      rec_low = rec[0]
+      rec_high = rec[1]
 
       sess.run(variables.local_variables_initializer())
       sess.run([prec_op, rec_op])
@@ -2562,7 +2556,7 @@ class StreamingFPRThresholdsTest(test.TestCase):
     predictions = random_ops.random_uniform(
         (10, 3), maxval=1, dtype=dtypes_lib.float32, seed=1)
     labels = random_ops.random_uniform(
-        (10, 3), maxval=1, dtype=dtypes_lib.int64, seed=2)
+        (10, 3), maxval=2, dtype=dtypes_lib.int64, seed=2)
     thresholds = [0, 0.5, 1.0]
     fpr, fpr_op = metrics.streaming_false_positive_rate_at_thresholds(
         predictions, labels, thresholds)
@@ -2794,7 +2788,7 @@ class StreamingFNRThresholdsTest(test.TestCase):
     predictions = random_ops.random_uniform(
         (10, 3), maxval=1, dtype=dtypes_lib.float32, seed=1)
     labels = random_ops.random_uniform(
-        (10, 3), maxval=1, dtype=dtypes_lib.int64, seed=2)
+        (10, 3), maxval=2, dtype=dtypes_lib.int64, seed=2)
     thresholds = [0, 0.5, 1.0]
     fnr, fnr_op = metrics.streaming_false_negative_rate_at_thresholds(
         predictions, labels, thresholds)
-- 
GitLab


From 7a253f3da99c3692d464a8dd95d8280d4cd8973a Mon Sep 17 00:00:00 2001
From: Igor Saprykin <isaprykin@google.com>
Date: Thu, 19 Oct 2017 16:16:29 -0700
Subject: [PATCH 584/909] Fix random_forest_mnist.py and eliminate a
 contrib.learn reference to skcompat.

PiperOrigin-RevId: 172815173
---
 .../examples/learn/random_forest_mnist.py     | 65 ++++++++++---------
 1 file changed, 36 insertions(+), 29 deletions(-)

diff --git a/tensorflow/examples/learn/random_forest_mnist.py b/tensorflow/examples/learn/random_forest_mnist.py
index 3c09990ea1..72c935cdae 100644
--- a/tensorflow/examples/learn/random_forest_mnist.py
+++ b/tensorflow/examples/learn/random_forest_mnist.py
@@ -1,4 +1,4 @@
-   # Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -21,18 +21,14 @@ import argparse
 import sys
 import tempfile
 
-# pylint: disable=g-backslash-continuation
-from tensorflow.contrib.learn.python.learn\
-        import metric_spec
-from tensorflow.contrib.learn.python.learn.estimators\
-        import estimator
-from tensorflow.contrib.tensor_forest.client\
-        import eval_metrics
-from tensorflow.contrib.tensor_forest.client\
-        import random_forest
-from tensorflow.contrib.tensor_forest.python\
-        import tensor_forest
+import numpy
+
+from tensorflow.contrib.learn.python.learn import metric_spec
+from tensorflow.contrib.tensor_forest.client import eval_metrics
+from tensorflow.contrib.tensor_forest.client import random_forest
+from tensorflow.contrib.tensor_forest.python import tensor_forest
 from tensorflow.examples.tutorials.mnist import input_data
+from tensorflow.python.estimator.inputs import numpy_io
 from tensorflow.python.platform import app
 
 FLAGS = None
@@ -41,16 +37,15 @@ FLAGS = None
 def build_estimator(model_dir):
   """Build an estimator."""
   params = tensor_forest.ForestHParams(
-      num_classes=10, num_features=784,
-      num_trees=FLAGS.num_trees, max_nodes=FLAGS.max_nodes)
+      num_classes=10,
+      num_features=784,
+      num_trees=FLAGS.num_trees,
+      max_nodes=FLAGS.max_nodes)
   graph_builder_class = tensor_forest.RandomForestGraphs
   if FLAGS.use_training_loss:
     graph_builder_class = tensor_forest.TrainingLossForest
-  # Use the SKCompat wrapper, which gives us a convenient way to split
-  # in-memory data like MNIST into batches.
-  return estimator.SKCompat(random_forest.TensorForestEstimator(
-      params, graph_builder_class=graph_builder_class,
-      model_dir=model_dir))
+  return random_forest.TensorForestEstimator(
+      params, graph_builder_class=graph_builder_class, model_dir=model_dir)
 
 
 def train_and_eval():
@@ -62,18 +57,30 @@ def train_and_eval():
 
   mnist = input_data.read_data_sets(FLAGS.data_dir, one_hot=False)
 
-  est.fit(x=mnist.train.images, y=mnist.train.labels,
-          batch_size=FLAGS.batch_size)
+  train_input_fn = numpy_io.numpy_input_fn(
+      x={'images': mnist.train.images},
+      y=mnist.train.labels.astype(numpy.int32),
+      batch_size=FLAGS.batch_size,
+      num_epochs=None,
+      shuffle=True)
+  est.fit(input_fn=train_input_fn, steps=None)
 
   metric_name = 'accuracy'
-  metric = {metric_name:
-            metric_spec.MetricSpec(
-                eval_metrics.get_metric(metric_name),
-                prediction_key=eval_metrics.get_prediction_key(metric_name))}
-
-  results = est.score(x=mnist.test.images, y=mnist.test.labels,
-                      batch_size=FLAGS.batch_size,
-                      metrics=metric)
+  metric = {
+      metric_name:
+          metric_spec.MetricSpec(
+              eval_metrics.get_metric(metric_name),
+              prediction_key=eval_metrics.get_prediction_key(metric_name))
+  }
+
+  test_input_fn = numpy_io.numpy_input_fn(
+      x={'images': mnist.test.images},
+      y=mnist.test.labels.astype(numpy.int32),
+      num_epochs=1,
+      batch_size=FLAGS.batch_size,
+      shuffle=False)
+
+  results = est.evaluate(input_fn=test_input_fn, metrics=metric)
   for key in sorted(results):
     print('%s: %s' % (key, results[key]))
 
-- 
GitLab


From 60a03dfc7dbde7acf58ffaeef897eb3ebb98603f Mon Sep 17 00:00:00 2001
From: Michael Case <mikecase@google.com>
Date: Thu, 19 Oct 2017 16:18:46 -0700
Subject: [PATCH 585/909] Move s3 file system support from contrib/ to
 core/platform/.

PiperOrigin-RevId: 172815422
---
 tensorflow/BUILD                                              | 2 +-
 tensorflow/contrib/makefile/Makefile                          | 1 +
 tensorflow/core/platform/default/build_config.bzl             | 2 +-
 tensorflow/{contrib => core/platform}/s3/BUILD                | 0
 tensorflow/{contrib => core/platform}/s3/s3_crypto.cc         | 2 +-
 tensorflow/{contrib => core/platform}/s3/s3_crypto.h          | 0
 tensorflow/{contrib => core/platform}/s3/s3_file_system.cc    | 4 ++--
 tensorflow/{contrib => core/platform}/s3/s3_file_system.h     | 0
 .../{contrib => core/platform}/s3/s3_file_system_test.cc      | 2 +-
 9 files changed, 7 insertions(+), 6 deletions(-)
 rename tensorflow/{contrib => core/platform}/s3/BUILD (100%)
 rename tensorflow/{contrib => core/platform}/s3/s3_crypto.cc (98%)
 rename tensorflow/{contrib => core/platform}/s3/s3_crypto.h (100%)
 rename tensorflow/{contrib => core/platform}/s3/s3_file_system.cc (99%)
 rename tensorflow/{contrib => core/platform}/s3/s3_file_system.h (100%)
 rename tensorflow/{contrib => core/platform}/s3/s3_file_system_test.cc (99%)

diff --git a/tensorflow/BUILD b/tensorflow/BUILD
index d5c56cdc18..d7d6d5fc77 100644
--- a/tensorflow/BUILD
+++ b/tensorflow/BUILD
@@ -414,7 +414,6 @@ filegroup(
         "//tensorflow/contrib/remote_fused_graph/pylib:all_files",
         "//tensorflow/contrib/resampler:all_files",
         "//tensorflow/contrib/rnn:all_files",
-        "//tensorflow/contrib/s3:all_files",
         "//tensorflow/contrib/saved_model:all_files",
         "//tensorflow/contrib/saved_model/cc/saved_model:all_files",
         "//tensorflow/contrib/seq2seq:all_files",
@@ -468,6 +467,7 @@ filegroup(
         "//tensorflow/core/platform/cloud:all_files",
         "//tensorflow/core/platform/default/build_config:all_files",
         "//tensorflow/core/platform/hadoop:all_files",
+        "//tensorflow/core/platform/s3:all_files",
         "//tensorflow/core/profiler:all_files",
         "//tensorflow/core/profiler/internal:all_files",
         "//tensorflow/core/profiler/internal/advisor:all_files",
diff --git a/tensorflow/contrib/makefile/Makefile b/tensorflow/contrib/makefile/Makefile
index be7c790ee9..3dcff3d4a3 100644
--- a/tensorflow/contrib/makefile/Makefile
+++ b/tensorflow/contrib/makefile/Makefile
@@ -502,6 +502,7 @@ $(wildcard tensorflow/core/platform/google/*) \
 $(wildcard tensorflow/core/platform/google/*/*) \
 $(wildcard tensorflow/core/platform/jpeg.*) \
 $(wildcard tensorflow/core/platform/png.*) \
+$(wildcard tensorflow/core/platform/s3/*) \
 $(wildcard tensorflow/core/platform/stream_executor.*) \
 $(wildcard tensorflow/core/platform/windows/*) \
 $(wildcard tensorflow/core/user_ops/*.cu.cc) \
diff --git a/tensorflow/core/platform/default/build_config.bzl b/tensorflow/core/platform/default/build_config.bzl
index 2c14ea917c..e4518a8e2f 100644
--- a/tensorflow/core/platform/default/build_config.bzl
+++ b/tensorflow/core/platform/default/build_config.bzl
@@ -467,7 +467,7 @@ def tf_additional_core_deps():
       "//conditions:default": [],
   }) + select({
       "//tensorflow:with_s3_support": [
-          "//tensorflow/contrib/s3:s3_file_system",
+          "//tensorflow/core/platform/s3:s3_file_system",
       ],
       "//conditions:default": [],
   })
diff --git a/tensorflow/contrib/s3/BUILD b/tensorflow/core/platform/s3/BUILD
similarity index 100%
rename from tensorflow/contrib/s3/BUILD
rename to tensorflow/core/platform/s3/BUILD
diff --git a/tensorflow/contrib/s3/s3_crypto.cc b/tensorflow/core/platform/s3/s3_crypto.cc
similarity index 98%
rename from tensorflow/contrib/s3/s3_crypto.cc
rename to tensorflow/core/platform/s3/s3_crypto.cc
index 1450384dc0..14bbed19a5 100644
--- a/tensorflow/contrib/s3/s3_crypto.cc
+++ b/tensorflow/core/platform/s3/s3_crypto.cc
@@ -12,7 +12,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-#include "tensorflow/contrib/s3/s3_crypto.h"
+#include "tensorflow/core/platform/s3/s3_crypto.h"
 #include <openssl/hmac.h>
 #include <openssl/sha.h>
 
diff --git a/tensorflow/contrib/s3/s3_crypto.h b/tensorflow/core/platform/s3/s3_crypto.h
similarity index 100%
rename from tensorflow/contrib/s3/s3_crypto.h
rename to tensorflow/core/platform/s3/s3_crypto.h
diff --git a/tensorflow/contrib/s3/s3_file_system.cc b/tensorflow/core/platform/s3/s3_file_system.cc
similarity index 99%
rename from tensorflow/contrib/s3/s3_file_system.cc
rename to tensorflow/core/platform/s3/s3_file_system.cc
index daced83145..51c85592bf 100644
--- a/tensorflow/contrib/s3/s3_file_system.cc
+++ b/tensorflow/core/platform/s3/s3_file_system.cc
@@ -12,10 +12,10 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-#include "tensorflow/contrib/s3/s3_file_system.h"
-#include "tensorflow/contrib/s3/s3_crypto.h"
 #include "tensorflow/core/lib/io/path.h"
 #include "tensorflow/core/platform/mutex.h"
+#include "tensorflow/core/platform/s3/s3_file_system.h"
+#include "tensorflow/core/platform/s3/s3_crypto.h"
 
 #include <aws/core/Aws.h>
 #include <aws/core/utils/FileSystemUtils.h>
diff --git a/tensorflow/contrib/s3/s3_file_system.h b/tensorflow/core/platform/s3/s3_file_system.h
similarity index 100%
rename from tensorflow/contrib/s3/s3_file_system.h
rename to tensorflow/core/platform/s3/s3_file_system.h
diff --git a/tensorflow/contrib/s3/s3_file_system_test.cc b/tensorflow/core/platform/s3/s3_file_system_test.cc
similarity index 99%
rename from tensorflow/contrib/s3/s3_file_system_test.cc
rename to tensorflow/core/platform/s3/s3_file_system_test.cc
index 949281fad4..0b42f5fcec 100644
--- a/tensorflow/contrib/s3/s3_file_system_test.cc
+++ b/tensorflow/core/platform/s3/s3_file_system_test.cc
@@ -13,7 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/contrib/s3/s3_file_system.h"
+#include "tensorflow/core/platform/s3/s3_file_system.h"
 
 #include "tensorflow/core/lib/core/status_test_util.h"
 #include "tensorflow/core/lib/gtl/stl_util.h"
-- 
GitLab


From d88cccebc7f61078d775d26f4714a06bc4002fcf Mon Sep 17 00:00:00 2001
From: Justine Tunney <jart@google.com>
Date: Thu, 19 Oct 2017 16:20:06 -0700
Subject: [PATCH 586/909] Rename SNAPPY to TF_USE_SNAPPY

This way there's less risk of it conflicting with downstream BUILD rules.

PiperOrigin-RevId: 172815580
---
 tensorflow/contrib/cmake/external/snappy.cmake | 2 +-
 tensorflow/core/BUILD                          | 2 +-
 tensorflow/core/platform/posix/port.cc         | 8 ++++----
 tensorflow/core/platform/windows/port.cc       | 8 ++++----
 4 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/tensorflow/contrib/cmake/external/snappy.cmake b/tensorflow/contrib/cmake/external/snappy.cmake
index a35d8654fb..2d2451521c 100644
--- a/tensorflow/contrib/cmake/external/snappy.cmake
+++ b/tensorflow/contrib/cmake/external/snappy.cmake
@@ -47,4 +47,4 @@ ExternalProject_Add(snappy
 )
 
 # actually enables snappy in the source code
-add_definitions(-DSNAPPY)
\ No newline at end of file
+add_definitions(-DTF_USE_SNAPPY)
diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 5ab84fec5b..d198a796a7 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -1410,7 +1410,7 @@ cc_library(
     hdrs = LIB_INTERNAL_PUBLIC_HEADERS,
     copts = tf_copts(),
     defines = tf_additional_lib_defines() + [
-                  "SNAPPY",
+                  "TF_USE_SNAPPY",
               ] + tf_additional_verbs_lib_defines() +
               tf_additional_mpi_lib_defines() +
               tf_additional_gdr_lib_defines(),
diff --git a/tensorflow/core/platform/posix/port.cc b/tensorflow/core/platform/posix/port.cc
index 3b17bac808..93a59348c8 100644
--- a/tensorflow/core/platform/posix/port.cc
+++ b/tensorflow/core/platform/posix/port.cc
@@ -29,7 +29,7 @@ limitations under the License.
 #include <stdlib.h>
 #include <string.h>
 #include <unistd.h>
-#ifdef SNAPPY
+#ifdef TF_USE_SNAPPY
 #include "snappy.h"
 #endif
 #if (defined(__APPLE__) && defined(__MACH__)) || defined(__FreeBSD__)
@@ -126,7 +126,7 @@ void AdjustFilenameForLogging(string* filename) {
 }
 
 bool Snappy_Compress(const char* input, size_t length, string* output) {
-#ifdef SNAPPY
+#ifdef TF_USE_SNAPPY
   output->resize(snappy::MaxCompressedLength(length));
   size_t outlen;
   snappy::RawCompress(input, length, &(*output)[0], &outlen);
@@ -139,7 +139,7 @@ bool Snappy_Compress(const char* input, size_t length, string* output) {
 
 bool Snappy_GetUncompressedLength(const char* input, size_t length,
                                   size_t* result) {
-#ifdef SNAPPY
+#ifdef TF_USE_SNAPPY
   return snappy::GetUncompressedLength(input, length, result);
 #else
   return false;
@@ -147,7 +147,7 @@ bool Snappy_GetUncompressedLength(const char* input, size_t length,
 }
 
 bool Snappy_Uncompress(const char* input, size_t length, char* output) {
-#ifdef SNAPPY
+#ifdef TF_USE_SNAPPY
   return snappy::RawUncompress(input, length, output);
 #else
   return false;
diff --git a/tensorflow/core/platform/windows/port.cc b/tensorflow/core/platform/windows/port.cc
index 85b53e07c4..e327d53949 100644
--- a/tensorflow/core/platform/windows/port.cc
+++ b/tensorflow/core/platform/windows/port.cc
@@ -20,7 +20,7 @@ limitations under the License.
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
-#ifdef SNAPPY
+#ifdef TF_USE_SNAPPY
 #include "snappy.h"
 #endif
 
@@ -118,7 +118,7 @@ void AdjustFilenameForLogging(string* filename) {
 }
 
 bool Snappy_Compress(const char* input, size_t length, string* output) {
-#ifdef SNAPPY
+#ifdef TF_USE_SNAPPY
   output->resize(snappy::MaxCompressedLength(length));
   size_t outlen;
   snappy::RawCompress(input, length, &(*output)[0], &outlen);
@@ -131,7 +131,7 @@ bool Snappy_Compress(const char* input, size_t length, string* output) {
 
 bool Snappy_GetUncompressedLength(const char* input, size_t length,
                                   size_t* result) {
-#ifdef SNAPPY
+#ifdef TF_USE_SNAPPY
   return snappy::GetUncompressedLength(input, length, result);
 #else
   return false;
@@ -139,7 +139,7 @@ bool Snappy_GetUncompressedLength(const char* input, size_t length,
 }
 
 bool Snappy_Uncompress(const char* input, size_t length, char* output) {
-#ifdef SNAPPY
+#ifdef TF_USE_SNAPPY
   return snappy::RawUncompress(input, length, output);
 #else
   return false;
-- 
GitLab


From f2250bfe85b59c9fba128aad9993417eca711d75 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 19 Oct 2017 16:24:56 -0700
Subject: [PATCH 587/909] Replace http://mirror.bazel.build with
 https://mirror.bazel.build

PiperOrigin-RevId: 172816169
---
 WORKSPACE                                     |  2 +-
 tensorflow/contrib/cmake/external/cub.cmake   |  2 +-
 tensorflow/contrib/cmake/external/gif.cmake   |  2 +-
 tensorflow/contrib/cmake/external/jpeg.cmake  |  2 +-
 tensorflow/contrib/cmake/external/lmdb.cmake  |  2 +-
 .../contrib/makefile/download_dependencies.sh |  8 +-
 tensorflow/workspace.bzl                      | 90 +++++++++----------
 7 files changed, 54 insertions(+), 54 deletions(-)

diff --git a/WORKSPACE b/WORKSPACE
index 1bf1069f88..b40913801b 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -5,7 +5,7 @@ http_archive(
     sha256 = "110fe68753413777944b473c25eed6368c4a0487cee23a7bac1b13cc49d3e257",
     strip_prefix = "rules_closure-4af89ef1db659eb41f110df189b67d4cf14073e1",
     urls = [
-        "http://mirror.bazel.build/github.com/bazelbuild/rules_closure/archive/4af89ef1db659eb41f110df189b67d4cf14073e1.tar.gz",
+        "https://mirror.bazel.build/github.com/bazelbuild/rules_closure/archive/4af89ef1db659eb41f110df189b67d4cf14073e1.tar.gz",
         "https://github.com/bazelbuild/rules_closure/archive/4af89ef1db659eb41f110df189b67d4cf14073e1.tar.gz",  # 2017-08-28
     ],
 )
diff --git a/tensorflow/contrib/cmake/external/cub.cmake b/tensorflow/contrib/cmake/external/cub.cmake
index d98579d207..e03026b1b0 100644
--- a/tensorflow/contrib/cmake/external/cub.cmake
+++ b/tensorflow/contrib/cmake/external/cub.cmake
@@ -14,7 +14,7 @@
 # ==============================================================================
 include (ExternalProject)
 
-set(cub_URL http://mirror.bazel.build/github.com/NVlabs/cub/archive/1.7.3.zip)
+set(cub_URL https://mirror.bazel.build/github.com/NVlabs/cub/archive/1.7.3.zip)
 set(cub_HASH SHA256=b7ead9e291d34ffa8074243541c1380d63be63f88de23de8ee548db573b72ebe)
 set(cub_BUILD ${CMAKE_CURRENT_BINARY_DIR}/cub/src/cub)
 set(cub_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/cub/src/cub)
diff --git a/tensorflow/contrib/cmake/external/gif.cmake b/tensorflow/contrib/cmake/external/gif.cmake
index 5cb719b878..3d53c51fff 100644
--- a/tensorflow/contrib/cmake/external/gif.cmake
+++ b/tensorflow/contrib/cmake/external/gif.cmake
@@ -15,7 +15,7 @@
 include (ExternalProject)
 
 set(gif_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/external/gif_archive/giflib-5.1.4/)
-set(gif_URL http://mirror.bazel.build/ufpr.dl.sourceforge.net/project/giflib/giflib-5.1.4.tar.gz)
+set(gif_URL https://mirror.bazel.build/ufpr.dl.sourceforge.net/project/giflib/giflib-5.1.4.tar.gz)
 set(gif_HASH SHA256=34a7377ba834397db019e8eb122e551a49c98f49df75ec3fcc92b9a794a4f6d1)
 set(gif_INSTALL ${CMAKE_BINARY_DIR}/gif/install)
 set(gif_BUILD ${CMAKE_BINARY_DIR}/gif/src/gif)
diff --git a/tensorflow/contrib/cmake/external/jpeg.cmake b/tensorflow/contrib/cmake/external/jpeg.cmake
index 058f554b8f..d9a165e856 100644
--- a/tensorflow/contrib/cmake/external/jpeg.cmake
+++ b/tensorflow/contrib/cmake/external/jpeg.cmake
@@ -15,7 +15,7 @@
 include (ExternalProject)
 
 set(jpeg_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/external/jpeg_archive)
-set(jpeg_URL http://mirror.bazel.build/www.ijg.org/files/jpegsrc.v9a.tar.gz)
+set(jpeg_URL https://mirror.bazel.build/www.ijg.org/files/jpegsrc.v9a.tar.gz)
 set(jpeg_HASH SHA256=3a753ea48d917945dd54a2d97de388aa06ca2eb1066cbfdc6652036349fe05a7)
 set(jpeg_BUILD ${CMAKE_CURRENT_BINARY_DIR}/jpeg/src/jpeg)
 set(jpeg_INSTALL ${CMAKE_CURRENT_BINARY_DIR}/jpeg/install)
diff --git a/tensorflow/contrib/cmake/external/lmdb.cmake b/tensorflow/contrib/cmake/external/lmdb.cmake
index 28ec833bab..79971b7cfc 100644
--- a/tensorflow/contrib/cmake/external/lmdb.cmake
+++ b/tensorflow/contrib/cmake/external/lmdb.cmake
@@ -15,7 +15,7 @@
 include (ExternalProject)
 
 set(lmdb_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/external/lmdb)
-set(lmdb_URL http://mirror.bazel.build/github.com/LMDB/lmdb/archive/LMDB_0.9.19.tar.gz)
+set(lmdb_URL https://mirror.bazel.build/github.com/LMDB/lmdb/archive/LMDB_0.9.19.tar.gz)
 set(lmdb_HASH SHA256=108532fb94c6f227558d45be3f3347b52539f0f58290a7bb31ec06c462d05326)
 set(lmdb_BUILD ${CMAKE_BINARY_DIR}/lmdb/src/lmdb)
 set(lmdb_INSTALL ${CMAKE_BINARY_DIR}/lmdb/install)
diff --git a/tensorflow/contrib/makefile/download_dependencies.sh b/tensorflow/contrib/makefile/download_dependencies.sh
index 39c89628d9..f0b9658e3d 100755
--- a/tensorflow/contrib/makefile/download_dependencies.sh
+++ b/tensorflow/contrib/makefile/download_dependencies.sh
@@ -20,11 +20,11 @@ DOWNLOADS_DIR=tensorflow/contrib/makefile/downloads
 BZL_FILE_PATH=tensorflow/workspace.bzl
 
 EIGEN_URL="$(grep -o 'http.*bitbucket.org/eigen/eigen/get/.*tar\.gz' "${BZL_FILE_PATH}" | grep -v bazel-mirror | head -n1)"
-GEMMLOWP_URL="$(grep -o 'http://mirror.bazel.build/github.com/google/gemmlowp/.*zip' "${BZL_FILE_PATH}" | head -n1)"
+GEMMLOWP_URL="$(grep -o 'https://mirror.bazel.build/github.com/google/gemmlowp/.*zip' "${BZL_FILE_PATH}" | head -n1)"
 GOOGLETEST_URL="https://github.com/google/googletest/archive/release-1.8.0.tar.gz"
-NSYNC_URL="$(grep -o 'http://mirror.bazel.build/github.com/google/nsync/.*tar\.gz' "${BZL_FILE_PATH}" | head -n1)"
-PROTOBUF_URL="$(grep -o 'http://mirror.bazel.build/github.com/google/protobuf/.*tar\.gz' "${BZL_FILE_PATH}" | head -n1)"
-RE2_URL="$(grep -o 'http://mirror.bazel.build/github.com/google/re2/.*tar\.gz' "${BZL_FILE_PATH}" | head -n1)"
+NSYNC_URL="$(grep -o 'https://mirror.bazel.build/github.com/google/nsync/.*tar\.gz' "${BZL_FILE_PATH}" | head -n1)"
+PROTOBUF_URL="$(grep -o 'https://mirror.bazel.build/github.com/google/protobuf/.*tar\.gz' "${BZL_FILE_PATH}" | head -n1)"
+RE2_URL="$(grep -o 'https://mirror.bazel.build/github.com/google/re2/.*tar\.gz' "${BZL_FILE_PATH}" | head -n1)"
 FFT2D_URL="$(grep -o 'http.*fft\.tgz' "${BZL_FILE_PATH}" | grep -v bazel-mirror | head -n1)"
 
 # TODO(petewarden): Some new code in Eigen triggers a clang bug with iOS arm64,
diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 54559edbea..a863aa18dd 100644
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -157,7 +157,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   mkl_repository(
       name = "mkl",
       urls = [
-          "http://mirror.bazel.build/github.com/01org/mkl-dnn/releases/download/v0.9/mklml_lnx_2018.0.20170720.tgz",
+          "https://mirror.bazel.build/github.com/01org/mkl-dnn/releases/download/v0.9/mklml_lnx_2018.0.20170720.tgz",
           # "https://github.com/01org/mkl-dnn/releases/download/v0.9/mklml_lnx_2018.0.20170720.tgz",
       ],
       sha256 = "57ba56c4c243f403ff78f417ff854ef50b9eddf4a610a917b7c95e7fa8553a4b",
@@ -174,7 +174,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       name = "mkl_dnn",
       urls = [
           "https://github.com/01org/mkl-dnn/archive/b01e3a55a07be62172e713bcd2644c5176360212.tar.gz",
-          "http://mirror.bazel.build/github.com/01org/mkl-dnn/archive/b01e3a55a07be62172e713bcd2644c5176360212.tar.gz",
+          "https://mirror.bazel.build/github.com/01org/mkl-dnn/archive/b01e3a55a07be62172e713bcd2644c5176360212.tar.gz",
       ],
       sha256 = "0d529ad4c49dc799e6df07c2b88b115d0668735da15fb3b3862d28d33fa68165",
       strip_prefix = "mkl-dnn-b01e3a55a07be62172e713bcd2644c5176360212",
@@ -185,7 +185,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       name = "eigen_archive",
       urls = [
           "https://bitbucket.org/eigen/eigen/get/429aa5254200.tar.gz",
-          "http://mirror.bazel.build/bitbucket.org/eigen/eigen/get/429aa5254200.tar.gz",
+          "https://mirror.bazel.build/bitbucket.org/eigen/eigen/get/429aa5254200.tar.gz",
       ],
       sha256 = "61d8b6fc4279dd1dda986fb1677d15e3d641c07a3ea5abe255790b1f0c0c14e9",
       strip_prefix = "eigen-eigen-429aa5254200",
@@ -198,7 +198,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       sha256 = "970285762565c7890c6c087d262b0a18286e7d0384f13a37786d8521773bc969",
       strip_prefix = "tools-0e906ebc527eab1cdbf7adabff5b474da9562e9f/arm-bcm2708/arm-rpi-4.9.3-linux-gnueabihf",
       urls = [
-          "http://mirror.bazel.build/github.com/raspberrypi/tools/archive/0e906ebc527eab1cdbf7adabff5b474da9562e9f.tar.gz",
+          "https://mirror.bazel.build/github.com/raspberrypi/tools/archive/0e906ebc527eab1cdbf7adabff5b474da9562e9f.tar.gz",
           # "https://github.com/raspberrypi/tools/archive/0e906ebc527eab1cdbf7adabff5b474da9562e9f.tar.gz",
       ],
   )
@@ -206,7 +206,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   native.new_http_archive(
       name = "libxsmm_archive",
       urls = [
-          "http://mirror.bazel.build/github.com/hfp/libxsmm/archive/1.8.1.tar.gz",
+          "https://mirror.bazel.build/github.com/hfp/libxsmm/archive/1.8.1.tar.gz",
           # "https://github.com/hfp/libxsmm/archive/1.8.1.tar.gz",
       ],
       sha256 = "2ade869c3f42f23b5263c7d594aa3c7e5e61ac6a3afcaf5d6e42899d2a7986ce",
@@ -222,7 +222,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   native.new_http_archive(
       name = "ortools_archive",
       urls = [
-          "http://mirror.bazel.build/github.com/google/or-tools/archive/253f7955c6a1fd805408fba2e42ac6d45b312d15.tar.gz",
+          "https://mirror.bazel.build/github.com/google/or-tools/archive/253f7955c6a1fd805408fba2e42ac6d45b312d15.tar.gz",
           # "https://github.com/google/or-tools/archive/253f7955c6a1fd805408fba2e42ac6d45b312d15.tar.gz",
       ],
       sha256 = "932075525642b04ac6f1b50589f1df5cd72ec2f448b721fd32234cf183f0e755",
@@ -233,7 +233,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   native.http_archive(
       name = "com_googlesource_code_re2",
       urls = [
-          "http://mirror.bazel.build/github.com/google/re2/archive/b94b7cd42e9f02673cd748c1ac1d16db4052514c.tar.gz",
+          "https://mirror.bazel.build/github.com/google/re2/archive/b94b7cd42e9f02673cd748c1ac1d16db4052514c.tar.gz",
           # "https://github.com/google/re2/archive/b94b7cd42e9f02673cd748c1ac1d16db4052514c.tar.gz",
       ],
       sha256 = "bd63550101e056427c9e7ff12a408c1c8b74e9803f393ca916b2926fc2c4906f",
@@ -243,7 +243,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   native.http_archive(
       name = "gemmlowp",
       urls = [
-          "http://mirror.bazel.build/github.com/google/gemmlowp/archive/010bb3e71a26ca1d0884a167081d092b43563996.zip"
+          "https://mirror.bazel.build/github.com/google/gemmlowp/archive/010bb3e71a26ca1d0884a167081d092b43563996.zip"
           # "https://github.com/google/gemmlowp/archive/010bb3e71a26ca1d0884a167081d092b43563996.zip",
       ],
       sha256 = "dd2557072bde12141419cb8320a9c25e6ec41a8ae53c2ac78c076a347bb46d9d",
@@ -253,7 +253,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   native.new_http_archive(
       name = "farmhash_archive",
       urls = [
-          "http://mirror.bazel.build/github.com/google/farmhash/archive/816a4ae622e964763ca0862d9dbd19324a1eaf45.tar.gz",
+          "https://mirror.bazel.build/github.com/google/farmhash/archive/816a4ae622e964763ca0862d9dbd19324a1eaf45.tar.gz",
           # "https://github.com/google/farmhash/archive/816a4ae622e964763ca0862d9dbd19324a1eaf45.tar.gz",
       ],
       sha256 = "6560547c63e4af82b0f202cb710ceabb3f21347a4b996db565a411da5b17aba0",
@@ -269,7 +269,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   native.new_http_archive(
       name = "highwayhash",
       urls = [
-          "http://mirror.bazel.build/github.com/google/highwayhash/archive/dfcb97ca4fe9277bf9dc1802dd979b071896453b.tar.gz",
+          "https://mirror.bazel.build/github.com/google/highwayhash/archive/dfcb97ca4fe9277bf9dc1802dd979b071896453b.tar.gz",
           # "https://github.com/google/highwayhash/archive/dfcb97ca4fe9277bf9dc1802dd979b071896453b.tar.gz",
       ],
       sha256 = "0f30a15b1566d93f146c8d149878a06e91d9bb7ec2cfd76906df62a82be4aac9",
@@ -280,7 +280,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   native.new_http_archive(
       name = "nasm",
       urls = [
-          "http://mirror.bazel.build/www.nasm.us/pub/nasm/releasebuilds/2.12.02/nasm-2.12.02.tar.bz2",
+          "https://mirror.bazel.build/www.nasm.us/pub/nasm/releasebuilds/2.12.02/nasm-2.12.02.tar.bz2",
           "http://pkgs.fedoraproject.org/repo/pkgs/nasm/nasm-2.12.02.tar.bz2/d15843c3fb7db39af80571ee27ec6fad/nasm-2.12.02.tar.bz2",
       ],
       sha256 = "00b0891c678c065446ca59bcee64719d0096d54d6886e6e472aeee2e170ae324",
@@ -291,7 +291,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   temp_workaround_http_archive(
       name = "jpeg",
       urls = [
-          "http://mirror.bazel.build/github.com/libjpeg-turbo/libjpeg-turbo/archive/1.5.1.tar.gz",
+          "https://mirror.bazel.build/github.com/libjpeg-turbo/libjpeg-turbo/archive/1.5.1.tar.gz",
           # "https://github.com/libjpeg-turbo/libjpeg-turbo/archive/1.5.1.tar.gz",
       ],
       sha256 = "c15a9607892113946379ccea3ca8b85018301b200754f209453ab21674268e77",
@@ -303,7 +303,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   native.new_http_archive(
       name = "png_archive",
       urls = [
-          "http://mirror.bazel.build/github.com/glennrp/libpng/archive/v1.2.53.tar.gz",
+          "https://mirror.bazel.build/github.com/glennrp/libpng/archive/v1.2.53.tar.gz",
           # "https://github.com/glennrp/libpng/archive/v1.2.53.tar.gz",
       ],
       sha256 = "716c59c7dfc808a4c368f8ada526932be72b2fcea11dd85dc9d88b1df1dfe9c2",
@@ -314,7 +314,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   native.new_http_archive(
       name = "sqlite_archive",
       urls = [
-          "http://mirror.bazel.build/www.sqlite.org/2017/sqlite-amalgamation-3200000.zip",
+          "https://mirror.bazel.build/www.sqlite.org/2017/sqlite-amalgamation-3200000.zip",
           "http://www.sqlite.org/2017/sqlite-amalgamation-3200000.zip",
       ],
       sha256 = "208780b3616f9de0aeb50822b7a8f5482f6515193859e91ed61637be6ad74fd4",
@@ -325,7 +325,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   native.new_http_archive(
       name = "gif_archive",
       urls = [
-          "http://mirror.bazel.build/ufpr.dl.sourceforge.net/project/giflib/giflib-5.1.4.tar.gz",
+          "https://mirror.bazel.build/ufpr.dl.sourceforge.net/project/giflib/giflib-5.1.4.tar.gz",
           "http://pilotfiber.dl.sourceforge.net/project/giflib/giflib-5.1.4.tar.gz",
       ],
       sha256 = "34a7377ba834397db019e8eb122e551a49c98f49df75ec3fcc92b9a794a4f6d1",
@@ -336,7 +336,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   native.new_http_archive(
       name = "six_archive",
       urls = [
-          "http://mirror.bazel.build/pypi.python.org/packages/source/s/six/six-1.10.0.tar.gz",
+          "https://mirror.bazel.build/pypi.python.org/packages/source/s/six/six-1.10.0.tar.gz",
           "https://pypi.python.org/packages/source/s/six/six-1.10.0.tar.gz",
       ],
       sha256 = "105f8d68616f8248e24bf0e9372ef04d3cc10104f1980f54d57b2ce73a5ad56a",
@@ -347,7 +347,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   native.new_http_archive(
       name = "org_python_pypi_backports_weakref",
       urls = [
-          "http://mirror.bazel.build/pypi.python.org/packages/bc/cc/3cdb0a02e7e96f6c70bd971bc8a90b8463fda83e264fa9c5c1c98ceabd81/backports.weakref-1.0rc1.tar.gz",
+          "https://mirror.bazel.build/pypi.python.org/packages/bc/cc/3cdb0a02e7e96f6c70bd971bc8a90b8463fda83e264fa9c5c1c98ceabd81/backports.weakref-1.0rc1.tar.gz",
           "https://pypi.python.org/packages/bc/cc/3cdb0a02e7e96f6c70bd971bc8a90b8463fda83e264fa9c5c1c98ceabd81/backports.weakref-1.0rc1.tar.gz",
       ],
       sha256 = "8813bf712a66b3d8b85dc289e1104ed220f1878cf981e2fe756dfaabe9a82892",
@@ -358,7 +358,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   native.new_http_archive(
       name = "com_github_andreif_codegen",
       urls = [
-          "http://mirror.bazel.build/github.com/andreif/codegen/archive/1.0.tar.gz",
+          "https://mirror.bazel.build/github.com/andreif/codegen/archive/1.0.tar.gz",
           # "https://github.com/andreif/codegen/archive/1.0.tar.gz",
       ],
       sha256 = "2dadd04a2802de27e0fe5a19b76538f6da9d39ff244036afa00c1bba754de5ee",
@@ -371,7 +371,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       licenses = ["notice"],  # Python 2.0
       sha256_urls = {
           "b5556e921715ddb9242c076cae3963f483aa47266c5e37ea4c187f77cc79501c": [
-              "http://mirror.bazel.build/docs.python.org/2.7/_sources/license.txt",
+              "https://mirror.bazel.build/docs.python.org/2.7/_sources/license.txt",
               "https://docs.python.org/2.7/_sources/license.txt",
           ],
       },
@@ -387,7 +387,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   patched_http_archive(
       name = "protobuf_archive",
       urls = [
-          "http://mirror.bazel.build/github.com/google/protobuf/archive/b04e5cba356212e4e8c66c61bbe0c3a20537c5b9.tar.gz",
+          "https://mirror.bazel.build/github.com/google/protobuf/archive/b04e5cba356212e4e8c66c61bbe0c3a20537c5b9.tar.gz",
       ],
       sha256 = "e178a25c52efcb6b05988bdbeace4c0d3f2d2fe5b46696d1d9898875c3803d6a",
       strip_prefix = "protobuf-b04e5cba356212e4e8c66c61bbe0c3a20537c5b9",
@@ -410,7 +410,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   native.http_archive(
       name = "com_google_protobuf",
       urls = [
-          "http://mirror.bazel.build/github.com/google/protobuf/archive/0b059a3d8a8f8aa40dde7bea55edca4ec5dfea66.tar.gz",
+          "https://mirror.bazel.build/github.com/google/protobuf/archive/0b059a3d8a8f8aa40dde7bea55edca4ec5dfea66.tar.gz",
       ],
       sha256 = "6d43b9d223ce09e5d4ce8b0060cb8a7513577a35a64c7e3dad10f0703bf3ad93",
       strip_prefix = "protobuf-0b059a3d8a8f8aa40dde7bea55edca4ec5dfea66",
@@ -420,7 +420,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   native.http_archive(
       name = "com_google_protobuf_cc",
       urls = [
-          "http://mirror.bazel.build/github.com/google/protobuf/archive/0b059a3d8a8f8aa40dde7bea55edca4ec5dfea66.tar.gz",
+          "https://mirror.bazel.build/github.com/google/protobuf/archive/0b059a3d8a8f8aa40dde7bea55edca4ec5dfea66.tar.gz",
       ],
       sha256 = "6d43b9d223ce09e5d4ce8b0060cb8a7513577a35a64c7e3dad10f0703bf3ad93",
       strip_prefix = "protobuf-0b059a3d8a8f8aa40dde7bea55edca4ec5dfea66",
@@ -429,7 +429,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   native.http_archive(
       name = "nsync",
       urls = [
-          "http://mirror.bazel.build/github.com/google/nsync/archive/ad722c76c6e6653f66be2e1f69521b7f7517da55.tar.gz",
+          "https://mirror.bazel.build/github.com/google/nsync/archive/ad722c76c6e6653f66be2e1f69521b7f7517da55.tar.gz",
           # "https://github.com/google/nsync/archive/ad722c76c6e6653f66be2e1f69521b7f7517da55.tar.gz",
       ],
       sha256 = "7dd8ca49319f77e8226cd020a9210a525f88ac26e7041c59c95418223a1cdf55",
@@ -439,7 +439,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   native.http_archive(
       name = "com_google_googletest",
       urls = [
-          "http://mirror.bazel.build/github.com/google/googletest/archive/9816b96a6ddc0430671693df90192bbee57108b6.zip",
+          "https://mirror.bazel.build/github.com/google/googletest/archive/9816b96a6ddc0430671693df90192bbee57108b6.zip",
           # "https://github.com/google/googletest/archive/9816b96a6ddc0430671693df90192bbee57108b6.zip",
       ],
       sha256 = "9cbca84c4256bed17df2c8f4d00c912c19d247c11c9ba6647cd6dd5b5c996b8d",
@@ -449,7 +449,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   native.http_archive(
       name = "com_github_gflags_gflags",
       urls = [
-          "http://mirror.bazel.build/github.com/gflags/gflags/archive/f8a0efe03aa69b3336d8e228b37d4ccb17324b88.tar.gz",
+          "https://mirror.bazel.build/github.com/gflags/gflags/archive/f8a0efe03aa69b3336d8e228b37d4ccb17324b88.tar.gz",
           # "https://github.com/gflags/gflags/archive/f8a0efe03aa69b3336d8e228b37d4ccb17324b88.tar.gz",
       ],
       sha256 = "4d222fab8f1ede4709cdff417d15a1336f862d7334a81abf76d09c15ecf9acd1",
@@ -465,7 +465,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       name = "pcre",
       sha256 = "ccdf7e788769838f8285b3ee672ed573358202305ee361cfec7a4a4fb005bbc7",
       urls = [
-          "http://mirror.bazel.build/ftp.exim.org/pub/pcre/pcre-8.39.tar.gz",
+          "https://mirror.bazel.build/ftp.exim.org/pub/pcre/pcre-8.39.tar.gz",
           "http://ftp.exim.org/pub/pcre/pcre-8.39.tar.gz",
       ],
       strip_prefix = "pcre-8.39",
@@ -476,7 +476,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       name = "swig",
       sha256 = "58a475dbbd4a4d7075e5fe86d4e54c9edde39847cdb96a3053d87cb64a23a453",
       urls = [
-          "http://mirror.bazel.build/ufpr.dl.sourceforge.net/project/swig/swig/swig-3.0.8/swig-3.0.8.tar.gz",
+          "https://mirror.bazel.build/ufpr.dl.sourceforge.net/project/swig/swig/swig-3.0.8/swig-3.0.8.tar.gz",
           "http://ufpr.dl.sourceforge.net/project/swig/swig/swig-3.0.8/swig-3.0.8.tar.gz",
           "http://pilotfiber.dl.sourceforge.net/project/swig/swig/swig-3.0.8/swig-3.0.8.tar.gz",
       ],
@@ -488,7 +488,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       name = "curl",
       sha256 = "ff3e80c1ca6a068428726cd7dd19037a47cc538ce58ef61c59587191039b2ca6",
       urls = [
-          "http://mirror.bazel.build/curl.haxx.se/download/curl-7.49.1.tar.gz",
+          "https://mirror.bazel.build/curl.haxx.se/download/curl-7.49.1.tar.gz",
           "https://curl.haxx.se/download/curl-7.49.1.tar.gz",
       ],
       strip_prefix = "curl-7.49.1",
@@ -518,7 +518,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   patched_http_archive(
       name = "grpc",
       urls = [
-          "http://mirror.bazel.build/github.com/grpc/grpc/archive/781fd6f6ea03645a520cd5c675da67ab61f87e4b.tar.gz",
+          "https://mirror.bazel.build/github.com/grpc/grpc/archive/781fd6f6ea03645a520cd5c675da67ab61f87e4b.tar.gz",
           # "https://github.com/grpc/grpc/archive/781fd6f6ea03645a520cd5c675da67ab61f87e4b.tar.gz",
       ],
       sha256 = "2004635e6a078acfac8ffa71738397796be4f8fb72f572cc44ecee5d99511d9f",
@@ -542,7 +542,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       name = "linenoise",
       sha256 = "7f51f45887a3d31b4ce4fa5965210a5e64637ceac12720cfce7954d6a2e812f7",
       urls = [
-          "http://mirror.bazel.build/github.com/antirez/linenoise/archive/c894b9e59f02203dbe4e2be657572cf88c4230c3.tar.gz",
+          "https://mirror.bazel.build/github.com/antirez/linenoise/archive/c894b9e59f02203dbe4e2be657572cf88c4230c3.tar.gz",
           # "https://github.com/antirez/linenoise/archive/c894b9e59f02203dbe4e2be657572cf88c4230c3.tar.gz",
       ],
       strip_prefix = "linenoise-c894b9e59f02203dbe4e2be657572cf88c4230c3",
@@ -554,7 +554,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   temp_workaround_http_archive(
       name = "llvm",
       urls = [
-          "http://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/bb3c660e87f59abb665570a31b01ab125ec4c10e.tar.gz",
+          "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/bb3c660e87f59abb665570a31b01ab125ec4c10e.tar.gz",
           "https://github.com/llvm-mirror/llvm/archive/bb3c660e87f59abb665570a31b01ab125ec4c10e.tar.gz",
       ],
       sha256 = "caab6d7978e6771cb4e9b5b89607c5370de8aa642913c6c14e892468194c94e4",
@@ -566,7 +566,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   native.new_http_archive(
       name = "lmdb",
       urls = [
-          "http://mirror.bazel.build/github.com/LMDB/lmdb/archive/LMDB_0.9.19.tar.gz",
+          "https://mirror.bazel.build/github.com/LMDB/lmdb/archive/LMDB_0.9.19.tar.gz",
           # "https://github.com/LMDB/lmdb/archive/LMDB_0.9.19.tar.gz",
       ],
       sha256 = "108532fb94c6f227558d45be3f3347b52539f0f58290a7bb31ec06c462d05326",
@@ -577,7 +577,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   native.new_http_archive(
       name = "jsoncpp_git",
       urls = [
-          "http://mirror.bazel.build/github.com/open-source-parsers/jsoncpp/archive/11086dd6a7eba04289944367ca82cea71299ed70.tar.gz",
+          "https://mirror.bazel.build/github.com/open-source-parsers/jsoncpp/archive/11086dd6a7eba04289944367ca82cea71299ed70.tar.gz",
           # "https://github.com/open-source-parsers/jsoncpp/archive/11086dd6a7eba04289944367ca82cea71299ed70.tar.gz",
       ],
       sha256 = "07d34db40593d257324ec5fb9debc4dc33f29f8fb44e33a2eeb35503e61d0fe2",
@@ -593,7 +593,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   patched_http_archive(
       name = "boringssl",
       urls = [
-          "http://mirror.bazel.build/github.com/google/boringssl/archive/e3860009a091cd1bd2bc189cdbc3c6d095abde84.tar.gz",
+          "https://mirror.bazel.build/github.com/google/boringssl/archive/e3860009a091cd1bd2bc189cdbc3c6d095abde84.tar.gz",
           # "https://github.com/google/boringssl/archive/e3860009a091cd1bd2bc189cdbc3c6d095abde84.tar.gz",  # 2017-07-07
       ],
       sha256 = "02f5950f93c4fd3691771c07c9d04cf2999ab01383ff99da345249e93b0fcfb2",
@@ -605,7 +605,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   native.new_http_archive(
       name = "zlib_archive",
       urls = [
-          "http://mirror.bazel.build/zlib.net/zlib-1.2.8.tar.gz",
+          "https://mirror.bazel.build/zlib.net/zlib-1.2.8.tar.gz",
           "http://zlib.net/fossils/zlib-1.2.8.tar.gz",
       ],
       sha256 = "36658cb768a54c1d4dec43c3116c27ed893e88b02ecfcb44f2166f9c0b7f2a0d",
@@ -621,7 +621,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   native.new_http_archive(
       name = "fft2d",
       urls = [
-          "http://mirror.bazel.build/www.kurims.kyoto-u.ac.jp/~ooura/fft.tgz",
+          "https://mirror.bazel.build/www.kurims.kyoto-u.ac.jp/~ooura/fft.tgz",
           "http://www.kurims.kyoto-u.ac.jp/~ooura/fft.tgz",
       ],
       sha256 = "52bb637c70b971958ec79c9c8752b1df5ff0218a4db4510e60826e0cb79b5296",
@@ -631,7 +631,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   temp_workaround_http_archive(
       name = "snappy",
       urls = [
-          "http://mirror.bazel.build/github.com/google/snappy/archive/1.1.4.tar.gz",
+          "https://mirror.bazel.build/github.com/google/snappy/archive/1.1.4.tar.gz",
           # "https://github.com/google/snappy/archive/1.1.4.tar.gz",
       ],
       sha256 = "2f7504c73d85bac842e893340333be8cb8561710642fc9562fccdd9d2c3fcc94",
@@ -643,7 +643,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   temp_workaround_http_archive(
       name = "nccl_archive",
       urls = [
-          "http://mirror.bazel.build/github.com/nvidia/nccl/archive/03d856977ecbaac87e598c0c4bafca96761b9ac7.tar.gz",
+          "https://mirror.bazel.build/github.com/nvidia/nccl/archive/03d856977ecbaac87e598c0c4bafca96761b9ac7.tar.gz",
           # "https://github.com/nvidia/nccl/archive/03d856977ecbaac87e598c0c4bafca96761b9ac7.tar.gz",
       ],
       sha256 = "2ca86fb6179ecbff789cc67c836139c1bbc0324ed8c04643405a30bf26325176",
@@ -668,7 +668,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       name = "junit",
       jar_sha256 = "59721f0805e223d84b90677887d9ff567dc534d7c502ca903c0c2b17f05c116a",
       jar_urls = [
-          "http://mirror.bazel.build/repo1.maven.org/maven2/junit/junit/4.12/junit-4.12.jar",
+          "https://mirror.bazel.build/repo1.maven.org/maven2/junit/junit/4.12/junit-4.12.jar",
           "http://repo1.maven.org/maven2/junit/junit/4.12/junit-4.12.jar",
           "http://maven.ibiblio.org/maven2/junit/junit/4.12/junit-4.12.jar",
       ],
@@ -681,7 +681,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       name = "org_hamcrest_core",
       jar_sha256 = "66fdef91e9739348df7a096aa384a5685f4e875584cce89386a7a47251c4d8e9",
       jar_urls = [
-          "http://mirror.bazel.build/repo1.maven.org/maven2/org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar",
+          "https://mirror.bazel.build/repo1.maven.org/maven2/org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar",
           "http://repo1.maven.org/maven2/org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar",
           "http://maven.ibiblio.org/maven2/org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar",
       ],
@@ -692,7 +692,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   temp_workaround_http_archive(
       name = "jemalloc",
       urls = [
-          "http://mirror.bazel.build/github.com/jemalloc/jemalloc/archive/4.4.0.tar.gz",
+          "https://mirror.bazel.build/github.com/jemalloc/jemalloc/archive/4.4.0.tar.gz",
           # "https://github.com/jemalloc/jemalloc/archive/4.4.0.tar.gz",
       ],
       sha256 = "3c8f25c02e806c3ce0ab5fb7da1817f89fc9732709024e2a81b6b82f7cc792a8",
@@ -704,7 +704,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   native.new_http_archive(
       name = "com_google_pprof",
       urls = [
-          "http://mirror.bazel.build/github.com/google/pprof/archive/c0fb62ec88c411cc91194465e54db2632845b650.tar.gz",
+          "https://mirror.bazel.build/github.com/google/pprof/archive/c0fb62ec88c411cc91194465e54db2632845b650.tar.gz",
           # "https://github.com/google/pprof/archive/c0fb62ec88c411cc91194465e54db2632845b650.tar.gz",
       ],
       sha256 = "e0928ca4aa10ea1e0551e2d7ce4d1d7ea2d84b2abbdef082b0da84268791d0c4",
@@ -715,7 +715,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   native.new_http_archive(
       name = "cub_archive",
       urls = [
-          "http://mirror.bazel.build/github.com/NVlabs/cub/archive/1.7.3.zip",
+          "https://mirror.bazel.build/github.com/NVlabs/cub/archive/1.7.3.zip",
           # "https://github.com/NVlabs/cub/archive/1.7.3.zip",
       ],
       sha256 = "b7ead9e291d34ffa8074243541c1380d63be63f88de23de8ee548db573b72ebe",
@@ -732,7 +732,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       name = "cython",
       sha256 = "6dcd30b5ceb887b2b965ee7ceb82ea3acb5f0642fe2206c7636b45acea4798e5",
       urls = [
-          "http://mirror.bazel.build/github.com/cython/cython/archive/3732784c45cfb040a5b0936951d196f83a12ea17.tar.gz",
+          "https://mirror.bazel.build/github.com/cython/cython/archive/3732784c45cfb040a5b0936951d196f83a12ea17.tar.gz",
           "https://github.com/cython/cython/archive/3732784c45cfb040a5b0936951d196f83a12ea17.tar.gz",
       ],
       strip_prefix = "cython-3732784c45cfb040a5b0936951d196f83a12ea17",
@@ -742,7 +742,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   native.http_archive(
       name = "bazel_toolchains",
       urls = [
-          "http://mirror.bazel.build/github.com/bazelbuild/bazel-toolchains/archive/b2b4b38433bf2d1159360855ea4004378308711b.tar.gz",
+          "https://mirror.bazel.build/github.com/bazelbuild/bazel-toolchains/archive/b2b4b38433bf2d1159360855ea4004378308711b.tar.gz",
           # "https://github.com/bazelbuild/bazel-toolchains/archive/b2b4b38433bf2d1159360855ea4004378308711b.tar.gz",
       ],
       sha256 = "46187270ca04ff8109980f45c3438fabfe48695e163789096eb82ee097ffe685",
-- 
GitLab


From e0e4f693978dcaf5bf4ecbc18e6926bdf33b2870 Mon Sep 17 00:00:00 2001
From: Eugene Brevdo <ebrevdo@google.com>
Date: Thu, 19 Oct 2017 16:28:57 -0700
Subject: [PATCH 588/909] [tf.contrib.seq2seq] Reserve -1s in GatherTree for
 error states.

GatherTree now emits end_token after the first decoded end_token in the path,
instead of -1s at the end of each sequence.

PiperOrigin-RevId: 172816652
---
 .../seq2seq/kernels/beam_search_ops.cc        |  7 +++--
 .../seq2seq/kernels/beam_search_ops_gpu.cu.cc | 25 +++++++++------
 .../contrib/seq2seq/ops/beam_search_ops.cc    | 11 ++++---
 .../kernel_tests/beam_search_ops_test.py      | 31 ++++++++++---------
 4 files changed, 45 insertions(+), 29 deletions(-)

diff --git a/tensorflow/contrib/seq2seq/kernels/beam_search_ops.cc b/tensorflow/contrib/seq2seq/kernels/beam_search_ops.cc
index 95273e2b33..64973ccccd 100644
--- a/tensorflow/contrib/seq2seq/kernels/beam_search_ops.cc
+++ b/tensorflow/contrib/seq2seq/kernels/beam_search_ops.cc
@@ -112,7 +112,7 @@ struct GatherTree<CPUDevice, int32> {
     const int32 max_time = parent_ids.dimension(0);
     const int32 batch_size = parent_ids.dimension(1);
     const int32 beam_width = parent_ids.dimension(2);
-    beams.setConstant(-1);
+    beams.setConstant(end_token);
 
     auto DoWork = [&, ctx, end_token](int start_batch_beam,
                                       int limit_batch_beam) {
@@ -138,10 +138,13 @@ struct GatherTree<CPUDevice, int32> {
           beams(level, batch, beam) = step_ids(level, batch, parent);
           parent = parent_ids(level, batch, parent);
         }
+        // Not necessary when using a BeamSearchDecoder, but necessary
+        // when a user feeds in possibly broken trajectory (i.e., non-eos
+        // entries in a beam following eos entries).
         bool finished = false;
         for (int32 time = 0; time < max_seq_len_b; ++time) {
           if (finished) {
-            beams(time, batch, beam) = -1;
+            beams(time, batch, beam) = end_token;
           } else if (beams(time, batch, beam) == end_token) {
             finished = true;
           }
diff --git a/tensorflow/contrib/seq2seq/kernels/beam_search_ops_gpu.cu.cc b/tensorflow/contrib/seq2seq/kernels/beam_search_ops_gpu.cu.cc
index e71efc48ce..bc28d492fe 100644
--- a/tensorflow/contrib/seq2seq/kernels/beam_search_ops_gpu.cu.cc
+++ b/tensorflow/contrib/seq2seq/kernels/beam_search_ops_gpu.cu.cc
@@ -46,24 +46,31 @@ __global__ void GatherTreeOpKernel(const int32 batch_size, const int32 max_time,
     const int32 initial_beam_ix = GET_IX(max_seq_len_b - 1, beam);
     beams[initial_beam_ix] = ldg(step_ids + initial_beam_ix);
     int32 parent = ldg(parent_ids + initial_beam_ix);
+    bool found_bad = false;
     for (int32 level = max_seq_len_b - 2; level >= 0; --level) {
       const int32 level_beam_ix = GET_IX(level, beam);
       const int32 level_parent_ix = GET_IX(level, parent);
       if (parent < 0 || parent > beam_width) {
         beams[level_beam_ix] = -1;
         parent = -1;
+        found_bad = true;
       } else {
         beams[level_beam_ix] = ldg(step_ids + level_parent_ix);
         parent = ldg(parent_ids + level_parent_ix);
       }
     }
-    bool finished = false;
-    for (int32 time = 0; time < max_seq_len_b; ++time) {
-      const int32 level_beam_ix = GET_IX(time, beam);
-      if (finished) {
-        beams[level_beam_ix] = -1;
-      } else if (beams[level_beam_ix] == end_token) {
-        finished = true;
+    // Not necessary when using a BeamSearchDecoder, but necessary
+    // when a user feeds in possibly broken trajectory (i.e., non-eos
+    // entries in a beam following eos entries).
+    if (!found_bad) {
+      bool finished = false;
+      for (int32 time = 0; time < max_seq_len_b; ++time) {
+        const int32 level_beam_ix = GET_IX(time, beam);
+        if (finished) {
+          beams[level_beam_ix] = end_token;
+        } else if (beams[level_beam_ix] == end_token) {
+          finished = true;
+        }
       }
     }
 #undef GET_IX
@@ -80,8 +87,8 @@ struct GatherTree<GPUDevice, T> {
     const int32 max_time = parent_ids.dimension(0);
     const int32 batch_size = parent_ids.dimension(1);
     const int32 beam_width = parent_ids.dimension(2);
-    // First kernel launch to zero things out
-    beams.device(d) = beams.constant(T(-1));
+    // First kernel launch to "zero" things out
+    beams.device(d) = beams.constant(end_token);
 
     CudaLaunchConfig config = GetCudaLaunchConfig(batch_size * beam_width, d);
     // clang-format off
diff --git a/tensorflow/contrib/seq2seq/ops/beam_search_ops.cc b/tensorflow/contrib/seq2seq/ops/beam_search_ops.cc
index 231504bfbb..71539b6f59 100644
--- a/tensorflow/contrib/seq2seq/ops/beam_search_ops.cc
+++ b/tensorflow/contrib/seq2seq/ops/beam_search_ops.cc
@@ -53,11 +53,14 @@ REGISTER_OP("GatherTree")
     .Doc(R"doc(
 Calculates the full beams from the per-step ids and parent beam ids.
 
-This op implements the following mathematical equations:
+On CPU, if an out of bound parent id is found, an error is returned.
+On GPU, if an out of bound parent id is found, a -1 is stored in the
+corresponding output value and the execution for that beam returns early.
 
-```python
-TODO(ebrevdo): fill in
-```
+For a given beam, past the time step containing the first decoded `end_token`
+all values are filled in with `end_token`.
+
+TODO(ebrevdo): fill in the remainder of this docstring.
 
 step_ids: `[max_time, batch_size, beam_width]`.
 parent_ids: `[max_time, batch_size, beam_width]`.
diff --git a/tensorflow/contrib/seq2seq/python/kernel_tests/beam_search_ops_test.py b/tensorflow/contrib/seq2seq/python/kernel_tests/beam_search_ops_test.py
index f301314872..277c5b6ef7 100644
--- a/tensorflow/contrib/seq2seq/python/kernel_tests/beam_search_ops_test.py
+++ b/tensorflow/contrib/seq2seq/python/kernel_tests/beam_search_ops_test.py
@@ -36,24 +36,26 @@ class GatherTreeTest(test.TestCase):
 
   def testGatherTreeOne(self):
     # (max_time = 4, batch_size = 1, beams = 3)
+    end_token = 10
     step_ids = _transpose_batch_time(
         [[[1, 2, 3], [4, 5, 6], [7, 8, 9], [-1, -1, -1]]])
     parent_ids = _transpose_batch_time(
         [[[0, 0, 0], [0, 1, 1], [2, 1, 2], [-1, -1, -1]]])
     max_sequence_lengths = [3]
-    expected_result = _transpose_batch_time(
-        [[[2, 2, 2], [6, 5, 6], [7, 8, 9], [-1, -1, -1]]])
+    expected_result = _transpose_batch_time([[[2, 2, 2], [6, 5, 6], [7, 8, 9],
+                                              [10, 10, 10]]])
     beams = beam_search_ops.gather_tree(
         step_ids=step_ids,
         parent_ids=parent_ids,
         max_sequence_lengths=max_sequence_lengths,
-        end_token=10)
+        end_token=end_token)
     with self.test_session(use_gpu=True):
       self.assertAllEqual(expected_result, beams.eval())
 
   def testBadParentValuesOnCPU(self):
     # (batch_size = 1, max_time = 4, beams = 3)
     # bad parent in beam 1 time 1
+    end_token = 10
     step_ids = _transpose_batch_time(
         [[[1, 2, 3], [4, 5, 6], [7, 8, 9], [-1, -1, -1]]])
     parent_ids = _transpose_batch_time(
@@ -64,7 +66,7 @@ class GatherTreeTest(test.TestCase):
           step_ids=step_ids,
           parent_ids=parent_ids,
           max_sequence_lengths=max_sequence_lengths,
-          end_token=10)
+          end_token=end_token)
     with self.test_session():
       with self.assertRaisesOpError(
           r"parent id -1 at \(batch, time, beam\) == \(0, 0, 1\)"):
@@ -77,19 +79,20 @@ class GatherTreeTest(test.TestCase):
       return
     # (max_time = 4, batch_size = 1, beams = 3)
     # bad parent in beam 1 time 1; appears as a negative index at time 0
+    end_token = 10
     step_ids = _transpose_batch_time(
         [[[1, 2, 3], [4, 5, 6], [7, 8, 9], [-1, -1, -1]]])
     parent_ids = _transpose_batch_time(
         [[[0, 0, 0], [0, -1, 1], [2, 1, 2], [-1, -1, -1]]])
     max_sequence_lengths = [3]
-    expected_result = _transpose_batch_time(
-        [[[2, -1, 2], [6, 5, 6], [7, 8, 9], [-1, -1, -1]]])
+    expected_result = _transpose_batch_time([[[2, -1, 2], [6, 5, 6], [7, 8, 9],
+                                              [10, 10, 10]]])
     with ops.device("/device:GPU:0"):
       beams = beam_search_ops.gather_tree(
           step_ids=step_ids,
           parent_ids=parent_ids,
           max_sequence_lengths=max_sequence_lengths,
-          end_token=10)
+          end_token=end_token)
     with self.test_session(use_gpu=True):
       self.assertAllEqual(expected_result, beams.eval())
 
@@ -115,24 +118,24 @@ class GatherTreeTest(test.TestCase):
       self.assertEqual((max_time, batch_size, beam_width), beams.shape)
       beams_value = beams.eval()
       for b in range(batch_size):
-        # Past max_sequence_lengths[b], we emit all -1s.
+        # Past max_sequence_lengths[b], we emit all end tokens.
         b_value = beams_value[max_sequence_lengths[b]:, b, :]
-        self.assertAllClose(b_value, -1. * np.ones_like(b_value))
+        self.assertAllClose(b_value, end_token * np.ones_like(b_value))
       for batch, beam in itertools.product(
           range(batch_size), range(beam_width)):
         v = np.squeeze(beams_value[:, batch, beam])
         if end_token in v:
+          found_bad = np.where(v == -1)[0]
+          self.assertEqual(0, len(found_bad))
           found = np.where(v == end_token)[0]
-          # Should be up to 1 instance of end_token per beam.
-          self.assertEqual(len(found), 1)
-          found = found[0]
+          found = found[0]  # First occurrence of end_token.
           # If an end_token is found, everything before it should be a
           # valid id and everything after it should be -1.
           if found > 0:
             self.assertAllEqual(
                 v[:found - 1] >= 0, np.ones_like(v[:found - 1], dtype=bool))
-          self.assertAllClose(
-              v[found + 1:], -1 * np.ones_like(v[found + 1:]))
+          self.assertAllClose(v[found + 1:],
+                              end_token * np.ones_like(v[found + 1:]))
 
 
 if __name__ == "__main__":
-- 
GitLab


From 2977dccc96c343ca85cb00b50672b36c99656532 Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Thu, 19 Oct 2017 16:42:14 -0700
Subject: [PATCH 589/909] Context-specific C API to set options other than
 configproto (still unused)

PiperOrigin-RevId: 172818175
---
 tensorflow/c/eager/c_api.cc         | 17 ++++++++++----
 tensorflow/c/eager/c_api.h          | 20 ++++++++++++++--
 tensorflow/c/eager/c_api_internal.h |  4 ++++
 tensorflow/c/eager/c_api_test.cc    | 36 ++++++++++++++---------------
 tensorflow/python/eager/context.py  | 16 ++++++++-----
 tensorflow/python/pywrap_tfe.i      | 16 ++++++++++++-
 6 files changed, 78 insertions(+), 31 deletions(-)

diff --git a/tensorflow/c/eager/c_api.cc b/tensorflow/c/eager/c_api.cc
index 514a4010bc..334c02bff9 100644
--- a/tensorflow/c/eager/c_api.cc
+++ b/tensorflow/c/eager/c_api.cc
@@ -54,9 +54,18 @@ string DeviceName(tensorflow::Device* d) {
 
 extern "C" {
 
-TFE_Context* TFE_NewContext(const TF_SessionOptions* opts, TF_Status* status) {
+TFE_ContextOptions* TFE_NewContextOptions() { return new TFE_ContextOptions; }
+
+void TFE_ContextOptionsSetConfig(TFE_ContextOptions* options, const void* proto,
+                                 size_t proto_len, TF_Status* status) {
+  TF_SetConfig(&options->session_options, proto, proto_len, status);
+}
+
+void TFE_DeleteContextOptions(TFE_ContextOptions* options) { delete options; }
+
+TFE_Context* TFE_NewContext(const TFE_ContextOptions* opts, TF_Status* status) {
   TF_Graph* graph = TF_NewGraph();
-  TF_Session* session = TF_NewSession(graph, opts, status);
+  TF_Session* session = TF_NewSession(graph, &opts->session_options, status);
   if (status->status.ok()) {
     if (session->device_mgr == nullptr || session->devices.empty()) {
       status->status = tensorflow::errors::InvalidArgument(
@@ -72,8 +81,8 @@ TFE_Context* TFE_NewContext(const TF_SessionOptions* opts, TF_Status* status) {
 
   TFE_Context* ret = new TFE_Context(session);
   ret->pflr.reset(new tensorflow::ProcessFunctionLibraryRuntime(
-      ret->session->device_mgr, opts->options.env, TF_GRAPH_DEF_VERSION,
-      &ret->func_lib_def, {}));
+      ret->session->device_mgr, opts->session_options.options.env,
+      TF_GRAPH_DEF_VERSION, &ret->func_lib_def, {}));
   ret->rendezvous =
       new tensorflow::IntraProcessRendezvous(ret->session->device_mgr);
 
diff --git a/tensorflow/c/eager/c_api.h b/tensorflow/c/eager/c_api.h
index 9bfa63711b..201cb222c9 100644
--- a/tensorflow/c/eager/c_api.h
+++ b/tensorflow/c/eager/c_api.h
@@ -43,14 +43,30 @@ limitations under the License.
 extern "C" {
 #endif
 
+typedef struct TFE_ContextOptions TFE_ContextOptions;
+
+// Return a new options object.
+TF_CAPI_EXPORT extern TFE_ContextOptions* TFE_NewContextOptions();
+
+// Set the config in TF_ContextOptions.options.
+// config should be a serialized tensorflow.ConfigProto proto.
+// If config was not parsed successfully as a ConfigProto, record the
+// error information in *status.
+TF_CAPI_EXPORT extern void TFE_ContextOptionsSetConfig(
+    TFE_ContextOptions* options, const void* proto, size_t proto_len,
+    TF_Status* status);
+
+// Destroy an options object.
+TF_CAPI_EXPORT extern void TFE_DeleteContextOptions(TFE_ContextOptions*);
+
 // "Context" under which operations/functions are executed. It encapsulates
 // things like the available devices, resource manager etc.
 //
 // TODO(ashankar): Merge with TF_Session?
 typedef struct TFE_Context TFE_Context;
 
-TF_CAPI_EXPORT extern TFE_Context* TFE_NewContext(const TF_SessionOptions* opts,
-                                                  TF_Status* status);
+TF_CAPI_EXPORT extern TFE_Context* TFE_NewContext(
+    const TFE_ContextOptions* opts, TF_Status* status);
 TF_CAPI_EXPORT extern void TFE_DeleteContext(TFE_Context* ctx, TF_Status* status);
 TF_CAPI_EXPORT extern TF_DeviceList* TFE_ContextListDevices(TFE_Context* ctx,
                                                             TF_Status* status);
diff --git a/tensorflow/c/eager/c_api_internal.h b/tensorflow/c/eager/c_api_internal.h
index 712526f170..7a440a5a7e 100644
--- a/tensorflow/c/eager/c_api_internal.h
+++ b/tensorflow/c/eager/c_api_internal.h
@@ -35,6 +35,10 @@ limitations under the License.
 #include "tensorflow/core/platform/mutex.h"
 #include "tensorflow/core/platform/thread_annotations.h"
 
+struct TFE_ContextOptions {
+  TF_SessionOptions session_options;
+};
+
 struct TFE_Context {
   explicit TFE_Context(TF_Session* s) : session(s) {}
 
diff --git a/tensorflow/c/eager/c_api_test.cc b/tensorflow/c/eager/c_api_test.cc
index 72e0fe8a15..5344956ee7 100644
--- a/tensorflow/c/eager/c_api_test.cc
+++ b/tensorflow/c/eager/c_api_test.cc
@@ -62,10 +62,10 @@ TFE_Op* MatMulOp(TFE_Context* ctx, TFE_TensorHandle* a, TFE_TensorHandle* b) {
 void BM_InitOp(int iters) {
   tensorflow::testing::StopTiming();
   TF_Status* status = TF_NewStatus();
-  TF_SessionOptions* opts = TF_NewSessionOptions();
+  TFE_ContextOptions* opts = TFE_NewContextOptions();
   TFE_Context* ctx = TFE_NewContext(opts, status);
   CHECK_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
-  TF_DeleteSessionOptions(opts);
+  TFE_DeleteContextOptions(opts);
 
   TFE_TensorHandle* m = TestMatrixTensorHandle();
   tensorflow::testing::StartTiming();
@@ -84,10 +84,10 @@ BENCHMARK(BM_InitOp);
 void BM_Execute(int iters) {
   tensorflow::testing::StopTiming();
   TF_Status* status = TF_NewStatus();
-  TF_SessionOptions* opts = TF_NewSessionOptions();
+  TFE_ContextOptions* opts = TFE_NewContextOptions();
   TFE_Context* ctx = TFE_NewContext(opts, status);
   CHECK_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
-  TF_DeleteSessionOptions(opts);
+  TFE_DeleteContextOptions(opts);
 
   TFE_TensorHandle* m = TestMatrixTensorHandle();
   TFE_Op* matmul = MatMulOp(ctx, m, m);
@@ -109,9 +109,9 @@ BENCHMARK(BM_Execute);
 
 TEST(CAPI, Context) {
   TF_Status* status = TF_NewStatus();
-  TF_SessionOptions* opts = TF_NewSessionOptions();
+  TFE_ContextOptions* opts = TFE_NewContextOptions();
   TFE_Context* ctx = TFE_NewContext(opts, status);
-  TF_DeleteSessionOptions(opts);
+  TFE_DeleteContextOptions(opts);
 
   TF_DeviceList* devices = TFE_ContextListDevices(ctx, status);
   EXPECT_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
@@ -150,9 +150,9 @@ TEST(CAPI, TensorHandle) {
 TEST(CAPI, TensorHandleCopyBetweenDevices) {
   std::unique_ptr<TF_Status, decltype(&TF_DeleteStatus)> status(
       TF_NewStatus(), TF_DeleteStatus);
-  TF_SessionOptions* opts = TF_NewSessionOptions();
+  TFE_ContextOptions* opts = TFE_NewContextOptions();
   TFE_Context* ctx = TFE_NewContext(opts, status.get());
-  TF_DeleteSessionOptions(opts);
+  TFE_DeleteContextOptions(opts);
   ASSERT_EQ(TF_OK, TF_GetCode(status.get())) << TF_Message(status.get());
 
   TFE_TensorHandle* hcpu = TestMatrixTensorHandle();
@@ -218,10 +218,10 @@ TEST(CAPI, TensorHandleCopyBetweenDevices) {
 
 TEST(CAPI, Execute) {
   TF_Status* status = TF_NewStatus();
-  TF_SessionOptions* opts = TF_NewSessionOptions();
+  TFE_ContextOptions* opts = TFE_NewContextOptions();
   TFE_Context* ctx = TFE_NewContext(opts, status);
   CHECK_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
-  TF_DeleteSessionOptions(opts);
+  TFE_DeleteContextOptions(opts);
 
   TFE_TensorHandle* m = TestMatrixTensorHandle();
   TFE_Op* matmul = MatMulOp(ctx, m, m);
@@ -285,10 +285,10 @@ string MatMulFunction() {
 
 TEST(CAPI, FunctionDefAndExecute) {
   TF_Status* status = TF_NewStatus();
-  TF_SessionOptions* opts = TF_NewSessionOptions();
+  TFE_ContextOptions* opts = TFE_NewContextOptions();
   TFE_Context* ctx = TFE_NewContext(opts, status);
   CHECK_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
-  TF_DeleteSessionOptions(opts);
+  TFE_DeleteContextOptions(opts);
 
   string function_def = MatMulFunction();
   TFE_ContextAddFunctionDef(ctx, function_def.data(), function_def.size(),
@@ -326,10 +326,10 @@ TEST(CAPI, FunctionDefAndExecute) {
 void BM_ExecuteFunction(int iters) {
   tensorflow::testing::StopTiming();
   TF_Status* status = TF_NewStatus();
-  TF_SessionOptions* opts = TF_NewSessionOptions();
+  TFE_ContextOptions* opts = TFE_NewContextOptions();
   TFE_Context* ctx = TFE_NewContext(opts, status);
   CHECK_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
-  TF_DeleteSessionOptions(opts);
+  TFE_DeleteContextOptions(opts);
 
   string function_def = MatMulFunction();
   TFE_ContextAddFunctionDef(ctx, function_def.data(), function_def.size(),
@@ -406,10 +406,10 @@ TEST(CAPI, Variables) {
   // Variables use resource handles, so this is really a test for resource
   // tensor handling.
   TF_Status* status = TF_NewStatus();
-  TF_SessionOptions* opts = TF_NewSessionOptions();
+  TFE_ContextOptions* opts = TFE_NewContextOptions();
   TFE_Context* ctx = TFE_NewContext(opts, status);
   ASSERT_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
-  TF_DeleteSessionOptions(opts);
+  TFE_DeleteContextOptions(opts);
 
   TFE_TensorHandle* var_handle = CreateVariable(ctx, 12.0, status);
   ASSERT_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
@@ -446,10 +446,10 @@ TEST(CAPI, Variables) {
 void BM_ReadVariable(int iters) {
   tensorflow::testing::StopTiming();
   TF_Status* status = TF_NewStatus();
-  TF_SessionOptions* opts = TF_NewSessionOptions();
+  TFE_ContextOptions* opts = TFE_NewContextOptions();
   TFE_Context* ctx = TFE_NewContext(opts, status);
   CHECK_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
-  TF_DeleteSessionOptions(opts);
+  TFE_DeleteContextOptions(opts);
 
   TFE_TensorHandle* var_handle = CreateVariable(ctx, 5.0, status);
   CHECK_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
diff --git a/tensorflow/python/eager/context.py b/tensorflow/python/eager/context.py
index aa7cba56de..58581283d2 100644
--- a/tensorflow/python/eager/context.py
+++ b/tensorflow/python/eager/context.py
@@ -26,7 +26,6 @@ import threading
 from tensorflow.python import pywrap_tensorflow
 from tensorflow.python.framework import device as pydev
 from tensorflow.python.framework import errors
-from tensorflow.python.util import compat
 from tensorflow.python.util import tf_contextlib
 
 GRAPH_MODE = 0
@@ -103,11 +102,16 @@ class Context(object):
       if self._context_handle is not None:
         return
       assert self._context_devices is None
-      opts = pywrap_tensorflow.TF_NewSessionOptions(
-          target=compat.as_bytes(""), config=self._config)
-      with errors.raise_exception_on_not_ok_status() as status:
-        self._context_handle = pywrap_tensorflow.TFE_NewContext(opts, status)
-        pywrap_tensorflow.TF_DeleteSessionOptions(opts)
+      opts = pywrap_tensorflow.TFE_NewContextOptions()
+      try:
+        with errors.raise_exception_on_not_ok_status() as status:
+          if self._config is not None:
+            config_str = self._config.SerializeToString()
+            pywrap_tensorflow.TFE_ContextOptionsSetConfig(
+                opts, config_str, len(config_str), status)
+          self._context_handle = pywrap_tensorflow.TFE_NewContext(opts, status)
+      finally:
+        pywrap_tensorflow.TFE_DeleteContextOptions(opts)
       # Store list of devices
       self._context_devices = []
       with errors.raise_exception_on_not_ok_status() as status:
diff --git a/tensorflow/python/pywrap_tfe.i b/tensorflow/python/pywrap_tfe.i
index 5c624a9c12..36c09c20c2 100644
--- a/tensorflow/python/pywrap_tfe.i
+++ b/tensorflow/python/pywrap_tfe.i
@@ -30,12 +30,25 @@ limitations under the License.
 %rename("%s") TFE_Py_TapeDeleteTrace;
 %rename("%s") TFE_Py_TapeRecordOperation;
 %rename("%s") TFE_Py_TapeExport;
-
+%rename("%s") TFE_NewContextOptions;
+%rename("%s") TFE_ContextOptionsSetConfig;
+%rename("%s") TFE_DeleteContextOptions;
 
 %{
 #include "tensorflow/python/eager/pywrap_tfe.h"
 %}
 
+%typemap(in) (const void* proto) {
+  char* c_string;
+  Py_ssize_t py_size;
+  // PyBytes_AsStringAndSize() does not copy but simply interprets the input
+  if (PyBytes_AsStringAndSize($input, &c_string, &py_size) == -1) {
+    // Python has raised an error (likely TypeError or UnicodeEncodeError).
+    SWIG_fail;
+  }
+  $1 = static_cast<void*>(c_string);
+}
+
 %typemap(out) TF_DataType {
   $result = PyInt_FromLong($1);
 }
@@ -165,3 +178,4 @@ limitations under the License.
 %typemap(in, numinputs=0) TF_Status *out_status;
 %typemap(freearg) (TF_Status* out_status);
 %typemap(argout) (TFE_OutputTensorHandles* outputs, TF_Status* out_status);
+%typemap(in) (const void* proto);
-- 
GitLab


From e885d1abdce5db4a67e0b3ba85dbcc708f856645 Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Thu, 19 Oct 2017 16:42:45 -0700
Subject: [PATCH 590/909] One less error message in gradients_function

PiperOrigin-RevId: 172818233
---
 tensorflow/python/eager/backprop.py      | 11 ++++-------
 tensorflow/python/eager/backprop_test.py |  8 ++++++++
 2 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/tensorflow/python/eager/backprop.py b/tensorflow/python/eager/backprop.py
index da17be05b7..9580e84847 100644
--- a/tensorflow/python/eager/backprop.py
+++ b/tensorflow/python/eager/backprop.py
@@ -396,12 +396,11 @@ def implicit_grad(f):
   return grad_fn
 
 
-def _get_arg_spec(f, params):
+def _get_arg_spec(f, params, param_args):
   args = tf_inspect.getargspec(f).args
   if params is None:
     if not args:
-      raise ValueError("When params is None the differentiated function cannot"
-                       " only take arguments by *args and **kwds.")
+      return range(len(param_args))
     return range(len(args))
   elif all(isinstance(x, six.string_types) for x in params):
     return [args.index(n) for n in params]
@@ -560,10 +559,9 @@ def val_and_grad_function(f, params=None):
    ValueError: if the params are not all strings or all integers.
   """
 
-  parameter_positions = _get_arg_spec(f, params)
-
   def decorated(*args, **kwds):
     """Computes the value and gradient of the decorated function."""
+    parameter_positions = _get_arg_spec(f, params, args)
     dy = kwds.pop("dy", None)
     if dy is not None:
       dy = ops.convert_to_tensor(dy)
@@ -616,10 +614,9 @@ def make_vjp(f, params=None):
 
   """
 
-  parameter_positions = _get_arg_spec(f, params)
-
   def decorated(*args, **kwds):
     """Computes the value and gradient of the decorated function."""
+    parameter_positions = _get_arg_spec(f, params, args)
     assert not kwds, "The gradient function can't take keyword arguments."
     tape.push_new_tape()
     sources = []
diff --git a/tensorflow/python/eager/backprop_test.py b/tensorflow/python/eager/backprop_test.py
index 95d5f0adcb..7da8eb0c9b 100644
--- a/tensorflow/python/eager/backprop_test.py
+++ b/tensorflow/python/eager/backprop_test.py
@@ -381,6 +381,14 @@ class BackpropTest(test.TestCase):
         [tensor_shape.TensorShape(s).as_proto() for s in shape_list],
         backprop.make_attr([pywrap_tensorflow.TF_ATTR_SHAPE], shape_list))
 
+  def testArgsGradientFunction(self):
+
+    def f(*args):
+      return args[0] * args[0]
+
+    grad = backprop.gradients_function(f)
+    self.assertAllEqual(grad(1.0)[0], 2.0)
+
   def testMultiValueConvertToTensor(self):
     x = resource_variable_ops.ResourceVariable(
         initial_value=array_ops.constant([1.0]), name='x')
-- 
GitLab


From f1054553eafc74df8be9425c3344e71af98962ad Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 19 Oct 2017 17:31:49 -0700
Subject: [PATCH 591/909] Add missing backslash in macro in
 mkl_transpose_op.cc. Fix erroneous formatting that resulted from it. Fix
 return type for function template MKLTranspose2D. Define MKL_Complex8 and
 MKL_Complex16 macros before including the MKL headers. Only conjugate but
 don't transpose if conjugate=true && perm[0] == 0 && perm[1] == 1.

PiperOrigin-RevId: 172824073
---
 tensorflow/core/kernels/mkl_transpose_op.cc | 120 ++++++++++----------
 1 file changed, 63 insertions(+), 57 deletions(-)

diff --git a/tensorflow/core/kernels/mkl_transpose_op.cc b/tensorflow/core/kernels/mkl_transpose_op.cc
index 89a1d5e8a7..764d4c9400 100644
--- a/tensorflow/core/kernels/mkl_transpose_op.cc
+++ b/tensorflow/core/kernels/mkl_transpose_op.cc
@@ -18,6 +18,9 @@ limitations under the License.
 #ifdef INTEL_MKL
 #define EIGEN_USE_THREADS
 
+#include "tensorflow/core/framework/numeric_types.h"
+#define MKL_Complex8 tensorflow::complex64
+#define MKL_Complex16 tensorflow::complex128
 #include "mkl_trans.h"
 #include "tensorflow/core/kernels/transpose_functor.h"
 #include "tensorflow/core/kernels/transpose_op.h"
@@ -41,7 +44,7 @@ namespace tensorflow {
 
 namespace {
 template <typename T>
-void MKLTranspose2D(const char trans, const Tensor& in, Tensor* out) {}
+Status MKLTranspose2D(const char trans, const Tensor& in, Tensor* out);
 
 // Documentation here: https://software.intel.com/en-us/node/520863
 // Parameters: (ordering:row-major, operation:transpose, num_rows, num_cols,
@@ -54,70 +57,73 @@ void MKLTranspose2D(const char trans, const Tensor& in, Tensor* out) {}
     mkl_##PREFIX##omatcopy('R', trans, in.dim_size(0), in.dim_size(1), 1,     \
                            in.flat<T>().data(), in.dim_size(1),               \
                            out->flat<T>().data(), in.dim_size(0));            \
-    return Status::OK();
+    return Status::OK();                                                      \
   }
 
-  INSTANTIATE(float, s)
-  INSTANTIATE(double, d)
-  INSTANTIATE(complex64, c)
-  INSTANTIATE(complex128, z)
+INSTANTIATE(float, s)
+INSTANTIATE(double, d)
+INSTANTIATE(complex64, c)
+INSTANTIATE(complex128, z)
 #undef INSTANTIATE
 
-  static const char kMKLTranspose = 'T';
-  static const char kMKLConjugateTranspose = 'C';
-
-  }  // namespace tensorflow
-
-  Status MklTransposeCpuOp::DoTranspose(OpKernelContext* ctx, const Tensor& in,
-                                        gtl::ArraySlice<int32> perm,
-                                        Tensor* out) {
-    if (in.dims() == 2) {
-      switch (in.dtype()) {
-        case DT_FLOAT:
-          return MKLTranspose2D<float>(kMKLTranspose, in, out);
-        case DT_DOUBLE:
-          return MKLTranspose2D<double>(kMKLTranspose, in, out);
-        case DT_COMPLEX64:
-          return MKLTranspose2D<complex64>(kMKLTranspose, in, out);
-        case DT_COMPLEX128:
-          return MKLTranspose2D<complex128>(kMKLTranspose, in, out);
-        default:
-          break;
-      }
+static const char kMKLTranspose = 'T';
+static const char kMKLConjugateTranspose = 'C';
+
+}  // namespace
+
+Status MklTransposeCpuOp::DoTranspose(OpKernelContext* ctx, const Tensor& in,
+                                      gtl::ArraySlice<int32> perm,
+                                      Tensor* out) {
+  if (in.dims() == 2) {
+    if (perm[0] == 0 && perm[1] == 1) {
+      return Status::OK();
+    }
+    switch (in.dtype()) {
+      case DT_FLOAT:
+        return MKLTranspose2D<float>(kMKLTranspose, in, out);
+      case DT_DOUBLE:
+        return MKLTranspose2D<double>(kMKLTranspose, in, out);
+      case DT_COMPLEX64:
+        return MKLTranspose2D<complex64>(kMKLTranspose, in, out);
+      case DT_COMPLEX128:
+        return MKLTranspose2D<complex128>(kMKLTranspose, in, out);
+      default:
+        break;
     }
-    // Fallback to eigen if transpose parameters not supported by MKL
-    typedef Eigen::ThreadPoolDevice CPUDevice;
-    return ::tensorflow::DoTranspose(ctx->eigen_device<CPUDevice>(), in, perm,
-                                     out);
   }
-
-  Status MklConjugateTransposeCpuOp::DoTranspose(OpKernelContext* ctx,
-                                                 const Tensor& in,
-                                                 gtl::ArraySlice<int32> perm,
-                                                 Tensor* out) {
-    if (in.dims() == 2) {
-      // TODO(rmlarsen): By setting lda and ldb, we could use the MKL kernels
-      // for any transpose that can be reduced to swapping the last two
-      // dimensions in a rank-3 tensor. We can even run each outer dimension in
-      // a separate thread.
-      switch (in.dtype()) {
-        case DT_FLOAT:
-          return MKLTranspose2D<float>(kMKLTranspose, in, out);
-        case DT_DOUBLE:
-          return MKLTranspose2D<double>(kMKLTranspose, in, out);
-        case DT_COMPLEX64:
-          return MKLTranspose2D<complex64>(kMKLConjugateTranspose, in, out);
-        case DT_COMPLEX128:
-          return MKLTranspose2D<complex128>(kMKLConjugateTranspose, in, out);
-        default:
-          break;
-      }
+  // Fallback to eigen if transpose parameters not supported by MKL
+  typedef Eigen::ThreadPoolDevice CPUDevice;
+  return ::tensorflow::DoTranspose(ctx->eigen_device<CPUDevice>(), in, perm,
+                                   out);
+}
+
+Status MklConjugateTransposeCpuOp::DoTranspose(OpKernelContext* ctx,
+                                               const Tensor& in,
+                                               gtl::ArraySlice<int32> perm,
+                                               Tensor* out) {
+  if (in.dims() == 2 && perm[0] == 1 && perm[1] == 0) {
+    // TODO(rmlarsen): By setting lda and ldb, we could use the MKL kernels
+    // for any transpose that can be reduced to swapping the last two
+    // dimensions in a rank-3 tensor. We can even run each outer dimension in
+    // a separate thread.
+    switch (in.dtype()) {
+      case DT_FLOAT:
+        return MKLTranspose2D<float>(kMKLTranspose, in, out);
+      case DT_DOUBLE:
+        return MKLTranspose2D<double>(kMKLTranspose, in, out);
+      case DT_COMPLEX64:
+        return MKLTranspose2D<complex64>(kMKLConjugateTranspose, in, out);
+      case DT_COMPLEX128:
+        return MKLTranspose2D<complex128>(kMKLConjugateTranspose, in, out);
+      default:
+        break;
     }
-    // Fallback to eigen if transpose parameters not supported by MKL
-    typedef Eigen::ThreadPoolDevice CPUDevice;
-    return ::tensorflow::DoConjugateTranspose(ctx->eigen_device<CPUDevice>(),
-                                              in, perm, out);
   }
+  // Fallback to eigen if transpose parameters not supported by MKL
+  typedef Eigen::ThreadPoolDevice CPUDevice;
+  return ::tensorflow::DoConjugateTranspose(ctx->eigen_device<CPUDevice>(), in,
+                                            perm, out);
+}
 
 }  // namespace tensorflow
 
-- 
GitLab


From e7654b99c46a479d61c1fd96a9f4710682acf4da Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Thu, 19 Oct 2017 17:44:32 -0700
Subject: [PATCH 592/909] Adds tfe.IsolateTest, an Eager-agnostic abstraction
 for isolating resources

Switches Eager unit tests to use IsolateTest, so their resources have unique
container names.

PiperOrigin-RevId: 172825317
---
 tensorflow/contrib/eager/python/tfe.py        |  2 +
 tensorflow/python/eager/graph_callable.py     | 10 +++
 tensorflow/python/framework/test_util.py      | 64 ++++++++++++++++-
 tensorflow/python/framework/test_util_test.py | 71 +++++++++++++++++++
 .../resource_variable_ops_test.py             | 11 ++-
 .../python/ops/resource_variable_ops.py       | 11 +++
 6 files changed, 164 insertions(+), 5 deletions(-)

diff --git a/tensorflow/contrib/eager/python/tfe.py b/tensorflow/contrib/eager/python/tfe.py
index 25942aadfb..4ed258f6ff 100644
--- a/tensorflow/contrib/eager/python/tfe.py
+++ b/tensorflow/contrib/eager/python/tfe.py
@@ -53,6 +53,7 @@ To use, at program startup, call `tfe.enable_eager_execution()`.
 @@in_eager_mode
 @@in_graph_mode
 
+@@IsolateTest
 @@run_test_in_graph_and_eager_modes
 """
 
@@ -84,6 +85,7 @@ from tensorflow.python.eager.execution_callbacks import nan_callback
 from tensorflow.python.eager.execution_callbacks import seterr
 from tensorflow.python.framework.ops import enable_eager_execution
 from tensorflow.python.framework.ops import eager_run as run
+from tensorflow.python.framework.test_util import IsolateTest
 from tensorflow.python.framework.test_util import run_in_graph_and_eager_modes as run_test_in_graph_and_eager_modes
 from tensorflow.python.ops.resource_variable_ops import ResourceVariable as Variable
 from tensorflow.python.util.all_util import remove_undocumented
diff --git a/tensorflow/python/eager/graph_callable.py b/tensorflow/python/eager/graph_callable.py
index 3aba164630..0ec83636a0 100644
--- a/tensorflow/python/eager/graph_callable.py
+++ b/tensorflow/python/eager/graph_callable.py
@@ -312,11 +312,21 @@ def _graph_callable_internal(func, shape_and_dtypes):
   Returns:
     Callable graph object.
   """
+  container = tf_ops.get_default_graph()._container  # pylint: disable=protected-access
+  container_prefix = tf_ops.get_default_graph()._container_prefix  # pylint: disable=protected-access
   with context.graph_mode():
     # This graph will store both the initialization and the call version of the
     # wrapped function. It will later be used by the backprop code to build the
     # backprop graph, if necessary.
     tmp_graph = tf_ops.Graph()
+    # Inherit the container from the original graph to create resources at user
+    # expected containers. Also inherits the container prefix, since this is
+    # used for error checking when isolating Eager execution (the container
+    # prefix at creation must match the container prefix when used, and
+    # variables returned from the graph callable will be used in the outside
+    # context).
+    tmp_graph._container = container  # pylint: disable=protected-access
+    tmp_graph._container_prefix = container_prefix  # pylint: disable=protected-access
     with tmp_graph.as_default():
       # Placeholders for the non-variable inputs.
       func_inputs = _get_graph_callable_inputs(shape_and_dtypes)
diff --git a/tensorflow/python/framework/test_util.py b/tensorflow/python/framework/test_util.py
index c681ffb514..a01bf02deb 100644
--- a/tensorflow/python/framework/test_util.py
+++ b/tensorflow/python/framework/test_util.py
@@ -47,6 +47,7 @@ from tensorflow.python import pywrap_tensorflow
 from tensorflow.python.client import device_lib
 from tensorflow.python.client import session
 from tensorflow.python.eager import context
+from tensorflow.python.eager import tape
 from tensorflow.python.framework import device as pydev
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
@@ -391,6 +392,66 @@ def with_c_api(cls):
   return cls
 
 
+class IsolateTest(object):
+  """A context manager which isolates resources in its block.
+
+  Provides an Eager-agnostic abstraction for preventing the sharing of
+  variables and other resources.
+
+  In graph mode, resource handle ops are only executed in a particular Session,
+  isolating them from resources with the same name in other Graphs. In Eager,
+  separate Sessions do not exist, so resources (particularly ResourceVariables)
+  would be shared implicitly if a resource of the same name were created
+  anywhere in a Python process. Multiple handles to the same resource would
+  cause several issues, and so this type of sharing will raise an exception.
+
+  Using resources with the same name in a single Python process may be useful
+  (especially for unit tests), so this context manager provides an abstraction
+  for isolating resources. Using a resource created in one Isolation environment
+  in another is an error.
+
+  Example usage in Eager mode:
+
+  ```python
+  import tensorflow as tf
+  # Import subject to change
+  from tensorflow.contrib.eager.python import tfe
+
+  tfe.enable_eager_execution()
+
+  for hyperparameter in [1, 2, 3]:
+    with tfe.IsolateTest():
+      v = tfe.Variable(name="v", initial_value=hyperparameter)
+      # train model, test results ...
+  ```
+
+  IsolateTest is currently exposed through contrib.eager, but it creates a new
+  default Graph and provides equivalent safety in graph mode.
+  """
+
+  def __init__(self):
+    if context.in_eager_mode() and tape.could_possibly_record():
+      raise ValueError("Cannot isolate Eager execution with an active tape.")
+    # In Eager, Graphs set a container which isolates resources, and maintain a
+    # VariableStore which caches ResourceVariable objects created through
+    # get_variable. So setting the default Graph has the side effect of
+    # isolating Eager resources.
+    with context.eager_mode():
+      # Create the graph in Eager mode, as this provides stricter semantics
+      # (i.e. has a unique container prefix). This prevents implicit sharing
+      # when a Graph-mode graph is created and then Eager mode is enabled (an
+      # error through enable_eager_execution, but common with context managers
+      # in unit tests).
+      self._graph_as_default_context_manager = ops.Graph().as_default()
+
+  def __enter__(self):
+    self._graph_as_default_context_manager.__enter__()
+
+  def __exit__(self, type_arg, value_arg, traceback_arg):
+    return self._graph_as_default_context_manager.__exit__(
+        type_arg, value_arg, traceback_arg)
+
+
 def run_in_graph_and_eager_modes(__unused__=None, graph=None, config=None,
                                  use_gpu=False, force_gpu=False,
                                  reset_test=True):
@@ -440,9 +501,8 @@ def run_in_graph_and_eager_modes(__unused__=None, graph=None, config=None,
           with context.device("/device:CPU:0"):
             f(self, **kwargs)
 
-      eager_graph = graph or ops.Graph()
       with context.eager_mode():
-        with eager_graph.as_default():
+        with IsolateTest():
           run_eager_mode()
 
     return decorated
diff --git a/tensorflow/python/framework/test_util_test.py b/tensorflow/python/framework/test_util_test.py
index 6129fa2e0d..b2f8d62095 100644
--- a/tensorflow/python/framework/test_util_test.py
+++ b/tensorflow/python/framework/test_util_test.py
@@ -27,12 +27,16 @@ from google.protobuf import text_format
 
 from tensorflow.core.framework import graph_pb2
 from tensorflow.core.protobuf import meta_graph_pb2
+from tensorflow.python.client import session
+from tensorflow.python.eager import context
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import random_ops
+from tensorflow.python.ops import resource_variable_ops
+from tensorflow.python.ops import variables
 from tensorflow.python.platform import googletest
 
 
@@ -325,5 +329,72 @@ class TestUtilTest(test_util.TensorFlowTestCase):
     self.assertEqual(a_rand, b_rand)
 
 
+@test_util.with_c_api
+class IsolationTest(test_util.TensorFlowTestCase):
+
+  @test_util.run_in_graph_and_eager_modes()
+  def test_variable_reuse_exception(self):
+    with test_util.IsolateTest(), session.Session():
+      first_container_variable = resource_variable_ops.ResourceVariable(
+          name="first_container_variable",
+          initial_value=1)
+      if context.in_graph_mode():
+        self.evaluate([variables.global_variables_initializer()])
+    with test_util.IsolateTest():
+      if context.in_graph_mode():
+        with self.assertRaises(RuntimeError):
+          self.evaluate(first_container_variable.read_value())
+      else:
+        with self.assertRaises(ValueError):
+          first_container_variable.read_value()
+
+  @test_util.run_in_graph_and_eager_modes()
+  def test_variable_reuse_exception_nested(self):
+    with test_util.IsolateTest(), session.Session():
+      first_container_variable = resource_variable_ops.ResourceVariable(
+          name="first_container_variable",
+          initial_value=1)
+      if context.in_graph_mode():
+        self.evaluate([variables.global_variables_initializer()])
+      with test_util.IsolateTest(), session.Session():
+        if context.in_graph_mode():
+          with self.assertRaises(RuntimeError):
+            self.evaluate(first_container_variable.read_value())
+        else:
+          with self.assertRaises(ValueError):
+            first_container_variable.read_value()
+
+  @test_util.run_in_graph_and_eager_modes()
+  def test_no_sharing(self):
+    with test_util.IsolateTest(), session.Session():
+      first_container_variable = resource_variable_ops.ResourceVariable(
+          name="same_name",
+          initial_value=1)
+      if context.in_graph_mode():
+        self.evaluate([variables.global_variables_initializer()])
+      with test_util.IsolateTest(), session.Session():
+        second_container_variable = resource_variable_ops.ResourceVariable(
+            name="same_name",
+            initial_value=2)
+        if context.in_graph_mode():
+          self.evaluate([variables.global_variables_initializer()])
+        self.assertEqual(
+            2, self.evaluate(second_container_variable.read_value()))
+      self.assertEqual(1, self.evaluate(first_container_variable.read_value()))
+
+  def test_graph_mode_isolation(self):
+    with context.graph_mode():
+      # Even if we've (accidentally) called IsolateTest in Graph mode, it should
+      # provide Eager isolation.
+      with test_util.IsolateTest():
+        with context.eager_mode():
+          first_container_variable = resource_variable_ops.ResourceVariable(
+              name="first_container_variable",
+              initial_value=1)
+      with context.eager_mode():
+        with self.assertRaises(ValueError):
+          first_container_variable.read_value()
+
+
 if __name__ == "__main__":
   googletest.main()
diff --git a/tensorflow/python/kernel_tests/resource_variable_ops_test.py b/tensorflow/python/kernel_tests/resource_variable_ops_test.py
index 23676223dc..cf4b61674f 100644
--- a/tensorflow/python/kernel_tests/resource_variable_ops_test.py
+++ b/tensorflow/python/kernel_tests/resource_variable_ops_test.py
@@ -309,12 +309,15 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase):
     self.evaluate(variables.global_variables_initializer())
 
     w = resource_variable_ops.var_handle_op(
-        dtype=v.dtype.base_dtype, shape=v.get_shape(), shared_name="var4")
+        dtype=v.dtype.base_dtype, shape=v.get_shape(), shared_name="var4",
+        # Needed in Eager since we get a unique container name by default.
+        container=ops.get_default_graph()._container)
     w_read = resource_variable_ops.read_variable_op(w, v.dtype.base_dtype)
     self.assertEqual(300.0, self.evaluate(w_read))
 
     x = resource_variable_ops.var_handle_op(
-        dtype=v.dtype.base_dtype, shape=v.get_shape(), shared_name="var5")
+        dtype=v.dtype.base_dtype, shape=v.get_shape(), shared_name="var5",
+        container=ops.get_default_graph()._container)
     with self.assertRaisesOpError("Resource .*/var5/.* does not exist"):
       x_read = resource_variable_ops.read_variable_op(x, v.dtype.base_dtype)
       self.evaluate(x_read)
@@ -328,7 +331,9 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase):
       self.evaluate(variables.global_variables_initializer())
 
     w = resource_variable_ops.var_handle_op(
-        dtype=v.dtype.base_dtype, shape=v.get_shape(), shared_name="foo/var6")
+        dtype=v.dtype.base_dtype, shape=v.get_shape(), shared_name="foo/var6",
+        # Needed in Eager since we get a unique container name by default.
+        container=ops.get_default_graph()._container)
     w_read = resource_variable_ops.read_variable_op(w, v.dtype.base_dtype)
     self.assertEqual(300.0, self.evaluate(w_read))
 
diff --git a/tensorflow/python/ops/resource_variable_ops.py b/tensorflow/python/ops/resource_variable_ops.py
index dd3f167145..aa45752a9d 100644
--- a/tensorflow/python/ops/resource_variable_ops.py
+++ b/tensorflow/python/ops/resource_variable_ops.py
@@ -270,6 +270,9 @@ class ResourceVariable(variables.Variable):
       collections = list(collections) + [ops.GraphKeys.TRAINABLE_VARIABLES]
     self._save_slice_info = None
     self._in_graph_mode = context.in_graph_mode()
+    # Save the graph's container prefix for error checking. Reading the value of
+    # the ResourceVariable from another Graph in Eager mode is an error.
+    self._container_prefix = ops.get_default_graph()._container_prefix  # pylint: disable=protected-access
     with ops.control_dependencies(None):
       with ops.name_scope(name, "Variable", []
                           if init_from_fn else [initial_value]) as name:
@@ -577,7 +580,15 @@ class ResourceVariable(variables.Variable):
 
     Returns:
      the read operation.
+    Raises:
+      ValueError: if the ResourceVariable was created in another isolation
+        environment or graph.
     """
+    if (not self._in_graph_mode and
+        self._container_prefix != ops.get_default_graph()._container_prefix):  # pylint: disable=protected-access
+      raise ValueError(
+          "Attempted to read a variable from another isolation environment"
+          " or Graph")
     with ops.name_scope("Read"):
       # Ensure we read the variable in the same device as the handle.
       with ops.device(self._handle_device):
-- 
GitLab


From db07ee27b75f5efecf3f3706ec1a11e4cd05da54 Mon Sep 17 00:00:00 2001
From: Asim Shankar <ashankar@google.com>
Date: Thu, 19 Oct 2017 17:58:38 -0700
Subject: [PATCH 593/909] Fix bug introduced in
 https://github.com/tensorflow/tensorflow/commit/dc442f4ce2d3b11b56721337fe2b9e2282be93be

Potentially invalid pointers passed to GraphConstructor::Construct()

PiperOrigin-RevId: 172826567
---
 tensorflow/core/graph/graph_constructor.cc | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/tensorflow/core/graph/graph_constructor.cc b/tensorflow/core/graph/graph_constructor.cc
index 92b4843221..b2c193b050 100644
--- a/tensorflow/core/graph/graph_constructor.cc
+++ b/tensorflow/core/graph/graph_constructor.cc
@@ -1068,10 +1068,16 @@ Status ImportGraphDef(const ImportGraphDefOptions& opts, const GraphDef& gdef,
   refiner->set_graph_def_version(
       std::min(refiner->graph_def_version(), gdef.versions().producer()));
 
-  return GraphConstructor::Construct(
-      opts, gdef.node(), &gdef.versions(), &gdef.library(), g, refiner,
-      &results->return_tensors, &results->return_nodes,
-      &results->unused_input_map_keys);
+  if (results == nullptr) {
+    return GraphConstructor::Construct(opts, gdef.node(), &gdef.versions(),
+                                       &gdef.library(), g, refiner, nullptr,
+                                       nullptr, nullptr);
+  } else {
+    return GraphConstructor::Construct(
+        opts, gdef.node(), &gdef.versions(), &gdef.library(), g, refiner,
+        &results->return_tensors, &results->return_nodes,
+        &results->unused_input_map_keys);
+  }
 }
 
 void CopyGraph(const Graph& src, Graph* dest) {
-- 
GitLab


From fa4d04ab99d45eb317e39c1a6b8848bbc47ebe0e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 19 Oct 2017 17:58:50 -0700
Subject: [PATCH 594/909] Address Metrics TODOs, in particular we'd like them
 to work in Graph mode. To get this to work, we add support for capturing
 tensors from outside the function graph in graph mode to eager/function.py.
 Also get unique names and variable scopes working.

PiperOrigin-RevId: 172826589
---
 tensorflow/contrib/eager/python/BUILD         |   9 +-
 .../contrib/eager/python/evaluator_test.py    |   2 +-
 .../contrib/eager/python/metrics_impl.py      | 156 ++++++++++++------
 .../contrib/eager/python/metrics_test.py      |  51 ++++++
 tensorflow/python/eager/function.py           |  61 +++++--
 tensorflow/python/eager/function_test.py      |  18 ++
 tensorflow/tools/ci_build/ci_sanity.sh        |   1 +
 7 files changed, 234 insertions(+), 64 deletions(-)

diff --git a/tensorflow/contrib/eager/python/BUILD b/tensorflow/contrib/eager/python/BUILD
index 0c61630aa8..702136e3e4 100644
--- a/tensorflow/contrib/eager/python/BUILD
+++ b/tensorflow/contrib/eager/python/BUILD
@@ -132,11 +132,12 @@ py_library(
         "//tensorflow/python:array_ops",
         "//tensorflow/python:dtypes",
         "//tensorflow/python:framework_ops",
-        "//tensorflow/python:init_ops",
         "//tensorflow/python:layers_base",
         "//tensorflow/python:math_ops",
         "//tensorflow/python:util",
         "//tensorflow/python:variable_scope",
+        "//tensorflow/python/eager:context",
+        "//tensorflow/python/eager:function",
     ],
 )
 
@@ -146,6 +147,10 @@ py_test(
     srcs_version = "PY2AND3",
     deps = [
         ":metrics",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:dtypes",
+        "//tensorflow/python:variables",
+        "//tensorflow/python/eager:context",
         "//tensorflow/python/eager:test",
     ],
 )
@@ -160,6 +165,8 @@ py_library(
     deps = [
         ":datasets",
         ":metrics",
+        "//tensorflow/python/eager:context",
+        "//tensorflow/python/eager:function",
     ],
 )
 
diff --git a/tensorflow/contrib/eager/python/evaluator_test.py b/tensorflow/contrib/eager/python/evaluator_test.py
index 099e10e230..b18463c31a 100644
--- a/tensorflow/contrib/eager/python/evaluator_test.py
+++ b/tensorflow/contrib/eager/python/evaluator_test.py
@@ -86,7 +86,7 @@ class EvaluatorTest(test.TestCase):
     for v in e.metric_variables:
       p = v.name.split("/")[0]
       prefix_count[p] = prefix_count.get(p, 0) + 1
-    self.assertEqual({"outer-mean": 2, "mean": 2}, prefix_count)
+    self.assertEqual({"outer_mean": 2, "mean": 2}, prefix_count)
 
   def testDataset(self):
     e = SimpleEvaluator(IdentityModel())
diff --git a/tensorflow/contrib/eager/python/metrics_impl.py b/tensorflow/contrib/eager/python/metrics_impl.py
index 63a0f8d9a4..2a624b218c 100644
--- a/tensorflow/contrib/eager/python/metrics_impl.py
+++ b/tensorflow/contrib/eager/python/metrics_impl.py
@@ -18,6 +18,10 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import re
+
+from tensorflow.python.eager import context
+from tensorflow.python.eager import function
 from tensorflow.python.framework import dtypes
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import init_ops
@@ -25,55 +29,69 @@ from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import variable_scope
 
 
+_to_replace = re.compile("[^A-Za-z0-9.]")
+
+
 class Metric(object):
   """A metric holds state for aggregating statistics over an evaluation run.
 
   Users will use Evaluator.add_metric() to add Metric objects to their
-  evaluation, call them in each step, and then use
-  Evaluator.all_metric_results() at the end.
+  evaluation, call them in each step (treating the object as a callable),
+  and then use Evaluator.all_metric_results() at the end.
 
   Descendants will implement:
-  * call(): Should follow this pattern:
-      if not self.built:
-        self.var = self.add_variable(...)
-      self.add_update(self.var.assign_add(...))
-  * aggregate(): Adds in the state from a list of metrics of the same type
-    as `self`.  (Default of summing all the variables will be fine for most
-    descendants.)
-  * result(): Computes and returns a final value for the metric
+  * `build()`: All variables should be created in this method, by calling
+    `self.add_variable()` as in: `self.var = self.add_variable(...)`
+    build() will be called in the first invocation of `__call__()`, with
+    the same arguments passed `call()`.
+  * `call()`: Has all updates to variables, as in:
+      self.var.assign_add(...)
+  * `result()`: Computes and returns a final value for the metric
     from the variables in `self`.
+
+  Decendants may override, but usually won't need to:
+  * `aggregate()`: Adds in the state from a list of metrics of the same type
+    as `self`.  (Default is to sum all the variables.)
+  * `reset()`: Reset all variables to their initial state. (Default is to
+    zero all the variables.)
+  Note that users should not call `aggregate()` or `reset()`, they are for
+  use by TensorFlow infrastructure.
   """
 
   def __init__(self, name=None):
-    self.built = False
+    self._built = False
     self._vars = []
     self._updates = []
-    self._name = name or self.__class__.__name__
-    # TODO(josh11b): Need some way to make sure two Metrics in the same
-    # Network have distinct names. Maybe we can get a unique name from
-    # a name/variable scope?
-    # TODO(josh11b): self._in_graph_mode = context.in_graph_mode()
+    name = name or self.__class__.__name__
+    # Replace things like spaces in name to create a valid scope name.
+    scope_name = _to_replace.sub("_", name)
+    # We create the variable scope now to get the unique name that will
+    # be used as a variable prefix when build() calls add_variable().
+    with variable_scope.variable_scope(
+        None, default_name=scope_name, use_resource=True, reuse=False) as scope:
+      pos = scope.name.rfind(scope_name)
+      self._name = name + scope.name[pos + len(scope_name):]
+      self._scope = scope
+    if context.in_graph_mode():
+      # We make self.call() into a graph callable here, so that we can
+      # return a single op that performs all of the variable updates.
+      self.call = function.defun(self.call)
 
   # ---- API for users ----
   def __call__(self, *args, **kwargs):
-    # TODO(josh11b): If self._in_graph_mode is true, make self.call() into a
-    # graph callable here, so that variable updates happen without requiring
-    # a separate fetch.
-    # TODO(josh11b): Do we need a separate build() method to separate
-    # initialization from each update? If so, how do we get the arguments
-    # to it?  We *could* just pass in *args and **kwargs...
-    if not self.built:
-      # TODO(ashankar): Set up container isolation so there is no chance
-      # distinct metrics objects accidentally share variables.
-      # TODO(josh11b): Replace things like spaces in self._name to create
-      # a valid scope name.
-      with variable_scope.variable_scope(
-          self._name, use_resource=True, reuse=False):
-        ret = self.call(*args, **kwargs)
-      self.built = True
-    else:
-      ret = self.call(*args, **kwargs)
-    return ret
+    """Returns op to execute to update this metric for these inputs.
+
+    Returns None if eager execution is enabled.
+
+    Args:
+      *args:
+      **kwargs: A mini-batch of inputs to the Metric, passed on to `call()`.
+    """
+    if not self._built:
+      with variable_scope.variable_scope(self._scope):
+        self.build(*args, **kwargs)
+      self._built = True
+    return self.call(*args, **kwargs)
 
   @property
   def name(self):
@@ -84,10 +102,43 @@ class Metric(object):
     return self._vars
 
   # ---- To be implemented by descendants ---
+  def build(self, *args, **kwargs):
+    """Method to create variables.
+
+    Called by `__call__()` before `call()` for the first time.
+
+    Args:
+      *args:
+      **kwargs: The arguments to the first invocation of `__call__()`.
+       `build()` may use the shape and/or dtype of these arguments
+       when deciding how to create variables.
+    """
+    raise NotImplementedError("Metrics must define a build() member function")
+
   def call(self, *args, **kwargs):
-    """Accumulates statistics for the metric."""
+    """Accumulates statistics for the metric. Users should use __call__ instead.
+
+    Note: This function is executed as a graph function in graph mode.
+    This means:
+    a) Operations on the same resource are executed in textual order.
+       This should make it easier to do things like add the updated
+       value of a variable to another, for example.
+    b) You don't need to worry about collecting the update ops to execute.
+       All update ops added to the graph by this function will be executed.
+    As a result, code should generally work the same way with graph or
+    eager execution.
+
+    Args:
+      *args:
+      **kwargs: A mini-batch of inputs to the Metric, as passed to
+        `__call__()`.
+    """
     raise NotImplementedError("Metrics must define a call() member function")
 
+  def result(self):  # TODO(josh11b): Add an optional summary_writer parameter.
+    """Computes and returns a final value for the metric."""
+    raise NotImplementedError("Metrics must define a result() member function")
+
   # We can support two different strategies of for doing data-parallel
   # distributed metric computations:
   # * Put metric variables on the first device and rely on small
@@ -123,16 +174,19 @@ class Metric(object):
       self._vars[i].assign_add(math_ops.add_n([m._vars[i] for m in metrics]))
     # pylint: enable=protected-access
 
-  def result(self):  # TODO(josh11b): Add an optional summary_writer parameter.
-    """Computes and returns a final value for the metric."""
-    raise NotImplementedError("Metrics must define a result() member function")
+  def reset(self):
+    """Reset this metric to a freshly initialized state.
+
+    Default implementation zeros all the metric variables.
+    """
+    for v in self._vars:
+      v.assign(math_ops.zeros_like(v))
 
   # ---- For use by descendants ---
   def add_variable(self, name, shape=None, dtype=None, initializer=None):
     """***Only for use by descendants of Metric***."""
-    if self.built:
-      raise RuntimeError("Can't call add_variable() after a Metric has been "
-                         "built in the first call().")
+    if self._built:
+      raise RuntimeError("Can't call add_variable() except in build().")
     v = variable_scope.get_variable(name, shape, dtype, initializer,
                                     trainable=False, use_resource=True)
     self._vars.append(v)
@@ -144,6 +198,15 @@ class Mean(Metric):
   # TODO(josh11b): Maybe have a dtype argument that defaults to tf.float64?
   # Or defaults to type of the input if it is tf.float32, else tf.float64?
 
+  def build(self, values, weights=None):
+    del values, weights  # build() does not use call's arguments
+    self.numer = self.add_variable(name="numer", shape=(),
+                                   dtype=dtypes.float64,
+                                   initializer=init_ops.zeros_initializer)
+    self.denom = self.add_variable(name="denom", shape=(),
+                                   dtype=dtypes.float64,
+                                   initializer=init_ops.zeros_initializer)
+
   def call(self, values, weights=None):
     """Accumulate statistics for computing the mean.
 
@@ -154,13 +217,6 @@ class Mean(Metric):
       values: Tensor with the per-example value.
       weights: Optional weighting of each example. Defaults to 1.
     """
-    if not self.built:  # False only in the first call().
-      self.numer = self.add_variable(name="numer", shape=(),
-                                     dtype=dtypes.float64,
-                                     initializer=init_ops.zeros_initializer)
-      self.denom = self.add_variable(name="denom", shape=(),
-                                     dtype=dtypes.float64,
-                                     initializer=init_ops.zeros_initializer)
     if weights is None:
       self.denom.assign_add(
           math_ops.cast(array_ops.size(values), dtypes.float64))
@@ -179,6 +235,10 @@ class Mean(Metric):
 class Accuracy(Mean):
   """Calculates how often `predictions` matches `labels`."""
 
+  def build(self, labels, predictions, weights=None):
+    del labels, predictions, weights
+    super(Accuracy, self).build(None)  # Arguments are unused
+
   def call(self, labels, predictions, weights=None):
     """Accumulate accuracy statistics.
 
diff --git a/tensorflow/contrib/eager/python/metrics_test.py b/tensorflow/contrib/eager/python/metrics_test.py
index 089bad5a0e..bfb79cd72e 100644
--- a/tensorflow/contrib/eager/python/metrics_test.py
+++ b/tensorflow/contrib/eager/python/metrics_test.py
@@ -19,7 +19,11 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.contrib.eager.python import metrics
+from tensorflow.python.eager import context
 from tensorflow.python.eager import test
+from tensorflow.python.framework import dtypes
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import variables
 
 
 class MetricsTest(test.TestCase):
@@ -56,6 +60,53 @@ class MetricsTest(test.TestCase):
     m([7], [2])  # 0 correct, weight 1
     self.assertEqual(2.5/5, m.result().numpy())
 
+  def testTwoMeans(self):
+    # Verify two metrics with the same class and name don't
+    # accidentally share state.
+    m1 = metrics.Mean()
+    m2 = metrics.Mean()
+    m1(0)
+    m2(2)
+    self.assertEqual(0, m1.result().numpy())
+    self.assertEqual(2, m2.result().numpy())
+    self.assertNotEqual(m1.name, m2.name)
+
+  def testNamesWithSpaces(self):
+    # Verify two metrics with the same class and name don't
+    # accidentally share state.
+    m1 = metrics.Mean("has space")
+    m2 = metrics.Mean("has space")
+    m2(2)
+    m1(0)
+    self.assertEqual(m1.name, "has space")
+    self.assertEqual(m1.numer.name, "has_space/numer:0")
+    self.assertEqual(m2.name, "has space_1")
+    self.assertEqual(m2.numer.name, "has_space_1/numer:0")
+
+  def testGraph(self):
+    with context.graph_mode(), self.test_session() as sess:
+      m = metrics.Mean()
+      p = array_ops.placeholder(dtypes.float32)
+      accumulate = m(p)
+      variables.global_variables_initializer().run()
+      sess.run(accumulate, feed_dict={p: [1, 10, 100]})
+      sess.run(accumulate, feed_dict={p: 1000})
+      sess.run(accumulate, feed_dict={p: [10000, 100000]})
+      self.assertAllEqual(m.result().eval(), 111111.0/6)
+
+  def testTwoMeansGraph(self):
+    # Verify two metrics with the same class and name don't
+    # accidentally share state.
+    with context.graph_mode(), self.test_session() as sess:
+      m1 = metrics.Mean()
+      m2 = metrics.Mean()
+      accumulate1 = m1(0)
+      accumulate2 = m2(2)
+      variables.global_variables_initializer().run()
+      sess.run([accumulate1, accumulate2])
+      self.assertEqual(0, m1.result().eval())
+      self.assertEqual(2, m2.result().eval())
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py
index da49517cf9..e675ee8988 100644
--- a/tensorflow/python/eager/function.py
+++ b/tensorflow/python/eager/function.py
@@ -79,6 +79,22 @@ def capture_tensors(captures):
     _scoped_captures.tensors = old
 
 
+def capture_value(tensor_map, value, dtype, name):
+  """Capture a value from outside the function, to pass in as an extra arg."""
+  captured_value = tensor_map.get(ops.tensor_id(value), None)
+  if captured_value is None:
+    captured_value = graph_placeholder(
+        dtype=dtype or value.dtype, shape=value.shape, name=name)
+    if captured_value.dtype == dtypes.resource:
+      captured_value._handle_data = value._handle_data  # pylint: disable=protected-access
+    tensor_map[ops.tensor_id(value)] = (value, captured_value)
+  else:
+    captured_value = captured_value[1]
+  tape.record_operation("captured_value", [captured_value], [value],
+                        lambda x: [x])
+  return captured_value
+
+
 def _convert_to_graph_tensor(value, dtype=None, name=None, as_ref=False):
   """Captures a Tensor while building a graph mode function.
 
@@ -100,18 +116,33 @@ def _convert_to_graph_tensor(value, dtype=None, name=None, as_ref=False):
   if tensor_map is None:
     # Capturing is not enabled.
     return constant_op.constant(value.numpy())
-  captured_value = tensor_map.get(ops.tensor_id(value), None)
-  if captured_value is None:
-    captured_value = graph_placeholder(
-        dtype=dtype or value.dtype, shape=value.shape, name=name)
-    if captured_value.dtype == dtypes.resource:
-      captured_value._handle_data = value._handle_data  # pylint: disable=protected-access
-    tensor_map[ops.tensor_id(value)] = (value, captured_value)
-  else:
-    captured_value = captured_value[1]
-  tape.record_operation("captured_value", [captured_value], [value],
-                        lambda x: [x])
-  return captured_value
+  return capture_value(tensor_map, value, dtype, name)
+
+
+class CapturingGraph(ops.Graph):
+
+  def __init__(self, captures):
+    super(CapturingGraph, self).__init__()
+    self._building_function = True
+    self.captures = captures
+
+  def create_op(
+      self,
+      op_type,
+      inputs,
+      dtypes,  # pylint: disable=redefined-outer-name
+      input_types=None,
+      name=None,
+      attrs=None,
+      op_def=None,
+      compute_shapes=True,
+      compute_device=True):
+    for i, inp in enumerate(inputs):
+      if inp.graph is not self:
+        inputs[i] = capture_value(self.captures, inp, inp.dtype, inp.op.name)
+    return super(CapturingGraph, self).create_op(
+        op_type, inputs, dtypes, input_types, name, attrs, op_def,
+        compute_shapes, compute_device)
 
 
 # TODO(apassos): it'd be really nice if we could scope this registration.
@@ -325,6 +356,8 @@ class _GraphModeFunction(object):
           name="FunctionCall",
           compute_shapes=False)
       result = op.outputs
+      if not result:
+        return op
       for i, s in enumerate(self._output_shapes):
         result[i].set_shape(s)
     else:
@@ -381,7 +414,8 @@ def _get_defun_inputs(args):
 def _defun_internal(name, func, args, kwds):
   """Defines and returns graph-mode version of func."""
   with context.graph_mode():
-    tmp_graph = ops.Graph()
+    captures = {}
+    tmp_graph = CapturingGraph(captures)
     # Copy the graph collections to ensure summaries and other things work. This
     # lets the function access (but not mutate) collections of the containing
     # graph, such as the global step and the summary writer collections.
@@ -392,7 +426,6 @@ def _defun_internal(name, func, args, kwds):
     with tmp_graph.as_default():
       func_inputs = _get_defun_inputs(args)
 
-      captures = {}
       with capture_tensors(captures):
         func_outputs = func(*func_inputs, **kwds)
       ids = list(sorted(captures.keys()))
diff --git a/tensorflow/python/eager/function_test.py b/tensorflow/python/eager/function_test.py
index fb647f5c21..a4c351e8c9 100644
--- a/tensorflow/python/eager/function_test.py
+++ b/tensorflow/python/eager/function_test.py
@@ -32,6 +32,7 @@ from tensorflow.python.ops import clip_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.ops import variable_scope
+from tensorflow.python.ops import variables
 
 
 class FunctionTest(test.TestCase):
@@ -68,6 +69,23 @@ class FunctionTest(test.TestCase):
 
     self.assertAllEqual(step(), 2.0)
 
+  def testGraphModeCaptureVariable(self):
+    with context.graph_mode(), self.test_session() as sess:
+
+      class HasAVar(object):
+
+        def __init__(self):
+          self.v = resource_variable_ops.ResourceVariable(1.0)
+
+        def call(self):
+          return self.v * 2
+
+      o = HasAVar()
+      variables.global_variables_initializer().run()
+      call = function.defun(o.call)
+      op = call()
+      self.assertAllEqual(sess.run(op), 2.0)
+
   def testTensorConversionWithDefun(self):
 
     @function.defun
diff --git a/tensorflow/tools/ci_build/ci_sanity.sh b/tensorflow/tools/ci_build/ci_sanity.sh
index 4e72d025a2..1703cae1e5 100755
--- a/tensorflow/tools/ci_build/ci_sanity.sh
+++ b/tensorflow/tools/ci_build/ci_sanity.sh
@@ -95,6 +95,7 @@ do_pylint() {
 "^tensorflow/python/platform/default/_googletest\.py.*\[E0102.*function\salready\sdefined "\
 "^tensorflow/python/feature_column/feature_column_test\.py.*\[E0110.*abstract-class-instantiated "\
 "^tensorflow/contrib/layers/python/layers/feature_column\.py.*\[E0110.*abstract-class-instantiated "\
+"^tensorflow/contrib/eager/python/metrics_impl\.py.*\[E0202.*method-hidden "\
 "^tensorflow/python/platform/gfile\.py.*\[E0301.*non-iterator "\
 "^tensorflow/python/keras/_impl/keras/callbacks\.py.*\[E1133.*not-an-iterable"
 
-- 
GitLab


From 3715cffc6e2338cf2fc6ad6aba5c1d00ce598bfd Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Thu, 19 Oct 2017 18:27:01 -0700
Subject: [PATCH 595/909] Internal change.

PiperOrigin-RevId: 172829126
---
 tensorflow/core/framework/api_def.proto      |  5 +-
 tensorflow/core/framework/op_gen_lib.cc      | 18 +++++++
 tensorflow/core/framework/op_gen_lib_test.cc | 50 ++++++++++++++++++--
 3 files changed, 68 insertions(+), 5 deletions(-)

diff --git a/tensorflow/core/framework/api_def.proto b/tensorflow/core/framework/api_def.proto
index 987caee250..98c38efc0e 100644
--- a/tensorflow/core/framework/api_def.proto
+++ b/tensorflow/core/framework/api_def.proto
@@ -51,7 +51,8 @@ message ApiDef {
   // endpoints are deprecated).
   message Endpoint {
     // Name should be either like "CamelCaseName" or
-    // "Package.CamelCaseName".
+    // "Package.CamelCaseName". Client-language-specific ApiDefs may
+    // use a snake_case convention instead of CamelCase.
     string name = 1;
 
     // First GraphDef version at which the op is disallowed.
@@ -74,7 +75,7 @@ message ApiDef {
   }
   repeated Arg in_arg = 4;
   repeated Arg out_arg = 5;
-  // List of post-rename in_arg names to specify new argument order.
+  // List of original in_arg names to specify new argument order.
   // Length of arg_order should be either empty to keep current order
   // or match size of in_arg.
   repeated string arg_order = 11;
diff --git a/tensorflow/core/framework/op_gen_lib.cc b/tensorflow/core/framework/op_gen_lib.cc
index cfaca897ba..1e93e9be09 100644
--- a/tensorflow/core/framework/op_gen_lib.cc
+++ b/tensorflow/core/framework/op_gen_lib.cc
@@ -412,6 +412,8 @@ void InitApiDefFromOpDef(const OpDef& op_def, ApiDef* api_def) {
     api_in_arg->set_name(op_in_arg.name());
     api_in_arg->set_rename_to(op_in_arg.name());
     api_in_arg->set_description(op_in_arg.description());
+
+    *api_def->add_arg_order() = op_in_arg.name();
   }
   for (const auto& op_out_arg : op_def.output_arg()) {
     auto* api_out_arg = api_def->add_out_arg();
@@ -503,6 +505,22 @@ Status MergeApiDefs(ApiDef* base_api_def, const ApiDef& new_api_def) {
   }
   // Merge arg order
   if (new_api_def.arg_order_size() > 0) {
+    // Validate that new arg_order is correct.
+    if (new_api_def.arg_order_size() != base_api_def->arg_order_size()) {
+      return errors::FailedPrecondition(
+          "Invalid number of arguments ", new_api_def.arg_order_size(), " for ",
+          base_api_def->graph_op_name(),
+          ". Expected: ", base_api_def->arg_order_size());
+    }
+    if (!std::is_permutation(new_api_def.arg_order().begin(),
+                             new_api_def.arg_order().end(),
+                             base_api_def->arg_order().begin())) {
+      return errors::FailedPrecondition(
+          "Invalid arg_order: ", str_util::Join(new_api_def.arg_order(), ", "),
+          " for ", base_api_def->graph_op_name(),
+          ". All elements in arg_order override must match base arg_order: ",
+          str_util::Join(base_api_def->arg_order(), ", "));
+    }
     base_api_def->clear_arg_order();
     std::copy(
         new_api_def.arg_order().begin(), new_api_def.arg_order().end(),
diff --git a/tensorflow/core/framework/op_gen_lib_test.cc b/tensorflow/core/framework/op_gen_lib_test.cc
index b7ee6db991..da9b4dfbb1 100644
--- a/tensorflow/core/framework/op_gen_lib_test.cc
+++ b/tensorflow/core/framework/op_gen_lib_test.cc
@@ -207,6 +207,8 @@ attr {
   name: "attr_a"
   rename_to: "attr_a"
 }
+arg_order: "arg_a"
+arg_order: "arg_b"
 )";
   OpList op_list;
   protobuf::TextFormat::ParseFromString(kTestOpList, &op_list);  // NOLINT
@@ -331,8 +333,8 @@ op {
     name: "arg_c"
     rename_to: "arg_cc"
   }
-  arg_order: "arg_aa"
   arg_order: "arg_b"
+  arg_order: "arg_a"
 }
 )";
   OpList op_list;
@@ -351,8 +353,8 @@ op {
   EXPECT_EQ("arg_cc", api_def->out_arg(0).rename_to());
 
   ASSERT_EQ(2, api_def->arg_order_size());
-  EXPECT_EQ("arg_aa", api_def->arg_order(0));
-  EXPECT_EQ("arg_b", api_def->arg_order(1));
+  EXPECT_EQ("arg_b", api_def->arg_order(0));
+  EXPECT_EQ("arg_a", api_def->arg_order(1));
 }
 
 TEST(OpGenLibTest, ApiDefOverrideDescriptions) {
@@ -411,5 +413,47 @@ op {
   auto status = api_map.LoadApiDef(api_def1);
   ASSERT_EQ(tensorflow::error::FAILED_PRECONDITION, status.code());
 }
+
+TEST(OpGenLibTest, ApiDefInvalidArgOrder) {
+  const string api_def1 = R"(
+op {
+  graph_op_name: "testop"
+  arg_order: "arg_a"
+  arg_order: "unexpected_arg"
+}
+)";
+
+  const string api_def2 = R"(
+op {
+  graph_op_name: "testop"
+  arg_order: "arg_a"
+}
+)";
+
+  const string api_def3 = R"(
+op {
+  graph_op_name: "testop"
+  arg_order: "arg_a"
+  arg_order: "arg_a"
+}
+)";
+
+  OpList op_list;
+  protobuf::TextFormat::ParseFromString(kTestOpList, &op_list);  // NOLINT
+  ApiDefMap api_map(op_list);
+  TF_CHECK_OK(api_map.LoadApiDef(kTestApiDef));
+
+  // Loading with incorrect arg name in arg_order should fail.
+  auto status = api_map.LoadApiDef(api_def1);
+  ASSERT_EQ(tensorflow::error::FAILED_PRECONDITION, status.code());
+
+  // Loading with incorrect number of args in arg_order should fail.
+  status = api_map.LoadApiDef(api_def2);
+  ASSERT_EQ(tensorflow::error::FAILED_PRECONDITION, status.code());
+
+  // Loading with the same argument twice in arg_order should fail.
+  status = api_map.LoadApiDef(api_def3);
+  ASSERT_EQ(tensorflow::error::FAILED_PRECONDITION, status.code());
+}
 }  // namespace
 }  // namespace tensorflow
-- 
GitLab


From 0671c0b2546dbea87e231d336d5f4c0573a01964 Mon Sep 17 00:00:00 2001
From: David Soergel <soergel@google.com>
Date: Thu, 19 Oct 2017 19:01:19 -0700
Subject: [PATCH 596/909] Usability improvements regarding export signature
 generation.

* Log report of which signatures are produced and which TF Serving APIs are targeted.
* Improve docstrings for signature_def builders, explaining the TF Serving API constraints.
* Accept a single Tensor as a prediction output (which will be named 'output').

PiperOrigin-RevId: 172831366
---
 tensorflow/python/estimator/export/export.py  | 56 +++++++++++++++++--
 .../python/estimator/export/export_output.py  | 10 ++--
 .../estimator/export/export_output_test.py    | 14 ++---
 .../saved_model/signature_def_utils_impl.py   | 23 ++++++--
 4 files changed, 80 insertions(+), 23 deletions(-)

diff --git a/tensorflow/python/estimator/export/export.py b/tensorflow/python/estimator/export/export.py
index e2e20f0d71..31e9933c6f 100644
--- a/tensorflow/python/estimator/export/export.py
+++ b/tensorflow/python/estimator/export/export.py
@@ -33,6 +33,7 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import parsing_ops
 from tensorflow.python.platform import gfile
 from tensorflow.python.platform import tf_logging as logging
+from tensorflow.python.saved_model import signature_constants
 from tensorflow.python.saved_model import signature_def_utils
 from tensorflow.python.util import compat
 
@@ -47,8 +48,8 @@ class ServingInputReceiver(collections.namedtuple(
   """A return type for a serving_input_receiver_fn.
 
   The expected return values are:
-    features: A dict of string to `Tensor` or `SparseTensor`, specifying the
-      features to be passed to the model.
+    features: A `Tensor`, `SparseTensor`, or dict of string to `Tensor` or
+      `SparseTensor`, specifying the features to be passed to the model.
     receiver_tensors: a `Tensor`, or a dict of string to `Tensor`, specifying
       input nodes where this receiver expects to be fed by default.  Typically,
       this is a single placeholder expecting serialized `tf.Example` protos.
@@ -193,13 +194,14 @@ def build_all_signature_defs(receiver_tensors,
     raise ValueError('export_outputs must be a dict.')
 
   signature_def_map = {}
+  excluded_signatures = {}
   for output_key, export_output in export_outputs.items():
     signature_name = '{}'.format(output_key or 'None')
     try:
       signature = export_output.as_signature_def(receiver_tensors)
       signature_def_map[signature_name] = signature
-    except ValueError:
-      pass
+    except ValueError as e:
+      excluded_signatures[signature_name] = str(e)
 
   if receiver_tensors_alternatives:
     for receiver_name, receiver_tensors_alt in (
@@ -213,8 +215,10 @@ def build_all_signature_defs(receiver_tensors,
         try:
           signature = export_output.as_signature_def(receiver_tensors_alt)
           signature_def_map[signature_name] = signature
-        except ValueError:
-          pass
+        except ValueError as e:
+          excluded_signatures[signature_name] = str(e)
+
+  _log_signature_report(signature_def_map, excluded_signatures)
 
   # The above calls to export_output.as_signature_def should return only
   # valid signatures; if there is a validity problem, they raise ValueError,
@@ -224,6 +228,46 @@ def build_all_signature_defs(receiver_tensors,
           if signature_def_utils.is_valid_signature(v)}
 
 
+_FRIENDLY_METHOD_NAMES = {
+    signature_constants.CLASSIFY_METHOD_NAME: 'Classify',
+    signature_constants.REGRESS_METHOD_NAME: 'Regress',
+    signature_constants.PREDICT_METHOD_NAME: 'Predict',
+}
+
+
+def _log_signature_report(signature_def_map, excluded_signatures):
+  """Log a report of which signatures were produced."""
+  sig_names_by_method_name = collections.defaultdict(list)
+
+  # We'll collect whatever method_names are present, but also we want to make
+  # sure to output a line for each of the three standard methods even if they
+  # have no signatures.
+  for method_name in _FRIENDLY_METHOD_NAMES:
+    sig_names_by_method_name[method_name] = []
+
+  for signature_name, sig in signature_def_map.items():
+    sig_names_by_method_name[sig.method_name].append(signature_name)
+
+  # TODO(b/67733540): consider printing the full signatures, not just names
+  for method_name, sig_names in sig_names_by_method_name.items():
+    if method_name in _FRIENDLY_METHOD_NAMES:
+      method_name = _FRIENDLY_METHOD_NAMES[method_name]
+    logging.info('Signatures INCLUDED in export for {}: {}'.format(
+        method_name, sig_names if sig_names else 'None'))
+
+  if excluded_signatures:
+    logging.info('Signatures EXCLUDED from export because they cannot be '
+                 'be served via TensorFlow Serving APIs:')
+    for signature_name, message in excluded_signatures.items():
+      logging.info('\'{}\' : {}'.format(signature_name, message))
+
+  if not signature_def_map:
+    logging.warn('Export includes no signatures!')
+  elif (signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY
+        not in signature_def_map):
+    logging.warn('Export includes no default signature!')
+
+
 # When we create a timestamped directory, there is a small chance that the
 # directory already exists because another worker is also writing exports.
 # In this case we just wait one second to get a new timestamp and try again.
diff --git a/tensorflow/python/estimator/export/export_output.py b/tensorflow/python/estimator/export/export_output.py
index 7c7f92872e..863af6d41d 100644
--- a/tensorflow/python/estimator/export/export_output.py
+++ b/tensorflow/python/estimator/export/export_output.py
@@ -150,6 +150,9 @@ class RegressionOutput(ExportOutput):
     return signature_def_utils.regression_signature_def(examples, self.value)
 
 
+_SINGLE_OUTPUT_DEFAULT_NAME = 'output'
+
+
 class PredictOutput(ExportOutput):
   """Represents the output of a generic prediction head.
 
@@ -162,16 +165,15 @@ class PredictOutput(ExportOutput):
     """Constructor for PredictOutput.
 
     Args:
-      outputs: A dict of string to `Tensor` representing the predictions.
+      outputs: A `Tensor` or a dict of string to `Tensor` representing the
+        predictions.
 
     Raises:
       ValueError: if the outputs is not dict, or any of its keys are not
           strings, or any of its values are not `Tensor`s.
     """
     if not isinstance(outputs, dict):
-      raise ValueError(
-          'Prediction outputs must be given as a dict of string to Tensor; '
-          'got {}'.format(outputs))
+      outputs = {_SINGLE_OUTPUT_DEFAULT_NAME: outputs}
     for key, value in outputs.items():
       if not isinstance(key, six.string_types):
         raise ValueError(
diff --git a/tensorflow/python/estimator/export/export_output_test.py b/tensorflow/python/estimator/export/export_output_test.py
index 035a9a143e..7090e53d80 100644
--- a/tensorflow/python/estimator/export/export_output_test.py
+++ b/tensorflow/python/estimator/export/export_output_test.py
@@ -199,20 +199,18 @@ class ExportOutputTest(test.TestCase):
         signature_constants.CLASSIFY_METHOD_NAME)
     self.assertEqual(actual_signature_def, expected_signature_def)
 
-  def test_predict_output_constructor(self):
-    """Tests that no errors are raised when input is expected."""
+  def test_predict_outputs_valid(self):
+    """Tests that no errors are raised when provided outputs are valid."""
     outputs = {
         "output0": constant_op.constant([0]),
-        u"output1": constant_op.constant([1]),
+        u"output1": constant_op.constant(["foo"]),
     }
     export_output_lib.PredictOutput(outputs)
 
-  def test_predict_output_outputs_invalid(self):
-    with self.assertRaisesRegexp(
-        ValueError,
-        "Prediction outputs must be given as a dict of string to Tensor"):
-      export_output_lib.PredictOutput(constant_op.constant([0]))
+    # Single Tensor is OK too
+    export_output_lib.PredictOutput(constant_op.constant([0]))
 
+  def test_predict_outputs_invalid(self):
     with self.assertRaisesRegexp(
         ValueError,
         "Prediction output key must be a string"):
diff --git a/tensorflow/python/saved_model/signature_def_utils_impl.py b/tensorflow/python/saved_model/signature_def_utils_impl.py
index 564befeb0b..240ea61aa5 100644
--- a/tensorflow/python/saved_model/signature_def_utils_impl.py
+++ b/tensorflow/python/saved_model/signature_def_utils_impl.py
@@ -56,9 +56,13 @@ def build_signature_def(inputs=None, outputs=None, method_name=None):
 def regression_signature_def(examples, predictions):
   """Creates regression signature from given examples and predictions.
 
+  This function produces signatures intended for use with the TensorFlow Serving
+  Regress API (tensorflow_serving/apis/prediction_service.proto), and so
+  constrains the input and output types to those allowed by TensorFlow Serving.
+
   Args:
-    examples: `Tensor`.
-    predictions: `Tensor`.
+    examples: A string `Tensor`, expected to accept serialized tf.Examples.
+    predictions: A float `Tensor`.
 
   Returns:
     A regression-flavored signature_def.
@@ -93,10 +97,15 @@ def regression_signature_def(examples, predictions):
 def classification_signature_def(examples, classes, scores):
   """Creates classification signature from given examples and predictions.
 
+  This function produces signatures intended for use with the TensorFlow Serving
+  Classify API (tensorflow_serving/apis/prediction_service.proto), and so
+  constrains the input and output types to those allowed by TensorFlow Serving.
+
   Args:
-    examples: `Tensor`.
-    classes: `Tensor`.
-    scores: `Tensor`.
+    examples: A string `Tensor`, expected to accept serialized tf.Examples.
+    classes: A string `Tensor`.  Note that the ClassificationResponse message
+      requires that class labels are strings, not integers or anything else.
+    scores: a float `Tensor`.
 
   Returns:
     A classification-flavored signature_def.
@@ -140,6 +149,10 @@ def classification_signature_def(examples, classes, scores):
 def predict_signature_def(inputs, outputs):
   """Creates prediction signature from given inputs and outputs.
 
+  This function produces signatures intended for use with the TensorFlow Serving
+  Predict API (tensorflow_serving/apis/prediction_service.proto). This API
+  imposes no constraints on the input and output types.
+
   Args:
     inputs: dict of string to `Tensor`.
     outputs: dict of string to `Tensor`.
-- 
GitLab


From c2f91136bdfbc0103f01a932566ef46ad4ba9054 Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Thu, 19 Oct 2017 19:14:44 -0700
Subject: [PATCH 597/909] Fixes build breakage (#13843)

---
 .../contrib/framework/python/ops/accumulate_n_v2_eager_test.py  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/framework/python/ops/accumulate_n_v2_eager_test.py b/tensorflow/contrib/framework/python/ops/accumulate_n_v2_eager_test.py
index 8c618838bf..f3453f89fa 100644
--- a/tensorflow/contrib/framework/python/ops/accumulate_n_v2_eager_test.py
+++ b/tensorflow/contrib/framework/python/ops/accumulate_n_v2_eager_test.py
@@ -79,6 +79,6 @@ class AccumulateNV2EagerTest(test_util.TensorFlowTestCase):
 
 
 if __name__ == "__main__":
-  eager_context.enable_eager_execution()
+  ops.enable_eager_execution()
   test.main()
 
-- 
GitLab


From 932a68370cf3fc076b66f918d1745bce40030f43 Mon Sep 17 00:00:00 2001
From: Mahdi Abavisani <mahdi.abavisani@rutgers.edu>
Date: Thu, 19 Oct 2017 22:20:44 -0400
Subject: [PATCH 598/909] Update resnet.py (#13828)

Test with mnist test set. Previously it was testing on the training set.
---
 tensorflow/examples/learn/resnet.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/examples/learn/resnet.py b/tensorflow/examples/learn/resnet.py
index 33a09bb6e0..1e0966475b 100755
--- a/tensorflow/examples/learn/resnet.py
+++ b/tensorflow/examples/learn/resnet.py
@@ -190,8 +190,8 @@ def main(unused_args):
 
   # Calculate accuracy.
   test_input_fn = tf.estimator.inputs.numpy_input_fn(
-      x={X_FEATURE: mnist.train.images},
-      y=mnist.train.labels.astype(np.int32),
+      x={X_FEATURE: mnist.test.images},
+      y=mnist.test.labels.astype(np.int32),
       num_epochs=1,
       shuffle=False)
   scores = classifier.evaluate(input_fn=test_input_fn)
-- 
GitLab


From bbf1985db1e1b9cddaa04819e29f98f81600f49c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jeroen=20B=C3=A9dorf?= <jeroen@bedorf.net>
Date: Fri, 20 Oct 2017 04:27:14 +0200
Subject: [PATCH 599/909] Fix MPI and Verbs compilation when not using GPUs
 (#13800)

* Fix build errors when building without CUDA support, but with MPI / verbs support

* Fix buildifier errors

* Moved stream_executor dependency after PR feedback
---
 tensorflow/core/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 6e434ef49d..013ed2e8fd 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -2179,6 +2179,7 @@ tf_cuda_library(
         ":lib",
         ":lib_internal",
         ":protos_all_cc",
+        ":stream_executor",
         "//third_party/eigen3",
     ] + if_static([":gpu_runtime_impl"]),
 )
-- 
GitLab


From 58121b8b13597d3285f121f02bd2a512bc76be17 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 19 Oct 2017 20:09:38 -0700
Subject: [PATCH 600/909] Pull out a non-test-only class HloRunnerBase from
 HloTestBase so that it can be used as a library for running HloModule on
 given platform. Also add a function to read HloModule from a HloProto file,
 and a function to make fake input literals for given HloModule.

PiperOrigin-RevId: 172835863
---
 tensorflow/compiler/xla/service/BUILD         |  23 ++
 tensorflow/compiler/xla/service/hlo_runner.cc | 199 ++++++++++++++++++
 tensorflow/compiler/xla/service/hlo_runner.h  | 100 +++++++++
 tensorflow/compiler/xla/tests/BUILD           |  12 +-
 .../compiler/xla/tests/hlo_test_base.cc       | 114 +---------
 tensorflow/compiler/xla/tests/hlo_test_base.h |  26 +--
 6 files changed, 335 insertions(+), 139 deletions(-)
 create mode 100644 tensorflow/compiler/xla/service/hlo_runner.cc
 create mode 100644 tensorflow/compiler/xla/service/hlo_runner.h

diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD
index 1ef329365e..8f5105aa53 100644
--- a/tensorflow/compiler/xla/service/BUILD
+++ b/tensorflow/compiler/xla/service/BUILD
@@ -2066,6 +2066,29 @@ tf_cc_test(
     ],
 )
 
+cc_library(
+    name = "hlo_runner",
+    srcs = ["hlo_runner.cc"],
+    hdrs = ["hlo_runner.h"],
+    deps = [
+        ":executable",
+        ":hlo",
+        ":transfer_manager",
+        "//tensorflow/compiler/xla:shape_util",
+        "//tensorflow/compiler/xla:status_macros",
+        "//tensorflow/compiler/xla:statusor",
+        "//tensorflow/compiler/xla:types",
+        "//tensorflow/compiler/xla:util",
+        "//tensorflow/compiler/xla:xla_data_proto",
+        "//tensorflow/compiler/xla/service:backend",
+        "//tensorflow/compiler/xla/service:compiler",
+        "//tensorflow/core:core_cpu_internal",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:stream_executor_no_cuda",
+        "//third_party/eigen3",
+    ],
+)
+
 # -----------------------------------------------------------------------------
 
 filegroup(
diff --git a/tensorflow/compiler/xla/service/hlo_runner.cc b/tensorflow/compiler/xla/service/hlo_runner.cc
new file mode 100644
index 0000000000..d5d7042a02
--- /dev/null
+++ b/tensorflow/compiler/xla/service/hlo_runner.cc
@@ -0,0 +1,199 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/service/hlo_runner.h"
+
+#include <set>
+#include <string>
+#include <utility>
+
+#define EIGEN_USE_THREADS
+
+#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "tensorflow/compiler/xla/layout_util.h"
+#include "tensorflow/compiler/xla/ptr_util.h"
+#include "tensorflow/compiler/xla/service/backend.h"
+#include "tensorflow/compiler/xla/service/executable.h"
+#include "tensorflow/compiler/xla/service/hlo_computation.h"
+#include "tensorflow/compiler/xla/service/transfer_manager.h"
+#include "tensorflow/compiler/xla/shape_util.h"
+#include "tensorflow/compiler/xla/status_macros.h"
+#include "tensorflow/compiler/xla/statusor.h"
+#include "tensorflow/compiler/xla/types.h"
+#include "tensorflow/core/common_runtime/eigen_thread_pool.h"
+#include "tensorflow/core/platform/logging.h"
+#include "tensorflow/core/platform/types.h"
+
+namespace se = ::perftools::gputools;
+
+namespace xla {
+
+/*static*/ StatusOr<std::unique_ptr<HloModule>>
+HloRunner::ReadModuleFromHloProtoFile(const char* filename,
+                                      const DebugOptions& debug_options) {
+  HloProto proto;
+  TF_RETURN_IF_ERROR(tensorflow::ReadBinaryProto(tensorflow::Env::Default(),
+                                                 filename, &proto));
+  HloModuleConfig config;
+  config.set_debug_options(debug_options);
+  TF_ASSIGN_OR_RETURN(auto module, HloModule::CreateFromProto(
+                                       proto.hlo_module(),
+                                       VersionedComputationHandle(), config));
+  return std::move(module);
+}
+
+// Define this in .cc file to avoid having to include eigen or forward declare
+// these types in the header.
+struct HloRunner::EigenThreadPoolWrapper {
+  std::unique_ptr<EigenThreadPoolWrapper> pool;
+  std::unique_ptr<Eigen::ThreadPoolDevice> device;
+};
+
+HloRunner::HloRunner() {}
+
+HloRunner::HloRunner(se::Platform* platform) {
+  BackendOptions backend_options;
+  backend_options.set_platform(platform);
+  backend_ = Backend::CreateBackend(backend_options).ConsumeValueOrDie();
+  VLOG(1) << "Created HloRunner for platform: " << platform->Name();
+}
+
+HloRunner::~HloRunner() {
+  // Deallocate all the memory allocated during the tests.
+  for (auto& allocation : allocations_) {
+    backend().default_stream_executor()->Deallocate(&allocation);
+  }
+}
+
+StatusOr<se::DeviceMemoryBase> HloRunner::Execute(
+    std::unique_ptr<HloModule> module,
+    tensorflow::gtl::ArraySlice<se::DeviceMemoryBase> arguments,
+    Shape* result_shape) {
+  TF_ASSIGN_OR_RETURN(
+      std::unique_ptr<Executable> executable,
+      backend().compiler()->Compile(std::move(module),
+                                    backend().default_stream_executor()));
+
+  se::Stream stream(backend().default_stream_executor());
+  stream.Init();
+
+  ExecutableRunOptions run_options;
+  run_options.set_stream(&stream);
+  run_options.set_allocator(backend().memory_allocator());
+  run_options.set_inter_op_thread_pool(backend().inter_op_thread_pool());
+  run_options.set_intra_op_thread_pool(
+      backend().eigen_intra_op_thread_pool_device());
+
+  HloExecutionProfile hlo_execution_profile;
+  ServiceExecutableRunOptions service_run_options(
+      run_options, backend().StreamBorrower(),
+      backend().inter_op_thread_pool());
+  TF_ASSIGN_OR_RETURN(
+      se::DeviceMemoryBase result,
+      executable->ExecuteOnStream(&service_run_options, arguments,
+                                  &hlo_execution_profile));
+  TF_RET_CHECK(stream.BlockHostUntilDone());
+
+  allocations_.push_back(result);
+
+  *result_shape = executable->result_shape();
+
+  if (ShapeUtil::IsTuple(*result_shape)) {
+    // We must record element buffers of tuples as well to avoid leaks.
+    DCHECK(!ShapeUtil::IsNestedTuple(*result_shape));
+    TF_ASSIGN_OR_RETURN(
+        std::vector<se::DeviceMemoryBase> element_buffers,
+        backend().transfer_manager()->ShallowCopyTupleFromDevice(
+            backend().default_stream_executor(), result, *result_shape));
+
+    // A tuple may contain the same buffer in more than one element. Keep track
+    // of the buffers already added to avoid duplicates in allocations_.
+    std::set<void*> added_opaques;
+    for (auto element_buffer : element_buffers) {
+      if (added_opaques.count(element_buffer.opaque()) == 0) {
+        CHECK(element_buffer.opaque() != nullptr);
+        added_opaques.insert(element_buffer.opaque());
+        allocations_.push_back(element_buffer);
+      }
+    }
+  }
+
+  return result;
+}
+
+se::DeviceMemoryBase HloRunner::TransferToDevice(const Literal& literal) {
+  // Allocate memory on the device using the stream executor.
+  int64 allocation_size =
+      backend().transfer_manager()->GetByteSizeRequirement(literal.shape());
+  se::DeviceMemoryBase allocation =
+      backend().default_stream_executor()->AllocateArray<uint8>(
+          allocation_size);
+  allocations_.push_back(allocation);
+
+  TF_CHECK_OK(backend().transfer_manager()->TransferLiteralToDevice(
+      backend().default_stream_executor(), literal, &allocation));
+
+  return allocation;
+}
+
+std::unique_ptr<Literal> HloRunner::TransferFromDevice(
+    const Shape& shape, se::DeviceMemoryBase device_base) {
+  auto literal = MakeUnique<Literal>();
+  TF_CHECK_OK(backend().transfer_manager()->TransferLiteralFromDevice(
+      backend().default_stream_executor(), device_base, shape, shape,
+      literal.get()));
+  return literal;
+}
+
+std::unique_ptr<Literal> HloRunner::ExecuteAndTransfer(
+    std::unique_ptr<HloModule> module,
+    tensorflow::gtl::ArraySlice<se::DeviceMemoryBase> arguments) {
+  Shape result_shape;
+  se::DeviceMemoryBase device_base =
+      Execute(std::move(module), arguments, &result_shape).ValueOrDie();
+  return TransferFromDevice(result_shape, device_base);
+}
+
+template <>
+std::unique_ptr<Literal> HloRunner::Execute(
+    std::unique_ptr<HloModule> module,
+    const tensorflow::gtl::ArraySlice<std::unique_ptr<Literal>>& literals) {
+  std::vector<se::DeviceMemoryBase> arguments;
+  for (const auto& literal : literals) {
+    arguments.push_back(TransferToDevice(*literal));
+  }
+  return ExecuteAndTransfer(std::move(module), arguments);
+}
+
+template <>
+std::unique_ptr<Literal> HloRunner::Execute(
+    std::unique_ptr<HloModule> module,
+    const tensorflow::gtl::ArraySlice<Literal*>& literals) {
+  std::vector<se::DeviceMemoryBase> arguments;
+  for (const auto& literal : literals) {
+    arguments.push_back(TransferToDevice(*literal));
+  }
+  return ExecuteAndTransfer(std::move(module), arguments);
+}
+
+Backend& HloRunner::backend() {
+  if (!backend_) {
+    backend_ = Backend::CreateDefaultBackend().ConsumeValueOrDie();
+    VLOG(1) << "executing on platform " << backend().platform()->Name();
+  }
+  return *backend_;
+}
+
+}  // namespace xla
diff --git a/tensorflow/compiler/xla/service/hlo_runner.h b/tensorflow/compiler/xla/service/hlo_runner.h
new file mode 100644
index 0000000000..d74a1b59a8
--- /dev/null
+++ b/tensorflow/compiler/xla/service/hlo_runner.h
@@ -0,0 +1,100 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_COMPILER_XLA_SERVICE_HLO_RUNNER_H_
+#define TENSORFLOW_COMPILER_XLA_SERVICE_HLO_RUNNER_H_
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "tensorflow/compiler/xla/service/backend.h"
+#include "tensorflow/compiler/xla/service/compiler.h"
+#include "tensorflow/compiler/xla/service/hlo_computation.h"
+#include "tensorflow/compiler/xla/service/hlo_module.h"
+#include "tensorflow/compiler/xla/statusor.h"
+#include "tensorflow/compiler/xla/types.h"
+#include "tensorflow/compiler/xla/xla_data.pb.h"
+#include "tensorflow/core/lib/gtl/array_slice.h"
+#include "tensorflow/core/platform/stream_executor_no_cuda.h"
+
+namespace xla {
+
+// A base class for running an HloModule. This executes the given HloModule on a
+// certain backend directly without using the client interface. HloModule can be
+// explicitly built, or loaded from a serialization file (e.g., hlo proto file).
+class HloRunner {
+ public:
+  HloRunner();
+
+  HloRunner(::perftools::gputools::Platform* platform);
+
+  ~HloRunner();
+
+  // Reads the binary proto file in xla.HloProto format, creates and returns the
+  // HloModule.
+  static StatusOr<std::unique_ptr<HloModule>> ReadModuleFromHloProtoFile(
+      const char* filename, const DebugOptions& debug_options);
+
+  // Executes the given module with given literals as input and returns the
+  // result as a Literal. The LiteralPtr type accepts Literal* or
+  // std::unique_ptr<Literal>.
+  template <typename LiteralPtr>
+  std::unique_ptr<Literal> Execute(
+      std::unique_ptr<HloModule> module,
+      const tensorflow::gtl::ArraySlice<LiteralPtr>& literals);
+
+  // Executes the given module and returns a global data handle.
+  StatusOr<perftools::gputools::DeviceMemoryBase> Execute(
+      std::unique_ptr<HloModule> module,
+      tensorflow::gtl::ArraySlice<perftools::gputools::DeviceMemoryBase>
+          arguments,
+      Shape* result_shape);
+
+  // Transfers the given literal to the device and returns the data handle.
+  perftools::gputools::DeviceMemoryBase TransferToDevice(
+      const Literal& literal);
+
+  // Transfers the array referred to by the given handle from the device and
+  // returns as a Literal.
+  std::unique_ptr<Literal> TransferFromDevice(
+      const Shape& shape, perftools::gputools::DeviceMemoryBase device_base);
+
+  // Executes the given module and return the result as a Literal.
+  std::unique_ptr<Literal> ExecuteAndTransfer(
+      std::unique_ptr<HloModule> module,
+      tensorflow::gtl::ArraySlice<perftools::gputools::DeviceMemoryBase>
+          arguments);
+
+  // If backend is not created in the constructor, creates and returns the
+  // default backend. If creation fails, crashes the program.
+  //
+  // This creates the backend lazily so it's possible to instantiate an
+  // HloRunner in a program without any backends linked in.
+  Backend& backend();
+
+ private:
+  struct EigenThreadPoolWrapper;
+
+  std::vector<perftools::gputools::DeviceMemoryBase> allocations_;
+
+  std::unique_ptr<EigenThreadPoolWrapper> thread_pool_wrapper_;
+
+  std::unique_ptr<Backend> backend_;
+};
+
+}  // namespace xla
+
+#endif  // TENSORFLOW_COMPILER_XLA_SERVICE_HLO_RUNNER_H_
diff --git a/tensorflow/compiler/xla/tests/BUILD b/tensorflow/compiler/xla/tests/BUILD
index b02d906d93..43127925e6 100644
--- a/tensorflow/compiler/xla/tests/BUILD
+++ b/tensorflow/compiler/xla/tests/BUILD
@@ -102,28 +102,18 @@ cc_library(
     deps = [
         ":literal_test_util",
         "//tensorflow/compiler/xla:shape_layout",
-        "//tensorflow/compiler/xla:shape_util",
-        "//tensorflow/compiler/xla:status_macros",
         "//tensorflow/compiler/xla:statusor",
         "//tensorflow/compiler/xla:types",
         "//tensorflow/compiler/xla:util",
         "//tensorflow/compiler/xla:xla_data_proto",
         "//tensorflow/compiler/xla/legacy_flags:debug_options_flags",
-        "//tensorflow/compiler/xla/service",
         "//tensorflow/compiler/xla/service:backend",
-        "//tensorflow/compiler/xla/service:compiler",
         "//tensorflow/compiler/xla/service:computation_layout",
-        "//tensorflow/compiler/xla/service:computation_placer",
-        "//tensorflow/compiler/xla/service:executable",
         "//tensorflow/compiler/xla/service:hlo",
-        "//tensorflow/compiler/xla/service:hlo_execution_profile",
-        "//tensorflow/compiler/xla/service:hlo_graph_dumper",
-        "//tensorflow/compiler/xla/service:transfer_manager",
-        "//tensorflow/core:core_cpu_internal",
+        "//tensorflow/compiler/xla/service:hlo_runner",
         "//tensorflow/core:lib",
         "//tensorflow/core:stream_executor_no_cuda",
         "//tensorflow/core:test",
-        "//third_party/eigen3",
     ],
 )
 
diff --git a/tensorflow/compiler/xla/tests/hlo_test_base.cc b/tensorflow/compiler/xla/tests/hlo_test_base.cc
index 26513d6ce8..3e244fbfd9 100644
--- a/tensorflow/compiler/xla/tests/hlo_test_base.cc
+++ b/tensorflow/compiler/xla/tests/hlo_test_base.cc
@@ -19,24 +19,9 @@ limitations under the License.
 #include <string>
 #include <utility>
 
-#define EIGEN_USE_THREADS
-
-#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
-#include "tensorflow/compiler/xla/layout_util.h"
 #include "tensorflow/compiler/xla/legacy_flags/debug_options_flags.h"
 #include "tensorflow/compiler/xla/ptr_util.h"
-#include "tensorflow/compiler/xla/service/backend.h"
-#include "tensorflow/compiler/xla/service/computation_layout.h"
-#include "tensorflow/compiler/xla/service/executable.h"
-#include "tensorflow/compiler/xla/service/hlo_computation.h"
-#include "tensorflow/compiler/xla/service/hlo_execution_profile.h"
-#include "tensorflow/compiler/xla/service/hlo_instruction.h"
-#include "tensorflow/compiler/xla/service/transfer_manager.h"
-#include "tensorflow/compiler/xla/shape_layout.h"
-#include "tensorflow/compiler/xla/shape_util.h"
-#include "tensorflow/compiler/xla/status_macros.h"
 #include "tensorflow/compiler/xla/types.h"
-#include "tensorflow/core/common_runtime/eigen_thread_pool.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/test.h"
 #include "tensorflow/core/platform/types.h"
@@ -45,22 +30,6 @@ namespace se = ::perftools::gputools;
 
 namespace xla {
 
-// Define this in .cc file to avoid having to include eigen or forward declare
-// these types in the header.
-struct HloTestBase::EigenThreadPoolWrapper {
-  std::unique_ptr<EigenThreadPoolWrapper> pool;
-  std::unique_ptr<Eigen::ThreadPoolDevice> device;
-};
-
-HloTestBase::HloTestBase() {}
-
-HloTestBase::~HloTestBase() {
-  // Deallocate all the memory allocated during the tests.
-  for (auto& allocation : allocations_) {
-    backend().default_stream_executor()->Deallocate(&allocation);
-  }
-}
-
 /* static */
 std::unique_ptr<HloModule> HloTestBase::CreateNewModule() {
   HloModuleConfig config;
@@ -80,98 +49,25 @@ StatusOr<perftools::gputools::DeviceMemoryBase> HloTestBase::Execute(
     tensorflow::gtl::ArraySlice<perftools::gputools::DeviceMemoryBase>
         arguments,
     Shape* result_shape) {
-  TF_ASSIGN_OR_RETURN(
-      std::unique_ptr<Executable> executable,
-      backend().compiler()->Compile(std::move(module),
-                                    backend().default_stream_executor()));
-
-  se::Stream stream(backend().default_stream_executor());
-  stream.Init();
-
-  ExecutableRunOptions run_options;
-  run_options.set_stream(&stream);
-  run_options.set_allocator(backend().memory_allocator());
-  run_options.set_inter_op_thread_pool(backend().inter_op_thread_pool());
-  run_options.set_intra_op_thread_pool(
-      backend().eigen_intra_op_thread_pool_device());
-
-  HloExecutionProfile hlo_execution_profile;
-  ServiceExecutableRunOptions service_run_options(
-      run_options, backend().StreamBorrower(),
-      backend().inter_op_thread_pool());
-  TF_ASSIGN_OR_RETURN(
-      se::DeviceMemoryBase result,
-      executable->ExecuteOnStream(&service_run_options, arguments,
-                                  &hlo_execution_profile));
-  TF_RET_CHECK(stream.BlockHostUntilDone());
-
-  allocations_.push_back(result);
-
-  *result_shape = executable->result_shape();
-
-  if (ShapeUtil::IsTuple(*result_shape)) {
-    // We must record element buffers of tuples as well to avoid leaks.
-    DCHECK(!ShapeUtil::IsNestedTuple(*result_shape));
-    TF_ASSIGN_OR_RETURN(
-        std::vector<se::DeviceMemoryBase> element_buffers,
-        backend().transfer_manager()->ShallowCopyTupleFromDevice(
-            backend().default_stream_executor(), result, *result_shape));
-
-    // A tuple may contain the same buffer in more than one element. Keep track
-    // of the buffers already added to avoid duplicates in allocations_.
-    std::set<void*> added_opaques;
-    for (auto element_buffer : element_buffers) {
-      if (added_opaques.count(element_buffer.opaque()) == 0) {
-        CHECK(element_buffer.opaque() != nullptr);
-        added_opaques.insert(element_buffer.opaque());
-        allocations_.push_back(element_buffer);
-      }
-    }
-  }
-
-  return result;
+  return runner_.Execute(std::move(module), arguments, result_shape);
 }
 
 se::DeviceMemoryBase HloTestBase::TransferToDevice(const Literal& literal) {
-  // Allocate memory on the device using the stream executor.
-  int64 allocation_size =
-      backend().transfer_manager()->GetByteSizeRequirement(literal.shape());
-  se::DeviceMemoryBase allocation =
-      backend().default_stream_executor()->AllocateArray<uint8>(
-          allocation_size);
-  allocations_.push_back(allocation);
-
-  TF_CHECK_OK(backend().transfer_manager()->TransferLiteralToDevice(
-      backend().default_stream_executor(), literal, &allocation));
-
-  return allocation;
+  return runner_.TransferToDevice(literal);
 }
 
 std::unique_ptr<Literal> HloTestBase::TransferFromDevice(
     const Shape& shape, se::DeviceMemoryBase device_base) {
-  auto literal = MakeUnique<Literal>();
-  TF_CHECK_OK(backend().transfer_manager()->TransferLiteralFromDevice(
-      backend().default_stream_executor(), device_base, shape, shape,
-      literal.get()));
-  return literal;
+  return runner_.TransferFromDevice(shape, device_base);
 }
 
 std::unique_ptr<Literal> HloTestBase::ExecuteAndTransfer(
     std::unique_ptr<HloModule> module,
     tensorflow::gtl::ArraySlice<se::DeviceMemoryBase> arguments) {
-  Shape result_shape;
-  se::DeviceMemoryBase device_base =
-      Execute(std::move(module), arguments, &result_shape).ValueOrDie();
-  return TransferFromDevice(result_shape, device_base);
+  return runner_.ExecuteAndTransfer(std::move(module), arguments);
 }
 
-Backend& HloTestBase::backend() {
-  if (!backend_) {
-    backend_ = Backend::CreateDefaultBackend().ConsumeValueOrDie();
-    VLOG(1) << "executing on platform " << backend().platform()->Name();
-  }
-  return *backend_;
-}
+Backend& HloTestBase::backend() { return runner_.backend(); }
 
 /* static */
 string HloTestBase::TestName() {
diff --git a/tensorflow/compiler/xla/tests/hlo_test_base.h b/tensorflow/compiler/xla/tests/hlo_test_base.h
index 275f1f5c7b..7f068dce36 100644
--- a/tensorflow/compiler/xla/tests/hlo_test_base.h
+++ b/tensorflow/compiler/xla/tests/hlo_test_base.h
@@ -21,12 +21,12 @@ limitations under the License.
 #include <vector>
 
 #include "tensorflow/compiler/xla/service/backend.h"
-#include "tensorflow/compiler/xla/service/compiler.h"
-#include "tensorflow/compiler/xla/service/hlo_computation.h"
+#include "tensorflow/compiler/xla/service/computation_layout.h"
 #include "tensorflow/compiler/xla/service/hlo_module.h"
+#include "tensorflow/compiler/xla/service/hlo_runner.h"
+#include "tensorflow/compiler/xla/shape_layout.h"
 #include "tensorflow/compiler/xla/statusor.h"
 #include "tensorflow/compiler/xla/tests/literal_test_util.h"
-#include "tensorflow/compiler/xla/types.h"
 #include "tensorflow/compiler/xla/xla_data.pb.h"
 #include "tensorflow/core/lib/gtl/array_slice.h"
 #include "tensorflow/core/platform/stream_executor_no_cuda.h"
@@ -39,10 +39,9 @@ namespace xla {
 // building a graph of HLO instructions to run.
 class HloTestBase : public ::testing::Test {
  protected:
-  struct EigenThreadPoolWrapper;
-  HloTestBase();
+  HloTestBase() {}
 
-  ~HloTestBase() override;
+  ~HloTestBase() override {}
 
   // Creates a new HLO module for a test. The module created will have
   // TestName() for its name; it will also automatically populate its debug
@@ -102,23 +101,12 @@ class HloTestBase : public ::testing::Test {
 
   static string TestName();
 
-  // Creates (if necessary) and returns the default backend.  If creation fails,
-  // crashes the program.
-  //
-  // This creates the backend lazily so it's possible to instantiate an
-  // HloTestBase in a program without any backends linked in.
+  // Returns the backend owned by the HloRunner.
   Backend& backend();
 
-  // This vector contains handles of all the device memory allocations performed
-  // by the test. These are deallocated on destruction of the test object.
-  std::vector<perftools::gputools::DeviceMemoryBase> allocations_;
+  HloRunner runner_;
 
   ErrorSpec error_spec_{0.0001};
-
-  std::unique_ptr<EigenThreadPoolWrapper> thread_pool_wrapper_;
-
- private:
-  std::unique_ptr<Backend> backend_;  // Lazily populated. Access via backend().
 };
 
 }  // namespace xla
-- 
GitLab


From aa9ddb2006cba090a53ea978a6ec78bea8245805 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 19 Oct 2017 20:10:03 -0700
Subject: [PATCH 601/909] Add a tool which reads the Hlo module proto and
 convert it into JSON format.

PiperOrigin-RevId: 172835881
---
 tensorflow/compiler/xla/tools/BUILD           | 12 +++
 .../compiler/xla/tools/hlo_proto_to_json.cc   | 91 +++++++++++++++++++
 2 files changed, 103 insertions(+)
 create mode 100644 tensorflow/compiler/xla/tools/hlo_proto_to_json.cc

diff --git a/tensorflow/compiler/xla/tools/BUILD b/tensorflow/compiler/xla/tools/BUILD
index 0451537af7..759921dce5 100644
--- a/tensorflow/compiler/xla/tools/BUILD
+++ b/tensorflow/compiler/xla/tools/BUILD
@@ -210,6 +210,18 @@ tf_cc_binary(
     ],
 )
 
+tf_cc_binary(
+    name = "hlo_proto_to_json",
+    srcs = ["hlo_proto_to_json.cc"],
+    deps = [
+        "//tensorflow/compiler/xla:statusor",
+        "//tensorflow/compiler/xla:util",
+        "//tensorflow/compiler/xla/service:hlo_proto",
+        "//tensorflow/core:framework_internal",
+        "//tensorflow/core:lib",
+    ],
+)
+
 # -----------------------------------------------------------------------------
 
 filegroup(
diff --git a/tensorflow/compiler/xla/tools/hlo_proto_to_json.cc b/tensorflow/compiler/xla/tools/hlo_proto_to_json.cc
new file mode 100644
index 0000000000..4e02e17db6
--- /dev/null
+++ b/tensorflow/compiler/xla/tools/hlo_proto_to_json.cc
@@ -0,0 +1,91 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+// Usage:
+//   hlo_proto_to_json --input_file=some_binary_proto
+//   --output_file=path_to_dump_output
+//
+// Reads one serilized Hlo module, convert it into JSON format and dump into
+// some output directory. some_binaray_proto is obtained by serializing Hlo
+// module to disk using --xla_dump_hlo_proto_to debug optoin.
+
+#include <stdio.h>
+#include <string>
+#include <vector>
+
+#include "tensorflow/compiler/xla/service/hlo.pb.h"
+#include "tensorflow/compiler/xla/statusor.h"
+#include "tensorflow/compiler/xla/util.h"
+#include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/platform/env.h"
+#include "tensorflow/core/platform/init_main.h"
+#include "tensorflow/core/platform/logging.h"
+#include "tensorflow/core/util/command_line_flags.h"
+
+using tensorflow::Env;
+using xla::string;
+
+namespace xla {
+namespace tools {
+
+StatusOr<string> ToJson(const tensorflow::protobuf::Message& message) {
+  string json_output;
+  tensorflow::protobuf::util::JsonPrintOptions json_options;
+  json_options.add_whitespace = true;
+  json_options.always_print_primitive_fields = true;
+  auto status = tensorflow::protobuf::util::MessageToJsonString(
+      message, &json_output, json_options);
+  if (!status.ok()) {
+    return InternalError("MessageToJsonString failed: %s",
+                         status.error_message().data());
+  }
+  return json_output;
+}
+
+void RealMain(const string& input, const string& output) {
+  HloProto hlo_proto;
+  TF_CHECK_OK(tensorflow::ReadBinaryProto(tensorflow::Env::Default(), input,
+                                          &hlo_proto))
+      << "Can't open, read, or parse input file " << input;
+
+  auto statusor = ToJson(hlo_proto);
+  QCHECK(statusor.ok()) << "Error converting " << input << " to JSON."
+                        << statusor.status();
+
+  TF_CHECK_OK(tensorflow::WriteStringToFile(tensorflow::Env::Default(), output,
+                                            statusor.ValueOrDie()));
+}
+
+}  // namespace tools
+}  // namespace xla
+
+int main(int argc, char** argv) {
+  string input_file, output_file;
+  const std::vector<tensorflow::Flag> flag_list = {
+      tensorflow::Flag("input_file", &input_file, "file to convert."),
+      tensorflow::Flag("output_file", &output_file, "converted file"),
+  };
+  const string usage = tensorflow::Flags::Usage(argv[0], flag_list);
+  bool parse_ok = tensorflow::Flags::Parse(&argc, argv, flag_list);
+  tensorflow::port::InitMain(usage.c_str(), &argc, &argv);
+  QCHECK(parse_ok && argc == 1) << "\n" << usage;
+
+  QCHECK(!input_file.empty()) << "--input_file is required";
+  QCHECK(!output_file.empty()) << "--output_file is required";
+
+  xla::tools::RealMain(input_file, output_file);
+
+  return 0;
+}
-- 
GitLab


From 7a1ddf26aed9166af69a560e644abd3f0d4f8ecf Mon Sep 17 00:00:00 2001
From: Sang Han <jjangsangy@users.noreply.github.com>
Date: Thu, 19 Oct 2017 20:24:13 -0700
Subject: [PATCH 602/909] Fix casting to size_t for mkl conv filter dims
 (#13831)

---
 tensorflow/core/kernels/mkl_conv_ops.cc | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/kernels/mkl_conv_ops.cc b/tensorflow/core/kernels/mkl_conv_ops.cc
index 57661e8b10..369f632fb4 100644
--- a/tensorflow/core/kernels/mkl_conv_ops.cc
+++ b/tensorflow/core/kernels/mkl_conv_ops.cc
@@ -288,8 +288,10 @@ class MklConv2DOp : public OpKernel {
     mkl_filter_output_mkl_shape.SetMklLayout(mkl_context.prim_fwd,
                                              dnnResourceFilter);
 
-    size_t filter_sizes[4] = {filter.dim_size(0), filter.dim_size(1),
-                              filter.dim_size(2), filter.dim_size(3)};
+    size_t filter_sizes[4] = {static_cast<size_t>(filter.dim_size(0)),
+                              static_cast<size_t>(filter.dim_size(1)),
+                              static_cast<size_t>(filter.dim_size(2)),
+                              static_cast<size_t>(filter.dim_size(3))};
     mkl_filter_output_mkl_shape.SetTfLayout(filter.dims(), filter_sizes,
                                             mkl_context.filter_strides);
 
-- 
GitLab


From 513f7df42e4eadfcd241a3be695af6fd426b734e Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Thu, 19 Oct 2017 20:34:53 -0700
Subject: [PATCH 603/909] Add `int64` out_idx` support for
 `listdiff`/`list_diff`/`setdiff1d` (#13839)

* Add `int64` out_idx` support for `listdiff`/`list_diff`/`setdiff1d`

This fix tries to add `int64` `out_idx` support for `listdiff`/`list_diff`/`setdiff1d`.
As was specified in docs (`tf.setdiff1d.__doc__`), it is possible to specify
`tf.int32` or `tf.int64` for the type of the output idx. However,
the `tf.int64` kernel has not been registered. As a consequence,
an error will be thrown out if `tf.int64` is used.

This fix adds `int64` out_idx` support for `listdiff`/`list_diff`/`setdiff1d`

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Add template for signature matching of ListDiff kernel.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Add test cases for `int64` out_idx support for `tf.listdiff`/`setdiff1d`

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Add test case for int32 (missed in the last commit)

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/core/kernels/listdiff_op.cc        | 16 ++++++++++-----
 .../python/kernel_tests/listdiff_op_test.py   | 20 ++++++++++---------
 2 files changed, 22 insertions(+), 14 deletions(-)

diff --git a/tensorflow/core/kernels/listdiff_op.cc b/tensorflow/core/kernels/listdiff_op.cc
index d303bdd560..d28a2729d4 100644
--- a/tensorflow/core/kernels/listdiff_op.cc
+++ b/tensorflow/core/kernels/listdiff_op.cc
@@ -24,12 +24,13 @@ limitations under the License.
 #include "tensorflow/core/lib/core/status.h"
 
 namespace tensorflow {
-template <typename T>
+template <typename T, typename Tidx>
 class ListDiffOp : public OpKernel {
  public:
   explicit ListDiffOp(OpKernelConstruction* context) : OpKernel(context) {
     const DataType dt = DataTypeToEnum<T>::v();
-    OP_REQUIRES_OK(context, context->MatchSignature({dt, dt}, {dt, DT_INT32}));
+    const DataType dtidx = DataTypeToEnum<Tidx>::v();
+    OP_REQUIRES_OK(context, context->MatchSignature({dt, dt}, {dt, dtidx}));
   }
 
   void Compute(OpKernelContext* context) override {
@@ -72,9 +73,9 @@ class ListDiffOp : public OpKernel {
 
     Tensor* indices = nullptr;
     OP_REQUIRES_OK(context, context->allocate_output(1, {out_size}, &indices));
-    auto Tindices = indices->vec<int32>();
+    auto Tindices = indices->vec<Tidx>();
 
-    for (int i = 0, p = 0; i < static_cast<int32>(x_size); ++i) {
+    for (Tidx i = 0, p = 0; i < static_cast<Tidx>(x_size); ++i) {
       if (y_set.count(Tx(i)) == 0) {
         OP_REQUIRES(context, p < out_size,
                     errors::InvalidArgument(
@@ -95,7 +96,12 @@ class ListDiffOp : public OpKernel {
                               .Device(DEVICE_CPU)                \
                               .TypeConstraint<type>("T")         \
                               .TypeConstraint<int32>("out_idx"), \
-                          ListDiffOp<type>)
+                          ListDiffOp<type, int32>)               \
+  REGISTER_KERNEL_BUILDER(Name("ListDiff")                       \
+                              .Device(DEVICE_CPU)                \
+                              .TypeConstraint<type>("T")         \
+                              .TypeConstraint<int64>("out_idx"), \
+                          ListDiffOp<type, int64>)
 
 TF_CALL_REAL_NUMBER_TYPES(REGISTER_LISTDIFF);
 REGISTER_LISTDIFF(string);
diff --git a/tensorflow/python/kernel_tests/listdiff_op_test.py b/tensorflow/python/kernel_tests/listdiff_op_test.py
index 4f053d2a21..ee86cf0b24 100644
--- a/tensorflow/python/kernel_tests/listdiff_op_test.py
+++ b/tensorflow/python/kernel_tests/listdiff_op_test.py
@@ -41,15 +41,17 @@ class ListDiffTest(test.TestCase):
         y = [compat.as_bytes(str(a)) for a in y]
         out = [compat.as_bytes(str(a)) for a in out]
       for diff_func in [array_ops.setdiff1d]:
-        with self.test_session() as sess:
-          x_tensor = ops.convert_to_tensor(x, dtype=dtype)
-          y_tensor = ops.convert_to_tensor(y, dtype=dtype)
-          out_tensor, idx_tensor = diff_func(x_tensor, y_tensor)
-          tf_out, tf_idx = sess.run([out_tensor, idx_tensor])
-        self.assertAllEqual(tf_out, out)
-        self.assertAllEqual(tf_idx, idx)
-        self.assertEqual(1, out_tensor.get_shape().ndims)
-        self.assertEqual(1, idx_tensor.get_shape().ndims)
+        for index_dtype in [dtypes.int32, dtypes.int64]:
+          with self.test_session() as sess:
+            x_tensor = ops.convert_to_tensor(x, dtype=dtype)
+            y_tensor = ops.convert_to_tensor(y, dtype=dtype)
+            out_tensor, idx_tensor = diff_func(x_tensor, y_tensor,
+                                               index_dtype=index_dtype)
+            tf_out, tf_idx = sess.run([out_tensor, idx_tensor])
+          self.assertAllEqual(tf_out, out)
+          self.assertAllEqual(tf_idx, idx)
+          self.assertEqual(1, out_tensor.get_shape().ndims)
+          self.assertEqual(1, idx_tensor.get_shape().ndims)
 
   def testBasic1(self):
     x = [1, 2, 3, 4]
-- 
GitLab


From 492ddb55a9b31a07026b7d82a2f9bcac29f4ee65 Mon Sep 17 00:00:00 2001
From: Suharsh Sivakumar <suharshs@google.com>
Date: Thu, 19 Oct 2017 21:09:44 -0700
Subject: [PATCH 604/909] Add support for fused batch norm to fake quantize
 rewriter.

PiperOrigin-RevId: 172839124
---
 tensorflow/contrib/quantize/BUILD             |  33 +-
 .../quantize/python/copy_graph_test.py        |   2 +-
 .../quantize/python/fold_batch_norms.py       | 269 ++++++++++++-
 .../quantize/python/fold_batch_norms_test.py  | 372 ++++++------------
 .../contrib/quantize/python/graph_matcher.py  | 200 ++++++++++
 .../quantize/python/graph_matcher_test.py     | 130 ++++++
 .../python/quantize_parameterized_test.py     | 212 +++++-----
 7 files changed, 855 insertions(+), 363 deletions(-)
 create mode 100644 tensorflow/contrib/quantize/python/graph_matcher.py
 create mode 100644 tensorflow/contrib/quantize/python/graph_matcher_test.py

diff --git a/tensorflow/contrib/quantize/BUILD b/tensorflow/contrib/quantize/BUILD
index 7ff186bc2a..0d6c71965c 100644
--- a/tensorflow/contrib/quantize/BUILD
+++ b/tensorflow/contrib/quantize/BUILD
@@ -13,6 +13,34 @@ py_library(
     deps = [],
 )
 
+py_library(
+    name = "graph_matcher",
+    srcs = [
+        "python/graph_matcher.py",
+    ],
+    srcs_version = "PY2AND3",
+    deps = [],
+)
+
+py_test(
+    name = "graph_matcher_test",
+    size = "small",
+    srcs = ["python/graph_matcher_test.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":graph_matcher",
+        "//tensorflow/contrib/layers:layers_py",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:dtypes",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:init_ops",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:nn_ops",
+        "//tensorflow/python:platform_test",
+    ],
+)
+
 py_library(
     name = "input_to_ops",
     srcs = ["python/input_to_ops.py"],
@@ -43,6 +71,7 @@ py_library(
     srcs_version = "PY2AND3",
     deps = [
         ":common",
+        ":graph_matcher",
         ":input_to_ops",
         "//tensorflow/contrib/graph_editor:graph_editor_py",
         "//tensorflow/python:array_ops",
@@ -58,6 +87,7 @@ py_test(
     srcs_version = "PY2AND3",
     deps = [
         ":fold_batch_norms",
+        ":graph_matcher",
         "//tensorflow/contrib/layers:layers_py",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:dtypes",
@@ -147,10 +177,11 @@ py_test(
 
 py_test(
     name = "quantize_parameterized_test",
-    size = "medium",
+    size = "large",
     srcs = ["python/quantize_parameterized_test.py"],
     srcs_version = "PY2AND3",
     deps = [
+        ":fold_batch_norms",
         ":quantize",
         "//tensorflow/contrib/layers:layers_py",
         "//tensorflow/python:array_ops",
diff --git a/tensorflow/contrib/quantize/python/copy_graph_test.py b/tensorflow/contrib/quantize/python/copy_graph_test.py
index 0889f12de6..7ff9ad9f84 100644
--- a/tensorflow/contrib/quantize/python/copy_graph_test.py
+++ b/tensorflow/contrib/quantize/python/copy_graph_test.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Tests for tensorflow.quantized.mangle.copy_graph."""
+"""Tests for copy_graph."""
 
 from __future__ import absolute_import
 from __future__ import division
diff --git a/tensorflow/contrib/quantize/python/fold_batch_norms.py b/tensorflow/contrib/quantize/python/fold_batch_norms.py
index c416689510..647d404400 100644
--- a/tensorflow/contrib/quantize/python/fold_batch_norms.py
+++ b/tensorflow/contrib/quantize/python/fold_batch_norms.py
@@ -21,7 +21,9 @@ from __future__ import print_function
 import re
 from tensorflow.contrib import graph_editor
 from tensorflow.contrib.quantize.python import common
+from tensorflow.contrib.quantize.python import graph_matcher
 from tensorflow.contrib.quantize.python import input_to_ops
+from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import nn
@@ -29,7 +31,7 @@ from tensorflow.python.ops import nn_ops
 
 
 def FoldBatchNorms(graph):
-  """Finds batch norm layers in the graph, folds them into preceding layers.
+  """Finds batch norm layers and folds them into preceding layers.
 
   Folding only affects the following layers: Conv2D, fully connected, depthwise
   convolution.
@@ -40,10 +42,269 @@ def FoldBatchNorms(graph):
   Raises:
     ValueError: When batch norm folding fails.
   """
-  # Fail immediately when the graph contains unsupported fused batch norm ops.
-  if any(op for op in graph.get_operations() if op.type == 'FusedBatchNorm'):
-    raise ValueError('Fused batch norm is not supported')
+  _FoldFusedBatchNorms(graph)
+  _FoldUnfusedBatchNorms(graph)
 
+
+def _FoldFusedBatchNorms(graph):
+  """Finds fused batch norm layers and folds them into preceding layers.
+
+  Folding only affects the following layers: Conv2D, fully connected, depthwise
+  convolution.
+
+  Args:
+    graph: Graph to walk and modify.
+
+  Raises:
+    ValueError: When batch norm folding fails.
+  """
+  for match in _FindFusedBatchNorms(graph):
+    scope, sep, _ = match.layer_op.name.rpartition('/')
+    # Make sure new ops are added to `graph` and put on the same device as
+    # `bn_op`. The '/' (i.e. `sep`) ensures that we reuse the existing scope
+    # named `scope`. Otherwise, TF creates a unique scope whose name starts with
+    # `scope`.
+    with graph.as_default(), graph.name_scope(scope + sep), ops.device(
+        match.bn_op.device):
+      # new weights = old weights * gamma / sqrt(variance + epsilon)
+      # new biases = -mean * gamma / sqrt(variance + epsilon) + beta
+      multiplier_tensor = match.gamma_tensor * math_ops.rsqrt(
+          match.variance_tensor + match.bn_op.get_attr('epsilon'))
+      bias_tensor = math_ops.subtract(
+          match.beta_tensor, match.mean_tensor * multiplier_tensor, name='bias')
+
+      # The shape of depthwise weights is different, so we need to reshape the
+      # multiplier_tensor to ensure that the scaled_weight_tensor has the
+      # expected shape.
+      if match.layer_op.type == 'DepthwiseConv2dNative':
+        new_shape = [
+            match.weight_tensor.get_shape().as_list()[2],
+            match.weight_tensor.get_shape().as_list()[3]
+        ]
+        multiplier_tensor = array_ops.reshape(
+            multiplier_tensor, new_shape, name='scale_reshape')
+
+      # TODO(suharshs): This naming of the following ops needs to carefully
+      # follow the naming expected by quantize.py. Generalize the quantize code
+      # to not require these delicate naming conventions.
+      scaled_weight_tensor = math_ops.multiply(
+          match.weight_tensor, multiplier_tensor, name='mul_fold')
+
+      new_layer_tensor = _CloneWithNewOperands(
+          match.layer_op, match.input_tensor, scaled_weight_tensor)
+
+      bias_add_tensor = math_ops.add(
+          new_layer_tensor, bias_tensor, name='add_fold')
+
+      nodes_modified_count = graph_editor.reroute_ts(bias_add_tensor,
+                                                     match.output_tensor)
+      if nodes_modified_count != 1:
+        raise ValueError(
+            'Unexpected inputs to op: %s' % match.output_tensor.name)
+
+
+def _CloneWithNewOperands(layer_op, input_tensor, weight_tensor):
+  """Clones layer_op with input_tensor and weight_tensor as new inputs."""
+  new_layer_name = layer_op.name.split('/')[-1] + '_Fold'
+  if layer_op.type == 'Conv2D':
+    return nn_ops.conv2d(
+        input_tensor,
+        weight_tensor,
+        strides=layer_op.get_attr('strides'),
+        padding=layer_op.get_attr('padding'),
+        use_cudnn_on_gpu=layer_op.get_attr('use_cudnn_on_gpu'),
+        data_format=layer_op.get_attr('data_format'),
+        name=new_layer_name)
+  elif layer_op.type == 'MatMul':
+    return math_ops.matmul(
+        input_tensor,
+        weight_tensor,
+        transpose_a=layer_op.get_attr('transpose_a'),
+        transpose_b=layer_op.get_attr('transpose_b'),
+        name=new_layer_name)
+  elif layer_op.type == 'DepthwiseConv2dNative':
+    return nn.depthwise_conv2d(
+        input_tensor,
+        weight_tensor,
+        strides=layer_op.get_attr('strides'),
+        padding=layer_op.get_attr('padding'),
+        name=new_layer_name)
+  else:
+    raise ValueError('Cannot handle operation of type: %s' % layer_op.type)
+
+
+def _FindFusedBatchNorms(graph):
+  """Finds all ops and tensors related to found FusedBatchNorms.
+
+  Args:
+    graph: Graph to inspect.
+
+  Yields:
+    _FusedBatchNormMatches.
+  """
+  input_pattern = graph_matcher.OpTypePattern('*')
+  weight_pattern = graph_matcher.OpTypePattern('*')
+  gamma_pattern = graph_matcher.OpTypePattern('*')
+  beta_pattern = graph_matcher.OpTypePattern('*')
+  mean_pattern = graph_matcher.OpTypePattern('*')
+  variance_pattern = graph_matcher.OpTypePattern('*')
+
+  conv_pattern = graph_matcher.OpTypePattern(
+      'Conv2D|DepthwiseConv2dNative', inputs=[input_pattern, weight_pattern])
+  # MatMul has a Reshape between it and FusedBatchNorm.
+  matmul_pattern = graph_matcher.OpTypePattern(
+      'MatMul', inputs=[input_pattern, weight_pattern])
+  matmul_reshape_pattern = graph_matcher.OpTypePattern(
+      'Reshape', inputs=[matmul_pattern,
+                         graph_matcher.OpTypePattern('*')])
+
+  conv_batch_norm_pattern = graph_matcher.OpTypePattern(
+      'FusedBatchNorm',
+      inputs=[
+          conv_pattern, gamma_pattern, beta_pattern, mean_pattern,
+          variance_pattern
+      ])
+  matmul_batch_norm_pattern = graph_matcher.OpTypePattern(
+      'FusedBatchNorm',
+      inputs=[
+          matmul_reshape_pattern, gamma_pattern, beta_pattern, mean_pattern,
+          variance_pattern
+      ])
+  matmul_bn_output_reshape_pattern = graph_matcher.OpTypePattern(
+      'Reshape',
+      inputs=[matmul_batch_norm_pattern,
+              graph_matcher.OpTypePattern('*')])
+
+  conv_matcher = graph_matcher.GraphMatcher(conv_batch_norm_pattern)
+  matmul_matcher = graph_matcher.GraphMatcher(matmul_bn_output_reshape_pattern)
+
+  def _GetCommonTensors(match_result):
+    """Gets tensors needed for FusedBatchNormMatch from match_result."""
+    input_tensor = match_result.get_tensor(input_pattern)
+    weight_tensor = match_result.get_tensor(weight_pattern)
+    gamma_tensor = match_result.get_tensor(gamma_pattern)
+    beta_tensor = match_result.get_tensor(beta_pattern)
+    # FusedBatchNorm in training is different from that in inference. It takes
+    # empty 'mean' and empty 'variance', and produces the mean and the variance
+    # of the batch. Therefore, when is_training is true, mean_tensor and
+    # variance_tensor point to 1st and 2nd (0-based) output of bn_op,
+    # respectively; when is_training is false, they point to bn_op's inputs.
+    is_training = bn_op.get_attr('is_training')
+    if is_training:
+      mean_tensor = bn_op.outputs[1]
+      variance_tensor = bn_op.outputs[2]
+    else:
+      mean_tensor = match_result.get_tensor(mean_pattern)
+      variance_tensor = match_result.get_tensor(variance_pattern)
+    return (input_tensor, weight_tensor, gamma_tensor, beta_tensor, mean_tensor,
+            variance_tensor)
+
+  for match_result in conv_matcher.match_graph(graph):
+    layer_op = match_result.get_op(conv_pattern)
+    bn_op = match_result.get_op(conv_batch_norm_pattern)
+    # In the case of convolution the output_tensor is the output of bn_op.
+    output_tensor = bn_op.outputs[0]
+
+    (input_tensor, weight_tensor, gamma_tensor, beta_tensor, mean_tensor,
+     variance_tensor) = _GetCommonTensors(match_result)
+    yield _FusedBatchNormMatch(
+        layer_op=layer_op,
+        bn_op=bn_op,
+        output_tensor=output_tensor,
+        input_tensor=input_tensor,
+        weight_tensor=weight_tensor,
+        gamma_tensor=gamma_tensor,
+        beta_tensor=beta_tensor,
+        mean_tensor=mean_tensor,
+        variance_tensor=variance_tensor)
+
+  for match_result in matmul_matcher.match_graph(graph):
+    layer_op = match_result.get_op(matmul_pattern)
+    bn_op = match_result.get_op(matmul_batch_norm_pattern)
+    # In the MatMul case, the output of batch norm is reshaped back into a
+    # 2D tensor, so the output_tensor is the output of the Reshape op.
+    output_reshape_op = match_result.get_op(matmul_bn_output_reshape_pattern)
+    output_tensor = output_reshape_op.outputs[0]
+
+    (input_tensor, weight_tensor, gamma_tensor, beta_tensor, mean_tensor,
+     variance_tensor) = _GetCommonTensors(match_result)
+    yield _FusedBatchNormMatch(
+        layer_op=layer_op,
+        bn_op=bn_op,
+        output_tensor=output_tensor,
+        input_tensor=input_tensor,
+        weight_tensor=weight_tensor,
+        gamma_tensor=gamma_tensor,
+        beta_tensor=beta_tensor,
+        mean_tensor=mean_tensor,
+        variance_tensor=variance_tensor)
+
+
+class _FusedBatchNormMatch(object):
+  """Contains all information related to a found FusedBatchNorm."""
+
+  def __init__(self, layer_op, bn_op, output_tensor, input_tensor,
+               weight_tensor, gamma_tensor, beta_tensor, mean_tensor,
+               variance_tensor):
+    self._layer_op = layer_op
+    self._bn_op = bn_op
+    self._output_tensor = output_tensor
+    self._input_tensor = input_tensor
+    self._weight_tensor = weight_tensor
+    self._gamma_tensor = gamma_tensor
+    self._beta_tensor = beta_tensor
+    self._mean_tensor = mean_tensor
+    self._variance_tensor = variance_tensor
+
+  @property
+  def layer_op(self):
+    return self._layer_op
+
+  @property
+  def bn_op(self):
+    return self._bn_op
+
+  @property
+  def output_tensor(self):
+    return self._output_tensor
+
+  @property
+  def input_tensor(self):
+    return self._input_tensor
+
+  @property
+  def weight_tensor(self):
+    return self._weight_tensor
+
+  @property
+  def gamma_tensor(self):
+    return self._gamma_tensor
+
+  @property
+  def beta_tensor(self):
+    return self._beta_tensor
+
+  @property
+  def mean_tensor(self):
+    return self._mean_tensor
+
+  @property
+  def variance_tensor(self):
+    return self._variance_tensor
+
+
+def _FoldUnfusedBatchNorms(graph):
+  """Finds unfused batch norm layers and folds them into preceding layers.
+
+  Folding only affects the following layers: Conv2D, fully connected, depthwise
+  convolution.
+
+  Args:
+    graph: Graph to walk and modify.
+
+  Raises:
+    ValueError: When batch norm folding fails.
+  """
   input_to_ops_map = input_to_ops.InputToOps(graph)
 
   for bn in common.BatchNormGroups(graph):
diff --git a/tensorflow/contrib/quantize/python/fold_batch_norms_test.py b/tensorflow/contrib/quantize/python/fold_batch_norms_test.py
index ddedb0a2c0..5a66b38b15 100644
--- a/tensorflow/contrib/quantize/python/fold_batch_norms_test.py
+++ b/tensorflow/contrib/quantize/python/fold_batch_norms_test.py
@@ -18,7 +18,6 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import copy
 from tensorflow.contrib.layers.python.layers import layers
 from tensorflow.contrib.quantize.python import fold_batch_norms
 from tensorflow.python.framework import dtypes
@@ -35,57 +34,32 @@ conv2d = layers.conv2d
 fully_connected = layers.fully_connected
 separable_conv2d = layers.separable_conv2d
 
-_DEFAULT_BATCH_NORM_PARAMS = {
-    'center': True,
-    'scale': True,
-    'decay': 1.0 - 0.003,
-    'fused': False,
-}
-
 
 # TODO(suharshs): Use parameterized test once OSS TF supports it.
 class FoldBatchNormsTest(test_util.TensorFlowTestCase):
 
   def _RunTestOverParameters(self, test_fn):
     parameters_list = [
-        # (relu, relu_op_name, with_bypass)
-        (nn_ops.relu6, 'Relu6', False),
-        (nn_ops.relu, 'Relu', False),
-        (nn_ops.relu6, 'Relu6', True),
-        (nn_ops.relu, 'Relu', True),
+        # (relu, relu_op_name, with_bypass, has_scaling, fused_batch_norm)
+        (nn_ops.relu6, 'Relu6', False, False, False),
+        (nn_ops.relu, 'Relu', False, False, False),
+        (nn_ops.relu6, 'Relu6', True, False, False),
+        (nn_ops.relu, 'Relu', True, False, False),
+        (nn_ops.relu6, 'Relu6', False, True, False),
+        (nn_ops.relu, 'Relu', False, True, False),
+        (nn_ops.relu6, 'Relu6', True, True, False),
+        (nn_ops.relu, 'Relu', True, True, False),
+        # Fused batch norm always has scaling enabled.
+        (nn_ops.relu6, 'Relu6', False, True, True),
+        (nn_ops.relu, 'Relu', False, True, True),
+        (nn_ops.relu6, 'Relu6', True, True, True),
+        (nn_ops.relu, 'Relu', True, True, True),
     ]
-    for parameters in parameters_list:
-      test_fn(parameters[0], parameters[1], parameters[2])
-
-  def testFailsWithFusedBatchNorm(self):
-    self._RunTestOverParameters(self._TestFailsWithFusedBatchNorm)
+    for params in parameters_list:
+      test_fn(params[0], params[1], params[2], params[3], params[4])
 
-  def _TestFailsWithFusedBatchNorm(self, relu, relu_op_name, with_bypass):
-    """Tests that batch norm fails when fused batch norm ops are present."""
-    g = ops.Graph()
-    with g.as_default():
-      batch_size, height, width = 5, 128, 128
-      inputs = array_ops.zeros((batch_size, height, width, 3))
-      out_depth = 3 if with_bypass else 32
-      stride = 1 if with_bypass else 2
-      activation_fn = None if with_bypass else relu
-      batch_norm_params = _DEFAULT_BATCH_NORM_PARAMS.copy()
-      batch_norm_params['fused'] = True
-      scope = 'test/test2' if with_bypass else 'test'
-      node = conv2d(inputs, out_depth, [5, 5], stride=stride, padding='SAME',
-                    weights_initializer=self._WeightInit(0.09),
-                    activation_fn=activation_fn,
-                    normalizer_fn=batch_norm,
-                    normalizer_params=batch_norm_params,
-                    scope=scope)
-      if with_bypass:
-        node = math_ops.add(inputs, node, name='test/Add')
-        relu(node, name='test/' + relu_op_name)
-
-      with self.assertRaises(ValueError):
-        fold_batch_norms.FoldBatchNorms(g)
-
-  def _TestFoldConv2d(self, relu, relu_op_name, with_bypass):
+  def _TestFoldConv2d(self, relu, relu_op_name, with_bypass, has_scaling,
+                      fused_batch_norm):
     """Tests folding cases: inputs -> Conv2d with batch norm -> Relu*.
 
     Args:
@@ -93,6 +67,8 @@ class FoldBatchNormsTest(test_util.TensorFlowTestCase):
       relu_op_name: String, name of the Relu* operation.
       with_bypass: Bool, when true there is an extra connection added from
         inputs to just before Relu*.
+      has_scaling: Bool, when true the batch norm has scaling.
+      fused_batch_norm: Bool, when true the batch norm is fused.
     """
     g = ops.Graph()
     with g.as_default():
@@ -102,12 +78,17 @@ class FoldBatchNormsTest(test_util.TensorFlowTestCase):
       stride = 1 if with_bypass else 2
       activation_fn = None if with_bypass else relu
       scope = 'test/test2' if with_bypass else 'test'
-      node = conv2d(inputs, out_depth, [5, 5], stride=stride, padding='SAME',
-                    weights_initializer=self._WeightInit(0.09),
-                    activation_fn=activation_fn,
-                    normalizer_fn=batch_norm,
-                    normalizer_params=_DEFAULT_BATCH_NORM_PARAMS,
-                    scope=scope)
+      node = conv2d(
+          inputs,
+          out_depth, [5, 5],
+          stride=stride,
+          padding='SAME',
+          weights_initializer=self._WeightInit(0.09),
+          activation_fn=activation_fn,
+          normalizer_fn=batch_norm,
+          normalizer_params=self._BatchNormParams(
+              scale=has_scaling, fused=fused_batch_norm),
+          scope=scope)
       if with_bypass:
         node = math_ops.add(inputs, node, name='test/Add')
         relu(node, name='test/' + relu_op_name)
@@ -116,9 +97,10 @@ class FoldBatchNormsTest(test_util.TensorFlowTestCase):
 
     folded_mul = g.get_operation_by_name(scope + '/mul_fold')
     self.assertEqual(folded_mul.type, 'Mul')
-    self._AssertInputOpsAre(folded_mul,
-                            [scope + '/weights/read',
-                             scope + '/BatchNorm/batchnorm/mul'])
+    self._AssertInputOpsAre(folded_mul, [
+        scope + '/weights/read',
+        self._BatchNormMultiplierName(scope, has_scaling, fused_batch_norm)
+    ])
     self._AssertOutputGoesToOps(folded_mul, g, [scope + '/convolution_Fold'])
 
     folded_conv = g.get_operation_by_name(scope + '/convolution_Fold')
@@ -129,16 +111,18 @@ class FoldBatchNormsTest(test_util.TensorFlowTestCase):
 
     folded_add = g.get_operation_by_name(scope + '/add_fold')
     self.assertEqual(folded_add.type, 'Add')
-    self._AssertInputOpsAre(folded_add,
-                            [scope + '/convolution_Fold',
-                             scope + '/BatchNorm/batchnorm/sub'])
+    self._AssertInputOpsAre(folded_add, [
+        scope + '/convolution_Fold',
+        self._BathNormBiasName(scope, fused_batch_norm)
+    ])
     output_op_names = ['test/Add' if with_bypass else 'test/' + relu_op_name]
     self._AssertOutputGoesToOps(folded_add, g, output_op_names)
 
   def testFoldConv2d(self):
     self._RunTestOverParameters(self._TestFoldConv2d)
 
-  def _TestFoldConv2dUnknownShape(self, relu, relu_op_name, with_bypass):
+  def _TestFoldConv2dUnknownShape(self, relu, relu_op_name, with_bypass,
+                                  has_scaling, fused_batch_norm):
     """Tests folding cases: inputs -> Conv2d with batch norm -> Relu*.
 
     Tests that folding works even with an input shape where some dimensions are
@@ -149,6 +133,8 @@ class FoldBatchNormsTest(test_util.TensorFlowTestCase):
       relu_op_name: String, name of the Relu* operation.
       with_bypass: Bool, when true there is an extra connection added from
         inputs to just before Relu*.
+      has_scaling: Bool, when true the batch norm has scaling.
+      fused_batch_norm: Bool, when true the batch norm is fused.
     """
     g = ops.Graph()
     with g.as_default():
@@ -165,7 +151,8 @@ class FoldBatchNormsTest(test_util.TensorFlowTestCase):
           weights_initializer=self._WeightInit(0.09),
           activation_fn=activation_fn,
           normalizer_fn=batch_norm,
-          normalizer_params=_DEFAULT_BATCH_NORM_PARAMS,
+          normalizer_params=self._BatchNormParams(
+              scale=has_scaling, fused=fused_batch_norm),
           scope=scope)
       if with_bypass:
         node = math_ops.add(inputs, node, name='test/Add')
@@ -176,7 +163,8 @@ class FoldBatchNormsTest(test_util.TensorFlowTestCase):
     folded_mul = g.get_operation_by_name(scope + '/mul_fold')
     self.assertEqual(folded_mul.type, 'Mul')
     self._AssertInputOpsAre(folded_mul, [
-        scope + '/weights/read', scope + '/BatchNorm/batchnorm/mul'
+        scope + '/weights/read',
+        self._BatchNormMultiplierName(scope, has_scaling, fused_batch_norm)
     ])
     self._AssertOutputGoesToOps(folded_mul, g, [scope + '/convolution_Fold'])
 
@@ -188,7 +176,8 @@ class FoldBatchNormsTest(test_util.TensorFlowTestCase):
     folded_add = g.get_operation_by_name(scope + '/add_fold')
     self.assertEqual(folded_add.type, 'Add')
     self._AssertInputOpsAre(folded_add, [
-        scope + '/convolution_Fold', scope + '/BatchNorm/batchnorm/sub'
+        scope + '/convolution_Fold',
+        self._BathNormBiasName(scope, fused_batch_norm)
     ])
     output_op_names = ['test/Add' if with_bypass else 'test/' + relu_op_name]
     self._AssertOutputGoesToOps(folded_add, g, output_op_names)
@@ -196,62 +185,8 @@ class FoldBatchNormsTest(test_util.TensorFlowTestCase):
   def testFoldConv2dUnknownShape(self):
     self._RunTestOverParameters(self._TestFoldConv2dUnknownShape)
 
-  def _TestFoldConv2dWithoutScale(self, relu, relu_op_name, with_bypass):
-    """Tests folding cases: inputs -> Conv2d with batch norm -> Relu*.
-
-    Args:
-      relu: Callable that returns an Operation, a factory method for the Relu*.
-      relu_op_name: String, name of the Relu* operation.
-      with_bypass: Bool, when true there is an extra connection added from
-        inputs to just before Relu*.
-    """
-    g = ops.Graph()
-    with g.as_default():
-      batch_size, height, width = 5, 128, 128
-      inputs = array_ops.zeros((batch_size, height, width, 3))
-      out_depth = 3 if with_bypass else 32
-      stride = 1 if with_bypass else 2
-      activation_fn = None if with_bypass else relu
-      bn_params = copy.copy(_DEFAULT_BATCH_NORM_PARAMS)
-      bn_params['scale'] = False
-      scope = 'test/test2' if with_bypass else 'test'
-      node = conv2d(inputs, out_depth, [5, 5], stride=stride, padding='SAME',
-                    weights_initializer=self._WeightInit(0.09),
-                    activation_fn=activation_fn,
-                    normalizer_fn=batch_norm,
-                    normalizer_params=bn_params,
-                    scope=scope)
-      if with_bypass:
-        node = math_ops.add(inputs, node, name='test/Add')
-        relu(node, name='test/' + relu_op_name)
-
-      fold_batch_norms.FoldBatchNorms(g)
-
-    folded_mul = g.get_operation_by_name(scope + '/mul_fold')
-    self.assertEqual(folded_mul.type, 'Mul')
-    self._AssertInputOpsAre(folded_mul,
-                            [scope + '/weights/read',
-                             scope + '/BatchNorm/batchnorm/Rsqrt'])
-    self._AssertOutputGoesToOps(folded_mul, g, [scope + '/convolution_Fold'])
-
-    folded_conv = g.get_operation_by_name(scope + '/convolution_Fold')
-    self.assertEqual(folded_conv.type, 'Conv2D')
-    self._AssertInputOpsAre(folded_conv,
-                            [scope + '/mul_fold', inputs.op.name])
-    self._AssertOutputGoesToOps(folded_conv, g, [scope + '/add_fold'])
-
-    folded_add = g.get_operation_by_name(scope + '/add_fold')
-    self.assertEqual(folded_add.type, 'Add')
-    self._AssertInputOpsAre(folded_add,
-                            [scope + '/convolution_Fold',
-                             scope + '/BatchNorm/batchnorm/sub'])
-    output_op_names = ['test/Add' if with_bypass else 'test/' + relu_op_name]
-    self._AssertOutputGoesToOps(folded_add, g, output_op_names)
-
-  def testFoldConv2dWithoutScale(self):
-    self._RunTestOverParameters(self._TestFoldConv2dWithoutScale)
-
-  def _TestFoldFullyConnectedLayer(self, relu, relu_op_name, with_bypass):
+  def _TestFoldFullyConnectedLayer(self, relu, relu_op_name, with_bypass,
+                                   has_scaling, fused_batch_norm):
     """Tests folding cases: inputs -> FC with batch norm -> Relu*.
 
     Args:
@@ -259,6 +194,8 @@ class FoldBatchNormsTest(test_util.TensorFlowTestCase):
       relu_op_name: String, name of the Relu* operation.
       with_bypass: Bool, when true there is an extra connection added from
         inputs to just before Relu*.
+      has_scaling: Bool, when true the batch norm has scaling.
+      fused_batch_norm: Bool, when true the batch norm is fused.
     """
     g = ops.Graph()
     with g.as_default():
@@ -267,12 +204,15 @@ class FoldBatchNormsTest(test_util.TensorFlowTestCase):
       out_depth = 256 if with_bypass else 128
       activation_fn = None if with_bypass else relu
       scope = 'test/test2' if with_bypass else 'test'
-      node = fully_connected(inputs, out_depth,
-                             weights_initializer=self._WeightInit(0.03),
-                             activation_fn=activation_fn,
-                             normalizer_fn=batch_norm,
-                             normalizer_params=_DEFAULT_BATCH_NORM_PARAMS,
-                             scope=scope)
+      node = fully_connected(
+          inputs,
+          out_depth,
+          weights_initializer=self._WeightInit(0.03),
+          activation_fn=activation_fn,
+          normalizer_fn=batch_norm,
+          normalizer_params=self._BatchNormParams(
+              scale=has_scaling, fused=fused_batch_norm),
+          scope=scope)
       if with_bypass:
         node = math_ops.add(inputs, node, name='test/Add')
         relu(node, name='test/' + relu_op_name)
@@ -281,9 +221,10 @@ class FoldBatchNormsTest(test_util.TensorFlowTestCase):
 
     folded_mul = g.get_operation_by_name(scope + '/mul_fold')
     self.assertEqual(folded_mul.type, 'Mul')
-    self._AssertInputOpsAre(folded_mul,
-                            [scope + '/weights/read',
-                             scope + '/BatchNorm/batchnorm/mul'])
+    self._AssertInputOpsAre(folded_mul, [
+        scope + '/weights/read',
+        self._BatchNormMultiplierName(scope, has_scaling, fused_batch_norm)
+    ])
     self._AssertOutputGoesToOps(folded_mul, g, [scope + '/MatMul_Fold'])
 
     folded_conv = g.get_operation_by_name(scope + '/MatMul_Fold')
@@ -294,71 +235,18 @@ class FoldBatchNormsTest(test_util.TensorFlowTestCase):
 
     folded_add = g.get_operation_by_name(scope + '/add_fold')
     self.assertEqual(folded_add.type, 'Add')
-    self._AssertInputOpsAre(folded_add,
-                            [scope + '/MatMul_Fold',
-                             scope + '/BatchNorm/batchnorm/sub'])
+    self._AssertInputOpsAre(folded_add, [
+        scope + '/MatMul_Fold',
+        self._BathNormBiasName(scope, fused_batch_norm)
+    ])
     output_op_names = ['test/Add' if with_bypass else 'test/' + relu_op_name]
     self._AssertOutputGoesToOps(folded_add, g, output_op_names)
 
   def testFoldFullyConnectedLayer(self):
     self._RunTestOverParameters(self._TestFoldFullyConnectedLayer)
 
-  def _TestFoldFullyConnectedLayerWithoutScale(self, relu, relu_op_name,
-                                               with_bypass):
-    """Tests folding cases: inputs -> FC with batch norm -> Relu*.
-
-    Args:
-      relu: Callable that returns an Operation, a factory method for the Relu*.
-      relu_op_name: String, name of the Relu* operation.
-      with_bypass: Bool, when true there is an extra connection added from
-        inputs to just before Relu*.
-    """
-    g = ops.Graph()
-    with g.as_default():
-      batch_size, depth = 5, 256
-      inputs = array_ops.zeros((batch_size, depth))
-      out_depth = 256 if with_bypass else 128
-      activation_fn = None if with_bypass else relu
-      bn_params = copy.copy(_DEFAULT_BATCH_NORM_PARAMS)
-      bn_params['scale'] = False
-      scope = 'test/test2' if with_bypass else 'test'
-      node = fully_connected(inputs, out_depth,
-                             weights_initializer=self._WeightInit(0.03),
-                             activation_fn=activation_fn,
-                             normalizer_fn=batch_norm,
-                             normalizer_params=bn_params,
-                             scope=scope)
-      if with_bypass:
-        node = math_ops.add(inputs, node, name='test/Add')
-        relu(node, name='test/' + relu_op_name)
-
-      fold_batch_norms.FoldBatchNorms(g)
-
-    folded_mul = g.get_operation_by_name(scope + '/mul_fold')
-    self.assertEqual(folded_mul.type, 'Mul')
-    self._AssertInputOpsAre(folded_mul,
-                            [scope + '/weights/read',
-                             scope + '/BatchNorm/batchnorm/Rsqrt'])
-    self._AssertOutputGoesToOps(folded_mul, g, [scope + '/MatMul_Fold'])
-
-    folded_conv = g.get_operation_by_name(scope + '/MatMul_Fold')
-    self.assertEqual(folded_conv.type, 'MatMul')
-    self._AssertInputOpsAre(folded_conv,
-                            [scope + '/mul_fold', inputs.op.name])
-    self._AssertOutputGoesToOps(folded_conv, g, [scope + '/add_fold'])
-
-    folded_add = g.get_operation_by_name(scope + '/add_fold')
-    self.assertEqual(folded_add.type, 'Add')
-    self._AssertInputOpsAre(folded_add,
-                            [scope + '/MatMul_Fold',
-                             scope + '/BatchNorm/batchnorm/sub'])
-    output_op_names = ['test/Add' if with_bypass else 'test/' + relu_op_name]
-    self._AssertOutputGoesToOps(folded_add, g, output_op_names)
-
-  def testFoldFullyConnectedLayerWithoutScale(self):
-    self._RunTestOverParameters(self._TestFoldFullyConnectedLayerWithoutScale)
-
-  def _TestFoldDepthwiseConv2d(self, relu, relu_op_name, with_bypass):
+  def _TestFoldDepthwiseConv2d(self, relu, relu_op_name, with_bypass,
+                               has_scaling, fused_batch_norm):
     """Tests folding: inputs -> DepthwiseConv2d with batch norm -> Relu*.
 
     Args:
@@ -366,6 +254,8 @@ class FoldBatchNormsTest(test_util.TensorFlowTestCase):
       relu_op_name: String, name of the Relu* operation.
       with_bypass: Bool, when true there is an extra connection added from
         inputs to just before Relu*.
+      has_scaling: Bool, when true the batch norm has scaling.
+      fused_batch_norm: Bool, when true the batch norm is fused.
     """
     g = ops.Graph()
     with g.as_default():
@@ -374,13 +264,18 @@ class FoldBatchNormsTest(test_util.TensorFlowTestCase):
       stride = 1 if with_bypass else 2
       activation_fn = None if with_bypass else relu
       scope = 'test/test2' if with_bypass else 'test'
-      node = separable_conv2d(inputs, None, [5, 5], stride=stride,
-                              depth_multiplier=1.0, padding='SAME',
-                              weights_initializer=self._WeightInit(0.09),
-                              activation_fn=activation_fn,
-                              normalizer_fn=batch_norm,
-                              normalizer_params=_DEFAULT_BATCH_NORM_PARAMS,
-                              scope=scope)
+      node = separable_conv2d(
+          inputs,
+          None, [5, 5],
+          stride=stride,
+          depth_multiplier=1.0,
+          padding='SAME',
+          weights_initializer=self._WeightInit(0.09),
+          activation_fn=activation_fn,
+          normalizer_fn=batch_norm,
+          normalizer_params=self._BatchNormParams(
+              scale=has_scaling, fused=fused_batch_norm),
+          scope=scope)
       if with_bypass:
         node = math_ops.add(inputs, node, name='test/Add')
         relu(node, name='test/' + relu_op_name)
@@ -396,9 +291,10 @@ class FoldBatchNormsTest(test_util.TensorFlowTestCase):
 
     scale_reshape = g.get_operation_by_name(scope + '/scale_reshape')
     self.assertEqual(scale_reshape.type, 'Reshape')
-    self._AssertInputOpsAre(scale_reshape,
-                            [scope + '/BatchNorm/batchnorm/mul',
-                             scope + '/scale_reshape/shape'])
+    self._AssertInputOpsAre(scale_reshape, [
+        self._BatchNormMultiplierName(scope, has_scaling, fused_batch_norm),
+        scope + '/scale_reshape/shape'
+    ])
     self._AssertOutputGoesToOps(scale_reshape, g, [scope + '/mul_fold'])
 
     folded_conv = g.get_operation_by_name(scope + '/depthwise_Fold')
@@ -409,77 +305,35 @@ class FoldBatchNormsTest(test_util.TensorFlowTestCase):
 
     folded_add = g.get_operation_by_name(scope + '/add_fold')
     self.assertEqual(folded_add.type, 'Add')
-    self._AssertInputOpsAre(folded_add,
-                            [scope + '/depthwise_Fold',
-                             scope + '/BatchNorm/batchnorm/sub'])
+    self._AssertInputOpsAre(folded_add, [
+        scope + '/depthwise_Fold',
+        self._BathNormBiasName(scope, fused_batch_norm)
+    ])
     output_op_names = ['test/Add' if with_bypass else 'test/' + relu_op_name]
     self._AssertOutputGoesToOps(folded_add, g, output_op_names)
 
   def testFoldDepthwiseConv2d(self):
     self._RunTestOverParameters(self._TestFoldDepthwiseConv2d)
 
-  def _TestFoldDepthwiseConv2dWithoutScale(self, relu, relu_op_name,
-                                           with_bypass):
-    """Tests folding: inputs -> DepthwiseConv2d with batch norm -> Relu*.
-
-    Args:
-      relu: Callable that returns an Operation, a factory method for the Relu*.
-      relu_op_name: String, name of the Relu* operation.
-      with_bypass: Bool, when true there is an extra connection added from
-        inputs to just before Relu*.
-    """
-    g = ops.Graph()
-    with g.as_default():
-      batch_size, height, width = 5, 128, 128
-      inputs = array_ops.zeros((batch_size, height, width, 3))
-      stride = 1 if with_bypass else 2
-      activation_fn = None if with_bypass else relu
-      bn_params = copy.copy(_DEFAULT_BATCH_NORM_PARAMS)
-      bn_params['scale'] = False
-      scope = 'test/test2' if with_bypass else 'test'
-      node = separable_conv2d(inputs, None, [5, 5], stride=stride,
-                              depth_multiplier=1.0, padding='SAME',
-                              weights_initializer=self._WeightInit(0.09),
-                              activation_fn=activation_fn,
-                              normalizer_fn=batch_norm,
-                              normalizer_params=bn_params,
-                              scope=scope)
-      if with_bypass:
-        node = math_ops.add(inputs, node, name='test/Add')
-        relu(node, name='test/' + relu_op_name)
-
-      fold_batch_norms.FoldBatchNorms(g)
-
-    folded_mul = g.get_operation_by_name(scope + '/mul_fold')
-    self.assertEqual(folded_mul.type, 'Mul')
-    self._AssertInputOpsAre(folded_mul,
-                            [scope + '/depthwise_weights/read',
-                             scope + '/scale_reshape'])
-    self._AssertOutputGoesToOps(folded_mul, g, [scope + '/depthwise_Fold'])
-
-    scale_reshape = g.get_operation_by_name(scope + '/scale_reshape')
-    self.assertEqual(scale_reshape.type, 'Reshape')
-    self._AssertInputOpsAre(scale_reshape,
-                            [scope + '/BatchNorm/batchnorm/Rsqrt',
-                             scope + '/scale_reshape/shape'])
-    self._AssertOutputGoesToOps(scale_reshape, g, [scope + '/mul_fold'])
-
-    folded_conv = g.get_operation_by_name(scope + '/depthwise_Fold')
-    self.assertEqual(folded_conv.type, 'DepthwiseConv2dNative')
-    self._AssertInputOpsAre(folded_conv,
-                            [scope + '/mul_fold', inputs.op.name])
-    self._AssertOutputGoesToOps(folded_conv, g, [scope + '/add_fold'])
-
-    folded_add = g.get_operation_by_name(scope + '/add_fold')
-    self.assertEqual(folded_add.type, 'Add')
-    self._AssertInputOpsAre(folded_add,
-                            [scope + '/depthwise_Fold',
-                             scope + '/BatchNorm/batchnorm/sub'])
-    output_op_names = ['test/Add' if with_bypass else 'test/' + relu_op_name]
-    self._AssertOutputGoesToOps(folded_add, g, output_op_names)
-
-  def testFoldDepthwiseConv2dWithoutScale(self):
-    self._RunTestOverParameters(self._TestFoldDepthwiseConv2dWithoutScale)
+  def _BatchNormParams(self, scale=True, fused=False):
+    return {
+        'center': True,
+        'scale': scale,
+        'decay': 1.0 - 0.003,
+        'fused': fused
+    }
+
+  def _BatchNormMultiplierName(self, scope, has_scaling, fused):
+    if has_scaling:
+      if fused:
+        return scope + '/mul'
+      return scope + '/BatchNorm/batchnorm/mul'
+    return scope + '/BatchNorm/batchnorm/Rsqrt'
+
+  def _BathNormBiasName(self, scope, fused):
+    if fused:
+      return scope + '/bias'
+    return scope + '/BatchNorm/batchnorm/sub'
 
   def _WeightInit(self, stddev):
     """Returns a truncated normal variable initializer.
diff --git a/tensorflow/contrib/quantize/python/graph_matcher.py b/tensorflow/contrib/quantize/python/graph_matcher.py
new file mode 100644
index 0000000000..e3581cc559
--- /dev/null
+++ b/tensorflow/contrib/quantize/python/graph_matcher.py
@@ -0,0 +1,200 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Utilities that match patterns in a tf.Graph."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+
+class OpTypePattern(object):
+  """A tree pattern that matches TF expressions with certain op types."""
+
+  def __init__(self, op_type, name=None, inputs=None):
+    """Initializes an OpTypePattern.
+
+    Args:
+      op_type: string that specifies the allowed types of the root. It can be
+        (1) an op type, e.g. 'Conv2D',
+        (2) '*', i.e. wildcard, or
+        (3) multiple op types separated by '|', e.g., 'Relu|Relu6'.
+        We could use regex strings, which might be worthwhile when we have many
+        similar TF op types.
+      name: Optional string. The name of the pattern that can be looked up in
+        MatchResult.
+      inputs: Optional list of `OpTypePattern`s or strings that specify the
+        patterns for the inputs of a matching op. If None, this pattern accepts
+        any inputs of a matching op.
+    """
+    self._op_type = op_type
+    self._name = name
+    if inputs is None:
+      inputs = []
+    self._inputs = [
+        input_pattern if isinstance(input_pattern, OpTypePattern) else
+        OpTypePattern(input_pattern) for input_pattern in inputs
+    ]
+
+  @property
+  def op_type(self):
+    return self._op_type
+
+  @property
+  def inputs(self):
+    return self._inputs
+
+  @property
+  def name(self):
+    return self._name
+
+
+class MatchResult(object):
+  r"""Encapsulates the result of a match done by GraphMatcher.
+
+  MatchResult contains a map from OpTypePattern to the matching op and tensor.
+  When the matching op has multiple output tensors, the matching tensor is the
+  output tensor used by the matching op of the parent pattern. E.g., when we
+  match graph
+
+      -         +
+     / \y0   y1/ \
+    x    split    z
+          |
+          y         (nodes are ops; edges are going up)
+
+  against add_pattern defined as
+
+    y1_pattern = OpTypePattern('*')
+    z_pattern = OpTypePattern('*')
+    add_pattern = OpTypePattern('+', inputs=[y1_pattern, z_pattern])
+
+  the matching op of `y1_pattern` is `split`, and the matching tensor of
+  `y1_pattern`
+  is `y1` not `y0`.
+  """
+
+  def __init__(self):
+    self._pattern_to_op_tensor = {}
+    self._name_to_pattern = {}
+
+  def add(self, pattern, op, tensor):
+    self._pattern_to_op_tensor[pattern] = op, tensor
+    if pattern.name is not None:
+      if pattern.name in self._name_to_pattern:
+        raise ValueError(
+            'Name %s is already bound to another pattern' % pattern.name)
+      self._name_to_pattern[pattern.name] = pattern
+
+  def _to_pattern(self, pattern_or_name):
+    if isinstance(pattern_or_name, OpTypePattern):
+      return pattern_or_name
+
+    if isinstance(pattern_or_name, str):
+      return self._name_to_pattern[pattern_or_name]
+
+    raise ValueError('pattern_or_name has type %s. Expect OpTypePattern or str.'
+                     % type(pattern_or_name))
+
+  def get_op(self, pattern_or_name):
+    return self._pattern_to_op_tensor[self._to_pattern(pattern_or_name)][0]
+
+  def get_tensor(self, pattern_or_name):
+    return self._pattern_to_op_tensor[self._to_pattern(pattern_or_name)][1]
+
+
+class GraphMatcher(object):
+  """Checks if a particular subgraph matches a given pattern."""
+
+  def __init__(self, pattern):
+    """Initializes a GraphMatcher.
+
+    Args:
+      pattern: The `OpTypePattern` against which `GraphMatcher` matches
+        subgraphs.
+    """
+    self._pattern = pattern
+
+  def _match_pattern(self, pattern, op, tensor):
+    """Returns whether an TF expression rooted at `op` matches `pattern`.
+
+    If there is a match, adds to `self._match_result` the matching op and tensor
+    with key `pattern`.
+
+    Args:
+      pattern: An `OpTypePattern`.
+      op: A `tf.Operation` to match against the pattern.
+      tensor: the output `tf.Tensor` of `op` that is used by the matching op of
+        `pattern`'s parent. Can be None if `pattern` is already the root of the
+        pattern tree.
+
+    Returns:
+      True if an TF expression rooted at `op` matches `pattern`.
+    """
+    if pattern.op_type != '*':
+      if op.type not in pattern.op_type.split('|'):
+        return False
+
+    self._match_result.add(pattern, op, tensor)
+
+    if not pattern.inputs:
+      # If pattern.inputs is empty, skips the rest and accepts all the inputs.
+      return True
+
+    return len(op.inputs) == len(pattern.inputs) and all([
+        self._match_pattern(input_pattern, input_tensor.op, input_tensor)
+        for input_tensor, input_pattern in zip(op.inputs, pattern.inputs)
+    ])
+
+  def match_op(self, op):
+    """Matches `op` against `self._pattern`.
+
+    Args:
+      op: `tf.Operation` to match against the pattern.
+
+    Returns:
+      Returns a `MatchResult` if `op` matches the pattern; otherwise, returns
+      None.
+    """
+    self._match_result = MatchResult()
+    if not self._match_pattern(self._pattern, op, tensor=None):
+      return None
+    return self._match_result
+
+  def match_ops(self, ops):
+    """Matches each operation in `ops` against `self._pattern`.
+
+    Args:
+      ops: collection of `tf.Operation` to match against the pattern.
+
+    Yields:
+      `MatchResult` for each `tf.Operation` that matches the pattern.
+    """
+    for op in ops:
+      match_result = self.match_op(op)
+      if match_result:
+        yield match_result
+
+  def match_graph(self, graph):
+    """Matches each operation in `graph` against `self._pattern`.
+
+    Args:
+      graph: `tf.Graph` containing operations to match.
+
+    Yields:
+      `MatchResult` for each `tf.Operation` in `graph` that matches the pattern.
+    """
+    # Python 3.3.2+ implements `yield from`, but for now:
+    for match_result in self.match_ops(graph.get_operations()):
+      yield match_result
diff --git a/tensorflow/contrib/quantize/python/graph_matcher_test.py b/tensorflow/contrib/quantize/python/graph_matcher_test.py
new file mode 100644
index 0000000000..e1572865e4
--- /dev/null
+++ b/tensorflow/contrib/quantize/python/graph_matcher_test.py
@@ -0,0 +1,130 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for graph_matcher."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.contrib.framework.python import ops as contrib_ops
+from tensorflow.contrib.layers.python.layers import initializers
+from tensorflow.contrib.layers.python.layers import layers
+from tensorflow.contrib.quantize.python import graph_matcher
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import init_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import nn_ops
+from tensorflow.python.platform import googletest
+
+
+class GraphMatcherTest(test_util.TensorFlowTestCase):
+
+  def test_conv_layer(self):
+    g = ops.Graph()
+    with g.as_default():
+      inputs = array_ops.placeholder(dtypes.float32, shape=[8, 5, 5, 3])
+
+    with contrib_ops.arg_scope(
+        [layers.batch_norm], fused=True, is_training=True, trainable=True):
+      return layers.convolution(
+          inputs,
+          num_outputs=16,
+          kernel_size=3,
+          stride=1,
+          padding='VALID',
+          activation_fn=nn_ops.relu,
+          normalizer_fn=layers.batch_norm,
+          normalizer_params={},
+          weights_initializer=initializers.xavier_initializer(),
+          weights_regularizer=None,
+          biases_initializer=init_ops.zeros_initializer(),
+          biases_regularizer=None,
+          reuse=None,
+          trainable=True,
+          scope=None)
+
+    inputs_pattern = graph_matcher.OpTypePattern('*', name='inputs')
+    relu_pattern = graph_matcher.OpTypePattern(
+        'Relu',
+        name='relu',
+        inputs=[
+            graph_matcher.OpTypePattern(
+                'FusedBatchNorm',
+                inputs=[
+                    graph_matcher.OpTypePattern(
+                        'Conv2D', inputs=[inputs_pattern, '*']), '*', '*', '*',
+                    '*'
+                ])
+        ])
+    matcher = graph_matcher.GraphMatcher(relu_pattern)
+    match_results = list(matcher.match_graph(g))
+    self.assertEqual(1, len(match_results))
+    match_result = match_results[0]
+    self.assertEqual(match_result.get_tensor(inputs_pattern), inputs)
+    self.assertEqual(match_result.get_tensor('inputs'), inputs)
+
+  def test_multiple_outputs(self):
+    #   -         +
+    #  / \y0   y1/ \
+    # x    split    z
+    #       |
+    #       y         (nodes are ops; edges are going up)
+    g = ops.Graph()
+    with g.as_default():
+      x = array_ops.placeholder(dtypes.float32, shape=[1], name='x')
+      y = array_ops.placeholder(dtypes.float32, shape=[2], name='y')
+      y0, y1 = array_ops.split(y, num_or_size_splits=2, axis=0)
+      z = array_ops.placeholder(dtypes.float32, shape=[1], name='z')
+      math_ops.add(x, y0)
+      math_ops.subtract(y1, z)
+
+    y1_pattern = graph_matcher.OpTypePattern('*')
+    minus_pattern = graph_matcher.OpTypePattern('Sub', inputs=[y1_pattern, '*'])
+    matcher = graph_matcher.GraphMatcher(minus_pattern)
+
+    match_results = list(matcher.match_graph(g))
+    self.assertEqual(1, len(match_results))
+    match_result = match_results[0]
+
+    self.assertEqual(y0.op, y1.op)
+    self.assertEqual(match_result.get_op(y1_pattern), y1.op)
+    self.assertEqual(match_result.get_tensor(y1_pattern), y1)
+
+  def test_oneof_pattern(self):
+    #   -   +
+    #  / \ / \
+    # x   y   z
+    g = ops.Graph()
+    with g.as_default():
+      x = array_ops.placeholder(dtypes.float32, shape=[], name='x')
+      y = array_ops.placeholder(dtypes.float32, shape=[], name='y')
+      z = array_ops.placeholder(dtypes.float32, shape=[], name='z')
+      plus = x + y
+      minus = y - z
+
+    add_or_sub_pattern = graph_matcher.OpTypePattern(
+        'Add|Sub', inputs=['*', '*'])
+    matcher = graph_matcher.GraphMatcher(add_or_sub_pattern)
+    self.assertEqual([
+        match_result.get_op(add_or_sub_pattern)
+        for match_result in matcher.match_graph(g)
+    ], [plus.op, minus.op])
+
+
+if __name__ == '__main__':
+  googletest.main()
diff --git a/tensorflow/contrib/quantize/python/quantize_parameterized_test.py b/tensorflow/contrib/quantize/python/quantize_parameterized_test.py
index b5a32a7266..31fcd66dfb 100644
--- a/tensorflow/contrib/quantize/python/quantize_parameterized_test.py
+++ b/tensorflow/contrib/quantize/python/quantize_parameterized_test.py
@@ -19,6 +19,7 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.contrib.layers.python.layers import layers
+from tensorflow.contrib.quantize.python import fold_batch_norms
 from tensorflow.contrib.quantize.python import quantize
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util
@@ -35,18 +36,11 @@ conv2d = layers.conv2d
 fully_connected = layers.fully_connected
 separable_conv2d = layers.separable_conv2d
 
-_DEFAULT_BATCH_NORM_PARAMS = {
-    'center': True,
-    'scale': True,
-    'decay': 1.0 - 0.003,
-    'fused': False,
-}
 
-
-# TODO(suharshs): Use parameterized test once OSS TF supports it.
 class QuantizeTest(test_util.TensorFlowTestCase):
 
-  def _RunTestOverParameters(self, test_fn):
+  def _RunWithoutBatchNormTestOverParameters(self, test_fn):
+    # TODO(suharshs): Use parameterized test once OSS TF supports it.
     parameters_list = [
         # (activation, activation_op_name, with_bypass, delay)
         (nn_ops.relu6, 'Relu6', False, None),
@@ -60,10 +54,10 @@ class QuantizeTest(test_util.TensorFlowTestCase):
         (array_ops.identity, 'Identity', True, None),
         (nn_ops.relu6, 'Relu6', True, 5000),
         (nn_ops.relu, 'Relu', True, 5000),
-        (array_ops.identity, 'Identity', True, 5000)
+        (array_ops.identity, 'Identity', True, 5000),
     ]
-    for parameters in parameters_list:
-      test_fn(parameters[0], parameters[1], parameters[2], parameters[3])
+    for params in parameters_list:
+      test_fn(params[0], params[1], params[2], params[3])
 
   def _TestQuantize_Conv2dWithoutBatchNorm(self, activation, activation_op_name,
                                            with_bypass, delay):
@@ -137,7 +131,8 @@ class QuantizeTest(test_util.TensorFlowTestCase):
     self._AssertOutputGoesToOps(act_quant, graph, [output_op_name])
 
   def testQuantize_Conv2dWithoutBatchNorm(self):
-    self._RunTestOverParameters(self._TestQuantize_Conv2dWithoutBatchNorm)
+    self._RunWithoutBatchNormTestOverParameters(
+        self._TestQuantize_Conv2dWithoutBatchNorm)
 
   def _TestQuantize_FCWithoutBatchNorm(self, activation, activation_op_name,
                                        with_bypass, delay):
@@ -210,7 +205,8 @@ class QuantizeTest(test_util.TensorFlowTestCase):
     self._AssertOutputGoesToOps(act_quant, graph, [output_op_name])
 
   def testQuantize_FCWithoutBatchNorm(self):
-    self._RunTestOverParameters(self._TestQuantize_FCWithoutBatchNorm)
+    self._RunWithoutBatchNormTestOverParameters(
+        self._TestQuantize_FCWithoutBatchNorm)
 
   def _TestQuantize_DepthwiseConv2dWithoutBatchNorm(
       self, activation, activation_op_name, with_bypass, delay):
@@ -284,11 +280,43 @@ class QuantizeTest(test_util.TensorFlowTestCase):
     self._AssertOutputGoesToOps(act_quant, graph, [output_op_name])
 
   def testQuantize_DepthwiseConv2dWithoutBatchNorm(self):
-    self._RunTestOverParameters(
+    self._RunWithoutBatchNormTestOverParameters(
         self._TestQuantize_DepthwiseConv2dWithoutBatchNorm)
 
+  def _RunBatchNormTestOverParameters(self, test_fn):
+    # TODO(suharshs): Use parameterized test once OSS TF supports it.
+    parameters_list = [
+        # (activation, activation_op_name, with_bypass, delay, fused_batch_norm)
+        (nn_ops.relu6, 'Relu6', False, None, False),
+        (nn_ops.relu, 'Relu', False, None, False),
+        (array_ops.identity, 'Identity', False, None, False),
+        (nn_ops.relu6, 'Relu6', False, 5000, False),
+        (nn_ops.relu, 'Relu', False, 5000, False),
+        (array_ops.identity, 'Identity', False, 5000, False),
+        (nn_ops.relu6, 'Relu6', True, None, False),
+        (nn_ops.relu, 'Relu', True, None, False),
+        (array_ops.identity, 'Identity', True, None, False),
+        (nn_ops.relu6, 'Relu6', True, 5000, False),
+        (nn_ops.relu, 'Relu', True, 5000, False),
+        (array_ops.identity, 'Identity', True, 5000, False),
+        (nn_ops.relu6, 'Relu6', False, None, True),
+        (nn_ops.relu, 'Relu', False, None, True),
+        (array_ops.identity, 'Identity', False, None, True),
+        (nn_ops.relu6, 'Relu6', False, 5000, True),
+        (nn_ops.relu, 'Relu', False, 5000, True),
+        (array_ops.identity, 'Identity', False, 5000, True),
+        (nn_ops.relu6, 'Relu6', True, None, True),
+        (nn_ops.relu, 'Relu', True, None, True),
+        (array_ops.identity, 'Identity', True, None, True),
+        (nn_ops.relu6, 'Relu6', True, 5000, True),
+        (nn_ops.relu, 'Relu', True, 5000, True),
+        (array_ops.identity, 'Identity', True, 5000, True)
+    ]
+    for params in parameters_list:
+      test_fn(params[0], params[1], params[2], params[3], params[4])
+
   def _TestQuantize_Conv2dWithBatchNorm(self, activation, activation_op_name,
-                                        with_bypass, delay):
+                                        with_bypass, delay, fused_batch_norm):
     """Tests quantization: inputs -> Conv2d with batch norm -> Activation.
 
     Args:
@@ -298,25 +326,29 @@ class QuantizeTest(test_util.TensorFlowTestCase):
       with_bypass: Bool, when true there is an extra connection added from
         inputs to just before Activation.
       delay: Int (optional), delay in number of steps until quantization starts.
+      fused_batch_norm: Bool, when true use FusedBatchNorm.
     """
     self._testQuantize_Conv2dWithBatchNorm(
         activation,
         activation_op_name,
         with_bypass,
         delay,
+        fused_batch_norm,
         use_ema=True)
     self._testQuantize_Conv2dWithBatchNorm(
         activation,
         activation_op_name,
         with_bypass,
         delay,
+        fused_batch_norm,
         use_ema=False)
 
   def testQuantize_Conv2dWithBatchNorm(self):
-    self._RunTestOverParameters(self._TestQuantize_Conv2dWithBatchNorm)
+    self._RunBatchNormTestOverParameters(self._TestQuantize_Conv2dWithBatchNorm)
 
   def _testQuantize_Conv2dWithBatchNorm(self, activation, activation_op_name,
-                                        with_bypass, delay, use_ema):
+                                        with_bypass, delay, fused_batch_norm,
+                                        use_ema):
     """Tests quantization: inputs -> Conv2d with batch norm -> Activation.
 
     Args:
@@ -326,6 +358,7 @@ class QuantizeTest(test_util.TensorFlowTestCase):
       with_bypass: Bool, when true there is an extra connection added from
         inputs to just before Activation.
       delay: Int (optional), delay in number of steps until quantization starts.
+      fused_batch_norm: Bool, when true use FusedBatchNorm.
       use_ema: Bool, when true uses EMA quantization for BN folded weights.
     """
     graph = ops.Graph()
@@ -337,39 +370,29 @@ class QuantizeTest(test_util.TensorFlowTestCase):
       stride = 1 if with_bypass else 2
       out_depth = 3 if with_bypass else 32
       scope = 'test/test2' if with_bypass else 'test'
-      node = conv2d(inputs, out_depth, [5, 5], stride=stride, padding='SAME',
-                    weights_initializer=self._WeightInit(0.09),
-                    activation_fn=None,
-                    normalizer_fn=batch_norm,
-                    normalizer_params=_DEFAULT_BATCH_NORM_PARAMS,
-                    scope=scope)
-      # Manually fold the batch norm.
-      weights = graph.get_operation_by_name(scope + '/weights/read').outputs[0]
-      bn_mult = (graph.get_operation_by_name(scope + '/BatchNorm/batchnorm/mul')
-                 .outputs[0])
-      mul_fold = math_ops.multiply(weights, bn_mult, name=scope + '/mul_fold')
-      stride = [stride, stride]
-      conv_fold = nn_ops.convolution(
-          input=inputs,
-          filter=mul_fold,
+      node = conv2d(
+          inputs,
+          out_depth, [5, 5],
+          stride=stride,
           padding='SAME',
-          strides=stride,
-          data_format='NHWC',
-          name=scope + '/convolution_Fold')
-      bn_bias = (graph.get_operation_by_name(scope + '/BatchNorm/batchnorm/sub')
-                 .outputs[0])
-      add_fold = math_ops.add(conv_fold, bn_bias, name=scope + '/add_fold')
+          weights_initializer=self._WeightInit(0.09),
+          activation_fn=None,
+          normalizer_fn=batch_norm,
+          normalizer_params=self._BatchNormParams(fused_batch_norm),
+          scope=scope)
+
       # Manually add a bypass (optionaly) and an activation.
       if with_bypass:
-        node = math_ops.add(inputs, add_fold, name='test/Add')
-      else:
-        node = add_fold
+        node = math_ops.add(inputs, node, name='test/Add')
+
       node = activation(node, name='test/' + activation_op_name)
 
       update_barrier = control_flow_ops.no_op(name='update_barrier')
       with ops.control_dependencies([update_barrier]):
         array_ops.identity(node, name='control_dependency')
 
+      fold_batch_norms.FoldBatchNorms(graph)
+
       quantize.Quantize(
           graph, quant_delay=delay, quantize_folded_weights_use_ema=use_ema)
 
@@ -413,7 +436,7 @@ class QuantizeTest(test_util.TensorFlowTestCase):
     self._AssertOutputGoesToOps(act_quant, graph, [output_op_name])
 
   def _TestQuantize_FCWithBatchNorm(self, activation, activation_op_name,
-                                    with_bypass, delay):
+                                    with_bypass, delay, fused_batch_norm):
     """Tests quantization: inputs -> FC with batch norm -> Activation.
 
     Args:
@@ -423,25 +446,29 @@ class QuantizeTest(test_util.TensorFlowTestCase):
       with_bypass: Bool, when true there is an extra connection added from
         inputs to just before Activation.
       delay: Int (optional), delay in number of steps until quantization starts.
+      fused_batch_norm: Bool, when true use FusedBatchNorm.
     """
     self._testQuantize_FCWithBatchNorm(
         activation,
         activation_op_name,
         with_bypass,
         delay,
+        fused_batch_norm,
         use_ema=True)
     self._testQuantize_FCWithBatchNorm(
         activation,
         activation_op_name,
         with_bypass,
         delay,
+        fused_batch_norm,
         use_ema=False)
 
   def testQuantize_FCWithBatchNorm(self):
-    self._RunTestOverParameters(self._TestQuantize_FCWithBatchNorm)
+    self._RunBatchNormTestOverParameters(self._TestQuantize_FCWithBatchNorm)
 
   def _testQuantize_FCWithBatchNorm(self, activation, activation_op_name,
-                                    with_bypass, delay, use_ema):
+                                    with_bypass, delay, fused_batch_norm,
+                                    use_ema):
     """Tests quantization: inputs -> FC with batch norm -> Activation.
 
     Args:
@@ -451,6 +478,7 @@ class QuantizeTest(test_util.TensorFlowTestCase):
       with_bypass: Bool, when true there is an extra connection added from
         inputs to just before Activation.
       delay: Int (optional), delay in number of steps until quantization starts.
+      fused_batch_norm: Bool, when true use FusedBatchNorm.
       use_ema: Bool, when true uses EMA quantization for BN folded weights.
     """
     graph = ops.Graph()
@@ -461,32 +489,27 @@ class QuantizeTest(test_util.TensorFlowTestCase):
       inputs = array_ops.zeros((batch_size, depth))
       out_depth = 256 if with_bypass else 128
       scope = 'test/test2' if with_bypass else 'test'
-      node = fully_connected(inputs, out_depth,
-                             weights_initializer=self._WeightInit(0.03),
-                             activation_fn=None,
-                             normalizer_fn=batch_norm,
-                             normalizer_params=_DEFAULT_BATCH_NORM_PARAMS,
-                             scope=scope)
-      # Manually fold the batch norm.
-      weights = graph.get_operation_by_name(scope + '/weights/read').outputs[0]
-      bn_mult = (graph.get_operation_by_name(scope + '/BatchNorm/batchnorm/mul')
-                 .outputs[0])
-      mul_fold = math_ops.multiply(weights, bn_mult, name=scope + '/mul_fold')
-      fc_fold = math_ops.matmul(inputs, mul_fold, name=scope + '/MatMul_Fold')
-      bn_bias = (graph.get_operation_by_name(scope + '/BatchNorm/batchnorm/sub')
-                 .outputs[0])
-      add_fold = math_ops.add(fc_fold, bn_bias, name=scope + '/add_fold')
+      node = fully_connected(
+          inputs,
+          out_depth,
+          weights_initializer=self._WeightInit(0.03),
+          activation_fn=None,
+          normalizer_fn=batch_norm,
+          normalizer_params=self._BatchNormParams(fused_batch_norm),
+          scope=scope)
+
       # Manually add a bypass (optionaly) and an activation.
       if with_bypass:
-        node = math_ops.add(inputs, add_fold, name='test/Add')
-      else:
-        node = add_fold
+        node = math_ops.add(inputs, node, name='test/Add')
+
       node = activation(node, name='test/' + activation_op_name)
 
       update_barrier = control_flow_ops.no_op(name='update_barrier')
       with ops.control_dependencies([update_barrier]):
         array_ops.identity(node, name='control_dependency')
 
+      fold_batch_norms.FoldBatchNorms(graph)
+
       quantize.Quantize(
           graph, quant_delay=delay, quantize_folded_weights_use_ema=use_ema)
 
@@ -530,7 +553,8 @@ class QuantizeTest(test_util.TensorFlowTestCase):
     self._AssertOutputGoesToOps(act_quant, graph, [output_op_name])
 
   def _TestQuantize_DepthwiseConv2dWithBatchNorm(
-      self, activation, activation_op_name, with_bypass, delay):
+      self, activation, activation_op_name, with_bypass, delay,
+      fused_batch_norm):
     """Tests quantization: inputs -> DWConv2d with batch norm -> Activation.
 
     Args:
@@ -540,26 +564,30 @@ class QuantizeTest(test_util.TensorFlowTestCase):
       with_bypass: Bool, when true there is an extra connection added from
         inputs to just before Activation.
       delay: Int (optional), delay in number of steps until quantization starts.
+      fused_batch_norm: Bool, when true use FusedBatchNorm.
     """
     self._testQuantize_DepthwiseConv2dWithBatchNorm(
         activation,
         activation_op_name,
         with_bypass,
         delay,
+        fused_batch_norm,
         use_ema=True)
     self._testQuantize_DepthwiseConv2dWithBatchNorm(
         activation,
         activation_op_name,
         with_bypass,
         delay,
+        fused_batch_norm,
         use_ema=False)
 
   def testQuantize_DepthwiseConv2dWithBatchNorm(self):
-    self._RunTestOverParameters(
-        self._TestQuantize_DepthwiseConv2dWithoutBatchNorm)
+    self._RunBatchNormTestOverParameters(
+        self._TestQuantize_DepthwiseConv2dWithBatchNorm)
 
   def _testQuantize_DepthwiseConv2dWithBatchNorm(
-      self, activation, activation_op_name, with_bypass, delay, use_ema):
+      self, activation, activation_op_name, with_bypass, delay,
+      fused_batch_norm, use_ema):
     """Tests quantization: inputs -> DWConv2d with batch norm -> Activation.
 
     Args:
@@ -569,6 +597,7 @@ class QuantizeTest(test_util.TensorFlowTestCase):
       with_bypass: Bool, when true there is an extra connection added from
         inputs to just before Activation.
       delay: Int (optional), delay in number of steps until quantization starts.
+      fused_batch_norm: Bool, when true use FusedBatchNorm.
       use_ema: Bool, when true uses EMA quantization for BN folded weights.
     """
     graph = ops.Graph()
@@ -579,46 +608,30 @@ class QuantizeTest(test_util.TensorFlowTestCase):
       inputs = array_ops.zeros((batch_size, height, width, depth))
       stride = 1 if with_bypass else 2
       scope = 'test/test2' if with_bypass else 'test'
-      node = separable_conv2d(inputs, None, [5, 5], stride=stride,
-                              depth_multiplier=1.0, padding='SAME',
-                              weights_initializer=self._WeightInit(0.09),
-                              activation_fn=None,
-                              normalizer_fn=batch_norm,
-                              normalizer_params=_DEFAULT_BATCH_NORM_PARAMS,
-                              scope=scope)
-      # Manually fold the batch norm.
-      weights = (graph.get_operation_by_name(scope + '/depthwise_weights/read')
-                 .outputs[0])
-      bn_mult = (graph.get_operation_by_name(scope + '/BatchNorm/batchnorm/mul')
-                 .outputs[0])
-      new_shape = [
-          weights.get_shape().as_list()[2], weights.get_shape().as_list()[3]
-      ]
-      bn_mult_reshaped = array_ops.reshape(
-          bn_mult, new_shape, name=scope + '/gamma_reshape')
-      mul_fold = math_ops.multiply(
-          weights, bn_mult_reshaped, name=scope + '/mul_fold')
-      stride = [1, stride, stride, 1]
-      conv_fold = nn_ops.depthwise_conv2d(
-          input=inputs,
-          filter=mul_fold,
+      node = separable_conv2d(
+          inputs,
+          None, [5, 5],
+          stride=stride,
+          depth_multiplier=1.0,
           padding='SAME',
-          strides=stride,
-          name=scope + '/depthwise_Fold')
-      bn_bias = (graph.get_operation_by_name(scope + '/BatchNorm/batchnorm/sub')
-                 .outputs[0])
-      add_fold = math_ops.add(conv_fold, bn_bias, name=scope + '/add_fold')
+          weights_initializer=self._WeightInit(0.09),
+          activation_fn=None,
+          normalizer_fn=batch_norm,
+          normalizer_params=self._BatchNormParams(fused_batch_norm),
+          scope=scope)
+
       # Manually add a bypass (optionaly) and an activation.
       if with_bypass:
-        node = math_ops.add(inputs, add_fold, name='test/Add')
-      else:
-        node = add_fold
+        node = math_ops.add(inputs, node, name='test/Add')
+
       node = activation(node, name='test/' + activation_op_name)
 
       update_barrier = control_flow_ops.no_op(name='update_barrier')
       with ops.control_dependencies([update_barrier]):
         array_ops.identity(node, name='control_dependency')
 
+      fold_batch_norms.FoldBatchNorms(graph)
+
       quantize.Quantize(
           graph, quant_delay=delay, quantize_folded_weights_use_ema=use_ema)
     quantization_node_name = 'FakeQuantWithMinMaxVars'
@@ -660,6 +673,9 @@ class QuantizeTest(test_util.TensorFlowTestCase):
                       if delay else 'control_dependency')
     self._AssertOutputGoesToOps(act_quant, graph, [output_op_name])
 
+  def _BatchNormParams(self, fused=False):
+    return {'center': True, 'scale': True, 'decay': 1.0 - 0.003, 'fused': fused}
+
   def _WeightInit(self, stddev):
     """Returns truncated normal variable initializer.
 
-- 
GitLab


From 49f9c6f890c938955fa2d448ac5b556b9a6d9aa0 Mon Sep 17 00:00:00 2001
From: powderluv <powderluv@users.noreply.github.com>
Date: Thu, 19 Oct 2017 23:07:55 -0700
Subject: [PATCH 605/909] Fix ../makefile/download_dependencies.sh on OSX
 (#13845)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

wget expects parameters before the URL on OSX (tested on
version 1.16 and 1.19)

It would fail trying to use -P as a URL

Resolving -p... failed: nodename nor servname provided, or not known.
wget: unable to resolve host address ‘-p’
---
 tensorflow/contrib/makefile/download_dependencies.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/makefile/download_dependencies.sh b/tensorflow/contrib/makefile/download_dependencies.sh
index 39c89628d9..a63cd89e89 100755
--- a/tensorflow/contrib/makefile/download_dependencies.sh
+++ b/tensorflow/contrib/makefile/download_dependencies.sh
@@ -54,7 +54,7 @@ download_and_extract() {
   elif [[ "${url}" == *zip ]]; then
     tempdir=$(mktemp -d)
     tempdir2=$(mktemp -d)
-    wget ${url} -P ${tempdir}
+    wget -P ${tempdir} ${url}
     unzip ${tempdir}/* -d ${tempdir2}
     # unzip has no strip components, so unzip to a temp dir, and move the files
     # we want from the tempdir to destination.
-- 
GitLab


From a528ccdbfe6e4dadad4d982099e8ea5be93fe96f Mon Sep 17 00:00:00 2001
From: Jinze Bai <baijinze1994@163.com>
Date: Fri, 20 Oct 2017 23:20:02 +0800
Subject: [PATCH 606/909] Add GPU support and improve performance for tf.diag
 and tf.diag_part (#13666)

* improve tf.diag and tf.diag_part in CPU and GPU

* add comment

* make changes of DiagOp according to reviews

* tidy indent

* remove uesless comment prefix

* add shard function for DiagOp

* add benchmark for diag_op_test in core/kernel

* change symbol order in BUILD file

* remove empty line for Sanity Checks

* add some comments and fix benchmark throughput ratio for DiagOp
---
 tensorflow/core/graph/testlib.cc              |  18 ++
 tensorflow/core/graph/testlib.h               |   6 +
 tensorflow/core/kernels/BUILD                 |  18 ++
 tensorflow/core/kernels/diag_op.cc            | 295 +++++++++++-------
 tensorflow/core/kernels/diag_op.h             |  43 +++
 tensorflow/core/kernels/diag_op_gpu.cu.cc     | 150 +++++++++
 tensorflow/core/kernels/diag_op_test.cc       |  54 ++++
 tensorflow/core/ops/array_ops.cc              |  10 +-
 tensorflow/core/ops/array_ops_test.cc         |  13 +-
 .../python/kernel_tests/diag_op_test.py       |  64 +++-
 10 files changed, 538 insertions(+), 133 deletions(-)
 create mode 100644 tensorflow/core/kernels/diag_op.h
 create mode 100644 tensorflow/core/kernels/diag_op_gpu.cu.cc
 create mode 100644 tensorflow/core/kernels/diag_op_test.cc

diff --git a/tensorflow/core/graph/testlib.cc b/tensorflow/core/graph/testlib.cc
index be52438747..172471e34b 100644
--- a/tensorflow/core/graph/testlib.cc
+++ b/tensorflow/core/graph/testlib.cc
@@ -480,6 +480,24 @@ Node* Conv2D(Graph* g, Node* in0, Node* in1) {
   return ret;
 }
 
+Node* Diag(Graph* g, Node* in, DataType type) {
+  Node* ret;
+  TF_CHECK_OK(NodeBuilder(g->NewName("n"), "Diag")
+                  .Input(in)
+                  .Attr("T", type)
+                  .Finalize(g, &ret));
+  return ret;
+}
+
+Node* DiagPart(Graph* g, Node* in, DataType type) {
+  Node* ret;
+  TF_CHECK_OK(NodeBuilder(g->NewName("n"), "DiagPart")
+                  .Input(in)
+                  .Attr("T", type)
+                  .Finalize(g, &ret));
+  return ret;
+}
+
 void ToGraphDef(Graph* g, GraphDef* gdef) { g->ToGraphDef(gdef); }
 
 }  // end namespace graph
diff --git a/tensorflow/core/graph/testlib.h b/tensorflow/core/graph/testlib.h
index a38809e6b4..06597778bb 100644
--- a/tensorflow/core/graph/testlib.h
+++ b/tensorflow/core/graph/testlib.h
@@ -199,6 +199,12 @@ Node* BiasAdd(Graph* g, Node* value, Node* bias);
 // Add a Conv2D node in "g".
 Node* Conv2D(Graph* g, Node* in0, Node* in1);
 
+// Add a Diag node in "g".
+Node* Diag(Graph* g, Node* in, DataType type);
+
+// Add a DiagPart node in "g".
+Node* DiagPart(Graph* g, Node* in, DataType type);
+
 }  // end namespace graph
 }  // end namespace test
 }  // end namespace tensorflow
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 3a06189d72..f5700346fd 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -2912,6 +2912,24 @@ tf_cuda_cc_test(
     ],
 )
 
+tf_cuda_cc_test(
+    name = "diag_op_test",
+    size = "small",
+    srcs = ["diag_op_test.cc"],
+    deps = [
+        ":diag_op",
+        ":ops_testutil",
+        ":ops_util",
+        "//tensorflow/core:core_cpu",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+        "//tensorflow/core:testlib",
+    ],
+)
+
 # conv_grad_ops currently has to be built with conv_ops*.
 # TODO(josh11b, zhengxq): put these a separate libraries in ":nn" below once
 # conv_ops_gpu.h has be separated into its own library.
diff --git a/tensorflow/core/kernels/diag_op.cc b/tensorflow/core/kernels/diag_op.cc
index c800859d90..be862b82f1 100644
--- a/tensorflow/core/kernels/diag_op.cc
+++ b/tensorflow/core/kernels/diag_op.cc
@@ -14,65 +14,32 @@ limitations under the License.
 ==============================================================================*/
 
 // See docs in ../ops/array_ops.cc
+
+#define EIGEN_USE_THREADS
+
+#if GOOGLE_CUDA
+#define EIGEN_USE_GPU
+#endif  // GOOGLE_CUDA
+
+#include "tensorflow/core/kernels/diag_op.h"
+
+#include <algorithm>
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/framework/tensor_types.h"
+#include "tensorflow/core/platform/types.h"
 #include "tensorflow/core/platform/logging.h"
+#include "tensorflow/core/util/work_sharder.h"
 
 namespace tensorflow {
-namespace {
-template <typename T, size_t NumDims, size_t DoubleNumDims>
-class DiagonalGenerator {
- public:
-  explicit DiagonalGenerator(const Tensor& diagonal) : diagonal_(diagonal) {
-    static_assert(DoubleNumDims == 2 * NumDims,
-                  "The second size must be the double of the first size.");
-    CHECK_EQ(diagonal.dims(), NumDims);
-  }
-  T operator()(
-      const Eigen::array<Eigen::DenseIndex, DoubleNumDims>& coordinates) const {
-    Eigen::array<Eigen::DenseIndex, NumDims> index;
-    for (size_t i = 0; i < NumDims; ++i) {
-      if (coordinates[i] != coordinates[NumDims + i]) {
-        return T(0);
-      }
-      index[i] = coordinates[i];
-    }
-    return diagonal_.tensor<T, NumDims>()(index);
-  }
 
- private:
-  Tensor diagonal_;
-};
-
-template <typename T, size_t NumDims>
-class DiagonalExtractor {
- public:
-  explicit DiagonalExtractor(const Tensor& tensor) : tensor_(tensor) {
-    CHECK_EQ(tensor.dims(), 2 * NumDims);
-  }
-  T operator()(const Eigen::array<Eigen::Index, NumDims>& coordinates) const {
-    Eigen::array<Eigen::Index, 2 * NumDims> index;
-    for (size_t j = 0; j < NumDims; ++j){
-      index[j] = coordinates[j];
-    }
-    for (size_t j = NumDims; j < 2 * NumDims; ++j){
-      index[j] = index[j - NumDims];
-    }
-    return tensor_.tensor<T, 2 * NumDims>()(index);
-  }
-
- private:
-  Tensor tensor_;
-};
-  
-}  // namespace
+typedef Eigen::ThreadPoolDevice CPUDevice;
+typedef Eigen::GpuDevice GPUDevice;
 
 // Generate the diagonal tensor with the diagonal set to the input tensor.
-// It only allows up to rank 3 input tensor, so the output tensor is up to
-// rank 6.
-template <typename T>
+template <typename Device, typename T>
 class DiagOp : public OpKernel {
  public:
   explicit DiagOp(OpKernelConstruction* context) : OpKernel(context) {}
@@ -80,9 +47,8 @@ class DiagOp : public OpKernel {
   void Compute(OpKernelContext* context) override {
     const Tensor& diagonal = context->input(0);
     const int num_dims = diagonal.dims();
-    OP_REQUIRES(context, 1 <= num_dims && num_dims <= 3,
-                errors::InvalidArgument("Expected 1 <= dims <= 3, got shape ",
-                                        diagonal.shape().DebugString()));
+    OP_REQUIRES(context, 0 != num_dims, errors::InvalidArgument(
+        "Input must be at least rank 1, got 0"));
     TensorShape out_shape;
     for (int i = 0; i < num_dims; ++i) {
       out_shape.AddDim(diagonal.dim_size(i));
@@ -93,45 +59,17 @@ class DiagOp : public OpKernel {
     Tensor* output_tensor = nullptr;
     OP_REQUIRES_OK(context,
                    context->allocate_output(0, out_shape, &output_tensor));
-    switch (num_dims) {
-      case 1:
-        output_tensor->tensor<T, 2>() = output_tensor->tensor<T, 2>().generate(
-            DiagonalGenerator<T, 1, 2>(diagonal));
-        break;
-      case 2:
-        output_tensor->tensor<T, 4>() = output_tensor->tensor<T, 4>().generate(
-            DiagonalGenerator<T, 2, 4>(diagonal));
-        break;
-      case 3:
-        output_tensor->tensor<T, 6>() = output_tensor->tensor<T, 6>().generate(
-            DiagonalGenerator<T, 3, 6>(diagonal));
-        break;
-      default:
-        context->SetStatus(errors::Unimplemented(
-            "Diagonal of rank ", num_dims, " tensor is not supported yet."));
-        return;
-    }
+    functor::DiagFunctor<Device, T> diagFunc;
+    Status s = diagFunc(context,
+                        diagonal.NumElements(),
+                        diagonal.flat<T>().data(),
+                        output_tensor->flat<T>().data());
+    OP_REQUIRES_OK(context, s);
   }
 };
 
-#define REGISTER_DIAGOP(T) \
-  REGISTER_KERNEL_BUILDER( \
-      Name("Diag").Device(DEVICE_CPU).TypeConstraint<T>("T"), DiagOp<T>)
-
-REGISTER_DIAGOP(double);
-REGISTER_DIAGOP(float);
-REGISTER_DIAGOP(int32);
-REGISTER_DIAGOP(int64);
-REGISTER_DIAGOP(complex64);
-REGISTER_DIAGOP(complex128);
-
-#undef REGISTER_DIAGOP
-
-
-// Generate the diagonal tensor with the diagonal set to the input tensor.
-// It only allows rank 2, 4, or 6 input tensor, so the output tensor is 
-// rank 1, 2, or 3.
-template <typename T>
+// Extract the diagonal tensor with the diagonal set to the input tensor.
+template <typename Device, typename T>
 class DiagPartOp : public OpKernel {
  public:
   explicit DiagPartOp(OpKernelConstruction* context) : OpKernel(context) {}
@@ -140,9 +78,9 @@ class DiagPartOp : public OpKernel {
     const Tensor& tensor = context->input(0);
     const int num_dims = tensor.dims();
     const int out_dims = num_dims / 2;
-    OP_REQUIRES(context, 2 == num_dims || 4 == num_dims || 6 == num_dims, 
-                errors::InvalidArgument("The rank of the tensor should be 2, \
-                                         4, or 6, got shape ",
+    OP_REQUIRES(context, 0 == num_dims % 2,
+                errors::InvalidArgument("The rank of the tensor should be \
+                                         even and positive, got shape ",
                                         tensor.shape().DebugString()));
     for (int i = 0; i < out_dims; i++){
       OP_REQUIRES(context, tensor.dim_size(i) == tensor.dim_size(i + out_dims),
@@ -160,39 +98,158 @@ class DiagPartOp : public OpKernel {
     Tensor* output = nullptr;
     OP_REQUIRES_OK(context,
                    context->allocate_output(0, out_shape, &output));
+    functor::DiagPartFunctor<Device, T> diagPartFunc;
+    Status s = diagPartFunc(context,
+                            out_shape.num_elements(),
+                            tensor.flat<T>().data(),
+                            output->flat<T>().data());
+    OP_REQUIRES_OK(context, s);
+  }
+};
 
-    switch (num_dims) {
-      case 2:
-        output->tensor<T, 1>() = output->tensor<T, 1>().generate(
-          DiagonalExtractor<T, 1>(tensor));
-        break; 
-      case 4:
-        output->tensor<T, 2>() = output->tensor<T, 2>().generate(
-          DiagonalExtractor<T, 2>(tensor));
-        break;
-      case 6:
-        output->tensor<T, 3>() = output->tensor<T, 3>().generate(
-          DiagonalExtractor<T, 3>(tensor));
-        break;      
-      default:
-        context->SetStatus(errors::Unimplemented(
-          "Diagonal of rank ", num_dims, " tensor is not supported yet."));
-        return;
-    }
+// Implementation of the functor specialization for CPU.
+// 
+// According to the diagonal definition,
+// `output[i1,..., ik, i1,..., ik] = input[i1,..., ik]`,
+//
+// Let the rank of input is [s1,..., sk], then any offset of input's
+// pointer can be represent by coordinate [i1,..., ik],
+// where `index = i1*(s2*...*sk) + i2*(s3*...*sk) +... + ik`
+//
+// Let new_index is the offset of output's pointer with coordinate 
+// [i1,..., ik, i1,..., ik], then we have
+// `new_index = i1*(s2*...sk*s1*...*sk) + i2*(s3*...*sk*s1*...*sk) +... + \
+//              ik*(s1*...*sk) + i1*(s2*...*sk) + i2*(s3*...*sk) +... + ik
+//            = (i1*(s2*...*sk) + i2*(s3*...*sk) +... + ik) * (1 + s1*...*sk)
+//            = index * (1 + s1*...*sk)
+//
+// Let `size = s1*...*sk`, we finally have `new_index = index * (1 + size)`,
+// which is the transfer function we use below.
+// This trick make our implementations clear and easy to be parallel.
+namespace functor {
+template <typename T>
+struct DiagFunctor<CPUDevice, T> {
+  EIGEN_ALWAYS_INLINE Status
+  operator() (OpKernelContext* context, const int64 size,
+              const T* in, T* out) {
+    // This subprocess is responsible for writing values in index range
+    // [start*size, limit*size)
+    auto subDiag = [in, out, size](int64 start, int64 limit) {
+      std::fill(out + size * start, out + size * limit, T());
+      for (int64 index = start; index < limit; ++index) {
+        out[(1 + size) * index] = in[index];
+      }
+    };
+
+    // Here, 5 is a empirical factor of cost_per_unit.
+    auto worker_threads = *(context->device()->tensorflow_cpu_worker_threads());
+    Shard(worker_threads.num_threads, worker_threads.workers, size,
+        5 * size, subDiag);
+    return Status::OK();
+  }
+};
+
+template <typename T>
+struct DiagPartFunctor<CPUDevice, T> {
+  EIGEN_ALWAYS_INLINE Status
+  operator() (OpKernelContext* context, const int64 size,
+              const T* in, T* out) {
+    // This subprocess is responsible for extracting values in index range
+    // [start, limit)
+    auto subDiagPart = [in, out, size](int64 start, int64 limit) {
+      for (int64 index = start; index < limit; ++index) {
+        out[index] = in[(1 + size) * index];
+      }
+    };
+
+    // Here, 5 is a empirical factor of cost_per_unit.
+    auto worker_threads = *(context->device()->tensorflow_cpu_worker_threads());
+    Shard(worker_threads.num_threads, worker_threads.workers, size,
+        5, subDiagPart);
+    return Status::OK();
   }
 };
+}  // namespace functor
 
-#define REGISTER_DIAGPARTOP(T) \
-  REGISTER_KERNEL_BUILDER( \
-      Name("DiagPart").Device(DEVICE_CPU).TypeConstraint<T>("T"), DiagPartOp<T>)
 
-REGISTER_DIAGPARTOP(double);
-REGISTER_DIAGPARTOP(float);
-REGISTER_DIAGPARTOP(int32);
-REGISTER_DIAGPARTOP(int64);
-REGISTER_DIAGPARTOP(complex64);
-REGISTER_DIAGPARTOP(complex128);
+// Register the CPU kernels.
+#define REGISTER_DIAGOP(T)                                    \
+  REGISTER_KERNEL_BUILDER(                                    \
+      Name("Diag").Device(DEVICE_CPU).TypeConstraint<T>("T"), \
+      DiagOp<CPUDevice, T>)
 
+TF_CALL_double(REGISTER_DIAGOP);
+TF_CALL_float(REGISTER_DIAGOP);
+TF_CALL_int32(REGISTER_DIAGOP);
+TF_CALL_int64(REGISTER_DIAGOP);
+TF_CALL_complex64(REGISTER_DIAGOP);
+TF_CALL_complex128(REGISTER_DIAGOP);
+#undef REGISTER_DIAGOP
+
+#define REGISTER_DIAGPARTOP(T)                                    \
+  REGISTER_KERNEL_BUILDER(                                        \
+      Name("DiagPart").Device(DEVICE_CPU).TypeConstraint<T>("T"), \
+      DiagPartOp<CPUDevice, T>)
+
+TF_CALL_double(REGISTER_DIAGPARTOP);
+TF_CALL_float(REGISTER_DIAGPARTOP);
+TF_CALL_int32(REGISTER_DIAGPARTOP);
+TF_CALL_int64(REGISTER_DIAGPARTOP);
+TF_CALL_complex64(REGISTER_DIAGPARTOP);
+TF_CALL_complex128(REGISTER_DIAGPARTOP);
 #undef REGISTER_DIAGPARTOP
-  
+
+// Register the GPU kernels.
+#ifdef GOOGLE_CUDA
+
+// Forward declarations of the functor specializations for GPU.
+namespace functor {
+extern template struct DiagFunctor<GPUDevice, double>;
+extern template struct DiagFunctor<GPUDevice, float>;
+extern template struct DiagFunctor<GPUDevice, int32>;
+extern template struct DiagFunctor<GPUDevice, int64>;
+extern template struct DiagFunctor<GPUDevice, complex64>;
+extern template struct DiagFunctor<GPUDevice, complex128>;
+}  // namespace functor
+
+#define REGISTER_DIAGOP_GPU(T)                                \
+  REGISTER_KERNEL_BUILDER(                                    \
+      Name("Diag").Device(DEVICE_GPU).TypeConstraint<T>("T"), \
+      DiagOp<GPUDevice, T>)
+
+TF_CALL_double(REGISTER_DIAGOP_GPU);
+TF_CALL_float(REGISTER_DIAGOP_GPU);
+TF_CALL_int32(REGISTER_DIAGOP_GPU);
+TF_CALL_int64(REGISTER_DIAGOP_GPU);
+TF_CALL_complex64(REGISTER_DIAGOP_GPU);
+TF_CALL_complex128(REGISTER_DIAGOP_GPU);
+#undef REGISTER_DIAGOP_GPU
+
+// Forward declarations of the functor specializations for GPU.
+namespace functor {
+extern template struct DiagPartFunctor<GPUDevice, double>;
+extern template struct DiagPartFunctor<GPUDevice, float>;
+extern template struct DiagPartFunctor<GPUDevice, int32>;
+extern template struct DiagPartFunctor<GPUDevice, int64>;
+extern template struct DiagPartFunctor<GPUDevice, complex64>;
+extern template struct DiagPartFunctor<GPUDevice, complex128>;
+}  // namespace functor
+
+#define REGISTER_DIAGPARTOP_GPU(T)                                \
+  REGISTER_KERNEL_BUILDER(                                        \
+      Name("DiagPart").Device(DEVICE_GPU).TypeConstraint<T>("T"), \
+      DiagPartOp<GPUDevice, T>)
+
+TF_CALL_double(REGISTER_DIAGPARTOP_GPU);
+TF_CALL_float(REGISTER_DIAGPARTOP_GPU);
+TF_CALL_int32(REGISTER_DIAGPARTOP_GPU);
+TF_CALL_int64(REGISTER_DIAGPARTOP_GPU);
+TF_CALL_complex64(REGISTER_DIAGPARTOP_GPU);
+TF_CALL_complex128(REGISTER_DIAGPARTOP_GPU);
+#undef REGISTER_DIAGPARTOP_GPU
+
+#endif  // GOOGLE_CUDA
+
+
 }  // namespace tensorflow
+
diff --git a/tensorflow/core/kernels/diag_op.h b/tensorflow/core/kernels/diag_op.h
new file mode 100644
index 0000000000..c6ca6a2047
--- /dev/null
+++ b/tensorflow/core/kernels/diag_op.h
@@ -0,0 +1,43 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_CORE_KERNELS_DIAG_OP_H_
+#define TENSORFLOW_CORE_KERNELS_DIAG_OP_H_
+
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/framework/tensor_types.h"
+#include "tensorflow/core/platform/types.h"
+
+namespace tensorflow {
+
+namespace functor {
+
+template <typename Device, typename T>
+struct DiagFunctor {
+  Status operator() (OpKernelContext* context, const int64 size,
+                     const T* in, T* out);
+};
+
+template <typename Device, typename T>
+struct DiagPartFunctor {
+  Status operator() (OpKernelContext* context, const int64 size,
+                     const T* in, T* out);
+};
+
+}  // namespace functor
+
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_CORE_KERNELS_DIAG_OP_H_
diff --git a/tensorflow/core/kernels/diag_op_gpu.cu.cc b/tensorflow/core/kernels/diag_op_gpu.cu.cc
new file mode 100644
index 0000000000..9878f347d2
--- /dev/null
+++ b/tensorflow/core/kernels/diag_op_gpu.cu.cc
@@ -0,0 +1,150 @@
+/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+
+#define EIGEN_USE_GPU
+
+#include <complex>
+#include "tensorflow/core/framework/register_types.h"
+#include "tensorflow/core/util/cuda_kernel_helper.h"
+#include "tensorflow/core/kernels/diag_op.h"
+
+namespace tensorflow {
+namespace functor {
+
+typedef Eigen::GpuDevice GPUDevice;
+
+template <typename T>
+__global__ void DiagCudaKernel(const int num_threads,
+                               const int64 size,
+                               const T* in,
+                               T* out) {
+  CUDA_1D_KERNEL_LOOP(index, num_threads) {
+    out[(1 + size) * index] = in[index];
+  }
+}
+
+template <typename T>
+__global__ void ZeroCudaKernel(const int num_threads,
+                               T* out) {
+  CUDA_1D_KERNEL_LOOP(index, num_threads) {
+    out[index] = T(0);
+  }
+}
+
+template <typename T>
+struct DiagFunctor<GPUDevice, T> {
+  EIGEN_ALWAYS_INLINE Status
+  operator() (OpKernelContext* context, const int64 size,
+              const T* in, T* out) {
+    // CudaLaunchConfig uses an int for virtual_thread_count,
+    // so this may overflow in extreme cases.
+    if (size && (size * size / size) != size) {
+      return errors::Internal(
+          "DiagOp got input size too large.");
+    }
+
+    // Empty tensor couldn't launch the kernel.
+    if (size == 0) {
+      return Status::OK();
+    }
+    const GPUDevice& device = context->eigen_device<GPUDevice>();
+
+    // Set output memory with zero elements.
+    CudaLaunchConfig zero_config = GetCudaLaunchConfig(size*size, device);
+    ZeroCudaKernel<<<zero_config.block_count,
+                     zero_config.thread_per_block,
+                     0, device.stream()>>>(
+        zero_config.virtual_thread_count, out);
+    auto err = cudaGetLastError();
+    if (err != cudaSuccess) {
+      return errors::Internal(
+          "Could not launch DiagOp kernel: ",
+          cudaGetErrorString(err), ".");
+    }
+
+    // Fill the diagonal elements
+    CudaLaunchConfig diag_config = GetCudaLaunchConfig(size, device);
+    DiagCudaKernel<<<diag_config.block_count,
+                     diag_config.thread_per_block,
+                     0, device.stream()>>>(
+        diag_config.virtual_thread_count, size, in, out);
+    err = cudaGetLastError();
+    if (err != cudaSuccess) {
+      return errors::Internal(
+          "Could not launch DiagOp kernel: ",
+          cudaGetErrorString(err), ".");
+    }
+    return Status::OK();
+  }
+};
+
+template struct DiagFunctor<GPUDevice, double>;
+template struct DiagFunctor<GPUDevice, float>;
+template struct DiagFunctor<GPUDevice, int32>;
+template struct DiagFunctor<GPUDevice, int64>;
+template struct DiagFunctor<GPUDevice, complex64>;
+template struct DiagFunctor<GPUDevice, complex128>;
+
+
+template <typename T>
+__global__ void DiagPartCudaKernel(const int num_threads,
+                                   const int64 size,
+                                   const T* in,
+                                   T* out) {
+  CUDA_1D_KERNEL_LOOP(index, num_threads) {
+    out[index] = in[(1 + size) * index];
+  }
+}
+
+template <typename T>
+struct DiagPartFunctor<GPUDevice, T> {
+  EIGEN_ALWAYS_INLINE Status
+  operator() (OpKernelContext* context, const int64 size,
+              const T* in, T* out) {
+    // Empty tensor couldn't launch the kernel.
+    if (size == 0) {
+      return Status::OK();
+    }
+    const GPUDevice& device = context->eigen_device<GPUDevice>();
+
+    // Extract the diagonal elements.
+    CudaLaunchConfig diag_config = GetCudaLaunchConfig(size, device);
+    DiagPartCudaKernel<<<diag_config.block_count,
+                     diag_config.thread_per_block,
+                     0, device.stream()>>>(
+        diag_config.virtual_thread_count, size, in, out);
+    auto err = cudaGetLastError();
+    if (err != cudaSuccess) {
+      return errors::Internal(
+          "Could not launch DiagPartOp kernel: ",
+          cudaGetErrorString(err), ".");
+    }
+    return Status::OK();
+  }
+};
+
+template struct DiagPartFunctor<GPUDevice, double>;
+template struct DiagPartFunctor<GPUDevice, float>;
+template struct DiagPartFunctor<GPUDevice, int32>;
+template struct DiagPartFunctor<GPUDevice, int64>;
+template struct DiagPartFunctor<GPUDevice, complex64>;
+template struct DiagPartFunctor<GPUDevice, complex128>;
+
+}  // end namespace functor
+}  // end namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/diag_op_test.cc b/tensorflow/core/kernels/diag_op_test.cc
new file mode 100644
index 0000000000..2d1417854c
--- /dev/null
+++ b/tensorflow/core/kernels/diag_op_test.cc
@@ -0,0 +1,54 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/common_runtime/kernel_benchmark_testlib.h"
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/platform/test.h"
+#include "tensorflow/core/platform/test_benchmark.h"
+
+namespace tensorflow {
+
+template <typename T>
+static Graph* Diag(int n, DataType type) {
+  Graph* g = new Graph(OpRegistry::Global());
+  Tensor in(type, TensorShape({n}));
+  in.flat<T>().setRandom();
+  Node* out = test::graph::Diag(g, test::graph::Constant(g, in), type);
+  test::graph::DiagPart(g, out, type);
+  return g;
+}
+
+#define BM_DiagDev(N, T, TFTYPE, DEVICE)                           \
+  static void BM_Diag##_##N##_##TFTYPE##_##DEVICE(int iters) {   \
+    testing::UseRealTime();                                     \
+    testing::ItemsProcessed(static_cast<int64>(iters) * N * N); \
+    test::Benchmark(#DEVICE, Diag<T>(N, TFTYPE)).Run(iters);    \
+  }                                                             \
+  BENCHMARK(BM_Diag##_##N##_##TFTYPE##_##DEVICE);
+
+#define BM_Diag(N)                                       \
+  BM_DiagDev(N, int, DT_INT32, cpu);                     \
+  BM_DiagDev(N, float, DT_FLOAT, cpu);                   \
+  BM_DiagDev(N, std::complex<float>, DT_COMPLEX64, cpu); \
+  BM_DiagDev(N, int, DT_INT32, gpu);                     \
+  BM_DiagDev(N, float, DT_FLOAT, gpu);                   \
+  BM_DiagDev(N, std::complex<float>, DT_COMPLEX64, gpu);
+
+BM_Diag(16);
+BM_Diag(128);
+BM_Diag(512);
+
+}  // end namespace tensorflow
+
diff --git a/tensorflow/core/ops/array_ops.cc b/tensorflow/core/ops/array_ops.cc
index 15b09c2c16..c5935141f8 100644
--- a/tensorflow/core/ops/array_ops.cc
+++ b/tensorflow/core/ops/array_ops.cc
@@ -739,7 +739,7 @@ REGISTER_OP("Diag")
     .Attr("T: {float, double, int32, int64, complex64, complex128}")
     .SetShapeFn([](InferenceContext* c) {
       ShapeHandle in = c->input(0);
-      TF_RETURN_IF_ERROR(c->WithRankAtMost(in, 3, &in));
+      TF_RETURN_IF_ERROR(c->WithRankAtLeast(in, 1, &in));
       // Output shape is original concatenated with itself.
       ShapeHandle out;
       TF_RETURN_IF_ERROR(c->Concatenate(in, in, &out));
@@ -767,7 +767,7 @@ tf.diag(diagonal) ==> [[1, 0, 0, 0]
                        [0, 0, 0, 4]]
 ```
 
-diagonal: Rank k tensor where k is at most 3.
+diagonal: Rank k tensor where k is at most 1.
 )doc");
 
 // --------------------------------------------------------------------------
@@ -783,9 +783,9 @@ REGISTER_OP("DiagPart")
       }
       // Rank must be even, and result will have rank <rank/2>.
       const int32 rank = c->Rank(in);
-      if ((rank % 2) != 0 || rank > 6) {
+      if ((rank % 2) != 0 || rank <= 0) {
         return errors::InvalidArgument(
-            "Input must have even rank <= 6, input rank is ", rank);
+            "Input must have even and non-zero rank, input rank is ", rank);
       }
       const int32 mid = rank / 2;
 
@@ -820,7 +820,7 @@ For example:
 tf.diag_part(input) ==> [1, 2, 3, 4]
 ```
 
-input: Rank k tensor where k is 2, 4, or 6.
+input: Rank k tensor where k is even and not zero.
 diagonal: The extracted diagonal.
 
 )doc");
diff --git a/tensorflow/core/ops/array_ops_test.cc b/tensorflow/core/ops/array_ops_test.cc
index a5d7a32e05..94eb120175 100644
--- a/tensorflow/core/ops/array_ops_test.cc
+++ b/tensorflow/core/ops/array_ops_test.cc
@@ -186,21 +186,20 @@ TEST(ArrayOpsTest, Identity_ShapeFnHandles) {
 TEST(ArrayOpsTest, Diag_ShapeFn) {
   ShapeInferenceTestOp op("Diag");
   INFER_OK(op, "?", "?");
-  INFER_OK(op, "[]", "[]");
   INFER_OK(op, "[1,?,3]", "[d0_0,d0_1,d0_2,d0_0,d0_1,d0_2]");
-  INFER_ERROR("Shape must be at most rank 3 but is rank 4", op, "[?,1,2,3]");
+  INFER_OK(op, "[?,1,2,3]", "[d0_0,d0_1,d0_2,d0_3,d0_0,d0_1,d0_2,d0_3]");
+  INFER_ERROR("Shape must be at least rank 1 but is rank 0", op, "[]");
 }
 
 TEST(ArrayOpsTest, DiagPart_ShapeFn) {
   ShapeInferenceTestOp op("DiagPart");
   INFER_OK(op, "?", "?");
-  INFER_OK(op, "[]", "[]");
   INFER_OK(op, "[1,?,?,4]", "[d0_0,d0_3]");
   INFER_OK(op, "[1,?,3,?,4,3]", "[d0_0,d0_4,d0_2|d0_5]");
-  INFER_ERROR("Input must have even rank <= 6, input rank is 1", op, "[?]");
-  INFER_ERROR("Input must have even rank <= 6, input rank is 3", op, "[1,2,3]");
-  INFER_ERROR("Input must have even rank <= 6, input rank is 8", op,
-              "[1,2,3,?,?,?,?,?]");
+  INFER_OK(op, "[1,2,3,?,?,?,?,4]", "[d0_0,d0_1,d0_2,d0_7]");
+  INFER_ERROR("Input must have even and non-zero rank", op, "[]");
+  INFER_ERROR("Input must have even and non-zero rank", op, "[?]");
+  INFER_ERROR("Input must have even and non-zero rank", op, "[1,2,3]");
   INFER_ERROR("Dimensions must be equal, but are 2 and 10", op, "[1,2,?,10]");
 }
 
diff --git a/tensorflow/python/kernel_tests/diag_op_test.py b/tensorflow/python/kernel_tests/diag_op_test.py
index f0b7885732..6cfa9b37fe 100644
--- a/tensorflow/python/kernel_tests/diag_op_test.py
+++ b/tensorflow/python/kernel_tests/diag_op_test.py
@@ -279,7 +279,7 @@ class MatrixDiagPartTest(test.TestCase):
 
 class DiagTest(test.TestCase):
 
-  def diagOp(self, diag, dtype, expected_ans, use_gpu=False):
+  def _diagOp(self, diag, dtype, expected_ans, use_gpu):
     with self.test_session(use_gpu=use_gpu):
       tf_ans = array_ops.diag(ops.convert_to_tensor(diag.astype(dtype)))
       out = tf_ans.eval()
@@ -290,6 +290,10 @@ class DiagTest(test.TestCase):
     self.assertShapeEqual(expected_ans, tf_ans)
     self.assertShapeEqual(diag, tf_ans_inv)
 
+  def diagOp(self, diag, dtype, expected_ans):
+    self._diagOp(diag, dtype, expected_ans, False)
+    self._diagOp(diag, dtype, expected_ans, True)
+
   def testEmptyTensor(self):
     x = np.array([])
     expected_ans = np.empty([0, 0])
@@ -400,13 +404,53 @@ class DiagTest(test.TestCase):
           dtype=dtype)
       self.diagOp(x, dtype, expected_ans)
 
+  def testRankFourNumberTensor(self):
+    for dtype in [np.float32, np.float64, np.int64, np.int32]:
+      # Input with shape [2, 1, 2, 3]
+      x = np.array([[[[ 1,  2,  3],
+                      [ 4,  5,  6]]],
+                    [[[ 7,  8,  9],
+                      [10, 11, 12]]]], dtype=dtype)
+      # Output with shape [2, 1, 2, 3, 2, 1, 2, 3]
+      expected_ans = np.array(
+          [[[[[[[[1, 0, 0], [0, 0, 0]]],
+               [[[0, 0, 0], [0, 0, 0]]]],
+              [[[[0, 2, 0], [0, 0, 0]]],
+               [[[0, 0, 0], [0, 0, 0]]]],
+              [[[[0, 0, 3], [0, 0, 0]]],
+               [[[0, 0, 0], [0, 0, 0]]]]],
+             [[[[[0, 0, 0], [4, 0, 0]]],
+               [[[0, 0, 0], [0, 0, 0]]]],
+              [[[[0, 0, 0], [0, 5, 0]]],
+               [[[0, 0, 0], [0, 0, 0]]]],
+              [[[[0, 0, 0], [0, 0, 6]]],
+               [[[0, 0, 0], [0, 0, 0]]]]]]],
+
+           [[[[[[[0, 0, 0], [0, 0, 0]]],
+               [[[7, 0, 0], [0, 0, 0]]]],
+              [[[[0, 0, 0], [0, 0, 0]]],
+               [[[0, 8, 0], [0, 0, 0]]]],
+              [[[[0, 0, 0], [0, 0, 0]]],
+               [[[0, 0, 9], [0, 0, 0]]]]],
+             [[[[[0, 0, 0], [0, 0, 0]]],
+               [[[0, 0, 0], [10, 0, 0]]]],
+              [[[[0, 0, 0], [0, 0, 0]]],
+               [[[0, 0, 0], [0, 11, 0]]]],
+              [[[[0, 0, 0], [0, 0, 0]]],
+               [[[0, 0, 0], [0, 0, 12]]]]]]]], dtype=dtype)
+      self.diagOp(x, dtype, expected_ans)
+
+  def testInvalidRank(self):
+    with self.assertRaisesRegexp(ValueError, "must be at least rank 1"):
+      array_ops.diag(0.0)
+
 
 class DiagPartOpTest(test.TestCase):
 
   def setUp(self):
     np.random.seed(0)
 
-  def diagPartOp(self, tensor, dtype, expected_ans, use_gpu=False):
+  def _diagPartOp(self, tensor, dtype, expected_ans, use_gpu):
     with self.test_session(use_gpu=use_gpu):
       tensor = ops.convert_to_tensor(tensor.astype(dtype))
       tf_ans_inv = array_ops.diag_part(tensor)
@@ -414,6 +458,10 @@ class DiagPartOpTest(test.TestCase):
     self.assertAllClose(inv_out, expected_ans)
     self.assertShapeEqual(expected_ans, tf_ans_inv)
 
+  def diagPartOp(self, tensor, dtype, expected_ans):
+    self._diagPartOp(tensor, dtype, expected_ans, False)
+    self._diagPartOp(tensor, dtype, expected_ans, True)
+
   def testRankTwoFloatTensor(self):
     x = np.random.rand(3, 3)
     i = np.arange(3)
@@ -451,11 +499,23 @@ class DiagPartOpTest(test.TestCase):
     self.diagPartOp(x, np.float32, expected_ans)
     self.diagPartOp(x, np.float64, expected_ans)
 
+  def testRankEightComplexTensor(self):
+    x = np.random.rand(2, 2, 2, 3, 2, 2, 2, 3)
+    i = np.arange(2)[:, None, None, None]
+    j = np.arange(2)[:, None, None]
+    k = np.arange(2)[:, None]
+    l = np.arange(3)
+    expected_ans = x[i, j, k, l, i, j, k, l]
+    self.diagPartOp(x, np.complex64, expected_ans)
+    self.diagPartOp(x, np.complex128, expected_ans)
+
   def testOddRank(self):
     w = np.random.rand(2)
     x = np.random.rand(2, 2, 2)
     self.assertRaises(ValueError, self.diagPartOp, w, np.float32, 0)
     self.assertRaises(ValueError, self.diagPartOp, x, np.float32, 0)
+    with self.assertRaises(ValueError):
+      array_ops.diag_part(0.0)
 
   def testUnevenDimensions(self):
     w = np.random.rand(2, 5)
-- 
GitLab


From 93871a811eab7457f8e36ee4905234aa1a9ea8c8 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 20 Oct 2017 09:13:44 -0700
Subject: [PATCH 607/909] Remove duplicated `smart_cond()` code.

PiperOrigin-RevId: 172891249
---
 .../training/python/training/bucket_ops.py    |  4 +--
 tensorflow/python/BUILD                       |  1 +
 tensorflow/python/training/input.py           | 26 ++++---------------
 3 files changed, 8 insertions(+), 23 deletions(-)

diff --git a/tensorflow/contrib/training/python/training/bucket_ops.py b/tensorflow/contrib/training/python/training/bucket_ops.py
index 5523cc375f..95fbc50cba 100644
--- a/tensorflow/contrib/training/python/training/bucket_ops.py
+++ b/tensorflow/contrib/training/python/training/bucket_ops.py
@@ -31,6 +31,7 @@ from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
 from tensorflow.python.framework import tensor_util
+from tensorflow.python.layers import utils
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import data_flow_ops
@@ -47,7 +48,6 @@ _dtypes = input_py._dtypes
 _store_sparse_tensors = input_py._store_sparse_tensors
 _validate_keep_input = input_py._validate_keep_input
 _shapes = input_py._shapes
-_smart_cond = input_py._smart_cond
 _which_queue = input_py._which_queue
 
 # pylint: enable=protected-access
@@ -239,7 +239,7 @@ def bucket(tensors,
       ]
       return control_flow_ops.group(*enqueues, name="group_enqueues")
 
-    maybe_enqueue = _smart_cond(
+    maybe_enqueue = utils.smart_cond(
         keep_input,
         enqueue_which,
         control_flow_ops.no_op)
diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 21cdaec477..e63c554e47 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -2638,6 +2638,7 @@ py_library(
         ":init_ops",
         ":io_ops",
         ":io_ops_gen",
+        ":layers_base",
         ":lib",
         ":lookup_ops",
         ":math_ops",
diff --git a/tensorflow/python/training/input.py b/tensorflow/python/training/input.py
index 704017c244..36f97960dd 100644
--- a/tensorflow/python/training/input.py
+++ b/tensorflow/python/training/input.py
@@ -32,7 +32,7 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import sparse_tensor
 from tensorflow.python.framework import tensor_shape
-from tensorflow.python.framework import tensor_util
+from tensorflow.python.layers import utils
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import data_flow_ops
@@ -413,22 +413,6 @@ def _as_original_type(original_tensors, tensor_list):
     return tensor_list
 
 
-def _smart_cond(pred, if_true, if_false):
-  """A `tf.cond` that does nothing when the condition is static."""
-  pred = ops.convert_to_tensor(pred)
-  static_pred = tensor_util.constant_value(pred)
-  if static_pred is not None:
-    if static_pred:
-      return if_true()
-    else:
-      return if_false()
-  else:
-    return control_flow_ops.cond(
-        pred,
-        if_true,
-        if_false)
-
-
 def _store_sparse_tensors(tensor_list, enqueue_many, keep_input,
                           shared_map_ops=None):
   """Store SparseTensors for feeding into batch, etc.
@@ -480,13 +464,13 @@ def _store_sparse_tensors(tensor_list, enqueue_many, keep_input,
     map_op_name = shared_map_op.name if shared_map_op else None
     def _maybe_store_sparse(t, map_op_name, keep_input):
       """Conditionally store a single sparse Tensor."""
-      return _smart_cond(
+      return utils.smart_cond(
           keep_input,
           lambda: _store_sparse(t, shared_name=map_op_name),
           lambda: constant_op.constant(-1, dtypes.int64))
     def _maybe_store_many_sparse(t, map_op_name, keep_input):
       """Conditionally store multiple sparse Tensors."""
-      out_tensor = _smart_cond(
+      out_tensor = utils.smart_cond(
           keep_input,
           lambda: _store_many_sparse(t, shared_name=map_op_name),
           lambda: -1 * array_ops.ones(array_ops.shape(t)[0:1], dtypes.int64))
@@ -667,7 +651,7 @@ def _enqueue_join(queue, tensor_list_list, enqueue_many, keep_input):
     enqueue_ops = [enqueue_fn(_select_which_to_enqueue(x, keep_input))
                    for x in tensor_list_list]
   else:
-    enqueue_ops = [_smart_cond(
+    enqueue_ops = [utils.smart_cond(
         keep_input,
         lambda: enqueue_fn(tl),  # pylint:disable=cell-var-from-loop
         control_flow_ops.no_op) for tl in tensor_list_list]
@@ -684,7 +668,7 @@ def _enqueue(queue, tensor_list, threads, enqueue_many, keep_input):
     enqueue_ops = [
         enqueue_fn(_select_which_to_enqueue(tensor_list, keep_input))] * threads
   else:
-    enqueue_ops = [_smart_cond(
+    enqueue_ops = [utils.smart_cond(
         keep_input,
         lambda: enqueue_fn(tensor_list),
         control_flow_ops.no_op)] * threads
-- 
GitLab


From 5c24b8b1e5f3f1145e123a5a159b958ea9fc8c3d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 20 Oct 2017 09:17:03 -0700
Subject: [PATCH 608/909] XLA refactoring

PiperOrigin-RevId: 172891551
---
 .../xla/legacy_flags/debug_options_flags.cc   |  6 ++---
 tensorflow/compiler/xla/protobuf_util.cc      | 25 -----------------
 tensorflow/compiler/xla/protobuf_util.h       | 13 +++------
 .../compiler/xla/service/cpu/cpu_compiler.cc  | 27 +++++++++----------
 .../compiler/xla/service/gpu/gpu_compiler.cc  | 10 +++----
 tensorflow/compiler/xla/xla.proto             |  4 +--
 6 files changed, 27 insertions(+), 58 deletions(-)

diff --git a/tensorflow/compiler/xla/legacy_flags/debug_options_flags.cc b/tensorflow/compiler/xla/legacy_flags/debug_options_flags.cc
index 8892bfbe92..f2cdd9669c 100644
--- a/tensorflow/compiler/xla/legacy_flags/debug_options_flags.cc
+++ b/tensorflow/compiler/xla/legacy_flags/debug_options_flags.cc
@@ -206,9 +206,9 @@ void AllocateFlags() {
            flag_values->xla_gpu_disable_multi_streaming(),
            "If true, multi-streaming in the GPU backend is disabled."),
        tensorflow::Flag(
-           "xla_dump_debug_json_to",
-           flag_values->mutable_xla_dump_debug_json_to(),
-           "Dump compilation artifacts as JSON into this directory."),
+           "xla_dump_hlo_proto_to",
+           flag_values->mutable_xla_dump_hlo_proto_to(),
+           "Dump compilation artifacts as proto binary into this directory."),
        tensorflow::Flag(
            "xla_test_all_output_layouts",
            bool_setter_for(&DebugOptions::set_xla_test_all_output_layouts),
diff --git a/tensorflow/compiler/xla/protobuf_util.cc b/tensorflow/compiler/xla/protobuf_util.cc
index c032cb8dc5..787725e884 100644
--- a/tensorflow/compiler/xla/protobuf_util.cc
+++ b/tensorflow/compiler/xla/protobuf_util.cc
@@ -37,20 +37,6 @@ bool ProtobufEquals(const tensorflow::protobuf::Message& m1,
   return (serialized1 == serialized2);
 }
 
-StatusOr<string> ToJson(const tensorflow::protobuf::Message& message) {
-  string json_output;
-  tensorflow::protobuf::util::JsonPrintOptions json_options;
-  json_options.add_whitespace = true;
-  json_options.always_print_primitive_fields = true;
-  auto status = tensorflow::protobuf::util::MessageToJsonString(
-      message, &json_output, json_options);
-  if (!status.ok()) {
-    return InternalError("MessageToJsonString failed: %s",
-                         status.error_message().data());
-  }
-  return json_output;
-}
-
 namespace {
 
 string SanitizeFilename(const string& file_name) {
@@ -65,17 +51,6 @@ string SanitizeFilename(const string& file_name) {
 
 }  // namespace
 
-Status DumpJsonToDirectory(const tensorflow::protobuf::Message& message,
-                           const string& directory, const string& file_name) {
-  TF_ASSIGN_OR_RETURN(const string json_output, ToJson(message));
-
-  tensorflow::Env* env = tensorflow::Env::Default();
-  TF_RETURN_IF_ERROR(env->RecursivelyCreateDir(directory));
-  string safe_file_name = SanitizeFileName(file_name) + ".json";
-  const string path = tensorflow::io::JoinPath(directory, safe_file_name);
-  return tensorflow::WriteStringToFile(env, path, json_output);
-}
-
 Status DumpProtoToDirectory(const tensorflow::protobuf::Message& message,
                             const string& directory, const string& file_name) {
   tensorflow::Env* env = tensorflow::Env::Default();
diff --git a/tensorflow/compiler/xla/protobuf_util.h b/tensorflow/compiler/xla/protobuf_util.h
index 7accb22e0c..3667621367 100644
--- a/tensorflow/compiler/xla/protobuf_util.h
+++ b/tensorflow/compiler/xla/protobuf_util.h
@@ -32,17 +32,12 @@ namespace protobuf_util {
 extern bool ProtobufEquals(const tensorflow::protobuf::Message& m1,
                            const tensorflow::protobuf::Message& m2);
 
-// Returns 'message' as a JSON string.
-StatusOr<string> ToJson(const tensorflow::protobuf::Message& message);
-
-// Writes the given message in binary proto or JSON format to the path formed by
-// joining 'directory/file_name.pb' (or file_name.json). The 'directory' is
-// recursively created if it doesn't already exist, and the 'file_name' is
-// sanitized by replacing illegal characters with underscore '_'.
+// Writes the given message in binary proto to the path formed by joining
+// 'directory/file_name.pb'. The 'directory' is recursively created if it
+// doesn't already exist, and the 'file_name' is sanitized by replacing
+// illegal characters with underscore '_'.
 Status DumpProtoToDirectory(const tensorflow::protobuf::Message& message,
                             const string& directory, const string& file_name);
-Status DumpJsonToDirectory(const tensorflow::protobuf::Message& message,
-                           const string& directory, const string& file_name);
 
 }  // namespace protobuf_util
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
index ce4d109214..06e7ec0c7c 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
@@ -475,8 +475,8 @@ StatusOr<std::unique_ptr<Executable>> CpuCompiler::Compile(
   // ownership is std::moved.
   const bool embed_ir_in_executable =
       module->config().debug_options().xla_embed_ir_in_executable();
-  const string dump_debug_json_to =
-      module->config().debug_options().xla_dump_debug_json_to();
+  const string xla_dump_hlo_proto_to =
+      module->config().debug_options().xla_dump_hlo_proto_to();
 
   if (options::CpuParallelBackendRequested(module->config())) {
     VLOG(1) << "Using parallel cpu backend";
@@ -496,10 +496,10 @@ StatusOr<std::unique_ptr<Executable>> CpuCompiler::Compile(
     // print one ourselves.
     XLA_VLOG_LINES(2, assignment->ToString());
 
-    if (!dump_debug_json_to.empty()) {
+    if (!xla_dump_hlo_proto_to.empty()) {
       HloProto proto = MakeHloProto(*module, *assignment);
-      TF_RETURN_IF_ERROR(protobuf_util::DumpJsonToDirectory(
-          proto, dump_debug_json_to, module->name()));
+      TF_RETURN_IF_ERROR(protobuf_util::DumpProtoToDirectory(
+          proto, xla_dump_hlo_proto_to, module->name()));
     }
 
     // If we are using the parallel CPU backend, we need to create map from
@@ -603,12 +603,11 @@ StatusOr<std::unique_ptr<Executable>> CpuCompiler::Compile(
     // print one ourselves.
     XLA_VLOG_LINES(2, assignment->ToString());
 
-    if (!dump_debug_json_to.empty()) {
+    if (!xla_dump_hlo_proto_to.empty()) {
       HloProto proto = MakeHloProto(*module, *assignment);
-      TF_RETURN_IF_ERROR(protobuf_util::DumpJsonToDirectory(
-          proto, dump_debug_json_to, module->name()));
+      TF_RETURN_IF_ERROR(protobuf_util::DumpProtoToDirectory(
+          proto, xla_dump_hlo_proto_to, module->name()));
     }
-
     // Each computation is a single function.  Emit all embedded computations
     // before the entry computation. The order of computations returned from
     // GetEmbeddedComputations guarantees that a called computation occurs
@@ -775,12 +774,12 @@ CpuCompiler::CompileAheadOfTime(std::vector<std::unique_ptr<HloModule>> modules,
     // print one ourselves.
     XLA_VLOG_LINES(2, assignment->ToString());
 
-    const string dump_debug_json_to =
-        module->config().debug_options().xla_dump_debug_json_to();
-    if (!dump_debug_json_to.empty()) {
+    const string xla_dump_hlo_proto_to =
+        module->config().debug_options().xla_dump_hlo_proto_to();
+    if (!xla_dump_hlo_proto_to.empty()) {
       HloProto proto = MakeHloProto(*module, *assignment);
-      TF_RETURN_IF_ERROR(protobuf_util::DumpJsonToDirectory(
-          proto, dump_debug_json_to, module->name()));
+      TF_RETURN_IF_ERROR(protobuf_util::DumpProtoToDirectory(
+          proto, xla_dump_hlo_proto_to, module->name()));
     }
 
     IrEmitter ir_emitter(*module, *assignment, &llvm_module,
diff --git a/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc b/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc
index 3e16e4e3c4..9c7ca9ea38 100644
--- a/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc
+++ b/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc
@@ -318,12 +318,12 @@ StatusOr<std::unique_ptr<Executable>> GpuCompiler::Compile(
   // print one ourselves.
   XLA_VLOG_LINES(2, buffer_assignment->ToString());
 
-  const string dump_debug_json_to =
-      module->config().debug_options().xla_dump_debug_json_to();
-  if (!dump_debug_json_to.empty()) {
+  const string xla_dump_hlo_proto_to =
+      module->config().debug_options().xla_dump_hlo_proto_to();
+  if (!xla_dump_hlo_proto_to.empty()) {
     HloProto proto = MakeHloProto(*module, *buffer_assignment);
-    TF_RETURN_IF_ERROR(protobuf_util::DumpJsonToDirectory(
-        proto, dump_debug_json_to, module->name()));
+    TF_RETURN_IF_ERROR(protobuf_util::DumpProtoToDirectory(
+        proto, xla_dump_hlo_proto_to, module->name()));
   }
 
   IrEmitterContext ir_emitter_context(module.get(), buffer_assignment.get(),
diff --git a/tensorflow/compiler/xla/xla.proto b/tensorflow/compiler/xla/xla.proto
index 7f4bd26d1b..ce3c3eee68 100644
--- a/tensorflow/compiler/xla/xla.proto
+++ b/tensorflow/compiler/xla/xla.proto
@@ -82,8 +82,8 @@ message DebugOptions {
   // Dump all HLO modules as text into the provided directory path.
   string xla_generate_hlo_text_to = 7;
 
-  // Dump compilation artifacts as JSON into this directory.
-  string xla_dump_debug_json_to = 8;
+  // Dump compilation artifacts in binary proto into this directory.
+  string xla_dump_hlo_proto_to = 8;
 
   // Instrument the computation to collect per-HLO cycle counts.
   bool xla_hlo_profile = 9;
-- 
GitLab


From f86588ce8fb38ab3a6afc21eb08d2a2097b56adc Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 20 Oct 2017 09:50:28 -0700
Subject: [PATCH 609/909] Added gradient op for QR decomposition

PiperOrigin-RevId: 172895297
---
 tensorflow/python/kernel_tests/qr_op_test.py | 66 ++++++++++++++++++--
 tensorflow/python/ops/linalg_grad.py         | 42 +++++++++++--
 2 files changed, 98 insertions(+), 10 deletions(-)

diff --git a/tensorflow/python/kernel_tests/qr_op_test.py b/tensorflow/python/kernel_tests/qr_op_test.py
index b4fd89bd03..8848c15e76 100644
--- a/tensorflow/python/kernel_tests/qr_op_test.py
+++ b/tensorflow/python/kernel_tests/qr_op_test.py
@@ -22,6 +22,7 @@ import numpy as np
 
 from tensorflow.python.framework import constant_op
 from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import gradient_checker
 from tensorflow.python.ops import linalg_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import random_ops
@@ -140,11 +141,11 @@ def _GetQrOpTest(dtype_, shape_, full_matrices_, use_static_shape_):
       x_reshape = np.reshape(x_np, (-1, x_np.shape[-2], x_np.shape[-1]))
       for i in range(new_first_dim):
         if full_matrices_:
-          np_q_reshape[i,:,:], _ = \
-                np.linalg.qr(x_reshape[i,:,:], mode="complete")
+          np_q_reshape[i, :, :], _ = np.linalg.qr(
+              x_reshape[i, :, :], mode="complete")
         else:
-          np_q_reshape[i,:,:], _ = \
-                np.linalg.qr(x_reshape[i,:,:], mode="reduced")
+          np_q_reshape[i, :, :], _ = np.linalg.qr(
+              x_reshape[i, :, :], mode="reduced")
       np_q = np.reshape(np_q_reshape, q_dims)
       CompareOrthogonal(self, np_q, q_tf_val, min(shape_[-2:]))
       CheckApproximation(self, x_np, q_tf_val, r_tf_val)
@@ -153,6 +154,46 @@ def _GetQrOpTest(dtype_, shape_, full_matrices_, use_static_shape_):
   return Test
 
 
+class QrGradOpTest(test.TestCase):
+  pass
+
+
+def _GetQrGradOpTest(dtype_, shape_, full_matrices_):
+
+  def Test(self):
+    np.random.seed(42)
+    a = np.random.uniform(low=-1.0, high=1.0, size=shape_).astype(dtype_)
+    if dtype_ in [np.complex64, np.complex128]:
+      a += 1j * np.random.uniform(
+          low=-1.0, high=1.0, size=shape_).astype(dtype_)
+    # Optimal stepsize for central difference is O(epsilon^{1/3}).
+    epsilon = np.finfo(dtype_).eps
+    delta = 0.1 * epsilon**(1.0 / 3.0)
+    if dtype_ in [np.float32, np.complex64]:
+      tol = 3e-2
+    else:
+      tol = 1e-6
+    with self.test_session(use_gpu=True):
+      tf_a = constant_op.constant(a)
+      tf_b = linalg_ops.qr(tf_a, full_matrices=full_matrices_)
+      for b in tf_b:
+        x_init = np.random.uniform(
+            low=-1.0, high=1.0, size=shape_).astype(dtype_)
+        if dtype_ in [np.complex64, np.complex128]:
+          x_init += 1j * np.random.uniform(
+              low=-1.0, high=1.0, size=shape_).astype(dtype_)
+        theoretical, numerical = gradient_checker.compute_gradient(
+            tf_a,
+            tf_a.get_shape().as_list(),
+            b,
+            b.get_shape().as_list(),
+            x_init_value=x_init,
+            delta=delta)
+        self.assertAllClose(theoretical, numerical, atol=tol, rtol=tol)
+
+  return Test
+
+
 if __name__ == "__main__":
   for dtype in np.float32, np.float64, np.complex64, np.complex128:
     for rows in 1, 2, 5, 10, 32, 100:
@@ -168,4 +209,21 @@ if __name__ == "__main__":
               _AddTest(QrOpTest, "Qr", name,
                        _GetQrOpTest(dtype, shape, full_matrices,
                                     use_static_shape))
+
+  # TODO(pfau): Get working with complex types.
+  # TODO(pfau): Get working with full_matrices when rows != cols
+  # TODO(pfau): Get working when rows < cols
+  # TODO(pfau): Get working with shapeholders (dynamic shapes)
+  for full_matrices in False, True:
+    for dtype in np.float32, np.float64:
+      for rows in 1, 2, 5, 10:
+        for cols in 1, 2, 5, 10:
+          if rows == cols or (not full_matrices and rows > cols):
+            for batch_dims in [(), (3,)] + [(3, 2)] * (max(rows, cols) < 10):
+              shape = batch_dims + (rows, cols)
+              name = "%s_%s_full_%s" % (dtype.__name__,
+                                        "_".join(map(str, shape)),
+                                        full_matrices)
+              _AddTest(QrGradOpTest, "QrGrad", name,
+                       _GetQrGradOpTest(dtype, shape, full_matrices))
   test.main()
diff --git a/tensorflow/python/ops/linalg_grad.py b/tensorflow/python/ops/linalg_grad.py
index ec263591e1..8a76fe3ce5 100644
--- a/tensorflow/python/ops/linalg_grad.py
+++ b/tensorflow/python/ops/linalg_grad.py
@@ -81,6 +81,36 @@ def _CholeskyGrad(op, grad):
   return grad_a * 0.5
 
 
+@ops.RegisterGradient("Qr")
+def _QrGrad(op, dq, dr):
+  """Gradient for Qr."""
+  q, r = op.outputs
+  if q.dtype.is_complex:
+    raise NotImplementedError("QrGrad not implemented for dtype: %s" % q.dtype)
+  if (r.shape.ndims is None or r.shape.as_list()[-2] is None or
+      r.shape.as_list()[-1] is None):
+    raise NotImplementedError("QrGrad not implemented with dynamic shapes.")
+  if r.shape[-2].value != r.shape[-1].value:
+    raise NotImplementedError("QrGrad not implemented when ncols > nrows "
+                              "or full_matrices is true and ncols != nrows.")
+
+  qdq = math_ops.matmul(q, dq, adjoint_a=True)
+  qdq_ = qdq - _linalg.adjoint(qdq)
+  rdr = math_ops.matmul(r, dr, adjoint_b=True)
+  rdr_ = rdr - _linalg.adjoint(rdr)
+  tril = array_ops.matrix_band_part(qdq_ + rdr_, -1, 0)
+
+  def _TriangularSolve(x, r):
+    """Equiv to matmul(x, adjoint(matrix_inverse(r))) if r is upper-tri."""
+    return _linalg.adjoint(
+        linalg_ops.matrix_triangular_solve(
+            r, _linalg.adjoint(x), lower=False, adjoint=False))
+
+  grad_a = math_ops.matmul(q, dr + _TriangularSolve(tril, r))
+  grad_b = _TriangularSolve(dq - math_ops.matmul(q, qdq), r)
+  return grad_a + grad_b
+
+
 @ops.RegisterGradient("MatrixSolve")
 def _MatrixSolveGrad(op, grad):
   """Gradient for MatrixSolve."""
@@ -105,7 +135,7 @@ def _MatrixSolveLsGrad(op, grad):
   #   b) Implement a symmetric rank-k update op instead of computing
   #      x*z + transpose(x*z). This pattern occurs other places in TensorFlow.
 
-  def _overdetermined(op, grad):
+  def _Overdetermined(op, grad):
     """Gradients for the overdetermined case of MatrixSolveLs.
 
     This is the backprop for the solution to the normal equations of the first
@@ -130,7 +160,7 @@ def _MatrixSolveLsGrad(op, grad):
     grad_b = math_ops.matmul(a, z)
     return (grad_a, grad_b, None)
 
-  def _underdetermined(op, grad):
+  def _Underdetermined(op, grad):
     """Gradients for the underdetermined case of MatrixSolveLs.
 
     This is the backprop for the solution to the normal equations of the second
@@ -162,16 +192,16 @@ def _MatrixSolveLsGrad(op, grad):
   matrix_shape = op.inputs[0].get_shape()[-2:]
   if matrix_shape.is_fully_defined():
     if matrix_shape[-2] >= matrix_shape[-1]:
-      return _overdetermined(op, grad)
+      return _Overdetermined(op, grad)
     else:
-      return _underdetermined(op, grad)
+      return _Underdetermined(op, grad)
   else:
     # We have to defer determining the shape to runtime and use
     # conditional execution of the appropriate graph.
     matrix_shape = array_ops.shape(op.inputs[0])[-2:]
     return control_flow_ops.cond(matrix_shape[-2] >= matrix_shape[-1],
-                                 lambda: _overdetermined(op, grad),
-                                 lambda: _underdetermined(op, grad))
+                                 lambda: _Overdetermined(op, grad),
+                                 lambda: _Underdetermined(op, grad))
 
 
 @ops.RegisterGradient("MatrixTriangularSolve")
-- 
GitLab


From c91dadb3737395de6b09f4f52596d7ce202eff8f Mon Sep 17 00:00:00 2001
From: Max Galkin <maxgalkin@google.com>
Date: Fri, 20 Oct 2017 10:43:56 -0700
Subject: [PATCH 610/909] Minor change: extra logging to help understand the
 effects of OptimizeGraph and PruneGraph calls.

PiperOrigin-RevId: 172902338
---
 tensorflow/core/grappler/grappler_item_builder.cc | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/grappler/grappler_item_builder.cc b/tensorflow/core/grappler/grappler_item_builder.cc
index 54d60cd7aa..3f6183b6f1 100644
--- a/tensorflow/core/grappler/grappler_item_builder.cc
+++ b/tensorflow/core/grappler/grappler_item_builder.cc
@@ -450,12 +450,16 @@ std::unique_ptr<GrapplerItem> GrapplerItemFromMetaGraphDef(
   }
 
   // Optimize the graph (function inlining, l1 optimizations, etc).
+  VLOG(1) << "Number of nodes in graph before OptimizeGraph: "
+          << new_item->graph.node_size();
   Status optimize_status =
       OptimizeGraph(new_item->graph, &new_item->graph, cfg);
   if (!optimize_status.ok()) {
     LOG(ERROR) << "Graph preprocessing failed: " << optimize_status;
     return nullptr;
   }
+  VLOG(1) << "Number of nodes in graph after OptimizeGraph: "
+          << new_item->graph.node_size();
 
   if (cfg.prune_graph) {
     VLOG(1) << "Pruning graph...";
@@ -464,7 +468,8 @@ std::unique_ptr<GrapplerItem> GrapplerItemFromMetaGraphDef(
       LOG(ERROR) << "Pruning failed: " << status.error_message();
       return nullptr;
     }
-    VLOG(1) << "Pruning ran succesfully.";
+    VLOG(1) << "Number of nodes in graph after pruning: "
+            << new_item->graph.node_size();
   }
 
   // Validate feed, fetch and init nodes
-- 
GitLab


From 8f7439888c7c3ea7f188df64952cfb4f1e082ecc Mon Sep 17 00:00:00 2001
From: Akshay Agrawal <akshayka@google.com>
Date: Fri, 20 Oct 2017 10:45:35 -0700
Subject: [PATCH 611/909] Patch dynamic_rnn to work in Eager mode

PiperOrigin-RevId: 172902635
---
 tensorflow/contrib/rnn/BUILD                  |   2 +
 .../rnn/python/kernel_tests/core_rnn_test.py  | 364 +++++++++++-------
 tensorflow/python/BUILD                       |   1 +
 tensorflow/python/kernel_tests/BUILD          |   2 +
 tensorflow/python/kernel_tests/rnn_test.py    |  91 +++--
 tensorflow/python/ops/rnn.py                  |  76 ++--
 6 files changed, 339 insertions(+), 197 deletions(-)

diff --git a/tensorflow/contrib/rnn/BUILD b/tensorflow/contrib/rnn/BUILD
index 571d299ad9..29ba26d75d 100644
--- a/tensorflow/contrib/rnn/BUILD
+++ b/tensorflow/contrib/rnn/BUILD
@@ -156,6 +156,7 @@ cuda_py_tests(
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:control_flow_ops",
         "//tensorflow/python:framework_for_generated_wrappers",
+        "//tensorflow/python:framework_test_lib",
         "//tensorflow/python:gradients",
         "//tensorflow/python:init_ops",
         "//tensorflow/python:math_ops",
@@ -165,6 +166,7 @@ cuda_py_tests(
         "//tensorflow/python:util",
         "//tensorflow/python:variable_scope",
         "//tensorflow/python:variables",
+        "//tensorflow/python/eager:context",
     ],
     shard_count = 10,
 )
diff --git a/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_test.py b/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_test.py
index 2fa033632a..12def6dcc8 100644
--- a/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_test.py
+++ b/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_test.py
@@ -25,10 +25,12 @@ from six.moves import xrange  # pylint: disable=redefined-builtin
 
 from tensorflow.contrib import rnn as rnn_lib
 from tensorflow.core.protobuf import config_pb2
+from tensorflow.python.eager import context
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops as ops_lib
 from tensorflow.python.framework import tensor_shape
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import gradients_impl
@@ -881,6 +883,7 @@ class LSTMTest(test.TestCase):
     # Smoke test, this should not raise an error
     rnn.dynamic_rnn(cell, inputs, dtype=dtypes.float32)
 
+  @test_util.run_in_graph_and_eager_modes()
   def testDynamicRNNWithTupleStates(self):
     num_units = 3
     input_size = 5
@@ -888,13 +891,20 @@ class LSTMTest(test.TestCase):
     num_proj = 4
     max_length = 8
     sequence_length = [4, 6]
+    in_graph_mode = context.in_graph_mode()
     with self.test_session(graph=ops_lib.Graph()) as sess:
       initializer = init_ops.random_uniform_initializer(
           -0.01, 0.01, seed=self._seed)
-      inputs = max_length * [
-          array_ops.placeholder(
-              dtypes.float32, shape=(None, input_size))
-      ]
+      if in_graph_mode:
+        inputs = max_length * [
+            array_ops.placeholder(
+                dtypes.float32, shape=(None, input_size))
+        ]
+      else:
+        inputs = max_length * [
+            constant_op.constant(
+                np.random.randn(batch_size, input_size).astype(np.float32))
+        ]
       inputs_c = array_ops.stack(inputs)
       cell = rnn_cell.LSTMCell(
           num_units,
@@ -924,21 +934,34 @@ class LSTMTest(test.TestCase):
       self.assertEqual(state_dynamic[0], state_dynamic.c)
       self.assertEqual(state_dynamic[1], state_dynamic.h)
 
-      variables_lib.global_variables_initializer().run()
-
-      input_value = np.random.randn(batch_size, input_size)
-      outputs_static_v = sess.run(outputs_static,
-                                  feed_dict={inputs[0]: input_value})
-      outputs_dynamic_v = sess.run(outputs_dynamic,
-                                   feed_dict={inputs[0]: input_value})
-      self.assertAllEqual(outputs_static_v, outputs_dynamic_v)
-
-      state_static_v = sess.run(state_static,
-                                feed_dict={inputs[0]: input_value})
-      state_dynamic_v = sess.run(state_dynamic,
-                                 feed_dict={inputs[0]: input_value})
-      self.assertAllEqual(np.hstack(state_static_v), np.hstack(state_dynamic_v))
+      if in_graph_mode:
+        variables_lib.global_variables_initializer().run()
+        input_value = np.random.randn(batch_size, input_size)
+        outputs_static = sess.run(
+            outputs_static, feed_dict={
+                inputs[0]: input_value
+            })
+        outputs_dynamic = sess.run(
+            outputs_dynamic, feed_dict={
+                inputs[0]: input_value
+            })
+        state_static = sess.run(
+            state_static, feed_dict={
+                inputs[0]: input_value
+            })
+        state_dynamic = sess.run(
+            state_dynamic, feed_dict={
+                inputs[0]: input_value
+            })
+
+      if in_graph_mode:
+        self.assertAllEqual(outputs_static, outputs_dynamic)
+      else:
+        self.assertAllEqual(
+            array_ops.stack(outputs_static).numpy(), outputs_dynamic.numpy())
+      self.assertAllEqual(np.hstack(state_static), np.hstack(state_dynamic))
 
+  @test_util.run_in_graph_and_eager_modes()
   def testDynamicRNNWithNestedTupleStates(self):
     num_units = 3
     input_size = 5
@@ -946,13 +969,20 @@ class LSTMTest(test.TestCase):
     num_proj = 4
     max_length = 8
     sequence_length = [4, 6]
+    in_graph_mode = context.in_graph_mode()
     with self.test_session(graph=ops_lib.Graph()) as sess:
       initializer = init_ops.random_uniform_initializer(
           -0.01, 0.01, seed=self._seed)
-      inputs = max_length * [
-          array_ops.placeholder(
-              dtypes.float32, shape=(None, input_size))
-      ]
+      if in_graph_mode:
+        inputs = max_length * [
+            array_ops.placeholder(
+                dtypes.float32, shape=(None, input_size))
+        ]
+      else:
+        inputs = max_length * [
+            constant_op.constant(
+                np.random.randn(batch_size, input_size).astype(np.float32))
+        ]
       inputs_c = array_ops.stack(inputs)
 
       def _cell(i):
@@ -993,20 +1023,34 @@ class LSTMTest(test.TestCase):
             sequence_length=sequence_length,
             scope=scope)
 
-      variables_lib.global_variables_initializer().run()
-
-      input_value = np.random.randn(batch_size, input_size)
-      outputs_static_v = sess.run(outputs_static,
-                                  feed_dict={inputs[0]: input_value})
-      outputs_dynamic_v = sess.run(outputs_dynamic,
-                                   feed_dict={inputs[0]: input_value})
-      self.assertAllEqual(outputs_static_v, outputs_dynamic_v)
-
-      state_static_v = sess.run(nest.flatten(state_static),
-                                feed_dict={inputs[0]: input_value})
-      state_dynamic_v = sess.run(nest.flatten(state_dynamic),
-                                 feed_dict={inputs[0]: input_value})
-      self.assertAllEqual(np.hstack(state_static_v), np.hstack(state_dynamic_v))
+      if in_graph_mode:
+        input_value = np.random.randn(batch_size, input_size)
+        variables_lib.global_variables_initializer().run()
+        outputs_static = sess.run(
+            outputs_static, feed_dict={
+                inputs[0]: input_value
+            })
+        outputs_dynamic = sess.run(
+            outputs_dynamic, feed_dict={
+                inputs[0]: input_value
+            })
+        state_static = sess.run(
+            nest.flatten(state_static), feed_dict={
+                inputs[0]: input_value
+            })
+        state_dynamic = sess.run(
+            nest.flatten(state_dynamic), feed_dict={
+                inputs[0]: input_value
+            })
+
+      if in_graph_mode:
+        self.assertAllEqual(outputs_static, outputs_dynamic)
+      else:
+        self.assertAllEqual(
+            array_ops.stack(outputs_static).numpy(), outputs_dynamic.numpy())
+        state_static = [s.numpy() for s in nest.flatten(state_static)]
+        state_dynamic = [s.numpy() for s in nest.flatten(state_dynamic)]
+      self.assertAllEqual(np.hstack(state_static), np.hstack(state_dynamic))
 
   def _testDynamicEquivalentToStaticRNN(self, use_gpu, use_sequence_length):
     time_steps = 8
@@ -1015,21 +1059,22 @@ class LSTMTest(test.TestCase):
     input_size = 5
     batch_size = 2
 
-    input_values = np.random.randn(time_steps, batch_size, input_size)
+    input_values = np.random.randn(time_steps, batch_size, input_size).astype(
+        np.float32)
 
     if use_sequence_length:
       sequence_length = np.random.randint(0, time_steps, size=batch_size)
     else:
       sequence_length = None
 
-    ########### Step 1: Run static graph and generate readouts
-    with self.test_session(use_gpu=use_gpu, graph=ops_lib.Graph()) as sess:
-      concat_inputs = array_ops.placeholder(
-          dtypes.float32, shape=(time_steps, batch_size, input_size))
-      inputs = array_ops.unstack(concat_inputs)
+    in_graph_mode = context.in_graph_mode()
+
+    # TODO(b/68017812): Eager ignores operation seeds, so we need to create a
+    # single cell and reuse it across the static and dynamic RNNs. Remove this
+    # special case once is fixed.
+    if not in_graph_mode:
       initializer = init_ops.random_uniform_initializer(
           -0.01, 0.01, seed=self._seed)
-
       cell = rnn_cell.LSTMCell(
           num_units,
           use_peepholes=True,
@@ -1037,63 +1082,85 @@ class LSTMTest(test.TestCase):
           num_proj=num_proj,
           state_is_tuple=False)
 
+    ########### Step 1: Run static graph and generate readouts
+    with self.test_session(use_gpu=use_gpu, graph=ops_lib.Graph()) as sess:
+      if in_graph_mode:
+        concat_inputs = array_ops.placeholder(
+            dtypes.float32, shape=(time_steps, batch_size, input_size))
+      else:
+        concat_inputs = constant_op.constant(input_values)
+      inputs = array_ops.unstack(concat_inputs)
+      initializer = init_ops.random_uniform_initializer(
+          -0.01, 0.01, seed=self._seed)
+
+      # TODO(akshayka): Remove special case once b/68017812 is fixed.
+      if in_graph_mode:
+        cell = rnn_cell.LSTMCell(
+            num_units,
+            use_peepholes=True,
+            initializer=initializer,
+            num_proj=num_proj,
+            state_is_tuple=False)
+
       with variable_scope.variable_scope("dynamic_scope"):
         outputs_static, state_static = rnn.static_rnn(
             cell, inputs, sequence_length=sequence_length, dtype=dtypes.float32)
 
-      feeds = {concat_inputs: input_values}
-
-      # Initialize
-      variables_lib.global_variables_initializer().run(feed_dict=feeds)
-
-      # Generate gradients of sum of outputs w.r.t. inputs
-      static_gradients = gradients_impl.gradients(
-          outputs_static + [state_static], [concat_inputs])
-
-      # Generate gradients of individual outputs w.r.t. inputs
-      static_individual_gradients = nest.flatten([
-          gradients_impl.gradients(y, [concat_inputs])
-          for y in [outputs_static[0], outputs_static[-1], state_static]
-      ])
-
-      # Generate gradients of individual variables w.r.t. inputs
-      trainable_variables = ops_lib.get_collection(
-          ops_lib.GraphKeys.TRAINABLE_VARIABLES)
-      assert len(trainable_variables) > 1, ("Count of trainable variables: %d" %
-                                            len(trainable_variables))
-      # pylint: disable=bad-builtin
-      static_individual_variable_gradients = nest.flatten([
-          gradients_impl.gradients(y, trainable_variables)
-          for y in [outputs_static[0], outputs_static[-1], state_static]
-      ])
-
-      # Test forward pass
-      values_static = sess.run(outputs_static, feed_dict=feeds)
-      (state_value_static,) = sess.run((state_static,), feed_dict=feeds)
-
-      # Test gradients to inputs and variables w.r.t. outputs & final state
-      static_grad_values = sess.run(static_gradients, feed_dict=feeds)
-
-      static_individual_grad_values = sess.run(static_individual_gradients,
-                                               feed_dict=feeds)
-
-      static_individual_var_grad_values = sess.run(
-          static_individual_variable_gradients, feed_dict=feeds)
+      if in_graph_mode:
+        # Generate gradients and run sessions to obtain outputs
+        feeds = {concat_inputs: input_values}
+        # Initialize
+        variables_lib.global_variables_initializer().run(feed_dict=feeds)
+        # Generate gradients of sum of outputs w.r.t. inputs
+        static_gradients = gradients_impl.gradients(
+            outputs_static + [state_static], [concat_inputs])
+        # Generate gradients of individual outputs w.r.t. inputs
+        static_individual_gradients = nest.flatten([
+            gradients_impl.gradients(y, [concat_inputs])
+            for y in [outputs_static[0], outputs_static[-1], state_static]
+        ])
+        # Generate gradients of individual variables w.r.t. inputs
+        trainable_variables = ops_lib.get_collection(
+            ops_lib.GraphKeys.TRAINABLE_VARIABLES)
+        assert len(trainable_variables) > 1, (
+            "Count of trainable variables: %d" % len(trainable_variables))
+        # pylint: disable=bad-builtin
+        static_individual_variable_gradients = nest.flatten([
+            gradients_impl.gradients(y, trainable_variables)
+            for y in [outputs_static[0], outputs_static[-1], state_static]
+        ])
+        # Test forward pass
+        values_static = sess.run(outputs_static, feed_dict=feeds)
+        (state_value_static,) = sess.run((state_static,), feed_dict=feeds)
+
+        # Test gradients to inputs and variables w.r.t. outputs & final state
+        static_grad_values = sess.run(static_gradients, feed_dict=feeds)
+
+        static_individual_grad_values = sess.run(static_individual_gradients,
+                                                 feed_dict=feeds)
+
+        static_individual_var_grad_values = sess.run(
+            static_individual_variable_gradients, feed_dict=feeds)
 
     ########## Step 2: Run dynamic graph and generate readouts
     with self.test_session(use_gpu=use_gpu, graph=ops_lib.Graph()) as sess:
-      concat_inputs = array_ops.placeholder(
-          dtypes.float32, shape=(time_steps, batch_size, input_size))
-      inputs = array_ops.unstack(concat_inputs)
+      if in_graph_mode:
+        concat_inputs = array_ops.placeholder(
+            dtypes.float32, shape=(time_steps, batch_size, input_size))
+      else:
+        concat_inputs = constant_op.constant(input_values)
       initializer = init_ops.random_uniform_initializer(
           -0.01, 0.01, seed=self._seed)
 
-      cell = rnn_cell.LSTMCell(
-          num_units,
-          use_peepholes=True,
-          initializer=initializer,
-          num_proj=num_proj,
-          state_is_tuple=False)
+      # TODO(akshayka): Remove this special case once b/68017812 is
+      # fixed.
+      if in_graph_mode:
+        cell = rnn_cell.LSTMCell(
+            num_units,
+            use_peepholes=True,
+            initializer=initializer,
+            num_proj=num_proj,
+            state_is_tuple=False)
 
       with variable_scope.variable_scope("dynamic_scope"):
         outputs_dynamic, state_dynamic = rnn.dynamic_rnn(
@@ -1104,72 +1171,83 @@ class LSTMTest(test.TestCase):
             dtype=dtypes.float32)
         split_outputs_dynamic = array_ops.unstack(outputs_dynamic, time_steps)
 
-      feeds = {concat_inputs: input_values}
+      if in_graph_mode:
+        feeds = {concat_inputs: input_values}
 
-      # Initialize
-      variables_lib.global_variables_initializer().run(feed_dict=feeds)
+        # Initialize
+        variables_lib.global_variables_initializer().run(feed_dict=feeds)
+
+        # Generate gradients of sum of outputs w.r.t. inputs
+        dynamic_gradients = gradients_impl.gradients(
+            split_outputs_dynamic + [state_dynamic], [concat_inputs])
+
+        # Generate gradients of several individual outputs w.r.t. inputs
+        dynamic_individual_gradients = nest.flatten([
+            gradients_impl.gradients(y, [concat_inputs])
+            for y in
+            [split_outputs_dynamic[0], split_outputs_dynamic[-1], state_dynamic]
+        ])
+
+        # Generate gradients of individual variables w.r.t. inputs
+        trainable_variables = ops_lib.get_collection(
+            ops_lib.GraphKeys.TRAINABLE_VARIABLES)
+        assert len(trainable_variables) > 1, (
+            "Count of trainable variables: %d" % len(trainable_variables))
+        dynamic_individual_variable_gradients = nest.flatten([
+            gradients_impl.gradients(y, trainable_variables)
+            for y in
+            [split_outputs_dynamic[0], split_outputs_dynamic[-1], state_dynamic]
+        ])
 
-      # Generate gradients of sum of outputs w.r.t. inputs
-      dynamic_gradients = gradients_impl.gradients(
-          split_outputs_dynamic + [state_dynamic], [concat_inputs])
-
-      # Generate gradients of several individual outputs w.r.t. inputs
-      dynamic_individual_gradients = nest.flatten([
-          gradients_impl.gradients(y, [concat_inputs])
-          for y in
-          [split_outputs_dynamic[0], split_outputs_dynamic[-1], state_dynamic]
-      ])
-
-      # Generate gradients of individual variables w.r.t. inputs
-      trainable_variables = ops_lib.get_collection(
-          ops_lib.GraphKeys.TRAINABLE_VARIABLES)
-      assert len(trainable_variables) > 1, ("Count of trainable variables: %d" %
-                                            len(trainable_variables))
-      dynamic_individual_variable_gradients = nest.flatten([
-          gradients_impl.gradients(y, trainable_variables)
-          for y in
-          [split_outputs_dynamic[0], split_outputs_dynamic[-1], state_dynamic]
-      ])
-
-      # Test forward pass
-      values_dynamic = sess.run(split_outputs_dynamic, feed_dict=feeds)
-      (state_value_dynamic,) = sess.run((state_dynamic,), feed_dict=feeds)
-
-      # Test gradients to inputs and variables w.r.t. outputs & final state
-      dynamic_grad_values = sess.run(dynamic_gradients, feed_dict=feeds)
-
-      dynamic_individual_grad_values = sess.run(dynamic_individual_gradients,
-                                                feed_dict=feeds)
-
-      dynamic_individual_var_grad_values = sess.run(
-          dynamic_individual_variable_gradients, feed_dict=feeds)
+        # Test forward pass
+        values_dynamic = sess.run(split_outputs_dynamic, feed_dict=feeds)
+        (state_value_dynamic,) = sess.run((state_dynamic,), feed_dict=feeds)
+
+        # Test gradients to inputs and variables w.r.t. outputs & final state
+        dynamic_grad_values = sess.run(dynamic_gradients, feed_dict=feeds)
+
+        dynamic_individual_grad_values = sess.run(dynamic_individual_gradients,
+                                                  feed_dict=feeds)
+
+        dynamic_individual_var_grad_values = sess.run(
+            dynamic_individual_variable_gradients, feed_dict=feeds)
 
     ######### Step 3: Comparisons
+    if not in_graph_mode:
+      values_static = outputs_static
+      values_dynamic = split_outputs_dynamic
+      state_value_static = state_static
+      state_value_dynamic = state_dynamic
+
     self.assertEqual(len(values_static), len(values_dynamic))
     for (value_static, value_dynamic) in zip(values_static, values_dynamic):
       self.assertAllEqual(value_static, value_dynamic)
     self.assertAllEqual(state_value_static, state_value_dynamic)
 
-    self.assertAllEqual(static_grad_values, dynamic_grad_values)
+    if in_graph_mode:
+
+      self.assertAllEqual(static_grad_values, dynamic_grad_values)
 
-    self.assertEqual(
-        len(static_individual_grad_values), len(dynamic_individual_grad_values))
-    self.assertEqual(
-        len(static_individual_var_grad_values),
-        len(dynamic_individual_var_grad_values))
+      self.assertEqual(
+          len(static_individual_grad_values),
+          len(dynamic_individual_grad_values))
+      self.assertEqual(
+          len(static_individual_var_grad_values),
+          len(dynamic_individual_var_grad_values))
 
-    for i, (a, b) in enumerate(
-        zip(static_individual_grad_values, dynamic_individual_grad_values)):
-      tf_logging.info("Comparing individual gradients iteration %d" % i)
-      self.assertAllEqual(a, b)
+      for i, (a, b) in enumerate(
+          zip(static_individual_grad_values, dynamic_individual_grad_values)):
+        tf_logging.info("Comparing individual gradients iteration %d" % i)
+        self.assertAllEqual(a, b)
 
-    for i, (a, b) in enumerate(
-        zip(static_individual_var_grad_values,
-            dynamic_individual_var_grad_values)):
-      tf_logging.info("Comparing individual variable gradients iteration %d" %
-                      i)
-      self.assertAllEqual(a, b)
+      for i, (a, b) in enumerate(
+          zip(static_individual_var_grad_values,
+              dynamic_individual_var_grad_values)):
+        tf_logging.info("Comparing individual variable gradients iteration %d" %
+                        i)
+        self.assertAllEqual(a, b)
 
+  @test_util.run_in_graph_and_eager_modes()
   def testDynamicEquivalentToStaticRNN(self):
     self._testDynamicEquivalentToStaticRNN(
         use_gpu=False, use_sequence_length=False)
diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index e63c554e47..b7aa7bbf6b 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -1978,6 +1978,7 @@ py_library(
         ":tensor_array_ops",
         ":util",
         ":variable_scope",
+        "//tensorflow/python/eager:context",
     ],
 )
 
diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD
index dece290f83..e6848edc12 100644
--- a/tensorflow/python/kernel_tests/BUILD
+++ b/tensorflow/python/kernel_tests/BUILD
@@ -2297,6 +2297,7 @@ cuda_py_test(
         "//tensorflow/python:control_flow_ops",
         "//tensorflow/python:data_flow_grad",
         "//tensorflow/python:framework_for_generated_wrappers",
+        "//tensorflow/python:framework_test_lib",
         "//tensorflow/python:gradients",
         "//tensorflow/python:init_ops",
         "//tensorflow/python:nn_grad",
@@ -2305,6 +2306,7 @@ cuda_py_test(
         "//tensorflow/python:sparse_grad",
         "//tensorflow/python:tensor_array_grad",
         "//tensorflow/python:variables",
+        "//tensorflow/python/eager:context",
     ],
     shard_count = 10,
     tags = ["no_windows"],
diff --git a/tensorflow/python/kernel_tests/rnn_test.py b/tensorflow/python/kernel_tests/rnn_test.py
index a644e6a44f..d8f4b439e3 100644
--- a/tensorflow/python/kernel_tests/rnn_test.py
+++ b/tensorflow/python/kernel_tests/rnn_test.py
@@ -26,9 +26,12 @@ import numpy as np
 from tensorflow.contrib import rnn as contrib_rnn
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.python.client import session
+from tensorflow.python.eager import context
+from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops as ops_lib
 from tensorflow.python.framework import tensor_shape
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import gradients_impl
@@ -82,9 +85,13 @@ class RNNTest(test.TestCase):
     self._seed = 23489
     np.random.seed(self._seed)
 
+  @test_util.run_in_graph_and_eager_modes()
   def testInvalidSequenceLengthShape(self):
     cell = Plus1RNNCell()
-    inputs = [array_ops.placeholder(dtypes.float32, shape=(3, 4))]
+    if context.in_graph_mode():
+      inputs = [array_ops.placeholder(dtypes.float32, shape=(3, 4))]
+    else:
+      inputs = [constant_op.constant(np.ones((3, 4)))]
     with self.assertRaisesRegexp(ValueError, "must be a vector"):
       rnn.dynamic_rnn(
           cell,
@@ -92,45 +99,77 @@ class RNNTest(test.TestCase):
           dtype=dtypes.float32,
           sequence_length=[[4]])
 
+  @test_util.run_in_graph_and_eager_modes()
   def testBatchSizeFromInput(self):
     cell = Plus1RNNCell()
+    in_graph_mode = context.in_graph_mode()
     # With static batch size
-    inputs = array_ops.placeholder(dtypes.float32, shape=(3, 4, 5))
+    if in_graph_mode:
+      inputs = array_ops.placeholder(dtypes.float32, shape=(3, 4, 5))
+      initial_state = array_ops.placeholder(dtypes.float32, shape=(3, 5))
+    else:
+      inputs = np.zeros((3, 4, 5), dtype=np.float32)
+      initial_state = np.zeros((3, 5), dtype=np.float32)
+
     # - Without initial_state
     outputs, state = rnn.dynamic_rnn(cell, inputs, dtype=dtypes.float32)
-    self.assertEqual(3, outputs.shape[0].value)
-    self.assertEqual(3, state.shape[0].value)
+    if in_graph_mode:
+      self.assertEqual(3, outputs.shape[0].value)
+      self.assertEqual(3, state.shape[0].value)
+    else:
+      self.assertEqual(3, outputs.shape[0])
+      self.assertEqual(3, state.shape[0])
+
     # - With initial_state
     outputs, state = rnn.dynamic_rnn(
-        cell,
-        inputs,
-        initial_state=array_ops.placeholder(dtypes.float32, shape=(3, 5)))
-    self.assertEqual(3, outputs.shape[0].value)
-    self.assertEqual(3, state.shape[0].value)
+        cell, inputs, initial_state=initial_state)
+    if in_graph_mode:
+      self.assertEqual(3, outputs.shape[0].value)
+      self.assertEqual(3, state.shape[0].value)
+    else:
+      self.assertEqual(3, outputs.shape[0])
+      self.assertEqual(3, state.shape[0])
+
     # Without static batch size
-    inputs = array_ops.placeholder(dtypes.float32, shape=(None, 4, 5))
-    # - Without initial_state
-    outputs, state = rnn.dynamic_rnn(cell, inputs, dtype=dtypes.float32)
-    self.assertEqual(None, outputs.shape[0].value)
-    self.assertEqual(None, state.shape[0].value)
-    # - With initial_state
-    outputs, state = rnn.dynamic_rnn(
-        cell,
-        inputs,
-        initial_state=array_ops.placeholder(dtypes.float32, shape=(None, 5)))
-    self.assertEqual(None, outputs.shape[0].value)
-    self.assertEqual(None, state.shape[0].value)
+    # Tensor shapes are fully determined in Eager mode, so only run this
+    # test in graph mode.
+    if in_graph_mode:
+      inputs = array_ops.placeholder(dtypes.float32, shape=(None, 4, 5))
+      # - Without initial_state
+      outputs, state = rnn.dynamic_rnn(cell, inputs, dtype=dtypes.float32)
+      self.assertEqual(None, outputs.shape[0].value)
+      self.assertEqual(None, state.shape[0].value)
+      # - With initial_state
+      outputs, state = rnn.dynamic_rnn(
+          cell,
+          inputs,
+          initial_state=array_ops.placeholder(dtypes.float32, shape=(None, 5)))
+      self.assertEqual(None, outputs.shape[0].value)
+      self.assertEqual(None, state.shape[0].value)
 
+  @test_util.run_in_graph_and_eager_modes()
   def testScalarStateIsAccepted(self):
     cell = ScalarStateRNNCell()
-    inputs = array_ops.placeholder(dtypes.float32, shape=(1, 4, 1))
+    in_graph_mode = context.in_graph_mode()
+
+    if in_graph_mode:
+      inputs = array_ops.placeholder(dtypes.float32, shape=(1, 4, 1))
+    else:
+      inputs = np.array([[[1], [2], [3], [4]]], dtype=np.float32)
+
     with self.test_session() as sess:
       outputs, state = rnn.dynamic_rnn(
           cell, inputs, dtype=dtypes.float32, sequence_length=[4])
-      outputs, state = sess.run(
-          [outputs, state], feed_dict={inputs: [[[1], [2], [3], [4]]]})
-    self.assertAllEqual(outputs, [[[1], [2], [3], [4]]])
-    self.assertEqual(state, 4)
+      if in_graph_mode:
+        outputs, state = sess.run(
+            [outputs, state], feed_dict={inputs: [[[1], [2], [3], [4]]]})
+
+    if in_graph_mode:
+      self.assertAllEqual(outputs, np.array([[[1], [2], [3], [4]]]))
+      self.assertEqual(state, 4)
+    else:
+      self.assertAllEqual(outputs.numpy(), np.array([[[1], [2], [3], [4]]]))
+      self.assertEqual(state.numpy(), 4)
 
 
 ######### Benchmarking RNN code
diff --git a/tensorflow/python/ops/rnn.py b/tensorflow/python/ops/rnn.py
index b174956e60..21c7ed361d 100644
--- a/tensorflow/python/ops/rnn.py
+++ b/tensorflow/python/ops/rnn.py
@@ -27,6 +27,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from tensorflow.python.eager import context
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
@@ -576,8 +577,9 @@ def dynamic_rnn(cell, inputs, sequence_length=None, initial_state=None,
   # determined by the parent scope, or is set to place the cached
   # Variable using the same placement as for the rest of the RNN.
   with vs.variable_scope(scope or "rnn") as varscope:
-    if varscope.caching_device is None:
-      varscope.set_caching_device(lambda op: op.device)
+    if context.in_graph_mode():
+      if varscope.caching_device is None:
+        varscope.set_caching_device(lambda op: op.device)
     batch_size = _best_effort_input_batch_size(flat_input)
 
     if initial_state is not None:
@@ -595,7 +597,7 @@ def dynamic_rnn(cell, inputs, sequence_length=None, initial_state=None,
           ["Expected shape for Tensor %s is " % x.name,
            packed_shape, " but saw shape: ", x_shape])
 
-    if sequence_length is not None:
+    if context.in_graph_mode() and sequence_length is not None:
       # Perform some shape validation
       with ops.control_dependencies(
           [_assert_has_shape(sequence_length, [batch_size])]):
@@ -718,14 +720,19 @@ def _dynamic_rnn_loop(cell,
                                         size=time_steps,
                                         tensor_array_name=base_name + name)
 
-  output_ta = tuple(_create_ta("output_%d" % i,
-                               _infer_state_dtype(dtype, state))
-                    for i in range(len(flat_output_size)))
-  input_ta = tuple(_create_ta("input_%d" % i, flat_input[i].dtype)
-                   for i in range(len(flat_input)))
-
-  input_ta = tuple(ta.unstack(input_)
-                   for ta, input_ in zip(input_ta, flat_input))
+  in_graph_mode = context.in_graph_mode()
+  if in_graph_mode:
+    output_ta = tuple(_create_ta("output_%d" % i,
+                                 _infer_state_dtype(dtype, state))
+                      for i in range(len(flat_output_size)))
+    input_ta = tuple(_create_ta("input_%d" % i, flat_input[i].dtype)
+                     for i in range(len(flat_input)))
+    input_ta = tuple(ta.unstack(input_)
+                     for ta, input_ in zip(input_ta, flat_input))
+  else:
+    output_ta = tuple([0 for _ in range(time_steps.numpy())]
+                      for i in range(len(flat_output_size)))
+    input_ta = flat_input
 
   def _time_step(time, output_ta_t, state):
     """Take a time step of the dynamic RNN.
@@ -739,10 +746,13 @@ def _dynamic_rnn_loop(cell,
       The tuple (time + 1, output_ta_t with updated flow, new_state).
     """
 
-    input_t = tuple(ta.read(time) for ta in input_ta)
-    # Restore some shape information
-    for input_, shape in zip(input_t, inputs_got_shape):
-      input_.set_shape(shape[1:])
+    if in_graph_mode:
+      input_t = tuple(ta.read(time) for ta in input_ta)
+      # Restore some shape information
+      for input_, shape in zip(input_t, inputs_got_shape):
+        input_.set_shape(shape[1:])
+    else:
+      input_t = tuple(ta[time.numpy()] for ta in input_ta)
 
     input_t = nest.pack_sequence_as(structure=inputs, flat_sequence=input_t)
     call_cell = lambda: cell(input_t, state)
@@ -764,8 +774,12 @@ def _dynamic_rnn_loop(cell,
     # Pack state if using state tuples
     output = nest.flatten(output)
 
-    output_ta_t = tuple(
-        ta.write(time, out) for ta, out in zip(output_ta_t, output))
+    if in_graph_mode:
+      output_ta_t = tuple(
+          ta.write(time, out) for ta, out in zip(output_ta_t, output))
+    else:
+      for ta, out in zip(output_ta_t, output):
+        ta[time.numpy()] = out
 
     return (time + 1, output_ta_t, new_state)
 
@@ -777,16 +791,20 @@ def _dynamic_rnn_loop(cell,
       swap_memory=swap_memory)
 
   # Unpack final output if not using output tuples.
-  final_outputs = tuple(ta.stack() for ta in output_final_ta)
-
-  # Restore some shape information
-  for output, output_size in zip(final_outputs, flat_output_size):
-    shape = _concat(
-        [const_time_steps, const_batch_size], output_size, static=True)
-    output.set_shape(shape)
+  if in_graph_mode:
+    final_outputs = tuple(ta.stack() for ta in output_final_ta)
+    # Restore some shape information
+    for output, output_size in zip(final_outputs, flat_output_size):
+      shape = _concat(
+          [const_time_steps, const_batch_size], output_size, static=True)
+      output.set_shape(shape)
+  else:
+    final_outputs = output_final_ta
 
   final_outputs = nest.pack_sequence_as(
       structure=cell.output_size, flat_sequence=final_outputs)
+  if not in_graph_mode:
+    final_outputs = array_ops.stack(final_outputs, axis=0)
 
   return (final_outputs, final_state)
 
@@ -967,8 +985,9 @@ def raw_rnn(cell, loop_fn,
   # determined by the parent scope, or is set to place the cached
   # Variable using the same placement as for the rest of the RNN.
   with vs.variable_scope(scope or "rnn") as varscope:
-    if varscope.caching_device is None:
-      varscope.set_caching_device(lambda op: op.device)
+    if context.in_graph_mode():
+      if varscope.caching_device is None:
+        varscope.set_caching_device(lambda op: op.device)
 
     time = constant_op.constant(0, dtype=dtypes.int32)
     (elements_finished, next_input, initial_state, emit_structure,
@@ -1166,8 +1185,9 @@ def static_rnn(cell,
   # determined by the parent scope, or is set to place the cached
   # Variable using the same placement as for the rest of the RNN.
   with vs.variable_scope(scope or "rnn") as varscope:
-    if varscope.caching_device is None:
-      varscope.set_caching_device(lambda op: op.device)
+    if context.in_graph_mode():
+      if varscope.caching_device is None:
+        varscope.set_caching_device(lambda op: op.device)
 
     # Obtain the first sequence of the input
     first_input = inputs
-- 
GitLab


From 0f5683d629c6607d1baeaa44ecd264321ae05abc Mon Sep 17 00:00:00 2001
From: Jianwei Xie <xiejw@google.com>
Date: Fri, 20 Oct 2017 10:45:51 -0700
Subject: [PATCH 612/909] Migrate the iris example to use TF core API.

PiperOrigin-RevId: 172902682
---
 tensorflow/examples/learn/iris.py | 101 ++++++++++++++++++++++--------
 1 file changed, 74 insertions(+), 27 deletions(-)

diff --git a/tensorflow/examples/learn/iris.py b/tensorflow/examples/learn/iris.py
index 33e8d45801..0a50b3ba87 100644
--- a/tensorflow/examples/learn/iris.py
+++ b/tensorflow/examples/learn/iris.py
@@ -17,47 +17,94 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import numpy as np
-from sklearn import datasets
-from sklearn import metrics
-from sklearn import model_selection
+import os
+import urllib
 
 import tensorflow as tf
 
+# Data sets
+IRIS_TRAINING = 'iris_training.csv'
+IRIS_TRAINING_URL = 'http://download.tensorflow.org/data/iris_training.csv'
 
-X_FEATURE = 'x'  # Name of the input feature.
+IRIS_TEST = 'iris_test.csv'
+IRIS_TEST_URL = 'http://download.tensorflow.org/data/iris_test.csv'
+
+FEATURE_KEYS = ['sepal_length', 'sepal_width', 'petal_length', 'petal_width']
+
+
+def maybe_download_iris_data(file_name, download_url):
+  """Downloads the file and returns the number of data."""
+  if not os.path.exists(file_name):
+    raw = urllib.urlopen(download_url).read()
+    with open(file_name, 'w') as f:
+      f.write(raw)
+
+  # The first line is a comma-separated string. The first one is the number of
+  # total data in the file.
+  with open(file_name, 'r') as f:
+    first_line = f.readline()
+  num_elements = first_line.split(',')[0]
+  return int(num_elements)
+
+
+def input_fn(file_name, num_data, batch_size, is_training):
+  """Creates an input_fn required by Estimator train/evaluate."""
+  # If the data sets aren't stored locally, download them.
+
+  def _parse_csv(rows_string_tensor):
+    """Takes the string input tensor and returns tuple of (features, labels)."""
+    # Last dim is the label.
+    num_features = len(FEATURE_KEYS)
+    num_columns = num_features + 1
+    columns = tf.decode_csv(rows_string_tensor,
+                            record_defaults=[[]] * num_columns)
+    features = dict(zip(FEATURE_KEYS, columns[:num_features]))
+    labels = tf.cast(columns[num_features], tf.int32)
+    return features, labels
+
+  def _input_fn():
+    """The input_fn."""
+    dataset = tf.data.TextLineDataset([file_name])
+    # Skip the first line (which does not have data).
+    dataset = dataset.skip(1)
+    dataset = dataset.map(_parse_csv)
+
+    if is_training:
+      # For this small dataset, which can fit into memory, to achieve true
+      # randomness, the shuffle buffer size is set as the total number of
+      # elements in the dataset.
+      dataset = dataset.shuffle(num_data)
+      dataset = dataset.repeat()
+
+    dataset = dataset.batch(batch_size)
+    iterator = dataset.make_one_shot_iterator()
+    features, labels = iterator.get_next()
+    return features, labels
+
+  return _input_fn
 
 
 def main(unused_argv):
-  # Load dataset.
-  iris = datasets.load_iris()
-  x_train, x_test, y_train, y_test = model_selection.train_test_split(
-      iris.data, iris.target, test_size=0.2, random_state=42)
+  tf.logging.set_verbosity(tf.logging.INFO)
+
+  num_training_data = maybe_download_iris_data(
+      IRIS_TRAINING, IRIS_TRAINING_URL)
+  num_test_data = maybe_download_iris_data(IRIS_TEST, IRIS_TEST_URL)
 
   # Build 3 layer DNN with 10, 20, 10 units respectively.
   feature_columns = [
-      tf.feature_column.numeric_column(
-          X_FEATURE, shape=np.array(x_train).shape[1:])]
+      tf.feature_column.numeric_column(key, shape=1) for key in FEATURE_KEYS]
   classifier = tf.estimator.DNNClassifier(
       feature_columns=feature_columns, hidden_units=[10, 20, 10], n_classes=3)
 
   # Train.
-  train_input_fn = tf.estimator.inputs.numpy_input_fn(
-      x={X_FEATURE: x_train}, y=y_train, num_epochs=None, shuffle=True)
-  classifier.train(input_fn=train_input_fn, steps=200)
-
-  # Predict.
-  test_input_fn = tf.estimator.inputs.numpy_input_fn(
-      x={X_FEATURE: x_test}, y=y_test, num_epochs=1, shuffle=False)
-  predictions = classifier.predict(input_fn=test_input_fn)
-  y_predicted = np.array(list(p['class_ids'] for p in predictions))
-  y_predicted = y_predicted.reshape(np.array(y_test).shape)
-
-  # Score with sklearn.
-  score = metrics.accuracy_score(y_test, y_predicted)
-  print('Accuracy (sklearn): {0:f}'.format(score))
-
-  # Score with tensorflow.
+  train_input_fn = input_fn(IRIS_TRAINING, num_training_data, batch_size=32,
+                            is_training=True)
+  classifier.train(input_fn=train_input_fn, steps=400)
+
+  # Eval.
+  test_input_fn = input_fn(IRIS_TEST, num_test_data, batch_size=32,
+                           is_training=False)
   scores = classifier.evaluate(input_fn=test_input_fn)
   print('Accuracy (tensorflow): {0:f}'.format(scores['accuracy']))
 
-- 
GitLab


From ff0530067435fea5c51605c2e7dfd55f6fe8dfe1 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 20 Oct 2017 11:06:03 -0700
Subject: [PATCH 613/909] Avoid silent variable sharing with ResourceVariable
 class.

PiperOrigin-RevId: 172905986
---
 tensorflow/contrib/eager/python/BUILD         |  2 +-
 tensorflow/contrib/eager/python/saver_test.py | 13 +++-----
 tensorflow/python/eager/backprop_test.py      |  5 +--
 tensorflow/python/eager/function_test.py      |  4 +--
 .../resource_variable_ops_test.py             | 32 ++++++++++++++-----
 .../python/ops/resource_variable_ops.py       | 16 ++++++++++
 tensorflow/python/training/saver_test.py      |  3 +-
 7 files changed, 53 insertions(+), 22 deletions(-)

diff --git a/tensorflow/contrib/eager/python/BUILD b/tensorflow/contrib/eager/python/BUILD
index 702136e3e4..ace17424fe 100644
--- a/tensorflow/contrib/eager/python/BUILD
+++ b/tensorflow/contrib/eager/python/BUILD
@@ -86,7 +86,7 @@ cuda_py_test(
         "//tensorflow/python:client",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python/eager:graph_callable",
-        "//tensorflow/python:platform_test",
+        "//tensorflow/python/eager:test",
         "//tensorflow/python:variables",
     ],
 )
diff --git a/tensorflow/contrib/eager/python/saver_test.py b/tensorflow/contrib/eager/python/saver_test.py
index 29af2b531f..c89554e6dd 100644
--- a/tensorflow/contrib/eager/python/saver_test.py
+++ b/tensorflow/contrib/eager/python/saver_test.py
@@ -22,6 +22,7 @@ import os
 from tensorflow.contrib.eager.python import saver as _saver
 from tensorflow.python.eager import context
 from tensorflow.python.eager import graph_callable
+from tensorflow.python.eager import test
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
@@ -29,7 +30,6 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import init_ops
 from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.ops import variable_scope
-from tensorflow.python.platform import test
 
 
 class SaverTest(test.TestCase):
@@ -38,7 +38,7 @@ class SaverTest(test.TestCase):
     return '/device:GPU:0' if context.num_gpus() else '/device:CPU:0'
 
   def testBasics(self):
-    with context.eager_mode(), ops.device(self._dev()):
+    with ops.device(self._dev()):
       v1 = resource_variable_ops.ResourceVariable(1.0, name='v1')
       def model():
         return array_ops.constant(2.0) * v1
@@ -55,7 +55,7 @@ class SaverTest(test.TestCase):
       self.assertEqual(v1.read_value().numpy(), 1.0)
 
   def testRestoreOnCreate(self):
-    with context.eager_mode(), ops.device(self._dev()):
+    with ops.device(self._dev()):
       def model(init_val):
         v1 = resource_variable_ops.ResourceVariable(init_val, name='v1')
         return array_ops.constant(1.0) * v1, v1
@@ -71,12 +71,9 @@ class SaverTest(test.TestCase):
           # Value is from checkpoint, but not from argument.
           ret, _ = model(2.0)
           self.assertEqual(ret.numpy(), 1.0)
-          # Create it a second time won't re-assign the checkpoint value.
-          v1_2 = resource_variable_ops.ResourceVariable(3.0, name='v1')
-          self.assertEqual(v1_2.read_value().numpy(), 3.0)
 
   def testRestoreNotFound(self):
-    with context.eager_mode(), ops.device(self._dev()):
+    with ops.device(self._dev()):
       def model(v):
         return array_ops.constant(1.0) * v
 
@@ -92,7 +89,7 @@ class SaverTest(test.TestCase):
           _ = model(resource_variable_ops.ResourceVariable(1.0, name='v2'))
 
   def testSaveRestoreGraphCallable(self):
-    with context.eager_mode(), ops.device(self._dev()):
+    with ops.device(self._dev()):
       @graph_callable.graph_callable(
           [graph_callable.ShapeAndDtype(shape=(), dtype=dtypes.float32)])
       def model(x):
diff --git a/tensorflow/python/eager/backprop_test.py b/tensorflow/python/eager/backprop_test.py
index 7da8eb0c9b..9ba5913c65 100644
--- a/tensorflow/python/eager/backprop_test.py
+++ b/tensorflow/python/eager/backprop_test.py
@@ -292,7 +292,7 @@ class BackpropTest(test.TestCase):
     self.assertEqual(grad.numpy(), 6.0)
 
   def testGradientTapeVariable(self):
-    v = resource_variable_ops.ResourceVariable(1.0)
+    v = resource_variable_ops.ResourceVariable(1.0, name='v')
     with backprop.GradientTape() as g:
       y = v * v
     grad = g.gradient(y, [v])[0]
@@ -457,7 +457,8 @@ class BackpropTest(test.TestCase):
         add_n.append(1)
     context.context().add_post_execution_callback(callback)
 
-    v = resource_variable_ops.ResourceVariable(constant_op.constant(2.0))
+    v = resource_variable_ops.ResourceVariable(constant_op.constant(2.0),
+                                               name='v')
     def fn():
       outputs = []
       for _ in range(20):
diff --git a/tensorflow/python/eager/function_test.py b/tensorflow/python/eager/function_test.py
index a4c351e8c9..33bedb59f3 100644
--- a/tensorflow/python/eager/function_test.py
+++ b/tensorflow/python/eager/function_test.py
@@ -57,7 +57,7 @@ class FunctionTest(test.TestCase):
     self.assertAllEqual(out, math_ops.matmul(t, t).numpy())
 
   def testGraphModeWithGradients(self):
-    v = resource_variable_ops.ResourceVariable(1.0)
+    v = resource_variable_ops.ResourceVariable(1.0, name='v')
 
     @function.defun
     def step():
@@ -156,7 +156,7 @@ class FunctionTest(test.TestCase):
     g(constant_op.constant(1.0))
 
   def testGradientTensorConversionWithDefun(self):
-    three = resource_variable_ops.ResourceVariable(3.0)
+    three = resource_variable_ops.ResourceVariable(3.0, name='v')
 
     @function.defun
     def f(x):
diff --git a/tensorflow/python/kernel_tests/resource_variable_ops_test.py b/tensorflow/python/kernel_tests/resource_variable_ops_test.py
index cf4b61674f..10f9a72c7b 100644
--- a/tensorflow/python/kernel_tests/resource_variable_ops_test.py
+++ b/tensorflow/python/kernel_tests/resource_variable_ops_test.py
@@ -181,7 +181,7 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase):
   @test_util.run_in_graph_and_eager_modes()
   def testInitFnDtype(self):
     v = resource_variable_ops.ResourceVariable(
-        initial_value=lambda: 1, dtype=dtypes.float32)
+        initial_value=lambda: 1, dtype=dtypes.float32, name="var0")
     self.assertEqual(dtypes.float32, v.value().dtype)
 
   @test_util.run_in_graph_and_eager_modes()
@@ -192,26 +192,27 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase):
 
   @test_util.run_in_graph_and_eager_modes()
   def testInitializeAllVariables(self):
-    v = resource_variable_ops.ResourceVariable(1, dtype=dtypes.float32)
+    v = resource_variable_ops.ResourceVariable(1, dtype=dtypes.float32,
+                                               name="var0")
     self.evaluate(variables.global_variables_initializer())
     self.assertEqual(1.0, self.evaluate(v.value()))
 
   @test_util.run_in_graph_and_eager_modes()
   def testOperatorOverload(self):
-    v = resource_variable_ops.ResourceVariable(1.0)
+    v = resource_variable_ops.ResourceVariable(1.0, name="var0")
     self.evaluate(variables.global_variables_initializer())
     self.assertEqual(2.0, self.evaluate(v + v))
 
   @test_util.run_in_graph_and_eager_modes()
   def testAssignMethod(self):
-    v = resource_variable_ops.ResourceVariable(1.0)
+    v = resource_variable_ops.ResourceVariable(1.0, name="var0")
     self.evaluate(variables.global_variables_initializer())
     self.evaluate(v.assign(2.0))
     self.assertEqual(2.0, self.evaluate(v.value()))
 
   @test_util.run_in_graph_and_eager_modes()
   def testLoad(self):
-    v = resource_variable_ops.ResourceVariable(1.0)
+    v = resource_variable_ops.ResourceVariable(1.0, name="var0")
     self.evaluate(variables.global_variables_initializer())
     v.load(2.0)
     self.assertEqual(2.0, self.evaluate(v.value()))
@@ -237,21 +238,21 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase):
 
   @test_util.run_in_graph_and_eager_modes()
   def testAssignAddMethod(self):
-    v = resource_variable_ops.ResourceVariable(1.0)
+    v = resource_variable_ops.ResourceVariable(1.0, name="var0")
     self.evaluate(variables.global_variables_initializer())
     self.evaluate(v.assign_add(1.0))
     self.assertEqual(2.0, self.evaluate(v.value()))
 
   @test_util.run_in_graph_and_eager_modes()
   def testAssignSubMethod(self):
-    v = resource_variable_ops.ResourceVariable(3.0)
+    v = resource_variable_ops.ResourceVariable(3.0, name="var0")
     self.evaluate(variables.global_variables_initializer())
     self.evaluate(v.assign_sub(1.0))
     self.assertEqual(2.0, self.evaluate(v.value()))
 
   @test_util.run_in_graph_and_eager_modes()
   def testDestroyResource(self):
-    v = resource_variable_ops.ResourceVariable(3.0)
+    v = resource_variable_ops.ResourceVariable(3.0, name="var0")
     self.evaluate(variables.global_variables_initializer())
     self.assertEqual(3.0, self.evaluate(v.value()))
     self.evaluate(resource_variable_ops.destroy_resource_op(v.handle))
@@ -443,6 +444,21 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase):
         resource_variable_ops.destroy_resource_op(var._handle,
                                                   ignore_lookup_error=False)
 
+  def testSharingViaResourceVariableObject(self):
+    with context.eager_mode():
+      _ = resource_variable_ops.ResourceVariable(1.0, name="var0")
+      with self.assertRaisesRegexp(ValueError,
+                                   "'var0' already created"):
+        _ = resource_variable_ops.ResourceVariable(2.0, name="var0")
+      with ops.Graph().as_default():
+        _ = resource_variable_ops.ResourceVariable(2.0, name="var0")
+
+  def testVariableNameMissing(self):
+    with context.eager_mode():
+      with self.assertRaisesRegexp(ValueError,
+                                   "Variables need to have explicit names"):
+        _ = resource_variable_ops.ResourceVariable(1.0)
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/ops/resource_variable_ops.py b/tensorflow/python/ops/resource_variable_ops.py
index aa45752a9d..c94ddb0627 100644
--- a/tensorflow/python/ops/resource_variable_ops.py
+++ b/tensorflow/python/ops/resource_variable_ops.py
@@ -49,6 +49,16 @@ def _eager_safe_variable_handle(shape, dtype, shared_name, name, graph_mode):
                                                    container=container)
   if graph_mode:
     return handle
+
+  # We do not want two distinct ResourceVariable objects for the same
+  # underlying resource in the runtime.
+  # When in eager mode, explicitly ensure so here. When in graph mode, it's
+  # ensured by always generating different variable names.
+  exists = gen_resource_variable_ops.var_is_initialized_op(handle)
+  if exists:
+    raise ValueError("variable object with name '%s' already created. Use "
+                     "get_variable() if reuse is desired." %
+                     shared_name)
   with context.graph_mode(), ops.Graph().as_default():
     h = gen_resource_variable_ops.var_handle_op(shape=shape, dtype=dtype,
                                                 shared_name=shared_name,
@@ -273,6 +283,12 @@ class ResourceVariable(variables.Variable):
     # Save the graph's container prefix for error checking. Reading the value of
     # the ResourceVariable from another Graph in Eager mode is an error.
     self._container_prefix = ops.get_default_graph()._container_prefix  # pylint: disable=protected-access
+    if not self._in_graph_mode and not name:
+      # TODO(ashankar,josh11b): make this unnecessary using the same
+      # logic as in layer
+      raise ValueError("Variables need to have explicit names when eager "
+                       "execution is enabled")
+
     with ops.control_dependencies(None):
       with ops.name_scope(name, "Variable", []
                           if init_from_fn else [initial_value]) as name:
diff --git a/tensorflow/python/training/saver_test.py b/tensorflow/python/training/saver_test.py
index aeb8eaffe8..4abff1d106 100644
--- a/tensorflow/python/training/saver_test.py
+++ b/tensorflow/python/training/saver_test.py
@@ -233,7 +233,8 @@ class SaverTest(test.TestCase):
   def testResourceSaveRestoreCachingDevice(self):
     save_path = os.path.join(self.get_temp_dir(), "resource_cache")
     with self.test_session(graph=ops_lib.Graph()) as sess:
-      v = resource_variable_ops.ResourceVariable([1], caching_device="/cpu:0")
+      v = resource_variable_ops.ResourceVariable([1], caching_device="/cpu:0",
+                                                 name="v")
       if context.in_graph_mode():
         self.evaluate(variables.global_variables_initializer())
       else:
-- 
GitLab


From 017a5021a7fdc713357fceecf31068ae5090afaf Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 20 Oct 2017 11:13:03 -0700
Subject: [PATCH 614/909] [XLA:CPU] Do not assign parallel tasks to
 instructions which forward pointers (GetTupleElement and Bitcast), because
 the process of outlining the instruction into a parallel computation forces
 the pointed-to buffer to be materialized.

PiperOrigin-RevId: 172907063
---
 tensorflow/compiler/xla/service/cpu/parallel_task_assignment.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tensorflow/compiler/xla/service/cpu/parallel_task_assignment.cc b/tensorflow/compiler/xla/service/cpu/parallel_task_assignment.cc
index 5afb2e67ff..c2213c8f2e 100644
--- a/tensorflow/compiler/xla/service/cpu/parallel_task_assignment.cc
+++ b/tensorflow/compiler/xla/service/cpu/parallel_task_assignment.cc
@@ -136,6 +136,8 @@ int64 ParallelTaskAssignment::GetTargetParallelTaskCount(
       instruction->opcode() == HloOpcode::kCall ||
       instruction->opcode() == HloOpcode::kCustomCall ||
       instruction->opcode() == HloOpcode::kSelectAndScatter ||
+      instruction->opcode() == HloOpcode::kGetTupleElement ||
+      instruction->opcode() == HloOpcode::kBitcast ||
       (instruction->opcode() == HloOpcode::kConvolution &&
        PotentiallyImplementedAsEigenConvolution(*instruction)) ||
       PotentiallyImplementedAsEigenDot(*instruction) ||
-- 
GitLab


From 86908c30c4c0adf92fa14ed6f1d92616177c1b89 Mon Sep 17 00:00:00 2001
From: Jianwei Xie <xiejw@google.com>
Date: Fri, 20 Oct 2017 11:13:45 -0700
Subject: [PATCH 615/909] Step 1: Large refactoring toward wrapping input_fn
 and TPU infeed into tf.while_loop

PiperOrigin-RevId: 172907182
---
 .../contrib/tpu/python/tpu/tpu_estimator.py   | 1313 +++++++++--------
 1 file changed, 664 insertions(+), 649 deletions(-)

diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
index 04e0719a1b..805de16468 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
@@ -20,6 +20,7 @@ from __future__ import division
 from __future__ import print_function
 
 import collections
+from contextlib import contextmanager
 import copy
 import threading
 import six
@@ -38,6 +39,7 @@ from tensorflow.core.protobuf import config_pb2
 from tensorflow.python.estimator import estimator as estimator_lib
 from tensorflow.python.estimator import model_fn as model_fn_lib
 from tensorflow.python.estimator import util
+from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
@@ -57,12 +59,15 @@ from tensorflow.python.training import training_util
 
 _INITIAL_LOSS = 1e7
 _ZERO_LOSS = 0.
-_DEFAULT_NAME_SCOPE = 'tpu_estimator'
+_TPU_ESTIMATOR = 'tpu_estimator'
 _ITERATIONS_PER_LOOP_VAR = 'iterations_per_loop'
 _BATCH_SIZE_KEY = 'batch_size'
 _CROSS_REPLICA_SUM_OP = 'CrossReplicaSum'
 _RESERVED_PARAMS_KEYS = [_BATCH_SIZE_KEY]
 
+# TODO(b/65703635): Flip the value and remove all dead code.
+_WRAP_INPUT_FN_INTO_WHILE_LOOP = False
+
 
 def _create_global_step(graph):
   graph = graph or ops.get_default_graph()
@@ -81,17 +86,25 @@ def _create_global_step(graph):
                      ops.GraphKeys.GLOBAL_STEP])
 
 
-def _create_iterations_per_loop():
-  with variable_scope.variable_scope(_DEFAULT_NAME_SCOPE,
-                                     reuse=variable_scope.AUTO_REUSE):
-    return variable_scope.get_variable(
-        _ITERATIONS_PER_LOOP_VAR,
-        initializer=init_ops.zeros_initializer(),
-        shape=[],
-        dtype=dtypes.int32,
-        trainable=False,
-        collections=[],
-        use_resource=True)
+def _create_or_get_iterations_per_loop():
+  graph = ops.get_default_graph()
+  iter_vars = graph.get_collection(_TPU_ESTIMATOR)
+  if len(iter_vars) == 1:
+    return iter_vars[0]
+  elif len(iter_vars) > 1:
+    raise RuntimeError('Multiple iterations_per_loop_var in collection.')
+
+  with ops.colocate_with(training_util.get_global_step()):
+    with variable_scope.variable_scope(_TPU_ESTIMATOR,
+                                       reuse=variable_scope.AUTO_REUSE):
+      return variable_scope.get_variable(
+          _ITERATIONS_PER_LOOP_VAR,
+          initializer=init_ops.zeros_initializer(),
+          shape=[],
+          dtype=dtypes.int32,
+          trainable=False,
+          collections=[_TPU_ESTIMATOR],
+          use_resource=True)
 
 
 def _sync_variables_ops():
@@ -127,64 +140,209 @@ _DEFAULT_COORDINATOR_JOB_NAME = 'coordinator'
 _LOCAL_MASTERS = ('', 'local')
 
 
-def _tpu_job(run_config, mode):
-  """Returns the job name to use to place TPU computations on.
-
-  Args:
-    run_config: The tpu_config.RunConfig used for this custom estimator.
-    mode: A model_fn_lib.ModeKeys value.
+class _TPUContext(object):
+  """A context holds immutable states of TPU computation.
 
-  Returns:
-    A string containing the job name, or None if no job should be specified.
+  This immutable object holds TPUEstimator config, train/eval batch size, and
+  `TPUEstimator.use_tpu`, which is expected to be passed around. It also
+  provides utility functions, basded on the current state, to determine other
+  information commonly required by TPU computation, such as TPU device names,
+  TPU hosts, shard batch size, etc.
 
-  Raises:
-    ValueError: If the user needs to specify a tpu_job_name, because we are
-      unable to infer the job name automatically, or if the user-specified job
-      names are inappropriate.
+  N.B. As `mode` is not immutable state in Estimator, but essential to
+  distinguish between TPU training and evaluation, a common usage for
+  _TPUContext with `mode` is as follows:
+  ```
+  with _ctx.with_mode(mode) as ctx:
+    if ctx.is_running_on_cpu():
+       ...
+  ```
   """
-  # If the user specifies the tpu_job_name, use that.
-  if run_config.tpu_config.tpu_job_name:
-    return run_config.tpu_config.tpu_job_name
-
-  # The tpu job is determined by the run_config. Right now, this method is
-  # required as tpu_config is not part of the RunConfig.
-  master = (run_config.evaluation_master if mode == model_fn_lib.ModeKeys.EVAL
-            else run_config.master)
-  if master in _LOCAL_MASTERS:
-    return None
-
-  if (not run_config.session_config or
-      not run_config.session_config.cluster_def.job):
-    return _DEFAULT_JOB_NAME
-  cluster_def = run_config.session_config.cluster_def
-  job_names = set([job.name for job in cluster_def.job])
-  if _DEFAULT_JOB_NAME in job_names:
-    # b/37868888 tracks allowing ClusterSpec propagation to reuse job names.
-    raise ValueError('Currently, tpu_worker is not an allowed job name.')
-  if len(job_names) == 1:
-    return cluster_def.job[0].name
-  if len(job_names) == 2:
-    if _DEFAULT_COORDINATOR_JOB_NAME in job_names:
-      job_names.remove(_DEFAULT_COORDINATOR_JOB_NAME)
-      return job_names.pop()
-    # TODO(b/67716447): Include more sophisticated heuristics.
-  raise ValueError(
-      'Could not infer TPU job name. Please specify a tpu_job_name as part of '
-      'your TPUConfig.')
-
-
-def _is_running_on_cpu(use_tpu, mode, eval_batch_size):
-  """Determines whether the input_fn and model_fn should be invoked on CPU."""
-  return ((not use_tpu) or mode == model_fn_lib.ModeKeys.PREDICT or
-          (mode == model_fn_lib.ModeKeys.EVAL and eval_batch_size is None))
-
-
-def _per_shard_batch_size(global_batch_size, run_config, use_tpu):
-  """Returns the batch size for each shard."""
-  if use_tpu:
-    return global_batch_size // run_config.tpu_config.num_shards
-  else:
-    return global_batch_size
+
+  def __init__(self, config, train_batch_size, eval_batch_size, use_tpu):
+    self._config = config
+    self._train_batch_size = train_batch_size
+    self._eval_batch_size = eval_batch_size
+    self._use_tpu = use_tpu
+    self._num_shards_or_none = self._config.tpu_config.num_shards
+    self._mode = None
+
+  def _assert_mode(self):
+    if self._mode is None:
+      raise RuntimeError(
+          '`mode` needs to be set via contextmanager `with_mode`.')
+    return self._mode
+
+  @property
+  def num_of_cores_per_host(self):
+    num_cores = self.num_cores
+    return min(num_cores, 8)
+
+  @contextmanager
+  def with_mode(self, mode):
+    new_ctx = copy.copy(self)  # Shallow copy is enough.
+    new_ctx._mode = mode  # pylint: disable=protected-access
+    yield new_ctx
+
+  @property
+  def mode(self):
+    return self._assert_mode()
+
+  @property
+  def num_cores(self):
+    # TODO(xiejw): Adds lazy num_shards initialization.
+    return self._num_shards_or_none
+
+  @property
+  def num_hosts(self):
+    return self.num_cores // self.num_of_cores_per_host
+
+  @property
+  def config(self):
+    return self._config
+
+  def is_input_sharded_per_core(self):
+    """Return true if input_fn is invoked per-core (other than per-host)."""
+    self._assert_mode()
+    return (self._mode == model_fn_lib.ModeKeys.TRAIN and
+            not self._config.tpu_config.per_host_input_for_training)
+
+  def is_running_on_cpu(self):
+    """Determines whether the input_fn and model_fn should be invoked on CPU."""
+    mode = self._assert_mode()
+    return ((not self._use_tpu) or mode == model_fn_lib.ModeKeys.PREDICT or
+            (mode == model_fn_lib.ModeKeys.EVAL and
+             self._eval_batch_size is None))
+
+  @property
+  def batch_size_for_input_fn(self):
+    """Returns the shard batch size for `input_fn`."""
+    mode = self._assert_mode()
+    # Special case for eval.
+    if mode == model_fn_lib.ModeKeys.EVAL and self._eval_batch_size is None:
+      return None
+    if self.is_running_on_cpu():
+      if mode == model_fn_lib.ModeKeys.TRAIN:
+        return self._train_batch_size
+      if mode == model_fn_lib.ModeKeys.EVAL:
+        return self._eval_batch_size
+      return None
+
+    global_batch_size = (self._train_batch_size if
+                         mode == model_fn_lib.ModeKeys.TRAIN
+                         else self._eval_batch_size)
+    # On TPU
+    return (global_batch_size // self.num_cores
+            if self.is_input_sharded_per_core() else global_batch_size)
+
+  @property
+  def batch_size_for_model_fn(self):
+    """Returns the shard batch size for `model_fn`."""
+    mode = self._assert_mode()
+    # Special case for eval.
+    if mode == model_fn_lib.ModeKeys.EVAL and self._eval_batch_size is None:
+      return None
+    if self.is_running_on_cpu():
+      if mode == model_fn_lib.ModeKeys.TRAIN:
+        return self._train_batch_size
+      if mode == model_fn_lib.ModeKeys.EVAL:
+        return self._eval_batch_size
+      return None
+
+    # On TPU. always sharded per core.
+    if mode == model_fn_lib.ModeKeys.TRAIN:
+      return self._train_batch_size // self.num_cores
+    else:
+      return self._eval_batch_size // self.num_cores
+
+  @property
+  def master_job(self):
+    """Returns the job name to use to place TPU computations on.
+
+    Returns:
+      A string containing the job name, or None if no job should be specified.
+
+    Raises:
+      ValueError: If the user needs to specify a tpu_job_name, because we are
+        unable to infer the job name automatically, or if the user-specified job
+        names are inappropriate.
+    """
+    run_config = self._config
+    # If the user specifies the tpu_job_name, use that.
+    if run_config.tpu_config.tpu_job_name:
+      return run_config.tpu_config.tpu_job_name
+
+    # The tpu job is determined by the run_config. Right now, this method is
+    # required as tpu_config is not part of the RunConfig.
+    mode = self._assert_mode()
+    master = (run_config.evaluation_master if mode == model_fn_lib.ModeKeys.EVAL
+              else run_config.master)
+    if master in _LOCAL_MASTERS:
+      return None
+
+    if (not run_config.session_config or
+        not run_config.session_config.cluster_def.job):
+      return _DEFAULT_JOB_NAME
+    cluster_def = run_config.session_config.cluster_def
+    job_names = set([job.name for job in cluster_def.job])
+    if _DEFAULT_JOB_NAME in job_names:
+      # b/37868888 tracks allowing ClusterSpec propagation to reuse job names.
+      raise ValueError('Currently, tpu_worker is not an allowed job name.')
+    if len(job_names) == 1:
+      return cluster_def.job[0].name
+    if len(job_names) == 2:
+      if _DEFAULT_COORDINATOR_JOB_NAME in job_names:
+        job_names.remove(_DEFAULT_COORDINATOR_JOB_NAME)
+        return job_names.pop()
+      # TODO(b/67716447): Include more sophisticated heuristics.
+    raise ValueError(
+        'Could not infer TPU job name. Please specify a tpu_job_name as part '
+        'of your TPUConfig.')
+
+  @property
+  def tpu_host_placement_function(self):
+    """Returns the TPU host place function."""
+    master = self.master_job
+    def _placement_function(_sentinal=None, core_id=None, host_id=None):  # pylint: disable=invalid-name
+      assert _sentinal is None
+      if core_id is not None and host_id is not None:
+        raise RuntimeError(
+            'core_id and host_id can have only one non-None value.')
+
+      if master is None:
+        return '/replica:0/task:0/device:CPU:0'
+      else:
+        # This assumes that if using more than 8 shards,
+        # the job configuration varies 'task'.
+        if core_id is not None:
+          host_id = core_id / 8
+        return '/job:%s/task:%d/device:CPU:0' % (master, host_id)
+    return _placement_function
+
+  @property
+  def tpu_device_placement_function(self):
+    master = self.master_job
+    job_device = '' if master is None else ('/job:%s' % master)
+    def _placement_function(i):
+      return '%s/task:%d/device:TPU:%d' % (job_device, i / 8, i % 8)
+    return _placement_function
+
+  @property
+  def tpu_ordinal_function(self):
+    """Returns the TPU ordinal fn."""
+    def _tpu_ordinal_function(index):
+      """Return the TPU ordinal associated with a shard.
+
+      Required because the enqueue ops are placed on CPU.
+
+      Args:
+        index: the shard index
+
+      Returns:
+        The ordinal of the TPU device the shard's infeed should be placed on.
+      """
+      return index % 8
+    return _tpu_ordinal_function
 
 
 class _SIGNAL(object):
@@ -319,11 +477,16 @@ class _InfeedThreadController(_InfeedOutfeedThreadBaseController):
           logging.info('Stop Infeed input thread.')
           return
 
-        iterations = signal
-        for i in range(iterations):
-          logging.debug('Infeed enqueue for iteration (%d, %d)', count, i)
+        if _WRAP_INPUT_FN_INTO_WHILE_LOOP:
+          # Enqueue batches for next loop.
           session.run(enqueue_ops)
-        count += 1
+        else:
+          iterations = signal
+          for i in range(iterations):
+            logging.debug('Infeed enqueue for iteration (%d, %d)', count, i)
+            session.run(enqueue_ops)
+          count += 1
+
     except Exception:  # pylint: disable=broad-except
       logging.error(
           'Failed running infeed, closing session.\n'
@@ -346,17 +509,16 @@ class TPUInfeedOutfeedSessionHook(session_run_hook.SessionRunHook):
      dequeue.
   """
 
-  def __init__(self, run_config, mode, enqueue_fn, dequeue_ops=None):
-    self._tpu_job = _tpu_job(run_config, mode)
-    self._enqueue_fn = enqueue_fn
+  def __init__(self, ctx, enqueue_ops, dequeue_ops=None):
+    self._master_job = ctx.master_job
+    self._enqueue_ops = enqueue_ops
     self._dequeue_ops = dequeue_ops
 
   def begin(self):
-    self._enqueue_ops = self._enqueue_fn()
-    self._iterations_per_loop_var = _create_iterations_per_loop()
-    logging.info('TPU job name %s', self._tpu_job)
-    self._init_op = [tpu.initialize_system(job=self._tpu_job)]
-    self._finalize_op = [tpu.shutdown_system(job=self._tpu_job)]
+    logging.info('TPU job name %s', self._master_job)
+    self._iterations_per_loop_var = _create_or_get_iterations_per_loop()
+    self._init_op = [tpu.initialize_system(job=self._master_job)]
+    self._finalize_op = [tpu.shutdown_system(job=self._master_job)]
 
   def after_create_session(self, session, coord):
     logging.info('Init TPU system')
@@ -378,6 +540,7 @@ class TPUInfeedOutfeedSessionHook(session_run_hook.SessionRunHook):
     iterations = run_context.session.run(self._iterations_per_loop_var)
     self._infeed_thd_controller.send_next_batch_signal(iterations)
     if self._dequeue_ops is not None:
+      # TODO(xiejw): Refactor the outfeed dequeue into tf.while_loop.
       logging.info('Dequeue next batch of data from outfeed.')
       self._outfeed_thd_controller.send_next_batch_signal(iterations)
 
@@ -439,7 +602,7 @@ class _TPUStopAtStepHook(session_run_hook.SessionRunHook):
     if self._global_step_tensor is None:
       raise RuntimeError('Global step should be created.')
 
-    self._iterations_per_loop_var = _create_iterations_per_loop()
+    self._iterations_per_loop_var = _create_or_get_iterations_per_loop()
 
   def after_create_session(self, session, coord):
     global_step = session.run(self._global_step_tensor)
@@ -474,360 +637,288 @@ class _SetEvalIterationsHook(session_run_hook.SessionRunHook):
     self._num_steps = num_steps
 
   def begin(self):
-    self._iterations_per_loop_var = _create_iterations_per_loop()
+    self._iterations_per_loop_var = _create_or_get_iterations_per_loop()
 
   def after_create_session(self, session, coord):
     self._iterations_per_loop_var.load(self._num_steps, session=session)
 
 
-class _PerShardOutput(object):
-  """Wraps input_fn's outputs into per-shard outputs.
-
-  Used so that the model_fn can distinguish between sharded input and unsharded
-  inputs (e.g., for export_savedmodel()).
-  """
-
-  def __init__(self, output):
-    self.output = output
-
-  def as_list(self):
-    return self.output
-
+def generate_per_core_enqueue_ops_fn_for_host(
+    ctx, input_fn, inputs_structure_recorder):
+  """Generates infeed enqueue ops for per-core input_fn on a single host."""
+  infeed_queue_holder = {'instance': None}
+
+  def enqueue_ops_fn():
+    """A fn returns enqueue_ops."""
+    num_cores_per_host = ctx.num_of_cores_per_host
+    per_host_sharded_inputs = []
+    for core_ordinal in range(num_cores_per_host):
+      with ops.name_scope('ordinal_%d' % (core_ordinal)):
+        inputs = input_fn()
+        if isinstance(inputs, tuple):
+          features, labels = inputs
+        else:
+          features, labels = inputs, None
 
-class _InputsHolder(object):
-  """A inputs holder holds the `features` and `labels' for TPU system.
+        inputs_structure_recorder.validate_and_record_structure(
+            features, labels)
+        flattened_inputs = (
+            inputs_structure_recorder.flatten_features_and_labels(
+                features, labels))
+        per_host_sharded_inputs.append(flattened_inputs)
 
-  Model inputs returned by the `input_fn` can have one of the following forms:
+    infeed_queue = tpu_feed.InfeedQueue(
+        number_of_tuple_elements=len(per_host_sharded_inputs[0]))
+    infeed_queue_holder['instance'] = infeed_queue
+    infeed_queue.set_configuration_from_sharded_input_tensors(
+        per_host_sharded_inputs)
+
+    per_host_enqueue_ops = infeed_queue.generate_enqueue_ops(
+        per_host_sharded_inputs,
+        tpu_ordinal_function=ctx.tpu_ordinal_function)
+    return per_host_enqueue_ops
+  return enqueue_ops_fn, (lambda: infeed_queue_holder['instance'])
+
+
+class _InputPipeline(object):
+  """`_InputPipeline` handles invoking `input_fn` and piping to infeed queue.
+
+  `_InputPipeline` abstracts the per-core/per-host `input_fn` invocation from
+  call site.  To be precise, based on the configuration in `_TPUContext`,  it
+  invokes `input_fn` for all cores (usually multi-host TPU training) or for one
+  host (usually for single-host TPU evaluation), and sends all `features` and
+  `labels` returned by `input_fn` to TPU infeed. For per-core invocation,
+  `features` and `labels` are piped to infeed directly, one tuple for each
+  core. For per-host invocation,  `features` and `labels` are split at host
+  (with respect to `batch_axis`) and piped to all cores accordingly.
+
+  In addition, flatten/unflatten are handled by `_InputPipeline` also.  Model
+  inputs returned by the `input_fn` can have one of the following forms:
   1. features
   2. (features, labels)
 
   Internally, form 1 is reformed to `(features, None)` as features and labels
   are passed separatedly to underlying methods. For TPU training, TPUEstimator
-  expects multiple `features` and `labels` tuples one for each shard.
-
-  In addition, TPUEstimator allows various different structures for inputs
-  (namely `features` and `labels`).  `features` can be `Tensor` or dict of
-  string name to `Tensor`, and `labels` could be `None`, `Tensor`, or dict of
-  string name to `Tensor`. TPU infeed/outfeed library expects flattened tensor
-  list. So, `features` and `labels` need to be flattened, before infeed enqueue,
-  and the structure of them needs to be recorded, in order to restore them after
-  infeed dequeue.
-
-  `_InputsHolder` could hold the `features` and `labels` tuple for all shards
-  (usually multi-host TPU training) or for one host (usually for single-host TPU
-  evaluation), records the structure details (including presence, dict or single
-  tensor, dict names), validates the structure consistency cross all shards, and
-  encapsulates the flatten/unflatten logic.
+  may expect multiple `features` and `labels` tuples one for each core.
+
+  TPUEstimator allows various different structures for inputs (namely `features`
+  and `labels`).  `features` can be `Tensor` or dict of string name to `Tensor`,
+  and `labels` could be `None`, `Tensor`, or dict of string name to `Tensor`.
+  TPU infeed/outfeed library expects flattened tensor list. So, `features` and
+  `labels` need to be flattened, before infeed enqueue, and the structure of
+  them needs to be recorded, in order to restore them after infeed dequeue.
   """
 
-  def __init__(self, features=None, labels=None, num_shards=None):
-    """Constructor.
-
-    Args:
-      features: features for one host or a list of features one for each shard
-        (must be type `_PerShardOutput`). Once provided, the corresponding
-        `labels` should be set also and this `_InputsHolder` is frozen to
-        prevent from future modification. If `None`, it is expected to add
-        features and labels for each shard by calling `append_tuple` later.
-      labels: labels for one host or a list of labels one for each shard
-        (must be type `_PerShardOutput`).
-      num_shards: Number of shards in the TPU system. Must be provided unless it
-        can be deduced from `features`.
-
-    Raises:
-      ValueError: If both `sharded_features` and `num_shards` are `None`.
-    """
-    # Holds the features and labels for all shards.
-    self._feature_list = []
-    self._label_list = []
-
-    # Holds the structure of inputs
-    self._feature_names = []
-    self._label_names = []
-    self._has_labels = False
-
-    # Internal state.
-    self._initialized = False
-    self._frozen = False
-    self._sharded = False
-
-    if features is None:
-      if num_shards is None:
-        raise ValueError(
-            '`features` and `num_shards` cannot be both None')
-      self._num_shards = num_shards
-    elif isinstance(features, _PerShardOutput):
-      self._from_sharded_inputs(features, labels, num_shards)
-    else:
-      if num_shards is None:
-        raise ValueError(
-            '`num_shards` cannot be None for unsharded features.')
-      self._from_unsharded_inputs(features, labels, num_shards)
-
-  def _from_unsharded_inputs(self, features, labels, num_shards):
-    """Initializes the inputs with unsharded features and labels."""
-    self._num_shards = num_shards
-    if labels is not None:
-      self._has_labels = True
-      self.append_tuple((features, labels))
-    else:
-      self.append_tuple(features)
-
-    self._sharded = False
-    self._frozen = True
-
-  def _from_sharded_inputs(self, sharded_features, sharded_labels, num_shards):
-    """Initializes the inputs with sharded features and labels."""
-    if not isinstance(sharded_features, _PerShardOutput):
-      raise ValueError('`sharded_features` must have type `_PerShardOutput`.')
-    features = sharded_features.as_list()
-
-    if num_shards is not None and num_shards != len(features):
-      raise ValueError(
-          '`num_shards` should be same as the length of sharded_features.')
+  class InputsStructureRecorder(object):
+    """The recorder to record inputs structure."""
+
+    def __init__(self):
+      # Holds the structure of inputs
+      self._feature_names = []
+      self._label_names = []
+      self._has_labels = False
+
+      # Internal state.
+      self._initialized = False
+
+    def has_labels(self):
+      return self._has_labels
+
+    def validate_and_record_structure(self, features, labels):
+      """Validates and records the structure of features` and `labels`."""
+      def _extract_key_names(tensor_or_dict):
+        if tensor_or_dict is None:
+          return []
+        return tensor_or_dict.keys() if isinstance(tensor_or_dict, dict) else []
+
+      # Extract structure.
+      has_labels = labels is not None
+      feature_names = _extract_key_names(features)
+      label_names = _extract_key_names(labels)
+
+      if self._initialized:
+        # Verify the structure is same. The following should never happen.
+        assert feature_names == self._feature_names, 'feature keys mismatched'
+        assert label_names == self._label_names, 'label keys mismatched'
+        assert has_labels == self._has_labels, 'label presence mismatched'
+      else:
+        # Record structure.
+        self._initialized = True
+        self._feature_names = feature_names
+        self._label_names = label_names
+        self._has_labels = has_labels
+
+    def flatten_features_and_labels(self, features, labels):
+      """Flattens the `features` and `labels` to a single tensor list."""
+      flattened_inputs = []
+      if self._feature_names:
+        # We need a fixed ordering for enqueueing and dequeueing.
+        flattened_inputs.extend([features[name]
+                                 for name in self._feature_names])
+      else:
+        flattened_inputs.append(features)
 
-    self._num_shards = len(features)
-    if not self._num_shards:
-      raise ValueError('`sharded_features` should not be empty.')
+      if labels is not None:
+        if self._label_names:
+          # We need a fixed ordering for enqueueing and dequeueing.
+          flattened_inputs.extend([labels[name] for name in self._label_names])
+        else:
+          flattened_inputs.append(labels)
+      return flattened_inputs
+
+    def unflatten_features_and_labels(self, flattened_inputs):
+      """Restores the flattened inputs to original features and labels form.
+
+      Args:
+        flattened_inputs: Flattened inputs for each shard.
+
+      Returns:
+        A tuple of (`features`, `labels`), where `labels` could be None.
+        Each one, if present, should have identical structure (single tensor vs
+        dict) as the one returned by input_fn.
+
+      Raises:
+        ValueError: If the number of expected tensors from `flattened_inputs`
+          mismatches the recorded structure.
+      """
+      expected_num_features = (len(self._feature_names) if self._feature_names
+                               else 1)
+      if self._has_labels:
+        expected_num_labels = (len(self._label_names) if self._label_names
+                               else 1)
+      else:
+        expected_num_labels = 0
 
-    if sharded_labels is not None:
-      if not isinstance(sharded_labels, _PerShardOutput):
-        raise ValueError('sharded_labels` must have type `_PerShardOutput`.')
+      expected_num_tensors = expected_num_features + expected_num_labels
 
-      self._has_labels = True
-      labels = sharded_labels.as_list()
-      if self._num_shards != len(labels):
+      if expected_num_tensors != len(flattened_inputs):
         raise ValueError(
-            'Length of `sharded_features` and `sharded_labels` mismatch.')
-
-    if self._has_labels:
-      for (f, l) in zip(features, labels):
-        self.append_tuple((f, l))
-    else:
-      for f in features:
-        self.append_tuple(f)
-
-    self._sharded = True
-    self._frozen = True
-
-  def _extract_key_names(self, tensor_or_dict):
-    if tensor_or_dict is None:
-      return []
-
-    return tensor_or_dict.keys() if isinstance(tensor_or_dict, dict) else []
-
-  def _validate(self, features, labels):
-    has_labels = labels is not None
-    feature_names = self._extract_key_names(features)
-    label_names = self._extract_key_names(labels)
-
-    if self._initialized:
-      self._sharded = True
-      # The following should never happen.
-      assert feature_names == self._feature_names, 'feature keys mismatched'
-      assert label_names == self._label_names, 'label keys mismatched'
-      assert has_labels == self._has_labels, 'label presence mismatched'
-    else:
-      self._initialized = True
-      self._feature_names = feature_names
-      self._label_names = label_names
-      self._has_labels = has_labels
-
-  @property
-  def sharded(self):
-    if not self._frozen:
-      raise RuntimeError('_InputsHolder has not been frozen yet.')
-    return self._sharded
-
-  @property
-  def num_shards(self):
-    if not self._frozen:
-      raise RuntimeError('_InputsHolder has not been frozen yet.')
-    return self._num_shards
-
-  def append_tuple(self, inputs):
-    """Appends `inputs` for one shard into holder.
-
-    Args:
-      inputs: The return from `input_fn`, which could be features or tuple of
-        (features, labels). After the first `inputs` appended into
-        `_InputsHolder`, the structure of `features` and `labels is recorded.
-        Any future invocation should provide the `inputs` with same structure.
-
-    Raises:
-      RuntimeError: If the internal data has been frozen already.
-    """
-    if self._frozen:
-      raise RuntimeError('InputsHolder has frozen, which cannot be mutated.')
-
-    # input_fn may return either features or (features, labels)
-    if isinstance(inputs, tuple):
-      features, labels = inputs
-    else:
-      features, labels = inputs, None
-
-    self._validate(features, labels)
-
-    self._feature_list.append(features)
-    if labels is not None:
-      self._label_list.append(labels)
-
-  def as_features_and_labels_tuple(self):
-    """Returns features and labels as grouped tuple.
-
-    This is intended to be used to pass features and labels for all shards from
-    input_fn to model_fn as the parent class `Estimator` does not have the
-    concept of shards. So, grouped tuple is required.
-
-    Once called, the internal data is frozen and `append_tuple` cannot be
-    invoked anymore.
-
-    Returns:
-      A tuple of features and labels. Both have type `_PerShardOutput`, holding
-      the inputs for all shards. `labels` could be `None`.
-
-    Raises:
-      RuntimeError: If the internal data has not been initialized.
-    """
-    self._frozen = True
-    if not self._initialized:
-      raise RuntimeError('InputsHolder has not been initialized.')
-
-    assert len(self._feature_list) == self._num_shards
-    if not self._label_list or all(l is None for l in self._label_list):
-      return _PerShardOutput(self._feature_list), None
-
-    assert len(self._label_list) == self._num_shards
-    return (_PerShardOutput(self._feature_list),
-            _PerShardOutput(self._label_list))
-
-  def as_sharded_flattened_inputs(self):
-    """Flatten the features and label as tensor lists for all shards.
-
-    Flattened tensor list contains all tensors in `features` (dict) and `labels`
-    (dict). Conceptually, it has the predicated structure like:
-
-    ```python
-    flatten_list = []
-    for name in features:
-      flatten_list.append(features[name])
-    for name in labels:
-      flatten_list.append(labels[name])
-    ```
-
-    This method handles the label is None case and single tensor case nicely.
-
-    Once called, the internal data is frozen and `append_tuple` cannot be
-    invokded anymore.
-
-    Returns:
-      A list of flattened inputs one for each shard.
-
-    Raises:
-      RuntimeError: If the internal data has not been initialized.
-      ValueError: If the inputs are sharded.
-    """
-    self._frozen = True
-    if not self._initialized:
-      raise RuntimeError('InputsHolder has not been initialized.')
-    if not self._sharded:
-      raise ValueError('Inputs are not sharded.')
-
-    sharded_inputs = []
-
-    for shard in range(self._num_shards):
-      flattened_inputs = self._as_flattened_inputs(
-          self._feature_list[shard],
-          self._label_list[shard] if self._has_labels else None)
-      sharded_inputs.append(flattened_inputs)
-
-    return sharded_inputs
-
-  def as_flattened_inputs(self):
-    """Flatten the features and label as a single tensor list for one host."""
-    self._frozen = True
-    if not self._initialized:
-      raise RuntimeError('InputsHolder has not been initialized.')
-    if self._sharded:
-      raise ValueError('Inputs are sharded.')
-
-    return self._as_flattened_inputs(
-        self._feature_list[0],
-        self._label_list[0] if self._has_labels else None)
-
-  def _as_flattened_inputs(self, features, labels):
-    """Flattens the `features` and `labels` to a single tensor list."""
-    flattened_inputs = []
-    if self._feature_names:
-      # We need a fixed ordering for enqueueing and dequeueing.
-      flattened_inputs.extend([features[name] for name in self._feature_names])
-    else:
-      flattened_inputs.append(features)
-
-    if labels is not None:
-      if self._label_names:
-        # We need a fixed ordering for enqueueing and dequeueing.
-        flattened_inputs.extend([labels[name] for name in self._label_names])
+            'The number of flattened tensors mismatches expected num. '
+            'Expected {}, got {}'.format(expected_num_tensors,
+                                         len(flattened_inputs)))
+      if self._feature_names:
+        unflattened_features = dict(
+            zip(self._feature_names, flattened_inputs[:expected_num_features]))
+      else:
+        # Single tensor case
+        unflattened_features = flattened_inputs[0]
+
+      if expected_num_labels == 0:
+        unflattened_label = None
+      elif self._label_names:
+        unflattened_label = dict(zip(self._label_names,
+                                     flattened_inputs[expected_num_features:]))
       else:
-        flattened_inputs.append(labels)
-    return flattened_inputs
+        # Single tensor case.
+        unflattened_label = flattened_inputs[expected_num_features]
 
-  def unflatten_features_and_labels(self, flattened_inputs):
-    """Restores the flattened inputs to original features and labels form.
+      return unflattened_features, unflattened_label
 
-    Once called, the internal data is frozen and `append_tuple` cannot be
-    invokded anymore.
+  def __init__(self, input_fn, batch_axis, ctx):
+    """Constructor.
 
     Args:
-      flattened_inputs: Flattened inputs for one each, which should be created
-      by the `as_sharded_flattened_inputs` API.
-
-    Returns:
-      A tuple of (`features`, `labels`), where `labels` could be None.
-      Each one, if present, should have identical structure (single tensor vs
-      dict) as the one returned by input_fn.
+      input_fn: input fn for train or eval.
+      batch_axis: A python tuple of int values describing how each tensor
+        produced by the Estimator `input_fn` should be split across the TPU
+        compute shards.
+      ctx: A `_TPUContext` instance with mode.
 
     Raises:
-      RuntimeError: If the internal data has not been initialized.
-      ValueError: If the number of expected tensors from `flattened_inputs`
-        mismatches the recorded structure.
+      ValueError: If both `sharded_features` and `num_cores` are `None`.
     """
-    self._frozen = True
-    if not self._initialized:
-      raise RuntimeError('InputsHolder has not been initialized.')
-
-    expected_num_features = (len(self._feature_names) if self._feature_names
-                             else 1)
-    if self._has_labels:
-      expected_num_labels = (len(self._label_names) if self._label_names
-                             else 1)
-    else:
-      expected_num_labels = 0
-
-    expected_num_tensors = expected_num_features + expected_num_labels
+    self._inputs_structure_recorder = _InputPipeline.InputsStructureRecorder()
+
+    self._sharded_per_core = ctx.is_input_sharded_per_core()
+    self._input_fn = input_fn
+    self._infeed_queue = None
+    self._ctx = ctx
+    self._batch_axis = batch_axis
+
+  def generate_infeed_enqueue_ops_and_dequeue_fn(self):
+    """Generates infeed enqueue ops and dequeue_fn."""
+    # While tf.while_loop is called, the body function, which invokes
+    # `enqueue_fn` passed in, is called to construct the graph. So, input_fn
+    # structure is recorded.
+    enqueue_ops = self._invoke_input_fn_and_record_structure()
+
+    def dequeue_fn():
+      """dequeue_fn is used by TPU to retrieve the tensors."""
+      values = self._infeed_queue.generate_dequeue_op()
+      # The unflatten process uses the structure information recorded above.
+      return self._inputs_structure_recorder.unflatten_features_and_labels(
+          values)
+
+    return (enqueue_ops, dequeue_fn)
+
+  def _invoke_input_fn_and_record_structure(self):
+    if self._sharded_per_core:
+      # Per-Core input pipeline deployment.
+      tpu_host_placement_fn = self._ctx.tpu_host_placement_function
+      enqueue_ops = []
+      infeed_queues = []
+
+      # Invoke input pipeline for each core and placed on the corresponding
+      # host.
+      num_hosts = self._ctx.num_hosts
+      for host_id in range(num_hosts):
+        host_device = tpu_host_placement_fn(host_id=host_id)
+        with ops.device(host_device):
+          with ops.name_scope('input_pipeline_task%d' % (host_id)):
+            enqueue_ops_fn, infeed_queue_getter = (
+                generate_per_core_enqueue_ops_fn_for_host(
+                    self._ctx, self._input_fn, self._inputs_structure_recorder))
+
+            if _WRAP_INPUT_FN_INTO_WHILE_LOOP:
+              enqueue_ops.append(_wrap_computation_in_while_loop(
+                  device=host_device, op_fn=enqueue_ops_fn))
+            else:
+              enqueue_ops.append(enqueue_ops_fn())
+            # Infeed_queue_getter must be called after enqueue_ops_fn is called.
+            infeed_queues.append(infeed_queue_getter())
+
+      # infeed_queue is used to generate dequeue ops. The only thing it uses for
+      # dequeue is dtypes and types. So, any one can be used. Here, grab the
+      # first one.
+      self._infeed_queue = infeed_queues[0]
+      return enqueue_ops
 
-    if expected_num_tensors != len(flattened_inputs):
-      raise ValueError(
-          'The number of flattened tensors mismatches expected num. '
-          'Expected {}, got {}'.format(expected_num_tensors,
-                                       len(flattened_inputs)))
-    if self._feature_names:
-      unflattened_features = dict(zip(self._feature_names,
-                                      flattened_inputs[:expected_num_features]))
-    else:
-      # Single tensor case
-      unflattened_features = flattened_inputs[0]
-
-    if expected_num_labels == 0:
-      unflattened_label = None
-    elif self._label_names:
-      unflattened_label = dict(zip(self._label_names,
-                                   flattened_inputs[expected_num_features:]))
     else:
-      # Single tensor case.
-      unflattened_label = flattened_inputs[expected_num_features]
-
-    return unflattened_features, unflattened_label
+      # TODO(b/67051042): Extend this to multi-host support.
+      host_id = 0
+      host_device = self._ctx.tpu_host_placement_function(host_id=host_id)
+      def enqueue_fn():
+        with ops.device(host_device):
+          with ops.name_scope('input_pipeline_task%d' % (host_id)):
+            inputs = self._input_fn()
+            if isinstance(inputs, tuple):
+              features, labels = inputs
+            else:
+              features, labels = inputs, None
+            self._inputs_structure_recorder.validate_and_record_structure(
+                features, labels)
+            unsharded_tensor_list = (
+                self._inputs_structure_recorder.flatten_features_and_labels(
+                    features, labels))
+
+            self._infeed_queue = tpu_feed.InfeedQueue(
+                tuple_types=[t.dtype for t in unsharded_tensor_list],
+                tuple_shapes=[t.shape for t in unsharded_tensor_list],
+                shard_dimensions=self._batch_axis)
+            self._infeed_queue.set_number_of_shards(self._ctx.num_cores)
+
+            def placement_fn(core_id):
+              return self._ctx.tpu_host_placement_function(core_id=core_id)
+            return (
+                self._infeed_queue.split_inputs_and_generate_enqueue_ops(
+                    unsharded_tensor_list,
+                    placement_function=placement_fn))
+
+      if _WRAP_INPUT_FN_INTO_WHILE_LOOP:
+        return _wrap_computation_in_while_loop(device=host_device,
+                                               op_fn=enqueue_fn)
+      else:
+        return enqueue_fn()
 
 
 class _ModelFnWrapper(object):
@@ -840,20 +931,17 @@ class _ModelFnWrapper(object):
   train and eval step.
   """
 
-  def __init__(self, model_fn, config, params, mode, train_batch_size,
-               eval_batch_size):
+  def __init__(self, model_fn, config, params, ctx):
     self._model_fn = model_fn
     self._config = config
     self._params = params
-    self._mode = mode
-    self._train_batch_size = train_batch_size
-    self._eval_batch_size = eval_batch_size
+    self._ctx = ctx
 
   def call_without_tpu(self, features, labels):
     # Let CrossShardOptimizer be called without TPU in model_fn, since it's
     # common to set the train_op even when running evaluate() or predict().
     with tpu_function.tpu_shard_context(1):
-      return self._call_model_fn(features, labels, use_tpu=False)
+      return self._call_model_fn(features, labels)
 
   def convert_to_single_tpu_train_step(self, dequeue_fn):
     """Converts user provided model_fn` as a single train step on TPU.
@@ -883,7 +971,7 @@ class _ModelFnWrapper(object):
       features, labels = dequeue_fn()
 
       estimator_spec = self._verify_estimator_spec(
-          self._call_model_fn(features, labels, use_tpu=True))
+          self._call_model_fn(features, labels))
       loss, train_op = estimator_spec.loss, estimator_spec.train_op
       with ops.control_dependencies([train_op]):
         return array_ops.identity(loss)
@@ -915,13 +1003,13 @@ class _ModelFnWrapper(object):
       A tuple of eval_fn and eval_metrics. The eval_fn representing the eval
       step for TPU. and eval_metrics is an `_EvalMetrics` instance.
     """
-    eval_metrics = _EvalMetrics()
+    eval_metrics = _EvalMetrics(self._ctx)
 
     def eval_step(total_loss):
       """Evaluation step function for use inside a while loop."""
       features, labels = dequeue_fn()
 
-      tpu_estimator_spec = self._call_model_fn(features, labels, use_tpu=True)
+      tpu_estimator_spec = self._call_model_fn(features, labels)
       if not isinstance(tpu_estimator_spec, TPUEstimatorSpec):
         raise RuntimeError(
             'estimator_spec used by TPU evaluation must have type'
@@ -935,11 +1023,7 @@ class _ModelFnWrapper(object):
         return math_ops.add(total_loss, loss)
     return eval_step, eval_metrics
 
-  @property
-  def config(self):
-    return self._config
-
-  def _call_model_fn(self, features, labels, use_tpu):
+  def _call_model_fn(self, features, labels):
     """Calls the model_fn with required parameters."""
     model_fn_args = util.fn_args(self._model_fn)
     kwargs = {}
@@ -950,12 +1034,11 @@ class _ModelFnWrapper(object):
 
     if 'labels' in model_fn_args:
       kwargs['labels'] = labels
-    else:
-      if labels is not None:
-        raise ValueError(
-            'model_fn does not take labels, but input_fn returns labels.')
+    elif labels is not None:
+      raise ValueError(
+          'model_fn does not take labels, but input_fn returns labels.')
     if 'mode' in model_fn_args:
-      kwargs['mode'] = self._mode
+      kwargs['mode'] = self._ctx.mode
     if 'config' in model_fn_args:
       kwargs['config'] = config
     if 'params' in model_fn_args:
@@ -966,16 +1049,16 @@ class _ModelFnWrapper(object):
           'model_fn ({}) does not include params argument, '
           'required by TPUEstimator to pass batch size as '
           'params[\'batch_size\']'.format(self._model_fn))
-    if self._mode == model_fn_lib.ModeKeys.TRAIN:
-      params[_BATCH_SIZE_KEY] = _per_shard_batch_size(
-          self._train_batch_size, config, use_tpu)
-    elif (self._mode == model_fn_lib.ModeKeys.EVAL and
-          self._eval_batch_size is not None):
-      params[_BATCH_SIZE_KEY] = _per_shard_batch_size(
-          self._eval_batch_size, config, use_tpu)
+
+    batch_size_for_model_fn = self._ctx.batch_size_for_model_fn
+    if batch_size_for_model_fn is not None:
+      params[_BATCH_SIZE_KEY] = batch_size_for_model_fn
 
     estimator_spec = self._model_fn(features=features, **kwargs)
-    if (not use_tpu) and isinstance(estimator_spec, TPUEstimatorSpec):
+    if (self._ctx.is_running_on_cpu() and
+        isinstance(estimator_spec, TPUEstimatorSpec)):
+      # The estimator_spec will be passed to `Estimator` directly, which expects
+      # type `EstimatorSpec`.
       return estimator_spec.as_estimator_spec()
     else:
       return estimator_spec
@@ -998,7 +1081,8 @@ class _ModelFnWrapper(object):
 class _EvalMetrics(object):
   """Class wraps TPUEstimator.eval_metrics."""
 
-  def __init__(self):
+  def __init__(self, ctx):
+    self._ctx = ctx
     self._metric_fn = None
     self._is_dict = False
     self._tensor_keys = []
@@ -1081,7 +1165,7 @@ class _EvalMetrics(object):
       raise RuntimeError('Eval metrics have not been recorded yet')
     return self._tensors
 
-  def to_metric_metric_ops_for_tpu(self, run_config, dummy_update_op):
+  def to_metric_metric_ops_for_tpu(self, dummy_update_op):
     """Creates the eval_metric_ops now based on the TPU outfeed.
 
     `eval_metric_ops` is defined in `EstimatorSpec`. From all shards, tensors
@@ -1090,7 +1174,6 @@ class _EvalMetrics(object):
     metric fn.
 
     Args:
-      run_config: A `RunConfig` instance.
       dummy_update_op: A dummy update op.
 
     Returns:
@@ -1102,9 +1185,7 @@ class _EvalMetrics(object):
       RuntimeError: If outfeed tensor is scalar.
     """
 
-    num_shards = run_config.tpu_config.num_shards
-    job = _tpu_job(run_config, model_fn_lib.ModeKeys.EVAL)
-    job_device = '' if job is None else ('/job:%s' % job)
+    num_cores = self._ctx.num_cores
 
     # For each i, dequeue_ops[i] is a list containing the tensors from all
     # shards. This list is concatenated later.
@@ -1113,8 +1194,9 @@ class _EvalMetrics(object):
       dequeue_ops.append([])
 
     # Outfeed ops execute on each JF node.
-    for i in xrange(num_shards):
-      with ops.device('%s/task:%d/device:TPU:%d' % (job_device, i / 8, i % 8)):
+    tpu_device_placement_fn = self._ctx.tpu_device_placement_function
+    for i in xrange(num_cores):
+      with ops.device(tpu_device_placement_fn(i)):
         outfeed_tensors = tpu_ops.outfeed_dequeue_tuple(
             dtypes=self._tensor_dtypes, shapes=self._tensor_shapes)
         for j, item in enumerate(outfeed_tensors):
@@ -1122,7 +1204,7 @@ class _EvalMetrics(object):
 
     # It is assumed evaluation always happends on single host TPU system. So,
     # place all ops on tpu host if possible.
-    with ops.device('{}/device:CPU:0'.format(job_device)):
+    with ops.device(self._ctx.tpu_host_placement_function(core_id=0)):
       for i, item in enumerate(dequeue_ops):
         if dequeue_ops[i][0].shape.ndims == 0:
           raise RuntimeError(
@@ -1167,9 +1249,9 @@ class TPUEstimator(estimator_lib.Estimator):
   specify `train_batch_size` in constructor, and then get the batch size for
   each shard in `input_fn` and `model_fn` by `params['batch_size']`. If
   `TPUConfig.per_host_input_for_training` is `True`, `input_fn` is invoked per
-  host rather than per shard. In this case, a global batch size is transformed a
+  host rather than per core. In this case, a global batch size is transformed a
   per-host batch size in params for `input_fn`, but `model_fn` still gets
-  per-shard batch size.
+  per-core batch size.
 
   For evaluation, if `eval_batch_size` is None, it is executed on CPU, even if
   `use_tpu` is `True`. If `eval_batch_size` is not `None`, it is executed on
@@ -1327,9 +1409,7 @@ class TPUEstimator(estimator_lib.Estimator):
     # We cannot store config and params in this constructor as parent
     # constructor might change them, such as assigning a temp dir for
     # config.model_dir.
-    model_function = _augment_model_fn(model_fn, train_batch_size,
-                                       eval_batch_size, use_tpu,
-                                       batch_axis)
+    model_function = self._augment_model_fn(model_fn, batch_axis)
 
     # Passing non-None params as wrapped model_fn has it.
     params = params or {}
@@ -1338,12 +1418,13 @@ class TPUEstimator(estimator_lib.Estimator):
         model_dir=model_dir,
         config=config,
         params=params)
-    self._use_tpu = use_tpu
-    self._train_batch_size = train_batch_size
-    self._eval_batch_size = eval_batch_size
     self._iterations_per_training_loop = (
         self._config.tpu_config.iterations_per_loop)
 
+    # All properties passed to _TPUContext are immutable.
+    self._ctx = _TPUContext(self._config, train_batch_size, eval_batch_size,
+                            use_tpu)
+
   def _create_global_step(self, graph):
     """Creates a global step suitable for TPUs.
 
@@ -1359,10 +1440,10 @@ class TPUEstimator(estimator_lib.Estimator):
     return _create_global_step(graph)
 
   def _convert_train_steps_to_hooks(self, steps, max_steps):
-    if _is_running_on_cpu(self._use_tpu, model_fn_lib.ModeKeys.TRAIN,
-                          self._eval_batch_size):
-      return super(TPUEstimator, self)._convert_train_steps_to_hooks(
-          steps, max_steps)
+    with self._ctx.with_mode(model_fn_lib.ModeKeys.TRAIN) as ctx:
+      if ctx.is_running_on_cpu():
+        return super(TPUEstimator, self)._convert_train_steps_to_hooks(
+            steps, max_steps)
 
     # On TPU.
     if steps is None and max_steps is None:
@@ -1380,9 +1461,9 @@ class TPUEstimator(estimator_lib.Estimator):
                                steps, max_steps)]
 
   def _convert_eval_steps_to_hooks(self, steps):
-    if _is_running_on_cpu(self._use_tpu, model_fn_lib.ModeKeys.EVAL,
-                          self._eval_batch_size):
-      return super(TPUEstimator, self)._convert_eval_steps_to_hooks(steps)
+    with self._ctx.with_mode(model_fn_lib.ModeKeys.EVAL) as ctx:
+      if ctx.is_running_on_cpu():
+        return super(TPUEstimator, self)._convert_eval_steps_to_hooks(steps)
 
     if steps is None:
       raise ValueError('Evaluate `steps` must be set on TPU. Cannot be `None`.')
@@ -1422,197 +1503,115 @@ class TPUEstimator(estimator_lib.Estimator):
     if 'config' in input_fn_args:
       kwargs['config'] = config
 
-    # Setting the batch size in params first. This helps user to have same
-    # input_fn for use_tpu=True/False.
-    if mode == model_fn_lib.ModeKeys.TRAIN:
-      kwargs['params'][_BATCH_SIZE_KEY] = (
-          _per_shard_batch_size(self._train_batch_size, config, self._use_tpu)
-          if not config.tpu_config.per_host_input_for_training else
-          self._train_batch_size)
-    elif (mode == model_fn_lib.ModeKeys.EVAL and
-          self._eval_batch_size is not None):
-      # For TPU evaluation, input_fn is invoked for one host (instead of shard).
-      kwargs['params'][_BATCH_SIZE_KEY] = self._eval_batch_size
-
-    if _is_running_on_cpu(self._use_tpu, mode, self._eval_batch_size):
-      with ops.device('/device:CPU:0'):
-        return input_fn(**kwargs)
-
-    job = _tpu_job(config, mode)
-    def placement_function(index):
-      if job is None:
-        return '/replica:0/task:0/device:CPU:0'
-      else:
-        return '/job:%s/task:%d/device:CPU:0' % (job, index / 8)
+    with self._ctx.with_mode(mode) as ctx:
+      # Setting the batch size in params first. This helps user to have same
+      # input_fn for use_tpu=True/False.
+      batch_size_for_input_fn = ctx.batch_size_for_input_fn
+      if batch_size_for_input_fn is not None:
+        kwargs['params'][_BATCH_SIZE_KEY] = batch_size_for_input_fn
 
-    if mode == model_fn_lib.ModeKeys.TRAIN:
-      if not config.tpu_config.per_host_input_for_training:
-        # Now for TPU training.
-        num_shards = config.tpu_config.num_shards
-        inputs = _InputsHolder(num_shards=num_shards)
-        for i in range(config.tpu_config.num_shards):
-          with ops.device(placement_function(i)):
-            inputs.append_tuple(input_fn(**kwargs))
-        return inputs.as_features_and_labels_tuple()
-      else:
-        # TODO(xiejw): Extend this to multi-host support.
-        with ops.device(placement_function(0)):
+      if ctx.is_running_on_cpu():
+        with ops.device('/device:CPU:0'):
           return input_fn(**kwargs)
 
-    # Now for TPU evaluation.
-    with ops.device(placement_function(0)):
-      return input_fn(**kwargs)
-
-
-# TODO(b/64607814): Ensure batch_axis works with nested structures.
-def _create_infeed_enqueue_ops_and_dequeue_fn(inputs_holder, run_config,
-                                              batch_axis, mode):
-  """Utility to convert input_fn to enqueue and dequeue fns for TPU.
-
-  Args:
-    inputs_holder: An `_InputsHolder` holding features and labels.
-    run_config: A `RunConfig` instance.
-    batch_axis: A python list of batch dimensions.
-    mode: ModeKeys
-
-  Returns:
-    A tuple of (dequeue_fn, enqueue_fn)
-  """
-  if inputs_holder.sharded:
-    sharded_inputs = inputs_holder.as_sharded_flattened_inputs()
-
-    infeed_queue = tpu_feed.InfeedQueue(
-        number_of_tuple_elements=len(sharded_inputs[0]))
-    infeed_queue.set_configuration_from_sharded_input_tensors(sharded_inputs)
-  else:
-    unsharded_inputs = inputs_holder.as_flattened_inputs()
-    infeed_queue = tpu_feed.InfeedQueue(
-        tuple_types=[t.dtype for t in unsharded_inputs],
-        tuple_shapes=[t.shape for t in unsharded_inputs],
-        shard_dimensions=batch_axis)
-    infeed_queue.set_number_of_shards(inputs_holder.num_shards)
-
-  def dequeue_fn():
-    """dequeue_fn is used by the train_step in TPU to retrieve the tensors."""
-    values = infeed_queue.generate_dequeue_op()
-    return inputs_holder.unflatten_features_and_labels(values)
-
-  def tpu_ordinal_function(index):
-    """Return the TPU ordinal associated with a shard.
-
-    Required because the enqueue ops are placed on CPU.
-
-    Args:
-      index: the shard index
-
-    Returns:
-      The ordinal of the TPU device the shard's infeed should be placed on.
-    """
-    return index % 8
-
-  def enqueue_fn():
-    """enqueue_fn is used to add ops to the graph to send tensors."""
-    if inputs_holder.sharded:
-      return infeed_queue.generate_enqueue_ops(
-          sharded_inputs, tpu_ordinal_function=tpu_ordinal_function)
-    else:
-      job = _tpu_job(run_config, mode)
-      def placement_function(index):
-        if job is None:
-          return '/replica:0/task:0/device:CPU:0'
-        else:
-          # This assumes that if using more than 8 shards,
-          # the job configuration varies 'task'.
-          return '/job:%s/task:%d/device:CPU:0' % (job, index / 8)
-      return infeed_queue.split_inputs_and_generate_enqueue_ops(
-          unsharded_inputs, placement_function=placement_function)
-
-  return (dequeue_fn, enqueue_fn)
-
-
-def _augment_model_fn(model_fn, train_batch_size, eval_batch_size, use_tpu,
-                      batch_axis):
-  """Returns a new model_fn, which wraps the TPU support."""
-
-  def _model_fn(features, labels, mode, config, params):
-    """A Estimator `model_fn` for TPUEstimator."""
-    model_fn_wrapper = _ModelFnWrapper(model_fn, config, params, mode,
-                                       train_batch_size, eval_batch_size)
-
-    # TODO(jhseu): Move to PREDICT to TPU.
-    if _is_running_on_cpu(use_tpu, mode, eval_batch_size):
-      logging.info('Running %s on CPU', mode)
-      return model_fn_wrapper.call_without_tpu(features, labels)
-
-    inputs = _InputsHolder(features=features, labels=labels,
-                           num_shards=config.tpu_config.num_shards)
-
-    dequeue_fn, enqueue_fn = _create_infeed_enqueue_ops_and_dequeue_fn(
-        inputs, config, batch_axis, mode)
-
-    if mode == model_fn_lib.ModeKeys.TRAIN:
-      loss = _train_on_tpu_system(model_fn_wrapper, dequeue_fn)
-      hooks = [
-          TPUInfeedOutfeedSessionHook(config, mode, enqueue_fn),
-          training.LoggingTensorHook(
-              {'loss': array_ops.identity(loss),
-               'step': training.get_global_step()},
-              every_n_secs=30)
-      ]
-      summary.scalar(model_fn_lib.LOSS_METRIC_KEY, loss)
-      with ops.control_dependencies([loss]):
-        update_ops = _sync_variables_ops()
-
-      # Validate the TPU training graph to catch basic errors
-      _validate_tpu_training_graph()
-
-      return model_fn_lib.EstimatorSpec(
-          mode,
-          loss=loss,
-          training_hooks=hooks,
-          train_op=control_flow_ops.group(*update_ops))
-
-    # Now eval.
-    total_loss, eval_metric_ops = _eval_on_tpu_system(
-        model_fn_wrapper, dequeue_fn)
-    iterations_per_loop_var = _create_iterations_per_loop()
-    mean_loss = math_ops.div(
-        total_loss,
-        math_ops.cast(iterations_per_loop_var, dtype=total_loss.dtype))
-
-    # Creates a dummy metric update_op for all metrics. Estimator expects all
-    # metrics in eval_metric_ops have update_op and calls them one by one. The
-    # real metric update_ops are invoked in a separated thread. So, here give
-    # Estimator the dummy op for all metrics.
-    with ops.control_dependencies([mean_loss]):
-      # After TPU evaluation computation is done (the mean_loss tensor), reads
-      # all variables back from TPU and updates the eval step counter properly.
-      internal_ops_to_run = _sync_variables_ops()
-      internal_ops_to_run.append(
-          _increase_eval_step_op(iterations_per_loop_var))
-      with ops.control_dependencies(internal_ops_to_run):
-        dummy_update_op = control_flow_ops.no_op()
-
-    eval_metric_ops, eval_update_ops = (
-        eval_metric_ops.to_metric_metric_ops_for_tpu(
-            config, dummy_update_op))
-    hooks = [
-        TPUInfeedOutfeedSessionHook(config, mode, enqueue_fn, eval_update_ops),
-    ]
-
-    return model_fn_lib.EstimatorSpec(
-        mode,
-        loss=mean_loss,
-        evaluation_hooks=hooks,
-        eval_metric_ops=eval_metric_ops)
-  return _model_fn
-
-
-def _eval_on_tpu_system(model_fn_wrapper, dequeue_fn):
+      # For TPU computation, input_fn should be invoked in a tf.while_loop for
+      # performance. While constructing the tf.while_loop, the structure of
+      # inputs returned by the `input_fn` needs to be recorded. The structure
+      # includes whether features or labels is dict or single Tensor, dict keys,
+      # tensor shapes, and dtypes. The recorded structure is used to create the
+      # infeed dequeue ops, which must be wrapped and passed as a Fn, called
+      # inside the TPU computation, as the TPU computation is wrapped inside a
+      # tf.while_loop also. So, we either pass input_fn to model_fn or pass
+      # dequeue_fn to model_fn. Here, `input_fn` is passed directly as
+      # `features` in `model_fn` signature.
+      def _input_fn():
+        return input_fn(**kwargs)
+      return _input_fn
+
+  def _augment_model_fn(self, model_fn, batch_axis):
+    """Returns a new model_fn, which wraps the TPU support."""
+
+    def _model_fn(features, labels, mode, config, params):
+      """A Estimator `model_fn` for TPUEstimator."""
+      with self._ctx.with_mode(mode) as ctx:
+        model_fn_wrapper = _ModelFnWrapper(model_fn, config, params, ctx)
+
+        # TODO(jhseu): Move to PREDICT to TPU.
+        if ctx.is_running_on_cpu():
+          logging.info('Running %s on CPU', mode)
+          return model_fn_wrapper.call_without_tpu(features, labels)
+
+        assert labels is None, '`labels` passed to `model_fn` must be `None`.'
+        # TPUEstimator._call_input_fn passes `input_fn` as features to here.
+        assert callable(features), '`input_fn` is not callable.'
+        input_fn = features
+
+        input_holders = _InputPipeline(input_fn, batch_axis, ctx)
+        enqueue_ops, dequeue_fn = (
+            input_holders.generate_infeed_enqueue_ops_and_dequeue_fn())
+
+        if mode == model_fn_lib.ModeKeys.TRAIN:
+          loss = _train_on_tpu_system(ctx, model_fn_wrapper, dequeue_fn)
+          hooks = [
+              TPUInfeedOutfeedSessionHook(ctx, enqueue_ops),
+              training.LoggingTensorHook(
+                  {'loss': array_ops.identity(loss),
+                   'step': training.get_global_step()},
+                  every_n_secs=30)
+          ]
+          summary.scalar(model_fn_lib.LOSS_METRIC_KEY, loss)
+          with ops.control_dependencies([loss]):
+            update_ops = _sync_variables_ops()
+
+          # Validate the TPU training graph to catch basic errors
+          _validate_tpu_training_graph()
+
+          return model_fn_lib.EstimatorSpec(
+              mode,
+              loss=loss,
+              training_hooks=hooks,
+              train_op=control_flow_ops.group(*update_ops))
+
+        # Now eval.
+        total_loss, eval_metric_ops = _eval_on_tpu_system(
+            ctx, model_fn_wrapper, dequeue_fn)
+        iterations_per_loop_var = _create_or_get_iterations_per_loop()
+        mean_loss = math_ops.div(
+            total_loss,
+            math_ops.cast(iterations_per_loop_var, dtype=total_loss.dtype))
+
+        # Creates a dummy metric update_op for all metrics. Estimator expects
+        # all metrics in eval_metric_ops have update_op and calls them one by
+        # one. The real metric update_ops are invoked in a separated thread. So,
+        # here give Estimator the dummy op for all metrics.
+        with ops.control_dependencies([mean_loss]):
+          # After TPU evaluation computation is done (the mean_loss tensor),
+          # reads all variables back from TPU and updates the eval step counter
+          # properly
+          internal_ops_to_run = _sync_variables_ops()
+          internal_ops_to_run.append(
+              _increase_eval_step_op(iterations_per_loop_var))
+          with ops.control_dependencies(internal_ops_to_run):
+            dummy_update_op = control_flow_ops.no_op()
+
+        eval_metric_ops, eval_update_ops = (
+            eval_metric_ops.to_metric_metric_ops_for_tpu(dummy_update_op))
+        hooks = [
+            TPUInfeedOutfeedSessionHook(ctx, enqueue_ops, eval_update_ops),
+        ]
+
+        return model_fn_lib.EstimatorSpec(
+            mode,
+            loss=mean_loss,
+            evaluation_hooks=hooks,
+            eval_metric_ops=eval_metric_ops)
+    return _model_fn
+
+
+def _eval_on_tpu_system(ctx, model_fn_wrapper, dequeue_fn):
   """Executes `model_fn_wrapper` multiple times on all TPU shards."""
-  config = model_fn_wrapper.config.tpu_config
-  num_shards = config.num_shards
-  iterations_per_loop_var = _create_iterations_per_loop()
+  num_cores = ctx.num_cores
+  iterations_per_loop_var = _create_or_get_iterations_per_loop()
 
   single_tpu_eval_step, eval_metric_ops = (
       model_fn_wrapper.convert_to_single_tpu_eval_step(dequeue_fn))
@@ -1625,15 +1624,15 @@ def _eval_on_tpu_system(model_fn_wrapper, dequeue_fn):
 
   (loss,) = tpu.shard(multi_tpu_eval_steps_on_single_shard,
                       inputs=[],
-                      num_shards=num_shards,
+                      num_shards=num_cores,
                       outputs_from_all_shards=False)
   return loss, eval_metric_ops
 
 
-def _train_on_tpu_system(model_fn_wrapper, dequeue_fn):
+def _train_on_tpu_system(ctx, model_fn_wrapper, dequeue_fn):
   """Executes `model_fn_wrapper` multiple times on all TPU shards."""
-  num_shards = model_fn_wrapper.config.tpu_config.num_shards
-  iterations_per_loop_var = _create_iterations_per_loop()
+  num_cores = ctx.num_cores
+  iterations_per_loop_var = _create_or_get_iterations_per_loop()
 
   single_tpu_train_step = model_fn_wrapper.convert_to_single_tpu_train_step(
       dequeue_fn)
@@ -1647,11 +1646,27 @@ def _train_on_tpu_system(model_fn_wrapper, dequeue_fn):
 
   (loss,) = tpu.shard(multi_tpu_train_steps_on_single_shard,
                       inputs=[],
-                      num_shards=num_shards,
+                      num_shards=num_cores,
                       outputs_from_all_shards=False)
   return loss
 
 
+def _wrap_computation_in_while_loop(device, op_fn):
+  """Wraps the ops generated by `op_fn` in tf.while_loop."""
+  def computation(i):
+    with ops.control_dependencies(op_fn()):
+      return i + 1
+
+  iterations_per_loop_var = _create_or_get_iterations_per_loop()
+  # By setting parallel_iterations=1, the parallel execution in while_loop is
+  # basically turned off.
+  with ops.device(device):
+    iterations = array_ops.identity(iterations_per_loop_var)
+    return control_flow_ops.while_loop(
+        lambda i: i < iterations,
+        computation, [constant_op.constant(0)], parallel_iterations=1)
+
+
 def _validate_tpu_training_graph():
   """Validate graph before running distributed training.
 
-- 
GitLab


From 71bdc0efa737e3094033f0c6ea3779b1fc3c8a94 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 20 Oct 2017 11:34:40 -0700
Subject: [PATCH 616/909] Formatting metric_ops.

PiperOrigin-RevId: 172910546
---
 .../contrib/metrics/python/ops/metric_ops.py  | 591 +++++++++++-------
 1 file changed, 382 insertions(+), 209 deletions(-)

diff --git a/tensorflow/contrib/metrics/python/ops/metric_ops.py b/tensorflow/contrib/metrics/python/ops/metric_ops.py
index 85c8e9038a..09485c4fa2 100644
--- a/tensorflow/contrib/metrics/python/ops/metric_ops.py
+++ b/tensorflow/contrib/metrics/python/ops/metric_ops.py
@@ -56,7 +56,10 @@ def _safe_div(numerator, denominator, name):
       name=name)
 
 
-def _create_local(name, shape, collections=None, validate_shape=True,
+def _create_local(name,
+                  shape,
+                  collections=None,
+                  validate_shape=True,
                   dtype=dtypes.float32):
   """Creates a new local variable.
 
@@ -87,7 +90,9 @@ def _assert_weights_rank(weights, values):
   return check_ops.assert_rank_in(weights, (0, array_ops.rank(values)))
 
 
-def _count_condition(values, weights=None, metrics_collections=None,
+def _count_condition(values,
+                     weights=None,
+                     metrics_collections=None,
                      updates_collections=None):
   """Sums the weights of cases where the given values are True.
 
@@ -134,7 +139,9 @@ def _count_condition(values, weights=None, metrics_collections=None,
   return value_tensor, update_op
 
 
-def streaming_true_positives(predictions, labels, weights=None,
+def streaming_true_positives(predictions,
+                             labels,
+                             weights=None,
                              metrics_collections=None,
                              updates_collections=None,
                              name=None):
@@ -168,12 +175,17 @@ def streaming_true_positives(predictions, labels, weights=None,
       tuple.
   """
   return metrics.true_positives(
-      predictions=predictions, labels=labels, weights=weights,
+      predictions=predictions,
+      labels=labels,
+      weights=weights,
       metrics_collections=metrics_collections,
-      updates_collections=updates_collections, name=name)
+      updates_collections=updates_collections,
+      name=name)
 
 
-def streaming_true_negatives(predictions, labels, weights=None,
+def streaming_true_negatives(predictions,
+                             labels,
+                             weights=None,
                              metrics_collections=None,
                              updates_collections=None,
                              name=None):
@@ -206,20 +218,22 @@ def streaming_true_negatives(predictions, labels, weights=None,
       either `metrics_collections` or `updates_collections` are not a list or
       tuple.
   """
-  with variable_scope.variable_scope(
-      name, 'true_negatives', (predictions, labels, weights)):
+  with variable_scope.variable_scope(name, 'true_negatives',
+                                     (predictions, labels, weights)):
 
     predictions, labels, weights = _remove_squeezable_dimensions(
         predictions=math_ops.cast(predictions, dtype=dtypes.bool),
         labels=math_ops.cast(labels, dtype=dtypes.bool),
         weights=weights)
-    is_true_negative = math_ops.logical_and(math_ops.equal(labels, False),
-                                            math_ops.equal(predictions, False))
+    is_true_negative = math_ops.logical_and(
+        math_ops.equal(labels, False), math_ops.equal(predictions, False))
     return _count_condition(is_true_negative, weights, metrics_collections,
                             updates_collections)
 
 
-def streaming_false_positives(predictions, labels, weights=None,
+def streaming_false_positives(predictions,
+                              labels,
+                              weights=None,
                               metrics_collections=None,
                               updates_collections=None,
                               name=None):
@@ -253,12 +267,17 @@ def streaming_false_positives(predictions, labels, weights=None,
       tuple.
   """
   return metrics.false_positives(
-      predictions=predictions, labels=labels, weights=weights,
+      predictions=predictions,
+      labels=labels,
+      weights=weights,
       metrics_collections=metrics_collections,
-      updates_collections=updates_collections, name=name)
+      updates_collections=updates_collections,
+      name=name)
 
 
-def streaming_false_negatives(predictions, labels, weights=None,
+def streaming_false_negatives(predictions,
+                              labels,
+                              weights=None,
                               metrics_collections=None,
                               updates_collections=None,
                               name=None):
@@ -291,9 +310,12 @@ def streaming_false_negatives(predictions, labels, weights=None,
       or tuple.
   """
   return metrics.false_negatives(
-      predictions=predictions, labels=labels, weights=weights,
+      predictions=predictions,
+      labels=labels,
+      weights=weights,
       metrics_collections=metrics_collections,
-      updates_collections=updates_collections, name=name)
+      updates_collections=updates_collections,
+      name=name)
 
 
 # TODO(ptucker): Move this somewhere common, to share with ops/losses/losses.py.
@@ -317,17 +339,18 @@ def _broadcast_weights(weights, values):
   with ops.name_scope(None, 'broadcast_weights', (values, weights)) as scope:
     weights_shape = weights.get_shape()
     values_shape = values.get_shape()
-    if (weights_shape.is_fully_defined() and
-        values_shape.is_fully_defined() and
+    if (weights_shape.is_fully_defined() and values_shape.is_fully_defined() and
         weights_shape.is_compatible_with(values_shape)):
       return weights
     with ops.control_dependencies((_assert_weights_rank(weights, values),)):
-      return math_ops.multiply(
-          weights, array_ops.ones_like(values), name=scope)
+      return math_ops.multiply(weights, array_ops.ones_like(values), name=scope)
 
 
-def streaming_mean(values, weights=None, metrics_collections=None,
-                   updates_collections=None, name=None):
+def streaming_mean(values,
+                   weights=None,
+                   metrics_collections=None,
+                   updates_collections=None,
+                   name=None):
   """Computes the (weighted) mean of the given values.
 
   The `streaming_mean` function creates two local variables, `total` and `count`
@@ -365,12 +388,18 @@ def streaming_mean(values, weights=None, metrics_collections=None,
       or tuple.
   """
   return metrics.mean(
-      values=values, weights=weights, metrics_collections=metrics_collections,
-      updates_collections=updates_collections, name=name)
+      values=values,
+      weights=weights,
+      metrics_collections=metrics_collections,
+      updates_collections=updates_collections,
+      name=name)
 
 
-def streaming_mean_tensor(values, weights=None, metrics_collections=None,
-                          updates_collections=None, name=None):
+def streaming_mean_tensor(values,
+                          weights=None,
+                          metrics_collections=None,
+                          updates_collections=None,
+                          name=None):
   """Computes the element-wise (weighted) mean of the given tensors.
 
   In contrast to the `streaming_mean` function which returns a scalar with the
@@ -412,12 +441,18 @@ def streaming_mean_tensor(values, weights=None, metrics_collections=None,
       or tuple.
   """
   return metrics.mean_tensor(
-      values=values, weights=weights, metrics_collections=metrics_collections,
-      updates_collections=updates_collections, name=name)
+      values=values,
+      weights=weights,
+      metrics_collections=metrics_collections,
+      updates_collections=updates_collections,
+      name=name)
 
 
-def streaming_accuracy(predictions, labels, weights=None,
-                       metrics_collections=None, updates_collections=None,
+def streaming_accuracy(predictions,
+                       labels,
+                       weights=None,
+                       metrics_collections=None,
+                       updates_collections=None,
                        name=None):
   """Calculates how often `predictions` matches `labels`.
 
@@ -462,13 +497,19 @@ def streaming_accuracy(predictions, labels, weights=None,
       tuple.
   """
   return metrics.accuracy(
-      predictions=predictions, labels=labels, weights=weights,
+      predictions=predictions,
+      labels=labels,
+      weights=weights,
       metrics_collections=metrics_collections,
-      updates_collections=updates_collections, name=name)
+      updates_collections=updates_collections,
+      name=name)
 
 
-def streaming_precision(predictions, labels, weights=None,
-                        metrics_collections=None, updates_collections=None,
+def streaming_precision(predictions,
+                        labels,
+                        weights=None,
+                        metrics_collections=None,
+                        updates_collections=None,
                         name=None):
   """Computes the precision of the predictions with respect to the labels.
 
@@ -512,13 +553,19 @@ def streaming_precision(predictions, labels, weights=None,
       tuple.
   """
   return metrics.precision(
-      predictions=predictions, labels=labels, weights=weights,
+      predictions=predictions,
+      labels=labels,
+      weights=weights,
       metrics_collections=metrics_collections,
-      updates_collections=updates_collections, name=name)
+      updates_collections=updates_collections,
+      name=name)
 
 
-def streaming_recall(predictions, labels, weights=None,
-                     metrics_collections=None, updates_collections=None,
+def streaming_recall(predictions,
+                     labels,
+                     weights=None,
+                     metrics_collections=None,
+                     updates_collections=None,
                      name=None):
   """Computes the recall of the predictions with respect to the labels.
 
@@ -560,12 +607,17 @@ def streaming_recall(predictions, labels, weights=None,
       tuple.
   """
   return metrics.recall(
-      predictions=predictions, labels=labels, weights=weights,
+      predictions=predictions,
+      labels=labels,
+      weights=weights,
       metrics_collections=metrics_collections,
-      updates_collections=updates_collections, name=name)
+      updates_collections=updates_collections,
+      name=name)
 
 
-def _true_negatives(labels, predictions, weights=None,
+def _true_negatives(labels,
+                    predictions,
+                    weights=None,
                     metrics_collections=None,
                     updates_collections=None,
                     name=None):
@@ -597,20 +649,22 @@ def _true_negatives(labels, predictions, weights=None,
       either `metrics_collections` or `updates_collections` are not a list or
       tuple.
   """
-  with variable_scope.variable_scope(
-      name, 'true_negatives', (predictions, labels, weights)):
+  with variable_scope.variable_scope(name, 'true_negatives',
+                                     (predictions, labels, weights)):
 
     predictions, labels, weights = _remove_squeezable_dimensions(
         predictions=math_ops.cast(predictions, dtype=dtypes.bool),
         labels=math_ops.cast(labels, dtype=dtypes.bool),
         weights=weights)
-    is_true_negative = math_ops.logical_and(math_ops.equal(labels, False),
-                                            math_ops.equal(predictions, False))
+    is_true_negative = math_ops.logical_and(
+        math_ops.equal(labels, False), math_ops.equal(predictions, False))
     return _count_condition(is_true_negative, weights, metrics_collections,
                             updates_collections)
 
 
-def streaming_false_positive_rate(predictions, labels, weights=None,
+def streaming_false_positive_rate(predictions,
+                                  labels,
+                                  weights=None,
                                   metrics_collections=None,
                                   updates_collections=None,
                                   name=None):
@@ -657,30 +711,35 @@ def streaming_false_positive_rate(predictions, labels, weights=None,
       either `metrics_collections` or `updates_collections` are not a list or
       tuple.
   """
-  with variable_scope.variable_scope(
-      name, 'false_positive_rate', (predictions, labels, weights)):
+  with variable_scope.variable_scope(name, 'false_positive_rate',
+                                     (predictions, labels, weights)):
     predictions, labels, weights = _remove_squeezable_dimensions(
         predictions=math_ops.cast(predictions, dtype=dtypes.bool),
         labels=math_ops.cast(labels, dtype=dtypes.bool),
         weights=weights)
 
     false_p, false_positives_update_op = metrics.false_positives(
-        labels, predictions, weights, metrics_collections=None,
-        updates_collections=None, name=None)
+        labels,
+        predictions,
+        weights,
+        metrics_collections=None,
+        updates_collections=None,
+        name=None)
     true_n, true_negatives_update_op = _true_negatives(
-        labels, predictions, weights, metrics_collections=None,
-        updates_collections=None, name=None)
+        labels,
+        predictions,
+        weights,
+        metrics_collections=None,
+        updates_collections=None,
+        name=None)
 
     def compute_fpr(fp, tn, name):
       return array_ops.where(
-          math_ops.greater(fp + tn, 0),
-          math_ops.div(fp, fp + tn),
-          0,
-          name)
+          math_ops.greater(fp + tn, 0), math_ops.div(fp, fp + tn), 0, name)
 
     fpr = compute_fpr(false_p, true_n, 'value')
-    update_op = compute_fpr(
-        false_positives_update_op, true_negatives_update_op, 'update_op')
+    update_op = compute_fpr(false_positives_update_op, true_negatives_update_op,
+                            'update_op')
 
     if metrics_collections:
       ops.add_to_collections(metrics_collections, fpr)
@@ -691,7 +750,9 @@ def streaming_false_positive_rate(predictions, labels, weights=None,
     return fpr, update_op
 
 
-def streaming_false_negative_rate(predictions, labels, weights=None,
+def streaming_false_negative_rate(predictions,
+                                  labels,
+                                  weights=None,
                                   metrics_collections=None,
                                   updates_collections=None,
                                   name=None):
@@ -738,30 +799,35 @@ def streaming_false_negative_rate(predictions, labels, weights=None,
       either `metrics_collections` or `updates_collections` are not a list or
       tuple.
   """
-  with variable_scope.variable_scope(
-      name, 'false_negative_rate', (predictions, labels, weights)):
+  with variable_scope.variable_scope(name, 'false_negative_rate',
+                                     (predictions, labels, weights)):
     predictions, labels, weights = _remove_squeezable_dimensions(
         predictions=math_ops.cast(predictions, dtype=dtypes.bool),
         labels=math_ops.cast(labels, dtype=dtypes.bool),
         weights=weights)
 
     false_n, false_negatives_update_op = metrics.false_negatives(
-        labels, predictions, weights, metrics_collections=None,
-        updates_collections=None, name=None)
+        labels,
+        predictions,
+        weights,
+        metrics_collections=None,
+        updates_collections=None,
+        name=None)
     true_p, true_positives_update_op = metrics.true_positives(
-        labels, predictions, weights, metrics_collections=None,
-        updates_collections=None, name=None)
+        labels,
+        predictions,
+        weights,
+        metrics_collections=None,
+        updates_collections=None,
+        name=None)
 
     def compute_fnr(fn, tp, name):
       return array_ops.where(
-          math_ops.greater(fn + tp, 0),
-          math_ops.div(fn, fn + tp),
-          0,
-          name)
+          math_ops.greater(fn + tp, 0), math_ops.div(fn, fn + tp), 0, name)
 
     fnr = compute_fnr(false_n, true_p, 'value')
-    update_op = compute_fnr(
-        false_negatives_update_op, true_positives_update_op, 'update_op')
+    update_op = compute_fnr(false_negatives_update_op, true_positives_update_op,
+                            'update_op')
 
     if metrics_collections:
       ops.add_to_collections(metrics_collections, fnr)
@@ -772,8 +838,11 @@ def streaming_false_negative_rate(predictions, labels, weights=None,
     return fnr, update_op
 
 
-def _streaming_confusion_matrix_at_thresholds(
-    predictions, labels, thresholds, weights=None, includes=None):
+def _streaming_confusion_matrix_at_thresholds(predictions,
+                                              labels,
+                                              thresholds,
+                                              weights=None,
+                                              includes=None):
   """Computes true_positives, false_negatives, true_negatives, false_positives.
 
   This function creates up to four local variables, `true_positives`,
@@ -861,8 +930,8 @@ def _streaming_confusion_matrix_at_thresholds(
   if weights is not None:
     broadcast_weights = weights_broadcast_ops.broadcast_weights(
         math_ops.to_float(weights), predictions)
-    weights_tiled = array_ops.tile(array_ops.reshape(
-        broadcast_weights, [1, -1]), [num_thresholds, 1])
+    weights_tiled = array_ops.tile(
+        array_ops.reshape(broadcast_weights, [1, -1]), [num_thresholds, 1])
     thresh_tiled.get_shape().assert_is_compatible_with(
         weights_tiled.get_shape())
   else:
@@ -877,8 +946,9 @@ def _streaming_confusion_matrix_at_thresholds(
         math_ops.logical_and(label_is_pos, pred_is_pos))
     if weights_tiled is not None:
       is_true_positive *= weights_tiled
-    update_ops['tp'] = state_ops.assign_add(
-        true_positives, math_ops.reduce_sum(is_true_positive, 1))
+    update_ops['tp'] = state_ops.assign_add(true_positives,
+                                            math_ops.reduce_sum(
+                                                is_true_positive, 1))
     values['tp'] = true_positives
 
   if 'fn' in includes:
@@ -887,8 +957,9 @@ def _streaming_confusion_matrix_at_thresholds(
         math_ops.logical_and(label_is_pos, pred_is_neg))
     if weights_tiled is not None:
       is_false_negative *= weights_tiled
-    update_ops['fn'] = state_ops.assign_add(
-        false_negatives, math_ops.reduce_sum(is_false_negative, 1))
+    update_ops['fn'] = state_ops.assign_add(false_negatives,
+                                            math_ops.reduce_sum(
+                                                is_false_negative, 1))
     values['fn'] = false_negatives
 
   if 'tn' in includes:
@@ -897,8 +968,9 @@ def _streaming_confusion_matrix_at_thresholds(
         math_ops.logical_and(label_is_neg, pred_is_neg))
     if weights_tiled is not None:
       is_true_negative *= weights_tiled
-    update_ops['tn'] = state_ops.assign_add(
-        true_negatives, math_ops.reduce_sum(is_true_negative, 1))
+    update_ops['tn'] = state_ops.assign_add(true_negatives,
+                                            math_ops.reduce_sum(
+                                                is_true_negative, 1))
     values['tn'] = true_negatives
 
   if 'fp' in includes:
@@ -907,36 +979,45 @@ def _streaming_confusion_matrix_at_thresholds(
         math_ops.logical_and(label_is_neg, pred_is_pos))
     if weights_tiled is not None:
       is_false_positive *= weights_tiled
-    update_ops['fp'] = state_ops.assign_add(
-        false_positives, math_ops.reduce_sum(is_false_positive, 1))
+    update_ops['fp'] = state_ops.assign_add(false_positives,
+                                            math_ops.reduce_sum(
+                                                is_false_positive, 1))
     values['fp'] = false_positives
 
   return values, update_ops
 
 
-def streaming_true_positives_at_thresholds(
-    predictions, labels, thresholds, weights=None):
+def streaming_true_positives_at_thresholds(predictions,
+                                           labels,
+                                           thresholds,
+                                           weights=None):
   values, update_ops = _streaming_confusion_matrix_at_thresholds(
       predictions, labels, thresholds, weights=weights, includes=('tp',))
   return values['tp'], update_ops['tp']
 
 
-def streaming_false_negatives_at_thresholds(
-    predictions, labels, thresholds, weights=None):
+def streaming_false_negatives_at_thresholds(predictions,
+                                            labels,
+                                            thresholds,
+                                            weights=None):
   values, update_ops = _streaming_confusion_matrix_at_thresholds(
       predictions, labels, thresholds, weights=weights, includes=('fn',))
   return values['fn'], update_ops['fn']
 
 
-def streaming_false_positives_at_thresholds(
-    predictions, labels, thresholds, weights=None):
+def streaming_false_positives_at_thresholds(predictions,
+                                            labels,
+                                            thresholds,
+                                            weights=None):
   values, update_ops = _streaming_confusion_matrix_at_thresholds(
       predictions, labels, thresholds, weights=weights, includes=('fp',))
   return values['fp'], update_ops['fp']
 
 
-def streaming_true_negatives_at_thresholds(
-    predictions, labels, thresholds, weights=None):
+def streaming_true_negatives_at_thresholds(predictions,
+                                           labels,
+                                           thresholds,
+                                           weights=None):
   values, update_ops = _streaming_confusion_matrix_at_thresholds(
       predictions, labels, thresholds, weights=weights, includes=('tn',))
   return values['tn'], update_ops['tn']
@@ -996,8 +1077,8 @@ def streaming_curve_points(labels=None,
       either `metrics_collections` or `updates_collections` are not a list or
       tuple.
   """
-  with variable_scope.variable_scope(name, 'curve_points', (labels, predictions,
-                                                            weights)):
+  with variable_scope.variable_scope(name, 'curve_points',
+                                     (labels, predictions, weights)):
     if curve != 'ROC' and curve != 'PR':
       raise ValueError('curve must be either ROC or PR, %s unknown' % (curve))
     kepsilon = 1e-7  # to account for floating point imprecisions
@@ -1038,9 +1119,14 @@ def streaming_curve_points(labels=None,
     return points, update_op
 
 
-def streaming_auc(predictions, labels, weights=None, num_thresholds=200,
-                  metrics_collections=None, updates_collections=None,
-                  curve='ROC', name=None):
+def streaming_auc(predictions,
+                  labels,
+                  weights=None,
+                  num_thresholds=200,
+                  metrics_collections=None,
+                  updates_collections=None,
+                  curve='ROC',
+                  name=None):
   """Computes the approximate AUC via a Riemann sum.
 
   The `streaming_auc` function creates four local variables, `true_positives`,
@@ -1097,14 +1183,24 @@ def streaming_auc(predictions, labels, weights=None, num_thresholds=200,
       tuple.
   """
   return metrics.auc(
-      predictions=predictions, labels=labels, weights=weights,
-      metrics_collections=metrics_collections, num_thresholds=num_thresholds,
-      curve=curve, updates_collections=updates_collections, name=name)
+      predictions=predictions,
+      labels=labels,
+      weights=weights,
+      metrics_collections=metrics_collections,
+      num_thresholds=num_thresholds,
+      curve=curve,
+      updates_collections=updates_collections,
+      name=name)
 
 
-def streaming_specificity_at_sensitivity(
-    predictions, labels, sensitivity, weights=None, num_thresholds=200,
-    metrics_collections=None, updates_collections=None, name=None):
+def streaming_specificity_at_sensitivity(predictions,
+                                         labels,
+                                         sensitivity,
+                                         weights=None,
+                                         num_thresholds=200,
+                                         metrics_collections=None,
+                                         updates_collections=None,
+                                         name=None):
   """Computes the specificity at a given sensitivity.
 
   The `streaming_specificity_at_sensitivity` function creates four local
@@ -1154,15 +1250,24 @@ def streaming_specificity_at_sensitivity(
       or `updates_collections` are not a list or tuple.
   """
   return metrics.specificity_at_sensitivity(
-      sensitivity=sensitivity, num_thresholds=num_thresholds,
-      predictions=predictions, labels=labels, weights=weights,
+      sensitivity=sensitivity,
+      num_thresholds=num_thresholds,
+      predictions=predictions,
+      labels=labels,
+      weights=weights,
       metrics_collections=metrics_collections,
-      updates_collections=updates_collections, name=name)
+      updates_collections=updates_collections,
+      name=name)
 
 
-def streaming_sensitivity_at_specificity(
-    predictions, labels, specificity, weights=None, num_thresholds=200,
-    metrics_collections=None, updates_collections=None, name=None):
+def streaming_sensitivity_at_specificity(predictions,
+                                         labels,
+                                         specificity,
+                                         weights=None,
+                                         num_thresholds=200,
+                                         metrics_collections=None,
+                                         updates_collections=None,
+                                         name=None):
   """Computes the sensitivity at a given specificity.
 
   The `streaming_sensitivity_at_specificity` function creates four local
@@ -1212,16 +1317,23 @@ def streaming_sensitivity_at_specificity(
       or `updates_collections` are not a list or tuple.
   """
   return metrics.sensitivity_at_specificity(
-      specificity=specificity, num_thresholds=num_thresholds,
-      predictions=predictions, labels=labels, weights=weights,
+      specificity=specificity,
+      num_thresholds=num_thresholds,
+      predictions=predictions,
+      labels=labels,
+      weights=weights,
       metrics_collections=metrics_collections,
-      updates_collections=updates_collections, name=name)
+      updates_collections=updates_collections,
+      name=name)
 
 
-def streaming_precision_at_thresholds(predictions, labels, thresholds,
+def streaming_precision_at_thresholds(predictions,
+                                      labels,
+                                      thresholds,
                                       weights=None,
                                       metrics_collections=None,
-                                      updates_collections=None, name=None):
+                                      updates_collections=None,
+                                      name=None):
   """Computes precision values for different `thresholds` on `predictions`.
 
   The `streaming_precision_at_thresholds` function creates four local variables,
@@ -1266,14 +1378,21 @@ def streaming_precision_at_thresholds(predictions, labels, thresholds,
   """
   return metrics.precision_at_thresholds(
       thresholds=thresholds,
-      predictions=predictions, labels=labels, weights=weights,
+      predictions=predictions,
+      labels=labels,
+      weights=weights,
       metrics_collections=metrics_collections,
-      updates_collections=updates_collections, name=name)
+      updates_collections=updates_collections,
+      name=name)
 
 
-def streaming_recall_at_thresholds(predictions, labels, thresholds,
-                                   weights=None, metrics_collections=None,
-                                   updates_collections=None, name=None):
+def streaming_recall_at_thresholds(predictions,
+                                   labels,
+                                   thresholds,
+                                   weights=None,
+                                   metrics_collections=None,
+                                   updates_collections=None,
+                                   name=None):
   """Computes various recall values for different `thresholds` on `predictions`.
 
   The `streaming_recall_at_thresholds` function creates four local variables,
@@ -1316,14 +1435,21 @@ def streaming_recall_at_thresholds(predictions, labels, thresholds,
   """
   return metrics.recall_at_thresholds(
       thresholds=thresholds,
-      predictions=predictions, labels=labels, weights=weights,
+      predictions=predictions,
+      labels=labels,
+      weights=weights,
       metrics_collections=metrics_collections,
-      updates_collections=updates_collections, name=name)
+      updates_collections=updates_collections,
+      name=name)
 
 
-def streaming_false_positive_rate_at_thresholds(
-    predictions, labels, thresholds, weights=None, metrics_collections=None,
-    updates_collections=None, name=None):
+def streaming_false_positive_rate_at_thresholds(predictions,
+                                                labels,
+                                                thresholds,
+                                                weights=None,
+                                                metrics_collections=None,
+                                                updates_collections=None,
+                                                name=None):
   """Computes various fpr values for different `thresholds` on `predictions`.
 
   The `streaming_false_positive_rate_at_thresholds` function creates two
@@ -1365,20 +1491,19 @@ def streaming_false_positive_rate_at_thresholds(
       either `metrics_collections` or `updates_collections` are not a list or
       tuple.
   """
-  with variable_scope.variable_scope(
-      name, 'false_positive_rate_at_thresholds',
-      (predictions, labels, weights)):
+  with variable_scope.variable_scope(name, 'false_positive_rate_at_thresholds',
+                                     (predictions, labels, weights)):
     values, update_ops = _streaming_confusion_matrix_at_thresholds(
         predictions, labels, thresholds, weights, includes=('fp', 'tn'))
 
     # Avoid division by zero.
     epsilon = 1e-7
+
     def compute_fpr(fp, tn, name):
       return math_ops.div(fp, epsilon + fp + tn, name='fpr_' + name)
 
     fpr = compute_fpr(values['fp'], values['tn'], 'value')
-    update_op = compute_fpr(
-        update_ops['fp'], update_ops['tn'], 'update_op')
+    update_op = compute_fpr(update_ops['fp'], update_ops['tn'], 'update_op')
 
     if metrics_collections:
       ops.add_to_collections(metrics_collections, fpr)
@@ -1389,9 +1514,13 @@ def streaming_false_positive_rate_at_thresholds(
     return fpr, update_op
 
 
-def streaming_false_negative_rate_at_thresholds(
-    predictions, labels, thresholds, weights=None, metrics_collections=None,
-    updates_collections=None, name=None):
+def streaming_false_negative_rate_at_thresholds(predictions,
+                                                labels,
+                                                thresholds,
+                                                weights=None,
+                                                metrics_collections=None,
+                                                updates_collections=None,
+                                                name=None):
   """Computes various fnr values for different `thresholds` on `predictions`.
 
   The `streaming_false_negative_rate_at_thresholds` function creates two
@@ -1433,20 +1562,19 @@ def streaming_false_negative_rate_at_thresholds(
       either `metrics_collections` or `updates_collections` are not a list or
       tuple.
   """
-  with variable_scope.variable_scope(
-      name, 'false_negative_rate_at_thresholds',
-      (predictions, labels, weights)):
+  with variable_scope.variable_scope(name, 'false_negative_rate_at_thresholds',
+                                     (predictions, labels, weights)):
     values, update_ops = _streaming_confusion_matrix_at_thresholds(
         predictions, labels, thresholds, weights, includes=('fn', 'tp'))
 
     # Avoid division by zero.
     epsilon = 1e-7
+
     def compute_fnr(fn, tp, name):
       return math_ops.div(fn, epsilon + fn + tp, name='fnr_' + name)
 
     fnr = compute_fnr(values['fn'], values['tp'], 'value')
-    update_op = compute_fnr(
-        update_ops['fn'], update_ops['tp'], 'update_op')
+    update_op = compute_fnr(update_ops['fn'], update_ops['tp'], 'update_op')
 
     if metrics_collections:
       ops.add_to_collections(metrics_collections, fnr)
@@ -1469,8 +1597,12 @@ def _at_k_name(name, k=None, class_id=None):
 
 @deprecated('2016-11-08', 'Please use `streaming_sparse_recall_at_k`, '
             'and reshape labels from [batch_size] to [batch_size, 1].')
-def streaming_recall_at_k(predictions, labels, k, weights=None,
-                          metrics_collections=None, updates_collections=None,
+def streaming_recall_at_k(predictions,
+                          labels,
+                          k,
+                          weights=None,
+                          metrics_collections=None,
+                          updates_collections=None,
                           name=None):
   """Computes the recall@k of the predictions with respect to dense labels.
 
@@ -1516,11 +1648,8 @@ def streaming_recall_at_k(predictions, labels, k, weights=None,
       tuple.
   """
   in_top_k = math_ops.to_float(nn.in_top_k(predictions, labels, k))
-  return streaming_mean(in_top_k,
-                        weights,
-                        metrics_collections,
-                        updates_collections,
-                        name or _at_k_name('recall', k))
+  return streaming_mean(in_top_k, weights, metrics_collections,
+                        updates_collections, name or _at_k_name('recall', k))
 
 
 # TODO(ptucker): Validate range of values in labels?
@@ -1599,10 +1728,14 @@ def streaming_sparse_recall_at_k(predictions,
     are not a list or tuple.
   """
   return metrics.recall_at_k(
-      k=k, class_id=class_id,
-      predictions=predictions, labels=labels, weights=weights,
+      k=k,
+      class_id=class_id,
+      predictions=predictions,
+      labels=labels,
+      weights=weights,
       metrics_collections=metrics_collections,
-      updates_collections=updates_collections, name=name)
+      updates_collections=updates_collections,
+      name=name)
 
 
 # TODO(ptucker): Validate range of values in labels?
@@ -1684,10 +1817,14 @@ def streaming_sparse_precision_at_k(predictions,
       are not a list or tuple.
   """
   return metrics.sparse_precision_at_k(
-      k=k, class_id=class_id,
-      predictions=predictions, labels=labels, weights=weights,
+      k=k,
+      class_id=class_id,
+      predictions=predictions,
+      labels=labels,
+      weights=weights,
       metrics_collections=metrics_collections,
-      updates_collections=updates_collections, name=name)
+      updates_collections=updates_collections,
+      name=name)
 
 
 # TODO(ptucker): Validate range of values in labels?
@@ -1766,9 +1903,8 @@ def streaming_sparse_precision_at_top_k(top_k_predictions,
     ValueError: If `top_k_predictions` has rank < 2.
   """
   default_name = _at_k_name('precision', class_id=class_id)
-  with ops.name_scope(
-      name, default_name,
-      (top_k_predictions, labels, weights)) as name_scope:
+  with ops.name_scope(name, default_name,
+                      (top_k_predictions, labels, weights)) as name_scope:
     return metrics_impl._sparse_precision_at_top_k(  # pylint: disable=protected-access
         labels=labels,
         predictions_idx=top_k_predictions,
@@ -1848,8 +1984,8 @@ def sparse_recall_at_top_k(labels,
     are not a list or tuple.
   """
   default_name = _at_k_name('recall', class_id=class_id)
-  with ops.name_scope(name, default_name, (top_k_predictions, labels,
-                                           weights)) as name_scope:
+  with ops.name_scope(name, default_name,
+                      (top_k_predictions, labels, weights)) as name_scope:
     return metrics_impl._sparse_recall_at_top_k(  # pylint: disable=protected-access
         labels=labels,
         predictions_idx=top_k_predictions,
@@ -1919,9 +2055,13 @@ def streaming_sparse_average_precision_at_k(predictions,
       value matches `metric`.
   """
   return metrics.sparse_average_precision_at_k(
-      k=k, predictions=predictions, labels=labels, weights=weights,
+      k=k,
+      predictions=predictions,
+      labels=labels,
+      weights=weights,
       metrics_collections=metrics_collections,
-      updates_collections=updates_collections, name=name)
+      updates_collections=updates_collections,
+      name=name)
 
 
 def streaming_sparse_average_precision_at_top_k(top_k_predictions,
@@ -1987,7 +2127,9 @@ def streaming_sparse_average_precision_at_top_k(top_k_predictions,
       name=name)
 
 
-def streaming_mean_absolute_error(predictions, labels, weights=None,
+def streaming_mean_absolute_error(predictions,
+                                  labels,
+                                  weights=None,
                                   metrics_collections=None,
                                   updates_collections=None,
                                   name=None):
@@ -2035,12 +2177,18 @@ def streaming_mean_absolute_error(predictions, labels, weights=None,
       tuple.
   """
   return metrics.mean_absolute_error(
-      predictions=predictions, labels=labels, weights=weights,
+      predictions=predictions,
+      labels=labels,
+      weights=weights,
       metrics_collections=metrics_collections,
-      updates_collections=updates_collections, name=name)
+      updates_collections=updates_collections,
+      name=name)
 
 
-def streaming_mean_relative_error(predictions, labels, normalizer, weights=None,
+def streaming_mean_relative_error(predictions,
+                                  labels,
+                                  normalizer,
+                                  weights=None,
                                   metrics_collections=None,
                                   updates_collections=None,
                                   name=None):
@@ -2089,12 +2237,18 @@ def streaming_mean_relative_error(predictions, labels, normalizer, weights=None,
       tuple.
   """
   return metrics.mean_relative_error(
-      normalizer=normalizer, predictions=predictions, labels=labels,
-      weights=weights, metrics_collections=metrics_collections,
-      updates_collections=updates_collections, name=name)
+      normalizer=normalizer,
+      predictions=predictions,
+      labels=labels,
+      weights=weights,
+      metrics_collections=metrics_collections,
+      updates_collections=updates_collections,
+      name=name)
 
 
-def streaming_mean_squared_error(predictions, labels, weights=None,
+def streaming_mean_squared_error(predictions,
+                                 labels,
+                                 weights=None,
                                  metrics_collections=None,
                                  updates_collections=None,
                                  name=None):
@@ -2142,12 +2296,17 @@ def streaming_mean_squared_error(predictions, labels, weights=None,
       tuple.
   """
   return metrics.mean_squared_error(
-      predictions=predictions, labels=labels, weights=weights,
+      predictions=predictions,
+      labels=labels,
+      weights=weights,
       metrics_collections=metrics_collections,
-      updates_collections=updates_collections, name=name)
+      updates_collections=updates_collections,
+      name=name)
 
 
-def streaming_root_mean_squared_error(predictions, labels, weights=None,
+def streaming_root_mean_squared_error(predictions,
+                                      labels,
+                                      weights=None,
                                       metrics_collections=None,
                                       updates_collections=None,
                                       name=None):
@@ -2195,9 +2354,12 @@ def streaming_root_mean_squared_error(predictions, labels, weights=None,
       tuple.
   """
   return metrics.root_mean_squared_error(
-      predictions=predictions, labels=labels, weights=weights,
+      predictions=predictions,
+      labels=labels,
+      weights=weights,
       metrics_collections=metrics_collections,
-      updates_collections=updates_collections, name=name)
+      updates_collections=updates_collections,
+      name=name)
 
 
 def streaming_covariance(predictions,
@@ -2253,8 +2415,8 @@ def streaming_covariance(predictions,
     ValueError: If labels and predictions are of different sizes or if either
       `metrics_collections` or `updates_collections` are not a list or tuple.
   """
-  with variable_scope.variable_scope(
-      name, 'covariance', (predictions, labels, weights)):
+  with variable_scope.variable_scope(name, 'covariance',
+                                     (predictions, labels, weights)):
     predictions, labels, weights = _remove_squeezable_dimensions(
         predictions, labels, weights)
     predictions.get_shape().assert_is_compatible_with(labels.get_shape())
@@ -2298,22 +2460,22 @@ def streaming_covariance(predictions,
     # prev_mean_label is E[y_A] in the update equation
     prev_mean_label = update_mean_label - delta_mean_label
 
-    unweighted_batch_coresiduals = (
-        (predictions - batch_mean_prediction) * (labels - batch_mean_label))
+    unweighted_batch_coresiduals = ((predictions - batch_mean_prediction) *
+                                    (labels - batch_mean_label))
     # batch_comoment is C_B in the update equation
     if weights is None:
       batch_comoment = math_ops.reduce_sum(unweighted_batch_coresiduals)
     else:
-      batch_comoment = math_ops.reduce_sum(unweighted_batch_coresiduals *
-                                           weights)
+      batch_comoment = math_ops.reduce_sum(
+          unweighted_batch_coresiduals * weights)
 
     # View delta_comoment as = C_AB - C_A in the update equation above.
     # Since C_A is stored in a var, by how much do we need to increment that var
     # to make the var = C_AB?
-    delta_comoment = (batch_comoment +
-                      (prev_mean_prediction - batch_mean_prediction) *
-                      (prev_mean_label - batch_mean_label) *
-                      (prev_count * batch_count / update_count))
+    delta_comoment = (
+        batch_comoment + (prev_mean_prediction - batch_mean_prediction) *
+        (prev_mean_label - batch_mean_label) *
+        (prev_count * batch_count / update_count))
     update_comoment = state_ops.assign_add(comoment, delta_comoment)
 
     covariance = array_ops.where(
@@ -2387,8 +2549,8 @@ def streaming_pearson_correlation(predictions,
       `weights` is the wrong size, or if either `metrics_collections` or
       `updates_collections` are not a `list` or `tuple`.
   """
-  with variable_scope.variable_scope(
-      name, 'pearson_r', (predictions, labels, weights)):
+  with variable_scope.variable_scope(name, 'pearson_r',
+                                     (predictions, labels, weights)):
     predictions, labels, weights = _remove_squeezable_dimensions(
         predictions, labels, weights)
     predictions.get_shape().assert_is_compatible_with(labels.get_shape())
@@ -2405,13 +2567,14 @@ def streaming_pearson_correlation(predictions,
 
     pearson_r = math_ops.truediv(
         cov,
-        math_ops.multiply(math_ops.sqrt(var_predictions),
-                          math_ops.sqrt(var_labels)),
+        math_ops.multiply(
+            math_ops.sqrt(var_predictions), math_ops.sqrt(var_labels)),
         name='pearson_r')
     update_op = math_ops.truediv(
         update_cov,
-        math_ops.multiply(math_ops.sqrt(update_var_predictions),
-                          math_ops.sqrt(update_var_labels)),
+        math_ops.multiply(
+            math_ops.sqrt(update_var_predictions),
+            math_ops.sqrt(update_var_labels)),
         name='update_op')
 
   if metrics_collections:
@@ -2425,7 +2588,10 @@ def streaming_pearson_correlation(predictions,
 
 # TODO(nsilberman): add a 'normalized' flag so that the user can request
 # normalization if the inputs are not normalized.
-def streaming_mean_cosine_distance(predictions, labels, dim, weights=None,
+def streaming_mean_cosine_distance(predictions,
+                                   labels,
+                                   dim,
+                                   weights=None,
                                    metrics_collections=None,
                                    updates_collections=None,
                                    name=None):
@@ -2471,12 +2637,11 @@ def streaming_mean_cosine_distance(predictions, labels, dim, weights=None,
       predictions, labels, weights)
   predictions.get_shape().assert_is_compatible_with(labels.get_shape())
   radial_diffs = math_ops.multiply(predictions, labels)
-  radial_diffs = math_ops.reduce_sum(radial_diffs,
-                                     reduction_indices=[dim,],
-                                     keep_dims=True)
-  mean_distance, update_op = streaming_mean(radial_diffs, weights,
-                                            None,
-                                            None,
+  radial_diffs = math_ops.reduce_sum(
+      radial_diffs, reduction_indices=[
+          dim,
+      ], keep_dims=True)
+  mean_distance, update_op = streaming_mean(radial_diffs, weights, None, None,
                                             name or 'mean_cosine_distance')
   mean_distance = math_ops.subtract(1.0, mean_distance)
   update_op = math_ops.subtract(1.0, update_op)
@@ -2490,7 +2655,9 @@ def streaming_mean_cosine_distance(predictions, labels, dim, weights=None,
   return mean_distance, update_op
 
 
-def streaming_percentage_less(values, threshold, weights=None,
+def streaming_percentage_less(values,
+                              threshold,
+                              weights=None,
                               metrics_collections=None,
                               updates_collections=None,
                               name=None):
@@ -2530,9 +2697,12 @@ def streaming_percentage_less(values, threshold, weights=None,
       or tuple.
   """
   return metrics.percentage_below(
-      values=values, threshold=threshold, weights=weights,
+      values=values,
+      threshold=threshold,
+      weights=weights,
       metrics_collections=metrics_collections,
-      updates_collections=updates_collections, name=name)
+      updates_collections=updates_collections,
+      name=name)
 
 
 def streaming_mean_iou(predictions,
@@ -2584,9 +2754,13 @@ def streaming_mean_iou(predictions,
       tuple.
   """
   return metrics.mean_iou(
-      num_classes=num_classes, predictions=predictions, labels=labels,
-      weights=weights, metrics_collections=metrics_collections,
-      updates_collections=updates_collections, name=name)
+      num_classes=num_classes,
+      predictions=predictions,
+      labels=labels,
+      weights=weights,
+      metrics_collections=metrics_collections,
+      updates_collections=updates_collections,
+      name=name)
 
 
 def _next_array_size(required_size, growth_factor=1.5):
@@ -2601,9 +2775,9 @@ def _next_array_size(required_size, growth_factor=1.5):
     tf.Tensor with dtype=int32 giving the next array size.
   """
   exponent = math_ops.ceil(
-      math_ops.log(math_ops.cast(required_size, dtypes.float32))
-      / math_ops.log(math_ops.cast(growth_factor, dtypes.float32)))
-  return math_ops.cast(math_ops.ceil(growth_factor ** exponent), dtypes.int32)
+      math_ops.log(math_ops.cast(required_size, dtypes.float32)) / math_ops.log(
+          math_ops.cast(growth_factor, dtypes.float32)))
+  return math_ops.cast(math_ops.ceil(growth_factor**exponent), dtypes.int32)
 
 
 def streaming_concat(values,
@@ -2660,8 +2834,7 @@ def streaming_concat(values,
     if not 0 <= axis < ndim:
       raise ValueError('axis = %r not in [0, %r)' % (axis, ndim))
 
-    fixed_shape = [dim.value for n, dim in enumerate(values_shape)
-                   if n != axis]
+    fixed_shape = [dim.value for n, dim in enumerate(values_shape) if n != axis]
     if any(value is None for value in fixed_shape):
       raise ValueError('all dimensions of `values` other than the dimension to '
                        'concatenate along must have statically known size')
@@ -2804,14 +2977,14 @@ def _remove_squeezable_dimensions(predictions, labels, weights):
       # Use static rank.
       if weights_rank - predictions_rank == 1:
         weights = array_ops.squeeze(weights, [-1])
-    elif (weights_rank is None) or (
-        weights_shape.dims[-1].is_compatible_with(1)):
+    elif (weights_rank is
+          None) or (weights_shape.dims[-1].is_compatible_with(1)):
       # Use dynamic rank
       weights = control_flow_ops.cond(
-          math_ops.equal(array_ops.rank(weights),
-                         math_ops.add(array_ops.rank(predictions), 1)),
-          lambda: array_ops.squeeze(weights, [-1]),
-          lambda: weights)
+          math_ops.equal(
+              array_ops.rank(weights),
+              math_ops.add(array_ops.rank(predictions), 1)),
+          lambda: array_ops.squeeze(weights, [-1]), lambda: weights)
   return predictions, labels, weights
 
 
-- 
GitLab


From fafff08cbc3b952d60ee98914c234bb6af09b968 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 20 Oct 2017 11:57:40 -0700
Subject: [PATCH 617/909] Adds the k-MC2 algorithm for efficient seeding of
 mini batch k-means in TensorFlow.

PiperOrigin-RevId: 172914154
---
 .../contrib/factorization/g3doc/kmeans.md     |  12 +-
 .../factorization/kernels/clustering_ops.cc   |  52 +++++++
 .../kernels/clustering_ops_test.cc            |  56 ++++++++
 .../factorization/ops/clustering_ops.cc       |  19 +++
 .../kernel_tests/clustering_ops_test.py       |  57 ++++++++
 .../python/ops/clustering_ops.py              | 127 ++++++++++++++++--
 6 files changed, 307 insertions(+), 16 deletions(-)

diff --git a/tensorflow/contrib/factorization/g3doc/kmeans.md b/tensorflow/contrib/factorization/g3doc/kmeans.md
index b55c9d09ad..c1843f0bf0 100644
--- a/tensorflow/contrib/factorization/g3doc/kmeans.md
+++ b/tensorflow/contrib/factorization/g3doc/kmeans.md
@@ -24,7 +24,11 @@ the full-batch version.
 approach for computing the initial cluster assignments that is expensive but is
 typically less prone to getting stuck in bad local minima.
 
-We provide distributed implementations of both full-batch and mini-batch
-K-Means algorithm. Both K-Means++ and random initialization are supported.
-The user can also choose between **Cosine** and **Squared Euclidean** distance
-metrics.
+**[k-MC2](https://www.aaai.org/ocs/index.php/AAAI/AAAI16/paper/view/12147/11759)**
+provides a very fast seeding method that provides high quality centers
+comparable to K-Means++ seeding. k-MC2 works particularly well if it is combined
+with Mini-batch K-Means.
+
+We provide distributed implementations of both full-batch and mini-batch K-Means
+algorithm. K-Means++, k-MC2 and random initialization are supported. The user
+can also choose between **Cosine** and **Squared Euclidean** distance metrics.
diff --git a/tensorflow/contrib/factorization/kernels/clustering_ops.cc b/tensorflow/contrib/factorization/kernels/clustering_ops.cc
index a2136c08bb..dd61f59585 100644
--- a/tensorflow/contrib/factorization/kernels/clustering_ops.cc
+++ b/tensorflow/contrib/factorization/kernels/clustering_ops.cc
@@ -224,6 +224,58 @@ class KmeansPlusPlusInitializationOp : public OpKernel {
 REGISTER_KERNEL_BUILDER(Name("KmeansPlusPlusInitialization").Device(DEVICE_CPU),
                         KmeansPlusPlusInitializationOp);
 
+// Implementation of one single Markov Chain for the k-MC^2 algorithm
+class KMC2ChainInitializationOp : public OpKernel {
+ public:
+  explicit KMC2ChainInitializationOp(OpKernelConstruction* context)
+      : OpKernel(context) {
+    OP_REQUIRES_OK(context,
+                   context->MatchSignature({DT_FLOAT, DT_INT64}, {DT_INT64}));
+  }
+
+  void Compute(OpKernelContext* context) override {
+    const Tensor& distances_tensor = context->input(0);
+    const Tensor& seed_tensor = context->input(1);
+    OP_REQUIRES(context, TensorShapeUtils::IsVector(distances_tensor.shape()),
+                InvalidArgument("Input distances should be a vector."));
+    OP_REQUIRES(context, TensorShapeUtils::IsScalar(seed_tensor.shape()),
+                InvalidArgument("Input seed should be a scalar."));
+    const int64 num_points = distances_tensor.dim_size(0);
+    const int64 seed = seed_tensor.scalar<int64>()();
+    OP_REQUIRES(context, num_points > 0,
+                InvalidArgument("Expected distances_tensor.size() > 0."));
+
+    random::PhiloxRandom random(seed);
+    random::SimplePhilox rng(&random);
+
+    auto distances = distances_tensor.flat<float>();
+    // Set the initial state of the Markov chain to be the first candidate.
+    int64 selected_index = 0;
+    float selected_distance = distances(selected_index);
+    // Build a Markov chain of length num_points.
+    for (int64 i = 1; i < num_points; ++i) {
+      const float candidate_distance = distances(i);
+      // Set the next state of the Markov chain to be the candidate with
+      // probability min(1, candidate_distance/selected_distance).
+      if (candidate_distance > rng.RandFloat() * selected_distance) {
+        selected_index = i;
+        selected_distance = candidate_distance;
+      }
+    }
+
+    Tensor* output_sampled_index_tensor;
+    OP_REQUIRES_OK(context,
+                   context->allocate_output(0, TensorShape({}),
+                                            &output_sampled_index_tensor));
+    auto output = output_sampled_index_tensor->scalar<int64>();
+    // Return the last state of the Markov chain as the new center.
+    output() = selected_index;
+  }
+};
+
+REGISTER_KERNEL_BUILDER(Name("KMC2ChainInitialization").Device(DEVICE_CPU),
+                        KMC2ChainInitializationOp);
+
 // Operator for computing the nearest neighbors for a set of points.
 class NearestNeighborsOp : public OpKernel {
  public:
diff --git a/tensorflow/contrib/factorization/kernels/clustering_ops_test.cc b/tensorflow/contrib/factorization/kernels/clustering_ops_test.cc
index c4a96b048d..8172a7cebb 100644
--- a/tensorflow/contrib/factorization/kernels/clustering_ops_test.cc
+++ b/tensorflow/contrib/factorization/kernels/clustering_ops_test.cc
@@ -116,6 +116,62 @@ RUN_BM_KmeansPlusPlusInitialization(k3RetriesPerSample);
 #undef RUN_BM_KmeansPlusPlusInitialization
 #undef BENCHMARK_KMEANS_PLUS_PLUS
 
+Graph* SetUpKMC2Initialization(int num_points) {
+  Graph* g = new Graph(OpRegistry::Global());
+  Tensor distances(DT_FLOAT, TensorShape({num_points}));
+  Tensor seed(DT_INT64, TensorShape({}));
+  distances.flat<float>().setRandom();
+  seed.flat<int64>().setConstant(12345);
+
+  TF_CHECK_OK(
+      NodeBuilder("KMC2ChainInitializationOp", "KMC2ChainInitialization")
+          .Input(test::graph::Constant(g, distances))
+          .Input(test::graph::Constant(g, seed))
+          .Finalize(g, nullptr /* node */));
+  return g;
+}
+
+template <int num_points, int num_to_sample, int num_dims>
+void BM_KMC2Initialization(int iters) {
+  testing::StopTiming();
+  testing::ItemsProcessed(static_cast<int64>(iters) * num_points * num_dims *
+                          num_to_sample);
+  testing::UseRealTime();
+  Graph* g = SetUpKMC2Initialization(num_points);
+  testing::StartTiming();
+  test::Benchmark("cpu", g).Run(iters);
+}
+#define BENCHMARK_KMC2(p, c, d)                           \
+  void BM_KMC2Initialization_##p##_##c##_##d(int iters) { \
+    BM_KMC2Initialization<p, c, d>(iters);                \
+  }                                                       \
+  BENCHMARK(BM_KMC2Initialization_##p##_##c##_##d);
+
+#define RUN_BM_KMC2Initialization                   \
+  BENCHMARK_KMC2(k10Points, k2Centers, k100Dim);    \
+  BENCHMARK_KMC2(k10Points, k5Centers, k100Dim);    \
+  BENCHMARK_KMC2(k10Points, k10Centers, k100Dim);   \
+  BENCHMARK_KMC2(k100Points, k10Centers, k100Dim);  \
+  BENCHMARK_KMC2(k100Points, k20Centers, k100Dim);  \
+  BENCHMARK_KMC2(k100Points, k50Centers, k100Dim);  \
+  BENCHMARK_KMC2(k100Points, k100Centers, k100Dim); \
+  BENCHMARK_KMC2(k1kPoints, k100Centers, k100Dim);  \
+  BENCHMARK_KMC2(k1kPoints, k200Centers, k100Dim);  \
+  BENCHMARK_KMC2(k1kPoints, k500Centers, k100Dim);  \
+  BENCHMARK_KMC2(k1kPoints, k1kCenters, k100Dim);   \
+  BENCHMARK_KMC2(k10kPoints, k100Centers, k100Dim); \
+  BENCHMARK_KMC2(k10kPoints, k200Centers, k100Dim); \
+  BENCHMARK_KMC2(k10kPoints, k500Centers, k100Dim); \
+  BENCHMARK_KMC2(k10kPoints, k1kCenters, k100Dim);  \
+  BENCHMARK_KMC2(k1MPoints, k100Centers, k100Dim);  \
+  BENCHMARK_KMC2(k1MPoints, k200Centers, k100Dim);  \
+  BENCHMARK_KMC2(k1MPoints, k500Centers, k100Dim);  \
+  BENCHMARK_KMC2(k1MPoints, k1kCenters, k100Dim)
+
+RUN_BM_KMC2Initialization;
+#undef RUN_BM_KMC2Initialization
+#undef BENCHMARK_KMC2
+
 Graph* SetUpNearestNeighbors(int num_dims, int num_points, int num_centers,
                              int k) {
   Graph* g = new Graph(OpRegistry::Global());
diff --git a/tensorflow/contrib/factorization/ops/clustering_ops.cc b/tensorflow/contrib/factorization/ops/clustering_ops.cc
index f2dfcf7ed0..2686702c1d 100644
--- a/tensorflow/contrib/factorization/ops/clustering_ops.cc
+++ b/tensorflow/contrib/factorization/ops/clustering_ops.cc
@@ -44,6 +44,25 @@ num_retries_per_sample: Scalar. For each row that is sampled, this parameter
 samples: Matrix of shape (num_to_sample, d). The sampled rows.
 )");
 
+REGISTER_OP("KMC2ChainInitialization")
+    .Input("distances: float32")
+    .Input("seed: int64")
+    .Output("index: int64")
+    .SetShapeFn(shape_inference::ScalarShape)
+    .Doc(R"(
+Returns the index of a data point that should be added to the seed set.
+
+Entries in distances are assumed to be squared distances of candidate points to
+the already sampled centers in the seed set. The op constructs one Markov chain
+of the k-MC^2 algorithm and returns the index of one candidate point to be added
+as an additional cluster center.
+
+distances: Vector with squared distances to the closest previously sampled
+  cluster center for each candidate point.
+seed: Scalar. Seed for initializing the random number generator.
+index: Scalar with the index of the sampled point.
+)");
+
 REGISTER_OP("NearestNeighbors")
     .Input("points: float32")
     .Input("centers: float32")
diff --git a/tensorflow/contrib/factorization/python/kernel_tests/clustering_ops_test.py b/tensorflow/contrib/factorization/python/kernel_tests/clustering_ops_test.py
index 450f64063a..1322f7ce5f 100644
--- a/tensorflow/contrib/factorization/python/kernel_tests/clustering_ops_test.py
+++ b/tensorflow/contrib/factorization/python/kernel_tests/clustering_ops_test.py
@@ -55,6 +55,63 @@ class KmeansPlusPlusInitializationTest(test.TestCase):
       self.runTestWithSeed(seed)
 
 
+class KMC2InitializationTest(test.TestCase):
+
+  def runTestWithSeed(self, seed):
+    with self.test_session():
+      distances = np.zeros(1000).astype(np.float32)
+      distances[6] = 10e7
+      distances[4] = 10e3
+
+      sampled_point = clustering_ops.kmc2_chain_initialization(distances, seed)
+      self.assertEquals(sampled_point.eval(), 6)
+      distances[6] = 0.0
+      sampled_point = clustering_ops.kmc2_chain_initialization(distances, seed)
+      self.assertEquals(sampled_point.eval(), 4)
+
+  def testBasic(self):
+    for seed in range(100):
+      self.runTestWithSeed(seed)
+
+
+class KMC2InitializationLargeTest(test.TestCase):
+
+  def setUp(self):
+    self._distances = np.zeros(1001)
+    self._distances[500] = 100.0
+    self._distances[1000] = 50.0
+
+  def testBasic(self):
+    with self.test_session():
+      counts = {}
+      seed = 0
+      for i in range(50):
+        sample = clustering_ops.kmc2_chain_initialization(
+            self._distances, seed + i).eval()
+        counts[sample] = counts.get(sample, 0) + 1
+      self.assertEquals(len(counts), 2)
+      self.assertTrue(500 in counts)
+      self.assertTrue(1000 in counts)
+      self.assertGreaterEqual(counts[500], 5)
+      self.assertGreaterEqual(counts[1000], 5)
+
+
+class KMC2InitializationCornercaseTest(test.TestCase):
+
+  def setUp(self):
+    self._distances = np.zeros(10)
+
+  def runTestWithSeed(self, seed):
+    with self.test_session():
+      sampled_point = clustering_ops.kmc2_chain_initialization(
+          self._distances, seed)
+      self.assertEquals(sampled_point.eval(), 0)
+
+  def testBasic(self):
+    for seed in range(100):
+      self.runTestWithSeed(seed)
+
+
 # A simple test that can be verified by hand.
 class NearestCentersTest(test.TestCase):
 
diff --git a/tensorflow/contrib/factorization/python/ops/clustering_ops.py b/tensorflow/contrib/factorization/python/ops/clustering_ops.py
index d7320aeb3d..96cc80ce24 100644
--- a/tensorflow/contrib/factorization/python/ops/clustering_ops.py
+++ b/tensorflow/contrib/factorization/python/ops/clustering_ops.py
@@ -50,6 +50,7 @@ COSINE_DISTANCE = 'cosine'
 
 RANDOM_INIT = 'random'
 KMEANS_PLUS_PLUS_INIT = 'kmeans_plus_plus'
+KMC2_INIT = 'kmc2'
 
 # The name of the variable holding the cluster centers. Used by the Estimator.
 CLUSTERS_VAR_NAME = 'clusters'
@@ -66,7 +67,8 @@ class KMeans(object):
                use_mini_batch=False,
                mini_batch_steps_per_iteration=1,
                random_seed=0,
-               kmeans_plus_plus_num_retries=2):
+               kmeans_plus_plus_num_retries=2,
+               kmc2_chain_length=200):
     """Creates an object for generating KMeans clustering graph.
 
     This class implements the following variants of K-means algorithm:
@@ -95,7 +97,8 @@ class KMeans(object):
     exactly like a full-batch version.
 
     Args:
-      inputs: An input tensor or list of input tensors
+      inputs: An input tensor or list of input tensors. It is assumed that the
+        data points have been previously randomly permuted.
       num_clusters: An integer tensor specifying the number of clusters. This
         argument is ignored if initial_clusters is a tensor or numpy array.
       initial_clusters: Specifies the clusters used during initialization. One
@@ -104,6 +107,7 @@ class KMeans(object):
         - a function f(inputs, k) that returns up to k centers from `inputs`.
         - "random": Choose centers randomly from `inputs`.
         - "kmeans_plus_plus": Use kmeans++ to choose centers from `inputs`.
+        - "kmc2": Use the fast k-MC2 algorithm to choose centers from `inputs`.
         In the last three cases, one batch of `inputs` may not yield
         `num_clusters` centers, in which case initialization will require
         multiple batches until enough centers are chosen. In the case of
@@ -121,13 +125,17 @@ class KMeans(object):
         additional points to draw from the current distribution before selecting
         the best. If a negative value is specified, a heuristic is used to
         sample O(log(num_to_sample)) additional points.
+      kmc2_chain_length: Determines how many candidate points are used by the
+        k-MC2 algorithm to produce one new cluster centers. If a (mini-)batch
+        contains less points, one new cluster center is generated from the
+        (mini-)batch.
 
     Raises:
       ValueError: An invalid argument was passed to initial_clusters or
         distance_metric.
     """
     if isinstance(initial_clusters, str) and initial_clusters not in [
-        RANDOM_INIT, KMEANS_PLUS_PLUS_INIT
+        RANDOM_INIT, KMEANS_PLUS_PLUS_INIT, KMC2_INIT
     ]:
       raise ValueError(
           "Unsupported initialization algorithm '%s'" % initial_clusters)
@@ -141,6 +149,7 @@ class KMeans(object):
     self._mini_batch_steps_per_iteration = int(mini_batch_steps_per_iteration)
     self._random_seed = random_seed
     self._kmeans_plus_plus_num_retries = kmeans_plus_plus_num_retries
+    self._kmc2_chain_length = kmc2_chain_length
 
   @classmethod
   def _distance_graph(cls, inputs, clusters, distance_metric):
@@ -302,9 +311,10 @@ class KMeans(object):
     else:
       cluster_centers_updated = cluster_centers
       update_in_steps = None
-      cluster_counts = (variable_scope.variable(
-          array_ops.ones([num_clusters], dtype=dtypes.int64))
-                        if self._use_mini_batch else None)
+      cluster_counts = (
+          variable_scope.variable(
+              array_ops.ones([num_clusters], dtype=dtypes.int64))
+          if self._use_mini_batch else None)
     return (cluster_centers, cluster_centers_initialized, cluster_counts,
             cluster_centers_updated, update_in_steps)
 
@@ -359,7 +369,7 @@ class KMeans(object):
     init_op = _InitializeClustersOpFactory(
         self._inputs, num_clusters, initial_clusters, self._distance_metric,
         self._random_seed, self._kmeans_plus_plus_num_retries,
-        cluster_centers_var, cluster_centers_updated,
+        self._kmc2_chain_length, cluster_centers_var, cluster_centers_updated,
         cluster_centers_initialized).op()
     cluster_centers = cluster_centers_var
 
@@ -520,8 +530,9 @@ class KMeans(object):
                         array_ops.reshape(array_ops.shape(inp)[0], [-1])),
                     [-1, 1]), cluster_idx, num_clusters))
     with ops.colocate_with(cluster_centers, ignore_existing=True):
-      new_clusters_centers = math_ops.add_n(cluster_sums) / (math_ops.cast(
-          math_ops.add_n(cluster_counts), cluster_sums[0].dtype) + epsilon)
+      new_clusters_centers = math_ops.add_n(cluster_sums) / (
+          math_ops.cast(math_ops.add_n(cluster_counts), cluster_sums[0].dtype) +
+          epsilon)
       if self._clusters_l2_normalized():
         new_clusters_centers = nn_impl.l2_normalize(new_clusters_centers, dim=1)
     return state_ops.assign(cluster_centers, new_clusters_centers)
@@ -548,9 +559,12 @@ class _InitializeClustersOpFactory(object):
         cluster_centers_initialized := true
   """
 
+  # TODO(ccolby): Refactor this class so that kmc2 isn't so much a special case.
+
   def __init__(self, inputs, num_clusters, initial_clusters, distance_metric,
-               random_seed, kmeans_plus_plus_num_retries, cluster_centers,
-               cluster_centers_updated, cluster_centers_initialized):
+               random_seed, kmeans_plus_plus_num_retries, kmc2_chain_length,
+               cluster_centers, cluster_centers_updated,
+               cluster_centers_initialized):
     """Creates an op factory.
 
     Args:
@@ -560,6 +574,7 @@ class _InitializeClustersOpFactory(object):
       distance_metric: See KMeans constructor.
       random_seed: See KMeans constructor.
       kmeans_plus_plus_num_retries: See KMeans constructor.
+      kmc2_chain_length: See KMeans constructor.
       cluster_centers: The TF variable holding the initial centers. It may
           already contain some centers when the op is executed.
       cluster_centers_updated: A second TF variable to hold a copy of the
@@ -575,6 +590,7 @@ class _InitializeClustersOpFactory(object):
     self._distance_metric = distance_metric
     self._random_seed = random_seed
     self._kmeans_plus_plus_num_retries = kmeans_plus_plus_num_retries
+    self._kmc2_chain_length = kmc2_chain_length
     self._cluster_centers = cluster_centers
     self._cluster_centers_updated = cluster_centers_updated
     self._cluster_centers_initialized = cluster_centers_initialized
@@ -604,6 +620,90 @@ class _InitializeClustersOpFactory(object):
         math_ops.to_int64(self._num_remaining), self._random_seed,
         self._kmeans_plus_plus_num_retries)
 
+  def _kmc2_multiple_centers(self):
+    """Adds new initial cluster centers using the k-MC2 algorithm.
+
+    In each call to the op, the provided batch is split into subsets based on
+    the specified `kmc2_chain_length`. On each subset, a single Markov chain of
+    the k-MC2 algorithm is used to add *one* new center cluster center. If there
+    are less than `kmc2_chain_length` points in the subset, a single center is
+    added using one Markov chain on the full input. It is assumed that the
+    provided batch has previously been randomly permuted. Otherwise, k-MC2 may
+    return suboptimal centers.
+
+    Returns:
+      An op that adds new cluster centers.
+    """
+    # The op only operates on the first shard of data.
+    first_shard = self._inputs[0]
+    # Number of points in the input that can be used.
+    batch_size = array_ops.shape(first_shard)[0]
+    # Maximum number of subsets such that the size of each subset is at least
+    # `kmc2_chain_length`. Final subsets may be larger.
+    max_to_sample = math_ops.cast(
+        batch_size / self._kmc2_chain_length, dtype=dtypes.int32)
+    # We sample at least one new center and at most all remaining centers.
+    num_to_sample = math_ops.maximum(
+        math_ops.minimum(self._num_remaining, max_to_sample), 1)
+
+    def _cond(i, _):
+      """Stopping condition for the while loop."""
+      return math_ops.less(i, num_to_sample)
+
+    def _body(i, _):
+      """Body that adds a single new center based on a subset."""
+
+      def _sample_random():
+        """Returns a random point as a cluster center."""
+        # By assumption the batch is reshuffled and _sample_random is always
+        # called for i=0. Hence, we simply return the first point.
+        new_center = array_ops.reshape(first_shard[0], [1, -1])
+        if self._distance_metric == COSINE_DISTANCE:
+          new_center = nn_impl.l2_normalize(new_center, dim=1)
+        return new_center
+
+      def _sample_kmc2_chain():
+        """Returns previous centers as well as a new center sampled using k-MC2.
+        """
+        # Extract the subset from the underlying batch.
+        start = i * self._kmc2_chain_length
+        end = start + self._kmc2_chain_length
+        subset = first_shard[start:end]
+        # Compute the distances from points in the subset to previous centers.
+        _, distances = gen_clustering_ops.nearest_neighbors(
+            subset, self._cluster_centers, 1)
+        # Sample index of new center using k-MC2 Markov chain.
+        new_center_index = gen_clustering_ops.kmc2_chain_initialization(
+            array_ops.squeeze(distances), self._random_seed)
+        # Extract actual new center.
+        newly_sampled_center = array_ops.reshape(subset[new_center_index],
+                                                 [1, -1])
+        # Return concatenation with previously sampled centers.
+        if self._distance_metric == COSINE_DISTANCE:
+          newly_sampled_center = nn_impl.l2_normalize(
+              newly_sampled_center, dim=1)
+        return array_ops.concat([self._cluster_centers, newly_sampled_center],
+                                0)
+
+      # Obtain a random point if there are no previously sampled centers.
+      # Otherwise, construct a k-MC2 Markov chain.
+      new_centers = control_flow_ops.cond(
+          math_ops.equal(self._num_selected, 0), _sample_random,
+          _sample_kmc2_chain)
+      # Assign new cluster centers to underlying variable.
+      assigned_centers = state_ops.assign(
+          self._cluster_centers, new_centers, validate_shape=False)
+      if self._cluster_centers_updated is not self._cluster_centers:
+        assigned_centers = state_ops.assign(
+            self._cluster_centers_updated,
+            assigned_centers,
+            validate_shape=False)
+      return i + 1, self._num_clusters - array_ops.shape(assigned_centers)[0]
+
+    # Add num_to_sample new data points.
+    _, num_remaining = control_flow_ops.while_loop(_cond, _body, [0, 0])
+    return num_remaining
+
   def _greedy_batch_sampler(self, sampler):
     # If the input dataset size is smaller than the number of centers
     # remaining, choose the entire input dataset as centers. This can happen
@@ -657,7 +757,10 @@ class _InitializeClustersOpFactory(object):
     with ops.control_dependencies([
         check_ops.assert_positive(self._num_remaining),
     ]):
-      num_now_remaining = self._add_new_centers()
+      if self._initial_clusters == KMC2_INIT:
+        num_now_remaining = self._kmc2_multiple_centers()
+      else:
+        num_now_remaining = self._add_new_centers()
       return control_flow_ops.cond(
           math_ops.equal(num_now_remaining, 0),
           lambda: state_ops.assign(self._cluster_centers_initialized, True),
-- 
GitLab


From 58bdae2f8b7bead45537092f39f6fa6fd15c50d0 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 20 Oct 2017 12:08:44 -0700
Subject: [PATCH 618/909] Work around for compiler bug in GCC on Android.

PiperOrigin-RevId: 172915900
---
 tensorflow/core/kernels/transpose_functor.h | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/tensorflow/core/kernels/transpose_functor.h b/tensorflow/core/kernels/transpose_functor.h
index 9781fe3b61..add4635331 100644
--- a/tensorflow/core/kernels/transpose_functor.h
+++ b/tensorflow/core/kernels/transpose_functor.h
@@ -201,17 +201,26 @@ Status DoTransposeImpl(const Device& d, const Tensor& in,
 
     case DT_COMPLEX64:
       if (conjugate) {
-        Transpose<Device, complex64, true>::run(d, in, perm, out);
+#if defined(__ANDROID__) and !defined(__clang__)
+        // Workaround for GCC compiler bug in Android toolchain.
+        return errors::Unimplemented(
+            "Conjugate transpose of complex64 not supported for GCC on "
+            "Android.");
+#else
+        Transpose<Device, complex64, /*conjugate=*/true>::run(d, in, perm, out);
+#endif
       } else {
-        Transpose<Device, complex64, false>::run(d, in, perm, out);
+        Transpose<Device, uint64>::run(d, in, perm, out);
       }
       break;
 
     case DT_COMPLEX128:
       if (conjugate) {
-        Transpose<Device, complex128, true>::run(d, in, perm, out);
+        Transpose<Device, complex128, /*conjugate=*/true>::run(d, in, perm,
+                                                               out);
       } else {
-        Transpose<Device, complex128, false>::run(d, in, perm, out);
+        Transpose<Device, complex128, /*conjugate=*/false>::run(d, in, perm,
+                                                                out);
       }
       break;
 
-- 
GitLab


From b68a3f2e445cdc749f380387b910f6eac72e5dcf Mon Sep 17 00:00:00 2001
From: Yunxing Dai <yunxing@google.com>
Date: Fri, 20 Oct 2017 12:26:09 -0700
Subject: [PATCH 619/909] Iterating through a map in protobuf is essentially
 nondeterministic. This CL enables us to traverse the map in a deterministic
 order by sorting the keys first.

PiperOrigin-RevId: 172918084
---
 tensorflow/compiler/xla/service/user_computation.cc | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/tensorflow/compiler/xla/service/user_computation.cc b/tensorflow/compiler/xla/service/user_computation.cc
index b3506b72bf..065d2580c6 100644
--- a/tensorflow/compiler/xla/service/user_computation.cc
+++ b/tensorflow/compiler/xla/service/user_computation.cc
@@ -20,6 +20,7 @@ limitations under the License.
 #include <stack>
 #include <unordered_map>
 #include <utility>
+#include <vector>
 
 #include "tensorflow/compiler/xla/layout_util.h"
 #include "tensorflow/compiler/xla/literal_util.h"
@@ -1843,10 +1844,17 @@ UserComputation::GetEmbeddedComputations(
   XLA_VLOG_LINES(3, session_computation_.DebugString());
 
   std::vector<VersionedComputationHandle> computations;
+  std::vector<int64> sorted_handles;
   for (const auto& handle_request : session_computation_.requests()) {
-    int64 handle_value = handle_request.first;
+    sorted_handles.push_back(handle_request.first);
+  }
+  std::sort(sorted_handles.begin(), sorted_handles.end());
+  for (int64 handle : sorted_handles) {
+    const auto& handle_request = session_computation_.requests().find(handle);
+    CHECK(handle_request != session_computation_.requests().end());
+    int64 handle_value = handle_request->first;
     if (handle_value <= version) {
-      const OperationRequest& request = handle_request.second;
+      const OperationRequest& request = handle_request->second;
       switch (request.request().op_case()) {
         case OpRequest::kCallRequest: {
           CHECK_EQ(1, request.embedded_computation_versions_size());
-- 
GitLab


From aada11e19a1ceb901f490aa3c064f2778cb2acf2 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 20 Oct 2017 12:46:29 -0700
Subject: [PATCH 620/909] Exposes the read_batch_size argument to
 read_batch_features.

PiperOrigin-RevId: 172920603
---
 .../learn/python/learn/learn_io/graph_io.py        | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/tensorflow/contrib/learn/python/learn/learn_io/graph_io.py b/tensorflow/contrib/learn/python/learn/learn_io/graph_io.py
index bdb88b89bb..4b34fc6284 100644
--- a/tensorflow/contrib/learn/python/learn/learn_io/graph_io.py
+++ b/tensorflow/contrib/learn/python/learn/learn_io/graph_io.py
@@ -442,7 +442,8 @@ def read_keyed_batch_features(file_pattern,
                               feature_queue_capacity=100,
                               num_enqueue_threads=2,
                               parse_fn=None,
-                              name=None):
+                              name=None,
+                              read_batch_size=None):
   """Adds operations to read, queue, batch and parse `Example` protos.
 
   Given file pattern (or list of files), will setup a queue for file names,
@@ -482,6 +483,8 @@ def read_keyed_batch_features(file_pattern,
     parse_fn: Parsing function, takes `Example` Tensor returns parsed
       representation. If `None`, no parsing is done.
     name: Name of resulting op.
+    read_batch_size: An int or scalar `Tensor` specifying the number of
+      records to read at once. If `None`, defaults to `batch_size`.
 
   Returns:
     Returns tuple of:
@@ -493,6 +496,7 @@ def read_keyed_batch_features(file_pattern,
   """
 
   with ops.name_scope(name, 'read_batch_features', [file_pattern]) as scope:
+    if read_batch_size is None: read_batch_size = batch_size
     keys, examples = read_keyed_batch_examples(
         file_pattern,
         batch_size,
@@ -501,7 +505,7 @@ def read_keyed_batch_features(file_pattern,
         num_epochs=num_epochs,
         queue_capacity=queue_capacity,
         num_threads=reader_num_threads,
-        read_batch_size=batch_size,
+        read_batch_size=read_batch_size,
         parse_fn=parse_fn,
         name=scope)
     # Parse the example.
@@ -727,7 +731,8 @@ def read_batch_features(file_pattern,
                         reader_num_threads=1,
                         num_enqueue_threads=2,
                         parse_fn=None,
-                        name=None):
+                        name=None,
+                        read_batch_size=None):
   """Adds operations to read, queue, batch and parse `Example` protos.
 
   Given file pattern (or list of files), will setup a queue for file names,
@@ -768,6 +773,8 @@ def read_batch_features(file_pattern,
     parse_fn: Parsing function, takes `Example` Tensor returns parsed
       representation. If `None`, no parsing is done.
     name: Name of resulting op.
+    read_batch_size: An int or scalar `Tensor` specifying the number of
+      records to read at once. If `None`, defaults to `batch_size`.
 
   Returns:
     A dict of `Tensor` or `SparseTensor` objects for each in `features`.
@@ -786,6 +793,7 @@ def read_batch_features(file_pattern,
       reader_num_threads=reader_num_threads,
       feature_queue_capacity=feature_queue_capacity,
       num_enqueue_threads=num_enqueue_threads,
+      read_batch_size=read_batch_size,
       parse_fn=parse_fn,
       name=name)
   return features
-- 
GitLab


From 5c331cfd573984287778aab02794dd86ba1f3006 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 20 Oct 2017 12:47:57 -0700
Subject: [PATCH 621/909] The new array class provides a way to simplify the
 implementation of these classes by eliminating a large number of duplicated
 code.

Removing the old API is non-trivial because of the existing users
outside of tensorflow.

PiperOrigin-RevId: 172920837
---
 .../compiler/xla/client/computation_builder.h |  41 +++---
 tensorflow/compiler/xla/layout_util.cc        |   4 +
 tensorflow/compiler/xla/layout_util.h         |   1 +
 tensorflow/compiler/xla/literal_util.h        | 121 ++++++++----------
 4 files changed, 85 insertions(+), 82 deletions(-)

diff --git a/tensorflow/compiler/xla/client/computation_builder.h b/tensorflow/compiler/xla/client/computation_builder.h
index cdd9c8847f..93c2a80678 100644
--- a/tensorflow/compiler/xla/client/computation_builder.h
+++ b/tensorflow/compiler/xla/client/computation_builder.h
@@ -138,6 +138,11 @@ class ComputationBuilder {
   ComputationDataHandle ConstantR2(
       std::initializer_list<std::initializer_list<NativeT>> values);
   template <typename NativeT>
+  ComputationDataHandle ConstantFromArrayWithLayout(
+      const Array<NativeT>& values, const Layout& layout);
+  template <typename NativeT>
+  ComputationDataHandle ConstantFromArray(const Array<NativeT>& values);
+  template <typename NativeT>
   ComputationDataHandle ConstantR2FromArray2DWithLayout(
       const Array2D<NativeT>& values, const Layout& layout);
   template <typename NativeT>
@@ -910,48 +915,54 @@ ComputationDataHandle ComputationBuilder::ConstantR2(
 }
 
 template <typename NativeT>
-ComputationDataHandle ComputationBuilder::ConstantR2FromArray2DWithLayout(
-    const Array2D<NativeT>& values, const Layout& layout) {
+ComputationDataHandle ComputationBuilder::ConstantFromArrayWithLayout(
+    const Array<NativeT>& values, const Layout& layout) {
   return ConstantOp([&values, &layout](Literal* literal) {
-    literal->PopulateR2FromArray2DWithLayout(values, layout);
+    literal->PopulateFromArrayWithLayout(values, layout);
   });
 }
 
+template <typename NativeT>
+ComputationDataHandle ComputationBuilder::ConstantFromArray(
+    const Array<NativeT>& values) {
+  return ConstantOp(
+      [&values](Literal* literal) { literal->PopulateFromArray(values); });
+}
+
+template <typename NativeT>
+ComputationDataHandle ComputationBuilder::ConstantR2FromArray2DWithLayout(
+    const Array2D<NativeT>& values, const Layout& layout) {
+  return ConstantFromArrayWithLayout(values, layout);
+}
+
 template <typename NativeT>
 ComputationDataHandle ComputationBuilder::ConstantR2FromArray2D(
     const Array2D<NativeT>& values) {
-  return ConstantOp(
-      [&values](Literal* literal) { literal->PopulateR2FromArray2D(values); });
+  return ConstantFromArray(values);
 }
 
 template <typename NativeT>
 ComputationDataHandle ComputationBuilder::ConstantR3FromArray3DWithLayout(
     const Array3D<NativeT>& values, const Layout& layout) {
-  return ConstantOp([&values, &layout](Literal* literal) {
-    literal->PopulateR3FromArray3DWithLayout(values, layout);
-  });
+  return ConstantFromArrayWithLayout(values, layout);
 }
 
 template <typename NativeT>
 ComputationDataHandle ComputationBuilder::ConstantR3FromArray3D(
     const Array3D<NativeT>& values) {
-  return ConstantOp(
-      [&values](Literal* literal) { literal->PopulateR3FromArray3D(values); });
+  return ConstantFromArray(values);
 }
 
 template <typename NativeT>
 ComputationDataHandle ComputationBuilder::ConstantR4FromArray4DWithLayout(
     const Array4D<NativeT>& values, const Layout& layout) {
-  return ConstantOp([&values, &layout](Literal* literal) {
-    literal->PopulateR4FromArray4DWithLayout(values, layout);
-  });
+  return ConstantFromArrayWithLayout(values, layout);
 }
 
 template <typename NativeT>
 ComputationDataHandle ComputationBuilder::ConstantR4FromArray4D(
     const Array4D<NativeT>& values) {
-  return ConstantOp(
-      [&values](Literal* literal) { literal->PopulateR4FromArray4D(values); });
+  return ConstantFromArray(values);
 }
 
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/layout_util.cc b/tensorflow/compiler/xla/layout_util.cc
index 011fc3c194..5c2cc2a7a9 100644
--- a/tensorflow/compiler/xla/layout_util.cc
+++ b/tensorflow/compiler/xla/layout_util.cc
@@ -83,6 +83,10 @@ Layout CreateDefaultLayoutForRank(int64 rank) {
   return CreateDefaultLayoutForRank(shape.dimensions_size());
 }
 
+/* static */ Layout LayoutUtil::GetDefaultLayoutForRank(int64 rank) {
+  return CreateDefaultLayoutForRank(rank);
+}
+
 /* static */ Layout LayoutUtil::GetDefaultLayoutForR2() {
   return CreateDefaultLayoutForRank(2);
 }
diff --git a/tensorflow/compiler/xla/layout_util.h b/tensorflow/compiler/xla/layout_util.h
index 5de0a653f6..bc42e22229 100644
--- a/tensorflow/compiler/xla/layout_util.h
+++ b/tensorflow/compiler/xla/layout_util.h
@@ -40,6 +40,7 @@ class LayoutUtil {
   static Layout GetDefaultLayoutForShape(const Shape& shape);
 
   // Helper functions that create default layouts for various ranks.
+  static Layout GetDefaultLayoutForRank(int64 rank);
   static Layout GetDefaultLayoutForR2();
   static Layout GetDefaultLayoutForR3();
   static Layout GetDefaultLayoutForR4();
diff --git a/tensorflow/compiler/xla/literal_util.h b/tensorflow/compiler/xla/literal_util.h
index e8cee732d4..4063cb05a9 100644
--- a/tensorflow/compiler/xla/literal_util.h
+++ b/tensorflow/compiler/xla/literal_util.h
@@ -334,6 +334,11 @@ class Literal {
   // WithLayout use the default XLA layout for the literal's linear
   // representation in memory.
   template <typename NativeT>
+  static std::unique_ptr<Literal> CreateFromArray(const Array<NativeT>& values);
+  template <typename NativeT>
+  static std::unique_ptr<Literal> CreateFromArrayWithLayout(
+      const Array<NativeT>& values, const Layout& layout);
+  template <typename NativeT>
   static std::unique_ptr<Literal> CreateR2FromArray2D(
       const Array2D<NativeT>& values);
   template <typename NativeT>
@@ -481,6 +486,11 @@ class Literal {
       std::initializer_list<std::initializer_list<NativeT>> values,
       const Layout& layout);
   template <typename NativeT>
+  void PopulateFromArray(const Array<NativeT>& values);
+  template <typename NativeT>
+  void PopulateFromArrayWithLayout(const Array<NativeT>& values,
+                                   const Layout& layout);
+  template <typename NativeT>
   void PopulateR2FromArray2D(const Array2D<NativeT>& values);
   template <typename NativeT>
   void PopulateR2FromArray2DWithLayout(const Array2D<NativeT>& values,
@@ -816,33 +826,42 @@ template <typename NativeT>
 }
 
 template <typename NativeT>
-/* static */ std::unique_ptr<Literal> Literal::CreateR2FromArray2DWithLayout(
-    const Array2D<NativeT>& values, const Layout& layout) {
+/* static */ std::unique_ptr<Literal> Literal::CreateFromArrayWithLayout(
+    const Array<NativeT>& values, const Layout& layout) {
   auto literal = MakeUnique<Literal>();
-  literal->PopulateR2FromArray2DWithLayout(values, layout);
+  literal->PopulateFromArrayWithLayout(values, layout);
   return literal;
 }
 
+template <typename NativeT>
+/* static */ std::unique_ptr<Literal> Literal::CreateFromArray(
+    const Array<NativeT>& values) {
+  return CreateFromArrayWithLayout(
+      values, LayoutUtil::GetDefaultLayoutForRank(values.num_dimensions()));
+}
+
+template <typename NativeT>
+/* static */ std::unique_ptr<Literal> Literal::CreateR2FromArray2DWithLayout(
+    const Array2D<NativeT>& values, const Layout& layout) {
+  return CreateFromArrayWithLayout(values, layout);
+}
+
 template <typename NativeT>
 /* static */ std::unique_ptr<Literal> Literal::CreateR2FromArray2D(
     const Array2D<NativeT>& values) {
-  return CreateR2FromArray2DWithLayout(values,
-                                       LayoutUtil::GetDefaultLayoutForR2());
+  return CreateFromArray(values);
 }
 
 template <typename NativeT>
 /* static */ std::unique_ptr<Literal> Literal::CreateR3FromArray3DWithLayout(
     const Array3D<NativeT>& values, const Layout& layout) {
-  auto literal = MakeUnique<Literal>();
-  literal->PopulateR3FromArray3DWithLayout(values, layout);
-  return literal;
+  return CreateFromArrayWithLayout(values, layout);
 }
 
 template <typename NativeT>
 /* static */ std::unique_ptr<Literal> Literal::CreateR3FromArray3D(
     const Array3D<NativeT>& values) {
-  return CreateR3FromArray3DWithLayout(values,
-                                       LayoutUtil::GetDefaultLayoutForR3());
+  return CreateFromArray(values);
 }
 
 template <typename NativeT>
@@ -901,16 +920,13 @@ template <typename NativeT>
 template <typename NativeT>
 /* static */ std::unique_ptr<Literal> Literal::CreateR4FromArray4D(
     const Array4D<NativeT>& values) {
-  return CreateR4FromArray4DWithLayout(values,
-                                       LayoutUtil::GetDefaultLayoutForR4());
+  return CreateFromArray(values);
 }
 
 template <typename NativeT>
 /* static */ std::unique_ptr<Literal> Literal::CreateR4FromArray4DWithLayout(
     const Array4D<NativeT>& values, const Layout& layout) {
-  auto literal = MakeUnique<Literal>();
-  literal->PopulateR4FromArray4DWithLayout(values, layout);
-  return literal;
+  return CreateFromArrayWithLayout(values, layout);
 }
 
 template <typename NativeT>
@@ -1070,82 +1086,53 @@ void Literal::PopulateR2(
 }
 
 template <typename NativeT>
-void Literal::PopulateR2FromArray2DWithLayout(const Array2D<NativeT>& values,
-                                              const Layout& layout) {
+void Literal::PopulateFromArrayWithLayout(const Array<NativeT>& values,
+                                          const Layout& layout) {
   *mutable_shape() = ShapeUtil::MakeShapeWithLayout(
-      primitive_util::NativeToPrimitiveType<NativeT>(),
-      {values.height(), values.width()}, AsInt64Slice(layout.minor_to_major()));
+      primitive_util::NativeToPrimitiveType<NativeT>(), values.dimensions(),
+      AsInt64Slice(layout.minor_to_major()));
+  Reserve(values.num_elements());
+  values.Each([this](tensorflow::gtl::ArraySlice<int64> indices,
+                     NativeT value) { this->Set(indices, value); });
+}
 
-  const int64 dim1_size = values.width();
-  const int64 dim0_size = values.height();
-  CHECK_EQ(dim0_size, shape().dimensions(0));
-  CHECK_EQ(dim1_size, shape().dimensions(1));
-  Reserve(dim1_size * dim0_size);
-  for (int64 dim0 = 0; dim0 < dim0_size; ++dim0) {
-    for (int64 dim1 = 0; dim1 < dim1_size; ++dim1) {
-      Set({dim0, dim1}, values(dim0, dim1));
-    }
-  }
+template <typename NativeT>
+void Literal::PopulateFromArray(const Array<NativeT>& values) {
+  PopulateFromArrayWithLayout(
+      values, LayoutUtil::GetDefaultLayoutForRank(values.num_dimensions()));
+}
+
+template <typename NativeT>
+void Literal::PopulateR2FromArray2DWithLayout(const Array2D<NativeT>& values,
+                                              const Layout& layout) {
+  PopulateFromArrayWithLayout(values, layout);
 }
 
 template <typename NativeT>
 void Literal::PopulateR2FromArray2D(const Array2D<NativeT>& values) {
-  PopulateR2FromArray2DWithLayout(values, LayoutUtil::GetDefaultLayoutForR2());
+  PopulateFromArray(values);
 }
 
 template <typename NativeT>
 void Literal::PopulateR3FromArray3DWithLayout(const Array3D<NativeT>& values,
                                               const Layout& layout) {
-  *mutable_shape() = ShapeUtil::MakeShapeWithLayout(
-      primitive_util::NativeToPrimitiveType<NativeT>(),
-      {values.n1(), values.n2(), values.n3()},
-      AsInt64Slice(layout.minor_to_major()));
-
-  CHECK_EQ(values.n1(), shape().dimensions(0));
-  CHECK_EQ(values.n2(), shape().dimensions(1));
-  CHECK_EQ(values.n3(), shape().dimensions(2));
-  Reserve(values.n1() * values.n2() * values.n3());
-  for (int64 dim0 = 0; dim0 < values.n1(); ++dim0) {
-    for (int64 dim1 = 0; dim1 < values.n2(); ++dim1) {
-      for (int64 dim2 = 0; dim2 < values.n3(); ++dim2) {
-        Set({dim0, dim1, dim2}, values(dim0, dim1, dim2));
-      }
-    }
-  }
+  PopulateFromArrayWithLayout(values, layout);
 }
 
 template <typename NativeT>
 void Literal::PopulateR3FromArray3D(const Array3D<NativeT>& values) {
-  PopulateR3FromArray3DWithLayout(values, LayoutUtil::GetDefaultLayoutForR3());
+  PopulateFromArray(values);
 }
 
 template <typename NativeT>
 void Literal::PopulateR4FromArray4DWithLayout(const Array4D<NativeT>& values,
                                               const Layout& layout) {
-  *mutable_shape() = ShapeUtil::MakeShapeWithLayout(
-      primitive_util::NativeToPrimitiveType<NativeT>(),
-      {values.planes(), values.depth(), values.height(), values.width()},
-      AsInt64Slice(layout.minor_to_major()));
-
-  CHECK_EQ(values.n1(), shape().dimensions(0));
-  CHECK_EQ(values.n2(), shape().dimensions(1));
-  CHECK_EQ(values.n3(), shape().dimensions(2));
-  CHECK_EQ(values.n4(), shape().dimensions(3));
-  Reserve(values.n1() * values.n2() * values.n3() * values.n4());
-  for (int64 dim0 = 0; dim0 < values.n1(); ++dim0) {
-    for (int64 dim1 = 0; dim1 < values.n2(); ++dim1) {
-      for (int64 dim2 = 0; dim2 < values.n3(); ++dim2) {
-        for (int64 dim3 = 0; dim3 < values.n4(); ++dim3) {
-          Set({dim0, dim1, dim2, dim3}, values(dim0, dim1, dim2, dim3));
-        }
-      }
-    }
-  }
+  PopulateFromArrayWithLayout(values, layout);
 }
 
 template <typename NativeT>
 void Literal::PopulateR4FromArray4D(const Array4D<NativeT>& values) {
-  PopulateR4FromArray4DWithLayout(values, LayoutUtil::GetDefaultLayoutForR4());
+  PopulateFromArray(values);
 }
 
 template <typename NativeT, typename FnType>
-- 
GitLab


From 5bb971864220e0afdb5587680f444d3779a0f2cf Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Fri, 20 Oct 2017 12:56:51 -0700
Subject: [PATCH 622/909] TFE: Raises an error when attempting to save multiple
 ResourceVariable objects with the same shared_name.

The only way to get multiple objects is if they're created in different Graphs/IsolateTest contexts. Previously this snuck by because of a list -> dictionary conversion without key checking.

Allows the same object to be passed multiple times (so people don't need to de-duplicate their lists).

PiperOrigin-RevId: 172921932
---
 tensorflow/contrib/eager/python/saver_test.py | 34 +++++++++++++++++++
 tensorflow/python/training/saver.py           |  9 ++++-
 2 files changed, 42 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/eager/python/saver_test.py b/tensorflow/contrib/eager/python/saver_test.py
index c89554e6dd..1605435d8d 100644
--- a/tensorflow/contrib/eager/python/saver_test.py
+++ b/tensorflow/contrib/eager/python/saver_test.py
@@ -54,6 +54,40 @@ class SaverTest(test.TestCase):
       saver.restore(ckpt_prefix)
       self.assertEqual(v1.read_value().numpy(), 1.0)
 
+  def testSameNameNoClobbering(self):
+    with context.eager_mode(), ops.device(self._dev()):
+      # Note that this test purposefully uses Graphs rather than
+      # IsolateTest. Users are more likely to accidentally create the same
+      # variable name this way.
+      first_graph = ops.Graph()
+      with first_graph.as_default():
+        v1_first_graph = resource_variable_ops.ResourceVariable(1.0, name='v1')
+      with ops.Graph().as_default():
+        v1_second_graph = resource_variable_ops.ResourceVariable(2.0, name='v1')
+        saver = _saver.Saver([v1_first_graph, v1_second_graph])
+      ckpt_prefix = os.path.join(test.get_temp_dir(), 'ckpt')
+      with self.assertRaisesRegexp(ValueError, 'v1'):
+        saver.save(ckpt_prefix)
+
+  def testDifferentGraphError(self):
+    with context.eager_mode(), ops.device(self._dev()):
+      with ops.Graph().as_default():
+        v1 = resource_variable_ops.ResourceVariable(1.0, name='v1')
+      with ops.Graph().as_default():
+        saver = _saver.Saver([v1])
+        ckpt_prefix = os.path.join(test.get_temp_dir(), 'ckpt')
+        with self.assertRaisesRegexp(ValueError, 'Graph'):
+          saver.save(ckpt_prefix)
+
+  def testSameObjectOK(self):
+    with context.eager_mode(), ops.device(self._dev()):
+      v1 = resource_variable_ops.ResourceVariable(1.0, name='v1')
+      # While different objects with the same shared_name are not good, passing
+      # in the same object multiple times is fine.
+      saver = _saver.Saver([v1, v1])
+      ckpt_prefix = os.path.join(test.get_temp_dir(), 'ckpt')
+      saver.save(ckpt_prefix)
+
   def testRestoreOnCreate(self):
     with ops.device(self._dev()):
       def model(init_val):
diff --git a/tensorflow/python/training/saver.py b/tensorflow/python/training/saver.py
index b1926f4eaf..c4c1df22eb 100644
--- a/tensorflow/python/training/saver.py
+++ b/tensorflow/python/training/saver.py
@@ -557,7 +557,14 @@ class BaseSaverBuilder(object):
           if not isinstance(var, resource_variable_ops.ResourceVariable):
             raise ValueError("Can only save/restore ResourceVariable eager "
                              "mode is enabled, type: %s." % type(var))
-          names_to_saveables[var._shared_name] = var
+          set_var = names_to_saveables.setdefault(var._shared_name, var)
+          if set_var is not var:
+            raise ValueError(
+                ("Two different ResourceVariable objects with the same "
+                 "shared_name '%s' were passed to the Saver. This likely means "
+                 "that they were created in different Graphs or isolation "
+                 "contexts, and may not be checkpointed together.") % (
+                     var._shared_name,))
 
       # pylint: enable=protected-access
     return names_to_saveables
-- 
GitLab


From 54503483ef987c6488d7bc2bd3c4b1d34fbd1f26 Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Fri, 20 Oct 2017 13:01:41 -0700
Subject: [PATCH 623/909] Enables silent copies of eager tensors for
 specially-constructed contexts.

PiperOrigin-RevId: 172922467
---
 tensorflow/c/eager/BUILD            |  3 +-
 tensorflow/c/eager/c_api.cc         | 64 +++++++++++++++++++++++++----
 tensorflow/c/eager/c_api.h          | 16 ++++++++
 tensorflow/c/eager/c_api_internal.h |  3 ++
 tensorflow/c/eager/c_api_test.cc    | 46 +++++++++++++++++++++
 5 files changed, 122 insertions(+), 10 deletions(-)

diff --git a/tensorflow/c/eager/BUILD b/tensorflow/c/eager/BUILD
index 96f3c3e195..c77896b80b 100644
--- a/tensorflow/c/eager/BUILD
+++ b/tensorflow/c/eager/BUILD
@@ -3,6 +3,7 @@ licenses(["notice"])  # Apache 2.0
 
 load(
     "//tensorflow:tensorflow.bzl",
+    "tf_cuda_cc_test",
     "tf_cc_test",
     "tf_copts",
     "tf_cuda_library",
@@ -50,7 +51,7 @@ tf_cuda_library(
     ],
 )
 
-tf_cc_test(
+tf_cuda_cc_test(
     name = "c_api_test",
     srcs = ["c_api_test.cc"],
     deps = [
diff --git a/tensorflow/c/eager/c_api.cc b/tensorflow/c/eager/c_api.cc
index 334c02bff9..28ea2edee4 100644
--- a/tensorflow/c/eager/c_api.cc
+++ b/tensorflow/c/eager/c_api.cc
@@ -61,6 +61,11 @@ void TFE_ContextOptionsSetConfig(TFE_ContextOptions* options, const void* proto,
   TF_SetConfig(&options->session_options, proto, proto_len, status);
 }
 
+void TFE_ContextOptionsSetDevicePlacementPolicy(
+    TFE_ContextOptions* options, TFE_ContextDevicePlacementPolicy policy) {
+  options->policy = policy;
+}
+
 void TFE_DeleteContextOptions(TFE_ContextOptions* options) { delete options; }
 
 TFE_Context* TFE_NewContext(const TFE_ContextOptions* opts, TF_Status* status) {
@@ -80,6 +85,7 @@ TFE_Context* TFE_NewContext(const TFE_ContextOptions* opts, TF_Status* status) {
   }
 
   TFE_Context* ret = new TFE_Context(session);
+  ret->policy = opts->policy;
   ret->pflr.reset(new tensorflow::ProcessFunctionLibraryRuntime(
       ret->session->device_mgr, opts->session_options.options.env,
       TF_GRAPH_DEF_VERSION, &ret->func_lib_def, {}));
@@ -417,8 +423,10 @@ void TFE_OpSetAttrShapeList(TFE_Op* op, const char* attr_name,
 namespace {
 
 tensorflow::Status ValidateInputTypeAndPlacement(
-    tensorflow::Device* host_device, tensorflow::Device* op_device, TFE_Op* op,
-    const tensorflow::OpKernel* kernel) {
+    TFE_Context* ctx, tensorflow::Device* host_device,
+    tensorflow::Device* op_device, TFE_Op* op,
+    const tensorflow::OpKernel* kernel,
+    std::vector<TFE_TensorHandle*>* copied_tensors) {
   const tensorflow::MemoryTypeVector& memtypes = kernel->input_memory_types();
   if (memtypes.size() != op->inputs.size()) {
     return tensorflow::errors::InvalidArgument(
@@ -430,11 +438,42 @@ tensorflow::Status ValidateInputTypeAndPlacement(
     const tensorflow::Device* actual_device =
         op->input_devices[i] == nullptr ? host_device : op->input_devices[i];
     if (expected_device != actual_device) {
-      return tensorflow::errors::InvalidArgument(
-          "cannot compute ", op->name, " as input #", i,
-          " was expected to be on ", expected_device->name(),
-          " but is actually on ", actual_device->name(),
-          " (operation running on ", op_device->name(), ")");
+      switch (ctx->policy) {
+        case TFE_DEVICE_PLACEMENT_EXPLICIT:
+          return tensorflow::errors::InvalidArgument(
+              "cannot compute ", op->name, " as input #", i,
+              " was expected to be on ", expected_device->name(),
+              " but is actually on ", actual_device->name(),
+              " (operation running on ", op_device->name(), ")");
+        case TFE_DEVICE_PLACEMENT_WARN:
+          LOG(WARNING) << "before computing " << op->name << " input #" << i
+                       << " was expected to be on " << expected_device->name()
+                       << " but is actually on " << actual_device->name()
+                       << " (operation running on " << op_device->name()
+                       << "). This triggers a copy which can be a performance "
+                          "bottleneck.";
+          break;
+        case TFE_DEVICE_PLACEMENT_SILENT:  // Do nothing.
+          break;
+      }
+      // We are only here if the policy is warn or silent copies, so we should
+      // trigger a copy.
+      TFE_TensorHandle original{op->inputs[i], op->input_devices[i]};
+      TF_Status* s = TF_NewStatus();
+      TFE_TensorHandle* copied_tensor = TFE_TensorHandleCopyToDevice(
+          &original, ctx, expected_device->name().c_str(), s);
+      if (!s->status.ok()) {
+        tensorflow::Status status = s->status;
+        delete s;
+        return tensorflow::errors::Internal(
+            "Failed copying input tensor from ", actual_device->name(), " to ",
+            expected_device->name(), " in order to run ", op->name, ": ",
+            status.error_message());
+      }
+      op->inputs[i] = copied_tensor->t;
+      copied_tensors->push_back(copied_tensor);
+      op->input_devices[i] = copied_tensor->d;
+      delete s;
     }
     if (op->inputs[i].dtype() != kernel->input_type(i)) {
       return tensorflow::errors::InvalidArgument(
@@ -477,10 +516,14 @@ void TFE_Execute(TFE_Op* op, TFE_TensorHandle** retvals, int* num_retvals,
     }
     tensorflow::gtl::InsertOrUpdate(&(ctx->kernel_cache), cache_key, kernel);
   }
-  status->status = ValidateInputTypeAndPlacement(ctx->devices()[0], device, op,
-                                                 kernel->kernel());
+  std::vector<TFE_TensorHandle*> copied_tensors;
+  status->status = ValidateInputTypeAndPlacement(
+      ctx, ctx->devices()[0], device, op, kernel->kernel(), &copied_tensors);
   output_memory_types = &kernel->kernel()->output_memory_types();
   if (!status->status.ok()) {
+    for (auto* t : copied_tensors) {
+      TFE_DeleteTensorHandle(t);
+    }
     return;
   }
   // WARNING: kernel->Run utilizes the FunctionLibraryRuntime
@@ -492,6 +535,9 @@ void TFE_Execute(TFE_Op* op, TFE_TensorHandle** retvals, int* num_retvals,
   // sense for FunctionLibraryRuntime to ensure thread-safe access to
   // FunctionLibraryDefinition?).
   status->status = kernel->Run(&op->inputs, &outputs);
+  for (auto* t : copied_tensors) {
+    TFE_DeleteTensorHandle(t);
+  }
   if (!status->status.ok()) return;
   *num_retvals = std::min<int>(*num_retvals, outputs.size());
   for (int i = 0; i < *num_retvals; ++i) {
diff --git a/tensorflow/c/eager/c_api.h b/tensorflow/c/eager/c_api.h
index 201cb222c9..865580c5f3 100644
--- a/tensorflow/c/eager/c_api.h
+++ b/tensorflow/c/eager/c_api.h
@@ -56,6 +56,22 @@ TF_CAPI_EXPORT extern void TFE_ContextOptionsSetConfig(
     TFE_ContextOptions* options, const void* proto, size_t proto_len,
     TF_Status* status);
 
+// Controls how to act when we try to run an operation on a given device but
+// some input tensors are not on that device.
+typedef enum TFE_ContextDevicePlacementPolicy {
+  // The default: running operations with input tensors on the wrong device will
+  // fail.
+  TFE_DEVICE_PLACEMENT_EXPLICIT = 0,
+  // Copy the tensor to the right device but log a warning.
+  TFE_DEVICE_PLACEMENT_WARN = 1,
+  // Silently copy the tensor, which has a performance cost since the
+  // operation will be blocked till the copy completes.
+  TFE_DEVICE_PLACEMENT_SILENT = 2,
+} TFE_ContextDevicePlacementPolicy;
+
+TF_CAPI_EXPORT extern void TFE_ContextOptionsSetDevicePlacementPolicy(
+    TFE_ContextOptions*, TFE_ContextDevicePlacementPolicy);
+
 // Destroy an options object.
 TF_CAPI_EXPORT extern void TFE_DeleteContextOptions(TFE_ContextOptions*);
 
diff --git a/tensorflow/c/eager/c_api_internal.h b/tensorflow/c/eager/c_api_internal.h
index 7a440a5a7e..0971e2ab2f 100644
--- a/tensorflow/c/eager/c_api_internal.h
+++ b/tensorflow/c/eager/c_api_internal.h
@@ -37,11 +37,14 @@ limitations under the License.
 
 struct TFE_ContextOptions {
   TF_SessionOptions session_options;
+  TFE_ContextDevicePlacementPolicy policy{TFE_DEVICE_PLACEMENT_EXPLICIT};
 };
 
 struct TFE_Context {
   explicit TFE_Context(TF_Session* s) : session(s) {}
 
+  TFE_ContextDevicePlacementPolicy policy;
+
   // TFE_Context is an extension of TF_Session. And TF_Session needs a TF_Graph.
   TF_Session* session;
   tensorflow::Rendezvous* rendezvous;
diff --git a/tensorflow/c/eager/c_api_test.cc b/tensorflow/c/eager/c_api_test.cc
index 5344956ee7..4af91b8853 100644
--- a/tensorflow/c/eager/c_api_test.cc
+++ b/tensorflow/c/eager/c_api_test.cc
@@ -216,6 +216,52 @@ TEST(CAPI, TensorHandleCopyBetweenDevices) {
   EXPECT_EQ(TF_OK, TF_GetCode(status.get())) << TF_Message(status.get());
 }
 
+TEST(CAPI, TensorHandleSilentCopy) {
+  std::unique_ptr<TF_Status, decltype(&TF_DeleteStatus)> status(
+      TF_NewStatus(), TF_DeleteStatus);
+  TFE_ContextOptions* opts = TFE_NewContextOptions();
+  TFE_ContextOptionsSetDevicePlacementPolicy(opts, TFE_DEVICE_PLACEMENT_SILENT);
+  TFE_Context* ctx = TFE_NewContext(opts, status.get());
+  TFE_DeleteContextOptions(opts);
+  ASSERT_EQ(TF_OK, TF_GetCode(status.get())) << TF_Message(status.get());
+
+  TFE_TensorHandle* hcpu = TestMatrixTensorHandle();
+  TF_Tensor* t = TFE_TensorHandleResolve(hcpu, status.get());
+  ASSERT_EQ(TF_OK, TF_GetCode(status.get())) << TF_Message(status.get());
+
+  TF_DeviceList* devices = TFE_ContextListDevices(ctx, status.get());
+  ASSERT_EQ(TF_OK, TF_GetCode(status.get())) << TF_Message(status.get());
+  const int num_devices = TF_DeviceListCount(devices);
+
+  // Disable the test if no GPU is present.
+  if (num_devices > 1) {
+    const int device_to_use = 1;
+    const string name(TF_DeviceListName(devices, device_to_use, status.get()));
+    ASSERT_TRUE(TF_GetCode(status.get()) == TF_OK) << TF_Message(status.get());
+
+    TFE_TensorHandle* hgpu =
+        TFE_TensorHandleCopyToDevice(hcpu, ctx, name.c_str(), status.get());
+    ASSERT_TRUE(TF_GetCode(status.get()) == TF_OK) << TF_Message(status.get());
+
+    TFE_Op* matmul = MatMulOp(ctx, hcpu, hgpu);
+    TFE_OpSetDevice(matmul, name.c_str(), status.get());
+    ASSERT_TRUE(TF_GetCode(status.get()) == TF_OK) << TF_Message(status.get());
+    TFE_TensorHandle* retvals[1];
+    int num_retvals = 1;
+    TFE_Execute(matmul, &retvals[0], &num_retvals, status.get());
+    ASSERT_TRUE(TF_GetCode(status.get()) == TF_OK) << TF_Message(status.get());
+    TFE_DeleteOp(matmul);
+    TFE_DeleteTensorHandle(retvals[0]);
+    TFE_DeleteTensorHandle(hgpu);
+  }
+
+  TF_DeleteDeviceList(devices);
+  TF_DeleteTensor(t);
+  TFE_DeleteTensorHandle(hcpu);
+  TFE_DeleteContext(ctx, status.get());
+  EXPECT_EQ(TF_OK, TF_GetCode(status.get())) << TF_Message(status.get());
+}
+
 TEST(CAPI, Execute) {
   TF_Status* status = TF_NewStatus();
   TFE_ContextOptions* opts = TFE_NewContextOptions();
-- 
GitLab


From e65fbbc9dc608d97977b17e05250b015d65aa027 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 20 Oct 2017 13:03:50 -0700
Subject: [PATCH 624/909] Expose tf.contrib.framework.current_arg_scope()

PiperOrigin-RevId: 172922818
---
 tensorflow/contrib/framework/__init__.py             | 1 +
 tensorflow/contrib/framework/python/ops/arg_scope.py | 7 ++++---
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/tensorflow/contrib/framework/__init__.py b/tensorflow/contrib/framework/__init__.py
index 2081a11f47..8421ba7c04 100644
--- a/tensorflow/contrib/framework/__init__.py
+++ b/tensorflow/contrib/framework/__init__.py
@@ -37,6 +37,7 @@ See the @{$python/contrib.framework} guide.
 
 @@arg_scope
 @@add_arg_scope
+@@current_arg_scope
 @@has_arg_scope
 @@arg_scoped_arguments
 
diff --git a/tensorflow/contrib/framework/python/ops/arg_scope.py b/tensorflow/contrib/framework/python/ops/arg_scope.py
index 9c194ec202..2bce00fde2 100644
--- a/tensorflow/contrib/framework/python/ops/arg_scope.py
+++ b/tensorflow/contrib/framework/python/ops/arg_scope.py
@@ -67,6 +67,7 @@ from tensorflow.python.util import tf_decorator
 
 __all__ = ['arg_scope',
            'add_arg_scope',
+           'current_arg_scope',
            'has_arg_scope',
            'arg_scoped_arguments']
 
@@ -83,7 +84,7 @@ def _get_arg_stack():
     return _ARGSTACK
 
 
-def _current_arg_scope():
+def current_arg_scope():
   stack = _get_arg_stack()
   return stack[-1]
 
@@ -144,7 +145,7 @@ def arg_scope(list_ops_or_scope, **kwargs):
       raise TypeError('list_ops_or_scope must either be a list/tuple or reused'
                       'scope (i.e. dict)')
     try:
-      current_scope = _current_arg_scope().copy()
+      current_scope = current_arg_scope().copy()
       for op in list_ops_or_scope:
         key_op = _key_op(op)
         if not has_arg_scope(op):
@@ -172,7 +173,7 @@ def add_arg_scope(func):
     A tuple with the decorated function func_with_args().
   """
   def func_with_args(*args, **kwargs):
-    current_scope = _current_arg_scope()
+    current_scope = current_arg_scope()
     current_args = kwargs
     key_func = _key_op(func)
     if key_func in current_scope:
-- 
GitLab


From d2d9a6c7cc3b4f8c068054082a0fa2f2b95bb3d6 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 20 Oct 2017 13:19:42 -0700
Subject: [PATCH 625/909] Add AdaptiveSharedBatchScheduler which processes
 batches at a variable rate which can be adjusted based on external feedback. 
 For reasonable feedback, this scheduler should deliver better latency than
 the SharedBatchScheduler.

PiperOrigin-RevId: 172924803
---
 tensorflow/contrib/batching/BUILD             |  22 +
 .../adaptive_shared_batch_scheduler.h         | 463 ++++++++++++++++++
 .../adaptive_shared_batch_scheduler_test.cc   | 438 +++++++++++++++++
 tensorflow/contrib/batching/batch_scheduler.h |   2 +-
 4 files changed, 924 insertions(+), 1 deletion(-)
 create mode 100644 tensorflow/contrib/batching/adaptive_shared_batch_scheduler.h
 create mode 100644 tensorflow/contrib/batching/adaptive_shared_batch_scheduler_test.cc

diff --git a/tensorflow/contrib/batching/BUILD b/tensorflow/contrib/batching/BUILD
index 1555a3427f..ae3f48f1b2 100644
--- a/tensorflow/contrib/batching/BUILD
+++ b/tensorflow/contrib/batching/BUILD
@@ -69,6 +69,28 @@ tf_cc_test(
     ],
 )
 
+cc_library(
+    name = "adaptive_shared_batch_scheduler",
+    hdrs = ["adaptive_shared_batch_scheduler.h"],
+    deps = [
+        ":batch_scheduler",
+        "//tensorflow/contrib/batching/util:periodic_function_dynamic",
+        "//tensorflow/core:lib",
+    ],
+)
+
+tf_cc_test(
+    name = "adaptive_shared_batch_scheduler_test",
+    srcs = ["adaptive_shared_batch_scheduler_test.cc"],
+    deps = [
+        ":adaptive_shared_batch_scheduler",
+        "//tensorflow/contrib/batching/test_util:fake_clock_env",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+    ],
+)
+
 cc_library(
     name = "basic_batch_scheduler",
     hdrs = ["basic_batch_scheduler.h"],
diff --git a/tensorflow/contrib/batching/adaptive_shared_batch_scheduler.h b/tensorflow/contrib/batching/adaptive_shared_batch_scheduler.h
new file mode 100644
index 0000000000..ac32f09639
--- /dev/null
+++ b/tensorflow/contrib/batching/adaptive_shared_batch_scheduler.h
@@ -0,0 +1,463 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef THIRD_PARTY_TENSORFLOW_CONTRIB_BATCHING_ADAPTIVE_SHARED_BATCH_SCHEDULER_H_
+#define THIRD_PARTY_TENSORFLOW_CONTRIB_BATCHING_ADAPTIVE_SHARED_BATCH_SCHEDULER_H_
+
+#include <functional>
+#include <memory>
+#include <queue>
+#include <unordered_map>
+#include <vector>
+
+#include "tensorflow/contrib/batching/batch_scheduler.h"
+#include "tensorflow/contrib/batching/util/periodic_function.h"
+#include "tensorflow/core/lib/core/errors.h"
+#include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/lib/core/threadpool.h"
+#include "tensorflow/core/platform/cpu_info.h"
+#include "tensorflow/core/platform/env.h"
+#include "tensorflow/core/platform/thread_annotations.h"
+#include "tensorflow/core/platform/types.h"
+
+namespace tensorflow {
+namespace serving {
+namespace internal {
+template <typename TaskType>
+class ASBSBatch;
+
+template <typename TaskType>
+class ASBSQueue;
+}  // namespace internal
+
+// Shared batch scheduler designed to minimize latency. The scheduler keeps
+// track of a number of queues (one per model or model version) which are
+// continuously enqueuing requests. The scheduler groups the requests into
+// batches which it periodically sends off for processing (see
+// shared_batch_scheduler.h for more details). The AdaptiveSharedBatchScheduler
+// prioritizes batches by age (i.e. the batch's oldest request) irrespective of
+// queue. The scheduler will process the oldest batch at an adjustable rate,
+// regardless of batch size. The user can provide feedback to help set this rate
+// to achieve some goal (i.e. minimize overall latency, limit cpu usage, etc).
+//
+// The rate (or rather, the corresponding period) is adjusted each time a batch
+// is processed, using an exponentially weighted moving average to smooth
+// potentially noisy feedback:
+// ewma_feedback = ((N - 1) * ewma_feedback + feedback()) / N
+// period *= (1 + K * emwa_feedback)
+//
+// Some potential use cases:
+// Hardware Accelerators (GPUs & TPUs) - If some phase of batch processing
+//   involves serial processing by a device, from a latency perspective it is
+//   desirable to keep the device evenly loaded, avoiding the need to wait for
+//   the device to process prior batches.
+//   feedback = num_pending_on_device() - desired_pending.
+// CPU utilization - If the batch processing is cpu dominated, you can reap
+//   latency gains when underutilized by increasing the processing rate, but
+//   back the rate off when the load increases to avoid overload.
+//   feedback = cpu_rate() - desired_cpu_rate.
+
+template <typename TaskType>
+class AdaptiveSharedBatchScheduler
+    : public std::enable_shared_from_this<
+          AdaptiveSharedBatchScheduler<TaskType>> {
+ public:
+  struct Options {
+    // The name to use for the pool of batch threads.
+    string thread_pool_name = {"batch_threads"};
+    // Number of batch processing threads; equivalently the maximum number of
+    // concurrently running batches.
+    int64 num_batch_threads = port::NumSchedulableCPUs();
+    // The environment to use (typically only overridden by test code).
+    Env* env = Env::Default();
+    // Initial batch scheduling period in microseconds. Will be altered for
+    // non-zero rate_feedback.
+    double initial_scheduling_period_micros = 500;
+    // Minimum batch scheduling period in microseconds. Recommend setting this
+    // value greater than 0, otherwise it may take a while to recover from a
+    // sustained time of negative scheduling_period_feedback (which may occur
+    // under low load).
+    double min_scheduling_period_micros = 100;
+    // Maximum batch scheduling period in microseconds.
+    double max_scheduling_period_micros = 10000;
+    // Feedback function used to modify the scheduling period each time a batch
+    // is scheduled.  Should return values roughly O(1), with positive values
+    // resulting in an increased period.
+    std::function<double()> scheduling_period_feedback = [] { return 0.; };
+    // To handle potentially noisy scheduling_period_feedback, the period is
+    // adjusted using an exponentially weighted moving average over the previous
+    // feedback_smoothing_batches batches.  Must be greater than 0.
+    int64 feedback_smoothing_batches = 10;
+  };
+
+  // Ownership is shared between the caller of Create() and any queues created
+  // via AddQueue().
+  static Status Create(
+      const Options& options,
+      std::shared_ptr<AdaptiveSharedBatchScheduler<TaskType>>* scheduler);
+
+  struct QueueOptions {
+    // Maximum size of each batch.
+    int max_batch_size = 1000;
+    // Maximum number of enqueued (i.e. non-scheduled) batches.
+    int max_enqueued_batches = 10;
+  };
+
+  using BatchProcessor = std::function<void(std::unique_ptr<Batch<TaskType>>)>;
+
+  // Adds queue (and its callback) to be managed by this scheduler.
+  Status AddQueue(const QueueOptions& options,
+                  BatchProcessor process_batch_callback,
+                  std::unique_ptr<BatchScheduler<TaskType>>* queue);
+
+ private:
+  // access to AddBatch, RemoveQueue, GetEnv.
+  friend class internal::ASBSQueue<TaskType>;
+
+  explicit AdaptiveSharedBatchScheduler(const Options& options);
+
+  // Batch scheduling function which runs every scheduling_period_ microseconds.
+  void ProcessOneBatch();
+
+  // Notifies scheduler of non-empty batch which is eligible for processing.
+  void AddBatch(internal::ASBSBatch<TaskType>*);
+
+  // Removes queue from scheduler.
+  void RemoveQueue(const internal::ASBSQueue<TaskType>* queue);
+
+  Env* GetEnv() const { return options_.env; }
+
+  const Options options_;
+
+  struct BatchCompare {
+    bool operator()(const internal::ASBSBatch<TaskType>* a,
+                    const internal::ASBSBatch<TaskType>* b);
+  };
+
+  // Collection of batches added by AddBatch, ordered by age. Owned by scheduler
+  // until they are released for processing.
+  std::priority_queue<const internal::ASBSBatch<TaskType>*,
+                      std::vector<internal::ASBSBatch<TaskType>*>, BatchCompare>
+      batches_ GUARDED_BY(mu_);
+
+  // Unowned queues and callbacks added by AddQueue.
+  std::unordered_map<const internal::ASBSQueue<TaskType>*, BatchProcessor>
+      queues_and_callbacks_ GUARDED_BY(mu_);
+
+  mutex mu_;
+
+  // Responsible for running ProcessOneBatch. PeriodicFunction was used in order
+  // to check for deletion so that the thread can be shut down.
+  std::unique_ptr<PeriodicFunction> scheduling_thread_;
+
+  // Responsible for running the batch processing callbacks.
+  std::unique_ptr<thread::ThreadPool> batch_thread_pool_;
+
+  // Time interval in microseconds between successive ProcessOneBatch calls.
+  double scheduling_period_;
+
+  // Exponentially weighted moving average of
+  // options_.scheduling_period_feedback() evaluated in each ProcessOneBatch
+  // call.
+  double ewma_feedback_ = 0;
+
+  TF_DISALLOW_COPY_AND_ASSIGN(AdaptiveSharedBatchScheduler);
+};
+
+//////////////////////////////////////////////////////////
+// Implementation details follow. API users need not read.
+
+namespace internal {
+// Consolidates tasks into batches, passing them off to the
+// AdaptiveSharedBatchScheduler for processing.
+template <typename TaskType>
+class ASBSQueue : public BatchScheduler<TaskType> {
+ public:
+  using QueueOptions =
+      typename AdaptiveSharedBatchScheduler<TaskType>::QueueOptions;
+
+  ASBSQueue(std::shared_ptr<AdaptiveSharedBatchScheduler<TaskType>> scheduler,
+            const QueueOptions& options);
+
+  ~ASBSQueue() override;
+
+  // Adds task to current batch. Fails if the task size is larger than the batch
+  // size or if the current batch is full and this queue's number of outstanding
+  // batches is at its maximum.
+  Status Schedule(std::unique_ptr<TaskType>* task) override;
+
+  // Number of tasks waiting to be scheduled.
+  size_t NumEnqueuedTasks() const override;
+
+  // Number of size 1 tasks which could currently be scheduled without failing.
+  size_t SchedulingCapacity() const override;
+
+  // Notifies queue that a batch is about to be scheduled; the queue should not
+  // place any more tasks in this batch.
+  void ReleaseBatch(const ASBSBatch<TaskType>* batch);
+
+ private:
+  std::shared_ptr<AdaptiveSharedBatchScheduler<TaskType>> scheduler_;
+  const QueueOptions options_;
+  // Owned by scheduler_.
+  ASBSBatch<TaskType>* current_batch_ GUARDED_BY(mu_) = nullptr;
+  int64 num_enqueued_batches_ GUARDED_BY(mu_) = 0;
+  int64 num_enqueued_tasks_ GUARDED_BY(mu_) = 0;
+  mutable mutex mu_;
+  TF_DISALLOW_COPY_AND_ASSIGN(ASBSQueue);
+};
+
+// Batch which remembers when and by whom it was created.
+template <typename TaskType>
+class ASBSBatch : public Batch<TaskType> {
+ public:
+  ASBSBatch(ASBSQueue<TaskType>* queue, int64 creation_time_micros)
+      : queue_(queue), creation_time_micros_(creation_time_micros) {}
+
+  ~ASBSBatch() override {}
+
+  ASBSQueue<TaskType>* queue() const { return queue_; }
+
+  int64 creation_time_micros() const { return creation_time_micros_; }
+
+ private:
+  ASBSQueue<TaskType>* queue_;
+  const int64 creation_time_micros_;
+  TF_DISALLOW_COPY_AND_ASSIGN(ASBSBatch);
+};
+}  // namespace internal
+
+// ---------------- AdaptiveSharedBatchScheduler ----------------
+
+template <typename TaskType>
+Status AdaptiveSharedBatchScheduler<TaskType>::Create(
+    const Options& options,
+    std::shared_ptr<AdaptiveSharedBatchScheduler<TaskType>>* scheduler) {
+  if (options.num_batch_threads < 1) {
+    return errors::InvalidArgument("num_batch_threads must be positive; was ",
+                                   options.num_batch_threads);
+  }
+  if (options.min_scheduling_period_micros < 0) {
+    return errors::InvalidArgument(
+        "min_scheduling_period_micros must be >= 0; was ",
+        options.min_scheduling_period_micros);
+  }
+  if (options.min_scheduling_period_micros >
+      options.initial_scheduling_period_micros) {
+    return errors::InvalidArgument(
+        "initial_scheduling_period_micros (",
+        options.initial_scheduling_period_micros,
+        ") must be >= min_scheduling_period_micros (",
+        options.min_scheduling_period_micros, ")");
+  }
+  if (options.initial_scheduling_period_micros >
+      options.max_scheduling_period_micros) {
+    return errors::InvalidArgument(
+        "initial_scheduling_period_micros (",
+        options.initial_scheduling_period_micros,
+        ") must be <= max_scheduling_period_micros (",
+        options.max_scheduling_period_micros, ")");
+  }
+  if (options.feedback_smoothing_batches < 1) {
+    return errors::InvalidArgument(
+        "feedback_smoothing_batches must be positive; was ",
+        options.feedback_smoothing_batches);
+  }
+  scheduler->reset(new AdaptiveSharedBatchScheduler<TaskType>(options));
+  return Status::OK();
+}
+
+template <typename TaskType>
+AdaptiveSharedBatchScheduler<TaskType>::AdaptiveSharedBatchScheduler(
+    const Options& options)
+    : options_(options),
+      scheduling_period_(options.initial_scheduling_period_micros) {
+  PeriodicFunction::Options opts;
+  opts.thread_name_prefix = "scheduling_thread";
+  opts.env = GetEnv();
+  scheduling_thread_.reset(
+      new PeriodicFunction([this] { ProcessOneBatch(); }, 0, opts));
+  batch_thread_pool_.reset(new thread::ThreadPool(
+      GetEnv(), options.thread_pool_name, options.num_batch_threads));
+}
+
+template <typename TaskType>
+Status AdaptiveSharedBatchScheduler<TaskType>::AddQueue(
+    const QueueOptions& options, BatchProcessor process_batch_callback,
+    std::unique_ptr<BatchScheduler<TaskType>>* queue) {
+  if (options.max_batch_size <= 0) {
+    return errors::InvalidArgument("max_batch_size must be positive; was ",
+                                   options.max_batch_size);
+  }
+  if (options.max_enqueued_batches <= 0) {
+    return errors::InvalidArgument(
+        "max_enqueued_batches must be positive; was ",
+        options.max_enqueued_batches);
+  }
+  internal::ASBSQueue<TaskType>* asbs_queue_raw;
+  queue->reset(asbs_queue_raw = new internal::ASBSQueue<TaskType>(
+                   this->shared_from_this(), options));
+  mutex_lock l(mu_);
+  queues_and_callbacks_[asbs_queue_raw] = process_batch_callback;
+  return Status::OK();
+}
+
+template <typename TaskType>
+void AdaptiveSharedBatchScheduler<TaskType>::AddBatch(
+    internal::ASBSBatch<TaskType>* batch) {
+  mutex_lock l(mu_);
+  batches_.push(batch);
+}
+
+template <typename TaskType>
+void AdaptiveSharedBatchScheduler<TaskType>::RemoveQueue(
+    const internal::ASBSQueue<TaskType>* queue) {
+  mutex_lock l(mu_);
+  queues_and_callbacks_.erase(queue);
+}
+
+template <typename TaskType>
+void AdaptiveSharedBatchScheduler<TaskType>::ProcessOneBatch() {
+  static const double kFeedbackMultiplier = .001;
+  internal::ASBSBatch<TaskType>* batch = nullptr;
+  BatchProcessor callback;
+  const int64 start_time_micros = GetEnv()->NowMicros();
+  {
+    mutex_lock l(mu_);
+    if (!batches_.empty()) {
+      batch = batches_.top();
+      batches_.pop();
+      callback = queues_and_callbacks_[batch->queue()];
+    }
+  }
+  if (batch != nullptr) {
+    double feedback = options_.scheduling_period_feedback();
+    const int64 N = options_.feedback_smoothing_batches;
+    ewma_feedback_ = ((N - 1) * ewma_feedback_ + feedback) / N;
+    scheduling_period_ *= (1 + kFeedbackMultiplier * ewma_feedback_);
+    if (scheduling_period_ < options_.min_scheduling_period_micros) {
+      scheduling_period_ = options_.min_scheduling_period_micros;
+    } else if (scheduling_period_ > options_.max_scheduling_period_micros) {
+      scheduling_period_ = options_.max_scheduling_period_micros;
+    }
+    // Queue may destroy itself after ReleaseBatch is called.
+    batch->queue()->ReleaseBatch(batch);
+    batch_thread_pool_->Schedule([callback, batch] {
+      callback(std::unique_ptr<Batch<TaskType>>(batch));
+    });
+  }
+  const int64 sleep_time =
+      scheduling_period_ - (GetEnv()->NowMicros() - start_time_micros);
+  if (sleep_time > 0) {
+    GetEnv()->SleepForMicroseconds(sleep_time);
+  }
+}
+
+template <typename TaskType>
+bool AdaptiveSharedBatchScheduler<TaskType>::BatchCompare::operator()(
+    const internal::ASBSBatch<TaskType>* a,
+    const internal::ASBSBatch<TaskType>* b) {
+  return a->creation_time_micros() > b->creation_time_micros();
+}
+
+// ---------------- ASBSQueue ----------------
+
+namespace internal {
+template <typename TaskType>
+ASBSQueue<TaskType>::ASBSQueue(
+    std::shared_ptr<AdaptiveSharedBatchScheduler<TaskType>> scheduler,
+    const QueueOptions& options)
+    : scheduler_(scheduler), options_(options) {}
+
+template <typename TaskType>
+ASBSQueue<TaskType>::~ASBSQueue() {
+  // Wait until last batch has been scheduled.
+  const int kSleepMicros = 1000;
+  for (;;) {
+    {
+      mutex_lock l(mu_);
+      if (num_enqueued_batches_ == 0) {
+        break;
+      }
+    }
+    scheduler_->GetEnv()->SleepForMicroseconds(kSleepMicros);
+  }
+  scheduler_->RemoveQueue(this);
+}
+
+template <typename TaskType>
+Status ASBSQueue<TaskType>::Schedule(std::unique_ptr<TaskType>* task) {
+  bool added_new_batch = false;
+  size_t size = (*task)->size();
+  if (size > options_.max_batch_size) {
+    return errors::InvalidArgument("Task size ", size,
+                                   " is larger than maximum batch size ",
+                                   options_.max_batch_size);
+  }
+  {
+    mutex_lock l(mu_);
+    // Current batch is full, create another if allowed.
+    if (current_batch_ &&
+        current_batch_->size() + size > options_.max_batch_size) {
+      if (num_enqueued_batches_ >= options_.max_enqueued_batches) {
+        return errors::Unavailable("The batch scheduling queue is full");
+      }
+      current_batch_->Close();
+      current_batch_ = nullptr;
+    }
+    if (!current_batch_) {
+      added_new_batch = true;
+      num_enqueued_batches_++;
+      current_batch_ =
+          new ASBSBatch<TaskType>(this, scheduler_->GetEnv()->NowMicros());
+    }
+    current_batch_->AddTask(std::move(*task));
+    num_enqueued_tasks_++;
+  }
+  if (added_new_batch) scheduler_->AddBatch(current_batch_);
+  return Status::OK();
+}
+
+template <typename TaskType>
+void ASBSQueue<TaskType>::ReleaseBatch(const ASBSBatch<TaskType>* batch) {
+  mutex_lock l(mu_);
+  num_enqueued_batches_--;
+  num_enqueued_tasks_ -= batch->num_tasks();
+  if (batch == current_batch_) {
+    current_batch_->Close();
+    current_batch_ = nullptr;
+  }
+}
+
+template <typename TaskType>
+size_t ASBSQueue<TaskType>::NumEnqueuedTasks() const {
+  mutex_lock l(mu_);
+  return num_enqueued_tasks_;
+}
+
+template <typename TaskType>
+size_t ASBSQueue<TaskType>::SchedulingCapacity() const {
+  mutex_lock l(mu_);
+  const int current_batch_capacity =
+      current_batch_ ? options_.max_batch_size - current_batch_->size() : 0;
+  const int spare_batches =
+      options_.max_enqueued_batches - num_enqueued_batches_;
+  return spare_batches * options_.max_batch_size + current_batch_capacity;
+}
+}  // namespace internal
+}  // namespace serving
+}  // namespace tensorflow
+
+#endif  // THIRD_PARTY_TENSORFLOW_CONTRIB_BATCHING_ADAPTIVE_SHARED_BATCH_SCHEDULER_H_
diff --git a/tensorflow/contrib/batching/adaptive_shared_batch_scheduler_test.cc b/tensorflow/contrib/batching/adaptive_shared_batch_scheduler_test.cc
new file mode 100644
index 0000000000..a07cd6d834
--- /dev/null
+++ b/tensorflow/contrib/batching/adaptive_shared_batch_scheduler_test.cc
@@ -0,0 +1,438 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/contrib/batching/adaptive_shared_batch_scheduler.h"
+
+#include "tensorflow/contrib/batching/test_util/fake_clock_env.h"
+#include "tensorflow/core/lib/core/notification.h"
+#include "tensorflow/core/lib/core/status_test_util.h"
+#include "tensorflow/core/platform/macros.h"
+#include "tensorflow/core/platform/test.h"
+
+namespace tensorflow {
+namespace serving {
+namespace anonymous {
+
+class FakeTask : public BatchTask {
+ public:
+  explicit FakeTask(size_t size) : size_(size) {}
+
+  ~FakeTask() override = default;
+
+  size_t size() const override { return size_; }
+
+ private:
+  const size_t size_;
+
+  TF_DISALLOW_COPY_AND_ASSIGN(FakeTask);
+};
+
+// Creates a FakeTask of size 'task_size', and calls 'scheduler->Schedule()' on
+// that task. Returns the resulting status.
+Status ScheduleTask(size_t task_size, BatchScheduler<FakeTask>* scheduler) {
+  std::unique_ptr<FakeTask> task(new FakeTask(task_size));
+  Status status = scheduler->Schedule(&task);
+  // Schedule() should have consumed 'task' iff it returned Status::OK.
+  CHECK_EQ(status.ok(), task == nullptr);
+  return status;
+}
+
+// Creates a thread that waits on 'start' and then advances the fake clock in
+// 'env' in a loop until 'stop' is notified. Useful for allowing objects that
+// use the clock to be destroyed.
+std::unique_ptr<Thread> CreateFakeClockAdvancerThread(
+    test_util::FakeClockEnv* env, Notification* start, Notification* stop) {
+  return std::unique_ptr<Thread>(Env::Default()->StartThread(
+      {}, "FakeClockAdvancerThread", [env, start, stop] {
+        start->WaitForNotification();
+        while (!stop->HasBeenNotified()) {
+          env->AdvanceByMicroseconds(10);
+          Env::Default()->SleepForMicroseconds(10);
+        }
+      }));
+}
+
+TEST(AdaptiveSharedBatchSchedulerTest, Basic) {
+  for (const bool delete_scheduler_early : {false, true}) {
+    for (const bool delete_queue_1_early : {false, true}) {
+      int queue_0_tasks = 0;
+      auto queue_0_callback =
+          [&queue_0_tasks](std::unique_ptr<Batch<FakeTask>> batch) {
+            ASSERT_TRUE(batch->IsClosed());
+            EXPECT_GT(batch->num_tasks(), 0);
+            for (int i = 0; i < batch->num_tasks(); i++) {
+              queue_0_tasks += batch->task(i).size();
+            }
+          };
+      int queue_1_tasks = 0;
+      auto queue_1_callback =
+          [&queue_1_tasks](std::unique_ptr<Batch<FakeTask>> batch) {
+            ASSERT_TRUE(batch->IsClosed());
+            EXPECT_GT(batch->num_tasks(), 0);
+            for (int i = 0; i < batch->num_tasks(); i++) {
+              queue_1_tasks += batch->task(i).size();
+            }
+          };
+      {
+        std::shared_ptr<AdaptiveSharedBatchScheduler<FakeTask>> scheduler;
+        TF_ASSERT_OK(
+            AdaptiveSharedBatchScheduler<FakeTask>::Create({}, &scheduler));
+
+        // Create two queues.
+        std::unique_ptr<BatchScheduler<FakeTask>> queue_0;
+        TF_ASSERT_OK(scheduler->AddQueue({}, queue_0_callback, &queue_0));
+        std::unique_ptr<BatchScheduler<FakeTask>> queue_1;
+        TF_ASSERT_OK(scheduler->AddQueue({}, queue_1_callback, &queue_1));
+
+        if (delete_scheduler_early) {
+          // Delete our copy of the scheduler. The queues should keep it alive
+          // under the covers.
+          scheduler = nullptr;
+        }
+        // Submit tasks to the two queues, and (optionally) remove the queues.
+        TF_ASSERT_OK(ScheduleTask(1, queue_0.get()));
+        TF_ASSERT_OK(ScheduleTask(2, queue_1.get()));
+        TF_ASSERT_OK(ScheduleTask(3, queue_0.get()));
+        TF_ASSERT_OK(ScheduleTask(4, queue_1.get()));
+        if (delete_queue_1_early) {
+          queue_1 = nullptr;
+        }
+        TF_ASSERT_OK(ScheduleTask(5, queue_0.get()));
+      }
+      EXPECT_EQ(queue_0_tasks, 9);
+      EXPECT_EQ(queue_1_tasks, 6);
+    }
+  }
+}
+
+TEST(AdaptiveSharedBatchSchedulerTest, BadOptions) {
+  using Scheduler = AdaptiveSharedBatchScheduler<FakeTask>;
+  std::shared_ptr<Scheduler> scheduler;
+  Scheduler::Options options;
+  options.num_batch_threads = 0;
+  EXPECT_FALSE(Scheduler::Create(options, &scheduler).ok());
+  options = Scheduler::Options();
+  options.min_scheduling_period_micros = 50;
+  options.max_scheduling_period_micros = 100;
+  options.initial_scheduling_period_micros = 1;
+  EXPECT_FALSE(Scheduler::Create(options, &scheduler).ok());
+  options = Scheduler::Options();
+  options.min_scheduling_period_micros = 50;
+  options.max_scheduling_period_micros = 100;
+  options.initial_scheduling_period_micros = 1000;
+  EXPECT_FALSE(Scheduler::Create(options, &scheduler).ok());
+  options = Scheduler::Options();
+  options.min_scheduling_period_micros = 100;
+  options.max_scheduling_period_micros = 50;
+  options.initial_scheduling_period_micros = 75;
+  EXPECT_FALSE(Scheduler::Create(options, &scheduler).ok());
+  options = Scheduler::Options();
+  options.feedback_smoothing_batches = 0;
+  EXPECT_FALSE(Scheduler::Create(options, &scheduler).ok());
+}
+
+TEST(AdaptiveSharedBatchSchedulerTest, ObeysQueueOptions) {
+  test_util::FakeClockEnv env(Env::Default());
+  Notification start_teardown, stop_teardown;
+  std::unique_ptr<Thread> teardown_thread =
+      CreateFakeClockAdvancerThread(&env, &start_teardown, &stop_teardown);
+  {
+    AdaptiveSharedBatchScheduler<FakeTask>::Options options;
+    options.initial_scheduling_period_micros = 1000;
+    options.env = &env;
+    std::shared_ptr<AdaptiveSharedBatchScheduler<FakeTask>> scheduler;
+    TF_ASSERT_OK(
+        AdaptiveSharedBatchScheduler<FakeTask>::Create(options, &scheduler));
+    std::unique_ptr<BatchScheduler<FakeTask>> queue_0;
+    std::unique_ptr<BatchScheduler<FakeTask>> queue_1;
+    int queue_0_tasks = 0;
+    int queue_1_tasks = 0;
+    auto queue_0_callback = [&queue_0_tasks,
+                             &env](std::unique_ptr<Batch<FakeTask>> batch) {
+      ASSERT_TRUE(batch->IsClosed());
+      EXPECT_GT(batch->num_tasks(), 0);
+      for (int i = 0; i < batch->num_tasks(); i++) {
+        queue_0_tasks += batch->task(i).size();
+      }
+      env.SleepForMicroseconds(1);
+    };
+    auto queue_1_callback = [&queue_1_tasks,
+                             &env](std::unique_ptr<Batch<FakeTask>> batch) {
+      ASSERT_TRUE(batch->IsClosed());
+      EXPECT_GT(batch->num_tasks(), 0);
+      for (int i = 0; i < batch->num_tasks(); i++) {
+        queue_1_tasks += batch->task(i).size();
+      }
+      env.SleepForMicroseconds(1);
+    };
+    AdaptiveSharedBatchScheduler<FakeTask>::QueueOptions queue_options;
+    queue_options.max_batch_size = 10;
+    queue_options.max_enqueued_batches = 0;
+    // Queue must have max_enqueued_batchs > 1.
+    EXPECT_FALSE(
+        scheduler->AddQueue(queue_options, queue_0_callback, &queue_0).ok());
+    queue_options.max_enqueued_batches = 2;
+    TF_ASSERT_OK(
+        scheduler->AddQueue(queue_options, queue_0_callback, &queue_0));
+    queue_options.max_batch_size = 0;
+    // Queue must have max_batch_size > 0.
+    EXPECT_FALSE(
+        scheduler->AddQueue(queue_options, queue_1_callback, &queue_1).ok());
+    queue_options.max_batch_size = 2;
+    queue_options.max_enqueued_batches = 1;
+    TF_ASSERT_OK(
+        scheduler->AddQueue(queue_options, queue_1_callback, &queue_1));
+
+    // Wait for scheduling_thread to sleep.
+    env.BlockUntilThreadsAsleep(1);
+    // Task larger than max_batch_size shouldn't schedule.
+    EXPECT_FALSE(ScheduleTask(15, queue_0.get()).ok());
+    TF_ASSERT_OK(ScheduleTask(5, queue_0.get()));
+    TF_ASSERT_OK(ScheduleTask(5, queue_0.get()));
+    env.AdvanceByMicroseconds(1);
+
+    // Task larger than max_batch_size shouldn't schedule.
+    EXPECT_FALSE(ScheduleTask(3, queue_1.get()).ok());
+    TF_ASSERT_OK(ScheduleTask(1, queue_1.get()));
+    TF_ASSERT_OK(ScheduleTask(1, queue_1.get()));
+    env.AdvanceByMicroseconds(1);
+    // Exceeds max_enqueued_batches, shouldn't schedule.
+    EXPECT_FALSE(ScheduleTask(1, queue_1.get()).ok());
+
+    TF_ASSERT_OK(ScheduleTask(5, queue_0.get()));
+    // Exceeds max_enqueued_batches, shouldn't schedule.
+    EXPECT_FALSE(ScheduleTask(6, queue_0.get()).ok());
+    TF_ASSERT_OK(ScheduleTask(4, queue_0.get()));
+
+    // Batches should be processed in order from oldest to newest.
+    env.AdvanceByMicroseconds(1000);
+    env.BlockUntilThreadsAsleep(2);
+    EXPECT_EQ(queue_0_tasks, 10);
+    EXPECT_EQ(queue_1_tasks, 0);
+
+    env.AdvanceByMicroseconds(1000);
+    env.BlockUntilThreadsAsleep(2);
+    EXPECT_EQ(queue_0_tasks, 10);
+    EXPECT_EQ(queue_1_tasks, 2);
+
+    env.AdvanceByMicroseconds(1000);
+    env.BlockUntilThreadsAsleep(2);
+    EXPECT_EQ(queue_0_tasks, 19);
+    EXPECT_EQ(queue_1_tasks, 2);
+    start_teardown.Notify();
+  }
+  stop_teardown.Notify();
+}
+
+TEST(AdaptiveSharedBatchSchedulerTest, RateFeedback) {
+  test_util::FakeClockEnv env(Env::Default());
+  Notification start_teardown, stop_teardown;
+  std::unique_ptr<Thread> teardown_thread =
+      CreateFakeClockAdvancerThread(&env, &start_teardown, &stop_teardown);
+  {
+    double feedback = 0;
+    AdaptiveSharedBatchScheduler<FakeTask>::Options options;
+    options.initial_scheduling_period_micros = 1000;
+    options.min_scheduling_period_micros = 200;
+    options.max_scheduling_period_micros = 2000;
+    options.env = &env;
+    options.scheduling_period_feedback = [&feedback] { return feedback; };
+    options.feedback_smoothing_batches = 1;
+    std::shared_ptr<AdaptiveSharedBatchScheduler<FakeTask>> scheduler;
+    TF_ASSERT_OK(
+        AdaptiveSharedBatchScheduler<FakeTask>::Create(options, &scheduler));
+    std::unique_ptr<BatchScheduler<FakeTask>> queue;
+    int scheduled_items = 0;
+    auto queue_callback = [&scheduled_items,
+                           &env](std::unique_ptr<Batch<FakeTask>> batch) {
+      ASSERT_TRUE(batch->IsClosed());
+      EXPECT_GT(batch->num_tasks(), 0);
+      scheduled_items = 0;
+      for (int i = 0; i < batch->num_tasks(); i++) {
+        scheduled_items += batch->task(i).size();
+      }
+      env.SleepForMicroseconds(1);
+    };
+
+    TF_ASSERT_OK(scheduler->AddQueue({}, queue_callback, &queue));
+
+    // Wait for scheduling_thread to sleep.
+    env.BlockUntilThreadsAsleep(1);
+    // Enqueue 6 batches.
+    for (int i = 0; i < 6; i++) {
+      TF_ASSERT_OK(ScheduleTask(900 + i, queue.get()));
+      env.AdvanceByMicroseconds(1);
+    }
+    feedback = -500;
+    env.AdvanceByMicroseconds(994);
+    env.BlockUntilThreadsAsleep(2);  // scheduling period = 500 usec.
+    EXPECT_EQ(scheduled_items, 900);
+    env.AdvanceByMicroseconds(500);
+    env.BlockUntilThreadsAsleep(2);  // scheduling period = 250 usec.
+    EXPECT_EQ(scheduled_items, 901);
+    feedback = 0;
+    env.AdvanceByMicroseconds(250);
+    env.BlockUntilThreadsAsleep(2);  // scheduling period = 250 usec.
+    EXPECT_EQ(scheduled_items, 902);
+    feedback = 10000;  // large feedback should hit max_scheduling_period.
+    env.AdvanceByMicroseconds(250);
+    env.BlockUntilThreadsAsleep(2);  // scheduling period = 2000 usec.
+    EXPECT_EQ(scheduled_items, 903);
+    feedback = -10000;  // large feedback should hit min_scheduling_period.
+    env.AdvanceByMicroseconds(1999);
+    // No callback scheduled, only scheduling thread sleeping.
+    env.BlockUntilThreadsAsleep(1);
+    EXPECT_EQ(scheduled_items, 903);
+    env.AdvanceByMicroseconds(1);
+    env.BlockUntilThreadsAsleep(2);  // scheduling period = 200 usec.
+    EXPECT_EQ(scheduled_items, 904);
+    env.AdvanceByMicroseconds(200);
+    env.BlockUntilThreadsAsleep(2);
+    EXPECT_EQ(scheduled_items, 905);
+    start_teardown.Notify();
+  }
+  stop_teardown.Notify();
+}
+
+TEST(AdaptiveSharedBatchSchedulerTest, FeedbackSmoothing) {
+  test_util::FakeClockEnv env(Env::Default());
+  Notification start_teardown, stop_teardown;
+  std::unique_ptr<Thread> teardown_thread =
+      CreateFakeClockAdvancerThread(&env, &start_teardown, &stop_teardown);
+  {
+    double feedback = 0;
+    AdaptiveSharedBatchScheduler<FakeTask>::Options options;
+    options.initial_scheduling_period_micros = 1000;
+    options.env = &env;
+    options.scheduling_period_feedback = [&feedback] { return feedback; };
+    options.feedback_smoothing_batches = 3;
+    std::shared_ptr<AdaptiveSharedBatchScheduler<FakeTask>> scheduler;
+    TF_ASSERT_OK(
+        AdaptiveSharedBatchScheduler<FakeTask>::Create(options, &scheduler));
+    std::unique_ptr<BatchScheduler<FakeTask>> queue;
+    int scheduled_items = 0;
+    auto queue_callback = [&scheduled_items,
+                           &env](std::unique_ptr<Batch<FakeTask>> batch) {
+      ASSERT_TRUE(batch->IsClosed());
+      EXPECT_GT(batch->num_tasks(), 0);
+      scheduled_items = 0;
+      for (int i = 0; i < batch->num_tasks(); i++) {
+        scheduled_items += batch->task(i).size();
+      }
+      env.SleepForMicroseconds(1);
+    };
+
+    TF_ASSERT_OK(scheduler->AddQueue({}, queue_callback, &queue));
+
+    // Wait for scheduling_thread to sleep.
+    env.BlockUntilThreadsAsleep(1);
+    // Enqueue 4 batches.
+    for (int i = 0; i < 4; i++) {
+      TF_ASSERT_OK(ScheduleTask(900 + i, queue.get()));
+      env.AdvanceByMicroseconds(1);
+    }
+    feedback = -300;
+    env.AdvanceByMicroseconds(996);
+    env.BlockUntilThreadsAsleep(2);
+    // ewma_feedback = 100, scheduling_period = 900.
+    EXPECT_EQ(scheduled_items, 900);
+    env.AdvanceByMicroseconds(899);
+    // No callback scheduled, only scheduling thread sleeping.
+    env.BlockUntilThreadsAsleep(1);
+    EXPECT_EQ(scheduled_items, 900);
+    env.AdvanceByMicroseconds(1);
+    env.BlockUntilThreadsAsleep(2);
+    // ewma_feedback = 167, scheduling_period = 750.
+    EXPECT_EQ(scheduled_items, 901);
+    env.AdvanceByMicroseconds(749);
+    // No callback scheduled, only scheduling thread sleeping.
+    env.BlockUntilThreadsAsleep(1);
+    EXPECT_EQ(scheduled_items, 901);
+    feedback = 1000 / 3.;
+    env.AdvanceByMicroseconds(1);
+    env.BlockUntilThreadsAsleep(2);
+    // emwa_feedback = 0, scheduling_period = 750.
+    EXPECT_EQ(scheduled_items, 902);
+    env.AdvanceByMicroseconds(749);
+    // No callback scheduled, only scheduling thread sleeping.
+    env.BlockUntilThreadsAsleep(1);
+    EXPECT_EQ(scheduled_items, 902);
+    env.AdvanceByMicroseconds(1);
+    env.BlockUntilThreadsAsleep(2);
+    EXPECT_EQ(scheduled_items, 903);
+    start_teardown.Notify();
+  }
+  stop_teardown.Notify();
+}
+
+TEST(AdaptiveSharedBatchSchedulerTest, QueueCapacityInfo) {
+  test_util::FakeClockEnv env(Env::Default());
+  Notification start_teardown, stop_teardown;
+  std::unique_ptr<Thread> teardown_thread =
+      CreateFakeClockAdvancerThread(&env, &start_teardown, &stop_teardown);
+  {
+    AdaptiveSharedBatchScheduler<FakeTask>::Options options;
+    options.initial_scheduling_period_micros = 1000;
+    options.env = &env;
+    std::shared_ptr<AdaptiveSharedBatchScheduler<FakeTask>> scheduler;
+    TF_ASSERT_OK(
+        AdaptiveSharedBatchScheduler<FakeTask>::Create(options, &scheduler));
+    std::unique_ptr<BatchScheduler<FakeTask>> queue;
+    int scheduled_items = 0;
+    auto queue_callback = [&scheduled_items,
+                           &env](std::unique_ptr<Batch<FakeTask>> batch) {
+      ASSERT_TRUE(batch->IsClosed());
+      EXPECT_GT(batch->num_tasks(), 0);
+      scheduled_items = 0;
+      for (int i = 0; i < batch->num_tasks(); i++) {
+        scheduled_items += batch->task(i).size();
+      }
+      env.SleepForMicroseconds(1);
+    };
+    AdaptiveSharedBatchScheduler<FakeTask>::QueueOptions queue_options;
+    queue_options.max_batch_size = 10;
+    queue_options.max_enqueued_batches = 10;
+    TF_ASSERT_OK(scheduler->AddQueue(queue_options, queue_callback, &queue));
+
+    // Wait for scheduling_thread to sleep.
+    env.BlockUntilThreadsAsleep(1);
+    // Enqueue 3 tasks.
+    EXPECT_EQ(queue->NumEnqueuedTasks(), 0);
+    EXPECT_EQ(queue->SchedulingCapacity(), 100);
+    TF_ASSERT_OK(ScheduleTask(5, queue.get()));
+    EXPECT_EQ(queue->NumEnqueuedTasks(), 1);
+    EXPECT_EQ(queue->SchedulingCapacity(), 95);
+    env.AdvanceByMicroseconds(1);
+    TF_ASSERT_OK(ScheduleTask(6, queue.get()));
+    EXPECT_EQ(queue->NumEnqueuedTasks(), 2);
+    EXPECT_EQ(queue->SchedulingCapacity(), 84);
+    env.AdvanceByMicroseconds(1);
+    TF_ASSERT_OK(ScheduleTask(1, queue.get()));
+    EXPECT_EQ(queue->NumEnqueuedTasks(), 3);
+    EXPECT_EQ(queue->SchedulingCapacity(), 83);
+
+    env.AdvanceByMicroseconds(998);
+    env.BlockUntilThreadsAsleep(2);
+    EXPECT_EQ(scheduled_items, 5);
+    env.AdvanceByMicroseconds(1000);
+    env.BlockUntilThreadsAsleep(2);
+    EXPECT_EQ(scheduled_items, 7);
+    start_teardown.Notify();
+  }
+  stop_teardown.Notify();
+}
+}  // namespace anonymous
+}  // namespace serving
+}  // namespace tensorflow
diff --git a/tensorflow/contrib/batching/batch_scheduler.h b/tensorflow/contrib/batching/batch_scheduler.h
index 7c41ad8818..a5072f439a 100644
--- a/tensorflow/contrib/batching/batch_scheduler.h
+++ b/tensorflow/contrib/batching/batch_scheduler.h
@@ -78,7 +78,7 @@ template <typename TaskType>
 class Batch {
  public:
   Batch() = default;
-  ~Batch();  // Blocks until the batch is closed.
+  virtual ~Batch();  // Blocks until the batch is closed.
 
   // Appends 'task' to the batch. After calling AddTask(), the newly-added task
   // can be accessed via task(num_tasks()-1) or mutable_task(num_tasks()-1).
-- 
GitLab


From 32eb07bf7b4cf5c9f5ee14e1f4cbe18b1eba6c4d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 20 Oct 2017 13:44:10 -0700
Subject: [PATCH 626/909] Simplify the graph generated for contrib/summaries in
 the "always summarize" and "never summarize" cases by skipping the `cond`.

PiperOrigin-RevId: 172928083
---
 tensorflow/contrib/summary/BUILD               |  2 +-
 tensorflow/contrib/summary/summary_ops.py      | 11 +++++------
 tensorflow/contrib/summary/summary_ops_test.py |  4 ++--
 3 files changed, 8 insertions(+), 9 deletions(-)

diff --git a/tensorflow/contrib/summary/BUILD b/tensorflow/contrib/summary/BUILD
index d09ad48e10..bcb2d74b4a 100644
--- a/tensorflow/contrib/summary/BUILD
+++ b/tensorflow/contrib/summary/BUILD
@@ -43,9 +43,9 @@ py_library(
     deps = [
         ":gen_summary_ops",
         "//tensorflow/python:constant_op",
-        "//tensorflow/python:control_flow_ops",
         "//tensorflow/python:dtypes",
         "//tensorflow/python:framework_ops",
+        "//tensorflow/python:layers_base",
         "//tensorflow/python:summary_op_util",
         "//tensorflow/python:training",
         "//tensorflow/python/eager:context",
diff --git a/tensorflow/contrib/summary/summary_ops.py b/tensorflow/contrib/summary/summary_ops.py
index c8d0c14e19..ba3619bfc9 100644
--- a/tensorflow/contrib/summary/summary_ops.py
+++ b/tensorflow/contrib/summary/summary_ops.py
@@ -24,11 +24,10 @@ from tensorflow.python.eager import context
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
-from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.layers import utils
 from tensorflow.python.ops import summary_op_util
 from tensorflow.python.training import training_util
 
-
 # Name for a collection which is expected to have at most a single boolean
 # Tensor. If this tensor is True the summary ops will record summaries.
 _SHOULD_RECORD_SUMMARIES_NAME = "ShouldRecordSummaries"
@@ -38,7 +37,7 @@ def should_record_summaries():
   """Returns boolean Tensor which is true if summaries should be recorded."""
   should_record_collection = ops.get_collection(_SHOULD_RECORD_SUMMARIES_NAME)
   if not should_record_collection:
-    return constant_op.constant(False)
+    return False
   if len(should_record_collection) != 1:
     raise ValueError(
         "More than one tensor specified for whether summaries "
@@ -56,13 +55,13 @@ def record_summaries_every_n_global_steps(n):
 def always_record_summaries():
   """Sets the should_record_summaries Tensor to always true."""
   collection_ref = ops.get_collection_ref(_SHOULD_RECORD_SUMMARIES_NAME)
-  collection_ref[:] = [constant_op.constant(True)]
+  collection_ref[:] = [True]
 
 
 def never_record_summaries():
   """Sets the should_record_summaries Tensor to always false."""
   collection_ref = ops.get_collection_ref(_SHOULD_RECORD_SUMMARIES_NAME)
-  collection_ref[:] = [constant_op.constant(False)]
+  collection_ref[:] = [False]
 
 
 def create_summary_file_writer(logdir,
@@ -106,7 +105,7 @@ def summary_writer_function(name, tensor, function, family=None):
       function(tag, scope)
       return True
 
-  return control_flow_ops.cond(
+  return utils.smart_cond(
       should_record_summaries(), record, _nothing, name="")
 
 
diff --git a/tensorflow/contrib/summary/summary_ops_test.py b/tensorflow/contrib/summary/summary_ops_test.py
index 6958ee8dd8..2cd4fce5b3 100644
--- a/tensorflow/contrib/summary/summary_ops_test.py
+++ b/tensorflow/contrib/summary/summary_ops_test.py
@@ -40,9 +40,9 @@ class TargetTest(test_util.TensorFlowTestCase):
       summary_ops.create_summary_file_writer(logdir, max_queue=0, name='t0')
 
   def testShouldRecordSummary(self):
-    self.assertFalse(summary_ops.should_record_summaries().numpy())
+    self.assertFalse(summary_ops.should_record_summaries())
     summary_ops.always_record_summaries()
-    self.assertTrue(summary_ops.should_record_summaries().numpy())
+    self.assertTrue(summary_ops.should_record_summaries())
 
   def testSummaryOps(self):
     training_util.get_or_create_global_step()
-- 
GitLab


From afd9224b3f6148e4c115e0da8ad4e57a8b47e383 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 20 Oct 2017 13:59:55 -0700
Subject: [PATCH 627/909] Updating to latest stable version of llvm repo (same
 used to generate latest in gcr.io/cloud-marketplace/google/clang-debian8)

PiperOrigin-RevId: 172930105
---
 tensorflow/tools/ci_build/install/build_and_install_clang.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/tools/ci_build/install/build_and_install_clang.sh b/tensorflow/tools/ci_build/install/build_and_install_clang.sh
index 3fb9964948..9966434477 100755
--- a/tensorflow/tools/ci_build/install/build_and_install_clang.sh
+++ b/tensorflow/tools/ci_build/install/build_and_install_clang.sh
@@ -16,7 +16,7 @@
 
 set -ex
 
-LLVM_SVN_REVISION="299268"
+LLVM_SVN_REVISION="314281"
 CLANG_TMP_DIR=/tmp/clang-build
 
 mkdir "$CLANG_TMP_DIR"
-- 
GitLab


From b2dcbca94c928181986488f3dccdcc9568926988 Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Fri, 20 Oct 2017 14:23:37 -0700
Subject: [PATCH 628/909] Add a BUILD file for makefile build.

PiperOrigin-RevId: 172933558
---
 tensorflow/BUILD                              |  2 ++
 tensorflow/contrib/makefile/BUILD             | 31 +++++++++++++++++++
 tensorflow/contrib/makefile/Makefile          |  3 +-
 .../contrib/makefile/build_all_linux.sh       |  3 +-
 tensorflow/python/kernel_tests/BUILD          |  5 +--
 tensorflow/tools/benchmark/BUILD              |  9 ++++++
 third_party/eigen3/BUILD                      |  9 ++++++
 third_party/fft2d/BUILD                       |  9 ++++++
 8 files changed, 65 insertions(+), 6 deletions(-)
 create mode 100644 tensorflow/contrib/makefile/BUILD

diff --git a/tensorflow/BUILD b/tensorflow/BUILD
index d7d6d5fc77..d4396bacbf 100644
--- a/tensorflow/BUILD
+++ b/tensorflow/BUILD
@@ -400,6 +400,7 @@ filegroup(
         "//tensorflow/contrib/linear_optimizer:all_files",
         "//tensorflow/contrib/lookup:all_files",
         "//tensorflow/contrib/losses:all_files",
+        "//tensorflow/contrib/makefile:all_files",
         "//tensorflow/contrib/meta_graph_transform:all_files",
         "//tensorflow/contrib/metrics:all_files",
         "//tensorflow/contrib/mpi_collectives:all_files",
@@ -513,6 +514,7 @@ filegroup(
         "//tensorflow/tools/api/golden:all_files",
         "//tensorflow/tools/api/lib:all_files",
         "//tensorflow/tools/api/tests:all_files",
+        "//tensorflow/tools/benchmark:all_files",
         "//tensorflow/tools/build_info:all_files",
         "//tensorflow/tools/common:all_files",
         "//tensorflow/tools/compatibility:all_files",
diff --git a/tensorflow/contrib/makefile/BUILD b/tensorflow/contrib/makefile/BUILD
new file mode 100644
index 0000000000..a8dd59f32a
--- /dev/null
+++ b/tensorflow/contrib/makefile/BUILD
@@ -0,0 +1,31 @@
+# Necessary build rules for makefile build in our CI.
+
+licenses(["notice"])  # Apache 2.0
+
+package(default_visibility = ["//visibility:private"])
+
+filegroup(
+    name = "all_files",
+    srcs = glob(
+        ["**/*"],
+        exclude = ["**/OWNERS"],
+    ),
+    visibility = ["//tensorflow:__subpackages__"],
+)
+
+sh_test(
+    name = "build_all_linux",
+    size = "enormous",
+    srcs = ["build_all_linux.sh"],
+    data = [
+        "//tensorflow:all_opensource_files",
+        "//third_party/eigen3:all_files",
+        "//third_party/fft2d:all_files",
+    ],
+    tags = [
+        "manual",
+        "no_gpu",
+        "no_oss",
+        "notap",
+    ],
+)
diff --git a/tensorflow/contrib/makefile/Makefile b/tensorflow/contrib/makefile/Makefile
index 3dcff3d4a3..cb23dd6dab 100644
--- a/tensorflow/contrib/makefile/Makefile
+++ b/tensorflow/contrib/makefile/Makefile
@@ -14,7 +14,7 @@
 # Host compilation settings
 
 # Find where we're running from, so we can store generated files here.
-MAKEFILE_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
+MAKEFILE_DIR ?= $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
 HAS_GEN_HOST_PROTOC := \
 $(shell test -f $(MAKEFILE_DIR)/gen/protobuf-host/bin/protoc && echo "true" ||\
 echo "false")
@@ -71,6 +71,7 @@ HOST_LDOPTS += -L/usr/local/lib
 
 HOST_INCLUDES := \
 -I. \
+-I$(MAKEFILE_DIR)/../../../ \
 -I$(MAKEFILE_DIR)/downloads/ \
 -I$(MAKEFILE_DIR)/downloads/eigen \
 -I$(MAKEFILE_DIR)/downloads/gemmlowp \
diff --git a/tensorflow/contrib/makefile/build_all_linux.sh b/tensorflow/contrib/makefile/build_all_linux.sh
index 5d73f697f4..a440633cfc 100755
--- a/tensorflow/contrib/makefile/build_all_linux.sh
+++ b/tensorflow/contrib/makefile/build_all_linux.sh
@@ -44,4 +44,5 @@ tensorflow/contrib/makefile/compile_linux_protobuf.sh
 # Build TensorFlow.
 make -j"${JOB_COUNT}" -f tensorflow/contrib/makefile/Makefile \
   OPTFLAGS="-O3 -march=native" \
-  HOST_CXXFLAGS="--std=c++11 -march=native"
+  HOST_CXXFLAGS="--std=c++11 -march=native" \
+  MAKEFILE_DIR=$SCRIPT_DIR
diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD
index e6848edc12..0e36c3498a 100644
--- a/tensorflow/python/kernel_tests/BUILD
+++ b/tensorflow/python/kernel_tests/BUILD
@@ -2700,10 +2700,7 @@ tf_py_test(
         "//tensorflow/python:variables",
     ],
     shard_count = 3,
-    tags = [
-        "no_windows_gpu",
-        "nozapfhahn",
-    ],
+    tags = ["no_windows_gpu"],
 )
 
 tf_py_test(
diff --git a/tensorflow/tools/benchmark/BUILD b/tensorflow/tools/benchmark/BUILD
index 048035f2b1..caa6629c49 100644
--- a/tensorflow/tools/benchmark/BUILD
+++ b/tensorflow/tools/benchmark/BUILD
@@ -89,3 +89,12 @@ tf_cc_binary(
     visibility = ["//visibility:public"],
     deps = [":benchmark_model_lib"],
 )
+
+filegroup(
+    name = "all_files",
+    srcs = glob(
+        ["**/*"],
+        exclude = ["**/OWNERS"],
+    ),
+    visibility = ["//tensorflow:__subpackages__"],
+)
diff --git a/third_party/eigen3/BUILD b/third_party/eigen3/BUILD
index f38a26717e..ad87477b7a 100644
--- a/third_party/eigen3/BUILD
+++ b/third_party/eigen3/BUILD
@@ -38,3 +38,12 @@ cc_library(
         "@local_config_sycl//sycl:sycl",
     ],
 )
+
+filegroup(
+    name = "all_files",
+    srcs = glob(
+        ["**/*"],
+        exclude = ["**/OWNERS"],
+    ),
+    visibility = ["//tensorflow:__subpackages__"],
+)
diff --git a/third_party/fft2d/BUILD b/third_party/fft2d/BUILD
index 93ea06e81b..8135442482 100644
--- a/third_party/fft2d/BUILD
+++ b/third_party/fft2d/BUILD
@@ -28,3 +28,12 @@ filegroup(
     name = "fft2d_headers_srcs",
     srcs = ["fft.h"],
 )
+
+filegroup(
+    name = "all_files",
+    srcs = glob(
+        ["**/*"],
+        exclude = ["**/OWNERS"],
+    ),
+    visibility = ["//tensorflow:__subpackages__"],
+)
-- 
GitLab


From d1e7382af7b99dad2455d9b7eaf34989a75f26d2 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 20 Oct 2017 14:48:27 -0700
Subject: [PATCH 629/909] Automated g4 rollback of changelist 172924803

PiperOrigin-RevId: 172936802
---
 tensorflow/contrib/batching/BUILD             |  22 -
 .../adaptive_shared_batch_scheduler.h         | 463 ------------------
 .../adaptive_shared_batch_scheduler_test.cc   | 438 -----------------
 tensorflow/contrib/batching/batch_scheduler.h |   2 +-
 4 files changed, 1 insertion(+), 924 deletions(-)
 delete mode 100644 tensorflow/contrib/batching/adaptive_shared_batch_scheduler.h
 delete mode 100644 tensorflow/contrib/batching/adaptive_shared_batch_scheduler_test.cc

diff --git a/tensorflow/contrib/batching/BUILD b/tensorflow/contrib/batching/BUILD
index ae3f48f1b2..1555a3427f 100644
--- a/tensorflow/contrib/batching/BUILD
+++ b/tensorflow/contrib/batching/BUILD
@@ -69,28 +69,6 @@ tf_cc_test(
     ],
 )
 
-cc_library(
-    name = "adaptive_shared_batch_scheduler",
-    hdrs = ["adaptive_shared_batch_scheduler.h"],
-    deps = [
-        ":batch_scheduler",
-        "//tensorflow/contrib/batching/util:periodic_function_dynamic",
-        "//tensorflow/core:lib",
-    ],
-)
-
-tf_cc_test(
-    name = "adaptive_shared_batch_scheduler_test",
-    srcs = ["adaptive_shared_batch_scheduler_test.cc"],
-    deps = [
-        ":adaptive_shared_batch_scheduler",
-        "//tensorflow/contrib/batching/test_util:fake_clock_env",
-        "//tensorflow/core:lib",
-        "//tensorflow/core:test",
-        "//tensorflow/core:test_main",
-    ],
-)
-
 cc_library(
     name = "basic_batch_scheduler",
     hdrs = ["basic_batch_scheduler.h"],
diff --git a/tensorflow/contrib/batching/adaptive_shared_batch_scheduler.h b/tensorflow/contrib/batching/adaptive_shared_batch_scheduler.h
deleted file mode 100644
index ac32f09639..0000000000
--- a/tensorflow/contrib/batching/adaptive_shared_batch_scheduler.h
+++ /dev/null
@@ -1,463 +0,0 @@
-/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#ifndef THIRD_PARTY_TENSORFLOW_CONTRIB_BATCHING_ADAPTIVE_SHARED_BATCH_SCHEDULER_H_
-#define THIRD_PARTY_TENSORFLOW_CONTRIB_BATCHING_ADAPTIVE_SHARED_BATCH_SCHEDULER_H_
-
-#include <functional>
-#include <memory>
-#include <queue>
-#include <unordered_map>
-#include <vector>
-
-#include "tensorflow/contrib/batching/batch_scheduler.h"
-#include "tensorflow/contrib/batching/util/periodic_function.h"
-#include "tensorflow/core/lib/core/errors.h"
-#include "tensorflow/core/lib/core/status.h"
-#include "tensorflow/core/lib/core/threadpool.h"
-#include "tensorflow/core/platform/cpu_info.h"
-#include "tensorflow/core/platform/env.h"
-#include "tensorflow/core/platform/thread_annotations.h"
-#include "tensorflow/core/platform/types.h"
-
-namespace tensorflow {
-namespace serving {
-namespace internal {
-template <typename TaskType>
-class ASBSBatch;
-
-template <typename TaskType>
-class ASBSQueue;
-}  // namespace internal
-
-// Shared batch scheduler designed to minimize latency. The scheduler keeps
-// track of a number of queues (one per model or model version) which are
-// continuously enqueuing requests. The scheduler groups the requests into
-// batches which it periodically sends off for processing (see
-// shared_batch_scheduler.h for more details). The AdaptiveSharedBatchScheduler
-// prioritizes batches by age (i.e. the batch's oldest request) irrespective of
-// queue. The scheduler will process the oldest batch at an adjustable rate,
-// regardless of batch size. The user can provide feedback to help set this rate
-// to achieve some goal (i.e. minimize overall latency, limit cpu usage, etc).
-//
-// The rate (or rather, the corresponding period) is adjusted each time a batch
-// is processed, using an exponentially weighted moving average to smooth
-// potentially noisy feedback:
-// ewma_feedback = ((N - 1) * ewma_feedback + feedback()) / N
-// period *= (1 + K * emwa_feedback)
-//
-// Some potential use cases:
-// Hardware Accelerators (GPUs & TPUs) - If some phase of batch processing
-//   involves serial processing by a device, from a latency perspective it is
-//   desirable to keep the device evenly loaded, avoiding the need to wait for
-//   the device to process prior batches.
-//   feedback = num_pending_on_device() - desired_pending.
-// CPU utilization - If the batch processing is cpu dominated, you can reap
-//   latency gains when underutilized by increasing the processing rate, but
-//   back the rate off when the load increases to avoid overload.
-//   feedback = cpu_rate() - desired_cpu_rate.
-
-template <typename TaskType>
-class AdaptiveSharedBatchScheduler
-    : public std::enable_shared_from_this<
-          AdaptiveSharedBatchScheduler<TaskType>> {
- public:
-  struct Options {
-    // The name to use for the pool of batch threads.
-    string thread_pool_name = {"batch_threads"};
-    // Number of batch processing threads; equivalently the maximum number of
-    // concurrently running batches.
-    int64 num_batch_threads = port::NumSchedulableCPUs();
-    // The environment to use (typically only overridden by test code).
-    Env* env = Env::Default();
-    // Initial batch scheduling period in microseconds. Will be altered for
-    // non-zero rate_feedback.
-    double initial_scheduling_period_micros = 500;
-    // Minimum batch scheduling period in microseconds. Recommend setting this
-    // value greater than 0, otherwise it may take a while to recover from a
-    // sustained time of negative scheduling_period_feedback (which may occur
-    // under low load).
-    double min_scheduling_period_micros = 100;
-    // Maximum batch scheduling period in microseconds.
-    double max_scheduling_period_micros = 10000;
-    // Feedback function used to modify the scheduling period each time a batch
-    // is scheduled.  Should return values roughly O(1), with positive values
-    // resulting in an increased period.
-    std::function<double()> scheduling_period_feedback = [] { return 0.; };
-    // To handle potentially noisy scheduling_period_feedback, the period is
-    // adjusted using an exponentially weighted moving average over the previous
-    // feedback_smoothing_batches batches.  Must be greater than 0.
-    int64 feedback_smoothing_batches = 10;
-  };
-
-  // Ownership is shared between the caller of Create() and any queues created
-  // via AddQueue().
-  static Status Create(
-      const Options& options,
-      std::shared_ptr<AdaptiveSharedBatchScheduler<TaskType>>* scheduler);
-
-  struct QueueOptions {
-    // Maximum size of each batch.
-    int max_batch_size = 1000;
-    // Maximum number of enqueued (i.e. non-scheduled) batches.
-    int max_enqueued_batches = 10;
-  };
-
-  using BatchProcessor = std::function<void(std::unique_ptr<Batch<TaskType>>)>;
-
-  // Adds queue (and its callback) to be managed by this scheduler.
-  Status AddQueue(const QueueOptions& options,
-                  BatchProcessor process_batch_callback,
-                  std::unique_ptr<BatchScheduler<TaskType>>* queue);
-
- private:
-  // access to AddBatch, RemoveQueue, GetEnv.
-  friend class internal::ASBSQueue<TaskType>;
-
-  explicit AdaptiveSharedBatchScheduler(const Options& options);
-
-  // Batch scheduling function which runs every scheduling_period_ microseconds.
-  void ProcessOneBatch();
-
-  // Notifies scheduler of non-empty batch which is eligible for processing.
-  void AddBatch(internal::ASBSBatch<TaskType>*);
-
-  // Removes queue from scheduler.
-  void RemoveQueue(const internal::ASBSQueue<TaskType>* queue);
-
-  Env* GetEnv() const { return options_.env; }
-
-  const Options options_;
-
-  struct BatchCompare {
-    bool operator()(const internal::ASBSBatch<TaskType>* a,
-                    const internal::ASBSBatch<TaskType>* b);
-  };
-
-  // Collection of batches added by AddBatch, ordered by age. Owned by scheduler
-  // until they are released for processing.
-  std::priority_queue<const internal::ASBSBatch<TaskType>*,
-                      std::vector<internal::ASBSBatch<TaskType>*>, BatchCompare>
-      batches_ GUARDED_BY(mu_);
-
-  // Unowned queues and callbacks added by AddQueue.
-  std::unordered_map<const internal::ASBSQueue<TaskType>*, BatchProcessor>
-      queues_and_callbacks_ GUARDED_BY(mu_);
-
-  mutex mu_;
-
-  // Responsible for running ProcessOneBatch. PeriodicFunction was used in order
-  // to check for deletion so that the thread can be shut down.
-  std::unique_ptr<PeriodicFunction> scheduling_thread_;
-
-  // Responsible for running the batch processing callbacks.
-  std::unique_ptr<thread::ThreadPool> batch_thread_pool_;
-
-  // Time interval in microseconds between successive ProcessOneBatch calls.
-  double scheduling_period_;
-
-  // Exponentially weighted moving average of
-  // options_.scheduling_period_feedback() evaluated in each ProcessOneBatch
-  // call.
-  double ewma_feedback_ = 0;
-
-  TF_DISALLOW_COPY_AND_ASSIGN(AdaptiveSharedBatchScheduler);
-};
-
-//////////////////////////////////////////////////////////
-// Implementation details follow. API users need not read.
-
-namespace internal {
-// Consolidates tasks into batches, passing them off to the
-// AdaptiveSharedBatchScheduler for processing.
-template <typename TaskType>
-class ASBSQueue : public BatchScheduler<TaskType> {
- public:
-  using QueueOptions =
-      typename AdaptiveSharedBatchScheduler<TaskType>::QueueOptions;
-
-  ASBSQueue(std::shared_ptr<AdaptiveSharedBatchScheduler<TaskType>> scheduler,
-            const QueueOptions& options);
-
-  ~ASBSQueue() override;
-
-  // Adds task to current batch. Fails if the task size is larger than the batch
-  // size or if the current batch is full and this queue's number of outstanding
-  // batches is at its maximum.
-  Status Schedule(std::unique_ptr<TaskType>* task) override;
-
-  // Number of tasks waiting to be scheduled.
-  size_t NumEnqueuedTasks() const override;
-
-  // Number of size 1 tasks which could currently be scheduled without failing.
-  size_t SchedulingCapacity() const override;
-
-  // Notifies queue that a batch is about to be scheduled; the queue should not
-  // place any more tasks in this batch.
-  void ReleaseBatch(const ASBSBatch<TaskType>* batch);
-
- private:
-  std::shared_ptr<AdaptiveSharedBatchScheduler<TaskType>> scheduler_;
-  const QueueOptions options_;
-  // Owned by scheduler_.
-  ASBSBatch<TaskType>* current_batch_ GUARDED_BY(mu_) = nullptr;
-  int64 num_enqueued_batches_ GUARDED_BY(mu_) = 0;
-  int64 num_enqueued_tasks_ GUARDED_BY(mu_) = 0;
-  mutable mutex mu_;
-  TF_DISALLOW_COPY_AND_ASSIGN(ASBSQueue);
-};
-
-// Batch which remembers when and by whom it was created.
-template <typename TaskType>
-class ASBSBatch : public Batch<TaskType> {
- public:
-  ASBSBatch(ASBSQueue<TaskType>* queue, int64 creation_time_micros)
-      : queue_(queue), creation_time_micros_(creation_time_micros) {}
-
-  ~ASBSBatch() override {}
-
-  ASBSQueue<TaskType>* queue() const { return queue_; }
-
-  int64 creation_time_micros() const { return creation_time_micros_; }
-
- private:
-  ASBSQueue<TaskType>* queue_;
-  const int64 creation_time_micros_;
-  TF_DISALLOW_COPY_AND_ASSIGN(ASBSBatch);
-};
-}  // namespace internal
-
-// ---------------- AdaptiveSharedBatchScheduler ----------------
-
-template <typename TaskType>
-Status AdaptiveSharedBatchScheduler<TaskType>::Create(
-    const Options& options,
-    std::shared_ptr<AdaptiveSharedBatchScheduler<TaskType>>* scheduler) {
-  if (options.num_batch_threads < 1) {
-    return errors::InvalidArgument("num_batch_threads must be positive; was ",
-                                   options.num_batch_threads);
-  }
-  if (options.min_scheduling_period_micros < 0) {
-    return errors::InvalidArgument(
-        "min_scheduling_period_micros must be >= 0; was ",
-        options.min_scheduling_period_micros);
-  }
-  if (options.min_scheduling_period_micros >
-      options.initial_scheduling_period_micros) {
-    return errors::InvalidArgument(
-        "initial_scheduling_period_micros (",
-        options.initial_scheduling_period_micros,
-        ") must be >= min_scheduling_period_micros (",
-        options.min_scheduling_period_micros, ")");
-  }
-  if (options.initial_scheduling_period_micros >
-      options.max_scheduling_period_micros) {
-    return errors::InvalidArgument(
-        "initial_scheduling_period_micros (",
-        options.initial_scheduling_period_micros,
-        ") must be <= max_scheduling_period_micros (",
-        options.max_scheduling_period_micros, ")");
-  }
-  if (options.feedback_smoothing_batches < 1) {
-    return errors::InvalidArgument(
-        "feedback_smoothing_batches must be positive; was ",
-        options.feedback_smoothing_batches);
-  }
-  scheduler->reset(new AdaptiveSharedBatchScheduler<TaskType>(options));
-  return Status::OK();
-}
-
-template <typename TaskType>
-AdaptiveSharedBatchScheduler<TaskType>::AdaptiveSharedBatchScheduler(
-    const Options& options)
-    : options_(options),
-      scheduling_period_(options.initial_scheduling_period_micros) {
-  PeriodicFunction::Options opts;
-  opts.thread_name_prefix = "scheduling_thread";
-  opts.env = GetEnv();
-  scheduling_thread_.reset(
-      new PeriodicFunction([this] { ProcessOneBatch(); }, 0, opts));
-  batch_thread_pool_.reset(new thread::ThreadPool(
-      GetEnv(), options.thread_pool_name, options.num_batch_threads));
-}
-
-template <typename TaskType>
-Status AdaptiveSharedBatchScheduler<TaskType>::AddQueue(
-    const QueueOptions& options, BatchProcessor process_batch_callback,
-    std::unique_ptr<BatchScheduler<TaskType>>* queue) {
-  if (options.max_batch_size <= 0) {
-    return errors::InvalidArgument("max_batch_size must be positive; was ",
-                                   options.max_batch_size);
-  }
-  if (options.max_enqueued_batches <= 0) {
-    return errors::InvalidArgument(
-        "max_enqueued_batches must be positive; was ",
-        options.max_enqueued_batches);
-  }
-  internal::ASBSQueue<TaskType>* asbs_queue_raw;
-  queue->reset(asbs_queue_raw = new internal::ASBSQueue<TaskType>(
-                   this->shared_from_this(), options));
-  mutex_lock l(mu_);
-  queues_and_callbacks_[asbs_queue_raw] = process_batch_callback;
-  return Status::OK();
-}
-
-template <typename TaskType>
-void AdaptiveSharedBatchScheduler<TaskType>::AddBatch(
-    internal::ASBSBatch<TaskType>* batch) {
-  mutex_lock l(mu_);
-  batches_.push(batch);
-}
-
-template <typename TaskType>
-void AdaptiveSharedBatchScheduler<TaskType>::RemoveQueue(
-    const internal::ASBSQueue<TaskType>* queue) {
-  mutex_lock l(mu_);
-  queues_and_callbacks_.erase(queue);
-}
-
-template <typename TaskType>
-void AdaptiveSharedBatchScheduler<TaskType>::ProcessOneBatch() {
-  static const double kFeedbackMultiplier = .001;
-  internal::ASBSBatch<TaskType>* batch = nullptr;
-  BatchProcessor callback;
-  const int64 start_time_micros = GetEnv()->NowMicros();
-  {
-    mutex_lock l(mu_);
-    if (!batches_.empty()) {
-      batch = batches_.top();
-      batches_.pop();
-      callback = queues_and_callbacks_[batch->queue()];
-    }
-  }
-  if (batch != nullptr) {
-    double feedback = options_.scheduling_period_feedback();
-    const int64 N = options_.feedback_smoothing_batches;
-    ewma_feedback_ = ((N - 1) * ewma_feedback_ + feedback) / N;
-    scheduling_period_ *= (1 + kFeedbackMultiplier * ewma_feedback_);
-    if (scheduling_period_ < options_.min_scheduling_period_micros) {
-      scheduling_period_ = options_.min_scheduling_period_micros;
-    } else if (scheduling_period_ > options_.max_scheduling_period_micros) {
-      scheduling_period_ = options_.max_scheduling_period_micros;
-    }
-    // Queue may destroy itself after ReleaseBatch is called.
-    batch->queue()->ReleaseBatch(batch);
-    batch_thread_pool_->Schedule([callback, batch] {
-      callback(std::unique_ptr<Batch<TaskType>>(batch));
-    });
-  }
-  const int64 sleep_time =
-      scheduling_period_ - (GetEnv()->NowMicros() - start_time_micros);
-  if (sleep_time > 0) {
-    GetEnv()->SleepForMicroseconds(sleep_time);
-  }
-}
-
-template <typename TaskType>
-bool AdaptiveSharedBatchScheduler<TaskType>::BatchCompare::operator()(
-    const internal::ASBSBatch<TaskType>* a,
-    const internal::ASBSBatch<TaskType>* b) {
-  return a->creation_time_micros() > b->creation_time_micros();
-}
-
-// ---------------- ASBSQueue ----------------
-
-namespace internal {
-template <typename TaskType>
-ASBSQueue<TaskType>::ASBSQueue(
-    std::shared_ptr<AdaptiveSharedBatchScheduler<TaskType>> scheduler,
-    const QueueOptions& options)
-    : scheduler_(scheduler), options_(options) {}
-
-template <typename TaskType>
-ASBSQueue<TaskType>::~ASBSQueue() {
-  // Wait until last batch has been scheduled.
-  const int kSleepMicros = 1000;
-  for (;;) {
-    {
-      mutex_lock l(mu_);
-      if (num_enqueued_batches_ == 0) {
-        break;
-      }
-    }
-    scheduler_->GetEnv()->SleepForMicroseconds(kSleepMicros);
-  }
-  scheduler_->RemoveQueue(this);
-}
-
-template <typename TaskType>
-Status ASBSQueue<TaskType>::Schedule(std::unique_ptr<TaskType>* task) {
-  bool added_new_batch = false;
-  size_t size = (*task)->size();
-  if (size > options_.max_batch_size) {
-    return errors::InvalidArgument("Task size ", size,
-                                   " is larger than maximum batch size ",
-                                   options_.max_batch_size);
-  }
-  {
-    mutex_lock l(mu_);
-    // Current batch is full, create another if allowed.
-    if (current_batch_ &&
-        current_batch_->size() + size > options_.max_batch_size) {
-      if (num_enqueued_batches_ >= options_.max_enqueued_batches) {
-        return errors::Unavailable("The batch scheduling queue is full");
-      }
-      current_batch_->Close();
-      current_batch_ = nullptr;
-    }
-    if (!current_batch_) {
-      added_new_batch = true;
-      num_enqueued_batches_++;
-      current_batch_ =
-          new ASBSBatch<TaskType>(this, scheduler_->GetEnv()->NowMicros());
-    }
-    current_batch_->AddTask(std::move(*task));
-    num_enqueued_tasks_++;
-  }
-  if (added_new_batch) scheduler_->AddBatch(current_batch_);
-  return Status::OK();
-}
-
-template <typename TaskType>
-void ASBSQueue<TaskType>::ReleaseBatch(const ASBSBatch<TaskType>* batch) {
-  mutex_lock l(mu_);
-  num_enqueued_batches_--;
-  num_enqueued_tasks_ -= batch->num_tasks();
-  if (batch == current_batch_) {
-    current_batch_->Close();
-    current_batch_ = nullptr;
-  }
-}
-
-template <typename TaskType>
-size_t ASBSQueue<TaskType>::NumEnqueuedTasks() const {
-  mutex_lock l(mu_);
-  return num_enqueued_tasks_;
-}
-
-template <typename TaskType>
-size_t ASBSQueue<TaskType>::SchedulingCapacity() const {
-  mutex_lock l(mu_);
-  const int current_batch_capacity =
-      current_batch_ ? options_.max_batch_size - current_batch_->size() : 0;
-  const int spare_batches =
-      options_.max_enqueued_batches - num_enqueued_batches_;
-  return spare_batches * options_.max_batch_size + current_batch_capacity;
-}
-}  // namespace internal
-}  // namespace serving
-}  // namespace tensorflow
-
-#endif  // THIRD_PARTY_TENSORFLOW_CONTRIB_BATCHING_ADAPTIVE_SHARED_BATCH_SCHEDULER_H_
diff --git a/tensorflow/contrib/batching/adaptive_shared_batch_scheduler_test.cc b/tensorflow/contrib/batching/adaptive_shared_batch_scheduler_test.cc
deleted file mode 100644
index a07cd6d834..0000000000
--- a/tensorflow/contrib/batching/adaptive_shared_batch_scheduler_test.cc
+++ /dev/null
@@ -1,438 +0,0 @@
-/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#include "tensorflow/contrib/batching/adaptive_shared_batch_scheduler.h"
-
-#include "tensorflow/contrib/batching/test_util/fake_clock_env.h"
-#include "tensorflow/core/lib/core/notification.h"
-#include "tensorflow/core/lib/core/status_test_util.h"
-#include "tensorflow/core/platform/macros.h"
-#include "tensorflow/core/platform/test.h"
-
-namespace tensorflow {
-namespace serving {
-namespace anonymous {
-
-class FakeTask : public BatchTask {
- public:
-  explicit FakeTask(size_t size) : size_(size) {}
-
-  ~FakeTask() override = default;
-
-  size_t size() const override { return size_; }
-
- private:
-  const size_t size_;
-
-  TF_DISALLOW_COPY_AND_ASSIGN(FakeTask);
-};
-
-// Creates a FakeTask of size 'task_size', and calls 'scheduler->Schedule()' on
-// that task. Returns the resulting status.
-Status ScheduleTask(size_t task_size, BatchScheduler<FakeTask>* scheduler) {
-  std::unique_ptr<FakeTask> task(new FakeTask(task_size));
-  Status status = scheduler->Schedule(&task);
-  // Schedule() should have consumed 'task' iff it returned Status::OK.
-  CHECK_EQ(status.ok(), task == nullptr);
-  return status;
-}
-
-// Creates a thread that waits on 'start' and then advances the fake clock in
-// 'env' in a loop until 'stop' is notified. Useful for allowing objects that
-// use the clock to be destroyed.
-std::unique_ptr<Thread> CreateFakeClockAdvancerThread(
-    test_util::FakeClockEnv* env, Notification* start, Notification* stop) {
-  return std::unique_ptr<Thread>(Env::Default()->StartThread(
-      {}, "FakeClockAdvancerThread", [env, start, stop] {
-        start->WaitForNotification();
-        while (!stop->HasBeenNotified()) {
-          env->AdvanceByMicroseconds(10);
-          Env::Default()->SleepForMicroseconds(10);
-        }
-      }));
-}
-
-TEST(AdaptiveSharedBatchSchedulerTest, Basic) {
-  for (const bool delete_scheduler_early : {false, true}) {
-    for (const bool delete_queue_1_early : {false, true}) {
-      int queue_0_tasks = 0;
-      auto queue_0_callback =
-          [&queue_0_tasks](std::unique_ptr<Batch<FakeTask>> batch) {
-            ASSERT_TRUE(batch->IsClosed());
-            EXPECT_GT(batch->num_tasks(), 0);
-            for (int i = 0; i < batch->num_tasks(); i++) {
-              queue_0_tasks += batch->task(i).size();
-            }
-          };
-      int queue_1_tasks = 0;
-      auto queue_1_callback =
-          [&queue_1_tasks](std::unique_ptr<Batch<FakeTask>> batch) {
-            ASSERT_TRUE(batch->IsClosed());
-            EXPECT_GT(batch->num_tasks(), 0);
-            for (int i = 0; i < batch->num_tasks(); i++) {
-              queue_1_tasks += batch->task(i).size();
-            }
-          };
-      {
-        std::shared_ptr<AdaptiveSharedBatchScheduler<FakeTask>> scheduler;
-        TF_ASSERT_OK(
-            AdaptiveSharedBatchScheduler<FakeTask>::Create({}, &scheduler));
-
-        // Create two queues.
-        std::unique_ptr<BatchScheduler<FakeTask>> queue_0;
-        TF_ASSERT_OK(scheduler->AddQueue({}, queue_0_callback, &queue_0));
-        std::unique_ptr<BatchScheduler<FakeTask>> queue_1;
-        TF_ASSERT_OK(scheduler->AddQueue({}, queue_1_callback, &queue_1));
-
-        if (delete_scheduler_early) {
-          // Delete our copy of the scheduler. The queues should keep it alive
-          // under the covers.
-          scheduler = nullptr;
-        }
-        // Submit tasks to the two queues, and (optionally) remove the queues.
-        TF_ASSERT_OK(ScheduleTask(1, queue_0.get()));
-        TF_ASSERT_OK(ScheduleTask(2, queue_1.get()));
-        TF_ASSERT_OK(ScheduleTask(3, queue_0.get()));
-        TF_ASSERT_OK(ScheduleTask(4, queue_1.get()));
-        if (delete_queue_1_early) {
-          queue_1 = nullptr;
-        }
-        TF_ASSERT_OK(ScheduleTask(5, queue_0.get()));
-      }
-      EXPECT_EQ(queue_0_tasks, 9);
-      EXPECT_EQ(queue_1_tasks, 6);
-    }
-  }
-}
-
-TEST(AdaptiveSharedBatchSchedulerTest, BadOptions) {
-  using Scheduler = AdaptiveSharedBatchScheduler<FakeTask>;
-  std::shared_ptr<Scheduler> scheduler;
-  Scheduler::Options options;
-  options.num_batch_threads = 0;
-  EXPECT_FALSE(Scheduler::Create(options, &scheduler).ok());
-  options = Scheduler::Options();
-  options.min_scheduling_period_micros = 50;
-  options.max_scheduling_period_micros = 100;
-  options.initial_scheduling_period_micros = 1;
-  EXPECT_FALSE(Scheduler::Create(options, &scheduler).ok());
-  options = Scheduler::Options();
-  options.min_scheduling_period_micros = 50;
-  options.max_scheduling_period_micros = 100;
-  options.initial_scheduling_period_micros = 1000;
-  EXPECT_FALSE(Scheduler::Create(options, &scheduler).ok());
-  options = Scheduler::Options();
-  options.min_scheduling_period_micros = 100;
-  options.max_scheduling_period_micros = 50;
-  options.initial_scheduling_period_micros = 75;
-  EXPECT_FALSE(Scheduler::Create(options, &scheduler).ok());
-  options = Scheduler::Options();
-  options.feedback_smoothing_batches = 0;
-  EXPECT_FALSE(Scheduler::Create(options, &scheduler).ok());
-}
-
-TEST(AdaptiveSharedBatchSchedulerTest, ObeysQueueOptions) {
-  test_util::FakeClockEnv env(Env::Default());
-  Notification start_teardown, stop_teardown;
-  std::unique_ptr<Thread> teardown_thread =
-      CreateFakeClockAdvancerThread(&env, &start_teardown, &stop_teardown);
-  {
-    AdaptiveSharedBatchScheduler<FakeTask>::Options options;
-    options.initial_scheduling_period_micros = 1000;
-    options.env = &env;
-    std::shared_ptr<AdaptiveSharedBatchScheduler<FakeTask>> scheduler;
-    TF_ASSERT_OK(
-        AdaptiveSharedBatchScheduler<FakeTask>::Create(options, &scheduler));
-    std::unique_ptr<BatchScheduler<FakeTask>> queue_0;
-    std::unique_ptr<BatchScheduler<FakeTask>> queue_1;
-    int queue_0_tasks = 0;
-    int queue_1_tasks = 0;
-    auto queue_0_callback = [&queue_0_tasks,
-                             &env](std::unique_ptr<Batch<FakeTask>> batch) {
-      ASSERT_TRUE(batch->IsClosed());
-      EXPECT_GT(batch->num_tasks(), 0);
-      for (int i = 0; i < batch->num_tasks(); i++) {
-        queue_0_tasks += batch->task(i).size();
-      }
-      env.SleepForMicroseconds(1);
-    };
-    auto queue_1_callback = [&queue_1_tasks,
-                             &env](std::unique_ptr<Batch<FakeTask>> batch) {
-      ASSERT_TRUE(batch->IsClosed());
-      EXPECT_GT(batch->num_tasks(), 0);
-      for (int i = 0; i < batch->num_tasks(); i++) {
-        queue_1_tasks += batch->task(i).size();
-      }
-      env.SleepForMicroseconds(1);
-    };
-    AdaptiveSharedBatchScheduler<FakeTask>::QueueOptions queue_options;
-    queue_options.max_batch_size = 10;
-    queue_options.max_enqueued_batches = 0;
-    // Queue must have max_enqueued_batchs > 1.
-    EXPECT_FALSE(
-        scheduler->AddQueue(queue_options, queue_0_callback, &queue_0).ok());
-    queue_options.max_enqueued_batches = 2;
-    TF_ASSERT_OK(
-        scheduler->AddQueue(queue_options, queue_0_callback, &queue_0));
-    queue_options.max_batch_size = 0;
-    // Queue must have max_batch_size > 0.
-    EXPECT_FALSE(
-        scheduler->AddQueue(queue_options, queue_1_callback, &queue_1).ok());
-    queue_options.max_batch_size = 2;
-    queue_options.max_enqueued_batches = 1;
-    TF_ASSERT_OK(
-        scheduler->AddQueue(queue_options, queue_1_callback, &queue_1));
-
-    // Wait for scheduling_thread to sleep.
-    env.BlockUntilThreadsAsleep(1);
-    // Task larger than max_batch_size shouldn't schedule.
-    EXPECT_FALSE(ScheduleTask(15, queue_0.get()).ok());
-    TF_ASSERT_OK(ScheduleTask(5, queue_0.get()));
-    TF_ASSERT_OK(ScheduleTask(5, queue_0.get()));
-    env.AdvanceByMicroseconds(1);
-
-    // Task larger than max_batch_size shouldn't schedule.
-    EXPECT_FALSE(ScheduleTask(3, queue_1.get()).ok());
-    TF_ASSERT_OK(ScheduleTask(1, queue_1.get()));
-    TF_ASSERT_OK(ScheduleTask(1, queue_1.get()));
-    env.AdvanceByMicroseconds(1);
-    // Exceeds max_enqueued_batches, shouldn't schedule.
-    EXPECT_FALSE(ScheduleTask(1, queue_1.get()).ok());
-
-    TF_ASSERT_OK(ScheduleTask(5, queue_0.get()));
-    // Exceeds max_enqueued_batches, shouldn't schedule.
-    EXPECT_FALSE(ScheduleTask(6, queue_0.get()).ok());
-    TF_ASSERT_OK(ScheduleTask(4, queue_0.get()));
-
-    // Batches should be processed in order from oldest to newest.
-    env.AdvanceByMicroseconds(1000);
-    env.BlockUntilThreadsAsleep(2);
-    EXPECT_EQ(queue_0_tasks, 10);
-    EXPECT_EQ(queue_1_tasks, 0);
-
-    env.AdvanceByMicroseconds(1000);
-    env.BlockUntilThreadsAsleep(2);
-    EXPECT_EQ(queue_0_tasks, 10);
-    EXPECT_EQ(queue_1_tasks, 2);
-
-    env.AdvanceByMicroseconds(1000);
-    env.BlockUntilThreadsAsleep(2);
-    EXPECT_EQ(queue_0_tasks, 19);
-    EXPECT_EQ(queue_1_tasks, 2);
-    start_teardown.Notify();
-  }
-  stop_teardown.Notify();
-}
-
-TEST(AdaptiveSharedBatchSchedulerTest, RateFeedback) {
-  test_util::FakeClockEnv env(Env::Default());
-  Notification start_teardown, stop_teardown;
-  std::unique_ptr<Thread> teardown_thread =
-      CreateFakeClockAdvancerThread(&env, &start_teardown, &stop_teardown);
-  {
-    double feedback = 0;
-    AdaptiveSharedBatchScheduler<FakeTask>::Options options;
-    options.initial_scheduling_period_micros = 1000;
-    options.min_scheduling_period_micros = 200;
-    options.max_scheduling_period_micros = 2000;
-    options.env = &env;
-    options.scheduling_period_feedback = [&feedback] { return feedback; };
-    options.feedback_smoothing_batches = 1;
-    std::shared_ptr<AdaptiveSharedBatchScheduler<FakeTask>> scheduler;
-    TF_ASSERT_OK(
-        AdaptiveSharedBatchScheduler<FakeTask>::Create(options, &scheduler));
-    std::unique_ptr<BatchScheduler<FakeTask>> queue;
-    int scheduled_items = 0;
-    auto queue_callback = [&scheduled_items,
-                           &env](std::unique_ptr<Batch<FakeTask>> batch) {
-      ASSERT_TRUE(batch->IsClosed());
-      EXPECT_GT(batch->num_tasks(), 0);
-      scheduled_items = 0;
-      for (int i = 0; i < batch->num_tasks(); i++) {
-        scheduled_items += batch->task(i).size();
-      }
-      env.SleepForMicroseconds(1);
-    };
-
-    TF_ASSERT_OK(scheduler->AddQueue({}, queue_callback, &queue));
-
-    // Wait for scheduling_thread to sleep.
-    env.BlockUntilThreadsAsleep(1);
-    // Enqueue 6 batches.
-    for (int i = 0; i < 6; i++) {
-      TF_ASSERT_OK(ScheduleTask(900 + i, queue.get()));
-      env.AdvanceByMicroseconds(1);
-    }
-    feedback = -500;
-    env.AdvanceByMicroseconds(994);
-    env.BlockUntilThreadsAsleep(2);  // scheduling period = 500 usec.
-    EXPECT_EQ(scheduled_items, 900);
-    env.AdvanceByMicroseconds(500);
-    env.BlockUntilThreadsAsleep(2);  // scheduling period = 250 usec.
-    EXPECT_EQ(scheduled_items, 901);
-    feedback = 0;
-    env.AdvanceByMicroseconds(250);
-    env.BlockUntilThreadsAsleep(2);  // scheduling period = 250 usec.
-    EXPECT_EQ(scheduled_items, 902);
-    feedback = 10000;  // large feedback should hit max_scheduling_period.
-    env.AdvanceByMicroseconds(250);
-    env.BlockUntilThreadsAsleep(2);  // scheduling period = 2000 usec.
-    EXPECT_EQ(scheduled_items, 903);
-    feedback = -10000;  // large feedback should hit min_scheduling_period.
-    env.AdvanceByMicroseconds(1999);
-    // No callback scheduled, only scheduling thread sleeping.
-    env.BlockUntilThreadsAsleep(1);
-    EXPECT_EQ(scheduled_items, 903);
-    env.AdvanceByMicroseconds(1);
-    env.BlockUntilThreadsAsleep(2);  // scheduling period = 200 usec.
-    EXPECT_EQ(scheduled_items, 904);
-    env.AdvanceByMicroseconds(200);
-    env.BlockUntilThreadsAsleep(2);
-    EXPECT_EQ(scheduled_items, 905);
-    start_teardown.Notify();
-  }
-  stop_teardown.Notify();
-}
-
-TEST(AdaptiveSharedBatchSchedulerTest, FeedbackSmoothing) {
-  test_util::FakeClockEnv env(Env::Default());
-  Notification start_teardown, stop_teardown;
-  std::unique_ptr<Thread> teardown_thread =
-      CreateFakeClockAdvancerThread(&env, &start_teardown, &stop_teardown);
-  {
-    double feedback = 0;
-    AdaptiveSharedBatchScheduler<FakeTask>::Options options;
-    options.initial_scheduling_period_micros = 1000;
-    options.env = &env;
-    options.scheduling_period_feedback = [&feedback] { return feedback; };
-    options.feedback_smoothing_batches = 3;
-    std::shared_ptr<AdaptiveSharedBatchScheduler<FakeTask>> scheduler;
-    TF_ASSERT_OK(
-        AdaptiveSharedBatchScheduler<FakeTask>::Create(options, &scheduler));
-    std::unique_ptr<BatchScheduler<FakeTask>> queue;
-    int scheduled_items = 0;
-    auto queue_callback = [&scheduled_items,
-                           &env](std::unique_ptr<Batch<FakeTask>> batch) {
-      ASSERT_TRUE(batch->IsClosed());
-      EXPECT_GT(batch->num_tasks(), 0);
-      scheduled_items = 0;
-      for (int i = 0; i < batch->num_tasks(); i++) {
-        scheduled_items += batch->task(i).size();
-      }
-      env.SleepForMicroseconds(1);
-    };
-
-    TF_ASSERT_OK(scheduler->AddQueue({}, queue_callback, &queue));
-
-    // Wait for scheduling_thread to sleep.
-    env.BlockUntilThreadsAsleep(1);
-    // Enqueue 4 batches.
-    for (int i = 0; i < 4; i++) {
-      TF_ASSERT_OK(ScheduleTask(900 + i, queue.get()));
-      env.AdvanceByMicroseconds(1);
-    }
-    feedback = -300;
-    env.AdvanceByMicroseconds(996);
-    env.BlockUntilThreadsAsleep(2);
-    // ewma_feedback = 100, scheduling_period = 900.
-    EXPECT_EQ(scheduled_items, 900);
-    env.AdvanceByMicroseconds(899);
-    // No callback scheduled, only scheduling thread sleeping.
-    env.BlockUntilThreadsAsleep(1);
-    EXPECT_EQ(scheduled_items, 900);
-    env.AdvanceByMicroseconds(1);
-    env.BlockUntilThreadsAsleep(2);
-    // ewma_feedback = 167, scheduling_period = 750.
-    EXPECT_EQ(scheduled_items, 901);
-    env.AdvanceByMicroseconds(749);
-    // No callback scheduled, only scheduling thread sleeping.
-    env.BlockUntilThreadsAsleep(1);
-    EXPECT_EQ(scheduled_items, 901);
-    feedback = 1000 / 3.;
-    env.AdvanceByMicroseconds(1);
-    env.BlockUntilThreadsAsleep(2);
-    // emwa_feedback = 0, scheduling_period = 750.
-    EXPECT_EQ(scheduled_items, 902);
-    env.AdvanceByMicroseconds(749);
-    // No callback scheduled, only scheduling thread sleeping.
-    env.BlockUntilThreadsAsleep(1);
-    EXPECT_EQ(scheduled_items, 902);
-    env.AdvanceByMicroseconds(1);
-    env.BlockUntilThreadsAsleep(2);
-    EXPECT_EQ(scheduled_items, 903);
-    start_teardown.Notify();
-  }
-  stop_teardown.Notify();
-}
-
-TEST(AdaptiveSharedBatchSchedulerTest, QueueCapacityInfo) {
-  test_util::FakeClockEnv env(Env::Default());
-  Notification start_teardown, stop_teardown;
-  std::unique_ptr<Thread> teardown_thread =
-      CreateFakeClockAdvancerThread(&env, &start_teardown, &stop_teardown);
-  {
-    AdaptiveSharedBatchScheduler<FakeTask>::Options options;
-    options.initial_scheduling_period_micros = 1000;
-    options.env = &env;
-    std::shared_ptr<AdaptiveSharedBatchScheduler<FakeTask>> scheduler;
-    TF_ASSERT_OK(
-        AdaptiveSharedBatchScheduler<FakeTask>::Create(options, &scheduler));
-    std::unique_ptr<BatchScheduler<FakeTask>> queue;
-    int scheduled_items = 0;
-    auto queue_callback = [&scheduled_items,
-                           &env](std::unique_ptr<Batch<FakeTask>> batch) {
-      ASSERT_TRUE(batch->IsClosed());
-      EXPECT_GT(batch->num_tasks(), 0);
-      scheduled_items = 0;
-      for (int i = 0; i < batch->num_tasks(); i++) {
-        scheduled_items += batch->task(i).size();
-      }
-      env.SleepForMicroseconds(1);
-    };
-    AdaptiveSharedBatchScheduler<FakeTask>::QueueOptions queue_options;
-    queue_options.max_batch_size = 10;
-    queue_options.max_enqueued_batches = 10;
-    TF_ASSERT_OK(scheduler->AddQueue(queue_options, queue_callback, &queue));
-
-    // Wait for scheduling_thread to sleep.
-    env.BlockUntilThreadsAsleep(1);
-    // Enqueue 3 tasks.
-    EXPECT_EQ(queue->NumEnqueuedTasks(), 0);
-    EXPECT_EQ(queue->SchedulingCapacity(), 100);
-    TF_ASSERT_OK(ScheduleTask(5, queue.get()));
-    EXPECT_EQ(queue->NumEnqueuedTasks(), 1);
-    EXPECT_EQ(queue->SchedulingCapacity(), 95);
-    env.AdvanceByMicroseconds(1);
-    TF_ASSERT_OK(ScheduleTask(6, queue.get()));
-    EXPECT_EQ(queue->NumEnqueuedTasks(), 2);
-    EXPECT_EQ(queue->SchedulingCapacity(), 84);
-    env.AdvanceByMicroseconds(1);
-    TF_ASSERT_OK(ScheduleTask(1, queue.get()));
-    EXPECT_EQ(queue->NumEnqueuedTasks(), 3);
-    EXPECT_EQ(queue->SchedulingCapacity(), 83);
-
-    env.AdvanceByMicroseconds(998);
-    env.BlockUntilThreadsAsleep(2);
-    EXPECT_EQ(scheduled_items, 5);
-    env.AdvanceByMicroseconds(1000);
-    env.BlockUntilThreadsAsleep(2);
-    EXPECT_EQ(scheduled_items, 7);
-    start_teardown.Notify();
-  }
-  stop_teardown.Notify();
-}
-}  // namespace anonymous
-}  // namespace serving
-}  // namespace tensorflow
diff --git a/tensorflow/contrib/batching/batch_scheduler.h b/tensorflow/contrib/batching/batch_scheduler.h
index a5072f439a..7c41ad8818 100644
--- a/tensorflow/contrib/batching/batch_scheduler.h
+++ b/tensorflow/contrib/batching/batch_scheduler.h
@@ -78,7 +78,7 @@ template <typename TaskType>
 class Batch {
  public:
   Batch() = default;
-  virtual ~Batch();  // Blocks until the batch is closed.
+  ~Batch();  // Blocks until the batch is closed.
 
   // Appends 'task' to the batch. After calling AddTask(), the newly-added task
   // can be accessed via task(num_tasks()-1) or mutable_task(num_tasks()-1).
-- 
GitLab


From 37fd951790d7ad27c679c925c28b01ca73875738 Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Fri, 20 Oct 2017 14:49:56 -0700
Subject: [PATCH 630/909] Simplifies capturing code in graph_callable to use
 recent function improvements.

PiperOrigin-RevId: 172937003
---
 tensorflow/python/eager/graph_callable.py     | 57 +++----------------
 .../python/ops/resource_variable_ops.py       | 15 ++---
 2 files changed, 14 insertions(+), 58 deletions(-)

diff --git a/tensorflow/python/eager/graph_callable.py b/tensorflow/python/eager/graph_callable.py
index 0ec83636a0..7f7a8c4a88 100644
--- a/tensorflow/python/eager/graph_callable.py
+++ b/tensorflow/python/eager/graph_callable.py
@@ -45,28 +45,6 @@ def _default_initializer(name, shape, dtype):
   return initializer[0]
 
 
-class _VariableFromResource(resource_variable_ops.ResourceVariable):
-  """Variable object from a preexisting resource.
-
-  Required because the ResourceVariable constructor creates the resource handle,
-  and here we want to use a preexisting one.
-  """
-
-  def __init__(self, resource, dtype, name, shape):
-    self._handle = resource
-    self._graph_shape = tensor_shape.as_shape(shape)
-    self._handle_device = resource.device
-    self._handle_name = name
-    self._cached_value = None
-    self._initializer_op = None
-    self._caching_device = None
-    self._dtype = dtype
-    self._constraint = None
-    self._in_graph_mode = context.in_graph_mode()
-    if self._in_graph_mode:
-      self._graph_element = self.read_value()
-
-
 class _CapturedVariable(object):
   """Variable captured by graph_callable.
 
@@ -137,17 +115,11 @@ class _VariableCapturingScope(object):
                        trainable=True, collections=None, caching_device=None,  # pylint: disable=redefined-outer-name
                        partitioner=None, validate_shape=True,
                        use_resource=None):
-      del getter, regularizer, partitioner, validate_shape, use_resource
-      del collections, initializer, trainable, reuse, caching_device
+      del getter, regularizer, partitioner, validate_shape, use_resource, dtype
+      del collections, initializer, trainable, reuse, caching_device, shape,
       assert name in self.variables
       v = self.variables[name]
-      v.placeholder = array_ops.placeholder(dtype=dtypes.resource, shape=shape)
-      # TODO(apassos) remove the need for this by correctly dealing with shape
-      # inference.
-      v.placeholder._handle_data = v.variable.handle._handle_data  # pylint: disable=protected-access
-      return _VariableFromResource(
-          v.placeholder, dtype=dtypes.as_dtype(dtype), name=name,
-          shape=v.shape)
+      return v.variable
 
     scope = variable_scope.get_variable_scope()
     with variable_scope.variable_scope(scope, custom_getter=_custom_getter):
@@ -181,14 +153,12 @@ class _VariableCapturingScope(object):
       v = _CapturedVariable(name, initializer, shape, dtype, trainable)
       self.variables[name] = v
 
-      graph_mode_resource = resource_variable_ops.var_handle_op(
-          shared_name=name, shape=shape, dtype=dtype)
+      graph_mode_resource = v.variable.handle
       if initializer is None:
         initializer = _default_initializer(name, shape, dtype)
       resource_variable_ops.assign_variable_op(
           graph_mode_resource, initializer(shape, dtype))
-      return _VariableFromResource(
-          graph_mode_resource, dtype, name, shape=v.shape)
+      return v.variable
 
     scope = variable_scope.get_variable_scope()
     with variable_scope.variable_scope(scope, custom_getter=_custom_getter):
@@ -220,13 +190,6 @@ class _FunctionObject(function._GraphModeFunction):  # pylint: disable=protected
   def variables(self):
     return [x.variable for x in self._variables]
 
-  def __call__(self, *args, **kwds):
-    kwds.pop("want_gradients", False)
-    if kwds:
-      raise ValueError("graph_callable functions do not take keyword args")
-    values = [x.variable.handle for x in self._variables]
-    return super(_FunctionObject, self).__call__(*(values + list(args)))
-
 
 class _InitializingFunctionObject(object):
   """Responsible for deciding which version of func-to-object to call.
@@ -318,7 +281,8 @@ def _graph_callable_internal(func, shape_and_dtypes):
     # This graph will store both the initialization and the call version of the
     # wrapped function. It will later be used by the backprop code to build the
     # backprop graph, if necessary.
-    tmp_graph = tf_ops.Graph()
+    captures = {}
+    tmp_graph = function.CapturingGraph(captures)
     # Inherit the container from the original graph to create resources at user
     # expected containers. Also inherits the container prefix, since this is
     # used for error checking when isolating Eager execution (the container
@@ -342,7 +306,6 @@ def _graph_callable_internal(func, shape_and_dtypes):
       # variables. As a side-effect this will populate the variable capturing
       # scope's view of which variables exist.
       variable_captures = _VariableCapturingScope()
-      captures = {}
       with variable_captures.initializing_scope(), function.capture_tensors(
           captures):
         func_outputs = func(*func_inputs)
@@ -366,7 +329,6 @@ def _graph_callable_internal(func, shape_and_dtypes):
 
   sorted_variables = sorted(variable_captures.variables.values(),
                             key=lambda x: x.name)
-  variable_placeholders = [x.placeholder for x in sorted_variables]
   ids = list(sorted(captures.keys()))
   if ids:
     extra_inputs, extra_placeholders = zip(*[captures[x] for x in ids])
@@ -377,7 +339,6 @@ def _graph_callable_internal(func, shape_and_dtypes):
   flat_inputs = [x for x in nest.flatten(func_inputs)
                  if isinstance(x, tf_ops.Tensor)]
   placeholder_inputs = flat_inputs+ list(extra_placeholders)
-  all_inputs = variable_placeholders + placeholder_inputs
 
   func_def_outputs = [x for x in outputs_list if isinstance(x, tf_ops.Tensor)]
   initializer_function_def = function.make_function_def(
@@ -407,13 +368,13 @@ def _graph_callable_internal(func, shape_and_dtypes):
   captured_function_def = function.make_function_def(
       tmp_graph,
       capturing_operations,
-      all_inputs,
+      placeholder_inputs,
       capture_func_def_outputs)
   function._register_with_name(function._inference_name(func.__name__),  # pylint: disable=protected-access
                                captured_function_def)
   captured_function = _FunctionObject(
       sorted_variables,
-      all_inputs,
+      placeholder_inputs,
       extra_inputs,
       captured_function_def,
       tmp_graph,
diff --git a/tensorflow/python/ops/resource_variable_ops.py b/tensorflow/python/ops/resource_variable_ops.py
index c94ddb0627..71e1fb0297 100644
--- a/tensorflow/python/ops/resource_variable_ops.py
+++ b/tensorflow/python/ops/resource_variable_ops.py
@@ -26,7 +26,6 @@ from tensorflow.python.eager import tape
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
-from tensorflow.python.framework import tensor_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gen_array_ops
 from tensorflow.python.ops import gen_resource_variable_ops
@@ -315,7 +314,7 @@ class ResourceVariable(variables.Variable):
               self._handle_device = (
                   self._handle.device if self._in_graph_mode else
                   context.get_default_context().device_name)
-              self._graph_shape = initial_value.get_shape()
+              self._shape = initial_value.get_shape()
           else:
             initial_value = initial_value()
             with ops.name_scope("Initializer"):
@@ -330,7 +329,7 @@ class ResourceVariable(variables.Variable):
             self._handle_device = (
                 self._handle.device if self._in_graph_mode else
                 context.get_default_context().device_name)
-            self._graph_shape = initial_value.get_shape()
+            self._shape = initial_value.get_shape()
         # pylint: enable=protected-access
 
         # Or get the initial value from a Tensor or Python object.
@@ -355,7 +354,7 @@ class ResourceVariable(variables.Variable):
               graph_mode=self._in_graph_mode)
           self._handle_device = (self._handle.device if self._in_graph_mode else
                                  context.get_default_context().device_name)
-          self._graph_shape = initial_value.get_shape()
+          self._shape = initial_value.get_shape()
 
         self._initial_value = initial_value if self._in_graph_mode else None
         self._handle_name = handle_name + ":0"
@@ -422,7 +421,7 @@ class ResourceVariable(variables.Variable):
     self._handle = g.as_graph_element(
         ops.prepend_name_scope(
             variable_def.variable_name, import_scope=import_scope))
-    self._graph_shape = tensor_shape.TensorShape(
+    self._shape = tensor_shape.TensorShape(
         self._handle.op.get_attr("shape"))
     self._handle_device = self._handle.device
     self._handle_name = self._handle.name
@@ -502,11 +501,7 @@ class ResourceVariable(variables.Variable):
   @property
   def shape(self):
     """The shape of this variable."""
-    if self._in_graph_mode:
-      return self._graph_shape
-    return tensor_shape.TensorShape(
-        tensor_util.constant_value(
-            gen_resource_variable_ops.variable_shape(self._handle)))
+    return self._shape
 
   @property
   def create(self):
-- 
GitLab


From 41df2cec28274cff4538f4735202471f8da45ce8 Mon Sep 17 00:00:00 2001
From: ashankar <ashankar@google.com>
Date: Fri, 20 Oct 2017 14:06:54 -0800
Subject: [PATCH 631/909] Testing pending CL: 172939383

---
 tensorflow/contrib/eager/README.OPENSOURCE.md |  15 +
 tensorflow/contrib/eager/README.md            |  74 +-
 .../contrib/eager/python/examples/BUILD       | 134 +++
 .../eager/python/examples/cart_pole.py        | 282 ++++++
 .../eager/python/examples/cart_pole_helper.py |  60 ++
 .../python/examples/linear_regression.py      | 197 ++++
 .../python/examples/notebooks/1_basics.ipynb  | 529 +++++++++++
 .../examples/notebooks/2_gradients.ipynb      | 864 ++++++++++++++++++
 .../examples/notebooks/3_datasets.ipynb       | 218 +++++
 .../examples/tests/cart_pole_helper_test.py   |  51 ++
 .../python/examples/tests/cart_pole_test.py   | 162 ++++
 .../examples/tests/linear_regression_test.py  | 114 +++
 .../eager/python/examples/tests/spinn_test.py | 311 +++++++
 13 files changed, 2999 insertions(+), 12 deletions(-)
 create mode 100644 tensorflow/contrib/eager/README.OPENSOURCE.md
 create mode 100644 tensorflow/contrib/eager/python/examples/BUILD
 create mode 100644 tensorflow/contrib/eager/python/examples/cart_pole.py
 create mode 100644 tensorflow/contrib/eager/python/examples/cart_pole_helper.py
 create mode 100644 tensorflow/contrib/eager/python/examples/linear_regression.py
 create mode 100644 tensorflow/contrib/eager/python/examples/notebooks/1_basics.ipynb
 create mode 100644 tensorflow/contrib/eager/python/examples/notebooks/2_gradients.ipynb
 create mode 100644 tensorflow/contrib/eager/python/examples/notebooks/3_datasets.ipynb
 create mode 100644 tensorflow/contrib/eager/python/examples/tests/cart_pole_helper_test.py
 create mode 100644 tensorflow/contrib/eager/python/examples/tests/cart_pole_test.py
 create mode 100644 tensorflow/contrib/eager/python/examples/tests/linear_regression_test.py
 create mode 100644 tensorflow/contrib/eager/python/examples/tests/spinn_test.py

diff --git a/tensorflow/contrib/eager/README.OPENSOURCE.md b/tensorflow/contrib/eager/README.OPENSOURCE.md
new file mode 100644
index 0000000000..a4a3af08cf
--- /dev/null
+++ b/tensorflow/contrib/eager/README.OPENSOURCE.md
@@ -0,0 +1,15 @@
+TensorFlow has many kernels for doing (deep) learning and data manipulation.
+There are typically assembled into computational graphs which can run
+efficiently in a variety of environments.
+
+We are exploring an alternative interaction, where kernels are invoked
+immediately and call this "eager execution". We are hoping to retain the
+benefits of graphs while improving usability with benefits like:
+
+- Immediate error messages and easier debugging
+- Flexibility to use Python datastructures and control flow
+- Reduced boilerplate
+
+Eager execution is under active development.
+There are not many developer-facing materials yet, but stay tuned for updates
+in this directory.
diff --git a/tensorflow/contrib/eager/README.md b/tensorflow/contrib/eager/README.md
index a4a3af08cf..fe577fa7eb 100644
--- a/tensorflow/contrib/eager/README.md
+++ b/tensorflow/contrib/eager/README.md
@@ -1,15 +1,65 @@
-TensorFlow has many kernels for doing (deep) learning and data manipulation.
-There are typically assembled into computational graphs which can run
-efficiently in a variety of environments.
+# TensorFlow Eager Execution
 
-We are exploring an alternative interaction, where kernels are invoked
-immediately and call this "eager execution". We are hoping to retain the
-benefits of graphs while improving usability with benefits like:
+> *WARNING*: This is a preview/pre-alpha version. The API and performance
+> characteristics are subject to change.
 
-- Immediate error messages and easier debugging
-- Flexibility to use Python datastructures and control flow
-- Reduced boilerplate
 
-Eager execution is under active development.
-There are not many developer-facing materials yet, but stay tuned for updates
-in this directory.
+Eager execution is an experimental interface to TensorFlow that provides an
+imperative programming style (à la [NumPy](http://www.numpy.org)). When you
+enable eager execution, TensorFlow operations execute immediately; you do not
+execute a pre-constructed graph with
+[`Session.run()`](https://www.tensorflow.org/api_docs/python/tf/Session).
+
+For example, consider a simple computation in TensorFlow:
+
+```python
+x = tf.placeholder(tf.float32, shape=[1, 1])
+m = tf.matmul(x, x)
+
+with tf.Session() as sess:
+  print(sess.run(m, feed_dict={x: [[2.]]}))
+
+# Will print [[4.]]
+```
+
+Eager execution makes this much simpler:
+
+```python
+x = [[2.]]
+m = tf.matmul(x, x)
+
+print(m)
+```
+
+## Installation
+
+Since eager execution is not yet part of a TensorFlow release, using it requires
+either [building from source](https://www.tensorflow.org/install/install_sources)
+or the latest nightly builds. The nightly builds are available as:
+
+- [`pip` packages](https://github.com/tensorflow/tensorflow/blob/master/README.md#installation) and
+
+- [docker](https://hub.docker.com/r/tensorflow/tensorflow/) images.
+
+For example, to run the latest nightly docker image:
+
+```sh
+# If you have a GPU, use https://github.com/NVIDIA/nvidia-docker
+nvidia-docker pull tensorflow/tensorflow:nightly-gpu
+nvidia-docker run -it -p 8888:8888 tensorflow/tensorflow:nightly-gpu
+
+# If you do not have a GPU, use the CPU-only image
+docker pull tensorflow/tensorflow:nightly
+docker run -it -p 8888:8888 tensorflow/tensorflow:nightly
+```
+
+And then visit http://localhost:8888 in your browser for a Jupyter notebook
+environment. Try out the notebooks below.
+
+## Documentation
+
+For an introduction to TensorFlow eager execution, see the Jupyter notebooks:
+
+- [Basic Usage](examples/notebooks/1_basics.ipynb)
+- [Gradients](examples/notebooks/2_gradients.ipynb)
+- [Importing Data](examples/notebooks/3_datasets.ipynb)
diff --git a/tensorflow/contrib/eager/python/examples/BUILD b/tensorflow/contrib/eager/python/examples/BUILD
new file mode 100644
index 0000000000..3604139819
--- /dev/null
+++ b/tensorflow/contrib/eager/python/examples/BUILD
@@ -0,0 +1,134 @@
+# Description:
+#   Open-source examples and tutorials for TensorFlow Eager Execution.
+
+licenses(["notice"])  # Apache 2.0
+
+exports_files(["LICENSE"])
+
+load("//tensorflow:tensorflow.bzl", "py_test")
+load("//tensorflow:tensorflow.bzl", "cuda_py_test")
+
+py_binary(
+    name = "linear_regression",
+    srcs = ["linear_regression.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        "//tensorflow:tensorflow_py",
+        "//tensorflow/contrib/eager/python:tfe",
+        "@six_archive//:six",
+    ],
+)
+
+py_library(
+    name = "cart_pole_helper",
+    srcs = ["cart_pole_helper.py"],
+    srcs_version = "PY2AND3",
+)
+
+py_library(
+    name = "spinn",
+    srcs = ["spinn.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        "//tensorflow:tensorflow_py",
+        "//third_party/py/numpy",
+        "@six_archive//:six",
+    ],
+)
+
+py_binary(
+    name = "cart_pole",
+    srcs = ["cart_pole.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":cart_pole_helper",
+        "//tensorflow:tensorflow_py",
+        "//tensorflow/contrib/eager/python:tfe",
+        "@six_archive//:six",
+    ],
+)
+
+py_binary(
+    name = "spinn_prep_data",
+    srcs = ["spinn_prep_data.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        "//tensorflow:tensorflow_py",
+    ],
+)
+
+py_binary(
+    name = "spinn_train",
+    srcs = ["spinn_train.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":spinn",
+        "//tensorflow:tensorflow_py",
+        "//tensorflow/contrib/eager/python:tfe",
+        "//third_party/py/numpy",
+        "@six_archive//:six",
+    ],
+)
+
+cuda_py_test(
+    name = "linear_regression_test",
+    size = "small",
+    srcs = ["tests/linear_regression_test.py"],
+    additional_deps = [
+        ":linear_regression",
+        "//tensorflow/python/eager:test",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework_test_lib",
+    ],
+)
+
+py_test(
+    name = "cart_pole_helper_test",
+    srcs = ["tests/cart_pole_helper_test.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":cart_pole_helper",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework_test_lib",
+    ],
+)
+
+cuda_py_test(
+    name = "cart_pole_test",
+    size = "small",
+    srcs = ["tests/cart_pole_test.py"],
+    additional_deps = [
+        ":cart_pole",
+        "//tensorflow/python/eager:test",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:training",
+    ],
+)
+
+cuda_py_test(
+    name = "spinn_test",
+    size = "medium",
+    srcs = ["tests/spinn_test.py"],
+    additional_deps = [
+        ":spinn",
+        "//third_party/py/numpy",
+        "//tensorflow:tensorflow_py",
+        "//tensorflow/python/eager:test",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework_test_lib",
+    ],
+)
+
+filegroup(
+    name = "all_files",
+    srcs = glob(
+        ["**/*"],
+        exclude = [
+            "**/METADATA",
+            "**/OWNERS",
+        ],
+    ),
+    visibility = ["//tensorflow:__subpackages__"],
+)
diff --git a/tensorflow/contrib/eager/python/examples/cart_pole.py b/tensorflow/contrib/eager/python/examples/cart_pole.py
new file mode 100644
index 0000000000..56235e4039
--- /dev/null
+++ b/tensorflow/contrib/eager/python/examples/cart_pole.py
@@ -0,0 +1,282 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+r"""TensorFlow Eager Execution Example: OpenAI Gym CartPole.
+
+Solves the cart-pole problem with policy gradient-based reinforcement learning.
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import argparse
+import sys
+
+import gym
+import numpy as np
+from six.moves import input  # pylint: disable=redefined-builtin
+from six.moves import xrange  # pylint: disable=redefined-builtin
+import tensorflow as tf
+
+from tensorflow.contrib.eager.python import tfe
+from tensorflow.contrib.eager.python.examples import cart_pole_helper
+
+
+class PolicyNetwork(object):
+  """Policy network for the cart-pole reinforcement learning problem.
+
+  The forward path of the network takes an observation from the cart-pole
+  environment (length-4 vector) and outputs an action.
+  """
+
+  def __init__(self, hidden_size, train_logdir=None):
+    """Constructor of PolicyNetwork.
+
+    Args:
+      hidden_size: Size of the hidden layer, as an `int`.
+      train_logdir: The directory in which summaries will be written for
+        TensorBoard during training (optional).
+    """
+    self._hidden_layer = tf.layers.Dense(hidden_size, activation=tf.nn.elu)
+    self._output_layer = tf.layers.Dense(1)
+
+    # Gradient function.
+    self._grad_fn = tfe.implicit_gradients(
+        self._get_cross_entropy_and_save_actions)
+
+    # Support for TensorBoard summaries. Once training has started, use:
+    #   tensorboard --logdir=<train_logdir>
+    self._summary_writer = (tfe.SummaryWriter(train_logdir) if train_logdir
+                            else None)
+
+  def forward(self, inputs):
+    """Given inputs, calculate logits and action.
+
+    Args:
+      inputs: Observations from a step in the cart-pole environment, of shape
+        `(batch_size, input_size)`
+
+    Returns:
+      logits: the logits output by the output layer. This can be viewed as the
+        likelihood vales of choosing the left (0) action. Shape:
+        `(batch_size, 1)`.
+      actions: randomly selected actions ({0, 1}) based on the logits. Shape:
+        `(batch_size, 1)`.
+    """
+    hidden = self._hidden_layer(inputs)
+    logits = self._output_layer(hidden)
+
+    # Probability of selecting the left action.
+    left_p = tf.nn.sigmoid(logits)
+    # Probabilities of selecting the left and right actions.
+    left_right_ps = tf.concat([left_p, 1.0 - left_p], 1)
+    # Randomly-generated actions based on the probabilities.
+    actions = tf.multinomial(tf.log(left_right_ps), 1)
+    return logits, actions
+
+  def _get_cross_entropy_and_save_actions(self, inputs):
+    """Given inputs, get the sigmoid cross entropy and save selection action.
+
+    Args:
+      inputs: Observation from a step in the cart-pole environment.
+
+    Returns:
+      The sigmoid cross-entropy loss given the selected action and logits, based
+        on the assumption that the selected action was rewarded by the
+        environment.
+    """
+    logits, actions = self.forward(inputs)
+
+    # N.B.: This is an important step. We save the value of the `actions` in a
+    # member variable for use with the RL environment. In classic TensorFlow
+    # (non-eager execution), it is less straightfoward to access intermediate
+    # computation results in this manner (c.f., `tf.Session.partial_run()`).
+    self._current_actions = actions
+
+    labels = 1.0 - tf.cast(actions, tf.float32)
+    return tf.nn.sigmoid_cross_entropy_with_logits(labels=labels, logits=logits)
+
+  def train(self,
+            cart_pole_env,
+            optimizer,
+            discount_rate,
+            num_games,
+            max_steps_per_game):
+    """Train the PolicyNetwork by playing `num_games` games in `cart_pole_env`.
+
+    Arguments:
+      cart_pole_env: The cart-pole gym environment object.
+      optimizer: A TensorFlow `Optimizer` object to be used in this training
+        (e.g., `tf.train.AdamOptimizer`).
+      discount_rate: Reward discounting rate.
+      num_games: Number of games to run per parameter update.
+      max_steps_per_game: Maximum number of steps to run in each game.
+
+    Returns:
+      Step counts from all games, as a `list` of `int`.
+    """
+    all_gradient_lists = []
+    all_rewards = []
+    for _ in xrange(num_games):
+      obs = cart_pole_env.reset()
+      game_rewards = []
+      game_gradient_lists = []
+      for _ in xrange(max_steps_per_game):
+        # TODO(cais): Can we save the tf.constant() call?
+        grad_list, var_list = zip(*self._grad_fn(tf.constant([obs])))
+        game_gradient_lists.append(grad_list)
+
+        action = self._current_actions.numpy()[0][0]
+        obs, reward, done, _ = cart_pole_env.step(action)
+        game_rewards.append(reward)
+        if reward != 1.0 or done:
+          break
+
+      all_gradient_lists.append(game_gradient_lists)
+      all_rewards.append(game_rewards)
+
+    normalized_rewards = cart_pole_helper.discount_and_normalize_rewards(
+        all_rewards, discount_rate)
+    all_grads_and_vars = self._scale_and_average_gradients(var_list,
+                                                           all_gradient_lists,
+                                                           normalized_rewards)
+    optimizer.apply_gradients(all_grads_and_vars)
+    step_counts = [len(rewards) for rewards in all_rewards]
+
+    if self._summary_writer:
+      self._summary_writer.scalar("mean_step_count", np.mean(step_counts))
+      self._summary_writer.step()
+
+    return step_counts
+
+  def _scale_and_average_gradients(self,
+                                   variable_list,
+                                   all_gradient_lists,
+                                   normalized_rewards):
+    """Scale gradient tensors with normalized rewards."""
+    num_games = len(all_gradient_lists)
+    grads_and_vars = []
+    for j, var in enumerate(variable_list):
+      scaled_gradients = []
+      for g in xrange(int(num_games)):
+        num_steps = len(all_gradient_lists[g])
+        for s in xrange(num_steps):
+          scaled_gradients.append(
+              all_gradient_lists[g][s][j] * normalized_rewards[g][s])
+      mean_scaled_gradients = sum(scaled_gradients) / len(scaled_gradients)
+      grads_and_vars.append((mean_scaled_gradients, var))
+    return grads_and_vars
+
+  def play(self, cart_pole_env, max_steps=None, render=False):
+    """Play a game in the cart-pole gym environment.
+
+    Args:
+      cart_pole_env: The cart-pole gym environment object.
+      max_steps: Maximum number of steps to run in the game.
+      render: Whether the game state is to be rendered on the screen.
+    """
+    if render:
+      input("\nAbout to play a game with rendering. Press Enter to continue: ")
+
+    steps = 0
+    obs = cart_pole_env.reset()
+    while True:
+      # TODO(cais): Can we save the tf.constant() call?
+      _, actions = self.forward(tf.constant([obs]))
+      if render:
+        cart_pole_env.render()
+      obs, reward, done, _ = cart_pole_env.step(actions.numpy()[0][0])
+      steps += 1
+      if done or reward != 1.0 or max_steps is not None and steps >= max_steps:
+        break
+
+
+def main(_):
+  tf.set_random_seed(0)
+
+  cart_pole_env = gym.make("CartPole-v0")
+  cart_pole_env.seed(0)
+  cart_pole_env.reset()
+
+  device = "gpu:0" if tfe.num_gpus() else "cpu:0"
+  print("Using device: %s" % device)
+
+  with tf.device(device):
+    policy_network = PolicyNetwork(FLAGS.hidden_size, train_logdir=FLAGS.logdir)
+    optimizer = tf.train.AdamOptimizer(FLAGS.learning_rate)
+
+    # Training loop.
+    for i in xrange(FLAGS.num_iterations):
+      step_counts = policy_network.train(
+          cart_pole_env,
+          optimizer,
+          FLAGS.discount_rate,
+          FLAGS.num_games_per_iteration,
+          FLAGS.max_steps_per_game)
+      print("Iteration %d: step counts = %s; mean = %g" % (
+          i, step_counts, np.mean(step_counts)))
+      sys.stdout.flush()
+
+    # Optional playing after training, with rendering.
+    if FLAGS.play_after_training:
+      policy_network.play(cart_pole_env,
+                          max_steps=FLAGS.max_steps_per_game,
+                          render=True)
+
+
+if __name__ == "__main__":
+  parser = argparse.ArgumentParser()
+  parser.add_argument(
+      "--hidden_size",
+      type=int,
+      default=5,
+      help="Size of the hidden layer of the policy network.")
+  parser.add_argument(
+      "--discount_rate",
+      type=float,
+      default=0.95,
+      help="Reward discounting rate.")
+  parser.add_argument(
+      "--learning_rate",
+      type=float,
+      default=0.05,
+      help="Learning rate to be used during training.")
+  parser.add_argument(
+      "--num_iterations",
+      type=int,
+      default=100,
+      help="Number of training iterations.")
+  parser.add_argument(
+      "--num_games_per_iteration",
+      type=int,
+      default=20,
+      help="Number of games to run in each training iteration.")
+  parser.add_argument(
+      "--max_steps_per_game",
+      type=int,
+      default=1000,
+      help="Maximum number of steps to run in each game.")
+  parser.add_argument(
+      "--logdir",
+      type=str,
+      default=None,
+      help="logdir in which TensorBoard summaries will be written (optional).")
+  parser.add_argument(
+      "--play_after_training",
+      action="store_true",
+      help="Play a game after training (with rendering).")
+
+  FLAGS, unparsed = parser.parse_known_args()
+  tfe.run(main=main, argv=[sys.argv[0]] + unparsed)
diff --git a/tensorflow/contrib/eager/python/examples/cart_pole_helper.py b/tensorflow/contrib/eager/python/examples/cart_pole_helper.py
new file mode 100644
index 0000000000..1b80f90165
--- /dev/null
+++ b/tensorflow/contrib/eager/python/examples/cart_pole_helper.py
@@ -0,0 +1,60 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Helper functions for reinforcement learning in the cart-pole problem."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+
+def discount_rewards(rewards, discount_rate):
+  """Discout reward values with discount rate.
+
+  Args:
+    rewards: A sequence of reward values in time.
+    discount_rate: (`float`) reward discounting rate (e.g., 0.95).
+
+  Returns:
+    Discounted reward values.
+  """
+  discounted = []
+  for reward in reversed(rewards):
+    discounted.append(
+        (discounted[-1] if discounted else 0.0) * discount_rate + reward)
+  return list(reversed(discounted))
+
+
+def discount_and_normalize_rewards(reward_sequences, discount_rate):
+  """Perform discounting on a number of reward sequences; then normalize values.
+
+  Args:
+    reward_sequences: an `iterable` of reward sequences.
+    discount_rate: reward discounting rate (e.g., 0.95).
+
+  Returns:
+    A `list` of reward value `list`s, discounted and normalized.
+  """
+  discounted = []
+  for sequence in reward_sequences:
+    discounted.append(discount_rewards(sequence, discount_rate))
+  discounted = np.array(discounted)
+
+  # Compute overall mean and stddev.
+  flattened = np.concatenate(discounted)
+  mean = np.mean(flattened)
+  std = np.std(flattened)
+  return [((d - mean) / std) for d in discounted]
diff --git a/tensorflow/contrib/eager/python/examples/linear_regression.py b/tensorflow/contrib/eager/python/examples/linear_regression.py
new file mode 100644
index 0000000000..538d6d4225
--- /dev/null
+++ b/tensorflow/contrib/eager/python/examples/linear_regression.py
@@ -0,0 +1,197 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+# pylint: disable=line-too-long
+r"""TensorFlow Eager Execution Example: Linear Regression.
+
+This example shows how to use TensorFlow Eager Execution to fit a simple linear
+regression model using some synthesized data. Specifically, it illustrates how
+to define the forward path of the linear model and the loss function, as well
+as how to obtain the gradients of the loss function with respect to the
+variables and update the variables with the gradients.
+"""
+# pylint: enable=line-too-long
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import argparse
+import sys
+
+import numpy as np
+from six.moves import xrange  # pylint: disable=redefined-builtin
+import tensorflow as tf
+
+# TODO(cais): Use tf.contrib.eager namespace when ready.
+from tensorflow.contrib.eager.python import tfe
+
+
+class DataGenerator(object):
+  """Generates synthetic data for linear regression."""
+
+  def __init__(self, w, b, noise_level, batch_size):
+    self._w = w
+    self._b = b
+    self._noise_level = noise_level
+    self._batch_size = batch_size
+    self._ndims = w.shape[0]
+
+  def next_batch(self):
+    """Generate a synthetic batch of xs and ys."""
+    xs = tf.random_normal([self._batch_size, self._ndims])
+    ys = (tf.matmul(xs, self._w) + self._b +
+          self._noise_level * tf.random_normal([self._batch_size, 1]))
+    return xs, ys
+
+
+class LinearModel(object):
+  """A TensorFlow linear regression model.
+
+  Uses TensorFlow's eager execution.
+
+  For those familiar with TensorFlow graphs, notice the absence of
+  `tf.Session`. The `forward()` method here immediately executes and
+  returns output values. The `loss()` method immediately compares the
+  output of `forward()` with the target adn returns the MSE loss value.
+  The `fit()` performs gradient-descent training on the model's weights
+  and bias.
+  """
+
+  def __init__(self):
+    """Constructs a LinearModel object."""
+    self._hidden_layer = tf.layers.Dense(1)
+
+    # loss_value_and_grad_fn is a function that when invoked, will return the
+    # loss value and the gradients of loss with respect to the variables. It has
+    # the same input arguments as `self.loss()`.
+    self._loss_value_and_grad_fn = tfe.implicit_value_and_gradients(self.loss)
+
+  @property
+  def weights(self):
+    """Get values of weights as a numpy array."""
+    return self._hidden_layer.variables[0].read_value().numpy()
+
+  @property
+  def biases(self):
+    """Get values of biases as a numpy array."""
+    return self._hidden_layer.variables[1].read_value().numpy()
+
+  def forward(self, xs):
+    """Invoke the linear model.
+
+    Args:
+      xs: input features, as a tensor of size [batch_size, ndims].
+
+    Returns:
+      ys: the predictions of the linear mode, as a tensor of size [batch_size]
+    """
+    # Note: Unlike classic TensorFlow, operations such as self._hidden_layer
+    # will execute the underlying computation immediately.
+    return self._hidden_layer(xs)
+
+  def loss(self, xs, ys):
+    """Loss of the linear model.
+
+    Args:
+      xs: input features, as a tensor of size [batch_size, ndims].
+      ys: the target values of y, as a tensor of size [batch_size].
+
+    Returns:
+      The mean square error loss value.
+    """
+    return tf.reduce_mean(tf.square(self.forward(xs) - ys))
+
+  def fit(self,
+          batch_fn,
+          optimizer,
+          num_iters,
+          verbose=False,
+          logdir=None):
+    """Fit the linear-regression model.
+
+    Args:
+      batch_fn: A function, which when called without any arguments, returns a
+        batch of xs and ys for training.
+      optimizer: The TensorFlow Optimizer object to be used.
+      num_iters: Number of training iterations to perform.
+      verbose: If true, will print out loss values at every iteration.
+      logdir: The directory in which summaries will be written for TensorBoard
+        (optional).
+    """
+    if logdir:
+      # Support for TensorBoard summaries. Once training has started, use:
+      #   tensorboard --logdir=<logdir>
+      summary_writer = tfe.SummaryWriter(logdir)
+
+    # Training loop.
+    for i in xrange(num_iters):
+      # Generate a (mini-)batch of data for training.
+      xs, ys = batch_fn()
+
+      # Call the function obtained above to get the loss and gradient values at
+      # the specific training batch. The function has the same input arguments
+      # as the forward function, i.e., `linear_loss()`.
+      loss_value, grads_and_vars = self._loss_value_and_grad_fn(xs, ys)
+      if verbose:
+        print("Iteration %d: loss = %s" % (i, loss_value.numpy()))
+
+      # Send the gradients to the optimizer and update the Variables, i.e., `w`
+      # and `b`.
+      optimizer.apply_gradients(grads_and_vars)
+
+      if logdir:
+        summary_writer.scalar("loss", loss_value)
+        summary_writer.step()
+
+
+def main(_):
+  # Ground-truth constants.
+  true_w = np.array([[-2.0], [4.0], [1.0]], dtype=np.float32)
+  true_b = np.array([0.5], dtype=np.float32)
+  noise_level = 0.01
+
+  # Training constants.
+  batch_size = 64
+  learning_rate = 0.1
+  num_iters = 20
+
+  print("True w: %s" % true_w)
+  print("True b: %s\n" % true_b)
+
+  device = "gpu:0" if tfe.num_gpus() else "cpu:0"
+  print("Using device: %s" % device)
+  with tf.device(device):
+    linear_model = LinearModel()
+
+    optimizer = tf.train.GradientDescentOptimizer(learning_rate)
+    data_gen = DataGenerator(true_w, true_b, noise_level, batch_size)
+    linear_model.fit(data_gen.next_batch, optimizer, num_iters, verbose=True,
+                     logdir=FLAGS.logdir)
+
+  print("\nAfter training: w = %s" % linear_model.weights)
+  print("\nAfter training: b = %s" % linear_model.biases)
+
+
+if __name__ == "__main__":
+  parser = argparse.ArgumentParser()
+  parser.add_argument(
+      "--logdir",
+      type=str,
+      default=None,
+      help="logdir in which TensorBoard summaries will be written (optional).")
+  FLAGS, unparsed = parser.parse_known_args()
+
+  # Use tfe.run() instead of tf.app.run() for eager execution.
+  tfe.run(main=main, argv=[sys.argv[0]] + unparsed)
diff --git a/tensorflow/contrib/eager/python/examples/notebooks/1_basics.ipynb b/tensorflow/contrib/eager/python/examples/notebooks/1_basics.ipynb
new file mode 100644
index 0000000000..9c2e6f15b4
--- /dev/null
+++ b/tensorflow/contrib/eager/python/examples/notebooks/1_basics.ipynb
@@ -0,0 +1,529 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "colab_type": "text",
+        "id": "U9i2Dsh-ziXr"
+      },
+      "source": [
+        "# Eager Execution Tutorial: Basics\n",
+        "\n",
+        "This notebook introduces the basics of using TensorFlow's eager execution capabilities. It covers concepts such as:\n",
+        "\n",
+        "* Importing required packages\n",
+        "* Enabling eager execution\n",
+        "* Creating and using TensorFlow Tensors and Variables\n",
+        "* Using TensorFlow interactively\n",
+        "* Using GPUs with eager execution enabled\n",
+        "\n",
+        "This notebook does *not* cover modeling topics, such as gradients."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "colab_type": "text",
+        "id": "z1JcS5iBXMRO"
+      },
+      "source": [
+        "# Step 1: Import Eager\n",
+        "\n",
+        "The key imports for eager execution are the following:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 0,
+      "metadata": {
+        "cellView": "code",
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          }
+        },
+        "colab_type": "code",
+        "id": "RlIWhyeLoYnG"
+      },
+      "outputs": [],
+      "source": [
+        "# Import TensorFlow.\n",
+        "import tensorflow as tf\n",
+        "\n",
+        "# Import TensorFlow eager execution support (subject to future changes).\n",
+        "from tensorflow.contrib.eager.python import tfe"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "colab_type": "text",
+        "id": "H9UySOPLXdaw"
+      },
+      "source": [
+        "# Step 2: Enable eager execution\n",
+        "\n",
+        "All future TensorFlow calls will execute the\n",
+        "underlying TensorFlow ops immediately:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 0,
+      "metadata": {
+        "cellView": "code",
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          }
+        },
+        "colab_type": "code",
+        "id": "WPTUfGq6kJ5w"
+      },
+      "outputs": [],
+      "source": [
+        "tfe.enable_eager_execution()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "colab_type": "text",
+        "id": "twBfWd5xyu_d"
+      },
+      "source": [
+        "# Step 3: Interactively Use TensorFlow!\n",
+        "\n",
+        "Now you can call TensorFlow functions and get results, immediately! No more `tf.Sessions`!\n",
+        "\n",
+        "TensorFlow will automatically wrap native Python types for you with operator overloading for TensorFlow Tensors."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 0,
+      "metadata": {
+        "cellView": "code",
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          }
+        },
+        "colab_type": "code",
+        "id": "ngUe237Wt48W"
+      },
+      "outputs": [],
+      "source": [
+        "print(tf.add(1, 2))\n",
+        "print(tf.add([1, 2], [3, 4]))\n",
+        "print(tf.square(5))\n",
+        "print(tf.reduce_sum([1, 2, 3]))\n",
+        "print(tf.encode_base64(\"hello world\"))\n",
+        "print(\"\")\n",
+        "\n",
+        "x = tf.constant(2)\n",
+        "y = tf.constant(3)\n",
+        "print(x * y + 1)\n",
+        "\n",
+        "# Most TensorFlow ops are directly usable with eager execution, giving\n",
+        "# results immediately.\n",
+        "print(tf.contrib.signal.hamming_window(x * y + 1))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "colab_type": "text",
+        "id": "IDY4WsYRhP81"
+      },
+      "source": [
+        "Numpy arrays are supported, too:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 0,
+      "metadata": {
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          }
+        },
+        "colab_type": "code",
+        "id": "lCUWzso6mbqR"
+      },
+      "outputs": [],
+      "source": [
+        "import numpy as np\n",
+        "\n",
+        "ones = np.ones([3, 3])\n",
+        "\n",
+        "print(\"numpy 3x3 matrix of 1s:\")\n",
+        "print(ones)\n",
+        "print(\"\")\n",
+        "\n",
+        "print(\"Multiplied by 42:\")\n",
+        "print(tf.multiply(ones, 42))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "colab_type": "text",
+        "id": "PBNP8yTRfu_X"
+      },
+      "source": [
+        "# Step 4: Define and Print TensorFlow Variables\n",
+        "\n",
+        "To define TensorFlow variables, use the `get_variable()` function as follows:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 0,
+      "metadata": {
+        "cellView": "code",
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          }
+        },
+        "colab_type": "code",
+        "id": "3Twf_Rw-gQFM"
+      },
+      "outputs": [],
+      "source": [
+        "x = tf.get_variable(name=\"x\", shape=[1], dtype=tf.float32, initializer=tf.zeros_initializer)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "colab_type": "text",
+        "id": "45G7094TxsMb"
+      },
+      "source": [
+        "## Printing TensorFlow Variables"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 0,
+      "metadata": {
+        "cellView": "code",
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          }
+        },
+        "colab_type": "code",
+        "id": "UJBJeZ5XxuwA"
+      },
+      "outputs": [],
+      "source": [
+        "# This does NOT print the Variable's actual value:\n",
+        "print(\"Printing a TensorFlow Variable:\")\n",
+        "print(x)\n",
+        "print(\"\")\n",
+        "\n",
+        "# A TensorFlow variable represents a reference to a tensor.\n",
+        "# The `read_value()` method provides access to the current value of the\n",
+        "# variable. Tensorflow Variables are automatically initialized according to the\n",
+        "# semantics defined in tf.get_variable().\n",
+        "print(\"Printing a TensorFlow Variable's value using .read_value():\")\n",
+        "print(x.read_value())\n",
+        "print(\"\")\n",
+        "\n",
+        "print(\"Printing a TensorFlow Variable's value using .read_value().numpy():\")\n",
+        "print(x.read_value().numpy())"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "colab_type": "text",
+        "id": "2njjWHcTpBEn"
+      },
+      "source": [
+        "## Changing a TensorFlow Variable's value\n",
+        "\n",
+        "To change a TensorFlow Variable's value, use its `.assign()` or `.assign_add()` method:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 0,
+      "metadata": {
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          }
+        },
+        "colab_type": "code",
+        "id": "v3wr6Erbo_hB"
+      },
+      "outputs": [],
+      "source": [
+        "x.assign(42)\n",
+        "print(x.read_value())\n",
+        "\n",
+        "x.assign_add(3)\n",
+        "print(x.read_value())"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "colab_type": "text",
+        "id": "uhtynjHVpTB5"
+      },
+      "source": [
+        "## Use a Variable just like any other Tensor"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 0,
+      "metadata": {
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          }
+        },
+        "colab_type": "code",
+        "id": "7PbktdnHoehR"
+      },
+      "outputs": [],
+      "source": [
+        "print(x + 3)\n",
+        "\n",
+        "# This code will broadcast the value across the list of numbers:\n",
+        "print(x * [1, 2, 4])"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "colab_type": "text",
+        "id": "GVChqwlwy1SI"
+      },
+      "source": [
+        "# Step 5: Debug Errors with Instant Feedback\n",
+        "\n",
+        "TensorFlow's eager execution helps you identify and debug runtime issues through interactive exploration of code snippets.\n",
+        "\n",
+        "Below, we'll define a length-4 vector, and attempt two `tf.slice()` operations,\n",
+        "one being legal and the other being illegal, leading to a runtime error that is\n",
+        "raised immediately."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 0,
+      "metadata": {
+        "cellView": "code",
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          }
+        },
+        "colab_type": "code",
+        "id": "23ap04N0v4k0"
+      },
+      "outputs": [],
+      "source": [
+        "vector = tf.constant([10.0, 20.0, 30.0, 40.0])"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 0,
+      "metadata": {
+        "cellView": "code",
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          }
+        },
+        "colab_type": "code",
+        "id": "FCUMsIYxxRRa"
+      },
+      "outputs": [],
+      "source": [
+        "# Works, because the values of `begin` and `size` (the 2nd and 3rd input\n",
+        "# arguments) are within the bound of `vector`.\n",
+        "print(tf.slice(vector, [1], [3]))"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 0,
+      "metadata": {
+        "cellView": "code",
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          }
+        },
+        "colab_type": "code",
+        "id": "T8me2oCNxpFp"
+      },
+      "outputs": [],
+      "source": [
+        "# The following does NOT work, because the value of `size` (the 3rd\n",
+        "# argument) causes the indices to go out of the bounds of `vector`. The\n",
+        "# error is raised immediately.\n",
+        "try:\n",
+        "  print(tf.slice(vector, [1], [4]))\n",
+        "except tf.OpError as e:\n",
+        "  print(\"Caught error: %s\" % e)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "colab_type": "text",
+        "id": "irxJhAgar84v"
+      },
+      "source": [
+        "# Step 6: Using the GPU\n",
+        "\n",
+        "You can place Tensors on the GPU by calling a Tensor's `.gpu()` method.\n",
+        "\n",
+        "The first operation executing on the GPU may be slow as TensorFlow initializes. Subsequent uses will be much faster."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 0,
+      "metadata": {
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          }
+        },
+        "colab_type": "code",
+        "id": "7J4N9baqaKCL"
+      },
+      "outputs": [],
+      "source": [
+        "# The example code from here on will work only if your notebook\n",
+        "# is running on a machine with a functional CUDA GPU. The following\n",
+        "# line checks that.\n",
+        "is_gpu_available = tfe.num_gpus() \u003e 0\n",
+        "\n",
+        "# Create some Tensors\n",
+        "SIZE = 1000\n",
+        "cpu_tensor = tf.random_normal([SIZE, SIZE])\n",
+        "\n",
+        "if is_gpu_available:\n",
+        "  gpu_tensor = cpu_tensor.gpu()"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 0,
+      "metadata": {
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          }
+        },
+        "colab_type": "code",
+        "id": "4E-2n7VbzY1n"
+      },
+      "outputs": [],
+      "source": [
+        "# Time a CPU-based matrix multiplication\n",
+        "\n",
+        "print(\"Time to conduct matmul on CPU:\")\n",
+        "%time tf.matmul(cpu_tensor, cpu_tensor)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 0,
+      "metadata": {
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          }
+        },
+        "colab_type": "code",
+        "id": "vbSFW-T5zhZF"
+      },
+      "outputs": [],
+      "source": [
+        "# Time GPU-based matrix multiplications.\n",
+        "\n",
+        "if is_gpu_available:\n",
+        "  # First use of the GPU will be slow:\n",
+        "  print(\"Time to conduct first matmul on GPU:\")\n",
+        "  %time tf.matmul(gpu_tensor, gpu_tensor)\n",
+        "  print()\n",
+        "\n",
+        "  # Subsequent uses are much faster:\n",
+        "  print(\"Time to conduct second matmul on GPU:\")\n",
+        "  %time tf.matmul(gpu_tensor, gpu_tensor)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 0,
+      "metadata": {
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          }
+        },
+        "colab_type": "code",
+        "id": "E5pIOe3Rz7iW"
+      },
+      "outputs": [],
+      "source": [
+        "# Second timing demo for GPUs, after it has been used once:\n",
+        "\n",
+        "cpu_tensor = tf.random_normal([SIZE, SIZE])\n",
+        "print(\"Time to conduct CPU matmul:\")\n",
+        "%time tf.matmul(cpu_tensor, cpu_tensor)\n",
+        "print()\n",
+        "\n",
+        "if is_gpu_available:\n",
+        "  gpu_tensor = cpu_tensor.gpu()\n",
+        "  print(\"Time to conduct GPU matmul:\")\n",
+        "  %time tf.matmul(gpu_tensor, gpu_tensor)"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "default_view": {},
+      "name": "Eager Execution Tutorial: Basics",
+      "provenance": [
+        {
+          "file_id": "0B0kLcpwLFwKEVm9XNkFueGk4bTg",
+          "timestamp": 1504118841551
+        }
+      ],
+      "version": "0.3.2",
+      "views": {}
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/tensorflow/contrib/eager/python/examples/notebooks/2_gradients.ipynb b/tensorflow/contrib/eager/python/examples/notebooks/2_gradients.ipynb
new file mode 100644
index 0000000000..5e0ec5cf8a
--- /dev/null
+++ b/tensorflow/contrib/eager/python/examples/notebooks/2_gradients.ipynb
@@ -0,0 +1,864 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "colab_type": "text",
+        "id": "vDJ4XzMqodTy"
+      },
+      "source": [
+        "# Eager Execution: Working with Gradients\n",
+        "\n",
+        "This notebook demonstrates:\n",
+        "\n",
+        "* How to get gradients using TensorFlow's eager execution capabilities\n",
+        "* How to apply the gradients so you can update your variables"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "colab_type": "text",
+        "id": "GQJysDM__Qb0"
+      },
+      "source": [
+        "# Setup: Import eager and enable eager execution.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 0,
+      "metadata": {
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          }
+        },
+        "colab_type": "code",
+        "id": "OiMPZStlibBv"
+      },
+      "outputs": [],
+      "source": [
+        "# Import TensorFlow.\n",
+        "import tensorflow as tf\n",
+        "\n",
+        "# Import TensorFlow eager execution support (subject to future changes).\n",
+        "from tensorflow.contrib.eager.python import tfe\n",
+        "\n",
+        "# Enable eager execution.\n",
+        "tfe.enable_eager_execution()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "colab_type": "text",
+        "id": "1CLWJl0QliB0"
+      },
+      "source": [
+        "# Fitting a Simple Linear Model"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "colab_type": "text",
+        "id": "-39gouo7mtgu"
+      },
+      "source": [
+        "## Step 1: Synthesize some data\n",
+        "\n",
+        "To demonstrate fitting a model with TensorFlow's eager execution, we'll fit a linear model to some synthesized data (which includes some noise).\n",
+        "\n",
+        "In the code, we  use the variable names `w` and `b` to represent the single weight and bias we'll use to fit our model."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 0,
+      "metadata": {
+        "cellView": "code",
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          }
+        },
+        "colab_type": "code",
+        "id": "rQsdCg9PfIL-"
+      },
+      "outputs": [],
+      "source": [
+        "# The constants we'll try to fit our variables to:\n",
+        "true_w = 3\n",
+        "true_b = 2\n",
+        "\n",
+        "NUM_EXAMPLES = 1000\n",
+        "\n",
+        "# Our inputs:\n",
+        "inputs = tf.random_normal(shape=[NUM_EXAMPLES, 1])\n",
+        "\n",
+        "# Our labels, with noise:\n",
+        "noise = tf.random_normal(shape=[NUM_EXAMPLES, 1])\n",
+        "labels = inputs * true_w + true_b + noise"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 3,
+      "metadata": {
+        "cellView": "code",
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          },
+          "height": 360,
+          "output_extras": [
+            {
+              "item_id": 1
+            }
+          ]
+        },
+        "colab_type": "code",
+        "executionInfo": {
+          "elapsed": 127,
+          "status": "ok",
+          "timestamp": 1505502830690,
+          "user": {
+            "displayName": "",
+            "photoUrl": "",
+            "userId": ""
+          },
+          "user_tz": 240
+        },
+        "id": "O4lsC4ckAcar",
+        "outputId": "2f760690-cafb-4777-b970-91d839f99faf"
+      },
+      "outputs": [
+        {
+          "data": {
+            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAesAAAFXCAYAAACC+2avAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzsnXt8VPWd99+TK7kykxtJQIebqZfaqogtrhKNa1ooEKl9\nCrpVn9ZNW6x9VWsbCi7aVUt01NZ9tq21KVZlFey2YkQNohhj3QWK2liCF5RIBCc3yEwmIZnMTOY8\nf/zmzJwzSSBAYibh+369eIU5c87vXLh8zvdu0TRNQxAEQRCEmCVurC9AEARBEISjI2ItCIIgCDGO\niLUgCIIgxDgi1oIgCIIQ44hYC4IgCEKMI2ItCIIgCDHOiIj16tWrufjii1m8eHF4269//Wvmz5/P\n0qVLWbp0Ka+//vpInEoQBEEQTjksI1Fn/eabb5KWlkZFRQWbN28GlFinpaXx7W9/+6QvUhAEQRBO\nZUbEsr7wwgvJzMwcsF36rQiCIAjCyTOqMesnn3ySsrIybr/9drq6ukbzVIIgCIIwYRk1sb722mt5\n5ZVXqK6uJicnh8rKytE6lSAIgiBMaEZNrLOysrBYLAB885vfZPfu3cc8RtzmgiAIgjCQhJFaKFpo\n29vbyc3NBeDll1+mqKjomGtYLBba2yeuuzw3N0Pubxwzke9vIt8byP2Nd06F+zsWIyLWt912Gzt3\n7sTtdnPZZZfxwx/+kJ07d/Lee+8RFxfH1KlTueuuu0biVIIgCIJwyjEiYv3ggw8O2Hb11VePxNKC\nIAiCcMojHcwEQRAEIcYRsRYEQRCEGEfEWhAEQRBiHBFrQRAEQYhxRKwFQRAEIcYRsRYEQRCEGEfE\nWhAEQRBiHBFrQRAEQYhxRKwFQRAEIcYRsRYEQRCEGEfEWhAEQRBiHBFrQRAEQYhxRKwFQRAEIcYR\nsRYEQRCEGEfEWhAEQRBiHBFrQRAEQYhxRKwFQRAEIcYRsRYEQRCEGCdhrC9AEARBOHXo6HCzcmUt\nTU2Z2O2dOBwl2GzWsb6smEfEWhAEQfjMWLmylurq6wAL9fUasJ6qqqVjfVkxj7jBBUEQhM+MpqZM\nwBL6ZAl9Fo6FiLUgCILwmWG3dwJa6JOG3e4Zy8sZN4gbXBAEQfjMcDhKgPWhmLUHh+Pysb6kcYGI\ntSAIgvCZYbNZJUZ9AogbXBAEQRBiHBFrQRAEQYhxRKwFQRAEIcYRsRYEQRCEGEfEWhAEQRBiHBFr\nQRAEQYhxRKwFQRAEIcYRsRYEQRCEGEfEWhAEQRBiHBFrQRAEQYhxRKwFQRAEIcYRsRYEQRCEGEfE\nWhAEQRBiHBFrQRAEQYhxRKwFQRAEIcYRsRYEQRCEGEfEWhAEQRBiHBFrQRAEQYhxRKwFQRAEIcYR\nsRYEQRCEGEfEWhAEQRBiHBFrQRAEQYhxRKwFQRAEIcYRsRYEQRCEGEfEWhAEQRBiHBFrQRAEQYhx\nRKwFQRAEIcYRsRYEQRCEGCdhrC9AEARBODE6OtysXFmL02mjsLADh6MEm806rGOamjKx2zuHdYww\n9oyIWK9evZrXXnuN7OxsNm/eDEBnZye33norn376KdOmTeOhhx4iIyNjJE4nCIIgACtX1lJdfR1g\nATRgPVVVS037RIuzz9dDTc33AQv19YMfI8QeI+IG//rXv866detM237/+98zb948XnrpJb70pS/x\nyCOPjMSpBEEQhBBNTZkooQawhD6b0QW9vv4qqquvZ/v27mMeI8QeIyLWF154IZmZ5j/wbdu2sXSp\neltbunQpr7zyykicShAEQQhht3eiLGoADbvdM2CfaEGH7GMeI8Qeoxaz7ujoICcnB4Dc3FxcLtdo\nnUoQBOGUxOEoAdaHYtYuHI7LAbPru61tD1AM2AAXkyY5sVr/CBxi3rwMHI5FY3cDwrCJuQSz3NyJ\nHdeW+xvfTOT7m8j3BhPz/uLi+klOTgQgOTmBnJwMsrIyuPnm5w2x7DKmTbuPgoJzaG7ew8GDt6PH\nuDMyNpKdncFNNz3Pxx+nM2NGFw8/vJCsrNhLOJuIf37Hw6iJdXZ2NocOHSInJ4f29naysrKGdVx7\ne9doXdKYk5ubIfc3jpnI9zeR7w0m7v2Vlz8XFuVduzT6+lSy2N69KRhd3zk5Z/LCC5dRWtrPwYOR\n7Xv3pnDjjYOvEUtM1D8/neG8iIxYnbWmaabPJSUlPPPMMwBs2rSJK664YqROJQiCIDB0gtlQsezB\ntg8nSU0Ye0bEsr7tttvYuXMnbrebyy67jB/+8Id897vf5Uc/+hF/+ctfKCws5D/+4z9G4lSCIAhC\nCLu9M1R+pdzauijrsWxVruUJx7JXrZrDrl2VuFzTsNkOsnr1EtaufWvQNYTYYkTE+sEHHxx0+2OP\nPTYSywuCIAiDMFSCmc1mHdSVXVn5Nk7nKsBCb6/G2rXrhxR2IbaIuQQzQRAEYXjoojxYTHewTmWD\nubyHEnYhthCxFgRBmIAYu5vpncrsdk1c3uMUEWtBEIQYYai+3SfSz3swK/rpp+cgLu/xiYi1IAhC\njDCYNVxVtXTI7UdjsOQzcXmPX0SsBUEQYoShyqhOpLxKEscmFiLWgiAIMcJQpVjm7S7a2t6ltJSw\nS3ywphojYUXLOM3YQcRaEAQhRhjKGjZub2t7F6dzFU6nconX1T1AaelU7r770mEL6XBF+ETc78Lo\nIGItCIIQIwxlDRu3l5aC0xlxibvdZ/KnPy06rjahwxVh6W4WO4xYu1FBEARh9IluGQpqPvXxCOlw\nRXg4IziFzwaxrAVBEMYRuku8ttaPx5MCLAQ0CgoODXuNoWLjQ51LktTGHhFrQRCEcYTuEr/hhv+i\npiYBeBY4xNtvu3G53ANiz4PFp4crwlLqFTuIWAuCIIxDmpsLgF5gOWChtVWjomJg7Hmo+LSI8PhC\nxFoQBGEcEG0hFxT4qK+fwrFiz5IkNjGQBDNBEITPiI4ON+Xlmygt3UZ5+TO4XO5hf69byPX1V1Fd\nfT0QoLBwN8dKAJMksYmBWNaCIAifEdEu6V27KqmtvS4cZz5aSVW0hdzcXEBt7SIqKgaOyDQiSWIT\nAxFrQRCEz4howXU6P09FRe2Qgmx0WRcUNFNf/xSQAXgoKPAcdUSmznCTxKRbWWwjbnBBEIRBOJbL\n+kQwu6RdwLts3Up4/aO7rBOBa4DFwLWhzyNHtJu9oqJ2RNcXTg6xrAVBEAZhNFptOhwl7NpVidP5\neeBdYCW9vRaqq9X6DkcJfX3r2LEjDjiMz5cWLsdqbs7B7AbPOalriUYS0WIbsawFQRAG4XjFaziW\nuM1mpbb2OsrK3KSkFA5Y32azkpychNv9bdzun1JTsyJs4UZb3QUFLeHzLVv21Elb/pKIFtuIZS0I\ngjAIw+3ypROxxDupr3+RurqXKS6OHxD71WPI5eXPhCxq8/pDvSREJ4r5fAkmy/94eoMb0WPV+/Yl\nUFhYSXZ2ETNn9kgiWowhYi0IgjAIx5tFHRHZGmABbvcWqqvT2LXrCWprrx+QrOVwlODzPcL27V1A\nNj5ffzhuPdhLQnSiWGnpNoZr+R8teczo7geNuXNlslYsImItCIIwCMfbajMisunAFvTOYk7n4kE7\ni9lsVpKSUnG7vwdYqKnRSEpaf9SXBKPotrXtAcoYjuV/PCVhEquOTUSsBUEQBmEoa3So7brI1tW1\n4HafyXAEcDChPNpLgtkKLqawsJK8vLMpKurl7ruHtvyPJsjH6+4XxgYRa0EQhEEYyhodarsusi6X\nm8svfwKnczFDCaAu+Pv3t6CSuoYnlGbRtZGXdzZbt15Bbm4GH3xwgPLyTYO6uo8myNI0ZXwgYi0I\nQswylo06IsLoBmrC9dD79iUQbaV2dLi55ZaXQiVXh5gzJ5kvfnEdzc052O0eVq26wCSkPl8PNTXf\nBzqBDVitXoqLE44plEcT3aO5uo8myDJZa3wgYi0IQswyGrXOwyXSMcwJ3Bauhy4srCTaGl65spYt\nW24Mb9u2bQNlZQG2br0CgPLyTab7sFofCO1rBa5l+vRnqaq6Ilz+NdTLiS66+/bF09HRRGNjEeXl\nz/Doo2VHdXWLII9/RKwFQYhZxjb5Se8Y9rzpGrKzi5g7V1mpBQUt+HwJvPZaErABWIgS4AyamvrD\nK0XfB2SjOphtAdJoa9uDyzXnmC8nuuhef/3TNDSswum0sHu3xo03PoHdjsSeJzAi1oIgxCxjmfwU\n6RjWhdGSnjmzJyygRotZ7fMgUAD00NbWTmkphnGWkTXmzQvyzjsP43SuwpgxPtyXE+Vuj+xXV6ex\nY8cVSOx54iJiLQhCzDKWyU+RF4WFDBVXHmgxfw5YRHLy7TidP8XptFFfr7Fgwe8oKzPex1dYtuwt\nnM7IsVu3gs023HKsQxhfIOCQuLonOCLWgiDELCMtQEdLWIv+bvXqOUReFAI4HFcOSG6LtvyhO/T7\n01Eu7nSgiwMHMnn11SVHPba3N5He3psoLKwkK6uIjo697Ntnp7z8mQGx63nz0qmp2YCawNXF/Pky\nHWuiI2ItCMIpw2Ax4fvuu5yVK2upqwvgdicDl1FfP5nBktmGEvTaWj8eTwrKCteAg4BqdgIaHR2V\nA65F9xps3Qq9vX7Uf8dv0NOTwFlnfUJDw3SczgwaGjz4fM/z+OPfCl8DJGG1eoGDzJuXwaOPXkN/\n/4BTCBMIEWtBEE4ZBosJR7fbhI3ANYPGi4dKALvhhv+ipkYD/gDk4PMlocqyAGpwuQoHWMjmHuEp\nqGQ2C273Il5//U7g1vA1bd/+AKCEuqRkfTjWDarrWVaWdch51sLEQMRaEIRThsES1gbGndOJjhfr\nFvXWrRj27WTz5oMUFf03gcCh0PbbAQuapqGywy3ActMYzGhr3eEooa7uZdzuyDX090+PuqZsQL0s\nqPGa0h70VEPEWhCEU4bBEtYqKl41CbjV+j7FxS5TIlnEot5AJLHrRYLBVSGR1YDHMQusDzWF+OjC\narNZ+fKX+9myJXINOTkHaWszZ4+D7hno5ni6ngkTAxFrQRBOGWw2azhG3dSUSUXFq1GJZB4cjuUD\nEsn27YtHucctwL1YLFPQNLMQQzvmDG0L8Klp2/vvv0lJyRFmzQqYXOIWSwD1IqASxs49N430dHP2\nOOiegSWha0mjsLABh+O6UXteQuwgYi0IQkwQnby1atUcKivfHvFWoyfSFa2jowmIxImTk9fg9Z6F\nWZyt6CIKO1BW9W3AfcDZwBG83ttoaNhCQ8P1pvM2NxcAV4XPd/jws2zYcMWA61Cegc2hZ+LG4bgO\nTYNlyzawd2/KZ96SVfjsELEWBCEmiBbRXbsqw4lUx9NqdLDyrNzcjPD3J9IVbfLkqTidG9FLsU47\nrZDZsz1s3/4A3d2ZBAKTQmumAW8CPwXeAGzAOcBiw2rpA8473OYvg5WyRbcy/SxbsgqfHSLWgiDE\nBNEi6nJN40QSqQaznJ999vrw98cSxsHEvrPzU4yW9ZEjlfzqV9excmUtjY2pHD78AR5PBt3de1Ad\nzGxEOp8ZO6C5gJ1AO3v2OLnhBicPPbT4pJq/yDzqUwMRa0EQYoJoEbXZDtLbG/mcn38oPOSioKAZ\nSAxNtTK7fo8lXqtWzWHnzntoa8sjPv4Q3d3puFzu8PGDiX12dpGp21h2dtGAkq/k5DXARcAelCir\nzmeZmR34fHfg9Z4P7AXuBiz4/VqosckLJCWlnrC7X+ZRnxqIWAuCMGocz4jLaOty9eolrF0b+ezz\n+amuVpOt1DSsaxisucn+/QHgSeBrwOQB4lVZ+TYtLbOAawgGLWzbplFREXEdDyb2M2d2snu3uT94\n9H59fRcBS1Au7/tISSmktBQcjjKWLXuL+vqrgM2mYyCD7ds/xe3+HsNxYw/2PB2OEpKTN4Zi1tIT\nfKIiYi0IwqhxPMlcg8Vjq6rs4d+Xlm4jInQZGEVv61bYtesJnM6bUC5oNYayuHjKAPFSmd1O1DSt\nLmAhdXUBGhubqKx8m/37W4gujVq1ag67dlXick3DZjvA6tVl3HnndswJZkfC1wNTuOyyI1RVqa5j\nEeu3K+qYLlQN9fDc2EM9z6efvkaaokxwRKwFQRg1oq3PfftSB8xr1jSGZX2b3b0ejKKn+mqvRu8+\nBhamTz+DqqqBGdXRmd2wAbd7Epde+if8/n9HdR4zD+6oqKgNJ7v19mosXVpJd7debuUDmoHvh86g\nAclApP+ncQ71oUP30NMzlbi4w8yblw4khLqfHduNLfHpUxcRa0EQRo3oeGpHx14aGswZ3sCg1uJQ\nfbhVL20f8ERo3URgAZFsbDia6EXHn5XYXoXfHwh9tqLizVU0NZ1BRcWrNDamYRTJSBexxSjX9lVA\nDSrT+wPgX2lufi18zqMNJHG53CQlDS+5TOLTpy4i1oIgjBrRceh9++wGoeykrq6Vvr4pKAt1IWAN\nW4tDuXxVL20Vu1ax6eXo4lVY2EBeXvCoojdz5pFQ/LkTeDG09QXgI4zdydzun1Bfr85dWLiWgS5v\njYgrezLqheFFIAd4gYKCwYV0sLjzcEutxnJkqDC2iFgLgjBqRFuU5eXP0NBgFkTzAI3lYWtxKJev\nUbCUIK4LZYV7cDiuO2YmtX58bW0LHs9PDedfh8redtHb68bvj8S0s7KmM3euOmdb27s4nStQrvh7\ngQySklbS359Mf/9cVDvQBcBfBj3/8cTxT0bYhYmFiLUgCJ8ZRqHdv99rGl6RkuKntHR92FocyuUb\n/QJgFLSKilePWfqkH19auo36eqM73A/00tX1KZr2C4wx7Vmz+sOu+VtvPURPzyaOHPkYv//HgA2f\nL5Kdrr94NDfnDCq2xxN3PpFua8LERMRaEITPDKPQKnd2RIxLSzEJ0VAu32gB7Orq5NVXf4guaD7f\nOh5/fNmAc+vH7dsXT0dHE93d8Zhd25OBa9G05zCKqdXqxeG4ElDiWVNzI2ZvwDVEZ6dDGna7e1Cx\ntdu1YcedJaFM0BGxFgRhTDhW/HWopKxoAUxMXItR0LZvjxtwzOHDxjnQG1HZ4CrrOzPTS3d3C8Hg\nTaG9zVOtiosThmy4EkloM2enT5q0i9Wrl/G9731EtNg+/XT04BBJKBOOjYi1IAhjwtEypI9GtGD2\n9+dgtpAPDzjmpptqQhncnahJWJF4dFzcM+Tnazidk0N7LwDuwGqdQXFxAqtWXRAuN2tr2wMUo2q5\nXUyatAuLxU1m5l407S7a2s5HDez4MZdc8is0bSrwGCpbXDVoOZ77loQyQUfEWhCEMWG43c2i9yso\n8Jmszby8Vlpa9PGSrfT2urDbN2GzHWDTpjJmzLDz8cdqAIfK1r4NYzwaDvPHP17OkiVr6OubgcXy\nMZdcks4f/nAlmgaXXfY4LS0/ALYA55KYuJaUlCx6erLwes8EvkZv72Ss1gdQHcwUfv+Foc9DN2g5\nFif6QiNMPESsBUEYE4abPBW934IFv+OKKx6hrs5CMHiY/v4errjiEIcPp/L++014vSo5rLfXRXHx\nL5k9+4vs21cPfA7owezG9jFvXjq//e1H9PWpnt2appGVtR5Ng5KS9bS0fAEl1KpEzO/vxu83J5Op\nuHU2Q3U0G6pBiyAMFxFrQRDGhOEmT0Xv19xcQFvbuwQCqrlKe7vGe+9VUl9/RSimq++7Ba/3Lhoa\nLMDVKFFNxyioU6Y0AqezdStE13qvXFkbcp13o4+1VEQnk6k1580LkpS0nrq6AG53K8aOZhJrFk6W\nURfrkpIS0tPTiYuLIyEhgT//+c+jfUpBEMaI4xncEZ08ZZyqZTx2sCSrDz4wj89U4zTBZjsQmtTV\nCfQxUFQvJTPzfk4/fSYdHXvp7k4JZXfrDVKeBRIpKPDQ1FRApGb6d6huZQNbnVqt71Nc7MLh+Ao2\nmxWXy80ttzzP9u1/ALKZNy+Iw/GVkXvIwinJqIu1xWJh/fr1TJ48+dg7C4IwrhnKtR0t4qtWzaG7\n20Ni4lr6+3PIzm7i7bcTaWubA3RTX78E2ExV1VJWrDiDmprb8fnsQCtvvHGE9HSLaXympn1ISclL\n+P3dJCSsIRBIAaZjdkt3A5NJTw8wa1ZPqO3p86HvazDXSa8LvSQsAZ4DJmOxrCEjYzpz5/aQlGRs\nxLLc9EJis1l5/PFvfRaPWziFGHWx1jSNYDA42qcRBCEGGMq1HS3iu3ZV4nTmoeK8GbS3dwA/wxgH\nbmrKZN++JhYufJ5gMNKk5PDhDcTH/4NJk9agaTPw+/fh9f6UhgYbEXd3MlBCxPX9D1QG94N0d/tp\nbEwNradPwUrH7GrPCZVYbWbfvgQ6OtxkZ5/HzJlHcDiWhsW5o8NNRcXwPAmCcDJ8Jpb1jTfeiMVi\nYdmyZXzzm98c7VMKgjBGDFUXHC3iym3dBhgbjAxsKnL11c8RDH4u6rsM+vtn0t9fTmFhJU7nl1FC\nrH/vB3YDS1HWsga8AawGLHg8GocP672+F6Ji1fuARabr1jOxy8s30dCwCqfTEuopHvEWqNptFdeu\nr19CX99fSE5OEvEWRpxRF+uNGzeSm5tLR0cH3/72t5k5cyYXXnjhaJ9WEIQxYKi64GgRV7HlwtBn\nN7AntIKKERcWNrBq1RIuvvgg8CEDZ0C3As/jdPaihHmx4ftEYAZKhDOIWM+6ld1FZmYuUGmYnnUd\ncB9wNoWFDTgc14Xv6WjeAn1spr7+jh1xuN3SHlQYeUZdrHNzcwHIysriyiuvZPfu3UcV69zcjNG+\npDFF7m98M5Hv70Tv7fBhNzfdVMPHH6czY0YXjz66hKwsszX56KNlrFixMbRPN2vXfotLLnmMlhYN\nFS+OuMCnTbuPd965iRUraggGVwGfALejYtDtKCs6H7gUaADsqIEaBSj394LQmkYSME7n6u6+j6lT\nz8XpXBzeIzW1kEWLjvDwwzeZrr+oqMf0olFU1EtubgZOp41ob4DF8qlpm9Np+8z+zkzkv5sw8e/v\nWIyqWPf29hIMBklLS6Onp4c33niDm2+++ajHtLd3jeYljSm5uRlyf+OYiXx/J3Nv5eXPhePRu3Zp\n9PUNZk3G8+tfLzJtOf/8PGpqNgD6HGkACzk5Z9LfH8/evSmh7XagAvgD8AVU/PkHRIu8Emz98/6o\n78wtSW222WRkfAw8hbK+D5OW9iF7987lO9+pNrmvf/zjL/DGG5W4XNOw2Q5w221ltLd3UVjYgdHi\nLyxs4ItftFJTY9zm+kz+zkzkv5twatzfsRhVsT506BA333wzFouF/v5+Fi9ezCWXXDKapxQEYYQY\nbhnWiQ6baG4uQLXhfAyj6L3zzm7OO28PZ51lrImeDEwFFpGfX09Ly2Sik8JU0xPlyk5IsBIIGL/L\nMp1j5swedu7sBH4Y3tbevoH29qvCCXB5eWdjt3fi8/nD7u7eXo21a9dTVWUfxOWvXOdJSdIeVBh5\nRlWsTzvtNKqrq0fzFIIgjBLD7TA2VFLZYOValZVvG9qGHgkd14MxvqxpBTidNxIM3kNZ2XoaG1Np\nb3+Pnh6NuLg/cs45mZx//jq2b+/A7Y4khcFeVCMSK3APA/uFb8Bq9VJcnIDDcTnnnVdLdOKa/nun\n8/M4nUuor9ewWv/IYC8jQ7UClRi1MBpIBzNBEAZluBbzUEllt9zyElu2qGzv+nqNF164g0DgrvDn\nBQvWsWDBOmpqfIbzgJpkZaGz024Yp9kTfnHYts1Fbu6DdHUB3IPFkk1c3Kf09/8EJdQagUAPkYSy\nbiwWK0uWBHA4rgx7B1SSmwvVSjQNleR2KcqKj7QKhUMYhV+6kQljwcBZcoIgCCiLWYkUgMb+/R9S\nXv4MLpfbtJ/NZuW++y7Hbvewb18ql1/+BCUlz7FtWwuqMxiAhUDAjlH8X3stgaSkROLjm4GvEmnr\n+S7gQtP2hs9lfnF4hvb2NPr7LwJmoWnX0N9fBNRgtT5KYWElKhltOSpLfDmTJ3cDsGzZW+F72LSp\njEmTfhnabwnwMzIz/5NJk+5AJao9BbiYNy+DsrL1nHfes5SVrT8h13ZHh5vy8k2Ulm4b9BkKwrEQ\ny1oQhEFxOEro61vHK68ECAS6cLu7qK6eyvbtf+Svf/22KX5tdJmDhtO5EZXBvQG4FiX6jRgt1N7e\nZKqrl2Ox3INxUIYS2Pvwem+jokJ1MTO72l1EN1BRMenFTJv2JJ98YkH913Ynykp2091dSHV1PHBZ\nKCb9MHl5ZzNp0gy83sgLRFzcJLzefwuvXVhYyUMPXXfStdLDDSkIwlCIWAuCMCg2m5Xk5CQCAWPj\nko20ta3hllseISkpNRx/bmxMY2AfbjXVCjaj6qKTgcdR86SnAN9ATbmaiu76jhxfAPyOLVuslJc/\nw+rVc9Bd7Q0NGVHJY16U21qjo6MJj2cFSvwvBHYBd4X214UdnE7V5ASexBzbzjZdR17e2SPS1ORE\nk/AEQUfEWhCEIYkWGV2Et2/vwu3+HrqlOGXKHShhzkANuuhFiV8Lqq92I5oWaRmqLG5QrmY/8L+Y\nG5skAT+jr+9RqqsTqav7G8XF8Tz99Bxuuul5tm0zCmwLaWk9/PM/r6exsQin02ilw8Dr7yISz+4h\nM/NeZs48C7vdg8/Xbyq9Gqn49FBJeIIwXESsBeEURs/YdjptFBZ2DCjPihYZFVceaIEePpyAcRBG\nQsJdBAIbUNnZk5k82YXbbRRNN/BrlKtcubaTk9fg988kGExBNTbRXd7fwe22UF2t3MdJSQA/B+ag\nLOrvEx9fFWoN+gy7dycSEWM9acyGPiHL6+3E6707fK3p6ZVs3apmTbtc7lEpvRoqCU8QhouItSCc\nwkTHmqNjqQ5HCUeOPMJrr2kEAk7i4rK45JKH2Lv3CGoaVTdwMYFAPkbxPuusc5g5s4emptcGtVhV\n1na84RgbfX3TgY+BuahxlQuAHAa6jzNRLvUl4evs6ckIX++WLY/Q16eL8SLgDmy2WcyfH4fDsZxv\nfGMnu3dH1szOLgqvM1Q51skyWusKpw4i1oJwCjBUg5NjxVJtNitPPfUvpm3l5ZtoabkFXXgtltvR\ntHOIxH5d7NnzNh99VITNtodHHilD0+CVV+7E75+NillfS2Lievx+o4A3Ar8wrZuRkYHHE+0+1qiv\nbzadr7+8jtPXAAAgAElEQVT/AKWl27DbO5k58wzee89oxV/A7NkJVFVdBsDMmUdCAzkiDVIEIdYR\nsRaEcc5wOo0NlY18tFjqcAVe07JQFnEVqnd3F8FgJb29quPX0qWVzJ07Db//34kI8wbmz8/kf/7n\nDrzeuSh39udN606ePJudO6/kllseYfv2LiAbn6+fn/98Hps3dxAMRlzdmvYL6uvVveXn/wJz0th7\nfPRRIeXlz+BwlIhLWhiXiFgLwjgnWojr6h6guDjPJNpDWdC6cKmYtcskXMMVeNU0pNLw+bemc7lc\nBQPOHxfnYc8eNxbLLJQrfSHK9R1Zd9IkJwBJSanhZLaaGo2kpPV85Svp1NQsN5wzsnZPTz6RjmgN\nwApcLls45l1VtXTYLunhtlwVhNFGxFoQxjnRQuh2n0l19SKM8eehLGg9ljrYoITIum6ghq1bCZdR\n9fWtY8eOOI4c2Y/ff6bp/Kq1Z+RcmtaI3T47dP5O4EWCwSAtLbOAr6FqoTcCC4iLu51g8MvAEVpa\nfkBFxeZBXzSefnoO77xTidM5DeVWj2SS9/a2ohLXQL0IbEHPAt+3L/64BFjqo4VYQcRaEMY5g2ds\nm+PPJ+L6LShopr7+KVRJViK9vUuorp7MSy+t4Z/+yYbb/R3gEeAAZrezF2OrT6/XRm1tVyi2nQXc\nZth3I3ANKSl9XHbZ0/zP/2Tg8fhQXczcvPiik/nzC03rt7Q0cMstzbhc01D/hX0/tE4asAO/f7ph\n//0YG6h89NEdfPnLTtzunzAcAZb6aCFWELEWhHGOLsR1dQHc7kkol7Kyns1WpMbTT885DjduIsZy\nLF1Yvd6LeP31N1EW60qUtfwEaiBHO6qLsdFFvQGP59rQPtEzoPuA5wgEGnj11UmGLO6rgY34/d+n\noeEOpky5k9bWTCCHlpZp1NT0oCzq7xPp7f0ukAt8E3gUVfaVazqf13s+Xm8iwxVgqY8WYgURa0EY\n5+iubJfLTUVFbbhcyuG4nIqKod24RiEvKurh7rsvNQl5c7O5bEpZyhpwhP5+O5GuY1ZUE5PridRG\n3wecg5o9nQc0AR+iyq6Mk7KSgCX4/Xpf8IENWDyeWSQnt2O2yO8AZgPVqBeEbuAW8vP/MzQ+MxX4\nDip2bbT6+1CW//AEWJLRhFhBxFoQJgjRtbwdHW7q6gIMZUVGx2O3blWJafooy/37WzAL3QcoUfwq\nmnY/0EYkVmxsF2pDCfWi0P7LUeJ9F8oK3xD6eRi4OXSMGo9pPp9qwKJp+4AZmIXc+HKgERdXyeLF\nm1m9+uusXbuerVuht9eC8jJsJDXVj9V6EKdzRegY87jM4T5TQRgrRKwFYYKycmUtbncyRgFsa3sX\nl2vOoCVYbvdsqqt7eeGFZwkEfoBqevI4CQkHiY930dcHyjLdiKaBGtDxBMpSNSd5KYu6m0gnsjwi\nVvi1obUnh36BalGqhFUJ//+GjrkTTZvOpEmfmu7DYslG0yLXnpmZHxbVqio75eXPhLK/rcByFi3a\nyN13XxdOWLPbzeMyBSHWEbEWhAmKEuPLiCR7fYDTuWKISVYaKuY7hUAgDlV+dROwhUDgCwQC/wvM\nAv7VsP8TKAs3A9iHxXIP8fF5BAKTQts04K+AhylTPqa11Xiut4F+LJZ7yMgoJDHxQw4f/hg4HdUi\nVG9peit9fRZaWlwUFlaSl3c2druH7m6LqT/4vHlB071Hu68ffngJ/f3xYiUL4xYRa0GYYOix6P37\nA8ALRMqjGgAL+/bFU16+iX37EigsrKS7Ow+PJxXoR8V6VwHPM3Bs5YOYXdFeIq7opSQn34HFkkIg\ncD1qulYkOe3ccx8hGFxDe7sVSEFlmF+IpnnxeBaQmPhbIuVWoCzvFoyu9by8s009vCsqjLHkr5ie\nQbT7OitrYGmaIIwnRKwFYYKgi3RdXWu4NElZqA8CU1GZ0y/S0dFEQ8Oq8PcLFqwjI8PCn/6Ui7KI\nLaj4cXTCVzbmmHIbcC9wJtCL16u3En0KJeSRY1991UJcXAoqSWwjymqPZJn7/YVRax9BxbUHTwST\nWLJwqiFiLQgThEjC2POYRfZzKMsYrFYv2dlFoVnO6vvm5hxefPEqtmypxOPJRAnkQuBhzHHoD4B7\nUOVQn6Cs9dNQ/43obvR7UWIcB/wB1VAlm2CwjWBwNsYs78j1pQE+EhPvxO+/ECXUXwX+jB7DLixs\nwOG4bljPYbCmJ7m5GcN/kIIQg4hYC8I4xihMjY3NKGs0Oqv6g9C2i0lNbaGpqdXwvYuWlgYuuiie\n1FQfHs8hIsM0koG1wNmo+dR+4N8M696DuQ57PxExVo1U4EbD9/eGfkZf35vArcyf/0fee68Bl6uA\nYPBBEhPzSEhwMW9eBg89dJ0pGexoXcgG6zr27LPXj+RjF4TPHBFrQRjHDBxxuQFlFW/AYulE0yaj\nksImAz/B6ZyDsnZ19/X7tLTcTkuLGiepYtjxeDyRrl9qNvVUVDmWbhF3hn4+jxLfhahY9FMoV/jn\nQvsaLegzQ9fXgYpPzwQ+4owzTufsszfj82XidN4aPm9f30ZgOe+8U3nU+46uH5euY8JEJG6sL0AQ\nhBMnWpiURfssYCEpCVSZlJVIzPkaVLz4Z6i4snnSVV7e2cTFTQltawLuIxA4DTW+8gPUCwGooRv/\nhnKTXwO8SE5OV+j35aiMbo9hfw34O6rL2b+EzvuvQCVnn51OVdXSIZqwdOJ0JvGlL71MefkzuFzu\nQe/bKMh2ux7rVueVrmPCREAsa0EYx+Tnt2N2KX+KsliXk529FqfT+F02A8XQQ3QS1/79h4hY6SsN\nx98R+mVHZY4b1+rC69Vbieq11A+jeocfRjVKuZWMjN/j9f4Wv/8H4WN1oR28x/mLwG243Zbw1Kyf\n/ewC3n//TZStoWq5jYIsXceEiYiItSCMA4aK0VosAZRL+xxUYtZNJCb+ioUL13PTTZdTVnYHXu8Z\nKBHXk8f0lqC7gHxgLSkpUykuDuDz+QkGe1FCrTcyIfTzDOA6VH11E+aXhAz6+oyNS14GvoDKLs9A\nxbxtBAIF5OYewOmcjHLHv0hjo5fzzvt/ZGbmUlhYSUdHPl7vx8BZKE+B2YK++urn8HrvDp970qQ7\ncDi+G35WkikuTERErAVhHDBUjLa5uQBlyR5BWco1zJ49i6qqpZSXb8LrvQtdnJOSKklIuJ2enjhg\nGiqurGqws7PvIzm5kOrqG9HHWMI+zILsDH13AGVdr0G9JAAsJCnpMIHAGjStCJVsdhvKotbLxzR6\nexPp7b2JwsJKenoScbt/gsdjwePRcDo3AuUUFq7F6bwRlQneHzpeXdP+/V48Hv2zcu9bLGdIJzJh\nwiNiLQjjgH374ol0IusKfdZdx06MYyA7OytDx6QSsUq34PPdh893H2bX9qNAKocPF1BX10JEBK8F\nqlCZ4fkogc5CDc643XD8htC+GoFAK5p2t+E7NaVLfc5An1kNVvLyzgagvn7g4I7U1Azi4n5PMHgm\nyoJ/mMREF37/atzugee12Q6OwBMWhNhGxFoQxgEdHU2ozmJKrDo6lCA7HCVs2/Ys3d0RIXc6M7nh\nho20tzehRk0aB20UYnZtu4Dv0NtrobdXL69agcoeTw/9Mo67/H3U8T5SUp6gtBS2bp0V9V1a6Pca\neXlO2toy0NuPFhR4SEpKHSRGrXHwYDvB4C8M2+8jIeE0/P7I2gkJXSQmPoHNdpBNm5aMwBMWhNhG\nxFoQxgHRjUy6u/MpLd2G3d5JWlob3d03YxS3mpofkJFRScQa34PK3Nbjyrqr20qk3MsKTCUh4X7i\n4vz4fBehksOMAtwedbwPTfuE1auXs2tXdUjw9VjyLs48Mxjq5Z3Ntm3Gmux14USwxsZUDh/eS1aW\nnVmz1g8i+oXYbAdMa3/taykSlxZOKUSsBWEcMHPmEXbvjoiVxzOJ+vqrqK/XyMy8n4Edyyx0dWWh\nOoFtQcWoVwE5KDd2GrAas8t6OZBIIHAPSsD/GZXR/RyRCVr5qASzT9AbpHi9LoqLf8mMGbPp6FiD\nxTILm62ZTZuWMWOGHYDS0m2ma2xuzglN7oL4+ATmzp2KwzEfm83Keef9P5Mwx8W9z2OPLeI3v5EM\nb+HURcRaEGIUYwZ4QcERFixYR3NzDvv3f4jbXR7ay0JcXA4DO5Y9h7Ki70fVNH+CyuZuAaaj/ukb\nBb6XSExZjzHXYIyFq45lFtRkrFzD8Vvweu/ivffUfmVl66mq+qHpXqLLsvLzD1FSsh6n8/NAN/X1\nSwA1DWzTpjKKi+/A650LHCEY/Cm/+c1msaSFUxoRa0GIUaIzwBcseCRUB52NcZqWGg+5jr/+tZfu\n7k+Bi1CW8OmoxiMbMVvRG1ATuIwCvxeoNHzuIjLUg9DPPOC7od8/aTg+zbSfPtXLWGbmcJTQ17eO\nHTvigMP8/e+dtLYas8U3huutZ8ywc+aZc0ICrpAuZMKpjoi1IMQI0bXU+/aZrd/t27twu7+HLqiZ\nmfeTnh7gwAE7s2YFuPRSqKkxCq4+0jJ6cEYGkEpy8hr6+opQJVnXAveRklLIxRd3smePO9yCNLJe\nu2Gdr6EsbTvwIcaBH8apXvX1Gjt33oPXO4nu7kwCgRRUh7PJRJLZrEAaBQVt4WcRbYlLFzLhVEfE\nWhDGEKNAt7Xtwem8CbBRX69RWFiJ2fo1dyCLi8vB6VyK07mFhgYbCQnNmEVZH2kZPTiji4yMOI4c\nyUEN2zgHZWmfTmlpAJhMS8vNqCSyDajGJMmobmcuVAw8DeU670MN67gPOJ1Jk97D5TIniLW0nAbc\nYDi/XtJ1DsrVvhyVABeplT6RLmRHG+4hCOMdEWtBGEPMgzjK0OueIZ3u7ngWLPgdzc0F2O0efL5+\namoiohsMtqISwFTcNxBIxizKHwLrAB8JCXeQmmqnt7cVvz+Prq6bQsdGyrL0TmDLlr2FuW3oY6hE\ntXZUDFwvq1qMst5/HfrcH2rCsiHqOj7GPPAjncjMaj9KvFfQ3Pxa+LmcSBeyW255iS1b1JSv+noN\nn28djz++7LjWEIRYRcRaEMaQgYM4VN0zWPB4FvHOO5U888ylVFa+zYEDqRQWVpKdXcTMmT3s2NGD\nx6N3KFPlUCrTuwg4hEokawb6uPLKqTz++DJKS7dRX38VqtXnFNO5LZaZVFS8SkGBL6r+OQnV4/t7\nqKYo0Znnt6EEWo9xL0QJsB/1wvBjIrHpDSi3ezfqBaAGZWWfvKtbxcONYQOZUyRMHESsBWEM0F22\n+/cHUMlaKlksISGDQCAiOE7n5/n615/D6VyFXtvc3d3K4cOddHbOwFwjnQccxOxyvg/4N/7+919Q\nWrqNtrY9QDHKlW22xHt7J1Fd/VVycx8gLq6SYPBclKguBJ4hMfGXTJ6sceiQUcg7iMTBdXe7FWWx\nbwQuQAm1up+MjB4uuSSN5uYUCgr+Avhpbn52hMqx9AEk+rUdPsn1BCF2ELEWhDEgeg611foAxcVT\n8PniTK5u2EN7+ySU8H0K3IbHsxGP5ybMMWA97mu2llUi1ye0tEyipUUD4oiLewzoIRj8FuamKWnA\nb2lvn40S/UuIWMQp+P134fGsJGJFd6GsZz0uvjD0nQc129ofWidyPxkZbTz00HWjEkueNy+dmprI\ntc2blz7i5xCEsULEWhDGgGj39/TpZ1BVdQUul5va2kiNMXyf/v77gVtRcd9OlGgbY8CHgZ8Dp6Hi\nwy4iItsM/BfG0q1g8FGUqNeiXNyXoBLMsgFzJzRlraeg11/7fJ9HxbHdKBd2H6rZyixUL3Er8+f3\n8NFHHQZvQCRJzelcQUXF6NRMP/TQYpKSamlq6sduD+BwLBrxcwjCWCFiLQhjwGClSR0dbm699QW8\n3lTgXVQf799hsVhD+3Whz3c210w7iSR96f29P48S4FuBNxgqLq72vxPlEg9gdqufTWLiLvx+Y1xc\nb1eqZ3FvBCJWfn7+PVRV/V+WLXsr1B5VT1J7At3CHq2aaRmNKUxkRKwF4SQYrFxI0zhmCdGqVXPY\ntasSl2saNtsBVq8uY+XKWmpqMlGCGBHI/v5VKKH7J1Ss2Si8XSir1rgtK7R/AcrCPow5lpsVtX8i\ng7ce3cP8+Vbee68y1GnsCPA14uJuJxiczWA13IcO5bFs2VuG2Lhu4SeG1tyA3R44mUcuCKckItaC\ncBJEdxnbseNOLJZEWlq+SHQbTSOVlW+H3MRq2tXatetDFmc8oAshqCztIpYsWU9dXStudwCz8LpQ\n7m/jtkmoGHRC6LNuMetx5vej9j8Ns3gfAdYQF5cNTGLTpq+wdu3bNDVl8v77/43X+wsi5VnmGu5A\noIv6+u8BZRQWqpeR3t5EdDe61erF4bhyJB69IJxSiFgLwkkQHXtubc3E7KbeyL598dxww5Ns394F\nZDNvXj8HD9pMx+lWeH19AsqtHRHA5OSPqaqqoKTkJdzuuURiyZ+gBnTEoVzZXyAu7m0SE0/Dau0h\nP9/PO++sQpVyAVyKcksfQrnN1QuFwije7cDdBIMWtm1TLxL6y4YqrzKWZx1Cud3PRDVJ0T0IFvLy\nzmbu3E6qqyO13MXFCdKoRBBOABFrQTgJomPPaqqV0UpN49Chf9DQMBNVp2yhpkajsHAtRoFsa3uX\nRx5ZwvPPr6e/H2ANMAP4kOeeUz2yOzo+QM2n/hmq3KsIVaOs1igsrKS2dkVYDE8/3YHRnR5xbx9C\nJY3prURdpKTcyec+d0FoSMh0ol8kdDIzP6S39ymUlR4EWiksTCMvz0Jb236czhWhPbVQOdbxdyIT\nBGEgItaCcBI4HCXs2mWM6YJRhAsLG+juzid6KEZW1nSCwXtCrTgP4XTm8POf/5WMjM/hdn8ntJ+b\nxMTf8KMfHaCx8QX6+rJRTU+CqCEdlqg1i/jRj14KNQc5hNc7jYHu7TtQGdr5JCSsITGxCJvtIK+/\nfiOZmVmUl3dSXR003YOxWcm5506ltdU8lzovL4etW6/A5ZpDRcVmkzBL0pcgjAwi1oJwEthsVmpr\nr6OiQh9l6QbWceCAlY6OvWRl2Wlvfx9lyUYEsKOjiba2eIwNTF555U5SUuyoEqgkQMPvn857730F\n+CbKMs4Grg8d86RpzY8+eoeGBqMlvQqze/sQytJ+ArievLxK6uuVkObmZtDe3oXDUUJX1yb++te1\n9PfnkJfXyurVXw/fb0tLtOcgKyzmgwmz9OsWhJFBxFo45TlZQRlMpMrLN9HQsCpUvuQCfoXqo51D\ncvJenM6fAq9hFD6//zz8/q8DT2F0b0cGX6SjMrvNk6+s1qmkprbgdJ6FWUhPJ9L0pBs1IUvPFrdw\n6JCV8877T7KzizjrLB93330pNpuVjAwrfv8PUUM4VMz6vvsms3JlLR988AnGF4BJk/6Ow/HdIZ9N\ndAIerBdLWxBOABFr4ZTnZAVlMLGPTjxLTEwmISEPm+0AaWmz+fBDGwOzsveimo34MIuuPviiG5X8\npR8zGYsFtmy5iNLSF4nUQOvrdaJGUBpFXwM+ADz4fB04nbfjdFrYvVtj69YHKC7OGzCas6kp0/CM\nzE1OZs8+86gvNtHPQeZSC8KJIWItnPKcrKAMJvYFBUeor9cTsRrw+1fj96syrbi421GiOR01ZcuF\nSkzTUIMyEjGKrsXyDzTtDSAXaMNYhlVSMpnKyrfxeH6KEtInAC8WSxslJalYLI/wt78l0Nv7CX7/\n5NCx/4pqQ/p703273WdSXb1owGhOu91jeEZ6k5PNwCJmzVp/1Gcjc6kFYWQQsRZOeU5WUAYT+4IC\nH2ZXduT7YLAIZeUeRDUNKUSJbyJKjL9NxH39Ppr2A5S4bgD+D/AUcXFTyM9vZu3aMr73vY9QQl2D\ncnHXk5CQQnp6DqtWzaGy8m2ami6goaGVQOBaw5V7MFvi3YCFtjYbmZn3Ehc3hXnzgjgcX6Gi4lXT\nM7Ja36e42HXM7G7JBheEkUHEWjjlOVlBGUzsm5qMiVjdmEVxHzAXZSk3oTK0dSt6DZo2GX1spGpu\nYkW5x53AfwM/Ixi04HSqeLLdrlFf/yKq8cgW4Iv4/X+jujqVHTueprVVTzp7LOo6JqFqtnWL/VpU\nYxM3Hk8u8G2SktZjs1kHeUbLhxXXl2xwQRgZRKyFU56TFZTBxN5siS5g0iR9OEcD5vnOD2O0ujVt\nKuaksNND3+k9wfVhHjVAOi+++AlPPTWX6upGlFDrDUgWAxtob3cb1r8KPclNZZtnYh7c8SAwFfg+\najZ2JCQgoisIY8uoi/Xrr7/O2rVr0TSNq6++mu9+d+jMUUGINYzJY0VFPeGM6ejv7HaNp5+eg6ZB\nRUUtH3zQR1LSSgKByWhaFgkJCeTkvMGhQzMwzneO7tsdH3+Q/v7lKOFNA/4GrEe5rI3DPJSL3e9f\nxHXX3YHqIKYnkaWH9rMQF5dNMOgCnkPVWbtRZWT7UF3QjIlsn0OJPOgxdIkxC0JsMKpiHQwGufvu\nu3nsscfIy8vjG9/4BldccQWzZs0azdMKwogRnTzW1xfJFDd/52LXrofp6cnH7U5GtQA9D11Uu7s1\nurvvZaBLvBdju87MzG5crl+i3OTdKGv6d8TFHSYYfCp0nC7cABb6+magyrgexNyx7A4CgWyUq/si\nlBv9bsP390ZdS1doTY3MzGYuv3y9xJgFIUYYVbH+xz/+gd1uZ+rUqQB87WtfY9u2bSLWwrjhaJni\n5u+2hAdzRFzKUzBbrlNR85/10qckoAKYTGbm/Vx+eT61tfmodqLGcqtzCAZ3EElYM2drJyU10tc3\nGbgg6nznh873o9Dn56K+n0JcXCWZmflcfHEQTfPT3PxsyJX/LWleIggxxKiKdWtrKwUFBeHPU6ZM\nYffu3aN5SkE4LvQZ0sYhGw899NWwUB0tU9z8XRpmIcxhYLb1p6h48JbQfsbM7CyqqpYye/bTUesk\nomZbzyYya/paVO/wmUAj55+vMWXKeurqWnC7jefrwzzCMtqqn0QwuBq3WyM9fSO//vWiE3+QgiCM\nKqMq1pqmjebygnDSRGZIR4ZsJCVFXN3G5LGiol7uvjviFjZ+19a2B6fzUpT1qgGfEB9/iJSU/Xi9\nOaSmdjB3bhJJSX/hwAErDQ31GIWzp6cJgNTUZjweo6C+jZqQFT2M42z07O3333+A555bSmNjE5dd\npiey7UG9GNQYzrMAWE1CwukEgx0Egz8I3YmFl1/uw+VyizUtCDHKqIp1fn4+Tqcz/Lm1tZW8vLyj\nHpObmzGalzTmyP3FFk6nMdlL/Xz5Zbj55s08/PBCiopO49lnrx/02I8//pitWz/C651OUpKb5OT7\n6euLCGt//4P097t5//2vMmuWPXzcsmUbaGiwY8z61rQscnMzyM+fTUuLMRu8ELOl3YNyg98U3max\n5JKbm8HNN+8OCfUSYD5KqA9jjImXlc3i2Wf/lWXLnuJPf5ocWkPD5UpizZo3ePrpa07mccY04+3v\n5vEi9zexGVWxPvfcc/nkk0/49NNPyc3N5YUXXuCXv/zlUY9pb+866vfjGX1YwkTls7y/kRoQUVjY\ngbI8jVauxp/+dA11dXdywQWn09ycg93eyaOPltHfHx8+trj4v/F6VUJXX9/AMiz4HL29izjnnDWc\nddaF4evcuzcF1RAl0go0Le1+PvjgAG1tjcBqIpb0PZhd1wdRLvGI0H75ywHa27tC6+qubiuwHKv1\nAdzuVeFrfuGFRzj33Cc57bROMjPvx+M5K3TMQvbufW3C/v2Uf3vjm1Ph/o7FqIp1fHw8a9as4Tvf\n+Q6apvGNb3xDksuEESE6S9vne4SkpNTjFm+Ho4QdO35Pa2ukhSf4AQutrZnU1NwYPseKFea4rsrC\nNoqzLvzmjmB9fRdRX78k3IpUNTHRO5KloHqEZ1BS8gRO57+gLO40EhPfRNM6CQSM1xYAesOJYfPm\nBXnooa8Aegx9Sfj4KVPexGJJQLnmu4EFBAIZNDRYaGhYQWHhWjyeReHrLShoobx8k0zIEoQYZNTr\nrOfPn8/8+fNH+zTCKUZ0lvb27V243SrufDzDOGw2K7m5X6S19RuGrZtRYmseB/nxx+mmYxMT38Pn\n08up9gNWLJZVaNoUVCb4wtA6R8JrNDVl8vTTc/D5nmf79k85csSH378aj8cSilXrE7bgnHOCfPCB\nJ6pF6BPAdSxePPD+VAxdnyftxuc7Pfyyoa7j58A09KSzrKzpzJ0bicd3dSXIhCxBiFGkg5kwLonO\n0lZzno8+jGMo13lHxweYLeJ/oCxRv2n71KkdpvXmzTuNurprUAKryq1UUuUToWP+GlrrJlQzkhfZ\nv99LRcWr3HnnpVRWvs3WreD361neVlRWOeiZ521tB+jtjVxDYuJHLFwYqX8+WjigtHQbZsv/QmAR\nqu5aY9as/rAY5+ZmcP75zx7zGQqCMDaIWAvjkugWnz5fPzU1Rx/GMdQozKys6TidxqQuG5BGUtLf\n8fkiLmhN8wMRgfzb3zKJuLKNomhDJXlp5OfXc/75f2H7dhdu909wuy1UV2vs2lUZVZetZ3nvITPz\nAOnpnTQ2FnHWWekEg/fQ2WnHZjvIpk3fZMaMSLLa0cZ75ucbx2lG3PLJyZlkZ1fS2FhEefkzOBwl\n5OZmyIQsQYhhRKyFcUl0r2qXy01SkhLv/PxD+Hx+Skqeo6OjiezsImbOPGKY0+wGati6FcrLn+G0\n047Q0BA993k+gUAzxlpop3MzYBbIwTuBvQtYsFrfp67u/2KzWSkt3UZ9fUTQW1ryMQu8L3TeFfT2\n/gaPZzVOp1qvrGxod/TRmrZYLAHMDViUWz47243TuSo8xxrW8+yz18uELEGIYUSshQmBUbzLyzdR\nXX0jSvwiohSZ01wDLKe3V1m5Cxaso6xsPXV1AdzuScDngQcJBmcCT6JaeU5mxoxuYKBAKsv7DlQ8\nuAOYDnhITvawbNlb2O2dFBT4TFZrMNiIWeCdwCpAw++fynDd0UezhpubC1DDO9TLSUrKc5SWQmNj\nUagP95kAAB2VSURBVOhFwLy+DOsQhNhFxFqYcETE1Ni9y0J2dhFz565n61bo7Y1s37LFD8SRlLSP\nSy9NYceO9/H7jT221wCn8cYbbXz8cdMQ8fJ/Ae4HvoxyN/8Tra1NtLbGU1+vYbM1oAR9BtCIGk/5\nKOACckhI8HHmmU9y8KATtzsXo5C3tb2Ly6WGhETHp49mDUeuU5VxlZYqC728/JmQRS3ubkEYL4hY\nCxOOiEh1YU4QcwNJJCe3mJK21Pzoa+nr0/jb39bQ3z8Ts+V8EbAEp1Nj6dJKamuvo69vHVu39hMM\ndqFqnp/D3GnsTuDfw59drmYiPb9dwC9DP28DLAQCGtOmraOjw4/bnYkS9rMAC07nCioqlAteud87\nqa9/kc2bXyQ//xCbNpWZ4tg6Qwm5uLsFYfwhYi1MOHQx2rcvno6OylDMugefzx9yj3cCG7Bavbjd\nzcC3ULHddPr6JqHKsIyWc6T0yuWahs1mJTk5iWBQj1u7gD9hFvjZUZ+Nru0tqOlYz5v22bEjLtTA\nxAIsxVjGFXGFW1Bu/GsIBi3hF4j6+h8OeA5DubXF3S0I44+4sb4AQTgROjrclJdvorR0G+Xlz+By\nucPf6WL05z/PZ+7cacTHJwAaBw7o7nErcC3Tp2eRnNwN/A8qE/tS1HCM6cDtKOt3FZAMPAW4sNkO\nAkZXuxvVuSwdJewQGdox1Gd96EdX1D6HMQu8uYzLbu8M7Wd277tc007gCQqCMJ4Qy1oYM06mZejR\nSpaG2ieSYBaJ1WZm5vL66z6MFmvEoq4M/VKfU1LuZNOmbwJGV3sNKiFtPpFe3/XAdeidxPLz/8E5\n56Tw1lsPEAza6OvbT1/fYlR2trLwi4sT8PnSTOVn8CbgJjHxQ1avXoamESr5ysKY+Ka/QAiCMHER\nsRbGjOEI7lAcrWRpqH2ys4v4whfWsWNHHHAYny8Nl+t0Iv20zRYrmMurZs/+AmvXvk1T00cUFPhY\nsOB3vPZaGr293ai49TWhdeopK3s93EnM4bjB9BLicrmpqNBjxgEcjiux2azh8jOVld4K3ArY8Ps1\n1q5dD2CqzY6LqyQ/HzZtWjKsZyYIwvhFxFoYM4YjuDC4BR6dkd3W9i6NjbOprHw7FKtuort7CkYL\n9PDhvRw4kIjb/RNAjcMsLFwL5KFi1p+iOnzplu1HGC3xDz98h927fwxsob5+Cvn573DxxbBt23J0\nK1qNpnRTVXXLkPetu+n1+9LLuxyOEqqqluJyufnSl17G7Y5MBDPHrNXPL3zhbLZuveL4HrogCOMS\nEWthzBhux6zBLHCHoyTkEv48cASncwVf//rDIctT1Vfr61qtD5Ca6sfpXAG8gVHwsrKm09PTh9t9\nLSr+vBHoBeJJT7fS3R3pbOb1TkElhy0HOmlp6aalxQU4UAlk76EakMwIdwY7mls/+r5eeukOzjjj\ni8yceYR587yDdGTTpMOYIJyiiFgLY8ZwSog6OtzU1bWiMqe7gIXU1QUAyMs7G6cz4gJWiVadKAs5\nsj9kk5WVHJpdbSznctHR0YT6Z2BM9Ipj0qQP+dKXckNWs3EQhje0dgORUiy9i9mtqBj2tVRXR9z6\nQ8Xmoz0LXu9cdu9ewu7dkUYtA5+NlFwJwqmIiLUwZgynhGjlytqw21qJ4gbc7klUVNSGRk1GLE2b\n7QC9vS+i1y4b909N3R/6HEnqSk1tCVnincCDoTOqY71eDYvlEcrK9CYqicBpgHGKlTG+fQ7K6s4I\nb9Nd10PF5gc2V4mUiG3fHsfOnZcPsMyl5EoQTk2kdEuIaQa29vQBC2lqysThKKGsbD3nnfcsZWXr\n2bSpDKvVO+j+2dlFoX1fo6wswM6dV5KXdzaRUq5CoMh07JtvJlFVtZTSUg3l+p5i+F5PSoOI0Kah\nLHe1TXUecw8am+/ocOPz9ZCYeCeqocq9wFfDx+ovJIIgCCCWtRCj6K7j/ftbMDcoSQYmY7d7BrXM\ni4vfCrmgzfvPnNkzYF+zZbsA1S50cfjYI0eacbnchvi4RiQBbQEqLn4xSqi/Sn7+b9C0Plpbn0OP\no1dUbB7gAbDbPaxcWUtNzfdRVv2LZGZm0Nv7K/z+81Gu9oU0Nb02sg9VEIRxi4i1EJNEXMeq21hm\nppf09BaysuzMmrWeVasuoLx804A4sB4Hb2xM5fDhvUPuv2LFGezc+Qlxcb9H09rQtGyU5RwZien3\n9/GlL71McXE8mzYtYfHi/6Kt7V5UMtmnXHppOllZ7tCam3E4bmDZsrdobdXj6CrePm1aIYWFkU5q\nDsflLFv2FsYGLTNnPovdnkF19VVE9wQfbu25IAgTFxFrIWYwJmIpi7oTo5ht3fp/wvuqyVqROPCO\nHXdywQX/v717D66yvvM4/s4dSAI5QIBEuiGAEay2TC11YVxCsY0SwKBopXWkRZuV0sEx7Qw3124t\n3VBTrbZDhyJip1AqWNYkUAhVA4RWKcvWTTEqZYg0CLmS5DQJhlzI2T8eTs41yUlyDufJyef1jyR5\n8jy/x4if/G7f379QVTWelBQb+/bdicVyj5frjbra+/f/DZttKvZtXUbFskSMFd23AGeBWVitVyks\nXAgcYO7cmRQUrMAepnFxO/rorR/qPsMabMye7VhwVlv7IcYsVAuw8PqCMc8V7mvXHtA8tYgorMU8\nPM+Jfg3jPGnPbUru88A1NaMpKjIWf5WW2igpeZ709AleVl4bVcpsNvszXgVGYdTyjsGo8x2O8yEc\nsIeKitFERUW4PLOqarzHO2zYcAenTm2msXEyHR2tdHZ67iNft+6oS3GT5OTN5OU9isWS4LHCvbfj\nMUVk+NACMzEN9wBOSLjavXjMfZuSo0421/853uV7rdYZFBau6F6k1VNdbSOclwMP4Dhw4zxGr95+\nTSy1tR9y7twZl2c6/wJhr1V+773/Q2VlCq2t99HZGef1evf3nDDh1u6hbvf30l5qEQH1rMVE3Lcy\n/eu/dhET00RFxWjWrj3iUmTEfcjY4LywrAXn3qx9LrukpBqr1blKWTze64I7evUjRpyisvJx4G3g\nBSIj4/nqVyPIy3MMs3uOCuwBFpGQ8DyTJ6fS0HCW8vIUsrPfICmpvcfiJjq+UkS8UViLaTiOthxF\nQ8NZ3n13NE1NI4H5lJaOwbl2uMWSwNGjj/LUUwc5caKZrq6RjBr1X3z6adL178nEOQjtK8dd63I3\n0dLSRXGxZ4979OirhIe/CtTT1TWRq1dPYN9j3dlp429/20xj4z9Zu9Y+x96Ja489DhhDevpE4FPK\nyjZQWRlGWZmNhQt/1UPBEx1fKSLeKazFNOxBlZ2dT1mZY07Xfq6z+/ytxZJAdPQorNYngDCamowg\njI6OoqLimNeeqc3m8hG5uf9Gbu4ujh6toqnJ0eP+9NOP6ezcdP3j3TiOtQQIo7LyNpYuzae6ehoQ\nAdTg3LNPSDhDenqj28pv43urqpJU01tE+kVhLUHR2/GYnoVQjLlfb/O37td6C0LnZ9XWfkBl5SPA\nCUpLLZw6Vcirr36ZoqIyjCpmxtx3Z2eM030XERb2PDabYw82NFJdzfW2NQNfJyrqP/nsZ79w/ZeE\n5S7z0KrpLSKDobCWoOjteEz3cHPupdo5iqZ0Ar/AmKOezJkzZzl/fjqpqSlenwVZwHPAOowe8hKW\nLv0B7e3P4dqTHwf8DmNOu4nY2Gu0tPwEo6zoFYzKaP/h8j2xsVO89pg1Dy0ig6WwlqDo7XhMz3Bb\n7lIYpKHByoIFu64vLmsB/oF9q9XVqzbuv38zpaVrenyWUVrU8XFbW6rb12MxVoR/B8ee6k20tKzC\nqP8dS2Rkk8u2LIhlzhz7QjdXmocWkcFSWEvA+XIetfPQcF/h5r5PGTbjHLaNjZNdnlldfRqoAyYB\nTURHv097u+PZMTEfc/Wq4+Pw8L8QE3Mzra2OeyYm3sq8eYc5e3YkKSlW2tvDXY6wTE4u46WXHvXr\nvyNVLhMRO4W1BFxP51E79557Kh/qjWtP+Z/ANYzDMIxqYBbLRS9D369h1P22MW9eM7Gxjmd/97uZ\nfOtbRiETi+Ui+fnfIDfXtcb41KmfsnfvCurqjIM6GhutREc79/4fHVS49jYtICKisJaA8zbk7d57\ndi8f2ltYTZpUh2Pl9SGc545HjPgB+fkP88QT53Ad2o4HrEAR77wziowMG3v3Oupul5be7vKMvDxj\nq1hP88z+HtrubVpAREQVzCTgfKnK5R5Wb74J2dlv0Nho7b7GXiXs3XerMHrKBzAWejm+b8aMO0hN\nTfFS4awZo/DJclpbV7hUN3O/f0ZG8fUiLF9mz547AFi27CSf+cxmFizY79Euf1DlMhHpjXrWEnB9\nrYb2drBFa2sUhYXLgV0899yXWbfuKCUlnVitMcDNGNXGwFix7Riurq39kIwMSEq6wsKFO6iqGk9S\n0mWgg2PHYl3mod17r84nfZWWHqKk5C1Gjap2mR+/eHEPZWUr8PcwtVaMi0hvFNYyIN4WRCUmxvf6\n9Z7mdD0XjD0HrMIeqJ6lPH+CUdP7MAAjRjzDzTfPor7+LJWV36Gy0kJpqY2srF28+ebd3W2Jiemk\ntXU39pO2ej4cxCg9arWGYbXux3PPt/+HqbViXER6o7CWAfG2IMo4PrLnr/cURp5bq27FOBrTGA72\n/PoM4JcYx1oa27UmT95BRMStVFZauq9zPuXKOewTEp4nPX2i18NBjLY6lx5twbPmuIapReTGUljL\ngHhbEFVfbyU7e7+X86h774m6b+OaNOk0V69eBuppb48lKanNrUjKOVpaEl32OZ84EU56uvftYO5t\nnTLlZrZv77l4iethHwtJTt7MuHFpNDaeIyHhM0yb5jgFTFuuRORGUFjLgHjbJ716dZHP51E7y8tb\nQFvbDv7yl3CgHputDat1JRBGUZG3gy+Wc+edr2G1Ovd468nLM+a43ed9fS332dNhH/ZtWYmJD3Zv\n3bLTlisRuREU1jIg3vZJL1z4v7ifRz1lSkGfC6YslgRiYqKxWpdgzEN3YAR9JpDgtd73nDlxFBW9\nhrElq5k5c+KwWBJYv/4LLFu2n7//PYk//nEbqak3M2WKY7GZL4u3+jN/rC1XInIjKKxlQLztk25s\njMJ5fjc9PdLrcLOd8xCyMWy+H1iBo7e8B1jutSf80ktLiI4+SkXFNVJSOsnLWwzAsmX7XRarffTR\nHj76aEX3YrPBcB7m96USm4iIvyisxS+MHuV8jIAdQVTU/1FefgvZ2W/0OI9rDCHbe9MzgCqce6kj\nR3aQkbGrx+pm3nq/jY2TXe7hz9XbzsP8PVVi05YrEQkEhbX4hdHDHIOx//l3dHQ8S1lZGGVlrvO4\nnr3p/wYex3FutKOXmpFB9/nWvs4LWyyf0NoamNXb5887rxL3XolNRCQQFNbiF3l5CwgL28mxY9do\nauqgq8v7PK7nnukXcD43Oioql8jIz2CxXGTjxvsAKC8fhXNIfvzxKI/n238JGDNmOg0Nz2CzJREW\nVk1q6nTS0nb5pcebmtrMqVMa8haRG09hLT5raLDy1FN/vL5q+zJz5sTx0ktLsFgSsFgSiI6Oxmpd\njrE4zHuouS/IioyMp7PTfu0YOjpS6ej4Bq2tNnJzd7F9ewoNDX93uV99/VngHpe2uf4S8DWysnax\nffsK/Gnr1kza2jTkLSI3nsJafLZu3VEOH7YPWdsoKnqN6Oij3cPAjmHiTGDP9TlnXELNfUHWqFEN\nxMUZ+5g/+eQ8Vms29gM39u/v5NSpXxAbm4QxFx4HtDB2bIpH227EquyxYzXkLSLBobAWn3lWEoun\nouJa99cdw8QJwHIyMjznlh2FRzqxWkfQ1PQdmprGMHv2LqZOnUBh4Rjsq8BttjAqK22MGPEMsAl7\nwE+btsujbVqVLSKhTGEtPjMC0V6TOxb4gKQkxypvX4aJ7QuyMjKKKS1d2v35iorR7N17B7CLwsJm\nHD3pZq5ds7gVRfG8r1Zli0goU1iLz/LyFnDy5C+prjZqcsMSYEf31/szTOzeE66t/ZCHH4aUFBsx\nMVW0ta3u/lpExA/Yvv3fe72fVmWLSChTWIvPLJYEJk26jepqx1B4VdX4Ad3LuSdcW/uhy2lZ8fHb\naWtzPCM19TZ/NF9EZMhSWIvPvJ07PdC5YeeecEYGLqdlRURYcV79nZbWNui2i4gMZQpr8Zn7udPJ\nyZvJy3t00Pd1HxKfMyee6GjNP4uI2CmsxWfuq8EnTLgViyWhuyBJZaWF5OSGfh8T6bk4bLGOmRQR\ncaKwHqYGcg5zT9ujPKuS9e+YyIEsDtM50iIynCish6mBnMPc0/aoG3VMpHNA19Z+QGXlasCic6RF\nJOQprIcJ957oxx/H0t+A7akH3FOP29+9X9cefBbGXuyv+9x+EZGhSmE9TLj3pJOTc+mpfndf3EN4\n40ajmIkxZ93Y3eMeSO+9N54V1GKv/1kVy0QktAUsrLds2cLrr7/OuHHjAMjJyWHevHmBepz0wT3o\nxo6dwuzZA1tx3VMIJybGU1fX3OMzB9v7de/BJyeXMWFCl1aMi0jIC2jPeuXKlaxcuTKQjxAfuQfd\ntGnXBtzL9TWE+1uvu69hc88580e1qExEhoWAhrXNZgvk7aUf/Fk729cQ7u8z+xo2V0lRERmuAhrW\nu3fvprCwkNtuu43169cTHx8fyMdJL/wZdL6GcH+feaNWlYuIDDVhtkF0f1euXMnly5c9Pp+Tk8Os\nWbOwWCyEhYXx4osvUldXR25u7qAaKwNXX29l9eoizp+PIzW1ma1bMxk71lxDyA8//Dtef91Y3Q02\nvva1Pezd+/VgN0tEJOgGFda+unTpEqtWreLAgQN9Xuu8QCnUuC/AupGys/NdCpdkZfl/X/Jg36+x\n0cratUddeuxmmpMO5s8v0EL53UDvN9QNh/frS8CGwevq6khMTATgrbfeIi0tLVCPEh84hpitQBFv\nvgnZ2W+YqvKX5qRFRLwLWFj/9Kc/5aOPPiI8PJybbrqJH/3oR4F6lPjAsSisCFhOa2sYhYUD3/vs\nbeW2L78diohI/wUsrPPy8gJ1axmADRvu4NSpzVRVTcJmG/wiLm8rtwsKVviruSIi4kQVzIaJzZvf\nu3685Wv0VrnM3mMuL4+goaGCcePSmDr1isdwuVZui4jcOArrYcIRrpnAHkaO7CAjA49tV44e8x5g\nA5WVYbz/vudwube91vX1VrKz9+skLBERP1NYDxOOcE0AlpOR4Qhf5/nnf/yjEyOA43DuOZeXjyI7\nO9+jHrh95faGDV9g1qxfcfHiOvpbC1zHXYqI9E5hPUz0VsjE9TSr3RjD5M04D5dfvnyGsrKnsQdx\ne/sOfvObh7vvkZ2dz8WLtzKQoXF/H/ghIhJqFNbDRG/bolznnxeRkPA8kycn09Cw+fqc9accPZqA\ncxCfOBHu5R4tDOQkL81/i4j0TmEtbvPPY0hPn8j27fe5XJOWthXnIIZ6L/e4D2OuO5bk5DLy8h4d\nwPN13KWIiDuFtfhU63vOnDiKil4D4oFm5syJ87hHTMxhzp4dSUqKtV8nYvnzkBERkVB0Q8qN9keo\nl5Qbqu/nSynQofx+vgjl9wvldwO931A3HN6vL+pZB0ioVfhSKVARkeBRWAeIKnyJiIi/hPd9iQyE\nVjiLiIi/KKwDJCXlnxirpkErnEVEZDA0DB4gWuGsymQiIv6isA6Q/i7ICsVgU2UyERH/UFibRCgG\nm+btRUT8Q3PWJhGKwaZ5exER/1DP2iRCseSm5u1FRPxDYW0SoRhsKqQiIuIfCmuTULCJiEhPNGct\nIiJicgprERERk1NYi4iImJzCWkRExOQU1iIiIiansBYRETE5hbWIiIjJKaxFRERMTmEtIiJicgpr\nERERk1NYi4iImJzCWkRExOQU1iIiIiansBYRETE5hbWIiIjJKaxFRERMTmEtIiJicgprERERk1NY\ni4iImJzCWkRExOQU1iIiIiansBYRETE5hbWIiIjJKaxFRERMTmEtIiJicgprERERk1NYi4iImJzC\nWkRExOQU1iIiIiansBYRETE5hbWIiIjJKaxFRERMTmEtIiJicoMK68OHD7N48WJmzpzJBx984PK1\nbdu2kZGRwcKFC/nzn/88qEaKiIgMZ4MK67S0NLZs2cLs2bNdPl9eXk5RURGHDh1i+/btPPvss9hs\ntkE1VEREZLgaVFhPnTqVKVOmeARxcXExmZmZREZGMnnyZFJSUjh9+vSgGioiIjJcBWTOuqamhqSk\npO6PJ06cSE1NTSAeJSIiEvIi+7pg5cqVXL582ePzOTk5LFiwwOv3eBvyDgsLG0DzREREpM+w/vWv\nf93vm06aNImqqqruj6urq5kwYYJP35uYGN/v5w0ler+hLZTfL5TfDfR+Q12ov19f/DYM7tybXrBg\nAYcOHaK9vZ1PPvmECxcu8LnPfc5fjxIRERlWwmyDWKb99ttvs2nTJhobGxk9ejQzZszglVdeAYyt\nW/v27SMyMpKnn36au+66y2+NFhERGU4GFdYiIiISeKpgJiIiYnIKaxEREZNTWIuIiJicacN6x44d\nzJgxA6vVGuym+NXPf/5z7rvvPpYuXcrjjz9OXV1dsJvkV3l5eSxcuJCsrCzWrFlDS0tLsJvkN73V\nwh/Kjh8/zr333ss999zDyy+/HOzm+NXGjRuZO3cuS5YsCXZTAqK6upoVK1aQmZnJkiVL2LlzZ7Cb\n5Dft7e089NBDLF26lCVLlrBly5ZgNykgurq6uP/++1m1alWv15kyrKurq3n33XdJTk4OdlP87tvf\n/jb79++noKCA+fPnh9x/gHfddRcHDx6ksLCQlJQUtm3bFuwm+U1PtfCHsq6uLjZt2sSOHTv4wx/+\nwMGDBykvLw92s/zmgQceYMeOHcFuRsBERESwYcMGDh06xJ49e9i9e3fI/Pyio6PZuXMnBQUFFBQU\ncPz48ZAsW71z506mTZvW53WmDOvc3FzWrl0b7GYERGxsbPefW1tbCQ835Y9gwObOndv9TrNmzaK6\nujrILfKfnmrhD2WnT58mJSWFm266iaioKBYtWkRxcXGwm+U3X/ziFxk9enSwmxEwiYmJzJw5EzD+\n3zJt2jRqa2uD3Cr/GTlyJGD0sjs7O4PcGv+rrq6mpKSEhx56qM9r+6xgdqMdOXKEpKQkbrnllmA3\nJWBefPFFCgsLiY+PD6lhK3f79u1j0aJFwW6G9MJbHf/3338/iC2Sgbp48SJnzpwJqQJUXV1dPPDA\nA1y4cIFHHnkkpN4NHB3T5ubmPq8NSlj3VG/8qaeeYtu2bbz66qvdnxuKvZi+6qnn5OSQk5PDyy+/\nzG9/+1vWrFkThFYOnC/14rdu3UpUVNSQmyscSC38oWwo/v0ST1euXOHJJ59k48aNLqN3Q114eDgF\nBQW0tLSwevVqzp07x/Tp04PdLL84duwY48ePZ+bMmZw8ebLP64MS1j3VGz979iyXLl0iKysLm81G\nTU0Ny5Yt4/e//z3jxo27wa0cOF/rqS9evJgnnnhiyIV1X++Xn59PSUnJkBw1GEgt/KFs0qRJVFZW\ndn9cU1Pjcx1/MYfOzk6efPJJsrKy+MpXvhLs5gREXFwcX/rSl/jTn/4UMmH93nvvceTIEUpKSmhr\na+PKlSusXbuWvLw8r9ebasI0LS2Nd955h+LiYo4cOcLEiRPJz88fUkHdl4qKiu4/FxcXM3Xq1CC2\nxv+OHz/OK6+8wtatW4mOjg52cwImVHqkt99+OxcuXODSpUu0t7dz8OBB7r777mA3y69C5WfVk40b\nNzJ9+nS++c1vBrspftXQ0NA9PHz16lVOnDgRUv+//N73vsexY8coLi7mZz/7GXfeeWePQQ0mnLN2\nFhYWFnJ/0V544QXOnz9PeHg4ycnJPPvss8Fukl/9+Mc/pqOjg8ceewyAz3/+8/zwhz8MbqP8xLkW\n/qpVq1xq4Q9VERERPPPMMzz22GPYbDYefPBBn1amDhXf//73OXnyJFarlfnz57NmzRqWLVsW7Gb5\nzV//+lcOHDhAWloaS5cuJSwsjJycHObNmxfspg1aXV0d69evp6uri66uLjIzM0lPTw92s4JGtcFF\nRERMzlTD4CIiIuJJYS0iImJyCmsRERGTU1iLiIiYnMJaRETE5BTWIiIiJqewFhERMTmFtYiIiMn9\nPyQ+uNKCpR6MAAAAAElFTkSuQmCC\n",
+            "text/plain": [
+              "\u003cmatplotlib.figure.Figure at 0xa813090\u003e"
+            ]
+          },
+          "metadata": {
+            "tags": []
+          },
+          "output_type": "display_data"
+        }
+      ],
+      "source": [
+        "# Plot the Data (Optional)\n",
+        "\n",
+        "import matplotlib.pyplot as plt\n",
+        "\n",
+        "plt.scatter(inputs.numpy(), labels.numpy())\n",
+        "plt.show()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "colab_type": "text",
+        "id": "JaFHyAG9nDET"
+      },
+      "source": [
+        "## Step 2: Define our TensorFlow variables\n",
+        "\n",
+        "We'll use Keras's object-oriented [`Dense`](https://www.tensorflow.org/api_docs/python/tf/contrib/keras/layers/Dense) layer to create our variables. In this case, we'll create a `Dense` layer with a single weight and bias.\n",
+        "\n",
+        "(**Note**: We're using the implementation of `Dense` found in `tf.layers.Dense` though the documentation link is for `tf.contrib.keras.layers.Dense`. When TensorFlow 1.4 is released, the documentation will also be in `tf.layers.Dense`) "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 4,
+      "metadata": {
+        "cellView": "code",
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          },
+          "height": 34,
+          "output_extras": [
+            {
+              "item_id": 1
+            }
+          ]
+        },
+        "colab_type": "code",
+        "executionInfo": {
+          "elapsed": 22,
+          "status": "ok",
+          "timestamp": 1505502830753,
+          "user": {
+            "displayName": "",
+            "photoUrl": "",
+            "userId": ""
+          },
+          "user_tz": 240
+        },
+        "id": "z9r-ZeyrXu3A",
+        "outputId": "6230a7a3-29fe-4d08-f101-da80425bad82"
+      },
+      "outputs": [
+        {
+          "data": {
+            "text/plain": [
+              "[]"
+            ]
+          },
+          "execution_count": 4,
+          "metadata": {
+            "tags": []
+          },
+          "output_type": "execute_result"
+        }
+      ],
+      "source": [
+        "# Create TensorFlow Variables using Keras's Dense layer.\n",
+        "\n",
+        "wb = tf.layers.Dense(units=1, use_bias=True)\n",
+        "\n",
+        "# We can access the underlying TensorFlow variables using wb.variables.\n",
+        "# However, the variables won't exist until the dimensions of the input\n",
+        "# tensors are known. Once the dimensions of the input tensors are known,\n",
+        "# Keras can create and initialize the variables. Until then, Keras will\n",
+        "# report the variables as an empty list: [].\n",
+        "\n",
+        "wb.variables"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "colab_type": "text",
+        "id": "docKLUaonYG_"
+      },
+      "source": [
+        "## Step 3: Define our loss function\n",
+        "\n",
+        "Our loss function is the standard L2 loss (where we reduce the loss to its mean across its inputs)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 0,
+      "metadata": {
+        "cellView": "code",
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          }
+        },
+        "colab_type": "code",
+        "id": "0_w8ZJSCtuY7"
+      },
+      "outputs": [],
+      "source": [
+        "def loss_fn(inputs, labels, wb):\n",
+        "  \"\"\"Calculates the mean L2 loss for our linear model.\"\"\"\n",
+        "  predictions = wb(inputs)\n",
+        "  return tf.reduce_mean(tf.square(predictions - labels))"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 6,
+      "metadata": {
+        "cellView": "code",
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          },
+          "height": 34,
+          "output_extras": [
+            {
+              "item_id": 1
+            }
+          ]
+        },
+        "colab_type": "code",
+        "executionInfo": {
+          "elapsed": 24,
+          "status": "ok",
+          "timestamp": 1505502830875,
+          "user": {
+            "displayName": "",
+            "photoUrl": "",
+            "userId": ""
+          },
+          "user_tz": 240
+        },
+        "id": "RkNbXoXkpjVH",
+        "outputId": "c36fc98d-3a57-4074-901d-c10ae017ae3f"
+      },
+      "outputs": [
+        {
+          "data": {
+            "text/plain": [
+              "\u003ctf.Tensor: id=40, shape=(), dtype=float32, numpy=7.3549819\u003e"
+            ]
+          },
+          "execution_count": 6,
+          "metadata": {
+            "tags": []
+          },
+          "output_type": "execute_result"
+        }
+      ],
+      "source": [
+        "# Test loss function (optional).\n",
+        "\n",
+        "loss_fn(inputs, labels, wb)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 7,
+      "metadata": {
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          },
+          "height": 51,
+          "output_extras": [
+            {
+              "item_id": 1
+            }
+          ]
+        },
+        "colab_type": "code",
+        "executionInfo": {
+          "elapsed": 57,
+          "status": "ok",
+          "timestamp": 1505502830981,
+          "user": {
+            "displayName": "",
+            "photoUrl": "",
+            "userId": ""
+          },
+          "user_tz": 240
+        },
+        "id": "K_7beXoHOU7t",
+        "outputId": "1ad0856a-02ec-4117-a6c0-b41030981d87"
+      },
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "w: tf.Tensor([[ 1.56891453]], shape=(1, 1), dtype=float32)\n",
+            "b: tf.Tensor([ 0.], shape=(1,), dtype=float32)\n"
+          ]
+        }
+      ],
+      "source": [
+        "# At this point, the variables exist, and can now be queried:\n",
+        "\n",
+        "w, b = wb.variables\n",
+        "print(\"w: \" + str(w.read_value()))\n",
+        "print(\"b: \" + str(b.read_value()))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "colab_type": "text",
+        "id": "YIlebeb_qYtC"
+      },
+      "source": [
+        "## Step 4: Create our gradients function using `implicit_value_and_gradients()`\n",
+        "\n",
+        "With a loss function defined, we can calculate gradients and apply them to our variables to update them.\n",
+        "\n",
+        "To calculate the gradients, we wrap our loss function using the `implicit_value_and_gradients()` function.\n",
+        "\n",
+        "`implicit_value_and_gradients()` returns a function that accepts the same inputs as the function passed in, and returns a tuple consisting of:\n",
+        "\n",
+        "1. the value returned by the function passed in (in this case, the loss calculated by `calculate_linear_model_loss()`), and\n",
+        "1. a list of tuples consisting of:\n",
+        "  1. The value of the gradient (a `tf.Tensor`) with respect to a given variable\n",
+        "  1. The corresponding variable (`tf.Variable`)\n",
+        "\n",
+        "Test it out below to get a feel for what it does. Notice how the first value of the returned tuple (the loss) is the same as the value returned in the cell above that tests our loss function."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 0,
+      "metadata": {
+        "cellView": "code",
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          }
+        },
+        "colab_type": "code",
+        "id": "v1spZQ4NwW1U"
+      },
+      "outputs": [],
+      "source": [
+        "# Produce our gradients function. See description above for details about\n",
+        "# the returned function's signature.\n",
+        "\n",
+        "value_and_gradients_fn = tfe.implicit_value_and_gradients(loss_fn)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 9,
+      "metadata": {
+        "cellView": "code",
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          },
+          "height": 153,
+          "output_extras": [
+            {
+              "item_id": 1
+            }
+          ]
+        },
+        "colab_type": "code",
+        "executionInfo": {
+          "elapsed": 46,
+          "status": "ok",
+          "timestamp": 1505502831114,
+          "user": {
+            "displayName": "",
+            "photoUrl": "",
+            "userId": ""
+          },
+          "user_tz": 240
+        },
+        "id": "21WMcpsmFFLd",
+        "outputId": "f51b3171-33f5-4f87-8bf7-0be2dc8edc8a"
+      },
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "Outputs of value_and_gradients_fn:\n",
+            "Loss: tf.Tensor(7.35498, shape=(), dtype=float32)\n",
+            "\n",
+            "Gradient: tf.Tensor([[-3.00773573]], shape=(1, 1), dtype=float32)\n",
+            "Variable: \u003ctf.Variable 'dense/kernel:0' shape=(1, 1) dtype=float32\u003e\n",
+            "\n",
+            "Gradient: tf.Tensor([-4.06519032], shape=(1,), dtype=float32)\n",
+            "Variable: \u003ctf.Variable 'dense/bias:0' shape=(1,) dtype=float32\u003e\n"
+          ]
+        }
+      ],
+      "source": [
+        "# Show outputs of value_and_gradients_fn.\n",
+        "\n",
+        "print(\"Outputs of value_and_gradients_fn:\")\n",
+        "\n",
+        "value, grads_and_vars = value_and_gradients_fn(inputs, labels, wb)\n",
+        "\n",
+        "print('Loss: {}'.format(value))\n",
+        "for (grad, var) in grads_and_vars:\n",
+        "  print(\"\")\n",
+        "  print('Gradient: {}\\nVariable: {}'.format(grad, var))"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "colab_type": "text",
+        "id": "JVDWpL9VYWdP"
+      },
+      "source": [
+        "## Step 5: Create an optimizer\n",
+        "\n",
+        "We'll use a `GradientDescentOptimizer` to fit our model."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 0,
+      "metadata": {
+        "cellView": "code",
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          }
+        },
+        "colab_type": "code",
+        "id": "DudNEebMKDWN"
+      },
+      "outputs": [],
+      "source": [
+        "optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.1)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "colab_type": "text",
+        "id": "YBeJYxY8YaiO"
+      },
+      "source": [
+        "### Step 5a: Test Our Optimizer\n",
+        "\n",
+        "Now we have everything needed to start fitting our variables to the data!\n",
+        "\n",
+        "In the next cell, we'll demo these capabilities. We'll:\n",
+        "\n",
+        "1. Print the current values of `w` and `b`\n",
+        "1. Calculate the loss and gradients\n",
+        "1. Apply the gradients\n",
+        "1. Print out the new values of `w` and `b`\n",
+        "\n",
+        "You can run the cell multiple times. Each time, you should see the values of `w` and `b` get closer to their true values of 3 and 2."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 11,
+      "metadata": {
+        "cellView": "code",
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          },
+          "height": 102,
+          "output_extras": [
+            {
+              "item_id": 1
+            }
+          ]
+        },
+        "colab_type": "code",
+        "executionInfo": {
+          "elapsed": 103,
+          "status": "ok",
+          "timestamp": 1505502831285,
+          "user": {
+            "displayName": "",
+            "photoUrl": "",
+            "userId": ""
+          },
+          "user_tz": 240
+        },
+        "id": "diDZfrMJM3OC",
+        "outputId": "d585fff0-ecb3-4e98-9b33-bbae07a95d8c"
+      },
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "Values of w, b, BEFORE applying gradients:\n",
+            "(array([[ 1.56891453]], dtype=float32), array([ 0.], dtype=float32))\n",
+            "()\n",
+            "Values of w, b, AFTER applying gradients:\n",
+            "(array([[ 1.86968815]], dtype=float32), array([ 0.40651903], dtype=float32))\n"
+          ]
+        }
+      ],
+      "source": [
+        "# Test the optimizer.\n",
+        "\n",
+        "print(\"Values of w, b, BEFORE applying gradients:\")\n",
+        "w, b = wb.variables\n",
+        "print(w.read_value().numpy(), b.read_value().numpy())\n",
+        "print()\n",
+        "\n",
+        "# Calculate the gradients:\n",
+        "empirical_loss, gradients_and_variables = value_and_gradients_fn(\n",
+        "    inputs, labels, wb)\n",
+        "optimizer.apply_gradients(gradients_and_variables)\n",
+        "\n",
+        "print(\"Values of w, b, AFTER applying gradients:\")\n",
+        "print(w.read_value().numpy(), b.read_value().numpy())"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "colab_type": "text",
+        "id": "61TgeLVlKEQp"
+      },
+      "source": [
+        "## Step 6: Create a training loop\n",
+        "\n",
+        "Of course, now we can simply turn all of this code into a self-standing training loop. We'll also capture our loss and approximations of `w` and `b` and plot them over time."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 12,
+      "metadata": {
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          },
+          "height": 397,
+          "output_extras": [
+            {
+              "item_id": 1
+            },
+            {
+              "item_id": 2
+            }
+          ]
+        },
+        "colab_type": "code",
+        "executionInfo": {
+          "elapsed": 225,
+          "status": "ok",
+          "timestamp": 1505502831550,
+          "user": {
+            "displayName": "",
+            "photoUrl": "",
+            "userId": ""
+          },
+          "user_tz": 240
+        },
+        "id": "VukGe-huNaJ4",
+        "outputId": "f0a8d665-1910-477c-d8ab-c94ccdc4afcd"
+      },
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "[2.111051321029663, 2.3047544956207275, 2.4602210521698, 2.5850086212158203, 2.6851789951324463, 2.7655951976776123, 2.830157995223999, 2.8819968700408936, 2.9236228466033936, 2.9570505619049072]\n"
+          ]
+        },
+        {
+          "data": {
+            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAd0AAAFXCAYAAADnFpTQAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3Xd4FFUbBfAzu+m9koSShBQCSC+igIAgRRGkChJEiggo\nHURAEBQBQeADRcWCha50ULFLk6IivYRQQwskhPS6O/P9sckmm4Rkk2x2difn9zz7bLuZvC8JHO7M\n7FxBkiQJREREVOlUchdARERUVTB0iYiIzIShS0REZCYMXSIiIjNh6BIREZkJQ5eIiMhMjArdlJQU\njB8/Hk8//TS6d++OkydPVnZdREREiiMY8znd6dOno2XLlujbty80Gg0yMzPh4uJijvqIiIgUo9TQ\nTU1NRa9evfDbb7+ZqyYiIiJFKnX38s2bN+Hp6YkZM2agd+/emD17NjIzM81RGxERkaKUGroajQbn\nzp3DoEGDsH37djg4OOCzzz4zR21ERESKUmro+vv7w9/fHw0bNgQAdO3aFefOnSvxa3g5ZyIioqJs\nShvg4+ODgIAAXL16FbVr18aRI0cQGhpa4tcIgoC4uBSTFSkHX19Xq+8BUEYfSugBYB+WRAk9AMro\nQwk9ALo+jFFq6ALArFmzMHXqVGg0GtSqVQsLFy6sUHFERERVkVGhW7duXWzdurWyayEiIlI0XpGK\niIjITBi6REREZsLQJSIiMhOGLhERkZkwdImIiMyEoUtERCbRuXM7uUuweAxdIiIyCUEQ5C7B4hn1\nOV0iIqKy+OijFTh69BAEQYUhQ4ajU6fOuH8/HnPmzER6ehq0Wi2mTJmOJ59sgwUL3kZU1HkAArp3\n74nnn39B7vIrDUOXiEhh5s6dhd27d5h0mz169MLcue8aNXbv3t9x+XI01qz5Fg8eJODll4egadNm\n+PXXn9Cq1eN48cVhkCQJmZmZOH/+POLi7uGbbzYBANLSUk1at6Xh7mUiIjKp06dP4qmnugIAPD29\n0LRpc5w/fw716j2CH37Yha+++hyXLkXD0dERtWrVwp07t7F8+RIcPXoYTk7OMldfuTjTJSJSmLlz\n3zV6VloZCq80l/e8ceOm+Oijz3H48EEsWDAXAwcOxuDBA/D11xtx9Ohh7Ny5DX/88StmzHhLjrLN\ngjNdIiIyifxwbYbff/8VoijiwYMHOHXqBOrXfwSxsbHw8PDEs8/2wrPP9sLFixeQmJgIUdSiffsn\n8fLLoxEdHSVzF5WLM10iIjKJvLOX27d/EmfPnsbQoS9AEFR49dXx8PT0wp4932PjxrWwsbGBk5Mz\nZs16G7GxsXj99TcgSSIEQcDo0eNk7qJyCVIlrThv7esjKmmNR2vvQwk9AOzDkiihB0AZfSihB8D4\n9XS5e5mIiMhMGLpERERmwtAlIiIyE4YuERGRmTB0iYiIzIShS0REZCYMXSIismjHjx/DmTOn9M93\n7NiKn3/+0STbXrv2K5Nsx1gMXSIismjHjx/D6dP5odurV1907fqMSba9Zo15Q5dXpCIiogrbsGEN\n7O3t0bfvAHzwwVJcvnwJK1Z8gmPH/sGPP+7C7NnzDMZHRV3Ahx8ug0aTDWdnN7z55hx4eXlj8+ZN\n2LlzG2xsbBAcXBujR4/Fzp1boVbb4Ndf92DixNfx779/w8nJCQMHDsa4caNQp04ETp48gczMTMya\nNRdr136FK1cuo2PHzhg5cgwAYMaMqYiLu4fs7Cz07/8CevTohVWrViI7OwvDh0eidu0QzJ49D7/8\nsgebN2+CVqtB/foNMGXKdJOuE8zQJSJSGOe5s2Bv4qX9snr0QloJiyg0btwM3367Hn37DkBU1AXk\n5ORAq9Xi1KkTaNy4mcFYjUaD5csX4733liEsrBY2bdqGTz/9CDNmvIX167/Bli27YWNjg7S0VDg7\nu+C55/rqQxYA/v33b4Pt2dra4Ysv1mDz5k2YPn0KvvpqPVxcXDFgQC8MGBAJNzc3zJw5B66ursjK\nysLIkUPQvn1HjB49Ftu2bcaXX64HAFy/fg2///4LVq36Emq1GkuXLsIvv+wx2awaYOgSEZEJRETU\nRVTUeaSnp8PW1hYREXVx/vw5nDx5HJMmTTMYGxNzHVeuXMakSa9BrVYhO1sDHx9fAEBYWDjmzn0T\n7dp1wBNPdDDqe7dt2w4AEBoahpCQUHh6egEAqlevgXv37sLNzQ3ffbcBBw7sAwDcu3cPN2/GoH79\nBgYrIv3779+4eDEKI0cOgSRJyM7OhpeXV0X/aAwwdImIFCZt7rslzkorg42NDfz9A/Djj7vQsGFj\nhIWF4/jxf3H79i0EBQUXGi0hJCQUn3zyZZFrL7///gqcOPEfDh7cjzVrvsSaNd+W+r1tbe0A6BZc\nsLW11b8uCAK0Wi2OHz+G//77F5999jXs7OwwbtwoZGdnF7MlCd26dceoUa+V40/AODyRioiITKJx\n46bYuHEdmjRphkaNmmDHjq0ID69TZFxgYDAePEjEmTOnAeh2N1+9egUAcPduLJo2bY4xY8YhLS0N\nGRnpcHJyQlpaWrnrSktLhaurK+zs7HD9+jWcPXtG/56trS20Wi0AoHnzR7F37+948OABACA5ORmx\nsbHl/r7F4UyXiIhMonHjpli79is0aNAQ9vYOsLe3L3I8F9DNit99dxGWL38fy5cvQnZ2Dp5//gXU\nqhWId96ZnRuwEvr3HwhnZxe0adMOs2a9gb/+2o+JE183OLGppJOc8t5r1ao1duzYisGDn0dgYBAa\nNGioH9OzZ2+89NJARETUxezZ8/Dyy2MwefJrEEUJtra2mDx5Gvz9/U32Z8Sl/R5CSctNWXsfSugB\nYB+WRAk9AMroQwk9AFzaj4iIyOIwdImIiMyEoUtERGQmDF0iIiIzYegSERGZCUOXiIjITBi6RERk\ndt99txFZWVlyl2F2DF0iIjK7zZs3Iisrs9j3RFE0czXmw9AlIqIK27BhDbZu1V0n+YMPlmLCBN2S\neseO/YN582YbjN2yZRPi4+MwbtxovPTSSwCAzp3bYeXK5Rg2bBDOnDmF/v17Ijk5CQBw4cJ5jBs3\nCgCQmZmJhQvfwciRL2H48ME4eHC/uVo0CV4GkohIgbyaNyj29YRjZ4p9vazjCyvL0n79+g3Et99u\nxIcfforQ0BqIi0tBZmYGGjRoiLFjJ+aOMry8Y94lHb/5ZjWaN38UM2a8hdTUVIwcOQQtWz4Ke3sH\no+qUG0OXiIgqrCxL++lIuTcdtVqN9u07Fnq/qH/+OYpDhw5g48Y1AHSLJdy9G4vAwGCT9VKZGLpE\nRApk7Ay1vOMLK9vSfkXZ2dkbLF6gVqshirrgzc7OP+FKkiS8++5i1KoVWKF65cJjukREZBLGLu0H\nAE5OzgbL9RVeeycgoDqios4DAPbt+0P/+qOPPoYtWzbpn0dHR5myhUpn1Ey3Y8eOcHFxgUqlgo2N\nDbZs2VLZdRERkZUxdmk/AOjZsxemTh2PgAB/LFmyssgSfUOHjsR7770DFxcXNG3avMDrL+ODD5bi\npZcGAgD8/QOwaNH/Kq8pEzNqab9OnTph27ZtcHd3N2qjFy9ehKdnQIWLk5OSlpuy9j6U0APAPiyJ\nEnoAlNGHEnoATLy0nyRJZfrc1IABA5CTk2P0eCIioqrAqNAVBAEjRoxA37598d1335U6/sSJE/jw\nQ+uZ7hMREZmDUcd0N23aBF9fXyQkJGDYsGEICQlBixYtHjq+Ro0aWLp0Ebp164769R8xWbFERETW\nzKhjugWtXLkSzs7OGDZs2EPH/PDDD3j22WfRvHlzHDlyBDY2/GQSERFRqWmYkZEBURTh7OyM9PR0\nHDx4EGPHji3xa7p3747nn38B3323EXPnvosJE6aYrGBzUdLBfWvvQwk9AOzDkiihB0AZfSihB8D4\nE6lKDd34+HiMHTsWgiBAq9WiR48eaNu2bakbfvfd97Bv3594//2F6NatOyIi6hpVEBERkVKVeiJV\nrVq1sHPnTuzYsQO7d+/GK6+8YtSGPTw88f77y5GdnY0JE8ZAo9FUuFgiIrJMsbF3MGTIAJNuMzr6\nIg4f/kv//ODB/Vi//huTbFuupQUr9YpU3bo9g759n8d//x3DqlUfVea3IiIimRW+wEVFXbp0EUeO\n5Idu27btEBn5kkm2XdLSgpWp0s9wmj9/Efbv34tFi95F165PP/SSYEREZN00Gg3eeWc2Ll68gNq1\nQzFr1tuwt7c3GHPr1k0sW7YYSUmJcHBwwHvvLYCLiw/++OM3fP3151Cr1XB2dsHy5R/jiy9WITs7\nG6dPn8TgwcOQlZWJCxfOYdKkaViw4G3Y2dkjOjoKiYkPMGPGW9iz53ucPXsa9es3wMyZcwAAS5a8\nh6ioc8jKykKHDp0wfPgrBksLenh4YMWKT/D330fw5ZefIScnBzVq1MTMmXPg4GD6lYsqPXS9vLyx\nePH/MGxYJCZMeBW7d/8MtVpd2d+WiKjKmjvXHrt3m/af9x49NJg7t+TdsTEx1zFjxhw0aNAQCxe+\ng+3bN2PgwMEGYxYvXoBp02aiRo2aOHfuDObOnYslS1bim2++wLJlH8HHxwdpaamwsbHByy+PRlTU\neUyc+DoAYM+e7w1m06mpKfj0069w8OA+vPHGJKxa9RVq1w7BiBEv4tKlaISFhWPUqNfg6uoKURQx\nYcIYXLlyyWBpQTc3NyQlJWLNmi+xYsXHsLd3wPr132DTpnUYOvRlk/4ZAmZaZah79x7o1asPduzY\nhs8//wSjR5d89jMREVkfPz9/NGjQEADQtesz2LLlW4PQzcjIwJkzJzF79hsFFjjQ3Tds2Bjz589B\nx46d0b79k0Z9vzZtngAAhISEwcvLG7VrhwAAatcOQWzsbYSFheP333/Grl07oNVqkZBwH1evXkVI\nSBgKLi149uwZXLt2BWPGjIAkSdBoNGjQoFHF/0CKYbYP0C5YsAQHD+7HggXvoEuXbrlNExGRqc2d\nm1XqrLQyFD6mW/gQrySJcHV1w5dfrte/lveRoalTZ+D8+bM4dOggRox4EatXryv1+9nZ2QEAVCqV\n/nHec61Wizt3bmPTpvVYvXotnJ1dsGDB2wbLBObXJaFly8cwZ867ZWm3XMy2tJ+Pjw/ee28pMjMz\nMWHCa2W6ljMREVm+2Ng7OHtWty7vr7/+jEaNmhi87+TkjICA6vjzz9/0r124cAGA7lhvvXqPYMSI\nUfDw8MS9e3fh5ORksPxfSYq7zlNaWhocHR3h5OSMhIT7OHLkkEEtedt+5JGGOH36JG7dugkAyMrK\nxI0bMWXo3HhmvVRUz5690aPHduzevQOrV3+KkSPHmPPbExFRJQoKCsa2bd9h4cK3ERwcgl69+hUZ\nM2fOu3j//YX45psvodVq0LNnD/Tv/yI+/ngFbt68AQBo3rwlwsLCUa2aH9at+xrDh0di8OCHXwUR\nKP7M6bCwcISHRyAysh+qVfNDo0aN9e/lLS3o4+OLFSs+wcyZczB37kxkZ+dAEASMHDkGtWoFVvBP\npJg6y3oZSGM97AojcXFxeOKJlsjMzMSffx7S74O3NEq6Soq196GEHgD2YUmU0AOgjD6U0ANg4qX9\nTMnX1xcLFy5Beno6Jk0ay93MRERUZZg9dAGgV6++ePrpZ3Ho0EF8/fVqOUogIiIyO1lCVxAELF78\nP3h4eOCdd97C9evX5CiDiIjIrGQJXQDw8/PD/PmLkZ6ehsmTxxV75hkREZGSyBa6ANCv3wB06dIN\nBw7sw5o1X8lZChERUaWTNXQFQcCSJSvg7u6BuXNnVdrnooiIiCyBrKELAP7+AZg3byHS0lK5m5mI\nyEoZu7Tfnj3f4/79eDNUZJlkD10AGDBgEDp16ox9+/7Ehg1r5S6HiIjKwZil/X78cTfi4uKKfa8q\nfITUIkJXEAQsXfoBXF3d8NZbM3H79i25SyIiojLKW9pv8OD+mD17epFF4vfu/R0XLpzHvHmzMXx4\nJLKystCxY0d88smHGDHiRfz5528YN24UoqJ0l4ZMSkpE//49AegC+eOPV2DkyJcwdOgg7Nq13ez9\nmYJFhC4AVK9eA++8swApKcmYMmU8dzMTEVVA8+bOxd5MNb44MTHX0afP81i3bjOcnJywfftmg/c7\ndOiEevXqY86cd/Hll+v1a+26u3tg9eq16NSpSzFb1c2ev/9+J1xcXPH559/g88+/wa5d2xEbe6dM\n9VkCiwldABg06EV06NARv//+K779doPc5RARURkUXtrv1KmTRcZIkoTCc6pOnTqXuu2//z6Cn376\nAcOGDcIrr7yE5OQkqzz51qwLHpRGEAQsW/Yh2rV7DLNnz0CHDh3h7x8gd1lERFbn2DHjVucp7/ji\nlLa038M4OjrqH6vVakiS7thudnZ2gVESJk16HS1bPlbRMmVlUTNdAKhZsxbmzJmHpKRETJ06gbuZ\niYisRGlL+wGAs7Mz0tJSH7qNgIAauHDhHAAYLAH46KOPY9u2LdBoNACAGzdikJWVacryzcLiQhcA\nhgwZhieeaI9ffvkJW7Z8K3c5RERkhLyl/QYP7o+UlORil/Z7+ulnsWTJQv2JVIVnxy+8EInt27di\n+PDBSE5O1r/eo0cvBAfXxogRgzFkyAAsWbIQWq220nsyNbMv7WesmJjraNfuMdjZ2eLAgX/g5+dn\nosqMo6Tlpqy9DyX0ALAPS6KEHgBl9KGEHgALXtrPWIGBQZg9+20kJiZi2rRJ3M1MRERWz2JDFwCG\nDXsZrVu3xZ4932PHjq1yl0NERFQhFh26KpUK//vfSjg5OWHGjKm4d++e3CURERGVm0WHLgDUrh2C\nN9+cg4SEBMyYMVXucoiIiMrN4kMXAEaMGIVWrR7H7t07rPbSX0RERFYRuiqVCitWfAQHBwdMnz4F\n8fFVd4UKIiKyXlYRugAQEhKGGTPeQnx8PGbO5G5mIiKyPlYTugDwyitj0KLFo9ixYxt++GG33OUQ\nERGViVWFrlqtxooVH8Pe3h7Tpk1CQsJ9uUsiIiIymlWFLgCEh9fBG2/MQlzcPbz55htyl0NERGQ0\nqwtdABgzZiyaNWuOrVu/w08//Sh3OUREREaxytDV7Wb+BHZ2dnj99YlITHwgd0lERESlssrQBYCI\niLp4/fUZuHs3FrNnz5C7HCIiolJZbegCwGuvTUDjxk3x7bcb8OuvP8ldDhERUYmsOnRtbGzwwQef\nwNbWFlOnTkRSUqLcJRERET2UVYcuANSrVx+TJ0/DnTu3MWfOm3KXQ0RE9FBWH7oAMH78ZDRo0Agb\nNqzFH3/8Jnc5RERExVJE6Nra2uKDDz6BjY0NJk8eh5SUZLlLIiIiKkIRoQsADRo0xMSJU3H79i3M\nnTtb7nKIiIiKUEzoAsDEiVNRv34DrF37Ffbt+1PucoiIiAwYHbqiKKJ3794YPXp0ZdZTIXZ2dvjg\ng4+hVqsxefI4pKamyF0SERGRntGhu2bNGoSGhlZmLSbRqFETjB8/CTduxGDevDlyl0NERKRnVOjG\nxsZi37596N+/f2XXYxKTJ7+BunXr4auvvsDBg/vlLoeIiAiAkaG7YMECTJs2DYIgVHY9JmFvb48V\nKz6GSqXCxIljkZaWJndJREREsCltwN69e+Hj44N69erh6NGjRm/Y19e1QoVVVJcuHTBt2jS89957\nWLZsAT744IMyb0PuHkxFCX0ooQeAfVgSJfQAKKMPJfRgLEGSJKmkAcuWLcOuXbugVquRlZWFtLQ0\ndO7cGYsXLy5xw3Fx8p/ElJmZiaeeegIXL0Zh5849ePzxNkZ/ra+vq0X0UFFK6EMJPQDsw5IooQdA\nGX0ooQfA+P84lLp7efLkydi7dy9+//13LFu2DK1atSo1cC2Fg4MDli//CCqVChMmvIr09HS5SyIi\noipMUZ/TLU6LFo9i9OixuHbtKhYunCd3OUREVIWVKXQfffRRrFq1qrJqqTRvvPEmQkPD8NlnH+Po\n0SNyl0NERFWU4me6AODo6Ijlyz8GAEyc+CoyMjJkroiIiKqiKhG6ANCq1WN45ZUxuHz5EhYtmi93\nOUREVAVVmdAFgBkz3kJwcG2sWrUS//77t9zlEBFRFVOlQtfJyQkrVnwMURQxYcKryMzMlLskIiKq\nQqpU6ALA44+3wcsvj0J09EUsWfKe3OUQEVEVUuVCFwDefHMuAgODsXLlchw/fkzucoiIqIqokqHr\n7OyM5ctX6nczZ2VlyV0SERFVAVUydAGgbdt2GDp0BC5cOI///c86rrBFRETWrcqGLgC89dY7qFUr\nECtWLMOpUyfkLoeIiBSuSoeui4srli37EFqtFuPHv4rs7Gy5SyIiIgWr0qELAO3bP4kXXxyGc+fO\nYPnyJXKXQ0REClblQxcA5s6dhxo1amL58iU4c+a03OUQEZFCMXQBuLq6YenSD6DRaDB+/Bjk5OTI\nXRIRESkQQzdXx45PYdCgF3HmzCl8+OH/5C6HiIgUiKFbwNtvz4e/fwCWLl2E06e5m5mIiEyLoVuA\nu7sHli5dgZycHAwdOhSpqalyl0RERArC0C2kc+duiIwcgv/++w8DBvRGcnKS3CUREZFCMHSL8f77\nyzFo0CD8889R9O3bEwkJ9+UuiYiIFIChWwwbGxusWbMGgwa9iJMnj6N372cRFxcnd1lERGTlGLoP\noVarsWzZhxg+fCTOnz+LXr2exp07t+Uui4iIrBhDtwQqlQoLFy7Bq6+OR3T0RfTs2Q03bsTIXRYR\nEVkphm4pBEHAnDnzMGXKG7h+/Rp69uyGK1cuy10WERFZIYauEQRBwBtvvIlZs+bi1q2beO65pxEV\ndUHusoiIyMowdMtg/PjJmD9/Ee7ejUWvXk/j9OlTcpdERERWhKFbRiNHjsGSJSuQkJCAPn2exfHj\nx+QuiYiIrARDtxyGDBmGDz9chZSUZPTt2xNHjhyWuyQiIrICDN1yev75F/DZZ18hMzMDAwf2xoED\n++QuiYiILBxDtwJ69uyNr75aD41Gg0GD+uG3336WuyQiIrJgDN0K6tr1aaxb9x1UKhVeemkQfvhh\nt9wlERGRhWLomkCHDh2xceNW2NnZ4+WXh2Dbts1yl0RERBaIoWsirVu3xebNO+Ds7IIxY17Ghg1r\n5S6JiIgsDEPXhFq0eBTbtu2Gp6cnJk58DatXfyZ3SUREZEEYuibWqFETbN/+I3x9q2HGjKn4+OMP\n5S6JiIgsBEO3EtSrVx87d+5BQEB1zJ37JpYuXQRJkuQui4iIZMbQrSRhYeHYuXMPAgODsGjRfCxY\n8A6Dl4ioimPoVqLg4NrYuXMPQkJCsWLFUsyePZ3BS0RUhTF0K1mNGjWxc+dPqFu3Hj777BNMnToR\noijKXRYREcmAoWsGfn5+2L79RzRo0Ahr136FceNGQ6PRyF0WERGZGUPXTLy9vbFt2240b94Cmzdv\nwujRI5CTkyN3WUREZEYMXTPy8PDE5s078fjjbbBr13YMHz4YmZmZcpdFRERmwtA1MxcXV2zcuBXt\n2z+Jn3/egyFDBiI9PV3usoiIyAwYujJwcnLC2rXfokuXbti79w8MGtQPqakpcpdFRESVrNTQzc7O\nRv/+/dGrVy/06NEDK1euNEddiufg4IAvv1yHHj164dChg+jfvxeSkhLlLouIiCqRTWkD7OzssGbN\nGjg6OkKr1eKFF15Au3bt0KhRI3PUp2h2dnb49NMvYW9vjy1bvkWfPj3w3Xc74O3tLXdpRERUCYza\nvezo6AhAN+vlR11My8bGBitXfooXXxyK06dPok+f7rh7967cZRERUSUodaYLAKIook+fPoiJiUFk\nZGTps9zgYHiJRa+8lHDsTLHDvZo3KPZ1WcerhCI9VGY9XwFwGDkan3++Cr16PY2tW3ejevUaFd9+\ngT6s6s+/oNweLKaeco5HzHWLqofjOd4SxisiL4CH/v0uzKjQValU2LFjB1JTU/Hqq6/i0qVLCAsL\nK/Fr1CqhyGu+vq4P+QZFx1rC+MI9VHY9n376Mby83LFo0SL07v0M/vjjDwQHB1d4+3l9yP3nWZHx\napVgUfWUZ/xDv8ZK6i843uBrLaCe8ozXP7eQeso7vrh/a+Wsp8zjoYy8MJYglfFiwCtXroSzszOG\nDRtW4ri4OOs+G9fX11WWHiRJwtKli7B48QJUr14D27btRkhIyf/BKYlcfZiSEnoA2IclUUIPgDL6\nsPgeRBHIzISQmQEh9x4ZmRCyMiFkZgKZGRAyMuE+dJBRmyt1ppuQkABbW1u4uroiMzMThw8fxiuv\nvFLhPqh4giBg6tTpcHBwxDvvzEbPnk9jy5ZdqFu3ntylERHJy8gAzHtfN7bg89z3szLzt5M7Hpm5\n28nIHZf3fna2cbWZKnTj4uIwffp0iKIIURTxzDPPoH379sYVQeU2duwEODo6YMaM19G79zP47rsd\naNiwsdxlEREVJUlAdjaE9DQI6em5tzT9PdLTIaQ95D1JA9fEFMMAzMrSPy9XAJa1fEEAHB0hOThA\ncnCE5OICyccXkoM9JAdHIO91BwdIjo6Avb3hcwcHuBj5vUoN3YiICGzfvr2CLVF5jBgxCg4Ojpg8\neRz69OmBTZu2onnzlnKXRUTWSJKAjIwioWd4nw4UfC2taEgWHZf7ulZb7tIcCpZZXAB6+0BydCga\ngA4ORQPRwQGSfe57jo4FxjoCDvaGz/O2aWsLCGU7NluYyUKX5BUZOQQODg4YO3YU+vV7Dhs2bMbj\nj7eRuywiqkyiqAuylBQIqakQUpINH6emQJWaCkg5cI5/UCQQ9Y/T8h8jIx2CCdbzllQqSE7OkJyc\nACcniN4+kJyc9K9JTk6QnAs8dnIGDN43fM+rpi/i00VdANo7AHZ2FQ5AS8bQtQJ9+z4POzt7jB49\nHAMH9sGaNZvQvv2TcpdFRAVJku64YEpKbiimFB+aqbrHKv17KbrX8h6npEBISzU6IJ2KK8XWVh9u\nors7pIDqucH38PArGJYlhSTs7U0bir6ukCz5RCoTY+haiR49noODw3oMH/4iBg9+HqtXr0GXLk/L\nXRaR9cvJyZ095oeeKi0lPwALhmZaam5gFgzRlPyvL+fFgyQ7O0iurpBcXCEGBUN0dc197gLJxS3/\nsasrJFfc0+i4AAAgAElEQVQ3iC4ukFxc4FHTDwlZAJxzw9HRUReMtram/TMik2HoWpHOnbth/frN\nGDJkIIYOjcSnn36JHj16yV0WkbxEEUJyEoTERKiSEiEkJkJISoQqMbH415ISgdRkeCcl6YKynMtr\nSmo1JBddOIoB1SE560JRdHXLD0gXV/2Y/OB0g+iS/1hycdHNHsvD1xXaKjRLVAKGrpVp164DNm3a\nhkGD+mPkyKH48MNV6N9/oNxlEVWMkcGpSnxQJECF5KQyHauUnJwAd3eInl6QagUWmUnqQzMvLAuG\npqsrRGfdPRwdFX3skSoHQ9cKPfZYa2zZshMDBvTB2LGjkJWVhcGDX5K7LKrqSgzOB/qQzAvSigan\n6O4BsXp1iPXqQ/LwgOTuAbHQveThAdHDE5KHJ0R3D0ju7oC9PXx9XfGAM0SSAUPXSjVr1gLbtn2P\n559/DpMnj0NmZgZefnm03GWRUmRkQBUfB9X9eKjux0OIj4fq/n2o7scDmalwi40zTXB6eEKsXgNi\n/UfyQ1Iflh6FXvPUvwc7u0psnqjyMHStWMOGjbBjxx707dsDM2dOQ0ZGJsaNmyh3WWSJ0tL0AaoP\n0fgCz+/H54bsfaji43UXLShB3hFIyckZoocHg5PISAxdKxcRURe7du1B3749MW/eW8jISMfrr8+A\nwGNNyiVJhiEaHwchNyzzn+cFqm52KqSnl75Ze3uI3j7QhIVD8vaG6O2ju/n4QPLxzX3uDc/QWojX\n2up21TI4icqEoasAISFh2LlTN+NdsuQ9ZGZmYvbstxm81kKSdB9FiYszDMr4eMNdvPfv658bc8at\n5OCgC9HwiEIh6gvJx0cfonnPJWcX404MqmKfqyQyJYauQgQGBmHXrp/Qt28PrFy5HBkZ6Zg/f7Hc\nZVVdkqQ7eSg2Fqo7t6G6GwukJcL5+q1Cx0lzH2dllb5JR0eIPr7Q1K2nuwpQboDqZ6O5AZoXrnB2\n5tm1RBaGoasgAQHVsWPHHvTv/xxWr/4MWVlZ+Prr1XKXpTzp6VDF3oH6bm6g6oP1DtR37kAVeweq\nu7HFzkYLXj1IcnKG6OMDTf1HdCFaIDAfGqJEZNUYugpTrVo1bN/+PQYM6IN1677BvXt3MG/eYtSu\nHSJ3aZZPo4Hq3l1daBYIT/Wd27rHsXd0AZuU+NBNSCoVRN9qutmof4D+pg2oDrewIDywdc4PUafi\nLuBHRErG0FUgLy9vbN26C6+8Mgy//PIL9u/fjwkTpmDs2ImwL++Vb6yZJEF4kKAL0oKz0dhYqGIL\nzFTj7pX4kRfRwwNiQAA0TZvlBmkARL8AiAHVIfr76+59fAGbh/y18nWFhsdCiao0hq5Cubm5Y+PG\nrfjzzz2YMGEiFi2ajy1bvsWiRcvQrl0HucsznbQ0qO8WmJnmBqsqNm+GGgvV3TslHjOVHBwg+gcg\np9XjEIsJUq2fP0T/AN0ViIiIKoChq2CCIGDAgAFo0aINFi2aj9WrP0O/fj3Rp08/vP32Qvj5+cld\n4sPlnoikvhEDJMXB4eKV/BlqgWBVJSc9fBMqFUQ/f90xU78AXaDm7uoV/fz1wSq5e/CEIyIyC4Zu\nFeDm5o758xdjwIBBmDZtErZt24Jff/0FM2fOxtChL0OtVstTWFoa1DdioI65BlXMdaivX4c6RndT\nxVyHKiVZP9S10JeKnp4Qa9SEpnkLaP0Dip2hij6+gFy9EREVg6FbhTRq1AQ//PAb1q79GvPnv40Z\nM17Hpk0b8P77/0OTJs1M/w2zs6G6dVMfpLowvaZ7fP06VPFxxX6Z5OQEbWAQcgJbQxsYBKd6dZDs\n6gWtf26g+gcADg6mr5eIqJIxdKsYtVqNoUNH4JlneuDtt2dh8+ZN6Nr1SQwdOgIzZ74Fd3cP4zcm\nirqPzsRch+r6NYNZqjrmOlR3bkMQxSJfJtnaQluzFjSPNIA2MAjawCCIuffawGBIPj4Gu3udfF2R\nxROQiEgBGLpVVLVq1fDRR59h0KAXMW3aJHz11Rf4/vtdePvt+ejb93nd1awkCUJCAtS5s1OVfvdv\n7u7gmzcgZGcX2bYkCBADqiPn0ccKhGkQxKBg3b1/AHf7ElGVxNCt4to2boIDH32OXz/7GCd3bkPW\nqyNxcdZ0NPX0hGNsLFRpqcV+nejjkztTDS4UrEHQ1qhV/kW5iYgUjKGrdFlZUF+OLjBLzdv9mzt7\nTUgAAAzOvQEAEu4jOeE+Yn184dGmLVA7JDdYdTNVba1AwMVFro6IiKwWQ1cJJAlCfDxsoqOgvhgF\ndXQUbC5GQX0pGrh9C17FXPBBsreHtlYgNI2b5odpkC5Qf4m+iCnz38btO7cReOEC3hs6Ak891VWG\nxoiIlIWha01EEapbN2Fz8QLUFy/mh2t0FFQPHhQZrq1eA2jfHhkBNQ1OVBKDgiBW8wNUqmK/Taem\nzXHwmR5YunQRPv30Iwwa1B/du/fEu+++hxo1alZ2l0REisXQtUQ5OVBfvQL1xagCs9eLsLl0sci6\nqJJKBW1wbeS0ehza8Aho6kRAWycC2vA6kFxc4evritRynPnr4uKCOXPm4fnnX8C0aZPwww+78Oef\nv2PatJkYOXI0bG1tTdUtEVGVwdCVU1oabC5dzA/V3Fmr+uoVCBqNwVDJwQHa0HBo6tTJD9fwCGhD\nQiv1pKV69epj5849+PbbDXj77VmYO/dNfPvtBixe/D+0avVYpX1fIiIlYuiagXD/ftHjrdEXob55\no8hY0d0DmibN8kO1Th1owiMg1gqU7WM2KpUKL7wwGF27Po13352Ldeu+QY8eXRAZOQSzZ78NLy9v\nWeoiIrI2DF1TkSSobt8qsEv4ItQXL8AmOgqq+/eLDNf6+SP7iQ76UNXWiYAmPAJStWoWex1gLy9v\nLFv2IQYMiMS0aZOwfv0a7NnzPd56ax4GDoyE6iHHiImISIehW1YaDdTXrhaatUZBHR1d5DOtkkoF\nMTAIWc1bFtglXAfaOhGQ3NxlaqDiWrV6DL/9th9ffPEpFi2aj4kTX8OGDWuxePH/UL/+I3KXR0Rk\nsRi6D5OeDpvTJwuEq+5sYfWVyxBycgyGSnZ20IaGI7tAqGrCI6ANDVPsNYJtbW0xZsxYPPdcb8ya\nNR3ff78TnTq1xahRr2Hq1Olw4ed4iYiKYOgCEFKSYXPqJGxOnoDNqeOwOXkCuHIZnoU+3yq6uELT\nsBG0deoW2CVcB2JQcJW9rGH16jXw5Zdr8dtvP2P69Nfx8ccfYMeOrZg/fzGeeeZZ3eUkiYgIQBUM\nXSE5CTanT+kC9uR/uvsrlw3GiG7uQLt2yKgdVuCEpgjdNYMZIsV66qmuOHCgHVasWIIPP1yOYcMi\n0blzVyxY8D6CgoLlLo+IyCIoOnSF5KQiM9giAevugewn2kPTqAk0TZoip1ETiMG14VvNrVyfb63K\nHB0dMX36bPTtOwBvvDEZv/76Mw4e3I9Jk17Hq6+Oh52dndwlEhHJSjGhW6aAbdwUmsZN9AHL2atp\nhYfXwdatu7F163eYM+dNLFjwDjZv3oRFi5ahbdt2cpdHRCQbqwzdIgF74jhsrl4xGKML2A7QNG7C\ngJWBIAjo128AOnfuioUL5+Grr75Anz7Pol+/AZg7dz6qVasmd4lERGZn8aGrD9gTx/NnsKUFbOOm\nupObGLCyc3f3wHvvLcWAAYMwbdpkbNnyLX755Se8+eYcDBkyDOoqegIaEVVNFhW6QlJi0V3EJQRs\nTpOm0DRqwoC1Ak2bNsdPP/2Br79ejQUL3sEbb0zGpk3r8P77y9GoURO5yyMiMgvZQteogPXwQHa7\nJ3Nnr00YsFZOrVZjxIhX8OyzPTFnzkxs27YFXbp0wPDhIzF9+iy4WfEFQ4iIjGGW0DUI2JPHYXvy\nONTXrhqMYcBWHX5+/li16ku88MKLmD59Cr744lPs2rUD8+YtRK9effnZXiJSrMoJ3T/+gOPev2Bz\n6oRxAdu4KcTAIAZsFdO+/ZPYu/cwVq5cjuXLl2DUqOFYv34tFi1agtDQcLnLIyIyucoJ3U6dkHcR\nQIOAzTsGy4ClXPb29pgy5Q306dMfM2ZMxR9//Ib27R/HuHGTMGHCFDgo9DKaRFQ1VU7oTp+OpPD6\nDFgyWu3aIdi4cSu+/34XZs16A0uXLsLWrd/lnvncW+7yiIhMotS12GJjYzFkyBA888wz6NGjB9as\nWVP6VhcuRHaPXjwmS2UiCAJ69HgOf/31D0aNeg03bsRg4MA+6NevH44d+wdSoWthExFZm1JDV61W\nY8aMGfjxxx+xadMmrF+/HpcvXy7ty4jKzcXFFfPmLcSvv+5HixaPYuvWrXj66U7o0OFxfPbZx0hI\nKLo+MRGRNSg1dH19fVGvXj0AgLOzM0JDQ3Hv3r1KL4yoQYOG+P77X/DTTz+hZ8/euHQpGrNmTUej\nRhEYNWoY9u/fC1EU5S6TiMhoZTqme/PmTVy4cAGNGjWqrHqIDKhUKnTt2hXNmrVGfHw8Nm/ehHXr\nvsb27VuxfftWBAUFIzJyCAYOjIS/f4Dc5RIRlUiQjDxQlpaWhhdffBGvvvoqnnrqqRLHBgej2BnI\nsWNpxY5v3ty52NflHK9SqYr0YE315ynYhyXUU57xeT3kjZckCX//fRTr13+DXbu2Iz39LADAwcER\nLi4ucHBwgCAIFlN/npgYFeKKWbnK0v/8C4/39XU16EPuesozvmAPllBPecf7+roiMLD4vT3WUD8A\ntGzpavV5Aej+fhvDqJmuRqPB+PHj8dxzz5UauHlUqqIF+Pq6PmRs8duQe3zhHuSup7zj8/qwlHrK\nM16lUhmMf/bZznj22c5ISkpCSIgKqakpyMzMQGZmBtRqNVxcXJCUdB9hYWEWUX9JX2MNf/6Fxxd8\nbAn1lGd83nNLqaf844v/AmupX/c11p8XxjJqpjtt2jR4enpixowZRm+4uP/RW5PC/5u3Vkrow9ge\nTp8+hQ0b1mDLlu+QlJQIAGjbth0iI4ege/eesn/mVwk/C0AZfSihB0AZfSihB6Dk/1QUVGpGHzt2\nDLt378aRI0fQq1cv9O7dG/v3769wgUSm1rBhIyxcuASnTkXh448/R5s2T+Dgwf0YM+ZlNGpUBzNn\nvo6zZ8/IXSYRVWFGH9MtK2v/n4uS/vdl7X1UpIcrVy5hw4Z12LhxHeLidGfdN2vWHJGRL6F3775w\ncTHuf6emoISfBaCMPpTQA6CMPpTQA2DCmS6RNQsJCcOsWXNx4sR5fPPNRnTp0g0nThzHlCnj0aBB\nHUyc+Br++ecoL7xBRGbB0KUqwdbWFk8/3R3r1n2H48fPYcaM2fDx8cWGDWvRvXtntGvXCqtWrcT9\n+7zwBhFVHoYuVTkBAdUxadLr+PvvE9i8eSd69eqDq1ev4K23ZqJRozoYOXIo9u79gxfeICKTk20R\neyK5qVQqtG//JNq3fxL379/Hli2bsH79GuzcuQ07d25DYGAQXnhhMF54YTCqV68hd7lEVMmys4H0\ndCA9XShwr3uclpb/WkZG0TEbNxr3PXgi1UMo6eC+tfdhzh4kScKxY/9g/fo12L59K9LT06BSqdCx\n41OIjHwJXbp0g62tbbm2rYSfBaCMPpTQA6CMPsrSgyiimMDLv8/IKDksC79XOEA1mvIv0GNsknKm\nS1SAIAho0eJRtGjxKObNW4gdO7Zh/fpv8Ntvv+C3336Br281DBgwCIMHD0FISNELbxCRjiQBaWlA\naqqAlBQBKSmGj9PSdI+1WiA+3v6hQVg4LE1BpZLg5AQ4OenuvbzEAs91rzk7S3B0zB9jeF/0NehX\nkS8ZZ7oPoYT/QQLK6MMSejh37iw2bFiDzZs34cGDBwCA1q3bIjJyCJ599jk4OjqWug1L6MMUlNCH\nEnoATN+HJAFZWXnhmB+SqanIDUvd4/zwzH+vuK+RpPKHpL19ySFX8N7R0XCMs/PDw9LREbC3N/2q\ns8Z+ZIih+xD8S2k5LKmHzMxM7NnzPdatW4MDB/YCANzc3NGv3/OIjHwJDRs+fDEQS+qjIpTQhxJ6\nAPL70GhQKAx1j4ubZRYOybzHea/n5JQvjezsJLi6SnBxAVxcdI9dXQFXVwnOzvmPdWN0z11cJNSs\n6YTs7LQiM0y12sR/WJWMoVtBSvtLac0stYdr165i48a12LhxPWJj7wAAGjduisjIIejTpx/c3NwN\nxltqH2WlhD4srQdJ0h2rTEwUkJgoICkp7x548CD/eeH309JUSE6Wyr3bVRAMw9DZuWAwokBA5j/P\nC1NdkOaHp719+Xq3tJ9FeTF0K0hJvwjW3oel96DRaPD7779i/fpv8OuvP0Or1cLR0RE9e/ZGZORL\naNXqMQiCYPF9GEsJfVRWD5mZKBSQMAjJwqFZ8P3sbOOD09ZWgru7BC8vFRwdtUVmjwXDMO/1ggGa\n956Tk+l3s5aVEn6fAIZuhSnpF8Ha+7CmHmJj7+Dbbzdg/fo1uHbtKgAgLCwckZEvYeTIobCzc5O5\nwoqzpp/Hw5TUQ04ODELz4YFZ9P3MTOMTTK2W4OEhwd0d8PCQ9Dd3d6nQ86Lv54Wl0n8W1oShW0FK\n+kWw9j6ssQdRFHHo0EGsW/cNfvhhF7KysgAAoaFhaN26rf4WEFBd5krLzlp+Hnlnz8bHC7h/X0B8\nvID4eBXu3xeQnm6PO3dy9KFZcBduWXbVCoIuFN3dJXh65gem4fOi73t46HbXVnSWaS0/i5IooQeA\noVthSvpFsPY+rL2HBw8SsG3bZhw48Cf27z+A1NT8XkJCQvUB3KbNE1YRwnL+PDIzDUM0Li7vsapQ\nuOoeZ2QYl2pubg+bZepC82GzUFfXsq+nakrW/ncDUEYPAEO3wpT0i2DtfSihB0DXx507D3DmzCn8\n9ddBHDp0AEeOHEZKSrJ+TO3aIWjT5gk8/ngbtGnzhEVeCcuUP4+cHCAhQReexYWmLlhV+sepqaWH\nqL29BB8fw5u3twQfH1H/PDTUCZKUCg8PCW5ugI2VXrFACX83lNADwNCtMCX9Ilh7H0roASi+D61W\naxDChw8fMgjh4ODaBiFco0ZNc5ddREk/D61Wd7Zt4QDNn5EWfE+FxMTSQ9TGJi808wPU17domOa9\n7uxc+m5bJf9OWRsl9AAwdCtMSb8I1t6HEnoAjOtDq9Xi7NnTBiGcnJykfz8oKNgghGvWrFXZZUOj\nAe7dE3DnjoDYWBUyMx1x7VpWkRlpfLyAhAQBolhy4qlUEry8Cs9Ciz7OC1N398q5kEFV+Z2ydEro\nAWDoVpiSfhGsvQ8l9ACUrw+tVotz587gr78O4NChgzh8+BCSkhL17wcGBqNNm/wTs2rVCizT9tPS\ngNhYAbdvq/SheueOgNu38x/fu1d6kHp46EKy5Bmp7ubpKcl+4YOq/DtlaZTQA8DQrTAl/SJYex9K\n6AEwTR+6ED6LQ4cO4K+/DuLw4b8KhXAQWrdui8cea4v69dtDrQ7MDVEVYmMF3LmjC1LdTYXk5IeH\nqZ2dBH9/CQEBIgICJAQESPD3FxEW5gBb23T4+OhC1ctLQjnXgJANf6cshxJ6AIwPXSs9fYCoalKr\n1ahTpxHc3BqjcePx6NVLwokT93DqVDyuXMnErVu22LTJD5s21QBg99DtuLtLqFFDRPPmulD195dQ\nvXr+44AA3ey0uN26vr4OiIvTVl6TRArG0CWyEJIEJCejwK5e3Wy04K7e2FjdCUiGaufedMdLfXyy\n4eAQj5yca0hMPI2srCsAbgG4CT8/EW3ahKB9+1Zo3botAgODIMh9SSKiKoShS2QGWi1w6xZw5oyq\nwK7egrt7da+VdGEGJyfdDLRuXU3uzFTM3eWrm6FWr67b3as7XuoKoCFE8RGcP38Ohw8fxF9/peDw\n4YPYtu0Atm37BgBQo0ZN/WeEW7dui6CgYIYwUSXiMd2HUNJxBmvvw1p6SEkBrl9X4do1Fa5fF3D9\nukr//ObNkldv8fExPG4aEKAL1bxdvQEBItzcKn4WryiKuHDhfG4IH8Thwwdx//59/fs1atTUnxnd\nunVbBAfXLhLC1vLzKIkSegCU0YcSegB4IlWFKekXwdr7sJQetFrdmb7Fher16wLu3y/+0kQ+PiKC\ngiSEhqrh5ZVtcGJSQIAIP7/yr9BSUaIoIirqAg4dOoBDh/7CoUMHDEK4evUaBiFcu3YIqlVzs4if\nR0VYyu9URSmhDyX0ADB0K0xJvwjW3oc5e0hNhT5M84JVF6oq3LhR/EowtrYSAgMlBAWJ+ltwcP5z\nFxfz91FekiQhKuoC/vrrAA4f1oVwfHy8/n1//wA0bdoEgYEhqFMnAuHhEQgPrwNvb28Zqy47a/hZ\nGEMJfSihB4BnLxMVSxR1s9W8UL12LT9Ur18v7iQlHW9vEQ0aFAxV3ew1KEg3a5X7c6emIggC6tat\nh7p162HEiFcgSRIuXozSh/CRI4ewZ8+eIl/n7e2tD+D8WwRq1qwFlZwXJyayMAxdUpz0dBjMVAvu\nAo6JUSErq+hs1cZGQq1aEho00BQJ1aAg3fHUqkgQBERE1EVERF0MHz4SAGBjo8GRI/8hOvoiLl6M\nwqVLuvu//z6CI0cOGXy9o6MjQkPDUadOnQKhHIGQkFDYy7VPnUhGDF2yOpIE3L1reGy14Gz13r3i\nZ1aenhLq1Ss6Uw0K0p35a60XvTc3T09PtGjxKFq0eNTg9czMTFy9egXR0VEFwvgiLl+OxpkzpwzG\nqlQqBAUFo06dCISF1cndVa2bIbu7e5izHSKz4j8zZJEkSbcb+MIFFWJjgTNn7PWhGhOjKnbJNrVa\nQs2aEtq10+hDVXevu7m7y9BIFeLg4IB69eqjXr36Bq+LooibN2/khvFF/cw4OjoKP/+8Bz//bLi7\nulo1v9wwDjc4bhwQUJ0fZyKrx9AlWUmS7mL6Fy6oEBWlu124oMbFiyokJRX8B1Z3dSU3Nwnh4WKB\nMJX0M9caNThbtUQqlQqBgUEIDAxCp05dDN67f/8+oqOj9Luqo6OjcOlSNA4e3I+DB/cbjHV2dkF4\neDjCwyMMZsjBwbVha23XoaQqi/9EkdnExeWHa37Iqoss76ZWSwgJEfHEEyIiIkS0bGkPb+80BAWJ\n8OCeR0Xx9vaGt3drPPZYa4PX09PTcflydIEw1s2Qz507ixMnjhuMtbGxQe3aIQXCOFx/7+Ji3Bml\nRObC0CWTu39fKBSsulvhz7GqVBJq15bQurUGdevqAjYiQkRoqGjwuVVfX3vExYlm7oLk5OTkhIYN\nG6Nhw8YGr2s0GsTEXEN0dLTBSVzR0RcRHX0RP/6422B89eo1DM6mzpsh+/i4mLMdIj2GLpXbgwdA\nVJS60K5hVZGP3QiChKAgCS1b5hiEa1iYCAcHmYonq2RjY4OQkDCEhISha9en9a9LkoR79+4VOYkr\nOjoK+/b9iX37/jTYjouLC/z9A+DvHwA/P//cez+D1/z8/OHk5GTuFknhGLpUqqQk4MIFtUGwRkWp\nij1LODBQRJcuGkREaBERIaJuXV248t8uqkyCIMDPzw9+fn5o27adwXupqSkFPt6kmyHfvHkdt2/f\nxqVL0SVu193dA/7+/vDzC4C/v39uKPvnhnL+Y378iYzF0CW9lBTkBqraIFxjY4uGa61aIp56SpM7\na9Wibl0R4eEinJ1lKJyoBC4urmjatDmaNm2ufy3vKkhZWVm4d+8u7t6NRWxsLO7evYPY2FjExt5B\nbOyd3NfvICrqQonfw8vLq5hgDjAI6WrV/HjCFzF0q6LUVODixfwzhfPC9fbtouFao4aIjh01ubNW\n3ey1Tp38SxsSWTN7e3vUqhWIWrUCSxyXkZGBe/fuFgjm/HDOC+Zbt27i/PmzD92GIAjw9vbRB3HB\nXdsFw9nHxxc2PA1fsfiTVbD0dODff4HDh230s9eoKBVu3CgargEBIjp00Oh3CeftHnblyZ9EcHR0\nRFBQMIKCgkscl5aWhrt3Y/VBnB/M+Y+vXLlc5GIhBalUKvj6Vis0Yy46g7a2612TDkNXITQa3a7h\n48fVOH5chf/+081gRREAHPXjqlUT8cQT+WcL581eeeEIoopzdnZGSEgoQkJCSxyXmppisBu74K7t\n/F3a53Hy5PGHbsPGxgZeXl5wc3OHu7s73N09Ctx76J97eHjAza3ovVopFwy3MgxdKyRJQEyMgOPH\n1fjvP13InjqlNrhKk5OThJYttWjZ0gaBgZn62aunp4yFExEA3XHmsDBXhIWFP3SMJElITk4q9hjz\n3bt3ERt7B8nJiUhIeICYmOvIzs4uUw2urm7FhLW7QVjnv+ZpEOCOjo68Olg5MXStQEICcOJEXsDq\nQrbgx3JUKgl164po1kyLZs1ENG2qm73a2OSdMJIjY/VEVB6CIOhnrBERdYsdk3dCmCRJyMjIQHJy\nEhITE5GUlISkpAe597rniYmJ+vcL3sfEXEdKSnKZarOzs9PPmjnLLhuGroXJyABOn87bTawL2mvX\nDI/BBgaKeO65HDRtqgvZhg21PGuYqAoTBAFOTk5wcnKCv39Amb9eq9UiOTnJIKSTkhILBHhigZth\nkF+/fg05OWX7j72rq5s+gH18vGBraw9HR139jo6OcHJy1t87ORV87lRgXP69s7Pu3hrCnKErI60W\niI5W4b//VPpZ7PnzKmg0+bttPD0ldOyoyQ1YLZo0EeHrK8lYNREpjVqthqenFzw9vcr8tWWdZRcM\n7piY6zh79rTJ+rC3ty8S2oXDOu/2sJA3DHvDcLezs6vwbnWGrplIEnD7tqA/Bnv8uBonTqiRlpb/\nA7S3l9CkiW43cdOmulvt2hJ46ISILFVFZ9ne3s6IibmH9PR0ZGSkF3ufd8vIyEB6elqh+4LjdK+l\npaUjOTkZsbGxyMhIhyia5jKyarW6SFjnzcT3799r1DZKDd2ZM2di79698Pb2xu7du0sbTrmSkqDf\nRZx3NnHBKzgJgoSICBFNm4r6WWzduiLs7GQsmojIzFQqFZydneFcScfIJElCVlZWgSDXBXZ6enEB\nXtmry4cAAAsRSURBVFyQFwx9w/vExESkp6eVafd6qaHbp08fvPjii5g2bVqFGleyrCzg7FmVwdnE\nly4ZHluoXl1E9+45aNpUN5Nt3FjLz8ASEVUyQRDg4OAABweHcu0+N4ZJQ7dFixa4detWhQpSElEE\nLl/WHYfNm8meOaNCTk7+PmBXV91C6rrdxLqZrL8/j8MSESlRWS7vyWO6pbh7V3ccNu9kpxMn1EhJ\nyQ9YW1sJDRqI+mOwzZrplqZTFb3oExERVXEM3QIkCTh/XoUDB9Q4fhw4csS5yPWIw8K06NYt/2Sn\nRx4xXPuViIjoYSotdH19reOA5Y0bwG+/6W6//w7cvZv/np+fCj17Ao8+CrRqBbRoAXh4qAGoAVjP\naiHW8rMoiRJ6ANiHJVFCD4Ay+lBCD8YyKnQlqezHI+PiUsr8NeaQlAQcPGiD/fvV2L/fBpcv589k\nq1UT0a+fFu3aadCzpyMcHVMMPq6TkwPExclQdAXkXbHGmimhB4B9WBIl9AAoow8l9AAY/x+HUkN3\nypQpOHr0KBITE9GhQweMGzcOffv2rXCB5pKVBfzzj1ofsidOqCCKuiR1dpbQpYsG7dpp0K6d7tKJ\neSHr62t9AUtERJat1NBdunSpOeowGVHUfXxn3z5dyB49mr8QgI2NbhGAdu10t2bNtOCa0kREZC6K\nOJHq+nUB+/frdhkfOKBGQkL+LuN69fJCVoPHH9dy8XUiIpKNVYZuQoLuuGzebPb69fyQrV5dxMCB\nOWjXToMnntDCz4+fjyUiIstgFaGbkQEcPZp/XPb0aRUkSbfL2M1NwjPP5Ohns6GhvFYxERFZJosM\nXa0WOHVKpd9l/PffamRl6ZLUzk5Cmzb5u4wbNdKtG0tERGTpLCKuJAm4elXAvn26kD140AZJSfnT\n1YYN80O2VSstnJxkLJaIiKicZAvde/cEHDyYv8v45s3847KBgSJ69tTtMm7TRgsfHx6XJSIi62e2\n0E1NBY4cUetns+fP56/C4+kp6UO2XTsNgoMZskREpDyVFro5OcDx4/nHZf/9Vw2NRrfL2MFBQvv2\nugtStG+vQYMGXCCAiIiUr1JCt2dP4M8/XZCaqgtZQZDQpImov/JTy5ZaODhUxncmIiKyXJUSurt3\nAyEhEvr10+0ybttWAw+PyvhORERE1qNSQvfaNcDJKa0yNk1ERGS1KuVIalBQZWyViIjIuvH0JSIi\nIjNh6BIREZkJQ5eIiMhMGLpERERmwtAlIiIyE4YuERGRmTB0iYiIzIShS0REZCYMXSIiIjNh6BIR\nEZkJQ5eIiMhMGLpERERmwtAlIiIyE4YuERGRmTB0iYiIzIShS0REZCYMXSIiIjNh6BIREZkJQ5eI\niMhMGLpERERmwtAlIiIyE4YuERGRmTB0iYiIzIShS0REZCYMXSIiIjNh6BIREZkJQ5eIiMhMGLpE\nRERmwtAlIiIyE4YuERGRmRgVuvv370e3bt3QtWtXfPbZZ5VdExERkSKVGrqiKGLevHlYvXo1vv/+\ne/zwww+4fPmyOWojIiJSlFJD99SpUwgKCkKNGjVga2uL7t274/fffzdHbURERIpSaujevXsXAQEB\n+ud+fn64d+9epRZFRESkRKWGriRJ5qiDiIhI8WxKG+Dv74/bt2/rn9+9exfVqlUrdcO+vq4Vq8wC\nKKEHQBl9KKEHgH1YEiX0ACijDyX0YKxSZ7oNGzZETEwMbt26hezsbPzwww/o1KmTOWojIiJSlFJn\numq1GrNnz8bw4cMhSRL69euH0NBQc9RGRESkKILEg7ZERERmwStSERERmQlDl4iIyEwYukRERGZS\n6olUZbF//34sWLAAkiShb9++eOWVV0y5ebOYOXMm9u7dC29vb+zevVvucsolNjYW06ZNQ3x8PNRq\nNfr3748hQ4bIXVaZZWdnIzIyEjk5OdBqtejatSvGjh0rd1nlIooi+vbtCz8/P6xatUrucsqlY8eO\ncHFxgUqlgo2NDbZs2SJ3SeWSkpKCN998E9HR0VCpVFiwYAEaN24sd1lGu3r1KiZNmgRBECBJEm7c\nuIEJEyZY5d/xr7/+Glu2bIEgCKhTpw4W/r+9u3mJag8DOP6dHKRQexElCyzIjCySFr1AEyamSTXV\nxGCLNiVRbdIow14oghYJLfoHWkREEBEaRG1EszGmQiuGYIgwIhhMKkRT5yXPnOcu4l64G+89x7nz\na7rPZz1n+A6HmYcznHmmo4P8/HzTWY7cunXrr/fCv/qslQxJp9NSX18vsVhMfvz4IXv37pWhoaFM\nPX3WDAwMSDQaFb/fbzrFtS9fvkg0GhURkcnJSdmxY0dOngsRkXg8LiIilmVJU1OTRCIRw0Xu3Lx5\nU9ra2uT48eOmU1yrq6uTsbEx0xmzdvbsWbl//76IiExPT8vExIThIvfS6bT4fD4ZHh42neLYyMiI\n1NXVSSqVEhGRkydPSldXl+EqZ96/fy9+v19SqZRYliWHDx+WT58+zXhMxr5e/l12NG/YsIH58+eb\nzpiV0tJSqqqqACgoKKCioiJnV3fOmzcP+HnVa1mW4Rp3RkZGePr0KU1NTaZTZkVEsG3bdMasTE5O\nMjg4SDAYBMDr9VJYWGi4yr1wOMyyZcv+tqo3l9i2TSKRwLIsksnkv1q89Cv58OED69evJz8/n7y8\nPDZu3Eh3d/eMx2Rs6OqO5l9TLBbj3bt3VFdXm05xxbZtAoEAPp8Pn8+Xk6/j6tWrtLe34/F4TKfM\nisfj4ciRIwSDQe7du2c6x5VYLMaiRYs4f/48+/fv59KlSySTSdNZrj1+/Jjdu3ebznBl8eLFNDc3\nU1tbS01NDUVFRWzZssV0liOVlZUMDAwwPj5OIpEgFArx+fPnGY/J2NAV/bnvL2dqaorW1lYuXLhA\nQUGB6RxX5syZw4MHDwiFQkQiEYaGhkwnOdLX10dJSQlVVVU5/x65e/cunZ2d3Lhxgzt37jA4OGg6\nyTHLsohGoxw8eJCuri7mzp2bs/8RPj09TW9vLzt37jSd4sr379/p6enhyZMn9Pf3E4/Hc+4+moqK\nCo4ePUpzczPHjh1j9erVeL0z3yqVsaHrdkez+m9YlkVrayv79u2jvr7edM6sFRYWsmnTJvr7+02n\nOPL69Wt6e3vZvn07bW1tvHz5kvb2dtNZrpSWlgJQXFxMQ0MDb9++NVzkXFlZGWVlZaxbtw6AxsZG\notGo4Sp3QqEQa9eupbi42HSKK+FwmPLychYuXEheXh4NDQ28efPGdJZjwWCQzs5Obt++zYIFC1i+\nfPmMj8/Y0P2ddjTn+hUJ/LwLe+XKlRw6dMh0imujo6NMTEwAkEwmef78OStWrDBc5czp06fp6+uj\np6eH69evs3nzZq5du2Y6y7FEIsHU1BQA8XicZ8+eUVlZabjKuZKSEpYsWcLHjx8BePHiRc6utX30\n6BF+v990hmtLly4lEomQSqUQkZw9F6OjowAMDw/T3d39j+ckYz8Z+l12NP95NTI2NkZtbS0tLS1/\n3XSRK169esXDhw9ZtWoVgUAAj8fDqVOnqKmpMZ3myNevXzl37hy2bWPbNrt27WLbtm2ms/6Xvn37\nxokTJ/B4PKTTafbs2cPWrVtNZ7ly8eJFzpw5g2VZlJeX09HRYTrJsWQySTgc5sqVK6ZTXKuurqax\nsZFAIIDX62XNmjUcOHDAdJZjLS0tjI+P4/V6uXz5MkVFM/9jku5eVkoppbJEN1IppZRSWaJDVyml\nlMoSHbpKKaVUlujQVUoppbJEh65SSimVJTp0lVJKqSzRoauUUkpliQ5dpZRSKkv+AO2e4yf8wTuC\nAAAAAElFTkSuQmCC\n",
+            "text/plain": [
+              "\u003cmatplotlib.figure.Figure at 0xc1dc310\u003e"
+            ]
+          },
+          "metadata": {
+            "tags": []
+          },
+          "output_type": "display_data"
+        }
+      ],
+      "source": [
+        "# Train our variables.\n",
+        "\n",
+        "# numpy is used for its asscalar() function.\n",
+        "import numpy as np\n",
+        "\n",
+        "num_training_steps = 10\n",
+        "\n",
+        "def train_model(inputs, labels, wb, optimizer, num_training_steps):\n",
+        "  loss_at_step = []\n",
+        "  w_at_step = []\n",
+        "  b_at_step = []\n",
+        "  for step_num in range(num_training_steps):\n",
+        "    loss, gradients_and_variables = value_and_gradients_fn(inputs, labels, wb)\n",
+        "    loss_at_step.append(np.asscalar(loss.numpy()))\n",
+        "    \n",
+        "    optimizer.apply_gradients(gradients_and_variables)\n",
+        "    w, b = wb.variables\n",
+        "    w_at_step.append(np.asscalar(w.read_value().numpy()))\n",
+        "    b_at_step.append(np.asscalar(b.read_value().numpy()))\n",
+        "\n",
+        "  print(w_at_step)\n",
+        "  t = range(0, num_training_steps)\n",
+        "  plt.plot(t, loss_at_step, 'k',\n",
+        "           t, w_at_step, 'r',\n",
+        "           t, [true_w] * num_training_steps, 'r--',\n",
+        "           t, b_at_step, 'b',\n",
+        "           t, [true_b] * num_training_steps, 'b--')\n",
+        "  plt.legend(['loss', 'w estimate', 'w true', 'b estimate', 'b true'])\n",
+        "  plt.show()\n",
+        "\n",
+        "train_model(inputs, labels, wb, optimizer, num_training_steps)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "colab_type": "text",
+        "id": "UNurY9VJ-hpH"
+      },
+      "source": [
+        "## Other Ways to Compute Gradients\n",
+        "\n",
+        "Using our loss function as an example (`calculate_linear_model_loss()`), there are several other ways we could compute gradients:\n",
+        "\n",
+        "1. `tfe.implicit_gradients()`\n",
+        "1. `tfe.gradients_function()`\n",
+        "1. `tfe.implicit_value_and_gradients()`\n",
+        "1. `tfe.value_and_gradients_function()`\n",
+        "\n",
+        "Each of these functions does the following:\n",
+        "* Wraps a function.\n",
+        "* Returns a function with the same input signature as the wrapped function.\n",
+        "\n",
+        "They differ only in what information they return.\n",
+        "\n",
+        "### Gradients-only functions\n",
+        "\n",
+        "The following two functions return a function that returns only the variables' gradients:\n",
+        "\n",
+        "1. `tfe.gradients_function()`: Returns the partial derivatives of the function `f()` with respect to the parameters of `f()`.\n",
+        "1. `tfe.implicit_gradients()`: Returns the partial derivatives of the function `f()` with respect to the trainable parameters (`tf.Variable`) used by `f()`.\n",
+        "\n",
+        "In our example above, the `tf.layers.Dense` object encapsulates the trainable parameters.\n",
+        "\n",
+        "### Value and gradients functions\n",
+        "\n",
+        "The following two functions are identical to their counterparts above, except that they also return the value of the wrapped function.\n",
+        "\n",
+        "1. `tfe.implicit_value_and_gradients()`\n",
+        "1. `tfe.value_and_gradients_function()`\n",
+        "\n",
+        "### Gradient demos\n",
+        "\n",
+        "In the demos below, we show examples for the `implicit_*` functions, since our existing loss function works seamlessly with these versions. (The other versions require that your parameters are tensors and tensors only; in our example, we're using a `Dense` layer.)\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 13,
+      "metadata": {
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          },
+          "height": 85,
+          "output_extras": [
+            {
+              "item_id": 1
+            }
+          ]
+        },
+        "colab_type": "code",
+        "executionInfo": {
+          "elapsed": 100,
+          "status": "ok",
+          "timestamp": 1505502831671,
+          "user": {
+            "displayName": "",
+            "photoUrl": "",
+            "userId": ""
+          },
+          "user_tz": 240
+        },
+        "id": "aEoCftnfAIH5",
+        "outputId": "72f1c1dc-a574-463f-f860-c4e5f48fcdaa"
+      },
+      "outputs": [
+        {
+          "data": {
+            "text/plain": [
+              "[(\u003ctf.Tensor: id=673, shape=(1, 1), dtype=float32, numpy=array([[-0.26846504]], dtype=float32)\u003e,\n",
+              "  \u003ctf.Variable 'dense/kernel:0' shape=(1, 1) dtype=float32\u003e),\n",
+              " (\u003ctf.Tensor: id=671, shape=(1,), dtype=float32, numpy=array([-0.32890949], dtype=float32)\u003e,\n",
+              "  \u003ctf.Variable 'dense/bias:0' shape=(1,) dtype=float32\u003e)]"
+            ]
+          },
+          "execution_count": 13,
+          "metadata": {
+            "tags": []
+          },
+          "output_type": "execute_result"
+        }
+      ],
+      "source": [
+        "# tfe.implicit_gradients() demo\n",
+        "gradients_fn = tfe.implicit_gradients(loss_fn)\n",
+        "\n",
+        "# Returns only gradients and variables:\n",
+        "gradients_fn(inputs, labels, wb)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 14,
+      "metadata": {
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          },
+          "height": 102,
+          "output_extras": [
+            {
+              "item_id": 1
+            }
+          ]
+        },
+        "colab_type": "code",
+        "executionInfo": {
+          "elapsed": 88,
+          "status": "ok",
+          "timestamp": 1505502831785,
+          "user": {
+            "displayName": "",
+            "photoUrl": "",
+            "userId": ""
+          },
+          "user_tz": 240
+        },
+        "id": "bbgCUdCzAVhH",
+        "outputId": "152aa9b6-9e42-4b7e-848a-9423c0b1929c"
+      },
+      "outputs": [
+        {
+          "data": {
+            "text/plain": [
+              "(\u003ctf.Tensor: id=688, shape=(), dtype=float32, numpy=1.0623235\u003e,\n",
+              " [(\u003ctf.Tensor: id=720, shape=(1, 1), dtype=float32, numpy=array([[-0.26846504]], dtype=float32)\u003e,\n",
+              "   \u003ctf.Variable 'dense/kernel:0' shape=(1, 1) dtype=float32\u003e),\n",
+              "  (\u003ctf.Tensor: id=718, shape=(1,), dtype=float32, numpy=array([-0.32890949], dtype=float32)\u003e,\n",
+              "   \u003ctf.Variable 'dense/bias:0' shape=(1,) dtype=float32\u003e)])"
+            ]
+          },
+          "execution_count": 14,
+          "metadata": {
+            "tags": []
+          },
+          "output_type": "execute_result"
+        }
+      ],
+      "source": [
+        "# tfe.implicit_value_and_gradients() demo\n",
+        "value_gradients_fn = tfe.implicit_value_and_gradients(loss_fn)\n",
+        "\n",
+        "# Returns only gradients:\n",
+        "value_gradients_fn(inputs, labels, wb)"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "default_view": {},
+      "last_runtime": {
+        "build_target": "",
+        "kind": "local"
+      },
+      "name": "Eager Execution Tutorial: Working with Gradients",
+      "provenance": [],
+      "version": "0.3.2",
+      "views": {}
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/tensorflow/contrib/eager/python/examples/notebooks/3_datasets.ipynb b/tensorflow/contrib/eager/python/examples/notebooks/3_datasets.ipynb
new file mode 100644
index 0000000000..ff0ff4a6a7
--- /dev/null
+++ b/tensorflow/contrib/eager/python/examples/notebooks/3_datasets.ipynb
@@ -0,0 +1,218 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "colab_type": "text",
+        "id": "U9i2Dsh-ziXr"
+      },
+      "source": [
+        "# Eager Execution Tutorial: Importing Data\n",
+        "\n",
+        "This notebook demonstrates the use of the [`tf.contrib.data.Dataset` API](https://www.tensorflow.org/programmers_guide/datasets) to build pipelines to feed data to your program. It covers:\n",
+        "\n",
+        "* Creating a `Dataset`.\n",
+        "* Iteration over a `Dataset` with eager execution enabled.\n",
+        "\n",
+        "We recommend using the `Dataset`s API for building performant, complex input pipelines from simple, re-usable pieces that will feed your model's training or evaluation loops.\n",
+        "\n",
+        "If you're familiar with TensorFlow graphs, the API for constructing the `Dataset` object remains exactly the same when eager execution is enabled, but the process of iterating over elements of the dataset is slightly different.  You will use a Pythonic `Iterator()` class instead of using `make_one_shot_iterator()` and `get_next()`. As a result, the discussion on iterators in the [Programmer's Guide](https://www.tensorflow.org/programmers_guide/datasets) is not relevant when eager execution is enabled."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "colab_type": "text",
+        "id": "z1JcS5iBXMRO"
+      },
+      "source": [
+        "# Setup: Enable eager execution\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 0,
+      "metadata": {
+        "cellView": "code",
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          }
+        },
+        "colab_type": "code",
+        "id": "RlIWhyeLoYnG"
+      },
+      "outputs": [],
+      "source": [
+        "# Import TensorFlow.\n",
+        "import tensorflow as tf\n",
+        "\n",
+        "# Import TensorFlow eager execution support (subject to future changes).\n",
+        "from tensorflow.contrib.eager.python import tfe\n",
+        "\n",
+        "# Enable eager execution\n",
+        "tfe.enable_eager_execution()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "colab_type": "text",
+        "id": "H9UySOPLXdaw"
+      },
+      "source": [
+        "# Step 1: Create a source `Dataset`\n",
+        "\n",
+        "Create a _source_ dataset using one of the factory functions like [`Dataset.from_tensors`](https://www.tensorflow.org/api_docs/python/tf/contrib/data/Dataset#from_tensors), [`Dataset.from_tensor_slices`](https://www.tensorflow.org/api_docs/python/tf/contrib/data/Dataset#from_tensor_slices) or using objects that read from files like [`TextLineDataset`](https://www.tensorflow.org/api_docs/python/tf/contrib/data/TextLineDataset) or [`TFRecordDataset`](https://www.tensorflow.org/api_docs/python/tf/contrib/data/TFRecordDataset). See the [Programmer's Guide](https://www.google.com/url?sa=D\u0026q=https%3A%2F%2Fwww.tensorflow.org%2Fprogrammers_guide%2Fdatasets%23reading_input_data) for more information."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 0,
+      "metadata": {
+        "cellView": "code",
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          }
+        },
+        "colab_type": "code",
+        "id": "WPTUfGq6kJ5w"
+      },
+      "outputs": [],
+      "source": [
+        "ds_tensors = tf.contrib.data.Dataset.from_tensor_slices([1, 2, 3, 4, 5, 6])\n",
+        "\n",
+        "# Create a CSV file\n",
+        "import tempfile\n",
+        "_, filename = tempfile.mkstemp()\n",
+        "with open(filename, 'w') as f:\n",
+        "  f.write(\"\"\"Line 1\n",
+        "Line 2\n",
+        "Line 3\n",
+        "  \"\"\")\n",
+        "ds_file = tf.contrib.data.TextLineDataset(filename)\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "colab_type": "text",
+        "id": "twBfWd5xyu_d"
+      },
+      "source": [
+        "# Step 2: Apply transformations\n",
+        "\n",
+        "Use the transformations functions like [`map`](https://www.tensorflow.org/api_docs/python/tf/contrib/data/Dataset#map), [`batch`](https://www.tensorflow.org/api_docs/python/tf/contrib/data/Dataset#batch), [`shuffle`](https://www.tensorflow.org/api_docs/python/tf/contrib/data/Dataset#shuffle) etc. to apply transformations to the records of the dataset. See the [API documentation for `tf.contrib.data.Dataset`](https://www.tensorflow.org/api_docs/python/tf/contrib/data/Dataset) for details."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 0,
+      "metadata": {
+        "cellView": "code",
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          }
+        },
+        "colab_type": "code",
+        "id": "ngUe237Wt48W"
+      },
+      "outputs": [],
+      "source": [
+        "ds_tensors = ds_tensors.map(tf.square).shuffle(2).batch(2)\n",
+        "ds_file = ds_file.batch(2)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "colab_type": "text",
+        "id": "IDY4WsYRhP81"
+      },
+      "source": [
+        "# Step 3: Iterate\n",
+        "\n",
+        "Use `tfe.Iterator` on the `Dataset` object to get a Python iterator over the contents of the dataset.\n",
+        "\n",
+        "If you're familiar with the use of `Dataset`s in TensorFlow graphs, note that this process of iteration is different. Here there are no calls to `Dataset.make_one_shot_iterator()` and no `get_next()` calls."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 5,
+      "metadata": {
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          },
+          "height": 153,
+          "output_extras": [
+            {
+              "item_id": 1
+            }
+          ]
+        },
+        "colab_type": "code",
+        "executionInfo": {
+          "elapsed": 201,
+          "status": "ok",
+          "timestamp": 1505952405928,
+          "user": {
+            "displayName": "",
+            "photoUrl": "",
+            "userId": ""
+          },
+          "user_tz": 420
+        },
+        "id": "lCUWzso6mbqR",
+        "outputId": "ec027d30-96c6-4ea4-9ee1-ef74ec1ae29a"
+      },
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "Elements of ds_tensors:\n",
+            "tf.Tensor([4 9], shape=(2,), dtype=int32)\n",
+            "tf.Tensor([16 25], shape=(2,), dtype=int32)\n",
+            "tf.Tensor([36  1], shape=(2,), dtype=int32)\n",
+            "\n",
+            "Elements in ds_file:\n",
+            "tf.Tensor(['Line 1' 'Line 2'], shape=(2,), dtype=string)\n",
+            "tf.Tensor(['Line 3' '  '], shape=(2,), dtype=string)\n"
+          ]
+        }
+      ],
+      "source": [
+        "print('Elements of ds_tensors:')\n",
+        "for x in tfe.Iterator(ds_tensors):\n",
+        "  print(x)\n",
+        "\n",
+        "print('\\nElements in ds_file:')\n",
+        "for x in tfe.Iterator(ds_file):\n",
+        "  print(x)"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "default_view": {},
+      "last_runtime": {
+        "build_target": "",
+        "kind": "local"
+      },
+      "name": "Eager Execution Tutorial: Importing Data",
+      "provenance": [],
+      "version": "0.3.2",
+      "views": {}
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/tensorflow/contrib/eager/python/examples/tests/cart_pole_helper_test.py b/tensorflow/contrib/eager/python/examples/tests/cart_pole_helper_test.py
new file mode 100644
index 0000000000..7a213e9e03
--- /dev/null
+++ b/tensorflow/contrib/eager/python/examples/tests/cart_pole_helper_test.py
@@ -0,0 +1,51 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.contrib.eager.python.examples import cart_pole_helper
+from tensorflow.python.framework import test_util
+from tensorflow.python.platform import test
+
+
+class RewardDiscountingTest(test_util.TensorFlowTestCase):
+
+  def testDiscountingRewards(self):
+    rewards = [0.0, 10.0, 20.0]
+    discount_rate = 0.9
+    self.assertAllClose(
+        [10 * discount_rate + 20 * discount_rate * discount_rate,
+         10 + 20 * discount_rate, 20],
+        cart_pole_helper.discount_rewards(rewards, discount_rate))
+    self.assertAllClose(
+        [-1.2], cart_pole_helper.discount_rewards([-1.2], discount_rate))
+    self.assertEqual([], cart_pole_helper.discount_rewards([], discount_rate))
+
+  def testDiscountAndNormalizeRewardSequences(self):
+    rewards1 = [0.0, 10.0, 20.0]
+    rewards2 = [0.0, 5.0, -5.0]
+    reward_sequences = [rewards1, rewards2]
+    discount_rate = 0.9
+    dn = cart_pole_helper.discount_and_normalize_rewards(reward_sequences,
+                                                         discount_rate)
+    self.assertAllClose(
+        [[1.03494653, 1.24685514, 0.64140196],
+         [-0.83817424, -0.83439016, -1.25063922]], dn)
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/contrib/eager/python/examples/tests/cart_pole_test.py b/tensorflow/contrib/eager/python/examples/tests/cart_pole_test.py
new file mode 100644
index 0000000000..dc1381cc04
--- /dev/null
+++ b/tensorflow/contrib/eager/python/examples/tests/cart_pole_test.py
@@ -0,0 +1,162 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Unit test for cart-pole reinforcement learning under eager exection."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import gc
+import glob
+import os
+import shutil
+import tempfile
+import time
+
+import gym
+import numpy as np
+
+from tensorflow.contrib.eager.python.examples import cart_pole
+from tensorflow.python.eager import context
+from tensorflow.python.eager import test
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import random_seed
+from tensorflow.python.framework import test_util
+from tensorflow.python.platform import tf_logging as logging
+from tensorflow.python.training import training
+
+
+class CartPoleTest(test_util.TensorFlowTestCase):
+
+  def setUp(self):
+    super(CartPoleTest, self).setUp()
+    self._tmp_logdir = tempfile.mkdtemp()
+
+  def tearDown(self):
+    shutil.rmtree(self._tmp_logdir)
+    super(CartPoleTest, self).tearDown()
+
+  def testGetLogitsAndAction(self):
+    hidden_size = 5
+    policy_network = cart_pole.PolicyNetwork(hidden_size)
+
+    dummy_inputs = np.array([[0.1, 0.3, 0.2, 0.5],
+                             [0.0, -0.2, 0.6, -0.8]], dtype=np.float32)
+    logits, actions = policy_network.forward(constant_op.constant(dummy_inputs))
+
+    self.assertEqual((2, 1), logits.shape)
+    self.assertEqual(dtypes.float32, logits.dtype)
+    self.assertEqual((2, 1), actions.shape)
+    self.assertEqual(dtypes.int64, actions.dtype)
+
+  def testCrossEntropy(self):
+    hidden_size = 5
+    policy_network = cart_pole.PolicyNetwork(hidden_size)
+
+    dummy_inputs = np.array([[0.1, 0.3, 0.2, 0.5],
+                             [0.0, -0.2, 0.6, -0.8]], dtype=np.float32)
+    cross_entropy = policy_network._get_cross_entropy_and_save_actions(
+        constant_op.constant(dummy_inputs))
+
+    self.assertEqual((2, 1), cross_entropy.shape)
+    self.assertEqual(dtypes.float32, cross_entropy.dtype)
+
+  def testPlayAGame(self):
+    hidden_size = 5
+    cart_pole_env = gym.make("CartPole-v0")
+    cart_pole_env.seed(0)
+    cart_pole_env.reset()
+
+    device = "gpu:0" if context.context().num_gpus() > 0 else "cpu:0"
+    logging.info("device = %s", device)
+    with context.device(device):
+      policy_network = cart_pole.PolicyNetwork(hidden_size)
+      policy_network.play(cart_pole_env, max_steps=10, render=False)
+
+  def testTrain(self):
+    hidden_size = 5
+    num_games_per_iteration = 5
+    max_steps_per_game = 10
+    discount_rate = 0.95
+    learning_rate = 0.02
+
+    cart_pole_env = gym.make("CartPole-v0")
+    cart_pole_env.reset()
+
+    device = "gpu:0" if context.context().num_gpus() > 0 else "cpu:0"
+    logging.info("device = %s", device)
+    with context.device(device):
+      policy_network = cart_pole.PolicyNetwork(hidden_size,
+                                               train_logdir=self._tmp_logdir)
+      optimizer = training.AdamOptimizer(learning_rate)
+      policy_network.train(
+          cart_pole_env,
+          optimizer,
+          discount_rate,
+          num_games_per_iteration,
+          max_steps_per_game)
+      self.assertTrue(glob.glob(os.path.join(self._tmp_logdir, "events.out.*")))
+
+
+class EagerCartPoleTrainingBenchmark(test.Benchmark):
+
+  def benchmarkEagerCartPolePolicyNetworkTraining(self):
+    burn_in_iterations = 1
+    benchmark_iterations = 2
+    num_games_per_iteration = 10
+    max_steps_per_game = 100
+    discount_rate = 0.95
+    learning_rate = 0.02
+
+    cart_pole_env = gym.make("CartPole-v0")
+    cart_pole_env.seed(0)
+    random_seed.set_random_seed(0)
+    cart_pole_env.reset()
+
+    hidden_size = 5
+    policy_network = cart_pole.PolicyNetwork(hidden_size)
+    optimizer = training.AdamOptimizer(learning_rate)
+
+    # Perform burn-in.
+    for _ in xrange(burn_in_iterations):
+      policy_network.train(
+          cart_pole_env,
+          optimizer,
+          discount_rate,
+          num_games_per_iteration,
+          max_steps_per_game)
+
+    gc.collect()
+    start_time = time.time()
+    for _ in xrange(benchmark_iterations):
+      policy_network.train(
+          cart_pole_env,
+          optimizer,
+          discount_rate,
+          num_games_per_iteration,
+          max_steps_per_game)
+    wall_time = time.time() - start_time
+    # Named "examples"_per_sec to conform with other benchmarks.
+    extras = {"examples_per_sec": benchmark_iterations / wall_time}
+    self.report_benchmark(
+        name="EagerCartPoleReinforcementLearning",
+        iters=benchmark_iterations,
+        wall_time=wall_time,
+        extras=extras)
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/contrib/eager/python/examples/tests/linear_regression_test.py b/tensorflow/contrib/eager/python/examples/tests/linear_regression_test.py
new file mode 100644
index 0000000000..aee0b3d0dd
--- /dev/null
+++ b/tensorflow/contrib/eager/python/examples/tests/linear_regression_test.py
@@ -0,0 +1,114 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Unit tests for linear regression example under TensorFlow eager execution."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import glob
+import os
+import shutil
+import tempfile
+import time
+
+import numpy as np
+import tensorflow as tf
+
+from tensorflow.contrib.eager.python.examples import linear_regression
+from tensorflow.python.eager import context
+from tensorflow.python.eager import test
+from tensorflow.python.framework import test_util
+from tensorflow.python.platform import tf_logging as logging
+
+
+def _create_data_gen_for_test():
+  true_w = np.array([[1.0], [-0.5], [2.0]], dtype=np.float32)
+  true_b = np.array([1.0], dtype=np.float32)
+  noise_level = 0
+  batch_size = 64
+  return (
+      true_w, true_b, noise_level, batch_size,
+      linear_regression.DataGenerator(true_w, true_b, noise_level, batch_size))
+
+
+class LinearRegressionTest(test_util.TensorFlowTestCase):
+
+  def setUp(self):
+    super(LinearRegressionTest, self).setUp()
+    self._tmp_logdir = tempfile.mkdtemp()
+
+  def tearDown(self):
+    shutil.rmtree(self._tmp_logdir)
+    super(LinearRegressionTest, self).tearDown()
+
+  def testSyntheticBatch(self):
+    _, _, _, batch_size, data_gen = _create_data_gen_for_test()
+
+    xs, ys = data_gen.next_batch()
+    self.assertEqual((batch_size, 3), xs.shape)
+    self.assertEqual((batch_size, 1), ys.shape)
+    self.assertEqual(tf.float32, xs.dtype)
+    self.assertEqual(tf.float32, ys.dtype)
+
+  def testLinearRegression(self):
+    true_w, true_b, _, _, data_gen = _create_data_gen_for_test()
+
+    learning_rate = 0.1
+    num_iters = 40
+
+    device = "gpu:0" if context.context().num_gpus() > 0 else "cpu:0"
+    logging.info("device = %s", device)
+    with context.device(device):
+      linear_model = linear_regression.LinearModel()
+      optimizer = tf.train.GradientDescentOptimizer(learning_rate)
+      linear_model.fit(data_gen.next_batch, optimizer, num_iters,
+                       logdir=self._tmp_logdir)
+
+      self.assertAllClose(true_w, linear_model.weights, rtol=1e-2)
+      self.assertAllClose(true_b, linear_model.biases, rtol=1e-2)
+      self.assertTrue(glob.glob(os.path.join(self._tmp_logdir, "events.out.*")))
+
+
+class EagerLinearRegressionBenchmark(test.Benchmark):
+
+  def benchmarkEagerLinearRegression(self):
+    _, _, _, _, data_gen = _create_data_gen_for_test()
+
+    learning_rate = 0.1
+    num_burnin_iters = 10
+    num_iters = 200
+
+    device = "gpu:0" if context.context().num_gpus() > 0 else "cpu:0"
+    logging.info("device = %s", device)
+    with context.device(device):
+      linear_model = linear_regression.LinearModel()
+      optimizer = tf.train.GradientDescentOptimizer(learning_rate)
+
+      # Perform burn-in.
+      linear_model.fit(data_gen.next_batch, optimizer, num_burnin_iters)
+
+      start_time = time.time()
+      linear_model.fit(data_gen.next_batch, optimizer, num_iters)
+      wall_time = time.time() - start_time
+
+      self.report_benchmark(
+          name="EagerLinearRegression",
+          iters=num_iters,
+          wall_time=wall_time)
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/contrib/eager/python/examples/tests/spinn_test.py b/tensorflow/contrib/eager/python/examples/tests/spinn_test.py
new file mode 100644
index 0000000000..9c8b691b98
--- /dev/null
+++ b/tensorflow/contrib/eager/python/examples/tests/spinn_test.py
@@ -0,0 +1,311 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import collections
+import gc
+import time
+
+import numpy as np
+import tensorflow as tf
+
+from tensorflow.contrib.eager.python import tfe
+from tensorflow.contrib.eager.python.examples import spinn
+from tensorflow.python.eager import test
+from tensorflow.python.framework import test_util
+
+
+def _generate_synthetic_snli_data_batch(sequence_length,
+                                        batch_size,
+                                        vocab_size):
+  """Generate a fake batch of SNLI data for testing."""
+  with tf.device("cpu:0"):
+    labels = tf.random_uniform([batch_size], minval=1, maxval=4, dtype=tf.int64)
+    prem = tf.random_uniform(
+        (sequence_length, batch_size), maxval=vocab_size, dtype=tf.int64)
+    prem_trans = tf.constant(np.array(
+        [[3, 3, 2, 3, 3, 3, 2, 2, 2, 3, 3, 3,
+          2, 3, 3, 2, 2, 3, 3, 3, 2, 2, 2, 2,
+          3, 2, 2]] * batch_size, dtype=np.int64).T)
+    hypo = tf.random_uniform(
+        (sequence_length, batch_size), maxval=vocab_size, dtype=tf.int64)
+    hypo_trans = tf.constant(np.array(
+        [[3, 3, 2, 3, 3, 3, 2, 2, 2, 3, 3, 3,
+          2, 3, 3, 2, 2, 3, 3, 3, 2, 2, 2, 2,
+          3, 2, 2]] * batch_size, dtype=np.int64).T)
+  if tfe.num_gpus():
+    labels = labels.gpu()
+    prem = prem.gpu()
+    prem_trans = prem_trans.gpu()
+    hypo = hypo.gpu()
+    hypo_trans = hypo_trans.gpu()
+  return labels, prem, prem_trans, hypo, hypo_trans
+
+
+def _snli_classifier_config(d_embed, d_out):
+  config_tuple = collections.namedtuple(
+      "Config", ["d_hidden", "d_proj", "d_tracker", "predict",
+                 "embed_dropout", "mlp_dropout", "n_mlp_layers", "d_mlp",
+                 "d_out", "projection", "lr"])
+  config = config_tuple(
+      d_hidden=d_embed,
+      d_proj=d_embed * 2,
+      d_tracker=8,
+      predict=False,
+      embed_dropout=0.1,
+      mlp_dropout=0.1,
+      n_mlp_layers=2,
+      d_mlp=32,
+      d_out=d_out,
+      projection=True,
+      lr=2e-3)
+  return config
+
+
+class SpinnTest(test_util.TensorFlowTestCase):
+
+  def setUp(self):
+    super(SpinnTest, self).setUp()
+    self._test_device = "gpu:0" if tfe.num_gpus() else "cpu:0"
+
+  def testBundle(self):
+    with tf.device(self._test_device):
+      lstm_iter = [np.array([[0, 1], [2, 3]], dtype=np.float32),
+                   np.array([[0, -1], [-2, -3]], dtype=np.float32),
+                   np.array([[0, 2], [4, 6]], dtype=np.float32),
+                   np.array([[0, -2], [-4, -6]], dtype=np.float32)]
+      out = spinn._bundle(lstm_iter)
+
+      self.assertEqual(2, len(out))
+      self.assertEqual(tf.float32, out[0].dtype)
+      self.assertEqual(tf.float32, out[1].dtype)
+      self.assertAllEqual(np.array([[0, 2, 0, -2, 0, 4, 0, -4]]).T,
+                          out[0].numpy())
+      self.assertAllEqual(np.array([[1, 3, -1, -3, 2, 6, -2, -6]]).T,
+                          out[1].numpy())
+
+  def testUnbunbdle(self):
+    with tf.device(self._test_device):
+      state = [np.array([[0, 1, 2], [3, 4, 5]], dtype=np.float32),
+               np.array([[0, -1, -2], [-3, -4, -5]], dtype=np.float32)]
+      out = spinn._unbundle(state)
+
+      self.assertEqual(2, len(out))
+      self.assertEqual(tf.float32, out[0].dtype)
+      self.assertEqual(tf.float32, out[1].dtype)
+      self.assertAllEqual(np.array([[0, 1, 2, 0, -1, -2]]),
+                          out[0].numpy())
+      self.assertAllEqual(np.array([[3, 4, 5, -3, -4, -5]]),
+                          out[1].numpy())
+
+  def testReduce(self):
+    with tf.device(self._test_device):
+      batch_size = 3
+      size = 10
+      tracker_size = 8
+      reducer = spinn.Reduce(size, tracker_size=tracker_size)
+
+      left_in = []
+      right_in = []
+      tracking = []
+      for _ in range(batch_size):
+        left_in.append(tf.random_normal((1, size * 2)))
+        right_in.append(tf.random_normal((1, size * 2)))
+        tracking.append(tf.random_normal((1, tracker_size * 2)))
+
+      out = reducer(left_in, right_in, tracking=tracking)
+      self.assertEqual(batch_size, len(out))
+      self.assertEqual(tf.float32, out[0].dtype)
+      self.assertEqual((1, size * 2), out[0].shape)
+
+  def testReduceTreeLSTM(self):
+    with tf.device(self._test_device):
+      size = 10
+      tracker_size = 8
+      reducer = spinn.Reduce(size, tracker_size=tracker_size)
+
+      lstm_in = np.array([[0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
+                          [0, -1, -2, -3, -4, -5, -6, -7, -8, -9]],
+                         dtype=np.float32)
+      c1 = np.array([[0, 1], [2, 3]], dtype=np.float32)
+      c2 = np.array([[0, -1], [-2, -3]], dtype=np.float32)
+
+      h, c = reducer._tree_lstm(c1, c2, lstm_in)
+      self.assertEqual(tf.float32, h.dtype)
+      self.assertEqual(tf.float32, c.dtype)
+      self.assertEqual((2, 2), h.shape)
+      self.assertEqual((2, 2), c.shape)
+
+  def testTracker(self):
+    with tf.device(self._test_device):
+      batch_size = 2
+      size = 10
+      tracker_size = 8
+      buffer_length = 18
+      stack_size = 3
+
+      tracker = spinn.Tracker(tracker_size, False)
+      tracker.reset_state()
+
+      # Create dummy inputs for testing.
+      bufs = []
+      buf = []
+      for _ in range(buffer_length):
+        buf.append(tf.random_normal((batch_size, size * 2)))
+      bufs.append(buf)
+      self.assertEqual(1, len(bufs))
+      self.assertEqual(buffer_length, len(bufs[0]))
+      self.assertEqual((batch_size, size * 2), bufs[0][0].shape)
+
+      stacks = []
+      stack = []
+      for _ in range(stack_size):
+        stack.append(tf.random_normal((batch_size, size * 2)))
+      stacks.append(stack)
+      self.assertEqual(1, len(stacks))
+      self.assertEqual(3, len(stacks[0]))
+      self.assertEqual((batch_size, size * 2), stacks[0][0].shape)
+
+      for _ in range(2):
+        out1, out2 = tracker(bufs, stacks)
+        self.assertIsNone(out2)
+        self.assertEqual(batch_size, len(out1))
+        self.assertEqual(tf.float32, out1[0].dtype)
+        self.assertEqual((1, tracker_size * 2), out1[0].shape)
+
+        self.assertEqual(tf.float32, tracker.state.c.dtype)
+        self.assertEqual((batch_size, tracker_size), tracker.state.c.shape)
+        self.assertEqual(tf.float32, tracker.state.h.dtype)
+        self.assertEqual((batch_size, tracker_size), tracker.state.h.shape)
+
+  def testSPINN(self):
+    with tf.device(self._test_device):
+      embedding_dims = 10
+      d_tracker = 8
+      sequence_length = 15
+      num_transitions = 27
+
+      config_tuple = collections.namedtuple(
+          "Config", ["d_hidden", "d_proj", "d_tracker", "predict"])
+      config = config_tuple(
+          embedding_dims, embedding_dims * 2, d_tracker, False)
+      s = spinn.SPINN(config)
+
+      # Create some fake data.
+      buffers = tf.random_normal((sequence_length, 1, config.d_proj))
+      transitions = np.array(
+          [[3], [3], [2], [3], [3], [3], [2], [2], [2], [3], [3], [3],
+           [2], [3], [3], [2], [2], [3], [3], [3], [2], [2], [2], [2],
+           [3], [2], [2]], dtype=np.int32)
+      self.assertEqual(tf.int32, transitions.dtype)
+      self.assertEqual((num_transitions, 1), transitions.shape)
+
+      out = s(buffers, transitions, training=True)
+      self.assertEqual(tf.float32, out.dtype)
+      self.assertEqual((1, embedding_dims), out.shape)
+
+  def testSNLIClassifierAndTrainer(self):
+    with tf.device(self._test_device):
+      vocab_size = 40
+      batch_size = 2
+      d_embed = 10
+      sequence_length = 15
+      d_out = 4
+
+      config = _snli_classifier_config(d_embed, d_out)
+
+      # Create fake embedding matrix.
+      embed = tf.random_normal((vocab_size, d_embed))
+
+      model = spinn.SNLIClassifier(config, embed)
+      trainer = spinn.SNLIClassifierTrainer(model, config.lr)
+
+      (labels, prem, prem_trans, hypo,
+       hypo_trans) = _generate_synthetic_snli_data_batch(sequence_length,
+                                                         batch_size,
+                                                         vocab_size)
+
+      # Invoke model under non-training mode.
+      logits = model(prem, prem_trans, hypo, hypo_trans, training=False)
+      self.assertEqual(tf.float32, logits.dtype)
+      self.assertEqual((batch_size, d_out), logits.shape)
+
+      # Invoke model under training model.
+      logits = model(prem, prem_trans, hypo, hypo_trans, training=True)
+      self.assertEqual(tf.float32, logits.dtype)
+      self.assertEqual((batch_size, d_out), logits.shape)
+
+      # Calculate loss.
+      loss1 = trainer.loss(labels, logits)
+      self.assertEqual(tf.float32, loss1.dtype)
+      self.assertEqual((), loss1.shape)
+
+      loss2, logits = trainer.train_batch(
+          labels, prem, prem_trans, hypo, hypo_trans)
+      self.assertEqual(tf.float32, loss2.dtype)
+      self.assertEqual((), loss2.shape)
+      self.assertEqual(tf.float32, logits.dtype)
+      self.assertEqual((batch_size, d_out), logits.shape)
+      # Training on the batch should have led to a change in the loss value.
+      self.assertNotEqual(loss1.numpy(), loss2.numpy())
+
+
+class EagerSpinnSNLIClassifierBenchmark(test.Benchmark):
+
+  def benchmarkEagerSpinnSNLIClassifier(self):
+    test_device = "gpu:0" if tfe.num_gpus() else "cpu:0"
+    with tf.device(test_device):
+      burn_in_iterations = 2
+      benchmark_iterations = 10
+
+      vocab_size = 1000
+      batch_size = 128
+      sequence_length = 15
+      d_embed = 200
+      d_out = 4
+
+      embed = tf.random_normal((vocab_size, d_embed))
+
+      config = _snli_classifier_config(d_embed, d_out)
+      model = spinn.SNLIClassifier(config, embed)
+      trainer = spinn.SNLIClassifierTrainer(model, config.lr)
+
+      (labels, prem, prem_trans, hypo,
+       hypo_trans) = _generate_synthetic_snli_data_batch(sequence_length,
+                                                         batch_size,
+                                                         vocab_size)
+
+      for _ in range(burn_in_iterations):
+        trainer.train_batch(labels, prem, prem_trans, hypo, hypo_trans)
+
+      gc.collect()
+      start_time = time.time()
+      for _ in xrange(benchmark_iterations):
+        trainer.train_batch(labels, prem, prem_trans, hypo, hypo_trans)
+      wall_time = time.time() - start_time
+      # Named "examples"_per_sec to conform with other benchmarks.
+      extras = {"examples_per_sec": benchmark_iterations / wall_time}
+      self.report_benchmark(
+          name="Eager_SPINN_SNLIClassifier_Benchmark",
+          iters=benchmark_iterations,
+          wall_time=wall_time,
+          extras=extras)
+
+
+if __name__ == "__main__":
+  test.main()
-- 
GitLab


From 66b1f43839ccbfe7e44df004fb92d505ab6ed942 Mon Sep 17 00:00:00 2001
From: Francois Chollet <fchollet@google.com>
Date: Fri, 20 Oct 2017 15:29:41 -0700
Subject: [PATCH 632/909] Make Network compatible with eager mode. Currently it
 only allows to instantiate a Network in eager mode using the regular Keras
 API, and call it on eager tensors.

PiperOrigin-RevId: 172942569
---
 .../keras/_impl/keras/engine/topology.py      |   2 +-
 .../keras/_impl/keras/integration_test.py     |   4 +-
 tensorflow/python/keras/_impl/keras/models.py |   2 +
 tensorflow/python/layers/base.py              | 198 +++++++++++-------
 tensorflow/python/layers/base_test.py         | 132 +++++++++---
 5 files changed, 228 insertions(+), 110 deletions(-)

diff --git a/tensorflow/python/keras/_impl/keras/engine/topology.py b/tensorflow/python/keras/_impl/keras/engine/topology.py
index d9454ee8d1..c0be023b36 100644
--- a/tensorflow/python/keras/_impl/keras/engine/topology.py
+++ b/tensorflow/python/keras/_impl/keras/engine/topology.py
@@ -776,7 +776,7 @@ class Network(tf_base_layers.Network, Layer):
     if cache_key in self._output_mask_cache:
       return self._output_mask_cache[cache_key]
     else:
-      _, output_masks, _ = self._run_internal_graph(inputs, masks)
+      _, output_masks = self._run_internal_graph(inputs, masks)
       return output_masks
 
   def get_config(self):
diff --git a/tensorflow/python/keras/_impl/keras/integration_test.py b/tensorflow/python/keras/_impl/keras/integration_test.py
index d7d20e5698..7110036848 100644
--- a/tensorflow/python/keras/_impl/keras/integration_test.py
+++ b/tensorflow/python/keras/_impl/keras/integration_test.py
@@ -192,10 +192,12 @@ class KerasIntegrationTest(test.TestCase):
       model.compile(loss='categorical_crossentropy',
                     optimizer='rmsprop',
                     metrics=['accuracy'])
+      self.assertEqual(len(model.losses), 2)
+      self.assertEqual(len(model.updates), 2)
       history = model.fit(x_train, y_train, epochs=10, batch_size=16,
                           validation_data=(x_test, y_test),
                           verbose=2)
-      self.assertGreater(history.history['val_acc'][-1], 0.85)
+      self.assertGreater(history.history['val_acc'][-1], 0.84)
 
   def test_vector_classification_shared_model(self):
     # Test that functional models that feature internal updates
diff --git a/tensorflow/python/keras/_impl/keras/models.py b/tensorflow/python/keras/_impl/keras/models.py
index 6e55c429e9..06941e4bac 100644
--- a/tensorflow/python/keras/_impl/keras/models.py
+++ b/tensorflow/python/keras/_impl/keras/models.py
@@ -420,6 +420,8 @@ class Sequential(Model):
     # Used by Layer base class.
     self._dtype = None
     self._activity_regularizer = None
+    self._per_input_losses = {}
+    self._per_input_updates = {}
 
     # The following properties are not actually used by Keras;
     # they exist for compatibility with TF's variable scoping mechanism.
diff --git a/tensorflow/python/layers/base.py b/tensorflow/python/layers/base.py
index 99a30657ef..91e18b2ba5 100644
--- a/tensorflow/python/layers/base.py
+++ b/tensorflow/python/layers/base.py
@@ -508,6 +508,7 @@ class Layer(object):
     input_list = nest.flatten(inputs)
 
     in_graph_mode = context.in_graph_mode()
+    in_deferred_mode = isinstance(input_list[0], _DeferredTensor)
     # Ensure the Layer, if being reused, is working with inputs from
     # the same graph as where it was created.
     if in_graph_mode:
@@ -515,6 +516,7 @@ class Layer(object):
         ops._get_graph_from_inputs(input_list, graph=self.graph)  # pylint: disable=protected-access
       except ValueError as e:
         raise ValueError('Input graph and Layer graph are not the same: %s' % e)
+    if in_graph_mode or in_deferred_mode:
       user_kwargs = copy.copy(kwargs)
 
     # Handle Keras mask propagation from previous layer to current layer.
@@ -553,6 +555,7 @@ class Layer(object):
               raise ValueError('activity_regularizer currently unsupported in '
                                'Eager mode. Found an activity_regularizer in '
                                '%s(%s).' % (self.__class__.__name__, self))
+          if not in_graph_mode and not in_deferred_mode:
             # TODO(agarwal): support _keras_history in Eager mode.
             for x in input_list:
               if hasattr(x, '_keras_history'):
@@ -581,13 +584,26 @@ class Layer(object):
         if call_has_scope_arg:
           kwargs['scope'] = scope
         # Check input assumptions set after layer building, e.g. input shape.
-        if in_graph_mode:
+        if in_graph_mode or in_deferred_mode:
           self._assert_input_compatibility(inputs)
-        outputs = self.call(inputs, *args, **kwargs)
 
-        if outputs is None:
-          raise ValueError('A layer\'s `call` method should return a Tensor '
-                           'or a list of Tensors, not None.')
+        if not in_deferred_mode:
+          outputs = self.call(inputs, *args, **kwargs)
+          if outputs is None:
+            raise ValueError('A layer\'s `call` method should return a Tensor '
+                             'or a list of Tensors, not None.')
+        else:
+          # Deferred mode behavior: use `_compute_output_shape` to
+          # infer the number of outputs of the layer and their shapes.
+          output_shapes = self._compute_output_shape(input_shapes)
+          output_shapes = nest.flatten(output_shapes)
+          outputs = [
+              # TODO(fchollet): name the deferred tensors?
+              _DeferredTensor(shape=shape, dtype=self._dtype)
+              for shape in output_shapes
+          ]
+          if len(outputs) == 1:
+            outputs = outputs[0]
 
         if in_graph_mode:
           # Apply activity regularization.
@@ -600,16 +616,18 @@ class Layer(object):
                 activity_regularization = self._activity_regularizer(output)
               self.add_loss(activity_regularization)
 
-        # Handle mask computation and propagation to the next layer.
-        if hasattr(self, 'compute_mask'):
-          output_mask = self.compute_mask(inputs, previous_mask)
-          if isinstance(outputs, list):
-            if output_mask is None:
-              output_mask = [None for _ in range(len(outputs))]
-            for x, m in zip(outputs, output_mask):
-              x._keras_mask = m  # pylint: disable=protected-access
-          else:
-            outputs._keras_mask = output_mask  # pylint: disable=protected-access
+        if not in_deferred_mode:
+          # TODO(fchollet): consider how masking will work with deferred mode.
+          # Handle mask computation and propagation to the next layer.
+          if hasattr(self, 'compute_mask'):
+            output_mask = self.compute_mask(inputs, previous_mask)
+            if isinstance(outputs, list):
+              if output_mask is None:
+                output_mask = [None for _ in range(len(outputs))]
+              for x, m in zip(outputs, output_mask):
+                x._keras_mask = m  # pylint: disable=protected-access
+            else:
+              outputs._keras_mask = output_mask  # pylint: disable=protected-access
 
     if in_graph_mode:
       # If all input tensors have history metadata,
@@ -631,14 +649,16 @@ class Layer(object):
         else:
           outputs = output_ls_copy
 
+      # Update global default collections.
+      _add_elements_to_collection(self.updates, ops.GraphKeys.UPDATE_OPS)
+
+    if in_deferred_mode or in_graph_mode:
+      if _have_all_keras_metadata(inputs):
         # Add an inbound node to the layer, so it can keep track of this call.
         # This updates the layer history of the output tensor(s).
         self._add_inbound_node(
             input_tensors=inputs, output_tensors=outputs, arguments=user_kwargs)
 
-      # Update global default collections.
-      _add_elements_to_collection(self.updates, ops.GraphKeys.UPDATE_OPS)
-
     self.built = True
     return outputs
 
@@ -692,7 +712,6 @@ class Layer(object):
         arguments: dictionary of keyword arguments that were passed to the
             `call` method of the layer at the call that created the node.
     """
-    assert context.in_graph_mode()
     input_tensors = nest.flatten(input_tensors)
     output_tensors = nest.flatten(output_tensors)
 
@@ -1251,6 +1270,34 @@ class Node(object):
     }
 
 
+class _DeferredTensor(object):
+  """Tensor-like object used to build graphs of layers in Eager mode.
+
+  When calling a layer on a DeferredTensor, the layer will not perform any
+  computation and will simply perfom shape inference to return new
+  DeferredTensors with appropriate shape information. Thus DeferredTensor
+  behaves like a graph-mode Tensor when manipulated by layers.
+  """
+
+  def __init__(self, shape, dtype, name=None):
+    self.shape = tensor_shape.TensorShape(shape)
+    self.dtype = dtypes.as_dtype(dtype)
+    self.name = name
+
+  def get_shape(self):
+    return self.shape
+
+  def __str__(self):
+    return "DeferredTensor('%s', shape=%s, dtype=%s)" % (self.name,
+                                                         self.get_shape(),
+                                                         self.dtype.name)
+
+  def __repr__(self):
+    return "<_DeferredTensor '%s' shape=%s dtype=%s>" % (self.name,
+                                                         self.get_shape(),
+                                                         self.dtype.name)
+
+
 class InputLayer(Layer):
   """Layer to be used as an entry point into a Network (a graph of layers).
 
@@ -1283,8 +1330,6 @@ class InputLayer(Layer):
                input_tensor=None,
                sparse=False,
                name=None):
-    if context.in_eager_mode():
-      raise RuntimeError('InputLayer not supported in Eager mode.')
     super(InputLayer, self).__init__(dtype=dtype, name=name)
     self.built = True
     self.sparse = sparse
@@ -1299,16 +1344,24 @@ class InputLayer(Layer):
       else:
         batch_input_shape = None
 
-      if sparse:
-        input_tensor = array_ops.sparse_placeholder(
+      if context.in_eager_mode():
+        # In eager mode, create a temporary placeholder to call the layer on.
+        input_tensor = _DeferredTensor(
             shape=batch_input_shape,
             dtype=dtype,
             name=self.name)
       else:
-        input_tensor = array_ops.placeholder(
-            shape=batch_input_shape,
-            dtype=dtype,
-            name=self.name)
+        # In graph mode, create a graph placeholder to call the layer on.
+        if sparse:
+          input_tensor = array_ops.sparse_placeholder(
+              shape=batch_input_shape,
+              dtype=dtype,
+              name=self.name)
+        else:
+          input_tensor = array_ops.placeholder(
+              shape=batch_input_shape,
+              dtype=dtype,
+              name=self.name)
 
       # For compatibility with Keras API.
       self.is_placeholder = True
@@ -1375,8 +1428,6 @@ def Input(  # pylint: disable=invalid-name
   Raises:
     RuntimeError: If called in Eager mode.
   """
-  if context.in_eager_mode():
-    raise RuntimeError('Input not supported in Eager mode.')
   input_layer = InputLayer(
       input_shape=shape,
       batch_size=batch_size,
@@ -1440,9 +1491,10 @@ class Network(Layer):
   """
 
   def __init__(self, inputs, outputs, name=None):  # pylint: disable=super-init-not-called
-    # TODO(agarwal): Make Network work in Eager mode.
     if context.in_eager_mode():
-      raise RuntimeError('Network not supported in Eager mode.')
+      # TODO(fchollet): check that all inputs and outputs are DeferredTensors.
+      pass
+
     # Set layer name and scope
     if isinstance(name, vs.VariableScope):
       base_name = name.name
@@ -1919,16 +1971,17 @@ class Network(Layer):
       masks = [None for _ in range(len(inputs))]
     else:
       masks = nest.flatten(mask)
-    # Try to retrieve cached outputs if the layer has already been called
-    # on these exact inputs.
-    cache_key = _object_list_uid(inputs) + '_' + _object_list_uid(masks)
-    if cache_key in self._output_tensor_cache:
-      # Cache hit.
-      return self._output_tensor_cache[cache_key]
-    else:
-      # Cache miss: actually apply the network graph to the new inputs.
-      output_tensors, _, _ = self._run_internal_graph(inputs, masks)
-      return output_tensors
+
+    if context.in_graph_mode():
+      # Try to retrieve cached outputs if the layer has already been called
+      # on these exact inputs.
+      cache_key = _object_list_uid(inputs) + '_' + _object_list_uid(masks)
+      if cache_key in self._output_tensor_cache:
+        # Cache hit.
+        return self._output_tensor_cache[cache_key]
+    # Actually apply the network graph to the new inputs.
+    outputs, _ = self._run_internal_graph(inputs, masks)
+    return outputs
 
   def _compute_output_shape(self, input_shape):
     if isinstance(input_shape, list):
@@ -2091,6 +2144,7 @@ class Network(Layer):
               if 'mask' in estimator_util.fn_args(layer.call):
                 if 'mask' not in kwargs:
                   kwargs['mask'] = computed_mask
+
               output_tensors = nest.flatten(
                   layer.call(computed_tensor, **kwargs))
               if hasattr(layer, 'compute_mask'):
@@ -2121,18 +2175,19 @@ class Network(Layer):
               ]
               layer.add_loss(regularization_losses, computed_tensors)
 
-          # Update model updates and losses:
-          # Keep track of updates that depend on the inputs
-          # (e.g. BN updates).
-          self.add_update(layer.get_updates_for(computed_tensors), inputs)
-          # Keep track of unconditional updates (e.g. a counter).
-          self.add_update(layer.get_updates_for(None), None)
-          # Keep track of losses that depend on the inputs
-          # (e.g. activity regularizers).
-          self.add_loss(layer.get_losses_for(computed_tensors), inputs)
-          # Keep track of unconditional losses
-          # (e.g. weight regularizers).
-          self.add_loss(layer.get_losses_for(None), None)
+          if context.in_graph_mode():
+            # Update model updates and losses:
+            # Keep track of updates that depend on the inputs
+            # (e.g. BN updates).
+            self.add_update(layer.get_updates_for(computed_tensors), inputs)
+            # Keep track of unconditional updates (e.g. a counter).
+            self.add_update(layer.get_updates_for(None), None)
+            # Keep track of losses that depend on the inputs
+            # (e.g. activity regularizers).
+            self.add_loss(layer.get_losses_for(computed_tensors), inputs)
+            # Keep track of unconditional losses
+            # (e.g. weight regularizers).
+            self.add_loss(layer.get_losses_for(None), None)
 
           # Update tensor_map.
           for x, y, mask in zip(reference_output_tensors, output_tensors,
@@ -2149,31 +2204,26 @@ class Network(Layer):
       output_tensors.append(tensor)
       output_masks.append(mask)
 
-    # Update cache;
-    # keys are based on ids on input tensors and inputs masks.
-    cache_key = _object_list_uid(inputs) + '_' + _object_list_uid(masks)
-
     if len(output_tensors) == 1:
       output_tensors = output_tensors[0]
-      self._output_tensor_cache[cache_key] = output_tensors
-    else:
-      self._output_tensor_cache[cache_key] = output_tensors
-
-    if len(output_masks) == 1:
-      output_masks = output_masks[0]
-      self._output_mask_cache[cache_key] = output_masks
-    else:
-      self._output_mask_cache[cache_key] = output_masks
-
-    if output_shapes is not None:
-      input_shapes = [_static_shape(x) for x in inputs]
-      cache_key = _object_list_uid(input_shapes)
-      if len(output_shapes) == 1:
+      if output_shapes is not None:
         output_shapes = output_shapes[0]
+      if output_masks is not None:
+        output_masks = output_masks[0]
+
+    if context.in_graph_mode():
+      # Update cache;
+      # keys are based on ids on input tensors and inputs masks.
+      cache_key = _object_list_uid(inputs) + '_' + _object_list_uid(masks)
+      self._output_tensor_cache[cache_key] = output_tensors
+      if output_masks is not None:
+        self._output_mask_cache[cache_key] = output_masks
+      if output_shapes is not None:
+        input_shapes = [_static_shape(x) for x in inputs]
+        cache_key = _object_list_uid(input_shapes)
         self._output_shape_cache[cache_key] = output_shapes
-      else:
-        self._output_shape_cache[cache_key] = output_shapes
-    return output_tensors, output_masks, output_shapes
+
+    return output_tensors, output_masks
 
 
 def _is_tensor_or_tensor_list(v):
diff --git a/tensorflow/python/layers/base_test.py b/tensorflow/python/layers/base_test.py
index 813a2fe755..71eff2f965 100644
--- a/tensorflow/python/layers/base_test.py
+++ b/tensorflow/python/layers/base_test.py
@@ -20,6 +20,8 @@ from __future__ import print_function
 
 import copy
 
+import numpy as np
+
 from tensorflow.python.eager import context
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
@@ -41,13 +43,13 @@ class BaseLayerTest(test.TestCase):
   @test_util.run_in_graph_and_eager_modes()
   def testLayerProperties(self):
     layer = base_layers.Layer(name='my_layer')
-    self.assertListEqual(layer.variables, [])
-    self.assertListEqual(layer.trainable_variables, [])
-    self.assertListEqual(layer.non_trainable_variables, [])
+    self.assertEqual(layer.variables, [])
+    self.assertEqual(layer.trainable_variables, [])
+    self.assertEqual(layer.non_trainable_variables, [])
     if context.in_graph_mode():
       # updates, losses only suppported in GRAPH mode
-      self.assertListEqual(layer.updates, [])
-      self.assertListEqual(layer.losses, [])
+      self.assertEqual(layer.updates, [])
+      self.assertEqual(layer.losses, [])
     self.assertEqual(layer.built, False)
     layer = base_layers.Layer(name='my_layer', trainable=False)
     self.assertEqual(layer.trainable, False)
@@ -60,11 +62,11 @@ class BaseLayerTest(test.TestCase):
     variable = layer.add_variable(
         'my_var', [2, 2], initializer=init_ops.zeros_initializer())
     self.assertEqual(variable.name, 'my_layer/my_var:0')
-    self.assertListEqual(layer.variables, [variable])
-    self.assertListEqual(layer.trainable_variables, [variable])
-    self.assertListEqual(layer.non_trainable_variables, [])
+    self.assertEqual(layer.variables, [variable])
+    self.assertEqual(layer.trainable_variables, [variable])
+    self.assertEqual(layer.non_trainable_variables, [])
     if context.in_graph_mode():
-      self.assertListEqual(
+      self.assertEqual(
           layer.variables,
           ops.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES))
 
@@ -74,9 +76,9 @@ class BaseLayerTest(test.TestCase):
         'non_trainable_var', [2, 2],
         initializer=init_ops.zeros_initializer(),
         trainable=False)
-    self.assertListEqual(layer.variables, [variable, variable_2])
-    self.assertListEqual(layer.trainable_variables, [variable])
-    self.assertListEqual(layer.non_trainable_variables, [variable_2])
+    self.assertEqual(layer.variables, [variable, variable_2])
+    self.assertEqual(layer.trainable_variables, [variable])
+    self.assertEqual(layer.non_trainable_variables, [variable_2])
     if context.in_graph_mode():
       self.assertEqual(
           len(ops.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES)), 1)
@@ -105,8 +107,8 @@ class BaseLayerTest(test.TestCase):
       inputs = random_ops.random_uniform((5,), seed=1)
       layer.apply(inputs)
       layer.apply(inputs)
-      self.assertListEqual([v.name for v in layer.variables],
-                           ['my_layer/my_var:0'])
+      self.assertEqual([v.name for v in layer.variables],
+                       ['my_layer/my_var:0'])
 
       # Creating a layer with no scope leads to lazy construction of
       # the scope at apply() time.  It uses scope "<current scope>/base_name"
@@ -120,7 +122,7 @@ class BaseLayerTest(test.TestCase):
         # The variables were created outside of the Layer, and
         # reuse=True, so the Layer does not own them and they are not
         # stored in its collection.
-        self.assertListEqual(lazy_layer.variables, [])
+        self.assertEqual(lazy_layer.variables, [])
         self.assertEqual(lazy_layer._scope.name, 'new_scope/my_layer')
 
       # Creating a layer with no scope leads to lazy construction of
@@ -135,7 +137,7 @@ class BaseLayerTest(test.TestCase):
         # The variables were created outside of the Layer, and
         # reuse=True, so the Layer does not own them and they are not
         # stored in its collection.
-        self.assertListEqual(lazy_layer.variables, [])
+        self.assertEqual(lazy_layer.variables, [])
         self.assertEqual(lazy_layer._scope.name, 'new_scope')
 
       # Checking for graph equality is only done in GRAPH mode.
@@ -183,14 +185,14 @@ class BaseLayerTest(test.TestCase):
     outputs = layer.apply(inputs)
     self.assertEqual(layer.built, True)
     self.assertEqual(outputs.op.name, 'my_layer/add')
-    self.assertListEqual([v.name
-                          for v in layer.variables], ['my_layer/my_var:0'])
+    self.assertEqual([v.name
+                      for v in layer.variables], ['my_layer/my_var:0'])
     with self.assertRaisesRegexp(ValueError,
                                  'my_layer/this_will_break_on_second_call'):
       layer.apply(inputs)
     # The list of variables hasn't changed.
-    self.assertListEqual([v.name
-                          for v in layer.variables], ['my_layer/my_var:0'])
+    self.assertEqual([v.name
+                      for v in layer.variables], ['my_layer/my_var:0'])
 
   @test_util.run_in_graph_and_eager_modes()
   def testDeepCopy(self):
@@ -435,8 +437,8 @@ class BaseLayerTest(test.TestCase):
     dense_layer.add_update(0, inputs=a)
     dense_layer.add_update(1, inputs=None)
 
-    self.assertListEqual(dense_layer.get_updates_for(a), [0])
-    self.assertListEqual(dense_layer.get_updates_for(None), [1])
+    self.assertEqual(dense_layer.get_updates_for(a), [0])
+    self.assertEqual(dense_layer.get_updates_for(None), [1])
 
   def test_get_losses_for(self):
     a = base_layers.Input(shape=(2,))
@@ -444,8 +446,8 @@ class BaseLayerTest(test.TestCase):
     dense_layer.add_loss(0, inputs=a)
     dense_layer.add_loss(1, inputs=None)
 
-    self.assertListEqual(dense_layer.get_losses_for(a), [0])
-    self.assertListEqual(dense_layer.get_losses_for(None), [1])
+    self.assertEqual(dense_layer.get_losses_for(a), [0])
+    self.assertEqual(dense_layer.get_losses_for(None), [1])
 
   def testTopologicalAttributes(self):
     # test layer attributes / methods related to cross-layer connectivity.
@@ -612,7 +614,7 @@ class NetworkTest(test.TestCase):
     a = base_layers.Input(shape=(32,), name='input_a')
     b = base_layers.Input(shape=(32,), name='input_b')
 
-    self.assertListEqual(a.get_shape().as_list(), [None, 32])
+    self.assertEqual(a.get_shape().as_list(), [None, 32])
     a_layer, a_node_index, a_tensor_index = a._keras_history
     b_layer, _, _ = b._keras_history
     self.assertEqual(len(a_layer._inbound_nodes), 1)
@@ -620,11 +622,11 @@ class NetworkTest(test.TestCase):
     node = a_layer._inbound_nodes[a_node_index]
     self.assertEqual(node.outbound_layer, a_layer)
 
-    self.assertListEqual(node.inbound_layers, [])
-    self.assertListEqual(node.input_tensors, [a])
-    self.assertListEqual(node.input_shapes, [(None, 32)])
-    self.assertListEqual(node.output_tensors, [a])
-    self.assertListEqual(node.output_shapes, [(None, 32)])
+    self.assertEqual(node.inbound_layers, [])
+    self.assertEqual(node.input_tensors, [a])
+    self.assertEqual(node.input_shapes, [(None, 32)])
+    self.assertEqual(node.output_tensors, [a])
+    self.assertEqual(node.output_shapes, [(None, 32)])
 
     dense = core_layers.Dense(16, name='dense_1')
     dense(a)
@@ -632,12 +634,12 @@ class NetworkTest(test.TestCase):
 
     self.assertEqual(len(dense._inbound_nodes), 2)
     self.assertEqual(len(dense._outbound_nodes), 0)
-    self.assertListEqual(dense._inbound_nodes[0].inbound_layers, [a_layer])
+    self.assertEqual(dense._inbound_nodes[0].inbound_layers, [a_layer])
     self.assertEqual(dense._inbound_nodes[0].outbound_layer, dense)
-    self.assertListEqual(dense._inbound_nodes[1].inbound_layers, [b_layer])
+    self.assertEqual(dense._inbound_nodes[1].inbound_layers, [b_layer])
     self.assertEqual(dense._inbound_nodes[1].outbound_layer, dense)
-    self.assertListEqual(dense._inbound_nodes[0].input_tensors, [a])
-    self.assertListEqual(dense._inbound_nodes[1].input_tensors, [b])
+    self.assertEqual(dense._inbound_nodes[0].input_tensors, [a])
+    self.assertEqual(dense._inbound_nodes[1].input_tensors, [b])
 
     # Test config
     config_0 = dense._inbound_nodes[0].get_config()
@@ -889,5 +891,67 @@ class NetworkTest(test.TestCase):
       self.assertAllEqual(self.evaluate(a * mask), self.evaluate(b))
 
 
+class DeferredModeTest(test.TestCase):
+
+  def testDeferredTensorAttributes(self):
+    x = base_layers._DeferredTensor(shape=(None, 2), dtype='float32', name='x')
+    self.assertEqual(str(x),
+                     'DeferredTensor(\'x\', shape=(?, 2), dtype=float32)')
+    self.assertEqual(repr(x),
+                     '<_DeferredTensor \'x\' shape=(?, 2) dtype=float32>')
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testSimpleNetworkBuilding(self):
+    inputs = base_layers.Input(shape=(32,))
+    if context.in_eager_mode():
+      self.assertIsInstance(inputs, base_layers._DeferredTensor)
+      self.assertEqual(inputs.dtype.name, 'float32')
+      self.assertEqual(inputs.shape.as_list(), [None, 32])
+
+    x = core_layers.Dense(2)(inputs)
+    if context.in_eager_mode():
+      self.assertIsInstance(x, base_layers._DeferredTensor)
+      self.assertEqual(x.dtype.name, 'float32')
+      self.assertEqual(x.shape.as_list(), [None, 2])
+
+    outputs = core_layers.Dense(4)(x)
+    network = base_layers.Network(inputs, outputs)
+    self.assertIsInstance(network, base_layers.Network)
+
+    if context.in_eager_mode():
+      # It should be possible to call such a network on EagerTensors.
+      inputs = constant_op.constant(
+          np.random.random((10, 32)).astype('float32'))
+      outputs = network(inputs)
+      self.assertEqual(outputs.shape.as_list(), [10, 4])
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testMultiIONetworkbuilding(self):
+    input_a = base_layers.Input(shape=(32,))
+    input_b = base_layers.Input(shape=(16,))
+    a = core_layers.Dense(16)(input_a)
+
+    class AddLayer(base_layers.Layer):
+
+      def call(self, inputs):
+        return inputs[0] + inputs[1]
+
+      def _compute_output_shape(self, input_shape):
+        return input_shape[0]
+
+    c = AddLayer()([a, input_b])  # pylint: disable=not-callable
+    c = core_layers.Dense(2)(c)
+
+    network = base_layers.Network([input_a, input_b], [a, c])
+    if context.in_eager_mode():
+      a_val = constant_op.constant(
+          np.random.random((10, 32)).astype('float32'))
+      b_val = constant_op.constant(
+          np.random.random((10, 16)).astype('float32'))
+      outputs = network([a_val, b_val])
+      self.assertEqual(len(outputs), 2)
+      self.assertEqual(outputs[0].shape.as_list(), [10, 16])
+      self.assertEqual(outputs[1].shape.as_list(), [10, 2])
+
 if __name__ == '__main__':
   test.main()
-- 
GitLab


From 703182d854e704ba32770342a2cac28022f7814d Mon Sep 17 00:00:00 2001
From: Mingxing Tan <tanmingxing@google.com>
Date: Fri, 20 Oct 2017 15:33:13 -0700
Subject: [PATCH 633/909] Add performance guide for fused decode_and_crop_jpeg
 optimization.

PiperOrigin-RevId: 172943116
---
 .../docs_src/performance/performance_guide.md | 34 +++++++++++++++++++
 1 file changed, 34 insertions(+)

diff --git a/tensorflow/docs_src/performance/performance_guide.md b/tensorflow/docs_src/performance/performance_guide.md
index 30fb91f9d9..06bb40f64d 100644
--- a/tensorflow/docs_src/performance/performance_guide.md
+++ b/tensorflow/docs_src/performance/performance_guide.md
@@ -87,6 +87,40 @@ the Dataset API is still strongly recommended. Try to avoid the following:
 sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})
 ```
 
+#### Fused decode and crop
+
+If inputs are JPEG images that also require cropping, use fused
+@{tf.image.decode_and_crop_jpeg} to speed up preprocessing.
+`tf.image.decode_and_crop_jpeg` only decodes the part of
+the image within the crop window. This significantly speeds up the process if
+the crop window is much smaller than the full image. For imagenet data, this
+approach could speed up the input pipeline by up to 30%.
+
+Example Usage:
+
+```python
+def _image_preprocess_fn(image_buffer):
+    # image_buffer 1-D string Tensor representing the raw JPEG image buffer.
+
+    # Extract image shape from raw JPEG image buffer.
+    image_shape = tf.image.extract_jpeg_shape(image_buffer)
+
+    # Get a crop window with distorted bounding box.
+    sample_distorted_bounding_box = tf.image.sample_distorted_bounding_box(
+      image_shape, ...)
+    bbox_begin, bbox_size, distort_bbox = sample_distorted_bounding_box
+
+    # Decode and crop image.
+    offset_y, offset_x, _ = tf.unstack(bbox_begin)
+    target_height, target_width, _ = tf.unstack(bbox_size)
+    crop_window = tf.stack([offset_y, offset_x, target_height, target_width])
+    cropped_image = tf.image.decode_and_crop_jpeg(image, crop_window)
+```
+
+`tf.image.decode_and_crop_jpeg` is available on all platforms. There is no speed
+up on Windows due to the use of `libjpeg` vs. `libjpeg-turbo` on other
+platforms.
+
 #### Use large files
 
 Reading large numbers of small files significantly impacts I/O performance.
-- 
GitLab


From 985031a10194c219f7e5f532a703b8b07e85faac Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Fri, 20 Oct 2017 15:35:10 -0700
Subject: [PATCH 634/909] Allows
 tfe.enable_eager_execution(device_policy=tfe.DEVICE_POLICY_WARN).

PiperOrigin-RevId: 172943398
---
 tensorflow/contrib/eager/README.OPENSOURCE.md |  15 -
 tensorflow/contrib/eager/README.md            |  74 +-
 .../contrib/eager/python/examples/BUILD       | 134 ---
 .../eager/python/examples/cart_pole.py        | 282 ------
 .../eager/python/examples/cart_pole_helper.py |  60 --
 .../python/examples/linear_regression.py      | 197 ----
 .../python/examples/notebooks/1_basics.ipynb  | 529 -----------
 .../examples/notebooks/2_gradients.ipynb      | 864 ------------------
 .../examples/notebooks/3_datasets.ipynb       | 218 -----
 .../examples/tests/cart_pole_helper_test.py   |  51 --
 .../python/examples/tests/cart_pole_test.py   | 162 ----
 .../examples/tests/linear_regression_test.py  | 114 ---
 .../eager/python/examples/tests/spinn_test.py | 311 -------
 tensorflow/contrib/eager/python/tfe.py        |   7 +
 tensorflow/python/eager/context.py            |  23 +-
 tensorflow/python/eager/ops_test.py           |  17 +
 tensorflow/python/framework/ops.py            |  32 +-
 tensorflow/python/pywrap_tfe.i                |   6 +
 18 files changed, 92 insertions(+), 3004 deletions(-)
 delete mode 100644 tensorflow/contrib/eager/README.OPENSOURCE.md
 delete mode 100644 tensorflow/contrib/eager/python/examples/BUILD
 delete mode 100644 tensorflow/contrib/eager/python/examples/cart_pole.py
 delete mode 100644 tensorflow/contrib/eager/python/examples/cart_pole_helper.py
 delete mode 100644 tensorflow/contrib/eager/python/examples/linear_regression.py
 delete mode 100644 tensorflow/contrib/eager/python/examples/notebooks/1_basics.ipynb
 delete mode 100644 tensorflow/contrib/eager/python/examples/notebooks/2_gradients.ipynb
 delete mode 100644 tensorflow/contrib/eager/python/examples/notebooks/3_datasets.ipynb
 delete mode 100644 tensorflow/contrib/eager/python/examples/tests/cart_pole_helper_test.py
 delete mode 100644 tensorflow/contrib/eager/python/examples/tests/cart_pole_test.py
 delete mode 100644 tensorflow/contrib/eager/python/examples/tests/linear_regression_test.py
 delete mode 100644 tensorflow/contrib/eager/python/examples/tests/spinn_test.py

diff --git a/tensorflow/contrib/eager/README.OPENSOURCE.md b/tensorflow/contrib/eager/README.OPENSOURCE.md
deleted file mode 100644
index a4a3af08cf..0000000000
--- a/tensorflow/contrib/eager/README.OPENSOURCE.md
+++ /dev/null
@@ -1,15 +0,0 @@
-TensorFlow has many kernels for doing (deep) learning and data manipulation.
-There are typically assembled into computational graphs which can run
-efficiently in a variety of environments.
-
-We are exploring an alternative interaction, where kernels are invoked
-immediately and call this "eager execution". We are hoping to retain the
-benefits of graphs while improving usability with benefits like:
-
-- Immediate error messages and easier debugging
-- Flexibility to use Python datastructures and control flow
-- Reduced boilerplate
-
-Eager execution is under active development.
-There are not many developer-facing materials yet, but stay tuned for updates
-in this directory.
diff --git a/tensorflow/contrib/eager/README.md b/tensorflow/contrib/eager/README.md
index fe577fa7eb..a4a3af08cf 100644
--- a/tensorflow/contrib/eager/README.md
+++ b/tensorflow/contrib/eager/README.md
@@ -1,65 +1,15 @@
-# TensorFlow Eager Execution
+TensorFlow has many kernels for doing (deep) learning and data manipulation.
+There are typically assembled into computational graphs which can run
+efficiently in a variety of environments.
 
-> *WARNING*: This is a preview/pre-alpha version. The API and performance
-> characteristics are subject to change.
+We are exploring an alternative interaction, where kernels are invoked
+immediately and call this "eager execution". We are hoping to retain the
+benefits of graphs while improving usability with benefits like:
 
+- Immediate error messages and easier debugging
+- Flexibility to use Python datastructures and control flow
+- Reduced boilerplate
 
-Eager execution is an experimental interface to TensorFlow that provides an
-imperative programming style (à la [NumPy](http://www.numpy.org)). When you
-enable eager execution, TensorFlow operations execute immediately; you do not
-execute a pre-constructed graph with
-[`Session.run()`](https://www.tensorflow.org/api_docs/python/tf/Session).
-
-For example, consider a simple computation in TensorFlow:
-
-```python
-x = tf.placeholder(tf.float32, shape=[1, 1])
-m = tf.matmul(x, x)
-
-with tf.Session() as sess:
-  print(sess.run(m, feed_dict={x: [[2.]]}))
-
-# Will print [[4.]]
-```
-
-Eager execution makes this much simpler:
-
-```python
-x = [[2.]]
-m = tf.matmul(x, x)
-
-print(m)
-```
-
-## Installation
-
-Since eager execution is not yet part of a TensorFlow release, using it requires
-either [building from source](https://www.tensorflow.org/install/install_sources)
-or the latest nightly builds. The nightly builds are available as:
-
-- [`pip` packages](https://github.com/tensorflow/tensorflow/blob/master/README.md#installation) and
-
-- [docker](https://hub.docker.com/r/tensorflow/tensorflow/) images.
-
-For example, to run the latest nightly docker image:
-
-```sh
-# If you have a GPU, use https://github.com/NVIDIA/nvidia-docker
-nvidia-docker pull tensorflow/tensorflow:nightly-gpu
-nvidia-docker run -it -p 8888:8888 tensorflow/tensorflow:nightly-gpu
-
-# If you do not have a GPU, use the CPU-only image
-docker pull tensorflow/tensorflow:nightly
-docker run -it -p 8888:8888 tensorflow/tensorflow:nightly
-```
-
-And then visit http://localhost:8888 in your browser for a Jupyter notebook
-environment. Try out the notebooks below.
-
-## Documentation
-
-For an introduction to TensorFlow eager execution, see the Jupyter notebooks:
-
-- [Basic Usage](examples/notebooks/1_basics.ipynb)
-- [Gradients](examples/notebooks/2_gradients.ipynb)
-- [Importing Data](examples/notebooks/3_datasets.ipynb)
+Eager execution is under active development.
+There are not many developer-facing materials yet, but stay tuned for updates
+in this directory.
diff --git a/tensorflow/contrib/eager/python/examples/BUILD b/tensorflow/contrib/eager/python/examples/BUILD
deleted file mode 100644
index 3604139819..0000000000
--- a/tensorflow/contrib/eager/python/examples/BUILD
+++ /dev/null
@@ -1,134 +0,0 @@
-# Description:
-#   Open-source examples and tutorials for TensorFlow Eager Execution.
-
-licenses(["notice"])  # Apache 2.0
-
-exports_files(["LICENSE"])
-
-load("//tensorflow:tensorflow.bzl", "py_test")
-load("//tensorflow:tensorflow.bzl", "cuda_py_test")
-
-py_binary(
-    name = "linear_regression",
-    srcs = ["linear_regression.py"],
-    srcs_version = "PY2AND3",
-    deps = [
-        "//tensorflow:tensorflow_py",
-        "//tensorflow/contrib/eager/python:tfe",
-        "@six_archive//:six",
-    ],
-)
-
-py_library(
-    name = "cart_pole_helper",
-    srcs = ["cart_pole_helper.py"],
-    srcs_version = "PY2AND3",
-)
-
-py_library(
-    name = "spinn",
-    srcs = ["spinn.py"],
-    srcs_version = "PY2AND3",
-    deps = [
-        "//tensorflow:tensorflow_py",
-        "//third_party/py/numpy",
-        "@six_archive//:six",
-    ],
-)
-
-py_binary(
-    name = "cart_pole",
-    srcs = ["cart_pole.py"],
-    srcs_version = "PY2AND3",
-    deps = [
-        ":cart_pole_helper",
-        "//tensorflow:tensorflow_py",
-        "//tensorflow/contrib/eager/python:tfe",
-        "@six_archive//:six",
-    ],
-)
-
-py_binary(
-    name = "spinn_prep_data",
-    srcs = ["spinn_prep_data.py"],
-    srcs_version = "PY2AND3",
-    deps = [
-        "//tensorflow:tensorflow_py",
-    ],
-)
-
-py_binary(
-    name = "spinn_train",
-    srcs = ["spinn_train.py"],
-    srcs_version = "PY2AND3",
-    deps = [
-        ":spinn",
-        "//tensorflow:tensorflow_py",
-        "//tensorflow/contrib/eager/python:tfe",
-        "//third_party/py/numpy",
-        "@six_archive//:six",
-    ],
-)
-
-cuda_py_test(
-    name = "linear_regression_test",
-    size = "small",
-    srcs = ["tests/linear_regression_test.py"],
-    additional_deps = [
-        ":linear_regression",
-        "//tensorflow/python/eager:test",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:framework_test_lib",
-    ],
-)
-
-py_test(
-    name = "cart_pole_helper_test",
-    srcs = ["tests/cart_pole_helper_test.py"],
-    srcs_version = "PY2AND3",
-    deps = [
-        ":cart_pole_helper",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:framework_test_lib",
-    ],
-)
-
-cuda_py_test(
-    name = "cart_pole_test",
-    size = "small",
-    srcs = ["tests/cart_pole_test.py"],
-    additional_deps = [
-        ":cart_pole",
-        "//tensorflow/python/eager:test",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:framework",
-        "//tensorflow/python:framework_test_lib",
-        "//tensorflow/python:training",
-    ],
-)
-
-cuda_py_test(
-    name = "spinn_test",
-    size = "medium",
-    srcs = ["tests/spinn_test.py"],
-    additional_deps = [
-        ":spinn",
-        "//third_party/py/numpy",
-        "//tensorflow:tensorflow_py",
-        "//tensorflow/python/eager:test",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:framework_test_lib",
-    ],
-)
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/eager/python/examples/cart_pole.py b/tensorflow/contrib/eager/python/examples/cart_pole.py
deleted file mode 100644
index 56235e4039..0000000000
--- a/tensorflow/contrib/eager/python/examples/cart_pole.py
+++ /dev/null
@@ -1,282 +0,0 @@
-# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-r"""TensorFlow Eager Execution Example: OpenAI Gym CartPole.
-
-Solves the cart-pole problem with policy gradient-based reinforcement learning.
-"""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import argparse
-import sys
-
-import gym
-import numpy as np
-from six.moves import input  # pylint: disable=redefined-builtin
-from six.moves import xrange  # pylint: disable=redefined-builtin
-import tensorflow as tf
-
-from tensorflow.contrib.eager.python import tfe
-from tensorflow.contrib.eager.python.examples import cart_pole_helper
-
-
-class PolicyNetwork(object):
-  """Policy network for the cart-pole reinforcement learning problem.
-
-  The forward path of the network takes an observation from the cart-pole
-  environment (length-4 vector) and outputs an action.
-  """
-
-  def __init__(self, hidden_size, train_logdir=None):
-    """Constructor of PolicyNetwork.
-
-    Args:
-      hidden_size: Size of the hidden layer, as an `int`.
-      train_logdir: The directory in which summaries will be written for
-        TensorBoard during training (optional).
-    """
-    self._hidden_layer = tf.layers.Dense(hidden_size, activation=tf.nn.elu)
-    self._output_layer = tf.layers.Dense(1)
-
-    # Gradient function.
-    self._grad_fn = tfe.implicit_gradients(
-        self._get_cross_entropy_and_save_actions)
-
-    # Support for TensorBoard summaries. Once training has started, use:
-    #   tensorboard --logdir=<train_logdir>
-    self._summary_writer = (tfe.SummaryWriter(train_logdir) if train_logdir
-                            else None)
-
-  def forward(self, inputs):
-    """Given inputs, calculate logits and action.
-
-    Args:
-      inputs: Observations from a step in the cart-pole environment, of shape
-        `(batch_size, input_size)`
-
-    Returns:
-      logits: the logits output by the output layer. This can be viewed as the
-        likelihood vales of choosing the left (0) action. Shape:
-        `(batch_size, 1)`.
-      actions: randomly selected actions ({0, 1}) based on the logits. Shape:
-        `(batch_size, 1)`.
-    """
-    hidden = self._hidden_layer(inputs)
-    logits = self._output_layer(hidden)
-
-    # Probability of selecting the left action.
-    left_p = tf.nn.sigmoid(logits)
-    # Probabilities of selecting the left and right actions.
-    left_right_ps = tf.concat([left_p, 1.0 - left_p], 1)
-    # Randomly-generated actions based on the probabilities.
-    actions = tf.multinomial(tf.log(left_right_ps), 1)
-    return logits, actions
-
-  def _get_cross_entropy_and_save_actions(self, inputs):
-    """Given inputs, get the sigmoid cross entropy and save selection action.
-
-    Args:
-      inputs: Observation from a step in the cart-pole environment.
-
-    Returns:
-      The sigmoid cross-entropy loss given the selected action and logits, based
-        on the assumption that the selected action was rewarded by the
-        environment.
-    """
-    logits, actions = self.forward(inputs)
-
-    # N.B.: This is an important step. We save the value of the `actions` in a
-    # member variable for use with the RL environment. In classic TensorFlow
-    # (non-eager execution), it is less straightfoward to access intermediate
-    # computation results in this manner (c.f., `tf.Session.partial_run()`).
-    self._current_actions = actions
-
-    labels = 1.0 - tf.cast(actions, tf.float32)
-    return tf.nn.sigmoid_cross_entropy_with_logits(labels=labels, logits=logits)
-
-  def train(self,
-            cart_pole_env,
-            optimizer,
-            discount_rate,
-            num_games,
-            max_steps_per_game):
-    """Train the PolicyNetwork by playing `num_games` games in `cart_pole_env`.
-
-    Arguments:
-      cart_pole_env: The cart-pole gym environment object.
-      optimizer: A TensorFlow `Optimizer` object to be used in this training
-        (e.g., `tf.train.AdamOptimizer`).
-      discount_rate: Reward discounting rate.
-      num_games: Number of games to run per parameter update.
-      max_steps_per_game: Maximum number of steps to run in each game.
-
-    Returns:
-      Step counts from all games, as a `list` of `int`.
-    """
-    all_gradient_lists = []
-    all_rewards = []
-    for _ in xrange(num_games):
-      obs = cart_pole_env.reset()
-      game_rewards = []
-      game_gradient_lists = []
-      for _ in xrange(max_steps_per_game):
-        # TODO(cais): Can we save the tf.constant() call?
-        grad_list, var_list = zip(*self._grad_fn(tf.constant([obs])))
-        game_gradient_lists.append(grad_list)
-
-        action = self._current_actions.numpy()[0][0]
-        obs, reward, done, _ = cart_pole_env.step(action)
-        game_rewards.append(reward)
-        if reward != 1.0 or done:
-          break
-
-      all_gradient_lists.append(game_gradient_lists)
-      all_rewards.append(game_rewards)
-
-    normalized_rewards = cart_pole_helper.discount_and_normalize_rewards(
-        all_rewards, discount_rate)
-    all_grads_and_vars = self._scale_and_average_gradients(var_list,
-                                                           all_gradient_lists,
-                                                           normalized_rewards)
-    optimizer.apply_gradients(all_grads_and_vars)
-    step_counts = [len(rewards) for rewards in all_rewards]
-
-    if self._summary_writer:
-      self._summary_writer.scalar("mean_step_count", np.mean(step_counts))
-      self._summary_writer.step()
-
-    return step_counts
-
-  def _scale_and_average_gradients(self,
-                                   variable_list,
-                                   all_gradient_lists,
-                                   normalized_rewards):
-    """Scale gradient tensors with normalized rewards."""
-    num_games = len(all_gradient_lists)
-    grads_and_vars = []
-    for j, var in enumerate(variable_list):
-      scaled_gradients = []
-      for g in xrange(int(num_games)):
-        num_steps = len(all_gradient_lists[g])
-        for s in xrange(num_steps):
-          scaled_gradients.append(
-              all_gradient_lists[g][s][j] * normalized_rewards[g][s])
-      mean_scaled_gradients = sum(scaled_gradients) / len(scaled_gradients)
-      grads_and_vars.append((mean_scaled_gradients, var))
-    return grads_and_vars
-
-  def play(self, cart_pole_env, max_steps=None, render=False):
-    """Play a game in the cart-pole gym environment.
-
-    Args:
-      cart_pole_env: The cart-pole gym environment object.
-      max_steps: Maximum number of steps to run in the game.
-      render: Whether the game state is to be rendered on the screen.
-    """
-    if render:
-      input("\nAbout to play a game with rendering. Press Enter to continue: ")
-
-    steps = 0
-    obs = cart_pole_env.reset()
-    while True:
-      # TODO(cais): Can we save the tf.constant() call?
-      _, actions = self.forward(tf.constant([obs]))
-      if render:
-        cart_pole_env.render()
-      obs, reward, done, _ = cart_pole_env.step(actions.numpy()[0][0])
-      steps += 1
-      if done or reward != 1.0 or max_steps is not None and steps >= max_steps:
-        break
-
-
-def main(_):
-  tf.set_random_seed(0)
-
-  cart_pole_env = gym.make("CartPole-v0")
-  cart_pole_env.seed(0)
-  cart_pole_env.reset()
-
-  device = "gpu:0" if tfe.num_gpus() else "cpu:0"
-  print("Using device: %s" % device)
-
-  with tf.device(device):
-    policy_network = PolicyNetwork(FLAGS.hidden_size, train_logdir=FLAGS.logdir)
-    optimizer = tf.train.AdamOptimizer(FLAGS.learning_rate)
-
-    # Training loop.
-    for i in xrange(FLAGS.num_iterations):
-      step_counts = policy_network.train(
-          cart_pole_env,
-          optimizer,
-          FLAGS.discount_rate,
-          FLAGS.num_games_per_iteration,
-          FLAGS.max_steps_per_game)
-      print("Iteration %d: step counts = %s; mean = %g" % (
-          i, step_counts, np.mean(step_counts)))
-      sys.stdout.flush()
-
-    # Optional playing after training, with rendering.
-    if FLAGS.play_after_training:
-      policy_network.play(cart_pole_env,
-                          max_steps=FLAGS.max_steps_per_game,
-                          render=True)
-
-
-if __name__ == "__main__":
-  parser = argparse.ArgumentParser()
-  parser.add_argument(
-      "--hidden_size",
-      type=int,
-      default=5,
-      help="Size of the hidden layer of the policy network.")
-  parser.add_argument(
-      "--discount_rate",
-      type=float,
-      default=0.95,
-      help="Reward discounting rate.")
-  parser.add_argument(
-      "--learning_rate",
-      type=float,
-      default=0.05,
-      help="Learning rate to be used during training.")
-  parser.add_argument(
-      "--num_iterations",
-      type=int,
-      default=100,
-      help="Number of training iterations.")
-  parser.add_argument(
-      "--num_games_per_iteration",
-      type=int,
-      default=20,
-      help="Number of games to run in each training iteration.")
-  parser.add_argument(
-      "--max_steps_per_game",
-      type=int,
-      default=1000,
-      help="Maximum number of steps to run in each game.")
-  parser.add_argument(
-      "--logdir",
-      type=str,
-      default=None,
-      help="logdir in which TensorBoard summaries will be written (optional).")
-  parser.add_argument(
-      "--play_after_training",
-      action="store_true",
-      help="Play a game after training (with rendering).")
-
-  FLAGS, unparsed = parser.parse_known_args()
-  tfe.run(main=main, argv=[sys.argv[0]] + unparsed)
diff --git a/tensorflow/contrib/eager/python/examples/cart_pole_helper.py b/tensorflow/contrib/eager/python/examples/cart_pole_helper.py
deleted file mode 100644
index 1b80f90165..0000000000
--- a/tensorflow/contrib/eager/python/examples/cart_pole_helper.py
+++ /dev/null
@@ -1,60 +0,0 @@
-# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Helper functions for reinforcement learning in the cart-pole problem."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import numpy as np
-
-
-def discount_rewards(rewards, discount_rate):
-  """Discout reward values with discount rate.
-
-  Args:
-    rewards: A sequence of reward values in time.
-    discount_rate: (`float`) reward discounting rate (e.g., 0.95).
-
-  Returns:
-    Discounted reward values.
-  """
-  discounted = []
-  for reward in reversed(rewards):
-    discounted.append(
-        (discounted[-1] if discounted else 0.0) * discount_rate + reward)
-  return list(reversed(discounted))
-
-
-def discount_and_normalize_rewards(reward_sequences, discount_rate):
-  """Perform discounting on a number of reward sequences; then normalize values.
-
-  Args:
-    reward_sequences: an `iterable` of reward sequences.
-    discount_rate: reward discounting rate (e.g., 0.95).
-
-  Returns:
-    A `list` of reward value `list`s, discounted and normalized.
-  """
-  discounted = []
-  for sequence in reward_sequences:
-    discounted.append(discount_rewards(sequence, discount_rate))
-  discounted = np.array(discounted)
-
-  # Compute overall mean and stddev.
-  flattened = np.concatenate(discounted)
-  mean = np.mean(flattened)
-  std = np.std(flattened)
-  return [((d - mean) / std) for d in discounted]
diff --git a/tensorflow/contrib/eager/python/examples/linear_regression.py b/tensorflow/contrib/eager/python/examples/linear_regression.py
deleted file mode 100644
index 538d6d4225..0000000000
--- a/tensorflow/contrib/eager/python/examples/linear_regression.py
+++ /dev/null
@@ -1,197 +0,0 @@
-# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-# pylint: disable=line-too-long
-r"""TensorFlow Eager Execution Example: Linear Regression.
-
-This example shows how to use TensorFlow Eager Execution to fit a simple linear
-regression model using some synthesized data. Specifically, it illustrates how
-to define the forward path of the linear model and the loss function, as well
-as how to obtain the gradients of the loss function with respect to the
-variables and update the variables with the gradients.
-"""
-# pylint: enable=line-too-long
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import argparse
-import sys
-
-import numpy as np
-from six.moves import xrange  # pylint: disable=redefined-builtin
-import tensorflow as tf
-
-# TODO(cais): Use tf.contrib.eager namespace when ready.
-from tensorflow.contrib.eager.python import tfe
-
-
-class DataGenerator(object):
-  """Generates synthetic data for linear regression."""
-
-  def __init__(self, w, b, noise_level, batch_size):
-    self._w = w
-    self._b = b
-    self._noise_level = noise_level
-    self._batch_size = batch_size
-    self._ndims = w.shape[0]
-
-  def next_batch(self):
-    """Generate a synthetic batch of xs and ys."""
-    xs = tf.random_normal([self._batch_size, self._ndims])
-    ys = (tf.matmul(xs, self._w) + self._b +
-          self._noise_level * tf.random_normal([self._batch_size, 1]))
-    return xs, ys
-
-
-class LinearModel(object):
-  """A TensorFlow linear regression model.
-
-  Uses TensorFlow's eager execution.
-
-  For those familiar with TensorFlow graphs, notice the absence of
-  `tf.Session`. The `forward()` method here immediately executes and
-  returns output values. The `loss()` method immediately compares the
-  output of `forward()` with the target adn returns the MSE loss value.
-  The `fit()` performs gradient-descent training on the model's weights
-  and bias.
-  """
-
-  def __init__(self):
-    """Constructs a LinearModel object."""
-    self._hidden_layer = tf.layers.Dense(1)
-
-    # loss_value_and_grad_fn is a function that when invoked, will return the
-    # loss value and the gradients of loss with respect to the variables. It has
-    # the same input arguments as `self.loss()`.
-    self._loss_value_and_grad_fn = tfe.implicit_value_and_gradients(self.loss)
-
-  @property
-  def weights(self):
-    """Get values of weights as a numpy array."""
-    return self._hidden_layer.variables[0].read_value().numpy()
-
-  @property
-  def biases(self):
-    """Get values of biases as a numpy array."""
-    return self._hidden_layer.variables[1].read_value().numpy()
-
-  def forward(self, xs):
-    """Invoke the linear model.
-
-    Args:
-      xs: input features, as a tensor of size [batch_size, ndims].
-
-    Returns:
-      ys: the predictions of the linear mode, as a tensor of size [batch_size]
-    """
-    # Note: Unlike classic TensorFlow, operations such as self._hidden_layer
-    # will execute the underlying computation immediately.
-    return self._hidden_layer(xs)
-
-  def loss(self, xs, ys):
-    """Loss of the linear model.
-
-    Args:
-      xs: input features, as a tensor of size [batch_size, ndims].
-      ys: the target values of y, as a tensor of size [batch_size].
-
-    Returns:
-      The mean square error loss value.
-    """
-    return tf.reduce_mean(tf.square(self.forward(xs) - ys))
-
-  def fit(self,
-          batch_fn,
-          optimizer,
-          num_iters,
-          verbose=False,
-          logdir=None):
-    """Fit the linear-regression model.
-
-    Args:
-      batch_fn: A function, which when called without any arguments, returns a
-        batch of xs and ys for training.
-      optimizer: The TensorFlow Optimizer object to be used.
-      num_iters: Number of training iterations to perform.
-      verbose: If true, will print out loss values at every iteration.
-      logdir: The directory in which summaries will be written for TensorBoard
-        (optional).
-    """
-    if logdir:
-      # Support for TensorBoard summaries. Once training has started, use:
-      #   tensorboard --logdir=<logdir>
-      summary_writer = tfe.SummaryWriter(logdir)
-
-    # Training loop.
-    for i in xrange(num_iters):
-      # Generate a (mini-)batch of data for training.
-      xs, ys = batch_fn()
-
-      # Call the function obtained above to get the loss and gradient values at
-      # the specific training batch. The function has the same input arguments
-      # as the forward function, i.e., `linear_loss()`.
-      loss_value, grads_and_vars = self._loss_value_and_grad_fn(xs, ys)
-      if verbose:
-        print("Iteration %d: loss = %s" % (i, loss_value.numpy()))
-
-      # Send the gradients to the optimizer and update the Variables, i.e., `w`
-      # and `b`.
-      optimizer.apply_gradients(grads_and_vars)
-
-      if logdir:
-        summary_writer.scalar("loss", loss_value)
-        summary_writer.step()
-
-
-def main(_):
-  # Ground-truth constants.
-  true_w = np.array([[-2.0], [4.0], [1.0]], dtype=np.float32)
-  true_b = np.array([0.5], dtype=np.float32)
-  noise_level = 0.01
-
-  # Training constants.
-  batch_size = 64
-  learning_rate = 0.1
-  num_iters = 20
-
-  print("True w: %s" % true_w)
-  print("True b: %s\n" % true_b)
-
-  device = "gpu:0" if tfe.num_gpus() else "cpu:0"
-  print("Using device: %s" % device)
-  with tf.device(device):
-    linear_model = LinearModel()
-
-    optimizer = tf.train.GradientDescentOptimizer(learning_rate)
-    data_gen = DataGenerator(true_w, true_b, noise_level, batch_size)
-    linear_model.fit(data_gen.next_batch, optimizer, num_iters, verbose=True,
-                     logdir=FLAGS.logdir)
-
-  print("\nAfter training: w = %s" % linear_model.weights)
-  print("\nAfter training: b = %s" % linear_model.biases)
-
-
-if __name__ == "__main__":
-  parser = argparse.ArgumentParser()
-  parser.add_argument(
-      "--logdir",
-      type=str,
-      default=None,
-      help="logdir in which TensorBoard summaries will be written (optional).")
-  FLAGS, unparsed = parser.parse_known_args()
-
-  # Use tfe.run() instead of tf.app.run() for eager execution.
-  tfe.run(main=main, argv=[sys.argv[0]] + unparsed)
diff --git a/tensorflow/contrib/eager/python/examples/notebooks/1_basics.ipynb b/tensorflow/contrib/eager/python/examples/notebooks/1_basics.ipynb
deleted file mode 100644
index 9c2e6f15b4..0000000000
--- a/tensorflow/contrib/eager/python/examples/notebooks/1_basics.ipynb
+++ /dev/null
@@ -1,529 +0,0 @@
-{
-  "cells": [
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "U9i2Dsh-ziXr"
-      },
-      "source": [
-        "# Eager Execution Tutorial: Basics\n",
-        "\n",
-        "This notebook introduces the basics of using TensorFlow's eager execution capabilities. It covers concepts such as:\n",
-        "\n",
-        "* Importing required packages\n",
-        "* Enabling eager execution\n",
-        "* Creating and using TensorFlow Tensors and Variables\n",
-        "* Using TensorFlow interactively\n",
-        "* Using GPUs with eager execution enabled\n",
-        "\n",
-        "This notebook does *not* cover modeling topics, such as gradients."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "z1JcS5iBXMRO"
-      },
-      "source": [
-        "# Step 1: Import Eager\n",
-        "\n",
-        "The key imports for eager execution are the following:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "cellView": "code",
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          }
-        },
-        "colab_type": "code",
-        "id": "RlIWhyeLoYnG"
-      },
-      "outputs": [],
-      "source": [
-        "# Import TensorFlow.\n",
-        "import tensorflow as tf\n",
-        "\n",
-        "# Import TensorFlow eager execution support (subject to future changes).\n",
-        "from tensorflow.contrib.eager.python import tfe"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "H9UySOPLXdaw"
-      },
-      "source": [
-        "# Step 2: Enable eager execution\n",
-        "\n",
-        "All future TensorFlow calls will execute the\n",
-        "underlying TensorFlow ops immediately:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "cellView": "code",
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          }
-        },
-        "colab_type": "code",
-        "id": "WPTUfGq6kJ5w"
-      },
-      "outputs": [],
-      "source": [
-        "tfe.enable_eager_execution()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "twBfWd5xyu_d"
-      },
-      "source": [
-        "# Step 3: Interactively Use TensorFlow!\n",
-        "\n",
-        "Now you can call TensorFlow functions and get results, immediately! No more `tf.Sessions`!\n",
-        "\n",
-        "TensorFlow will automatically wrap native Python types for you with operator overloading for TensorFlow Tensors."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "cellView": "code",
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          }
-        },
-        "colab_type": "code",
-        "id": "ngUe237Wt48W"
-      },
-      "outputs": [],
-      "source": [
-        "print(tf.add(1, 2))\n",
-        "print(tf.add([1, 2], [3, 4]))\n",
-        "print(tf.square(5))\n",
-        "print(tf.reduce_sum([1, 2, 3]))\n",
-        "print(tf.encode_base64(\"hello world\"))\n",
-        "print(\"\")\n",
-        "\n",
-        "x = tf.constant(2)\n",
-        "y = tf.constant(3)\n",
-        "print(x * y + 1)\n",
-        "\n",
-        "# Most TensorFlow ops are directly usable with eager execution, giving\n",
-        "# results immediately.\n",
-        "print(tf.contrib.signal.hamming_window(x * y + 1))"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "IDY4WsYRhP81"
-      },
-      "source": [
-        "Numpy arrays are supported, too:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          }
-        },
-        "colab_type": "code",
-        "id": "lCUWzso6mbqR"
-      },
-      "outputs": [],
-      "source": [
-        "import numpy as np\n",
-        "\n",
-        "ones = np.ones([3, 3])\n",
-        "\n",
-        "print(\"numpy 3x3 matrix of 1s:\")\n",
-        "print(ones)\n",
-        "print(\"\")\n",
-        "\n",
-        "print(\"Multiplied by 42:\")\n",
-        "print(tf.multiply(ones, 42))"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "PBNP8yTRfu_X"
-      },
-      "source": [
-        "# Step 4: Define and Print TensorFlow Variables\n",
-        "\n",
-        "To define TensorFlow variables, use the `get_variable()` function as follows:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "cellView": "code",
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          }
-        },
-        "colab_type": "code",
-        "id": "3Twf_Rw-gQFM"
-      },
-      "outputs": [],
-      "source": [
-        "x = tf.get_variable(name=\"x\", shape=[1], dtype=tf.float32, initializer=tf.zeros_initializer)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "45G7094TxsMb"
-      },
-      "source": [
-        "## Printing TensorFlow Variables"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "cellView": "code",
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          }
-        },
-        "colab_type": "code",
-        "id": "UJBJeZ5XxuwA"
-      },
-      "outputs": [],
-      "source": [
-        "# This does NOT print the Variable's actual value:\n",
-        "print(\"Printing a TensorFlow Variable:\")\n",
-        "print(x)\n",
-        "print(\"\")\n",
-        "\n",
-        "# A TensorFlow variable represents a reference to a tensor.\n",
-        "# The `read_value()` method provides access to the current value of the\n",
-        "# variable. Tensorflow Variables are automatically initialized according to the\n",
-        "# semantics defined in tf.get_variable().\n",
-        "print(\"Printing a TensorFlow Variable's value using .read_value():\")\n",
-        "print(x.read_value())\n",
-        "print(\"\")\n",
-        "\n",
-        "print(\"Printing a TensorFlow Variable's value using .read_value().numpy():\")\n",
-        "print(x.read_value().numpy())"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "2njjWHcTpBEn"
-      },
-      "source": [
-        "## Changing a TensorFlow Variable's value\n",
-        "\n",
-        "To change a TensorFlow Variable's value, use its `.assign()` or `.assign_add()` method:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          }
-        },
-        "colab_type": "code",
-        "id": "v3wr6Erbo_hB"
-      },
-      "outputs": [],
-      "source": [
-        "x.assign(42)\n",
-        "print(x.read_value())\n",
-        "\n",
-        "x.assign_add(3)\n",
-        "print(x.read_value())"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "uhtynjHVpTB5"
-      },
-      "source": [
-        "## Use a Variable just like any other Tensor"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          }
-        },
-        "colab_type": "code",
-        "id": "7PbktdnHoehR"
-      },
-      "outputs": [],
-      "source": [
-        "print(x + 3)\n",
-        "\n",
-        "# This code will broadcast the value across the list of numbers:\n",
-        "print(x * [1, 2, 4])"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "GVChqwlwy1SI"
-      },
-      "source": [
-        "# Step 5: Debug Errors with Instant Feedback\n",
-        "\n",
-        "TensorFlow's eager execution helps you identify and debug runtime issues through interactive exploration of code snippets.\n",
-        "\n",
-        "Below, we'll define a length-4 vector, and attempt two `tf.slice()` operations,\n",
-        "one being legal and the other being illegal, leading to a runtime error that is\n",
-        "raised immediately."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "cellView": "code",
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          }
-        },
-        "colab_type": "code",
-        "id": "23ap04N0v4k0"
-      },
-      "outputs": [],
-      "source": [
-        "vector = tf.constant([10.0, 20.0, 30.0, 40.0])"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "cellView": "code",
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          }
-        },
-        "colab_type": "code",
-        "id": "FCUMsIYxxRRa"
-      },
-      "outputs": [],
-      "source": [
-        "# Works, because the values of `begin` and `size` (the 2nd and 3rd input\n",
-        "# arguments) are within the bound of `vector`.\n",
-        "print(tf.slice(vector, [1], [3]))"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "cellView": "code",
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          }
-        },
-        "colab_type": "code",
-        "id": "T8me2oCNxpFp"
-      },
-      "outputs": [],
-      "source": [
-        "# The following does NOT work, because the value of `size` (the 3rd\n",
-        "# argument) causes the indices to go out of the bounds of `vector`. The\n",
-        "# error is raised immediately.\n",
-        "try:\n",
-        "  print(tf.slice(vector, [1], [4]))\n",
-        "except tf.OpError as e:\n",
-        "  print(\"Caught error: %s\" % e)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "irxJhAgar84v"
-      },
-      "source": [
-        "# Step 6: Using the GPU\n",
-        "\n",
-        "You can place Tensors on the GPU by calling a Tensor's `.gpu()` method.\n",
-        "\n",
-        "The first operation executing on the GPU may be slow as TensorFlow initializes. Subsequent uses will be much faster."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          }
-        },
-        "colab_type": "code",
-        "id": "7J4N9baqaKCL"
-      },
-      "outputs": [],
-      "source": [
-        "# The example code from here on will work only if your notebook\n",
-        "# is running on a machine with a functional CUDA GPU. The following\n",
-        "# line checks that.\n",
-        "is_gpu_available = tfe.num_gpus() \u003e 0\n",
-        "\n",
-        "# Create some Tensors\n",
-        "SIZE = 1000\n",
-        "cpu_tensor = tf.random_normal([SIZE, SIZE])\n",
-        "\n",
-        "if is_gpu_available:\n",
-        "  gpu_tensor = cpu_tensor.gpu()"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          }
-        },
-        "colab_type": "code",
-        "id": "4E-2n7VbzY1n"
-      },
-      "outputs": [],
-      "source": [
-        "# Time a CPU-based matrix multiplication\n",
-        "\n",
-        "print(\"Time to conduct matmul on CPU:\")\n",
-        "%time tf.matmul(cpu_tensor, cpu_tensor)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          }
-        },
-        "colab_type": "code",
-        "id": "vbSFW-T5zhZF"
-      },
-      "outputs": [],
-      "source": [
-        "# Time GPU-based matrix multiplications.\n",
-        "\n",
-        "if is_gpu_available:\n",
-        "  # First use of the GPU will be slow:\n",
-        "  print(\"Time to conduct first matmul on GPU:\")\n",
-        "  %time tf.matmul(gpu_tensor, gpu_tensor)\n",
-        "  print()\n",
-        "\n",
-        "  # Subsequent uses are much faster:\n",
-        "  print(\"Time to conduct second matmul on GPU:\")\n",
-        "  %time tf.matmul(gpu_tensor, gpu_tensor)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          }
-        },
-        "colab_type": "code",
-        "id": "E5pIOe3Rz7iW"
-      },
-      "outputs": [],
-      "source": [
-        "# Second timing demo for GPUs, after it has been used once:\n",
-        "\n",
-        "cpu_tensor = tf.random_normal([SIZE, SIZE])\n",
-        "print(\"Time to conduct CPU matmul:\")\n",
-        "%time tf.matmul(cpu_tensor, cpu_tensor)\n",
-        "print()\n",
-        "\n",
-        "if is_gpu_available:\n",
-        "  gpu_tensor = cpu_tensor.gpu()\n",
-        "  print(\"Time to conduct GPU matmul:\")\n",
-        "  %time tf.matmul(gpu_tensor, gpu_tensor)"
-      ]
-    }
-  ],
-  "metadata": {
-    "colab": {
-      "default_view": {},
-      "name": "Eager Execution Tutorial: Basics",
-      "provenance": [
-        {
-          "file_id": "0B0kLcpwLFwKEVm9XNkFueGk4bTg",
-          "timestamp": 1504118841551
-        }
-      ],
-      "version": "0.3.2",
-      "views": {}
-    }
-  },
-  "nbformat": 4,
-  "nbformat_minor": 0
-}
diff --git a/tensorflow/contrib/eager/python/examples/notebooks/2_gradients.ipynb b/tensorflow/contrib/eager/python/examples/notebooks/2_gradients.ipynb
deleted file mode 100644
index 5e0ec5cf8a..0000000000
--- a/tensorflow/contrib/eager/python/examples/notebooks/2_gradients.ipynb
+++ /dev/null
@@ -1,864 +0,0 @@
-{
-  "cells": [
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "vDJ4XzMqodTy"
-      },
-      "source": [
-        "# Eager Execution: Working with Gradients\n",
-        "\n",
-        "This notebook demonstrates:\n",
-        "\n",
-        "* How to get gradients using TensorFlow's eager execution capabilities\n",
-        "* How to apply the gradients so you can update your variables"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "GQJysDM__Qb0"
-      },
-      "source": [
-        "# Setup: Import eager and enable eager execution.\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          }
-        },
-        "colab_type": "code",
-        "id": "OiMPZStlibBv"
-      },
-      "outputs": [],
-      "source": [
-        "# Import TensorFlow.\n",
-        "import tensorflow as tf\n",
-        "\n",
-        "# Import TensorFlow eager execution support (subject to future changes).\n",
-        "from tensorflow.contrib.eager.python import tfe\n",
-        "\n",
-        "# Enable eager execution.\n",
-        "tfe.enable_eager_execution()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "1CLWJl0QliB0"
-      },
-      "source": [
-        "# Fitting a Simple Linear Model"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "-39gouo7mtgu"
-      },
-      "source": [
-        "## Step 1: Synthesize some data\n",
-        "\n",
-        "To demonstrate fitting a model with TensorFlow's eager execution, we'll fit a linear model to some synthesized data (which includes some noise).\n",
-        "\n",
-        "In the code, we  use the variable names `w` and `b` to represent the single weight and bias we'll use to fit our model."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "cellView": "code",
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          }
-        },
-        "colab_type": "code",
-        "id": "rQsdCg9PfIL-"
-      },
-      "outputs": [],
-      "source": [
-        "# The constants we'll try to fit our variables to:\n",
-        "true_w = 3\n",
-        "true_b = 2\n",
-        "\n",
-        "NUM_EXAMPLES = 1000\n",
-        "\n",
-        "# Our inputs:\n",
-        "inputs = tf.random_normal(shape=[NUM_EXAMPLES, 1])\n",
-        "\n",
-        "# Our labels, with noise:\n",
-        "noise = tf.random_normal(shape=[NUM_EXAMPLES, 1])\n",
-        "labels = inputs * true_w + true_b + noise"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 3,
-      "metadata": {
-        "cellView": "code",
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          },
-          "height": 360,
-          "output_extras": [
-            {
-              "item_id": 1
-            }
-          ]
-        },
-        "colab_type": "code",
-        "executionInfo": {
-          "elapsed": 127,
-          "status": "ok",
-          "timestamp": 1505502830690,
-          "user": {
-            "displayName": "",
-            "photoUrl": "",
-            "userId": ""
-          },
-          "user_tz": 240
-        },
-        "id": "O4lsC4ckAcar",
-        "outputId": "2f760690-cafb-4777-b970-91d839f99faf"
-      },
-      "outputs": [
-        {
-          "data": {
-            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAesAAAFXCAYAAACC+2avAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzsnXt8VPWd99+TK7kykxtJQIebqZfaqogtrhKNa1ooEKl9\nCrpVn9ZNW6x9VWsbCi7aVUt01NZ9tq21KVZlFey2YkQNohhj3QWK2liCF5RIBCc3yEwmIZnMTOY8\nf/zmzJwzSSBAYibh+369eIU5c87vXLh8zvdu0TRNQxAEQRCEmCVurC9AEARBEISjI2ItCIIgCDGO\niLUgCIIgxDgi1oIgCIIQ44hYC4IgCEKMI2ItCIIgCDHOiIj16tWrufjii1m8eHF4269//Wvmz5/P\n0qVLWbp0Ka+//vpInEoQBEEQTjksI1Fn/eabb5KWlkZFRQWbN28GlFinpaXx7W9/+6QvUhAEQRBO\nZUbEsr7wwgvJzMwcsF36rQiCIAjCyTOqMesnn3ySsrIybr/9drq6ukbzVIIgCIIwYRk1sb722mt5\n5ZVXqK6uJicnh8rKytE6lSAIgiBMaEZNrLOysrBYLAB885vfZPfu3cc8RtzmgiAIgjCQhJFaKFpo\n29vbyc3NBeDll1+mqKjomGtYLBba2yeuuzw3N0Pubxwzke9vIt8byP2Nd06F+zsWIyLWt912Gzt3\n7sTtdnPZZZfxwx/+kJ07d/Lee+8RFxfH1KlTueuuu0biVIIgCIJwyjEiYv3ggw8O2Hb11VePxNKC\nIAiCcMojHcwEQRAEIcYRsRYEQRCEGEfEWhAEQRBiHBFrQRAEQYhxRKwFQRAEIcYRsRYEQRCEGEfE\nWhAEQRBiHBFrQRAEQYhxRKwFQRAEIcYRsRYEQRCEGEfEWhAEQRBiHBFrQRAEQYhxRKwFQRAEIcYR\nsRYEQRCEGEfEWhAEQRBiHBFrQRAEQYhxRKwFQRAEIcYRsRYEQRCEGCdhrC9AEARBOHXo6HCzcmUt\nTU2Z2O2dOBwl2GzWsb6smEfEWhAEQfjMWLmylurq6wAL9fUasJ6qqqVjfVkxj7jBBUEQhM+MpqZM\nwBL6ZAl9Fo6FiLUgCILwmWG3dwJa6JOG3e4Zy8sZN4gbXBAEQfjMcDhKgPWhmLUHh+Pysb6kcYGI\ntSAIgvCZYbNZJUZ9AogbXBAEQRBiHBFrQRAEQYhxRKwFQRAEIcYRsRYEQRCEGEfEWhAEQRBiHBFr\nQRAEQYhxRKwFQRAEIcYRsRYEQRCEGEfEWhAEQRBiHBFrQRAEQYhxRKwFQRAEIcYRsRYEQRCEGEfE\nWhAEQRBiHBFrQRAEQYhxRKwFQRAEIcYRsRYEQRCEGEfEWhAEQRBiHBFrQRAEQYhxRKwFQRAEIcYR\nsRYEQRCEGEfEWhAEQRBiHBFrQRAEQYhxRKwFQRAEIcYRsRYEQRCEGEfEWhAEQRBiHBFrQRAEQYhx\nRKwFQRAEIcYRsRYEQRCEGCdhrC9AEARBODE6OtysXFmL02mjsLADh6MEm806rGOamjKx2zuHdYww\n9oyIWK9evZrXXnuN7OxsNm/eDEBnZye33norn376KdOmTeOhhx4iIyNjJE4nCIIgACtX1lJdfR1g\nATRgPVVVS037RIuzz9dDTc33AQv19YMfI8QeI+IG//rXv866detM237/+98zb948XnrpJb70pS/x\nyCOPjMSpBEEQhBBNTZkooQawhD6b0QW9vv4qqquvZ/v27mMeI8QeIyLWF154IZmZ5j/wbdu2sXSp\neltbunQpr7zyykicShAEQQhht3eiLGoADbvdM2CfaEGH7GMeI8Qeoxaz7ujoICcnB4Dc3FxcLtdo\nnUoQBOGUxOEoAdaHYtYuHI7LAbPru61tD1AM2AAXkyY5sVr/CBxi3rwMHI5FY3cDwrCJuQSz3NyJ\nHdeW+xvfTOT7m8j3BhPz/uLi+klOTgQgOTmBnJwMsrIyuPnm5w2x7DKmTbuPgoJzaG7ew8GDt6PH\nuDMyNpKdncFNNz3Pxx+nM2NGFw8/vJCsrNhLOJuIf37Hw6iJdXZ2NocOHSInJ4f29naysrKGdVx7\ne9doXdKYk5ubIfc3jpnI9zeR7w0m7v2Vlz8XFuVduzT6+lSy2N69KRhd3zk5Z/LCC5dRWtrPwYOR\n7Xv3pnDjjYOvEUtM1D8/neG8iIxYnbWmaabPJSUlPPPMMwBs2rSJK664YqROJQiCIDB0gtlQsezB\ntg8nSU0Ye0bEsr7tttvYuXMnbrebyy67jB/+8Id897vf5Uc/+hF/+ctfKCws5D/+4z9G4lSCIAhC\nCLu9M1R+pdzauijrsWxVruUJx7JXrZrDrl2VuFzTsNkOsnr1EtaufWvQNYTYYkTE+sEHHxx0+2OP\nPTYSywuCIAiDMFSCmc1mHdSVXVn5Nk7nKsBCb6/G2rXrhxR2IbaIuQQzQRAEYXjoojxYTHewTmWD\nubyHEnYhthCxFgRBmIAYu5vpncrsdk1c3uMUEWtBEIQYYai+3SfSz3swK/rpp+cgLu/xiYi1IAhC\njDCYNVxVtXTI7UdjsOQzcXmPX0SsBUEQYoShyqhOpLxKEscmFiLWgiAIMcJQpVjm7S7a2t6ltJSw\nS3ywphojYUXLOM3YQcRaEAQhRhjKGjZub2t7F6dzFU6nconX1T1AaelU7r770mEL6XBF+ETc78Lo\nIGItCIIQIwxlDRu3l5aC0xlxibvdZ/KnPy06rjahwxVh6W4WO4xYu1FBEARh9IluGQpqPvXxCOlw\nRXg4IziFzwaxrAVBEMYRuku8ttaPx5MCLAQ0CgoODXuNoWLjQ51LktTGHhFrQRCEcYTuEr/hhv+i\npiYBeBY4xNtvu3G53ANiz4PFp4crwlLqFTuIWAuCIIxDmpsLgF5gOWChtVWjomJg7Hmo+LSI8PhC\nxFoQBGEcEG0hFxT4qK+fwrFiz5IkNjGQBDNBEITPiI4ON+Xlmygt3UZ5+TO4XO5hf69byPX1V1Fd\nfT0QoLBwN8dKAJMksYmBWNaCIAifEdEu6V27KqmtvS4cZz5aSVW0hdzcXEBt7SIqKgaOyDQiSWIT\nAxFrQRCEz4howXU6P09FRe2Qgmx0WRcUNFNf/xSQAXgoKPAcdUSmznCTxKRbWWwjbnBBEIRBOJbL\n+kQwu6RdwLts3Up4/aO7rBOBa4DFwLWhzyNHtJu9oqJ2RNcXTg6xrAVBEAZhNFptOhwl7NpVidP5\neeBdYCW9vRaqq9X6DkcJfX3r2LEjDjiMz5cWLsdqbs7B7AbPOalriUYS0WIbsawFQRAG4XjFaziW\nuM1mpbb2OsrK3KSkFA5Y32azkpychNv9bdzun1JTsyJs4UZb3QUFLeHzLVv21Elb/pKIFtuIZS0I\ngjAIw+3ypROxxDupr3+RurqXKS6OHxD71WPI5eXPhCxq8/pDvSREJ4r5fAkmy/94eoMb0WPV+/Yl\nUFhYSXZ2ETNn9kgiWowhYi0IgjAIx5tFHRHZGmABbvcWqqvT2LXrCWprrx+QrOVwlODzPcL27V1A\nNj5ffzhuPdhLQnSiWGnpNoZr+R8teczo7geNuXNlslYsImItCIIwCMfbajMisunAFvTOYk7n4kE7\ni9lsVpKSUnG7vwdYqKnRSEpaf9SXBKPotrXtAcoYjuV/PCVhEquOTUSsBUEQBmEoa3So7brI1tW1\n4HafyXAEcDChPNpLgtkKLqawsJK8vLMpKurl7ruHtvyPJsjH6+4XxgYRa0EQhEEYyhodarsusi6X\nm8svfwKnczFDCaAu+Pv3t6CSuoYnlGbRtZGXdzZbt15Bbm4GH3xwgPLyTYO6uo8myNI0ZXwgYi0I\nQswylo06IsLoBmrC9dD79iUQbaV2dLi55ZaXQiVXh5gzJ5kvfnEdzc052O0eVq26wCSkPl8PNTXf\nBzqBDVitXoqLE44plEcT3aO5uo8myDJZa3wgYi0IQswyGrXOwyXSMcwJ3Bauhy4srCTaGl65spYt\nW24Mb9u2bQNlZQG2br0CgPLyTab7sFofCO1rBa5l+vRnqaq6Ilz+NdTLiS66+/bF09HRRGNjEeXl\nz/Doo2VHdXWLII9/RKwFQYhZxjb5Se8Y9rzpGrKzi5g7V1mpBQUt+HwJvPZaErABWIgS4AyamvrD\nK0XfB2SjOphtAdJoa9uDyzXnmC8nuuhef/3TNDSswum0sHu3xo03PoHdjsSeJzAi1oIgxCxjmfwU\n6RjWhdGSnjmzJyygRotZ7fMgUAD00NbWTmkphnGWkTXmzQvyzjsP43SuwpgxPtyXE+Vuj+xXV6ex\nY8cVSOx54iJiLQhCzDKWyU+RF4WFDBVXHmgxfw5YRHLy7TidP8XptFFfr7Fgwe8oKzPex1dYtuwt\nnM7IsVu3gs023HKsQxhfIOCQuLonOCLWgiDELCMtQEdLWIv+bvXqOUReFAI4HFcOSG6LtvyhO/T7\n01Eu7nSgiwMHMnn11SVHPba3N5He3psoLKwkK6uIjo697Ntnp7z8mQGx63nz0qmp2YCawNXF/Pky\nHWuiI2ItCMIpw2Ax4fvuu5yVK2upqwvgdicDl1FfP5nBktmGEvTaWj8eTwrKCteAg4BqdgIaHR2V\nA65F9xps3Qq9vX7Uf8dv0NOTwFlnfUJDw3SczgwaGjz4fM/z+OPfCl8DJGG1eoGDzJuXwaOPXkN/\n/4BTCBMIEWtBEE4ZBosJR7fbhI3ANYPGi4dKALvhhv+ipkYD/gDk4PMlocqyAGpwuQoHWMjmHuEp\nqGQ2C273Il5//U7g1vA1bd/+AKCEuqRkfTjWDarrWVaWdch51sLEQMRaEIRThsES1gbGndOJjhfr\nFvXWrRj27WTz5oMUFf03gcCh0PbbAQuapqGywy3ActMYzGhr3eEooa7uZdzuyDX090+PuqZsQL0s\nqPGa0h70VEPEWhCEU4bBEtYqKl41CbjV+j7FxS5TIlnEot5AJLHrRYLBVSGR1YDHMQusDzWF+OjC\narNZ+fKX+9myJXINOTkHaWszZ4+D7hno5ni6ngkTAxFrQRBOGWw2azhG3dSUSUXFq1GJZB4cjuUD\nEsn27YtHucctwL1YLFPQNLMQQzvmDG0L8Klp2/vvv0lJyRFmzQqYXOIWSwD1IqASxs49N430dHP2\nOOiegSWha0mjsLABh+O6UXteQuwgYi0IQkwQnby1atUcKivfHvFWoyfSFa2jowmIxImTk9fg9Z6F\nWZyt6CIKO1BW9W3AfcDZwBG83ttoaNhCQ8P1pvM2NxcAV4XPd/jws2zYcMWA61Cegc2hZ+LG4bgO\nTYNlyzawd2/KZ96SVfjsELEWBCEmiBbRXbsqw4lUx9NqdLDyrNzcjPD3J9IVbfLkqTidG9FLsU47\nrZDZsz1s3/4A3d2ZBAKTQmumAW8CPwXeAGzAOcBiw2rpA8473OYvg5WyRbcy/SxbsgqfHSLWgiDE\nBNEi6nJN40QSqQaznJ999vrw98cSxsHEvrPzU4yW9ZEjlfzqV9excmUtjY2pHD78AR5PBt3de1Ad\nzGxEOp8ZO6C5gJ1AO3v2OLnhBicPPbT4pJq/yDzqUwMRa0EQYoJoEbXZDtLbG/mcn38oPOSioKAZ\nSAxNtTK7fo8lXqtWzWHnzntoa8sjPv4Q3d3puFzu8PGDiX12dpGp21h2dtGAkq/k5DXARcAelCir\nzmeZmR34fHfg9Z4P7AXuBiz4/VqosckLJCWlnrC7X+ZRnxqIWAuCMGocz4jLaOty9eolrF0b+ezz\n+amuVpOt1DSsaxisucn+/QHgSeBrwOQB4lVZ+TYtLbOAawgGLWzbplFREXEdDyb2M2d2snu3uT94\n9H59fRcBS1Au7/tISSmktBQcjjKWLXuL+vqrgM2mYyCD7ds/xe3+HsNxYw/2PB2OEpKTN4Zi1tIT\nfKIiYi0IwqhxPMlcg8Vjq6rs4d+Xlm4jInQZGEVv61bYtesJnM6bUC5oNYayuHjKAPFSmd1O1DSt\nLmAhdXUBGhubqKx8m/37W4gujVq1ag67dlXick3DZjvA6tVl3HnndswJZkfC1wNTuOyyI1RVqa5j\nEeu3K+qYLlQN9fDc2EM9z6efvkaaokxwRKwFQRg1oq3PfftSB8xr1jSGZX2b3b0ejKKn+mqvRu8+\nBhamTz+DqqqBGdXRmd2wAbd7Epde+if8/n9HdR4zD+6oqKgNJ7v19mosXVpJd7debuUDmoHvh86g\nAclApP+ncQ71oUP30NMzlbi4w8yblw4khLqfHduNLfHpUxcRa0EQRo3oeGpHx14aGswZ3sCg1uJQ\nfbhVL20f8ERo3URgAZFsbDia6EXHn5XYXoXfHwh9tqLizVU0NZ1BRcWrNDamYRTJSBexxSjX9lVA\nDSrT+wPgX2lufi18zqMNJHG53CQlDS+5TOLTpy4i1oIgjBrRceh9++wGoeykrq6Vvr4pKAt1IWAN\nW4tDuXxVL20Vu1ax6eXo4lVY2EBeXvCoojdz5pFQ/LkTeDG09QXgI4zdydzun1Bfr85dWLiWgS5v\njYgrezLqheFFIAd4gYKCwYV0sLjzcEutxnJkqDC2iFgLgjBqRFuU5eXP0NBgFkTzAI3lYWtxKJev\nUbCUIK4LZYV7cDiuO2YmtX58bW0LHs9PDedfh8redtHb68bvj8S0s7KmM3euOmdb27s4nStQrvh7\ngQySklbS359Mf/9cVDvQBcBfBj3/8cTxT0bYhYmFiLUgCJ8ZRqHdv99rGl6RkuKntHR92FocyuUb\n/QJgFLSKilePWfqkH19auo36eqM73A/00tX1KZr2C4wx7Vmz+sOu+VtvPURPzyaOHPkYv//HgA2f\nL5Kdrr94NDfnDCq2xxN3PpFua8LERMRaEITPDKPQKnd2RIxLSzEJ0VAu32gB7Orq5NVXf4guaD7f\nOh5/fNmAc+vH7dsXT0dHE93d8Zhd25OBa9G05zCKqdXqxeG4ElDiWVNzI2ZvwDVEZ6dDGna7e1Cx\ntdu1YcedJaFM0BGxFgRhTDhW/HWopKxoAUxMXItR0LZvjxtwzOHDxjnQG1HZ4CrrOzPTS3d3C8Hg\nTaG9zVOtiosThmy4EkloM2enT5q0i9Wrl/G9731EtNg+/XT04BBJKBOOjYi1IAhjwtEypI9GtGD2\n9+dgtpAPDzjmpptqQhncnahJWJF4dFzcM+Tnazidk0N7LwDuwGqdQXFxAqtWXRAuN2tr2wMUo2q5\nXUyatAuLxU1m5l407S7a2s5HDez4MZdc8is0bSrwGCpbXDVoOZ77loQyQUfEWhCEMWG43c2i9yso\n8Jmszby8Vlpa9PGSrfT2urDbN2GzHWDTpjJmzLDz8cdqAIfK1r4NYzwaDvPHP17OkiVr6OubgcXy\nMZdcks4f/nAlmgaXXfY4LS0/ALYA55KYuJaUlCx6erLwes8EvkZv72Ss1gdQHcwUfv+Foc9DN2g5\nFif6QiNMPESsBUEYE4abPBW934IFv+OKKx6hrs5CMHiY/v4errjiEIcPp/L++014vSo5rLfXRXHx\nL5k9+4vs21cPfA7owezG9jFvXjq//e1H9PWpnt2appGVtR5Ng5KS9bS0fAEl1KpEzO/vxu83J5Op\nuHU2Q3U0G6pBiyAMFxFrQRDGhOEmT0Xv19xcQFvbuwQCqrlKe7vGe+9VUl9/RSimq++7Ba/3Lhoa\nLMDVKFFNxyioU6Y0AqezdStE13qvXFkbcp13o4+1VEQnk6k1580LkpS0nrq6AG53K8aOZhJrFk6W\nURfrkpIS0tPTiYuLIyEhgT//+c+jfUpBEMaI4xncEZ08ZZyqZTx2sCSrDz4wj89U4zTBZjsQmtTV\nCfQxUFQvJTPzfk4/fSYdHXvp7k4JZXfrDVKeBRIpKPDQ1FRApGb6d6huZQNbnVqt71Nc7MLh+Ao2\nmxWXy80ttzzP9u1/ALKZNy+Iw/GVkXvIwinJqIu1xWJh/fr1TJ48+dg7C4IwrhnKtR0t4qtWzaG7\n20Ni4lr6+3PIzm7i7bcTaWubA3RTX78E2ExV1VJWrDiDmprb8fnsQCtvvHGE9HSLaXympn1ISclL\n+P3dJCSsIRBIAaZjdkt3A5NJTw8wa1ZPqO3p86HvazDXSa8LvSQsAZ4DJmOxrCEjYzpz5/aQlGRs\nxLLc9EJis1l5/PFvfRaPWziFGHWx1jSNYDA42qcRBCEGGMq1HS3iu3ZV4nTmoeK8GbS3dwA/wxgH\nbmrKZN++JhYufJ5gMNKk5PDhDcTH/4NJk9agaTPw+/fh9f6UhgYbEXd3MlBCxPX9D1QG94N0d/tp\nbEwNradPwUrH7GrPCZVYbWbfvgQ6OtxkZ5/HzJlHcDiWhsW5o8NNRcXwPAmCcDJ8Jpb1jTfeiMVi\nYdmyZXzzm98c7VMKgjBGDFUXHC3iym3dBhgbjAxsKnL11c8RDH4u6rsM+vtn0t9fTmFhJU7nl1FC\nrH/vB3YDS1HWsga8AawGLHg8GocP672+F6Ji1fuARabr1jOxy8s30dCwCqfTEuopHvEWqNptFdeu\nr19CX99fSE5OEvEWRpxRF+uNGzeSm5tLR0cH3/72t5k5cyYXXnjhaJ9WEIQxYKi64GgRV7HlwtBn\nN7AntIKKERcWNrBq1RIuvvgg8CEDZ0C3As/jdPaihHmx4ftEYAZKhDOIWM+6ld1FZmYuUGmYnnUd\ncB9wNoWFDTgc14Xv6WjeAn1spr7+jh1xuN3SHlQYeUZdrHNzcwHIysriyiuvZPfu3UcV69zcjNG+\npDFF7m98M5Hv70Tv7fBhNzfdVMPHH6czY0YXjz66hKwsszX56KNlrFixMbRPN2vXfotLLnmMlhYN\nFS+OuMCnTbuPd965iRUraggGVwGfALejYtDtKCs6H7gUaADsqIEaBSj394LQmkYSME7n6u6+j6lT\nz8XpXBzeIzW1kEWLjvDwwzeZrr+oqMf0olFU1EtubgZOp41ob4DF8qlpm9Np+8z+zkzkv5sw8e/v\nWIyqWPf29hIMBklLS6Onp4c33niDm2+++ajHtLd3jeYljSm5uRlyf+OYiXx/J3Nv5eXPhePRu3Zp\n9PUNZk3G8+tfLzJtOf/8PGpqNgD6HGkACzk5Z9LfH8/evSmh7XagAvgD8AVU/PkHRIu8Emz98/6o\n78wtSW222WRkfAw8hbK+D5OW9iF7987lO9+pNrmvf/zjL/DGG5W4XNOw2Q5w221ltLd3UVjYgdHi\nLyxs4ItftFJTY9zm+kz+zkzkv5twatzfsRhVsT506BA333wzFouF/v5+Fi9ezCWXXDKapxQEYYQY\nbhnWiQ6baG4uQLXhfAyj6L3zzm7OO28PZ51lrImeDEwFFpGfX09Ly2Sik8JU0xPlyk5IsBIIGL/L\nMp1j5swedu7sBH4Y3tbevoH29qvCCXB5eWdjt3fi8/nD7u7eXo21a9dTVWUfxOWvXOdJSdIeVBh5\nRlWsTzvtNKqrq0fzFIIgjBLD7TA2VFLZYOValZVvG9qGHgkd14MxvqxpBTidNxIM3kNZ2XoaG1Np\nb3+Pnh6NuLg/cs45mZx//jq2b+/A7Y4khcFeVCMSK3APA/uFb8Bq9VJcnIDDcTnnnVdLdOKa/nun\n8/M4nUuor9ewWv/IYC8jQ7UClRi1MBpIBzNBEAZluBbzUEllt9zyElu2qGzv+nqNF164g0DgrvDn\nBQvWsWDBOmpqfIbzgJpkZaGz024Yp9kTfnHYts1Fbu6DdHUB3IPFkk1c3Kf09/8EJdQagUAPkYSy\nbiwWK0uWBHA4rgx7B1SSmwvVSjQNleR2KcqKj7QKhUMYhV+6kQljwcBZcoIgCCiLWYkUgMb+/R9S\nXv4MLpfbtJ/NZuW++y7Hbvewb18ql1/+BCUlz7FtWwuqMxiAhUDAjlH8X3stgaSkROLjm4GvEmnr\n+S7gQtP2hs9lfnF4hvb2NPr7LwJmoWnX0N9fBNRgtT5KYWElKhltOSpLfDmTJ3cDsGzZW+F72LSp\njEmTfhnabwnwMzIz/5NJk+5AJao9BbiYNy+DsrL1nHfes5SVrT8h13ZHh5vy8k2Ulm4b9BkKwrEQ\ny1oQhEFxOEro61vHK68ECAS6cLu7qK6eyvbtf+Svf/22KX5tdJmDhtO5EZXBvQG4FiX6jRgt1N7e\nZKqrl2Ox3INxUIYS2Pvwem+jokJ1MTO72l1EN1BRMenFTJv2JJ98YkH913Ynykp2091dSHV1PHBZ\nKCb9MHl5ZzNp0gy83sgLRFzcJLzefwuvXVhYyUMPXXfStdLDDSkIwlCIWAuCMCg2m5Xk5CQCAWPj\nko20ta3hllseISkpNRx/bmxMY2AfbjXVCjaj6qKTgcdR86SnAN9ATbmaiu76jhxfAPyOLVuslJc/\nw+rVc9Bd7Q0NGVHJY16U21qjo6MJj2cFSvwvBHYBd4X214UdnE7V5ASexBzbzjZdR17e2SPS1ORE\nk/AEQUfEWhCEIYkWGV2Et2/vwu3+HrqlOGXKHShhzkANuuhFiV8Lqq92I5oWaRmqLG5QrmY/8L+Y\nG5skAT+jr+9RqqsTqav7G8XF8Tz99Bxuuul5tm0zCmwLaWk9/PM/r6exsQin02ilw8Dr7yISz+4h\nM/NeZs48C7vdg8/Xbyq9Gqn49FBJeIIwXESsBeEURs/YdjptFBZ2DCjPihYZFVceaIEePpyAcRBG\nQsJdBAIbUNnZk5k82YXbbRRNN/BrlKtcubaTk9fg988kGExBNTbRXd7fwe22UF2t3MdJSQA/B+ag\nLOrvEx9fFWoN+gy7dycSEWM9acyGPiHL6+3E6707fK3p6ZVs3apmTbtc7lEpvRoqCU8QhouItSCc\nwkTHmqNjqQ5HCUeOPMJrr2kEAk7i4rK45JKH2Lv3CGoaVTdwMYFAPkbxPuusc5g5s4emptcGtVhV\n1na84RgbfX3TgY+BuahxlQuAHAa6jzNRLvUl4evs6ckIX++WLY/Q16eL8SLgDmy2WcyfH4fDsZxv\nfGMnu3dH1szOLgqvM1Q51skyWusKpw4i1oJwCjBUg5NjxVJtNitPPfUvpm3l5ZtoabkFXXgtltvR\ntHOIxH5d7NnzNh99VITNtodHHilD0+CVV+7E75+NillfS2Lievx+o4A3Ar8wrZuRkYHHE+0+1qiv\nbzadr7+8jtPXAAAgAElEQVT/AKWl27DbO5k58wzee89oxV/A7NkJVFVdBsDMmUdCAzkiDVIEIdYR\nsRaEcc5wOo0NlY18tFjqcAVe07JQFnEVqnd3F8FgJb29quPX0qWVzJ07Db//34kI8wbmz8/kf/7n\nDrzeuSh39udN606ePJudO6/kllseYfv2LiAbn6+fn/98Hps3dxAMRlzdmvYL6uvVveXn/wJz0th7\nfPRRIeXlz+BwlIhLWhiXiFgLwjgnWojr6h6guDjPJNpDWdC6cKmYtcskXMMVeNU0pNLw+bemc7lc\nBQPOHxfnYc8eNxbLLJQrfSHK9R1Zd9IkJwBJSanhZLaaGo2kpPV85Svp1NQsN5wzsnZPTz6RjmgN\nwApcLls45l1VtXTYLunhtlwVhNFGxFoQxjnRQuh2n0l19SKM8eehLGg9ljrYoITIum6ghq1bCZdR\n9fWtY8eOOI4c2Y/ff6bp/Kq1Z+RcmtaI3T47dP5O4EWCwSAtLbOAr6FqoTcCC4iLu51g8MvAEVpa\nfkBFxeZBXzSefnoO77xTidM5DeVWj2SS9/a2ohLXQL0IbEHPAt+3L/64BFjqo4VYQcRaEMY5g2ds\nm+PPJ+L6LShopr7+KVRJViK9vUuorp7MSy+t4Z/+yYbb/R3gEeAAZrezF2OrT6/XRm1tVyi2nQXc\nZth3I3ANKSl9XHbZ0/zP/2Tg8fhQXczcvPiik/nzC03rt7Q0cMstzbhc01D/hX0/tE4asAO/f7ph\n//0YG6h89NEdfPnLTtzunzAcAZb6aCFWELEWhHGOLsR1dQHc7kkol7Kyns1WpMbTT885DjduIsZy\nLF1Yvd6LeP31N1EW60qUtfwEaiBHO6qLsdFFvQGP59rQPtEzoPuA5wgEGnj11UmGLO6rgY34/d+n\noeEOpky5k9bWTCCHlpZp1NT0oCzq7xPp7f0ukAt8E3gUVfaVazqf13s+Xm8iwxVgqY8WYgURa0EY\n5+iubJfLTUVFbbhcyuG4nIqKod24RiEvKurh7rsvNQl5c7O5bEpZyhpwhP5+O5GuY1ZUE5PridRG\n3wecg5o9nQc0AR+iyq6Mk7KSgCX4/Xpf8IENWDyeWSQnt2O2yO8AZgPVqBeEbuAW8vP/MzQ+MxX4\nDip2bbT6+1CW//AEWJLRhFhBxFoQJgjRtbwdHW7q6gIMZUVGx2O3blWJafooy/37WzAL3QcoUfwq\nmnY/0EYkVmxsF2pDCfWi0P7LUeJ9F8oK3xD6eRi4OXSMGo9pPp9qwKJp+4AZmIXc+HKgERdXyeLF\nm1m9+uusXbuerVuht9eC8jJsJDXVj9V6EKdzRegY87jM4T5TQRgrRKwFYYKycmUtbncyRgFsa3sX\nl2vOoCVYbvdsqqt7eeGFZwkEfoBqevI4CQkHiY930dcHyjLdiKaBGtDxBMpSNSd5KYu6m0gnsjwi\nVvi1obUnh36BalGqhFUJ//+GjrkTTZvOpEmfmu7DYslG0yLXnpmZHxbVqio75eXPhLK/rcByFi3a\nyN13XxdOWLPbzeMyBSHWEbEWhAmKEuPLiCR7fYDTuWKISVYaKuY7hUAgDlV+dROwhUDgCwQC/wvM\nAv7VsP8TKAs3A9iHxXIP8fF5BAKTQts04K+AhylTPqa11Xiut4F+LJZ7yMgoJDHxQw4f/hg4HdUi\nVG9peit9fRZaWlwUFlaSl3c2druH7m6LqT/4vHlB071Hu68ffngJ/f3xYiUL4xYRa0GYYOix6P37\nA8ALRMqjGgAL+/bFU16+iX37EigsrKS7Ow+PJxXoR8V6VwHPM3Bs5YOYXdFeIq7opSQn34HFkkIg\ncD1qulYkOe3ccx8hGFxDe7sVSEFlmF+IpnnxeBaQmPhbIuVWoCzvFoyu9by8s009vCsqjLHkr5ie\nQbT7OitrYGmaIIwnRKwFYYKgi3RdXWu4NElZqA8CU1GZ0y/S0dFEQ8Oq8PcLFqwjI8PCn/6Ui7KI\nLaj4cXTCVzbmmHIbcC9wJtCL16u3En0KJeSRY1991UJcXAoqSWwjymqPZJn7/YVRax9BxbUHTwST\nWLJwqiFiLQgThEjC2POYRfZzKMsYrFYv2dlFoVnO6vvm5hxefPEqtmypxOPJRAnkQuBhzHHoD4B7\nUOVQn6Cs9dNQ/43obvR7UWIcB/wB1VAlm2CwjWBwNsYs78j1pQE+EhPvxO+/ECXUXwX+jB7DLixs\nwOG4bljPYbCmJ7m5GcN/kIIQg4hYC8I4xihMjY3NKGs0Oqv6g9C2i0lNbaGpqdXwvYuWlgYuuiie\n1FQfHs8hIsM0koG1wNmo+dR+4N8M696DuQ57PxExVo1U4EbD9/eGfkZf35vArcyf/0fee68Bl6uA\nYPBBEhPzSEhwMW9eBg89dJ0pGexoXcgG6zr27LPXj+RjF4TPHBFrQRjHDBxxuQFlFW/AYulE0yaj\nksImAz/B6ZyDsnZ19/X7tLTcTkuLGiepYtjxeDyRrl9qNvVUVDmWbhF3hn4+jxLfhahY9FMoV/jn\nQvsaLegzQ9fXgYpPzwQ+4owzTufsszfj82XidN4aPm9f30ZgOe+8U3nU+46uH5euY8JEJG6sL0AQ\nhBMnWpiURfssYCEpCVSZlJVIzPkaVLz4Z6i4snnSVV7e2cTFTQltawLuIxA4DTW+8gPUCwGooRv/\nhnKTXwO8SE5OV+j35aiMbo9hfw34O6rL2b+EzvuvQCVnn51OVdXSIZqwdOJ0JvGlL71MefkzuFzu\nQe/bKMh2ux7rVueVrmPCREAsa0EYx+Tnt2N2KX+KsliXk529FqfT+F02A8XQQ3QS1/79h4hY6SsN\nx98R+mVHZY4b1+rC69Vbieq11A+jeocfRjVKuZWMjN/j9f4Wv/8H4WN1oR28x/mLwG243Zbw1Kyf\n/ewC3n//TZStoWq5jYIsXceEiYiItSCMA4aK0VosAZRL+xxUYtZNJCb+ioUL13PTTZdTVnYHXu8Z\nKBHXk8f0lqC7gHxgLSkpUykuDuDz+QkGe1FCrTcyIfTzDOA6VH11E+aXhAz6+oyNS14GvoDKLs9A\nxbxtBAIF5OYewOmcjHLHv0hjo5fzzvt/ZGbmUlhYSUdHPl7vx8BZKE+B2YK++urn8HrvDp970qQ7\ncDi+G35WkikuTERErAVhHDBUjLa5uQBlyR5BWco1zJ49i6qqpZSXb8LrvQtdnJOSKklIuJ2enjhg\nGiqurGqws7PvIzm5kOrqG9HHWMI+zILsDH13AGVdr0G9JAAsJCnpMIHAGjStCJVsdhvKotbLxzR6\nexPp7b2JwsJKenoScbt/gsdjwePRcDo3AuUUFq7F6bwRlQneHzpeXdP+/V48Hv2zcu9bLGdIJzJh\nwiNiLQjjgH374ol0IusKfdZdx06MYyA7OytDx6QSsUq34PPdh893H2bX9qNAKocPF1BX10JEBK8F\nqlCZ4fkogc5CDc643XD8htC+GoFAK5p2t+E7NaVLfc5An1kNVvLyzgagvn7g4I7U1Azi4n5PMHgm\nyoJ/mMREF37/atzugee12Q6OwBMWhNhGxFoQxgEdHU2ozmJKrDo6lCA7HCVs2/Ys3d0RIXc6M7nh\nho20tzehRk0aB20UYnZtu4Dv0NtrobdXL69agcoeTw/9Mo67/H3U8T5SUp6gtBS2bp0V9V1a6Pca\neXlO2toy0NuPFhR4SEpKHSRGrXHwYDvB4C8M2+8jIeE0/P7I2gkJXSQmPoHNdpBNm5aMwBMWhNhG\nxFoQxgHRjUy6u/MpLd2G3d5JWlob3d03YxS3mpofkJFRScQa34PK3Nbjyrqr20qk3MsKTCUh4X7i\n4vz4fBehksOMAtwedbwPTfuE1auXs2tXdUjw9VjyLs48Mxjq5Z3Ntm3Gmux14USwxsZUDh/eS1aW\nnVmz1g8i+oXYbAdMa3/taykSlxZOKUSsBWEcMHPmEXbvjoiVxzOJ+vqrqK/XyMy8n4Edyyx0dWWh\nOoFtQcWoVwE5KDd2GrAas8t6OZBIIHAPSsD/GZXR/RyRCVr5qASzT9AbpHi9LoqLf8mMGbPp6FiD\nxTILm62ZTZuWMWOGHYDS0m2ma2xuzglN7oL4+ATmzp2KwzEfm83Keef9P5Mwx8W9z2OPLeI3v5EM\nb+HURcRaEGIUYwZ4QcERFixYR3NzDvv3f4jbXR7ay0JcXA4DO5Y9h7Ki70fVNH+CyuZuAaaj/ukb\nBb6XSExZjzHXYIyFq45lFtRkrFzD8Vvweu/ivffUfmVl66mq+qHpXqLLsvLzD1FSsh6n8/NAN/X1\nSwA1DWzTpjKKi+/A650LHCEY/Cm/+c1msaSFUxoRa0GIUaIzwBcseCRUB52NcZqWGg+5jr/+tZfu\n7k+Bi1CW8OmoxiMbMVvRG1ATuIwCvxeoNHzuIjLUg9DPPOC7od8/aTg+zbSfPtXLWGbmcJTQ17eO\nHTvigMP8/e+dtLYas8U3huutZ8ywc+aZc0ICrpAuZMKpjoi1IMQI0bXU+/aZrd/t27twu7+HLqiZ\nmfeTnh7gwAE7s2YFuPRSqKkxCq4+0jJ6cEYGkEpy8hr6+opQJVnXAveRklLIxRd3smePO9yCNLJe\nu2Gdr6EsbTvwIcaBH8apXvX1Gjt33oPXO4nu7kwCgRRUh7PJRJLZrEAaBQVt4WcRbYlLFzLhVEfE\nWhDGEKNAt7Xtwem8CbBRX69RWFiJ2fo1dyCLi8vB6VyK07mFhgYbCQnNmEVZH2kZPTiji4yMOI4c\nyUEN2zgHZWmfTmlpAJhMS8vNqCSyDajGJMmobmcuVAw8DeU670MN67gPOJ1Jk97D5TIniLW0nAbc\nYDi/XtJ1DsrVvhyVABeplT6RLmRHG+4hCOMdEWtBGEPMgzjK0OueIZ3u7ngWLPgdzc0F2O0efL5+\namoiohsMtqISwFTcNxBIxizKHwLrAB8JCXeQmmqnt7cVvz+Prq6bQsdGyrL0TmDLlr2FuW3oY6hE\ntXZUDFwvq1qMst5/HfrcH2rCsiHqOj7GPPAjncjMaj9KvFfQ3Pxa+LmcSBeyW255iS1b1JSv+noN\nn28djz++7LjWEIRYRcRaEMaQgYM4VN0zWPB4FvHOO5U888ylVFa+zYEDqRQWVpKdXcTMmT3s2NGD\nx6N3KFPlUCrTuwg4hEokawb6uPLKqTz++DJKS7dRX38VqtXnFNO5LZaZVFS8SkGBL6r+OQnV4/t7\nqKYo0Znnt6EEWo9xL0QJsB/1wvBjIrHpDSi3ezfqBaAGZWWfvKtbxcONYQOZUyRMHESsBWEM0F22\n+/cHUMlaKlksISGDQCAiOE7n5/n615/D6VyFXtvc3d3K4cOddHbOwFwjnQccxOxyvg/4N/7+919Q\nWrqNtrY9QDHKlW22xHt7J1Fd/VVycx8gLq6SYPBclKguBJ4hMfGXTJ6sceiQUcg7iMTBdXe7FWWx\nbwQuQAm1up+MjB4uuSSN5uYUCgr+Avhpbn52hMqx9AEk+rUdPsn1BCF2ELEWhDEgeg611foAxcVT\n8PniTK5u2EN7+ySU8H0K3IbHsxGP5ybMMWA97mu2llUi1ye0tEyipUUD4oiLewzoIRj8FuamKWnA\nb2lvn40S/UuIWMQp+P134fGsJGJFd6GsZz0uvjD0nQc129ofWidyPxkZbTz00HWjEkueNy+dmprI\ntc2blz7i5xCEsULEWhDGgGj39/TpZ1BVdQUul5va2kiNMXyf/v77gVtRcd9OlGgbY8CHgZ8Dp6Hi\nwy4iItsM/BfG0q1g8FGUqNeiXNyXoBLMsgFzJzRlraeg11/7fJ9HxbHdKBd2H6rZyixUL3Er8+f3\n8NFHHQZvQCRJzelcQUXF6NRMP/TQYpKSamlq6sduD+BwLBrxcwjCWCFiLQhjwGClSR0dbm699QW8\n3lTgXVQf799hsVhD+3Whz3c210w7iSR96f29P48S4FuBNxgqLq72vxPlEg9gdqufTWLiLvx+Y1xc\nb1eqZ3FvBCJWfn7+PVRV/V+WLXsr1B5VT1J7At3CHq2aaRmNKUxkRKwF4SQYrFxI0zhmCdGqVXPY\ntasSl2saNtsBVq8uY+XKWmpqMlGCGBHI/v5VKKH7J1Ss2Si8XSir1rgtK7R/AcrCPow5lpsVtX8i\ng7ce3cP8+Vbee68y1GnsCPA14uJuJxiczWA13IcO5bFs2VuG2Lhu4SeG1tyA3R44mUcuCKckItaC\ncBJEdxnbseNOLJZEWlq+SHQbTSOVlW+H3MRq2tXatetDFmc8oAshqCztIpYsWU9dXStudwCz8LpQ\n7m/jtkmoGHRC6LNuMetx5vej9j8Ns3gfAdYQF5cNTGLTpq+wdu3bNDVl8v77/43X+wsi5VnmGu5A\noIv6+u8BZRQWqpeR3t5EdDe61erF4bhyJB69IJxSiFgLwkkQHXtubc3E7KbeyL598dxww5Ns394F\nZDNvXj8HD9pMx+lWeH19AsqtHRHA5OSPqaqqoKTkJdzuuURiyZ+gBnTEoVzZXyAu7m0SE0/Dau0h\nP9/PO++sQpVyAVyKcksfQrnN1QuFwije7cDdBIMWtm1TLxL6y4YqrzKWZx1Cud3PRDVJ0T0IFvLy\nzmbu3E6qqyO13MXFCdKoRBBOABFrQTgJomPPaqqV0UpN49Chf9DQMBNVp2yhpkajsHAtRoFsa3uX\nRx5ZwvPPr6e/H2ANMAP4kOeeUz2yOzo+QM2n/hmq3KsIVaOs1igsrKS2dkVYDE8/3YHRnR5xbx9C\nJY3prURdpKTcyec+d0FoSMh0ol8kdDIzP6S39ymUlR4EWiksTCMvz0Jb236czhWhPbVQOdbxdyIT\nBGEgItaCcBI4HCXs2mWM6YJRhAsLG+juzid6KEZW1nSCwXtCrTgP4XTm8POf/5WMjM/hdn8ntJ+b\nxMTf8KMfHaCx8QX6+rJRTU+CqCEdlqg1i/jRj14KNQc5hNc7jYHu7TtQGdr5JCSsITGxCJvtIK+/\nfiOZmVmUl3dSXR003YOxWcm5506ltdU8lzovL4etW6/A5ZpDRcVmkzBL0pcgjAwi1oJwEthsVmpr\nr6OiQh9l6QbWceCAlY6OvWRl2Wlvfx9lyUYEsKOjiba2eIwNTF555U5SUuyoEqgkQMPvn857730F\n+CbKMs4Grg8d86RpzY8+eoeGBqMlvQqze/sQytJ+ArievLxK6uuVkObmZtDe3oXDUUJX1yb++te1\n9PfnkJfXyurVXw/fb0tLtOcgKyzmgwmz9OsWhJFBxFo45TlZQRlMpMrLN9HQsCpUvuQCfoXqo51D\ncvJenM6fAq9hFD6//zz8/q8DT2F0b0cGX6SjMrvNk6+s1qmkprbgdJ6FWUhPJ9L0pBs1IUvPFrdw\n6JCV8877T7KzizjrLB93330pNpuVjAwrfv8PUUM4VMz6vvsms3JlLR988AnGF4BJk/6Ow/HdIZ9N\ndAIerBdLWxBOABFr4ZTnZAVlMLGPTjxLTEwmISEPm+0AaWmz+fBDGwOzsveimo34MIuuPviiG5X8\npR8zGYsFtmy5iNLSF4nUQOvrdaJGUBpFXwM+ADz4fB04nbfjdFrYvVtj69YHKC7OGzCas6kp0/CM\nzE1OZs8+86gvNtHPQeZSC8KJIWItnPKcrKAMJvYFBUeor9cTsRrw+1fj96syrbi421GiOR01ZcuF\nSkzTUIMyEjGKrsXyDzTtDSAXaMNYhlVSMpnKyrfxeH6KEtInAC8WSxslJalYLI/wt78l0Nv7CX7/\n5NCx/4pqQ/p703273WdSXb1owGhOu91jeEZ6k5PNwCJmzVp/1Gcjc6kFYWQQsRZOeU5WUAYT+4IC\nH2ZXduT7YLAIZeUeRDUNKUSJbyJKjL9NxH39Ppr2A5S4bgD+D/AUcXFTyM9vZu3aMr73vY9QQl2D\ncnHXk5CQQnp6DqtWzaGy8m2ami6goaGVQOBaw5V7MFvi3YCFtjYbmZn3Ehc3hXnzgjgcX6Gi4lXT\nM7Ja36e42HXM7G7JBheEkUHEWjjlOVlBGUzsm5qMiVjdmEVxHzAXZSk3oTK0dSt6DZo2GX1spGpu\nYkW5x53AfwM/Ixi04HSqeLLdrlFf/yKq8cgW4Iv4/X+jujqVHTueprVVTzp7LOo6JqFqtnWL/VpU\nYxM3Hk8u8G2SktZjs1kHeUbLhxXXl2xwQRgZRKyFU56TFZTBxN5siS5g0iR9OEcD5vnOD2O0ujVt\nKuaksNND3+k9wfVhHjVAOi+++AlPPTWX6upGlFDrDUgWAxtob3cb1r8KPclNZZtnYh7c8SAwFfg+\najZ2JCQgoisIY8uoi/Xrr7/O2rVr0TSNq6++mu9+d+jMUUGINYzJY0VFPeGM6ejv7HaNp5+eg6ZB\nRUUtH3zQR1LSSgKByWhaFgkJCeTkvMGhQzMwzneO7tsdH3+Q/v7lKOFNA/4GrEe5rI3DPJSL3e9f\nxHXX3YHqIKYnkaWH9rMQF5dNMOgCnkPVWbtRZWT7UF3QjIlsn0OJPOgxdIkxC0JsMKpiHQwGufvu\nu3nsscfIy8vjG9/4BldccQWzZs0azdMKwogRnTzW1xfJFDd/52LXrofp6cnH7U5GtQA9D11Uu7s1\nurvvZaBLvBdju87MzG5crl+i3OTdKGv6d8TFHSYYfCp0nC7cABb6+magyrgexNyx7A4CgWyUq/si\nlBv9bsP390ZdS1doTY3MzGYuv3y9xJgFIUYYVbH+xz/+gd1uZ+rUqQB87WtfY9u2bSLWwrjhaJni\n5u+2hAdzRFzKUzBbrlNR85/10qckoAKYTGbm/Vx+eT61tfmodqLGcqtzCAZ3EElYM2drJyU10tc3\nGbgg6nznh873o9Dn56K+n0JcXCWZmflcfHEQTfPT3PxsyJX/LWleIggxxKiKdWtrKwUFBeHPU6ZM\nYffu3aN5SkE4LvQZ0sYhGw899NWwUB0tU9z8XRpmIcxhYLb1p6h48JbQfsbM7CyqqpYye/bTUesk\nomZbzyYya/paVO/wmUAj55+vMWXKeurqWnC7jefrwzzCMtqqn0QwuBq3WyM9fSO//vWiE3+QgiCM\nKqMq1pqmjebygnDSRGZIR4ZsJCVFXN3G5LGiol7uvjviFjZ+19a2B6fzUpT1qgGfEB9/iJSU/Xi9\nOaSmdjB3bhJJSX/hwAErDQ31GIWzp6cJgNTUZjweo6C+jZqQFT2M42z07O3333+A555bSmNjE5dd\npiey7UG9GNQYzrMAWE1CwukEgx0Egz8I3YmFl1/uw+VyizUtCDHKqIp1fn4+Tqcz/Lm1tZW8vLyj\nHpObmzGalzTmyP3FFk6nMdlL/Xz5Zbj55s08/PBCiopO49lnrx/02I8//pitWz/C651OUpKb5OT7\n6euLCGt//4P097t5//2vMmuWPXzcsmUbaGiwY8z61rQscnMzyM+fTUuLMRu8ELOl3YNyg98U3max\n5JKbm8HNN+8OCfUSYD5KqA9jjImXlc3i2Wf/lWXLnuJPf5ocWkPD5UpizZo3ePrpa07mccY04+3v\n5vEi9zexGVWxPvfcc/nkk0/49NNPyc3N5YUXXuCXv/zlUY9pb+866vfjGX1YwkTls7y/kRoQUVjY\ngbI8jVauxp/+dA11dXdywQWn09ycg93eyaOPltHfHx8+trj4v/F6VUJXX9/AMiz4HL29izjnnDWc\nddaF4evcuzcF1RAl0go0Le1+PvjgAG1tjcBqIpb0PZhd1wdRLvGI0H75ywHa27tC6+qubiuwHKv1\nAdzuVeFrfuGFRzj33Cc57bROMjPvx+M5K3TMQvbufW3C/v2Uf3vjm1Ph/o7FqIp1fHw8a9as4Tvf\n+Q6apvGNb3xDksuEESE6S9vne4SkpNTjFm+Ho4QdO35Pa2ukhSf4AQutrZnU1NwYPseKFea4rsrC\nNoqzLvzmjmB9fRdRX78k3IpUNTHRO5KloHqEZ1BS8gRO57+gLO40EhPfRNM6CQSM1xYAesOJYfPm\nBXnooa8Aegx9Sfj4KVPexGJJQLnmu4EFBAIZNDRYaGhYQWHhWjyeReHrLShoobx8k0zIEoQYZNTr\nrOfPn8/8+fNH+zTCKUZ0lvb27V243SrufDzDOGw2K7m5X6S19RuGrZtRYmseB/nxx+mmYxMT38Pn\n08up9gNWLJZVaNoUVCb4wtA6R8JrNDVl8vTTc/D5nmf79k85csSH378aj8cSilXrE7bgnHOCfPCB\nJ6pF6BPAdSxePPD+VAxdnyftxuc7Pfyyoa7j58A09KSzrKzpzJ0bicd3dSXIhCxBiFGkg5kwLonO\n0lZzno8+jGMo13lHxweYLeJ/oCxRv2n71KkdpvXmzTuNurprUAKryq1UUuUToWP+GlrrJlQzkhfZ\nv99LRcWr3HnnpVRWvs3WreD361neVlRWOeiZ521tB+jtjVxDYuJHLFwYqX8+WjigtHQbZsv/QmAR\nqu5aY9as/rAY5+ZmcP75zx7zGQqCMDaIWAvjkugWnz5fPzU1Rx/GMdQozKys6TidxqQuG5BGUtLf\n8fkiLmhN8wMRgfzb3zKJuLKNomhDJXlp5OfXc/75f2H7dhdu909wuy1UV2vs2lUZVZetZ3nvITPz\nAOnpnTQ2FnHWWekEg/fQ2WnHZjvIpk3fZMaMSLLa0cZ75ucbx2lG3PLJyZlkZ1fS2FhEefkzOBwl\n5OZmyIQsQYhhRKyFcUl0r2qXy01SkhLv/PxD+Hx+Skqeo6OjiezsImbOPGKY0+wGati6FcrLn+G0\n047Q0BA993k+gUAzxlpop3MzYBbIwTuBvQtYsFrfp67u/2KzWSkt3UZ9fUTQW1ryMQu8L3TeFfT2\n/gaPZzVOp1qvrGxod/TRmrZYLAHMDViUWz47243TuSo8xxrW8+yz18uELEGIYUSshQmBUbzLyzdR\nXX0jSvwiohSZ01wDLKe3V1m5Cxaso6xsPXV1AdzuScDngQcJBmcCT6JaeU5mxoxuYKBAKsv7DlQ8\nuAOYDnhITvawbNlb2O2dFBT4TFZrMNiIWeCdwCpAw++fynDd0UezhpubC1DDO9TLSUrKc5SWQmNj\nUagP95kAAB2VSURBVOhFwLy+DOsQhNhFxFqYcETE1Ni9y0J2dhFz565n61bo7Y1s37LFD8SRlLSP\nSy9NYceO9/H7jT221wCn8cYbbXz8cdMQ8fJ/Ae4HvoxyN/8Tra1NtLbGU1+vYbM1oAR9BtCIGk/5\nKOACckhI8HHmmU9y8KATtzsXo5C3tb2Ly6WGhETHp49mDUeuU5VxlZYqC728/JmQRS3ubkEYL4hY\nCxOOiEh1YU4QcwNJJCe3mJK21Pzoa+nr0/jb39bQ3z8Ts+V8EbAEp1Nj6dJKamuvo69vHVu39hMM\ndqFqnp/D3GnsTuDfw59drmYiPb9dwC9DP28DLAQCGtOmraOjw4/bnYkS9rMAC07nCioqlAteud87\nqa9/kc2bXyQ//xCbNpWZ4tg6Qwm5uLsFYfwhYi1MOHQx2rcvno6OylDMugefzx9yj3cCG7Bavbjd\nzcC3ULHddPr6JqHKsIyWc6T0yuWahs1mJTk5iWBQj1u7gD9hFvjZUZ+Nru0tqOlYz5v22bEjLtTA\nxAIsxVjGFXGFW1Bu/GsIBi3hF4j6+h8OeA5DubXF3S0I44+4sb4AQTgROjrclJdvorR0G+Xlz+By\nucPf6WL05z/PZ+7cacTHJwAaBw7o7nErcC3Tp2eRnNwN/A8qE/tS1HCM6cDtKOt3FZAMPAW4sNkO\nAkZXuxvVuSwdJewQGdox1Gd96EdX1D6HMQu8uYzLbu8M7Wd277tc007gCQqCMJ4Qy1oYM06mZejR\nSpaG2ieSYBaJ1WZm5vL66z6MFmvEoq4M/VKfU1LuZNOmbwJGV3sNKiFtPpFe3/XAdeidxPLz/8E5\n56Tw1lsPEAza6OvbT1/fYlR2trLwi4sT8PnSTOVn8CbgJjHxQ1avXoamESr5ysKY+Ka/QAiCMHER\nsRbGjOEI7lAcrWRpqH2ys4v4whfWsWNHHHAYny8Nl+t0Iv20zRYrmMurZs/+AmvXvk1T00cUFPhY\nsOB3vPZaGr293ai49TWhdeopK3s93EnM4bjB9BLicrmpqNBjxgEcjiux2azh8jOVld4K3ArY8Ps1\n1q5dD2CqzY6LqyQ/HzZtWjKsZyYIwvhFxFoYM4YjuDC4BR6dkd3W9i6NjbOprHw7FKtuort7CkYL\n9PDhvRw4kIjb/RNAjcMsLFwL5KFi1p+iOnzplu1HGC3xDz98h927fwxsob5+Cvn573DxxbBt23J0\nK1qNpnRTVXXLkPetu+n1+9LLuxyOEqqqluJyufnSl17G7Y5MBDPHrNXPL3zhbLZuveL4HrogCOMS\nEWthzBhux6zBLHCHoyTkEv48cASncwVf//rDIctT1Vfr61qtD5Ca6sfpXAG8gVHwsrKm09PTh9t9\nLSr+vBHoBeJJT7fS3R3pbOb1TkElhy0HOmlp6aalxQU4UAlk76EakMwIdwY7mls/+r5eeukOzjjj\ni8yceYR587yDdGTTpMOYIJyiiFgLY8ZwSog6OtzU1bWiMqe7gIXU1QUAyMs7G6cz4gJWiVadKAs5\nsj9kk5WVHJpdbSznctHR0YT6Z2BM9Ipj0qQP+dKXckNWs3EQhje0dgORUiy9i9mtqBj2tVRXR9z6\nQ8Xmoz0LXu9cdu9ewu7dkUYtA5+NlFwJwqmIiLUwZgynhGjlytqw21qJ4gbc7klUVNSGRk1GLE2b\n7QC9vS+i1y4b909N3R/6HEnqSk1tCVnincCDoTOqY71eDYvlEcrK9CYqicBpgHGKlTG+fQ7K6s4I\nb9Nd10PF5gc2V4mUiG3fHsfOnZcPsMyl5EoQTk2kdEuIaQa29vQBC2lqysThKKGsbD3nnfcsZWXr\n2bSpDKvVO+j+2dlFoX1fo6wswM6dV5KXdzaRUq5CoMh07JtvJlFVtZTSUg3l+p5i+F5PSoOI0Kah\nLHe1TXUecw8am+/ocOPz9ZCYeCeqocq9wFfDx+ovJIIgCCCWtRCj6K7j/ftbMDcoSQYmY7d7BrXM\ni4vfCrmgzfvPnNkzYF+zZbsA1S50cfjYI0eacbnchvi4RiQBbQEqLn4xSqi/Sn7+b9C0Plpbn0OP\no1dUbB7gAbDbPaxcWUtNzfdRVv2LZGZm0Nv7K/z+81Gu9oU0Nb02sg9VEIRxi4i1EJNEXMeq21hm\nppf09BaysuzMmrWeVasuoLx804A4sB4Hb2xM5fDhvUPuv2LFGezc+Qlxcb9H09rQtGyU5RwZien3\n9/GlL71McXE8mzYtYfHi/6Kt7V5UMtmnXHppOllZ7tCam3E4bmDZsrdobdXj6CrePm1aIYWFkU5q\nDsflLFv2FsYGLTNnPovdnkF19VVE9wQfbu25IAgTFxFrIWYwJmIpi7oTo5ht3fp/wvuqyVqROPCO\nHXdywQX/v717D66yvvM4/s4dSAI5QIBEuiGAEay2TC11YVxCsY0SwKBopXWkRZuV0sEx7Qw3124t\n3VBTrbZDhyJip1AqWNYkUAhVA4RWKcvWTTEqZYg0CLmS5DQJhlzI2T8eTs41yUlyDufJyef1jyR5\n8jy/x4if/G7f379QVTWelBQb+/bdicVyj5frjbra+/f/DZttKvZtXUbFskSMFd23AGeBWVitVyks\nXAgcYO7cmRQUrMAepnFxO/rorR/qPsMabMye7VhwVlv7IcYsVAuw8PqCMc8V7mvXHtA8tYgorMU8\nPM+Jfg3jPGnPbUru88A1NaMpKjIWf5WW2igpeZ709AleVl4bVcpsNvszXgVGYdTyjsGo8x2O8yEc\nsIeKitFERUW4PLOqarzHO2zYcAenTm2msXEyHR2tdHZ67iNft+6oS3GT5OTN5OU9isWS4LHCvbfj\nMUVk+NACMzEN9wBOSLjavXjMfZuSo0421/853uV7rdYZFBau6F6k1VNdbSOclwMP4Dhw4zxGr95+\nTSy1tR9y7twZl2c6/wJhr1V+773/Q2VlCq2t99HZGef1evf3nDDh1u6hbvf30l5qEQH1rMVE3Lcy\n/eu/dhET00RFxWjWrj3iUmTEfcjY4LywrAXn3qx9LrukpBqr1blKWTze64I7evUjRpyisvJx4G3g\nBSIj4/nqVyPIy3MMs3uOCuwBFpGQ8DyTJ6fS0HCW8vIUsrPfICmpvcfiJjq+UkS8UViLaTiOthxF\nQ8NZ3n13NE1NI4H5lJaOwbl2uMWSwNGjj/LUUwc5caKZrq6RjBr1X3z6adL178nEOQjtK8dd63I3\n0dLSRXGxZ4979OirhIe/CtTT1TWRq1dPYN9j3dlp429/20xj4z9Zu9Y+x96Ja489DhhDevpE4FPK\nyjZQWRlGWZmNhQt/1UPBEx1fKSLeKazFNOxBlZ2dT1mZY07Xfq6z+/ytxZJAdPQorNYngDCamowg\njI6OoqLimNeeqc3m8hG5uf9Gbu4ujh6toqnJ0eP+9NOP6ezcdP3j3TiOtQQIo7LyNpYuzae6ehoQ\nAdTg3LNPSDhDenqj28pv43urqpJU01tE+kVhLUHR2/GYnoVQjLlfb/O37td6C0LnZ9XWfkBl5SPA\nCUpLLZw6Vcirr36ZoqIyjCpmxtx3Z2eM030XERb2PDabYw82NFJdzfW2NQNfJyrqP/nsZ79w/ZeE\n5S7z0KrpLSKDobCWoOjteEz3cHPupdo5iqZ0Ar/AmKOezJkzZzl/fjqpqSlenwVZwHPAOowe8hKW\nLv0B7e3P4dqTHwf8DmNOu4nY2Gu0tPwEo6zoFYzKaP/h8j2xsVO89pg1Dy0ig6WwlqDo7XhMz3Bb\n7lIYpKHByoIFu64vLmsB/oF9q9XVqzbuv38zpaVrenyWUVrU8XFbW6rb12MxVoR/B8ee6k20tKzC\nqP8dS2Rkk8u2LIhlzhz7QjdXmocWkcFSWEvA+XIetfPQcF/h5r5PGTbjHLaNjZNdnlldfRqoAyYB\nTURHv097u+PZMTEfc/Wq4+Pw8L8QE3Mzra2OeyYm3sq8eYc5e3YkKSlW2tvDXY6wTE4u46WXHvXr\nvyNVLhMRO4W1BFxP51E79557Kh/qjWtP+Z/ANYzDMIxqYBbLRS9D369h1P22MW9eM7Gxjmd/97uZ\nfOtbRiETi+Ui+fnfIDfXtcb41KmfsnfvCurqjIM6GhutREc79/4fHVS49jYtICKisJaA8zbk7d57\ndi8f2ltYTZpUh2Pl9SGc545HjPgB+fkP88QT53Ad2o4HrEAR77wziowMG3v3Oupul5be7vKMvDxj\nq1hP88z+HtrubVpAREQVzCTgfKnK5R5Wb74J2dlv0Nho7b7GXiXs3XerMHrKBzAWejm+b8aMO0hN\nTfFS4awZo/DJclpbV7hUN3O/f0ZG8fUiLF9mz547AFi27CSf+cxmFizY79Euf1DlMhHpjXrWEnB9\nrYb2drBFa2sUhYXLgV0899yXWbfuKCUlnVitMcDNGNXGwFix7Riurq39kIwMSEq6wsKFO6iqGk9S\n0mWgg2PHYl3mod17r84nfZWWHqKk5C1Gjap2mR+/eHEPZWUr8PcwtVaMi0hvFNYyIN4WRCUmxvf6\n9Z7mdD0XjD0HrMIeqJ6lPH+CUdP7MAAjRjzDzTfPor7+LJWV36Gy0kJpqY2srF28+ebd3W2Jiemk\ntXU39pO2ej4cxCg9arWGYbXux3PPt/+HqbViXER6o7CWAfG2IMo4PrLnr/cURp5bq27FOBrTGA72\n/PoM4JcYx1oa27UmT95BRMStVFZauq9zPuXKOewTEp4nPX2i18NBjLY6lx5twbPmuIapReTGUljL\ngHhbEFVfbyU7e7+X86h774m6b+OaNOk0V69eBuppb48lKanNrUjKOVpaEl32OZ84EU56uvftYO5t\nnTLlZrZv77l4iethHwtJTt7MuHFpNDaeIyHhM0yb5jgFTFuuRORGUFjLgHjbJ716dZHP51E7y8tb\nQFvbDv7yl3CgHputDat1JRBGUZG3gy+Wc+edr2G1Ovd468nLM+a43ed9fS332dNhH/ZtWYmJD3Zv\n3bLTlisRuREU1jIg3vZJL1z4v7ifRz1lSkGfC6YslgRiYqKxWpdgzEN3YAR9JpDgtd73nDlxFBW9\nhrElq5k5c+KwWBJYv/4LLFu2n7//PYk//nEbqak3M2WKY7GZL4u3+jN/rC1XInIjKKxlQLztk25s\njMJ5fjc9PdLrcLOd8xCyMWy+H1iBo7e8B1jutSf80ktLiI4+SkXFNVJSOsnLWwzAsmX7XRarffTR\nHj76aEX3YrPBcB7m96USm4iIvyisxS+MHuV8jIAdQVTU/1FefgvZ2W/0OI9rDCHbe9MzgCqce6kj\nR3aQkbGrx+pm3nq/jY2TXe7hz9XbzsP8PVVi05YrEQkEhbX4hdHDHIOx//l3dHQ8S1lZGGVlrvO4\nnr3p/wYex3FutKOXmpFB9/nWvs4LWyyf0NoamNXb5887rxL3XolNRCQQFNbiF3l5CwgL28mxY9do\nauqgq8v7PK7nnukXcD43Oioql8jIz2CxXGTjxvsAKC8fhXNIfvzxKI/n238JGDNmOg0Nz2CzJREW\nVk1q6nTS0nb5pcebmtrMqVMa8haRG09hLT5raLDy1FN/vL5q+zJz5sTx0ktLsFgSsFgSiI6Oxmpd\njrE4zHuouS/IioyMp7PTfu0YOjpS6ej4Bq2tNnJzd7F9ewoNDX93uV99/VngHpe2uf4S8DWysnax\nffsK/Gnr1kza2jTkLSI3nsJafLZu3VEOH7YPWdsoKnqN6Oij3cPAjmHiTGDP9TlnXELNfUHWqFEN\nxMUZ+5g/+eQ8Vms29gM39u/v5NSpXxAbm4QxFx4HtDB2bIpH227EquyxYzXkLSLBobAWn3lWEoun\nouJa99cdw8QJwHIyMjznlh2FRzqxWkfQ1PQdmprGMHv2LqZOnUBh4Rjsq8BttjAqK22MGPEMsAl7\nwE+btsujbVqVLSKhTGEtPjMC0V6TOxb4gKQkxypvX4aJ7QuyMjKKKS1d2v35iorR7N17B7CLwsJm\nHD3pZq5ds7gVRfG8r1Zli0goU1iLz/LyFnDy5C+prjZqcsMSYEf31/szTOzeE66t/ZCHH4aUFBsx\nMVW0ta3u/lpExA/Yvv3fe72fVmWLSChTWIvPLJYEJk26jepqx1B4VdX4Ad3LuSdcW/uhy2lZ8fHb\naWtzPCM19TZ/NF9EZMhSWIvPvJ07PdC5YeeecEYGLqdlRURYcV79nZbWNui2i4gMZQpr8Zn7udPJ\nyZvJy3t00Pd1HxKfMyee6GjNP4uI2CmsxWfuq8EnTLgViyWhuyBJZaWF5OSGfh8T6bk4bLGOmRQR\ncaKwHqYGcg5zT9ujPKuS9e+YyIEsDtM50iIynCish6mBnMPc0/aoG3VMpHNA19Z+QGXlasCic6RF\nJOQprIcJ957oxx/H0t+A7akH3FOP29+9X9cefBbGXuyv+9x+EZGhSmE9TLj3pJOTc+mpfndf3EN4\n40ajmIkxZ93Y3eMeSO+9N54V1GKv/1kVy0QktAUsrLds2cLrr7/OuHHjAMjJyWHevHmBepz0wT3o\nxo6dwuzZA1tx3VMIJybGU1fX3OMzB9v7de/BJyeXMWFCl1aMi0jIC2jPeuXKlaxcuTKQjxAfuQfd\ntGnXBtzL9TWE+1uvu69hc88580e1qExEhoWAhrXNZgvk7aUf/Fk729cQ7u8z+xo2V0lRERmuAhrW\nu3fvprCwkNtuu43169cTHx8fyMdJL/wZdL6GcH+feaNWlYuIDDVhtkF0f1euXMnly5c9Pp+Tk8Os\nWbOwWCyEhYXx4osvUldXR25u7qAaKwNXX29l9eoizp+PIzW1ma1bMxk71lxDyA8//Dtef91Y3Q02\nvva1Pezd+/VgN0tEJOgGFda+unTpEqtWreLAgQN9Xuu8QCnUuC/AupGys/NdCpdkZfl/X/Jg36+x\n0cratUddeuxmmpMO5s8v0EL53UDvN9QNh/frS8CGwevq6khMTATgrbfeIi0tLVCPEh84hpitQBFv\nvgnZ2W+YqvKX5qRFRLwLWFj/9Kc/5aOPPiI8PJybbrqJH/3oR4F6lPjAsSisCFhOa2sYhYUD3/vs\nbeW2L78diohI/wUsrPPy8gJ1axmADRvu4NSpzVRVTcJmG/wiLm8rtwsKVviruSIi4kQVzIaJzZvf\nu3685Wv0VrnM3mMuL4+goaGCcePSmDr1isdwuVZui4jcOArrYcIRrpnAHkaO7CAjA49tV44e8x5g\nA5WVYbz/vudwube91vX1VrKz9+skLBERP1NYDxOOcE0AlpOR4Qhf5/nnf/yjEyOA43DuOZeXjyI7\nO9+jHrh95faGDV9g1qxfcfHiOvpbC1zHXYqI9E5hPUz0VsjE9TSr3RjD5M04D5dfvnyGsrKnsQdx\ne/sOfvObh7vvkZ2dz8WLtzKQoXF/H/ghIhJqFNbDRG/bolznnxeRkPA8kycn09Cw+fqc9accPZqA\ncxCfOBHu5R4tDOQkL81/i4j0TmEtbvPPY0hPn8j27fe5XJOWthXnIIZ6L/e4D2OuO5bk5DLy8h4d\nwPN13KWIiDuFtfhU63vOnDiKil4D4oFm5syJ87hHTMxhzp4dSUqKtV8nYvnzkBERkVB0Q8qN9keo\nl5Qbqu/nSynQofx+vgjl9wvldwO931A3HN6vL+pZB0ioVfhSKVARkeBRWAeIKnyJiIi/hPd9iQyE\nVjiLiIi/KKwDJCXlnxirpkErnEVEZDA0DB4gWuGsymQiIv6isA6Q/i7ICsVgU2UyERH/UFibRCgG\nm+btRUT8Q3PWJhGKwaZ5exER/1DP2iRCseSm5u1FRPxDYW0SoRhsKqQiIuIfCmuTULCJiEhPNGct\nIiJicgprERERk1NYi4iImJzCWkRExOQU1iIiIiansBYRETE5hbWIiIjJKaxFRERMTmEtIiJicgpr\nERERk1NYi4iImJzCWkRExOQU1iIiIiansBYRETE5hbWIiIjJKaxFRERMTmEtIiJicgprERERk1NY\ni4iImJzCWkRExOQU1iIiIiansBYRETE5hbWIiIjJKaxFRERMTmEtIiJicgprERERk1NYi4iImJzC\nWkRExOQU1iIiIiansBYRETE5hbWIiIjJKaxFRERMTmEtIiJicoMK68OHD7N48WJmzpzJBx984PK1\nbdu2kZGRwcKFC/nzn/88qEaKiIgMZ4MK67S0NLZs2cLs2bNdPl9eXk5RURGHDh1i+/btPPvss9hs\ntkE1VEREZLgaVFhPnTqVKVOmeARxcXExmZmZREZGMnnyZFJSUjh9+vSgGioiIjJcBWTOuqamhqSk\npO6PJ06cSE1NTSAeJSIiEvIi+7pg5cqVXL582ePzOTk5LFiwwOv3eBvyDgsLG0DzREREpM+w/vWv\nf93vm06aNImqqqruj6urq5kwYYJP35uYGN/v5w0ler+hLZTfL5TfDfR+Q12ov19f/DYM7tybXrBg\nAYcOHaK9vZ1PPvmECxcu8LnPfc5fjxIRERlWwmyDWKb99ttvs2nTJhobGxk9ejQzZszglVdeAYyt\nW/v27SMyMpKnn36au+66y2+NFhERGU4GFdYiIiISeKpgJiIiYnIKaxEREZNTWIuIiJicacN6x44d\nzJgxA6vVGuym+NXPf/5z7rvvPpYuXcrjjz9OXV1dsJvkV3l5eSxcuJCsrCzWrFlDS0tLsJvkN73V\nwh/Kjh8/zr333ss999zDyy+/HOzm+NXGjRuZO3cuS5YsCXZTAqK6upoVK1aQmZnJkiVL2LlzZ7Cb\n5Dft7e089NBDLF26lCVLlrBly5ZgNykgurq6uP/++1m1alWv15kyrKurq3n33XdJTk4OdlP87tvf\n/jb79++noKCA+fPnh9x/gHfddRcHDx6ksLCQlJQUtm3bFuwm+U1PtfCHsq6uLjZt2sSOHTv4wx/+\nwMGDBykvLw92s/zmgQceYMeOHcFuRsBERESwYcMGDh06xJ49e9i9e3fI/Pyio6PZuXMnBQUFFBQU\ncPz48ZAsW71z506mTZvW53WmDOvc3FzWrl0b7GYERGxsbPefW1tbCQ835Y9gwObOndv9TrNmzaK6\nujrILfKfnmrhD2WnT58mJSWFm266iaioKBYtWkRxcXGwm+U3X/ziFxk9enSwmxEwiYmJzJw5EzD+\n3zJt2jRqa2uD3Cr/GTlyJGD0sjs7O4PcGv+rrq6mpKSEhx56qM9r+6xgdqMdOXKEpKQkbrnllmA3\nJWBefPFFCgsLiY+PD6lhK3f79u1j0aJFwW6G9MJbHf/3338/iC2Sgbp48SJnzpwJqQJUXV1dPPDA\nA1y4cIFHHnkkpN4NHB3T5ubmPq8NSlj3VG/8qaeeYtu2bbz66qvdnxuKvZi+6qnn5OSQk5PDyy+/\nzG9/+1vWrFkThFYOnC/14rdu3UpUVNSQmyscSC38oWwo/v0ST1euXOHJJ59k48aNLqN3Q114eDgF\nBQW0tLSwevVqzp07x/Tp04PdLL84duwY48ePZ+bMmZw8ebLP64MS1j3VGz979iyXLl0iKysLm81G\nTU0Ny5Yt4/e//z3jxo27wa0cOF/rqS9evJgnnnhiyIV1X++Xn59PSUnJkBw1GEgt/KFs0qRJVFZW\ndn9cU1Pjcx1/MYfOzk6efPJJsrKy+MpXvhLs5gREXFwcX/rSl/jTn/4UMmH93nvvceTIEUpKSmhr\na+PKlSusXbuWvLw8r9ebasI0LS2Nd955h+LiYo4cOcLEiRPJz88fUkHdl4qKiu4/FxcXM3Xq1CC2\nxv+OHz/OK6+8wtatW4mOjg52cwImVHqkt99+OxcuXODSpUu0t7dz8OBB7r777mA3y69C5WfVk40b\nNzJ9+nS++c1vBrspftXQ0NA9PHz16lVOnDgRUv+//N73vsexY8coLi7mZz/7GXfeeWePQQ0mnLN2\nFhYWFnJ/0V544QXOnz9PeHg4ycnJPPvss8Fukl/9+Mc/pqOjg8ceewyAz3/+8/zwhz8MbqP8xLkW\n/qpVq1xq4Q9VERERPPPMMzz22GPYbDYefPBBn1amDhXf//73OXnyJFarlfnz57NmzRqWLVsW7Gb5\nzV//+lcOHDhAWloaS5cuJSwsjJycHObNmxfspg1aXV0d69evp6uri66uLjIzM0lPTw92s4JGtcFF\nRERMzlTD4CIiIuJJYS0iImJyCmsRERGTU1iLiIiYnMJaRETE5BTWIiIiJqewFhERMTmFtYiIiMn9\nPyQ+uNKCpR6MAAAAAElFTkSuQmCC\n",
-            "text/plain": [
-              "\u003cmatplotlib.figure.Figure at 0xa813090\u003e"
-            ]
-          },
-          "metadata": {
-            "tags": []
-          },
-          "output_type": "display_data"
-        }
-      ],
-      "source": [
-        "# Plot the Data (Optional)\n",
-        "\n",
-        "import matplotlib.pyplot as plt\n",
-        "\n",
-        "plt.scatter(inputs.numpy(), labels.numpy())\n",
-        "plt.show()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "JaFHyAG9nDET"
-      },
-      "source": [
-        "## Step 2: Define our TensorFlow variables\n",
-        "\n",
-        "We'll use Keras's object-oriented [`Dense`](https://www.tensorflow.org/api_docs/python/tf/contrib/keras/layers/Dense) layer to create our variables. In this case, we'll create a `Dense` layer with a single weight and bias.\n",
-        "\n",
-        "(**Note**: We're using the implementation of `Dense` found in `tf.layers.Dense` though the documentation link is for `tf.contrib.keras.layers.Dense`. When TensorFlow 1.4 is released, the documentation will also be in `tf.layers.Dense`) "
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 4,
-      "metadata": {
-        "cellView": "code",
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          },
-          "height": 34,
-          "output_extras": [
-            {
-              "item_id": 1
-            }
-          ]
-        },
-        "colab_type": "code",
-        "executionInfo": {
-          "elapsed": 22,
-          "status": "ok",
-          "timestamp": 1505502830753,
-          "user": {
-            "displayName": "",
-            "photoUrl": "",
-            "userId": ""
-          },
-          "user_tz": 240
-        },
-        "id": "z9r-ZeyrXu3A",
-        "outputId": "6230a7a3-29fe-4d08-f101-da80425bad82"
-      },
-      "outputs": [
-        {
-          "data": {
-            "text/plain": [
-              "[]"
-            ]
-          },
-          "execution_count": 4,
-          "metadata": {
-            "tags": []
-          },
-          "output_type": "execute_result"
-        }
-      ],
-      "source": [
-        "# Create TensorFlow Variables using Keras's Dense layer.\n",
-        "\n",
-        "wb = tf.layers.Dense(units=1, use_bias=True)\n",
-        "\n",
-        "# We can access the underlying TensorFlow variables using wb.variables.\n",
-        "# However, the variables won't exist until the dimensions of the input\n",
-        "# tensors are known. Once the dimensions of the input tensors are known,\n",
-        "# Keras can create and initialize the variables. Until then, Keras will\n",
-        "# report the variables as an empty list: [].\n",
-        "\n",
-        "wb.variables"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "docKLUaonYG_"
-      },
-      "source": [
-        "## Step 3: Define our loss function\n",
-        "\n",
-        "Our loss function is the standard L2 loss (where we reduce the loss to its mean across its inputs)."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "cellView": "code",
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          }
-        },
-        "colab_type": "code",
-        "id": "0_w8ZJSCtuY7"
-      },
-      "outputs": [],
-      "source": [
-        "def loss_fn(inputs, labels, wb):\n",
-        "  \"\"\"Calculates the mean L2 loss for our linear model.\"\"\"\n",
-        "  predictions = wb(inputs)\n",
-        "  return tf.reduce_mean(tf.square(predictions - labels))"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 6,
-      "metadata": {
-        "cellView": "code",
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          },
-          "height": 34,
-          "output_extras": [
-            {
-              "item_id": 1
-            }
-          ]
-        },
-        "colab_type": "code",
-        "executionInfo": {
-          "elapsed": 24,
-          "status": "ok",
-          "timestamp": 1505502830875,
-          "user": {
-            "displayName": "",
-            "photoUrl": "",
-            "userId": ""
-          },
-          "user_tz": 240
-        },
-        "id": "RkNbXoXkpjVH",
-        "outputId": "c36fc98d-3a57-4074-901d-c10ae017ae3f"
-      },
-      "outputs": [
-        {
-          "data": {
-            "text/plain": [
-              "\u003ctf.Tensor: id=40, shape=(), dtype=float32, numpy=7.3549819\u003e"
-            ]
-          },
-          "execution_count": 6,
-          "metadata": {
-            "tags": []
-          },
-          "output_type": "execute_result"
-        }
-      ],
-      "source": [
-        "# Test loss function (optional).\n",
-        "\n",
-        "loss_fn(inputs, labels, wb)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 7,
-      "metadata": {
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          },
-          "height": 51,
-          "output_extras": [
-            {
-              "item_id": 1
-            }
-          ]
-        },
-        "colab_type": "code",
-        "executionInfo": {
-          "elapsed": 57,
-          "status": "ok",
-          "timestamp": 1505502830981,
-          "user": {
-            "displayName": "",
-            "photoUrl": "",
-            "userId": ""
-          },
-          "user_tz": 240
-        },
-        "id": "K_7beXoHOU7t",
-        "outputId": "1ad0856a-02ec-4117-a6c0-b41030981d87"
-      },
-      "outputs": [
-        {
-          "name": "stdout",
-          "output_type": "stream",
-          "text": [
-            "w: tf.Tensor([[ 1.56891453]], shape=(1, 1), dtype=float32)\n",
-            "b: tf.Tensor([ 0.], shape=(1,), dtype=float32)\n"
-          ]
-        }
-      ],
-      "source": [
-        "# At this point, the variables exist, and can now be queried:\n",
-        "\n",
-        "w, b = wb.variables\n",
-        "print(\"w: \" + str(w.read_value()))\n",
-        "print(\"b: \" + str(b.read_value()))"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "YIlebeb_qYtC"
-      },
-      "source": [
-        "## Step 4: Create our gradients function using `implicit_value_and_gradients()`\n",
-        "\n",
-        "With a loss function defined, we can calculate gradients and apply them to our variables to update them.\n",
-        "\n",
-        "To calculate the gradients, we wrap our loss function using the `implicit_value_and_gradients()` function.\n",
-        "\n",
-        "`implicit_value_and_gradients()` returns a function that accepts the same inputs as the function passed in, and returns a tuple consisting of:\n",
-        "\n",
-        "1. the value returned by the function passed in (in this case, the loss calculated by `calculate_linear_model_loss()`), and\n",
-        "1. a list of tuples consisting of:\n",
-        "  1. The value of the gradient (a `tf.Tensor`) with respect to a given variable\n",
-        "  1. The corresponding variable (`tf.Variable`)\n",
-        "\n",
-        "Test it out below to get a feel for what it does. Notice how the first value of the returned tuple (the loss) is the same as the value returned in the cell above that tests our loss function."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "cellView": "code",
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          }
-        },
-        "colab_type": "code",
-        "id": "v1spZQ4NwW1U"
-      },
-      "outputs": [],
-      "source": [
-        "# Produce our gradients function. See description above for details about\n",
-        "# the returned function's signature.\n",
-        "\n",
-        "value_and_gradients_fn = tfe.implicit_value_and_gradients(loss_fn)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 9,
-      "metadata": {
-        "cellView": "code",
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          },
-          "height": 153,
-          "output_extras": [
-            {
-              "item_id": 1
-            }
-          ]
-        },
-        "colab_type": "code",
-        "executionInfo": {
-          "elapsed": 46,
-          "status": "ok",
-          "timestamp": 1505502831114,
-          "user": {
-            "displayName": "",
-            "photoUrl": "",
-            "userId": ""
-          },
-          "user_tz": 240
-        },
-        "id": "21WMcpsmFFLd",
-        "outputId": "f51b3171-33f5-4f87-8bf7-0be2dc8edc8a"
-      },
-      "outputs": [
-        {
-          "name": "stdout",
-          "output_type": "stream",
-          "text": [
-            "Outputs of value_and_gradients_fn:\n",
-            "Loss: tf.Tensor(7.35498, shape=(), dtype=float32)\n",
-            "\n",
-            "Gradient: tf.Tensor([[-3.00773573]], shape=(1, 1), dtype=float32)\n",
-            "Variable: \u003ctf.Variable 'dense/kernel:0' shape=(1, 1) dtype=float32\u003e\n",
-            "\n",
-            "Gradient: tf.Tensor([-4.06519032], shape=(1,), dtype=float32)\n",
-            "Variable: \u003ctf.Variable 'dense/bias:0' shape=(1,) dtype=float32\u003e\n"
-          ]
-        }
-      ],
-      "source": [
-        "# Show outputs of value_and_gradients_fn.\n",
-        "\n",
-        "print(\"Outputs of value_and_gradients_fn:\")\n",
-        "\n",
-        "value, grads_and_vars = value_and_gradients_fn(inputs, labels, wb)\n",
-        "\n",
-        "print('Loss: {}'.format(value))\n",
-        "for (grad, var) in grads_and_vars:\n",
-        "  print(\"\")\n",
-        "  print('Gradient: {}\\nVariable: {}'.format(grad, var))"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "JVDWpL9VYWdP"
-      },
-      "source": [
-        "## Step 5: Create an optimizer\n",
-        "\n",
-        "We'll use a `GradientDescentOptimizer` to fit our model."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "cellView": "code",
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          }
-        },
-        "colab_type": "code",
-        "id": "DudNEebMKDWN"
-      },
-      "outputs": [],
-      "source": [
-        "optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.1)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "YBeJYxY8YaiO"
-      },
-      "source": [
-        "### Step 5a: Test Our Optimizer\n",
-        "\n",
-        "Now we have everything needed to start fitting our variables to the data!\n",
-        "\n",
-        "In the next cell, we'll demo these capabilities. We'll:\n",
-        "\n",
-        "1. Print the current values of `w` and `b`\n",
-        "1. Calculate the loss and gradients\n",
-        "1. Apply the gradients\n",
-        "1. Print out the new values of `w` and `b`\n",
-        "\n",
-        "You can run the cell multiple times. Each time, you should see the values of `w` and `b` get closer to their true values of 3 and 2."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 11,
-      "metadata": {
-        "cellView": "code",
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          },
-          "height": 102,
-          "output_extras": [
-            {
-              "item_id": 1
-            }
-          ]
-        },
-        "colab_type": "code",
-        "executionInfo": {
-          "elapsed": 103,
-          "status": "ok",
-          "timestamp": 1505502831285,
-          "user": {
-            "displayName": "",
-            "photoUrl": "",
-            "userId": ""
-          },
-          "user_tz": 240
-        },
-        "id": "diDZfrMJM3OC",
-        "outputId": "d585fff0-ecb3-4e98-9b33-bbae07a95d8c"
-      },
-      "outputs": [
-        {
-          "name": "stdout",
-          "output_type": "stream",
-          "text": [
-            "Values of w, b, BEFORE applying gradients:\n",
-            "(array([[ 1.56891453]], dtype=float32), array([ 0.], dtype=float32))\n",
-            "()\n",
-            "Values of w, b, AFTER applying gradients:\n",
-            "(array([[ 1.86968815]], dtype=float32), array([ 0.40651903], dtype=float32))\n"
-          ]
-        }
-      ],
-      "source": [
-        "# Test the optimizer.\n",
-        "\n",
-        "print(\"Values of w, b, BEFORE applying gradients:\")\n",
-        "w, b = wb.variables\n",
-        "print(w.read_value().numpy(), b.read_value().numpy())\n",
-        "print()\n",
-        "\n",
-        "# Calculate the gradients:\n",
-        "empirical_loss, gradients_and_variables = value_and_gradients_fn(\n",
-        "    inputs, labels, wb)\n",
-        "optimizer.apply_gradients(gradients_and_variables)\n",
-        "\n",
-        "print(\"Values of w, b, AFTER applying gradients:\")\n",
-        "print(w.read_value().numpy(), b.read_value().numpy())"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "61TgeLVlKEQp"
-      },
-      "source": [
-        "## Step 6: Create a training loop\n",
-        "\n",
-        "Of course, now we can simply turn all of this code into a self-standing training loop. We'll also capture our loss and approximations of `w` and `b` and plot them over time."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 12,
-      "metadata": {
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          },
-          "height": 397,
-          "output_extras": [
-            {
-              "item_id": 1
-            },
-            {
-              "item_id": 2
-            }
-          ]
-        },
-        "colab_type": "code",
-        "executionInfo": {
-          "elapsed": 225,
-          "status": "ok",
-          "timestamp": 1505502831550,
-          "user": {
-            "displayName": "",
-            "photoUrl": "",
-            "userId": ""
-          },
-          "user_tz": 240
-        },
-        "id": "VukGe-huNaJ4",
-        "outputId": "f0a8d665-1910-477c-d8ab-c94ccdc4afcd"
-      },
-      "outputs": [
-        {
-          "name": "stdout",
-          "output_type": "stream",
-          "text": [
-            "[2.111051321029663, 2.3047544956207275, 2.4602210521698, 2.5850086212158203, 2.6851789951324463, 2.7655951976776123, 2.830157995223999, 2.8819968700408936, 2.9236228466033936, 2.9570505619049072]\n"
-          ]
-        },
-        {
-          "data": {
-            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAd0AAAFXCAYAAADnFpTQAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3Xd4FFUbBfAzu+m9koSShBQCSC+igIAgRRGkChJEiggo\nHURAEBQBQeADRcWCha50ULFLk6IivYRQQwskhPS6O/P9sckmm4Rkk2x2difn9zz7bLuZvC8JHO7M\n7FxBkiQJREREVOlUchdARERUVTB0iYiIzIShS0REZCYMXSIiIjNh6BIREZkJQ5eIiMhMjArdlJQU\njB8/Hk8//TS6d++OkydPVnZdREREiiMY8znd6dOno2XLlujbty80Gg0yMzPh4uJijvqIiIgUo9TQ\nTU1NRa9evfDbb7+ZqyYiIiJFKnX38s2bN+Hp6YkZM2agd+/emD17NjIzM81RGxERkaKUGroajQbn\nzp3DoEGDsH37djg4OOCzzz4zR21ERESKUmro+vv7w9/fHw0bNgQAdO3aFefOnSvxa3g5ZyIioqJs\nShvg4+ODgIAAXL16FbVr18aRI0cQGhpa4tcIgoC4uBSTFSkHX19Xq+8BUEYfSugBYB+WRAk9AMro\nQwk9ALo+jFFq6ALArFmzMHXqVGg0GtSqVQsLFy6sUHFERERVkVGhW7duXWzdurWyayEiIlI0XpGK\niIjITBi6REREZsLQJSIiMhOGLhERkZkwdImIiMyEoUtERCbRuXM7uUuweAxdIiIyCUEQ5C7B4hn1\nOV0iIqKy+OijFTh69BAEQYUhQ4ajU6fOuH8/HnPmzER6ehq0Wi2mTJmOJ59sgwUL3kZU1HkAArp3\n74nnn39B7vIrDUOXiEhh5s6dhd27d5h0mz169MLcue8aNXbv3t9x+XI01qz5Fg8eJODll4egadNm\n+PXXn9Cq1eN48cVhkCQJmZmZOH/+POLi7uGbbzYBANLSUk1at6Xh7mUiIjKp06dP4qmnugIAPD29\n0LRpc5w/fw716j2CH37Yha+++hyXLkXD0dERtWrVwp07t7F8+RIcPXoYTk7OMldfuTjTJSJSmLlz\n3zV6VloZCq80l/e8ceOm+Oijz3H48EEsWDAXAwcOxuDBA/D11xtx9Ohh7Ny5DX/88StmzHhLjrLN\ngjNdIiIyifxwbYbff/8VoijiwYMHOHXqBOrXfwSxsbHw8PDEs8/2wrPP9sLFixeQmJgIUdSiffsn\n8fLLoxEdHSVzF5WLM10iIjKJvLOX27d/EmfPnsbQoS9AEFR49dXx8PT0wp4932PjxrWwsbGBk5Mz\nZs16G7GxsXj99TcgSSIEQcDo0eNk7qJyCVIlrThv7esjKmmNR2vvQwk9AOzDkiihB0AZfSihB8D4\n9XS5e5mIiMhMGLpERERmwtAlIiIyE4YuERGRmTB0iYiIzIShS0REZCYMXSIismjHjx/DmTOn9M93\n7NiKn3/+0STbXrv2K5Nsx1gMXSIismjHjx/D6dP5odurV1907fqMSba9Zo15Q5dXpCIiogrbsGEN\n7O3t0bfvAHzwwVJcvnwJK1Z8gmPH/sGPP+7C7NnzDMZHRV3Ahx8ug0aTDWdnN7z55hx4eXlj8+ZN\n2LlzG2xsbBAcXBujR4/Fzp1boVbb4Ndf92DixNfx779/w8nJCQMHDsa4caNQp04ETp48gczMTMya\nNRdr136FK1cuo2PHzhg5cgwAYMaMqYiLu4fs7Cz07/8CevTohVWrViI7OwvDh0eidu0QzJ49D7/8\nsgebN2+CVqtB/foNMGXKdJOuE8zQJSJSGOe5s2Bv4qX9snr0QloJiyg0btwM3367Hn37DkBU1AXk\n5ORAq9Xi1KkTaNy4mcFYjUaD5csX4733liEsrBY2bdqGTz/9CDNmvIX167/Bli27YWNjg7S0VDg7\nu+C55/rqQxYA/v33b4Pt2dra4Ysv1mDz5k2YPn0KvvpqPVxcXDFgQC8MGBAJNzc3zJw5B66ursjK\nysLIkUPQvn1HjB49Ftu2bcaXX64HAFy/fg2///4LVq36Emq1GkuXLsIvv+wx2awaYOgSEZEJRETU\nRVTUeaSnp8PW1hYREXVx/vw5nDx5HJMmTTMYGxNzHVeuXMakSa9BrVYhO1sDHx9fAEBYWDjmzn0T\n7dp1wBNPdDDqe7dt2w4AEBoahpCQUHh6egEAqlevgXv37sLNzQ3ffbcBBw7sAwDcu3cPN2/GoH79\nBgYrIv3779+4eDEKI0cOgSRJyM7OhpeXV0X/aAwwdImIFCZt7rslzkorg42NDfz9A/Djj7vQsGFj\nhIWF4/jxf3H79i0EBQUXGi0hJCQUn3zyZZFrL7///gqcOPEfDh7cjzVrvsSaNd+W+r1tbe0A6BZc\nsLW11b8uCAK0Wi2OHz+G//77F5999jXs7OwwbtwoZGdnF7MlCd26dceoUa+V40/AODyRioiITKJx\n46bYuHEdmjRphkaNmmDHjq0ID69TZFxgYDAePEjEmTOnAeh2N1+9egUAcPduLJo2bY4xY8YhLS0N\nGRnpcHJyQlpaWrnrSktLhaurK+zs7HD9+jWcPXtG/56trS20Wi0AoHnzR7F37+948OABACA5ORmx\nsbHl/r7F4UyXiIhMonHjpli79is0aNAQ9vYOsLe3L3I8F9DNit99dxGWL38fy5cvQnZ2Dp5//gXU\nqhWId96ZnRuwEvr3HwhnZxe0adMOs2a9gb/+2o+JE183OLGppJOc8t5r1ao1duzYisGDn0dgYBAa\nNGioH9OzZ2+89NJARETUxezZ8/Dyy2MwefJrEEUJtra2mDx5Gvz9/U32Z8Sl/R5CSctNWXsfSugB\nYB+WRAk9AMroQwk9AFzaj4iIyOIwdImIiMyEoUtERGQmDF0iIiIzYegSERGZCUOXiIjITBi6RERk\ndt99txFZWVlyl2F2DF0iIjK7zZs3Iisrs9j3RFE0czXmw9AlIqIK27BhDbZu1V0n+YMPlmLCBN2S\neseO/YN582YbjN2yZRPi4+MwbtxovPTSSwCAzp3bYeXK5Rg2bBDOnDmF/v17Ijk5CQBw4cJ5jBs3\nCgCQmZmJhQvfwciRL2H48ME4eHC/uVo0CV4GkohIgbyaNyj29YRjZ4p9vazjCyvL0n79+g3Et99u\nxIcfforQ0BqIi0tBZmYGGjRoiLFjJ+aOMry8Y94lHb/5ZjWaN38UM2a8hdTUVIwcOQQtWz4Ke3sH\no+qUG0OXiIgqrCxL++lIuTcdtVqN9u07Fnq/qH/+OYpDhw5g48Y1AHSLJdy9G4vAwGCT9VKZGLpE\nRApk7Ay1vOMLK9vSfkXZ2dkbLF6gVqshirrgzc7OP+FKkiS8++5i1KoVWKF65cJjukREZBLGLu0H\nAE5OzgbL9RVeeycgoDqios4DAPbt+0P/+qOPPoYtWzbpn0dHR5myhUpn1Ey3Y8eOcHFxgUqlgo2N\nDbZs2VLZdRERkZUxdmk/AOjZsxemTh2PgAB/LFmyssgSfUOHjsR7770DFxcXNG3avMDrL+ODD5bi\npZcGAgD8/QOwaNH/Kq8pEzNqab9OnTph27ZtcHd3N2qjFy9ehKdnQIWLk5OSlpuy9j6U0APAPiyJ\nEnoAlNGHEnoATLy0nyRJZfrc1IABA5CTk2P0eCIioqrAqNAVBAEjRoxA37598d1335U6/sSJE/jw\nQ+uZ7hMREZmDUcd0N23aBF9fXyQkJGDYsGEICQlBixYtHjq+Ro0aWLp0Ebp164769R8xWbFERETW\nzKhjugWtXLkSzs7OGDZs2EPH/PDDD3j22WfRvHlzHDlyBDY2/GQSERFRqWmYkZEBURTh7OyM9PR0\nHDx4EGPHji3xa7p3747nn38B3323EXPnvosJE6aYrGBzUdLBfWvvQwk9AOzDkiihB0AZfSihB8D4\nE6lKDd34+HiMHTsWgiBAq9WiR48eaNu2bakbfvfd97Bv3594//2F6NatOyIi6hpVEBERkVKVeiJV\nrVq1sHPnTuzYsQO7d+/GK6+8YtSGPTw88f77y5GdnY0JE8ZAo9FUuFgiIrJMsbF3MGTIAJNuMzr6\nIg4f/kv//ODB/Vi//huTbFuupQUr9YpU3bo9g759n8d//x3DqlUfVea3IiIimRW+wEVFXbp0EUeO\n5Idu27btEBn5kkm2XdLSgpWp0s9wmj9/Efbv34tFi95F165PP/SSYEREZN00Gg3eeWc2Ll68gNq1\nQzFr1tuwt7c3GHPr1k0sW7YYSUmJcHBwwHvvLYCLiw/++OM3fP3151Cr1XB2dsHy5R/jiy9WITs7\nG6dPn8TgwcOQlZWJCxfOYdKkaViw4G3Y2dkjOjoKiYkPMGPGW9iz53ucPXsa9es3wMyZcwAAS5a8\nh6ioc8jKykKHDp0wfPgrBksLenh4YMWKT/D330fw5ZefIScnBzVq1MTMmXPg4GD6lYsqPXS9vLyx\nePH/MGxYJCZMeBW7d/8MtVpd2d+WiKjKmjvXHrt3m/af9x49NJg7t+TdsTEx1zFjxhw0aNAQCxe+\ng+3bN2PgwMEGYxYvXoBp02aiRo2aOHfuDObOnYslS1bim2++wLJlH8HHxwdpaamwsbHByy+PRlTU\neUyc+DoAYM+e7w1m06mpKfj0069w8OA+vPHGJKxa9RVq1w7BiBEv4tKlaISFhWPUqNfg6uoKURQx\nYcIYXLlyyWBpQTc3NyQlJWLNmi+xYsXHsLd3wPr132DTpnUYOvRlk/4ZAmZaZah79x7o1asPduzY\nhs8//wSjR5d89jMREVkfPz9/NGjQEADQtesz2LLlW4PQzcjIwJkzJzF79hsFFjjQ3Tds2Bjz589B\nx46d0b79k0Z9vzZtngAAhISEwcvLG7VrhwAAatcOQWzsbYSFheP333/Grl07oNVqkZBwH1evXkVI\nSBgKLi149uwZXLt2BWPGjIAkSdBoNGjQoFHF/0CKYbYP0C5YsAQHD+7HggXvoEuXbrlNExGRqc2d\nm1XqrLQyFD6mW/gQrySJcHV1w5dfrte/lveRoalTZ+D8+bM4dOggRox4EatXryv1+9nZ2QEAVCqV\n/nHec61Wizt3bmPTpvVYvXotnJ1dsGDB2wbLBObXJaFly8cwZ867ZWm3XMy2tJ+Pjw/ee28pMjMz\nMWHCa2W6ljMREVm+2Ng7OHtWty7vr7/+jEaNmhi87+TkjICA6vjzz9/0r124cAGA7lhvvXqPYMSI\nUfDw8MS9e3fh5ORksPxfSYq7zlNaWhocHR3h5OSMhIT7OHLkkEEtedt+5JGGOH36JG7dugkAyMrK\nxI0bMWXo3HhmvVRUz5690aPHduzevQOrV3+KkSPHmPPbExFRJQoKCsa2bd9h4cK3ERwcgl69+hUZ\nM2fOu3j//YX45psvodVq0LNnD/Tv/yI+/ngFbt68AQBo3rwlwsLCUa2aH9at+xrDh0di8OCHXwUR\nKP7M6bCwcISHRyAysh+qVfNDo0aN9e/lLS3o4+OLFSs+wcyZczB37kxkZ+dAEASMHDkGtWoFVvBP\npJg6y3oZSGM97AojcXFxeOKJlsjMzMSffx7S74O3NEq6Soq196GEHgD2YUmU0AOgjD6U0ANg4qX9\nTMnX1xcLFy5Beno6Jk0ay93MRERUZZg9dAGgV6++ePrpZ3Ho0EF8/fVqOUogIiIyO1lCVxAELF78\nP3h4eOCdd97C9evX5CiDiIjIrGQJXQDw8/PD/PmLkZ6ehsmTxxV75hkREZGSyBa6ANCv3wB06dIN\nBw7sw5o1X8lZChERUaWTNXQFQcCSJSvg7u6BuXNnVdrnooiIiCyBrKELAP7+AZg3byHS0lK5m5mI\nyEoZu7Tfnj3f4/79eDNUZJlkD10AGDBgEDp16ox9+/7Ehg1r5S6HiIjKwZil/X78cTfi4uKKfa8q\nfITUIkJXEAQsXfoBXF3d8NZbM3H79i25SyIiojLKW9pv8OD+mD17epFF4vfu/R0XLpzHvHmzMXx4\nJLKystCxY0d88smHGDHiRfz5528YN24UoqJ0l4ZMSkpE//49AegC+eOPV2DkyJcwdOgg7Nq13ez9\nmYJFhC4AVK9eA++8swApKcmYMmU8dzMTEVVA8+bOxd5MNb44MTHX0afP81i3bjOcnJywfftmg/c7\ndOiEevXqY86cd/Hll+v1a+26u3tg9eq16NSpSzFb1c2ev/9+J1xcXPH559/g88+/wa5d2xEbe6dM\n9VkCiwldABg06EV06NARv//+K779doPc5RARURkUXtrv1KmTRcZIkoTCc6pOnTqXuu2//z6Cn376\nAcOGDcIrr7yE5OQkqzz51qwLHpRGEAQsW/Yh2rV7DLNnz0CHDh3h7x8gd1lERFbn2DHjVucp7/ji\nlLa038M4OjrqH6vVakiS7thudnZ2gVESJk16HS1bPlbRMmVlUTNdAKhZsxbmzJmHpKRETJ06gbuZ\niYisRGlL+wGAs7Mz0tJSH7qNgIAauHDhHAAYLAH46KOPY9u2LdBoNACAGzdikJWVacryzcLiQhcA\nhgwZhieeaI9ffvkJW7Z8K3c5RERkhLyl/QYP7o+UlORil/Z7+ulnsWTJQv2JVIVnxy+8EInt27di\n+PDBSE5O1r/eo0cvBAfXxogRgzFkyAAsWbIQWq220nsyNbMv7WesmJjraNfuMdjZ2eLAgX/g5+dn\nosqMo6Tlpqy9DyX0ALAPS6KEHgBl9KGEHgALXtrPWIGBQZg9+20kJiZi2rRJ3M1MRERWz2JDFwCG\nDXsZrVu3xZ4932PHjq1yl0NERFQhFh26KpUK//vfSjg5OWHGjKm4d++e3CURERGVm0WHLgDUrh2C\nN9+cg4SEBMyYMVXucoiIiMrN4kMXAEaMGIVWrR7H7t07rPbSX0RERFYRuiqVCitWfAQHBwdMnz4F\n8fFVd4UKIiKyXlYRugAQEhKGGTPeQnx8PGbO5G5mIiKyPlYTugDwyitj0KLFo9ixYxt++GG33OUQ\nERGViVWFrlqtxooVH8Pe3h7Tpk1CQsJ9uUsiIiIymlWFLgCEh9fBG2/MQlzcPbz55htyl0NERGQ0\nqwtdABgzZiyaNWuOrVu/w08//Sh3OUREREaxytDV7Wb+BHZ2dnj99YlITHwgd0lERESlssrQBYCI\niLp4/fUZuHs3FrNnz5C7HCIiolJZbegCwGuvTUDjxk3x7bcb8OuvP8ldDhERUYmsOnRtbGzwwQef\nwNbWFlOnTkRSUqLcJRERET2UVYcuANSrVx+TJ0/DnTu3MWfOm3KXQ0RE9FBWH7oAMH78ZDRo0Agb\nNqzFH3/8Jnc5RERExVJE6Nra2uKDDz6BjY0NJk8eh5SUZLlLIiIiKkIRoQsADRo0xMSJU3H79i3M\nnTtb7nKIiIiKUEzoAsDEiVNRv34DrF37Ffbt+1PucoiIiAwYHbqiKKJ3794YPXp0ZdZTIXZ2dvjg\ng4+hVqsxefI4pKamyF0SERGRntGhu2bNGoSGhlZmLSbRqFETjB8/CTduxGDevDlyl0NERKRnVOjG\nxsZi37596N+/f2XXYxKTJ7+BunXr4auvvsDBg/vlLoeIiAiAkaG7YMECTJs2DYIgVHY9JmFvb48V\nKz6GSqXCxIljkZaWJndJREREsCltwN69e+Hj44N69erh6NGjRm/Y19e1QoVVVJcuHTBt2jS89957\nWLZsAT744IMyb0PuHkxFCX0ooQeAfVgSJfQAKKMPJfRgLEGSJKmkAcuWLcOuXbugVquRlZWFtLQ0\ndO7cGYsXLy5xw3Fx8p/ElJmZiaeeegIXL0Zh5849ePzxNkZ/ra+vq0X0UFFK6EMJPQDsw5IooQdA\nGX0ooQfA+P84lLp7efLkydi7dy9+//13LFu2DK1atSo1cC2Fg4MDli//CCqVChMmvIr09HS5SyIi\noipMUZ/TLU6LFo9i9OixuHbtKhYunCd3OUREVIWVKXQfffRRrFq1qrJqqTRvvPEmQkPD8NlnH+Po\n0SNyl0NERFWU4me6AODo6Ijlyz8GAEyc+CoyMjJkroiIiKqiKhG6ANCq1WN45ZUxuHz5EhYtmi93\nOUREVAVVmdAFgBkz3kJwcG2sWrUS//77t9zlEBFRFVOlQtfJyQkrVnwMURQxYcKryMzMlLskIiKq\nQqpU6ALA44+3wcsvj0J09EUsWfKe3OUQEVEVUuVCFwDefHMuAgODsXLlchw/fkzucoiIqIqokqHr\n7OyM5ctX6nczZ2VlyV0SERFVAVUydAGgbdt2GDp0BC5cOI///c86rrBFRETWrcqGLgC89dY7qFUr\nECtWLMOpUyfkLoeIiBSuSoeui4srli37EFqtFuPHv4rs7Gy5SyIiIgWr0qELAO3bP4kXXxyGc+fO\nYPnyJXKXQ0REClblQxcA5s6dhxo1amL58iU4c+a03OUQEZFCMXQBuLq6YenSD6DRaDB+/Bjk5OTI\nXRIRESkQQzdXx45PYdCgF3HmzCl8+OH/5C6HiIgUiKFbwNtvz4e/fwCWLl2E06e5m5mIiEyLoVuA\nu7sHli5dgZycHAwdOhSpqalyl0RERArC0C2kc+duiIwcgv/++w8DBvRGcnKS3CUREZFCMHSL8f77\nyzFo0CD8889R9O3bEwkJ9+UuiYiIFIChWwwbGxusWbMGgwa9iJMnj6N372cRFxcnd1lERGTlGLoP\noVarsWzZhxg+fCTOnz+LXr2exp07t+Uui4iIrBhDtwQqlQoLFy7Bq6+OR3T0RfTs2Q03bsTIXRYR\nEVkphm4pBEHAnDnzMGXKG7h+/Rp69uyGK1cuy10WERFZIYauEQRBwBtvvIlZs+bi1q2beO65pxEV\ndUHusoiIyMowdMtg/PjJmD9/Ee7ejUWvXk/j9OlTcpdERERWhKFbRiNHjsGSJSuQkJCAPn2exfHj\nx+QuiYiIrARDtxyGDBmGDz9chZSUZPTt2xNHjhyWuyQiIrICDN1yev75F/DZZ18hMzMDAwf2xoED\n++QuiYiILBxDtwJ69uyNr75aD41Gg0GD+uG3336WuyQiIrJgDN0K6tr1aaxb9x1UKhVeemkQfvhh\nt9wlERGRhWLomkCHDh2xceNW2NnZ4+WXh2Dbts1yl0RERBaIoWsirVu3xebNO+Ds7IIxY17Ghg1r\n5S6JiIgsDEPXhFq0eBTbtu2Gp6cnJk58DatXfyZ3SUREZEEYuibWqFETbN/+I3x9q2HGjKn4+OMP\n5S6JiIgsBEO3EtSrVx87d+5BQEB1zJ37JpYuXQRJkuQui4iIZMbQrSRhYeHYuXMPAgODsGjRfCxY\n8A6Dl4ioimPoVqLg4NrYuXMPQkJCsWLFUsyePZ3BS0RUhTF0K1mNGjWxc+dPqFu3Hj777BNMnToR\noijKXRYREcmAoWsGfn5+2L79RzRo0Ahr136FceNGQ6PRyF0WERGZGUPXTLy9vbFt2240b94Cmzdv\nwujRI5CTkyN3WUREZEYMXTPy8PDE5s078fjjbbBr13YMHz4YmZmZcpdFRERmwtA1MxcXV2zcuBXt\n2z+Jn3/egyFDBiI9PV3usoiIyAwYujJwcnLC2rXfokuXbti79w8MGtQPqakpcpdFRESVrNTQzc7O\nRv/+/dGrVy/06NEDK1euNEddiufg4IAvv1yHHj164dChg+jfvxeSkhLlLouIiCqRTWkD7OzssGbN\nGjg6OkKr1eKFF15Au3bt0KhRI3PUp2h2dnb49NMvYW9vjy1bvkWfPj3w3Xc74O3tLXdpRERUCYza\nvezo6AhAN+vlR11My8bGBitXfooXXxyK06dPok+f7rh7967cZRERUSUodaYLAKIook+fPoiJiUFk\nZGTps9zgYHiJRa+8lHDsTLHDvZo3KPZ1WcerhCI9VGY9XwFwGDkan3++Cr16PY2tW3ejevUaFd9+\ngT6s6s+/oNweLKaeco5HzHWLqofjOd4SxisiL4CH/v0uzKjQValU2LFjB1JTU/Hqq6/i0qVLCAsL\nK/Fr1CqhyGu+vq4P+QZFx1rC+MI9VHY9n376Mby83LFo0SL07v0M/vjjDwQHB1d4+3l9yP3nWZHx\napVgUfWUZ/xDv8ZK6i843uBrLaCe8ozXP7eQeso7vrh/a+Wsp8zjoYy8MJYglfFiwCtXroSzszOG\nDRtW4ri4OOs+G9fX11WWHiRJwtKli7B48QJUr14D27btRkhIyf/BKYlcfZiSEnoA2IclUUIPgDL6\nsPgeRBHIzISQmQEh9x4ZmRCyMiFkZgKZGRAyMuE+dJBRmyt1ppuQkABbW1u4uroiMzMThw8fxiuv\nvFLhPqh4giBg6tTpcHBwxDvvzEbPnk9jy5ZdqFu3ntylERHJy8gAzHtfN7bg89z3szLzt5M7Hpm5\n28nIHZf3fna2cbWZKnTj4uIwffp0iKIIURTxzDPPoH379sYVQeU2duwEODo6YMaM19G79zP47rsd\naNiwsdxlEREVJUlAdjaE9DQI6em5tzT9PdLTIaQ95D1JA9fEFMMAzMrSPy9XAJa1fEEAHB0hOThA\ncnCE5OICyccXkoM9JAdHIO91BwdIjo6Avb3hcwcHuBj5vUoN3YiICGzfvr2CLVF5jBgxCg4Ojpg8\neRz69OmBTZu2onnzlnKXRUTWSJKAjIwioWd4nw4UfC2taEgWHZf7ulZb7tIcCpZZXAB6+0BydCga\ngA4ORQPRwQGSfe57jo4FxjoCDvaGz/O2aWsLCGU7NluYyUKX5BUZOQQODg4YO3YU+vV7Dhs2bMbj\nj7eRuywiqkyiqAuylBQIqakQUpINH6emQJWaCkg5cI5/UCQQ9Y/T8h8jIx2CCdbzllQqSE7OkJyc\nACcniN4+kJyc9K9JTk6QnAs8dnIGDN43fM+rpi/i00VdANo7AHZ2FQ5AS8bQtQJ9+z4POzt7jB49\nHAMH9sGaNZvQvv2TcpdFRAVJku64YEpKbiimFB+aqbrHKv17KbrX8h6npEBISzU6IJ2KK8XWVh9u\nors7pIDqucH38PArGJYlhSTs7U0bir6ukCz5RCoTY+haiR49noODw3oMH/4iBg9+HqtXr0GXLk/L\nXRaR9cvJyZ095oeeKi0lPwALhmZaam5gFgzRlPyvL+fFgyQ7O0iurpBcXCEGBUN0dc197gLJxS3/\nsasrJFfc0+i4AAAgAElEQVQ3iC4ukFxc4FHTDwlZAJxzw9HRUReMtram/TMik2HoWpHOnbth/frN\nGDJkIIYOjcSnn36JHj16yV0WkbxEEUJyEoTERKiSEiEkJkJISoQqMbH415ISgdRkeCcl6YKynMtr\nSmo1JBddOIoB1SE560JRdHXLD0gXV/2Y/OB0g+iS/1hycdHNHsvD1xXaKjRLVAKGrpVp164DNm3a\nhkGD+mPkyKH48MNV6N9/oNxlEVWMkcGpSnxQJECF5KQyHauUnJwAd3eInl6QagUWmUnqQzMvLAuG\npqsrRGfdPRwdFX3skSoHQ9cKPfZYa2zZshMDBvTB2LGjkJWVhcGDX5K7LKrqSgzOB/qQzAvSigan\n6O4BsXp1iPXqQ/LwgOTuAbHQveThAdHDE5KHJ0R3D0ju7oC9PXx9XfGAM0SSAUPXSjVr1gLbtn2P\n559/DpMnj0NmZgZefnm03GWRUmRkQBUfB9X9eKjux0OIj4fq/n2o7scDmalwi40zTXB6eEKsXgNi\n/UfyQ1Iflh6FXvPUvwc7u0psnqjyMHStWMOGjbBjxx707dsDM2dOQ0ZGJsaNmyh3WWSJ0tL0AaoP\n0fgCz+/H54bsfaji43UXLShB3hFIyckZoocHg5PISAxdKxcRURe7du1B3749MW/eW8jISMfrr8+A\nwGNNyiVJhiEaHwchNyzzn+cFqm52KqSnl75Ze3uI3j7QhIVD8vaG6O2ju/n4QPLxzX3uDc/QWojX\n2up21TI4icqEoasAISFh2LlTN+NdsuQ9ZGZmYvbstxm81kKSdB9FiYszDMr4eMNdvPfv658bc8at\n5OCgC9HwiEIh6gvJx0cfonnPJWcX404MqmKfqyQyJYauQgQGBmHXrp/Qt28PrFy5HBkZ6Zg/f7Hc\nZVVdkqQ7eSg2Fqo7t6G6GwukJcL5+q1Cx0lzH2dllb5JR0eIPr7Q1K2nuwpQboDqZ6O5AZoXrnB2\n5tm1RBaGoasgAQHVsWPHHvTv/xxWr/4MWVlZ+Prr1XKXpTzp6VDF3oH6bm6g6oP1DtR37kAVeweq\nu7HFzkYLXj1IcnKG6OMDTf1HdCFaIDAfGqJEZNUYugpTrVo1bN/+PQYM6IN1677BvXt3MG/eYtSu\nHSJ3aZZPo4Hq3l1daBYIT/Wd27rHsXd0AZuU+NBNSCoVRN9qutmof4D+pg2oDrewIDywdc4PUafi\nLuBHRErG0FUgLy9vbN26C6+8Mgy//PIL9u/fjwkTpmDs2ImwL++Vb6yZJEF4kKAL0oKz0dhYqGIL\nzFTj7pX4kRfRwwNiQAA0TZvlBmkARL8AiAHVIfr76+59fAGbh/y18nWFhsdCiao0hq5Cubm5Y+PG\nrfjzzz2YMGEiFi2ajy1bvsWiRcvQrl0HucsznbQ0qO8WmJnmBqsqNm+GGgvV3TslHjOVHBwg+gcg\np9XjEIsJUq2fP0T/AN0ViIiIKoChq2CCIGDAgAFo0aINFi2aj9WrP0O/fj3Rp08/vP32Qvj5+cld\n4sPlnoikvhEDJMXB4eKV/BlqgWBVJSc9fBMqFUQ/f90xU78AXaDm7uoV/fz1wSq5e/CEIyIyC4Zu\nFeDm5o758xdjwIBBmDZtErZt24Jff/0FM2fOxtChL0OtVstTWFoa1DdioI65BlXMdaivX4c6RndT\nxVyHKiVZP9S10JeKnp4Qa9SEpnkLaP0Dip2hij6+gFy9EREVg6FbhTRq1AQ//PAb1q79GvPnv40Z\nM17Hpk0b8P77/0OTJs1M/w2zs6G6dVMfpLowvaZ7fP06VPFxxX6Z5OQEbWAQcgJbQxsYBKd6dZDs\n6gWtf26g+gcADg6mr5eIqJIxdKsYtVqNoUNH4JlneuDtt2dh8+ZN6Nr1SQwdOgIzZ74Fd3cP4zcm\nirqPzsRch+r6NYNZqjrmOlR3bkMQxSJfJtnaQluzFjSPNIA2MAjawCCIuffawGBIPj4Gu3udfF2R\nxROQiEgBGLpVVLVq1fDRR59h0KAXMW3aJHz11Rf4/vtdePvt+ejb93nd1awkCUJCAtS5s1OVfvdv\n7u7gmzcgZGcX2bYkCBADqiPn0ccKhGkQxKBg3b1/AHf7ElGVxNCt4to2boIDH32OXz/7GCd3bkPW\nqyNxcdZ0NPX0hGNsLFRpqcV+nejjkztTDS4UrEHQ1qhV/kW5iYgUjKGrdFlZUF+OLjBLzdv9mzt7\nTUgAAAzOvQEAEu4jOeE+Yn184dGmLVA7JDdYdTNVba1AwMVFro6IiKwWQ1cJJAlCfDxsoqOgvhgF\ndXQUbC5GQX0pGrh9C17FXPBBsreHtlYgNI2b5odpkC5Qf4m+iCnz38btO7cReOEC3hs6Ak891VWG\nxoiIlIWha01EEapbN2Fz8QLUFy/mh2t0FFQPHhQZrq1eA2jfHhkBNQ1OVBKDgiBW8wNUqmK/Taem\nzXHwmR5YunQRPv30Iwwa1B/du/fEu+++hxo1alZ2l0REisXQtUQ5OVBfvQL1xagCs9eLsLl0sci6\nqJJKBW1wbeS0ehza8Aho6kRAWycC2vA6kFxc4evritRynPnr4uKCOXPm4fnnX8C0aZPwww+78Oef\nv2PatJkYOXI0bG1tTdUtEVGVwdCVU1oabC5dzA/V3Fmr+uoVCBqNwVDJwQHa0HBo6tTJD9fwCGhD\nQiv1pKV69epj5849+PbbDXj77VmYO/dNfPvtBixe/D+0avVYpX1fIiIlYuiagXD/ftHjrdEXob55\no8hY0d0DmibN8kO1Th1owiMg1gqU7WM2KpUKL7wwGF27Po13352Ldeu+QY8eXRAZOQSzZ78NLy9v\nWeoiIrI2DF1TkSSobt8qsEv4ItQXL8AmOgqq+/eLDNf6+SP7iQ76UNXWiYAmPAJStWoWex1gLy9v\nLFv2IQYMiMS0aZOwfv0a7NnzPd56ax4GDoyE6iHHiImISIehW1YaDdTXrhaatUZBHR1d5DOtkkoF\nMTAIWc1bFtglXAfaOhGQ3NxlaqDiWrV6DL/9th9ffPEpFi2aj4kTX8OGDWuxePH/UL/+I3KXR0Rk\nsRi6D5OeDpvTJwuEq+5sYfWVyxBycgyGSnZ20IaGI7tAqGrCI6ANDVPsNYJtbW0xZsxYPPdcb8ya\nNR3ff78TnTq1xahRr2Hq1Olw4ed4iYiKYOgCEFKSYXPqJGxOnoDNqeOwOXkCuHIZnoU+3yq6uELT\nsBG0deoW2CVcB2JQcJW9rGH16jXw5Zdr8dtvP2P69Nfx8ccfYMeOrZg/fzGeeeZZ3eUkiYgIQBUM\nXSE5CTanT+kC9uR/uvsrlw3GiG7uQLt2yKgdVuCEpgjdNYMZIsV66qmuOHCgHVasWIIPP1yOYcMi\n0blzVyxY8D6CgoLlLo+IyCIoOnSF5KQiM9giAevugewn2kPTqAk0TZoip1ETiMG14VvNrVyfb63K\nHB0dMX36bPTtOwBvvDEZv/76Mw4e3I9Jk17Hq6+Oh52dndwlEhHJSjGhW6aAbdwUmsZN9AHL2atp\nhYfXwdatu7F163eYM+dNLFjwDjZv3oRFi5ahbdt2cpdHRCQbqwzdIgF74jhsrl4xGKML2A7QNG7C\ngJWBIAjo128AOnfuioUL5+Grr75Anz7Pol+/AZg7dz6qVasmd4lERGZn8aGrD9gTx/NnsKUFbOOm\nupObGLCyc3f3wHvvLcWAAYMwbdpkbNnyLX755Se8+eYcDBkyDOoqegIaEVVNFhW6QlJi0V3EJQRs\nTpOm0DRqwoC1Ak2bNsdPP/2Br79ejQUL3sEbb0zGpk3r8P77y9GoURO5yyMiMgvZQteogPXwQHa7\nJ3Nnr00YsFZOrVZjxIhX8OyzPTFnzkxs27YFXbp0wPDhIzF9+iy4WfEFQ4iIjGGW0DUI2JPHYXvy\nONTXrhqMYcBWHX5+/li16ku88MKLmD59Cr744lPs2rUD8+YtRK9effnZXiJSrMoJ3T/+gOPev2Bz\n6oRxAdu4KcTAIAZsFdO+/ZPYu/cwVq5cjuXLl2DUqOFYv34tFi1agtDQcLnLIyIyucoJ3U6dkHcR\nQIOAzTsGy4ClXPb29pgy5Q306dMfM2ZMxR9//Ib27R/HuHGTMGHCFDgo9DKaRFQ1VU7oTp+OpPD6\nDFgyWu3aIdi4cSu+/34XZs16A0uXLsLWrd/lnvncW+7yiIhMotS12GJjYzFkyBA888wz6NGjB9as\nWVP6VhcuRHaPXjwmS2UiCAJ69HgOf/31D0aNeg03bsRg4MA+6NevH44d+wdSoWthExFZm1JDV61W\nY8aMGfjxxx+xadMmrF+/HpcvXy7ty4jKzcXFFfPmLcSvv+5HixaPYuvWrXj66U7o0OFxfPbZx0hI\nKLo+MRGRNSg1dH19fVGvXj0AgLOzM0JDQ3Hv3r1KL4yoQYOG+P77X/DTTz+hZ8/euHQpGrNmTUej\nRhEYNWoY9u/fC1EU5S6TiMhoZTqme/PmTVy4cAGNGjWqrHqIDKhUKnTt2hXNmrVGfHw8Nm/ehHXr\nvsb27VuxfftWBAUFIzJyCAYOjIS/f4Dc5RIRlUiQjDxQlpaWhhdffBGvvvoqnnrqqRLHBgej2BnI\nsWNpxY5v3ty52NflHK9SqYr0YE315ynYhyXUU57xeT3kjZckCX//fRTr13+DXbu2Iz39LADAwcER\nLi4ucHBwgCAIFlN/npgYFeKKWbnK0v/8C4/39XU16EPuesozvmAPllBPecf7+roiMLD4vT3WUD8A\ntGzpavV5Aej+fhvDqJmuRqPB+PHj8dxzz5UauHlUqqIF+Pq6PmRs8duQe3zhHuSup7zj8/qwlHrK\nM16lUhmMf/bZznj22c5ISkpCSIgKqakpyMzMQGZmBtRqNVxcXJCUdB9hYWEWUX9JX2MNf/6Fxxd8\nbAn1lGd83nNLqaf844v/AmupX/c11p8XxjJqpjtt2jR4enpixowZRm+4uP/RW5PC/5u3Vkrow9ge\nTp8+hQ0b1mDLlu+QlJQIAGjbth0iI4ege/eesn/mVwk/C0AZfSihB0AZfSihB6Dk/1QUVGpGHzt2\nDLt378aRI0fQq1cv9O7dG/v3769wgUSm1rBhIyxcuASnTkXh448/R5s2T+Dgwf0YM+ZlNGpUBzNn\nvo6zZ8/IXSYRVWFGH9MtK2v/n4uS/vdl7X1UpIcrVy5hw4Z12LhxHeLidGfdN2vWHJGRL6F3775w\ncTHuf6emoISfBaCMPpTQA6CMPpTQA2DCmS6RNQsJCcOsWXNx4sR5fPPNRnTp0g0nThzHlCnj0aBB\nHUyc+Br++ecoL7xBRGbB0KUqwdbWFk8/3R3r1n2H48fPYcaM2fDx8cWGDWvRvXtntGvXCqtWrcT9\n+7zwBhFVHoYuVTkBAdUxadLr+PvvE9i8eSd69eqDq1ev4K23ZqJRozoYOXIo9u79gxfeICKTk20R\neyK5qVQqtG//JNq3fxL379/Hli2bsH79GuzcuQ07d25DYGAQXnhhMF54YTCqV68hd7lEVMmys4H0\ndCA9XShwr3uclpb/WkZG0TEbNxr3PXgi1UMo6eC+tfdhzh4kScKxY/9g/fo12L59K9LT06BSqdCx\n41OIjHwJXbp0g62tbbm2rYSfBaCMPpTQA6CMPsrSgyiimMDLv8/IKDksC79XOEA1mvIv0GNsknKm\nS1SAIAho0eJRtGjxKObNW4gdO7Zh/fpv8Ntvv+C3336Br281DBgwCIMHD0FISNELbxCRjiQBaWlA\naqqAlBQBKSmGj9PSdI+1WiA+3v6hQVg4LE1BpZLg5AQ4OenuvbzEAs91rzk7S3B0zB9jeF/0NehX\nkS8ZZ7oPoYT/QQLK6MMSejh37iw2bFiDzZs34cGDBwCA1q3bIjJyCJ599jk4OjqWug1L6MMUlNCH\nEnoATN+HJAFZWXnhmB+SqanIDUvd4/zwzH+vuK+RpPKHpL19ySFX8N7R0XCMs/PDw9LREbC3N/2q\ns8Z+ZIih+xD8S2k5LKmHzMxM7NnzPdatW4MDB/YCANzc3NGv3/OIjHwJDRs+fDEQS+qjIpTQhxJ6\nAPL70GhQKAx1j4ubZRYOybzHea/n5JQvjezsJLi6SnBxAVxcdI9dXQFXVwnOzvmPdWN0z11cJNSs\n6YTs7LQiM0y12sR/WJWMoVtBSvtLac0stYdr165i48a12LhxPWJj7wAAGjduisjIIejTpx/c3NwN\nxltqH2WlhD4srQdJ0h2rTEwUkJgoICkp7x548CD/eeH309JUSE6Wyr3bVRAMw9DZuWAwokBA5j/P\nC1NdkOaHp719+Xq3tJ9FeTF0K0hJvwjW3oel96DRaPD7779i/fpv8OuvP0Or1cLR0RE9e/ZGZORL\naNXqMQiCYPF9GEsJfVRWD5mZKBSQMAjJwqFZ8P3sbOOD09ZWgru7BC8vFRwdtUVmjwXDMO/1ggGa\n956Tk+l3s5aVEn6fAIZuhSnpF8Ha+7CmHmJj7+Dbbzdg/fo1uHbtKgAgLCwckZEvYeTIobCzc5O5\nwoqzpp/Hw5TUQ04ODELz4YFZ9P3MTOMTTK2W4OEhwd0d8PCQ9Dd3d6nQ86Lv54Wl0n8W1oShW0FK\n+kWw9j6ssQdRFHHo0EGsW/cNfvhhF7KysgAAoaFhaN26rf4WEFBd5krLzlp+Hnlnz8bHC7h/X0B8\nvID4eBXu3xeQnm6PO3dy9KFZcBduWXbVCoIuFN3dJXh65gem4fOi73t46HbXVnSWaS0/i5IooQeA\noVthSvpFsPY+rL2HBw8SsG3bZhw48Cf27z+A1NT8XkJCQvUB3KbNE1YRwnL+PDIzDUM0Li7vsapQ\nuOoeZ2QYl2pubg+bZepC82GzUFfXsq+nakrW/ncDUEYPAEO3wpT0i2DtfSihB0DXx507D3DmzCn8\n9ddBHDp0AEeOHEZKSrJ+TO3aIWjT5gk8/ngbtGnzhEVeCcuUP4+cHCAhQReexYWmLlhV+sepqaWH\nqL29BB8fw5u3twQfH1H/PDTUCZKUCg8PCW5ugI2VXrFACX83lNADwNCtMCX9Ilh7H0roASi+D61W\naxDChw8fMgjh4ODaBiFco0ZNc5ddREk/D61Wd7Zt4QDNn5EWfE+FxMTSQ9TGJi808wPU17domOa9\n7uxc+m5bJf9OWRsl9AAwdCtMSb8I1t6HEnoAjOtDq9Xi7NnTBiGcnJykfz8oKNgghGvWrFXZZUOj\nAe7dE3DnjoDYWBUyMx1x7VpWkRlpfLyAhAQBolhy4qlUEry8Cs9Ciz7OC1N398q5kEFV+Z2ydEro\nAWDoVpiSfhGsvQ8l9ACUrw+tVotz587gr78O4NChgzh8+BCSkhL17wcGBqNNm/wTs2rVCizT9tPS\ngNhYAbdvq/SheueOgNu38x/fu1d6kHp46EKy5Bmp7ubpKcl+4YOq/DtlaZTQA8DQrTAl/SJYex9K\n6AEwTR+6ED6LQ4cO4K+/DuLw4b8KhXAQWrdui8cea4v69dtDrQ7MDVEVYmMF3LmjC1LdTYXk5IeH\nqZ2dBH9/CQEBIgICJAQESPD3FxEW5gBb23T4+OhC1ctLQjnXgJANf6cshxJ6AIwPXSs9fYCoalKr\n1ahTpxHc3BqjcePx6NVLwokT93DqVDyuXMnErVu22LTJD5s21QBg99DtuLtLqFFDRPPmulD195dQ\nvXr+44AA3ey0uN26vr4OiIvTVl6TRArG0CWyEJIEJCejwK5e3Wy04K7e2FjdCUiGaufedMdLfXyy\n4eAQj5yca0hMPI2srCsAbgG4CT8/EW3ahKB9+1Zo3botAgODIMh9SSKiKoShS2QGWi1w6xZw5oyq\nwK7egrt7da+VdGEGJyfdDLRuXU3uzFTM3eWrm6FWr67b3as7XuoKoCFE8RGcP38Ohw8fxF9/peDw\n4YPYtu0Atm37BgBQo0ZN/WeEW7dui6CgYIYwUSXiMd2HUNJxBmvvw1p6SEkBrl9X4do1Fa5fF3D9\nukr//ObNkldv8fExPG4aEKAL1bxdvQEBItzcKn4WryiKuHDhfG4IH8Thwwdx//59/fs1atTUnxnd\nunVbBAfXLhLC1vLzKIkSegCU0YcSegB4IlWFKekXwdr7sJQetFrdmb7Fher16wLu3y/+0kQ+PiKC\ngiSEhqrh5ZVtcGJSQIAIP7/yr9BSUaIoIirqAg4dOoBDh/7CoUMHDEK4evUaBiFcu3YIqlVzs4if\nR0VYyu9URSmhDyX0ADB0K0xJvwjW3oc5e0hNhT5M84JVF6oq3LhR/EowtrYSAgMlBAWJ+ltwcP5z\nFxfz91FekiQhKuoC/vrrAA4f1oVwfHy8/n1//wA0bdoEgYEhqFMnAuHhEQgPrwNvb28Zqy47a/hZ\nGEMJfSihB4BnLxMVSxR1s9W8UL12LT9Ur18v7iQlHW9vEQ0aFAxV3ew1KEg3a5X7c6emIggC6tat\nh7p162HEiFcgSRIuXozSh/CRI4ewZ8+eIl/n7e2tD+D8WwRq1qwFlZwXJyayMAxdUpz0dBjMVAvu\nAo6JUSErq+hs1cZGQq1aEho00BQJ1aAg3fHUqkgQBERE1EVERF0MHz4SAGBjo8GRI/8hOvoiLl6M\nwqVLuvu//z6CI0cOGXy9o6MjQkPDUadOnQKhHIGQkFDYy7VPnUhGDF2yOpIE3L1reGy14Gz13r3i\nZ1aenhLq1Ss6Uw0K0p35a60XvTc3T09PtGjxKFq0eNTg9czMTFy9egXR0VEFwvgiLl+OxpkzpwzG\nqlQqBAUFo06dCISF1cndVa2bIbu7e5izHSKz4j8zZJEkSbcb+MIFFWJjgTNn7PWhGhOjKnbJNrVa\nQs2aEtq10+hDVXevu7m7y9BIFeLg4IB69eqjXr36Bq+LooibN2/khvFF/cw4OjoKP/+8Bz//bLi7\nulo1v9wwDjc4bhwQUJ0fZyKrx9AlWUmS7mL6Fy6oEBWlu124oMbFiyokJRX8B1Z3dSU3Nwnh4WKB\nMJX0M9caNThbtUQqlQqBgUEIDAxCp05dDN67f/8+oqOj9Luqo6OjcOlSNA4e3I+DB/cbjHV2dkF4\neDjCwyMMZsjBwbVha23XoaQqi/9EkdnExeWHa37Iqoss76ZWSwgJEfHEEyIiIkS0bGkPb+80BAWJ\n8OCeR0Xx9vaGt3drPPZYa4PX09PTcflydIEw1s2Qz507ixMnjhuMtbGxQe3aIQXCOFx/7+Ji3Bml\nRObC0CWTu39fKBSsulvhz7GqVBJq15bQurUGdevqAjYiQkRoqGjwuVVfX3vExYlm7oLk5OTkhIYN\nG6Nhw8YGr2s0GsTEXEN0dLTBSVzR0RcRHX0RP/6422B89eo1DM6mzpsh+/i4mLMdIj2GLpXbgwdA\nVJS60K5hVZGP3QiChKAgCS1b5hiEa1iYCAcHmYonq2RjY4OQkDCEhISha9en9a9LkoR79+4VOYkr\nOjoK+/b9iX37/jTYjouLC/z9A+DvHwA/P//cez+D1/z8/OHk5GTuFknhGLpUqqQk4MIFtUGwRkWp\nij1LODBQRJcuGkREaBERIaJuXV248t8uqkyCIMDPzw9+fn5o27adwXupqSkFPt6kmyHfvHkdt2/f\nxqVL0SVu193dA/7+/vDzC4C/v39uKPvnhnL+Y378iYzF0CW9lBTkBqraIFxjY4uGa61aIp56SpM7\na9Wibl0R4eEinJ1lKJyoBC4urmjatDmaNm2ufy3vKkhZWVm4d+8u7t6NRWxsLO7evYPY2FjExt5B\nbOyd3NfvICrqQonfw8vLq5hgDjAI6WrV/HjCFzF0q6LUVODixfwzhfPC9fbtouFao4aIjh01ubNW\n3ey1Tp38SxsSWTN7e3vUqhWIWrUCSxyXkZGBe/fuFgjm/HDOC+Zbt27i/PmzD92GIAjw9vbRB3HB\nXdsFw9nHxxc2PA1fsfiTVbD0dODff4HDh230s9eoKBVu3CgargEBIjp00Oh3CeftHnblyZ9EcHR0\nRFBQMIKCgkscl5aWhrt3Y/VBnB/M+Y+vXLlc5GIhBalUKvj6Vis0Yy46g7a2612TDkNXITQa3a7h\n48fVOH5chf/+081gRREAHPXjqlUT8cQT+WcL581eeeEIoopzdnZGSEgoQkJCSxyXmppisBu74K7t\n/F3a53Hy5PGHbsPGxgZeXl5wc3OHu7s73N09Ctx76J97eHjAza3ovVopFwy3MgxdKyRJQEyMgOPH\n1fjvP13InjqlNrhKk5OThJYttWjZ0gaBgZn62aunp4yFExEA3XHmsDBXhIWFP3SMJElITk4q9hjz\n3bt3ERt7B8nJiUhIeICYmOvIzs4uUw2urm7FhLW7QVjnv+ZpEOCOjo68Olg5MXStQEICcOJEXsDq\nQrbgx3JUKgl164po1kyLZs1ENG2qm73a2OSdMJIjY/VEVB6CIOhnrBERdYsdk3dCmCRJyMjIQHJy\nEhITE5GUlISkpAe597rniYmJ+vcL3sfEXEdKSnKZarOzs9PPmjnLLhuGroXJyABOn87bTawL2mvX\nDI/BBgaKeO65HDRtqgvZhg21PGuYqAoTBAFOTk5wcnKCv39Amb9eq9UiOTnJIKSTkhILBHhigZth\nkF+/fg05OWX7j72rq5s+gH18vGBraw9HR139jo6OcHJy1t87ORV87lRgXP69s7Pu3hrCnKErI60W\niI5W4b//VPpZ7PnzKmg0+bttPD0ldOyoyQ1YLZo0EeHrK8lYNREpjVqthqenFzw9vcr8tWWdZRcM\n7piY6zh79rTJ+rC3ty8S2oXDOu/2sJA3DHvDcLezs6vwbnWGrplIEnD7tqA/Bnv8uBonTqiRlpb/\nA7S3l9CkiW43cdOmulvt2hJ46ISILFVFZ9ne3s6IibmH9PR0ZGSkF3ufd8vIyEB6elqh+4LjdK+l\npaUjOTkZsbGxyMhIhyia5jKyarW6SFjnzcT3799r1DZKDd2ZM2di79698Pb2xu7du0sbTrmSkqDf\nRZx3NnHBKzgJgoSICBFNm4r6WWzduiLs7GQsmojIzFQqFZydneFcScfIJElCVlZWgSDXBXZ6enEB\nXtmry4cAAAsRSURBVFyQFwx9w/vExESkp6eVafd6qaHbp08fvPjii5g2bVqFGleyrCzg7FmVwdnE\nly4ZHluoXl1E9+45aNpUN5Nt3FjLz8ASEVUyQRDg4OAABweHcu0+N4ZJQ7dFixa4detWhQpSElEE\nLl/WHYfNm8meOaNCTk7+PmBXV91C6rrdxLqZrL8/j8MSESlRWS7vyWO6pbh7V3ccNu9kpxMn1EhJ\nyQ9YW1sJDRqI+mOwzZrplqZTFb3oExERVXEM3QIkCTh/XoUDB9Q4fhw4csS5yPWIw8K06NYt/2Sn\nRx4xXPuViIjoYSotdH19reOA5Y0bwG+/6W6//w7cvZv/np+fCj17Ao8+CrRqBbRoAXh4qAGoAVjP\naiHW8rMoiRJ6ANiHJVFCD4Ay+lBCD8YyKnQlqezHI+PiUsr8NeaQlAQcPGiD/fvV2L/fBpcv589k\nq1UT0a+fFu3aadCzpyMcHVMMPq6TkwPExclQdAXkXbHGmimhB4B9WBIl9AAoow8l9AAY/x+HUkN3\nypQpOHr0KBITE9GhQweMGzcOffv2rXCB5pKVBfzzj1ofsidOqCCKuiR1dpbQpYsG7dpp0K6d7tKJ\neSHr62t9AUtERJat1NBdunSpOeowGVHUfXxn3z5dyB49mr8QgI2NbhGAdu10t2bNtOCa0kREZC6K\nOJHq+nUB+/frdhkfOKBGQkL+LuN69fJCVoPHH9dy8XUiIpKNVYZuQoLuuGzebPb69fyQrV5dxMCB\nOWjXToMnntDCz4+fjyUiIstgFaGbkQEcPZp/XPb0aRUkSbfL2M1NwjPP5Ohns6GhvFYxERFZJosM\nXa0WOHVKpd9l/PffamRl6ZLUzk5Cmzb5u4wbNdKtG0tERGTpLCKuJAm4elXAvn26kD140AZJSfnT\n1YYN80O2VSstnJxkLJaIiKicZAvde/cEHDyYv8v45s3847KBgSJ69tTtMm7TRgsfHx6XJSIi62e2\n0E1NBY4cUetns+fP56/C4+kp6UO2XTsNgoMZskREpDyVFro5OcDx4/nHZf/9Vw2NRrfL2MFBQvv2\nugtStG+vQYMGXCCAiIiUr1JCt2dP4M8/XZCaqgtZQZDQpImov/JTy5ZaODhUxncmIiKyXJUSurt3\nAyEhEvr10+0ybttWAw+PyvhORERE1qNSQvfaNcDJKa0yNk1ERGS1KuVIalBQZWyViIjIuvH0JSIi\nIjNh6BIREZkJQ5eIiMhMGLpERERmwtAlIiIyE4YuERGRmTB0iYiIzIShS0REZCYMXSIiIjNh6BIR\nEZkJQ5eIiMhMGLpERERmwtAlIiIyE4YuERGRmTB0iYiIzIShS0REZCYMXSIiIjNh6BIREZkJQ5eI\niMhMGLpERERmwtAlIiIyE4YuERGRmTB0iYiIzIShS0REZCYMXSIiIjNh6BIREZkJQ5eIiMhMGLpE\nRERmwtAlIiIyE4YuERGRmRgVuvv370e3bt3QtWtXfPbZZ5VdExERkSKVGrqiKGLevHlYvXo1vv/+\ne/zwww+4fPmyOWojIiJSlFJD99SpUwgKCkKNGjVga2uL7t274/fffzdHbURERIpSaujevXsXAQEB\n+ud+fn64d+9epRZFRESkRKWGriRJ5qiDiIhI8WxKG+Dv74/bt2/rn9+9exfVqlUrdcO+vq4Vq8wC\nKKEHQBl9KKEHgH1YEiX0ACijDyX0YKxSZ7oNGzZETEwMbt26hezsbPzwww/o1KmTOWojIiJSlFJn\numq1GrNnz8bw4cMhSRL69euH0NBQc9RGRESkKILEg7ZERERmwStSERERmQlDl4iIyEwYukRERGZS\n6olUZbF//34sWLAAkiShb9++eOWVV0y5ebOYOXMm9u7dC29vb+zevVvucsolNjYW06ZNQ3x8PNRq\nNfr3748hQ4bIXVaZZWdnIzIyEjk5OdBqtejatSvGjh0rd1nlIooi+vbtCz8/P6xatUrucsqlY8eO\ncHFxgUqlgo2NDbZs2SJ3SeWSkpKCN998E9HR0VCpVFiwYAEaN24sd1lGu3r1KiZNmgRBECBJEm7c\nuIEJEyZY5d/xr7/+Glu2bIEgCKhTpw4W/r+9u3mJag8DOP6dHKRQexElCyzIjCySFr1AEyamSTXV\nxGCLNiVRbdIow14oghYJLfoHWkREEBEaRG1EszGmQiuGYIgwIhhMKkRT5yXPnOcu4l64G+89x7nz\na7rPZz1n+A6HmYcznHmmo4P8/HzTWY7cunXrr/fCv/qslQxJp9NSX18vsVhMfvz4IXv37pWhoaFM\nPX3WDAwMSDQaFb/fbzrFtS9fvkg0GhURkcnJSdmxY0dOngsRkXg8LiIilmVJU1OTRCIRw0Xu3Lx5\nU9ra2uT48eOmU1yrq6uTsbEx0xmzdvbsWbl//76IiExPT8vExIThIvfS6bT4fD4ZHh42neLYyMiI\n1NXVSSqVEhGRkydPSldXl+EqZ96/fy9+v19SqZRYliWHDx+WT58+zXhMxr5e/l12NG/YsIH58+eb\nzpiV0tJSqqqqACgoKKCioiJnV3fOmzcP+HnVa1mW4Rp3RkZGePr0KU1NTaZTZkVEsG3bdMasTE5O\nMjg4SDAYBMDr9VJYWGi4yr1wOMyyZcv+tqo3l9i2TSKRwLIsksnkv1q89Cv58OED69evJz8/n7y8\nPDZu3Eh3d/eMx2Rs6OqO5l9TLBbj3bt3VFdXm05xxbZtAoEAPp8Pn8+Xk6/j6tWrtLe34/F4TKfM\nisfj4ciRIwSDQe7du2c6x5VYLMaiRYs4f/48+/fv59KlSySTSdNZrj1+/Jjdu3ebznBl8eLFNDc3\nU1tbS01NDUVFRWzZssV0liOVlZUMDAwwPj5OIpEgFArx+fPnGY/J2NAV/bnvL2dqaorW1lYuXLhA\nQUGB6RxX5syZw4MHDwiFQkQiEYaGhkwnOdLX10dJSQlVVVU5/x65e/cunZ2d3Lhxgzt37jA4OGg6\nyTHLsohGoxw8eJCuri7mzp2bs/8RPj09TW9vLzt37jSd4sr379/p6enhyZMn9Pf3E4/Hc+4+moqK\nCo4ePUpzczPHjh1j9erVeL0z3yqVsaHrdkez+m9YlkVrayv79u2jvr7edM6sFRYWsmnTJvr7+02n\nOPL69Wt6e3vZvn07bW1tvHz5kvb2dtNZrpSWlgJQXFxMQ0MDb9++NVzkXFlZGWVlZaxbtw6AxsZG\notGo4Sp3QqEQa9eupbi42HSKK+FwmPLychYuXEheXh4NDQ28efPGdJZjwWCQzs5Obt++zYIFC1i+\nfPmMj8/Y0P2ddjTn+hUJ/LwLe+XKlRw6dMh0imujo6NMTEwAkEwmef78OStWrDBc5czp06fp6+uj\np6eH69evs3nzZq5du2Y6y7FEIsHU1BQA8XicZ8+eUVlZabjKuZKSEpYsWcLHjx8BePHiRc6utX30\n6BF+v990hmtLly4lEomQSqUQkZw9F6OjowAMDw/T3d39j+ckYz8Z+l12NP95NTI2NkZtbS0tLS1/\n3XSRK169esXDhw9ZtWoVgUAAj8fDqVOnqKmpMZ3myNevXzl37hy2bWPbNrt27WLbtm2ms/6Xvn37\nxokTJ/B4PKTTafbs2cPWrVtNZ7ly8eJFzpw5g2VZlJeX09HRYTrJsWQySTgc5sqVK6ZTXKuurqax\nsZFAIIDX62XNmjUcOHDAdJZjLS0tjI+P4/V6uXz5MkVFM/9jku5eVkoppbJEN1IppZRSWaJDVyml\nlMoSHbpKKaVUlujQVUoppbJEh65SSimVJTp0lVJKqSzRoauUUkpliQ5dpZRSKkv+AO2e4yf8wTuC\nAAAAAElFTkSuQmCC\n",
-            "text/plain": [
-              "\u003cmatplotlib.figure.Figure at 0xc1dc310\u003e"
-            ]
-          },
-          "metadata": {
-            "tags": []
-          },
-          "output_type": "display_data"
-        }
-      ],
-      "source": [
-        "# Train our variables.\n",
-        "\n",
-        "# numpy is used for its asscalar() function.\n",
-        "import numpy as np\n",
-        "\n",
-        "num_training_steps = 10\n",
-        "\n",
-        "def train_model(inputs, labels, wb, optimizer, num_training_steps):\n",
-        "  loss_at_step = []\n",
-        "  w_at_step = []\n",
-        "  b_at_step = []\n",
-        "  for step_num in range(num_training_steps):\n",
-        "    loss, gradients_and_variables = value_and_gradients_fn(inputs, labels, wb)\n",
-        "    loss_at_step.append(np.asscalar(loss.numpy()))\n",
-        "    \n",
-        "    optimizer.apply_gradients(gradients_and_variables)\n",
-        "    w, b = wb.variables\n",
-        "    w_at_step.append(np.asscalar(w.read_value().numpy()))\n",
-        "    b_at_step.append(np.asscalar(b.read_value().numpy()))\n",
-        "\n",
-        "  print(w_at_step)\n",
-        "  t = range(0, num_training_steps)\n",
-        "  plt.plot(t, loss_at_step, 'k',\n",
-        "           t, w_at_step, 'r',\n",
-        "           t, [true_w] * num_training_steps, 'r--',\n",
-        "           t, b_at_step, 'b',\n",
-        "           t, [true_b] * num_training_steps, 'b--')\n",
-        "  plt.legend(['loss', 'w estimate', 'w true', 'b estimate', 'b true'])\n",
-        "  plt.show()\n",
-        "\n",
-        "train_model(inputs, labels, wb, optimizer, num_training_steps)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "UNurY9VJ-hpH"
-      },
-      "source": [
-        "## Other Ways to Compute Gradients\n",
-        "\n",
-        "Using our loss function as an example (`calculate_linear_model_loss()`), there are several other ways we could compute gradients:\n",
-        "\n",
-        "1. `tfe.implicit_gradients()`\n",
-        "1. `tfe.gradients_function()`\n",
-        "1. `tfe.implicit_value_and_gradients()`\n",
-        "1. `tfe.value_and_gradients_function()`\n",
-        "\n",
-        "Each of these functions does the following:\n",
-        "* Wraps a function.\n",
-        "* Returns a function with the same input signature as the wrapped function.\n",
-        "\n",
-        "They differ only in what information they return.\n",
-        "\n",
-        "### Gradients-only functions\n",
-        "\n",
-        "The following two functions return a function that returns only the variables' gradients:\n",
-        "\n",
-        "1. `tfe.gradients_function()`: Returns the partial derivatives of the function `f()` with respect to the parameters of `f()`.\n",
-        "1. `tfe.implicit_gradients()`: Returns the partial derivatives of the function `f()` with respect to the trainable parameters (`tf.Variable`) used by `f()`.\n",
-        "\n",
-        "In our example above, the `tf.layers.Dense` object encapsulates the trainable parameters.\n",
-        "\n",
-        "### Value and gradients functions\n",
-        "\n",
-        "The following two functions are identical to their counterparts above, except that they also return the value of the wrapped function.\n",
-        "\n",
-        "1. `tfe.implicit_value_and_gradients()`\n",
-        "1. `tfe.value_and_gradients_function()`\n",
-        "\n",
-        "### Gradient demos\n",
-        "\n",
-        "In the demos below, we show examples for the `implicit_*` functions, since our existing loss function works seamlessly with these versions. (The other versions require that your parameters are tensors and tensors only; in our example, we're using a `Dense` layer.)\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 13,
-      "metadata": {
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          },
-          "height": 85,
-          "output_extras": [
-            {
-              "item_id": 1
-            }
-          ]
-        },
-        "colab_type": "code",
-        "executionInfo": {
-          "elapsed": 100,
-          "status": "ok",
-          "timestamp": 1505502831671,
-          "user": {
-            "displayName": "",
-            "photoUrl": "",
-            "userId": ""
-          },
-          "user_tz": 240
-        },
-        "id": "aEoCftnfAIH5",
-        "outputId": "72f1c1dc-a574-463f-f860-c4e5f48fcdaa"
-      },
-      "outputs": [
-        {
-          "data": {
-            "text/plain": [
-              "[(\u003ctf.Tensor: id=673, shape=(1, 1), dtype=float32, numpy=array([[-0.26846504]], dtype=float32)\u003e,\n",
-              "  \u003ctf.Variable 'dense/kernel:0' shape=(1, 1) dtype=float32\u003e),\n",
-              " (\u003ctf.Tensor: id=671, shape=(1,), dtype=float32, numpy=array([-0.32890949], dtype=float32)\u003e,\n",
-              "  \u003ctf.Variable 'dense/bias:0' shape=(1,) dtype=float32\u003e)]"
-            ]
-          },
-          "execution_count": 13,
-          "metadata": {
-            "tags": []
-          },
-          "output_type": "execute_result"
-        }
-      ],
-      "source": [
-        "# tfe.implicit_gradients() demo\n",
-        "gradients_fn = tfe.implicit_gradients(loss_fn)\n",
-        "\n",
-        "# Returns only gradients and variables:\n",
-        "gradients_fn(inputs, labels, wb)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 14,
-      "metadata": {
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          },
-          "height": 102,
-          "output_extras": [
-            {
-              "item_id": 1
-            }
-          ]
-        },
-        "colab_type": "code",
-        "executionInfo": {
-          "elapsed": 88,
-          "status": "ok",
-          "timestamp": 1505502831785,
-          "user": {
-            "displayName": "",
-            "photoUrl": "",
-            "userId": ""
-          },
-          "user_tz": 240
-        },
-        "id": "bbgCUdCzAVhH",
-        "outputId": "152aa9b6-9e42-4b7e-848a-9423c0b1929c"
-      },
-      "outputs": [
-        {
-          "data": {
-            "text/plain": [
-              "(\u003ctf.Tensor: id=688, shape=(), dtype=float32, numpy=1.0623235\u003e,\n",
-              " [(\u003ctf.Tensor: id=720, shape=(1, 1), dtype=float32, numpy=array([[-0.26846504]], dtype=float32)\u003e,\n",
-              "   \u003ctf.Variable 'dense/kernel:0' shape=(1, 1) dtype=float32\u003e),\n",
-              "  (\u003ctf.Tensor: id=718, shape=(1,), dtype=float32, numpy=array([-0.32890949], dtype=float32)\u003e,\n",
-              "   \u003ctf.Variable 'dense/bias:0' shape=(1,) dtype=float32\u003e)])"
-            ]
-          },
-          "execution_count": 14,
-          "metadata": {
-            "tags": []
-          },
-          "output_type": "execute_result"
-        }
-      ],
-      "source": [
-        "# tfe.implicit_value_and_gradients() demo\n",
-        "value_gradients_fn = tfe.implicit_value_and_gradients(loss_fn)\n",
-        "\n",
-        "# Returns only gradients:\n",
-        "value_gradients_fn(inputs, labels, wb)"
-      ]
-    }
-  ],
-  "metadata": {
-    "colab": {
-      "default_view": {},
-      "last_runtime": {
-        "build_target": "",
-        "kind": "local"
-      },
-      "name": "Eager Execution Tutorial: Working with Gradients",
-      "provenance": [],
-      "version": "0.3.2",
-      "views": {}
-    }
-  },
-  "nbformat": 4,
-  "nbformat_minor": 0
-}
diff --git a/tensorflow/contrib/eager/python/examples/notebooks/3_datasets.ipynb b/tensorflow/contrib/eager/python/examples/notebooks/3_datasets.ipynb
deleted file mode 100644
index ff0ff4a6a7..0000000000
--- a/tensorflow/contrib/eager/python/examples/notebooks/3_datasets.ipynb
+++ /dev/null
@@ -1,218 +0,0 @@
-{
-  "cells": [
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "U9i2Dsh-ziXr"
-      },
-      "source": [
-        "# Eager Execution Tutorial: Importing Data\n",
-        "\n",
-        "This notebook demonstrates the use of the [`tf.contrib.data.Dataset` API](https://www.tensorflow.org/programmers_guide/datasets) to build pipelines to feed data to your program. It covers:\n",
-        "\n",
-        "* Creating a `Dataset`.\n",
-        "* Iteration over a `Dataset` with eager execution enabled.\n",
-        "\n",
-        "We recommend using the `Dataset`s API for building performant, complex input pipelines from simple, re-usable pieces that will feed your model's training or evaluation loops.\n",
-        "\n",
-        "If you're familiar with TensorFlow graphs, the API for constructing the `Dataset` object remains exactly the same when eager execution is enabled, but the process of iterating over elements of the dataset is slightly different.  You will use a Pythonic `Iterator()` class instead of using `make_one_shot_iterator()` and `get_next()`. As a result, the discussion on iterators in the [Programmer's Guide](https://www.tensorflow.org/programmers_guide/datasets) is not relevant when eager execution is enabled."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "z1JcS5iBXMRO"
-      },
-      "source": [
-        "# Setup: Enable eager execution\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "cellView": "code",
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          }
-        },
-        "colab_type": "code",
-        "id": "RlIWhyeLoYnG"
-      },
-      "outputs": [],
-      "source": [
-        "# Import TensorFlow.\n",
-        "import tensorflow as tf\n",
-        "\n",
-        "# Import TensorFlow eager execution support (subject to future changes).\n",
-        "from tensorflow.contrib.eager.python import tfe\n",
-        "\n",
-        "# Enable eager execution\n",
-        "tfe.enable_eager_execution()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "H9UySOPLXdaw"
-      },
-      "source": [
-        "# Step 1: Create a source `Dataset`\n",
-        "\n",
-        "Create a _source_ dataset using one of the factory functions like [`Dataset.from_tensors`](https://www.tensorflow.org/api_docs/python/tf/contrib/data/Dataset#from_tensors), [`Dataset.from_tensor_slices`](https://www.tensorflow.org/api_docs/python/tf/contrib/data/Dataset#from_tensor_slices) or using objects that read from files like [`TextLineDataset`](https://www.tensorflow.org/api_docs/python/tf/contrib/data/TextLineDataset) or [`TFRecordDataset`](https://www.tensorflow.org/api_docs/python/tf/contrib/data/TFRecordDataset). See the [Programmer's Guide](https://www.google.com/url?sa=D\u0026q=https%3A%2F%2Fwww.tensorflow.org%2Fprogrammers_guide%2Fdatasets%23reading_input_data) for more information."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "cellView": "code",
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          }
-        },
-        "colab_type": "code",
-        "id": "WPTUfGq6kJ5w"
-      },
-      "outputs": [],
-      "source": [
-        "ds_tensors = tf.contrib.data.Dataset.from_tensor_slices([1, 2, 3, 4, 5, 6])\n",
-        "\n",
-        "# Create a CSV file\n",
-        "import tempfile\n",
-        "_, filename = tempfile.mkstemp()\n",
-        "with open(filename, 'w') as f:\n",
-        "  f.write(\"\"\"Line 1\n",
-        "Line 2\n",
-        "Line 3\n",
-        "  \"\"\")\n",
-        "ds_file = tf.contrib.data.TextLineDataset(filename)\n"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "twBfWd5xyu_d"
-      },
-      "source": [
-        "# Step 2: Apply transformations\n",
-        "\n",
-        "Use the transformations functions like [`map`](https://www.tensorflow.org/api_docs/python/tf/contrib/data/Dataset#map), [`batch`](https://www.tensorflow.org/api_docs/python/tf/contrib/data/Dataset#batch), [`shuffle`](https://www.tensorflow.org/api_docs/python/tf/contrib/data/Dataset#shuffle) etc. to apply transformations to the records of the dataset. See the [API documentation for `tf.contrib.data.Dataset`](https://www.tensorflow.org/api_docs/python/tf/contrib/data/Dataset) for details."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 0,
-      "metadata": {
-        "cellView": "code",
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          }
-        },
-        "colab_type": "code",
-        "id": "ngUe237Wt48W"
-      },
-      "outputs": [],
-      "source": [
-        "ds_tensors = ds_tensors.map(tf.square).shuffle(2).batch(2)\n",
-        "ds_file = ds_file.batch(2)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "colab_type": "text",
-        "id": "IDY4WsYRhP81"
-      },
-      "source": [
-        "# Step 3: Iterate\n",
-        "\n",
-        "Use `tfe.Iterator` on the `Dataset` object to get a Python iterator over the contents of the dataset.\n",
-        "\n",
-        "If you're familiar with the use of `Dataset`s in TensorFlow graphs, note that this process of iteration is different. Here there are no calls to `Dataset.make_one_shot_iterator()` and no `get_next()` calls."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 5,
-      "metadata": {
-        "colab": {
-          "autoexec": {
-            "startup": false,
-            "wait_interval": 0
-          },
-          "height": 153,
-          "output_extras": [
-            {
-              "item_id": 1
-            }
-          ]
-        },
-        "colab_type": "code",
-        "executionInfo": {
-          "elapsed": 201,
-          "status": "ok",
-          "timestamp": 1505952405928,
-          "user": {
-            "displayName": "",
-            "photoUrl": "",
-            "userId": ""
-          },
-          "user_tz": 420
-        },
-        "id": "lCUWzso6mbqR",
-        "outputId": "ec027d30-96c6-4ea4-9ee1-ef74ec1ae29a"
-      },
-      "outputs": [
-        {
-          "name": "stdout",
-          "output_type": "stream",
-          "text": [
-            "Elements of ds_tensors:\n",
-            "tf.Tensor([4 9], shape=(2,), dtype=int32)\n",
-            "tf.Tensor([16 25], shape=(2,), dtype=int32)\n",
-            "tf.Tensor([36  1], shape=(2,), dtype=int32)\n",
-            "\n",
-            "Elements in ds_file:\n",
-            "tf.Tensor(['Line 1' 'Line 2'], shape=(2,), dtype=string)\n",
-            "tf.Tensor(['Line 3' '  '], shape=(2,), dtype=string)\n"
-          ]
-        }
-      ],
-      "source": [
-        "print('Elements of ds_tensors:')\n",
-        "for x in tfe.Iterator(ds_tensors):\n",
-        "  print(x)\n",
-        "\n",
-        "print('\\nElements in ds_file:')\n",
-        "for x in tfe.Iterator(ds_file):\n",
-        "  print(x)"
-      ]
-    }
-  ],
-  "metadata": {
-    "colab": {
-      "default_view": {},
-      "last_runtime": {
-        "build_target": "",
-        "kind": "local"
-      },
-      "name": "Eager Execution Tutorial: Importing Data",
-      "provenance": [],
-      "version": "0.3.2",
-      "views": {}
-    }
-  },
-  "nbformat": 4,
-  "nbformat_minor": 0
-}
diff --git a/tensorflow/contrib/eager/python/examples/tests/cart_pole_helper_test.py b/tensorflow/contrib/eager/python/examples/tests/cart_pole_helper_test.py
deleted file mode 100644
index 7a213e9e03..0000000000
--- a/tensorflow/contrib/eager/python/examples/tests/cart_pole_helper_test.py
+++ /dev/null
@@ -1,51 +0,0 @@
-# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-from tensorflow.contrib.eager.python.examples import cart_pole_helper
-from tensorflow.python.framework import test_util
-from tensorflow.python.platform import test
-
-
-class RewardDiscountingTest(test_util.TensorFlowTestCase):
-
-  def testDiscountingRewards(self):
-    rewards = [0.0, 10.0, 20.0]
-    discount_rate = 0.9
-    self.assertAllClose(
-        [10 * discount_rate + 20 * discount_rate * discount_rate,
-         10 + 20 * discount_rate, 20],
-        cart_pole_helper.discount_rewards(rewards, discount_rate))
-    self.assertAllClose(
-        [-1.2], cart_pole_helper.discount_rewards([-1.2], discount_rate))
-    self.assertEqual([], cart_pole_helper.discount_rewards([], discount_rate))
-
-  def testDiscountAndNormalizeRewardSequences(self):
-    rewards1 = [0.0, 10.0, 20.0]
-    rewards2 = [0.0, 5.0, -5.0]
-    reward_sequences = [rewards1, rewards2]
-    discount_rate = 0.9
-    dn = cart_pole_helper.discount_and_normalize_rewards(reward_sequences,
-                                                         discount_rate)
-    self.assertAllClose(
-        [[1.03494653, 1.24685514, 0.64140196],
-         [-0.83817424, -0.83439016, -1.25063922]], dn)
-
-
-if __name__ == "__main__":
-  test.main()
diff --git a/tensorflow/contrib/eager/python/examples/tests/cart_pole_test.py b/tensorflow/contrib/eager/python/examples/tests/cart_pole_test.py
deleted file mode 100644
index dc1381cc04..0000000000
--- a/tensorflow/contrib/eager/python/examples/tests/cart_pole_test.py
+++ /dev/null
@@ -1,162 +0,0 @@
-# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Unit test for cart-pole reinforcement learning under eager exection."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import gc
-import glob
-import os
-import shutil
-import tempfile
-import time
-
-import gym
-import numpy as np
-
-from tensorflow.contrib.eager.python.examples import cart_pole
-from tensorflow.python.eager import context
-from tensorflow.python.eager import test
-from tensorflow.python.framework import constant_op
-from tensorflow.python.framework import dtypes
-from tensorflow.python.framework import random_seed
-from tensorflow.python.framework import test_util
-from tensorflow.python.platform import tf_logging as logging
-from tensorflow.python.training import training
-
-
-class CartPoleTest(test_util.TensorFlowTestCase):
-
-  def setUp(self):
-    super(CartPoleTest, self).setUp()
-    self._tmp_logdir = tempfile.mkdtemp()
-
-  def tearDown(self):
-    shutil.rmtree(self._tmp_logdir)
-    super(CartPoleTest, self).tearDown()
-
-  def testGetLogitsAndAction(self):
-    hidden_size = 5
-    policy_network = cart_pole.PolicyNetwork(hidden_size)
-
-    dummy_inputs = np.array([[0.1, 0.3, 0.2, 0.5],
-                             [0.0, -0.2, 0.6, -0.8]], dtype=np.float32)
-    logits, actions = policy_network.forward(constant_op.constant(dummy_inputs))
-
-    self.assertEqual((2, 1), logits.shape)
-    self.assertEqual(dtypes.float32, logits.dtype)
-    self.assertEqual((2, 1), actions.shape)
-    self.assertEqual(dtypes.int64, actions.dtype)
-
-  def testCrossEntropy(self):
-    hidden_size = 5
-    policy_network = cart_pole.PolicyNetwork(hidden_size)
-
-    dummy_inputs = np.array([[0.1, 0.3, 0.2, 0.5],
-                             [0.0, -0.2, 0.6, -0.8]], dtype=np.float32)
-    cross_entropy = policy_network._get_cross_entropy_and_save_actions(
-        constant_op.constant(dummy_inputs))
-
-    self.assertEqual((2, 1), cross_entropy.shape)
-    self.assertEqual(dtypes.float32, cross_entropy.dtype)
-
-  def testPlayAGame(self):
-    hidden_size = 5
-    cart_pole_env = gym.make("CartPole-v0")
-    cart_pole_env.seed(0)
-    cart_pole_env.reset()
-
-    device = "gpu:0" if context.context().num_gpus() > 0 else "cpu:0"
-    logging.info("device = %s", device)
-    with context.device(device):
-      policy_network = cart_pole.PolicyNetwork(hidden_size)
-      policy_network.play(cart_pole_env, max_steps=10, render=False)
-
-  def testTrain(self):
-    hidden_size = 5
-    num_games_per_iteration = 5
-    max_steps_per_game = 10
-    discount_rate = 0.95
-    learning_rate = 0.02
-
-    cart_pole_env = gym.make("CartPole-v0")
-    cart_pole_env.reset()
-
-    device = "gpu:0" if context.context().num_gpus() > 0 else "cpu:0"
-    logging.info("device = %s", device)
-    with context.device(device):
-      policy_network = cart_pole.PolicyNetwork(hidden_size,
-                                               train_logdir=self._tmp_logdir)
-      optimizer = training.AdamOptimizer(learning_rate)
-      policy_network.train(
-          cart_pole_env,
-          optimizer,
-          discount_rate,
-          num_games_per_iteration,
-          max_steps_per_game)
-      self.assertTrue(glob.glob(os.path.join(self._tmp_logdir, "events.out.*")))
-
-
-class EagerCartPoleTrainingBenchmark(test.Benchmark):
-
-  def benchmarkEagerCartPolePolicyNetworkTraining(self):
-    burn_in_iterations = 1
-    benchmark_iterations = 2
-    num_games_per_iteration = 10
-    max_steps_per_game = 100
-    discount_rate = 0.95
-    learning_rate = 0.02
-
-    cart_pole_env = gym.make("CartPole-v0")
-    cart_pole_env.seed(0)
-    random_seed.set_random_seed(0)
-    cart_pole_env.reset()
-
-    hidden_size = 5
-    policy_network = cart_pole.PolicyNetwork(hidden_size)
-    optimizer = training.AdamOptimizer(learning_rate)
-
-    # Perform burn-in.
-    for _ in xrange(burn_in_iterations):
-      policy_network.train(
-          cart_pole_env,
-          optimizer,
-          discount_rate,
-          num_games_per_iteration,
-          max_steps_per_game)
-
-    gc.collect()
-    start_time = time.time()
-    for _ in xrange(benchmark_iterations):
-      policy_network.train(
-          cart_pole_env,
-          optimizer,
-          discount_rate,
-          num_games_per_iteration,
-          max_steps_per_game)
-    wall_time = time.time() - start_time
-    # Named "examples"_per_sec to conform with other benchmarks.
-    extras = {"examples_per_sec": benchmark_iterations / wall_time}
-    self.report_benchmark(
-        name="EagerCartPoleReinforcementLearning",
-        iters=benchmark_iterations,
-        wall_time=wall_time,
-        extras=extras)
-
-
-if __name__ == "__main__":
-  test.main()
diff --git a/tensorflow/contrib/eager/python/examples/tests/linear_regression_test.py b/tensorflow/contrib/eager/python/examples/tests/linear_regression_test.py
deleted file mode 100644
index aee0b3d0dd..0000000000
--- a/tensorflow/contrib/eager/python/examples/tests/linear_regression_test.py
+++ /dev/null
@@ -1,114 +0,0 @@
-# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Unit tests for linear regression example under TensorFlow eager execution."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import glob
-import os
-import shutil
-import tempfile
-import time
-
-import numpy as np
-import tensorflow as tf
-
-from tensorflow.contrib.eager.python.examples import linear_regression
-from tensorflow.python.eager import context
-from tensorflow.python.eager import test
-from tensorflow.python.framework import test_util
-from tensorflow.python.platform import tf_logging as logging
-
-
-def _create_data_gen_for_test():
-  true_w = np.array([[1.0], [-0.5], [2.0]], dtype=np.float32)
-  true_b = np.array([1.0], dtype=np.float32)
-  noise_level = 0
-  batch_size = 64
-  return (
-      true_w, true_b, noise_level, batch_size,
-      linear_regression.DataGenerator(true_w, true_b, noise_level, batch_size))
-
-
-class LinearRegressionTest(test_util.TensorFlowTestCase):
-
-  def setUp(self):
-    super(LinearRegressionTest, self).setUp()
-    self._tmp_logdir = tempfile.mkdtemp()
-
-  def tearDown(self):
-    shutil.rmtree(self._tmp_logdir)
-    super(LinearRegressionTest, self).tearDown()
-
-  def testSyntheticBatch(self):
-    _, _, _, batch_size, data_gen = _create_data_gen_for_test()
-
-    xs, ys = data_gen.next_batch()
-    self.assertEqual((batch_size, 3), xs.shape)
-    self.assertEqual((batch_size, 1), ys.shape)
-    self.assertEqual(tf.float32, xs.dtype)
-    self.assertEqual(tf.float32, ys.dtype)
-
-  def testLinearRegression(self):
-    true_w, true_b, _, _, data_gen = _create_data_gen_for_test()
-
-    learning_rate = 0.1
-    num_iters = 40
-
-    device = "gpu:0" if context.context().num_gpus() > 0 else "cpu:0"
-    logging.info("device = %s", device)
-    with context.device(device):
-      linear_model = linear_regression.LinearModel()
-      optimizer = tf.train.GradientDescentOptimizer(learning_rate)
-      linear_model.fit(data_gen.next_batch, optimizer, num_iters,
-                       logdir=self._tmp_logdir)
-
-      self.assertAllClose(true_w, linear_model.weights, rtol=1e-2)
-      self.assertAllClose(true_b, linear_model.biases, rtol=1e-2)
-      self.assertTrue(glob.glob(os.path.join(self._tmp_logdir, "events.out.*")))
-
-
-class EagerLinearRegressionBenchmark(test.Benchmark):
-
-  def benchmarkEagerLinearRegression(self):
-    _, _, _, _, data_gen = _create_data_gen_for_test()
-
-    learning_rate = 0.1
-    num_burnin_iters = 10
-    num_iters = 200
-
-    device = "gpu:0" if context.context().num_gpus() > 0 else "cpu:0"
-    logging.info("device = %s", device)
-    with context.device(device):
-      linear_model = linear_regression.LinearModel()
-      optimizer = tf.train.GradientDescentOptimizer(learning_rate)
-
-      # Perform burn-in.
-      linear_model.fit(data_gen.next_batch, optimizer, num_burnin_iters)
-
-      start_time = time.time()
-      linear_model.fit(data_gen.next_batch, optimizer, num_iters)
-      wall_time = time.time() - start_time
-
-      self.report_benchmark(
-          name="EagerLinearRegression",
-          iters=num_iters,
-          wall_time=wall_time)
-
-
-if __name__ == "__main__":
-  test.main()
diff --git a/tensorflow/contrib/eager/python/examples/tests/spinn_test.py b/tensorflow/contrib/eager/python/examples/tests/spinn_test.py
deleted file mode 100644
index 9c8b691b98..0000000000
--- a/tensorflow/contrib/eager/python/examples/tests/spinn_test.py
+++ /dev/null
@@ -1,311 +0,0 @@
-# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import collections
-import gc
-import time
-
-import numpy as np
-import tensorflow as tf
-
-from tensorflow.contrib.eager.python import tfe
-from tensorflow.contrib.eager.python.examples import spinn
-from tensorflow.python.eager import test
-from tensorflow.python.framework import test_util
-
-
-def _generate_synthetic_snli_data_batch(sequence_length,
-                                        batch_size,
-                                        vocab_size):
-  """Generate a fake batch of SNLI data for testing."""
-  with tf.device("cpu:0"):
-    labels = tf.random_uniform([batch_size], minval=1, maxval=4, dtype=tf.int64)
-    prem = tf.random_uniform(
-        (sequence_length, batch_size), maxval=vocab_size, dtype=tf.int64)
-    prem_trans = tf.constant(np.array(
-        [[3, 3, 2, 3, 3, 3, 2, 2, 2, 3, 3, 3,
-          2, 3, 3, 2, 2, 3, 3, 3, 2, 2, 2, 2,
-          3, 2, 2]] * batch_size, dtype=np.int64).T)
-    hypo = tf.random_uniform(
-        (sequence_length, batch_size), maxval=vocab_size, dtype=tf.int64)
-    hypo_trans = tf.constant(np.array(
-        [[3, 3, 2, 3, 3, 3, 2, 2, 2, 3, 3, 3,
-          2, 3, 3, 2, 2, 3, 3, 3, 2, 2, 2, 2,
-          3, 2, 2]] * batch_size, dtype=np.int64).T)
-  if tfe.num_gpus():
-    labels = labels.gpu()
-    prem = prem.gpu()
-    prem_trans = prem_trans.gpu()
-    hypo = hypo.gpu()
-    hypo_trans = hypo_trans.gpu()
-  return labels, prem, prem_trans, hypo, hypo_trans
-
-
-def _snli_classifier_config(d_embed, d_out):
-  config_tuple = collections.namedtuple(
-      "Config", ["d_hidden", "d_proj", "d_tracker", "predict",
-                 "embed_dropout", "mlp_dropout", "n_mlp_layers", "d_mlp",
-                 "d_out", "projection", "lr"])
-  config = config_tuple(
-      d_hidden=d_embed,
-      d_proj=d_embed * 2,
-      d_tracker=8,
-      predict=False,
-      embed_dropout=0.1,
-      mlp_dropout=0.1,
-      n_mlp_layers=2,
-      d_mlp=32,
-      d_out=d_out,
-      projection=True,
-      lr=2e-3)
-  return config
-
-
-class SpinnTest(test_util.TensorFlowTestCase):
-
-  def setUp(self):
-    super(SpinnTest, self).setUp()
-    self._test_device = "gpu:0" if tfe.num_gpus() else "cpu:0"
-
-  def testBundle(self):
-    with tf.device(self._test_device):
-      lstm_iter = [np.array([[0, 1], [2, 3]], dtype=np.float32),
-                   np.array([[0, -1], [-2, -3]], dtype=np.float32),
-                   np.array([[0, 2], [4, 6]], dtype=np.float32),
-                   np.array([[0, -2], [-4, -6]], dtype=np.float32)]
-      out = spinn._bundle(lstm_iter)
-
-      self.assertEqual(2, len(out))
-      self.assertEqual(tf.float32, out[0].dtype)
-      self.assertEqual(tf.float32, out[1].dtype)
-      self.assertAllEqual(np.array([[0, 2, 0, -2, 0, 4, 0, -4]]).T,
-                          out[0].numpy())
-      self.assertAllEqual(np.array([[1, 3, -1, -3, 2, 6, -2, -6]]).T,
-                          out[1].numpy())
-
-  def testUnbunbdle(self):
-    with tf.device(self._test_device):
-      state = [np.array([[0, 1, 2], [3, 4, 5]], dtype=np.float32),
-               np.array([[0, -1, -2], [-3, -4, -5]], dtype=np.float32)]
-      out = spinn._unbundle(state)
-
-      self.assertEqual(2, len(out))
-      self.assertEqual(tf.float32, out[0].dtype)
-      self.assertEqual(tf.float32, out[1].dtype)
-      self.assertAllEqual(np.array([[0, 1, 2, 0, -1, -2]]),
-                          out[0].numpy())
-      self.assertAllEqual(np.array([[3, 4, 5, -3, -4, -5]]),
-                          out[1].numpy())
-
-  def testReduce(self):
-    with tf.device(self._test_device):
-      batch_size = 3
-      size = 10
-      tracker_size = 8
-      reducer = spinn.Reduce(size, tracker_size=tracker_size)
-
-      left_in = []
-      right_in = []
-      tracking = []
-      for _ in range(batch_size):
-        left_in.append(tf.random_normal((1, size * 2)))
-        right_in.append(tf.random_normal((1, size * 2)))
-        tracking.append(tf.random_normal((1, tracker_size * 2)))
-
-      out = reducer(left_in, right_in, tracking=tracking)
-      self.assertEqual(batch_size, len(out))
-      self.assertEqual(tf.float32, out[0].dtype)
-      self.assertEqual((1, size * 2), out[0].shape)
-
-  def testReduceTreeLSTM(self):
-    with tf.device(self._test_device):
-      size = 10
-      tracker_size = 8
-      reducer = spinn.Reduce(size, tracker_size=tracker_size)
-
-      lstm_in = np.array([[0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
-                          [0, -1, -2, -3, -4, -5, -6, -7, -8, -9]],
-                         dtype=np.float32)
-      c1 = np.array([[0, 1], [2, 3]], dtype=np.float32)
-      c2 = np.array([[0, -1], [-2, -3]], dtype=np.float32)
-
-      h, c = reducer._tree_lstm(c1, c2, lstm_in)
-      self.assertEqual(tf.float32, h.dtype)
-      self.assertEqual(tf.float32, c.dtype)
-      self.assertEqual((2, 2), h.shape)
-      self.assertEqual((2, 2), c.shape)
-
-  def testTracker(self):
-    with tf.device(self._test_device):
-      batch_size = 2
-      size = 10
-      tracker_size = 8
-      buffer_length = 18
-      stack_size = 3
-
-      tracker = spinn.Tracker(tracker_size, False)
-      tracker.reset_state()
-
-      # Create dummy inputs for testing.
-      bufs = []
-      buf = []
-      for _ in range(buffer_length):
-        buf.append(tf.random_normal((batch_size, size * 2)))
-      bufs.append(buf)
-      self.assertEqual(1, len(bufs))
-      self.assertEqual(buffer_length, len(bufs[0]))
-      self.assertEqual((batch_size, size * 2), bufs[0][0].shape)
-
-      stacks = []
-      stack = []
-      for _ in range(stack_size):
-        stack.append(tf.random_normal((batch_size, size * 2)))
-      stacks.append(stack)
-      self.assertEqual(1, len(stacks))
-      self.assertEqual(3, len(stacks[0]))
-      self.assertEqual((batch_size, size * 2), stacks[0][0].shape)
-
-      for _ in range(2):
-        out1, out2 = tracker(bufs, stacks)
-        self.assertIsNone(out2)
-        self.assertEqual(batch_size, len(out1))
-        self.assertEqual(tf.float32, out1[0].dtype)
-        self.assertEqual((1, tracker_size * 2), out1[0].shape)
-
-        self.assertEqual(tf.float32, tracker.state.c.dtype)
-        self.assertEqual((batch_size, tracker_size), tracker.state.c.shape)
-        self.assertEqual(tf.float32, tracker.state.h.dtype)
-        self.assertEqual((batch_size, tracker_size), tracker.state.h.shape)
-
-  def testSPINN(self):
-    with tf.device(self._test_device):
-      embedding_dims = 10
-      d_tracker = 8
-      sequence_length = 15
-      num_transitions = 27
-
-      config_tuple = collections.namedtuple(
-          "Config", ["d_hidden", "d_proj", "d_tracker", "predict"])
-      config = config_tuple(
-          embedding_dims, embedding_dims * 2, d_tracker, False)
-      s = spinn.SPINN(config)
-
-      # Create some fake data.
-      buffers = tf.random_normal((sequence_length, 1, config.d_proj))
-      transitions = np.array(
-          [[3], [3], [2], [3], [3], [3], [2], [2], [2], [3], [3], [3],
-           [2], [3], [3], [2], [2], [3], [3], [3], [2], [2], [2], [2],
-           [3], [2], [2]], dtype=np.int32)
-      self.assertEqual(tf.int32, transitions.dtype)
-      self.assertEqual((num_transitions, 1), transitions.shape)
-
-      out = s(buffers, transitions, training=True)
-      self.assertEqual(tf.float32, out.dtype)
-      self.assertEqual((1, embedding_dims), out.shape)
-
-  def testSNLIClassifierAndTrainer(self):
-    with tf.device(self._test_device):
-      vocab_size = 40
-      batch_size = 2
-      d_embed = 10
-      sequence_length = 15
-      d_out = 4
-
-      config = _snli_classifier_config(d_embed, d_out)
-
-      # Create fake embedding matrix.
-      embed = tf.random_normal((vocab_size, d_embed))
-
-      model = spinn.SNLIClassifier(config, embed)
-      trainer = spinn.SNLIClassifierTrainer(model, config.lr)
-
-      (labels, prem, prem_trans, hypo,
-       hypo_trans) = _generate_synthetic_snli_data_batch(sequence_length,
-                                                         batch_size,
-                                                         vocab_size)
-
-      # Invoke model under non-training mode.
-      logits = model(prem, prem_trans, hypo, hypo_trans, training=False)
-      self.assertEqual(tf.float32, logits.dtype)
-      self.assertEqual((batch_size, d_out), logits.shape)
-
-      # Invoke model under training model.
-      logits = model(prem, prem_trans, hypo, hypo_trans, training=True)
-      self.assertEqual(tf.float32, logits.dtype)
-      self.assertEqual((batch_size, d_out), logits.shape)
-
-      # Calculate loss.
-      loss1 = trainer.loss(labels, logits)
-      self.assertEqual(tf.float32, loss1.dtype)
-      self.assertEqual((), loss1.shape)
-
-      loss2, logits = trainer.train_batch(
-          labels, prem, prem_trans, hypo, hypo_trans)
-      self.assertEqual(tf.float32, loss2.dtype)
-      self.assertEqual((), loss2.shape)
-      self.assertEqual(tf.float32, logits.dtype)
-      self.assertEqual((batch_size, d_out), logits.shape)
-      # Training on the batch should have led to a change in the loss value.
-      self.assertNotEqual(loss1.numpy(), loss2.numpy())
-
-
-class EagerSpinnSNLIClassifierBenchmark(test.Benchmark):
-
-  def benchmarkEagerSpinnSNLIClassifier(self):
-    test_device = "gpu:0" if tfe.num_gpus() else "cpu:0"
-    with tf.device(test_device):
-      burn_in_iterations = 2
-      benchmark_iterations = 10
-
-      vocab_size = 1000
-      batch_size = 128
-      sequence_length = 15
-      d_embed = 200
-      d_out = 4
-
-      embed = tf.random_normal((vocab_size, d_embed))
-
-      config = _snli_classifier_config(d_embed, d_out)
-      model = spinn.SNLIClassifier(config, embed)
-      trainer = spinn.SNLIClassifierTrainer(model, config.lr)
-
-      (labels, prem, prem_trans, hypo,
-       hypo_trans) = _generate_synthetic_snli_data_batch(sequence_length,
-                                                         batch_size,
-                                                         vocab_size)
-
-      for _ in range(burn_in_iterations):
-        trainer.train_batch(labels, prem, prem_trans, hypo, hypo_trans)
-
-      gc.collect()
-      start_time = time.time()
-      for _ in xrange(benchmark_iterations):
-        trainer.train_batch(labels, prem, prem_trans, hypo, hypo_trans)
-      wall_time = time.time() - start_time
-      # Named "examples"_per_sec to conform with other benchmarks.
-      extras = {"examples_per_sec": benchmark_iterations / wall_time}
-      self.report_benchmark(
-          name="Eager_SPINN_SNLIClassifier_Benchmark",
-          iters=benchmark_iterations,
-          wall_time=wall_time,
-          extras=extras)
-
-
-if __name__ == "__main__":
-  test.main()
diff --git a/tensorflow/contrib/eager/python/tfe.py b/tensorflow/contrib/eager/python/tfe.py
index 4ed258f6ff..3810d96950 100644
--- a/tensorflow/contrib/eager/python/tfe.py
+++ b/tensorflow/contrib/eager/python/tfe.py
@@ -55,6 +55,10 @@ To use, at program startup, call `tfe.enable_eager_execution()`.
 
 @@IsolateTest
 @@run_test_in_graph_and_eager_modes
+
+@@DEVICE_PLACEMENT_EXPLICIT
+@@DEVICE_PLACEMENT_WARN
+@@DEVICE_PLACEMENT_SILENT
 """
 
 from __future__ import absolute_import
@@ -71,6 +75,9 @@ from tensorflow.contrib.eager.python.saver import Saver
 from tensorflow.contrib.eager.python.summary_writer import SummaryWriter
 from tensorflow.python.eager import backprop
 from tensorflow.python.eager import function
+from tensorflow.python.eager.context import DEVICE_PLACEMENT_EXPLICIT
+from tensorflow.python.eager.context import DEVICE_PLACEMENT_WARN
+from tensorflow.python.eager.context import DEVICE_PLACEMENT_SILENT
 from tensorflow.python.eager.context import in_eager_mode
 from tensorflow.python.eager.context import in_graph_mode
 from tensorflow.python.eager.context import list_devices
diff --git a/tensorflow/python/eager/context.py b/tensorflow/python/eager/context.py
index 58581283d2..c5eedb7c9c 100644
--- a/tensorflow/python/eager/context.py
+++ b/tensorflow/python/eager/context.py
@@ -42,6 +42,10 @@ _device_parsing_cache = {}
 
 _MAXINT32 = 2**31 - 1
 
+DEVICE_PLACEMENT_EXPLICIT = pywrap_tensorflow.TFE_DEVICE_PLACEMENT_EXPLICIT
+DEVICE_PLACEMENT_WARN = pywrap_tensorflow.TFE_DEVICE_PLACEMENT_WARN
+DEVICE_PLACEMENT_SILENT = pywrap_tensorflow.TFE_DEVICE_PLACEMENT_SILENT
+
 
 # TODO(agarwal): better name ?
 class _EagerContext(threading.local):
@@ -62,13 +66,22 @@ class _EagerContext(threading.local):
 class Context(object):
   """Environment in which eager operations execute."""
 
-  def __init__(self, config=None):
+  def __init__(self, config=None, device_policy=None):
     """Creates a new Context.
 
     Args:
       config: (Optional.) A `ConfigProto` protocol buffer with configuration
-      options for the Context. Note that a lot of these options may be
-      currently unimplemented or irrelevant for EAGER mode.
+       options for the Context. Note that a lot of these options may be
+       currently unimplemented or irrelevant when eager execution is enabled.
+      device_policy: (Optional.) What policy to use when trying to run an
+       operation on a device with inputs which are not on that device.
+       Valid values:
+         tfe.DEVICE_PLACEMENT_EXPLICIT: raises an error if the placement is not
+           correct.
+         tfe.DEVICE_PLACEMENT_WARN: copies the tensors which are not on the
+           right device but raises a warning.
+         tfe.DEVICE_PLACEMENT_SILENT: silently copies the tensors. This might
+           hide performance problems.
     """
     self._eager_context = _EagerContext()
     self._context_handle = None
@@ -78,6 +91,7 @@ class Context(object):
     self._config = config
     self._seed = None
     self._initialize_lock = threading.Lock()
+    self._device_policy = device_policy
 
   def _set_global_seed(self, seed):
     """Set a global eager mode seed for random ops."""
@@ -109,6 +123,9 @@ class Context(object):
             config_str = self._config.SerializeToString()
             pywrap_tensorflow.TFE_ContextOptionsSetConfig(
                 opts, config_str, len(config_str), status)
+          if self._device_policy is not None:
+            pywrap_tensorflow.TFE_ContextOptionsSetDevicePlacementPolicy(
+                opts, self._device_policy)
           self._context_handle = pywrap_tensorflow.TFE_NewContext(opts, status)
       finally:
         pywrap_tensorflow.TFE_DeleteContextOptions(opts)
diff --git a/tensorflow/python/eager/ops_test.py b/tensorflow/python/eager/ops_test.py
index 2ebb625f9f..1cd3826755 100644
--- a/tensorflow/python/eager/ops_test.py
+++ b/tensorflow/python/eager/ops_test.py
@@ -266,6 +266,23 @@ class OpsTest(test_util.TensorFlowTestCase):
     shape = array_ops.shape(value)
     self.assertEqual([1], shape.numpy())
 
+  def testSilentCopy(self):
+    if not context.context().num_gpus():
+      self.skipTest('No GPUs found')
+    # Temporarily replace the context
+    # pylint: disable=protected-access
+    del context._context
+    try:
+      context._context = context.Context(
+          device_policy=context.DEVICE_PLACEMENT_SILENT)
+      cpu_tensor = constant_op.constant(1.0)
+      gpu_tensor = cpu_tensor.gpu()
+      self.assertAllEqual(cpu_tensor + gpu_tensor, 2.0)
+    finally:
+      del context._context
+      context._context = context.Context()
+    # pylint: enable=protected-access
+
   def testRandomUniform(self):
     scalar_shape = constant_op.constant([], dtype=dtypes.int32)
 
diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index ef708a4703..b45cb2e0c6 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -4559,7 +4559,7 @@ class _DefaultGraphStack(_DefaultStack):  # pylint: disable=protected-access
 _default_graph_stack = _DefaultGraphStack()
 
 
-def enable_eager_execution():
+def enable_eager_execution(config=None, device_policy=None):
   """Enables, for the rest of the lifetime of this program, eager execution.
 
   If not called immediately on startup risks creating breakage and bugs.
@@ -4574,8 +4574,24 @@ def enable_eager_execution():
   assert tf.multiply(6, 7).numpy() == 42
   ```
 
+  Args:
+    config: (Optional.) A `ConfigProto` protocol buffer with configuration
+     options for the Context. Note that a lot of these options may be
+     currently unimplemented or irrelevant when eager execution is enabled.
+    device_policy: (Optional.) What policy to use when trying to run an
+     operation on a device with inputs which are not on that device.
+     Valid values:
+       tfe.DEVICE_PLACEMENT_EXPLICIT: raises an error if the placement is not
+         correct.
+       tfe.DEVICE_PLACEMENT_WARN: copies the tensors which are not on the
+         right device but raises a warning.
+       tfe.DEVICE_PLACEMENT_SILENT: silently copies the tensors. This might
+         hide performance problems.
+
   Raises:
-    ValueError: If this method has already been invoked in the current process.
+    ValueError: If trying to create a context after using graph operations
+     or if trying to create a context with nontrivial options which differ
+     from those of the existing context.
   """
   # pylint: disable=protected-access
   if context._default_mode == context.GRAPH_MODE:
@@ -4586,6 +4602,18 @@ def enable_eager_execution():
       raise ValueError(
           "tfe.enable_eager_execution has to be called at program startup.")
   context._default_mode = context.EAGER_MODE
+  if context._context is None:
+    context._context = context.Context(config=config,
+                                       device_policy=device_policy)
+  elif ((config is not None and config is not context._context._config)
+        or (device_policy is not None
+            and device_policy is not context._context._device_policy)):
+    raise ValueError("Trying to change the options of an active eager"
+                     " execution. Context config: %s, specified config:"
+                     " %s. Context device policy: %s; specified device"
+                     " policy: %s." % (config, context._context._config,
+                                       device_policy,
+                                       context._context._device_policy))
 
 
 def eager_run(main=None, argv=None):
diff --git a/tensorflow/python/pywrap_tfe.i b/tensorflow/python/pywrap_tfe.i
index 36c09c20c2..fa36b77311 100644
--- a/tensorflow/python/pywrap_tfe.i
+++ b/tensorflow/python/pywrap_tfe.i
@@ -32,6 +32,7 @@ limitations under the License.
 %rename("%s") TFE_Py_TapeExport;
 %rename("%s") TFE_NewContextOptions;
 %rename("%s") TFE_ContextOptionsSetConfig;
+%rename("%s") TFE_ContextOptionsSetDevicePlacementPolicy;
 %rename("%s") TFE_DeleteContextOptions;
 
 %{
@@ -101,6 +102,11 @@ limitations under the License.
   }
 }
 
+%rename("%s") TFE_ContextDevicePlacementPolicy;
+%rename("%s") TFE_DEVICE_PLACEMENT_EXPLICIT;
+%rename("%s") TFE_DEVICE_PLACEMENT_WARN;
+%rename("%s") TFE_DEVICE_PLACEMENT_SILENT;
+
 %include "tensorflow/c/eager/c_api.h"
 
 %typemap(in) TFE_InputTensorHandles* inputs (TFE_InputTensorHandles temp) {
-- 
GitLab


From ccfd9c1e50934e4b16d40d4d5d87206ad871996d Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Fri, 20 Oct 2017 15:40:34 -0700
Subject: [PATCH 635/909] Log Hlo IR during AOT compilation

PiperOrigin-RevId: 172944165
---
 tensorflow/compiler/xla/service/cpu/cpu_compiler.cc | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
index 06e7ec0c7c..99b5035c2d 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
@@ -757,8 +757,14 @@ CpuCompiler::CompileAheadOfTime(std::vector<std::unique_ptr<HloModule>> modules,
     HloModule* module = modules[i].get();
     VLOG(1) << "Compiling ahead-of-time: " << module->name();
 
+    VLOG(2) << "Before optimization:";
+    XLA_VLOG_LINES(2, module->ToString());
+
     TF_RETURN_IF_ERROR(RunHloPasses(module, /*is_aot_compile=*/true));
 
+    VLOG(2) << "After optimization:";
+    XLA_VLOG_LINES(2, module->ToString());
+
     TF_ASSIGN_OR_RETURN(
         SequentialHloOrdering::HloModuleSequence module_sequence,
         CreateMemoryMinimizingSequence(*module, BufferSizeBytesFunction()));
-- 
GitLab


From ebcae4a5e3bf5c840d73a0d90f1b5bf01a68f82c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 20 Oct 2017 15:55:17 -0700
Subject: [PATCH 636/909] Add streaming_precision_recall_at_equal_thresholds

This helper method computes streaming tp, fp, tn, fp, precision, and recall for the user in a way that exhibits O(T + N) time and space complexity (instead of O(T * N)), where T is the number of thresholds and N is the size of the predictions tensor.

Thanks to Frank Chu for the efficient algorithm!

PiperOrigin-RevId: 172946073
---
 .../contrib/metrics/python/ops/metric_ops.py  | 180 ++++++++++++++++++
 .../metrics/python/ops/metric_ops_test.py     | 164 ++++++++++++++++
 2 files changed, 344 insertions(+)

diff --git a/tensorflow/contrib/metrics/python/ops/metric_ops.py b/tensorflow/contrib/metrics/python/ops/metric_ops.py
index 09485c4fa2..5a4c0c4358 100644
--- a/tensorflow/contrib/metrics/python/ops/metric_ops.py
+++ b/tensorflow/contrib/metrics/python/ops/metric_ops.py
@@ -22,6 +22,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import collections as collections_lib
+
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
@@ -1076,6 +1078,9 @@ def streaming_curve_points(labels=None,
       `weights` is not `None` and its shape doesn't match `predictions`, or if
       either `metrics_collections` or `updates_collections` are not a list or
       tuple.
+
+  TODO(chizeng): Consider rewriting this method to make use of logic within the
+  streaming_precision_recall_at_equal_thresholds method (to improve run time).
   """
   with variable_scope.variable_scope(name, 'curve_points',
                                      (labels, predictions, weights)):
@@ -1193,6 +1198,181 @@ def streaming_auc(predictions,
       name=name)
 
 
+def streaming_precision_recall_at_equal_thresholds(predictions,
+                                                   labels,
+                                                   num_thresholds=None,
+                                                   weights=None,
+                                                   name=None,
+                                                   use_locking=None):
+  """A helper method for creating metrics related to precision-recall curves.
+
+  These values are true positives, false negatives, true negatives, false
+  positives, precision, and recall. This function returns a data structure that
+  contains ops within it.
+
+  Unlike _streaming_confusion_matrix_at_thresholds (which exhibits O(T * N)
+  space and run time), this op exhibits O(T + N) space and run time, where T is
+  the number of thresholds and N is the size of the predictions tensor. Hence,
+  it may be advantageous to use this function when `predictions` is big.
+
+  For instance, prefer this method for per-pixel classification tasks, for which
+  the predictions tensor may be very large.
+
+  Each number in `predictions`, a float in `[0, 1]`, is compared with its
+  corresponding label in `labels`, and counts as a single tp/fp/tn/fn value at
+  each threshold. This is then multiplied with `weights` which can be used to
+  reweight certain values, or more commonly used for masking values.
+
+  Args:
+    predictions: A floating point `Tensor` of arbitrary shape and whose values
+      are in the range `[0, 1]`.
+    labels: A bool `Tensor` whose shape matches `predictions`.
+    num_thresholds: Optional; Number of thresholds, evenly distributed in
+      `[0, 1]`. Should be `>= 2`. Defaults to 201. Note that the number of bins
+      is 1 less than `num_thresholds`. Using an even `num_thresholds` value
+      instead of an odd one may yield unfriendly edges for bins.
+    weights: Optional; If provided, a `Tensor` that has the same dtype as,
+      and broadcastable to, `predictions`. This tensor is multplied by counts.
+    name: Optional; variable_scope name. If not provided, the string
+      'precision_recall_at_equal_threshold' is used.
+    use_locking: Optional; If True, the op will be protected by a lock.
+      Otherwise, the behavior is undefined, but may exhibit less contention.
+      Defaults to True.
+
+  Returns:
+    result: A named tuple (See PrecisionRecallData within the implementation of
+      this function) with properties that are variables of shape
+      `[num_thresholds]`. The names of the properties are tp, fp, tn, fn,
+      precision, recall, thresholds.
+    update_op: An op that accumulates values.
+
+  Raises:
+    ValueError: If `predictions` and `labels` have mismatched shapes, or if
+      `weights` is not `None` and its shape doesn't match `predictions`, or if
+      `includes` contains invalid keys.
+  """
+  # Disable the invalid-name checker so that we can capitalize the name.
+  # pylint: disable=invalid-name
+  PrecisionRecallData = collections_lib.namedtuple(
+      'PrecisionRecallData',
+      ['tp', 'fp', 'tn', 'fn', 'precision', 'recall', 'thresholds'])
+  # pylint: enable=invalid-name
+
+  if num_thresholds is None:
+    num_thresholds = 201
+
+  if weights is None:
+    weights = 1.0
+
+  if use_locking is None:
+    use_locking = True
+
+  check_ops.assert_type(labels, dtypes.bool)
+
+  dtype = predictions.dtype
+  with variable_scope.variable_scope(name,
+                                     'precision_recall_at_equal_thresholds',
+                                     (labels, predictions, weights)):
+    # Make sure that predictions are within [0.0, 1.0].
+    with ops.control_dependencies([
+        check_ops.assert_greater_equal(
+            predictions,
+            math_ops.cast(0.0, dtype=predictions.dtype),
+            message='predictions must be in [0, 1]'),
+        check_ops.assert_less_equal(
+            predictions,
+            math_ops.cast(1.0, dtype=predictions.dtype),
+            message='predictions must be in [0, 1]')
+    ]):
+      predictions, labels, weights = _remove_squeezable_dimensions(
+          predictions=predictions, labels=labels, weights=weights)
+
+    predictions.get_shape().assert_is_compatible_with(labels.get_shape())
+
+    # We cast to float to ensure we have 0.0 or 1.0.
+    f_labels = math_ops.cast(labels, dtype)
+
+    # Get weighted true/false labels.
+    true_labels = f_labels * weights
+    false_labels = (1.0 - f_labels) * weights
+
+    # Flatten predictions and labels.
+    predictions = array_ops.reshape(predictions, [-1])
+    true_labels = array_ops.reshape(true_labels, [-1])
+    false_labels = array_ops.reshape(false_labels, [-1])
+
+    # To compute TP/FP/TN/FN, we are measuring a binary classifier
+    #   C(t) = (predictions >= t)
+    # at each threshold 't'. So we have
+    #   TP(t) = sum( C(t) * true_labels )
+    #   FP(t) = sum( C(t) * false_labels )
+    #
+    # But, computing C(t) requires computation for each t. To make it fast,
+    # observe that C(t) is a cumulative integral, and so if we have
+    #   thresholds = [t_0, ..., t_{n-1}];  t_0 < ... < t_{n-1}
+    # where n = num_thresholds, and if we can compute the bucket function
+    #   B(i) = Sum( (predictions == t), t_i <= t < t{i+1} )
+    # then we get
+    #   C(t_i) = sum( B(j), j >= i )
+    # which is the reversed cumulative sum in tf.cumsum().
+    #
+    # We can compute B(i) efficiently by taking advantage of the fact that
+    # our thresholds are evenly distributed, in that
+    #   width = 1.0 / (num_thresholds - 1)
+    #   thresholds = [0.0, 1*width, 2*width, 3*width, ..., 1.0]
+    # Given a prediction value p, we can map it to its bucket by
+    #   bucket_index(p) = floor( p * (num_thresholds - 1) )
+    # so we can use tf.scatter_add() to update the buckets in one pass.
+    #
+    # This implementation exhibits a run time and space complexity of O(T + N),
+    # where T is the number of thresholds and N is the size of predictions.
+    # Metrics that rely on _streaming_confusion_matrix_at_thresholds instead
+    # exhibit a complexity of O(T * N).
+
+    # Compute the bucket indices for each prediction value.
+    bucket_indices = math_ops.cast(
+        math_ops.floor(predictions * (num_thresholds - 1)), dtypes.int32)
+
+    with ops.name_scope('variables'):
+      tp_buckets_v = _create_local(
+          'tp_buckets', shape=[num_thresholds], dtype=dtype)
+      fp_buckets_v = _create_local(
+          'fp_buckets', shape=[num_thresholds], dtype=dtype)
+
+    with ops.name_scope('update_op'):
+      update_tp = state_ops.scatter_add(
+          tp_buckets_v, bucket_indices, true_labels, use_locking=use_locking)
+      update_fp = state_ops.scatter_add(
+          fp_buckets_v, bucket_indices, false_labels, use_locking=use_locking)
+
+    # Set up the cumulative sums to compute the actual metrics.
+    tp = math_ops.cumsum(tp_buckets_v, reverse=True, name='tp')
+    fp = math_ops.cumsum(fp_buckets_v, reverse=True, name='fp')
+    # fn = sum(true_labels) - tp
+    #    = sum(tp_buckets) - tp
+    #    = tp[0] - tp
+    # Similarly,
+    # tn = fp[0] - fp
+    tn = fp[0] - fp
+    fn = tp[0] - tp
+
+    # We use a minimum to prevent division by 0.
+    epsilon = 1e-7
+    precision = tp / math_ops.maximum(epsilon, tp + fp)
+    recall = tp / math_ops.maximum(epsilon, tp + fn)
+
+    result = PrecisionRecallData(
+        tp=tp,
+        fp=fp,
+        tn=tn,
+        fn=fn,
+        precision=precision,
+        recall=recall,
+        thresholds=math_ops.lin_space(0.0, 1.0, num_thresholds))
+    update_op = control_flow_ops.group(update_tp, update_fp)
+    return result, update_op
+
+
 def streaming_specificity_at_sensitivity(predictions,
                                          labels,
                                          sensitivity,
diff --git a/tensorflow/contrib/metrics/python/ops/metric_ops_test.py b/tensorflow/contrib/metrics/python/ops/metric_ops_test.py
index f288fceef6..f24bec7f11 100644
--- a/tensorflow/contrib/metrics/python/ops/metric_ops_test.py
+++ b/tensorflow/contrib/metrics/python/ops/metric_ops_test.py
@@ -1970,6 +1970,170 @@ class StreamingAUCTest(test.TestCase):
         self.assertAlmostEqual(expected_auc, auc.eval(), 2)
 
 
+class StreamingPrecisionRecallAtEqualThresholdsTest(test.TestCase):
+
+  def setUp(self):
+    np.random.seed(1)
+    ops.reset_default_graph()
+
+  def _testResultsEqual(self, expected_dict, gotten_result):
+    """Tests that 2 results (dicts) represent the same data.
+
+    Args:
+      expected_dict: A dictionary with keys that are the names of properties
+        of PrecisionRecallData and whose values are lists of floats.
+      gotten_result: A PrecisionRecallData object.
+    """
+    gotten_dict = {k: t.eval() for k, t in gotten_result._asdict().items()}
+    self.assertItemsEqual(
+        list(expected_dict.keys()), list(gotten_dict.keys()))
+
+    for key, expected_values in expected_dict.items():
+      self.assertAllClose(expected_values, gotten_dict[key])
+
+  def _testCase(self, predictions, labels, expected_result, weights=None):
+    """Performs a test given a certain scenario of labels, predictions, weights.
+
+    Args:
+      predictions: The predictions tensor. Of type float32.
+      labels: The labels tensor. Of type bool.
+      expected_result: The expected result (dict) that maps to tensors.
+      weights: Optional weights tensor.
+    """
+    with self.test_session() as sess:
+      predictions_tensor = constant_op.constant(
+          predictions, dtype=dtypes_lib.float32)
+      labels_tensor = constant_op.constant(labels, dtype=dtypes_lib.bool)
+      weights_tensor = None
+      if weights:
+        weights_tensor = constant_op.constant(weights, dtype=dtypes_lib.float32)
+      gotten_result, update_op = (
+          metric_ops.streaming_precision_recall_at_equal_thresholds(
+              predictions=predictions_tensor,
+              labels=labels_tensor,
+              num_thresholds=3,
+              weights=weights_tensor))
+
+      sess.run(variables.local_variables_initializer())
+      sess.run(update_op)
+
+      self._testResultsEqual(expected_result, gotten_result)
+
+  def testVars(self):
+    metric_ops.streaming_precision_recall_at_equal_thresholds(
+        predictions=constant_op.constant([0.42], dtype=dtypes_lib.float32),
+        labels=constant_op.constant([True], dtype=dtypes_lib.bool))
+    _assert_local_variables(
+        self,
+        (
+            'precision_recall_at_equal_thresholds/variables/tp_buckets:0',
+            'precision_recall_at_equal_thresholds/variables/fp_buckets:0'
+        ))
+
+  def testVarsWithName(self):
+    metric_ops.streaming_precision_recall_at_equal_thresholds(
+        predictions=constant_op.constant([0.42], dtype=dtypes_lib.float32),
+        labels=constant_op.constant([True], dtype=dtypes_lib.bool),
+        name='foo')
+    _assert_local_variables(
+        self, ('foo/variables/tp_buckets:0', 'foo/variables/fp_buckets:0'))
+
+  def testValuesAreIdempotent(self):
+    predictions = constant_op.constant(
+        np.random.uniform(size=(10, 3)), dtype=dtypes_lib.float32)
+    labels = constant_op.constant(
+        np.random.uniform(size=(10, 3)) > 0.5, dtype=dtypes_lib.bool)
+
+    result, update_op = (
+        metric_ops.streaming_precision_recall_at_equal_thresholds(
+            predictions=predictions, labels=labels))
+
+    with self.test_session() as sess:
+      # Run several updates.
+      sess.run(variables.local_variables_initializer())
+      for _ in range(3):
+        sess.run(update_op)
+
+      # Then verify idempotency.
+      initial_result = {k: value.eval().tolist() for k, value in
+                        result._asdict().items()}
+      for _ in range(3):
+        self._testResultsEqual(initial_result, result)
+
+  def testAllTruePositives(self):
+    self._testCase([[1]], [[True]], {
+        'tp': [1, 1, 1],
+        'fp': [0, 0, 0],
+        'tn': [0, 0, 0],
+        'fn': [0, 0, 0],
+        'precision': [1.0, 1.0, 1.0],
+        'recall': [1.0, 1.0, 1.0],
+        'thresholds': [0.0, 0.5, 1.0],
+    })
+
+  def testAllTrueNegatives(self):
+    self._testCase([[0]], [[False]], {
+        'tp': [0, 0, 0],
+        'fp': [1, 0, 0],
+        'tn': [0, 1, 1],
+        'fn': [0, 0, 0],
+        'precision': [0.0, 0.0, 0.0],
+        'recall': [0.0, 0.0, 0.0],
+        'thresholds': [0.0, 0.5, 1.0],
+    })
+
+  def testAllFalsePositives(self):
+    self._testCase([[1]], [[False]], {
+        'tp': [0, 0, 0],
+        'fp': [1, 1, 1],
+        'tn': [0, 0, 0],
+        'fn': [0, 0, 0],
+        'precision': [0.0, 0.0, 0.0],
+        'recall': [0.0, 0.0, 0.0],
+        'thresholds': [0.0, 0.5, 1.0],
+    })
+
+  def testAllFalseNegatives(self):
+    self._testCase([[0]], [[True]], {
+        'tp': [1, 0, 0],
+        'fp': [0, 0, 0],
+        'tn': [0, 0, 0],
+        'fn': [0, 1, 1],
+        'precision': [1.0, 0.0, 0.0],
+        'recall': [1.0, 0.0, 0.0],
+        'thresholds': [0.0, 0.5, 1.0],
+    })
+
+  def testManyValues(self):
+    self._testCase(
+        [[0.2, 0.3, 0.4, 0.6, 0.7, 0.8]],
+        [[True, False, False, True, True, True]],
+        {
+            'tp': [4, 3, 0],
+            'fp': [2, 0, 0],
+            'tn': [0, 2, 2],
+            'fn': [0, 1, 4],
+            'precision': [2.0 / 3.0, 1.0, 0.0],
+            'recall': [1.0, 0.75, 0.0],
+            'thresholds': [0.0, 0.5, 1.0],
+        })
+
+  def testManyValuesWithWeights(self):
+    self._testCase(
+        [[0.2, 0.3, 0.4, 0.6, 0.7, 0.8]],
+        [[True, False, False, True, True, True]],
+        {
+            'tp': [1.5, 1.5, 0.0],
+            'fp': [2.5, 0.0, 0.0],
+            'tn': [0.0, 2.5, 2.5],
+            'fn': [0.0, 0.0, 1.5],
+            'precision': [0.375, 1.0, 0.0],
+            'recall': [1.0, 1.0, 0.0],
+            'thresholds': [0.0, 0.5, 1.0],
+        },
+        weights=[0.0, 0.5, 2.0, 0.0, 0.5, 1.0])
+
+
 class StreamingSpecificityAtSensitivityTest(test.TestCase):
 
   def setUp(self):
-- 
GitLab


From 8ff33271ea4de89e6ff662fe8e479c1fcf56fe77 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Fri, 20 Oct 2017 15:55:57 -0700
Subject: [PATCH 637/909] Dump the computation's SessionModule as part of the
 tf_compile rule.

PiperOrigin-RevId: 172946149
---
 tensorflow/compiler/aot/compile.cc    | 6 +++---
 tensorflow/compiler/aot/flags.cc      | 5 ++---
 tensorflow/compiler/aot/flags.h       | 2 +-
 tensorflow/compiler/aot/tfcompile.bzl | 3 +++
 4 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/tensorflow/compiler/aot/compile.cc b/tensorflow/compiler/aot/compile.cc
index eac8da0ab1..77c4ec88cb 100644
--- a/tensorflow/compiler/aot/compile.cc
+++ b/tensorflow/compiler/aot/compile.cc
@@ -97,11 +97,11 @@ Status CompileGraph(const GraphDef& graph_def, const tf2xla::Config& config,
   TF_RETURN_IF_ERROR(ConvertGraphDefToXla(graph_def, config, client,
                                           &computation,
                                           &compile_result->has_context_arg));
-  if (!flags.debug_dir.empty()) {
+  if (!flags.out_session_module.empty()) {
     TF_ASSIGN_OR_RETURN(std::unique_ptr<xla::SessionModule> module,
                         computation.Snapshot());
-    string file = io::JoinPath(flags.debug_dir, "tfcompile_xla_module.pb");
-    TF_RETURN_IF_ERROR(WriteBinaryProto(Env::Default(), file, *module));
+    TF_RETURN_IF_ERROR(
+        WriteBinaryProto(Env::Default(), flags.out_session_module, *module));
   }
   xla::cpu::CpuAotCompilationOptions aot_opts(
       flags.target_triple, flags.target_cpu, flags.target_features,
diff --git a/tensorflow/compiler/aot/flags.cc b/tensorflow/compiler/aot/flags.cc
index 5aff10346f..7c2f27e550 100644
--- a/tensorflow/compiler/aot/flags.cc
+++ b/tensorflow/compiler/aot/flags.cc
@@ -33,9 +33,6 @@ void AppendMainFlags(std::vector<Flag>* flag_list, MainFlags* flags) {
        "fetch nodes will be dumped to stdout in a comma-separated list.  "
        "Typically used to format arguments for other tools, e.g. "
        "freeze_graph."},
-      {"debug_dir", &flags->debug_dir,
-       "Specifies a directory to dump debugging information, including "
-       "rewritten graphs and the XLA HLO module."},
       // Flags controlling the XLA ahead-of-time compilation, that correspond to
       // the fields of xla::cpu::CpuAotCompilationOptions.
       //
@@ -64,6 +61,8 @@ void AppendMainFlags(std::vector<Flag>* flag_list, MainFlags* flags) {
        "namespaces are given, within the global namespace."},
       {"out_object", &flags->out_object, "Output object file name."},
       {"out_header", &flags->out_header, "Output header file name."},
+      {"out_session_module", &flags->out_session_module,
+       "Output session module proto."},
       {"gen_name_to_index", &flags->gen_name_to_index,
        "Generate name-to-index data for Lookup{Arg,Result}Index methods."},
       {"gen_program_shape", &flags->gen_program_shape,
diff --git a/tensorflow/compiler/aot/flags.h b/tensorflow/compiler/aot/flags.h
index 3246dbf95c..3519659e3a 100644
--- a/tensorflow/compiler/aot/flags.h
+++ b/tensorflow/compiler/aot/flags.h
@@ -29,7 +29,6 @@ struct MainFlags {
   string graph;
   string config;
   bool dump_fetch_nodes = false;
-  string debug_dir;
   string target_triple;
   string target_cpu;
   string target_features;
@@ -37,6 +36,7 @@ struct MainFlags {
   string cpp_class;
   string out_object;
   string out_header;
+  string out_session_module;
 
   // C++ codegen options
   bool gen_name_to_index = false;
diff --git a/tensorflow/compiler/aot/tfcompile.bzl b/tensorflow/compiler/aot/tfcompile.bzl
index 4888760acd..0ecfbedcb4 100644
--- a/tensorflow/compiler/aot/tfcompile.bzl
+++ b/tensorflow/compiler/aot/tfcompile.bzl
@@ -129,6 +129,7 @@ def tf_library(name, graph, config,
   # Rule that runs tfcompile to produce the header and object file.
   header_file = name + ".h"
   object_file = name + ".o"
+  session_module_pb = name + "_session_module.pb"
   ep = ("__" + PACKAGE_NAME + "__" + name).replace("/", "_")
   native.genrule(
       name=("gen_" + name),
@@ -139,6 +140,7 @@ def tf_library(name, graph, config,
       outs=[
           header_file,
           object_file,
+          session_module_pb,
       ],
       cmd=("$(location " + tfcompile_tool + ")" +
            " --graph=$(location " + tfcompile_graph + ")" +
@@ -148,6 +150,7 @@ def tf_library(name, graph, config,
            " --target_triple=" + target_llvm_triple() +
            " --out_header=$(@D)/" + header_file +
            " --out_object=$(@D)/" + object_file +
+           " --out_session_module=$(@D)/" + session_module_pb +
            " " + (tfcompile_flags or "")),
       tools=[tfcompile_tool],
       visibility=visibility,
-- 
GitLab


From c0ca50a47724363e5edb4de6afe28f5c60cd2eda Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Yan=20Facai=20=28=E9=A2=9C=E5=8F=91=E6=89=8D=29?=
 <facai.yan@gmail.com>
Date: Sat, 21 Oct 2017 07:08:48 +0800
Subject: [PATCH 638/909] ENH: add Relu6GradGrad (#13268)

* ENH: add Relu6GradGrad

* TST: add test case

* CLN: import nn_grad

* TST: add init value
---
 tensorflow/python/BUILD               | 13 ++++++++
 tensorflow/python/ops/nn_grad.py      |  7 ++++
 tensorflow/python/ops/nn_grad_test.py | 48 +++++++++++++++++++++++++++
 3 files changed, 68 insertions(+)
 create mode 100644 tensorflow/python/ops/nn_grad_test.py

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index b7aa7bbf6b..933e0e3e8c 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -4201,6 +4201,19 @@ cuda_py_test(
     main = "client/session_benchmark.py",
 )
 
+cuda_py_test(
+    name = "nn_grad_test",
+    size = "small",
+    srcs = ["ops/nn_grad_test.py"],
+    additional_deps = [
+        ":client_testlib",
+        ":framework_for_generated_wrappers",
+        ":nn_grad",
+        ":nn_ops",
+        "//third_party/py/numpy",
+    ],
+)
+
 py_library(
     name = "tf_item",
     srcs = [
diff --git a/tensorflow/python/ops/nn_grad.py b/tensorflow/python/ops/nn_grad.py
index c7c745142b..557f39fb42 100644
--- a/tensorflow/python/ops/nn_grad.py
+++ b/tensorflow/python/ops/nn_grad.py
@@ -352,6 +352,13 @@ def _Relu6Grad(op, grad):
   return gen_nn_ops._relu6_grad(grad, op.outputs[0])  # pylint: disable=protected-access
 
 
+@ops.RegisterGradient("Relu6Grad")
+def _Relu6GradGrad(op, grad):
+  x = op.inputs[1]
+  return (gen_nn_ops._relu6_grad(grad, x), array_ops.zeros(
+      shape=array_ops.shape(x), dtype=x.dtype))
+
+
 @ops.RegisterGradient("Elu")
 def _EluGrad(op, grad):
   return gen_nn_ops._elu_grad(grad, op.outputs[0])
diff --git a/tensorflow/python/ops/nn_grad_test.py b/tensorflow/python/ops/nn_grad_test.py
new file mode 100644
index 0000000000..f7541c0e89
--- /dev/null
+++ b/tensorflow/python/ops/nn_grad_test.py
@@ -0,0 +1,48 @@
+# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for Python ops defined in nn_grad.py."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.ops import gradient_checker
+from tensorflow.python.ops import gradients_impl
+from tensorflow.python.ops import nn_grad
+from tensorflow.python.ops import nn_ops
+from tensorflow.python.platform import test
+
+
+class Relu6OpTest(test.TestCase):
+  def testRelu6GradGrad(self):
+    inputs = constant_op.constant([[-2, -1, 1, 3], [5, 7, 8, 9]],
+                                  dtype=dtypes.float32)
+    x_init_value = np.array([[-3.5, -1.5, 2, 4], [4.5, 7.5, 8.5, 11]])
+    r = nn_ops.relu6(inputs)
+    r_g = gradients_impl.gradients(r, inputs)[0]
+    with self.test_session():
+      error = gradient_checker.compute_gradient_error(
+        inputs, inputs.get_shape().as_list(),
+        r_g, r_g.get_shape().as_list(),
+        x_init_value=x_init_value)
+      self.assertLess(error, 1e-4)
+
+
+if __name__ == "__main__":
+  test.main()
-- 
GitLab


From 9c825d32c9423980e1b263a50360e03e833b69a6 Mon Sep 17 00:00:00 2001
From: Jinze Bai <baijinze1994@163.com>
Date: Sat, 21 Oct 2017 07:12:31 +0800
Subject: [PATCH 639/909] Merge two GPU kernel launching to one in DiagOp.
 (#13859)

---
 tensorflow/core/kernels/diag_op_gpu.cu.cc | 49 +++++++++--------------
 1 file changed, 19 insertions(+), 30 deletions(-)

diff --git a/tensorflow/core/kernels/diag_op_gpu.cu.cc b/tensorflow/core/kernels/diag_op_gpu.cu.cc
index 9878f347d2..684f00ea61 100644
--- a/tensorflow/core/kernels/diag_op_gpu.cu.cc
+++ b/tensorflow/core/kernels/diag_op_gpu.cu.cc
@@ -33,15 +33,12 @@ __global__ void DiagCudaKernel(const int num_threads,
                                const T* in,
                                T* out) {
   CUDA_1D_KERNEL_LOOP(index, num_threads) {
-    out[(1 + size) * index] = in[index];
-  }
-}
-
-template <typename T>
-__global__ void ZeroCudaKernel(const int num_threads,
-                               T* out) {
-  CUDA_1D_KERNEL_LOOP(index, num_threads) {
-    out[index] = T(0);
+    // Fill the diagonal elements or set to zero in other place. 
+    if (index % (1 + size) == 0) {
+      out[index] = in[index / (1 + size)];
+    } else {
+      out[index] = T(0);
+    }
   }
 }
 
@@ -50,39 +47,30 @@ struct DiagFunctor<GPUDevice, T> {
   EIGEN_ALWAYS_INLINE Status
   operator() (OpKernelContext* context, const int64 size,
               const T* in, T* out) {
-    // CudaLaunchConfig uses an int for virtual_thread_count,
-    // so this may overflow in extreme cases.
-    if (size && (size * size / size) != size) {
-      return errors::Internal(
-          "DiagOp got input size too large.");
-    }
-
     // Empty tensor couldn't launch the kernel.
     if (size == 0) {
       return Status::OK();
     }
-    const GPUDevice& device = context->eigen_device<GPUDevice>();
 
-    // Set output memory with zero elements.
-    CudaLaunchConfig zero_config = GetCudaLaunchConfig(size*size, device);
-    ZeroCudaKernel<<<zero_config.block_count,
-                     zero_config.thread_per_block,
-                     0, device.stream()>>>(
-        zero_config.virtual_thread_count, out);
-    auto err = cudaGetLastError();
-    if (err != cudaSuccess) {
+    // CudaLaunchConfig uses an int for virtual_thread_count,
+    // so this may overflow for `size*size` in extreme cases,
+    // here is checking the multiplication overflow for integer.
+    if (size && (int(size * size) / size) != size) {
       return errors::Internal(
-          "Could not launch DiagOp kernel: ",
-          cudaGetErrorString(err), ".");
+          "DiagOp got input size too large.");
     }
+    int virtual_thread_count = int(size * size);
 
-    // Fill the diagonal elements
-    CudaLaunchConfig diag_config = GetCudaLaunchConfig(size, device);
+    // Launch the GPU kernel.
+    const GPUDevice& device = context->eigen_device<GPUDevice>();
+    CudaLaunchConfig diag_config = GetCudaLaunchConfig(
+        virtual_thread_count, device);
     DiagCudaKernel<<<diag_config.block_count,
                      diag_config.thread_per_block,
                      0, device.stream()>>>(
         diag_config.virtual_thread_count, size, in, out);
-    err = cudaGetLastError();
+
+    auto err = cudaGetLastError();
     if (err != cudaSuccess) {
       return errors::Internal(
           "Could not launch DiagOp kernel: ",
@@ -127,6 +115,7 @@ struct DiagPartFunctor<GPUDevice, T> {
                      diag_config.thread_per_block,
                      0, device.stream()>>>(
         diag_config.virtual_thread_count, size, in, out);
+
     auto err = cudaGetLastError();
     if (err != cudaSuccess) {
       return errors::Internal(
-- 
GitLab


From 49483793695247f27332c7db0b9740e95a5de3db Mon Sep 17 00:00:00 2001
From: "Joshua V. Dillon" <jvdillon@google.com>
Date: Fri, 20 Oct 2017 16:28:55 -0700
Subject: [PATCH 640/909] Make `tf.contrib.distributions` quadrature family
 accept a `Tensor` for `quadrature_grid_and_probs` argument.

PiperOrigin-RevId: 172950094
---
 .../bijectors/masked_autoregressive_test.py   |  4 +-
 .../python/kernel_tests/independent_test.py   |  4 +-
 .../kernel_tests/mixture_same_family_test.py  |  6 +-
 .../kernel_tests/poisson_lognormal_test.py    | 28 +++++--
 .../kernel_tests/vector_diffeomixture_test.py | 53 +++++++++++---
 .../vector_sinh_arcsinh_diag_test.py          | 12 +--
 .../python/ops/poisson_lognormal.py           | 53 +++++++-------
 .../distributions/python/ops/test_util.py     | 34 +++++----
 .../python/ops/vector_diffeomixture.py        | 46 ++++++------
 tensorflow/python/ops/distributions/util.py   | 73 ++++++++++++++++++-
 10 files changed, 216 insertions(+), 97 deletions(-)

diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/masked_autoregressive_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/masked_autoregressive_test.py
index 98c09545ac..25a9b6f5fe 100644
--- a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/masked_autoregressive_test.py
+++ b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/masked_autoregressive_test.py
@@ -111,7 +111,7 @@ class MaskedAutoregressiveFlowTest(test_util.VectorDistributionTestHelpers,
           event_shape=[dims],
           validate_args=True)
       self.run_test_sample_consistent_log_prob(
-          sess=sess,
+          sess_run_fn=sess.run,
           dist=dist,
           num_samples=int(1e5),
           radius=1.,
@@ -130,7 +130,7 @@ class MaskedAutoregressiveFlowTest(test_util.VectorDistributionTestHelpers,
           event_shape=[dims],
           validate_args=True)
       self.run_test_sample_consistent_log_prob(
-          sess=sess,
+          sess_run_fn=sess.run,
           dist=dist,
           num_samples=int(1e5),
           radius=1.,
diff --git a/tensorflow/contrib/distributions/python/kernel_tests/independent_test.py b/tensorflow/contrib/distributions/python/kernel_tests/independent_test.py
index 7a321db4b2..dcc66e8972 100644
--- a/tensorflow/contrib/distributions/python/kernel_tests/independent_test.py
+++ b/tensorflow/contrib/distributions/python/kernel_tests/independent_test.py
@@ -23,7 +23,6 @@ import numpy as np
 
 from tensorflow.contrib.distributions.python.ops import independent as independent_lib
 from tensorflow.contrib.distributions.python.ops import mvn_diag as mvn_diag_lib
-from tensorflow.contrib.distributions.python.ops import test_util
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops.distributions import normal as normal_lib
 from tensorflow.python.platform import test
@@ -41,8 +40,7 @@ def try_import(name):  # pylint: disable=invalid-name
 stats = try_import("scipy.stats")
 
 
-class ProductDistributionTest(
-    test_util.VectorDistributionTestHelpers, test.TestCase):
+class ProductDistributionTest(test.TestCase):
 
   def testSampleAndLogProbUnivariate(self):
     loc = np.float32([-1., 1])
diff --git a/tensorflow/contrib/distributions/python/kernel_tests/mixture_same_family_test.py b/tensorflow/contrib/distributions/python/kernel_tests/mixture_same_family_test.py
index ee4f989dac..ece6bc077d 100644
--- a/tensorflow/contrib/distributions/python/kernel_tests/mixture_same_family_test.py
+++ b/tensorflow/contrib/distributions/python/kernel_tests/mixture_same_family_test.py
@@ -94,10 +94,10 @@ class MixtureSameFamilyTest(test_util.VectorDistributionTestHelpers,
               loc=[[-1., 1], [1, -1]], scale_identity_multiplier=[1., 0.5]))
       # Ball centered at component0's mean.
       self.run_test_sample_consistent_log_prob(
-          sess, gm, radius=1., center=[-1., 1], rtol=0.02)
+          sess.run, gm, radius=1., center=[-1., 1], rtol=0.02)
       # Larger ball centered at component1's mean.
       self.run_test_sample_consistent_log_prob(
-          sess, gm, radius=1., center=[1., -1], rtol=0.02)
+          sess.run, gm, radius=1., center=[1., -1], rtol=0.02)
 
   def testLogCdf(self):
     with self.test_session() as sess:
@@ -122,7 +122,7 @@ class MixtureSameFamilyTest(test_util.VectorDistributionTestHelpers,
           mixture_distribution=categorical_lib.Categorical(probs=[0.3, 0.7]),
           components_distribution=mvn_diag_lib.MultivariateNormalDiag(
               loc=[[-1., 1], [1, -1]], scale_identity_multiplier=[1., 0.5]))
-      self.run_test_sample_consistent_mean_covariance(sess, gm)
+      self.run_test_sample_consistent_mean_covariance(sess.run, gm)
 
   def testVarianceConsistentCovariance(self):
     with self.test_session() as sess:
diff --git a/tensorflow/contrib/distributions/python/kernel_tests/poisson_lognormal_test.py b/tensorflow/contrib/distributions/python/kernel_tests/poisson_lognormal_test.py
index 3ded4159d8..3c0147b8cf 100644
--- a/tensorflow/contrib/distributions/python/kernel_tests/poisson_lognormal_test.py
+++ b/tensorflow/contrib/distributions/python/kernel_tests/poisson_lognormal_test.py
@@ -22,6 +22,8 @@ import numpy as np
 
 from tensorflow.contrib.distributions.python.ops import poisson_lognormal
 from tensorflow.contrib.distributions.python.ops import test_util
+from tensorflow.python.framework import dtypes
+from tensorflow.python.ops import array_ops
 from tensorflow.python.platform import test
 
 
@@ -38,7 +40,7 @@ class PoissonLogNormalQuadratureCompoundTest(
               np.polynomial.hermite.hermgauss(deg=10)),
           validate_args=True)
       self.run_test_sample_consistent_log_prob(
-          sess, pln, rtol=0.1)
+          sess.run, pln, rtol=0.1)
 
   def testMeanVariance(self):
     with self.test_session() as sess:
@@ -49,7 +51,7 @@ class PoissonLogNormalQuadratureCompoundTest(
               np.polynomial.hermite.hermgauss(deg=10)),
           validate_args=True)
       self.run_test_sample_consistent_mean_variance(
-          sess, pln, rtol=0.02)
+          sess.run, pln, rtol=0.02)
 
   def testSampleProbConsistentBroadcastScalar(self):
     with self.test_session() as sess:
@@ -60,7 +62,7 @@ class PoissonLogNormalQuadratureCompoundTest(
               np.polynomial.hermite.hermgauss(deg=10)),
           validate_args=True)
       self.run_test_sample_consistent_log_prob(
-          sess, pln, rtol=0.1, atol=0.01)
+          sess.run, pln, rtol=0.1, atol=0.01)
 
   def testMeanVarianceBroadcastScalar(self):
     with self.test_session() as sess:
@@ -71,7 +73,7 @@ class PoissonLogNormalQuadratureCompoundTest(
               np.polynomial.hermite.hermgauss(deg=10)),
           validate_args=True)
       self.run_test_sample_consistent_mean_variance(
-          sess, pln, rtol=0.1, atol=0.01)
+          sess.run, pln, rtol=0.1, atol=0.01)
 
   def testSampleProbConsistentBroadcastBoth(self):
     with self.test_session() as sess:
@@ -82,7 +84,7 @@ class PoissonLogNormalQuadratureCompoundTest(
               np.polynomial.hermite.hermgauss(deg=10)),
           validate_args=True)
       self.run_test_sample_consistent_log_prob(
-          sess, pln, rtol=0.1, atol=0.08)
+          sess.run, pln, rtol=0.1, atol=0.08)
 
   def testMeanVarianceBroadcastBoth(self):
     with self.test_session() as sess:
@@ -93,7 +95,21 @@ class PoissonLogNormalQuadratureCompoundTest(
               np.polynomial.hermite.hermgauss(deg=10)),
           validate_args=True)
       self.run_test_sample_consistent_mean_variance(
-          sess, pln, rtol=0.1, atol=0.01)
+          sess.run, pln, rtol=0.1, atol=0.01)
+
+  def testSampleProbConsistentDynamicQuadrature(self):
+    with self.test_session() as sess:
+      qgrid = array_ops.placeholder(dtype=dtypes.float32)
+      qprobs = array_ops.placeholder(dtype=dtypes.float32)
+      g, p = np.polynomial.hermite.hermgauss(deg=10)
+      pln = poisson_lognormal.PoissonLogNormalQuadratureCompound(
+          loc=-2.,
+          scale=1.1,
+          quadrature_grid_and_probs=(g, p),
+          validate_args=True)
+      self.run_test_sample_consistent_log_prob(
+          lambda x: sess.run(x, feed_dict={qgrid: g, qprobs: p}),
+          pln, rtol=0.1)
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/contrib/distributions/python/kernel_tests/vector_diffeomixture_test.py b/tensorflow/contrib/distributions/python/kernel_tests/vector_diffeomixture_test.py
index aea4d42503..de4a221f7b 100644
--- a/tensorflow/contrib/distributions/python/kernel_tests/vector_diffeomixture_test.py
+++ b/tensorflow/contrib/distributions/python/kernel_tests/vector_diffeomixture_test.py
@@ -22,6 +22,8 @@ import numpy as np
 
 from tensorflow.contrib.distributions.python.ops import test_util
 from tensorflow.contrib.distributions.python.ops import vector_diffeomixture as vector_diffeomixture_lib
+from tensorflow.python.framework import dtypes
+from tensorflow.python.ops import array_ops
 from tensorflow.python.ops.distributions import normal as normal_lib
 from tensorflow.python.ops.linalg import linear_operator_diag as linop_diag_lib
 from tensorflow.python.ops.linalg import linear_operator_identity as linop_identity_lib
@@ -55,10 +57,10 @@ class VectorDiffeomixtureTest(
           validate_args=True)
       # Ball centered at component0's mean.
       self.run_test_sample_consistent_log_prob(
-          sess, vdm, radius=2., center=0., rtol=0.005)
+          sess.run, vdm, radius=2., center=0., rtol=0.005)
       # Larger ball centered at component1's mean.
       self.run_test_sample_consistent_log_prob(
-          sess, vdm, radius=4., center=2., rtol=0.005)
+          sess.run, vdm, radius=4., center=2., rtol=0.005)
 
   def testSampleProbConsistentBroadcastMixNonStandardBase(self):
     with self.test_session() as sess:
@@ -83,10 +85,10 @@ class VectorDiffeomixtureTest(
           validate_args=True)
       # Ball centered at component0's mean.
       self.run_test_sample_consistent_log_prob(
-          sess, vdm, radius=2., center=1., rtol=0.006)
+          sess.run, vdm, radius=2., center=1., rtol=0.006)
       # Larger ball centered at component1's mean.
       self.run_test_sample_consistent_log_prob(
-          sess, vdm, radius=4., center=3., rtol=0.009)
+          sess.run, vdm, radius=4., center=3., rtol=0.009)
 
   def testSampleProbConsistentBroadcastMixBatch(self):
     with self.test_session() as sess:
@@ -114,10 +116,10 @@ class VectorDiffeomixtureTest(
           validate_args=True)
       # Ball centered at component0's mean.
       self.run_test_sample_consistent_log_prob(
-          sess, vdm, radius=2., center=0., rtol=0.005)
+          sess.run, vdm, radius=2., center=0., rtol=0.005)
       # Larger ball centered at component1's mean.
       self.run_test_sample_consistent_log_prob(
-          sess, vdm, radius=4., center=2., rtol=0.005)
+          sess.run, vdm, radius=4., center=2., rtol=0.005)
 
   def testMeanCovarianceNoBatch(self):
     with self.test_session() as sess:
@@ -141,7 +143,7 @@ class VectorDiffeomixtureTest(
           ],
           validate_args=True)
       self.run_test_sample_consistent_mean_covariance(
-          sess, vdm, rtol=0.02, cov_rtol=0.06)
+          sess.run, vdm, rtol=0.02, cov_rtol=0.06)
 
   def testMeanCovarianceNoBatchUncenteredNonStandardBase(self):
     with self.test_session() as sess:
@@ -165,7 +167,7 @@ class VectorDiffeomixtureTest(
           ],
           validate_args=True)
       self.run_test_sample_consistent_mean_covariance(
-          sess, vdm, num_samples=int(1e6), rtol=0.01, cov_atol=0.025)
+          sess.run, vdm, num_samples=int(1e6), rtol=0.01, cov_atol=0.025)
 
   def testMeanCovarianceBatch(self):
     with self.test_session() as sess:
@@ -192,7 +194,40 @@ class VectorDiffeomixtureTest(
           ],
           validate_args=True)
       self.run_test_sample_consistent_mean_covariance(
-          sess, vdm, rtol=0.02, cov_rtol=0.06)
+          sess.run, vdm, rtol=0.02, cov_rtol=0.06)
+
+  def testSampleProbConsistentDynamicQuadrature(self):
+    with self.test_session() as sess:
+      qgrid = array_ops.placeholder(dtype=dtypes.float32)
+      qprobs = array_ops.placeholder(dtype=dtypes.float32)
+      g, p = np.polynomial.hermite.hermgauss(deg=8)
+      dims = 4
+      vdm = vector_diffeomixture_lib.VectorDiffeomixture(
+          mix_loc=[[0.], [1.]],
+          mix_scale=[1.],
+          distribution=normal_lib.Normal(0., 1.),
+          loc=[
+              None,
+              np.float32([2.]*dims),
+          ],
+          scale=[
+              linop_identity_lib.LinearOperatorScaledIdentity(
+                  num_rows=dims,
+                  multiplier=np.float32(1.1),
+                  is_positive_definite=True),
+              linop_diag_lib.LinearOperatorDiag(
+                  diag=np.linspace(2.5, 3.5, dims, dtype=np.float32),
+                  is_positive_definite=True),
+          ],
+          quadrature_grid_and_probs=(g, p),
+          validate_args=True)
+      # Ball centered at component0's mean.
+      sess_run_fn = lambda x: sess.run(x, feed_dict={qgrid: g, qprobs: p})
+      self.run_test_sample_consistent_log_prob(
+          sess_run_fn, vdm, radius=2., center=0., rtol=0.005)
+      # Larger ball centered at component1's mean.
+      self.run_test_sample_consistent_log_prob(
+          sess_run_fn, vdm, radius=4., center=2., rtol=0.005)
 
   # TODO(jvdillon): We've tested that (i) .sample and .log_prob are consistent,
   # (ii) .mean, .stddev etc... and .sample are consistent. However, we haven't
diff --git a/tensorflow/contrib/distributions/python/kernel_tests/vector_sinh_arcsinh_diag_test.py b/tensorflow/contrib/distributions/python/kernel_tests/vector_sinh_arcsinh_diag_test.py
index a5d837d454..2bc6a926dd 100644
--- a/tensorflow/contrib/distributions/python/kernel_tests/vector_sinh_arcsinh_diag_test.py
+++ b/tensorflow/contrib/distributions/python/kernel_tests/vector_sinh_arcsinh_diag_test.py
@@ -210,15 +210,15 @@ class VectorSinhArcsinhDiagTest(test_util.VectorDistributionTestHelpers,
           validate_args=True)
 
       self.run_test_sample_consistent_log_prob(
-          sess, sasnorm, radius=1.0, center=0., rtol=0.1)
+          sess.run, sasnorm, radius=1.0, center=0., rtol=0.1)
       self.run_test_sample_consistent_log_prob(
-          sess,
+          sess.run,
           sasnorm,
           radius=1.0,
           center=-0.15,
           rtol=0.1)
       self.run_test_sample_consistent_log_prob(
-          sess,
+          sess.run,
           sasnorm,
           radius=1.0,
           center=0.15,
@@ -237,15 +237,15 @@ class VectorSinhArcsinhDiagTest(test_util.VectorDistributionTestHelpers,
           validate_args=True)
 
       self.run_test_sample_consistent_log_prob(
-          sess, sasnorm, radius=1.0, center=0., rtol=0.1)
+          sess.run, sasnorm, radius=1.0, center=0., rtol=0.1)
       self.run_test_sample_consistent_log_prob(
-          sess,
+          sess.run,
           sasnorm,
           radius=1.0,
           center=-0.15,
           rtol=0.1)
       self.run_test_sample_consistent_log_prob(
-          sess,
+          sess.run,
           sasnorm,
           radius=1.0,
           center=0.15,
diff --git a/tensorflow/contrib/distributions/python/ops/poisson_lognormal.py b/tensorflow/contrib/distributions/python/ops/poisson_lognormal.py
index 80d4e2dc5e..8a95038a3c 100644
--- a/tensorflow/contrib/distributions/python/ops/poisson_lognormal.py
+++ b/tensorflow/contrib/distributions/python/ops/poisson_lognormal.py
@@ -20,6 +20,7 @@ from __future__ import print_function
 
 import numpy as np
 
+from tensorflow.contrib.distributions.python.ops import distribution_util
 from tensorflow.contrib.distributions.python.ops import poisson as poisson_lib
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
@@ -29,7 +30,6 @@ from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import random_ops
 from tensorflow.python.ops.distributions import categorical as categorical_lib
 from tensorflow.python.ops.distributions import distribution as distribution_lib
-from tensorflow.python.ops.distributions import util as distribution_util
 
 
 __all__ = [
@@ -55,8 +55,10 @@ class PoissonLogNormalQuadratureCompound(distribution_lib.Distribution):
   ```
 
   where `lambda(z) = exp(sqrt(2) scale z + loc)` and the `prob,grid` terms
-  are from [Gauss--Hermite quadrature](
-  https://en.wikipedia.org/wiki/Gauss%E2%80%93Hermite_quadrature). Note that
+  are from [numerical quadrature](
+  https://en.wikipedia.org/wiki/Numerical_integration) (default:
+  [Gauss--Hermite quadrature](
+  https://en.wikipedia.org/wiki/Gauss%E2%80%93Hermite_quadrature)). Note that
   the second line made the substitution:
   `z(l) = (log(l) - loc) / (sqrt(2) scale)` which implies `lambda(z)` [above]
   and `dl = sqrt(2) scale lambda(z) dz`
@@ -65,8 +67,11 @@ class PoissonLogNormalQuadratureCompound(distribution_lib.Distribution):
   Poisson rate parameter. Unfortunately, the non-approximate distribution lacks
   an analytical probability density function (pdf). Therefore the
   `PoissonLogNormalQuadratureCompound` class implements an approximation based
-  on [Gauss-Hermite quadrature](
-  https://en.wikipedia.org/wiki/Gauss%E2%80%93Hermite_quadrature).
+  on [numerical quadrature](
+  https://en.wikipedia.org/wiki/Numerical_integration) (default:
+  [Gauss--Hermite quadrature](
+  https://en.wikipedia.org/wiki/Gauss%E2%80%93Hermite_quadrature)).
+
   Note: although the `PoissonLogNormalQuadratureCompound` is approximately the
   Poisson-LogNormal compound distribution, it is itself a valid distribution.
   Viz., it possesses a `sample`, `log_prob`, `mean`, `variance`, etc. which are
@@ -76,9 +81,11 @@ class PoissonLogNormalQuadratureCompound(distribution_lib.Distribution):
 
   The `PoissonLogNormalQuadratureCompound` approximates a Poisson-LogNormal
   [compound distribution](
-  https://en.wikipedia.org/wiki/Compound_probability_distribution).
-  Using variable-substitution and [Gauss-Hermite quadrature](
-  https://en.wikipedia.org/wiki/Gauss%E2%80%93Hermite_quadrature) we can
+  https://en.wikipedia.org/wiki/Compound_probability_distribution). Using
+  variable-substitution and [numerical quadrature](
+  https://en.wikipedia.org/wiki/Numerical_integration) (default:
+  [Gauss--Hermite quadrature](
+  https://en.wikipedia.org/wiki/Gauss%E2%80%93Hermite_quadrature)) we can
   redefine the distribution to be a parameter-less convex combination of `deg`
   different Poisson samples.
 
@@ -125,9 +132,10 @@ class PoissonLogNormalQuadratureCompound(distribution_lib.Distribution):
         the LogNormal prior.
       scale: `float`-like (batch of) scalar `Tensor`; the scale parameter of
         the LogNormal prior.
-      quadrature_grid_and_probs: Python pair of `list`-like objects representing
-        the sample points and the corresponding (possibly normalized) weight.
-        When `None`, defaults to: `np.polynomial.hermite.hermgauss(deg=8)`.
+      quadrature_grid_and_probs: Python pair of `float`-like `Tensor`s
+        representing the sample points and the corresponding (possibly
+        normalized) weight.  When `None`, defaults to:
+        `np.polynomial.hermite.hermgauss(deg=8)`.
       validate_args: Python `bool`, default `False`. When `True` distribution
         parameters are checked for validity despite possibly degrading runtime
         performance. When `False` invalid inputs may silently render incorrect
@@ -140,8 +148,6 @@ class PoissonLogNormalQuadratureCompound(distribution_lib.Distribution):
 
     Raises:
       TypeError: if `loc.dtype != scale[0].dtype`.
-      ValueError: if `quadrature_grid_and_probs is not None` and
-        `len(quadrature_grid_and_probs[0]) != len(quadrature_grid_and_probs[1])`
     """
     parameters = locals()
     with ops.name_scope(name, values=[loc, scale]):
@@ -157,21 +163,14 @@ class PoissonLogNormalQuadratureCompound(distribution_lib.Distribution):
             "loc.dtype(\"{}\") does not match scale.dtype(\"{}\")".format(
                 loc.dtype.name, scale.dtype.name))
 
-      if quadrature_grid_and_probs is None:
-        grid, probs = np.polynomial.hermite.hermgauss(deg=8)
-      else:
-        grid, probs = tuple(quadrature_grid_and_probs)
-        if len(grid) != len(probs):
-          raise ValueError("`quadrature_grid_and_probs` must be a `tuple` of "
-                           "same-length list-like objects")
-      grid = grid.astype(dtype.as_numpy_dtype)
-      probs = probs.astype(dtype.as_numpy_dtype)
-      probs /= np.linalg.norm(probs, ord=1)
+      grid, probs = distribution_util.process_quadrature_grid_and_probs(
+          quadrature_grid_and_probs, dtype, validate_args)
       self._quadrature_grid = grid
       self._quadrature_probs = probs
+      self._quadrature_size = distribution_util.dimension_size(probs, axis=0)
 
       self._mixture_distribution = categorical_lib.Categorical(
-          logits=np.log(probs),
+          logits=math_ops.log(self._quadrature_probs),
           validate_args=validate_args,
           allow_nan_stats=allow_nan_stats)
 
@@ -254,10 +253,10 @@ class PoissonLogNormalQuadratureCompound(distribution_lib.Distribution):
                 [batch_size])),
         seed=distribution_util.gen_new_seed(
             seed, "poisson_lognormal_quadrature_compound"))
-    # Stride `quadrature_degree` for `batch_size` number of times.
+    # Stride `quadrature_size` for `batch_size` number of times.
     offset = math_ops.range(start=0,
-                            limit=batch_size * len(self.quadrature_probs),
-                            delta=len(self.quadrature_probs),
+                            limit=batch_size * self._quadrature_size,
+                            delta=self._quadrature_size,
                             dtype=ids.dtype)
     ids += offset
     rate = array_ops.gather(
diff --git a/tensorflow/contrib/distributions/python/ops/test_util.py b/tensorflow/contrib/distributions/python/ops/test_util.py
index 631ffc1bac..77f2a39273 100644
--- a/tensorflow/contrib/distributions/python/ops/test_util.py
+++ b/tensorflow/contrib/distributions/python/ops/test_util.py
@@ -38,7 +38,7 @@ class DiscreteScalarDistributionTestHelpers(object):
   """DiscreteScalarDistributionTestHelpers."""
 
   def run_test_sample_consistent_log_prob(
-      self, sess, dist,
+      self, sess_run_fn, dist,
       num_samples=int(1e5), num_threshold=int(1e3), seed=42,
       rtol=1e-2, atol=0.):
     """Tests that sample/log_prob are consistent with each other.
@@ -51,7 +51,9 @@ class DiscreteScalarDistributionTestHelpers(object):
     are consistent.
 
     Args:
-      sess: Tensorflow session.
+      sess_run_fn: Python `callable` taking `list`-like of `Tensor`s and
+        returning a list of results after running one "step" of TensorFlow
+        computation, typically set to `sess.run`.
       dist: Distribution instance or object which implements `sample`,
         `log_prob`, `event_shape_tensor` and `batch_shape_tensor`.
       num_samples: Python `int` scalar indicating the number of Monte-Carlo
@@ -87,7 +89,7 @@ class DiscreteScalarDistributionTestHelpers(object):
       probs = math_ops.exp(dist.log_prob(edges))
       probs = array_ops.reshape(probs, shape=[-1, batch_size])[:, b]
 
-      [counts_, probs_] = sess.run([counts, probs])
+      [counts_, probs_] = sess_run_fn([counts, probs])
       valid = counts_ > num_threshold
       probs_ = probs_[valid]
       counts_ = counts_[valid]
@@ -95,7 +97,7 @@ class DiscreteScalarDistributionTestHelpers(object):
                           rtol=rtol, atol=atol)
 
   def run_test_sample_consistent_mean_variance(
-      self, sess, dist,
+      self, sess_run_fn, dist,
       num_samples=int(1e5), seed=24,
       rtol=1e-2, atol=0.):
     """Tests that sample/mean/variance are consistent with each other.
@@ -104,7 +106,9 @@ class DiscreteScalarDistributionTestHelpers(object):
     to the same distribution.
 
     Args:
-      sess: Tensorflow session.
+      sess_run_fn: Python `callable` taking `list`-like of `Tensor`s and
+        returning a list of results after running one "step" of TensorFlow
+        computation, typically set to `sess.run`.
       dist: Distribution instance or object which implements `sample`,
         `log_prob`, `event_shape_tensor` and `batch_shape_tensor`.
       num_samples: Python `int` scalar indicating the number of Monte-Carlo
@@ -130,7 +134,7 @@ class DiscreteScalarDistributionTestHelpers(object):
         mean_,
         variance_,
         stddev_
-    ] = sess.run([
+    ] = sess_run_fn([
         sample_mean,
         sample_variance,
         sample_stddev,
@@ -187,7 +191,7 @@ class VectorDistributionTestHelpers(object):
 
   def run_test_sample_consistent_log_prob(
       self,
-      sess,
+      sess_run_fn,
       dist,
       num_samples=int(1e5),
       radius=1.,
@@ -240,7 +244,9 @@ class VectorDistributionTestHelpers(object):
       https://en.wikipedia.org/wiki/Importance_sampling.
 
     Args:
-      sess: Tensorflow session.
+      sess_run_fn: Python `callable` taking `list`-like of `Tensor`s and
+        returning a list of results after running one "step" of TensorFlow
+        computation, typically set to `sess.run`.
       dist: Distribution instance or object which implements `sample`,
         `log_prob`, `event_shape_tensor` and `batch_shape_tensor`. The
         distribution must have non-zero probability of sampling every point
@@ -301,8 +307,8 @@ class VectorDistributionTestHelpers(object):
       init_op = variables_ops.global_variables_initializer()
 
     # Execute graph.
-    sess.run(init_op)
-    [batch_shape_, actual_volume_, sample_volume_] = sess.run([
+    sess_run_fn(init_op)
+    [batch_shape_, actual_volume_, sample_volume_] = sess_run_fn([
         batch_shape, actual_volume, sample_volume])
 
     # Check results.
@@ -312,7 +318,7 @@ class VectorDistributionTestHelpers(object):
 
   def run_test_sample_consistent_mean_covariance(
       self,
-      sess,
+      sess_run_fn,
       dist,
       num_samples=int(1e5),
       seed=24,
@@ -326,7 +332,9 @@ class VectorDistributionTestHelpers(object):
     to the same distribution.
 
     Args:
-      sess: Tensorflow session.
+      sess_run_fn: Python `callable` taking `list`-like of `Tensor`s and
+        returning a list of results after running one "step" of TensorFlow
+        computation, typically set to `sess.run`.
       dist: Distribution instance or object which implements `sample`,
         `log_prob`, `event_shape_tensor` and `batch_shape_tensor`.
       num_samples: Python `int` scalar indicating the number of Monte-Carlo
@@ -360,7 +368,7 @@ class VectorDistributionTestHelpers(object):
         covariance_,
         variance_,
         stddev_
-    ] = sess.run([
+    ] = sess_run_fn([
         sample_mean,
         sample_covariance,
         sample_variance,
diff --git a/tensorflow/contrib/distributions/python/ops/vector_diffeomixture.py b/tensorflow/contrib/distributions/python/ops/vector_diffeomixture.py
index 33dad811a9..92043d6a08 100644
--- a/tensorflow/contrib/distributions/python/ops/vector_diffeomixture.py
+++ b/tensorflow/contrib/distributions/python/ops/vector_diffeomixture.py
@@ -73,8 +73,10 @@ class VectorDiffeomixture(distribution_lib.Distribution):
   denotes matrix multiplication.  However, the non-approximate distribution does
   not have an analytical probability density function (pdf). Therefore the
   `VectorDiffeomixture` class implements an approximation based on
-  [Gauss-Hermite quadrature](
-  https://en.wikipedia.org/wiki/Gauss%E2%80%93Hermite_quadrature). I.e., in
+  [numerical quadrature](
+  https://en.wikipedia.org/wiki/Numerical_integration) (default:
+  [Gauss--Hermite quadrature](
+  https://en.wikipedia.org/wiki/Gauss%E2%80%93Hermite_quadrature)). I.e., in
   Note: although the `VectorDiffeomixture` is approximately the
   `SoftmaxNormal-Distribution` compound distribution, it is itself a valid
   distribution. It possesses a `sample`, `log_prob`, `mean`, `covariance` which
@@ -109,8 +111,10 @@ class VectorDiffeomixture(distribution_lib.Distribution):
   The `VectorDiffeomixture` approximates a SoftmaxNormal-mixed ("prior")
   [compound distribution](
   https://en.wikipedia.org/wiki/Compound_probability_distribution).
-  Using variable-substitution and [Gauss-Hermite quadrature](
-  https://en.wikipedia.org/wiki/Gauss%E2%80%93Hermite_quadrature) we can
+  Using variable-substitution and [numerical quadrature](
+  https://en.wikipedia.org/wiki/Numerical_integration) (default:
+  [Gauss--Hermite quadrature](
+  https://en.wikipedia.org/wiki/Gauss%E2%80%93Hermite_quadrature)) we can
   redefine the distribution to be a parameter-less convex combination of `K`
   different affine combinations of a `d` iid samples from `distribution`.
 
@@ -141,7 +145,7 @@ class VectorDiffeomixture(distribution_lib.Distribution):
   and,
 
   ```none
-  grid, weight = np.polynomial.hermite.hermgauss(quadrature_degree)
+  grid, weight = np.polynomial.hermite.hermgauss(quadrature_size)
   prob[k]   = weight[k] / sqrt(pi)
   lambda[k; i] = sigmoid(mix_loc[k] + sqrt(2) mix_scale[k] grid[i])
   ```
@@ -248,9 +252,10 @@ class VectorDiffeomixture(distribution_lib.Distribution):
         `k`-th element represents the `scale` used for the `k`-th affine
         transformation. `LinearOperator`s must have shape `[B1, ..., Bb, d, d]`,
         `b >= 0`, i.e., characterizes `b`-batches of `d x d` matrices
-      quadrature_grid_and_probs: Python pair of `list`-like objects representing
-        the sample points and the corresponding (possibly normalized) weight.
-        When `None`, defaults to: `np.polynomial.hermite.hermgauss(deg=8)`.
+      quadrature_grid_and_probs: Python pair of `float`-like `Tensor`s
+        representing the sample points and the corresponding (possibly
+        normalized) weight.  When `None`, defaults to:
+        `np.polynomial.hermite.hermgauss(deg=8)`.
       validate_args: Python `bool`, default `False`. When `True` distribution
         parameters are checked for validity despite possibly degrading runtime
         performance. When `False` invalid inputs may silently render incorrect
@@ -317,24 +322,17 @@ class VectorDiffeomixture(distribution_lib.Distribution):
         raise NotImplementedError("Currently only bimixtures are supported; "
                                   "len(scale)={} is not 2.".format(len(scale)))
 
-      if quadrature_grid_and_probs is None:
-        grid, probs = np.polynomial.hermite.hermgauss(deg=8)
-      else:
-        grid, probs = tuple(quadrature_grid_and_probs)
-        if len(grid) != len(probs):
-          raise ValueError("`quadrature_grid_and_probs` must be a `tuple` of "
-                           "same-length list-like objects")
-      grid = grid.astype(dtype.as_numpy_dtype)
-      probs = probs.astype(dtype.as_numpy_dtype)
-      probs /= np.linalg.norm(probs, ord=1)
+      grid, probs = distribution_util.process_quadrature_grid_and_probs(
+          quadrature_grid_and_probs, dtype, validate_args)
       self._quadrature_grid = grid
       self._quadrature_probs = probs
+      self._quadrature_size = distribution_util.dimension_size(probs, axis=0)
 
       # Note: by creating the logits as `log(prob)` we ensure that
       # `self.mixture_distribution.logits` is equivalent to
       # `math_ops.log(self.mixture_distribution.probs)`.
       self._mixture_distribution = categorical_lib.Categorical(
-          logits=np.log(probs),
+          logits=math_ops.log(probs),
           validate_args=validate_args,
           allow_nan_stats=allow_nan_stats)
 
@@ -361,10 +359,10 @@ class VectorDiffeomixture(distribution_lib.Distribution):
                                validate_args=validate_args,
                                name="interpolated_affine_{}".format(k))
           for k, (loc_, scale_) in enumerate(zip(
-              interpolate_loc(len(self._quadrature_grid),
+              interpolate_loc(self._quadrature_size,
                               self._interpolate_weight,
                               loc),
-              interpolate_scale(len(self._quadrature_grid),
+              interpolate_scale(self._quadrature_size,
                                 self._interpolate_weight,
                                 scale)))]
 
@@ -463,10 +461,10 @@ class VectorDiffeomixture(distribution_lib.Distribution):
         seed=distribution_util.gen_new_seed(
             seed, "vector_diffeomixture"))
 
-    # Stride `quadrature_degree` for `batch_size` number of times.
+    # Stride `quadrature_size` for `batch_size` number of times.
     offset = math_ops.range(start=0,
-                            limit=batch_size * len(self.quadrature_probs),
-                            delta=len(self.quadrature_probs),
+                            limit=batch_size * self._quadrature_size,
+                            delta=self._quadrature_size,
                             dtype=ids.dtype)
 
     weight = array_ops.gather(
diff --git a/tensorflow/python/ops/distributions/util.py b/tensorflow/python/ops/distributions/util.py
index f261d996b5..41b86f7940 100644
--- a/tensorflow/python/ops/distributions/util.py
+++ b/tensorflow/python/ops/distributions/util.py
@@ -29,6 +29,7 @@ from tensorflow.python.framework import tensor_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import check_ops
 from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import linalg_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import nn
 
@@ -1049,13 +1050,77 @@ def dimension_size(x, axis):
   """Returns the size of a specific dimension."""
   # Since tf.gather isn't "constant-in, constant-out", we must first check the
   # static shape or fallback to dynamic shape.
-  num_rows = (None if x.get_shape().ndims is None
-              else x.get_shape()[axis].value)
-  if num_rows is not None:
-    return num_rows
+  s = x.shape.with_rank_at_least(axis + 1)[axis].value
+  if axis > -1 and s is not None:
+    return s
   return array_ops.shape(x)[axis]
 
 
+def process_quadrature_grid_and_probs(
+    quadrature_grid_and_probs, dtype, validate_args, name=None):
+  """Validates quadrature grid, probs or computes them as necessary.
+
+  Args:
+    quadrature_grid_and_probs: Python pair of `float`-like `Tensor`s
+      representing the sample points and the corresponding (possibly
+      normalized) weight.  When `None`, defaults to:
+      `np.polynomial.hermite.hermgauss(deg=8)`.
+    dtype: The expected `dtype` of `grid` and `probs`.
+    validate_args: Python `bool`, default `False`. When `True` distribution
+      parameters are checked for validity despite possibly degrading runtime
+      performance. When `False` invalid inputs may silently render incorrect
+      outputs.
+    name: Python `str` name prefixed to Ops created by this class.
+
+  Returns:
+     quadrature_grid_and_probs: Python pair of `float`-like `Tensor`s
+      representing the sample points and the corresponding (possibly
+      normalized) weight.
+
+  Raises:
+    ValueError: if `quadrature_grid_and_probs is not None` and
+      `len(quadrature_grid_and_probs[0]) != len(quadrature_grid_and_probs[1])`
+  """
+  with ops.name_scope(name, "process_quadrature_grid_and_probs",
+                      [quadrature_grid_and_probs]):
+    if quadrature_grid_and_probs is None:
+      grid, probs = np.polynomial.hermite.hermgauss(deg=8)
+      grid = grid.astype(dtype.as_numpy_dtype)
+      probs = probs.astype(dtype.as_numpy_dtype)
+      probs /= np.linalg.norm(probs, ord=1, keepdims=True)
+      grid = ops.convert_to_tensor(grid, name="grid", dtype=dtype)
+      probs = ops.convert_to_tensor(probs, name="probs", dtype=dtype)
+      return grid, probs
+
+    grid, probs = tuple(quadrature_grid_and_probs)
+    grid = ops.convert_to_tensor(grid, name="grid", dtype=dtype)
+    probs = ops.convert_to_tensor(probs, name="unnormalized_probs",
+                                  dtype=dtype)
+    probs /= linalg_ops.norm(probs, ord=1, axis=-1, keep_dims=True,
+                             name="probs")
+
+    def _static_dim_size(x, axis):
+      """Returns the static size of a specific dimension or `None`."""
+      return x.shape.with_rank_at_least(axis + 1)[axis].value
+
+    m, n = _static_dim_size(probs, axis=0), _static_dim_size(grid, axis=0)
+    if m is not None and n is not None:
+      if m != n:
+        raise ValueError("`quadrature_grid_and_probs` must be a `tuple` of "
+                         "same-length zero-th-dimension `Tensor`s "
+                         "(saw lengths {}, {})".format(m, n))
+    elif validate_args:
+      grid = control_flow_ops.with_dependencies([
+          check_ops.assert_equal(
+              dimension_size(probs, axis=0),
+              dimension_size(grid, axis=0),
+              message=("`quadrature_grid_and_probs` must be a `tuple` of "
+                       "same-length zero-th-dimension `Tensor`s")),
+      ], grid)
+
+    return grid, probs
+
+
 class AppendDocstring(object):
   """Helper class to promote private subclass docstring to public counterpart.
 
-- 
GitLab


From c77090a0ae61fc69fcdff7c58be9feb6121e3bd4 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Fri, 20 Oct 2017 16:36:31 -0700
Subject: [PATCH 641/909] Fix issues where int64 crops could not be passed to
 batch_to_space. (#13862)

* Fix issues where int64 crops could not be passed to batch_to_space.

This fix tries to address the issue where int64 `crops` could
not be passed to `batch_to_space` even though both int32 and
int64 are specified as supported in the docs (tf.batch_to_space.__doc__)

The reason is that BatchToSpace kernel puts a constraint of int32 to crops
data types.

This fix removed the constraint so that int64 `crops` could be supported.

NOTE: Just removing the constraint should work and it is not necessary
to add specification to the kernel class template, as `SubtleMustCopyFlat`
called in the class already correctly handled both int32 and int64 cases.
Besides, other data types (e.g., float or double) will not be passed to the
kernel as they are guarded by the specification in `array_ops.cc`.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Also remove int64/int32 type constraints for SpaceToBatch kernels

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Add test cases for int64 crops of batch_to_space and space_to_batch

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Fix test failures.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/core/kernels/batchtospace_op.cc    | 50 ++++++++-----------
 tensorflow/core/kernels/spacetobatch_op.cc    | 50 ++++++++-----------
 .../kernel_tests/batchtospace_op_test.py      | 36 +++++++------
 3 files changed, 65 insertions(+), 71 deletions(-)

diff --git a/tensorflow/core/kernels/batchtospace_op.cc b/tensorflow/core/kernels/batchtospace_op.cc
index 99b5d3daaa..c1c0d6d329 100644
--- a/tensorflow/core/kernels/batchtospace_op.cc
+++ b/tensorflow/core/kernels/batchtospace_op.cc
@@ -249,40 +249,34 @@ class BatchToSpaceOp : public OpKernel {
   Tensor block_shape_;
 };
 
-#define REGISTER(T)                                                  \
-  REGISTER_KERNEL_BUILDER(Name("BatchToSpaceND")                     \
-                              .Device(DEVICE_CPU)                    \
-                              .TypeConstraint<T>("T")                \
-                              .TypeConstraint<int32>("Tblock_shape") \
-                              .TypeConstraint<int32>("Tcrops")       \
-                              .HostMemory("block_shape")             \
-                              .HostMemory("crops"),                  \
-                          BatchToSpaceNDOp<CPUDevice, T>);           \
-  REGISTER_KERNEL_BUILDER(Name("BatchToSpace")                       \
-                              .Device(DEVICE_CPU)                    \
-                              .TypeConstraint<T>("T")                \
-                              .TypeConstraint<int32>("Tidx")         \
-                              .HostMemory("crops"),                  \
+#define REGISTER(T)                                        \
+  REGISTER_KERNEL_BUILDER(Name("BatchToSpaceND")           \
+                              .Device(DEVICE_CPU)          \
+                              .TypeConstraint<T>("T")      \
+                              .HostMemory("block_shape")   \
+                              .HostMemory("crops"),        \
+                          BatchToSpaceNDOp<CPUDevice, T>); \
+  REGISTER_KERNEL_BUILDER(Name("BatchToSpace")             \
+                              .Device(DEVICE_CPU)          \
+                              .TypeConstraint<T>("T")      \
+                              .HostMemory("crops"),        \
                           BatchToSpaceOp<CPUDevice, T>);
 
 TF_CALL_REAL_NUMBER_TYPES(REGISTER);
 #undef REGISTER
 
 #if GOOGLE_CUDA
-#define REGISTER(T)                                                  \
-  REGISTER_KERNEL_BUILDER(Name("BatchToSpaceND")                     \
-                              .Device(DEVICE_GPU)                    \
-                              .TypeConstraint<T>("T")                \
-                              .TypeConstraint<int32>("Tblock_shape") \
-                              .TypeConstraint<int32>("Tcrops")       \
-                              .HostMemory("block_shape")             \
-                              .HostMemory("crops"),                  \
-                          BatchToSpaceNDOp<GPUDevice, T>);           \
-  REGISTER_KERNEL_BUILDER(Name("BatchToSpace")                       \
-                              .Device(DEVICE_GPU)                    \
-                              .TypeConstraint<T>("T")                \
-                              .TypeConstraint<int32>("Tidx")         \
-                              .HostMemory("crops"),                  \
+#define REGISTER(T)                                        \
+  REGISTER_KERNEL_BUILDER(Name("BatchToSpaceND")           \
+                              .Device(DEVICE_GPU)          \
+                              .TypeConstraint<T>("T")      \
+                              .HostMemory("block_shape")   \
+                              .HostMemory("crops"),        \
+                          BatchToSpaceNDOp<GPUDevice, T>); \
+  REGISTER_KERNEL_BUILDER(Name("BatchToSpace")             \
+                              .Device(DEVICE_GPU)          \
+                              .TypeConstraint<T>("T")      \
+                              .HostMemory("crops"),        \
                           BatchToSpaceOp<GPUDevice, T>);
 
 TF_CALL_GPU_NUMBER_TYPES(REGISTER);
diff --git a/tensorflow/core/kernels/spacetobatch_op.cc b/tensorflow/core/kernels/spacetobatch_op.cc
index c513683918..95c1f5e7e8 100644
--- a/tensorflow/core/kernels/spacetobatch_op.cc
+++ b/tensorflow/core/kernels/spacetobatch_op.cc
@@ -248,40 +248,34 @@ class SpaceToBatchOp : public OpKernel {
   Tensor block_shape_;
 };
 
-#define REGISTER(T)                                                  \
-  REGISTER_KERNEL_BUILDER(Name("SpaceToBatchND")                     \
-                              .Device(DEVICE_CPU)                    \
-                              .TypeConstraint<T>("T")                \
-                              .TypeConstraint<int32>("Tblock_shape") \
-                              .TypeConstraint<int32>("Tpaddings")    \
-                              .HostMemory("block_shape")             \
-                              .HostMemory("paddings"),               \
-                          SpaceToBatchNDOp<CPUDevice, T>);           \
-  REGISTER_KERNEL_BUILDER(Name("SpaceToBatch")                       \
-                              .Device(DEVICE_CPU)                    \
-                              .TypeConstraint<T>("T")                \
-                              .TypeConstraint<int32>("Tpaddings")    \
-                              .HostMemory("paddings"),               \
+#define REGISTER(T)                                        \
+  REGISTER_KERNEL_BUILDER(Name("SpaceToBatchND")           \
+                              .Device(DEVICE_CPU)          \
+                              .TypeConstraint<T>("T")      \
+                              .HostMemory("block_shape")   \
+                              .HostMemory("paddings"),     \
+                          SpaceToBatchNDOp<CPUDevice, T>); \
+  REGISTER_KERNEL_BUILDER(Name("SpaceToBatch")             \
+                              .Device(DEVICE_CPU)          \
+                              .TypeConstraint<T>("T")      \
+                              .HostMemory("paddings"),     \
                           SpaceToBatchOp<CPUDevice, T>);
 
 TF_CALL_REAL_NUMBER_TYPES(REGISTER);
 #undef REGISTER
 
 #if GOOGLE_CUDA
-#define REGISTER(T)                                                  \
-  REGISTER_KERNEL_BUILDER(Name("SpaceToBatchND")                     \
-                              .Device(DEVICE_GPU)                    \
-                              .TypeConstraint<T>("T")                \
-                              .TypeConstraint<int32>("Tblock_shape") \
-                              .TypeConstraint<int32>("Tpaddings")    \
-                              .HostMemory("block_shape")             \
-                              .HostMemory("paddings"),               \
-                          SpaceToBatchNDOp<GPUDevice, T>);           \
-  REGISTER_KERNEL_BUILDER(Name("SpaceToBatch")                       \
-                              .Device(DEVICE_GPU)                    \
-                              .TypeConstraint<T>("T")                \
-                              .TypeConstraint<int32>("Tpaddings")    \
-                              .HostMemory("paddings"),               \
+#define REGISTER(T)                                        \
+  REGISTER_KERNEL_BUILDER(Name("SpaceToBatchND")           \
+                              .Device(DEVICE_GPU)          \
+                              .TypeConstraint<T>("T")      \
+                              .HostMemory("block_shape")   \
+                              .HostMemory("paddings"),     \
+                          SpaceToBatchNDOp<GPUDevice, T>); \
+  REGISTER_KERNEL_BUILDER(Name("SpaceToBatch")             \
+                              .Device(DEVICE_GPU)          \
+                              .TypeConstraint<T>("T")      \
+                              .HostMemory("paddings"),     \
                           SpaceToBatchOp<GPUDevice, T>);
 
 TF_CALL_GPU_NUMBER_TYPES(REGISTER);
diff --git a/tensorflow/python/kernel_tests/batchtospace_op_test.py b/tensorflow/python/kernel_tests/batchtospace_op_test.py
index 8ec93119f2..0c802476a0 100644
--- a/tensorflow/python/kernel_tests/batchtospace_op_test.py
+++ b/tensorflow/python/kernel_tests/batchtospace_op_test.py
@@ -24,6 +24,7 @@ from __future__ import print_function
 
 import numpy as np
 
+from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
@@ -52,14 +53,15 @@ class BatchToSpaceDepthToSpace(test.TestCase, PythonOpImpl):
   def testDepthToSpaceTranspose(self):
     x = np.arange(20 * 5 * 8 * 7, dtype=np.float32).reshape([20, 5, 8, 7])
     block_size = 2
-    crops = np.zeros((2, 2), dtype=np.int32)
-    y1 = self.batch_to_space(x, crops, block_size=block_size)
-    y2 = array_ops.transpose(
-        array_ops.depth_to_space(
-            array_ops.transpose(x, [3, 1, 2, 0]), block_size=block_size),
-        [3, 1, 2, 0])
-    with self.test_session():
-      self.assertAllEqual(y1.eval(), y2.eval())
+    for crops_dtype in [dtypes.int64, dtypes.int32]:
+      crops = array_ops.zeros((2, 2), dtype=crops_dtype)
+      y1 = self.batch_to_space(x, crops, block_size=block_size)
+      y2 = array_ops.transpose(
+          array_ops.depth_to_space(
+              array_ops.transpose(x, [3, 1, 2, 0]), block_size=block_size),
+          [3, 1, 2, 0])
+      with self.test_session():
+        self.assertAllEqual(y1.eval(), y2.eval())
 
 
 class BatchToSpaceDepthToSpaceCpp(BatchToSpaceDepthToSpace, CppOpImpl):
@@ -287,9 +289,10 @@ class BatchToSpaceGradientCppTest(BatchToSpaceGradientTest, CppOpImpl):
 class BatchToSpaceNDGradientTest(test.TestCase):
 
   # Check the gradients.
-  def _checkGrad(self, x, block_shape, crops):
+  def _checkGrad(self, x, block_shape, crops, crops_dtype):
     block_shape = np.array(block_shape)
-    crops = np.array(crops).reshape((len(block_shape), 2))
+    crops = constant_op.constant(
+        np.array(crops).reshape((len(block_shape), 2)), crops_dtype)
     with self.test_session():
       tf_x = ops.convert_to_tensor(x)
       tf_y = array_ops.batch_to_space_nd(tf_x, block_shape, crops)
@@ -304,23 +307,26 @@ class BatchToSpaceNDGradientTest(test.TestCase):
 
     self.assertAllClose(x_jacob_t, x_jacob_n, rtol=1e-2, atol=epsilon)
 
-  def _compare(self, input_shape, block_shape, crops):
+  def _compare(self, input_shape, block_shape, crops, crops_dtype):
     input_shape = list(input_shape)
     input_shape[0] *= np.prod(block_shape)
     x = np.random.normal(
         0, 1, np.prod(input_shape)).astype(np.float32).reshape(input_shape)
-    self._checkGrad(x, block_shape, crops)
+    self._checkGrad(x, block_shape, crops, crops_dtype)
 
   # Don't use very large numbers as dimensions here as the result is tensor
   # with cartesian product of the dimensions.
   def testSmall(self):
-    self._compare([1, 2, 3, 5], [2, 2], [[0, 0], [0, 0]])
+    for dtype in [dtypes.int64, dtypes.int32]:
+      self._compare([1, 2, 3, 5], [2, 2], [[0, 0], [0, 0]], dtype)
 
   def testSmall2(self):
-    self._compare([2, 4, 3, 2], [2, 2], [[0, 0], [0, 0]])
+    for dtype in [dtypes.int64, dtypes.int32]:
+      self._compare([2, 4, 3, 2], [2, 2], [[0, 0], [0, 0]], dtype)
 
   def testSmallCrop1x1(self):
-    self._compare([1, 2, 3, 5], [2, 2], [[1, 1], [1, 1]])
+    for dtype in [dtypes.int64, dtypes.int32]:
+      self._compare([1, 2, 3, 5], [2, 2], [[1, 1], [1, 1]], dtype)
 
 
 if __name__ == "__main__":
-- 
GitLab


From a5fe66b1519668505c0daf5f2d93a4d532cedda1 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 20 Oct 2017 16:32:24 -0700
Subject: [PATCH 642/909] Removed some unnecessary broadcasts in binary ops
 where only one input needs broadcasting (which is a fairly common case, even
 in the fallback path).

PiperOrigin-RevId: 172950493
---
 tensorflow/core/kernels/cwise_ops_common.h | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/tensorflow/core/kernels/cwise_ops_common.h b/tensorflow/core/kernels/cwise_ops_common.h
index 9a05e1500f..2454620776 100644
--- a/tensorflow/core/kernels/cwise_ops_common.h
+++ b/tensorflow/core/kernels/cwise_ops_common.h
@@ -410,10 +410,20 @@ struct BinaryFunctor<CPUDevice, Functor, NDIMS, false> {
       }
     }
 
-    // Fallback path. Always work and probably slower.
-    auto lhs = in0.broadcast(bcast0);
-    auto rhs = in1.broadcast(bcast1);
-    Assign(dev, out, lhs.binaryExpr(rhs, func));
+    // Fallback path. Always works and probably slower.
+    if (AllOne<NDIMS>(bcast0) && AllOne<NDIMS>(bcast1)) {
+      Assign(dev, out, in0.binaryExpr(in1, func));
+    } else if (AllOne<NDIMS>(bcast0)) {
+      auto rhs = in1.broadcast(bcast1);
+      Assign(dev, out, in0.binaryExpr(rhs, func));
+    } else if (AllOne<NDIMS>(bcast1)) {
+      auto lhs = in0.broadcast(bcast0);
+      Assign(dev, out, lhs.binaryExpr(in1, func));
+    } else {
+      auto lhs = in0.broadcast(bcast0);
+      auto rhs = in1.broadcast(bcast1);
+      Assign(dev, out, lhs.binaryExpr(rhs, func));
+    }
   }
 };
 
-- 
GitLab


From f758b24a825e9787bd0bc89c5e2869116e5384fe Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Fri, 20 Oct 2017 16:57:13 -0700
Subject: [PATCH 643/909] Variable name for the eager test (#13873)

---
 .../contrib/framework/python/ops/accumulate_n_v2_eager_test.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/framework/python/ops/accumulate_n_v2_eager_test.py b/tensorflow/contrib/framework/python/ops/accumulate_n_v2_eager_test.py
index f3453f89fa..5f086ea8cc 100644
--- a/tensorflow/contrib/framework/python/ops/accumulate_n_v2_eager_test.py
+++ b/tensorflow/contrib/framework/python/ops/accumulate_n_v2_eager_test.py
@@ -64,7 +64,8 @@ class AccumulateNV2EagerTest(test_util.TensorFlowTestCase):
     np.random.seed(42)
     num_inputs = 3
     input_vars = [
-        resource_variable_ops.ResourceVariable(10.0 * np.random.random())
+        resource_variable_ops.ResourceVariable(10.0 * np.random.random(), 
+                                               name="t1")
         for i in range(0, num_inputs)
     ]
 
-- 
GitLab


From 29c7b46585aabab6b1a1677324667c2d5720181c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 20 Oct 2017 17:26:46 -0700
Subject: [PATCH 644/909] Adding the Stanford Tensorflow class to community
 resources.

PiperOrigin-RevId: 172956049
---
 tensorflow/docs_src/community/welcome.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/docs_src/community/welcome.md b/tensorflow/docs_src/community/welcome.md
index 4991783a53..c4f78051f0 100644
--- a/tensorflow/docs_src/community/welcome.md
+++ b/tensorflow/docs_src/community/welcome.md
@@ -29,6 +29,7 @@ The TensorFlow community has created many great projects around TensorFlow, incl
 * [Sublime Tensorflow - A plugin for Sublime Text](https://github.com/baptisteArnaud/Sublime-Tensorflow)
 * [Edward - A library for probabilistic modeling, inference, and criticism](http://edwardlib.org) ([Github](https://github.com/blei-lab/edward), [Forum](https://discourse.edwardlib.org))
 * [GPflow - Gaussian processes in TensorFlow](https://github.com/GPflow/GPflow)
+* [CS 20SI: Tensorflow for Deep Learning Research](https://web.stanford.edu/class/cs20si/) - Please note, this course was designed with TensorFlow v0.12, so some of the notes may be out of date - but it's still a great resource.
 
 ## TensorFlow Communities Around the World
 
-- 
GitLab


From df8bce63d6de6e728e69eb9f45862b816f88a0db Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Fri, 20 Oct 2017 17:40:40 -0700
Subject: [PATCH 645/909] Fix crash when `int64` axis is passed to
 `tf.reduce_sum` (#13863)

* Fix crash when `int64` axis is passed to `tf.reduce_sum`

This fix tries to fix the crash triggered by `int64` axis passed
to `tf.reduce_sum`:
```
ubuntu@ubuntu:~/tensorflow2$ (cd && python)
Python 2.7.12 (default, Nov 19 2016, 06:48:10)
[GCC 5.4.0 20160609] on linux2
Type "help", "copyright", "credits" or "license" for more information.
>>> import tensorflow as tf
>>> v = tf.reduce_sum([1,2,3], tf.constant(0, tf.int64))
2017-10-20 15:55:06.993430: F tensorflow/core/framework/tensor.cc:601] Check failed: dtype() == expected_dtype (9 vs. 3)
ubuntu@ubuntu:~/tensorflow2$
```

The issue is caused by the fact that shape inference in `common_shape_fns.cc`
only assumes int32 without proper handling of diffent types. In `math_ops.cc`
both int32 and int64 are mentioned.

NOTE that this fix does not address the issue that int64 is not supported.
To allow int64 axis it is more than adding a template in `ReductionOp` as the type
of the axis seems to be decided by some other ways in Eigen.

This fix merely fixed the crash so that an error message will return without
exit from the python program "No OpKernel was registered to support Op 'Sum' with these attrs".

Still, I think its worth to at least allow the program to continue in case of unsupported kernel.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Update implementation with a template helper function.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/core/framework/common_shape_fns.cc | 58 ++++++++++++-------
 1 file changed, 37 insertions(+), 21 deletions(-)

diff --git a/tensorflow/core/framework/common_shape_fns.cc b/tensorflow/core/framework/common_shape_fns.cc
index 4796c3c00a..315c99d32b 100644
--- a/tensorflow/core/framework/common_shape_fns.cc
+++ b/tensorflow/core/framework/common_shape_fns.cc
@@ -1020,6 +1020,29 @@ Status UnknownShape(shape_inference::InferenceContext* c) {
   return Status::OK();
 }
 
+template <typename T>
+Status ReductionShapeHelper(const Tensor* reduction_indices_t,
+                            const int32 input_rank,
+                            std::set<int64>& true_indices) {
+  auto reduction_indices = reduction_indices_t->flat<T>();
+  for (int i = 0; i < reduction_indices_t->NumElements(); ++i) {
+    const T reduction_index = reduction_indices(i);
+    if (reduction_index < -input_rank || reduction_index >= input_rank) {
+      return errors::InvalidArgument("Invalid reduction dimension ",
+                                     reduction_index, " for input with ",
+                                     input_rank, " dimensions.");
+    }
+
+    auto wrapped_index = reduction_index;
+    if (wrapped_index < 0) {
+      wrapped_index += input_rank;
+    }
+
+    true_indices.insert(wrapped_index);
+  }
+  return Status::OK();
+}
+
 Status ReductionShape(InferenceContext* c) {
   ShapeHandle input = c->input(0);
 
@@ -1050,22 +1073,16 @@ Status ReductionShape(InferenceContext* c) {
   }
 
   const int32 input_rank = c->Rank(input);
-  std::set<int32> true_indices;
-  auto reduction_indices = reduction_indices_t->flat<int32>();
-  for (int i = 0; i < reduction_indices_t->NumElements(); ++i) {
-    int32 reduction_index = reduction_indices(i);
-    if (reduction_index < -input_rank || reduction_index >= input_rank) {
-      return errors::InvalidArgument("Invalid reduction dimension ",
-                                     reduction_index, " for input with ",
-                                     input_rank, " dimensions.");
-    }
-
-    int32 wrapped_index = reduction_index;
-    if (wrapped_index < 0) {
-      wrapped_index += input_rank;
-    }
-
-    true_indices.insert(wrapped_index);
+  std::set<int64> true_indices;
+  if (reduction_indices_t->dtype() == DataType::DT_INT32) {
+    TF_RETURN_IF_ERROR(ReductionShapeHelper<int32>(reduction_indices_t,
+                                                   input_rank, true_indices));
+  } else if (reduction_indices_t->dtype() == DataType::DT_INT64) {
+    TF_RETURN_IF_ERROR(ReductionShapeHelper<int64>(reduction_indices_t,
+                                                   input_rank, true_indices));
+  } else {
+    return errors::InvalidArgument(
+        "reduction_indices can only be int32 or int64");
   }
 
   std::vector<DimensionHandle> dims;
@@ -1319,11 +1336,10 @@ Status ScatterNdUpdateShape(InferenceContext* c) {
       Status s = c->Merge(prefix_indices, prefix_updates, &unused);
       if (!s.ok()) {
         return errors::InvalidArgument(
-            "The outer ", num_outer_dims,
-            " dimensions of indices.shape=", c->DebugString(indices_shape),
-            " must match the outer ", num_outer_dims,
-            " dimensions of updates.shape=", c->DebugString(updates_shape),
-            ": ", s.error_message());
+            "The outer ", num_outer_dims, " dimensions of indices.shape=",
+            c->DebugString(indices_shape), " must match the outer ",
+            num_outer_dims, " dimensions of updates.shape=",
+            c->DebugString(updates_shape), ": ", s.error_message());
       }
 
       ShapeHandle input_suffix;
-- 
GitLab


From d7409d32bba5ffa89141ec5427780f68a3b6942d Mon Sep 17 00:00:00 2001
From: Simone Cirillo <my.accounts@gmx.se>
Date: Sat, 21 Oct 2017 02:44:08 +0200
Subject: [PATCH 646/909] Fix import of spatial_softmax from
 tensorflow.contrib.layers (#13833)

---
 tensorflow/contrib/layers/__init__.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/contrib/layers/__init__.py b/tensorflow/contrib/layers/__init__.py
index d8ab7c2d70..d309ba958d 100644
--- a/tensorflow/contrib/layers/__init__.py
+++ b/tensorflow/contrib/layers/__init__.py
@@ -47,6 +47,7 @@ See the @{$python/contrib.layers} guide.
 @@separable_conv2d
 @@separable_convolution2d
 @@softmax
+@@spatial_softmax
 @@stack
 @@unit_norm
 @@bow_encoder
-- 
GitLab


From 62df65c7255e2a8878cd29f66fe80ff8952de157 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 20 Oct 2017 17:46:17 -0700
Subject: [PATCH 647/909] Add dtype argument to Mean and Accuracy
 object-oriented metrics.

PiperOrigin-RevId: 172957714
---
 .../contrib/eager/python/metrics_impl.py      | 27 +++++++++++--------
 .../contrib/eager/python/metrics_test.py      | 20 ++++++++++++++
 2 files changed, 36 insertions(+), 11 deletions(-)

diff --git a/tensorflow/contrib/eager/python/metrics_impl.py b/tensorflow/contrib/eager/python/metrics_impl.py
index 2a624b218c..2139c2b4b9 100644
--- a/tensorflow/contrib/eager/python/metrics_impl.py
+++ b/tensorflow/contrib/eager/python/metrics_impl.py
@@ -198,13 +198,19 @@ class Mean(Metric):
   # TODO(josh11b): Maybe have a dtype argument that defaults to tf.float64?
   # Or defaults to type of the input if it is tf.float32, else tf.float64?
 
-  def build(self, values, weights=None):
-    del values, weights  # build() does not use call's arguments
+  def __init__(self, name=None, dtype=dtypes.float64):
+    super(Mean, self).__init__(name=name)
+    self.dtype = dtype
+
+  def build(self, *args, **kwargs):
+    # build() does not use call's arguments, by using *args, **kwargs
+    # we make it easier to inherit from Mean().
+    del args, kwargs
     self.numer = self.add_variable(name="numer", shape=(),
-                                   dtype=dtypes.float64,
+                                   dtype=self.dtype,
                                    initializer=init_ops.zeros_initializer)
     self.denom = self.add_variable(name="denom", shape=(),
-                                   dtype=dtypes.float64,
+                                   dtype=self.dtype,
                                    initializer=init_ops.zeros_initializer)
 
   def call(self, values, weights=None):
@@ -219,13 +225,13 @@ class Mean(Metric):
     """
     if weights is None:
       self.denom.assign_add(
-          math_ops.cast(array_ops.size(values), dtypes.float64))
+          math_ops.cast(array_ops.size(values), self.dtype))
       values = math_ops.reduce_sum(values)
-      self.numer.assign_add(math_ops.cast(values, dtypes.float64))
+      self.numer.assign_add(math_ops.cast(values, self.dtype))
     else:
-      weights = math_ops.cast(weights, dtypes.float64)
+      weights = math_ops.cast(weights, self.dtype)
       self.denom.assign_add(math_ops.reduce_sum(weights))
-      values = math_ops.cast(values, dtypes.float64) * weights
+      values = math_ops.cast(values, self.dtype) * weights
       self.numer.assign_add(math_ops.reduce_sum(values))
 
   def result(self):
@@ -235,9 +241,8 @@ class Mean(Metric):
 class Accuracy(Mean):
   """Calculates how often `predictions` matches `labels`."""
 
-  def build(self, labels, predictions, weights=None):
-    del labels, predictions, weights
-    super(Accuracy, self).build(None)  # Arguments are unused
+  def __init__(self, name=None, dtype=dtypes.float64):
+    super(Accuracy, self).__init__(name=name, dtype=dtype)
 
   def call(self, labels, predictions, weights=None):
     """Accumulate accuracy statistics.
diff --git a/tensorflow/contrib/eager/python/metrics_test.py b/tensorflow/contrib/eager/python/metrics_test.py
index bfb79cd72e..9743666c89 100644
--- a/tensorflow/contrib/eager/python/metrics_test.py
+++ b/tensorflow/contrib/eager/python/metrics_test.py
@@ -34,6 +34,8 @@ class MetricsTest(test.TestCase):
     m(1000)
     m([10000.0, 100000.0])
     self.assertEqual(111111.0/6, m.result().numpy())
+    self.assertEqual(dtypes.float64, m.dtype)
+    self.assertEqual(dtypes.float64, m.result().dtype)
 
   def testWeightedMean(self):
     m = metrics.Mean()
@@ -41,6 +43,14 @@ class MetricsTest(test.TestCase):
     m([500000, 5000, 500])  # weights of 1 each
     self.assertNear(535521/4.5, m.result().numpy(), 0.001)
 
+  def testMeanDtype(self):
+    # Can override default dtype of float64.
+    m = metrics.Mean(dtype=dtypes.float32)
+    m([0, 2])
+    self.assertEqual(1, m.result().numpy())
+    self.assertEqual(dtypes.float32, m.dtype)
+    self.assertEqual(dtypes.float32, m.result().dtype)
+
   def testAccuracy(self):
     m = metrics.Accuracy()
     m([0, 1, 2, 3], [0, 0, 0, 0])  # 1 correct
@@ -49,6 +59,8 @@ class MetricsTest(test.TestCase):
     m([6], [6])  # 1 correct
     m([7], [2])  # 0 correct
     self.assertEqual(3.0/8, m.result().numpy())
+    self.assertEqual(dtypes.float64, m.dtype)
+    self.assertEqual(dtypes.float64, m.result().dtype)
 
   def testWeightedAccuracy(self):
     m = metrics.Accuracy()
@@ -60,6 +72,14 @@ class MetricsTest(test.TestCase):
     m([7], [2])  # 0 correct, weight 1
     self.assertEqual(2.5/5, m.result().numpy())
 
+  def testAccuracyDtype(self):
+    # Can override default dtype of float64.
+    m = metrics.Accuracy(dtype=dtypes.float32)
+    m([0, 0], [0, 1])
+    self.assertEqual(0.5, m.result().numpy())
+    self.assertEqual(dtypes.float32, m.dtype)
+    self.assertEqual(dtypes.float32, m.result().dtype)
+
   def testTwoMeans(self):
     # Verify two metrics with the same class and name don't
     # accidentally share state.
-- 
GitLab


From 0d6a2e35312c71cf8a145a7c40d69883e254daee Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Fri, 20 Oct 2017 18:19:13 -0700
Subject: [PATCH 648/909] Internal change.

PiperOrigin-RevId: 172960439
---
 tensorflow/contrib/makefile/proto_text_pb_cc_files.txt | 1 +
 tensorflow/contrib/makefile/proto_text_pb_h_files.txt  | 1 +
 tensorflow/contrib/makefile/tf_pb_text_files.txt       | 1 +
 tensorflow/contrib/makefile/tf_proto_files.txt         | 1 +
 4 files changed, 4 insertions(+)

diff --git a/tensorflow/contrib/makefile/proto_text_pb_cc_files.txt b/tensorflow/contrib/makefile/proto_text_pb_cc_files.txt
index 5ade8942af..938c4a53ab 100644
--- a/tensorflow/contrib/makefile/proto_text_pb_cc_files.txt
+++ b/tensorflow/contrib/makefile/proto_text_pb_cc_files.txt
@@ -24,6 +24,7 @@ tensorflow/core/framework/summary.pb.cc
 tensorflow/core/framework/step_stats.pb.cc
 tensorflow/core/framework/resource_handle.pb.cc
 tensorflow/core/framework/remote_fused_graph_execute_info.pb.cc
+tensorflow/core/framework/api_def.pb.cc
 tensorflow/core/framework/op_def.pb.cc
 tensorflow/core/framework/node_def.pb.cc
 tensorflow/core/framework/log_memory.pb.cc
diff --git a/tensorflow/contrib/makefile/proto_text_pb_h_files.txt b/tensorflow/contrib/makefile/proto_text_pb_h_files.txt
index 1f0ad06cdc..aa91b2f954 100644
--- a/tensorflow/contrib/makefile/proto_text_pb_h_files.txt
+++ b/tensorflow/contrib/makefile/proto_text_pb_h_files.txt
@@ -25,6 +25,7 @@ tensorflow/core/framework/summary.pb.h
 tensorflow/core/framework/step_stats.pb.h
 tensorflow/core/framework/resource_handle.pb.h
 tensorflow/core/framework/remote_fused_graph_execute_info.pb.h
+tensorflow/core/framework/api_def.pb.h
 tensorflow/core/framework/op_def.pb.h
 tensorflow/core/framework/node_def.pb.h
 tensorflow/core/framework/log_memory.pb.h
diff --git a/tensorflow/contrib/makefile/tf_pb_text_files.txt b/tensorflow/contrib/makefile/tf_pb_text_files.txt
index c39257ffa9..b5431df2eb 100644
--- a/tensorflow/contrib/makefile/tf_pb_text_files.txt
+++ b/tensorflow/contrib/makefile/tf_pb_text_files.txt
@@ -17,6 +17,7 @@ tensorflow/core/framework/summary.pb_text.cc
 tensorflow/core/framework/step_stats.pb_text.cc
 tensorflow/core/framework/resource_handle.pb_text.cc
 tensorflow/core/framework/remote_fused_graph_execute_info.pb_text.cc
+tensorflow/core/framework/api_def.pb_text.cc
 tensorflow/core/framework/op_def.pb_text.cc
 tensorflow/core/framework/node_def.pb_text.cc
 tensorflow/core/framework/log_memory.pb_text.cc
diff --git a/tensorflow/contrib/makefile/tf_proto_files.txt b/tensorflow/contrib/makefile/tf_proto_files.txt
index a1a9aa7190..d569bde637 100644
--- a/tensorflow/contrib/makefile/tf_proto_files.txt
+++ b/tensorflow/contrib/makefile/tf_proto_files.txt
@@ -30,6 +30,7 @@ tensorflow/core/framework/step_stats.proto
 tensorflow/core/framework/resource_handle.proto
 tensorflow/core/framework/remote_fused_graph_execute_info.proto
 tensorflow/core/framework/reader_base.proto
+tensorflow/core/framework/api_def.proto
 tensorflow/core/framework/op_def.proto
 tensorflow/core/framework/node_def.proto
 tensorflow/core/framework/log_memory.proto
-- 
GitLab


From 93e8f3c67d82c2d43b8dddd4cb8d7f02259d0e7e Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Fri, 20 Oct 2017 18:20:05 -0700
Subject: [PATCH 649/909] Adding Python ApiDef overrides.

PiperOrigin-RevId: 172960496
---
 tensorflow/core/BUILD                         |   5 +
 .../core/api_def/python_api/api_def_B.pbtxt   |  18 ++
 .../core/api_def/python_api/api_def_C.pbtxt   |  15 ++
 .../core/api_def/python_api/api_def_D.pbtxt   |  54 ++++++
 .../core/api_def/python_api/api_def_E.pbtxt   |  30 +++
 .../core/api_def/python_api/api_def_F.pbtxt   |  21 +++
 .../core/api_def/python_api/api_def_H.pbtxt   |   6 +
 .../core/api_def/python_api/api_def_I.pbtxt   |  15 ++
 .../core/api_def/python_api/api_def_L.pbtxt   |  24 +++
 .../core/api_def/python_api/api_def_M.pbtxt   |  78 ++++++++
 .../core/api_def/python_api/api_def_Q.pbtxt   |  27 +++
 .../core/api_def/python_api/api_def_R.pbtxt   |  36 ++++
 .../core/api_def/python_api/api_def_S.pbtxt   |  36 ++++
 tensorflow/tools/api/tests/BUILD              |  15 ++
 .../tools/api/tests/api_compatibility_test.py | 177 ++++++++++++++++++
 .../tools/api/tests/convert_from_multiline.cc |  63 +++++++
 16 files changed, 620 insertions(+)
 create mode 100644 tensorflow/core/api_def/python_api/api_def_B.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_C.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_D.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_E.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_F.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_H.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_I.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_L.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_M.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_Q.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_R.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_S.pbtxt
 create mode 100644 tensorflow/tools/api/tests/convert_from_multiline.cc

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index d198a796a7..6ad93a73f4 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -3326,6 +3326,11 @@ filegroup(
     data = glob(["api_def/base_api/*"]),
 )
 
+filegroup(
+    name = "python_api_def",
+    data = glob(["api_def/python_api/*"]),
+)
+
 tf_cc_test(
     name = "api_test",
     srcs = ["api_def/api_test.cc"],
diff --git a/tensorflow/core/api_def/python_api/api_def_B.pbtxt b/tensorflow/core/api_def/python_api/api_def_B.pbtxt
new file mode 100644
index 0000000000..9b5df58eba
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_B.pbtxt
@@ -0,0 +1,18 @@
+op {
+  graph_op_name: "BitwiseAnd"
+  endpoint {
+    name: "bitwise.bitwise_and"
+  }
+}
+op {
+  graph_op_name: "BitwiseOr"
+  endpoint {
+    name: "bitwise.bitwise_or"
+  }
+}
+op {
+  graph_op_name: "BitwiseXor"
+  endpoint {
+    name: "bitwise.bitwise_xor"
+  }
+}
diff --git a/tensorflow/core/api_def/python_api/api_def_C.pbtxt b/tensorflow/core/api_def/python_api/api_def_C.pbtxt
new file mode 100644
index 0000000000..cf8d0622be
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_C.pbtxt
@@ -0,0 +1,15 @@
+op {
+  graph_op_name: "Cholesky"
+  endpoint {
+    name: "cholesky"
+  }
+  endpoint {
+    name: "linalg.cholesky"
+  }
+}
+op {
+  graph_op_name: "CropAndResize"
+  endpoint {
+    name: "image.crop_and_resize"
+  }
+}
diff --git a/tensorflow/core/api_def/python_api/api_def_D.pbtxt b/tensorflow/core/api_def/python_api/api_def_D.pbtxt
new file mode 100644
index 0000000000..12e0dbec1c
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_D.pbtxt
@@ -0,0 +1,54 @@
+op {
+  graph_op_name: "DecodeAndCropJpeg"
+  endpoint {
+    name: "image.decode_and_crop_jpeg"
+  }
+}
+op {
+  graph_op_name: "DecodeBmp"
+  endpoint {
+    name: "image.decode_bmp"
+  }
+}
+op {
+  graph_op_name: "DecodeGif"
+  endpoint {
+    name: "image.decode_gif"
+  }
+}
+op {
+  graph_op_name: "DecodeJpeg"
+  endpoint {
+    name: "image.decode_jpeg"
+  }
+}
+op {
+  graph_op_name: "DecodePng"
+  endpoint {
+    name: "image.decode_png"
+  }
+}
+op {
+  graph_op_name: "DepthwiseConv2dNative"
+  endpoint {
+    name: "nn.depthwise_conv2d_native"
+  }
+}
+op {
+  graph_op_name: "DepthwiseConv2dNativeBackpropFilter"
+  endpoint {
+    name: "nn.depthwise_conv2d_native_backprop_filter"
+  }
+}
+op {
+  graph_op_name: "DepthwiseConv2dNativeBackpropInput"
+  endpoint {
+    name: "nn.depthwise_conv2d_native_backprop_input"
+  }
+}
+op {
+  graph_op_name: "DrawBoundingBoxes"
+  endpoint {
+    name: "image.draw_bounding_boxes"
+  }
+}
diff --git a/tensorflow/core/api_def/python_api/api_def_E.pbtxt b/tensorflow/core/api_def/python_api/api_def_E.pbtxt
new file mode 100644
index 0000000000..f6871f7138
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_E.pbtxt
@@ -0,0 +1,30 @@
+op {
+  graph_op_name: "Elu"
+  endpoint {
+    name: "nn.elu"
+  }
+}
+op {
+  graph_op_name: "EncodeJpeg"
+  endpoint {
+    name: "image.encode_jpeg"
+  }
+}
+op {
+  graph_op_name: "EncodePng"
+  endpoint {
+    name: "image.encode_png"
+  }
+}
+op {
+  graph_op_name: "ExtractGlimpse"
+  endpoint {
+    name: "image.extract_glimpse"
+  }
+}
+op {
+  graph_op_name: "ExtractJpegShape"
+  endpoint {
+    name: "image.extract_jpeg_shape"
+  }
+}
diff --git a/tensorflow/core/api_def/python_api/api_def_F.pbtxt b/tensorflow/core/api_def/python_api/api_def_F.pbtxt
new file mode 100644
index 0000000000..844a1348a3
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_F.pbtxt
@@ -0,0 +1,21 @@
+op {
+  graph_op_name: "FFT"
+  endpoint {
+    name: "fft"
+  }
+  endpoint {
+    name: "spectral.fft"
+  }
+}
+op {
+  graph_op_name: "FractionalAvgPool"
+  endpoint {
+    name: "nn.fractional_avg_pool"
+  }
+}
+op {
+  graph_op_name: "FractionalMaxPool"
+  endpoint {
+    name: "nn.fractional_max_pool"
+  }
+}
diff --git a/tensorflow/core/api_def/python_api/api_def_H.pbtxt b/tensorflow/core/api_def/python_api/api_def_H.pbtxt
new file mode 100644
index 0000000000..55998189f4
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_H.pbtxt
@@ -0,0 +1,6 @@
+op {
+  graph_op_name: "HSVToRGB"
+  endpoint {
+    name: "image.hsv_to_rgb"
+  }
+}
diff --git a/tensorflow/core/api_def/python_api/api_def_I.pbtxt b/tensorflow/core/api_def/python_api/api_def_I.pbtxt
new file mode 100644
index 0000000000..6c794fab0d
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_I.pbtxt
@@ -0,0 +1,15 @@
+op {
+  graph_op_name: "IFFT"
+  endpoint {
+    name: "ifft"
+  }
+  endpoint {
+    name: "spectral.ifft"
+  }
+}
+op {
+  graph_op_name: "Invert"
+  endpoint {
+    name: "bitwise.invert"
+  }
+}
diff --git a/tensorflow/core/api_def/python_api/api_def_L.pbtxt b/tensorflow/core/api_def/python_api/api_def_L.pbtxt
new file mode 100644
index 0000000000..38ba26a8e8
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_L.pbtxt
@@ -0,0 +1,24 @@
+op {
+  graph_op_name: "L2Loss"
+  endpoint {
+    name: "nn.l2_loss"
+  }
+}
+op {
+  graph_op_name: "LRN"
+  endpoint {
+    name: "nn.local_response_normalization"
+  }
+  endpoint {
+    name: "nn.lrn"
+  }
+}
+op {
+  graph_op_name: "LinSpace"
+  endpoint {
+    name: "lin_space"
+  }
+  endpoint {
+    name: "linspace"
+  }
+}
diff --git a/tensorflow/core/api_def/python_api/api_def_M.pbtxt b/tensorflow/core/api_def/python_api/api_def_M.pbtxt
new file mode 100644
index 0000000000..154071f6bc
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_M.pbtxt
@@ -0,0 +1,78 @@
+op {
+  graph_op_name: "MatrixBandPart"
+  endpoint {
+    name: "linalg.band_part"
+  }
+  endpoint {
+    name: "matrix_band_part"
+  }
+}
+op {
+  graph_op_name: "MatrixDeterminant"
+  endpoint {
+    name: "linalg.det"
+  }
+  endpoint {
+    name: "matrix_determinant"
+  }
+}
+op {
+  graph_op_name: "MatrixDiag"
+  endpoint {
+    name: "linalg.diag"
+  }
+  endpoint {
+    name: "matrix_diag"
+  }
+}
+op {
+  graph_op_name: "MatrixDiagPart"
+  endpoint {
+    name: "linalg.diag_part"
+  }
+  endpoint {
+    name: "matrix_diag_part"
+  }
+}
+op {
+  graph_op_name: "MatrixInverse"
+  endpoint {
+    name: "linalg.inv"
+  }
+  endpoint {
+    name: "matrix_inverse"
+  }
+}
+op {
+  graph_op_name: "MatrixSetDiag"
+  endpoint {
+    name: "linalg.set_diag"
+  }
+  endpoint {
+    name: "matrix_set_diag"
+  }
+}
+op {
+  graph_op_name: "MatrixSolve"
+  endpoint {
+    name: "linalg.solve"
+  }
+  endpoint {
+    name: "matrix_solve"
+  }
+}
+op {
+  graph_op_name: "MatrixTriangularSolve"
+  endpoint {
+    name: "linalg.triangular_solve"
+  }
+  endpoint {
+    name: "matrix_triangular_solve"
+  }
+}
+op {
+  graph_op_name: "MaxPoolWithArgmax"
+  endpoint {
+    name: "nn.max_pool_with_argmax"
+  }
+}
diff --git a/tensorflow/core/api_def/python_api/api_def_Q.pbtxt b/tensorflow/core/api_def/python_api/api_def_Q.pbtxt
new file mode 100644
index 0000000000..cba032880f
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_Q.pbtxt
@@ -0,0 +1,27 @@
+op {
+  graph_op_name: "Qr"
+  endpoint {
+    name: "linalg.qr"
+  }
+  endpoint {
+    name: "qr"
+  }
+}
+op {
+  graph_op_name: "QuantizedAvgPool"
+  endpoint {
+    name: "nn.quantized_avg_pool"
+  }
+}
+op {
+  graph_op_name: "QuantizedMaxPool"
+  endpoint {
+    name: "nn.quantized_max_pool"
+  }
+}
+op {
+  graph_op_name: "QuantizedReluX"
+  endpoint {
+    name: "nn.quantized_relu_x"
+  }
+}
diff --git a/tensorflow/core/api_def/python_api/api_def_R.pbtxt b/tensorflow/core/api_def/python_api/api_def_R.pbtxt
new file mode 100644
index 0000000000..9a57e72be0
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_R.pbtxt
@@ -0,0 +1,36 @@
+op {
+  graph_op_name: "RGBToHSV"
+  endpoint {
+    name: "image.rgb_to_hsv"
+  }
+}
+op {
+  graph_op_name: "Relu"
+  endpoint {
+    name: "nn.relu"
+  }
+}
+op {
+  graph_op_name: "ResizeArea"
+  endpoint {
+    name: "image.resize_area"
+  }
+}
+op {
+  graph_op_name: "ResizeBicubic"
+  endpoint {
+    name: "image.resize_bicubic"
+  }
+}
+op {
+  graph_op_name: "ResizeBilinear"
+  endpoint {
+    name: "image.resize_bilinear"
+  }
+}
+op {
+  graph_op_name: "ResizeNearestNeighbor"
+  endpoint {
+    name: "image.resize_nearest_neighbor"
+  }
+}
diff --git a/tensorflow/core/api_def/python_api/api_def_S.pbtxt b/tensorflow/core/api_def/python_api/api_def_S.pbtxt
new file mode 100644
index 0000000000..9c7a39038e
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_S.pbtxt
@@ -0,0 +1,36 @@
+op {
+  graph_op_name: "SdcaFprint"
+  endpoint {
+    name: "train.sdca_fprint"
+  }
+}
+op {
+  graph_op_name: "SdcaOptimizer"
+  endpoint {
+    name: "train.sdca_optimizer"
+  }
+}
+op {
+  graph_op_name: "SdcaShrinkL1"
+  endpoint {
+    name: "train.sdca_shrink_l1"
+  }
+}
+op {
+  graph_op_name: "Selu"
+  endpoint {
+    name: "nn.selu"
+  }
+}
+op {
+  graph_op_name: "Softplus"
+  endpoint {
+    name: "nn.softplus"
+  }
+}
+op {
+  graph_op_name: "Softsign"
+  endpoint {
+    name: "nn.softsign"
+  }
+}
diff --git a/tensorflow/tools/api/tests/BUILD b/tensorflow/tools/api/tests/BUILD
index e99cc0572f..a913e35101 100644
--- a/tensorflow/tools/api/tests/BUILD
+++ b/tensorflow/tools/api/tests/BUILD
@@ -11,10 +11,15 @@ exports_files([
     "API_UPDATE_WARNING.txt",
 ])
 
+load("//tensorflow:tensorflow.bzl", "tf_cc_binary")
+
 py_test(
     name = "api_compatibility_test",
     srcs = ["api_compatibility_test.py"],
     data = [
+        ":convert_from_multiline",
+        "//tensorflow/core:base_api_def",
+        "//tensorflow/core:python_api_def",
         "//tensorflow/tools/api/golden:api_golden",
         "//tensorflow/tools/api/tests:API_UPDATE_WARNING.txt",
         "//tensorflow/tools/api/tests:README.txt",
@@ -23,6 +28,7 @@ py_test(
     deps = [
         "//tensorflow:tensorflow_py",
         "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework_test_lib",
         "//tensorflow/python:lib",
         "//tensorflow/python:platform",
         "//tensorflow/tools/api/lib:python_object_to_proto_visitor",
@@ -31,6 +37,15 @@ py_test(
     ],
 )
 
+tf_cc_binary(
+    name = "convert_from_multiline",
+    srcs = ["convert_from_multiline.cc"],
+    deps = [
+        "//tensorflow/core:lib",
+        "//tensorflow/core:op_gen_lib",
+    ],
+)
+
 filegroup(
     name = "all_files",
     srcs = glob(
diff --git a/tensorflow/tools/api/tests/api_compatibility_test.py b/tensorflow/tools/api/tests/api_compatibility_test.py
index 1ffa8fc26c..f350c12d41 100644
--- a/tensorflow/tools/api/tests/api_compatibility_test.py
+++ b/tensorflow/tools/api/tests/api_compatibility_test.py
@@ -28,8 +28,11 @@ from __future__ import division
 from __future__ import print_function
 
 import argparse
+from collections import defaultdict
+from operator import attrgetter
 import os
 import re
+import subprocess
 import sys
 import unittest
 
@@ -37,6 +40,7 @@ import tensorflow as tf
 
 from google.protobuf import text_format
 
+from tensorflow.core.framework import api_def_pb2
 from tensorflow.python.lib.io import file_io
 from tensorflow.python.platform import resource_loader
 from tensorflow.python.platform import test
@@ -64,6 +68,11 @@ _API_GOLDEN_FOLDER = 'tensorflow/tools/api/golden'
 _TEST_README_FILE = 'tensorflow/tools/api/tests/README.txt'
 _UPDATE_WARNING_FILE = 'tensorflow/tools/api/tests/API_UPDATE_WARNING.txt'
 
+_ALPHABET = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
+_CONVERT_FROM_MULTILINE_SCRIPT = 'tensorflow/tools/api/tests/convert_from_multiline'
+_BASE_API_DIR = 'tensorflow/core/api_def/base_api'
+_PYTHON_API_DIR = 'tensorflow/core/api_def/python_api'
+
 
 def _KeyToFilePath(key):
   """From a given key, construct a filepath."""
@@ -88,6 +97,30 @@ def _FileNameToKey(filename):
   return api_object_key
 
 
+def _GetSymbol(symbol_id):
+  """Get TensorFlow symbol based on the given identifier.
+
+  Args:
+    symbol_id: Symbol identifier in the form module1.module2. ... .sym.
+
+  Returns:
+    Symbol corresponding to the given id.
+  """
+  # Ignore first module which should be tensorflow
+  symbol_id_split = symbol_id.split('.')[1:]
+  symbol = tf
+  for sym in symbol_id_split:
+    symbol = getattr(symbol, sym)
+  return symbol
+
+
+def _IsGenModule(module_name):
+  if not module_name:
+    return False
+  module_name_split = module_name.split('.')
+  return module_name_split[-1].startswith('gen_')
+
+
 class ApiCompatibilityTest(test.TestCase):
 
   def __init__(self, *args, **kwargs):
@@ -229,6 +262,150 @@ class ApiCompatibilityTest(test.TestCase):
         update_goldens=FLAGS.update_goldens)
 
 
+class ApiDefTest(test.TestCase):
+
+  def __init__(self, *args, **kwargs):
+    super(ApiDefTest, self).__init__(*args, **kwargs)
+    self._first_cap_pattern = re.compile('(.)([A-Z][a-z]+)')
+    self._all_cap_pattern = re.compile('([a-z0-9])([A-Z])')
+
+  def _GenerateLowerCaseOpName(self, op_name):
+    lower_case_name = self._first_cap_pattern.sub(r'\1_\2', op_name)
+    return self._all_cap_pattern.sub(r'\1_\2', lower_case_name).lower()
+
+  def _CreatePythonApiDef(self, base_api_def, endpoint_names):
+    """Creates Python ApiDef that overrides base_api_def if needed.
+
+    Args:
+      base_api_def: (api_def_pb2.ApiDef) base ApiDef instance.
+      endpoint_names: List of Python endpoint names.
+
+    Returns:
+      api_def_pb2.ApiDef instance with overrides for base_api_def
+      if module.name endpoint is different from any existing
+      endpoints in base_api_def. Otherwise, returns None.
+    """
+    endpoint_names_set = set(endpoint_names)
+    base_endpoint_names_set = {
+        self._GenerateLowerCaseOpName(endpoint.name)
+        for endpoint in base_api_def.endpoint}
+
+    if endpoint_names_set == base_endpoint_names_set:
+      return None  # All endpoints are the same
+
+    api_def = api_def_pb2.ApiDef()
+    api_def.graph_op_name = base_api_def.graph_op_name
+
+    for endpoint_name in sorted(endpoint_names):
+      new_endpoint = api_def.endpoint.add()
+      new_endpoint.name = endpoint_name
+
+    return api_def
+
+  def _GetBaseApiMap(self):
+    """Get a map from graph op name to its base ApiDef.
+
+    Returns:
+      Dictionary mapping graph op name to corresponding ApiDef.
+    """
+    # Convert base ApiDef in Multiline format to Proto format.
+    converted_base_api_dir = os.path.join(
+        test.get_temp_dir(), 'temp_base_api_defs')
+    subprocess.check_call(
+        [os.path.join(resource_loader.get_root_dir_with_all_resources(),
+                      _CONVERT_FROM_MULTILINE_SCRIPT),
+         _BASE_API_DIR, converted_base_api_dir])
+
+    name_to_base_api_def = {}
+    base_api_files = file_io.get_matching_files(
+        os.path.join(converted_base_api_dir, 'api_def_*.pbtxt'))
+    for base_api_file in base_api_files:
+      if file_io.file_exists(base_api_file):
+        api_defs = api_def_pb2.ApiDefs()
+        text_format.Merge(
+            file_io.read_file_to_string(base_api_file), api_defs)
+        for api_def in api_defs.op:
+          lower_case_name = self._GenerateLowerCaseOpName(api_def.graph_op_name)
+          name_to_base_api_def[lower_case_name] = api_def
+    return name_to_base_api_def
+
+  @unittest.skipUnless(
+      sys.version_info.major == 2 and os.uname()[0] == 'Linux',
+      'API compabitility test goldens are generated using python2 on Linux.')
+  def testAPIDefCompatibility(self):
+    # Get base ApiDef
+    name_to_base_api_def = self._GetBaseApiMap()
+    # Extract Python API
+    visitor = python_object_to_proto_visitor.PythonObjectToProtoVisitor()
+    public_api_visitor = public_api.PublicAPIVisitor(visitor)
+    public_api_visitor.do_not_descend_map['tf'].append('contrib')
+    traverse.traverse(tf, public_api_visitor)
+    proto_dict = visitor.GetProtos()
+
+    # Map from first character of op name to Python ApiDefs.
+    api_def_map = defaultdict(api_def_pb2.ApiDefs)
+    # We need to override all endpoints even if 1 endpoint differs from base
+    # ApiDef. So, we first create a map from an op to all its endpoints.
+    op_to_endpoint_name = defaultdict(list)
+
+    # Generate map from generated python op to endpoint names.
+    for public_module, value in proto_dict.items():
+      module_obj = _GetSymbol(public_module)
+      for sym in value.tf_module.member_method:
+        obj = getattr(module_obj, sym.name)
+
+        # Check if object is defined in gen_* module. That is,
+        # the object has been generated from OpDef.
+        if hasattr(obj, '__module__') and _IsGenModule(obj.__module__):
+          if obj.__name__ not in name_to_base_api_def:
+            # Symbol might be defined only in Python and not generated from
+            # C++ api.
+            continue
+          relative_public_module = public_module[len('tensorflow.'):]
+          full_name = (relative_public_module + '.' + sym.name
+                       if relative_public_module else sym.name)
+          op_to_endpoint_name[obj].append(full_name)
+
+    # Generate Python ApiDef overrides.
+    for op, endpoint_names in op_to_endpoint_name.items():
+      api_def = self._CreatePythonApiDef(
+          name_to_base_api_def[op.__name__], endpoint_names)
+      if api_def:
+        api_defs = api_def_map[op.__name__[0].upper()]
+        api_defs.op.extend([api_def])
+
+    for key in _ALPHABET:
+      # Get new ApiDef for the given key.
+      new_api_defs_str = ''
+      if key in api_def_map:
+        new_api_defs = api_def_map[key]
+        new_api_defs.op.sort(key=attrgetter('graph_op_name'))
+        new_api_defs_str = str(new_api_defs)
+
+      # Get current ApiDef for the given key.
+      api_defs_file_path = os.path.join(
+          _PYTHON_API_DIR, 'api_def_%s.pbtxt' % key)
+      old_api_defs_str = ''
+      if file_io.file_exists(api_defs_file_path):
+        old_api_defs_str = file_io.read_file_to_string(api_defs_file_path)
+
+      if old_api_defs_str == new_api_defs_str:
+        continue
+
+      if FLAGS.update_goldens:
+        if not new_api_defs_str:
+          logging.info('Deleting %s...' % api_defs_file_path)
+          file_io.delete_file(api_defs_file_path)
+        else:
+          logging.info('Updating %s...' % api_defs_file_path)
+          file_io.write_string_to_file(api_defs_file_path, new_api_defs_str)
+      else:
+        self.assertMultiLineEqual(
+            old_api_defs_str, new_api_defs_str,
+            'To update golden API files, run api_compatibility_test locally '
+            'with --update_goldens=True flag.')
+
+
 if __name__ == '__main__':
   parser = argparse.ArgumentParser()
   parser.add_argument(
diff --git a/tensorflow/tools/api/tests/convert_from_multiline.cc b/tensorflow/tools/api/tests/convert_from_multiline.cc
new file mode 100644
index 0000000000..5c5aaa4f06
--- /dev/null
+++ b/tensorflow/tools/api/tests/convert_from_multiline.cc
@@ -0,0 +1,63 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+// Converts all *.pbtxt files in a directory from Multiline to proto format.
+#include "tensorflow/core/framework/op_gen_lib.h"
+#include "tensorflow/core/lib/io/path.h"
+#include "tensorflow/core/platform/env.h"
+#include "tensorflow/core/platform/init_main.h"
+
+namespace tensorflow {
+
+namespace {
+constexpr char kApiDefFilePattern[] = "*.pbtxt";
+
+Status ConvertFilesFromMultiline(const string& input_dir,
+                                 const string& output_dir) {
+  Env* env = Env::Default();
+
+  const string file_pattern = io::JoinPath(input_dir, kApiDefFilePattern);
+  std::vector<string> matching_paths;
+  TF_CHECK_OK(env->GetMatchingPaths(file_pattern, &matching_paths));
+
+  if (!env->IsDirectory(output_dir).ok()) {
+    TF_RETURN_IF_ERROR(env->CreateDir(output_dir));
+  }
+
+  for (const auto& path : matching_paths) {
+    string contents;
+    TF_RETURN_IF_ERROR(tensorflow::ReadFileToString(env, path, &contents));
+    contents = tensorflow::PBTxtFromMultiline(contents);
+    string output_path = io::JoinPath(output_dir, io::Basename(path));
+    // Write contents to output_path
+    TF_RETURN_IF_ERROR(
+        tensorflow::WriteStringToFile(env, output_path, contents));
+  }
+  return Status::OK();
+}
+}  // namespace
+}  // namespace tensorflow
+
+int main(int argc, char* argv[]) {
+  tensorflow::port::InitMain(argv[0], &argc, &argv);
+
+  const std::string usage =
+      "Usage: convert_from_multiline input_dir output_dir";
+  if (argc != 3) {
+    std::cerr << usage << std::endl;
+    return -1;
+  }
+  TF_CHECK_OK(tensorflow::ConvertFilesFromMultiline(argv[1], argv[2]));
+  return 0;
+}
-- 
GitLab


From ba49d85832918837c2d568545f73cc3b2e47763c Mon Sep 17 00:00:00 2001
From: Bjarke Hammersholt Roune <broune@google.com>
Date: Fri, 20 Oct 2017 19:44:11 -0700
Subject: [PATCH 650/909] Slight change to reduce_test to avoid generating inf,
 which was triggering an inf detector unnecessarily.

PiperOrigin-RevId: 172965466
---
 tensorflow/compiler/xla/tests/reduce_test.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/compiler/xla/tests/reduce_test.cc b/tensorflow/compiler/xla/tests/reduce_test.cc
index b48b3a2bdb..794e5a4920 100644
--- a/tensorflow/compiler/xla/tests/reduce_test.cc
+++ b/tensorflow/compiler/xla/tests/reduce_test.cc
@@ -457,7 +457,7 @@ XLA_TEST_F(ReduceTest, Reshape_111x2x25Reduce_111x50_To_R1) {
   const Shape input_shape = ShapeUtil::MakeShape(F32, {rows, 2, cols / 2});
   auto input = builder.Parameter(0, input_shape, "input");
   auto zero = builder.ConstantR0<float>(0.0);
-  auto log_ = builder.Log(input);
+  auto log_ = builder.Tanh(input);
   auto reshape = builder.Reshape(log_, {rows, cols});
   builder.Reduce(reshape, zero, add_f32, /*dimensions_to_reduce=*/{0});
 
@@ -473,7 +473,7 @@ XLA_TEST_F(ReduceTest, Reshape_111x2x25Reduce_111x50_To_R1) {
     for (int64 colno = 0; colno < cols / 2; ++colno) {
       float column_sum = 0;
       for (int64 rowno = 0; rowno < rows; ++rowno) {
-        column_sum += log(input_data(rowno, major, colno));
+        column_sum += tanh(input_data(rowno, major, colno));
       }
       expected.push_back(column_sum);
     }
-- 
GitLab


From 9d55c249c18745fdd2e4e50b8faa3eef2aac4f90 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Sat, 21 Oct 2017 10:06:45 -0700
Subject: [PATCH 651/909] Fix doc in TF_CALL_ when invoked in mobile platform
 (#13881)

* Fix doc in TF_CALL_ when defined(IS_MOBILE_PLATFORM) && !defined(__ANDROID_TYPES_FULL__)

This is a small doc fix that includes bool as part of the types
that is supported in mobile (IS_MOBILE_PLATFORM && !__ANDROID_TYPES_FULL__),
as bool is clearly invoked in the following define.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Also add bool to android full version.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/core/framework/register_types.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/framework/register_types.h b/tensorflow/core/framework/register_types.h
index 61e722e57b..c31ab18cc1 100644
--- a/tensorflow/core/framework/register_types.h
+++ b/tensorflow/core/framework/register_types.h
@@ -87,7 +87,7 @@ limitations under the License.
 
 #elif defined(__ANDROID_TYPES_FULL__)
 
-// Only half, float, int32, int64, and quantized types are supported.
+// Only half, float, int32, int64, bool, and quantized types are supported.
 #define TF_CALL_float(m) m(float)
 #define TF_CALL_double(m)
 #define TF_CALL_int32(m) m(::tensorflow::int32)
@@ -117,7 +117,7 @@ limitations under the License.
 
 #else  // defined(IS_MOBILE_PLATFORM) && !defined(__ANDROID_TYPES_FULL__)
 
-// Only float and int32 are supported.
+// Only float, int32, and bool are supported.
 #define TF_CALL_float(m) m(float)
 #define TF_CALL_double(m)
 #define TF_CALL_int32(m) m(::tensorflow::int32)
-- 
GitLab


From a699458107fd2c6960cda61c6bfef4cd03025887 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Sat, 21 Oct 2017 11:49:50 -0700
Subject: [PATCH 652/909] Update pin for bazel-toolchains to latest version

PiperOrigin-RevId: 173002530
---
 tensorflow/workspace.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index a863aa18dd..8ba8748aae 100644
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -742,9 +742,9 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   native.http_archive(
       name = "bazel_toolchains",
       urls = [
-          "https://mirror.bazel.build/github.com/bazelbuild/bazel-toolchains/archive/b2b4b38433bf2d1159360855ea4004378308711b.tar.gz",
-          # "https://github.com/bazelbuild/bazel-toolchains/archive/b2b4b38433bf2d1159360855ea4004378308711b.tar.gz",
+          "https://mirror.bazel.build/github.com/bazelbuild/bazel-toolchains/archive/af4681c3d19f063f090222ec3d04108c4e0ca255.tar.gz",
+          # "https://github.com/bazelbuild/bazel-toolchains/archive/af4681c3d19f063f090222ec3d04108c4e0ca255.tar.gz",
       ],
-      sha256 = "46187270ca04ff8109980f45c3438fabfe48695e163789096eb82ee097ffe685",
-      strip_prefix = "bazel-toolchains-b2b4b38433bf2d1159360855ea4004378308711b",
+      sha256 = "d58bb2d6c8603f600d522b6104d6192a65339aa26cbba9f11ff5c4b36dedb928",
+      strip_prefix = "bazel-toolchains-af4681c3d19f063f090222ec3d04108c4e0ca255",
   )
-- 
GitLab


From d1183ca6a245cd0b498c46fd1079909ebc4abc3a Mon Sep 17 00:00:00 2001
From: Vijay Vasudevan <vrv@google.com>
Date: Sat, 21 Oct 2017 13:43:01 -0700
Subject: [PATCH 653/909] Give each variable a unique name in
 accumulate_n_v2_eager_test. (#13886)

---
 .../contrib/framework/python/ops/accumulate_n_v2_eager_test.py  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/framework/python/ops/accumulate_n_v2_eager_test.py b/tensorflow/contrib/framework/python/ops/accumulate_n_v2_eager_test.py
index 5f086ea8cc..c2229bb8ad 100644
--- a/tensorflow/contrib/framework/python/ops/accumulate_n_v2_eager_test.py
+++ b/tensorflow/contrib/framework/python/ops/accumulate_n_v2_eager_test.py
@@ -65,7 +65,7 @@ class AccumulateNV2EagerTest(test_util.TensorFlowTestCase):
     num_inputs = 3
     input_vars = [
         resource_variable_ops.ResourceVariable(10.0 * np.random.random(), 
-                                               name="t1")
+                                               name="t%d" % i)
         for i in range(0, num_inputs)
     ]
 
-- 
GitLab


From b927df57f0c09ea62a855795e340d6daf70553df Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Sat, 21 Oct 2017 22:05:18 -0700
Subject: [PATCH 654/909] Update protobuf.cmake to
 b04e5cba356212e4e8c66c61bbe0c3a20537c5b9 (#13893)

This fix tries to address the issue raised in 8187 where
protobuf.cmake used different version as bazel.

The reason for discrepancy was due to the fact that a customerized
protobuf was needed with Windows patch. Since the patch has been
merged in (https://github.com/google/protobuf/pull/2203),
it makes sense to update protobuf.cmake so that the same version
of cmake is used.

This fix fixes 8187.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/contrib/cmake/external/protobuf.cmake | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/cmake/external/protobuf.cmake b/tensorflow/contrib/cmake/external/protobuf.cmake
index d600d8c3c0..1e300e21df 100644
--- a/tensorflow/contrib/cmake/external/protobuf.cmake
+++ b/tensorflow/contrib/cmake/external/protobuf.cmake
@@ -15,8 +15,8 @@
 include (ExternalProject)
 
 set(PROTOBUF_INCLUDE_DIRS ${CMAKE_CURRENT_BINARY_DIR}/protobuf/src/protobuf/src)
-set(PROTOBUF_URL https://github.com/mrry/protobuf.git)  # Includes MSVC fix.
-set(PROTOBUF_TAG 1d2c7b6c7376f396c8c7dd9b6afd2d4f83f3cb05)
+set(PROTOBUF_URL https://github.com/google/protobuf.git)
+set(PROTOBUF_TAG b04e5cba356212e4e8c66c61bbe0c3a20537c5b9)
 
 if(WIN32)
   set(protobuf_STATIC_LIBRARIES 
-- 
GitLab


From 17096081eed7881c0b8ce3c32b5e9795619e27bb Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Sat, 21 Oct 2017 22:14:40 -0700
Subject: [PATCH 655/909] Improve resize_bicubic performance by reorganizing
 loops (#13840)

* Improve resize_bicubic performance by reorganizing loops

This fix tries to address the issue raised in 13693 where
performance of `resize_bicubic` is not on par with opencv.

This fix rearranges the loops so that it is the same for
num_channel=40 and num_channel=3:

Pre-fix:
```
CHANNEL=40
opencv: 145.08ms
tf: 314.26ms

CHANNEL=3
opencv: 11.95ms
tf: 8.95ms
```

Post-fix:
```
CHANNEL=40
opencv: 144.25ms
tf: 214.55ms

CHANNEL=3
opencv: 11.78ms
tf: 14.07ms
```

This fix fixes 13693.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Keep special handling of `num_channels=3` for `resize_bicubic`

This commit keeps special handling of `num_channels=3` for
`resize_bicubic`:
Without special handling:
```
opencv: 11.78ms
tf: 14.07ms
```
With special handling:
```
opencv: 11.74ms
tf: 9.46ms
```

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Expand Benchmark test for resize_bicubic

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Update from review feedback.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/core/kernels/resize_bicubic_op.cc  | 85 +++++++++++--------
 .../core/kernels/resize_bicubic_op_test.cc    | 20 ++++-
 2 files changed, 67 insertions(+), 38 deletions(-)

diff --git a/tensorflow/core/kernels/resize_bicubic_op.cc b/tensorflow/core/kernels/resize_bicubic_op.cc
index 1c43e77e7c..1a9cf4c640 100644
--- a/tensorflow/core/kernels/resize_bicubic_op.cc
+++ b/tensorflow/core/kernels/resize_bicubic_op.cc
@@ -20,7 +20,6 @@ limitations under the License.
 #include <algorithm>
 #include <array>
 
-#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
@@ -29,6 +28,7 @@ limitations under the License.
 #include "tensorflow/core/kernels/image_resizer_state.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/platform/logging.h"
+#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 
 namespace tensorflow {
 namespace {
@@ -235,6 +235,7 @@ inline void interpolate_with_caching(
 
   const T* input_b_ptr = input_data.data();
   float* output_y_ptr = output_data.data();
+  std::vector<float> cached_value(num_channels == 3 ? 0 : 4 * num_channels, 0);
 
   for (int64 b = 0; b < resizer_state.batch_size;
        ++b, input_b_ptr += in_batch_width) {
@@ -248,6 +249,7 @@ inline void interpolate_with_caching(
       const T* y_ptr_1 = input_b_ptr + y_wai.index_1 * in_row_width;
       const T* y_ptr_2 = input_b_ptr + y_wai.index_2 * in_row_width;
       const T* y_ptr_3 = input_b_ptr + y_wai.index_3 * in_row_width;
+
       if (num_channels == 3) {
         // Manually unroll case of 3 channels.
         float cached_value_0[4] = {0};
@@ -330,48 +332,61 @@ inline void interpolate_with_caching(
                       x_wai.weight_2, x_wai.weight_3);
         }
       } else {
-        for (int64 c = 0; c < num_channels; ++c) {
-          float cached_value[4] = {0};
-          for (int64 x = 0; x < resizer_state.out_width; ++x) {
-            const WeightsAndIndices& x_wai = x_wais[x];
-            // Shift values in cached_value to fill first 'advance' values.
-            switch (x_wai.advance) {
-              case 3:
-                cached_value[0] = cached_value[1];
-                cached_value[1] = cached_value[2];
-                cached_value[2] = cached_value[3];
-                break;
-              case 2:
-                cached_value[0] = cached_value[2];
-                cached_value[1] = cached_value[3];
-                break;
-              case 1: {
-                cached_value[0] = cached_value[3];
-                break;
+        for (int64 x = 0; x < resizer_state.out_width; ++x) {
+          const WeightsAndIndices& x_wai = x_wais[x];
+          // Shift values in cached_value to fill first 'advance' values.
+          switch (x_wai.advance) {
+            case 3:
+              for (int64 c = 0; c < num_channels; ++c) {
+                cached_value[4 * c + 0] = cached_value[4 * c + 1];
+                cached_value[4 * c + 1] = cached_value[4 * c + 2];
+                cached_value[4 * c + 2] = cached_value[4 * c + 3];
+              }
+              break;
+            case 2:
+              for (int64 c = 0; c < num_channels; ++c) {
+                cached_value[4 * c + 0] = cached_value[4 * c + 2];
+                cached_value[4 * c + 1] = cached_value[4 * c + 3];
+              }
+              break;
+            case 1: {
+              for (int64 c = 0; c < num_channels; ++c) {
+                cached_value[4 * c + 0] = cached_value[4 * c + 3];
               }
+              break;
             }
+          }
 
-            // Set the remaining '4-advance' values by computing.
-            switch (x_wai.advance) {
-              case 0:
-                cached_value[0] = ComputeYInterpolation(
+          // Set the remaining '4-advance' values by computing.
+          switch (x_wai.advance) {
+            case 0:
+              for (int64 c = 0; c < num_channels; ++c) {
+                cached_value[4 * c + 0] = ComputeYInterpolation(
                     0, c, y_wai, y_ptr_0, y_ptr_1, y_ptr_2, y_ptr_3, x_wai);
-                TF_FALLTHROUGH_INTENDED;
-              case 1:
-                cached_value[1] = ComputeYInterpolation(
+              }
+              TF_FALLTHROUGH_INTENDED;
+            case 1:
+              for (int64 c = 0; c < num_channels; ++c) {
+                cached_value[4 * c + 1] = ComputeYInterpolation(
                     1, c, y_wai, y_ptr_0, y_ptr_1, y_ptr_2, y_ptr_3, x_wai);
-                TF_FALLTHROUGH_INTENDED;
-              case 2:
-                cached_value[2] = ComputeYInterpolation(
+              }
+              TF_FALLTHROUGH_INTENDED;
+            case 2:
+              for (int64 c = 0; c < num_channels; ++c) {
+                cached_value[4 * c + 2] = ComputeYInterpolation(
                     2, c, y_wai, y_ptr_0, y_ptr_1, y_ptr_2, y_ptr_3, x_wai);
-                TF_FALLTHROUGH_INTENDED;
-              case 3:
-                cached_value[3] = ComputeYInterpolation(
+              }
+              TF_FALLTHROUGH_INTENDED;
+            case 3:
+              for (int64 c = 0; c < num_channels; ++c) {
+                cached_value[4 * c + 3] = ComputeYInterpolation(
                     3, c, y_wai, y_ptr_0, y_ptr_1, y_ptr_2, y_ptr_3, x_wai);
-                break;
-            }
+              }
+              break;
+          }
+          for (int64 c = 0; c < num_channels; ++c) {
             output_y_ptr[x * num_channels + c] =
-                Compute(cached_value, x_wai.weight_0, x_wai.weight_1,
+                Compute(&cached_value[4 * c], x_wai.weight_0, x_wai.weight_1,
                         x_wai.weight_2, x_wai.weight_3);
           }
         }
diff --git a/tensorflow/core/kernels/resize_bicubic_op_test.cc b/tensorflow/core/kernels/resize_bicubic_op_test.cc
index ae14d2804e..9e10fec423 100644
--- a/tensorflow/core/kernels/resize_bicubic_op_test.cc
+++ b/tensorflow/core/kernels/resize_bicubic_op_test.cc
@@ -251,14 +251,15 @@ TEST_F(ResizeBicubicOpTest, TestAreaRandomDataSeveralInputsSizes4Channels) {
   RunManyRandomTests(4);
 }
 
-static Graph* ResizeBicubic(int batch_size, int size, int channels) {
+static Graph* ResizeBicubic(int batch_size, int size, int channels,
+                            float scale_y = 0.3, float scale_x = 0.7) {
   Graph* g = new Graph(OpRegistry::Global());
   Tensor input(DT_FLOAT, TensorShape({batch_size, size, size, channels}));
   input.flat<float>().setRandom();
   Tensor shape(DT_INT32, TensorShape({2}));
   auto shape_t = shape.flat<int32>();
-  shape_t(0) = 0.3 * size;
-  shape_t(1) = 0.7 * size;
+  shape_t(0) = scale_y * size;
+  shape_t(1) = scale_x * size;
   test::graph::Binary(g, "ResizeBicubic", test::graph::Constant(g, input),
                       test::graph::Constant(g, shape));
   return g;
@@ -285,4 +286,17 @@ BM_ResizeBicubicDev(32, 128, 3);
 BM_ResizeBicubicDev(32, 512, 3);
 BM_ResizeBicubicDev(32, 1024, 3);
 
+#define BM_ResizeBicubicExpand(BATCH, SIZE, CHANNELS)                          \
+  static void BM_ResizeBicubicExpand##_##BATCH##_##SIZE##_##CHANNELS(int iters) { \
+    testing::ItemsProcessed(static_cast<int64>(iters) * BATCH * SIZE * SIZE *  \
+                            CHANNELS * 8 * 8);                                 \
+    test::Benchmark("cpu", ResizeBicubic(BATCH, SIZE, CHANNELS, 8, 8))         \
+        .Run(iters);                                                           \
+  }                                                                            \
+  BENCHMARK(BM_ResizeBicubicExpand##_##BATCH##_##SIZE##_##CHANNELS);
+
+BM_ResizeBicubicExpand(12, 48, 1);
+BM_ResizeBicubicExpand(12, 48, 3);
+BM_ResizeBicubicExpand(12, 48, 40);
+
 }  // end namespace tensorflow
-- 
GitLab


From 1c1dad105a57bb13711492a8ba5ab9d10c91b5df Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Sat, 21 Oct 2017 23:01:42 -0700
Subject: [PATCH 656/909] Add int64 axis support for reduction ops. (#13891)

* Add int64 axis support for reduction ops.

This fix is a follow up to PR 13863. In PR 13863 the
program crash is fixed if int64 axis is passed to reduction ops,
e.g. reduce_sum, reduce_max, etc. However, 13863 does not
process the case of int64 support, it merely fixes the crash.

This fix adds the support for int64 axis of reduction ops.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Add int64 axis support for mean, prod, sum

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Add int64 axis support for min and max.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Add int64 axis support for reduce_all and reduce_any

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Add test cases for int64 axis support of reduce_any and reduce_all

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/core/kernels/reduction_ops_all.cc  | 16 +++-
 tensorflow/core/kernels/reduction_ops_any.cc  | 16 +++-
 .../core/kernels/reduction_ops_common.cc      | 22 +++--
 .../core/kernels/reduction_ops_common.h       | 27 +++---
 tensorflow/core/kernels/reduction_ops_max.cc  | 90 +++++++++++++------
 tensorflow/core/kernels/reduction_ops_mean.cc | 68 +++++++++-----
 tensorflow/core/kernels/reduction_ops_min.cc  | 90 +++++++++++++------
 tensorflow/core/kernels/reduction_ops_prod.cc | 68 +++++++++-----
 tensorflow/core/kernels/reduction_ops_sum.cc  | 90 +++++++++++++------
 .../python/kernel_tests/reduction_ops_test.py | 52 +++++++++++
 10 files changed, 391 insertions(+), 148 deletions(-)

diff --git a/tensorflow/core/kernels/reduction_ops_all.cc b/tensorflow/core/kernels/reduction_ops_all.cc
index 41abc2b957..4a34c4ef51 100644
--- a/tensorflow/core/kernels/reduction_ops_all.cc
+++ b/tensorflow/core/kernels/reduction_ops_all.cc
@@ -22,7 +22,13 @@ REGISTER_KERNEL_BUILDER(
         .TypeConstraint<int32>("Tidx")
         .Device(DEVICE_CPU)
         .HostMemory("reduction_indices"),
-    ReductionOp<CPUDevice, bool, Eigen::internal::AndReducer>);
+    ReductionOp<CPUDevice, bool, int32, Eigen::internal::AndReducer>);
+REGISTER_KERNEL_BUILDER(
+    Name("All")
+        .TypeConstraint<int64>("Tidx")
+        .Device(DEVICE_CPU)
+        .HostMemory("reduction_indices"),
+    ReductionOp<CPUDevice, bool, int64, Eigen::internal::AndReducer>);
 
 #if GOOGLE_CUDA
 REGISTER_KERNEL_BUILDER(
@@ -30,7 +36,13 @@ REGISTER_KERNEL_BUILDER(
         .TypeConstraint<int32>("Tidx")
         .Device(DEVICE_GPU)
         .HostMemory("reduction_indices"),
-    ReductionOp<GPUDevice, bool, Eigen::internal::AndReducer>);
+    ReductionOp<GPUDevice, bool, int32, Eigen::internal::AndReducer>);
+REGISTER_KERNEL_BUILDER(
+    Name("All")
+        .TypeConstraint<int64>("Tidx")
+        .Device(DEVICE_GPU)
+        .HostMemory("reduction_indices"),
+    ReductionOp<GPUDevice, bool, int64, Eigen::internal::AndReducer>);
 #endif
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/kernels/reduction_ops_any.cc b/tensorflow/core/kernels/reduction_ops_any.cc
index a2087cc3b7..6c0519de95 100644
--- a/tensorflow/core/kernels/reduction_ops_any.cc
+++ b/tensorflow/core/kernels/reduction_ops_any.cc
@@ -22,7 +22,13 @@ REGISTER_KERNEL_BUILDER(
         .TypeConstraint<int32>("Tidx")
         .Device(DEVICE_CPU)
         .HostMemory("reduction_indices"),
-    ReductionOp<CPUDevice, bool, Eigen::internal::OrReducer>);
+    ReductionOp<CPUDevice, bool, int32, Eigen::internal::OrReducer>);
+REGISTER_KERNEL_BUILDER(
+    Name("Any")
+        .TypeConstraint<int64>("Tidx")
+        .Device(DEVICE_CPU)
+        .HostMemory("reduction_indices"),
+    ReductionOp<CPUDevice, bool, int64, Eigen::internal::OrReducer>);
 
 #if GOOGLE_CUDA
 REGISTER_KERNEL_BUILDER(
@@ -30,7 +36,13 @@ REGISTER_KERNEL_BUILDER(
         .TypeConstraint<int32>("Tidx")
         .Device(DEVICE_GPU)
         .HostMemory("reduction_indices"),
-    ReductionOp<GPUDevice, bool, Eigen::internal::OrReducer>);
+    ReductionOp<GPUDevice, bool, int32, Eigen::internal::OrReducer>);
+REGISTER_KERNEL_BUILDER(
+    Name("Any")
+        .TypeConstraint<int64>("Tidx")
+        .Device(DEVICE_GPU)
+        .HostMemory("reduction_indices"),
+    ReductionOp<GPUDevice, bool, int64, Eigen::internal::OrReducer>);
 #endif
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/kernels/reduction_ops_common.cc b/tensorflow/core/kernels/reduction_ops_common.cc
index 5eba4288ac..8daab0d6be 100644
--- a/tensorflow/core/kernels/reduction_ops_common.cc
+++ b/tensorflow/core/kernels/reduction_ops_common.cc
@@ -57,13 +57,12 @@ gtl::InlinedVector<int32, 8> ReductionHelper::permutation() {
   return perm;
 }
 
-Status ReductionHelper::Simplify(const Tensor& data, const Tensor& axis,
-                                 const bool keep_dims) {
-  // bitmap[i] indicates whether to reduce data along i-th axis.
-  gtl::InlinedVector<bool, 4> bitmap(data.dims(), false);
-  auto axis_vec = axis.flat<int32>();
+template <typename Tperm>
+Status SimplifyHelper(const Tensor& data, const Tensor& axis,
+                      gtl::InlinedVector<bool, 4>& bitmap) {
+  auto axis_vec = axis.flat<Tperm>();
   for (int64 i = 0; i < axis.NumElements(); ++i) {
-    int32 index = axis_vec(i);
+    Tperm index = axis_vec(i);
     if (index < -data.dims() || index >= data.dims()) {
       return errors::InvalidArgument("Invalid reduction dimension (", index,
                                      " for input with ", data.dims(),
@@ -72,7 +71,18 @@ Status ReductionHelper::Simplify(const Tensor& data, const Tensor& axis,
     index = (index + data.dims()) % data.dims();
     bitmap[index] = true;
   }
+  return Status::OK();
+}
 
+Status ReductionHelper::Simplify(const Tensor& data, const Tensor& axis,
+                                 const bool keep_dims) {
+  // bitmap[i] indicates whether to reduce data along i-th axis.
+  gtl::InlinedVector<bool, 4> bitmap(data.dims(), false);
+  if (axis.dtype() == DT_INT32) {
+    TF_RETURN_IF_ERROR(SimplifyHelper<int32>(data, axis, bitmap));
+  } else {
+    TF_RETURN_IF_ERROR(SimplifyHelper<int64>(data, axis, bitmap));
+  }
   // Output tensor's dim sizes.
   out_shape_.clear();
   for (int i = 0; i < data.dims(); ++i) {
diff --git a/tensorflow/core/kernels/reduction_ops_common.h b/tensorflow/core/kernels/reduction_ops_common.h
index 71af9d88dc..9da992ccd1 100644
--- a/tensorflow/core/kernels/reduction_ops_common.h
+++ b/tensorflow/core/kernels/reduction_ops_common.h
@@ -25,6 +25,7 @@ limitations under the License.
 
 #include "third_party/eigen3/Eigen/Core"
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+
 #include "tensorflow/core/framework/numeric_op.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
@@ -42,7 +43,7 @@ typedef Eigen::ThreadPoolDevice CPUDevice;
 typedef Eigen::GpuDevice GPUDevice;
 #ifdef TENSORFLOW_USE_SYCL
 typedef Eigen::SyclDevice SYCLDevice;
-#endif // TENSORFLOW_USE_SYCL
+#endif  // TENSORFLOW_USE_SYCL
 
 template <typename Device>
 struct Constants {
@@ -68,11 +69,13 @@ struct ConstantsBase {
   const Eigen::IndexList<Eigen::type2index<1>> kOne;
   const Eigen::IndexList<Eigen::type2index<0>, Eigen::type2index<2>> kZeroTwo;
 };
-template<> struct Constants<CPUDevice> : ConstantsBase{};
+template <>
+struct Constants<CPUDevice> : ConstantsBase {};
 #ifdef TENSORFLOW_USE_SYCL
-template<> struct Constants<SYCLDevice> : ConstantsBase{};
-#endif // TENSORFLOW_USE_SYCL
-#endif // EIGEN_HAS_INDEX_LIST
+template <>
+struct Constants<SYCLDevice> : ConstantsBase {};
+#endif  // TENSORFLOW_USE_SYCL
+#endif  // EIGEN_HAS_INDEX_LIST
 
 class ReductionHelper {
  public:
@@ -131,12 +134,13 @@ class ReductionHelper {
 
 // For operations where the output is a reduction function along some
 // dimensions of the input.
-template <typename Device, class T, typename Reducer>
+template <typename Device, class T, typename Tperm, typename Reducer>
 class ReductionOp : public OpKernel {
  public:
   explicit ReductionOp(OpKernelConstruction* ctx) : OpKernel(ctx) {
     const DataType dt = DataTypeToEnum<T>::v();
-    OP_REQUIRES_OK(ctx, ctx->MatchSignature({dt, DT_INT32}, {dt}));
+    const DataType pt = DataTypeToEnum<Tperm>::v();
+    OP_REQUIRES_OK(ctx, ctx->MatchSignature({dt, pt}, {dt}));
 
     OP_REQUIRES_OK(ctx, ctx->GetAttr("keep_dims", &keep_dims_));
   }
@@ -266,20 +270,19 @@ struct ReduceFunctorBase {
   }
 
   template <typename OUT_T>
-  static void FillIdentity(const Device& d, OUT_T out,
-                           const Reducer& reducer) {
+  static void FillIdentity(const Device& d, OUT_T out, const Reducer& reducer) {
     FillIdentityEigenImpl(d, out, reducer);
   }
 };
 
 template <typename Reducer>
 struct ReduceFunctor<CPUDevice, Reducer>
-        : ReduceFunctorBase<CPUDevice, Reducer>{};
+    : ReduceFunctorBase<CPUDevice, Reducer> {};
 #if TENSORFLOW_USE_SYCL
 template <typename Reducer>
 struct ReduceFunctor<SYCLDevice, Reducer>
-        : ReduceFunctorBase<SYCLDevice, Reducer>{};
-#endif // TENSORFLOW_USE_SYCL
+    : ReduceFunctorBase<SYCLDevice, Reducer> {};
+#endif  // TENSORFLOW_USE_SYCL
 
 }  // namespace functor
 }  // namespace tensorflow
diff --git a/tensorflow/core/kernels/reduction_ops_max.cc b/tensorflow/core/kernels/reduction_ops_max.cc
index 4ca5c11a48..9cf953f4bf 100644
--- a/tensorflow/core/kernels/reduction_ops_max.cc
+++ b/tensorflow/core/kernels/reduction_ops_max.cc
@@ -17,26 +17,39 @@ limitations under the License.
 
 namespace tensorflow {
 
-#define REGISTER_CPU_KERNELS(type)        \
-  REGISTER_KERNEL_BUILDER(                \
-      Name("Max")                         \
-          .Device(DEVICE_CPU)             \
-          .TypeConstraint<type>("T")      \
-          .TypeConstraint<int32>("Tidx"), \
-      ReductionOp<CPUDevice, type, Eigen::internal::MaxReducer<type>>);
+#define REGISTER_CPU_KERNELS(type)                                             \
+  REGISTER_KERNEL_BUILDER(                                                     \
+      Name("Max")                                                              \
+          .Device(DEVICE_CPU)                                                  \
+          .TypeConstraint<type>("T")                                           \
+          .TypeConstraint<int32>("Tidx"),                                      \
+      ReductionOp<CPUDevice, type, int32, Eigen::internal::MaxReducer<type>>); \
+  REGISTER_KERNEL_BUILDER(                                                     \
+      Name("Max")                                                              \
+          .Device(DEVICE_CPU)                                                  \
+          .TypeConstraint<type>("T")                                           \
+          .TypeConstraint<int64>("Tidx"),                                      \
+      ReductionOp<CPUDevice, type, int64, Eigen::internal::MaxReducer<type>>);
 TF_CALL_REAL_NUMBER_TYPES(REGISTER_CPU_KERNELS);
 #undef REGISTER_CPU_KERNELS
 
 #if GOOGLE_CUDA
 
-#define REGISTER_GPU_KERNELS(type)          \
-  REGISTER_KERNEL_BUILDER(                  \
-      Name("Max")                           \
-          .Device(DEVICE_GPU)               \
-          .TypeConstraint<type>("T")        \
-          .TypeConstraint<int32>("Tidx")    \
-          .HostMemory("reduction_indices"), \
-      ReductionOp<GPUDevice, type, Eigen::internal::MaxReducer<type>>);
+#define REGISTER_GPU_KERNELS(type)                                             \
+  REGISTER_KERNEL_BUILDER(                                                     \
+      Name("Max")                                                              \
+          .Device(DEVICE_GPU)                                                  \
+          .TypeConstraint<type>("T")                                           \
+          .TypeConstraint<int32>("Tidx")                                       \
+          .HostMemory("reduction_indices"),                                    \
+      ReductionOp<GPUDevice, type, int32, Eigen::internal::MaxReducer<type>>); \
+  REGISTER_KERNEL_BUILDER(                                                     \
+      Name("Max")                                                              \
+          .Device(DEVICE_GPU)                                                  \
+          .TypeConstraint<type>("T")                                           \
+          .TypeConstraint<int64>("Tidx")                                       \
+          .HostMemory("reduction_indices"),                                    \
+      ReductionOp<GPUDevice, type, int64, Eigen::internal::MaxReducer<type>>);
 REGISTER_GPU_KERNELS(float);
 REGISTER_GPU_KERNELS(double);
 REGISTER_GPU_KERNELS(int64);
@@ -52,21 +65,37 @@ REGISTER_KERNEL_BUILDER(
         .HostMemory("output")
         .TypeConstraint<int32>("T")
         .TypeConstraint<int32>("Tidx"),
-    ReductionOp<CPUDevice, int32, Eigen::internal::MaxReducer<int32>>);
+    ReductionOp<CPUDevice, int32, int32, Eigen::internal::MaxReducer<int32>>);
+REGISTER_KERNEL_BUILDER(
+    Name("Max")
+        .Device(DEVICE_GPU)
+        .HostMemory("reduction_indices")
+        .HostMemory("input")
+        .HostMemory("output")
+        .TypeConstraint<int32>("T")
+        .TypeConstraint<int64>("Tidx"),
+    ReductionOp<CPUDevice, int32, int64, Eigen::internal::MaxReducer<int32>>);
 
 #undef REGISTER_GPU_KERNELS
 
 #endif
 
 #ifdef TENSORFLOW_USE_SYCL
-#define REGISTER_SYCL_KERNELS(type)         \
-  REGISTER_KERNEL_BUILDER(                  \
-      Name("Max")                           \
-          .Device(DEVICE_SYCL)              \
-          .TypeConstraint<type>("T")        \
-          .TypeConstraint<int32>("Tidx")    \
-          .HostMemory("reduction_indices"), \
-      ReductionOp<SYCLDevice, type, Eigen::internal::MaxReducer<type>>);
+#define REGISTER_SYCL_KERNELS(type)                                        \
+  REGISTER_KERNEL_BUILDER(Name("Max")                                      \
+                              .Device(DEVICE_SYCL)                         \
+                              .TypeConstraint<type>("T")                   \
+                              .TypeConstraint<int32>("Tidx")               \
+                              .HostMemory("reduction_indices"),            \
+                          ReductionOp<SYCLDevice, type, int32,             \
+                                      Eigen::internal::MaxReducer<type>>); \
+  REGISTER_KERNEL_BUILDER(Name("Max")                                      \
+                              .Device(DEVICE_SYCL)                         \
+                              .TypeConstraint<type>("T")                   \
+                              .TypeConstraint<int64>("Tidx")               \
+                              .HostMemory("reduction_indices"),            \
+                          ReductionOp<SYCLDevice, type, int64,             \
+                                      Eigen::internal::MaxReducer<type>>);
 REGISTER_SYCL_KERNELS(float);
 REGISTER_SYCL_KERNELS(double);
 
@@ -78,8 +107,17 @@ REGISTER_KERNEL_BUILDER(
         .HostMemory("output")
         .TypeConstraint<int32>("T")
         .TypeConstraint<int32>("Tidx"),
-    ReductionOp<CPUDevice, int32, Eigen::internal::MaxReducer<int32>>);
+    ReductionOp<CPUDevice, int32, int32, Eigen::internal::MaxReducer<int32>>);
+REGISTER_KERNEL_BUILDER(
+    Name("Max")
+        .Device(DEVICE_SYCL)
+        .HostMemory("reduction_indices")
+        .HostMemory("input")
+        .HostMemory("output")
+        .TypeConstraint<int32>("T")
+        .TypeConstraint<int64>("Tidx"),
+    ReductionOp<CPUDevice, int32, int64, Eigen::internal::MaxReducer<int32>>);
 #undef REGISTER_SYCL_KERNELS
-#endif // TENSORFLOW_USE_SYCL
+#endif  // TENSORFLOW_USE_SYCL
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/kernels/reduction_ops_mean.cc b/tensorflow/core/kernels/reduction_ops_mean.cc
index 5b01de8ddb..f61589f913 100644
--- a/tensorflow/core/kernels/reduction_ops_mean.cc
+++ b/tensorflow/core/kernels/reduction_ops_mean.cc
@@ -17,26 +17,39 @@ limitations under the License.
 
 namespace tensorflow {
 
-#define REGISTER_CPU_KERNELS(type)        \
-  REGISTER_KERNEL_BUILDER(                \
-      Name("Mean")                        \
-          .Device(DEVICE_CPU)             \
-          .TypeConstraint<type>("T")      \
-          .TypeConstraint<int32>("Tidx"), \
-      ReductionOp<CPUDevice, type, Eigen::internal::MeanReducer<type>>);
+#define REGISTER_CPU_KERNELS(type)                                          \
+  REGISTER_KERNEL_BUILDER(Name("Mean")                                      \
+                              .Device(DEVICE_CPU)                           \
+                              .TypeConstraint<type>("T")                    \
+                              .TypeConstraint<int32>("Tidx"),               \
+                          ReductionOp<CPUDevice, type, int32,               \
+                                      Eigen::internal::MeanReducer<type>>); \
+  REGISTER_KERNEL_BUILDER(Name("Mean")                                      \
+                              .Device(DEVICE_CPU)                           \
+                              .TypeConstraint<type>("T")                    \
+                              .TypeConstraint<int64>("Tidx"),               \
+                          ReductionOp<CPUDevice, type, int64,               \
+                                      Eigen::internal::MeanReducer<type>>);
 TF_CALL_NUMBER_TYPES(REGISTER_CPU_KERNELS);
 #undef REGISTER_CPU_KERNELS
 
 #if GOOGLE_CUDA
 
-#define REGISTER_GPU_KERNELS(type)          \
-  REGISTER_KERNEL_BUILDER(                  \
-      Name("Mean")                          \
-          .Device(DEVICE_GPU)               \
-          .TypeConstraint<type>("T")        \
-          .TypeConstraint<int32>("Tidx")    \
-          .HostMemory("reduction_indices"), \
-      ReductionOp<GPUDevice, type, Eigen::internal::MeanReducer<type>>);
+#define REGISTER_GPU_KERNELS(type)                                          \
+  REGISTER_KERNEL_BUILDER(Name("Mean")                                      \
+                              .Device(DEVICE_GPU)                           \
+                              .TypeConstraint<type>("T")                    \
+                              .TypeConstraint<int32>("Tidx")                \
+                              .HostMemory("reduction_indices"),             \
+                          ReductionOp<GPUDevice, type, int32,               \
+                                      Eigen::internal::MeanReducer<type>>); \
+  REGISTER_KERNEL_BUILDER(Name("Mean")                                      \
+                              .Device(DEVICE_GPU)                           \
+                              .TypeConstraint<type>("T")                    \
+                              .TypeConstraint<int64>("Tidx")                \
+                              .HostMemory("reduction_indices"),             \
+                          ReductionOp<GPUDevice, type, int64,               \
+                                      Eigen::internal::MeanReducer<type>>);
 TF_CALL_GPU_NUMBER_TYPES(REGISTER_GPU_KERNELS);
 TF_CALL_complex64(REGISTER_GPU_KERNELS);
 TF_CALL_complex128(REGISTER_GPU_KERNELS);
@@ -45,17 +58,24 @@ TF_CALL_complex128(REGISTER_GPU_KERNELS);
 #endif
 
 #ifdef TENSORFLOW_USE_SYCL
-#define REGISTER_SYCL_KERNELS(type)         \
-  REGISTER_KERNEL_BUILDER(                  \
-      Name("Mean")                          \
-          .Device(DEVICE_SYCL)              \
-          .TypeConstraint<type>("T")        \
-          .TypeConstraint<int32>("Tidx")    \
-          .HostMemory("reduction_indices"), \
-      ReductionOp<SYCLDevice, type, Eigen::internal::MeanReducer<type>>);
+#define REGISTER_SYCL_KERNELS(type)                                         \
+  REGISTER_KERNEL_BUILDER(Name("Mean")                                      \
+                              .Device(DEVICE_SYCL)                          \
+                              .TypeConstraint<type>("T")                    \
+                              .TypeConstraint<int32>("Tidx")                \
+                              .HostMemory("reduction_indices"),             \
+                          ReductionOp<SYCLDevice, type, int32,              \
+                                      Eigen::internal::MeanReducer<type>>); \
+  REGISTER_KERNEL_BUILDER(Name("Mean")                                      \
+                              .Device(DEVICE_SYCL)                          \
+                              .TypeConstraint<type>("T")                    \
+                              .TypeConstraint<int64>("Tidx")                \
+                              .HostMemory("reduction_indices"),             \
+                          ReductionOp<SYCLDevice, type, int64,              \
+                                      Eigen::internal::MeanReducer<type>>);
 REGISTER_SYCL_KERNELS(float);
 REGISTER_SYCL_KERNELS(double);
 #undef REGISTER_SYCL_KERNELS
-#endif // TENSORFLOW_USE_SYCL
+#endif  // TENSORFLOW_USE_SYCL
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/kernels/reduction_ops_min.cc b/tensorflow/core/kernels/reduction_ops_min.cc
index 1e394bea41..807ac0a456 100644
--- a/tensorflow/core/kernels/reduction_ops_min.cc
+++ b/tensorflow/core/kernels/reduction_ops_min.cc
@@ -17,26 +17,39 @@ limitations under the License.
 
 namespace tensorflow {
 
-#define REGISTER_CPU_KERNELS(type)        \
-  REGISTER_KERNEL_BUILDER(                \
-      Name("Min")                         \
-          .Device(DEVICE_CPU)             \
-          .TypeConstraint<type>("T")      \
-          .TypeConstraint<int32>("Tidx"), \
-      ReductionOp<CPUDevice, type, Eigen::internal::MinReducer<type>>);
+#define REGISTER_CPU_KERNELS(type)                                             \
+  REGISTER_KERNEL_BUILDER(                                                     \
+      Name("Min")                                                              \
+          .Device(DEVICE_CPU)                                                  \
+          .TypeConstraint<type>("T")                                           \
+          .TypeConstraint<int32>("Tidx"),                                      \
+      ReductionOp<CPUDevice, type, int32, Eigen::internal::MinReducer<type>>); \
+  REGISTER_KERNEL_BUILDER(                                                     \
+      Name("Min")                                                              \
+          .Device(DEVICE_CPU)                                                  \
+          .TypeConstraint<type>("T")                                           \
+          .TypeConstraint<int64>("Tidx"),                                      \
+      ReductionOp<CPUDevice, type, int64, Eigen::internal::MinReducer<type>>);
 TF_CALL_REAL_NUMBER_TYPES(REGISTER_CPU_KERNELS);
 #undef REGISTER_CPU_KERNELS
 
 #if GOOGLE_CUDA
 
-#define REGISTER_GPU_KERNELS(type)          \
-  REGISTER_KERNEL_BUILDER(                  \
-      Name("Min")                           \
-          .Device(DEVICE_GPU)               \
-          .TypeConstraint<type>("T")        \
-          .TypeConstraint<int32>("Tidx")    \
-          .HostMemory("reduction_indices"), \
-      ReductionOp<GPUDevice, type, Eigen::internal::MinReducer<type>>);
+#define REGISTER_GPU_KERNELS(type)                                             \
+  REGISTER_KERNEL_BUILDER(                                                     \
+      Name("Min")                                                              \
+          .Device(DEVICE_GPU)                                                  \
+          .TypeConstraint<type>("T")                                           \
+          .TypeConstraint<int32>("Tidx")                                       \
+          .HostMemory("reduction_indices"),                                    \
+      ReductionOp<GPUDevice, type, int32, Eigen::internal::MinReducer<type>>); \
+  REGISTER_KERNEL_BUILDER(                                                     \
+      Name("Min")                                                              \
+          .Device(DEVICE_GPU)                                                  \
+          .TypeConstraint<type>("T")                                           \
+          .TypeConstraint<int64>("Tidx")                                       \
+          .HostMemory("reduction_indices"),                                    \
+      ReductionOp<GPUDevice, type, int64, Eigen::internal::MinReducer<type>>);
 REGISTER_GPU_KERNELS(float);
 REGISTER_GPU_KERNELS(double);
 
@@ -51,21 +64,37 @@ REGISTER_KERNEL_BUILDER(
         .HostMemory("output")
         .TypeConstraint<int32>("T")
         .TypeConstraint<int32>("Tidx"),
-    ReductionOp<CPUDevice, int32, Eigen::internal::MinReducer<int32>>);
+    ReductionOp<CPUDevice, int32, int32, Eigen::internal::MinReducer<int32>>);
+REGISTER_KERNEL_BUILDER(
+    Name("Min")
+        .Device(DEVICE_GPU)
+        .HostMemory("reduction_indices")
+        .HostMemory("input")
+        .HostMemory("output")
+        .TypeConstraint<int32>("T")
+        .TypeConstraint<int64>("Tidx"),
+    ReductionOp<CPUDevice, int32, int64, Eigen::internal::MinReducer<int32>>);
 
 #undef REGISTER_GPU_KERNELS
 
 #endif
 
 #ifdef TENSORFLOW_USE_SYCL
-#define REGISTER_SYCL_KERNELS(type)         \
-  REGISTER_KERNEL_BUILDER(                  \
-      Name("Min")                           \
-          .Device(DEVICE_SYCL)              \
-          .TypeConstraint<type>("T")        \
-          .TypeConstraint<int32>("Tidx")    \
-          .HostMemory("reduction_indices"), \
-      ReductionOp<SYCLDevice, type, Eigen::internal::MinReducer<type>>);
+#define REGISTER_SYCL_KERNELS(type)                                        \
+  REGISTER_KERNEL_BUILDER(Name("Min")                                      \
+                              .Device(DEVICE_SYCL)                         \
+                              .TypeConstraint<type>("T")                   \
+                              .TypeConstraint<int32>("Tidx")               \
+                              .HostMemory("reduction_indices"),            \
+                          ReductionOp<SYCLDevice, type, int32,             \
+                                      Eigen::internal::MinReducer<type>>); \
+  REGISTER_KERNEL_BUILDER(Name("Min")                                      \
+                              .Device(DEVICE_SYCL)                         \
+                              .TypeConstraint<type>("T")                   \
+                              .TypeConstraint<int64>("Tidx")               \
+                              .HostMemory("reduction_indices"),            \
+                          ReductionOp<SYCLDevice, type, int64,             \
+                                      Eigen::internal::MinReducer<type>>);
 REGISTER_SYCL_KERNELS(float);
 REGISTER_SYCL_KERNELS(double);
 
@@ -77,8 +106,17 @@ REGISTER_KERNEL_BUILDER(
         .HostMemory("output")
         .TypeConstraint<int32>("T")
         .TypeConstraint<int32>("Tidx"),
-    ReductionOp<CPUDevice, int32, Eigen::internal::MinReducer<int32>>);
+    ReductionOp<CPUDevice, int32, int32, Eigen::internal::MinReducer<int32>>);
+REGISTER_KERNEL_BUILDER(
+    Name("Min")
+        .Device(DEVICE_SYCL)
+        .HostMemory("reduction_indices")
+        .HostMemory("input")
+        .HostMemory("output")
+        .TypeConstraint<int32>("T")
+        .TypeConstraint<int64>("Tidx"),
+    ReductionOp<CPUDevice, int32, int64, Eigen::internal::MinReducer<int32>>);
 #undef REGISTER_SYCL_KERNELS
-#endif // TENSORFLOW_USE_SYCL
+#endif  // TENSORFLOW_USE_SYCL
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/kernels/reduction_ops_prod.cc b/tensorflow/core/kernels/reduction_ops_prod.cc
index 33f6ae6bae..e9b23df746 100644
--- a/tensorflow/core/kernels/reduction_ops_prod.cc
+++ b/tensorflow/core/kernels/reduction_ops_prod.cc
@@ -17,26 +17,39 @@ limitations under the License.
 
 namespace tensorflow {
 
-#define REGISTER_CPU_KERNELS(type)        \
-  REGISTER_KERNEL_BUILDER(                \
-      Name("Prod")                        \
-          .Device(DEVICE_CPU)             \
-          .TypeConstraint<type>("T")      \
-          .TypeConstraint<int32>("Tidx"), \
-      ReductionOp<CPUDevice, type, Eigen::internal::ProdReducer<type>>);
+#define REGISTER_CPU_KERNELS(type)                                          \
+  REGISTER_KERNEL_BUILDER(Name("Prod")                                      \
+                              .Device(DEVICE_CPU)                           \
+                              .TypeConstraint<type>("T")                    \
+                              .TypeConstraint<int32>("Tidx"),               \
+                          ReductionOp<CPUDevice, type, int32,               \
+                                      Eigen::internal::ProdReducer<type>>); \
+  REGISTER_KERNEL_BUILDER(Name("Prod")                                      \
+                              .Device(DEVICE_CPU)                           \
+                              .TypeConstraint<type>("T")                    \
+                              .TypeConstraint<int64>("Tidx"),               \
+                          ReductionOp<CPUDevice, type, int64,               \
+                                      Eigen::internal::ProdReducer<type>>);
 TF_CALL_NUMBER_TYPES(REGISTER_CPU_KERNELS);
 #undef REGISTER_CPU_KERNELS
 
 #if GOOGLE_CUDA
 
-#define REGISTER_GPU_KERNELS(type)          \
-  REGISTER_KERNEL_BUILDER(                  \
-      Name("Prod")                          \
-          .Device(DEVICE_GPU)               \
-          .TypeConstraint<type>("T")        \
-          .TypeConstraint<int32>("Tidx")    \
-          .HostMemory("reduction_indices"), \
-      ReductionOp<GPUDevice, type, Eigen::internal::ProdReducer<type>>);
+#define REGISTER_GPU_KERNELS(type)                                          \
+  REGISTER_KERNEL_BUILDER(Name("Prod")                                      \
+                              .Device(DEVICE_GPU)                           \
+                              .TypeConstraint<type>("T")                    \
+                              .TypeConstraint<int32>("Tidx")                \
+                              .HostMemory("reduction_indices"),             \
+                          ReductionOp<GPUDevice, type, int32,               \
+                                      Eigen::internal::ProdReducer<type>>); \
+  REGISTER_KERNEL_BUILDER(Name("Prod")                                      \
+                              .Device(DEVICE_GPU)                           \
+                              .TypeConstraint<type>("T")                    \
+                              .TypeConstraint<int64>("Tidx")                \
+                              .HostMemory("reduction_indices"),             \
+                          ReductionOp<GPUDevice, type, int64,               \
+                                      Eigen::internal::ProdReducer<type>>);
 TF_CALL_GPU_NUMBER_TYPES(REGISTER_GPU_KERNELS);
 TF_CALL_int32(REGISTER_GPU_KERNELS);
 TF_CALL_complex64(REGISTER_GPU_KERNELS);
@@ -46,18 +59,25 @@ TF_CALL_complex128(REGISTER_GPU_KERNELS);
 #endif
 
 #ifdef TENSORFLOW_USE_SYCL
-#define REGISTER_SYCL_KERNELS(type)         \
-  REGISTER_KERNEL_BUILDER(                  \
-      Name("Prod")                          \
-          .Device(DEVICE_SYCL)              \
-          .TypeConstraint<type>("T")        \
-          .TypeConstraint<int32>("Tidx")    \
-          .HostMemory("reduction_indices"), \
-      ReductionOp<SYCLDevice, type, Eigen::internal::ProdReducer<type>>);
+#define REGISTER_SYCL_KERNELS(type)                                         \
+  REGISTER_KERNEL_BUILDER(Name("Prod")                                      \
+                              .Device(DEVICE_SYCL)                          \
+                              .TypeConstraint<type>("T")                    \
+                              .TypeConstraint<int32>("Tidx")                \
+                              .HostMemory("reduction_indices"),             \
+                          ReductionOp<SYCLDevice, type, int32,              \
+                                      Eigen::internal::ProdReducer<type>>); \
+  REGISTER_KERNEL_BUILDER(Name("Prod")                                      \
+                              .Device(DEVICE_SYCL)                          \
+                              .TypeConstraint<type>("T")                    \
+                              .TypeConstraint<int64>("Tidx")                \
+                              .HostMemory("reduction_indices"),             \
+                          ReductionOp<SYCLDevice, type, int64,              \
+                                      Eigen::internal::ProdReducer<type>>);
 REGISTER_SYCL_KERNELS(int32);
 REGISTER_SYCL_KERNELS(float);
 REGISTER_SYCL_KERNELS(double);
 #undef REGISTER_SYCL_KERNELS
-#endif // TENSORFLOW_USE_SYCL
+#endif  // TENSORFLOW_USE_SYCL
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/kernels/reduction_ops_sum.cc b/tensorflow/core/kernels/reduction_ops_sum.cc
index c1f4f3475a..5318d8c133 100644
--- a/tensorflow/core/kernels/reduction_ops_sum.cc
+++ b/tensorflow/core/kernels/reduction_ops_sum.cc
@@ -17,26 +17,39 @@ limitations under the License.
 
 namespace tensorflow {
 
-#define REGISTER_CPU_KERNELS(type)        \
-  REGISTER_KERNEL_BUILDER(                \
-      Name("Sum")                         \
-          .Device(DEVICE_CPU)             \
-          .TypeConstraint<type>("T")      \
-          .TypeConstraint<int32>("Tidx"), \
-      ReductionOp<CPUDevice, type, Eigen::internal::SumReducer<type>>);
+#define REGISTER_CPU_KERNELS(type)                                             \
+  REGISTER_KERNEL_BUILDER(                                                     \
+      Name("Sum")                                                              \
+          .Device(DEVICE_CPU)                                                  \
+          .TypeConstraint<type>("T")                                           \
+          .TypeConstraint<int32>("Tidx"),                                      \
+      ReductionOp<CPUDevice, type, int32, Eigen::internal::SumReducer<type>>); \
+  REGISTER_KERNEL_BUILDER(                                                     \
+      Name("Sum")                                                              \
+          .Device(DEVICE_CPU)                                                  \
+          .TypeConstraint<type>("T")                                           \
+          .TypeConstraint<int64>("Tidx"),                                      \
+      ReductionOp<CPUDevice, type, int64, Eigen::internal::SumReducer<type>>);
 TF_CALL_NUMBER_TYPES(REGISTER_CPU_KERNELS);
 #undef REGISTER_CPU_KERNELS
 
 #if GOOGLE_CUDA
 
-#define REGISTER_GPU_KERNELS(type)          \
-  REGISTER_KERNEL_BUILDER(                  \
-      Name("Sum")                           \
-          .Device(DEVICE_GPU)               \
-          .TypeConstraint<type>("T")        \
-          .TypeConstraint<int32>("Tidx")    \
-          .HostMemory("reduction_indices"), \
-      ReductionOp<GPUDevice, type, Eigen::internal::SumReducer<type>>);
+#define REGISTER_GPU_KERNELS(type)                                             \
+  REGISTER_KERNEL_BUILDER(                                                     \
+      Name("Sum")                                                              \
+          .Device(DEVICE_GPU)                                                  \
+          .TypeConstraint<type>("T")                                           \
+          .TypeConstraint<int32>("Tidx")                                       \
+          .HostMemory("reduction_indices"),                                    \
+      ReductionOp<GPUDevice, type, int32, Eigen::internal::SumReducer<type>>); \
+  REGISTER_KERNEL_BUILDER(                                                     \
+      Name("Sum")                                                              \
+          .Device(DEVICE_GPU)                                                  \
+          .TypeConstraint<type>("T")                                           \
+          .TypeConstraint<int64>("Tidx")                                       \
+          .HostMemory("reduction_indices"),                                    \
+      ReductionOp<GPUDevice, type, int64, Eigen::internal::SumReducer<type>>);
 TF_CALL_GPU_NUMBER_TYPES(REGISTER_GPU_KERNELS);
 TF_CALL_complex64(REGISTER_GPU_KERNELS);
 TF_CALL_complex128(REGISTER_GPU_KERNELS);
@@ -53,19 +66,35 @@ REGISTER_KERNEL_BUILDER(
         .HostMemory("input")
         .HostMemory("output")
         .HostMemory("reduction_indices"),
-    ReductionOp<CPUDevice, int32, Eigen::internal::SumReducer<int32>>);
+    ReductionOp<CPUDevice, int32, int32, Eigen::internal::SumReducer<int32>>);
+REGISTER_KERNEL_BUILDER(
+    Name("Sum")
+        .Device(DEVICE_GPU)
+        .TypeConstraint<int32>("T")
+        .TypeConstraint<int64>("Tidx")
+        .HostMemory("input")
+        .HostMemory("output")
+        .HostMemory("reduction_indices"),
+    ReductionOp<CPUDevice, int32, int64, Eigen::internal::SumReducer<int32>>);
 
 #endif
 
 #ifdef TENSORFLOW_USE_SYCL
-#define REGISTER_SYCL_KERNELS(type)         \
-  REGISTER_KERNEL_BUILDER(                  \
-      Name("Sum")                           \
-          .Device(DEVICE_SYCL)              \
-          .TypeConstraint<type>("T")        \
-          .TypeConstraint<int32>("Tidx")    \
-          .HostMemory("reduction_indices"), \
-      ReductionOp<SYCLDevice, type, Eigen::internal::SumReducer<type>>);
+#define REGISTER_SYCL_KERNELS(type)                                        \
+  REGISTER_KERNEL_BUILDER(Name("Sum")                                      \
+                              .Device(DEVICE_SYCL)                         \
+                              .TypeConstraint<type>("T")                   \
+                              .TypeConstraint<int32>("Tidx")               \
+                              .HostMemory("reduction_indices"),            \
+                          ReductionOp<SYCLDevice, type, int32,             \
+                                      Eigen::internal::SumReducer<type>>); \
+  REGISTER_KERNEL_BUILDER(Name("Sum")                                      \
+                              .Device(DEVICE_SYCL)                         \
+                              .TypeConstraint<type>("T")                   \
+                              .TypeConstraint<int64>("Tidx")               \
+                              .HostMemory("reduction_indices"),            \
+                          ReductionOp<SYCLDevice, type, int64,             \
+                                      Eigen::internal::SumReducer<type>>);
 REGISTER_SYCL_KERNELS(float);
 REGISTER_SYCL_KERNELS(double);
 
@@ -77,8 +106,17 @@ REGISTER_KERNEL_BUILDER(
         .HostMemory("input")
         .HostMemory("output")
         .HostMemory("reduction_indices"),
-    ReductionOp<CPUDevice, int32, Eigen::internal::SumReducer<int32>>);
+    ReductionOp<CPUDevice, int32, int32, Eigen::internal::SumReducer<int32>>);
+REGISTER_KERNEL_BUILDER(
+    Name("Sum")
+        .Device(DEVICE_SYCL)
+        .TypeConstraint<int32>("T")
+        .TypeConstraint<int64>("Tidx")
+        .HostMemory("input")
+        .HostMemory("output")
+        .HostMemory("reduction_indices"),
+    ReductionOp<CPUDevice, int32, int64, Eigen::internal::SumReducer<int32>>);
 #undef REGISTER_SYCL_KERNELS
-#endif // TENSORFLOW_USE_SYCL
+#endif  // TENSORFLOW_USE_SYCL
 
 }  // namespace tensorflow
diff --git a/tensorflow/python/kernel_tests/reduction_ops_test.py b/tensorflow/python/kernel_tests/reduction_ops_test.py
index c794351fe9..2dc65b1384 100644
--- a/tensorflow/python/kernel_tests/reduction_ops_test.py
+++ b/tensorflow/python/kernel_tests/reduction_ops_test.py
@@ -163,6 +163,13 @@ class SumReductionTest(BaseReductionTest):
       reduction_axes = tuple(reduction_axes)
     return np.sum(x, axis=reduction_axes, keepdims=keep_dims)
 
+  def testAxesType(self):
+    for dtype in [dtypes.int64, dtypes.int32]:
+      with self.test_session(use_gpu=True) as sess:
+        v = math_ops.reduce_sum([0, 0], constant_op.constant(0, dtype=dtype))
+        tf_v = sess.run(v)
+      self.assertAllEqual(tf_v, 0)
+
   def testInfinity(self):
     for dtype in [np.float32, np.float64]:
       for special_value_x in [-np.inf, np.inf]:
@@ -193,6 +200,7 @@ class SumReductionTest(BaseReductionTest):
       tf_out_mean = sess.run(tf_mean)
     self.assertAllClose(tf_out_mean, 1.)
 
+
   def testFloat32(self):
     for rank in range(1, _MAX_RANK + 1):
       np_arr = self._makeIncremental((2,) * rank, dtypes.float32)
@@ -369,6 +377,13 @@ class MeanReductionTest(BaseReductionTest):
       return np_sum // count
     return np_sum / count
 
+  def testAxesType(self):
+    for dtype in [dtypes.int64, dtypes.int32]:
+      with self.test_session(use_gpu=True) as sess:
+        v = math_ops.reduce_mean([0, 0], constant_op.constant(0, dtype=dtype))
+        tf_v = sess.run(v)
+      self.assertAllEqual(tf_v, 0)
+
   def testInfinity(self):
     for dtype in [np.float32, np.float64]:
       for special_value_x in [-np.inf, np.inf]:
@@ -435,6 +450,13 @@ class ProdReductionTest(BaseReductionTest):
       reduction_axes = tuple(reduction_axes)
     return np.prod(x, axis=reduction_axes, keepdims=keep_dims)
 
+  def testAxesType(self):
+    for dtype in [dtypes.int64, dtypes.int32]:
+      with self.test_session(use_gpu=True) as sess:
+        v = math_ops.reduce_prod([0, 0], constant_op.constant(0, dtype=dtype))
+        tf_v = sess.run(v)
+      self.assertAllEqual(tf_v, 0)
+
   def testInfinity(self):
     for dtype in [np.float32, np.float64]:
       for special_value_x in [-np.inf, np.inf]:
@@ -531,6 +553,13 @@ class MinReductionTest(test.TestCase):
     self._compare(x, reduction_axes, True, use_gpu=True)
     self._compare(x, reduction_axes, True, use_gpu=False)
 
+  def testAxesType(self):
+    for dtype in [dtypes.int64, dtypes.int32]:
+      with self.test_session(use_gpu=True) as sess:
+        v = math_ops.reduce_min([0, 0], constant_op.constant(0, dtype=dtype))
+        tf_v = sess.run(v)
+      self.assertAllEqual(tf_v, 0)
+
   def testInfinity(self):
     for dtype in [np.float32, np.float64]:
       for special_value_x in [-np.inf, np.inf]:
@@ -637,6 +666,13 @@ class MaxReductionTest(test.TestCase):
     self._compare(x, reduction_axes, True, use_gpu=True)
     self._compare(x, reduction_axes, True, use_gpu=False)
 
+  def testAxesType(self):
+    for dtype in [dtypes.int64, dtypes.int32]:
+      with self.test_session(use_gpu=True) as sess:
+        v = math_ops.reduce_max([0, 0], constant_op.constant(0, dtype=dtype))
+        tf_v = sess.run(v)
+      self.assertAllEqual(tf_v, 0)
+
   def testInfinity(self):
     for dtype in [np.float32, np.float64]:
       for special_value_x in [-np.inf, np.inf]:
@@ -757,6 +793,14 @@ class AllReductionTest(test.TestCase):
     self._compare(x, reduction_axes, True, use_gpu=True)
     self._compare(x, reduction_axes, True, use_gpu=False)
 
+  def testAxesType(self):
+    for dtype in [dtypes.int64, dtypes.int32]:
+      with self.test_session(use_gpu=True) as sess:
+        v = math_ops.reduce_all([True, True],
+                                constant_op.constant(0, dtype=dtype))
+        tf_v = sess.run(v)
+      self.assertAllEqual(tf_v, True)
+
   def testAll3D(self):
     # Create a 3D array of bools and reduce across all possible
     # dimensions
@@ -798,6 +842,14 @@ class AnyReductionTest(test.TestCase):
     self._compare(x, reduction_axes, True, use_gpu=True)
     self._compare(x, reduction_axes, True, use_gpu=False)
 
+  def testAxesType(self):
+    for dtype in [dtypes.int64, dtypes.int32]:
+      with self.test_session(use_gpu=True) as sess:
+        v = math_ops.reduce_any([True, True],
+                                constant_op.constant(0, dtype=dtype))
+        tf_v = sess.run(v)
+      self.assertAllEqual(tf_v, True)
+
   def testAll3D(self):
     # Create a 3D array of bools and reduce across all possible
     # dimensions
-- 
GitLab


From c9cb5a58d5e174e9870c40328d6be427ccf8be54 Mon Sep 17 00:00:00 2001
From: formath <jinpengliu@163.com>
Date: Sun, 22 Oct 2017 14:02:50 +0800
Subject: [PATCH 657/909] protobuf lib path bug fix for benckmark on osx
 (#13878)

---
 tensorflow/contrib/makefile/Makefile | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tensorflow/contrib/makefile/Makefile b/tensorflow/contrib/makefile/Makefile
index 3dcff3d4a3..e970e50d2e 100644
--- a/tensorflow/contrib/makefile/Makefile
+++ b/tensorflow/contrib/makefile/Makefile
@@ -190,6 +190,10 @@ LIBFLAGS :=
 
 # If we're on OS X, make sure that globals aren't stripped out.
 ifeq ($(TARGET),OSX)
+ifeq ($(HAS_GEN_HOST_PROTOC),true)
+	LIBFLAGS += -L$(MAKEFILE_DIR)/gen/protobuf-host/lib
+	export LD_LIBRARY_PATH=$(MAKEFILE_DIR)/gen/protobuf-host/lib
+endif
 	LDFLAGS += -all_load
 endif
 # Make sure that we don't strip global constructors on Linux.
-- 
GitLab


From bfa4ec194a595f7ed466f2e5af391c81e98786bc Mon Sep 17 00:00:00 2001
From: Tayo Oguntebi <10927929+tayo@users.noreply.github.com>
Date: Sat, 21 Oct 2017 23:08:17 -0700
Subject: [PATCH 658/909] Update node_def.proto comments (#13874)

The device field had outdated comments.

Note: We could consider adding tpu as an example here, e.g. "gpu" | "cpu" | "tpu".  Thoughts?
---
 tensorflow/core/framework/node_def.proto | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/framework/node_def.proto b/tensorflow/core/framework/node_def.proto
index 53aa03108a..1fd2e50b51 100644
--- a/tensorflow/core/framework/node_def.proto
+++ b/tensorflow/core/framework/node_def.proto
@@ -35,7 +35,7 @@ message NodeDef {
   // CONSTRAINT ::= ("job:" JOB_NAME)
   //              | ("replica:" [1-9][0-9]*)
   //              | ("task:" [1-9][0-9]*)
-  //              | ( ("gpu" | "cpu") ":" ([1-9][0-9]* | "*") )
+  //              | ("device:" ("gpu" | "cpu") ":" ([1-9][0-9]* | "*") )
   //
   // Valid values for this string include:
   // * "/job:worker/replica:0/task:1/device:GPU:3"  (full specification)
-- 
GitLab


From 40c475b48c091a70ad8061c1508dff6ded2d2af6 Mon Sep 17 00:00:00 2001
From: formath <jinpengliu@163.com>
Date: Mon, 23 Oct 2017 03:05:08 +0800
Subject: [PATCH 659/909] add segment_reduction_ops to tf_op_files (#13901)

---
 tensorflow/contrib/makefile/tf_op_files.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/contrib/makefile/tf_op_files.txt b/tensorflow/contrib/makefile/tf_op_files.txt
index a8690a04ad..8b77c99cb5 100644
--- a/tensorflow/contrib/makefile/tf_op_files.txt
+++ b/tensorflow/contrib/makefile/tf_op_files.txt
@@ -264,3 +264,4 @@ tensorflow/core/kernels/spacetobatch_functor.cc
 tensorflow/core/kernels/spacetobatch_op.cc
 tensorflow/core/kernels/batchtospace_op.cc
 tensorflow/core/kernels/warn_about_ints.cc
+tensorflow/core/kernels/segment_reduction_ops.cc
-- 
GitLab


From fd8d517b97da8b41ad4088b2fc68080393f26b55 Mon Sep 17 00:00:00 2001
From: Yunxing Dai <yunxing@google.com>
Date: Sun, 22 Oct 2017 16:48:19 -0700
Subject: [PATCH 660/909] Add tests for convolution 1D RELNOTES: n/a

PiperOrigin-RevId: 173060283
---
 .../compiler/xla/tests/convolution_test.cc    | 79 ++++++++++++++-----
 1 file changed, 61 insertions(+), 18 deletions(-)

diff --git a/tensorflow/compiler/xla/tests/convolution_test.cc b/tensorflow/compiler/xla/tests/convolution_test.cc
index a7089c2897..0cc2e5fb7e 100644
--- a/tensorflow/compiler/xla/tests/convolution_test.cc
+++ b/tensorflow/compiler/xla/tests/convolution_test.cc
@@ -508,21 +508,35 @@ XLA_TEST_F(ConvolutionTest, Convolve2D_1x3x3x5_3x3x5x5_Valid) {
                            error_spec_);
 }
 
-XLA_TEST_F(ConvolutionTest, Convolve1D_Valid) {
+struct Convolve1DTestParam {
+  int64 input_feature;
+  int64 output_feature;
+  int64 batch;
+  int64 window_size;
+  int64 num_windows;
+};
+
+class Convolve1D1WindowTest
+    : public ConvolutionTest,
+      public ::testing::WithParamInterface<Convolve1DTestParam> {};
+
+XLA_TEST_P(Convolve1D1WindowTest, Convolve1D1Window) {
   ComputationBuilder builder(client_, TestName());
-  int64 output_feature = 1;
-  int64 input_feature = 64;
-  int64 batch = 1;
-  int64 length = 1;
-  std::vector<int64> input_dims = {batch, 4 + length - 1, input_feature};
-  std::vector<int64> filter_dims = {4, input_feature, output_feature};
+  int64 input_feature = GetParam().input_feature;
+  int64 output_feature = GetParam().output_feature;
+  int64 batch = GetParam().batch;
+  int64 num_windows = GetParam().num_windows;
+  int64 window_size = GetParam().window_size;
+  std::vector<int64> input_dims = {batch, window_size + num_windows - 1,
+                                   input_feature};
+  std::vector<int64> filter_dims = {window_size, input_feature, output_feature};
   Shape input_shape = ShapeUtil::MakeShape(F32, input_dims);
   Shape filter_shape = ShapeUtil::MakeShape(F32, filter_dims);
   {
     auto input = builder.Parameter(0, input_shape, "input");
     auto filter = builder.Parameter(1, filter_shape, "filter");
 
-    // Tensorflow dimension numbers for 2D convolution.
+    // Tensorflow dimension numbers for 1D convolution.
     ConvolutionDimensionNumbers dnums;
     dnums.set_input_batch_dimension(0);
     dnums.set_output_batch_dimension(0);
@@ -538,28 +552,57 @@ XLA_TEST_F(ConvolutionTest, Convolve1D_Valid) {
   }
 
   std::vector<float> input_elems(ShapeUtil::ElementsIn(input_shape), 1.0);
-  // std::iota(input_elems.begin(), input_elems.end(), 1.0f);
   auto input_r1 = Literal::CreateR1<float>(input_elems);
-  auto input_r4 = input_r1->Reshape(input_dims).ConsumeValueOrDie();
+  auto input_r3 = input_r1->Reshape(input_dims).ConsumeValueOrDie();
 
   std::vector<float> filter_elems(ShapeUtil::ElementsIn(filter_shape), 1.0);
-  // std::iota(filter_elems.begin(), filter_elems.end(), 1.0f);
 
   auto filter_r1 = Literal::CreateR1<float>(filter_elems);
-  auto filter_r4 = filter_r1->Reshape(filter_dims).ConsumeValueOrDie();
+  auto filter_r3 = filter_r1->Reshape(filter_dims).ConsumeValueOrDie();
 
-  std::vector<float> expect_elems(batch * output_feature * length, 256);
+  std::vector<float> expect_elems(batch * output_feature * num_windows,
+                                  window_size * input_feature);
   auto expected_r1 = Literal::CreateR1<float>(expect_elems);
-  auto expected_r4 =
-      expected_r1->Reshape({batch, length, output_feature}).ConsumeValueOrDie();
+  auto expected_r3 = expected_r1->Reshape({batch, num_windows, output_feature})
+                         .ConsumeValueOrDie();
 
-  auto input_literal = client_->TransferToServer(*input_r4).ConsumeValueOrDie();
+  auto input_literal = client_->TransferToServer(*input_r3).ConsumeValueOrDie();
   auto filter_literal =
-      client_->TransferToServer(*filter_r4).ConsumeValueOrDie();
-  ComputeAndCompareLiteral(&builder, *expected_r4,
+      client_->TransferToServer(*filter_r3).ConsumeValueOrDie();
+  ComputeAndCompareLiteral(&builder, *expected_r3,
                            {input_literal.get(), filter_literal.get()},
                            error_spec_);
 }
 
+INSTANTIATE_TEST_CASE_P(
+    Convolve1D1WindowTest_Instantiation, Convolve1D1WindowTest,
+    ::testing::Values(Convolve1DTestParam{1, 1, 1, 1, 2},
+                      Convolve1DTestParam{160, 1, 1, 5, 1},
+                      Convolve1DTestParam{24, 1, 1, 20, 1},
+                      Convolve1DTestParam{30, 1, 1, 20, 1},
+                      Convolve1DTestParam{23, 1, 1, 20, 20},
+                      Convolve1DTestParam{25, 1, 1, 20, 1},
+                      Convolve1DTestParam{24, 1, 1, 10, 5},
+                      Convolve1DTestParam{160, 1, 1, 10, 1},
+                      Convolve1DTestParam{255, 1, 1, 3, 1},
+                      Convolve1DTestParam{130, 1, 1, 1, 3},
+                      Convolve1DTestParam{64, 1, 1, 1, 1},
+                      Convolve1DTestParam{128, 1, 1, 1, 1},
+                      Convolve1DTestParam{139, 1, 1, 128, 1},
+                      Convolve1DTestParam{1, 10, 10, 1, 10},
+                      Convolve1DTestParam{1, 10, 130, 1, 2},
+                      Convolve1DTestParam{1, 10, 130, 1, 1},
+                      Convolve1DTestParam{1, 64, 64, 1, 10},
+                      Convolve1DTestParam{1, 65, 65, 1, 1},
+                      Convolve1DTestParam{1, 128, 128, 1, 1},
+                      Convolve1DTestParam{128, 128, 128, 128, 1},
+                      Convolve1DTestParam{1, 128, 128, 1, 1},
+                      Convolve1DTestParam{2, 2, 2, 2, 1},
+                      Convolve1DTestParam{161, 1, 1, 10, 1},
+                      Convolve1DTestParam{900, 1, 1, 10, 1},
+                      Convolve1DTestParam{640, 3, 3, 128, 1})
+
+);
+
 }  // namespace
 }  // namespace xla
-- 
GitLab


From 690003cc015d6d56630d5836adb6769729bd9c3d Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Sun, 22 Oct 2017 22:33:18 -0700
Subject: [PATCH 661/909] Add `int64` type `multiples` support for `tf.tile`
 (#13884)

* Add `int64` type `multiples` support for `tf.tile`

In the doc of `tf.tile` (tf.tile.__doc__) both `int32`
and `int64` are supported for `multiples`. However, the kernel
for `int64` is not registered yet.

This fix adds the support of `int64` `multiples` so that the
behavior matches the description of the docs.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Update functors for int64 multiples support in `tf.tile`

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Update test cases for int64 of multiples in `tf.tile`

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Add GPU and non GPU tests

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* format with clang-format -i

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Move Tmultiples after T (as it is  auxilliary)

And use `use_gpu=True`

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/core/kernels/tile_functor.h        |  39 +--
 tensorflow/core/kernels/tile_functor_cpu.cc   |  12 +-
 .../core/kernels/tile_functor_gpu.cu.cc       |  12 +-
 tensorflow/core/kernels/tile_ops.cc           | 249 +++++++-----------
 .../python/kernel_tests/shape_ops_test.py     |  18 +-
 5 files changed, 149 insertions(+), 181 deletions(-)

diff --git a/tensorflow/core/kernels/tile_functor.h b/tensorflow/core/kernels/tile_functor.h
index 28af2dace3..189be9239b 100644
--- a/tensorflow/core/kernels/tile_functor.h
+++ b/tensorflow/core/kernels/tile_functor.h
@@ -17,6 +17,7 @@ limitations under the License.
 #define TENSORFLOW_KERNELS_TILE_FUNCTOR_H_
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/platform/types.h"
@@ -29,13 +30,13 @@ namespace internal {
 template <typename Device, typename T>
 void TileSimple(const Device& d, Tensor* out, const Tensor& in);
 
-template <typename Device, typename T, int NDIM>
+template <typename Device, typename T, typename Tmultiples, int NDIM>
 void TileUsingEigen(const Device& d, Tensor* out, const Tensor& in,
-                    const gtl::ArraySlice<int32>& broadcast_array) {
+                    const gtl::ArraySlice<Tmultiples>& broadcast_array) {
   auto x = in.tensor<T, NDIM>();
   auto y = out->tensor<T, NDIM>();
 
-  Eigen::array<int32, NDIM> b;
+  Eigen::array<Tmultiples, NDIM> b;
   for (int i = 0; i < NDIM; ++i) b[i] = broadcast_array[i];
   if (Eigen::internal::is_same<Device, Eigen::GpuDevice>::value) {
     // Use 32bit indexing to speed up the computations
@@ -45,9 +46,9 @@ void TileUsingEigen(const Device& d, Tensor* out, const Tensor& in,
   }
 }
 
-template <typename Device, typename T>
+template <typename Device, typename T, typename Tmultiples>
 void TileUsingEigen(const Device& d, Tensor* out, const Tensor& in,
-                    const gtl::ArraySlice<int32>&) {
+                    const gtl::ArraySlice<Tmultiples>&) {
   auto x = in.tensor<T, 0>();
   auto y = out->tensor<T, 0>();
   // In the scalar case we simply copy the input.
@@ -58,34 +59,42 @@ void TileUsingEigen(const Device& d, Tensor* out, const Tensor& in,
 
 namespace functor {
 
-template <typename Device, typename T>
+template <typename Device, typename T, typename Tmultiples>
 struct Tile {
   void operator()(const Device& d, Tensor* out, const Tensor& in,
-                  const gtl::ArraySlice<int32> broadcast_array) const {
+                  const gtl::ArraySlice<Tmultiples> broadcast_array) const {
     switch (in.dims()) {
       case 0:
-        internal::TileUsingEigen<Device, T>(d, out, in, broadcast_array);
+        internal::TileUsingEigen<Device, T, Tmultiples>(d, out, in,
+                                                        broadcast_array);
         break;
       case 1:
-        internal::TileUsingEigen<Device, T, 1>(d, out, in, broadcast_array);
+        internal::TileUsingEigen<Device, T, Tmultiples, 1>(d, out, in,
+                                                           broadcast_array);
         break;
       case 2:
-        internal::TileUsingEigen<Device, T, 2>(d, out, in, broadcast_array);
+        internal::TileUsingEigen<Device, T, Tmultiples, 2>(d, out, in,
+                                                           broadcast_array);
         break;
       case 3:
-        internal::TileUsingEigen<Device, T, 3>(d, out, in, broadcast_array);
+        internal::TileUsingEigen<Device, T, Tmultiples, 3>(d, out, in,
+                                                           broadcast_array);
         break;
       case 4:
-        internal::TileUsingEigen<Device, T, 4>(d, out, in, broadcast_array);
+        internal::TileUsingEigen<Device, T, Tmultiples, 4>(d, out, in,
+                                                           broadcast_array);
         break;
       case 5:
-        internal::TileUsingEigen<Device, T, 5>(d, out, in, broadcast_array);
+        internal::TileUsingEigen<Device, T, Tmultiples, 5>(d, out, in,
+                                                           broadcast_array);
         break;
       case 6:
-        internal::TileUsingEigen<Device, T, 6>(d, out, in, broadcast_array);
+        internal::TileUsingEigen<Device, T, Tmultiples, 6>(d, out, in,
+                                                           broadcast_array);
         break;
       case 7:
-        internal::TileUsingEigen<Device, T, 7>(d, out, in, broadcast_array);
+        internal::TileUsingEigen<Device, T, Tmultiples, 7>(d, out, in,
+                                                           broadcast_array);
         break;
       default:
         internal::TileSimple<Device, T>(d, out, in);
diff --git a/tensorflow/core/kernels/tile_functor_cpu.cc b/tensorflow/core/kernels/tile_functor_cpu.cc
index 5952d49221..b2fd669541 100644
--- a/tensorflow/core/kernels/tile_functor_cpu.cc
+++ b/tensorflow/core/kernels/tile_functor_cpu.cc
@@ -15,10 +15,10 @@ limitations under the License.
 
 #define EIGEN_USE_THREADS
 
+#include "tensorflow/core/kernels/tile_functor.h"
 #include "tensorflow/core/framework/attr_value.pb.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/kernels/ops_util.h"
-#include "tensorflow/core/kernels/tile_functor.h"
 
 namespace tensorflow {
 
@@ -51,7 +51,9 @@ namespace functor {
 typedef Eigen::ThreadPoolDevice CPUDevice;
 
 // Register functors used for Tile functor.
-#define DEFINE_TYPE(T) template struct Tile<CPUDevice, T>;
+#define DEFINE_TYPE(T)                       \
+  template struct Tile<CPUDevice, T, int32>; \
+  template struct Tile<CPUDevice, T, int64>;
 
 TF_CALL_bool(DEFINE_TYPE);
 TF_CALL_float(DEFINE_TYPE);
@@ -70,7 +72,9 @@ TF_CALL_string(DEFINE_TYPE);
 #ifdef TENSORFLOW_USE_SYCL
 typedef Eigen::SyclDevice SYCLDevice;
 
-#define DEFINE_TYPE(T) template struct Tile<SYCLDevice, T>;
+#define DEFINE_TYPE(T)                        \
+  template struct Tile<SYCLDevice, T, int32>; \
+  template struct Tile<SYCLDevice, T, int64>;
 
 TF_CALL_bool(DEFINE_TYPE);
 TF_CALL_float(DEFINE_TYPE);
@@ -81,7 +85,7 @@ TF_CALL_int16(DEFINE_TYPE);
 TF_CALL_int64(DEFINE_TYPE);
 
 #undef DEFINE_TYPE
-#endif // TENSORFLOW_USE_SYCL
+#endif  // TENSORFLOW_USE_SYCL
 
 }  // end namespace functor
 }  // end namespace tensorflow
diff --git a/tensorflow/core/kernels/tile_functor_gpu.cu.cc b/tensorflow/core/kernels/tile_functor_gpu.cu.cc
index 1c61c3030a..5a36e7567b 100644
--- a/tensorflow/core/kernels/tile_functor_gpu.cu.cc
+++ b/tensorflow/core/kernels/tile_functor_gpu.cu.cc
@@ -18,10 +18,11 @@ limitations under the License.
 #define EIGEN_USE_GPU
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
-#include "tensorflow/core/kernels/tile_functor.h"
+
+#include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/kernels/ops_util.h"
+#include "tensorflow/core/kernels/tile_functor.h"
 #include "tensorflow/core/util/cuda_kernel_helper.h"
-#include "tensorflow/core/framework/register_types.h"
 
 namespace tensorflow {
 namespace internal {
@@ -60,7 +61,8 @@ void TileSimple(const Device& d, Tensor* out, const Tensor& in) {
     host_buf[ndims + i] = out_strides[i];
     host_buf[ndims * 2 + i] = in.dim_size(i);
   }
-  // Copies the input strides, output strides and input dimension sizes to the device.
+  // Copies the input strides, output strides and input dimension sizes to the
+  // device.
   auto num_bytes = sizeof(int64) * host_buf.size();
   auto dev_buf = d.allocate(num_bytes);
   // NOTE: host_buf is not allocated by CudaHostAllocator, and
@@ -84,7 +86,9 @@ namespace functor {
 typedef Eigen::GpuDevice GPUDevice;
 
 // Register functors used for Tile functor.
-#define DEFINE_TYPE(T) template struct Tile<GPUDevice, T>;
+#define DEFINE_TYPE(T)                       \
+  template struct Tile<GPUDevice, T, int32>; \
+  template struct Tile<GPUDevice, T, int64>;
 
 TF_CALL_int16(DEFINE_TYPE);
 TF_CALL_int32(DEFINE_TYPE);
diff --git a/tensorflow/core/kernels/tile_ops.cc b/tensorflow/core/kernels/tile_ops.cc
index c49ebc0685..4c496a12c2 100644
--- a/tensorflow/core/kernels/tile_ops.cc
+++ b/tensorflow/core/kernels/tile_ops.cc
@@ -42,14 +42,14 @@ typedef Eigen::ThreadPoolDevice CPUDevice;
 typedef Eigen::GpuDevice GPUDevice;
 #ifdef TENSORFLOW_USE_SYCL
 typedef Eigen::SyclDevice SYCLDevice;
-#endif // TENSORFLOW_USE_SYCL
+#endif  // TENSORFLOW_USE_SYCL
 
 // Forward declarations of functors that will be defined in tile_ops_impl.h
 namespace functor {
-template <typename Device, typename T>
+template <typename Device, typename T, typename Tmultiple>
 struct Tile {
   void operator()(const Device& d, Tensor* out, const Tensor& in,
-                  const gtl::ArraySlice<int32> broadcast_array) const;
+                  const gtl::ArraySlice<Tmultiple> broadcast_array) const;
 };
 
 template <typename Device, typename T, int NDIM>
@@ -80,7 +80,7 @@ struct ReduceAndReshape {
 }  // namespace functor
 
 // --------------------------------------------------------------------------
-template <typename Device>
+template <typename Device, typename Tmultiples>
 class TileOp : public OpKernel {
  public:
   explicit TileOp(OpKernelConstruction* context) : OpKernel(context) {}
@@ -105,8 +105,8 @@ class TileOp : public OpKernel {
       return;
     }
 
-    const gtl::ArraySlice<int32> multiples_array(multiples.flat<int32>().data(),
-                                                 input_dims);
+    const gtl::ArraySlice<Tmultiples> multiples_array(
+        multiples.flat<Tmultiples>().data(), input_dims);
     TensorShape output_shape;
     for (int i = 0; i < input_dims; ++i) {
       OP_REQUIRES(
@@ -125,10 +125,10 @@ class TileOp : public OpKernel {
     // If there's no output, there's nothing to do.
     if (output_shape.num_elements() == 0) return;
 
-#define HANDLE_TYPE(DT)                                        \
-  if (context->input(0).dtype() == DT) {                       \
-    HandleCase<DT>(context, multiples_array, result);          \
-    return;                                                    \
+#define HANDLE_TYPE(DT)                               \
+  if (context->input(0).dtype() == DT) {              \
+    HandleCase<DT>(context, multiples_array, result); \
+    return;                                           \
   }
 
 #define HANDLE_TYPE_NAME(T) HANDLE_TYPE(DataTypeToEnum<T>::value)
@@ -158,27 +158,27 @@ class TileOp : public OpKernel {
  private:
   template <DataType DT>
   void HandleCaseImpl(OpKernelContext* context,
-                      const gtl::ArraySlice<int32>& multiples_array,
+                      const gtl::ArraySlice<Tmultiples>& multiples_array,
                       Tensor* result) {
     typedef typename EnumToDataType<DT>::Type T;
-    functor::Tile<Device, T>() (
-        context->eigen_device<Device>(), result,
-        context->input(0), multiples_array);
+    functor::Tile<Device, T, Tmultiples>()(context->eigen_device<Device>(),
+                                           result, context->input(0),
+                                           multiples_array);
   }
 
   template <DataType DT>
   void HandleCase(OpKernelContext* context,
-                  const gtl::ArraySlice<int32>& multiples_array,
+                  const gtl::ArraySlice<Tmultiples>& multiples_array,
                   Tensor* result);
 
   TF_DISALLOW_COPY_AND_ASSIGN(TileOp);
 };
 
-template <typename Device>
+template <typename Device, typename Tmultiples>
 template <DataType DT>
-inline void TileOp<Device>::HandleCase(
-    OpKernelContext* context, const gtl::ArraySlice<int32>& multiples_array,
-    Tensor* result) {
+inline void TileOp<Device, Tmultiples>::HandleCase(
+    OpKernelContext* context,
+    const gtl::ArraySlice<Tmultiples>& multiples_array, Tensor* result) {
   // TODO(vrv): print out the device name if useful. Currently disabled to avoid
   // having to use RTTI.
   LOG(FATAL) << "TileOp: Invalid combination of Device, DT: "
@@ -186,25 +186,28 @@ inline void TileOp<Device>::HandleCase(
              << DataTypeString(DT);
 }
 
-#define HANDLE_CASE(device, dtype)                                     \
-  template <>                                                          \
-  template <>                                                          \
-  void TileOp<device>::HandleCase<dtype>(                              \
-      OpKernelContext * context,                                       \
-      const gtl::ArraySlice<int32>& multiples_array, Tensor* result) { \
-    HandleCaseImpl<dtype>(context, multiples_array, result);           \
+#define HANDLE_CASE(device, dtype, Tmultiples)                              \
+  template <>                                                               \
+  template <>                                                               \
+  void TileOp<device, Tmultiples>::HandleCase<dtype>(                       \
+      OpKernelContext * context,                                            \
+      const gtl::ArraySlice<Tmultiples>& multiples_array, Tensor* result) { \
+    HandleCaseImpl<dtype>(context, multiples_array, result);                \
   }
 
-#define HANDLE_TYPE_NAME_CPU(T) \
-  HANDLE_CASE(CPUDevice, DataTypeToEnum<T>::value);
+#define HANDLE_TYPE_NAME_CPU(T)                            \
+  HANDLE_CASE(CPUDevice, DataTypeToEnum<T>::value, int32); \
+  HANDLE_CASE(CPUDevice, DataTypeToEnum<T>::value, int64);
 
-#define HANDLE_TYPE_NAME_GPU(T) \
-  HANDLE_CASE(GPUDevice, DataTypeToEnum<T>::value);
+#define HANDLE_TYPE_NAME_GPU(T)                            \
+  HANDLE_CASE(GPUDevice, DataTypeToEnum<T>::value, int32); \
+  HANDLE_CASE(GPUDevice, DataTypeToEnum<T>::value, int64);
 
 #ifdef TENSORFLOW_USE_SYCL
-#define HANDLE_TYPE_NAME_SYCL(T) \
-  HANDLE_CASE(SYCLDevice, DataTypeToEnum<T>::value);
-#endif // TENSORFLOW_USE_SYCL
+#define HANDLE_TYPE_NAME_SYCL(T)                            \
+  HANDLE_CASE(SYCLDevice, DataTypeToEnum<T>::value, int32); \
+  HANDLE_CASE(SYCLDevice, DataTypeToEnum<T>::value, int64);
+#endif  // TENSORFLOW_USE_SYCL
 
 TF_CALL_bool(HANDLE_TYPE_NAME_CPU);
 TF_CALL_float(HANDLE_TYPE_NAME_CPU);
@@ -235,13 +238,13 @@ TF_CALL_double(HANDLE_TYPE_NAME_SYCL);
 TF_CALL_int16(HANDLE_TYPE_NAME_SYCL);
 TF_CALL_int32(HANDLE_TYPE_NAME_SYCL);
 TF_CALL_int64(HANDLE_TYPE_NAME_SYCL);
-#endif // TENSORFLOW_USE_SYCL
+#endif  // TENSORFLOW_USE_SYCL
 
 #undef HANDLE_TYPE_NAME_CPU
 #undef HANDLE_TYPE_NAME_GPU
 #ifdef TENSORFLOW_USE_SYCL
 #undef HANDLE_TYPE_NAME_SYCL
-#endif // TENSORFLOW_USE_SYCL
+#endif  // TENSORFLOW_USE_SYCL
 #undef HANDLE_CASE
 
 // --------------------------------------------------------------------------
@@ -494,7 +497,7 @@ TF_CALL_int16(HANDLE_TYPE_NAME_SYCL);
 TF_CALL_int32(HANDLE_TYPE_NAME_SYCL);
 TF_CALL_int64(HANDLE_TYPE_NAME_SYCL);
 #undef HANDLE_TYPE_NAME_SYCL
-#endif // TENSORFLOW_USE_SYCL
+#endif  // TENSORFLOW_USE_SYCL
 
 #undef HANDLE_TYPE_NAME_CPU
 #undef HANDLE_TYPE_NAME_GPU
@@ -505,127 +508,73 @@ REGISTER_KERNEL_BUILDER(Name("Tile")
                             .Device(DEVICE_CPU)
                             .HostMemory("multiples")
                             .TypeConstraint<int32>("Tmultiples"),
-                        TileOp<CPUDevice>);
+                        TileOp<CPUDevice, int32>);
+REGISTER_KERNEL_BUILDER(Name("Tile")
+                            .Device(DEVICE_CPU)
+                            .HostMemory("multiples")
+                            .TypeConstraint<int64>("Tmultiples"),
+                        TileOp<CPUDevice, int64>);
 REGISTER_KERNEL_BUILDER(
     Name("TileGrad").Device(DEVICE_CPU).HostMemory("multiples"),
     TileGradientOp<CPUDevice>);
 
 #if GOOGLE_CUDA
-
-REGISTER_KERNEL_BUILDER(Name("Tile")
-                            .Device(DEVICE_GPU)
-                            .TypeConstraint<float>("T")
-                            .TypeConstraint<int32>("Tmultiples")
-                            .HostMemory("multiples"),
-                        TileOp<GPUDevice>);
-REGISTER_KERNEL_BUILDER(Name("Tile")
-                            .Device(DEVICE_GPU)
-                            .TypeConstraint<double>("T")
-                            .TypeConstraint<int32>("Tmultiples")
-                            .HostMemory("multiples"),
-                        TileOp<GPUDevice>);
-REGISTER_KERNEL_BUILDER(Name("Tile")
-                            .Device(DEVICE_GPU)
-                            .TypeConstraint<Eigen::half>("T")
-                            .TypeConstraint<int32>("Tmultiples")
-                            .HostMemory("multiples"),
-                        TileOp<GPUDevice>);
-REGISTER_KERNEL_BUILDER(Name("Tile")
-                            .Device(DEVICE_GPU)
-                            .TypeConstraint<int16>("T")
-                            .TypeConstraint<int32>("Tmultiples")
-                            .HostMemory("multiples"),
-                        TileOp<GPUDevice>);
-REGISTER_KERNEL_BUILDER(Name("Tile")
-                            .Device(DEVICE_GPU)
-                            .TypeConstraint<int32>("T")
-                            .TypeConstraint<int32>("Tmultiples")
-                            .HostMemory("multiples"),
-                        TileOp<GPUDevice>);
-REGISTER_KERNEL_BUILDER(Name("Tile")
-                            .Device(DEVICE_GPU)
-                            .TypeConstraint<complex64>("T")
-                            .TypeConstraint<int32>("Tmultiples")
-                            .HostMemory("multiples"),
-                        TileOp<GPUDevice>);
-REGISTER_KERNEL_BUILDER(Name("Tile")
-                            .Device(DEVICE_GPU)
-                            .TypeConstraint<complex128>("T")
-                            .TypeConstraint<int32>("Tmultiples")
-                            .HostMemory("multiples"),
-                        TileOp<GPUDevice>);
-
-REGISTER_KERNEL_BUILDER(Name("TileGrad")
-                            .Device(DEVICE_GPU)
-                            .TypeConstraint<float>("T")
-                            .TypeConstraint<int32>("Tmultiples")
-                            .HostMemory("multiples"),
-                        TileGradientOp<GPUDevice>);
-REGISTER_KERNEL_BUILDER(Name("TileGrad")
-                            .Device(DEVICE_GPU)
-                            .TypeConstraint<double>("T")
-                            .TypeConstraint<int32>("Tmultiples")
-                            .HostMemory("multiples"),
-                        TileGradientOp<GPUDevice>);
-REGISTER_KERNEL_BUILDER(Name("TileGrad")
-                            .Device(DEVICE_GPU)
-                            .TypeConstraint<Eigen::half>("T")
-                            .TypeConstraint<int32>("Tmultiples")
-                            .HostMemory("multiples"),
-                        TileGradientOp<GPUDevice>);
-REGISTER_KERNEL_BUILDER(Name("TileGrad")
-                            .Device(DEVICE_GPU)
-                            .TypeConstraint<int16>("T")
-                            .TypeConstraint<int32>("Tmultiples")
-                            .HostMemory("multiples"),
-                        TileGradientOp<GPUDevice>);
-REGISTER_KERNEL_BUILDER(Name("TileGrad")
-                            .Device(DEVICE_GPU)
-                            .TypeConstraint<int32>("T")
-                            .TypeConstraint<int32>("Tmultiples")
-                            .HostMemory("multiples"),
-                        TileGradientOp<GPUDevice>);
-REGISTER_KERNEL_BUILDER(Name("TileGrad")
-                            .Device(DEVICE_GPU)
-                            .TypeConstraint<complex64>("T")
-                            .TypeConstraint<int32>("Tmultiples")
-                            .HostMemory("multiples"),
-                        TileGradientOp<GPUDevice>);
-REGISTER_KERNEL_BUILDER(Name("TileGrad")
-                            .Device(DEVICE_GPU)
-                            .TypeConstraint<complex128>("T")
-                            .TypeConstraint<int32>("Tmultiples")
-                            .HostMemory("multiples"),
-                        TileGradientOp<GPUDevice>);
-
+#define REGISTER_GPU(type)                                         \
+  REGISTER_KERNEL_BUILDER(Name("Tile")                             \
+                              .Device(DEVICE_GPU)                  \
+                              .TypeConstraint<type>("T")           \
+                              .TypeConstraint<int32>("Tmultiples") \
+                              .HostMemory("multiples"),            \
+                          TileOp<GPUDevice, int32>);               \
+  REGISTER_KERNEL_BUILDER(Name("Tile")                             \
+                              .Device(DEVICE_GPU)                  \
+                              .TypeConstraint<type>("T")           \
+                              .TypeConstraint<int64>("Tmultiples") \
+                              .HostMemory("multiples"),            \
+                          TileOp<GPUDevice, int64>);               \
+  REGISTER_KERNEL_BUILDER(Name("TileGrad")                         \
+                              .Device(DEVICE_GPU)                  \
+                              .TypeConstraint<type>("T")           \
+                              .TypeConstraint<int32>("Tmultiples") \
+                              .HostMemory("multiples"),            \
+                          TileGradientOp<GPUDevice>);
+
+TF_CALL_float(REGISTER_GPU);
+TF_CALL_double(REGISTER_GPU);
+TF_CALL_half(REGISTER_GPU);
+TF_CALL_int16(REGISTER_GPU);
+TF_CALL_int32(REGISTER_GPU);
+TF_CALL_complex64(REGISTER_GPU);
+TF_CALL_complex128(REGISTER_GPU)
+
+#undef REGISTER_GPU
 #endif  // GOOGLE_CUDA
 
 #ifdef TENSORFLOW_USE_SYCL
-REGISTER_KERNEL_BUILDER(Name("Tile")
-                            .Device(DEVICE_SYCL)
-                            .TypeConstraint<float>("T")
-                            .TypeConstraint<int32>("Tmultiples")
-                            .HostMemory("multiples"),
-                        TileOp<SYCLDevice>);
-REGISTER_KERNEL_BUILDER(Name("Tile")
-                            .Device(DEVICE_SYCL)
-                            .TypeConstraint<double>("T")
-                            .TypeConstraint<int32>("Tmultiples")
-                            .HostMemory("multiples"),
-                        TileOp<SYCLDevice>);
-
-REGISTER_KERNEL_BUILDER(Name("TileGrad")
-                            .Device(DEVICE_SYCL)
-                            .TypeConstraint<float>("T")
-                            .TypeConstraint<int32>("Tmultiples")
-                            .HostMemory("multiples"),
-                        TileGradientOp<SYCLDevice>);
-REGISTER_KERNEL_BUILDER(Name("TileGrad")
-                            .Device(DEVICE_SYCL)
-                            .TypeConstraint<double>("T")
-                            .TypeConstraint<int32>("Tmultiples")
-                            .HostMemory("multiples"),
-                        TileGradientOp<SYCLDevice>);
-#endif // TENSORFLOW_USE_SYCL
+#define REGISTER_SYCL(type)                                        \
+  REGISTER_KERNEL_BUILDER(Name("Tile")                             \
+                              .Device(DEVICE_SYCL)                 \
+                              .TypeConstraint<type>("T")           \
+                              .TypeConstraint<int32>("Tmultiples") \
+                              .HostMemory("multiples"),            \
+                          TileOp<SYCLDevice, int32>);              \
+  REGISTER_KERNEL_BUILDER(Name("Tile")                             \
+                              .Device(DEVICE_SYCL)                 \
+                              .TypeConstraint<type>("T")           \
+                              .TypeConstraint<int64>("Tmultiples") \
+                              .HostMemory("multiples"),            \
+                          TileOp<SYCLDevice, int64>);              \
+  REGISTER_KERNEL_BUILDER(Name("TileGrad")                         \
+                              .Device(DEVICE_SYCL)                 \
+                              .TypeConstraint<type>("T")           \
+                              .TypeConstraint<int32>("Tmultiples") \
+                              .HostMemory("multiples"),            \
+                          TileGradientOp<SYCLDevice>);
+
+    TF_CALL_float(REGISTER_SYCL);
+TF_CALL_double(REGISTER_SYCL);
+
+#undef REGISTER_SYCL
+#endif  // TENSORFLOW_USE_SYCL
 
 }  // namespace tensorflow
diff --git a/tensorflow/python/kernel_tests/shape_ops_test.py b/tensorflow/python/kernel_tests/shape_ops_test.py
index 52cf904528..a9fc699b21 100644
--- a/tensorflow/python/kernel_tests/shape_ops_test.py
+++ b/tensorflow/python/kernel_tests/shape_ops_test.py
@@ -411,14 +411,16 @@ class TileTest(test.TestCase):
       self.assertEqual(7, result)
 
   def testSimple(self):
-    with self.test_session():
-      inp = np.random.rand(4, 1).astype(np.float32)
-      a = constant_op.constant(inp)
-      tiled = array_ops.tile(a, [1, 4])
-      result = tiled.eval()
-    self.assertEqual(result.shape, (4, 4))
-    self.assertEqual([4, 4], tiled.get_shape())
-    self.assertTrue((result == np.tile(inp, (1, 4))).all())
+    # multiples could be int32 or int64
+    for dtype in [dtypes.int32, dtypes.int64]:
+      with self.test_session(use_gpu=True):
+        inp = np.random.rand(4, 1).astype(np.float32)
+        a = constant_op.constant(inp)
+        tiled = array_ops.tile(a, constant_op.constant([1, 4], dtype=dtype))
+        result = tiled.eval()
+      self.assertEqual(result.shape, (4, 4))
+      self.assertEqual([4, 4], tiled.get_shape())
+      self.assertTrue((result == np.tile(inp, (1, 4))).all())
 
   def testIdentityTileAndGrad(self):
     with self.test_session():
-- 
GitLab


From 0d437c3beb14c08b5b9c08d806de91d7f3d2c0e3 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Sun, 22 Oct 2017 22:50:07 -0700
Subject: [PATCH 662/909] Add int64 padding support for MirrorPad (#13907)

* Add int64 padding support for MirrorPad

This fix adds int64 padding support for `MirrorPad`.
In the `array_ops.cc` the `MirrorPad`/`MirrorPadGrad`
has been specified as supporting int64 padding. The related
kernels does not have the int64 padding registered though.
This fix adds the int64 padding support. This fix also adds
additional test cases for coverage.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Update template for CPU and GPU support of int64 paddings.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Add int64 padding support for MirrorPad

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Put eigen header first like before, just in case.
---
 tensorflow/core/kernels/mirror_pad_op.cc      | 200 +++++++++++-------
 tensorflow/core/kernels/mirror_pad_op.h       |  13 +-
 .../core/kernels/mirror_pad_op_cpu_impl.h     |  12 +-
 .../core/kernels/mirror_pad_op_gpu.cu.cc      |  32 ++-
 tensorflow/python/kernel_tests/pad_op_test.py |  19 ++
 5 files changed, 177 insertions(+), 99 deletions(-)

diff --git a/tensorflow/core/kernels/mirror_pad_op.cc b/tensorflow/core/kernels/mirror_pad_op.cc
index e3643f9447..fbdeaf43eb 100644
--- a/tensorflow/core/kernels/mirror_pad_op.cc
+++ b/tensorflow/core/kernels/mirror_pad_op.cc
@@ -18,10 +18,10 @@ limitations under the License.
 #define EIGEN_USE_THREADS
 
 #include "tensorflow/core/kernels/mirror_pad_op.h"
-
 #include <string>
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+
 #include "tensorflow/core/framework/op.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
@@ -35,7 +35,7 @@ limitations under the License.
 
 namespace tensorflow {
 
-template <typename Device, typename T>
+template <typename Device, typename T, typename Tpaddings>
 class MirrorPadOp : public OpKernel {
  public:
   explicit MirrorPadOp(OpKernelConstruction* context) : OpKernel(context) {
@@ -82,10 +82,10 @@ class MirrorPadOp : public OpKernel {
 
     // Compute the shape of the output tensor, and allocate it.
     TensorShape output_shape;
-    TTypes<int32>::ConstMatrix paddings = in1.matrix<int32>();
+    typename TTypes<Tpaddings>::ConstMatrix paddings = in1.matrix<Tpaddings>();
     for (int d = 0; d < dims; ++d) {
-      const int32 before = paddings(d, 0);  // Pad before existing elements.
-      const int32 after = paddings(d, 1);   // Pad after existing elements.
+      const Tpaddings before = paddings(d, 0);  // Pad before existing elements.
+      const Tpaddings after = paddings(d, 1);   // Pad after existing elements.
       OP_REQUIRES(context, before >= 0 && after >= 0,
                   errors::InvalidArgument("paddings must be non-negative: ",
                                           before, " ", after));
@@ -121,7 +121,7 @@ class MirrorPadOp : public OpKernel {
 
 #define MIRROR_PAD_CASE(i)                                                \
   case i: {                                                               \
-    functor::MirrorPad<Device, T, i>()(                                   \
+    functor::MirrorPad<Device, T, Tpaddings, i>()(                        \
         context->eigen_device<Device>(), To32Bit(output->tensor<T, i>()), \
         To32Bit(in0.tensor<T, i>()), paddings, offset_);                  \
     break;                                                                \
@@ -152,20 +152,25 @@ using GpuDevice = Eigen::GpuDevice;
 namespace functor {
 // Forward declarations of the functor specializations defined in the sharded
 // files.
-#define DECLARE_CPU_SPEC(T, i)                                               \
-  template <>                                                                \
-  void MirrorPad<CpuDevice, T, i>::operator()(                               \
-      const CpuDevice&, typename TTypes<T, i, int32>::Tensor,                \
-      typename TTypes<T, i, int32>::ConstTensor, TTypes<int32>::ConstMatrix, \
-      int);                                                                  \
-  extern template struct MirrorPad<CpuDevice, T, i>;
-
-#define DECLARE_CPU_SPECS(T) \
-  DECLARE_CPU_SPEC(T, 1);    \
-  DECLARE_CPU_SPEC(T, 2);    \
-  DECLARE_CPU_SPEC(T, 3);    \
-  DECLARE_CPU_SPEC(T, 4);    \
-  DECLARE_CPU_SPEC(T, 5);
+#define DECLARE_CPU_SPEC(T, Tpaddings, i)                     \
+  template <>                                                 \
+  void MirrorPad<CpuDevice, T, Tpaddings, i>::operator()(     \
+      const CpuDevice&, typename TTypes<T, i, int32>::Tensor, \
+      typename TTypes<T, i, int32>::ConstTensor,              \
+      TTypes<Tpaddings>::ConstMatrix, int);                   \
+  extern template struct MirrorPad<CpuDevice, T, Tpaddings, i>;
+
+#define DECLARE_CPU_SPECS(T)     \
+  DECLARE_CPU_SPEC(T, int32, 1); \
+  DECLARE_CPU_SPEC(T, int32, 2); \
+  DECLARE_CPU_SPEC(T, int32, 3); \
+  DECLARE_CPU_SPEC(T, int32, 4); \
+  DECLARE_CPU_SPEC(T, int32, 5); \
+  DECLARE_CPU_SPEC(T, int64, 1); \
+  DECLARE_CPU_SPEC(T, int64, 2); \
+  DECLARE_CPU_SPEC(T, int64, 3); \
+  DECLARE_CPU_SPEC(T, int64, 4); \
+  DECLARE_CPU_SPEC(T, int64, 5);
 
 TF_CALL_POD_TYPES(DECLARE_CPU_SPECS);
 
@@ -179,7 +184,13 @@ TF_CALL_POD_TYPES(DECLARE_CPU_SPECS);
                               .TypeConstraint<type>("T")          \
                               .TypeConstraint<int32>("Tpaddings") \
                               .HostMemory("paddings"),            \
-                          MirrorPadOp<CpuDevice, type>);
+                          MirrorPadOp<CpuDevice, type, int32>);   \
+  REGISTER_KERNEL_BUILDER(Name("MirrorPad")                       \
+                              .Device(DEVICE_CPU)                 \
+                              .TypeConstraint<type>("T")          \
+                              .TypeConstraint<int64>("Tpaddings") \
+                              .HostMemory("paddings"),            \
+                          MirrorPadOp<CpuDevice, type, int64>);
 
 // Note that we do register for bool type, but not in the gradient op.
 TF_CALL_POD_TYPES(REGISTER_KERNEL);
@@ -188,20 +199,25 @@ TF_CALL_POD_TYPES(REGISTER_KERNEL);
 #if GOOGLE_CUDA
 namespace functor {
 // Forward declarations of the functor specializations for GPU.
-#define DECLARE_GPU_SPEC(T, i)                                               \
-  template <>                                                                \
-  void MirrorPad<GpuDevice, T, i>::operator()(                               \
-      const GpuDevice&, typename TTypes<T, i, int32>::Tensor,                \
-      typename TTypes<T, i, int32>::ConstTensor, TTypes<int32>::ConstMatrix, \
-      int);                                                                  \
-  extern template struct MirrorPad<GpuDevice, T, i>;
-
-#define DECLARE_GPU_SPECS(T) \
-  DECLARE_GPU_SPEC(T, 1);    \
-  DECLARE_GPU_SPEC(T, 2);    \
-  DECLARE_GPU_SPEC(T, 3);    \
-  DECLARE_GPU_SPEC(T, 4);    \
-  DECLARE_GPU_SPEC(T, 5);
+#define DECLARE_GPU_SPEC(T, Tpaddings, i)                     \
+  template <>                                                 \
+  void MirrorPad<GpuDevice, T, Tpaddings, i>::operator()(     \
+      const GpuDevice&, typename TTypes<T, i, int32>::Tensor, \
+      typename TTypes<T, i, int32>::ConstTensor,              \
+      TTypes<Tpaddings>::ConstMatrix, int);                   \
+  extern template struct MirrorPad<GpuDevice, T, Tpaddings, i>;
+
+#define DECLARE_GPU_SPECS(T)     \
+  DECLARE_GPU_SPEC(T, int32, 1); \
+  DECLARE_GPU_SPEC(T, int32, 2); \
+  DECLARE_GPU_SPEC(T, int32, 3); \
+  DECLARE_GPU_SPEC(T, int32, 4); \
+  DECLARE_GPU_SPEC(T, int32, 5); \
+  DECLARE_GPU_SPEC(T, int64, 1); \
+  DECLARE_GPU_SPEC(T, int64, 2); \
+  DECLARE_GPU_SPEC(T, int64, 3); \
+  DECLARE_GPU_SPEC(T, int64, 4); \
+  DECLARE_GPU_SPEC(T, int64, 5);
 
 TF_CALL_GPU_NUMBER_TYPES(DECLARE_GPU_SPECS);
 #undef DECLARE_GPU_SPECS
@@ -215,14 +231,20 @@ TF_CALL_GPU_NUMBER_TYPES(DECLARE_GPU_SPECS);
                               .TypeConstraint<T>("T")             \
                               .TypeConstraint<int32>("Tpaddings") \
                               .HostMemory("paddings"),            \
-                          MirrorPadOp<GpuDevice, T>)
+                          MirrorPadOp<GpuDevice, T, int32>);      \
+  REGISTER_KERNEL_BUILDER(Name("MirrorPad")                       \
+                              .Device(DEVICE_GPU)                 \
+                              .TypeConstraint<T>("T")             \
+                              .TypeConstraint<int64>("Tpaddings") \
+                              .HostMemory("paddings"),            \
+                          MirrorPadOp<GpuDevice, T, int64>);
 
 TF_CALL_GPU_NUMBER_TYPES(REGISTER_GPU_KERNEL);
 #undef REGISTER_GPU_KERNEL
 #endif  // GOOGLE_CUDA
 
 // Gradient op.
-template <typename Device, typename T>
+template <typename Device, typename T, typename Tpaddings>
 class MirrorPadGradOp : public OpKernel {
  public:
   explicit MirrorPadGradOp(OpKernelConstruction* context) : OpKernel(context) {
@@ -269,10 +291,10 @@ class MirrorPadGradOp : public OpKernel {
 
     // Compute the shape of the output tensor, and allocate it.
     TensorShape output_shape;
-    TTypes<int32>::ConstMatrix paddings = in1.matrix<int32>();
+    typename TTypes<Tpaddings>::ConstMatrix paddings = in1.matrix<Tpaddings>();
     for (int d = 0; d < dims; ++d) {
-      const int32 before = paddings(d, 0);  // Pad before existing elements.
-      const int32 after = paddings(d, 1);   // Pad after existing elements.
+      const Tpaddings before = paddings(d, 0);  // Pad before existing elements.
+      const Tpaddings after = paddings(d, 1);   // Pad after existing elements.
       OP_REQUIRES(context, before >= 0 && after >= 0,
                   errors::InvalidArgument("Paddings must be non-negative: ",
                                           before, ", ", after));
@@ -308,7 +330,7 @@ class MirrorPadGradOp : public OpKernel {
 
 #define MIRROR_PAD_GRAD_CASE(k)                                           \
   case k: {                                                               \
-    functor::MirrorPadGrad<Device, T, k>()(                               \
+    functor::MirrorPadGrad<Device, T, Tpaddings, k>()(                    \
         context->eigen_device<Device>(), To32Bit(output->tensor<T, k>()), \
         To32Bit(in0.tensor<T, k>()), paddings, offset_,                   \
         To32Bit(scratch.tensor<T, k>()));                                 \
@@ -337,33 +359,45 @@ class MirrorPadGradOp : public OpKernel {
 namespace functor {
 // Forward declarations of the functor specializations defined in the sharded
 // files.
-#define DECLARE_CPU_SPEC(T, k)                                               \
-  template <>                                                                \
-  void MirrorPadGrad<CpuDevice, T, k>::operator()(                           \
-      const CpuDevice&, typename TTypes<T, k, int32>::Tensor,                \
-      typename TTypes<T, k, int32>::ConstTensor, TTypes<int32>::ConstMatrix, \
-      int, typename TTypes<T, k, int32>::Tensor);                            \
-  extern template struct MirrorPadGrad<CpuDevice, T, k>;
-
-#define DECLARE_CPU_SPECS(T) \
-  DECLARE_CPU_SPEC(T, 1);    \
-  DECLARE_CPU_SPEC(T, 2);    \
-  DECLARE_CPU_SPEC(T, 3);    \
-  DECLARE_CPU_SPEC(T, 4);    \
-  DECLARE_CPU_SPEC(T, 5);
+#define DECLARE_CPU_SPEC(T, Tpaddings, k)                     \
+  template <>                                                 \
+  void MirrorPadGrad<CpuDevice, T, Tpaddings, k>::operator()( \
+      const CpuDevice&, typename TTypes<T, k, int32>::Tensor, \
+      typename TTypes<T, k, int32>::ConstTensor,              \
+      TTypes<Tpaddings>::ConstMatrix, int,                    \
+      typename TTypes<T, k, int32>::Tensor);                  \
+  extern template struct MirrorPadGrad<CpuDevice, T, Tpaddings, k>;
+
+#define DECLARE_CPU_SPECS(T)     \
+  DECLARE_CPU_SPEC(T, int32, 1); \
+  DECLARE_CPU_SPEC(T, int32, 2); \
+  DECLARE_CPU_SPEC(T, int32, 3); \
+  DECLARE_CPU_SPEC(T, int32, 4); \
+  DECLARE_CPU_SPEC(T, int32, 5); \
+  DECLARE_CPU_SPEC(T, int64, 1); \
+  DECLARE_CPU_SPEC(T, int64, 2); \
+  DECLARE_CPU_SPEC(T, int64, 3); \
+  DECLARE_CPU_SPEC(T, int64, 4); \
+  DECLARE_CPU_SPEC(T, int64, 5);
 
 TF_CALL_NUMBER_TYPES(DECLARE_CPU_SPECS);
 #undef DECLARE_CPU_SPECS
 #undef DECLARE_CPU_SPEC
 }  // namespace functor
 
-#define REGISTER_KERNEL(type)                                     \
-  REGISTER_KERNEL_BUILDER(Name("MirrorPadGrad")                   \
-                              .Device(DEVICE_CPU)                 \
-                              .TypeConstraint<type>("T")          \
-                              .TypeConstraint<int32>("Tpaddings") \
-                              .HostMemory("paddings"),            \
-                          MirrorPadGradOp<CpuDevice, type>);
+#define REGISTER_KERNEL(type)                                       \
+  REGISTER_KERNEL_BUILDER(Name("MirrorPadGrad")                     \
+                              .Device(DEVICE_CPU)                   \
+                              .TypeConstraint<type>("T")            \
+                              .TypeConstraint<int32>("Tpaddings")   \
+                              .HostMemory("paddings"),              \
+                          MirrorPadGradOp<CpuDevice, type, int32>); \
+  REGISTER_KERNEL_BUILDER(Name("MirrorPadGrad")                     \
+                              .Device(DEVICE_CPU)                   \
+                              .TypeConstraint<type>("T")            \
+                              .TypeConstraint<int64>("Tpaddings")   \
+                              .HostMemory("paddings"),              \
+                          MirrorPadGradOp<CpuDevice, type, int64>);
 
 TF_CALL_NUMBER_TYPES(REGISTER_KERNEL);
 #undef REGISTER_KERNEL
@@ -371,20 +405,26 @@ TF_CALL_NUMBER_TYPES(REGISTER_KERNEL);
 #if GOOGLE_CUDA
 namespace functor {
 // Forward declarations of the functor specializations for GPU.
-#define DECLARE_GPU_SPEC(T, k)                                               \
-  template <>                                                                \
-  void MirrorPadGrad<GpuDevice, T, k>::operator()(                           \
-      const GpuDevice&, typename TTypes<T, k, int32>::Tensor,                \
-      typename TTypes<T, k, int32>::ConstTensor, TTypes<int32>::ConstMatrix, \
-      int, typename TTypes<T, k, int32>::Tensor);                            \
-  extern template struct MirrorPadGrad<GpuDevice, T, k>;
-
-#define DECLARE_GPU_SPECS(T) \
-  DECLARE_GPU_SPEC(T, 1);    \
-  DECLARE_GPU_SPEC(T, 2);    \
-  DECLARE_GPU_SPEC(T, 3);    \
-  DECLARE_GPU_SPEC(T, 4);    \
-  DECLARE_GPU_SPEC(T, 5);
+#define DECLARE_GPU_SPEC(T, Tpaddings, k)                     \
+  template <>                                                 \
+  void MirrorPadGrad<GpuDevice, T, Tpaddings, k>::operator()( \
+      const GpuDevice&, typename TTypes<T, k, int32>::Tensor, \
+      typename TTypes<T, k, int32>::ConstTensor,              \
+      TTypes<Tpaddings>::ConstMatrix, int,                    \
+      typename TTypes<T, k, int32>::Tensor);                  \
+  extern template struct MirrorPadGrad<GpuDevice, T, Tpaddings, k>;
+
+#define DECLARE_GPU_SPECS(T)     \
+  DECLARE_GPU_SPEC(T, int32, 1); \
+  DECLARE_GPU_SPEC(T, int32, 2); \
+  DECLARE_GPU_SPEC(T, int32, 3); \
+  DECLARE_GPU_SPEC(T, int32, 4); \
+  DECLARE_GPU_SPEC(T, int32, 5); \
+  DECLARE_GPU_SPEC(T, int64, 1); \
+  DECLARE_GPU_SPEC(T, int64, 2); \
+  DECLARE_GPU_SPEC(T, int64, 3); \
+  DECLARE_GPU_SPEC(T, int64, 4); \
+  DECLARE_GPU_SPEC(T, int64, 5);
 
 TF_CALL_GPU_NUMBER_TYPES(DECLARE_GPU_SPECS);
 #undef DECLARE_GPU_SPECS
@@ -398,7 +438,13 @@ TF_CALL_GPU_NUMBER_TYPES(DECLARE_GPU_SPECS);
                               .TypeConstraint<T>("T")             \
                               .TypeConstraint<int32>("Tpaddings") \
                               .HostMemory("paddings"),            \
-                          MirrorPadGradOp<GpuDevice, T>)
+                          MirrorPadGradOp<GpuDevice, T, int32>);  \
+  REGISTER_KERNEL_BUILDER(Name("MirrorPadGrad")                   \
+                              .Device(DEVICE_GPU)                 \
+                              .TypeConstraint<T>("T")             \
+                              .TypeConstraint<int64>("Tpaddings") \
+                              .HostMemory("paddings"),            \
+                          MirrorPadGradOp<GpuDevice, T, int64>);
 
 TF_CALL_GPU_NUMBER_TYPES(REGISTER_GPU_KERNEL);
 #undef REGISTER_GPU_KERNEL
diff --git a/tensorflow/core/kernels/mirror_pad_op.h b/tensorflow/core/kernels/mirror_pad_op.h
index b83d2223d0..81150a9e79 100644
--- a/tensorflow/core/kernels/mirror_pad_op.h
+++ b/tensorflow/core/kernels/mirror_pad_op.h
@@ -64,9 +64,8 @@ class TensorMirrorPadOp
       StorageKind;
   typedef typename Eigen::internal::traits<TensorMirrorPadOp>::Index Index;
 
-  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
-  TensorMirrorPadOp(const XprType& expr, const PaddingDimensions& padding_dims,
-                    Index offset)
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorMirrorPadOp(
+      const XprType& expr, const PaddingDimensions& padding_dims, Index offset)
       : xpr_(expr), padding_dims_(padding_dims), offset_(offset) {}
 
   EIGEN_DEVICE_FUNC
@@ -336,12 +335,12 @@ namespace functor {
 
 // offset argument must be either 0 or 1. This controls whether the boundary
 // values are replicated (offset == 0) or not replicated (offset == 1).
-template <typename Device, typename T, int Dims>
+template <typename Device, typename T, typename Tpaddings, int Dims>
 struct MirrorPad {
   void operator()(const Device& device,
                   typename TTypes<T, Dims, int32>::Tensor output,
                   typename TTypes<T, Dims, int32>::ConstTensor input,
-                  TTypes<int32>::ConstMatrix padding, int offset) {
+                  typename TTypes<Tpaddings>::ConstMatrix padding, int offset) {
     Eigen::array<Eigen::IndexPair<int32>, Dims> padding_dims;
 
     for (int i = 0; i < Dims; ++i) {
@@ -363,12 +362,12 @@ struct MirrorPad {
 
 // offset argument must be either 0 or 1. This controls whether the boundary
 // values are replicated (offset == 0) or not replicated (offset == 1).
-template <typename Device, typename T, int Dims>
+template <typename Device, typename T, typename Tpaddings, int Dims>
 struct MirrorPadGrad {
   void operator()(const Device& device,
                   typename TTypes<T, Dims, int32>::Tensor output,
                   typename TTypes<T, Dims, int32>::ConstTensor input,
-                  TTypes<int32>::ConstMatrix paddings, int offset,
+                  typename TTypes<Tpaddings>::ConstMatrix paddings, int offset,
                   typename TTypes<T, Dims, int32>::Tensor scratch) {
     // Copy the gradient input into the scratch buffer.
     scratch.device(device) = input;
diff --git a/tensorflow/core/kernels/mirror_pad_op_cpu_impl.h b/tensorflow/core/kernels/mirror_pad_op_cpu_impl.h
index 9864f5633a..bb22b2aa91 100644
--- a/tensorflow/core/kernels/mirror_pad_op_cpu_impl.h
+++ b/tensorflow/core/kernels/mirror_pad_op_cpu_impl.h
@@ -25,13 +25,17 @@ namespace tensorflow {
 
 using CpuDevice = Eigen::ThreadPoolDevice;
 
-#define DEFINE_CPU_SPECS(T) \
-  template struct functor::MirrorPad<CpuDevice, T, CPU_PROVIDED_IXDIM>;
+#define DEFINE_CPU_SPECS(T)                                                    \
+  template struct functor::MirrorPad<CpuDevice, T, int32, CPU_PROVIDED_IXDIM>; \
+  template struct functor::MirrorPad<CpuDevice, T, int64, CPU_PROVIDED_IXDIM>;
 TF_CALL_POD_TYPES(DEFINE_CPU_SPECS);
 #undef DEFINE_CPU_SPECS
 
-#define DEFINE_CPU_SPECS(T) \
-  template struct functor::MirrorPadGrad<CpuDevice, T, CPU_PROVIDED_IXDIM>;
+#define DEFINE_CPU_SPECS(T)                                   \
+  template struct functor::MirrorPadGrad<CpuDevice, T, int32, \
+                                         CPU_PROVIDED_IXDIM>; \
+  template struct functor::MirrorPadGrad<CpuDevice, T, int64, \
+                                         CPU_PROVIDED_IXDIM>;
 TF_CALL_NUMBER_TYPES(DEFINE_CPU_SPECS);
 #undef DEFINE_CPU_SPECS
 
diff --git a/tensorflow/core/kernels/mirror_pad_op_gpu.cu.cc b/tensorflow/core/kernels/mirror_pad_op_gpu.cu.cc
index 8074aa9624..dbd0a9bd8f 100644
--- a/tensorflow/core/kernels/mirror_pad_op_gpu.cu.cc
+++ b/tensorflow/core/kernels/mirror_pad_op_gpu.cu.cc
@@ -25,17 +25,27 @@ namespace tensorflow {
 
 using GpuDevice = Eigen::GpuDevice;
 
-#define DEFINE_GPU_SPECS(T)                                \
-  template struct functor::MirrorPad<GpuDevice, T, 1>;     \
-  template struct functor::MirrorPad<GpuDevice, T, 2>;     \
-  template struct functor::MirrorPad<GpuDevice, T, 3>;     \
-  template struct functor::MirrorPad<GpuDevice, T, 4>;     \
-  template struct functor::MirrorPad<GpuDevice, T, 5>;     \
-  template struct functor::MirrorPadGrad<GpuDevice, T, 1>; \
-  template struct functor::MirrorPadGrad<GpuDevice, T, 2>; \
-  template struct functor::MirrorPadGrad<GpuDevice, T, 3>; \
-  template struct functor::MirrorPadGrad<GpuDevice, T, 4>; \
-  template struct functor::MirrorPadGrad<GpuDevice, T, 5>;
+#define DEFINE_GPU_SPECS(T)                                       \
+  template struct functor::MirrorPad<GpuDevice, T, int32, 1>;     \
+  template struct functor::MirrorPad<GpuDevice, T, int32, 2>;     \
+  template struct functor::MirrorPad<GpuDevice, T, int32, 3>;     \
+  template struct functor::MirrorPad<GpuDevice, T, int32, 4>;     \
+  template struct functor::MirrorPad<GpuDevice, T, int32, 5>;     \
+  template struct functor::MirrorPad<GpuDevice, T, int64, 1>;     \
+  template struct functor::MirrorPad<GpuDevice, T, int64, 2>;     \
+  template struct functor::MirrorPad<GpuDevice, T, int64, 3>;     \
+  template struct functor::MirrorPad<GpuDevice, T, int64, 4>;     \
+  template struct functor::MirrorPad<GpuDevice, T, int64, 5>;     \
+  template struct functor::MirrorPadGrad<GpuDevice, T, int32, 1>; \
+  template struct functor::MirrorPadGrad<GpuDevice, T, int32, 2>; \
+  template struct functor::MirrorPadGrad<GpuDevice, T, int32, 3>; \
+  template struct functor::MirrorPadGrad<GpuDevice, T, int32, 4>; \
+  template struct functor::MirrorPadGrad<GpuDevice, T, int32, 5>; \
+  template struct functor::MirrorPadGrad<GpuDevice, T, int64, 1>; \
+  template struct functor::MirrorPadGrad<GpuDevice, T, int64, 2>; \
+  template struct functor::MirrorPadGrad<GpuDevice, T, int64, 3>; \
+  template struct functor::MirrorPadGrad<GpuDevice, T, int64, 4>; \
+  template struct functor::MirrorPadGrad<GpuDevice, T, int64, 5>;
 
 TF_CALL_GPU_NUMBER_TYPES(DEFINE_GPU_SPECS);
 #undef DEFINE_GPU_SPECS
diff --git a/tensorflow/python/kernel_tests/pad_op_test.py b/tensorflow/python/kernel_tests/pad_op_test.py
index 1af43e6067..2c766e3640 100644
--- a/tensorflow/python/kernel_tests/pad_op_test.py
+++ b/tensorflow/python/kernel_tests/pad_op_test.py
@@ -193,6 +193,25 @@ class PadOpTest(test.TestCase):
       with self.assertRaisesRegexp(ValueError, "Unknown padding mode"):
         array_ops.pad(x, [[1, 0], [2, 1]], mode="weird").eval()
 
+  def testPaddingTypes(self):
+    paddings = [[1, 0], [2, 3], [0, 2]]
+    inputs = np.random.randint(-100, 100, (4, 4, 3)).astype(np.float32)
+    for mode in ("CONSTANT", "REFLECT", "SYMMETRIC", "reflect", "symmetric",
+                 "constant"):
+      for padding_dtype in [dtypes.int32, dtypes.int64]:
+        np_val = self._npPad(inputs,
+                             paddings,
+                             mode=mode,
+                             constant_values=0)
+        with self.test_session(use_gpu=True):
+          tf_val = array_ops.pad(inputs,
+                                 constant_op.constant(paddings, padding_dtype),
+                                 mode=mode,
+                                 constant_values=0)
+          out = tf_val.eval()
+        self.assertAllEqual(np_val, out)
+        self.assertShapeEqual(np_val, tf_val)
+
   def testIntTypes(self):
     # TODO(touts): Figure out why the padding tests do not work on GPU
     # for int types and rank > 2.
-- 
GitLab


From ac0004e71120e237989422bd4a7441df72613072 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Sun, 22 Oct 2017 22:50:20 -0700
Subject: [PATCH 663/909] Add int64 shape support on GPU for stateless random
 ops. (#13908)

* Add int64 shape support on GPU for stateless random ops.

This fix adds int64 shape support on GPU for stateless random ops
`StatelessRandomUniform`, `StatelessRandomNormal`, `StatelessTruncatedNormal`.

The int64 shape for stateless random ops is already supported on CPU
with int32/int64 processed properly through `MakeShape`.

However, on GPU a type constraint `.TypeConstraint<int32>("T")`
has been improperly added. Such a type constraint actually prevents
an int64 shape type to run on GPU. (As a comparision, no type constraint
on CPU).

This fix removes the type constraint and allows int64 shape to be run on GPU.

This fix also adds test cases for int64 shape support on stateless random ops.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Add test cases for int64 shape support for stateless random ops.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Add int32 to shape types tested.
---
 .../kernel_tests/stateless_random_ops_test.py    | 16 ++++++++++++++++
 tensorflow/core/kernels/stateless_random_ops.cc  |  3 ---
 2 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/tensorflow/contrib/stateless/python/kernel_tests/stateless_random_ops_test.py b/tensorflow/contrib/stateless/python/kernel_tests/stateless_random_ops_test.py
index 9a36bdc2f9..cd4d46aa07 100644
--- a/tensorflow/contrib/stateless/python/kernel_tests/stateless_random_ops_test.py
+++ b/tensorflow/contrib/stateless/python/kernel_tests/stateless_random_ops_test.py
@@ -20,6 +20,7 @@ from __future__ import print_function
 
 import numpy as np
 from tensorflow.contrib import stateless
+from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import random_seed
 from tensorflow.python.ops import array_ops
@@ -79,6 +80,21 @@ class StatelessOpsTest(test.TestCase):
             for s1, v1 in values:
               self.assertEqual(s0 == s1, np.all(v0 == v1))
 
+  def testShapeType(self):
+    with self.test_session(use_gpu=True):
+      for shape_dtype in [dtypes.int32, dtypes.int64]:
+        seed_t = array_ops.placeholder(dtypes.int64, shape=[2])
+        seeds = [(x, y) for x in range(5) for y in range(5)] * 3
+        for stateless_op, _ in CASES:
+          for shape in (), (3,), (2, 5):
+            pure = stateless_op(constant_op.constant(shape, dtype=shape_dtype),
+                                seed=seed_t)
+            values = [(seed, pure.eval(feed_dict={seed_t: seed}))
+                      for seed in seeds]
+            for s0, v0 in values:
+              for s1, v1 in values:
+                self.assertEqual(s0 == s1, np.all(v0 == v1))
+
 
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/core/kernels/stateless_random_ops.cc b/tensorflow/core/kernels/stateless_random_ops.cc
index 79d0c07acd..f6fb0a121d 100644
--- a/tensorflow/core/kernels/stateless_random_ops.cc
+++ b/tensorflow/core/kernels/stateless_random_ops.cc
@@ -137,7 +137,6 @@ TF_CALL_double(REGISTER);
           .Device(DEVICE_GPU)                                          \
           .HostMemory("shape")                                         \
           .HostMemory("seed")                                          \
-          .TypeConstraint<int32>("T")                                  \
           .TypeConstraint<TYPE>("dtype"),                              \
       StatelessRandomOp<GPUDevice, random::UniformDistribution<        \
                                        random::PhiloxRandom, TYPE> >); \
@@ -146,7 +145,6 @@ TF_CALL_double(REGISTER);
           .Device(DEVICE_GPU)                                          \
           .HostMemory("shape")                                         \
           .HostMemory("seed")                                          \
-          .TypeConstraint<int32>("T")                                  \
           .TypeConstraint<TYPE>("dtype"),                              \
       StatelessRandomOp<GPUDevice, random::NormalDistribution<         \
                                        random::PhiloxRandom, TYPE> >); \
@@ -155,7 +153,6 @@ TF_CALL_double(REGISTER);
           .Device(DEVICE_GPU)                                          \
           .HostMemory("shape")                                         \
           .HostMemory("seed")                                          \
-          .TypeConstraint<int32>("T")                                  \
           .TypeConstraint<TYPE>("dtype"),                              \
       StatelessRandomOp<                                               \
           GPUDevice,                                                   \
-- 
GitLab


From 9b9cbbe2a69b7fcec72d82f271cb90839c3035b7 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Sun, 22 Oct 2017 23:02:28 -0700
Subject: [PATCH 664/909] Add int64 Tperm type support for `Transpose` (#13909)

* Add int64 Tperm type support for `Transpose`

This fix adds int64 Tperm support for `Transpose`. In
`array_ops.cc`, `Transpose` and `ConjugateTranspose`
have been specified as accepting int32 and int64 perm
types. However, only int32 kernels has been registered.

This fix adds the int64 perm support by removing
the constraint on Tperm, resolve the type at runtime,
and copying the data type accordingly to correctly handle
the int64/int32 types.

Additional tests have been added as well.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Add test cases for int64 of perm in Transpose.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Add namespace to hide PermutationHelper

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Enable use_gpu=True for perm type test.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* extra // namespace annotation

* Adding a comment about int32 casting that should be safe.

Permutations only contain values that refer to dimensions, and the maximum number of dimensions we have is 254, so an int32 is always safe here.
---
 tensorflow/core/kernels/transpose_op.cc       | 134 ++++++++++--------
 .../python/kernel_tests/transpose_op_test.py  |  13 ++
 2 files changed, 85 insertions(+), 62 deletions(-)

diff --git a/tensorflow/core/kernels/transpose_op.cc b/tensorflow/core/kernels/transpose_op.cc
index e151b38d90..20f0edf309 100644
--- a/tensorflow/core/kernels/transpose_op.cc
+++ b/tensorflow/core/kernels/transpose_op.cc
@@ -91,6 +91,26 @@ REGISTER_KERNEL_BUILDER(Name("InvertPermutation")
                         InvertPermutationOp);
 #endif  // TENSORFLOW_USE_SYCL
 
+namespace {
+template <typename Tperm>
+Status PermutationHelper(const Tensor& perm, const int dims,
+                         std::vector<int32>* permutation) {
+  auto Vperm = perm.vec<Tperm>();
+  if (dims != Vperm.size()) {
+    return errors::InvalidArgument("transpose expects a vector of size ", dims,
+                                   ". But input(1) is a vector of size ",
+                                   Vperm.size());
+  }
+  // using volatile instead of SubtleMustCopy here so that the
+  // asynchrony boundary is permutation.
+  const volatile Tperm* perm_begin =
+      reinterpret_cast<const volatile Tperm*>(Vperm.data());
+  *permutation = std::vector<int32>(perm_begin, perm_begin + dims);
+
+  return Status::OK();
+}
+}  // namespace
+
 // output = TransposeOp(T<any> input, T<int32> perm) takes a tensor
 // of type T and rank N, and a permutation of 0, 1, ..., N-1. It
 // shuffles the dimensions of the input tensor according to permutation.
@@ -113,17 +133,16 @@ void TransposeOp::Compute(OpKernelContext* ctx) {
   OP_REQUIRES(ctx, TensorShapeUtils::IsVector(perm.shape()),
               errors::InvalidArgument("perm must be a vector, not ",
                                       perm.shape().DebugString()));
-  auto Vperm = perm.vec<int32>();
+
+  // Although Tperm may be an int64 type, an int32 is sufficient to hold
+  // dimension range values, so the narrowing here should be safe.
+  std::vector<int32> permutation;
   const int dims = input.dims();
-  OP_REQUIRES(ctx, dims == Vperm.size(),
-              errors::InvalidArgument(
-                  "transpose expects a vector of size ", input.dims(),
-                  ". But input(1) is a vector of size ", Vperm.size()));
-  // using volatile instead of SubtleMustCopy here so that the
-  // asynchrony boundary is permutation.
-  const volatile int32* perm_begin =
-      reinterpret_cast<const volatile int32*>(Vperm.data());
-  const std::vector<int32> permutation(perm_begin, perm_begin + dims);
+  if (perm.dtype() == DT_INT32) {
+    OP_REQUIRES_OK(ctx, PermutationHelper<int32>(perm, dims, &permutation));
+  } else {
+    OP_REQUIRES_OK(ctx, PermutationHelper<int64>(perm, dims, &permutation));
+  }
   TensorShape shape;
 
   // Check whether permutation is a permutation of integers of [0 .. dims).
@@ -142,10 +161,9 @@ void TransposeOp::Compute(OpKernelContext* ctx) {
     }
   }
   for (int i = 0; i < dims; ++i) {
-    OP_REQUIRES(
-        ctx, bits[i],
-        errors::InvalidArgument(i, " is missing from {",
-                                str_util::Join(permutation, ","), "}."));
+    OP_REQUIRES(ctx, bits[i], errors::InvalidArgument(
+                                  i, " is missing from {",
+                                  str_util::Join(permutation, ","), "}."));
   }
 
   // 0-D, 1-D, and identity transposes do nothing.
@@ -185,18 +203,16 @@ Status ConjugateTransposeCpuOp::DoTranspose(OpKernelContext* ctx,
 }
 
 #ifdef INTEL_MKL
-#define REGISTER(T)                                           \
-  REGISTER_KERNEL_BUILDER(Name("Transpose")                   \
-                              .Device(DEVICE_CPU)             \
-                              .TypeConstraint<T>("T")         \
-                              .TypeConstraint<int32>("Tperm") \
-                              .HostMemory("perm"),            \
-                          MklTransposeCpuOp);                 \
-  REGISTER_KERNEL_BUILDER(Name("ConjugateTranspose")          \
-                              .Device(DEVICE_CPU)             \
-                              .TypeConstraint<T>("T")         \
-                              .TypeConstraint<int32>("Tperm") \
-                              .HostMemory("perm"),            \
+#define REGISTER(T)                                   \
+  REGISTER_KERNEL_BUILDER(Name("Transpose")           \
+                              .Device(DEVICE_CPU)     \
+                              .TypeConstraint<T>("T") \
+                              .HostMemory("perm"),    \
+                          MklTransposeCpuOp);         \
+  REGISTER_KERNEL_BUILDER(Name("ConjugateTranspose")  \
+                              .Device(DEVICE_CPU)     \
+                              .TypeConstraint<T>("T") \
+                              .HostMemory("perm"),    \
                           MklConjugateTransposeCpuOp);
 TF_CALL_ALL_TYPES(REGISTER);
 REGISTER(bfloat16);
@@ -204,18 +220,16 @@ REGISTER(bfloat16);
 
 #else  // INTEL_MKL
 
-#define REGISTER(T)                                           \
-  REGISTER_KERNEL_BUILDER(Name("Transpose")                   \
-                              .Device(DEVICE_CPU)             \
-                              .TypeConstraint<T>("T")         \
-                              .TypeConstraint<int32>("Tperm") \
-                              .HostMemory("perm"),            \
-                          TransposeCpuOp);                    \
-  REGISTER_KERNEL_BUILDER(Name("ConjugateTranspose")          \
-                              .Device(DEVICE_CPU)             \
-                              .TypeConstraint<T>("T")         \
-                              .TypeConstraint<int32>("Tperm") \
-                              .HostMemory("perm"),            \
+#define REGISTER(T)                                   \
+  REGISTER_KERNEL_BUILDER(Name("Transpose")           \
+                              .Device(DEVICE_CPU)     \
+                              .TypeConstraint<T>("T") \
+                              .HostMemory("perm"),    \
+                          TransposeCpuOp);            \
+  REGISTER_KERNEL_BUILDER(Name("ConjugateTranspose")  \
+                              .Device(DEVICE_CPU)     \
+                              .TypeConstraint<T>("T") \
+                              .HostMemory("perm"),    \
                           ConjugateTransposeCpuOp);
 TF_CALL_ALL_TYPES(REGISTER)
 REGISTER(bfloat16);
@@ -238,18 +252,16 @@ Status ConjugateTransposeGpuOp::DoTranspose(OpKernelContext* ctx,
                                             perm, out);
 }
 
-#define REGISTER(T)                                           \
-  REGISTER_KERNEL_BUILDER(Name("Transpose")                   \
-                              .Device(DEVICE_GPU)             \
-                              .TypeConstraint<T>("T")         \
-                              .TypeConstraint<int32>("Tperm") \
-                              .HostMemory("perm"),            \
-                          TransposeGpuOp);                    \
-  REGISTER_KERNEL_BUILDER(Name("ConjugateTranspose")          \
-                              .Device(DEVICE_GPU)             \
-                              .TypeConstraint<T>("T")         \
-                              .TypeConstraint<int32>("Tperm") \
-                              .HostMemory("perm"),            \
+#define REGISTER(T)                                   \
+  REGISTER_KERNEL_BUILDER(Name("Transpose")           \
+                              .Device(DEVICE_GPU)     \
+                              .TypeConstraint<T>("T") \
+                              .HostMemory("perm"),    \
+                          TransposeGpuOp);            \
+  REGISTER_KERNEL_BUILDER(Name("ConjugateTranspose")  \
+                              .Device(DEVICE_GPU)     \
+                              .TypeConstraint<T>("T") \
+                              .HostMemory("perm"),    \
                           ConjugateTransposeGpuOp);
 TF_CALL_POD_TYPES(REGISTER);
 #undef REGISTER
@@ -270,18 +282,16 @@ Status ConjugateTransposeSyclOp::DoTranspose(OpKernelContext* ctx,
   return ::tensorflow::DoConjugateTranspose(ctx->eigen_device<SYCLDevice>(), in,
                                             perm, out);
 }
-#define REGISTER(T)                                           \
-  REGISTER_KERNEL_BUILDER(Name("Transpose")                   \
-                              .Device(DEVICE_SYCL)            \
-                              .TypeConstraint<T>("T")         \
-                              .TypeConstraint<int32>("Tperm") \
-                              .HostMemory("perm"),            \
-                          TransposeSyclOp);                   \
-  REGISTER_KERNEL_BUILDER(Name("ConjugateTranspose")          \
-                              .Device(DEVICE_SYCL)            \
-                              .TypeConstraint<T>("T")         \
-                              .TypeConstraint<int32>("Tperm") \
-                              .HostMemory("perm"),            \
+#define REGISTER(T)                                   \
+  REGISTER_KERNEL_BUILDER(Name("Transpose")           \
+                              .Device(DEVICE_SYCL)    \
+                              .TypeConstraint<T>("T") \
+                              .HostMemory("perm"),    \
+                          TransposeSyclOp);           \
+  REGISTER_KERNEL_BUILDER(Name("ConjugateTranspose")  \
+                              .Device(DEVICE_SYCL)    \
+                              .TypeConstraint<T>("T") \
+                              .HostMemory("perm"),    \
                           ConjugateTransposeSyclOp);
 TF_CALL_POD_TYPES(REGISTER);
 #undef REGISTER
diff --git a/tensorflow/python/kernel_tests/transpose_op_test.py b/tensorflow/python/kernel_tests/transpose_op_test.py
index 3b352937c8..c551d9c3d0 100644
--- a/tensorflow/python/kernel_tests/transpose_op_test.py
+++ b/tensorflow/python/kernel_tests/transpose_op_test.py
@@ -317,6 +317,19 @@ class TransposeTest(test.TestCase):
         np.arange(0, 8).reshape([2, 4]).astype(np.float32),
         np.array([1, 0]).astype(np.int32))
 
+  def testPermType(self):
+    for perm_dtype in [np.int64, np.int32]:
+      x = np.arange(0, 8).reshape([2, 4]).astype(np.float32)
+      p = np.array([1, 0]).astype(perm_dtype)
+      np_ans = np.copy(x).transpose(p)
+      with self.test_session(use_gpu=True):
+        inx = ops.convert_to_tensor(x)
+        inp = constant_op.constant(p)
+        y = array_ops.transpose(inx, inp)
+        tf_ans = y.eval()
+        self.assertShapeEqual(np_ans, y)
+        self.assertAllEqual(np_ans, tf_ans)
+
   def testHalf(self):
     self._compare(np.arange(0, 21).reshape([3, 7]).astype(np.float16))
     self._compare(np.arange(0, 210).reshape([2, 3, 5, 7]).astype(np.float16))
-- 
GitLab


From eea089bdb66597c9e66180d39b94eea2c17be93e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 23 Oct 2017 06:00:06 -0700
Subject: [PATCH 665/909] K-FAC: Multi-tower support for ConvDiagonalFB.

PiperOrigin-RevId: 173105412
---
 .../python/kernel_tests/fisher_blocks_test.py | 202 +++++++++++++++++-
 .../kernel_tests/layer_collection_test.py     |   6 +-
 .../contrib/kfac/python/ops/fisher_blocks.py  |  70 ++++--
 .../kfac/python/ops/layer_collection.py       |  27 ++-
 4 files changed, 271 insertions(+), 34 deletions(-)

diff --git a/tensorflow/contrib/kfac/python/kernel_tests/fisher_blocks_test.py b/tensorflow/contrib/kfac/python/kernel_tests/fisher_blocks_test.py
index 9b13756e62..80855da2e9 100644
--- a/tensorflow/contrib/kfac/python/kernel_tests/fisher_blocks_test.py
+++ b/tensorflow/contrib/kfac/python/kernel_tests/fisher_blocks_test.py
@@ -328,17 +328,11 @@ class FullyConnectedDiagonalFB(test.TestCase):
       multiply_result: Result of FisherBlock.multiply(params)
       multiply_inverse_result: Result of FisherBlock.multiply_inverse(params)
     """
-
-    def _as_tensors(tensor_or_tuple):
-      if isinstance(tensor_or_tuple, (tuple, list)):
-        return tuple(ops.convert_to_tensor(t) for t in tensor_or_tuple)
-      return ops.convert_to_tensor(tensor_or_tuple)
-
     with ops.Graph().as_default(), self.test_session() as sess:
-      inputs = [_as_tensors(i) for i in inputs]
-      outputs = [_as_tensors(o) for o in outputs]
-      output_grads = [_as_tensors(og) for og in output_grads]
-      params = _as_tensors(params)
+      inputs = as_tensors(inputs)
+      outputs = as_tensors(outputs)
+      output_grads = as_tensors(output_grads)
+      params = as_tensors(params)
 
       block = fb.FullyConnectedDiagonalFB(
           lc.LayerCollection(), has_bias=isinstance(params, (tuple, list)))
@@ -464,6 +458,188 @@ class FullyConnectedKFACBasicFBTest(test.TestCase):
       self.assertAllClose(output_flat, explicit)
 
 
+class ConvDiagonalFBTest(test.TestCase):
+
+  def setUp(self):
+    super(ConvDiagonalFBTest, self).setUp()
+
+    self.batch_size = 2
+    self.height = 8
+    self.width = 4
+    self.input_channels = 6
+    self.output_channels = 3
+    self.kernel_size = 1
+
+    self.inputs = np.random.randn(self.batch_size, self.height, self.width,
+                                  self.input_channels).astype(np.float32)
+    self.outputs = np.zeros(
+        [self.batch_size, self.height, self.width,
+         self.output_channels]).astype(np.float32)
+    self.output_grads = np.random.randn(
+        self.batch_size, self.height, self.width, self.output_channels).astype(
+            np.float32)
+    self.w = np.random.randn(self.kernel_size, self.kernel_size,
+                             self.input_channels, self.output_channels).astype(
+                                 np.float32)
+    self.b = np.random.randn(self.output_channels).astype(np.float32)
+
+  def fisherApprox(self, has_bias=False):
+    """Fisher approximation using default inputs."""
+    if has_bias:
+      inputs = np.concatenate(
+          [self.inputs,
+           np.ones([self.batch_size, self.height, self.width, 1])],
+          axis=-1)
+    else:
+      inputs = self.inputs
+    return self.buildDiagonalFisherApproximation(inputs, self.output_grads,
+                                                 self.kernel_size)
+
+  def buildDiagonalFisherApproximation(self, inputs, output_grads, kernel_size):
+    r"""Builds explicit diagonal Fisher approximation.
+
+    Fisher's diagonal is (d loss / d w)'s elements squared for
+      d/dw = E[\sum_{loc} outer(input_{loc}, output_grad_{loc})]
+
+    where the expectation is taken over examples and the sum over (x, y)
+    locations upon which the convolution is applied.
+
+    Args:
+      inputs: np.array of shape [batch_size, height, width, input_channels].
+      output_grads: np.array of shape [batch_size, height, width,
+        output_channels].
+      kernel_size: int. height and width of kernel.
+
+    Returns:
+      Diagonal np.array of shape [num_params, num_params] for num_params =
+      kernel_size^2 * input_channels * output_channels.
+    """
+    batch_size, height, width, input_channels = inputs.shape
+    assert output_grads.shape[0] == batch_size
+    assert output_grads.shape[1] == height
+    assert output_grads.shape[2] == width
+    output_channels = output_grads.shape[3]
+
+    # If kernel_size == 1, then we don't need to worry about capturing context
+    # around the pixel upon which a convolution is applied. This makes testing
+    # easier.
+    assert kernel_size == 1, "kernel_size != 1 isn't supported."
+    num_locations = height * width
+    inputs = np.reshape(inputs, [batch_size, num_locations, input_channels])
+    output_grads = np.reshape(output_grads,
+                              [batch_size, num_locations, output_channels])
+
+    fisher_diag = np.zeros((input_channels, output_channels))
+    for i in range(batch_size):
+      # Each example's approximation is a square(sum-of-outer-products).
+      example_fisher_diag = np.zeros((input_channels, output_channels))
+      for j in range(num_locations):
+        example_fisher_diag += np.outer(inputs[i, j], output_grads[i, j])
+      fisher_diag += np.square(example_fisher_diag)
+
+    # Normalize by batch_size (not num_locations).
+    return np.diag(fisher_diag.flatten()) / batch_size
+
+  def testMultiply(self):
+    result, _ = self.runFisherBlockOps(self.w, [self.inputs], [self.outputs],
+                                       [self.output_grads])
+
+    # Construct Fisher-vector product.
+    expected_result = self.fisherApprox().dot(self.w.flatten())
+    expected_result = expected_result.reshape([
+        self.kernel_size, self.kernel_size, self.input_channels,
+        self.output_channels
+    ])
+
+    self.assertAllClose(expected_result, result)
+
+  def testMultiplyInverse(self):
+    _, result = self.runFisherBlockOps(self.w, [self.inputs], [self.outputs],
+                                       [self.output_grads])
+
+    # Construct inverse Fisher-vector product.
+    expected_result = np.linalg.inv(self.fisherApprox()).dot(self.w.flatten())
+    expected_result = expected_result.reshape([
+        self.kernel_size, self.kernel_size, self.input_channels,
+        self.output_channels
+    ])
+
+    self.assertAllClose(expected_result, result, atol=1e-3)
+
+  def testRegisterAdditionalMinibatch(self):
+    """Ensure 1 big minibatch and 2 small minibatches are equivalent."""
+    multiply_result_big, multiply_inverse_result_big = self.runFisherBlockOps(
+        self.w, [self.inputs], [self.outputs], [self.output_grads])
+    multiply_result_small, multiply_inverse_result_small = (
+        self.runFisherBlockOps(self.w,
+                               np.split(self.inputs, 2),
+                               np.split(self.outputs, 2),
+                               np.split(self.output_grads, 2)))
+
+    self.assertAllClose(multiply_result_big, multiply_result_small)
+    self.assertAllClose(multiply_inverse_result_big,
+                        multiply_inverse_result_small)
+
+  def testMultiplyHasBias(self):
+    result, _ = self.runFisherBlockOps((self.w, self.b), [self.inputs],
+                                       [self.outputs], [self.output_grads])
+    # Clone 'b' along 'input_channels' dimension.
+    b_filter = np.tile(
+        np.reshape(self.b, [1, 1, 1, self.output_channels]),
+        [self.kernel_size, self.kernel_size, 1, 1])
+    params = np.concatenate([self.w, b_filter], axis=2)
+    expected_result = self.fisherApprox(True).dot(params.flatten())
+
+    # Extract 'b' from concatenated parameters.
+    expected_result = expected_result.reshape([
+        self.kernel_size, self.kernel_size, self.input_channels + 1,
+        self.output_channels
+    ])
+    expected_result = (expected_result[:, :, 0:-1, :], np.reshape(
+        expected_result[:, :, -1, :], [self.output_channels]))
+
+    self.assertEqual(len(result), 2)
+    self.assertAllClose(expected_result[0], result[0])
+    self.assertAllClose(expected_result[1], result[1])
+
+  def runFisherBlockOps(self, params, inputs, outputs, output_grads):
+    """Run Ops guaranteed by FisherBlock interface.
+
+    Args:
+      params: Tensor or 2-tuple of Tensors. Represents weights or weights and
+        bias of this layer.
+      inputs: list of Tensors of shape [batch_size, input_size]. Inputs to
+        layer.
+      outputs: list of Tensors of shape [batch_size, output_size].
+        Preactivations produced by layer.
+      output_grads: list of Tensors of shape [batch_size, output_size].
+        Gradient of loss with respect to 'outputs'.
+
+    Returns:
+      multiply_result: Result of FisherBlock.multiply(params)
+      multiply_inverse_result: Result of FisherBlock.multiply_inverse(params)
+    """
+    with ops.Graph().as_default(), self.test_session() as sess:
+      inputs = as_tensors(inputs)
+      outputs = as_tensors(outputs)
+      output_grads = as_tensors(output_grads)
+      params = as_tensors(params)
+
+      block = fb.ConvDiagonalFB(
+          lc.LayerCollection(), params, strides=[1, 1, 1, 1], padding='SAME')
+      for (i, o) in zip(inputs, outputs):
+        block.register_additional_minibatch(i, o)
+
+      block.instantiate_factors((output_grads,), damping=0.0)
+
+      sess.run(tf_variables.global_variables_initializer())
+      sess.run(block._factor.make_covariance_update_op(0.0))
+      multiply_result = sess.run(block.multiply(params))
+      multiply_inverse_result = sess.run(block.multiply_inverse(params))
+
+    return multiply_result, multiply_inverse_result
+
+
 class ConvKFCBasicFBTest(test.TestCase):
 
   def _testConvKFCBasicFBInitParams(self, params):
@@ -583,5 +759,11 @@ class ConvKFCBasicFBTest(test.TestCase):
       self.assertAllClose(output_flat, explicit)
 
 
+def as_tensors(tensor_or_tuple):
+  """Converts a potentially nested tuple of np.array to Tensors."""
+  if isinstance(tensor_or_tuple, (tuple, list)):
+    return tuple(as_tensors(t) for t in tensor_or_tuple)
+  return ops.convert_to_tensor(tensor_or_tuple)
+
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/contrib/kfac/python/kernel_tests/layer_collection_test.py b/tensorflow/contrib/kfac/python/kernel_tests/layer_collection_test.py
index 53d40da586..b444e87170 100644
--- a/tensorflow/contrib/kfac/python/kernel_tests/layer_collection_test.py
+++ b/tensorflow/contrib/kfac/python/kernel_tests/layer_collection_test.py
@@ -89,6 +89,10 @@ class LayerCollectionTest(test.TestCase):
       lc.register_conv2d(
           array_ops.constant(4), [1, 1, 1, 1], 'SAME',
           array_ops.ones((1, 1, 1, 1)), array_ops.constant(3))
+      lc.register_conv2d(
+          array_ops.constant(4), [1, 1, 1, 1], 'SAME',
+          array_ops.ones((1, 1, 1, 1)), array_ops.constant(3),
+          approx=layer_collection.APPROX_DIAGONAL_NAME)
       lc.register_generic(
           array_ops.constant(5), 16, approx=layer_collection.APPROX_FULL_NAME)
       lc.register_generic(
@@ -96,7 +100,7 @@ class LayerCollectionTest(test.TestCase):
           16,
           approx=layer_collection.APPROX_DIAGONAL_NAME)
 
-      self.assertEqual(5, len(lc.get_blocks()))
+      self.assertEqual(6, len(lc.get_blocks()))
 
   def testRegisterBlocksMultipleRegistrations(self):
     with ops.Graph().as_default():
diff --git a/tensorflow/contrib/kfac/python/ops/fisher_blocks.py b/tensorflow/contrib/kfac/python/ops/fisher_blocks.py
index 6cca2272d7..5e822b5fe3 100644
--- a/tensorflow/contrib/kfac/python/ops/fisher_blocks.py
+++ b/tensorflow/contrib/kfac/python/ops/fisher_blocks.py
@@ -227,7 +227,7 @@ class FullyConnectedDiagonalFB(FisherBlock):
   'w'. For an example 'x' that produces layer inputs 'a' and output
   preactivations 's',
 
-    v(x, y, w) = vec( x (d loss / d s)^T )
+    v(x, y, w) = vec( a (d loss / d s)^T )
 
   This FisherBlock tracks Fisher(params)[i, i] for all indices 'i' corresponding
   to the layer's parameters 'w'.
@@ -309,13 +309,29 @@ class FullyConnectedDiagonalFB(FisherBlock):
 class ConvDiagonalFB(FisherBlock):
   """FisherBlock for convolutional layers using a diagonal approx.
 
-  Unlike NaiveDiagonalFB this uses the low-variance "sum of squares" estimator.
-  """
+  Estimates the Fisher Information matrix's diagonal entries for a convolutional
+  layer. Unlike NaiveDiagonalFB this uses the low-variance "sum of squares"
+  estimator.
 
-  # TODO(jamesmartens): add units tests for this class
+  Let 'params' be a vector parameterizing a model and 'i' an arbitrary index
+  into it. We are interested in Fisher(params)[i, i]. This is,
 
-  def __init__(self, layer_collection, params, inputs, outputs, strides,
-               padding):
+    Fisher(params)[i, i] = E[ v(x, y, params) v(x, y, params)^T ][i, i]
+                         = E[ v(x, y, params)[i] ^ 2 ]
+
+  Consider a convoluational layer in this model with (unshared) filter matrix
+  'w'. For an example image 'x' that produces layer inputs 'a' and output
+  preactivations 's',
+
+    v(x, y, w) = vec( sum_{loc} a_{loc} (d loss / d s_{loc})^T )
+
+  where 'loc' is a single (x, y) location in an image.
+
+  This FisherBlock tracks Fisher(params)[i, i] for all indices 'i' corresponding
+  to the layer's parameters 'w'.
+  """
+
+  def __init__(self, layer_collection, params, strides, padding):
     """Creates a ConvDiagonalFB block.
 
     Args:
@@ -325,37 +341,39 @@ class ConvDiagonalFB(FisherBlock):
         kernel alone, a Tensor of shape [kernel_height, kernel_width,
         in_channels, out_channels]. If kernel and bias, a tuple of 2 elements
         containing the previous and a Tensor of shape [out_channels].
-      inputs: A Tensor of shape [batch_size, height, width, in_channels].
-        Input activations to this layer.
-      outputs: A Tensor of shape [batch_size, height, width, out_channels].
-        Output pre-activations from this layer.
       strides: The stride size in this layer (1-D Tensor of length 4).
-      padding: The padding in this layer (1-D of Tensor length 4).
+      padding: The padding in this layer (e.g. "SAME").
     """
-    self._inputs = inputs
-    self._outputs = outputs
-    self._strides = strides
+    self._inputs = []
+    self._outputs = []
+    self._strides = tuple(strides) if isinstance(strides, list) else strides
     self._padding = padding
     self._has_bias = isinstance(params, (tuple, list))
 
     fltr = params[0] if self._has_bias else params
     self._filter_shape = tuple(fltr.shape.as_list())
 
-    input_shape = tuple(inputs.shape.as_list())
-    self._num_locations = (
-        input_shape[1] * input_shape[2] // (strides[1] * strides[2]))
-
     super(ConvDiagonalFB, self).__init__(layer_collection)
 
   def instantiate_factors(self, grads_list, damping):
+    # Concatenate inputs, grads_list into single Tensors.
+    inputs = _concat_along_batch_dim(self._inputs)
+    grads_list = tuple(_concat_along_batch_dim(grads) for grads in grads_list)
+
+    # Infer number of locations upon which convolution is applied.
+    inputs_shape = tuple(inputs.shape.as_list())
+    self._num_locations = (
+        inputs_shape[1] * inputs_shape[2] //
+        (self._strides[1] * self._strides[2]))
+
     if NORMALIZE_DAMPING_POWER:
       damping /= self._num_locations**NORMALIZE_DAMPING_POWER
     self._damping = damping
 
     self._factor = self._layer_collection.make_or_get_factor(
         fisher_factors.ConvDiagonalFactor,
-        (self._inputs, grads_list, self._filter_shape, self._strides,
-         self._padding, self._has_bias))
+        (inputs, grads_list, self._filter_shape, self._strides, self._padding,
+         self._has_bias))
 
   def multiply_inverse(self, vector):
     reshaped_vect = utils.layer_params_to_mat2d(vector)
@@ -370,6 +388,18 @@ class ConvDiagonalFB(FisherBlock):
   def tensors_to_compute_grads(self):
     return self._outputs
 
+  def register_additional_minibatch(self, inputs, outputs):
+    """Registers an additional minibatch to the FisherBlock.
+
+    Args:
+      inputs: Tensor of shape [batch_size, height, width, input_size]. Inputs to
+        the convolution.
+      outputs: Tensor of shape [batch_size, height, width, output_size]. Layer
+        preactivations.
+    """
+    self._inputs.append(inputs)
+    self._outputs.append(outputs)
+
 
 class KroneckerProductFB(FisherBlock):
   """A base class for FisherBlocks with separate input and output factors.
diff --git a/tensorflow/contrib/kfac/python/ops/layer_collection.py b/tensorflow/contrib/kfac/python/ops/layer_collection.py
index beb8ef136e..10ef554351 100644
--- a/tensorflow/contrib/kfac/python/ops/layer_collection.py
+++ b/tensorflow/contrib/kfac/python/ops/layer_collection.py
@@ -273,9 +273,9 @@ class LayerCollection(object):
                           fb.ConvKFCBasicFB(self, params, inputs, outputs,
                                             strides, padding))
     elif approx == APPROX_DIAGONAL_NAME:
-      self.register_block(params,
-                          fb.ConvDiagonalFB(self, params, inputs, outputs,
-                                            strides, padding))
+      block = fb.ConvDiagonalFB(self, params, strides, padding)
+      block.register_additional_minibatch(inputs, outputs)
+      self.register_block(params, block)
 
   def register_generic(self, params, batch_size, approx=APPROX_DIAGONAL_NAME):
     params = params if isinstance(params, (tuple, list)) else (params,)
@@ -379,6 +379,27 @@ class LayerCollection(object):
     self._loss_dict[name] = loss
 
   def make_or_get_factor(self, cls, args):
+    """Insert 'cls(args)' into 'self.fisher_factors' if not already present.
+
+    Wraps constructor in 'tf.variable_scope()' to ensure variables constructed
+    in 'cls.__init__' are placed under this LayerCollection's scope.
+
+    Args:
+      cls: Class that implements FisherFactor.
+      args: Tuple of arguments to pass into 'cls's constructor. Must be
+        hashable.
+
+    Returns:
+      Instance of 'cls' found in self.fisher_factors.
+    """
+    try:
+      hash(args)
+    except TypeError:
+      raise TypeError((
+          "Unable to use (cls, args) = ({}, {}) as a key in "
+          "LayerCollection.fisher_factors. The pair cannot be hashed."
+      ).format(cls, args))
+
     with variable_scope.variable_scope(self._var_scope):
       return utils.setdefault(self.fisher_factors, (cls, args),
                               lambda: cls(*args))
-- 
GitLab


From dc13a8e2f7cfd56121347f5596f8b5a770da41c9 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 23 Oct 2017 06:03:28 -0700
Subject: [PATCH 666/909] Fix import of meta graphs with partitioned variables
 into a scope.

Saver inspects SliceInfo to decide the variable name when creating a
checkpoint. Before this fix even if a partitioned variable ("weights")
was imported into a scope "a" it would still be checkpointed as ("weights")
instead of ("a/weights") since import_scoped_meta_graph was not adjusting
the SliceInfo.

WARNING: if you use import_meta_graph on graphs with partitioned_variables WITH an import_scope argument AND then create a Saver to write/read checkpoints this change
may break your checkpoint loading.
PiperOrigin-RevId: 173105796
---
 .../python/framework/meta_graph_test.py       | 39 +++++++++++++++++++
 tensorflow/python/ops/variables.py            |  3 +-
 2 files changed, 41 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/framework/meta_graph_test.py b/tensorflow/python/framework/meta_graph_test.py
index 65abb69599..06cee46bf6 100644
--- a/tensorflow/python/framework/meta_graph_test.py
+++ b/tensorflow/python/framework/meta_graph_test.py
@@ -36,8 +36,10 @@ from tensorflow.python.ops import data_flow_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import metrics
 from tensorflow.python.ops import nn_ops
+from tensorflow.python.ops import partitioned_variables
 from tensorflow.python.ops import random_ops
 from tensorflow.python.ops import resource_variable_ops
+from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import gfile
 from tensorflow.python.platform import test
@@ -657,5 +659,42 @@ class MetaGraphWithVariableScopeTest(test.TestCase):
         initializer = variables.local_variables_initializer()
 
 
+class ExportImportAcrossScopesTest(test.TestCase):
+
+  def testPartionedVariables(self):
+    def make_graph_with_partitioned_variables():
+      variable_scope.get_variable(
+          name="weights",
+          partitioner=partitioned_variables.fixed_size_partitioner(3, axis=0),
+          initializer=random_ops.truncated_normal([100, 10]))
+    self._testExportImportAcrossScopes(make_graph_with_partitioned_variables)
+
+  def _testExportImportAcrossScopes(self, graph_fn):
+    """Tests export and importing a graph across scopes.
+
+    Args:
+      graph_fn: A closure that creates a graph on the current scope.
+    """
+    with ops.Graph().as_default() as original_graph:
+      with variable_scope.variable_scope("dropA/dropB/keepA"):
+        graph_fn()
+    exported_meta_graph_def = meta_graph.export_scoped_meta_graph(
+        graph=original_graph,
+        export_scope="dropA/dropB")[0]
+
+    with ops.Graph().as_default() as imported_graph:
+      meta_graph.import_scoped_meta_graph(
+          exported_meta_graph_def,
+          import_scope="importA")
+
+    with ops.Graph().as_default() as expected_graph:
+      with variable_scope.variable_scope("importA/keepA"):
+        graph_fn()
+
+    result = meta_graph.export_scoped_meta_graph(graph=imported_graph)[0]
+    expected = meta_graph.export_scoped_meta_graph(graph=expected_graph)[0]
+    self.assertProtoEquals(expected, result)
+
+
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/ops/variables.py b/tensorflow/python/ops/variables.py
index 90b4f25d81..0272f77176 100644
--- a/tensorflow/python/ops/variables.py
+++ b/tensorflow/python/ops/variables.py
@@ -394,7 +394,8 @@ class Variable(object):
                                import_scope=import_scope))
     if variable_def.HasField("save_slice_info_def"):
       self._save_slice_info = Variable.SaveSliceInfo(
-          save_slice_info_def=variable_def.save_slice_info_def)
+          save_slice_info_def=variable_def.save_slice_info_def,
+          import_scope=import_scope)
     else:
       self._save_slice_info = None
     self._caching_device = None
-- 
GitLab


From 670dddf4ad81c67fc76b370bf7b9d77263824358 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 23 Oct 2017 06:53:02 -0700
Subject: [PATCH 667/909] Multi-minibatch support for
 tf.contrib.kfac.fisher_blocks.FullyConnectedKFACBasicFB.

PiperOrigin-RevId: 173109677
---
 .../python/kernel_tests/fisher_blocks_test.py | 41 ++++++++--------
 .../contrib/kfac/python/ops/fisher_blocks.py  | 48 ++++++++++++++-----
 .../contrib/kfac/python/ops/fisher_factors.py |  8 ++++
 .../kfac/python/ops/layer_collection.py       |  6 +--
 4 files changed, 69 insertions(+), 34 deletions(-)

diff --git a/tensorflow/contrib/kfac/python/kernel_tests/fisher_blocks_test.py b/tensorflow/contrib/kfac/python/kernel_tests/fisher_blocks_test.py
index 80855da2e9..85ac08a1eb 100644
--- a/tensorflow/contrib/kfac/python/kernel_tests/fisher_blocks_test.py
+++ b/tensorflow/contrib/kfac/python/kernel_tests/fisher_blocks_test.py
@@ -356,50 +356,51 @@ class FullyConnectedKFACBasicFBTest(test.TestCase):
       random_seed.set_random_seed(200)
       inputs = array_ops.constant([1., 2.])
       outputs = array_ops.constant([3., 4.])
-      block = fb.FullyConnectedKFACBasicFB(lc.LayerCollection(), inputs,
-                                           outputs)
+      block = fb.FullyConnectedKFACBasicFB(lc.LayerCollection())
+      block.register_additional_minibatch(inputs, outputs)
 
-      self.assertAllEqual(outputs, block.tensors_to_compute_grads())
+      self.assertAllEqual([outputs], block.tensors_to_compute_grads())
 
   def testInstantiateFactorsHasBias(self):
     with ops.Graph().as_default():
       random_seed.set_random_seed(200)
       inputs = array_ops.constant([[1., 2.], [3., 4.]])
       outputs = array_ops.constant([[3., 4.], [5., 6.]])
-      block = fb.FullyConnectedKFACBasicFB(
-          lc.LayerCollection(), inputs, outputs, has_bias=True)
+      block = fb.FullyConnectedKFACBasicFB(lc.LayerCollection(), has_bias=True)
+      block.register_additional_minibatch(inputs, outputs)
 
       grads = outputs**2
-      block.instantiate_factors((grads,), 0.5)
+      block.instantiate_factors(([grads],), 0.5)
 
   def testInstantiateFactorsNoBias(self):
     with ops.Graph().as_default():
       random_seed.set_random_seed(200)
       inputs = array_ops.constant([[1., 2.], [3., 4.]])
       outputs = array_ops.constant([[3., 4.], [5., 6.]])
-      block = fb.FullyConnectedKFACBasicFB(
-          lc.LayerCollection(), inputs, outputs, has_bias=False)
+      block = fb.FullyConnectedKFACBasicFB(lc.LayerCollection(), has_bias=False)
+      block.register_additional_minibatch(inputs, outputs)
 
       grads = outputs**2
-      block.instantiate_factors((grads,), 0.5)
+      block.instantiate_factors(([grads],), 0.5)
 
   def testMultiplyInverseTuple(self):
     with ops.Graph().as_default(), self.test_session() as sess:
       random_seed.set_random_seed(200)
       inputs = array_ops.constant([[1., 2., 3.], [3., 4., 5.], [5., 6., 7.]])
       outputs = array_ops.constant([[3., 4.], [5., 6.]])
-      block = fb.FullyConnectedKFACBasicFB(
-          lc.LayerCollection(), inputs, outputs, has_bias=False)
+      block = fb.FullyConnectedKFACBasicFB(lc.LayerCollection(), has_bias=False)
+      block.register_additional_minibatch(inputs, outputs)
       grads = outputs**2
-      block.instantiate_factors((grads,), 0.5)
+      block.instantiate_factors(([grads],), 0.5)
 
       # Make sure our inverse is something other than the identity.
       sess.run(tf_variables.global_variables_initializer())
       sess.run(block._input_factor.make_inverse_update_ops())
       sess.run(block._output_factor.make_inverse_update_ops())
 
-      vector = (np.arange(2, 6).reshape(2, 2).astype(np.float32), np.arange(
-          1, 3).reshape(2, 1).astype(np.float32))
+      vector = (
+          np.arange(2, 6).reshape(2, 2).astype(np.float32),  #
+          np.arange(1, 3).reshape(2, 1).astype(np.float32))
       output = block.multiply_inverse((array_ops.constant(vector[0]),
                                        array_ops.constant(vector[1])))
 
@@ -413,10 +414,10 @@ class FullyConnectedKFACBasicFBTest(test.TestCase):
       random_seed.set_random_seed(200)
       inputs = array_ops.constant([[1., 2.], [3., 4.]])
       outputs = array_ops.constant([[3., 4.], [5., 6.]])
-      block = fb.FullyConnectedKFACBasicFB(
-          lc.LayerCollection(), inputs, outputs, has_bias=False)
+      block = fb.FullyConnectedKFACBasicFB(lc.LayerCollection(), has_bias=False)
+      block.register_additional_minibatch(inputs, outputs)
       grads = outputs**2
-      block.instantiate_factors((grads,), 0.5)
+      block.instantiate_factors(([grads],), 0.5)
 
       # Make sure our inverse is something other than the identity.
       sess.run(tf_variables.global_variables_initializer())
@@ -436,11 +437,11 @@ class FullyConnectedKFACBasicFBTest(test.TestCase):
       inputs = array_ops.zeros([32, input_dim])
       outputs = array_ops.zeros([32, output_dim])
       params = array_ops.zeros([input_dim, output_dim])
-      block = fb.FullyConnectedKFACBasicFB(
-          lc.LayerCollection(), inputs, outputs, has_bias=False)
+      block = fb.FullyConnectedKFACBasicFB(lc.LayerCollection(), has_bias=False)
+      block.register_additional_minibatch(inputs, outputs)
       grads = outputs**2
       damping = 0.  # This test is only valid without damping.
-      block.instantiate_factors((grads,), damping)
+      block.instantiate_factors(([grads],), damping)
 
       sess.run(state_ops.assign(block._input_factor._cov, _make_psd(3)))
       sess.run(state_ops.assign(block._output_factor._cov, _make_psd(2)))
diff --git a/tensorflow/contrib/kfac/python/ops/fisher_blocks.py b/tensorflow/contrib/kfac/python/ops/fisher_blocks.py
index 5e822b5fe3..754c2cc853 100644
--- a/tensorflow/contrib/kfac/python/ops/fisher_blocks.py
+++ b/tensorflow/contrib/kfac/python/ops/fisher_blocks.py
@@ -367,7 +367,7 @@ class ConvDiagonalFB(FisherBlock):
         (self._strides[1] * self._strides[2]))
 
     if NORMALIZE_DAMPING_POWER:
-      damping /= self._num_locations**NORMALIZE_DAMPING_POWER
+      damping /= self._num_locations ** NORMALIZE_DAMPING_POWER
     self._damping = damping
 
     self._factor = self._layer_collection.make_or_get_factor(
@@ -478,34 +478,60 @@ class FullyConnectedKFACBasicFB(KroneckerProductFB):
   K-FAC paper (https://arxiv.org/abs/1503.05671)
   """
 
-  def __init__(self, layer_collection, inputs, outputs, has_bias=False):
+  def __init__(self, layer_collection, has_bias=False):
     """Creates a FullyConnectedKFACBasicFB block.
 
     Args:
       layer_collection: The collection of all layers in the K-FAC approximate
           Fisher information matrix to which this FisherBlock belongs.
-      inputs: The Tensor of input activations to this layer.
-      outputs: The Tensor of output pre-activations from this layer.
       has_bias: Whether the component Kronecker factors have an additive bias.
           (Default: False)
     """
-    self._inputs = inputs
-    self._outputs = outputs
+    self._inputs = []
+    self._outputs = []
     self._has_bias = has_bias
 
     super(FullyConnectedKFACBasicFB, self).__init__(layer_collection)
 
   def instantiate_factors(self, grads_list, damping):
-    self._input_factor = self._layer_collection.make_or_get_factor(
-        fisher_factors.FullyConnectedKroneckerFactor,
-        ((self._inputs,), self._has_bias))
-    self._output_factor = self._layer_collection.make_or_get_factor(
-        fisher_factors.FullyConnectedKroneckerFactor, (grads_list,))
+    """Instantiate Kronecker Factors for this FisherBlock.
+
+    Args:
+      grads_list: List of list of Tensors. grads_list[i][j] is the
+        gradient of the loss with respect to 'outputs' from source 'i' and
+        tower 'j'. Each Tensor has shape [tower_minibatch_size, output_size].
+      damping: 0-D Tensor or float. 'damping' * identity is approximately added
+        to this FisherBlock's Fisher approximation.
+    """
+    # TODO(b/68033310): Validate which of,
+    #   (1) summing on a single device (as below), or
+    #   (2) on each device in isolation and aggregating
+    # is faster.
+    inputs = _concat_along_batch_dim(self._inputs)
+    grads_list = tuple(_concat_along_batch_dim(grads) for grads in grads_list)
+
+    self._input_factor = self._layer_collection.make_or_get_factor(  #
+        fisher_factors.FullyConnectedKroneckerFactor,  #
+        ((inputs,), self._has_bias))
+    self._output_factor = self._layer_collection.make_or_get_factor(  #
+        fisher_factors.FullyConnectedKroneckerFactor,  #
+        (grads_list,))
     self._register_damped_input_and_output_inverses(damping)
 
   def tensors_to_compute_grads(self):
     return self._outputs
 
+  def register_additional_minibatch(self, inputs, outputs):
+    """Registers an additional minibatch to the FisherBlock.
+
+    Args:
+      inputs: Tensor of shape [batch_size, input_size]. Inputs to the
+        matrix-multiply.
+      outputs: Tensor of shape [batch_size, output_size]. Layer preactivations.
+    """
+    self._inputs.append(inputs)
+    self._outputs.append(outputs)
+
 
 class ConvKFCBasicFB(KroneckerProductFB):
   """FisherBlock for 2D convolutional layers using the basic KFC approx.
diff --git a/tensorflow/contrib/kfac/python/ops/fisher_factors.py b/tensorflow/contrib/kfac/python/ops/fisher_factors.py
index 86a1782fcf..b8b524406c 100644
--- a/tensorflow/contrib/kfac/python/ops/fisher_factors.py
+++ b/tensorflow/contrib/kfac/python/ops/fisher_factors.py
@@ -573,6 +573,14 @@ class FullyConnectedKroneckerFactor(InverseProvidingFactor):
   """
 
   def __init__(self, tensors, has_bias=False):
+    """Instantiate FullyConnectedKroneckerFactor.
+
+    Args:
+      tensors: List of Tensors of shape [batch_size, n]. Represents either a
+        layer's inputs or its output's gradients.
+      has_bias: bool. If True, assume this factor is for the layer's inputs and
+        append '1' to each row.
+    """
     # The tensor argument is either a tensor of input activations or a tensor of
     # output pre-activation gradients.
     self._has_bias = has_bias
diff --git a/tensorflow/contrib/kfac/python/ops/layer_collection.py b/tensorflow/contrib/kfac/python/ops/layer_collection.py
index 10ef554351..ceb1131f28 100644
--- a/tensorflow/contrib/kfac/python/ops/layer_collection.py
+++ b/tensorflow/contrib/kfac/python/ops/layer_collection.py
@@ -255,9 +255,9 @@ class LayerCollection(object):
                                approx=APPROX_KRONECKER_NAME):
     has_bias = isinstance(params, (tuple, list))
     if approx == APPROX_KRONECKER_NAME:
-      self.register_block(params,
-                          fb.FullyConnectedKFACBasicFB(self, inputs, outputs,
-                                                       has_bias))
+      block = fb.FullyConnectedKFACBasicFB(self, has_bias)
+      block.register_additional_minibatch(inputs, outputs)
+      self.register_block(params, block)
     elif approx == APPROX_DIAGONAL_NAME:
       block = fb.FullyConnectedDiagonalFB(self, has_bias)
       block.register_additional_minibatch(inputs, outputs)
-- 
GitLab


From 434695921de7cfd713b789533173e1e0c3fc7691 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 23 Oct 2017 08:00:39 -0700
Subject: [PATCH 668/909] K-FAC: _check_registration() supports multiple
 towers.

PiperOrigin-RevId: 173115870
---
 .../kernel_tests/layer_collection_test.py     | 14 ++++++--
 tensorflow/contrib/kfac/python/ops/BUILD      |  2 ++
 .../contrib/kfac/python/ops/fisher_blocks.py  | 34 +++++++++++++++++++
 .../kfac/python/ops/layer_collection.py       | 10 +++---
 4 files changed, 54 insertions(+), 6 deletions(-)

diff --git a/tensorflow/contrib/kfac/python/kernel_tests/layer_collection_test.py b/tensorflow/contrib/kfac/python/kernel_tests/layer_collection_test.py
index b444e87170..1da811dc0a 100644
--- a/tensorflow/contrib/kfac/python/kernel_tests/layer_collection_test.py
+++ b/tensorflow/contrib/kfac/python/kernel_tests/layer_collection_test.py
@@ -313,10 +313,20 @@ class LayerCollectionTest(test.TestCase):
       self.assertTrue(all([var.name.startswith(scope) for var in variables]))
 
   def testGetUseCountMap(self):
+    """Ensure get_use_count_map() sums 'num_registered_minibatches'."""
+
+    class MockFisherBlock(object):
+
+      num_registered_minibatches = 2
+
     lc = layer_collection.LayerCollection()
-    lc.fisher_blocks = {'a': 1, ('a', 'c'): 2, ('b', 'c'): 2}
+    lc.fisher_blocks = {
+        'a': MockFisherBlock(),
+        ('a', 'c'): MockFisherBlock(),
+        ('b', 'c'): MockFisherBlock()
+    }
     use_count_map = lc.get_use_count_map()
-    self.assertDictEqual({'a': 2, 'b': 1, 'c': 2}, use_count_map)
+    self.assertDictEqual({'a': 4, 'b': 2, 'c': 4}, use_count_map)
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/contrib/kfac/python/ops/BUILD b/tensorflow/contrib/kfac/python/ops/BUILD
index 8b82f6e314..5d5046c9ec 100644
--- a/tensorflow/contrib/kfac/python/ops/BUILD
+++ b/tensorflow/contrib/kfac/python/ops/BUILD
@@ -113,7 +113,9 @@ py_library(
         "//tensorflow/python:framework_ops",
         "//tensorflow/python:math_ops",
         "//tensorflow/python:platform",
+        "//tensorflow/python:util",
         "//tensorflow/python:variable_scope",
+        "@six_archive//:six",
     ],
 )
 
diff --git a/tensorflow/contrib/kfac/python/ops/fisher_blocks.py b/tensorflow/contrib/kfac/python/ops/fisher_blocks.py
index 754c2cc853..7ef755c35e 100644
--- a/tensorflow/contrib/kfac/python/ops/fisher_blocks.py
+++ b/tensorflow/contrib/kfac/python/ops/fisher_blocks.py
@@ -114,6 +114,14 @@ class FisherBlock(object):
     """
     pass
 
+  @abc.abstractproperty
+  def num_registered_minibatches(self):
+    """Number of minibatches registered for this FisherBlock.
+
+    Typically equal to the number of towers in a multi-tower setup.
+    """
+    pass
+
 
 class FullFB(FisherBlock):
   """FisherBlock using a full matrix estimate (no approximations).
@@ -164,6 +172,10 @@ class FullFB(FisherBlock):
   def tensors_to_compute_grads(self):
     return self._params
 
+  @property
+  def num_registered_minibatches(self):
+    return 1  # Multiple minibatches not supported.
+
 
 class NaiveDiagonalFB(FisherBlock):
   """FisherBlock using a diagonal matrix approximation.
@@ -209,6 +221,10 @@ class NaiveDiagonalFB(FisherBlock):
   def tensors_to_compute_grads(self):
     return self._params
 
+  @property
+  def num_registered_minibatches(self):
+    return 1  # Multiple minibatches not supported.
+
 
 class FullyConnectedDiagonalFB(FisherBlock):
   """FisherBlock for fully-connected (dense) layers using a diagonal approx.
@@ -305,6 +321,12 @@ class FullyConnectedDiagonalFB(FisherBlock):
     self._inputs.append(inputs)
     self._outputs.append(outputs)
 
+  @property
+  def num_registered_minibatches(self):
+    result = len(self._inputs)
+    assert result == len(self._outputs)
+    return result
+
 
 class ConvDiagonalFB(FisherBlock):
   """FisherBlock for convolutional layers using a diagonal approx.
@@ -400,6 +422,10 @@ class ConvDiagonalFB(FisherBlock):
     self._inputs.append(inputs)
     self._outputs.append(outputs)
 
+  @property
+  def num_registered_minibatches(self):
+    return len(self._inputs)
+
 
 class KroneckerProductFB(FisherBlock):
   """A base class for FisherBlocks with separate input and output factors.
@@ -532,6 +558,10 @@ class FullyConnectedKFACBasicFB(KroneckerProductFB):
     self._inputs.append(inputs)
     self._outputs.append(outputs)
 
+  @property
+  def num_registered_minibatches(self):
+    return 1  # Multiple minibatches not supported.
+
 
 class ConvKFCBasicFB(KroneckerProductFB):
   """FisherBlock for 2D convolutional layers using the basic KFC approx.
@@ -591,6 +621,10 @@ class ConvKFCBasicFB(KroneckerProductFB):
   def tensors_to_compute_grads(self):
     return self._outputs
 
+  @property
+  def num_registered_minibatches(self):
+    return 1  # Multiple minibatches not supported.
+
 
 def _concat_along_batch_dim(tensor_list):
   """Concatenate tensors along batch (first) dimension.
diff --git a/tensorflow/contrib/kfac/python/ops/layer_collection.py b/tensorflow/contrib/kfac/python/ops/layer_collection.py
index ceb1131f28..49279954dc 100644
--- a/tensorflow/contrib/kfac/python/ops/layer_collection.py
+++ b/tensorflow/contrib/kfac/python/ops/layer_collection.py
@@ -27,6 +27,8 @@ from __future__ import print_function
 from collections import defaultdict
 from collections import OrderedDict
 
+import six
+
 from tensorflow.contrib.kfac.python.ops import fisher_blocks as fb
 from tensorflow.contrib.kfac.python.ops import loss_functions as lf
 from tensorflow.contrib.kfac.python.ops import utils
@@ -82,8 +84,8 @@ class LayerParametersDict(OrderedDict):
     return key
 
 
-# TODO(duckworthd): add capability for LayerCollection to be "finalized"
-# and do this when it gets used by FisherEstimator / KfacOptimizer
+# TODO(b/68034464): add capability for LayerCollection to be "finalized"
+# and do this when it gets used by FisherEstimator / KfacOptimizer.
 
 
 class LayerCollection(object):
@@ -211,10 +213,10 @@ class LayerCollection(object):
   def get_use_count_map(self):
     """Returns a dict of variables to their number of registrations."""
     vars_to_uses = defaultdict(int)
-    for key in self.fisher_blocks.keys():
+    for key, block in six.iteritems(self.fisher_blocks):
       key = key if isinstance(key, (tuple, list)) else (key,)
       for k in key:
-        vars_to_uses[k] += 1
+        vars_to_uses[k] += block.num_registered_minibatches
     return vars_to_uses
 
   def get_blocks(self):
-- 
GitLab


From ed03c433ae5c89075717476f9bc1e731de656c6f Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Mon, 23 Oct 2017 15:26:02 +0000
Subject: [PATCH 669/909] Add int64 type `multiples` support for TileGrad.

This fix is a follow up of #13884 to add int64 type of `multiples`
support for TileGrad for completeness.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/core/kernels/tile_ops.cc | 89 +++++++++++++++++++----------
 1 file changed, 58 insertions(+), 31 deletions(-)

diff --git a/tensorflow/core/kernels/tile_ops.cc b/tensorflow/core/kernels/tile_ops.cc
index 4c496a12c2..fa5afe6a31 100644
--- a/tensorflow/core/kernels/tile_ops.cc
+++ b/tensorflow/core/kernels/tile_ops.cc
@@ -248,7 +248,7 @@ TF_CALL_int64(HANDLE_TYPE_NAME_SYCL);
 #undef HANDLE_CASE
 
 // --------------------------------------------------------------------------
-template <typename Device>
+template <typename Device, typename Tmultiples>
 class TileGradientOp : public OpKernel {
  public:
   explicit TileGradientOp(OpKernelConstruction* context) : OpKernel(context) {}
@@ -273,10 +273,10 @@ class TileGradientOp : public OpKernel {
       return;
     }
 
-    const gtl::ArraySlice<int32> multiples_array(multiples.flat<int32>().data(),
-                                                 input_dims);
+    const gtl::ArraySlice<Tmultiples> multiples_array(
+        multiples.flat<Tmultiples>().data(), input_dims);
     TensorShape output_shape;
-    std::vector<int32> input_dim_size_vec;
+    std::vector<Tmultiples> input_dim_size_vec;
     for (int i = 0; i < input_dims; ++i) {
       OP_REQUIRES(
           context, multiples_array[i] > 0,
@@ -337,19 +337,19 @@ class TileGradientOp : public OpKernel {
  private:
   template <DataType DT, int NDIM>
   void HandleCase(OpKernelContext* context,
-                  const std::vector<int32>& input_dims,
-                  const gtl::ArraySlice<int32>& multiples_array,
+                  const std::vector<Tmultiples>& input_dims,
+                  const gtl::ArraySlice<Tmultiples>& multiples_array,
                   Tensor* result);
 
   template <DataType DT, int NDIM>
   void HandleCaseImpl(OpKernelContext* context,
-                      const std::vector<int32>& input_dims,
-                      const gtl::ArraySlice<int32>& multiples_array,
+                      const std::vector<Tmultiples>& input_dims,
+                      const gtl::ArraySlice<Tmultiples>& multiples_array,
                       Tensor* result) {
     typedef typename EnumToDataType<DT>::Type T;
 
     bool reduction_only = true;
-    std::vector<int> reduction_dims;
+    std::vector<Tmultiples> reduction_dims;
 
     for (int i = 0; i < NDIM; ++i) {
       if (input_dims[i] > multiples_array[i] && multiples_array[i] > 1) {
@@ -411,7 +411,8 @@ class TileGradientOp : public OpKernel {
 
   template <typename T, int NDIM, int REDUCENDIM>
   void HandleReduce(OpKernelContext* context,
-                    const std::vector<int32>& reduce_dim_in, Tensor* result) {
+                    const std::vector<Tmultiples>& reduce_dim_in,
+                    Tensor* result) {
     static_assert(NDIM >= REDUCENDIM, "Too many reduced dimensions");
     Eigen::DSizes<Eigen::DenseIndex, REDUCENDIM> reduce_dim;
     Eigen::DSizes<Eigen::DenseIndex, NDIM> reshape_dim;
@@ -432,34 +433,41 @@ class TileGradientOp : public OpKernel {
   TF_DISALLOW_COPY_AND_ASSIGN(TileGradientOp);
 };
 
-template <typename Device>
+template <typename Device, typename Tmultiples>
 template <DataType DT, int NDIM>
-inline void TileGradientOp<Device>::HandleCase(
-    OpKernelContext* context, const std::vector<int32>& input_dims,
-    const gtl::ArraySlice<int32>& multiples_array, Tensor* result) {
+inline void TileGradientOp<Device, Tmultiples>::HandleCase(
+    OpKernelContext* context, const std::vector<Tmultiples>& input_dims,
+    const gtl::ArraySlice<Tmultiples>& multiples_array, Tensor* result) {
   LOG(FATAL) << "TileGradientOp: Invalid combination of Device, DT and NDIM: "
              << MakeTypeIndex<Device>().name() << ", " << DataTypeString(DT)
              << ", " << NDIM;
 }
 
-#define HANDLE_CASE(device, T, dtype, ndim)                                    \
+#define HANDLE_CASE(device, T, dtype, Tmultiples, ndim)                        \
   template <>                                                                  \
   template <>                                                                  \
-  void TileGradientOp<device>::HandleCase<dtype, ndim>(                        \
-      OpKernelContext * context, const std::vector<int32>& input_dims,         \
-      const gtl::ArraySlice<int32>& multiples_array, Tensor* result) {         \
+  void TileGradientOp<device, Tmultiples>::HandleCase<dtype, ndim>(            \
+      OpKernelContext * context, const std::vector<Tmultiples>& input_dims,    \
+      const gtl::ArraySlice<Tmultiples>& multiples_array, Tensor* result) {    \
     HandleCaseImpl<dtype, ndim>(context, input_dims, multiples_array, result); \
   }
 
 // 0-D handled specially above
-#define HANDLE_CASE_DIM(device, T, dtype) \
-  HANDLE_CASE(device, T, dtype, 1);       \
-  HANDLE_CASE(device, T, dtype, 2);       \
-  HANDLE_CASE(device, T, dtype, 3);       \
-  HANDLE_CASE(device, T, dtype, 4);       \
-  HANDLE_CASE(device, T, dtype, 5);       \
-  HANDLE_CASE(device, T, dtype, 6);       \
-  HANDLE_CASE(device, T, dtype, 7);
+#define HANDLE_CASE_DIM(device, T, dtype)  \
+  HANDLE_CASE(device, T, dtype, int32, 1); \
+  HANDLE_CASE(device, T, dtype, int32, 2); \
+  HANDLE_CASE(device, T, dtype, int32, 3); \
+  HANDLE_CASE(device, T, dtype, int32, 4); \
+  HANDLE_CASE(device, T, dtype, int32, 5); \
+  HANDLE_CASE(device, T, dtype, int32, 6); \
+  HANDLE_CASE(device, T, dtype, int32, 7); \
+  HANDLE_CASE(device, T, dtype, int64, 1); \
+  HANDLE_CASE(device, T, dtype, int64, 2); \
+  HANDLE_CASE(device, T, dtype, int64, 3); \
+  HANDLE_CASE(device, T, dtype, int64, 4); \
+  HANDLE_CASE(device, T, dtype, int64, 5); \
+  HANDLE_CASE(device, T, dtype, int64, 6); \
+  HANDLE_CASE(device, T, dtype, int64, 7);
 
 #define HANDLE_TYPE_NAME_CPU(T) \
   HANDLE_CASE_DIM(CPUDevice, T, DataTypeToEnum<T>::value);
@@ -514,9 +522,16 @@ REGISTER_KERNEL_BUILDER(Name("Tile")
                             .HostMemory("multiples")
                             .TypeConstraint<int64>("Tmultiples"),
                         TileOp<CPUDevice, int64>);
-REGISTER_KERNEL_BUILDER(
-    Name("TileGrad").Device(DEVICE_CPU).HostMemory("multiples"),
-    TileGradientOp<CPUDevice>);
+REGISTER_KERNEL_BUILDER(Name("TileGrad")
+                            .Device(DEVICE_CPU)
+                            .HostMemory("multiples")
+                            .TypeConstraint<int32>("Tmultiples"),
+                        TileGradientOp<CPUDevice, int32>);
+REGISTER_KERNEL_BUILDER(Name("TileGrad")
+                            .Device(DEVICE_CPU)
+                            .HostMemory("multiples")
+                            .TypeConstraint<int64>("Tmultiples"),
+                        TileGradientOp<CPUDevice, int64>);
 
 #if GOOGLE_CUDA
 #define REGISTER_GPU(type)                                         \
@@ -537,7 +552,13 @@ REGISTER_KERNEL_BUILDER(
                               .TypeConstraint<type>("T")           \
                               .TypeConstraint<int32>("Tmultiples") \
                               .HostMemory("multiples"),            \
-                          TileGradientOp<GPUDevice>);
+                          TileGradientOp<GPUDevice, int32>);       \
+  REGISTER_KERNEL_BUILDER(Name("TileGrad")                         \
+                              .Device(DEVICE_GPU)                  \
+                              .TypeConstraint<type>("T")           \
+                              .TypeConstraint<int64>("Tmultiples") \
+                              .HostMemory("multiples"),            \
+                          TileGradientOp<GPUDevice, int64>);
 
 TF_CALL_float(REGISTER_GPU);
 TF_CALL_double(REGISTER_GPU);
@@ -569,7 +590,13 @@ TF_CALL_complex128(REGISTER_GPU)
                               .TypeConstraint<type>("T")           \
                               .TypeConstraint<int32>("Tmultiples") \
                               .HostMemory("multiples"),            \
-                          TileGradientOp<SYCLDevice>);
+                          TileGradientOp<SYCLDevice, int32>);      \
+  REGISTER_KERNEL_BUILDER(Name("TileGrad")                         \
+                              .Device(DEVICE_SYCL)                 \
+                              .TypeConstraint<type>("T")           \
+                              .TypeConstraint<int64>("Tmultiples") \
+                              .HostMemory("multiples"),            \
+                          TileGradientOp<SYCLDevice, int64>);
 
     TF_CALL_float(REGISTER_SYCL);
 TF_CALL_double(REGISTER_SYCL);
-- 
GitLab


From 2845bfcd64cea4405135b3c7034e9aa28896dff4 Mon Sep 17 00:00:00 2001
From: Tim Harley <tharley@google.com>
Date: Mon, 23 Oct 2017 08:54:57 -0700
Subject: [PATCH 670/909] Avoid listing all modified Enter/RefEnter nodes on
 INFO, use VLOG(1) instead.

Leave a single, simple, message on INFO.

PiperOrigin-RevId: 173121726
---
 tensorflow/core/debug/debug_graph_utils.cc | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/tensorflow/core/debug/debug_graph_utils.cc b/tensorflow/core/debug/debug_graph_utils.cc
index 2559808b59..4539ea5c0c 100644
--- a/tensorflow/core/debug/debug_graph_utils.cc
+++ b/tensorflow/core/debug/debug_graph_utils.cc
@@ -221,21 +221,26 @@ Status DebugNodeInserter::InsertNodes(
 }
 
 void DebugNodeInserter::DeparallelizeWhileLoops(Graph* graph, Device* device) {
+  bool deparallelized_a_loop = false;
   for (Node* node : graph->nodes()) {
     if (node->IsEnter()) {
       const AttrValue* parallel_iterations =
           node->attrs().Find("parallel_iterations");
       if (parallel_iterations && parallel_iterations->i() > 1) {
-        LOG(INFO) << "For debugging, tfdbg is changing the "
-                  << "parallel_iterations attribute of the Enter/RefEnter "
-                  << "node \"" << node->name() << "\" on device \""
-                  << device->name() << "\" from " << parallel_iterations->i()
-                  << " to 1. (This does not affect subsequent non-debug "
-                  << "runs.)";
+        deparallelized_a_loop = true;
+        VLOG(1) << "Changing the parallel_iterations attribute of the "
+                << "Enter/RefEnter node \"" << node->name() << "\" on device \""
+                << device->name() << "\" from " << parallel_iterations->i()
+                << " to 1.";
         node->AddAttr<int64>("parallel_iterations", 1);
       }
     }
   }
+  if (deparallelized_a_loop) {
+    LOG(INFO) << "For debugging, tfdbg has set the parallel_iterations "
+              << "attribute of all scheduled Enter/RefEnter nodes to 1. (This "
+              << "does not affect subsequent non-debug runs.)";
+  }
 }
 
 // static
-- 
GitLab


From 4f7503a876e20e6d58c9aec3f44214b98bcfdbbb Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 23 Oct 2017 08:55:00 -0700
Subject: [PATCH 671/909] K-FAC: Support for registering multiple minibatches
 with register_fully_connected()

PiperOrigin-RevId: 173121735
---
 .../kernel_tests/layer_collection_test.py     | 67 +++++++++++++++++++
 .../kfac/python/ops/layer_collection.py       | 64 +++++++++++++++---
 .../kfac/python/ops/layer_collection_lib.py   |  1 +
 3 files changed, 122 insertions(+), 10 deletions(-)

diff --git a/tensorflow/contrib/kfac/python/kernel_tests/layer_collection_test.py b/tensorflow/contrib/kfac/python/kernel_tests/layer_collection_test.py
index 1da811dc0a..432937d803 100644
--- a/tensorflow/contrib/kfac/python/kernel_tests/layer_collection_test.py
+++ b/tensorflow/contrib/kfac/python/kernel_tests/layer_collection_test.py
@@ -282,6 +282,73 @@ class LayerCollectionTest(test.TestCase):
       single_loss = sess.run(lc.total_loss())
       self.assertAlmostEqual(7.6983433, single_loss)
 
+  def testRegisterFullyConnectedReuse(self):
+    """Ensure the 'reuse' keyword argument function as intended."""
+    with ops.Graph().as_default():
+      inputs = [
+          array_ops.ones([2, 10]),  #
+          array_ops.zeros([5, 10])
+      ]
+      outputs = [
+          array_ops.zeros([2, 5]),  #
+          array_ops.ones([5, 5])
+      ]
+      params = (
+          variable_scope.get_variable('w', [10, 5]),  #
+          variable_scope.get_variable('b', [5]))
+
+      # Fails on second if reuse=False.
+      lc = layer_collection.LayerCollection()
+      lc.register_fully_connected(params, inputs[0], outputs[0])
+      with self.assertRaises(ValueError):
+        lc.register_fully_connected(params, inputs[1], outputs[1], reuse=False)
+
+      # Succeeds on second if reuse=True.
+      lc = layer_collection.LayerCollection()
+      lc.register_fully_connected(params, inputs[0], outputs[0])
+      lc.register_fully_connected(params, inputs[1], outputs[1], reuse=True)
+
+      # Fails on second if reuse=VARIABLE_SCOPE and no variable reuse.
+      lc = layer_collection.LayerCollection()
+      lc.register_fully_connected(params, inputs[0], outputs[0])
+      with self.assertRaises(ValueError):
+        lc.register_fully_connected(
+            params,
+            inputs[1],
+            outputs[1],
+            reuse=layer_collection.VARIABLE_SCOPE)
+
+      # Succeeds on second if reuse=VARIABLE_SCOPE and variable reuse.
+      lc = layer_collection.LayerCollection()
+      lc.register_fully_connected(params, inputs[0], outputs[0])
+      with variable_scope.variable_scope(
+          variable_scope.get_variable_scope(), reuse=True):
+        lc.register_fully_connected(
+            params,
+            inputs[1],
+            outputs[1],
+            reuse=layer_collection.VARIABLE_SCOPE)
+
+      # Fails if block type changes.
+      lc = layer_collection.LayerCollection()
+      lc.register_fully_connected(
+          params,
+          inputs[0],
+          outputs[0],
+          approx=layer_collection.APPROX_KRONECKER_NAME)
+      with self.assertRaises(ValueError):
+        lc.register_fully_connected(
+            params,
+            inputs[1],
+            outputs[1],
+            approx=layer_collection.APPROX_DIAGONAL_NAME,
+            reuse=True)
+
+      # Fails if reuse requested but no FisherBlock exists.
+      lc = layer_collection.LayerCollection()
+      with self.assertRaises(KeyError):
+        lc.register_fully_connected(params, inputs[0], outputs[0], reuse=True)
+
   def testMakeOrGetFactor(self):
     with ops.Graph().as_default():
       random_seed.set_random_seed(200)
diff --git a/tensorflow/contrib/kfac/python/ops/layer_collection.py b/tensorflow/contrib/kfac/python/ops/layer_collection.py
index 49279954dc..cd711d0561 100644
--- a/tensorflow/contrib/kfac/python/ops/layer_collection.py
+++ b/tensorflow/contrib/kfac/python/ops/layer_collection.py
@@ -39,10 +39,15 @@ from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.util import nest
 
 
+# Names for various approximations that can be requested for Fisher blocks.
 APPROX_KRONECKER_NAME = "kron"
 APPROX_DIAGONAL_NAME = "diagonal"
 APPROX_FULL_NAME = "full"
 
+# Possible value for 'reuse' keyword argument. Sets 'reuse' to
+# tf.get_variable_scope().reuse.
+VARIABLE_SCOPE = "VARIABLE_SCOPE"
+
 # TODO(jamesmartens): need to add find_canonical_output back into this somewhere
 
 
@@ -254,18 +259,57 @@ class LayerCollection(object):
                                params,
                                inputs,
                                outputs,
-                               approx=APPROX_KRONECKER_NAME):
+                               approx=APPROX_KRONECKER_NAME,
+                               reuse=VARIABLE_SCOPE):
+    """Registers a fully connnected layer.
+
+    Args:
+      params: Tensor or 2-tuple of Tensors corresponding to weight and bias of
+        this layer. Weight matrix should have shape [input_size, output_size].
+        Bias should have shape [output_size].
+      inputs: Tensor of shape [batch_size, input_size]. Inputs to layer.
+      outputs: Tensor of shape [batch_size, output_size]. Preactivations
+        produced by layer.
+      approx: str. One of APPROX_KRONECKER_NAME or APPROX_DIAGONAL_NAME.
+      reuse: bool or str.  If True, reuse an existing FisherBlock. If False,
+        create a new FisherBlock.  If VARIABLE_SCOPE, use
+        tf.get_variable_scope().reuse.
+
+    Raises:
+      ValueError: For improper value to 'approx'.
+      KeyError: If reuse == True but no FisherBlock found for 'params'.
+      ValueError: If reuse == True and FisherBlock found but of the wrong type.
+    """
+    approx_to_block_types = {
+        APPROX_KRONECKER_NAME: fb.FullyConnectedKFACBasicFB,
+        APPROX_DIAGONAL_NAME: fb.FullyConnectedDiagonalFB,
+    }
+
+    if approx not in approx_to_block_types:
+      raise ValueError("Bad value {} for approx.".format(approx))
+
+    block_type = approx_to_block_types[approx]
     has_bias = isinstance(params, (tuple, list))
-    if approx == APPROX_KRONECKER_NAME:
-      block = fb.FullyConnectedKFACBasicFB(self, has_bias)
-      block.register_additional_minibatch(inputs, outputs)
-      self.register_block(params, block)
-    elif approx == APPROX_DIAGONAL_NAME:
-      block = fb.FullyConnectedDiagonalFB(self, has_bias)
-      block.register_additional_minibatch(inputs, outputs)
-      self.register_block(params, block)
+
+    if reuse == VARIABLE_SCOPE:
+      reuse = variable_scope.get_variable_scope().reuse
+
+    if reuse:
+      block = self.fisher_blocks.get(params, None)
+      if block is None:
+        raise KeyError(
+            "Reuse requested but no FisherBlock found for params {}.".format(
+                params))
+      if not isinstance(block, block_type):
+        raise ValueError(
+            "Requested block of type {} but block of type {} already exists "
+            "for params {}.".format(block_type, type(block), params))
+
     else:
-      raise ValueError("Bad value {} for approx.".format(approx))
+      block = block_type(self, has_bias)
+      self.register_block(params, block)
+
+    block.register_additional_minibatch(inputs, outputs)
 
   def register_conv2d(self, params, strides, padding, inputs, outputs,
                       approx=APPROX_KRONECKER_NAME):
diff --git a/tensorflow/contrib/kfac/python/ops/layer_collection_lib.py b/tensorflow/contrib/kfac/python/ops/layer_collection_lib.py
index 63a9b173bc..d6bf61a210 100644
--- a/tensorflow/contrib/kfac/python/ops/layer_collection_lib.py
+++ b/tensorflow/contrib/kfac/python/ops/layer_collection_lib.py
@@ -35,6 +35,7 @@ _allowed_symbols = [
     "APPROX_KRONECKER_NAME",
     "APPROX_DIAGONAL_NAME",
     "APPROX_FULL_NAME",
+    "VARIABLE_SCOPE",
 ]
 
 remove_undocumented(__name__, allowed_exception_list=_allowed_symbols)
-- 
GitLab


From fc56349b7f1afec33c88358a06dab12dda5736a4 Mon Sep 17 00:00:00 2001
From: Derek Murray <mrry@google.com>
Date: Mon, 23 Oct 2017 09:34:30 -0700
Subject: [PATCH 672/909] [tf.data] Convert dataset arguments to tensors as
 early as possible.

This change raises a `TypeError` earlier if (for example) the `batch_size`
argument to `Dataset.batch()` has the incorrect type.

PiperOrigin-RevId: 173126678
---
 tensorflow/python/data/ops/dataset_ops.py | 29 +++++++++++++----------
 1 file changed, 17 insertions(+), 12 deletions(-)

diff --git a/tensorflow/python/data/ops/dataset_ops.py b/tensorflow/python/data/ops/dataset_ops.py
index 5f2e6296a8..151556994f 100644
--- a/tensorflow/python/data/ops/dataset_ops.py
+++ b/tensorflow/python/data/ops/dataset_ops.py
@@ -1057,21 +1057,21 @@ class RangeDataset(Dataset):
   def _parse_args(self, *args):
     if len(args) == 1:
       self._start = self._build_tensor(0, "start")
-      self._stop = args[0]
+      self._stop = self._build_tensor(args[0], "stop")
       self._step = self._build_tensor(1, "step")
     elif len(args) == 2:
-      self._start = args[0]
-      self._stop = args[1]
+      self._start = self._build_tensor(args[0], "start")
+      self._stop = self._build_tensor(args[1], "stop")
       self._step = self._build_tensor(1, "step")
     elif len(args) == 3:
-      self._start = args[0]
-      self._stop = args[1]
-      self._step = args[2]
+      self._start = self._build_tensor(args[0], "start")
+      self._stop = self._build_tensor(args[1], "stop")
+      self._step = self._build_tensor(args[2], "step")
     else:
       raise ValueError("Invalid arguments to RangeDataset: %s" % str(args))
 
   def _build_tensor(self, int64_value, name):
-    return constant_op.constant(int64_value, dtype=dtypes.int64, name=name)
+    return ops.convert_to_tensor(int64_value, dtype=dtypes.int64, name=name)
 
   def _as_variant_tensor(self):
     return gen_dataset_ops.range_dataset(
@@ -1217,7 +1217,8 @@ class BatchDataset(Dataset):
     """See `Dataset.batch()` for details."""
     super(BatchDataset, self).__init__()
     self._input_dataset = input_dataset
-    self._batch_size = batch_size
+    self._batch_size = ops.convert_to_tensor(batch_size, dtype=dtypes.int64,
+                                             name="batch_size")
 
   def _as_variant_tensor(self):
     return gen_dataset_ops.batch_dataset(
@@ -1285,7 +1286,8 @@ class PaddedBatchDataset(Dataset):
     """See `Dataset.batch()` for details."""
     super(PaddedBatchDataset, self).__init__()
     self._input_dataset = input_dataset
-    self._batch_size = batch_size
+    self._batch_size = ops.convert_to_tensor(batch_size, dtype=dtypes.int64,
+                                             name="batch_size")
     padding_values = (padding_values if padding_values is not None else
                       self._default_padding(input_dataset))
     self._padded_shapes = nest.map_structure_up_to(
@@ -1509,8 +1511,10 @@ class InterleaveDataset(Dataset):
     self._map_func = tf_map_func
     self._map_func.add_to_graph(ops.get_default_graph())
 
-    self._cycle_length = ops.convert_to_tensor(cycle_length, dtype=dtypes.int64)
-    self._block_length = ops.convert_to_tensor(block_length, dtype=dtypes.int64)
+    self._cycle_length = ops.convert_to_tensor(cycle_length, dtype=dtypes.int64,
+                                               name="cycle_length")
+    self._block_length = ops.convert_to_tensor(block_length, dtype=dtypes.int64,
+                                               name="block_length")
 
   def _as_variant_tensor(self):
     return gen_dataset_ops.interleave_dataset(
@@ -1587,7 +1591,8 @@ class PrefetchDataset(Dataset):
     """See `Dataset.prefetch()` for details."""
     super(PrefetchDataset, self).__init__()
     self._input_dataset = input_dataset
-    self._buffer_size = ops.convert_to_tensor(buffer_size, dtype=dtypes.int64)
+    self._buffer_size = ops.convert_to_tensor(buffer_size, dtype=dtypes.int64,
+                                              name="buffer_size")
 
   def _as_variant_tensor(self):
     return gen_dataset_ops.prefetch_dataset(
-- 
GitLab


From 46ab25e4de87bc6873697e28f747fb8ae3579755 Mon Sep 17 00:00:00 2001
From: David Majnemer <majnemer@google.com>
Date: Mon, 23 Oct 2017 09:36:32 -0700
Subject: [PATCH 673/909] [XLA] Add support for convolutions with no spatial
 dimensions

PiperOrigin-RevId: 173126950
---
 .../xla/client/computation_builder.cc         |  2 +-
 .../xla/service/algebraic_simplifier.cc       | 27 ++++++++++++-------
 .../xla/service/algebraic_simplifier.h        | 11 +++++---
 .../compiler/xla/service/hlo_cost_analysis.cc |  4 ++-
 .../compiler/xla/service/hlo_evaluator.cc     |  2 +-
 .../compiler/xla/service/shape_inference.cc   | 10 ++-----
 6 files changed, 32 insertions(+), 24 deletions(-)

diff --git a/tensorflow/compiler/xla/client/computation_builder.cc b/tensorflow/compiler/xla/client/computation_builder.cc
index dcbdb3525e..b9977fb2f8 100644
--- a/tensorflow/compiler/xla/client/computation_builder.cc
+++ b/tensorflow/compiler/xla/client/computation_builder.cc
@@ -663,7 +663,7 @@ bool ComputationBuilder::VerifyConvolution(
     return false;
   }
   int num_dims = ShapeUtil::Rank(lhs_shape);
-  if (num_dims < 3) {
+  if (num_dims < 2) {
     NoteError(InvalidArgument(
         "Convolution expects argument arrays with >= 3 dimensions. "
         "Got: %s and %s",
diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier.cc b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
index 39e8430ed3..8b3886cc7a 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
@@ -201,17 +201,18 @@ class AlgebraicSimplifierVisitor : public DfsHloVisitorWithDefault {
   static bool Run(
       HloComputation* computation, bool is_layout_sensitive,
       AlgebraicSimplifier::ValidBitcastCallback valid_bitcast_callback,
-      bool enable_dot_simplification);
+      bool enable_dot_simplification, bool enable_conv_simplification);
 
  private:
   explicit AlgebraicSimplifierVisitor(
       HloComputation* computation, bool is_layout_sensitive,
       AlgebraicSimplifier::ValidBitcastCallback valid_bitcast_callback,
-      bool enable_dot_simplification)
+      bool enable_dot_simplification, bool enable_conv_simplification)
       : computation_(computation),
         is_layout_sensitive_(is_layout_sensitive),
         valid_bitcast_callback_(std::move(valid_bitcast_callback)),
-        enable_dot_simplification_(enable_dot_simplification) {}
+        enable_dot_simplification_(enable_dot_simplification),
+        enable_conv_simplification_(enable_conv_simplification) {}
 
   // Convenience method for replacing an instruction with a bitcast.
   void ReplaceWithBitcast(HloInstruction* instruction);
@@ -287,15 +288,18 @@ class AlgebraicSimplifierVisitor : public DfsHloVisitorWithDefault {
 
   // Disable dot simplication on platforms where it causes a slowdown.
   bool enable_dot_simplification_;
+
+  // Disable convolution simplication on platforms where it causes a slowdown.
+  bool enable_conv_simplification_;
 };
 
 bool AlgebraicSimplifierVisitor::Run(
     HloComputation* computation, bool is_layout_sensitive,
     AlgebraicSimplifier::ValidBitcastCallback valid_bitcast_callback,
-    bool enable_dot_simplification) {
-  AlgebraicSimplifierVisitor visitor(computation, is_layout_sensitive,
-                                     std::move(valid_bitcast_callback),
-                                     enable_dot_simplification);
+    bool enable_dot_simplification, bool enable_conv_simplification) {
+  AlgebraicSimplifierVisitor visitor(
+      computation, is_layout_sensitive, std::move(valid_bitcast_callback),
+      enable_dot_simplification, enable_conv_simplification);
   TF_CHECK_OK(computation->Accept(&visitor));
   return visitor.changed_;
 }
@@ -1459,6 +1463,9 @@ Status AlgebraicSimplifierVisitor::HandleTranspose(HloInstruction* transpose) {
 Status AlgebraicSimplifierVisitor::HandleConvolution(
     HloInstruction* convolution, HloInstruction* lhs, HloInstruction* rhs,
     const Window& window) {
+  if (!enable_conv_simplification_) {
+    return Status::OK();
+  }
   // HandleConvolution tries to replace a convolution with a DOT instruction.
   //
   // Only add when bitcasts can be used:
@@ -1962,9 +1969,9 @@ StatusOr<bool> AlgebraicSimplifier::Run(HloModule* module) {
                  "AlgebraicSimplifier::Run(), before:\n" + module->ToString());
   bool changed = false;
   for (auto* comp : module->MakeNonfusionComputations()) {
-    if (AlgebraicSimplifierVisitor::Run(comp, is_layout_sensitive_,
-                                        valid_bitcast_callback_,
-                                        enable_dot_simplification_)) {
+    if (AlgebraicSimplifierVisitor::Run(
+            comp, is_layout_sensitive_, valid_bitcast_callback_,
+            enable_dot_simplification_, enable_conv_simplification_)) {
       changed = true;
     }
   }
diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier.h b/tensorflow/compiler/xla/service/algebraic_simplifier.h
index 4295a3227a..a9f476178c 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier.h
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier.h
@@ -40,11 +40,13 @@ class AlgebraicSimplifier : public HloPassInterface {
   // bitcasts.
   AlgebraicSimplifier(bool is_layout_sensitive,
                       ValidBitcastCallback valid_bitcast_callback,
-                      bool enable_dot_simplification = true)
+                      bool enable_dot_simplification = true,
+                      bool enable_conv_simplification = true)
       : is_layout_sensitive_(is_layout_sensitive),
         valid_bitcast_callback_(std::move(valid_bitcast_callback)),
-        enable_dot_simplification_(enable_dot_simplification) {}
-  ~AlgebraicSimplifier() override {}
+        enable_dot_simplification_(enable_dot_simplification),
+        enable_conv_simplification_(enable_conv_simplification) {}
+  ~AlgebraicSimplifier() override = default;
   tensorflow::StringPiece name() const override { return "algsimp"; }
 
   // Run algebraic simplification on the given computation. Returns whether the
@@ -57,6 +59,9 @@ class AlgebraicSimplifier : public HloPassInterface {
 
   // Enable dot simplication on platforms where it is profitable.
   bool enable_dot_simplification_;
+
+  // Enable convolution simplication on platforms where it is profitable.
+  bool enable_conv_simplification_;
 };
 
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/service/hlo_cost_analysis.cc b/tensorflow/compiler/xla/service/hlo_cost_analysis.cc
index 84d55d4b5f..ca99fd6de8 100644
--- a/tensorflow/compiler/xla/service/hlo_cost_analysis.cc
+++ b/tensorflow/compiler/xla/service/hlo_cost_analysis.cc
@@ -398,7 +398,9 @@ Status HloCostAnalysis::HandleConvolution(HloInstruction* convolution,
   // For each output element, we do one fma per element in the kernel at some
   // given output feature index.
   const int64 fmas_per_output_element =
-      ShapeUtil::ElementsIn(rhs_instruction->shape()) / output_features;
+      output_features > 0
+          ? ShapeUtil::ElementsIn(rhs_instruction->shape()) / output_features
+          : 0;
   const int64 output_elements = ShapeUtil::ElementsIn(convolution->shape());
   current_properties_[kFlopsKey] =
       output_elements * fmas_per_output_element * kFmaFlops;
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator.cc b/tensorflow/compiler/xla/service/hlo_evaluator.cc
index 5fd891835d..e8f88427da 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator.cc
+++ b/tensorflow/compiler/xla/service/hlo_evaluator.cc
@@ -547,7 +547,7 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault {
     const auto& dnums = conv->convolution_dimension_numbers();
     const int64 num_spatial_dims = dnums.spatial_dimensions_size();
     CHECK_EQ(num_spatial_dims, dnums.kernel_spatial_dimensions_size());
-    CHECK_GE(num_spatial_dims, 1);
+    CHECK_GE(num_spatial_dims, 0);
     CHECK_EQ(window.dimensions_size(), num_spatial_dims);
 
     const auto lhs_rank = ShapeUtil::Rank(lhs_shape);
diff --git a/tensorflow/compiler/xla/service/shape_inference.cc b/tensorflow/compiler/xla/service/shape_inference.cc
index 6be6b77e85..1df1022442 100644
--- a/tensorflow/compiler/xla/service/shape_inference.cc
+++ b/tensorflow/compiler/xla/service/shape_inference.cc
@@ -1385,14 +1385,8 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(
         "Window: %s",
         window.DebugString().c_str());
   }
-  int num_spatial_dims = dnums.spatial_dimensions_size();
-  if (num_spatial_dims < 1) {
-    return InvalidArgument(
-        "Convolution requires at least one spatial dimension.\n"
-        "Window: %s",
-        window.DebugString().c_str());
-  }
 
+  const int num_spatial_dims = dnums.spatial_dimensions_size();
   if (window.dimensions_size() != num_spatial_dims) {
     return InvalidArgument(
         "Window must have same number of dimensions as dimension numbers.\n"
@@ -1400,7 +1394,7 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(
         window.DebugString().c_str(), dnums.DebugString().c_str());
   }
 
-  int num_dims = num_spatial_dims + 2;
+  const int num_dims = num_spatial_dims + 2;
   if (ShapeUtil::Rank(lhs) != num_dims) {
     return InvalidArgument(
         "The LHS argument to a convolution should have rank %d.\n"
-- 
GitLab


From 03b02ffc9e542a7f40d98debd711e537f7f3bb04 Mon Sep 17 00:00:00 2001
From: Justine Tunney <jart@google.com>
Date: Mon, 23 Oct 2017 09:44:24 -0700
Subject: [PATCH 674/909] Put Bazel mirror URLs first

PiperOrigin-RevId: 173127955
---
 tensorflow/workspace.bzl | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 8ba8748aae..02540bd843 100644
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -173,8 +173,8 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   native.new_http_archive(
       name = "mkl_dnn",
       urls = [
-          "https://github.com/01org/mkl-dnn/archive/b01e3a55a07be62172e713bcd2644c5176360212.tar.gz",
           "https://mirror.bazel.build/github.com/01org/mkl-dnn/archive/b01e3a55a07be62172e713bcd2644c5176360212.tar.gz",
+          "https://github.com/01org/mkl-dnn/archive/b01e3a55a07be62172e713bcd2644c5176360212.tar.gz",
       ],
       sha256 = "0d529ad4c49dc799e6df07c2b88b115d0668735da15fb3b3862d28d33fa68165",
       strip_prefix = "mkl-dnn-b01e3a55a07be62172e713bcd2644c5176360212",
@@ -184,8 +184,8 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   native.new_http_archive(
       name = "eigen_archive",
       urls = [
-          "https://bitbucket.org/eigen/eigen/get/429aa5254200.tar.gz",
           "https://mirror.bazel.build/bitbucket.org/eigen/eigen/get/429aa5254200.tar.gz",
+          "https://bitbucket.org/eigen/eigen/get/429aa5254200.tar.gz",
       ],
       sha256 = "61d8b6fc4279dd1dda986fb1677d15e3d641c07a3ea5abe255790b1f0c0c14e9",
       strip_prefix = "eigen-eigen-429aa5254200",
-- 
GitLab


From 4ec6f2b07c08ddab479541cad0c61f169c1f816f Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Mon, 23 Oct 2017 09:59:21 -0700
Subject: [PATCH 675/909] Switching contrib.summaries API to be
 context-manager-centric

PiperOrigin-RevId: 173129793
---
 tensorflow/contrib/summary/summary_ops.py     | 33 ++++++-
 .../contrib/summary/summary_ops_test.py       | 89 ++++++++++---------
 tensorflow/python/eager/context.py            |  6 +-
 3 files changed, 82 insertions(+), 46 deletions(-)

diff --git a/tensorflow/contrib/summary/summary_ops.py b/tensorflow/contrib/summary/summary_ops.py
index ba3619bfc9..30a9398ee5 100644
--- a/tensorflow/contrib/summary/summary_ops.py
+++ b/tensorflow/contrib/summary/summary_ops.py
@@ -27,6 +27,7 @@ from tensorflow.python.framework import ops
 from tensorflow.python.layers import utils
 from tensorflow.python.ops import summary_op_util
 from tensorflow.python.training import training_util
+from tensorflow.python.util import tf_contextlib
 
 # Name for a collection which is expected to have at most a single boolean
 # Tensor. If this tensor is True the summary ops will record summaries.
@@ -46,22 +47,50 @@ def should_record_summaries():
 
 
 # TODO(apassos) consider how to handle local step here.
+@tf_contextlib.contextmanager
 def record_summaries_every_n_global_steps(n):
   """Sets the should_record_summaries Tensor to true if global_step % n == 0."""
   collection_ref = ops.get_collection_ref(_SHOULD_RECORD_SUMMARIES_NAME)
+  old = collection_ref[:]
   collection_ref[:] = [training_util.get_global_step() % n == 0]
+  yield
+  collection_ref[:] = old
 
 
+@tf_contextlib.contextmanager
 def always_record_summaries():
   """Sets the should_record_summaries Tensor to always true."""
   collection_ref = ops.get_collection_ref(_SHOULD_RECORD_SUMMARIES_NAME)
+  old = collection_ref[:]
   collection_ref[:] = [True]
+  yield
+  collection_ref[:] = old
 
 
+@tf_contextlib.contextmanager
 def never_record_summaries():
   """Sets the should_record_summaries Tensor to always false."""
   collection_ref = ops.get_collection_ref(_SHOULD_RECORD_SUMMARIES_NAME)
+  old = collection_ref[:]
   collection_ref[:] = [False]
+  yield
+  collection_ref[:] = old
+
+
+class SummaryWriter(object):
+
+  def __init__(self, resource):
+    self._resource = resource
+
+  def set_as_default(self):
+    context.context().summary_writer_resource = self._resource
+
+  @tf_contextlib.contextmanager
+  def as_default(self):
+    old = context.context().summary_writer_resource
+    context.context().summary_writer_resource = self._resource
+    yield
+    context.context().summary_writer_resource = old
 
 
 def create_summary_file_writer(logdir,
@@ -77,9 +106,11 @@ def create_summary_file_writer(logdir,
   if filename_suffix is None:
     filename_suffix = constant_op.constant("")
   resource = gen_summary_ops.summary_writer(shared_name=name)
+  # TODO(apassos) ensure the initialization op runs when in graph mode; consider
+  # calling session.run here.
   gen_summary_ops.create_summary_file_writer(resource, logdir, max_queue,
                                              flush_secs, filename_suffix)
-  context.context().summary_writer_resource = resource
+  return SummaryWriter(resource)
 
 
 def _nothing():
diff --git a/tensorflow/contrib/summary/summary_ops_test.py b/tensorflow/contrib/summary/summary_ops_test.py
index 2cd4fce5b3..405a92a726 100644
--- a/tensorflow/contrib/summary/summary_ops_test.py
+++ b/tensorflow/contrib/summary/summary_ops_test.py
@@ -41,60 +41,65 @@ class TargetTest(test_util.TensorFlowTestCase):
 
   def testShouldRecordSummary(self):
     self.assertFalse(summary_ops.should_record_summaries())
-    summary_ops.always_record_summaries()
-    self.assertTrue(summary_ops.should_record_summaries())
+    with summary_ops.always_record_summaries():
+      self.assertTrue(summary_ops.should_record_summaries())
 
   def testSummaryOps(self):
     training_util.get_or_create_global_step()
     logdir = tempfile.mkdtemp()
-    summary_ops.create_summary_file_writer(logdir, max_queue=0, name='t0')
-    summary_ops.always_record_summaries()
-    summary_ops.generic('tensor', 1, '')
-    summary_ops.scalar('scalar', 2.0)
-    summary_ops.histogram('histogram', [1.0])
-    summary_ops.image('image', [[[[1.0]]]])
-    summary_ops.audio('audio', [[1.0]], 1.0, 1)
-    # The working condition of the ops is tested in the C++ test so we just
-    # test here that we're calling them correctly.
-    self.assertTrue(gfile.Exists(logdir))
+    with summary_ops.create_summary_file_writer(
+        logdir, max_queue=0,
+        name='t0').as_default(), summary_ops.always_record_summaries():
+      summary_ops.generic('tensor', 1, '')
+      summary_ops.scalar('scalar', 2.0)
+      summary_ops.histogram('histogram', [1.0])
+      summary_ops.image('image', [[[[1.0]]]])
+      summary_ops.audio('audio', [[1.0]], 1.0, 1)
+      # The working condition of the ops is tested in the C++ test so we just
+      # test here that we're calling them correctly.
+      self.assertTrue(gfile.Exists(logdir))
 
   def testDefunSummarys(self):
     training_util.get_or_create_global_step()
     logdir = tempfile.mkdtemp()
-    summary_ops.create_summary_file_writer(logdir, max_queue=0, name='t1')
-    summary_ops.always_record_summaries()
-
-    @function.defun
-    def write():
-      summary_ops.scalar('scalar', 2.0)
-
-    write()
-
-    self.assertTrue(gfile.Exists(logdir))
-    files = gfile.ListDirectory(logdir)
-    self.assertEqual(len(files), 1)
-    records = list(tf_record.tf_record_iterator(os.path.join(logdir, files[0])))
-    self.assertEqual(len(records), 2)
-    event = event_pb2.Event()
-    event.ParseFromString(records[1])
-    self.assertEqual(event.summary.value[0].simple_value, 2.0)
+    with summary_ops.create_summary_file_writer(
+        logdir, max_queue=0,
+        name='t1').as_default(), summary_ops.always_record_summaries():
+
+      @function.defun
+      def write():
+        summary_ops.scalar('scalar', 2.0)
+
+      write()
+
+      self.assertTrue(gfile.Exists(logdir))
+      files = gfile.ListDirectory(logdir)
+      self.assertEqual(len(files), 1)
+      records = list(
+          tf_record.tf_record_iterator(os.path.join(logdir, files[0])))
+      self.assertEqual(len(records), 2)
+      event = event_pb2.Event()
+      event.ParseFromString(records[1])
+      self.assertEqual(event.summary.value[0].simple_value, 2.0)
 
   def testSummaryName(self):
     training_util.get_or_create_global_step()
     logdir = tempfile.mkdtemp()
-    summary_ops.create_summary_file_writer(logdir, max_queue=0, name='t2')
-    summary_ops.always_record_summaries()
-
-    summary_ops.scalar('scalar', 2.0)
-
-    self.assertTrue(gfile.Exists(logdir))
-    files = gfile.ListDirectory(logdir)
-    self.assertEqual(len(files), 1)
-    records = list(tf_record.tf_record_iterator(os.path.join(logdir, files[0])))
-    self.assertEqual(len(records), 2)
-    event = event_pb2.Event()
-    event.ParseFromString(records[1])
-    self.assertEqual(event.summary.value[0].tag, 'scalar')
+    with summary_ops.create_summary_file_writer(
+        logdir, max_queue=0,
+        name='t2').as_default(), summary_ops.always_record_summaries():
+
+      summary_ops.scalar('scalar', 2.0)
+
+      self.assertTrue(gfile.Exists(logdir))
+      files = gfile.ListDirectory(logdir)
+      self.assertEqual(len(files), 1)
+      records = list(
+          tf_record.tf_record_iterator(os.path.join(logdir, files[0])))
+      self.assertEqual(len(records), 2)
+      event = event_pb2.Event()
+      event.ParseFromString(records[1])
+      self.assertEqual(event.summary.value[0].tag, 'scalar')
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/python/eager/context.py b/tensorflow/python/eager/context.py
index c5eedb7c9c..92f4e15c05 100644
--- a/tensorflow/python/eager/context.py
+++ b/tensorflow/python/eager/context.py
@@ -58,6 +58,7 @@ class _EagerContext(threading.local):
     self.mode = _default_mode
     self.scope_name = ""
     self.recording_summaries = False
+    self.summary_writer_resource = None
     self.scalar_cache = {}
 
 
@@ -86,7 +87,6 @@ class Context(object):
     self._eager_context = _EagerContext()
     self._context_handle = None
     self._context_devices = None
-    self._summary_writer_resource = None
     self._post_execution_callbacks = []
     self._config = config
     self._seed = None
@@ -213,12 +213,12 @@ class Context(object):
   @property
   def summary_writer_resource(self):
     """Returns summary writer resource."""
-    return self._summary_writer_resource
+    return self._eager_context.summary_writer_resource
 
   @summary_writer_resource.setter
   def summary_writer_resource(self, resource):
     """Sets summary writer resource."""
-    self._summary_writer_resource = resource
+    self._eager_context.summary_writer_resource = resource
 
   @property
   def device_name(self):
-- 
GitLab


From 3ed049b673e1a2b58e197fd8429ed81a015cd351 Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Mon, 23 Oct 2017 09:59:25 -0700
Subject: [PATCH 676/909] Allows calling keras layers in eager mode.

PiperOrigin-RevId: 173129805
---
 tensorflow/python/keras/_impl/keras/engine/topology.py  | 3 +++
 tensorflow/python/keras/_impl/keras/layers/core_test.py | 9 +++++++++
 2 files changed, 12 insertions(+)

diff --git a/tensorflow/python/keras/_impl/keras/engine/topology.py b/tensorflow/python/keras/_impl/keras/engine/topology.py
index c0be023b36..f9be782f85 100644
--- a/tensorflow/python/keras/_impl/keras/engine/topology.py
+++ b/tensorflow/python/keras/_impl/keras/engine/topology.py
@@ -26,6 +26,7 @@ import os
 import numpy as np
 from six.moves import zip  # pylint: disable=redefined-builtin
 
+from tensorflow.python.eager import context
 from tensorflow.python.framework import tensor_shape
 from tensorflow.python.keras._impl.keras import backend as K
 from tensorflow.python.keras._impl.keras.utils import conv_utils
@@ -250,6 +251,8 @@ class Layer(tf_base_layers.Layer):
     """
     # Actually call the layer (optionally building it).
     output = super(Layer, self).__call__(inputs, **kwargs)
+    if context.in_eager_mode():
+      return output
 
     # Update learning phase info.
     output_tensors = _to_list(output)
diff --git a/tensorflow/python/keras/_impl/keras/layers/core_test.py b/tensorflow/python/keras/_impl/keras/layers/core_test.py
index 5b15895c41..9cdebd375c 100644
--- a/tensorflow/python/keras/_impl/keras/layers/core_test.py
+++ b/tensorflow/python/keras/_impl/keras/layers/core_test.py
@@ -20,8 +20,11 @@ from __future__ import print_function
 
 import numpy as np
 
+from tensorflow.python.eager import context
+from tensorflow.python.framework import constant_op
 from tensorflow.python.keras._impl import keras
 from tensorflow.python.keras._impl.keras import testing_utils
+from tensorflow.python.ops import init_ops
 from tensorflow.python.platform import test
 
 
@@ -198,6 +201,12 @@ class CoreLayersTest(test.TestCase):
       self.assertEqual(layer.kernel.constraint, k_constraint)
       self.assertEqual(layer.bias.constraint, b_constraint)
 
+  def test_eager_dense(self):
+    with context.eager_mode():
+      l = keras.layers.Dense(units=3,
+                             kernel_initializer=init_ops.zeros_initializer())
+      self.assertAllEqual(l(constant_op.constant([[1.0]])), [[0., 0., 0.]])
+
   def test_activity_regularization(self):
     with self.test_session():
       layer = keras.layers.ActivityRegularization(l1=0.1)
-- 
GitLab


From 0e56ffb7b7cddaf3f0521747d2fade90c56b586f Mon Sep 17 00:00:00 2001
From: Shanqing Cai <cais@google.com>
Date: Mon, 23 Oct 2017 10:26:34 -0700
Subject: [PATCH 677/909] Fix breakages in OSS builds

See example breakages logs at:
http://ci.tensorflow.org/job/tensorflow-cl-cpu-python3-pip/10847/console
http://ci.tensorflow.org/job/tensorflow-cl-gpu/11008/console

1. CL/172477381 added the no_oss tag to tests with oss_serial tags, which broke the logic of OSS_SERIAL tests in pip.sh and run_pip_test.sh. This CL fixes that.

2. The nccl_kernels BUILD target in contrib/nccl/BUILD was missing some dependencies. This CL adds the missing ones.

Fixes: #13918
PiperOrigin-RevId: 173133914
---
 tensorflow/contrib/nccl/BUILD                     | 2 ++
 tensorflow/tools/ci_build/builds/run_pip_tests.sh | 1 +
 2 files changed, 3 insertions(+)

diff --git a/tensorflow/contrib/nccl/BUILD b/tensorflow/contrib/nccl/BUILD
index 5e7263ff62..3aa3215a5f 100644
--- a/tensorflow/contrib/nccl/BUILD
+++ b/tensorflow/contrib/nccl/BUILD
@@ -74,8 +74,10 @@ tf_kernel_library(
         "kernels/nccl_rewrite.cc",
     ],
     deps = [
+        "//tensorflow/core:core_cpu",
         "//tensorflow/core:framework",
         "//tensorflow/core:gpu_headers_lib",
+        "//tensorflow/core:lib",
         "//tensorflow/core:proto_text",
         "@nccl_archive//:nccl",
     ],
diff --git a/tensorflow/tools/ci_build/builds/run_pip_tests.sh b/tensorflow/tools/ci_build/builds/run_pip_tests.sh
index 43d5c5ff3b..29680e6882 100755
--- a/tensorflow/tools/ci_build/builds/run_pip_tests.sh
+++ b/tensorflow/tools/ci_build/builds/run_pip_tests.sh
@@ -78,6 +78,7 @@ ln -s $(pwd)/tensorflow ${PIP_TEST_ROOT}/tensorflow
 # tests with no_pip_gpu tag.
 PIP_TEST_FILTER_TAG="-no_pip,-no_oss"
 if [[ ${IS_OSS_SERIAL} == "1" ]]; then
+  PIP_TEST_FILTER_TAG="$(echo "${PIP_TEST_FILTER_TAG}" | sed s/-no_oss//)"
   PIP_TEST_FILTER_TAG="${PIP_TEST_FILTER_TAG},oss_serial"
 else
   PIP_TEST_FILTER_TAG="${PIP_TEST_FILTER_TAG},-oss_serial"
-- 
GitLab


From 57f3e529d935e6b08a6c0a3a418ad367d9314fde Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 23 Oct 2017 10:44:59 -0700
Subject: [PATCH 678/909] Internal change

PiperOrigin-RevId: 173136642
---
 tensorflow/compiler/xla/service/cpu/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/compiler/xla/service/cpu/BUILD b/tensorflow/compiler/xla/service/cpu/BUILD
index c71eca0d39..136cbe7cb7 100644
--- a/tensorflow/compiler/xla/service/cpu/BUILD
+++ b/tensorflow/compiler/xla/service/cpu/BUILD
@@ -525,6 +525,7 @@ cc_library(
 tf_cc_test(
     name = "cpu_runtime_test",
     srcs = ["cpu_runtime_test.cc"],
+    tags = ["optonly"],
     deps = [
         ":cpu_runtime",
         ":runtime_matmul",
-- 
GitLab


From 1038927c096ecc81ca48665871d1be390444b121 Mon Sep 17 00:00:00 2001
From: Saurabh Saxena <srbs@google.com>
Date: Mon, 23 Oct 2017 11:07:10 -0700
Subject: [PATCH 679/909] Add SerializeIterator op that serializes an
 IteratorResource into a variant tensor. Add DeserializeIterator op that
 builds IteratorResource from a variant tensor. Move BundleReaderWrapper and
 BundleWriterWrapper from dataset.h to iterator_ops.cc. Add generic key-value
 store interfaces IteratorStateReader and IteratorStateWriter for
 reading/writing state of iterators. Get rid of IteratorBundleReader and
 IteratorBundleWriter.

PiperOrigin-RevId: 173140858
---
 .../contrib/data/python/kernel_tests/BUILD    |   4 +
 .../python/kernel_tests/iterator_ops_test.py  |  29 +-
 .../kernel_tests/range_dataset_op_test.py     |  67 ++--
 .../kernel_tests/reader_dataset_ops_test.py   |  25 +-
 tensorflow/core/BUILD                         |   1 +
 tensorflow/core/framework/iterator.proto      |  17 +
 tensorflow/core/kernels/BUILD                 |   1 +
 tensorflow/core/kernels/dataset.h             | 189 ++++------
 tensorflow/core/kernels/iterator_ops.cc       | 355 +++++++++++++-----
 tensorflow/core/kernels/parse_tensor_op.cc    |   1 +
 tensorflow/core/kernels/range_dataset_op.cc   |  11 +-
 tensorflow/core/kernels/reader_dataset_ops.cc |  17 +-
 tensorflow/core/kernels/repeat_dataset_op.cc  |  13 +-
 .../core/ops/compat/ops_history.v1.pbtxt      |  24 --
 tensorflow/core/ops/dataset_ops.cc            |  42 ++-
 tensorflow/python/kernel_tests/BUILD          |   5 +
 .../python/kernel_tests/iterator_ops_test.py  |  29 +-
 .../kernel_tests/range_dataset_op_test.py     |  67 ++--
 .../kernel_tests/reader_dataset_ops_test.py   |  26 +-
 19 files changed, 537 insertions(+), 386 deletions(-)
 create mode 100644 tensorflow/core/framework/iterator.proto

diff --git a/tensorflow/contrib/data/python/kernel_tests/BUILD b/tensorflow/contrib/data/python/kernel_tests/BUILD
index c34c9dad9b..b3175e3e56 100644
--- a/tensorflow/contrib/data/python/kernel_tests/BUILD
+++ b/tensorflow/contrib/data/python/kernel_tests/BUILD
@@ -185,6 +185,7 @@ py_test(
         "//tensorflow/python:function",
         "//tensorflow/python:functional_ops",
         "//tensorflow/python:gradients",
+        "//tensorflow/python:io_ops",
         "//tensorflow/python:math_ops",
         "//tensorflow/python:parsing_ops",
         "//tensorflow/python:script_ops",
@@ -252,6 +253,8 @@ py_test(
         "//tensorflow/python:dtypes",
         "//tensorflow/python:errors",
         "//tensorflow/python:framework_ops",
+        "//tensorflow/python:io_ops",
+        "//tensorflow/python:parsing_ops",
         "//tensorflow/python:platform",
         "//tensorflow/python:tensor_shape",
         "//tensorflow/python:variables",
@@ -274,6 +277,7 @@ py_test(
         "//tensorflow/python:dtypes",
         "//tensorflow/python:errors",
         "//tensorflow/python:framework_ops",
+        "//tensorflow/python:io_ops",
         "//tensorflow/python:lib",
         "//tensorflow/python:parsing_ops",
         "//tensorflow/python:tensor_shape",
diff --git a/tensorflow/contrib/data/python/kernel_tests/iterator_ops_test.py b/tensorflow/contrib/data/python/kernel_tests/iterator_ops_test.py
index 20f6d6ba34..bda9a2a4a3 100644
--- a/tensorflow/contrib/data/python/kernel_tests/iterator_ops_test.py
+++ b/tensorflow/contrib/data/python/kernel_tests/iterator_ops_test.py
@@ -35,6 +35,7 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import functional_ops
 from tensorflow.python.ops import gen_dataset_ops
 from tensorflow.python.ops import gradients_impl
+from tensorflow.python.ops import io_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import parsing_ops
 from tensorflow.python.ops import script_ops
@@ -538,9 +539,23 @@ class IteratorTest(test.TestCase):
 
   def testIncorrectIteratorRestore(self):
 
-    def _iterator_checkpoint_prefix():
+    def _path():
       return os.path.join(self.get_temp_dir(), "iterator")
 
+    def _save_op(iterator_resource):
+      iterator_state_variant = gen_dataset_ops.serialize_iterator(
+          iterator_resource)
+      save_op = io_ops.write_file(
+          _path(), parsing_ops.serialize_tensor(iterator_state_variant))
+      return save_op
+
+    def _restore_op(iterator_resource):
+      iterator_state_variant = parsing_ops.parse_tensor(
+          io_ops.read_file(_path()), dtypes.variant)
+      restore_op = gen_dataset_ops.deserialize_iterator(iterator_resource,
+                                                        iterator_state_variant)
+      return restore_op
+
     def _build_range_dataset_graph():
       start = 1
       stop = 10
@@ -548,22 +563,18 @@ class IteratorTest(test.TestCase):
                                            stop).make_initializable_iterator()
       init_op = iterator.initializer
       get_next = iterator.get_next()
-      path = _iterator_checkpoint_prefix()
-      save_op = gen_dataset_ops.save_iterator(iterator._iterator_resource, path)
-      restore_op = gen_dataset_ops.restore_iterator(iterator._iterator_resource,
-                                                    path)
+      save_op = _save_op(iterator._iterator_resource)
+      restore_op = _restore_op(iterator._iterator_resource)
       return init_op, get_next, save_op, restore_op
 
     def _build_reader_dataset_graph():
       filenames = ["test"]  # Does not exist but we don't care in this test.
-      path = _iterator_checkpoint_prefix()
       iterator = readers.FixedLengthRecordDataset(
           filenames, 1, 0, 0).make_initializable_iterator()
       init_op = iterator.initializer
       get_next_op = iterator.get_next()
-      save_op = gen_dataset_ops.save_iterator(iterator._iterator_resource, path)
-      restore_op = gen_dataset_ops.restore_iterator(iterator._iterator_resource,
-                                                    path)
+      save_op = _save_op(iterator._iterator_resource)
+      restore_op = _restore_op(iterator._iterator_resource)
       return init_op, get_next_op, save_op, restore_op
 
     # Saving iterator for RangeDataset graph.
diff --git a/tensorflow/contrib/data/python/kernel_tests/range_dataset_op_test.py b/tensorflow/contrib/data/python/kernel_tests/range_dataset_op_test.py
index c8a0072809..c944eb4a49 100644
--- a/tensorflow/contrib/data/python/kernel_tests/range_dataset_op_test.py
+++ b/tensorflow/contrib/data/python/kernel_tests/range_dataset_op_test.py
@@ -29,6 +29,8 @@ from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gen_dataset_ops
+from tensorflow.python.ops import io_ops
+from tensorflow.python.ops import parsing_ops
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import gfile
 from tensorflow.python.platform import test
@@ -193,6 +195,21 @@ class RangeDatasetTest(test.TestCase):
   def _iterator_checkpoint_prefix(self):
     return os.path.join(self.get_temp_dir(), "iterator")
 
+  def _save_op(self, iterator_resource):
+    iterator_state_variant = gen_dataset_ops.serialize_iterator(
+        iterator_resource)
+    save_op = io_ops.write_file(
+        self._iterator_checkpoint_prefix(),
+        parsing_ops.serialize_tensor(iterator_state_variant))
+    return save_op
+
+  def _restore_op(self, iterator_resource):
+    iterator_state_variant = parsing_ops.parse_tensor(
+        io_ops.read_file(self._iterator_checkpoint_prefix()), dtypes.variant)
+    restore_op = gen_dataset_ops.deserialize_iterator(iterator_resource,
+                                                      iterator_state_variant)
+    return restore_op
+
   def testSaveRestore(self):
 
     def _build_graph(start, stop):
@@ -200,10 +217,8 @@ class RangeDatasetTest(test.TestCase):
                                            stop).make_initializable_iterator()
       init_op = iterator.initializer
       get_next = iterator.get_next()
-      path = self._iterator_checkpoint_prefix()
-      save_op = gen_dataset_ops.save_iterator(iterator._iterator_resource, path)
-      restore_op = gen_dataset_ops.restore_iterator(iterator._iterator_resource,
-                                                    path)
+      save_op = self._save_op(iterator._iterator_resource)
+      restore_op = self._restore_op(iterator._iterator_resource)
       return init_op, get_next, save_op, restore_op
 
     # Saving and restoring in different sessions.
@@ -246,14 +261,13 @@ class RangeDatasetTest(test.TestCase):
 
   def testRestoreWithoutBuildingDatasetGraph(self):
 
-    def _build_graph(start, stop, num_epochs, path):
+    def _build_graph(start, stop, num_epochs):
       dataset = dataset_ops.Dataset.range(start, stop).repeat(num_epochs)
       iterator = dataset.make_initializable_iterator()
       init_op = iterator.initializer
       get_next = iterator.get_next()
-      save_op = gen_dataset_ops.save_iterator(iterator._iterator_resource, path)
-      restore_op = gen_dataset_ops.restore_iterator(iterator._iterator_resource,
-                                                    path)
+      save_op = self._save_op(iterator._iterator_resource)
+      restore_op = self._restore_op(iterator._iterator_resource)
       return init_op, get_next, save_op, restore_op
 
     # Saving and restoring in different sessions.
@@ -262,10 +276,8 @@ class RangeDatasetTest(test.TestCase):
     num_epochs = 5
     break_point = 5
     break_epoch = 3
-    path = self._iterator_checkpoint_prefix()
     with ops.Graph().as_default() as g:
-      init_op, get_next, save_op, _ = _build_graph(start, stop, num_epochs,
-                                                   path)
+      init_op, get_next, save_op, _ = _build_graph(start, stop, num_epochs)
       with self.test_session(graph=g) as sess:
         sess.run(variables.global_variables_initializer())
         sess.run(init_op)
@@ -282,8 +294,7 @@ class RangeDatasetTest(test.TestCase):
       output_shapes = tensor_shape.scalar()
       iterator = iterator_ops.Iterator.from_structure(output_types,
                                                       output_shapes)
-      restore_op = gen_dataset_ops.restore_iterator(iterator._iterator_resource,
-                                                    path)
+      restore_op = self._restore_op(iterator._iterator_resource)
       get_next = iterator.get_next()
       with self.test_session(graph=g) as sess:
         sess.run(restore_op)
@@ -302,10 +313,8 @@ class RangeDatasetTest(test.TestCase):
       iterator = dataset.make_initializable_iterator()
       init_op = iterator.initializer
       get_next = iterator.get_next()
-      path = self._iterator_checkpoint_prefix()
-      save_op = gen_dataset_ops.save_iterator(iterator._iterator_resource, path)
-      restore_op = gen_dataset_ops.restore_iterator(iterator._iterator_resource,
-                                                    path)
+      save_op = self._save_op(iterator._iterator_resource)
+      restore_op = self._restore_op(iterator._iterator_resource)
       return init_op, get_next, save_op, restore_op
 
     # Saving and restoring in different sessions.
@@ -343,10 +352,8 @@ class RangeDatasetTest(test.TestCase):
       iterator = dataset.make_initializable_iterator()
       init_op = iterator.initializer
       get_next = iterator.get_next()
-      path = self._iterator_checkpoint_prefix()
-      save_op = gen_dataset_ops.save_iterator(iterator._iterator_resource, path)
-      restore_op = gen_dataset_ops.restore_iterator(iterator._iterator_resource,
-                                                    path)
+      save_op = self._save_op(iterator._iterator_resource)
+      restore_op = self._restore_op(iterator._iterator_resource)
       return init_op, get_next, save_op, restore_op
 
     # Saving and restoring in different sessions.
@@ -379,10 +386,8 @@ class RangeDatasetTest(test.TestCase):
                                            stop).make_initializable_iterator()
       init_op = iterator.initializer
       get_next = iterator.get_next()
-      path = self._iterator_checkpoint_prefix()
-      save_op = gen_dataset_ops.save_iterator(iterator._iterator_resource, path)
-      restore_op = gen_dataset_ops.restore_iterator(iterator._iterator_resource,
-                                                    path)
+      save_op = self._save_op(iterator._iterator_resource)
+      restore_op = self._restore_op(iterator._iterator_resource)
       return init_op, get_next, save_op, restore_op
 
     start = 2
@@ -424,10 +429,8 @@ class RangeDatasetTest(test.TestCase):
           start, stop).repeat(num_epochs).make_initializable_iterator()
       init_op = iterator.initializer
       get_next = iterator.get_next()
-      path = self._iterator_checkpoint_prefix()
-      save_op = gen_dataset_ops.save_iterator(iterator._iterator_resource, path)
-      restore_op = gen_dataset_ops.restore_iterator(iterator._iterator_resource,
-                                                    path)
+      save_op = self._save_op(iterator._iterator_resource)
+      restore_op = self._restore_op(iterator._iterator_resource)
       return init_op, get_next, save_op, restore_op
 
     start = 2
@@ -471,10 +474,8 @@ class RangeDatasetTest(test.TestCase):
           start, stop).repeat(num_epochs).make_initializable_iterator()
       init_op = iterator.initializer
       get_next = iterator.get_next()
-      path = self._iterator_checkpoint_prefix()
-      save_op = gen_dataset_ops.save_iterator(iterator._iterator_resource, path)
-      restore_op = gen_dataset_ops.restore_iterator(iterator._iterator_resource,
-                                                    path)
+      save_op = self._save_op(iterator._iterator_resource)
+      restore_op = self._restore_op(iterator._iterator_resource)
       return init_op, get_next, save_op, restore_op
 
     start = 2
diff --git a/tensorflow/contrib/data/python/kernel_tests/reader_dataset_ops_test.py b/tensorflow/contrib/data/python/kernel_tests/reader_dataset_ops_test.py
index c9f88f3dfc..2682e8bdfa 100644
--- a/tensorflow/contrib/data/python/kernel_tests/reader_dataset_ops_test.py
+++ b/tensorflow/contrib/data/python/kernel_tests/reader_dataset_ops_test.py
@@ -33,6 +33,7 @@ from tensorflow.python.framework import tensor_shape
 from tensorflow.python.lib.io import python_io
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gen_dataset_ops
+from tensorflow.python.ops import io_ops
 from tensorflow.python.ops import parsing_ops
 from tensorflow.python.platform import test
 from tensorflow.python.util import compat
@@ -276,18 +277,31 @@ class FixedLengthRecordReaderTest(test.TestCase):
   def _iterator_checkpoint_path(self):
     return os.path.join(self.get_temp_dir(), "iterator")
 
+  def _save_op(self, iterator_resource):
+    iterator_state_variant = gen_dataset_ops.serialize_iterator(
+        iterator_resource)
+    save_op = io_ops.write_file(
+        self._iterator_checkpoint_path(),
+        parsing_ops.serialize_tensor(iterator_state_variant))
+    return save_op
+
+  def _restore_op(self, iterator_resource):
+    iterator_state_variant = parsing_ops.parse_tensor(
+        io_ops.read_file(self._iterator_checkpoint_path()), dtypes.variant)
+    restore_op = gen_dataset_ops.deserialize_iterator(iterator_resource,
+                                                      iterator_state_variant)
+    return restore_op
+
   def _build_iterator_graph(self, num_epochs):
     filenames = self._createFiles()
-    path = self._iterator_checkpoint_path()
     dataset = (readers.FixedLengthRecordDataset(
         filenames, self._record_bytes, self._header_bytes, self._footer_bytes)
                .repeat(num_epochs))
     iterator = dataset.make_initializable_iterator()
     init_op = iterator.initializer
     get_next_op = iterator.get_next()
-    save_op = gen_dataset_ops.save_iterator(iterator._iterator_resource, path)
-    restore_op = gen_dataset_ops.restore_iterator(iterator._iterator_resource,
-                                                  path)
+    save_op = self._save_op(iterator._iterator_resource)
+    restore_op = self._restore_op(iterator._iterator_resource)
     return init_op, get_next_op, save_op, restore_op
 
   def _restore_iterator(self):
@@ -295,8 +309,7 @@ class FixedLengthRecordReaderTest(test.TestCase):
     output_shapes = tensor_shape.scalar()
     iterator = iterator_ops.Iterator.from_structure(output_types, output_shapes)
     get_next = iterator.get_next()
-    restore_op = gen_dataset_ops.restore_iterator(
-        iterator._iterator_resource, self._iterator_checkpoint_path())
+    restore_op = self._restore_op(iterator._iterator_resource)
     return restore_op, get_next
 
   def testSaveRestore(self):
diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 6ad93a73f4..c4f880da9d 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -163,6 +163,7 @@ CORE_PROTO_SRCS = [
     "framework/function.proto",
     "framework/graph.proto",
     "framework/graph_transfer_info.proto",
+    "framework/iterator.proto",
     "framework/kernel_def.proto",
     "framework/log_memory.proto",
     "framework/node_def.proto",
diff --git a/tensorflow/core/framework/iterator.proto b/tensorflow/core/framework/iterator.proto
new file mode 100644
index 0000000000..7e5f5ea2e0
--- /dev/null
+++ b/tensorflow/core/framework/iterator.proto
@@ -0,0 +1,17 @@
+syntax = "proto3";
+
+package tensorflow;
+option cc_enable_arenas = true;
+option java_outer_classname = "IteratorProtos";
+option java_multiple_files = true;
+option java_package = "org.tensorflow.util";
+
+// Protocol buffer representing the metadata for an iterator's state stored
+// as a Variant tensor.
+message IteratorStateMetadata {
+  // A user-specified version string.
+  string version = 1;
+
+  // Keys for tensors in the VariantTensorDataProto.
+  repeated string keys = 2;
+}
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index d931f12f6d..f5bfa60199 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -6061,6 +6061,7 @@ tf_kernel_library(
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
+        "//tensorflow/core:protos_all_cc",
     ],
 )
 
diff --git a/tensorflow/core/kernels/dataset.h b/tensorflow/core/kernels/dataset.h
index f9ffc4e065..a906113466 100644
--- a/tensorflow/core/kernels/dataset.h
+++ b/tensorflow/core/kernels/dataset.h
@@ -17,12 +17,14 @@ limitations under the License.
 
 #include <memory>
 
+#include "tensorflow/core/common_runtime/graph_runner.h"
 #include "tensorflow/core/framework/graph.pb.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/resource_mgr.h"
 #include "tensorflow/core/framework/variant_encode_decode.h"
 #include "tensorflow/core/framework/variant_tensor_data.h"
 #include "tensorflow/core/graph/graph.h"
+#include "tensorflow/core/graph/graph_constructor.h"
 #include "tensorflow/core/graph/graph_def_builder.h"
 #include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/lib/strings/strcat.h"
@@ -39,54 +41,25 @@ namespace tensorflow {
 
 class ResourceMgr;
 
-class BundleReaderWrapper {
+// Interface for reading values from a key-value store.
+// Used for restoring iterator state.
+class IteratorStateReader {
  public:
-  BundleReaderWrapper(BundleReader* bundle_reader)
-      : bundle_reader_(bundle_reader) {}
+  virtual Status ReadScalar(StringPiece key, int64* val) = 0;
+  virtual Status ReadScalar(StringPiece key, string* val) = 0;
+  virtual bool Contains(StringPiece key) = 0;
 
-  // Reads a scalar value.
-  template <typename T>
-  Status ReadScalar(StringPiece key, T* val) {
-    Tensor val_t = Tensor(DataTypeToEnum<T>::v(), TensorShape({}));
-    TF_RETURN_IF_ERROR(Lookup(key, &val_t));
-    *val = val_t.scalar<T>()();
-    return Status::OK();
-  }
-
-  bool Contains(StringPiece key) { return bundle_reader_->Contains(key); }
-
- private:
-  Status Lookup(StringPiece key, Tensor* val) {
-    return bundle_reader_->Lookup(key, val);
-  }
-
-  BundleReader* bundle_reader_;
+  virtual ~IteratorStateReader() {}
 };
 
-class BundleWriterWrapper {
+// Interface for writing values to a key-value store.
+// Used for saving iterator state.
+class IteratorStateWriter {
  public:
-  // Note: We intentionally do not provide a constructor that builds a
-  // BundleWriter from the checkpoint path because we want the caller to be
-  // in-charge of calling BundleWriter::Finish(). If we expose the Finish()
-  // method here it may be called pre-maturely by users of this object.
-  explicit BundleWriterWrapper(BundleWriter* bundle_writer)
-      : bundle_writer_(bundle_writer) {}
-
-  // Writes a scalar value.
-  template <typename T>
-  Status WriteScalar(StringPiece key, const T val) {
-    Tensor val_t = Tensor(DataTypeToEnum<T>::v(), TensorShape({}));
-    val_t.scalar<T>()() = val;
-    TF_RETURN_IF_ERROR(Add(key, val_t));
-    return Status::OK();
-  }
+  virtual Status WriteScalar(StringPiece key, const int64& val) = 0;
+  virtual Status WriteScalar(StringPiece key, const string& val) = 0;
 
- private:
-  Status Add(StringPiece key, const Tensor& val) {
-    return bundle_writer_->Add(key, val);
-  }
-
-  BundleWriter* bundle_writer_;
+  virtual ~IteratorStateWriter() {}
 };
 
 // Wrapper around GraphDefBuilder. Used to serialize Dataset graph.
@@ -249,10 +222,6 @@ class IteratorContext {
 // range of outputs is typically represented by an `DatasetBase`,
 // defined below.
 class IteratorBase {
- protected:
-  class IteratorBundleReader;
-  class IteratorBundleWriter;
-
  public:
   virtual ~IteratorBase() {}
 
@@ -284,87 +253,53 @@ class IteratorBase {
   virtual const std::vector<PartialTensorShape>& output_shapes() const = 0;
 
   // Saves the state of this iterator.
-  virtual Status Save(OpKernelContext* ctx, const string& path) {
-    BundleWriter bundle_writer(ctx->env(), path);
-    TF_RETURN_IF_ERROR(bundle_writer.status());
-    IteratorBundleWriter writer(&bundle_writer);
-    TF_RETURN_IF_ERROR(Save(ctx, &writer));
-    return bundle_writer.Finish();
+  virtual Status Save(IteratorStateWriter* writer) {
+    if (is_exhausted_) {
+      LOG(INFO) << "Iterator exhausted.";
+      return writer->WriteScalar(kIteratorExhausted, kIteratorExhausted);
+    } else {
+      return SaveInternal(writer);
+    }
   }
 
-  virtual Status Restore(OpKernelContext* ctx, const string& path) {
-    if (!(ctx->env()->FileExists(MetaFilename(path)).ok())) {
-      return errors::NotFound(
-          "Failed to restore Iterator state. No file found at ",
-          MetaFilename(path));
+  // Restores the state of this iterator.
+  virtual Status Restore(OpKernelContext* ctx, IteratorStateReader* reader) {
+    if (reader->Contains(kIteratorExhausted)) {
+      LOG(INFO) << "Iterator exhausted. Nothing to restore.";
+      is_exhausted_ = true;
+      return Status::OK();
+    } else {
+      return RestoreInternal(ctx, reader);
     }
-    BundleReader bundle_reader(ctx->env(), path);
-    TF_RETURN_IF_ERROR(bundle_reader.status());
-    IteratorBundleReader reader(&bundle_reader);
-    return Restore(ctx, &reader);
   }
 
   static const char kIteratorExhausted[];
 
  protected:
   // This is needed so that sub-classes of IteratorBase can call
-  // `RestoreInternal` on their parent iterators, e.g., in
+  // `SaveInternal` on their parent iterators, e.g., in
   // `RepeatDataasetOp::Dataset`.
-  class IteratorBundleReader : public BundleReaderWrapper {
-   public:
-    IteratorBundleReader(BundleReader* bundle_reader)
-        : BundleReaderWrapper(bundle_reader) {}
-
-    // Restores the state of a parent iterator recursively.
-    Status RestoreParent(OpKernelContext* ctx,
-                         const std::unique_ptr<IteratorBase>& parent) {
-      return parent->RestoreInternal(ctx, this);
-    }
-  };
+  Status SaveParent(IteratorStateWriter* writer,
+                    const std::unique_ptr<IteratorBase>& parent) {
+    return parent->SaveInternal(writer);
+  }
 
   // This is needed so that sub-classes of IteratorBase can call
-  // `SaveInternal` on their parent iterators, e.g., in
+  // `RestoreInternal` on their parent iterators, e.g., in
   // `RepeatDataasetOp::Dataset`.
-  class IteratorBundleWriter : public BundleWriterWrapper {
-   public:
-    IteratorBundleWriter(BundleWriter* bundle_writer)
-        : BundleWriterWrapper(bundle_writer) {}
-    // Saves the state of a parent iterator recursively.
-    Status SaveParent(OpKernelContext* ctx,
-                      const std::unique_ptr<IteratorBase>& parent) {
-      return parent->SaveInternal(ctx, this);
-    }
-  };
-
-  virtual Status Save(OpKernelContext* ctx, IteratorBundleWriter* writer) {
-    if (is_exhausted_) {
-      LOG(INFO) << "Iterator exhausted.";
-      return writer->WriteScalar<string>(kIteratorExhausted,
-                                         kIteratorExhausted);
-    } else {
-      return SaveInternal(ctx, writer);
-    }
+  Status RestoreParent(OpKernelContext* ctx, IteratorStateReader* reader,
+                       const std::unique_ptr<IteratorBase>& parent) {
+    return parent->RestoreInternal(ctx, reader);
   }
 
-  // Saves the state of this iterator.
-  virtual Status SaveInternal(OpKernelContext* ctx,
-                              IteratorBundleWriter* writer) {
+  // Saves the state of this iterator recursively.
+  virtual Status SaveInternal(IteratorStateWriter* writer) {
     return errors::Unimplemented("SaveInternal");
   }
 
-  virtual Status Restore(OpKernelContext* ctx, IteratorBundleReader* reader) {
-    if (reader->Contains(kIteratorExhausted)) {
-      LOG(INFO) << "Iterator exhausted. Nothing to restore.";
-      is_exhausted_ = true;
-      return Status::OK();
-    } else {
-      return RestoreInternal(ctx, reader);
-    }
-  }
-
-  // Restores the state of this iterator.
+  // Restores the state of this iterator recursively.
   virtual Status RestoreInternal(OpKernelContext* ctx,
-                                 IteratorBundleReader* reader) {
+                                 IteratorStateReader* reader) {
     return errors::Unimplemented("RestoreInternal");
   }
 
@@ -404,7 +339,7 @@ class DatasetBase : public core::RefCounted {
   virtual string DebugString() = 0;
 
   // Serializes the dataset and writes it to the `writer`.
-  virtual Status Save(BundleWriterWrapper* writer) const {
+  virtual Status Save(IteratorStateWriter* writer) const {
     return errors::Unimplemented("DatasetBase::Save");
   }
 
@@ -435,20 +370,14 @@ class GraphDatasetBase : public DatasetBase {
 
   const string op_name() const { return op_name_; }
 
-  Status Save(BundleWriterWrapper* writer) const override {
-    GraphDefBuilder b;
-    DatasetGraphDefBuilder db(&b);
-    Node* node = nullptr;
-    TF_RETURN_IF_ERROR(AsGraphDefInternal(&db, &node));
-    string output_name = node->name();
-    GraphDef graph_def;
-    TF_RETURN_IF_ERROR(b.ToGraphDef(&graph_def));
+  Status Save(IteratorStateWriter* writer) const override {
     string serialized_graph_def;
-    graph_def.SerializeToString(&serialized_graph_def);
+    string output_node;
+    TF_RETURN_IF_ERROR(Serialize(&serialized_graph_def, &output_node));
     TF_RETURN_IF_ERROR(
-        writer->WriteScalar<string>(kDatasetGraphKey, serialized_graph_def));
+        writer->WriteScalar(kDatasetGraphKey, serialized_graph_def));
     TF_RETURN_IF_ERROR(
-        writer->WriteScalar<string>(kDatasetGraphOutputNodeKey, output_name));
+        writer->WriteScalar(kDatasetGraphOutputNodeKey, output_node));
     return Status::OK();
   }
 
@@ -460,6 +389,18 @@ class GraphDatasetBase : public DatasetBase {
   static const char kDatasetGraphOutputNodeKey[];
 
  private:
+  Status Serialize(string* serialized_graph_def, string* output_node) const {
+    GraphDefBuilder b;
+    DatasetGraphDefBuilder db(&b);
+    Node* node = nullptr;
+    TF_RETURN_IF_ERROR(AsGraphDefInternal(&db, &node));
+    *output_node = node->name();
+    GraphDef graph_def;
+    TF_RETURN_IF_ERROR(b.ToGraphDef(&graph_def));
+    graph_def.SerializeToString(serialized_graph_def);
+    return Status::OK();
+  }
+
   const string op_name_;
 };
 
@@ -505,18 +446,18 @@ class DatasetIterator : public IteratorBase {
     return GetNextInternal(ctx, out_tensors, end_of_sequence);
   }
 
- protected:
-  Status Save(OpKernelContext* ctx, IteratorBundleWriter* writer) final {
+  Status Save(IteratorStateWriter* writer) final {
     TF_RETURN_IF_ERROR(dataset()->Save(writer));
-    return IteratorBase::Save(ctx, writer);
+    return IteratorBase::Save(writer);
   }
 
+ protected:
   // Internal implementation of GetNext that is wrapped in tracing logic.
   virtual Status GetNextInternal(IteratorContext* ctx,
                                  std::vector<Tensor>* out_tensors,
                                  bool* end_of_sequence) = 0;
 
-  string full_name(const string& name) {
+  string full_name(const string& name) const {
     return strings::StrCat(prefix(), ":", name);
   }
 
diff --git a/tensorflow/core/kernels/iterator_ops.cc b/tensorflow/core/kernels/iterator_ops.cc
index df13edc83a..b7c1fff2a9 100644
--- a/tensorflow/core/kernels/iterator_ops.cc
+++ b/tensorflow/core/kernels/iterator_ops.cc
@@ -16,9 +16,11 @@ limitations under the License.
 
 #include "tensorflow/core/common_runtime/function.h"
 #include "tensorflow/core/common_runtime/graph_runner.h"
+#include "tensorflow/core/framework/iterator.pb.h"
 #include "tensorflow/core/framework/partial_tensor_shape.h"
 #include "tensorflow/core/framework/resource_op_kernel.h"
 #include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/framework/variant_op_registry.h"
 #include "tensorflow/core/graph/graph_constructor.h"
 #include "tensorflow/core/kernels/ops_util.h"
 #include "tensorflow/core/lib/core/threadpool.h"
@@ -35,6 +37,8 @@ namespace {
 // See documentation in ../ops/dataset_ops.cc for a high-level
 // description of the following ops.
 
+const char kIteratorVariantTypeName[] = "tensorflow::Iterator";
+
 Status VerifyTypesMatch(const DataTypeVector& expected,
                         const DataTypeVector& received) {
   if (expected.size() != received.size()) {
@@ -93,10 +97,10 @@ class IteratorResource : public ResourceBase {
     }
   }
 
-  Status Save(OpKernelContext* ctx, const string& path) {
+  Status Save(IteratorStateWriter* writer) {
     std::shared_ptr<IteratorBase> captured_iterator(iterator_);
     if (captured_iterator) {
-      return captured_iterator->Save(ctx, path);
+      return captured_iterator->Save(writer);
     } else {
       return errors::FailedPrecondition(
           "Save() failed because the iterator has not been initialized. "
@@ -105,49 +109,34 @@ class IteratorResource : public ResourceBase {
     }
   }
 
-  Status Restore(OpKernelContext* ctx, const string& path) {
-    if (!(ctx->env()->FileExists(MetaFilename(path)).ok())) {
-      return errors::NotFound(
-          "Failed to restore Iterator state. No file found at ",
-          MetaFilename(path));
-    }
-
-    BundleReader bundle_reader(ctx->env(), path);
-    TF_RETURN_IF_ERROR(bundle_reader.status());
-    BundleReaderWrapper reader(&bundle_reader);
-    if (reader.Contains(GraphDatasetBase::kDatasetGraphKey)) {
-      string serialized_graph_def;
-      TF_RETURN_IF_ERROR(reader.ReadScalar(GraphDatasetBase::kDatasetGraphKey,
-                                           &serialized_graph_def));
-      GraphDef graph_def;
-      graph_def.ParseFromString(serialized_graph_def);
-      // TODO(srbs): Is there a way of getting the op registry of the original
-      // graph.
-      Graph graph(OpRegistry::Global());
-      TF_RETURN_IF_ERROR(ImportGraphDef({}, graph_def, &graph, nullptr));
-      string output_node;
-      TF_RETURN_IF_ERROR(reader.ReadScalar(
-          GraphDatasetBase::kDatasetGraphOutputNodeKey, &output_node));
-      std::vector<Tensor> outputs;
-      GraphRunner graph_runner(ctx->env());
-      TF_RETURN_IF_ERROR(graph_runner.Run(&graph, ctx->function_library(), {},
-                                          {output_node}, &outputs));
-      DatasetBase* dataset;
-      TF_RETURN_IF_ERROR(GetDatasetFromVariantTensor(outputs[0], &dataset));
-      TF_RETURN_IF_ERROR(set_iterator(dataset->MakeIterator("Iterator")));
-    } else if (reader.Contains(IteratorBase::kIteratorExhausted)) {
-      TF_RETURN_IF_ERROR(set_iterator(std::unique_ptr<IteratorBase>(
-          new ExhaustedIterator(output_dtypes_, output_shapes_))));
+  Status Restore(OpKernelContext* ctx, IteratorStateReader* reader) {
+    string serialized_graph_def;
+    TF_RETURN_IF_ERROR(reader->ReadScalar(GraphDatasetBase::kDatasetGraphKey,
+                                          &serialized_graph_def));
+    GraphDef graph_def;
+    if (!graph_def.ParseFromString(serialized_graph_def)) {
+      return errors::Internal("Error parsing dataset GraphDef.");
     }
+    string output_node;
+    TF_RETURN_IF_ERROR(reader->ReadScalar(
+        GraphDatasetBase::kDatasetGraphOutputNodeKey, &output_node));
+    DatasetBase* dataset = nullptr;
+    Graph graph(OpRegistry::Global());
+    TF_RETURN_IF_ERROR(ImportGraphDef({}, graph_def, &graph, nullptr));
+    std::vector<Tensor> outputs;
+    GraphRunner graph_runner(ctx->env());
+    TF_RETURN_IF_ERROR(graph_runner.Run(&graph, ctx->function_library(), {},
+                                        {output_node}, &outputs));
+    TF_RETURN_IF_ERROR(GetDatasetFromVariantTensor(outputs[0], &dataset));
+
+    TF_RETURN_IF_ERROR(set_iterator(dataset->MakeIterator("Iterator")));
     std::shared_ptr<IteratorBase> captured_iterator(iterator_);
 
     if (captured_iterator) {
-      // TODO(srbs): Figure a way to pass bundle_reader here.
-      return captured_iterator->Restore(ctx, path);
+      return captured_iterator->Restore(ctx, reader);
     } else {
       return errors::FailedPrecondition(
-          "Failed to restore iterator from ", path,
-          ". Make sure the checkpoint ",
+          "Failed to restore iterator. Make sure the checkpoint ",
           "is not corrupt. If the checkpoint does not contain the GraphDef, ",
           "you will need to initialize your iterator before restoring.");
     }
@@ -174,43 +163,194 @@ class IteratorResource : public ResourceBase {
   }
 
  private:
-  // A no-op iterator which always sets end_of_sequence = true. An instance of
-  // this is returned when attempting to restore an exhausted iterator. This is
-  // needed because the Dataset GraphDef may not have been saved for exhausted
-  // iterators so the actual Iterator can not be built.
-  class ExhaustedIterator : public IteratorBase {
-   public:
-    ExhaustedIterator(const DataTypeVector& output_dtypes,
-                      const std::vector<PartialTensorShape>& output_shapes)
-        : output_dtypes_(output_dtypes), output_shapes_(output_shapes) {}
-    Status GetNext(IteratorContext* ctx, std::vector<Tensor>* out_tensors,
-                   bool* end_of_sequence) final {
-      *end_of_sequence = true;
-      return Status::OK();
-    }
+  std::shared_ptr<IteratorBase> iterator_;
+  const DataTypeVector output_dtypes_;
+  const std::vector<PartialTensorShape> output_shapes_;
+};
+
+// Helper class for reading data from a VariantTensorData object.
+class VariantTensorDataReader : public IteratorStateReader {
+ public:
+  explicit VariantTensorDataReader(const VariantTensorData* data)
+      : data_(data) {
+    PreProcess();
+  }
+
+  // Returns OK iff the initialization was successful, i.e.,
+  // pre-processing did not have errors.
+  Status status() const { return status_; }
+
+  Status ReadScalar(StringPiece key, int64* val) override {
+    return ReadScalarInternal(key, val);
+  }
+
+  Status ReadScalar(StringPiece key, string* val) override {
+    return ReadScalarInternal(key, val);
+  }
 
-    const DataTypeVector& output_dtypes() const override {
-      return output_dtypes_;
+  bool Contains(StringPiece key) override {
+    return map_.find(key.ToString()) != map_.end();
+  }
+
+ private:
+  void PreProcess() {
+    string metadata;
+    data_->get_metadata(&metadata);
+    IteratorStateMetadata proto;
+    if (!proto.ParseFromString(metadata)) {
+      status_ = errors::Internal("Error parsing IteratorStateMetadata.");
+      return;
+    }
+    size_t num_entries = proto.keys_size();
+    CHECK_EQ(num_entries, data_->tensors_size());
+    for (size_t i = 0; i < num_entries; i++) {
+      map_[proto.keys(i)] = i;
     }
+  }
 
-    const std::vector<PartialTensorShape>& output_shapes() const override {
-      return output_shapes_;
+  template <typename T>
+  Status ReadScalarInternal(StringPiece key, T* val) {
+    if (map_.find(key.ToString()) == map_.end()) {
+      return errors::NotFound(key);
     }
+    *val = data_->tensors(map_[key.ToString()]).scalar<T>()();
+    return Status::OK();
+  }
 
-    virtual const std::vector<PartialTensorShape>& output_shapes() {
-      return output_shapes_;
+  std::map<string, size_t> map_;
+  const VariantTensorData* data_;  // Not owned.
+  Status status_;
+};
+
+// Helper class for writing data to a VariantTensorData object.
+class VariantTensorDataWriter : public IteratorStateWriter {
+ public:
+  // Does not take ownership of data.
+  explicit VariantTensorDataWriter(VariantTensorData* data) : data_(data) {}
+
+  Status WriteScalar(StringPiece key, const int64& val) override {
+    return WriteScalarInternal(key, val);
+  }
+
+  Status WriteScalar(StringPiece key, const string& val) override {
+    return WriteScalarInternal(key, val);
+  }
+
+  // Writes the metadata to `data_`.
+  Status Flush() {
+    string metadata;
+    if (!metadata_proto_.SerializeToString(&metadata)) {
+      return errors::Internal("Unable to serialize IteratorStateMetadata.");
     }
+    data_->set_metadata(metadata);
+    return Status::OK();
+  }
 
-   private:
-    const DataTypeVector output_dtypes_;
-    const std::vector<PartialTensorShape> output_shapes_;
-  };
+ private:
+  template <typename T>
+  Status WriteScalarInternal(StringPiece key, const T& val) {
+    // Write key to the metadata proto. This gets written to `data_`
+    // when `Flush()` is called. We do this lazily to avoid multiple
+    // serialization calls.
+    metadata_proto_.add_keys(key.ToString());
+
+    // Update tensors.
+    Tensor val_t = Tensor(DataTypeToEnum<T>::v(), TensorShape({}));
+    val_t.scalar<T>()() = val;
+    *(data_->add_tensors()) = std::move(val_t);
+    return Status::OK();
+  }
 
-  std::shared_ptr<IteratorBase> iterator_;
-  const DataTypeVector output_dtypes_;
-  const std::vector<PartialTensorShape> output_shapes_;
+  VariantTensorData* data_;
+  // TODO(srbs): Set the version string.
+  IteratorStateMetadata metadata_proto_;
+};
+
+// Wrapper for encoding/decoding the iterator state stored in a Variant tensor.
+// The get() method returns an IteratorStateReader which can be used
+// to restore iterator state.
+//
+// Usage example:
+//
+// Encoding:
+//
+//   Tensor t(DT_VARIANT, TensorShape({}));
+//   t->scalar<Variant>()() = IteratorStateVariant(iterator_resource);
+//
+// Encode() sets the type_name of the VariantTensorData object to
+// IteratorStateVariant::TypeName().
+//
+// Decoding:
+//
+//   Variant v = <VariantTensorDataProto object>;
+//   DecodeUnaryVariant(&v);
+//   IteratorStateVariant* wrapper = v.get<IteratorStateVariant>();
+//   iterator_resource->Restore(ctx, wrapper->get())
+//
+// The type_name of the VariantTensorData object to be decoded must
+// match IteratorStateVariant::TypeName().
+class IteratorStateVariant {
+ public:
+  IteratorStateVariant() : data_(nullptr) {}
+  IteratorStateVariant(const IteratorStateVariant& other) : data_(nullptr) {
+    if (other.data_) {
+      Decode(*other.data_);
+    }
+  }
+  // Initializes this object with the current state of the iterator so
+  // that it can be written on the next call to Encode().
+  Status InitializeFromIterator(IteratorResource* iterator_resource) {
+    data_.reset(new VariantTensorData());
+    data_->set_type_name(TypeName());
+    VariantTensorDataWriter writer(data_.get());
+    TF_RETURN_IF_ERROR(iterator_resource->Save(&writer));
+    TF_RETURN_IF_ERROR(writer.Flush());
+    return Status::OK();
+  }
+  string TypeName() const { return kIteratorVariantTypeName; }
+  void Encode(VariantTensorData* data) const { *data = *data_; }
+  bool Decode(const VariantTensorData& data) {
+    if (data.type_name() != TypeName()) {
+      return false;
+    }
+    std::unique_ptr<VariantTensorData> tensor_data(new VariantTensorData);
+    *tensor_data = data;
+    std::unique_ptr<VariantTensorDataReader> reader(
+        new VariantTensorDataReader(tensor_data.get()));
+    status_ = reader->status();
+    if (!status_.ok()) {
+      return false;
+    }
+    data_ = std::move(tensor_data);
+    reader_ = std::move(reader);
+    return true;
+  }
+  IteratorStateReader* get() { return reader_.get(); }
+  Status status() const { return status_; }
+  string DebugString() const {
+    if (data_) {
+      return strings::StrCat("IteratorStateVariant<",
+                             "data: ", data_->DebugString(),
+                             " status: ", status_.ToString(), ">");
+    } else {
+      return strings::StrCat("IteratorStateVariant<empty>");
+    }
+  }
+
+ private:
+  std::unique_ptr<IteratorStateReader> reader_;
+  Status status_;
+  std::unique_ptr<VariantTensorData> data_;
 };
 
+// Register the reader class in the global variant decode_fn registry
+// so that a Variant containing a serialized representation of iterator state
+// can be decoded using DecodeUnaryVariant. If we don't do this we will need
+// to manually decode the returned Variant using MaybeDecodeAndCopy in
+// DeserializeIteratorOp which is not recommended.
+REGISTER_UNARY_VARIANT_DECODE_FUNCTION(IteratorStateVariant,
+                                       kIteratorVariantTypeName);
+
 // TODO(mrry): Can we simply use the template kernel here?
 class IteratorHandleOp : public ResourceOpKernel<IteratorResource> {
  public:
@@ -294,37 +434,6 @@ class ToSingleElementOp : public OpKernel {
   }
 };
 
-class SaveIteratorOp : public OpKernel {
- public:
-  explicit SaveIteratorOp(OpKernelConstruction* ctx) : OpKernel(ctx) {}
-
-  void Compute(OpKernelContext* ctx) override {
-    IteratorResource* iterator_resource;
-    OP_REQUIRES_OK(
-        ctx, LookupResource(ctx, HandleFromInput(ctx, 0), &iterator_resource));
-    OP_REQUIRES(ctx, TensorShapeUtils::IsScalar(ctx->input(1).shape()),
-                errors::InvalidArgument("SaveIteratorOp: path must be scalar"));
-    const string& path = ctx->input(1).scalar<string>()();
-    OP_REQUIRES_OK(ctx, iterator_resource->Save(ctx, path));
-  }
-};
-
-class RestoreIteratorOp : public OpKernel {
- public:
-  explicit RestoreIteratorOp(OpKernelConstruction* ctx) : OpKernel(ctx) {}
-
-  void Compute(OpKernelContext* ctx) override {
-    IteratorResource* iterator_resource;
-    OP_REQUIRES_OK(
-        ctx, LookupResource(ctx, HandleFromInput(ctx, 0), &iterator_resource));
-    OP_REQUIRES(
-        ctx, TensorShapeUtils::IsScalar(ctx->input(1).shape()),
-        errors::InvalidArgument("RestoreIteratorOp: path must be scalar"));
-    const string& path = ctx->input(1).scalar<string>()();
-    OP_REQUIRES_OK(ctx, iterator_resource->Restore(ctx, path));
-  }
-};
-
 class OneShotIteratorOp : public AsyncOpKernel {
  public:
   explicit OneShotIteratorOp(OpKernelConstruction* ctx)
@@ -644,15 +753,55 @@ class IteratorFromStringHandleOp : public OpKernel {
   std::vector<PartialTensorShape> output_shapes_;
 };
 
+class SerializeIteratorOp : public OpKernel {
+ public:
+  explicit SerializeIteratorOp(OpKernelConstruction* ctx) : OpKernel(ctx) {}
+
+  void Compute(OpKernelContext* ctx) override {
+    const Tensor& resource_handle_t = ctx->input(0);
+    OP_REQUIRES(ctx, TensorShapeUtils::IsScalar(resource_handle_t.shape()),
+                errors::InvalidArgument("resource_handle must be a scalar"));
+
+    // Validate that the handle corresponds to a real resource, and
+    // that it is an IteratorResource.
+    IteratorResource* iterator_resource;
+    OP_REQUIRES_OK(
+        ctx, LookupResource(ctx, HandleFromInput(ctx, 0), &iterator_resource));
+    iterator_resource->Unref();
+    Tensor* variant_t;
+    OP_REQUIRES_OK(ctx, ctx->allocate_output(0, TensorShape({}), &variant_t));
+    IteratorStateVariant v;
+    OP_REQUIRES_OK(ctx, v.InitializeFromIterator(iterator_resource));
+    variant_t->scalar<Variant>()() = v;
+  }
+};
+
+class DeserializeIteratorOp : public OpKernel {
+ public:
+  explicit DeserializeIteratorOp(OpKernelConstruction* ctx) : OpKernel(ctx) {}
+
+  void Compute(OpKernelContext* ctx) override {
+    // Validate that the handle corresponds to a real resource, and
+    // that it is an IteratorResource.
+    IteratorResource* iterator_resource;
+    OP_REQUIRES_OK(
+        ctx, LookupResource(ctx, HandleFromInput(ctx, 0), &iterator_resource));
+
+    Variant variant = ctx->input(1).scalar<Variant>()();
+    auto* wrapper = variant.get<IteratorStateVariant>();
+    OP_REQUIRES(ctx, wrapper != nullptr,
+                errors::InvalidArgument(
+                    "DeserializeIteratorOp: Unable to parse variant tensor."));
+    OP_REQUIRES_OK(ctx, wrapper->status());
+    OP_REQUIRES_OK(ctx, iterator_resource->Restore(ctx, wrapper->get()));
+  }
+};
+
 REGISTER_KERNEL_BUILDER(Name("Iterator").Device(DEVICE_CPU), IteratorHandleOp);
 REGISTER_KERNEL_BUILDER(Name("MakeIterator").Device(DEVICE_CPU),
                         MakeIteratorOp);
 REGISTER_KERNEL_BUILDER(Name("DatasetToSingleElement").Device(DEVICE_CPU),
                         ToSingleElementOp);
-REGISTER_KERNEL_BUILDER(Name("SaveIterator").Device(DEVICE_CPU),
-                        SaveIteratorOp);
-REGISTER_KERNEL_BUILDER(Name("RestoreIterator").Device(DEVICE_CPU),
-                        RestoreIteratorOp);
 REGISTER_KERNEL_BUILDER(Name("OneShotIterator").Device(DEVICE_CPU),
                         OneShotIteratorOp);
 REGISTER_KERNEL_BUILDER(Name("IteratorGetNext").Device(DEVICE_CPU),
@@ -661,6 +810,10 @@ REGISTER_KERNEL_BUILDER(Name("IteratorToStringHandle").Device(DEVICE_CPU),
                         IteratorToStringHandleOp);
 REGISTER_KERNEL_BUILDER(Name("IteratorFromStringHandle").Device(DEVICE_CPU),
                         IteratorFromStringHandleOp);
+REGISTER_KERNEL_BUILDER(Name("SerializeIterator").Device(DEVICE_CPU),
+                        SerializeIteratorOp);
+REGISTER_KERNEL_BUILDER(Name("DeserializeIterator").Device(DEVICE_CPU),
+                        DeserializeIteratorOp);
 
 }  // namespace
 
diff --git a/tensorflow/core/kernels/parse_tensor_op.cc b/tensorflow/core/kernels/parse_tensor_op.cc
index ab91a6ef67..6b599612ad 100644
--- a/tensorflow/core/kernels/parse_tensor_op.cc
+++ b/tensorflow/core/kernels/parse_tensor_op.cc
@@ -92,6 +92,7 @@ class SerializeTensorOp : public OpKernel {
       Name("SerializeTensor").Device(DEVICE_CPU).TypeConstraint<T>("T"), \
       SerializeTensorOp<T>);
 TF_CALL_ALL_TYPES(REGISTER)
+TF_CALL_variant(REGISTER)
 #undef REGISTER
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/kernels/range_dataset_op.cc b/tensorflow/core/kernels/range_dataset_op.cc
index a57c21a590..7adfcc4f8d 100644
--- a/tensorflow/core/kernels/range_dataset_op.cc
+++ b/tensorflow/core/kernels/range_dataset_op.cc
@@ -112,19 +112,16 @@ class RangeDatasetOp : public DatasetOpKernel {
       }
 
      protected:
-      Status SaveInternal(OpKernelContext* ctx,
-                          IteratorBundleWriter* writer) override {
+      Status SaveInternal(IteratorStateWriter* writer) override {
         mutex_lock l(mu_);
-        TF_RETURN_IF_ERROR(
-            writer->WriteScalar<int64>(full_name("next"), next_));
+        TF_RETURN_IF_ERROR(writer->WriteScalar(full_name("next"), next_));
         return Status::OK();
       }
 
       Status RestoreInternal(OpKernelContext* ctx,
-                             IteratorBundleReader* reader) override {
+                             IteratorStateReader* reader) override {
         mutex_lock l(mu_);
-        TF_RETURN_IF_ERROR(
-            reader->ReadScalar<int64>(full_name("next"), &next_));
+        TF_RETURN_IF_ERROR(reader->ReadScalar(full_name("next"), &next_));
         return Status::OK();
       }
 
diff --git a/tensorflow/core/kernels/reader_dataset_ops.cc b/tensorflow/core/kernels/reader_dataset_ops.cc
index b455c28e07..fb88c55f73 100644
--- a/tensorflow/core/kernels/reader_dataset_ops.cc
+++ b/tensorflow/core/kernels/reader_dataset_ops.cc
@@ -356,31 +356,30 @@ class FixedLengthRecordDatasetOp : public DatasetOpKernel {
       }
 
      protected:
-      Status SaveInternal(OpKernelContext* ctx,
-                          IteratorBundleWriter* writer) override {
+      Status SaveInternal(IteratorStateWriter* writer) override {
         mutex_lock l(mu_);
-        TF_RETURN_IF_ERROR(writer->WriteScalar<int64>(
-            full_name("current_file_index"), current_file_index_));
+        TF_RETURN_IF_ERROR(writer->WriteScalar(full_name("current_file_index"),
+                                               current_file_index_));
 
         // `input_buffer_` is empty if
         // 1. GetNext has not been called even once.
         // 2. All files have been read and iterator has been exhausted.
         int64 current_pos = input_buffer_ ? input_buffer_->Tell() : -1;
         TF_RETURN_IF_ERROR(
-            writer->WriteScalar<int64>(full_name("current_pos"), current_pos));
+            writer->WriteScalar(full_name("current_pos"), current_pos));
         return Status::OK();
       }
 
       Status RestoreInternal(OpKernelContext* ctx,
-                             IteratorBundleReader* reader) override {
+                             IteratorStateReader* reader) override {
         mutex_lock l(mu_);
         int64 current_file_index;
-        TF_RETURN_IF_ERROR(reader->ReadScalar<int64>(
-            full_name("current_file_index"), &current_file_index));
+        TF_RETURN_IF_ERROR(reader->ReadScalar(full_name("current_file_index"),
+                                              &current_file_index));
         current_file_index_ = size_t(current_file_index);
         int64 current_pos;
         TF_RETURN_IF_ERROR(
-            reader->ReadScalar<int64>(full_name("current_pos"), &current_pos));
+            reader->ReadScalar(full_name("current_pos"), &current_pos));
 
         // Seek to current_pos.
         input_buffer_.reset();
diff --git a/tensorflow/core/kernels/repeat_dataset_op.cc b/tensorflow/core/kernels/repeat_dataset_op.cc
index 5d836927d2..9813e99a70 100644
--- a/tensorflow/core/kernels/repeat_dataset_op.cc
+++ b/tensorflow/core/kernels/repeat_dataset_op.cc
@@ -124,19 +124,18 @@ class RepeatDatasetOp : public UnaryDatasetOpKernel {
       }
 
      protected:
-      Status SaveInternal(OpKernelContext* ctx,
-                          IteratorBundleWriter* writer) override {
+      Status SaveInternal(IteratorStateWriter* writer) override {
         mutex_lock l(mu_);
-        TF_RETURN_IF_ERROR(writer->WriteScalar<int64>(full_name("i"), i_));
-        TF_RETURN_IF_ERROR(writer->SaveParent(ctx, input_impl_));
+        TF_RETURN_IF_ERROR(writer->WriteScalar(full_name("i"), i_));
+        TF_RETURN_IF_ERROR(SaveParent(writer, input_impl_));
         return Status::OK();
       }
 
       Status RestoreInternal(OpKernelContext* ctx,
-                             IteratorBundleReader* reader) override {
+                             IteratorStateReader* reader) override {
         mutex_lock l(mu_);
-        TF_RETURN_IF_ERROR(reader->ReadScalar<int64>(full_name("i"), &i_));
-        TF_RETURN_IF_ERROR(reader->RestoreParent(ctx, input_impl_));
+        TF_RETURN_IF_ERROR(reader->ReadScalar(full_name("i"), &i_));
+        TF_RETURN_IF_ERROR(RestoreParent(ctx, reader, input_impl_));
         return Status::OK();
       }
 
diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index 6772024263..c5ceb14a09 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -28753,18 +28753,6 @@ op {
   }
   is_stateful: true
 }
-op {
-  name: "RestoreIterator"
-  input_arg {
-    name: "iterator"
-    type: DT_RESOURCE
-  }
-  input_arg {
-    name: "path"
-    type: DT_STRING
-  }
-  is_stateful: true
-}
 op {
   name: "RestoreSlice"
   input_arg {
@@ -29548,18 +29536,6 @@ op {
   }
   is_stateful: true
 }
-op {
-  name: "SaveIterator"
-  input_arg {
-    name: "iterator"
-    type: DT_RESOURCE
-  }
-  input_arg {
-    name: "path"
-    type: DT_STRING
-  }
-  is_stateful: true
-}
 op {
   name: "SaveSlices"
   input_arg {
diff --git a/tensorflow/core/ops/dataset_ops.cc b/tensorflow/core/ops/dataset_ops.cc
index 566049179a..8b77e3f9f0 100644
--- a/tensorflow/core/ops/dataset_ops.cc
+++ b/tensorflow/core/ops/dataset_ops.cc
@@ -598,24 +598,6 @@ This operation may be executed multiple times. Each execution will reset the
 iterator in `iterator` to the first element of `dataset`.
 )doc");
 
-REGISTER_OP("SaveIterator")
-    .Input("iterator: resource")
-    .Input("path: string")
-    .SetShapeFn(shape_inference::NoOutputs)
-    .Doc(R"doc(
-Saves the state of the `iterator` at `path`.
-
-This state can be restored using "RestoreIterator".
-)doc");
-
-REGISTER_OP("RestoreIterator")
-    .Input("iterator: resource")
-    .Input("path: string")
-    .SetShapeFn(shape_inference::NoOutputs)
-    .Doc(R"doc(
-Restores the state of the `iterator` from the checkpoint saved at `path` using "SaveIterator".
-)doc");
-
 REGISTER_OP("OneShotIterator")
     .Output("handle: resource")
     .Attr("dataset_factory: func")
@@ -737,4 +719,28 @@ output_shapes: If specified, defines the shape of each tuple component in an
   element produced by the resulting iterator.
 )doc");
 
+REGISTER_OP("SerializeIterator")
+    .Input("resource_handle: resource")
+    .Output("serialized: variant")
+    .SetShapeFn(shape_inference::ScalarShape)
+    .Doc(R"doc(
+Converts the given `resource_handle` representing an iterator to a variant tensor.
+
+resource_handle: A handle to an iterator resource.
+serialized: A variant tensor storing the state of the iterator contained in the
+  resource.
+)doc");
+
+REGISTER_OP("DeserializeIterator")
+    .Input("resource_handle: resource")
+    .Input("serialized: variant")
+    .SetShapeFn(shape_inference::NoOutputs)
+    .Doc(R"doc(
+Converts the given variant tensor to an iterator and stores it in the given resource.
+
+resource_handle: A handle to an iterator resource.
+serialized: A variant tensor storing the state of the iterator contained in the
+  resource.
+)doc");
+
 }  // namespace tensorflow
diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD
index 0e36c3498a..b02bae95fd 100644
--- a/tensorflow/python/kernel_tests/BUILD
+++ b/tensorflow/python/kernel_tests/BUILD
@@ -2886,7 +2886,9 @@ tf_py_test(
         "//tensorflow/python:dataset_ops_gen",
         "//tensorflow/python:dtypes",
         "//tensorflow/python:errors",
+        "//tensorflow/python:io_ops",
         "//tensorflow/python:framework_ops",
+        "//tensorflow/python:parsing_ops",
         "//tensorflow/python:platform",
         "//tensorflow/python:tensor_shape",
         "//tensorflow/python:variables",
@@ -2907,7 +2909,9 @@ tf_py_test(
         "//tensorflow/python:dtypes",
         "//tensorflow/python:errors",
         "//tensorflow/python:framework_ops",
+        "//tensorflow/python:io_ops",
         "//tensorflow/python:lib",
+        "//tensorflow/python:parsing_ops",
         "//tensorflow/python:tensor_shape",
         "//tensorflow/python:util",
         "//tensorflow/python/data/ops:iterator_ops",
@@ -3022,6 +3026,7 @@ tf_py_test(
         "//tensorflow/python:function",
         "//tensorflow/python:functional_ops",
         "//tensorflow/python:gradients",
+        "//tensorflow/python:io_ops",
         "//tensorflow/python:math_ops",
         "//tensorflow/python:parsing_ops",
         "//tensorflow/python:script_ops",
diff --git a/tensorflow/python/kernel_tests/iterator_ops_test.py b/tensorflow/python/kernel_tests/iterator_ops_test.py
index b5ec9f7db0..2128ef4ae1 100644
--- a/tensorflow/python/kernel_tests/iterator_ops_test.py
+++ b/tensorflow/python/kernel_tests/iterator_ops_test.py
@@ -35,6 +35,7 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import functional_ops
 from tensorflow.python.ops import gen_dataset_ops
 from tensorflow.python.ops import gradients_impl
+from tensorflow.python.ops import io_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import parsing_ops
 from tensorflow.python.ops import script_ops
@@ -538,9 +539,23 @@ class IteratorTest(test.TestCase):
 
   def testIncorrectIteratorRestore(self):
 
-    def _iterator_checkpoint_prefix():
+    def _path():
       return os.path.join(self.get_temp_dir(), "iterator")
 
+    def _save_op(iterator_resource):
+      iterator_state_variant = gen_dataset_ops.serialize_iterator(
+          iterator_resource)
+      save_op = io_ops.write_file(
+          _path(), parsing_ops.serialize_tensor(iterator_state_variant))
+      return save_op
+
+    def _restore_op(iterator_resource):
+      iterator_state_variant = parsing_ops.parse_tensor(
+          io_ops.read_file(_path()), dtypes.variant)
+      restore_op = gen_dataset_ops.deserialize_iterator(iterator_resource,
+                                                        iterator_state_variant)
+      return restore_op
+
     def _build_range_dataset_graph():
       start = 1
       stop = 10
@@ -548,22 +563,18 @@ class IteratorTest(test.TestCase):
                                            stop).make_initializable_iterator()
       init_op = iterator.initializer
       get_next = iterator.get_next()
-      path = _iterator_checkpoint_prefix()
-      save_op = gen_dataset_ops.save_iterator(iterator._iterator_resource, path)
-      restore_op = gen_dataset_ops.restore_iterator(iterator._iterator_resource,
-                                                    path)
+      save_op = _save_op(iterator._iterator_resource)
+      restore_op = _restore_op(iterator._iterator_resource)
       return init_op, get_next, save_op, restore_op
 
     def _build_reader_dataset_graph():
       filenames = ["test"]  # Does not exist but we don't care in this test.
-      path = _iterator_checkpoint_prefix()
       iterator = readers.FixedLengthRecordDataset(
           filenames, 1, 0, 0).make_initializable_iterator()
       init_op = iterator.initializer
       get_next_op = iterator.get_next()
-      save_op = gen_dataset_ops.save_iterator(iterator._iterator_resource, path)
-      restore_op = gen_dataset_ops.restore_iterator(iterator._iterator_resource,
-                                                    path)
+      save_op = _save_op(iterator._iterator_resource)
+      restore_op = _restore_op(iterator._iterator_resource)
       return init_op, get_next_op, save_op, restore_op
 
     # Saving iterator for RangeDataset graph.
diff --git a/tensorflow/python/kernel_tests/range_dataset_op_test.py b/tensorflow/python/kernel_tests/range_dataset_op_test.py
index 8291967155..0c530522b8 100644
--- a/tensorflow/python/kernel_tests/range_dataset_op_test.py
+++ b/tensorflow/python/kernel_tests/range_dataset_op_test.py
@@ -27,6 +27,8 @@ from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gen_dataset_ops
+from tensorflow.python.ops import io_ops
+from tensorflow.python.ops import parsing_ops
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import gfile
 from tensorflow.python.platform import test
@@ -169,6 +171,21 @@ class RangeDatasetTest(test.TestCase):
   def _iterator_checkpoint_prefix(self):
     return os.path.join(self.get_temp_dir(), "iterator")
 
+  def _save_op(self, iterator_resource):
+    iterator_state_variant = gen_dataset_ops.serialize_iterator(
+        iterator_resource)
+    save_op = io_ops.write_file(
+        self._iterator_checkpoint_prefix(),
+        parsing_ops.serialize_tensor(iterator_state_variant))
+    return save_op
+
+  def _restore_op(self, iterator_resource):
+    iterator_state_variant = parsing_ops.parse_tensor(
+        io_ops.read_file(self._iterator_checkpoint_prefix()), dtypes.variant)
+    restore_op = gen_dataset_ops.deserialize_iterator(iterator_resource,
+                                                      iterator_state_variant)
+    return restore_op
+
   def testSaveRestore(self):
 
     def _build_graph(start, stop):
@@ -176,10 +193,8 @@ class RangeDatasetTest(test.TestCase):
                                            stop).make_initializable_iterator()
       init_op = iterator.initializer
       get_next = iterator.get_next()
-      path = self._iterator_checkpoint_prefix()
-      save_op = gen_dataset_ops.save_iterator(iterator._iterator_resource, path)
-      restore_op = gen_dataset_ops.restore_iterator(iterator._iterator_resource,
-                                                    path)
+      save_op = self._save_op(iterator._iterator_resource)
+      restore_op = self._restore_op(iterator._iterator_resource)
       return init_op, get_next, save_op, restore_op
 
     # Saving and restoring in different sessions.
@@ -222,14 +237,13 @@ class RangeDatasetTest(test.TestCase):
 
   def testRestoreWithoutBuildingDatasetGraph(self):
 
-    def _build_graph(start, stop, num_epochs, path):
+    def _build_graph(start, stop, num_epochs):
       dataset = dataset_ops.Dataset.range(start, stop).repeat(num_epochs)
       iterator = dataset.make_initializable_iterator()
       init_op = iterator.initializer
       get_next = iterator.get_next()
-      save_op = gen_dataset_ops.save_iterator(iterator._iterator_resource, path)
-      restore_op = gen_dataset_ops.restore_iterator(iterator._iterator_resource,
-                                                    path)
+      save_op = self._save_op(iterator._iterator_resource)
+      restore_op = self._restore_op(iterator._iterator_resource)
       return init_op, get_next, save_op, restore_op
 
     # Saving and restoring in different sessions.
@@ -238,10 +252,8 @@ class RangeDatasetTest(test.TestCase):
     num_epochs = 5
     break_point = 5
     break_epoch = 3
-    path = self._iterator_checkpoint_prefix()
     with ops.Graph().as_default() as g:
-      init_op, get_next, save_op, _ = _build_graph(start, stop, num_epochs,
-                                                   path)
+      init_op, get_next, save_op, _ = _build_graph(start, stop, num_epochs)
       with self.test_session(graph=g) as sess:
         sess.run(variables.global_variables_initializer())
         sess.run(init_op)
@@ -258,8 +270,7 @@ class RangeDatasetTest(test.TestCase):
       output_shapes = tensor_shape.scalar()
       iterator = iterator_ops.Iterator.from_structure(output_types,
                                                       output_shapes)
-      restore_op = gen_dataset_ops.restore_iterator(iterator._iterator_resource,
-                                                    path)
+      restore_op = self._restore_op(iterator._iterator_resource)
       get_next = iterator.get_next()
       with self.test_session(graph=g) as sess:
         sess.run(restore_op)
@@ -278,10 +289,8 @@ class RangeDatasetTest(test.TestCase):
       iterator = dataset.make_initializable_iterator()
       init_op = iterator.initializer
       get_next = iterator.get_next()
-      path = self._iterator_checkpoint_prefix()
-      save_op = gen_dataset_ops.save_iterator(iterator._iterator_resource, path)
-      restore_op = gen_dataset_ops.restore_iterator(iterator._iterator_resource,
-                                                    path)
+      save_op = self._save_op(iterator._iterator_resource)
+      restore_op = self._restore_op(iterator._iterator_resource)
       return init_op, get_next, save_op, restore_op
 
     # Saving and restoring in different sessions.
@@ -319,10 +328,8 @@ class RangeDatasetTest(test.TestCase):
       iterator = dataset.make_initializable_iterator()
       init_op = iterator.initializer
       get_next = iterator.get_next()
-      path = self._iterator_checkpoint_prefix()
-      save_op = gen_dataset_ops.save_iterator(iterator._iterator_resource, path)
-      restore_op = gen_dataset_ops.restore_iterator(iterator._iterator_resource,
-                                                    path)
+      save_op = self._save_op(iterator._iterator_resource)
+      restore_op = self._restore_op(iterator._iterator_resource)
       return init_op, get_next, save_op, restore_op
 
     # Saving and restoring in different sessions.
@@ -355,10 +362,8 @@ class RangeDatasetTest(test.TestCase):
                                            stop).make_initializable_iterator()
       init_op = iterator.initializer
       get_next = iterator.get_next()
-      path = self._iterator_checkpoint_prefix()
-      save_op = gen_dataset_ops.save_iterator(iterator._iterator_resource, path)
-      restore_op = gen_dataset_ops.restore_iterator(iterator._iterator_resource,
-                                                    path)
+      save_op = self._save_op(iterator._iterator_resource)
+      restore_op = self._restore_op(iterator._iterator_resource)
       return init_op, get_next, save_op, restore_op
 
     start = 2
@@ -400,10 +405,8 @@ class RangeDatasetTest(test.TestCase):
           start, stop).repeat(num_epochs).make_initializable_iterator()
       init_op = iterator.initializer
       get_next = iterator.get_next()
-      path = self._iterator_checkpoint_prefix()
-      save_op = gen_dataset_ops.save_iterator(iterator._iterator_resource, path)
-      restore_op = gen_dataset_ops.restore_iterator(iterator._iterator_resource,
-                                                    path)
+      save_op = self._save_op(iterator._iterator_resource)
+      restore_op = self._restore_op(iterator._iterator_resource)
       return init_op, get_next, save_op, restore_op
 
     start = 2
@@ -447,10 +450,8 @@ class RangeDatasetTest(test.TestCase):
           start, stop).repeat(num_epochs).make_initializable_iterator()
       init_op = iterator.initializer
       get_next = iterator.get_next()
-      path = self._iterator_checkpoint_prefix()
-      save_op = gen_dataset_ops.save_iterator(iterator._iterator_resource, path)
-      restore_op = gen_dataset_ops.restore_iterator(iterator._iterator_resource,
-                                                    path)
+      save_op = self._save_op(iterator._iterator_resource)
+      restore_op = self._restore_op(iterator._iterator_resource)
       return init_op, get_next, save_op, restore_op
 
     start = 2
diff --git a/tensorflow/python/kernel_tests/reader_dataset_ops_test.py b/tensorflow/python/kernel_tests/reader_dataset_ops_test.py
index 38420328ef..c8e7333b4b 100644
--- a/tensorflow/python/kernel_tests/reader_dataset_ops_test.py
+++ b/tensorflow/python/kernel_tests/reader_dataset_ops_test.py
@@ -31,6 +31,8 @@ from tensorflow.python.framework import tensor_shape
 from tensorflow.python.lib.io import python_io
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gen_dataset_ops
+from tensorflow.python.ops import io_ops
+from tensorflow.python.ops import parsing_ops
 from tensorflow.python.platform import test
 from tensorflow.python.util import compat
 
@@ -273,18 +275,31 @@ class FixedLengthRecordReaderTest(test.TestCase):
   def _iterator_checkpoint_path(self):
     return os.path.join(self.get_temp_dir(), "iterator")
 
+  def _save_op(self, iterator_resource):
+    iterator_state_variant = gen_dataset_ops.serialize_iterator(
+        iterator_resource)
+    save_op = io_ops.write_file(
+        self._iterator_checkpoint_path(),
+        parsing_ops.serialize_tensor(iterator_state_variant))
+    return save_op
+
+  def _restore_op(self, iterator_resource):
+    iterator_state_variant = parsing_ops.parse_tensor(
+        io_ops.read_file(self._iterator_checkpoint_path()), dtypes.variant)
+    restore_op = gen_dataset_ops.deserialize_iterator(iterator_resource,
+                                                      iterator_state_variant)
+    return restore_op
+
   def _build_iterator_graph(self, num_epochs):
     filenames = self._createFiles()
-    path = self._iterator_checkpoint_path()
     dataset = (readers.FixedLengthRecordDataset(
         filenames, self._record_bytes, self._header_bytes, self._footer_bytes)
                .repeat(num_epochs))
     iterator = dataset.make_initializable_iterator()
     init_op = iterator.initializer
     get_next_op = iterator.get_next()
-    save_op = gen_dataset_ops.save_iterator(iterator._iterator_resource, path)
-    restore_op = gen_dataset_ops.restore_iterator(iterator._iterator_resource,
-                                                  path)
+    save_op = self._save_op(iterator._iterator_resource)
+    restore_op = self._restore_op(iterator._iterator_resource)
     return init_op, get_next_op, save_op, restore_op
 
   def _restore_iterator(self):
@@ -292,8 +307,7 @@ class FixedLengthRecordReaderTest(test.TestCase):
     output_shapes = tensor_shape.scalar()
     iterator = iterator_ops.Iterator.from_structure(output_types, output_shapes)
     get_next = iterator.get_next()
-    restore_op = gen_dataset_ops.restore_iterator(
-        iterator._iterator_resource, self._iterator_checkpoint_path())
+    restore_op = self._restore_op(iterator._iterator_resource)
     return restore_op, get_next
 
   def testSaveRestore(self):
-- 
GitLab


From 5e23e0e67ac565d56de7680ccb8d8ccc6a0d2179 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 23 Oct 2017 11:12:26 -0700
Subject: [PATCH 680/909] [XLA] Erase cloned instructions on the fly when
 merging fusion nodes.

This avoids the awkward situation where an RNG which is clearly eligible for fusion becomes ineligible mid-fusion because it suddenly has an extra (dead) user.

PiperOrigin-RevId: 173141716
---
 tensorflow/compiler/xla/service/hlo_instruction.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tensorflow/compiler/xla/service/hlo_instruction.cc b/tensorflow/compiler/xla/service/hlo_instruction.cc
index f24953051a..0669a86863 100644
--- a/tensorflow/compiler/xla/service/hlo_instruction.cc
+++ b/tensorflow/compiler/xla/service/hlo_instruction.cc
@@ -716,10 +716,12 @@ void HloInstruction::MergeFusionInstructionIntoMultiOutput(
 
   // Fuse the root instruction and generate multiple outputs.
   FuseInstructionIntoMultiOutput(unfused_root);
+  TF_CHECK_OK(unfused_root->parent()->RemoveInstruction(unfused_root));
   // The rest instructions are of normal fusing.
   for (int64 i = 1; i < unfused_instructions.size(); i++) {
     auto instruction = unfused_instructions[i];
     FuseInstruction(instruction);
+    TF_CHECK_OK(instruction->parent()->RemoveInstruction(instruction));
   }
 }
 
-- 
GitLab


From 01b6b063811f60764a9604ffcb8cef611f41afa3 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 23 Oct 2017 11:29:58 -0700
Subject: [PATCH 681/909] Cut tracing memory cost

PiperOrigin-RevId: 173144626
---
 .../core/distributed_runtime/master_session.cc     | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/tensorflow/core/distributed_runtime/master_session.cc b/tensorflow/core/distributed_runtime/master_session.cc
index 995422644a..f7fce1d0ec 100644
--- a/tensorflow/core/distributed_runtime/master_session.cc
+++ b/tensorflow/core/distributed_runtime/master_session.cc
@@ -746,18 +746,22 @@ void MasterSession::ReffedClientGraph::ProcessStats(int64 step_id,
                  Status::OK());
   }
   // Assemble all stats for this timeline into a merged StepStats.
-  StepStats step_stats_proto;
   if (pss->collect_timeline) {
-    step_stats_proto = pss->rpc_stats;
+    StepStats step_stats_proto;
+    step_stats_proto.Swap(&pss->rpc_stats);
     for (size_t i = 0; i < partitions_.size(); ++i) {
-      const StepStats& ss = pss->step_stats[i];
-      step_stats_proto.MergeFrom(ss);
+      step_stats_proto.MergeFrom(pss->step_stats[i]);
+      pss->step_stats[i].Clear();
     }
-    stats_publisher_->PublishStatsProto(step_stats_proto);
+    pss->step_stats.clear();
     // Copy the stats back, but only for on-demand profiling to avoid slowing
     // down calls that trigger the automatic profiling.
     if (options.trace_level() == RunOptions::FULL_TRACE) {
       resp->mutable_step_stats()->Swap(&step_stats_proto);
+    } else {
+      // If FULL_TRACE, it can be fetched from Session API, no need for
+      // duplicated publishing.
+      stats_publisher_->PublishStatsProto(step_stats_proto);
     }
   }
 }
-- 
GitLab


From 9f8523640848f5891d31049b86d5d310cf0f843d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 23 Oct 2017 11:37:06 -0700
Subject: [PATCH 682/909] Update ops-related pbtxt files.

PiperOrigin-RevId: 173145770
---
 .../core/ops/compat/ops_history.v1.pbtxt      | 24 ++++++++
 tensorflow/core/ops/ops.pbtxt                 | 57 ++++++++++---------
 2 files changed, 54 insertions(+), 27 deletions(-)

diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index c5ceb14a09..92037c1997 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -9582,6 +9582,18 @@ op {
     }
   }
 }
+op {
+  name: "DeserializeIterator"
+  input_arg {
+    name: "resource_handle"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "serialized"
+    type: DT_VARIANT
+  }
+  is_stateful: true
+}
 op {
   name: "DeserializeManySparse"
   input_arg {
@@ -31465,6 +31477,18 @@ op {
     }
   }
 }
+op {
+  name: "SerializeIterator"
+  input_arg {
+    name: "resource_handle"
+    type: DT_RESOURCE
+  }
+  output_arg {
+    name: "serialized"
+    type: DT_VARIANT
+  }
+  is_stateful: true
+}
 op {
   name: "SerializeManySparse"
   input_arg {
diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index 623f5457bb..c037c99c19 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -6997,6 +6997,21 @@ op {
   summary: "Dequantize the \'input\' tensor into a float Tensor."
   description: "[min_range, max_range] are scalar floats that specify the range for\nthe \'input\' data. The \'mode\' attribute controls exactly which calculations are\nused to convert the float values to their quantized equivalents.\n\nIn \'MIN_COMBINED\' mode, each value of the tensor will undergo the following:\n\n```\nif T == qint8, in[i] += (range(T) + 1)/ 2.0\nout[i] = min_range + (in[i]* (max_range - min_range) / range(T))\n```\nhere `range(T) = numeric_limits<T>::max() - numeric_limits<T>::min()`\n\n*MIN_COMBINED Mode Example*\n\nIf the input comes from a QuantizedRelu6, the output type is\nquint8 (range of 0-255) but the possible range of QuantizedRelu6 is\n0-6.  The min_range and max_range values are therefore 0.0 and 6.0.\nDequantize on quint8 will take each value, cast to float, and multiply\nby 6 / 255.\nNote that if quantizedtype is qint8, the operation will additionally add\neach value by 128 prior to casting.\n\nIf the mode is \'MIN_FIRST\', then this approach is used:\n\n```c++\nnumber_of_steps = 1 << (# of bits in T)\nrange_adjust = number_of_steps / (number_of_steps - 1)\nrange = (range_max - range_min) * range_adjust\nrange_scale = range / number_of_steps\nconst double offset_input = static_cast<double>(input) - lowest_quantized;\nresult = range_min + ((input - numeric_limits<T>::min()) * range_scale)\n```\n\n*SCALED mode Example*\n\n`SCALED` mode matches the quantization approach used in\n`QuantizeAndDequantize{V2|V3}`.\n\nIf the mode is `SCALED`, we do not use the full range of the output type,\nchoosing to elide the lowest possible value for symmetry (e.g., output range is\n-127 to 127, not -128 to 127 for signed 8 bit quantization), so that 0.0 maps to\n0.\n\nWe first find the range of values in our tensor. The\nrange we use is always centered on 0, so we find m such that\n```c++\n  m = max(abs(input_min), abs(input_max))\n```\n\nOur input tensor range is then `[-m, m]`.\n\nNext, we choose our fixed-point quantization buckets, `[min_fixed, max_fixed]`.\nIf T is signed, this is\n```\n  num_bits = sizeof(T) * 8\n  [min_fixed, max_fixed] =\n      [-(1 << (num_bits - 1) - 1), (1 << (num_bits - 1)) - 1]\n```\n\nOtherwise, if T is unsigned, the fixed-point range is\n```\n  [min_fixed, max_fixed] = [0, (1 << num_bits) - 1]\n```\n\nFrom this we compute our scaling factor, s:\n```c++\n  s = (2 * m) / (max_fixed - min_fixed)\n```\n\nNow we can dequantize the elements of our tensor:\n```c++\nresult = input * s\n```"
 }
+op {
+  name: "DeserializeIterator"
+  input_arg {
+    name: "resource_handle"
+    description: "A handle to an iterator resource."
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "serialized"
+    description: "A variant tensor storing the state of the iterator contained in the\nresource."
+    type: DT_VARIANT
+  }
+  summary: "Converts the given variant tensor to an iterator and stores it in the given resource."
+  is_stateful: true
+}
 op {
   name: "DeserializeManySparse"
   input_arg {
@@ -23025,19 +23040,6 @@ op {
   description: "Reads a tensor stored in one or several files. If there are several files (for\ninstance because a tensor was saved as slices), `file_pattern` may contain\nwildcard symbols (`*` and `?`) in the filename portion only, not in the\ndirectory portion.\n\nIf a `file_pattern` matches several files, `preferred_shard` can be used to hint\nin which file the requested tensor is likely to be found. This op will first\nopen the file at index `preferred_shard` in the list of matching files and try\nto restore tensors from that file.  Only if some tensors or tensor slices are\nnot found in that first file, then the Op opens all the files. Setting\n`preferred_shard` to match the value passed as the `shard` input\nof a matching `Save` Op may speed up Restore.  This attribute only affects\nperformance, not correctness.  The default value -1 means files are processed in\norder.\n\nSee also `RestoreSlice`."
   is_stateful: true
 }
-op {
-  name: "RestoreIterator"
-  input_arg {
-    name: "iterator"
-    type: DT_RESOURCE
-  }
-  input_arg {
-    name: "path"
-    type: DT_STRING
-  }
-  summary: "Restores the state of the `iterator` from the checkpoint saved at `path` using \"SaveIterator\"."
-  is_stateful: true
-}
 op {
   name: "RestoreSlice"
   input_arg {
@@ -23632,20 +23634,6 @@ op {
   description: "The size of `tensor_names` must match the number of tensors in `data`. `data[i]`\nis written to `filename` with name `tensor_names[i]`.\n\nSee also `SaveSlices`."
   is_stateful: true
 }
-op {
-  name: "SaveIterator"
-  input_arg {
-    name: "iterator"
-    type: DT_RESOURCE
-  }
-  input_arg {
-    name: "path"
-    type: DT_STRING
-  }
-  summary: "Saves the state of the `iterator` at `path`."
-  description: "This state can be restored using \"RestoreIterator\"."
-  is_stateful: true
-}
 op {
   name: "SaveSlices"
   input_arg {
@@ -24990,6 +24978,21 @@ op {
   }
   summary: "Computes gradients for the scaled exponential linear (Selu) operation."
 }
+op {
+  name: "SerializeIterator"
+  input_arg {
+    name: "resource_handle"
+    description: "A handle to an iterator resource."
+    type: DT_RESOURCE
+  }
+  output_arg {
+    name: "serialized"
+    description: "A variant tensor storing the state of the iterator contained in the\nresource."
+    type: DT_VARIANT
+  }
+  summary: "Converts the given `resource_handle` representing an iterator to a variant tensor."
+  is_stateful: true
+}
 op {
   name: "SerializeManySparse"
   input_arg {
-- 
GitLab


From d25397281cda45e56693979a20d8e622a0b00b29 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 23 Oct 2017 11:45:44 -0700
Subject: [PATCH 683/909] Go: Update generated wrapper functions for TensorFlow
 ops.

PiperOrigin-RevId: 173147086
---
 tensorflow/go/op/wrappers.go | 224 ++++++++++++++++++-----------------
 1 file changed, 116 insertions(+), 108 deletions(-)

diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go
index c117711c81..b3b317013f 100644
--- a/tensorflow/go/op/wrappers.go
+++ b/tensorflow/go/op/wrappers.go
@@ -4932,80 +4932,6 @@ func PriorityQueueV2(scope *Scope, shapes []tf.Shape, optional ...PriorityQueueV
 	return op.Output(0)
 }
 
-// FIFOQueueV2Attr is an optional argument to FIFOQueueV2.
-type FIFOQueueV2Attr func(optionalAttr)
-
-// FIFOQueueV2Shapes sets the optional shapes attribute to value.
-//
-// value: The shape of each component in a value. The length of this attr must
-// be either 0 or the same as the length of component_types. If the length of
-// this attr is 0, the shapes of queue elements are not constrained, and
-// only one element may be dequeued at a time.
-// If not specified, defaults to <>
-//
-// REQUIRES: len(value) >= 0
-func FIFOQueueV2Shapes(value []tf.Shape) FIFOQueueV2Attr {
-	return func(m optionalAttr) {
-		m["shapes"] = value
-	}
-}
-
-// FIFOQueueV2Capacity sets the optional capacity attribute to value.
-//
-// value: The upper bound on the number of elements in this queue.
-// Negative numbers mean no limit.
-// If not specified, defaults to -1
-func FIFOQueueV2Capacity(value int64) FIFOQueueV2Attr {
-	return func(m optionalAttr) {
-		m["capacity"] = value
-	}
-}
-
-// FIFOQueueV2Container sets the optional container attribute to value.
-//
-// value: If non-empty, this queue is placed in the given container.
-// Otherwise, a default container is used.
-// If not specified, defaults to ""
-func FIFOQueueV2Container(value string) FIFOQueueV2Attr {
-	return func(m optionalAttr) {
-		m["container"] = value
-	}
-}
-
-// FIFOQueueV2SharedName sets the optional shared_name attribute to value.
-//
-// value: If non-empty, this queue will be shared under the given name
-// across multiple sessions.
-// If not specified, defaults to ""
-func FIFOQueueV2SharedName(value string) FIFOQueueV2Attr {
-	return func(m optionalAttr) {
-		m["shared_name"] = value
-	}
-}
-
-// A queue that produces elements in first-in first-out order.
-//
-// Arguments:
-//	component_types: The type of each component in a value.
-//
-// Returns The handle to the queue.
-func FIFOQueueV2(scope *Scope, component_types []tf.DataType, optional ...FIFOQueueV2Attr) (handle tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"component_types": component_types}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "FIFOQueueV2",
-
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
 // StridedSliceAttr is an optional argument to StridedSlice.
 type StridedSliceAttr func(optionalAttr)
 
@@ -5385,6 +5311,101 @@ func DynamicStitch(scope *Scope, indices []tf.Output, data []tf.Output) (merged
 	return op.Output(0)
 }
 
+// FIFOQueueV2Attr is an optional argument to FIFOQueueV2.
+type FIFOQueueV2Attr func(optionalAttr)
+
+// FIFOQueueV2Shapes sets the optional shapes attribute to value.
+//
+// value: The shape of each component in a value. The length of this attr must
+// be either 0 or the same as the length of component_types. If the length of
+// this attr is 0, the shapes of queue elements are not constrained, and
+// only one element may be dequeued at a time.
+// If not specified, defaults to <>
+//
+// REQUIRES: len(value) >= 0
+func FIFOQueueV2Shapes(value []tf.Shape) FIFOQueueV2Attr {
+	return func(m optionalAttr) {
+		m["shapes"] = value
+	}
+}
+
+// FIFOQueueV2Capacity sets the optional capacity attribute to value.
+//
+// value: The upper bound on the number of elements in this queue.
+// Negative numbers mean no limit.
+// If not specified, defaults to -1
+func FIFOQueueV2Capacity(value int64) FIFOQueueV2Attr {
+	return func(m optionalAttr) {
+		m["capacity"] = value
+	}
+}
+
+// FIFOQueueV2Container sets the optional container attribute to value.
+//
+// value: If non-empty, this queue is placed in the given container.
+// Otherwise, a default container is used.
+// If not specified, defaults to ""
+func FIFOQueueV2Container(value string) FIFOQueueV2Attr {
+	return func(m optionalAttr) {
+		m["container"] = value
+	}
+}
+
+// FIFOQueueV2SharedName sets the optional shared_name attribute to value.
+//
+// value: If non-empty, this queue will be shared under the given name
+// across multiple sessions.
+// If not specified, defaults to ""
+func FIFOQueueV2SharedName(value string) FIFOQueueV2Attr {
+	return func(m optionalAttr) {
+		m["shared_name"] = value
+	}
+}
+
+// A queue that produces elements in first-in first-out order.
+//
+// Arguments:
+//	component_types: The type of each component in a value.
+//
+// Returns The handle to the queue.
+func FIFOQueueV2(scope *Scope, component_types []tf.DataType, optional ...FIFOQueueV2Attr) (handle tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"component_types": component_types}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "FIFOQueueV2",
+
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Converts the given `resource_handle` representing an iterator to a variant tensor.
+//
+// Arguments:
+//	resource_handle: A handle to an iterator resource.
+//
+// Returns A variant tensor storing the state of the iterator contained in the
+// resource.
+func SerializeIterator(scope *Scope, resource_handle tf.Output) (serialized tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "SerializeIterator",
+		Input: []tf.Input{
+			resource_handle,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Return a tensor with the same shape and contents as the input tensor or value.
 func Identity(scope *Scope, input tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
@@ -5575,40 +5596,6 @@ func IteratorGetNext(scope *Scope, iterator tf.Output, output_types []tf.DataTyp
 	return components
 }
 
-// Restores the state of the `iterator` from the checkpoint saved at `path` using "SaveIterator".
-//
-// Returns the created operation.
-func RestoreIterator(scope *Scope, iterator tf.Output, path tf.Output) (o *tf.Operation) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "RestoreIterator",
-		Input: []tf.Input{
-			iterator, path,
-		},
-	}
-	return scope.AddOperation(opspec)
-}
-
-// Saves the state of the `iterator` at `path`.
-//
-// This state can be restored using "RestoreIterator".
-//
-// Returns the created operation.
-func SaveIterator(scope *Scope, iterator tf.Output, path tf.Output) (o *tf.Operation) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "SaveIterator",
-		Input: []tf.Input{
-			iterator, path,
-		},
-	}
-	return scope.AddOperation(opspec)
-}
-
 // Makes a new iterator from the given `dataset` and stores it in `iterator`.
 //
 // This operation may be executed multiple times. Each execution will reset the
@@ -5919,6 +5906,27 @@ func TensorArrayConcatV2(scope *Scope, handle tf.Output, flow_in tf.Output, dtyp
 	return op.Output(0), op.Output(1)
 }
 
+// Converts the given variant tensor to an iterator and stores it in the given resource.
+//
+// Arguments:
+//	resource_handle: A handle to an iterator resource.
+//	serialized: A variant tensor storing the state of the iterator contained in the
+// resource.
+//
+// Returns the created operation.
+func DeserializeIterator(scope *Scope, resource_handle tf.Output, serialized tf.Output) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "DeserializeIterator",
+		Input: []tf.Input{
+			resource_handle, serialized,
+		},
+	}
+	return scope.AddOperation(opspec)
+}
+
 // Concatenates tensors along one dimension.
 //
 // Arguments:
-- 
GitLab


From 14506b6a5b26f3351a6d0b8097f0e3d3f2ddf82c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 23 Oct 2017 12:44:54 -0700
Subject: [PATCH 684/909] [TF:XLA] Replace a Mul in XlaArgMinMaxOp with an And

PiperOrigin-RevId: 173155697
---
 tensorflow/compiler/tf2xla/kernels/BUILD      |  1 +
 .../compiler/tf2xla/kernels/index_ops.cc      | 19 ++++++++++++++-----
 2 files changed, 15 insertions(+), 5 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/kernels/BUILD b/tensorflow/compiler/tf2xla/kernels/BUILD
index f44d61de68..4ee7989824 100644
--- a/tensorflow/compiler/tf2xla/kernels/BUILD
+++ b/tensorflow/compiler/tf2xla/kernels/BUILD
@@ -83,6 +83,7 @@ tf_kernel_library(
         "//tensorflow/compiler/tf2xla:xla_compiler",
         "//tensorflow/compiler/tf2xla/ops:sendrecv_ops",
         "//tensorflow/compiler/xla:literal_util",
+        "//tensorflow/compiler/xla:shape_util",
         "//tensorflow/compiler/xla:util",
         "//tensorflow/compiler/xla:xla_data_proto",
         "//tensorflow/compiler/xla/client:client_library",
diff --git a/tensorflow/compiler/tf2xla/kernels/index_ops.cc b/tensorflow/compiler/tf2xla/kernels/index_ops.cc
index db7d556630..b8769b3ea2 100644
--- a/tensorflow/compiler/tf2xla/kernels/index_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/index_ops.cc
@@ -22,6 +22,7 @@ limitations under the License.
 #include "tensorflow/compiler/tf2xla/xla_op_kernel.h"
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
 #include "tensorflow/compiler/xla/client/lib/arithmetic.h"
+#include "tensorflow/compiler/xla/shape_util.h"
 #include "tensorflow/core/framework/kernel_def_builder.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
@@ -82,16 +83,24 @@ void XlaArgMinMaxOp::Compile(XlaOpKernelContext* ctx) {
   std::iota(broadcast_dims.begin(), broadcast_dims.begin() + axis, 0);
   std::iota(broadcast_dims.begin() + axis, broadcast_dims.end(), axis + 1);
   // Compute a mask that has 1s for elements equal to the maximum.
-  xla::ComputationDataHandle mask = b->ConvertElementType(
+  xla::ComputationDataHandle partial_mask = b->ConvertElementType(
       b->Eq(input, input_max, broadcast_dims), xla_index_type);
 
-  // Multiply by the vector [0, 1, 2, ...] to convert each 1 into its index.
-  // TODO(phawkins): add a bitwise And operator to HLO, use a bitwise and
-  // instead of a multiplication here.
+  // In order to make identity elements for a bitwise And, we:
+  //   Left shift the 1 to the leftmost bit, yielding 0x10...0
+  //   Arithmetic right shift the 1 back to the rightmost bit, yielding 0xFF...F
+  int32 bits_in_type =
+      xla::ShapeUtil::ByteSizeOfPrimitiveType(xla_index_type) * 8 - 1;
+  xla::ComputationDataHandle shift_amount =
+      XlaHelpers::IntegerLiteral(b, index_type, bits_in_type);
+  xla::ComputationDataHandle full_mask = b->ShiftRightArithmetic(
+      b->ShiftLeft(partial_mask, shift_amount), shift_amount);
+
+  // And with the vector [0, 1, 2, ...] to convert each 0xFF...F into its index.
   xla::ComputationDataHandle iota;
   OP_REQUIRES_OK(ctx, XlaHelpers::Iota(b, index_type, axis_size, &iota));
   xla::ComputationDataHandle product =
-      b->Mul(mask, iota, /*broadcast_dimensions=*/{axis});
+      b->And(full_mask, iota, /*broadcast_dimensions=*/{axis});
 
   // If there are multiple maximum elements, choose the one with the highest
   // index.
-- 
GitLab


From 1a5e8ead650606aba57ec830c82000a6c38e5695 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 23 Oct 2017 13:08:06 -0700
Subject: [PATCH 685/909] Automated g4 rollback of changelist 172936802

PiperOrigin-RevId: 173158990
---
 tensorflow/contrib/batching/BUILD             |  22 +
 .../adaptive_shared_batch_scheduler.h         | 463 ++++++++++++++++++
 .../adaptive_shared_batch_scheduler_test.cc   | 438 +++++++++++++++++
 tensorflow/contrib/batching/batch_scheduler.h |   2 +-
 4 files changed, 924 insertions(+), 1 deletion(-)
 create mode 100644 tensorflow/contrib/batching/adaptive_shared_batch_scheduler.h
 create mode 100644 tensorflow/contrib/batching/adaptive_shared_batch_scheduler_test.cc

diff --git a/tensorflow/contrib/batching/BUILD b/tensorflow/contrib/batching/BUILD
index 1555a3427f..ae3f48f1b2 100644
--- a/tensorflow/contrib/batching/BUILD
+++ b/tensorflow/contrib/batching/BUILD
@@ -69,6 +69,28 @@ tf_cc_test(
     ],
 )
 
+cc_library(
+    name = "adaptive_shared_batch_scheduler",
+    hdrs = ["adaptive_shared_batch_scheduler.h"],
+    deps = [
+        ":batch_scheduler",
+        "//tensorflow/contrib/batching/util:periodic_function_dynamic",
+        "//tensorflow/core:lib",
+    ],
+)
+
+tf_cc_test(
+    name = "adaptive_shared_batch_scheduler_test",
+    srcs = ["adaptive_shared_batch_scheduler_test.cc"],
+    deps = [
+        ":adaptive_shared_batch_scheduler",
+        "//tensorflow/contrib/batching/test_util:fake_clock_env",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+    ],
+)
+
 cc_library(
     name = "basic_batch_scheduler",
     hdrs = ["basic_batch_scheduler.h"],
diff --git a/tensorflow/contrib/batching/adaptive_shared_batch_scheduler.h b/tensorflow/contrib/batching/adaptive_shared_batch_scheduler.h
new file mode 100644
index 0000000000..a0606427a5
--- /dev/null
+++ b/tensorflow/contrib/batching/adaptive_shared_batch_scheduler.h
@@ -0,0 +1,463 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef THIRD_PARTY_TENSORFLOW_CONTRIB_BATCHING_ADAPTIVE_SHARED_BATCH_SCHEDULER_H_
+#define THIRD_PARTY_TENSORFLOW_CONTRIB_BATCHING_ADAPTIVE_SHARED_BATCH_SCHEDULER_H_
+
+#include <functional>
+#include <memory>
+#include <queue>
+#include <unordered_map>
+#include <vector>
+
+#include "tensorflow/contrib/batching/batch_scheduler.h"
+#include "tensorflow/contrib/batching/util/periodic_function.h"
+#include "tensorflow/core/lib/core/errors.h"
+#include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/lib/core/threadpool.h"
+#include "tensorflow/core/platform/cpu_info.h"
+#include "tensorflow/core/platform/env.h"
+#include "tensorflow/core/platform/thread_annotations.h"
+#include "tensorflow/core/platform/types.h"
+
+namespace tensorflow {
+namespace serving {
+namespace internal {
+template <typename TaskType>
+class ASBSBatch;
+
+template <typename TaskType>
+class ASBSQueue;
+}  // namespace internal
+
+// Shared batch scheduler designed to minimize latency. The scheduler keeps
+// track of a number of queues (one per model or model version) which are
+// continuously enqueuing requests. The scheduler groups the requests into
+// batches which it periodically sends off for processing (see
+// shared_batch_scheduler.h for more details). The AdaptiveSharedBatchScheduler
+// prioritizes batches by age (i.e. the batch's oldest request) irrespective of
+// queue. The scheduler will process the oldest batch at an adjustable rate,
+// regardless of batch size. The user can provide feedback to help set this rate
+// to achieve some goal (i.e. minimize overall latency, limit cpu usage, etc).
+//
+// The rate (or rather, the corresponding period) is adjusted each time a batch
+// is processed, using an exponentially weighted moving average to smooth
+// potentially noisy feedback:
+// ewma_feedback = ((N - 1) * ewma_feedback + feedback()) / N
+// period *= (1 + K * emwa_feedback)
+//
+// Some potential use cases:
+// Hardware Accelerators (GPUs & TPUs) - If some phase of batch processing
+//   involves serial processing by a device, from a latency perspective it is
+//   desirable to keep the device evenly loaded, avoiding the need to wait for
+//   the device to process prior batches.
+//   feedback = num_pending_on_device() - desired_pending.
+// CPU utilization - If the batch processing is cpu dominated, you can reap
+//   latency gains when underutilized by increasing the processing rate, but
+//   back the rate off when the load increases to avoid overload.
+//   feedback = cpu_rate() - desired_cpu_rate.
+
+template <typename TaskType>
+class AdaptiveSharedBatchScheduler
+    : public std::enable_shared_from_this<
+          AdaptiveSharedBatchScheduler<TaskType>> {
+ public:
+  struct Options {
+    // The name to use for the pool of batch threads.
+    string thread_pool_name = {"batch_threads"};
+    // Number of batch processing threads; equivalently the maximum number of
+    // concurrently running batches.
+    int64 num_batch_threads = port::NumSchedulableCPUs();
+    // The environment to use (typically only overridden by test code).
+    Env* env = Env::Default();
+    // Initial batch scheduling period in microseconds. Will be altered for
+    // non-zero rate_feedback.
+    double initial_scheduling_period_micros = 500;
+    // Minimum batch scheduling period in microseconds. Recommend setting this
+    // value greater than 0, otherwise it may take a while to recover from a
+    // sustained time of negative scheduling_period_feedback (which may occur
+    // under low load).
+    double min_scheduling_period_micros = 100;
+    // Maximum batch scheduling period in microseconds.
+    double max_scheduling_period_micros = 10000;
+    // Feedback function used to modify the scheduling period each time a batch
+    // is scheduled.  Should return values roughly O(1), with positive values
+    // resulting in an increased period.
+    std::function<double()> scheduling_period_feedback{[] { return 0.; }};
+    // To handle potentially noisy scheduling_period_feedback, the period is
+    // adjusted using an exponentially weighted moving average over the previous
+    // feedback_smoothing_batches batches.  Must be greater than 0.
+    int64 feedback_smoothing_batches = 10;
+  };
+
+  // Ownership is shared between the caller of Create() and any queues created
+  // via AddQueue().
+  static Status Create(
+      const Options& options,
+      std::shared_ptr<AdaptiveSharedBatchScheduler<TaskType>>* scheduler);
+
+  struct QueueOptions {
+    // Maximum size of each batch.
+    int max_batch_size = 1000;
+    // Maximum number of enqueued (i.e. non-scheduled) batches.
+    int max_enqueued_batches = 10;
+  };
+
+  using BatchProcessor = std::function<void(std::unique_ptr<Batch<TaskType>>)>;
+
+  // Adds queue (and its callback) to be managed by this scheduler.
+  Status AddQueue(const QueueOptions& options,
+                  BatchProcessor process_batch_callback,
+                  std::unique_ptr<BatchScheduler<TaskType>>* queue);
+
+ private:
+  // access to AddBatch, RemoveQueue, GetEnv.
+  friend class internal::ASBSQueue<TaskType>;
+
+  explicit AdaptiveSharedBatchScheduler(const Options& options);
+
+  // Batch scheduling function which runs every scheduling_period_ microseconds.
+  void ProcessOneBatch();
+
+  // Notifies scheduler of non-empty batch which is eligible for processing.
+  void AddBatch(internal::ASBSBatch<TaskType>*);
+
+  // Removes queue from scheduler.
+  void RemoveQueue(const internal::ASBSQueue<TaskType>* queue);
+
+  Env* GetEnv() const { return options_.env; }
+
+  const Options options_;
+
+  struct BatchCompare {
+    bool operator()(const internal::ASBSBatch<TaskType>* a,
+                    const internal::ASBSBatch<TaskType>* b);
+  };
+
+  // Collection of batches added by AddBatch, ordered by age. Owned by scheduler
+  // until they are released for processing.
+  std::priority_queue<const internal::ASBSBatch<TaskType>*,
+                      std::vector<internal::ASBSBatch<TaskType>*>, BatchCompare>
+      batches_ GUARDED_BY(mu_);
+
+  // Unowned queues and callbacks added by AddQueue.
+  std::unordered_map<const internal::ASBSQueue<TaskType>*, BatchProcessor>
+      queues_and_callbacks_ GUARDED_BY(mu_);
+
+  mutex mu_;
+
+  // Responsible for running ProcessOneBatch. PeriodicFunction was used in order
+  // to check for deletion so that the thread can be shut down.
+  std::unique_ptr<PeriodicFunction> scheduling_thread_;
+
+  // Responsible for running the batch processing callbacks.
+  std::unique_ptr<thread::ThreadPool> batch_thread_pool_;
+
+  // Time interval in microseconds between successive ProcessOneBatch calls.
+  double scheduling_period_;
+
+  // Exponentially weighted moving average of
+  // options_.scheduling_period_feedback() evaluated in each ProcessOneBatch
+  // call.
+  double ewma_feedback_ = 0;
+
+  TF_DISALLOW_COPY_AND_ASSIGN(AdaptiveSharedBatchScheduler);
+};
+
+//////////////////////////////////////////////////////////
+// Implementation details follow. API users need not read.
+
+namespace internal {
+// Consolidates tasks into batches, passing them off to the
+// AdaptiveSharedBatchScheduler for processing.
+template <typename TaskType>
+class ASBSQueue : public BatchScheduler<TaskType> {
+ public:
+  using QueueOptions =
+      typename AdaptiveSharedBatchScheduler<TaskType>::QueueOptions;
+
+  ASBSQueue(std::shared_ptr<AdaptiveSharedBatchScheduler<TaskType>> scheduler,
+            const QueueOptions& options);
+
+  ~ASBSQueue() override;
+
+  // Adds task to current batch. Fails if the task size is larger than the batch
+  // size or if the current batch is full and this queue's number of outstanding
+  // batches is at its maximum.
+  Status Schedule(std::unique_ptr<TaskType>* task) override;
+
+  // Number of tasks waiting to be scheduled.
+  size_t NumEnqueuedTasks() const override;
+
+  // Number of size 1 tasks which could currently be scheduled without failing.
+  size_t SchedulingCapacity() const override;
+
+  // Notifies queue that a batch is about to be scheduled; the queue should not
+  // place any more tasks in this batch.
+  void ReleaseBatch(const ASBSBatch<TaskType>* batch);
+
+ private:
+  std::shared_ptr<AdaptiveSharedBatchScheduler<TaskType>> scheduler_;
+  const QueueOptions options_;
+  // Owned by scheduler_.
+  ASBSBatch<TaskType>* current_batch_ GUARDED_BY(mu_) = nullptr;
+  int64 num_enqueued_batches_ GUARDED_BY(mu_) = 0;
+  int64 num_enqueued_tasks_ GUARDED_BY(mu_) = 0;
+  mutable mutex mu_;
+  TF_DISALLOW_COPY_AND_ASSIGN(ASBSQueue);
+};
+
+// Batch which remembers when and by whom it was created.
+template <typename TaskType>
+class ASBSBatch : public Batch<TaskType> {
+ public:
+  ASBSBatch(ASBSQueue<TaskType>* queue, int64 creation_time_micros)
+      : queue_(queue), creation_time_micros_(creation_time_micros) {}
+
+  ~ASBSBatch() override {}
+
+  ASBSQueue<TaskType>* queue() const { return queue_; }
+
+  int64 creation_time_micros() const { return creation_time_micros_; }
+
+ private:
+  ASBSQueue<TaskType>* queue_;
+  const int64 creation_time_micros_;
+  TF_DISALLOW_COPY_AND_ASSIGN(ASBSBatch);
+};
+}  // namespace internal
+
+// ---------------- AdaptiveSharedBatchScheduler ----------------
+
+template <typename TaskType>
+Status AdaptiveSharedBatchScheduler<TaskType>::Create(
+    const Options& options,
+    std::shared_ptr<AdaptiveSharedBatchScheduler<TaskType>>* scheduler) {
+  if (options.num_batch_threads < 1) {
+    return errors::InvalidArgument("num_batch_threads must be positive; was ",
+                                   options.num_batch_threads);
+  }
+  if (options.min_scheduling_period_micros < 0) {
+    return errors::InvalidArgument(
+        "min_scheduling_period_micros must be >= 0; was ",
+        options.min_scheduling_period_micros);
+  }
+  if (options.min_scheduling_period_micros >
+      options.initial_scheduling_period_micros) {
+    return errors::InvalidArgument(
+        "initial_scheduling_period_micros (",
+        options.initial_scheduling_period_micros,
+        ") must be >= min_scheduling_period_micros (",
+        options.min_scheduling_period_micros, ")");
+  }
+  if (options.initial_scheduling_period_micros >
+      options.max_scheduling_period_micros) {
+    return errors::InvalidArgument(
+        "initial_scheduling_period_micros (",
+        options.initial_scheduling_period_micros,
+        ") must be <= max_scheduling_period_micros (",
+        options.max_scheduling_period_micros, ")");
+  }
+  if (options.feedback_smoothing_batches < 1) {
+    return errors::InvalidArgument(
+        "feedback_smoothing_batches must be positive; was ",
+        options.feedback_smoothing_batches);
+  }
+  scheduler->reset(new AdaptiveSharedBatchScheduler<TaskType>(options));
+  return Status::OK();
+}
+
+template <typename TaskType>
+AdaptiveSharedBatchScheduler<TaskType>::AdaptiveSharedBatchScheduler(
+    const Options& options)
+    : options_(options),
+      scheduling_period_(options.initial_scheduling_period_micros) {
+  PeriodicFunction::Options opts;
+  opts.thread_name_prefix = "scheduling_thread";
+  opts.env = GetEnv();
+  scheduling_thread_.reset(
+      new PeriodicFunction([this] { ProcessOneBatch(); }, 0, opts));
+  batch_thread_pool_.reset(new thread::ThreadPool(
+      GetEnv(), options.thread_pool_name, options.num_batch_threads));
+}
+
+template <typename TaskType>
+Status AdaptiveSharedBatchScheduler<TaskType>::AddQueue(
+    const QueueOptions& options, BatchProcessor process_batch_callback,
+    std::unique_ptr<BatchScheduler<TaskType>>* queue) {
+  if (options.max_batch_size <= 0) {
+    return errors::InvalidArgument("max_batch_size must be positive; was ",
+                                   options.max_batch_size);
+  }
+  if (options.max_enqueued_batches <= 0) {
+    return errors::InvalidArgument(
+        "max_enqueued_batches must be positive; was ",
+        options.max_enqueued_batches);
+  }
+  internal::ASBSQueue<TaskType>* asbs_queue_raw;
+  queue->reset(asbs_queue_raw = new internal::ASBSQueue<TaskType>(
+                   this->shared_from_this(), options));
+  mutex_lock l(mu_);
+  queues_and_callbacks_[asbs_queue_raw] = process_batch_callback;
+  return Status::OK();
+}
+
+template <typename TaskType>
+void AdaptiveSharedBatchScheduler<TaskType>::AddBatch(
+    internal::ASBSBatch<TaskType>* batch) {
+  mutex_lock l(mu_);
+  batches_.push(batch);
+}
+
+template <typename TaskType>
+void AdaptiveSharedBatchScheduler<TaskType>::RemoveQueue(
+    const internal::ASBSQueue<TaskType>* queue) {
+  mutex_lock l(mu_);
+  queues_and_callbacks_.erase(queue);
+}
+
+template <typename TaskType>
+void AdaptiveSharedBatchScheduler<TaskType>::ProcessOneBatch() {
+  static const double kFeedbackMultiplier = .001;
+  internal::ASBSBatch<TaskType>* batch = nullptr;
+  BatchProcessor callback;
+  const int64 start_time_micros = GetEnv()->NowMicros();
+  {
+    mutex_lock l(mu_);
+    if (!batches_.empty()) {
+      batch = batches_.top();
+      batches_.pop();
+      callback = queues_and_callbacks_[batch->queue()];
+    }
+  }
+  if (batch != nullptr) {
+    double feedback = options_.scheduling_period_feedback();
+    const int64 N = options_.feedback_smoothing_batches;
+    ewma_feedback_ = ((N - 1) * ewma_feedback_ + feedback) / N;
+    scheduling_period_ *= (1 + kFeedbackMultiplier * ewma_feedback_);
+    if (scheduling_period_ < options_.min_scheduling_period_micros) {
+      scheduling_period_ = options_.min_scheduling_period_micros;
+    } else if (scheduling_period_ > options_.max_scheduling_period_micros) {
+      scheduling_period_ = options_.max_scheduling_period_micros;
+    }
+    // Queue may destroy itself after ReleaseBatch is called.
+    batch->queue()->ReleaseBatch(batch);
+    batch_thread_pool_->Schedule([callback, batch] {
+      callback(std::unique_ptr<Batch<TaskType>>(batch));
+    });
+  }
+  const int64 sleep_time =
+      scheduling_period_ - (GetEnv()->NowMicros() - start_time_micros);
+  if (sleep_time > 0) {
+    GetEnv()->SleepForMicroseconds(sleep_time);
+  }
+}
+
+template <typename TaskType>
+bool AdaptiveSharedBatchScheduler<TaskType>::BatchCompare::operator()(
+    const internal::ASBSBatch<TaskType>* a,
+    const internal::ASBSBatch<TaskType>* b) {
+  return a->creation_time_micros() > b->creation_time_micros();
+}
+
+// ---------------- ASBSQueue ----------------
+
+namespace internal {
+template <typename TaskType>
+ASBSQueue<TaskType>::ASBSQueue(
+    std::shared_ptr<AdaptiveSharedBatchScheduler<TaskType>> scheduler,
+    const QueueOptions& options)
+    : scheduler_(scheduler), options_(options) {}
+
+template <typename TaskType>
+ASBSQueue<TaskType>::~ASBSQueue() {
+  // Wait until last batch has been scheduled.
+  const int kSleepMicros = 1000;
+  for (;;) {
+    {
+      mutex_lock l(mu_);
+      if (num_enqueued_batches_ == 0) {
+        break;
+      }
+    }
+    scheduler_->GetEnv()->SleepForMicroseconds(kSleepMicros);
+  }
+  scheduler_->RemoveQueue(this);
+}
+
+template <typename TaskType>
+Status ASBSQueue<TaskType>::Schedule(std::unique_ptr<TaskType>* task) {
+  bool added_new_batch = false;
+  size_t size = (*task)->size();
+  if (size > options_.max_batch_size) {
+    return errors::InvalidArgument("Task size ", size,
+                                   " is larger than maximum batch size ",
+                                   options_.max_batch_size);
+  }
+  {
+    mutex_lock l(mu_);
+    // Current batch is full, create another if allowed.
+    if (current_batch_ &&
+        current_batch_->size() + size > options_.max_batch_size) {
+      if (num_enqueued_batches_ >= options_.max_enqueued_batches) {
+        return errors::Unavailable("The batch scheduling queue is full");
+      }
+      current_batch_->Close();
+      current_batch_ = nullptr;
+    }
+    if (!current_batch_) {
+      added_new_batch = true;
+      num_enqueued_batches_++;
+      current_batch_ =
+          new ASBSBatch<TaskType>(this, scheduler_->GetEnv()->NowMicros());
+    }
+    current_batch_->AddTask(std::move(*task));
+    num_enqueued_tasks_++;
+  }
+  if (added_new_batch) scheduler_->AddBatch(current_batch_);
+  return Status::OK();
+}
+
+template <typename TaskType>
+void ASBSQueue<TaskType>::ReleaseBatch(const ASBSBatch<TaskType>* batch) {
+  mutex_lock l(mu_);
+  num_enqueued_batches_--;
+  num_enqueued_tasks_ -= batch->num_tasks();
+  if (batch == current_batch_) {
+    current_batch_->Close();
+    current_batch_ = nullptr;
+  }
+}
+
+template <typename TaskType>
+size_t ASBSQueue<TaskType>::NumEnqueuedTasks() const {
+  mutex_lock l(mu_);
+  return num_enqueued_tasks_;
+}
+
+template <typename TaskType>
+size_t ASBSQueue<TaskType>::SchedulingCapacity() const {
+  mutex_lock l(mu_);
+  const int current_batch_capacity =
+      current_batch_ ? options_.max_batch_size - current_batch_->size() : 0;
+  const int spare_batches =
+      options_.max_enqueued_batches - num_enqueued_batches_;
+  return spare_batches * options_.max_batch_size + current_batch_capacity;
+}
+}  // namespace internal
+}  // namespace serving
+}  // namespace tensorflow
+
+#endif  // THIRD_PARTY_TENSORFLOW_CONTRIB_BATCHING_ADAPTIVE_SHARED_BATCH_SCHEDULER_H_
diff --git a/tensorflow/contrib/batching/adaptive_shared_batch_scheduler_test.cc b/tensorflow/contrib/batching/adaptive_shared_batch_scheduler_test.cc
new file mode 100644
index 0000000000..a07cd6d834
--- /dev/null
+++ b/tensorflow/contrib/batching/adaptive_shared_batch_scheduler_test.cc
@@ -0,0 +1,438 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/contrib/batching/adaptive_shared_batch_scheduler.h"
+
+#include "tensorflow/contrib/batching/test_util/fake_clock_env.h"
+#include "tensorflow/core/lib/core/notification.h"
+#include "tensorflow/core/lib/core/status_test_util.h"
+#include "tensorflow/core/platform/macros.h"
+#include "tensorflow/core/platform/test.h"
+
+namespace tensorflow {
+namespace serving {
+namespace anonymous {
+
+class FakeTask : public BatchTask {
+ public:
+  explicit FakeTask(size_t size) : size_(size) {}
+
+  ~FakeTask() override = default;
+
+  size_t size() const override { return size_; }
+
+ private:
+  const size_t size_;
+
+  TF_DISALLOW_COPY_AND_ASSIGN(FakeTask);
+};
+
+// Creates a FakeTask of size 'task_size', and calls 'scheduler->Schedule()' on
+// that task. Returns the resulting status.
+Status ScheduleTask(size_t task_size, BatchScheduler<FakeTask>* scheduler) {
+  std::unique_ptr<FakeTask> task(new FakeTask(task_size));
+  Status status = scheduler->Schedule(&task);
+  // Schedule() should have consumed 'task' iff it returned Status::OK.
+  CHECK_EQ(status.ok(), task == nullptr);
+  return status;
+}
+
+// Creates a thread that waits on 'start' and then advances the fake clock in
+// 'env' in a loop until 'stop' is notified. Useful for allowing objects that
+// use the clock to be destroyed.
+std::unique_ptr<Thread> CreateFakeClockAdvancerThread(
+    test_util::FakeClockEnv* env, Notification* start, Notification* stop) {
+  return std::unique_ptr<Thread>(Env::Default()->StartThread(
+      {}, "FakeClockAdvancerThread", [env, start, stop] {
+        start->WaitForNotification();
+        while (!stop->HasBeenNotified()) {
+          env->AdvanceByMicroseconds(10);
+          Env::Default()->SleepForMicroseconds(10);
+        }
+      }));
+}
+
+TEST(AdaptiveSharedBatchSchedulerTest, Basic) {
+  for (const bool delete_scheduler_early : {false, true}) {
+    for (const bool delete_queue_1_early : {false, true}) {
+      int queue_0_tasks = 0;
+      auto queue_0_callback =
+          [&queue_0_tasks](std::unique_ptr<Batch<FakeTask>> batch) {
+            ASSERT_TRUE(batch->IsClosed());
+            EXPECT_GT(batch->num_tasks(), 0);
+            for (int i = 0; i < batch->num_tasks(); i++) {
+              queue_0_tasks += batch->task(i).size();
+            }
+          };
+      int queue_1_tasks = 0;
+      auto queue_1_callback =
+          [&queue_1_tasks](std::unique_ptr<Batch<FakeTask>> batch) {
+            ASSERT_TRUE(batch->IsClosed());
+            EXPECT_GT(batch->num_tasks(), 0);
+            for (int i = 0; i < batch->num_tasks(); i++) {
+              queue_1_tasks += batch->task(i).size();
+            }
+          };
+      {
+        std::shared_ptr<AdaptiveSharedBatchScheduler<FakeTask>> scheduler;
+        TF_ASSERT_OK(
+            AdaptiveSharedBatchScheduler<FakeTask>::Create({}, &scheduler));
+
+        // Create two queues.
+        std::unique_ptr<BatchScheduler<FakeTask>> queue_0;
+        TF_ASSERT_OK(scheduler->AddQueue({}, queue_0_callback, &queue_0));
+        std::unique_ptr<BatchScheduler<FakeTask>> queue_1;
+        TF_ASSERT_OK(scheduler->AddQueue({}, queue_1_callback, &queue_1));
+
+        if (delete_scheduler_early) {
+          // Delete our copy of the scheduler. The queues should keep it alive
+          // under the covers.
+          scheduler = nullptr;
+        }
+        // Submit tasks to the two queues, and (optionally) remove the queues.
+        TF_ASSERT_OK(ScheduleTask(1, queue_0.get()));
+        TF_ASSERT_OK(ScheduleTask(2, queue_1.get()));
+        TF_ASSERT_OK(ScheduleTask(3, queue_0.get()));
+        TF_ASSERT_OK(ScheduleTask(4, queue_1.get()));
+        if (delete_queue_1_early) {
+          queue_1 = nullptr;
+        }
+        TF_ASSERT_OK(ScheduleTask(5, queue_0.get()));
+      }
+      EXPECT_EQ(queue_0_tasks, 9);
+      EXPECT_EQ(queue_1_tasks, 6);
+    }
+  }
+}
+
+TEST(AdaptiveSharedBatchSchedulerTest, BadOptions) {
+  using Scheduler = AdaptiveSharedBatchScheduler<FakeTask>;
+  std::shared_ptr<Scheduler> scheduler;
+  Scheduler::Options options;
+  options.num_batch_threads = 0;
+  EXPECT_FALSE(Scheduler::Create(options, &scheduler).ok());
+  options = Scheduler::Options();
+  options.min_scheduling_period_micros = 50;
+  options.max_scheduling_period_micros = 100;
+  options.initial_scheduling_period_micros = 1;
+  EXPECT_FALSE(Scheduler::Create(options, &scheduler).ok());
+  options = Scheduler::Options();
+  options.min_scheduling_period_micros = 50;
+  options.max_scheduling_period_micros = 100;
+  options.initial_scheduling_period_micros = 1000;
+  EXPECT_FALSE(Scheduler::Create(options, &scheduler).ok());
+  options = Scheduler::Options();
+  options.min_scheduling_period_micros = 100;
+  options.max_scheduling_period_micros = 50;
+  options.initial_scheduling_period_micros = 75;
+  EXPECT_FALSE(Scheduler::Create(options, &scheduler).ok());
+  options = Scheduler::Options();
+  options.feedback_smoothing_batches = 0;
+  EXPECT_FALSE(Scheduler::Create(options, &scheduler).ok());
+}
+
+TEST(AdaptiveSharedBatchSchedulerTest, ObeysQueueOptions) {
+  test_util::FakeClockEnv env(Env::Default());
+  Notification start_teardown, stop_teardown;
+  std::unique_ptr<Thread> teardown_thread =
+      CreateFakeClockAdvancerThread(&env, &start_teardown, &stop_teardown);
+  {
+    AdaptiveSharedBatchScheduler<FakeTask>::Options options;
+    options.initial_scheduling_period_micros = 1000;
+    options.env = &env;
+    std::shared_ptr<AdaptiveSharedBatchScheduler<FakeTask>> scheduler;
+    TF_ASSERT_OK(
+        AdaptiveSharedBatchScheduler<FakeTask>::Create(options, &scheduler));
+    std::unique_ptr<BatchScheduler<FakeTask>> queue_0;
+    std::unique_ptr<BatchScheduler<FakeTask>> queue_1;
+    int queue_0_tasks = 0;
+    int queue_1_tasks = 0;
+    auto queue_0_callback = [&queue_0_tasks,
+                             &env](std::unique_ptr<Batch<FakeTask>> batch) {
+      ASSERT_TRUE(batch->IsClosed());
+      EXPECT_GT(batch->num_tasks(), 0);
+      for (int i = 0; i < batch->num_tasks(); i++) {
+        queue_0_tasks += batch->task(i).size();
+      }
+      env.SleepForMicroseconds(1);
+    };
+    auto queue_1_callback = [&queue_1_tasks,
+                             &env](std::unique_ptr<Batch<FakeTask>> batch) {
+      ASSERT_TRUE(batch->IsClosed());
+      EXPECT_GT(batch->num_tasks(), 0);
+      for (int i = 0; i < batch->num_tasks(); i++) {
+        queue_1_tasks += batch->task(i).size();
+      }
+      env.SleepForMicroseconds(1);
+    };
+    AdaptiveSharedBatchScheduler<FakeTask>::QueueOptions queue_options;
+    queue_options.max_batch_size = 10;
+    queue_options.max_enqueued_batches = 0;
+    // Queue must have max_enqueued_batchs > 1.
+    EXPECT_FALSE(
+        scheduler->AddQueue(queue_options, queue_0_callback, &queue_0).ok());
+    queue_options.max_enqueued_batches = 2;
+    TF_ASSERT_OK(
+        scheduler->AddQueue(queue_options, queue_0_callback, &queue_0));
+    queue_options.max_batch_size = 0;
+    // Queue must have max_batch_size > 0.
+    EXPECT_FALSE(
+        scheduler->AddQueue(queue_options, queue_1_callback, &queue_1).ok());
+    queue_options.max_batch_size = 2;
+    queue_options.max_enqueued_batches = 1;
+    TF_ASSERT_OK(
+        scheduler->AddQueue(queue_options, queue_1_callback, &queue_1));
+
+    // Wait for scheduling_thread to sleep.
+    env.BlockUntilThreadsAsleep(1);
+    // Task larger than max_batch_size shouldn't schedule.
+    EXPECT_FALSE(ScheduleTask(15, queue_0.get()).ok());
+    TF_ASSERT_OK(ScheduleTask(5, queue_0.get()));
+    TF_ASSERT_OK(ScheduleTask(5, queue_0.get()));
+    env.AdvanceByMicroseconds(1);
+
+    // Task larger than max_batch_size shouldn't schedule.
+    EXPECT_FALSE(ScheduleTask(3, queue_1.get()).ok());
+    TF_ASSERT_OK(ScheduleTask(1, queue_1.get()));
+    TF_ASSERT_OK(ScheduleTask(1, queue_1.get()));
+    env.AdvanceByMicroseconds(1);
+    // Exceeds max_enqueued_batches, shouldn't schedule.
+    EXPECT_FALSE(ScheduleTask(1, queue_1.get()).ok());
+
+    TF_ASSERT_OK(ScheduleTask(5, queue_0.get()));
+    // Exceeds max_enqueued_batches, shouldn't schedule.
+    EXPECT_FALSE(ScheduleTask(6, queue_0.get()).ok());
+    TF_ASSERT_OK(ScheduleTask(4, queue_0.get()));
+
+    // Batches should be processed in order from oldest to newest.
+    env.AdvanceByMicroseconds(1000);
+    env.BlockUntilThreadsAsleep(2);
+    EXPECT_EQ(queue_0_tasks, 10);
+    EXPECT_EQ(queue_1_tasks, 0);
+
+    env.AdvanceByMicroseconds(1000);
+    env.BlockUntilThreadsAsleep(2);
+    EXPECT_EQ(queue_0_tasks, 10);
+    EXPECT_EQ(queue_1_tasks, 2);
+
+    env.AdvanceByMicroseconds(1000);
+    env.BlockUntilThreadsAsleep(2);
+    EXPECT_EQ(queue_0_tasks, 19);
+    EXPECT_EQ(queue_1_tasks, 2);
+    start_teardown.Notify();
+  }
+  stop_teardown.Notify();
+}
+
+TEST(AdaptiveSharedBatchSchedulerTest, RateFeedback) {
+  test_util::FakeClockEnv env(Env::Default());
+  Notification start_teardown, stop_teardown;
+  std::unique_ptr<Thread> teardown_thread =
+      CreateFakeClockAdvancerThread(&env, &start_teardown, &stop_teardown);
+  {
+    double feedback = 0;
+    AdaptiveSharedBatchScheduler<FakeTask>::Options options;
+    options.initial_scheduling_period_micros = 1000;
+    options.min_scheduling_period_micros = 200;
+    options.max_scheduling_period_micros = 2000;
+    options.env = &env;
+    options.scheduling_period_feedback = [&feedback] { return feedback; };
+    options.feedback_smoothing_batches = 1;
+    std::shared_ptr<AdaptiveSharedBatchScheduler<FakeTask>> scheduler;
+    TF_ASSERT_OK(
+        AdaptiveSharedBatchScheduler<FakeTask>::Create(options, &scheduler));
+    std::unique_ptr<BatchScheduler<FakeTask>> queue;
+    int scheduled_items = 0;
+    auto queue_callback = [&scheduled_items,
+                           &env](std::unique_ptr<Batch<FakeTask>> batch) {
+      ASSERT_TRUE(batch->IsClosed());
+      EXPECT_GT(batch->num_tasks(), 0);
+      scheduled_items = 0;
+      for (int i = 0; i < batch->num_tasks(); i++) {
+        scheduled_items += batch->task(i).size();
+      }
+      env.SleepForMicroseconds(1);
+    };
+
+    TF_ASSERT_OK(scheduler->AddQueue({}, queue_callback, &queue));
+
+    // Wait for scheduling_thread to sleep.
+    env.BlockUntilThreadsAsleep(1);
+    // Enqueue 6 batches.
+    for (int i = 0; i < 6; i++) {
+      TF_ASSERT_OK(ScheduleTask(900 + i, queue.get()));
+      env.AdvanceByMicroseconds(1);
+    }
+    feedback = -500;
+    env.AdvanceByMicroseconds(994);
+    env.BlockUntilThreadsAsleep(2);  // scheduling period = 500 usec.
+    EXPECT_EQ(scheduled_items, 900);
+    env.AdvanceByMicroseconds(500);
+    env.BlockUntilThreadsAsleep(2);  // scheduling period = 250 usec.
+    EXPECT_EQ(scheduled_items, 901);
+    feedback = 0;
+    env.AdvanceByMicroseconds(250);
+    env.BlockUntilThreadsAsleep(2);  // scheduling period = 250 usec.
+    EXPECT_EQ(scheduled_items, 902);
+    feedback = 10000;  // large feedback should hit max_scheduling_period.
+    env.AdvanceByMicroseconds(250);
+    env.BlockUntilThreadsAsleep(2);  // scheduling period = 2000 usec.
+    EXPECT_EQ(scheduled_items, 903);
+    feedback = -10000;  // large feedback should hit min_scheduling_period.
+    env.AdvanceByMicroseconds(1999);
+    // No callback scheduled, only scheduling thread sleeping.
+    env.BlockUntilThreadsAsleep(1);
+    EXPECT_EQ(scheduled_items, 903);
+    env.AdvanceByMicroseconds(1);
+    env.BlockUntilThreadsAsleep(2);  // scheduling period = 200 usec.
+    EXPECT_EQ(scheduled_items, 904);
+    env.AdvanceByMicroseconds(200);
+    env.BlockUntilThreadsAsleep(2);
+    EXPECT_EQ(scheduled_items, 905);
+    start_teardown.Notify();
+  }
+  stop_teardown.Notify();
+}
+
+TEST(AdaptiveSharedBatchSchedulerTest, FeedbackSmoothing) {
+  test_util::FakeClockEnv env(Env::Default());
+  Notification start_teardown, stop_teardown;
+  std::unique_ptr<Thread> teardown_thread =
+      CreateFakeClockAdvancerThread(&env, &start_teardown, &stop_teardown);
+  {
+    double feedback = 0;
+    AdaptiveSharedBatchScheduler<FakeTask>::Options options;
+    options.initial_scheduling_period_micros = 1000;
+    options.env = &env;
+    options.scheduling_period_feedback = [&feedback] { return feedback; };
+    options.feedback_smoothing_batches = 3;
+    std::shared_ptr<AdaptiveSharedBatchScheduler<FakeTask>> scheduler;
+    TF_ASSERT_OK(
+        AdaptiveSharedBatchScheduler<FakeTask>::Create(options, &scheduler));
+    std::unique_ptr<BatchScheduler<FakeTask>> queue;
+    int scheduled_items = 0;
+    auto queue_callback = [&scheduled_items,
+                           &env](std::unique_ptr<Batch<FakeTask>> batch) {
+      ASSERT_TRUE(batch->IsClosed());
+      EXPECT_GT(batch->num_tasks(), 0);
+      scheduled_items = 0;
+      for (int i = 0; i < batch->num_tasks(); i++) {
+        scheduled_items += batch->task(i).size();
+      }
+      env.SleepForMicroseconds(1);
+    };
+
+    TF_ASSERT_OK(scheduler->AddQueue({}, queue_callback, &queue));
+
+    // Wait for scheduling_thread to sleep.
+    env.BlockUntilThreadsAsleep(1);
+    // Enqueue 4 batches.
+    for (int i = 0; i < 4; i++) {
+      TF_ASSERT_OK(ScheduleTask(900 + i, queue.get()));
+      env.AdvanceByMicroseconds(1);
+    }
+    feedback = -300;
+    env.AdvanceByMicroseconds(996);
+    env.BlockUntilThreadsAsleep(2);
+    // ewma_feedback = 100, scheduling_period = 900.
+    EXPECT_EQ(scheduled_items, 900);
+    env.AdvanceByMicroseconds(899);
+    // No callback scheduled, only scheduling thread sleeping.
+    env.BlockUntilThreadsAsleep(1);
+    EXPECT_EQ(scheduled_items, 900);
+    env.AdvanceByMicroseconds(1);
+    env.BlockUntilThreadsAsleep(2);
+    // ewma_feedback = 167, scheduling_period = 750.
+    EXPECT_EQ(scheduled_items, 901);
+    env.AdvanceByMicroseconds(749);
+    // No callback scheduled, only scheduling thread sleeping.
+    env.BlockUntilThreadsAsleep(1);
+    EXPECT_EQ(scheduled_items, 901);
+    feedback = 1000 / 3.;
+    env.AdvanceByMicroseconds(1);
+    env.BlockUntilThreadsAsleep(2);
+    // emwa_feedback = 0, scheduling_period = 750.
+    EXPECT_EQ(scheduled_items, 902);
+    env.AdvanceByMicroseconds(749);
+    // No callback scheduled, only scheduling thread sleeping.
+    env.BlockUntilThreadsAsleep(1);
+    EXPECT_EQ(scheduled_items, 902);
+    env.AdvanceByMicroseconds(1);
+    env.BlockUntilThreadsAsleep(2);
+    EXPECT_EQ(scheduled_items, 903);
+    start_teardown.Notify();
+  }
+  stop_teardown.Notify();
+}
+
+TEST(AdaptiveSharedBatchSchedulerTest, QueueCapacityInfo) {
+  test_util::FakeClockEnv env(Env::Default());
+  Notification start_teardown, stop_teardown;
+  std::unique_ptr<Thread> teardown_thread =
+      CreateFakeClockAdvancerThread(&env, &start_teardown, &stop_teardown);
+  {
+    AdaptiveSharedBatchScheduler<FakeTask>::Options options;
+    options.initial_scheduling_period_micros = 1000;
+    options.env = &env;
+    std::shared_ptr<AdaptiveSharedBatchScheduler<FakeTask>> scheduler;
+    TF_ASSERT_OK(
+        AdaptiveSharedBatchScheduler<FakeTask>::Create(options, &scheduler));
+    std::unique_ptr<BatchScheduler<FakeTask>> queue;
+    int scheduled_items = 0;
+    auto queue_callback = [&scheduled_items,
+                           &env](std::unique_ptr<Batch<FakeTask>> batch) {
+      ASSERT_TRUE(batch->IsClosed());
+      EXPECT_GT(batch->num_tasks(), 0);
+      scheduled_items = 0;
+      for (int i = 0; i < batch->num_tasks(); i++) {
+        scheduled_items += batch->task(i).size();
+      }
+      env.SleepForMicroseconds(1);
+    };
+    AdaptiveSharedBatchScheduler<FakeTask>::QueueOptions queue_options;
+    queue_options.max_batch_size = 10;
+    queue_options.max_enqueued_batches = 10;
+    TF_ASSERT_OK(scheduler->AddQueue(queue_options, queue_callback, &queue));
+
+    // Wait for scheduling_thread to sleep.
+    env.BlockUntilThreadsAsleep(1);
+    // Enqueue 3 tasks.
+    EXPECT_EQ(queue->NumEnqueuedTasks(), 0);
+    EXPECT_EQ(queue->SchedulingCapacity(), 100);
+    TF_ASSERT_OK(ScheduleTask(5, queue.get()));
+    EXPECT_EQ(queue->NumEnqueuedTasks(), 1);
+    EXPECT_EQ(queue->SchedulingCapacity(), 95);
+    env.AdvanceByMicroseconds(1);
+    TF_ASSERT_OK(ScheduleTask(6, queue.get()));
+    EXPECT_EQ(queue->NumEnqueuedTasks(), 2);
+    EXPECT_EQ(queue->SchedulingCapacity(), 84);
+    env.AdvanceByMicroseconds(1);
+    TF_ASSERT_OK(ScheduleTask(1, queue.get()));
+    EXPECT_EQ(queue->NumEnqueuedTasks(), 3);
+    EXPECT_EQ(queue->SchedulingCapacity(), 83);
+
+    env.AdvanceByMicroseconds(998);
+    env.BlockUntilThreadsAsleep(2);
+    EXPECT_EQ(scheduled_items, 5);
+    env.AdvanceByMicroseconds(1000);
+    env.BlockUntilThreadsAsleep(2);
+    EXPECT_EQ(scheduled_items, 7);
+    start_teardown.Notify();
+  }
+  stop_teardown.Notify();
+}
+}  // namespace anonymous
+}  // namespace serving
+}  // namespace tensorflow
diff --git a/tensorflow/contrib/batching/batch_scheduler.h b/tensorflow/contrib/batching/batch_scheduler.h
index 7c41ad8818..a5072f439a 100644
--- a/tensorflow/contrib/batching/batch_scheduler.h
+++ b/tensorflow/contrib/batching/batch_scheduler.h
@@ -78,7 +78,7 @@ template <typename TaskType>
 class Batch {
  public:
   Batch() = default;
-  ~Batch();  // Blocks until the batch is closed.
+  virtual ~Batch();  // Blocks until the batch is closed.
 
   // Appends 'task' to the batch. After calling AddTask(), the newly-added task
   // can be accessed via task(num_tasks()-1) or mutable_task(num_tasks()-1).
-- 
GitLab


From 2ceaad624dd749bd21ce1cad58143acb9366f297 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 23 Oct 2017 13:08:26 -0700
Subject: [PATCH 686/909] Rename add_layer -> track_layer in Network.

PiperOrigin-RevId: 173159032
---
 tensorflow/contrib/eager/python/network.py      | 16 ++++++++--------
 tensorflow/contrib/eager/python/network_test.py |  4 ++--
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/tensorflow/contrib/eager/python/network.py b/tensorflow/contrib/eager/python/network.py
index bebc595df0..8ae5099546 100644
--- a/tensorflow/contrib/eager/python/network.py
+++ b/tensorflow/contrib/eager/python/network.py
@@ -40,7 +40,7 @@ class Network(base.Layer):
     print(d.name)
     print(d.variables)
   - Note that name provided to __init__ is only for error messages?
-  - Detect layers used in __call__ that weren't registered with add_layer.
+  - Detect layers used in __call__ that weren't registered with track_layer.
   - Convert inputs to __call__ to tensors.
   - Prevent variables from being created after the first __call__?
     (Think about restoring from a checkpoint).
@@ -52,10 +52,10 @@ class Network(base.Layer):
     self._container = uuid.uuid4().hex
     self._layers = collections.OrderedDict()
 
-  def add_layer(self, layer):
-    """Add a Layer to this Network.
+  def track_layer(self, layer):
+    """Track a Layer in this Network.
 
-    `Network` requires that all `Layer`s used in `call()` be added so that the
+    `Network` requires that all `Layer`s used in `call()` be tracked so that the
     `Network` can export a complete list of variables.
 
     Args:
@@ -66,14 +66,14 @@ class Network(base.Layer):
 
     Raises:
       RuntimeError: If __init__ has not been called.
-      TypeError: If layer is the wrong type.
-      ValueError: If a layer with the same name has already been added.
+      TypeError: If `layer` is the wrong type.
+      ValueError: If a `Layer` with the same name has already been added.
     """
     if not hasattr(self, "_layers"):
       raise RuntimeError("Need to call Network.__init__ before adding layers")
     if not isinstance(layer, base.Layer):
       raise TypeError(
-          "Network.add_layer() passed type %s, not a tf.layers.Layer" %
+          "Network.track_layer() passed type %s, not a tf.layers.Layer" %
           (type(layer),))
     if layer.name in self._layers:
       if self._layers[layer.name] is layer:
@@ -189,7 +189,7 @@ class Sequential(Network):
     super(Sequential, self).__init__(name=name)
     if layers:
       for l in layers:
-        self.add_layer(l)
+        self.track_layer(l)
 
   def call(self, inputs):
     """Call each Layer in the order they were added."""
diff --git a/tensorflow/contrib/eager/python/network_test.py b/tensorflow/contrib/eager/python/network_test.py
index f0dcae85ee..f43ce3acda 100644
--- a/tensorflow/contrib/eager/python/network_test.py
+++ b/tensorflow/contrib/eager/python/network_test.py
@@ -27,7 +27,7 @@ class MyNetwork(network.Network):
 
   def __init__(self):
     super(MyNetwork, self).__init__(name="abcd")
-    self.l1 = self.add_layer(core.Dense(1, use_bias=False))
+    self.l1 = self.track_layer(core.Dense(1, use_bias=False))
 
   def call(self, x):
     return self.l1(x)
@@ -94,7 +94,7 @@ class SequentialTest(test.TestCase):
 
     # Add a second layer to the network.
     l2 = core.Dense(1, use_bias=False)
-    net.add_layer(l2)
+    net.track_layer(l2)
 
     # Set the second layer's weights so it multiplies by 11
     net(constant_op.constant([[2.0]]))  # Create l2's variables
-- 
GitLab


From 3f96f6f956e0e0a6b960e664db4f1e1f2d9b9967 Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Mon, 23 Oct 2017 13:35:30 -0700
Subject: [PATCH 687/909] Functions in graph mode can call other functions in
 graph mode.

PiperOrigin-RevId: 173163050
---
 tensorflow/python/eager/function.py      |  3 +++
 tensorflow/python/eager/function_test.py | 13 +++++++++++++
 2 files changed, 16 insertions(+)

diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py
index e675ee8988..5afc9d295e 100644
--- a/tensorflow/python/eager/function.py
+++ b/tensorflow/python/eager/function.py
@@ -347,6 +347,9 @@ class _GraphModeFunction(object):
       g = ops.get_default_graph()
       if self._fdef.name not in g._functions:  # pylint: disable=protected-access
         g._add_function(self._fdef)  # pylint: disable=protected-access
+      for f in self._graph._functions.values():  # pylint: disable=protected-access
+        if f.name not in g._functions:  # pylint: disable=protected-access
+          g._add_function(f)  # pylint: disable=protected-access
       signature = self._fdef.definition.signature
       args = list(tensor_inputs) + self._extra_inputs
       op = g.create_op(
diff --git a/tensorflow/python/eager/function_test.py b/tensorflow/python/eager/function_test.py
index 33bedb59f3..3722f9dfa5 100644
--- a/tensorflow/python/eager/function_test.py
+++ b/tensorflow/python/eager/function_test.py
@@ -86,6 +86,19 @@ class FunctionTest(test.TestCase):
       op = call()
       self.assertAllEqual(sess.run(op), 2.0)
 
+  def testGraphModeManyFunctions(self):
+    with context.graph_mode(), self.test_session():
+
+      @function.defun
+      def f(x):
+        return x * x
+
+      @function.defun
+      def g(x):
+        return f(x) + 1
+
+      self.assertAllEqual(g(constant_op.constant(2.0)).eval(), 5.0)
+
   def testTensorConversionWithDefun(self):
 
     @function.defun
-- 
GitLab


From a0ee701f73cc80a56b41c2452006e166e0b835e6 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Mon, 23 Oct 2017 13:51:45 -0700
Subject: [PATCH 688/909] Remove contrib/xla_tf_graph/

It isn't needed anymore.

PiperOrigin-RevId: 173165310
---
 tensorflow/BUILD                              |   1 -
 tensorflow/compiler/xla/BUILD                 |   1 -
 tensorflow/contrib/xla_tf_graph/BUILD         |  67 -----
 tensorflow/contrib/xla_tf_graph/README.md     |   8 -
 .../contrib/xla_tf_graph/xla_tf_graph_util.cc | 247 ------------------
 .../contrib/xla_tf_graph/xla_tf_graph_util.h  |  72 -----
 .../xla_tf_graph/xla_tf_graph_util_test.cc    | 134 ----------
 7 files changed, 530 deletions(-)
 delete mode 100644 tensorflow/contrib/xla_tf_graph/BUILD
 delete mode 100644 tensorflow/contrib/xla_tf_graph/README.md
 delete mode 100644 tensorflow/contrib/xla_tf_graph/xla_tf_graph_util.cc
 delete mode 100644 tensorflow/contrib/xla_tf_graph/xla_tf_graph_util.h
 delete mode 100644 tensorflow/contrib/xla_tf_graph/xla_tf_graph_util_test.cc

diff --git a/tensorflow/BUILD b/tensorflow/BUILD
index d4396bacbf..673e433a8a 100644
--- a/tensorflow/BUILD
+++ b/tensorflow/BUILD
@@ -448,7 +448,6 @@ filegroup(
         "//tensorflow/contrib/training:all_files",
         "//tensorflow/contrib/util:all_files",
         "//tensorflow/contrib/verbs:all_files",
-        "//tensorflow/contrib/xla_tf_graph:all_files",
         "//tensorflow/core:all_files",
         "//tensorflow/core/debug:all_files",
         "//tensorflow/core/distributed_runtime:all_files",
diff --git a/tensorflow/compiler/xla/BUILD b/tensorflow/compiler/xla/BUILD
index e51bbffcd0..0129c51a09 100644
--- a/tensorflow/compiler/xla/BUILD
+++ b/tensorflow/compiler/xla/BUILD
@@ -7,7 +7,6 @@ package_group(
     packages = [
         "//tensorflow/compiler/...",
         "//tensorflow/contrib/tpu/...",
-        "//tensorflow/contrib/xla_tf_graph/...",
     ],
 )
 
diff --git a/tensorflow/contrib/xla_tf_graph/BUILD b/tensorflow/contrib/xla_tf_graph/BUILD
deleted file mode 100644
index 4a3a2de9b5..0000000000
--- a/tensorflow/contrib/xla_tf_graph/BUILD
+++ /dev/null
@@ -1,67 +0,0 @@
-# Description:
-#   contains parts of TensorFlow that are experimental or unstable and which are not supported.
-
-package(
-    default_visibility = ["//visibility:public"],
-)
-
-licenses(["notice"])  # Apache 2.0
-
-exports_files(["LICENSE"])
-
-load("//tensorflow:tensorflow.bzl", "tf_cc_test")
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-)
-
-cc_library(
-    name = "xla_tf_graph_util",
-    srcs = [
-        "xla_tf_graph_util.cc",
-    ],
-    hdrs = [
-        "xla_tf_graph_util.h",
-    ],
-    deps = [
-        "//tensorflow/compiler/tf2xla:xla_compiler",
-        "//tensorflow/compiler/xla:status_macros",
-        "//tensorflow/compiler/xla/client",
-        "//tensorflow/compiler/xla/client:client_library",
-        "//tensorflow/core:core_cpu",
-        "//tensorflow/core:framework",
-        "//tensorflow/core:lib",
-    ],
-)
-
-tf_cc_test(
-    name = "xla_tf_graph_util_test",
-    srcs = ["xla_tf_graph_util_test.cc"],
-    linkstatic = 1,
-    tags = ["nomac"],  # b/63908145
-    deps = [
-        ":xla_tf_graph_util",
-        "//tensorflow/cc:cc_ops",
-        "//tensorflow/cc:function_ops",
-        "//tensorflow/cc:scope",
-        "//tensorflow/compiler/jit:xla_cpu_jit",
-        "//tensorflow/compiler/tf2xla:xla_compiler",
-        "//tensorflow/compiler/xla:shape_util",
-        "//tensorflow/compiler/xla/client:client_library",
-        "//tensorflow/compiler/xla/service:hlo_module_config",
-        "//tensorflow/core:core_cpu_internal",
-        "//tensorflow/core:framework_internal",
-        "//tensorflow/core:ops",
-        "//tensorflow/core:tensorflow",
-        "//tensorflow/core:test",
-        "//tensorflow/core:test_main",
-        "//tensorflow/core/kernels:cwise_op",
-    ],
-)
diff --git a/tensorflow/contrib/xla_tf_graph/README.md b/tensorflow/contrib/xla_tf_graph/README.md
deleted file mode 100644
index a374189e81..0000000000
--- a/tensorflow/contrib/xla_tf_graph/README.md
+++ /dev/null
@@ -1,8 +0,0 @@
-# Xla Tf Graph
-
-## Description
-
-This module contains utilities to treat xla representation as tf graph to support mobile SOC experiments and leverage tf tools.
-
-Maintainers:
-- Satoshi Kataoka (satok@google.com, github.com/satok16)
diff --git a/tensorflow/contrib/xla_tf_graph/xla_tf_graph_util.cc b/tensorflow/contrib/xla_tf_graph/xla_tf_graph_util.cc
deleted file mode 100644
index 302aa6457a..0000000000
--- a/tensorflow/contrib/xla_tf_graph/xla_tf_graph_util.cc
+++ /dev/null
@@ -1,247 +0,0 @@
-/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#include "tensorflow/contrib/xla_tf_graph/xla_tf_graph_util.h"
-
-#include "tensorflow/compiler/tf2xla/xla_op_registry.h"
-#include "tensorflow/compiler/xla/client/client_library.h"
-#include "tensorflow/core/platform/protobuf.h"
-
-namespace tensorflow {
-namespace xla_tf_graph {
-
-namespace {
-
-constexpr const char* const GRAPH_NAME = "xla_tf_graph";
-constexpr const char* const NODE_NAME_PREFIX = "xla";
-
-Status ConvertPrimitiveTypeToDataType(const xla::PrimitiveType p_type,
-                                      DataType* d_type) {
-  switch (p_type) {
-    case xla::PRED:
-      *d_type = DT_BOOL;
-      return Status::OK();
-    case xla::S8:
-      *d_type = DT_INT8;
-      return Status::OK();
-    case xla::S16:
-      *d_type = DT_INT16;
-      return Status::OK();
-    case xla::S32:
-      *d_type = DT_INT32;
-      return Status::OK();
-    case xla::S64:
-      *d_type = DT_INT64;
-      return Status::OK();
-    case xla::U8:
-      *d_type = DT_UINT8;
-      return Status::OK();
-    case xla::U16:
-      *d_type = DT_UINT16;
-      return Status::OK();
-    case xla::F16:
-      *d_type = DT_HALF;
-      return Status::OK();
-    case xla::F32:
-      *d_type = DT_FLOAT;
-      return Status::OK();
-    case xla::F64:
-      *d_type = DT_DOUBLE;
-      return Status::OK();
-    default:
-      return errors::InvalidArgument(
-          "Unsupported PrimitiveType in ConvertPrimitiveTypeToDataType ",
-          xla::PrimitiveType_Name(p_type));
-  }
-}
-
-Status ConvertXlaShapeToTensorShapeType(const xla::Shape& xla_shape,
-                                        std::vector<TensorShape>* tensor_shapes,
-                                        std::vector<DataType>* data_types) {
-  switch (xla_shape.element_type()) {
-    case xla::TUPLE: {
-      for (const xla::Shape& element_shape : xla_shape.tuple_shapes()) {
-        if (element_shape.element_type() == xla::TUPLE) {
-          return errors::InvalidArgument("Nested tuple is not allowed.");
-        }
-        TF_RETURN_IF_ERROR(ConvertXlaShapeToTensorShapeType(
-            element_shape, tensor_shapes, data_types));
-      }
-      return Status::OK();
-    }
-    case xla::PRED:
-    case xla::S8:
-    case xla::S16:
-    case xla::S32:
-    case xla::S64:
-    case xla::U8:
-    case xla::U16:
-    case xla::U32:
-    case xla::U64:
-    case xla::F16:
-    case xla::F32:
-    case xla::F64: {
-      TensorShape shape;
-      DataType type;
-      TF_RETURN_IF_ERROR(
-          ConvertPrimitiveTypeToDataType(xla_shape.element_type(), &type));
-      for (const int64& dim : xla_shape.dimensions()) {
-        shape.AddDim(dim);
-      }
-      tensor_shapes->emplace_back(shape);
-      data_types->emplace_back(type);
-      return Status::OK();
-    }
-    default:
-      return errors::InvalidArgument(
-          "Unsupported PrimitiveType in ConvertXlaShapeToTensorShapeType ",
-          xla::PrimitiveType_Name(xla_shape.element_type()));
-  }
-}
-
-string BuildXlaNodeName(const xla::OperationRequest& operation_request,
-                        const string& xla_op_type, const string& suffix) {
-  const string name = strings::StrCat(
-      NODE_NAME_PREFIX, "/", operation_request.output_handle().handle(), "/",
-      xla_op_type);
-  if (suffix.empty()) {
-    return name;
-  } else {
-    return strings::StrCat(name, "/", suffix);
-  }
-}
-
-string BuildXlaNodeName(const xla::OperationRequest& operation_request,
-                        const string& xla_op_type) {
-  return BuildXlaNodeName(operation_request, xla_op_type, "");
-}
-
-string BuildXlaNodeOp(const protobuf::Message& msg, const string& suffix) {
-  return strings::StrCat(msg.GetDescriptor()->name(), "/", suffix);
-}
-
-string BuildXlaNodeOp(const protobuf::Message& msg) {
-  return BuildXlaNodeOp(msg, "");
-}
-
-Status ConvertOpRequestToXlaNode(const xla::OperationRequest& operation_request,
-                                 XlaNode* xla_node) {
-  const xla::OpRequest& op_request = operation_request.request();
-  switch (op_request.op_case()) {
-    case xla::OpRequest::kBinaryOpRequest: {
-      const xla::BinaryOpRequest& op = op_request.binary_op_request();
-      xla_node->op_type =
-          BuildXlaNodeOp(op, xla::BinaryOperation_Name(op.binop()));
-      xla_node->name = BuildXlaNodeName(operation_request, xla_node->op_type);
-      xla_node->input_ids.emplace_back(std::make_tuple(op.lhs().handle(), 0));
-      xla_node->input_ids.emplace_back(std::make_tuple(op.rhs().handle(), 0));
-      for (const int64& dim : op.broadcast_dimensions()) {
-        xla_node->broadcast_dimensions.emplace_back(dim);
-      }
-      break;
-    }
-    case xla::OpRequest::kParameterRequest: {
-      const xla::ParameterRequest& op = op_request.parameter_request();
-      xla_node->op_type = BuildXlaNodeOp(op, "");
-      xla_node->name =
-          BuildXlaNodeName(operation_request, xla_node->op_type, op.name());
-      break;
-    }
-    case xla::OpRequest::kVariadicOpRequest: {
-      const xla::VariadicOpRequest& op = op_request.variadic_op_request();
-      xla_node->op_type =
-          BuildXlaNodeOp(op, xla::VariadicOperation_Name(op.varop()));
-      xla_node->name = BuildXlaNodeName(operation_request, xla_node->op_type);
-      for (const xla::ComputationDataHandle& handle : op.operands()) {
-        xla_node->input_ids.emplace_back(std::make_tuple(handle.handle(), 0));
-      }
-      break;
-    }
-    case xla::OpRequest::kGetTupleElementRequest: {
-      const xla::GetTupleElementRequest& op =
-          op_request.get_tuple_element_request();
-      xla_node->op_type = BuildXlaNodeOp(op);
-      xla_node->name = BuildXlaNodeName(operation_request, xla_node->op_type);
-      xla_node->input_ids.emplace_back(
-          std::make_tuple(op.operand().handle(), op.index()));
-      break;
-    }
-    default:
-      // TODO(satok): Implement all possible cases.
-      LOG(FATAL) << "Op request: " << op_request.op_case()
-                 << " is not supported yet.";
-      break;
-  }
-
-  CHECK(!xla_node->name.empty());
-  CHECK(!xla_node->op_type.empty());
-
-  TF_RETURN_IF_ERROR(ConvertXlaShapeToTensorShapeType(
-      operation_request.output_shape(), &xla_node->output_shapes,
-      &xla_node->output_data_types));
-  return Status::OK();
-}
-
-void SetupXlaCpuClient(std::unique_ptr<FunctionLibraryDefinition>* flib_def,
-                       std::unique_ptr<XlaCompiler>* compiler) {
-  xla::Client* client = xla::ClientLibrary::LocalClientOrDie();
-  XlaOpRegistry::RegisterCompilationKernels();
-
-  FunctionDefLibrary flib;
-  flib_def->reset(new FunctionLibraryDefinition(OpRegistry::Global(), flib));
-
-  // Setup compiler options
-  XlaCompiler::Options options;
-  DeviceType device_type(DEVICE_CPU_XLA_JIT);
-  options.device_type = &device_type;
-  options.flib_def = flib_def->get();
-  options.client = client;
-  compiler->reset(new XlaCompiler(options));
-}
-
-}  // namespace
-
-xla::StatusOr<std::unique_ptr<xla::SessionModule>>
-ConvertTfGraphToXlaSessionModule(const std::vector<XlaCompiler::Argument>& args,
-                                 std::unique_ptr<Graph> graph) {
-  CHECK(graph);
-
-  std::unique_ptr<FunctionLibraryDefinition> flib_def;
-  std::unique_ptr<XlaCompiler> compiler;
-
-  SetupXlaCpuClient(&flib_def, &compiler);
-
-  // Compile graph and build computation
-  XlaCompiler::CompilationResult result;
-  TF_CHECK_OK(compiler->CompileGraph(XlaCompiler::CompileOptions(), GRAPH_NAME,
-                                     std::move(graph), args, &result));
-
-  return result.computation->Snapshot();
-}
-
-xla::StatusOr<std::unordered_map<int64, XlaNode>>
-ConvertXlaSessionModuleToXlaNodes(const xla::SessionModule& session_module) {
-  std::unordered_map<int64, XlaNode> xla_nodes;
-  for (const auto& operation_request : session_module.entry().requests()) {
-    XlaNode xla_node;
-    TF_RETURN_IF_ERROR(
-        ConvertOpRequestToXlaNode(operation_request.second, &xla_node));
-    xla_nodes.emplace(operation_request.first, xla_node);
-  }
-  return std::move(xla_nodes);
-}
-
-}  // namespace xla_tf_graph
-}  // namespace tensorflow
diff --git a/tensorflow/contrib/xla_tf_graph/xla_tf_graph_util.h b/tensorflow/contrib/xla_tf_graph/xla_tf_graph_util.h
deleted file mode 100644
index e635290851..0000000000
--- a/tensorflow/contrib/xla_tf_graph/xla_tf_graph_util.h
+++ /dev/null
@@ -1,72 +0,0 @@
-/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#ifndef TENSORFLOW_CONTRIB_XLA_TF_GRAPH_XLA_TF_GRAPH_UTIL_H_
-#define TENSORFLOW_CONTRIB_XLA_TF_GRAPH_XLA_TF_GRAPH_UTIL_H_
-
-#include <unordered_map>
-
-#include "tensorflow/compiler/tf2xla/xla_compiler.h"
-#include "tensorflow/compiler/xla/client/client.h"
-#include "tensorflow/compiler/xla/status_macros.h"
-#include "tensorflow/core/framework/function.h"
-#include "tensorflow/core/graph/graph.h"
-#include "tensorflow/core/platform/macros.h"
-
-namespace tensorflow {
-namespace xla_tf_graph {
-
-// A set of utilities to handle xla computation requests.
-// These utilities help developers leverage existing tools to work with
-// xla computations, also provide a way to support TensorFlow ops by
-// implementing xla computations so that they can do experiments on their
-// specialized environments.
-
-// A structure to represent typed attributes of TensorFlow graph node.
-// This structure contains op specific attributes as members so that
-// we can treat them explicitly.
-struct XlaNode {
-  // Unique node name
-  string name;
-  // Op type of xla computation
-  string op_type;
-  // List of pair of unique id and port of input node.
-  // We store this value instead
-  // of node name in order not to wait for all XlaNodes to be constructed.
-  std::vector<std::tuple<int64, int>> input_ids;
-  // Oputput shapes
-  std::vector<TensorShape> output_shapes;
-  // Output data types
-  std::vector<DataType> output_data_types;
-
-  //---------------------------
-  // Op specific attributes
-  // #xla::OpRequest::kBinaryOpRequest
-  std::vector<int64> broadcast_dimensions;
-};
-
-// Convert a tf graph to a xla session module
-xla::StatusOr<std::unique_ptr<xla::SessionModule>>
-ConvertTfGraphToXlaSessionModule(const std::vector<XlaCompiler::Argument>& args,
-                                 std::unique_ptr<Graph> graph);
-
-// Convert a xla session module to a map to XlaNode from unique id
-xla::StatusOr<std::unordered_map<int64, XlaNode>>
-ConvertXlaSessionModuleToXlaNodes(const xla::SessionModule& session_module);
-
-}  // namespace xla_tf_graph
-}  // namespace tensorflow
-
-#endif  // TENSORFLOW_CONTRIB_XLA_TF_GRAPH_XLA_TF_GRAPH_UTIL_H_
diff --git a/tensorflow/contrib/xla_tf_graph/xla_tf_graph_util_test.cc b/tensorflow/contrib/xla_tf_graph/xla_tf_graph_util_test.cc
deleted file mode 100644
index 144269303e..0000000000
--- a/tensorflow/contrib/xla_tf_graph/xla_tf_graph_util_test.cc
+++ /dev/null
@@ -1,134 +0,0 @@
-/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#include "tensorflow/contrib/xla_tf_graph/xla_tf_graph_util.h"
-#include "tensorflow/cc/framework/scope.h"
-#include "tensorflow/cc/ops/function_ops.h"
-#include "tensorflow/cc/ops/standard_ops.h"
-#include "tensorflow/compiler/tf2xla/xla_op_registry.h"
-#include "tensorflow/compiler/xla/client/client_library.h"
-#include "tensorflow/compiler/xla/service/hlo_module_config.h"
-#include "tensorflow/compiler/xla/shape_util.h"
-#include "tensorflow/core/platform/test.h"
-
-namespace tensorflow {
-namespace xla_tf_graph {
-
-static std::unique_ptr<Graph> BuildAddGraph() {
-  Scope scope = Scope::NewRootScope().ExitOnError();
-  auto a = ops::_Arg(scope.WithOpName("A"), DT_INT32, 0);
-  auto b = ops::_Arg(scope.WithOpName("B"), DT_INT32, 1);
-  // See tf2xla/kernels/binary_ops.cc
-  auto c = ops::Add(scope.WithOpName("C"), a, b);
-  auto d = ops::_Retval(scope.WithOpName("D"), c, 0);
-  std::unique_ptr<Graph> graph(new Graph(OpRegistry::Global()));
-  TF_CHECK_OK(scope.ToGraph(graph.get()));
-  return graph;
-}
-
-static std::vector<XlaCompiler::Argument> BuildAddGraphArguments() {
-  // Builds a description of the arguments.
-  std::vector<XlaCompiler::Argument> args(2);
-  args[0].kind = XlaCompiler::Argument::kParameter;
-  args[0].type = DT_INT32;
-  // Difference of dimension will add extra broadcast_dimensions.
-  // broadcast_dimension generates an additional HloInstruction
-  // in user_computation.cc
-  args[0].shape = xla::ShapeUtil::MakeShape(xla::S32, {2, 2});
-  args[1].kind = XlaCompiler::Argument::kParameter;
-  args[1].type = DT_INT32;
-  args[1].shape = xla::ShapeUtil::MakeShape(xla::S32, {2});
-  return args;
-}
-
-// CAVEAT: Debug purpose only.
-// This function dumps a protobuf string format of HloModule.
-static void DumpHloGraphForDebug(const std::vector<XlaCompiler::Argument>& args,
-                                 std::unique_ptr<Graph> graph) {
-  std::unique_ptr<FunctionLibraryDefinition> flib_def;
-  std::unique_ptr<FunctionLibraryRuntime> flr;
-  std::unique_ptr<XlaCompiler> compiler;
-
-  xla::Client* client = xla::ClientLibrary::LocalClientOrDie();
-  XlaOpRegistry::RegisterCompilationKernels();
-
-  FunctionDefLibrary flib;
-  flib_def.reset(new FunctionLibraryDefinition(OpRegistry::Global(), flib));
-
-  // Compiles the graph.
-  XlaCompiler::Options options;
-  DeviceType device_type("XLA_CPU_JIT");
-  options.device_type = &device_type;
-  options.client = client;
-  options.flib_def = flib_def.get();
-  compiler.reset(new XlaCompiler(options));
-
-  // Compile graph
-  XlaCompiler::CompilationResult result;
-  TF_CHECK_OK(compiler->CompileGraph(XlaCompiler::CompileOptions(), "dump",
-                                     std::move(graph), args, &result));
-
-  // Convert to hlo
-  xla::Computation& computation = *result.computation;
-
-  xla::Service* service(
-      static_cast<xla::Service*>(xla::ClientLibrary::GetXlaService(
-          static_cast<xla::LocalClient*>(client)->platform())));
-  const xla::ComputationTracker& computation_tracker =
-      service->computation_tracker();
-
-  auto user_computation_status =
-      computation_tracker.Resolve(computation.handle());
-  TF_CHECK_OK(user_computation_status.status());
-  auto user_computation = user_computation_status.ConsumeValueOrDie();
-  xla::VersionedComputationHandle versioned_handle =
-      user_computation->GetVersionedHandle();
-  std::unique_ptr<xla::HloModule> hlo_module =
-      std::move(computation_tracker
-                    .BuildHloModule(versioned_handle, xla::HloModuleConfig())
-                    .ValueOrDie());
-  VLOG(1) << "--- DUMP HLO ---";
-  VLOG(1) << hlo_module->ToString();
-}
-
-TEST(XlaTfGraphUtil, ConvertTfGraphToSessionModule) {
-  // Builds a description of the arguments.
-  std::vector<XlaCompiler::Argument> args = BuildAddGraphArguments();
-  std::unique_ptr<Graph> graph = BuildAddGraph();
-
-  TF_ASSERT_OK_AND_ASSIGN(
-      std::unique_ptr<xla::SessionModule> session_module,
-      ConvertTfGraphToXlaSessionModule(args, std::move(graph)));
-
-  ASSERT_EQ(4, session_module->entry().requests_size());
-
-  VLOG(1) << "--- DUMP ---";
-  VLOG(1) << session_module->DebugString();
-  DumpHloGraphForDebug(args, BuildAddGraph());
-}
-
-TEST(XlaTfGraphUtil, ConvertXlaSessionModuleToXlaNodes) {
-  std::vector<XlaCompiler::Argument> args = BuildAddGraphArguments();
-  std::unique_ptr<Graph> graph = BuildAddGraph();
-  TF_ASSERT_OK_AND_ASSIGN(
-      std::unique_ptr<xla::SessionModule> session_module,
-      ConvertTfGraphToXlaSessionModule(args, std::move(graph)));
-  TF_ASSERT_OK_AND_ASSIGN(auto xla_nodes,
-                          ConvertXlaSessionModuleToXlaNodes(*session_module));
-  EXPECT_EQ(session_module->entry().requests_size(), xla_nodes.size());
-}
-
-}  // namespace xla_tf_graph
-}  // namespace tensorflow
-- 
GitLab


From 3f30e6424fa3b8e890f9360d1661e61c2d1625a5 Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Mon, 23 Oct 2017 14:26:02 -0700
Subject: [PATCH 689/909] Makes gradients_function exception-safe.

PiperOrigin-RevId: 173170394
---
 tensorflow/python/eager/backprop.py      | 57 +++++++++++-------------
 tensorflow/python/eager/backprop_test.py | 15 +++++++
 2 files changed, 40 insertions(+), 32 deletions(-)

diff --git a/tensorflow/python/eager/backprop.py b/tensorflow/python/eager/backprop.py
index 9580e84847..9d86ac77f8 100644
--- a/tensorflow/python/eager/backprop.py
+++ b/tensorflow/python/eager/backprop.py
@@ -335,15 +335,18 @@ def implicit_val_and_grad(f):
   def grad_fn(*args):
     """Computes the gradient of the wrapped function."""
     tape.push_new_tape()
-    end_node = f(*args)
-    variables = tape.top_tape_watched_variables()
+    try:
+      end_node = f(*args)
+      variables = tape.top_tape_watched_variables()
+    finally:
+      popped_tape = tape.pop_tape()
     sources = [x.handle for x in variables]
 
     if not sources:
       raise ValueError("no trainable variables were accessed while the "
                        "function was being computed.")
     grad = imperative_grad.imperative_grad(_default_vspace,
-                                           tape.pop_tape(),
+                                           popped_tape,
                                            nest.flatten(end_node),
                                            sources)
     return end_node, list(zip(grad, variables))
@@ -561,25 +564,12 @@ def val_and_grad_function(f, params=None):
 
   def decorated(*args, **kwds):
     """Computes the value and gradient of the decorated function."""
-    parameter_positions = _get_arg_spec(f, params, args)
     dy = kwds.pop("dy", None)
-    if dy is not None:
-      dy = ops.convert_to_tensor(dy)
-    assert not kwds, "The gradient function can't take keyword arguments."
-    tape.push_new_tape()
-    sources = []
-    args = [
-        ops.convert_to_tensor(args[i]) if i in parameter_positions else args[i]
-        for i in range(len(args))
-    ]
-    args = _ensure_unique_tensor_objects(parameter_positions, args)
-    for i in parameter_positions:
-      sources.append(args[i])
-      tape.watch(args[i])
-    result = f(*args)
-    return result, imperative_grad.imperative_grad(
-        _default_vspace, tape.pop_tape(), nest.flatten(result), sources,
-        output_gradients=nest.flatten(dy) if dy is not None else None)
+    if kwds:
+      raise ValueError("Functions to be differentiated cannot "
+                       "receive keyword arguments.")
+    val, vjp = make_vjp(f, params)(*args, **kwds)
+    return val, vjp(dy=dy)
 
   return decorated
 
@@ -619,17 +609,20 @@ def make_vjp(f, params=None):
     parameter_positions = _get_arg_spec(f, params, args)
     assert not kwds, "The gradient function can't take keyword arguments."
     tape.push_new_tape()
-    sources = []
-    args = [
-        ops.convert_to_tensor(args[i]) if i in parameter_positions else args[i]
-        for i in range(len(args))
-    ]
-    args = _ensure_unique_tensor_objects(parameter_positions, args)
-    for i in parameter_positions:
-      sources.append(args[i])
-      tape.watch(args[i])
-    result = f(*args)
-    t = tape.pop_tape()
+    try:
+      sources = []
+      args = [
+          ops.convert_to_tensor(args[i])
+          if i in parameter_positions else args[i]
+          for i in range(len(args))
+      ]
+      args = _ensure_unique_tensor_objects(parameter_positions, args)
+      for i in parameter_positions:
+        sources.append(args[i])
+        tape.watch(args[i])
+        result = f(*args)
+    finally:
+      t = tape.pop_tape()
     def vjp(dy=None):
       return imperative_grad.imperative_grad(
           _default_vspace, t, nest.flatten(result), sources,
diff --git a/tensorflow/python/eager/backprop_test.py b/tensorflow/python/eager/backprop_test.py
index 9ba5913c65..628f254b18 100644
--- a/tensorflow/python/eager/backprop_test.py
+++ b/tensorflow/python/eager/backprop_test.py
@@ -389,6 +389,21 @@ class BackpropTest(test.TestCase):
     grad = backprop.gradients_function(f)
     self.assertAllEqual(grad(1.0)[0], 2.0)
 
+  def testExceptionSafety(self):
+
+    def f(unused_x):
+      raise ValueError()
+
+    try:
+      backprop.gradients_function(f)(1.0)
+    except ValueError:
+      pass
+
+    def real_f(x):
+      return x * x
+
+    self.assertAllEqual(backprop.gradients_function(real_f)(1.0)[0], 2.0)
+
   def testMultiValueConvertToTensor(self):
     x = resource_variable_ops.ResourceVariable(
         initial_value=array_ops.constant([1.0]), name='x')
-- 
GitLab


From 15022cb1564d3cadaa9b676f3e93c6f16a3a298e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 23 Oct 2017 14:35:53 -0700
Subject: [PATCH 690/909] Remove name_scope from convolutional calls.

PiperOrigin-RevId: 173171871
---
 .../quantize/python/fold_batch_norms_test.py  | 12 +++++-----
 .../python/quantize_parameterized_test.py     |  4 ++--
 tensorflow/python/layers/convolutional.py     | 22 ++++++++-----------
 3 files changed, 17 insertions(+), 21 deletions(-)

diff --git a/tensorflow/contrib/quantize/python/fold_batch_norms_test.py b/tensorflow/contrib/quantize/python/fold_batch_norms_test.py
index 5a66b38b15..2cecf68514 100644
--- a/tensorflow/contrib/quantize/python/fold_batch_norms_test.py
+++ b/tensorflow/contrib/quantize/python/fold_batch_norms_test.py
@@ -101,9 +101,9 @@ class FoldBatchNormsTest(test_util.TensorFlowTestCase):
         scope + '/weights/read',
         self._BatchNormMultiplierName(scope, has_scaling, fused_batch_norm)
     ])
-    self._AssertOutputGoesToOps(folded_mul, g, [scope + '/convolution_Fold'])
+    self._AssertOutputGoesToOps(folded_mul, g, [scope + '/Conv2D_Fold'])
 
-    folded_conv = g.get_operation_by_name(scope + '/convolution_Fold')
+    folded_conv = g.get_operation_by_name(scope + '/Conv2D_Fold')
     self.assertEqual(folded_conv.type, 'Conv2D')
     self._AssertInputOpsAre(folded_conv,
                             [scope + '/mul_fold', inputs.op.name])
@@ -112,7 +112,7 @@ class FoldBatchNormsTest(test_util.TensorFlowTestCase):
     folded_add = g.get_operation_by_name(scope + '/add_fold')
     self.assertEqual(folded_add.type, 'Add')
     self._AssertInputOpsAre(folded_add, [
-        scope + '/convolution_Fold',
+        scope + '/Conv2D_Fold',
         self._BathNormBiasName(scope, fused_batch_norm)
     ])
     output_op_names = ['test/Add' if with_bypass else 'test/' + relu_op_name]
@@ -166,9 +166,9 @@ class FoldBatchNormsTest(test_util.TensorFlowTestCase):
         scope + '/weights/read',
         self._BatchNormMultiplierName(scope, has_scaling, fused_batch_norm)
     ])
-    self._AssertOutputGoesToOps(folded_mul, g, [scope + '/convolution_Fold'])
+    self._AssertOutputGoesToOps(folded_mul, g, [scope + '/Conv2D_Fold'])
 
-    folded_conv = g.get_operation_by_name(scope + '/convolution_Fold')
+    folded_conv = g.get_operation_by_name(scope + '/Conv2D_Fold')
     self.assertEqual(folded_conv.type, 'Conv2D')
     self._AssertInputOpsAre(folded_conv, [scope + '/mul_fold', inputs.op.name])
     self._AssertOutputGoesToOps(folded_conv, g, [scope + '/add_fold'])
@@ -176,7 +176,7 @@ class FoldBatchNormsTest(test_util.TensorFlowTestCase):
     folded_add = g.get_operation_by_name(scope + '/add_fold')
     self.assertEqual(folded_add.type, 'Add')
     self._AssertInputOpsAre(folded_add, [
-        scope + '/convolution_Fold',
+        scope + '/Conv2D_Fold',
         self._BathNormBiasName(scope, fused_batch_norm)
     ])
     output_op_names = ['test/Add' if with_bypass else 'test/' + relu_op_name]
diff --git a/tensorflow/contrib/quantize/python/quantize_parameterized_test.py b/tensorflow/contrib/quantize/python/quantize_parameterized_test.py
index 31fcd66dfb..3e62f95bd6 100644
--- a/tensorflow/contrib/quantize/python/quantize_parameterized_test.py
+++ b/tensorflow/contrib/quantize/python/quantize_parameterized_test.py
@@ -101,7 +101,7 @@ class QuantizeTest(test_util.TensorFlowTestCase):
         scope + '/weights/read'
     ]
     self._AssertInputOpsAre(weights_quant, expected_inputs)
-    output_op_name = scope + '/convolution'
+    output_op_name = scope + '/Conv2D'
     self._AssertOutputGoesToOps(weights_quant, graph, [output_op_name])
 
     if with_bypass:
@@ -407,7 +407,7 @@ class QuantizeTest(test_util.TensorFlowTestCase):
     ]
     self._AssertInputOpsAre(weights_quant, expected_inputs)
     output_op_name = scope + ('/weights_quant/delayed_quant/Switch_1'
-                              if (delay and use_ema) else '/convolution_Fold')
+                              if (delay and use_ema) else '/Conv2D_Fold')
     self._AssertOutputGoesToOps(weights_quant, graph, [output_op_name])
 
     if with_bypass:
diff --git a/tensorflow/python/layers/convolutional.py b/tensorflow/python/layers/convolutional.py
index 9850cd33b0..c983d3803b 100644
--- a/tensorflow/python/layers/convolutional.py
+++ b/tensorflow/python/layers/convolutional.py
@@ -153,22 +153,18 @@ class _Conv(base.Layer):
       self.bias = None
     self.input_spec = base.InputSpec(ndim=self.rank + 2,
                                      axes={channel_axis: input_dim})
-    with ops.name_scope(None, 'convolution', [self.kernel]) as name:
-      self._convolution_op = nn_ops.Convolution(
-          input_shape,
-          filter_shape=self.kernel.get_shape(),
-          dilation_rate=self.dilation_rate,
-          strides=self.strides,
-          padding=self.padding.upper(),
-          data_format=utils.convert_data_format(self.data_format,
-                                                self.rank + 2),
-          name=name)
+    self._convolution_op = nn_ops.Convolution(
+        input_shape,
+        filter_shape=self.kernel.get_shape(),
+        dilation_rate=self.dilation_rate,
+        strides=self.strides,
+        padding=self.padding.upper(),
+        data_format=utils.convert_data_format(self.data_format,
+                                              self.rank + 2))
     self.built = True
 
   def call(self, inputs):
-    # TODO(agarwal): do we need this name_scope ?
-    with ops.name_scope(None, 'convolution', [inputs, self.kernel]):
-      outputs = self._convolution_op(inputs, self.kernel)
+    outputs = self._convolution_op(inputs, self.kernel)
 
     if self.use_bias:
       if self.data_format == 'channels_first':
-- 
GitLab


From c319703d3669c9eec51f17ffc5e7e586b9608074 Mon Sep 17 00:00:00 2001
From: Asim Shankar <ashankar@google.com>
Date: Mon, 23 Oct 2017 14:36:55 -0700
Subject: [PATCH 691/909] Java: Update release to 1.4.0-rc1

PiperOrigin-RevId: 173172018
---
 tensorflow/java/maven/libtensorflow/pom.xml     | 2 +-
 tensorflow/java/maven/libtensorflow_jni/pom.xml | 2 +-
 tensorflow/java/maven/pom.xml                   | 2 +-
 tensorflow/java/maven/proto/pom.xml             | 2 +-
 tensorflow/java/maven/tensorflow/pom.xml        | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/tensorflow/java/maven/libtensorflow/pom.xml b/tensorflow/java/maven/libtensorflow/pom.xml
index 6cc1102930..3714570876 100644
--- a/tensorflow/java/maven/libtensorflow/pom.xml
+++ b/tensorflow/java/maven/libtensorflow/pom.xml
@@ -6,7 +6,7 @@
   <parent>
     <groupId>org.tensorflow</groupId>
     <artifactId>parentpom</artifactId>
-    <version>1.3.0</version>
+    <version>1.4.0-rc1</version>
     <relativePath>../</relativePath>
   </parent>
   <artifactId>libtensorflow</artifactId>
diff --git a/tensorflow/java/maven/libtensorflow_jni/pom.xml b/tensorflow/java/maven/libtensorflow_jni/pom.xml
index 0b22844898..9f7eb40253 100644
--- a/tensorflow/java/maven/libtensorflow_jni/pom.xml
+++ b/tensorflow/java/maven/libtensorflow_jni/pom.xml
@@ -6,7 +6,7 @@
   <parent>
     <groupId>org.tensorflow</groupId>
     <artifactId>parentpom</artifactId>
-    <version>1.3.0</version>
+    <version>1.4.0-rc1</version>
     <relativePath>../</relativePath>
   </parent>
   <artifactId>libtensorflow_jni</artifactId>
diff --git a/tensorflow/java/maven/pom.xml b/tensorflow/java/maven/pom.xml
index 0a3552d756..fac0a8bc26 100644
--- a/tensorflow/java/maven/pom.xml
+++ b/tensorflow/java/maven/pom.xml
@@ -6,7 +6,7 @@
   <modelVersion>4.0.0</modelVersion>
   <groupId>org.tensorflow</groupId>
   <artifactId>parentpom</artifactId>
-  <version>1.3.0</version>
+  <version>1.4.0-rc1</version>
   <packaging>pom</packaging>
 
   <url>https://www.tensorflow.org</url>
diff --git a/tensorflow/java/maven/proto/pom.xml b/tensorflow/java/maven/proto/pom.xml
index b76b28aa15..135ee0f2d2 100644
--- a/tensorflow/java/maven/proto/pom.xml
+++ b/tensorflow/java/maven/proto/pom.xml
@@ -6,7 +6,7 @@
   <parent>
     <groupId>org.tensorflow</groupId>
     <artifactId>parentpom</artifactId>
-    <version>1.3.0</version>
+    <version>1.4.0-rc1</version>
     <relativePath>../</relativePath>
   </parent>
   <artifactId>proto</artifactId>
diff --git a/tensorflow/java/maven/tensorflow/pom.xml b/tensorflow/java/maven/tensorflow/pom.xml
index c2af55f5ce..771482ba64 100644
--- a/tensorflow/java/maven/tensorflow/pom.xml
+++ b/tensorflow/java/maven/tensorflow/pom.xml
@@ -6,7 +6,7 @@
   <parent>
     <groupId>org.tensorflow</groupId>
     <artifactId>parentpom</artifactId>
-    <version>1.3.0</version>
+    <version>1.4.0-rc1</version>
     <relativePath>../</relativePath>
   </parent>
   <artifactId>tensorflow</artifactId>
-- 
GitLab


From 4cd64cac16ccd22a2d956d6957ecfda0ff67ee89 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 23 Oct 2017 14:40:39 -0700
Subject: [PATCH 692/909] Make Optimizer.minimize work when eager execution is
 enabled.

PiperOrigin-RevId: 173172604
---
 tensorflow/python/BUILD                      |   1 +
 tensorflow/python/eager/backprop.py          |   2 +-
 tensorflow/python/eager/function_test.py     |   2 +-
 tensorflow/python/framework/test_util.py     |   4 +-
 tensorflow/python/training/optimizer.py      |  49 ++++-
 tensorflow/python/training/optimizer_test.py | 211 +++++++++++--------
 6 files changed, 174 insertions(+), 95 deletions(-)

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index b7aa7bbf6b..4382eeb9a8 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -2658,6 +2658,7 @@ py_library(
         ":util",
         ":variable_scope",
         ":variables",
+        "//tensorflow/python/eager:backprop",
         "//third_party/py/numpy",
         "@six_archive//:six",
     ],
diff --git a/tensorflow/python/eager/backprop.py b/tensorflow/python/eager/backprop.py
index 9d86ac77f8..bdc4ce3252 100644
--- a/tensorflow/python/eager/backprop.py
+++ b/tensorflow/python/eager/backprop.py
@@ -343,7 +343,7 @@ def implicit_val_and_grad(f):
     sources = [x.handle for x in variables]
 
     if not sources:
-      raise ValueError("no trainable variables were accessed while the "
+      raise ValueError("No trainable variables were accessed while the "
                        "function was being computed.")
     grad = imperative_grad.imperative_grad(_default_vspace,
                                            popped_tape,
diff --git a/tensorflow/python/eager/function_test.py b/tensorflow/python/eager/function_test.py
index 3722f9dfa5..b4b704401a 100644
--- a/tensorflow/python/eager/function_test.py
+++ b/tensorflow/python/eager/function_test.py
@@ -138,7 +138,7 @@ class FunctionTest(test.TestCase):
             'v', initializer=constant_op.constant(1.0))
         return x * constant_op.constant(2.0)
       with self.assertRaisesRegexp(ValueError,
-                                   'no trainable variables were accessed'):
+                                   'No trainable variables were accessed'):
         backprop.implicit_val_and_grad(f)()
 
   def testDefunCallBackpropUsingSameObjectForMultipleArguments(self):
diff --git a/tensorflow/python/framework/test_util.py b/tensorflow/python/framework/test_util.py
index a01bf02deb..e545f6de8e 100644
--- a/tensorflow/python/framework/test_util.py
+++ b/tensorflow/python/framework/test_util.py
@@ -683,8 +683,10 @@ class TensorFlowTestCase(googletest.TestCase):
     elif isinstance(tensors, dict):
       assert not tensors, "Only support empty dict now."
       return dict()
+    elif tensors is None:
+      return None
     else:
-      raise ValueError("Unsupported type.")
+      raise ValueError("Unsupported type %s." % type(tensors))
 
   def evaluate(self, tensors):
     """Evaluates tensors and returns numpy values.
diff --git a/tensorflow/python/training/optimizer.py b/tensorflow/python/training/optimizer.py
index 86ba8e2c8e..82fc4edbcd 100644
--- a/tensorflow/python/training/optimizer.py
+++ b/tensorflow/python/training/optimizer.py
@@ -22,6 +22,7 @@ from __future__ import print_function
 
 import abc
 
+from tensorflow.python.eager import backprop
 from tensorflow.python.eager import context
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
@@ -335,6 +336,16 @@ class Optimizer(object):
 
     Raises:
       ValueError: If some of the variables are not `Variable` objects.
+
+      @compatibility(eager):
+      When eager execution is enabled, `loss` should be a Python function that
+      takes elements of `var_list` as arguments and computes the value to be
+      minimized. If `var_list` is None, `loss` should take no arguments.
+      Minimization (and gradient computation) is done with respect to the
+      elements of `var_list` if not None, else with respect to any trainable
+      variables created during the execution of the `loss` function.
+      `gate_gradients`, `aggregation_method`, `colocate_gradients_with_ops` and
+      `grad_loss` are ignored when eager execution is enabled.
     """
     grads_and_vars = self.compute_gradients(
         loss, var_list=var_list, gate_gradients=gate_gradients,
@@ -385,7 +396,32 @@ class Optimizer(object):
     Raises:
       TypeError: If `var_list` contains anything else than `Variable` objects.
       ValueError: If some arguments are invalid.
+
+      @compatibility(eager):
+      When eager execution is enabled, `loss` should be a Python function that
+      takes elements of `var_list` as arguments and computes the value to be
+      minimized. If `var_list` is None, `loss` should take no arguments.
+      Gradient computation is done with respect to the elements of `var_list` if
+      not None, else with respect to any trainable variables created during the
+      execution of the `loss` function.
+      `gate_gradients`, `aggregation_method`, `colocate_gradients_with_ops` and
+      `grad_loss` are ignored when eager execution is enabled.
     """
+    if context.in_eager_mode():
+      if grad_loss is not None:
+        raise ValueError("`grad_loss` argument to Optimizer.compute_gradients "
+                         "not supported when eager execution is enabled.")
+      if not callable(loss):
+        raise ValueError("`loss` passed to Optimizer.compute_gradients should "
+                         "be a function when eager execution is enabled.")
+      # TODO(agarwal): consider passing parameters to the `loss` function.
+      if var_list is None:
+        return backprop.implicit_grad(loss)()
+      else:
+        var_list = nest.flatten(var_list)
+        grads = backprop.gradients_function(loss)(*var_list)
+        grads_and_vars = list(zip(grads, var_list))
+        return grads_and_vars
     if gate_gradients not in [Optimizer.GATE_NONE, Optimizer.GATE_OP,
                               Optimizer.GATE_GRAPH]:
       raise ValueError("gate_gradients must be one of: Optimizer.GATE_NONE, "
@@ -489,11 +525,14 @@ class Optimizer(object):
       else:
         with ops.control_dependencies([self._finish(update_ops, "update")]):
           with ops.colocate_with(global_step):
-            apply_updates = state_ops.assign_add(global_step, 1, name=name).op
-
-      train_op = ops.get_collection_ref(ops.GraphKeys.TRAIN_OP)
-      if apply_updates not in train_op:
-        train_op.append(apply_updates)
+            apply_updates = state_ops.assign_add(global_step, 1, name=name)
+
+      if context.in_graph_mode():
+        if isinstance(apply_updates, ops.Tensor):
+          apply_updates = apply_updates.op
+        train_op = ops.get_collection_ref(ops.GraphKeys.TRAIN_OP)
+        if apply_updates not in train_op:
+          train_op.append(apply_updates)
 
       return apply_updates
 
diff --git a/tensorflow/python/training/optimizer_test.py b/tensorflow/python/training/optimizer_test.py
index c7eb9bc412..6bdae39073 100644
--- a/tensorflow/python/training/optimizer_test.py
+++ b/tensorflow/python/training/optimizer_test.py
@@ -18,12 +18,15 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from tensorflow.python.eager import context
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import clip_ops
 from tensorflow.python.ops import gradients_impl
+from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.ops import state_ops
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import test
@@ -32,26 +35,34 @@ from tensorflow.python.training import gradient_descent
 
 class OptimizerTest(test.TestCase):
 
+  @test_util.run_in_graph_and_eager_modes()
   def testBasic(self):
-    for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
-      with self.test_session():
-        var0 = variables.Variable([1.0, 2.0], dtype=dtype)
-        var1 = variables.Variable([3.0, 4.0], dtype=dtype)
-        cost = 5 * var0 + 3 * var1
-        global_step = variables.Variable(
-            array_ops.zeros([], dtypes.int64), name='global_step')
-        sgd_op = gradient_descent.GradientDescentOptimizer(3.0)
-        opt_op = sgd_op.minimize(cost, global_step, [var0, var1])
+    for i, dtype in enumerate([dtypes.half, dtypes.float32, dtypes.float64]):
+      # Note that we name the variables uniquely here since the variables don't
+      # seem to be getting deleted at the end of the loop.
+      var0 = resource_variable_ops.ResourceVariable([1.0, 2.0], dtype=dtype,
+                                                    name='a_%d' % i)
+      var1 = resource_variable_ops.ResourceVariable([3.0, 4.0], dtype=dtype,
+                                                    name='b_%d' % i)
+      def loss(v0, v1):
+        return 5 * v0 + 3 * v1
+      # Note that for eager execution, minimize expects a function instead of a
+      # Tensor.
+      cost = loss if context.in_eager_mode() else loss(var0, var1)
+      global_step = resource_variable_ops.ResourceVariable(
+          array_ops.zeros([], dtypes.int64), name='global_step_%d' % i)
+      sgd_op = gradient_descent.GradientDescentOptimizer(3.0)
 
-        variables.global_variables_initializer().run()
-        # Fetch params to validate initial values
-        self.assertAllClose([1.0, 2.0], var0.eval())
-        self.assertAllClose([3.0, 4.0], var1.eval())
-        # Run 1 step of sgd through optimizer
-        opt_op.run()
-        # Validate updated params
-        self.assertAllClose([-14., -13.], var0.eval())
-        self.assertAllClose([-6., -5.], var1.eval())
+      self.evaluate(variables.global_variables_initializer())
+      # Fetch params to validate initial values
+      self.assertAllClose([1.0, 2.0], self.evaluate(var0))
+      self.assertAllClose([3.0, 4.0], self.evaluate(var1))
+      # Run 1 step of sgd through optimizer
+      opt_op = sgd_op.minimize(cost, global_step, [var0, var1])
+      self.evaluate(opt_op)
+      # Validate updated params
+      self.assertAllClose([-14., -13.], self.evaluate(var0))
+      self.assertAllClose([-6., -5.], self.evaluate(var1))
 
   def testAggregationMethod(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
@@ -103,86 +114,112 @@ class OptimizerTest(test.TestCase):
         self.assertAllClose([3.0 - 3 * 3 * 42.0, 4.0 - 3 * 3 * (-42.0)],
                             var1.eval())
 
+  @test_util.run_in_graph_and_eager_modes()
   def testNoVariables(self):
     for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
-      with self.test_session():
-        var0 = variables.Variable([1.0, 2.0], dtype=dtype, trainable=False)
-        var1 = variables.Variable([3.0, 4.0], dtype=dtype, trainable=False)
-        cost = 5 * var0 + var1
-        sgd_op = gradient_descent.GradientDescentOptimizer(3.0)
-        with self.assertRaisesRegexp(ValueError, 'No variables'):
-          sgd_op.minimize(cost)
+      # pylint: disable=cell-var-from-loop
+      def loss():
+        var0 = resource_variable_ops.ResourceVariable(
+            [1.0, 2.0], dtype=dtype, trainable=False, name='a')
+        var1 = resource_variable_ops.ResourceVariable(
+            [3.0, 4.0], dtype=dtype, trainable=False, name='b')
+        return 5 * var0 + var1
+      # pylint: enable=cell-var-from-loop
+      cost = loss if context.in_eager_mode() else loss()
+      sgd_op = gradient_descent.GradientDescentOptimizer(3.0)
+      with self.assertRaisesRegexp(ValueError, 'No.*variables'):
+        sgd_op.minimize(cost)
 
+  @test_util.run_in_graph_and_eager_modes()
   def testNoGradients(self):
-    for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
-      with self.test_session():
-        var0 = variables.Variable([1.0, 2.0], dtype=dtype)
-        var1 = variables.Variable([3.0, 4.0], dtype=dtype)
-        cost = 5 * var0
-        global_step = variables.Variable(
-            array_ops.zeros([], dtypes.int64), name='global_step')
-        sgd_op = gradient_descent.GradientDescentOptimizer(3.0)
-        with self.assertRaisesRegexp(ValueError, 'No gradients'):
-          # var1 has no gradient
-          sgd_op.minimize(cost, global_step, [var1])
+    for i, dtype in enumerate([dtypes.half, dtypes.float32, dtypes.float64]):
+      # Note that we name the variables uniquely here since the variables don't
+      # seem to be getting deleted at the end of the loop.
+      var0 = resource_variable_ops.ResourceVariable([1.0, 2.0], dtype=dtype,
+                                                    name='a%d' % i)
+      var1 = resource_variable_ops.ResourceVariable([3.0, 4.0], dtype=dtype,
+                                                    name='b%d' % i)
+      # pylint: disable=cell-var-from-loop
+      def loss(_):
+        return 5 * var0
+      # pylint: enable=cell-var-from-loop
+      cost = loss if context.in_eager_mode() else loss(var1)
+      sgd_op = gradient_descent.GradientDescentOptimizer(3.0)
+      with self.assertRaisesRegexp(ValueError, 'No gradients'):
+        # var1 has no gradient
+        sgd_op.minimize(cost, var_list=[var1])
 
+  @test_util.run_in_graph_and_eager_modes()
   def testNoGradientsForAnyVariables_Minimize(self):
-    for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
-      with self.test_session():
-        var0 = variables.Variable([1.0, 2.0], dtype=dtype)
-        var1 = variables.Variable([3.0, 4.0], dtype=dtype)
-        cost = constant_op.constant(5.0)
-        global_step = variables.Variable(
-            array_ops.zeros([], dtypes.int64), name='global_step')
-        sgd_op = gradient_descent.GradientDescentOptimizer(3.0)
-        with self.assertRaisesRegexp(ValueError,
-                                     'No gradients provided for any variable'):
-          sgd_op.minimize(cost, global_step, [var0, var1])
+    for i, dtype in enumerate([dtypes.half, dtypes.float32, dtypes.float64]):
+      # Note that we name the variables uniquely here since the variables don't
+      # seem to be getting deleted at the end of the loop.
+      var0 = resource_variable_ops.ResourceVariable([1.0, 2.0], dtype=dtype,
+                                                    name='a_%d' % i)
+      var1 = resource_variable_ops.ResourceVariable([3.0, 4.0], dtype=dtype,
+                                                    name='b_%d' % i)
+      def loss(unused_v1, unused_v2):
+        return constant_op.constant(5.0)
+      cost = loss if context.in_eager_mode() else loss(var0, var1)
+      sgd_op = gradient_descent.GradientDescentOptimizer(3.0)
+      with self.assertRaisesRegexp(ValueError,
+                                   'No gradients provided for any variable'):
+        sgd_op.minimize(cost, var_list=[var0, var1])
 
+  @test_util.run_in_graph_and_eager_modes()
   def testNoGradientsForAnyVariables_ApplyGradients(self):
-    for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
-      with self.test_session():
-        var0 = variables.Variable([1.0, 2.0], dtype=dtype)
-        var1 = variables.Variable([3.0, 4.0], dtype=dtype)
-        sgd_op = gradient_descent.GradientDescentOptimizer(3.0)
-        with self.assertRaisesRegexp(ValueError,
-                                     'No gradients provided for any variable'):
-          sgd_op.apply_gradients([(None, var0), (None, var1)])
+    for i, dtype in enumerate([dtypes.half, dtypes.float32, dtypes.float64]):
+      # Note that we name the variables uniquely here since the variables don't
+      # seem to be getting deleted at the end of the loop.
+      var0 = resource_variable_ops.ResourceVariable([1.0, 2.0], dtype=dtype,
+                                                    name='a_%d' % i)
+      var1 = resource_variable_ops.ResourceVariable([3.0, 4.0], dtype=dtype,
+                                                    name='b_%d' % i)
+      sgd_op = gradient_descent.GradientDescentOptimizer(3.0)
+      with self.assertRaisesRegexp(ValueError,
+                                   'No gradients provided for any variable'):
+        sgd_op.apply_gradients([(None, var0), (None, var1)])
 
+  @test_util.run_in_graph_and_eager_modes()
   def testGradientsAsVariables(self):
-    for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
-      with self.test_session() as sess:
-        var0 = variables.Variable([1.0, 2.0], dtype=dtype)
-        var1 = variables.Variable([3.0, 4.0], dtype=dtype)
-        cost = 5 * var0 + 3 * var1
-        global_step = variables.Variable(
-            array_ops.zeros([], dtypes.int64), name='global_step')
-        sgd_op = gradient_descent.GradientDescentOptimizer(3.0)
-        grads_and_vars = sgd_op.compute_gradients(cost, [var0, var1])
-        # Convert gradients to tf.Variables
-        converted_grads = [
-            variables.Variable(array_ops.zeros([2], dtype))
-            for i in grads_and_vars
-        ]
-        convert_ops = [
-            state_ops.assign(converted_grads[i], gv[0])
-            for i, gv in enumerate(grads_and_vars)
-        ]
-
-        converted_grads_and_vars = list(zip(converted_grads, [var0, var1]))
-        opt_op = sgd_op.apply_gradients(converted_grads_and_vars, global_step)
+    for i, dtype in enumerate([dtypes.half, dtypes.float32, dtypes.float64]):
+      # Note that we name the variables uniquely here since the variables don't
+      # seem to be getting deleted at the end of the loop.
+      var0 = resource_variable_ops.ResourceVariable([1.0, 2.0], dtype=dtype,
+                                                    name='a%d' % i)
+      var1 = resource_variable_ops.ResourceVariable([3.0, 4.0], dtype=dtype,
+                                                    name='b%d' % i)
+      def loss(v0, v1):
+        return 5 * v0 + 3 * v1
+      cost = loss if context.in_eager_mode() else loss(var0, var1)
+      sgd_op = gradient_descent.GradientDescentOptimizer(3.0)
+      grads_and_vars = sgd_op.compute_gradients(cost, [var0, var1])
+      # Convert gradients to tf.Variables
+      converted_grads = [
+          resource_variable_ops.ResourceVariable(array_ops.zeros([2], dtype),
+                                                 name='c_%d_%d' % (i, j))
+          for j, gv in enumerate(grads_and_vars)
+      ]
+      convert_ops = [
+          state_ops.assign(converted_grads[j], gv[0])
+          for j, gv in enumerate(grads_and_vars)
+      ]
 
-        variables.global_variables_initializer().run()
-        # Run convert_ops to achieve the gradietns converting
-        sess.run(convert_ops)
-        # Fetch params to validate initial values
-        self.assertAllClose([1.0, 2.0], var0.eval())
-        self.assertAllClose([3.0, 4.0], var1.eval())
-        # Run 1 step of sgd through optimizer
-        opt_op.run()
-        # Validate updated params
-        self.assertAllClose([-14., -13.], var0.eval())
-        self.assertAllClose([-6., -5.], var1.eval())
+      self.evaluate(variables.global_variables_initializer())
+      # Run convert_ops to achieve the gradietns converting
+      self.evaluate(convert_ops)
+      # Fetch params to validate initial values
+      self.assertAllClose([1.0, 2.0], self.evaluate(var0))
+      self.assertAllClose([3.0, 4.0], self.evaluate(var1))
+
+      # Run 1 step of sgd through optimizer
+      converted_grads_and_vars = list(zip(converted_grads, [var0, var1]))
+      opt_op = sgd_op.apply_gradients(converted_grads_and_vars)
+      self.evaluate(opt_op)
+
+      # Validate updated params
+      self.assertAllClose([-14., -13.], self.evaluate(var0))
+      self.assertAllClose([-6., -5.], self.evaluate(var1))
 
   def testTrainOp(self):
     with self.test_session():
-- 
GitLab


From 4f127e9019ff32f5c165550d535e4ad0fa587dd6 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 23 Oct 2017 14:42:03 -0700
Subject: [PATCH 693/909] Infer
 `tf.contrib.distributions.RelaxedOneHotCategorical` `dtype` from arguments.

PiperOrigin-RevId: 173172808
---
 .../relaxed_onehot_categorical_test.py        |  8 ++++++
 .../python/ops/relaxed_onehot_categorical.py  | 25 +++++++++++++------
 2 files changed, 25 insertions(+), 8 deletions(-)

diff --git a/tensorflow/contrib/distributions/python/kernel_tests/relaxed_onehot_categorical_test.py b/tensorflow/contrib/distributions/python/kernel_tests/relaxed_onehot_categorical_test.py
index 8c8363fe3f..faae9da6ad 100644
--- a/tensorflow/contrib/distributions/python/kernel_tests/relaxed_onehot_categorical_test.py
+++ b/tensorflow/contrib/distributions/python/kernel_tests/relaxed_onehot_categorical_test.py
@@ -164,6 +164,14 @@ class RelaxedOneHotCategoricalTest(test.TestCase):
         self.assertAllEqual([5, 3],
                             dist.sample(5).eval(feed_dict=feed_dict).shape)
 
+  def testDTypes(self):
+    # check that sampling and log_prob work for a range of dtypes
+    with self.test_session():
+      for dtype in (dtypes.float16, dtypes.float32, dtypes.float64):
+        logits = random_ops.random_uniform(shape=[3, 3], dtype=dtype)
+        dist = relaxed_onehot_categorical.RelaxedOneHotCategorical(
+            temperature=0.5, logits=logits)
+        dist.log_prob(dist.sample())
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/contrib/distributions/python/ops/relaxed_onehot_categorical.py b/tensorflow/contrib/distributions/python/ops/relaxed_onehot_categorical.py
index 699cf45a73..b6becfa9fc 100644
--- a/tensorflow/contrib/distributions/python/ops/relaxed_onehot_categorical.py
+++ b/tensorflow/contrib/distributions/python/ops/relaxed_onehot_categorical.py
@@ -130,7 +130,7 @@ class ExpRelaxedOneHotCategorical(distribution.Distribution):
       temperature,
       logits=None,
       probs=None,
-      dtype=dtypes.float32,
+      dtype=None,
       validate_args=False,
       allow_nan_stats=True,
       name="ExpRelaxedOneHotCategorical"):
@@ -150,7 +150,8 @@ class ExpRelaxedOneHotCategorical(distribution.Distribution):
         `N - 1` dimensions index into a batch of independent distributions and
         the last dimension represents a vector of probabilities for each
         class. Only one of `logits` or `probs` should be passed in.
-      dtype: The type of the event samples (default: float32).
+      dtype: The type of the event samples (default: inferred from
+        logits/probs).
       validate_args: Python `bool`, default `False`. When `True` distribution
         parameters are checked for validity despite possibly degrading runtime
         performance. When `False` invalid inputs may silently render incorrect
@@ -163,14 +164,21 @@ class ExpRelaxedOneHotCategorical(distribution.Distribution):
     """
     parameters = locals()
     with ops.name_scope(name, values=[logits, probs, temperature]):
+
+      self._logits, self._probs = distribution_util.get_logits_and_probs(
+          name=name, logits=logits, probs=probs, validate_args=validate_args,
+          multidimensional=True)
+
+      if dtype is None:
+        dtype = self._logits.dtype
+        if not validate_args:
+          temperature = math_ops.cast(temperature, dtype)
+
       with ops.control_dependencies([check_ops.assert_positive(temperature)]
                                     if validate_args else []):
         self._temperature = array_ops.identity(temperature, name="temperature")
         self._temperature_2d = array_ops.reshape(temperature, [-1, 1],
                                                  name="temperature_2d")
-      self._logits, self._probs = distribution_util.get_logits_and_probs(
-          name=name, logits=logits, probs=probs, validate_args=validate_args,
-          multidimensional=True)
 
       logits_shape_static = self._logits.get_shape().with_rank_at_least(1)
       if logits_shape_static.ndims is not None:
@@ -230,7 +238,7 @@ class ExpRelaxedOneHotCategorical(distribution.Distribution):
 
   def _sample_n(self, n, seed=None):
     sample_shape = array_ops.concat([[n], array_ops.shape(self.logits)], 0)
-    logits = self.logits * array_ops.ones(sample_shape)
+    logits = self.logits * array_ops.ones(sample_shape, dtype=self.dtype)
     logits_2d = array_ops.reshape(logits, [-1, self.event_size])
     # Uniform variates must be sampled from the open-interval `(0, 1)` rather
     # than `[0, 1)`. To do so, we use `np.finfo(self.dtype.as_numpy_dtype).tiny`
@@ -368,7 +376,7 @@ class RelaxedOneHotCategorical(
       temperature,
       logits=None,
       probs=None,
-      dtype=dtypes.float32,
+      dtype=None,
       validate_args=False,
       allow_nan_stats=True,
       name="RelaxedOneHotCategorical"):
@@ -388,7 +396,8 @@ class RelaxedOneHotCategorical(
         dimensions index into a batch of independent distributions and the last
         dimension represents a vector of probabilities for each class. Only one
         of `logits` or `probs` should be passed in.
-      dtype: The type of the event samples (default: float32).
+      dtype: The type of the event samples (default: inferred from
+        logits/probs).
       validate_args: Unused in this distribution.
       allow_nan_stats: Python `bool`, default `True`. If `False`, raise an
         exception if a statistic (e.g. mean/mode/etc...) is undefined for any
-- 
GitLab


From f226eb3717a0df815579178f4393d4e68cbe08fc Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 23 Oct 2017 14:42:57 -0700
Subject: [PATCH 694/909] [XLA] Adds a C64 type to XLA, with actual compilation
 support coming soon.

PiperOrigin-RevId: 173172916
---
 tensorflow/compiler/tf2xla/type_util.cc       |   3 +
 tensorflow/compiler/xla/literal_util.cc       | 101 +++++++++++++++++-
 tensorflow/compiler/xla/literal_util.h        |  26 +++++
 tensorflow/compiler/xla/literal_util_test.cc  |  85 ++++++++++++++-
 tensorflow/compiler/xla/primitive_util.cc     |  18 ++++
 tensorflow/compiler/xla/primitive_util.h      |  15 +++
 .../compiler/xla/service/hlo_evaluator.cc     |   3 +
 tensorflow/compiler/xla/shape_util.cc         |   6 ++
 tensorflow/compiler/xla/shape_util.h          |   3 +
 tensorflow/compiler/xla/shape_util_test.cc    |   4 +
 .../xla/tests/client_library_test_base.cc     |   6 +-
 .../compiler/xla/tests/literal_test_util.cc   |  52 ++++++++-
 tensorflow/compiler/xla/types.h               |   2 +
 tensorflow/compiler/xla/xla_data.proto        |   4 +
 14 files changed, 317 insertions(+), 11 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/type_util.cc b/tensorflow/compiler/tf2xla/type_util.cc
index c698488776..1efbe0ffb1 100644
--- a/tensorflow/compiler/tf2xla/type_util.cc
+++ b/tensorflow/compiler/tf2xla/type_util.cc
@@ -58,6 +58,9 @@ Status DataTypeToPrimitiveType(DataType data_type, xla::PrimitiveType* type) {
     case tensorflow::DT_DOUBLE:
       *type = xla::F64;
       return Status::OK();
+    case tensorflow::DT_COMPLEX64:
+      *type = xla::C64;
+      return Status::OK();
     case tensorflow::DT_QUINT8:
       *type = xla::U8;
       return Status::OK();
diff --git a/tensorflow/compiler/xla/literal_util.cc b/tensorflow/compiler/xla/literal_util.cc
index 79e40c1262..413b85e3ba 100644
--- a/tensorflow/compiler/xla/literal_util.cc
+++ b/tensorflow/compiler/xla/literal_util.cc
@@ -173,6 +173,8 @@ Status Literal::Copy(const Literal& src_literal,
       return CopyRange<float>(src_literal, src_base, dest_base, copy_size);
     case F64:
       return CopyRange<double>(src_literal, src_base, dest_base, copy_size);
+    case C64:
+      return CopyRange<complex64>(src_literal, src_base, dest_base, copy_size);
     case PRED:
       return CopyRange<bool>(src_literal, src_base, dest_base, copy_size);
     default:
@@ -522,6 +524,10 @@ string Literal::GetAsString(
       return tensorflow::strings::StrCat(Get<float>(multi_index));
     case F64:
       return tensorflow::strings::StrCat(Get<double>(multi_index));
+    case C64: {
+      complex64 c = Get<complex64>(multi_index);
+      return tensorflow::strings::StrCat("(", c.real(), ", ", c.imag(), ")");
+    }
     case F16:
       return tensorflow::strings::StrCat(Get<half>(multi_index));
     default:
@@ -716,6 +722,8 @@ void* Literal::MutableInternalData() {
       return reinterpret_cast<void*>(f32s_.data());
     case F64:
       return reinterpret_cast<void*>(f64s_.data());
+    case C64:
+      return reinterpret_cast<void*>(c64s_.data());
     case F16:
       return reinterpret_cast<void*>(f16s_.data());
     default:
@@ -754,6 +762,9 @@ void Literal::Reserve(int64 num_elements) {
     case F64:
       Resize<double>(num_elements, 0);
       break;
+    case C64:
+      Resize<complex64>(num_elements, 0);
+      break;
     case F16:
       Resize<half>(num_elements, static_cast<half>(0.0f));
       break;
@@ -790,6 +801,9 @@ tensorflow::Status Literal::ValidateLiteral() const {
     case F64:
       actual = f64s_size();
       break;
+    case C64:
+      actual = c64s_size();
+      break;
     case F16:
       actual = f16s().size() / sizeof(half);
       break;
@@ -843,6 +857,26 @@ std::unique_ptr<Literal> ConvertBetweenNativeTypes(const Literal& src_literal) {
   return result_literal;
 }
 
+template <PrimitiveType primitive_src_type>
+std::unique_ptr<Literal> ConvertToC64(const Literal& src_literal) {
+  auto result_literal = MakeUnique<Literal>();
+  Shape* result_shape = result_literal->mutable_shape();
+  *result_shape = src_literal.shape();
+  result_shape->set_element_type(C64);
+  result_literal->Reserve(ShapeUtil::ElementsIn(*result_shape));
+  using NativeSrcT =
+      typename primitive_util::PrimitiveTypeToNative<primitive_src_type>::type;
+  tensorflow::gtl::ArraySlice<NativeSrcT> src_data =
+      src_literal.GetArraySlice<NativeSrcT>();
+  tensorflow::gtl::MutableArraySlice<complex64> dest_data =
+      result_literal->GetMutableArraySlice<complex64>();
+  int64 num_elements = ShapeUtil::ElementsIn(src_literal.shape());
+  for (int64 i = 0; i < num_elements; ++i) {
+    dest_data[i] = complex64(static_cast<float>(src_data[i]), 0);
+  }
+  return result_literal;
+}
+
 template <PrimitiveType primitive_src_type, PrimitiveType primitive_dest_type>
 std::unique_ptr<Literal> ConvertIfTypesMatch(const Literal& src_literal) {
   CHECK_EQ(primitive_src_type, src_literal.shape().element_type());
@@ -870,6 +904,8 @@ StatusOr<std::unique_ptr<Literal>> ConvertIfDestTypeMatches(
     CONVERT_IF_TYPES_MATCH(F32)
     CONVERT_IF_TYPES_MATCH(F64)
 #undef CONVERT_IF_TYPES_MATCH
+    case C64:
+      return ConvertToC64<primitive_src_type>(src_literal);
     // Other types are not yet supported.
     default:
       return InvalidArgument(
@@ -966,6 +1002,8 @@ bool Literal::operator==(const Literal& other) const {
         return EqualElements<double>(*this, other, 0, &multi_index);
       case F16:
         return EqualElements<half>(*this, other, 0, &multi_index);
+      case C64:
+        return EqualElements<complex64>(*this, other, 0, &multi_index);
       default:
         LOG(FATAL) << "Unimplemented: Literal::Equal for type "
                    << PrimitiveType_Name(shape().element_type());
@@ -1065,6 +1103,12 @@ tensorflow::gtl::MutableArraySlice<double> Literal::GetMutableArraySlice() {
                                                     values->size());
 }
 
+template <>
+tensorflow::gtl::MutableArraySlice<complex64> Literal::GetMutableArraySlice() {
+  auto values = mutable_c64s();
+  return {values->data(), values->size()};
+}
+
 template <>
 tensorflow::gtl::MutableArraySlice<half> Literal::GetMutableArraySlice<half>() {
   // TODO - there is an endianess problem here. fix it, or wait for uint16
@@ -1144,6 +1188,13 @@ tensorflow::gtl::ArraySlice<half> Literal::GetArraySlice<half>() const {
                                            f16s().size() / sizeof(half));
 }
 
+template <>
+tensorflow::gtl::ArraySlice<complex64> Literal::GetArraySlice<complex64>()
+    const {
+  CHECK_EQ(shape().element_type(), C64);
+  return c64s();
+}
+
 template <typename NativeT>
 static bool AllElementsEqualValue(const Literal& literal, NativeT value) {
   for (int64 i = 0; i < ShapeUtil::ElementsIn(literal.shape()); ++i) {
@@ -1211,6 +1262,15 @@ bool Literal::IsAllFloat(float value) const {
   }
 }
 
+bool Literal::IsAllComplex(complex64 value) const {
+  switch (shape().element_type()) {
+    case C64:
+      return AllElementsEqualValue<complex64>(*this, value);
+    default:
+      return false;
+  }
+}
+
 bool Literal::IsZero(tensorflow::gtl::ArraySlice<int64> indices) const {
   switch (shape().element_type()) {
     case U8:
@@ -1229,6 +1289,8 @@ bool Literal::IsZero(tensorflow::gtl::ArraySlice<int64> indices) const {
       return Get<float>(indices) == 0.0f;
     case F64:
       return Get<double>(indices) == 0.0;
+    case C64:
+      return Get<complex64>(indices) == complex64(0.0f, 0.0f);
     case F16:
       return Get<half>(indices) == static_cast<half>(0.0f);
     case PRED:
@@ -1298,12 +1360,27 @@ void Literal::Resize<half>(int64 num_elements, half value) {
   mutable_f16s()->resize(num_elements, value);
 }
 
+template <>
+void Literal::Resize<complex64>(int64 num_elements, complex64 value) {
+  CHECK_EQ(ShapeUtil::ElementsIn(shape()), num_elements);
+  mutable_c64s()->resize(num_elements, value);
+}
+
 template <typename RepeatedFieldT, typename NativeT>
-static void CopyToRepeatedField(RepeatedFieldT* dest,
-                                const std::vector<NativeT>& src) {
+void CopyToRepeatedField(RepeatedFieldT* dest,
+                         const std::vector<NativeT>& src) {
   *dest = RepeatedFieldT(src.begin(), src.end());
 }
 
+template <>
+void CopyToRepeatedField<tensorflow::protobuf::RepeatedField<float>, complex64>(
+    tensorflow::protobuf::RepeatedField<float>* dest,
+    const std::vector<complex64>& src) {
+  *dest = tensorflow::protobuf::RepeatedField<float>(
+      reinterpret_cast<const float*>(src.data()),
+      reinterpret_cast<const float*>(src.data()) + src.size() * 2);
+}
+
 LiteralProto Literal::ToProto() const {
   LiteralProto proto;
   proto.Clear();
@@ -1338,6 +1415,9 @@ LiteralProto Literal::ToProto() const {
     case F64:
       CopyToRepeatedField(proto.mutable_f64s(), f64s());
       break;
+    case C64:
+      CopyToRepeatedField(proto.mutable_c64s(), c64s());
+      break;
     case TUPLE:
       for (const auto& tuple : tuple_literals()) {
         *proto.add_tuple_literals() = tuple.ToProto();
@@ -1351,11 +1431,21 @@ LiteralProto Literal::ToProto() const {
 }
 
 template <typename RepeatedFieldT, typename NativeT>
-static void CopyFromRepeatedField(std::vector<NativeT>* dest,
-                                  const RepeatedFieldT& src) {
+void CopyFromRepeatedField(std::vector<NativeT>* dest,
+                           const RepeatedFieldT& src) {
   *dest = std::vector<NativeT>(src.begin(), src.end());
 }
 
+template <>
+void CopyFromRepeatedField<tensorflow::protobuf::RepeatedField<float>,
+                           complex64>(
+    std::vector<complex64>* dest,
+    const tensorflow::protobuf::RepeatedField<float>& src) {
+  *dest = std::vector<complex64>(
+      reinterpret_cast<const complex64*>(src.data()),
+      reinterpret_cast<const complex64*>(src.data()) + src.size() / 2);
+}
+
 void Literal::CopyFromProto(const LiteralProto& literal_proto) {
   if (!literal_proto.has_shape()) {
     return;
@@ -1394,6 +1484,9 @@ void Literal::CopyFromProto(const LiteralProto& literal_proto) {
     case F64:
       CopyFromRepeatedField(mutable_f64s(), literal_proto.f64s());
       break;
+    case C64:
+      CopyFromRepeatedField(mutable_c64s(), literal_proto.c64s());
+      break;
     case TUPLE:
       for (const auto& proto : literal_proto.tuple_literals()) {
         mutable_tuple_literals()->push_back(Literal(proto));
diff --git a/tensorflow/compiler/xla/literal_util.h b/tensorflow/compiler/xla/literal_util.h
index 4063cb05a9..a1e288829f 100644
--- a/tensorflow/compiler/xla/literal_util.h
+++ b/tensorflow/compiler/xla/literal_util.h
@@ -159,6 +159,10 @@ class Literal {
   const std::vector<double>& f64s() const { return f64s_; }
   std::vector<double>* mutable_f64s() { return &f64s_; }
 
+  int c64s_size() const { return c64s().size(); }
+  const std::vector<complex64>& c64s() const { return c64s_; }
+  std::vector<complex64>* mutable_c64s() { return &c64s_; }
+
   int tuple_literals_size() const { return tuple_literals().size(); }
   const Literal& tuple_literals(int i) const { return tuple_literals_[i]; }
   Literal* add_tuple_literals() {
@@ -560,6 +564,17 @@ class Literal {
   // e.g. -0.5.
   bool IsAllFloat(float value) const;
 
+  // Like IsAll(const Literal&, int8), except we check whether the literal is
+  // equal to a particular complex number.
+  //
+  // If the literal is not a complex value, this always returns false.
+  //
+  // This casts value to the type of literal, then compares using ==.  The usual
+  // admonishments about floating-point equality checks apply.  We expect you to
+  // use this to check for complex values that can be expressed precisely as
+  // float pairs e.g. (-0.5, 1.0).
+  bool IsAllComplex(complex64 value) const;
+
   // Returns whether this literal is zero at the specified index. This literal
   // must be an array.
   bool IsZero(tensorflow::gtl::ArraySlice<int64> indices) const;
@@ -610,6 +625,7 @@ class Literal {
   std::vector<half> f16s_;
   std::vector<float> f32s_;
   std::vector<double> f64s_;
+  std::vector<complex64> c64s_;
   std::vector<Literal> tuple_literals_;
 };
 
@@ -658,6 +674,10 @@ tensorflow::gtl::ArraySlice<double> Literal::GetArraySlice<double>() const;
 template <>
 tensorflow::gtl::ArraySlice<half> Literal::GetArraySlice<half>() const;
 
+template <>
+tensorflow::gtl::ArraySlice<complex64> Literal::GetArraySlice<complex64>()
+    const;
+
 template <>
 tensorflow::gtl::MutableArraySlice<bool> Literal::GetMutableArraySlice();
 
@@ -694,6 +714,9 @@ tensorflow::gtl::MutableArraySlice<double> Literal::GetMutableArraySlice();
 template <>
 tensorflow::gtl::MutableArraySlice<half> Literal::GetMutableArraySlice();
 
+template <>
+tensorflow::gtl::MutableArraySlice<complex64> Literal::GetMutableArraySlice();
+
 template <>
 void Literal::Resize<bool>(int64 num_elements, bool value);
 
@@ -724,6 +747,9 @@ void Literal::Resize<double>(int64 num_elements, double value);
 template <>
 void Literal::Resize<half>(int64 num_elements, half value);
 
+template <>
+void Literal::Resize<complex64>(int64 num_elements, complex64 value);
+
 template <typename NativeT>
 /* static */ std::unique_ptr<Literal> Literal::CreateR0(NativeT value) {
   auto literal = MakeUnique<Literal>();
diff --git a/tensorflow/compiler/xla/literal_util_test.cc b/tensorflow/compiler/xla/literal_util_test.cc
index e7dedd0821..a9af4849e2 100644
--- a/tensorflow/compiler/xla/literal_util_test.cc
+++ b/tensorflow/compiler/xla/literal_util_test.cc
@@ -107,6 +107,9 @@ TEST_F(LiteralUtilTest, LiteralScalarToString) {
 
   auto f16_lit = Literal::CreateR0<half>(static_cast<half>(0.5f));
   ASSERT_EQ("0.5", f16_lit->ToString());
+
+  auto c64_lit = Literal::CreateR0<complex64>({3.14f, 2.78f});
+  ASSERT_EQ("(3.14, 2.78)", c64_lit->ToString());
 }
 
 TEST_F(LiteralUtilTest, LiteralVectorToString) {
@@ -331,6 +334,19 @@ TEST_F(LiteralUtilTest, TupleEquality) {
   EXPECT_NE(*tuple1, *different_tuple);
 }
 
+TEST_F(LiteralUtilTest, C64Equality) {
+  // Test equality with tuples.
+  auto vector = Literal::CreateR1<complex64>({{1.0, 2.0}, {3.0, 4.0}});
+
+  // Tuple with the same elements. One element is shared with the original
+  // tuple, the other is a clone of the element in the original tuple.
+  auto vector_clone = Literal::CreateR1<complex64>({{1.0, 2.0}, {3.0, 4.0}});
+  EXPECT_EQ(*vector, *vector_clone);
+
+  auto vector_reversed = Literal::CreateR1<complex64>({{3.0, 4.0}, {1.0, 2.0}});
+  EXPECT_NE(*vector, *vector_reversed);
+}
+
 TEST_F(LiteralUtilTest, IsAllTuple) {
   auto element1 = Literal::CreateR0<float>(0.0);
   auto element2 = Literal::CreateR2<float>({{0.0, 0.0}, {0.0, 0.0}});
@@ -381,6 +397,9 @@ TEST_F(LiteralUtilTest, IsAll) {
   EXPECT_FALSE(Literal::CreateR2<half>({{h8}, {h9}})->IsAll(8));
   EXPECT_FALSE(Literal::CreateR2<half>({{h9}, {h8}})->IsAll(8));
 
+  complex64 c8_9 = {8, 9};
+  EXPECT_FALSE(Literal::CreateR2<complex64>({{c8_9}, {c8_9}})->IsAll(8));
+
   auto uint64_max = std::numeric_limits<uint64>::max();
   EXPECT_FALSE(Literal::CreateR2<uint64>(
                    {{uint64_max, uint64_max}, {uint64_max, uint64_max}})
@@ -411,6 +430,25 @@ TEST_F(LiteralUtilTest, IsAllFloat) {
       Literal::CreateR2<double>({{0, 0, 0}, {0, .1, 0}})->IsAllFloat(0));
 }
 
+TEST_F(LiteralUtilTest, IsAllComplex) {
+  // IsAllComplex always returns false when the literal is not complex.
+  EXPECT_FALSE(Literal::CreateR0<bool>(false)->IsAllComplex(0));
+  EXPECT_FALSE(Literal::CreateR0<int8>(0)->IsAllComplex(0));
+  EXPECT_FALSE(Literal::CreateR0<uint8>(0)->IsAllComplex(0));
+  EXPECT_FALSE(Literal::CreateR0<int>(0)->IsAllComplex(0));
+  EXPECT_FALSE(Literal::CreateR0<float>(0)->IsAllComplex(0));
+  EXPECT_FALSE(Literal::CreateR0<double>(0)->IsAllComplex(0));
+
+  complex64 c8_9 = {8, 9};
+  complex64 c7_9 = {7, 9};
+  EXPECT_TRUE(Literal::CreateR2<complex64>({{c8_9}, {c8_9}})
+                  ->IsAllComplex({8.0f, 9.0f}));
+  EXPECT_FALSE(Literal::CreateR2<complex64>({{c7_9}, {c8_9}})
+                   ->IsAllComplex({8.0f, 9.0f}));
+  EXPECT_FALSE(Literal::CreateR2<complex64>({{c8_9}, {c7_9}})
+                   ->IsAllComplex({8.0f, 9.0f}));
+}
+
 TEST_F(LiteralUtilTest, IsZero) {
   auto scalar_zero = Literal::CreateR0<float>(0.0f);
   auto scalar_one = Literal::CreateR0<float>(1.0f);
@@ -422,12 +460,17 @@ TEST_F(LiteralUtilTest, IsZero) {
   EXPECT_TRUE(array->IsZero({0, 2}));
   EXPECT_TRUE(array->IsZero({1, 1}));
   EXPECT_FALSE(array->IsZero({1, 2}));
+
+  auto complex_zero = Literal::CreateR0<complex64>(0.0f);
+  auto complex_nonzero = Literal::CreateR0<complex64>(0.5f);
+  EXPECT_TRUE(complex_zero->IsZero({}));
+  EXPECT_FALSE(complex_nonzero->IsZero({}));
 }
 
 template <typename T>
 class LiteralUtilTestTemplated : public ::testing::Test {};
 
-using TestedTypes = ::testing::Types<float, int32, uint32>;
+using TestedTypes = ::testing::Types<float, int32, uint32, complex64>;
 TYPED_TEST_CASE(LiteralUtilTestTemplated, TestedTypes);
 
 TYPED_TEST(LiteralUtilTestTemplated, Relayout2x2) {
@@ -626,13 +669,28 @@ TEST_F(LiteralUtilTest, PopulateR1S64) {
   EXPECT_EQ(output, *expected);
 }
 
-TEST_F(LiteralUtilTest, PopulateR2U64) {
+TEST_F(LiteralUtilTest, PopulateR1U64) {
   Literal output;
   output.PopulateR1<uint64>({{77, 88}});
   auto expected = Literal::CreateR1<uint64>({{77, 88}});
   EXPECT_EQ(output, *expected);
 }
 
+TEST_F(LiteralUtilTest, PopulateR1C64) {
+  Literal output;
+  output.PopulateR1<complex64>({{77, 88}});
+  auto expected = Literal::CreateR1<complex64>({{77, 88}});
+  EXPECT_EQ(output, *expected);
+}
+
+TEST_F(LiteralUtilTest, PopulateR2C64) {
+  Literal output;
+  output.PopulateR2<complex64>({{{7, 8}, {9, 10}}, {{1, 2}, {3, 4}}});
+  auto expected =
+      Literal::CreateR2<complex64>({{{7, 8}, {9, 10}}, {{1, 2}, {3, 4}}});
+  EXPECT_EQ(output, *expected);
+}
+
 TEST_F(LiteralUtilTest, PopulateWithValueR0F32) {
   Literal output;
   output.PopulateWithValue<float>(2.5f, {});
@@ -654,6 +712,14 @@ TEST_F(LiteralUtilTest, PopulateWithValueR2U64) {
   EXPECT_EQ(output, *expected);
 }
 
+TEST_F(LiteralUtilTest, PopulateWithValueR2C64) {
+  Literal output;
+  output.PopulateWithValue<complex64>({4, 2}, {2, 2});
+  auto expected =
+      Literal::CreateR2<complex64>({{{4, 2}, {4, 2}}, {{4, 2}, {4, 2}}});
+  EXPECT_EQ(output, *expected);
+}
+
 TEST_F(LiteralUtilTest, PopulateWithValueR0F16) {
   Literal output;
   half h(0.25f);
@@ -919,6 +985,11 @@ TEST_F(LiteralUtilTest, ConvertIfTypesMatch) {
     {{0.0, 19.0, 0.0, 21.0}, {22.0, 0.0, 24.0, 0.0}},
     {{26.0, 0.0, 28.0, 0.0}, {0.0, 31.0, 0.0, 33.0}},
   }}, layout_r4_dim0major_);
+  auto c64 = Literal::CreateR4WithLayout<complex64>({{
+    {{10.0f, 0.0f, 12.0f, 0.0f}, {0.0f, 15.0f, 0.0f, 17.0f}},
+    {{0.0f, 19.0f, 0.0f, 21.0f}, {22.0f, 0.0f, 24.0f, 0.0f}},
+    {{26.0f, 0.0f, 28.0f, 0.0f}, {0.0f, 31.0f, 0.0f, 33.0f}},
+  }}, layout_r4_dim0major_);
   // clang-format on
   std::unique_ptr<Literal> conv;
 
@@ -961,12 +1032,22 @@ TEST_F(LiteralUtilTest, ConvertIfTypesMatch) {
   conv = u32->Convert(F16).ConsumeValueOrDie();
   EXPECT_EQ(*conv, *f16);
 
+  conv = s32->Convert(C64).ConsumeValueOrDie();
+  EXPECT_EQ(*conv, *c64);
+
+  conv = f16->Convert(C64).ConsumeValueOrDie();
+  EXPECT_EQ(*conv, *c64);
+
   EXPECT_EQ(s32->Convert(TUPLE).status().code(),
             tensorflow::error::INVALID_ARGUMENT);
   EXPECT_EQ(s32->Convert(S16).status().code(),
             tensorflow::error::INVALID_ARGUMENT);
   EXPECT_EQ(s32->Convert(U16).status().code(),
             tensorflow::error::INVALID_ARGUMENT);
+  EXPECT_EQ(c64->Convert(F32).status().code(),
+            tensorflow::error::INVALID_ARGUMENT);
+  EXPECT_EQ(c64->Convert(S32).status().code(),
+            tensorflow::error::INVALID_ARGUMENT);
 }
 
 TEST_F(LiteralUtilTest, CopyFromProto_Bool) {
diff --git a/tensorflow/compiler/xla/primitive_util.cc b/tensorflow/compiler/xla/primitive_util.cc
index e4e37177a2..2113b5e06f 100644
--- a/tensorflow/compiler/xla/primitive_util.cc
+++ b/tensorflow/compiler/xla/primitive_util.cc
@@ -83,10 +83,17 @@ PrimitiveType NativeToPrimitiveType<half>() {
   return F16;
 }
 
+template <>
+PrimitiveType NativeToPrimitiveType<complex64>() {
+  return C64;
+}
+
 bool IsFloatingPointType(PrimitiveType type) {
   return type == F16 || type == F32 || type == F64;
 }
 
+bool IsComplexType(PrimitiveType type) { return type == C64; }
+
 bool IsSignedIntegralType(PrimitiveType type) {
   return type == S8 || type == S16 || type == S32 || type == S64;
 }
@@ -121,6 +128,7 @@ int BitWidth(PrimitiveType type) {
     case U64:
     case S64:
     case F64:
+    case C64:
       return 64;
 
     case TUPLE:
@@ -134,5 +142,15 @@ int BitWidth(PrimitiveType type) {
   }
 }
 
+PrimitiveType ComplexComponentType(PrimitiveType complex_type) {
+  switch (complex_type) {
+    case C64:
+      return F32;
+    default:
+      LOG(FATAL) << "Primitive type is not complex: "
+                 << PrimitiveType_Name(complex_type);
+  }
+}
+
 }  // namespace primitive_util
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/primitive_util.h b/tensorflow/compiler/xla/primitive_util.h
index 162a11c7d2..a49c8b86fc 100644
--- a/tensorflow/compiler/xla/primitive_util.h
+++ b/tensorflow/compiler/xla/primitive_util.h
@@ -78,8 +78,14 @@ PrimitiveType NativeToPrimitiveType<double>();
 template <>
 PrimitiveType NativeToPrimitiveType<half>();
 
+// Complex
+template <>
+PrimitiveType NativeToPrimitiveType<complex64>();
+
 bool IsFloatingPointType(PrimitiveType type);
 
+bool IsComplexType(PrimitiveType type);
+
 bool IsSignedIntegralType(PrimitiveType type);
 
 bool IsUnsignedIntegralType(PrimitiveType type);
@@ -89,6 +95,10 @@ bool IsIntegralType(PrimitiveType type);
 // Returns the number of bits in the representation for a given type.
 int BitWidth(PrimitiveType type);
 
+// Returns the real, imag component type underlying the given complex type.
+// LOG(FATAL)'s if complex_type is not complex.
+PrimitiveType ComplexComponentType(PrimitiveType complex_type);
+
 // Returns the native type (eg, float) corresponding to the given template
 // parameter XLA primitive type (eg, F32).
 template <PrimitiveType>
@@ -157,6 +167,11 @@ struct PrimitiveTypeToNative<F16> {
   using type = half;
 };
 
+// Complex
+template <>
+struct PrimitiveTypeToNative<C64> {
+  using type = complex64;
+};
 }  // namespace primitive_util
 }  // namespace xla
 
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator.cc b/tensorflow/compiler/xla/service/hlo_evaluator.cc
index e8f88427da..fa6a8f3d53 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator.cc
+++ b/tensorflow/compiler/xla/service/hlo_evaluator.cc
@@ -1265,6 +1265,9 @@ HloEvaluator::HloEvaluator() {
   });
   typed_visitors_[F32] = MakeUnique<TypedVisitor<float>>(this);
   typed_visitors_[F64] = MakeUnique<TypedVisitor<double>>(this);
+  typed_visitors_[C64] = MakeUnique<FunctionVisitor>([](HloInstruction*) {
+    return Unimplemented("unhandled primitive type: C64.");
+  });
   typed_visitors_[TUPLE] = MakeUnique<FunctionVisitor>([](HloInstruction*) {
     return Unimplemented("unhandled primitive type: TUPLE.");
   });
diff --git a/tensorflow/compiler/xla/shape_util.cc b/tensorflow/compiler/xla/shape_util.cc
index af583bed62..fa4f71414d 100644
--- a/tensorflow/compiler/xla/shape_util.cc
+++ b/tensorflow/compiler/xla/shape_util.cc
@@ -281,6 +281,10 @@ StatusOr<Shape> MakeShapeWithLayoutInternal(
   }
 }
 
+/* static */ bool ShapeUtil::ElementIsComplex(const Shape& shape) {
+  return primitive_util::IsComplexType(shape.element_type());
+}
+
 /* static */ bool ShapeUtil::ElementIsFloating(const Shape& shape) {
   return primitive_util::IsFloatingPointType(shape.element_type());
 }
@@ -592,6 +596,8 @@ StatusOr<Shape> ParseShapeStringInternal(tensorflow::StringPiece* s) {
       return sizeof(float);
     case F64:
       return sizeof(double);
+    case C64:
+      return sizeof(complex64);
     default:
       LOG(FATAL) << "Unhandled primitive type " << primitive_type;
   }
diff --git a/tensorflow/compiler/xla/shape_util.h b/tensorflow/compiler/xla/shape_util.h
index c5800acaf1..8f8d4a73c9 100644
--- a/tensorflow/compiler/xla/shape_util.h
+++ b/tensorflow/compiler/xla/shape_util.h
@@ -291,6 +291,9 @@ class ShapeUtil {
   // Returns whether the element type of the shape is floating point.
   static bool ElementIsFloating(const Shape& shape);
 
+  // Returns whether the element type of the shape is complex.
+  static bool ElementIsComplex(const Shape& shape);
+
   // Returns whether the element type has the given bit width.
   static bool ElementHasBitWidth(const Shape& shape, int bits);
 
diff --git a/tensorflow/compiler/xla/shape_util_test.cc b/tensorflow/compiler/xla/shape_util_test.cc
index 79945b9c77..0ba542ad1b 100644
--- a/tensorflow/compiler/xla/shape_util_test.cc
+++ b/tensorflow/compiler/xla/shape_util_test.cc
@@ -218,6 +218,10 @@ TEST(ShapeUtilTest, ByteSizeOfWithoutPadding) {
   EXPECT_EQ(8, ShapeUtil::ByteSizeOfPrimitiveType(F64));
   EXPECT_EQ(8, ShapeUtil::ByteSizeOf(ShapeUtil::MakeShape(F64, {})));
   EXPECT_EQ(1600, ShapeUtil::ByteSizeOf(ShapeUtil::MakeShape(F64, {10, 20})));
+
+  EXPECT_EQ(8, ShapeUtil::ByteSizeOfPrimitiveType(C64));
+  EXPECT_EQ(8, ShapeUtil::ByteSizeOf(ShapeUtil::MakeShape(C64, {})));
+  EXPECT_EQ(1600, ShapeUtil::ByteSizeOf(ShapeUtil::MakeShape(C64, {10, 20})));
 }
 
 TEST(ShapeUtilTest, ByteSizeOfWithPadding) {
diff --git a/tensorflow/compiler/xla/tests/client_library_test_base.cc b/tensorflow/compiler/xla/tests/client_library_test_base.cc
index a60d3e50bd..065bce7e31 100644
--- a/tensorflow/compiler/xla/tests/client_library_test_base.cc
+++ b/tensorflow/compiler/xla/tests/client_library_test_base.cc
@@ -254,7 +254,8 @@ tensorflow::Status ClientLibraryTestBase::ComputeAndCompareLiteralWithStatus(
     tensorflow::gtl::ArraySlice<GlobalData*> arguments,
     const Shape* shape_with_layout) {
   TF_ASSIGN_OR_RETURN(auto computation, builder->Build());
-  if (ShapeUtil::ElementIsFloating(expected.shape())) {
+  if (ShapeUtil::ElementIsFloating(expected.shape()) ||
+      ShapeUtil::ElementIsComplex(expected.shape())) {
     LOG(WARNING) << "performing exact comparison of floating point numbers";
   } else {
     TF_RET_CHECK(ShapeUtil::ElementIsIntegral(expected.shape()) ||
@@ -282,7 +283,8 @@ tensorflow::Status ClientLibraryTestBase::ComputeAndCompareLiteralWithStatus(
     ComputationBuilder* builder, const Literal& expected,
     tensorflow::gtl::ArraySlice<GlobalData*> arguments, ErrorSpec error,
     const Shape* shape_with_layout) {
-  TF_RET_CHECK(ShapeUtil::ElementIsFloating(expected.shape()));
+  TF_RET_CHECK(ShapeUtil::ElementIsFloating(expected.shape()) ||
+               ShapeUtil::ElementIsComplex(expected.shape()));
   TF_ASSIGN_OR_RETURN(auto computation, builder->Build());
   auto expect_near = [&](const Literal& actual, const string& error_message) {
     LiteralTestUtil::ExpectNear(expected, actual, error, error_message);
diff --git a/tensorflow/compiler/xla/tests/literal_test_util.cc b/tensorflow/compiler/xla/tests/literal_test_util.cc
index 2876a79dd8..95a52ecd2f 100644
--- a/tensorflow/compiler/xla/tests/literal_test_util.cc
+++ b/tensorflow/compiler/xla/tests/literal_test_util.cc
@@ -156,6 +156,15 @@ template <>
 ::testing::AssertionResult CompareEqual<double>(double lhs, double rhs) {
   return CompareFloatsBitwiseEqual<double, uint64>(lhs, rhs);
 }
+template <>
+::testing::AssertionResult CompareEqual<complex64>(complex64 lhs,
+                                                   complex64 rhs) {
+  auto res = CompareEqual<float>(lhs.real(), rhs.real());
+  if (!res) {
+    return res;
+  }
+  return CompareEqual<float>(lhs.imag(), rhs.imag());
+}
 
 // A recursive function which iterates through every index of expected and
 // actual literal and compares their values elementwise. Returns true if all
@@ -235,6 +244,9 @@ bool ExpectLiteralsEqual(const Literal& expected, const Literal& actual,
     case F64:
       match = ExpectLiteralsEqual<double>(expected, actual, &multi_index, 0);
       break;
+    case C64:
+      match = ExpectLiteralsEqual<complex64>(expected, actual, &multi_index, 0);
+      break;
     case TUPLE: {
       bool tuple_match = true;
       for (int i = 0; i < actual.tuple_literals_size(); ++i) {
@@ -325,6 +337,9 @@ class NearComparator {
       case F64:
         ExpectLiteralsNear<double>(expected, actual, 0);
         break;
+      case C64:
+        ExpectLiteralsNear<complex64>(expected, actual, 0);
+        break;
       default:
         LOG(FATAL) << "Unsupported primitive type in near comparator: "
                    << PrimitiveType_Name(expected.shape().element_type())
@@ -365,6 +380,19 @@ class NearComparator {
   }
 
  private:
+  template <typename NativeT>
+  bool NanMismatch(NativeT lhs, NativeT rhs) {
+    return std::isnan(lhs) != std::isnan(rhs);
+  }
+
+  template <typename NativeT>
+  void ExpectNear(NativeT expected, NativeT actual,
+                  const ::testing::Message& message) {
+    EXPECT_NEAR(expected, actual, error_.abs)
+        << "expected:\n  " << expected << "\n\tvs actual:\n  " << actual << "\n"
+        << message;
+  }
+
   // EXPECTs that the two given scalar values are within the error bound. Keeps
   // track of how many mismatches have occurred to keep the size of the output
   // manageable.
@@ -390,7 +418,7 @@ class NearComparator {
         "index %s abs_diff %f rel_err %f",
         LiteralTestUtil::MultiIndexAsString(multi_index_).c_str(), abs_diff,
         rel_err);
-    bool nan_mismatch = std::isnan(actual) != std::isnan(expected);
+    bool nan_mismatch = NanMismatch<NativeT>(expected, actual);
     bool mismatch =
         (nan_mismatch || (abs_diff >= error_.abs && rel_err >= error_.rel));
     if (mismatch) {
@@ -398,11 +426,12 @@ class NearComparator {
       abs_expected_miscompare_sum_ += std::abs(expected);
       const int64 kMaxFailures = 2;
       if (num_miscompares_ < kMaxFailures) {
-        EXPECT_NEAR(expected, actual, error_.abs)
-            << "mismatch at index "
+        ::testing::Message msg;
+        msg << "mismatch at index "
             << LiteralTestUtil::MultiIndexAsString(multi_index_) << " abs diff "
             << abs_diff << " rel err " << rel_err << " failure #"
             << num_miscompares_;
+        ExpectNear<NativeT>(expected, actual, msg);
       } else if (num_miscompares_ == kMaxFailures) {
         LOG(ERROR)
             << "reached max 'loud' failure count; silently proceeding...";
@@ -470,6 +499,23 @@ class NearComparator {
   std::vector<int64> max_abs_multi_index_;
 };
 
+template <>
+bool NearComparator::NanMismatch<complex64>(complex64 lhs, complex64 rhs) {
+  return std::isnan(lhs.real()) != std::isnan(rhs.real()) ||
+         std::isnan(lhs.imag()) != std::isnan(rhs.imag());
+}
+
+template <>
+void NearComparator::ExpectNear<complex64>(complex64 expected, complex64 actual,
+                                           const ::testing::Message& message) {
+  EXPECT_NEAR(expected.real(), actual.real(), error_.abs)
+      << "expected:\n  " << expected << "\n\tvs actual:\n  " << actual << "\n"
+      << message;
+  EXPECT_NEAR(expected.imag(), actual.imag(), error_.abs)
+      << "expected:\n  " << expected << "\n\tvs actual:\n  " << actual << "\n"
+      << message;
+}
+
 }  // namespace
 
 /* static */ ::testing::AssertionResult LiteralTestUtil::Near(
diff --git a/tensorflow/compiler/xla/types.h b/tensorflow/compiler/xla/types.h
index ea8b4b7b98..3d78466107 100644
--- a/tensorflow/compiler/xla/types.h
+++ b/tensorflow/compiler/xla/types.h
@@ -35,6 +35,8 @@ using ::tensorflow::uint16;
 using ::tensorflow::uint32;
 using ::tensorflow::uint64;
 
+typedef std::complex<float> complex64;
+
 using ::Eigen::half;
 
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/xla_data.proto b/tensorflow/compiler/xla/xla_data.proto
index eae284afb7..7ad61fab81 100644
--- a/tensorflow/compiler/xla/xla_data.proto
+++ b/tensorflow/compiler/xla/xla_data.proto
@@ -48,6 +48,9 @@ enum PrimitiveType {
   F32 = 11;
   F64 = 12;
 
+  // Complex values of fixed width.
+  C64 = 15;
+
   // A tuple is a polymorphic sequence; e.g. a shape that holds different
   // sub-shapes. They are used for things like returning multiple values from a
   // computation; e.g. a computation that returns weights and biases may have a
@@ -305,6 +308,7 @@ message LiteralProto {
   repeated uint64 u64s = 7;
   repeated float f32s = 8;
   repeated double f64s = 9;
+  repeated float c64s = 12;  // Stored as interleaved real, imag floats.
   repeated LiteralProto tuple_literals = 10;
   bytes f16s = 11;  // Note: the F16s are encoded in little endian byte order
 }
-- 
GitLab


From 555c63d173a41ceaba89513bb5f1b4ac9a4c86e4 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Mon, 23 Oct 2017 15:06:19 -0700
Subject: [PATCH 695/909] Automated g4 rollback of changelist 172946149

PiperOrigin-RevId: 173176850
---
 tensorflow/compiler/aot/compile.cc    | 6 +++---
 tensorflow/compiler/aot/flags.cc      | 5 +++--
 tensorflow/compiler/aot/flags.h       | 2 +-
 tensorflow/compiler/aot/tfcompile.bzl | 3 ---
 4 files changed, 7 insertions(+), 9 deletions(-)

diff --git a/tensorflow/compiler/aot/compile.cc b/tensorflow/compiler/aot/compile.cc
index 77c4ec88cb..eac8da0ab1 100644
--- a/tensorflow/compiler/aot/compile.cc
+++ b/tensorflow/compiler/aot/compile.cc
@@ -97,11 +97,11 @@ Status CompileGraph(const GraphDef& graph_def, const tf2xla::Config& config,
   TF_RETURN_IF_ERROR(ConvertGraphDefToXla(graph_def, config, client,
                                           &computation,
                                           &compile_result->has_context_arg));
-  if (!flags.out_session_module.empty()) {
+  if (!flags.debug_dir.empty()) {
     TF_ASSIGN_OR_RETURN(std::unique_ptr<xla::SessionModule> module,
                         computation.Snapshot());
-    TF_RETURN_IF_ERROR(
-        WriteBinaryProto(Env::Default(), flags.out_session_module, *module));
+    string file = io::JoinPath(flags.debug_dir, "tfcompile_xla_module.pb");
+    TF_RETURN_IF_ERROR(WriteBinaryProto(Env::Default(), file, *module));
   }
   xla::cpu::CpuAotCompilationOptions aot_opts(
       flags.target_triple, flags.target_cpu, flags.target_features,
diff --git a/tensorflow/compiler/aot/flags.cc b/tensorflow/compiler/aot/flags.cc
index 7c2f27e550..5aff10346f 100644
--- a/tensorflow/compiler/aot/flags.cc
+++ b/tensorflow/compiler/aot/flags.cc
@@ -33,6 +33,9 @@ void AppendMainFlags(std::vector<Flag>* flag_list, MainFlags* flags) {
        "fetch nodes will be dumped to stdout in a comma-separated list.  "
        "Typically used to format arguments for other tools, e.g. "
        "freeze_graph."},
+      {"debug_dir", &flags->debug_dir,
+       "Specifies a directory to dump debugging information, including "
+       "rewritten graphs and the XLA HLO module."},
       // Flags controlling the XLA ahead-of-time compilation, that correspond to
       // the fields of xla::cpu::CpuAotCompilationOptions.
       //
@@ -61,8 +64,6 @@ void AppendMainFlags(std::vector<Flag>* flag_list, MainFlags* flags) {
        "namespaces are given, within the global namespace."},
       {"out_object", &flags->out_object, "Output object file name."},
       {"out_header", &flags->out_header, "Output header file name."},
-      {"out_session_module", &flags->out_session_module,
-       "Output session module proto."},
       {"gen_name_to_index", &flags->gen_name_to_index,
        "Generate name-to-index data for Lookup{Arg,Result}Index methods."},
       {"gen_program_shape", &flags->gen_program_shape,
diff --git a/tensorflow/compiler/aot/flags.h b/tensorflow/compiler/aot/flags.h
index 3519659e3a..3246dbf95c 100644
--- a/tensorflow/compiler/aot/flags.h
+++ b/tensorflow/compiler/aot/flags.h
@@ -29,6 +29,7 @@ struct MainFlags {
   string graph;
   string config;
   bool dump_fetch_nodes = false;
+  string debug_dir;
   string target_triple;
   string target_cpu;
   string target_features;
@@ -36,7 +37,6 @@ struct MainFlags {
   string cpp_class;
   string out_object;
   string out_header;
-  string out_session_module;
 
   // C++ codegen options
   bool gen_name_to_index = false;
diff --git a/tensorflow/compiler/aot/tfcompile.bzl b/tensorflow/compiler/aot/tfcompile.bzl
index 0ecfbedcb4..4888760acd 100644
--- a/tensorflow/compiler/aot/tfcompile.bzl
+++ b/tensorflow/compiler/aot/tfcompile.bzl
@@ -129,7 +129,6 @@ def tf_library(name, graph, config,
   # Rule that runs tfcompile to produce the header and object file.
   header_file = name + ".h"
   object_file = name + ".o"
-  session_module_pb = name + "_session_module.pb"
   ep = ("__" + PACKAGE_NAME + "__" + name).replace("/", "_")
   native.genrule(
       name=("gen_" + name),
@@ -140,7 +139,6 @@ def tf_library(name, graph, config,
       outs=[
           header_file,
           object_file,
-          session_module_pb,
       ],
       cmd=("$(location " + tfcompile_tool + ")" +
            " --graph=$(location " + tfcompile_graph + ")" +
@@ -150,7 +148,6 @@ def tf_library(name, graph, config,
            " --target_triple=" + target_llvm_triple() +
            " --out_header=$(@D)/" + header_file +
            " --out_object=$(@D)/" + object_file +
-           " --out_session_module=$(@D)/" + session_module_pb +
            " " + (tfcompile_flags or "")),
       tools=[tfcompile_tool],
       visibility=visibility,
-- 
GitLab


From fbb71d767d890ead9007e713ae77dd223df232bd Mon Sep 17 00:00:00 2001
From: Sarah Maddox <sarahmaddox@users.noreply.github.com>
Date: Tue, 24 Oct 2017 09:20:44 +1100
Subject: [PATCH 696/909] Add links and fix typs

Add links to the docs in GitHub, to make it easier for contributors to find them. Also fix some typos in the names of GitHub and TensorFlow, and standardise capitalisation in headings.
---
 .../docs_src/community/documentation.md       | 39 ++++++++++---------
 1 file changed, 21 insertions(+), 18 deletions(-)

diff --git a/tensorflow/docs_src/community/documentation.md b/tensorflow/docs_src/community/documentation.md
index 655506b098..77d4e0caec 100644
--- a/tensorflow/docs_src/community/documentation.md
+++ b/tensorflow/docs_src/community/documentation.md
@@ -1,6 +1,6 @@
 # Writing TensorFlow Documentation
 
-We welcome contributions to the Tensorflow documentation from the community.
+We welcome contributions to the TensorFlow documentation from the community.
 This document explains how you can contribute to that documentation. In
 particular, this document explains the following:
 
@@ -8,28 +8,30 @@ particular, this document explains the following:
 * How to make conformant edits.
 * How to build and test your documentation changes before you submit them.
 
-You can view Tensorflow documentation on https://www.tensorflow.org, and you
-can view and edit the raw files on Github. We're publishing our docs on Github
-so everybody can contribute. Whatever gets checked in tensorflow/docs_src will
-be published soon after on https://www.tensorflow.org. 
+You can view TensorFlow documentation on https://www.tensorflow.org, and you
+can view and edit the raw files on
+[GitHub](https://www.tensorflow.org/code/tensorflow/docs_src/). 
+We're publishing our docs on GitHub so everybody can contribute. Whatever gets
+checked in to `tensorflow/docs_src` will be published soon after on
+https://www.tensorflow.org. 
 
 Republishing TensorFlow documentation in different forms is absolutely allowed,
 but we are unlikely to accept other documentation formats (or the tooling to
 generate them) into our repository. If you do choose to republish our
 documentation in another form, please be sure to include:
 
-* The version of the API this represents (i.e. r1.0, master, etc.)
+* The version of the API this represents (for example, r1.0, master, etc.)
 * The commit or version from which the documentation was generated
 * Where to get the latest documentation (that is, https://www.tensorflow.org)
 * The Apache 2.0 license.
 
-## A Note on Versions
+## A note on versions
 
 tensorflow.org, at root, shows documentation for the latest stable binary.  This
 is the documentation you should be reading if you are using `pip` to install
 TensorFlow.
 
-However, most developers will contribute documentation into the master Github
+However, most developers will contribute documentation into the master GitHub
 branch, which is published, occasionally,
 at [tensorflow.org/versions/master](https://www.tensorflow.org/versions/master).
 
@@ -49,8 +51,9 @@ in the code:
 To modify the reference documentation, you edit the appropriate code comments.
 
 Non-reference documentation (for example, the TensorFlow installation guides) is
-authored by humans. This documentation is located in the `tensorflow/docs_src`
-directory.  Each subdirectory of `docs_src` contains a set of related Tensorflow
+authored by humans. This documentation is located in the
+[`tensorflow/docs_src`](https://www.tensorflow.org/code/tensorflow/docs_src/)
+directory.  Each subdirectory of `docs_src` contains a set of related TensorFlow
 documentation. For example, the TensorFlow installation guides are all in the
 `docs_src/install` directory.
 
@@ -183,7 +186,7 @@ documentation in the `/tmp/tfdocs` dir:
 
 Note: You must set `src_dir` and `output_dir` to absolute file paths.
 
-## Generating Python API Documentation
+## Generating Python API documentation
 
 Ops, classes, and utility functions are defined in Python modules, such as
 `image_ops.py`. Python modules contain a module docstring. For example:
@@ -216,7 +219,7 @@ the following:
 Only top level modules (currently just `tf` and `tfdbg`) need to be manually
 added to the generate script.
 
-### Sealing Modules
+### Sealing modules
 
 Because the doc generator walks all visible symbols, and descends into anything
 it finds, it will document any accidentally exposed symbols. If a module only
@@ -242,7 +245,7 @@ following options for dealing with them:
 
 We'll discuss these options in detail below.
 
-#### Private Symbols and Imports
+#### Private symbols and imports
 
 The easiest way to conform to the API sealing expectations is to make non-public
 symbols private (by prepending an underscore _). The doc generator respects
@@ -288,7 +291,7 @@ are public. All `@@`s will eventually be removed. If you see them, however,
 please do not randomly delete them as they are still in use by some of our
 systems.
 
-#### Traversal Blacklist
+#### Traversal blacklist
 
 If all else fails, you may add entries to the traversal blacklist in
 `generate_lib.py.` **Almost all entries in this list are an abuse of its
@@ -311,7 +314,7 @@ flags, ...) included for platform abstraction can be documented without
 documenting their interior. Its use beyond this purpose is a shortcut that may
 be acceptable for contrib, but not for core tensorflow.
 
-## Op Documentation Style Guide
+## Op documentation style guide
 
 Long, descriptive module-level documentation for modules should go in the API
 Guides in `docs_src/api_guides/python`.
@@ -334,7 +337,7 @@ is [here](https://daringfireball.net/projects/markdown/). You are allowed to
 use [MathJax](https://www.mathjax.org) notation for equations (see above for
 restrictions).
 
-### Writing About Code
+### Writing about code
 
 Put backticks around these things when they're used in text:
 
@@ -375,7 +378,7 @@ Two notes about backticks for code samples in Markdown:
    However, do NOT indent four spaces and use backticks simultaneously. Use one
    or the other.
 
-### Tensor Dimensions
+### Tensor dimensions
 
 When you're talking about a tensor in general, don't capitalize the word tensor.
 When you're talking about the specific object that's provided to an op as an
@@ -500,7 +503,7 @@ def foo(x, y, name="bar"):
   """
 ```
 
-## Description of the Docstring Sections
+## Description of the docstring sections
 
 This section details each of the elements in docstrings.
 
-- 
GitLab


From fd182dd02e431e7a7f16bd0ad1547405e591bc82 Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Mon, 23 Oct 2017 15:19:06 -0700
Subject: [PATCH 697/909] metrics.Mean writes a summary.

PiperOrigin-RevId: 173178780
---
 tensorflow/contrib/eager/python/BUILD         |  1 +
 .../contrib/eager/python/metrics_impl.py      |  5 +++-
 .../contrib/eager/python/metrics_test.py      | 28 +++++++++++++++++++
 3 files changed, 33 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/eager/python/BUILD b/tensorflow/contrib/eager/python/BUILD
index ace17424fe..a83012e17b 100644
--- a/tensorflow/contrib/eager/python/BUILD
+++ b/tensorflow/contrib/eager/python/BUILD
@@ -129,6 +129,7 @@ py_library(
     srcs_version = "PY2AND3",
     visibility = ["//tensorflow:internal"],
     deps = [
+        "//tensorflow/contrib/summary:summary_ops",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:dtypes",
         "//tensorflow/python:framework_ops",
diff --git a/tensorflow/contrib/eager/python/metrics_impl.py b/tensorflow/contrib/eager/python/metrics_impl.py
index 2139c2b4b9..959ee735b0 100644
--- a/tensorflow/contrib/eager/python/metrics_impl.py
+++ b/tensorflow/contrib/eager/python/metrics_impl.py
@@ -20,6 +20,7 @@ from __future__ import print_function
 
 import re
 
+from tensorflow.contrib.summary import summary_ops
 from tensorflow.python.eager import context
 from tensorflow.python.eager import function
 from tensorflow.python.framework import dtypes
@@ -235,7 +236,9 @@ class Mean(Metric):
       self.numer.assign_add(math_ops.reduce_sum(values))
 
   def result(self):
-    return self.numer / self.denom
+    t = self.numer / self.denom
+    summary_ops.scalar(name=self.name, tensor=t)
+    return t
 
 
 class Accuracy(Mean):
diff --git a/tensorflow/contrib/eager/python/metrics_test.py b/tensorflow/contrib/eager/python/metrics_test.py
index 9743666c89..1880e762d4 100644
--- a/tensorflow/contrib/eager/python/metrics_test.py
+++ b/tensorflow/contrib/eager/python/metrics_test.py
@@ -18,12 +18,20 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import os
+import tempfile
+
 from tensorflow.contrib.eager.python import metrics
+from tensorflow.contrib.summary import summary_ops
+from tensorflow.core.util import event_pb2
 from tensorflow.python.eager import context
 from tensorflow.python.eager import test
 from tensorflow.python.framework import dtypes
+from tensorflow.python.lib.io import tf_record
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import variables
+from tensorflow.python.platform import gfile
+from tensorflow.python.training import training_util
 
 
 class MetricsTest(test.TestCase):
@@ -37,6 +45,26 @@ class MetricsTest(test.TestCase):
     self.assertEqual(dtypes.float64, m.dtype)
     self.assertEqual(dtypes.float64, m.result().dtype)
 
+  def testWriteSummaries(self):
+    m = metrics.Mean()
+    m([1, 10, 100])
+    training_util.get_or_create_global_step()
+    logdir = tempfile.mkdtemp()
+    with summary_ops.create_summary_file_writer(
+        logdir, max_queue=0,
+        name="t0").as_default(), summary_ops.always_record_summaries():
+      m.result()  # As a side-effect will write summaries.
+
+    self.assertTrue(gfile.Exists(logdir))
+    files = gfile.ListDirectory(logdir)
+    self.assertEqual(len(files), 1)
+    records = list(
+        tf_record.tf_record_iterator(os.path.join(logdir, files[0])))
+    self.assertEqual(len(records), 2)
+    event = event_pb2.Event()
+    event.ParseFromString(records[1])
+    self.assertEqual(event.summary.value[0].simple_value, 37.0)
+
   def testWeightedMean(self):
     m = metrics.Mean()
     m([1, 100, 100000], weights=[1, 0.2, 0.3])
-- 
GitLab


From 57023e1b6c7d27e76d6f41d80b95402b9d93c467 Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Mon, 23 Oct 2017 15:59:39 -0700
Subject: [PATCH 698/909] tf.constant takes numpy dtypes in eager mode as well

PiperOrigin-RevId: 173184568
---
 tensorflow/python/eager/tensor_test.py     | 5 +++++
 tensorflow/python/framework/constant_op.py | 5 ++++-
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/eager/tensor_test.py b/tensorflow/python/eager/tensor_test.py
index e31c03c08d..b52bbe44d4 100644
--- a/tensorflow/python/eager/tensor_test.py
+++ b/tensorflow/python/eager/tensor_test.py
@@ -23,6 +23,7 @@ import numpy as np
 from tensorflow.python.eager import context
 from tensorflow.python.eager import core
 from tensorflow.python.eager import test
+from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util
@@ -102,6 +103,10 @@ class TFETensorTest(test_util.TensorFlowTestCase):
     t = _create_tensor(n)
     self.assertAllEqual([[1, 2], [3, 4]], t)
 
+  def testConstantDtype(self):
+    self.assertEqual(constant_op.constant(1.0, dtype=np.int64).dtype,
+                     dtypes.int64)
+
   def testTensorAndNumpyMatrix(self):
     expected = np.array([[1.0, 2.0], [3.0, 4.0]], np.float32)
     actual = _create_tensor([[1.0, 2.0], [3.0, 4.0]])
diff --git a/tensorflow/python/framework/constant_op.py b/tensorflow/python/framework/constant_op.py
index 34848af53b..d51e142da1 100644
--- a/tensorflow/python/framework/constant_op.py
+++ b/tensorflow/python/framework/constant_op.py
@@ -108,7 +108,10 @@ def convert_to_eager_tensor(value, ctx, dtype=None):
           dtype, value.dtype))
     return value
   if dtype is not None:
-    dtype = dtype.as_datatype_enum
+    try:
+      dtype = dtype.as_datatype_enum
+    except AttributeError:
+      dtype = dtypes.as_dtype(dtype).as_datatype_enum
   device = ctx.device_name
   handle = ctx._handle  # pylint: disable=protected-access
   if isinstance(value, (float,) + six.integer_types):
-- 
GitLab


From 6bcc00668094be8daf8465b8689bbed5ab285b2d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 23 Oct 2017 16:24:08 -0700
Subject: [PATCH 699/909] Fix error in topk heap launch code. It assumed
 sizeof(struct) == sizeof(struct components).

PiperOrigin-RevId: 173188044
---
 tensorflow/core/kernels/topk_op_gpu.cu.cc      | 2 +-
 tensorflow/python/kernel_tests/topk_op_test.py | 7 +++++++
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/kernels/topk_op_gpu.cu.cc b/tensorflow/core/kernels/topk_op_gpu.cu.cc
index 10a7602dc4..ca296d5aa0 100644
--- a/tensorflow/core/kernels/topk_op_gpu.cu.cc
+++ b/tensorflow/core/kernels/topk_op_gpu.cu.cc
@@ -379,7 +379,7 @@ cudaError LaunchTopKKernel(const cudaStream_t& stream, int num_shards,
   // Use as many shards as possible.
   if (num_shards <= 0) {
     constexpr auto shared_memory_size = 48 << 10;  // 48 KB
-    const auto heap_size = k * (sizeof(int) + sizeof(T));
+    const auto heap_size = k * sizeof(Entry<T>);
     // shared_memory_size = (num_shards + 1) * heap_size <=>
     num_shards = shared_memory_size / heap_size - 1;
     if (num_shards <= 0) {
diff --git a/tensorflow/python/kernel_tests/topk_op_test.py b/tensorflow/python/kernel_tests/topk_op_test.py
index a8e7799cab..efb5b9f364 100644
--- a/tensorflow/python/kernel_tests/topk_op_test.py
+++ b/tensorflow/python/kernel_tests/topk_op_test.py
@@ -100,6 +100,13 @@ class TopKTest(test.TestCase):
     inputs = [[0.1, 0.3, 0.2, 0.4], [0.1, 0.3, 0.4, 0.2]]
     self._validateTopK(inputs, 2, [[0.4, 0.3], [0.4, 0.3]], [[3, 1], [2, 1]])
 
+  def testTop3(self):
+    k = 5
+    inputs = np.random.permutation(np.linspace(0, 100, 6140, dtype=np.float64))
+    indices = np.argsort(-inputs)[:k]
+    values = -np.sort(-inputs)[:k]
+    self._validateTopK(inputs, k, values, indices)
+
   def _testLargeSort(self, dtype):
     b = 10
     n = 5000
-- 
GitLab


From ba1c7b8c67a09ea7af321860ef73a203bf27567a Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Mon, 23 Oct 2017 16:31:11 -0700
Subject: [PATCH 700/909] tf.confusion_matrix works with eager execution
 enabled.

PiperOrigin-RevId: 173188867
---
 tensorflow/python/kernel_tests/confusion_matrix_test.py | 6 ++++--
 tensorflow/python/ops/check_ops.py                      | 6 +++++-
 tensorflow/python/ops/control_flow_ops.py               | 3 +++
 3 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/kernel_tests/confusion_matrix_test.py b/tensorflow/python/kernel_tests/confusion_matrix_test.py
index 2f56540a31..670a625f0f 100644
--- a/tensorflow/python/kernel_tests/confusion_matrix_test.py
+++ b/tensorflow/python/kernel_tests/confusion_matrix_test.py
@@ -23,6 +23,7 @@ import numpy as np
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors_impl
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import confusion_matrix
 from tensorflow.python.ops import math_ops
@@ -32,6 +33,7 @@ from tensorflow.python.platform import test
 
 class ConfusionMatrixTest(test.TestCase):
 
+  @test_util.run_in_graph_and_eager_modes()
   def testExample(self):
     """This is a test of the example provided in pydoc."""
     with self.test_session():
@@ -41,8 +43,8 @@ class ConfusionMatrixTest(test.TestCase):
           [0, 0, 1, 0, 0],
           [0, 0, 0, 0, 0],
           [0, 0, 0, 0, 1]
-      ], confusion_matrix.confusion_matrix(
-          labels=[1, 2, 4], predictions=[2, 2, 4]).eval())
+      ], self.evaluate(confusion_matrix.confusion_matrix(
+          labels=[1, 2, 4], predictions=[2, 2, 4])))
 
   def _testConfMatrix(self, labels, predictions, truth, weights=None,
                       num_classes=None):
diff --git a/tensorflow/python/ops/check_ops.py b/tensorflow/python/ops/check_ops.py
index fb48175285..ceee009104 100644
--- a/tensorflow/python/ops/check_ops.py
+++ b/tensorflow/python/ops/check_ops.py
@@ -229,10 +229,14 @@ def assert_non_negative(x, data=None, summarize=None, message=None, name=None):
   with ops.name_scope(name, 'assert_non_negative', [x, data]):
     x = ops.convert_to_tensor(x, name='x')
     if data is None:
+      if context.in_eager_mode():
+        name = str(x)
+      else:
+        name = x.name
       data = [
           message,
           'Condition x >= 0 did not hold element-wise:',
-          'x (%s) = ' % x.name, x]
+          'x (%s) = ' % name, x]
     zero = ops.convert_to_tensor(0, dtype=x.dtype)
     return assert_less_equal(zero, x, data=data, summarize=summarize)
 
diff --git a/tensorflow/python/ops/control_flow_ops.py b/tensorflow/python/ops/control_flow_ops.py
index 29aac913f0..f584d93aa2 100644
--- a/tensorflow/python/ops/control_flow_ops.py
+++ b/tensorflow/python/ops/control_flow_ops.py
@@ -116,6 +116,7 @@ def Assert(condition, data, summarize=None, name=None):
   Returns:
     assert_op: An `Operation` that, when executed, raises a
     `tf.errors.InvalidArgumentError` if `condition` is not true.
+    @compatibility{eager} returns None.
   """
   with ops.name_scope(name, "Assert", [condition, data]) as name:
     xs = ops.convert_n_to_tensor(data)
@@ -132,6 +133,8 @@ def Assert(condition, data, summarize=None, name=None):
             condition, data, summarize, name="Assert")
       guarded_assert = cond(
           condition, no_op, true_assert, name="AssertGuard")
+      if context.in_eager_mode():
+        return
       return guarded_assert.op
 
 
-- 
GitLab


From 4f35cce3ed24455d30c46132ce3202db66009b31 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 23 Oct 2017 16:52:56 -0700
Subject: [PATCH 701/909] Adds the ability to adjust the normalized value in
 batchnorm. By using random adjustments, it is possible to improve the model
 generalization.

PiperOrigin-RevId: 173192179
---
 tensorflow/python/layers/normalization.py     |  96 ++++++++++++----
 .../python/layers/normalization_test.py       | 106 ++++++++++++++++++
 ...nsorflow.layers.-batch-normalization.pbtxt |   2 +-
 .../tools/api/golden/tensorflow.layers.pbtxt  |   2 +-
 4 files changed, 180 insertions(+), 26 deletions(-)

diff --git a/tensorflow/python/layers/normalization.py b/tensorflow/python/layers/normalization.py
index df2b97f03e..74246189b5 100644
--- a/tensorflow/python/layers/normalization.py
+++ b/tensorflow/python/layers/normalization.py
@@ -101,6 +101,18 @@ class BatchNormalization(base.Layer):
       Normalization", which creates virtual sub-batches which are each
       normalized separately (with shared gamma, beta, and moving statistics).
       Must divide the actual batch size during execution.
+    adjustment: A function taking the `Tensor` containing the (dynamic) shape of
+      the input tensor and returning a pair (scale, bias) to apply to the
+      normalized values (before gamma and beta), only during training. For
+      example, if axis==-1,
+        `adjustment = lambda shape: (
+          tf.random_uniform(shape[-1:], 0.93, 1.07),
+          tf.random_uniform(shape[-1:], -0.1, 0.1))`
+      will scale the normalized value by up to 7% up or down, then shift the
+      result by up to 0.1 (with independent scaling and bias for each feature
+      but shared across all examples), and finally apply gamma and/or beta. If
+      `None`, no adjustment is applied. Cannot be specified if
+      virtual_batch_size is specified.
     name: A string, the name of the layer.
   """
 
@@ -124,6 +136,7 @@ class BatchNormalization(base.Layer):
                fused=None,
                trainable=True,
                virtual_batch_size=None,
+               adjustment=None,
                name=None,
                **kwargs):
     super(BatchNormalization, self).__init__(
@@ -143,6 +156,7 @@ class BatchNormalization(base.Layer):
     self.gamma_constraint = gamma_constraint
     self.renorm = renorm
     self.virtual_batch_size = virtual_batch_size
+    self.adjustment = adjustment
     if fused is None:
       fused = True
 
@@ -192,6 +206,9 @@ class BatchNormalization(base.Layer):
       if 0 in self.axis:
         raise ValueError('When using virtual_batch_size, the batch dimension '
                          'must be 0 and thus axis cannot include 0')
+      if self.adjustment is not None:
+        raise ValueError('When using virtual_batch_size, adjustment cannot '
+                         'be specified')
 
     if self.fused:
       # Currently fused batch norm doesn't support renorm and beta/gamma
@@ -204,7 +221,8 @@ class BatchNormalization(base.Layer):
                     self.axis in [[1], [3]] and
                     self.beta_regularizer is None and
                     self.gamma_regularizer is None and
-                    self.virtual_batch_size is None)
+                    self.virtual_batch_size is None and
+                    self.adjustment is None)
       # TODO(chrisying): fused batch norm is currently not supported for
       # multi-axis batch norm and by extension virtual batches. In some cases,
       # it might be possible to use fused batch norm but would require reshaping
@@ -482,11 +500,41 @@ class BatchNormalization(base.Layer):
     if self.virtual_batch_size is not None:
       del reduction_axes[1]     # Do not reduce along virtual batch dim
 
-    scale, offset = self.gamma, self.beta
+    # Broadcasting only necessary for single-axis batch norm where the axis is
+    # not the last dimension
+    broadcast_shape = [1] * ndims
+    broadcast_shape[self.axis[0]] = input_shape[self.axis[0]].value
+    def _broadcast(v):
+      if (v is not None and
+          len(v.get_shape()) != ndims and
+          reduction_axes != list(range(ndims - 1))):
+        return array_ops.reshape(v, broadcast_shape)
+      return v
+
+    scale, offset = _broadcast(self.gamma), _broadcast(self.beta)
+
+    def _compose_transforms(scale, offset, then_scale, then_offset):
+      if then_scale is not None:
+        scale *= then_scale
+        offset *= then_scale
+      if then_offset is not None:
+        offset += then_offset
+      return (scale, offset)
 
     # Determine a boolean value for `training`: could be True, False, or None.
     training_value = utils.constant_value(training)
     if training_value is not False:
+      if self.adjustment:
+        adj_scale, adj_bias = self.adjustment(array_ops.shape(inputs))
+        # Adjust only during training.
+        adj_scale = utils.smart_cond(training,
+                                     lambda: adj_scale,
+                                     lambda: array_ops.ones_like(adj_scale))
+        adj_bias = utils.smart_cond(training,
+                                    lambda: adj_bias,
+                                    lambda: array_ops.zeros_like(adj_bias))
+        scale, offset = _compose_transforms(adj_scale, adj_bias, scale, offset)
+
       # Some of the computations here are not necessary when training==False
       # but not a constant. However, this makes the code simpler.
       keep_dims = self.virtual_batch_size is not None or len(self.axis) > 1
@@ -508,13 +556,9 @@ class BatchNormalization(base.Layer):
         # When training, the normalized values (say, x) will be transformed as
         # x * gamma + beta without renorm, and (x * r + d) * gamma + beta
         # = x * (r * gamma) + (d * gamma + beta) with renorm.
-        scale = array_ops.stop_gradient(r, name='renorm_r')
-        offset = array_ops.stop_gradient(d, name='renorm_d')
-        if self.gamma is not None:
-          scale *= self.gamma
-          offset *= self.gamma
-        if self.beta is not None:
-          offset += self.beta
+        r = _broadcast(array_ops.stop_gradient(r, name='renorm_r'))
+        d = _broadcast(array_ops.stop_gradient(d, name='renorm_d'))
+        scale, offset = _compose_transforms(r, d, scale, offset)
       else:
         new_mean, new_variance = mean, variance
 
@@ -542,24 +586,14 @@ class BatchNormalization(base.Layer):
     else:
       mean, variance = self.moving_mean, self.moving_variance
 
-    # Broadcasting only necessary for single-axis batch norm where the axis is
-    # not the last dimension
-    broadcast_shape = [1] * ndims
-    broadcast_shape[self.axis[0]] = input_shape[self.axis[0]].value
-    rank = len(inputs.get_shape())
-    def _broadcast(v):
-      if (v is not None and
-          len(v.get_shape()) != rank and
-          reduction_axes != list(range(ndims))[:-1]):
-        return array_ops.reshape(v, broadcast_shape)
-      return v
-
     outputs = nn.batch_normalization(inputs,
                                      _broadcast(mean),
                                      _broadcast(variance),
-                                     _broadcast(offset),
-                                     _broadcast(scale),
+                                     offset,
+                                     scale,
                                      self.epsilon)
+    # If some components of the shape got lost due to adjustments, fix that.
+    outputs.set_shape(input_shape)
 
     if self.virtual_batch_size is not None:
       return undo_virtual_batching(outputs)
@@ -589,7 +623,8 @@ def batch_normalization(inputs,
                         renorm_clipping=None,
                         renorm_momentum=0.99,
                         fused=None,
-                        virtual_batch_size=None):
+                        virtual_batch_size=None,
+                        adjustment=None):
   """Functional interface for the batch normalization layer.
 
   Reference: http://arxiv.org/abs/1502.03167
@@ -667,6 +702,18 @@ def batch_normalization(inputs,
       Normalization", which creates virtual sub-batches which are each
       normalized separately (with shared gamma, beta, and moving statistics).
       Must divide the actual batch size during execution.
+    adjustment: A function taking the `Tensor` containing the (dynamic) shape of
+      the input tensor and returning a pair (scale, bias) to apply to the
+      normalized values (before gamma and beta), only during training. For
+      example, if axis==-1,
+        `adjustment = lambda shape: (
+          tf.random_uniform(shape[-1:], 0.93, 1.07),
+          tf.random_uniform(shape[-1:], -0.1, 0.1))`
+      will scale the normalized value by up to 7% up or down, then shift the
+      result by up to 0.1 (with independent scaling and bias for each feature
+      but shared across all examples), and finally apply gamma and/or beta. If
+      `None`, no adjustment is applied. Cannot be specified if
+      virtual_batch_size is specified.
 
   Returns:
     Output tensor.
@@ -691,6 +738,7 @@ def batch_normalization(inputs,
       fused=fused,
       trainable=trainable,
       virtual_batch_size=virtual_batch_size,
+      adjustment=adjustment,
       name=name,
       _reuse=reuse,
       _scope=name)
diff --git a/tensorflow/python/layers/normalization_test.py b/tensorflow/python/layers/normalization_test.py
index f8d9d2948c..90ebdc8c86 100644
--- a/tensorflow/python/layers/normalization_test.py
+++ b/tensorflow/python/layers/normalization_test.py
@@ -823,6 +823,112 @@ class BNTest(test.TestCase):
         self.assertAllClose(y_train, yt_val_train, atol=1e-5)
         self.assertAllClose(y_test, yt_val_test, atol=1e-5)
 
+  def testAdjustment(self):
+    shape = (4, 3)
+    xt = array_ops.placeholder(dtypes.float32, shape)
+    momentum = 0.99
+    gamma = 2.
+    beta = 3.
+    epsilon = 0.001
+    adjust_scale = random_ops.random_uniform(shape[-1:], 0.5, 1.5)
+    adjust_bias = random_ops.random_uniform(shape[-1:], -.2, .2)
+    bn = normalization_layers.BatchNormalization(
+        axis=1,
+        gamma_initializer=init_ops.constant_initializer(gamma),
+        beta_initializer=init_ops.constant_initializer(beta),
+        epsilon=epsilon,
+        momentum=momentum,
+        adjustment=lambda _: (adjust_scale, adjust_bias))
+    training = array_ops.placeholder(dtypes.bool)
+    yt = bn.apply(xt, training=training)
+
+    moving_mean = 0.
+    moving_variance = 1.
+    with self.test_session(use_gpu=True) as sess:
+      sess.run(variables.global_variables_initializer())
+      for _ in range(5):
+        x = np.random.random(shape)
+        yt_val_train, adj_scale_val, adj_bias_val = sess.run(
+            [yt, adjust_scale, adjust_bias] + bn.updates,
+            feed_dict={xt: x, training: True})[:3]
+        yt_val_test = sess.run([yt] + bn.updates,
+                               feed_dict={xt: x, training: False})[0]
+
+        mean = x.mean(0)
+        variance = x.var(0)
+        y_train = (((x - mean) / (variance + epsilon) ** 0.5) * adj_scale_val +
+                   adj_bias_val) * gamma + beta
+        moving_mean += (mean - moving_mean) * (1. - momentum)
+        moving_variance += (variance - moving_variance) * (1. - momentum)
+
+        y_test = ((x - moving_mean) / (moving_variance + epsilon) ** 0.5 *
+                  gamma) + beta
+
+        self.assertAllClose(y_train, yt_val_train, atol=1e-5)
+        self.assertAllClose(y_test, yt_val_test, atol=1e-5)
+
+  def testRenormWithAdjustment(self):
+    shape = (4, 3)
+    xt = array_ops.placeholder(dtypes.float32, shape)
+    momentum = 0.99
+    renorm_momentum = 0.8
+    rmax = 1.1
+    rmin = 0.9
+    dmax = 0.1
+    gamma = 2.
+    beta = 3.
+    epsilon = 0.001
+    adjust_scale = random_ops.random_uniform(shape[-1:], 0.5, 1.5)
+    adjust_bias = random_ops.random_uniform(shape[-1:], -.2, .2)
+    bn = normalization_layers.BatchNormalization(
+        axis=1,
+        gamma_initializer=init_ops.constant_initializer(gamma),
+        beta_initializer=init_ops.constant_initializer(beta),
+        epsilon=epsilon,
+        momentum=momentum,
+        renorm=True,
+        renorm_clipping={'rmax': rmax, 'rmin': rmin, 'dmax': dmax},
+        renorm_momentum=renorm_momentum,
+        adjustment=lambda _: (adjust_scale, adjust_bias))
+    training = array_ops.placeholder(dtypes.bool)
+    yt = bn.apply(xt, training=training)
+
+    moving_mean = 0.
+    moving_variance = 1.
+    renorm_mean = renorm_stddev = 0.
+    renorm_weight = 0.
+    with self.test_session(use_gpu=True) as sess:
+      sess.run(variables.global_variables_initializer())
+      for _ in range(5):
+        x = np.random.random(shape)
+        yt_val_train, adj_scale_val, adj_bias_val = sess.run(
+            [yt, adjust_scale, adjust_bias] + bn.updates,
+            feed_dict={xt: x, training: True})[:3]
+        yt_val_test = sess.run([yt] + bn.updates,
+                               feed_dict={xt: x, training: False})[0]
+
+        mean = x.mean(0)
+        stddev = np.sqrt(x.var(0) + epsilon)
+        adj_mean = renorm_mean + (1. - renorm_weight) * mean
+        adj_stddev = renorm_stddev + (1. - renorm_weight) * stddev
+        r = (stddev / adj_stddev).clip(rmin, rmax)
+        d = ((mean - adj_mean) / adj_stddev).clip(-dmax, dmax)
+        y_train = (((x - mean) / stddev * r + d) * adj_scale_val +
+                   adj_bias_val) * gamma + beta
+        renorm_mean += (mean - renorm_mean) * (1. - renorm_momentum)
+        renorm_stddev += (stddev - renorm_stddev) * (1. - renorm_momentum)
+        renorm_weight += (1. - renorm_weight) * (1. - renorm_momentum)
+        moving_mean += (renorm_mean / renorm_weight -
+                        moving_mean) * (1. - momentum)
+        moving_variance += ((renorm_stddev / renorm_weight) ** 2 - epsilon -
+                            moving_variance) * (1. - momentum)
+
+        y_test = ((x - moving_mean) / (moving_variance + epsilon) ** 0.5 *
+                  gamma) + beta
+
+        self.assertAllClose(y_train, yt_val_train, atol=1e-5)
+        self.assertAllClose(y_test, yt_val_test, atol=1e-5)
+
   def testGhostBNNegativeVirtualBatch(self):
     shape = [6, 5, 4, 3]
     inp = random_ops.random_uniform(shape, seed=1)
diff --git a/tensorflow/tools/api/golden/tensorflow.layers.-batch-normalization.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.-batch-normalization.pbtxt
index c66af13850..6e07b911a4 100644
--- a/tensorflow/tools/api/golden/tensorflow.layers.-batch-normalization.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.layers.-batch-normalization.pbtxt
@@ -81,7 +81,7 @@ tf_class {
   }
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'axis\', \'momentum\', \'epsilon\', \'center\', \'scale\', \'beta_initializer\', \'gamma_initializer\', \'moving_mean_initializer\', \'moving_variance_initializer\', \'beta_regularizer\', \'gamma_regularizer\', \'beta_constraint\', \'gamma_constraint\', \'renorm\', \'renorm_clipping\', \'renorm_momentum\', \'fused\', \'trainable\', \'virtual_batch_size\', \'name\'], varargs=None, keywords=kwargs, defaults=[\'-1\', \'0.99\', \'0.001\', \'True\', \'True\', \'<tensorflow.python.ops.init_ops.Zeros object instance>\', \'<tensorflow.python.ops.init_ops.Ones object instance>\', \'<tensorflow.python.ops.init_ops.Zeros object instance>\', \'<tensorflow.python.ops.init_ops.Ones object instance>\', \'None\', \'None\', \'None\', \'None\', \'False\', \'None\', \'0.99\', \'None\', \'True\', \'None\', \'None\'], "
+    argspec: "args=[\'self\', \'axis\', \'momentum\', \'epsilon\', \'center\', \'scale\', \'beta_initializer\', \'gamma_initializer\', \'moving_mean_initializer\', \'moving_variance_initializer\', \'beta_regularizer\', \'gamma_regularizer\', \'beta_constraint\', \'gamma_constraint\', \'renorm\', \'renorm_clipping\', \'renorm_momentum\', \'fused\', \'trainable\', \'virtual_batch_size\', \'adjustment\', \'name\'], varargs=None, keywords=kwargs, defaults=[\'-1\', \'0.99\', \'0.001\', \'True\', \'True\', \'<tensorflow.python.ops.init_ops.Zeros object instance>\', \'<tensorflow.python.ops.init_ops.Ones object instance>\', \'<tensorflow.python.ops.init_ops.Zeros object instance>\', \'<tensorflow.python.ops.init_ops.Ones object instance>\', \'None\', \'None\', \'None\', \'None\', \'False\', \'None\', \'0.99\', \'None\', \'True\', \'None\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_loss"
diff --git a/tensorflow/tools/api/golden/tensorflow.layers.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.pbtxt
index dad514b534..c45d6e6c05 100644
--- a/tensorflow/tools/api/golden/tensorflow.layers.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.layers.pbtxt
@@ -90,7 +90,7 @@ tf_module {
   }
   member_method {
     name: "batch_normalization"
-    argspec: "args=[\'inputs\', \'axis\', \'momentum\', \'epsilon\', \'center\', \'scale\', \'beta_initializer\', \'gamma_initializer\', \'moving_mean_initializer\', \'moving_variance_initializer\', \'beta_regularizer\', \'gamma_regularizer\', \'beta_constraint\', \'gamma_constraint\', \'training\', \'trainable\', \'name\', \'reuse\', \'renorm\', \'renorm_clipping\', \'renorm_momentum\', \'fused\', \'virtual_batch_size\'], varargs=None, keywords=None, defaults=[\'-1\', \'0.99\', \'0.001\', \'True\', \'True\', \'<tensorflow.python.ops.init_ops.Zeros object instance>\', \'<tensorflow.python.ops.init_ops.Ones object instance>\', \'<tensorflow.python.ops.init_ops.Zeros object instance>\', \'<tensorflow.python.ops.init_ops.Ones object instance>\', \'None\', \'None\', \'None\', \'None\', \'False\', \'True\', \'None\', \'None\', \'False\', \'None\', \'0.99\', \'None\', \'None\'], "
+    argspec: "args=[\'inputs\', \'axis\', \'momentum\', \'epsilon\', \'center\', \'scale\', \'beta_initializer\', \'gamma_initializer\', \'moving_mean_initializer\', \'moving_variance_initializer\', \'beta_regularizer\', \'gamma_regularizer\', \'beta_constraint\', \'gamma_constraint\', \'training\', \'trainable\', \'name\', \'reuse\', \'renorm\', \'renorm_clipping\', \'renorm_momentum\', \'fused\', \'virtual_batch_size\', \'adjustment\'], varargs=None, keywords=None, defaults=[\'-1\', \'0.99\', \'0.001\', \'True\', \'True\', \'<tensorflow.python.ops.init_ops.Zeros object instance>\', \'<tensorflow.python.ops.init_ops.Ones object instance>\', \'<tensorflow.python.ops.init_ops.Zeros object instance>\', \'<tensorflow.python.ops.init_ops.Ones object instance>\', \'None\', \'None\', \'None\', \'None\', \'False\', \'True\', \'None\', \'None\', \'False\', \'None\', \'0.99\', \'None\', \'None\', \'None\'], "
   }
   member_method {
     name: "conv1d"
-- 
GitLab


From ef3964f8ba27723c8db48af4f10801d5b7432db9 Mon Sep 17 00:00:00 2001
From: Eugene Brevdo <ebrevdo@google.com>
Date: Mon, 23 Oct 2017 17:04:42 -0700
Subject: [PATCH 702/909] Make BasicLSTMCell build its variables like a proper
 tf.layers.Layer.

PiperOrigin-RevId: 173193748
---
 tensorflow/python/ops/rnn_cell_impl.py        | 108 ++++++++++++++----
 ...flow.nn.rnn_cell.-basic-l-s-t-m-cell.pbtxt |   5 +-
 2 files changed, 89 insertions(+), 24 deletions(-)

diff --git a/tensorflow/python/ops/rnn_cell_impl.py b/tensorflow/python/ops/rnn_cell_impl.py
index fb7b6d11a5..65b0407008 100644
--- a/tensorflow/python/ops/rnn_cell_impl.py
+++ b/tensorflow/python/ops/rnn_cell_impl.py
@@ -159,17 +159,17 @@ class RNNCell(base_layer.Layer):
     """Run this RNN cell on inputs, starting from the given state.
 
     Args:
-      inputs: `2-D` tensor with shape `[batch_size x input_size]`.
+      inputs: `2-D` tensor with shape `[batch_size, input_size]`.
       state: if `self.state_size` is an integer, this should be a `2-D Tensor`
-        with shape `[batch_size x self.state_size]`.  Otherwise, if
+        with shape `[batch_size, self.state_size]`.  Otherwise, if
         `self.state_size` is a tuple of integers, this should be a tuple
-        with shapes `[batch_size x s] for s in self.state_size`.
+        with shapes `[batch_size, s] for s in self.state_size`.
       scope: VariableScope for the created subgraph; defaults to class name.
 
     Returns:
       A pair containing:
 
-      - Output: A `2-D` tensor with shape `[batch_size x self.output_size]`.
+      - Output: A `2-D` tensor with shape `[batch_size, self.output_size]`.
       - New state: Either a single `2-D` tensor, or a tuple of tensors matching
         the arity and shapes of `state`.
     """
@@ -229,11 +229,11 @@ class RNNCell(base_layer.Layer):
 
     Returns:
       If `state_size` is an int or TensorShape, then the return value is a
-      `N-D` tensor of shape `[batch_size x state_size]` filled with zeros.
+      `N-D` tensor of shape `[batch_size, state_size]` filled with zeros.
 
       If `state_size` is a nested list or tuple, then the return value is
       a nested list or tuple (of the same structure) of `2-D` tensors with
-      the shapes `[batch_size x s]` for each s in `state_size`.
+      the shapes `[batch_size, s]` for each s in `state_size`.
     """
     # Try to use the last cached zero_state. This is done to avoid recreating
     # zeros, especially when eager execution is enabled.
@@ -285,6 +285,45 @@ class BasicRNNCell(RNNCell):
     return output, output
 
 
+class _LayerRNNCell(RNNCell):
+  """Subclass of RNNCells that act like proper `tf.Layer` objects.
+
+  For backwards compatibility purposes, most `RNNCell` instances allow their
+  `call` methods to instantiate variables via `tf.get_variable`.  The underlying
+  variable scope thus keeps track of any variables, and returning cached
+  versions.  This is atypical of `tf.layer` objects, which separate this
+  part of layer building into a `build` method that is only called once.
+
+  Here we provide a subclass for `RNNCell` objects that act exactly as
+  `Layer` objects do.  They must provide a `build` method and their
+  `call` methods do not access Variables `tf.get_variable`.
+  """
+
+  def __call__(self, inputs, state, scope=None):
+    """Run this RNN cell on inputs, starting from the given state.
+
+    Args:
+      inputs: `2-D` tensor with shape `[batch_size, input_size]`.
+      state: if `self.state_size` is an integer, this should be a `2-D Tensor`
+        with shape `[batch_size, self.state_size]`.  Otherwise, if
+        `self.state_size` is a tuple of integers, this should be a tuple
+        with shapes `[batch_size, s] for s in self.state_size`.
+      scope: `VariableScope` for the created subgraph; if not provided,
+        defaults to standard `tf.layers.Layer` behavior.
+
+    Returns:
+      A pair containing:
+
+      - Output: A `2-D` tensor with shape `[batch_size, self.output_size]`.
+      - New state: Either a single `2-D` tensor, or a tuple of tensors matching
+        the arity and shapes of `state`.
+    """
+    # Bypass RNNCell's variable capturing semantics for LayerRNNCell.
+    # Instead, it is up to subclasses to provide a proper build
+    # method.  See the class docstring for more details.
+    return base_layer.Layer.__call__(self, inputs, state, scope=scope)
+
+
 class GRUCell(RNNCell):
   """Gated Recurrent Unit cell (cf. http://arxiv.org/abs/1406.1078).
 
@@ -374,7 +413,7 @@ class LSTMStateTuple(_LSTMStateTuple):
     return c.dtype
 
 
-class BasicLSTMCell(RNNCell):
+class BasicLSTMCell(_LayerRNNCell):
   """Basic LSTM recurrent network cell.
 
   The implementation is based on: http://arxiv.org/abs/1409.2329.
@@ -390,7 +429,7 @@ class BasicLSTMCell(RNNCell):
   """
 
   def __init__(self, num_units, forget_bias=1.0,
-               state_is_tuple=True, activation=None, reuse=None):
+               state_is_tuple=True, activation=None, reuse=None, name=None):
     """Initialize the basic LSTM cell.
 
     Args:
@@ -405,11 +444,14 @@ class BasicLSTMCell(RNNCell):
       reuse: (optional) Python boolean describing whether to reuse variables
         in an existing scope.  If not `True`, and the existing scope already has
         the given variables, an error is raised.
+      name: String, the name of the layer. Layers with the same name will
+        share weights, but to avoid mistakes we require reuse=True in such
+        cases.
 
       When restoring from CudnnLSTM-trained checkpoints, must use
-      CudnnCompatibleLSTMCell instead.
+      `CudnnCompatibleLSTMCell` instead.
     """
-    super(BasicLSTMCell, self).__init__(_reuse=reuse)
+    super(BasicLSTMCell, self).__init__(_reuse=reuse, name=name)
     if not state_is_tuple:
       logging.warn("%s: Using a concatenated state is slower and will soon be "
                    "deprecated.  Use state_is_tuple=True.", self)
@@ -428,15 +470,35 @@ class BasicLSTMCell(RNNCell):
   def output_size(self):
     return self._num_units
 
+  def build(self, inputs_shape):
+    if inputs_shape.ndims != 2:
+      raise ValueError("Expected inputs.shape to be rank 2, saw shape: %s"
+                       % inputs_shape)
+    if inputs_shape[1].value is None:
+      raise ValueError("Expected inputs.shape[-1] to be known, saw shape: %s"
+                       % inputs_shape)
+
+    input_depth = inputs_shape[1].value
+    h_depth = self._num_units
+    self._kernel = self.add_variable(
+        _WEIGHTS_VARIABLE_NAME,
+        shape=[input_depth + h_depth, 4 * self._num_units])
+    self._bias = self.add_variable(
+        _BIAS_VARIABLE_NAME,
+        shape=[4 * self._num_units],
+        initializer=init_ops.constant_initializer(0.0, dtype=self.dtype))
+
+    self._built = True
+
   def call(self, inputs, state):
     """Long short-term memory cell (LSTM).
 
     Args:
-      inputs: `2-D` tensor with shape `[batch_size x input_size]`.
+      inputs: `2-D` tensor with shape `[batch_size, input_size]`.
       state: An `LSTMStateTuple` of state tensors, each shaped
-        `[batch_size x self.state_size]`, if `state_is_tuple` has been set to
+        `[batch_size, self.state_size]`, if `state_is_tuple` has been set to
         `True`.  Otherwise, a `Tensor` shaped
-        `[batch_size x 2 * self.state_size]`.
+        `[batch_size, 2 * self.state_size]`.
 
     Returns:
       A pair containing the new hidden state, and the new state (either a
@@ -451,11 +513,13 @@ class BasicLSTMCell(RNNCell):
     else:
       c, h = array_ops.split(value=state, num_or_size_splits=2, axis=one)
 
-    if self._linear is None:
-      self._linear = _Linear([inputs, h], 4 * self._num_units, True)
+    gate_inputs = math_ops.matmul(
+        array_ops.concat([inputs, h], 1), self._kernel)
+    gate_inputs = nn_ops.bias_add(gate_inputs, self._bias)
+
     # i = input_gate, j = new_input, f = forget_gate, o = output_gate
     i, j, f, o = array_ops.split(
-        value=self._linear([inputs, h]), num_or_size_splits=4, axis=one)
+        value=gate_inputs, num_or_size_splits=4, axis=one)
 
     forget_bias_tensor = constant_op.constant(self._forget_bias, dtype=f.dtype)
     # Note that using `add` and `multiply` instead of `+` and `*` gives a
@@ -585,16 +649,16 @@ class LSTMCell(RNNCell):
     """Run one step of LSTM.
 
     Args:
-      inputs: input Tensor, 2D, batch x num_units.
+      inputs: input Tensor, 2D, `[batch, num_units].
       state: if `state_is_tuple` is False, this must be a state Tensor,
-        `2-D, batch x state_size`.  If `state_is_tuple` is True, this must be a
+        `2-D, [batch, state_size]`.  If `state_is_tuple` is True, this must be a
         tuple of state Tensors, both `2-D`, with column sizes `c_state` and
         `m_state`.
 
     Returns:
       A tuple containing:
 
-      - A `2-D, [batch x output_dim]`, Tensor representing the output of the
+      - A `2-D, [batch, output_dim]`, Tensor representing the output of the
         LSTM after reading `inputs` when previous state was `state`.
         Here output_dim is:
            num_proj if num_proj was set,
@@ -1143,7 +1207,7 @@ class _Linear(object):
   """Linear map: sum_i(args[i] * W[i]), where W[i] is a variable.
 
   Args:
-    args: a 2D Tensor or a list of 2D, batch x n, Tensors.
+    args: a 2D Tensor or a list of 2D, batch, n, Tensors.
     output_size: int, second dimension of weight variable.
     dtype: data type for variables.
     build_bias: boolean, whether to build a bias variable.
@@ -1225,7 +1289,7 @@ def _linear(args,
   """Linear map: sum_i(args[i] * W[i]), where W[i] is a variable.
 
   Args:
-    args: a 2D Tensor or a list of 2D, batch x n, Tensors.
+    args: a 2D Tensor or a list of 2D, batch, n, Tensors.
     output_size: int, second dimension of W[i].
     bias: boolean, whether to add a bias term or not.
     bias_initializer: starting value to initialize the bias
@@ -1233,7 +1297,7 @@ def _linear(args,
     kernel_initializer: starting value to initialize the weight.
 
   Returns:
-    A 2D Tensor with shape [batch x output_size] equal to
+    A 2D Tensor with shape `[batch, output_size]` equal to
     sum_i(args[i] * W[i]), where W[i]s are newly created matrices.
 
   Raises:
diff --git a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-basic-l-s-t-m-cell.pbtxt b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-basic-l-s-t-m-cell.pbtxt
index 3254a62af1..b8e27cc6cb 100644
--- a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-basic-l-s-t-m-cell.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-basic-l-s-t-m-cell.pbtxt
@@ -1,6 +1,7 @@
 path: "tensorflow.nn.rnn_cell.BasicLSTMCell"
 tf_class {
   is_instance: "<class \'tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell\'>"
+  is_instance: "<class \'tensorflow.python.ops.rnn_cell_impl._LayerRNNCell\'>"
   is_instance: "<class \'tensorflow.python.ops.rnn_cell_impl.RNNCell\'>"
   is_instance: "<class \'tensorflow.python.layers.base.Layer\'>"
   is_instance: "<type \'object\'>"
@@ -90,7 +91,7 @@ tf_class {
   }
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'num_units\', \'forget_bias\', \'state_is_tuple\', \'activation\', \'reuse\'], varargs=None, keywords=None, defaults=[\'1.0\', \'True\', \'None\', \'None\'], "
+    argspec: "args=[\'self\', \'num_units\', \'forget_bias\', \'state_is_tuple\', \'activation\', \'reuse\', \'name\'], varargs=None, keywords=None, defaults=[\'1.0\', \'True\', \'None\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_loss"
@@ -110,7 +111,7 @@ tf_class {
   }
   member_method {
     name: "build"
-    argspec: "args=[\'self\', \'_\'], varargs=None, keywords=None, defaults=None"
+    argspec: "args=[\'self\', \'inputs_shape\'], varargs=None, keywords=None, defaults=None"
   }
   member_method {
     name: "call"
-- 
GitLab


From f1c6f5688de0a6fb3b8f016a7232c293b64689da Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Mon, 23 Oct 2017 17:07:22 -0700
Subject: [PATCH 703/909] EagerTensor.__copy__ and __deepcopy__

PiperOrigin-RevId: 173194115
---
 tensorflow/python/eager/tensor_test.py | 12 ++++++++++++
 tensorflow/python/framework/ops.py     |  9 +++++++++
 2 files changed, 21 insertions(+)

diff --git a/tensorflow/python/eager/tensor_test.py b/tensorflow/python/eager/tensor_test.py
index b52bbe44d4..2b7b5c727a 100644
--- a/tensorflow/python/eager/tensor_test.py
+++ b/tensorflow/python/eager/tensor_test.py
@@ -18,6 +18,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import copy
+
 import numpy as np
 
 from tensorflow.python.eager import context
@@ -103,6 +105,16 @@ class TFETensorTest(test_util.TensorFlowTestCase):
     t = _create_tensor(n)
     self.assertAllEqual([[1, 2], [3, 4]], t)
 
+  def testCopy(self):
+    t = constant_op.constant(1.0)
+    tt = copy.copy(t)
+    self.assertAllEqual(tt, 1.0)
+    del tt
+    tt = copy.deepcopy(t)
+    self.assertAllEqual(tt, 1.0)
+    del tt
+    self.assertAllEqual(t, 1.0)
+
   def testConstantDtype(self):
     self.assertEqual(constant_op.constant(1.0, dtype=np.int64).dtype,
                      dtypes.int64)
diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index b45cb2e0c6..b3caebce70 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -624,6 +624,15 @@ class _EagerTensorBase(Tensor):
   def _numpy(self):
     raise NotImplementedError()
 
+  def __copy__(self):
+    # Eager Tensors are immutable so it's safe to return themselves as a copy.
+    return self
+
+  def __deepcopy__(self, memo):
+    # Eager Tensors are immutable so it's safe to return themselves as a copy.
+    del memo
+    return self
+
   def _datatype_enum(self):
     raise NotImplementedError()
 
-- 
GitLab


From d75d5529d569d8f72cb215d3696db1feb1d9f033 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 23 Oct 2017 18:49:48 -0700
Subject: [PATCH 704/909] Revert to 64-bit indexing in
 extract_image_patches_op.h if the input/output tensor have more than 2^32
 elements.

PiperOrigin-RevId: 173203430
---
 .../core/kernels/extract_image_patches_op.h   | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/tensorflow/core/kernels/extract_image_patches_op.h b/tensorflow/core/kernels/extract_image_patches_op.h
index 2abbed15e5..9d34daca64 100644
--- a/tensorflow/core/kernels/extract_image_patches_op.h
+++ b/tensorflow/core/kernels/extract_image_patches_op.h
@@ -32,11 +32,20 @@ struct ExtractImagePatchesForward {
                   typename TTypes<T, 4>::Tensor output) {
     // Need to swap row/col when calling Eigen, because our data is in
     // NHWC format while Eigen assumes NWHC format.
-    To32Bit(output).device(d) =
-        To32Bit(input)
-            .extract_image_patches(patch_cols, patch_rows, stride_cols,
-                                   stride_rows, rate_cols, rate_rows, padding)
-            .reshape(output.dimensions());
+    const int64 N = std::max(input.size(), output.size());
+    if (N <= std::numeric_limits<Index32>::max()) {
+      To32Bit(output).device(d) =
+          To32Bit(input)
+              .extract_image_patches(patch_cols, patch_rows, stride_cols,
+                                     stride_rows, rate_cols, rate_rows, padding)
+              .reshape(output.dimensions());
+    } else {
+      output.device(d) =
+          input
+              .extract_image_patches(patch_cols, patch_rows, stride_cols,
+                                     stride_rows, rate_cols, rate_rows, padding)
+              .reshape(output.dimensions());
+    }
   }
 };
 
-- 
GitLab


From 1b46f888f28d67e52e4f40393d39410c74cfbb58 Mon Sep 17 00:00:00 2001
From: Sarah Maddox <sarahmaddox@google.com>
Date: Tue, 24 Oct 2017 14:01:39 +1100
Subject: [PATCH 705/909] Standardised caps on Virtualenv.

---
 tensorflow/docs_src/install/install_linux.md | 36 ++++++++++----------
 tensorflow/docs_src/install/install_mac.md   | 30 ++++++++--------
 2 files changed, 33 insertions(+), 33 deletions(-)

diff --git a/tensorflow/docs_src/install/install_linux.md b/tensorflow/docs_src/install/install_linux.md
index 9d204cc246..b641f403f5 100644
--- a/tensorflow/docs_src/install/install_linux.md
+++ b/tensorflow/docs_src/install/install_linux.md
@@ -77,22 +77,22 @@ TensorFlow with GPU support, but only if you do the following:
 You must pick the mechanism by which you install TensorFlow. The
 supported choices are as follows:
 
-  * [virtualenv](#InstallingVirtualenv)
+  * [Virtualenv](#InstallingVirtualenv)
   * ["native" pip](#InstallingNativePip)
   * [Docker](#InstallingDocker)
   * [Anaconda](#InstallingAnaconda)
   * installing from sources, which is documented in
     [a separate guide](https://www.tensorflow.org/install/install_sources).
 
-**We recommend the virtualenv installation.**
+**We recommend the Virtualenv installation.**
 [Virtualenv](https://virtualenv.pypa.io/en/stable/)
 is a virtual Python environment isolated from other Python development,
 incapable of interfering with or being affected by other Python programs
-on the same machine.  During the virtualenv installation process,
+on the same machine.  During the Virtualenv installation process,
 you will install not only TensorFlow but also all the packages that
 TensorFlow requires.  (This is actually pretty easy.)
 To start working with TensorFlow, you simply need to "activate" the
-virtual environment.  All in all, virtualenv provides a safe and
+virtual environment.  All in all, Virtualenv provides a safe and
 reliable mechanism for installing and running TensorFlow.
 
 Native pip installs TensorFlow directly on your system without going
@@ -121,26 +121,26 @@ Use that package at your own risk.
 
 
 <a name="InstallingVirtualenv"></a>
-## Installing with virtualenv
+## Installing with Virtualenv
 
 Take the following steps to install TensorFlow with Virtualenv:
 
-  1. Install pip and virtualenv by issuing one of the following commands:
+  1. Install pip and Virtualenv by issuing one of the following commands:
 
      <pre>$ <b>sudo apt-get install python-pip python-dev python-virtualenv</b> # for Python 2.7
      $ <b>sudo apt-get install python3-pip python3-dev python-virtualenv</b> # for Python 3.n</pre>
 
-  2. Create a virtualenv environment by issuing one of the following commands:
+  2. Create a Virtualenv environment by issuing one of the following commands:
 
      <pre>$ <b>virtualenv --system-site-packages</b> <i>targetDirectory</i> # for Python 2.7
      $ <b>virtualenv --system-site-packages -p python3</b> <i>targetDirectory</i> # for Python 3.n</pre>
 
      where <code><em>targetDirectory</em></code> specifies the top of the
-     virtualenv tree.  Our instructions assume that
+     Virtualenv tree.  Our instructions assume that
      <code><em>targetDirectory</em></code> is `~/tensorflow`, but you may
      choose any directory.
 
-  3. Activate the virtualenv environment by issuing one of the following
+  3. Activate the Virtualenv environment by issuing one of the following
      commands:
 
      <pre>$ <b>source ~/tensorflow/bin/activate</b> # bash, sh, ksh, or zsh
@@ -156,18 +156,18 @@ Take the following steps to install TensorFlow with Virtualenv:
      <pre>(tensorflow)$ <b>easy_install -U pip</b></pre>
 
   5. Issue one of the following commands to install TensorFlow in the active
-     virtualenv environment:
+     Virtualenv environment:
 
      <pre>(tensorflow)$ <b>pip install --upgrade tensorflow</b>      # for Python 2.7
      (tensorflow)$ <b>pip3 install --upgrade tensorflow</b>     # for Python 3.n
      (tensorflow)$ <b>pip install --upgrade tensorflow-gpu</b>  # for Python 2.7 and GPU
      (tensorflow)$ <b>pip3 install --upgrade tensorflow-gpu</b> # for Python 3.n and GPU</pre>
 
-     If the preceding command succeeds, skip Step 6. If the preceding
+     If the above command succeeds, skip Step 6. If the preceding
      command fails, perform Step 6.
 
   6. (Optional) If Step 5 failed (typically because you invoked a pip version
-     lower than 8.1), install TensorFlow in the active virtualenv environment
+     lower than 8.1), install TensorFlow in the active Virtualenv environment
      by issuing a command of the following format:
 
      <pre>(tensorflow)$ <b>pip install --upgrade</b> <i>tfBinaryURL</i>   # Python 2.7
@@ -181,7 +181,7 @@ Take the following steps to install TensorFlow with Virtualenv:
      [here](#the_url_of_the_tensorflow_python_package).  For example, if you
      are installing TensorFlow for Linux, Python 3.4, and CPU-only support,
      issue the following command to install TensorFlow in the active
-     virtualenv environment:
+     Virtualenv environment:
 
      <pre>(tensorflow)$ <b>pip3 install --upgrade \
      https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0rc0-cp34-cp34m-linux_x86_64.whl</b></pre>
@@ -195,14 +195,14 @@ If you encounter installation problems, see
 After installing TensorFlow,
 [validate the installation](#ValidateYourInstallation).
 
-Note that you must activate the virtualenv environment each time you
-use TensorFlow. If the virtualenv environment is not currently active,
+Note that you must activate the Virtualenv environment each time you
+use TensorFlow. If the Virtualenv environment is not currently active,
 invoke one of the following commands:
 
 <pre>$ <b>source ~/tensorflow/bin/activate</b>      # bash, sh, ksh, or zsh
 $ <b>source ~/tensorflow/bin/activate.csh</b>  # csh or tcsh</pre>
 
-When the virtualenv environment is active, you may run
+When the Virtualenv environment is active, you may run
 TensorFlow programs from this shell.  Your prompt will become
 the following to indicate that your tensorflow environment is active:
 
@@ -490,11 +490,11 @@ To validate your TensorFlow installation, do the following:
 
 ### Prepare your environment
 
-If you installed on native pip, virtualenv, or Anaconda, then
+If you installed on native pip, Virtualenv, or Anaconda, then
 do the following:
 
   1. Start a terminal.
-  2. If you installed with virtualenv or Anaconda, activate your container.
+  2. If you installed with Virtualenv or Anaconda, activate your container.
   3. If you installed TensorFlow source code, navigate to any
      directory *except* one containing TensorFlow source code.
 
diff --git a/tensorflow/docs_src/install/install_mac.md b/tensorflow/docs_src/install/install_mac.md
index 6da22784bf..c95c27cd10 100644
--- a/tensorflow/docs_src/install/install_mac.md
+++ b/tensorflow/docs_src/install/install_mac.md
@@ -8,21 +8,21 @@ Note: As of version 1.2, TensorFlow no longer provides GPU support on macOS.
 
 You must pick the mechanism by which you install TensorFlow. The supported choices are as follows:
 
-  * virtualenv
+  * Virtualenv
   * "native" pip
   * Docker
   * installing from sources, which is documented in
     [a separate guide](https://www.tensorflow.org/install/install_sources).
 
-**We recommend the virtualenv installation.**
+**We recommend the Virtualenv installation.**
 [Virtualenv](https://virtualenv.pypa.io/en/stable)
 is a virtual Python environment isolated from other Python development,
 incapable of interfering with or being affected by other Python programs
-on the same machine.  During the virtualenv installation process,
+on the same machine.  During the Virtualenv installation process,
 you will install not only TensorFlow but also all the packages that
 TensorFlow requires.  (This is actually pretty easy.)
 To start working with TensorFlow, you simply need to "activate" the
-virtual environment.  All in all, virtualenv provides a safe and
+virtual environment.  All in all, Virtualenv provides a safe and
 reliable mechanism for installing and running TensorFlow.
 
 Native pip installs TensorFlow directly on your system without going through
@@ -48,30 +48,30 @@ However, within Anaconda, we recommend installing TensorFlow with the
 That is, the TensorFlow team neither tests nor maintains the conda package.
 Use that package at your own risk.
 
-## Installing with virtualenv
+## Installing with Virtualenv
 
 Take the following steps to install TensorFlow with Virtualenv:
 
   1. Start a terminal (a shell). You'll perform all subsequent steps
      in this shell.
 
-  2. Install pip and virtualenv by issuing the following commands:
+  2. Install pip and Virtualenv by issuing the following commands:
 
      <pre> $ <b>sudo easy_install pip</b>
      $ <b>pip install --upgrade virtualenv</b> </pre>
 
-  3. Create a virtualenv environment by issuing a command of one
+  3. Create a Virtualenv environment by issuing a command of one
      of the following formats:
 
      <pre> $ <b>virtualenv --system-site-packages</b> <i>targetDirectory</i> # for Python 2.7
      $ <b>virtualenv --system-site-packages -p python3</b> <i>targetDirectory</i> # for Python 3.n
      </pre>
 
-     where <i>targetDirectory</i> identifies the top of the virtualenv tree.
+     where <i>targetDirectory</i> identifies the top of the Virtualenv tree.
      Our instructions assume that <i>targetDirectory</i>
      is `~/tensorflow`, but you may choose any directory.
 
-  4. Activate the virtualenv environment by issuing one of the
+  4. Activate the Virtualenv environment by issuing one of the
      following commands:
 
      <pre>$ <b>source ~/tensorflow/bin/activate</b>      # If using bash, sh, ksh, or zsh
@@ -93,7 +93,7 @@ Take the following steps to install TensorFlow with Virtualenv:
 
   7. Optional. If Step 6 failed (typically because you invoked a pip version
      lower than 8.1), install TensorFlow in the active
-     virtualenv environment by issuing a command of the following format:
+     Virtualenv environment by issuing a command of the following format:
 
      <pre> $ <b>pip install --upgrade</b> <i>tfBinaryURL</i>   # Python 2.7
      $ <b>pip3 install --upgrade</b> <i>tfBinaryURL</i>  # Python 3.n </pre>
@@ -121,8 +121,8 @@ After installing TensorFlow,
 [validate your installation](#ValidateYourInstallation)
 to confirm that the installation worked properly.
 
-Note that you must activate the virtualenv environment each time you
-use TensorFlow in a new shell.  If the virtualenv environment is not
+Note that you must activate the Virtualenv environment each time you
+use TensorFlow in a new shell.  If the Virtualenv environment is not
 currently active (that is, the prompt is not `(tensorflow)`, invoke
 one of the following commands:
 
@@ -134,7 +134,7 @@ tensorflow environment is active:
 
 <pre> (tensorflow)$ </pre>
 
-When the virtualenv environment is active, you may run
+When the Virtualenv environment is active, you may run
 TensorFlow programs from this shell.
 
 When you are done using TensorFlow, you may deactivate the
@@ -353,11 +353,11 @@ To validate your TensorFlow installation, do the following:
 
 ### Prepare your environment
 
-If you installed on native pip, virtualenv, or Anaconda, then
+If you installed on native pip, Virtualenv, or Anaconda, then
 do the following:
 
   1. Start a terminal.
-  2. If you installed with virtualenv or Anaconda, activate your container.
+  2. If you installed with Virtualenv or Anaconda, activate your container.
   3. If you installed TensorFlow source code, navigate to any
      directory *except* one containing TensorFlow source code.
 
-- 
GitLab


From 538c8ed28f3306ed724165b566c0a3d2ea817331 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 23 Oct 2017 19:53:55 -0700
Subject: [PATCH 706/909] Updating gpu toolchain

PiperOrigin-RevId: 173207602
---
 third_party/toolchains/gpus/crosstool/CROSSTOOL | 2 +-
 third_party/toolchains/gpus/cuda/BUILD          | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/third_party/toolchains/gpus/crosstool/CROSSTOOL b/third_party/toolchains/gpus/crosstool/CROSSTOOL
index 224b8912f6..a47e0c7cd7 100644
--- a/third_party/toolchains/gpus/crosstool/CROSSTOOL
+++ b/third_party/toolchains/gpus/crosstool/CROSSTOOL
@@ -296,7 +296,7 @@ toolchain {
   cxx_builtin_include_directory: "/usr/include/x86_64-linux-gnu/c++/5.4.0"
   cxx_builtin_include_directory: "/usr/include/c++/5.4.0/backward"
   cxx_builtin_include_directory: "/usr/local/include"
-  cxx_builtin_include_directory: "/usr/local/lib/clang/5.0.0/include"
+  cxx_builtin_include_directory: "/usr/local/lib/clang/6.0.0/include"
   cxx_builtin_include_directory: "/usr/include/x86_64-linux-gnu"
   cxx_builtin_include_directory: "/usr/include"
 }
diff --git a/third_party/toolchains/gpus/cuda/BUILD b/third_party/toolchains/gpus/cuda/BUILD
index 36be86cd10..39136de99c 100644
--- a/third_party/toolchains/gpus/cuda/BUILD
+++ b/third_party/toolchains/gpus/cuda/BUILD
@@ -1347,7 +1347,7 @@ genrule(
         "cuda/lib/libcupti.so.8.0",
     ],
     cmd = """
-if [ -d "$(@D)/extras" ]; then rm $(@D)/extras -drf; fi && if [ -d "$(@D)/include" ]; then rm $(@D)/include -drf; fi && if [ -d "$(@D)/lib" ]; then rm $(@D)/lib -drf; fi && if [ -d "$(@D)/nvvm" ]; then rm $(@D)/nvvm -drf; fi && cp "/usr/local/cuda-8.0/targets/x86_64-linux/lib/stubs/libcuda.so" "$(@D)/cuda/lib/libcuda.so" && cp "/usr/local/cuda-8.0/targets/x86_64-linux/lib/libcudart.so.8.0.61" "$(@D)/cuda/lib/libcudart.so.8.0" && cp "/usr/local/cuda-8.0/targets/x86_64-linux/lib/libcudart_static.a" "$(@D)/cuda/lib/libcudart_static.a" && cp "/usr/local/cuda-8.0/targets/x86_64-linux/lib/libcublas.so.8.0.71" "$(@D)/cuda/lib/libcublas.so.8.0" && cp "/usr/local/cuda-8.0/targets/x86_64-linux/lib/libcusolver.so.8.0.61" "$(@D)/cuda/lib/libcusolver.so.8.0" && cp "/usr/local/cuda-8.0/targets/x86_64-linux/lib/libcurand.so.8.0.61" "$(@D)/cuda/lib/libcurand.so.8.0" && cp "/usr/local/cuda-8.0/targets/x86_64-linux/lib/libcufft.so.8.0.61" "$(@D)/cuda/lib/libcufft.so.8.0" && cp "/usr/lib/x86_64-linux-gnu/libcudnn.so.6.0.21" "$(@D)/cuda/lib/libcudnn.so.6" && cp "/usr/local/cuda-8.0/extras/CUPTI/lib64/libcupti.so.8.0.61" "$(@D)/cuda/lib/libcupti.so.8.0"
+if [ -d "$(@D)/extras" ]; then rm $(@D)/extras -drf; fi && if [ -d "$(@D)/include" ]; then rm $(@D)/include -drf; fi && if [ -d "$(@D)/lib" ]; then rm $(@D)/lib -drf; fi && if [ -d "$(@D)/nvvm" ]; then rm $(@D)/nvvm -drf; fi && cp "/usr/local/cuda-8.0/targets/x86_64-linux/lib/stubs/libcuda.so" "$(@D)/cuda/lib/libcuda.so" && cp "/usr/local/cuda-8.0/targets/x86_64-linux/lib/libcudart.so.8.0.61" "$(@D)/cuda/lib/libcudart.so.8.0" && cp "/usr/local/cuda-8.0/targets/x86_64-linux/lib/libcudart_static.a" "$(@D)/cuda/lib/libcudart_static.a" && cp "/usr/local/cuda-8.0/targets/x86_64-linux/lib/libcublas.so.8.0.88" "$(@D)/cuda/lib/libcublas.so.8.0" && cp "/usr/local/cuda-8.0/targets/x86_64-linux/lib/libcusolver.so.8.0.61" "$(@D)/cuda/lib/libcusolver.so.8.0" && cp "/usr/local/cuda-8.0/targets/x86_64-linux/lib/libcurand.so.8.0.61" "$(@D)/cuda/lib/libcurand.so.8.0" && cp "/usr/local/cuda-8.0/targets/x86_64-linux/lib/libcufft.so.8.0.61" "$(@D)/cuda/lib/libcufft.so.8.0" && cp "/usr/lib/x86_64-linux-gnu/libcudnn.so.6.0.21" "$(@D)/cuda/lib/libcudnn.so.6" && cp "/usr/local/cuda-8.0/extras/CUPTI/lib64/libcupti.so.8.0.61" "$(@D)/cuda/lib/libcupti.so.8.0"
    """,
 )
 
-- 
GitLab


From b20c66a2ad6055602b680ba8f7c8f359e104fd6b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 23 Oct 2017 20:16:22 -0700
Subject: [PATCH 707/909] nsync update: portability fixes for MacOS, s390x.

PiperOrigin-RevId: 173208878
---
 tensorflow/workspace.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 02540bd843..4d577fc246 100644
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -429,11 +429,11 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   native.http_archive(
       name = "nsync",
       urls = [
-          "https://mirror.bazel.build/github.com/google/nsync/archive/ad722c76c6e6653f66be2e1f69521b7f7517da55.tar.gz",
-          # "https://github.com/google/nsync/archive/ad722c76c6e6653f66be2e1f69521b7f7517da55.tar.gz",
+          "https://mirror.bazel.build/github.com/google/nsync/archive/839fcc53ff9be58218ed55397deb3f8376a1444e.tar.gz",
+          # "https://github.com/google/nsync/archive/839fcc53ff9be58218ed55397deb3f8376a1444e.tar.gz",
       ],
-      sha256 = "7dd8ca49319f77e8226cd020a9210a525f88ac26e7041c59c95418223a1cdf55",
-      strip_prefix = "nsync-ad722c76c6e6653f66be2e1f69521b7f7517da55",
+      sha256 = "124d105edb0313ef2d7f5bb86ec94d9f8de95479e55641c4254ffa8f795e9b37",
+      strip_prefix = "nsync-839fcc53ff9be58218ed55397deb3f8376a1444e",
   )
 
   native.http_archive(
-- 
GitLab


From 5bef42720aac651957139d78e4edf0f7bcda1a5f Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Mon, 23 Oct 2017 21:53:28 -0700
Subject: [PATCH 708/909] Use := instead of ?= to set MAKEFILES_DIR to fix
 Android build flakiness

---
 tensorflow/contrib/makefile/Makefile | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/makefile/Makefile b/tensorflow/contrib/makefile/Makefile
index 525cf2cd41..c138fa0c1e 100644
--- a/tensorflow/contrib/makefile/Makefile
+++ b/tensorflow/contrib/makefile/Makefile
@@ -14,7 +14,9 @@
 # Host compilation settings
 
 # Find where we're running from, so we can store generated files here.
-MAKEFILE_DIR ?= $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
+ifeq ($(origin MAKEFILE_DIR), undefined)
+	MAKEFILE_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
+endif
 HAS_GEN_HOST_PROTOC := \
 $(shell test -f $(MAKEFILE_DIR)/gen/protobuf-host/bin/protoc && echo "true" ||\
 echo "false")
-- 
GitLab


From abbb80460f36f40641a42a03a04347143e2cc0ad Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Mon, 23 Oct 2017 21:54:19 -0700
Subject: [PATCH 709/909] Internal change.

PiperOrigin-RevId: 173213868
---
 tensorflow/contrib/makefile/Makefile | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/makefile/Makefile b/tensorflow/contrib/makefile/Makefile
index cb23dd6dab..81024c26a4 100644
--- a/tensorflow/contrib/makefile/Makefile
+++ b/tensorflow/contrib/makefile/Makefile
@@ -14,7 +14,10 @@
 # Host compilation settings
 
 # Find where we're running from, so we can store generated files here.
-MAKEFILE_DIR ?= $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
+ifeq ($(origin MAKEFILE_DIR), undefined)
+	MAKEFILE_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
+endif
+
 HAS_GEN_HOST_PROTOC := \
 $(shell test -f $(MAKEFILE_DIR)/gen/protobuf-host/bin/protoc && echo "true" ||\
 echo "false")
-- 
GitLab


From b7de55e9ea79d1b6b1987834015b37ee59da5a99 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 23 Oct 2017 21:55:58 -0700
Subject: [PATCH 710/909] Handle non-Layer callables and "training" arguments
 in Sequential. Create "add()" method (analogous to Keras') now that
 add/track_layer() is inappropriate to use.

PiperOrigin-RevId: 173213945
---
 tensorflow/contrib/eager/python/BUILD         |  3 ++
 tensorflow/contrib/eager/python/network.py    | 42 +++++++++++++----
 .../contrib/eager/python/network_test.py      | 46 ++++++++++++++++++-
 3 files changed, 80 insertions(+), 11 deletions(-)

diff --git a/tensorflow/contrib/eager/python/BUILD b/tensorflow/contrib/eager/python/BUILD
index a83012e17b..3d7d307778 100644
--- a/tensorflow/contrib/eager/python/BUILD
+++ b/tensorflow/contrib/eager/python/BUILD
@@ -192,6 +192,7 @@ py_library(
         "//tensorflow/python:framework_ops",
         "//tensorflow/python:layers_base",
         "//tensorflow/python:variable_scope",
+        "//tensorflow/python/estimator:util",
     ],
 )
 
@@ -203,6 +204,8 @@ py_test(
         ":network",
         "//tensorflow/python:constant_op",
         "//tensorflow/python:layers",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:nn_ops",
         "//tensorflow/python/eager:test",
     ],
 )
diff --git a/tensorflow/contrib/eager/python/network.py b/tensorflow/contrib/eager/python/network.py
index 8ae5099546..28aed7628e 100644
--- a/tensorflow/contrib/eager/python/network.py
+++ b/tensorflow/contrib/eager/python/network.py
@@ -23,6 +23,7 @@ import uuid
 
 import six
 
+from tensorflow.python.estimator import util as estimator_util
 from tensorflow.python.framework import ops
 from tensorflow.python.layers import base
 from tensorflow.python.ops import variable_scope
@@ -174,26 +175,47 @@ class Network(base.Layer):
 
 
 class Sequential(Network):
-  """Represents a linear sequence of Layers.
+  """Represents a linear sequence of Layers or functions.
 
-  The output of each layer is provided as the input to the next.
+  The output of each layer/function is provided as the input to the next.
   The inputs passed to `__call__` are passed to the inputs of the first
   Layer, and it returns the outputs of the last Layer.
 
   Args:
-    layers: An optional sequence of tf.layers.Layer objects.
+    layers_funcs: An optional sequence where each element is either a
+      tf.layers.Layer object or a callable.
     name: An optional string name to use for this Network.
   """
 
-  def __init__(self, layers=None, name=None):
+  def __init__(self, layers_funcs=None, name=None):
     super(Sequential, self).__init__(name=name)
-    if layers:
-      for l in layers:
-        self.track_layer(l)
+    self._layers_funcs = []
+    if layers_funcs:
+      for l in layers_funcs:
+        self.add(l)
+
+  def add(self, layer_func):
+    if isinstance(layer_func, base.Layer):
+      args = estimator_util.fn_args(layer_func.call)
+      self.track_layer(layer_func)
+    elif callable(layer_func):
+      args = estimator_util.fn_args(layer_func)
+    else:
+      raise TypeError(
+          "Sequential.add() takes only tf.layers.Layer objects or callables; "
+          "not '%s' of type '%s'." % (layer_func, type(layer_func)))
+    self._layers_funcs.append((("training" in args), layer_func))
 
-  def call(self, inputs):
+  def call(self, inputs, training=None):
     """Call each Layer in the order they were added."""
     # TODO(josh11b): Support "mode" and maybe other arguments
-    for l in self.layers:
-      inputs = l(inputs)
+    if training is None:
+      for _, l in self._layers_funcs:
+        inputs = l(inputs)
+    else:
+      for has_training_arg, l in self._layers_funcs:
+        if has_training_arg:
+          inputs = l(inputs, training)
+        else:
+          inputs = l(inputs)
     return inputs
diff --git a/tensorflow/contrib/eager/python/network_test.py b/tensorflow/contrib/eager/python/network_test.py
index f43ce3acda..94cb73ae72 100644
--- a/tensorflow/contrib/eager/python/network_test.py
+++ b/tensorflow/contrib/eager/python/network_test.py
@@ -20,6 +20,8 @@ from tensorflow.contrib.eager.python import network
 from tensorflow.python.eager import test
 from tensorflow.python.framework import constant_op
 from tensorflow.python.layers import core
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import nn_ops
 
 
 # pylint: disable=not-callable
@@ -94,7 +96,7 @@ class SequentialTest(test.TestCase):
 
     # Add a second layer to the network.
     l2 = core.Dense(1, use_bias=False)
-    net.track_layer(l2)
+    net.add(l2)
 
     # Set the second layer's weights so it multiplies by 11
     net(constant_op.constant([[2.0]]))  # Create l2's variables
@@ -102,6 +104,48 @@ class SequentialTest(test.TestCase):
     l2.trainable_variables[0].assign([[11.0]])
     self.assertEqual(231.0, net(constant_op.constant([[7.0]])).numpy())
 
+  def testFunctions(self):
+    # Create a sequential network with one function.
+    net = network.Sequential([nn_ops.relu])
+    two = constant_op.constant(2.0)
+    self.assertEqual(2.0, net(two).numpy())
+    self.assertEqual(0.0, net(-two).numpy())
+    # Add a second function.
+    net.add(math_ops.negative)
+    self.assertEqual(-2.0, net(two).numpy())
+
+  def testTrainingLayer(self):
+    net = network.Sequential([core.Dropout(0.99999)])
+    two = constant_op.constant(2.0)
+    self.assertEqual(2.0, net(two).numpy())
+    self.assertEqual(2.0, net(two, training=False).numpy())
+    for _ in range(20):
+      with_dropout = net(two, training=True).numpy()
+      self.assertIn(with_dropout, [0.0, 2.0])
+      if with_dropout == 0.0:
+        return
+    # Should only fail spuriously 1 in 10^100 runs.
+    self.fail("Didn't see dropout happen after 20 tries.")
+
+  def testTrainingFunction(self):
+    # Output depends on value of "training".
+    def add_training(input_value, training=None):
+      if training is None:
+        return input_value
+      elif training:
+        return input_value + 1
+      return input_value - 1
+
+    # Passing a "training" argument to double would cause an error.
+    def double(input_value):
+      return 2 * input_value
+
+    net = network.Sequential([add_training, double])
+    two = constant_op.constant(2)
+    self.assertEqual(4, net(two).numpy())
+    self.assertEqual(2, net(two, training=False).numpy())
+    self.assertEqual(6, net(two, training=True).numpy())
+
 
 if __name__ == "__main__":
   test.main()
-- 
GitLab


From 20199e91b3503881ce9a4253d64fa783f731230f Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Mon, 23 Oct 2017 22:00:03 -0700
Subject: [PATCH 711/909] Don't prematurely return streams

PiperOrigin-RevId: 173214110
---
 tensorflow/compiler/xla/client/local_client.cc | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/tensorflow/compiler/xla/client/local_client.cc b/tensorflow/compiler/xla/client/local_client.cc
index c885b815eb..15c744ecd3 100644
--- a/tensorflow/compiler/xla/client/local_client.cc
+++ b/tensorflow/compiler/xla/client/local_client.cc
@@ -175,10 +175,15 @@ StatusOr<std::unique_ptr<ScopedShapedBuffer>> LocalExecutable::Run(
   TF_RETURN_IF_ERROR(ValidateExecutionOptions(arguments, options, *backend_));
 
   ExecutableRunOptions actual_options = options;
+
+  Backend::StreamPtr stream;
   if (options.stream() == nullptr) {
+    // NB!  The lifetime of `stream` needs to match the lifetime of
+    // `actual_options` (otherwise we will end up using a returned stream in
+    // ExecuteOnStreamWrapper), which is why it isn't declared in the inner "if"
+    // scope.
     TF_ASSIGN_OR_RETURN(
-        Backend::StreamPtr stream,
-        BorrowStreamForDevice(options.device_ordinal(), backend_));
+        stream, BorrowStreamForDevice(options.device_ordinal(), backend_));
     actual_options.set_stream(stream.get());
   }
   if (options.allocator() == nullptr) {
-- 
GitLab


From 48591e00fe917bfeecc31e501ef133447b81e161 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 24 Oct 2017 08:25:47 -0700
Subject: [PATCH 712/909] Better error message if you pass a list to
 tf.group().

PiperOrigin-RevId: 173260210
---
 tensorflow/python/ops/control_flow_ops.py      | 7 ++++++-
 tensorflow/python/ops/control_flow_ops_test.py | 9 +++++++++
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/ops/control_flow_ops.py b/tensorflow/python/ops/control_flow_ops.py
index f584d93aa2..dcdbeefb70 100644
--- a/tensorflow/python/ops/control_flow_ops.py
+++ b/tensorflow/python/ops/control_flow_ops.py
@@ -2918,7 +2918,6 @@ def group(*inputs, **kwargs):
 
   Args:
     *inputs: Zero or more tensors to group.
-    **kwargs: Optional parameters to pass when constructing the NodeDef.
     name: A name for this operation (optional).
 
   Returns:
@@ -2940,6 +2939,12 @@ def group(*inputs, **kwargs):
     # Sorts *inputs according to their devices.
     ops_on_device = {}  # device -> operations specified on the device.
     for inp in inputs:
+      if not hasattr(inp, "device"):
+        if isinstance(inp, list):
+          raise TypeError("To call tf.group() with a list, use "
+                          "tf.group(*[...]) not tf.group([...]).")
+        raise TypeError("Expected tf.group() expected Tensor arguments not "
+                        "'%s' with type '%s'" % (inp, type(inp)))
       dev = inp.device
       if dev in ops_on_device:
         ops_on_device[dev].append(inp)
diff --git a/tensorflow/python/ops/control_flow_ops_test.py b/tensorflow/python/ops/control_flow_ops_test.py
index d4e66ff1b3..34c405f293 100644
--- a/tensorflow/python/ops/control_flow_ops_test.py
+++ b/tensorflow/python/ops/control_flow_ops_test.py
@@ -114,6 +114,15 @@ class GroupTestCase(test_util.TensorFlowTestCase):
              device: "/task:2" }
     """, self._StripGraph(gd))
 
+  def testPassingList(self):
+    with ops.Graph().as_default():
+      a = constant_op.constant(0, name="a")
+      b = constant_op.constant(0, name="b")
+      with self.assertRaises(TypeError):
+        control_flow_ops.group([a.op, b.op])
+      with self.assertRaises(TypeError):
+        control_flow_ops.group(1, 2)
+
 
 class ShapeTestCase(test_util.TensorFlowTestCase):
 
-- 
GitLab


From c89ebd31d6d729704d77a712c7103f0a7e5353e1 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 24 Oct 2017 08:36:47 -0700
Subject: [PATCH 713/909] Creating a fix for a threading issue with ffmpeg_lib
 in third_party.

Bug is at:

#5804

Fix is to add a unique identifier to each temp file name. The id is unique to
the process. Multiple processes could still have a conflict, though even there
the odds do go down somewhat with this fix.

PiperOrigin-RevId: 173261202
---
 tensorflow/contrib/ffmpeg/default/BUILD       | 12 +++
 .../contrib/ffmpeg/default/ffmpeg_lib.cc      | 16 +++-
 .../ffmpeg/default/ffmpeg_lib_utility_test.cc | 80 +++++++++++++++++++
 3 files changed, 106 insertions(+), 2 deletions(-)
 create mode 100644 tensorflow/contrib/ffmpeg/default/ffmpeg_lib_utility_test.cc

diff --git a/tensorflow/contrib/ffmpeg/default/BUILD b/tensorflow/contrib/ffmpeg/default/BUILD
index 05fc658d80..949ae9ad9e 100644
--- a/tensorflow/contrib/ffmpeg/default/BUILD
+++ b/tensorflow/contrib/ffmpeg/default/BUILD
@@ -23,6 +23,18 @@ cc_library(
     ],
 )
 
+tf_cc_test(
+    name = "ffmpeg_lib_utility_test",
+    srcs = ["ffmpeg_lib_utility_test.cc"],
+    deps = [
+        ":ffmpeg_lib",
+        "//tensorflow/core:framework_internal",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+    ],
+)
+
 tf_cc_test(
     name = "ffmpeg_lib_installed_test",
     srcs = ["ffmpeg_lib_test.cc"],
diff --git a/tensorflow/contrib/ffmpeg/default/ffmpeg_lib.cc b/tensorflow/contrib/ffmpeg/default/ffmpeg_lib.cc
index b417a70b6e..545a4386d0 100644
--- a/tensorflow/contrib/ffmpeg/default/ffmpeg_lib.cc
+++ b/tensorflow/contrib/ffmpeg/default/ffmpeg_lib.cc
@@ -198,6 +198,14 @@ string BuildWavFile(int32 samples_per_second, int32 channel_count,
   return data;
 }
 
+// Returns a unique number every time it is called.
+int64 UniqueId() {
+  static mutex mu(LINKER_INITIALIZED);
+  static int64 id = 0;
+  mutex_lock l(mu);
+  return ++id;
+}
+
 }  // namespace
 
 string GetTempFilename(const string& extension) {
@@ -208,8 +216,12 @@ string GetTempFilename(const string& extension) {
     }
     struct stat statbuf;
     if (!stat(dir, &statbuf) && S_ISDIR(statbuf.st_mode)) {
-      string tmp_filepath =
-          io::JoinPath(dir, StrCat("tmp_file_XXXXXX", ".", extension));
+      // UniqueId is added here because mkstemps is not as thread safe as it
+      // looks. https://github.com/tensorflow/tensorflow/issues/5804 shows
+      // the problem.
+      string tmp_filepath = io::JoinPath(
+          dir,
+          StrCat("tmp_file_tensorflow_", UniqueId(), "_XXXXXX.", extension));
       int fd = mkstemps(&tmp_filepath[0], extension.length() + 1);
       if (fd < 0) {
         LOG(FATAL) << "Failed to create temp file.";
diff --git a/tensorflow/contrib/ffmpeg/default/ffmpeg_lib_utility_test.cc b/tensorflow/contrib/ffmpeg/default/ffmpeg_lib_utility_test.cc
new file mode 100644
index 0000000000..7176f3b550
--- /dev/null
+++ b/tensorflow/contrib/ffmpeg/default/ffmpeg_lib_utility_test.cc
@@ -0,0 +1,80 @@
+// Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// =============================================================================
+
+#include "tensorflow/contrib/ffmpeg/ffmpeg_lib.h"
+
+#include <array>
+#include <set>
+#include <string>
+#include <vector>
+
+#include "tensorflow/core/lib/core/threadpool.h"
+#include "tensorflow/core/platform/env.h"
+#include "tensorflow/core/platform/mutex.h"
+#include "tensorflow/core/platform/test.h"
+
+namespace tensorflow {
+namespace ffmpeg {
+namespace {
+
+TEST(FfmpegLibTest, TestTempDirectoryThreading) {
+  // Testing a fix for a bug that allowed different threads to create
+  // conflicting temp files.
+  // See github.com/tensorflow/tensorflow/issues/5804 for details.
+  const int32 kNumThreads = 10;
+  const int32 kNumWorkItems = 10000;
+  static constexpr size_t kStringsPerItem = 100;
+  Env* environment = Env::Default();
+  thread::ThreadPool pool(environment, "test", kNumThreads);
+
+  mutex mu;
+  std::vector<string> temp_filenames;
+  temp_filenames.reserve(kNumWorkItems * kStringsPerItem);
+
+  // Queue a large number of work items for the threads to process. Each work
+  // item creates a temp file and then deletes it.
+  for (int i = 0; i < kNumWorkItems; ++i) {
+    pool.Schedule([&mu, &temp_filenames, environment]() {
+      std::array<string, kStringsPerItem> buffer;
+      for (int32 j = 0; j < kStringsPerItem; ++j) {
+        buffer[j] = GetTempFilename("mp3");
+        TF_QCHECK_OK(environment->DeleteFile(buffer[j]));
+      }
+      mutex_lock l(mu);
+      for (const auto& fn : buffer) {
+        temp_filenames.push_back(fn);
+      }
+    });
+  }
+
+  // Wait until all work items are complete.
+  while (true) {
+    mutex_lock l(mu);
+    if (temp_filenames.size() == kNumWorkItems * kStringsPerItem) {
+      break;
+    }
+  }
+
+  // Check that no duplicates are created.
+  std::set<string> unique_filenames;
+  mutex_lock l(mu);
+  for (const auto& fn : temp_filenames) {
+    ASSERT_TRUE(unique_filenames.insert(fn).second);
+  }
+}
+
+}  // namespace
+}  // namespace ffmpeg
+}  // namespace tensorflow
-- 
GitLab


From d312011c0220affe5fc8ec5c0d0d61a605c79ed7 Mon Sep 17 00:00:00 2001
From: Chris Leary <leary@google.com>
Date: Tue, 24 Oct 2017 08:59:50 -0700
Subject: [PATCH 714/909] [XLA] Elide whitespace in symbols in
 BatchNormRewriter.

PiperOrigin-RevId: 173263867
---
 tensorflow/compiler/xla/service/batchnorm_rewriter.cc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/compiler/xla/service/batchnorm_rewriter.cc b/tensorflow/compiler/xla/service/batchnorm_rewriter.cc
index 427294dfc6..abe881cd1a 100644
--- a/tensorflow/compiler/xla/service/batchnorm_rewriter.cc
+++ b/tensorflow/compiler/xla/service/batchnorm_rewriter.cc
@@ -83,11 +83,11 @@ class BatchNormRewriterVisitor : public DfsHloVisitorWithDefault {
 
   HloComputation* GetScalarBinaryComputation(PrimitiveType primitive_type,
                                              HloOpcode opcode) {
-    HloComputation::Builder b("scalar computation");
+    HloComputation::Builder b("scalar_computation");
     auto scalar_lhs = b.AddInstruction(HloInstruction::CreateParameter(
-        0, ShapeUtil::MakeShape(F32, {}), "scalar lhs"));
+        0, ShapeUtil::MakeShape(F32, {}), "scalar_lhs"));
     auto scalar_rhs = b.AddInstruction(HloInstruction::CreateParameter(
-        1, ShapeUtil::MakeShape(F32, {}), "scalar rhs"));
+        1, ShapeUtil::MakeShape(F32, {}), "scalar_rhs"));
     auto scalar_op = b.AddInstruction(
         HloInstruction::CreateBinary(ShapeUtil::MakeShape(primitive_type, {}),
                                      opcode, scalar_lhs, scalar_rhs));
-- 
GitLab


From 86895d4a87a4d2cf2e1106b3fa3c176378d1029a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 24 Oct 2017 09:22:56 -0700
Subject: [PATCH 715/909] Provide better debug information on true_classes
 assertion

PiperOrigin-RevId: 173266690
---
 tensorflow/core/kernels/candidate_sampler_ops.cc | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/tensorflow/core/kernels/candidate_sampler_ops.cc b/tensorflow/core/kernels/candidate_sampler_ops.cc
index 9e8b122801..e937c4f11b 100644
--- a/tensorflow/core/kernels/candidate_sampler_ops.cc
+++ b/tensorflow/core/kernels/candidate_sampler_ops.cc
@@ -44,9 +44,11 @@ class BaseCandidateSamplerOp : public OpKernel {
     OP_REQUIRES(context, true_classes.dims() == 2,
                 errors::InvalidArgument("true_classes must be a matrix"));
     const int32 batch_size = true_classes.dim_size(0);
-    OP_REQUIRES(context, true_classes.dim_size(1) == num_true_,
-                errors::InvalidArgument("true_classes must have "
-                                        "num_true columns"));
+    OP_REQUIRES(
+        context, true_classes.dim_size(1) == num_true_,
+        errors::InvalidArgument("true_classes must have "
+                                "num_true columns, expected: ",
+                                true_classes.dim_size(1), " was: ", num_true_));
     CHECK(sampler_) << "CandidateSamplerOp did not set sampler_";
 
     if (unique_) {
-- 
GitLab


From 58b071639d97afdbc5ac5e222a4be81dcb344962 Mon Sep 17 00:00:00 2001
From: Katherine Wu <kathywu@google.com>
Date: Tue, 24 Oct 2017 10:08:03 -0700
Subject: [PATCH 716/909] Added a dataset page to the api guide

PiperOrigin-RevId: 173272637
---
 .../api_guides/python/input_dataset.md        | 81 +++++++++++++++++++
 .../api_guides/python/reading_data.md         | 23 ++++--
 .../docs_src/programmers_guide/datasets.md    |  2 +-
 3 files changed, 98 insertions(+), 8 deletions(-)
 create mode 100644 tensorflow/docs_src/api_guides/python/input_dataset.md

diff --git a/tensorflow/docs_src/api_guides/python/input_dataset.md b/tensorflow/docs_src/api_guides/python/input_dataset.md
new file mode 100644
index 0000000000..2798d76be9
--- /dev/null
+++ b/tensorflow/docs_src/api_guides/python/input_dataset.md
@@ -0,0 +1,81 @@
+# `Dataset` Input Pipeline
+[TOC]
+
+@{tf.data.Dataset} allows you to build complex input pipelines. See the
+@{$datasets$programmer's guide} for an in-depth explanation of how to use this
+API.
+
+## Reader classes
+
+Classes that create a dataset from input files.
+
+*   @{tf.data.FixedLengthRecordDataset}
+*   @{tf.data.TextLineDataset}
+*   @{tf.data.TFRecordDataset}
+
+## Creating new datasets
+
+Static methods in `Dataset` that create new datasets.
+
+*   @{tf.data.Dataset.from_generator}
+*   @{tf.data.Dataset.from_sparse_tensor_slices}
+*   @{tf.data.Dataset.from_tensor_slices}
+*   @{tf.data.Dataset.from_tensors}
+*   @{tf.data.Dataset.list_files}
+*   @{tf.data.Dataset.range}
+*   @{tf.data.Dataset.zip}
+
+## Transformations on existing datasets
+
+These functions transform an existing dataset, and return a new dataset. Calls
+can be chained together, as shown in the example below:
+
+```
+train_data = train_data.batch(100).shuffle().repeat()
+```
+
+*   @{tf.data.Dataset.apply}
+*   @{tf.data.Dataset.batch}
+*   @{tf.data.Dataset.cache}
+*   @{tf.data.Dataset.concatenate}
+*   @{tf.data.Dataset.filter}
+*   @{tf.data.Dataset.flat_map}
+*   @{tf.data.Dataset.interleave}
+*   @{tf.data.Dataset.map}
+*   @{tf.data.Dataset.padded_batch}
+*   @{tf.data.Dataset.prefetch}
+*   @{tf.data.Dataset.repeat}
+*   @{tf.data.Dataset.shard}
+*   @{tf.data.Dataset.shuffle}
+*   @{tf.data.Dataset.skip}
+*   @{tf.data.Dataset.take}
+
+### Custom transformation functions
+
+Custom transformation functions can be applied to a `Dataset` using @{tf.data.Dataset.apply}. Below are custom transformation functions from `tf.contrib.data`:
+
+*   @{tf.contrib.data.batch_and_drop_remainder}
+*   @{tf.contrib.data.dense_to_sparse_batch}
+*   @{tf.contrib.data.enumerate_dataset}
+*   @{tf.contrib.data.group_by_window}
+*   @{tf.contrib.data.ignore_errors}
+*   @{tf.contrib.data.rejection_resample}
+*   @{tf.contrib.data.sloppy_interleave}
+*   @{tf.contrib.data.unbatch}
+
+## Iterating over datasets
+
+These functions make a @{tf.data.Iterator} from a `Dataset`.
+
+*   @{tf.data.Dataset.make_initializable_iterator}
+*   @{tf.data.Dataset.make_one_shot_iterator}
+
+The `Iterator` class also contains static methods that create a @{tf.data.Iterator} that can be used with multiple `Dataset` objects.
+
+*   @{tf.data.Iterator.from_structure}
+*   @{tf.data.Iterator.from_string_handle}
+
+## Extra functions from `tf.contrib.data`
+
+*   @{tf.contrib.data.read_batch_features}
+
diff --git a/tensorflow/docs_src/api_guides/python/reading_data.md b/tensorflow/docs_src/api_guides/python/reading_data.md
index 8b6196ea34..7609ca91d0 100644
--- a/tensorflow/docs_src/api_guides/python/reading_data.md
+++ b/tensorflow/docs_src/api_guides/python/reading_data.md
@@ -3,16 +3,25 @@
 Note: The preferred way to feed data into a tensorflow program is using the
 @{$datasets$Datasets API}.
 
-There are three other methods of getting data into a TensorFlow program:
+There are four methods of getting data into a TensorFlow program:
 
+*   `Dataset` API: Easily construct a complex input pipeline. (preferred method)
 *   Feeding: Python code provides the data when running each step.
-*   Reading from files: an input pipeline reads the data from files
+*   `QueueRunner`: a queue-based input pipeline reads the data from files
     at the beginning of a TensorFlow graph.
 *   Preloaded data: a constant or variable in the TensorFlow graph holds
     all the data (for small data sets).
 
 [TOC]
 
+## Dataset API
+
+See the @{$datasets$programmer's guide} for an in-depth explanation of
+@{tf.data.Dataset}. The `Dataset` API allows you to extract and preprocess data
+from different input/file formats, and apply transformations such as batch,
+shuffle, and map to the dataset. This is an improved version of the old input
+methods, feeding and `QueueRunner`.
+
 ## Feeding
 
 TensorFlow's feed mechanism lets you inject data into any Tensor in a
@@ -22,7 +31,7 @@ graph.
 Supply feed data through the `feed_dict` argument to a run() or eval() call
 that initiates computation.
 
-Note: "Feeding" is the least efficient way to feed data into a tensorflow
+Warning: "Feeding" is the least efficient way to feed data into a tensorflow
 program and should only be used for small experiments and debugging.
 
 ```python
@@ -44,9 +53,9 @@ in
 [`tensorflow/examples/tutorials/mnist/fully_connected_feed.py`](https://www.tensorflow.org/code/tensorflow/examples/tutorials/mnist/fully_connected_feed.py),
 and is described in the @{$mechanics$MNIST tutorial}.
 
-## Reading from files
+## `QueueRunner`
 
-A typical pipeline for reading records from files has the following stages:
+A typical queue-based pipeline for reading records from files has the following stages:
 
 1.  The list of filenames
 2.  *Optional* filename shuffling
@@ -57,8 +66,8 @@ A typical pipeline for reading records from files has the following stages:
 7.  *Optional* preprocessing
 8.  Example queue
 
-Note: This section discusses implementing input pipelines using the
-queue-based APIs which can be cleanly replaced by the ${$datasets$Dataset API}.
+Warning: This section discusses implementing input pipelines using the
+queue-based APIs which can be cleanly replaced by the @{$datasets$Dataset API}.
 
 ### Filenames, shuffling, and epoch limits
 
diff --git a/tensorflow/docs_src/programmers_guide/datasets.md b/tensorflow/docs_src/programmers_guide/datasets.md
index fd1c927539..38e5612fb4 100644
--- a/tensorflow/docs_src/programmers_guide/datasets.md
+++ b/tensorflow/docs_src/programmers_guide/datasets.md
@@ -1,6 +1,6 @@
 # Importing Data
 
-The `Dataset` API enables you to build complex input pipelines from
+The @{tf.data.Dataset$`Dataset`} API enables you to build complex input pipelines from
 simple, reusable pieces. For example, the pipeline for an image model might
 aggregate data from files in a distributed file system, apply random
 perturbations to each image, and merge randomly selected images into a batch
-- 
GitLab


From 377dd3d0d51f93f22eadfd18f4186c27d8506d69 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 24 Oct 2017 10:11:42 -0700
Subject: [PATCH 717/909] Use tf.where instead of multiplies when masking
 probabilities in the BeamSearchDecoder.

PiperOrigin-RevId: 173273139
---
 .../seq2seq/python/ops/beam_search_decoder.py     | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/tensorflow/contrib/seq2seq/python/ops/beam_search_decoder.py b/tensorflow/contrib/seq2seq/python/ops/beam_search_decoder.py
index a88d4f5b8b..5be0c92243 100644
--- a/tensorflow/contrib/seq2seq/python/ops/beam_search_decoder.py
+++ b/tensorflow/contrib/seq2seq/python/ops/beam_search_decoder.py
@@ -715,12 +715,6 @@ def _mask_probs(probs, eos_token, finished):
     probability on the EOS token.
   """
   vocab_size = array_ops.shape(probs)[2]
-  finished_mask = math_ops.cast(array_ops.expand_dims(finished, 2), probs.dtype)
-  not_finished_mask = math_ops.cast(
-      array_ops.expand_dims(math_ops.logical_not(finished), 2),
-      probs.dtype)
-  # These examples are not finished and we leave them
-  non_finished_examples = not_finished_mask * probs
   # All finished examples are replaced with a vector that has all
   # probability on EOS
   finished_row = array_ops.one_hot(
@@ -729,8 +723,13 @@ def _mask_probs(probs, eos_token, finished):
       dtype=probs.dtype,
       on_value=0.,
       off_value=probs.dtype.min)
-  finished_examples = finished_mask * finished_row
-  return finished_examples + non_finished_examples
+  finished_probs = array_ops.tile(
+      array_ops.reshape(finished_row, [1, 1, -1]),
+      array_ops.concat([array_ops.shape(finished), [1]], 0))
+  finished_mask = array_ops.tile(
+      array_ops.expand_dims(finished, 2), [1, 1, vocab_size])
+
+  return array_ops.where(finished_mask, finished_probs, probs)
 
 
 def _maybe_tensor_gather_helper(gather_indices, gather_from, batch_size,
-- 
GitLab


From 1bbec9e4e9c5d3fbbc2fa2b58841435e86dbf76a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 24 Oct 2017 10:39:16 -0700
Subject: [PATCH 718/909] * Add GPU implementation of LogDeterminant op. *
 Switch GPU implementation of Determinant to use the more numerically stable
 kernel as well. * Change behavior for Determinant on matrices with
 (numerically) infinite determinants to match the behavior of
 numpy.linalg.det: Return inf for matrix with infinite determinant. * Misc.
 cleanup in code working around missing support for complex in the NVCC
 compiler.

PiperOrigin-RevId: 173277377
---
 tensorflow/core/kernels/cuda_solvers.h        |  10 -
 .../core/kernels/cuda_solvers_gpu.cu.cc       | 156 +-----------
 tensorflow/core/kernels/determinant_op.cc     | 224 ++++++++++++++----
 tensorflow/core/kernels/determinant_op.h      |  47 ++++
 .../core/kernels/determinant_op_gpu.cu.cc     | 168 +++++++++++++
 tensorflow/core/kernels/linalg_ops_common.h   |  15 +-
 tensorflow/core/kernels/matrix_inverse_op.cc  |   1 -
 tensorflow/core/kernels/matrix_solve_op.cc    |   9 +-
 .../kernels/matrix_triangular_solve_op.cc     |  10 +-
 .../kernel_tests/determinant_op_test.py       |   5 +-
 10 files changed, 414 insertions(+), 231 deletions(-)
 create mode 100644 tensorflow/core/kernels/determinant_op.h
 create mode 100644 tensorflow/core/kernels/determinant_op_gpu.cu.cc

diff --git a/tensorflow/core/kernels/cuda_solvers.h b/tensorflow/core/kernels/cuda_solvers.h
index eb720b191f..af27eb6c47 100644
--- a/tensorflow/core/kernels/cuda_solvers.h
+++ b/tensorflow/core/kernels/cuda_solvers.h
@@ -410,16 +410,6 @@ class DeviceLapackInfo : public ScratchSpace<int> {
 
 namespace functor {
 
-// Helper functor to compute the product of diagonal elements in all matrices
-// in a flattened batch.
-template <typename Device, typename Scalar>
-struct DeterminantFromPivotedLUFunctor {
-  void operator()(const Device& device,
-                  typename TTypes<Scalar, 3>::ConstTensor lu_factor,
-                  const int* pivots, typename TTypes<Scalar, 1>::Tensor output,
-                  int* info);
-};
-
 // Helper functor to set a batch of matrices to the identity.
 // TODO(rmlarsen): Use this kernel to replace the horribly inefficient tf.eye
 // op.
diff --git a/tensorflow/core/kernels/cuda_solvers_gpu.cu.cc b/tensorflow/core/kernels/cuda_solvers_gpu.cu.cc
index 4171f9d68e..84330c041a 100644
--- a/tensorflow/core/kernels/cuda_solvers_gpu.cu.cc
+++ b/tensorflow/core/kernels/cuda_solvers_gpu.cu.cc
@@ -29,159 +29,11 @@ namespace functor {
 
 typedef Eigen::GpuDevice GPUDevice;
 
-namespace {
-
-// Hacks around missing support for complex arithmetic in nvcc.
-template <typename Scalar>
-__device__ inline Scalar Multiply(Scalar x, Scalar y) {
-  return x * y;
-}
-
-template <>
-__device__ inline cuComplex Multiply(cuComplex x, cuComplex y) {
-  return cuCmulf(x, y);
-}
-
-template <>
-__device__ inline cuDoubleComplex Multiply(cuDoubleComplex x,
-                                           cuDoubleComplex y) {
-  return cuCmul(x, y);
-}
-
-template <typename Scalar>
-__device__ inline Scalar Negate(Scalar x) {
-  return -x;
-}
-
-template <>
-__device__ inline cuComplex Negate(cuComplex x) {
-  return make_cuComplex(-cuCrealf(x), -cuCimagf(x));
-}
-
-template <>
-__device__ inline cuDoubleComplex Negate(cuDoubleComplex x) {
-  return make_cuDoubleComplex(-cuCreal(x), -cuCimag(x));
-}
-
-template <typename Scalar>
-__device__ inline bool IsFinite(Scalar x) {
-  return Eigen::numext::isfinite(x);
-}
-
-template <>
-__device__ inline bool IsFinite(cuComplex x) {
-  return Eigen::numext::isfinite(cuCrealf(x)) &&
-         Eigen::numext::isfinite(cuCimagf(x));
-}
-
-template <>
-__device__ inline bool IsFinite(cuDoubleComplex x) {
-  return Eigen::numext::isfinite(cuCreal(x)) &&
-         Eigen::numext::isfinite(cuCimag(x));
-}
-
-template <typename Scalar>
-struct Const {
-  template <typename RealScalar>
-  __device__ static inline Scalar make_const(const RealScalar x) {
-    return Scalar(x);
-  }
-};
-
-template <>
-struct Const<cuComplex> {
-  template <typename RealScalar>
-  __device__ static inline cuComplex make_const(const RealScalar x) {
-    return make_cuComplex(x, 0.0f);
-  }
-};
-
-template <>
-struct Const<cuDoubleComplex> {
-  template <typename RealScalar>
-  __device__ static inline cuDoubleComplex make_const(const RealScalar x) {
-    return make_cuDoubleComplex(x, 0.0f);
-  }
-};
-
-}  // namespace
-
-template <typename Scalar>
-__global__ void DeterminantFromPivotedLUKernel(int nthreads, int n,
-                                               const Scalar* lu_factor,
-                                               const int* all_pivots,
-                                               Scalar* dst, int* info) {
-  const int matrix_size = n * n;
-  const int stride = n + 1;
-  // We only parallelize over batches here. Performance is not critical,
-  // since this cheap O(n) kernel always follows an O(n^3) LU factorization.
-  // The main purpose is to avoid having to copy the LU decomposition to
-  // host memory.
-  CUDA_1D_KERNEL_LOOP(o_idx, nthreads) {
-    // Compute the order of the permutation from the number of transpositions
-    // encoded in the pivot array, see:
-    // http://icl.cs.utk.edu/lapack-forum/viewtopic.php?f=2&t=340
-    const int* pivots = all_pivots + o_idx * n;
-    int order = 0;
-    for (int i = 0; i < n - 1; ++i) {
-      // Notice: Internally, the cuBlas code uses Fortran convention (1-based)
-      // indexing so we expect pivots[i] == i + 1 for rows that were not moved.
-      order += pivots[i] != (i + 1);
-    }
-
-    // Compute the product of the diagonal elements of U from the partially
-    // pivoted LU factorization.
-    // TODO(rmlarsen): This naive implementation (matching that in Eigen used
-    // for the CPU kernel) is pathetically unstable. Should we implement
-    // log-determinant instead (a different set of ops altogether) or something
-    // like the method used in the old LINPACK code:
-    // http://www.netlib.org/linpack/dgedi.f ?
-    int i_idx = matrix_size * o_idx;
-    Scalar prod = lu_factor[i_idx];
-    for (int i = 1; i < n; ++i) {
-      i_idx += stride;
-      prod = Multiply(prod, lu_factor[i_idx]);
-    }
-    // Finally set the determinant to (-1)^order * prod(diag(U)).
-    dst[o_idx] = order % 2 ? Negate(prod) : prod;
-
-    // We write a magic value into the info array if the result was infinite.
-    if (!IsFinite(prod)) {
-      info[o_idx] = kint32min;
-    }
-  }
-}
-
-template <typename Scalar>
-struct DeterminantFromPivotedLUFunctor<GPUDevice, Scalar> {
-  void operator()(const GPUDevice& device,
-                  typename TTypes<Scalar, 3>::ConstTensor lu_factor,
-                  const int* pivots, typename TTypes<Scalar, 1>::Tensor output,
-                  int* info) {
-    using CudaType = typename CUDAComplexT<Scalar>::type;
-    const int64 num_matrices = output.size();
-    const int64 n = lu_factor.dimension(2);
-    const CudaType* lu_factor_ptr =
-        reinterpret_cast<const CudaType*>(lu_factor.data());
-    CudaType* output_ptr = reinterpret_cast<CudaType*>(output.data());
-    CudaLaunchConfig config = GetCudaLaunchConfig(num_matrices, device);
-    DeterminantFromPivotedLUKernel<<<
-        config.block_count, config.thread_per_block, 0, device.stream()>>>(
-        config.virtual_thread_count, n, lu_factor_ptr, pivots, output_ptr,
-        info);
-  }
-};
-
-template struct DeterminantFromPivotedLUFunctor<GPUDevice, float>;
-template struct DeterminantFromPivotedLUFunctor<GPUDevice, double>;
-template struct DeterminantFromPivotedLUFunctor<GPUDevice, complex64>;
-template struct DeterminantFromPivotedLUFunctor<GPUDevice, complex128>;
-
 template <typename Scalar>
 __global__ void EyeKernel(Cuda3DLaunchConfig config, int batch_size, int m,
                           int n, Scalar* matrix_batch_ptr) {
   const int matrix_size = m * n;
-  const Scalar one = Const<Scalar>::make_const(1.0);
+  const Scalar one = Scalar(1);
   CUDA_AXIS_KERNEL_LOOP(batch, config.virtual_thread_count, x) {
     if (batch >= batch_size) {
       break;
@@ -205,16 +57,14 @@ template <typename Scalar>
 struct EyeFunctor<GPUDevice, Scalar> {
   void operator()(const GPUDevice& device,
                   typename TTypes<Scalar, 3>::Tensor matrix_batch) {
-    using CudaType = typename CUDAComplexT<Scalar>::type;
     const int batch_size = matrix_batch.dimension(0);
     const int m = matrix_batch.dimension(1);
     const int n = matrix_batch.dimension(2);
-    CudaType* matrix_batch_ptr =
-        reinterpret_cast<CudaType*>(matrix_batch.data());
     Cuda3DLaunchConfig config = GetCuda3DLaunchConfig(batch_size, m, n, device,
                                                       EyeKernel<Scalar>, 0, 0);
     EyeKernel<<<config.block_count, config.thread_per_block, 0,
-                device.stream()>>>(config, batch_size, m, n, matrix_batch_ptr);
+                device.stream()>>>(config, batch_size, m, n,
+                                   matrix_batch.data());
   }
 };
 
diff --git a/tensorflow/core/kernels/determinant_op.cc b/tensorflow/core/kernels/determinant_op.cc
index 876dbff030..b06f42384e 100644
--- a/tensorflow/core/kernels/determinant_op.cc
+++ b/tensorflow/core/kernels/determinant_op.cc
@@ -14,10 +14,13 @@ limitations under the License.
 ==============================================================================*/
 
 // See docs in ../ops/linalg_ops.cc.
+
 #include <cmath>
 
 #if GOOGLE_CUDA
 #define EIGEN_USE_GPU
+#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "tensorflow/core/kernels/determinant_op.h"
 #endif
 
 #include "third_party/eigen3/Eigen/LU"
@@ -31,23 +34,24 @@ limitations under the License.
 #include "tensorflow/core/platform/types.h"
 
 #if GOOGLE_CUDA
-#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 #include "tensorflow/core/kernels/cuda_solvers.h"
 #include "tensorflow/core/kernels/fill_functor.h"
 #endif
 
 namespace tensorflow {
 
-// A helper function to compute the sign and absolute value of the
-// log of the determinant of inputs via a partially pivoted LU
+// A helper function to compute the sign and absolute value of the log of the
+// determinant of inputs via a partially pivoted LU
 // factorization.
 //
-// Returns the sign in 'sign' and the log determinant in 'logdet'
+// Returns the log of the absolute value of the determinant, and its sign in
+// 'sign'.
 template <class Scalar>
-static void SLogDet(
+static typename Eigen::NumTraits<Scalar>::Real SLogDet(
     const Eigen::Matrix<Scalar, Eigen::Dynamic, Eigen::Dynamic>& inputs,
-    Scalar* sign, Scalar* log_abs_det) {
-  *log_abs_det = 0;
+    Scalar* sign) {
+  using RealScalar = typename Eigen::NumTraits<Scalar>::Real;
+  RealScalar log_abs_det = 0;
   *sign = 1;
   // An empty matrix' determinant is defined to be 1.
   // (https://en.wikipedia.org/wiki/Determinant)
@@ -58,27 +62,25 @@ static void SLogDet(
     Eigen::Matrix<Scalar, Dynamic, Dynamic> LU = lu.matrixLU();
     *sign = lu.permutationP().determinant();
     auto diag = LU.diagonal().array().eval();
-    auto abs_diag = diag.cwiseAbs().template cast<Scalar>().eval();
-    *log_abs_det += abs_diag.log().sum();
+    auto abs_diag = diag.cwiseAbs().eval();
+    log_abs_det += abs_diag.log().sum();
     *sign *= (diag / abs_diag).prod();
   }
-  if (!Eigen::numext::isfinite(*log_abs_det)) {
+  if (!Eigen::numext::isfinite(log_abs_det)) {
     *sign = 0;
-    *log_abs_det = std::log(0.0);
+    log_abs_det =
+        log_abs_det > 0 ? -std::log(RealScalar(0)) : std::log(RealScalar(0));
   }
+  return log_abs_det;
 }
 
 template <class Scalar>
 class LogDeterminantOp : public LinearAlgebraOp<Scalar> {
  public:
-  typedef LinearAlgebraOp<Scalar> Base;
+  INHERIT_LINALG_TYPEDEFS(Scalar);
 
   explicit LogDeterminantOp(OpKernelConstruction* context) : Base(context) {}
 
-  using TensorShapes = typename Base::TensorShapes;
-  using MatrixMaps = typename Base::MatrixMaps;
-  using ConstMatrixMaps = typename Base::ConstMatrixMaps;
-
   TensorShapes GetOutputMatrixShapes(
       const TensorShapes& input_matrix_shapes) const final {
     return TensorShapes({TensorShape({}), TensorShape({})});
@@ -87,9 +89,9 @@ class LogDeterminantOp : public LinearAlgebraOp<Scalar> {
   void ComputeMatrix(OpKernelContext* context, const ConstMatrixMaps& inputs,
                      MatrixMaps* outputs) final {
     Scalar sign;
-    Scalar log_abs_det;
-    SLogDet(Eigen::Matrix<Scalar, Eigen::Dynamic, Eigen::Dynamic>(inputs[0]),
-            &sign, &log_abs_det);
+    const RealScalar log_abs_det = SLogDet(
+        Eigen::Matrix<Scalar, Eigen::Dynamic, Eigen::Dynamic>(inputs[0]),
+        &sign);
 
     outputs->at(0)(0, 0) = sign;
     outputs->at(1)(0, 0) = log_abs_det;
@@ -99,14 +101,10 @@ class LogDeterminantOp : public LinearAlgebraOp<Scalar> {
 template <class Scalar>
 class DeterminantOp : public LinearAlgebraOp<Scalar> {
  public:
-  typedef LinearAlgebraOp<Scalar> Base;
+  INHERIT_LINALG_TYPEDEFS(Scalar);
 
   explicit DeterminantOp(OpKernelConstruction* context) : Base(context) {}
 
-  using TensorShapes = typename Base::TensorShapes;
-  using MatrixMaps = typename Base::MatrixMaps;
-  using ConstMatrixMaps = typename Base::ConstMatrixMaps;
-
   TensorShapes GetOutputMatrixShapes(
       const TensorShapes& input_matrix_shape) const final {
     return TensorShapes({TensorShape({})});
@@ -115,15 +113,10 @@ class DeterminantOp : public LinearAlgebraOp<Scalar> {
   void ComputeMatrix(OpKernelContext* context, const ConstMatrixMaps& inputs,
                      MatrixMaps* outputs) final {
     Scalar sign;
-    Scalar log_abs_det;
-    SLogDet(Eigen::Matrix<Scalar, Eigen::Dynamic, Eigen::Dynamic>(inputs[0]),
-            &sign, &log_abs_det);
-    Scalar determinant = sign * std::exp(log_abs_det);
-    // TODO(rmlarsen): Don't fail on infinite determinants, since that could
-    // be a valid result and the user should check for it instead.
-    OP_REQUIRES(context, Eigen::numext::isfinite(determinant),
-                errors::InvalidArgument("The determinant is not finite."));
-    outputs->at(0)(0, 0) = determinant;
+    const RealScalar log_abs_det = SLogDet(
+        Eigen::Matrix<Scalar, Eigen::Dynamic, Eigen::Dynamic>(inputs[0]),
+        &sign);
+    outputs->at(0)(0, 0) = sign * std::exp(log_abs_det);
   }
 };
 
@@ -171,7 +164,7 @@ class DeterminantOpGpu : public AsyncOpKernel {
       return;
     }
 
-    // TODO(rmlarsen): Convert to std::make_unique when available.
+    // TODO(rmlarsen): Convert to absl::make_unique when available.
     std::unique_ptr<CudaSolver> solver(new CudaSolver(context));
 
     // Reuse the input buffer or make a copy for the factorization step,
@@ -255,18 +248,160 @@ class DeterminantOpGpu : public AsyncOpKernel {
         for (int i = 0; i < host_infos[0].size(); ++i) {
           // It is OK for a matrix to be singular (signaled by info > 0),
           // corresponding to determinant of zero, but we do want to catch
-          // invalid arguments to GetrfBatched.
+          // invalid arguments to Getrf{Batched}.
           OP_REQUIRES_ASYNC(
-              context,
-              host_infos[0].data()[i] >= 0 ||
-                  host_infos[0].data()[i] == kint32min,
+              context, host_infos[0](i) >= 0,
               errors::InvalidArgument("Invalid input argument no. ",
                                       host_infos[0].data()[i],
                                       " for batch index ", i, "."),
               done);
+        }
+      }
+      done();
+    };
+    CudaSolver::CheckLapackInfoAndDeleteSolverAsync(std::move(solver), dev_info,
+                                                    std::move(info_checker));
+  }
+};
+
+template <class Scalar>
+class LogDeterminantOpGpu : public AsyncOpKernel {
+ public:
+  explicit LogDeterminantOpGpu(OpKernelConstruction* context)
+      : AsyncOpKernel(context) {}
+
+  void ComputeAsync(OpKernelContext* context, DoneCallback done) final {
+    const Tensor& input = context->input(0);
+    const int ndims = input.dims();
+    const int64 n = input.dim_size(ndims - 1);
+    // Validate inputs.
+    OP_REQUIRES_ASYNC(
+        context, ndims >= 2,
+        errors::InvalidArgument("Input must have rank >= 2, got ", ndims),
+        done);
+    OP_REQUIRES_ASYNC(
+        context, input.dim_size(ndims - 2) == n,
+        errors::InvalidArgument("Input matrices must be square, got",
+                                input.dim_size(ndims - 2), " != ", n),
+        done);
+
+    // Allocate output.
+    TensorShape out_shape;
+    for (int dim = 0; dim < ndims - 2; ++dim) {
+      out_shape.AddDim(input.dim_size(dim));
+    }
+    out_shape.AppendShape(TensorShape({}));
+    Tensor* sign;
+    OP_REQUIRES_OK_ASYNC(context, context->allocate_output(0, out_shape, &sign),
+                         done);
+    Tensor* log_abs_det;
+    OP_REQUIRES_OK_ASYNC(
+        context, context->allocate_output(1, out_shape, &log_abs_det), done);
+
+    // By definition, the determinant of an empty matrix is equal to one.
+    const GPUDevice& d = context->eigen_device<GPUDevice>();
+    if (input.NumElements() == 0) {
+      functor::SetOneFunctor<GPUDevice, Scalar> one_func;
+      one_func(d, sign->template flat<Scalar>());
+      functor::SetZeroFunctor<GPUDevice, Scalar> zero_func;
+      zero_func(d, log_abs_det->template flat<Scalar>());
+      done();
+      return;
+    }
+
+    // TODO(rmlarsen): Convert to absl::make_unique when available.
+    std::unique_ptr<CudaSolver> solver(new CudaSolver(context));
+
+    // Reuse the input buffer or make a copy for the factorization step,
+    // depending on whether this ops owns it exclusively.
+    Tensor input_copy;
+    OP_REQUIRES_OK_ASYNC(
+        context,
+        solver->forward_input_or_allocate_scoped_tensor(
+            {0}, DataTypeToEnum<Scalar>::value, input.shape(), &input_copy),
+        done);
+    if (!input.SharesBufferWith(input_copy)) {
+      d.memcpy(input_copy.flat<Scalar>().data(), input.flat<Scalar>().data(),
+               input.NumElements() * sizeof(Scalar));
+    }
+    auto input_copy_reshaped = input_copy.template flat_inner_dims<Scalar, 3>();
+    const int64 batch_size = input_copy_reshaped.dimension(0);
+
+    // Allocate pivots on the device.
+    Tensor pivots;
+    OP_REQUIRES_OK_ASYNC(
+        context,
+        solver->allocate_scoped_tensor(DataTypeToEnum<int>::value,
+                                       TensorShape{batch_size, n}, &pivots),
+        done);
+    auto pivots_mat = pivots.template matrix<int>();
+
+    // Prepare pointer arrays for cuBlas' batch interface.
+    // TODO(rmlarsen): Find a way to encode pointer arrays in pinned host memory
+    // without the ugly casting.
+    auto input_copy_ptrs = solver->GetScratchSpace<uint8>(
+        sizeof(Scalar*) * batch_size, "input_copy_ptrs",
+        /* on_host */ true);
+
+    // Compute the partially pivoted LU factorization(s) of the matrix/matrices.
+    std::vector<DeviceLapackInfo> dev_info;
+    if (n / batch_size <= 128) {
+      // For small matrices or large batch sizes, we use the batched interface
+      // from cuBlas.
+      const Scalar** input_copy_ptrs_base =
+          reinterpret_cast<const Scalar**>(input_copy_ptrs.mutable_data());
+      for (int batch = 0; batch < batch_size; ++batch) {
+        input_copy_ptrs_base[batch] = &input_copy_reshaped(batch, 0, 0);
+      }
+      dev_info.push_back(
+          solver->GetDeviceLapackInfo(batch_size, "getrfBatched"));
+      OP_REQUIRES_OK_ASYNC(
+          context,
+          solver->GetrfBatched(n, input_copy_ptrs_base, n, pivots_mat.data(),
+                               &dev_info.back(), batch_size),
+          done);
+    } else {
+      // For large matrices or small batch sizes we use the non-batched
+      // interface from cuSolver, which is much faster for large matrices.
+      dev_info.push_back(solver->GetDeviceLapackInfo(batch_size, "getrf"));
+      for (int batch = 0; batch < batch_size; ++batch) {
+        OP_REQUIRES_OK_ASYNC(
+            context,
+            solver->Getrf(n, n, &input_copy_reshaped(batch, 0, 0), n,
+                          &pivots_mat(batch, 0), &dev_info.back()(batch)),
+            done);
+      }
+    }
+
+    auto input_copy_reshaped_const =
+        const_cast<const Tensor*>(&input_copy)
+            ->template flat_inner_dims<Scalar, 3>();
+    auto sign_reshaped = sign->flat<Scalar>();
+    auto log_abs_det_reshaped = log_abs_det->flat<Scalar>();
+    // Compute the determinant for each batch as (-1)^s * prod(diag(U)),
+    // where s is the order of the permutation encoded in pivots and U is the
+    // upper triangular factor of the LU factorization, which is written to
+    // input_copy by the Getrf{Batched} kernel.
+    functor::LogDeterminantFromPivotedLUFunctor<GPUDevice, Scalar> functor;
+    functor(d, input_copy_reshaped_const, pivots_mat.data(), sign_reshaped,
+            log_abs_det_reshaped);
+
+    // Register callback to check info after kernels finish.
+    auto info_checker = [context, done](
+                            const Status& status,
+                            const std::vector<HostLapackInfo>& host_infos) {
+      if (!status.ok() && errors::IsInvalidArgument(status) &&
+          !host_infos.empty()) {
+        for (int i = 0; i < host_infos[0].size(); ++i) {
+          // It is OK for a matrix to be singular (signaled by info > 0),
+          // corresponding to determinant of zero, but we do want to catch
+          // invalid arguments to Getrf{Batched}.
           OP_REQUIRES_ASYNC(
-              context, host_infos[0].data()[i] != kint32min,
-              errors::InvalidArgument("The determinant is not finite."), done);
+              context, host_infos[0](i) >= 0,
+              errors::InvalidArgument("Invalid input argument no. ",
+                                      host_infos[0].data()[i],
+                                      " for batch index ", i, "."),
+              done);
         }
       }
       done();
@@ -282,6 +417,15 @@ REGISTER_LINALG_OP_GPU("MatrixDeterminant", (DeterminantOpGpu<complex64>),
                        complex64);
 REGISTER_LINALG_OP_GPU("MatrixDeterminant", (DeterminantOpGpu<complex128>),
                        complex128);
+
+REGISTER_LINALG_OP_GPU("LogMatrixDeterminant", (LogDeterminantOpGpu<float>),
+                       float);
+REGISTER_LINALG_OP_GPU("LogMatrixDeterminant", (LogDeterminantOpGpu<double>),
+                       double);
+REGISTER_LINALG_OP_GPU("LogMatrixDeterminant", (LogDeterminantOpGpu<complex64>),
+                       complex64);
+REGISTER_LINALG_OP_GPU("LogMatrixDeterminant",
+                       (LogDeterminantOpGpu<complex128>), complex128);
 #endif  // GOOGLE_CUDA
 
 REGISTER_LINALG_OP("MatrixDeterminant", (DeterminantOp<float>), float);
diff --git a/tensorflow/core/kernels/determinant_op.h b/tensorflow/core/kernels/determinant_op.h
new file mode 100644
index 0000000000..e931e328e4
--- /dev/null
+++ b/tensorflow/core/kernels/determinant_op.h
@@ -0,0 +1,47 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef THIRD_PARTY_TENSORFLOW_CORE_KERNELS_DETERMINANT_OP_H_
+#define THIRD_PARTY_TENSORFLOW_CORE_KERNELS_DETERMINANT_OP_H_
+
+#include "tensorflow/core/framework/tensor_types.h"
+
+namespace tensorflow {
+namespace functor {
+
+// Helper functor to compute Determinant from a partially pivoted LU
+// factorization.
+template <typename Device, typename Scalar>
+struct DeterminantFromPivotedLUFunctor {
+  void operator()(const Device& device,
+                  typename TTypes<Scalar, 3>::ConstTensor lu_factor,
+                  const int* pivots, typename TTypes<Scalar, 1>::Tensor output,
+                  int* info);
+};
+
+// Helper functor to compute sign and log of the absolute value of the
+// determinant from a partially pivoted LU factorization.
+template <typename Device, typename Scalar>
+struct LogDeterminantFromPivotedLUFunctor {
+  void operator()(const Device& device,
+                  typename TTypes<Scalar, 3>::ConstTensor lu_factor,
+                  const int* pivots, typename TTypes<Scalar, 1>::Tensor sign,
+                  typename TTypes<Scalar, 1>::Tensor log_abs_det);
+};
+
+}  // namespace functor
+}  // namespace tensorflow
+
+#endif  // THIRD_PARTY_TENSORFLOW_CORE_KERNELS_DETERMINANT_OP_H_
diff --git a/tensorflow/core/kernels/determinant_op_gpu.cu.cc b/tensorflow/core/kernels/determinant_op_gpu.cu.cc
new file mode 100644
index 0000000000..c866204c97
--- /dev/null
+++ b/tensorflow/core/kernels/determinant_op_gpu.cu.cc
@@ -0,0 +1,168 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+
+#define EIGEN_USE_GPU
+
+#include "tensorflow/core/kernels/determinant_op.h"
+
+#include <complex>
+#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "tensorflow/core/framework/tensor_types.h"
+#include "tensorflow/core/kernels/cuda_solvers.h"
+#include "tensorflow/core/util/cuda_kernel_helper.h"
+
+namespace tensorflow {
+namespace functor {
+
+typedef Eigen::GpuDevice GPUDevice;
+namespace {
+__device__ int PermutationOrder(int n, const int* pivots) {
+  // Compute the order of the permutation from the number of transpositions
+  // encoded in the pivot array, see:
+  // http://icl.cs.utk.edu/lapack-forum/viewtopic.php?f=2&t=340
+  int order = 0;
+  for (int i = 0; i < n - 1; ++i) {
+    // Notice: Internally, the cuBlas code uses Fortran convention (1-based)
+    // indexing so we expect pivots[i] == i + 1 for rows that were not moved.
+    order += pivots[i] != (i + 1);
+  }
+  return order;
+}
+
+#if defined(__CUDACC__)
+// Hack around missing support for complex in NVCC.
+template <typename T>
+__device__ inline std::complex<T> complex_multiply(const std::complex<T>& a,
+                                                   const std::complex<T>& b) {
+  const T a_real = Eigen::numext::real(a);
+  const T a_imag = Eigen::numext::imag(a);
+  const T b_real = Eigen::numext::real(b);
+  const T b_imag = Eigen::numext::imag(b);
+  return std::complex<T>(a_real * b_real - a_imag * b_imag,
+                         a_real * b_imag + a_imag * b_real);
+}
+__device__ inline complex64 operator*(const complex64& a, const complex64& b) {
+  return complex_multiply<float>(a, b);
+}
+__device__ inline complex64 operator*(const complex64& a, const float& b) {
+  return complex64(Eigen::numext::real(a) * b, Eigen::numext::imag(a) * b);
+}
+__device__ inline complex64 operator/(const complex64& a, const float& b) {
+  const float inv_b = 1.0f / b;
+  return a * inv_b;
+}
+__device__ inline complex128 operator*(const complex128& a,
+                                       const complex128& b) {
+  return complex_multiply<double>(a, b);
+}
+__device__ inline complex128 operator*(const complex128& a, const double& b) {
+  return complex128(Eigen::numext::real(a) * b, Eigen::numext::imag(a) * b);
+}
+__device__ inline complex128 operator/(const complex128& a, const double& b) {
+  const double inv_b = 1.0 / b;
+  return a * inv_b;
+}
+#endif
+}  // namespace
+
+// This kernel computes either determinant or log_abs_determinant, depending
+// on the value of the template parameter. If compute_log_abs_det is false,
+// the sign argument is ignored.
+template <typename Scalar, bool compute_log_abs_det = true>
+__global__ void DeterminantFromPivotedLUKernel(int nthreads, int n,
+                                               const Scalar* lu_factor,
+                                               const int* all_pivots,
+                                               Scalar* sign,
+                                               Scalar* log_abs_det) {
+  typedef typename Eigen::NumTraits<Scalar>::Real RealScalar;
+  const int matrix_size = n * n;
+  const int stride = n + 1;
+  // We only parallelize over batches here. Performance is not critical,
+  // since this cheap O(n) kernel always follows an O(n^3) LU factorization.
+  // The main purpose is to avoid having to copy the LU decomposition to
+  // host memory.
+  CUDA_1D_KERNEL_LOOP(o_idx, nthreads) {
+    // Initialize sign to (-1)^order.
+    const int order = PermutationOrder(n, all_pivots + o_idx * n);
+    Scalar prod_sign = order % 2 ? Scalar(-1) : Scalar(1);
+    RealScalar sum_log_abs_det = RealScalar(0);
+    int i_idx = matrix_size * o_idx;
+    for (int i = 0; i < n; ++i, i_idx += stride) {
+      const RealScalar abs_i = Eigen::numext::abs(lu_factor[i_idx]);
+      sum_log_abs_det += Eigen::numext::log(abs_i);
+      prod_sign = prod_sign * (lu_factor[i_idx] / abs_i);
+    }
+    if (!Eigen::numext::isfinite(sum_log_abs_det)) {
+      prod_sign = Scalar(0);
+      sum_log_abs_det = sum_log_abs_det > 0 ? -Eigen::numext::log(RealScalar(0))
+                                            : Eigen::numext::log(RealScalar(0));
+    }
+    if (compute_log_abs_det) {
+      sign[o_idx] = prod_sign;
+      log_abs_det[o_idx] = Scalar(sum_log_abs_det);
+    } else {
+      log_abs_det[o_idx] = prod_sign * Eigen::numext::exp(sum_log_abs_det);
+    }
+  }
+}
+
+template <typename Scalar>
+struct DeterminantFromPivotedLUFunctor<GPUDevice, Scalar> {
+  void operator()(const GPUDevice& device,
+                  typename TTypes<Scalar, 3>::ConstTensor lu_factor,
+                  const int* pivots, typename TTypes<Scalar, 1>::Tensor output,
+                  int* info) {
+    const int64 num_matrices = output.size();
+    const int64 n = lu_factor.dimension(2);
+    CudaLaunchConfig config = GetCudaLaunchConfig(num_matrices, device);
+    DeterminantFromPivotedLUKernel<Scalar, /*compute_log_abs_det=*/false>
+        <<<config.block_count, config.thread_per_block, 0, device.stream()>>>(
+            config.virtual_thread_count, n, lu_factor.data(), pivots, nullptr,
+            output.data());
+  }
+};
+
+template struct DeterminantFromPivotedLUFunctor<GPUDevice, float>;
+template struct DeterminantFromPivotedLUFunctor<GPUDevice, double>;
+template struct DeterminantFromPivotedLUFunctor<GPUDevice, complex64>;
+template struct DeterminantFromPivotedLUFunctor<GPUDevice, complex128>;
+
+template <typename Scalar>
+struct LogDeterminantFromPivotedLUFunctor<GPUDevice, Scalar> {
+  void operator()(const GPUDevice& device,
+                  typename TTypes<Scalar, 3>::ConstTensor lu_factor,
+                  const int* pivots, typename TTypes<Scalar, 1>::Tensor sign,
+                  typename TTypes<Scalar, 1>::Tensor log_abs_det) {
+    const int64 num_matrices = sign.size();
+    const int64 n = lu_factor.dimension(2);
+    CudaLaunchConfig config = GetCudaLaunchConfig(num_matrices, device);
+    DeterminantFromPivotedLUKernel<Scalar, /*compute_log_abs_det=*/true>
+        <<<config.block_count, config.thread_per_block, 0, device.stream()>>>(
+            config.virtual_thread_count, n, lu_factor.data(), pivots,
+            sign.data(), log_abs_det.data());
+  }
+};
+
+template struct LogDeterminantFromPivotedLUFunctor<GPUDevice, float>;
+template struct LogDeterminantFromPivotedLUFunctor<GPUDevice, double>;
+template struct LogDeterminantFromPivotedLUFunctor<GPUDevice, complex64>;
+template struct LogDeterminantFromPivotedLUFunctor<GPUDevice, complex128>;
+
+}  // namespace functor
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/linalg_ops_common.h b/tensorflow/core/kernels/linalg_ops_common.h
index 1d31786728..f7c3f1950b 100644
--- a/tensorflow/core/kernels/linalg_ops_common.h
+++ b/tensorflow/core/kernels/linalg_ops_common.h
@@ -172,13 +172,14 @@ extern template class LinearAlgebraOp<complex128>;
 
 }  // namespace tensorflow
 
-#define INHERIT_LINALG_TYPEDEFS(Scalar)                   \
-  typedef LinearAlgebraOp<Scalar> Base;                   \
-  using Matrix = typename Base::Matrix;                   \
-  using MatrixMap = typename Base::MatrixMap;             \
-  using MatrixMaps = typename Base::MatrixMaps;           \
-  using ConstMatrixMap = typename Base::ConstMatrixMap;   \
-  using ConstMatrixMaps = typename Base::ConstMatrixMaps; \
+#define INHERIT_LINALG_TYPEDEFS(Scalar)                       \
+  typedef LinearAlgebraOp<Scalar> Base;                       \
+  using RealScalar = typename Eigen::NumTraits<Scalar>::Real; \
+  using Matrix = typename Base::Matrix;                       \
+  using MatrixMap = typename Base::MatrixMap;                 \
+  using MatrixMaps = typename Base::MatrixMaps;               \
+  using ConstMatrixMap = typename Base::ConstMatrixMap;       \
+  using ConstMatrixMaps = typename Base::ConstMatrixMaps;     \
   using TensorShapes = typename Base::TensorShapes;
 
 #define REGISTER_LINALG_OP_CPU(OpName, OpClass, Scalar) \
diff --git a/tensorflow/core/kernels/matrix_inverse_op.cc b/tensorflow/core/kernels/matrix_inverse_op.cc
index 64edfe470d..cae84f52d7 100644
--- a/tensorflow/core/kernels/matrix_inverse_op.cc
+++ b/tensorflow/core/kernels/matrix_inverse_op.cc
@@ -69,7 +69,6 @@ class MatrixInverseOp : public LinearAlgebraOp<Scalar> {
     // a result of basic user mistakes, such as providing integer valued
     // matrices that are exactly singular, or due to underflow if this
     // code is run with denormals being flushed to zero.
-    using RealScalar = typename Base::RealScalar;
     const RealScalar min_abs_pivot =
         lu_decomposition.matrixLU().diagonal().cwiseAbs().minCoeff();
     OP_REQUIRES(context, min_abs_pivot > RealScalar(0),
diff --git a/tensorflow/core/kernels/matrix_solve_op.cc b/tensorflow/core/kernels/matrix_solve_op.cc
index 2e4098dfab..169f3dae76 100644
--- a/tensorflow/core/kernels/matrix_solve_op.cc
+++ b/tensorflow/core/kernels/matrix_solve_op.cc
@@ -44,18 +44,12 @@ static const char kErrMsg[] = "Input matrix is not invertible.";
 template <class Scalar>
 class MatrixSolveOp : public LinearAlgebraOp<Scalar> {
  public:
-  typedef LinearAlgebraOp<Scalar> Base;
+  INHERIT_LINALG_TYPEDEFS(Scalar);
 
   explicit MatrixSolveOp(OpKernelConstruction* context) : Base(context) {
     OP_REQUIRES_OK(context, context->GetAttr("adjoint", &adjoint_));
   }
 
-  using TensorShapes = typename Base::TensorShapes;
-  using Matrix = typename Base::Matrix;
-  using MatrixMaps = typename Base::MatrixMaps;
-  using ConstMatrixMap = typename Base::ConstMatrixMap;
-  using ConstMatrixMaps = typename Base::ConstMatrixMaps;
-
   void ValidateInputMatrixShapes(
       OpKernelContext* context,
       const TensorShapes& input_matrix_shapes) const final {
@@ -102,7 +96,6 @@ class MatrixSolveOp : public LinearAlgebraOp<Scalar> {
     // a result of basic user mistakes such providing integer valued
     // matrices that are exactly singular, or due to underflow if this
     // code is run with denormals being flushed to zero.
-    using RealScalar = typename Base::RealScalar;
     const RealScalar min_abs_pivot =
         lu_decomposition.matrixLU().diagonal().cwiseAbs().minCoeff();
     OP_REQUIRES(context, min_abs_pivot > RealScalar(0),
diff --git a/tensorflow/core/kernels/matrix_triangular_solve_op.cc b/tensorflow/core/kernels/matrix_triangular_solve_op.cc
index 953f37fa02..6f7e6a7496 100644
--- a/tensorflow/core/kernels/matrix_triangular_solve_op.cc
+++ b/tensorflow/core/kernels/matrix_triangular_solve_op.cc
@@ -47,7 +47,7 @@ perftools::gputools::DeviceMemory<Scalar> AsDeviceMemory(
 template <class Scalar>
 class MatrixTriangularSolveOp : public LinearAlgebraOp<Scalar> {
  public:
-  typedef LinearAlgebraOp<Scalar> Base;
+  INHERIT_LINALG_TYPEDEFS(Scalar);
 
   explicit MatrixTriangularSolveOp(OpKernelConstruction* context)
       : Base(context), lower_(true), adjoint_(false) {
@@ -55,13 +55,6 @@ class MatrixTriangularSolveOp : public LinearAlgebraOp<Scalar> {
     OP_REQUIRES_OK(context, context->GetAttr("adjoint", &adjoint_));
   }
 
-  using TensorShapes = typename Base::TensorShapes;
-  using Matrix = typename Base::Matrix;
-  using MatrixMap = typename Base::MatrixMap;
-  using MatrixMaps = typename Base::MatrixMaps;
-  using ConstMatrixMap = typename Base::ConstMatrixMap;
-  using ConstMatrixMaps = typename Base::ConstMatrixMaps;
-
   void ValidateInputMatrixShapes(
       OpKernelContext* context,
       const TensorShapes& input_matrix_shapes) const final {
@@ -97,7 +90,6 @@ class MatrixTriangularSolveOp : public LinearAlgebraOp<Scalar> {
       // an empty set of equation as the empty matrix.
       return;
     }
-    using RealScalar = typename Base::RealScalar;
     const RealScalar min_abs_pivot = matrix.diagonal().cwiseAbs().minCoeff();
     OP_REQUIRES(context, min_abs_pivot > RealScalar(0),
                 errors::InvalidArgument("Input matrix is not invertible."));
diff --git a/tensorflow/python/kernel_tests/determinant_op_test.py b/tensorflow/python/kernel_tests/determinant_op_test.py
index 7368fbc4a1..222038b22e 100644
--- a/tensorflow/python/kernel_tests/determinant_op_test.py
+++ b/tensorflow/python/kernel_tests/determinant_op_test.py
@@ -126,11 +126,10 @@ class DeterminantOpTest(test.TestCase):
     self._compareDeterminant(
         np.random.rand(3, 4, 5, 2, 2).astype(np.complex128))
 
-  def testOverflow(self):
+  def testInfiniteDeterminant(self):
     max_double = np.finfo("d").max
     huge_matrix = np.array([[max_double, 0.0], [0.0, max_double]])
-    with self.assertRaisesOpError("not finite"):
-      self._compareDeterminant(huge_matrix)
+    self._compareDeterminant(huge_matrix)
 
   def testNonSquareMatrix(self):
     # When the determinant of a non-square matrix is attempted we should return
-- 
GitLab


From 720efa37a4e93d5833e6e928993790f2523f0d85 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Tue, 24 Oct 2017 10:41:58 -0700
Subject: [PATCH 719/909] Roll forward CL 171084886

171084886 had to be rolled back twice due to various open source build issues.
I'm trying again, now that I think I've addressed all the pertinent issues.

Original CL description:

Don't use dlsym to resolve symbols in the CPU JIT

Instead of resolving symbols via dlsym when JITting for the CPU backend, use a
registry based mechanism.  This lets us kill off the --export_dynamic hack that
we used to need for CustomCall on the CPU backend.

PiperOrigin-RevId: 173277862
---
 tensorflow/compiler/tf2xla/kernels/BUILD      |   3 +-
 .../index_ops_kernel_argmax_float_1d.cc       |   3 +
 .../index_ops_kernel_argmax_float_2d.cc       |   3 +
 tensorflow/compiler/xla/service/cpu/BUILD     |  12 ++
 .../cpu/custom_call_target_registry.cc        |  39 ++++
 .../service/cpu/custom_call_target_registry.h |  74 +++++++
 .../xla/service/cpu/simple_orc_jit.cc         | 198 ++++++++++--------
 tensorflow/compiler/xla/tests/BUILD           |   3 +-
 .../compiler/xla/tests/custom_call_test.cc    |  14 +-
 tensorflow/compiler/xla/xla.bzl               |   8 -
 10 files changed, 259 insertions(+), 98 deletions(-)
 create mode 100644 tensorflow/compiler/xla/service/cpu/custom_call_target_registry.cc
 create mode 100644 tensorflow/compiler/xla/service/cpu/custom_call_target_registry.h

diff --git a/tensorflow/compiler/tf2xla/kernels/BUILD b/tensorflow/compiler/tf2xla/kernels/BUILD
index 4ee7989824..2b43e313eb 100644
--- a/tensorflow/compiler/tf2xla/kernels/BUILD
+++ b/tensorflow/compiler/tf2xla/kernels/BUILD
@@ -5,7 +5,6 @@ package(
 )
 
 load("//tensorflow:tensorflow.bzl", "tf_kernel_library")
-load("//tensorflow/compiler/xla:xla.bzl", "export_dynamic_linkopts")
 
 tf_kernel_library(
     name = "xla_ops",
@@ -153,6 +152,7 @@ cc_library(
     srcs = ["index_ops_kernel_argmax_float_1d.cc"],
     visibility = ["//visibility:public"],
     deps = [
+        "//tensorflow/compiler/xla/service/cpu:custom_call_target_registry",
         "//tensorflow/core:framework_lite",
         "//third_party/eigen3",
     ],
@@ -164,6 +164,7 @@ cc_library(
     srcs = ["index_ops_kernel_argmax_float_2d.cc"],
     visibility = ["//visibility:public"],
     deps = [
+        "//tensorflow/compiler/xla/service/cpu:custom_call_target_registry",
         "//tensorflow/core:framework_lite",
         "//third_party/eigen3",
     ],
diff --git a/tensorflow/compiler/tf2xla/kernels/index_ops_kernel_argmax_float_1d.cc b/tensorflow/compiler/tf2xla/kernels/index_ops_kernel_argmax_float_1d.cc
index afbd64ca50..47cf8c6675 100644
--- a/tensorflow/compiler/tf2xla/kernels/index_ops_kernel_argmax_float_1d.cc
+++ b/tensorflow/compiler/tf2xla/kernels/index_ops_kernel_argmax_float_1d.cc
@@ -16,6 +16,7 @@ limitations under the License.
 #define EIGEN_USE_THREADS
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "tensorflow/compiler/xla/service/cpu/custom_call_target_registry.h"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/platform/dynamic_annotations.h"
 #include "tensorflow/core/platform/macros.h"
@@ -47,3 +48,5 @@ EIGEN_STRONG_INLINE void argmax_float_1d_xla_impl(void* out, void** data) {
 extern "C" void TF_EXPORT argmax_float_1d_xla_impl(void* out, void** data) {
   tensorflow::argmax_float_1d_xla_impl(out, data);
 }
+
+REGISTER_CUSTOM_CALL_TARGET(argmax_float_1d_xla_impl);
diff --git a/tensorflow/compiler/tf2xla/kernels/index_ops_kernel_argmax_float_2d.cc b/tensorflow/compiler/tf2xla/kernels/index_ops_kernel_argmax_float_2d.cc
index 841ff2f4df..9b83392d8f 100644
--- a/tensorflow/compiler/tf2xla/kernels/index_ops_kernel_argmax_float_2d.cc
+++ b/tensorflow/compiler/tf2xla/kernels/index_ops_kernel_argmax_float_2d.cc
@@ -16,6 +16,7 @@ limitations under the License.
 #define EIGEN_USE_THREADS
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+#include "tensorflow/compiler/xla/service/cpu/custom_call_target_registry.h"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/platform/dynamic_annotations.h"
 #include "tensorflow/core/platform/macros.h"
@@ -49,3 +50,5 @@ EIGEN_STRONG_INLINE void argmax_float_2d_xla_impl(void* out, void** data) {
 extern "C" void TF_EXPORT argmax_float_2d_xla_impl(void* out, void** data) {
   tensorflow::argmax_float_2d_xla_impl(out, data);
 }
+
+REGISTER_CUSTOM_CALL_TARGET(argmax_float_2d_xla_impl);
diff --git a/tensorflow/compiler/xla/service/cpu/BUILD b/tensorflow/compiler/xla/service/cpu/BUILD
index 136cbe7cb7..56bc1a6706 100644
--- a/tensorflow/compiler/xla/service/cpu/BUILD
+++ b/tensorflow/compiler/xla/service/cpu/BUILD
@@ -153,6 +153,7 @@ cc_library(
         ":cpu_runtime_avx",
         ":cpu_runtime_neon",
         ":cpu_runtime_sse4_1",
+        ":custom_call_target_registry",
         ":disassembler",
         ":external_constant_pool",
         ":runtime_conv2d",
@@ -719,6 +720,17 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "custom_call_target_registry",
+    srcs = [
+        "custom_call_target_registry.cc",
+    ],
+    hdrs = [
+        "custom_call_target_registry.h",
+    ],
+    visibility = ["//visibility:public"],
+)
+
 # -----------------------------------------------------------------------------
 
 filegroup(
diff --git a/tensorflow/compiler/xla/service/cpu/custom_call_target_registry.cc b/tensorflow/compiler/xla/service/cpu/custom_call_target_registry.cc
new file mode 100644
index 0000000000..5f5803874b
--- /dev/null
+++ b/tensorflow/compiler/xla/service/cpu/custom_call_target_registry.cc
@@ -0,0 +1,39 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/service/cpu/custom_call_target_registry.h"
+
+namespace xla {
+namespace cpu {
+
+CustomCallTargetRegistry* CustomCallTargetRegistry::Global() {
+  static auto* registry = new CustomCallTargetRegistry;
+  return registry;
+}
+
+void CustomCallTargetRegistry::Register(const std::string& symbol,
+                                        void* address) {
+  std::lock_guard<std::mutex> lock(mu_);
+  registered_symbols_[symbol] = address;
+}
+
+void* CustomCallTargetRegistry::Lookup(const std::string& symbol) const {
+  std::lock_guard<std::mutex> lock(mu_);
+  auto it = registered_symbols_.find(symbol);
+  return it == registered_symbols_.end() ? nullptr : it->second;
+}
+
+}  // namespace cpu
+}  // namespace xla
diff --git a/tensorflow/compiler/xla/service/cpu/custom_call_target_registry.h b/tensorflow/compiler/xla/service/cpu/custom_call_target_registry.h
new file mode 100644
index 0000000000..2994642356
--- /dev/null
+++ b/tensorflow/compiler/xla/service/cpu/custom_call_target_registry.h
@@ -0,0 +1,74 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef THIRD_PARTY_TENSORFLOW_COMPILER_XLA_SERVICE_CPU_CUSTOM_CALL_TARGET_REGISTRY_H_
+#define THIRD_PARTY_TENSORFLOW_COMPILER_XLA_SERVICE_CPU_CUSTOM_CALL_TARGET_REGISTRY_H_
+
+// This file is depended on by kernels that have to build for mobile devices.
+// For this reason, we avoid relying on TensorFlow and instead only use the
+// standard C++ library.
+
+#include <mutex>  // NOLINT
+#include <string>
+#include <unordered_map>
+
+namespace xla {
+namespace cpu {
+
+// The CPU JIT compiler uses this registry to resolve symbolic CustomCall
+// targets; so when using the CPU JIT, CustomCall targets need to be registered
+// here with the symbol name used in the CustomCall.
+//
+// The XLA AOT compiler links using a standard offline linker; so when compiling
+// in AOT mode, you *also* need to make sure the name of the callee (presumably
+// implemented in C++) matches up with the symbolic name used in the CustomCall.
+//
+// We maintain the registry in both the JIT and the AOT cases for simplicity,
+// but we only use it when running in JIT mode.
+class CustomCallTargetRegistry {
+ public:
+  static CustomCallTargetRegistry* Global();
+
+  void Register(const std::string& symbol, void* address);
+  void* Lookup(const std::string& symbol) const;
+
+ private:
+  std::unordered_map<std::string, void*> registered_symbols_;
+  mutable std::mutex mu_;
+};
+
+class RegisterCustomCallTarget {
+ public:
+  explicit RegisterCustomCallTarget(const std::string& name, void* address) {
+    CustomCallTargetRegistry::Global()->Register(name, address);
+  }
+};
+
+#define REGISTER_CUSTOM_CALL_CONCAT(a, b) a##b
+
+#define REGISTER_CUSTOM_CALL_TARGET_WITH_SYM_HELPER(symbol, address, counter) \
+  static ::xla::cpu::RegisterCustomCallTarget REGISTER_CUSTOM_CALL_CONCAT(    \
+      custom_call_target_register, counter)(symbol,                           \
+                                            reinterpret_cast<void*>(address))
+
+#define REGISTER_CUSTOM_CALL_TARGET_WITH_SYM(symbol, address) \
+  REGISTER_CUSTOM_CALL_TARGET_WITH_SYM_HELPER(symbol, address, __COUNTER__)
+
+#define REGISTER_CUSTOM_CALL_TARGET(function) \
+  REGISTER_CUSTOM_CALL_TARGET_WITH_SYM(#function, function)
+
+}  // namespace cpu
+}  // namespace xla
+
+#endif  // THIRD_PARTY_TENSORFLOW_COMPILER_XLA_SERVICE_CPU_CUSTOM_CALL_TARGET_REGISTRY_H_
diff --git a/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc b/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc
index cfffb3fbc3..fdf02e5b42 100644
--- a/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc
+++ b/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc
@@ -31,6 +31,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/cpu/cpu_runtime_avx.h"
 #include "tensorflow/compiler/xla/service/cpu/cpu_runtime_neon.h"
 #include "tensorflow/compiler/xla/service/cpu/cpu_runtime_sse4_1.h"
+#include "tensorflow/compiler/xla/service/cpu/custom_call_target_registry.h"
 #include "tensorflow/compiler/xla/service/cpu/runtime_conv2d.h"
 #include "tensorflow/compiler/xla/service/cpu/runtime_fork_join.h"
 #include "tensorflow/compiler/xla/service/cpu/runtime_matmul.h"
@@ -43,81 +44,6 @@ namespace xla {
 namespace cpu {
 namespace {
 
-// Converts a symbol 'name' into the form expected by dlsym().
-std::string CanonicalizeSymbol(const std::string& name) {
-#if defined(__APPLE__)
-  // On Mac OS X, dlsym() expects names not to be prefixed with a leading
-  // underscore.
-  if (!name.empty() && name.front() == '_') {
-    return name.substr(1);
-  }
-#endif
-  return name;
-}
-
-class JITSymbolTable {
- public:
-  JITSymbolTable() { Populate(); }
-
-  void* Lookup(llvm::StringRef jit_symbol_name) const {
-    auto it = jit_symbol_table_.find(jit_symbol_name);
-    return it == jit_symbol_table_.end() ? nullptr : it->getValue();
-  }
-
-  static bool MustBeInTable(llvm::StringRef name) {
-    // In particular, names starting with
-    // runtime::kXlaCpuRuntimeSymbolNamePrefix should not be dlsym'ed.
-    return name.startswith(runtime::kXlaCpuRuntimeSymbolNamePrefix);
-  }
-
- private:
-  void AddJITSymbolToTable(llvm::StringRef jit_symbol_name,
-                           llvm::StringRef cpp_symbol_name,
-                           void* jit_symbol_value) {
-    // The JIT symbol name and the C++ symbol name (with an extern "C" linkage)
-    // need to match, otherwise AOT links will fail.
-    CHECK(jit_symbol_name == cpp_symbol_name);
-    CHECK(jit_symbol_table_.insert({jit_symbol_name, jit_symbol_value}).second);
-  }
-
-  void Populate() {
-#define ADD_JIT_SYMBOL_TO_TABLE(base_name)                       \
-  do {                                                           \
-    AddJITSymbolToTable(                                         \
-        xla::cpu::runtime::k##base_name##SymbolName,             \
-        "__xla_cpu_runtime_" #base_name,                         \
-        reinterpret_cast<void*>(__xla_cpu_runtime_##base_name)); \
-  } while (false)
-
-    ADD_JIT_SYMBOL_TO_TABLE(AcquireInfeedBufferForDequeue);
-    ADD_JIT_SYMBOL_TO_TABLE(ReleaseInfeedBufferAfterDequeue);
-    ADD_JIT_SYMBOL_TO_TABLE(AcquireOutfeedBufferForPopulation);
-    ADD_JIT_SYMBOL_TO_TABLE(ReleaseOutfeedBufferAfterPopulation);
-    ADD_JIT_SYMBOL_TO_TABLE(ExpV8F32AVX);
-    ADD_JIT_SYMBOL_TO_TABLE(LogV8F32AVX);
-    ADD_JIT_SYMBOL_TO_TABLE(ExpV4F32SSE);
-    ADD_JIT_SYMBOL_TO_TABLE(LogV4F32SSE);
-    ADD_JIT_SYMBOL_TO_TABLE(ExpV4F32NEON);
-    ADD_JIT_SYMBOL_TO_TABLE(LogV4F32NEON);
-    ADD_JIT_SYMBOL_TO_TABLE(EigenConvF32);
-    ADD_JIT_SYMBOL_TO_TABLE(EigenMatMulF32);
-    ADD_JIT_SYMBOL_TO_TABLE(EigenMatMulF64);
-    ADD_JIT_SYMBOL_TO_TABLE(EigenSingleThreadedConvF32);
-    ADD_JIT_SYMBOL_TO_TABLE(EigenSingleThreadedMatMulF32);
-    ADD_JIT_SYMBOL_TO_TABLE(EigenSingleThreadedMatMulF64);
-    ADD_JIT_SYMBOL_TO_TABLE(ParallelForkJoin);
-
-#undef ADD_JIT_SYMBOL_TO_TABLE
-  }
-
-  llvm::StringMap<void*> jit_symbol_table_;
-};
-
-const JITSymbolTable& GetJITSymbolTable() {
-  static JITSymbolTable* symbol_table = new JITSymbolTable;
-  return *symbol_table;
-}
-
 // A simple SymbolResolver that delegates to the host dynamic linker.
 class SimpleResolver : public llvm::JITSymbolResolver {
  public:
@@ -125,7 +51,6 @@ class SimpleResolver : public llvm::JITSymbolResolver {
       : external_constant_pool_(external_constant_pool) {}
 
   llvm::JITSymbol findSymbol(const std::string& name) override {
-    string name_as_string(name);
     if (const uint8* from_constant_pool =
             external_constant_pool_->Find(string(name))) {
       return llvm::JITEvaluatedSymbol(
@@ -133,13 +58,7 @@ class SimpleResolver : public llvm::JITSymbolResolver {
           llvm::JITSymbolFlags::None);
     }
 
-    std::string canonical_name = CanonicalizeSymbol(name);
-    const JITSymbolTable& jit_symbol_table = GetJITSymbolTable();
-
-    void* func_addr = JITSymbolTable::MustBeInTable(canonical_name)
-                          ? jit_symbol_table.Lookup(canonical_name)
-                          : dlsym(RTLD_DEFAULT, canonical_name.c_str());
-
+    void* func_addr = CustomCallTargetRegistry::Global()->Lookup(name);
     if (func_addr == nullptr) {
       return nullptr;
     }
@@ -255,5 +174,118 @@ llvm::JITSymbol SimpleOrcJIT::FindSymbol(const std::string& name) {
   return nullptr;
 }
 
+namespace {
+// Register some known symbols with the CustomCallTargetRegistry.
+bool RegisterKnownJITSymbols() {
+  CustomCallTargetRegistry* registry = CustomCallTargetRegistry::Global();
+
+#define REGISTER_CPU_RUNTIME_SYMBOL(base_name)                                \
+  do {                                                                        \
+    auto* function_address =                                                  \
+        reinterpret_cast<void*>(__xla_cpu_runtime_##base_name);               \
+    registry->Register(xla::cpu::runtime::k##base_name##SymbolName,           \
+                       function_address);                                     \
+    CHECK_EQ(                                                                 \
+        tensorflow::StringPiece(xla::cpu::runtime::k##base_name##SymbolName), \
+        "__xla_cpu_runtime_" #base_name);                                     \
+  } while (false)
+
+  REGISTER_CPU_RUNTIME_SYMBOL(AcquireInfeedBufferForDequeue);
+  REGISTER_CPU_RUNTIME_SYMBOL(AcquireOutfeedBufferForPopulation);
+  REGISTER_CPU_RUNTIME_SYMBOL(EigenConvF32);
+  REGISTER_CPU_RUNTIME_SYMBOL(EigenMatMulF32);
+  REGISTER_CPU_RUNTIME_SYMBOL(EigenMatMulF64);
+  REGISTER_CPU_RUNTIME_SYMBOL(EigenSingleThreadedConvF32);
+  REGISTER_CPU_RUNTIME_SYMBOL(EigenSingleThreadedMatMulF32);
+  REGISTER_CPU_RUNTIME_SYMBOL(EigenSingleThreadedMatMulF64);
+  REGISTER_CPU_RUNTIME_SYMBOL(ExpV4F32NEON);
+  REGISTER_CPU_RUNTIME_SYMBOL(ExpV4F32SSE);
+  REGISTER_CPU_RUNTIME_SYMBOL(ExpV8F32AVX);
+  REGISTER_CPU_RUNTIME_SYMBOL(LogV4F32NEON);
+  REGISTER_CPU_RUNTIME_SYMBOL(LogV4F32SSE);
+  REGISTER_CPU_RUNTIME_SYMBOL(LogV8F32AVX);
+  REGISTER_CPU_RUNTIME_SYMBOL(ParallelForkJoin);
+  REGISTER_CPU_RUNTIME_SYMBOL(ReleaseInfeedBufferAfterDequeue);
+  REGISTER_CPU_RUNTIME_SYMBOL(ReleaseOutfeedBufferAfterPopulation);
+
+#undef REGISTER_CPU_RUNTIME_SYMBOL
+
+#define REGISTER_LIBM_SYMBOL(name)                                    \
+  do {                                                                \
+    /* Register both the F32 and F64 variants of the libm symbol.  */ \
+    registry->Register(#name "f", reinterpret_cast<void*>(name##f));  \
+    registry->Register(#name, reinterpret_cast<void*>(name));         \
+  } while (false)
+
+  REGISTER_LIBM_SYMBOL(acos);
+  REGISTER_LIBM_SYMBOL(acosh);
+  REGISTER_LIBM_SYMBOL(asin);
+  REGISTER_LIBM_SYMBOL(asinh);
+  REGISTER_LIBM_SYMBOL(atan);
+  REGISTER_LIBM_SYMBOL(atan2);
+  REGISTER_LIBM_SYMBOL(atanh);
+  REGISTER_LIBM_SYMBOL(cbrt);
+  REGISTER_LIBM_SYMBOL(ceil);
+  REGISTER_LIBM_SYMBOL(copysign);
+  REGISTER_LIBM_SYMBOL(cos);
+  REGISTER_LIBM_SYMBOL(cosh);
+  REGISTER_LIBM_SYMBOL(erf);
+  REGISTER_LIBM_SYMBOL(erfc);
+  REGISTER_LIBM_SYMBOL(exp);
+  REGISTER_LIBM_SYMBOL(exp2);
+  REGISTER_LIBM_SYMBOL(expm1);
+  REGISTER_LIBM_SYMBOL(fabs);
+  REGISTER_LIBM_SYMBOL(fdim);
+  REGISTER_LIBM_SYMBOL(floor);
+  REGISTER_LIBM_SYMBOL(fma);
+  REGISTER_LIBM_SYMBOL(fmax);
+  REGISTER_LIBM_SYMBOL(fmin);
+  REGISTER_LIBM_SYMBOL(fmod);
+  REGISTER_LIBM_SYMBOL(frexp);
+  REGISTER_LIBM_SYMBOL(hypot);
+  REGISTER_LIBM_SYMBOL(ilogb);
+  REGISTER_LIBM_SYMBOL(ldexp);
+  REGISTER_LIBM_SYMBOL(lgamma);
+  REGISTER_LIBM_SYMBOL(llrint);
+  REGISTER_LIBM_SYMBOL(llround);
+  REGISTER_LIBM_SYMBOL(log);
+  REGISTER_LIBM_SYMBOL(log10);
+  REGISTER_LIBM_SYMBOL(log1p);
+  REGISTER_LIBM_SYMBOL(log2);
+  REGISTER_LIBM_SYMBOL(logb);
+  REGISTER_LIBM_SYMBOL(lrint);
+  REGISTER_LIBM_SYMBOL(lround);
+  REGISTER_LIBM_SYMBOL(modf);
+  REGISTER_LIBM_SYMBOL(nan);
+  REGISTER_LIBM_SYMBOL(nearbyint);
+  REGISTER_LIBM_SYMBOL(nextafter);
+  REGISTER_LIBM_SYMBOL(nexttoward);
+  REGISTER_LIBM_SYMBOL(pow);
+  REGISTER_LIBM_SYMBOL(remainder);
+  REGISTER_LIBM_SYMBOL(remquo);
+  REGISTER_LIBM_SYMBOL(rint);
+  REGISTER_LIBM_SYMBOL(round);
+  REGISTER_LIBM_SYMBOL(scalbln);
+  REGISTER_LIBM_SYMBOL(scalbn);
+  REGISTER_LIBM_SYMBOL(sin);
+  REGISTER_LIBM_SYMBOL(sincos);
+  REGISTER_LIBM_SYMBOL(sinh);
+  REGISTER_LIBM_SYMBOL(sqrt);
+  REGISTER_LIBM_SYMBOL(tan);
+  REGISTER_LIBM_SYMBOL(tanh);
+  REGISTER_LIBM_SYMBOL(tgamma);
+  REGISTER_LIBM_SYMBOL(trunc);
+
+#undef REGISTER_LIBM_SYMBOL
+
+  registry->Register("memcpy", reinterpret_cast<void*>(memcpy));
+  registry->Register("memmove", reinterpret_cast<void*>(memmove));
+  registry->Register("memset", reinterpret_cast<void*>(memset));
+  return true;
+}
+
+bool unused = RegisterKnownJITSymbols();
+}  // namespace
+
 }  // namespace cpu
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/tests/BUILD b/tensorflow/compiler/xla/tests/BUILD
index 43127925e6..2ea7b9bd8e 100644
--- a/tensorflow/compiler/xla/tests/BUILD
+++ b/tensorflow/compiler/xla/tests/BUILD
@@ -23,7 +23,6 @@ filegroup(
     ]),
 )
 
-load("//tensorflow/compiler/xla:xla.bzl", "export_dynamic_linkopts")
 load("//tensorflow/compiler/xla/tests:build_defs.bzl", "xla_test")
 load("//tensorflow/compiler/xla/tests:build_defs.bzl", "xla_test_library")
 load("//tensorflow/compiler/xla/tests:build_defs.bzl", "generate_backend_suites")
@@ -988,13 +987,13 @@ xla_test(
 xla_test(
     name = "custom_call_test",
     srcs = ["custom_call_test.cc"],
-    linkopts = export_dynamic_linkopts,
     deps = [
         "//tensorflow/compiler/xla:literal_util",
         "//tensorflow/compiler/xla:shape_util",
         "//tensorflow/compiler/xla:util",
         "//tensorflow/compiler/xla:xla_data_proto",
         "//tensorflow/compiler/xla/service:hlo",
+        "//tensorflow/compiler/xla/service/cpu:custom_call_target_registry",
         "//tensorflow/compiler/xla/tests:hlo_test_base",
         "//tensorflow/compiler/xla/tests:literal_test_util",
         "//tensorflow/compiler/xla/tests:xla_internal_test_main",
diff --git a/tensorflow/compiler/xla/tests/custom_call_test.cc b/tensorflow/compiler/xla/tests/custom_call_test.cc
index 342478bc74..74f73a1ddc 100644
--- a/tensorflow/compiler/xla/tests/custom_call_test.cc
+++ b/tensorflow/compiler/xla/tests/custom_call_test.cc
@@ -18,6 +18,7 @@ limitations under the License.
 
 #include "tensorflow/compiler/xla/literal_util.h"
 #include "tensorflow/compiler/xla/ptr_util.h"
+#include "tensorflow/compiler/xla/service/cpu/custom_call_target_registry.h"
 #include "tensorflow/compiler/xla/service/hlo_computation.h"
 #include "tensorflow/compiler/xla/service/hlo_instruction.h"
 #include "tensorflow/compiler/xla/service/hlo_module.h"
@@ -31,19 +32,19 @@ limitations under the License.
 #include "tensorflow/core/platform/macros.h"
 #include "tensorflow/core/platform/test.h"
 
-
-extern "C" void TF_EXPORT R0F32Add2(float* out, float** in) {
+namespace {
+void R0F32Add2(float* out, float** in) {
   TF_ANNOTATE_MEMORY_IS_INITIALIZED(in, sizeof(float*));
   *out = **in + 2.0f;
 }
 
-extern "C" void TF_EXPORT R2F32ReduceSum(float* out, float** in) {
+void R2F32ReduceSum(float* out, float** in) {
   TF_ANNOTATE_MEMORY_IS_INITIALIZED(in, sizeof(float) * 4);
   float* array = in[0];
   *out = array[0] + array[1] + array[2] + array[3];
 }
 
-extern "C" void TF_EXPORT Add1ToValues(float* out, float** in) {
+void Add1ToValues(float* out, float** in) {
   TF_ANNOTATE_MEMORY_IS_INITIALIZED(in, sizeof(float) * 4);
   float* array = in[0];
   out[0] = array[0] + 1;
@@ -51,6 +52,11 @@ extern "C" void TF_EXPORT Add1ToValues(float* out, float** in) {
   out[2] = array[2] + 1;
   out[3] = array[3] + 1;
 }
+}  // namespace
+
+REGISTER_CUSTOM_CALL_TARGET(R0F32Add2);
+REGISTER_CUSTOM_CALL_TARGET(R2F32ReduceSum);
+REGISTER_CUSTOM_CALL_TARGET(Add1ToValues);
 
 namespace xla {
 namespace {
diff --git a/tensorflow/compiler/xla/xla.bzl b/tensorflow/compiler/xla/xla.bzl
index 22e70ec97a..3fa5bcc1df 100644
--- a/tensorflow/compiler/xla/xla.bzl
+++ b/tensorflow/compiler/xla/xla.bzl
@@ -17,11 +17,3 @@ def xla_proto_library(name, srcs=[], deps=[], visibility=None, testonly=0):
                    protoc="@protobuf_archive//:protoc",
                    testonly=testonly,
                    visibility=visibility,)
-
-# Flags required for modules that export symbols that are to be called by the
-# XLA CustomCall operator. CustomCall must be able to find symbols with dlsym(),
-# which on Linux requires we link with --export-dynamic.
-export_dynamic_linkopts = select({
-    "//tensorflow:darwin": [],
-    "//conditions:default": ["-Wl,--export-dynamic"],
-})
-- 
GitLab


From 34f5d001c4224c8c1f4ce615bcb1e76610e95673 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 24 Oct 2017 13:06:29 -0700
Subject: [PATCH 720/909] Fix K-FAC's loss_functions.insert_slice_in_zeros and
 add tests

PiperOrigin-RevId: 173299853
---
 .../contrib/kfac/python/kernel_tests/BUILD    | 12 +++++
 .../kernel_tests/loss_functions_test.py       | 44 +++++++++++++++++++
 .../contrib/kfac/python/ops/loss_functions.py | 11 ++---
 3 files changed, 62 insertions(+), 5 deletions(-)
 create mode 100644 tensorflow/contrib/kfac/python/kernel_tests/loss_functions_test.py

diff --git a/tensorflow/contrib/kfac/python/kernel_tests/BUILD b/tensorflow/contrib/kfac/python/kernel_tests/BUILD
index 1b2a5cdd38..fd4f588741 100644
--- a/tensorflow/contrib/kfac/python/kernel_tests/BUILD
+++ b/tensorflow/contrib/kfac/python/kernel_tests/BUILD
@@ -79,6 +79,18 @@ py_test(
     ],
 )
 
+py_test(
+    name = "loss_functions_test",
+    srcs = ["loss_functions_test.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        "//tensorflow/contrib/kfac/python/ops:loss_functions",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:constant_op",
+    ],
+)
+
 py_test(
     name = "optimizer_test",
     srcs = ["optimizer_test.py"],
diff --git a/tensorflow/contrib/kfac/python/kernel_tests/loss_functions_test.py b/tensorflow/contrib/kfac/python/kernel_tests/loss_functions_test.py
new file mode 100644
index 0000000000..86dd839896
--- /dev/null
+++ b/tensorflow/contrib/kfac/python/kernel_tests/loss_functions_test.py
@@ -0,0 +1,44 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for tf.contrib.kfac.loss_functions."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.contrib.kfac.python.ops import loss_functions
+from tensorflow.python.framework import constant_op
+from tensorflow.python.ops import array_ops
+from tensorflow.python.platform import test
+
+
+class InsertSliceInZerosTest(test.TestCase):
+
+  def testBadShape(self):
+    bad_shaped_ones = array_ops.ones(shape=[1, 3])  # n.b. shape[1] != 1
+    with self.assertRaises(ValueError):
+      loss_functions.insert_slice_in_zeros(bad_shaped_ones, 1, 42, 17)
+
+  def test3d(self):
+    input_tensor = constant_op.constant([[[1, 2]], [[3, 4]]])
+    expected_output_array = [[[1, 2], [0, 0]], [[3, 4], [0, 0]]]
+    op = loss_functions.insert_slice_in_zeros(input_tensor, 1, 2, 0)
+    with self.test_session() as sess:
+      actual_output_array = sess.run(op)
+    self.assertAllEqual(expected_output_array, actual_output_array)
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/contrib/kfac/python/ops/loss_functions.py b/tensorflow/contrib/kfac/python/ops/loss_functions.py
index d80382b9cf..0b5c3d4928 100644
--- a/tensorflow/contrib/kfac/python/ops/loss_functions.py
+++ b/tensorflow/contrib/kfac/python/ops/loss_functions.py
@@ -632,11 +632,12 @@ class MultiBernoulliNegativeLogProbLoss(DistributionNegativeLogProbLoss,
 
 
 def insert_slice_in_zeros(slice_to_insert, dim, dim_size, position):
-  """Inserts slice into a larger tensors of zeros.
+  """Inserts slice into a larger tensor of zeros.
 
-  Forms a new tensor that which is the same shape as slice_, except that
+  Forms a new tensor which is the same shape as slice_to_insert, except that
   the dimension given by 'dim' is expanded to the size given by 'dim_size'.
-  'position' determines the position (index) of the slice in that dimension.
+  'position' determines the position (index) at which to insert the slice within
+  that dimension.
 
   Assumes slice_to_insert.shape[dim] = 1.
 
@@ -644,7 +645,7 @@ def insert_slice_in_zeros(slice_to_insert, dim, dim_size, position):
     slice_to_insert: The slice to insert.
     dim: The dimension which to expand with zeros.
     dim_size: The new size of the 'dim' dimension.
-    position: The position of 'slice_' in the new tensor.
+    position: The position of 'slice_to_insert' in the new tensor.
 
   Returns:
     The new tensor.
@@ -662,4 +663,4 @@ def insert_slice_in_zeros(slice_to_insert, dim, dim_size, position):
   before[dim] = position
   after[dim] = dim_size - position - 1
 
-  return array_ops.pad(slice_to_insert, zip(before, after))
+  return array_ops.pad(slice_to_insert, list(zip(before, after)))
-- 
GitLab


From 6d6c8e012bb4498bbdd75de3f64f0cab72b1391c Mon Sep 17 00:00:00 2001
From: Jeremy Sharpe <jeremy.adamson.sharpe@gmail.com>
Date: Tue, 24 Oct 2017 16:16:09 -0400
Subject: [PATCH 721/909] Fix a typo of "Jenkins".

---
 tensorflow/tools/ci_build/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/tools/ci_build/README.md b/tensorflow/tools/ci_build/README.md
index acef833909..202fcb9101 100644
--- a/tensorflow/tools/ci_build/README.md
+++ b/tensorflow/tools/ci_build/README.md
@@ -50,7 +50,7 @@ and tests. Click on **Details** to see the results from Jenkins or the internal
 CI system.
 
 Results from Jenkins are displayed in the Jenkins UI. For more information,
-see the [Jenkns documentation](https://jenkins.io/doc/).
+see the [Jenkins documentation](https://jenkins.io/doc/).
 
 Results from the internal CI system are displayed in the Build Status UI. In
 this UI, to see the logs for a failed build:
-- 
GitLab


From 03f1105003c7e30127ed9449524c36d2c384b79c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 24 Oct 2017 13:30:04 -0700
Subject: [PATCH 722/909] Better support for Metrics in graph mode: * Avoid
 situation where variables are created in the wrong graph. * Add an
 init_variables() method that returns an op that will initialize   any
 non-initialized variables.

PiperOrigin-RevId: 173302832
---
 tensorflow/contrib/eager/python/BUILD         |  3 ++
 .../contrib/eager/python/metrics_impl.py      | 32 ++++++++++++++++++-
 .../contrib/eager/python/metrics_test.py      | 11 +++++--
 3 files changed, 42 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/eager/python/BUILD b/tensorflow/contrib/eager/python/BUILD
index 3d7d307778..ee2ec79141 100644
--- a/tensorflow/contrib/eager/python/BUILD
+++ b/tensorflow/contrib/eager/python/BUILD
@@ -131,10 +131,13 @@ py_library(
     deps = [
         "//tensorflow/contrib/summary:summary_ops",
         "//tensorflow/python:array_ops",
+        "//tensorflow/python:constant_op",
+        "//tensorflow/python:control_flow_ops",
         "//tensorflow/python:dtypes",
         "//tensorflow/python:framework_ops",
         "//tensorflow/python:layers_base",
         "//tensorflow/python:math_ops",
+        "//tensorflow/python:resource_variable_ops",
         "//tensorflow/python:util",
         "//tensorflow/python:variable_scope",
         "//tensorflow/python/eager:context",
diff --git a/tensorflow/contrib/eager/python/metrics_impl.py b/tensorflow/contrib/eager/python/metrics_impl.py
index 959ee735b0..77a84e006e 100644
--- a/tensorflow/contrib/eager/python/metrics_impl.py
+++ b/tensorflow/contrib/eager/python/metrics_impl.py
@@ -23,16 +23,30 @@ import re
 from tensorflow.contrib.summary import summary_ops
 from tensorflow.python.eager import context
 from tensorflow.python.eager import function
+from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import init_ops
 from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.ops import variable_scope
 
 
 _to_replace = re.compile("[^A-Za-z0-9.]")
 
 
+def _init_var(v):
+  def do_init(v):
+    with ops.control_dependencies([v.assign(v.initial_value)]):
+      return constant_op.constant(True)
+  return control_flow_ops.cond(
+      resource_variable_ops.var_is_initialized_op(v._handle),  # pylint: disable=protected-access
+      lambda: constant_op.constant(False),
+      lambda: do_init(v))
+
+
 class Metric(object):
   """A metric holds state for aggregating statistics over an evaluation run.
 
@@ -76,7 +90,10 @@ class Metric(object):
     if context.in_graph_mode():
       # We make self.call() into a graph callable here, so that we can
       # return a single op that performs all of the variable updates.
+      self._construction_scope = ops.get_default_graph().as_default
       self.call = function.defun(self.call)
+    else:
+      self._construction_scope = context.eager_mode
 
   # ---- API for users ----
   def __call__(self, *args, **kwargs):
@@ -89,7 +106,8 @@ class Metric(object):
       **kwargs: A mini-batch of inputs to the Metric, passed on to `call()`.
     """
     if not self._built:
-      with variable_scope.variable_scope(self._scope):
+      with variable_scope.variable_scope(
+          self._scope), self._construction_scope():
         self.build(*args, **kwargs)
       self._built = True
     return self.call(*args, **kwargs)
@@ -102,6 +120,18 @@ class Metric(object):
   def variables(self):
     return self._vars
 
+  def init_variables(self):
+    """Return an op for initializing this Metric's uninitialized variables.
+
+    Only for graph execution. Should be called after variables are created
+    in the first execution of __call__().
+
+    Returns:
+      An op to run.
+    """
+    assert context.in_graph_mode()
+    return control_flow_ops.group(*[_init_var(v) for v in self._vars])
+
   # ---- To be implemented by descendants ---
   def build(self, *args, **kwargs):
     """Method to create variables.
diff --git a/tensorflow/contrib/eager/python/metrics_test.py b/tensorflow/contrib/eager/python/metrics_test.py
index 1880e762d4..fce6be1761 100644
--- a/tensorflow/contrib/eager/python/metrics_test.py
+++ b/tensorflow/contrib/eager/python/metrics_test.py
@@ -29,7 +29,6 @@ from tensorflow.python.eager import test
 from tensorflow.python.framework import dtypes
 from tensorflow.python.lib.io import tf_record
 from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import variables
 from tensorflow.python.platform import gfile
 from tensorflow.python.training import training_util
 
@@ -136,11 +135,16 @@ class MetricsTest(test.TestCase):
       m = metrics.Mean()
       p = array_ops.placeholder(dtypes.float32)
       accumulate = m(p)
-      variables.global_variables_initializer().run()
+      init_op = m.init_variables()
+      init_op.run()
       sess.run(accumulate, feed_dict={p: [1, 10, 100]})
       sess.run(accumulate, feed_dict={p: 1000})
       sess.run(accumulate, feed_dict={p: [10000, 100000]})
       self.assertAllEqual(m.result().eval(), 111111.0/6)
+      # Second init is ignored, since the variables are already initialized.
+      init_op.run()
+      sess.run(accumulate, feed_dict={p: 7})
+      self.assertAllEqual(m.result().eval(), 111118.0/7)
 
   def testTwoMeansGraph(self):
     # Verify two metrics with the same class and name don't
@@ -150,7 +154,8 @@ class MetricsTest(test.TestCase):
       m2 = metrics.Mean()
       accumulate1 = m1(0)
       accumulate2 = m2(2)
-      variables.global_variables_initializer().run()
+      m1.init_variables().run()
+      m2.init_variables().run()
       sess.run([accumulate1, accumulate2])
       self.assertEqual(0, m1.result().eval())
       self.assertEqual(2, m2.result().eval())
-- 
GitLab


From 73f8b044ea7333b25ef5c9841c1e072e45ad5890 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 24 Oct 2017 13:48:27 -0700
Subject: [PATCH 723/909] replace min for std:min to avoid issues with clang
 compilation e.g.:
 http://ci.tensorflow.org/job/nightly-matrix-linux-gpu-clang/159/console

PiperOrigin-RevId: 173305545
---
 tensorflow/contrib/rnn/kernels/lstm_ops_gpu.cu.cc  |  4 ++--
 tensorflow/core/kernels/reduction_gpu_kernels.cu.h | 10 +++++-----
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/tensorflow/contrib/rnn/kernels/lstm_ops_gpu.cu.cc b/tensorflow/contrib/rnn/kernels/lstm_ops_gpu.cu.cc
index d82676ff7e..6d3758fef1 100644
--- a/tensorflow/contrib/rnn/kernels/lstm_ops_gpu.cu.cc
+++ b/tensorflow/contrib/rnn/kernels/lstm_ops_gpu.cu.cc
@@ -209,7 +209,7 @@ void LSTMBlockCellFpropWithCUDA(
   // Use 2D blocks. The number of threads per block is equal to x * y, where x =
   // min(batch_size, 8) and y = 32. See above for guidance on number of
   // threads.
-  dim3 block_dim_2d(min(batch_size, 8), 32);
+  dim3 block_dim_2d(std::min(batch_size, 8), 32);
   dim3 grid_dim_2d(Eigen::divup(batch_size, static_cast<int>(block_dim_2d.x)),
                    Eigen::divup(cell_size, static_cast<int>(block_dim_2d.y)));
 
@@ -323,7 +323,7 @@ void LSTMBlockCellBpropWithCUDA(
     const bool use_peephole) {
   const cudaStream_t& cu_stream = GetCudaStream(ctx);
 
-  dim3 block_dim_2d(min(batch_size, 8), 32);
+  dim3 block_dim_2d(std::min(batch_size, 8), 32);
   dim3 grid_dim_2d(Eigen::divup(batch_size, static_cast<int>(block_dim_2d.x)),
                    Eigen::divup(cell_size, static_cast<int>(block_dim_2d.y)));
 
diff --git a/tensorflow/core/kernels/reduction_gpu_kernels.cu.h b/tensorflow/core/kernels/reduction_gpu_kernels.cu.h
index be9a611881..36ca7f834f 100644
--- a/tensorflow/core/kernels/reduction_gpu_kernels.cu.h
+++ b/tensorflow/core/kernels/reduction_gpu_kernels.cu.h
@@ -460,7 +460,7 @@ void LaunchScalarReduction(OpKernelContext* ctx, OUT_T out, IN_T in,
     return;
   } else if (in_size <= 1 << 19) {
     const int num_threads = 256;
-    const int num_blocks = min(32, Eigen::divup(in_size, num_threads));
+    const int num_blocks = std::min(32, Eigen::divup(in_size, num_threads));
     // it seems like tailoring this to the GPU
     // would be more effective, but all attempts
     // at making this a multiple of the number of
@@ -557,13 +557,13 @@ void LaunchColumnReduction_LTE16Cols(OpKernelContext* ctx, OUT_T out, IN_T in,
                                      int extent_x, int extent_y, Op op, T init,
                                      const cudaStream_t& cu_stream) {
   int rows_per_warp = 32 / extent_y;
-  dim3 block_dim(32, min(Eigen::divup(extent_x, rows_per_warp), 32), 1);
+  dim3 block_dim(32, std::min(Eigen::divup(extent_x, rows_per_warp), 32), 1);
   dim3 grid_dim(1,
                 Eigen::divup(static_cast<unsigned int>(extent_x),
                              rows_per_warp * block_dim.y),
                 1);
 
-  grid_dim.y = min((int)grid_dim.y, 32);
+  grid_dim.y = std::min((int)grid_dim.y, 32);
 
   if (grid_dim.y > 2 && grid_dim.y < 32) {
     int log2 = Log2Floor(grid_dim.y);
@@ -596,10 +596,10 @@ template <typename T, typename Op, typename OUT_T, typename IN_T>
 void LaunchColumnReduction_LTE4096Cols(OpKernelContext* ctx, OUT_T out, IN_T in,
                                        int extent_x, int extent_y, Op op,
                                        T init, const cudaStream_t& cu_stream) {
-  dim3 block_dim(32, min(extent_x, 32), 1);
+  dim3 block_dim(32, std::min(extent_x, 32), 1);
   dim3 grid_dim((extent_y + 31) / 32, 1, 1);
 
-  if (grid_dim.x < 16) grid_dim.y = min((extent_x + 31) / 32, 32);
+  if (grid_dim.x < 16) grid_dim.y = std::min((extent_x + 31) / 32, 32);
 
   if (grid_dim.y > 2 && grid_dim.y < 32) {
     int log2 = Log2Floor(grid_dim.y);
-- 
GitLab


From bf1fad214febef6af5c101d8f953d0109c46dfbb Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 24 Oct 2017 13:55:04 -0700
Subject: [PATCH 724/909] Fix NCCL rewrite bug when rerunning sessions
 (assigned device id is not stable). Fix collocate_gradients for initial
 losses. Remove NcclBroadcast gradient test for now. The generated AddN to
 accumulate the broadcast outputs before passing it to the gradient function
 is CPU only and cannot be collocated with NcclBroadcast on the GPU.

PiperOrigin-RevId: 173306409
---
 .../contrib/nccl/kernels/nccl_rewrite.cc      |  9 +-
 .../contrib/nccl/python/ops/nccl_ops_test.py  | 10 +--
 tensorflow/python/ops/gradients_impl.py       | 87 +++++++++----------
 3 files changed, 54 insertions(+), 52 deletions(-)

diff --git a/tensorflow/contrib/nccl/kernels/nccl_rewrite.cc b/tensorflow/contrib/nccl/kernels/nccl_rewrite.cc
index 94a77c59da..a4de46a93f 100644
--- a/tensorflow/contrib/nccl/kernels/nccl_rewrite.cc
+++ b/tensorflow/contrib/nccl/kernels/nccl_rewrite.cc
@@ -117,6 +117,7 @@ Status ReplaceBroadcast(Graph* graph, Node* node) {
   TF_RETURN_IF_ERROR(GetNodeAttr(node->attrs(), "T", &dtype));
   int send_dev = node->assigned_device_name_index();
   int num_devices = 0;  // Number of distinct devices, incremented below.
+  std::vector<int> recv_index_map;  // Map device name index to stable index.
 
   // Map device name index to nodes that take the broadcast as input.
   std::vector<std::forward_list<NodeBuilder::NodeOut>> out_nodes_map;
@@ -126,9 +127,11 @@ Status ReplaceBroadcast(Graph* graph, Node* node) {
                       : edge->dst()->assigned_device_name_index();
     if (out_nodes_map.size() <= dst_dev) {
       out_nodes_map.resize(dst_dev + 1);
+      recv_index_map.resize(dst_dev + 1);
     }
     auto it = out_nodes_map.begin() + dst_dev;
     if (it->empty()) {
+      recv_index_map[dst_dev] = num_devices;
       ++num_devices;
     }
     it->emplace_front(NodeBuilder::NodeOut(edge->dst(), edge->dst_input()));
@@ -211,16 +214,18 @@ Status ReplaceBroadcast(Graph* graph, Node* node) {
     if (out_nodes_map[recv_dev].empty()) {
       continue;
     }
+    int recv_index = recv_index_map[recv_dev];
     if (is_fully_defined) {
       // If the shape is fully defined, define one const node per device.
-      NodeBuilder shape_builder(strings::StrCat(shape_name, recv_dev), "Const");
+      NodeBuilder shape_builder(strings::StrCat(shape_name, recv_index),
+                                "Const");
       shape_builder.Attr("value", tensor_proto).Attr("dtype", DT_INT32);
       TF_RETURN_IF_ERROR(shape_builder.Finalize(graph, &shape_node));
       shape_node->set_assigned_device_name_index(recv_dev);
     }
     Node* recv_node;
     TF_RETURN_IF_ERROR(
-        make_builder("_NcclBroadcastRecv", strings::StrCat("Recv_", recv_dev))
+        make_builder("_NcclBroadcastRecv", strings::StrCat("Recv_", recv_index))
             .Input(shape_node)
             .Finalize(graph, &recv_node));
     recv_node->set_assigned_device_name_index(recv_dev);
diff --git a/tensorflow/contrib/nccl/python/ops/nccl_ops_test.py b/tensorflow/contrib/nccl/python/ops/nccl_ops_test.py
index 255409303a..0b13e3595e 100644
--- a/tensorflow/contrib/nccl/python/ops/nccl_ops_test.py
+++ b/tensorflow/contrib/nccl/python/ops/nccl_ops_test.py
@@ -117,7 +117,8 @@ class NcclTestCase(test.TestCase):
       inputs = [array_ops.placeholder(t.dtype, t.shape) for t in tensors]
       reduce_tensors = nccl_reduce(inputs, devices)
       losses = _DeviceTensors(tensors, [t.device for t in reduce_tensors])
-      grads = gradients.gradients(reduce_tensors, inputs, losses)
+      grads = gradients.gradients(
+          reduce_tensors, inputs, losses, colocate_gradients_with_ops=True)
       return [g for g in grads if g is not None]
 
     self._Test(_Gradient, numpy_fn)
@@ -159,7 +160,7 @@ class BroadcastTest(NcclTestCase):
   def testBroadcastSingleDevice(self):
     # Broadcasts on a single device are removed completely during rewrite.
     self._Test(_NcclBroadcast, lambda x, y: x,
-               (['/device:GPU:0', '/device:GPU:0']))
+               (['/device:GPU:0', '/device:GPU:0'],))
 
   def testBroadcastToCpuError(self):
     # Broadcasts to CPU is not supported.
@@ -167,10 +168,7 @@ class BroadcastTest(NcclTestCase):
         errors.NotFoundError,
         "No registered '_NcclBroadcastRecv' OpKernel for CPU devices"):
       self._Test(_NcclBroadcast, lambda x, y: x,
-                 (['/device:GPU:0', '/device:CPU:0']))
-
-  def testBroadcastGrad(self):
-    self._TestGradient(_NcclBroadcast, lambda x, y: x + y)
+                 (['/device:GPU:0', '/device:CPU:0'],))
 
 
 class CombinedTest(NcclTestCase):
diff --git a/tensorflow/python/ops/gradients_impl.py b/tensorflow/python/ops/gradients_impl.py
index f7b72eb82f..eb34a35a2b 100644
--- a/tensorflow/python/ops/gradients_impl.py
+++ b/tensorflow/python/ops/gradients_impl.py
@@ -227,53 +227,52 @@ def _DefaultGradYs(grad_ys, ys, colocate_gradients_with_ops):
   for i in xrange(len(grad_ys)):
     grad_y = grad_ys[i]
     y = ys[i]
-    if grad_y is None:
-      if y.dtype.is_complex:
-        raise TypeError(
-            "Gradients of complex tensors must set grad_ys (y.dtype = %r)" %
-            y.dtype)
-      with _maybe_colocate_with(y.op, colocate_gradients_with_ops):
+    with _maybe_colocate_with(y.op, colocate_gradients_with_ops):
+      if grad_y is None:
+        if y.dtype.is_complex:
+          raise TypeError(
+              "Gradients of complex tensors must set grad_ys (y.dtype = %r)" %
+              y.dtype)
         new_grad_ys.append(array_ops.fill(
             array_ops.shape(y), constant_op.constant(
                 1, dtype=y.dtype, name="grad_ys_%d" % i)))
-      continue
-    if y.dtype.is_floating or y.dtype.is_integer:
-      if not grad_y.dtype.is_floating and not grad_y.dtype.is_integer:
-        raise TypeError("Gradient type %s generated for real or "
-                        "integer-valued tensor %s with type %s must be "
-                        "real or integer" %
-                        (dtypes.as_dtype(grad_y.dtype).name, y,
-                         dtypes.as_dtype(y.dtype).name))
-    elif y.dtype.is_complex:
-      if not grad_y.dtype.is_complex:
-        raise TypeError("Gradient type %s generated for complex-valued "
-                        "tensor %s with type %s must be real" %
-                        (dtypes.as_dtype(grad_y.dtype).name, y,
-                         dtypes.as_dtype(y.dtype).name))
-    else:
-      raise TypeError("Tensor %s with type %s must be numeric "
-                      "to obtain a default gradient" %
-                      (y, dtypes.as_dtype(y.dtype).name))
-    # Create a grad_y tensor in the name scope of the gradient.
-    # Required for TensorArrays to identify which gradient call a
-    # grad_y value is coming from.
-    if isinstance(grad_y, ops.IndexedSlices):
-      new_grad_ys.append(
-          ops.IndexedSlices(
-              indices=(array_ops.identity(grad_y.indices,
-                                          name="grad_ys_%d_indices" % i)
-                       if isinstance(grad_y.indices, ops.Tensor)
-                       else grad_y.indices),
-              values=(array_ops.identity(grad_y.values,
-                                         name="grad_ys_%d_values" % i)
-                      if isinstance(grad_y.values, ops.Tensor)
-                      else grad_y.values),
-              dense_shape=(array_ops.identity(grad_y.dense_shape,
-                                              name="grad_ys_%d_shape" % i)
-                           if isinstance(grad_y.dense_shape, ops.Tensor)
-                           else grad_y.dense_shape)))
-    else:
-      new_grad_ys.append(array_ops.identity(grad_y, name="grad_ys_%d" % i))
+        continue
+      if y.dtype.is_floating or y.dtype.is_integer:
+        if not grad_y.dtype.is_floating and not grad_y.dtype.is_integer:
+          raise TypeError("Gradient type %s generated for real or "
+                          "integer-valued tensor %s with type %s must be "
+                          "real or integer" %
+                          (dtypes.as_dtype(grad_y.dtype).name, y,
+                           dtypes.as_dtype(y.dtype).name))
+      elif y.dtype.is_complex:
+        if not grad_y.dtype.is_complex:
+          raise TypeError("Gradient type %s generated for complex-valued "
+                          "tensor %s with type %s must be real" %
+                          (dtypes.as_dtype(grad_y.dtype).name, y,
+                           dtypes.as_dtype(y.dtype).name))
+      else:
+        raise TypeError("Tensor %s with type %s must be numeric "
+                        "to obtain a default gradient" %
+                        (y, dtypes.as_dtype(y.dtype).name))
+      # Create a grad_y tensor in the name scope of the gradient.
+      # Required for TensorArrays to identify which gradient call a
+      # grad_y value is coming from.
+      if isinstance(grad_y, ops.IndexedSlices):
+        new_grad_ys.append(
+            ops.IndexedSlices(
+                indices=(array_ops.identity(
+                    grad_y.indices, name="grad_ys_%d_indices" % i)
+                         if isinstance(grad_y.indices, ops.Tensor) else
+                         grad_y.indices),
+                values=(array_ops.identity(
+                    grad_y.values, name="grad_ys_%d_values" % i) if isinstance(
+                        grad_y.values, ops.Tensor) else grad_y.values),
+                dense_shape=(array_ops.identity(
+                    grad_y.dense_shape, name="grad_ys_%d_shape" % i)
+                             if isinstance(grad_y.dense_shape, ops.Tensor) else
+                             grad_y.dense_shape)))
+      else:
+        new_grad_ys.append(array_ops.identity(grad_y, name="grad_ys_%d" % i))
 
   return new_grad_ys
 
-- 
GitLab


From 134daeb4151349acf8c2b3c22f5aebc3e429d756 Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Tue, 24 Oct 2017 13:55:22 -0700
Subject: [PATCH 725/909] Eager reuse story is False instead of AUTO_REUSE.

We want variables with eager execution to have object semantics instead of name semantics and this is a small step in that direction.

This means that the functional style layer invocations (tf.layers.dense() etc.)
will NOT work when eager execution is enabled. Instead, use of the object-oriented
layers is advised.

PiperOrigin-RevId: 173306447
---
 .../kernel_tests/variable_scope_test.py       | 62 +++++++++----------
 tensorflow/python/ops/variable_scope.py       | 62 +++++++++----------
 2 files changed, 60 insertions(+), 64 deletions(-)

diff --git a/tensorflow/python/kernel_tests/variable_scope_test.py b/tensorflow/python/kernel_tests/variable_scope_test.py
index 0ea58b4402..29f583d5ba 100644
--- a/tensorflow/python/kernel_tests/variable_scope_test.py
+++ b/tensorflow/python/kernel_tests/variable_scope_test.py
@@ -39,7 +39,6 @@ from tensorflow.python.platform import test
 
 class VariableScopeTest(test.TestCase):
 
-  @test_util.run_in_graph_and_eager_modes()
   def testGetVar(self):
     vs = variable_scope._get_default_variable_store()
     v = vs.get_variable("v", [1])
@@ -52,7 +51,6 @@ class VariableScopeTest(test.TestCase):
     v1 = vs.get_variable("v", [1], use_resource=True)
     self.assertTrue(isinstance(v1, resource_variable_ops.ResourceVariable))
 
-  @test_util.run_in_graph_and_eager_modes()
   def testNameExists(self):
     vs = variable_scope._get_default_variable_store()
     # No check by default, so we can both create and get existing names.
@@ -60,15 +58,14 @@ class VariableScopeTest(test.TestCase):
     v1 = vs.get_variable("v", [1])
     self.assertEqual(v, v1)
 
-    if context.in_graph_mode():
-      # When reuse is False, we fail when variables are already there.
-      vs.get_variable("w", [1], reuse=False)  # That's ok.
-      with self.assertRaises(ValueError):
-        vs.get_variable("v", [1], reuse=False)  # That fails.
-      # When reuse is True, we fail when variables are new.
-      vs.get_variable("v", [1], reuse=True)  # That's ok.
-      with self.assertRaises(ValueError):
-        vs.get_variable("u", [1], reuse=True)  # That fails.
+    # When reuse is False, we fail when variables are already there.
+    vs.get_variable("w", [1], reuse=False)  # That's ok.
+    with self.assertRaises(ValueError):
+      vs.get_variable("v", [1], reuse=False)  # That fails.
+    # When reuse is True, we fail when variables are new.
+    vs.get_variable("v", [1], reuse=True)  # That's ok.
+    with self.assertRaises(ValueError):
+      vs.get_variable("u", [1], reuse=True)  # That fails.
 
   @test_util.run_in_graph_and_eager_modes()
   def testNamelessStore(self):
@@ -224,10 +221,12 @@ class VariableScopeTest(test.TestCase):
         self.assertAllClose(self.evaluate(losses[1]), 0.4)
         self.assertAllClose(self.evaluate(losses[2]), 0.5)
       with variable_scope.variable_scope("foo", reuse=True):
-        v = variable_scope.get_variable("v",
-                                        [])  # "v" is alredy there, reused
-        losses = ops.get_collection(ops.GraphKeys.REGULARIZATION_LOSSES)
-        self.assertEqual(3, len(losses))  # No new loss added.
+        # reuse=True is for now only supported when eager execution is disabled.
+        if context.in_graph_mode():
+          v = variable_scope.get_variable("v",
+                                          [])  # "v" is alredy there, reused
+          losses = ops.get_collection(ops.GraphKeys.REGULARIZATION_LOSSES)
+          self.assertEqual(3, len(losses))  # No new loss added.
 
   @test_util.run_in_graph_and_eager_modes()
   def testInitializeFromValue(self):
@@ -439,20 +438,20 @@ class VariableScopeTest(test.TestCase):
       with variable_scope.variable_scope(vs, reuse=False) as jump_no_reuse:
         self.assertFalse(jump_no_reuse.reuse)
 
-  @test_util.run_in_graph_and_eager_modes()
   def testVarScopeGetOrCreateReuse(self):
-    def test_value(value):
-      x = constant_op.constant(value)
-      with variable_scope.variable_scope("testVarScopeGetOrCreateReuse_bar",
-                                         reuse=variable_scope.AUTO_REUSE):
-        _ = state_ops.assign(variable_scope.get_variable("var", []), x)
-      with variable_scope.variable_scope("testVarScopeGetOrCreateReuse_bar",
-                                         reuse=variable_scope.AUTO_REUSE):
-        _ = variable_scope.get_variable("var", [])
-      self.assertEqual(value, self.evaluate(x))
-    test_value(42.)  # Variable is created.
-    test_value(13.)  # Variable is reused hereafter.
-    test_value(17.)
+    with self.test_session():
+      def test_value(value):
+        x = constant_op.constant(value)
+        with variable_scope.variable_scope("testVarScopeGetOrCreateReuse_bar",
+                                           reuse=variable_scope.AUTO_REUSE):
+          _ = state_ops.assign(variable_scope.get_variable("var", []), x)
+        with variable_scope.variable_scope("testVarScopeGetOrCreateReuse_bar",
+                                           reuse=variable_scope.AUTO_REUSE):
+          _ = variable_scope.get_variable("var", [])
+        self.assertEqual(value, x.eval())
+      test_value(42.)  # Variable is created.
+      test_value(13.)  # Variable is reused hereafter.
+      test_value(17.)
 
   def testVarOpScope(self):
     with self.test_session():
@@ -745,9 +744,10 @@ class VariableScopeTest(test.TestCase):
                        ops.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES))
 
     # Check that local variable respects `reuse`.
-    with variable_scope.variable_scope(outer, "default", reuse=True):
-      self.assertEqual(
-          variable_scope.get_local_variable("w", []).name, "outer/w:0")
+    if context.in_graph_mode():
+      with variable_scope.variable_scope(outer, "default", reuse=True):
+        self.assertEqual(
+            variable_scope.get_local_variable("w", []).name, "outer/w:0")
 
   def testGetVarWithDevice(self):
     g = ops.Graph()
diff --git a/tensorflow/python/ops/variable_scope.py b/tensorflow/python/ops/variable_scope.py
index 22048a0cef..8c5c639b68 100644
--- a/tensorflow/python/ops/variable_scope.py
+++ b/tensorflow/python/ops/variable_scope.py
@@ -259,8 +259,8 @@ class _VariableStore(object):
         applying it on a newly created variable will be added to the collection
         GraphKeys.REGULARIZATION_LOSSES and can be used for regularization.
       reuse: a Boolean, None, or tf.AUTO_REUSE. Controls reuse or creation
-        of variables. In Eager mode, this argument is always forced to be
-        tf.AUTO_REUSE.
+        of variables. When eager execution is enabled  this argument is always
+        forced to be False.
       trainable: If `True` also add the variable to the graph collection
         `GraphKeys.TRAINABLE_VARIABLES` (see `tf.Variable`).
       collections: List of graph collections keys to add the `Variable` to.
@@ -279,7 +279,8 @@ class _VariableStore(object):
       use_resource: If False, creates a regular Variable. If True, creates
         instead an experimental ResourceVariable which has well-defined
         semantics. Defaults to False (will later change to True).
-        In Eager mode, this argument is always forced to be true.
+        When eager execution is enabled this argument is always forced to be
+        true.
       custom_getter: Callable that takes as a first argument the true getter,
         and allows overwriting the internal get_variable method.
         The signature of `custom_getter` should match that of this method,
@@ -314,7 +315,7 @@ class _VariableStore(object):
           "Passed a custom_getter which is not callable: %s" % custom_getter)
 
     if context.in_eager_mode():
-      reuse = AUTO_REUSE
+      reuse = False
       use_resource = True
 
     # If a *_ref type is passed in an error would be triggered further down the
@@ -506,7 +507,7 @@ class _VariableStore(object):
     """
     if context.in_eager_mode():
       raise NotImplementedError("Partitioned variables are not yet supported "
-                                "in Eager mode.")
+                                "when eager execution is enabled.")
 
     initializing_from_value = initializer is not None and isinstance(
         initializer, ops.Tensor)
@@ -710,15 +711,6 @@ class _VariableStore(object):
     Raises:
       ValueError: See documentation of get_variable above.
     """
-    # Fast-path for get_variable in eager mode when the variable already
-    # exists. Note this skips error validation code, so mismatched shapes and
-    # dtypes will be caught when the variable is used instead of when the call
-    # to get_variable happens.
-    if context.in_eager_mode():
-      v = self._vars.get(name, None)
-      if v is not None:
-        return v
-
     # Set to true if initializer is a constant.
     initializing_from_value = False
     if initializer is not None and not callable(initializer):
@@ -732,6 +724,9 @@ class _VariableStore(object):
     if name in self._vars:
       # Here we handle the case when returning an existing variable.
       if reuse is False:
+        if context.in_eager_mode():
+          raise ValueError(
+              "Trying to recreate existing variable: %s" % self._vars[name])
         tb = self._vars[name].op.traceback[::-1]
         # Throw away internal tf entries and only take a few lines.
         tb = [x for x in tb if "tensorflow/python" not in x[0]][:3]
@@ -875,8 +870,8 @@ class VariableScope(object):
     initializer: default initializer passed to get_variable.
     regularizer: default regularizer passed to get_variable.
     reuse: Boolean, None, or tf.AUTO_REUSE, setting the reuse in
-      get_variable. In Eager mode, this argument is always forced to be
-      tf.AUTO_REUSE.
+      get_variable. When eager execution is enabled this argument is always
+      forced to be False.
     caching_device: string, callable, or None: the caching device passed to
       get_variable.
     partitioner: callable or `None`: the partitioner passed to `get_variable`.
@@ -885,8 +880,8 @@ class VariableScope(object):
     dtype: default type passed to get_variable (defaults to DT_FLOAT).
     use_resource: if False, create a normal Variable; if True create an
       experimental ResourceVariable with well-defined semantics. Defaults
-      to False (will later change to True). In Eager mode, this argument is
-      always forced to be True.
+      to False (will later change to True). When eager execution is enabled
+      this argument is always forced to be True.
     constraint: An optional projection function to be applied to the variable
       after being updated by an `Optimizer` (e.g. used to implement norm
       constraints or value constraints for layer weights). The function must
@@ -923,10 +918,10 @@ class VariableScope(object):
     if context.in_eager_mode():
       if self._caching_device is not None:
         raise NotImplementedError("Caching devices is not yet supported "
-                                  "in Eager mode.")
+                                  "when eager execution is enabled.")
       if self._partitioner is not None:
         raise NotImplementedError("Partitioned variables are not yet supported "
-                                  "in Eager mode.")
+                                  "when eager execution is enabled.")
       self._reuse = AUTO_REUSE
       self._use_resource = True
 
@@ -989,7 +984,8 @@ class VariableScope(object):
   def set_use_resource(self, use_resource):
     """Sets whether to use ResourceVariables for this scope."""
     if context.in_eager_mode() and not use_resource:
-      raise ValueError("In eager mode, use_resource cannot be set to false.")
+      raise ValueError("When eager execution is enabled, "
+                       "use_resource cannot be set to false.")
     self._use_resource = use_resource
 
   def set_regularizer(self, regularizer):
@@ -1000,14 +996,14 @@ class VariableScope(object):
     """Set caching_device for this scope."""
     if context.in_eager_mode():
       raise NotImplementedError("Caching devices are not yet supported "
-                                "in Eager mode.")
+                                "when eager execution is enabled.")
     self._caching_device = caching_device
 
   def set_partitioner(self, partitioner):
     """Set partitioner for this scope."""
     if partitioner and context.in_eager_mode():
       raise NotImplementedError("Partitioned variables are not yet supported "
-                                "in Eager mode.")
+                                "when eager execution is enabled.")
     self._partitioner = partitioner
 
   def set_custom_getter(self, custom_getter):
@@ -1062,7 +1058,7 @@ class VariableScope(object):
       if use_resource is None:
         use_resource = self._use_resource
     else:
-      reuse = AUTO_REUSE
+      reuse = False
       use_resource = True
 
     full_name = self.name + "/" + name if self.name else name
@@ -1108,7 +1104,7 @@ class VariableScope(object):
     """Gets an existing variable with this name or create a new one."""
     if context.in_eager_mode():
       raise NotImplementedError("Partitioned variables are not yet supported "
-                                "in Eager mode.")
+                                "when eager execution is enabled.")
     if initializer is None:
       initializer = self._initializer
     if regularizer is None:
@@ -1259,8 +1255,8 @@ Args:
       must be known.
   use_resource: If False, creates a regular Variable. If true, creates an
     experimental ResourceVariable instead with well-defined semantics.
-    Defaults to False (will later change to True). In Eager mode, this argument
-    is always forced to be True.
+    Defaults to False (will later change to True). When eager execution is
+    enabled this argument is always forced to be True.
   custom_getter: Callable that takes as a first argument the true getter, and
     allows overwriting the internal get_variable method.
     The signature of `custom_getter` should match that of this method,
@@ -1721,14 +1717,14 @@ class variable_scope(object):  # pylint: disable=invalid-name
       reuse: `True`, None, or tf.AUTO_REUSE; if `True`, we go into reuse mode
         for this scope as well as all sub-scopes; if tf.AUTO_REUSE, we create
         variables if they do not exist, and return them otherwise; if None, we
-        inherit the parent scope's reuse flag. In Eager mode, this argument is
-        always forced to be tf.AUTO_REUSE.
+        inherit the parent scope's reuse flag. When eager execution is enabled,
+        this argument is always forced to be tf.AUTO_REUSE.
       dtype: type of variables created in this scope (defaults to the type
         in the passed scope, or inherited from parent scope).
       use_resource: If False, all variables will be regular Variables. If True,
         experimental ResourceVariables with well-defined semantics will be used
-        instead. Defaults to False (will later change to True). In Eager mode,
-        this argument is always forced to be True.
+        instead. Defaults to False (will later change to True). When eager
+        execution is enabled this argument is always forced to be True.
       constraint: An optional projection function to be applied to the variable
         after being updated by an `Optimizer` (e.g. used to implement norm
         constraints or value constraints for layer weights). The function must
@@ -1935,8 +1931,8 @@ def variable(initial_value=None,
         caching_device=caching_device, name=name, dtype=dtype)
   elif not use_resource and context.in_eager_mode():
     raise RuntimeError(
-        "VariableScope should use resource variable in Eager mode, but "
-        "use_resource is False."
+        "VariableScope should use resource variable when eager execution is"
+        " enabled, but use_resource is False."
     )
   else:
     return variables.Variable(
-- 
GitLab


From de1b4a8a75ae3a50f4fa7480efb1177d79abf553 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 24 Oct 2017 14:00:10 -0700
Subject: [PATCH 726/909] Refactor K-FAC FisherEstimator

PiperOrigin-RevId: 173307212
---
 .../contrib/kfac/python/kernel_tests/BUILD    |   2 +
 .../python/kernel_tests/estimator_test.py     |  68 +++++++---
 .../contrib/kfac/python/ops/estimator.py      | 116 +++++++++---------
 3 files changed, 115 insertions(+), 71 deletions(-)

diff --git a/tensorflow/contrib/kfac/python/kernel_tests/BUILD b/tensorflow/contrib/kfac/python/kernel_tests/BUILD
index fd4f588741..8980f03092 100644
--- a/tensorflow/contrib/kfac/python/kernel_tests/BUILD
+++ b/tensorflow/contrib/kfac/python/kernel_tests/BUILD
@@ -13,6 +13,8 @@ py_test(
     deps = [
         "//tensorflow/contrib/kfac/python/ops:fisher_estimator",
         "//tensorflow/contrib/kfac/python/ops:layer_collection",
+        "//tensorflow/contrib/kfac/python/ops:utils",
+        "//tensorflow/python:array_ops",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:dtypes",
         "//tensorflow/python:framework_ops",
diff --git a/tensorflow/contrib/kfac/python/kernel_tests/estimator_test.py b/tensorflow/contrib/kfac/python/kernel_tests/estimator_test.py
index 281274d884..b52a7b52a7 100644
--- a/tensorflow/contrib/kfac/python/kernel_tests/estimator_test.py
+++ b/tensorflow/contrib/kfac/python/kernel_tests/estimator_test.py
@@ -20,42 +20,80 @@ from __future__ import print_function
 
 from tensorflow.contrib.kfac.python.ops import estimator
 from tensorflow.contrib.kfac.python.ops import layer_collection as lc
+from tensorflow.contrib.kfac.python.ops import utils
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
+from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import init_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import random_ops
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.platform import test
 
+_ALL_ESTIMATION_MODES = ["gradients", "empirical", "curvature_prop", "exact"]
+
 
 class EstimatorTest(test.TestCase):
 
-  def testEstimatorInitManualRegistration(self):
-    with ops.Graph().as_default():
-      layer_collection = lc.LayerCollection()
+  def setUp(self):
+    self._graph = ops.Graph()
+    with self._graph.as_default():
+      self.layer_collection = lc.LayerCollection()
 
-      inputs = random_ops.random_normal((2, 2), dtype=dtypes.float32)
-      weights = variable_scope.get_variable(
-          'w', shape=(2, 2), dtype=dtypes.float32)
-      bias = variable_scope.get_variable(
-          'b', initializer=init_ops.zeros_initializer(), shape=(2, 1))
-      output = math_ops.matmul(inputs, weights) + bias
+      self.inputs = random_ops.random_normal((2, 2), dtype=dtypes.float32)
+      self.weights = variable_scope.get_variable(
+          "w", shape=(2, 2), dtype=dtypes.float32)
+      self.bias = variable_scope.get_variable(
+          "b", initializer=init_ops.zeros_initializer(), shape=(2, 1))
+      self.output = math_ops.matmul(self.inputs, self.weights) + self.bias
 
       # Only register the weights.
-      layer_collection.register_fully_connected((weights,), inputs, output)
+      self.layer_collection.register_fully_connected(
+          params=(self.weights,), inputs=self.inputs, outputs=self.output)
 
-      outputs = math_ops.tanh(output)
-      layer_collection.register_categorical_predictive_distribution(outputs)
+      self.outputs = math_ops.tanh(self.output)
+      self.targets = array_ops.zeros_like(self.outputs)
+      self.layer_collection.register_categorical_predictive_distribution(
+          logits=self.outputs, targets=self.targets)
 
+  def testEstimatorInitManualRegistration(self):
+    with self._graph.as_default():
       # We should be able to build an estimator for only the registered vars.
-      estimator.FisherEstimator([weights], 0.1, 0.2, layer_collection)
+      estimator.FisherEstimator([self.weights], 0.1, 0.2, self.layer_collection)
 
       # Check that we throw an error if we try to build an estimator for vars
       # that were not manually registered.
       with self.assertRaises(ValueError):
-        estimator.FisherEstimator([weights, bias], 0.1, 0.2, layer_collection)
+        estimator.FisherEstimator([self.weights, self.bias], 0.1, 0.2,
+                                  self.layer_collection)
+
+      # Check that we throw an error if we don't include registered variables,
+      # i.e. self.weights
+      with self.assertRaises(ValueError):
+        estimator.FisherEstimator([], 0.1, 0.2, self.layer_collection)
+
+  @test.mock.patch.object(utils.SubGraph, "variable_uses", return_value=42)
+  def testVariableWrongNumberOfUses(self, mock_uses):
+    with self.assertRaises(ValueError):
+      estimator.FisherEstimator([self.weights], 0.1, 0.2, self.layer_collection)
+
+  def testInvalidEstimationMode(self):
+    with self.assertRaises(ValueError):
+      estimator.FisherEstimator([self.weights], 0.1, 0.2, self.layer_collection,
+                                "not_a_real_mode")
+
+  def testModeListCorrect(self):
+    with self._graph.as_default():
+      est = estimator.FisherEstimator([self.weights], 0.1, 0.2,
+                                      self.layer_collection)
+    self.assertItemsEqual(_ALL_ESTIMATION_MODES, est._gradient_fns.keys())
+
+  def testAllModesBuild(self):
+    for mode in _ALL_ESTIMATION_MODES:
+      with self._graph.as_default():
+        estimator.FisherEstimator([self.weights], 0.1, 0.2,
+                                  self.layer_collection, mode)
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/contrib/kfac/python/ops/estimator.py b/tensorflow/contrib/kfac/python/ops/estimator.py
index c81086416c..6e2c9ecdce 100644
--- a/tensorflow/contrib/kfac/python/ops/estimator.py
+++ b/tensorflow/contrib/kfac/python/ops/estimator.py
@@ -80,6 +80,12 @@ class FisherEstimator(object):
     self._layers = layer_collection
     self._layers.create_subgraph()
     self._check_registration(variables)
+    self._gradient_fns = {
+        "gradients": self._get_grads_lists_gradients,
+        "empirical": self._get_grads_lists_empirical,
+        "curvature_prop": self._get_grads_lists_curvature_prop,
+        "exact": self._get_grads_lists_exact
+    }
     setup = self._setup(cov_ema_decay)
     self.cov_update_op, self.inv_update_op, self.inv_updates_dict = setup
 
@@ -201,75 +207,73 @@ class FisherEstimator(object):
     Raises:
       ValueError: If estimation_mode was improperly specified at construction.
     """
-    damping = self.damping
-
     fisher_blocks_list = self._layers.get_blocks()
-
     tensors_to_compute_grads = [
         fb.tensors_to_compute_grads() for fb in fisher_blocks_list
     ]
-    tensors_to_compute_grads_flat = nest.flatten(tensors_to_compute_grads)
-
-    if self._estimation_mode == "gradients":
-      grads_flat = gradients_impl.gradients(self._layers.total_sampled_loss(),
-                                            tensors_to_compute_grads_flat)
-      grads_all = nest.pack_sequence_as(tensors_to_compute_grads, grads_flat)
-      grads_lists = tuple((grad,) for grad in grads_all)
-
-    elif self._estimation_mode == "empirical":
-      grads_flat = gradients_impl.gradients(self._layers.total_loss(),
-                                            tensors_to_compute_grads_flat)
-      grads_all = nest.pack_sequence_as(tensors_to_compute_grads, grads_flat)
-      grads_lists = tuple((grad,) for grad in grads_all)
-
-    elif self._estimation_mode == "curvature_prop":
-      loss_inputs = list(loss.inputs for loss in self._layers.losses)
-      loss_inputs_flat = nest.flatten(loss_inputs)
-
-      transformed_random_signs = list(loss.multiply_fisher_factor(
-          utils.generate_random_signs(loss.fisher_factor_inner_shape))
-                                      for loss in self._layers.losses)
-
-      transformed_random_signs_flat = nest.flatten(transformed_random_signs)
-
-      grads_flat = gradients_impl.gradients(loss_inputs_flat,
-                                            tensors_to_compute_grads_flat,
-                                            grad_ys
-                                            =transformed_random_signs_flat)
-      grads_all = nest.pack_sequence_as(tensors_to_compute_grads, grads_flat)
-      grads_lists = tuple((grad,) for grad in grads_all)
-
-    elif self._estimation_mode == "exact":
-      # Loop over all coordinates of all losses.
-      grads_all = []
-      for loss in self._layers.losses:
-        for index in np.ndindex(*loss.fisher_factor_inner_static_shape[1:]):
-          transformed_one_hot = loss.multiply_fisher_factor_replicated_one_hot(
-              index)
-          grads_flat = gradients_impl.gradients(loss.inputs,
-                                                tensors_to_compute_grads_flat,
-                                                grad_ys=transformed_one_hot)
-          grads_all.append(nest.pack_sequence_as(tensors_to_compute_grads,
-                                                 grads_flat))
-
-      grads_lists = zip(*grads_all)
-
-    else:
+
+    try:
+      grads_lists = self._gradient_fns[self._estimation_mode](
+          tensors_to_compute_grads)
+    except KeyError:
       raise ValueError("Unrecognized value {} for estimation_mode.".format(
           self._estimation_mode))
 
     for grads_list, fb in zip(grads_lists, fisher_blocks_list):
-      fb.instantiate_factors(grads_list, damping)
+      fb.instantiate_factors(grads_list, self.damping)
 
     cov_updates = [
         factor.make_covariance_update_op(cov_ema_decay)
         for factor in self._layers.get_factors()
     ]
-    inv_updates = {
-        op.name: op
-        for factor in self._layers.get_factors()
-        for op in factor.make_inverse_update_ops()
-    }
+    inv_updates = {op.name: op for op in self._get_all_inverse_update_ops()}
 
     return control_flow_ops.group(*cov_updates), control_flow_ops.group(
         *inv_updates.values()), inv_updates
+
+  def _get_all_inverse_update_ops(self):
+    for factor in self._layers.get_factors():
+      for op in factor.make_inverse_update_ops():
+        yield op
+
+  def _get_grads_lists_gradients(self, tensors):
+    grads_flat = gradients_impl.gradients(self._layers.total_sampled_loss(),
+                                          nest.flatten(tensors))
+    grads_all = nest.pack_sequence_as(tensors, grads_flat)
+    return tuple((grad,) for grad in grads_all)
+
+  def _get_grads_lists_empirical(self, tensors):
+    grads_flat = gradients_impl.gradients(self._layers.total_loss(),
+                                          nest.flatten(tensors))
+    grads_all = nest.pack_sequence_as(tensors, grads_flat)
+    return tuple((grad,) for grad in grads_all)
+
+  def _get_transformed_random_signs(self):
+    transformed_random_signs = []
+    for loss in self._layers.losses:
+      transformed_random_signs.append(
+          loss.multiply_fisher_factor(
+              utils.generate_random_signs(loss.fisher_factor_inner_shape)))
+    return transformed_random_signs
+
+  def _get_grads_lists_curvature_prop(self, tensors):
+    loss_inputs = list(loss.inputs for loss in self._layers.losses)
+    transformed_random_signs = self._get_transformed_random_signs()
+    grads_flat = gradients_impl.gradients(
+        nest.flatten(loss_inputs),
+        nest.flatten(tensors),
+        grad_ys=nest.flatten(transformed_random_signs))
+    grads_all = nest.pack_sequence_as(tensors, grads_flat)
+    return tuple((grad,) for grad in grads_all)
+
+  def _get_grads_lists_exact(self, tensors):
+    # Loop over all coordinates of all losses.
+    grads_all = []
+    for loss in self._layers.losses:
+      for index in np.ndindex(*loss.fisher_factor_inner_static_shape[1:]):
+        transformed_one_hot = loss.multiply_fisher_factor_replicated_one_hot(
+            index)
+        grads_flat = gradients_impl.gradients(
+            loss.inputs, nest.flatten(tensors), grad_ys=transformed_one_hot)
+        grads_all.append(nest.pack_sequence_as(tensors, grads_flat))
+    return zip(*grads_all)
-- 
GitLab


From 01365dbc2c257ff2ab409a2a5122a06739272737 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 24 Oct 2017 14:10:14 -0700
Subject: [PATCH 727/909] Allow lists to be passed to tf.group().

PiperOrigin-RevId: 173308794
---
 tensorflow/python/ops/control_flow_ops.py      |  7 +++++--
 tensorflow/python/ops/control_flow_ops_test.py | 14 +++++++++++---
 2 files changed, 16 insertions(+), 5 deletions(-)

diff --git a/tensorflow/python/ops/control_flow_ops.py b/tensorflow/python/ops/control_flow_ops.py
index dcdbeefb70..10d8e01304 100644
--- a/tensorflow/python/ops/control_flow_ops.py
+++ b/tensorflow/python/ops/control_flow_ops.py
@@ -2910,7 +2910,7 @@ def _GroupControlDeps(dev, deps, name=None):
 def group(*inputs, **kwargs):
   """Create an op that groups multiple operations.
 
-  When this op finishes, all ops in `input` have finished. This op has no
+  When this op finishes, all ops in `inputs` have finished. This op has no
   output.
 
   See also @{tf.tuple$tuple} and
@@ -2938,7 +2938,10 @@ def group(*inputs, **kwargs):
 
     # Sorts *inputs according to their devices.
     ops_on_device = {}  # device -> operations specified on the device.
-    for inp in inputs:
+    for inp in nest.flatten(inputs):
+      if not hasattr(inp, "device"):
+        raise TypeError("Expected tf.group() expected Tensor arguments not "
+                        "'%s' with type '%s'" % (inp, type(inp)))
       if not hasattr(inp, "device"):
         if isinstance(inp, list):
           raise TypeError("To call tf.group() with a list, use "
diff --git a/tensorflow/python/ops/control_flow_ops_test.py b/tensorflow/python/ops/control_flow_ops_test.py
index 34c405f293..3e8f39dd24 100644
--- a/tensorflow/python/ops/control_flow_ops_test.py
+++ b/tensorflow/python/ops/control_flow_ops_test.py
@@ -115,11 +115,19 @@ class GroupTestCase(test_util.TensorFlowTestCase):
     """, self._StripGraph(gd))
 
   def testPassingList(self):
-    with ops.Graph().as_default():
+    with ops.Graph().as_default() as g:
       a = constant_op.constant(0, name="a")
       b = constant_op.constant(0, name="b")
-      with self.assertRaises(TypeError):
-        control_flow_ops.group([a.op, b.op])
+      control_flow_ops.group([a.op, b.op], name="root")
+    gd = g.as_graph_def()
+    self.assertProtoEquals("""
+      node { name: "a" op: "Const"}
+      node { name: "b" op: "Const"}
+      node { name: "root" op: "NoOp" input: "^a" input: "^b" }
+    """, self._StripGraph(gd))
+
+  def testPassingNonTensors(self):
+    with ops.Graph().as_default():
       with self.assertRaises(TypeError):
         control_flow_ops.group(1, 2)
 
-- 
GitLab


From 177bd25e5d75ab4b21d9aa25e1cba5ff9dbfddc9 Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Tue, 24 Oct 2017 14:33:17 -0700
Subject: [PATCH 728/909] Nice error messages when using queues / batching in
 eager mode.

PiperOrigin-RevId: 173312134
---
 tensorflow/python/ops/data_flow_ops.py |  5 +++++
 tensorflow/python/training/input.py    | 21 +++++++++++++++++++++
 2 files changed, 26 insertions(+)

diff --git a/tensorflow/python/ops/data_flow_ops.py b/tensorflow/python/ops/data_flow_ops.py
index 41dd7f1467..62845a9f8b 100644
--- a/tensorflow/python/ops/data_flow_ops.py
+++ b/tensorflow/python/ops/data_flow_ops.py
@@ -146,7 +146,12 @@ class QueueBase(object):
 
     Raises:
       ValueError: If one of the arguments is invalid.
+      ValueError: If eager execution is enabled.
     """
+    if context.in_eager_mode():
+      raise ValueError(
+          "Queues are not supported in TensorFlow with eager execution. "
+          "Instead, use tf.data to get data into your model.")
     self._dtypes = dtypes
     if shapes is not None:
       if len(shapes) != len(dtypes):
diff --git a/tensorflow/python/training/input.py b/tensorflow/python/training/input.py
index 36f97960dd..b999dbedb6 100644
--- a/tensorflow/python/training/input.py
+++ b/tensorflow/python/training/input.py
@@ -27,6 +27,7 @@ import collections
 
 from six.moves import xrange  # pylint: disable=redefined-builtin
 
+from tensorflow.python.eager import context
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
@@ -146,6 +147,10 @@ def input_producer(input_tensor,
   Raises:
     ValueError: If the shape of the input cannot be inferred from the arguments.
   """
+  if context.in_eager_mode():
+    raise ValueError(
+        "Queue-using input pipelines are not supported when eager execution is"
+        " enabled. Please use tf.data to ingest data into your model instead.")
   with ops.name_scope(name, "input_producer", [input_tensor]):
     input_tensor = ops.convert_to_tensor(input_tensor, name="input_tensor")
     element_shape = input_tensor.shape[1:].merge_with(element_shape)
@@ -685,6 +690,10 @@ def _batch(tensors, batch_size, keep_input, num_threads=1, capacity=32,
            allow_smaller_final_batch=False, shared_name=None,
            name=None):
   """Helper function for `batch` and `maybe_batch`."""
+  if context.in_eager_mode():
+    raise ValueError(
+        "Queue-using input pipelines are not supported when eager execution is"
+        " enabled. Please use tf.data to ingest data into your model instead.")
   tensor_list = _as_tensor_list(tensors)
   with ops.name_scope(name, "batch", list(tensor_list) + [keep_input]) as name:
     tensor_list = _validate(tensor_list)
@@ -718,6 +727,10 @@ def _batch_join(tensors_list, batch_size, keep_input, capacity=32,
                 enqueue_many=False, shapes=None, dynamic_pad=False,
                 allow_smaller_final_batch=False, shared_name=None, name=None):
   """Helper function for `batch_join` and `maybe_batch_join`."""
+  if context.in_eager_mode():
+    raise ValueError(
+        "Queue-using input pipelines are not supported when eager execution is"
+        " enabled. Please use tf.data to ingest data into your model instead.")
   tensor_list_list = _as_tensor_list_list(tensors_list)
   with ops.name_scope(name, "batch_join",
                       _flatten(tensor_list_list) + [keep_input]) as name:
@@ -748,6 +761,10 @@ def _shuffle_batch(tensors, batch_size, capacity, min_after_dequeue,
                    shapes=None, allow_smaller_final_batch=False,
                    shared_name=None, name=None):
   """Helper function for `shuffle_batch` and `maybe_shuffle_batch`."""
+  if context.in_eager_mode():
+    raise ValueError(
+        "Queue-using input pipelines are not supported when eager execution is"
+        " enabled. Please use tf.data to ingest data into your model instead.")
   tensor_list = _as_tensor_list(tensors)
   with ops.name_scope(name, "shuffle_batch",
                       list(tensor_list) + [keep_input]) as name:
@@ -788,6 +805,10 @@ def _shuffle_batch_join(tensors_list, batch_size, capacity,
                         allow_smaller_final_batch=False, shared_name=None,
                         name=None):
   """Helper function for `shuffle_batch_join` and `maybe_shuffle_batch_join`."""
+  if context.in_eager_mode():
+    raise ValueError(
+        "Queue-using input pipelines are not supported when eager execution is"
+        " enabled. Please use tf.data to ingest data into your model instead.")
   tensor_list_list = _as_tensor_list_list(tensors_list)
   with ops.name_scope(name, "shuffle_batch_join",
                       _flatten(tensor_list_list) + [keep_input]) as name:
-- 
GitLab


From dfc7b26b0dc0dd54038a1be3b31b05bd39c1e79f Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Tue, 24 Oct 2017 14:42:15 -0700
Subject: [PATCH 729/909] Exception instead of crashing on resource.numpy()

PiperOrigin-RevId: 173313459
---
 tensorflow/python/framework/ops.py                          | 5 +++++
 .../python/kernel_tests/resource_variable_ops_test.py       | 6 ++++++
 2 files changed, 11 insertions(+)

diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index b3caebce70..94c29c89df 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -615,7 +615,12 @@ class _EagerTensorBase(Tensor):
     Returns:
       A numpy array that may share memory with the Tensor object. Any changes
       to one may be reflected in the other.
+
+    Raises:
+      ValueError: if the type of this Tensor is not representable in numpy.
     """
+    if self.dtype == dtypes.resource:
+      raise ValueError("Resource handles are not convertible to numpy.")
     return self.cpu()._numpy()  # pylint: disable=protected-access
 
   def __array__(self):
diff --git a/tensorflow/python/kernel_tests/resource_variable_ops_test.py b/tensorflow/python/kernel_tests/resource_variable_ops_test.py
index 10f9a72c7b..a2a1e1dcd8 100644
--- a/tensorflow/python/kernel_tests/resource_variable_ops_test.py
+++ b/tensorflow/python/kernel_tests/resource_variable_ops_test.py
@@ -178,6 +178,12 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase):
       self.assertEqual(v.handle.op.colocation_groups(),
                        v.initializer.inputs[1].op.colocation_groups())
 
+  def testHandleNumpy(self):
+    with context.eager_mode():
+      with self.assertRaises(ValueError):
+        resource_variable_ops.ResourceVariable(
+            1.0, name="handle-numpy").handle.numpy()
+
   @test_util.run_in_graph_and_eager_modes()
   def testInitFnDtype(self):
     v = resource_variable_ops.ResourceVariable(
-- 
GitLab


From 8d1a4fa09cb40ee98ecddc99f207f17b05176897 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 24 Oct 2017 14:45:30 -0700
Subject: [PATCH 730/909] Add a MultiHandler that can conditionally apply
 handling logic based on presence of input Tensors.

PiperOrigin-RevId: 173314020
---
 .../python/slim/data/tfexample_decoder.py     | 34 +++++++++++++
 .../slim/data/tfexample_decoder_test.py       | 51 +++++++++++++++++++
 2 files changed, 85 insertions(+)

diff --git a/tensorflow/contrib/slim/python/slim/data/tfexample_decoder.py b/tensorflow/contrib/slim/python/slim/data/tfexample_decoder.py
index 7a56df9e97..0544404e9e 100644
--- a/tensorflow/contrib/slim/python/slim/data/tfexample_decoder.py
+++ b/tensorflow/contrib/slim/python/slim/data/tfexample_decoder.py
@@ -243,6 +243,40 @@ class LookupTensor(Tensor):
     return self._table.lookup(unmapped_tensor)
 
 
+class BackupHandler(ItemHandler):
+  """An ItemHandler that tries two ItemHandlers in order."""
+
+  def __init__(self, handler, backup):
+    """Initializes the BackupHandler handler.
+
+    If the first Handler's tensors_to_item returns a Tensor with no elements,
+    the second Handler is used.
+
+    Args:
+      handler: The primary ItemHandler.
+      backup: The backup ItemHandler.
+
+    Raises:
+      ValueError: if either is not an ItemHandler.
+    """
+    if not isinstance(handler, ItemHandler):
+      raise ValueError('Primary handler is of type %s instead of ItemHandler'
+                       % type(handler))
+    if not isinstance(backup, ItemHandler):
+      raise ValueError('Backup handler is of type %s instead of ItemHandler'
+                       % type(backup))
+    self._handler = handler
+    self._backup = backup
+    super(BackupHandler, self).__init__(handler.keys + backup.keys)
+
+  def tensors_to_item(self, keys_to_tensors):
+    item = self._handler.tensors_to_item(keys_to_tensors)
+    return control_flow_ops.cond(
+        pred=math_ops.equal(math_ops.reduce_prod(array_ops.shape(item)), 0),
+        true_fn=lambda: self._backup.tensors_to_item(keys_to_tensors),
+        false_fn=lambda: item)
+
+
 class SparseTensor(ItemHandler):
   """An ItemHandler for SparseTensors."""
 
diff --git a/tensorflow/contrib/slim/python/slim/data/tfexample_decoder_test.py b/tensorflow/contrib/slim/python/slim/data/tfexample_decoder_test.py
index 9c5a14d006..d783d4fef4 100644
--- a/tensorflow/contrib/slim/python/slim/data/tfexample_decoder_test.py
+++ b/tensorflow/contrib/slim/python/slim/data/tfexample_decoder_test.py
@@ -843,5 +843,56 @@ class TFExampleDecoderTest(test.TestCase):
 
     self.assertAllClose([2, 0, 1], obtained_class_ids)
 
+  def testDecodeExampleWithBackupHandlerLookup(self):
+
+    example1 = example_pb2.Example(
+        features=feature_pb2.Features(
+            feature={
+                'image/object/class/text':
+                    self._BytesFeature(np.array(['cat', 'dog', 'guinea pig'])),
+                'image/object/class/label':
+                    self._EncodedInt64Feature(np.array([42, 10, 900]))
+            }))
+    example2 = example_pb2.Example(
+        features=feature_pb2.Features(
+            feature={
+                'image/object/class/text':
+                    self._BytesFeature(np.array(['cat', 'dog', 'guinea pig'])),
+            }))
+    example3 = example_pb2.Example(
+        features=feature_pb2.Features(
+            feature={
+                'image/object/class/label':
+                    self._EncodedInt64Feature(np.array([42, 10, 901]))
+            }))
+    # 'dog' -> 0, 'guinea pig' -> 1, 'cat' -> 2
+    table = lookup_ops.index_table_from_tensor(
+        constant_op.constant(['dog', 'guinea pig', 'cat']))
+    keys_to_features = {
+        'image/object/class/text': parsing_ops.VarLenFeature(dtypes.string),
+        'image/object/class/label': parsing_ops.VarLenFeature(dtypes.int64),
+    }
+    backup_handler = tfexample_decoder.BackupHandler(
+        handler=tfexample_decoder.Tensor('image/object/class/label'),
+        backup=tfexample_decoder.LookupTensor('image/object/class/text', table))
+    items_to_handlers = {
+        'labels': backup_handler,
+    }
+    decoder = tfexample_decoder.TFExampleDecoder(keys_to_features,
+                                                 items_to_handlers)
+    obtained_class_ids_each_example = []
+    with self.test_session() as sess:
+      sess.run(lookup_ops.tables_initializer())
+      for example in [example1, example2, example3]:
+        serialized_example = array_ops.reshape(
+            example.SerializeToString(), shape=[])
+        obtained_class_ids_each_example.append(
+            decoder.decode(serialized_example)[0].eval())
+
+    self.assertAllClose([42, 10, 900], obtained_class_ids_each_example[0])
+    self.assertAllClose([2, 0, 1], obtained_class_ids_each_example[1])
+    self.assertAllClose([42, 10, 901], obtained_class_ids_each_example[2])
+
+
 if __name__ == '__main__':
   test.main()
-- 
GitLab


From b20e0b28eec6245ce734d78cdb26dbf2d92c87ba Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 24 Oct 2017 15:07:22 -0700
Subject: [PATCH 731/909] Add transitional and temporary include file to merge
 grpc and grpc::internal namespaces while versions of gRPC are in transition

PiperOrigin-RevId: 173317900
---
 .../contrib/verbs/grpc_verbs_service_impl.cc  |  8 ++---
 .../contrib/verbs/grpc_verbs_service_impl.h   | 11 ++++++-
 tensorflow/core/distributed_runtime/rpc/BUILD | 11 +++++++
 .../rpc/grpc_master_service_impl.cc           | 32 +++++++++----------
 .../rpc/grpc_master_service_impl.h            | 15 +++++----
 .../rpc/grpc_namespace_compat.h               | 32 +++++++++++++++++++
 .../rpc/grpc_worker_service_impl.cc           |  4 +--
 .../rpc/grpc_worker_service_impl.h            |  1 +
 8 files changed, 84 insertions(+), 30 deletions(-)
 create mode 100644 tensorflow/core/distributed_runtime/rpc/grpc_namespace_compat.h

diff --git a/tensorflow/contrib/verbs/grpc_verbs_service_impl.cc b/tensorflow/contrib/verbs/grpc_verbs_service_impl.cc
index a1fbea57dd..cff765d1e8 100644
--- a/tensorflow/contrib/verbs/grpc_verbs_service_impl.cc
+++ b/tensorflow/contrib/verbs/grpc_verbs_service_impl.cc
@@ -43,21 +43,21 @@ VerbsService::Stub::Stub(
     const std::shared_ptr< ::grpc::ChannelInterface>& channel)
     : channel_(channel),
       rpcmethod_GetRemoteAddress_(grpcVerbsService_method_names[0],
-                                  ::grpc::internal::RpcMethod::NORMAL_RPC,
+                                  ::grpc::RpcMethod::NORMAL_RPC,
                                   channel) {}
 
 ::grpc::Status VerbsService::Stub::GetRemoteAddress(
     ::grpc::ClientContext* context, const GetRemoteAddressRequest& request,
     GetRemoteAddressResponse* response) {
-  return ::grpc::internal::BlockingUnaryCall(
+  return ::grpc::BlockingUnaryCall(
       channel_.get(), rpcmethod_GetRemoteAddress_, context, request, response);
 }
 
 VerbsService::AsyncService::AsyncService() {
   for (int i = 0; i < 1; ++i) {
-    AddMethod(new ::grpc::internal::RpcServiceMethod(
+    AddMethod(new ::grpc::RpcServiceMethod(
         grpcVerbsService_method_names[i],
-        ::grpc::internal::RpcMethod::NORMAL_RPC,
+        ::grpc::RpcMethod::NORMAL_RPC,
         nullptr));
     ::grpc::Service::MarkMethodAsync(i);
   }
diff --git a/tensorflow/contrib/verbs/grpc_verbs_service_impl.h b/tensorflow/contrib/verbs/grpc_verbs_service_impl.h
index 86431ca030..6e2bf86dac 100644
--- a/tensorflow/contrib/verbs/grpc_verbs_service_impl.h
+++ b/tensorflow/contrib/verbs/grpc_verbs_service_impl.h
@@ -28,6 +28,15 @@ limitations under the License.
 #include "tensorflow/contrib/verbs/verbs_service.pb.h"
 
 namespace grpc {
+
+// ensure internal namespace exists
+namespace internal {
+// bring in contents of external namespace
+using namespace ::grpc;
+}  // namespace internal
+// bring in contents of internal namespace
+using namespace internal;
+
 class CompletionQueue;
 class Channel;
 class RpcService;
@@ -61,7 +70,7 @@ class VerbsService GRPC_FINAL {
 
    private:
     std::shared_ptr< ::grpc::ChannelInterface> channel_;
-    const ::grpc::internal::RpcMethod rpcmethod_GetRemoteAddress_;
+    const ::grpc::RpcMethod rpcmethod_GetRemoteAddress_;
   };
   static std::unique_ptr<Stub> NewStub(
       const std::shared_ptr< ::grpc::ChannelInterface>& channel,
diff --git a/tensorflow/core/distributed_runtime/rpc/BUILD b/tensorflow/core/distributed_runtime/rpc/BUILD
index a8af124e2b..5190288e88 100644
--- a/tensorflow/core/distributed_runtime/rpc/BUILD
+++ b/tensorflow/core/distributed_runtime/rpc/BUILD
@@ -182,6 +182,7 @@ cc_library(
     srcs = ["grpc_worker_service_impl.cc"],
     hdrs = ["grpc_worker_service_impl.h"],
     deps = [
+        ":grpc_namespace_compat",
         ":grpc_serialization_traits",
         "//tensorflow/core:worker_proto_cc",
         "//tensorflow/core/distributed_runtime:worker_interface",
@@ -228,12 +229,22 @@ cc_library(
     srcs = ["grpc_master_service_impl.cc"],
     hdrs = ["grpc_master_service_impl.h"],
     deps = [
+        ":grpc_namespace_compat",
         ":grpc_serialization_traits",
         "//tensorflow/core:master_proto_cc",
         "@grpc//:grpc++_unsecure",
     ],
 )
 
+cc_library(
+    name = "grpc_namespace_compat",
+    srcs = [],
+    hdrs = ["grpc_namespace_compat.h"],
+    deps = [
+        "@grpc//:grpc++_unsecure",
+    ],
+)
+
 cc_library(
     name = "grpc_serialization_traits",
     srcs = [],
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_master_service_impl.cc b/tensorflow/core/distributed_runtime/rpc/grpc_master_service_impl.cc
index 17d0047eb2..d998d51058 100644
--- a/tensorflow/core/distributed_runtime/rpc/grpc_master_service_impl.cc
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_master_service_impl.cc
@@ -49,74 +49,74 @@ MasterService::Stub::Stub(
     const std::shared_ptr< ::grpc::ChannelInterface>& channel)
     : channel_(channel),
       rpcmethod_CreateSession_(grpcMasterService_method_names[0],
-                               ::grpc::internal::RpcMethod::NORMAL_RPC, channel),
+                               ::grpc::RpcMethod::NORMAL_RPC, channel),
       rpcmethod_ExtendSession_(grpcMasterService_method_names[1],
-                               ::grpc::internal::RpcMethod::NORMAL_RPC, channel),
+                               ::grpc::RpcMethod::NORMAL_RPC, channel),
       rpcmethod_PartialRunSetup_(grpcMasterService_method_names[2],
-                                 ::grpc::internal::RpcMethod::NORMAL_RPC, channel),
+                                 ::grpc::RpcMethod::NORMAL_RPC, channel),
       rpcmethod_RunStep_(grpcMasterService_method_names[3],
-                         ::grpc::internal::RpcMethod::NORMAL_RPC, channel),
+                         ::grpc::RpcMethod::NORMAL_RPC, channel),
       rpcmethod_CloseSession_(grpcMasterService_method_names[4],
-                              ::grpc::internal::RpcMethod::NORMAL_RPC, channel),
+                              ::grpc::RpcMethod::NORMAL_RPC, channel),
       rpcmethod_ListDevices_(grpcMasterService_method_names[5],
-                             ::grpc::internal::RpcMethod::NORMAL_RPC, channel),
+                             ::grpc::RpcMethod::NORMAL_RPC, channel),
       rpcmethod_Reset_(grpcMasterService_method_names[6],
-                       ::grpc::internal::RpcMethod::NORMAL_RPC, channel) {}
+                       ::grpc::RpcMethod::NORMAL_RPC, channel) {}
 
 ::grpc::Status MasterService::Stub::CreateSession(
     ::grpc::ClientContext* context, const CreateSessionRequest& request,
     CreateSessionResponse* response) {
-  return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_CreateSession_,
+  return ::grpc::BlockingUnaryCall(channel_.get(), rpcmethod_CreateSession_,
                                    context, request, response);
 }
 
 ::grpc::Status MasterService::Stub::ExtendSession(
     ::grpc::ClientContext* context, const ExtendSessionRequest& request,
     ExtendSessionResponse* response) {
-  return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_ExtendSession_,
+  return ::grpc::BlockingUnaryCall(channel_.get(), rpcmethod_ExtendSession_,
                                    context, request, response);
 }
 
 ::grpc::Status MasterService::Stub::PartialRunSetup(
     ::grpc::ClientContext* context, const PartialRunSetupRequest& request,
     PartialRunSetupResponse* response) {
-  return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_PartialRunSetup_,
+  return ::grpc::BlockingUnaryCall(channel_.get(), rpcmethod_PartialRunSetup_,
                                    context, request, response);
 }
 
 ::grpc::Status MasterService::Stub::RunStep(::grpc::ClientContext* context,
                                             const RunStepRequest& request,
                                             RunStepResponse* response) {
-  return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_RunStep_, context,
+  return ::grpc::BlockingUnaryCall(channel_.get(), rpcmethod_RunStep_, context,
                                    request, response);
 }
 
 ::grpc::Status MasterService::Stub::CloseSession(
     ::grpc::ClientContext* context, const CloseSessionRequest& request,
     CloseSessionResponse* response) {
-  return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_CloseSession_,
+  return ::grpc::BlockingUnaryCall(channel_.get(), rpcmethod_CloseSession_,
                                    context, request, response);
 }
 
 ::grpc::Status MasterService::Stub::ListDevices(
     ::grpc::ClientContext* context, const ListDevicesRequest& request,
     ListDevicesResponse* response) {
-  return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_ListDevices_,
+  return ::grpc::BlockingUnaryCall(channel_.get(), rpcmethod_ListDevices_,
                                    context, request, response);
 }
 
 ::grpc::Status MasterService::Stub::Reset(::grpc::ClientContext* context,
                                           const ResetRequest& request,
                                           ResetResponse* response) {
-  return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_Reset_, context,
+  return ::grpc::BlockingUnaryCall(channel_.get(), rpcmethod_Reset_, context,
                                    request, response);
 }
 
 MasterService::AsyncService::AsyncService() {
   for (int i = 0; i < 7; ++i) {
-    AddMethod(new ::grpc::internal::RpcServiceMethod(
+    AddMethod(new ::grpc::RpcServiceMethod(
         grpcMasterService_method_names[i],
-        ::grpc::internal::RpcMethod::NORMAL_RPC,
+        ::grpc::RpcMethod::NORMAL_RPC,
         nullptr));
     ::grpc::Service::MarkMethodAsync(i);
   }
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_master_service_impl.h b/tensorflow/core/distributed_runtime/rpc/grpc_master_service_impl.h
index 412395c526..131de2863f 100644
--- a/tensorflow/core/distributed_runtime/rpc/grpc_master_service_impl.h
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_master_service_impl.h
@@ -25,6 +25,7 @@ limitations under the License.
 #include "grpc++/impl/codegen/stub_options.h"
 #include "grpc++/impl/codegen/sync_stream.h"
 
+#include "tensorflow/core/distributed_runtime/rpc/grpc_namespace_compat.h"
 #include "tensorflow/core/distributed_runtime/rpc/grpc_serialization_traits.h"
 #include "tensorflow/core/protobuf/master.pb.h"
 
@@ -107,13 +108,13 @@ class MasterService final {
 
    private:
     std::shared_ptr< ::grpc::ChannelInterface> channel_;
-    const ::grpc::internal::RpcMethod rpcmethod_CreateSession_;
-    const ::grpc::internal::RpcMethod rpcmethod_ExtendSession_;
-    const ::grpc::internal::RpcMethod rpcmethod_PartialRunSetup_;
-    const ::grpc::internal::RpcMethod rpcmethod_RunStep_;
-    const ::grpc::internal::RpcMethod rpcmethod_CloseSession_;
-    const ::grpc::internal::RpcMethod rpcmethod_ListDevices_;
-    const ::grpc::internal::RpcMethod rpcmethod_Reset_;
+    const ::grpc::RpcMethod rpcmethod_CreateSession_;
+    const ::grpc::RpcMethod rpcmethod_ExtendSession_;
+    const ::grpc::RpcMethod rpcmethod_PartialRunSetup_;
+    const ::grpc::RpcMethod rpcmethod_RunStep_;
+    const ::grpc::RpcMethod rpcmethod_CloseSession_;
+    const ::grpc::RpcMethod rpcmethod_ListDevices_;
+    const ::grpc::RpcMethod rpcmethod_Reset_;
   };
   static std::unique_ptr<Stub> NewStub(
       const std::shared_ptr< ::grpc::ChannelInterface>& channel,
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_namespace_compat.h b/tensorflow/core/distributed_runtime/rpc/grpc_namespace_compat.h
new file mode 100644
index 0000000000..c178927f5d
--- /dev/null
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_namespace_compat.h
@@ -0,0 +1,32 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef THIRD_PARTY_TENSORFLOW_CORE_DISTRIBUTED_RUNTIME_RPC_GRPC_NAMESPACE_COMPAT_H_
+#define THIRD_PARTY_TENSORFLOW_CORE_DISTRIBUTED_RUNTIME_RPC_GRPC_NAMESPACE_COMPAT_H_
+
+// This file is a transitional place-holder until gRPC versions consistently
+// use namespace grpc::internal for library-internal structures
+
+namespace grpc {
+// ensure internal namespace exists
+namespace internal {
+// bring in contents of external namespace
+using namespace ::grpc;
+}  // namespace internal
+// bring in contents of internal namespace
+using namespace internal;
+}  // namespace grpc
+
+#endif  // THIRD_PARTY_TENSORFLOW_CORE_DISTRIBUTED_RUNTIME_RPC_GRPC_NAMESPACE_COMPAT_H_
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_worker_service_impl.cc b/tensorflow/core/distributed_runtime/rpc/grpc_worker_service_impl.cc
index 348c6dc98b..80a2f89337 100644
--- a/tensorflow/core/distributed_runtime/rpc/grpc_worker_service_impl.cc
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_worker_service_impl.cc
@@ -58,9 +58,9 @@ namespace grpc {
 
 WorkerService::AsyncService::AsyncService() {
   for (int i = 0; i < kGrpcNumWorkerMethods; ++i) {
-    AddMethod(new ::grpc::internal::RpcServiceMethod(
+    AddMethod(new ::grpc::RpcServiceMethod(
         GrpcWorkerMethodName(static_cast<GrpcWorkerMethod>(i)),
-        ::grpc::internal::RpcMethod::NORMAL_RPC, nullptr));
+        ::grpc::RpcMethod::NORMAL_RPC, nullptr));
     ::grpc::Service::MarkMethodAsync(i);
   }
 }
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_worker_service_impl.h b/tensorflow/core/distributed_runtime/rpc/grpc_worker_service_impl.h
index e9862a61a3..c8a8b5778e 100644
--- a/tensorflow/core/distributed_runtime/rpc/grpc_worker_service_impl.h
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_worker_service_impl.h
@@ -26,6 +26,7 @@ limitations under the License.
 #include "grpc++/impl/codegen/sync_stream.h"
 #include "grpc++/support/byte_buffer.h"
 
+#include "tensorflow/core/distributed_runtime/rpc/grpc_namespace_compat.h"
 #include "tensorflow/core/distributed_runtime/rpc/grpc_serialization_traits.h"
 #include "tensorflow/core/distributed_runtime/tensor_coding.h"
 #include "tensorflow/core/protobuf/worker.pb.h"
-- 
GitLab


From 488408c2cefcac507b325da4dd779a9015f7b53f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 24 Oct 2017 15:23:58 -0700
Subject: [PATCH 732/909] Removes unnecessary cast and warning in auc
 calculation.

PiperOrigin-RevId: 173320547
---
 tensorflow/python/estimator/BUILD          | 1 -
 tensorflow/python/estimator/canned/head.py | 4 ----
 2 files changed, 5 deletions(-)

diff --git a/tensorflow/python/estimator/BUILD b/tensorflow/python/estimator/BUILD
index 9670827e41..13fbfe9f53 100644
--- a/tensorflow/python/estimator/BUILD
+++ b/tensorflow/python/estimator/BUILD
@@ -543,7 +543,6 @@ py_library(
         "//tensorflow/python:math_ops",
         "//tensorflow/python:metrics",
         "//tensorflow/python:nn",
-        "//tensorflow/python:platform",
         "//tensorflow/python:sparse_tensor",
         "//tensorflow/python:string_ops",
         "//tensorflow/python:summary",
diff --git a/tensorflow/python/estimator/canned/head.py b/tensorflow/python/estimator/canned/head.py
index 1cc82c5055..f26e54ff49 100644
--- a/tensorflow/python/estimator/canned/head.py
+++ b/tensorflow/python/estimator/canned/head.py
@@ -40,7 +40,6 @@ from tensorflow.python.ops import nn
 from tensorflow.python.ops import string_ops
 from tensorflow.python.ops import weights_broadcast_ops
 from tensorflow.python.ops.losses import losses
-from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.saved_model import signature_constants
 from tensorflow.python.summary import summary
 
@@ -314,9 +313,6 @@ def _predictions_mean(predictions, weights=None, name=None):
 def _auc(labels, predictions, weights=None, curve='ROC', name=None):
   with ops.name_scope(name, 'auc', (predictions, labels, weights)) as scope:
     predictions = math_ops.to_float(predictions, name='predictions')
-    if labels.dtype.base_dtype != dtypes.bool:
-      logging.warning('Casting %s labels to bool.', labels.dtype)
-      labels = math_ops.cast(labels, dtypes.bool)
     if weights is not None:
       weights = weights_broadcast_ops.broadcast_weights(weights, predictions)
     return metrics_lib.auc(
-- 
GitLab


From 64ba163dc8fa1bdf780cbbb67811f9adce05e325 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 24 Oct 2017 15:24:15 -0700
Subject: [PATCH 733/909] Evaluator changes: * Get graph execution working, by
 adding init_variables() and   supporting graph mode in evaluate_on_dataset().
 * Use track_*() instead of add_*() to match Network. * Fill in several doc
 strings. * Detect metric added to two different Evaluators. * Return prefix
 along with metrics from metrics property.

PiperOrigin-RevId: 173320585
---
 tensorflow/contrib/eager/python/BUILD         |  4 +
 tensorflow/contrib/eager/python/evaluator.py  | 99 ++++++++++++++++---
 .../contrib/eager/python/evaluator_test.py    | 51 ++++++++--
 tensorflow/tools/ci_build/ci_sanity.sh        |  1 +
 4 files changed, 132 insertions(+), 23 deletions(-)

diff --git a/tensorflow/contrib/eager/python/BUILD b/tensorflow/contrib/eager/python/BUILD
index ee2ec79141..bbbf72d632 100644
--- a/tensorflow/contrib/eager/python/BUILD
+++ b/tensorflow/contrib/eager/python/BUILD
@@ -169,6 +169,9 @@ py_library(
     deps = [
         ":datasets",
         ":metrics",
+        "//tensorflow/python:control_flow_ops",
+        "//tensorflow/python:errors",
+        "//tensorflow/python:framework_ops",
         "//tensorflow/python/eager:context",
         "//tensorflow/python/eager:function",
     ],
@@ -182,6 +185,7 @@ py_test(
         ":evaluator",
         ":metrics",
         "//tensorflow/python/data/ops:dataset_ops",
+        "//tensorflow/python/eager:context",
         "//tensorflow/python/eager:test",
     ],
 )
diff --git a/tensorflow/contrib/eager/python/evaluator.py b/tensorflow/contrib/eager/python/evaluator.py
index d757e976ee..67f545e838 100644
--- a/tensorflow/contrib/eager/python/evaluator.py
+++ b/tensorflow/contrib/eager/python/evaluator.py
@@ -22,6 +22,11 @@ import six
 
 from tensorflow.contrib.eager.python import datasets
 from tensorflow.contrib.eager.python import metrics
+from tensorflow.python.eager import context
+from tensorflow.python.eager import function
+from tensorflow.python.framework import errors_impl
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import control_flow_ops
 
 
 class Evaluator(object):
@@ -37,7 +42,7 @@ class Evaluator(object):
   the evaluate_on_dataset() method.
 
   Implementers of Evaluators should
-  (a) Call `add_metric()` and/or `add_evaluator()` in __init__().
+  (a) Call `track_metric()` and/or `track_evaluator()` in __init__().
   (b) Override the `call()` method. It will be passed the output of the
       model's `eval_data()` method, and should call its contained metrics
       (treating them as callables) and any child Evaluators (using their
@@ -51,12 +56,36 @@ class Evaluator(object):
     self._model = model
     self._metrics = {}
     self._evaluators = {}
+    if context.in_graph_mode():
+      self.call = function.defun(self.call)
 
   # ---- API for users ----
   def __call__(self, *args, **kwargs):
-    """Update metrics with a minibatch of input examples."""
+    """Update metrics with a minibatch of input examples.
+
+    Args:
+      *args:
+      **kwargs: Arguments representing an input mini-batch of examples to
+        pass to self.model.eval_data().
+
+    Returns:
+      The op to execute or None if executing eagerly.
+    """
     return self.call(self._model.eval_data(*args, **kwargs))
 
+  def init_variables(self):
+    """Return an op for initializing all contained uninitialized variables.
+
+    Only for graph execution. Should be called after variables are created
+    in the first execution of __call__().
+
+    Returns:
+      An op.
+    """
+    assert context.in_graph_mode()
+    return control_flow_ops.group(
+        *[m.init_variables() for _, m in self.metrics])
+
   def all_metric_results(self):  # TODO(josh11b): Add optional summary_writer.
     """Returns dict mapping metric name -> value."""
     results = {}
@@ -69,14 +98,45 @@ class Evaluator(object):
 
   def evaluate_on_dataset(self, dataset, *args, **kwargs):
     """Convenience method for performing an eval on a Dataset."""
+    # TODO(josh11b): Add optional summary_writer.
+    if context.in_graph_mode():
+      # TODO(josh11b): Return an dict of tensors to pass to session.run()
+      # instead of running using the default session here.
+      sess = ops.get_default_session()
+      call_op = self.__call__(dataset.make_one_shot_iterator().get_next(),
+                              *args, **kwargs)
+      init_op = self.init_variables()
+      results_op = self.all_metric_results()
+      sess.run(init_op)
+      try:
+        while True:
+          sess.run(call_op)
+      except errors_impl.OutOfRangeError:
+        pass
+      return sess.run(results_op)
+    # Eager case
     for example in datasets.Iterator(dataset):
       self.__call__(example, *args, **kwargs)
-    # TODO(josh11b): Add optional summary_writer.
     return self.all_metric_results()
 
   # ---- To be implemented by descendants ---
   def call(self, eval_data):
-    """Update metrics using the output of self.model."""
+    """Update metrics using the output of self.model.
+
+    Note: This function is executed as a graph function in graph mode.
+    This means:
+    a) Operations on the same resource are executed in textual order.
+       This should make it easier to do things like add the updated
+       value of a variable to another, for example.
+    b) You don't need to worry about collecting the update ops to execute.
+       All update ops added to the graph by this function will be executed.
+    As a result, code should generally work the same way with graph or
+    eager execution.
+
+    Args:
+      eval_data: The output of self.model.eval_data() on a mini-batch of
+        examples.
+    """
     raise NotImplementedError("Evaluators must define a call member function.")
 
   # ---- For use by descendants ---
@@ -84,10 +144,11 @@ class Evaluator(object):
   def model(self):
     return self._model
 
-  def add_metric(self, metric):
+  def track_metric(self, metric):
     """Add a Metric to be tracked.
 
-    Rule: metrics can only be in one `Evaluator`.
+    Metrics can only be tracked by one `Evaluator`. Metrics must be
+    tracked or they will not appear in `all_metric_results()`.
 
     Args:
       metric: A `Metric` object.
@@ -98,14 +159,15 @@ class Evaluator(object):
     Raises:
       RuntimeError: If called before __init__.
       TypeError: If `metric` is not of the correct type.
-      ValueError: If there is a name collision between Metrics.
+      ValueError: If there is a name collision between Metrics or `metric`
+        has already been added to another `Evaluator`.
     """
     if not hasattr(self, "_metrics"):
       raise RuntimeError(
           "Need to call Evaluator.__init__ before adding metrics")
     if not isinstance(metric, metrics.Metric):
       raise TypeError(
-          "Evaluator.add_metric() passed type %s, not a tfe.metrics.Metric" %
+          "Evaluator.track_metric() passed type %s, not a tfe.metrics.Metric" %
           (type(metric),))
     if metric.name in self._metrics:
       if metric is self._metrics[metric.name]:
@@ -113,10 +175,16 @@ class Evaluator(object):
       raise ValueError(
           "Attempt to add two Metrics with the name '%s' to the same Evaluator "
           "'%s'" % (metric.name, self.name))
+    # pylint: disable=protected-access
+    if hasattr(metric, "_added_to_an_evaluator"):
+      raise ValueError("Metric %s already added to Evaluator %s" %
+                       (metric.name, metric._added_to_an_evaluator))
+    metric._added_to_an_evaluator = self.__class__.__name__
+    # pylint: enable=protected-access
     self._metrics[metric.name] = metric
     return metric
 
-  def add_evaluator(self, prefix, evaluator):
+  def track_evaluator(self, prefix, evaluator):
     """Add a contained `Evaluator`.
 
     This is for delegating to another `Evaluator`, e.g. for when you have a
@@ -141,7 +209,7 @@ class Evaluator(object):
           "Need to call Evaluator.__init__ before adding evaluators")
     if not isinstance(evaluator, Evaluator):
       raise TypeError(
-          "Evaluator.add_evaluator() passed type %s, not a tfe.Evaluator." %
+          "Evaluator.track_evaluator() passed type %s, not a tfe.Evaluator." %
           (type(evaluator),))
     if prefix in self._evaluators:
       if evaluator is self._evaluators[prefix]:
@@ -162,11 +230,12 @@ class Evaluator(object):
 
   @property
   def metrics(self):
+    """Returns a list of (prefix, metric) pairs."""
     m = []
     for metric in six.itervalues(self._metrics):
-      m.append(metric)
-    for evaluator in six.itervalues(self._evaluators):
-      m += evaluator.metrics
+      m.append(("", metric))
+    for prefix, evaluator in six.iteritems(self._evaluators):
+      m += [(prefix + "/" + p, m) for p, m in evaluator.metrics]
     return m
 
 
@@ -196,8 +265,8 @@ class SparseSoftmaxEvaluator(Evaluator):
     super(SparseSoftmaxEvaluator, self).__init__(model)
     # TODO(josh11b): Expand this to include everything from the standard
     # SparseSoftmax Head.
-    self.avg_loss = self.add_metric(metrics.Mean("Avg_Loss"))
-    self.accuracy = self.add_metric(metrics.Accuracy())
+    self.avg_loss = self.track_metric(metrics.Mean("Avg Loss"))
+    self.accuracy = self.track_metric(metrics.Accuracy())
     self.loss_key = loss_key
     self.label_key = label_key
     self.predicted_class_key = predicted_class_key
diff --git a/tensorflow/contrib/eager/python/evaluator_test.py b/tensorflow/contrib/eager/python/evaluator_test.py
index b18463c31a..71e9fa40a8 100644
--- a/tensorflow/contrib/eager/python/evaluator_test.py
+++ b/tensorflow/contrib/eager/python/evaluator_test.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 from tensorflow.contrib.eager.python import evaluator
 from tensorflow.contrib.eager.python import metrics
 from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.eager import context
 from tensorflow.python.eager import test
 
 
@@ -40,7 +41,7 @@ class SimpleEvaluator(evaluator.Evaluator):
 
   def __init__(self, model):
     super(SimpleEvaluator, self).__init__(model)
-    self.mean = self.add_metric(metrics.Mean("mean"))
+    self.mean = self.track_metric(metrics.Mean("mean"))
 
   def call(self, eval_data):
     self.mean(eval_data)
@@ -50,8 +51,8 @@ class DelegatingEvaluator(evaluator.Evaluator):
 
   def __init__(self, model):
     super(DelegatingEvaluator, self).__init__(model)
-    self.sub = self.add_evaluator("inner", SimpleEvaluator(model))
-    self.mean = self.add_metric(metrics.Mean("outer-mean"))
+    self.sub = self.track_evaluator("inner", SimpleEvaluator(model))
+    self.mean = self.track_metric(metrics.Mean("outer-mean"))
 
   def call(self, eval_data):
     # Keys here come from PrefixLModel, which adds "l_".
@@ -88,13 +89,21 @@ class EvaluatorTest(test.TestCase):
       prefix_count[p] = prefix_count.get(p, 0) + 1
     self.assertEqual({"outer_mean": 2, "mean": 2}, prefix_count)
 
-  def testDataset(self):
+  def testDatasetEager(self):
     e = SimpleEvaluator(IdentityModel())
     ds = dataset_ops.Dataset.from_tensor_slices([3.0, 5.0, 7.0, 9.0])
     results = e.evaluate_on_dataset(ds)
     self.assertEqual(set(["mean"]), set(results.keys()))
     self.assertEqual(6.0, results["mean"].numpy())
 
+  def testDatasetGraph(self):
+    with context.graph_mode(), self.test_session():
+      e = SimpleEvaluator(IdentityModel())
+      ds = dataset_ops.Dataset.from_tensor_slices([3.0, 5.0, 7.0, 9.0])
+      results = e.evaluate_on_dataset(ds)
+      self.assertEqual(set(["mean"]), set(results.keys()))
+      self.assertEqual(6.0, results["mean"])
+
   def testModelProperty(self):
     m = IdentityModel()
     e = SimpleEvaluator(m)
@@ -102,8 +111,34 @@ class EvaluatorTest(test.TestCase):
 
   def testMetricsProperty(self):
     e = DelegatingEvaluator(PrefixLModel())
-    names = set([m.name for m in e.metrics])
-    self.assertEqual(set(["outer-mean", "mean"]), names)
+    names = set([(p, m.name) for p, m in e.metrics])
+    self.assertEqual(set([("", "outer-mean"), ("inner/", "mean")]), names)
+
+  def testSharedMetric(self):
+
+    class MetricArgEvaluator(evaluator.Evaluator):
+
+      def __init__(self, model, m):
+        super(MetricArgEvaluator, self).__init__(model)
+        self.m = self.track_metric(m)
+
+    metric = metrics.Mean("mean")
+    model = IdentityModel()
+    e = MetricArgEvaluator(model, metric)
+    with self.assertRaisesRegexp(ValueError, "already added"):
+      MetricArgEvaluator(model, metric)
+    del e
+
+  def testMetricTrackedTwice(self):
+
+    class MetricTwiceEvaluator(evaluator.Evaluator):
+
+      def __init__(self, model):
+        super(MetricTwiceEvaluator, self).__init__(model)
+        self.m = self.track_metric(metrics.Mean("mean"))
+        self.track_metric(self.m)  # okay to track same metric again
+
+    MetricTwiceEvaluator(IdentityModel())
 
 
 class SparseSoftmaxEvaluatorTest(test.TestCase):
@@ -115,8 +150,8 @@ class SparseSoftmaxEvaluatorTest(test.TestCase):
        e.label_key: [1, 2, 3],
        e.predicted_class_key: [1, 1, 3]})
     results = e.all_metric_results()
-    self.assertEqual(set(["Avg_Loss", "Accuracy"]), set(results.keys()))
-    self.assertEqual(2.0, results["Avg_Loss"].numpy())
+    self.assertEqual(set(["Avg Loss", "Accuracy"]), set(results.keys()))
+    self.assertEqual(2.0, results["Avg Loss"].numpy())
     self.assertEqual(0.75, results["Accuracy"].numpy())
 
 
diff --git a/tensorflow/tools/ci_build/ci_sanity.sh b/tensorflow/tools/ci_build/ci_sanity.sh
index 1703cae1e5..26053de4e9 100755
--- a/tensorflow/tools/ci_build/ci_sanity.sh
+++ b/tensorflow/tools/ci_build/ci_sanity.sh
@@ -95,6 +95,7 @@ do_pylint() {
 "^tensorflow/python/platform/default/_googletest\.py.*\[E0102.*function\salready\sdefined "\
 "^tensorflow/python/feature_column/feature_column_test\.py.*\[E0110.*abstract-class-instantiated "\
 "^tensorflow/contrib/layers/python/layers/feature_column\.py.*\[E0110.*abstract-class-instantiated "\
+"^tensorflow/contrib/eager/python/evaluator\.py.*\[E0202.*method-hidden "\
 "^tensorflow/contrib/eager/python/metrics_impl\.py.*\[E0202.*method-hidden "\
 "^tensorflow/python/platform/gfile\.py.*\[E0301.*non-iterator "\
 "^tensorflow/python/keras/_impl/keras/callbacks\.py.*\[E1133.*not-an-iterable"
-- 
GitLab


From 5b9cdb2dcbb057701b0ffb0ec4e0ab555a53390b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 24 Oct 2017 15:48:40 -0700
Subject: [PATCH 734/909] Adding the batch norm adjustment to contrib/layers.

PiperOrigin-RevId: 173324074
---
 .../contrib/layers/python/layers/layers.py    | 20 +++++++++++++++++--
 .../layers/python/layers/layers_test.py       | 20 +++++++++++++++++++
 2 files changed, 38 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/layers/python/layers/layers.py b/tensorflow/contrib/layers/python/layers/layers.py
index 29ab281b1a..deeafdf300 100644
--- a/tensorflow/contrib/layers/python/layers/layers.py
+++ b/tensorflow/contrib/layers/python/layers/layers.py
@@ -463,7 +463,8 @@ def batch_norm(inputs,
                scope=None,
                renorm=False,
                renorm_clipping=None,
-               renorm_decay=0.99):
+               renorm_decay=0.99,
+               adjustment=None):
   """Adds a Batch Normalization layer from http://arxiv.org/abs/1502.03167.
 
     "Batch Normalization: Accelerating Deep Network Training by Reducing
@@ -546,6 +547,17 @@ def batch_norm(inputs,
       and should be neither too small (which would add noise) nor too large
       (which would give stale estimates). Note that `decay` is still applied
       to get the means and variances for inference.
+    adjustment: A function taking the `Tensor` containing the (dynamic) shape of
+      the input tensor and returning a pair (scale, bias) to apply to the
+      normalized values (before gamma and beta), only during training. For
+      example,
+        `adjustment = lambda shape: (
+          tf.random_uniform(shape[-1:], 0.93, 1.07),
+          tf.random_uniform(shape[-1:], -0.1, 0.1))`
+      will scale the normalized value by up to 7% up or down, then shift the
+      result by up to 0.1 (with independent scaling and bias for each feature
+      but shared across all examples), and finally apply gamma and/or beta. If
+      `None`, no adjustment is applied.
 
   Returns:
     A `Tensor` representing the output of the operation.
@@ -569,7 +581,10 @@ def batch_norm(inputs,
   #   implementation in normalization_layers.BatchNormalization.
   inputs = ops.convert_to_tensor(inputs)
   rank = inputs.get_shape().ndims
-  possible_to_fuse = batch_weights is None and not renorm and rank in [2, 4]
+  possible_to_fuse = (batch_weights is None and
+                      not renorm and
+                      rank in [2, 4] and
+                      adjustment is None)
   if fused and possible_to_fuse and (
       zero_debias_moving_mean or rank == 2 or
       updates_collections is not ops.GraphKeys.UPDATE_OPS):
@@ -636,6 +651,7 @@ def batch_norm(inputs,
           renorm=renorm,
           renorm_clipping=renorm_clipping,
           renorm_momentum=renorm_decay,
+          adjustment=adjustment,
           name=sc.name,
           _scope=sc,
           _reuse=reuse,
diff --git a/tensorflow/contrib/layers/python/layers/layers_test.py b/tensorflow/contrib/layers/python/layers/layers_test.py
index 1040ad3ca7..7c77e905f7 100644
--- a/tensorflow/contrib/layers/python/layers/layers_test.py
+++ b/tensorflow/contrib/layers/python/layers/layers_test.py
@@ -2644,6 +2644,26 @@ class BatchNormTest(test.TestCase):
                                        zero_debias_moving_mean=True)
       sess.run(variables_lib.global_variables_initializer())
 
+  def testAdjustmentCreated(self):
+    # Tests that the adjustment is appropriately passed to and used by the core
+    # BN layer.
+    all_adjustments = []
+    def _create_adjustment(shape):
+      adjustments = [array_ops.ones(shape[-1:]), array_ops.zeros(shape[-1:])]
+      all_adjustments.extend(adjustments)
+      return adjustments
+    depth = 8
+    images = array_ops.zeros([10, 5, 5, depth])
+    output = _layers.batch_norm(
+        images,
+        is_training=True,
+        adjustment=_create_adjustment)
+    self.assertListEqual(output.shape.as_list(), images.shape.as_list())
+    self.assertEqual(len(all_adjustments), 2)
+    self.assertListEqual(all_adjustments[0].shape.as_list(), [depth])
+    self.assertListEqual(all_adjustments[1].shape.as_list(), [depth])
+
+
 class LayerNormTest(test.TestCase):
 
   def testUnknownShape(self):
-- 
GitLab


From e74557db4b9f7b57f32118f4bc349f0a56c1e208 Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Tue, 24 Oct 2017 15:50:05 -0700
Subject: [PATCH 735/909] bool() for eager variables

PiperOrigin-RevId: 173324238
---
 .../python/kernel_tests/resource_variable_ops_test.py       | 5 +++++
 tensorflow/python/ops/resource_variable_ops.py              | 6 ++++++
 tensorflow/python/ops/rnn_cell_impl.py                      | 2 +-
 3 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/kernel_tests/resource_variable_ops_test.py b/tensorflow/python/kernel_tests/resource_variable_ops_test.py
index a2a1e1dcd8..32edc5be7f 100644
--- a/tensorflow/python/kernel_tests/resource_variable_ops_test.py
+++ b/tensorflow/python/kernel_tests/resource_variable_ops_test.py
@@ -68,6 +68,11 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase):
       self.assertAllEqual(variable.numpy(), 1.0)
       self.assertAllEqual(variable.initialized_value().numpy(), 1.0)
 
+  def testEagerBool(self):
+    with context.eager_mode():
+      v = resource_variable_ops.ResourceVariable(False, name="bool_test")
+      self.assertAllEqual(bool(v), False)
+
   def testAssignVariableDtypeMismatchEager(self):
     with context.eager_mode():
       handle = resource_variable_ops.var_handle_op(
diff --git a/tensorflow/python/ops/resource_variable_ops.py b/tensorflow/python/ops/resource_variable_ops.py
index 71e1fb0297..ce81a32924 100644
--- a/tensorflow/python/ops/resource_variable_ops.py
+++ b/tensorflow/python/ops/resource_variable_ops.py
@@ -478,6 +478,12 @@ class ResourceVariable(variables.Variable):
         pass  # 'NoneType' object has no attribute 'eager_mode' when context has
               # been unloaded. Will catch other module unloads as well.
 
+  def __nonzero__(self):
+    return self.__bool__()
+
+  def __bool__(self):
+    return bool(self.read_value())
+
   @property
   def dtype(self):
     """The dtype of this variable."""
diff --git a/tensorflow/python/ops/rnn_cell_impl.py b/tensorflow/python/ops/rnn_cell_impl.py
index 65b0407008..1825e98259 100644
--- a/tensorflow/python/ops/rnn_cell_impl.py
+++ b/tensorflow/python/ops/rnn_cell_impl.py
@@ -698,7 +698,7 @@ class LSTMCell(RNNCell):
     i, j, f, o = array_ops.split(
         value=lstm_matrix, num_or_size_splits=4, axis=1)
     # Diagonal connections
-    if self._use_peepholes and not self._w_f_diag:
+    if self._use_peepholes and self._w_f_diag is None:
       scope = vs.get_variable_scope()
       with vs.variable_scope(
           scope, initializer=self._initializer) as unit_scope:
-- 
GitLab


From fa5921b6cefa3e877348cfa5158143fbc764bc8c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 24 Oct 2017 15:50:41 -0700
Subject: [PATCH 736/909] Exposes precision_at_top_k under tf.metrics.

PiperOrigin-RevId: 173324359
---
 .../contrib/metrics/python/ops/metric_ops.py  |   2 +-
 .../python/kernel_tests/metrics_test.py       | 155 ++++++++++++++----
 tensorflow/python/ops/metrics.py              |   1 +
 tensorflow/python/ops/metrics_impl.py         |  23 ++-
 .../tools/api/golden/tensorflow.metrics.pbtxt |   4 +
 5 files changed, 142 insertions(+), 43 deletions(-)

diff --git a/tensorflow/contrib/metrics/python/ops/metric_ops.py b/tensorflow/contrib/metrics/python/ops/metric_ops.py
index 5a4c0c4358..675c49dfc3 100644
--- a/tensorflow/contrib/metrics/python/ops/metric_ops.py
+++ b/tensorflow/contrib/metrics/python/ops/metric_ops.py
@@ -2085,7 +2085,7 @@ def streaming_sparse_precision_at_top_k(top_k_predictions,
   default_name = _at_k_name('precision', class_id=class_id)
   with ops.name_scope(name, default_name,
                       (top_k_predictions, labels, weights)) as name_scope:
-    return metrics_impl._sparse_precision_at_top_k(  # pylint: disable=protected-access
+    return metrics_impl.precision_at_top_k(
         labels=labels,
         predictions_idx=top_k_predictions,
         class_id=class_id,
diff --git a/tensorflow/python/kernel_tests/metrics_test.py b/tensorflow/python/kernel_tests/metrics_test.py
index 2472b2a2a6..f21b0dfeab 100644
--- a/tensorflow/python/kernel_tests/metrics_test.py
+++ b/tensorflow/python/kernel_tests/metrics_test.py
@@ -1827,6 +1827,38 @@ def _test_sparse_precision_at_k(predictions,
       test_case.assertEqual(expected, metric.eval())
 
 
+def _test_precision_at_top_k(
+    predictions_idx,
+    labels,
+    expected,
+    k=None,
+    class_id=None,
+    weights=None,
+    test_case=None):
+  with ops.Graph().as_default() as g, test_case.test_session(g):
+    if weights is not None:
+      weights = constant_op.constant(weights, dtypes_lib.float32)
+    metric, update = metrics.precision_at_top_k(
+        predictions_idx=constant_op.constant(predictions_idx, dtypes_lib.int32),
+        labels=labels,
+        k=k,
+        class_id=class_id,
+        weights=weights)
+
+    # Fails without initialized vars.
+    test_case.assertRaises(errors_impl.OpError, metric.eval)
+    test_case.assertRaises(errors_impl.OpError, update.eval)
+    variables.variables_initializer(variables.local_variables()).run()
+
+    # Run per-step op and assert expected values.
+    if math.isnan(expected):
+      test_case.assertTrue(math.isnan(update.eval()))
+      test_case.assertTrue(math.isnan(metric.eval()))
+    else:
+      test_case.assertEqual(expected, update.eval())
+      test_case.assertEqual(expected, metric.eval())
+
+
 def _test_sparse_average_precision_at_k(predictions,
                                         labels,
                                         k,
@@ -1858,6 +1890,7 @@ class SingleLabelSparsePrecisionTest(test.TestCase):
 
   def setUp(self):
     self._predictions = ((0.1, 0.3, 0.2, 0.4), (0.1, 0.2, 0.3, 0.4))
+    self._predictions_idx = [[3], [3]]
     indicator_labels = ((0, 0, 0, 1), (0, 0, 1, 0))
     class_labels = (3, 2)
     # Sparse vs dense, and 1d vs 2d labels should all be handled the same.
@@ -1868,6 +1901,8 @@ class SingleLabelSparsePrecisionTest(test.TestCase):
                 [[class_id] for class_id in class_labels], dtype=np.int64))
     self._test_sparse_precision_at_k = functools.partial(
         _test_sparse_precision_at_k, test_case=self)
+    self._test_precision_at_top_k = functools.partial(
+        _test_precision_at_top_k, test_case=self)
     self._test_sparse_average_precision_at_k = functools.partial(
         _test_sparse_average_precision_at_k, test_case=self)
 
@@ -1877,16 +1912,24 @@ class SingleLabelSparsePrecisionTest(test.TestCase):
       for class_id in (-1, 0, 1, 2, 4):
         self._test_sparse_precision_at_k(
             self._predictions, labels, k=1, expected=NAN, class_id=class_id)
+        self._test_precision_at_top_k(
+            self._predictions_idx, labels, k=1, expected=NAN, class_id=class_id)
 
   def test_at_k1(self):
     for labels in self._labels:
       # Class 3: 1 label, 2 predictions, 1 correct.
       self._test_sparse_precision_at_k(
           self._predictions, labels, k=1, expected=1.0 / 2, class_id=3)
+      self._test_precision_at_top_k(
+          self._predictions_idx, labels, k=1, expected=1.0 / 2, class_id=3)
 
       # All classes: 2 labels, 2 predictions, 1 correct.
       self._test_sparse_precision_at_k(
           self._predictions, labels, k=1, expected=1.0 / 2)
+      self._test_precision_at_top_k(
+          self._predictions_idx, labels, k=1, expected=1.0 / 2)
+      self._test_sparse_average_precision_at_k(
+          self._predictions, labels, k=1, expected=1.0 / 2)
 
 
 class MultiLabelSparsePrecisionTest(test.TestCase):
@@ -1894,6 +1937,8 @@ class MultiLabelSparsePrecisionTest(test.TestCase):
   def setUp(self):
     self._test_sparse_precision_at_k = functools.partial(
         _test_sparse_precision_at_k, test_case=self)
+    self._test_precision_at_top_k = functools.partial(
+        _test_precision_at_top_k, test_case=self)
     self._test_sparse_average_precision_at_k = functools.partial(
         _test_sparse_average_precision_at_k, test_case=self)
 
@@ -1905,6 +1950,7 @@ class MultiLabelSparsePrecisionTest(test.TestCase):
     labels = np.array([labels_ex1], dtype=np.int64)
     predictions_ex1 = (0.2, 0.1, 0.0, 0.4, 0.0, 0.5, 0.3)
     predictions = (predictions_ex1,)
+    predictions_idx_ex1 = (5, 3, 6, 0, 1)
     precision_ex1 = (0.0 / 1, 1.0 / 2, 1.0 / 3, 2.0 / 4)
     avg_precision_ex1 = (0.0 / 1, precision_ex1[1] / 2, precision_ex1[1] / 3,
                          (precision_ex1[1] + precision_ex1[3]) / 4)
@@ -1912,6 +1958,8 @@ class MultiLabelSparsePrecisionTest(test.TestCase):
       k = i + 1
       self._test_sparse_precision_at_k(
           predictions, labels, k, expected=precision_ex1[i])
+      self._test_precision_at_top_k(
+          (predictions_idx_ex1[:k],), labels, k=k, expected=precision_ex1[i])
       self._test_sparse_average_precision_at_k(
           predictions, labels, k, expected=avg_precision_ex1[i])
 
@@ -1920,6 +1968,7 @@ class MultiLabelSparsePrecisionTest(test.TestCase):
     labels = np.array([labels_ex2], dtype=np.int64)
     predictions_ex2 = (0.3, 0.5, 0.0, 0.4, 0.0, 0.1, 0.2)
     predictions = (predictions_ex2,)
+    predictions_idx_ex2 = (1, 3, 0, 6, 5)
     precision_ex2 = (0.0 / 1, 0.0 / 2, 1.0 / 3, 2.0 / 4)
     avg_precision_ex2 = (0.0 / 1, 0.0 / 2, precision_ex2[2] / 3,
                          (precision_ex2[2] + precision_ex2[3]) / 4)
@@ -1927,6 +1976,8 @@ class MultiLabelSparsePrecisionTest(test.TestCase):
       k = i + 1
       self._test_sparse_precision_at_k(
           predictions, labels, k, expected=precision_ex2[i])
+      self._test_precision_at_top_k(
+          (predictions_idx_ex2[:k],), labels, k=k, expected=precision_ex2[i])
       self._test_sparse_average_precision_at_k(
           predictions, labels, k, expected=avg_precision_ex2[i])
 
@@ -1942,8 +1993,11 @@ class MultiLabelSparsePrecisionTest(test.TestCase):
     ]
     for i in xrange(4):
       k = i + 1
+      predictions_idx = (predictions_idx_ex1[:k], predictions_idx_ex2[:k])
       self._test_sparse_precision_at_k(
           predictions, labels, k, expected=streaming_precision[i])
+      self._test_precision_at_top_k(
+          predictions_idx, labels, k=k, expected=streaming_precision[i])
       self._test_sparse_average_precision_at_k(
           predictions, labels, k, expected=streaming_average_precision[i])
 
@@ -1969,6 +2023,7 @@ class MultiLabelSparsePrecisionTest(test.TestCase):
     labels = np.array([labels_ex1], dtype=np.int64)
     predictions_ex1 = (0.2, 0.1, 0.0, 0.4, 0.0, 0.5, 0.3)
     predictions = (predictions_ex1,)
+    predictions_idx_ex1 = (5, 3, 6, 0, 1)
     precision_ex1 = (0.0 / 1, 1.0 / 2, 1.0 / 3, 2.0 / 4)
     avg_precision_ex1 = (0.0 / 1, precision_ex1[1] / 2, precision_ex1[1] / 3,
                          (precision_ex1[1] + precision_ex1[3]) / 4)
@@ -1976,12 +2031,15 @@ class MultiLabelSparsePrecisionTest(test.TestCase):
       k = i + 1
       self._test_sparse_precision_at_k(
           predictions, labels, k, expected=precision_ex1[i])
+      self._test_precision_at_top_k(
+          (predictions_idx_ex1[:k],), labels, k=k, expected=precision_ex1[i])
       self._test_sparse_average_precision_at_k(
           predictions, labels, k, expected=avg_precision_ex1[i])
 
   def test_three_labels_at_k5_no_predictions(self):
     predictions = [[0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9],
                    [0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6]]
+    predictions_idx = [[9, 4, 6, 2, 0], [5, 7, 2, 9, 6]]
     sparse_labels = _binary_2d_label_to_2d_sparse_value(
         [[0, 0, 1, 0, 0, 0, 0, 1, 1, 0], [0, 1, 1, 0, 0, 1, 0, 0, 0, 0]])
     dense_labels = np.array([[2, 7, 8], [1, 2, 5]], dtype=np.int64)
@@ -1991,10 +2049,13 @@ class MultiLabelSparsePrecisionTest(test.TestCase):
       for class_id in (-1, 1, 3, 8, 10):
         self._test_sparse_precision_at_k(
             predictions, labels, k=5, expected=NAN, class_id=class_id)
+        self._test_precision_at_top_k(
+            predictions_idx, labels, k=5, expected=NAN, class_id=class_id)
 
   def test_three_labels_at_k5_no_labels(self):
     predictions = [[0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9],
                    [0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6]]
+    predictions_idx = [[9, 4, 6, 2, 0], [5, 7, 2, 9, 6]]
     sparse_labels = _binary_2d_label_to_2d_sparse_value(
         [[0, 0, 1, 0, 0, 0, 0, 1, 1, 0], [0, 1, 1, 0, 0, 1, 0, 0, 0, 0]])
     dense_labels = np.array([[2, 7, 8], [1, 2, 5]], dtype=np.int64)
@@ -2004,10 +2065,13 @@ class MultiLabelSparsePrecisionTest(test.TestCase):
       for class_id in (0, 4, 6, 9):
         self._test_sparse_precision_at_k(
             predictions, labels, k=5, expected=0.0, class_id=class_id)
+        self._test_precision_at_top_k(
+            predictions_idx, labels, k=5, expected=0.0, class_id=class_id)
 
   def test_three_labels_at_k5(self):
     predictions = [[0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9],
                    [0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6]]
+    predictions_idx = [[9, 4, 6, 2, 0], [5, 7, 2, 9, 6]]
     sparse_labels = _binary_2d_label_to_2d_sparse_value(
         [[0, 0, 1, 0, 0, 0, 0, 1, 1, 0], [0, 1, 1, 0, 0, 1, 0, 0, 0, 0]])
     dense_labels = np.array([[2, 7, 8], [1, 2, 5]], dtype=np.int64)
@@ -2016,23 +2080,32 @@ class MultiLabelSparsePrecisionTest(test.TestCase):
       # Class 2: 2 labels, 2 correct predictions.
       self._test_sparse_precision_at_k(
           predictions, labels, k=5, expected=2.0 / 2, class_id=2)
+      self._test_precision_at_top_k(
+          predictions_idx, labels, k=5, expected=2.0 / 2, class_id=2)
 
       # Class 5: 1 label, 1 correct prediction.
       self._test_sparse_precision_at_k(
           predictions, labels, k=5, expected=1.0 / 1, class_id=5)
+      self._test_precision_at_top_k(
+          predictions_idx, labels, k=5, expected=1.0 / 1, class_id=5)
 
       # Class 7: 1 label, 1 incorrect prediction.
       self._test_sparse_precision_at_k(
           predictions, labels, k=5, expected=0.0 / 1, class_id=7)
+      self._test_precision_at_top_k(
+          predictions_idx, labels, k=5, expected=0.0 / 1, class_id=7)
 
       # All classes: 10 predictions, 3 correct.
       self._test_sparse_precision_at_k(
           predictions, labels, k=5, expected=3.0 / 10)
+      self._test_precision_at_top_k(
+          predictions_idx, labels, k=5, expected=3.0 / 10)
 
   def test_three_labels_at_k5_some_out_of_range(self):
     """Tests that labels outside the [0, n_classes) range are ignored."""
     predictions = [[0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9],
                    [0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6]]
+    predictions_idx = [[9, 4, 6, 2, 0], [5, 7, 2, 9, 6]]
     sp_labels = sparse_tensor.SparseTensorValue(
         indices=[[0, 0], [0, 1], [0, 2], [0, 3], [1, 0], [1, 1], [1, 2],
                  [1, 3]],
@@ -2043,24 +2116,34 @@ class MultiLabelSparsePrecisionTest(test.TestCase):
     # Class 2: 2 labels, 2 correct predictions.
     self._test_sparse_precision_at_k(
         predictions, sp_labels, k=5, expected=2.0 / 2, class_id=2)
+    self._test_precision_at_top_k(
+        predictions_idx, sp_labels, k=5, expected=2.0 / 2, class_id=2)
 
     # Class 5: 1 label, 1 correct prediction.
     self._test_sparse_precision_at_k(
         predictions, sp_labels, k=5, expected=1.0 / 1, class_id=5)
+    self._test_precision_at_top_k(
+        predictions_idx, sp_labels, k=5, expected=1.0 / 1, class_id=5)
 
     # Class 7: 1 label, 1 incorrect prediction.
     self._test_sparse_precision_at_k(
         predictions, sp_labels, k=5, expected=0.0 / 1, class_id=7)
+    self._test_precision_at_top_k(
+        predictions_idx, sp_labels, k=5, expected=0.0 / 1, class_id=7)
 
     # All classes: 10 predictions, 3 correct.
     self._test_sparse_precision_at_k(
         predictions, sp_labels, k=5, expected=3.0 / 10)
+    self._test_precision_at_top_k(
+        predictions_idx, sp_labels, k=5, expected=3.0 / 10)
 
   def test_3d_nan(self):
     predictions = [[[0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9],
                     [0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6]],
                    [[0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6],
                     [0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9]]]
+    predictions_idx = [[[9, 4, 6, 2, 0], [5, 7, 2, 9, 6]],
+                       [[5, 7, 2, 9, 6], [9, 4, 6, 2, 0]]]
     labels = _binary_3d_label_to_sparse_value(
         [[[0, 0, 1, 0, 0, 0, 0, 1, 1, 0], [0, 1, 1, 0, 0, 1, 0, 0, 0, 0]],
          [[0, 1, 1, 0, 0, 1, 0, 1, 0, 0], [0, 0, 1, 0, 0, 0, 0, 0, 1, 0]]])
@@ -2069,12 +2152,16 @@ class MultiLabelSparsePrecisionTest(test.TestCase):
     for class_id in (-1, 1, 3, 8, 10):
       self._test_sparse_precision_at_k(
           predictions, labels, k=5, expected=NAN, class_id=class_id)
+      self._test_precision_at_top_k(
+          predictions_idx, labels, k=5, expected=NAN, class_id=class_id)
 
   def test_3d_no_labels(self):
     predictions = [[[0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9],
                     [0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6]],
                    [[0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6],
                     [0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9]]]
+    predictions_idx = [[[9, 4, 6, 2, 0], [5, 7, 2, 9, 6]],
+                       [[5, 7, 2, 9, 6], [9, 4, 6, 2, 0]]]
     labels = _binary_3d_label_to_sparse_value(
         [[[0, 0, 1, 0, 0, 0, 0, 1, 1, 0], [0, 1, 1, 0, 0, 1, 0, 0, 0, 0]],
          [[0, 1, 1, 0, 0, 1, 0, 1, 0, 0], [0, 0, 1, 0, 0, 0, 0, 0, 1, 0]]])
@@ -2083,12 +2170,16 @@ class MultiLabelSparsePrecisionTest(test.TestCase):
     for class_id in (0, 4, 6, 9):
       self._test_sparse_precision_at_k(
           predictions, labels, k=5, expected=0.0, class_id=class_id)
+      self._test_precision_at_top_k(
+          predictions_idx, labels, k=5, expected=0.0, class_id=class_id)
 
   def test_3d(self):
     predictions = [[[0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9],
                     [0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6]],
                    [[0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6],
                     [0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9]]]
+    predictions_idx = [[[9, 4, 6, 2, 0], [5, 7, 2, 9, 6]],
+                       [[5, 7, 2, 9, 6], [9, 4, 6, 2, 0]]]
     labels = _binary_3d_label_to_sparse_value(
         [[[0, 0, 1, 0, 0, 0, 0, 1, 1, 0], [0, 1, 1, 0, 0, 1, 0, 0, 0, 0]],
          [[0, 1, 1, 0, 0, 1, 0, 1, 0, 0], [0, 0, 1, 0, 0, 0, 0, 0, 1, 0]]])
@@ -2096,80 +2187,84 @@ class MultiLabelSparsePrecisionTest(test.TestCase):
     # Class 2: 4 predictions, all correct.
     self._test_sparse_precision_at_k(
         predictions, labels, k=5, expected=4.0 / 4, class_id=2)
+    self._test_precision_at_top_k(
+        predictions_idx, labels, k=5, expected=4.0 / 4, class_id=2)
 
     # Class 5: 2 predictions, both correct.
     self._test_sparse_precision_at_k(
         predictions, labels, k=5, expected=2.0 / 2, class_id=5)
+    self._test_precision_at_top_k(
+        predictions_idx, labels, k=5, expected=2.0 / 2, class_id=5)
 
     # Class 7: 2 predictions, 1 correct.
     self._test_sparse_precision_at_k(
         predictions, labels, k=5, expected=1.0 / 2, class_id=7)
+    self._test_precision_at_top_k(
+        predictions_idx, labels, k=5, expected=1.0 / 2, class_id=7)
 
     # All classes: 20 predictions, 7 correct.
     self._test_sparse_precision_at_k(
         predictions, labels, k=5, expected=7.0 / 20)
+    self._test_precision_at_top_k(
+        predictions_idx, labels, k=5, expected=7.0 / 20)
 
   def test_3d_ignore_some(self):
     predictions = [[[0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9],
                     [0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6]],
                    [[0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6],
                     [0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9]]]
+    predictions_idx = [[[9, 4, 6, 2, 0], [5, 7, 2, 9, 6]],
+                       [[5, 7, 2, 9, 6], [9, 4, 6, 2, 0]]]
     labels = _binary_3d_label_to_sparse_value(
         [[[0, 0, 1, 0, 0, 0, 0, 1, 1, 0], [0, 1, 1, 0, 0, 1, 0, 0, 0, 0]],
          [[0, 1, 1, 0, 0, 1, 0, 1, 0, 0], [0, 0, 1, 0, 0, 0, 0, 0, 1, 0]]])
 
     # Class 2: 2 predictions, both correct.
     self._test_sparse_precision_at_k(
-        predictions,
-        labels,
-        k=5,
-        expected=2.0 / 2.0,
-        class_id=2,
+        predictions, labels, k=5, expected=2.0 / 2.0, class_id=2,
+        weights=[[1], [0]])
+    self._test_precision_at_top_k(
+        predictions_idx, labels, k=5, expected=2.0 / 2.0, class_id=2,
         weights=[[1], [0]])
 
     # Class 2: 2 predictions, both correct.
     self._test_sparse_precision_at_k(
-        predictions,
-        labels,
-        k=5,
-        expected=2.0 / 2.0,
-        class_id=2,
+        predictions, labels, k=5, expected=2.0 / 2.0, class_id=2,
+        weights=[[0], [1]])
+    self._test_precision_at_top_k(
+        predictions_idx, labels, k=5, expected=2.0 / 2.0, class_id=2,
         weights=[[0], [1]])
 
     # Class 7: 1 incorrect prediction.
     self._test_sparse_precision_at_k(
-        predictions,
-        labels,
-        k=5,
-        expected=0.0 / 1.0,
-        class_id=7,
+        predictions, labels, k=5, expected=0.0 / 1.0, class_id=7,
+        weights=[[1], [0]])
+    self._test_precision_at_top_k(
+        predictions_idx, labels, k=5, expected=0.0 / 1.0, class_id=7,
         weights=[[1], [0]])
 
     # Class 7: 1 correct prediction.
     self._test_sparse_precision_at_k(
-        predictions,
-        labels,
-        k=5,
-        expected=1.0 / 1.0,
-        class_id=7,
+        predictions, labels, k=5, expected=1.0 / 1.0, class_id=7,
+        weights=[[0], [1]])
+    self._test_precision_at_top_k(
+        predictions_idx, labels, k=5, expected=1.0 / 1.0, class_id=7,
         weights=[[0], [1]])
 
     # Class 7: no predictions.
     self._test_sparse_precision_at_k(
-        predictions,
-        labels,
-        k=5,
-        expected=NAN,
-        class_id=7,
+        predictions, labels, k=5, expected=NAN, class_id=7,
+        weights=[[1, 0], [0, 1]])
+    self._test_precision_at_top_k(
+        predictions_idx, labels, k=5, expected=NAN, class_id=7,
         weights=[[1, 0], [0, 1]])
 
     # Class 7: 2 predictions, 1 correct.
     self._test_sparse_precision_at_k(
-        predictions,
-        labels,
-        k=5,
-        expected=1.0 / 2.0,
-        class_id=7,
+        predictions, labels, k=5, expected=1.0 / 2.0, class_id=7,
+        weights=[[0, 1], [1, 0]])
+    self._test_precision_at_top_k(
+        predictions_idx, labels, k=5, expected=1.0 / 2.0, class_id=7,
         weights=[[0, 1], [1, 0]])
 
 
diff --git a/tensorflow/python/ops/metrics.py b/tensorflow/python/ops/metrics.py
index a4e2ef1dad..0465c77691 100644
--- a/tensorflow/python/ops/metrics.py
+++ b/tensorflow/python/ops/metrics.py
@@ -39,6 +39,7 @@
 @@sensitivity_at_specificity
 @@sparse_average_precision_at_k
 @@sparse_precision_at_k
+@@precision_at_top_k
 @@specificity_at_sensitivity
 @@true_negatives
 @@true_negatives_at_thresholds
diff --git a/tensorflow/python/ops/metrics_impl.py b/tensorflow/python/ops/metrics_impl.py
index 4c3ebb3aae..9273659a77 100644
--- a/tensorflow/python/ops/metrics_impl.py
+++ b/tensorflow/python/ops/metrics_impl.py
@@ -2889,14 +2889,14 @@ def _streaming_sparse_false_positive_at_k(labels,
     return var, state_ops.assign_add(var, batch_total_fp, name='update')
 
 
-def _sparse_precision_at_top_k(labels,
-                               predictions_idx,
-                               k=None,
-                               class_id=None,
-                               weights=None,
-                               metrics_collections=None,
-                               updates_collections=None,
-                               name=None):
+def precision_at_top_k(labels,
+                       predictions_idx,
+                       k=None,
+                       class_id=None,
+                       weights=None,
+                       metrics_collections=None,
+                       updates_collections=None,
+                       name=None):
   """Computes precision@k of the predictions with respect to sparse labels.
 
   Differs from `sparse_precision_at_k` in that predictions must be in the form
@@ -2915,7 +2915,7 @@ def _sparse_precision_at_top_k(labels,
       N >= 1. Commonly, N=1 and predictions has shape [batch size, k].
       The final dimension contains the top `k` predicted class indices.
       [D1, ... DN] must match `labels`.
-    k: Integer, k for @k metric.
+    k: Integer, k for @k metric. Only used for the default op name.
     class_id: Integer class ID for which we want binary metrics. This should be
       in range [0, num_classes], where num_classes is the last dimension of
       `predictions`. If `class_id` is outside this range, the method returns
@@ -2944,6 +2944,7 @@ def _sparse_precision_at_top_k(labels,
   """
   with ops.name_scope(name, _at_k_name('precision', k, class_id=class_id),
                       (predictions_idx, labels, weights)) as scope:
+    labels = _maybe_expand_labels(labels, predictions_idx)
     top_k_idx = math_ops.to_int64(predictions_idx)
     tp, tp_update = _streaming_sparse_true_positive_at_k(
         predictions_idx=top_k_idx, labels=labels, k=k, class_id=class_id,
@@ -3038,10 +3039,8 @@ def sparse_precision_at_k(labels,
   """
   with ops.name_scope(name, _at_k_name('precision', k, class_id=class_id),
                       (predictions, labels, weights)) as scope:
-    labels = _maybe_expand_labels(labels, predictions)
-
     _, top_k_idx = nn.top_k(predictions, k)
-    return _sparse_precision_at_top_k(
+    return precision_at_top_k(
         labels=labels,
         predictions_idx=top_k_idx,
         k=k,
diff --git a/tensorflow/tools/api/golden/tensorflow.metrics.pbtxt b/tensorflow/tools/api/golden/tensorflow.metrics.pbtxt
index daa3785034..2aab2c4a77 100644
--- a/tensorflow/tools/api/golden/tensorflow.metrics.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.metrics.pbtxt
@@ -68,6 +68,10 @@ tf_module {
     name: "precision_at_thresholds"
     argspec: "args=[\'labels\', \'predictions\', \'thresholds\', \'weights\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
   }
+  member_method {
+    name: "precision_at_top_k"
+    argspec: "args=[\'labels\', \'predictions_idx\', \'k\', \'class_id\', \'weights\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\'], "
+  }
   member_method {
     name: "recall"
     argspec: "args=[\'labels\', \'predictions\', \'weights\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
-- 
GitLab


From 8e7390ff4e0d9d173df5e193bf90af934e42f193 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 24 Oct 2017 16:13:09 -0700
Subject: [PATCH 737/909] Fix FusedConv2DBiasActivationOp for OIHW filter
 format.

The 'filter' variable wasn't initialized for OIHW filter format.

PiperOrigin-RevId: 173327533
---
 .../kernels/fused_conv2d_bias_activation_op.cc         | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/tensorflow/contrib/fused_conv/kernels/fused_conv2d_bias_activation_op.cc b/tensorflow/contrib/fused_conv/kernels/fused_conv2d_bias_activation_op.cc
index e4c39739f7..88306094ab 100644
--- a/tensorflow/contrib/fused_conv/kernels/fused_conv2d_bias_activation_op.cc
+++ b/tensorflow/contrib/fused_conv/kernels/fused_conv2d_bias_activation_op.cc
@@ -445,11 +445,11 @@ void LaunchFusedConv2DBiasActivationOp<GPUDevice, T, BiasType, ScaleType>::
       .set_zero_padding_width(padding_cols / 2);
 
   Tensor maybe_transformed_filter;
-  const Tensor* filter;
-  if (is_int8x4) {
-    // We have already checked filter is OIHW_VECT_I in the constructor.
-    filter = &filter_param;
-  } else if (filter_format == FORMAT_HWIO) {
+  const Tensor* filter = &filter_param;
+  // For qint8, we have already checked filter is OIHW_VECT_I in the
+  // constructor, but we need to test for is_int8x4 so the if block doesn't
+  // generate code for qint8.
+  if (!is_int8x4 && filter_format == FORMAT_HWIO) {
     // Shuffle filter tensor from HWIO to OIHW:
     OP_REQUIRES_OK(ctx, ctx->allocate_temp(
                             DataTypeToEnum<T>::value,
-- 
GitLab


From 56ceca431454635e8ea456cb35f9aeb7f62a8948 Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Tue, 24 Oct 2017 17:00:23 -0700
Subject: [PATCH 738/909] Disables storing variables in the default variable
 store for eager.

Also disables all functional layers until a non-default store is
implemented.

PiperOrigin-RevId: 173333446
---
 .../kernel_tests/variable_scope_test.py       |  1 -
 tensorflow/python/layers/convolutional.py     | 42 +++++++++
 tensorflow/python/layers/core.py              | 14 +++
 tensorflow/python/layers/core_test.py         | 87 ++++++++-----------
 tensorflow/python/layers/maxout.py            |  5 ++
 tensorflow/python/layers/normalization.py     |  8 +-
 tensorflow/python/layers/pooling.py           | 43 +++++++++
 tensorflow/python/ops/variable_scope.py       |  8 +-
 8 files changed, 153 insertions(+), 55 deletions(-)

diff --git a/tensorflow/python/kernel_tests/variable_scope_test.py b/tensorflow/python/kernel_tests/variable_scope_test.py
index 29f583d5ba..efeb25d095 100644
--- a/tensorflow/python/kernel_tests/variable_scope_test.py
+++ b/tensorflow/python/kernel_tests/variable_scope_test.py
@@ -67,7 +67,6 @@ class VariableScopeTest(test.TestCase):
     with self.assertRaises(ValueError):
       vs.get_variable("u", [1], reuse=True)  # That fails.
 
-  @test_util.run_in_graph_and_eager_modes()
   def testNamelessStore(self):
     vs = variable_scope._get_default_variable_store()
     vs.get_variable("v1", [2])
diff --git a/tensorflow/python/layers/convolutional.py b/tensorflow/python/layers/convolutional.py
index c983d3803b..6b371c618f 100644
--- a/tensorflow/python/layers/convolutional.py
+++ b/tensorflow/python/layers/convolutional.py
@@ -383,7 +383,14 @@ def conv1d(inputs,
 
   Returns:
     Output tensor.
+
+  Raises:
+    ValueError: if eager execution is enabled.
   """
+  if context.in_eager_mode():
+    raise ValueError(
+        'Functional layers are currently not compatible with eager execution.'
+        'Use tf.layers.Conv1D instead.')
   layer = Conv1D(
       filters=filters,
       kernel_size=kernel_size,
@@ -583,7 +590,14 @@ def conv2d(inputs,
 
   Returns:
     Output tensor.
+
+  Raises:
+    ValueError: if eager execution is enabled.
   """
+  if context.in_eager_mode():
+    raise ValueError(
+        'Functional layers are currently not compatible with eager execution.'
+        'Use tf.layers.Conv2D instead.')
   layer = Conv2D(
       filters=filters,
       kernel_size=kernel_size,
@@ -785,7 +799,14 @@ def conv3d(inputs,
 
   Returns:
     Output tensor.
+
+  Raises:
+    ValueError: if eager execution is enabled.
   """
+  if context.in_eager_mode():
+    raise ValueError(
+        'Functional layers are currently not compatible with eager execution.'
+        'Use tf.layers.Conv3D instead.')
   layer = Conv3D(
       filters=filters,
       kernel_size=kernel_size,
@@ -1104,7 +1125,14 @@ def separable_conv2d(inputs,
 
   Returns:
     Output tensor.
+
+  Raises:
+    ValueError: if eager execution is enabled.
   """
+  if context.in_eager_mode():
+    raise ValueError(
+        'Functional layers are currently not compatible with eager execution.'
+        'Use tf.layers.SeparableConv2D instead.')
   layer = SeparableConv2D(
       filters=filters,
       kernel_size=kernel_size,
@@ -1399,7 +1427,14 @@ def conv2d_transpose(inputs,
 
   Returns:
     Output tensor.
+
+  Raises:
+    ValueError: if eager execution is enabled.
   """
+  if context.in_eager_mode():
+    raise ValueError(
+        'Functional layers are currently not compatible with eager execution.'
+        'Use tf.layers.Conv2DTranspose instead.')
   layer = Conv2DTranspose(
       filters=filters,
       kernel_size=kernel_size,
@@ -1710,7 +1745,14 @@ def conv3d_transpose(inputs,
 
   Returns:
     Output tensor.
+
+  Raises:
+    ValueError: if eager execution is enabled.
   """
+  if context.in_eager_mode():
+    raise ValueError(
+        'Functional layers are currently not compatible with eager execution.'
+        'Use tf.layers.Conv3DTranspose instead.')
   layer = Conv3DTranspose(
       filters=filters,
       kernel_size=kernel_size,
diff --git a/tensorflow/python/layers/core.py b/tensorflow/python/layers/core.py
index ef9ff5790c..457bee5cff 100644
--- a/tensorflow/python/layers/core.py
+++ b/tensorflow/python/layers/core.py
@@ -231,7 +231,14 @@ def dense(
 
   Returns:
     Output tensor.
+
+  Raises:
+    ValueError: if eager execution is enabled.
   """
+  if context.in_eager_mode():
+    raise ValueError(
+        'Functional layers are currently not compatible with eager execution.'
+        'Use tf.layers.Dense instead.')
   layer = Dense(units,
                 activation=activation,
                 use_bias=use_bias,
@@ -333,7 +340,14 @@ def dropout(inputs,
 
   Returns:
     Output tensor.
+
+  Raises:
+    ValueError: if eager execution is enabled.
   """
+  if context.in_eager_mode():
+    raise ValueError(
+        'Functional layers are currently not compatible with eager execution.'
+        'Use tf.layers.Dropout instead.')
   layer = Dropout(rate, noise_shape=noise_shape, seed=seed, name=name)
   return layer.apply(inputs, training=training)
 
diff --git a/tensorflow/python/layers/core_test.py b/tensorflow/python/layers/core_test.py
index d917dcb69c..5184b372ff 100644
--- a/tensorflow/python/layers/core_test.py
+++ b/tensorflow/python/layers/core_test.py
@@ -203,21 +203,15 @@ class DenseTest(test.TestCase):
     self.assertEqual(len(loss_keys), 1)
     self.assertListEqual(dense.losses, loss_keys)
 
-  @test_util.run_in_graph_and_eager_modes()
   def testFunctionalDense(self):
-    inputs = random_ops.random_uniform((5, 3), seed=1)
-    outputs = core_layers.dense(
-        inputs, 2, activation=nn_ops.relu, name='my_dense')
-    if context.in_graph_mode():
+    with self.test_session():
+      inputs = random_ops.random_uniform((5, 3), seed=1)
+      outputs = core_layers.dense(
+          inputs, 2, activation=nn_ops.relu, name='my_dense')
       self.assertEqual(
           len(ops.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES)), 2)
       self.assertEqual(outputs.op.name, 'my_dense/Relu')
-    else:
-      self.assertEqual(
-          len(_get_variable_dict_from_varstore().values()), 2)
-    self.assertEqual(outputs.get_shape().as_list(), [5, 2])
 
-  @test_util.run_in_graph_and_eager_modes()
   def testFunctionalDenseTwice(self):
     inputs = random_ops.random_uniform((5, 3), seed=1)
     core_layers.dense(inputs, 2)
@@ -249,25 +243,21 @@ class DenseTest(test.TestCase):
         vars2 = variables.trainable_variables()
       self.assertEqual(vars1, vars2)
 
-  @test_util.run_in_graph_and_eager_modes()
   def testFunctionalDenseInitializerFromScope(self):
     with variable_scope.variable_scope(
-        'scope', initializer=init_ops.ones_initializer()):
+        'scope', initializer=init_ops.ones_initializer()), self.test_session():
       inputs = random_ops.random_uniform((5, 3), seed=1)
       core_layers.dense(inputs, 2)
-      self.evaluate(variables.global_variables_initializer())
+      variables.global_variables_initializer().run()
       weights = _get_variable_dict_from_varstore()
       self.assertEqual(len(weights), 2)
       # Check that the matrix weights got initialized to ones (from scope).
-      self.assertAllClose(
-          self.evaluate(weights['scope/dense/kernel'].read_value()),
-          np.ones((3, 2)))
+      self.assertAllClose(weights['scope/dense/kernel'].read_value().eval(),
+                          np.ones((3, 2)))
       # Check that the bias still got initialized to zeros.
-      self.assertAllClose(
-          self.evaluate(weights['scope/dense/bias'].read_value()),
-          np.zeros((2)))
+      self.assertAllClose(weights['scope/dense/bias'].read_value().eval(),
+                          np.zeros((2)))
 
-  @test_util.run_in_graph_and_eager_modes()
   def testFunctionalDenseWithCustomGetter(self):
     called = [0]
 
@@ -280,26 +270,26 @@ class DenseTest(test.TestCase):
       core_layers.dense(inputs, 2)
     self.assertEqual(called[0], 2)
 
-  @test_util.run_in_graph_and_eager_modes()
   def testFunctionalDenseInScope(self):
-    with variable_scope.variable_scope('test'):
-      inputs = random_ops.random_uniform((5, 3), seed=1)
-      core_layers.dense(inputs, 2, name='my_dense')
-      var_dict = _get_variable_dict_from_varstore()
-      var_key = 'test/my_dense/kernel'
-      self.assertEqual(var_dict[var_key].name, '%s:0' % var_key)
-    with variable_scope.variable_scope('test1') as scope:
-      inputs = random_ops.random_uniform((5, 3), seed=1)
-      core_layers.dense(inputs, 2, name=scope)
-      var_dict = _get_variable_dict_from_varstore()
-      var_key = 'test1/kernel'
-      self.assertEqual(var_dict[var_key].name, '%s:0' % var_key)
-    with variable_scope.variable_scope('test2'):
-      inputs = random_ops.random_uniform((5, 3), seed=1)
-      core_layers.dense(inputs, 2)
-      var_dict = _get_variable_dict_from_varstore()
-      var_key = 'test2/dense/kernel'
-      self.assertEqual(var_dict[var_key].name, '%s:0' % var_key)
+    with self.test_session():
+      with variable_scope.variable_scope('test'):
+        inputs = random_ops.random_uniform((5, 3), seed=1)
+        core_layers.dense(inputs, 2, name='my_dense')
+        var_dict = _get_variable_dict_from_varstore()
+        var_key = 'test/my_dense/kernel'
+        self.assertEqual(var_dict[var_key].name, '%s:0' % var_key)
+      with variable_scope.variable_scope('test1') as scope:
+        inputs = random_ops.random_uniform((5, 3), seed=1)
+        core_layers.dense(inputs, 2, name=scope)
+        var_dict = _get_variable_dict_from_varstore()
+        var_key = 'test1/kernel'
+        self.assertEqual(var_dict[var_key].name, '%s:0' % var_key)
+      with variable_scope.variable_scope('test2'):
+        inputs = random_ops.random_uniform((5, 3), seed=1)
+        core_layers.dense(inputs, 2)
+        var_dict = _get_variable_dict_from_varstore()
+        var_key = 'test2/dense/kernel'
+        self.assertEqual(var_dict[var_key].name, '%s:0' % var_key)
 
   @test_util.run_in_graph_and_eager_modes()
   def testComputeOutputShape(self):
@@ -389,17 +379,16 @@ class DropoutTest(test.TestCase):
     self.assertAlmostEqual(0., np_output.min())
     self.assertAllClose(np_output[:, 0, :], np_output[:, 1, :])
 
-  @test_util.run_in_graph_and_eager_modes()
   def testFunctionalDropout(self):
-    inputs = array_ops.ones((5, 5))
-    dropped = core_layers.dropout(inputs, 0.5, training=True, seed=1)
-    if context.in_graph_mode():
-      self.evaluate(variables.global_variables_initializer())
-    np_output = self.evaluate(dropped)
-    self.assertAlmostEqual(0., np_output.min())
-    dropped = core_layers.dropout(inputs, 0.5, training=False, seed=1)
-    np_output = self.evaluate(dropped)
-    self.assertAllClose(np.ones((5, 5)), np_output)
+    with self.test_session():
+      inputs = array_ops.ones((5, 5))
+      dropped = core_layers.dropout(inputs, 0.5, training=True, seed=1)
+      variables.global_variables_initializer().run()
+      np_output = self.evaluate(dropped)
+      self.assertAlmostEqual(0., np_output.min())
+      dropped = core_layers.dropout(inputs, 0.5, training=False, seed=1)
+      np_output = self.evaluate(dropped)
+      self.assertAllClose(np.ones((5, 5)), np_output)
 
   def testDynamicRate(self):
     with self.test_session() as sess:
diff --git a/tensorflow/python/layers/maxout.py b/tensorflow/python/layers/maxout.py
index 1ea36dbf6a..fa6c8cee97 100644
--- a/tensorflow/python/layers/maxout.py
+++ b/tensorflow/python/layers/maxout.py
@@ -20,6 +20,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from tensorflow.python.eager import context
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import gen_array_ops
@@ -50,6 +51,10 @@ def maxout(inputs, num_units, axis=-1, name=None):
    Raises:
     ValueError: if num_units is not multiple of number of features.
   """
+  if context.in_eager_mode():
+    raise ValueError(
+        'Functional layers are currently not compatible with eager execution.'
+        'use tf.contrib.layers.MaxOut instead')
   return MaxOut(num_units=num_units, axis=axis, name=name)(inputs)
 
 
diff --git a/tensorflow/python/layers/normalization.py b/tensorflow/python/layers/normalization.py
index 74246189b5..899be08020 100644
--- a/tensorflow/python/layers/normalization.py
+++ b/tensorflow/python/layers/normalization.py
@@ -717,7 +717,14 @@ def batch_normalization(inputs,
 
   Returns:
     Output tensor.
+
+  Raises:
+    ValueError: if eager execution is enabled.
   """
+  if context.in_eager_mode():
+    raise ValueError(
+        'Functional layers are currently not compatible with eager execution.'
+        'Use tf.layers.BactchNormalization instead.')
   layer = BatchNormalization(
       axis=axis,
       momentum=momentum,
@@ -749,4 +756,3 @@ def batch_normalization(inputs,
 
 BatchNorm = BatchNormalization
 batch_norm = batch_normalization
-
diff --git a/tensorflow/python/layers/pooling.py b/tensorflow/python/layers/pooling.py
index 6245ec5054..ec02ab032d 100644
--- a/tensorflow/python/layers/pooling.py
+++ b/tensorflow/python/layers/pooling.py
@@ -20,6 +20,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from tensorflow.python.eager import context
 from tensorflow.python.framework import tensor_shape
 from tensorflow.python.layers import base
 from tensorflow.python.layers import utils
@@ -144,7 +145,14 @@ def average_pooling1d(inputs, pool_size, strides,
 
   Returns:
     The output tensor, of rank 3.
+
+  Raises:
+    ValueError: if eager execution is enabled.
   """
+  if context.in_eager_mode():
+    raise ValueError(
+        'Functional layers are currently not compatible with eager execution.'
+        'Use tf.layers.AveragePooling1D instead.')
   layer = AveragePooling1D(pool_size=pool_size,
                            strides=strides,
                            padding=padding,
@@ -206,7 +214,14 @@ def max_pooling1d(inputs, pool_size, strides,
 
   Returns:
     The output tensor, of rank 3.
+
+  Raises:
+    ValueError: if eager execution is enabled.
   """
+  if context.in_eager_mode():
+    raise ValueError(
+        'Functional layers are currently not compatible with eager execution.'
+        'Use tf.layers.MaxPooling1D instead.')
   layer = MaxPooling1D(pool_size=pool_size,
                        strides=strides,
                        padding=padding,
@@ -344,7 +359,14 @@ def average_pooling2d(inputs,
 
   Returns:
     Output tensor.
+
+  Raises:
+    ValueError: if eager execution is enabled.
   """
+  if context.in_eager_mode():
+    raise ValueError(
+        'Functional layers are currently not compatible with eager execution.'
+        'Use tf.layers.AveragePooling2D instead.')
   layer = AveragePooling2D(pool_size=pool_size, strides=strides,
                            padding=padding, data_format=data_format,
                            name=name)
@@ -409,7 +431,14 @@ def max_pooling2d(inputs,
 
   Returns:
     Output tensor.
+
+  Raises:
+    ValueError: if eager execution is enabled.
   """
+  if context.in_eager_mode():
+    raise ValueError(
+        'Functional layers are currently not compatible with eager execution.'
+        'Use tf.layers.MaxPooling2D instead.')
   layer = MaxPooling2D(pool_size=pool_size, strides=strides,
                        padding=padding, data_format=data_format,
                        name=name)
@@ -560,7 +589,14 @@ def average_pooling3d(inputs,
 
   Returns:
     Output tensor.
+
+  Raises:
+    ValueError: if eager execution is enabled.
   """
+  if context.in_eager_mode():
+    raise ValueError(
+        'Functional layers are currently not compatible with eager execution.'
+        'Use tf.layers.AveragePooling3D instead.')
   layer = AveragePooling3D(pool_size=pool_size, strides=strides,
                            padding=padding, data_format=data_format,
                            name=name)
@@ -629,7 +665,14 @@ def max_pooling3d(inputs,
 
   Returns:
     Output tensor.
+
+  Raises:
+    ValueError: if eager execution is enabled.
   """
+  if context.in_eager_mode():
+    raise ValueError(
+        'Functional layers are currently not compatible with eager execution.'
+        'Use tf.layers.MaxPooling3D instead.')
   layer = MaxPooling3D(pool_size=pool_size, strides=strides,
                        padding=padding, data_format=data_format,
                        name=name)
diff --git a/tensorflow/python/ops/variable_scope.py b/tensorflow/python/ops/variable_scope.py
index 8c5c639b68..08be8574f3 100644
--- a/tensorflow/python/ops/variable_scope.py
+++ b/tensorflow/python/ops/variable_scope.py
@@ -724,9 +724,6 @@ class _VariableStore(object):
     if name in self._vars:
       # Here we handle the case when returning an existing variable.
       if reuse is False:
-        if context.in_eager_mode():
-          raise ValueError(
-              "Trying to recreate existing variable: %s" % self._vars[name])
         tb = self._vars[name].op.traceback[::-1]
         # Throw away internal tf entries and only take a few lines.
         tb = [x for x in tb if "tensorflow/python" not in x[0]][:3]
@@ -798,7 +795,10 @@ class _VariableStore(object):
           dtype=variable_dtype,
           validate_shape=validate_shape,
           constraint=constraint)
-    self._vars[name] = v
+    if context.in_graph_mode():
+      # In eager mode we do not want to keep default references to Variable
+      # objects as this will prevent their memory from being released.
+      self._vars[name] = v
     logging.vlog(1, "Created variable %s with shape %s and init %s", v.name,
                  format(shape), initializer)
 
-- 
GitLab


From 171dc9f32182bad58f811c4dedf8e435bb2508d6 Mon Sep 17 00:00:00 2001
From: Mark Daoust <markdaoust@google.com>
Date: Tue, 24 Oct 2017 17:01:41 -0700
Subject: [PATCH 739/909] Disambiguate links to "@{$estimators}".

PiperOrigin-RevId: 173333650
---
 tensorflow/docs_src/extend/estimators.md     |  2 +-
 tensorflow/docs_src/extend/index.md          |  2 +-
 tensorflow/docs_src/get_started/estimator.md |  2 +-
 tensorflow/docs_src/get_started/input_fn.md  |  2 +-
 tensorflow/docs_src/tutorials/layers.md      | 10 +++++-----
 tensorflow/docs_src/tutorials/linear.md      |  2 +-
 6 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/tensorflow/docs_src/extend/estimators.md b/tensorflow/docs_src/extend/estimators.md
index 5defade7ae..7e6507c584 100644
--- a/tensorflow/docs_src/extend/estimators.md
+++ b/tensorflow/docs_src/extend/estimators.md
@@ -44,7 +44,7 @@ feature columns, input functions, and `train()`/`evaluate()`/`predict()`
 operations. If you've never used tf.estimator before, or need a refresher,
 you should first review the following tutorials:
 
-*   @{$estimator$tf.estimator Quickstart}: Quick introduction to
+*   @{$get_started/estimator$tf.estimator Quickstart}: Quick introduction to
     training a neural network using tf.estimator.
 *   @{$wide$TensorFlow Linear Model Tutorial}: Introduction to
     feature columns, and an overview on building a linear classifier in
diff --git a/tensorflow/docs_src/extend/index.md b/tensorflow/docs_src/extend/index.md
index 5812caaffc..3f30b9a8c2 100644
--- a/tensorflow/docs_src/extend/index.md
+++ b/tensorflow/docs_src/extend/index.md
@@ -14,7 +14,7 @@ TensorFlow:
     add support for your own shared or distributed filesystem.
   * @{$new_data_formats$Custom Data Readers}, which details how to add support
     for your own file and record formats.
-  * @{$estimators$Creating Estimators in tf.contrib.learn}, which explains how
+  * @{$extend/estimators$Creating Estimators in tf.contrib.learn}, which explains how
     to write your own custom Estimator.  For example, you could build your
     own Estimator to implement some variation on standard linear regression.
 
diff --git a/tensorflow/docs_src/get_started/estimator.md b/tensorflow/docs_src/get_started/estimator.md
index 4f3a438d17..ab270d1408 100644
--- a/tensorflow/docs_src/get_started/estimator.md
+++ b/tensorflow/docs_src/get_started/estimator.md
@@ -400,7 +400,7 @@ second sample is *Iris virginica*.
     @{$linear$Large-scale Linear Models with TensorFlow}.
 
 *   To build your own Estimator using tf.estimator APIs, check out
-    @{$estimators$Creating Estimators in tf.estimator}.
+    @{$extend/estimators$Creating Estimators}.
 
 *   To experiment with neural network modeling and visualization in the browser,
     check out [Deep Playground](http://playground.tensorflow.org/).
diff --git a/tensorflow/docs_src/get_started/input_fn.md b/tensorflow/docs_src/get_started/input_fn.md
index 7706c07b1d..9d3af5d96a 100644
--- a/tensorflow/docs_src/get_started/input_fn.md
+++ b/tensorflow/docs_src/get_started/input_fn.md
@@ -11,7 +11,7 @@ median house values.
 The `input_fn` is used to pass feature and target data to the `train`,
 `evaluate`, and `predict` methods of the `Estimator`.
 The user can do feature engineering or pre-processing inside the `input_fn`.
-Here's an example taken from the @{$estimator$tf.estimator Quickstart tutorial}:
+Here's an example taken from the @{$get_started/estimator$tf.estimator Quickstart tutorial}:
 
 ```python
 import numpy as np
diff --git a/tensorflow/docs_src/tutorials/layers.md b/tensorflow/docs_src/tutorials/layers.md
index 8037c92c73..e808a3677f 100644
--- a/tensorflow/docs_src/tutorials/layers.md
+++ b/tensorflow/docs_src/tutorials/layers.md
@@ -192,7 +192,7 @@ def cnn_model_fn(features, labels, mode):
 The following sections (with headings corresponding to each code block above)
 dive deeper into the `tf.layers` code used to create each layer, as well as how
 to calculate loss, configure the training op, and generate predictions. If
-you're already experienced with CNNs and @{$estimators$TensorFlow `Estimator`s},
+you're already experienced with CNNs and @{$extend/estimators$TensorFlow `Estimator`s},
 and find the above code intuitive, you may want to skim these sections or just
 skip ahead to ["Training and Evaluating the CNN MNIST
 Classifier"](#training-and-evaluating-the-cnn-mnist-classifier).
@@ -536,8 +536,8 @@ if mode == tf.estimator.ModeKeys.TRAIN:
 ```
 
 > Note: For a more in-depth look at configuring training ops for Estimator model
-> functions, see @{$estimators#defining-the-training-op-for-the-model$"Defining
-> the training op for the model"} in the @{$estimators$"Creating Estimations in
+> functions, see @{$extend/estimators#defining-the-training-op-for-the-model$"Defining
+> the training op for the model"} in the @{$extend/estimators$"Creating Estimations in
 > tf.estimator"} tutorial.
 
 ### Add evaluation metrics
@@ -601,7 +601,7 @@ be saved (here, we specify the temp directory `/tmp/mnist_convnet_model`, but
 feel free to change to another directory of your choice).
 
 > Note: For an in-depth walkthrough of the TensorFlow `Estimator` API, see the
-> tutorial @{$estimators$"Creating Estimators in tf.estimator."}
+> tutorial @{$extend/estimators$"Creating Estimators in tf.estimator."}
 
 ### Set Up a Logging Hook {#set_up_a_logging_hook}
 
@@ -720,7 +720,7 @@ Here, we've achieved an accuracy of 97.3% on our test data set.
 To learn more about TensorFlow Estimators and CNNs in TensorFlow, see the
 following resources:
 
-*   @{$estimators$Creating Estimators in tf.estimator}. An
+*   @{$extend/estimators$Creating Estimators in tf.estimator}. An
     introduction to the TensorFlow Estimator API, which walks through
     configuring an Estimator, writing a model function, calculating loss, and
     defining a training op.
diff --git a/tensorflow/docs_src/tutorials/linear.md b/tensorflow/docs_src/tutorials/linear.md
index 4201a8021b..a6517549c3 100644
--- a/tensorflow/docs_src/tutorials/linear.md
+++ b/tensorflow/docs_src/tutorials/linear.md
@@ -16,7 +16,7 @@ give it a try. This overview uses code samples from the tutorial, but the
 tutorial walks through the code in greater detail.
 
 To understand this overview it will help to have some familiarity
-with basic machine learning concepts, and also with @{$estimator$tf.estimator}.
+with basic machine learning concepts, and also with @{$get_started/estimator$`tf.estimator`}.
 
 [TOC]
 
-- 
GitLab


From 2e2c1753b64a5aaa6a5fa30edf42297bc402f41b Mon Sep 17 00:00:00 2001
From: Igor Saprykin <isaprykin@google.com>
Date: Tue, 24 Oct 2017 17:06:15 -0700
Subject: [PATCH 740/909] Add a way to run ops using a step function to
 MonitoredSession.

With this method users have access to a raw Session while getting the benefit of recoverable behavior of MonitoredSession.

PiperOrigin-RevId: 173334319
---
 tensorflow/python/debug/wrappers/framework.py |   6 +-
 .../python/training/monitored_session.py      | 102 ++++++++++-
 .../python/training/monitored_session_test.py | 166 ++++++++++++++++++
 ...ain.-monitored-session.-step-context.pbtxt |  21 +++
 .../tensorflow.train.-monitored-session.pbtxt |   8 +
 ...ular-monitored-session.-step-context.pbtxt |  21 +++
 ...ow.train.-singular-monitored-session.pbtxt |   8 +
 7 files changed, 328 insertions(+), 4 deletions(-)
 create mode 100644 tensorflow/tools/api/golden/tensorflow.train.-monitored-session.-step-context.pbtxt
 create mode 100644 tensorflow/tools/api/golden/tensorflow.train.-singular-monitored-session.-step-context.pbtxt

diff --git a/tensorflow/python/debug/wrappers/framework.py b/tensorflow/python/debug/wrappers/framework.py
index 1947d74973..4e243cb6c9 100644
--- a/tensorflow/python/debug/wrappers/framework.py
+++ b/tensorflow/python/debug/wrappers/framework.py
@@ -551,6 +551,10 @@ class BaseDebugWrapperSession(session.SessionInterface):
     return (self._thread_name_filter_pattern and
             not self._thread_name_filter_pattern.match(thread_name))
 
+  def run_step_fn(self, step_fn):
+    return step_fn(
+        monitored_session.MonitoredSession.StepContext(self._sess, self.run))
+
   def partial_run_setup(self, fetches, feeds=None):
     """Sets up the feeds and fetches for partial runs in the session."""
     raise NotImplementedError(
@@ -792,7 +796,7 @@ class NonInteractiveDebugWrapperSession(BaseDebugWrapperSession):
 
   def __init__(self, sess, watch_fn=None, thread_name_filter=None,
                pass_through_operrors=False):
-    """Constructor of DumpingDebugWrapperSession.
+    """Constructor of NonInteractiveDebugWrapperSession.
 
     Args:
       sess: The TensorFlow `Session` object being wrapped.
diff --git a/tensorflow/python/training/monitored_session.py b/tensorflow/python/training/monitored_session.py
index e6162dd34b..2dd2114af0 100644
--- a/tensorflow/python/training/monitored_session.py
+++ b/tensorflow/python/training/monitored_session.py
@@ -25,6 +25,7 @@ import sys
 import six
 
 from tensorflow.core.protobuf import config_pb2
+from tensorflow.python.estimator import util
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
@@ -493,6 +494,7 @@ class _MonitoredSession(object):
       self._sess = _RecoverableSession(self._coordinated_creator)
     else:
       self._sess = self._coordinated_creator.create_session()
+    self._stop_requested_in_step_fn = False
 
   @property
   def graph(self):
@@ -520,10 +522,104 @@ class _MonitoredSession(object):
                           options=options,
                           run_metadata=run_metadata)
 
+  def run_step_fn(self, step_fn):
+    """Run ops using a step function.
+
+    Args:
+      step_fn: A function or a method with a single argument of type
+        `StepContext`.  The function may use methods of the argument to
+        perform computations with access to a raw session.
+
+        The returned value of the `step_fn` will be returned from `run_step_fn`,
+        unless a stop is requested.  In that case, the next `should_stop` call
+        will return True.
+
+        Example usage:
+        ```python
+           with tf.Graph().as_default():
+             c = tf.placeholder(dtypes.float32)
+             v = tf.add(c, 4.0)
+             w = tf.add(c, 0.5)
+
+             def step_fn(step_context):
+               a = step_context.session.run(fetches=v, feed_dict={c: 0.5})
+               if a <= 4.5:
+                 step_context.request_stop()
+               return step_context.run_with_hooks(fetches=w, feed_dict={c: 0.1})
+
+             with tf.MonitoredSession() as session:
+               while not session.should_stop():
+                 a = session.run_step_fn(step_fn)
+        ```
+        Hooks interact with the `run_with_hooks()` call inside the `step_fn`
+        as they do with a `MonitoredSession.run` call.
+
+    Returns:
+      Returns the returned value of `step_fn`.
+
+    Raises:
+      StopIteration: if `step_fn` has called `request_stop()`.  It may be
+        caught by `with tf.MonitoredSession()` to close the session.
+      ValueError: if `step_fn` doesn't have a single argument called
+        `step_context`. It may also optionally have `self` for cases when it
+        belongs to an object.
+    """
+    step_fn_arguments = util.fn_args(step_fn)
+    if step_fn_arguments != ('step_context',) and step_fn_arguments != (
+        'self',
+        'step_context',
+    ):
+      raise ValueError(
+          '`step_fn` may either have one `step_context` argument, or'
+          ' `self` and `step_context` arguments if it\'s an instance'
+          ' method. Got {} instead.'.format(step_fn_arguments))
+
+    try:
+      return step_fn(_MonitoredSession.StepContext(self._tf_sess(), self.run))
+    except StopIteration:
+      self._stop_requested_in_step_fn = True
+      raise
+
+  class StepContext(object):
+    """Control flow instrument for the `step_fn` from `run_step_fn()`.
+
+       Users of `step_fn` may perform `run()` calls without running hooks
+       by accessing the `session`.  A `run()` call with hooks may be performed
+       using `run_with_hooks()`.  Computation flow can be interrupted using
+       `request_stop()`.
+    """
+
+    def __init__(self, session, run_with_hooks_fn):
+      """Initializes the `step_context` argument for a `step_fn` invocation.
+
+      Args:
+        session: An instance of `tf.Session`.
+        run_with_hooks_fn: A function for running fetches and hooks.
+      """
+      self._session = session
+      self._run_with_hooks_fn = run_with_hooks_fn
+
+    @property
+    def session(self):
+      return self._session
+
+    def run_with_hooks(self, *args, **kwargs):
+      """Same as `MonitoredSession.run`. Accepts the same arguments."""
+      return self._run_with_hooks_fn(*args, **kwargs)
+
+    def request_stop(self):
+      """Exit the training loop by causing `should_stop()` to return `True`.
+
+         Causes `step_fn` to exit by raising an exception.
+
+      Raises:
+        StopIteration
+      """
+      raise StopIteration('step_fn has requested the iterations to stop.')
+
   def should_stop(self):
-    if self._sess:
-      return self._sess.should_stop()
-    return True
+    return (self._sess is None or self._sess.should_stop() or
+            self._stop_requested_in_step_fn)
 
   def close(self):
     self._close_internal()
diff --git a/tensorflow/python/training/monitored_session_test.py b/tensorflow/python/training/monitored_session_test.py
index 84d262935a..e729b79425 100644
--- a/tensorflow/python/training/monitored_session_test.py
+++ b/tensorflow/python/training/monitored_session_test.py
@@ -33,10 +33,12 @@ from tensorflow.core.protobuf import config_pb2
 from tensorflow.core.protobuf import debug_pb2
 from tensorflow.python.client import session as session_lib
 from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors_impl
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.ops import state_ops
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import test
@@ -1449,6 +1451,170 @@ class MonitoredSessionTest(test.TestCase):
       with monitored_session.MonitoredSession() as session:
         session.close()
 
+  def test_step_fn_example(self):
+    with ops.Graph().as_default():
+      c = array_ops.placeholder(dtypes.float32)
+      v = array_ops.identity(c)
+
+      def step_fn(step_context):
+        value = step_context.run_with_hooks(fetches=v, feed_dict={c: 3.2})
+        return value
+
+      with monitored_session.MonitoredSession() as session:
+        self.assertNear(3.2, session.run_step_fn(step_fn), 0.1)
+
+  def test_step_function_stops(self):
+    with ops.Graph().as_default():
+
+      def step_fn(step_context):
+        step_context.request_stop()
+
+      with monitored_session.MonitoredSession() as session:
+        self.assertEqual(None, session.run_step_fn(step_fn))
+        self.assertTrue(session.should_stop())
+
+  def test_step_request_stop_without_a_with_block(self):
+    with ops.Graph().as_default():
+
+      def step_fn(step_context):
+        step_context.request_stop()
+
+      session = monitored_session.MonitoredSession()
+      try:
+        self.assertEqual(None, session.run_step_fn(step_fn))
+      except StopIteration:
+        pass
+      self.assertTrue(session.should_stop())
+
+  def test_step_request_stop_in_a_loop(self):
+    with ops.Graph().as_default():
+      def step_fn(step_context):
+        step_context.request_stop()
+
+      with monitored_session.MonitoredSession() as session:
+        while not session.should_stop():
+          _ = session.run_step_fn(step_fn)
+          self.fail('An exception should be raised on the line above.')
+
+  def test_step_request_stop_with_returning_a_type(self):
+    with ops.Graph().as_default():
+
+      def step_fn(step_context):
+        del step_context
+        return 'a type'
+
+      with monitored_session.MonitoredSession() as session:
+        self.assertEqual('a type', session.run_step_fn(step_fn))
+
+  def test_step_with_extra_arguments(self):
+    with ops.Graph().as_default():
+
+      def step_fn(step_context, extra_foo):
+        del step_context, extra_foo
+
+      with monitored_session.MonitoredSession() as session:
+        with self.assertRaisesRegexp(
+            ValueError,
+            '`step_fn` may either have one `step_context` argument'):
+          self.assertEqual(None, session.run_step_fn(step_fn))
+
+  def test_step_fn_belongs_to_a_class(self):
+    with ops.Graph().as_default():
+      c = array_ops.placeholder(dtypes.float32)
+      v = array_ops.identity(c)
+
+      class Model(object):
+
+        def step_fn(self, step_context):
+          value = step_context.run_with_hooks(fetches=v, feed_dict={c: 3.2})
+          return value
+
+      with monitored_session.MonitoredSession() as session:
+        model = Model()
+        self.assertNear(3.2, session.run_step_fn(model.step_fn), 0.1)
+
+  def test_step_fn_belongs_to_a_class_and_has_extra_methods(self):
+    with ops.Graph().as_default():
+
+      class Model(object):
+
+        def step_fn(self, step_context, extra_foo):
+          del step_context, extra_foo
+
+      with monitored_session.MonitoredSession() as session:
+        with self.assertRaisesRegexp(
+            ValueError,
+            '`step_fn` may either have one `step_context` argument'):
+          model = Model()
+          self.assertEqual(None, session.run_step_fn(model.step_fn))
+
+  def test_step_fn_with_hooks(self):
+    with ops.Graph().as_default():
+      var = resource_variable_ops.ResourceVariable(0.0)
+
+      # This test higlights the interaction of hooks with
+      # `Monitoredsession.run_step_fn`.  The order of execution of operations
+      # below is:
+      #   0.  stage_0
+      #   1.  stage_1_0 or stage_1_1 in an undefined order
+      #   2.  stage_2
+
+      stage_0 = state_ops.assign_add(var, 0.3)
+      stage_1_0 = state_ops.assign_add(var, 0.7)
+      # The order of `stage_1_0` and `stage_1_1` is undefined by
+      # `MonitoredSession`, but we should be able to assert when both of them
+      # are complete.  To obtain a consistent result of adding two different
+      # constants to `var`, we rely on a control dependency and
+      # `ResourceVariable`.  Otherwise, it is possible that one of the
+      # additions overwites the result of the other addition.
+      with ops.control_dependencies([stage_1_0]):
+        stage_1_1 = state_ops.assign_add(var, 0.5)
+      stage_2 = state_ops.assign_add(var, 1.1)
+
+      class Hook(session_run_hook.SessionRunHook):
+
+        def __init__(self, testing):
+          self._testing = testing
+
+        def before_run(self, run_context):
+          return session_run_hook.SessionRunArgs(fetches=stage_1_0)
+
+        def after_run(self, run_context, run_values):
+          self._testing.assertNear(0.3 + 0.5 + 0.7,
+                                   run_context.session.run(var), 0.1)
+          self._testing.assertNear(0.3 + 0.5 + 0.7 + 1.1,
+                                   run_context.session.run(stage_2), 0.1)
+
+      def step_fn(step_context):
+        self.assertNear(0.3, step_context.session.run(stage_0), 0.1)
+        return step_context.run_with_hooks(fetches=stage_1_1)
+
+      with monitored_session.MonitoredSession(hooks=[Hook(self)]) as session:
+        self.assertEqual(0.3 + 0.5 + 0.7, session.run_step_fn(step_fn))
+
+  def test_step_fn_with_hooks_and_request_stop(self):
+    with ops.Graph().as_default():
+      trace_the_hook = {'before_run': False, 'after_run': False}
+
+      class Hook(session_run_hook.SessionRunHook):
+
+        def before_run(self, run_context):
+          trace_the_hook['before_run'] = True
+
+        def after_run(self, run_context, run_values):
+          trace_the_hook['after_run'] = True
+
+      def step_fn(step_context):
+        step_context.request_stop()
+
+      with monitored_session.MonitoredSession(hooks=[Hook()]) as session:
+        self.assertEqual(None, session.run_step_fn(step_fn))
+        self.assertTrue(session.should_stop())
+        # `step_context.request_stop()` in a step_fn interrupts the flow of
+        # running the hooks.
+        self.assertFalse(trace_the_hook['before_run'])
+        self.assertFalse(trace_the_hook['after_run'])
+
 
 class SingularMonitoredSessionTest(test.TestCase):
   """Tests SingularMonitoredSession."""
diff --git a/tensorflow/tools/api/golden/tensorflow.train.-monitored-session.-step-context.pbtxt b/tensorflow/tools/api/golden/tensorflow.train.-monitored-session.-step-context.pbtxt
new file mode 100644
index 0000000000..03efe6639e
--- /dev/null
+++ b/tensorflow/tools/api/golden/tensorflow.train.-monitored-session.-step-context.pbtxt
@@ -0,0 +1,21 @@
+path: "tensorflow.train.MonitoredSession.StepContext"
+tf_class {
+  is_instance: "<class \'tensorflow.python.training.monitored_session.StepContext\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "session"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'session\', \'run_with_hooks_fn\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "request_stop"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "run_with_hooks"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/tensorflow.train.-monitored-session.pbtxt b/tensorflow/tools/api/golden/tensorflow.train.-monitored-session.pbtxt
index 3a5cc015b4..09b7b3fb53 100644
--- a/tensorflow/tools/api/golden/tensorflow.train.-monitored-session.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.train.-monitored-session.pbtxt
@@ -3,6 +3,10 @@ tf_class {
   is_instance: "<class \'tensorflow.python.training.monitored_session.MonitoredSession\'>"
   is_instance: "<class \'tensorflow.python.training.monitored_session._MonitoredSession\'>"
   is_instance: "<type \'object\'>"
+  member {
+    name: "StepContext"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "graph"
     mtype: "<type \'property\'>"
@@ -19,6 +23,10 @@ tf_class {
     name: "run"
     argspec: "args=[\'self\', \'fetches\', \'feed_dict\', \'options\', \'run_metadata\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
   }
+  member_method {
+    name: "run_step_fn"
+    argspec: "args=[\'self\', \'step_fn\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "should_stop"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
diff --git a/tensorflow/tools/api/golden/tensorflow.train.-singular-monitored-session.-step-context.pbtxt b/tensorflow/tools/api/golden/tensorflow.train.-singular-monitored-session.-step-context.pbtxt
new file mode 100644
index 0000000000..36d8ce7ff8
--- /dev/null
+++ b/tensorflow/tools/api/golden/tensorflow.train.-singular-monitored-session.-step-context.pbtxt
@@ -0,0 +1,21 @@
+path: "tensorflow.train.SingularMonitoredSession.StepContext"
+tf_class {
+  is_instance: "<class \'tensorflow.python.training.monitored_session.StepContext\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "session"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'session\', \'run_with_hooks_fn\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "request_stop"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "run_with_hooks"
+    argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/tensorflow.train.-singular-monitored-session.pbtxt b/tensorflow/tools/api/golden/tensorflow.train.-singular-monitored-session.pbtxt
index 7caf837cc3..de0f2c1c1a 100644
--- a/tensorflow/tools/api/golden/tensorflow.train.-singular-monitored-session.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.train.-singular-monitored-session.pbtxt
@@ -3,6 +3,10 @@ tf_class {
   is_instance: "<class \'tensorflow.python.training.monitored_session.SingularMonitoredSession\'>"
   is_instance: "<class \'tensorflow.python.training.monitored_session._MonitoredSession\'>"
   is_instance: "<type \'object\'>"
+  member {
+    name: "StepContext"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "graph"
     mtype: "<type \'property\'>"
@@ -23,6 +27,10 @@ tf_class {
     name: "run"
     argspec: "args=[\'self\', \'fetches\', \'feed_dict\', \'options\', \'run_metadata\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
   }
+  member_method {
+    name: "run_step_fn"
+    argspec: "args=[\'self\', \'step_fn\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "should_stop"
     argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
-- 
GitLab


From e67f3af48c94c9456c3ff376dc30c82a4bf982cd Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Tue, 24 Oct 2017 17:45:17 -0700
Subject: [PATCH 741/909] Use 'LABEL maintainer=' in Dockerfile (#13961)

* Use 'LABEL maintainer=' in Dockerfile

This fix is a follow up of 13661 to replace `MAINTAINER`
with `LABEL maintainer=` in Dockerfile. The keyword
`MAINTAINER` has long been deprecated and is replaced by `LABEL`,
which is much more flexible and is easily searchable through `docker inspect`.

This fix replaces remaining `MAINTAINER` with `LABEL`.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Additional `MAITAINER` -> `LABEL`

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/tools/ci_build/Dockerfile.android              | 2 +-
 tensorflow/tools/ci_build/Dockerfile.cmake                | 2 +-
 tensorflow/tools/ci_build/Dockerfile.cpu                  | 2 +-
 tensorflow/tools/ci_build/Dockerfile.debian.jessie.cpu    | 2 +-
 tensorflow/tools/ci_build/Dockerfile.gpu                  | 2 +-
 tensorflow/tools/ci_build/Dockerfile.gpu_clang            | 2 +-
 tensorflow/tools/ci_build/Dockerfile.hadoop               | 2 +-
 tensorflow/tools/ci_build/Dockerfile.pi                   | 2 +-
 tensorflow/tools/ci_build/Dockerfile.pi-python3           | 2 +-
 tensorflow/tools/dist_test/Dockerfile                     | 2 +-
 tensorflow/tools/dist_test/Dockerfile.local               | 2 +-
 tensorflow/tools/dist_test/local/Dockerfile               | 2 +-
 tensorflow/tools/dist_test/server/Dockerfile              | 2 +-
 tensorflow/tools/dist_test/server/Dockerfile.test         | 2 +-
 tensorflow/tools/docker/Dockerfile.devel-gpu-cuda9-cudnn7 | 2 +-
 tensorflow/tools/gcs_test/Dockerfile                      | 2 +-
 16 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/tensorflow/tools/ci_build/Dockerfile.android b/tensorflow/tools/ci_build/Dockerfile.android
index facff47621..99a69d7b43 100644
--- a/tensorflow/tools/ci_build/Dockerfile.android
+++ b/tensorflow/tools/ci_build/Dockerfile.android
@@ -1,6 +1,6 @@
 FROM ubuntu:14.04
 
-MAINTAINER Jan Prach <jendap@google.com>
+LABEL maintainer="Jan Prach <jendap@google.com>"
 
 # Copy and run the install scripts.
 COPY install/*.sh /install/
diff --git a/tensorflow/tools/ci_build/Dockerfile.cmake b/tensorflow/tools/ci_build/Dockerfile.cmake
index 9013dc012d..37ba24d65a 100644
--- a/tensorflow/tools/ci_build/Dockerfile.cmake
+++ b/tensorflow/tools/ci_build/Dockerfile.cmake
@@ -14,7 +14,7 @@
 # ==============================================================================
 FROM ubuntu:16.04
 
-MAINTAINER Shanqing Cai <cais@google.com>
+LABEL maintainer="Shanqing Cai <cais@google.com>"
 
 # Copy and run the install scripts.
 COPY install/*.sh /install/
diff --git a/tensorflow/tools/ci_build/Dockerfile.cpu b/tensorflow/tools/ci_build/Dockerfile.cpu
index 206108930a..57a854a9df 100644
--- a/tensorflow/tools/ci_build/Dockerfile.cpu
+++ b/tensorflow/tools/ci_build/Dockerfile.cpu
@@ -1,6 +1,6 @@
 FROM ubuntu:14.04
 
-MAINTAINER Jan Prach <jendap@google.com>
+LABEL maintainer="Jan Prach <jendap@google.com>"
 
 # Copy and run the install scripts.
 COPY install/*.sh /install/
diff --git a/tensorflow/tools/ci_build/Dockerfile.debian.jessie.cpu b/tensorflow/tools/ci_build/Dockerfile.debian.jessie.cpu
index b914f51918..eb9d0d4dd0 100644
--- a/tensorflow/tools/ci_build/Dockerfile.debian.jessie.cpu
+++ b/tensorflow/tools/ci_build/Dockerfile.debian.jessie.cpu
@@ -1,6 +1,6 @@
 FROM debian:jessie
 
-MAINTAINER Jan Prach <jendap@google.com>
+LABEL maintainer="Jan Prach <jendap@google.com>"
 
 # Copy and run the install scripts.
 COPY install/*.sh /install/
diff --git a/tensorflow/tools/ci_build/Dockerfile.gpu b/tensorflow/tools/ci_build/Dockerfile.gpu
index 5d18295f68..2d46ccb6b1 100644
--- a/tensorflow/tools/ci_build/Dockerfile.gpu
+++ b/tensorflow/tools/ci_build/Dockerfile.gpu
@@ -1,6 +1,6 @@
 FROM nvidia/cuda:8.0-cudnn6-devel-ubuntu14.04
 
-MAINTAINER Jan Prach <jendap@google.com>
+LABEL maintainer="Jan Prach <jendap@google.com>"
 
 # In the Ubuntu 14.04 images, cudnn is placed in system paths. Move them to
 # /usr/local/cuda
diff --git a/tensorflow/tools/ci_build/Dockerfile.gpu_clang b/tensorflow/tools/ci_build/Dockerfile.gpu_clang
index c4342d17f5..0ecd8c75e0 100644
--- a/tensorflow/tools/ci_build/Dockerfile.gpu_clang
+++ b/tensorflow/tools/ci_build/Dockerfile.gpu_clang
@@ -1,6 +1,6 @@
 FROM nvidia/cuda:8.0-cudnn6-devel-ubuntu14.04
 
-MAINTAINER Ilya Biryukov <ibiryukov@google.com>
+LABEL maintainer="Ilya Biryukov <ibiryukov@google.com>"
 
 # In the Ubuntu 14.04 images, cudnn is placed in system paths. Move them to
 # /usr/local/cuda
diff --git a/tensorflow/tools/ci_build/Dockerfile.hadoop b/tensorflow/tools/ci_build/Dockerfile.hadoop
index 489493c26e..6010aedb33 100644
--- a/tensorflow/tools/ci_build/Dockerfile.hadoop
+++ b/tensorflow/tools/ci_build/Dockerfile.hadoop
@@ -1,6 +1,6 @@
 FROM ubuntu:14.04
 
-MAINTAINER Jonathan Hseu <jhseu@google.com>
+LABEL maintainer="Jonathan Hseu <jhseu@google.com>"
 
 # Copy and run the install scripts.
 COPY install/*.sh /install/
diff --git a/tensorflow/tools/ci_build/Dockerfile.pi b/tensorflow/tools/ci_build/Dockerfile.pi
index 2fddd6a2c0..75ef30d32b 100644
--- a/tensorflow/tools/ci_build/Dockerfile.pi
+++ b/tensorflow/tools/ci_build/Dockerfile.pi
@@ -1,6 +1,6 @@
 FROM ubuntu:14.04
 
-MAINTAINER Jan Prach <jendap@google.com>
+LABEL maintainer="Jan Prach <jendap@google.com>"
 
 # Copy and run the install scripts.
 COPY install/*.sh /install/
diff --git a/tensorflow/tools/ci_build/Dockerfile.pi-python3 b/tensorflow/tools/ci_build/Dockerfile.pi-python3
index 18b131ea19..b1c648ba30 100644
--- a/tensorflow/tools/ci_build/Dockerfile.pi-python3
+++ b/tensorflow/tools/ci_build/Dockerfile.pi-python3
@@ -1,6 +1,6 @@
 FROM ubuntu:14.04
 
-MAINTAINER Jan Prach <jendap@google.com>
+LABEL maintainer="Jan Prach <jendap@google.com>"
 
 # Copy and run the install scripts.
 COPY install/*.sh /install/
diff --git a/tensorflow/tools/dist_test/Dockerfile b/tensorflow/tools/dist_test/Dockerfile
index cd64e2c518..2a7605bbc9 100644
--- a/tensorflow/tools/dist_test/Dockerfile
+++ b/tensorflow/tools/dist_test/Dockerfile
@@ -20,7 +20,7 @@
 
 FROM ubuntu:16.04
 
-MAINTAINER Shanqing Cai <cais@google.com>
+LABEL maintainer="Shanqing Cai <cais@google.com>"
 
 RUN apt-get update
 RUN apt-get install -y \
diff --git a/tensorflow/tools/dist_test/Dockerfile.local b/tensorflow/tools/dist_test/Dockerfile.local
index 7a896ab611..795aeee1b5 100644
--- a/tensorflow/tools/dist_test/Dockerfile.local
+++ b/tensorflow/tools/dist_test/Dockerfile.local
@@ -19,7 +19,7 @@
 
 FROM ubuntu:16.04
 
-MAINTAINER Shanqing Cai <cais@google.com>
+LABEL maintainer="Shanqing Cai <cais@google.com>"
 
 # Pick up some TF dependencies.
 RUN apt-get update && apt-get install -y \
diff --git a/tensorflow/tools/dist_test/local/Dockerfile b/tensorflow/tools/dist_test/local/Dockerfile
index 96846f6564..383c3c2f4c 100644
--- a/tensorflow/tools/dist_test/local/Dockerfile
+++ b/tensorflow/tools/dist_test/local/Dockerfile
@@ -1,6 +1,6 @@
 FROM jpetazzo/dind
 
-MAINTAINER Shanqing Cai <cais@google.com>
+LABEL maintainer="Shanqing Cai <cais@google.com>"
 
 RUN apt-get update
 
diff --git a/tensorflow/tools/dist_test/server/Dockerfile b/tensorflow/tools/dist_test/server/Dockerfile
index fabc8a7105..1359428f11 100644
--- a/tensorflow/tools/dist_test/server/Dockerfile
+++ b/tensorflow/tools/dist_test/server/Dockerfile
@@ -19,7 +19,7 @@
 
 FROM ubuntu:16.04
 
-MAINTAINER Shanqing Cai <cais@google.com>
+LABEL maintainer="Shanqing Cai <cais@google.com>"
 
 # Pick up some TF dependencies
 RUN apt-get update && apt-get install -y \
diff --git a/tensorflow/tools/dist_test/server/Dockerfile.test b/tensorflow/tools/dist_test/server/Dockerfile.test
index 908af8af9b..ce7e783a1a 100644
--- a/tensorflow/tools/dist_test/server/Dockerfile.test
+++ b/tensorflow/tools/dist_test/server/Dockerfile.test
@@ -19,7 +19,7 @@
 
 FROM ubuntu:16.04
 
-MAINTAINER Shanqing Cai <cais@google.com>
+LABEL maintainer="Shanqing Cai <cais@google.com>"
 
 # Pick up some TF dependencies
 RUN apt-get update && apt-get install -y \
diff --git a/tensorflow/tools/docker/Dockerfile.devel-gpu-cuda9-cudnn7 b/tensorflow/tools/docker/Dockerfile.devel-gpu-cuda9-cudnn7
index 4558bc5293..64ebc4607a 100644
--- a/tensorflow/tools/docker/Dockerfile.devel-gpu-cuda9-cudnn7
+++ b/tensorflow/tools/docker/Dockerfile.devel-gpu-cuda9-cudnn7
@@ -1,6 +1,6 @@
 FROM nvidia/cuda:9.0-cudnn7-devel-ubuntu16.04
 
-MAINTAINER Gunhan Gulsoy <gunan@google.com>
+LABEL maintainer="Gunhan Gulsoy <gunan@google.com>"
 
 # It is possible to override these for releases.
 ARG TF_BRANCH=master
diff --git a/tensorflow/tools/gcs_test/Dockerfile b/tensorflow/tools/gcs_test/Dockerfile
index 5af753226f..69b554047b 100644
--- a/tensorflow/tools/gcs_test/Dockerfile
+++ b/tensorflow/tools/gcs_test/Dockerfile
@@ -1,6 +1,6 @@
 FROM ubuntu:16.04
 
-MAINTAINER Shanqing Cai <cais@google.com>
+LABEL maintainer="Shanqing Cai <cais@google.com>"
 
 RUN apt-get update
 RUN apt-get install -y \
-- 
GitLab


From 7ff50995aeffff8f534f6d9758a98ca9418e6816 Mon Sep 17 00:00:00 2001
From: Saurabh Saxena <srbs@google.com>
Date: Tue, 24 Oct 2017 18:03:26 -0700
Subject: [PATCH 742/909] Make Iterators saveable. Add
 tf.contrib.data.make_saveable_from_iterator(iterator) that builds a
 SaveableObject for an iterator so it can be saved/restored using tf.Saver.

PiperOrigin-RevId: 173340191
---
 tensorflow/contrib/data/BUILD                 |   1 +
 tensorflow/contrib/data/__init__.py           |   2 +
 .../contrib/data/python/kernel_tests/BUILD    |   2 +
 .../kernel_tests/range_dataset_op_test.py     | 133 ++++++++++++++++++
 tensorflow/contrib/data/python/ops/BUILD      |  13 ++
 .../contrib/data/python/ops/iterator_ops.py   |  77 ++++++++++
 6 files changed, 228 insertions(+)
 create mode 100644 tensorflow/contrib/data/python/ops/iterator_ops.py

diff --git a/tensorflow/contrib/data/BUILD b/tensorflow/contrib/data/BUILD
index ee96269a73..b485d78f5c 100644
--- a/tensorflow/contrib/data/BUILD
+++ b/tensorflow/contrib/data/BUILD
@@ -10,6 +10,7 @@ py_library(
     srcs_version = "PY2AND3",
     deps = [
         "//tensorflow/contrib/data/python/ops:dataset_ops",
+        "//tensorflow/contrib/data/python/ops:iterator_ops",
         "//tensorflow/contrib/data/python/ops:readers",
         "//tensorflow/contrib/data/python/ops:transformation_ops",
         "//tensorflow/python:util",
diff --git a/tensorflow/contrib/data/__init__.py b/tensorflow/contrib/data/__init__.py
index 7ff26e087b..e0aab1cd83 100644
--- a/tensorflow/contrib/data/__init__.py
+++ b/tensorflow/contrib/data/__init__.py
@@ -27,6 +27,7 @@ See the @{$datasets$Importing Data} Programmer's Guide for an overview.
 @@enumerate_dataset
 @@group_by_window
 @@ignore_errors
+@@make_saveable_from_iterator
 @@read_batch_features
 @@unbatch
 @@rejection_resample
@@ -49,6 +50,7 @@ from tensorflow.contrib.data.python.ops.dataset_ops import get_single_element
 from tensorflow.contrib.data.python.ops.enumerate_ops import enumerate_dataset
 from tensorflow.contrib.data.python.ops.error_ops import ignore_errors
 from tensorflow.contrib.data.python.ops.grouping import group_by_window
+from tensorflow.contrib.data.python.ops.iterator_ops import make_saveable_from_iterator
 from tensorflow.contrib.data.python.ops.readers import FixedLengthRecordDataset
 from tensorflow.contrib.data.python.ops.readers import read_batch_features
 from tensorflow.contrib.data.python.ops.readers import SqlDataset
diff --git a/tensorflow/contrib/data/python/kernel_tests/BUILD b/tensorflow/contrib/data/python/kernel_tests/BUILD
index b3175e3e56..96447abd7c 100644
--- a/tensorflow/contrib/data/python/kernel_tests/BUILD
+++ b/tensorflow/contrib/data/python/kernel_tests/BUILD
@@ -245,6 +245,7 @@ py_test(
     srcs_version = "PY2AND3",
     deps = [
         "//tensorflow/contrib/data/python/ops:dataset_ops",
+        "//tensorflow/contrib/data/python/ops:iterator_ops",
         "//tensorflow/contrib/data/python/ops:transformation_ops",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:client_testlib",
@@ -257,6 +258,7 @@ py_test(
         "//tensorflow/python:parsing_ops",
         "//tensorflow/python:platform",
         "//tensorflow/python:tensor_shape",
+        "//tensorflow/python:training",
         "//tensorflow/python:variables",
         "//tensorflow/python/data/ops:iterator_ops",
     ],
diff --git a/tensorflow/contrib/data/python/kernel_tests/range_dataset_op_test.py b/tensorflow/contrib/data/python/kernel_tests/range_dataset_op_test.py
index c944eb4a49..f59ac760dc 100644
--- a/tensorflow/contrib/data/python/kernel_tests/range_dataset_op_test.py
+++ b/tensorflow/contrib/data/python/kernel_tests/range_dataset_op_test.py
@@ -21,6 +21,7 @@ import os
 
 from tensorflow.contrib.data.python.ops import dataset_ops
 from tensorflow.contrib.data.python.ops import enumerate_ops
+from tensorflow.contrib.data.python.ops import iterator_ops as contrib_iterator_ops
 from tensorflow.python.data.ops import iterator_ops
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
@@ -34,6 +35,7 @@ from tensorflow.python.ops import parsing_ops
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import gfile
 from tensorflow.python.platform import test
+from tensorflow.python.training import saver as saver_lib
 
 
 class RangeDatasetTest(test.TestCase):
@@ -259,6 +261,137 @@ class RangeDatasetTest(test.TestCase):
         with self.assertRaises(errors.OutOfRangeError):
           sess.run(get_next)
 
+  def testSaveRestoreUsingSaverFromMetaGraph(self):
+
+    def _build_graph(start, stop):
+      iterator = dataset_ops.Dataset.range(start,
+                                           stop).make_initializable_iterator()
+      init_op = iterator.initializer
+      get_next = iterator.get_next()
+      ops.add_to_collection("iterator_ops", init_op)
+      ops.add_to_collection("iterator_ops", get_next)
+      saveable_obj = contrib_iterator_ops.make_saveable_from_iterator(iterator)
+      # Add the SaveableObject to the `SAVEABLE_OBJECTS` collection
+      # so that it can be automatically picked up by the Saver.
+      ops.add_to_collection(ops.GraphKeys.SAVEABLE_OBJECTS, saveable_obj)
+      saver = saver_lib.Saver()
+      return init_op, get_next, saver
+
+    start = 2
+    stop = 10
+    break_point = 5
+    path = self._iterator_checkpoint_prefix()
+    meta_filename = path + ".meta"
+
+    # Execute input pipeline for a few steps and save iterator state.
+    with ops.Graph().as_default() as g:
+      init_op, get_next, saver = _build_graph(start, stop)
+      with self.test_session(graph=g) as sess:
+        sess.run(variables.global_variables_initializer())
+        sess.run(init_op)
+        for i in range(start, break_point):
+          self.assertEqual(i, sess.run(get_next))
+        saver.save(sess, path)
+
+    # Build the saver from the MetaGraph using import_meta_graph and
+    # check that the iterator state is restored.
+    with ops.Graph().as_default() as g:
+      saver = saver_lib.import_meta_graph(meta_filename)
+      init_op, get_next = ops.get_collection("iterator_ops")
+      with self.test_session(graph=g) as sess:
+        saver.restore(sess, saver_lib.latest_checkpoint(self.get_temp_dir()))
+        for i in range(break_point, stop):
+          self.assertEqual(i, sess.run(get_next))
+        with self.assertRaises(errors.OutOfRangeError):
+          sess.run(get_next)
+
+  def testSaveRestoreUsingBuiltSaver(self):
+
+    def _build_graph(start, stop):
+      iterator = dataset_ops.Dataset.range(start,
+                                           stop).make_initializable_iterator()
+      init_op = iterator.initializer
+      get_next = iterator.get_next()
+      ops.add_to_collection("iterator_ops", init_op)
+      ops.add_to_collection("iterator_ops", get_next)
+      # Add the SaveableObject to the `SAVEABLE_OBJECTS` collection
+      # so that it can be automatically picked up by the Saver.
+      saveable_obj = contrib_iterator_ops.make_saveable_from_iterator(iterator)
+      ops.add_to_collection(ops.GraphKeys.SAVEABLE_OBJECTS, saveable_obj)
+      saver = saver_lib.Saver()
+      return init_op, get_next, saver
+
+    start = 2
+    stop = 10
+    stop_new = 15
+    break_point = 5
+    path = self._iterator_checkpoint_prefix()
+
+    # Execute input pipeline for a few steps and save iterator state.
+    with ops.Graph().as_default() as g:
+      init_op, get_next, saver = _build_graph(start, stop)
+      with self.test_session(graph=g) as sess:
+        sess.run(variables.global_variables_initializer())
+        sess.run(init_op)
+        for i in range(start, break_point):
+          self.assertEqual(i, sess.run(get_next))
+        saver.save(sess, path)
+
+    # Manually build a modified Graph and Saver instead of importing
+    # MetaGraph and verify that original iterator state gets restored.
+    with ops.Graph().as_default() as g:
+      init_op, get_next, saver = _build_graph(start, stop_new)
+      with self.test_session(graph=g) as sess:
+        saver.restore(sess, saver_lib.latest_checkpoint(self.get_temp_dir()))
+        for i in range(break_point, stop):
+          self.assertEqual(i, sess.run(get_next))
+        with self.assertRaises(errors.OutOfRangeError):
+          sess.run(get_next)
+
+  def testSaveRestoreUsingSaverThenInit(self):
+
+    def _build_graph(start, stop):
+      iterator = dataset_ops.Dataset.range(start,
+                                           stop).make_initializable_iterator()
+      init_op = iterator.initializer
+      get_next = iterator.get_next()
+      ops.add_to_collection("iterator_ops", init_op)
+      ops.add_to_collection("iterator_ops", get_next)
+      # Add the SaveableObject to the `SAVEABLE_OBJECTS` collection
+      # so that it can be automatically picked up by the Saver.
+      saveable_obj = contrib_iterator_ops.make_saveable_from_iterator(iterator)
+      ops.add_to_collection(ops.GraphKeys.SAVEABLE_OBJECTS, saveable_obj)
+      saver = saver_lib.Saver()
+      return init_op, get_next, saver
+
+    start = 2
+    stop = 10
+    stop_new = 15
+    break_point = 5
+    path = self._iterator_checkpoint_prefix()
+
+    # Execute input pipeline for a few steps and save iterator state.
+    with ops.Graph().as_default() as g:
+      init_op, get_next, saver = _build_graph(start, stop)
+      with self.test_session(graph=g) as sess:
+        sess.run(variables.global_variables_initializer())
+        sess.run(init_op)
+        for i in range(start, break_point):
+          self.assertEqual(i, sess.run(get_next))
+        saver.save(sess, path)
+
+    # Restore iterator state call and then call init_op for the iterator and
+    # verify that the new iterator hides the restored iterator.
+    with ops.Graph().as_default() as g:
+      init_op, get_next, saver = _build_graph(start, stop_new)
+      with self.test_session(graph=g) as sess:
+        saver.restore(sess, saver_lib.latest_checkpoint(self.get_temp_dir()))
+        sess.run(init_op)
+        for i in range(start, stop_new):
+          self.assertEqual(i, sess.run(get_next))
+        with self.assertRaises(errors.OutOfRangeError):
+          sess.run(get_next)
+
   def testRestoreWithoutBuildingDatasetGraph(self):
 
     def _build_graph(start, stop, num_epochs):
diff --git a/tensorflow/contrib/data/python/ops/BUILD b/tensorflow/contrib/data/python/ops/BUILD
index 2a9b41d6df..b17b02ee35 100644
--- a/tensorflow/contrib/data/python/ops/BUILD
+++ b/tensorflow/contrib/data/python/ops/BUILD
@@ -20,6 +20,19 @@ py_library(
     ],
 )
 
+py_library(
+    name = "iterator_ops",
+    srcs = [
+        "iterator_ops.py",
+    ],
+    srcs_version = "PY2AND3",
+    deps = [
+        "//tensorflow/python:dataset_ops_gen",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:training",
+    ],
+)
+
 py_library(
     name = "readers",
     srcs = [
diff --git a/tensorflow/contrib/data/python/ops/iterator_ops.py b/tensorflow/contrib/data/python/ops/iterator_ops.py
new file mode 100644
index 0000000000..d736029fb0
--- /dev/null
+++ b/tensorflow/contrib/data/python/ops/iterator_ops.py
@@ -0,0 +1,77 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Iterator ops."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import gen_dataset_ops
+from tensorflow.python.training import saver
+
+
+def make_saveable_from_iterator(iterator):
+  """Returns a SaveableObject for saving/restore iterator state using Saver.
+
+  Args:
+    iterator: Iterator.
+
+  For example:
+
+  ```python
+  with tf.Graph().as_default():
+    ds = tf.data.Dataset.range(10)
+    iterator = ds.make_initializable_iterator()
+    # Build the iterator SaveableObject.
+    saveable_obj = tf.contrib.data.make_saveable_from_iterator(iterator)
+    # Add the SaveableObject to the SAVEABLE_OBJECTS collection so
+    # it can be automatically saved using Saver.
+    tf.add_to_collection(tf.GraphKeys.SAVEABLE_OBJECTS, saveable_obj)
+    saver = tf.train.Saver()
+
+    while continue_training:
+      ... Perform training ...
+      if should_save_checkpoint:
+        saver.save()
+  ```
+
+  Note: When restoring the iterator, the existing iterator state is completely
+  discarded. This means that any changes you may have made to the Dataset
+  graph will be discarded as well! This includes the new Dataset graph
+  that you may have built during validation. So, while running validation,
+  make sure to run the initializer for the validation input pipeline after
+  restoring the checkpoint.
+
+  Note: Not all iterators support checkpointing yet. Attempting to save the
+  state of an unsupported iterator will throw an error.
+  """
+  return _Saveable(iterator._iterator_resource)  # pylint: disable=protected-access
+
+
+class _Saveable(saver.BaseSaverBuilder.SaveableObject):
+  """SaveableObject for saving/restoring iterator state."""
+
+  def __init__(self, iterator_resource):
+    serialized_iterator = gen_dataset_ops.serialize_iterator(iterator_resource)
+    specs = [
+        saver.BaseSaverBuilder.SaveSpec(serialized_iterator, "",
+                                        iterator_resource.name + "-state")
+    ]
+    super(_Saveable, self).__init__(iterator_resource, specs,
+                                    iterator_resource.name)
+
+  def restore(self, restored_tensors, unused_restored_shapes):
+    with ops.colocate_with(self.op):
+      return gen_dataset_ops.deserialize_iterator(self.op, restored_tensors[0])
-- 
GitLab


From f1ecdd6ea3eec5f75ba47676dbfa3c283b4e172a Mon Sep 17 00:00:00 2001
From: Michael Case <mikecase@google.com>
Date: Tue, 24 Oct 2017 18:07:59 -0700
Subject: [PATCH 743/909] Write common android build configs to .bazelrc

Add --config=android_arm and --config=android_arm64 Bazel configs
to add options needed for Android Bazel builds.

PiperOrigin-RevId: 173340664
---
 configure.py | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/configure.py b/configure.py
index 9da49b628d..ea3f598f3d 100644
--- a/configure.py
+++ b/configure.py
@@ -963,6 +963,19 @@ def set_monolithic():
   write_to_bazelrc('build --define framework_shared_object=true')
 
 
+def create_android_bazelrc_configs():
+  # Flags for --config=android
+  write_to_bazelrc('build:android --crosstool_top=//external:android/crosstool')
+  write_to_bazelrc(
+      'build:android --host_crosstool_top=@bazel_tools//tools/cpp:toolchain')
+  # Flags for --config=android_arm
+  write_to_bazelrc('build:android_arm --config=android')
+  write_to_bazelrc('build:android_arm --cpu=armeabi-v7a')
+  # Flags for --config=android_arm64
+  write_to_bazelrc('build:android_arm64 --config=android')
+  write_to_bazelrc('build:android_arm64 --cpu=arm64-v8a')
+
+
 def main():
   # Make a copy of os.environ to be clear when functions and getting and setting
   # environment variables.
@@ -1032,7 +1045,7 @@ def main():
   set_cc_opt_flags(environ_cp)
   set_mkl()
   set_monolithic()
-
+  create_android_bazelrc_configs()
 
 if __name__ == '__main__':
   main()
-- 
GitLab


From e384e28a97bc9f1da1402c93087e0394e5a0168c Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Tue, 24 Oct 2017 18:55:42 -0700
Subject: [PATCH 744/909] Re-land: Dump the computation's SessionModule as part
 of the tf_compile rule.

Nondeterminism in the SessionModule proto dumped by aot/compile.cc was
causing problems for some users. Re-landed with the SessionModule proto
being generated in a different genrule (so as not to disturb existing
users), and with more determinism in the dumped proto.

PiperOrigin-RevId: 173344189
---
 tensorflow/compiler/aot/compile.cc    | 10 +++++++---
 tensorflow/compiler/aot/flags.cc      |  5 ++---
 tensorflow/compiler/aot/flags.h       |  2 +-
 tensorflow/compiler/aot/tfcompile.bzl | 28 +++++++++++++++++++++++++++
 4 files changed, 38 insertions(+), 7 deletions(-)

diff --git a/tensorflow/compiler/aot/compile.cc b/tensorflow/compiler/aot/compile.cc
index eac8da0ab1..2b8cc6024c 100644
--- a/tensorflow/compiler/aot/compile.cc
+++ b/tensorflow/compiler/aot/compile.cc
@@ -97,11 +97,15 @@ Status CompileGraph(const GraphDef& graph_def, const tf2xla::Config& config,
   TF_RETURN_IF_ERROR(ConvertGraphDefToXla(graph_def, config, client,
                                           &computation,
                                           &compile_result->has_context_arg));
-  if (!flags.debug_dir.empty()) {
+  if (!flags.out_session_module.empty()) {
     TF_ASSIGN_OR_RETURN(std::unique_ptr<xla::SessionModule> module,
                         computation.Snapshot());
-    string file = io::JoinPath(flags.debug_dir, "tfcompile_xla_module.pb");
-    TF_RETURN_IF_ERROR(WriteBinaryProto(Env::Default(), file, *module));
+    // Serialize the SessionModule deterministically so that all the outputs of
+    // a tf_library genrule are deterministic.
+    string proto;
+    TF_RET_CHECK(SerializeToStringDeterministic(*module, &proto));
+    TF_RETURN_IF_ERROR(
+        WriteStringToFile(Env::Default(), flags.out_session_module, proto));
   }
   xla::cpu::CpuAotCompilationOptions aot_opts(
       flags.target_triple, flags.target_cpu, flags.target_features,
diff --git a/tensorflow/compiler/aot/flags.cc b/tensorflow/compiler/aot/flags.cc
index 5aff10346f..7c2f27e550 100644
--- a/tensorflow/compiler/aot/flags.cc
+++ b/tensorflow/compiler/aot/flags.cc
@@ -33,9 +33,6 @@ void AppendMainFlags(std::vector<Flag>* flag_list, MainFlags* flags) {
        "fetch nodes will be dumped to stdout in a comma-separated list.  "
        "Typically used to format arguments for other tools, e.g. "
        "freeze_graph."},
-      {"debug_dir", &flags->debug_dir,
-       "Specifies a directory to dump debugging information, including "
-       "rewritten graphs and the XLA HLO module."},
       // Flags controlling the XLA ahead-of-time compilation, that correspond to
       // the fields of xla::cpu::CpuAotCompilationOptions.
       //
@@ -64,6 +61,8 @@ void AppendMainFlags(std::vector<Flag>* flag_list, MainFlags* flags) {
        "namespaces are given, within the global namespace."},
       {"out_object", &flags->out_object, "Output object file name."},
       {"out_header", &flags->out_header, "Output header file name."},
+      {"out_session_module", &flags->out_session_module,
+       "Output session module proto."},
       {"gen_name_to_index", &flags->gen_name_to_index,
        "Generate name-to-index data for Lookup{Arg,Result}Index methods."},
       {"gen_program_shape", &flags->gen_program_shape,
diff --git a/tensorflow/compiler/aot/flags.h b/tensorflow/compiler/aot/flags.h
index 3246dbf95c..3519659e3a 100644
--- a/tensorflow/compiler/aot/flags.h
+++ b/tensorflow/compiler/aot/flags.h
@@ -29,7 +29,6 @@ struct MainFlags {
   string graph;
   string config;
   bool dump_fetch_nodes = false;
-  string debug_dir;
   string target_triple;
   string target_cpu;
   string target_features;
@@ -37,6 +36,7 @@ struct MainFlags {
   string cpp_class;
   string out_object;
   string out_header;
+  string out_session_module;
 
   // C++ codegen options
   bool gen_name_to_index = false;
diff --git a/tensorflow/compiler/aot/tfcompile.bzl b/tensorflow/compiler/aot/tfcompile.bzl
index 4888760acd..2adb1dc65e 100644
--- a/tensorflow/compiler/aot/tfcompile.bzl
+++ b/tensorflow/compiler/aot/tfcompile.bzl
@@ -165,6 +165,34 @@ def tf_library(name, graph, config,
       tags=tags,
   )
 
+  # Rule that runs tfcompile to produce the SessionModule proto, useful for
+  # debugging.  TODO(b/64813587): Once the SessionModule proto is
+  # deterministic, move this into the main rule above.
+  session_module_pb = name + "_session_module.pb"
+  native.genrule(
+      name=(name + "_session_module"),
+      srcs=[
+          tfcompile_graph,
+          config,
+      ],
+      outs=[
+          session_module_pb,
+      ],
+      cmd=("$(location " + tfcompile_tool + ")" +
+           " --graph=$(location " + tfcompile_graph + ")" +
+           " --config=$(location " + config + ")" +
+           " --entry_point=" + ep +
+           " --cpp_class=" + cpp_class +
+           " --target_triple=" + target_llvm_triple() +
+           " --out_session_module=$(@D)/" + session_module_pb +
+           " " + (tfcompile_flags or "")),
+      tools=[tfcompile_tool],
+      visibility=visibility,
+      testonly=testonly,
+      local=1,
+      tags=tags,
+  )
+
   # The cc_library rule packaging up the header and object file, and needed
   # kernel implementations.
   need_xla_data_proto = (tfcompile_flags and
-- 
GitLab


From 355e25ebcab64e833dfc987638c3e6c79d838266 Mon Sep 17 00:00:00 2001
From: Benoit Steiner <bsteiner@google.com>
Date: Tue, 24 Oct 2017 19:47:46 -0700
Subject: [PATCH 745/909] Merge changes from github. END_PUBLIC

---
Commit 9f8523640 authored by A. Unique TensorFlower<gardener@tensorflow.org>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Update ops-related pbtxt files.

PiperOrigin-RevId: 173145770

---
Commit 01b6b0638 authored by A. Unique TensorFlower<gardener@tensorflow.org>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Cut tracing memory cost

PiperOrigin-RevId: 173144626

---
Commit 5e23e0e67 authored by A. Unique TensorFlower<gardener@tensorflow.org>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
[XLA] Erase cloned instructions on the fly when merging fusion nodes.

This avoids the awkward situation where an RNG which is clearly eligible for fusion becomes ineligible mid-fusion because it suddenly has an extra (dead) user.

PiperOrigin-RevId: 173141716

---
Commit 1038927c0 authored by Saurabh Saxena<srbs@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Add SerializeIterator op that serializes an IteratorResource into a variant tensor.
Add DeserializeIterator op that builds IteratorResource from a variant tensor.
Move BundleReaderWrapper and BundleWriterWrapper from dataset.h to iterator_ops.cc.
Add generic key-value store interfaces IteratorStateReader and IteratorStateWriter for reading/writing state of iterators.
Get rid of IteratorBundleReader and IteratorBundleWriter.

PiperOrigin-RevId: 173140858

---
Commit 57f3e529d authored by A. Unique TensorFlower<gardener@tensorflow.org>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Internal change

PiperOrigin-RevId: 173136642

---
Commit 0e56ffb7b authored by Shanqing Cai<cais@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Fix breakages in OSS builds

See example breakages logs at:
http://ci.tensorflow.org/job/tensorflow-cl-cpu-python3-pip/10847/console
http://ci.tensorflow.org/job/tensorflow-cl-gpu/11008/console

1. CL/172477381 added the no_oss tag to tests with oss_serial tags, which broke the logic of OSS_SERIAL tests in pip.sh and run_pip_test.sh. This CL fixes that.

2. The nccl_kernels BUILD target in contrib/nccl/BUILD was missing some dependencies. This CL adds the missing ones.

Fixes: #13918
PiperOrigin-RevId: 173133914

---
Commit 3ed049b67 authored by Alexandre Passos<apassos@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Allows calling keras layers in eager mode.

PiperOrigin-RevId: 173129805

---
Commit 4ec6f2b07 authored by Alexandre Passos<apassos@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Switching contrib.summaries API to be context-manager-centric

PiperOrigin-RevId: 173129793

---
Commit 03b02ffc9 authored by Justine Tunney<jart@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Put Bazel mirror URLs first

PiperOrigin-RevId: 173127955

---
Commit 46ab25e4d authored by David Majnemer<majnemer@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
[XLA] Add support for convolutions with no spatial dimensions

PiperOrigin-RevId: 173126950

---
Commit fc56349b7 authored by Derek Murray<mrry@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
[tf.data] Convert dataset arguments to tensors as early as possible.

This change raises a `TypeError` earlier if (for example) the `batch_size`
argument to `Dataset.batch()` has the incorrect type.

PiperOrigin-RevId: 173126678

---
Commit 4f7503a87 authored by A. Unique TensorFlower<gardener@tensorflow.org>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
K-FAC: Support for registering multiple minibatches with register_fully_connected()

PiperOrigin-RevId: 173121735

---
Commit 2845bfcd6 authored by Tim Harley<tharley@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Avoid listing all modified Enter/RefEnter nodes on INFO, use VLOG(1) instead.

Leave a single, simple, message on INFO.

PiperOrigin-RevId: 173121726

---
Commit 434695921 authored by A. Unique TensorFlower<gardener@tensorflow.org>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
K-FAC: _check_registration() supports multiple towers.

PiperOrigin-RevId: 173115870

---
Commit 670dddf4a authored by A. Unique TensorFlower<gardener@tensorflow.org>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Multi-minibatch support for
tf.contrib.kfac.fisher_blocks.FullyConnectedKFACBasicFB.

PiperOrigin-RevId: 173109677

---
Commit dc13a8e2f authored by A. Unique TensorFlower<gardener@tensorflow.org>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Fix import of meta graphs with partitioned variables into a scope.

Saver inspects SliceInfo to decide the variable name when creating a
checkpoint. Before this fix even if a partitioned variable ("weights")
was imported into a scope "a" it would still be checkpointed as ("weights")
instead of ("a/weights") since import_scoped_meta_graph was not adjusting
the SliceInfo.

WARNING: if you use import_meta_graph on graphs with partitioned_variables WITH an import_scope argument AND then create a Saver to write/read checkpoints this change
may break your checkpoint loading.
PiperOrigin-RevId: 173105796

---
Commit eea089bdb authored by A. Unique TensorFlower<gardener@tensorflow.org>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
K-FAC: Multi-tower support for ConvDiagonalFB.

PiperOrigin-RevId: 173105412

---
Commit 9b9cbbe2a authored by Yong Tang<yong.tang.github@outlook.com>
Committed by Vijay Vasudevan<vrv@google.com>:
Add int64 Tperm type support for `Transpose` (#13909)

* Add int64 Tperm type support for `Transpose`

This fix adds int64 Tperm support for `Transpose`. In
`array_ops.cc`, `Transpose` and `ConjugateTranspose`
have been specified as accepting int32 and int64 perm
types. However, only int32 kernels has been registered.

This fix adds the int64 perm support by removing
the constraint on Tperm, resolve the type at runtime,
and copying the data type accordingly to correctly handle
the int64/int32 types.

Additional tests have been added as well.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Add test cases for int64 of perm in Transpose.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Add namespace to hide PermutationHelper

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Enable use_gpu=True for perm type test.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* extra // namespace annotation

* Adding a comment about int32 casting that should be safe.

Permutations only contain values that refer to dimensions, and the maximum number of dimensions we have is 254, so an int32 is always safe here.

---
Commit ac0004e71 authored by Yong Tang<yong.tang.github@outlook.com>
Committed by Vijay Vasudevan<vrv@google.com>:
Add int64 shape support on GPU for stateless random ops. (#13908)

* Add int64 shape support on GPU for stateless random ops.

This fix adds int64 shape support on GPU for stateless random ops
`StatelessRandomUniform`, `StatelessRandomNormal`, `StatelessTruncatedNormal`.

The int64 shape for stateless random ops is already supported on CPU
with int32/int64 processed properly through `MakeShape`.

However, on GPU a type constraint `.TypeConstraint<int32>("T")`
has been improperly added. Such a type constraint actually prevents
an int64 shape type to run on GPU. (As a comparision, no type constraint
on CPU).

This fix removes the type constraint and allows int64 shape to be run on GPU.

This fix also adds test cases for int64 shape support on stateless random ops.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Add test cases for int64 shape support for stateless random ops.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Add int32 to shape types tested.

---
Commit 0d437c3be authored by Yong Tang<yong.tang.github@outlook.com>
Committed by Vijay Vasudevan<vrv@google.com>:
Add int64 padding support for MirrorPad (#13907)

* Add int64 padding support for MirrorPad

This fix adds int64 padding support for `MirrorPad`.
In the `array_ops.cc` the `MirrorPad`/`MirrorPadGrad`
has been specified as supporting int64 padding. The related
kernels does not have the int64 padding registered though.
This fix adds the int64 padding support. This fix also adds
additional test cases for coverage.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Update template for CPU and GPU support of int64 paddings.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Add int64 padding support for MirrorPad

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Put eigen header first like before, just in case.

---
Commit 690003cc0 authored by Yong Tang<yong.tang.github@outlook.com>
Committed by Vijay Vasudevan<vrv@google.com>:
Add `int64` type `multiples` support for `tf.tile` (#13884)

* Add `int64` type `multiples` support for `tf.tile`

In the doc of `tf.tile` (tf.tile.__doc__) both `int32`
and `int64` are supported for `multiples`. However, the kernel
for `int64` is not registered yet.

This fix adds the support of `int64` `multiples` so that the
behavior matches the description of the docs.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Update functors for int64 multiples support in `tf.tile`

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Update test cases for int64 of multiples in `tf.tile`

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Add GPU and non GPU tests

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* format with clang-format -i

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Move Tmultiples after T (as it is  auxilliary)

And use `use_gpu=True`

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

---
Commit fd8d517b9 authored by Yunxing Dai<yunxing@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Add tests for convolution 1D
RELNOTES: n/a

PiperOrigin-RevId: 173060283

---
Commit 40c475b48 authored by formath<jinpengliu@163.com>
Committed by Vijay Vasudevan<vrv@google.com>:
add segment_reduction_ops to tf_op_files (#13901)

---
Commit bfa4ec194 authored by Tayo Oguntebi<10927929+tayo@users.noreply.github.com>
Committed by Vijay Vasudevan<vrv@google.com>:
Update node_def.proto comments (#13874)

The device field had outdated comments.

Note: We could consider adding tpu as an example here, e.g. "gpu" | "cpu" | "tpu".  Thoughts?
---
Commit c9cb5a58d authored by formath<jinpengliu@163.com>
Committed by Vijay Vasudevan<vrv@google.com>:
protobuf lib path bug fix for benckmark on osx (#13878)

---
Commit 1c1dad105 authored by Yong Tang<yong.tang.github@outlook.com>
Committed by Vijay Vasudevan<vrv@google.com>:
Add int64 axis support for reduction ops. (#13891)

* Add int64 axis support for reduction ops.

This fix is a follow up to PR 13863. In PR 13863 the
program crash is fixed if int64 axis is passed to reduction ops,
e.g. reduce_sum, reduce_max, etc. However, 13863 does not
process the case of int64 support, it merely fixes the crash.

This fix adds the support for int64 axis of reduction ops.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Add int64 axis support for mean, prod, sum

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Add int64 axis support for min and max.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Add int64 axis support for reduce_all and reduce_any

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Add test cases for int64 axis support of reduce_any and reduce_all

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

---
Commit 17096081e authored by Yong Tang<yong.tang.github@outlook.com>
Committed by Vijay Vasudevan<vrv@google.com>:
Improve resize_bicubic performance by reorganizing loops (#13840)

* Improve resize_bicubic performance by reorganizing loops

This fix tries to address the issue raised in 13693 where
performance of `resize_bicubic` is not on par with opencv.

This fix rearranges the loops so that it is the same for
num_channel=40 and num_channel=3:

Pre-fix:
```
CHANNEL=40
opencv: 145.08ms
tf: 314.26ms

CHANNEL=3
opencv: 11.95ms
tf: 8.95ms
```

Post-fix:
```
CHANNEL=40
opencv: 144.25ms
tf: 214.55ms

CHANNEL=3
opencv: 11.78ms
tf: 14.07ms
```

This fix fixes 13693.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Keep special handling of `num_channels=3` for `resize_bicubic`

This commit keeps special handling of `num_channels=3` for
`resize_bicubic`:
Without special handling:
```
opencv: 11.78ms
tf: 14.07ms
```
With special handling:
```
opencv: 11.74ms
tf: 9.46ms
```

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Expand Benchmark test for resize_bicubic

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Update from review feedback.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

---
Commit b927df57f authored by Yong Tang<yong.tang.github@outlook.com>
Committed by Vijay Vasudevan<vrv@google.com>:
Update protobuf.cmake to b04e5cba356212e4e8c66c61bbe0c3a20537c5b9 (#13893)

This fix tries to address the issue raised in 8187 where
protobuf.cmake used different version as bazel.

The reason for discrepancy was due to the fact that a customerized
protobuf was needed with Windows patch. Since the patch has been
merged in (https://github.com/google/protobuf/pull/2203),
it makes sense to update protobuf.cmake so that the same version
of cmake is used.

This fix fixes 8187.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
Commit d1183ca6a authored by Vijay Vasudevan<vrv@google.com>
Committed by GitHub<noreply@github.com>:
Give each variable a unique name in accumulate_n_v2_eager_test. (#13886)

---
Commit a69945810 authored by A. Unique TensorFlower<gardener@tensorflow.org>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Update pin for bazel-toolchains to latest version

PiperOrigin-RevId: 173002530

---
Commit 9d55c249c authored by Yong Tang<yong.tang.github@outlook.com>
Committed by Vijay Vasudevan<vrv@google.com>:
Fix doc in TF_CALL_ when invoked in mobile platform (#13881)

* Fix doc in TF_CALL_ when defined(IS_MOBILE_PLATFORM) && !defined(__ANDROID_TYPES_FULL__)

This is a small doc fix that includes bool as part of the types
that is supported in mobile (IS_MOBILE_PLATFORM && !__ANDROID_TYPES_FULL__),
as bool is clearly invoked in the following define.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Also add bool to android full version.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

---
Commit ba49d8583 authored by Bjarke Hammersholt Roune<broune@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Slight change to reduce_test to avoid generating inf, which was triggering an inf detector unnecessarily.

PiperOrigin-RevId: 172965466

---
Commit 93e8f3c67 authored by Anna R<annarev@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Adding Python ApiDef overrides.

PiperOrigin-RevId: 172960496

---
Commit 0d6a2e353 authored by Anna R<annarev@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Internal change.

PiperOrigin-RevId: 172960439

---
Commit 62df65c72 authored by A. Unique TensorFlower<gardener@tensorflow.org>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Add dtype argument to Mean and Accuracy object-oriented metrics.

PiperOrigin-RevId: 172957714

---
Commit d7409d32b authored by Simone Cirillo<my.accounts@gmx.se>
Committed by Vijay Vasudevan<vrv@google.com>:
Fix import of spatial_softmax from tensorflow.contrib.layers (#13833)

---
Commit df8bce63d authored by Yong Tang<yong.tang.github@outlook.com>
Committed by Vijay Vasudevan<vrv@google.com>:
Fix crash when `int64` axis is passed to `tf.reduce_sum` (#13863)

* Fix crash when `int64` axis is passed to `tf.reduce_sum`

This fix tries to fix the crash triggered by `int64` axis passed
to `tf.reduce_sum`:
```
ubuntu@ubuntu:~/tensorflow2$ (cd && python)
Python 2.7.12 (default, Nov 19 2016, 06:48:10)
[GCC 5.4.0 20160609] on linux2
Type "help", "copyright", "credits" or "license" for more information.
>>> import tensorflow as tf
>>> v = tf.reduce_sum([1,2,3], tf.constant(0, tf.int64))
2017-10-20 15:55:06.993430: F tensorflow/core/framework/tensor.cc:601] Check failed: dtype() == expected_dtype (9 vs. 3)
ubuntu@ubuntu:~/tensorflow2$
```

The issue is caused by the fact that shape inference in `common_shape_fns.cc`
only assumes int32 without proper handling of diffent types. In `math_ops.cc`
both int32 and int64 are mentioned.

NOTE that this fix does not address the issue that int64 is not supported.
To allow int64 axis it is more than adding a template in `ReductionOp` as the type
of the axis seems to be decided by some other ways in Eigen.

This fix merely fixed the crash so that an error message will return without
exit from the python program "No OpKernel was registered to support Op 'Sum' with these attrs".

Still, I think its worth to at least allow the program to continue in case of unsupported kernel.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Update implementation with a template helper function.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

---
Commit 29c7b4658 authored by A. Unique TensorFlower<gardener@tensorflow.org>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Adding the Stanford Tensorflow class to community resources.

PiperOrigin-RevId: 172956049

---
Commit f758b24a8 authored by Alexandre Passos<apassos@google.com>
Committed by Vijay Vasudevan<vrv@google.com>:
Variable name for the eager test (#13873)

---
Commit a5fe66b15 authored by A. Unique TensorFlower<gardener@tensorflow.org>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Removed some unnecessary broadcasts in binary ops where only one input needs
broadcasting (which is a fairly common case, even in the fallback path).

PiperOrigin-RevId: 172950493

---
Commit c77090a0a authored by Yong Tang<yong.tang.github@outlook.com>
Committed by Vijay Vasudevan<vrv@google.com>:
Fix issues where int64 crops could not be passed to batch_to_space. (#13862)

* Fix issues where int64 crops could not be passed to batch_to_space.

This fix tries to address the issue where int64 `crops` could
not be passed to `batch_to_space` even though both int32 and
int64 are specified as supported in the docs (tf.batch_to_space.__doc__)

The reason is that BatchToSpace kernel puts a constraint of int32 to crops
data types.

This fix removed the constraint so that int64 `crops` could be supported.

NOTE: Just removing the constraint should work and it is not necessary
to add specification to the kernel class template, as `SubtleMustCopyFlat`
called in the class already correctly handled both int32 and int64 cases.
Besides, other data types (e.g., float or double) will not be passed to the
kernel as they are guarded by the specification in `array_ops.cc`.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Also remove int64/int32 type constraints for SpaceToBatch kernels

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Add test cases for int64 crops of batch_to_space and space_to_batch

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Fix test failures.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

---
Commit 494837936 authored by Joshua V. Dillon<jvdillon@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Make `tf.contrib.distributions` quadrature family accept a `Tensor` for
`quadrature_grid_and_probs` argument.

PiperOrigin-RevId: 172950094

---
Commit 9c825d32c authored by Jinze Bai<baijinze1994@163.com>
Committed by Vijay Vasudevan<vrv@google.com>:
Merge two GPU kernel launching to one in DiagOp. (#13859)

---
Commit c0ca50a47 authored by Yan Facai (???)<facai.yan@gmail.com>
Committed by Vijay Vasudevan<vrv@google.com>:
ENH: add Relu6GradGrad (#13268)

* ENH: add Relu6GradGrad

* TST: add test case

* CLN: import nn_grad

* TST: add init value

---
Commit 8ff33271e authored by Justin Lebar<jlebar@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Dump the computation's SessionModule as part of the tf_compile rule.

PiperOrigin-RevId: 172946149

---
Commit ebcae4a5e authored by A. Unique TensorFlower<gardener@tensorflow.org>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Add streaming_precision_recall_at_equal_thresholds

This helper method computes streaming tp, fp, tn, fp, precision, and recall for the user in a way that exhibits O(T + N) time and space complexity (instead of O(T * N)), where T is the number of thresholds and N is the size of the predictions tensor.

Thanks to Frank Chu for the efficient algorithm!

PiperOrigin-RevId: 172946073

---
Commit ccfd9c1e5 authored by Sanjoy Das<sanjoy@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Log Hlo IR during AOT compilation

PiperOrigin-RevId: 172944165

---
Commit 985031a10 authored by Alexandre Passos<apassos@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Allows tfe.enable_eager_execution(device_policy=tfe.DEVICE_POLICY_WARN).

PiperOrigin-RevId: 172943398

---
Commit 703182d85 authored by Mingxing Tan<tanmingxing@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Add performance guide for fused decode_and_crop_jpeg optimization.

PiperOrigin-RevId: 172943116

---
Commit 66b1f4383 authored by Francois Chollet<fchollet@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Make Network compatible with eager mode. Currently it only allows to instantiate a Network in eager mode using the regular Keras API, and call it on eager tensors.

PiperOrigin-RevId: 172942569

---
Commit 41df2cec2 authored by ashankar<ashankar@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Testing pending CL: 172939383

---
Commit 37fd95179 authored by Alexandre Passos<apassos@google.com>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
Simplifies capturing code in graph_callable to use recent function improvements.

PiperOrigin-RevId: 172937003

---
Commit d1e7382af authored by A. Unique TensorFlower<gardener@tensorflow.org>
Committed by TensorFlower Gardener<gardener@tensorflow.org>:
BEGIN_PUBLIC
Automated g4 rollback of changelist 172924803

PiperOrigin-RevId: 173347587
---
 .gitignore                                    |   5 +
 README.md                                     |   9 +-
 RELEASE.md                                    |  81 ++++-
 configure.py                                  |   5 +-
 tensorflow/BUILD                              |   1 +
 tensorflow/c/c_api.h                          |   2 +-
 tensorflow/cc/gradients/math_grad.cc          |  32 +-
 tensorflow/cc/gradients/math_grad_test.cc     |  58 +++-
 tensorflow/compiler/jit/BUILD                 |   1 +
 tensorflow/compiler/plugin/BUILD              |  56 ++++
 tensorflow/compiler/plugin/README.md          |  16 +
 .../xla/service/hlo_computation_test.cc       |   2 +-
 tensorflow/compiler/xla/service/inliner.cc    |   6 +-
 .../compiler/xla/service/inliner_test.cc      |  39 +++
 .../contrib/all_reduce/python/all_reduce.py   |   2 +-
 tensorflow/contrib/boosted_trees/README.md    |   2 +-
 .../boosted_trees/examples/binary_mnist.py    |   2 +-
 .../contrib/boosted_trees/examples/mnist.py   |   2 +-
 tensorflow/contrib/cmake/external/cub.cmake   |   4 +-
 .../contrib/cmake/external/protobuf.cmake     |   4 +-
 .../contrib/cmake/tf_core_kernels.cmake       |   4 +
 tensorflow/contrib/cmake/tf_tests.cmake       |  90 +++---
 .../python/kernel_tests/cudnn_rnn_test.py     |   6 +-
 .../contrib/data/python/kernel_tests/BUILD    |   6 +
 tensorflow/contrib/framework/BUILD            |  29 +-
 .../framework/python/ops/accumulate_n_v2.py   | 111 +++++++
 .../python/ops/accumulate_n_v2_eager_test.py  |  85 +++++
 .../python/ops/accumulate_n_v2_test.py        | 123 ++++++++
 tensorflow/contrib/image/__init__.py          |   4 +
 .../python/kernel_tests/image_ops_test.py     |  33 +-
 .../contrib/image/python/ops/image_ops.py     | 294 +++++++++++------
 .../contrib/kfac/python/ops/loss_functions.py |   6 +-
 .../contrib/kfac/python/ops/op_queue.py       |   2 +-
 tensorflow/contrib/layers/__init__.py         |   1 +
 .../learn/python/learn/learn_runner.py        |   2 +-
 .../contrib/losses/python/losses/loss_ops.py  |  17 +-
 tensorflow/contrib/makefile/Makefile          |   4 +
 .../contrib/makefile/download_dependencies.sh |   2 +-
 tensorflow/contrib/makefile/tf_op_files.txt   |   1 +
 .../meta_graph_transform.py                   |   2 +-
 .../metrics/python/ops/metric_ops_test.py     |  54 +++-
 .../contrib/mpi_collectives/__init__.py       |   2 +-
 tensorflow/contrib/nn/__init__.py             |   2 +
 .../python/util/receptive_field.py            | 134 +++++++-
 .../python/util/receptive_field_test.py       |  56 ++++
 .../kernel_tests/stateless_random_ops_test.py |  16 +
 tensorflow/core/BUILD                         |  34 +-
 .../common_runtime/accumulate_n_optimizer.cc  | 191 ++++++++++++
 .../core/common_runtime/mkl_cpu_allocator.h   |  61 +++-
 .../common_runtime/mkl_cpu_allocator_test.cc  |  53 ++++
 tensorflow/core/framework/common_shape_fns.cc |  58 ++--
 tensorflow/core/framework/node_def.proto      |   2 +-
 tensorflow/core/framework/register_types.h    |   4 +-
 tensorflow/core/framework/rendezvous.cc       |   2 +-
 tensorflow/core/graph/graph.h                 |   2 +-
 tensorflow/core/graph/mkl_layout_pass.cc      |   2 +-
 tensorflow/core/graph/mkl_layout_pass_test.cc |   2 +-
 tensorflow/core/graph/testlib.cc              |  18 ++
 tensorflow/core/graph/testlib.h               |   6 +
 .../core/grappler/optimizers/model_pruner.cc  |   2 +-
 tensorflow/core/kernels/BUILD                 |  55 ++++
 tensorflow/core/kernels/batchtospace_op.cc    |  50 ++-
 tensorflow/core/kernels/conv_ops_gpu_3.cu.cc  |   2 +-
 .../core/kernels/crop_and_resize_op_test.cc   |   6 +-
 tensorflow/core/kernels/dataset.h             |   2 +-
 tensorflow/core/kernels/diag_op.cc            | 295 +++++++++++-------
 tensorflow/core/kernels/diag_op.h             |  43 +++
 tensorflow/core/kernels/diag_op_gpu.cu.cc     | 139 +++++++++
 tensorflow/core/kernels/diag_op_test.cc       |  54 ++++
 tensorflow/core/kernels/histogram_op.cc       | 147 +++++++++
 tensorflow/core/kernels/histogram_op.h        |  38 +++
 .../core/kernels/histogram_op_gpu.cu.cc       | 125 ++++++++
 tensorflow/core/kernels/listdiff_op.cc        |  16 +-
 tensorflow/core/kernels/map_stage_op.cc       |  12 +-
 tensorflow/core/kernels/mirror_pad_op.cc      | 200 +++++++-----
 tensorflow/core/kernels/mirror_pad_op.h       |  13 +-
 .../core/kernels/mirror_pad_op_cpu_impl.h     |  12 +-
 .../core/kernels/mirror_pad_op_gpu.cu.cc      |  32 +-
 tensorflow/core/kernels/mkl_conv_ops.cc       |   6 +-
 tensorflow/core/kernels/nth_element_op.cc     | 139 +++++++++
 tensorflow/core/kernels/nth_element_op.h      |  39 +++
 tensorflow/core/kernels/pad_op.cc             | 144 +++++++--
 tensorflow/core/kernels/pad_op.h              |  10 +-
 tensorflow/core/kernels/pad_op_gpu.cu.cc      |  20 +-
 tensorflow/core/kernels/reduction_ops_all.cc  |  16 +-
 tensorflow/core/kernels/reduction_ops_any.cc  |  16 +-
 .../core/kernels/reduction_ops_common.cc      |  22 +-
 .../core/kernels/reduction_ops_common.h       |  27 +-
 tensorflow/core/kernels/reduction_ops_max.cc  |  90 ++++--
 tensorflow/core/kernels/reduction_ops_mean.cc |  68 ++--
 tensorflow/core/kernels/reduction_ops_min.cc  |  90 ++++--
 tensorflow/core/kernels/reduction_ops_prod.cc |  68 ++--
 tensorflow/core/kernels/reduction_ops_sum.cc  |  90 ++++--
 tensorflow/core/kernels/resize_bicubic_op.cc  |  85 ++---
 .../core/kernels/resize_bicubic_op_test.cc    |  20 +-
 .../core/kernels/reverse_sequence_op.cc       |   3 +
 .../kernels/reverse_sequence_op_gpu.cu.cc     |   1 +
 tensorflow/core/kernels/scan_ops.cc           |  98 +++---
 tensorflow/core/kernels/sequence_ops.cc       |  48 +--
 tensorflow/core/kernels/sequence_ops_test.cc  | 148 +++++++++
 tensorflow/core/kernels/spacetobatch_op.cc    |  50 ++-
 tensorflow/core/kernels/sparse_matmul_op.h    |   6 +-
 tensorflow/core/kernels/stage_op.cc           |  14 +-
 .../core/kernels/stateless_random_ops.cc      |   3 -
 tensorflow/core/kernels/tile_functor.h        |  39 ++-
 tensorflow/core/kernels/tile_functor_cpu.cc   |  12 +-
 .../core/kernels/tile_functor_gpu.cu.cc       |  12 +-
 tensorflow/core/kernels/tile_ops.cc           | 249 ++++++---------
 tensorflow/core/kernels/transpose_op.cc       | 134 ++++----
 tensorflow/core/ops/array_ops.cc              |  44 ++-
 tensorflow/core/ops/array_ops_test.cc         |  13 +-
 tensorflow/core/ops/image_ops.cc              |  43 ++-
 tensorflow/core/ops/image_ops_test.cc         |   6 +-
 tensorflow/core/ops/math_ops.cc               |  72 ++++-
 tensorflow/core/ops/nn_ops.cc                 |  50 +++
 tensorflow/core/ops/nn_ops_test.cc            |  24 ++
 tensorflow/core/platform/s3/s3_crypto.cc      |   2 +-
 tensorflow/core/profiler/README.md            |   2 +-
 tensorflow/core/profiler/g3doc/options.md     |   2 +-
 tensorflow/core/public/version.h              |   4 +-
 .../api_guides/python/reading_data.md         |   2 +-
 tensorflow/docs_src/get_started/estimator.md  |  20 +-
 tensorflow/docs_src/install/install_c.md      |   2 +-
 tensorflow/docs_src/install/install_go.md     |   2 +-
 tensorflow/docs_src/install/install_java.md   |  18 +-
 tensorflow/docs_src/install/install_linux.md  |  38 ++-
 tensorflow/docs_src/install/install_mac.md    |  12 +-
 .../docs_src/install/install_sources.md       |  25 +-
 .../docs_src/install/install_windows.md       |   2 +-
 .../docs_src/performance/performance_guide.md |   2 +-
 .../performance/performance_models.md         |   2 +-
 .../docs_src/programmers_guide/datasets.md    |   2 +-
 .../docs_src/programmers_guide/graphs.md      |   4 +-
 .../docs_src/programmers_guide/saved_model.md |  33 ++
 tensorflow/docs_src/tutorials/wide.md         |   3 +-
 .../get_started/regression/imports85.py       |   2 +-
 .../linear_regression_categorical.py          |   2 +-
 tensorflow/examples/learn/resnet.py           |   4 +-
 .../tutorials/word2vec/word2vec_basic.py      |   6 +-
 tensorflow/java/BUILD                         |  39 +++
 .../processor/OperatorProcessor.java          | 164 ++++++++++
 .../javax.annotation.processing.Processor     |   1 +
 .../tensorflow/op/annotation/Operator.java    |   2 +-
 .../processor/OperatorProcessorTest.java      |  51 +++
 .../processor/operator/bad/BasicBad.java      |  22 ++
 .../processor/operator/good/BasicGood.java    |  21 ++
 tensorflow/python/BUILD                       |  13 +
 tensorflow/python/debug/cli/tensor_format.py  |   2 +-
 tensorflow/python/estimator/training.py       |   6 +-
 tensorflow/python/kernel_tests/BUILD          |  15 +
 .../kernel_tests/batchtospace_op_test.py      |  36 ++-
 .../python/kernel_tests/diag_op_test.py       |  64 +++-
 .../python/kernel_tests/listdiff_op_test.py   |  20 +-
 .../python/kernel_tests/metrics_test.py       |  51 ++-
 .../kernel_tests/nth_element_op_test.py       | 174 +++++++++++
 tensorflow/python/kernel_tests/pad_op_test.py |  28 ++
 .../python/kernel_tests/reduction_ops_test.py |  52 +++
 .../python/kernel_tests/scan_ops_test.py      |  18 ++
 .../python/kernel_tests/shape_ops_test.py     |  18 +-
 .../python/kernel_tests/slice_op_test.py      |  11 +
 .../python/kernel_tests/transpose_op_test.py  |  13 +
 tensorflow/python/ops/hidden_ops.txt          |   3 +
 tensorflow/python/ops/histogram_ops.py        |  31 +-
 tensorflow/python/ops/histogram_ops_test.py   |   8 +-
 tensorflow/python/ops/image_ops_test.py       |  29 +-
 tensorflow/python/ops/losses/losses_impl.py   |  22 +-
 tensorflow/python/ops/metrics_impl.py         |  14 +-
 tensorflow/python/ops/nn_grad.py              |  36 +++
 tensorflow/python/ops/nn_grad_test.py         |  48 +++
 tensorflow/python/ops/nn_ops.py               |  28 ++
 tensorflow/python/platform/self_check.py      |   8 +-
 .../tools/api/golden/tensorflow.losses.pbtxt  |   2 +-
 tensorflow/tools/ci_build/Dockerfile.pi       |   3 +
 .../tools/ci_build/Dockerfile.pi-python3      |  23 ++
 tensorflow/tools/ci_build/README.md           | 143 +++------
 .../tools/ci_build/builds/android_full.sh     |   4 +-
 .../tools/ci_build/builds/libtensorflow.sh    |  45 ++-
 .../tools/ci_build/install/install_golang.sh  |   2 +-
 .../install/install_pi_python3_toolchain.sh   |  29 ++
 .../ci_build/install/install_pi_toolchain.sh  |   2 +-
 .../tools/ci_build/linux/cpu/run_mkl.sh       |  36 +++
 tensorflow/tools/docker/Dockerfile            |   2 +-
 tensorflow/tools/docker/Dockerfile.devel      |   4 +-
 tensorflow/tools/docker/Dockerfile.devel-gpu  |  15 +-
 .../docker/Dockerfile.devel-gpu-cuda9-cudnn7  |  33 +-
 tensorflow/tools/docker/Dockerfile.gpu        |   2 +-
 tensorflow/tools/docker/README.md             |   1 +
 tensorflow/tools/pip_package/setup.py         |   5 +-
 tensorflow/workspace.bzl                      |  44 ++-
 third_party/aws.BUILD                         |   1 +
 .../CXX11/src/FixedPoint/PacketMathAVX2.h     |  51 ++-
 third_party/toolchains/cpus/arm/CROSSTOOL.tpl |   2 +-
 .../cpus/arm/arm_compiler_configure.bzl       |  11 +
 193 files changed, 5403 insertions(+), 1408 deletions(-)
 create mode 100644 tensorflow/compiler/plugin/BUILD
 create mode 100644 tensorflow/compiler/plugin/README.md
 create mode 100644 tensorflow/contrib/framework/python/ops/accumulate_n_v2.py
 create mode 100644 tensorflow/contrib/framework/python/ops/accumulate_n_v2_eager_test.py
 create mode 100644 tensorflow/contrib/framework/python/ops/accumulate_n_v2_test.py
 create mode 100644 tensorflow/core/common_runtime/accumulate_n_optimizer.cc
 create mode 100644 tensorflow/core/common_runtime/mkl_cpu_allocator_test.cc
 create mode 100644 tensorflow/core/kernels/diag_op.h
 create mode 100644 tensorflow/core/kernels/diag_op_gpu.cu.cc
 create mode 100644 tensorflow/core/kernels/diag_op_test.cc
 create mode 100644 tensorflow/core/kernels/histogram_op.cc
 create mode 100644 tensorflow/core/kernels/histogram_op.h
 create mode 100644 tensorflow/core/kernels/histogram_op_gpu.cu.cc
 create mode 100644 tensorflow/core/kernels/nth_element_op.cc
 create mode 100644 tensorflow/core/kernels/nth_element_op.h
 create mode 100644 tensorflow/core/kernels/sequence_ops_test.cc
 create mode 100644 tensorflow/java/src/gen/java/org/tensorflow/processor/OperatorProcessor.java
 create mode 100644 tensorflow/java/src/gen/resources/META-INF/services/javax.annotation.processing.Processor
 create mode 100644 tensorflow/java/src/test/java/org/tensorflow/processor/OperatorProcessorTest.java
 create mode 100644 tensorflow/java/src/test/resources/org/tensorflow/processor/operator/bad/BasicBad.java
 create mode 100644 tensorflow/java/src/test/resources/org/tensorflow/processor/operator/good/BasicGood.java
 create mode 100644 tensorflow/python/kernel_tests/nth_element_op_test.py
 create mode 100644 tensorflow/python/ops/nn_grad_test.py
 create mode 100644 tensorflow/tools/ci_build/Dockerfile.pi-python3
 create mode 100755 tensorflow/tools/ci_build/install/install_pi_python3_toolchain.sh
 create mode 100755 tensorflow/tools/ci_build/linux/cpu/run_mkl.sh

diff --git a/.gitignore b/.gitignore
index 09734fe497..9ae0d9c96f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -17,3 +17,8 @@ cmake_build/
 .idea/**
 /build/
 /tensorflow/core/util/version_info.cc
+/tensorflow/python/framework/fast_tensor_util.cpp
+Pods
+Podfile.lock
+*.pbxproj
+*.xcworkspacedata
diff --git a/README.md b/README.md
index 6339c57c95..24bbb6cec1 100644
--- a/README.md
+++ b/README.md
@@ -38,10 +38,11 @@ People who are a little more adventurous can also try our nightly binaries:
 
 **Nightly pip packages**
 * We are pleased to announce that TensorFlow now offers nightly pip packages
-under the [tf-nightly](https://pypi.python.org/pypi/tf-nightly) project on pypi.
-Simply run `pip install tf-nightly` in a clean environment to install the nightly
-tensorflow  build. We currently only support CPU packages on Linux, Mac, and Windows.
-GPU packages on all platforms will arrive soon!
+under the [tf-nightly](https://pypi.python.org/pypi/tf-nightly) and
+[tf-nightly-gpu](https://pypi.python.org/pypi/tf-nightly-gpu) project on pypi.
+Simply run `pip install tf-nightly` or `pip install tf-nightly-gpu` in a clean
+environment to install the nightly TensorFlow build. We support CPU and GPU
+packages on Linux, Mac, and Windows.
 
 
 **Individual whl files**
diff --git a/RELEASE.md b/RELEASE.md
index 2c6535c15d..4a33bce8b2 100644
--- a/RELEASE.md
+++ b/RELEASE.md
@@ -1,20 +1,51 @@
 # Release 1.4.0
 
 ## Major Features And Improvements
+* `tf.keras` is now part of the core TensorFlow API.
 * [`tf.data`](http://tensorflow.org/programmers_guide/datasets) is now part of
   the core TensorFlow API.
   * The API is now subject to backwards compatibility guarantees.
   * For a guide to migrating from the `tf.contrib.data` API, see the
-    [README] (https://github.com/tensorflow/tensorflow/blob/r1.4/tensorflow/contrib/data/README.md).
+    [README](https://github.com/tensorflow/tensorflow/blob/r1.4/tensorflow/contrib/data/README.md).
   * Major new features include `Dataset.from_generator()` (for building an input
     pipeline from a Python generator), and the `Dataset.apply()` method for
     applying custom transformation functions.
   * Several custom transformation functions have been added, including
     `tf.contrib.data.batch_and_drop_remainder()` and
     `tf.contrib.data.sloppy_interleave()`.
+* Add `train_and_evaluate` for simple distributed `Estimator` training.
+* Add `tf.spectral.dct` for computing the DCT-II.
+* Add Mel-Frequency Cepstral Coefficient support to `tf.contrib.signal`
+  (with GPU and gradient support).
+* Add a self-check on `import tensorflow` for Windows DLL issues.
+* Add NCHW support to `tf.depth_to_space` on GPU.
+* SinhArcsinh (scalar) distribution added to `contrib.distributions`.
+* Make `GANEstimator` opensource.
+* `Estimator.export_savedmodel()` now includes all valid serving signatures
+  that can be constructed from the Serving Input Receiver and all available
+  ExportOutputs. For instance, a classifier may provide regression- and
+  prediction-flavored outputs, in addition to the classification-flavored one.
+  Building signatures from these allows TF Serving to honor requests using the
+  different APIs (Classify, Regress, and Predict). Furthermore,
+  `serving_input_receiver_fn()` may now specify alternative subsets of nodes
+  that may act as inputs. This allows, for instance, producing a prediction
+  signature for a classifier that accepts raw `Tensors` instead of a serialized
+  `tf.Example`.
+* Add `tf.contrib.bayesflow.hmc`.
+* Add `tf.contrib.distributions.MixtureSameFamily`.
+* Make `Dataset.shuffle()` always reshuffles after each iteration by default.
+* Add `tf.contrib.bayesflow.metropolis_hastings`.
+* Add `log_rate` parameter to `tf.contrib.distributions.Poisson`.
+* Extend `tf.contrib.distributions.bijector` API to handle some non-injective
+  transforms.
 * Java:
-  * Generics (e.g., `Tensor<Integer>`) for improved type-safety (courtesy @andrewcmyers).
+  * Generics (e.g., `Tensor<Integer>`) for improved type-safety
+    (courtesy @andrewcmyers).
   * Support for multi-dimensional string tensors.
+  * Support loading of custom operations (e.g. many in `tf.contrib`) on Linux
+    and OS X
+* All our prebuilt binaries have been built with CUDA 8 and cuDNN 6.
+  We anticipate releasing TensorFlow 1.5 with CUDA 9 and cuDNN 7.
 
 ## Bug Fixes and Other Changes
 * `tf.nn.rnn_cell.DropoutWrapper` is now more careful about dropping out LSTM
@@ -26,11 +57,57 @@
 * Removed `tf.contrib.training.python_input`.  The same behavior, in a more
   flexible and reproducible package, is available via the new
   `tf.contrib.data.Dataset.from_generator` method!
+* Fix `tf.contrib.distributions.Affine` incorrectly computing log-det-jacobian.
+* Fix `tf.random_gamma` incorrectly handling non-batch, scalar draws.
+* Resolved a race condition in TensorForest TreePredictionsV4Op.
+* Google Cloud Storage file system and Hadoop file system support are now
+  default build options.
+* Custom op libraries must link against libtensorflow_framework.so
+  (installed at `tf.sysconfig.get_lib()`).
 
 ## Breaking Changes to the API
 * The signature of the `tf.contrib.data.rejection_resample()` function has been
   changed. It now returns a function that can be used as an argument to
   `Dataset.apply()`.
+* Remove `tf.contrib.data.Iterator.from_dataset()` method. Use
+  `Dataset.make_initializable_iterator()` instead.
+* Remove seldom used and unnecessary `tf.contrib.data.Iterator.dispose_op()`.
+* Reorder some TFGAN loss functions in a non-backwards compatible way.
+
+## Thanks to our Contributors
+
+This release contains contributions from many people at Google, as well as:
+
+4d55397500, Abdullah Alrasheed, abenmao, Adam Salvail, Aditya Dhulipala, Ag Ramesh,
+Akimasa Kimura, Alan Du, Alan Yee, Alexander, Amit Kushwaha, Amy, Andrei Costinescu,
+Andrei Nigmatulin, Andrew Erlichson, Andrew Myers, Andrew Stepanov, Androbin, AngryPowman,
+Anish Shah, Anton Daitche, Artsiom Chapialiou, asdf2014, Aseem Raj Baranwal, Ash Hall,
+Bart Kiers, Batchu Venkat Vishal, ben, Ben Barsdell, Bill Piel, Carl Thomé, Catalin Voss,
+Changming Sun, Chengzhi Chen, Chi Zeng, Chris Antaki, Chris Donahue, Chris Oelmueller,
+Chris Tava, Clayne Robison, Codrut, Courtial Florian, Dalmo Cirne, Dan J, Darren Garvey,
+David Kristoffersson, David Norman, David RöThlisberger, DavidNorman, Dhruv, DimanNe,
+Dorokhov, Duncan Mac-Vicar P, EdwardDixon, EMCP, error.d, FAIJUL, Fan Xia,
+Francois Xavier, Fred Reiss, Freedom" Koan-Sin Tan, Fritz Obermeyer, Gao, Xiang,
+Guenther Schmuelling, Guo Yejun (郭叶军), Hans Gaiser, HectorSVC, Hyungsuk Yoon,
+James Pruegsanusak, Jay Young, Jean Wanka, Jeff Carpenter, Jeremy Rutman, Jeroen BéDorf,
+Jett Jones, Jimmy Jia, jinghuangintel, jinze1994, JKurland, Joel Hestness, joetoth,
+John B Nelson, John Impallomeni, John Lawson, Jonas, Jonathan Dekhtiar, joshkyh, Jun Luan,
+Jun Mei, Kai Sasaki, Karl Lessard, karl@kubx.ca, Kb Sriram, Kenichi Ueno, Kevin Slagle,
+Kongsea, Lakshay Garg, lhlmgr, Lin Min, liu.guangcong, Loki Der Quaeler, Louie Helm,
+lucasmoura, Luke Iwanski, Lyndon White, Mahmoud Abuzaina, Marcel Puyat, Mark Aaron Shirley,
+Michele Colombo, MtDersvan, Namrata-Ibm, Nathan Luehr, Naurril, Nayana Thorat, Nicolas Lopez,
+Niranjan Hasabnis, Nolan Liu, Nouce, Oliver Hennigh, osdamv, Patrik Erdes,
+Patryk Chrabaszcz, Pavel Christof, Penghao Cen, postBG, Qingqing Cao, Qingying Chen, qjivy,
+Raphael, Rasmi, raymondxyang, Renze Yu, resec, Roffel, Ruben Vereecken, Ryohei Kuroki,
+sandipmgiri, Santiago Castro, Scott Kirkland, Sean Vig, Sebastian Raschka, Sebastian Weiss,
+Sergey Kolesnikov, Sergii Khomenko, Shahid, Shivam Kotwalia, Stuart Berg, Sumit Gouthaman,
+superzerg, Sven Mayer, tetris, Ti Zhou, Tiago Freitas Pereira, Tian Jin, Tomoaki Oiki,
+Vaibhav Sood, vfdev, Vivek Rane, Vladimir Moskva, wangqr, Weber Xie, Will Frey,
+Yan Facai (颜发才), yanivbl6, Yaroslav Bulatov, Yixing Lao, Yong Tang, youkaichao,
+Yuan (Terry) Tang, Yue Zhang, Yuxin Wu, Ziming Dong, ZxYuan, 黄璞
+
+We are also grateful to all who filed issues or helped resolve them, asked and
+answered questions, and were part of inspiring discussions.
 
 # Release 1.3.0
 
diff --git a/configure.py b/configure.py
index ea3f598f3d..425eae676c 100644
--- a/configure.py
+++ b/configure.py
@@ -989,6 +989,7 @@ def main():
   run_gen_git_source(environ_cp)
 
   if is_windows():
+    environ_cp['TF_NEED_S3'] = '0'
     environ_cp['TF_NEED_GCP'] = '0'
     environ_cp['TF_NEED_HDFS'] = '0'
     environ_cp['TF_NEED_JEMALLOC'] = '0'
@@ -1001,9 +1002,9 @@ def main():
   set_build_var(environ_cp, 'TF_NEED_JEMALLOC', 'jemalloc as malloc',
                 'with_jemalloc', True)
   set_build_var(environ_cp, 'TF_NEED_GCP', 'Google Cloud Platform',
-                'with_gcp_support', False, 'gcp')
+                'with_gcp_support', True, 'gcp')
   set_build_var(environ_cp, 'TF_NEED_HDFS', 'Hadoop File System',
-                'with_hdfs_support', False, 'hdfs')
+                'with_hdfs_support', True, 'hdfs')
   set_build_var(environ_cp, 'TF_NEED_S3', 'Amazon S3 File System',
                 'with_s3_support', True, 's3')
   set_build_var(environ_cp, 'TF_ENABLE_XLA', 'XLA JIT', 'with_xla_support',
diff --git a/tensorflow/BUILD b/tensorflow/BUILD
index 673e433a8a..20f02ad50a 100644
--- a/tensorflow/BUILD
+++ b/tensorflow/BUILD
@@ -323,6 +323,7 @@ filegroup(
         "//tensorflow/compiler/jit/kernels:all_files",
         "//tensorflow/compiler/jit/legacy_flags:all_files",
         "//tensorflow/compiler/jit/ops:all_files",
+        "//tensorflow/compiler/plugin:all_files",
         "//tensorflow/compiler/tests:all_files",
         "//tensorflow/compiler/tf2xla:all_files",
         "//tensorflow/compiler/tf2xla/cc:all_files",
diff --git a/tensorflow/c/c_api.h b/tensorflow/c/c_api.h
index 0c6bb53d01..1e8bfdc7b0 100644
--- a/tensorflow/c/c_api.h
+++ b/tensorflow/c/c_api.h
@@ -1153,7 +1153,7 @@ TF_CAPI_EXPORT extern TF_Function* TF_FunctionImportFunctionDef(
     const void* proto, size_t proto_len, TF_Status* status);
 
 // Sets function attribute named `attr_name` to value stored in `proto`.
-// If this attribute is already set to another value, it is overriden.
+// If this attribute is already set to another value, it is overridden.
 // `proto` should point to a sequence of bytes of length `proto_len`
 // representing a binary serialization of an AttrValue protocol
 // buffer.
diff --git a/tensorflow/cc/gradients/math_grad.cc b/tensorflow/cc/gradients/math_grad.cc
index 2417bf18a9..d7446b9560 100644
--- a/tensorflow/cc/gradients/math_grad.cc
+++ b/tensorflow/cc/gradients/math_grad.cc
@@ -13,6 +13,9 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
+#define _USE_MATH_DEFINES
+#include <cmath>
+
 #include "tensorflow/cc/ops/array_ops_internal.h"
 #include "tensorflow/cc/ops/math_ops_internal.h"
 #include "tensorflow/cc/ops/standard_ops.h"
@@ -200,8 +203,8 @@ Status TanhGrad(const Scope& scope, const Operation& op,
   // evaluated.
   Scope grad_scope = scope.WithControlDependencies(grad);
   auto y = ConjugateHelper(grad_scope, op.output(0));
-  grad_outputs->push_back(internal::TanhGrad(scope, y, grad));
-  return scope.status();
+  grad_outputs->push_back(internal::TanhGrad(grad_scope, y, grad));
+  return grad_scope.status();
 }
 REGISTER_GRADIENT_OP("Tanh", TanhGrad);
 
@@ -256,8 +259,8 @@ Status SigmoidGrad(const Scope& scope, const Operation& op,
   // evaluated.
   Scope grad_scope = scope.WithControlDependencies(grad);
   auto y = ConjugateHelper(grad_scope, op.output(0));
-  grad_outputs->push_back(internal::SigmoidGrad(scope, y, grad));
-  return scope.status();
+  grad_outputs->push_back(internal::SigmoidGrad(grad_scope, y, grad));
+  return grad_scope.status();
 }
 REGISTER_GRADIENT_OP("Sigmoid", SigmoidGrad);
 
@@ -696,15 +699,32 @@ Status MeanGrad(const Scope& scope, const Operation& op,
 }
 REGISTER_GRADIENT_OP("Mean", MeanGrad);
 
+Status ErfGrad(const Scope& scope, const Operation& op,
+               const std::vector<Output>& grad_inputs,
+               std::vector<Output>* grad_outputs) {
+  auto grad = grad_inputs[0];
+  auto two_over_root_pi = Cast(scope, Const(scope, 2 / std::sqrt(M_PI)),
+                               grad.type());
+  Scope grad_scope = scope.WithControlDependencies(grad);
+  auto x = ConjugateHelper(grad_scope, op.input(0));
+  // grad * 2/sqrt(pi) * exp(-x**2)
+  auto dx = Mul(grad_scope,
+                Mul(grad_scope, grad, two_over_root_pi),
+                Exp(grad_scope, Neg(grad_scope, Square(grad_scope, x))));
+  grad_outputs->push_back(dx);
+  return grad_scope.status();
+}
+REGISTER_GRADIENT_OP("Erf", ErfGrad);
+
 Status LgammaGrad(const Scope& scope, const Operation& op,
                   const std::vector<Output>& grad_inputs,
                   std::vector<Output>* grad_outputs) {
   auto grad = grad_inputs[0];
   Scope grad_scope = scope.WithControlDependencies(grad);
   auto x = ConjugateHelper(grad_scope, op.input(0));
-  auto dx = Mul(scope, grad, Digamma(scope, x));
+  auto dx = Mul(grad_scope, grad, Digamma(grad_scope, x));
   grad_outputs->push_back(dx);
-  return scope.status();
+  return grad_scope.status();
 }
 REGISTER_GRADIENT_OP("Lgamma", LgammaGrad);
 
diff --git a/tensorflow/cc/gradients/math_grad_test.cc b/tensorflow/cc/gradients/math_grad_test.cc
index a174f223ad..6313f41da5 100644
--- a/tensorflow/cc/gradients/math_grad_test.cc
+++ b/tensorflow/cc/gradients/math_grad_test.cc
@@ -64,7 +64,9 @@ class CWiseUnaryGradTest : public ::testing::Test {
     IMAG,
     CONJ,
     COMPLEX,
-    ANGLE
+    ANGLE,
+    LGAMMA,
+    ERF
   };
 
   template <typename X_T, typename Y_T>
@@ -168,6 +170,12 @@ class CWiseUnaryGradTest : public ::testing::Test {
       case ANGLE:
         y = Angle(scope_, x);
         break;
+      case LGAMMA:
+        y = Lgamma(scope_, x);
+        break;
+      case ERF:
+        y = Erf(scope_, x);
+        break;
     }
 
     float max_error;
@@ -503,6 +511,42 @@ TEST_F(CWiseUnaryGradTest, Angle) {
   TestCWiseGrad<complex64, float>(ANGLE, x_fn);
 }
 
+TEST_F(CWiseUnaryGradTest, Lgamma) {
+  auto x_fn = [this](const int i) {
+    return RV({-3.5, -2.5, -1.5, 1.0, 2.0, 3.5});
+  };
+  TestCWiseGrad<float, float>(LGAMMA, x_fn);
+}
+
+TEST_F(CWiseUnaryGradTest, Lgamma_Complex) {
+  auto x_fn = [this](const int i) {
+    return CRV({{-3.5, 0.5}, {-1.5, -0.5}, {1.5, -1.0}, {3.5, 1.0}});
+  };
+  // TODO(kbsriram)
+  // Add test when the lgamma kernel supports complex numbers
+  if (false) {
+    TestCWiseGrad<complex64, complex64>(LGAMMA, x_fn);
+  }
+}
+
+TEST_F(CWiseUnaryGradTest, Erf) {
+  auto x_fn = [this](const int i) {
+    return RV({-1.2, -1.0, -0.5, 0.3, 0.5, 1.3});
+  };
+  TestCWiseGrad<float, float>(ERF, x_fn);
+}
+
+TEST_F(CWiseUnaryGradTest, Erf_Complex) {
+  auto x_fn = [this](const int i) {
+    return CRV({{-1.2, 0.5}, {-0.5, -0.5}, {0.5, 0.5}, {1.2, -0.5}});
+  };
+  // TODO(kbsriram)
+  // Add test when the erf kernel supports complex numbers
+  if (false) {
+    TestCWiseGrad<complex64, complex64>(ERF, x_fn);
+  }
+}
+
 class MathGradTest : public ::testing::Test {
  protected:
   MathGradTest() : root_(Scope::NewRootScope().WithDevice("/cpu:0")) {}
@@ -821,17 +865,5 @@ TEST_F(NaryGradTest, Minimum) {
   RunTest(x, x_init_value, y, shape);
 }
 
-TEST_F(NaryGradTest, Lgamma) {
-  TensorShape shape({3, 2});
-  auto x = Placeholder(scope_, DT_FLOAT, Placeholder::Shape(shape));
-  auto y = Lgamma(scope_, x);
-  // Select values to avoid instability when computing finite differences.
-  // Ref: https://en.wikipedia.org/wiki/File:Gamma_plot.svg
-  Tensor x_init_value =
-      test::AsTensor<float>({-3.5f, -2.5f, -1.5f, 1.0f, 2.0f, 3.5f}, {3, 2});
-  RunTest(x, x_init_value, y, shape);
-  // TODO(suharshs): add test case for complex values
-}
-
 }  // namespace
 }  // namespace tensorflow
diff --git a/tensorflow/compiler/jit/BUILD b/tensorflow/compiler/jit/BUILD
index bf63b7e501..bf7d9cf14d 100644
--- a/tensorflow/compiler/jit/BUILD
+++ b/tensorflow/compiler/jit/BUILD
@@ -33,6 +33,7 @@ cc_library(
     deps = [
         ":xla_cpu_device",
         ":xla_cpu_jit",
+        "//tensorflow/compiler/plugin",
     ] + if_cuda_is_configured([
         ":xla_gpu_device",
         ":xla_gpu_jit",
diff --git a/tensorflow/compiler/plugin/BUILD b/tensorflow/compiler/plugin/BUILD
new file mode 100644
index 0000000000..c1edf2448c
--- /dev/null
+++ b/tensorflow/compiler/plugin/BUILD
@@ -0,0 +1,56 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""Configuration file for an XLA plugin.
+
+  please don't check in changes to this file. to prevent changes appearing
+  in git status, use:
+
+  git update-index --assume-unchanged tensorflow/compiler/plugin/BUILD
+
+  To add additional devices to the XLA subsystem, add targets to the
+  dependency list in the 'plugin' target. For instance:
+
+    deps = ["//tensorflow/compiler/plugin/example:plugin_lib"],
+
+  ** Please don't remove this file - it is supporting some 3rd party plugins **
+"""
+
+licenses(["notice"])
+
+package(
+    default_visibility = ["//visibility:public"],
+)
+
+cc_library(
+    name = "plugin",
+    deps = [
+        #"//tensorflow/compiler/plugin/example:example_lib",
+    ],
+)
+
+#-----------------------------------------------------------------------------
+
+filegroup(
+    name = "all_files",
+    srcs = glob(
+        ["**/*"],
+        exclude = [
+            "**/METADATA",
+            "**/OWNERS",
+        ],
+    ),
+    visibility = ["//tensorflow:__subpackages__"],
+)
diff --git a/tensorflow/compiler/plugin/README.md b/tensorflow/compiler/plugin/README.md
new file mode 100644
index 0000000000..9dd0d2bdab
--- /dev/null
+++ b/tensorflow/compiler/plugin/README.md
@@ -0,0 +1,16 @@
+3rd party XLA devices
+---------------------
+
+This directory is intended as a place for 3rd party XLA devices which are _not_
+integrated into the public repository.
+
+By adding entries to the BUILD target in this directory, a third party device
+can be included as a dependency of the JIT subsystem.
+
+For integration into the unit test system, see the files:
+
+- tensorflow/compiler/tests/plugin.bzl
+- tensorflow/compiler/xla/tests/plugin.bzl
+
+
+- 
diff --git a/tensorflow/compiler/xla/service/hlo_computation_test.cc b/tensorflow/compiler/xla/service/hlo_computation_test.cc
index ccab7bf348..7b7588f4ba 100644
--- a/tensorflow/compiler/xla/service/hlo_computation_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_computation_test.cc
@@ -310,7 +310,7 @@ TEST_F(HloComputationTest, DeepCopyArrayAtIndices) {
 }
 
 TEST_F(HloComputationTest, DeepCopyTupleAtIndices) {
-  // Test that DeepCopyInstruction properly copies elements of a a tuple as
+  // Test that DeepCopyInstruction properly copies elements of a tuple as
   // specified by the given indices.
   auto builder = HloComputation::Builder(TestName());
   auto constant1 = builder.AddInstruction(HloInstruction::CreateConstant(
diff --git a/tensorflow/compiler/xla/service/inliner.cc b/tensorflow/compiler/xla/service/inliner.cc
index 0682434bfb..6ea0f127d5 100644
--- a/tensorflow/compiler/xla/service/inliner.cc
+++ b/tensorflow/compiler/xla/service/inliner.cc
@@ -90,8 +90,12 @@ Status InlinerVisitor::HandleMap(
     // different than the map shape. Hence, a broadcast is needed, else the
     // cloned operand with new shape and operands work.
     if (root.opcode() != HloOpcode::kConstant) {
+      std::vector<HloInstruction*> params;
+      for (int64 o = 0; o < root.operands().size(); o++) {
+        params.push_back(operands[root.operand(o)->parameter_number()]);
+      }
       HloInstruction* placed_instruction = computation_->AddInstruction(
-          root.CloneWithNewOperands(map->shape(), operands));
+          root.CloneWithNewOperands(map->shape(), params));
       TF_RETURN_IF_ERROR(
           computation_->ReplaceInstruction(map, placed_instruction));
     } else {
diff --git a/tensorflow/compiler/xla/service/inliner_test.cc b/tensorflow/compiler/xla/service/inliner_test.cc
index 9d845c5545..7aa1c7c835 100644
--- a/tensorflow/compiler/xla/service/inliner_test.cc
+++ b/tensorflow/compiler/xla/service/inliner_test.cc
@@ -108,5 +108,44 @@ TEST_F(InlinerTest, MapConstant) {
   LiteralTestUtil::ExpectEqual(*result, *expected);
 }
 
+TEST_F(InlinerTest, MapSubtractOppositeOrder) {
+  Shape r0f32 = ShapeUtil::MakeShape(F32, {});
+
+  // Note that the parameter ordinals are in the opposite order to their
+  // position as operands
+  auto max_builder = HloComputation::Builder(TestName());
+  auto param1 = max_builder.AddInstruction(
+          HloInstruction::CreateParameter(1, r0f32, "x"));
+  auto param2 = max_builder.AddInstruction(
+          HloInstruction::CreateParameter(0, r0f32, "y"));
+  max_builder.AddInstruction(HloInstruction::CreateBinary(
+          param1->shape(), HloOpcode::kSubtract, param1, param2));
+  auto max_f32 = max_builder.Build();
+
+  auto builder = HloComputation::Builder("MapSubFunction");
+  auto lhs = builder.AddInstruction(
+    HloInstruction::CreateConstant(Literal::CreateR1<float>({1, 2, 3, 4})));
+  auto rhs = builder.AddInstruction(
+    HloInstruction::CreateConstant(Literal::CreateR1<float>({4, 3, 2, 1})));
+  builder.AddInstruction(
+    HloInstruction::CreateMap(lhs->shape(), {lhs, rhs}, max_f32.get()));
+
+  auto computation = builder.Build();
+  auto hlo_module = CreateNewModule();
+  hlo_module->AddEmbeddedComputation(std::move(max_f32));
+  hlo_module->AddEntryComputation(std::move(computation));
+
+  Inliner inliner;
+  EXPECT_TRUE(inliner.Run(hlo_module.get()).ValueOrDie());
+  EXPECT_THAT(hlo_module->entry_computation()->root_instruction(),
+          op::Subtract(rhs, lhs));
+
+  // Verify execution on CPU.
+  auto result = ExecuteAndTransfer(std::move(hlo_module), {});
+  auto expected = Literal::CreateR1<float>({3, 1, -1, -3});
+  LiteralTestUtil::ExpectEqual(*result, *expected);
+}
+
+
 }  // namespace
 }  // namespace xla
diff --git a/tensorflow/contrib/all_reduce/python/all_reduce.py b/tensorflow/contrib/all_reduce/python/all_reduce.py
index 22d7633ce2..a5057da9fd 100644
--- a/tensorflow/contrib/all_reduce/python/all_reduce.py
+++ b/tensorflow/contrib/all_reduce/python/all_reduce.py
@@ -191,7 +191,7 @@ def _ragged_split(tensor, pieces):
 
 
 def _ring_permutations(num_workers, num_subchunks, gpu_perm):
-  """"Generate an array of device index arrays, one for for each subchunk.
+  """"Generate an array of device index arrays, one for each subchunk.
 
   In the basic ring reduction algorithm there are size(T)/num_devices
   data chunks and each device process one chunk per tick, i.e. sending
diff --git a/tensorflow/contrib/boosted_trees/README.md b/tensorflow/contrib/boosted_trees/README.md
index 9ce700f1a1..7d30032e53 100644
--- a/tensorflow/contrib/boosted_trees/README.md
+++ b/tensorflow/contrib/boosted_trees/README.md
@@ -1,7 +1,7 @@
 # TF Boosted Trees (TFBT)
 
 TF Boosted trees is an implementation of a gradient boosting algorithm with
-trees used as week learners.
+trees used as weak learners.
 
 ## Examples
 Folder "examples" demonstrates how TFBT estimators can be used for various
diff --git a/tensorflow/contrib/boosted_trees/examples/binary_mnist.py b/tensorflow/contrib/boosted_trees/examples/binary_mnist.py
index c003b1de66..47ee3d816f 100644
--- a/tensorflow/contrib/boosted_trees/examples/binary_mnist.py
+++ b/tensorflow/contrib/boosted_trees/examples/binary_mnist.py
@@ -21,7 +21,7 @@ r"""Demonstrates multiclass MNIST TF Boosted trees example.
   python tensorflow/contrib/boosted_trees/examples/binary_mnist.py \
   --output_dir="/tmp/binary_mnist" --depth=4 --learning_rate=0.3 \
   --batch_size=10761 --examples_per_layer=10761 --eval_batch_size=1030 \
-  --num_eval_steps=1 --num_trees=10 --l2=1 --vmodule=training_ops=1 \
+  --num_eval_steps=1 --num_trees=10 --l2=1 --vmodule=training_ops=1
 
   When training is done, accuracy on eval data is reported. Point tensorboard
   to the directory for the run to see how the training progresses:
diff --git a/tensorflow/contrib/boosted_trees/examples/mnist.py b/tensorflow/contrib/boosted_trees/examples/mnist.py
index 0539d77720..817c6eb3e1 100644
--- a/tensorflow/contrib/boosted_trees/examples/mnist.py
+++ b/tensorflow/contrib/boosted_trees/examples/mnist.py
@@ -22,7 +22,7 @@ r"""Demonstrates multiclass MNIST TF Boosted trees example.
   python tensorflow/contrib/boosted_trees/examples/mnist.py \
   --output_dir="/tmp/mnist" --depth=4 --learning_rate=0.3 --batch_size=60000  \
   --examples_per_layer=60000 --eval_batch_size=10000 --num_eval_steps=1 \
-  --num_trees=10 --l2=1 --vmodule=training_ops=1 \
+  --num_trees=10 --l2=1 --vmodule=training_ops=1
 
   When training is done, accuracy on eval data is reported. Point tensorboard
   to the directory for the run to see how the training progresses:
diff --git a/tensorflow/contrib/cmake/external/cub.cmake b/tensorflow/contrib/cmake/external/cub.cmake
index e03026b1b0..8368898955 100644
--- a/tensorflow/contrib/cmake/external/cub.cmake
+++ b/tensorflow/contrib/cmake/external/cub.cmake
@@ -14,8 +14,8 @@
 # ==============================================================================
 include (ExternalProject)
 
-set(cub_URL https://mirror.bazel.build/github.com/NVlabs/cub/archive/1.7.3.zip)
-set(cub_HASH SHA256=b7ead9e291d34ffa8074243541c1380d63be63f88de23de8ee548db573b72ebe)
+set(cub_URL https://mirror.bazel.build/github.com/NVlabs/cub/archive/1.7.4.zip)
+set(cub_HASH SHA256=20a1a39fd97e5da7f40f5f2e7fd73fd2ea59f9dc4bb8a6c5f228aa543e727e31)
 set(cub_BUILD ${CMAKE_CURRENT_BINARY_DIR}/cub/src/cub)
 set(cub_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/cub/src/cub)
 set(cub_ARCHIVE_DIR ${CMAKE_CURRENT_BINARY_DIR}/external/cub_archive)
diff --git a/tensorflow/contrib/cmake/external/protobuf.cmake b/tensorflow/contrib/cmake/external/protobuf.cmake
index d600d8c3c0..1e300e21df 100644
--- a/tensorflow/contrib/cmake/external/protobuf.cmake
+++ b/tensorflow/contrib/cmake/external/protobuf.cmake
@@ -15,8 +15,8 @@
 include (ExternalProject)
 
 set(PROTOBUF_INCLUDE_DIRS ${CMAKE_CURRENT_BINARY_DIR}/protobuf/src/protobuf/src)
-set(PROTOBUF_URL https://github.com/mrry/protobuf.git)  # Includes MSVC fix.
-set(PROTOBUF_TAG 1d2c7b6c7376f396c8c7dd9b6afd2d4f83f3cb05)
+set(PROTOBUF_URL https://github.com/google/protobuf.git)
+set(PROTOBUF_TAG b04e5cba356212e4e8c66c61bbe0c3a20537c5b9)
 
 if(WIN32)
   set(protobuf_STATIC_LIBRARIES 
diff --git a/tensorflow/contrib/cmake/tf_core_kernels.cmake b/tensorflow/contrib/cmake/tf_core_kernels.cmake
index 46c680aad5..65565aad7e 100644
--- a/tensorflow/contrib/cmake/tf_core_kernels.cmake
+++ b/tensorflow/contrib/cmake/tf_core_kernels.cmake
@@ -33,6 +33,8 @@ else(tensorflow_BUILD_ALL_KERNELS)
      "${tensorflow_source_dir}/tensorflow/core/kernels/matmul_op.cc"
      "${tensorflow_source_dir}/tensorflow/core/kernels/no_op.h"
      "${tensorflow_source_dir}/tensorflow/core/kernels/no_op.cc"
+     "${tensorflow_source_dir}/tensorflow/core/kernels/ops_util.h"
+     "${tensorflow_source_dir}/tensorflow/core/kernels/ops_util.cc"
      "${tensorflow_source_dir}/tensorflow/core/kernels/sendrecv_ops.h"
      "${tensorflow_source_dir}/tensorflow/core/kernels/sendrecv_ops.cc"
   )
@@ -65,6 +67,8 @@ if(tensorflow_BUILD_CONTRIB_KERNELS)
       "${tensorflow_source_dir}/tensorflow/contrib/boosted_trees/ops/split_handler_ops.cc"
       "${tensorflow_source_dir}/tensorflow/contrib/boosted_trees/ops/stats_accumulator_ops.cc"
       "${tensorflow_source_dir}/tensorflow/contrib/boosted_trees/ops/training_ops.cc"
+      "${tensorflow_source_dir}/tensorflow/contrib/cudnn_rnn/kernels/cudnn_rnn_ops.cc"
+      "${tensorflow_source_dir}/tensorflow/contrib/cudnn_rnn/ops/cudnn_rnn_ops.cc"
       "${tensorflow_source_dir}/tensorflow/contrib/factorization/kernels/clustering_ops.cc"
       "${tensorflow_source_dir}/tensorflow/contrib/factorization/kernels/masked_matmul_ops.cc"
       "${tensorflow_source_dir}/tensorflow/contrib/factorization/kernels/wals_solver_ops.cc"
diff --git a/tensorflow/contrib/cmake/tf_tests.cmake b/tensorflow/contrib/cmake/tf_tests.cmake
index 1d58b1d416..ac55b9ea92 100644
--- a/tensorflow/contrib/cmake/tf_tests.cmake
+++ b/tensorflow/contrib/cmake/tf_tests.cmake
@@ -179,6 +179,9 @@ if (tensorflow_BUILD_PYTHON_TESTS)
 
   # exclude the ones we don't want
   set(tf_test_src_py_exclude
+    # generally excluded
+    "${tensorflow_source_dir}/tensorflow/python/kernel_tests/__init__.py"
+
     # Python source line inspection tests are flaky on Windows (b/36375074).
     "${tensorflow_source_dir}/tensorflow/python/debug/cli/analyzer_cli_test.py"
     "${tensorflow_source_dir}/tensorflow/python/debug/cli/profile_analyzer_cli_test.py"
@@ -188,19 +191,16 @@ if (tensorflow_BUILD_PYTHON_TESTS)
     "${tensorflow_source_dir}/tensorflow/python/debug/lib/dist_session_debug_grpc_test.py"
     "${tensorflow_source_dir}/tensorflow/python/debug/lib/session_debug_grpc_test.py"
     # generally not working
-    "${tensorflow_source_dir}/tensorflow/python/kernel_tests/__init__.py"
-    "${tensorflow_source_dir}/tensorflow/python/kernel_tests/benchmark_test.py"
-    "${tensorflow_source_dir}/tensorflow/python/kernel_tests/resource_variable_ops_test.py"
     "${tensorflow_source_dir}/tensorflow/python/profiler/pprof_profiler_test.py"
     # flaky test
     "${tensorflow_source_dir}/tensorflow/python/profiler/internal/run_metadata_test.py"
+    # Fails because uses data dependencies with bazel
     "${tensorflow_source_dir}/tensorflow/python/saved_model/saved_model_test.py"
     # requires scipy
     "${tensorflow_source_dir}/tensorflow/contrib/keras/python/keras/preprocessing/*_test.py"
     "${tensorflow_source_dir}/tensorflow/contrib/tfprof/python/tools/tfprof/pprof_profiler_test.py"
-    # flaky tests
+    # Takes very long to run without sharding (defined in bazel build file).
     "${tensorflow_source_dir}/tensorflow/python/kernel_tests/cwise_ops_test.py"
-    "${tensorflow_source_dir}/tensorflow/contrib/tfprof/python/tools/tfprof/internal/run_metadata_test.py"
     # Loading resources in contrib doesn't seem to work on Windows
     "${tensorflow_source_dir}/tensorflow/contrib/tensor_forest/client/random_forest_test.py"
     "${tensorflow_source_dir}/tensorflow/contrib/tensor_forest/python/tensor_forest_test.py"
@@ -213,47 +213,57 @@ if (tensorflow_BUILD_PYTHON_TESTS)
   if (WIN32)
     set(tf_test_src_py_exclude
       ${tf_test_src_py_exclude}
-      # generally excluded
-      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/__init__.py"
-
       # TODO: failing tests.
       # Nothing critical in here but should get this list down to []
       # The failing list is grouped by failure source
+
       # stl on windows handles overflows different
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/as_string_op_test.py"
-      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/cast_op_test.py"
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/string_to_number_op_test.py"
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/clip_ops_test.py"
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/tensor_array_ops_test.py"  # Needs portpicker.
-      # Matrix_set_diag failing on GPU on windows.
-      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/cholesky_op_test.py"
-      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/diag_op_test.py"
-      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/linalg_ops_test.py"
-      "${tensorflow_source_dir}/tensorflow/python/ops/init_ops.py"
+      # Numerical issues, calculations off.
+      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/concat_op_test.py"
+      "${tensorflow_source_dir}/tensorflow/contrib/factorization/python/ops/wals_test.py"
+      # Float division by zero
+      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/benchmark_test.py"
+      # Flaky, for unknown reasons. Cannot reproduce in terminal. Revisit once we can get stack traces.
+      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/batch_matmul_op_test.py"
+      # Flaky because of local cluster creation.
+      "${tensorflow_source_dir}/tensorflow/python/training/sync_replicas_optimizer_test.py"
+      "${tensorflow_source_dir}/tensorflow/python/debug/lib/session_debug_grpc_test.py"
+      "${tensorflow_source_dir}tensorflow/python/training/localhost_cluster_performance_test.py"
+      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/iterator_ops_cluster_test.py"
+      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/functional_ops_test.py"
+      "${tensorflow_source_dir}/tensorflow/contrib/data/python/kernel_tests/iterator_ops_cluster_test.py"
+      # Type error in testRemoteIteratorUsingRemoteCallOpDirectSessionGPUCPU.
+      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/iterator_ops_test.py"
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/self_adjoint_eig_op_test.py"
-      # misc
+      "${tensorflow_source_dir}/tensorflow/contrib/data/python/kernel_tests/iterator_ops_test.py"
+      # IteratorGetMax OutOfRangeError
       "${tensorflow_source_dir}/tensorflow/contrib/data/python/kernel_tests/batch_dataset_op_test.py"
-      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/variable_scope_test.py"
-      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/reshape_op_test.py"
-      "${tensorflow_source_dir}/tensorflow/python/training/evaluation_test.py"
-      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/neon_depthwise_conv_op_test.py"  # Depends on gemmlowp -> pthread.
+      # Depends on gemmlowp -> pthread
+      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/neon_depthwise_conv_op_test.py"
       # int32/int64 mixup
+      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/cast_op_test.py"
+      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/variable_scope_test.py"
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/functional_ops_test.py"
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/py_func_test.py"
+      # Windows file management related issues.
+      "${tensorflow_source_dir}/tensorflow/python/training/evaluation_test.py"
       # training tests
       "${tensorflow_source_dir}/tensorflow/python/training/basic_session_run_hooks_test.py"  # Needs tf.contrib fix.
       "${tensorflow_source_dir}/tensorflow/python/training/localhost_cluster_performance_test.py"  # Needs portpicker.
       "${tensorflow_source_dir}/tensorflow/python/training/quantize_training_test.py"  # Needs quantization ops to be included in windows.
       "${tensorflow_source_dir}/tensorflow/python/training/supervisor_test.py"  # Flaky I/O error on rename.
-      "${tensorflow_source_dir}/tensorflow/python/training/sync_replicas_optimizer_test.py"  # Needs portpicker.
       "${tensorflow_source_dir}/tensorflow/python/training/server_lib_test.py"  # Test occasionally deadlocks.
-      "${tensorflow_source_dir}/tensorflow/python/debug/lib/session_debug_multi_gpu_test.py"
+      "${tensorflow_source_dir}/tensorflow/python/debug/lib/session_debug_multi_gpu_test.py"  # Fails on multiple GPUs.
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/concat_op_test.py"  # numerical issues
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/linalg_grad_test.py"  # cudaSolver handle creation fails.
-
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/array_ops_test.py"  # depends on python/framework/test_ops
       # Dataset tests
-      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/dataset_constructor_op_test.py"
+      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/dataset_constructor_op_test.py"  # Segfaults on windows
+      "${tensorflow_source_dir}/tensorflow/contrib/data/python/kernel_tests/dataset_constructor_op_test.py"  # Segfaults on Windows.
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/iterator_ops_cluster_test.py"
       # Broken tensorboard test due to cmake issues.
       "${tensorflow_source_dir}/tensorflow/contrib/data/python/kernel_tests/iterator_ops_cluster_test.py"  # Needs portpicker
@@ -264,8 +274,6 @@ if (tensorflow_BUILD_PYTHON_TESTS)
       "${tensorflow_source_dir}/tensorflow/contrib/tensor_forest/python/kernel_tests/scatter_add_ndim_op_test.py"  # Bad placement.
       "${tensorflow_source_dir}/tensorflow/contrib/tensor_forest/python/topn_test.py"  # Results inaccurate
       "${tensorflow_source_dir}/tensorflow/python/ops/cloud/bigquery_reader_ops_test.py"  # No libcurl support
-      # Newly running on Windows since TensorBoard backend move. Fail on Windows and need debug.
-      "${tensorflow_source_dir}/tensorflow/contrib/data/python/kernel_tests/dataset_constructor_op_test.py"  # Segfaults on Windows.
       # Dask.Dataframe bugs on Window Build
       "${tensorflow_source_dir}/tensorflow/contrib/learn/python/learn/tests/dataframe/tensorflow_dataframe_test.py"
       "${tensorflow_source_dir}/tensorflow/contrib/learn/python/learn/learn_io/data_feeder_test.py"
@@ -274,37 +282,15 @@ if (tensorflow_BUILD_PYTHON_TESTS)
       # Need extra build
       "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/conditional_distribution_test.py"
       "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/conditional_transformed_distribution_test.py"
+      "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/estimator_test.py"
+      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/array_ops_test.py"  # depends on python/framework/test_ops
+      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/depthtospace_op_test.py"  # QuantizeV2
+      "${tensorflow_source_dir}/tensorflow/python/kernel_tests/spacetodepth_op_test.py"  # QuantizeV2
       # Windows Path
       "${tensorflow_source_dir}/tensorflow/contrib/framework/python/ops/checkpoint_ops_test.py" #TODO: Fix path
-      "${tensorflow_source_dir}/tensorflow/contrib/keras/python/keras/models_test.py"
-      # Related to Windows Multiprocessing https://github.com/fchollet/keras/issues/5071
-      "${tensorflow_source_dir}/tensorflow/contrib/keras/python/keras/engine/training_test.py"
-      "${tensorflow_source_dir}/tensorflow/contrib/keras/python/keras/utils/data_utils_test.py"
-      "${tensorflow_source_dir}/tensorflow/contrib/keras/python/keras/callbacks_test.py"
-      # Scipy needed
-      "${tensorflow_source_dir}/tensorflow/contrib/keras/python/keras/preprocessing/image_test.py"
-      "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/bijectors/sigmoid_test.py"
-      "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/binomial_test.py"
-      "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/chi2_test.py"
-      "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/geometric_test.py"
-      "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/inverse_gamma_test.py"
-      "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/logistic_test.py"
-      "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/mixture_test.py"
-      "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/mvn_diag_test.py"
-      "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/mvn_full_covariance_test.py"
-      "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/mvn_tril_test.py"
-      "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/negative_binomial_test.py"
-      "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/poisson_test.py"
-      "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/quantized_distribution_test.py"
-      "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/relaxed_bernoulli_test.py"
-      "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/relaxed_onehot_categorical_test.py"
-      "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/transformed_distribution_test.py"
-      "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/vector_student_t_test.py"
-      "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/wishart_test.py"
       "${tensorflow_source_dir}/tensorflow/contrib/factorization/python/ops/kmeans_test.py"
       "${tensorflow_source_dir}/tensorflow/contrib/learn/python/learn/estimators/kmeans_test.py"
-      # Failing with TF 1.3 (TODO)
-      "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/estimator_test.py"
+      # Numpy upgrade needed?
       "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/bijectors/sinh_arcsinh_test.py"
       # Test should only be run manually
       "${tensorflow_source_dir}/tensorflow/python/kernel_tests/reduction_ops_test_big.py"
diff --git a/tensorflow/contrib/cudnn_rnn/python/kernel_tests/cudnn_rnn_test.py b/tensorflow/contrib/cudnn_rnn/python/kernel_tests/cudnn_rnn_test.py
index 9e627bcaf4..1ce8954bb0 100644
--- a/tensorflow/contrib/cudnn_rnn/python/kernel_tests/cudnn_rnn_test.py
+++ b/tensorflow/contrib/cudnn_rnn/python/kernel_tests/cudnn_rnn_test.py
@@ -385,7 +385,7 @@ class CudnnRNNTestSaveRestore(TensorFlowTestCase):
       reset_op = state_ops.assign(
           opaque_params,
           array_ops.zeros(array_ops.shape(opaque_params), dtype=dtype))
-      # Passing graph explictly, otherwise an old sess would be reused.
+      # Passing graph explicitly, otherwise an old sess would be reused.
       with self.test_session(use_gpu=True, graph=g) as sess:
         sess.run(variables.global_variables_initializer())
         val = saver.save(sess, save_path)
@@ -436,7 +436,7 @@ class CudnnRNNTestSaveRestore(TensorFlowTestCase):
       save_path = os.path.join(self.get_temp_dir(),
                                "save-restore-variable-test2")
       saver = saver_lib.Saver()
-      # Passing graph explictly, otherwise an old sess would be reused.
+      # Passing graph explicitly, otherwise an old sess would be reused.
       with self.test_session(use_gpu=True, graph=g) as sess:
         sess.run(variables.global_variables_initializer())
         val = saver.save(sess, save_path)
@@ -484,7 +484,7 @@ class CudnnRNNTestSaveRestore(TensorFlowTestCase):
           array_ops.zeros(
               array_ops.shape(rnn.trainable_variables[0]), dtype=dtype))
 
-      # Passing graph explictly, otherwise an old sess would be reused.
+      # Passing graph explicitly, otherwise an old sess would be reused.
       with self.test_session(use_gpu=True, graph=g) as sess:
         sess.run(variables.global_variables_initializer())
         inputs, initial_state = model.SynthesizeInput(seq_length, batch_size)
diff --git a/tensorflow/contrib/data/python/kernel_tests/BUILD b/tensorflow/contrib/data/python/kernel_tests/BUILD
index 96447abd7c..5339ebb689 100644
--- a/tensorflow/contrib/data/python/kernel_tests/BUILD
+++ b/tensorflow/contrib/data/python/kernel_tests/BUILD
@@ -11,6 +11,9 @@ py_test(
     size = "small",
     srcs = ["batch_dataset_op_test.py"],
     srcs_version = "PY2AND3",
+    tags = [
+        "manual",  # b/67958604
+    ],
     deps = [
         "//tensorflow/contrib/data/python/ops:dataset_ops",
         "//tensorflow/contrib/data/python/ops:transformation_ops",
@@ -358,6 +361,9 @@ py_test(
     size = "small",
     srcs = ["sloppy_transformation_dataset_op_test.py"],
     srcs_version = "PY2AND3",
+    tags = [
+        "manual",  # b/67958761
+    ],
     deps = [
         "//tensorflow/contrib/data/python/ops:dataset_ops",
         "//tensorflow/contrib/data/python/ops:transformation_ops",
diff --git a/tensorflow/contrib/framework/BUILD b/tensorflow/contrib/framework/BUILD
index 6b0599ddd2..dd882acb8e 100644
--- a/tensorflow/contrib/framework/BUILD
+++ b/tensorflow/contrib/framework/BUILD
@@ -10,9 +10,8 @@ package(default_visibility = [
     "//tensorflow:__subpackages__",
 ])
 
-load("//tensorflow:tensorflow.bzl", "cuda_py_test")
-load("//tensorflow:tensorflow.bzl", "tf_custom_op_py_library")
 load("//tensorflow:tensorflow.bzl", "py_test")
+load("//tensorflow:tensorflow.bzl", "tf_custom_op_py_library")
 load("//tensorflow:tensorflow.bzl", "tf_custom_op_library")
 load("//tensorflow:tensorflow.bzl", "tf_gen_op_wrapper_py")
 load("//tensorflow:tensorflow.bzl", "tf_gen_op_libs")
@@ -27,6 +26,7 @@ tf_custom_op_py_library(
         "python/framework/experimental.py",
         "python/framework/tensor_util.py",
         "python/ops/__init__.py",
+        "python/ops/accumulate_n_v2.py",
         "python/ops/arg_scope.py",
         "python/ops/audio_ops.py",
         "python/ops/checkpoint_ops.py",
@@ -149,6 +149,31 @@ py_test(
     ],
 )
 
+py_test(
+    name = "accumulate_n_v2_test",
+    size = "small",
+    srcs = ["python/ops/accumulate_n_v2_test.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":framework_py",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework_for_generated_wrappers",
+    ],
+)
+
+py_test(
+    name = "accumulate_n_v2_eager_test",
+    size = "small",
+    srcs = ["python/ops/accumulate_n_v2_eager_test.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":framework_py",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework_for_generated_wrappers",
+        "//tensorflow/python/eager:backprop",
+    ],
+)
+
 py_test(
     name = "ops_test",
     size = "small",
diff --git a/tensorflow/contrib/framework/python/ops/accumulate_n_v2.py b/tensorflow/contrib/framework/python/ops/accumulate_n_v2.py
new file mode 100644
index 0000000000..a0667bd489
--- /dev/null
+++ b/tensorflow/contrib/framework/python/ops/accumulate_n_v2.py
@@ -0,0 +1,111 @@
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Ops that will eventually be folded into tensorflow/python/ops/math_ops.py
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+
+from tensorflow.python.eager import context
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import tensor_shape
+from tensorflow.python.ops import gen_math_ops
+from tensorflow.python.ops import math_ops
+
+
+
+def accumulate_n_v2(inputs, shape=None, tensor_dtype=None, name=None):
+  """Returns the element-wise sum of a list of tensors.
+
+  Optionally, pass `shape` and `tensor_dtype` for shape and type checking,
+  otherwise, these are inferred.
+
+  `tf.accumulate_n_v2` performs the same operation as `tf.add_n`, but does not
+  wait for all of its inputs to be ready before beginning to sum. This can
+  save memory if inputs are ready at different times, since minimum temporary
+  storage is proportional to the output size rather than the inputs size.
+
+  Unlike the original `accumulate_n`, `accumulate_n_v2` is differentiable.
+
+  For example:
+
+  ```python
+  a = tf.constant([[1, 2], [3, 4]])
+  b = tf.constant([[5, 0], [0, 6]])
+  tf.accumulate_n_v2([a, b, a])  # [[7, 4], [6, 14]]
+
+  # Explicitly pass shape and type
+  tf.accumulate_n_v2([a, b, a], shape=[2, 2], tensor_dtype=tf.int32)  
+                                                                   # [[7,  4],
+                                                                   #  [6, 14]]
+  ```
+
+  Args:
+    inputs: A list of `Tensor` objects, each with same shape and type.
+    shape: Shape of elements of `inputs`.
+    tensor_dtype: The type of `inputs`.
+    name: A name for the operation (optional).
+
+  Returns:
+    A `Tensor` of same shape and type as the elements of `inputs`.
+
+  Raises:
+    ValueError: If `inputs` don't all have same shape and dtype or the shape
+    cannot be inferred.
+  """
+  _INPUTS_ERR_MSG = ValueError("inputs must be a list of at least one Tensor"
+                               "with the same dtype and shape")
+  if not inputs or not isinstance(inputs, (list, tuple)):
+    raise _INPUTS_ERR_MSG
+  inputs = ops.convert_n_to_tensor_or_indexed_slices(inputs)
+  if not all(isinstance(x, ops.Tensor) for x in inputs):
+    raise _INPUTS_ERR_MSG
+  if not all(x.dtype == inputs[0].dtype for x in inputs):
+    raise _INPUTS_ERR_MSG
+  if shape is not None:
+    shape = tensor_shape.as_shape(shape)
+  else:
+    shape = tensor_shape.unknown_shape()
+  for input_tensor in inputs:
+    if isinstance(input_tensor, ops.Tensor):
+      shape = shape.merge_with(input_tensor.get_shape())
+
+  # tensor_dtype is for safety only; operator's output type computed in C++
+  if tensor_dtype is not None and tensor_dtype != inputs[0].dtype:
+    raise TypeError("tensor_dtype is {}, but input is of type {}"
+                    .format(tensor_dtype, inputs[0].dtype))
+
+  if len(inputs) == 1 and name is None:
+    return inputs[0]
+  elif len(inputs) == 1 and name is not None:
+    return array_ops.identity(inputs[0], name=name)
+  elif context.in_eager_mode():
+    # TemporaryVariable not currently supported in eager mode; fall back 
+    # onto AddN for now.
+    # TODO(frreiss) remove this once the lifetime of eager variables gets
+    # addressed
+    return math_ops.add_n(inputs, name=name)
+  else:
+    return gen_math_ops._accumulate_nv2(inputs, name=name, shape=shape)
+
+# The following code should eventually be merged into 
+# tensorflow/python/ops/math_grad.py
+@ops.RegisterGradient("AccumulateNV2")
+def _AddNGrad(op, grad):
+  """Same as gradient for AddN. Copies the gradient to all inputs."""
+  # Not broadcasting.
+  return [grad] * len(op.inputs)
+
diff --git a/tensorflow/contrib/framework/python/ops/accumulate_n_v2_eager_test.py b/tensorflow/contrib/framework/python/ops/accumulate_n_v2_eager_test.py
new file mode 100644
index 0000000000..c2229bb8ad
--- /dev/null
+++ b/tensorflow/contrib/framework/python/ops/accumulate_n_v2_eager_test.py
@@ -0,0 +1,85 @@
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for new version of accumulate_n op that will eventually go into 
+`ops.math_ops`.
+
+These test cases spefically exercise the `eager` APIs. They need to be in a 
+separate file from the remaining tests because eager mode is currently something
+you can turn on but can't turn off for the lifetime of the current process."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.contrib.framework.python.ops import accumulate_n_v2 as av2
+
+from tensorflow.python.eager import backprop
+from tensorflow.python.eager import context as eager_context
+from tensorflow.python.eager import tape
+
+
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes as dtypes_lib
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
+from tensorflow.python.ops import gradients
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import resource_variable_ops
+from tensorflow.python.platform import test
+
+
+
+class AccumulateNV2EagerTest(test_util.TensorFlowTestCase):
+  """Tests of the new, differentiable version of accumulate_n"""
+
+  def testMinimalEagerMode(self):
+    forty = constant_op.constant(40)
+    two = constant_op.constant(2)
+    answer = av2.accumulate_n_v2([forty, two])
+    self.assertEqual(42, answer.numpy())
+
+
+  def testFloat(self):
+    np.random.seed(12345)
+    x = [np.random.random((1, 2, 3, 4, 5)) - 0.5 for _ in range(5)]
+    tf_x = ops.convert_n_to_tensor(x)
+    with self.test_session(use_gpu=True):
+      self.assertAllClose(sum(x), av2.accumulate_n_v2(tf_x).numpy())
+      self.assertAllClose(x[0] * 5, av2.accumulate_n_v2([tf_x[0]] * 5).numpy())
+
+  def testGrad(self):
+    np.random.seed(42)
+    num_inputs = 3
+    input_vars = [
+        resource_variable_ops.ResourceVariable(10.0 * np.random.random(), 
+                                               name="t%d" % i)
+        for i in range(0, num_inputs)
+    ]
+
+    def fn(first, second, third):
+      return av2.accumulate_n_v2([first, second, third])
+
+    grad_fn = backprop.gradients_function(fn)      
+    grad = grad_fn(input_vars[0], input_vars[1], input_vars[2])
+    self.assertAllEqual(np.repeat(1.0, num_inputs), # d/dx (x + y + ...) = 1
+                        [elem.numpy() for elem in grad])
+
+
+
+if __name__ == "__main__":
+  ops.enable_eager_execution()
+  test.main()
+
diff --git a/tensorflow/contrib/framework/python/ops/accumulate_n_v2_test.py b/tensorflow/contrib/framework/python/ops/accumulate_n_v2_test.py
new file mode 100644
index 0000000000..3386e849d5
--- /dev/null
+++ b/tensorflow/contrib/framework/python/ops/accumulate_n_v2_test.py
@@ -0,0 +1,123 @@
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for new version of accumulate_n op that will eventually go into 
+`ops.math_ops`."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.contrib.framework.python.ops import accumulate_n_v2 as av2
+
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes as dtypes_lib
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
+from tensorflow.python.ops import gradients
+from tensorflow.python.ops import variables
+from tensorflow.python.platform import googletest
+
+
+
+class AccumulateNV2Test(test_util.TensorFlowTestCase):
+  """Tests of the new, differentiable version of accumulate_n"""
+
+  def testFloat(self):
+    np.random.seed(12345)
+    x = [np.random.random((1, 2, 3, 4, 5)) - 0.5 for _ in range(5)]
+    tf_x = ops.convert_n_to_tensor(x)
+    with self.test_session(use_gpu=True):
+      self.assertAllClose(sum(x), av2.accumulate_n_v2(tf_x).eval())
+      self.assertAllClose(x[0] * 5, av2.accumulate_n_v2([tf_x[0]] * 5).eval())
+
+  def testInt(self):
+    np.random.seed(54321)
+    x = [np.random.randint(-128, 128, (5, 4, 3, 2, 1)) for _ in range(6)]
+    tf_x = ops.convert_n_to_tensor(x)
+    with self.test_session(use_gpu=True):
+      self.assertAllEqual(sum(x), av2.accumulate_n_v2(tf_x).eval())
+      self.assertAllEqual(x[0] * 6, av2.accumulate_n_v2([tf_x[0]] * 6).eval())
+
+  def testGrad(self):
+    np.random.seed(42)
+    for num_inputs in range(1, 10):
+      with self.test_session(use_gpu=True) as sess:
+        input_vars = [
+            variables.Variable(10.0 * np.random.random())
+            for i in range(0, num_inputs)
+        ]
+        accum_n = av2.accumulate_n_v2(input_vars)
+        sess.run(variables.global_variables_initializer())
+        accum_n_grad = gradients.gradients(accum_n, input_vars)
+        self.assertAllEqual(np.repeat(1.0, num_inputs), # d/dx (x + y + ...) = 1
+                            [g.eval() for g in accum_n_grad])
+
+  # The tests below used to be in a separate class under cwise_ops_test.py,
+  # which did not run in the default test target.
+  # Putting them here so that everything that exercises AccumulateNV2 is in
+  # one place and the default build runs all unit tests.
+  def testSimple(self):
+    with self.test_session():
+      random_arrays = [
+          np.random.rand(16, 16, 16, 16).astype(np.float32) for _ in range(20)
+      ]
+      random_tensors = [
+          ops.convert_to_tensor(
+              x, dtype=dtypes_lib.float32) for x in random_arrays
+      ]
+      tf_val = av2.accumulate_n_v2(random_tensors)
+      np_val = random_arrays[0]
+      for random_array in random_arrays[1:]:
+        np_val += random_array
+      self.assertAllClose(np_val, tf_val.eval())
+
+  def testZeroArgs(self):
+    with self.test_session():
+      with self.assertRaises(ValueError):
+        tf_val = av2.accumulate_n_v2([])
+        tf_val.eval()
+
+  def testWrongShape(self):
+    with self.test_session():
+      with self.assertRaises(ValueError):
+        a = variables.Variable(0.2)
+        b = variables.Variable(0.1)
+        tf_val = av2.accumulate_n_v2([a,b], shape=[2,2]) # Should be shape=[]
+
+  def testIncompatibleShapes(self):
+    with self.test_session():
+      with self.assertRaises(ValueError):
+        a = variables.Variable(np.array([0.1,0.2]))
+        b = variables.Variable(np.array([[0.3],[0.4]]))
+        tf_val = av2.accumulate_n_v2([a,b]) 
+
+  def testWrongType(self):
+    with self.test_session():
+      with self.assertRaises(TypeError):
+        a = variables.Variable(0.2, dtype=np.float32)
+        b = variables.Variable(0.1, dtype=np.float32)
+        tf_val = av2.accumulate_n_v2([a,b], tensor_dtype=np.int32) 
+
+  def testWrongTypeOneInput(self):
+    # Scenario that used to trigger a bug, even when testWrongType() worked
+    with self.test_session():
+      with self.assertRaises(TypeError):
+        a = variables.Variable(0.2, dtype=np.float32)
+        tf_val = av2.accumulate_n_v2([a], tensor_dtype=np.int32) 
+
+
+if __name__ == "__main__":
+  googletest.main()
diff --git a/tensorflow/contrib/image/__init__.py b/tensorflow/contrib/image/__init__.py
index 59a322d3ca..d030dffade 100755
--- a/tensorflow/contrib/image/__init__.py
+++ b/tensorflow/contrib/image/__init__.py
@@ -26,6 +26,8 @@ projective transforms (including rotation) are supported.
 @@random_yiq_hsv
 @@rotate
 @@transform
+@@translate
+@@translations_to_projective_transforms
 @@bipartite_match
 @@single_image_random_dot_stereograms
 """
@@ -41,6 +43,8 @@ from tensorflow.contrib.image.python.ops.image_ops import angles_to_projective_t
 from tensorflow.contrib.image.python.ops.image_ops import compose_transforms
 from tensorflow.contrib.image.python.ops.image_ops import rotate
 from tensorflow.contrib.image.python.ops.image_ops import transform
+from tensorflow.contrib.image.python.ops.image_ops import translate
+from tensorflow.contrib.image.python.ops.image_ops import translations_to_projective_transforms
 from tensorflow.contrib.image.python.ops.single_image_random_dot_stereograms import single_image_random_dot_stereograms
 
 from tensorflow.python.util.all_util import remove_undocumented
diff --git a/tensorflow/contrib/image/python/kernel_tests/image_ops_test.py b/tensorflow/contrib/image/python/kernel_tests/image_ops_test.py
index b8a0706b61..b50177ae56 100644
--- a/tensorflow/contrib/image/python/kernel_tests/image_ops_test.py
+++ b/tensorflow/contrib/image/python/kernel_tests/image_ops_test.py
@@ -36,8 +36,8 @@ _DTYPES = set(
 class ImageOpsTest(test_util.TensorFlowTestCase):
 
   def test_zeros(self):
-    with self.test_session():
-      for dtype in _DTYPES:
+    for dtype in _DTYPES:
+      with self.test_session():
         for shape in [(5, 5), (24, 24), (2, 24, 24, 3)]:
           for angle in [0, 1, np.pi / 2.0]:
             image = array_ops.zeros(shape, dtype)
@@ -46,8 +46,8 @@ class ImageOpsTest(test_util.TensorFlowTestCase):
                 np.zeros(shape, dtype.as_numpy_dtype()))
 
   def test_rotate_even(self):
-    with self.test_session():
-      for dtype in _DTYPES:
+    for dtype in _DTYPES:
+      with self.test_session():
         image = array_ops.reshape(
             math_ops.cast(math_ops.range(36), dtype), (6, 6))
         image_rep = array_ops.tile(image[None, :, :, None], [3, 1, 1, 1])
@@ -68,8 +68,8 @@ class ImageOpsTest(test_util.TensorFlowTestCase):
                               [1, 7, 13, 19, 25, 31], [0, 6, 12, 18, 24, 30]]])
 
   def test_rotate_odd(self):
-    with self.test_session():
-      for dtype in _DTYPES:
+    for dtype in _DTYPES:
+      with self.test_session():
         image = array_ops.reshape(
             math_ops.cast(math_ops.range(25), dtype), (5, 5))
         image_rep = array_ops.tile(image[None, :, :, None], [3, 1, 1, 1])
@@ -87,9 +87,25 @@ class ImageOpsTest(test_util.TensorFlowTestCase):
                               [22, 17, 12, 7, 2], [23, 18, 13, 8, 3],
                               [24, 19, 14, 9, 4]]])
 
+  def test_translate(self):
+    for dtype in _DTYPES:
+      with self.test_session():
+        image = constant_op.constant(
+            [[1, 0, 1, 0],
+             [0, 1, 0, 1],
+             [1, 0, 1, 0],
+             [0, 1, 0, 1]], dtype=dtype)
+        translation = constant_op.constant([-1, -1], dtypes.float32)
+        image_translated = image_ops.translate(image, translation)
+        self.assertAllEqual(image_translated.eval(),
+                            [[1, 0, 1, 0],
+                             [0, 1, 0, 0],
+                             [1, 0, 1, 0],
+                             [0, 0, 0, 0]])
+
   def test_compose(self):
-    with self.test_session():
-      for dtype in _DTYPES:
+    for dtype in _DTYPES:
+      with self.test_session():
         image = constant_op.constant(
             [[1, 1, 1, 0],
              [1, 0, 0, 0],
@@ -246,4 +262,3 @@ class BipartiteMatchTest(test_util.TensorFlowTestCase):
 
 if __name__ == "__main__":
   googletest.main()
-
diff --git a/tensorflow/contrib/image/python/ops/image_ops.py b/tensorflow/contrib/image/python/ops/image_ops.py
index aef3e385b5..011ddeaa9a 100644
--- a/tensorflow/contrib/image/python/ops/image_ops.py
+++ b/tensorflow/contrib/image/python/ops/image_ops.py
@@ -37,16 +37,18 @@ _IMAGE_DTYPES = set(
 ops.RegisterShape("ImageProjectiveTransform")(common_shapes.call_cpp_shape_fn)
 
 
-def rotate(images, angles, interpolation="NEAREST"):
+def rotate(images, angles, interpolation="NEAREST", name=None):
   """Rotate image(s) by the passed angle(s) in radians.
 
   Args:
     images: A tensor of shape (num_images, num_rows, num_columns, num_channels)
        (NHWC), (num_rows, num_columns, num_channels) (HWC), or
-       (num_rows, num_columns) (HW).
+       (num_rows, num_columns) (HW). The rank must be statically known (the
+       shape is not `TensorShape(None)`.
     angles: A scalar angle to rotate all images by, or (if images has rank 4)
        a vector of length num_images, with an angle for each image in the batch.
     interpolation: Interpolation mode. Supported values: "NEAREST", "BILINEAR".
+    name: The name of the op.
 
   Returns:
     Image(s) with the same type and shape as `images`, rotated by the given
@@ -55,38 +57,77 @@ def rotate(images, angles, interpolation="NEAREST"):
   Raises:
     TypeError: If `image` is an invalid type.
   """
-  image_or_images = ops.convert_to_tensor(images, name="images")
-  if image_or_images.dtype.base_dtype not in _IMAGE_DTYPES:
-    raise TypeError("Invalid dtype %s." % image_or_images.dtype)
-  if len(image_or_images.get_shape()) == 2:
-    images = image_or_images[None, :, :, None]
-  elif len(image_or_images.get_shape()) == 3:
-    images = image_or_images[None, :, :, :]
-  elif len(image_or_images.get_shape()) == 4:
-    images = image_or_images
-  else:
-    raise TypeError("Images should have rank between 2 and 4.")
-
-  image_height = math_ops.cast(array_ops.shape(images)[1], dtypes.float32)[None]
-  image_width = math_ops.cast(array_ops.shape(images)[2], dtypes.float32)[None]
-  output = transform(
-      images,
-      angles_to_projective_transforms(angles, image_height, image_width),
-      interpolation=interpolation)
-  if len(image_or_images.get_shape()) == 2:
-    return output[0, :, :, 0]
-  elif len(image_or_images.get_shape()) == 3:
-    return output[0, :, :, :]
-  else:
-    return output
+  with ops.name_scope(name, "rotate"):
+    image_or_images = ops.convert_to_tensor(images)
+    if image_or_images.dtype.base_dtype not in _IMAGE_DTYPES:
+      raise TypeError("Invalid dtype %s." % image_or_images.dtype)
+    elif image_or_images.get_shape().ndims is None:
+      raise TypeError("image_or_images rank must be statically known")
+    elif len(image_or_images.get_shape()) == 2:
+      images = image_or_images[None, :, :, None]
+    elif len(image_or_images.get_shape()) == 3:
+      images = image_or_images[None, :, :, :]
+    elif len(image_or_images.get_shape()) == 4:
+      images = image_or_images
+    else:
+      raise TypeError("Images should have rank between 2 and 4.")
+
+    image_height = math_ops.cast(array_ops.shape(images)[1],
+                                 dtypes.float32)[None]
+    image_width = math_ops.cast(array_ops.shape(images)[2],
+                                dtypes.float32)[None]
+    output = transform(
+        images,
+        angles_to_projective_transforms(angles, image_height, image_width),
+        interpolation=interpolation)
+    if image_or_images.get_shape().ndims is None:
+      raise TypeError("image_or_images rank must be statically known")
+    elif len(image_or_images.get_shape()) == 2:
+      return output[0, :, :, 0]
+    elif len(image_or_images.get_shape()) == 3:
+      return output[0, :, :, :]
+    else:
+      return output
+
+
+def translate(images, translations, interpolation="NEAREST", name=None):
+  """Translate image(s) by the passed vectors(s).
 
+  Args:
+    images: A tensor of shape (num_images, num_rows, num_columns, num_channels)
+        (NHWC), (num_rows, num_columns, num_channels) (HWC), or
+        (num_rows, num_columns) (HW). The rank must be statically known (the
+        shape is not `TensorShape(None)`.
+    translations: A vector representing [dx, dy] or (if images has rank 4)
+        a matrix of length num_images, with a [dx, dy] vector for each image in
+        the batch.
+    interpolation: Interpolation mode. Supported values: "NEAREST", "BILINEAR".
+    name: The name of the op.
 
-def angles_to_projective_transforms(angles, image_height, image_width):
+  Returns:
+    Image(s) with the same type and shape as `images`, translated by the given
+        vector(s). Empty space due to the translation will be filled with zeros.
+
+  Raises:
+    TypeError: If `image` is an invalid type.
+  """
+  with ops.name_scope(name, "translate"):
+    return transform(
+        images,
+        translations_to_projective_transforms(translations),
+        interpolation=interpolation)
+
+
+def angles_to_projective_transforms(angles,
+                                    image_height,
+                                    image_width,
+                                    name=None):
   """Returns projective transform(s) for the given angle(s).
 
   Args:
     angles: A scalar angle to rotate all images by, or (for batches of images)
-      a vector with an angle to rotate each image in the batch.
+        a vector with an angle to rotate each image in the batch. The rank must
+        be statically known (the shape is not `TensorShape(None)`.
     image_height: Height of the image(s) to be transformed.
     image_width: Width of the image(s) to be transformed.
 
@@ -94,41 +135,89 @@ def angles_to_projective_transforms(angles, image_height, image_width):
     A tensor of shape (num_images, 8). Projective transforms which can be given
       to `tf.contrib.image.transform`.
   """
-  angle_or_angles = ops.convert_to_tensor(
-      angles, name="angles", dtype=dtypes.float32)
-  if len(angle_or_angles.get_shape()) == 0:  # pylint: disable=g-explicit-length-test
-    angles = angle_or_angles[None]
-  elif len(angle_or_angles.get_shape()) == 1:
-    angles = angle_or_angles
-  else:
-    raise TypeError("Angles should have rank 0 or 1.")
-  x_offset = ((image_width - 1) - (math_ops.cos(angles) *
-                                   (image_width - 1) - math_ops.sin(angles) *
-                                   (image_height - 1))) / 2.0
-  y_offset = ((image_height - 1) - (math_ops.sin(angles) *
-                                    (image_width - 1) + math_ops.cos(angles) *
-                                    (image_height - 1))) / 2.0
-  num_angles = array_ops.shape(angles)[0]
-  return array_ops.concat(
-      values=[
-          math_ops.cos(angles)[:, None],
-          -math_ops.sin(angles)[:, None],
-          x_offset[:, None],
-          math_ops.sin(angles)[:, None],
-          math_ops.cos(angles)[:, None],
-          y_offset[:, None],
-          array_ops.zeros((num_angles, 2), dtypes.float32),
-      ],
-      axis=1)
-
-
-def transform(images, transforms, interpolation="NEAREST"):
+  with ops.name_scope(name, "angles_to_projective_transforms"):
+    angle_or_angles = ops.convert_to_tensor(
+        angles, name="angles", dtype=dtypes.float32)
+    if len(angle_or_angles.get_shape()) == 0:  # pylint: disable=g-explicit-length-test
+      angles = angle_or_angles[None]
+    elif len(angle_or_angles.get_shape()) == 1:
+      angles = angle_or_angles
+    else:
+      raise TypeError("Angles should have rank 0 or 1.")
+    x_offset = ((image_width - 1) - (math_ops.cos(angles) *
+                                     (image_width - 1) - math_ops.sin(angles) *
+                                     (image_height - 1))) / 2.0
+    y_offset = ((image_height - 1) - (math_ops.sin(angles) *
+                                      (image_width - 1) + math_ops.cos(angles) *
+                                      (image_height - 1))) / 2.0
+    num_angles = array_ops.shape(angles)[0]
+    return array_ops.concat(
+        values=[
+            math_ops.cos(angles)[:, None],
+            -math_ops.sin(angles)[:, None],
+            x_offset[:, None],
+            math_ops.sin(angles)[:, None],
+            math_ops.cos(angles)[:, None],
+            y_offset[:, None],
+            array_ops.zeros((num_angles, 2), dtypes.float32),
+        ],
+        axis=1)
+
+
+def translations_to_projective_transforms(translations, name=None):
+  """Returns projective transform(s) for the given translation(s).
+
+  Args:
+      translations: A 2-element list representing [dx, dy] or a matrix of
+          2-element lists representing [dx, dy] to translate for each image
+          (for a batch of images). The rank must be statically known (the shape
+          is not `TensorShape(None)`.
+      name: The name of the op.
+
+  Returns:
+      A tensor of shape (num_images, 8) projective transforms which can be given
+          to `tf.contrib.image.transform`.
+  """
+  with ops.name_scope(name, "translations_to_projective_transforms"):
+    translation_or_translations = ops.convert_to_tensor(
+        translations, name="translations", dtype=dtypes.float32)
+    if translation_or_translations.get_shape().ndims is None:
+      raise TypeError(
+          "translation_or_translations rank must be statically known")
+    elif len(translation_or_translations.get_shape()) == 1:
+      translations = translation_or_translations[None]
+    elif len(translation_or_translations.get_shape()) == 2:
+      translations = translation_or_translations
+    else:
+      raise TypeError("Translations should have rank 1 or 2.")
+    num_translations = array_ops.shape(translations)[0]
+    # The translation matrix looks like:
+    #     [[1 0 -dx]
+    #      [0 1 -dy]
+    #      [0 0 1]]
+    # where the last entry is implicit.
+    # Translation matrices are always float32.
+    return array_ops.concat(
+        values=[
+            array_ops.ones((num_translations, 1), dtypes.float32),
+            array_ops.zeros((num_translations, 1), dtypes.float32),
+            -translations[:, 0, None],
+            array_ops.zeros((num_translations, 1), dtypes.float32),
+            array_ops.ones((num_translations, 1), dtypes.float32),
+            -translations[:, 1, None],
+            array_ops.zeros((num_translations, 2), dtypes.float32),
+        ],
+        axis=1)
+
+
+def transform(images, transforms, interpolation="NEAREST", name=None):
   """Applies the given transform(s) to the image(s).
 
   Args:
     images: A tensor of shape (num_images, num_rows, num_columns, num_channels)
        (NHWC), (num_rows, num_columns, num_channels) (HWC), or
-       (num_rows, num_columns) (HW).
+       (num_rows, num_columns) (HW). The rank must be statically known (the
+       shape is not `TensorShape(None)`.
     transforms: Projective transform matrix/matrices. A vector of length 8 or
        tensor of size N x 8. If one row of transforms is
        [a0, a1, a2, b0, b1, b2, c0, c1], then it maps the *output* point
@@ -146,34 +235,40 @@ def transform(images, transforms, interpolation="NEAREST"):
   Raises:
     TypeError: If `image` is an invalid type.
   """
-  image_or_images = ops.convert_to_tensor(images, name="images")
-  transform_or_transforms = ops.convert_to_tensor(
-      transforms, name="transforms", dtype=dtypes.float32)
-  if image_or_images.dtype.base_dtype not in _IMAGE_DTYPES:
-    raise TypeError("Invalid dtype %s." % image_or_images.dtype)
-  if len(image_or_images.get_shape()) == 2:
-    images = image_or_images[None, :, :, None]
-  elif len(image_or_images.get_shape()) == 3:
-    images = image_or_images[None, :, :, :]
-  elif len(image_or_images.get_shape()) == 4:
-    images = image_or_images
-  else:
-    raise TypeError("Images should have rank between 2 and 4.")
-
-  if len(transform_or_transforms.get_shape()) == 1:
-    transforms = transform_or_transforms[None]
-  elif len(transform_or_transforms.get_shape()) == 2:
-    transforms = transform_or_transforms
-  else:
-    raise TypeError("Transforms should have rank 1 or 2.")
-  output = gen_image_ops.image_projective_transform(
-      images, transforms, interpolation=interpolation.upper())
-  if len(image_or_images.get_shape()) == 2:
-    return output[0, :, :, 0]
-  elif len(image_or_images.get_shape()) == 3:
-    return output[0, :, :, :]
-  else:
-    return output
+  with ops.name_scope(name, "transform"):
+    image_or_images = ops.convert_to_tensor(images, name="images")
+    transform_or_transforms = ops.convert_to_tensor(
+        transforms, name="transforms", dtype=dtypes.float32)
+    if image_or_images.dtype.base_dtype not in _IMAGE_DTYPES:
+      raise TypeError("Invalid dtype %s." % image_or_images.dtype)
+    elif image_or_images.get_shape().ndims is None:
+      raise TypeError("image_or_images rank must be statically known")
+    elif len(image_or_images.get_shape()) == 2:
+      images = image_or_images[None, :, :, None]
+    elif len(image_or_images.get_shape()) == 3:
+      images = image_or_images[None, :, :, :]
+    elif len(image_or_images.get_shape()) == 4:
+      images = image_or_images
+    else:
+      raise TypeError("Images should have rank between 2 and 4.")
+
+    if len(transform_or_transforms.get_shape()) == 1:
+      transforms = transform_or_transforms[None]
+    elif transform_or_transforms.get_shape().ndims is None:
+      raise TypeError(
+          "transform_or_transforms rank must be statically known")
+    elif len(transform_or_transforms.get_shape()) == 2:
+      transforms = transform_or_transforms
+    else:
+      raise TypeError("Transforms should have rank 1 or 2.")
+    output = gen_image_ops.image_projective_transform(
+        images, transforms, interpolation=interpolation.upper())
+    if len(image_or_images.get_shape()) == 2:
+      return output[0, :, :, 0]
+    elif len(image_or_images.get_shape()) == 3:
+      return output[0, :, :, :]
+    else:
+      return output
 
 
 def compose_transforms(*transforms):
@@ -191,11 +286,12 @@ def compose_transforms(*transforms):
         order.
   """
   assert transforms, "transforms cannot be empty"
-  composed = _flat_transforms_to_matrices(transforms[0])
-  for tr in transforms[1:]:
-    # Multiply batches of matrices.
-    composed = math_ops.matmul(composed, _flat_transforms_to_matrices(tr))
-  return _transform_matrices_to_flat(composed)
+  with ops.name_scope("compose_transforms"):
+    composed = _flat_transforms_to_matrices(transforms[0])
+    for tr in transforms[1:]:
+      # Multiply batches of matrices.
+      composed = math_ops.matmul(composed, _flat_transforms_to_matrices(tr))
+    return _transform_matrices_to_flat(composed)
 
 
 def _flat_transforms_to_matrices(transforms):
@@ -211,8 +307,8 @@ def _flat_transforms_to_matrices(transforms):
 
 def _transform_matrices_to_flat(transform_matrices):
   # Flatten each matrix.
-  transforms = array_ops.reshape(
-      transform_matrices, constant_op.constant([-1, 9]))
+  transforms = array_ops.reshape(transform_matrices,
+                                 constant_op.constant([-1, 9]))
   # Divide each matrix by the last entry (normally 1).
   transforms /= transforms[:, 8:9]
   return transforms[:, :8]
@@ -260,10 +356,10 @@ def _image_projective_transform_grad(op, grad):
     return [output, None]
 
 
-def bipartite_match(
-    distance_mat,
-    num_valid_rows,
-    top_k=-1):
+def bipartite_match(distance_mat,
+                    num_valid_rows,
+                    top_k=-1,
+                    name="bipartite_match"):
   """Find bipartite matching based on a given distance matrix.
 
   A greedy bi-partite matching algorithm is used to obtain the matching with
@@ -282,6 +378,7 @@ def bipartite_match(
     top_k: A scalar that specifies the number of top-k matches to retrieve.
       If set to be negative, then is set according to the maximum number of
       matches from `distance_mat`.
+    name: The name of the op.
 
   Returns:
     row_to_col_match_indices: A vector of length num_rows, which is the number
@@ -292,7 +389,8 @@ def bipartite_match(
       If `col_to_row_match_indices[j]` is not -1, column j is matched to row
       `col_to_row_match_indices[j]`.
   """
-  result = gen_image_ops.bipartite_match(distance_mat, num_valid_rows, top_k)
+  result = gen_image_ops.bipartite_match(
+      distance_mat, num_valid_rows, top_k, name=name)
   return result
 
 
diff --git a/tensorflow/contrib/kfac/python/ops/loss_functions.py b/tensorflow/contrib/kfac/python/ops/loss_functions.py
index 0b5c3d4928..69d97f0b5b 100644
--- a/tensorflow/contrib/kfac/python/ops/loss_functions.py
+++ b/tensorflow/contrib/kfac/python/ops/loss_functions.py
@@ -104,7 +104,7 @@ class LossFunction(object):
 
   @abc.abstractmethod
   def multiply_hessian_factor_transpose(self, vector):
-    """Right-multiply a vector by the tranpose of a factor B of the Hessian.
+    """Right-multiply a vector by the transpose of a factor B of the Hessian.
 
     Here the 'Hessian' is the Hessian matrix (i.e. matrix of 2nd-derivatives)
     of the loss function with respect to its inputs.  Typically this will be
@@ -218,7 +218,7 @@ class NegativeLogProbLoss(LossFunction):
 
   @abc.abstractmethod
   def multiply_fisher_factor_transpose(self, vector):
-    """Right-multiply a vector by the tranpose of a factor B of the Fisher.
+    """Right-multiply a vector by the transpose of a factor B of the Fisher.
 
     Here the 'Fisher' is the Fisher information matrix (i.e. expected outer-
     product of gradients) with respect to the parameters of the underlying
@@ -397,7 +397,7 @@ class NormalMeanVarianceNegativeLogProbLoss(DistributionNegativeLogProbLoss):
 
   This class parameterizes a multivariate normal distribution with n independent
   dimensions. Unlike `NormalMeanNegativeLogProbLoss`, this class does not
-  assume the variance is held constant. The Fisher Information for for n = 1
+  assume the variance is held constant. The Fisher Information for n = 1
   is given by,
 
   F = [[1 / variance,                0],
diff --git a/tensorflow/contrib/kfac/python/ops/op_queue.py b/tensorflow/contrib/kfac/python/ops/op_queue.py
index 0617c5be4d..831870fca4 100644
--- a/tensorflow/contrib/kfac/python/ops/op_queue.py
+++ b/tensorflow/contrib/kfac/python/ops/op_queue.py
@@ -61,7 +61,7 @@ class OpQueue(object):
       sess: tf.Session.
 
     Returns:
-      Next Op chosen from from 'ops'.
+      Next Op chosen from 'ops'.
     """
     # In Python 3, type(next_op_name) == bytes. Calling bytes.decode('ascii')
     # returns a str.
diff --git a/tensorflow/contrib/layers/__init__.py b/tensorflow/contrib/layers/__init__.py
index d8ab7c2d70..d309ba958d 100644
--- a/tensorflow/contrib/layers/__init__.py
+++ b/tensorflow/contrib/layers/__init__.py
@@ -47,6 +47,7 @@ See the @{$python/contrib.layers} guide.
 @@separable_conv2d
 @@separable_convolution2d
 @@softmax
+@@spatial_softmax
 @@stack
 @@unit_norm
 @@bow_encoder
diff --git a/tensorflow/contrib/learn/python/learn/learn_runner.py b/tensorflow/contrib/learn/python/learn/learn_runner.py
index 9f9740ec49..2af723a0d6 100644
--- a/tensorflow/contrib/learn/python/learn/learn_runner.py
+++ b/tensorflow/contrib/learn/python/learn/learn_runner.py
@@ -165,7 +165,7 @@ def run(experiment_fn, output_dir=None, schedule=None, run_config=None,
       must be None.
       2) It accepts two arguments `run_config` and `hparams`, which should be
       used to create the `Estimator` (`run_config` passed as `config` to its
-      constructor; `hparams` used as the hyper-paremeters of the model).
+      constructor; `hparams` used as the hyper-parameters of the model).
       It must return an `Experiment`. For this case, `output_dir` must be None.
     output_dir: Base output directory [Deprecated].
     schedule: The name of the method in the `Experiment` to run.
diff --git a/tensorflow/contrib/losses/python/losses/loss_ops.py b/tensorflow/contrib/losses/python/losses/loss_ops.py
index 1d2477b8b7..7c523ad492 100644
--- a/tensorflow/contrib/losses/python/losses/loss_ops.py
+++ b/tensorflow/contrib/losses/python/losses/loss_ops.py
@@ -28,6 +28,7 @@ from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import nn
 from tensorflow.python.ops import nn_ops
 from tensorflow.python.util.deprecation import deprecated
+from tensorflow.python.util.deprecation import deprecated_args
 
 __all__ = ["absolute_difference",
            "add_loss",
@@ -623,8 +624,9 @@ def mean_pairwise_squared_error(
 
 
 @deprecated("2016-12-30", "Use tf.losses.cosine_distance instead.")
+@deprecated_args(None, "dim is deprecated, use axis instead", "dim")
 def cosine_distance(
-    predictions, labels=None, dim=None, weights=1.0, scope=None):
+    predictions, labels=None, axis=None, weights=1.0, scope=None, dim=None):
   """Adds a cosine-distance loss to the training procedure.
 
   Note that the function assumes that `predictions` and `labels` are already
@@ -633,10 +635,11 @@ def cosine_distance(
   Args:
     predictions: An arbitrary matrix.
     labels: A `Tensor` whose shape matches 'predictions'
-    dim: The dimension along which the cosine distance is computed.
+    axis: The dimension along which the cosine distance is computed.
     weights: Coefficients for the loss a scalar, a tensor of shape
       [batch_size] or a tensor whose shape matches `predictions`.
     scope: The scope for the operations performed in computing the loss.
+    dim: The old (deprecated) name for `axis`.
 
   Returns:
     A scalar `Tensor` representing the loss value.
@@ -645,8 +648,12 @@ def cosine_distance(
     ValueError: If `predictions` shape doesn't match `labels` shape, or
       `weights` is `None`.
   """
-  if dim is None:
-    raise ValueError("`dim` cannot be None.")
+  if dim is not None:
+    if axis is not None:
+      raise ValueError("Cannot specify both 'axis' and 'dim'")
+    axis = dim
+  if axis is None and dim is None:
+    raise ValueError("You must specify 'axis'.")
   with ops.name_scope(scope, "cosine_distance_loss",
                       [predictions, labels, weights]) as scope:
     predictions.get_shape().assert_is_compatible_with(labels.get_shape())
@@ -655,5 +662,5 @@ def cosine_distance(
     labels = math_ops.to_float(labels)
 
     radial_diffs = math_ops.multiply(predictions, labels)
-    losses = 1 - math_ops.reduce_sum(radial_diffs, reduction_indices=[dim,])
+    losses = 1 - math_ops.reduce_sum(radial_diffs, reduction_indices=[axis,])
     return compute_weighted_loss(losses, weights, scope=scope)
diff --git a/tensorflow/contrib/makefile/Makefile b/tensorflow/contrib/makefile/Makefile
index 81024c26a4..b582493131 100644
--- a/tensorflow/contrib/makefile/Makefile
+++ b/tensorflow/contrib/makefile/Makefile
@@ -194,6 +194,10 @@ LIBFLAGS :=
 
 # If we're on OS X, make sure that globals aren't stripped out.
 ifeq ($(TARGET),OSX)
+ifeq ($(HAS_GEN_HOST_PROTOC),true)
+	LIBFLAGS += -L$(MAKEFILE_DIR)/gen/protobuf-host/lib
+	export LD_LIBRARY_PATH=$(MAKEFILE_DIR)/gen/protobuf-host/lib
+endif
 	LDFLAGS += -all_load
 endif
 # Make sure that we don't strip global constructors on Linux.
diff --git a/tensorflow/contrib/makefile/download_dependencies.sh b/tensorflow/contrib/makefile/download_dependencies.sh
index f0b9658e3d..12e3f58930 100755
--- a/tensorflow/contrib/makefile/download_dependencies.sh
+++ b/tensorflow/contrib/makefile/download_dependencies.sh
@@ -54,7 +54,7 @@ download_and_extract() {
   elif [[ "${url}" == *zip ]]; then
     tempdir=$(mktemp -d)
     tempdir2=$(mktemp -d)
-    wget ${url} -P ${tempdir}
+    wget -P ${tempdir} ${url}
     unzip ${tempdir}/* -d ${tempdir2}
     # unzip has no strip components, so unzip to a temp dir, and move the files
     # we want from the tempdir to destination.
diff --git a/tensorflow/contrib/makefile/tf_op_files.txt b/tensorflow/contrib/makefile/tf_op_files.txt
index a8690a04ad..8b77c99cb5 100644
--- a/tensorflow/contrib/makefile/tf_op_files.txt
+++ b/tensorflow/contrib/makefile/tf_op_files.txt
@@ -264,3 +264,4 @@ tensorflow/core/kernels/spacetobatch_functor.cc
 tensorflow/core/kernels/spacetobatch_op.cc
 tensorflow/core/kernels/batchtospace_op.cc
 tensorflow/core/kernels/warn_about_ints.cc
+tensorflow/core/kernels/segment_reduction_ops.cc
diff --git a/tensorflow/contrib/meta_graph_transform/meta_graph_transform.py b/tensorflow/contrib/meta_graph_transform/meta_graph_transform.py
index 303c02dfa4..2932ae1c8d 100644
--- a/tensorflow/contrib/meta_graph_transform/meta_graph_transform.py
+++ b/tensorflow/contrib/meta_graph_transform/meta_graph_transform.py
@@ -749,7 +749,7 @@ def meta_graph_transform(
         base_meta_graph_def, meta_graph_def, collection_name,
         removed_op_names)
 
-  # Append newly added initalizers to collection.
+  # Append newly added initializers to collection.
   _add_new_inits_to_collection(meta_graph_def, updated_initializer_names)
 
   # Copy signature_defs, excluding any pruned nodes
diff --git a/tensorflow/contrib/metrics/python/ops/metric_ops_test.py b/tensorflow/contrib/metrics/python/ops/metric_ops_test.py
index f24bec7f11..6e038481e3 100644
--- a/tensorflow/contrib/metrics/python/ops/metric_ops_test.py
+++ b/tensorflow/contrib/metrics/python/ops/metric_ops_test.py
@@ -5856,7 +5856,7 @@ class StreamingMeanIOUTest(test.TestCase):
       sess.run(variables.local_variables_initializer())
       for _ in range(5):
         sess.run(update_op)
-      desired_output = np.mean([1.0 / 3.0, 2.0 / 4.0, 0.])
+      desired_output = np.mean([1.0 / 3.0, 2.0 / 4.0])
       self.assertAlmostEqual(desired_output, miou.eval())
 
   def testUpdateOpEvalIsAccumulatedConfusionMatrix(self):
@@ -5938,6 +5938,58 @@ class StreamingMeanIOUTest(test.TestCase):
       desired_miou = np.mean([2. / 4., 4. / 6.])
       self.assertAlmostEqual(desired_miou, miou.eval())
 
+  def testMissingClassInLabels(self):
+    labels = constant_op.constant([
+      [[0, 0, 1, 1, 0, 0],
+       [1, 0, 0, 0, 0, 1]],
+      [[1, 1, 1, 1, 1, 1],
+       [0, 0, 0, 0, 0, 0]]])
+    predictions = constant_op.constant([
+      [[0, 0, 2, 1, 1, 0],
+       [0, 1, 2, 2, 0, 1]],
+      [[0, 0, 2, 1, 1, 1],
+       [1, 1, 2, 0, 0, 0]]])
+    num_classes = 3
+    with self.test_session() as sess:
+      miou, update_op = metrics.streaming_mean_iou(
+          predictions, labels, num_classes)
+      sess.run(variables.local_variables_initializer())
+      self.assertAllEqual([[7, 4, 3], [3, 5, 2], [0, 0, 0]], update_op.eval())
+      self.assertAlmostEqual(
+        1 / 3 * (7 / (7 + 3 + 7) + 5 / (5 + 4 + 5) + 0 / (0 + 5 + 0)),
+        miou.eval())
+
+  def testMissingClassOverallSmall(self):
+    labels = constant_op.constant([0])
+    predictions = constant_op.constant([0])
+    num_classes = 2
+    with self.test_session() as sess:
+      miou, update_op = metrics.streaming_mean_iou(
+          predictions, labels, num_classes)
+      sess.run(variables.local_variables_initializer())
+      self.assertAllEqual([[1, 0], [0, 0]], update_op.eval())
+      self.assertAlmostEqual(1, miou.eval())
+
+  def testMissingClassOverallLarge(self):
+    labels = constant_op.constant([
+      [[0, 0, 1, 1, 0, 0],
+       [1, 0, 0, 0, 0, 1]],
+      [[1, 1, 1, 1, 1, 1],
+       [0, 0, 0, 0, 0, 0]]])
+    predictions = constant_op.constant([
+      [[0, 0, 1, 1, 0, 0],
+       [1, 1, 0, 0, 1, 1]],
+      [[0, 0, 0, 1, 1, 1],
+       [1, 1, 1, 0, 0, 0]]])
+    num_classes = 3
+    with self.test_session() as sess:
+      miou, update_op = metrics.streaming_mean_iou(
+          predictions, labels, num_classes)
+      sess.run(variables.local_variables_initializer())
+      self.assertAllEqual([[9, 5, 0], [3, 7, 0], [0, 0, 0]], update_op.eval())
+      self.assertAlmostEqual(
+        1 / 2 * (9 / (9 + 3 + 5) + 7 / (7 + 5 + 3)), miou.eval())
+
 
 class StreamingConcatTest(test.TestCase):
 
diff --git a/tensorflow/contrib/mpi_collectives/__init__.py b/tensorflow/contrib/mpi_collectives/__init__.py
index b94f7b0a35..9ed16a6f07 100644
--- a/tensorflow/contrib/mpi_collectives/__init__.py
+++ b/tensorflow/contrib/mpi_collectives/__init__.py
@@ -194,7 +194,7 @@ class DistributedOptimizer(tf.train.Optimizer):
 
     See Optimizer.compute_gradients() for more info.
 
-    In DistributedOptimizer, compute_gradients() is overriden to also
+    In DistributedOptimizer, compute_gradients() is overridden to also
     allreduce the gradients before returning them.
     """
     gradients = (super(DistributedOptimizer, self)
diff --git a/tensorflow/contrib/nn/__init__.py b/tensorflow/contrib/nn/__init__.py
index 7007e26bac..3bf795d19a 100644
--- a/tensorflow/contrib/nn/__init__.py
+++ b/tensorflow/contrib/nn/__init__.py
@@ -18,6 +18,7 @@
 @@deprecated_flipped_softmax_cross_entropy_with_logits
 @@deprecated_flipped_sparse_softmax_cross_entropy_with_logits
 @@deprecated_flipped_sigmoid_cross_entropy_with_logits
+@@nth_element
 @@rank_sampled_softmax_loss
 @@scaled_softplus
 """
@@ -31,6 +32,7 @@ from tensorflow.contrib.nn.python.ops.alpha_dropout import *
 from tensorflow.contrib.nn.python.ops.cross_entropy import *
 from tensorflow.contrib.nn.python.ops.sampling_ops import *
 from tensorflow.contrib.nn.python.ops.scaled_softplus import *
+from tensorflow.python.ops.nn_ops import nth_element
 # pylint: enable=unused-import,wildcard-import
 
 from tensorflow.python.util.all_util import remove_undocumented
diff --git a/tensorflow/contrib/receptive_field/python/util/receptive_field.py b/tensorflow/contrib/receptive_field/python/util/receptive_field.py
index db190a1a41..8b34465d21 100644
--- a/tensorflow/contrib/receptive_field/python/util/receptive_field.py
+++ b/tensorflow/contrib/receptive_field/python/util/receptive_field.py
@@ -27,13 +27,15 @@ import math
 from tensorflow.contrib.receptive_field.python.util import graph_compute_order
 from tensorflow.contrib.util import make_ndarray
 from tensorflow.python.platform import tf_logging as logging
+from tensorflow.python.framework import ops as framework_ops
+import numpy as np
 
 # White-listed layer operations, which do not affect the receptive field
 # computation.
 _UNCHANGED_RF_LAYER_OPS = [
-    "Softplus", "Relu", "BiasAdd", "Mul", "Add", "Const", "Identity",
-    "VariableV2", "Sub", "Rsqrt", "ConcatV2"
-]
+  'Add', 'BiasAdd', 'Ceil', 'ConcatV2', 'Const', 'Floor', 'Identity', 'Log',
+  'Mul', 'Pow', 'RealDiv', 'Relu', 'Round', 'Rsqrt', 'Softplus', 'Sub',
+  'VariableV2']
 
 # Different ways in which padding modes may be spelled.
 _VALID_PADDING = ["VALID", b"VALID"]
@@ -238,7 +240,8 @@ def _get_layer_params(node, name_to_order_node):
     padding_x = 0
     padding_y = 0
   else:
-    raise ValueError("Unknown layer op: %s" % node.op)
+    raise ValueError("Unknown layer for operation '%s': %s" %
+                     (node.name, node.op))
   return kernel_size_x, kernel_size_y, stride_x, stride_y, padding_x, padding_y
 
 
@@ -304,13 +307,103 @@ def _get_effective_padding_node_input(stride, padding,
   return stride * effective_padding_output + padding
 
 
-def compute_receptive_field_from_graph_def(graph_def, input_node, output_node):
-  """Computes receptive field (RF) parameters from a GraphDef object.
+class ReceptiveField:
+  """
+  Receptive field of a convolutional neural network.
+
+  Args:
+    size: Receptive field size.
+    stride: Effective stride.
+    padding: Effective padding.
+  """
+  def __init__(self, size, stride, padding):
+    self.size = np.asarray(size)
+    self.stride = np.asarray(stride)
+    self.padding = np.asarray(padding)
+
+  def compute_input_center_coordinates(self, y, axis=None):
+    """
+    Computes the center of the receptive field that generated a feature.
+
+    Args:
+      y: An array of feature coordinates with shape `(..., d)`, where `d` is the
+        number of dimensions of the coordinates.
+      axis: The dimensions for which to compute the input center coordinates.
+        If `None` (the default), compute the input center coordinates for all
+        dimensions.
+
+    Returns:
+      x: Center of the receptive field that generated the features, at the input
+        of the network.
+
+    Raises:
+      ValueError: If the number of dimensions of the feature coordinates does
+        not match the number of elements in `axis`.
+    """
+    # Use all dimensions.
+    if axis is None:
+      axis = range(self.size.size)
+    # Ensure axis is a list because tuples have different indexing behavior.
+    axis = list(axis)
+    y = np.asarray(y)
+    if y.shape[-1] != len(axis):
+      raise ValueError("Dimensionality of the feature coordinates `y` (%d) "
+                       "does not match dimensionality of `axis` (%d)" %
+                       (y.shape[-1], len(axis)))
+    return - self.padding[axis] + y * self.stride[axis] + \
+      (self.size[axis] - 1) / 2
+
+  def compute_feature_coordinates(self, x, axis=None):
+    """
+    Computes the position of a feature given the center of a receptive field.
+
+    Args:
+      x: An array of input center coordinates with shape `(..., d)`, where `d`
+        is the number of dimensions of the coordinates.
+      axis: The dimensions for which to compute the feature coordinates.
+        If `None` (the default), compute the feature coordinates for all
+        dimensions.
+
+    Returns:
+      y: Coordinates of the features.
+
+    Raises:
+      ValueError: If the number of dimensions of the input center coordinates
+        does not match the number of elements in `axis`.
+    """
+    # Use all dimensions.
+    if axis is None:
+      axis = range(self.size.size)
+    # Ensure axis is a list because tuples have different indexing behavior.
+    axis = list(axis)
+    x = np.asarray(x)
+    if x.shape[-1] != len(axis):
+      raise ValueError("Dimensionality of the input center coordinates `x` "
+                       "(%d) does not match dimensionality of `axis` (%d)" %
+                       (x.shape[-1], len(axis)))
+    return (x + self.padding[axis] + (1 - self.size[axis]) / 2) / \
+      self.stride[axis]
+
+  def __iter__(self):
+    return iter(np.concatenate([self.size, self.stride, self.padding]))
+
+
+def compute_receptive_field_from_graph_def(graph_def, input_node, output_node,
+                                           stop_propagation=None):
+  """Computes receptive field (RF) parameters from a Graph or GraphDef object.
+
+  The algorithm stops the calculation of the receptive field whenever it
+  encounters an operation in the list `stop_propagation`. Stopping the
+  calculation early can be useful to calculate the receptive field of a
+  subgraph such as a single branch of the
+  [inception network](https://arxiv.org/abs/1512.00567).
 
   Args:
-    graph_def: GraphDef object.
-    input_node: Name of the input node from graph.
-    output_node: Name of the output node from graph.
+    graph_def: Graph or GraphDef object.
+    input_node: Name of the input node or Tensor object from graph.
+    output_node: Name of the output node or Tensor object from graph.
+    stop_propagation: List of operation or scope names for which to stop the
+      propagation of the receptive field.
 
   Returns:
     rf_size_x: Receptive field size of network in the horizontal direction, with
@@ -331,6 +424,18 @@ def compute_receptive_field_from_graph_def(graph_def, input_node, output_node):
       cannot be found. For network criterion alignment, see
       photos/vision/features/delf/g3doc/rf_computation.md
   """
+  # Convert a graph to graph_def if necessary.
+  if isinstance(graph_def, framework_ops.Graph):
+    graph_def = graph_def.as_graph_def()
+
+  # Convert tensors to names.
+  if isinstance(input_node, framework_ops.Tensor):
+    input_node = input_node.op.name
+  if isinstance(output_node, framework_ops.Tensor):
+    output_node = output_node.op.name
+
+  stop_propagation = stop_propagation or []
+
   # Computes order of computation for a given graph.
   name_to_order_node = graph_compute_order.get_compute_order(
       graph_def=graph_def)
@@ -422,6 +527,10 @@ def compute_receptive_field_from_graph_def(graph_def, input_node, output_node):
 
       # Loop over this node's inputs and potentially propagate information down.
       for inp_name in node.input:
+        # Stop the propagation of the receptive field.
+        if any(inp_name.startswith(stop) for stop in stop_propagation):
+          logging.vlog(3, "Skipping explicitly ignored node %s.", node.name)
+          continue
         logging.vlog(4, "inp_name = %s", inp_name)
         inp_node = name_to_order_node[inp_name].node
         logging.vlog(4, "inp_node = \n%s", inp_node)
@@ -480,6 +589,7 @@ def compute_receptive_field_from_graph_def(graph_def, input_node, output_node):
     raise ValueError("Output node was not found")
   if input_node not in rf_sizes_x:
     raise ValueError("Input node was not found")
-  return (rf_sizes_x[input_node], rf_sizes_y[input_node],
-          effective_strides_x[input_node], effective_strides_y[input_node],
-          effective_paddings_x[input_node], effective_paddings_y[input_node])
+  return ReceptiveField(
+    (rf_sizes_x[input_node], rf_sizes_y[input_node]),
+    (effective_strides_x[input_node], effective_strides_y[input_node]),
+    (effective_paddings_x[input_node], effective_paddings_y[input_node]))
diff --git a/tensorflow/contrib/receptive_field/python/util/receptive_field_test.py b/tensorflow/contrib/receptive_field/python/util/receptive_field_test.py
index 2771389250..8d7d5440f6 100644
--- a/tensorflow/contrib/receptive_field/python/util/receptive_field_test.py
+++ b/tensorflow/contrib/receptive_field/python/util/receptive_field_test.py
@@ -25,6 +25,7 @@ from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import nn
 from tensorflow.python.platform import test
+import numpy as np
 
 
 def create_test_network_1():
@@ -150,6 +151,31 @@ def create_test_network_5():
   return g
 
 
+def create_test_network_6():
+  """Aligned network with dropout for test.
+
+  The graph is similar to create_test_network_1(), except that the right branch
+  has dropout normalization.
+
+  Returns:
+    g: Tensorflow graph object (Graph proto).
+  """
+  g = ops.Graph()
+  with g.as_default():
+    # An 8x8 test image.
+    x = array_ops.placeholder(dtypes.float32, (1, 8, 8, 1), name='input_image')
+    # Left branch.
+    l1 = slim.conv2d(x, 1, [1, 1], stride=4, scope='L1', padding='VALID')
+    # Right branch.
+    l2_pad = array_ops.pad(x, [[0, 0], [1, 0], [1, 0], [0, 0]])
+    l2 = slim.conv2d(l2_pad, 1, [3, 3], stride=2, scope='L2', padding='VALID')
+    l3 = slim.conv2d(l2, 1, [1, 1], stride=2, scope='L3', padding='VALID')
+    dropout = slim.dropout(l3)
+    # Addition.
+    nn.relu(l1 + dropout, name='output')
+  return g
+
+
 class RfUtilsTest(test.TestCase):
 
   def testComputeRFFromGraphDefAligned(self):
@@ -220,6 +246,36 @@ class RfUtilsTest(test.TestCase):
     self.assertEqual(effective_padding_x, 0)
     self.assertEqual(effective_padding_y, 0)
 
+  def testComputeRFFromGraphDefStopPropagation(self):
+    graph_def = create_test_network_6().as_graph_def()
+    input_node = 'input_image'
+    output_node = 'output'
+    # Compute the receptive field but stop the propagation for the random
+    # uniform variable of the dropout.
+    (receptive_field_x, receptive_field_y, effective_stride_x,
+     effective_stride_y, effective_padding_x, effective_padding_y) = (
+         receptive_field.compute_receptive_field_from_graph_def(
+             graph_def, input_node, output_node,
+             ['Dropout/dropout/random_uniform']))
+    self.assertEqual(receptive_field_x, 3)
+    self.assertEqual(receptive_field_y, 3)
+    self.assertEqual(effective_stride_x, 4)
+    self.assertEqual(effective_stride_y, 4)
+    self.assertEqual(effective_padding_x, 1)
+    self.assertEqual(effective_padding_y, 1)
+
+  def testComputeCoordinatesRoundtrip(self):
+    graph_def = create_test_network_1()
+    input_node = 'input_image'
+    output_node = 'output'
+    rf = receptive_field.compute_receptive_field_from_graph_def(
+      graph_def, input_node, output_node)
+
+    x = np.random.randint(0, 100, (50, 2))
+    y = rf.compute_feature_coordinates(x)
+    x2 = rf.compute_input_center_coordinates(y)
+
+    self.assertAllEqual(x, x2)
 
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/contrib/stateless/python/kernel_tests/stateless_random_ops_test.py b/tensorflow/contrib/stateless/python/kernel_tests/stateless_random_ops_test.py
index 9a36bdc2f9..cd4d46aa07 100644
--- a/tensorflow/contrib/stateless/python/kernel_tests/stateless_random_ops_test.py
+++ b/tensorflow/contrib/stateless/python/kernel_tests/stateless_random_ops_test.py
@@ -20,6 +20,7 @@ from __future__ import print_function
 
 import numpy as np
 from tensorflow.contrib import stateless
+from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import random_seed
 from tensorflow.python.ops import array_ops
@@ -79,6 +80,21 @@ class StatelessOpsTest(test.TestCase):
             for s1, v1 in values:
               self.assertEqual(s0 == s1, np.all(v0 == v1))
 
+  def testShapeType(self):
+    with self.test_session(use_gpu=True):
+      for shape_dtype in [dtypes.int32, dtypes.int64]:
+        seed_t = array_ops.placeholder(dtypes.int64, shape=[2])
+        seeds = [(x, y) for x in range(5) for y in range(5)] * 3
+        for stateless_op, _ in CASES:
+          for shape in (), (3,), (2, 5):
+            pure = stateless_op(constant_op.constant(shape, dtype=shape_dtype),
+                                seed=seed_t)
+            values = [(seed, pure.eval(feed_dict={seed_t: seed}))
+                      for seed in seeds]
+            for s0, v0 in values:
+              for s1, v1 in values:
+                self.assertEqual(s0 == s1, np.all(v0 == v1))
+
 
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index c4f880da9d..1c58aa3315 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -783,6 +783,7 @@ cc_library(
         "//tensorflow/core/kernels:dataset_ops",
         "//tensorflow/core/kernels:fake_quant_ops",
         "//tensorflow/core/kernels:function_ops",
+        "//tensorflow/core/kernels:histogram_op",
         "//tensorflow/core/kernels:image",
         "//tensorflow/core/kernels:io",
         "//tensorflow/core/kernels:linalg",
@@ -1943,6 +1944,7 @@ CORE_CPU_LIB_HEADERS = CORE_CPU_BASE_HDRS + [
 tf_cuda_library(
     name = "core_cpu_impl",
     srcs = [
+        "common_runtime/accumulate_n_optimizer.cc",
         "common_runtime/allocator_retry.cc",
         "common_runtime/bfc_allocator.cc",
         "common_runtime/build_graph_options.cc",
@@ -2178,6 +2180,7 @@ tf_cuda_library(
         ":lib",
         ":lib_internal",
         ":protos_all_cc",
+        ":stream_executor",
         "//third_party/eigen3",
     ] + if_static([":gpu_runtime_impl"]),
 )
@@ -2673,6 +2676,22 @@ tf_cc_tests(
     ],
 )
 
+tf_cc_test_mkl(
+    name = "mkl_runtime_tests",
+    size = "small",
+    srcs = ["common_runtime/mkl_cpu_allocator_test.cc"],
+    linkstatic = 1,
+    deps = [
+        ":core",
+        ":core_cpu",
+        ":framework",
+        ":framework_internal",
+        ":test",
+        ":test_main",
+        ":testlib",
+    ],
+)
+
 tf_cc_test_mkl(
     name = "mkl_related_tests",
     size = "small",
@@ -2700,7 +2719,20 @@ tf_cc_test_mkl(
         "//tensorflow/cc:sendrecv_ops",
         "//tensorflow/core/kernels:ops_util",
         "//third_party/eigen3",
-    ],
+    ] + if_mkl([
+        "//tensorflow/core/kernels:mkl_aggregate_ops",
+        "//tensorflow/core/kernels:mkl_concat_op",
+        "//tensorflow/core/kernels:mkl_conv_op",
+        "//tensorflow/core/kernels:mkl_cwise_ops_common",
+        "//tensorflow/core/kernels:mkl_fused_batch_norm_op",
+        "//tensorflow/core/kernels:mkl_identity_op",
+        "//tensorflow/core/kernels:mkl_input_conversion_op",
+        "//tensorflow/core/kernels:mkl_lrn_op",
+        "//tensorflow/core/kernels:mkl_pooling_ops",
+        "//tensorflow/core/kernels:mkl_relu_op",
+        "//tensorflow/core/kernels:mkl_reshape_op",
+        "//tensorflow/core/kernels:mkl_tfconv_op",
+    ]),
 )
 
 tf_cc_tests_gpu(
diff --git a/tensorflow/core/common_runtime/accumulate_n_optimizer.cc b/tensorflow/core/common_runtime/accumulate_n_optimizer.cc
new file mode 100644
index 0000000000..81cd44870e
--- /dev/null
+++ b/tensorflow/core/common_runtime/accumulate_n_optimizer.cc
@@ -0,0 +1,191 @@
+/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+
+#include "tensorflow/core/common_runtime/optimization_registry.h"
+#include "tensorflow/core/graph/node_builder.h"
+
+
+namespace tensorflow {
+namespace {
+
+Tensor make_zeros(const DataType& dtype, const TensorShapeProto& shape) {
+  Tensor tensor(dtype, TensorShape(shape));
+
+  // Conveniently, all numeric data types have 0x0 == zero.  Otherwise we would
+  // need a giant switch statement here.
+  memset(const_cast<char*>(tensor.tensor_data().data()), 0,
+         tensor.tensor_data().size());
+
+  return tensor;
+}
+
+// Replaces occurrences of the "AccumulateNV2" stub operator with a graph of
+// lower-level ops. The graph is equivalent (modulo certain corner cases)
+// to the semantics of the original accumulate_n() Python op in math_ops.py.
+// Implementing the op with a rewrite allows this new variant of accumulate_n 
+// to be differentiable.
+//
+// The binary code that generates AccumulateNV2 stub ops is located in a
+// dynamic library built out of tensorflow/contrib/framework. Ideally, this
+// class would also be in contrib, but calls to REGISTER_OPTIMIZATION() from
+// third-party libraries aren't currently supported.
+class AccumulateNV2RemovePass : public GraphOptimizationPass {
+ public:
+
+  Status Run(const GraphOptimizationPassOptions& options) override {
+    // TODO(freiss.oss@gmail.com): Substantial shared code with
+    // ParallelConcatRemovePass::Run(). Consider refactoring if someone makes
+    // a third similar rewrite.
+    if (options.graph == nullptr) {
+      // TODO(apassos) returning OK feels weird here as we can't do anything
+      // without a graph, but some tests require this.
+      return Status::OK();
+    }
+
+    Graph* g = options.graph->get();
+    if (g == nullptr) {
+      return errors::Internal(
+          "AccumulateNV2 removal should happen before partitioning and a "
+          "graph should be available.");
+    }
+
+    // Build up a todo list of ops to replace, *then* modify the graph
+    gtl::InlinedVector<Node*, 2> matches;
+    for (Node* n : g->op_nodes()) {
+      if (n->type_string() == "AccumulateNV2") {
+        matches.push_back(n);
+      }
+    }
+    for (Node* n : matches) {
+      TF_RETURN_IF_ERROR(rewriteNode(n, g));
+    }
+    return Status::OK();
+  }
+
+  Status rewriteNode(Node* n, Graph* g) {
+    AttrSlice n_attrs = n->attrs();
+    auto base_make_node = [n, g, &n_attrs](const string& op,
+                                           const string& name) {
+      NodeBuilder node_builder(name, op);
+
+      // The pieces of AccumulateNV2 should all be on the same node.
+      node_builder.Device(n->requested_device());
+      string colo;
+      if (GetNodeAttr(n_attrs, kColocationAttrName, &colo).ok()) {
+        node_builder.Attr(kColocationAttrName, colo);
+      }
+      return node_builder;
+    };
+    auto make_node = [n, g, &n_attrs, &base_make_node](string op) {
+      return base_make_node(
+          op, g->NewName(strings::StrCat(n->name(), "/Internal")));
+    };
+
+    DataType dtype;
+    TF_RETURN_IF_ERROR(GetNodeAttr(n_attrs, "T", &dtype));
+    TensorShapeProto shape;
+    TF_RETURN_IF_ERROR(GetNodeAttr(n_attrs, "shape", &shape));
+
+    std::vector<const Edge*> data_edges, control_edges;
+    for (const Edge* input_edge : n->in_edges()) {
+      if (input_edge->IsControlEdge()) {
+        control_edges.push_back(input_edge);
+      } else {
+        data_edges.push_back(input_edge);
+      }
+    }
+
+    // Create the following ops to replace the AccumulateNV2 placeholder:
+    Node* create_accumulator = nullptr;            // TemporaryVariable op
+    Node* initial_val = nullptr;                   // Const op
+    Node* initialize_accumulator = nullptr;        // Assign op
+    std::vector<Node*> add_values_to_accumulator;  // AssignAdd ops
+    Node* clean_up_accumulator = nullptr;          // DestroyTemporaryVariable
+
+    const string accumulator_name =
+        strings::StrCat(n->name(), "/Internal/Accumulator");
+    TF_RETURN_IF_ERROR(make_node("TemporaryVariable")
+                           .Attr("shape", shape)
+                           .Attr("dtype", dtype)
+                           .Attr("var_name", accumulator_name)
+                           .Finalize(g, &create_accumulator));
+    TF_RETURN_IF_ERROR(make_node("Const")
+                           .Attr("value", make_zeros(dtype, shape))
+                           .Attr("dtype", dtype)
+                           .Finalize(g, &initial_val));
+    TF_RETURN_IF_ERROR(make_node("Assign")
+                           .Attr("T", dtype)
+                           .Input(create_accumulator)  // ref: Ref(T)
+                           .Input(initial_val)         // value: T
+                           .Finalize(g, &initialize_accumulator));
+    for (int i = 0; i < data_edges.size(); ++i) {
+      Node* assignAdd;
+      TF_RETURN_IF_ERROR(make_node("AssignAdd")
+                             .Attr("T", dtype)
+                             .Attr("use_locking", true)
+                             .Input(initialize_accumulator)  // ref: Ref(T)
+                             .Input(data_edges[i]->src(),
+                                    data_edges[i]->src_output())  // value: T
+                             .Finalize(g, &assignAdd));
+
+      add_values_to_accumulator.push_back(assignAdd);
+    }
+
+    // Note that we use the original placeholder op's name here
+    TF_RETURN_IF_ERROR(base_make_node("DestroyTemporaryVariable", n->name())
+                           .Attr("T", dtype)
+                           .Attr("var_name", accumulator_name)
+                           .Input(initialize_accumulator)
+                           .Finalize(g, &clean_up_accumulator));
+
+    // Add edges to the graph to ensure that operations occur in the right
+    // order:
+    // 1. Do anything that had a control edge to the AccumulateNV2 placeholder
+    // 2. Initialize accumulator
+    // 3. Add input values to accumulator (already handled by data edges
+    //    added above)
+    // 4. Reclaim the buffer that held the accumulator
+    // 5. Do anything that depended on the AccumulateNV2 placeholder
+    for (const Edge* control_edge : control_edges) {
+      g->AddControlEdge(control_edge->src(), initialize_accumulator);
+    }
+
+    for (Node* assign_add : add_values_to_accumulator) {
+      g->AddControlEdge(assign_add, clean_up_accumulator);
+    }
+
+    for (const Edge* out_edge : n->out_edges()) {
+      if (out_edge->IsControlEdge()) {
+        g->AddControlEdge(clean_up_accumulator, out_edge->dst());
+      } else {
+        g->AddEdge(clean_up_accumulator, 0, out_edge->dst(),
+                   out_edge->dst_input());
+      }
+    }
+
+    // Remove the original AccumulateNV2 placeholder op.
+    // This removal modifies the op and must happen after we have finished
+    // using its incoming/outgoing edge sets.
+    g->RemoveNode(n);
+
+    return Status::OK();
+  }
+};
+REGISTER_OPTIMIZATION(OptimizationPassRegistry::PRE_PLACEMENT, 0,
+                      AccumulateNV2RemovePass);
+
+}  // namespace
+}  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/mkl_cpu_allocator.h b/tensorflow/core/common_runtime/mkl_cpu_allocator.h
index f16da10d7a..53e80b1ee3 100644
--- a/tensorflow/core/common_runtime/mkl_cpu_allocator.h
+++ b/tensorflow/core/common_runtime/mkl_cpu_allocator.h
@@ -21,9 +21,13 @@ limitations under the License.
 
 #ifdef INTEL_MKL
 
+#include <unistd.h>
+#include <cstdlib>
 #include <string>
 #include "tensorflow/core/common_runtime/bfc_allocator.h"
 #include "tensorflow/core/framework/allocator.h"
+#include "tensorflow/core/lib/strings/numbers.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/mem.h"
 
 #include "i_malloc.h"
@@ -46,10 +50,50 @@ class MklCPUAllocator : public Allocator {
  public:
   // Constructor and other standard functions
 
-  MklCPUAllocator() {
+  /// Environment variable that user can set to upper bound on memory allocation
+  static constexpr const char* kMaxLimitStr = "TF_MKL_ALLOC_MAX_BYTES";
+
+  /// Default upper limit on allocator size - 64GB
+  static const size_t kDefaultMaxLimit = 64LL << 30;
+
+  MklCPUAllocator() { TF_CHECK_OK(Initialize()); }
+
+  ~MklCPUAllocator() override { delete allocator_; }
+
+  Status Initialize() {
     VLOG(2) << "MklCPUAllocator: In MklCPUAllocator";
-    allocator_ =
-        new BFCAllocator(new MklSubAllocator, kMaxMemSize, kAllowGrowth, kName);
+
+    // Set upper bound on memory allocation to physical RAM available on the
+    // CPU unless explicitly specified by user
+    uint64 max_mem_bytes = kDefaultMaxLimit;
+#if defined(_SC_PHYS_PAGES) && defined(_SC_PAGESIZE)
+    max_mem_bytes =
+        (uint64)sysconf(_SC_PHYS_PAGES) * (uint64)sysconf(_SC_PAGESIZE);
+#endif
+    char* user_mem_bytes = getenv(kMaxLimitStr);
+
+    if (user_mem_bytes != NULL) {
+      uint64 user_val = 0;
+      if (!strings::safe_strtou64(user_mem_bytes, &user_val)) {
+        return errors::InvalidArgument("Invalid memory limit (", user_mem_bytes,
+                                       ") specified for MKL allocator through ",
+                                       kMaxLimitStr);
+      }
+#if defined(_SC_PHYS_PAGES) && defined(_SC_PAGESIZE)
+      if (user_val > max_mem_bytes) {
+        LOG(WARNING) << "The user specifed a memory limit " << kMaxLimitStr
+                     << "=" << user_val
+                     << " greater than available physical memory: "
+                     << max_mem_bytes
+                     << ". This could significantly reduce performance!";
+      }
+#endif
+      max_mem_bytes = user_val;
+    }
+
+    VLOG(1) << "MklCPUAllocator: Setting max_mem_bytes: " << max_mem_bytes;
+    allocator_ = new BFCAllocator(new MklSubAllocator, max_mem_bytes,
+                                  kAllowGrowth, kName);
 
     // For redirecting all allocations from MKL to this allocator
     // From: http://software.intel.com/en-us/node/528565
@@ -57,9 +101,9 @@ class MklCPUAllocator : public Allocator {
     i_calloc = CallocHook;
     i_realloc = ReallocHook;
     i_free = FreeHook;
-  }
 
-  ~MklCPUAllocator() override { delete allocator_; }
+    return Status::OK();
+  }
 
   inline string Name() override { return kName; }
 
@@ -71,6 +115,8 @@ class MklCPUAllocator : public Allocator {
     allocator_->DeallocateRaw(ptr);
   }
 
+  void GetStats(AllocatorStats* stats) { return allocator_->GetStats(stats); }
+
  private:
   // Hooks provided by this allocator for memory allocation routines from MKL
 
@@ -96,11 +142,6 @@ class MklCPUAllocator : public Allocator {
     TF_CHECK_OK(s);  // way to assert with an error message
   }
 
-  // TODO(jbobba): We should ideally move this into CPUOptions in config.proto.
-  /// Memory limit - 64GB
-  static const size_t kMaxMemSize =
-      static_cast<size_t>(64) * 1024 * 1024 * 1024;
-
   /// Do we allow growth in BFC Allocator
   static const bool kAllowGrowth = true;
 
diff --git a/tensorflow/core/common_runtime/mkl_cpu_allocator_test.cc b/tensorflow/core/common_runtime/mkl_cpu_allocator_test.cc
new file mode 100644
index 0000000000..a67411cd2e
--- /dev/null
+++ b/tensorflow/core/common_runtime/mkl_cpu_allocator_test.cc
@@ -0,0 +1,53 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifdef INTEL_MKL
+
+#include "tensorflow/core/common_runtime/mkl_cpu_allocator.h"
+
+#include "tensorflow/core/lib/core/status_test_util.h"
+#include "tensorflow/core/platform/logging.h"
+#include "tensorflow/core/platform/test.h"
+
+namespace tensorflow {
+
+TEST(MKLBFCAllocatorTest, TestMaxLimit) {
+  AllocatorStats stats;
+  setenv(MklCPUAllocator::kMaxLimitStr, "1000", 1);
+  MklCPUAllocator a;
+  TF_EXPECT_OK(a.Initialize());
+  a.GetStats(&stats);
+  EXPECT_EQ(stats.bytes_limit, 1000);
+
+  unsetenv(MklCPUAllocator::kMaxLimitStr);
+  TF_EXPECT_OK(a.Initialize());
+  a.GetStats(&stats);
+  uint64 max_mem_bytes = MklCPUAllocator::kDefaultMaxLimit;
+#if defined(_SC_PHYS_PAGES) && defined(_SC_PAGESIZE)
+  max_mem_bytes =
+      (uint64)sysconf(_SC_PHYS_PAGES) * (uint64)sysconf(_SC_PAGESIZE);
+#endif
+  EXPECT_EQ(stats.bytes_limit, max_mem_bytes);
+
+  setenv(MklCPUAllocator::kMaxLimitStr, "wrong-input", 1);
+  EXPECT_TRUE(errors::IsInvalidArgument(a.Initialize()));
+
+  setenv(MklCPUAllocator::kMaxLimitStr, "-20", 1);
+  EXPECT_TRUE(errors::IsInvalidArgument(a.Initialize()));
+}
+
+}  // namespace tensorflow
+
+#endif  // INTEL_MKL
diff --git a/tensorflow/core/framework/common_shape_fns.cc b/tensorflow/core/framework/common_shape_fns.cc
index 4796c3c00a..315c99d32b 100644
--- a/tensorflow/core/framework/common_shape_fns.cc
+++ b/tensorflow/core/framework/common_shape_fns.cc
@@ -1020,6 +1020,29 @@ Status UnknownShape(shape_inference::InferenceContext* c) {
   return Status::OK();
 }
 
+template <typename T>
+Status ReductionShapeHelper(const Tensor* reduction_indices_t,
+                            const int32 input_rank,
+                            std::set<int64>& true_indices) {
+  auto reduction_indices = reduction_indices_t->flat<T>();
+  for (int i = 0; i < reduction_indices_t->NumElements(); ++i) {
+    const T reduction_index = reduction_indices(i);
+    if (reduction_index < -input_rank || reduction_index >= input_rank) {
+      return errors::InvalidArgument("Invalid reduction dimension ",
+                                     reduction_index, " for input with ",
+                                     input_rank, " dimensions.");
+    }
+
+    auto wrapped_index = reduction_index;
+    if (wrapped_index < 0) {
+      wrapped_index += input_rank;
+    }
+
+    true_indices.insert(wrapped_index);
+  }
+  return Status::OK();
+}
+
 Status ReductionShape(InferenceContext* c) {
   ShapeHandle input = c->input(0);
 
@@ -1050,22 +1073,16 @@ Status ReductionShape(InferenceContext* c) {
   }
 
   const int32 input_rank = c->Rank(input);
-  std::set<int32> true_indices;
-  auto reduction_indices = reduction_indices_t->flat<int32>();
-  for (int i = 0; i < reduction_indices_t->NumElements(); ++i) {
-    int32 reduction_index = reduction_indices(i);
-    if (reduction_index < -input_rank || reduction_index >= input_rank) {
-      return errors::InvalidArgument("Invalid reduction dimension ",
-                                     reduction_index, " for input with ",
-                                     input_rank, " dimensions.");
-    }
-
-    int32 wrapped_index = reduction_index;
-    if (wrapped_index < 0) {
-      wrapped_index += input_rank;
-    }
-
-    true_indices.insert(wrapped_index);
+  std::set<int64> true_indices;
+  if (reduction_indices_t->dtype() == DataType::DT_INT32) {
+    TF_RETURN_IF_ERROR(ReductionShapeHelper<int32>(reduction_indices_t,
+                                                   input_rank, true_indices));
+  } else if (reduction_indices_t->dtype() == DataType::DT_INT64) {
+    TF_RETURN_IF_ERROR(ReductionShapeHelper<int64>(reduction_indices_t,
+                                                   input_rank, true_indices));
+  } else {
+    return errors::InvalidArgument(
+        "reduction_indices can only be int32 or int64");
   }
 
   std::vector<DimensionHandle> dims;
@@ -1319,11 +1336,10 @@ Status ScatterNdUpdateShape(InferenceContext* c) {
       Status s = c->Merge(prefix_indices, prefix_updates, &unused);
       if (!s.ok()) {
         return errors::InvalidArgument(
-            "The outer ", num_outer_dims,
-            " dimensions of indices.shape=", c->DebugString(indices_shape),
-            " must match the outer ", num_outer_dims,
-            " dimensions of updates.shape=", c->DebugString(updates_shape),
-            ": ", s.error_message());
+            "The outer ", num_outer_dims, " dimensions of indices.shape=",
+            c->DebugString(indices_shape), " must match the outer ",
+            num_outer_dims, " dimensions of updates.shape=",
+            c->DebugString(updates_shape), ": ", s.error_message());
       }
 
       ShapeHandle input_suffix;
diff --git a/tensorflow/core/framework/node_def.proto b/tensorflow/core/framework/node_def.proto
index 53aa03108a..1fd2e50b51 100644
--- a/tensorflow/core/framework/node_def.proto
+++ b/tensorflow/core/framework/node_def.proto
@@ -35,7 +35,7 @@ message NodeDef {
   // CONSTRAINT ::= ("job:" JOB_NAME)
   //              | ("replica:" [1-9][0-9]*)
   //              | ("task:" [1-9][0-9]*)
-  //              | ( ("gpu" | "cpu") ":" ([1-9][0-9]* | "*") )
+  //              | ("device:" ("gpu" | "cpu") ":" ([1-9][0-9]* | "*") )
   //
   // Valid values for this string include:
   // * "/job:worker/replica:0/task:1/device:GPU:3"  (full specification)
diff --git a/tensorflow/core/framework/register_types.h b/tensorflow/core/framework/register_types.h
index 61e722e57b..c31ab18cc1 100644
--- a/tensorflow/core/framework/register_types.h
+++ b/tensorflow/core/framework/register_types.h
@@ -87,7 +87,7 @@ limitations under the License.
 
 #elif defined(__ANDROID_TYPES_FULL__)
 
-// Only half, float, int32, int64, and quantized types are supported.
+// Only half, float, int32, int64, bool, and quantized types are supported.
 #define TF_CALL_float(m) m(float)
 #define TF_CALL_double(m)
 #define TF_CALL_int32(m) m(::tensorflow::int32)
@@ -117,7 +117,7 @@ limitations under the License.
 
 #else  // defined(IS_MOBILE_PLATFORM) && !defined(__ANDROID_TYPES_FULL__)
 
-// Only float and int32 are supported.
+// Only float, int32, and bool are supported.
 #define TF_CALL_float(m) m(float)
 #define TF_CALL_double(m)
 #define TF_CALL_int32(m) m(::tensorflow::int32)
diff --git a/tensorflow/core/framework/rendezvous.cc b/tensorflow/core/framework/rendezvous.cc
index 90426defa0..a9e4c1cfb1 100644
--- a/tensorflow/core/framework/rendezvous.cc
+++ b/tensorflow/core/framework/rendezvous.cc
@@ -210,7 +210,7 @@ class LocalRendezvousImpl : public Rendezvous {
     ItemQueue* queue = &table_[key_hash];
     if (queue->empty() || !queue->front()->IsSendValue()) {
       // There is no message to pick up.
-      // Only recv-related fileds need to be filled.
+      // Only recv-related fields need to be filled.
       Item* item = new Item;
       item->waiter = std::move(done);
       item->recv_args = recv_args;
diff --git a/tensorflow/core/graph/graph.h b/tensorflow/core/graph/graph.h
index 7c7f641265..c5dde722fa 100644
--- a/tensorflow/core/graph/graph.h
+++ b/tensorflow/core/graph/graph.h
@@ -639,7 +639,7 @@ class Graph {
   std::unordered_map<string, int> device_names_map_;
 
   // All the while contexts owned by this graph, keyed by frame name,
-  // corresonding to all the while loops contained in this graph (including
+  // corresponding to all the while loops contained in this graph (including
   // nested loops). The stored contexts are usually accessed via
   // AddWhileContext() or Node::while_ctx(), but this manages the lifetime.
   std::map<string, WhileContext> while_ctxs_;
diff --git a/tensorflow/core/graph/mkl_layout_pass.cc b/tensorflow/core/graph/mkl_layout_pass.cc
index f87a94a76a..f4c9073dee 100644
--- a/tensorflow/core/graph/mkl_layout_pass.cc
+++ b/tensorflow/core/graph/mkl_layout_pass.cc
@@ -543,7 +543,7 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
     string reason;
 
     // Substring that should be checked for in device name for CPU device.
-    const char* const kCPUDeviceSubStr = "cpu";
+    const char* const kCPUDeviceSubStr = "CPU";
 
     // If Op has been specifically assigned to a non-CPU device, then No.
     if (!n->assigned_device_name().empty() &&
diff --git a/tensorflow/core/graph/mkl_layout_pass_test.cc b/tensorflow/core/graph/mkl_layout_pass_test.cc
index a2b2f6530d..abc63e4f35 100644
--- a/tensorflow/core/graph/mkl_layout_pass_test.cc
+++ b/tensorflow/core/graph/mkl_layout_pass_test.cc
@@ -39,7 +39,7 @@ limitations under the License.
 namespace tensorflow {
 namespace {
 
-const char kCPUDevice[] = "/job:a/replica:0/task:0/cpu:0";
+const char kCPUDevice[] = "/job:a/replica:0/task:0/device:CPU:0";
 const char kGPUDevice[] = "/job:a/replica:0/task:0/device:GPU:0";
 
 static void InitGraph(const string& s, Graph* graph,
diff --git a/tensorflow/core/graph/testlib.cc b/tensorflow/core/graph/testlib.cc
index be52438747..172471e34b 100644
--- a/tensorflow/core/graph/testlib.cc
+++ b/tensorflow/core/graph/testlib.cc
@@ -480,6 +480,24 @@ Node* Conv2D(Graph* g, Node* in0, Node* in1) {
   return ret;
 }
 
+Node* Diag(Graph* g, Node* in, DataType type) {
+  Node* ret;
+  TF_CHECK_OK(NodeBuilder(g->NewName("n"), "Diag")
+                  .Input(in)
+                  .Attr("T", type)
+                  .Finalize(g, &ret));
+  return ret;
+}
+
+Node* DiagPart(Graph* g, Node* in, DataType type) {
+  Node* ret;
+  TF_CHECK_OK(NodeBuilder(g->NewName("n"), "DiagPart")
+                  .Input(in)
+                  .Attr("T", type)
+                  .Finalize(g, &ret));
+  return ret;
+}
+
 void ToGraphDef(Graph* g, GraphDef* gdef) { g->ToGraphDef(gdef); }
 
 }  // end namespace graph
diff --git a/tensorflow/core/graph/testlib.h b/tensorflow/core/graph/testlib.h
index a38809e6b4..06597778bb 100644
--- a/tensorflow/core/graph/testlib.h
+++ b/tensorflow/core/graph/testlib.h
@@ -199,6 +199,12 @@ Node* BiasAdd(Graph* g, Node* value, Node* bias);
 // Add a Conv2D node in "g".
 Node* Conv2D(Graph* g, Node* in0, Node* in1);
 
+// Add a Diag node in "g".
+Node* Diag(Graph* g, Node* in, DataType type);
+
+// Add a DiagPart node in "g".
+Node* DiagPart(Graph* g, Node* in, DataType type);
+
 }  // end namespace graph
 }  // end namespace test
 }  // end namespace tensorflow
diff --git a/tensorflow/core/grappler/optimizers/model_pruner.cc b/tensorflow/core/grappler/optimizers/model_pruner.cc
index e087621c3b..b9df196f83 100644
--- a/tensorflow/core/grappler/optimizers/model_pruner.cc
+++ b/tensorflow/core/grappler/optimizers/model_pruner.cc
@@ -104,7 +104,7 @@ Status ModelPruner::Optimize(Cluster* cluster, const GrapplerItem& item,
     // - Don't remove nodes that receive reference values, as those can be
     //   converting references to non-references. It is important to preserve
     //   these non-references since the partitioner will avoid sending
-    //   non-references accross partitions more than once.
+    //   non-references across partitions more than once.
     if (!rewriter.DrivesControlDependency(node) &&
         !rewriter.IsDrivenByControlDependency(node) &&
         !rewriter.IsConnectedToFunction(node) &&
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index f5bfa60199..92a0dbd0ab 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -2499,6 +2499,7 @@ cc_library(
         ":cross_op",
         ":cwise_op",
         ":fft_ops",
+        ":histogram_op",
         ":matmul_op",
         ":population_count_op",
         ":reduction_ops",
@@ -2635,6 +2636,24 @@ tf_kernel_library(
     deps = MATH_DEPS,
 )
 
+tf_cc_test(
+    name = "sequence_ops_test",
+    size = "small",
+    srcs = ["sequence_ops_test.cc"],
+    deps = [
+        ":ops_testutil",
+        ":ops_util",
+        ":sequence_ops",
+        "//tensorflow/core:core_cpu",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+        "//tensorflow/core:testlib",
+    ],
+)
+
 tf_cuda_cc_test(
     name = "cast_op_test",
     size = "small",
@@ -2893,6 +2912,24 @@ tf_cuda_cc_test(
     ],
 )
 
+tf_cuda_cc_test(
+    name = "diag_op_test",
+    size = "small",
+    srcs = ["diag_op_test.cc"],
+    deps = [
+        ":diag_op",
+        ":ops_testutil",
+        ":ops_util",
+        "//tensorflow/core:core_cpu",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+        "//tensorflow/core:testlib",
+    ],
+)
+
 # conv_grad_ops currently has to be built with conv_ops*.
 # TODO(josh11b, zhengxq): put these a separate libraries in ":nn" below once
 # conv_ops_gpu.h has be separated into its own library.
@@ -2993,6 +3030,7 @@ cc_library(
         ":in_topk_op",
         ":l2loss_op",
         ":lrn_op",
+        ":nth_element_op",
         ":relu_op",
         ":softmax_op",
         ":softplus_op",
@@ -3079,6 +3117,12 @@ tf_kernel_library(
     deps = NN_DEPS + if_cuda(["@cub_archive//:cub"]),
 )
 
+tf_kernel_library(
+    name = "nth_element_op",
+    prefix = "nth_element_op",
+    deps = NN_DEPS,
+)
+
 tf_kernel_library(
     name = "xent_op",
     prefix = "xent_op",
@@ -3096,6 +3140,17 @@ tf_kernel_library(
     ],
 )
 
+tf_kernel_library(
+    name = "histogram_op",
+    prefix = "histogram_op",
+    deps = [
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:lib_internal",
+        "//third_party/eigen3",
+    ] + if_cuda(["@cub_archive//:cub"]),
+)
+
 tf_kernel_library(
     name = "l2loss_op",
     prefix = "l2loss_op",
diff --git a/tensorflow/core/kernels/batchtospace_op.cc b/tensorflow/core/kernels/batchtospace_op.cc
index 99b5d3daaa..c1c0d6d329 100644
--- a/tensorflow/core/kernels/batchtospace_op.cc
+++ b/tensorflow/core/kernels/batchtospace_op.cc
@@ -249,40 +249,34 @@ class BatchToSpaceOp : public OpKernel {
   Tensor block_shape_;
 };
 
-#define REGISTER(T)                                                  \
-  REGISTER_KERNEL_BUILDER(Name("BatchToSpaceND")                     \
-                              .Device(DEVICE_CPU)                    \
-                              .TypeConstraint<T>("T")                \
-                              .TypeConstraint<int32>("Tblock_shape") \
-                              .TypeConstraint<int32>("Tcrops")       \
-                              .HostMemory("block_shape")             \
-                              .HostMemory("crops"),                  \
-                          BatchToSpaceNDOp<CPUDevice, T>);           \
-  REGISTER_KERNEL_BUILDER(Name("BatchToSpace")                       \
-                              .Device(DEVICE_CPU)                    \
-                              .TypeConstraint<T>("T")                \
-                              .TypeConstraint<int32>("Tidx")         \
-                              .HostMemory("crops"),                  \
+#define REGISTER(T)                                        \
+  REGISTER_KERNEL_BUILDER(Name("BatchToSpaceND")           \
+                              .Device(DEVICE_CPU)          \
+                              .TypeConstraint<T>("T")      \
+                              .HostMemory("block_shape")   \
+                              .HostMemory("crops"),        \
+                          BatchToSpaceNDOp<CPUDevice, T>); \
+  REGISTER_KERNEL_BUILDER(Name("BatchToSpace")             \
+                              .Device(DEVICE_CPU)          \
+                              .TypeConstraint<T>("T")      \
+                              .HostMemory("crops"),        \
                           BatchToSpaceOp<CPUDevice, T>);
 
 TF_CALL_REAL_NUMBER_TYPES(REGISTER);
 #undef REGISTER
 
 #if GOOGLE_CUDA
-#define REGISTER(T)                                                  \
-  REGISTER_KERNEL_BUILDER(Name("BatchToSpaceND")                     \
-                              .Device(DEVICE_GPU)                    \
-                              .TypeConstraint<T>("T")                \
-                              .TypeConstraint<int32>("Tblock_shape") \
-                              .TypeConstraint<int32>("Tcrops")       \
-                              .HostMemory("block_shape")             \
-                              .HostMemory("crops"),                  \
-                          BatchToSpaceNDOp<GPUDevice, T>);           \
-  REGISTER_KERNEL_BUILDER(Name("BatchToSpace")                       \
-                              .Device(DEVICE_GPU)                    \
-                              .TypeConstraint<T>("T")                \
-                              .TypeConstraint<int32>("Tidx")         \
-                              .HostMemory("crops"),                  \
+#define REGISTER(T)                                        \
+  REGISTER_KERNEL_BUILDER(Name("BatchToSpaceND")           \
+                              .Device(DEVICE_GPU)          \
+                              .TypeConstraint<T>("T")      \
+                              .HostMemory("block_shape")   \
+                              .HostMemory("crops"),        \
+                          BatchToSpaceNDOp<GPUDevice, T>); \
+  REGISTER_KERNEL_BUILDER(Name("BatchToSpace")             \
+                              .Device(DEVICE_GPU)          \
+                              .TypeConstraint<T>("T")      \
+                              .HostMemory("crops"),        \
                           BatchToSpaceOp<GPUDevice, T>);
 
 TF_CALL_GPU_NUMBER_TYPES(REGISTER);
diff --git a/tensorflow/core/kernels/conv_ops_gpu_3.cu.cc b/tensorflow/core/kernels/conv_ops_gpu_3.cu.cc
index 6e10b53cf7..9a00a091bd 100644
--- a/tensorflow/core/kernels/conv_ops_gpu_3.cu.cc
+++ b/tensorflow/core/kernels/conv_ops_gpu_3.cu.cc
@@ -394,7 +394,7 @@ __global__ void SwapDimension1And2InTensor3SmallDim(const T* input,
     int output_block_idx = SmallDim2 ? block_offset : block_offset * small_dim;
     int output_block_origin_idx = output_block_offset + output_block_idx;
 
-    // Store the tranposed memory region in shared memory to device.
+    // Store the transposed memory region in shared memory to device.
     if (x < tile_height) {
       for (int y = 0; y < small_dim; y++) {
         int output_idx = output_block_origin_idx + x +
diff --git a/tensorflow/core/kernels/crop_and_resize_op_test.cc b/tensorflow/core/kernels/crop_and_resize_op_test.cc
index 22c659b587..a35e1b0788 100644
--- a/tensorflow/core/kernels/crop_and_resize_op_test.cc
+++ b/tensorflow/core/kernels/crop_and_resize_op_test.cc
@@ -61,8 +61,12 @@ class CropAndResizeOpTest : public OpsTestBase {
 
 REGISTER_TEST(float)
 REGISTER_TEST(double)
-REGISTER_TEST(int8)
 REGISTER_TEST(uint8)
+REGISTER_TEST(uint16)
+REGISTER_TEST(int8)
+REGISTER_TEST(int16)
+REGISTER_TEST(int32)
+REGISTER_TEST(int64)
 
 #undef REGISTER_TEST
 
diff --git a/tensorflow/core/kernels/dataset.h b/tensorflow/core/kernels/dataset.h
index a906113466..a431889409 100644
--- a/tensorflow/core/kernels/dataset.h
+++ b/tensorflow/core/kernels/dataset.h
@@ -412,7 +412,7 @@ class DatasetIterator : public IteratorBase {
     // Owns one reference on the shared dataset resource.
     const DatasetType* dataset;
 
-    // Identifies the sequence of iterators leading up to to this iterator.
+    // Identifies the sequence of iterators leading up to this iterator.
     const string prefix;
   };
 
diff --git a/tensorflow/core/kernels/diag_op.cc b/tensorflow/core/kernels/diag_op.cc
index c800859d90..be862b82f1 100644
--- a/tensorflow/core/kernels/diag_op.cc
+++ b/tensorflow/core/kernels/diag_op.cc
@@ -14,65 +14,32 @@ limitations under the License.
 ==============================================================================*/
 
 // See docs in ../ops/array_ops.cc
+
+#define EIGEN_USE_THREADS
+
+#if GOOGLE_CUDA
+#define EIGEN_USE_GPU
+#endif  // GOOGLE_CUDA
+
+#include "tensorflow/core/kernels/diag_op.h"
+
+#include <algorithm>
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/framework/tensor_types.h"
+#include "tensorflow/core/platform/types.h"
 #include "tensorflow/core/platform/logging.h"
+#include "tensorflow/core/util/work_sharder.h"
 
 namespace tensorflow {
-namespace {
-template <typename T, size_t NumDims, size_t DoubleNumDims>
-class DiagonalGenerator {
- public:
-  explicit DiagonalGenerator(const Tensor& diagonal) : diagonal_(diagonal) {
-    static_assert(DoubleNumDims == 2 * NumDims,
-                  "The second size must be the double of the first size.");
-    CHECK_EQ(diagonal.dims(), NumDims);
-  }
-  T operator()(
-      const Eigen::array<Eigen::DenseIndex, DoubleNumDims>& coordinates) const {
-    Eigen::array<Eigen::DenseIndex, NumDims> index;
-    for (size_t i = 0; i < NumDims; ++i) {
-      if (coordinates[i] != coordinates[NumDims + i]) {
-        return T(0);
-      }
-      index[i] = coordinates[i];
-    }
-    return diagonal_.tensor<T, NumDims>()(index);
-  }
 
- private:
-  Tensor diagonal_;
-};
-
-template <typename T, size_t NumDims>
-class DiagonalExtractor {
- public:
-  explicit DiagonalExtractor(const Tensor& tensor) : tensor_(tensor) {
-    CHECK_EQ(tensor.dims(), 2 * NumDims);
-  }
-  T operator()(const Eigen::array<Eigen::Index, NumDims>& coordinates) const {
-    Eigen::array<Eigen::Index, 2 * NumDims> index;
-    for (size_t j = 0; j < NumDims; ++j){
-      index[j] = coordinates[j];
-    }
-    for (size_t j = NumDims; j < 2 * NumDims; ++j){
-      index[j] = index[j - NumDims];
-    }
-    return tensor_.tensor<T, 2 * NumDims>()(index);
-  }
-
- private:
-  Tensor tensor_;
-};
-  
-}  // namespace
+typedef Eigen::ThreadPoolDevice CPUDevice;
+typedef Eigen::GpuDevice GPUDevice;
 
 // Generate the diagonal tensor with the diagonal set to the input tensor.
-// It only allows up to rank 3 input tensor, so the output tensor is up to
-// rank 6.
-template <typename T>
+template <typename Device, typename T>
 class DiagOp : public OpKernel {
  public:
   explicit DiagOp(OpKernelConstruction* context) : OpKernel(context) {}
@@ -80,9 +47,8 @@ class DiagOp : public OpKernel {
   void Compute(OpKernelContext* context) override {
     const Tensor& diagonal = context->input(0);
     const int num_dims = diagonal.dims();
-    OP_REQUIRES(context, 1 <= num_dims && num_dims <= 3,
-                errors::InvalidArgument("Expected 1 <= dims <= 3, got shape ",
-                                        diagonal.shape().DebugString()));
+    OP_REQUIRES(context, 0 != num_dims, errors::InvalidArgument(
+        "Input must be at least rank 1, got 0"));
     TensorShape out_shape;
     for (int i = 0; i < num_dims; ++i) {
       out_shape.AddDim(diagonal.dim_size(i));
@@ -93,45 +59,17 @@ class DiagOp : public OpKernel {
     Tensor* output_tensor = nullptr;
     OP_REQUIRES_OK(context,
                    context->allocate_output(0, out_shape, &output_tensor));
-    switch (num_dims) {
-      case 1:
-        output_tensor->tensor<T, 2>() = output_tensor->tensor<T, 2>().generate(
-            DiagonalGenerator<T, 1, 2>(diagonal));
-        break;
-      case 2:
-        output_tensor->tensor<T, 4>() = output_tensor->tensor<T, 4>().generate(
-            DiagonalGenerator<T, 2, 4>(diagonal));
-        break;
-      case 3:
-        output_tensor->tensor<T, 6>() = output_tensor->tensor<T, 6>().generate(
-            DiagonalGenerator<T, 3, 6>(diagonal));
-        break;
-      default:
-        context->SetStatus(errors::Unimplemented(
-            "Diagonal of rank ", num_dims, " tensor is not supported yet."));
-        return;
-    }
+    functor::DiagFunctor<Device, T> diagFunc;
+    Status s = diagFunc(context,
+                        diagonal.NumElements(),
+                        diagonal.flat<T>().data(),
+                        output_tensor->flat<T>().data());
+    OP_REQUIRES_OK(context, s);
   }
 };
 
-#define REGISTER_DIAGOP(T) \
-  REGISTER_KERNEL_BUILDER( \
-      Name("Diag").Device(DEVICE_CPU).TypeConstraint<T>("T"), DiagOp<T>)
-
-REGISTER_DIAGOP(double);
-REGISTER_DIAGOP(float);
-REGISTER_DIAGOP(int32);
-REGISTER_DIAGOP(int64);
-REGISTER_DIAGOP(complex64);
-REGISTER_DIAGOP(complex128);
-
-#undef REGISTER_DIAGOP
-
-
-// Generate the diagonal tensor with the diagonal set to the input tensor.
-// It only allows rank 2, 4, or 6 input tensor, so the output tensor is 
-// rank 1, 2, or 3.
-template <typename T>
+// Extract the diagonal tensor with the diagonal set to the input tensor.
+template <typename Device, typename T>
 class DiagPartOp : public OpKernel {
  public:
   explicit DiagPartOp(OpKernelConstruction* context) : OpKernel(context) {}
@@ -140,9 +78,9 @@ class DiagPartOp : public OpKernel {
     const Tensor& tensor = context->input(0);
     const int num_dims = tensor.dims();
     const int out_dims = num_dims / 2;
-    OP_REQUIRES(context, 2 == num_dims || 4 == num_dims || 6 == num_dims, 
-                errors::InvalidArgument("The rank of the tensor should be 2, \
-                                         4, or 6, got shape ",
+    OP_REQUIRES(context, 0 == num_dims % 2,
+                errors::InvalidArgument("The rank of the tensor should be \
+                                         even and positive, got shape ",
                                         tensor.shape().DebugString()));
     for (int i = 0; i < out_dims; i++){
       OP_REQUIRES(context, tensor.dim_size(i) == tensor.dim_size(i + out_dims),
@@ -160,39 +98,158 @@ class DiagPartOp : public OpKernel {
     Tensor* output = nullptr;
     OP_REQUIRES_OK(context,
                    context->allocate_output(0, out_shape, &output));
+    functor::DiagPartFunctor<Device, T> diagPartFunc;
+    Status s = diagPartFunc(context,
+                            out_shape.num_elements(),
+                            tensor.flat<T>().data(),
+                            output->flat<T>().data());
+    OP_REQUIRES_OK(context, s);
+  }
+};
 
-    switch (num_dims) {
-      case 2:
-        output->tensor<T, 1>() = output->tensor<T, 1>().generate(
-          DiagonalExtractor<T, 1>(tensor));
-        break; 
-      case 4:
-        output->tensor<T, 2>() = output->tensor<T, 2>().generate(
-          DiagonalExtractor<T, 2>(tensor));
-        break;
-      case 6:
-        output->tensor<T, 3>() = output->tensor<T, 3>().generate(
-          DiagonalExtractor<T, 3>(tensor));
-        break;      
-      default:
-        context->SetStatus(errors::Unimplemented(
-          "Diagonal of rank ", num_dims, " tensor is not supported yet."));
-        return;
-    }
+// Implementation of the functor specialization for CPU.
+// 
+// According to the diagonal definition,
+// `output[i1,..., ik, i1,..., ik] = input[i1,..., ik]`,
+//
+// Let the rank of input is [s1,..., sk], then any offset of input's
+// pointer can be represent by coordinate [i1,..., ik],
+// where `index = i1*(s2*...*sk) + i2*(s3*...*sk) +... + ik`
+//
+// Let new_index is the offset of output's pointer with coordinate 
+// [i1,..., ik, i1,..., ik], then we have
+// `new_index = i1*(s2*...sk*s1*...*sk) + i2*(s3*...*sk*s1*...*sk) +... + \
+//              ik*(s1*...*sk) + i1*(s2*...*sk) + i2*(s3*...*sk) +... + ik
+//            = (i1*(s2*...*sk) + i2*(s3*...*sk) +... + ik) * (1 + s1*...*sk)
+//            = index * (1 + s1*...*sk)
+//
+// Let `size = s1*...*sk`, we finally have `new_index = index * (1 + size)`,
+// which is the transfer function we use below.
+// This trick make our implementations clear and easy to be parallel.
+namespace functor {
+template <typename T>
+struct DiagFunctor<CPUDevice, T> {
+  EIGEN_ALWAYS_INLINE Status
+  operator() (OpKernelContext* context, const int64 size,
+              const T* in, T* out) {
+    // This subprocess is responsible for writing values in index range
+    // [start*size, limit*size)
+    auto subDiag = [in, out, size](int64 start, int64 limit) {
+      std::fill(out + size * start, out + size * limit, T());
+      for (int64 index = start; index < limit; ++index) {
+        out[(1 + size) * index] = in[index];
+      }
+    };
+
+    // Here, 5 is a empirical factor of cost_per_unit.
+    auto worker_threads = *(context->device()->tensorflow_cpu_worker_threads());
+    Shard(worker_threads.num_threads, worker_threads.workers, size,
+        5 * size, subDiag);
+    return Status::OK();
+  }
+};
+
+template <typename T>
+struct DiagPartFunctor<CPUDevice, T> {
+  EIGEN_ALWAYS_INLINE Status
+  operator() (OpKernelContext* context, const int64 size,
+              const T* in, T* out) {
+    // This subprocess is responsible for extracting values in index range
+    // [start, limit)
+    auto subDiagPart = [in, out, size](int64 start, int64 limit) {
+      for (int64 index = start; index < limit; ++index) {
+        out[index] = in[(1 + size) * index];
+      }
+    };
+
+    // Here, 5 is a empirical factor of cost_per_unit.
+    auto worker_threads = *(context->device()->tensorflow_cpu_worker_threads());
+    Shard(worker_threads.num_threads, worker_threads.workers, size,
+        5, subDiagPart);
+    return Status::OK();
   }
 };
+}  // namespace functor
 
-#define REGISTER_DIAGPARTOP(T) \
-  REGISTER_KERNEL_BUILDER( \
-      Name("DiagPart").Device(DEVICE_CPU).TypeConstraint<T>("T"), DiagPartOp<T>)
 
-REGISTER_DIAGPARTOP(double);
-REGISTER_DIAGPARTOP(float);
-REGISTER_DIAGPARTOP(int32);
-REGISTER_DIAGPARTOP(int64);
-REGISTER_DIAGPARTOP(complex64);
-REGISTER_DIAGPARTOP(complex128);
+// Register the CPU kernels.
+#define REGISTER_DIAGOP(T)                                    \
+  REGISTER_KERNEL_BUILDER(                                    \
+      Name("Diag").Device(DEVICE_CPU).TypeConstraint<T>("T"), \
+      DiagOp<CPUDevice, T>)
 
+TF_CALL_double(REGISTER_DIAGOP);
+TF_CALL_float(REGISTER_DIAGOP);
+TF_CALL_int32(REGISTER_DIAGOP);
+TF_CALL_int64(REGISTER_DIAGOP);
+TF_CALL_complex64(REGISTER_DIAGOP);
+TF_CALL_complex128(REGISTER_DIAGOP);
+#undef REGISTER_DIAGOP
+
+#define REGISTER_DIAGPARTOP(T)                                    \
+  REGISTER_KERNEL_BUILDER(                                        \
+      Name("DiagPart").Device(DEVICE_CPU).TypeConstraint<T>("T"), \
+      DiagPartOp<CPUDevice, T>)
+
+TF_CALL_double(REGISTER_DIAGPARTOP);
+TF_CALL_float(REGISTER_DIAGPARTOP);
+TF_CALL_int32(REGISTER_DIAGPARTOP);
+TF_CALL_int64(REGISTER_DIAGPARTOP);
+TF_CALL_complex64(REGISTER_DIAGPARTOP);
+TF_CALL_complex128(REGISTER_DIAGPARTOP);
 #undef REGISTER_DIAGPARTOP
-  
+
+// Register the GPU kernels.
+#ifdef GOOGLE_CUDA
+
+// Forward declarations of the functor specializations for GPU.
+namespace functor {
+extern template struct DiagFunctor<GPUDevice, double>;
+extern template struct DiagFunctor<GPUDevice, float>;
+extern template struct DiagFunctor<GPUDevice, int32>;
+extern template struct DiagFunctor<GPUDevice, int64>;
+extern template struct DiagFunctor<GPUDevice, complex64>;
+extern template struct DiagFunctor<GPUDevice, complex128>;
+}  // namespace functor
+
+#define REGISTER_DIAGOP_GPU(T)                                \
+  REGISTER_KERNEL_BUILDER(                                    \
+      Name("Diag").Device(DEVICE_GPU).TypeConstraint<T>("T"), \
+      DiagOp<GPUDevice, T>)
+
+TF_CALL_double(REGISTER_DIAGOP_GPU);
+TF_CALL_float(REGISTER_DIAGOP_GPU);
+TF_CALL_int32(REGISTER_DIAGOP_GPU);
+TF_CALL_int64(REGISTER_DIAGOP_GPU);
+TF_CALL_complex64(REGISTER_DIAGOP_GPU);
+TF_CALL_complex128(REGISTER_DIAGOP_GPU);
+#undef REGISTER_DIAGOP_GPU
+
+// Forward declarations of the functor specializations for GPU.
+namespace functor {
+extern template struct DiagPartFunctor<GPUDevice, double>;
+extern template struct DiagPartFunctor<GPUDevice, float>;
+extern template struct DiagPartFunctor<GPUDevice, int32>;
+extern template struct DiagPartFunctor<GPUDevice, int64>;
+extern template struct DiagPartFunctor<GPUDevice, complex64>;
+extern template struct DiagPartFunctor<GPUDevice, complex128>;
+}  // namespace functor
+
+#define REGISTER_DIAGPARTOP_GPU(T)                                \
+  REGISTER_KERNEL_BUILDER(                                        \
+      Name("DiagPart").Device(DEVICE_GPU).TypeConstraint<T>("T"), \
+      DiagPartOp<GPUDevice, T>)
+
+TF_CALL_double(REGISTER_DIAGPARTOP_GPU);
+TF_CALL_float(REGISTER_DIAGPARTOP_GPU);
+TF_CALL_int32(REGISTER_DIAGPARTOP_GPU);
+TF_CALL_int64(REGISTER_DIAGPARTOP_GPU);
+TF_CALL_complex64(REGISTER_DIAGPARTOP_GPU);
+TF_CALL_complex128(REGISTER_DIAGPARTOP_GPU);
+#undef REGISTER_DIAGPARTOP_GPU
+
+#endif  // GOOGLE_CUDA
+
+
 }  // namespace tensorflow
+
diff --git a/tensorflow/core/kernels/diag_op.h b/tensorflow/core/kernels/diag_op.h
new file mode 100644
index 0000000000..c6ca6a2047
--- /dev/null
+++ b/tensorflow/core/kernels/diag_op.h
@@ -0,0 +1,43 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_CORE_KERNELS_DIAG_OP_H_
+#define TENSORFLOW_CORE_KERNELS_DIAG_OP_H_
+
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/framework/tensor_types.h"
+#include "tensorflow/core/platform/types.h"
+
+namespace tensorflow {
+
+namespace functor {
+
+template <typename Device, typename T>
+struct DiagFunctor {
+  Status operator() (OpKernelContext* context, const int64 size,
+                     const T* in, T* out);
+};
+
+template <typename Device, typename T>
+struct DiagPartFunctor {
+  Status operator() (OpKernelContext* context, const int64 size,
+                     const T* in, T* out);
+};
+
+}  // namespace functor
+
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_CORE_KERNELS_DIAG_OP_H_
diff --git a/tensorflow/core/kernels/diag_op_gpu.cu.cc b/tensorflow/core/kernels/diag_op_gpu.cu.cc
new file mode 100644
index 0000000000..684f00ea61
--- /dev/null
+++ b/tensorflow/core/kernels/diag_op_gpu.cu.cc
@@ -0,0 +1,139 @@
+/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+
+#define EIGEN_USE_GPU
+
+#include <complex>
+#include "tensorflow/core/framework/register_types.h"
+#include "tensorflow/core/util/cuda_kernel_helper.h"
+#include "tensorflow/core/kernels/diag_op.h"
+
+namespace tensorflow {
+namespace functor {
+
+typedef Eigen::GpuDevice GPUDevice;
+
+template <typename T>
+__global__ void DiagCudaKernel(const int num_threads,
+                               const int64 size,
+                               const T* in,
+                               T* out) {
+  CUDA_1D_KERNEL_LOOP(index, num_threads) {
+    // Fill the diagonal elements or set to zero in other place. 
+    if (index % (1 + size) == 0) {
+      out[index] = in[index / (1 + size)];
+    } else {
+      out[index] = T(0);
+    }
+  }
+}
+
+template <typename T>
+struct DiagFunctor<GPUDevice, T> {
+  EIGEN_ALWAYS_INLINE Status
+  operator() (OpKernelContext* context, const int64 size,
+              const T* in, T* out) {
+    // Empty tensor couldn't launch the kernel.
+    if (size == 0) {
+      return Status::OK();
+    }
+
+    // CudaLaunchConfig uses an int for virtual_thread_count,
+    // so this may overflow for `size*size` in extreme cases,
+    // here is checking the multiplication overflow for integer.
+    if (size && (int(size * size) / size) != size) {
+      return errors::Internal(
+          "DiagOp got input size too large.");
+    }
+    int virtual_thread_count = int(size * size);
+
+    // Launch the GPU kernel.
+    const GPUDevice& device = context->eigen_device<GPUDevice>();
+    CudaLaunchConfig diag_config = GetCudaLaunchConfig(
+        virtual_thread_count, device);
+    DiagCudaKernel<<<diag_config.block_count,
+                     diag_config.thread_per_block,
+                     0, device.stream()>>>(
+        diag_config.virtual_thread_count, size, in, out);
+
+    auto err = cudaGetLastError();
+    if (err != cudaSuccess) {
+      return errors::Internal(
+          "Could not launch DiagOp kernel: ",
+          cudaGetErrorString(err), ".");
+    }
+    return Status::OK();
+  }
+};
+
+template struct DiagFunctor<GPUDevice, double>;
+template struct DiagFunctor<GPUDevice, float>;
+template struct DiagFunctor<GPUDevice, int32>;
+template struct DiagFunctor<GPUDevice, int64>;
+template struct DiagFunctor<GPUDevice, complex64>;
+template struct DiagFunctor<GPUDevice, complex128>;
+
+
+template <typename T>
+__global__ void DiagPartCudaKernel(const int num_threads,
+                                   const int64 size,
+                                   const T* in,
+                                   T* out) {
+  CUDA_1D_KERNEL_LOOP(index, num_threads) {
+    out[index] = in[(1 + size) * index];
+  }
+}
+
+template <typename T>
+struct DiagPartFunctor<GPUDevice, T> {
+  EIGEN_ALWAYS_INLINE Status
+  operator() (OpKernelContext* context, const int64 size,
+              const T* in, T* out) {
+    // Empty tensor couldn't launch the kernel.
+    if (size == 0) {
+      return Status::OK();
+    }
+    const GPUDevice& device = context->eigen_device<GPUDevice>();
+
+    // Extract the diagonal elements.
+    CudaLaunchConfig diag_config = GetCudaLaunchConfig(size, device);
+    DiagPartCudaKernel<<<diag_config.block_count,
+                     diag_config.thread_per_block,
+                     0, device.stream()>>>(
+        diag_config.virtual_thread_count, size, in, out);
+
+    auto err = cudaGetLastError();
+    if (err != cudaSuccess) {
+      return errors::Internal(
+          "Could not launch DiagPartOp kernel: ",
+          cudaGetErrorString(err), ".");
+    }
+    return Status::OK();
+  }
+};
+
+template struct DiagPartFunctor<GPUDevice, double>;
+template struct DiagPartFunctor<GPUDevice, float>;
+template struct DiagPartFunctor<GPUDevice, int32>;
+template struct DiagPartFunctor<GPUDevice, int64>;
+template struct DiagPartFunctor<GPUDevice, complex64>;
+template struct DiagPartFunctor<GPUDevice, complex128>;
+
+}  // end namespace functor
+}  // end namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/diag_op_test.cc b/tensorflow/core/kernels/diag_op_test.cc
new file mode 100644
index 0000000000..2d1417854c
--- /dev/null
+++ b/tensorflow/core/kernels/diag_op_test.cc
@@ -0,0 +1,54 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/common_runtime/kernel_benchmark_testlib.h"
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/platform/test.h"
+#include "tensorflow/core/platform/test_benchmark.h"
+
+namespace tensorflow {
+
+template <typename T>
+static Graph* Diag(int n, DataType type) {
+  Graph* g = new Graph(OpRegistry::Global());
+  Tensor in(type, TensorShape({n}));
+  in.flat<T>().setRandom();
+  Node* out = test::graph::Diag(g, test::graph::Constant(g, in), type);
+  test::graph::DiagPart(g, out, type);
+  return g;
+}
+
+#define BM_DiagDev(N, T, TFTYPE, DEVICE)                           \
+  static void BM_Diag##_##N##_##TFTYPE##_##DEVICE(int iters) {   \
+    testing::UseRealTime();                                     \
+    testing::ItemsProcessed(static_cast<int64>(iters) * N * N); \
+    test::Benchmark(#DEVICE, Diag<T>(N, TFTYPE)).Run(iters);    \
+  }                                                             \
+  BENCHMARK(BM_Diag##_##N##_##TFTYPE##_##DEVICE);
+
+#define BM_Diag(N)                                       \
+  BM_DiagDev(N, int, DT_INT32, cpu);                     \
+  BM_DiagDev(N, float, DT_FLOAT, cpu);                   \
+  BM_DiagDev(N, std::complex<float>, DT_COMPLEX64, cpu); \
+  BM_DiagDev(N, int, DT_INT32, gpu);                     \
+  BM_DiagDev(N, float, DT_FLOAT, gpu);                   \
+  BM_DiagDev(N, std::complex<float>, DT_COMPLEX64, gpu);
+
+BM_Diag(16);
+BM_Diag(128);
+BM_Diag(512);
+
+}  // end namespace tensorflow
+
diff --git a/tensorflow/core/kernels/histogram_op.cc b/tensorflow/core/kernels/histogram_op.cc
new file mode 100644
index 0000000000..4e035286f6
--- /dev/null
+++ b/tensorflow/core/kernels/histogram_op.cc
@@ -0,0 +1,147 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+// See docs in ../ops/math_ops.cc.
+
+#define EIGEN_USE_THREADS
+
+#include "tensorflow/core/kernels/histogram_op.h"
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/framework/register_types.h"
+#include "tensorflow/core/framework/types.h"
+#include "tensorflow/core/lib/core/threadpool.h"
+#include "tensorflow/core/platform/types.h"
+
+namespace tensorflow {
+
+typedef Eigen::ThreadPoolDevice CPUDevice;
+typedef Eigen::GpuDevice GPUDevice;
+
+namespace functor {
+
+template <typename T, typename Tout>
+struct HistogramFixedWidthFunctor<CPUDevice, T, Tout> {
+  static Status Compute(OpKernelContext* context,
+                        const typename TTypes<T, 1>::ConstTensor& values,
+                        const typename TTypes<T, 1>::ConstTensor& value_range,
+                        int32 nbins, typename TTypes<Tout, 1>::Tensor& out) {
+    const CPUDevice& d = context->eigen_device<CPUDevice>();
+
+    Tensor index_to_bin_tensor;
+
+    TF_RETURN_IF_ERROR(context->forward_input_or_allocate_temp(
+        {0}, DataTypeToEnum<int32>::value, TensorShape({values.size()}),
+        &index_to_bin_tensor));
+    auto index_to_bin = index_to_bin_tensor.flat<int32>();
+
+    const double step = static_cast<double>(value_range(1) - value_range(0)) /
+                        static_cast<double>(nbins);
+
+    // The calculation is done by finding the slot of each value in `values`.
+    // With [a, b]:
+    //   step = (b - a) / nbins
+    //   (x - a) / step
+    // , then the entries are mapped to output.
+    index_to_bin.device(d) =
+        ((values.cwiseMax(value_range(0)) - values.constant(value_range(0)))
+             .template cast<double>() /
+         step)
+            .template cast<int32>()
+            .cwiseMin(nbins - 1);
+
+    out.setZero();
+    for (int32 i = 0; i < index_to_bin.size(); i++) {
+      out(index_to_bin(i)) += Tout(1);
+    }
+    return Status::OK();
+  }
+};
+
+}  // namespace functor
+
+template <typename Device, typename T, typename Tout>
+class HistogramFixedWidthOp : public OpKernel {
+ public:
+  explicit HistogramFixedWidthOp(OpKernelConstruction* ctx) : OpKernel(ctx) {}
+
+  void Compute(OpKernelContext* ctx) override {
+    const Tensor& values_tensor = ctx->input(0);
+    const Tensor& value_range_tensor = ctx->input(1);
+    const Tensor& nbins_tensor = ctx->input(2);
+
+    OP_REQUIRES(ctx, TensorShapeUtils::IsVector(value_range_tensor.shape()),
+                errors::InvalidArgument("value_range should be a vector."));
+    OP_REQUIRES(ctx, (value_range_tensor.shape().num_elements() == 2),
+                errors::InvalidArgument(
+                    "value_range should be a vector of 2 elements."));
+    OP_REQUIRES(ctx, TensorShapeUtils::IsScalar(nbins_tensor.shape()),
+                errors::InvalidArgument("nbins should be a scalar."));
+
+    const auto values = values_tensor.flat<T>();
+    const auto value_range = value_range_tensor.flat<T>();
+    const auto nbins = nbins_tensor.scalar<int32>()();
+
+    OP_REQUIRES(
+        ctx, (value_range(0) < value_range(1)),
+        errors::InvalidArgument("value_range should satisfy value_range[0] < "
+                                "value_range[1], but got '[",
+                                value_range(0), ", ", value_range(1), "]'"));
+    OP_REQUIRES(
+        ctx, (nbins > 0),
+        errors::InvalidArgument("nbins should be a positive number, but got '",
+                                nbins, "'"));
+
+    Tensor* out_tensor;
+    OP_REQUIRES_OK(ctx,
+                   ctx->allocate_output(0, TensorShape({nbins}), &out_tensor));
+    auto out = out_tensor->flat<Tout>();
+
+    OP_REQUIRES_OK(
+        ctx, functor::HistogramFixedWidthFunctor<Device, T, Tout>::Compute(
+                 ctx, values, value_range, nbins, out));
+  }
+};
+
+#define REGISTER_KERNELS(type)                                           \
+  REGISTER_KERNEL_BUILDER(Name("HistogramFixedWidth")                    \
+                              .Device(DEVICE_CPU)                        \
+                              .TypeConstraint<type>("T")                 \
+                              .TypeConstraint<int32>("dtype"),           \
+                          HistogramFixedWidthOp<CPUDevice, type, int32>) \
+  REGISTER_KERNEL_BUILDER(Name("HistogramFixedWidth")                    \
+                              .Device(DEVICE_CPU)                        \
+                              .TypeConstraint<type>("T")                 \
+                              .TypeConstraint<int64>("dtype"),           \
+                          HistogramFixedWidthOp<CPUDevice, type, int64>)
+
+TF_CALL_REAL_NUMBER_TYPES(REGISTER_KERNELS);
+#undef REGISTER_KERNELS
+
+#if GOOGLE_CUDA
+#define REGISTER_KERNELS(type)                                 \
+  REGISTER_KERNEL_BUILDER(Name("HistogramFixedWidth")          \
+                              .Device(DEVICE_GPU)              \
+                              .HostMemory("value_range")       \
+                              .HostMemory("nbins")             \
+                              .TypeConstraint<type>("T")       \
+                              .TypeConstraint<int32>("dtype"), \
+                          HistogramFixedWidthOp<GPUDevice, type, int32>)
+
+TF_CALL_GPU_NUMBER_TYPES(REGISTER_KERNELS);
+#undef REGISTER_KERNELS
+
+#endif  // GOOGLE_CUDA
+
+}  // end namespace tensorflow
diff --git a/tensorflow/core/kernels/histogram_op.h b/tensorflow/core/kernels/histogram_op.h
new file mode 100644
index 0000000000..1b253f7fed
--- /dev/null
+++ b/tensorflow/core/kernels/histogram_op.h
@@ -0,0 +1,38 @@
+/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_HISTOGRAM_OP_H_
+#define TENSORFLOW_HISTOGRAM_OP_H_
+
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/framework/tensor_types.h"
+#include "tensorflow/core/framework/types.h"
+#include "tensorflow/core/lib/core/errors.h"
+
+namespace tensorflow {
+namespace functor {
+
+template <typename Device, typename T, typename Tout>
+struct HistogramFixedWidthFunctor {
+  static Status Compute(OpKernelContext* context,
+                        const typename TTypes<T, 1>::ConstTensor& values,
+                        const typename TTypes<T, 1>::ConstTensor& value_range,
+                        int32 nbins, typename TTypes<Tout, 1>::Tensor& out);
+};
+
+}  // end namespace functor
+}  // end namespace tensorflow
+
+#endif  // TENSORFLOW_HISTOGRAM_OP_H_
diff --git a/tensorflow/core/kernels/histogram_op_gpu.cu.cc b/tensorflow/core/kernels/histogram_op_gpu.cu.cc
new file mode 100644
index 0000000000..c2bb958be8
--- /dev/null
+++ b/tensorflow/core/kernels/histogram_op_gpu.cu.cc
@@ -0,0 +1,125 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#if GOOGLE_CUDA
+
+#define EIGEN_USE_GPU
+
+#include "tensorflow/core/kernels/histogram_op.h"
+#include "external/cub_archive/cub/device/device_histogram.cuh"
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/framework/register_types.h"
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/framework/tensor_shape.h"
+#include "tensorflow/core/platform/logging.h"
+#include "tensorflow/core/platform/types.h"
+#include "tensorflow/core/util/cuda_kernel_helper.h"
+#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+
+namespace tensorflow {
+
+typedef Eigen::GpuDevice GPUDevice;
+
+namespace functor {
+
+// TODO(yongtang) int64 of atomicAdd is not supported yet.
+template <typename T, typename Tout>
+struct HistogramFixedWidthFunctor<GPUDevice, T, Tout> {
+  static Status Compute(OpKernelContext* context,
+                        const typename TTypes<T, 1>::ConstTensor& values,
+                        const typename TTypes<T, 1>::ConstTensor& value_range,
+                        int32 nbins, typename TTypes<Tout, 1>::Tensor& out) {
+    tensorflow::AllocatorAttributes pinned_allocator;
+    pinned_allocator.set_on_host(true);
+    pinned_allocator.set_gpu_compatible(true);
+
+    Tensor levels_tensor;
+    TF_RETURN_IF_ERROR(context->allocate_temp(
+        DataTypeToEnum<T>::value, TensorShape({nbins + 1}), &levels_tensor,
+        pinned_allocator));
+    auto levels = levels_tensor.flat<T>();
+
+    const double step = static_cast<double>(value_range(1) - value_range(0)) /
+                        static_cast<double>(nbins);
+    levels(0) = std::numeric_limits<T>::lowest();
+    for (int i = 1; i < nbins; i++) {
+      levels(i) =
+          static_cast<T>(static_cast<double>(value_range(0)) + step * i);
+    }
+    levels(nbins) = std::numeric_limits<T>::max();
+
+    size_t temp_storage_bytes = 0;
+    const T* d_samples = values.data();
+    Tout* d_histogram = out.data();
+    int num_levels = levels.size();
+    T* d_levels = levels.data();
+    int num_samples = values.size();
+    const cudaStream_t& stream = GetCudaStream(context);
+
+    // The first HistogramRange is to obtain the temp storage size required
+    // with d_temp_storage = NULL passed to the call.
+    auto err = cub::DeviceHistogram::HistogramRange(
+        /* d_temp_storage */ NULL,
+        /* temp_storage_bytes */ temp_storage_bytes,
+        /* d_samples */ d_samples,
+        /* d_histogram */ d_histogram,
+        /* num_levels */ num_levels,
+        /* d_levels */ d_levels,
+        /* num_samples */ num_samples,
+        /* stream */ stream);
+    if (err != cudaSuccess) {
+      return errors::Internal(
+          "Could not launch HistogramRange to get temp storage: ",
+          cudaGetErrorString(err), ".");
+    }
+
+    Tensor temp_storage;
+    TF_RETURN_IF_ERROR(context->allocate_temp(
+        DataTypeToEnum<int8>::value,
+        TensorShape({static_cast<int64>(temp_storage_bytes)}), &temp_storage));
+
+    void* d_temp_storage = temp_storage.flat<int8>().data();
+
+    // The second HistogramRange is to actual run with d_temp_storage
+    // allocated with temp_storage_bytes.
+    err = cub::DeviceHistogram::HistogramRange(
+        /* d_temp_storage */ d_temp_storage,
+        /* temp_storage_bytes */ temp_storage_bytes,
+        /* d_samples */ d_samples,
+        /* d_histogram */ d_histogram,
+        /* num_levels */ num_levels,
+        /* d_levels */ d_levels,
+        /* num_samples */ num_samples,
+        /* stream */ stream);
+    if (err != cudaSuccess) {
+      return errors::Internal("Could not launch HistogramRange: ",
+                              cudaGetErrorString(err), ".");
+    }
+
+    return Status::OK();
+  }
+};
+
+}  // end namespace functor
+
+#define REGISTER_GPU_SPEC(type) \
+  template struct functor::HistogramFixedWidthFunctor<GPUDevice, type, int32>;
+
+TF_CALL_GPU_NUMBER_TYPES(REGISTER_GPU_SPEC);
+#undef REGISTER_GPU_SPEC
+
+}  // namespace tensorflow
+
+#endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/listdiff_op.cc b/tensorflow/core/kernels/listdiff_op.cc
index d303bdd560..d28a2729d4 100644
--- a/tensorflow/core/kernels/listdiff_op.cc
+++ b/tensorflow/core/kernels/listdiff_op.cc
@@ -24,12 +24,13 @@ limitations under the License.
 #include "tensorflow/core/lib/core/status.h"
 
 namespace tensorflow {
-template <typename T>
+template <typename T, typename Tidx>
 class ListDiffOp : public OpKernel {
  public:
   explicit ListDiffOp(OpKernelConstruction* context) : OpKernel(context) {
     const DataType dt = DataTypeToEnum<T>::v();
-    OP_REQUIRES_OK(context, context->MatchSignature({dt, dt}, {dt, DT_INT32}));
+    const DataType dtidx = DataTypeToEnum<Tidx>::v();
+    OP_REQUIRES_OK(context, context->MatchSignature({dt, dt}, {dt, dtidx}));
   }
 
   void Compute(OpKernelContext* context) override {
@@ -72,9 +73,9 @@ class ListDiffOp : public OpKernel {
 
     Tensor* indices = nullptr;
     OP_REQUIRES_OK(context, context->allocate_output(1, {out_size}, &indices));
-    auto Tindices = indices->vec<int32>();
+    auto Tindices = indices->vec<Tidx>();
 
-    for (int i = 0, p = 0; i < static_cast<int32>(x_size); ++i) {
+    for (Tidx i = 0, p = 0; i < static_cast<Tidx>(x_size); ++i) {
       if (y_set.count(Tx(i)) == 0) {
         OP_REQUIRES(context, p < out_size,
                     errors::InvalidArgument(
@@ -95,7 +96,12 @@ class ListDiffOp : public OpKernel {
                               .Device(DEVICE_CPU)                \
                               .TypeConstraint<type>("T")         \
                               .TypeConstraint<int32>("out_idx"), \
-                          ListDiffOp<type>)
+                          ListDiffOp<type, int32>)               \
+  REGISTER_KERNEL_BUILDER(Name("ListDiff")                       \
+                              .Device(DEVICE_CPU)                \
+                              .TypeConstraint<type>("T")         \
+                              .TypeConstraint<int64>("out_idx"), \
+                          ListDiffOp<type, int64>)
 
 TF_CALL_REAL_NUMBER_TYPES(REGISTER_LISTDIFF);
 REGISTER_LISTDIFF(string);
diff --git a/tensorflow/core/kernels/map_stage_op.cc b/tensorflow/core/kernels/map_stage_op.cc
index 0168b57d35..7b5a464b72 100644
--- a/tensorflow/core/kernels/map_stage_op.cc
+++ b/tensorflow/core/kernels/map_stage_op.cc
@@ -111,15 +111,21 @@ class StagingMap : public ResourceBase {
   void notify_inserters_if_bounded(std::unique_lock<std::mutex>* lock) {
     if (has_capacity() || has_memory_limit()) {
       lock->unlock();
-      full_.notify_one();
+      // Notify all inserters. The removal of an element
+      // may make memory available for many inserters
+      // to insert new elements
+      full_.notify_all();
     }
   }
 
-  // Notify any removers waiting to extract values
+  // Notify all removers waiting to extract values
   // that data is now available
   void notify_removers(std::unique_lock<std::mutex>* lock) {
     lock->unlock();
-    not_empty_.notify_one();
+    // Notify all removers. This is because they are
+    // waiting for specific keys to appear in the map
+    // so we don't know which one to wake up.
+    not_empty_.notify_all();
   }
 
   bool has_capacity() const { return capacity_ > 0; }
diff --git a/tensorflow/core/kernels/mirror_pad_op.cc b/tensorflow/core/kernels/mirror_pad_op.cc
index e3643f9447..fbdeaf43eb 100644
--- a/tensorflow/core/kernels/mirror_pad_op.cc
+++ b/tensorflow/core/kernels/mirror_pad_op.cc
@@ -18,10 +18,10 @@ limitations under the License.
 #define EIGEN_USE_THREADS
 
 #include "tensorflow/core/kernels/mirror_pad_op.h"
-
 #include <string>
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+
 #include "tensorflow/core/framework/op.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
@@ -35,7 +35,7 @@ limitations under the License.
 
 namespace tensorflow {
 
-template <typename Device, typename T>
+template <typename Device, typename T, typename Tpaddings>
 class MirrorPadOp : public OpKernel {
  public:
   explicit MirrorPadOp(OpKernelConstruction* context) : OpKernel(context) {
@@ -82,10 +82,10 @@ class MirrorPadOp : public OpKernel {
 
     // Compute the shape of the output tensor, and allocate it.
     TensorShape output_shape;
-    TTypes<int32>::ConstMatrix paddings = in1.matrix<int32>();
+    typename TTypes<Tpaddings>::ConstMatrix paddings = in1.matrix<Tpaddings>();
     for (int d = 0; d < dims; ++d) {
-      const int32 before = paddings(d, 0);  // Pad before existing elements.
-      const int32 after = paddings(d, 1);   // Pad after existing elements.
+      const Tpaddings before = paddings(d, 0);  // Pad before existing elements.
+      const Tpaddings after = paddings(d, 1);   // Pad after existing elements.
       OP_REQUIRES(context, before >= 0 && after >= 0,
                   errors::InvalidArgument("paddings must be non-negative: ",
                                           before, " ", after));
@@ -121,7 +121,7 @@ class MirrorPadOp : public OpKernel {
 
 #define MIRROR_PAD_CASE(i)                                                \
   case i: {                                                               \
-    functor::MirrorPad<Device, T, i>()(                                   \
+    functor::MirrorPad<Device, T, Tpaddings, i>()(                        \
         context->eigen_device<Device>(), To32Bit(output->tensor<T, i>()), \
         To32Bit(in0.tensor<T, i>()), paddings, offset_);                  \
     break;                                                                \
@@ -152,20 +152,25 @@ using GpuDevice = Eigen::GpuDevice;
 namespace functor {
 // Forward declarations of the functor specializations defined in the sharded
 // files.
-#define DECLARE_CPU_SPEC(T, i)                                               \
-  template <>                                                                \
-  void MirrorPad<CpuDevice, T, i>::operator()(                               \
-      const CpuDevice&, typename TTypes<T, i, int32>::Tensor,                \
-      typename TTypes<T, i, int32>::ConstTensor, TTypes<int32>::ConstMatrix, \
-      int);                                                                  \
-  extern template struct MirrorPad<CpuDevice, T, i>;
-
-#define DECLARE_CPU_SPECS(T) \
-  DECLARE_CPU_SPEC(T, 1);    \
-  DECLARE_CPU_SPEC(T, 2);    \
-  DECLARE_CPU_SPEC(T, 3);    \
-  DECLARE_CPU_SPEC(T, 4);    \
-  DECLARE_CPU_SPEC(T, 5);
+#define DECLARE_CPU_SPEC(T, Tpaddings, i)                     \
+  template <>                                                 \
+  void MirrorPad<CpuDevice, T, Tpaddings, i>::operator()(     \
+      const CpuDevice&, typename TTypes<T, i, int32>::Tensor, \
+      typename TTypes<T, i, int32>::ConstTensor,              \
+      TTypes<Tpaddings>::ConstMatrix, int);                   \
+  extern template struct MirrorPad<CpuDevice, T, Tpaddings, i>;
+
+#define DECLARE_CPU_SPECS(T)     \
+  DECLARE_CPU_SPEC(T, int32, 1); \
+  DECLARE_CPU_SPEC(T, int32, 2); \
+  DECLARE_CPU_SPEC(T, int32, 3); \
+  DECLARE_CPU_SPEC(T, int32, 4); \
+  DECLARE_CPU_SPEC(T, int32, 5); \
+  DECLARE_CPU_SPEC(T, int64, 1); \
+  DECLARE_CPU_SPEC(T, int64, 2); \
+  DECLARE_CPU_SPEC(T, int64, 3); \
+  DECLARE_CPU_SPEC(T, int64, 4); \
+  DECLARE_CPU_SPEC(T, int64, 5);
 
 TF_CALL_POD_TYPES(DECLARE_CPU_SPECS);
 
@@ -179,7 +184,13 @@ TF_CALL_POD_TYPES(DECLARE_CPU_SPECS);
                               .TypeConstraint<type>("T")          \
                               .TypeConstraint<int32>("Tpaddings") \
                               .HostMemory("paddings"),            \
-                          MirrorPadOp<CpuDevice, type>);
+                          MirrorPadOp<CpuDevice, type, int32>);   \
+  REGISTER_KERNEL_BUILDER(Name("MirrorPad")                       \
+                              .Device(DEVICE_CPU)                 \
+                              .TypeConstraint<type>("T")          \
+                              .TypeConstraint<int64>("Tpaddings") \
+                              .HostMemory("paddings"),            \
+                          MirrorPadOp<CpuDevice, type, int64>);
 
 // Note that we do register for bool type, but not in the gradient op.
 TF_CALL_POD_TYPES(REGISTER_KERNEL);
@@ -188,20 +199,25 @@ TF_CALL_POD_TYPES(REGISTER_KERNEL);
 #if GOOGLE_CUDA
 namespace functor {
 // Forward declarations of the functor specializations for GPU.
-#define DECLARE_GPU_SPEC(T, i)                                               \
-  template <>                                                                \
-  void MirrorPad<GpuDevice, T, i>::operator()(                               \
-      const GpuDevice&, typename TTypes<T, i, int32>::Tensor,                \
-      typename TTypes<T, i, int32>::ConstTensor, TTypes<int32>::ConstMatrix, \
-      int);                                                                  \
-  extern template struct MirrorPad<GpuDevice, T, i>;
-
-#define DECLARE_GPU_SPECS(T) \
-  DECLARE_GPU_SPEC(T, 1);    \
-  DECLARE_GPU_SPEC(T, 2);    \
-  DECLARE_GPU_SPEC(T, 3);    \
-  DECLARE_GPU_SPEC(T, 4);    \
-  DECLARE_GPU_SPEC(T, 5);
+#define DECLARE_GPU_SPEC(T, Tpaddings, i)                     \
+  template <>                                                 \
+  void MirrorPad<GpuDevice, T, Tpaddings, i>::operator()(     \
+      const GpuDevice&, typename TTypes<T, i, int32>::Tensor, \
+      typename TTypes<T, i, int32>::ConstTensor,              \
+      TTypes<Tpaddings>::ConstMatrix, int);                   \
+  extern template struct MirrorPad<GpuDevice, T, Tpaddings, i>;
+
+#define DECLARE_GPU_SPECS(T)     \
+  DECLARE_GPU_SPEC(T, int32, 1); \
+  DECLARE_GPU_SPEC(T, int32, 2); \
+  DECLARE_GPU_SPEC(T, int32, 3); \
+  DECLARE_GPU_SPEC(T, int32, 4); \
+  DECLARE_GPU_SPEC(T, int32, 5); \
+  DECLARE_GPU_SPEC(T, int64, 1); \
+  DECLARE_GPU_SPEC(T, int64, 2); \
+  DECLARE_GPU_SPEC(T, int64, 3); \
+  DECLARE_GPU_SPEC(T, int64, 4); \
+  DECLARE_GPU_SPEC(T, int64, 5);
 
 TF_CALL_GPU_NUMBER_TYPES(DECLARE_GPU_SPECS);
 #undef DECLARE_GPU_SPECS
@@ -215,14 +231,20 @@ TF_CALL_GPU_NUMBER_TYPES(DECLARE_GPU_SPECS);
                               .TypeConstraint<T>("T")             \
                               .TypeConstraint<int32>("Tpaddings") \
                               .HostMemory("paddings"),            \
-                          MirrorPadOp<GpuDevice, T>)
+                          MirrorPadOp<GpuDevice, T, int32>);      \
+  REGISTER_KERNEL_BUILDER(Name("MirrorPad")                       \
+                              .Device(DEVICE_GPU)                 \
+                              .TypeConstraint<T>("T")             \
+                              .TypeConstraint<int64>("Tpaddings") \
+                              .HostMemory("paddings"),            \
+                          MirrorPadOp<GpuDevice, T, int64>);
 
 TF_CALL_GPU_NUMBER_TYPES(REGISTER_GPU_KERNEL);
 #undef REGISTER_GPU_KERNEL
 #endif  // GOOGLE_CUDA
 
 // Gradient op.
-template <typename Device, typename T>
+template <typename Device, typename T, typename Tpaddings>
 class MirrorPadGradOp : public OpKernel {
  public:
   explicit MirrorPadGradOp(OpKernelConstruction* context) : OpKernel(context) {
@@ -269,10 +291,10 @@ class MirrorPadGradOp : public OpKernel {
 
     // Compute the shape of the output tensor, and allocate it.
     TensorShape output_shape;
-    TTypes<int32>::ConstMatrix paddings = in1.matrix<int32>();
+    typename TTypes<Tpaddings>::ConstMatrix paddings = in1.matrix<Tpaddings>();
     for (int d = 0; d < dims; ++d) {
-      const int32 before = paddings(d, 0);  // Pad before existing elements.
-      const int32 after = paddings(d, 1);   // Pad after existing elements.
+      const Tpaddings before = paddings(d, 0);  // Pad before existing elements.
+      const Tpaddings after = paddings(d, 1);   // Pad after existing elements.
       OP_REQUIRES(context, before >= 0 && after >= 0,
                   errors::InvalidArgument("Paddings must be non-negative: ",
                                           before, ", ", after));
@@ -308,7 +330,7 @@ class MirrorPadGradOp : public OpKernel {
 
 #define MIRROR_PAD_GRAD_CASE(k)                                           \
   case k: {                                                               \
-    functor::MirrorPadGrad<Device, T, k>()(                               \
+    functor::MirrorPadGrad<Device, T, Tpaddings, k>()(                    \
         context->eigen_device<Device>(), To32Bit(output->tensor<T, k>()), \
         To32Bit(in0.tensor<T, k>()), paddings, offset_,                   \
         To32Bit(scratch.tensor<T, k>()));                                 \
@@ -337,33 +359,45 @@ class MirrorPadGradOp : public OpKernel {
 namespace functor {
 // Forward declarations of the functor specializations defined in the sharded
 // files.
-#define DECLARE_CPU_SPEC(T, k)                                               \
-  template <>                                                                \
-  void MirrorPadGrad<CpuDevice, T, k>::operator()(                           \
-      const CpuDevice&, typename TTypes<T, k, int32>::Tensor,                \
-      typename TTypes<T, k, int32>::ConstTensor, TTypes<int32>::ConstMatrix, \
-      int, typename TTypes<T, k, int32>::Tensor);                            \
-  extern template struct MirrorPadGrad<CpuDevice, T, k>;
-
-#define DECLARE_CPU_SPECS(T) \
-  DECLARE_CPU_SPEC(T, 1);    \
-  DECLARE_CPU_SPEC(T, 2);    \
-  DECLARE_CPU_SPEC(T, 3);    \
-  DECLARE_CPU_SPEC(T, 4);    \
-  DECLARE_CPU_SPEC(T, 5);
+#define DECLARE_CPU_SPEC(T, Tpaddings, k)                     \
+  template <>                                                 \
+  void MirrorPadGrad<CpuDevice, T, Tpaddings, k>::operator()( \
+      const CpuDevice&, typename TTypes<T, k, int32>::Tensor, \
+      typename TTypes<T, k, int32>::ConstTensor,              \
+      TTypes<Tpaddings>::ConstMatrix, int,                    \
+      typename TTypes<T, k, int32>::Tensor);                  \
+  extern template struct MirrorPadGrad<CpuDevice, T, Tpaddings, k>;
+
+#define DECLARE_CPU_SPECS(T)     \
+  DECLARE_CPU_SPEC(T, int32, 1); \
+  DECLARE_CPU_SPEC(T, int32, 2); \
+  DECLARE_CPU_SPEC(T, int32, 3); \
+  DECLARE_CPU_SPEC(T, int32, 4); \
+  DECLARE_CPU_SPEC(T, int32, 5); \
+  DECLARE_CPU_SPEC(T, int64, 1); \
+  DECLARE_CPU_SPEC(T, int64, 2); \
+  DECLARE_CPU_SPEC(T, int64, 3); \
+  DECLARE_CPU_SPEC(T, int64, 4); \
+  DECLARE_CPU_SPEC(T, int64, 5);
 
 TF_CALL_NUMBER_TYPES(DECLARE_CPU_SPECS);
 #undef DECLARE_CPU_SPECS
 #undef DECLARE_CPU_SPEC
 }  // namespace functor
 
-#define REGISTER_KERNEL(type)                                     \
-  REGISTER_KERNEL_BUILDER(Name("MirrorPadGrad")                   \
-                              .Device(DEVICE_CPU)                 \
-                              .TypeConstraint<type>("T")          \
-                              .TypeConstraint<int32>("Tpaddings") \
-                              .HostMemory("paddings"),            \
-                          MirrorPadGradOp<CpuDevice, type>);
+#define REGISTER_KERNEL(type)                                       \
+  REGISTER_KERNEL_BUILDER(Name("MirrorPadGrad")                     \
+                              .Device(DEVICE_CPU)                   \
+                              .TypeConstraint<type>("T")            \
+                              .TypeConstraint<int32>("Tpaddings")   \
+                              .HostMemory("paddings"),              \
+                          MirrorPadGradOp<CpuDevice, type, int32>); \
+  REGISTER_KERNEL_BUILDER(Name("MirrorPadGrad")                     \
+                              .Device(DEVICE_CPU)                   \
+                              .TypeConstraint<type>("T")            \
+                              .TypeConstraint<int64>("Tpaddings")   \
+                              .HostMemory("paddings"),              \
+                          MirrorPadGradOp<CpuDevice, type, int64>);
 
 TF_CALL_NUMBER_TYPES(REGISTER_KERNEL);
 #undef REGISTER_KERNEL
@@ -371,20 +405,26 @@ TF_CALL_NUMBER_TYPES(REGISTER_KERNEL);
 #if GOOGLE_CUDA
 namespace functor {
 // Forward declarations of the functor specializations for GPU.
-#define DECLARE_GPU_SPEC(T, k)                                               \
-  template <>                                                                \
-  void MirrorPadGrad<GpuDevice, T, k>::operator()(                           \
-      const GpuDevice&, typename TTypes<T, k, int32>::Tensor,                \
-      typename TTypes<T, k, int32>::ConstTensor, TTypes<int32>::ConstMatrix, \
-      int, typename TTypes<T, k, int32>::Tensor);                            \
-  extern template struct MirrorPadGrad<GpuDevice, T, k>;
-
-#define DECLARE_GPU_SPECS(T) \
-  DECLARE_GPU_SPEC(T, 1);    \
-  DECLARE_GPU_SPEC(T, 2);    \
-  DECLARE_GPU_SPEC(T, 3);    \
-  DECLARE_GPU_SPEC(T, 4);    \
-  DECLARE_GPU_SPEC(T, 5);
+#define DECLARE_GPU_SPEC(T, Tpaddings, k)                     \
+  template <>                                                 \
+  void MirrorPadGrad<GpuDevice, T, Tpaddings, k>::operator()( \
+      const GpuDevice&, typename TTypes<T, k, int32>::Tensor, \
+      typename TTypes<T, k, int32>::ConstTensor,              \
+      TTypes<Tpaddings>::ConstMatrix, int,                    \
+      typename TTypes<T, k, int32>::Tensor);                  \
+  extern template struct MirrorPadGrad<GpuDevice, T, Tpaddings, k>;
+
+#define DECLARE_GPU_SPECS(T)     \
+  DECLARE_GPU_SPEC(T, int32, 1); \
+  DECLARE_GPU_SPEC(T, int32, 2); \
+  DECLARE_GPU_SPEC(T, int32, 3); \
+  DECLARE_GPU_SPEC(T, int32, 4); \
+  DECLARE_GPU_SPEC(T, int32, 5); \
+  DECLARE_GPU_SPEC(T, int64, 1); \
+  DECLARE_GPU_SPEC(T, int64, 2); \
+  DECLARE_GPU_SPEC(T, int64, 3); \
+  DECLARE_GPU_SPEC(T, int64, 4); \
+  DECLARE_GPU_SPEC(T, int64, 5);
 
 TF_CALL_GPU_NUMBER_TYPES(DECLARE_GPU_SPECS);
 #undef DECLARE_GPU_SPECS
@@ -398,7 +438,13 @@ TF_CALL_GPU_NUMBER_TYPES(DECLARE_GPU_SPECS);
                               .TypeConstraint<T>("T")             \
                               .TypeConstraint<int32>("Tpaddings") \
                               .HostMemory("paddings"),            \
-                          MirrorPadGradOp<GpuDevice, T>)
+                          MirrorPadGradOp<GpuDevice, T, int32>);  \
+  REGISTER_KERNEL_BUILDER(Name("MirrorPadGrad")                   \
+                              .Device(DEVICE_GPU)                 \
+                              .TypeConstraint<T>("T")             \
+                              .TypeConstraint<int64>("Tpaddings") \
+                              .HostMemory("paddings"),            \
+                          MirrorPadGradOp<GpuDevice, T, int64>);
 
 TF_CALL_GPU_NUMBER_TYPES(REGISTER_GPU_KERNEL);
 #undef REGISTER_GPU_KERNEL
diff --git a/tensorflow/core/kernels/mirror_pad_op.h b/tensorflow/core/kernels/mirror_pad_op.h
index b83d2223d0..81150a9e79 100644
--- a/tensorflow/core/kernels/mirror_pad_op.h
+++ b/tensorflow/core/kernels/mirror_pad_op.h
@@ -64,9 +64,8 @@ class TensorMirrorPadOp
       StorageKind;
   typedef typename Eigen::internal::traits<TensorMirrorPadOp>::Index Index;
 
-  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
-  TensorMirrorPadOp(const XprType& expr, const PaddingDimensions& padding_dims,
-                    Index offset)
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorMirrorPadOp(
+      const XprType& expr, const PaddingDimensions& padding_dims, Index offset)
       : xpr_(expr), padding_dims_(padding_dims), offset_(offset) {}
 
   EIGEN_DEVICE_FUNC
@@ -336,12 +335,12 @@ namespace functor {
 
 // offset argument must be either 0 or 1. This controls whether the boundary
 // values are replicated (offset == 0) or not replicated (offset == 1).
-template <typename Device, typename T, int Dims>
+template <typename Device, typename T, typename Tpaddings, int Dims>
 struct MirrorPad {
   void operator()(const Device& device,
                   typename TTypes<T, Dims, int32>::Tensor output,
                   typename TTypes<T, Dims, int32>::ConstTensor input,
-                  TTypes<int32>::ConstMatrix padding, int offset) {
+                  typename TTypes<Tpaddings>::ConstMatrix padding, int offset) {
     Eigen::array<Eigen::IndexPair<int32>, Dims> padding_dims;
 
     for (int i = 0; i < Dims; ++i) {
@@ -363,12 +362,12 @@ struct MirrorPad {
 
 // offset argument must be either 0 or 1. This controls whether the boundary
 // values are replicated (offset == 0) or not replicated (offset == 1).
-template <typename Device, typename T, int Dims>
+template <typename Device, typename T, typename Tpaddings, int Dims>
 struct MirrorPadGrad {
   void operator()(const Device& device,
                   typename TTypes<T, Dims, int32>::Tensor output,
                   typename TTypes<T, Dims, int32>::ConstTensor input,
-                  TTypes<int32>::ConstMatrix paddings, int offset,
+                  typename TTypes<Tpaddings>::ConstMatrix paddings, int offset,
                   typename TTypes<T, Dims, int32>::Tensor scratch) {
     // Copy the gradient input into the scratch buffer.
     scratch.device(device) = input;
diff --git a/tensorflow/core/kernels/mirror_pad_op_cpu_impl.h b/tensorflow/core/kernels/mirror_pad_op_cpu_impl.h
index 9864f5633a..bb22b2aa91 100644
--- a/tensorflow/core/kernels/mirror_pad_op_cpu_impl.h
+++ b/tensorflow/core/kernels/mirror_pad_op_cpu_impl.h
@@ -25,13 +25,17 @@ namespace tensorflow {
 
 using CpuDevice = Eigen::ThreadPoolDevice;
 
-#define DEFINE_CPU_SPECS(T) \
-  template struct functor::MirrorPad<CpuDevice, T, CPU_PROVIDED_IXDIM>;
+#define DEFINE_CPU_SPECS(T)                                                    \
+  template struct functor::MirrorPad<CpuDevice, T, int32, CPU_PROVIDED_IXDIM>; \
+  template struct functor::MirrorPad<CpuDevice, T, int64, CPU_PROVIDED_IXDIM>;
 TF_CALL_POD_TYPES(DEFINE_CPU_SPECS);
 #undef DEFINE_CPU_SPECS
 
-#define DEFINE_CPU_SPECS(T) \
-  template struct functor::MirrorPadGrad<CpuDevice, T, CPU_PROVIDED_IXDIM>;
+#define DEFINE_CPU_SPECS(T)                                   \
+  template struct functor::MirrorPadGrad<CpuDevice, T, int32, \
+                                         CPU_PROVIDED_IXDIM>; \
+  template struct functor::MirrorPadGrad<CpuDevice, T, int64, \
+                                         CPU_PROVIDED_IXDIM>;
 TF_CALL_NUMBER_TYPES(DEFINE_CPU_SPECS);
 #undef DEFINE_CPU_SPECS
 
diff --git a/tensorflow/core/kernels/mirror_pad_op_gpu.cu.cc b/tensorflow/core/kernels/mirror_pad_op_gpu.cu.cc
index 8074aa9624..dbd0a9bd8f 100644
--- a/tensorflow/core/kernels/mirror_pad_op_gpu.cu.cc
+++ b/tensorflow/core/kernels/mirror_pad_op_gpu.cu.cc
@@ -25,17 +25,27 @@ namespace tensorflow {
 
 using GpuDevice = Eigen::GpuDevice;
 
-#define DEFINE_GPU_SPECS(T)                                \
-  template struct functor::MirrorPad<GpuDevice, T, 1>;     \
-  template struct functor::MirrorPad<GpuDevice, T, 2>;     \
-  template struct functor::MirrorPad<GpuDevice, T, 3>;     \
-  template struct functor::MirrorPad<GpuDevice, T, 4>;     \
-  template struct functor::MirrorPad<GpuDevice, T, 5>;     \
-  template struct functor::MirrorPadGrad<GpuDevice, T, 1>; \
-  template struct functor::MirrorPadGrad<GpuDevice, T, 2>; \
-  template struct functor::MirrorPadGrad<GpuDevice, T, 3>; \
-  template struct functor::MirrorPadGrad<GpuDevice, T, 4>; \
-  template struct functor::MirrorPadGrad<GpuDevice, T, 5>;
+#define DEFINE_GPU_SPECS(T)                                       \
+  template struct functor::MirrorPad<GpuDevice, T, int32, 1>;     \
+  template struct functor::MirrorPad<GpuDevice, T, int32, 2>;     \
+  template struct functor::MirrorPad<GpuDevice, T, int32, 3>;     \
+  template struct functor::MirrorPad<GpuDevice, T, int32, 4>;     \
+  template struct functor::MirrorPad<GpuDevice, T, int32, 5>;     \
+  template struct functor::MirrorPad<GpuDevice, T, int64, 1>;     \
+  template struct functor::MirrorPad<GpuDevice, T, int64, 2>;     \
+  template struct functor::MirrorPad<GpuDevice, T, int64, 3>;     \
+  template struct functor::MirrorPad<GpuDevice, T, int64, 4>;     \
+  template struct functor::MirrorPad<GpuDevice, T, int64, 5>;     \
+  template struct functor::MirrorPadGrad<GpuDevice, T, int32, 1>; \
+  template struct functor::MirrorPadGrad<GpuDevice, T, int32, 2>; \
+  template struct functor::MirrorPadGrad<GpuDevice, T, int32, 3>; \
+  template struct functor::MirrorPadGrad<GpuDevice, T, int32, 4>; \
+  template struct functor::MirrorPadGrad<GpuDevice, T, int32, 5>; \
+  template struct functor::MirrorPadGrad<GpuDevice, T, int64, 1>; \
+  template struct functor::MirrorPadGrad<GpuDevice, T, int64, 2>; \
+  template struct functor::MirrorPadGrad<GpuDevice, T, int64, 3>; \
+  template struct functor::MirrorPadGrad<GpuDevice, T, int64, 4>; \
+  template struct functor::MirrorPadGrad<GpuDevice, T, int64, 5>;
 
 TF_CALL_GPU_NUMBER_TYPES(DEFINE_GPU_SPECS);
 #undef DEFINE_GPU_SPECS
diff --git a/tensorflow/core/kernels/mkl_conv_ops.cc b/tensorflow/core/kernels/mkl_conv_ops.cc
index 57661e8b10..369f632fb4 100644
--- a/tensorflow/core/kernels/mkl_conv_ops.cc
+++ b/tensorflow/core/kernels/mkl_conv_ops.cc
@@ -288,8 +288,10 @@ class MklConv2DOp : public OpKernel {
     mkl_filter_output_mkl_shape.SetMklLayout(mkl_context.prim_fwd,
                                              dnnResourceFilter);
 
-    size_t filter_sizes[4] = {filter.dim_size(0), filter.dim_size(1),
-                              filter.dim_size(2), filter.dim_size(3)};
+    size_t filter_sizes[4] = {static_cast<size_t>(filter.dim_size(0)),
+                              static_cast<size_t>(filter.dim_size(1)),
+                              static_cast<size_t>(filter.dim_size(2)),
+                              static_cast<size_t>(filter.dim_size(3))};
     mkl_filter_output_mkl_shape.SetTfLayout(filter.dims(), filter_sizes,
                                             mkl_context.filter_strides);
 
diff --git a/tensorflow/core/kernels/nth_element_op.cc b/tensorflow/core/kernels/nth_element_op.cc
new file mode 100644
index 0000000000..da825e408c
--- /dev/null
+++ b/tensorflow/core/kernels/nth_element_op.cc
@@ -0,0 +1,139 @@
+/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+// See docs in ../ops/nn_ops.cc.
+#include "tensorflow/core/kernels/nth_element_op.h"
+
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/framework/register_types.h"
+#include "tensorflow/core/framework/types.h"
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/platform/logging.h"
+#include "tensorflow/core/util/work_sharder.h"
+#include <vector>
+#include <algorithm>
+#include <iostream>
+
+namespace tensorflow {
+
+typedef Eigen::ThreadPoolDevice CPUDevice;
+
+template <typename Device, typename T>
+class NthElementOp : public OpKernel {
+ public:
+  explicit NthElementOp(OpKernelConstruction* context) : OpKernel(context) {
+    OP_REQUIRES_OK(context, context->GetAttr("reverse", &reverse_));
+  }
+
+  void Compute(OpKernelContext* context) override {
+    // The second args is N, which must be a positive scalar.
+    const auto& n_in = context->input(1);
+    OP_REQUIRES(context, TensorShapeUtils::IsScalar(n_in.shape()),
+                errors::InvalidArgument("N must be scalar, got shape ",
+                                        n_in.shape().DebugString()));
+    int n = n_in.scalar<int32>()();
+    OP_REQUIRES(context, n >= 0,
+                errors::InvalidArgument("Need n >= 0, got ", n));
+
+    // The first args is input tensor, which must have 1 dimension at least.
+    const Tensor& input_in = context->input(0);
+    const int num_dims = input_in.dims();
+    OP_REQUIRES(context, num_dims >= 1,
+                errors::InvalidArgument("Input must be >= 1-D, got shape ",
+                                        input_in.shape().DebugString()));
+    // The last dimension of input tensor must be greater than N.
+    OP_REQUIRES(context, input_in.dim_size(num_dims-1) > n,
+                errors::InvalidArgument("Input must have at least n+1 columns"));
+
+    // std::nth_element only support the nth-smallest selection.
+    if (reverse_) {
+      n = input_in.dim_size(num_dims - 1) - n - 1;
+    }
+
+    // Assume input_shape is [d1,d2,...dk], and output_shape is [d1,d2...dk-1].
+    TensorShape out_shape;
+    for (int i = 0; i < num_dims-1; ++i) {
+      out_shape.AddDim(input_in.dim_size(i));
+    }
+    Tensor* output_tensor = nullptr;
+    OP_REQUIRES_OK(context,
+                   context->allocate_output(0, out_shape, &output_tensor));
+
+    functor::NthElementFunctor<Device, T> nthElementFunc;
+    nthElementFunc(context, input_in, *output_tensor, n, reverse_);
+  }
+
+ private:
+  bool reverse_;
+};
+
+namespace functor {
+
+template <typename T>
+struct NthElementFunctor<CPUDevice, T> {
+  void operator() (OpKernelContext* context,
+                   const Tensor& input_tensor,
+                   Tensor& output_tensor,
+                   int n,
+                   bool reverse) {
+    const T* input = input_tensor.flat<T>().data();
+    T* output = output_tensor.flat<T>().data();
+
+    // Assume input_shape is [d1,d2,...dk], and output_shape is [d1,d2...dk-1],
+    // then num_rows = d1*d2...dk-1, last_dim = dk.
+    const int num_rows = output_tensor.NumElements();
+    const int last_dim = input_tensor.dim_size(input_tensor.dims()-1);
+
+    // Allocate each row to different shard.
+    auto SubNthElement = [&, input, output, last_dim, n](int start,
+                                                         int limit) {
+      // std::nth_element would rearrange the array, so we need a new buffer.
+      std::vector<T> buf(last_dim);
+
+      for (int b = start; b < limit; ++b) {
+        // Copy from one row of elements to buffer
+        const T* input_start = input + b * last_dim;
+        const T* input_end = input + (b+1) * last_dim;
+        std::copy(input_start, input_end, buf.begin());
+
+        std::nth_element(buf.begin(), buf.begin()+n, buf.end());
+        // The element placed in the nth position is exactly the element that
+        // would occur in this position if the range was fully sorted.
+        output[b] = buf[n];
+      }
+    };
+
+    auto worker_threads = *(context->device()->tensorflow_cpu_worker_threads());
+    // The average time complexity of partition-based nth_element (BFPRT) is O(n),
+    // althought the worst time complexity could be O(n^2).
+    // Here, 20 is a empirical factor of cost_per_unit.
+    Shard(worker_threads.num_threads, worker_threads.workers, num_rows,
+          20 * last_dim, SubNthElement);
+  }
+};
+
+}  // namespace functor
+
+
+#define REGISTER_NTHOP(T)                                           \
+  REGISTER_KERNEL_BUILDER(                                          \
+      Name("NthElement").Device(DEVICE_CPU).TypeConstraint<T>("T"), \
+      NthElementOp<CPUDevice, T>)
+
+TF_CALL_REAL_NUMBER_TYPES(REGISTER_NTHOP);
+#undef REGISTER_NTHOP
+
+}  // end namespace tensorflow
+
diff --git a/tensorflow/core/kernels/nth_element_op.h b/tensorflow/core/kernels/nth_element_op.h
new file mode 100644
index 0000000000..11a6c996b0
--- /dev/null
+++ b/tensorflow/core/kernels/nth_element_op.h
@@ -0,0 +1,39 @@
+/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_NTH_ELEMENT_OP_H_
+#define TENSORFLOW_NTH_ELEMENT_OP_H_
+
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/framework/tensor_types.h"
+#include "tensorflow/core/platform/types.h"
+
+namespace tensorflow {
+
+namespace functor {
+
+template <typename Device, typename T>
+struct NthElementFunctor {
+  void operator() (OpKernelContext* context,
+                   const Tensor& input_tensor,
+                   Tensor& output_tensor,
+                   int n);
+};
+
+}  // namespace functor
+
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_NTH_ELEMENT_OP_H_
diff --git a/tensorflow/core/kernels/pad_op.cc b/tensorflow/core/kernels/pad_op.cc
index 6196c5ed93..eff3e4d92c 100644
--- a/tensorflow/core/kernels/pad_op.cc
+++ b/tensorflow/core/kernels/pad_op.cc
@@ -40,9 +40,9 @@ typedef Eigen::ThreadPoolDevice CPUDevice;
 typedef Eigen::GpuDevice GPUDevice;
 #ifdef TENSORFLOW_USE_SYCL
 typedef Eigen::SyclDevice SYCLDevice;
-#endif // TENSORFLOW_USE_SYCL
+#endif  // TENSORFLOW_USE_SYCL
 
-template <typename Device, typename T>
+template <typename Device, typename T, typename Tpadding>
 class PadOp : public OpKernel {
  public:
   explicit PadOp(OpKernelConstruction* context) : OpKernel(context) {}
@@ -82,10 +82,11 @@ class PadOp : public OpKernel {
 
     // Compute the shape of the output tensor, and allocate it.
     TensorShape output_shape;
-    TTypes<int32>::ConstMatrix paddings = in1.matrix<int32>();
+    typename TTypes<Tpadding>::ConstMatrix paddings = in1.matrix<Tpadding>();
     for (int d = 0; d < fixed_dims; ++d) {
-      const int32 before_d = paddings(d, 0);  // Pad before existing elements.
-      const int32 after_d = paddings(d, 1);   // Pad after existing elements.
+      const Tpadding before_d =
+          paddings(d, 0);                       // Pad before existing elements.
+      const Tpadding after_d = paddings(d, 1);  // Pad after existing elements.
       OP_REQUIRES(context, before_d >= 0 && after_d >= 0,
                   errors::InvalidArgument("Paddings must be non-negative: ",
                                           before_d, " ", after_d));
@@ -142,32 +143,47 @@ class PadOp : public OpKernel {
   template <int Dims>
   void Operate(OpKernelContext* context,
                typename TTypes<T, Dims>::ConstTensor input,
-               TTypes<int32>::ConstMatrix paddings, T pad_value,
+               typename TTypes<Tpadding>::ConstMatrix paddings, T pad_value,
                Tensor* output) {
     CHECK_EQ(Dims, paddings.dimension(0));
     CHECK_EQ(2, paddings.dimension(1));
-    Eigen::array<Eigen::IndexPair<int32>, Dims> paddings_array;
+    Eigen::array<Eigen::IndexPair<Tpadding>, Dims> paddings_array;
     for (int i = 0; i < Dims; ++i) {
       paddings_array[i] = {paddings(i, 0), paddings(i, 1)};
     }
-    functor::Pad<Device, T, Dims> functor;
+    functor::Pad<Device, T, Tpadding, Dims> functor;
     functor(context->eigen_device<Device>(), output->tensor<T, Dims>(), input,
             paddings_array, pad_value);
   }
 };
 
-#define REGISTER_KERNEL(type)                                 \
-  REGISTER_KERNEL_BUILDER(Name("Pad")                         \
-                              .Device(DEVICE_CPU)             \
-                              .TypeConstraint<type>("T")      \
-                              .HostMemory("paddings"),        \
-                          PadOp<CPUDevice, type>);            \
-  REGISTER_KERNEL_BUILDER(Name("PadV2")                       \
-                              .Device(DEVICE_CPU)             \
-                              .TypeConstraint<type>("T")      \
-                              .HostMemory("paddings")         \
-                              .HostMemory("constant_values"), \
-                          PadOp<CPUDevice, type>);
+#define REGISTER_KERNEL(type)                                     \
+  REGISTER_KERNEL_BUILDER(Name("Pad")                             \
+                              .Device(DEVICE_CPU)                 \
+                              .TypeConstraint<type>("T")          \
+                              .TypeConstraint<int32>("Tpaddings") \
+                              .HostMemory("paddings"),            \
+                          PadOp<CPUDevice, type, int32>);         \
+  REGISTER_KERNEL_BUILDER(Name("Pad")                             \
+                              .Device(DEVICE_CPU)                 \
+                              .TypeConstraint<type>("T")          \
+                              .TypeConstraint<int64>("Tpaddings") \
+                              .HostMemory("paddings"),            \
+                          PadOp<CPUDevice, type, int64>);         \
+  REGISTER_KERNEL_BUILDER(Name("PadV2")                           \
+                              .Device(DEVICE_CPU)                 \
+                              .TypeConstraint<type>("T")          \
+                              .TypeConstraint<int32>("Tpaddings") \
+                              .HostMemory("paddings")             \
+                              .HostMemory("constant_values"),     \
+                          PadOp<CPUDevice, type, int32>);         \
+  REGISTER_KERNEL_BUILDER(Name("PadV2")                           \
+                              .Device(DEVICE_CPU)                 \
+                              .TypeConstraint<type>("T")          \
+                              .TypeConstraint<int64>("Tpaddings") \
+                              .HostMemory("paddings")             \
+                              .HostMemory("constant_values"),     \
+                          PadOp<CPUDevice, type, int64>);
 
 TF_CALL_POD_TYPES(REGISTER_KERNEL);
 #undef REGISTER_KERNEL
@@ -177,11 +193,17 @@ TF_CALL_POD_TYPES(REGISTER_KERNEL);
 namespace functor {
 #define DECLARE_GPU_SPEC(T, Dims)                                         \
   template <>                                                             \
-  void Pad<GPUDevice, T, Dims>::operator()(                               \
+  void Pad<GPUDevice, T, int32, Dims>::operator()(                        \
       const GPUDevice& d, typename TTypes<T, Dims>::Tensor output,        \
       typename TTypes<T, Dims>::ConstTensor input,                        \
       Eigen::array<Eigen::IndexPair<int32>, Dims> paddings, T pad_value); \
-  extern template struct Pad<GPUDevice, T, Dims>;
+  extern template struct Pad<GPUDevice, T, int32, Dims>;                  \
+  template <>                                                             \
+  void Pad<GPUDevice, T, int64, Dims>::operator()(                        \
+      const GPUDevice& d, typename TTypes<T, Dims>::Tensor output,        \
+      typename TTypes<T, Dims>::ConstTensor input,                        \
+      Eigen::array<Eigen::IndexPair<int64>, Dims> paddings, T pad_value); \
+  extern template struct Pad<GPUDevice, T, int64, Dims>;
 
 #define DECLARE_GPU_SPECS(T) \
   DECLARE_GPU_SPEC(T, 0);    \
@@ -202,14 +224,27 @@ TF_CALL_GPU_NUMBER_TYPES(DECLARE_GPU_SPECS);
                               .TypeConstraint<T>("T")             \
                               .TypeConstraint<int32>("Tpaddings") \
                               .HostMemory("paddings"),            \
-                          PadOp<GPUDevice, T>);                   \
+                          PadOp<GPUDevice, T, int32>);            \
+  REGISTER_KERNEL_BUILDER(Name("Pad")                             \
+                              .Device(DEVICE_GPU)                 \
+                              .TypeConstraint<T>("T")             \
+                              .TypeConstraint<int64>("Tpaddings") \
+                              .HostMemory("paddings"),            \
+                          PadOp<GPUDevice, T, int64>);            \
   REGISTER_KERNEL_BUILDER(Name("PadV2")                           \
                               .Device(DEVICE_GPU)                 \
                               .TypeConstraint<T>("T")             \
                               .TypeConstraint<int32>("Tpaddings") \
                               .HostMemory("paddings")             \
                               .HostMemory("constant_values"),     \
-                          PadOp<GPUDevice, T>)
+                          PadOp<GPUDevice, T, int32>)             \
+  REGISTER_KERNEL_BUILDER(Name("PadV2")                           \
+                              .Device(DEVICE_GPU)                 \
+                              .TypeConstraint<T>("T")             \
+                              .TypeConstraint<int64>("Tpaddings") \
+                              .HostMemory("paddings")             \
+                              .HostMemory("constant_values"),     \
+                          PadOp<GPUDevice, T, int64>)
 
 TF_CALL_GPU_NUMBER_TYPES(REGISTER_GPU_KERNEL);
 
@@ -223,7 +258,15 @@ REGISTER_KERNEL_BUILDER(Name("Pad")
                             .HostMemory("input")
                             .HostMemory("paddings")
                             .HostMemory("output"),
-                        PadOp<CPUDevice, int32>);
+                        PadOp<CPUDevice, int32, int32>);
+REGISTER_KERNEL_BUILDER(Name("Pad")
+                            .Device(DEVICE_GPU)
+                            .TypeConstraint<int32>("T")
+                            .TypeConstraint<int64>("Tpaddings")
+                            .HostMemory("input")
+                            .HostMemory("paddings")
+                            .HostMemory("output"),
+                        PadOp<CPUDevice, int32, int64>);
 REGISTER_KERNEL_BUILDER(Name("PadV2")
                             .Device(DEVICE_GPU)
                             .TypeConstraint<int32>("T")
@@ -232,7 +275,16 @@ REGISTER_KERNEL_BUILDER(Name("PadV2")
                             .HostMemory("paddings")
                             .HostMemory("constant_values")
                             .HostMemory("output"),
-                        PadOp<CPUDevice, int32>);
+                        PadOp<CPUDevice, int32, int32>);
+REGISTER_KERNEL_BUILDER(Name("PadV2")
+                            .Device(DEVICE_GPU)
+                            .TypeConstraint<int32>("T")
+                            .TypeConstraint<int64>("Tpaddings")
+                            .HostMemory("input")
+                            .HostMemory("paddings")
+                            .HostMemory("constant_values")
+                            .HostMemory("output"),
+                        PadOp<CPUDevice, int32, int64>);
 #endif
 
 #ifdef TENSORFLOW_USE_SYCL
@@ -243,14 +295,27 @@ REGISTER_KERNEL_BUILDER(Name("PadV2")
                               .TypeConstraint<T>("T")             \
                               .TypeConstraint<int32>("Tpaddings") \
                               .HostMemory("paddings"),            \
-                          PadOp<SYCLDevice, T>);                  \
+                          PadOp<SYCLDevice, T, int32>);           \
+  REGISTER_KERNEL_BUILDER(Name("Pad")                             \
+                              .Device(DEVICE_SYCL)                \
+                              .TypeConstraint<T>("T")             \
+                              .TypeConstraint<int64>("Tpaddings") \
+                              .HostMemory("paddings"),            \
+                          PadOp<SYCLDevice, T, int64>);           \
   REGISTER_KERNEL_BUILDER(Name("PadV2")                           \
                               .Device(DEVICE_SYCL)                \
                               .TypeConstraint<T>("T")             \
                               .TypeConstraint<int32>("Tpaddings") \
                               .HostMemory("paddings")             \
                               .HostMemory("constant_values"),     \
-                          PadOp<SYCLDevice, T>)
+                          PadOp<SYCLDevice, T, int32>)            \
+  REGISTER_KERNEL_BUILDER(Name("PadV2")                           \
+                              .Device(DEVICE_SYCL)                \
+                              .TypeConstraint<T>("T")             \
+                              .TypeConstraint<int64>("Tpaddings") \
+                              .HostMemory("paddings")             \
+                              .HostMemory("constant_values"),     \
+                          PadOp<SYCLDevice, T, int64>)
 
 TF_CALL_GPU_NUMBER_TYPES_NO_HALF(REGISTER_SYCL_KERNEL);
 REGISTER_KERNEL_BUILDER(Name("Pad")
@@ -260,7 +325,15 @@ REGISTER_KERNEL_BUILDER(Name("Pad")
                             .HostMemory("input")
                             .HostMemory("paddings")
                             .HostMemory("output"),
-                        PadOp<CPUDevice, int32>);
+                        PadOp<CPUDevice, int32, int32>);
+REGISTER_KERNEL_BUILDER(Name("Pad")
+                            .Device(DEVICE_SYCL)
+                            .TypeConstraint<int32>("T")
+                            .TypeConstraint<int64>("Tpaddings")
+                            .HostMemory("input")
+                            .HostMemory("paddings")
+                            .HostMemory("output"),
+                        PadOp<CPUDevice, int32, int64>);
 REGISTER_KERNEL_BUILDER(Name("PadV2")
                             .Device(DEVICE_SYCL)
                             .TypeConstraint<int32>("T")
@@ -269,8 +342,17 @@ REGISTER_KERNEL_BUILDER(Name("PadV2")
                             .HostMemory("paddings")
                             .HostMemory("constant_values")
                             .HostMemory("output"),
-                        PadOp<CPUDevice, int32>);
+                        PadOp<CPUDevice, int32, int32>);
+REGISTER_KERNEL_BUILDER(Name("PadV2")
+                            .Device(DEVICE_SYCL)
+                            .TypeConstraint<int32>("T")
+                            .TypeConstraint<int64>("Tpaddings")
+                            .HostMemory("input")
+                            .HostMemory("paddings")
+                            .HostMemory("constant_values")
+                            .HostMemory("output"),
+                        PadOp<CPUDevice, int32, int64>);
 #undef REGISTER_SYCL_KERNEL
-#endif // TENSORFLOW_USE_SYCL
+#endif  // TENSORFLOW_USE_SYCL
 
 }  // end namespace tensorflow
diff --git a/tensorflow/core/kernels/pad_op.h b/tensorflow/core/kernels/pad_op.h
index 95a7c9a3ae..ee9e0f0330 100644
--- a/tensorflow/core/kernels/pad_op.h
+++ b/tensorflow/core/kernels/pad_op.h
@@ -25,13 +25,13 @@ namespace tensorflow {
 namespace functor {
 
 // Functor used by PadOp to do the computations.
-template <typename Device, typename T, int Dims>
+template <typename Device, typename T, typename Tpadding, int Dims>
 struct Pad {
   // Pad "input" into "output", as specified by "paddings" and "pad_value".
   // See pad_op.cc for details.
   void operator()(const Device& d, typename TTypes<T, Dims>::Tensor output,
                   typename TTypes<T, Dims>::ConstTensor input,
-                  Eigen::array<Eigen::IndexPair<int32>, Dims> paddings,
+                  Eigen::array<Eigen::IndexPair<Tpadding>, Dims> paddings,
                   T pad_value) {
     if (Eigen::internal::is_same<Device, Eigen::GpuDevice>::value &&
         (output.size() <= std::numeric_limits<int32>::max())) {
@@ -42,12 +42,12 @@ struct Pad {
   }
 };
 
-template <typename Device, typename T>
-struct Pad<Device, T, 0> {
+template <typename Device, typename T, typename Tpadding>
+struct Pad<Device, T, Tpadding, 0> {
   // In the scalar case we simply copy the input.
   void operator()(const Device& d, typename TTypes<T, 0>::Tensor output,
                   typename TTypes<T, 0>::ConstTensor input,
-                  Eigen::array<Eigen::IndexPair<int32>, 0>, T) {
+                  Eigen::array<Eigen::IndexPair<Tpadding>, 0>, T) {
     output.device(d) = input;
   }
 };
diff --git a/tensorflow/core/kernels/pad_op_gpu.cu.cc b/tensorflow/core/kernels/pad_op_gpu.cu.cc
index f98631df17..613ad62825 100644
--- a/tensorflow/core/kernels/pad_op_gpu.cu.cc
+++ b/tensorflow/core/kernels/pad_op_gpu.cu.cc
@@ -26,14 +26,18 @@ namespace tensorflow {
 typedef Eigen::GpuDevice GPUDevice;
 
 // Definition of the GPU implementations declared in pad_op.cc.
-#define DEFINE_GPU_SPECS(T)                      \
-  template struct functor::Pad<GPUDevice, T, 0>; \
-  template struct functor::Pad<GPUDevice, T, 1>; \
-  template struct functor::Pad<GPUDevice, T, 2>; \
-  template struct functor::Pad<GPUDevice, T, 3>; \
-  template struct functor::Pad<GPUDevice, T, 4>; \
-  template struct functor::Pad<GPUDevice, T, 5>; \
-  template struct functor::Pad<GPUDevice, T, 6>;
+#define DEFINE_GPU_PAD_SPECS(T, Tpadding)                  \
+  template struct functor::Pad<GPUDevice, T, Tpadding, 0>; \
+  template struct functor::Pad<GPUDevice, T, Tpadding, 1>; \
+  template struct functor::Pad<GPUDevice, T, Tpadding, 2>; \
+  template struct functor::Pad<GPUDevice, T, Tpadding, 3>; \
+  template struct functor::Pad<GPUDevice, T, Tpadding, 4>; \
+  template struct functor::Pad<GPUDevice, T, Tpadding, 5>; \
+  template struct functor::Pad<GPUDevice, T, Tpadding, 6>;
+
+#define DEFINE_GPU_SPECS(T)      \
+  DEFINE_GPU_PAD_SPECS(T, int32) \
+  DEFINE_GPU_PAD_SPECS(T, int64)
 
 TF_CALL_GPU_NUMBER_TYPES(DEFINE_GPU_SPECS);
 
diff --git a/tensorflow/core/kernels/reduction_ops_all.cc b/tensorflow/core/kernels/reduction_ops_all.cc
index 41abc2b957..4a34c4ef51 100644
--- a/tensorflow/core/kernels/reduction_ops_all.cc
+++ b/tensorflow/core/kernels/reduction_ops_all.cc
@@ -22,7 +22,13 @@ REGISTER_KERNEL_BUILDER(
         .TypeConstraint<int32>("Tidx")
         .Device(DEVICE_CPU)
         .HostMemory("reduction_indices"),
-    ReductionOp<CPUDevice, bool, Eigen::internal::AndReducer>);
+    ReductionOp<CPUDevice, bool, int32, Eigen::internal::AndReducer>);
+REGISTER_KERNEL_BUILDER(
+    Name("All")
+        .TypeConstraint<int64>("Tidx")
+        .Device(DEVICE_CPU)
+        .HostMemory("reduction_indices"),
+    ReductionOp<CPUDevice, bool, int64, Eigen::internal::AndReducer>);
 
 #if GOOGLE_CUDA
 REGISTER_KERNEL_BUILDER(
@@ -30,7 +36,13 @@ REGISTER_KERNEL_BUILDER(
         .TypeConstraint<int32>("Tidx")
         .Device(DEVICE_GPU)
         .HostMemory("reduction_indices"),
-    ReductionOp<GPUDevice, bool, Eigen::internal::AndReducer>);
+    ReductionOp<GPUDevice, bool, int32, Eigen::internal::AndReducer>);
+REGISTER_KERNEL_BUILDER(
+    Name("All")
+        .TypeConstraint<int64>("Tidx")
+        .Device(DEVICE_GPU)
+        .HostMemory("reduction_indices"),
+    ReductionOp<GPUDevice, bool, int64, Eigen::internal::AndReducer>);
 #endif
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/kernels/reduction_ops_any.cc b/tensorflow/core/kernels/reduction_ops_any.cc
index a2087cc3b7..6c0519de95 100644
--- a/tensorflow/core/kernels/reduction_ops_any.cc
+++ b/tensorflow/core/kernels/reduction_ops_any.cc
@@ -22,7 +22,13 @@ REGISTER_KERNEL_BUILDER(
         .TypeConstraint<int32>("Tidx")
         .Device(DEVICE_CPU)
         .HostMemory("reduction_indices"),
-    ReductionOp<CPUDevice, bool, Eigen::internal::OrReducer>);
+    ReductionOp<CPUDevice, bool, int32, Eigen::internal::OrReducer>);
+REGISTER_KERNEL_BUILDER(
+    Name("Any")
+        .TypeConstraint<int64>("Tidx")
+        .Device(DEVICE_CPU)
+        .HostMemory("reduction_indices"),
+    ReductionOp<CPUDevice, bool, int64, Eigen::internal::OrReducer>);
 
 #if GOOGLE_CUDA
 REGISTER_KERNEL_BUILDER(
@@ -30,7 +36,13 @@ REGISTER_KERNEL_BUILDER(
         .TypeConstraint<int32>("Tidx")
         .Device(DEVICE_GPU)
         .HostMemory("reduction_indices"),
-    ReductionOp<GPUDevice, bool, Eigen::internal::OrReducer>);
+    ReductionOp<GPUDevice, bool, int32, Eigen::internal::OrReducer>);
+REGISTER_KERNEL_BUILDER(
+    Name("Any")
+        .TypeConstraint<int64>("Tidx")
+        .Device(DEVICE_GPU)
+        .HostMemory("reduction_indices"),
+    ReductionOp<GPUDevice, bool, int64, Eigen::internal::OrReducer>);
 #endif
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/kernels/reduction_ops_common.cc b/tensorflow/core/kernels/reduction_ops_common.cc
index 5eba4288ac..8daab0d6be 100644
--- a/tensorflow/core/kernels/reduction_ops_common.cc
+++ b/tensorflow/core/kernels/reduction_ops_common.cc
@@ -57,13 +57,12 @@ gtl::InlinedVector<int32, 8> ReductionHelper::permutation() {
   return perm;
 }
 
-Status ReductionHelper::Simplify(const Tensor& data, const Tensor& axis,
-                                 const bool keep_dims) {
-  // bitmap[i] indicates whether to reduce data along i-th axis.
-  gtl::InlinedVector<bool, 4> bitmap(data.dims(), false);
-  auto axis_vec = axis.flat<int32>();
+template <typename Tperm>
+Status SimplifyHelper(const Tensor& data, const Tensor& axis,
+                      gtl::InlinedVector<bool, 4>& bitmap) {
+  auto axis_vec = axis.flat<Tperm>();
   for (int64 i = 0; i < axis.NumElements(); ++i) {
-    int32 index = axis_vec(i);
+    Tperm index = axis_vec(i);
     if (index < -data.dims() || index >= data.dims()) {
       return errors::InvalidArgument("Invalid reduction dimension (", index,
                                      " for input with ", data.dims(),
@@ -72,7 +71,18 @@ Status ReductionHelper::Simplify(const Tensor& data, const Tensor& axis,
     index = (index + data.dims()) % data.dims();
     bitmap[index] = true;
   }
+  return Status::OK();
+}
 
+Status ReductionHelper::Simplify(const Tensor& data, const Tensor& axis,
+                                 const bool keep_dims) {
+  // bitmap[i] indicates whether to reduce data along i-th axis.
+  gtl::InlinedVector<bool, 4> bitmap(data.dims(), false);
+  if (axis.dtype() == DT_INT32) {
+    TF_RETURN_IF_ERROR(SimplifyHelper<int32>(data, axis, bitmap));
+  } else {
+    TF_RETURN_IF_ERROR(SimplifyHelper<int64>(data, axis, bitmap));
+  }
   // Output tensor's dim sizes.
   out_shape_.clear();
   for (int i = 0; i < data.dims(); ++i) {
diff --git a/tensorflow/core/kernels/reduction_ops_common.h b/tensorflow/core/kernels/reduction_ops_common.h
index 71af9d88dc..9da992ccd1 100644
--- a/tensorflow/core/kernels/reduction_ops_common.h
+++ b/tensorflow/core/kernels/reduction_ops_common.h
@@ -25,6 +25,7 @@ limitations under the License.
 
 #include "third_party/eigen3/Eigen/Core"
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+
 #include "tensorflow/core/framework/numeric_op.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
@@ -42,7 +43,7 @@ typedef Eigen::ThreadPoolDevice CPUDevice;
 typedef Eigen::GpuDevice GPUDevice;
 #ifdef TENSORFLOW_USE_SYCL
 typedef Eigen::SyclDevice SYCLDevice;
-#endif // TENSORFLOW_USE_SYCL
+#endif  // TENSORFLOW_USE_SYCL
 
 template <typename Device>
 struct Constants {
@@ -68,11 +69,13 @@ struct ConstantsBase {
   const Eigen::IndexList<Eigen::type2index<1>> kOne;
   const Eigen::IndexList<Eigen::type2index<0>, Eigen::type2index<2>> kZeroTwo;
 };
-template<> struct Constants<CPUDevice> : ConstantsBase{};
+template <>
+struct Constants<CPUDevice> : ConstantsBase {};
 #ifdef TENSORFLOW_USE_SYCL
-template<> struct Constants<SYCLDevice> : ConstantsBase{};
-#endif // TENSORFLOW_USE_SYCL
-#endif // EIGEN_HAS_INDEX_LIST
+template <>
+struct Constants<SYCLDevice> : ConstantsBase {};
+#endif  // TENSORFLOW_USE_SYCL
+#endif  // EIGEN_HAS_INDEX_LIST
 
 class ReductionHelper {
  public:
@@ -131,12 +134,13 @@ class ReductionHelper {
 
 // For operations where the output is a reduction function along some
 // dimensions of the input.
-template <typename Device, class T, typename Reducer>
+template <typename Device, class T, typename Tperm, typename Reducer>
 class ReductionOp : public OpKernel {
  public:
   explicit ReductionOp(OpKernelConstruction* ctx) : OpKernel(ctx) {
     const DataType dt = DataTypeToEnum<T>::v();
-    OP_REQUIRES_OK(ctx, ctx->MatchSignature({dt, DT_INT32}, {dt}));
+    const DataType pt = DataTypeToEnum<Tperm>::v();
+    OP_REQUIRES_OK(ctx, ctx->MatchSignature({dt, pt}, {dt}));
 
     OP_REQUIRES_OK(ctx, ctx->GetAttr("keep_dims", &keep_dims_));
   }
@@ -266,20 +270,19 @@ struct ReduceFunctorBase {
   }
 
   template <typename OUT_T>
-  static void FillIdentity(const Device& d, OUT_T out,
-                           const Reducer& reducer) {
+  static void FillIdentity(const Device& d, OUT_T out, const Reducer& reducer) {
     FillIdentityEigenImpl(d, out, reducer);
   }
 };
 
 template <typename Reducer>
 struct ReduceFunctor<CPUDevice, Reducer>
-        : ReduceFunctorBase<CPUDevice, Reducer>{};
+    : ReduceFunctorBase<CPUDevice, Reducer> {};
 #if TENSORFLOW_USE_SYCL
 template <typename Reducer>
 struct ReduceFunctor<SYCLDevice, Reducer>
-        : ReduceFunctorBase<SYCLDevice, Reducer>{};
-#endif // TENSORFLOW_USE_SYCL
+    : ReduceFunctorBase<SYCLDevice, Reducer> {};
+#endif  // TENSORFLOW_USE_SYCL
 
 }  // namespace functor
 }  // namespace tensorflow
diff --git a/tensorflow/core/kernels/reduction_ops_max.cc b/tensorflow/core/kernels/reduction_ops_max.cc
index 4ca5c11a48..9cf953f4bf 100644
--- a/tensorflow/core/kernels/reduction_ops_max.cc
+++ b/tensorflow/core/kernels/reduction_ops_max.cc
@@ -17,26 +17,39 @@ limitations under the License.
 
 namespace tensorflow {
 
-#define REGISTER_CPU_KERNELS(type)        \
-  REGISTER_KERNEL_BUILDER(                \
-      Name("Max")                         \
-          .Device(DEVICE_CPU)             \
-          .TypeConstraint<type>("T")      \
-          .TypeConstraint<int32>("Tidx"), \
-      ReductionOp<CPUDevice, type, Eigen::internal::MaxReducer<type>>);
+#define REGISTER_CPU_KERNELS(type)                                             \
+  REGISTER_KERNEL_BUILDER(                                                     \
+      Name("Max")                                                              \
+          .Device(DEVICE_CPU)                                                  \
+          .TypeConstraint<type>("T")                                           \
+          .TypeConstraint<int32>("Tidx"),                                      \
+      ReductionOp<CPUDevice, type, int32, Eigen::internal::MaxReducer<type>>); \
+  REGISTER_KERNEL_BUILDER(                                                     \
+      Name("Max")                                                              \
+          .Device(DEVICE_CPU)                                                  \
+          .TypeConstraint<type>("T")                                           \
+          .TypeConstraint<int64>("Tidx"),                                      \
+      ReductionOp<CPUDevice, type, int64, Eigen::internal::MaxReducer<type>>);
 TF_CALL_REAL_NUMBER_TYPES(REGISTER_CPU_KERNELS);
 #undef REGISTER_CPU_KERNELS
 
 #if GOOGLE_CUDA
 
-#define REGISTER_GPU_KERNELS(type)          \
-  REGISTER_KERNEL_BUILDER(                  \
-      Name("Max")                           \
-          .Device(DEVICE_GPU)               \
-          .TypeConstraint<type>("T")        \
-          .TypeConstraint<int32>("Tidx")    \
-          .HostMemory("reduction_indices"), \
-      ReductionOp<GPUDevice, type, Eigen::internal::MaxReducer<type>>);
+#define REGISTER_GPU_KERNELS(type)                                             \
+  REGISTER_KERNEL_BUILDER(                                                     \
+      Name("Max")                                                              \
+          .Device(DEVICE_GPU)                                                  \
+          .TypeConstraint<type>("T")                                           \
+          .TypeConstraint<int32>("Tidx")                                       \
+          .HostMemory("reduction_indices"),                                    \
+      ReductionOp<GPUDevice, type, int32, Eigen::internal::MaxReducer<type>>); \
+  REGISTER_KERNEL_BUILDER(                                                     \
+      Name("Max")                                                              \
+          .Device(DEVICE_GPU)                                                  \
+          .TypeConstraint<type>("T")                                           \
+          .TypeConstraint<int64>("Tidx")                                       \
+          .HostMemory("reduction_indices"),                                    \
+      ReductionOp<GPUDevice, type, int64, Eigen::internal::MaxReducer<type>>);
 REGISTER_GPU_KERNELS(float);
 REGISTER_GPU_KERNELS(double);
 REGISTER_GPU_KERNELS(int64);
@@ -52,21 +65,37 @@ REGISTER_KERNEL_BUILDER(
         .HostMemory("output")
         .TypeConstraint<int32>("T")
         .TypeConstraint<int32>("Tidx"),
-    ReductionOp<CPUDevice, int32, Eigen::internal::MaxReducer<int32>>);
+    ReductionOp<CPUDevice, int32, int32, Eigen::internal::MaxReducer<int32>>);
+REGISTER_KERNEL_BUILDER(
+    Name("Max")
+        .Device(DEVICE_GPU)
+        .HostMemory("reduction_indices")
+        .HostMemory("input")
+        .HostMemory("output")
+        .TypeConstraint<int32>("T")
+        .TypeConstraint<int64>("Tidx"),
+    ReductionOp<CPUDevice, int32, int64, Eigen::internal::MaxReducer<int32>>);
 
 #undef REGISTER_GPU_KERNELS
 
 #endif
 
 #ifdef TENSORFLOW_USE_SYCL
-#define REGISTER_SYCL_KERNELS(type)         \
-  REGISTER_KERNEL_BUILDER(                  \
-      Name("Max")                           \
-          .Device(DEVICE_SYCL)              \
-          .TypeConstraint<type>("T")        \
-          .TypeConstraint<int32>("Tidx")    \
-          .HostMemory("reduction_indices"), \
-      ReductionOp<SYCLDevice, type, Eigen::internal::MaxReducer<type>>);
+#define REGISTER_SYCL_KERNELS(type)                                        \
+  REGISTER_KERNEL_BUILDER(Name("Max")                                      \
+                              .Device(DEVICE_SYCL)                         \
+                              .TypeConstraint<type>("T")                   \
+                              .TypeConstraint<int32>("Tidx")               \
+                              .HostMemory("reduction_indices"),            \
+                          ReductionOp<SYCLDevice, type, int32,             \
+                                      Eigen::internal::MaxReducer<type>>); \
+  REGISTER_KERNEL_BUILDER(Name("Max")                                      \
+                              .Device(DEVICE_SYCL)                         \
+                              .TypeConstraint<type>("T")                   \
+                              .TypeConstraint<int64>("Tidx")               \
+                              .HostMemory("reduction_indices"),            \
+                          ReductionOp<SYCLDevice, type, int64,             \
+                                      Eigen::internal::MaxReducer<type>>);
 REGISTER_SYCL_KERNELS(float);
 REGISTER_SYCL_KERNELS(double);
 
@@ -78,8 +107,17 @@ REGISTER_KERNEL_BUILDER(
         .HostMemory("output")
         .TypeConstraint<int32>("T")
         .TypeConstraint<int32>("Tidx"),
-    ReductionOp<CPUDevice, int32, Eigen::internal::MaxReducer<int32>>);
+    ReductionOp<CPUDevice, int32, int32, Eigen::internal::MaxReducer<int32>>);
+REGISTER_KERNEL_BUILDER(
+    Name("Max")
+        .Device(DEVICE_SYCL)
+        .HostMemory("reduction_indices")
+        .HostMemory("input")
+        .HostMemory("output")
+        .TypeConstraint<int32>("T")
+        .TypeConstraint<int64>("Tidx"),
+    ReductionOp<CPUDevice, int32, int64, Eigen::internal::MaxReducer<int32>>);
 #undef REGISTER_SYCL_KERNELS
-#endif // TENSORFLOW_USE_SYCL
+#endif  // TENSORFLOW_USE_SYCL
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/kernels/reduction_ops_mean.cc b/tensorflow/core/kernels/reduction_ops_mean.cc
index 5b01de8ddb..f61589f913 100644
--- a/tensorflow/core/kernels/reduction_ops_mean.cc
+++ b/tensorflow/core/kernels/reduction_ops_mean.cc
@@ -17,26 +17,39 @@ limitations under the License.
 
 namespace tensorflow {
 
-#define REGISTER_CPU_KERNELS(type)        \
-  REGISTER_KERNEL_BUILDER(                \
-      Name("Mean")                        \
-          .Device(DEVICE_CPU)             \
-          .TypeConstraint<type>("T")      \
-          .TypeConstraint<int32>("Tidx"), \
-      ReductionOp<CPUDevice, type, Eigen::internal::MeanReducer<type>>);
+#define REGISTER_CPU_KERNELS(type)                                          \
+  REGISTER_KERNEL_BUILDER(Name("Mean")                                      \
+                              .Device(DEVICE_CPU)                           \
+                              .TypeConstraint<type>("T")                    \
+                              .TypeConstraint<int32>("Tidx"),               \
+                          ReductionOp<CPUDevice, type, int32,               \
+                                      Eigen::internal::MeanReducer<type>>); \
+  REGISTER_KERNEL_BUILDER(Name("Mean")                                      \
+                              .Device(DEVICE_CPU)                           \
+                              .TypeConstraint<type>("T")                    \
+                              .TypeConstraint<int64>("Tidx"),               \
+                          ReductionOp<CPUDevice, type, int64,               \
+                                      Eigen::internal::MeanReducer<type>>);
 TF_CALL_NUMBER_TYPES(REGISTER_CPU_KERNELS);
 #undef REGISTER_CPU_KERNELS
 
 #if GOOGLE_CUDA
 
-#define REGISTER_GPU_KERNELS(type)          \
-  REGISTER_KERNEL_BUILDER(                  \
-      Name("Mean")                          \
-          .Device(DEVICE_GPU)               \
-          .TypeConstraint<type>("T")        \
-          .TypeConstraint<int32>("Tidx")    \
-          .HostMemory("reduction_indices"), \
-      ReductionOp<GPUDevice, type, Eigen::internal::MeanReducer<type>>);
+#define REGISTER_GPU_KERNELS(type)                                          \
+  REGISTER_KERNEL_BUILDER(Name("Mean")                                      \
+                              .Device(DEVICE_GPU)                           \
+                              .TypeConstraint<type>("T")                    \
+                              .TypeConstraint<int32>("Tidx")                \
+                              .HostMemory("reduction_indices"),             \
+                          ReductionOp<GPUDevice, type, int32,               \
+                                      Eigen::internal::MeanReducer<type>>); \
+  REGISTER_KERNEL_BUILDER(Name("Mean")                                      \
+                              .Device(DEVICE_GPU)                           \
+                              .TypeConstraint<type>("T")                    \
+                              .TypeConstraint<int64>("Tidx")                \
+                              .HostMemory("reduction_indices"),             \
+                          ReductionOp<GPUDevice, type, int64,               \
+                                      Eigen::internal::MeanReducer<type>>);
 TF_CALL_GPU_NUMBER_TYPES(REGISTER_GPU_KERNELS);
 TF_CALL_complex64(REGISTER_GPU_KERNELS);
 TF_CALL_complex128(REGISTER_GPU_KERNELS);
@@ -45,17 +58,24 @@ TF_CALL_complex128(REGISTER_GPU_KERNELS);
 #endif
 
 #ifdef TENSORFLOW_USE_SYCL
-#define REGISTER_SYCL_KERNELS(type)         \
-  REGISTER_KERNEL_BUILDER(                  \
-      Name("Mean")                          \
-          .Device(DEVICE_SYCL)              \
-          .TypeConstraint<type>("T")        \
-          .TypeConstraint<int32>("Tidx")    \
-          .HostMemory("reduction_indices"), \
-      ReductionOp<SYCLDevice, type, Eigen::internal::MeanReducer<type>>);
+#define REGISTER_SYCL_KERNELS(type)                                         \
+  REGISTER_KERNEL_BUILDER(Name("Mean")                                      \
+                              .Device(DEVICE_SYCL)                          \
+                              .TypeConstraint<type>("T")                    \
+                              .TypeConstraint<int32>("Tidx")                \
+                              .HostMemory("reduction_indices"),             \
+                          ReductionOp<SYCLDevice, type, int32,              \
+                                      Eigen::internal::MeanReducer<type>>); \
+  REGISTER_KERNEL_BUILDER(Name("Mean")                                      \
+                              .Device(DEVICE_SYCL)                          \
+                              .TypeConstraint<type>("T")                    \
+                              .TypeConstraint<int64>("Tidx")                \
+                              .HostMemory("reduction_indices"),             \
+                          ReductionOp<SYCLDevice, type, int64,              \
+                                      Eigen::internal::MeanReducer<type>>);
 REGISTER_SYCL_KERNELS(float);
 REGISTER_SYCL_KERNELS(double);
 #undef REGISTER_SYCL_KERNELS
-#endif // TENSORFLOW_USE_SYCL
+#endif  // TENSORFLOW_USE_SYCL
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/kernels/reduction_ops_min.cc b/tensorflow/core/kernels/reduction_ops_min.cc
index 1e394bea41..807ac0a456 100644
--- a/tensorflow/core/kernels/reduction_ops_min.cc
+++ b/tensorflow/core/kernels/reduction_ops_min.cc
@@ -17,26 +17,39 @@ limitations under the License.
 
 namespace tensorflow {
 
-#define REGISTER_CPU_KERNELS(type)        \
-  REGISTER_KERNEL_BUILDER(                \
-      Name("Min")                         \
-          .Device(DEVICE_CPU)             \
-          .TypeConstraint<type>("T")      \
-          .TypeConstraint<int32>("Tidx"), \
-      ReductionOp<CPUDevice, type, Eigen::internal::MinReducer<type>>);
+#define REGISTER_CPU_KERNELS(type)                                             \
+  REGISTER_KERNEL_BUILDER(                                                     \
+      Name("Min")                                                              \
+          .Device(DEVICE_CPU)                                                  \
+          .TypeConstraint<type>("T")                                           \
+          .TypeConstraint<int32>("Tidx"),                                      \
+      ReductionOp<CPUDevice, type, int32, Eigen::internal::MinReducer<type>>); \
+  REGISTER_KERNEL_BUILDER(                                                     \
+      Name("Min")                                                              \
+          .Device(DEVICE_CPU)                                                  \
+          .TypeConstraint<type>("T")                                           \
+          .TypeConstraint<int64>("Tidx"),                                      \
+      ReductionOp<CPUDevice, type, int64, Eigen::internal::MinReducer<type>>);
 TF_CALL_REAL_NUMBER_TYPES(REGISTER_CPU_KERNELS);
 #undef REGISTER_CPU_KERNELS
 
 #if GOOGLE_CUDA
 
-#define REGISTER_GPU_KERNELS(type)          \
-  REGISTER_KERNEL_BUILDER(                  \
-      Name("Min")                           \
-          .Device(DEVICE_GPU)               \
-          .TypeConstraint<type>("T")        \
-          .TypeConstraint<int32>("Tidx")    \
-          .HostMemory("reduction_indices"), \
-      ReductionOp<GPUDevice, type, Eigen::internal::MinReducer<type>>);
+#define REGISTER_GPU_KERNELS(type)                                             \
+  REGISTER_KERNEL_BUILDER(                                                     \
+      Name("Min")                                                              \
+          .Device(DEVICE_GPU)                                                  \
+          .TypeConstraint<type>("T")                                           \
+          .TypeConstraint<int32>("Tidx")                                       \
+          .HostMemory("reduction_indices"),                                    \
+      ReductionOp<GPUDevice, type, int32, Eigen::internal::MinReducer<type>>); \
+  REGISTER_KERNEL_BUILDER(                                                     \
+      Name("Min")                                                              \
+          .Device(DEVICE_GPU)                                                  \
+          .TypeConstraint<type>("T")                                           \
+          .TypeConstraint<int64>("Tidx")                                       \
+          .HostMemory("reduction_indices"),                                    \
+      ReductionOp<GPUDevice, type, int64, Eigen::internal::MinReducer<type>>);
 REGISTER_GPU_KERNELS(float);
 REGISTER_GPU_KERNELS(double);
 
@@ -51,21 +64,37 @@ REGISTER_KERNEL_BUILDER(
         .HostMemory("output")
         .TypeConstraint<int32>("T")
         .TypeConstraint<int32>("Tidx"),
-    ReductionOp<CPUDevice, int32, Eigen::internal::MinReducer<int32>>);
+    ReductionOp<CPUDevice, int32, int32, Eigen::internal::MinReducer<int32>>);
+REGISTER_KERNEL_BUILDER(
+    Name("Min")
+        .Device(DEVICE_GPU)
+        .HostMemory("reduction_indices")
+        .HostMemory("input")
+        .HostMemory("output")
+        .TypeConstraint<int32>("T")
+        .TypeConstraint<int64>("Tidx"),
+    ReductionOp<CPUDevice, int32, int64, Eigen::internal::MinReducer<int32>>);
 
 #undef REGISTER_GPU_KERNELS
 
 #endif
 
 #ifdef TENSORFLOW_USE_SYCL
-#define REGISTER_SYCL_KERNELS(type)         \
-  REGISTER_KERNEL_BUILDER(                  \
-      Name("Min")                           \
-          .Device(DEVICE_SYCL)              \
-          .TypeConstraint<type>("T")        \
-          .TypeConstraint<int32>("Tidx")    \
-          .HostMemory("reduction_indices"), \
-      ReductionOp<SYCLDevice, type, Eigen::internal::MinReducer<type>>);
+#define REGISTER_SYCL_KERNELS(type)                                        \
+  REGISTER_KERNEL_BUILDER(Name("Min")                                      \
+                              .Device(DEVICE_SYCL)                         \
+                              .TypeConstraint<type>("T")                   \
+                              .TypeConstraint<int32>("Tidx")               \
+                              .HostMemory("reduction_indices"),            \
+                          ReductionOp<SYCLDevice, type, int32,             \
+                                      Eigen::internal::MinReducer<type>>); \
+  REGISTER_KERNEL_BUILDER(Name("Min")                                      \
+                              .Device(DEVICE_SYCL)                         \
+                              .TypeConstraint<type>("T")                   \
+                              .TypeConstraint<int64>("Tidx")               \
+                              .HostMemory("reduction_indices"),            \
+                          ReductionOp<SYCLDevice, type, int64,             \
+                                      Eigen::internal::MinReducer<type>>);
 REGISTER_SYCL_KERNELS(float);
 REGISTER_SYCL_KERNELS(double);
 
@@ -77,8 +106,17 @@ REGISTER_KERNEL_BUILDER(
         .HostMemory("output")
         .TypeConstraint<int32>("T")
         .TypeConstraint<int32>("Tidx"),
-    ReductionOp<CPUDevice, int32, Eigen::internal::MinReducer<int32>>);
+    ReductionOp<CPUDevice, int32, int32, Eigen::internal::MinReducer<int32>>);
+REGISTER_KERNEL_BUILDER(
+    Name("Min")
+        .Device(DEVICE_SYCL)
+        .HostMemory("reduction_indices")
+        .HostMemory("input")
+        .HostMemory("output")
+        .TypeConstraint<int32>("T")
+        .TypeConstraint<int64>("Tidx"),
+    ReductionOp<CPUDevice, int32, int64, Eigen::internal::MinReducer<int32>>);
 #undef REGISTER_SYCL_KERNELS
-#endif // TENSORFLOW_USE_SYCL
+#endif  // TENSORFLOW_USE_SYCL
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/kernels/reduction_ops_prod.cc b/tensorflow/core/kernels/reduction_ops_prod.cc
index 33f6ae6bae..e9b23df746 100644
--- a/tensorflow/core/kernels/reduction_ops_prod.cc
+++ b/tensorflow/core/kernels/reduction_ops_prod.cc
@@ -17,26 +17,39 @@ limitations under the License.
 
 namespace tensorflow {
 
-#define REGISTER_CPU_KERNELS(type)        \
-  REGISTER_KERNEL_BUILDER(                \
-      Name("Prod")                        \
-          .Device(DEVICE_CPU)             \
-          .TypeConstraint<type>("T")      \
-          .TypeConstraint<int32>("Tidx"), \
-      ReductionOp<CPUDevice, type, Eigen::internal::ProdReducer<type>>);
+#define REGISTER_CPU_KERNELS(type)                                          \
+  REGISTER_KERNEL_BUILDER(Name("Prod")                                      \
+                              .Device(DEVICE_CPU)                           \
+                              .TypeConstraint<type>("T")                    \
+                              .TypeConstraint<int32>("Tidx"),               \
+                          ReductionOp<CPUDevice, type, int32,               \
+                                      Eigen::internal::ProdReducer<type>>); \
+  REGISTER_KERNEL_BUILDER(Name("Prod")                                      \
+                              .Device(DEVICE_CPU)                           \
+                              .TypeConstraint<type>("T")                    \
+                              .TypeConstraint<int64>("Tidx"),               \
+                          ReductionOp<CPUDevice, type, int64,               \
+                                      Eigen::internal::ProdReducer<type>>);
 TF_CALL_NUMBER_TYPES(REGISTER_CPU_KERNELS);
 #undef REGISTER_CPU_KERNELS
 
 #if GOOGLE_CUDA
 
-#define REGISTER_GPU_KERNELS(type)          \
-  REGISTER_KERNEL_BUILDER(                  \
-      Name("Prod")                          \
-          .Device(DEVICE_GPU)               \
-          .TypeConstraint<type>("T")        \
-          .TypeConstraint<int32>("Tidx")    \
-          .HostMemory("reduction_indices"), \
-      ReductionOp<GPUDevice, type, Eigen::internal::ProdReducer<type>>);
+#define REGISTER_GPU_KERNELS(type)                                          \
+  REGISTER_KERNEL_BUILDER(Name("Prod")                                      \
+                              .Device(DEVICE_GPU)                           \
+                              .TypeConstraint<type>("T")                    \
+                              .TypeConstraint<int32>("Tidx")                \
+                              .HostMemory("reduction_indices"),             \
+                          ReductionOp<GPUDevice, type, int32,               \
+                                      Eigen::internal::ProdReducer<type>>); \
+  REGISTER_KERNEL_BUILDER(Name("Prod")                                      \
+                              .Device(DEVICE_GPU)                           \
+                              .TypeConstraint<type>("T")                    \
+                              .TypeConstraint<int64>("Tidx")                \
+                              .HostMemory("reduction_indices"),             \
+                          ReductionOp<GPUDevice, type, int64,               \
+                                      Eigen::internal::ProdReducer<type>>);
 TF_CALL_GPU_NUMBER_TYPES(REGISTER_GPU_KERNELS);
 TF_CALL_int32(REGISTER_GPU_KERNELS);
 TF_CALL_complex64(REGISTER_GPU_KERNELS);
@@ -46,18 +59,25 @@ TF_CALL_complex128(REGISTER_GPU_KERNELS);
 #endif
 
 #ifdef TENSORFLOW_USE_SYCL
-#define REGISTER_SYCL_KERNELS(type)         \
-  REGISTER_KERNEL_BUILDER(                  \
-      Name("Prod")                          \
-          .Device(DEVICE_SYCL)              \
-          .TypeConstraint<type>("T")        \
-          .TypeConstraint<int32>("Tidx")    \
-          .HostMemory("reduction_indices"), \
-      ReductionOp<SYCLDevice, type, Eigen::internal::ProdReducer<type>>);
+#define REGISTER_SYCL_KERNELS(type)                                         \
+  REGISTER_KERNEL_BUILDER(Name("Prod")                                      \
+                              .Device(DEVICE_SYCL)                          \
+                              .TypeConstraint<type>("T")                    \
+                              .TypeConstraint<int32>("Tidx")                \
+                              .HostMemory("reduction_indices"),             \
+                          ReductionOp<SYCLDevice, type, int32,              \
+                                      Eigen::internal::ProdReducer<type>>); \
+  REGISTER_KERNEL_BUILDER(Name("Prod")                                      \
+                              .Device(DEVICE_SYCL)                          \
+                              .TypeConstraint<type>("T")                    \
+                              .TypeConstraint<int64>("Tidx")                \
+                              .HostMemory("reduction_indices"),             \
+                          ReductionOp<SYCLDevice, type, int64,              \
+                                      Eigen::internal::ProdReducer<type>>);
 REGISTER_SYCL_KERNELS(int32);
 REGISTER_SYCL_KERNELS(float);
 REGISTER_SYCL_KERNELS(double);
 #undef REGISTER_SYCL_KERNELS
-#endif // TENSORFLOW_USE_SYCL
+#endif  // TENSORFLOW_USE_SYCL
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/kernels/reduction_ops_sum.cc b/tensorflow/core/kernels/reduction_ops_sum.cc
index c1f4f3475a..5318d8c133 100644
--- a/tensorflow/core/kernels/reduction_ops_sum.cc
+++ b/tensorflow/core/kernels/reduction_ops_sum.cc
@@ -17,26 +17,39 @@ limitations under the License.
 
 namespace tensorflow {
 
-#define REGISTER_CPU_KERNELS(type)        \
-  REGISTER_KERNEL_BUILDER(                \
-      Name("Sum")                         \
-          .Device(DEVICE_CPU)             \
-          .TypeConstraint<type>("T")      \
-          .TypeConstraint<int32>("Tidx"), \
-      ReductionOp<CPUDevice, type, Eigen::internal::SumReducer<type>>);
+#define REGISTER_CPU_KERNELS(type)                                             \
+  REGISTER_KERNEL_BUILDER(                                                     \
+      Name("Sum")                                                              \
+          .Device(DEVICE_CPU)                                                  \
+          .TypeConstraint<type>("T")                                           \
+          .TypeConstraint<int32>("Tidx"),                                      \
+      ReductionOp<CPUDevice, type, int32, Eigen::internal::SumReducer<type>>); \
+  REGISTER_KERNEL_BUILDER(                                                     \
+      Name("Sum")                                                              \
+          .Device(DEVICE_CPU)                                                  \
+          .TypeConstraint<type>("T")                                           \
+          .TypeConstraint<int64>("Tidx"),                                      \
+      ReductionOp<CPUDevice, type, int64, Eigen::internal::SumReducer<type>>);
 TF_CALL_NUMBER_TYPES(REGISTER_CPU_KERNELS);
 #undef REGISTER_CPU_KERNELS
 
 #if GOOGLE_CUDA
 
-#define REGISTER_GPU_KERNELS(type)          \
-  REGISTER_KERNEL_BUILDER(                  \
-      Name("Sum")                           \
-          .Device(DEVICE_GPU)               \
-          .TypeConstraint<type>("T")        \
-          .TypeConstraint<int32>("Tidx")    \
-          .HostMemory("reduction_indices"), \
-      ReductionOp<GPUDevice, type, Eigen::internal::SumReducer<type>>);
+#define REGISTER_GPU_KERNELS(type)                                             \
+  REGISTER_KERNEL_BUILDER(                                                     \
+      Name("Sum")                                                              \
+          .Device(DEVICE_GPU)                                                  \
+          .TypeConstraint<type>("T")                                           \
+          .TypeConstraint<int32>("Tidx")                                       \
+          .HostMemory("reduction_indices"),                                    \
+      ReductionOp<GPUDevice, type, int32, Eigen::internal::SumReducer<type>>); \
+  REGISTER_KERNEL_BUILDER(                                                     \
+      Name("Sum")                                                              \
+          .Device(DEVICE_GPU)                                                  \
+          .TypeConstraint<type>("T")                                           \
+          .TypeConstraint<int64>("Tidx")                                       \
+          .HostMemory("reduction_indices"),                                    \
+      ReductionOp<GPUDevice, type, int64, Eigen::internal::SumReducer<type>>);
 TF_CALL_GPU_NUMBER_TYPES(REGISTER_GPU_KERNELS);
 TF_CALL_complex64(REGISTER_GPU_KERNELS);
 TF_CALL_complex128(REGISTER_GPU_KERNELS);
@@ -53,19 +66,35 @@ REGISTER_KERNEL_BUILDER(
         .HostMemory("input")
         .HostMemory("output")
         .HostMemory("reduction_indices"),
-    ReductionOp<CPUDevice, int32, Eigen::internal::SumReducer<int32>>);
+    ReductionOp<CPUDevice, int32, int32, Eigen::internal::SumReducer<int32>>);
+REGISTER_KERNEL_BUILDER(
+    Name("Sum")
+        .Device(DEVICE_GPU)
+        .TypeConstraint<int32>("T")
+        .TypeConstraint<int64>("Tidx")
+        .HostMemory("input")
+        .HostMemory("output")
+        .HostMemory("reduction_indices"),
+    ReductionOp<CPUDevice, int32, int64, Eigen::internal::SumReducer<int32>>);
 
 #endif
 
 #ifdef TENSORFLOW_USE_SYCL
-#define REGISTER_SYCL_KERNELS(type)         \
-  REGISTER_KERNEL_BUILDER(                  \
-      Name("Sum")                           \
-          .Device(DEVICE_SYCL)              \
-          .TypeConstraint<type>("T")        \
-          .TypeConstraint<int32>("Tidx")    \
-          .HostMemory("reduction_indices"), \
-      ReductionOp<SYCLDevice, type, Eigen::internal::SumReducer<type>>);
+#define REGISTER_SYCL_KERNELS(type)                                        \
+  REGISTER_KERNEL_BUILDER(Name("Sum")                                      \
+                              .Device(DEVICE_SYCL)                         \
+                              .TypeConstraint<type>("T")                   \
+                              .TypeConstraint<int32>("Tidx")               \
+                              .HostMemory("reduction_indices"),            \
+                          ReductionOp<SYCLDevice, type, int32,             \
+                                      Eigen::internal::SumReducer<type>>); \
+  REGISTER_KERNEL_BUILDER(Name("Sum")                                      \
+                              .Device(DEVICE_SYCL)                         \
+                              .TypeConstraint<type>("T")                   \
+                              .TypeConstraint<int64>("Tidx")               \
+                              .HostMemory("reduction_indices"),            \
+                          ReductionOp<SYCLDevice, type, int64,             \
+                                      Eigen::internal::SumReducer<type>>);
 REGISTER_SYCL_KERNELS(float);
 REGISTER_SYCL_KERNELS(double);
 
@@ -77,8 +106,17 @@ REGISTER_KERNEL_BUILDER(
         .HostMemory("input")
         .HostMemory("output")
         .HostMemory("reduction_indices"),
-    ReductionOp<CPUDevice, int32, Eigen::internal::SumReducer<int32>>);
+    ReductionOp<CPUDevice, int32, int32, Eigen::internal::SumReducer<int32>>);
+REGISTER_KERNEL_BUILDER(
+    Name("Sum")
+        .Device(DEVICE_SYCL)
+        .TypeConstraint<int32>("T")
+        .TypeConstraint<int64>("Tidx")
+        .HostMemory("input")
+        .HostMemory("output")
+        .HostMemory("reduction_indices"),
+    ReductionOp<CPUDevice, int32, int64, Eigen::internal::SumReducer<int32>>);
 #undef REGISTER_SYCL_KERNELS
-#endif // TENSORFLOW_USE_SYCL
+#endif  // TENSORFLOW_USE_SYCL
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/kernels/resize_bicubic_op.cc b/tensorflow/core/kernels/resize_bicubic_op.cc
index 1c43e77e7c..1a9cf4c640 100644
--- a/tensorflow/core/kernels/resize_bicubic_op.cc
+++ b/tensorflow/core/kernels/resize_bicubic_op.cc
@@ -20,7 +20,6 @@ limitations under the License.
 #include <algorithm>
 #include <array>
 
-#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
@@ -29,6 +28,7 @@ limitations under the License.
 #include "tensorflow/core/kernels/image_resizer_state.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/platform/logging.h"
+#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 
 namespace tensorflow {
 namespace {
@@ -235,6 +235,7 @@ inline void interpolate_with_caching(
 
   const T* input_b_ptr = input_data.data();
   float* output_y_ptr = output_data.data();
+  std::vector<float> cached_value(num_channels == 3 ? 0 : 4 * num_channels, 0);
 
   for (int64 b = 0; b < resizer_state.batch_size;
        ++b, input_b_ptr += in_batch_width) {
@@ -248,6 +249,7 @@ inline void interpolate_with_caching(
       const T* y_ptr_1 = input_b_ptr + y_wai.index_1 * in_row_width;
       const T* y_ptr_2 = input_b_ptr + y_wai.index_2 * in_row_width;
       const T* y_ptr_3 = input_b_ptr + y_wai.index_3 * in_row_width;
+
       if (num_channels == 3) {
         // Manually unroll case of 3 channels.
         float cached_value_0[4] = {0};
@@ -330,48 +332,61 @@ inline void interpolate_with_caching(
                       x_wai.weight_2, x_wai.weight_3);
         }
       } else {
-        for (int64 c = 0; c < num_channels; ++c) {
-          float cached_value[4] = {0};
-          for (int64 x = 0; x < resizer_state.out_width; ++x) {
-            const WeightsAndIndices& x_wai = x_wais[x];
-            // Shift values in cached_value to fill first 'advance' values.
-            switch (x_wai.advance) {
-              case 3:
-                cached_value[0] = cached_value[1];
-                cached_value[1] = cached_value[2];
-                cached_value[2] = cached_value[3];
-                break;
-              case 2:
-                cached_value[0] = cached_value[2];
-                cached_value[1] = cached_value[3];
-                break;
-              case 1: {
-                cached_value[0] = cached_value[3];
-                break;
+        for (int64 x = 0; x < resizer_state.out_width; ++x) {
+          const WeightsAndIndices& x_wai = x_wais[x];
+          // Shift values in cached_value to fill first 'advance' values.
+          switch (x_wai.advance) {
+            case 3:
+              for (int64 c = 0; c < num_channels; ++c) {
+                cached_value[4 * c + 0] = cached_value[4 * c + 1];
+                cached_value[4 * c + 1] = cached_value[4 * c + 2];
+                cached_value[4 * c + 2] = cached_value[4 * c + 3];
+              }
+              break;
+            case 2:
+              for (int64 c = 0; c < num_channels; ++c) {
+                cached_value[4 * c + 0] = cached_value[4 * c + 2];
+                cached_value[4 * c + 1] = cached_value[4 * c + 3];
+              }
+              break;
+            case 1: {
+              for (int64 c = 0; c < num_channels; ++c) {
+                cached_value[4 * c + 0] = cached_value[4 * c + 3];
               }
+              break;
             }
+          }
 
-            // Set the remaining '4-advance' values by computing.
-            switch (x_wai.advance) {
-              case 0:
-                cached_value[0] = ComputeYInterpolation(
+          // Set the remaining '4-advance' values by computing.
+          switch (x_wai.advance) {
+            case 0:
+              for (int64 c = 0; c < num_channels; ++c) {
+                cached_value[4 * c + 0] = ComputeYInterpolation(
                     0, c, y_wai, y_ptr_0, y_ptr_1, y_ptr_2, y_ptr_3, x_wai);
-                TF_FALLTHROUGH_INTENDED;
-              case 1:
-                cached_value[1] = ComputeYInterpolation(
+              }
+              TF_FALLTHROUGH_INTENDED;
+            case 1:
+              for (int64 c = 0; c < num_channels; ++c) {
+                cached_value[4 * c + 1] = ComputeYInterpolation(
                     1, c, y_wai, y_ptr_0, y_ptr_1, y_ptr_2, y_ptr_3, x_wai);
-                TF_FALLTHROUGH_INTENDED;
-              case 2:
-                cached_value[2] = ComputeYInterpolation(
+              }
+              TF_FALLTHROUGH_INTENDED;
+            case 2:
+              for (int64 c = 0; c < num_channels; ++c) {
+                cached_value[4 * c + 2] = ComputeYInterpolation(
                     2, c, y_wai, y_ptr_0, y_ptr_1, y_ptr_2, y_ptr_3, x_wai);
-                TF_FALLTHROUGH_INTENDED;
-              case 3:
-                cached_value[3] = ComputeYInterpolation(
+              }
+              TF_FALLTHROUGH_INTENDED;
+            case 3:
+              for (int64 c = 0; c < num_channels; ++c) {
+                cached_value[4 * c + 3] = ComputeYInterpolation(
                     3, c, y_wai, y_ptr_0, y_ptr_1, y_ptr_2, y_ptr_3, x_wai);
-                break;
-            }
+              }
+              break;
+          }
+          for (int64 c = 0; c < num_channels; ++c) {
             output_y_ptr[x * num_channels + c] =
-                Compute(cached_value, x_wai.weight_0, x_wai.weight_1,
+                Compute(&cached_value[4 * c], x_wai.weight_0, x_wai.weight_1,
                         x_wai.weight_2, x_wai.weight_3);
           }
         }
diff --git a/tensorflow/core/kernels/resize_bicubic_op_test.cc b/tensorflow/core/kernels/resize_bicubic_op_test.cc
index ae14d2804e..9e10fec423 100644
--- a/tensorflow/core/kernels/resize_bicubic_op_test.cc
+++ b/tensorflow/core/kernels/resize_bicubic_op_test.cc
@@ -251,14 +251,15 @@ TEST_F(ResizeBicubicOpTest, TestAreaRandomDataSeveralInputsSizes4Channels) {
   RunManyRandomTests(4);
 }
 
-static Graph* ResizeBicubic(int batch_size, int size, int channels) {
+static Graph* ResizeBicubic(int batch_size, int size, int channels,
+                            float scale_y = 0.3, float scale_x = 0.7) {
   Graph* g = new Graph(OpRegistry::Global());
   Tensor input(DT_FLOAT, TensorShape({batch_size, size, size, channels}));
   input.flat<float>().setRandom();
   Tensor shape(DT_INT32, TensorShape({2}));
   auto shape_t = shape.flat<int32>();
-  shape_t(0) = 0.3 * size;
-  shape_t(1) = 0.7 * size;
+  shape_t(0) = scale_y * size;
+  shape_t(1) = scale_x * size;
   test::graph::Binary(g, "ResizeBicubic", test::graph::Constant(g, input),
                       test::graph::Constant(g, shape));
   return g;
@@ -285,4 +286,17 @@ BM_ResizeBicubicDev(32, 128, 3);
 BM_ResizeBicubicDev(32, 512, 3);
 BM_ResizeBicubicDev(32, 1024, 3);
 
+#define BM_ResizeBicubicExpand(BATCH, SIZE, CHANNELS)                          \
+  static void BM_ResizeBicubicExpand##_##BATCH##_##SIZE##_##CHANNELS(int iters) { \
+    testing::ItemsProcessed(static_cast<int64>(iters) * BATCH * SIZE * SIZE *  \
+                            CHANNELS * 8 * 8);                                 \
+    test::Benchmark("cpu", ResizeBicubic(BATCH, SIZE, CHANNELS, 8, 8))         \
+        .Run(iters);                                                           \
+  }                                                                            \
+  BENCHMARK(BM_ResizeBicubicExpand##_##BATCH##_##SIZE##_##CHANNELS);
+
+BM_ResizeBicubicExpand(12, 48, 1);
+BM_ResizeBicubicExpand(12, 48, 3);
+BM_ResizeBicubicExpand(12, 48, 40);
+
 }  // end namespace tensorflow
diff --git a/tensorflow/core/kernels/reverse_sequence_op.cc b/tensorflow/core/kernels/reverse_sequence_op.cc
index 505c512cc4..d1980d4b65 100644
--- a/tensorflow/core/kernels/reverse_sequence_op.cc
+++ b/tensorflow/core/kernels/reverse_sequence_op.cc
@@ -175,6 +175,7 @@ class ReverseSequenceOp : public OpKernel {
   REGISTER_REVERSE_SEQUENCE(type, int64);
 
 TF_CALL_NUMBER_TYPES(REGISTER_REVERSE_SEQUENCE_LEN);
+TF_CALL_bool(REGISTER_REVERSE_SEQUENCE_LEN);
 
 #if GOOGLE_CUDA
 
@@ -200,6 +201,7 @@ namespace functor {
   DECLARE_GPU_SPEC_LEN(T, 5);
 
 TF_CALL_GPU_NUMBER_TYPES(DECLARE_GPU_SPECS);
+TF_CALL_bool(DECLARE_GPU_SPECS);
 
 }  // namespace functor
 
@@ -215,6 +217,7 @@ TF_CALL_GPU_NUMBER_TYPES(DECLARE_GPU_SPECS);
   REGISTER_REVERSE_SEQUENCE_GPU(type, int64);
 
 TF_CALL_GPU_NUMBER_TYPES(REGISTER_REVERSE_SEQUENCE_GPU_LEN);
+TF_CALL_bool(REGISTER_REVERSE_SEQUENCE_GPU_LEN);
 
 #undef REGISTER_REVERSE_SEQUENCE_GPU
 
diff --git a/tensorflow/core/kernels/reverse_sequence_op_gpu.cu.cc b/tensorflow/core/kernels/reverse_sequence_op_gpu.cu.cc
index 373fd60687..cb49f14525 100644
--- a/tensorflow/core/kernels/reverse_sequence_op_gpu.cu.cc
+++ b/tensorflow/core/kernels/reverse_sequence_op_gpu.cu.cc
@@ -39,6 +39,7 @@ typedef Eigen::GpuDevice GPUDevice;
   DEFINE_GPU_SPEC_LEN(T, 5);
 
 TF_CALL_GPU_NUMBER_TYPES(DEFINE_GPU_SPECS);
+TF_CALL_bool(DEFINE_GPU_SPECS);
 
 }  // end namespace tensorflow
 
diff --git a/tensorflow/core/kernels/scan_ops.cc b/tensorflow/core/kernels/scan_ops.cc
index cc434ab0ae..0a6848361a 100644
--- a/tensorflow/core/kernels/scan_ops.cc
+++ b/tensorflow/core/kernels/scan_ops.cc
@@ -35,7 +35,7 @@ namespace tensorflow {
 typedef Eigen::ThreadPoolDevice CPUDevice;
 typedef Eigen::GpuDevice GPUDevice;
 
-template <typename Device, class T, typename Reducer>
+template <typename Device, class T, typename Reducer, typename Tidx>
 class ScanOp : public OpKernel {
  public:
   explicit ScanOp(OpKernelConstruction* ctx) : OpKernel(ctx) {
@@ -51,8 +51,9 @@ class ScanOp : public OpKernel {
                 errors::InvalidArgument("ScanOp: axis must be a scalar, not ",
                                         tensor_axis.shape().DebugString()));
 
-    const int axis_arg = internal::SubtleMustCopy(tensor_axis.scalar<int>()());
-    const int axis = (axis_arg < 0) ? input.dims() + axis_arg : axis_arg;
+    const Tidx axis_arg =
+        internal::SubtleMustCopy(tensor_axis.scalar<Tidx>()());
+    const Tidx axis = (axis_arg < 0) ? input.dims() + axis_arg : axis_arg;
     OP_REQUIRES(ctx, FastBoundsCheck(axis, input.dims()),
                 errors::InvalidArgument(
                     "ScanOp: Expected scan axis in the range [", -input.dims(),
@@ -70,11 +71,11 @@ class ScanOp : public OpKernel {
 
     // Dim reduction.
     int64 reduced_shape[3] = {1, 1, 1};
-    for (int i = 0; i < axis; ++i) {
+    for (Tidx i = 0; i < axis; ++i) {
       reduced_shape[0] *= input.dim_size(i);
     }
     reduced_shape[1] = input.dim_size(axis);
-    for (int i = axis + 1; i < input.dims(); ++i) {
+    for (Tidx i = axis + 1; i < input.dims(); ++i) {
       reduced_shape[2] *= input.dim_size(i);
     }
 
@@ -112,51 +113,76 @@ TF_CALL_GPU_NUMBER_TYPES(DECLARE_FOR_ALL_REDUCERS);
 }  // namespace functor
 #endif  // GOOGLE_CUDA
 
-
 // Register Cumsum kernels
-#define REGISTER_CPU_KERNELS(type)        \
-  REGISTER_KERNEL_BUILDER(                \
-      Name("Cumsum")                      \
-          .Device(DEVICE_CPU)             \
-          .TypeConstraint<type>("T")      \
-          .TypeConstraint<int32>("Tidx"), \
-      ScanOp<CPUDevice, type, Eigen::internal::SumReducer<type>>)
+#define REGISTER_CPU_KERNELS(type)                                       \
+  REGISTER_KERNEL_BUILDER(                                               \
+      Name("Cumsum")                                                     \
+          .Device(DEVICE_CPU)                                            \
+          .TypeConstraint<type>("T")                                     \
+          .TypeConstraint<int32>("Tidx"),                                \
+      ScanOp<CPUDevice, type, Eigen::internal::SumReducer<type>, int32>) \
+  REGISTER_KERNEL_BUILDER(                                               \
+      Name("Cumsum")                                                     \
+          .Device(DEVICE_CPU)                                            \
+          .TypeConstraint<type>("T")                                     \
+          .TypeConstraint<int64>("Tidx"),                                \
+      ScanOp<CPUDevice, type, Eigen::internal::SumReducer<type>, int64>)
 TF_CALL_NUMBER_TYPES(REGISTER_CPU_KERNELS);
 #undef REGISTER_CPU_KERNELS
 
 #if GOOGLE_CUDA
-#define REGISTER_GPU_KERNELS(type)       \
-  REGISTER_KERNEL_BUILDER(               \
-      Name("Cumsum")                     \
-          .Device(DEVICE_GPU)            \
-          .TypeConstraint<type>("T")     \
-          .TypeConstraint<int32>("Tidx") \
-          .HostMemory("axis"),           \
-      ScanOp<GPUDevice, type, Eigen::internal::SumReducer<type>>)
+#define REGISTER_GPU_KERNELS(type)                                       \
+  REGISTER_KERNEL_BUILDER(                                               \
+      Name("Cumsum")                                                     \
+          .Device(DEVICE_GPU)                                            \
+          .TypeConstraint<type>("T")                                     \
+          .TypeConstraint<int32>("Tidx")                                 \
+          .HostMemory("axis"),                                           \
+      ScanOp<GPUDevice, type, Eigen::internal::SumReducer<type>, int32>) \
+  REGISTER_KERNEL_BUILDER(                                               \
+      Name("Cumsum")                                                     \
+          .Device(DEVICE_GPU)                                            \
+          .TypeConstraint<type>("T")                                     \
+          .TypeConstraint<int64>("Tidx")                                 \
+          .HostMemory("axis"),                                           \
+      ScanOp<GPUDevice, type, Eigen::internal::SumReducer<type>, int64>)
 TF_CALL_GPU_NUMBER_TYPES(REGISTER_GPU_KERNELS)
 #undef REGISTER_GPU_KERNELS
 #endif  // GOOGLE_CUDA
 
 // Register Cumprod kernels
-#define REGISTER_CPU_KERNELS(type)        \
-  REGISTER_KERNEL_BUILDER(                \
-      Name("Cumprod")                     \
-          .Device(DEVICE_CPU)             \
-          .TypeConstraint<type>("T")      \
-          .TypeConstraint<int32>("Tidx"), \
-      ScanOp<CPUDevice, type, Eigen::internal::ProdReducer<type>>)
+#define REGISTER_CPU_KERNELS(type)                                        \
+  REGISTER_KERNEL_BUILDER(                                                \
+      Name("Cumprod")                                                     \
+          .Device(DEVICE_CPU)                                             \
+          .TypeConstraint<type>("T")                                      \
+          .TypeConstraint<int32>("Tidx"),                                 \
+      ScanOp<CPUDevice, type, Eigen::internal::ProdReducer<type>, int32>) \
+  REGISTER_KERNEL_BUILDER(                                                \
+      Name("Cumprod")                                                     \
+          .Device(DEVICE_CPU)                                             \
+          .TypeConstraint<type>("T")                                      \
+          .TypeConstraint<int64>("Tidx"),                                 \
+      ScanOp<CPUDevice, type, Eigen::internal::ProdReducer<type>, int64>)
 TF_CALL_NUMBER_TYPES(REGISTER_CPU_KERNELS);
 #undef REGISTER_CPU_KERNELS
 
 #if GOOGLE_CUDA
-#define REGISTER_GPU_KERNELS(type)       \
-  REGISTER_KERNEL_BUILDER(               \
-      Name("Cumprod")                    \
-          .Device(DEVICE_GPU)            \
-          .TypeConstraint<type>("T")     \
-          .TypeConstraint<int32>("Tidx") \
-          .HostMemory("axis"),           \
-      ScanOp<GPUDevice, type, Eigen::internal::ProdReducer<type>>)
+#define REGISTER_GPU_KERNELS(type)                                        \
+  REGISTER_KERNEL_BUILDER(                                                \
+      Name("Cumprod")                                                     \
+          .Device(DEVICE_GPU)                                             \
+          .TypeConstraint<type>("T")                                      \
+          .TypeConstraint<int32>("Tidx")                                  \
+          .HostMemory("axis"),                                            \
+      ScanOp<GPUDevice, type, Eigen::internal::ProdReducer<type>, int32>) \
+  REGISTER_KERNEL_BUILDER(                                                \
+      Name("Cumprod")                                                     \
+          .Device(DEVICE_GPU)                                             \
+          .TypeConstraint<type>("T")                                      \
+          .TypeConstraint<int64>("Tidx")                                  \
+          .HostMemory("axis"),                                            \
+      ScanOp<GPUDevice, type, Eigen::internal::ProdReducer<type>, int64>)
 TF_CALL_GPU_NUMBER_TYPES(REGISTER_GPU_KERNELS)
 #undef REGISTER_GPU_KERNELS
 #endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/sequence_ops.cc b/tensorflow/core/kernels/sequence_ops.cc
index c8ea923020..e2e3758d87 100644
--- a/tensorflow/core/kernels/sequence_ops.cc
+++ b/tensorflow/core/kernels/sequence_ops.cc
@@ -96,7 +96,7 @@ TF_CALL_double(REGISTER_SYCL_KERNEL);
 TF_CALL_int32(REGISTER_SYCL_KERNEL);
 TF_CALL_int64(REGISTER_SYCL_KERNEL);
 #undef REGISTER_SYCL_KERNEL
-#endif // TENSORFLOW_USE_SYCL
+#endif  // TENSORFLOW_USE_SYCL
 
 TF_CALL_float(REGISTER_CPU_KERNEL);
 TF_CALL_double(REGISTER_CPU_KERNEL);
@@ -116,7 +116,7 @@ TF_CALL_int64(REGISTER_GPU_KERNEL);
 #undef REGISTER_CPU_KERNEL
 #undef REGISTER_GPU_KERNEL
 
-template <typename T>
+template <typename T, typename Tnum>
 class LinSpaceOp : public OpKernel {
  public:
   explicit LinSpaceOp(OpKernelConstruction* context) : OpKernel(context) {}
@@ -136,7 +136,7 @@ class LinSpaceOp : public OpKernel {
                                         num_in.shape().DebugString()));
     const T start = start_in.scalar<T>()();
     const T stop = stop_in.scalar<T>()();
-    const int32 num = num_in.scalar<int32>()();
+    const Tnum num = num_in.scalar<Tnum>()();
     OP_REQUIRES(context, num > 0,
                 errors::InvalidArgument("Requires num > 0: ", num));
     Tensor* out = nullptr;
@@ -147,34 +147,46 @@ class LinSpaceOp : public OpKernel {
       flat(0) = start;
     } else {
       const T step = (stop - start) / (num - 1);
-      for (int32 i = 0; i < num; ++i) flat(i) = start + step * i;
+      for (Tnum i = 0; i < num; ++i) flat(i) = start + step * i;
     }
   }
 };
 
-#define REGISTER_KERNEL(DEV, T)                              \
-  REGISTER_KERNEL_BUILDER(Name("LinSpace")                   \
-                              .Device(DEV)                   \
-                              .TypeConstraint<T>("T")        \
-                              .TypeConstraint<int32>("Tidx") \
-                              .HostMemory("start")           \
-                              .HostMemory("stop")            \
-                              .HostMemory("num")             \
-                              .HostMemory("output"),         \
-                          LinSpaceOp<T>);
-#define REGISTER_CPU_KERNEL(T) REGISTER_KERNEL(DEVICE_CPU, T)
+#define REGISTER_KERNEL(DEV, T, Tidx)                       \
+  REGISTER_KERNEL_BUILDER(Name("LinSpace")                  \
+                              .Device(DEV)                  \
+                              .TypeConstraint<T>("T")       \
+                              .TypeConstraint<Tidx>("Tidx") \
+                              .HostMemory("start")          \
+                              .HostMemory("stop")           \
+                              .HostMemory("num")            \
+                              .HostMemory("output"),        \
+                          LinSpaceOp<T, Tidx>);
+
+#define REGISTER_KERNEL_ALL_NUMS(dev, T) \
+  REGISTER_KERNEL(dev, T, int32);        \
+  REGISTER_KERNEL(dev, T, int64)
+
+#define REGISTER_CPU_KERNEL(T) REGISTER_KERNEL_ALL_NUMS(DEVICE_CPU, T)
 TF_CALL_float(REGISTER_CPU_KERNEL);
 TF_CALL_double(REGISTER_CPU_KERNEL);
 
 // NOTE(touts): We register the op on GPU but it still runs on CPU
 // because its inputs and outputs are tagged as HostMemory.
-#define REGISTER_GPU_KERNEL(T) REGISTER_KERNEL(DEVICE_GPU, T)
+#define REGISTER_GPU_KERNEL(T) REGISTER_KERNEL_ALL_NUMS(DEVICE_GPU, T)
 TF_CALL_float(REGISTER_GPU_KERNEL);
 TF_CALL_double(REGISTER_GPU_KERNEL);
+#undef REGISTER_GPU_KERNEL
 
 #ifdef TENSORFLOW_USE_SYCL
-#define REGISTER_SYCL_KERNEL(T) REGISTER_KERNEL(DEVICE_SYCL, T)
+#define REGISTER_SYCL_KERNEL(T) REGISTER_KERNEL_ALL_NUMS(DEVICE_SYCL, T)
 TF_CALL_float(REGISTER_SYCL_KERNEL);
 TF_CALL_double(REGISTER_SYCL_KERNEL);
-#endif // TENSORFLOW_USE_SYCL
+#undef REGISTER_SYCL_KERNEL
+#endif  // TENSORFLOW_USE_SYCL
+
+#undef REGISTER_CPU_KERNEL
+#undef REGISTER_KERNEL_ALL_NUMS
+#undef REGISTER_KERNEL
+
 }  // namespace tensorflow
diff --git a/tensorflow/core/kernels/sequence_ops_test.cc b/tensorflow/core/kernels/sequence_ops_test.cc
new file mode 100644
index 0000000000..5f0e0a69a8
--- /dev/null
+++ b/tensorflow/core/kernels/sequence_ops_test.cc
@@ -0,0 +1,148 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/framework/allocator.h"
+#include "tensorflow/core/framework/fake_input.h"
+#include "tensorflow/core/framework/node_def_builder.h"
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/framework/tensor_testutil.h"
+#include "tensorflow/core/framework/types.h"
+#include "tensorflow/core/framework/types.pb.h"
+#include "tensorflow/core/kernels/ops_testutil.h"
+#include "tensorflow/core/kernels/ops_util.h"
+#include "tensorflow/core/platform/test.h"
+
+namespace tensorflow {
+namespace {
+
+class RangeOpTest : public OpsTestBase {
+ protected:
+  void MakeOp(DataType input_type) {
+    TF_ASSERT_OK(NodeDefBuilder("myop", "Range")
+                     .Input(FakeInput(input_type))
+                     .Input(FakeInput(input_type))
+                     .Input(FakeInput(input_type))
+                     .Finalize(node_def()));
+    TF_ASSERT_OK(InitOp());
+  }
+};
+
+class LinSpaceOpTest : public OpsTestBase {
+ protected:
+  void MakeOp(DataType input_type, DataType index_type) {
+    TF_ASSERT_OK(NodeDefBuilder("myop", "LinSpace")
+                     .Input(FakeInput(input_type))
+                     .Input(FakeInput(input_type))
+                     .Input(FakeInput(index_type))
+                     .Finalize(node_def()));
+    TF_ASSERT_OK(InitOp());
+  }
+};
+
+TEST_F(RangeOpTest, Simple_D32) {
+  MakeOp(DT_INT32);
+
+  // Feed and run
+  AddInputFromArray<int32>(TensorShape({}), {0});
+  AddInputFromArray<int32>(TensorShape({}), {10});
+  AddInputFromArray<int32>(TensorShape({}), {2});
+  TF_ASSERT_OK(RunOpKernel());
+
+  // Check the output
+  Tensor expected(allocator(), DT_INT32, TensorShape({5}));
+  test::FillValues<int32>(&expected, {0, 2, 4, 6, 8});
+  test::ExpectTensorEqual<int32>(expected, *GetOutput(0));
+}
+
+TEST_F(RangeOpTest, Simple_Float) {
+  MakeOp(DT_FLOAT);
+
+  // Feed and run
+  AddInputFromArray<float>(TensorShape({}), {0.5});
+  AddInputFromArray<float>(TensorShape({}), {2});
+  AddInputFromArray<float>(TensorShape({}), {0.3});
+  TF_ASSERT_OK(RunOpKernel());
+
+  // Check the output
+  Tensor expected(allocator(), DT_FLOAT, TensorShape({5}));
+  test::FillValues<float>(&expected, {0.5, 0.8, 1.1, 1.4, 1.7});
+  test::ExpectTensorEqual<float>(expected, *GetOutput(0));
+}
+
+TEST_F(RangeOpTest, Large_Double) {
+  MakeOp(DT_DOUBLE);
+
+  // Feed and run
+  AddInputFromArray<double>(TensorShape({}), {0.0});
+  AddInputFromArray<double>(TensorShape({}), {10000});
+  AddInputFromArray<double>(TensorShape({}), {0.5});
+  TF_ASSERT_OK(RunOpKernel());
+
+  // Check the output
+  Tensor expected(allocator(), DT_DOUBLE, TensorShape({20000}));
+  std::vector<double> result;
+  for (int32 i = 0; i < 20000; ++i) result.push_back(i * 0.5);
+  test::FillValues<double>(&expected, gtl::ArraySlice<double>(result));
+  test::ExpectTensorEqual<double>(expected, *GetOutput(0));
+}
+
+TEST_F(LinSpaceOpTest, Simple_D32) {
+  MakeOp(DT_FLOAT, DT_INT32);
+
+  // Feed and run
+  AddInputFromArray<float>(TensorShape({}), {3.0});
+  AddInputFromArray<float>(TensorShape({}), {7.0});
+  AddInputFromArray<int32>(TensorShape({}), {3});
+  TF_ASSERT_OK(RunOpKernel());
+
+  // Check the output
+  Tensor expected(allocator(), DT_FLOAT, TensorShape({3}));
+  test::FillValues<float>(&expected, {3.0, 5.0, 7.0});
+  test::ExpectTensorEqual<float>(expected, *GetOutput(0));
+}
+
+TEST_F(LinSpaceOpTest, Single_D64) {
+  MakeOp(DT_FLOAT, DT_INT64);
+
+  // Feed and run
+  AddInputFromArray<float>(TensorShape({}), {9.0});
+  AddInputFromArray<float>(TensorShape({}), {100.0});
+  AddInputFromArray<int64>(TensorShape({}), {1});
+  TF_ASSERT_OK(RunOpKernel());
+
+  // Check the output
+  Tensor expected(allocator(), DT_FLOAT, TensorShape({1}));
+  test::FillValues<float>(&expected, {9.0});
+  test::ExpectTensorEqual<float>(expected, *GetOutput(0));
+}
+
+TEST_F(LinSpaceOpTest, Simple_Double) {
+  MakeOp(DT_DOUBLE, DT_INT32);
+
+  // Feed and run
+  AddInputFromArray<double>(TensorShape({}), {5.0});
+  AddInputFromArray<double>(TensorShape({}), {6.0});
+  AddInputFromArray<int32>(TensorShape({}), {6});
+  TF_ASSERT_OK(RunOpKernel());
+
+  // Check the output
+  Tensor expected(allocator(), DT_DOUBLE, TensorShape({6}));
+  test::FillValues<double>(&expected, {5.0, 5.2, 5.4, 5.6, 5.8, 6.0});
+  test::ExpectTensorEqual<double>(expected, *GetOutput(0));
+}
+
+}  // namespace
+}  // namespace tensorflow
diff --git a/tensorflow/core/kernels/spacetobatch_op.cc b/tensorflow/core/kernels/spacetobatch_op.cc
index c513683918..95c1f5e7e8 100644
--- a/tensorflow/core/kernels/spacetobatch_op.cc
+++ b/tensorflow/core/kernels/spacetobatch_op.cc
@@ -248,40 +248,34 @@ class SpaceToBatchOp : public OpKernel {
   Tensor block_shape_;
 };
 
-#define REGISTER(T)                                                  \
-  REGISTER_KERNEL_BUILDER(Name("SpaceToBatchND")                     \
-                              .Device(DEVICE_CPU)                    \
-                              .TypeConstraint<T>("T")                \
-                              .TypeConstraint<int32>("Tblock_shape") \
-                              .TypeConstraint<int32>("Tpaddings")    \
-                              .HostMemory("block_shape")             \
-                              .HostMemory("paddings"),               \
-                          SpaceToBatchNDOp<CPUDevice, T>);           \
-  REGISTER_KERNEL_BUILDER(Name("SpaceToBatch")                       \
-                              .Device(DEVICE_CPU)                    \
-                              .TypeConstraint<T>("T")                \
-                              .TypeConstraint<int32>("Tpaddings")    \
-                              .HostMemory("paddings"),               \
+#define REGISTER(T)                                        \
+  REGISTER_KERNEL_BUILDER(Name("SpaceToBatchND")           \
+                              .Device(DEVICE_CPU)          \
+                              .TypeConstraint<T>("T")      \
+                              .HostMemory("block_shape")   \
+                              .HostMemory("paddings"),     \
+                          SpaceToBatchNDOp<CPUDevice, T>); \
+  REGISTER_KERNEL_BUILDER(Name("SpaceToBatch")             \
+                              .Device(DEVICE_CPU)          \
+                              .TypeConstraint<T>("T")      \
+                              .HostMemory("paddings"),     \
                           SpaceToBatchOp<CPUDevice, T>);
 
 TF_CALL_REAL_NUMBER_TYPES(REGISTER);
 #undef REGISTER
 
 #if GOOGLE_CUDA
-#define REGISTER(T)                                                  \
-  REGISTER_KERNEL_BUILDER(Name("SpaceToBatchND")                     \
-                              .Device(DEVICE_GPU)                    \
-                              .TypeConstraint<T>("T")                \
-                              .TypeConstraint<int32>("Tblock_shape") \
-                              .TypeConstraint<int32>("Tpaddings")    \
-                              .HostMemory("block_shape")             \
-                              .HostMemory("paddings"),               \
-                          SpaceToBatchNDOp<GPUDevice, T>);           \
-  REGISTER_KERNEL_BUILDER(Name("SpaceToBatch")                       \
-                              .Device(DEVICE_GPU)                    \
-                              .TypeConstraint<T>("T")                \
-                              .TypeConstraint<int32>("Tpaddings")    \
-                              .HostMemory("paddings"),               \
+#define REGISTER(T)                                        \
+  REGISTER_KERNEL_BUILDER(Name("SpaceToBatchND")           \
+                              .Device(DEVICE_GPU)          \
+                              .TypeConstraint<T>("T")      \
+                              .HostMemory("block_shape")   \
+                              .HostMemory("paddings"),     \
+                          SpaceToBatchNDOp<GPUDevice, T>); \
+  REGISTER_KERNEL_BUILDER(Name("SpaceToBatch")             \
+                              .Device(DEVICE_GPU)          \
+                              .TypeConstraint<T>("T")      \
+                              .HostMemory("paddings"),     \
                           SpaceToBatchOp<GPUDevice, T>);
 
 TF_CALL_GPU_NUMBER_TYPES(REGISTER);
diff --git a/tensorflow/core/kernels/sparse_matmul_op.h b/tensorflow/core/kernels/sparse_matmul_op.h
index 308b641b54..cca52558ae 100644
--- a/tensorflow/core/kernels/sparse_matmul_op.h
+++ b/tensorflow/core/kernels/sparse_matmul_op.h
@@ -54,8 +54,9 @@ EIGEN_DEVICE_FUNC inline Packet pexpand_bf16_u(const Packet& from) {
 }
 
 // Specialization non-scalar version on non-sse.
+// Enable vectorization on z13 and higher
 #if defined(EIGEN_VECTORIZE_ALTIVEC) || defined(EIGEN_VECTORIZE_VSX) || \
-    defined(EIGEN_VECTORIZE_NEON)
+    defined(EIGEN_VECTORIZE_NEON) || defined(EIGEN_VECTORIZE_ZVECTOR)
 template <typename Packet>
 EIGEN_DEVICE_FUNC inline Packet4f pexpand_bf16_l(const Packet4f& from) {
   float r[4];
@@ -126,8 +127,9 @@ EIGEN_DEVICE_FUNC inline Packet pload2bf16(
 }
 
 // Specialization for pload4bf16 and pload2bf16 for non-sse.
+// Enable vectorization on z13 and higher.
 #if defined(EIGEN_VECTORIZE_ALTIVEC) || defined(EIGEN_VECTORIZE_VSX) || \
-    defined(EIGEN_VECTORIZE_NEON)
+    defined(EIGEN_VECTORIZE_NEON) || defined(EIGEN_VECTORIZE_ZVECTOR)
 template <>
 EIGEN_STRONG_INLINE Packet4f pload4bf16<Packet4f>(const float* from) {
   tensorflow::uint32 p[4];
diff --git a/tensorflow/core/kernels/stage_op.cc b/tensorflow/core/kernels/stage_op.cc
index 1717428adf..0fae46dea6 100644
--- a/tensorflow/core/kernels/stage_op.cc
+++ b/tensorflow/core/kernels/stage_op.cc
@@ -53,7 +53,10 @@ class Buffer : public ResourceBase {
   void notify_inserters_if_bounded(std::unique_lock<std::mutex>* lock) {
     if (IsBounded()) {
       lock->unlock();
-      full_cond_var_.notify_one();
+      // Notify all inserters. The removal of an element
+      // may make memory available for many inserters
+      // to insert new elements
+      full_cond_var_.notify_all();
     }
   }
 
@@ -115,9 +118,12 @@ class Buffer : public ResourceBase {
     buf_.push_back(std::move(*tuple));
 
     lock.unlock();
-    // maybe possible to optimize by reducing
-    // how often this signal is sent
-    non_empty_cond_var_.notify_one();
+    // Notify all removers. Removers
+    // may be peeking at a specific element or waiting
+    // for the element at the front of the deque.
+    // As we don't know the appropriate one to wake up
+    // we should wake them all.
+    non_empty_cond_var_.notify_all();
 
     return Status::OK();
   }
diff --git a/tensorflow/core/kernels/stateless_random_ops.cc b/tensorflow/core/kernels/stateless_random_ops.cc
index 79d0c07acd..f6fb0a121d 100644
--- a/tensorflow/core/kernels/stateless_random_ops.cc
+++ b/tensorflow/core/kernels/stateless_random_ops.cc
@@ -137,7 +137,6 @@ TF_CALL_double(REGISTER);
           .Device(DEVICE_GPU)                                          \
           .HostMemory("shape")                                         \
           .HostMemory("seed")                                          \
-          .TypeConstraint<int32>("T")                                  \
           .TypeConstraint<TYPE>("dtype"),                              \
       StatelessRandomOp<GPUDevice, random::UniformDistribution<        \
                                        random::PhiloxRandom, TYPE> >); \
@@ -146,7 +145,6 @@ TF_CALL_double(REGISTER);
           .Device(DEVICE_GPU)                                          \
           .HostMemory("shape")                                         \
           .HostMemory("seed")                                          \
-          .TypeConstraint<int32>("T")                                  \
           .TypeConstraint<TYPE>("dtype"),                              \
       StatelessRandomOp<GPUDevice, random::NormalDistribution<         \
                                        random::PhiloxRandom, TYPE> >); \
@@ -155,7 +153,6 @@ TF_CALL_double(REGISTER);
           .Device(DEVICE_GPU)                                          \
           .HostMemory("shape")                                         \
           .HostMemory("seed")                                          \
-          .TypeConstraint<int32>("T")                                  \
           .TypeConstraint<TYPE>("dtype"),                              \
       StatelessRandomOp<                                               \
           GPUDevice,                                                   \
diff --git a/tensorflow/core/kernels/tile_functor.h b/tensorflow/core/kernels/tile_functor.h
index 28af2dace3..189be9239b 100644
--- a/tensorflow/core/kernels/tile_functor.h
+++ b/tensorflow/core/kernels/tile_functor.h
@@ -17,6 +17,7 @@ limitations under the License.
 #define TENSORFLOW_KERNELS_TILE_FUNCTOR_H_
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/platform/types.h"
@@ -29,13 +30,13 @@ namespace internal {
 template <typename Device, typename T>
 void TileSimple(const Device& d, Tensor* out, const Tensor& in);
 
-template <typename Device, typename T, int NDIM>
+template <typename Device, typename T, typename Tmultiples, int NDIM>
 void TileUsingEigen(const Device& d, Tensor* out, const Tensor& in,
-                    const gtl::ArraySlice<int32>& broadcast_array) {
+                    const gtl::ArraySlice<Tmultiples>& broadcast_array) {
   auto x = in.tensor<T, NDIM>();
   auto y = out->tensor<T, NDIM>();
 
-  Eigen::array<int32, NDIM> b;
+  Eigen::array<Tmultiples, NDIM> b;
   for (int i = 0; i < NDIM; ++i) b[i] = broadcast_array[i];
   if (Eigen::internal::is_same<Device, Eigen::GpuDevice>::value) {
     // Use 32bit indexing to speed up the computations
@@ -45,9 +46,9 @@ void TileUsingEigen(const Device& d, Tensor* out, const Tensor& in,
   }
 }
 
-template <typename Device, typename T>
+template <typename Device, typename T, typename Tmultiples>
 void TileUsingEigen(const Device& d, Tensor* out, const Tensor& in,
-                    const gtl::ArraySlice<int32>&) {
+                    const gtl::ArraySlice<Tmultiples>&) {
   auto x = in.tensor<T, 0>();
   auto y = out->tensor<T, 0>();
   // In the scalar case we simply copy the input.
@@ -58,34 +59,42 @@ void TileUsingEigen(const Device& d, Tensor* out, const Tensor& in,
 
 namespace functor {
 
-template <typename Device, typename T>
+template <typename Device, typename T, typename Tmultiples>
 struct Tile {
   void operator()(const Device& d, Tensor* out, const Tensor& in,
-                  const gtl::ArraySlice<int32> broadcast_array) const {
+                  const gtl::ArraySlice<Tmultiples> broadcast_array) const {
     switch (in.dims()) {
       case 0:
-        internal::TileUsingEigen<Device, T>(d, out, in, broadcast_array);
+        internal::TileUsingEigen<Device, T, Tmultiples>(d, out, in,
+                                                        broadcast_array);
         break;
       case 1:
-        internal::TileUsingEigen<Device, T, 1>(d, out, in, broadcast_array);
+        internal::TileUsingEigen<Device, T, Tmultiples, 1>(d, out, in,
+                                                           broadcast_array);
         break;
       case 2:
-        internal::TileUsingEigen<Device, T, 2>(d, out, in, broadcast_array);
+        internal::TileUsingEigen<Device, T, Tmultiples, 2>(d, out, in,
+                                                           broadcast_array);
         break;
       case 3:
-        internal::TileUsingEigen<Device, T, 3>(d, out, in, broadcast_array);
+        internal::TileUsingEigen<Device, T, Tmultiples, 3>(d, out, in,
+                                                           broadcast_array);
         break;
       case 4:
-        internal::TileUsingEigen<Device, T, 4>(d, out, in, broadcast_array);
+        internal::TileUsingEigen<Device, T, Tmultiples, 4>(d, out, in,
+                                                           broadcast_array);
         break;
       case 5:
-        internal::TileUsingEigen<Device, T, 5>(d, out, in, broadcast_array);
+        internal::TileUsingEigen<Device, T, Tmultiples, 5>(d, out, in,
+                                                           broadcast_array);
         break;
       case 6:
-        internal::TileUsingEigen<Device, T, 6>(d, out, in, broadcast_array);
+        internal::TileUsingEigen<Device, T, Tmultiples, 6>(d, out, in,
+                                                           broadcast_array);
         break;
       case 7:
-        internal::TileUsingEigen<Device, T, 7>(d, out, in, broadcast_array);
+        internal::TileUsingEigen<Device, T, Tmultiples, 7>(d, out, in,
+                                                           broadcast_array);
         break;
       default:
         internal::TileSimple<Device, T>(d, out, in);
diff --git a/tensorflow/core/kernels/tile_functor_cpu.cc b/tensorflow/core/kernels/tile_functor_cpu.cc
index 5952d49221..b2fd669541 100644
--- a/tensorflow/core/kernels/tile_functor_cpu.cc
+++ b/tensorflow/core/kernels/tile_functor_cpu.cc
@@ -15,10 +15,10 @@ limitations under the License.
 
 #define EIGEN_USE_THREADS
 
+#include "tensorflow/core/kernels/tile_functor.h"
 #include "tensorflow/core/framework/attr_value.pb.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/kernels/ops_util.h"
-#include "tensorflow/core/kernels/tile_functor.h"
 
 namespace tensorflow {
 
@@ -51,7 +51,9 @@ namespace functor {
 typedef Eigen::ThreadPoolDevice CPUDevice;
 
 // Register functors used for Tile functor.
-#define DEFINE_TYPE(T) template struct Tile<CPUDevice, T>;
+#define DEFINE_TYPE(T)                       \
+  template struct Tile<CPUDevice, T, int32>; \
+  template struct Tile<CPUDevice, T, int64>;
 
 TF_CALL_bool(DEFINE_TYPE);
 TF_CALL_float(DEFINE_TYPE);
@@ -70,7 +72,9 @@ TF_CALL_string(DEFINE_TYPE);
 #ifdef TENSORFLOW_USE_SYCL
 typedef Eigen::SyclDevice SYCLDevice;
 
-#define DEFINE_TYPE(T) template struct Tile<SYCLDevice, T>;
+#define DEFINE_TYPE(T)                        \
+  template struct Tile<SYCLDevice, T, int32>; \
+  template struct Tile<SYCLDevice, T, int64>;
 
 TF_CALL_bool(DEFINE_TYPE);
 TF_CALL_float(DEFINE_TYPE);
@@ -81,7 +85,7 @@ TF_CALL_int16(DEFINE_TYPE);
 TF_CALL_int64(DEFINE_TYPE);
 
 #undef DEFINE_TYPE
-#endif // TENSORFLOW_USE_SYCL
+#endif  // TENSORFLOW_USE_SYCL
 
 }  // end namespace functor
 }  // end namespace tensorflow
diff --git a/tensorflow/core/kernels/tile_functor_gpu.cu.cc b/tensorflow/core/kernels/tile_functor_gpu.cu.cc
index 1c61c3030a..5a36e7567b 100644
--- a/tensorflow/core/kernels/tile_functor_gpu.cu.cc
+++ b/tensorflow/core/kernels/tile_functor_gpu.cu.cc
@@ -18,10 +18,11 @@ limitations under the License.
 #define EIGEN_USE_GPU
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
-#include "tensorflow/core/kernels/tile_functor.h"
+
+#include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/kernels/ops_util.h"
+#include "tensorflow/core/kernels/tile_functor.h"
 #include "tensorflow/core/util/cuda_kernel_helper.h"
-#include "tensorflow/core/framework/register_types.h"
 
 namespace tensorflow {
 namespace internal {
@@ -60,7 +61,8 @@ void TileSimple(const Device& d, Tensor* out, const Tensor& in) {
     host_buf[ndims + i] = out_strides[i];
     host_buf[ndims * 2 + i] = in.dim_size(i);
   }
-  // Copies the input strides, output strides and input dimension sizes to the device.
+  // Copies the input strides, output strides and input dimension sizes to the
+  // device.
   auto num_bytes = sizeof(int64) * host_buf.size();
   auto dev_buf = d.allocate(num_bytes);
   // NOTE: host_buf is not allocated by CudaHostAllocator, and
@@ -84,7 +86,9 @@ namespace functor {
 typedef Eigen::GpuDevice GPUDevice;
 
 // Register functors used for Tile functor.
-#define DEFINE_TYPE(T) template struct Tile<GPUDevice, T>;
+#define DEFINE_TYPE(T)                       \
+  template struct Tile<GPUDevice, T, int32>; \
+  template struct Tile<GPUDevice, T, int64>;
 
 TF_CALL_int16(DEFINE_TYPE);
 TF_CALL_int32(DEFINE_TYPE);
diff --git a/tensorflow/core/kernels/tile_ops.cc b/tensorflow/core/kernels/tile_ops.cc
index c49ebc0685..4c496a12c2 100644
--- a/tensorflow/core/kernels/tile_ops.cc
+++ b/tensorflow/core/kernels/tile_ops.cc
@@ -42,14 +42,14 @@ typedef Eigen::ThreadPoolDevice CPUDevice;
 typedef Eigen::GpuDevice GPUDevice;
 #ifdef TENSORFLOW_USE_SYCL
 typedef Eigen::SyclDevice SYCLDevice;
-#endif // TENSORFLOW_USE_SYCL
+#endif  // TENSORFLOW_USE_SYCL
 
 // Forward declarations of functors that will be defined in tile_ops_impl.h
 namespace functor {
-template <typename Device, typename T>
+template <typename Device, typename T, typename Tmultiple>
 struct Tile {
   void operator()(const Device& d, Tensor* out, const Tensor& in,
-                  const gtl::ArraySlice<int32> broadcast_array) const;
+                  const gtl::ArraySlice<Tmultiple> broadcast_array) const;
 };
 
 template <typename Device, typename T, int NDIM>
@@ -80,7 +80,7 @@ struct ReduceAndReshape {
 }  // namespace functor
 
 // --------------------------------------------------------------------------
-template <typename Device>
+template <typename Device, typename Tmultiples>
 class TileOp : public OpKernel {
  public:
   explicit TileOp(OpKernelConstruction* context) : OpKernel(context) {}
@@ -105,8 +105,8 @@ class TileOp : public OpKernel {
       return;
     }
 
-    const gtl::ArraySlice<int32> multiples_array(multiples.flat<int32>().data(),
-                                                 input_dims);
+    const gtl::ArraySlice<Tmultiples> multiples_array(
+        multiples.flat<Tmultiples>().data(), input_dims);
     TensorShape output_shape;
     for (int i = 0; i < input_dims; ++i) {
       OP_REQUIRES(
@@ -125,10 +125,10 @@ class TileOp : public OpKernel {
     // If there's no output, there's nothing to do.
     if (output_shape.num_elements() == 0) return;
 
-#define HANDLE_TYPE(DT)                                        \
-  if (context->input(0).dtype() == DT) {                       \
-    HandleCase<DT>(context, multiples_array, result);          \
-    return;                                                    \
+#define HANDLE_TYPE(DT)                               \
+  if (context->input(0).dtype() == DT) {              \
+    HandleCase<DT>(context, multiples_array, result); \
+    return;                                           \
   }
 
 #define HANDLE_TYPE_NAME(T) HANDLE_TYPE(DataTypeToEnum<T>::value)
@@ -158,27 +158,27 @@ class TileOp : public OpKernel {
  private:
   template <DataType DT>
   void HandleCaseImpl(OpKernelContext* context,
-                      const gtl::ArraySlice<int32>& multiples_array,
+                      const gtl::ArraySlice<Tmultiples>& multiples_array,
                       Tensor* result) {
     typedef typename EnumToDataType<DT>::Type T;
-    functor::Tile<Device, T>() (
-        context->eigen_device<Device>(), result,
-        context->input(0), multiples_array);
+    functor::Tile<Device, T, Tmultiples>()(context->eigen_device<Device>(),
+                                           result, context->input(0),
+                                           multiples_array);
   }
 
   template <DataType DT>
   void HandleCase(OpKernelContext* context,
-                  const gtl::ArraySlice<int32>& multiples_array,
+                  const gtl::ArraySlice<Tmultiples>& multiples_array,
                   Tensor* result);
 
   TF_DISALLOW_COPY_AND_ASSIGN(TileOp);
 };
 
-template <typename Device>
+template <typename Device, typename Tmultiples>
 template <DataType DT>
-inline void TileOp<Device>::HandleCase(
-    OpKernelContext* context, const gtl::ArraySlice<int32>& multiples_array,
-    Tensor* result) {
+inline void TileOp<Device, Tmultiples>::HandleCase(
+    OpKernelContext* context,
+    const gtl::ArraySlice<Tmultiples>& multiples_array, Tensor* result) {
   // TODO(vrv): print out the device name if useful. Currently disabled to avoid
   // having to use RTTI.
   LOG(FATAL) << "TileOp: Invalid combination of Device, DT: "
@@ -186,25 +186,28 @@ inline void TileOp<Device>::HandleCase(
              << DataTypeString(DT);
 }
 
-#define HANDLE_CASE(device, dtype)                                     \
-  template <>                                                          \
-  template <>                                                          \
-  void TileOp<device>::HandleCase<dtype>(                              \
-      OpKernelContext * context,                                       \
-      const gtl::ArraySlice<int32>& multiples_array, Tensor* result) { \
-    HandleCaseImpl<dtype>(context, multiples_array, result);           \
+#define HANDLE_CASE(device, dtype, Tmultiples)                              \
+  template <>                                                               \
+  template <>                                                               \
+  void TileOp<device, Tmultiples>::HandleCase<dtype>(                       \
+      OpKernelContext * context,                                            \
+      const gtl::ArraySlice<Tmultiples>& multiples_array, Tensor* result) { \
+    HandleCaseImpl<dtype>(context, multiples_array, result);                \
   }
 
-#define HANDLE_TYPE_NAME_CPU(T) \
-  HANDLE_CASE(CPUDevice, DataTypeToEnum<T>::value);
+#define HANDLE_TYPE_NAME_CPU(T)                            \
+  HANDLE_CASE(CPUDevice, DataTypeToEnum<T>::value, int32); \
+  HANDLE_CASE(CPUDevice, DataTypeToEnum<T>::value, int64);
 
-#define HANDLE_TYPE_NAME_GPU(T) \
-  HANDLE_CASE(GPUDevice, DataTypeToEnum<T>::value);
+#define HANDLE_TYPE_NAME_GPU(T)                            \
+  HANDLE_CASE(GPUDevice, DataTypeToEnum<T>::value, int32); \
+  HANDLE_CASE(GPUDevice, DataTypeToEnum<T>::value, int64);
 
 #ifdef TENSORFLOW_USE_SYCL
-#define HANDLE_TYPE_NAME_SYCL(T) \
-  HANDLE_CASE(SYCLDevice, DataTypeToEnum<T>::value);
-#endif // TENSORFLOW_USE_SYCL
+#define HANDLE_TYPE_NAME_SYCL(T)                            \
+  HANDLE_CASE(SYCLDevice, DataTypeToEnum<T>::value, int32); \
+  HANDLE_CASE(SYCLDevice, DataTypeToEnum<T>::value, int64);
+#endif  // TENSORFLOW_USE_SYCL
 
 TF_CALL_bool(HANDLE_TYPE_NAME_CPU);
 TF_CALL_float(HANDLE_TYPE_NAME_CPU);
@@ -235,13 +238,13 @@ TF_CALL_double(HANDLE_TYPE_NAME_SYCL);
 TF_CALL_int16(HANDLE_TYPE_NAME_SYCL);
 TF_CALL_int32(HANDLE_TYPE_NAME_SYCL);
 TF_CALL_int64(HANDLE_TYPE_NAME_SYCL);
-#endif // TENSORFLOW_USE_SYCL
+#endif  // TENSORFLOW_USE_SYCL
 
 #undef HANDLE_TYPE_NAME_CPU
 #undef HANDLE_TYPE_NAME_GPU
 #ifdef TENSORFLOW_USE_SYCL
 #undef HANDLE_TYPE_NAME_SYCL
-#endif // TENSORFLOW_USE_SYCL
+#endif  // TENSORFLOW_USE_SYCL
 #undef HANDLE_CASE
 
 // --------------------------------------------------------------------------
@@ -494,7 +497,7 @@ TF_CALL_int16(HANDLE_TYPE_NAME_SYCL);
 TF_CALL_int32(HANDLE_TYPE_NAME_SYCL);
 TF_CALL_int64(HANDLE_TYPE_NAME_SYCL);
 #undef HANDLE_TYPE_NAME_SYCL
-#endif // TENSORFLOW_USE_SYCL
+#endif  // TENSORFLOW_USE_SYCL
 
 #undef HANDLE_TYPE_NAME_CPU
 #undef HANDLE_TYPE_NAME_GPU
@@ -505,127 +508,73 @@ REGISTER_KERNEL_BUILDER(Name("Tile")
                             .Device(DEVICE_CPU)
                             .HostMemory("multiples")
                             .TypeConstraint<int32>("Tmultiples"),
-                        TileOp<CPUDevice>);
+                        TileOp<CPUDevice, int32>);
+REGISTER_KERNEL_BUILDER(Name("Tile")
+                            .Device(DEVICE_CPU)
+                            .HostMemory("multiples")
+                            .TypeConstraint<int64>("Tmultiples"),
+                        TileOp<CPUDevice, int64>);
 REGISTER_KERNEL_BUILDER(
     Name("TileGrad").Device(DEVICE_CPU).HostMemory("multiples"),
     TileGradientOp<CPUDevice>);
 
 #if GOOGLE_CUDA
-
-REGISTER_KERNEL_BUILDER(Name("Tile")
-                            .Device(DEVICE_GPU)
-                            .TypeConstraint<float>("T")
-                            .TypeConstraint<int32>("Tmultiples")
-                            .HostMemory("multiples"),
-                        TileOp<GPUDevice>);
-REGISTER_KERNEL_BUILDER(Name("Tile")
-                            .Device(DEVICE_GPU)
-                            .TypeConstraint<double>("T")
-                            .TypeConstraint<int32>("Tmultiples")
-                            .HostMemory("multiples"),
-                        TileOp<GPUDevice>);
-REGISTER_KERNEL_BUILDER(Name("Tile")
-                            .Device(DEVICE_GPU)
-                            .TypeConstraint<Eigen::half>("T")
-                            .TypeConstraint<int32>("Tmultiples")
-                            .HostMemory("multiples"),
-                        TileOp<GPUDevice>);
-REGISTER_KERNEL_BUILDER(Name("Tile")
-                            .Device(DEVICE_GPU)
-                            .TypeConstraint<int16>("T")
-                            .TypeConstraint<int32>("Tmultiples")
-                            .HostMemory("multiples"),
-                        TileOp<GPUDevice>);
-REGISTER_KERNEL_BUILDER(Name("Tile")
-                            .Device(DEVICE_GPU)
-                            .TypeConstraint<int32>("T")
-                            .TypeConstraint<int32>("Tmultiples")
-                            .HostMemory("multiples"),
-                        TileOp<GPUDevice>);
-REGISTER_KERNEL_BUILDER(Name("Tile")
-                            .Device(DEVICE_GPU)
-                            .TypeConstraint<complex64>("T")
-                            .TypeConstraint<int32>("Tmultiples")
-                            .HostMemory("multiples"),
-                        TileOp<GPUDevice>);
-REGISTER_KERNEL_BUILDER(Name("Tile")
-                            .Device(DEVICE_GPU)
-                            .TypeConstraint<complex128>("T")
-                            .TypeConstraint<int32>("Tmultiples")
-                            .HostMemory("multiples"),
-                        TileOp<GPUDevice>);
-
-REGISTER_KERNEL_BUILDER(Name("TileGrad")
-                            .Device(DEVICE_GPU)
-                            .TypeConstraint<float>("T")
-                            .TypeConstraint<int32>("Tmultiples")
-                            .HostMemory("multiples"),
-                        TileGradientOp<GPUDevice>);
-REGISTER_KERNEL_BUILDER(Name("TileGrad")
-                            .Device(DEVICE_GPU)
-                            .TypeConstraint<double>("T")
-                            .TypeConstraint<int32>("Tmultiples")
-                            .HostMemory("multiples"),
-                        TileGradientOp<GPUDevice>);
-REGISTER_KERNEL_BUILDER(Name("TileGrad")
-                            .Device(DEVICE_GPU)
-                            .TypeConstraint<Eigen::half>("T")
-                            .TypeConstraint<int32>("Tmultiples")
-                            .HostMemory("multiples"),
-                        TileGradientOp<GPUDevice>);
-REGISTER_KERNEL_BUILDER(Name("TileGrad")
-                            .Device(DEVICE_GPU)
-                            .TypeConstraint<int16>("T")
-                            .TypeConstraint<int32>("Tmultiples")
-                            .HostMemory("multiples"),
-                        TileGradientOp<GPUDevice>);
-REGISTER_KERNEL_BUILDER(Name("TileGrad")
-                            .Device(DEVICE_GPU)
-                            .TypeConstraint<int32>("T")
-                            .TypeConstraint<int32>("Tmultiples")
-                            .HostMemory("multiples"),
-                        TileGradientOp<GPUDevice>);
-REGISTER_KERNEL_BUILDER(Name("TileGrad")
-                            .Device(DEVICE_GPU)
-                            .TypeConstraint<complex64>("T")
-                            .TypeConstraint<int32>("Tmultiples")
-                            .HostMemory("multiples"),
-                        TileGradientOp<GPUDevice>);
-REGISTER_KERNEL_BUILDER(Name("TileGrad")
-                            .Device(DEVICE_GPU)
-                            .TypeConstraint<complex128>("T")
-                            .TypeConstraint<int32>("Tmultiples")
-                            .HostMemory("multiples"),
-                        TileGradientOp<GPUDevice>);
-
+#define REGISTER_GPU(type)                                         \
+  REGISTER_KERNEL_BUILDER(Name("Tile")                             \
+                              .Device(DEVICE_GPU)                  \
+                              .TypeConstraint<type>("T")           \
+                              .TypeConstraint<int32>("Tmultiples") \
+                              .HostMemory("multiples"),            \
+                          TileOp<GPUDevice, int32>);               \
+  REGISTER_KERNEL_BUILDER(Name("Tile")                             \
+                              .Device(DEVICE_GPU)                  \
+                              .TypeConstraint<type>("T")           \
+                              .TypeConstraint<int64>("Tmultiples") \
+                              .HostMemory("multiples"),            \
+                          TileOp<GPUDevice, int64>);               \
+  REGISTER_KERNEL_BUILDER(Name("TileGrad")                         \
+                              .Device(DEVICE_GPU)                  \
+                              .TypeConstraint<type>("T")           \
+                              .TypeConstraint<int32>("Tmultiples") \
+                              .HostMemory("multiples"),            \
+                          TileGradientOp<GPUDevice>);
+
+TF_CALL_float(REGISTER_GPU);
+TF_CALL_double(REGISTER_GPU);
+TF_CALL_half(REGISTER_GPU);
+TF_CALL_int16(REGISTER_GPU);
+TF_CALL_int32(REGISTER_GPU);
+TF_CALL_complex64(REGISTER_GPU);
+TF_CALL_complex128(REGISTER_GPU)
+
+#undef REGISTER_GPU
 #endif  // GOOGLE_CUDA
 
 #ifdef TENSORFLOW_USE_SYCL
-REGISTER_KERNEL_BUILDER(Name("Tile")
-                            .Device(DEVICE_SYCL)
-                            .TypeConstraint<float>("T")
-                            .TypeConstraint<int32>("Tmultiples")
-                            .HostMemory("multiples"),
-                        TileOp<SYCLDevice>);
-REGISTER_KERNEL_BUILDER(Name("Tile")
-                            .Device(DEVICE_SYCL)
-                            .TypeConstraint<double>("T")
-                            .TypeConstraint<int32>("Tmultiples")
-                            .HostMemory("multiples"),
-                        TileOp<SYCLDevice>);
-
-REGISTER_KERNEL_BUILDER(Name("TileGrad")
-                            .Device(DEVICE_SYCL)
-                            .TypeConstraint<float>("T")
-                            .TypeConstraint<int32>("Tmultiples")
-                            .HostMemory("multiples"),
-                        TileGradientOp<SYCLDevice>);
-REGISTER_KERNEL_BUILDER(Name("TileGrad")
-                            .Device(DEVICE_SYCL)
-                            .TypeConstraint<double>("T")
-                            .TypeConstraint<int32>("Tmultiples")
-                            .HostMemory("multiples"),
-                        TileGradientOp<SYCLDevice>);
-#endif // TENSORFLOW_USE_SYCL
+#define REGISTER_SYCL(type)                                        \
+  REGISTER_KERNEL_BUILDER(Name("Tile")                             \
+                              .Device(DEVICE_SYCL)                 \
+                              .TypeConstraint<type>("T")           \
+                              .TypeConstraint<int32>("Tmultiples") \
+                              .HostMemory("multiples"),            \
+                          TileOp<SYCLDevice, int32>);              \
+  REGISTER_KERNEL_BUILDER(Name("Tile")                             \
+                              .Device(DEVICE_SYCL)                 \
+                              .TypeConstraint<type>("T")           \
+                              .TypeConstraint<int64>("Tmultiples") \
+                              .HostMemory("multiples"),            \
+                          TileOp<SYCLDevice, int64>);              \
+  REGISTER_KERNEL_BUILDER(Name("TileGrad")                         \
+                              .Device(DEVICE_SYCL)                 \
+                              .TypeConstraint<type>("T")           \
+                              .TypeConstraint<int32>("Tmultiples") \
+                              .HostMemory("multiples"),            \
+                          TileGradientOp<SYCLDevice>);
+
+    TF_CALL_float(REGISTER_SYCL);
+TF_CALL_double(REGISTER_SYCL);
+
+#undef REGISTER_SYCL
+#endif  // TENSORFLOW_USE_SYCL
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/kernels/transpose_op.cc b/tensorflow/core/kernels/transpose_op.cc
index e151b38d90..20f0edf309 100644
--- a/tensorflow/core/kernels/transpose_op.cc
+++ b/tensorflow/core/kernels/transpose_op.cc
@@ -91,6 +91,26 @@ REGISTER_KERNEL_BUILDER(Name("InvertPermutation")
                         InvertPermutationOp);
 #endif  // TENSORFLOW_USE_SYCL
 
+namespace {
+template <typename Tperm>
+Status PermutationHelper(const Tensor& perm, const int dims,
+                         std::vector<int32>* permutation) {
+  auto Vperm = perm.vec<Tperm>();
+  if (dims != Vperm.size()) {
+    return errors::InvalidArgument("transpose expects a vector of size ", dims,
+                                   ". But input(1) is a vector of size ",
+                                   Vperm.size());
+  }
+  // using volatile instead of SubtleMustCopy here so that the
+  // asynchrony boundary is permutation.
+  const volatile Tperm* perm_begin =
+      reinterpret_cast<const volatile Tperm*>(Vperm.data());
+  *permutation = std::vector<int32>(perm_begin, perm_begin + dims);
+
+  return Status::OK();
+}
+}  // namespace
+
 // output = TransposeOp(T<any> input, T<int32> perm) takes a tensor
 // of type T and rank N, and a permutation of 0, 1, ..., N-1. It
 // shuffles the dimensions of the input tensor according to permutation.
@@ -113,17 +133,16 @@ void TransposeOp::Compute(OpKernelContext* ctx) {
   OP_REQUIRES(ctx, TensorShapeUtils::IsVector(perm.shape()),
               errors::InvalidArgument("perm must be a vector, not ",
                                       perm.shape().DebugString()));
-  auto Vperm = perm.vec<int32>();
+
+  // Although Tperm may be an int64 type, an int32 is sufficient to hold
+  // dimension range values, so the narrowing here should be safe.
+  std::vector<int32> permutation;
   const int dims = input.dims();
-  OP_REQUIRES(ctx, dims == Vperm.size(),
-              errors::InvalidArgument(
-                  "transpose expects a vector of size ", input.dims(),
-                  ". But input(1) is a vector of size ", Vperm.size()));
-  // using volatile instead of SubtleMustCopy here so that the
-  // asynchrony boundary is permutation.
-  const volatile int32* perm_begin =
-      reinterpret_cast<const volatile int32*>(Vperm.data());
-  const std::vector<int32> permutation(perm_begin, perm_begin + dims);
+  if (perm.dtype() == DT_INT32) {
+    OP_REQUIRES_OK(ctx, PermutationHelper<int32>(perm, dims, &permutation));
+  } else {
+    OP_REQUIRES_OK(ctx, PermutationHelper<int64>(perm, dims, &permutation));
+  }
   TensorShape shape;
 
   // Check whether permutation is a permutation of integers of [0 .. dims).
@@ -142,10 +161,9 @@ void TransposeOp::Compute(OpKernelContext* ctx) {
     }
   }
   for (int i = 0; i < dims; ++i) {
-    OP_REQUIRES(
-        ctx, bits[i],
-        errors::InvalidArgument(i, " is missing from {",
-                                str_util::Join(permutation, ","), "}."));
+    OP_REQUIRES(ctx, bits[i], errors::InvalidArgument(
+                                  i, " is missing from {",
+                                  str_util::Join(permutation, ","), "}."));
   }
 
   // 0-D, 1-D, and identity transposes do nothing.
@@ -185,18 +203,16 @@ Status ConjugateTransposeCpuOp::DoTranspose(OpKernelContext* ctx,
 }
 
 #ifdef INTEL_MKL
-#define REGISTER(T)                                           \
-  REGISTER_KERNEL_BUILDER(Name("Transpose")                   \
-                              .Device(DEVICE_CPU)             \
-                              .TypeConstraint<T>("T")         \
-                              .TypeConstraint<int32>("Tperm") \
-                              .HostMemory("perm"),            \
-                          MklTransposeCpuOp);                 \
-  REGISTER_KERNEL_BUILDER(Name("ConjugateTranspose")          \
-                              .Device(DEVICE_CPU)             \
-                              .TypeConstraint<T>("T")         \
-                              .TypeConstraint<int32>("Tperm") \
-                              .HostMemory("perm"),            \
+#define REGISTER(T)                                   \
+  REGISTER_KERNEL_BUILDER(Name("Transpose")           \
+                              .Device(DEVICE_CPU)     \
+                              .TypeConstraint<T>("T") \
+                              .HostMemory("perm"),    \
+                          MklTransposeCpuOp);         \
+  REGISTER_KERNEL_BUILDER(Name("ConjugateTranspose")  \
+                              .Device(DEVICE_CPU)     \
+                              .TypeConstraint<T>("T") \
+                              .HostMemory("perm"),    \
                           MklConjugateTransposeCpuOp);
 TF_CALL_ALL_TYPES(REGISTER);
 REGISTER(bfloat16);
@@ -204,18 +220,16 @@ REGISTER(bfloat16);
 
 #else  // INTEL_MKL
 
-#define REGISTER(T)                                           \
-  REGISTER_KERNEL_BUILDER(Name("Transpose")                   \
-                              .Device(DEVICE_CPU)             \
-                              .TypeConstraint<T>("T")         \
-                              .TypeConstraint<int32>("Tperm") \
-                              .HostMemory("perm"),            \
-                          TransposeCpuOp);                    \
-  REGISTER_KERNEL_BUILDER(Name("ConjugateTranspose")          \
-                              .Device(DEVICE_CPU)             \
-                              .TypeConstraint<T>("T")         \
-                              .TypeConstraint<int32>("Tperm") \
-                              .HostMemory("perm"),            \
+#define REGISTER(T)                                   \
+  REGISTER_KERNEL_BUILDER(Name("Transpose")           \
+                              .Device(DEVICE_CPU)     \
+                              .TypeConstraint<T>("T") \
+                              .HostMemory("perm"),    \
+                          TransposeCpuOp);            \
+  REGISTER_KERNEL_BUILDER(Name("ConjugateTranspose")  \
+                              .Device(DEVICE_CPU)     \
+                              .TypeConstraint<T>("T") \
+                              .HostMemory("perm"),    \
                           ConjugateTransposeCpuOp);
 TF_CALL_ALL_TYPES(REGISTER)
 REGISTER(bfloat16);
@@ -238,18 +252,16 @@ Status ConjugateTransposeGpuOp::DoTranspose(OpKernelContext* ctx,
                                             perm, out);
 }
 
-#define REGISTER(T)                                           \
-  REGISTER_KERNEL_BUILDER(Name("Transpose")                   \
-                              .Device(DEVICE_GPU)             \
-                              .TypeConstraint<T>("T")         \
-                              .TypeConstraint<int32>("Tperm") \
-                              .HostMemory("perm"),            \
-                          TransposeGpuOp);                    \
-  REGISTER_KERNEL_BUILDER(Name("ConjugateTranspose")          \
-                              .Device(DEVICE_GPU)             \
-                              .TypeConstraint<T>("T")         \
-                              .TypeConstraint<int32>("Tperm") \
-                              .HostMemory("perm"),            \
+#define REGISTER(T)                                   \
+  REGISTER_KERNEL_BUILDER(Name("Transpose")           \
+                              .Device(DEVICE_GPU)     \
+                              .TypeConstraint<T>("T") \
+                              .HostMemory("perm"),    \
+                          TransposeGpuOp);            \
+  REGISTER_KERNEL_BUILDER(Name("ConjugateTranspose")  \
+                              .Device(DEVICE_GPU)     \
+                              .TypeConstraint<T>("T") \
+                              .HostMemory("perm"),    \
                           ConjugateTransposeGpuOp);
 TF_CALL_POD_TYPES(REGISTER);
 #undef REGISTER
@@ -270,18 +282,16 @@ Status ConjugateTransposeSyclOp::DoTranspose(OpKernelContext* ctx,
   return ::tensorflow::DoConjugateTranspose(ctx->eigen_device<SYCLDevice>(), in,
                                             perm, out);
 }
-#define REGISTER(T)                                           \
-  REGISTER_KERNEL_BUILDER(Name("Transpose")                   \
-                              .Device(DEVICE_SYCL)            \
-                              .TypeConstraint<T>("T")         \
-                              .TypeConstraint<int32>("Tperm") \
-                              .HostMemory("perm"),            \
-                          TransposeSyclOp);                   \
-  REGISTER_KERNEL_BUILDER(Name("ConjugateTranspose")          \
-                              .Device(DEVICE_SYCL)            \
-                              .TypeConstraint<T>("T")         \
-                              .TypeConstraint<int32>("Tperm") \
-                              .HostMemory("perm"),            \
+#define REGISTER(T)                                   \
+  REGISTER_KERNEL_BUILDER(Name("Transpose")           \
+                              .Device(DEVICE_SYCL)    \
+                              .TypeConstraint<T>("T") \
+                              .HostMemory("perm"),    \
+                          TransposeSyclOp);           \
+  REGISTER_KERNEL_BUILDER(Name("ConjugateTranspose")  \
+                              .Device(DEVICE_SYCL)    \
+                              .TypeConstraint<T>("T") \
+                              .HostMemory("perm"),    \
                           ConjugateTransposeSyclOp);
 TF_CALL_POD_TYPES(REGISTER);
 #undef REGISTER
diff --git a/tensorflow/core/ops/array_ops.cc b/tensorflow/core/ops/array_ops.cc
index 14b87f0edf..c5935141f8 100644
--- a/tensorflow/core/ops/array_ops.cc
+++ b/tensorflow/core/ops/array_ops.cc
@@ -739,7 +739,7 @@ REGISTER_OP("Diag")
     .Attr("T: {float, double, int32, int64, complex64, complex128}")
     .SetShapeFn([](InferenceContext* c) {
       ShapeHandle in = c->input(0);
-      TF_RETURN_IF_ERROR(c->WithRankAtMost(in, 3, &in));
+      TF_RETURN_IF_ERROR(c->WithRankAtLeast(in, 1, &in));
       // Output shape is original concatenated with itself.
       ShapeHandle out;
       TF_RETURN_IF_ERROR(c->Concatenate(in, in, &out));
@@ -767,7 +767,7 @@ tf.diag(diagonal) ==> [[1, 0, 0, 0]
                        [0, 0, 0, 4]]
 ```
 
-diagonal: Rank k tensor where k is at most 3.
+diagonal: Rank k tensor where k is at most 1.
 )doc");
 
 // --------------------------------------------------------------------------
@@ -783,9 +783,9 @@ REGISTER_OP("DiagPart")
       }
       // Rank must be even, and result will have rank <rank/2>.
       const int32 rank = c->Rank(in);
-      if ((rank % 2) != 0 || rank > 6) {
+      if ((rank % 2) != 0 || rank <= 0) {
         return errors::InvalidArgument(
-            "Input must have even rank <= 6, input rank is ", rank);
+            "Input must have even and non-zero rank, input rank is ", rank);
       }
       const int32 mid = rank / 2;
 
@@ -820,7 +820,7 @@ For example:
 tf.diag_part(input) ==> [1, 2, 3, 4]
 ```
 
-input: Rank k tensor where k is 2, 4, or 6.
+input: Rank k tensor where k is even and not zero.
 diagonal: The extracted diagonal.
 
 )doc");
@@ -1175,7 +1175,7 @@ For example:
 #                  [20, 21, 22, 23]]]]
 # tensor 't' shape is [1, 2, 3, 4]
 
-# 'dims' is [3] or 'dims' is -1
+# 'dims' is [3] or 'dims' is [-1]
 reverse(t, dims) ==> [[[[ 3,  2,  1,  0],
                         [ 7,  6,  5,  4],
                         [ 11, 10, 9, 8]],
@@ -2283,6 +2283,8 @@ size(t) ==> 12
 
 namespace {
 
+// This SliceHelper processes the output shape of the `slice`
+// when the tensor of `sizes` is available.
 template <typename T>
 Status SliceHelper(InferenceContext* c, ShapeHandle begin_value,
                    const Tensor* sizes_value,
@@ -2308,7 +2310,6 @@ Status SliceHelper(InferenceContext* c, ShapeHandle begin_value,
 
   return Status::OK();
 }
-
 }  // namespace
 
 // --------------------------------------------------------------------------
@@ -2339,9 +2340,10 @@ REGISTER_OP("Slice")
       ShapeHandle begin_value;
       TF_RETURN_IF_ERROR(c->MakeShapeFromShapeTensor(1, &begin_value));
 
-      // NOTE(mrry): We can't use `MakeShapeFromShapeTensor` for `sizes` because
-      // it might contain -1, which can't be represented (-1 in the ShapeHandle
-      // would mean "unknown".
+      // We check the tensor value here and will only use
+      // `MakeShapeFromShapeTensor` when `sizes_value` is null.
+      // The reason is that `sizes`might contain -1, which can't
+      // be represented (-1 in the ShapeHandle would mean "unknown".
       const Tensor* sizes_value = c->input_tensor(2);
 
       if (sizes_value != nullptr) {
@@ -2361,6 +2363,28 @@ REGISTER_OP("Slice")
         c->set_output(0, c->MakeShape(dims));
         return Status::OK();
       } else {
+        // In case `sizes` is not available (`sizes_value` is null),
+        // we could try to use `MakeShapeFromShapeTensor` here.
+        // If sizes contain -1, we will simply consider it as `Unknown`.
+        // This is less than ideal but still an improvement of shape inference.
+        // The following is an example that returns [None, 1, None] with this
+        // code path:
+        //   z = tf.zeros((1, 2, 3))
+        //   m = tf.slice(z, [0, 0, 0], [tf.constant(1) + 0, 1, -1])
+        //   m.get_shape().as_list()
+        ShapeHandle sizes_value;
+        TF_RETURN_IF_ERROR(c->MakeShapeFromShapeTensor(2, &sizes_value));
+        if (c->RankKnown(sizes_value)) {
+          TF_RETURN_IF_ERROR(
+              c->WithRank(begin_value, c->Rank(sizes_value), &begin_value));
+          std::vector<DimensionHandle> dims;
+          for (int i = 0; i < c->Rank(sizes_value); ++i) {
+            dims.emplace_back(c->Dim(sizes_value, i));
+          }
+          c->set_output(0, c->MakeShape(dims));
+          return Status::OK();
+        }
+
         // We might know the rank of the input.
         if (c->RankKnown(input)) {
           c->set_output(0, c->UnknownShapeOfRank(c->Rank(input)));
diff --git a/tensorflow/core/ops/array_ops_test.cc b/tensorflow/core/ops/array_ops_test.cc
index a5d7a32e05..94eb120175 100644
--- a/tensorflow/core/ops/array_ops_test.cc
+++ b/tensorflow/core/ops/array_ops_test.cc
@@ -186,21 +186,20 @@ TEST(ArrayOpsTest, Identity_ShapeFnHandles) {
 TEST(ArrayOpsTest, Diag_ShapeFn) {
   ShapeInferenceTestOp op("Diag");
   INFER_OK(op, "?", "?");
-  INFER_OK(op, "[]", "[]");
   INFER_OK(op, "[1,?,3]", "[d0_0,d0_1,d0_2,d0_0,d0_1,d0_2]");
-  INFER_ERROR("Shape must be at most rank 3 but is rank 4", op, "[?,1,2,3]");
+  INFER_OK(op, "[?,1,2,3]", "[d0_0,d0_1,d0_2,d0_3,d0_0,d0_1,d0_2,d0_3]");
+  INFER_ERROR("Shape must be at least rank 1 but is rank 0", op, "[]");
 }
 
 TEST(ArrayOpsTest, DiagPart_ShapeFn) {
   ShapeInferenceTestOp op("DiagPart");
   INFER_OK(op, "?", "?");
-  INFER_OK(op, "[]", "[]");
   INFER_OK(op, "[1,?,?,4]", "[d0_0,d0_3]");
   INFER_OK(op, "[1,?,3,?,4,3]", "[d0_0,d0_4,d0_2|d0_5]");
-  INFER_ERROR("Input must have even rank <= 6, input rank is 1", op, "[?]");
-  INFER_ERROR("Input must have even rank <= 6, input rank is 3", op, "[1,2,3]");
-  INFER_ERROR("Input must have even rank <= 6, input rank is 8", op,
-              "[1,2,3,?,?,?,?,?]");
+  INFER_OK(op, "[1,2,3,?,?,?,?,4]", "[d0_0,d0_1,d0_2,d0_7]");
+  INFER_ERROR("Input must have even and non-zero rank", op, "[]");
+  INFER_ERROR("Input must have even and non-zero rank", op, "[?]");
+  INFER_ERROR("Input must have even and non-zero rank", op, "[1,2,3]");
   INFER_ERROR("Dimensions must be equal, but are 2 and 10", op, "[1,2,?,10]");
 }
 
diff --git a/tensorflow/core/ops/image_ops.cc b/tensorflow/core/ops/image_ops.cc
index a44bac60bf..e9bf29d172 100644
--- a/tensorflow/core/ops/image_ops.cc
+++ b/tensorflow/core/ops/image_ops.cc
@@ -151,7 +151,7 @@ REGISTER_OP("ResizeArea")
     .Input("images: T")
     .Input("size: int32")
     .Output("resized_images: float")
-    .Attr("T: {uint8, int8, int16, int32, int64, half, float, double}")
+    .Attr("T: {int8, uint8, int16, uint16, int32, int64, half, float, double}")
     .Attr("align_corners: bool = false")
     .SetShapeFn(ResizeShapeFn)
     .Doc(R"doc(
@@ -179,7 +179,7 @@ REGISTER_OP("ResizeBicubic")
     .Input("images: T")
     .Input("size: int32")
     .Output("resized_images: float")
-    .Attr("T: {uint8, int8, int16, int32, int64, half, float, double}")
+    .Attr("T: {int8, uint8, int16, uint16, int32, int64, half, float, double}")
     .Attr("align_corners: bool = false")
     .SetShapeFn(ResizeShapeFn)
     .Doc(R"doc(
@@ -227,7 +227,7 @@ REGISTER_OP("ResizeBilinear")
     .Input("images: T")
     .Input("size: int32")
     .Output("resized_images: float")
-    .Attr("T: {uint8, int8, int16, int32, int64, half, float, double}")
+    .Attr("T: {int8, uint8, int16, uint16, int32, int64, half, float, double}")
     .Attr("align_corners: bool = false")
     .SetShapeFn(ResizeShapeFn)
     .Doc(R"doc(
@@ -311,7 +311,7 @@ REGISTER_OP("ResizeNearestNeighbor")
     .Input("images: T")
     .Input("size: int32")
     .Output("resized_images: T")
-    .Attr("T: {uint8, int8, int16, int32, int64, half, float, double}")
+    .Attr("T: {int8, uint8, int16, uint16, int32, int64, half, float, double}")
     .Attr("align_corners: bool = false")
     .SetShapeFn(ResizeShapeFn)
     .Doc(R"doc(
@@ -453,7 +453,36 @@ REGISTER_OP("DecodeAndCropJpeg")
     .Attr("acceptable_fraction: float = 1.0")
     .Attr("dct_method: string = ''")
     .Output("image: uint8")
-    .SetShapeFn(DecodeImageShapeFn)
+    .SetShapeFn([](InferenceContext* c) {
+      ShapeHandle unused;
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 0, &unused));
+      DimensionHandle channels_dim = c->UnknownDim();
+      DimensionHandle h = c->UnknownDim();
+      DimensionHandle w = c->UnknownDim();
+
+      int32 channels;
+      TF_RETURN_IF_ERROR(c->GetAttr("channels", &channels));
+      if (channels != 0) {
+        if (channels < 0) {
+          return errors::InvalidArgument("channels must be non-negative, got ",
+                                         channels);
+        }
+        channels_dim = c->MakeDim(channels);
+      }
+
+      DimensionHandle unused_dim;
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 1, &unused));
+      TF_RETURN_IF_ERROR(c->WithValue(c->Dim(unused, 0), 4, &unused_dim));
+
+      const Tensor* crop_window = c->input_tensor(1);
+      if (crop_window != nullptr) {
+        auto crop_window_vec = crop_window->vec<int32>();
+        h = c->MakeDim(crop_window_vec(2));
+        w = c->MakeDim(crop_window_vec(3));
+      }
+      c->set_output(0, c->MakeShape({h, w, channels_dim}));
+      return Status::OK();
+    })
     .Doc(strings::StrCat(R"doc(
 Decode and Crop a JPEG-encoded image to a uint8 tensor.
 )doc",
@@ -1068,7 +1097,7 @@ REGISTER_OP("CropAndResize")
     .Input("box_ind: int32")
     .Input("crop_size: int32")
     .Output("crops: float")
-    .Attr("T: {uint8, int8, int16, int32, int64, half, float, double}")
+    .Attr("T: {uint8, uint16, int8, int16, int32, int64, half, float, double}")
     .Attr("method: {'bilinear'} = 'bilinear'")
     .Attr("extrapolation_value: float = 0")
     .SetShapeFn([](InferenceContext* c) {
@@ -1175,7 +1204,7 @@ REGISTER_OP("CropAndResizeGradBoxes")
     .Input("boxes: float")
     .Input("box_ind: int32")
     .Output("output: float")
-    .Attr("T: {uint8, int8, int16, int32, int64, half, float, double}")
+    .Attr("T: {uint8, uint16, int8, int16, int32, int64, half, float, double}")
     .Attr("method: {'bilinear'} = 'bilinear'")
     .SetShapeFn([](InferenceContext* c) {
       c->set_output(0, c->input(2));
diff --git a/tensorflow/core/ops/image_ops_test.cc b/tensorflow/core/ops/image_ops_test.cc
index c34b11a15e..5f0b391b0d 100644
--- a/tensorflow/core/ops/image_ops_test.cc
+++ b/tensorflow/core/ops/image_ops_test.cc
@@ -105,7 +105,7 @@ TEST(ImageOpsTest, DecodeAndCropJpeg_ShapeFn) {
                    .Input({"img", 0, DT_STRING})
                    .Input({"crop_window", 1, DT_INT32})
                    .Finalize(&op.node_def));
-  INFER_OK(op, "[];[]", "[?,?,?]");
+  INFER_OK(op, "[];[?]", "[?,?,?]");
 
   // Set the channel, so that part of output shape is known.
   TF_ASSERT_OK(NodeDefBuilder("test", op_name)
@@ -113,7 +113,7 @@ TEST(ImageOpsTest, DecodeAndCropJpeg_ShapeFn) {
                    .Input({"crop_window", 1, DT_INT32})
                    .Attr("channels", 4)
                    .Finalize(&op.node_def));
-  INFER_OK(op, "[];[]", "[?,?,4]");
+  INFER_OK(op, "[];[?]", "[?,?,4]");
 
   // Negative channel value is rejected.
   TF_ASSERT_OK(NodeDefBuilder("test", op_name)
@@ -139,7 +139,7 @@ TEST(ImageOpsTest, DecodeAndCropJpeg_InvalidCropWindow) {
                    .Input({"img", 0, DT_STRING})
                    .Input({"crop_window", 1, DT_INT32})
                    .Finalize(&op.node_def));
-  INFER_OK(op, "[];[]", "[?,?,?]");
+  INFER_OK(op, "[];[?]", "[?,?,?]");
 }
 
 TEST(ImageOpsTest, EncodeImage_ShapeFn) {
diff --git a/tensorflow/core/ops/math_ops.cc b/tensorflow/core/ops/math_ops.cc
index ab0bc258f7..61db896c51 100644
--- a/tensorflow/core/ops/math_ops.cc
+++ b/tensorflow/core/ops/math_ops.cc
@@ -49,6 +49,38 @@ inputs: Must all be the same size and shape.
 
 // --------------------------------------------------------------------------
 
+// Note that the following operator is just a placeholder and has no
+// associated kernel. The code in accumulate_n_optimizer.cc replaces
+// this placeholder with a graph of operators that do have kernels.
+// The Python code that generates instances of this op is currently in
+// contrib/framework/python/ops/accumulate_n_v2.py
+REGISTER_OP("AccumulateNV2")
+    .Input("inputs: N * T")
+    .Output("sum: T")
+    .Attr("N: int >= 1")
+    .Attr("T: numbertype")
+    .Attr("shape: shape")
+    .SetIsCommutative()
+    .SetIsAggregate()
+    .SetShapeFn(shape_inference::ExplicitShape)
+    .Doc(R"doc(
+Returns the element-wise sum of a list of tensors.
+
+`tf.accumulate_n_v2` performs the same operation as `tf.add_n`, but does not
+wait for all of its inputs to be ready before beginning to sum. This can
+save memory if inputs are ready at different times, since minimum temporary
+storage is proportional to the output size rather than the inputs size.
+
+Unlike the original `accumulate_n`, `accumulate_n_v2` is differentiable.
+
+Returns a `Tensor` of same shape and type as the elements of `inputs`.
+
+inputs: A list of `Tensor` objects, each with same shape and type.
+shape: Shape of elements of `inputs`.
+)doc");
+
+// --------------------------------------------------------------------------
+
 REGISTER_OP("BatchMatMul")
     .Input("x: T")
     .Input("y: T")
@@ -591,7 +623,7 @@ REGISTER_OP("TruncateDiv")
 Returns x / y element-wise for integer types.
 
 Truncation designates that negative numbers will round fractional quantities
-toward zero. I.e. -7 / 5 = 1. This matches C semantics but it is different
+toward zero. I.e. -7 / 5 = -1. This matches C semantics but it is different
 than Python semantics. See `FloorDiv` for a division function that matches
 Python Semantics.
 
@@ -2218,6 +2250,44 @@ product: Pairwise cross product of the vectors in `a` and `b`.
 
 // --------------------------------------------------------------------------
 
+REGISTER_OP("HistogramFixedWidth")
+    .Input("values: T")
+    .Input("value_range: T")
+    .Input("nbins: int32")
+    .Output("out: dtype")
+    .Attr("T: {int32, int64, float32, float64}")
+    .Attr("dtype: {int32, int64} = DT_INT32")
+    .SetShapeFn([](InferenceContext* c) {
+      c->set_output(0, c->UnknownShapeOfRank(1));
+      return Status::OK();
+    })
+    .Doc(R"doc(
+Return histogram of values.
+
+Given the tensor `values`, this operation returns a rank 1 histogram counting
+the number of entries in `values` that fall into every bin.  The bins are
+equal width and determined by the arguments `value_range` and `nbins`.
+
+```python
+# Bins will be:  (-inf, 1), [1, 2), [2, 3), [3, 4), [4, inf)
+nbins = 5
+value_range = [0.0, 5.0]
+new_values = [-1.0, 0.0, 1.5, 2.0, 5.0, 15]
+
+with tf.get_default_session() as sess:
+  hist = tf.histogram_fixed_width(new_values, value_range, nbins=5)
+  variables.global_variables_initializer().run()
+  sess.run(hist) => [2, 1, 1, 0, 2]
+```
+
+values:  Numeric `Tensor`.
+value_range:  Shape [2] `Tensor` of same `dtype` as `values`.
+  values <= value_range[0] will be mapped to hist[0],
+  values >= value_range[1] will be mapped to hist[-1].
+nbins:  Scalar `int32 Tensor`.  Number of histogram bins.
+out: A 1-D `Tensor` holding histogram of values.
+)doc");
+
 REGISTER_OP("Bincount")
     .Input("arr: int32")
     .Input("size: int32")
diff --git a/tensorflow/core/ops/nn_ops.cc b/tensorflow/core/ops/nn_ops.cc
index 5efa55b496..1d26660a4b 100644
--- a/tensorflow/core/ops/nn_ops.cc
+++ b/tensorflow/core/ops/nn_ops.cc
@@ -2260,6 +2260,56 @@ indices: The indices of `values` within the last dimension of `input`.
 
 // --------------------------------------------------------------------------
 
+REGISTER_OP("NthElement")
+    .Input("input: T")
+    .Input("n: int32")
+    .Output("values: T")
+    .Attr("reverse: bool = false")
+    .Attr("T: realnumbertype")
+    .SetShapeFn([](InferenceContext* c) {
+      ShapeHandle input;
+      TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(0), 1, &input));
+
+      // Get the n value from input tensor, and make sure which is a scalar.
+      DimensionHandle n_dim;
+      TF_RETURN_IF_ERROR(c->MakeDimForScalarInput(1, &n_dim));
+
+      // The last dimension of input tensor must be greater than N.
+      DimensionHandle last_dim = c->Dim(input, -1);
+      if (c->ValueKnown(last_dim) && c->ValueKnown(n_dim) &&
+          c->Value(last_dim) <= c->Value(n_dim)) {
+        return errors::InvalidArgument(
+            "Input must have last dimension > n = ", c->Value(n_dim), " but is ",
+            c->Value(last_dim));
+      }
+
+      // Reduce last_dim for output tensor
+      ShapeHandle s;
+      TF_RETURN_IF_ERROR(c->Subshape(input, 0, -1, &s));
+      c->set_output(0, s);
+      return Status::OK();
+    })
+    .Doc(R"doc(
+Finds values of the `n`-th order statistic for the last dmension.
+
+If the input is a vector (rank-1), finds the entries which is the nth-smallest
+value in the vector and outputs their values as scalar tensor.
+
+For matrices (resp. higher rank input), computes the entries which is the
+nth-smallest value in each row (resp. vector along the last dimension). Thus,
+
+    values.shape = input.shape[:-1]
+
+input: 1-D or higher with last dimension at least `n+1`.
+n: 0-D. Position of sorted vector to select along the last dimension (along
+  each row for matrices). Valid range of n is `[0, input.shape[:-1])`
+reverse: When set to True, find the nth-largest value in the vector and vice
+  versa.
+values: The `n`-th order statistic along each last dimensional slice.
+)doc");
+
+// --------------------------------------------------------------------------
+
 REGISTER_OP("FractionalMaxPool")
     .Input("value: T")
     .Output("output: T")
diff --git a/tensorflow/core/ops/nn_ops_test.cc b/tensorflow/core/ops/nn_ops_test.cc
index 4628b725f8..94ecf4d5db 100644
--- a/tensorflow/core/ops/nn_ops_test.cc
+++ b/tensorflow/core/ops/nn_ops_test.cc
@@ -81,6 +81,30 @@ TEST(NNOpsTest, TopKV2_ShapeFn) {
       op, "[1,2,3,4];[]");
 }
 
+TEST(NNOpsTest, NthElement_ShapeFn) {
+  ShapeInferenceTestOp op("NthElement");
+  op.input_tensors.resize(2);
+
+  Tensor n_t;
+  op.input_tensors[1] = &n_t;
+  n_t = test::AsScalar<int32>(20);
+
+  INFER_OK(op, "?;[]", "?");
+  INFER_OK(op, "[21];[]", "[]");
+  INFER_OK(op, "[2,?,?];[]", "[d0_0,d0_1]");
+  INFER_OK(op, "[?,3,?,21];[]", "[d0_0,d0_1,d0_2]");
+
+  INFER_ERROR("Shape must be at least rank 1 but is rank 0", op, "[];[]");
+  INFER_ERROR("Input must have last dimension > n = 20 but is 1", op,
+              "[1];[]");
+  INFER_ERROR("Input must have last dimension > n = 20 but is 20", op,
+              "[1,2,3,20];[]");
+  n_t = test::AsScalar<int32>(-1);
+  INFER_ERROR(
+     "Dimension size, given by scalar input 1, must be non-negative but is -1",
+     op, "[1,2,3,4];[]");
+}
+
 TEST(NNOpsTest, BatchNormWithGlobalNormalization_ShapeFn) {
   ShapeInferenceTestOp op("BatchNormWithGlobalNormalization");
 
diff --git a/tensorflow/core/platform/s3/s3_crypto.cc b/tensorflow/core/platform/s3/s3_crypto.cc
index 14bbed19a5..d7062a59d2 100644
--- a/tensorflow/core/platform/s3/s3_crypto.cc
+++ b/tensorflow/core/platform/s3/s3_crypto.cc
@@ -71,7 +71,7 @@ class S3Sha256OpenSSLImpl : public Aws::Utils::Crypto::Hash {
     SHA256_Init(&sha256);
 
     auto currentPos = stream.tellg();
-    if (currentPos == -1) {
+    if (currentPos == std::streampos(std::streamoff(-1))) {
       currentPos = 0;
       stream.clear();
     }
diff --git a/tensorflow/core/profiler/README.md b/tensorflow/core/profiler/README.md
index 92bce9c1ce..8ca26fa5dc 100644
--- a/tensorflow/core/profiler/README.md
+++ b/tensorflow/core/profiler/README.md
@@ -48,7 +48,7 @@ bazel-bin/tensorflow/python/profiler/profiler_ui \
 # Create options to profile the time and memory information.
 builder = tf.profiler.ProfileOptionBuilder
 opts = builder(builder.time_and_memory()).order_by('micros').build()
-# Create a profiling context, set contructor argument `trace_steps`, 
+# Create a profiling context, set constructor argument `trace_steps`, 
 # `dump_steps` to empty for explicit control.
 with tf.contrib.tfprof.ProfileContext('/tmp/train_dir',
                                       trace_steps=[],
diff --git a/tensorflow/core/profiler/g3doc/options.md b/tensorflow/core/profiler/g3doc/options.md
index ddee63ad42..4c73e372e3 100644
--- a/tensorflow/core/profiler/g3doc/options.md
+++ b/tensorflow/core/profiler/g3doc/options.md
@@ -43,7 +43,7 @@ In graph view, in means the number of hops in the <b>graph</b>.
 
 ### Times
 
-Most machines have mutli-core CPUs. Some installs one or more accelerators.
+Most machines have multi-core CPUs. Some installs one or more accelerators.
 Each accelerator usually performs massive parallel processing. The profiler
 tracks the accumulated processing times. Hence, the accumulated processing
 time is likely larger than the time of each step.
diff --git a/tensorflow/core/public/version.h b/tensorflow/core/public/version.h
index ccb861c93a..5d2298f7b7 100644
--- a/tensorflow/core/public/version.h
+++ b/tensorflow/core/public/version.h
@@ -19,12 +19,12 @@ limitations under the License.
 // TensorFlow uses semantic versioning, see http://semver.org/.
 
 #define TF_MAJOR_VERSION 1
-#define TF_MINOR_VERSION 3
+#define TF_MINOR_VERSION 4
 #define TF_PATCH_VERSION 0
 
 // TF_VERSION_SUFFIX is non-empty for pre-releases (e.g. "-alpha", "-alpha.1",
 // "-beta", "-rc", "-rc.1")
-#define TF_VERSION_SUFFIX ""
+#define TF_VERSION_SUFFIX "-rc0"
 
 #define TF_STR_HELPER(x) #x
 #define TF_STR(x) TF_STR_HELPER(x)
diff --git a/tensorflow/docs_src/api_guides/python/reading_data.md b/tensorflow/docs_src/api_guides/python/reading_data.md
index 7609ca91d0..b3ebaa0f0a 100644
--- a/tensorflow/docs_src/api_guides/python/reading_data.md
+++ b/tensorflow/docs_src/api_guides/python/reading_data.md
@@ -67,7 +67,7 @@ A typical queue-based pipeline for reading records from files has the following
 8.  Example queue
 
 Warning: This section discusses implementing input pipelines using the
-queue-based APIs which can be cleanly replaced by the @{$datasets$Dataset API}.
+queue-based APIs which can be cleanly replaced by the @{$datasets$Datasets API}.
 
 ### Filenames, shuffling, and epoch limits
 
diff --git a/tensorflow/docs_src/get_started/estimator.md b/tensorflow/docs_src/get_started/estimator.md
index ab270d1408..790de6679b 100644
--- a/tensorflow/docs_src/get_started/estimator.md
+++ b/tensorflow/docs_src/get_started/estimator.md
@@ -28,7 +28,7 @@ from __future__ import division
 from __future__ import print_function
 
 import os
-import urllib
+from six.moves.urllib.request import urlopen
 
 import numpy as np
 import tensorflow as tf
@@ -44,13 +44,13 @@ IRIS_TEST_URL = "http://download.tensorflow.org/data/iris_test.csv"
 def main():
   # If the training and test sets aren't stored locally, download them.
   if not os.path.exists(IRIS_TRAINING):
-    raw = urllib.urlopen(IRIS_TRAINING_URL).read()
-    with open(IRIS_TRAINING, "w") as f:
+    raw = urlopen(IRIS_TRAINING_URL).read()
+    with open(IRIS_TRAINING, "wb") as f:
       f.write(raw)
 
   if not os.path.exists(IRIS_TEST):
-    raw = urllib.urlopen(IRIS_TEST_URL).read()
-    with open(IRIS_TEST, "w") as f:
+    raw = urlopen(IRIS_TEST_URL).read()
+    with open(IRIS_TEST, "wb") as f:
       f.write(raw)
 
   # Load datasets.
@@ -167,7 +167,7 @@ from __future__ import division
 from __future__ import print_function
 
 import os
-import urllib
+from six.moves.urllib.request import urlopen
 
 import tensorflow as tf
 import numpy as np
@@ -184,13 +184,13 @@ them.
 
 ```python
 if not os.path.exists(IRIS_TRAINING):
-  raw = urllib.urlopen(IRIS_TRAINING_URL).read()
-  with open(IRIS_TRAINING,'w') as f:
+  raw = urlopen(IRIS_TRAINING_URL).read()
+  with open(IRIS_TRAINING,'wb') as f:
     f.write(raw)
 
 if not os.path.exists(IRIS_TEST):
-  raw = urllib.urlopen(IRIS_TEST_URL).read()
-  with open(IRIS_TEST,'w') as f:
+  raw = urlopen(IRIS_TEST_URL).read()
+  with open(IRIS_TEST,'wb') as f:
     f.write(raw)
 ```
 
diff --git a/tensorflow/docs_src/install/install_c.md b/tensorflow/docs_src/install/install_c.md
index 7ebf5c4a2c..586bb6dead 100644
--- a/tensorflow/docs_src/install/install_c.md
+++ b/tensorflow/docs_src/install/install_c.md
@@ -35,7 +35,7 @@ enable TensorFlow for C:
          OS="linux" # Change to "darwin" for Mac OS
          TARGET_DIRECTORY="/usr/local"
          curl -L \
-           "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-${OS}-x86_64-1.3.0.tar.gz" |
+           "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-${OS}-x86_64-1.4.0-rc0.tar.gz" |
            sudo tar -C $TARGET_DIRECTORY -xz
 
      The `tar` command extracts the TensorFlow C library into the `lib`
diff --git a/tensorflow/docs_src/install/install_go.md b/tensorflow/docs_src/install/install_go.md
index b991fd0f93..1d00661d83 100644
--- a/tensorflow/docs_src/install/install_go.md
+++ b/tensorflow/docs_src/install/install_go.md
@@ -35,7 +35,7 @@ steps to install this library and enable TensorFlow for Go:
          TF_TYPE="cpu" # Change to "gpu" for GPU support
          TARGET_DIRECTORY='/usr/local'
          curl -L \
-           "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-$(go env GOOS)-x86_64-1.3.0.tar.gz" |
+           "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-$(go env GOOS)-x86_64-1.4.0-rc0.tar.gz" |
          sudo tar -C $TARGET_DIRECTORY -xz
 
      The `tar` command extracts the TensorFlow C library into the `lib`
diff --git a/tensorflow/docs_src/install/install_java.md b/tensorflow/docs_src/install/install_java.md
index 2adcd4da73..3b3acfdcb3 100644
--- a/tensorflow/docs_src/install/install_java.md
+++ b/tensorflow/docs_src/install/install_java.md
@@ -34,7 +34,7 @@ following to the project's `pom.xml` to use the TensorFlow Java APIs:
 <dependency>
   <groupId>org.tensorflow</groupId>
   <artifactId>tensorflow</artifactId>
-  <version>1.3.0</version>
+  <version>1.4.0-rc0</version>
 </dependency>
 ```
 
@@ -63,7 +63,7 @@ As an example, these steps will create a Maven project that uses TensorFlow:
                <dependency>
                  <groupId>org.tensorflow</groupId>
                  <artifactId>tensorflow</artifactId>
-                 <version>1.3.0</version>
+                 <version>1.4.0-rc0</version>
                </dependency>
              </dependencies>
          </project>
@@ -122,7 +122,7 @@ refer to the simpler instructions above instead.
 Take the following steps to install TensorFlow for Java on Linux or Mac OS:
 
   1. Download
-     [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.3.0.jar),
+     [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.4.0-rc0.jar),
      which is the TensorFlow Java Archive (JAR).
 
   2. Decide whether you will run TensorFlow for Java on CPU(s) only or with
@@ -141,7 +141,7 @@ Take the following steps to install TensorFlow for Java on Linux or Mac OS:
          OS=$(uname -s | tr '[:upper:]' '[:lower:]')
          mkdir -p ./jni
          curl -L \
-           "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-${TF_TYPE}-${OS}-x86_64-1.3.0.tar.gz" |
+           "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-${TF_TYPE}-${OS}-x86_64-1.4.0-rc0.tar.gz" |
            tar -xz -C ./jni
 
 ### Install on Windows
@@ -149,10 +149,10 @@ Take the following steps to install TensorFlow for Java on Linux or Mac OS:
 Take the following steps to install TensorFlow for Java on Windows:
 
   1. Download
-     [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.3.0.jar),
+     [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.4.0-rc0.jar),
      which is the TensorFlow Java Archive (JAR).
   2. Download the following Java Native Interface (JNI) file appropriate for
-     [TensorFlow for Java on Windows](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-cpu-windows-x86_64-1.3.0.zip).
+     [TensorFlow for Java on Windows](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-cpu-windows-x86_64-1.4.0-rc0.zip).
   3. Extract this .zip file.
 
 
@@ -200,7 +200,7 @@ must be part of your `classpath`. For example, you can include the
 downloaded `.jar` in your `classpath` by using the `-cp` compilation flag
 as follows:
 
-<pre><b>javac -cp libtensorflow-1.3.0.jar HelloTF.java</b></pre>
+<pre><b>javac -cp libtensorflow-1.4.0-rc0.jar HelloTF.java</b></pre>
 
 
 ### Running
@@ -214,11 +214,11 @@ two files are available to the JVM:
 For example, the following command line executes the `HelloTF` program on Linux
 and Mac OS X:
 
-<pre><b>java -cp libtensorflow-1.3.0.jar:. -Djava.library.path=./jni HelloTF</b></pre>
+<pre><b>java -cp libtensorflow-1.4.0-rc0.jar:. -Djava.library.path=./jni HelloTF</b></pre>
 
 And the following command line executes the `HelloTF` program on Windows:
 
-<pre><b>java -cp libtensorflow-1.3.0.jar;. -Djava.library.path=jni HelloTF</b></pre>
+<pre><b>java -cp libtensorflow-1.4.0-rc0.jar;. -Djava.library.path=jni HelloTF</b></pre>
 
 If the program prints <tt>Hello from <i>version</i></tt>, you've successfully
 installed TensorFlow for Java and are ready to use the API.  If the program
diff --git a/tensorflow/docs_src/install/install_linux.md b/tensorflow/docs_src/install/install_linux.md
index 576099f054..9d204cc246 100644
--- a/tensorflow/docs_src/install/install_linux.md
+++ b/tensorflow/docs_src/install/install_linux.md
@@ -42,8 +42,20 @@ must be installed on your system:
     a list of supported GPU cards.
   * The libcupti-dev library, which is the NVIDIA CUDA Profile Tools Interface.
     This library provides advanced profiling support. To install this library,
-    issue the following command:
+    issue the following command for CUDA Toolkit >= 8.0:
 
+    <pre>
+    $ <b>sudo apt-get install cuda-command-line-tools</b>
+    </pre>
+    
+    and add its path to your `LD_LIBRARY_PATH` environment variable:
+
+    <pre> 
+    $ <b>export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda/extras/CUPTI/lib64</b> 
+    </pre>
+
+    For CUDA Toolkit <= 7.5 do:
+    
     <pre>
     $ <b>sudo apt-get install libcupti-dev</b>
     </pre>
@@ -172,7 +184,7 @@ Take the following steps to install TensorFlow with Virtualenv:
      virtualenv environment:
 
      <pre>(tensorflow)$ <b>pip3 install --upgrade \
-     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0dev-cp34-cp34m-linux_x86_64.whl</b></pre>
+     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0rc0-cp34-cp34m-linux_x86_64.whl</b></pre>
 
 If you encounter installation problems, see
 [Common Installation Problems](#common_installation_problems).
@@ -277,7 +289,7 @@ take the following steps:
 
      <pre>
      $ <b>sudo pip3 install --upgrade \
-     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0dev-cp34-cp34m-linux_x86_64.whl</b>
+     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0rc0-cp34-cp34m-linux_x86_64.whl</b>
      </pre>
 
      If this step fails, see
@@ -445,7 +457,7 @@ Take the following steps to install TensorFlow in an Anaconda environment:
   2. Create a conda environment named <tt>tensorflow</tt> to run a version
      of Python by invoking the following command:
 
-     <pre>$ <b>conda create -n tensorflow python=2.7 # or python=3.3, etc.</b></pre>
+     <pre>$ <b>conda create -n tensorflow pip python=2.7 # or python=3.3, etc.</b></pre>
 
   3. Activate the conda environment by issuing the following command:
 
@@ -464,7 +476,7 @@ Take the following steps to install TensorFlow in an Anaconda environment:
 
      <pre>
      (tensorflow)$ <b>pip install --ignore-installed --upgrade \
-     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0dev-cp34-cp34m-linux_x86_64.whl</b></pre>
+     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0rc0-cp34-cp34m-linux_x86_64.whl</b></pre>
 
 
 <a name="ValidateYourInstallation"></a>
@@ -632,14 +644,14 @@ This section documents the relevant values for Linux installations.
 CPU only:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0dev-cp27-none-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0rc0-cp27-none-linux_x86_64.whl
 </pre>
 
 
 GPU support:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.4.0dev-cp27-none-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.4.0rc0-cp27-none-linux_x86_64.whl
 </pre>
 
 Note that GPU support requires the NVIDIA hardware and software described in
@@ -651,14 +663,14 @@ Note that GPU support requires the NVIDIA hardware and software described in
 CPU only:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0dev-cp34-cp34m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0rc0-cp34-cp34m-linux_x86_64.whl
 </pre>
 
 
 GPU support:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.4.0dev-cp34-cp34m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.4.0rc0-cp34-cp34m-linux_x86_64.whl
 </pre>
 
 Note that GPU support requires the NVIDIA hardware and software described in
@@ -670,14 +682,14 @@ Note that GPU support requires the NVIDIA hardware and software described in
 CPU only:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0dev-cp35-cp35m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0rc0-cp35-cp35m-linux_x86_64.whl
 </pre>
 
 
 GPU support:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.4.0dev-cp35-cp35m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.4.0rc0-cp35-cp35m-linux_x86_64.whl
 </pre>
 
 
@@ -689,14 +701,14 @@ Note that GPU support requires the NVIDIA hardware and software described in
 CPU only:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0dev-cp36-cp36m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0rc0-cp36-cp36m-linux_x86_64.whl
 </pre>
 
 
 GPU support:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.4.0dev-cp36-cp36m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.4.0rc0-cp36-cp36m-linux_x86_64.whl
 </pre>
 
 
diff --git a/tensorflow/docs_src/install/install_mac.md b/tensorflow/docs_src/install/install_mac.md
index b6daeb0dd6..6da22784bf 100644
--- a/tensorflow/docs_src/install/install_mac.md
+++ b/tensorflow/docs_src/install/install_mac.md
@@ -109,7 +109,7 @@ Take the following steps to install TensorFlow with Virtualenv:
      TensorFlow in the active Virtualenv is as follows:
 
      <pre> $ <b>pip3 install --upgrade \
-     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.4.0dev-py2-none-any.whl</b></pre>
+     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.4.0rc0-py2-none-any.whl</b></pre>
 
 If you encounter installation problems, see
 [Common Installation Problems](#common-installation-problems).
@@ -230,7 +230,7 @@ take the following steps:
      issue the following command:
 
      <pre> $ <b>sudo pip3 install --upgrade \
-     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.4.0dev-py2-none-any.whl</b> </pre>
+     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.4.0rc0-py2-none-any.whl</b> </pre>
 
      If the preceding command fails, see
      [installation problems](#common-installation-problems).
@@ -321,7 +321,7 @@ Take the following steps to install TensorFlow in an Anaconda environment:
   2. Create a conda environment named `tensorflow`
      by invoking the following command:
 
-     <pre>$ <b>conda create -n tensorflow python=2.7 # or python=3.3, etc.</b></pre>
+     <pre>$ <b>conda create -n tensorflow pip python=2.7 # or python=3.3, etc.</b></pre>
 
   3. Activate the conda environment by issuing the following command:
 
@@ -339,7 +339,7 @@ Take the following steps to install TensorFlow in an Anaconda environment:
      TensorFlow for Python 2.7:
 
      <pre> (tensorflow)$ <b>pip install --ignore-installed --upgrade \
-     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.4.0dev-py2-none-any.whl</b></pre>
+     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.4.0rc0-py2-none-any.whl</b></pre>
 
 
 <a name="ValidateYourInstallation"></a>
@@ -512,7 +512,7 @@ This section documents the relevant values for Mac OS installations.
 
 
 <pre>
-https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.4.0dev-py2-none-any.whl
+https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.4.0rc0-py2-none-any.whl
 </pre>
 
 
@@ -520,7 +520,7 @@ https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.4.0dev-py2-none-a
 
 
 <pre>
-https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.4.0dev-py3-none-any.whl
+https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.4.0rc0-py3-none-any.whl
 </pre>
 
 
diff --git a/tensorflow/docs_src/install/install_sources.md b/tensorflow/docs_src/install/install_sources.md
index e6a4088656..b853d87816 100644
--- a/tensorflow/docs_src/install/install_sources.md
+++ b/tensorflow/docs_src/install/install_sources.md
@@ -137,8 +137,15 @@ The following NVIDIA <i>software</i> must be installed on your system:
     particularly the description of appending the appropriate pathname
     to your `LD_LIBRARY_PATH` environment variable.
 
-Finally, you must also install `libcupti-dev` by invoking the following
-command:
+Finally, you must also install `libcupti` which for Cuda Toolkit >= 8.0 you do via 
+
+<pre> $ <b>sudo apt-get install cuda-command-line-tools</b> </pre>
+
+and add its path to your `LD_LIBRARY_PATH` environment variable:
+
+<pre> $ <b>export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda/extras/CUPTI/lib64</b> </pre>
+
+For Cuda Toolkit <= 7.5, you install `libcupti-dev` by invoking the following command:
 
 <pre> $ <b>sudo apt-get install libcupti-dev</b> </pre>
 
@@ -342,10 +349,10 @@ Invoke `pip install` to install that pip package.
 The filename of the `.whl` file depends on your platform.
 For example, the following command will install the pip package
 
-for TensorFlow 1.4.0dev on Linux:
+for TensorFlow 1.4.0rc0 on Linux:
 
 <pre>
-$ <b>sudo pip install /tmp/tensorflow_pkg/tensorflow-1.4.0dev-py2-none-any.whl</b>
+$ <b>sudo pip install /tmp/tensorflow_pkg/tensorflow-1.4.0rc0-py2-none-any.whl</b>
 </pre>
 
 ## Validate your installation
@@ -434,8 +441,8 @@ Stack Overflow and specify the `tensorflow` tag.
 **Linux**
 <table>
 <tr><th>Version:</th><th>CPU/GPU:</th><th>Python Version:</th><th>Compiler:</th><th>Build Tools:</th><th>cuDNN:</th><th>CUDA:</th></tr>
-<tr><td>tensorflow-1.3.0</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>GCC 4.8</td><td>Bazel 0.4.5</td><td>N/A</td><td>N/A</td></tr>
-<tr><td>tensorflow_gpu-1.3.0</td><td>GPU</td><td>2.7, 3.3-3.6</td><td>GCC 4.8</td><td>Bazel 0.4.5</td><td>6</td><td>8</td></tr>
+<tr><td>tensorflow-1.4.0rc0</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>GCC 4.8</td><td>Bazel 0.4.5</td><td>N/A</td><td>N/A</td></tr>
+<tr><td>tensorflow_gpu-1.4.0rc0</td><td>GPU</td><td>2.7, 3.3-3.6</td><td>GCC 4.8</td><td>Bazel 0.4.5</td><td>6</td><td>8</td></tr>
 <tr><td>tensorflow-1.2.0</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>GCC 4.8</td><td>Bazel 0.4.5</td><td>N/A</td><td>N/A</td></tr>
 <tr><td>tensorflow_gpu-1.2.0</td><td>GPU</td><td>2.7, 3.3-3.6</td><td>GCC 4.8</td><td>Bazel 0.4.5</td><td>5.1</td><td>8</td></tr>
 <tr><td>tensorflow-1.1.0</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>GCC 4.8</td><td>Bazel 0.4.2</td><td>N/A</td><td>N/A</td></tr>
@@ -447,7 +454,7 @@ Stack Overflow and specify the `tensorflow` tag.
 **Mac**
 <table>
 <tr><th>Version:</th><th>CPU/GPU:</th><th>Python Version:</th><th>Compiler:</th><th>Build Tools:</th><th>cuDNN:</th><th>CUDA:</th></tr>
-<tr><td>tensorflow-1.3.0</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>Clang from xcode</td><td>Bazel 0.4.5</td><td>N/A</td><td>N/A</td></tr>
+<tr><td>tensorflow-1.4.0rc0</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>Clang from xcode</td><td>Bazel 0.4.5</td><td>N/A</td><td>N/A</td></tr>
 <tr><td>ttensorflow-1.2.0</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>Clang from xcode</td><td>Bazel 0.4.5</td><td>N/A</td><td>N/A</td></tr>
 <tr><td>ttensorflow-1.1.0</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>Clang from xcode</td><td>Bazel 0.4.2</td><td>N/A</td><td>N/A</td></tr>
 <tr><td>ttensorflow_gpu-1.1.0</td><td>GPU</td><td>2.7, 3.3-3.6</td><td>Clang from xcode</td><td>Bazel 0.4.2</td><td>5.1</td><td>8</td></tr>
@@ -458,8 +465,8 @@ Stack Overflow and specify the `tensorflow` tag.
 **Windows**
 <table>
 <tr><th>Version:</th><th>CPU/GPU:</th><th>Python Version:</th><th>Compiler:</th><th>Build Tools:</th><th>cuDNN:</th><th>CUDA:</th></tr>
-<tr><td>tensorflow-1.3.0</td><td>CPU</td><td>3.5-3.6</td><td>MSVC 2015 update 3</td><td>Cmake v3.6.3</td><td>N/A</td><td>N/A</td></tr>
-<tr><td>tensorflow_gpu-1.3.0</td><td>GPU</td><td>3.5-3.6</td><td>MSVC 2015 update 3</td><td>Cmake v3.6.3</td><td>6</td><td>8</td></tr>
+<tr><td>tensorflow-1.4.0rc0</td><td>CPU</td><td>3.5-3.6</td><td>MSVC 2015 update 3</td><td>Cmake v3.6.3</td><td>N/A</td><td>N/A</td></tr>
+<tr><td>tensorflow_gpu-1.4.0rc0</td><td>GPU</td><td>3.5-3.6</td><td>MSVC 2015 update 3</td><td>Cmake v3.6.3</td><td>6</td><td>8</td></tr>
 <tr><td>tensorflow-1.2.0</td><td>CPU</td><td>3.5-3.6</td><td>MSVC 2015 update 3</td><td>Cmake v3.6.3</td><td>N/A</td><td>N/A</td></tr>
 <tr><td>tensorflow_gpu-1.2.0</td><td>GPU</td><td>3.5-3.6</td><td>MSVC 2015 update 3</td><td>Cmake v3.6.3</td><td>5.1</td><td>8</td></tr>
 <tr><td>tensorflow-1.1.0</td><td>CPU</td><td>3.5</td><td>MSVC 2015 update 3</td><td>Cmake v3.6.3</td><td>N/A</td><td>N/A</td></tr>
diff --git a/tensorflow/docs_src/install/install_windows.md b/tensorflow/docs_src/install/install_windows.md
index ae8749c231..f0d580d803 100644
--- a/tensorflow/docs_src/install/install_windows.md
+++ b/tensorflow/docs_src/install/install_windows.md
@@ -105,7 +105,7 @@ Take the following steps to install TensorFlow in an Anaconda environment:
   2. Create a conda environment named <tt>tensorflow</tt>
      by invoking the following command:
 
-     <pre>C:\> <b>conda create -n tensorflow python=3.5</b> </pre>
+     <pre>C:\> <b>conda create -n tensorflow pip python=3.5</b> </pre>
 
   3. Activate the conda environment by issuing the following command:
 
diff --git a/tensorflow/docs_src/performance/performance_guide.md b/tensorflow/docs_src/performance/performance_guide.md
index 06bb40f64d..da556bd848 100644
--- a/tensorflow/docs_src/performance/performance_guide.md
+++ b/tensorflow/docs_src/performance/performance_guide.md
@@ -127,7 +127,7 @@ Reading large numbers of small files significantly impacts I/O performance.
 One approach to get maximum I/O throughput is to preprocess input data into
 larger (~100MB) `TFRecord` files. For smaller data sets (200MB-1GB), the best
 approach is often to load the entire data set into memory. The document
-[Downloading and converting to TFRecord format](https://github.com/tensorflow/models/tree/master/slim#Data)
+[Downloading and converting to TFRecord format](https://github.com/tensorflow/models/tree/master/research/slim#Data)
 includes information and scripts for creating `TFRecords` and this
 [script](https://github.com/tensorflow/models/tree/master/tutorials/image/cifar10_estimator/generate_cifar10_tfrecords.py)
 converts the CIFAR-10 data set into `TFRecords`.
diff --git a/tensorflow/docs_src/performance/performance_models.md b/tensorflow/docs_src/performance/performance_models.md
index 183bbc75a9..fcda19e74c 100644
--- a/tensorflow/docs_src/performance/performance_models.md
+++ b/tensorflow/docs_src/performance/performance_models.md
@@ -345,7 +345,7 @@ executing the main script
 *   **`num_gpus`**: Number of GPUs to use.
 *   **`data_dir`**: Path to data to process. If not set, synthetic data is used.
     To use Imagenet data use these
-    [instructions](https://github.com/tensorflow/models/tree/master/inception#getting-started)
+    [instructions](https://github.com/tensorflow/models/tree/master/research/inception#getting-started)
     as a starting point.
 *   **`batch_size`**: Batch size for each GPU.
 *   **`variable_update`**: The method for managing variables: `parameter_server`
diff --git a/tensorflow/docs_src/programmers_guide/datasets.md b/tensorflow/docs_src/programmers_guide/datasets.md
index 38e5612fb4..f458cbcef2 100644
--- a/tensorflow/docs_src/programmers_guide/datasets.md
+++ b/tensorflow/docs_src/programmers_guide/datasets.md
@@ -44,7 +44,7 @@ To start an input pipeline, you must define a *source*. For example, to
 construct a `Dataset` from some tensors in memory, you can use
 `tf.data.Dataset.from_tensors()` or
 `tf.data.Dataset.from_tensor_slices()`. Alternatively, if your input
-data are on disk in the recommend TFRecord format, you can construct a
+data are on disk in the recommended TFRecord format, you can construct a
 `tf.data.TFRecordDataset`.
 
 Once you have a `Dataset` object, you can *transform* it into a new `Dataset` by
diff --git a/tensorflow/docs_src/programmers_guide/graphs.md b/tensorflow/docs_src/programmers_guide/graphs.md
index 6ba8bb7a34..10f53fe8f2 100644
--- a/tensorflow/docs_src/programmers_guide/graphs.md
+++ b/tensorflow/docs_src/programmers_guide/graphs.md
@@ -404,8 +404,8 @@ y = tf.square(x)
 
 with tf.Session() as sess:
   # Feeding a value changes the result that is returned when you evaluate `y`.
-  print(sess.run(y, {x: [1.0, 2.0, 3.0]})  # => "[1.0, 4.0, 9.0]"
-  print(sess.run(y, {x: [0.0, 0.0, 5.0]})  # => "[0.0, 0.0, 25.0]"
+  print(sess.run(y, {x: [1.0, 2.0, 3.0]}))  # => "[1.0, 4.0, 9.0]"
+  print(sess.run(y, {x: [0.0, 0.0, 5.0]}))  # => "[0.0, 0.0, 25.0]"
 
   # Raises `tf.errors.InvalidArgumentError`, because you must feed a value for
   # a `tf.placeholder()` when evaluating a tensor that depends on it.
diff --git a/tensorflow/docs_src/programmers_guide/saved_model.md b/tensorflow/docs_src/programmers_guide/saved_model.md
index 9262143ad8..6bc2cbb9e3 100644
--- a/tensorflow/docs_src/programmers_guide/saved_model.md
+++ b/tensorflow/docs_src/programmers_guide/saved_model.md
@@ -158,6 +158,39 @@ Notes:
    optionally choose names for the variables in the checkpoint files.
 
 
+### Inspect variables in a checkpoint
+
+We can quickly inspect variables in a checkpoint with the 
+[`inspect_checkpoint`](https://www.tensorflow.org/code/tensorflow/python/tools/inspect_checkpoint.py) library.
+
+Continuing from the save/restore examples shown earlier:
+
+```python
+# import the inspect_checkpoint library
+from tensorflow.python.tools import inspect_checkpoint as chkp
+
+# print all tensors in checkpoint file
+chkp.print_tensors_in_checkpoint_file("/tmp/model.ckpt", tensor_name='', all_tensors=True)
+
+# tensor_name:  v1
+# [ 1.  1.  1.]
+# tensor_name:  v2
+# [-1. -1. -1. -1. -1.]
+
+# print only tensor v1 in checkpoint file
+chkp.print_tensors_in_checkpoint_file("/tmp/model.ckpt", tensor_name='v1', all_tensors=False)
+
+# tensor_name:  v1
+# [ 1.  1.  1.]
+
+# print only tensor v2 in checkpoint file
+chkp.print_tensors_in_checkpoint_file("/tmp/model.ckpt", tensor_name='v2', all_tensors=False)
+
+# tensor_name:  v2
+# [-1. -1. -1. -1. -1.]
+```
+
+
 <a name="models"></a>
 ## Overview of saving and restoring models
 
diff --git a/tensorflow/docs_src/tutorials/wide.md b/tensorflow/docs_src/tutorials/wide.md
index 3055c54021..6292c1a01e 100644
--- a/tensorflow/docs_src/tutorials/wide.md
+++ b/tensorflow/docs_src/tutorials/wide.md
@@ -426,8 +426,7 @@ m = tf.estimator.LinearClassifier(
     optimizer=tf.train.FtrlOptimizer(
       learning_rate=0.1,
       l1_regularization_strength=1.0,
-      l2_regularization_strength=1.0),
-    model_dir=model_dir)
+      l2_regularization_strength=1.0))
 ```
 
 One important difference between L1 and L2 regularization is that L1
diff --git a/tensorflow/examples/get_started/regression/imports85.py b/tensorflow/examples/get_started/regression/imports85.py
index 96a464920a..6bee556eb8 100644
--- a/tensorflow/examples/get_started/regression/imports85.py
+++ b/tensorflow/examples/get_started/regression/imports85.py
@@ -127,7 +127,7 @@ def dataset(y_name="price", train_fraction=0.7):
   def in_test_set(line):
     """Returns a boolean tensor, true if the line is in the training set."""
     # Items not in the training set are in the test set.
-    # This line must use `~` instead of `not` beacuse `not` only works on python
+    # This line must use `~` instead of `not` because `not` only works on python
     # booleans but we are dealing with symbolic tensors.
     return ~in_training_set(line)
 
diff --git a/tensorflow/examples/get_started/regression/linear_regression_categorical.py b/tensorflow/examples/get_started/regression/linear_regression_categorical.py
index 860d0e437c..e2ad415fbc 100644
--- a/tensorflow/examples/get_started/regression/linear_regression_categorical.py
+++ b/tensorflow/examples/get_started/regression/linear_regression_categorical.py
@@ -67,7 +67,7 @@ def main(argv):
 
   # The second way, appropriate for an unspecified vocabulary, is to create a
   # hashed column. It will create a fixed length list of weights, and
-  # automatically assign each input categort to a weight. Due to the
+  # automatically assign each input category to a weight. Due to the
   # pseudo-randomness of the process, some weights may be shared between
   # categories, while others will remain unused.
   make_column = tf.feature_column.categorical_column_with_hash_bucket(
diff --git a/tensorflow/examples/learn/resnet.py b/tensorflow/examples/learn/resnet.py
index 33a09bb6e0..1e0966475b 100755
--- a/tensorflow/examples/learn/resnet.py
+++ b/tensorflow/examples/learn/resnet.py
@@ -190,8 +190,8 @@ def main(unused_args):
 
   # Calculate accuracy.
   test_input_fn = tf.estimator.inputs.numpy_input_fn(
-      x={X_FEATURE: mnist.train.images},
-      y=mnist.train.labels.astype(np.int32),
+      x={X_FEATURE: mnist.test.images},
+      y=mnist.test.labels.astype(np.int32),
       num_epochs=1,
       shuffle=False)
   scores = classifier.evaluate(input_fn=test_input_fn)
diff --git a/tensorflow/examples/tutorials/word2vec/word2vec_basic.py b/tensorflow/examples/tutorials/word2vec/word2vec_basic.py
index 1fa2b14869..142e45a2e8 100644
--- a/tensorflow/examples/tutorials/word2vec/word2vec_basic.py
+++ b/tensorflow/examples/tutorials/word2vec/word2vec_basic.py
@@ -115,11 +115,9 @@ def generate_batch(batch_size, num_skips, skip_window):
   data_index += span
   for i in range(batch_size // num_skips):
     context_words = [w for w in range(span) if w != skip_window]
-    random.shuffle(context_words)
-    words_to_use = collections.deque(context_words)
-    for j in range(num_skips):
+    words_to_use = random.sample(context_words, num_skips)
+    for j, context_word in enumerate(words_to_use):
       batch[i * num_skips + j] = buffer[skip_window]
-      context_word = words_to_use.pop()
       labels[i * num_skips + j, 0] = buffer[context_word]
     if data_index == len(data):
       buffer[:] = data[:span]
diff --git a/tensorflow/java/BUILD b/tensorflow/java/BUILD
index a380bc2c71..d74cb32c5a 100644
--- a/tensorflow/java/BUILD
+++ b/tensorflow/java/BUILD
@@ -24,6 +24,7 @@ java_library(
     ],
     data = [":libtensorflow_jni"],
     javacopts = JAVACOPTS,
+    plugins = [":processor"],
     visibility = ["//visibility:public"],
 )
 
@@ -41,6 +42,21 @@ filegroup(
     ],
 )
 
+java_plugin(
+    name = "processor",
+    generates_api = True,
+    processor_class = "org.tensorflow.processor.OperatorProcessor",
+    visibility = ["//visibility:public"],
+    deps = [":processor_library"],
+)
+
+java_library(
+    name = "processor_library",
+    srcs = glob(["src/gen/java/org/tensorflow/processor/**/*.java"]),
+    javacopts = JAVACOPTS,
+    resources = glob(["src/gen/resources/META-INF/services/javax.annotation.processing.Processor"]),
+)
+
 filegroup(
     name = "java_op_sources",
     srcs = glob(["src/main/java/org/tensorflow/op/**/*.java"]) + [
@@ -264,6 +280,29 @@ tf_java_test(
     ],
 )
 
+#java_test(
+#    name = "OperatorProcessorTest",
+#    size = "small",
+#    srcs = ["src/test/java/org/tensorflow/processor/OperatorProcessorTest.java"],
+#    javacopts = JAVACOPTS,
+#    resources = [":processor_test_resources"],
+#    test_class = "org.tensorflow.processor.OperatorProcessorTest",
+#    deps = [
+#        ":processor_library",
+#        "@junit",
+#        "@com_google_testing_compile",
+#        "@com_google_truth",
+#    ],
+#)
+
+filegroup(
+    name = "processor_test_resources",
+    srcs = glob([
+        "src/test/resources/org/tensorflow/**/*.java",
+        "src/main/java/org/tensorflow/op/annotation/Operator.java",
+    ]),
+)
+
 filegroup(
     name = "libtensorflow_jni",
     srcs = select({
diff --git a/tensorflow/java/src/gen/java/org/tensorflow/processor/OperatorProcessor.java b/tensorflow/java/src/gen/java/org/tensorflow/processor/OperatorProcessor.java
new file mode 100644
index 0000000000..45e42878c7
--- /dev/null
+++ b/tensorflow/java/src/gen/java/org/tensorflow/processor/OperatorProcessor.java
@@ -0,0 +1,164 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+package org.tensorflow.processor;
+
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+import javax.annotation.processing.AbstractProcessor;
+import javax.annotation.processing.Filer;
+import javax.annotation.processing.Messager;
+import javax.annotation.processing.ProcessingEnvironment;
+import javax.annotation.processing.RoundEnvironment;
+import javax.lang.model.SourceVersion;
+import javax.lang.model.element.Element;
+import javax.lang.model.element.TypeElement;
+import javax.tools.Diagnostic.Kind;
+
+/**
+ * A compile-time Processor that aggregates classes annotated with {@link
+ * org.tensorflow.op.annotation.Operator} and generates the {@code Ops} convenience API. Please
+ * refer to the {@link org.tensorflow.op.annotation.Operator} annotation for details about the API
+ * generated for each annotated class.
+ *
+ * <p>Note that this processor can only be invoked once, in a single compilation run that includes
+ * all the {@code Operator} annotated source classes. The reason is that the {@code Ops} API is an
+ * "aggregating" API, and annotation processing does not permit modifying an already generated
+ * class.
+ *
+ * @see org.tensorflow.op.annotation.Operator
+ */
+public final class OperatorProcessor extends AbstractProcessor {
+
+  @Override
+  public SourceVersion getSupportedSourceVersion() {
+    return SourceVersion.latestSupported();
+  }
+
+  @Override
+  public synchronized void init(ProcessingEnvironment processingEnv) {
+    super.init(processingEnv);
+    messager = processingEnv.getMessager();
+    filer = processingEnv.getFiler();
+  }
+
+  @Override
+  public boolean process(Set<? extends TypeElement> annotations, RoundEnvironment roundEnv) {
+    // Nothing needs to be done at the end of all rounds.
+    if (roundEnv.processingOver()) {
+      return false;
+    }
+
+    // Nothing to look at in this round.
+    if (annotations.size() == 0) {
+      return false;
+    }
+
+    // We expect to be registered for exactly one annotation.
+    if (annotations.size() != 1) {
+      throw new IllegalStateException(
+          "Unexpected - multiple annotations registered: " + annotations);
+    }
+    TypeElement annotation = annotations.iterator().next();
+    Set<? extends Element> annotated = roundEnv.getElementsAnnotatedWith(annotation);
+
+    // If there are no annotated elements, claim the annotion but do nothing.
+    if (annotated.size() == 0) {
+      return true;
+    }
+
+    // This processor has to aggregate all op classes in one round, as it generates a single Ops
+    // API class which cannot be modified once generated. If we find an annotation after we've
+    // generated our code, flag the location of each such class.
+    if (hasRun) {
+      for (Element e : annotated) {
+        error(
+            e,
+            "The Operator processor has already processed @Operator annotated sources\n"
+                + "and written out an Ops API. It cannot process additional @Operator sources.\n"
+                + "One reason this can happen is if other annotation processors generate\n"
+                + "new @Operator source files.");
+      }
+      return true;
+    }
+
+    // Collect all classes tagged with our annotation.
+    Set<TypeElement> opClasses = new HashSet<TypeElement>();
+    if (!collectOpClasses(roundEnv, opClasses, annotation)) {
+      return true;
+    }
+
+    // Nothing to do when there are no tagged classes.
+    if (opClasses.isEmpty()) {
+      return true;
+    }
+
+    // TODO:(kbsriram) validate operator classes and generate Op API.
+    writeApi();
+    hasRun = true;
+    return true;
+  }
+
+  @Override
+  public Set<String> getSupportedAnnotationTypes() {
+    return Collections.singleton(String.format("%s.annotation.Operator", OP_PACKAGE));
+  }
+
+  private void writeApi() {
+    // Generate an empty class for now and get the build working correctly. This will be changed to
+    // generate the actual API once we've done with build-related changes.
+    // TODO:(kbsriram)
+    try (PrintWriter writer =
+        new PrintWriter(filer.createSourceFile(String.format("%s.Ops", OP_PACKAGE)).openWriter())) {
+      writer.println(String.format("package %s;", OP_PACKAGE));
+      writer.println("public class Ops{}");
+    } catch (IOException e) {
+      error(null, "Unexpected failure generating API: %s", e.getMessage());
+    }
+  }
+
+  private boolean collectOpClasses(
+      RoundEnvironment roundEnv, Set<TypeElement> opClasses, TypeElement annotation) {
+    boolean result = true;
+    for (Element e : roundEnv.getElementsAnnotatedWith(annotation)) {
+      // @Operator can only apply to types, so e must be a TypeElement.
+      if (!(e instanceof TypeElement)) {
+        error(
+            e,
+            "@Operator can only be applied to classes, but this is a %s",
+            e.getKind().toString());
+        result = false;
+        continue;
+      }
+      opClasses.add((TypeElement) e);
+    }
+    return result;
+  }
+
+  private void error(Element e, String message, Object... args) {
+    if (args != null && args.length > 0) {
+      message = String.format(message, args);
+    }
+    messager.printMessage(Kind.ERROR, message, e);
+  }
+
+  private Filer filer;
+  private Messager messager;
+  private boolean hasRun = false;
+  private static final String OP_PACKAGE = "org.tensorflow.op";
+}
diff --git a/tensorflow/java/src/gen/resources/META-INF/services/javax.annotation.processing.Processor b/tensorflow/java/src/gen/resources/META-INF/services/javax.annotation.processing.Processor
new file mode 100644
index 0000000000..9a4fc98a89
--- /dev/null
+++ b/tensorflow/java/src/gen/resources/META-INF/services/javax.annotation.processing.Processor
@@ -0,0 +1 @@
+org.tensorflow.processor.OperatorProcessor
diff --git a/tensorflow/java/src/main/java/org/tensorflow/op/annotation/Operator.java b/tensorflow/java/src/main/java/org/tensorflow/op/annotation/Operator.java
index 59476fb43d..3782240edb 100644
--- a/tensorflow/java/src/main/java/org/tensorflow/op/annotation/Operator.java
+++ b/tensorflow/java/src/main/java/org/tensorflow/op/annotation/Operator.java
@@ -54,7 +54,7 @@ import java.lang.annotation.Target;
  */
 @Documented
 @Target(ElementType.TYPE)
-@Retention(RetentionPolicy.CLASS)
+@Retention(RetentionPolicy.SOURCE)
 public @interface Operator {
   /**
    * Specify an optional group within the {@code Ops} class.
diff --git a/tensorflow/java/src/test/java/org/tensorflow/processor/OperatorProcessorTest.java b/tensorflow/java/src/test/java/org/tensorflow/processor/OperatorProcessorTest.java
new file mode 100644
index 0000000000..9fa1bad20d
--- /dev/null
+++ b/tensorflow/java/src/test/java/org/tensorflow/processor/OperatorProcessorTest.java
@@ -0,0 +1,51 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+package org.tensorflow.processor;
+
+import static com.google.testing.compile.CompilationSubject.assertThat;
+
+import com.google.testing.compile.Compilation;
+import com.google.testing.compile.Compiler;
+import com.google.testing.compile.JavaFileObjects;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Basic tests for {@link org.tensorflow.processor.operator.OperatorProcessor}. */
+@RunWith(JUnit4.class)
+public final class OperatorProcessorTest {
+
+  @Test
+  public void basicGood() {
+    Compilation compile = compile("org/tensorflow/processor/operator/good/BasicGood.java");
+    assertThat(compile).succeededWithoutWarnings();
+    assertThat(compile).generatedSourceFile("org.tensorflow.op.Ops");
+  }
+
+  @Test
+  public void basicBad() {
+    assertThat(compile("org/tensorflow/processor/operator/bad/BasicBad.java")).failed();
+  }
+
+  // Create a compilation unit that includes the @Operator annotation and processor.
+  private static Compilation compile(String path) {
+    return Compiler.javac()
+        .withProcessors(new OperatorProcessor())
+        .compile(
+            JavaFileObjects.forResource("src/main/java/org/tensorflow/op/annotation/Operator.java"),
+            JavaFileObjects.forResource(path));
+  }
+}
diff --git a/tensorflow/java/src/test/resources/org/tensorflow/processor/operator/bad/BasicBad.java b/tensorflow/java/src/test/resources/org/tensorflow/processor/operator/bad/BasicBad.java
new file mode 100644
index 0000000000..7d12857dfa
--- /dev/null
+++ b/tensorflow/java/src/test/resources/org/tensorflow/processor/operator/bad/BasicBad.java
@@ -0,0 +1,22 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+package org.tensorflow.processor.operator.bad;
+
+import org.tensorflow.op.annotation.Operator;
+
+public class BasicBad {
+  @Operator int foo;
+}
diff --git a/tensorflow/java/src/test/resources/org/tensorflow/processor/operator/good/BasicGood.java b/tensorflow/java/src/test/resources/org/tensorflow/processor/operator/good/BasicGood.java
new file mode 100644
index 0000000000..4cf175f00d
--- /dev/null
+++ b/tensorflow/java/src/test/resources/org/tensorflow/processor/operator/good/BasicGood.java
@@ -0,0 +1,21 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+package org.tensorflow.processor.operator.good;
+
+import org.tensorflow.op.annotation.Operator;
+
+@Operator
+public class BasicGood {}
diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 4382eeb9a8..953aa566f0 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -4202,6 +4202,19 @@ cuda_py_test(
     main = "client/session_benchmark.py",
 )
 
+cuda_py_test(
+    name = "nn_grad_test",
+    size = "small",
+    srcs = ["ops/nn_grad_test.py"],
+    additional_deps = [
+        ":client_testlib",
+        ":framework_for_generated_wrappers",
+        ":nn_grad",
+        ":nn_ops",
+        "//third_party/py/numpy",
+    ],
+)
+
 py_library(
     name = "tf_item",
     srcs = [
diff --git a/tensorflow/python/debug/cli/tensor_format.py b/tensorflow/python/debug/cli/tensor_format.py
index 7a5597db12..05ccf93f15 100644
--- a/tensorflow/python/debug/cli/tensor_format.py
+++ b/tensorflow/python/debug/cli/tensor_format.py
@@ -480,7 +480,7 @@ def _pad_string_to_length(string, length):
 
 
 def numeric_summary(tensor):
-  """Get a text summmary of a numeric tensor.
+  """Get a text summary of a numeric tensor.
 
   This summary is only available for numeric (int*, float*, complex*) and
   Boolean tensors.
diff --git a/tensorflow/python/estimator/training.py b/tensorflow/python/estimator/training.py
index 64b014a6b5..1131995b3e 100644
--- a/tensorflow/python/estimator/training.py
+++ b/tensorflow/python/estimator/training.py
@@ -199,7 +199,7 @@ class EvalSpec(
         evaluations on different data sets. Metrics for different evaluations
         are saved in separate folders, and appear separately in tensorboard.
       hooks: Iterable of `tf.train.SessionRunHook` objects to run
-        on all workers (including chief) during training.
+        during evaluation.
       exporters: Iterable of `Exporter`s, or a single one, or `None`.
         `exporters` will be invoked after each evaluation.
       start_delay_secs: Int. Start evaluating after waiting for this many
@@ -408,8 +408,8 @@ def train_and_evaluate(estimator, train_spec, eval_spec):
 
   Args:
     estimator: An `Estimator` instance to train and evaluate.
-    train_spec: A `TrainSpec instance to specify the training specification.
-    eval_spec: A `EvalSpec instance to specify the evaluation and export
+    train_spec: A `TrainSpec` instance to specify the training specification.
+    eval_spec: A `EvalSpec` instance to specify the evaluation and export
       specification.
 
   Raises:
diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD
index b02bae95fd..d8ecabcdea 100644
--- a/tensorflow/python/kernel_tests/BUILD
+++ b/tensorflow/python/kernel_tests/BUILD
@@ -903,6 +903,21 @@ cuda_py_test(
     ],
 )
 
+cuda_py_test(
+    name = "nth_element_op_test",
+    size = "small",
+    srcs = ["nth_element_op_test.py"],
+    additional_deps = [
+        "//third_party/py/numpy",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework_for_generated_wrappers",
+        "//tensorflow/python:gradients",
+        "//tensorflow/python:nn_grad",
+        "//tensorflow/python:nn_ops",
+    ],
+)
+
 tf_py_test(
     name = "unique_op_test",
     size = "small",
diff --git a/tensorflow/python/kernel_tests/batchtospace_op_test.py b/tensorflow/python/kernel_tests/batchtospace_op_test.py
index 8ec93119f2..0c802476a0 100644
--- a/tensorflow/python/kernel_tests/batchtospace_op_test.py
+++ b/tensorflow/python/kernel_tests/batchtospace_op_test.py
@@ -24,6 +24,7 @@ from __future__ import print_function
 
 import numpy as np
 
+from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
@@ -52,14 +53,15 @@ class BatchToSpaceDepthToSpace(test.TestCase, PythonOpImpl):
   def testDepthToSpaceTranspose(self):
     x = np.arange(20 * 5 * 8 * 7, dtype=np.float32).reshape([20, 5, 8, 7])
     block_size = 2
-    crops = np.zeros((2, 2), dtype=np.int32)
-    y1 = self.batch_to_space(x, crops, block_size=block_size)
-    y2 = array_ops.transpose(
-        array_ops.depth_to_space(
-            array_ops.transpose(x, [3, 1, 2, 0]), block_size=block_size),
-        [3, 1, 2, 0])
-    with self.test_session():
-      self.assertAllEqual(y1.eval(), y2.eval())
+    for crops_dtype in [dtypes.int64, dtypes.int32]:
+      crops = array_ops.zeros((2, 2), dtype=crops_dtype)
+      y1 = self.batch_to_space(x, crops, block_size=block_size)
+      y2 = array_ops.transpose(
+          array_ops.depth_to_space(
+              array_ops.transpose(x, [3, 1, 2, 0]), block_size=block_size),
+          [3, 1, 2, 0])
+      with self.test_session():
+        self.assertAllEqual(y1.eval(), y2.eval())
 
 
 class BatchToSpaceDepthToSpaceCpp(BatchToSpaceDepthToSpace, CppOpImpl):
@@ -287,9 +289,10 @@ class BatchToSpaceGradientCppTest(BatchToSpaceGradientTest, CppOpImpl):
 class BatchToSpaceNDGradientTest(test.TestCase):
 
   # Check the gradients.
-  def _checkGrad(self, x, block_shape, crops):
+  def _checkGrad(self, x, block_shape, crops, crops_dtype):
     block_shape = np.array(block_shape)
-    crops = np.array(crops).reshape((len(block_shape), 2))
+    crops = constant_op.constant(
+        np.array(crops).reshape((len(block_shape), 2)), crops_dtype)
     with self.test_session():
       tf_x = ops.convert_to_tensor(x)
       tf_y = array_ops.batch_to_space_nd(tf_x, block_shape, crops)
@@ -304,23 +307,26 @@ class BatchToSpaceNDGradientTest(test.TestCase):
 
     self.assertAllClose(x_jacob_t, x_jacob_n, rtol=1e-2, atol=epsilon)
 
-  def _compare(self, input_shape, block_shape, crops):
+  def _compare(self, input_shape, block_shape, crops, crops_dtype):
     input_shape = list(input_shape)
     input_shape[0] *= np.prod(block_shape)
     x = np.random.normal(
         0, 1, np.prod(input_shape)).astype(np.float32).reshape(input_shape)
-    self._checkGrad(x, block_shape, crops)
+    self._checkGrad(x, block_shape, crops, crops_dtype)
 
   # Don't use very large numbers as dimensions here as the result is tensor
   # with cartesian product of the dimensions.
   def testSmall(self):
-    self._compare([1, 2, 3, 5], [2, 2], [[0, 0], [0, 0]])
+    for dtype in [dtypes.int64, dtypes.int32]:
+      self._compare([1, 2, 3, 5], [2, 2], [[0, 0], [0, 0]], dtype)
 
   def testSmall2(self):
-    self._compare([2, 4, 3, 2], [2, 2], [[0, 0], [0, 0]])
+    for dtype in [dtypes.int64, dtypes.int32]:
+      self._compare([2, 4, 3, 2], [2, 2], [[0, 0], [0, 0]], dtype)
 
   def testSmallCrop1x1(self):
-    self._compare([1, 2, 3, 5], [2, 2], [[1, 1], [1, 1]])
+    for dtype in [dtypes.int64, dtypes.int32]:
+      self._compare([1, 2, 3, 5], [2, 2], [[1, 1], [1, 1]], dtype)
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/kernel_tests/diag_op_test.py b/tensorflow/python/kernel_tests/diag_op_test.py
index f0b7885732..6cfa9b37fe 100644
--- a/tensorflow/python/kernel_tests/diag_op_test.py
+++ b/tensorflow/python/kernel_tests/diag_op_test.py
@@ -279,7 +279,7 @@ class MatrixDiagPartTest(test.TestCase):
 
 class DiagTest(test.TestCase):
 
-  def diagOp(self, diag, dtype, expected_ans, use_gpu=False):
+  def _diagOp(self, diag, dtype, expected_ans, use_gpu):
     with self.test_session(use_gpu=use_gpu):
       tf_ans = array_ops.diag(ops.convert_to_tensor(diag.astype(dtype)))
       out = tf_ans.eval()
@@ -290,6 +290,10 @@ class DiagTest(test.TestCase):
     self.assertShapeEqual(expected_ans, tf_ans)
     self.assertShapeEqual(diag, tf_ans_inv)
 
+  def diagOp(self, diag, dtype, expected_ans):
+    self._diagOp(diag, dtype, expected_ans, False)
+    self._diagOp(diag, dtype, expected_ans, True)
+
   def testEmptyTensor(self):
     x = np.array([])
     expected_ans = np.empty([0, 0])
@@ -400,13 +404,53 @@ class DiagTest(test.TestCase):
           dtype=dtype)
       self.diagOp(x, dtype, expected_ans)
 
+  def testRankFourNumberTensor(self):
+    for dtype in [np.float32, np.float64, np.int64, np.int32]:
+      # Input with shape [2, 1, 2, 3]
+      x = np.array([[[[ 1,  2,  3],
+                      [ 4,  5,  6]]],
+                    [[[ 7,  8,  9],
+                      [10, 11, 12]]]], dtype=dtype)
+      # Output with shape [2, 1, 2, 3, 2, 1, 2, 3]
+      expected_ans = np.array(
+          [[[[[[[[1, 0, 0], [0, 0, 0]]],
+               [[[0, 0, 0], [0, 0, 0]]]],
+              [[[[0, 2, 0], [0, 0, 0]]],
+               [[[0, 0, 0], [0, 0, 0]]]],
+              [[[[0, 0, 3], [0, 0, 0]]],
+               [[[0, 0, 0], [0, 0, 0]]]]],
+             [[[[[0, 0, 0], [4, 0, 0]]],
+               [[[0, 0, 0], [0, 0, 0]]]],
+              [[[[0, 0, 0], [0, 5, 0]]],
+               [[[0, 0, 0], [0, 0, 0]]]],
+              [[[[0, 0, 0], [0, 0, 6]]],
+               [[[0, 0, 0], [0, 0, 0]]]]]]],
+
+           [[[[[[[0, 0, 0], [0, 0, 0]]],
+               [[[7, 0, 0], [0, 0, 0]]]],
+              [[[[0, 0, 0], [0, 0, 0]]],
+               [[[0, 8, 0], [0, 0, 0]]]],
+              [[[[0, 0, 0], [0, 0, 0]]],
+               [[[0, 0, 9], [0, 0, 0]]]]],
+             [[[[[0, 0, 0], [0, 0, 0]]],
+               [[[0, 0, 0], [10, 0, 0]]]],
+              [[[[0, 0, 0], [0, 0, 0]]],
+               [[[0, 0, 0], [0, 11, 0]]]],
+              [[[[0, 0, 0], [0, 0, 0]]],
+               [[[0, 0, 0], [0, 0, 12]]]]]]]], dtype=dtype)
+      self.diagOp(x, dtype, expected_ans)
+
+  def testInvalidRank(self):
+    with self.assertRaisesRegexp(ValueError, "must be at least rank 1"):
+      array_ops.diag(0.0)
+
 
 class DiagPartOpTest(test.TestCase):
 
   def setUp(self):
     np.random.seed(0)
 
-  def diagPartOp(self, tensor, dtype, expected_ans, use_gpu=False):
+  def _diagPartOp(self, tensor, dtype, expected_ans, use_gpu):
     with self.test_session(use_gpu=use_gpu):
       tensor = ops.convert_to_tensor(tensor.astype(dtype))
       tf_ans_inv = array_ops.diag_part(tensor)
@@ -414,6 +458,10 @@ class DiagPartOpTest(test.TestCase):
     self.assertAllClose(inv_out, expected_ans)
     self.assertShapeEqual(expected_ans, tf_ans_inv)
 
+  def diagPartOp(self, tensor, dtype, expected_ans):
+    self._diagPartOp(tensor, dtype, expected_ans, False)
+    self._diagPartOp(tensor, dtype, expected_ans, True)
+
   def testRankTwoFloatTensor(self):
     x = np.random.rand(3, 3)
     i = np.arange(3)
@@ -451,11 +499,23 @@ class DiagPartOpTest(test.TestCase):
     self.diagPartOp(x, np.float32, expected_ans)
     self.diagPartOp(x, np.float64, expected_ans)
 
+  def testRankEightComplexTensor(self):
+    x = np.random.rand(2, 2, 2, 3, 2, 2, 2, 3)
+    i = np.arange(2)[:, None, None, None]
+    j = np.arange(2)[:, None, None]
+    k = np.arange(2)[:, None]
+    l = np.arange(3)
+    expected_ans = x[i, j, k, l, i, j, k, l]
+    self.diagPartOp(x, np.complex64, expected_ans)
+    self.diagPartOp(x, np.complex128, expected_ans)
+
   def testOddRank(self):
     w = np.random.rand(2)
     x = np.random.rand(2, 2, 2)
     self.assertRaises(ValueError, self.diagPartOp, w, np.float32, 0)
     self.assertRaises(ValueError, self.diagPartOp, x, np.float32, 0)
+    with self.assertRaises(ValueError):
+      array_ops.diag_part(0.0)
 
   def testUnevenDimensions(self):
     w = np.random.rand(2, 5)
diff --git a/tensorflow/python/kernel_tests/listdiff_op_test.py b/tensorflow/python/kernel_tests/listdiff_op_test.py
index 4f053d2a21..ee86cf0b24 100644
--- a/tensorflow/python/kernel_tests/listdiff_op_test.py
+++ b/tensorflow/python/kernel_tests/listdiff_op_test.py
@@ -41,15 +41,17 @@ class ListDiffTest(test.TestCase):
         y = [compat.as_bytes(str(a)) for a in y]
         out = [compat.as_bytes(str(a)) for a in out]
       for diff_func in [array_ops.setdiff1d]:
-        with self.test_session() as sess:
-          x_tensor = ops.convert_to_tensor(x, dtype=dtype)
-          y_tensor = ops.convert_to_tensor(y, dtype=dtype)
-          out_tensor, idx_tensor = diff_func(x_tensor, y_tensor)
-          tf_out, tf_idx = sess.run([out_tensor, idx_tensor])
-        self.assertAllEqual(tf_out, out)
-        self.assertAllEqual(tf_idx, idx)
-        self.assertEqual(1, out_tensor.get_shape().ndims)
-        self.assertEqual(1, idx_tensor.get_shape().ndims)
+        for index_dtype in [dtypes.int32, dtypes.int64]:
+          with self.test_session() as sess:
+            x_tensor = ops.convert_to_tensor(x, dtype=dtype)
+            y_tensor = ops.convert_to_tensor(y, dtype=dtype)
+            out_tensor, idx_tensor = diff_func(x_tensor, y_tensor,
+                                               index_dtype=index_dtype)
+            tf_out, tf_idx = sess.run([out_tensor, idx_tensor])
+          self.assertAllEqual(tf_out, out)
+          self.assertAllEqual(tf_idx, idx)
+          self.assertEqual(1, out_tensor.get_shape().ndims)
+          self.assertEqual(1, idx_tensor.get_shape().ndims)
 
   def testBasic1(self):
     x = [1, 2, 3, 4]
diff --git a/tensorflow/python/kernel_tests/metrics_test.py b/tensorflow/python/kernel_tests/metrics_test.py
index f21b0dfeab..e5b7cbce7a 100644
--- a/tensorflow/python/kernel_tests/metrics_test.py
+++ b/tensorflow/python/kernel_tests/metrics_test.py
@@ -3426,7 +3426,7 @@ class MeanIOUTest(test.TestCase):
       sess.run(variables.local_variables_initializer())
       for _ in range(5):
         sess.run(update_op)
-      desired_output = np.mean([1.0 / 3.0, 2.0 / 4.0, 0.])
+      desired_output = np.mean([1.0 / 3.0, 2.0 / 4.0])
       self.assertAlmostEqual(desired_output, miou.eval())
 
   def testUpdateOpEvalIsAccumulatedConfusionMatrix(self):
@@ -3505,6 +3505,55 @@ class MeanIOUTest(test.TestCase):
       desired_miou = np.mean([2. / 4., 4. / 6.])
       self.assertAlmostEqual(desired_miou, miou.eval())
 
+  def testMissingClassInLabels(self):
+    labels = constant_op.constant([
+      [[0, 0, 1, 1, 0, 0],
+       [1, 0, 0, 0, 0, 1]],
+      [[1, 1, 1, 1, 1, 1],
+       [0, 0, 0, 0, 0, 0]]])
+    predictions = constant_op.constant([
+      [[0, 0, 2, 1, 1, 0],
+       [0, 1, 2, 2, 0, 1]],
+      [[0, 0, 2, 1, 1, 1],
+       [1, 1, 2, 0, 0, 0]]])
+    num_classes = 3
+    with self.test_session() as sess:
+      miou, update_op = metrics.mean_iou(labels, predictions, num_classes)
+      sess.run(variables.local_variables_initializer())
+      self.assertAllEqual([[7, 4, 3], [3, 5, 2], [0, 0, 0]], update_op.eval())
+      self.assertAlmostEqual(
+        1 / 3 * (7 / (7 + 3 + 7) + 5 / (5 + 4 + 5) + 0 / (0 + 5 + 0)),
+        miou.eval())
+
+  def testMissingClassOverallSmall(self):
+    labels = constant_op.constant([0])
+    predictions = constant_op.constant([0])
+    num_classes = 2
+    with self.test_session() as sess:
+      miou, update_op = metrics.mean_iou(labels, predictions, num_classes)
+      sess.run(variables.local_variables_initializer())
+      self.assertAllEqual([[1, 0], [0, 0]], update_op.eval())
+      self.assertAlmostEqual(1, miou.eval())
+
+  def testMissingClassOverallLarge(self):
+    labels = constant_op.constant([
+      [[0, 0, 1, 1, 0, 0],
+       [1, 0, 0, 0, 0, 1]],
+      [[1, 1, 1, 1, 1, 1],
+       [0, 0, 0, 0, 0, 0]]])
+    predictions = constant_op.constant([
+      [[0, 0, 1, 1, 0, 0],
+       [1, 1, 0, 0, 1, 1]],
+      [[0, 0, 0, 1, 1, 1],
+       [1, 1, 1, 0, 0, 0]]])
+    num_classes = 3
+    with self.test_session() as sess:
+      miou, update_op = metrics.mean_iou(labels, predictions, num_classes)
+      sess.run(variables.local_variables_initializer())
+      self.assertAllEqual([[9, 5, 0], [3, 7, 0], [0, 0, 0]], update_op.eval())
+      self.assertAlmostEqual(
+        1 / 2 * (9 / (9 + 3 + 5) + 7 / (7 + 5 + 3)), miou.eval())
+
 
 class MeanPerClassAccuracyTest(test.TestCase):
 
diff --git a/tensorflow/python/kernel_tests/nth_element_op_test.py b/tensorflow/python/kernel_tests/nth_element_op_test.py
new file mode 100644
index 0000000000..58cd46d2d5
--- /dev/null
+++ b/tensorflow/python/kernel_tests/nth_element_op_test.py
@@ -0,0 +1,174 @@
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+import tensorflow.python.ops.nn_grad  # pylint: disable=unused-import
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import nn_ops
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import gradients_impl
+from tensorflow.python.platform import test
+
+
+class NthElementTest(test.TestCase):
+
+  def _validateNthElement(self, inputs, dtype, n, reverse, expected_values):
+    np_expected_values = np.array(expected_values)
+    with self.test_session(use_gpu=False) as sess:
+      inputs_op = ops.convert_to_tensor(inputs, dtype=dtype)
+      values_op = nn_ops.nth_element(inputs_op, n, reverse=reverse)
+      values = sess.run(values_op)
+
+      self.assertShapeEqual(np_expected_values, values_op)
+      self.assertAllClose(np_expected_values, values)
+
+  def testExample1(self):
+    inputs = [2.2, 4.4, 1.1, 5.5, 3.3]
+    self._validateNthElement(inputs, dtypes.float32, 1, False, 2.2)
+    self._validateNthElement(inputs, dtypes.float32, 1, True, 4.4)
+
+  def testExample2(self):
+    inputs = [[2.2, 4.4, 1.1], [5.5, 3.3, 6.6]]
+    self._validateNthElement(inputs, dtypes.float64, 2, False, [4.4, 6.6])
+    self._validateNthElement(inputs, dtypes.float64, 2, True, [1.1, 3.3])
+
+  def testExample3(self):
+    inputs = [[[2, 4, 1], [5, -3, 6]],
+              [[7, 9, -8], [9, 0, 4]]]
+    self._validateNthElement(inputs, dtypes.int32, 0, False,
+                             [[1, -3], [-8, 0]])
+    self._validateNthElement(inputs, dtypes.int64, 0, True,
+                             [[4, 6], [9, 9]])
+
+  def _testFloatLargeInput(self, input_shape):
+    inputs = np.random.random_sample(input_shape)
+    n = np.random.randint(input_shape[-1])
+    sort_inputs = np.sort(inputs)
+    expected_values = sort_inputs[..., n]
+    self._validateNthElement(
+        inputs, dtypes.float32, n, False, expected_values)
+    expected_values = sort_inputs[..., ::-1][..., n]
+    self._validateNthElement(
+        inputs, dtypes.float64, n, True, expected_values)
+
+  def _testIntLargeInput(self, input_shape):
+    inputs = np.random.randint(-1e3, 1e3, input_shape)
+    n = np.random.randint(input_shape[-1])
+    sort_inputs = np.sort(inputs)
+    expected_values = sort_inputs[..., n]
+    self._validateNthElement(
+        inputs, dtypes.int32, n, False, expected_values)
+    expected_values = sort_inputs[..., ::-1][..., n]
+    self._validateNthElement(
+        inputs, dtypes.int64, n, True, expected_values)
+
+  def _testLargeInput(self, input_shape):
+    self._testFloatLargeInput(input_shape)
+    self._testIntLargeInput(input_shape)
+
+  def testLargeInput(self):
+    self._testLargeInput([1])
+    self._testLargeInput([10])
+    self._testLargeInput([5, 10])
+    self._testLargeInput([50, 100])
+    self._testLargeInput([50, 10000])
+    self._testLargeInput([50, 10, 100])
+    self._testLargeInput([50, 10, 10, 100])
+
+  def _testEnumerateN(self, input_shape):
+    inputs = np.random.random_sample(input_shape)
+    sort_inputs = np.sort(inputs)
+    for n in range(input_shape[-1]):
+      expected_values = sort_inputs[..., n]
+      self._validateNthElement(
+          inputs, dtypes.float32, n, False, expected_values)
+      expected_values = sort_inputs[..., ::-1][..., n]
+      self._validateNthElement(
+          inputs, dtypes.float64, n, True, expected_values)
+
+  def testEnumerateN(self):
+    self._testEnumerateN([1])
+    self._testEnumerateN([10])
+    self._testEnumerateN([10, 10])
+    self._testEnumerateN([10, 10, 10])
+    self._testEnumerateN([10, 10, 10, 10])
+
+  def testInvalidInput(self):
+    with self.assertRaisesRegexp(ValueError,
+                                 "at least rank 1 but is rank 0"):
+      nn_ops.nth_element(5, 0)
+
+  def testInvalidInputAtEval(self):
+    with self.test_session(use_gpu=False):
+      v = array_ops.placeholder(dtype=dtypes.float32)
+      with self.assertRaisesOpError("Input must be >= 1-D"):
+        nn_ops.nth_element(v, 0).eval(feed_dict={v: 5.0})
+
+  def testInvalidN(self):
+    with self.assertRaisesRegexp(ValueError,
+                                 "non-negative but is -1"):
+      nn_ops.nth_element([5], -1)
+    with self.assertRaisesRegexp(ValueError,
+                                 "scalar but has rank 1"):
+      nn_ops.nth_element([5, 6, 3], [1])
+
+  def testInvalidNAtEval(self):
+    inputs = [[0.1, 0.2], [0.3, 0.4]]
+    with self.test_session(use_gpu=False):
+      n = array_ops.placeholder(dtypes.int32)
+      values = nn_ops.nth_element(inputs, n)
+      with self.assertRaisesOpError("Need n >= 0, got -7"):
+        values.eval(feed_dict={n: -7})
+
+  def testNTooLarge(self):
+    inputs = [[0.1, 0.2], [0.3, 0.4]]
+    with self.assertRaisesRegexp(ValueError,
+                                 "must have last dimension > n = 2"):
+      nn_ops.nth_element(inputs, 2)
+
+  def testNTooLargeAtEval(self):
+    inputs = [[0.1, 0.2], [0.3, 0.4]]
+    with self.test_session(use_gpu=False):
+      n = array_ops.placeholder(dtypes.int32)
+      values = nn_ops.nth_element(inputs, n)
+      with self.assertRaisesOpError(r"Input must have at least n\+1 columns"):
+        values.eval(feed_dict={n: 2})
+
+  def testGradients(self):
+    with self.test_session(use_gpu=False) as sess:
+      inputs = array_ops.placeholder(dtypes.int32, shape=[3, 5])
+      values = nn_ops.nth_element(inputs, 3)
+      grad = sess.run(
+          gradients_impl.gradients(
+              values, inputs, grad_ys=[[-1., 2., 5.]]),
+          feed_dict={inputs: [[2, -1, 1000, 3, 1000],
+                              [1, 5, 2, 4, 3],
+                              [2, 2, 2, 2, 2],
+                             ]})
+    self.assertAllClose(grad[0], [[0, 0, -0.5, 0, -0.5],
+                                  [0, 0, 0, 2, 0],
+                                  [1, 1, 1, 1, 1],
+                                 ])
+
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/python/kernel_tests/pad_op_test.py b/tensorflow/python/kernel_tests/pad_op_test.py
index ca1f3f878f..2c766e3640 100644
--- a/tensorflow/python/kernel_tests/pad_op_test.py
+++ b/tensorflow/python/kernel_tests/pad_op_test.py
@@ -193,6 +193,25 @@ class PadOpTest(test.TestCase):
       with self.assertRaisesRegexp(ValueError, "Unknown padding mode"):
         array_ops.pad(x, [[1, 0], [2, 1]], mode="weird").eval()
 
+  def testPaddingTypes(self):
+    paddings = [[1, 0], [2, 3], [0, 2]]
+    inputs = np.random.randint(-100, 100, (4, 4, 3)).astype(np.float32)
+    for mode in ("CONSTANT", "REFLECT", "SYMMETRIC", "reflect", "symmetric",
+                 "constant"):
+      for padding_dtype in [dtypes.int32, dtypes.int64]:
+        np_val = self._npPad(inputs,
+                             paddings,
+                             mode=mode,
+                             constant_values=0)
+        with self.test_session(use_gpu=True):
+          tf_val = array_ops.pad(inputs,
+                                 constant_op.constant(paddings, padding_dtype),
+                                 mode=mode,
+                                 constant_values=0)
+          out = tf_val.eval()
+        self.assertAllEqual(np_val, out)
+        self.assertShapeEqual(np_val, tf_val)
+
   def testIntTypes(self):
     # TODO(touts): Figure out why the padding tests do not work on GPU
     # for int types and rank > 2.
@@ -284,6 +303,15 @@ class PadOpTest(test.TestCase):
     self.assertAllEqual(inp, out)
     self.assertShapeEqual(inp, tf_val)
 
+  def testPadTypes(self):
+    for dtype in [dtypes.int32, dtypes.int64]:
+      paddings = np.zeros((0, 2))
+      inp = np.asarray(7)
+      with self.test_session(use_gpu=True):
+        tf_val = array_ops.pad(inp, constant_op.constant(paddings, dtype=dtype))
+        out = tf_val.eval()
+      self.assertAllEqual(inp, out)
+      self.assertShapeEqual(inp, tf_val)
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/kernel_tests/reduction_ops_test.py b/tensorflow/python/kernel_tests/reduction_ops_test.py
index c794351fe9..2dc65b1384 100644
--- a/tensorflow/python/kernel_tests/reduction_ops_test.py
+++ b/tensorflow/python/kernel_tests/reduction_ops_test.py
@@ -163,6 +163,13 @@ class SumReductionTest(BaseReductionTest):
       reduction_axes = tuple(reduction_axes)
     return np.sum(x, axis=reduction_axes, keepdims=keep_dims)
 
+  def testAxesType(self):
+    for dtype in [dtypes.int64, dtypes.int32]:
+      with self.test_session(use_gpu=True) as sess:
+        v = math_ops.reduce_sum([0, 0], constant_op.constant(0, dtype=dtype))
+        tf_v = sess.run(v)
+      self.assertAllEqual(tf_v, 0)
+
   def testInfinity(self):
     for dtype in [np.float32, np.float64]:
       for special_value_x in [-np.inf, np.inf]:
@@ -193,6 +200,7 @@ class SumReductionTest(BaseReductionTest):
       tf_out_mean = sess.run(tf_mean)
     self.assertAllClose(tf_out_mean, 1.)
 
+
   def testFloat32(self):
     for rank in range(1, _MAX_RANK + 1):
       np_arr = self._makeIncremental((2,) * rank, dtypes.float32)
@@ -369,6 +377,13 @@ class MeanReductionTest(BaseReductionTest):
       return np_sum // count
     return np_sum / count
 
+  def testAxesType(self):
+    for dtype in [dtypes.int64, dtypes.int32]:
+      with self.test_session(use_gpu=True) as sess:
+        v = math_ops.reduce_mean([0, 0], constant_op.constant(0, dtype=dtype))
+        tf_v = sess.run(v)
+      self.assertAllEqual(tf_v, 0)
+
   def testInfinity(self):
     for dtype in [np.float32, np.float64]:
       for special_value_x in [-np.inf, np.inf]:
@@ -435,6 +450,13 @@ class ProdReductionTest(BaseReductionTest):
       reduction_axes = tuple(reduction_axes)
     return np.prod(x, axis=reduction_axes, keepdims=keep_dims)
 
+  def testAxesType(self):
+    for dtype in [dtypes.int64, dtypes.int32]:
+      with self.test_session(use_gpu=True) as sess:
+        v = math_ops.reduce_prod([0, 0], constant_op.constant(0, dtype=dtype))
+        tf_v = sess.run(v)
+      self.assertAllEqual(tf_v, 0)
+
   def testInfinity(self):
     for dtype in [np.float32, np.float64]:
       for special_value_x in [-np.inf, np.inf]:
@@ -531,6 +553,13 @@ class MinReductionTest(test.TestCase):
     self._compare(x, reduction_axes, True, use_gpu=True)
     self._compare(x, reduction_axes, True, use_gpu=False)
 
+  def testAxesType(self):
+    for dtype in [dtypes.int64, dtypes.int32]:
+      with self.test_session(use_gpu=True) as sess:
+        v = math_ops.reduce_min([0, 0], constant_op.constant(0, dtype=dtype))
+        tf_v = sess.run(v)
+      self.assertAllEqual(tf_v, 0)
+
   def testInfinity(self):
     for dtype in [np.float32, np.float64]:
       for special_value_x in [-np.inf, np.inf]:
@@ -637,6 +666,13 @@ class MaxReductionTest(test.TestCase):
     self._compare(x, reduction_axes, True, use_gpu=True)
     self._compare(x, reduction_axes, True, use_gpu=False)
 
+  def testAxesType(self):
+    for dtype in [dtypes.int64, dtypes.int32]:
+      with self.test_session(use_gpu=True) as sess:
+        v = math_ops.reduce_max([0, 0], constant_op.constant(0, dtype=dtype))
+        tf_v = sess.run(v)
+      self.assertAllEqual(tf_v, 0)
+
   def testInfinity(self):
     for dtype in [np.float32, np.float64]:
       for special_value_x in [-np.inf, np.inf]:
@@ -757,6 +793,14 @@ class AllReductionTest(test.TestCase):
     self._compare(x, reduction_axes, True, use_gpu=True)
     self._compare(x, reduction_axes, True, use_gpu=False)
 
+  def testAxesType(self):
+    for dtype in [dtypes.int64, dtypes.int32]:
+      with self.test_session(use_gpu=True) as sess:
+        v = math_ops.reduce_all([True, True],
+                                constant_op.constant(0, dtype=dtype))
+        tf_v = sess.run(v)
+      self.assertAllEqual(tf_v, True)
+
   def testAll3D(self):
     # Create a 3D array of bools and reduce across all possible
     # dimensions
@@ -798,6 +842,14 @@ class AnyReductionTest(test.TestCase):
     self._compare(x, reduction_axes, True, use_gpu=True)
     self._compare(x, reduction_axes, True, use_gpu=False)
 
+  def testAxesType(self):
+    for dtype in [dtypes.int64, dtypes.int32]:
+      with self.test_session(use_gpu=True) as sess:
+        v = math_ops.reduce_any([True, True],
+                                constant_op.constant(0, dtype=dtype))
+        tf_v = sess.run(v)
+      self.assertAllEqual(tf_v, True)
+
   def testAll3D(self):
     # Create a 3D array of bools and reduce across all possible
     # dimensions
diff --git a/tensorflow/python/kernel_tests/scan_ops_test.py b/tensorflow/python/kernel_tests/scan_ops_test.py
index 6b2b589a06..08b4a2aaae 100644
--- a/tensorflow/python/kernel_tests/scan_ops_test.py
+++ b/tensorflow/python/kernel_tests/scan_ops_test.py
@@ -20,6 +20,8 @@ from __future__ import print_function
 
 import numpy as np
 
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors_impl
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import gradient_checker
@@ -92,6 +94,14 @@ class CumsumTest(test.TestCase):
       for axis in (-1, 0):
         self._compareAll(x, axis)
 
+  def testAxisType(self):
+    for dtype in self.valid_dtypes:
+      x = np.arange(1, 6).reshape([5]).astype(dtype)
+      for axis_dtype in [dtypes.int64, dtypes.int32]:
+        with self.test_session(use_gpu=True):
+          axis = constant_op.constant(0, axis_dtype)
+          tf_out = math_ops.cumsum(x, axis).eval()
+
   def test1D(self):
     for dtype in self.valid_dtypes:
       x = np.arange(1, 6).reshape([5]).astype(dtype)
@@ -190,6 +200,14 @@ class CumprodTest(test.TestCase):
       for axis in (-1, 0):
         self._compareAll(x, axis)
 
+  def testAxisType(self):
+    for dtype in self.valid_dtypes:
+      x = np.arange(1, 6).reshape([5]).astype(dtype)
+      for axis_dtype in [dtypes.int64, dtypes.int32]:
+        with self.test_session(use_gpu=True):
+          axis = constant_op.constant(0, axis_dtype)
+          tf_out = math_ops.cumprod(x, axis).eval()
+
   def test1D(self):
     for dtype in self.valid_dtypes:
       x = np.arange(1, 6).reshape([5]).astype(dtype)
diff --git a/tensorflow/python/kernel_tests/shape_ops_test.py b/tensorflow/python/kernel_tests/shape_ops_test.py
index 52cf904528..a9fc699b21 100644
--- a/tensorflow/python/kernel_tests/shape_ops_test.py
+++ b/tensorflow/python/kernel_tests/shape_ops_test.py
@@ -411,14 +411,16 @@ class TileTest(test.TestCase):
       self.assertEqual(7, result)
 
   def testSimple(self):
-    with self.test_session():
-      inp = np.random.rand(4, 1).astype(np.float32)
-      a = constant_op.constant(inp)
-      tiled = array_ops.tile(a, [1, 4])
-      result = tiled.eval()
-    self.assertEqual(result.shape, (4, 4))
-    self.assertEqual([4, 4], tiled.get_shape())
-    self.assertTrue((result == np.tile(inp, (1, 4))).all())
+    # multiples could be int32 or int64
+    for dtype in [dtypes.int32, dtypes.int64]:
+      with self.test_session(use_gpu=True):
+        inp = np.random.rand(4, 1).astype(np.float32)
+        a = constant_op.constant(inp)
+        tiled = array_ops.tile(a, constant_op.constant([1, 4], dtype=dtype))
+        result = tiled.eval()
+      self.assertEqual(result.shape, (4, 4))
+      self.assertEqual([4, 4], tiled.get_shape())
+      self.assertTrue((result == np.tile(inp, (1, 4))).all())
 
   def testIdentityTileAndGrad(self):
     with self.test_session():
diff --git a/tensorflow/python/kernel_tests/slice_op_test.py b/tensorflow/python/kernel_tests/slice_op_test.py
index f6997e9c61..f415d9e70d 100644
--- a/tensorflow/python/kernel_tests/slice_op_test.py
+++ b/tensorflow/python/kernel_tests/slice_op_test.py
@@ -217,6 +217,17 @@ class SliceTest(test.TestCase):
     self.assertEqual(expected_val.shape, slice_t.get_shape())
     self.assertEqual(expected_val.shape, slice2_t.get_shape())
 
+  def testPartialShapeInference(self):
+    z = array_ops.zeros((1, 2, 3))
+    self.assertAllEqual(z.get_shape().as_list(), [1, 2, 3])
+
+    m1 = array_ops.slice(z, [0, 0, 0], [-1, -1, -1])
+    self.assertAllEqual(m1.get_shape().as_list(), [1, 2, 3])
+
+    m2 = array_ops.slice(z, [0, 0, 0], [constant_op.constant(1) + 0, 2, -1])
+    self.assertAllEqual(m2.get_shape().as_list(), [None, 2, None])
+
+
   def _testGradientSlice(self, input_shape, slice_begin, slice_size):
     with self.test_session(use_gpu=True):
       num_inputs = np.prod(input_shape)
diff --git a/tensorflow/python/kernel_tests/transpose_op_test.py b/tensorflow/python/kernel_tests/transpose_op_test.py
index 3b352937c8..c551d9c3d0 100644
--- a/tensorflow/python/kernel_tests/transpose_op_test.py
+++ b/tensorflow/python/kernel_tests/transpose_op_test.py
@@ -317,6 +317,19 @@ class TransposeTest(test.TestCase):
         np.arange(0, 8).reshape([2, 4]).astype(np.float32),
         np.array([1, 0]).astype(np.int32))
 
+  def testPermType(self):
+    for perm_dtype in [np.int64, np.int32]:
+      x = np.arange(0, 8).reshape([2, 4]).astype(np.float32)
+      p = np.array([1, 0]).astype(perm_dtype)
+      np_ans = np.copy(x).transpose(p)
+      with self.test_session(use_gpu=True):
+        inx = ops.convert_to_tensor(x)
+        inp = constant_op.constant(p)
+        y = array_ops.transpose(inx, inp)
+        tf_ans = y.eval()
+        self.assertShapeEqual(np_ans, y)
+        self.assertAllEqual(np_ans, tf_ans)
+
   def testHalf(self):
     self._compare(np.arange(0, 21).reshape([3, 7]).astype(np.float16))
     self._compare(np.arange(0, 210).reshape([2, 3, 5, 7]).astype(np.float16))
diff --git a/tensorflow/python/ops/hidden_ops.txt b/tensorflow/python/ops/hidden_ops.txt
index fcd378e3c0..86bc038e86 100644
--- a/tensorflow/python/ops/hidden_ops.txt
+++ b/tensorflow/python/ops/hidden_ops.txt
@@ -43,6 +43,7 @@ UniformCandidateSampler
 GenerateVocabRemapping
 LoadAndRemapMatrix
 
+
 # control_flow_ops
 Switch
 Merge
@@ -241,6 +242,7 @@ TensorSummaryV2
 
 # math_ops
 Abs
+AccumulateNV2
 AddN
 All
 Any
@@ -257,6 +259,7 @@ ComplexAbs
 Conj
 FloorDiv
 FloorMod
+HistogramFixedWidth
 Max
 Mean
 Min
diff --git a/tensorflow/python/ops/histogram_ops.py b/tensorflow/python/ops/histogram_ops.py
index c2077d51af..51e4be9343 100644
--- a/tensorflow/python/ops/histogram_ops.py
+++ b/tensorflow/python/ops/histogram_ops.py
@@ -28,6 +28,7 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import clip_ops
+from tensorflow.python.ops import gen_math_ops
 from tensorflow.python.ops import math_ops
 
 
@@ -69,30 +70,6 @@ def histogram_fixed_width(values,
   ```
   """
   with ops.name_scope(name, 'histogram_fixed_width',
-                      [values, value_range, nbins]) as scope:
-    values = ops.convert_to_tensor(values, name='values')
-    values = array_ops.reshape(values, [-1])
-    value_range = ops.convert_to_tensor(value_range, name='value_range')
-    nbins = ops.convert_to_tensor(nbins, dtype=dtypes.int32, name='nbins')
-    nbins_float = math_ops.cast(nbins, values.dtype)
-
-    # Map tensor values that fall within value_range to [0, 1].
-    scaled_values = math_ops.truediv(values - value_range[0],
-                                     value_range[1] - value_range[0],
-                                     name='scaled_values')
-
-    # map tensor values within the open interval value_range to {0,.., nbins-1},
-    # values outside the open interval will be zero or less, or nbins or more.
-    indices = math_ops.floor(nbins_float * scaled_values, name='indices')
-
-    # Clip edge cases (e.g. value = value_range[1]) or "outliers."
-    indices = math_ops.cast(
-        clip_ops.clip_by_value(indices, 0, nbins_float - 1), dtypes.int32)
-
-    # TODO(langmore) This creates an array of ones to add up and place in the
-    # bins.  This is inefficient, so replace when a better Op is available.
-    return math_ops.unsorted_segment_sum(
-        array_ops.ones_like(indices, dtype=dtype),
-        indices,
-        nbins,
-        name=scope)
+                      [values, value_range, nbins]) as name:
+    return gen_math_ops._histogram_fixed_width(values, value_range, nbins,
+                                               dtype=dtype, name=name)
diff --git a/tensorflow/python/ops/histogram_ops_test.py b/tensorflow/python/ops/histogram_ops_test.py
index e819e0234d..bf6e0296f6 100644
--- a/tensorflow/python/ops/histogram_ops_test.py
+++ b/tensorflow/python/ops/histogram_ops_test.py
@@ -36,7 +36,7 @@ class HistogramFixedWidthTest(test.TestCase):
     value_range = [0.0, 5.0]
     values = []
     expected_bin_counts = [0, 0, 0, 0, 0]
-    with self.test_session():
+    with self.test_session(use_gpu=True):
       hist = histogram_ops.histogram_fixed_width(values, value_range, nbins=5)
       self.assertEqual(dtypes.int32, hist.dtype)
       self.assertAllClose(expected_bin_counts, hist.eval())
@@ -47,7 +47,7 @@ class HistogramFixedWidthTest(test.TestCase):
     value_range = [0.0, 5.0]
     values = [-1.0, 0.0, 1.5, 2.0, 5.0, 15]
     expected_bin_counts = [2, 1, 1, 0, 2]
-    with self.test_session():
+    with self.test_session(use_gpu=True):
       hist = histogram_ops.histogram_fixed_width(
           values, value_range, nbins=5, dtype=dtypes.int64)
       self.assertEqual(dtypes.int64, hist.dtype)
@@ -59,7 +59,7 @@ class HistogramFixedWidthTest(test.TestCase):
     value_range = np.float64([0.0, 5.0])
     values = np.float64([-1.0, 0.0, 1.5, 2.0, 5.0, 15])
     expected_bin_counts = [2, 1, 1, 0, 2]
-    with self.test_session():
+    with self.test_session(use_gpu=True):
       hist = histogram_ops.histogram_fixed_width(values, value_range, nbins=5)
       self.assertEqual(dtypes.int32, hist.dtype)
       self.assertAllClose(expected_bin_counts, hist.eval())
@@ -70,7 +70,7 @@ class HistogramFixedWidthTest(test.TestCase):
     value_range = [0.0, 5.0]
     values = [[-1.0, 0.0, 1.5], [2.0, 5.0, 15]]
     expected_bin_counts = [2, 1, 1, 0, 2]
-    with self.test_session():
+    with self.test_session(use_gpu=True):
       hist = histogram_ops.histogram_fixed_width(values, value_range, nbins=5)
       self.assertEqual(dtypes.int32, hist.dtype)
       self.assertAllClose(expected_bin_counts, hist.eval())
diff --git a/tensorflow/python/ops/image_ops_test.py b/tensorflow/python/ops/image_ops_test.py
index ebbf581204..d1554b399f 100644
--- a/tensorflow/python/ops/image_ops_test.py
+++ b/tensorflow/python/ops/image_ops_test.py
@@ -1374,6 +1374,25 @@ class PadToBoundingBoxTest(test_util.TensorFlowTestCase):
     y = image_ops.pad_to_bounding_box(image, 0, 0, height, width)
     self.assertEqual(y.get_shape().as_list(), post_shape)
 
+  def testInt64(self):
+    x = [1, 2, 3,
+         4, 5, 6,
+         7, 8, 9]
+    x_shape = [3, 3, 1]
+
+    y = [0, 0, 0,
+         1, 2, 3,
+         4, 5, 6,
+         7, 8, 9]
+    y_shape = [4, 3, 1]
+    x = np.array(x).reshape(x_shape)
+    y = np.array(y).reshape(y_shape)
+
+    i = constant_op.constant([1, 0, 4, 3], dtype=dtypes.int64)
+    y_tf = image_ops.pad_to_bounding_box(x, i[0], i[1], i[2], i[3])
+    with self.test_session(use_gpu=True):
+      self.assertAllClose(y, y_tf.eval())
+
   def testNoOp(self):
     x_shape = [10, 10, 10]
     x = np.random.uniform(size=x_shape)
@@ -1672,8 +1691,8 @@ class ResizeImagesTest(test_util.TensorFlowTestCase):
              image_ops.ResizeMethod.BICUBIC,
              image_ops.ResizeMethod.AREA]
 
-  TYPES = [np.uint8, np.int8, np.int16, np.int32, np.int64,
-           np.float32, np.float64]
+  TYPES = [np.uint8, np.int8, np.uint16, np.int16, np.int32, np.int64,
+           np.float16, np.float32, np.float64]
 
   def _assertShapeInference(self, pre_shape, size, post_shape):
     # Try single image resize
@@ -2434,9 +2453,13 @@ class JpegTest(test_util.TensorFlowTestCase):
         y, x, h, w = crop_window
         image1_crop = image_ops.crop_to_bounding_box(image1, y, x, h, w)
 
-        # Combined crop+decode.
+        # Combined decode+crop.
         image2 = image_ops.decode_and_crop_jpeg(jpeg0, crop_window)
 
+        # Combined decode+crop should have the same shape inference
+        self.assertAllEqual(image1_crop.get_shape().as_list(),
+                            image2.get_shape().as_list())
+
         # CropAndDecode should be equal to DecodeJpeg+Crop.
         image1_crop, image2 = sess.run([image1_crop, image2])
         self.assertAllEqual(image1_crop, image2)
diff --git a/tensorflow/python/ops/losses/losses_impl.py b/tensorflow/python/ops/losses/losses_impl.py
index 752d260fba..55a18d28ca 100644
--- a/tensorflow/python/ops/losses/losses_impl.py
+++ b/tensorflow/python/ops/losses/losses_impl.py
@@ -27,6 +27,7 @@ from tensorflow.python.ops import nn
 from tensorflow.python.ops import nn_ops
 from tensorflow.python.ops import weights_broadcast_ops
 from tensorflow.python.ops.losses import util
+from tensorflow.python.util.deprecation import deprecated_args
 
 
 class Reduction(object):
@@ -230,10 +231,12 @@ def absolute_difference(
         losses, weights, scope, loss_collection, reduction=reduction)
 
 
+@deprecated_args(None, "dim is deprecated, use axis instead", "dim")
 def cosine_distance(
-    labels, predictions, dim=None, weights=1.0, scope=None,
+    labels, predictions, axis=None, weights=1.0, scope=None,
     loss_collection=ops.GraphKeys.LOSSES,
-    reduction=Reduction.SUM_BY_NONZERO_WEIGHTS):
+    reduction=Reduction.SUM_BY_NONZERO_WEIGHTS,
+    dim=None):
   """Adds a cosine-distance loss to the training procedure.
 
   Note that the function assumes that `predictions` and `labels` are already
@@ -242,13 +245,14 @@ def cosine_distance(
   Args:
     labels: `Tensor` whose shape matches 'predictions'
     predictions: An arbitrary matrix.
-    dim: The dimension along which the cosine distance is computed.
+    axis: The dimension along which the cosine distance is computed.
     weights: Optional `Tensor` whose rank is either 0, or the same rank as
       `labels`, and must be broadcastable to `labels` (i.e., all dimensions must
       be either `1`, or the same as the corresponding `losses` dimension).
     scope: The scope for the operations performed in computing the loss.
     loss_collection: collection to which this loss will be added.
     reduction: Type of reduction to apply to loss.
+    dim: The old (deprecated) name for `axis`.
 
   Returns:
     Weighted loss float `Tensor`. If `reduction` is `NONE`, this has the same
@@ -256,10 +260,14 @@ def cosine_distance(
 
   Raises:
     ValueError: If `predictions` shape doesn't match `labels` shape, or
-      `dim`, `labels`, `predictions` or `weights` is `None`.
+      `axis`, `labels`, `predictions` or `weights` is `None`.
   """
-  if dim is None:
-    raise ValueError("`dim` cannot be None.")
+  if dim is not None:
+    if axis is not None:
+      raise ValueError("Cannot specify both 'axis' and 'dim'")
+    axis = dim
+  if axis is None and dim is None:
+    raise ValueError("You must specify 'axis'.")
   if labels is None:
     raise ValueError("labels must not be None.")
   if predictions is None:
@@ -271,7 +279,7 @@ def cosine_distance(
     predictions.get_shape().assert_is_compatible_with(labels.get_shape())
 
     radial_diffs = math_ops.multiply(predictions, labels)
-    losses = 1 - math_ops.reduce_sum(radial_diffs, axis=(dim,), keep_dims=True)
+    losses = 1 - math_ops.reduce_sum(radial_diffs, axis=(axis,), keep_dims=True)
     return compute_weighted_loss(
         losses, weights, scope, loss_collection, reduction=reduction)
 
diff --git a/tensorflow/python/ops/metrics_impl.py b/tensorflow/python/ops/metrics_impl.py
index 9273659a77..10ff4be2dd 100644
--- a/tensorflow/python/ops/metrics_impl.py
+++ b/tensorflow/python/ops/metrics_impl.py
@@ -949,6 +949,12 @@ def mean_iou(labels,
       cm_diag = math_ops.to_float(array_ops.diag_part(total_cm))
       denominator = sum_over_row + sum_over_col - cm_diag
 
+      # The mean is only computed over classes that appear in the
+      # label or prediction tensor. If the denominator is 0, we need to
+      # ignore the class.
+      num_valid_entries = math_ops.reduce_sum(math_ops.cast(
+          math_ops.not_equal(denominator, 0), dtype=dtypes.float32))
+
       # If the value of the denominator is 0, set it to 1 to avoid
       # zero division.
       denominator = array_ops.where(
@@ -956,7 +962,13 @@ def mean_iou(labels,
           denominator,
           array_ops.ones_like(denominator))
       iou = math_ops.div(cm_diag, denominator)
-      return math_ops.reduce_mean(iou, name=name)
+
+      # If the number of valid entries is 0 (no classes) we return 0.
+      result = array_ops.where(
+          math_ops.greater(num_valid_entries, 0),
+          math_ops.reduce_sum(iou, name=name) / num_valid_entries,
+          0)
+      return result
 
     mean_iou_v = compute_mean_iou('mean_iou')
 
diff --git a/tensorflow/python/ops/nn_grad.py b/tensorflow/python/ops/nn_grad.py
index af610d8fdb..557f39fb42 100644
--- a/tensorflow/python/ops/nn_grad.py
+++ b/tensorflow/python/ops/nn_grad.py
@@ -352,6 +352,13 @@ def _Relu6Grad(op, grad):
   return gen_nn_ops._relu6_grad(grad, op.outputs[0])  # pylint: disable=protected-access
 
 
+@ops.RegisterGradient("Relu6Grad")
+def _Relu6GradGrad(op, grad):
+  x = op.inputs[1]
+  return (gen_nn_ops._relu6_grad(grad, x), array_ops.zeros(
+      shape=array_ops.shape(x), dtype=x.dtype))
+
+
 @ops.RegisterGradient("Elu")
 def _EluGrad(op, grad):
   return gen_nn_ops._elu_grad(grad, op.outputs[0])
@@ -934,3 +941,32 @@ def _TopKGrad(op, grad, _):
                                  validate_indices=False),
       in_shape), array_ops.zeros(
           [], dtype=dtypes.int32)]
+
+
+@ops.RegisterGradient("NthElement")
+def _NthElementGrad(op, grad):
+  """Return the gradients for NthElement.
+
+  Args:
+    op: The NthElementOp for which we need to generate gradients.
+    grad: Tensor. The gradients passed to the NthElementOp
+
+  Returns:
+    A list of two tensors, the first being the gradient w.r.t. the input,
+    the second being the gradient w.r.t. the N (None).
+  """
+  input = op.inputs[0]
+  output = op.outputs[0]
+
+  # Compute the number of elements which equal to output in each reduction
+  # dimension. If there are multiple elements then the gradient will be
+  # divided between them.
+  indicators = math_ops.cast(
+      math_ops.equal(array_ops.expand_dims(output, -1), input),
+      grad.dtype)
+
+  grad = array_ops.expand_dims(grad, -1)
+  num_selected = array_ops.expand_dims(
+      math_ops.reduce_sum(indicators, -1), -1)
+
+  return [math_ops.div(indicators, num_selected) * grad, None]
diff --git a/tensorflow/python/ops/nn_grad_test.py b/tensorflow/python/ops/nn_grad_test.py
new file mode 100644
index 0000000000..f7541c0e89
--- /dev/null
+++ b/tensorflow/python/ops/nn_grad_test.py
@@ -0,0 +1,48 @@
+# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for Python ops defined in nn_grad.py."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.ops import gradient_checker
+from tensorflow.python.ops import gradients_impl
+from tensorflow.python.ops import nn_grad
+from tensorflow.python.ops import nn_ops
+from tensorflow.python.platform import test
+
+
+class Relu6OpTest(test.TestCase):
+  def testRelu6GradGrad(self):
+    inputs = constant_op.constant([[-2, -1, 1, 3], [5, 7, 8, 9]],
+                                  dtype=dtypes.float32)
+    x_init_value = np.array([[-3.5, -1.5, 2, 4], [4.5, 7.5, 8.5, 11]])
+    r = nn_ops.relu6(inputs)
+    r_g = gradients_impl.gradients(r, inputs)[0]
+    with self.test_session():
+      error = gradient_checker.compute_gradient_error(
+        inputs, inputs.get_shape().as_list(),
+        r_g, r_g.get_shape().as_list(),
+        x_init_value=x_init_value)
+      self.assertLess(error, 1e-4)
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/python/ops/nn_ops.py b/tensorflow/python/ops/nn_ops.py
index 5f82323bfc..a37b68c6fa 100644
--- a/tensorflow/python/ops/nn_ops.py
+++ b/tensorflow/python/ops/nn_ops.py
@@ -2140,6 +2140,34 @@ def top_k(input, k=1, sorted=True, name=None):
   return gen_nn_ops._top_kv2(input, k=k, sorted=sorted, name=name)
 
 
+def nth_element(input, n, reverse=False, name=None):
+  r"""Finds values of the `n`-th order statistic for the last dmension.
+
+  If the input is a vector (rank-1), finds the entries which is the nth-smallest
+  value in the vector and outputs their values as scalar tensor.
+
+  For matrices (resp. higher rank input), computes the entries which is the
+  nth-smallest value in each row (resp. vector along the last dimension). Thus,
+
+      values.shape = input.shape[:-1]
+
+  Args:
+    input: 1-D or higher `Tensor` with last dimension at least `n+1`.
+    n: A `Tensor` of type `int32`.
+      0-D. Position of sorted vector to select along the last dimension (along
+      each row for matrices). Valid range of n is `[0, input.shape[:-1])`
+    reverse: An optional `bool`. Defaults to `False`.
+      When set to True, find the nth-largest value in the vector and vice
+      versa.
+    name: A name for the operation (optional).
+
+  Returns:
+    A `Tensor`. Has the same type as `input`.
+    The `n`-th order statistic along each last dimensional slice.
+  """
+  return gen_nn_ops.nth_element(input, n, reverse=reverse, name=name)
+
+
 def conv1d(value, filters, stride, padding,
            use_cudnn_on_gpu=None, data_format=None,
            name=None):
diff --git a/tensorflow/python/platform/self_check.py b/tensorflow/python/platform/self_check.py
index 39d38d7bbc..966a094e55 100644
--- a/tensorflow/python/platform/self_check.py
+++ b/tensorflow/python/platform/self_check.py
@@ -21,7 +21,13 @@ from __future__ import print_function
 import os
 
 
-from tensorflow.python.platform import build_info
+try:
+  from tensorflow.python.platform import build_info
+except ImportError:
+  raise ImportError("Could not import tensorflow. Do not import tensorflow "
+                    "from its source directory; change directory to outside "
+                    "the TensorFlow source tree, and relaunch your Python "
+                    "interpreter from there.")
 
 
 def preload_check():
diff --git a/tensorflow/tools/api/golden/tensorflow.losses.pbtxt b/tensorflow/tools/api/golden/tensorflow.losses.pbtxt
index 79443839b9..c1d190ae11 100644
--- a/tensorflow/tools/api/golden/tensorflow.losses.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.losses.pbtxt
@@ -18,7 +18,7 @@ tf_module {
   }
   member_method {
     name: "cosine_distance"
-    argspec: "args=[\'labels\', \'predictions\', \'dim\', \'weights\', \'scope\', \'loss_collection\', \'reduction\'], varargs=None, keywords=None, defaults=[\'None\', \'1.0\', \'None\', \'losses\', \'weighted_sum_by_nonzero_weights\'], "
+    argspec: "args=[\'labels\', \'predictions\', \'axis\', \'weights\', \'scope\', \'loss_collection\', \'reduction\', \'dim\'], varargs=None, keywords=None, defaults=[\'None\', \'1.0\', \'None\', \'losses\', \'weighted_sum_by_nonzero_weights\', \'None\'], "
   }
   member_method {
     name: "get_losses"
diff --git a/tensorflow/tools/ci_build/Dockerfile.pi b/tensorflow/tools/ci_build/Dockerfile.pi
index 9d12ededb8..2fddd6a2c0 100644
--- a/tensorflow/tools/ci_build/Dockerfile.pi
+++ b/tensorflow/tools/ci_build/Dockerfile.pi
@@ -14,6 +14,9 @@ RUN /install/install_proto3.sh
 RUN /install/install_buildifier.sh
 RUN /install/install_auditwheel.sh
 RUN /install/install_golang.sh
+
+# The following line installs the Python cross-compilation toolchain. All the
+# preceding dependencies should be kept in sync with the main CPU docker file.
 RUN /install/install_pi_toolchain.sh
 
 # Set up the master bazelrc configuration file.
diff --git a/tensorflow/tools/ci_build/Dockerfile.pi-python3 b/tensorflow/tools/ci_build/Dockerfile.pi-python3
new file mode 100644
index 0000000000..18b131ea19
--- /dev/null
+++ b/tensorflow/tools/ci_build/Dockerfile.pi-python3
@@ -0,0 +1,23 @@
+FROM ubuntu:14.04
+
+MAINTAINER Jan Prach <jendap@google.com>
+
+# Copy and run the install scripts.
+COPY install/*.sh /install/
+RUN /install/install_bootstrap_deb_packages.sh
+RUN add-apt-repository -y ppa:openjdk-r/ppa && \
+    add-apt-repository -y ppa:george-edison55/cmake-3.x
+RUN /install/install_deb_packages.sh
+RUN /install/install_pip_packages.sh
+RUN /install/install_bazel.sh
+RUN /install/install_proto3.sh
+RUN /install/install_buildifier.sh
+RUN /install/install_auditwheel.sh
+RUN /install/install_golang.sh
+
+# The following line installs the Python cross-compilation toolchain. All the
+# preceding dependencies should be kept in sync with the main CPU docker file.
+RUN /install/install_pi_python3_toolchain.sh
+
+# Set up the master bazelrc configuration file.
+COPY install/.bazelrc /etc/bazel.bazelrc
diff --git a/tensorflow/tools/ci_build/README.md b/tensorflow/tools/ci_build/README.md
index ad83669950..acef833909 100644
--- a/tensorflow/tools/ci_build/README.md
+++ b/tensorflow/tools/ci_build/README.md
@@ -1,115 +1,76 @@
 # TensorFlow Builds
 
-This directory contains all the files and setup instructions to run all
-the important builds and tests. **You can trivially run it yourself!** It also
-run continuous integration [ci.tensorflow.org](https://ci.tensorflow.org).
-
-
+This directory contains all the files and setup instructions to run all the
+important builds and tests. You can run it yourself!
 
 ## Run It Yourself
 
-1. Install [Docker](http://www.docker.com/). Follow instructions
-   [on the Docker site](https://docs.docker.com/installation/).
-
-   You can run all the jobs **without docker** if you are on mac or on linux
-   and you just don't want docker. Just install all the dependencies from
-   [Installing TensorFlow](https://www.tensorflow.org/install/).
-   Then run any of the one liners below without the
-   `tensorflow/tools/ci_build/ci_build.sh` in them.
-
-2. Clone tensorflow repository.
-
-   ```bash
-   git clone https://github.com/tensorflow/tensorflow.git
-   ```
-
-3. Go to tensorflow directory
-
-   ```bash
-   cd tensorflow
-   ```
-
-4. Build what you want, for example
-
-   ```bash
-   tensorflow/tools/ci_build/ci_build.sh CPU bazel test //tensorflow/...
-   ```
-   If you are using the Docker image on Windows or OS X, the Docker VM's default
-   memory limit may be too low to build TensorFlow. This can result in
-   strange-looking errors, e.g. the compilation may fail with `gcc: internal
-   compiler error: Killed (program cc1plus)`. Try increasing the memory limit in
-   the Docker preferences.
-
-
-## Jobs
-
-The jobs run by [ci.tensorflow.org](https://ci.tensorflow.org) include following:
-
-```bash
-# Note: You can run the following one-liners yourself if you have Docker. Run
-# without `tensorflow/tools/ci_build/ci_build.sh` on mac or linux without Docker.
-
-# build and run cpu tests
-tensorflow/tools/ci_build/ci_build.sh CPU bazel test //tensorflow/...
+You have two options when running TensorFlow tests locally on your
+machine. First, using docker, you can run our Continuous Integration
+(CI) scripts on tensorflow devel images. The other option is to install
+all TensorFlow dependencies on your machine and run the scripts
+natively on your system.
 
-# build and run gpu tests (note if you get unstable results you may be running
-# out of gpu memory - if so add "--jobs=1" argument)
-tensorflow/tools/ci_build/ci_build.sh GPU bazel test -c opt --config=cuda //tensorflow/...
+### Run TensorFlow CI Scripts using Docker
 
-# build pip with gpu support
-tensorflow/tools/ci_build/ci_build.sh GPU tensorflow/tools/ci_build/builds/pip.sh GPU -c opt --config=cuda
+1.  Install Docker following the [instructions on the docker website](https://docs.docker.com/engine/installation/).
 
-# build and run gpu tests using python 3
-CI_DOCKER_EXTRA_PARAMS="-e CI_BUILD_PYTHON=python3" tensorflow/tools/ci_build/ci_build.sh GPU tensorflow/tools/ci_build/builds/pip.sh GPU -c opt --config=cuda
+2.  Start a container with one of the devel images here:
+    https://hub.docker.com/r/tensorflow/tensorflow/tags/.
 
-# build android example app
-tensorflow/tools/ci_build/ci_build.sh ANDROID tensorflow/tools/ci_build/builds/android.sh
+3.  Based on your choice of the image, pick one of the scripts under
+    https://github.com/tensorflow/tensorflow/tree/master/tensorflow/tools/ci_build/linux
+    and run them from the TensorFlow repository root.
 
-# cmake cpu build and test
-tensorflow/tools/ci_build/ci_build.sh CPU tensorflow/tools/ci_build/builds/cmake.sh
+### Run TensorFlow CI Scripts Natively on your Machine
 
-# run bash inside the container
-CI_DOCKER_EXTRA_PARAMS='-it --rm' tensorflow/tools/ci_build/ci_build.sh CPU /bin/bash
-```
+1.  Follow the instructions at https://www.tensorflow.org/install/install_sources,
+    but stop when you get to the section "Configure the installation". You do not
+    need to configure the installation to run the CI scripts.
 
-**Note**: The set of jobs and how they are triggered is still evolving.
-There are builds for master branch on cpu, gpu and android. There is a build
-for incoming gerrit changes. Gpu tests and benchmark are coming soon. Check
-[ci.tensorflow.org](https://ci.tensorflow.org) for current jobs.
+2.  Pick the appropriate OS and python version you have installed,
+    and run the script under tensorflow/tools/ci_build/<OS>.
 
+## TensorFlow Continuous Integration
 
+To verify that new changes don’t break TensorFlow, we run builds and
+tests on either [Jenkins](https://jenkins-ci.org/) or a CI system
+internal to Google.
 
-## How Does TensorFlow Continuous Integration Work
+We can trigger builds and tests on updates to master or on each pull
+request. Contact one of the repository maintainers to trigger builds
+on your pull request.
 
-We use [jenkins](https://jenkins-ci.org/) as our continuous integration.
-It is running at [ci.tensorflow.org](https://ci.tensorflow.org).
-All the jobs are run within [docker](http://www.docker.com/) containers.
+### View CI Results
 
-Builds can be triggered by push to master, push a change set or manually.
-The build started in jenkins will first pull the git tree. Then jenkins builds
-a docker container (using one of those Dockerfile.* files in this directory).
-The build itself is run within the container itself.
+The Pull Request will show if the change passed or failed the checks.
 
-Source tree lives in jenkins job workspace. Docker container for jenkins
-are transient - deleted after the build. Containers build very fast thanks
-to docker caching. Individual builds are fast thanks to bazel caching.
+From the pull request, click **Show all checks** to see the list of builds
+and tests. Click on **Details** to see the results from Jenkins or the internal
+CI system.
 
+Results from Jenkins are displayed in the Jenkins UI. For more information,
+see the [Jenkns documentation](https://jenkins.io/doc/).
 
+Results from the internal CI system are displayed in the Build Status UI. In
+this UI, to see the logs for a failed build:
 
-## Implementation Details
+*   Click on the **INVOCATION LOG** tab to see the invocation log.
 
-* The ci_build.sh script create and run docker container with all dependencies.
-  The builds/with_the_same_user together with ci_build.sh creates an environment
-  which is the same inside the container as it is outside. The same user, group,
-  path, so that docker symlinks work inside and outside the container. You can
-  use it for your development. Edit files in your git clone directory. If you
-  run the ci_build.sh it gets this directory mapped inside the container and
-  build your tree.
+*   Click on the **ARTIFACTS** tab to see a list of all artifacts, including logs.
 
-* The unusual `bazel-ci_build-cache` directory is mapped to docker container
-  performing the build using docker's --volume parameter. This way we cache
-  bazel output between builds.
+*   Individual test logs may be available. To see these logs, from the **TARGETS**
+    tab, click on the failed target. Then, click on the **TARGET LOG** tab to see
+    its test log.
 
-* The `builds` directory within this folder contains shell scripts to run within
-  the container. They essentially contains workarounds for current limitations
-  of bazel.
+    If you’re looking at target that is sharded or a test that is flaky, then
+    the build tool divided the target into multiple shards or ran the test
+    multiple times. Each test log is specific to the shard, run, and attempt.
+    To see a specific log:
+    
+    1.  Click on the log icon that is on the right next to the shard, run,
+        and attempt number.
+        
+    2.  In the grid that appears on the right, click on the specific shard,
+        run, and attempt to view its log. You can also type the desired shard,
+        run, or attempt number in the field above its grid.
diff --git a/tensorflow/tools/ci_build/builds/android_full.sh b/tensorflow/tools/ci_build/builds/android_full.sh
index 63250e0a4d..9d449241e8 100755
--- a/tensorflow/tools/ci_build/builds/android_full.sh
+++ b/tensorflow/tools/ci_build/builds/android_full.sh
@@ -40,7 +40,7 @@ rm -rf ${AAR_LIB_TMP}
 for CPU in ${CPUS//,/ }
 do
     echo "========== Building native libs for Android ${CPU} =========="
-    bazel build -c opt --cpu=${CPU} \
+    bazel build -c opt --config=monolithic --cpu=${CPU} \
         --crosstool_top=//external:android/crosstool \
         --host_crosstool_top=@bazel_tools//tools/cpp:toolchain \
         //tensorflow/core:android_tensorflow_lib \
@@ -62,7 +62,7 @@ done
 # in assets/ dir (see https://github.com/bazelbuild/bazel/issues/2334)
 # TODO(gunan): remove extra flags once sandboxing is enabled for all builds.
 echo "========== Building TensorFlow Android Jar and Demo =========="
-bazel --bazelrc=/dev/null build -c opt --fat_apk_cpu=${CPUS} \
+bazel --bazelrc=/dev/null build -c opt --config=monolithic --fat_apk_cpu=${CPUS} \
     --spawn_strategy=sandboxed --genrule_strategy=sandboxed \
     //tensorflow/contrib/android:android_tensorflow_inference_java \
     //tensorflow/contrib/android:android_tensorflow_inference_java.aar \
diff --git a/tensorflow/tools/ci_build/builds/libtensorflow.sh b/tensorflow/tools/ci_build/builds/libtensorflow.sh
index 5052d3626c..26713dded8 100755
--- a/tensorflow/tools/ci_build/builds/libtensorflow.sh
+++ b/tensorflow/tools/ci_build/builds/libtensorflow.sh
@@ -78,9 +78,52 @@ function build_libtensorflow_tarball() {
     //tensorflow/tools/lib_package:libtensorflow_proto.zip
 
   mkdir -p ${DIR}
+
   cp bazel-bin/tensorflow/tools/lib_package/libtensorflow.tar.gz ${DIR}/libtensorflow${TARBALL_SUFFIX}.tar.gz
   cp bazel-bin/tensorflow/tools/lib_package/libtensorflow_jni.tar.gz ${DIR}/libtensorflow_jni${TARBALL_SUFFIX}.tar.gz
-  cp bazel-bin/tensorflow/java/libtensorflow.jar bazel-bin/tensorflow/java/libtensorflow-src.jar ${DIR}
+  cp bazel-bin/tensorflow/java/libtensorflow.jar ${DIR}
+  cp_normalized_srcjar bazel-bin/tensorflow/java/libtensorflow-src.jar ${DIR}/libtensorflow-src.jar
   cp bazel-genfiles/tensorflow/tools/lib_package/libtensorflow_proto.zip ${DIR}
   chmod -x ${DIR}/*
 }
+
+# Helper function to copy a srcjar after moving any source files
+# directly under the root to the "maven-style" src/main/java layout
+#
+# Source files generated by annotation processors appear directly
+# under the root of srcjars jars created by bazel, rather than under
+# the maven-style src/main/java subdirectory.
+#
+# Bazel manages annotation generated source as follows: First, it
+# calls javac with options that create generated files under a
+# bazel-out directory. Next, it archives the generated source files
+# into a srcjar directly under the root. There doesn't appear to be a
+# simple way to parameterize this from bazel, hence this helper to
+# "normalize" the srcjar layout.
+#
+# Arguments:
+#   src_jar - path to the original srcjar
+#   dest_jar - path to the destination
+# Returns:
+#   None
+function cp_normalized_srcjar() {
+  local src_jar="$1"
+  local dest_jar="$2"
+  if [[ -z "${src_jar}" || -z "${dest_jar}" ]]; then
+    echo "Unexpected: missing arguments" >&2
+    exit 2
+  fi
+  local tmp_dir
+  tmp_dir=$(mktemp -d)
+  cp "${src_jar}" "${tmp_dir}/orig.jar"
+  pushd "${tmp_dir}"
+  # Extract any src/ files
+  jar -xf "${tmp_dir}/orig.jar" src/
+  # Extract any org/ files under src/main/java
+  (mkdir -p src/main/java && cd src/main/java && jar -xf "${tmp_dir}/orig.jar" org/)
+  # Repackage src/
+  jar -cMf "${tmp_dir}/new.jar" src
+  popd
+  cp "${tmp_dir}/new.jar" "${dest_jar}"
+  rm -rf "${tmp_dir}"
+}
diff --git a/tensorflow/tools/ci_build/install/install_golang.sh b/tensorflow/tools/ci_build/install/install_golang.sh
index 596265b069..55c1674495 100755
--- a/tensorflow/tools/ci_build/install/install_golang.sh
+++ b/tensorflow/tools/ci_build/install/install_golang.sh
@@ -16,7 +16,7 @@
 
 set -ex
 
-GOLANG_URL="https://storage.googleapis.com/golang/go1.9.linux-amd64.tar.gz"
+GOLANG_URL="https://storage.googleapis.com/golang/go1.9.1.linux-amd64.tar.gz"
 
 sudo mkdir -p /usr/local
 wget -q -O - "${GOLANG_URL}" | sudo tar -C /usr/local -xz
diff --git a/tensorflow/tools/ci_build/install/install_pi_python3_toolchain.sh b/tensorflow/tools/ci_build/install/install_pi_python3_toolchain.sh
new file mode 100755
index 0000000000..9d8e3df3b5
--- /dev/null
+++ b/tensorflow/tools/ci_build/install/install_pi_python3_toolchain.sh
@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+dpkg --add-architecture armhf
+echo 'deb [arch=armhf] http://ports.ubuntu.com/ trusty main restricted universe multiverse' >> /etc/apt/sources.list.d/armhf.list
+echo 'deb [arch=armhf] http://ports.ubuntu.com/ trusty-updates main restricted universe multiverse' >> /etc/apt/sources.list.d/armhf.list
+echo 'deb [arch=armhf] http://ports.ubuntu.com/ trusty-security main restricted universe multiverse' >> /etc/apt/sources.list.d/armhf.list
+echo 'deb [arch=armhf] http://ports.ubuntu.com/ trusty-backports main restricted universe multiverse' >> /etc/apt/sources.list.d/armhf.list
+sed -i 's#deb http://archive.ubuntu.com/ubuntu/#deb [arch=amd64] http://archive.ubuntu.com/ubuntu/#g' /etc/apt/sources.list
+apt-get update
+apt-get install -y libpython3-all-dev:armhf
+echo "deb [arch=amd64] http://storage.googleapis.com/bazel-apt stable jdk1.8" | sudo tee /etc/apt/sources.list.d/bazel.list
+curl https://bazel.build/bazel-release.pub.gpg | sudo apt-key add -
+apt-get update
+rm -rf /usr/local/bin/bazel
+apt-get install -y bazel python3 python3-numpy python3-dev python3-pip
diff --git a/tensorflow/tools/ci_build/install/install_pi_toolchain.sh b/tensorflow/tools/ci_build/install/install_pi_toolchain.sh
index ef30ba58c2..03c43cc838 100755
--- a/tensorflow/tools/ci_build/install/install_pi_toolchain.sh
+++ b/tensorflow/tools/ci_build/install/install_pi_toolchain.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
diff --git a/tensorflow/tools/ci_build/linux/cpu/run_mkl.sh b/tensorflow/tools/ci_build/linux/cpu/run_mkl.sh
new file mode 100755
index 0000000000..dbf376be6f
--- /dev/null
+++ b/tensorflow/tools/ci_build/linux/cpu/run_mkl.sh
@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# ==============================================================================
+
+set -e
+set -x
+
+N_JOBS=$(grep -c ^processor /proc/cpuinfo)
+
+echo ""
+echo "Bazel will use ${N_JOBS} concurrent job(s)."
+echo ""
+
+# Run configure.
+export TF_NEED_CUDA=0
+export PYTHON_BIN_PATH=`which python2`
+yes "" | $PYTHON_BIN_PATH configure.py
+
+# Run bazel test command. Double test timeouts to avoid flakes.
+bazel test --test_tag_filters=-no_oss,-oss_serial,-gpu,-benchmark-test --test_lang_filters=py -k \
+    --jobs=${N_JOBS} --test_timeout 300,450,1200,3600 --build_tests_only \
+    --config=mkl --config=opt --test_output=errors -- \
+    //tensorflow/... -//tensorflow/compiler/... -//tensorflow/contrib/...
diff --git a/tensorflow/tools/docker/Dockerfile b/tensorflow/tools/docker/Dockerfile
index 07a972400d..024cb40eb4 100644
--- a/tensorflow/tools/docker/Dockerfile
+++ b/tensorflow/tools/docker/Dockerfile
@@ -1,6 +1,6 @@
 FROM ubuntu:16.04
 
-MAINTAINER Craig Citro <craigcitro@google.com>
+LABEL maintainer="Craig Citro <craigcitro@google.com>"
 
 # Pick up some TF dependencies
 RUN apt-get update && apt-get install -y --no-install-recommends \
diff --git a/tensorflow/tools/docker/Dockerfile.devel b/tensorflow/tools/docker/Dockerfile.devel
index 60a94504b7..20e1dcd085 100644
--- a/tensorflow/tools/docker/Dockerfile.devel
+++ b/tensorflow/tools/docker/Dockerfile.devel
@@ -1,6 +1,6 @@
 FROM ubuntu:16.04
 
-MAINTAINER Craig Citro <craigcitro@google.com>
+LABEL maintainer="Craig Citro <craigcitro@google.com>"
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
         build-essential \
@@ -72,7 +72,7 @@ RUN mkdir /bazel && \
 
 RUN git clone https://github.com/tensorflow/tensorflow.git && \
     cd tensorflow && \
-    git checkout r1.3
+    git checkout r1.4
 WORKDIR /tensorflow
 
 # TODO(craigcitro): Don't install the pip package, since it makes it
diff --git a/tensorflow/tools/docker/Dockerfile.devel-gpu b/tensorflow/tools/docker/Dockerfile.devel-gpu
index 04773376e9..21a44ee404 100644
--- a/tensorflow/tools/docker/Dockerfile.devel-gpu
+++ b/tensorflow/tools/docker/Dockerfile.devel-gpu
@@ -1,6 +1,6 @@
 FROM nvidia/cuda:8.0-cudnn6-devel-ubuntu16.04
 
-MAINTAINER Craig Citro <craigcitro@google.com>
+LABEL maintainer="Craig Citro <craigcitro@google.com>"
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
         build-essential \
@@ -73,20 +73,23 @@ RUN mkdir /bazel && \
 
 RUN git clone https://github.com/tensorflow/tensorflow.git && \
     cd tensorflow && \
-    git checkout r1.3
+    git checkout r1.4
 WORKDIR /tensorflow
 
 # Configure the build for our CUDA configuration.
 ENV CI_BUILD_PYTHON python
-ENV LD_LIBRARY_PATH /usr/local/cuda/lib64/stubs:/usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH
+ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH
 ENV TF_NEED_CUDA 1
 ENV TF_CUDA_COMPUTE_CAPABILITIES=3.0,3.5,5.2,6.0,6.1
 
-RUN ln -s /usr/local/cuda/lib64/stubs/libcuda.so /usr/local/cuda/lib64/stubs/libcuda.so.1
 
-RUN tensorflow/tools/ci_build/builds/configured GPU \
-    bazel build -c opt --config=cuda --cxxopt="-D_GLIBCXX_USE_CXX11_ABI=0" \
+RUN ln -s /usr/local/cuda/lib64/stubs/libcuda.so /usr/local/cuda/lib64/stubs/libcuda.so.1 && \
+    LD_LIBRARY_PATH=/usr/local/cuda/lib64/stubs:${LD_LIBRARY_PATH} \
+    tensorflow/tools/ci_build/builds/configured GPU \
+    bazel build -c opt --config=cuda \
+	--cxxopt="-D_GLIBCXX_USE_CXX11_ABI=0" \
         tensorflow/tools/pip_package:build_pip_package && \
+    rm /usr/local/cuda/lib64/stubs/libcuda.so.1 && \
     bazel-bin/tensorflow/tools/pip_package/build_pip_package /tmp/pip && \
     pip --no-cache-dir install --upgrade /tmp/pip/tensorflow-*.whl && \
     rm -rf /tmp/pip && \
diff --git a/tensorflow/tools/docker/Dockerfile.devel-gpu-cuda9-cudnn7 b/tensorflow/tools/docker/Dockerfile.devel-gpu-cuda9-cudnn7
index ac1a437031..4558bc5293 100644
--- a/tensorflow/tools/docker/Dockerfile.devel-gpu-cuda9-cudnn7
+++ b/tensorflow/tools/docker/Dockerfile.devel-gpu-cuda9-cudnn7
@@ -42,6 +42,7 @@ RUN pip --no-cache-dir install \
         scipy \
         sklearn \
         pandas \
+        wheel \
         && \
     python -m ipykernel.kernelspec
 
@@ -80,22 +81,32 @@ RUN git clone https://github.com/tensorflow/tensorflow.git && \
 WORKDIR /tensorflow
 
 # Configure the build for our CUDA configuration.
-ENV CI_BUILD_PYTHON python
-ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH
-ENV TF_NEED_CUDA 1
-ENV TF_CUDA_COMPUTE_CAPABILITIES 3.0,3.5,5.2,6.0,6.1
-ENV TF_CUDA_VERSION 9.0
-ENV TF_CUDNN_VERSION 7.0
+ENV CI_BUILD_PYTHON=python \
+    LD_LIBRARY_PATH=/usr/local/cuda/extras/CUPTI/lib64:${LD_LIBRARY_PATH} \
+    CUDNN_INSTALL_PATH=/usr/lib/x86_64-linux-gnu \
+    PYTHON_BIN_PATH=/usr/bin/python \
+    PYTHON_LIB_PATH=/usr/local/lib/python2.7/dist-packages \
+    TF_NEED_CUDA=1 \
+    TF_CUDA_VERSION=9.0 \
+    TF_CUDA_COMPUTE_CAPABILITIES=3.0,3.5,5.2,6.0,6.1,7.0 \
+    TF_CUDNN_VERSION=7
 RUN ./configure
 
-RUN LD_LIBRARY_PATH=/usr/local/cuda/lib64/stubs:${LD_LIBRARY_PATH} \
-    bazel build -c opt --config=cuda --cxxopt="-D_GLIBCXX_USE_CXX11_ABI=0" \
-        --jobs=${TF_AVAILABLE_CPUS} \
-        tensorflow/tools/pip_package:build_pip_package && \
-    mkdir -p /pip_pkg && \
+# Build and Install TensorFlow.
+RUN ln -s /usr/local/cuda/lib64/stubs/libcuda.so /usr/local/cuda/lib64/stubs/libcuda.so.1 && \
+    LD_LIBRARY_PATH=/usr/local/cuda/lib64/stubs:${LD_LIBRARY_PATH} \
+    bazel build -c opt \
+                --config=cuda \
+                --cxxopt="-D_GLIBCXX_USE_CXX11_ABI=0" \
+                --jobs=${TF_AVAILABLE_CPUS} \
+                tensorflow/tools/pip_package:build_pip_package && \
+    mkdir /pip_pkg && \
     bazel-bin/tensorflow/tools/pip_package/build_pip_package /pip_pkg
 
+# Clean up pip wheel and Bazel cache when done.
 RUN pip --no-cache-dir install --upgrade /pip_pkg/tensorflow-*.whl && \
+    rm -rf /pip_pkg && \
+    rm -rf /root/.cache
 
 WORKDIR /root
 
diff --git a/tensorflow/tools/docker/Dockerfile.gpu b/tensorflow/tools/docker/Dockerfile.gpu
index da83a30058..0571dd7391 100644
--- a/tensorflow/tools/docker/Dockerfile.gpu
+++ b/tensorflow/tools/docker/Dockerfile.gpu
@@ -1,6 +1,6 @@
 FROM nvidia/cuda:8.0-cudnn6-devel-ubuntu16.04
 
-MAINTAINER Craig Citro <craigcitro@google.com>
+LABEL maintainer="Craig Citro <craigcitro@google.com>"
 
 # Pick up some TF dependencies
 RUN apt-get update && apt-get install -y --no-install-recommends \
diff --git a/tensorflow/tools/docker/README.md b/tensorflow/tools/docker/README.md
index 3780bde2be..2e5a0038ed 100644
--- a/tensorflow/tools/docker/README.md
+++ b/tensorflow/tools/docker/README.md
@@ -41,6 +41,7 @@ Note: If you would have a problem running nvidia-docker you may try the old meth
 we have used. But it is not recommended. If you find a bug in nvidia-docker, please report
 it there and try using nvidia-docker as described above.
 
+    $ # The old, not recommended way to run docker with gpu support: 
     $ export CUDA_SO=$(\ls /usr/lib/x86_64-linux-gnu/libcuda.* | xargs -I{} echo '-v {}:{}')
     $ export DEVICES=$(\ls /dev/nvidia* | xargs -I{} echo '--device {}:{}')
     $ docker run -it -p 8888:8888 $CUDA_SO $DEVICES gcr.io/tensorflow/tensorflow:latest-gpu
diff --git a/tensorflow/tools/pip_package/setup.py b/tensorflow/tools/pip_package/setup.py
index c05d39e942..4f0de8f768 100644
--- a/tensorflow/tools/pip_package/setup.py
+++ b/tensorflow/tools/pip_package/setup.py
@@ -29,14 +29,14 @@ from setuptools.dist import Distribution
 # This version string is semver compatible, but incompatible with pip.
 # For pip, we will remove all '-' characters from this string, and use the
 # result for pip.
-_VERSION = '1.3.0'
+_VERSION = '1.4.0-rc0'
 
 REQUIRED_PACKAGES = [
     'enum34 >= 1.1.6',
     'numpy >= 1.12.1',
     'six >= 1.10.0',
     'protobuf >= 3.4.0',
-    'tensorflow-tensorboard >= 0.1.0, < 0.2.0',
+    'tensorflow-tensorboard >= 0.4.0rc1, < 0.5.0',
 ]
 
 project_name = 'tensorflow'
@@ -67,6 +67,7 @@ if sys.version_info < (3, 4):
 
 # pylint: disable=line-too-long
 CONSOLE_SCRIPTS = [
+    'freeze_graph = tensorflow.python.tools.freeze_graph:main',
     'saved_model_cli = tensorflow.python.tools.saved_model_cli:main',
     # We need to keep the TensorBoard command, even though the console script
     # is now declared by the tensorboard pip package. If we remove the
diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 4d577fc246..a14469a0be 100644
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -590,16 +590,13 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       actual = "@jsoncpp_git//:jsoncpp",
   )
 
-  patched_http_archive(
+  native.http_archive(
       name = "boringssl",
       urls = [
-          "https://mirror.bazel.build/github.com/google/boringssl/archive/e3860009a091cd1bd2bc189cdbc3c6d095abde84.tar.gz",
-          # "https://github.com/google/boringssl/archive/e3860009a091cd1bd2bc189cdbc3c6d095abde84.tar.gz",  # 2017-07-07
+          "https://mirror.bazel.build/github.com/google/boringssl/archive/a0fb951d2a26a8ee746b52f3ba81ab011a0af778.tar.gz",
       ],
-      sha256 = "02f5950f93c4fd3691771c07c9d04cf2999ab01383ff99da345249e93b0fcfb2",
-      strip_prefix = "boringssl-e3860009a091cd1bd2bc189cdbc3c6d095abde84",
-      # Add patch to boringssl code to support s390x
-      patch_file = str(Label("//third_party/boringssl:add_boringssl_s390x.patch")),
+      sha256 = "524ba98a56300149696481b4cb9ddebd0c7b7ac9b9f6edee81da2d2d7e5d2bb3",
+      strip_prefix = "boringssl-a0fb951d2a26a8ee746b52f3ba81ab011a0af778",
   )
 
   native.new_http_archive(
@@ -701,6 +698,31 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       repository = tf_repo_name,
   )
 
+  java_import_external(
+      name = "com_google_testing_compile",
+      jar_sha256 = "edc180fdcd9f740240da1a7a45673f46f59c5578d8cd3fbc912161f74b5aebb8",
+      jar_urls = [
+          "http://mirror.bazel.build/repo1.maven.org/maven2/com/google/testing/compile/compile-testing/0.11/compile-testing-0.11.jar",
+          "http://repo1.maven.org/maven2/com/google/testing/compile/compile-testing/0.11/compile-testing-0.11.jar",
+          "http://maven.ibiblio.org/maven2/com/google/testing/compile/compile-testing/0.11/compile-testing-0.11.jar",
+      ],
+      licenses = ["notice"],  # New BSD License
+      testonly_ = True,
+      deps = ["@com_google_guava", "@com_google_truth"],
+  )
+
+  java_import_external(
+      name = "com_google_truth",
+      jar_sha256 = "032eddc69652b0a1f8d458f999b4a9534965c646b8b5de0eba48ee69407051df",
+      jar_urls = [
+          "http://mirror.bazel.build/repo1.maven.org/maven2/com/google/truth/truth/0.32/truth-0.32.jar",
+          "http://repo1.maven.org/maven2/com/google/truth/truth/0.32/truth-0.32.jar",
+      ],
+      licenses = ["notice"],  # Apache 2.0
+      testonly_ = True,
+      deps = ["@com_google_guava"],
+  )
+
   native.new_http_archive(
       name = "com_google_pprof",
       urls = [
@@ -715,11 +737,11 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   native.new_http_archive(
       name = "cub_archive",
       urls = [
-          "https://mirror.bazel.build/github.com/NVlabs/cub/archive/1.7.3.zip",
-          # "https://github.com/NVlabs/cub/archive/1.7.3.zip",
+          "https://mirror.bazel.build/github.com/NVlabs/cub/archive/1.7.4.zip",
+          # "https://github.com/NVlabs/cub/archive/1.7.4.zip",
       ],
-      sha256 = "b7ead9e291d34ffa8074243541c1380d63be63f88de23de8ee548db573b72ebe",
-      strip_prefix = "cub-1.7.3",
+      sha256 = "20a1a39fd97e5da7f40f5f2e7fd73fd2ea59f9dc4bb8a6c5f228aa543e727e31",
+      strip_prefix = "cub-1.7.4",
       build_file = str(Label("//third_party:cub.BUILD")),
   )
 
diff --git a/third_party/aws.BUILD b/third_party/aws.BUILD
index 38b7e0e543..9d8e7946cd 100644
--- a/third_party/aws.BUILD
+++ b/third_party/aws.BUILD
@@ -18,6 +18,7 @@ cc_library(
         "@%ws%//tensorflow:darwin": glob([
             "aws-cpp-sdk-core/source/platform/linux-shared/*.cpp",
         ]),
+        "//conditions:default": [],
     }) + glob([
         "aws-cpp-sdk-core/include/**/*.h",
         "aws-cpp-sdk-core/source/*.cpp",
diff --git a/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/PacketMathAVX2.h b/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/PacketMathAVX2.h
index 078be83e0d..c210b1712c 100644
--- a/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/PacketMathAVX2.h
+++ b/third_party/eigen3/unsupported/Eigen/CXX11/src/FixedPoint/PacketMathAVX2.h
@@ -1,6 +1,35 @@
 #ifndef THIRD_PARTY_EIGEN3_UNSUPPORTED_EIGEN_CXX11_SRC_FIXEDPOINT_PACKETMATHAVX2_H_
 #define THIRD_PARTY_EIGEN3_UNSUPPORTED_EIGEN_CXX11_SRC_FIXEDPOINT_PACKETMATHAVX2_H_
 
+#ifdef _MSC_VER
+
+#include <immintrin.h>
+#include <emmintrin.h>
+#include <smmintrin.h>
+
+#endif
+
+inline int _mm256_extract_epi16_N0(const __m256i X)
+{
+	return _mm_extract_epi16(_mm256_extractf128_si256(X, 0 >> 3), 0 % 8);
+}
+
+inline int _mm256_extract_epi16_N1(const __m256i X)
+{
+	return _mm_extract_epi16(_mm256_extractf128_si256(X, 1 >> 3), 1 % 8);
+}
+
+inline int _mm256_extract_epi8_N0(const __m256i X)
+{
+	return _mm_extract_epi8(_mm256_extractf128_si256((X), 0 >> 4), 0 % 16);
+}
+
+inline int _mm256_extract_epi8_N1(const __m256i X)
+{
+	return _mm_extract_epi8(_mm256_extractf128_si256((X), 1 >> 4), 1 % 16);
+}
+
+
 namespace Eigen {
 namespace internal {
 
@@ -271,15 +300,15 @@ EIGEN_STRONG_INLINE QInt32 pfirst<Packet8q32i>(const Packet8q32i& a) {
 }
 template <>
 EIGEN_STRONG_INLINE QInt16 pfirst<Packet16q16i>(const Packet16q16i& a) {
-  return _mm256_extract_epi16(a.val, 0);
+  return _mm256_extract_epi16_N0(a.val);
 }
 template <>
 EIGEN_STRONG_INLINE QUInt8 pfirst<Packet32q8u>(const Packet32q8u& a) {
-  return static_cast<uint8_t>(_mm256_extract_epi8(a.val, 0));
+  return static_cast<uint8_t>(_mm256_extract_epi8_N0(a.val));
 }
 template <>
 EIGEN_STRONG_INLINE QInt8 pfirst<Packet32q8i>(const Packet32q8i& a) {
-  return _mm256_extract_epi8(a.val, 0);
+  return _mm256_extract_epi8_N0(a.val);
 }
 
 // Initialize to constant value.
@@ -391,7 +420,7 @@ EIGEN_STRONG_INLINE QInt16 predux_min<Packet16q16i>(const Packet16q16i& a) {
   tmp =
       _mm256_min_epi16(tmp, _mm256_shuffle_epi32(tmp, _MM_SHUFFLE(1, 0, 3, 2)));
   tmp = _mm256_min_epi16(tmp, _mm256_shuffle_epi32(tmp, 1));
-  return std::min(_mm256_extract_epi16(tmp, 0), _mm256_extract_epi16(tmp, 1));
+  return std::min(_mm256_extract_epi16_N0(tmp), _mm256_extract_epi16_N1(tmp));
 }
 template <>
 EIGEN_STRONG_INLINE QInt16 predux_max<Packet16q16i>(const Packet16q16i& a) {
@@ -399,7 +428,7 @@ EIGEN_STRONG_INLINE QInt16 predux_max<Packet16q16i>(const Packet16q16i& a) {
   tmp =
       _mm256_max_epi16(tmp, _mm256_shuffle_epi32(tmp, _MM_SHUFFLE(1, 0, 3, 2)));
   tmp = _mm256_max_epi16(tmp, _mm256_shuffle_epi32(tmp, 1));
-  return std::max(_mm256_extract_epi16(tmp, 0), _mm256_extract_epi16(tmp, 1));
+  return std::max(_mm256_extract_epi16_N0(tmp), _mm256_extract_epi16_N1(tmp));
 }
 
 template <>
@@ -410,8 +439,8 @@ EIGEN_STRONG_INLINE QUInt8 predux_min<Packet32q8u>(const Packet32q8u& a) {
   tmp = _mm256_min_epu8(tmp, _mm256_shuffle_epi32(tmp, 1));
   tmp = _mm256_min_epu8(tmp,
                         _mm256_shufflelo_epi16(tmp, _MM_SHUFFLE(1, 0, 3, 2)));
-  return std::min(static_cast<uint8_t>(_mm256_extract_epi8(tmp, 0)),
-                  static_cast<uint8_t>(_mm256_extract_epi8(tmp, 1)));
+  return std::min(static_cast<uint8_t>(_mm256_extract_epi8_N0(tmp)),
+                  static_cast<uint8_t>(_mm256_extract_epi8_N1(tmp)));
 }
 template <>
 EIGEN_STRONG_INLINE QUInt8 predux_max<Packet32q8u>(const Packet32q8u& a) {
@@ -421,8 +450,8 @@ EIGEN_STRONG_INLINE QUInt8 predux_max<Packet32q8u>(const Packet32q8u& a) {
   tmp = _mm256_max_epu8(tmp, _mm256_shuffle_epi32(tmp, 1));
   tmp = _mm256_max_epu8(tmp,
                         _mm256_shufflelo_epi16(tmp, _MM_SHUFFLE(1, 0, 3, 2)));
-  return std::max(static_cast<uint8_t>(_mm256_extract_epi8(tmp, 0)),
-                  static_cast<uint8_t>(_mm256_extract_epi8(tmp, 1)));
+  return std::max(static_cast<uint8_t>(_mm256_extract_epi8_N0(tmp)),
+                  static_cast<uint8_t>(_mm256_extract_epi8_N1(tmp)));
 }
 
 template <>
@@ -431,7 +460,7 @@ EIGEN_STRONG_INLINE QInt8 predux_min<Packet32q8i>(const Packet32q8i& a) {
   tmp = _mm256_min_epi8(tmp, _mm256_shuffle_epi32(tmp, _MM_SHUFFLE(1, 0, 3, 2)));
   tmp = _mm256_min_epi8(tmp, _mm256_shuffle_epi32(tmp, 1));
   tmp = _mm256_min_epi8(tmp, _mm256_shufflelo_epi16(tmp, _MM_SHUFFLE(1, 0, 3, 2)));
-  return std::min(_mm256_extract_epi8(tmp, 0), _mm256_extract_epi8(tmp, 1));
+  return std::min(_mm256_extract_epi8_N0(tmp), _mm256_extract_epi8_N1(tmp));
 }
 template <>
 EIGEN_STRONG_INLINE QInt8 predux_max<Packet32q8i>(const Packet32q8i& a) {
@@ -439,7 +468,7 @@ EIGEN_STRONG_INLINE QInt8 predux_max<Packet32q8i>(const Packet32q8i& a) {
   tmp = _mm256_max_epi8(tmp, _mm256_shuffle_epi32(tmp, _MM_SHUFFLE(1, 0, 3, 2)));
   tmp = _mm256_max_epi8(tmp, _mm256_shuffle_epi32(tmp, 1));
   tmp = _mm256_max_epi8(tmp, _mm256_shufflelo_epi16(tmp, _MM_SHUFFLE(1, 0, 3, 2)));
-  return std::max(_mm256_extract_epi8(tmp, 0), _mm256_extract_epi8(tmp, 1));
+  return std::max(_mm256_extract_epi8_N0(tmp), _mm256_extract_epi8_N1(tmp));
 }
 
 // Vectorized scaling of Packet32q8i by float.
diff --git a/third_party/toolchains/cpus/arm/CROSSTOOL.tpl b/third_party/toolchains/cpus/arm/CROSSTOOL.tpl
index ad7f5596d0..f0e17d1fe0 100644
--- a/third_party/toolchains/cpus/arm/CROSSTOOL.tpl
+++ b/third_party/toolchains/cpus/arm/CROSSTOOL.tpl
@@ -87,7 +87,7 @@ toolchain {
   cxx_flag: "-isystem"
   cxx_flag: "/usr/include/arm-linux-gnueabihf"
   cxx_flag: "-isystem"
-  cxx_flag: "/usr/include/python2.7"
+  cxx_flag: "%{PYTHON_INCLUDE_PATH}%"
   cxx_flag: "-isystem"
   cxx_flag: "/usr/include/"
   linker_flag: "-lstdc++"
diff --git a/third_party/toolchains/cpus/arm/arm_compiler_configure.bzl b/third_party/toolchains/cpus/arm/arm_compiler_configure.bzl
index 5eb3b7bb1c..ab6eac115c 100644
--- a/third_party/toolchains/cpus/arm/arm_compiler_configure.bzl
+++ b/third_party/toolchains/cpus/arm/arm_compiler_configure.bzl
@@ -11,9 +11,20 @@ def _tpl(repository_ctx, tpl, substitutions={}, out=None):
 
 
 def _arm_compiler_configure_impl(repository_ctx):
+  # We need to find a cross-compilation include directory for Python, so look
+  # for an environment variable. Be warned, this crosstool template is only
+  # regenerated on the first run of Bazel, so if you change the variable after
+  # it may not be reflected in later builds. Doing a shutdown and clean of Bazel
+  # doesn't fix this, you'll need to delete the generated file at something like:
+  # external/local_config_arm_compiler/CROSSTOOL in your Bazel install.
+  if "CROSSTOOL_PYTHON_INCLUDE_PATH" in repository_ctx.os.environ:
+    python_include_path = repository_ctx.os.environ["CROSSTOOL_PYTHON_INCLUDE_PATH"]
+  else:
+    python_include_path = "/usr/include/python2.7"
   _tpl(repository_ctx, "CROSSTOOL", {
       "%{ARM_COMPILER_PATH}%": str(repository_ctx.path(
           repository_ctx.attr.remote_config_repo)),
+      "%{PYTHON_INCLUDE_PATH}%": python_include_path,
   })
   repository_ctx.symlink(repository_ctx.attr.build_file, "BUILD")
 
-- 
GitLab


From 3a103d98d5830de5bde7c0713a247137f7f37804 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 24 Oct 2017 19:49:49 -0700
Subject: [PATCH 746/909] Update ops-related pbtxt files.

PiperOrigin-RevId: 173347713
---
 .../core/ops/compat/ops_history.v1.pbtxt      | 400 ++++++++++++++++++
 tensorflow/core/ops/ops.pbtxt                 | 171 +++++++-
 2 files changed, 563 insertions(+), 8 deletions(-)

diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index 92037c1997..e6aeb35e02 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -39,6 +39,54 @@ op {
     }
   }
 }
+op {
+  name: "AccumulateNV2"
+  input_arg {
+    name: "inputs"
+    type_attr: "T"
+    number_attr: "N"
+  }
+  output_arg {
+    name: "sum"
+    type_attr: "T"
+  }
+  attr {
+    name: "N"
+    type: "int"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+  attr {
+    name: "shape"
+    type: "shape"
+  }
+  is_aggregate: true
+  is_commutative: true
+}
 op {
   name: "AccumulatorApplyGradient"
   input_arg {
@@ -7653,6 +7701,65 @@ op {
     }
   }
 }
+op {
+  name: "CropAndResize"
+  input_arg {
+    name: "image"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "boxes"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "box_ind"
+    type: DT_INT32
+  }
+  input_arg {
+    name: "crop_size"
+    type: DT_INT32
+  }
+  output_arg {
+    name: "crops"
+    type: DT_FLOAT
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT8
+        type: DT_INT16
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_HALF
+        type: DT_FLOAT
+        type: DT_DOUBLE
+      }
+    }
+  }
+  attr {
+    name: "method"
+    type: "string"
+    default_value {
+      s: "bilinear"
+    }
+    allowed_values {
+      list {
+        s: "bilinear"
+      }
+    }
+  }
+  attr {
+    name: "extrapolation_value"
+    type: "float"
+    default_value {
+      f: 0
+    }
+  }
+}
 op {
   name: "CropAndResizeGradBoxes"
   input_arg {
@@ -7704,6 +7811,58 @@ op {
     }
   }
 }
+op {
+  name: "CropAndResizeGradBoxes"
+  input_arg {
+    name: "grads"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "image"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "boxes"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "box_ind"
+    type: DT_INT32
+  }
+  output_arg {
+    name: "output"
+    type: DT_FLOAT
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT8
+        type: DT_INT16
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_HALF
+        type: DT_FLOAT
+        type: DT_DOUBLE
+      }
+    }
+  }
+  attr {
+    name: "method"
+    type: "string"
+    default_value {
+      s: "bilinear"
+    }
+    allowed_values {
+      list {
+        s: "bilinear"
+      }
+    }
+  }
+}
 op {
   name: "CropAndResizeGradImage"
   input_arg {
@@ -13533,6 +13692,50 @@ op {
   }
   is_stateful: true
 }
+op {
+  name: "HistogramFixedWidth"
+  input_arg {
+    name: "values"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "value_range"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "nbins"
+    type: DT_INT32
+  }
+  output_arg {
+    name: "out"
+    type_attr: "dtype"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_FLOAT
+        type: DT_DOUBLE
+      }
+    }
+  }
+  attr {
+    name: "dtype"
+    type: "type"
+    default_value {
+      type: DT_INT32
+    }
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+}
 op {
   name: "HistogramSummary"
   input_arg {
@@ -19710,6 +19913,47 @@ op {
   }
   is_commutative: true
 }
+op {
+  name: "NthElement"
+  input_arg {
+    name: "input"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "n"
+    type: DT_INT32
+  }
+  output_arg {
+    name: "values"
+    type_attr: "T"
+  }
+  attr {
+    name: "reverse"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_UINT16
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+}
 op {
   name: "OneHot"
   input_arg {
@@ -25190,6 +25434,45 @@ op {
     }
   }
 }
+op {
+  name: "ResizeArea"
+  input_arg {
+    name: "images"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "size"
+    type: DT_INT32
+  }
+  output_arg {
+    name: "resized_images"
+    type: DT_FLOAT
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT8
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_UINT16
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_HALF
+        type: DT_FLOAT
+        type: DT_DOUBLE
+      }
+    }
+  }
+  attr {
+    name: "align_corners"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
+}
 op {
   name: "ResizeBicubic"
   input_arg {
@@ -25228,6 +25511,45 @@ op {
     }
   }
 }
+op {
+  name: "ResizeBicubic"
+  input_arg {
+    name: "images"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "size"
+    type: DT_INT32
+  }
+  output_arg {
+    name: "resized_images"
+    type: DT_FLOAT
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT8
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_UINT16
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_HALF
+        type: DT_FLOAT
+        type: DT_DOUBLE
+      }
+    }
+  }
+  attr {
+    name: "align_corners"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
+}
 op {
   name: "ResizeBicubicGrad"
   input_arg {
@@ -25298,6 +25620,45 @@ op {
     }
   }
 }
+op {
+  name: "ResizeBilinear"
+  input_arg {
+    name: "images"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "size"
+    type: DT_INT32
+  }
+  output_arg {
+    name: "resized_images"
+    type: DT_FLOAT
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT8
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_UINT16
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_HALF
+        type: DT_FLOAT
+        type: DT_DOUBLE
+      }
+    }
+  }
+  attr {
+    name: "align_corners"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
+}
 op {
   name: "ResizeBilinearGrad"
   input_arg {
@@ -25369,6 +25730,45 @@ op {
     }
   }
 }
+op {
+  name: "ResizeNearestNeighbor"
+  input_arg {
+    name: "images"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "size"
+    type: DT_INT32
+  }
+  output_arg {
+    name: "resized_images"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT8
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_UINT16
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_HALF
+        type: DT_FLOAT
+        type: DT_DOUBLE
+      }
+    }
+  }
+  attr {
+    name: "align_corners"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
+}
 op {
   name: "ResizeNearestNeighborGrad"
   input_arg {
diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index c037c99c19..a6886e465d 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -44,6 +44,58 @@ op {
   summary: "Computes the absolute value of a tensor."
   description: "Given a tensor `x`, this operation returns a tensor containing the absolute\nvalue of each element in `x`. For example, if x is an input element and y is\nan output element, this operation computes \\\\(y = |x|\\\\)."
 }
+op {
+  name: "AccumulateNV2"
+  input_arg {
+    name: "inputs"
+    description: "A list of `Tensor` objects, each with same shape and type."
+    type_attr: "T"
+    number_attr: "N"
+  }
+  output_arg {
+    name: "sum"
+    type_attr: "T"
+  }
+  attr {
+    name: "N"
+    type: "int"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT64
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_UINT16
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_COMPLEX128
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+  attr {
+    name: "shape"
+    type: "shape"
+    description: "Shape of elements of `inputs`."
+  }
+  summary: "Returns the element-wise sum of a list of tensors."
+  description: "`tf.accumulate_n_v2` performs the same operation as `tf.add_n`, but does not\nwait for all of its inputs to be ready before beginning to sum. This can\nsave memory if inputs are ready at different times, since minimum temporary\nstorage is proportional to the output size rather than the inputs size.\n\nUnlike the original `accumulate_n`, `accumulate_n_v2` is differentiable.\n\nReturns a `Tensor` of same shape and type as the elements of `inputs`."
+  is_aggregate: true
+  is_commutative: true
+}
 op {
   name: "AccumulatorApplyGradient"
   input_arg {
@@ -5639,6 +5691,7 @@ op {
     allowed_values {
       list {
         type: DT_UINT8
+        type: DT_UINT16
         type: DT_INT8
         type: DT_INT16
         type: DT_INT32
@@ -5706,6 +5759,7 @@ op {
     allowed_values {
       list {
         type: DT_UINT8
+        type: DT_UINT16
         type: DT_INT8
         type: DT_INT16
         type: DT_INT32
@@ -7086,7 +7140,7 @@ op {
   name: "Diag"
   input_arg {
     name: "diagonal"
-    description: "Rank k tensor where k is at most 3."
+    description: "Rank k tensor where k is at most 1."
     type_attr: "T"
   }
   output_arg {
@@ -7114,7 +7168,7 @@ op {
   name: "DiagPart"
   input_arg {
     name: "input"
-    description: "Rank k tensor where k is 2, 4, or 6."
+    description: "Rank k tensor where k is even and not zero."
     type_attr: "T"
   }
   output_arg {
@@ -10300,6 +10354,56 @@ op {
   description: "This op creates a hash table, specifying the type of its keys and values.\nBefore using the table you will have to initialize it.  After initialization the\ntable will be immutable."
   is_stateful: true
 }
+op {
+  name: "HistogramFixedWidth"
+  input_arg {
+    name: "values"
+    description: "Numeric `Tensor`."
+    type_attr: "T"
+  }
+  input_arg {
+    name: "value_range"
+    description: "Shape [2] `Tensor` of same `dtype` as `values`.\nvalues <= value_range[0] will be mapped to hist[0],\nvalues >= value_range[1] will be mapped to hist[-1]."
+    type_attr: "T"
+  }
+  input_arg {
+    name: "nbins"
+    description: "Scalar `int32 Tensor`.  Number of histogram bins."
+    type: DT_INT32
+  }
+  output_arg {
+    name: "out"
+    description: "A 1-D `Tensor` holding histogram of values."
+    type_attr: "dtype"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_FLOAT
+        type: DT_DOUBLE
+      }
+    }
+  }
+  attr {
+    name: "dtype"
+    type: "type"
+    default_value {
+      type: DT_INT32
+    }
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+  summary: "Return histogram of values."
+  description: "Given the tensor `values`, this operation returns a rank 1 histogram counting\nthe number of entries in `values` that fall into every bin.  The bins are\nequal width and determined by the arguments `value_range` and `nbins`.\n\n```python\n# Bins will be:  (-inf, 1), [1, 2), [2, 3), [3, 4), [4, inf)\nnbins = 5\nvalue_range = [0.0, 5.0]\nnew_values = [-1.0, 0.0, 1.5, 2.0, 5.0, 15]\n\nwith tf.get_default_session() as sess:\n  hist = tf.histogram_fixed_width(new_values, value_range, nbins=5)\n  variables.global_variables_initializer().run()\n  sess.run(hist) => [2, 1, 1, 0, 2]\n```"
+}
 op {
   name: "HistogramSummary"
   input_arg {
@@ -15237,6 +15341,53 @@ op {
   description: "*NOTE*: `NotEqual` supports broadcasting. More about broadcasting\n[here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)"
   is_commutative: true
 }
+op {
+  name: "NthElement"
+  input_arg {
+    name: "input"
+    description: "1-D or higher with last dimension at least `n+1`."
+    type_attr: "T"
+  }
+  input_arg {
+    name: "n"
+    description: "0-D. Position of sorted vector to select along the last dimension (along\neach row for matrices). Valid range of n is `[0, input.shape[:-1])`"
+    type: DT_INT32
+  }
+  output_arg {
+    name: "values"
+    description: "The `n`-th order statistic along each last dimensional slice."
+    type_attr: "T"
+  }
+  attr {
+    name: "reverse"
+    type: "bool"
+    default_value {
+      b: false
+    }
+    description: "When set to True, find the nth-largest value in the vector and vice\nversa."
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_INT64
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_UINT16
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+  summary: "Finds values of the `n`-th order statistic for the last dmension."
+  description: "If the input is a vector (rank-1), finds the entries which is the nth-smallest\nvalue in the vector and outputs their values as scalar tensor.\n\nFor matrices (resp. higher rank input), computes the entries which is the\nnth-smallest value in each row (resp. vector along the last dimension). Thus,\n\n    values.shape = input.shape[:-1]"
+}
 op {
   name: "OneHot"
   input_arg {
@@ -20814,9 +20965,10 @@ op {
     type: "type"
     allowed_values {
       list {
-        type: DT_UINT8
         type: DT_INT8
+        type: DT_UINT8
         type: DT_INT16
+        type: DT_UINT16
         type: DT_INT32
         type: DT_INT64
         type: DT_HALF
@@ -20858,9 +21010,10 @@ op {
     type: "type"
     allowed_values {
       list {
-        type: DT_UINT8
         type: DT_INT8
+        type: DT_UINT8
         type: DT_INT16
+        type: DT_UINT16
         type: DT_INT32
         type: DT_INT64
         type: DT_HALF
@@ -20939,9 +21092,10 @@ op {
     type: "type"
     allowed_values {
       list {
-        type: DT_UINT8
         type: DT_INT8
+        type: DT_UINT8
         type: DT_INT16
+        type: DT_UINT16
         type: DT_INT32
         type: DT_INT64
         type: DT_HALF
@@ -21021,9 +21175,10 @@ op {
     type: "type"
     allowed_values {
       list {
-        type: DT_UINT8
         type: DT_INT8
+        type: DT_UINT8
         type: DT_INT16
+        type: DT_UINT16
         type: DT_INT32
         type: DT_INT64
         type: DT_HALF
@@ -23255,7 +23410,7 @@ op {
     }
   }
   summary: "Reverses specific dimensions of a tensor."
-  description: "NOTE `tf.reverse` has now changed behavior in preparation for 1.0.\n`tf.reverse_v2` is currently an alias that will be deprecated before TF 1.0.\n\nGiven a `tensor`, and a `int32` tensor `axis` representing the set of\ndimensions of `tensor` to reverse. This operation reverses each dimension\n`i` for which there exists `j` s.t. `axis[j] == i`.\n\n`tensor` can have up to 8 dimensions. The number of dimensions specified\nin `axis` may be 0 or more entries. If an index is specified more than\nonce, a InvalidArgument error is raised.\n\nFor example:\n\n```\n# tensor \'t\' is [[[[ 0,  1,  2,  3],\n#                  [ 4,  5,  6,  7],\n#                  [ 8,  9, 10, 11]],\n#                 [[12, 13, 14, 15],\n#                  [16, 17, 18, 19],\n#                  [20, 21, 22, 23]]]]\n# tensor \'t\' shape is [1, 2, 3, 4]\n\n# \'dims\' is [3] or \'dims\' is -1\nreverse(t, dims) ==> [[[[ 3,  2,  1,  0],\n                        [ 7,  6,  5,  4],\n                        [ 11, 10, 9, 8]],\n                       [[15, 14, 13, 12],\n                        [19, 18, 17, 16],\n                        [23, 22, 21, 20]]]]\n\n# \'dims\' is \'[1]\' (or \'dims\' is \'[-3]\')\nreverse(t, dims) ==> [[[[12, 13, 14, 15],\n                        [16, 17, 18, 19],\n                        [20, 21, 22, 23]\n                       [[ 0,  1,  2,  3],\n                        [ 4,  5,  6,  7],\n                        [ 8,  9, 10, 11]]]]\n\n# \'dims\' is \'[2]\' (or \'dims\' is \'[-2]\')\nreverse(t, dims) ==> [[[[8, 9, 10, 11],\n                        [4, 5, 6, 7],\n                        [0, 1, 2, 3]]\n                       [[20, 21, 22, 23],\n                        [16, 17, 18, 19],\n                        [12, 13, 14, 15]]]]\n```"
+  description: "NOTE `tf.reverse` has now changed behavior in preparation for 1.0.\n`tf.reverse_v2` is currently an alias that will be deprecated before TF 1.0.\n\nGiven a `tensor`, and a `int32` tensor `axis` representing the set of\ndimensions of `tensor` to reverse. This operation reverses each dimension\n`i` for which there exists `j` s.t. `axis[j] == i`.\n\n`tensor` can have up to 8 dimensions. The number of dimensions specified\nin `axis` may be 0 or more entries. If an index is specified more than\nonce, a InvalidArgument error is raised.\n\nFor example:\n\n```\n# tensor \'t\' is [[[[ 0,  1,  2,  3],\n#                  [ 4,  5,  6,  7],\n#                  [ 8,  9, 10, 11]],\n#                 [[12, 13, 14, 15],\n#                  [16, 17, 18, 19],\n#                  [20, 21, 22, 23]]]]\n# tensor \'t\' shape is [1, 2, 3, 4]\n\n# \'dims\' is [3] or \'dims\' is [-1]\nreverse(t, dims) ==> [[[[ 3,  2,  1,  0],\n                        [ 7,  6,  5,  4],\n                        [ 11, 10, 9, 8]],\n                       [[15, 14, 13, 12],\n                        [19, 18, 17, 16],\n                        [23, 22, 21, 20]]]]\n\n# \'dims\' is \'[1]\' (or \'dims\' is \'[-3]\')\nreverse(t, dims) ==> [[[[12, 13, 14, 15],\n                        [16, 17, 18, 19],\n                        [20, 21, 22, 23]\n                       [[ 0,  1,  2,  3],\n                        [ 4,  5,  6,  7],\n                        [ 8,  9, 10, 11]]]]\n\n# \'dims\' is \'[2]\' (or \'dims\' is \'[-2]\')\nreverse(t, dims) ==> [[[[8, 9, 10, 11],\n                        [4, 5, 6, 7],\n                        [0, 1, 2, 3]]\n                       [[20, 21, 22, 23],\n                        [16, 17, 18, 19],\n                        [12, 13, 14, 15]]]]\n```"
 }
 op {
   name: "RightShift"
@@ -32065,7 +32220,7 @@ op {
     }
   }
   summary: "Returns x / y element-wise for integer types."
-  description: "Truncation designates that negative numbers will round fractional quantities\ntoward zero. I.e. -7 / 5 = 1. This matches C semantics but it is different\nthan Python semantics. See `FloorDiv` for a division function that matches\nPython Semantics.\n\n*NOTE*: `TruncateDiv` supports broadcasting. More about broadcasting\n[here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)"
+  description: "Truncation designates that negative numbers will round fractional quantities\ntoward zero. I.e. -7 / 5 = -1. This matches C semantics but it is different\nthan Python semantics. See `FloorDiv` for a division function that matches\nPython Semantics.\n\n*NOTE*: `TruncateDiv` supports broadcasting. More about broadcasting\n[here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)"
 }
 op {
   name: "TruncateMod"
-- 
GitLab


From 16953025097793d9748099ebf4296edca04a5366 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 24 Oct 2017 19:54:58 -0700
Subject: [PATCH 747/909] Go: Update generated wrapper functions for TensorFlow
 ops.

PiperOrigin-RevId: 173347919
---
 tensorflow/go/op/wrappers.go | 491 ++++++++++++++++++++++-------------
 1 file changed, 313 insertions(+), 178 deletions(-)

diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go
index b3b317013f..958ce6d040 100644
--- a/tensorflow/go/op/wrappers.go
+++ b/tensorflow/go/op/wrappers.go
@@ -1849,7 +1849,7 @@ func MatrixSetDiag(scope *Scope, input tf.Output, diagonal tf.Output) (output tf
 // ```
 //
 // Arguments:
-//	input: Rank k tensor where k is 2, 4, or 6.
+//	input: Rank k tensor where k is even and not zero.
 //
 // Returns The extracted diagonal.
 func DiagPart(scope *Scope, input tf.Output) (diagonal tf.Output) {
@@ -4140,7 +4140,7 @@ func TensorArraySplitV3(scope *Scope, handle tf.Output, value tf.Output, lengths
 // ```
 //
 // Arguments:
-//	diagonal: Rank k tensor where k is at most 3.
+//	diagonal: Rank k tensor where k is at most 1.
 func Diag(scope *Scope, diagonal tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
@@ -7833,6 +7833,113 @@ func MatrixSolveLs(scope *Scope, matrix tf.Output, rhs tf.Output, l2_regularizer
 	return op.Output(0)
 }
 
+// Adjust the saturation of one or more images.
+//
+// `images` is a tensor of at least 3 dimensions.  The last dimension is
+// interpretted as channels, and must be three.
+//
+// The input image is considered in the RGB colorspace. Conceptually, the RGB
+// colors are first mapped into HSV. A scale is then applied all the saturation
+// values, and then remapped back to RGB colorspace.
+//
+// Arguments:
+//	images: Images to adjust.  At least 3-D.
+//	scale: A float scale to add to the saturation.
+//
+// Returns The hue-adjusted image or images.
+func AdjustSaturation(scope *Scope, images tf.Output, scale tf.Output) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "AdjustSaturation",
+		Input: []tf.Input{
+			images, scale,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// SelfAdjointEigV2Attr is an optional argument to SelfAdjointEigV2.
+type SelfAdjointEigV2Attr func(optionalAttr)
+
+// SelfAdjointEigV2ComputeV sets the optional compute_v attribute to value.
+//
+// value: If `True` then eigenvectors will be computed and returned in `v`.
+// Otherwise, only the eigenvalues will be computed.
+// If not specified, defaults to true
+func SelfAdjointEigV2ComputeV(value bool) SelfAdjointEigV2Attr {
+	return func(m optionalAttr) {
+		m["compute_v"] = value
+	}
+}
+
+// Computes the eigen decomposition of one or more square self-adjoint matrices.
+//
+// Computes the eigenvalues and (optionally) eigenvectors of each inner matrix in
+// `input` such that `input[..., :, :] = v[..., :, :] * diag(e[..., :])`.
+//
+// ```python
+// # a is a tensor.
+// # e is a tensor of eigenvalues.
+// # v is a tensor of eigenvectors.
+// e, v = self_adjoint_eig(a)
+// e = self_adjoint_eig(a, compute_v=False)
+// ```
+//
+// Arguments:
+//	input: `Tensor` input of shape `[N, N]`.
+//
+// Returns Eigenvalues. Shape is `[N]`.Eigenvectors. Shape is `[N, N]`.
+func SelfAdjointEigV2(scope *Scope, input tf.Output, optional ...SelfAdjointEigV2Attr) (e tf.Output, v tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "SelfAdjointEigV2",
+		Input: []tf.Input{
+			input,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1)
+}
+
+// Computes the Eigen Decomposition of a batch of square self-adjoint matrices.
+//
+// DEPRECATED at GraphDef version 11: Use SelfAdjointEigV2 instead.
+//
+// The input is a tensor of shape `[..., M, M]` whose inner-most 2 dimensions
+// form square matrices, with the same constraints as the single matrix
+// SelfAdjointEig.
+//
+// The result is a [..., M+1, M] matrix with [..., 0,:] containing the
+// eigenvalues, and subsequent [...,1:, :] containing the eigenvectors.
+//
+// Arguments:
+//	input: Shape is `[..., M, M]`.
+//
+// Returns Shape is `[..., M+1, M]`.
+func SelfAdjointEig(scope *Scope, input tf.Output) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "SelfAdjointEig",
+		Input: []tf.Input{
+			input,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // FusedBatchNormGradAttr is an optional argument to FusedBatchNormGrad.
 type FusedBatchNormGradAttr func(optionalAttr)
 
@@ -10567,6 +10674,36 @@ func RandomGamma(scope *Scope, shape tf.Output, alpha tf.Output, optional ...Ran
 	return op.Output(0)
 }
 
+// Returns the element-wise sum of a list of tensors.
+//
+// `tf.accumulate_n_v2` performs the same operation as `tf.add_n`, but does not
+// wait for all of its inputs to be ready before beginning to sum. This can
+// save memory if inputs are ready at different times, since minimum temporary
+// storage is proportional to the output size rather than the inputs size.
+//
+// Unlike the original `accumulate_n`, `accumulate_n_v2` is differentiable.
+//
+// Returns a `Tensor` of same shape and type as the elements of `inputs`.
+//
+// Arguments:
+//	inputs: A list of `Tensor` objects, each with same shape and type.
+//	shape: Shape of elements of `inputs`.
+func AccumulateNV2(scope *Scope, inputs []tf.Output, shape tf.Shape) (sum tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"shape": shape}
+	opspec := tf.OpSpec{
+		Type: "AccumulateNV2",
+		Input: []tf.Input{
+			tf.OutputList(inputs),
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Computes the gradient for the inverse of `x` wrt its input.
 //
 // Specifically, `grad = -dy * y*y`, where `y = 1/x`, and `dy`
@@ -13641,35 +13778,6 @@ func Rint(scope *Scope, x tf.Output) (y tf.Output) {
 	return op.Output(0)
 }
 
-// Computes the Eigen Decomposition of a batch of square self-adjoint matrices.
-//
-// DEPRECATED at GraphDef version 11: Use SelfAdjointEigV2 instead.
-//
-// The input is a tensor of shape `[..., M, M]` whose inner-most 2 dimensions
-// form square matrices, with the same constraints as the single matrix
-// SelfAdjointEig.
-//
-// The result is a [..., M+1, M] matrix with [..., 0,:] containing the
-// eigenvalues, and subsequent [...,1:, :] containing the eigenvectors.
-//
-// Arguments:
-//	input: Shape is `[..., M, M]`.
-//
-// Returns Shape is `[..., M+1, M]`.
-func SelfAdjointEig(scope *Scope, input tf.Output) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "SelfAdjointEig",
-		Input: []tf.Input{
-			input,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
 // QuantizeV2Attr is an optional argument to QuantizeV2.
 type QuantizeV2Attr func(optionalAttr)
 
@@ -14310,7 +14418,7 @@ func QuantizedRelu(scope *Scope, features tf.Output, min_features tf.Output, max
 // #                  [20, 21, 22, 23]]]]
 // # tensor 't' shape is [1, 2, 3, 4]
 //
-// # 'dims' is [3] or 'dims' is -1
+// # 'dims' is [3] or 'dims' is [-1]
 // reverse(t, dims) ==> [[[[ 3,  2,  1,  0],
 //                         [ 7,  6,  5,  4],
 //                         [ 11, 10, 9, 8]],
@@ -14534,6 +14642,73 @@ func DepthwiseConv2dNativeBackpropInput(scope *Scope, input_sizes tf.Output, fil
 	return op.Output(0)
 }
 
+// MatrixSolveAttr is an optional argument to MatrixSolve.
+type MatrixSolveAttr func(optionalAttr)
+
+// MatrixSolveAdjoint sets the optional adjoint attribute to value.
+//
+// value: Boolean indicating whether to solve with `matrix` or its (block-wise)
+// adjoint.
+// If not specified, defaults to false
+func MatrixSolveAdjoint(value bool) MatrixSolveAttr {
+	return func(m optionalAttr) {
+		m["adjoint"] = value
+	}
+}
+
+// Solves systems of linear equations.
+//
+// `Matrix` is a tensor of shape `[..., M, M]` whose inner-most 2 dimensions
+// form square matrices. `Rhs` is a tensor of shape `[..., M, K]`. The `output` is
+// a tensor shape `[..., M, K]`.  If `adjoint` is `False` then each output matrix
+// satisfies `matrix[..., :, :] * output[..., :, :] = rhs[..., :, :]`.
+// If `adjoint` is `True` then each output matrix satisfies
+// `adjoint(matrix[..., :, :]) * output[..., :, :] = rhs[..., :, :]`.
+//
+// Arguments:
+//	matrix: Shape is `[..., M, M]`.
+//	rhs: Shape is `[..., M, K]`.
+//
+// Returns Shape is `[..., M, K]`.
+func MatrixSolve(scope *Scope, matrix tf.Output, rhs tf.Output, optional ...MatrixSolveAttr) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "MatrixSolve",
+		Input: []tf.Input{
+			matrix, rhs,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Transforms a Tensor into a serialized TensorProto proto.
+//
+// Arguments:
+//	tensor: A Tensor of type `T`.
+//
+// Returns A serialized TensorProto proto of the input tensor.
+func SerializeTensor(scope *Scope, tensor tf.Output) (serialized tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "SerializeTensor",
+		Input: []tf.Input{
+			tensor,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Get the value of the tensor specified by its handle.
 //
 // Arguments:
@@ -15915,7 +16090,7 @@ func RestoreV2(scope *Scope, prefix tf.Output, tensor_names tf.Output, shape_and
 // Returns x / y element-wise for integer types.
 //
 // Truncation designates that negative numbers will round fractional quantities
-// toward zero. I.e. -7 / 5 = 1. This matches C semantics but it is different
+// toward zero. I.e. -7 / 5 = -1. This matches C semantics but it is different
 // than Python semantics. See `FloorDiv` for a division function that matches
 // Python Semantics.
 //
@@ -20558,84 +20733,6 @@ func MaxPoolGradGradV2(scope *Scope, orig_input tf.Output, orig_output tf.Output
 	return op.Output(0)
 }
 
-// Adjust the saturation of one or more images.
-//
-// `images` is a tensor of at least 3 dimensions.  The last dimension is
-// interpretted as channels, and must be three.
-//
-// The input image is considered in the RGB colorspace. Conceptually, the RGB
-// colors are first mapped into HSV. A scale is then applied all the saturation
-// values, and then remapped back to RGB colorspace.
-//
-// Arguments:
-//	images: Images to adjust.  At least 3-D.
-//	scale: A float scale to add to the saturation.
-//
-// Returns The hue-adjusted image or images.
-func AdjustSaturation(scope *Scope, images tf.Output, scale tf.Output) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "AdjustSaturation",
-		Input: []tf.Input{
-			images, scale,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// SelfAdjointEigV2Attr is an optional argument to SelfAdjointEigV2.
-type SelfAdjointEigV2Attr func(optionalAttr)
-
-// SelfAdjointEigV2ComputeV sets the optional compute_v attribute to value.
-//
-// value: If `True` then eigenvectors will be computed and returned in `v`.
-// Otherwise, only the eigenvalues will be computed.
-// If not specified, defaults to true
-func SelfAdjointEigV2ComputeV(value bool) SelfAdjointEigV2Attr {
-	return func(m optionalAttr) {
-		m["compute_v"] = value
-	}
-}
-
-// Computes the eigen decomposition of one or more square self-adjoint matrices.
-//
-// Computes the eigenvalues and (optionally) eigenvectors of each inner matrix in
-// `input` such that `input[..., :, :] = v[..., :, :] * diag(e[..., :])`.
-//
-// ```python
-// # a is a tensor.
-// # e is a tensor of eigenvalues.
-// # v is a tensor of eigenvectors.
-// e, v = self_adjoint_eig(a)
-// e = self_adjoint_eig(a, compute_v=False)
-// ```
-//
-// Arguments:
-//	input: `Tensor` input of shape `[N, N]`.
-//
-// Returns Eigenvalues. Shape is `[N]`.Eigenvectors. Shape is `[N, N]`.
-func SelfAdjointEigV2(scope *Scope, input tf.Output, optional ...SelfAdjointEigV2Attr) (e tf.Output, v tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "SelfAdjointEigV2",
-		Input: []tf.Input{
-			input,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1)
-}
-
 // Computes second-order gradients of the maxpooling function.
 //
 // Arguments:
@@ -22256,6 +22353,62 @@ func QuantizedBiasAdd(scope *Scope, input tf.Output, bias tf.Output, min_input t
 	return op.Output(0), op.Output(1), op.Output(2)
 }
 
+// HistogramFixedWidthAttr is an optional argument to HistogramFixedWidth.
+type HistogramFixedWidthAttr func(optionalAttr)
+
+// HistogramFixedWidthDtype sets the optional dtype attribute to value.
+// If not specified, defaults to DT_INT32
+func HistogramFixedWidthDtype(value tf.DataType) HistogramFixedWidthAttr {
+	return func(m optionalAttr) {
+		m["dtype"] = value
+	}
+}
+
+// Return histogram of values.
+//
+// Given the tensor `values`, this operation returns a rank 1 histogram counting
+// the number of entries in `values` that fall into every bin.  The bins are
+// equal width and determined by the arguments `value_range` and `nbins`.
+//
+// ```python
+// # Bins will be:  (-inf, 1), [1, 2), [2, 3), [3, 4), [4, inf)
+// nbins = 5
+// value_range = [0.0, 5.0]
+// new_values = [-1.0, 0.0, 1.5, 2.0, 5.0, 15]
+//
+// with tf.get_default_session() as sess:
+//   hist = tf.histogram_fixed_width(new_values, value_range, nbins=5)
+//   variables.global_variables_initializer().run()
+//   sess.run(hist) => [2, 1, 1, 0, 2]
+// ```
+//
+// Arguments:
+//	values: Numeric `Tensor`.
+//	value_range: Shape [2] `Tensor` of same `dtype` as `values`.
+// values <= value_range[0] will be mapped to hist[0],
+// values >= value_range[1] will be mapped to hist[-1].
+//	nbins: Scalar `int32 Tensor`.  Number of histogram bins.
+//
+// Returns A 1-D `Tensor` holding histogram of values.
+func HistogramFixedWidth(scope *Scope, values tf.Output, value_range tf.Output, nbins tf.Output, optional ...HistogramFixedWidthAttr) (out tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "HistogramFixedWidth",
+		Input: []tf.Input{
+			values, value_range, nbins,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Quantized Batch normalization.
 //
 // This op is deprecated and will be removed in the future. Prefer
@@ -23714,6 +23867,55 @@ func SparseToDense(scope *Scope, sparse_indices tf.Output, output_shape tf.Outpu
 	return op.Output(0)
 }
 
+// NthElementAttr is an optional argument to NthElement.
+type NthElementAttr func(optionalAttr)
+
+// NthElementReverse sets the optional reverse attribute to value.
+//
+// value: When set to True, find the nth-largest value in the vector and vice
+// versa.
+// If not specified, defaults to false
+func NthElementReverse(value bool) NthElementAttr {
+	return func(m optionalAttr) {
+		m["reverse"] = value
+	}
+}
+
+// Finds values of the `n`-th order statistic for the last dmension.
+//
+// If the input is a vector (rank-1), finds the entries which is the nth-smallest
+// value in the vector and outputs their values as scalar tensor.
+//
+// For matrices (resp. higher rank input), computes the entries which is the
+// nth-smallest value in each row (resp. vector along the last dimension). Thus,
+//
+//     values.shape = input.shape[:-1]
+//
+// Arguments:
+//	input: 1-D or higher with last dimension at least `n+1`.
+//	n: 0-D. Position of sorted vector to select along the last dimension (along
+// each row for matrices). Valid range of n is `[0, input.shape[:-1])`
+//
+// Returns The `n`-th order statistic along each last dimensional slice.
+func NthElement(scope *Scope, input tf.Output, n tf.Output, optional ...NthElementAttr) (values tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "NthElement",
+		Input: []tf.Input{
+			input, n,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Computes asin of x element-wise.
 func Asin(scope *Scope, x tf.Output) (y tf.Output) {
 	if scope.Err() != nil {
@@ -26817,70 +27019,3 @@ func MatrixInverse(scope *Scope, input tf.Output, optional ...MatrixInverseAttr)
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
-
-// Transforms a Tensor into a serialized TensorProto proto.
-//
-// Arguments:
-//	tensor: A Tensor of type `T`.
-//
-// Returns A serialized TensorProto proto of the input tensor.
-func SerializeTensor(scope *Scope, tensor tf.Output) (serialized tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "SerializeTensor",
-		Input: []tf.Input{
-			tensor,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// MatrixSolveAttr is an optional argument to MatrixSolve.
-type MatrixSolveAttr func(optionalAttr)
-
-// MatrixSolveAdjoint sets the optional adjoint attribute to value.
-//
-// value: Boolean indicating whether to solve with `matrix` or its (block-wise)
-// adjoint.
-// If not specified, defaults to false
-func MatrixSolveAdjoint(value bool) MatrixSolveAttr {
-	return func(m optionalAttr) {
-		m["adjoint"] = value
-	}
-}
-
-// Solves systems of linear equations.
-//
-// `Matrix` is a tensor of shape `[..., M, M]` whose inner-most 2 dimensions
-// form square matrices. `Rhs` is a tensor of shape `[..., M, K]`. The `output` is
-// a tensor shape `[..., M, K]`.  If `adjoint` is `False` then each output matrix
-// satisfies `matrix[..., :, :] * output[..., :, :] = rhs[..., :, :]`.
-// If `adjoint` is `True` then each output matrix satisfies
-// `adjoint(matrix[..., :, :]) * output[..., :, :] = rhs[..., :, :]`.
-//
-// Arguments:
-//	matrix: Shape is `[..., M, M]`.
-//	rhs: Shape is `[..., M, K]`.
-//
-// Returns Shape is `[..., M, K]`.
-func MatrixSolve(scope *Scope, matrix tf.Output, rhs tf.Output, optional ...MatrixSolveAttr) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "MatrixSolve",
-		Input: []tf.Input{
-			matrix, rhs,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-- 
GitLab


From 557b0b27edff763c165ad59d10d49da8bccbec4f Mon Sep 17 00:00:00 2001
From: Mark Heffernan <meheff@google.com>
Date: Tue, 24 Oct 2017 20:05:39 -0700
Subject: [PATCH 748/909] Make HloRunner methods return StatusOr. Also move
 templated method definition of Execute into the header file.

PiperOrigin-RevId: 173348703
---
 tensorflow/compiler/xla/service/hlo_runner.cc | 40 +++++--------------
 tensorflow/compiler/xla/service/hlo_runner.h  | 24 ++++++++---
 .../compiler/xla/tests/hlo_test_base.cc       |  6 +--
 3 files changed, 31 insertions(+), 39 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_runner.cc b/tensorflow/compiler/xla/service/hlo_runner.cc
index d5d7042a02..9fdda38d2d 100644
--- a/tensorflow/compiler/xla/service/hlo_runner.cc
+++ b/tensorflow/compiler/xla/service/hlo_runner.cc
@@ -29,7 +29,6 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/hlo_computation.h"
 #include "tensorflow/compiler/xla/service/transfer_manager.h"
 #include "tensorflow/compiler/xla/shape_util.h"
-#include "tensorflow/compiler/xla/status_macros.h"
 #include "tensorflow/compiler/xla/statusor.h"
 #include "tensorflow/compiler/xla/types.h"
 #include "tensorflow/core/common_runtime/eigen_thread_pool.h"
@@ -133,7 +132,8 @@ StatusOr<se::DeviceMemoryBase> HloRunner::Execute(
   return result;
 }
 
-se::DeviceMemoryBase HloRunner::TransferToDevice(const Literal& literal) {
+StatusOr<se::DeviceMemoryBase> HloRunner::TransferToDevice(
+    const Literal& literal) {
   // Allocate memory on the device using the stream executor.
   int64 allocation_size =
       backend().transfer_manager()->GetByteSizeRequirement(literal.shape());
@@ -142,52 +142,30 @@ se::DeviceMemoryBase HloRunner::TransferToDevice(const Literal& literal) {
           allocation_size);
   allocations_.push_back(allocation);
 
-  TF_CHECK_OK(backend().transfer_manager()->TransferLiteralToDevice(
+  TF_RETURN_IF_ERROR(backend().transfer_manager()->TransferLiteralToDevice(
       backend().default_stream_executor(), literal, &allocation));
 
   return allocation;
 }
 
-std::unique_ptr<Literal> HloRunner::TransferFromDevice(
+StatusOr<std::unique_ptr<Literal>> HloRunner::TransferFromDevice(
     const Shape& shape, se::DeviceMemoryBase device_base) {
   auto literal = MakeUnique<Literal>();
-  TF_CHECK_OK(backend().transfer_manager()->TransferLiteralFromDevice(
+  TF_RETURN_IF_ERROR(backend().transfer_manager()->TransferLiteralFromDevice(
       backend().default_stream_executor(), device_base, shape, shape,
       literal.get()));
-  return literal;
+  return std::move(literal);
 }
 
-std::unique_ptr<Literal> HloRunner::ExecuteAndTransfer(
+StatusOr<std::unique_ptr<Literal>> HloRunner::ExecuteAndTransfer(
     std::unique_ptr<HloModule> module,
     tensorflow::gtl::ArraySlice<se::DeviceMemoryBase> arguments) {
   Shape result_shape;
-  se::DeviceMemoryBase device_base =
-      Execute(std::move(module), arguments, &result_shape).ValueOrDie();
+  TF_ASSIGN_OR_RETURN(se::DeviceMemoryBase device_base,
+                      Execute(std::move(module), arguments, &result_shape));
   return TransferFromDevice(result_shape, device_base);
 }
 
-template <>
-std::unique_ptr<Literal> HloRunner::Execute(
-    std::unique_ptr<HloModule> module,
-    const tensorflow::gtl::ArraySlice<std::unique_ptr<Literal>>& literals) {
-  std::vector<se::DeviceMemoryBase> arguments;
-  for (const auto& literal : literals) {
-    arguments.push_back(TransferToDevice(*literal));
-  }
-  return ExecuteAndTransfer(std::move(module), arguments);
-}
-
-template <>
-std::unique_ptr<Literal> HloRunner::Execute(
-    std::unique_ptr<HloModule> module,
-    const tensorflow::gtl::ArraySlice<Literal*>& literals) {
-  std::vector<se::DeviceMemoryBase> arguments;
-  for (const auto& literal : literals) {
-    arguments.push_back(TransferToDevice(*literal));
-  }
-  return ExecuteAndTransfer(std::move(module), arguments);
-}
-
 Backend& HloRunner::backend() {
   if (!backend_) {
     backend_ = Backend::CreateDefaultBackend().ConsumeValueOrDie();
diff --git a/tensorflow/compiler/xla/service/hlo_runner.h b/tensorflow/compiler/xla/service/hlo_runner.h
index d74a1b59a8..a4d7b653db 100644
--- a/tensorflow/compiler/xla/service/hlo_runner.h
+++ b/tensorflow/compiler/xla/service/hlo_runner.h
@@ -24,6 +24,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/compiler.h"
 #include "tensorflow/compiler/xla/service/hlo_computation.h"
 #include "tensorflow/compiler/xla/service/hlo_module.h"
+#include "tensorflow/compiler/xla/status_macros.h"
 #include "tensorflow/compiler/xla/statusor.h"
 #include "tensorflow/compiler/xla/types.h"
 #include "tensorflow/compiler/xla/xla_data.pb.h"
@@ -52,9 +53,9 @@ class HloRunner {
   // result as a Literal. The LiteralPtr type accepts Literal* or
   // std::unique_ptr<Literal>.
   template <typename LiteralPtr>
-  std::unique_ptr<Literal> Execute(
+  StatusOr<std::unique_ptr<Literal>> Execute(
       std::unique_ptr<HloModule> module,
-      const tensorflow::gtl::ArraySlice<LiteralPtr>& literals);
+      const tensorflow::gtl::ArraySlice<LiteralPtr> literals);
 
   // Executes the given module and returns a global data handle.
   StatusOr<perftools::gputools::DeviceMemoryBase> Execute(
@@ -64,16 +65,16 @@ class HloRunner {
       Shape* result_shape);
 
   // Transfers the given literal to the device and returns the data handle.
-  perftools::gputools::DeviceMemoryBase TransferToDevice(
+  StatusOr<perftools::gputools::DeviceMemoryBase> TransferToDevice(
       const Literal& literal);
 
   // Transfers the array referred to by the given handle from the device and
   // returns as a Literal.
-  std::unique_ptr<Literal> TransferFromDevice(
+  StatusOr<std::unique_ptr<Literal>> TransferFromDevice(
       const Shape& shape, perftools::gputools::DeviceMemoryBase device_base);
 
   // Executes the given module and return the result as a Literal.
-  std::unique_ptr<Literal> ExecuteAndTransfer(
+  StatusOr<std::unique_ptr<Literal>> ExecuteAndTransfer(
       std::unique_ptr<HloModule> module,
       tensorflow::gtl::ArraySlice<perftools::gputools::DeviceMemoryBase>
           arguments);
@@ -95,6 +96,19 @@ class HloRunner {
   std::unique_ptr<Backend> backend_;
 };
 
+template <typename LiteralPtr>
+StatusOr<std::unique_ptr<Literal>> HloRunner::Execute(
+    std::unique_ptr<HloModule> module,
+    const tensorflow::gtl::ArraySlice<LiteralPtr> literals) {
+  std::vector<perftools::gputools::DeviceMemoryBase> arguments;
+  for (const auto& literal : literals) {
+    TF_ASSIGN_OR_RETURN(perftools::gputools::DeviceMemoryBase argument,
+                        TransferToDevice(*literal));
+    arguments.push_back(argument);
+  }
+  return ExecuteAndTransfer(std::move(module), arguments);
+}
+
 }  // namespace xla
 
 #endif  // TENSORFLOW_COMPILER_XLA_SERVICE_HLO_RUNNER_H_
diff --git a/tensorflow/compiler/xla/tests/hlo_test_base.cc b/tensorflow/compiler/xla/tests/hlo_test_base.cc
index 3e244fbfd9..d73c05ff92 100644
--- a/tensorflow/compiler/xla/tests/hlo_test_base.cc
+++ b/tensorflow/compiler/xla/tests/hlo_test_base.cc
@@ -53,18 +53,18 @@ StatusOr<perftools::gputools::DeviceMemoryBase> HloTestBase::Execute(
 }
 
 se::DeviceMemoryBase HloTestBase::TransferToDevice(const Literal& literal) {
-  return runner_.TransferToDevice(literal);
+  return runner_.TransferToDevice(literal).ValueOrDie();
 }
 
 std::unique_ptr<Literal> HloTestBase::TransferFromDevice(
     const Shape& shape, se::DeviceMemoryBase device_base) {
-  return runner_.TransferFromDevice(shape, device_base);
+  return runner_.TransferFromDevice(shape, device_base).ValueOrDie();
 }
 
 std::unique_ptr<Literal> HloTestBase::ExecuteAndTransfer(
     std::unique_ptr<HloModule> module,
     tensorflow::gtl::ArraySlice<se::DeviceMemoryBase> arguments) {
-  return runner_.ExecuteAndTransfer(std::move(module), arguments);
+  return runner_.ExecuteAndTransfer(std::move(module), arguments).ValueOrDie();
 }
 
 Backend& HloTestBase::backend() { return runner_.backend(); }
-- 
GitLab


From 7bd701a29e958be1f836eabc498a67d742c35cdf Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 24 Oct 2017 20:37:06 -0700
Subject: [PATCH 749/909] Support more instructions in Hlo parser. - while,
 tuple, send/recv, get-tuple-element, call. - "device="

Also,
- Change HloModule::ToString to print computations in post order, so that a computation is defined before it's used.
- Add % before computation name when it's used.

PiperOrigin-RevId: 173350323
---
 .../compiler/xla/service/hlo_computation.cc   |   3 +-
 .../compiler/xla/service/hlo_instruction.cc   |  17 +-
 .../xla/service/hlo_instruction_test.cc       |   4 +-
 tensorflow/compiler/xla/service/hlo_module.cc |  14 +-
 .../compiler/xla/tools/parser/README.md       |  24 +-
 .../compiler/xla/tools/parser/hlo_lexer.cc    |   9 +
 .../compiler/xla/tools/parser/hlo_lexer.h     |   9 +-
 .../compiler/xla/tools/parser/hlo_parser.cc   | 323 ++++++++++++++----
 .../xla/tools/parser/hlo_parser_test.cc       | 117 ++++++-
 .../compiler/xla/tools/parser/hlo_token.h     |  12 +-
 10 files changed, 428 insertions(+), 104 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_computation.cc b/tensorflow/compiler/xla/service/hlo_computation.cc
index 51ead753f0..2285518a0e 100644
--- a/tensorflow/compiler/xla/service/hlo_computation.cc
+++ b/tensorflow/compiler/xla/service/hlo_computation.cc
@@ -379,7 +379,8 @@ string HloComputation::ToString(int nested_level) const {
     for (int i = 0; i < nested_level; i++) {
       s << "    ";
     }
-    s << "  " << instruction->ToString() << "\n";
+    s << "  " << (instruction == root_instruction_ ? "ROOT " : "")
+      << instruction->ToString() << "\n";
     if (instruction->opcode() == HloOpcode::kFusion) {
       s << instruction->fused_instructions_computation()->ToString(
                nested_level + 1)
diff --git a/tensorflow/compiler/xla/service/hlo_instruction.cc b/tensorflow/compiler/xla/service/hlo_instruction.cc
index 0669a86863..8e52d131a6 100644
--- a/tensorflow/compiler/xla/service/hlo_instruction.cc
+++ b/tensorflow/compiler/xla/service/hlo_instruction.cc
@@ -1855,16 +1855,20 @@ std::vector<string> HloInstruction::ExtraAttributesToString() const {
   }
 
   if (opcode() == HloOpcode::kWhile) {
-    extra.push_back(StrCat("condition=", while_condition()->name()));
-    extra.push_back(StrCat("body=", while_body()->name()));
+    extra.push_back(StrCat("condition=%", while_condition()->name()));
+    extra.push_back(StrCat("body=%", while_body()->name()));
   } else if (opcode() == HloOpcode::kSelectAndScatter) {
-    extra.push_back(StrCat("select=", select()->name()));
-    extra.push_back(StrCat("scatter=", scatter()->name()));
+    extra.push_back(StrCat("select=%", select()->name()));
+    extra.push_back(StrCat("scatter=%", scatter()->name()));
+  } else if (opcode() == HloOpcode::kCall || opcode() == HloOpcode::kMap ||
+             opcode() == HloOpcode::kReduceWindow ||
+             opcode() == HloOpcode::kReduce) {
+    extra.push_back(StrCat("to_apply=%", to_apply()->name()));
   } else if (!called_computations().empty()) {
     extra.push_back(StrCat(
         "calls=", Join(called_computations(), ", ",
                        [](string* out, const HloComputation* computation) {
-                         StrAppend(out, computation->name());
+                         StrAppend(out, "%", computation->name());
                        })));
   }
 
@@ -1875,6 +1879,9 @@ std::vector<string> HloInstruction::ExtraAttributesToString() const {
   if (opcode() == HloOpcode::kGetTupleElement) {
     extra.push_back(StrCat("index=", tuple_index()));
   }
+  if (device_assignment_.has_device()) {
+    extra.push_back(StrCat("device=", device_assignment_.device()));
+  }
   if (!control_successors_.empty()) {
     extra.push_back(StrCat(
         "control-successors=",
diff --git a/tensorflow/compiler/xla/service/hlo_instruction_test.cc b/tensorflow/compiler/xla/service/hlo_instruction_test.cc
index cdafc05d8c..9affecae60 100644
--- a/tensorflow/compiler/xla/service/hlo_instruction_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_instruction_test.cc
@@ -1203,13 +1203,13 @@ TEST_F(HloInstructionTest, Stringification) {
 
   EXPECT_EQ(fusion->ToString(false, false),
             "%fusion = f32[5,20]{1,0} fusion:kTransposeDot(f32[5,10]{1,0} %x, "
-            "f32[20,10]{1,0} %y), calls=fused_computation");
+            "f32[20,10]{1,0} %y), calls=%fused_computation");
 
   HloInstruction* loop = builder.AddInstruction(
       HloInstruction::CreateWhile(sout, computation, computation, x));
   EXPECT_EQ(loop->ToString(false, false),
             "%while = f32[5,20]{1,0} while(f32[5,10]{1,0} %x), "
-            "condition=TransposeDot, body=TransposeDot");
+            "condition=%TransposeDot, body=%TransposeDot");
 }
 
 }  // namespace
diff --git a/tensorflow/compiler/xla/service/hlo_module.cc b/tensorflow/compiler/xla/service/hlo_module.cc
index 9d4a994838..f7990fa789 100644
--- a/tensorflow/compiler/xla/service/hlo_module.cc
+++ b/tensorflow/compiler/xla/service/hlo_module.cc
@@ -153,11 +153,17 @@ void HloModule::ReplaceComputations(
 string HloModule::ToString() const {
   std::ostringstream s;
   s << "HloModule " << name() << ":\n\n";
-  s << "ENTRY " << entry_computation()->ToString() << "\n\n";
-  for (const HloComputation* computation : MakeNonfusionComputations()) {
-    if (computation != entry_computation()) {
-      s << computation->ToString() << "\n\n";
+  for (const HloComputation* computation : MakeComputationPostOrder()) {
+    // Fusion computations are emitted with their fusion instruction and
+    // therefore don't need to be emitted as a separate comptutation in the
+    // module.
+    if (computation->IsFusionComputation()) {
+      continue;
+    }
+    if (computation == entry_computation()) {
+      s << "ENTRY ";
     }
+    s << computation->ToString() << "\n\n";
   }
   return s.str();
 }
diff --git a/tensorflow/compiler/xla/tools/parser/README.md b/tensorflow/compiler/xla/tools/parser/README.md
index a334bc2b29..2feaa49db8 100644
--- a/tensorflow/compiler/xla/tools/parser/README.md
+++ b/tensorflow/compiler/xla/tools/parser/README.md
@@ -1,16 +1,22 @@
 # HloModule string syntax
 
-TODO: Support subcomputations (for fusion, reduce, while, ...).
+TODO: Support all subcomputations (for fusion, reduce, ...).
 
-TODO: Support ops that require extra attributes, e.g. dimensions, strides.
+TODO: Support all extra attributes, e.g. dimensions, strides.
 
 ```yacc
 hlo_module
-  : 'HloModule' name computation
+  : 'HloModule' name computations
+  ;
+
+computations
+  : computation
+  | computation computations
   ;
 
 computation
   : 'ENTRY' name param_list '->' shape instruction_list
+  | name param_list '->' shape instruction_list
   ;
 
 instruction_list
@@ -21,7 +27,8 @@ instruction_list1
   | instruction_list1 instruction
   ;
 instruction
-  : name '=' shape opcode operands
+  : 'ROOT' name '=' shape opcode operands extra_attributes
+  | name '=' shape opcode operands extra_attributes
   ;
 
 operands
@@ -36,6 +43,15 @@ operand
   : shape name
   ;
 
+extra_attributes
+  : /*empty*/
+  | ',' extra_attribute
+  | ',' extra_attribute extra_attributes
+  ;
+extra_attribute
+  : attribute_name attribute_value
+  ;
+
 param_list
   : '(' param_list1 ')'
   ;
diff --git a/tensorflow/compiler/xla/tools/parser/hlo_lexer.cc b/tensorflow/compiler/xla/tools/parser/hlo_lexer.cc
index 3e84ffcbd2..fba343de48 100644
--- a/tensorflow/compiler/xla/tools/parser/hlo_lexer.cc
+++ b/tensorflow/compiler/xla/tools/parser/hlo_lexer.cc
@@ -152,6 +152,7 @@ TokKind HloLexer::LexToken() {
 // name     ::= [a-zA-Z_][a-zA-Z0-9_.-]*:
 // keyword  ::= HloModule, ENTRY, ...
 // opcode   ::= add, greater-than, ...
+// attribute_name ::= condition, body, dimensions, ...
 TokKind HloLexer::LexIdentifier() {
   {
     auto consumable = RegexpStringPieceFromPointers(token_start_, buf_.end());
@@ -181,6 +182,13 @@ TokKind HloLexer::LexIdentifier() {
     return TokKind::kName;
   }
 
+  // If followed by '=', it's a attribute name.
+  if (PeekCurrentChar() == '=') {
+    str_val_.assign(token_start_, current_ptr_);
+    current_ptr_++;  // skip '='
+    return TokKind::kAttributeName;
+  }
+
   StringPiece identifier = StringPieceFromPointers(token_start_, current_ptr_);
 
   // See if this is a keyword.
@@ -195,6 +203,7 @@ TokKind HloLexer::LexIdentifier() {
   KEYWORD(false);
   KEYWORD(HloModule);
   KEYWORD(ENTRY);
+  KEYWORD(ROOT);
 
 #undef KEYWORD
 
diff --git a/tensorflow/compiler/xla/tools/parser/hlo_lexer.h b/tensorflow/compiler/xla/tools/parser/hlo_lexer.h
index 20278fd6cd..433a3a3601 100644
--- a/tensorflow/compiler/xla/tools/parser/hlo_lexer.h
+++ b/tensorflow/compiler/xla/tools/parser/hlo_lexer.h
@@ -39,8 +39,13 @@ class HloLexer {
   TokKind Lex() { return current_kind_ = LexToken(); }
   TokKind GetKind() const { return current_kind_; }
   string GetStrVal() const {
-    CHECK(GetKind() == TokKind::kName);
-    return str_val_;
+    switch (GetKind()) {
+      case TokKind::kName:
+      case TokKind::kAttributeName:
+        return str_val_;
+      default:
+        LOG(FATAL) << "This token does not have string value";
+    }
   }
   Shape GetShapeVal() const {
     CHECK(GetKind() == TokKind::kShape);
diff --git a/tensorflow/compiler/xla/tools/parser/hlo_parser.cc b/tensorflow/compiler/xla/tools/parser/hlo_parser.cc
index 57700493e6..a075d9057f 100644
--- a/tensorflow/compiler/xla/tools/parser/hlo_parser.cc
+++ b/tensorflow/compiler/xla/tools/parser/hlo_parser.cc
@@ -44,14 +44,26 @@ class HloParser {
  private:
   // ParseXXX returns false if an error occurred.
   bool ParseHloModule();
+  bool ParseComputations();
   bool ParseComputation();
-  bool ParseInstructionList(HloComputation::Builder* builder);
-  bool ParseInstruction(HloComputation::Builder* builder);
+  bool ParseInstructionList(HloComputation::Builder* builder,
+                            string* root_name);
+  bool ParseInstruction(HloComputation::Builder* builder, string* root_name);
   bool ParseLiteral(std::unique_ptr<Literal>* literal, const Shape& shape);
+  bool ParseOperands(std::vector<HloInstruction*>* operands);
+  // Fill parsed operands into 'operands' and expect a certain number of
+  // operands.
   bool ParseOperands(std::vector<HloInstruction*>* operands,
                      const int expected_size);
+
+  template <typename T>
+  bool ParseExtraAttribute(T* value, const string& expected_attribute);
+  template <typename T>
+  bool ParseAttributeValue(T* value);
+
   bool ParseParamList();
   bool ParseName(string* result);
+  bool ParseAttributeName(string* result);
   bool ParseShape(Shape* result);
   bool ParseOpcode(HloOpcode* result);
   bool ParseInt64(int64* result);
@@ -69,10 +81,14 @@ class HloParser {
   // Adds the instruction to the pool. Returns false and emits an error if the
   // instruction already exists.
   bool AddInstruction(const string& name, HloInstruction* instruction);
+  // Adds the computation to the pool. Returns false and emits an error if the
+  // computation already exists.
+  bool AddComputation(const string& name, HloComputation* computation);
 
   // The map from the instruction name to the instruction. This does not own the
   // instructions.
   std::unordered_map<string, HloInstruction*> instruction_pool_;
+  std::unordered_map<string, HloComputation*> computation_pool_;
 
   HloLexer lexer_;
   std::unique_ptr<HloModule> module_;
@@ -90,7 +106,7 @@ bool HloParser::Run() {
   return ParseHloModule();
 }
 
-// ::= 'HloModule' name computation
+// ::= 'HloModule' name computations
 bool HloParser::ParseHloModule() {
   if (lexer_.GetKind() != TokKind::kw_HloModule) {
     return TokenError("expects HloModule");
@@ -105,35 +121,63 @@ bool HloParser::ParseHloModule() {
 
   module_ = MakeUnique<HloModule>(name);
 
-  return ParseComputation();
+  return ParseComputations();
 }
 
-// computation ::= 'ENTRY' name param_list '->' shape instruction_list
+// computations ::= (computation)+
+bool HloParser::ParseComputations() {
+  do {
+    if (!ParseComputation()) {
+      return false;
+    }
+  } while (lexer_.GetKind() != TokKind::kEof);
+  return true;
+}
+
+// computation ::= ('ENTRY')? name param_list '->' shape instruction_list
 bool HloParser::ParseComputation() {
+  const bool is_entry_computation = EatIfPresent(TokKind::kw_ENTRY);
   string name;
-  if (!ParseToken(TokKind::kw_ENTRY, "expects 'ENTRY'") || !ParseName(&name)) {
+  if (!ParseName(&name)) {
     return false;
   }
   auto builder = MakeUnique<HloComputation::Builder>(name);
 
   Shape shape;
+  string root_name;
   if (!ParseParamList() || !ParseToken(TokKind::kArrow, "expects '->'") ||
-      !ParseShape(&shape) || !ParseInstructionList(builder.get())) {
+      !ParseShape(&shape) || !ParseInstructionList(builder.get(), &root_name)) {
     return false;
   }
-  module_->AddEntryComputation(builder->Build());
-  return true;
+
+  HloInstruction* root =
+      tensorflow::gtl::FindPtrOrNull(instruction_pool_, root_name);
+  // This means some instruction was marked as ROOT but we didn't find it in the
+  // pool, which should not happen.
+  if (!root_name.empty() && root == nullptr) {
+    LOG(FATAL) << "instruction " << root_name
+               << " was marked as ROOT but the parser has not seen it before";
+  }
+  // Now root can be either an existing instruction or a nullptr. If it's a
+  // nullptr, the implementation of Builder will set the last instruction as
+  // root instruction.
+  HloComputation* computation =
+      is_entry_computation
+          ? module_->AddEntryComputation(builder->Build(root))
+          : module_->AddEmbeddedComputation(builder->Build(root));
+  return AddComputation(name, computation);
 }
 
 // instruction_list ::= '{' instruction_list1 '}'
 // instruction_list1 ::= (instruction)+
-bool HloParser::ParseInstructionList(HloComputation::Builder* builder) {
+bool HloParser::ParseInstructionList(HloComputation::Builder* builder,
+                                     string* root_name) {
   if (!ParseToken(TokKind::kLbrace,
                   "expects '{' at the beginning of instruction list.")) {
     return false;
   }
   do {
-    if (!ParseInstruction(builder)) {
+    if (!ParseInstruction(builder, root_name)) {
       return false;
     }
   } while (lexer_.GetKind() != TokKind::kRbrace);
@@ -141,39 +185,47 @@ bool HloParser::ParseInstructionList(HloComputation::Builder* builder) {
                     "expects '}' at the end of instruction list.");
 }
 
-// instruction ::= name '=' shape opcode operands
-bool HloParser::ParseInstruction(HloComputation::Builder* builder) {
+// instruction ::= ('ROOT')? name '=' shape opcode operands (extra_attribute)*
+bool HloParser::ParseInstruction(HloComputation::Builder* builder,
+                                 string* root_name) {
   string name;
   Shape shape;
   HloOpcode opcode;
   std::vector<HloInstruction*> operands;
+  bool is_root = EatIfPresent(TokKind::kw_ROOT);
   if (!ParseName(&name) ||
       !ParseToken(TokKind::kEqual, "expects '=' in instruction") ||
       !ParseShape(&shape) || !ParseOpcode(&opcode)) {
     return false;
   }
+  if (is_root) {
+    *root_name = name;
+  }
+  HloInstruction* instruction;
   switch (opcode) {
     case HloOpcode::kParameter: {
       int64 parameter_number;
-      return ParseToken(TokKind::kLparen,
-                        "expects '(' before parameter number") &&
-             ParseInt64(&parameter_number) &&
-             ParseToken(TokKind::kRparen,
-                        "expects ')' after parameter number") &&
-             AddInstruction(
-                 name, builder->AddInstruction(HloInstruction::CreateParameter(
-                           parameter_number, shape, name)));
+      if (!ParseToken(TokKind::kLparen,
+                      "expects '(' before parameter number") ||
+          !ParseInt64(&parameter_number) ||
+          !ParseToken(TokKind::kRparen, "expects ')' after parameter number")) {
+        return false;
+      }
+      instruction = builder->AddInstruction(
+          HloInstruction::CreateParameter(parameter_number, shape, name));
+      break;
     }
     case HloOpcode::kConstant: {
       std::unique_ptr<Literal> literal;
-      return ParseToken(TokKind::kLparen,
-                        "expects '(' before parameter number") &&
-             ParseLiteral(&literal, shape) &&
-             ParseToken(TokKind::kRparen,
-                        "expects ')' after parameter number") &&
-             AddInstruction(
-                 name, builder->AddInstruction(
-                           HloInstruction::CreateConstant(std::move(literal))));
+      if (!ParseToken(TokKind::kLparen,
+                      "expects '(' before constant literal") ||
+          !ParseLiteral(&literal, shape) ||
+          !ParseToken(TokKind::kRparen, "expects ')' after constant literal")) {
+        return false;
+      }
+      instruction = builder->AddInstruction(
+          HloInstruction::CreateConstant(std::move(literal)));
+      break;
     }
     // Unary ops.
     case HloOpcode::kAbs:
@@ -192,10 +244,12 @@ bool HloParser::ParseInstruction(HloComputation::Builder* builder) {
     case HloOpcode::kSin:
     case HloOpcode::kSort:
     case HloOpcode::kTanh: {
-      return ParseOperands(&operands, /*expected_size=*/1) &&
-             AddInstruction(name,
-                            builder->AddInstruction(HloInstruction::CreateUnary(
-                                shape, opcode, operands[0])));
+      if (!ParseOperands(&operands, /*expected_size=*/1)) {
+        return false;
+      }
+      instruction = builder->AddInstruction(
+          HloInstruction::CreateUnary(shape, opcode, operands[0]));
+      break;
     }
     // Binary ops.
     case HloOpcode::kAdd:
@@ -218,46 +272,117 @@ bool HloParser::ParseInstruction(HloComputation::Builder* builder) {
     case HloOpcode::kShiftLeft:
     case HloOpcode::kShiftRightArithmetic:
     case HloOpcode::kShiftRightLogical: {
-      return ParseOperands(&operands, /*expected_size=*/2) &&
-             AddInstruction(
-                 name, builder->AddInstruction(HloInstruction::CreateBinary(
-                           shape, opcode, operands[0], operands[1])));
+      if (!ParseOperands(&operands, /*expected_size=*/2)) {
+        return false;
+      }
+      instruction = builder->AddInstruction(HloInstruction::CreateBinary(
+          shape, opcode, operands[0], operands[1]));
+      break;
     }
     // Ternary ops.
     case HloOpcode::kClamp:
     case HloOpcode::kSelect: {
-      return ParseOperands(&operands, /*expected_size=*/3) &&
-             AddInstruction(
-                 name,
-                 builder->AddInstruction(HloInstruction::CreateTernary(
-                     shape, opcode, operands[0], operands[1], operands[2])));
+      if (!ParseOperands(&operands, /*expected_size=*/3)) {
+        return false;
+      }
+      instruction = builder->AddInstruction(HloInstruction::CreateTernary(
+          shape, opcode, operands[0], operands[1], operands[2]));
+      break;
     }
     // Other supported ops.
     case HloOpcode::kConvert: {
-      return ParseOperands(&operands, /*expected_size=*/1) &&
-             AddInstruction(
-                 name, builder->AddInstruction(
-                           HloInstruction::CreateConvert(shape, operands[0])));
+      if (!ParseOperands(&operands, /*expected_size=*/1)) {
+        return false;
+      }
+      instruction = builder->AddInstruction(
+          HloInstruction::CreateConvert(shape, operands[0]));
+      break;
     }
     case HloOpcode::kCrossReplicaSum: {
-      return ParseOperands(&operands, /*expected_size=*/1) &&
-             AddInstruction(name, builder->AddInstruction(
-                                      HloInstruction::CreateCrossReplicaSum(
-                                          shape, operands[0])));
+      if (!ParseOperands(&operands, /*expected_size=*/1)) {
+        return false;
+      }
+      instruction = builder->AddInstruction(
+          HloInstruction::CreateCrossReplicaSum(shape, operands[0]));
+      break;
     }
     case HloOpcode::kReshape: {
-      return ParseOperands(&operands, /*expected_size=*/1) &&
-             AddInstruction(
-                 name, builder->AddInstruction(
-                           HloInstruction::CreateReshape(shape, operands[0])));
+      if (!ParseOperands(&operands, /*expected_size=*/1)) {
+        return false;
+      }
+      instruction = builder->AddInstruction(
+          HloInstruction::CreateReshape(shape, operands[0]));
+      break;
+    }
+    case HloOpcode::kTuple: {
+      if (!ParseOperands(&operands)) {
+        return false;
+      }
+      instruction =
+          builder->AddInstruction(HloInstruction::CreateTuple(operands));
+      break;
+    }
+    case HloOpcode::kWhile: {
+      HloComputation* condition;
+      HloComputation* body;
+      if (!ParseOperands(&operands, /*expected_size=*/1) ||
+          !ParseExtraAttribute(&condition,
+                               /*expected_attribute=*/"condition") ||
+          !ParseExtraAttribute(&body, /*expected_attribute=*/"body")) {
+        return false;
+      }
+      instruction = builder->AddInstruction(HloInstruction::CreateWhile(
+          shape, condition, body, /*init=*/operands[0]));
+      break;
+    }
+    case HloOpcode::kRecv: {
+      int64 channel_id;
+      if (!ParseOperands(&operands, /*expected_size=*/0) ||
+          !ParseExtraAttribute(&channel_id,
+                               /*expected_attribute=*/"channel_id")) {
+        return false;
+      }
+      instruction = builder->AddInstruction(
+          HloInstruction::CreateRecv(shape, channel_id));
+      break;
+    }
+    case HloOpcode::kSend: {
+      int64 channel_id;
+      if (!ParseOperands(&operands, /*expected_size=*/1) ||
+          !ParseExtraAttribute(&channel_id,
+                               /*expected_attribute=*/"channel_id")) {
+        return false;
+      }
+      instruction = builder->AddInstruction(
+          HloInstruction::CreateSend(operands[0], channel_id));
+      break;
+    }
+    case HloOpcode::kGetTupleElement: {
+      int64 index;
+      if (!ParseOperands(&operands, /*expected_size=*/1) ||
+          !ParseExtraAttribute(&index, /*expected_attribute=*/"index")) {
+        return false;
+      }
+      instruction = builder->AddInstruction(
+          HloInstruction::CreateGetTupleElement(shape, operands[0], index));
+      break;
+    }
+    case HloOpcode::kCall: {
+      HloComputation* to_apply;
+      if (!ParseOperands(&operands) ||
+          !ParseExtraAttribute(&to_apply,
+                               /*expected_attribute=*/"to_apply")) {
+        return false;
+      }
+      instruction = builder->AddInstruction(
+          HloInstruction::CreateCall(shape, operands, to_apply));
+      break;
     }
     case HloOpcode::kBroadcast:
-    case HloOpcode::kCall:
     case HloOpcode::kCustomCall:
     case HloOpcode::kConcatenate:
     case HloOpcode::kReducePrecision:
     case HloOpcode::kConvolution:
-    case HloOpcode::kGetTupleElement:
     case HloOpcode::kMap:
     case HloOpcode::kPad:
     case HloOpcode::kReduce:
@@ -269,22 +394,31 @@ bool HloParser::ParseInstruction(HloComputation::Builder* builder) {
     case HloOpcode::kDynamicSlice:
     case HloOpcode::kDynamicUpdateSlice:
     case HloOpcode::kTranspose:
-    case HloOpcode::kTuple:
-    case HloOpcode::kWhile:
     case HloOpcode::kFusion:
     case HloOpcode::kBatchNormTraining:
     case HloOpcode::kBatchNormInference:
     case HloOpcode::kInfeed:
     case HloOpcode::kOutfeed:
     case HloOpcode::kBatchNormGrad:
-    case HloOpcode::kRecv:
-    case HloOpcode::kSend:
     case HloOpcode::kUpdate:
     case HloOpcode::kIndex:
     case HloOpcode::kTrace:
       return TokenError(StrCat("parsing not yet implemented for op: ",
                                HloOpcodeString(opcode)));
   }
+  // Parse "device=".
+  if (lexer_.GetKind() == TokKind::kComma) {
+    int64 device;
+    if (!ParseExtraAttribute(&device, /*expected_attribute=*/"device")) {
+      return false;
+    }
+    OpDeviceAssignment assignment;
+    assignment.set_has_device(true);
+    assignment.set_device(device);
+    instruction->set_device_assignment(assignment);
+  }
+
+  return AddInstruction(name, instruction);
 }
 
 bool HloParser::ParseLiteral(std::unique_ptr<Literal>* literal,
@@ -322,8 +456,7 @@ bool HloParser::ParseLiteral(std::unique_ptr<Literal>* literal,
 //   ::= /*empty*/
 //   ::= operand (, operand)*
 // operand ::= shape name
-bool HloParser::ParseOperands(std::vector<HloInstruction*>* operands,
-                              const int expected_size) {
+bool HloParser::ParseOperands(std::vector<HloInstruction*>* operands) {
   if (!ParseToken(TokKind::kLparen,
                   "expects '(' at the beginning of operands")) {
     return false;
@@ -345,11 +478,57 @@ bool HloParser::ParseOperands(std::vector<HloInstruction*>* operands,
       operands->push_back(instruction);
     } while (EatIfPresent(TokKind::kComma));
   }
+  return ParseToken(TokKind::kRparen, "expects ')' at the end of operands");
+}
+
+bool HloParser::ParseOperands(std::vector<HloInstruction*>* operands,
+                              const int expected_size) {
+  if (!ParseOperands(operands)) {
+    return false;
+  }
   if (expected_size != operands->size()) {
     return TokenError(StrCat("expects ", expected_size, " operands, but has ",
                              operands->size(), " operands"));
   }
-  return ParseToken(TokKind::kRparen, "expects ')' at the end of operands");
+  return true;
+}
+
+// extra_attribute ::= ',' attribute_name value
+template <typename T>
+bool HloParser::ParseExtraAttribute(T* value,
+                                    const string& expected_attribute) {
+  if (!ParseToken(TokKind::kComma,
+                  "expects ',' in front of an extra attribute")) {
+    return false;
+  }
+  string attribute_name;
+  if (!ParseAttributeName(&attribute_name) &&
+      attribute_name != expected_attribute) {
+    return TokenError(StrCat("expects attribute name: ", expected_attribute));
+  }
+  if (!ParseAttributeValue(value)) {
+    return TokenError(
+        StrCat("expects value for attribute: ", expected_attribute));
+  }
+  return true;
+}
+
+template <>
+bool HloParser::ParseAttributeValue<HloComputation*>(HloComputation** value) {
+  string name;
+  if (!ParseName(&name)) {
+    return TokenError("expects computation name");
+  }
+  *value = tensorflow::gtl::FindPtrOrNull(computation_pool_, name);
+  if (*value == nullptr) {
+    return TokenError(StrCat("computation does not exist: ", name));
+  }
+  return true;
+}
+
+template <>
+bool HloParser::ParseAttributeValue<int64>(int64* value) {
+  return ParseInt64(value);
 }
 
 // param_list ::= '(' param_list1 ')'
@@ -418,6 +597,15 @@ bool HloParser::ParseName(string* result) {
   return true;
 }
 
+bool HloParser::ParseAttributeName(string* result) {
+  if (lexer_.GetKind() != TokKind::kAttributeName) {
+    return TokenError("expects attribute name");
+  }
+  *result = lexer_.GetStrVal();
+  lexer_.Lex();
+  return true;
+}
+
 bool HloParser::ParseOpcode(HloOpcode* result) {
   VLOG(1) << "ParseOpcode";
   if (lexer_.GetKind() != TokKind::kOpcode) {
@@ -488,6 +676,15 @@ bool HloParser::AddInstruction(const string& name,
   return true;
 }
 
+bool HloParser::AddComputation(const string& name,
+                               HloComputation* computation) {
+  auto result = computation_pool_.insert({name, computation});
+  if (!result.second) {
+    return TokenError(StrCat("computation already exists: ", name));
+  }
+  return true;
+}
+
 }  // namespace
 
 StatusOr<std::unique_ptr<HloModule>> Parse(StringPiece str) {
diff --git a/tensorflow/compiler/xla/tools/parser/hlo_parser_test.cc b/tensorflow/compiler/xla/tools/parser/hlo_parser_test.cc
index 4ecece3eac..5150e1f96d 100644
--- a/tensorflow/compiler/xla/tools/parser/hlo_parser_test.cc
+++ b/tensorflow/compiler/xla/tools/parser/hlo_parser_test.cc
@@ -45,7 +45,7 @@ ENTRY %axpy.v5 (alpha: f32[2,4], x: f32[2,4], y: f32[2,4]) -> f32[2,4] {
   %x = f32[2,4]{1,0} parameter(1)
   %multiply = f32[2,4]{1,0} multiply(f32[2,4]{1,0} %alpha, f32[2,4]{1,0} %x)
   %y = f32[2,4]{1,0} parameter(2)
-  %add = f32[2,4]{1,0} add(f32[2,4]{1,0} %multiply, f32[2,4]{1,0} %y)
+  ROOT %add = f32[2,4]{1,0} add(f32[2,4]{1,0} %multiply, f32[2,4]{1,0} %y)
 }
 
 )"
@@ -56,7 +56,7 @@ ENTRY %axpy.v5 (alpha: f32[2,4], x: f32[2,4], y: f32[2,4]) -> f32[2,4] {
 R"(HloModule constant_pred_module:
 
 ENTRY %constant_pred () -> pred[] {
-  %constant = pred[] constant(true)
+  ROOT %constant = pred[] constant(true)
 }
 
 )"
@@ -67,7 +67,7 @@ ENTRY %constant_pred () -> pred[] {
 R"(HloModule constant_s32_module:
 
 ENTRY %constant_s32 () -> s32[] {
-  %constant = s32[] constant(-42)
+  ROOT %constant = s32[] constant(-42)
 }
 
 )"
@@ -77,7 +77,7 @@ ENTRY %constant_s32 () -> s32[] {
 "ConstantF32", R"(HloModule ConstantF32_module:
 
 ENTRY %ConstantF32.v4 () -> f32[] {
-  %constant = f32[] constant(42)
+  ROOT %constant = f32[] constant(42)
 }
 
 )"
@@ -89,7 +89,7 @@ R"(HloModule add_constants_module:
 
 ENTRY %add_constants () -> f32[] {
   %constant = f32[] constant(3.14)
-  %add = f32[] add(f32[] %constant, f32[] %constant)
+  ROOT %add = f32[] add(f32[] %constant, f32[] %constant)
 }
 
 )"
@@ -103,7 +103,100 @@ ENTRY %SelectR1F32WithCmpR1F32sFromParamsSmall.v4 (v1: f32[4], v2: f32[4]) -> f3
   %v1 = f32[4]{0} parameter(0)
   %v2 = f32[4]{0} parameter(1)
   %greater-than = pred[4]{0} greater-than(f32[4]{0} %v1, f32[4]{0} %v2)
-  %select = f32[4]{0} select(pred[4]{0} %greater-than, f32[4]{0} %v1, f32[4]{0} %v2)
+  ROOT %select = f32[4]{0} select(pred[4]{0} %greater-than, f32[4]{0} %v1, f32[4]{0} %v2)
+}
+
+)"
+},
+// empty tuple
+{
+"EmptyTupleCreate",
+R"(HloModule EmptyTupleCreate_module:
+
+ENTRY %EmptyTupleCreate.v1 () -> () {
+  ROOT %tuple = () tuple()
+}
+
+)"
+},
+// tuple
+{
+"TupleCreate",
+R"(HloModule TupleCreate_module:
+
+ENTRY %TupleCreate.v4 (v1: f32[], v2: f32[3], v3: f32[2,3]) -> (f32[], f32[3], f32[2,3]) {
+  %v1 = f32[] parameter(0)
+  %v2 = f32[3]{0} parameter(1)
+  %v3 = f32[2,3]{1,0} parameter(2)
+  ROOT %tuple = (f32[], f32[3]{0}, f32[2,3]{1,0}) tuple(f32[] %v1, f32[3]{0} %v2, f32[2,3]{1,0} %v3)
+}
+
+)"
+},
+// int32 result = 0;
+// while (result < 5) { result = result + 1; }
+{
+"WhileWithScalarS32Result",
+R"(HloModule WhileWithScalarS32Result_module:
+
+%body.v3 (prev.1: s32[]) -> s32[] {
+  %constant = s32[] constant(1)
+  %prev.1 = s32[] parameter(0)
+  ROOT %add = s32[] add(s32[] %constant, s32[] %prev.1)
+}
+
+%condition.v3 (prev.2: s32[]) -> pred[] {
+  %constant.1 = s32[] constant(5)
+  %prev.2 = s32[] parameter(0)
+  ROOT %greater-than = pred[] greater-than(s32[] %constant.1, s32[] %prev.2)
+}
+
+ENTRY %WhileWithScalarS32Result.v2 () -> s32[] {
+  %constant.2 = s32[] constant(0)
+  ROOT %while = s32[] while(s32[] %constant.2), condition=%condition.v3, body=%body.v3
+}
+
+)"
+},
+// send and recv
+{
+"SendRecv",
+R"(HloModule TwoSendRecvBothWayRecvFist_module:
+
+ENTRY %TwoSendRecvBothWayRecvFist.v3 () -> f32[] {
+  %recv = f32[] recv(), channel_id=15
+  ROOT %constant = f32[] constant(2.1)
+  %send = () send(f32[] %constant), channel_id=16
+}
+
+)"
+},
+// get-tuple-element
+{
+"GetTupleElement",
+R"(HloModule GetTupleElement_module:
+
+ENTRY %GetTupleElement.v4 () -> s32[] {
+  %constant = f32[] constant(1.23)
+  %constant.1 = s32[] constant(4)
+  %tuple = (f32[], s32[]) tuple(f32[] %constant, s32[] %constant.1)
+  ROOT %get-tuple-element = s32[] get-tuple-element((f32[], s32[]) %tuple), index=1
+}
+
+)"
+},
+// call
+{
+"Call",
+R"(HloModule CallR0F32IdentityScalar_module:
+
+%Identity.v1 (x: f32[]) -> f32[] {
+  ROOT %x = f32[] parameter(0)
+}
+
+ENTRY %CallR0F32IdentityScalar.v2 () -> f32[] {
+  %constant = f32[] constant(42)
+  ROOT %call = f32[] call(f32[] %constant), to_apply=%Identity.v1
 }
 
 )"
@@ -223,18 +316,6 @@ ENTRY %ConstantWithExp.v4 () -> f32[] {
   // printed as "300".
 }
 
-TEST_F(HloParserTest, Tuple) {
-  const string original = R"(HloModule EmptyTupleCreate_module:
-
-ENTRY %EmptyTupleCreate.v1 () -> () {
-  %tuple = () tuple()
-}
-
-)";
-  auto result = Parse(original);
-  EXPECT_NE(tensorflow::Status::OK(), result.status());
-}
-
 }  // namespace
 }  // namespace tools
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/tools/parser/hlo_token.h b/tensorflow/compiler/xla/tools/parser/hlo_token.h
index 1f75e17c7f..1d56ea3478 100644
--- a/tensorflow/compiler/xla/tools/parser/hlo_token.h
+++ b/tensorflow/compiler/xla/tools/parser/hlo_token.h
@@ -41,15 +41,17 @@ enum class TokKind {
   // Keywords
   kw_HloModule,
   kw_ENTRY,
+  kw_ROOT,
   kw_true,
   kw_false,
 
   // Typed tokens.
-  kName,     // %foo
-  kShape,    // f32[2,3]{1,0}
-  kOpcode,   // add
-  kInt,      // 42
-  kDecimal,  // 4.2
+  kName,           // %foo
+  kAttributeName,  // dimensions=
+  kShape,          // f32[2,3]{1,0}
+  kOpcode,         // add
+  kInt,            // 42
+  kDecimal,        // 4.2
 };
 
 }  // namespace tools
-- 
GitLab


From 4652341b0a80f93aa06a1b8669f04cb825336af6 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 24 Oct 2017 21:10:50 -0700
Subject: [PATCH 750/909] Better hint in eager device placement error message.

PiperOrigin-RevId: 173352246
---
 tensorflow/c/eager/c_api.cc         | 16 ++++++++++++----
 tensorflow/python/eager/ops_test.py |  3 ++-
 2 files changed, 14 insertions(+), 5 deletions(-)

diff --git a/tensorflow/c/eager/c_api.cc b/tensorflow/c/eager/c_api.cc
index 28ea2edee4..8359de62b7 100644
--- a/tensorflow/c/eager/c_api.cc
+++ b/tensorflow/c/eager/c_api.cc
@@ -440,11 +440,19 @@ tensorflow::Status ValidateInputTypeAndPlacement(
     if (expected_device != actual_device) {
       switch (ctx->policy) {
         case TFE_DEVICE_PLACEMENT_EXPLICIT:
+          // TODO(xpan): See if we could bubble python related error up
+          // to python level.
           return tensorflow::errors::InvalidArgument(
-              "cannot compute ", op->name, " as input #", i,
-              " was expected to be on ", expected_device->name(),
-              " but is actually on ", actual_device->name(),
-              " (operation running on ", op_device->name(), ")");
+              "Tensors on conflicting devices:"
+              " cannot compute ",
+              op->name, " as input #", i, " was expected to be on ",
+              expected_device->name(), " but is actually on ",
+              actual_device->name(), " (operation running on ",
+              op_device->name(), ")",
+              " Tensors can be copied explicitly using .gpu() or .cpu(),"
+              " or transparently copied by using tfe.enable_eager_execution("
+              "tfe.DEVICE_PLACEMENT_SILENT). Copying tensors between devices"
+              " may slow down your model");
         case TFE_DEVICE_PLACEMENT_WARN:
           LOG(WARNING) << "before computing " << op->name << " input #" << i
                        << " was expected to be on " << expected_device->name()
diff --git a/tensorflow/python/eager/ops_test.py b/tensorflow/python/eager/ops_test.py
index 1cd3826755..e34587d5b1 100644
--- a/tensorflow/python/eager/ops_test.py
+++ b/tensorflow/python/eager/ops_test.py
@@ -249,7 +249,8 @@ class OpsTest(test_util.TensorFlowTestCase):
     # it should implicitly copy the tensor to host memory?
     with self.assertRaisesRegexp(
         errors.InvalidArgumentError,
-        'cannot compute Reshape as input #1 was expected to be on'):
+        'cannot compute Reshape as input #1 was expected to be on.*'
+        'using.*DEVICE_PLACEMENT_SILENT'):
       reshaped = array_ops.reshape(value, shape.gpu())
 
   def testInvalidInputDataType(self):
-- 
GitLab


From ba3fa7f7732bf74341debee3661337d8d836681f Mon Sep 17 00:00:00 2001
From: Kay Zhu <kayzhu@google.com>
Date: Tue, 24 Oct 2017 23:42:11 -0700
Subject: [PATCH 751/909] [XLA] Remove the assumption that the non-CPU backend
 is the default if more than 1 platforms exist, now that ComputeConstant no
 longer requires a dedicated CPU backend.

PiperOrigin-RevId: 173360476
---
 tensorflow/compiler/xla/service/platform_util.cc | 9 ---------
 tensorflow/compiler/xla/service/platform_util.h  | 7 +------
 2 files changed, 1 insertion(+), 15 deletions(-)

diff --git a/tensorflow/compiler/xla/service/platform_util.cc b/tensorflow/compiler/xla/service/platform_util.cc
index 4f915a0c2e..3a1818de82 100644
--- a/tensorflow/compiler/xla/service/platform_util.cc
+++ b/tensorflow/compiler/xla/service/platform_util.cc
@@ -84,15 +84,6 @@ PlatformUtil::GetSupportedPlatforms() {
     return NotFound("no platforms found");
   } else if (platforms.size() == 1) {
     return platforms[0];
-  } else if (platforms.size() == 2) {
-    // In the service we always link the cpu backend for ComputeConstant. So if
-    // one of the two platforms is CPU then pick the other (non-cpu) platform as
-    // the default.
-    if (platforms[0]->id() == se::host::kHostPlatformId) {
-      return platforms[1];
-    } else if (platforms[1]->id() == se::host::kHostPlatformId) {
-      return platforms[0];
-    }
   }
 
   // Multiple platforms present and we can't pick a reasonable default.
diff --git a/tensorflow/compiler/xla/service/platform_util.h b/tensorflow/compiler/xla/service/platform_util.h
index fe0281a69a..eac5737030 100644
--- a/tensorflow/compiler/xla/service/platform_util.h
+++ b/tensorflow/compiler/xla/service/platform_util.h
@@ -36,12 +36,7 @@ class PlatformUtil {
 
   // Convenience function which returns the default supported platform. If
   // exactly one supported platform is present, then this platform is the
-  // default platform. If exactly two supported platforms are present and one
-  // platform is CPU (host) then the non-CPU platform is default. This logic is
-  // used because the XLA service always links in the CPU backend to run
-  // ComputeConstant, so if exactly one other platform is linked in, we assume
-  // the intent is to execute on that non-CPU platform. If none of these
-  // conditions are met the function returns an error.
+  // default platform. Otherwise returns an error.
   static StatusOr<perftools::gputools::Platform*> GetDefaultPlatform();
 
   // Returns a vector of StreamExecutors for the given platform. The vector is
-- 
GitLab


From 7828529df0abbbe9bf6c324616857b6e20636a2a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 25 Oct 2017 03:36:25 -0700
Subject: [PATCH 752/909] Internal change

PiperOrigin-RevId: 173378236
---
 tensorflow/compiler/xla/tests/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/compiler/xla/tests/BUILD b/tensorflow/compiler/xla/tests/BUILD
index 2ea7b9bd8e..4e1be24b61 100644
--- a/tensorflow/compiler/xla/tests/BUILD
+++ b/tensorflow/compiler/xla/tests/BUILD
@@ -920,6 +920,7 @@ xla_test(
     name = "reduce_window_test",
     timeout = "long",
     srcs = [],
+    tags = ["optonly"],
     xla_test_library_deps = [":reduce_window_test_library"],
     deps = [],
 )
-- 
GitLab


From 4251fc8e038efd6e18188ee5c1d2dbfa8418e58a Mon Sep 17 00:00:00 2001
From: Shanqing Cai <cais@google.com>
Date: Wed, 25 Oct 2017 05:25:16 -0700
Subject: [PATCH 753/909] TFE: Adjust formatting of @compatibility(eager)

PiperOrigin-RevId: 173385051
---
 .../python/ops/resource_variable_ops.py       |  7 ++--
 tensorflow/python/training/optimizer.py       | 38 ++++++++++---------
 2 files changed, 24 insertions(+), 21 deletions(-)

diff --git a/tensorflow/python/ops/resource_variable_ops.py b/tensorflow/python/ops/resource_variable_ops.py
index ce81a32924..06c5a3bb2a 100644
--- a/tensorflow/python/ops/resource_variable_ops.py
+++ b/tensorflow/python/ops/resource_variable_ops.py
@@ -180,7 +180,7 @@ class ResourceVariable(variables.Variable):
 
     @compatibility(eager)
     When Eager Execution is enabled, the default for the `collections` argument
-    is None, which signifies that this Variable will not be added to any
+    is `None`, which signifies that this `Variable` will not be added to any
     collections.
     @end_compatibility
     """
@@ -257,8 +257,9 @@ class ResourceVariable(variables.Variable):
 
     @compatibility(eager)
     When Eager Execution is enabled, variables are never added to collections.
-    It is not implicitly added to the GLOBAL_VARIABLES or TRAINABLE_VARIABLES
-    collections, and the `collections` argument is ignored.
+    It is not implicitly added to the `GLOBAL_VARIABLES` or
+    `TRAINABLE_VARIABLES` collections, and the `collections` argument is
+    ignored.
     @end_compatibility
     """
     if initial_value is None:
diff --git a/tensorflow/python/training/optimizer.py b/tensorflow/python/training/optimizer.py
index 82fc4edbcd..d6ca52cd1b 100644
--- a/tensorflow/python/training/optimizer.py
+++ b/tensorflow/python/training/optimizer.py
@@ -337,15 +337,16 @@ class Optimizer(object):
     Raises:
       ValueError: If some of the variables are not `Variable` objects.
 
-      @compatibility(eager):
-      When eager execution is enabled, `loss` should be a Python function that
-      takes elements of `var_list` as arguments and computes the value to be
-      minimized. If `var_list` is None, `loss` should take no arguments.
-      Minimization (and gradient computation) is done with respect to the
-      elements of `var_list` if not None, else with respect to any trainable
-      variables created during the execution of the `loss` function.
-      `gate_gradients`, `aggregation_method`, `colocate_gradients_with_ops` and
-      `grad_loss` are ignored when eager execution is enabled.
+    @compatibility(eager)
+    When eager execution is enabled, `loss` should be a Python function that
+    takes elements of `var_list` as arguments and computes the value to be
+    minimized. If `var_list` is None, `loss` should take no arguments.
+    Minimization (and gradient computation) is done with respect to the
+    elements of `var_list` if not None, else with respect to any trainable
+    variables created during the execution of the `loss` function.
+    `gate_gradients`, `aggregation_method`, `colocate_gradients_with_ops` and
+    `grad_loss` are ignored when eager execution is enabled.
+    @end_compatibility
     """
     grads_and_vars = self.compute_gradients(
         loss, var_list=var_list, gate_gradients=gate_gradients,
@@ -397,15 +398,16 @@ class Optimizer(object):
       TypeError: If `var_list` contains anything else than `Variable` objects.
       ValueError: If some arguments are invalid.
 
-      @compatibility(eager):
-      When eager execution is enabled, `loss` should be a Python function that
-      takes elements of `var_list` as arguments and computes the value to be
-      minimized. If `var_list` is None, `loss` should take no arguments.
-      Gradient computation is done with respect to the elements of `var_list` if
-      not None, else with respect to any trainable variables created during the
-      execution of the `loss` function.
-      `gate_gradients`, `aggregation_method`, `colocate_gradients_with_ops` and
-      `grad_loss` are ignored when eager execution is enabled.
+    @compatibility(eager)
+    When eager execution is enabled, `loss` should be a Python function that
+    takes elements of `var_list` as arguments and computes the value to be
+    minimized. If `var_list` is None, `loss` should take no arguments.
+    Gradient computation is done with respect to the elements of `var_list` if
+    not None, else with respect to any trainable variables created during the
+    execution of the `loss` function.
+    `gate_gradients`, `aggregation_method`, `colocate_gradients_with_ops` and
+    `grad_loss` are ignored when eager execution is enabled.
+    @end_compatibility
     """
     if context.in_eager_mode():
       if grad_loss is not None:
-- 
GitLab


From cac07d9c48afcae3ef09c07b8e5cab4eba285b2e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 25 Oct 2017 06:41:31 -0700
Subject: [PATCH 754/909] Small doc cleanup

PiperOrigin-RevId: 173390627
---
 tensorflow/compiler/xla/service/hlo_constant_folding.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_constant_folding.cc b/tensorflow/compiler/xla/service/hlo_constant_folding.cc
index b30c7b417f..c05bbeb5c9 100644
--- a/tensorflow/compiler/xla/service/hlo_constant_folding.cc
+++ b/tensorflow/compiler/xla/service/hlo_constant_folding.cc
@@ -63,8 +63,8 @@ StatusOr<bool> HloConstantFolding::Run(HloModule* module) {
         continue;
       }
 
-      // Broadcasts dramatically increase the size of constants with is often
-      // detrimental to performance and memory capacity so do not fold
+      // Broadcasts dramatically increase the size of constants, which is often
+      // detrimental to performance and memory capacity, so do not fold
       // broadcasts.
       if (instruction->opcode() == HloOpcode::kBroadcast) {
         continue;
-- 
GitLab


From 3d5d8732508b52e8697b616670dea39beabe8bcb Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 25 Oct 2017 07:33:40 -0700
Subject: [PATCH 755/909] Remove unused dependency

PiperOrigin-RevId: 173395407
---
 tensorflow/compiler/xla/BUILD | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tensorflow/compiler/xla/BUILD b/tensorflow/compiler/xla/BUILD
index 0129c51a09..660f419e46 100644
--- a/tensorflow/compiler/xla/BUILD
+++ b/tensorflow/compiler/xla/BUILD
@@ -340,7 +340,6 @@ cc_library(
     hdrs = ["array.h"],
     deps = [
         ":types",
-        ":util",
         "//tensorflow/core:lib",
     ],
 )
-- 
GitLab


From 90278e68c03964fdbb0371357feb7cb1d86bd09b Mon Sep 17 00:00:00 2001
From: Taehoon Lee <me@taehoonlee.com>
Date: Wed, 25 Oct 2017 23:57:36 +0900
Subject: [PATCH 756/909] Fix typos

---
 .../boosted_trees/python/training/functions/gbdt_batch.py       | 2 +-
 tensorflow/examples/get_started/regression/test.py              | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/boosted_trees/python/training/functions/gbdt_batch.py b/tensorflow/contrib/boosted_trees/python/training/functions/gbdt_batch.py
index f8f4b43a07..5a917ca428 100644
--- a/tensorflow/contrib/boosted_trees/python/training/functions/gbdt_batch.py
+++ b/tensorflow/contrib/boosted_trees/python/training/functions/gbdt_batch.py
@@ -344,7 +344,7 @@ class GradientBoostedDecisionTreeModel(object):
                         learner_config.num_classes == 2)
 
   def _predict_and_return_dict(self, ensemble_handle, ensemble_stamp, mode):
-    """Runs prediciton and returns a dictionary of the prediction results.
+    """Runs prediction and returns a dictionary of the prediction results.
 
     Args:
       ensemble_handle: ensemble resource handle.
diff --git a/tensorflow/examples/get_started/regression/test.py b/tensorflow/examples/get_started/regression/test.py
index 652b44f543..0b1477ad96 100644
--- a/tensorflow/examples/get_started/regression/test.py
+++ b/tensorflow/examples/get_started/regression/test.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""A simple smoke test that runs these examples for 1 training iteraton."""
+"""A simple smoke test that runs these examples for 1 training iteration."""
 
 from __future__ import absolute_import
 from __future__ import division
-- 
GitLab


From 2eeb6df0bd7c329163a6a25dd111a25a7b9ad16f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 25 Oct 2017 08:34:05 -0700
Subject: [PATCH 757/909] Fix SIGSEGV in GraphRunner::Run when called with a
 function library for a non-CPU device.

PiperOrigin-RevId: 173401446
---
 tensorflow/core/common_runtime/graph_runner.cc | 11 +++++++++++
 tensorflow/python/framework/function_test.py   | 15 ++++++++++++---
 2 files changed, 23 insertions(+), 3 deletions(-)

diff --git a/tensorflow/core/common_runtime/graph_runner.cc b/tensorflow/core/common_runtime/graph_runner.cc
index d0f9e6ed18..a21304f7ef 100644
--- a/tensorflow/core/common_runtime/graph_runner.cc
+++ b/tensorflow/core/common_runtime/graph_runner.cc
@@ -109,6 +109,17 @@ Status GraphRunner::Run(Graph* graph, FunctionLibraryRuntime* function_library,
     return errors::NotFound("Cannot find a device for GraphRunner.");
   }
 
+  if (function_library && function_library->device() &&
+      function_library->device()->device_type() != cpu_device_->device_type()) {
+    // We are running on a CPU but the function library is for a non-CPU device,
+    // so just ignore the function_library.
+    // TODO(matthewmurray) Can we create a new FunctionLibraryRuntime that is
+    // identical to function_library except that it uses CPU?
+    VLOG(1) << "Cannot run on CPU device with a function library for a "
+            << function_library->device()->device_type() << " device.";
+    function_library = nullptr;
+  }
+
   // TODO(vrv): Instead of copying the entire graph, consider modifying
   // the existing graph, and then removing those removed edges.
   // prior to returning.
diff --git a/tensorflow/python/framework/function_test.py b/tensorflow/python/framework/function_test.py
index fea2129922..fbc1045b5b 100644
--- a/tensorflow/python/framework/function_test.py
+++ b/tensorflow/python/framework/function_test.py
@@ -309,8 +309,7 @@ class FunctionTest(test.TestCase):
       self.assertAllClose(y.eval(), 6.)
       self.assertAllClose(dx.eval(), 2.)
 
-  def testZNoDepOnY(self):
-
+  def _testZNoDepOnY(self, use_const_grad_ys):
     @function.Defun(dtypes.float32, dtypes.float32)
     def Foo(x, y):  # pylint: disable=unused-argument
       return x * 2
@@ -320,12 +319,22 @@ class FunctionTest(test.TestCase):
       x = constant_op.constant(1.0)
       y = constant_op.constant(2.0)
       z = Foo(x, y)
-      dx, dy = gradients_impl.gradients([z], [x, y])
+      if use_const_grad_ys:
+        dx, dy = gradients_impl.gradients([z], [x, y], grad_ys=[1.0])
+      else:
+        dx, dy = gradients_impl.gradients([z], [x, y])
       with session.Session() as sess:
         dx_val, dy_val = sess.run([dx, dy])
         self.assertEqual([2.0], dx_val)
         self.assertEqual([0.0], dy_val)
 
+  def testZNoDepOnY(self):
+    self._testZNoDepOnY(False)
+
+  def testZNoDepOnYConstGradYs(self):
+    # Tests for constant folding of grad_ys
+    self._testZNoDepOnY(True)
+
   def testDefineFunctionNoArgs(self):
 
     @function.Defun(func_name="AConstant")
-- 
GitLab


From 8357c3164689ed2ce5df4df69ef7439b3b2fce82 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 25 Oct 2017 08:56:23 -0700
Subject: [PATCH 758/909] First part of the refactoring allowing sparse
 multivalent feature columns. This change extends the split proto to allow
 feature ids within the feature columns.

PiperOrigin-RevId: 173403860
---
 .../estimator_batch/custom_export_strategy.py |   4 +
 tensorflow/contrib/boosted_trees/lib/BUILD    |  12 ++
 .../boosted_trees/lib/trees/decision_tree.cc  |  25 ++-
 .../lib/trees/decision_tree_test.cc           | 145 ++++++++++++------
 .../boosted_trees/lib/utils/batch_features.cc |   4 -
 .../lib/utils/batch_features_test.cc          |  13 --
 .../contrib/boosted_trees/lib/utils/example.h |  56 ++++++-
 .../boosted_trees/lib/utils/example_test.cc   |  81 ++++++++++
 .../lib/utils/examples_iterable.cc            |   2 -
 .../lib/utils/examples_iterable.h             |  29 +++-
 .../lib/utils/examples_iterable_test.cc       | 112 +++++++++-----
 .../lib/utils/sparse_column_iterable.h        |   2 +
 .../lib/utils/sparse_column_iterable_test.cc  |  10 +-
 .../boosted_trees/proto/tree_config.proto     |   3 +
 14 files changed, 376 insertions(+), 122 deletions(-)
 create mode 100644 tensorflow/contrib/boosted_trees/lib/utils/example_test.cc

diff --git a/tensorflow/contrib/boosted_trees/estimator_batch/custom_export_strategy.py b/tensorflow/contrib/boosted_trees/estimator_batch/custom_export_strategy.py
index a800c3ddc7..ef8dee91b6 100644
--- a/tensorflow/contrib/boosted_trees/estimator_batch/custom_export_strategy.py
+++ b/tensorflow/contrib/boosted_trees/estimator_batch/custom_export_strategy.py
@@ -149,6 +149,8 @@ def convert_to_universal_format(dtec, sorted_feature_names,
           split = gtflow_node.sparse_float_binary_split_default_left.split
           node.default_direction = (
               generic_tree_model_pb2.BinaryNode.LEFT)
+          # TODO(nponomareva): adjust this id assignement when we allow multi-
+          # column sparse tensors.
           feature_id = split.feature_column + num_dense
           inequality_test = node.inequality_left_child_test
           inequality_test.feature_id.id.value = sorted_feature_names[feature_id]
@@ -159,6 +161,8 @@ def convert_to_universal_format(dtec, sorted_feature_names,
           split = gtflow_node.sparse_float_binary_split_default_right.split
           node.default_direction = (
               generic_tree_model_pb2.BinaryNode.RIGHT)
+          # TODO(nponomareva): adjust this id assignement when we allow multi-
+          # column sparse tensors.
           feature_id = split.feature_column + num_dense
           inequality_test = node.inequality_left_child_test
           inequality_test.feature_id.id.value = sorted_feature_names[feature_id]
diff --git a/tensorflow/contrib/boosted_trees/lib/BUILD b/tensorflow/contrib/boosted_trees/lib/BUILD
index 70aa0284a6..107ff0d295 100644
--- a/tensorflow/contrib/boosted_trees/lib/BUILD
+++ b/tensorflow/contrib/boosted_trees/lib/BUILD
@@ -81,6 +81,18 @@ tf_cc_test(
     ],
 )
 
+tf_cc_test(
+    name = "example_test",
+    size = "small",
+    srcs = ["utils/example_test.cc"],
+    deps = [
+        ":utils",
+        "//tensorflow/core:tensor_testutil",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+    ],
+)
+
 tf_cc_test(
     name = "batch_features_test",
     size = "small",
diff --git a/tensorflow/contrib/boosted_trees/lib/trees/decision_tree.cc b/tensorflow/contrib/boosted_trees/lib/trees/decision_tree.cc
index bd70586393..f8750e7191 100644
--- a/tensorflow/contrib/boosted_trees/lib/trees/decision_tree.cc
+++ b/tensorflow/contrib/boosted_trees/lib/trees/decision_tree.cc
@@ -50,10 +50,15 @@ int DecisionTree::Traverse(const DecisionTreeConfig& config,
             current_node.sparse_float_binary_split_default_left().split();
         auto sparse_feature =
             example.sparse_float_features[split.feature_column()];
-        node_id = !sparse_feature.has_value() ||
-                          sparse_feature.get_value() <= split.threshold()
-                      ? split.left_id()
-                      : split.right_id();
+        // Feature id for the split when multivalent sparse float column, or 0
+        // by default.
+        const int32 feature_id = split.feature_id();
+
+        node_id =
+            !sparse_feature[feature_id].has_value() ||
+                    sparse_feature[feature_id].get_value() <= split.threshold()
+                ? split.left_id()
+                : split.right_id();
         break;
       }
       case TreeNode::kSparseFloatBinarySplitDefaultRight: {
@@ -61,10 +66,14 @@ int DecisionTree::Traverse(const DecisionTreeConfig& config,
             current_node.sparse_float_binary_split_default_right().split();
         auto sparse_feature =
             example.sparse_float_features[split.feature_column()];
-        node_id = sparse_feature.has_value() &&
-                          sparse_feature.get_value() <= split.threshold()
-                      ? split.left_id()
-                      : split.right_id();
+        // Feature id for the split when multivalent sparse float column, or 0
+        // by default.
+        const int32 feature_id = split.feature_id();
+        node_id =
+            sparse_feature[feature_id].has_value() &&
+                    sparse_feature[feature_id].get_value() <= split.threshold()
+                ? split.left_id()
+                : split.right_id();
         break;
       }
       case TreeNode::kCategoricalIdBinarySplit: {
diff --git a/tensorflow/contrib/boosted_trees/lib/trees/decision_tree_test.cc b/tensorflow/contrib/boosted_trees/lib/trees/decision_tree_test.cc
index c55d09807e..93924d429c 100644
--- a/tensorflow/contrib/boosted_trees/lib/trees/decision_tree_test.cc
+++ b/tensorflow/contrib/boosted_trees/lib/trees/decision_tree_test.cc
@@ -27,13 +27,14 @@ class DecisionTreeTest : public ::testing::Test {
  protected:
   DecisionTreeTest() : batch_features_(2) {
     // Create a batch of two examples having one dense float, two sparse float
-    // and one sparse int features.
+    // and one sparse int features, and one sparse multi-column float feature
+    // (SparseFM).
     // The first example is missing the second sparse feature column and the
     // second example is missing the first sparse feature column.
     // This looks like the following:
-    // Instance | DenseF1 | SparseF1 | SparseF2 | SparseI1 |
-    // 0        |   7     |   -3     |          |    3     |
-    // 1        |  -2     |          |   4      |          |
+    // Instance | DenseF1 | SparseF1 | SparseF2 | SparseI1 | SparseFM (3 cols)
+    // 0        |   7     |   -3     |          |    3     | 3.0 |   | 1.0
+    // 1        |  -2     |          |   4      |          | 1.5 |3.5|
     auto dense_float_matrix = test::AsTensor<float>({7.0f, -2.0f}, {2, 1});
     auto sparse_float_indices1 = test::AsTensor<int64>({0, 0}, {1, 2});
     auto sparse_float_values1 = test::AsTensor<float>({-3.0f});
@@ -44,11 +45,21 @@ class DecisionTreeTest : public ::testing::Test {
     auto sparse_int_indices1 = test::AsTensor<int64>({0, 0}, {1, 2});
     auto sparse_int_values1 = test::AsTensor<int64>({3});
     auto sparse_int_shape1 = test::AsTensor<int64>({2, 1});
+
+    // Multivalent sparse feature.
+    auto multi_sparse_float_indices =
+        test::AsTensor<int64>({0, 0, 0, 2, 1, 0, 1, 1}, {4, 2});
+    auto multi_sparse_float_values =
+        test::AsTensor<float>({3.0f, 1.0f, 1.5f, 3.5f});
+    auto multi_sparse_float_shape = test::AsTensor<int64>({2, 3});
+
     TF_EXPECT_OK(batch_features_.Initialize(
-        {dense_float_matrix}, {sparse_float_indices1, sparse_float_indices2},
-        {sparse_float_values1, sparse_float_values2},
-        {sparse_float_shape1, sparse_float_shape2}, {sparse_int_indices1},
-        {sparse_int_values1}, {sparse_int_shape1}));
+        {dense_float_matrix},
+        {sparse_float_indices1, sparse_float_indices2,
+         multi_sparse_float_indices},
+        {sparse_float_values1, sparse_float_values2, multi_sparse_float_values},
+        {sparse_float_shape1, sparse_float_shape2, multi_sparse_float_shape},
+        {sparse_int_indices1}, {sparse_int_values1}, {sparse_int_shape1}));
   }
 
   template <typename SplitType>
@@ -121,44 +132,90 @@ TEST_F(DecisionTreeTest, TraverseDenseBinarySplit) {
 }
 
 TEST_F(DecisionTreeTest, TraverseSparseBinarySplit) {
-  // Test first sparse feature which is missing for the second example.
-  DecisionTreeConfig tree_config1;
-  auto* split_node1 = tree_config1.add_nodes()
-                          ->mutable_sparse_float_binary_split_default_left()
-                          ->mutable_split();
-  split_node1->set_feature_column(0);
-  split_node1->set_threshold(-20.0f);
-  split_node1->set_left_id(1);
-  split_node1->set_right_id(2);
-  tree_config1.add_nodes()->mutable_leaf();
-  tree_config1.add_nodes()->mutable_leaf();
   auto example_iterable = batch_features_.examples_iterable(0, 2);
-
-  // Expect right child to be picked as !(-3 <= -20).
-  auto example_it = example_iterable.begin();
-  EXPECT_EQ(2, DecisionTree::Traverse(tree_config1, 0, *example_it));
-
-  // Expect left child to be picked as default direction.
-  EXPECT_EQ(1, DecisionTree::Traverse(tree_config1, 0, *++example_it));
-
+  // Split on SparseF1.
+  // Test first sparse feature which is missing for the second example.
+  {
+    DecisionTreeConfig tree_config;
+    auto* split_node = tree_config.add_nodes()
+                           ->mutable_sparse_float_binary_split_default_left()
+                           ->mutable_split();
+    split_node->set_feature_column(0);
+    split_node->set_threshold(-20.0f);
+    split_node->set_left_id(1);
+    split_node->set_right_id(2);
+    tree_config.add_nodes()->mutable_leaf();
+    tree_config.add_nodes()->mutable_leaf();
+
+    // Expect right child to be picked as !(-3 <= -20).
+    auto example_it = example_iterable.begin();
+    EXPECT_EQ(2, DecisionTree::Traverse(tree_config, 0, *example_it));
+
+    // Expect left child to be picked as default direction.
+    EXPECT_EQ(1, DecisionTree::Traverse(tree_config, 0, *++example_it));
+  }
+  // Split on SparseF2.
   // Test second sparse feature which is missing for the first example.
-  DecisionTreeConfig tree_config2;
-  auto* split_node2 = tree_config2.add_nodes()
-                          ->mutable_sparse_float_binary_split_default_right()
-                          ->mutable_split();
-  split_node2->set_feature_column(1);
-  split_node2->set_threshold(4.0f);
-  split_node2->set_left_id(1);
-  split_node2->set_right_id(2);
-  tree_config2.add_nodes()->mutable_leaf();
-  tree_config2.add_nodes()->mutable_leaf();
-
-  // Expect right child to be picked as default direction.
-  example_it = example_iterable.begin();
-  EXPECT_EQ(2, DecisionTree::Traverse(tree_config2, 0, *example_it));
-
-  // Expect left child to be picked as (4 <= 4).
-  EXPECT_EQ(1, DecisionTree::Traverse(tree_config2, 0, *++example_it));
+  {
+    DecisionTreeConfig tree_config;
+    auto* split_node = tree_config.add_nodes()
+                           ->mutable_sparse_float_binary_split_default_right()
+                           ->mutable_split();
+    split_node->set_feature_column(1);
+    split_node->set_threshold(4.0f);
+    split_node->set_left_id(1);
+    split_node->set_right_id(2);
+    tree_config.add_nodes()->mutable_leaf();
+    tree_config.add_nodes()->mutable_leaf();
+
+    // Expect right child to be picked as default direction.
+    auto example_it = example_iterable.begin();
+    EXPECT_EQ(2, DecisionTree::Traverse(tree_config, 0, *example_it));
+
+    // Expect left child to be picked as (4 <= 4).
+    EXPECT_EQ(1, DecisionTree::Traverse(tree_config, 0, *++example_it));
+  }
+  // Split on SparseFM.
+  // Test second sparse feature which is missing for the first example.
+  {
+    DecisionTreeConfig tree_config;
+    auto* split_node = tree_config.add_nodes()
+                           ->mutable_sparse_float_binary_split_default_right()
+                           ->mutable_split();
+    split_node->set_feature_column(2);
+
+    split_node->set_left_id(1);
+    split_node->set_right_id(2);
+    tree_config.add_nodes()->mutable_leaf();
+    tree_config.add_nodes()->mutable_leaf();
+
+    // Split on first column
+    split_node->set_feature_id(0);
+    split_node->set_threshold(2.0f);
+
+    // Both instances have this feature value.
+    auto example_it = example_iterable.begin();
+    EXPECT_EQ(2, DecisionTree::Traverse(tree_config, 0, *example_it));
+    EXPECT_EQ(1, DecisionTree::Traverse(tree_config, 0, *++example_it));
+
+    // Split on second column
+    split_node->set_feature_id(1);
+    split_node->set_threshold(5.0f);
+
+    // First instance does not have it (default right), second does have it.
+    example_it = example_iterable.begin();
+    EXPECT_EQ(2, DecisionTree::Traverse(tree_config, 0, *example_it));
+    EXPECT_EQ(1, DecisionTree::Traverse(tree_config, 0, *++example_it));
+
+    // Split on third column
+    split_node->set_feature_id(2);
+    split_node->set_threshold(3.0f);
+    example_it = example_iterable.begin();
+
+    // First instance has it, second does not (default right).
+    EXPECT_EQ(1, DecisionTree::Traverse(tree_config, 0, *example_it));
+    EXPECT_EQ(2, DecisionTree::Traverse(tree_config, 0, *++example_it));
+  }
 }
 
 TEST_F(DecisionTreeTest, TraverseCategoricalIdBinarySplit) {
diff --git a/tensorflow/contrib/boosted_trees/lib/utils/batch_features.cc b/tensorflow/contrib/boosted_trees/lib/utils/batch_features.cc
index 12b377dda7..cf4f9a097a 100644
--- a/tensorflow/contrib/boosted_trees/lib/utils/batch_features.cc
+++ b/tensorflow/contrib/boosted_trees/lib/utils/batch_features.cc
@@ -94,10 +94,6 @@ Status BatchFeatures::Initialize(
         shape_flat(0) == batch_size_,
         errors::InvalidArgument(
             "Sparse float feature shape incompatible with batch size."));
-    TF_CHECK_AND_RETURN_IF_ERROR(
-        shape_flat(1) <= 1,
-        errors::InvalidArgument(
-            "Sparse float features may not be multi-valent."));
     auto tensor_shape = TensorShape({shape_flat(0), shape_flat(1)});
     auto order_dims = sparse::SparseTensor::VarDimArray({0, 1});
     sparse_float_feature_columns_.emplace_back(sparse_float_feature_indices,
diff --git a/tensorflow/contrib/boosted_trees/lib/utils/batch_features_test.cc b/tensorflow/contrib/boosted_trees/lib/utils/batch_features_test.cc
index 7f523d527a..9de3e32b09 100644
--- a/tensorflow/contrib/boosted_trees/lib/utils/batch_features_test.cc
+++ b/tensorflow/contrib/boosted_trees/lib/utils/batch_features_test.cc
@@ -129,19 +129,6 @@ TEST_F(BatchFeaturesTest, SparseFloatFeatures_IncompatibleShape) {
                                 {sparse_float_feature_shape}, {}, {}, {}));
 }
 
-TEST_F(BatchFeaturesTest, SparseFloatFeatures_Multivalent) {
-  BatchFeatures batch_features(2);
-  auto sparse_float_feature_indices = AsTensor<int64>({0, 0, 1, 0}, {2, 2});
-  auto sparse_float_feature_values = AsTensor<float>({3.0f, 7.0f});
-  auto sparse_float_feature_shape = AsTensor<int64>({2, 2});
-  auto expected_error =
-      InvalidArgument("Sparse float features may not be multi-valent.");
-  EXPECT_EQ(expected_error, batch_features.Initialize(
-                                {}, {sparse_float_feature_indices},
-                                {sparse_float_feature_values},
-                                {sparse_float_feature_shape}, {}, {}, {}));
-}
-
 TEST_F(BatchFeaturesTest, SparseIntFeatures_WrongShapeIndices) {
   BatchFeatures batch_features(2);
   auto sparse_int_feature_indices = AsTensor<int64>({0, 0, 1, 0});
diff --git a/tensorflow/contrib/boosted_trees/lib/utils/example.h b/tensorflow/contrib/boosted_trees/lib/utils/example.h
index 4681eb06aa..9514416660 100644
--- a/tensorflow/contrib/boosted_trees/lib/utils/example.h
+++ b/tensorflow/contrib/boosted_trees/lib/utils/example.h
@@ -16,6 +16,8 @@
 #ifndef THIRD_PARTY_TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_UTILS_EXAMPLE_H_
 #define THIRD_PARTY_TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_UTILS_EXAMPLE_H_
 
+#include <algorithm>
+#include <unordered_map>
 #include <unordered_set>
 #include <vector>
 #include "tensorflow/contrib/boosted_trees/lib/utils/optional_value.h"
@@ -24,6 +26,56 @@ namespace tensorflow {
 namespace boosted_trees {
 namespace utils {
 
+// A matrix that given feature column id and feature value id will return
+// either a value or an optional. First index indicates feature column, second
+// index - the index of the value within this column - for single valued, it
+// will be 0.
+// Allows double-subscript access [][].
+template <class T>
+class SparseMatrix {
+  typedef std::vector<std::tuple<int32, int32, T>> SparseMap;
+
+  class Proxy {
+   public:
+    Proxy(const int32 feature_column_idx, const SparseMap& values)
+        : feature_column_idx_(feature_column_idx), values_(values) {}
+
+    OptionalValue<T> operator[](int feature_idx) const {
+      auto value_iter = std::find_if(
+          values_.begin(), values_.end(),
+          [this, &feature_idx](const std::tuple<int32, int32, T>& element) {
+            return std::get<0>(element) == feature_column_idx_ &&
+                   std::get<1>(element) == feature_idx;
+          });
+
+      if (value_iter == values_.end()) {
+        return OptionalValue<T>();
+      }
+      // There is this feature column and feature id.
+      return OptionalValue<T>(std::get<2>(*value_iter));
+    }
+
+   private:
+    int32 feature_column_idx_;
+    const SparseMap& values_;
+  };
+
+ public:
+  void addElement(const int32 feature_column_idx, const int32 feature_idx,
+                  const T value) {
+    values_.emplace_back(feature_column_idx, feature_idx, value);
+  }
+
+  void clear() { values_.clear(); }
+
+  Proxy operator[](int feature_column_idx) const {
+    return Proxy(feature_column_idx, values_);
+  }
+
+ private:
+  SparseMap values_;
+};
+
 // Holds data for one example and enables lookup by feature column.
 struct Example {
   // Default constructor creates an empty example.
@@ -35,7 +87,9 @@ struct Example {
   // Dense and sparse float features indexed by feature column.
   // TODO(salehay): figure out a design to support multivalent float features.
   std::vector<float> dense_float_features;
-  std::vector<OptionalValue<float>> sparse_float_features;
+  // Sparse float features are allowed to be multivalent and thus can be
+  // represented as a sparse matrix.
+  SparseMatrix<float> sparse_float_features;
 
   // Sparse integer features indexed by feature column.
   // Note that all integer features are assumed to be categorical, i.e. will
diff --git a/tensorflow/contrib/boosted_trees/lib/utils/example_test.cc b/tensorflow/contrib/boosted_trees/lib/utils/example_test.cc
new file mode 100644
index 0000000000..f78fd25022
--- /dev/null
+++ b/tensorflow/contrib/boosted_trees/lib/utils/example_test.cc
@@ -0,0 +1,81 @@
+// Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// =============================================================================
+#include "tensorflow/contrib/boosted_trees/lib/utils/example.h"
+#include "tensorflow/core/framework/tensor_testutil.h"
+#include "tensorflow/core/lib/core/status_test_util.h"
+#include "tensorflow/core/platform/test.h"
+
+namespace tensorflow {
+namespace boosted_trees {
+namespace utils {
+namespace {
+
+class ExampleTest : public ::testing::Test {};
+
+TEST_F(ExampleTest, TestSparseMatrix) {
+  // Create the following matrix:
+  // row id |   | 0.4 |  0.3
+  // 0      | 1 |     |   2
+  // 1      | 3 |  1  |   5
+  // 2      |   |     |  -4
+  // 3      |   |     |
+  SparseMatrix<float> matrix;
+  matrix.addElement(0, 1, 0.4f);
+  matrix.addElement(0, 2, 0.3f);
+  matrix.addElement(1, 0, 1.f);
+  matrix.addElement(1, 2, 2.f);
+  matrix.addElement(2, 0, 3.f);
+  matrix.addElement(2, 1, 1.f);
+  matrix.addElement(2, 2, 5.f);
+  matrix.addElement(3, 2, -4.f);
+
+  // Row 0.
+  EXPECT_FALSE(matrix[0][0].has_value());
+  EXPECT_TRUE(matrix[0][1].has_value());
+  EXPECT_EQ(0.4f, matrix[0][1].get_value());
+  EXPECT_TRUE(matrix[0][2].has_value());
+  EXPECT_EQ(0.3f, matrix[0][2].get_value());
+
+  // Row 1.
+  EXPECT_TRUE(matrix[1][0].has_value());
+  EXPECT_EQ(1.f, matrix[1][0].get_value());
+  EXPECT_FALSE(matrix[1][1].has_value());
+  EXPECT_TRUE(matrix[1][2].has_value());
+  EXPECT_EQ(2.f, matrix[1][2].get_value());
+
+  // Row 2.
+  EXPECT_TRUE(matrix[2][0].has_value());
+  EXPECT_EQ(3.f, matrix[2][0].get_value());
+  EXPECT_TRUE(matrix[2][1].has_value());
+  EXPECT_EQ(1.f, matrix[2][1].get_value());
+  EXPECT_TRUE(matrix[2][2].has_value());
+  EXPECT_EQ(5.f, matrix[2][2].get_value());
+
+  // Row 3.
+  EXPECT_FALSE(matrix[3][0].has_value());
+  EXPECT_FALSE(matrix[3][1].has_value());
+  EXPECT_TRUE(matrix[3][2].has_value());
+  EXPECT_EQ(-4.f, matrix[3][2].get_value());
+
+  // Row 4.
+  EXPECT_FALSE(matrix[4][0].has_value());
+  EXPECT_FALSE(matrix[4][1].has_value());
+  EXPECT_FALSE(matrix[4][2].has_value());
+}
+
+}  // namespace
+}  // namespace utils
+}  // namespace boosted_trees
+}  // namespace tensorflow
diff --git a/tensorflow/contrib/boosted_trees/lib/utils/examples_iterable.cc b/tensorflow/contrib/boosted_trees/lib/utils/examples_iterable.cc
index c73dc8e15d..3b287b1dcf 100644
--- a/tensorflow/contrib/boosted_trees/lib/utils/examples_iterable.cc
+++ b/tensorflow/contrib/boosted_trees/lib/utils/examples_iterable.cc
@@ -73,8 +73,6 @@ Iterator::Iterator(ExamplesIterable* iter, int64 example_idx)
   // Pre-size example features.
   example_.dense_float_features.resize(
       iter_->dense_float_column_values_.size());
-  example_.sparse_float_features.resize(
-      iter_->sparse_float_column_values_.size());
   example_.sparse_int_features.resize(iter_->sparse_int_column_values_.size());
 }
 
diff --git a/tensorflow/contrib/boosted_trees/lib/utils/examples_iterable.h b/tensorflow/contrib/boosted_trees/lib/utils/examples_iterable.h
index 67efb82a22..72b7486872 100644
--- a/tensorflow/contrib/boosted_trees/lib/utils/examples_iterable.h
+++ b/tensorflow/contrib/boosted_trees/lib/utils/examples_iterable.h
@@ -87,19 +87,34 @@ class ExamplesIterable {
 
       // Get sparse float values per column.
       auto& sparse_float_features = example_.sparse_float_features;
+      sparse_float_features.clear();
+      // Iterate through each sparse float feature column.
       for (size_t sparse_float_idx = 0;
-           sparse_float_idx < sparse_float_features.size();
+           sparse_float_idx < iter_->sparse_float_column_iterables_.size();
            ++sparse_float_idx) {
+        // Get range for values tensor.
         const auto& row_range =
             (*sparse_float_column_iterators_[sparse_float_idx]);
         DCHECK_EQ(example_idx_, row_range.example_idx);
+        // If the example has this feature column.
         if (row_range.start < row_range.end) {
-          DCHECK_EQ(1, row_range.end - row_range.start);
-          sparse_float_features[sparse_float_idx] = OptionalValue<float>(
-              iter_->sparse_float_column_values_[sparse_float_idx](
-                  row_range.start));
-        } else {
-          sparse_float_features[sparse_float_idx] = OptionalValue<float>();
+          // Retrieve original indices tensor.
+          const TTypes<int64>::ConstMatrix& indices =
+              iter_->sparse_float_column_iterables_[sparse_float_idx]
+                  .sparse_indices();
+
+          // For each value.
+          for (int64 row_idx = row_range.start; row_idx < row_range.end;
+               ++row_idx) {
+            // Get the feature id for the feature column and the value.
+            const int32 feature_id = indices(row_idx, 1);
+            DCHECK_EQ(example_idx_, indices(row_idx, 0));
+
+            // Save the value to our sparse matrix.
+            sparse_float_features.addElement(
+                sparse_float_idx, feature_id,
+                iter_->sparse_float_column_values_[sparse_float_idx](row_idx));
+          }
         }
       }
 
diff --git a/tensorflow/contrib/boosted_trees/lib/utils/examples_iterable_test.cc b/tensorflow/contrib/boosted_trees/lib/utils/examples_iterable_test.cc
index d93bcc8aa6..05c166edc6 100644
--- a/tensorflow/contrib/boosted_trees/lib/utils/examples_iterable_test.cc
+++ b/tensorflow/contrib/boosted_trees/lib/utils/examples_iterable_test.cc
@@ -26,17 +26,17 @@ class ExamplesIterableTest : public ::testing::Test {};
 
 TEST_F(ExamplesIterableTest, Iterate) {
   // Create a batch of 8 examples having one dense float, two sparse float and
-  // two sparse int features.
+  // two sparse int features. Second sparse float feature is multivalent.
   // The data looks like the following:
   // Instance | DenseF1 | SparseF1 | SparseF2 | SparseI1 | SparseI2 |
-  // 0        |   7     |   -3     |          |   1, 8   |          |
-  // 1        |  -2     |          |    4     |    0     |    7     |
-  // 2        |   8     |    0     |          |          |    13    |
-  // 3        |   1     |    5     |    7     |   2, 0   |    4     |
-  // 4        |   0     |    0     |          |          |    0     |
-  // 5        |  -4     |          |    9     |          |          |
-  // 6        |   7     |          |          |          |          |
-  // 7        |  -2     |          |   -4     |     5    |          |
+  // 0        |   7     |   -3     |    |  1  |   1, 8   |          |
+  // 1        |  -2     |          |  4 |     |    0     |    7     |
+  // 2        |   8     |    0     |    |  3  |          |    13    |
+  // 3        |   1     |    5     |  7 |     |   2, 0   |    4     |
+  // 4        |   0     |    0     |    | 4.3 |          |    0     |
+  // 5        |  -4     |          |  9 | 0.8 |          |          |
+  // 6        |   7     |          |    |     |          |          |
+  // 7        |  -2     |          | -4 |     |     5    |          |
   auto dense_float_tensor = test::AsTensor<float>(
       {7.0f, -2.0f, 8.0f, 1.0f, 0.0f, -4.0f, 7.0f, -2.0f}, {8, 1});
   auto sparse_float_indices1 =
@@ -45,10 +45,11 @@ TEST_F(ExamplesIterableTest, Iterate) {
   auto sparse_float_shape1 = TensorShape({8, 1});
   sparse::SparseTensor sparse_float_tensor1(
       sparse_float_indices1, sparse_float_values1, sparse_float_shape1);
-  auto sparse_float_indices2 =
-      test::AsTensor<int64>({1, 0, 3, 0, 5, 0, 7, 0}, {4, 2});
-  auto sparse_float_values2 = test::AsTensor<float>({4.0f, 7.0f, 9.0f, -4.0f});
-  auto sparse_float_shape2 = TensorShape({8, 1});
+  auto sparse_float_indices2 = test::AsTensor<int64>(
+      {0, 1, 1, 0, 2, 1, 3, 0, 4, 1, 5, 0, 5, 1, 7, 0}, {8, 2});
+  auto sparse_float_values2 =
+      test::AsTensor<float>({1.f, 4.0f, 3.f, 7.0f, 4.3f, 9.0f, 0.8f, -4.0f});
+  auto sparse_float_shape2 = TensorShape({8, 2});
   sparse::SparseTensor sparse_float_tensor2(
       sparse_float_indices2, sparse_float_values2, sparse_float_shape2);
   auto sparse_int_indices1 =
@@ -67,15 +68,19 @@ TEST_F(ExamplesIterableTest, Iterate) {
   auto validate_example_features = [](int64 example_idx,
                                       const Example& example) {
     EXPECT_EQ(1, example.dense_float_features.size());
-    EXPECT_EQ(2, example.sparse_float_features.size());
 
     switch (example_idx) {
       case 0: {
         EXPECT_EQ(0, example.example_idx);
         EXPECT_EQ(7.0f, example.dense_float_features[0]);
-        EXPECT_TRUE(example.sparse_float_features[0].has_value());
-        EXPECT_EQ(-3.0f, example.sparse_float_features[0].get_value());
-        EXPECT_FALSE(example.sparse_float_features[1].has_value());
+        // SparseF1.
+        EXPECT_TRUE(example.sparse_float_features[0][0].has_value());
+        EXPECT_EQ(-3.0f, example.sparse_float_features[0][0].get_value());
+        // SparseF2 - multivalent.
+        EXPECT_FALSE(example.sparse_float_features[1][0].has_value());
+        EXPECT_TRUE(example.sparse_float_features[1][1].has_value());
+        EXPECT_EQ(1.0f, example.sparse_float_features[1][1].get_value());
+
         EXPECT_EQ(2, example.sparse_int_features[0].size());
         EXPECT_EQ(1, example.sparse_int_features[0].count(1));
         EXPECT_EQ(1, example.sparse_int_features[0].count(8));
@@ -84,9 +89,13 @@ TEST_F(ExamplesIterableTest, Iterate) {
       case 1: {
         EXPECT_EQ(1, example.example_idx);
         EXPECT_EQ(-2.0f, example.dense_float_features[0]);
-        EXPECT_FALSE(example.sparse_float_features[0].has_value());
-        EXPECT_TRUE(example.sparse_float_features[1].has_value());
-        EXPECT_EQ(4.0f, example.sparse_float_features[1].get_value());
+        // SparseF1.
+        EXPECT_FALSE(example.sparse_float_features[0][0].has_value());
+        // SparseF2.
+        EXPECT_TRUE(example.sparse_float_features[1][0].has_value());
+        EXPECT_EQ(4.0f, example.sparse_float_features[1][0].get_value());
+        EXPECT_FALSE(example.sparse_float_features[1][1].has_value());
+
         EXPECT_EQ(1, example.sparse_int_features[0].size());
         EXPECT_EQ(1, example.sparse_int_features[0].count(0));
         EXPECT_EQ(1, example.sparse_int_features[1].size());
@@ -95,9 +104,14 @@ TEST_F(ExamplesIterableTest, Iterate) {
       case 2: {
         EXPECT_EQ(2, example.example_idx);
         EXPECT_EQ(8.0f, example.dense_float_features[0]);
-        EXPECT_TRUE(example.sparse_float_features[0].has_value());
-        EXPECT_EQ(0.0f, example.sparse_float_features[0].get_value());
-        EXPECT_FALSE(example.sparse_float_features[1].has_value());
+        // SparseF1.
+        EXPECT_TRUE(example.sparse_float_features[0][0].has_value());
+        EXPECT_EQ(0.0f, example.sparse_float_features[0][0].get_value());
+        // SparseF2.
+        EXPECT_FALSE(example.sparse_float_features[1][0].has_value());
+        EXPECT_TRUE(example.sparse_float_features[1][1].has_value());
+        EXPECT_EQ(3.f, example.sparse_float_features[1][1].get_value());
+
         EXPECT_EQ(0, example.sparse_int_features[0].size());
         EXPECT_EQ(1, example.sparse_int_features[1].size());
         EXPECT_EQ(1, example.sparse_int_features[1].count(13));
@@ -105,10 +119,14 @@ TEST_F(ExamplesIterableTest, Iterate) {
       case 3: {
         EXPECT_EQ(3, example.example_idx);
         EXPECT_EQ(1.0f, example.dense_float_features[0]);
-        EXPECT_TRUE(example.sparse_float_features[0].has_value());
-        EXPECT_EQ(5.0f, example.sparse_float_features[0].get_value());
-        EXPECT_TRUE(example.sparse_float_features[1].has_value());
-        EXPECT_EQ(7.0f, example.sparse_float_features[1].get_value());
+        // SparseF1.
+        EXPECT_TRUE(example.sparse_float_features[0][0].has_value());
+        EXPECT_EQ(5.0f, example.sparse_float_features[0][0].get_value());
+        // SparseF2.
+        EXPECT_TRUE(example.sparse_float_features[1][0].has_value());
+        EXPECT_EQ(7.0f, example.sparse_float_features[1][0].get_value());
+        EXPECT_FALSE(example.sparse_float_features[1][1].has_value());
+
         EXPECT_EQ(2, example.sparse_int_features[0].size());
         EXPECT_EQ(1, example.sparse_int_features[0].count(2));
         EXPECT_EQ(1, example.sparse_int_features[0].count(0));
@@ -118,9 +136,14 @@ TEST_F(ExamplesIterableTest, Iterate) {
       case 4: {
         EXPECT_EQ(4, example.example_idx);
         EXPECT_EQ(0.0f, example.dense_float_features[0]);
-        EXPECT_TRUE(example.sparse_float_features[0].has_value());
-        EXPECT_EQ(0.0f, example.sparse_float_features[0].get_value());
-        EXPECT_FALSE(example.sparse_float_features[1].has_value());
+        // SparseF1.
+        EXPECT_TRUE(example.sparse_float_features[0][0].has_value());
+        EXPECT_EQ(0.0f, example.sparse_float_features[0][0].get_value());
+        // SparseF2.
+        EXPECT_FALSE(example.sparse_float_features[1][0].has_value());
+        EXPECT_TRUE(example.sparse_float_features[1][1].has_value());
+        EXPECT_EQ(4.3f, example.sparse_float_features[1][1].get_value());
+
         EXPECT_EQ(0, example.sparse_int_features[0].size());
         EXPECT_EQ(1, example.sparse_int_features[1].size());
         EXPECT_EQ(1, example.sparse_int_features[1].count(0));
@@ -128,24 +151,37 @@ TEST_F(ExamplesIterableTest, Iterate) {
       case 5: {
         EXPECT_EQ(5, example.example_idx);
         EXPECT_EQ(-4.0f, example.dense_float_features[0]);
-        EXPECT_FALSE(example.sparse_float_features[0].has_value());
-        EXPECT_TRUE(example.sparse_float_features[1].has_value());
-        EXPECT_EQ(9.0f, example.sparse_float_features[1].get_value());
+        // SparseF1.
+        EXPECT_FALSE(example.sparse_float_features[0][0].has_value());
+        // SparseF2.
+        EXPECT_TRUE(example.sparse_float_features[1][0].has_value());
+        EXPECT_EQ(9.0f, example.sparse_float_features[1][0].get_value());
+        EXPECT_TRUE(example.sparse_float_features[1][1].has_value());
+        EXPECT_EQ(0.8f, example.sparse_float_features[1][1].get_value());
+
         EXPECT_EQ(0, example.sparse_int_features[0].size());
       } break;
       case 6: {
         EXPECT_EQ(6, example.example_idx);
         EXPECT_EQ(7.0f, example.dense_float_features[0]);
-        EXPECT_FALSE(example.sparse_float_features[0].has_value());
-        EXPECT_FALSE(example.sparse_float_features[1].has_value());
+        // SparseF1.
+        EXPECT_FALSE(example.sparse_float_features[0][0].has_value());
+        // SparseF2.
+        EXPECT_FALSE(example.sparse_float_features[1][0].has_value());
+        EXPECT_FALSE(example.sparse_float_features[1][1].has_value());
+
         EXPECT_EQ(0, example.sparse_int_features[0].size());
       } break;
       case 7: {
         EXPECT_EQ(7, example.example_idx);
         EXPECT_EQ(-2.0f, example.dense_float_features[0]);
-        EXPECT_FALSE(example.sparse_float_features[0].has_value());
-        EXPECT_TRUE(example.sparse_float_features[1].has_value());
-        EXPECT_EQ(-4.0f, example.sparse_float_features[1].get_value());
+        // SparseF1.
+        EXPECT_FALSE(example.sparse_float_features[0][0].has_value());
+        // SparseF2.
+        EXPECT_TRUE(example.sparse_float_features[1][0].has_value());
+        EXPECT_EQ(-4.0f, example.sparse_float_features[1][0].get_value());
+        EXPECT_FALSE(example.sparse_float_features[1][1].has_value());
+
         EXPECT_EQ(1, example.sparse_int_features[0].size());
         EXPECT_EQ(1, example.sparse_int_features[0].count(5));
       } break;
diff --git a/tensorflow/contrib/boosted_trees/lib/utils/sparse_column_iterable.h b/tensorflow/contrib/boosted_trees/lib/utils/sparse_column_iterable.h
index 78a5752730..9664c9d1c6 100644
--- a/tensorflow/contrib/boosted_trees/lib/utils/sparse_column_iterable.h
+++ b/tensorflow/contrib/boosted_trees/lib/utils/sparse_column_iterable.h
@@ -112,6 +112,8 @@ class SparseColumnIterable {
   int64 example_start() const { return example_start_; }
   int64 example_end() const { return example_end_; }
 
+  const TTypes<int64>::ConstMatrix& sparse_indices() const { return ix_; }
+
  private:
   // Sparse indices matrix.
   TTypes<int64>::ConstMatrix ix_;
diff --git a/tensorflow/contrib/boosted_trees/lib/utils/sparse_column_iterable_test.cc b/tensorflow/contrib/boosted_trees/lib/utils/sparse_column_iterable_test.cc
index 7792bd8c66..0138aae3db 100644
--- a/tensorflow/contrib/boosted_trees/lib/utils/sparse_column_iterable_test.cc
+++ b/tensorflow/contrib/boosted_trees/lib/utils/sparse_column_iterable_test.cc
@@ -34,19 +34,19 @@ TEST_F(SparseColumnIterableTest, Empty) {
 }
 
 TEST_F(SparseColumnIterableTest, Iterate) {
-  // 8 examples having 7 sparse features with the third multi-valent.
+  // 8 examples having 7 sparse features with the 3rd and 7th multi-valent.
   // This can be visualized like the following:
   // Instance | Sparse |
-  // 0        |   x    |
+  // 0        |  x     |
   // 1        |        |
   // 2        |        |
   // 3        |  xxx   |
-  // 4        |   x    |
+  // 4        |  x     |
   // 5        |        |
   // 6        |        |
-  // 7        |   xx   |
+  // 7        |  x x   |
   const auto indices =
-      AsTensor<int64>({0, 0, 3, 0, 3, 1, 3, 2, 4, 0, 7, 0, 7, 1}, {7, 2});
+      AsTensor<int64>({0, 0, 3, 0, 3, 1, 3, 2, 4, 0, 7, 0, 7, 2}, {7, 2});
 
   auto validate_example_range = [](const ExampleRowRange& range) {
     switch (range.example_idx) {
diff --git a/tensorflow/contrib/boosted_trees/proto/tree_config.proto b/tensorflow/contrib/boosted_trees/proto/tree_config.proto
index 2e9d45efd7..f14abf45a5 100644
--- a/tensorflow/contrib/boosted_trees/proto/tree_config.proto
+++ b/tensorflow/contrib/boosted_trees/proto/tree_config.proto
@@ -53,6 +53,9 @@ message DenseFloatBinarySplit {
   // Float feature column and split threshold describing
   // the rule feature <= threshold.
   int32 feature_column = 1;
+  // If feature column is multivalent, this holds the index of the feature for
+  // the split. Defaults to 0.
+  int32 feature_id = 5;
   float threshold = 2;
 
   // Node children indexing into a contiguous
-- 
GitLab


From d17db4b011d3c04cf7ff8caf4578032b4c0fc622 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 25 Oct 2017 09:03:21 -0700
Subject: [PATCH 759/909] Split Evaluator.evaluate_on_dataset() into two
 methods to separate graph construction from running in sessions. Otherwise it
 is very hard to avoid adding to the graph every time you do an eval.

Also:
* Stop making init_variables() conditional on whether the variables have
  already been initialized.
* Use new feature of group() to accept lists.
PiperOrigin-RevId: 173404673
---
 tensorflow/contrib/eager/python/evaluator.py  | 87 ++++++++++++++++---
 .../contrib/eager/python/evaluator_test.py    |  3 +-
 .../contrib/eager/python/metrics_impl.py      | 16 +---
 .../contrib/eager/python/metrics_test.py      |  4 +-
 4 files changed, 79 insertions(+), 31 deletions(-)

diff --git a/tensorflow/contrib/eager/python/evaluator.py b/tensorflow/contrib/eager/python/evaluator.py
index 67f545e838..633c747e5e 100644
--- a/tensorflow/contrib/eager/python/evaluator.py
+++ b/tensorflow/contrib/eager/python/evaluator.py
@@ -81,10 +81,18 @@ class Evaluator(object):
 
     Returns:
       An op.
+
+    Raises:
+      RuntimeError: if eager execution is enabled.
+
+    @compatibility(eager)
+    Only for graph execution.
+    @end_compatibility
     """
-    assert context.in_graph_mode()
-    return control_flow_ops.group(
-        *[m.init_variables() for _, m in self.metrics])
+    if context.in_eager_mode():
+      raise RuntimeError("Evaluator.init_variables() not needed when "
+                         "eager execution is enabled.")
+    return control_flow_ops.group([m.init_variables() for _, m in self.metrics])
 
   def all_metric_results(self):  # TODO(josh11b): Add optional summary_writer.
     """Returns dict mapping metric name -> value."""
@@ -97,28 +105,79 @@ class Evaluator(object):
     return results
 
   def evaluate_on_dataset(self, dataset, *args, **kwargs):
-    """Convenience method for performing an eval on a Dataset."""
+    """Convenience method for performing an eval on a Dataset.
+
+    Args:
+      dataset: Dataset object with the input data to evaluate on.
+      *args:
+      **kwargs: Optional additional arguments to __call__().
+
+    Returns:
+      @compatibility(eager)
+      When eager execution is enabled, this returns the result of performing
+      an evaluation as a dictionary. With graph execution, this returns a tuple
+      (init_op, call_op, results_op) which may be executed using this code:
+      ```python
+        sess.run(init_op)
+        try:
+          while True:
+            sess.run(call_op)
+        except tf.errors.OutOfRangeError:
+          pass
+        return sess.run(results_op)  # A dictionary
+
+        # equivalently:
+        return evaluator.run_evaluation(init_op, call_op, results_op, sess=sess)
+      ```
+      @end_compatibility
+    """
     # TODO(josh11b): Add optional summary_writer.
     if context.in_graph_mode():
-      # TODO(josh11b): Return an dict of tensors to pass to session.run()
-      # instead of running using the default session here.
-      sess = ops.get_default_session()
       call_op = self.__call__(dataset.make_one_shot_iterator().get_next(),
                               *args, **kwargs)
       init_op = self.init_variables()
       results_op = self.all_metric_results()
-      sess.run(init_op)
-      try:
-        while True:
-          sess.run(call_op)
-      except errors_impl.OutOfRangeError:
-        pass
-      return sess.run(results_op)
+      return (init_op, call_op, results_op)
     # Eager case
     for example in datasets.Iterator(dataset):
       self.__call__(example, *args, **kwargs)
     return self.all_metric_results()
 
+  @staticmethod
+  def run_evaluation(init_op, call_op, results_op, sess=None):
+    """Convenience method for running the ops returned by evaluate_on_dataset.
+
+    Args:
+      init_op: An op that initializes/resets evaluation state.
+      call_op: An op that updates evaluation state on a mini-batch of examples.
+        Must generate an tf.errors.OutOfRangeError when done.
+      results_op: A dictionary of tensors that compute the final evaluation
+        results from the evaulation state.
+      sess: The Session to run the evaluation in. Defaults to the default
+        Session.
+
+    Returns:
+      A dictionary of values, parallel to results_op.
+
+    Raises:
+      RuntimeError: if eager execution is enabled.
+
+    @compatibility(eager)
+    Only for graph execution.
+    @end_compatibility
+    """
+    if context.in_eager_mode():
+      raise RuntimeError("Evaluator.run_evaluation() not supported when "
+                         "eager execution is enabled.")
+    sess = sess or ops.get_default_session()
+    sess.run(init_op)
+    try:
+      while True:
+        sess.run(call_op)
+    except errors_impl.OutOfRangeError:
+      pass
+    return sess.run(results_op)
+
   # ---- To be implemented by descendants ---
   def call(self, eval_data):
     """Update metrics using the output of self.model.
diff --git a/tensorflow/contrib/eager/python/evaluator_test.py b/tensorflow/contrib/eager/python/evaluator_test.py
index 71e9fa40a8..4652a69081 100644
--- a/tensorflow/contrib/eager/python/evaluator_test.py
+++ b/tensorflow/contrib/eager/python/evaluator_test.py
@@ -100,7 +100,8 @@ class EvaluatorTest(test.TestCase):
     with context.graph_mode(), self.test_session():
       e = SimpleEvaluator(IdentityModel())
       ds = dataset_ops.Dataset.from_tensor_slices([3.0, 5.0, 7.0, 9.0])
-      results = e.evaluate_on_dataset(ds)
+      init_op, call_op, results_op = e.evaluate_on_dataset(ds)
+      results = e.run_evaluation(init_op, call_op, results_op)
       self.assertEqual(set(["mean"]), set(results.keys()))
       self.assertEqual(6.0, results["mean"])
 
diff --git a/tensorflow/contrib/eager/python/metrics_impl.py b/tensorflow/contrib/eager/python/metrics_impl.py
index 77a84e006e..6af0d65e08 100644
--- a/tensorflow/contrib/eager/python/metrics_impl.py
+++ b/tensorflow/contrib/eager/python/metrics_impl.py
@@ -23,30 +23,18 @@ import re
 from tensorflow.contrib.summary import summary_ops
 from tensorflow.python.eager import context
 from tensorflow.python.eager import function
-from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import init_ops
 from tensorflow.python.ops import math_ops
-from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.ops import variable_scope
 
 
 _to_replace = re.compile("[^A-Za-z0-9.]")
 
 
-def _init_var(v):
-  def do_init(v):
-    with ops.control_dependencies([v.assign(v.initial_value)]):
-      return constant_op.constant(True)
-  return control_flow_ops.cond(
-      resource_variable_ops.var_is_initialized_op(v._handle),  # pylint: disable=protected-access
-      lambda: constant_op.constant(False),
-      lambda: do_init(v))
-
-
 class Metric(object):
   """A metric holds state for aggregating statistics over an evaluation run.
 
@@ -121,7 +109,7 @@ class Metric(object):
     return self._vars
 
   def init_variables(self):
-    """Return an op for initializing this Metric's uninitialized variables.
+    """Return an op for initializing this Metric's variables.
 
     Only for graph execution. Should be called after variables are created
     in the first execution of __call__().
@@ -130,7 +118,7 @@ class Metric(object):
       An op to run.
     """
     assert context.in_graph_mode()
-    return control_flow_ops.group(*[_init_var(v) for v in self._vars])
+    return control_flow_ops.group([v.initializer for v in self._vars])
 
   # ---- To be implemented by descendants ---
   def build(self, *args, **kwargs):
diff --git a/tensorflow/contrib/eager/python/metrics_test.py b/tensorflow/contrib/eager/python/metrics_test.py
index fce6be1761..3ecbaeae69 100644
--- a/tensorflow/contrib/eager/python/metrics_test.py
+++ b/tensorflow/contrib/eager/python/metrics_test.py
@@ -141,10 +141,10 @@ class MetricsTest(test.TestCase):
       sess.run(accumulate, feed_dict={p: 1000})
       sess.run(accumulate, feed_dict={p: [10000, 100000]})
       self.assertAllEqual(m.result().eval(), 111111.0/6)
-      # Second init is ignored, since the variables are already initialized.
+      # Second init resets all the variables.
       init_op.run()
       sess.run(accumulate, feed_dict={p: 7})
-      self.assertAllEqual(m.result().eval(), 111118.0/7)
+      self.assertAllEqual(m.result().eval(), 7)
 
   def testTwoMeansGraph(self):
     # Verify two metrics with the same class and name don't
-- 
GitLab


From 4f86cf60254126d28f5f653d810de0a2cf1473c8 Mon Sep 17 00:00:00 2001
From: Jacques Pienaar <jpienaar@google.com>
Date: Wed, 25 Oct 2017 09:33:42 -0700
Subject: [PATCH 760/909] [TX2XLA] Add function name when reporting signature
 check failure.

PiperOrigin-RevId: 173408083
---
 tensorflow/compiler/tf2xla/xla_compiler.cc | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tensorflow/compiler/tf2xla/xla_compiler.cc b/tensorflow/compiler/tf2xla/xla_compiler.cc
index a82ef02e32..e49663b8b0 100644
--- a/tensorflow/compiler/tf2xla/xla_compiler.cc
+++ b/tensorflow/compiler/tf2xla/xla_compiler.cc
@@ -177,7 +177,9 @@ Status XlaCompiler::CompileFunction(
   const FunctionBody* fbody;
   TF_RETURN_IF_ERROR(FindFunctionBody(function, &fbody));
 
-  TF_RETURN_IF_ERROR(CheckSignature(fbody->arg_types, args));
+  TF_RETURN_WITH_CONTEXT_IF_ERROR(
+      CheckSignature(fbody->arg_types, args),
+      "Signature check failure while compiling: ", function.name());
 
   std::unique_ptr<Graph> graph(new Graph(options_.flib_def));
   CopyGraph(*fbody->graph, graph.get());
-- 
GitLab


From 7c1ff88d0e83b9b8378fb981c3a48a1ae698e12a Mon Sep 17 00:00:00 2001
From: Tayo Oguntebi <10927929+tayo@users.noreply.github.com>
Date: Wed, 25 Oct 2017 09:46:40 -0700
Subject: [PATCH 761/909] Update node_def.proto

Clarifying comments for valid device string in NodeDef, as discussed in PR #13874.

Notes:
1. The device string is as emitted by: tensorflow/python/framework/device.py to_string() function.
2. I notice our regex convention does not use '+' (e.g. XX* instead of X+).
---
 tensorflow/core/framework/node_def.proto | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/framework/node_def.proto b/tensorflow/core/framework/node_def.proto
index 1fd2e50b51..8fcee32e29 100644
--- a/tensorflow/core/framework/node_def.proto
+++ b/tensorflow/core/framework/node_def.proto
@@ -35,7 +35,7 @@ message NodeDef {
   // CONSTRAINT ::= ("job:" JOB_NAME)
   //              | ("replica:" [1-9][0-9]*)
   //              | ("task:" [1-9][0-9]*)
-  //              | ("device:" ("gpu" | "cpu") ":" ([1-9][0-9]* | "*") )
+  //              | ("device:" [A-Za-z]* ":" ([1-9][0-9]* | "*") )
   //
   // Valid values for this string include:
   // * "/job:worker/replica:0/task:1/device:GPU:3"  (full specification)
-- 
GitLab


From f1f60ac3e59b7cbbd2badef11cc3da42064fc695 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 25 Oct 2017 09:47:05 -0700
Subject: [PATCH 762/909] 1. Separate the special case BinaryFunctor when NDIMS
 == 2 into a template specialization. This prevents the NDIMS==2 optimization
 code from being compiled in the general case, which can lead to compile time
 errors when the underlying Eigen implementation becomes more strict about
 NDIMS.

2. Fix the 64-bit dimension() call on output in extract_image_patches_op.h when other operands have been cast to use 32-bit index.

PiperOrigin-RevId: 173409602
---
 tensorflow/core/kernels/cwise_ops_common.h    | 75 +++++++++++++++----
 .../core/kernels/extract_image_patches_op.h   |  5 +-
 2 files changed, 63 insertions(+), 17 deletions(-)

diff --git a/tensorflow/core/kernels/cwise_ops_common.h b/tensorflow/core/kernels/cwise_ops_common.h
index 2454620776..8295fa939e 100644
--- a/tensorflow/core/kernels/cwise_ops_common.h
+++ b/tensorflow/core/kernels/cwise_ops_common.h
@@ -305,6 +305,62 @@ struct BinaryFunctor<CPUDevice, Functor, NDIMS, false> {
     Assign(d, out, in.unaryExpr(Unary(scalar.data())));
   }
 
+  void BCast(const CPUDevice& dev,
+             typename TTypes<typename Functor::out_type, NDIMS>::Tensor out,
+             typename TTypes<typename Functor::in_type, NDIMS>::ConstTensor in0,
+             typename Eigen::array<Eigen::DenseIndex, NDIMS> bcast0,
+             typename TTypes<typename Functor::in_type, NDIMS>::ConstTensor in1,
+             typename Eigen::array<Eigen::DenseIndex, NDIMS> bcast1,
+             bool* error) {
+    typename Functor::func func;
+    if (AllOne<NDIMS>(bcast0) && AllOne<NDIMS>(bcast1)) {
+      Assign(dev, out, in0.binaryExpr(in1, func));
+    } else if (AllOne<NDIMS>(bcast0)) {
+      auto rhs = in1.broadcast(bcast1);
+      Assign(dev, out, in0.binaryExpr(rhs, func));
+    } else if (AllOne<NDIMS>(bcast1)) {
+      auto lhs = in0.broadcast(bcast0);
+      Assign(dev, out, lhs.binaryExpr(in1, func));
+    } else {
+      auto lhs = in0.broadcast(bcast0);
+      auto rhs = in1.broadcast(bcast1);
+      Assign(dev, out, lhs.binaryExpr(rhs, func));
+    }
+  }
+};
+
+// Partial specialization of BinaryFunctor<Device=CPUDevice, Functor, 2>
+// for functors with with no error checking.
+template <typename Functor>
+struct BinaryFunctor<CPUDevice, Functor, 2, false> {
+  enum { NDIMS = 2 };
+
+  void operator()(const CPUDevice& d, typename Functor::tout_type out,
+                  typename Functor::tin_type in0,
+                  typename Functor::tin_type in1, bool* error) {
+    Assign(d, out, in0.binaryExpr(in1, typename Functor::func()));
+  }
+
+  void Left(const CPUDevice& d, typename Functor::tout_type out,
+            typename Functor::tscalar_type scalar,
+            typename Functor::tin_type in, bool* error) {
+    typedef typename Functor::out_type Tout;
+    typedef typename Functor::in_type Tin;
+    typedef typename Functor::func Binary;
+    typedef typename Eigen::internal::scalar_left<Tout, Tin, Binary> Unary;
+    Assign(d, out, in.unaryExpr(Unary(scalar.data())));
+  }
+
+  void Right(const CPUDevice& d, typename Functor::tout_type out,
+             typename Functor::tin_type in,
+             typename Functor::tscalar_type scalar, bool* error) {
+    typedef typename Functor::out_type Tout;
+    typedef typename Functor::in_type Tin;
+    typedef typename Functor::func Binary;
+    typedef typename Eigen::internal::scalar_right<Tout, Tin, Binary> Unary;
+    Assign(d, out, in.unaryExpr(Unary(scalar.data())));
+  }
+
 #if !defined(EIGEN_HAS_INDEX_LIST)
   inline Eigen::DSizes<int, 2> NByOne(int n) {
     return Eigen::DSizes<int, 2>(n, 1);
@@ -334,8 +390,7 @@ struct BinaryFunctor<CPUDevice, Functor, NDIMS, false> {
              bool* error) {
     typedef typename Functor::in_type T;
     typename Functor::func func;
-    if ((NDIMS == 2) && Functor::use_bcast_optimization &&
-        use_bcast_optimization<T>::value) {
+    if (Functor::use_bcast_optimization && use_bcast_optimization<T>::value) {
       // Optimize for speed by using Eigen::type2index and avoid
       // .broadcast() when we know its a no-op.
       //
@@ -411,19 +466,9 @@ struct BinaryFunctor<CPUDevice, Functor, NDIMS, false> {
     }
 
     // Fallback path. Always works and probably slower.
-    if (AllOne<NDIMS>(bcast0) && AllOne<NDIMS>(bcast1)) {
-      Assign(dev, out, in0.binaryExpr(in1, func));
-    } else if (AllOne<NDIMS>(bcast0)) {
-      auto rhs = in1.broadcast(bcast1);
-      Assign(dev, out, in0.binaryExpr(rhs, func));
-    } else if (AllOne<NDIMS>(bcast1)) {
-      auto lhs = in0.broadcast(bcast0);
-      Assign(dev, out, lhs.binaryExpr(in1, func));
-    } else {
-      auto lhs = in0.broadcast(bcast0);
-      auto rhs = in1.broadcast(bcast1);
-      Assign(dev, out, lhs.binaryExpr(rhs, func));
-    }
+    auto lhs = in0.broadcast(bcast0);
+    auto rhs = in1.broadcast(bcast1);
+    Assign(dev, out, lhs.binaryExpr(rhs, func));
   }
 };
 
diff --git a/tensorflow/core/kernels/extract_image_patches_op.h b/tensorflow/core/kernels/extract_image_patches_op.h
index 9d34daca64..e430a23d20 100644
--- a/tensorflow/core/kernels/extract_image_patches_op.h
+++ b/tensorflow/core/kernels/extract_image_patches_op.h
@@ -34,11 +34,12 @@ struct ExtractImagePatchesForward {
     // NHWC format while Eigen assumes NWHC format.
     const int64 N = std::max(input.size(), output.size());
     if (N <= std::numeric_limits<Index32>::max()) {
-      To32Bit(output).device(d) =
+      auto output_32bit = To32Bit(output);
+      output_32bit.device(d) =
           To32Bit(input)
               .extract_image_patches(patch_cols, patch_rows, stride_cols,
                                      stride_rows, rate_cols, rate_rows, padding)
-              .reshape(output.dimensions());
+              .reshape(output_32bit.dimensions());
     } else {
       output.device(d) =
           input
-- 
GitLab


From 805594a4643dda027ff45be7edd2ab94f57a9dec Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 25 Oct 2017 10:22:46 -0700
Subject: [PATCH 763/909] [XLA] Set shape of -C equal to shape of C in
 A/pow(B,C) -> A*pow(B,-C).

PiperOrigin-RevId: 173414738
---
 .../xla/service/algebraic_simplifier.cc       |  9 +++--
 .../xla/service/algebraic_simplifier_test.cc  | 36 +++++++++++++++++++
 2 files changed, 43 insertions(+), 2 deletions(-)

diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier.cc b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
index 8b3886cc7a..ae26cc2d99 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
@@ -523,11 +523,16 @@ Status AlgebraicSimplifierVisitor::HandleDivide(HloInstruction* divide,
   // A/pow(B,C) => A*pow(B,-C)
   if (rhs->opcode() == HloOpcode::kPower) {
     VLOG(10) << "transform [A/pow(B,C) => A*pow(B,-C)]: " << divide->ToString();
+    // The output shape of the created negate operator should be the same as the
+    // input.
+    const Shape& negate_shape = rhs->operand(1)->shape();
     HloInstruction* negate =
         computation_->AddInstruction(HloInstruction::CreateUnary(
-            divide->shape(), HloOpcode::kNegate, rhs->mutable_operand(1)));
+            negate_shape, HloOpcode::kNegate, rhs->mutable_operand(1)));
+    // And the power operator should retain the output shape of the old one.
+    const Shape& new_power_shape = rhs->shape();
     HloInstruction* new_power = computation_->AddInstruction(
-        HloInstruction::CreateBinary(divide->shape(), HloOpcode::kPower,
+        HloInstruction::CreateBinary(new_power_shape, HloOpcode::kPower,
                                      rhs->mutable_operand(0), negate));
     return ReplaceWithNewInstruction(
         divide, HloInstruction::CreateBinary(
diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
index af502206e2..57be144b36 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
@@ -353,6 +353,42 @@ TEST_F(AlgebraicSimplifierTest, DivOfPower) {
               op::Multiply(param0, op::Power(param1, op::Negate(param2))));
 }
 
+// Test that broadcasting is done on the right step when simplifying A/pow(B,C)
+// to A*pow(B,-C).
+TEST_F(AlgebraicSimplifierTest, DivOfBroadcastingPower) {
+  Shape r0f32 = ShapeUtil::MakeShape(F32, {});
+  Shape r1f32 = ShapeUtil::MakeShape(F32, {7});
+  HloComputation::Builder builder(TestName());
+  HloInstruction* param0 = builder.AddInstruction(
+      HloInstruction::CreateParameter(0, r1f32, "param0"));
+  HloInstruction* param1 = builder.AddInstruction(
+      HloInstruction::CreateParameter(1, r1f32, "param1"));
+  HloInstruction* param2 = builder.AddInstruction(
+      HloInstruction::CreateParameter(2, r0f32, "param2"));
+  HloInstruction* power = builder.AddInstruction(
+      HloInstruction::CreateBinary(r1f32, HloOpcode::kPower, param1, param2));
+  builder.AddInstruction(
+      HloInstruction::CreateBinary(r1f32, HloOpcode::kDivide, param0, power));
+
+  auto module = CreateNewModule();
+  auto computation = module->AddEntryComputation(builder.Build());
+
+  EXPECT_THAT(computation->root_instruction(),
+              op::Divide(param0, op::Power(param1, param2)));
+
+  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
+                                 non_bitcasting_callback());
+  ASSERT_TRUE(simplifier.Run(module.get()).ValueOrDie());
+
+  ASSERT_THAT(computation->root_instruction(),
+              op::Multiply(param0, op::Power(param1, op::Negate(param2))));
+
+  const HloInstruction* negate =
+      computation->root_instruction()->operand(1)->operand(1);
+  const Shape& negate_shape = negate->shape();
+  EXPECT_EQ(0, negate_shape.dimensions_size());
+}
+
 // Test that A/1 is simplified to A for a scalar.
 TEST_F(AlgebraicSimplifierTest, DivOneScalar) {
   Shape r0f32 = ShapeUtil::MakeShape(F32, {});
-- 
GitLab


From f75a6b866524ce877321f0ef9f3508b9b6c705fc Mon Sep 17 00:00:00 2001
From: Michael Case <mikecase@google.com>
Date: Wed, 25 Oct 2017 10:29:23 -0700
Subject: [PATCH 764/909] MomentumOptimizer has two required args which causes
 error in optimize_loss.

optimize_loss function tries to initialize Optimizer like...
opt = OPTIMIZER_CLS_NAMES[optimizer](learning_rate=lr)

This does not work for MomentumOptimizer which also has required arg
of 'momentum'.

PiperOrigin-RevId: 173415707
---
 tensorflow/contrib/layers/python/layers/optimizers.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/layers/python/layers/optimizers.py b/tensorflow/contrib/layers/python/layers/optimizers.py
index 33db93b970..cdceea6fee 100644
--- a/tensorflow/contrib/layers/python/layers/optimizers.py
+++ b/tensorflow/contrib/layers/python/layers/optimizers.py
@@ -41,7 +41,7 @@ OPTIMIZER_CLS_NAMES = {
     "Adagrad": train.AdagradOptimizer,
     "Adam": train.AdamOptimizer,
     "Ftrl": train.FtrlOptimizer,
-    "Momentum": train.MomentumOptimizer,
+    "Momentum": lambda lr: train.MomentumOptimizer(lr, momentum=0.9),
     "RMSProp": train.RMSPropOptimizer,
     "SGD": train.GradientDescentOptimizer,
 }
-- 
GitLab


From 51d95238a6ccbf540bda0299548c9bbb8b304130 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 25 Oct 2017 11:29:17 -0700
Subject: [PATCH 765/909] Add tests for utils.SubGraph in K-FAC

PiperOrigin-RevId: 173425276
---
 .../kfac/python/kernel_tests/utils_test.py    | 33 +++++++++++++++++++
 1 file changed, 33 insertions(+)

diff --git a/tensorflow/contrib/kfac/python/kernel_tests/utils_test.py b/tensorflow/contrib/kfac/python/kernel_tests/utils_test.py
index 779a8179bb..55fe38e3e9 100644
--- a/tensorflow/contrib/kfac/python/kernel_tests/utils_test.py
+++ b/tensorflow/contrib/kfac/python/kernel_tests/utils_test.py
@@ -63,6 +63,39 @@ class SequenceDictTest(test.TestCase):
     self.assertItemsEqual(list(zip(keys, values)), seq_dict.items())
 
 
+class SubGraphTest(test.TestCase):
+
+  def testBasicGraph(self):
+    a = array_ops.constant([[1., 2.], [3., 4.]])
+    b = array_ops.constant([[5., 6.], [7., 8.]])
+    c = a + b
+    d = a * b
+    sub_graph = utils.SubGraph((c,))
+    self.assertTrue(sub_graph.is_member(a))
+    self.assertTrue(sub_graph.is_member(b))
+    self.assertTrue(sub_graph.is_member(c))
+    self.assertFalse(sub_graph.is_member(d))
+
+  def testRepeatedAdds(self):
+    a = array_ops.constant([[1., 2.], [3., 4.]])
+    b = array_ops.constant([[5., 6.], [7., 8.]])
+    c = a + b + a  # note that a appears twice in this graph
+    sub_graph = utils.SubGraph((c,))
+    self.assertTrue(sub_graph.is_member(a))
+    self.assertTrue(sub_graph.is_member(b))
+    self.assertTrue(sub_graph.is_member(c))
+
+  def testFilterList(self):
+    a = array_ops.constant([[1., 2.], [3., 4.]])
+    b = array_ops.constant([[5., 6.], [7., 8.]])
+    c = a + b
+    d = a * b
+    sub_graph = utils.SubGraph((c,))
+    input_list = [b, d]
+    filtered_list = sub_graph.filter_list(input_list)
+    self.assertEqual(filtered_list, [b])
+
+
 class UtilsTest(test.TestCase):
 
   def _fully_connected_layer_params(self):
-- 
GitLab


From 7ce9b664a7422e07cdda843f7256fdb3fd454082 Mon Sep 17 00:00:00 2001
From: Jiri Simsa <jsimsa@google.com>
Date: Wed, 25 Oct 2017 11:37:41 -0700
Subject: [PATCH 766/909] Reduce lock contention to prevent interop threadpool
 deadlocks.

PiperOrigin-RevId: 173426652
---
 tensorflow/contrib/data/python/kernel_tests/BUILD   |  3 ---
 tensorflow/core/kernels/map_and_batch_dataset_op.cc | 10 ++++++----
 2 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/tensorflow/contrib/data/python/kernel_tests/BUILD b/tensorflow/contrib/data/python/kernel_tests/BUILD
index 5339ebb689..b8cdb7b20d 100644
--- a/tensorflow/contrib/data/python/kernel_tests/BUILD
+++ b/tensorflow/contrib/data/python/kernel_tests/BUILD
@@ -11,9 +11,6 @@ py_test(
     size = "small",
     srcs = ["batch_dataset_op_test.py"],
     srcs_version = "PY2AND3",
-    tags = [
-        "manual",  # b/67958604
-    ],
     deps = [
         "//tensorflow/contrib/data/python/ops:dataset_ops",
         "//tensorflow/contrib/data/python/ops:transformation_ops",
diff --git a/tensorflow/core/kernels/map_and_batch_dataset_op.cc b/tensorflow/core/kernels/map_and_batch_dataset_op.cc
index 332a96ae03..f9f68a5418 100644
--- a/tensorflow/core/kernels/map_and_batch_dataset_op.cc
+++ b/tensorflow/core/kernels/map_and_batch_dataset_op.cc
@@ -287,10 +287,12 @@ class MapAndBatchDatasetOp : public UnaryDatasetOpKernel {
           EXCLUSIVE_LOCKS_REQUIRED(mu_) {
         port::Tracing::TraceMe activity(strings::StrCat(prefix(), "::Start"));
         // Initialize batch result.
-        mutex_lock l(batch_results_[batch_index].mu);
-        batch_results_[batch_index].output_allocated = false;
-        batch_results_[batch_index].counter.reset(
-            new BlockingCounter(dataset()->batch_size_));
+        {
+          mutex_lock l(batch_results_[batch_index].mu);
+          batch_results_[batch_index].output_allocated = false;
+          batch_results_[batch_index].counter.reset(
+              new BlockingCounter(dataset()->batch_size_));
+        }
         // Initialize invocation results.
         for (size_t i = 0; i < dataset()->batch_size_; ++i) {
           size_t index = ComputeInvocationIndex(batch_index, i);
-- 
GitLab


From 9469522bf13933bf9f53480e63d0ef36f7013b94 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 25 Oct 2017 11:54:39 -0700
Subject: [PATCH 767/909] Adds eager compatability message for Variable.

PiperOrigin-RevId: 173429067
---
 tensorflow/contrib/eager/python/tfe_test.py |  6 ++++++
 tensorflow/python/ops/variables.py          | 14 +++++++++++---
 2 files changed, 17 insertions(+), 3 deletions(-)

diff --git a/tensorflow/contrib/eager/python/tfe_test.py b/tensorflow/contrib/eager/python/tfe_test.py
index 6b5053125b..eabff7f0a8 100644
--- a/tensorflow/contrib/eager/python/tfe_test.py
+++ b/tensorflow/contrib/eager/python/tfe_test.py
@@ -24,6 +24,7 @@ from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import variables
 from tensorflow.python.platform import test
 
 
@@ -39,6 +40,11 @@ class TFETest(test_util.TensorFlowTestCase):
                                  r'indices = 7 is not in \[0, 3\)'):
       array_ops.gather([0, 1, 2], 7)
 
+  def testVariableError(self):
+    with self.assertRaisesRegexp(
+        RuntimeError, r'Variable not supported in Eager mode'):
+      variables.Variable(initial_value=1.0)
+
   def testGradients(self):
 
     def square(x):
diff --git a/tensorflow/python/ops/variables.py b/tensorflow/python/ops/variables.py
index 0272f77176..fd0aee3c33 100644
--- a/tensorflow/python/ops/variables.py
+++ b/tensorflow/python/ops/variables.py
@@ -118,6 +118,14 @@ class Variable(object):
   `trainable_variables()` returns the contents of this collection. The
   various `Optimizer` classes use this collection as the default list of
   variables to optimize.
+
+  @compatiblity(eager)
+  `tf.Variable` is not compatible with eager execution.  Use
+  `tfe.Variable` instead which is compatable with both eager execution
+  and graph construction.  See [the TensorFlow Eager Execution
+  guide](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/eager/python/g3doc/guide.md#variables-and-optimizers)
+  for details on how variables work in eager execution.
+  @end_compatiblity
   """
 
   def __init__(self,
@@ -188,11 +196,11 @@ class Variable(object):
       ValueError: If both `variable_def` and initial_value are specified.
       ValueError: If the initial value is not specified, or does not have a
         shape and `validate_shape` is `True`.
-      RuntimeError: If created in EAGER mode.
+      RuntimeError: If eager execution is enabled.
     """
     if not context.in_graph_mode():
-      raise RuntimeError("Variable not supported in Eager mode. "
-                         "Please use ResourceVariable instead")
+      raise RuntimeError("tf.Variable not supported in Eager mode. "
+                         "Please use tfe.Variable instead")
     if variable_def:
       # If variable_def is provided, recreates the variable from its fields.
       if initial_value:
-- 
GitLab


From 06674161dc5c6b860bea559c2eef0217d49d86ca Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 25 Oct 2017 12:11:43 -0700
Subject: [PATCH 768/909] Variable name remapping of saver/restore.

PiperOrigin-RevId: 173431453
---
 tensorflow/contrib/eager/python/saver.py      | 42 +++++++++----
 tensorflow/contrib/eager/python/saver_test.py | 40 ++++++++++++-
 .../core/kernels/resource_variable_ops.cc     |  2 +-
 tensorflow/python/training/saver_test.py      | 60 +++++++++++--------
 4 files changed, 102 insertions(+), 42 deletions(-)

diff --git a/tensorflow/contrib/eager/python/saver.py b/tensorflow/contrib/eager/python/saver.py
index 2bf11d3f20..404f77105a 100644
--- a/tensorflow/contrib/eager/python/saver.py
+++ b/tensorflow/contrib/eager/python/saver.py
@@ -29,22 +29,23 @@ from tensorflow.python.training import saver as _saver
 
 def _init_from_checkpoint(self, *args, **kwargs):
   """Overrides default init by loading value from checkpoint."""
-  self.old_init(*args, **kwargs)
   # pylint: disable=protected-access
-  if self._shared_name not in self.ckpt_var_cache:
+  self._old_init(*args, **kwargs)
+  ckpt_name = self._map_func(self._shared_name)
+  if ckpt_name not in self._ckpt_var_cache:
     raise errors.NotFoundError(None, None,
-                               "%s not found in checkpoint" % self._shared_name)
+                               "%s not found in checkpoint" % ckpt_name)
 
-  val = self.ckpt_var_cache[self._shared_name]
+  val = self._ckpt_var_cache.get(ckpt_name, None)
   if val is not None:
-    self.assign(self.ckpt_var_cache[self._shared_name])
+    self.assign(val)
     # Avoid assigning for the second time.
-    self.ckpt_var_cache[self._shared_name] = None
+    self._ckpt_var_cache[ckpt_name] = None
   # pylint: enable=protected-access
 
 
 @contextlib.contextmanager
-def restore_variables_on_create(save_path):
+def restore_variables_on_create(save_path, map_func=None):
   """ContextManager that restores variables on creation.
 
     When save_path is None (e.g. No checkpoint), does nothing.
@@ -59,19 +60,31 @@ def restore_variables_on_create(save_path):
 
   Args:
     save_path: The checkpoint file prefix.
+    map_func: A function that given the variable name as argument
+        and returns a variable name in checkpoint for restore. If
+        None, use the variable with the same name in checkpoint to restore.
+        It's an error that the mapped variable name doesn't exist in
+        checkpoint.
 
   Yields:
     Nothing.
 
   Raises:
     NotFoundError: If the variable is not found in checkpoint.
-    ValueError: If not used in eager mode.
+    ValueError: If not used in eager mode or map_func is not callable.
   """
   if context.in_graph_mode():
     raise ValueError(
         "Currently, restore_variables_on_create can only be used with "
         "eager execution enabled.")
   if save_path:
+    if map_func is None:
+      map_func_wrapper = lambda self, x: x
+    else:
+      if not callable(map_func):
+        raise ValueError("map_func must be callaled.")
+      map_func_wrapper = lambda self, x: map_func(x)
+
     ckpt_var_cache = dict()
     reader = checkpoint_utils.load_checkpoint(save_path)
     for k, _ in checkpoint_utils.list_variables(save_path):
@@ -82,8 +95,10 @@ def restore_variables_on_create(save_path):
     assert old_init, "ResourceVariable misses _init_from_args method."
     setattr(resource_variable_ops.ResourceVariable, "_init_from_args",
             _init_from_checkpoint)
-    setattr(resource_variable_ops.ResourceVariable, "old_init", old_init)
-    setattr(resource_variable_ops.ResourceVariable, "ckpt_var_cache",
+    setattr(resource_variable_ops.ResourceVariable, "_old_init", old_init)
+    setattr(resource_variable_ops.ResourceVariable, "_map_func",
+            map_func_wrapper)
+    setattr(resource_variable_ops.ResourceVariable, "_ckpt_var_cache",
             ckpt_var_cache)
   try:
     yield
@@ -93,8 +108,9 @@ def restore_variables_on_create(save_path):
     if save_path:
       setattr(resource_variable_ops.ResourceVariable, "_init_from_args",
               old_init)
-      setattr(resource_variable_ops.ResourceVariable, "old_init", None)
-      setattr(resource_variable_ops.ResourceVariable, "ckpt_var_cache", None)
+      setattr(resource_variable_ops.ResourceVariable, "_old_init", None)
+      setattr(resource_variable_ops.ResourceVariable, "_map_func", None)
+      setattr(resource_variable_ops.ResourceVariable, "_ckpt_var_cache", None)
 
 
 class Saver(object):
@@ -104,7 +120,7 @@ class Saver(object):
     session is not needed.
 
   Args:
-    var_list: A list of variables.
+    var_list: Same as tf.train.Saver.
   """
 
   def __init__(self, var_list):
diff --git a/tensorflow/contrib/eager/python/saver_test.py b/tensorflow/contrib/eager/python/saver_test.py
index 1605435d8d..3c69b90242 100644
--- a/tensorflow/contrib/eager/python/saver_test.py
+++ b/tensorflow/contrib/eager/python/saver_test.py
@@ -55,7 +55,7 @@ class SaverTest(test.TestCase):
       self.assertEqual(v1.read_value().numpy(), 1.0)
 
   def testSameNameNoClobbering(self):
-    with context.eager_mode(), ops.device(self._dev()):
+    with ops.device(self._dev()):
       # Note that this test purposefully uses Graphs rather than
       # IsolateTest. Users are more likely to accidentally create the same
       # variable name this way.
@@ -70,7 +70,7 @@ class SaverTest(test.TestCase):
         saver.save(ckpt_prefix)
 
   def testDifferentGraphError(self):
-    with context.eager_mode(), ops.device(self._dev()):
+    with ops.device(self._dev()):
       with ops.Graph().as_default():
         v1 = resource_variable_ops.ResourceVariable(1.0, name='v1')
       with ops.Graph().as_default():
@@ -80,7 +80,7 @@ class SaverTest(test.TestCase):
           saver.save(ckpt_prefix)
 
   def testSameObjectOK(self):
-    with context.eager_mode(), ops.device(self._dev()):
+    with ops.device(self._dev()):
       v1 = resource_variable_ops.ResourceVariable(1.0, name='v1')
       # While different objects with the same shared_name are not good, passing
       # in the same object multiple times is fine.
@@ -88,6 +88,40 @@ class SaverTest(test.TestCase):
       ckpt_prefix = os.path.join(test.get_temp_dir(), 'ckpt')
       saver.save(ckpt_prefix)
 
+  def testSaveByDict(self):
+    with ops.device(self._dev()):
+      v1 = resource_variable_ops.ResourceVariable(1.0, name='v1')
+      v2 = resource_variable_ops.ResourceVariable(1.0, name='v2')
+      def model():
+        return array_ops.constant(2.0) * v1 * v2
+
+      ckpt_prefix = os.path.join(test.get_temp_dir(), 'ckpt')
+
+      # Save the variables under different names.
+      _ = model()
+      saver = _saver.Saver({'ckpt/v1': v1, 'ckpt/v2': v2})
+      saver.save(ckpt_prefix)
+      v1.assign(2.0)
+      v2.assign(2.0)
+      self.assertEqual(v1.read_value().numpy(), 2.0)
+      self.assertEqual(v2.read_value().numpy(), 2.0)
+      # Can still restore it.
+      saver.restore(ckpt_prefix)
+      self.assertEqual(v1.read_value().numpy(), 1.0)
+      self.assertEqual(v1.read_value().numpy(), 1.0)
+      # However, cannot restore it with default name.
+      with self.assertRaisesOpError('not found in checkpoint'):
+        saver = _saver.Saver([v1, v2]).restore(ckpt_prefix)
+
+      # Can specify which variable in ckpt to restore to which variable.
+      def map_func(x):
+        return {'v3': 'ckpt/v1', 'v4': 'ckpt/v2'}.get(x, x)
+      with _saver.restore_variables_on_create(ckpt_prefix, map_func):
+        v3 = resource_variable_ops.ResourceVariable(2.0, name='v3')
+        v4 = resource_variable_ops.ResourceVariable(2.0, name='v4')
+      self.assertEqual(v3.read_value().numpy(), 1.0)
+      self.assertEqual(v4.read_value().numpy(), 1.0)
+
   def testRestoreOnCreate(self):
     with ops.device(self._dev()):
       def model(init_val):
diff --git a/tensorflow/core/kernels/resource_variable_ops.cc b/tensorflow/core/kernels/resource_variable_ops.cc
index 90db0c2b7b..a4db4abd7b 100644
--- a/tensorflow/core/kernels/resource_variable_ops.cc
+++ b/tensorflow/core/kernels/resource_variable_ops.cc
@@ -85,7 +85,7 @@ class ReadVariableOp : public OpKernel {
                 errors::NotFound(
                     "Error while reading resource variable ", handle.name(),
                     " from Container: ", handle.container(),
-                    ". This could mean that the variable was not initialized. ",
+                    ". This could mean that the variable was uninitialized. ",
                     status.ToString()));
 
     core::ScopedUnref s(variable);
diff --git a/tensorflow/python/training/saver_test.py b/tensorflow/python/training/saver_test.py
index 4abff1d106..744b17dd22 100644
--- a/tensorflow/python/training/saver_test.py
+++ b/tensorflow/python/training/saver_test.py
@@ -1299,20 +1299,20 @@ class KeepCheckpointEveryNHoursTest(test.TestCase):
 
 class SaveRestoreWithVariableNameMap(test.TestCase):
 
-  def testNonReshape(self):
+  def _testNonReshape(self, variable_op):
     save_path = os.path.join(self.get_temp_dir(), "non_reshape")
 
-    with self.test_session() as sess:
+    with self.test_session(graph=ops_lib.Graph()) as sess:
       # Build a graph with 2 parameter nodes, and Save and
       # Restore nodes for them.
-      v0 = variables.Variable(10.0, name="v0")
-      v1 = variables.Variable(20.0, name="v1")
+      v0 = variable_op(10.0, name="v0")
+      v1 = variable_op(20.0, name="v1")
       save = saver_module.Saver({"save_prefix/v0": v0, "save_prefix/v1": v1})
-      variables.global_variables_initializer().run()
+      self.evaluate(variables.global_variables_initializer())
 
       # Check that the parameter nodes have been initialized.
-      self.assertEqual(10.0, v0.eval())
-      self.assertEqual(20.0, v1.eval())
+      self.assertEqual(10.0, self.evaluate(v0))
+      self.assertEqual(20.0, self.evaluate(v1))
 
       # Save the initialized values in the file at "save_path"
       # Use a variable name map to set the saved tensor names
@@ -1327,40 +1327,50 @@ class SaveRestoreWithVariableNameMap(test.TestCase):
 
     # Verify that the mapped names are present in the Saved file and can be
     # Restored using remapped names.
-    with self.test_session() as sess:
-      v0 = variables.Variable(-1.0, name="v0")
-      v1 = variables.Variable(-1.0, name="v1")
+    with self.test_session(graph=ops_lib.Graph()) as sess:
+      v0 = variable_op(-1.0, name="v0")
+      v1 = variable_op(-1.0, name="v1")
 
-      with self.assertRaisesOpError("uninitialized value v0"):
-        sess.run(v0)
-      with self.assertRaisesOpError("uninitialized value v1"):
-        sess.run(v1)
+      if context.in_graph_mode():
+        with self.assertRaisesOpError("uninitialized"):
+          self.evaluate(v0)
+        with self.assertRaisesOpError("uninitialized"):
+          self.evaluate(v1)
 
       save = saver_module.Saver({"save_prefix/v0": v0, "save_prefix/v1": v1})
       save.restore(sess, save_path)
 
       # Check that the parameter nodes have been restored.
-      self.assertEqual(10.0, v0.eval())
-      self.assertEqual(20.0, v1.eval())
+      if context.in_graph_mode():
+        self.assertEqual(10.0, self.evaluate(v0))
+        self.assertEqual(20.0, self.evaluate(v1))
 
     # Add a prefix to the node names in the current graph and Restore using
     # remapped names.
-    with self.test_session() as sess:
-      v0 = variables.Variable(-1.0, name="restore_prefix/v0")
-      v1 = variables.Variable(-1.0, name="restore_prefix/v1")
+    with self.test_session(graph=ops_lib.Graph()) as sess:
+      v0 = variable_op(-1.0, name="restore_prefix/v0")
+      v1 = variable_op(-1.0, name="restore_prefix/v1")
 
-      with self.assertRaisesOpError("uninitialized value restore_prefix/v0"):
-        sess.run(v0)
-      with self.assertRaisesOpError("uninitialized value restore_prefix/v1"):
-        sess.run(v1)
+      if context.in_graph_mode():
+        with self.assertRaisesOpError("uninitialized"):
+          self.evaluate(v0)
+        with self.assertRaisesOpError("uninitialized"):
+          self.evaluate(v1)
 
       # Restore the saved values in the parameter nodes.
       save = saver_module.Saver({"save_prefix/v0": v0, "save_prefix/v1": v1})
       save.restore(sess, save_path)
 
       # Check that the parameter nodes have been restored.
-      self.assertEqual(10.0, v0.eval())
-      self.assertEqual(20.0, v1.eval())
+      self.assertEqual(10.0, self.evaluate(v0))
+      self.assertEqual(20.0, self.evaluate(v1))
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testNonReshapeResourceVariable(self):
+    self._testNonReshape(resource_variable_ops.ResourceVariable)
+
+  def testNonReshapeVariable(self):
+    self._testNonReshape(variables.Variable)
 
 
 class LatestCheckpointWithRelativePaths(test.TestCase):
-- 
GitLab


From df3b4444ccb61ef4858a6acf195d93f9c43d7b24 Mon Sep 17 00:00:00 2001
From: Mark Daoust <markdaoust@google.com>
Date: Wed, 25 Oct 2017 12:22:02 -0700
Subject: [PATCH 769/909] Dedent compatibility blocks

This is to avoid accidentally triggering markdown's 4-space code-formatting

PiperOrigin-RevId: 173432680
---
 tensorflow/tools/docs/pretty_docs.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tensorflow/tools/docs/pretty_docs.py b/tensorflow/tools/docs/pretty_docs.py
index 5ea9394865..92f50189dd 100644
--- a/tensorflow/tools/docs/pretty_docs.py
+++ b/tensorflow/tools/docs/pretty_docs.py
@@ -290,7 +290,9 @@ def _build_compatibility(compatibility):
   for key in sorted_keys:
 
     value = compatibility[key]
-    parts.append('\n\n#### %s compatibility\n%s\n' % (key, value))
+    # Dedent so that it does not trigger markdown code formatting.
+    value = textwrap.dedent(value)
+    parts.append('\n\n#### %s Compatibility\n%s\n' % (key.title(), value))
 
   return ''.join(parts)
 
-- 
GitLab


From 9ad48ae2652291b4ea6f4b03da8637f58d6fccf4 Mon Sep 17 00:00:00 2001
From: Mark Heffernan <meheff@google.com>
Date: Wed, 25 Oct 2017 13:37:14 -0700
Subject: [PATCH 770/909] Add de-fuser pass which replaces fusion nodes with
 their non-fused equivalents. This pass will be used in tools added in
 followup CLs. The motivation for this pass is running an HLO module which was
 built for one backend on a different backend for testing and debugging.
 De-fusing is necessary because different backends have different fusion
 support.

Also, fix problem with deletion of fused computation identified when testing
the defuser.

PiperOrigin-RevId: 173442671
---
 tensorflow/compiler/xla/service/BUILD         |  27 +++
 tensorflow/compiler/xla/service/defuser.cc    | 115 ++++++++++
 tensorflow/compiler/xla/service/defuser.h     |  41 ++++
 .../compiler/xla/service/defuser_test.cc      | 214 ++++++++++++++++++
 4 files changed, 397 insertions(+)
 create mode 100644 tensorflow/compiler/xla/service/defuser.cc
 create mode 100644 tensorflow/compiler/xla/service/defuser.h
 create mode 100644 tensorflow/compiler/xla/service/defuser_test.cc

diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD
index 8f5105aa53..fe5889efe1 100644
--- a/tensorflow/compiler/xla/service/BUILD
+++ b/tensorflow/compiler/xla/service/BUILD
@@ -1074,6 +1074,33 @@ tf_cc_test(
     ],
 )
 
+cc_library(
+    name = "defuser",
+    srcs = ["defuser.cc"],
+    hdrs = ["defuser.h"],
+    deps = [
+        ":call_graph",
+        ":hlo",
+        ":hlo_pass",
+        "//tensorflow/compiler/xla:status_macros",
+        "//tensorflow/compiler/xla:types",
+        "//tensorflow/compiler/xla:util",
+        "//tensorflow/core:lib",
+    ],
+)
+
+tf_cc_test(
+    name = "defuser_test",
+    srcs = ["defuser_test.cc"],
+    deps = [
+        ":defuser",
+        ":hlo_matchers",
+        "//tensorflow/compiler/xla:literal_util",
+        "//tensorflow/compiler/xla:shape_util",
+        "//tensorflow/compiler/xla/tests:hlo_verified_test_base",
+    ],
+)
+
 cc_library(
     name = "tuple_simplifier",
     srcs = ["tuple_simplifier.cc"],
diff --git a/tensorflow/compiler/xla/service/defuser.cc b/tensorflow/compiler/xla/service/defuser.cc
new file mode 100644
index 0000000000..d124f74d19
--- /dev/null
+++ b/tensorflow/compiler/xla/service/defuser.cc
@@ -0,0 +1,115 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/service/defuser.h"
+
+#include <algorithm>
+#include <memory>
+#include <numeric>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "tensorflow/compiler/xla/service/call_graph.h"
+#include "tensorflow/compiler/xla/service/hlo_computation.h"
+#include "tensorflow/compiler/xla/service/hlo_instruction.h"
+#include "tensorflow/compiler/xla/service/hlo_opcode.h"
+#include "tensorflow/compiler/xla/status_macros.h"
+#include "tensorflow/compiler/xla/types.h"
+#include "tensorflow/compiler/xla/util.h"
+#include "tensorflow/core/lib/core/errors.h"
+#include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/platform/logging.h"
+#include "tensorflow/core/platform/types.h"
+
+namespace xla {
+
+namespace {
+
+// Copy all the instructions in the given fusion instruction into the fusion
+// instruction's parent computation and replace the use of the fusion
+// instruction with the copy of the fusion expression root.
+Status Defuse(HloInstruction* fusion_instruction) {
+  VLOG(2) << "Defusing instruction: " << fusion_instruction->ToString();
+
+  HloComputation* fused_computation =
+      fusion_instruction->fused_instructions_computation();
+
+  // A map from fused instruction to its defused clone.
+  tensorflow::gtl::FlatMap<const HloInstruction*, HloInstruction*>
+      defused_instructions;
+  // Initialize map to contain the fusion instruction parameters mapping
+  // to the operands of the fusion instruction.
+  for (int64 i = 0; i < fusion_instruction->operand_count(); ++i) {
+    defused_instructions[fused_computation->parameter_instruction(i)] =
+        fusion_instruction->mutable_operand(i);
+  }
+
+  // Create a clone of each instruction of the fused computation in the same
+  // computation as the fusion instruction itself.
+  // TODO(b/68227302): Moving instruction to new computation rather than
+  // cloning and deleting.
+  for (HloInstruction* fused_instruction :
+       fused_computation->MakeInstructionPostOrder()) {
+    if (fused_instruction->opcode() == HloOpcode::kParameter) {
+      continue;
+    }
+    std::vector<HloInstruction*> new_operands;
+    for (HloInstruction* operand : fused_instruction->operands()) {
+      new_operands.push_back(defused_instructions.at(operand));
+    }
+    HloInstruction* defused_instruction =
+        fusion_instruction->parent()->AddInstruction(
+            fused_instruction->CloneWithNewOperands(fused_instruction->shape(),
+                                                    new_operands));
+    defused_instructions[fused_instruction] = defused_instruction;
+  }
+
+  TF_RETURN_IF_ERROR(fusion_instruction->ReplaceAllUsesWith(
+      defused_instructions.at(fusion_instruction->fused_expression_root())));
+
+  HloModule* module = fusion_instruction->parent()->parent();
+  TF_RETURN_IF_ERROR(
+      fusion_instruction->parent()->RemoveInstruction(fusion_instruction));
+  return module->RemoveEmbeddedComputation(fused_computation);
+}
+
+}  // namespace
+
+StatusOr<bool> Defuser::Run(HloModule* module) {
+  VLOG(1) << "Defusing module " << module->name();
+  XLA_VLOG_LINES(2, "Before defusion:\n" + module->ToString());
+
+  bool changed = false;
+  std::unique_ptr<CallGraph> call_graph = CallGraph::Build(module);
+  TF_RETURN_IF_ERROR(call_graph->VisitNodes(
+      [&](const CallGraphNode& call_graph_node) -> Status {
+        if (call_graph_node.computation()->IsFusionComputation()) {
+          TF_RET_CHECK(call_graph_node.caller_callsites().size() == 1);
+          HloInstruction* fusion_instruction =
+              call_graph_node.caller_callsites()[0].instruction();
+          TF_RETURN_IF_ERROR(Defuse(fusion_instruction));
+          changed = true;
+        }
+        return Status::OK();
+      },
+      /*visit_unreachable_nodes=*/true));
+
+  XLA_VLOG_LINES(2, "After defusion:\n" + module->ToString());
+
+  return changed;
+}
+
+}  // namespace xla
diff --git a/tensorflow/compiler/xla/service/defuser.h b/tensorflow/compiler/xla/service/defuser.h
new file mode 100644
index 0000000000..56b28fd22d
--- /dev/null
+++ b/tensorflow/compiler/xla/service/defuser.h
@@ -0,0 +1,41 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_COMPILER_XLA_SERVICE_DEFUSER_H_
+#define TENSORFLOW_COMPILER_XLA_SERVICE_DEFUSER_H_
+
+#include <utility>
+
+#include "tensorflow/compiler/xla/service/hlo_module.h"
+#include "tensorflow/compiler/xla/service/hlo_pass_interface.h"
+
+namespace xla {
+
+// A pass which replaces all fusion instructions with the equivalent un-fused
+// instructions.
+class Defuser : public HloPassInterface {
+ public:
+  Defuser() {}
+  ~Defuser() override {}
+  tensorflow::StringPiece name() const override { return "defuser"; }
+
+  // Run defusion on the given module. Returns whether the module was
+  // changed.
+  StatusOr<bool> Run(HloModule* module) override;
+};
+
+}  // namespace xla
+
+#endif  // TENSORFLOW_COMPILER_XLA_SERVICE_DEFUSER_H_
diff --git a/tensorflow/compiler/xla/service/defuser_test.cc b/tensorflow/compiler/xla/service/defuser_test.cc
new file mode 100644
index 0000000000..32b5c5d35f
--- /dev/null
+++ b/tensorflow/compiler/xla/service/defuser_test.cc
@@ -0,0 +1,214 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/service/defuser.h"
+
+#include "tensorflow/compiler/xla/literal_util.h"
+#include "tensorflow/compiler/xla/service/hlo_matchers.h"
+#include "tensorflow/compiler/xla/shape_util.h"
+#include "tensorflow/compiler/xla/tests/hlo_verified_test_base.h"
+
+namespace op = xla::testing::opcode_matchers;
+
+namespace xla {
+namespace {
+
+class DefuserTest : public HloVerifiedTestBase {
+ protected:
+  // Returns the number of fusion instructions in the module.
+  int FusionCount() {
+    int count = 0;
+    for (HloComputation* computation : module().computations()) {
+      if (computation->IsFusionComputation()) {
+        count++;
+      }
+    }
+    return count;
+  }
+
+  Defuser defuser_;
+  const Shape shape_ = ShapeUtil::MakeShape(F32, {2, 2});
+};
+
+TEST_F(DefuserTest, NoFusionInstruction) {
+  auto builder = HloComputation::Builder(TestName());
+  auto param0 =
+      builder.AddInstruction(HloInstruction::CreateParameter(0, shape_, "p0"));
+  auto param1 =
+      builder.AddInstruction(HloInstruction::CreateParameter(1, shape_, "p1"));
+  builder.AddInstruction(
+      HloInstruction::CreateBinary(shape_, HloOpcode::kAdd, param0, param1));
+
+  module().AddEntryComputation(builder.Build());
+  EXPECT_EQ(0, FusionCount());
+
+  EXPECT_FALSE(defuser_.Run(&module()).ValueOrDie());
+}
+
+TEST_F(DefuserTest, TrivialFusionInstructionAsRoot) {
+  auto builder = HloComputation::Builder(TestName());
+  auto param0 =
+      builder.AddInstruction(HloInstruction::CreateParameter(0, shape_, "p0"));
+  auto param1 =
+      builder.AddInstruction(HloInstruction::CreateParameter(1, shape_, "p1"));
+  auto add = builder.AddInstruction(
+      HloInstruction::CreateBinary(shape_, HloOpcode::kAdd, param0, param1));
+
+  auto computation = module().AddEntryComputation(builder.Build());
+  computation->CreateFusionInstruction({add},
+                                       HloInstruction::FusionKind::kLoop);
+
+  EXPECT_THAT(computation->root_instruction(), op::Fusion());
+
+  EXPECT_EQ(1, FusionCount());
+  EXPECT_TRUE(defuser_.Run(&module()).ValueOrDie());
+  EXPECT_EQ(0, FusionCount());
+
+  EXPECT_THAT(computation->root_instruction(),
+              op::Add(op::Parameter(), op::Parameter()));
+}
+
+TEST_F(DefuserTest, TrivialFusionInstructionNotAsRoot) {
+  auto builder = HloComputation::Builder(TestName());
+  auto param0 =
+      builder.AddInstruction(HloInstruction::CreateParameter(0, shape_, "p0"));
+  auto param1 =
+      builder.AddInstruction(HloInstruction::CreateParameter(1, shape_, "p1"));
+  auto add = builder.AddInstruction(
+      HloInstruction::CreateBinary(shape_, HloOpcode::kAdd, param0, param1));
+  builder.AddInstruction(
+      HloInstruction::CreateUnary(shape_, HloOpcode::kNegate, add));
+
+  auto computation = module().AddEntryComputation(builder.Build());
+  computation->CreateFusionInstruction({add},
+                                       HloInstruction::FusionKind::kLoop);
+
+  EXPECT_THAT(computation->root_instruction(), op::Negate(op::Fusion()));
+
+  EXPECT_EQ(1, FusionCount());
+  EXPECT_TRUE(defuser_.Run(&module()).ValueOrDie());
+  EXPECT_EQ(0, FusionCount());
+
+  EXPECT_THAT(computation->root_instruction(),
+              op::Negate(op::Add(op::Parameter(), op::Parameter())));
+}
+
+TEST_F(DefuserTest, NonTrivialFusionInstruction) {
+  auto builder = HloComputation::Builder(TestName());
+  auto param0 =
+      builder.AddInstruction(HloInstruction::CreateParameter(0, shape_, "p0"));
+  auto param1 =
+      builder.AddInstruction(HloInstruction::CreateParameter(1, shape_, "p1"));
+  auto param3 =
+      builder.AddInstruction(HloInstruction::CreateParameter(2, shape_, "p2"));
+  auto add = builder.AddInstruction(
+      HloInstruction::CreateBinary(shape_, HloOpcode::kAdd, param0, param1));
+  auto negate = builder.AddInstruction(
+      HloInstruction::CreateUnary(shape_, HloOpcode::kNegate, add));
+  auto sub = builder.AddInstruction(
+      HloInstruction::CreateBinary(shape_, HloOpcode::kSubtract, add, negate));
+  auto mul = builder.AddInstruction(
+      HloInstruction::CreateBinary(shape_, HloOpcode::kMultiply, sub, param3));
+  auto div = builder.AddInstruction(
+      HloInstruction::CreateBinary(shape_, HloOpcode::kDivide, mul, param3));
+  auto constant = builder.AddInstruction(HloInstruction::CreateConstant(
+      Literal::CreateR2<float>({{1.0, 2.0}, {3.0, 4.0}})));
+  auto add2 = builder.AddInstruction(
+      HloInstruction::CreateBinary(shape_, HloOpcode::kAdd, constant, div));
+
+  auto computation = module().AddEntryComputation(builder.Build());
+  computation->CreateFusionInstruction(
+      {add2, constant, div, mul, sub, negate, add},
+      HloInstruction::FusionKind::kLoop);
+
+  EXPECT_THAT(computation->root_instruction(), op::Fusion());
+
+  EXPECT_EQ(1, FusionCount());
+  EXPECT_TRUE(defuser_.Run(&module()).ValueOrDie());
+  EXPECT_EQ(0, FusionCount());
+
+  EXPECT_THAT(computation->root_instruction(),
+              op::Add(op::Constant(), op::Divide()));
+}
+
+TEST_F(DefuserTest, MultipleFusionInstructions) {
+  auto builder = HloComputation::Builder(TestName());
+  auto param0 =
+      builder.AddInstruction(HloInstruction::CreateParameter(0, shape_, "p0"));
+  auto param1 =
+      builder.AddInstruction(HloInstruction::CreateParameter(1, shape_, "p1"));
+  auto param3 =
+      builder.AddInstruction(HloInstruction::CreateParameter(2, shape_, "p2"));
+  auto add = builder.AddInstruction(
+      HloInstruction::CreateBinary(shape_, HloOpcode::kAdd, param0, param1));
+  auto negate = builder.AddInstruction(
+      HloInstruction::CreateUnary(shape_, HloOpcode::kNegate, add));
+  auto sub = builder.AddInstruction(
+      HloInstruction::CreateBinary(shape_, HloOpcode::kSubtract, add, negate));
+  auto mul = builder.AddInstruction(
+      HloInstruction::CreateBinary(shape_, HloOpcode::kMultiply, sub, param3));
+  auto div = builder.AddInstruction(
+      HloInstruction::CreateBinary(shape_, HloOpcode::kDivide, mul, param3));
+  auto constant = builder.AddInstruction(HloInstruction::CreateConstant(
+      Literal::CreateR2<float>({{1.0, 2.0}, {3.0, 4.0}})));
+  auto add2 = builder.AddInstruction(
+      HloInstruction::CreateBinary(shape_, HloOpcode::kAdd, constant, div));
+
+  auto computation = module().AddEntryComputation(builder.Build());
+  computation->CreateFusionInstruction({add2, constant, div, mul},
+                                       HloInstruction::FusionKind::kLoop);
+  computation->CreateFusionInstruction({sub, negate, add},
+                                       HloInstruction::FusionKind::kLoop);
+
+  EXPECT_THAT(computation->root_instruction(), op::Fusion());
+
+  EXPECT_EQ(2, FusionCount());
+  EXPECT_TRUE(defuser_.Run(&module()).ValueOrDie());
+  EXPECT_EQ(0, FusionCount());
+
+  EXPECT_THAT(computation->root_instruction(),
+              op::Add(op::Constant(), op::Divide()));
+}
+
+TEST_F(DefuserTest, NestedFusionInstructions) {
+  auto builder = HloComputation::Builder(TestName());
+  auto param0 =
+      builder.AddInstruction(HloInstruction::CreateParameter(0, shape_, "p0"));
+  auto param1 =
+      builder.AddInstruction(HloInstruction::CreateParameter(1, shape_, "p1"));
+  auto add = builder.AddInstruction(
+      HloInstruction::CreateBinary(shape_, HloOpcode::kAdd, param0, param1));
+  auto negate = builder.AddInstruction(
+      HloInstruction::CreateUnary(shape_, HloOpcode::kNegate, add));
+
+  auto computation = module().AddEntryComputation(builder.Build());
+  auto outer_fusion = computation->CreateFusionInstruction(
+      {negate, add}, HloInstruction::FusionKind::kLoop);
+  HloInstruction* fused_negate = outer_fusion->fused_expression_root();
+  ASSERT_EQ(fused_negate->opcode(), HloOpcode::kNegate);
+  outer_fusion->fused_instructions_computation()->CreateFusionInstruction(
+      {fused_negate}, HloInstruction::FusionKind::kLoop);
+
+  EXPECT_THAT(computation->root_instruction(), op::Fusion());
+
+  EXPECT_EQ(2, FusionCount());
+  EXPECT_TRUE(defuser_.Run(&module()).ValueOrDie());
+  EXPECT_EQ(0, FusionCount());
+
+  EXPECT_THAT(computation->root_instruction(), op::Negate(op::Add()));
+}
+
+}  // namespace
+}  // namespace xla
-- 
GitLab


From 772bebb16a6c359021ff789f89c6b7dfdd0d3126 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 25 Oct 2017 13:53:46 -0700
Subject: [PATCH 771/909] Better error message for ResourceVariable.set_shape

PiperOrigin-RevId: 173445012
---
 tensorflow/python/ops/resource_variable_ops.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tensorflow/python/ops/resource_variable_ops.py b/tensorflow/python/ops/resource_variable_ops.py
index 06c5a3bb2a..386fd204b6 100644
--- a/tensorflow/python/ops/resource_variable_ops.py
+++ b/tensorflow/python/ops/resource_variable_ops.py
@@ -680,6 +680,10 @@ class ResourceVariable(variables.Variable):
     """Unsupported."""
     raise NotImplementedError("ResourceVariable does not implement _ref()")
 
+  def set_shape(self, shape):
+    """Unsupported."""
+    raise NotImplementedError("ResourceVariable does not implement set_shape()")
+
   @staticmethod
   def _OverloadOperator(operator):  # pylint: disable=invalid-name
     """Defer an operator overload to `ops.Tensor`.
-- 
GitLab


From 433928745b5a76972a6128fe9b89f19b7a5e2d6c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 25 Oct 2017 14:02:16 -0700
Subject: [PATCH 772/909] Updated hlo parser tests to test "device=".

PiperOrigin-RevId: 173446328
---
 .../compiler/xla/tools/parser/hlo_parser_test.cc     | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/tensorflow/compiler/xla/tools/parser/hlo_parser_test.cc b/tensorflow/compiler/xla/tools/parser/hlo_parser_test.cc
index 5150e1f96d..2bf1cce1c0 100644
--- a/tensorflow/compiler/xla/tools/parser/hlo_parser_test.cc
+++ b/tensorflow/compiler/xla/tools/parser/hlo_parser_test.cc
@@ -100,8 +100,8 @@ ENTRY %add_constants () -> f32[] {
 R"(HloModule SelectR1F32WithCmpR1F32sFromParamsSmall_module:
 
 ENTRY %SelectR1F32WithCmpR1F32sFromParamsSmall.v4 (v1: f32[4], v2: f32[4]) -> f32[4] {
-  %v1 = f32[4]{0} parameter(0)
-  %v2 = f32[4]{0} parameter(1)
+  %v1 = f32[4]{0} parameter(0), device=1
+  %v2 = f32[4]{0} parameter(1), device=1
   %greater-than = pred[4]{0} greater-than(f32[4]{0} %v1, f32[4]{0} %v2)
   ROOT %select = f32[4]{0} select(pred[4]{0} %greater-than, f32[4]{0} %v1, f32[4]{0} %v2)
 }
@@ -164,9 +164,9 @@ ENTRY %WhileWithScalarS32Result.v2 () -> s32[] {
 R"(HloModule TwoSendRecvBothWayRecvFist_module:
 
 ENTRY %TwoSendRecvBothWayRecvFist.v3 () -> f32[] {
-  %recv = f32[] recv(), channel_id=15
-  ROOT %constant = f32[] constant(2.1)
-  %send = () send(f32[] %constant), channel_id=16
+  %recv = f32[] recv(), channel_id=15, device=1
+  ROOT %constant = f32[] constant(2.1), device=0
+  %send = () send(f32[] %constant), channel_id=16, device=0
 }
 
 )"
@@ -180,7 +180,7 @@ ENTRY %GetTupleElement.v4 () -> s32[] {
   %constant = f32[] constant(1.23)
   %constant.1 = s32[] constant(4)
   %tuple = (f32[], s32[]) tuple(f32[] %constant, s32[] %constant.1)
-  ROOT %get-tuple-element = s32[] get-tuple-element((f32[], s32[]) %tuple), index=1
+  ROOT %get-tuple-element = s32[] get-tuple-element((f32[], s32[]) %tuple), index=1, device=0
 }
 
 )"
-- 
GitLab


From b81c8eda6074d17142b1f6e64ab30b341a8338a0 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 25 Oct 2017 14:04:56 -0700
Subject: [PATCH 773/909] Better shape function for fixed width histogram op.

PiperOrigin-RevId: 173446796
---
 tensorflow/core/ops/math_ops.cc             |  9 ++++++++-
 tensorflow/python/ops/histogram_ops_test.py | 19 +++++++++++++++++++
 2 files changed, 27 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/ops/math_ops.cc b/tensorflow/core/ops/math_ops.cc
index 61db896c51..130e3ed781 100644
--- a/tensorflow/core/ops/math_ops.cc
+++ b/tensorflow/core/ops/math_ops.cc
@@ -2258,7 +2258,14 @@ REGISTER_OP("HistogramFixedWidth")
     .Attr("T: {int32, int64, float32, float64}")
     .Attr("dtype: {int32, int64} = DT_INT32")
     .SetShapeFn([](InferenceContext* c) {
-      c->set_output(0, c->UnknownShapeOfRank(1));
+      const Tensor* nbins_input = c->input_tensor(2);
+      if (nbins_input != nullptr) {
+        int64 nbins;
+        TF_RETURN_IF_ERROR(c->GetScalarFromTensor(nbins_input, &nbins));
+        c->set_output(0, c->Vector(nbins));
+      } else {
+        c->set_output(0, c->UnknownShapeOfRank(1));
+      }
       return Status::OK();
     })
     .Doc(R"doc(
diff --git a/tensorflow/python/ops/histogram_ops_test.py b/tensorflow/python/ops/histogram_ops_test.py
index bf6e0296f6..19ad6cd2ba 100644
--- a/tensorflow/python/ops/histogram_ops_test.py
+++ b/tensorflow/python/ops/histogram_ops_test.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 import numpy as np
 
 from tensorflow.python.framework import dtypes
+from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import histogram_ops
 from tensorflow.python.platform import test
 
@@ -75,6 +76,24 @@ class HistogramFixedWidthTest(test.TestCase):
       self.assertEqual(dtypes.int32, hist.dtype)
       self.assertAllClose(expected_bin_counts, hist.eval())
 
+  def test_shape_inference(self):
+    value_range = [0.0, 5.0]
+    values = [[-1.0, 0.0, 1.5], [2.0, 5.0, 15]]
+    expected_bin_counts = [2, 1, 1, 0, 2]
+    placeholder = array_ops.placeholder(dtypes.int32)
+    with self.test_session(use_gpu=True):
+      hist = histogram_ops.histogram_fixed_width(values, value_range, nbins=5)
+      self.assertAllEqual(hist.shape.as_list(), (5,))
+      self.assertEqual(dtypes.int32, hist.dtype)
+      self.assertAllClose(expected_bin_counts, hist.eval())
+
+      hist = histogram_ops.histogram_fixed_width(values, value_range,
+                                                 nbins=placeholder)
+      self.assertEquals(hist.shape.ndims, 1)
+      self.assertIs(hist.shape[0].value, None)
+      self.assertEqual(dtypes.int32, hist.dtype)
+      self.assertAllClose(expected_bin_counts, hist.eval({placeholder: 5}))
+
 
 if __name__ == '__main__':
   test.main()
-- 
GitLab


From 2b95f70b25101a02befd6495370cb16aaa722f6a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 25 Oct 2017 14:05:02 -0700
Subject: [PATCH 774/909] Show an error message in _einsum_reduction
 ValueError.

If `tf.einsum()` is called and the length of the axis labels do not match the
rank of the tensor, the ValueError now states what the problem is, which tensor
is causing it, and what the rank and length of the labels are.

RELNOTES: n/a
PiperOrigin-RevId: 173446812
---
 tensorflow/python/ops/special_math_ops.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/ops/special_math_ops.py b/tensorflow/python/ops/special_math_ops.py
index 87561cff92..fe3f734322 100644
--- a/tensorflow/python/ops/special_math_ops.py
+++ b/tensorflow/python/ops/special_math_ops.py
@@ -251,9 +251,13 @@ def _einsum_reduction(t0, t0_axis_labels, t1, t1_axis_labels, axes_to_sum):
       `t1_axis_labels`.
   """
   if len(t0_axis_labels) != len(t0.get_shape()):
-    raise ValueError()
+    raise ValueError(
+        'Tensor t0 of rank %d does not match einsum reduction of length %d' %
+        (len(t0.get_shape()), len(t0_axis_labels)))
   if len(t1_axis_labels) != len(t1.get_shape()):
-    raise ValueError()
+    raise ValueError(
+        'Tensor t1 of rank %d does not match einsum reduction of length %d' %
+        (len(t1.get_shape()), len(t1_axis_labels)))
 
   # This function computes the result of a two-argument einsum() using batch
   # matrix multiplication.  This involves
-- 
GitLab


From 64d3df30985a523d2dcd6b76a37a1fb420d2f2aa Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 25 Oct 2017 14:05:09 -0700
Subject: [PATCH 775/909] Adds loss_fn argument to multi_label_head.

PiperOrigin-RevId: 173446829
---
 tensorflow/contrib/estimator/BUILD            |  2 +
 .../estimator/python/estimator/head.py        | 84 +++++++++++++++++--
 .../estimator/python/estimator/head_test.py   | 80 ++++++++++++++++++
 3 files changed, 158 insertions(+), 8 deletions(-)

diff --git a/tensorflow/contrib/estimator/BUILD b/tensorflow/contrib/estimator/BUILD
index 89c26d1d2f..8a7d67b5c2 100644
--- a/tensorflow/contrib/estimator/BUILD
+++ b/tensorflow/contrib/estimator/BUILD
@@ -134,6 +134,7 @@ py_library(
         "//tensorflow/python/estimator:metric_keys",
         "//tensorflow/python/estimator:model_fn",
         "//tensorflow/python/estimator:prediction_keys",
+        "//tensorflow/python/estimator:util",
         "//tensorflow/python/ops/losses",
         "//tensorflow/python/saved_model:signature_constants",
     ],
@@ -154,6 +155,7 @@ py_test(
         "//tensorflow/python:dtypes",
         "//tensorflow/python:errors",
         "//tensorflow/python:framework_ops",
+        "//tensorflow/python:math_ops",
         "//tensorflow/python:sparse_tensor",
         "//tensorflow/python:string_ops",
         "//tensorflow/python:training",
diff --git a/tensorflow/contrib/estimator/python/estimator/head.py b/tensorflow/contrib/estimator/python/estimator/head.py
index d01b30d7f9..189f098005 100644
--- a/tensorflow/contrib/estimator/python/estimator/head.py
+++ b/tensorflow/contrib/estimator/python/estimator/head.py
@@ -19,6 +19,7 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.python.estimator import model_fn
+from tensorflow.python.estimator import util
 from tensorflow.python.estimator.canned import head as head_lib
 from tensorflow.python.estimator.canned import metric_keys
 from tensorflow.python.estimator.canned import prediction_keys
@@ -147,6 +148,7 @@ def multi_label_head(n_classes,
                      weight_column=None,
                      thresholds=None,
                      label_vocabulary=None,
+                     loss_fn=None,
                      name=None):
   """Creates a `_Head` for multi-label classification.
 
@@ -158,6 +160,12 @@ def multi_label_head(n_classes,
   multi-hot tensor of shape `[batch_size, n_classes]`, or as an integer
   `SparseTensor` of class indices.
 
+  Also supports custom `loss_fn`. `loss_fn` takes `(labels, logits)` or
+  `(labels, logits, features)` as arguments and returns unreduced loss with
+  shape `[batch_size, 1]`. `loss_fn` must support indicator `labels` with shape
+  `[batch_size, n_classes]`. Namely, the head applies `label_vocabulary` to the
+  input labels before passing them to `loss_fn`.
+
   Args:
     n_classes: Number of classes, must be greater than 1 (for 1 class, use
       `binary_classification_head`).
@@ -174,6 +182,7 @@ def multi_label_head(n_classes,
       [0, n_classes) or multi-hot Tensor. If given, labels must be SparseTensor
       string type and have any value in `label_vocabulary`. Also there will be
       errors if vocabulary is not provided and labels are string.
+    loss_fn: Optional loss function.
     name: name of the head. If provided, summary and metrics keys will be
       suffixed by `"/" + name`. Also used as `name_scope` when creating ops.
 
@@ -201,9 +210,11 @@ def multi_label_head(n_classes,
       raise ValueError(
           'Length of label_vocabulary must be n_classes ({}). '
           'Given: {}'.format(n_classes, len(label_vocabulary)))
+  if loss_fn:
+    _validate_loss_fn_args(loss_fn)
   return _MultiLabelHead(
       n_classes=n_classes, weight_column=weight_column, thresholds=thresholds,
-      label_vocabulary=label_vocabulary, name=name)
+      label_vocabulary=label_vocabulary, loss_fn=loss_fn, name=name)
 
 
 class _MultiLabelHead(head_lib._Head):  # pylint:disable=protected-access
@@ -214,11 +225,13 @@ class _MultiLabelHead(head_lib._Head):  # pylint:disable=protected-access
                weight_column=None,
                thresholds=None,
                label_vocabulary=None,
+               loss_fn=None,
                name=None):
     self._n_classes = n_classes
     self._weight_column = weight_column
     self._thresholds = thresholds
     self._label_vocabulary = label_vocabulary
+    self._loss_fn = loss_fn
     self._name = name
 
   @property
@@ -263,14 +276,19 @@ class _MultiLabelHead(head_lib._Head):  # pylint:disable=protected-access
 
   def create_loss(self, features, mode, logits, labels):
     """See `Head`."""
-    del mode, features  # Unused for this head.
+    del mode  # Unused for this head.
     processed_labels = self._process_labels(labels)
-    unweighted_loss = losses.sigmoid_cross_entropy(
-        multi_class_labels=processed_labels, logits=logits,
-        reduction=losses.Reduction.NONE)
-    # Averages loss over classes.
-    unweighted_loss = math_ops.reduce_mean(
-        unweighted_loss, axis=-1, keep_dims=True)
+    if self._loss_fn:
+      unweighted_loss = _call_loss_fn(
+          loss_fn=self._loss_fn, labels=processed_labels, logits=logits,
+          features=features)
+    else:
+      unweighted_loss = losses.sigmoid_cross_entropy(
+          multi_class_labels=processed_labels, logits=logits,
+          reduction=losses.Reduction.NONE)
+      # Averages loss over classes.
+      unweighted_loss = math_ops.reduce_mean(
+          unweighted_loss, axis=-1, keep_dims=True)
     return head_lib.LossAndLabels(
         unweighted_loss=unweighted_loss,
         processed_labels=processed_labels)
@@ -386,3 +404,53 @@ class _MultiLabelHead(head_lib._Head):  # pylint:disable=protected-access
                 threshold=threshold,
                 name=recall_key))
     return metric_ops
+
+
+def _validate_loss_fn_args(loss_fn):
+  """Validates loss_fn arguments.
+
+  Required arguments: labels, logits.
+  Optional arguments: features.
+
+  Args:
+    loss_fn: The loss function.
+  Raises:
+    ValueError: If the signature is unexpected.
+  """
+  loss_fn_args = util.fn_args(loss_fn)
+  for required_arg in ['labels', 'logits']:
+    if required_arg not in loss_fn_args:
+      raise ValueError(
+          'loss_fn must contain argument: {}. '
+          'Given arguments: {}'.format(required_arg, loss_fn_args))
+  invalid_args = list(set(loss_fn_args) - set(['labels', 'logits', 'features']))
+  if invalid_args:
+    raise ValueError('loss_fn has unexpected args: {}'.format(invalid_args))
+
+
+def _call_loss_fn(loss_fn, labels, logits, features):
+  """Calls loss_fn and checks the returned shape.
+
+  Args:
+    loss_fn: The loss function.
+    labels: Processed labels Tensor.
+    logits: Logits Tensor of shape [batch_size, logits_dimension].
+    features: Features dict.
+  Returns:
+    Loss Tensor with shape [batch_size, 1].
+  """
+  loss_fn_args = util.fn_args(loss_fn)
+  kwargs = {}
+  if 'features' in loss_fn_args:
+    kwargs['features'] = features
+  unweighted_loss = loss_fn(labels=labels, logits=logits, **kwargs)
+  batch_size = array_ops.shape(logits)[0]
+  loss_shape = array_ops.shape(unweighted_loss)
+  check_shape_op = control_flow_ops.Assert(
+      math_ops.reduce_all(math_ops.equal(loss_shape, [batch_size, 1])),
+      data=[
+          'loss_fn must return Tensor of shape [batch_size, 1]. Given: ',
+          loss_shape])
+  with ops.control_dependencies([check_shape_op]):
+    return array_ops.identity(unweighted_loss)
+
diff --git a/tensorflow/contrib/estimator/python/estimator/head_test.py b/tensorflow/contrib/estimator/python/estimator/head_test.py
index b7252f93ee..db7d96d508 100644
--- a/tensorflow/contrib/estimator/python/estimator/head_test.py
+++ b/tensorflow/contrib/estimator/python/estimator/head_test.py
@@ -33,6 +33,7 @@ from tensorflow.python.framework import ops
 from tensorflow.python.framework import sparse_tensor
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import string_ops
 from tensorflow.python.platform import test
 from tensorflow.python.saved_model import signature_constants
@@ -131,6 +132,37 @@ class MultiLabelHead(test.TestCase):
         r'Length of label_vocabulary must be n_classes \(3\). Given: 2'):
       head_lib.multi_label_head(n_classes=3, label_vocabulary=['foo', 'bar'])
 
+  def test_loss_fn_arg_labels_missing(self):
+    def _loss_fn(logits):
+      del logits  # Unused
+    with self.assertRaisesRegexp(
+        ValueError,
+        r'loss_fn must contain argument: labels\. '
+        r'Given arguments: \(\'logits\',\)'):
+      head_lib.multi_label_head(n_classes=3, loss_fn=_loss_fn)
+
+  def test_loss_fn_arg_logits_missing(self):
+    def _loss_fn(labels):
+      del labels  # unused
+    with self.assertRaisesRegexp(
+        ValueError,
+        r'loss_fn must contain argument: logits\. '
+        r'Given arguments: \(\'labels\',\)'):
+      head_lib.multi_label_head(n_classes=3, loss_fn=_loss_fn)
+
+  def test_loss_fn_arg_features_ok(self):
+    def _loss_fn(labels, logits, features):
+      del labels, logits, features  # Unused
+    head_lib.multi_label_head(n_classes=3, loss_fn=_loss_fn)
+
+  def test_loss_fn_arg_invalid(self):
+    def _loss_fn(labels, logits, name=None):
+      del labels, logits, name  # Unused
+    with self.assertRaisesRegexp(
+        ValueError,
+        r'loss_fn has unexpected args: \[\'name\'\]'):
+      head_lib.multi_label_head(n_classes=3, loss_fn=_loss_fn)
+
   def test_name(self):
     head = head_lib.multi_label_head(n_classes=4, name='foo')
     self.assertEqual('foo', head.name)
@@ -291,6 +323,54 @@ class MultiLabelHead(test.TestCase):
         actual_unweighted_loss.eval(
             {labels_placeholder: np.array([1, 1], dtype=np.int64)})
 
+  def test_eval_create_loss_loss_fn(self):
+    """Tests head.create_loss for eval mode and custom loss_fn."""
+    loss = np.array([[1.], [2.]], dtype=np.float32)
+    logits_input = np.array([[-10., 10.], [-15., 10.]], dtype=np.float32)
+    labels_input = np.array([[1, 0], [1, 1]], dtype=np.int64)
+    def _loss_fn(labels, logits):
+      check_labels = control_flow_ops.Assert(
+          math_ops.reduce_all(math_ops.equal(labels, labels_input)),
+          data=[labels])
+      check_logits = control_flow_ops.Assert(
+          math_ops.reduce_all(math_ops.equal(logits, logits_input)),
+          data=[logits])
+      with ops.control_dependencies([check_labels, check_logits]):
+        return constant_op.constant(loss)
+    head = head_lib.multi_label_head(n_classes=2, loss_fn=_loss_fn)
+
+    actual_unweighted_loss, _ = head.create_loss(
+        features={'x': np.array(((42,),), dtype=np.int32)},
+        mode=model_fn.ModeKeys.EVAL,
+        logits=logits_input,
+        labels=labels_input)
+    with self.test_session():
+      _initialize_variables(self, monitored_session.Scaffold())
+      self.assertAllClose(loss, actual_unweighted_loss.eval())
+
+  def test_eval_create_loss_loss_fn_wrong_shape(self):
+    """Tests custom loss_fn that returns Tensor of unexpected shape."""
+    loss = np.array([1., 2.], dtype=np.float32)
+    def _loss_fn(labels, logits):
+      del labels, logits  # Unused
+      return constant_op.constant(loss)
+    head = head_lib.multi_label_head(n_classes=2, loss_fn=_loss_fn)
+
+    logits = np.array([[-10., 10.], [-15., 10.]], dtype=np.float32)
+    labels = np.array([[1, 0], [1, 1]], dtype=np.int64)
+    actual_unweighted_loss, _ = head.create_loss(
+        features={'x': np.array(((42,),), dtype=np.int32)},
+        mode=model_fn.ModeKeys.EVAL,
+        logits=logits,
+        labels=labels)
+    with self.test_session():
+      _initialize_variables(self, monitored_session.Scaffold())
+      with self.assertRaisesRegexp(
+          errors.InvalidArgumentError,
+          r'loss_fn must return Tensor of shape \[batch_size, 1\]\. '
+          r'Given: \] \[2\]'):
+        actual_unweighted_loss.eval()
+
   def test_eval_labels_none(self):
     """Tests that error is raised when labels is None."""
     head = head_lib.multi_label_head(n_classes=2)
-- 
GitLab


From 7492e88c2c4899da56921b09ecfaf3bd18ef2995 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 25 Oct 2017 14:08:31 -0700
Subject: [PATCH 776/909] FusedConv2DBiasActivation never uses GEMM, so doesn't
 need a subclass of NodeProcessor.

PiperOrigin-RevId: 173447332
---
 tensorflow/core/grappler/optimizers/layout_optimizer.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/core/grappler/optimizers/layout_optimizer.cc b/tensorflow/core/grappler/optimizers/layout_optimizer.cc
index 11cab8099a..b364446ad7 100644
--- a/tensorflow/core/grappler/optimizers/layout_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/layout_optimizer.cc
@@ -54,6 +54,7 @@ std::set<string> GetOpsFormatSupported() {
                                            "BiasAddGrad",
                                            "FusedBatchNorm",
                                            "FusedBatchNormGrad",
+                                           "FusedConv2DBiasActivation",
                                            "MaxPool",
                                            "MaxPoolGrad"};
   return ops_format_supported;
-- 
GitLab


From fd42ab3cb32ca3a6ef72dc9efb674134f264a58b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 25 Oct 2017 14:09:48 -0700
Subject: [PATCH 777/909] Adds round_mode to QuantizeV2 op to select rounding
 algorithm. Options are half-away-from-zero and half-to-even.
 round_mode=HALF_TO_EVEN currently only applies when mode="SCALED".

PiperOrigin-RevId: 173447503
---
 tensorflow/core/kernels/BUILD                |  1 +
 tensorflow/core/kernels/quantize_op.cc       | 72 ++++++++++++++------
 tensorflow/core/kernels/quantize_op_test.cc  | 61 +++++++++++++++++
 tensorflow/core/ops/array_ops.cc             |  9 ++-
 tensorflow/python/ops/array_ops.py           | 21 ++++++
 tensorflow/tools/api/golden/tensorflow.pbtxt |  2 +-
 6 files changed, 143 insertions(+), 23 deletions(-)

diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 92a0dbd0ab..277b21f833 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -4869,6 +4869,7 @@ tf_kernel_library(
     deps = [
         ":concat_lib_hdrs",
         ":conv_ops",
+        ":cwise_op",
         ":eigen_helpers",
         ":image_resizer_state",
         ":ops_util",
diff --git a/tensorflow/core/kernels/quantize_op.cc b/tensorflow/core/kernels/quantize_op.cc
index fd34e13c29..75aa47cd6b 100644
--- a/tensorflow/core/kernels/quantize_op.cc
+++ b/tensorflow/core/kernels/quantize_op.cc
@@ -21,6 +21,7 @@ limitations under the License.
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/type_traits.h"
 #include "tensorflow/core/framework/types.h"
+#include "tensorflow/core/kernels/cwise_ops.h"
 #include "tensorflow/core/kernels/meta_support.h"
 #include "tensorflow/core/kernels/quantization_utils.h"
 #include "tensorflow/core/lib/core/errors.h"
@@ -31,6 +32,19 @@ enum {
   QUANTIZE_MODE_MIN_FIRST,
   QUANTIZE_MODE_SCALED,
 };
+enum {
+  // Round half away from zero: if the fraction of y is exactly 0.5, then
+  // round(y) = y + 0.5 if y > 0
+  // round(y) = y - 0.5 if y < 0
+  // E.g., -5.5 gets rounded to -6, -5.4 goes to -5,
+  // 5.4 goes to 5, and 5.5 goes to 6.
+  ROUND_HALF_AWAY_FROM_ZERO,
+  // Round half to even: if the fraction of y is exactly 0.5, then round(y) is
+  // the nearest even integer to y.
+  // E.g., 23.5 gets rounded to 24, 24.5 gets rounded to 24, while -23.5 becomes
+  // -24, and -24.5 gets rounded to 24.
+  ROUND_HALF_TO_EVEN,
+};
 }  // namespace
 
 namespace tensorflow {
@@ -66,6 +80,26 @@ class QuantizeV2Op : public OpKernel {
     } else if (mode_string == "SCALED") {
       mode_ = QUANTIZE_MODE_SCALED;
     }
+
+    string round_mode_string;
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("round_mode", &round_mode_string));
+    OP_REQUIRES(ctx,
+                (round_mode_string == "HALF_AWAY_FROM_ZERO" ||
+                 round_mode_string == "HALF_TO_EVEN"),
+                errors::InvalidArgument("Round mode string must be "
+                                        "'HALF_AWAY_FROM_ZERO' or "
+                                        "'HALF_TO_EVEN', is '" +
+                                        round_mode_string + "'"));
+    if (round_mode_string == "HALF_AWAY_FROM_ZERO") {
+      round_mode_ = ROUND_HALF_AWAY_FROM_ZERO;
+    } else if (round_mode_string == "HALF_TO_EVEN") {
+      OP_REQUIRES(ctx, mode_string == "SCALED",
+                  errors::InvalidArgument("Round mode 'HALF_TO_EVEN' "
+                                          "only supported for mode 'SCALED', "
+                                          "but mode is '" +
+                                          mode_string + "'."));
+      round_mode_ = ROUND_HALF_TO_EVEN;
+    }
   }
 
   void Compute(OpKernelContext* ctx) override {
@@ -151,40 +185,37 @@ class QuantizeV2Op : public OpKernel {
       typename TTypes<T>::Vec o = output->template flat<T>();
       static constexpr int num_bits = sizeof(T) * 8;
       const float max_abs = std::max(std::abs(min_range), std::abs(max_range));
-      bool is_signed = std::is_signed<T>::value;
+      const bool is_signed = std::is_signed<T>::value;
+      float target_range;
       if (is_signed) {
         max_range = max_abs;
         min_range = -max_abs;
         // If it is signed, we try to keep 0.0 being 0 and drop one bucket. For
         // example, if it is 8 bits, we have the range [-127, 127]. So for input
         // range of [-x, x], the scale should be 254/(2*x).
-        const float target_range =
-            static_cast<float>((uint64_t{1} << (num_bits - 1)) - 1);
-        const float scale_factor = target_range / max_abs;
-        // Note that std::round is used to round the number before the cast.
-        // std::round implements "round-half-away-zero",
-        // e.g., -5.5 gets rounded to -6, -5.4 goes to -5, 5.4 goes to 5,
-        // and 5.5 goes to 6.
-        o.device(ctx->template eigen_device<Device>()) =
-            (input.flat<float>().cwiseMin(max_range).cwiseMax(min_range) *
-             scale_factor)
-                .round()
-                .template cast<T>();
+        target_range = static_cast<float>((uint64_t{1} << (num_bits - 1)) - 1);
       } else {
         max_range = max_abs;
         min_range = 0.0;
         // If it is unsigned and num_bits == 8, the range with 8 bits is [0,
         // 255].  If the input range is [0, x], then the scale is x/255 instead
         // of 254 as in the case above.
-        const float target_range =
-            static_cast<float>((uint64_t{1} << num_bits) - 1);
-        const float scale_factor = target_range / max_abs;
-        // Because input is unsigned, we don't need to implement "round away
-        // from zero".  The fast path avoids unaryExpr.
+        target_range = static_cast<float>((uint64_t{1} << num_bits) - 1);
+      }
+      const float scale_factor = target_range / max_abs;
+      if (round_mode_ == ROUND_HALF_TO_EVEN) {
+        // scalar_round_op_google implements "round-half-to-even".
         o.device(ctx->template eigen_device<Device>()) =
             (input.flat<float>().cwiseMin(max_range).cwiseMax(min_range) *
-                 scale_factor +
-             0.5f)
+             scale_factor)
+                .unaryExpr(Eigen::internal::scalar_round_op_google<float>())
+                .template cast<T>();
+      } else if (round_mode_ == ROUND_HALF_AWAY_FROM_ZERO) {
+        // scalar_round_op implements "round-half-away-from-zero".
+        o.device(ctx->template eigen_device<Device>()) =
+            (input.flat<float>().cwiseMin(max_range).cwiseMax(min_range) *
+             scale_factor)
+                .unaryExpr(Eigen::internal::scalar_round_op<float>())
                 .template cast<T>();
       }
     }
@@ -201,6 +232,7 @@ class QuantizeV2Op : public OpKernel {
  private:
   float half_range_;
   int mode_;
+  int round_mode_;
 };
 
 REGISTER_KERNEL_BUILDER(
diff --git a/tensorflow/core/kernels/quantize_op_test.cc b/tensorflow/core/kernels/quantize_op_test.cc
index 8a370966b4..d2cc55a94d 100644
--- a/tensorflow/core/kernels/quantize_op_test.cc
+++ b/tensorflow/core/kernels/quantize_op_test.cc
@@ -82,6 +82,7 @@ TEST_F(QuantizedOpTest, QuantizeV2Quint8Scaled) {
   test::FillValues<float>(&expected_output_max, {255.0});
   test::ExpectTensorEqual<float>(expected_output_max, *GetOutput(2));
 }
+
 TEST_F(QuantizedOpTest, QuantizeV2Quint8ScaledSmallInputRange) {
   TF_ASSERT_OK(NodeDefBuilder("quantize_op", "QuantizeV2")
                    .Input(FakeInput(DT_FLOAT))
@@ -170,6 +171,66 @@ TEST_F(QuantizedOpTest, QuantizeV2Qint8ScaledSmallInputRange) {
   test::ExpectTensorEqual<float>(expected_output_max, *GetOutput(2));
 }
 
+TEST_F(QuantizedOpTest, QuantizeV2Qint8ScaledRoundToEven) {
+  TF_ASSERT_OK(NodeDefBuilder("quantize_op", "QuantizeV2")
+                   .Input(FakeInput(DT_FLOAT))
+                   .Input(FakeInput(DT_FLOAT))
+                   .Input(FakeInput(DT_FLOAT))
+                   .Attr("T", DataTypeToEnum<qint8>::v())
+                   .Attr("mode", "SCALED")
+                   .Attr("round_mode", "HALF_TO_EVEN")
+                   .Finalize(node_def()));
+  TF_ASSERT_OK(InitOp());
+  AddInputFromArray<float>(TensorShape({7}),
+                           {-126.5, 0.0, 1.0, 2.5, 3.5, 64.0, 127.0});
+  AddInputFromArray<float>(TensorShape({1}), {-127.0f});
+  AddInputFromArray<float>(TensorShape({1}), {-127.0f});
+  TF_ASSERT_OK(RunOpKernel());
+  Tensor expected(allocator(), DT_QINT8, TensorShape({7}));
+  // Input element 0.0 should map to 0.
+  // Input element 127.0 maps to 127.
+  test::FillValues<qint8>(&expected, {-126, 0, 1, 2, 4, 64, 127});
+  test::ExpectTensorEqual<qint8>(expected, *GetOutput(0));
+
+  Tensor expected_output_min(allocator(), DT_FLOAT, TensorShape({}));
+  test::FillValues<float>(&expected_output_min, {-127.0});
+  test::ExpectTensorEqual<float>(expected_output_min, *GetOutput(1));
+
+  Tensor expected_output_max(allocator(), DT_FLOAT, TensorShape({}));
+  test::FillValues<float>(&expected_output_max, {127.0});
+  test::ExpectTensorEqual<float>(expected_output_max, *GetOutput(2));
+}
+
+TEST_F(QuantizedOpTest, QuantizeV2Qint8ScaledRoundAwayFromZero) {
+  TF_ASSERT_OK(NodeDefBuilder("quantize_op", "QuantizeV2")
+                   .Input(FakeInput(DT_FLOAT))
+                   .Input(FakeInput(DT_FLOAT))
+                   .Input(FakeInput(DT_FLOAT))
+                   .Attr("T", DataTypeToEnum<qint8>::v())
+                   .Attr("mode", "SCALED")
+                   .Attr("round_mode", "HALF_AWAY_FROM_ZERO")
+                   .Finalize(node_def()));
+  TF_ASSERT_OK(InitOp());
+  AddInputFromArray<float>(TensorShape({7}),
+                           {-126.5, 0.0, 1.0, 2.5, 3.5, 64.0, 127.0});
+  AddInputFromArray<float>(TensorShape({1}), {-127.0f});
+  AddInputFromArray<float>(TensorShape({1}), {-127.0f});
+  TF_ASSERT_OK(RunOpKernel());
+  Tensor expected(allocator(), DT_QINT8, TensorShape({7}));
+  // Input element 0.0 should map to 0.
+  // Input element 127.0 maps to 127.
+  test::FillValues<qint8>(&expected, {-127, 0, 1, 3, 4, 64, 127});
+  test::ExpectTensorEqual<qint8>(expected, *GetOutput(0));
+
+  Tensor expected_output_min(allocator(), DT_FLOAT, TensorShape({}));
+  test::FillValues<float>(&expected_output_min, {-127.0});
+  test::ExpectTensorEqual<float>(expected_output_min, *GetOutput(1));
+
+  Tensor expected_output_max(allocator(), DT_FLOAT, TensorShape({}));
+  test::FillValues<float>(&expected_output_max, {127.0});
+  test::ExpectTensorEqual<float>(expected_output_max, *GetOutput(2));
+}
+
 TEST_F(QuantizedOpTest, QuantizeV2_32Bit) {
   TF_ASSERT_OK(NodeDefBuilder("quantize_op", "QuantizeV2")
                    .Input(FakeInput(DT_FLOAT))
diff --git a/tensorflow/core/ops/array_ops.cc b/tensorflow/core/ops/array_ops.cc
index c5935141f8..f73bc716d5 100644
--- a/tensorflow/core/ops/array_ops.cc
+++ b/tensorflow/core/ops/array_ops.cc
@@ -4859,6 +4859,9 @@ REGISTER_OP("QuantizeV2")
     .Output("output_max: float")
     .Attr("T: quantizedtype")
     .Attr("mode: {'MIN_COMBINED', 'MIN_FIRST', 'SCALED'} = 'MIN_COMBINED'")
+    .Attr(
+        "round_mode: {'HALF_AWAY_FROM_ZERO', 'HALF_TO_EVEN'} = "
+        "'HALF_AWAY_FROM_ZERO'")
     .SetShapeFn([](InferenceContext* c) {
       TF_RETURN_IF_ERROR(shape_inference::UnchangedShape(c));
       ShapeHandle unused;
@@ -4873,7 +4876,9 @@ Quantize the 'input' tensor of type float to 'output' tensor of type 'T'.
 
 [min_range, max_range] are scalar floats that specify the range for
 the 'input' data. The 'mode' attribute controls exactly which calculations are
-used to convert the float values to their quantized equivalents.
+used to convert the float values to their quantized equivalents.  The
+'round_mode' attribute controls which rounding tie-breaking algorithm is used
+when rounding float values to their quantized equivalents.
 
 In 'MIN_COMBINED' mode, each value of the tensor will undergo the following:
 
@@ -4950,7 +4955,7 @@ From this we compute our scaling factor, s:
 
 Now we can quantize the elements of our tensor:
 ```c++
-result = (input * s).round_to_nearest()
+result = round(input * s)
 ```
 
 One thing to watch out for is that the operator may choose to adjust the
diff --git a/tensorflow/python/ops/array_ops.py b/tensorflow/python/ops/array_ops.py
index c00efb16ba..97dc63ebb1 100644
--- a/tensorflow/python/ops/array_ops.py
+++ b/tensorflow/python/ops/array_ops.py
@@ -2526,3 +2526,24 @@ def gather(params, indices, validate_indices=None, name=None, axis=0):
 
 
 gather.__doc__ = gen_array_ops.gather_v2.__doc__
+
+
+# Define quantize_v2 here in order to make name the second-to-last attribute,
+# because round_mode was added later.
+def quantize_v2(input,
+                min_range,
+                max_range,
+                T,
+                mode="MIN_COMBINED",
+                name=None,
+                round_mode="HALF_AWAY_FROM_ZERO"):
+  return gen_array_ops.quantize_v2(input,
+                                   min_range,
+                                   max_range,
+                                   T=T,
+                                   mode=mode,
+                                   name=name,
+                                   round_mode=round_mode)
+
+
+quantize_v2.__doc__ = gen_array_ops.quantize_v2.__doc__
diff --git a/tensorflow/tools/api/golden/tensorflow.pbtxt b/tensorflow/tools/api/golden/tensorflow.pbtxt
index d56a59de72..1c6f3cc534 100644
--- a/tensorflow/tools/api/golden/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.pbtxt
@@ -1482,7 +1482,7 @@ tf_module {
   }
   member_method {
     name: "quantize_v2"
-    argspec: "args=[\'input\', \'min_range\', \'max_range\', \'T\', \'mode\', \'name\'], varargs=None, keywords=None, defaults=[\'MIN_COMBINED\', \'None\'], "
+    argspec: "args=[\'input\', \'min_range\', \'max_range\', \'T\', \'mode\', \'name\', \'round_mode\'], varargs=None, keywords=None, defaults=[\'MIN_COMBINED\', \'None\', \'HALF_AWAY_FROM_ZERO\'], "
   }
   member_method {
     name: "quantized_concat"
-- 
GitLab


From 9e3c3f186812c780edb4eee5b1e154b7e2f6bf23 Mon Sep 17 00:00:00 2001
From: Sergio Guadarrama <sguada@google.com>
Date: Wed, 25 Oct 2017 14:21:52 -0700
Subject: [PATCH 778/909] Expose the name of the name_scope as a property.
 Create tf.get_name_scope() helper function.

PiperOrigin-RevId: 173449188
---
 tensorflow/python/framework/ops.py            | 22 +++++++++++++++++++
 .../tensorflow.keras.backend.name_scope.pbtxt |  4 ++++
 .../api/golden/tensorflow.name_scope.pbtxt    |  4 ++++
 3 files changed, 30 insertions(+)

diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index 94c29c89df..61a5a4fcae 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -4694,6 +4694,24 @@ def get_default_graph():
   return _default_graph_stack.get_default()
 
 
+def get_name_scope():
+  """Returns the current name scope in the default_graph.
+
+  For example:
+
+  ```python
+  with tf.name_scope('scope1'):
+    with tf.name_scope('scope2'):
+      print(tf.get_name_scope())
+  ```
+  would print the string `scope1/scope2`.
+
+  Returns:
+    A string representing the current name scope.
+  """
+  return get_default_graph().get_name_scope()
+
+
 def _assert_same_graph(original_item, item):
   """Fail if the 2 items are from different graphs.
 
@@ -5019,6 +5037,10 @@ class name_scope(object):  # pylint: disable=invalid-name
   ```
   """
 
+  @property
+  def name(self):
+    return self._name
+
   def __init__(self, name, default_name=None, values=None):
     """Initialize the context manager.
 
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.backend.name_scope.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.backend.name_scope.pbtxt
index 43692a6c73..a2b98b1c27 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.backend.name_scope.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.backend.name_scope.pbtxt
@@ -2,6 +2,10 @@ path: "tensorflow.keras.backend.name_scope"
 tf_class {
   is_instance: "<class \'tensorflow.python.framework.ops.name_scope\'>"
   is_instance: "<type \'object\'>"
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
   member_method {
     name: "__init__"
     argspec: "args=[\'self\', \'name\', \'default_name\', \'values\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
diff --git a/tensorflow/tools/api/golden/tensorflow.name_scope.pbtxt b/tensorflow/tools/api/golden/tensorflow.name_scope.pbtxt
index 107f066c29..8041897013 100644
--- a/tensorflow/tools/api/golden/tensorflow.name_scope.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.name_scope.pbtxt
@@ -2,6 +2,10 @@ path: "tensorflow.name_scope"
 tf_class {
   is_instance: "<class \'tensorflow.python.framework.ops.name_scope\'>"
   is_instance: "<type \'object\'>"
+  member {
+    name: "name"
+    mtype: "<type \'property\'>"
+  }
   member_method {
     name: "__init__"
     argspec: "args=[\'self\', \'name\', \'default_name\', \'values\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
-- 
GitLab


From 83b39b0753026c5617d286929129e3529a41ce9e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 25 Oct 2017 14:36:18 -0700
Subject: [PATCH 779/909] Update ops-related pbtxt files.

PiperOrigin-RevId: 173451205
---
 .../core/ops/compat/ops_history.v1.pbtxt      | 67 +++++++++++++++++++
 tensorflow/core/ops/ops.pbtxt                 | 15 ++++-
 2 files changed, 81 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index e6aeb35e02..cec75f6799 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -22194,6 +22194,73 @@ op {
     }
   }
 }
+op {
+  name: "QuantizeV2"
+  input_arg {
+    name: "input"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "min_range"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "max_range"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "output_min"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "output_max"
+    type: DT_FLOAT
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT16
+        type: DT_QUINT16
+        type: DT_QINT32
+      }
+    }
+  }
+  attr {
+    name: "mode"
+    type: "string"
+    default_value {
+      s: "MIN_COMBINED"
+    }
+    allowed_values {
+      list {
+        s: "MIN_COMBINED"
+        s: "MIN_FIRST"
+        s: "SCALED"
+      }
+    }
+  }
+  attr {
+    name: "round_mode"
+    type: "string"
+    default_value {
+      s: "HALF_AWAY_FROM_ZERO"
+    }
+    allowed_values {
+      list {
+        s: "HALF_AWAY_FROM_ZERO"
+        s: "HALF_TO_EVEN"
+      }
+    }
+  }
+}
 op {
   name: "QuantizedAdd"
   input_arg {
diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index a6886e465d..78f0fda408 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -17435,8 +17435,21 @@ op {
       }
     }
   }
+  attr {
+    name: "round_mode"
+    type: "string"
+    default_value {
+      s: "HALF_AWAY_FROM_ZERO"
+    }
+    allowed_values {
+      list {
+        s: "HALF_AWAY_FROM_ZERO"
+        s: "HALF_TO_EVEN"
+      }
+    }
+  }
   summary: "Quantize the \'input\' tensor of type float to \'output\' tensor of type \'T\'."
-  description: "[min_range, max_range] are scalar floats that specify the range for\nthe \'input\' data. The \'mode\' attribute controls exactly which calculations are\nused to convert the float values to their quantized equivalents.\n\nIn \'MIN_COMBINED\' mode, each value of the tensor will undergo the following:\n\n```\nout[i] = (in[i] - min_range) * range(T) / (max_range - min_range)\nif T == qint8, out[i] -= (range(T) + 1) / 2.0\n```\nhere `range(T) = numeric_limits<T>::max() - numeric_limits<T>::min()`\n\n*MIN_COMBINED Mode Example*\n\nAssume the input is type float and has a possible range of [0.0, 6.0] and the\noutput type is quint8 ([0, 255]). The min_range and max_range values should be\nspecified as 0.0 and 6.0. Quantizing from float to quint8 will multiply each\nvalue of the input by 255/6 and cast to quint8.\n\nIf the output type was qint8 ([-128, 127]), the operation will additionally\nsubtract each value by 128 prior to casting, so that the range of values aligns\nwith the range of qint8.\n\nIf the mode is \'MIN_FIRST\', then this approach is used:\n\n```\nnumber_of_steps = 1 << (# of bits in T)\nrange_adjust = number_of_steps / (number_of_steps - 1)\nrange = (range_max - range_min) * range_adjust\nrange_scale = number_of_steps / range\nquantized = round(input * range_scale) - round(range_min * range_scale) +\n  numeric_limits<T>::min()\nquantized = max(quantized, numeric_limits<T>::min())\nquantized = min(quantized, numeric_limits<T>::max())\n```\n\nThe biggest difference between this and MIN_COMBINED is that the minimum range\nis rounded first, before it\'s subtracted from the rounded value. With\nMIN_COMBINED, a small bias is introduced where repeated iterations of quantizing\nand dequantizing will introduce a larger and larger error.\n\n*SCALED mode Example*\n\n`SCALED` mode matches the quantization approach used in\n`QuantizeAndDequantize{V2|V3}`.\n\nIf the mode is `SCALED`, we do not use the full range of the output type,\nchoosing to elide the lowest possible value for symmetry (e.g., output range is\n-127 to 127, not -128 to 127 for signed 8 bit quantization), so that 0.0 maps to\n0.\n\nWe first find the range of values in our tensor. The\nrange we use is always centered on 0, so we find m such that\n```c++\n  m = max(abs(input_min), abs(input_max))\n```\n\nOur input tensor range is then `[-m, m]`.\n\nNext, we choose our fixed-point quantization buckets, `[min_fixed, max_fixed]`.\nIf T is signed, this is\n```\n  num_bits = sizeof(T) * 8\n  [min_fixed, max_fixed] =\n      [-(1 << (num_bits - 1) - 1), (1 << (num_bits - 1)) - 1]\n```\n\nOtherwise, if T is unsigned, the fixed-point range is\n```\n  [min_fixed, max_fixed] = [0, (1 << num_bits) - 1]\n```\n\nFrom this we compute our scaling factor, s:\n```c++\n  s = (max_fixed - min_fixed) / (2 * m)\n```\n\nNow we can quantize the elements of our tensor:\n```c++\nresult = (input * s).round_to_nearest()\n```\n\nOne thing to watch out for is that the operator may choose to adjust the\nrequested minimum and maximum values slightly during the quantization process,\nso you should always use the output ports as the range for further calculations.\nFor example, if the requested minimum and maximum values are close to equal,\nthey will be separated by a small epsilon value to prevent ill-formed quantized\nbuffers from being created. Otherwise, you can end up with buffers where all the\nquantized values map to the same float value, which causes problems for\noperations that have to perform further calculations on them."
+  description: "[min_range, max_range] are scalar floats that specify the range for\nthe \'input\' data. The \'mode\' attribute controls exactly which calculations are\nused to convert the float values to their quantized equivalents.  The\n\'round_mode\' attribute controls which rounding tie-breaking algorithm is used\nwhen rounding float values to their quantized equivalents.\n\nIn \'MIN_COMBINED\' mode, each value of the tensor will undergo the following:\n\n```\nout[i] = (in[i] - min_range) * range(T) / (max_range - min_range)\nif T == qint8, out[i] -= (range(T) + 1) / 2.0\n```\nhere `range(T) = numeric_limits<T>::max() - numeric_limits<T>::min()`\n\n*MIN_COMBINED Mode Example*\n\nAssume the input is type float and has a possible range of [0.0, 6.0] and the\noutput type is quint8 ([0, 255]). The min_range and max_range values should be\nspecified as 0.0 and 6.0. Quantizing from float to quint8 will multiply each\nvalue of the input by 255/6 and cast to quint8.\n\nIf the output type was qint8 ([-128, 127]), the operation will additionally\nsubtract each value by 128 prior to casting, so that the range of values aligns\nwith the range of qint8.\n\nIf the mode is \'MIN_FIRST\', then this approach is used:\n\n```\nnumber_of_steps = 1 << (# of bits in T)\nrange_adjust = number_of_steps / (number_of_steps - 1)\nrange = (range_max - range_min) * range_adjust\nrange_scale = number_of_steps / range\nquantized = round(input * range_scale) - round(range_min * range_scale) +\n  numeric_limits<T>::min()\nquantized = max(quantized, numeric_limits<T>::min())\nquantized = min(quantized, numeric_limits<T>::max())\n```\n\nThe biggest difference between this and MIN_COMBINED is that the minimum range\nis rounded first, before it\'s subtracted from the rounded value. With\nMIN_COMBINED, a small bias is introduced where repeated iterations of quantizing\nand dequantizing will introduce a larger and larger error.\n\n*SCALED mode Example*\n\n`SCALED` mode matches the quantization approach used in\n`QuantizeAndDequantize{V2|V3}`.\n\nIf the mode is `SCALED`, we do not use the full range of the output type,\nchoosing to elide the lowest possible value for symmetry (e.g., output range is\n-127 to 127, not -128 to 127 for signed 8 bit quantization), so that 0.0 maps to\n0.\n\nWe first find the range of values in our tensor. The\nrange we use is always centered on 0, so we find m such that\n```c++\n  m = max(abs(input_min), abs(input_max))\n```\n\nOur input tensor range is then `[-m, m]`.\n\nNext, we choose our fixed-point quantization buckets, `[min_fixed, max_fixed]`.\nIf T is signed, this is\n```\n  num_bits = sizeof(T) * 8\n  [min_fixed, max_fixed] =\n      [-(1 << (num_bits - 1) - 1), (1 << (num_bits - 1)) - 1]\n```\n\nOtherwise, if T is unsigned, the fixed-point range is\n```\n  [min_fixed, max_fixed] = [0, (1 << num_bits) - 1]\n```\n\nFrom this we compute our scaling factor, s:\n```c++\n  s = (max_fixed - min_fixed) / (2 * m)\n```\n\nNow we can quantize the elements of our tensor:\n```c++\nresult = round(input * s)\n```\n\nOne thing to watch out for is that the operator may choose to adjust the\nrequested minimum and maximum values slightly during the quantization process,\nso you should always use the output ports as the range for further calculations.\nFor example, if the requested minimum and maximum values are close to equal,\nthey will be separated by a small epsilon value to prevent ill-formed quantized\nbuffers from being created. Otherwise, you can end up with buffers where all the\nquantized values map to the same float value, which causes problems for\noperations that have to perform further calculations on them."
 }
 op {
   name: "QuantizedAdd"
-- 
GitLab


From 22298967f352c5d54856b22b69b2a6486b01f23c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 25 Oct 2017 14:41:23 -0700
Subject: [PATCH 780/909] Adding learning rate decays found in Neural Optimizer
 Search with Reinforcement Learning [Bello et al, ICML2017] Also adding cosine
 decay.

PiperOrigin-RevId: 173451903
---
 .../python/training/learning_rate_decay.py    | 227 +++++++++++++++++-
 .../training/learning_rate_decay_test.py      |  93 +++++++
 tensorflow/python/training/training.py        |   3 +
 .../tools/api/golden/tensorflow.train.pbtxt   |  12 +
 4 files changed, 334 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/training/learning_rate_decay.py b/tensorflow/python/training/learning_rate_decay.py
index e4a7964aaf..bb7762c8c5 100644
--- a/tensorflow/python/training/learning_rate_decay.py
+++ b/tensorflow/python/training/learning_rate_decay.py
@@ -18,12 +18,14 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import math
+
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import math_ops
-
+from tensorflow.python.ops import random_ops
 
 def exponential_decay(learning_rate, global_step, decay_steps, decay_rate,
                       staircase=False, name=None):
@@ -412,3 +414,226 @@ def inverse_time_decay(learning_rate, global_step, decay_steps, decay_rate,
     const = math_ops.cast(constant_op.constant(1), learning_rate.dtype)
     denom = math_ops.add(const, math_ops.multiply(decay_rate, p))
     return math_ops.div(learning_rate, denom, name=name)
+
+
+def cosine_decay(learning_rate, global_step, decay_steps, name=None):
+  """Applies cosine decay to the learning rate.
+
+  See [Loshchilov & Hutter, ICLR2016], SGDR: Stochastic Gradient Descent
+  with Warm Restarts.
+
+  When training a model, it is often recommended to lower the learning rate as
+  the training progresses.  This function applies a cosine decay function
+  to a provided initial learning rate.  It requires a `global_step` value to
+  compute the decayed learning rate.  You can just pass a TensorFlow variable
+  that you increment at each training step.
+
+  The function returns the decayed learning rate.  It is computed as:
+  ```python
+  global_step = min(global_step, decay_steps)
+  decayed = 0.5 * (1 + cos(pi * global_step / decay_steps))
+  decayed_learning_rate = learning_rate * decayed
+  ```
+
+  Example usage:
+  ```python
+  decay_steps = 1000
+  lr_decayed = cosine_decay(learning_rate, global_step, decay_steps)
+  ```
+
+  Args:
+    learning_rate: A scalar `float32` or `float64` Tensor or a Python number.
+      The initial learning rate.
+    global_step: A scalar `int32` or `int64` `Tensor` or a Python number.
+      Global step to use for the decay computation.
+    decay_steps: A scalar `int32` or `int64` `Tensor` or a Python number.
+      Number of steps to decay over.
+    name: String. Optional name of the operation.  Defaults to 'CosineDecay'.
+  Returns:
+    A scalar `Tensor` of the same type as `learning_rate`.  The decayed
+    learning rate.
+  Raises:
+    ValueError: if `global_step` is not supplied.
+  """
+  if global_step is None:
+    raise ValueError("cosine decay requires global_step")
+  with ops.name_scope(name, "CosineDecay",
+                      [learning_rate, global_step]) as name:
+    learning_rate = ops.convert_to_tensor(learning_rate, name="learning_rate")
+    dtype = learning_rate.dtype
+    global_step = math_ops.cast(global_step, dtype)
+    decay_steps = math_ops.cast(decay_steps, dtype)
+    global_step = math_ops.minimum(global_step, decay_steps)
+    completed_fraction = global_step / decay_steps
+    cosine_decayed = 0.5 * (
+        1.0 + math_ops.cos(constant_op.constant(math.pi) * completed_fraction))
+
+    return math_ops.multiply(learning_rate, cosine_decayed)
+
+
+def linear_cosine_decay(learning_rate, global_step, decay_steps,
+                        num_periods=0.5, alpha=0.0, beta=0.001,
+                        name=None):
+  """Applies linear cosine decay to the learning rate.
+
+  See [Bello et al., ICML2017] Neural Optimizer Search with RL.
+  https://arxiv.org/abs/1709.07417
+
+  Note that linear cosine decay is more aggressive than cosine decay and
+  larger initial learning rates can typically be used.
+
+  When training a model, it is often recommended to lower the learning rate as
+  the training progresses.  This function applies a linear cosine decay function
+  to a provided initial learning rate.  It requires a `global_step` value to
+  compute the decayed learning rate.  You can just pass a TensorFlow variable
+  that you increment at each training step.
+
+  The function returns the decayed learning rate.  It is computed as:
+  ```python
+  global_step = min(global_step, decay_steps)
+  linear_decay = (decay_steps - global_step) / decay_steps)
+  cosine_decay = 0.5 * (
+      1 + cos(pi * 2 * num_periods * global_step / decay_steps))
+  decayed = (alpha + linear_decay) * cosine_decay + beta
+  decayed_learning_rate = learning_rate * decayed
+  ```
+
+  Example usage:
+  ```python
+  decay_steps = 1000
+  lr_decayed = linear_cosine_decay(learning_rate, global_step, decay_steps)
+  ```
+
+  Args:
+    learning_rate: A scalar `float32` or `float64` Tensor or a Python number.
+      The initial learning rate.
+    global_step: A scalar `int32` or `int64` `Tensor` or a Python number.
+      Global step to use for the decay computation.
+    decay_steps: A scalar `int32` or `int64` `Tensor` or a Python number.
+      Number of steps to decay over.
+    num_periods: Number of periods in the cosine part of the decay.
+      See computation above.
+    alpha: See computation above.
+    beta: See computation above.
+    name: String.  Optional name of the operation.  Defaults to
+      'LinearCosineDecay'.
+  Returns:
+    A scalar `Tensor` of the same type as `learning_rate`.  The decayed
+    learning rate.
+  Raises:
+    ValueError: if `global_step` is not supplied.
+  """
+  if global_step is None:
+    raise ValueError("linear cosine decay requires global_step")
+  with ops.name_scope(name, "LinearCosineDecay",
+                      [learning_rate, global_step]) as name:
+    learning_rate = ops.convert_to_tensor(learning_rate, name="learning_rate")
+    dtype = learning_rate.dtype
+    global_step = math_ops.cast(global_step, dtype)
+    decay_steps = math_ops.cast(decay_steps, dtype)
+    num_periods = math_ops.cast(num_periods, dtype)
+    global_step = math_ops.minimum(global_step, decay_steps)
+    alpha = math_ops.cast(alpha, dtype)
+    beta = math_ops.cast(beta, dtype)
+
+    linear_decayed = (decay_steps - global_step) / decay_steps
+    completed_fraction = global_step / decay_steps
+    fraction = 2.0 * num_periods * completed_fraction
+    cosine_decayed = 0.5 * (
+        1.0 + math_ops.cos(constant_op.constant(math.pi) * fraction))
+
+    linear_cosine_decayed = (alpha + linear_decayed) * cosine_decayed + beta
+    return math_ops.multiply(learning_rate, linear_cosine_decayed, name=name)
+
+
+def noisy_linear_cosine_decay(learning_rate, global_step, decay_steps,
+                              initial_variance=1.0, variance_decay=0.55,
+                              num_periods=0.5, alpha=0.0, beta=0.001,
+                              name=None):
+  """Applies noisy linear cosine decay to the learning rate.
+
+  See [Bello et al., ICML2017] Neural Optimizer Search with RL.
+  https://arxiv.org/abs/1709.07417
+
+  Note that linear cosine decay is more aggressive than cosine decay and
+  larger initial learning rates can typically be used.
+
+  When training a model, it is often recommended to lower the learning rate as
+  the training progresses.  This function applies a noisy linear
+  cosine decay function to a provided initial learning rate.
+  It requires a `global_step` value to compute the decayed learning rate.
+  You can just pass a TensorFlow variable that you increment at each
+  training step.
+
+  The function returns the decayed learning rate.  It is computed as:
+  ```python
+  global_step = min(global_step, decay_steps)
+  linear_decay = (decay_steps - global_step) / decay_steps)
+  cosine_decay = 0.5 * (
+      1 + cos(pi * 2 * num_periods * global_step / decay_steps))
+  decayed = (alpha + linear_decay + eps_t) * cosine_decay + beta
+  decayed_learning_rate = learning_rate * decayed
+  ```
+  where eps_t is 0-centered gaussian noise with variance
+  initial_variance / (1 + global_step) ** variance_decay
+
+  Example usage:
+  ```python
+  decay_steps = 1000
+  lr_decayed = noisy_linear_cosine_decay(
+    learning_rate, global_step, decay_steps)
+  ```
+
+  Args:
+    learning_rate: A scalar `float32` or `float64` Tensor or a Python number.
+      The initial learning rate.
+    global_step: A scalar `int32` or `int64` `Tensor` or a Python number.
+      Global step to use for the decay computation.
+    decay_steps: A scalar `int32` or `int64` `Tensor` or a Python number.
+      Number of steps to decay over.
+    initial_variance: initial variance for the noise. See computation above.
+    variance_decay: decay for the noise's variance. See computation above.
+    num_periods: Number of periods in the cosine part of the decay.
+      See computation above.
+    alpha: See computation above.
+    beta: See computation above.
+    name: String.  Optional name of the operation.  Defaults to
+      'NoisyLinearCosineDecay'.
+  Returns:
+    A scalar `Tensor` of the same type as `learning_rate`.  The decayed
+    learning rate.
+  Raises:
+    ValueError: if `global_step` is not supplied.
+  """
+  if global_step is None:
+    raise ValueError("noisy linear cosine decay requires global_step")
+  with ops.name_scope(name, "NoisyLinearCosineDecay",
+                      [learning_rate, global_step]) as name:
+    learning_rate = ops.convert_to_tensor(learning_rate, name="learning_rate")
+    dtype = learning_rate.dtype
+    global_step = math_ops.cast(global_step, dtype)
+    decay_steps = math_ops.cast(decay_steps, dtype)
+    global_step = math_ops.minimum(global_step, decay_steps)
+    initial_variance = math_ops.cast(initial_variance, dtype)
+    variance_decay = math_ops.cast(variance_decay, dtype)
+    num_periods = math_ops.cast(num_periods, dtype)
+    alpha = math_ops.cast(alpha, dtype)
+    beta = math_ops.cast(beta, dtype)
+
+    linear_decayed = (decay_steps - global_step) / decay_steps
+    variance = initial_variance / (
+        math_ops.pow(1.0 + global_step, variance_decay))
+    std = math_ops.sqrt(variance)
+    noisy_linear_decayed = (
+        linear_decayed + random_ops.random_normal(
+            linear_decayed.shape, stddev=std))
+
+    completed_fraction = global_step / decay_steps
+    fraction = 2.0 * num_periods * completed_fraction
+    cosine_decayed = 0.5 * (
+        1.0 + math_ops.cos(constant_op.constant(math.pi) * fraction))
+    noisy_linear_cosine_decayed = (
+        (alpha + noisy_linear_decayed) * cosine_decayed + beta)
+
+    return math_ops.multiply(
+        learning_rate, noisy_linear_cosine_decayed, name=name)
diff --git a/tensorflow/python/training/learning_rate_decay_test.py b/tensorflow/python/training/learning_rate_decay_test.py
index 77da3099fe..34c300eae7 100644
--- a/tensorflow/python/training/learning_rate_decay_test.py
+++ b/tensorflow/python/training/learning_rate_decay_test.py
@@ -340,5 +340,98 @@ class InverseDecayTest(test_util.TensorFlowTestCase):
         increment_step.op.run()
 
 
+class CosineDecayTest(test_util.TensorFlowTestCase):
+
+  def np_cosine_decay(self, step, decay_steps):
+    step = min(step, decay_steps)
+    completed_fraction = step / decay_steps
+    return 0.5 * (1.0 + math.cos(math.pi * completed_fraction))
+
+  def testDecay(self):
+    num_training_steps = 1000
+    initial_lr = 1.0
+    for step in range(0, 1500, 250):
+      with self.test_session():
+        decayed_lr = learning_rate_decay.cosine_decay(
+            initial_lr, step, num_training_steps)
+        expected = self.np_cosine_decay(step, num_training_steps)
+        self.assertAllClose(decayed_lr.eval(), expected, 1e-6)
+
+
+class LinearCosineDecayTest(test_util.TensorFlowTestCase):
+
+  def np_linear_cosine_decay(self,
+                             step,
+                             decay_steps,
+                             alpha=0.0,
+                             beta=0.001,
+                             num_periods=0.5):
+    step = min(step, decay_steps)
+    linear_decayed = float(decay_steps - step) / decay_steps
+    fraction = 2.0 * num_periods * step / float(decay_steps)
+    cosine_decayed = 0.5 * (1.0 + math.cos(math.pi * fraction))
+    return (alpha + linear_decayed) * cosine_decayed + beta
+
+  def testDefaultDecay(self):
+    num_training_steps = 1000
+    initial_lr = 1.0
+    for step in range(0, 1500, 250):
+      with self.test_session():
+        decayed_lr = learning_rate_decay.linear_cosine_decay(
+            initial_lr, step, num_training_steps)
+        expected = self.np_linear_cosine_decay(step, num_training_steps)
+        self.assertAllClose(decayed_lr.eval(), expected, 1e-6)
+
+  def testNonDefaultDecay(self):
+    num_training_steps = 1000
+    initial_lr = 1.0
+    for step in range(0, 1500, 250):
+      with self.test_session():
+        decayed_lr = learning_rate_decay.linear_cosine_decay(
+            initial_lr,
+            step,
+            num_training_steps,
+            alpha=0.1,
+            beta=1e-4,
+            num_periods=5)
+        expected = self.np_linear_cosine_decay(
+            step,
+            num_training_steps,
+            alpha=0.1,
+            beta=1e-4,
+            num_periods=5)
+        self.assertAllClose(decayed_lr.eval(), expected, 1e-6)
+
+
+class NoisyLinearCosineDecayTest(test_util.TensorFlowTestCase):
+
+  def testDefaultNoisyLinearCosine(self):
+    num_training_steps = 1000
+    initial_lr = 1.0
+    for step in range(0, 1500, 250):
+      with self.test_session():
+        # No numerical check because of noise
+        decayed_lr = learning_rate_decay.noisy_linear_cosine_decay(
+            initial_lr, step, num_training_steps)
+        decayed_lr.eval()
+
+  def testNonDefaultNoisyLinearCosine(self):
+    num_training_steps = 1000
+    initial_lr = 1.0
+    for step in range(0, 1500, 250):
+      with self.test_session():
+        # No numerical check because of noise
+        decayed_lr = learning_rate_decay.noisy_linear_cosine_decay(
+            initial_lr,
+            step,
+            num_training_steps,
+            initial_variance=0.5,
+            variance_decay=0.1,
+            alpha=0.1,
+            beta=1e-4,
+            num_periods=5)
+        decayed_lr.eval()
+
+
 if __name__ == "__main__":
   googletest.main()
diff --git a/tensorflow/python/training/training.py b/tensorflow/python/training/training.py
index 741dddc991..fa02ad84cc 100644
--- a/tensorflow/python/training/training.py
+++ b/tensorflow/python/training/training.py
@@ -37,6 +37,9 @@ See the @{$python/train} guide.
 @@clip_by_average_norm
 @@clip_by_global_norm
 @@global_norm
+@@cosine_decay
+@@linear_cosine_decay
+@@noisy_linear_cosine_decay
 @@exponential_decay
 @@inverse_time_decay
 @@natural_exp_decay
diff --git a/tensorflow/tools/api/golden/tensorflow.train.pbtxt b/tensorflow/tools/api/golden/tensorflow.train.pbtxt
index edc29e62dd..e73f6f6e63 100644
--- a/tensorflow/tools/api/golden/tensorflow.train.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.train.pbtxt
@@ -264,6 +264,10 @@ tf_module {
     name: "checkpoint_exists"
     argspec: "args=[\'checkpoint_prefix\'], varargs=None, keywords=None, defaults=None"
   }
+  member_method {
+    name: "cosine_decay"
+    argspec: "args=[\'learning_rate\', \'global_step\', \'decay_steps\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
   member_method {
     name: "create_global_step"
     argspec: "args=[\'graph\'], varargs=None, keywords=None, defaults=[\'None\'], "
@@ -328,6 +332,10 @@ tf_module {
     name: "limit_epochs"
     argspec: "args=[\'tensor\', \'num_epochs\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
   }
+  member_method {
+    name: "linear_cosine_decay"
+    argspec: "args=[\'learning_rate\', \'global_step\', \'decay_steps\', \'num_periods\', \'alpha\', \'beta\', \'name\'], varargs=None, keywords=None, defaults=[\'0.5\', \'0.0\', \'0.001\', \'None\'], "
+  }
   member_method {
     name: "list_variables"
     argspec: "args=[\'ckpt_dir_or_file\'], varargs=None, keywords=None, defaults=None"
@@ -364,6 +372,10 @@ tf_module {
     name: "natural_exp_decay"
     argspec: "args=[\'learning_rate\', \'global_step\', \'decay_steps\', \'decay_rate\', \'staircase\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'None\'], "
   }
+  member_method {
+    name: "noisy_linear_cosine_decay"
+    argspec: "args=[\'learning_rate\', \'global_step\', \'decay_steps\', \'initial_variance\', \'variance_decay\', \'num_periods\', \'alpha\', \'beta\', \'name\'], varargs=None, keywords=None, defaults=[\'1.0\', \'0.55\', \'0.5\', \'0.0\', \'0.001\', \'None\'], "
+  }
   member_method {
     name: "piecewise_constant"
     argspec: "args=[\'x\', \'boundaries\', \'values\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
-- 
GitLab


From 77d8f9ec51fa6daded8193841373d360fd978f1a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 25 Oct 2017 14:44:00 -0700
Subject: [PATCH 781/909] Go: Update generated wrapper functions for TensorFlow
 ops.

PiperOrigin-RevId: 173452304
---
 tensorflow/go/op/wrappers.go | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go
index 958ce6d040..cc8165e2c7 100644
--- a/tensorflow/go/op/wrappers.go
+++ b/tensorflow/go/op/wrappers.go
@@ -13789,11 +13789,21 @@ func QuantizeV2Mode(value string) QuantizeV2Attr {
 	}
 }
 
+// QuantizeV2RoundMode sets the optional round_mode attribute to value.
+// If not specified, defaults to "HALF_AWAY_FROM_ZERO"
+func QuantizeV2RoundMode(value string) QuantizeV2Attr {
+	return func(m optionalAttr) {
+		m["round_mode"] = value
+	}
+}
+
 // Quantize the 'input' tensor of type float to 'output' tensor of type 'T'.
 //
 // [min_range, max_range] are scalar floats that specify the range for
 // the 'input' data. The 'mode' attribute controls exactly which calculations are
-// used to convert the float values to their quantized equivalents.
+// used to convert the float values to their quantized equivalents.  The
+// 'round_mode' attribute controls which rounding tie-breaking algorithm is used
+// when rounding float values to their quantized equivalents.
 //
 // In 'MIN_COMBINED' mode, each value of the tensor will undergo the following:
 //
@@ -13870,7 +13880,7 @@ func QuantizeV2Mode(value string) QuantizeV2Attr {
 //
 // Now we can quantize the elements of our tensor:
 // ```c++
-// result = (input * s).round_to_nearest()
+// result = round(input * s)
 // ```
 //
 // One thing to watch out for is that the operator may choose to adjust the
-- 
GitLab


From 71beb2dbf8eb41ac03ea4cd52a771ad31b125e7e Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Wed, 25 Oct 2017 15:00:21 -0700
Subject: [PATCH 782/909] Remove unnecessary -ldl

We no longer use dlopen() in :simple_orc_jit (after cl/173277862), so the -ldl
flag is no longer needed.

PiperOrigin-RevId: 173454761
---
 tensorflow/compiler/xla/service/cpu/BUILD | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tensorflow/compiler/xla/service/cpu/BUILD b/tensorflow/compiler/xla/service/cpu/BUILD
index 56bc1a6706..ef8eed3f88 100644
--- a/tensorflow/compiler/xla/service/cpu/BUILD
+++ b/tensorflow/compiler/xla/service/cpu/BUILD
@@ -146,7 +146,6 @@ cc_library(
     name = "simple_orc_jit",
     srcs = ["simple_orc_jit.cc"],
     hdrs = ["simple_orc_jit.h"],
-    linkopts = ["-ldl"],
     deps = [
         ":compiler_functor",
         ":cpu_runtime",
-- 
GitLab


From 5fe90b57748714341d02b2b44a7ec8ff27123bc0 Mon Sep 17 00:00:00 2001
From: Yangzihao Wang <yangzihao@google.com>
Date: Wed, 25 Oct 2017 15:10:29 -0700
Subject: [PATCH 783/909] Force the CUDA runtime initialization before device
 creation. This is to avoid silent failure and garbage results produced when
 launching two TensorFlow programs simultaneously in two different processes.

PiperOrigin-RevId: 173456597
---
 .../core/common_runtime/gpu/gpu_device.cc     | 28 +++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/tensorflow/core/common_runtime/gpu/gpu_device.cc b/tensorflow/core/common_runtime/gpu/gpu_device.cc
index 12d44cc6b7..2c906ed220 100644
--- a/tensorflow/core/common_runtime/gpu/gpu_device.cc
+++ b/tensorflow/core/common_runtime/gpu/gpu_device.cc
@@ -652,6 +652,34 @@ Status BaseGPUDeviceFactory::CreateDevices(const SessionOptions& options,
   if (static_cast<size_t>(n) > valid_gpu_ids.size()) {
     n = valid_gpu_ids.size();
   }
+  // Save the original device.
+  int original_device = 0;
+  cudaError_t err = cudaGetDevice(&original_device);
+  if (err != cudaSuccess) {
+    return errors::Internal("cudaGetDevice() failed. Status: ",
+                            cudaGetErrorString(err));
+  }
+  // Force to implicitly initialize CUDA runtime on each valid GPU before
+  // CreateGPUDevice().
+  for (int gpu_id : valid_gpu_ids) {
+    err = cudaSetDevice(gpu_id);
+    if (err != cudaSuccess) {
+      return errors::Internal("cudaSetDevice() on GPU:", gpu_id,
+                              " failed. Status: ", cudaGetErrorString(err));
+    }
+    err = cudaFree(nullptr);
+    if (err != cudaSuccess) {
+      return errors::Internal(
+          "CUDA runtime implicit initialization on GPU:", gpu_id,
+          " failed. Status: ", cudaGetErrorString(err));
+    }
+  }
+  // Reset to the original device.
+  err = cudaSetDevice(original_device);
+  if (err != cudaSuccess) {
+    return errors::Internal("cudaSetDevice() on GPU:", original_device,
+                            " failed. Status: ", cudaGetErrorString(err));
+  }
   for (int i = 0; i < n; i++) {
     BaseGPUDevice* gpu_device;
     TF_RETURN_IF_ERROR(CreateGPUDevice(
-- 
GitLab


From df299e1a0c91f50acf4868c7bb3e0ea93b52db7b Mon Sep 17 00:00:00 2001
From: Suharsh Sivakumar <suharshs@google.com>
Date: Wed, 25 Oct 2017 15:10:45 -0700
Subject: [PATCH 784/909] Allow device specification for variables added by
 quantize graph rewriter.

PiperOrigin-RevId: 173456646
---
 tensorflow/contrib/quantize/BUILD             |   4 +
 .../contrib/quantize/python/quantize_graph.py |  46 +++++---
 .../quantize/python/quantize_graph_test.py    | 102 ++++++++++++++----
 .../contrib/quantize/python/quantize_test.py  |  23 ----
 4 files changed, 119 insertions(+), 56 deletions(-)

diff --git a/tensorflow/contrib/quantize/BUILD b/tensorflow/contrib/quantize/BUILD
index 0d6c71965c..2c0ffaf6c0 100644
--- a/tensorflow/contrib/quantize/BUILD
+++ b/tensorflow/contrib/quantize/BUILD
@@ -219,9 +219,13 @@ py_test(
     srcs_version = "PY2AND3",
     deps = [
         ":quantize_graph",
+        "//tensorflow/contrib/layers:layers_py",
+        "//tensorflow/python:array_ops",
         "//tensorflow/python:constant_op",
         "//tensorflow/python:framework_ops",
         "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:init_ops",
+        "//tensorflow/python:nn_ops",
         "//tensorflow/python:platform_test",
         "//tensorflow/python:variables",
     ],
diff --git a/tensorflow/contrib/quantize/python/quantize_graph.py b/tensorflow/contrib/quantize/python/quantize_graph.py
index aaf3e92b8e..d647bb94e8 100644
--- a/tensorflow/contrib/quantize/python/quantize_graph.py
+++ b/tensorflow/contrib/quantize/python/quantize_graph.py
@@ -25,7 +25,10 @@ from tensorflow.python.framework import ops
 from tensorflow.python.ops import variables
 
 
-def _create_graph(input_graph, is_training, elements=None):
+def _create_graph(input_graph,
+                  is_training,
+                  elements=None,
+                  device_name_or_function=None):
   """Returns a transformed training input_graph for simulated quantization.
 
   The forward pass has fake quantization ops inserted to simulate the error
@@ -36,12 +39,12 @@ def _create_graph(input_graph, is_training, elements=None):
     is_training: Whether quantizing training or eval graph.
     elements: (Optional) List of Tensors and Operations in input_graph whose
         corresponding elements in the new graph will be returned.
+    device_name_or_function: (Optional) The device name or function to use.
 
   Returns:
-    Returns a tuple(g, l) where:
     g is new tf.Graph that is rewritten for simulated quantization.
     l is a list of Tensors/Operations in g corresponding to the provided input
-        elements.
+        elements, if elements is not None.
 
   Raises:
     ValueError: If elements contains an element that isn't a tf.Tensor or
@@ -49,11 +52,14 @@ def _create_graph(input_graph, is_training, elements=None):
   """
   # TODO(suharshs): Describe the process in more detail in the doc string.
   g = copy_graph.CopyGraph(input_graph)
-  fold_batch_norms.FoldBatchNorms(g)
-  quantize.Quantize(g, is_training=is_training)
-  return_elements = []
+  with g.as_default():
+    with ops.device(device_name_or_function):
+      fold_batch_norms.FoldBatchNorms(g)
+      quantize.Quantize(g, is_training=is_training)
   if elements is None:
-    elements = []
+    return g
+
+  return_elements = []
   for element in elements:
     if isinstance(element, (ops.Tensor, variables.Variable)):
       return_elements.append(g.get_tensor_by_name(element.name))
@@ -66,7 +72,9 @@ def _create_graph(input_graph, is_training, elements=None):
   return g, return_elements
 
 
-def create_training_graph(input_graph, elements=None):
+def create_training_graph(input_graph,
+                          elements=None,
+                          device_name_or_function=None):
   """Returns a transformed training input_graph for simulated quantization.
 
   The forward pass has fake quantization ops inserted to simulate the error
@@ -76,21 +84,25 @@ def create_training_graph(input_graph, elements=None):
     input_graph: The tf.Graph to be transformed.
     elements: (Optional) List of Tensors and Operations in input_graph whose
         corresponding elements in the new graph will be returned.
+    device_name_or_function: (Optional) The device name or function to use.
 
   Returns:
-    Returns a tuple(g, l) where:
     g is new tf.Graph that is rewritten for simulated quantization.
     l is a list of Tensors/Operations in g corresponding to the provided input
-        elements.
+        elements, if elements is not None.
 
   Raises:
     ValueError: If elements contains an element that isn't a tf.Tensor or
         tf.Operation.
   """
-  return _create_graph(input_graph, True, elements)
+  return _create_graph(
+      input_graph=input_graph,
+      is_training=True,
+      elements=elements,
+      device_name_or_function=device_name_or_function)
 
 
-def create_eval_graph(input_graph, elements=None):
+def create_eval_graph(input_graph, elements=None, device_name_or_function=None):
   """Returns a transformed eval input_graph for simulated quantization.
 
   The forward pass has fake quantization ops inserted to simulate the error
@@ -100,15 +112,19 @@ def create_eval_graph(input_graph, elements=None):
     input_graph: The tf.Graph to be transformed.
     elements: (Optional) List of Tensors and Operations in input_graph whose
         corresponding elements in the new graph will be returned.
+    device_name_or_function: (Optional) The device name or function to use.
 
   Returns:
-    Returns a tuple(g, l) where:
     g is new tf.Graph that is rewritten for simulated quantization.
     l is a list of Tensors/Operations in g corresponding to the provided input
-        elements.
+        elements, if elements is not None.
 
   Raises:
     ValueError: If elements contains an element that isn't a tf.Tensor or
         tf.Operation.
   """
-  return _create_graph(input_graph, False, elements)
+  return _create_graph(
+      input_graph=input_graph,
+      is_training=False,
+      elements=elements,
+      device_name_or_function=device_name_or_function)
diff --git a/tensorflow/contrib/quantize/python/quantize_graph_test.py b/tensorflow/contrib/quantize/python/quantize_graph_test.py
index 382076672a..3407ace391 100644
--- a/tensorflow/contrib/quantize/python/quantize_graph_test.py
+++ b/tensorflow/contrib/quantize/python/quantize_graph_test.py
@@ -18,29 +18,41 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from tensorflow.contrib.layers.python.layers import layers
 from tensorflow.contrib.quantize.python import quantize_graph
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import init_ops
+from tensorflow.python.ops import nn_ops
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import googletest
 
 
-class QuantizeTest(test_util.TensorFlowTestCase):
+class QuantizeGraphTest(test_util.TensorFlowTestCase):
 
   # We have a lot of other tests that test the details of the rewrite, here we
   # just the specific features of the quantize_graph API.
   def testReturnedElementsTraining(self):
+    self._TestReturnElements(True)
+
+  def testReturnedElementsEval(self):
+    self._TestReturnElements(False)
+
+  def _TestReturnElements(self, is_training):
     graph = ops.Graph()
     with graph.as_default():
       a = constant_op.constant(1.0)
       b = variables.Variable(2.0)
       c = a + b
     elements = [a, b, c.op]
-    for element in elements:
-      print(element)
-    q_graph, returned_elements = quantize_graph.create_training_graph(
-        graph, elements=elements)
+    if is_training:
+      q_graph, returned_elements = quantize_graph.create_training_graph(
+          graph, elements=elements)
+    else:
+      q_graph, returned_elements = quantize_graph.create_eval_graph(
+          graph, elements=elements)
     # Make sure q_graph is different from graph.
     self.assertTrue(graph != q_graph)
     # Check that the returned elements are part of the new graph.
@@ -50,25 +62,79 @@ class QuantizeTest(test_util.TensorFlowTestCase):
     for element, returned_element in zip(elements, returned_elements):
       self.assertEqual(element.name, returned_element.name)
 
-  # We have a lot of other tests that test the details of the rewrite, here we
-  # just the specific features of the quantize_graph API.
-  def testReturnedElementsEval(self):
+  def testNoReturnElementsTraining(self):
+    self._TestNoReturnElements(True)
+
+  def testNoReturnElementsEval(self):
+    self._TestNoReturnElements(False)
+
+  def _TestNoReturnElements(self, is_training):
     graph = ops.Graph()
     with graph.as_default():
       a = constant_op.constant(1.0)
       b = variables.Variable(2.0)
-      c = a + b
-    elements = [a, b, c.op]
-    q_graph, returned_elements = quantize_graph.create_eval_graph(
-        graph, elements=elements)
+      _ = a + b
+    if is_training:
+      q_graph = quantize_graph.create_training_graph(graph)
+    else:
+      q_graph = quantize_graph.create_eval_graph(graph)
+    # Check that quantize_graph didn't return a tuple when elements isn't
+    # provided.
+    self.assertTrue(isinstance(q_graph, ops.Graph))
     # Make sure q_graph is different from graph.
     self.assertTrue(graph != q_graph)
-    # Check that the returned elements are part of the new graph.
-    for returned_element in returned_elements:
-      self.assertEqual(q_graph, returned_element.graph)
-    # Check that the elements match with the one from the input graph.
-    for element, returned_element in zip(elements, returned_elements):
-      self.assertEqual(element.name, returned_element.name)
+
+  def testDeviceNameTraining(self):
+    self._TestDeviceName(True)
+
+  def testDeviceNameEval(self):
+    self._TestDeviceName(False)
+
+  def _TestDeviceName(self, is_training):
+    graph = ops.Graph()
+    with graph.as_default():
+      batch_size, height, width, depth = 5, 128, 128, 3
+      inputs = array_ops.zeros((batch_size, height, width, depth))
+      conv = layers.conv2d(
+          inputs,
+          32, [5, 5],
+          stride=2,
+          padding='SAME',
+          weights_initializer=self._WeightInit(0.09),
+          activation_fn=None,
+          scope='test')
+      _ = nn_ops.relu6(conv)
+
+    device_name = '/job:oink/task:0/device:CPU:0'
+    if is_training:
+      q_graph = quantize_graph.create_training_graph(
+          graph, device_name_or_function=device_name)
+    else:
+      q_graph = quantize_graph.create_eval_graph(
+          graph, device_name_or_function=device_name)
+
+    orig_variable_names = set(
+        [v.name for v in graph.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)])
+    q_variables = q_graph.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)
+    # Ensure that variables were added.
+    self.assertTrue(len(orig_variable_names) < len(q_variables))
+    # All added variables should have the specified device name.
+    for var in q_variables:
+      if var.name not in orig_variable_names:
+        self.assertEqual(var.device, device_name)
+
+  def _WeightInit(self, stddev):
+    """Returns truncated normal variable initializer.
+
+    Function is defined purely to shorten the name so that it stops wrapping.
+
+    Args:
+      stddev: Standard deviation of normal variable.
+
+    Returns:
+      An initialized that initialzes with a truncated normal variable.
+    """
+    return init_ops.truncated_normal_initializer(stddev=stddev)
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/contrib/quantize/python/quantize_test.py b/tensorflow/contrib/quantize/python/quantize_test.py
index a6bd809bb7..4a82eac197 100644
--- a/tensorflow/contrib/quantize/python/quantize_test.py
+++ b/tensorflow/contrib/quantize/python/quantize_test.py
@@ -65,28 +65,5 @@ class QuantizeTest(test_util.TensorFlowTestCase):
     """
     return init_ops.truncated_normal_initializer(stddev=stddev)
 
-  def _AssertInputOpsAre(self, op, in_op_names):
-    """Asserts that all inputs to op come from in_op_names (disregarding order).
-
-    Args:
-      op: Operation to check inputs for.
-      in_op_names: List of strings, operations where all op's inputs should
-        come from.
-    """
-    expected_inputs = [in_op_name + ':0' for in_op_name in in_op_names]
-    self.assertItemsEqual([t.name for t in op.inputs], expected_inputs)
-
-  def _AssertOutputGoesToOps(self, op, graph, out_op_names):
-    """Asserts that outputs from op go to out_op_names (and perhaps others).
-
-    Args:
-      op: Operation to check outputs for.
-      graph: Graph where output operations are located.
-      out_op_names: List of strings, operations where op's outputs should go.
-    """
-    for out_op_name in out_op_names:
-      out_op = graph.get_operation_by_name(out_op_name)
-      self.assertIn(op.outputs[0].name, [str(t.name) for t in out_op.inputs])
-
 if __name__ == '__main__':
   googletest.main()
-- 
GitLab


From 2ece96046d32c205c78393441404604db8706b9b Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Mon, 23 Oct 2017 20:25:44 +0000
Subject: [PATCH 785/909] Add clang style check as part of the sanity check.

This fix is an effort to add clang style check as part of the sanity check.

In `CONTRIBUTING.md` it has been advised to run `clang-format --style=google file.cc`
so that Google coding style is conformed. However, there is no sanity check
in the current Jenkins build so current .cc and .h files in the repo are not really
conforming to the coding style.

This actually causes issues. In case a PR is submitted with `clang-format --style=google file.cc`,
the reviewer may see additional unrelated changes which might be a distraction. The
developer may also spent additional time to manually check for any discrepancies manually with additional
unrelated style changes.

This fix adds the clang-format check to the ci build so that when `ci_sanity.sh` is running,
it will use clang-format to make sure the code is conforming to the coding style as specified
in `CONTRIBUTING.md`.

One thing that might need to take notice is the header order of the Eigen library. See
https://github.com/tensorflow/tensorflow/pull/13907#issuecomment-338718110
for further details.

Basically, if Eigen headers could be placed in any order, then no additional steps are needed.
Otherwise, it is always possible to place the Eigen headers at the top, then leave one empty
line like:
```cpp

```

In this way, even a run of `clang-format -i --style=google file.cc` will still respect
the order and leave Eigen header at the top.

This PR is experimeal so it only checks `tensorflow/core/ops` directory. Other files could be
added if this PR is OK.

This PR also sanitizes all files in `tensorflow/core/ops` directory so that it conforms to
coding style requirement.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/tools/ci_build/ci_sanity.sh        |  6 +++-
 .../tools/ci_build/clang_format_check.sh      | 36 +++++++++++++++++++
 2 files changed, 41 insertions(+), 1 deletion(-)
 create mode 100755 tensorflow/tools/ci_build/clang_format_check.sh

diff --git a/tensorflow/tools/ci_build/ci_sanity.sh b/tensorflow/tools/ci_build/ci_sanity.sh
index 26053de4e9..7cf97dacf0 100755
--- a/tensorflow/tools/ci_build/ci_sanity.sh
+++ b/tensorflow/tools/ci_build/ci_sanity.sh
@@ -426,6 +426,10 @@ do_code_link_check() {
   tensorflow/tools/ci_build/code_link_check.sh
 }
 
+do_clang_format_check() {
+  CLANG_FORMAT=clang-format-3.8 tensorflow/tools/ci_build/clang_format_check.sh
+}
+
 do_check_load_py_test() {
   BUILD_CMD="bazel build ${BAZEL_FLAGS} //tensorflow/tools/pip_package:check_load_py_test"
   ${BUILD_CMD}
@@ -439,7 +443,7 @@ do_check_load_py_test() {
 }
 
 # Supply all sanity step commands and descriptions
-SANITY_STEPS=("do_pylint PYTHON2" "do_pylint PYTHON3" "do_buildifier" "do_bazel_nobuild" "do_pip_package_licenses_check" "do_lib_package_licenses_check" "do_java_package_licenses_check" "do_pip_smoke_test" "do_check_load_py_test" "do_code_link_check")
+SANITY_STEPS=("do_pylint PYTHON2" "do_pylint PYTHON3" "do_buildifier" "do_bazel_nobuild" "do_pip_package_licenses_check" "do_lib_package_licenses_check" "do_java_package_licenses_check" "do_pip_smoke_test" "do_check_load_py_test" "do_code_link_check" "do_clang_format_check")
 SANITY_STEPS_DESC=("Python 2 pylint" "Python 3 pylint" "buildifier check" "bazel nobuild" "pip: license check for external dependencies" "C library: license check for external dependencies" "Java Native Library: license check for external dependencies" "Pip Smoke Test: Checking py_test dependencies exist in pip package" "Check load py_test: Check that BUILD files with py_test target properly load py_test" "Code Link Check: Check there are no broken links")
 
 INCREMENTAL_FLAG=""
diff --git a/tensorflow/tools/ci_build/clang_format_check.sh b/tensorflow/tools/ci_build/clang_format_check.sh
new file mode 100755
index 0000000000..5f252d7f35
--- /dev/null
+++ b/tensorflow/tools/ci_build/clang_format_check.sh
@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+# please run this at root directory of tensorflow
+success=1
+
+CLANG_FORMAT=${CLANG_FORMAT:-clang-format}
+
+# only tensorflow/core/ops is checked at the moment for experimental purpose
+for filename in $(find tensorflow/core/ops -name *.h -o -name *.cc); do
+  $CLANG_FORMAT --style=google $filename | diff $filename - > /dev/null
+  if [ ! $? -eq 0 ]; then
+    success=0
+    echo File $filename is not properly formatted with "clang-format --style=google"
+  fi
+done
+
+if [ $success == 0 ]; then
+  echo Clang format check fails.
+  exit 1
+fi
+
+echo Clang format check success.
-- 
GitLab


From 44ce1cbe1bb153b93904ebb4c1b64d1079309065 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Mon, 23 Oct 2017 20:39:48 +0000
Subject: [PATCH 786/909] Add installation of clang-format-3.8, this will not
 conflict with clang

Add installation of clang-format-3.8, this will not conflict with clang
as command line needs to run with `clang-format-3.8 -i --style=Google`

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/tools/ci_build/install/install_deb_packages.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/tools/ci_build/install/install_deb_packages.sh b/tensorflow/tools/ci_build/install/install_deb_packages.sh
index da1f2199d0..03ca4716b4 100755
--- a/tensorflow/tools/ci_build/install/install_deb_packages.sh
+++ b/tensorflow/tools/ci_build/install/install_deb_packages.sh
@@ -41,6 +41,7 @@ apt-get install -y --no-install-recommends \
     autoconf \
     automake \
     build-essential \
+    clang-format-3.8 \
     curl \
     ffmpeg \
     git \
-- 
GitLab


From 5c9e7dc293cbeda6b085238fabacba02560424b4 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Mon, 23 Oct 2017 20:41:43 +0000
Subject: [PATCH 787/909] Sanitize files in tensorflow/core/ops with
 clang-format -i

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/core/ops/bitwise_ops.cc            |  20 ++-
 .../compat/backwards_compatibility_test.cc    |   5 +-
 tensorflow/core/ops/data_flow_ops.cc          |   3 -
 tensorflow/core/ops/image_ops.cc              |  56 +++---
 tensorflow/core/ops/linalg_ops.cc             |   1 -
 tensorflow/core/ops/math_grad_test.cc         |  17 +-
 tensorflow/core/ops/math_ops.cc               | 168 +++++++++++++-----
 tensorflow/core/ops/nn_ops.cc                 |  12 +-
 tensorflow/core/ops/nn_ops_test.cc            |  12 +-
 tensorflow/core/ops/sparse_ops_test.cc        |   4 +-
 tensorflow/core/ops/stateless_random_ops.cc   |   9 +-
 11 files changed, 196 insertions(+), 111 deletions(-)

diff --git a/tensorflow/core/ops/bitwise_ops.cc b/tensorflow/core/ops/bitwise_ops.cc
index 3156162b78..2889953bdb 100644
--- a/tensorflow/core/ops/bitwise_ops.cc
+++ b/tensorflow/core/ops/bitwise_ops.cc
@@ -56,35 +56,45 @@ representation of that entry.
 8- or 16-bit inputs and then aggregate the resulting counts.
 )doc");
 
-REGISTER_OP("BitwiseAnd").BINARY_BITWISE().Doc(R"doc(
+REGISTER_OP("BitwiseAnd")
+    .BINARY_BITWISE()
+    .Doc(R"doc(
 Elementwise computes the bitwise AND of `x` and `y`.
 
 The result will have those bits set, that are set in both `x` and `y`. The
 computation is performed on the underlying representations of `x` and `y`.
 )doc");
 
-REGISTER_OP("BitwiseOr").BINARY_BITWISE().Doc(R"doc(
+REGISTER_OP("BitwiseOr")
+    .BINARY_BITWISE()
+    .Doc(R"doc(
 Elementwise computes the bitwise OR of `x` and `y`.
 
 The result will have those bits set, that are set in `x`, `y` or both. The
 computation is performed on the underlying representations of `x` and `y`.
 )doc");
 
-REGISTER_OP("BitwiseXor").BINARY_BITWISE().Doc(R"doc(
+REGISTER_OP("BitwiseXor")
+    .BINARY_BITWISE()
+    .Doc(R"doc(
 Elementwise computes the bitwise XOR of `x` and `y`.
 
 The result will have those bits set, that are different in `x` and `y`. The
 computation is performed on the underlying representations of `x` and `y`.
 )doc");
 
-REGISTER_OP("LeftShift").BINARY_BITWISE().Doc(R"doc(
+REGISTER_OP("LeftShift")
+    .BINARY_BITWISE()
+    .Doc(R"doc(
 Elementwise computes the bitwise left-shift of `x` and `y`.
 
 If `y` is negative, or greater than or equal to the width of `x` in bits the
 result is implementation defined.
 )doc");
 
-REGISTER_OP("RightShift").BINARY_BITWISE().Doc(R"doc(
+REGISTER_OP("RightShift")
+    .BINARY_BITWISE()
+    .Doc(R"doc(
 Elementwise computes the bitwise right-shift of `x` and `y`.
 
 Performs a logical shift for unsigned integer types, and an arithmetic shift
diff --git a/tensorflow/core/ops/compat/backwards_compatibility_test.cc b/tensorflow/core/ops/compat/backwards_compatibility_test.cc
index 6e05ae4be4..add05d6610 100644
--- a/tensorflow/core/ops/compat/backwards_compatibility_test.cc
+++ b/tensorflow/core/ops/compat/backwards_compatibility_test.cc
@@ -25,9 +25,8 @@ namespace tensorflow {
 namespace {
 
 TEST(BackwardsCompatibilityTest, IsCompatible) {
-  OpCompatibilityLib compatibility("tensorflow/core/ops",
-                                   strings::StrCat("v", TF_MAJOR_VERSION),
-                                   nullptr);
+  OpCompatibilityLib compatibility(
+      "tensorflow/core/ops", strings::StrCat("v", TF_MAJOR_VERSION), nullptr);
 
   Env* env = Env::Default();
   int changed_ops = 0;
diff --git a/tensorflow/core/ops/data_flow_ops.cc b/tensorflow/core/ops/data_flow_ops.cc
index 8e24ea70cb..3b1ed217ce 100644
--- a/tensorflow/core/ops/data_flow_ops.cc
+++ b/tensorflow/core/ops/data_flow_ops.cc
@@ -2225,7 +2225,6 @@ this op will block until it does.   This Op is optimized for
 performance.
     )doc");
 
-
 REGISTER_OP("StageSize")
     .Output("size: int32")
     .Attr("capacity: int >= 0 = 0")
@@ -2354,7 +2353,6 @@ REGISTER_OP("MapIncompleteSize")
 Op returns the number of incomplete elements in the underlying container.
     )doc");
 
-
 REGISTER_OP("MapClear")
     .Attr("capacity: int >= 0 = 0")
     .Attr("memory_limit: int >= 0 = 0")
@@ -2367,7 +2365,6 @@ REGISTER_OP("MapClear")
 Op removes all elements in the underlying container.
     )doc");
 
-
 // OrderedMap
 REGISTER_OP("OrderedMapStage")
     .Input("key: int64")
diff --git a/tensorflow/core/ops/image_ops.cc b/tensorflow/core/ops/image_ops.cc
index e9bf29d172..c3f8006415 100644
--- a/tensorflow/core/ops/image_ops.cc
+++ b/tensorflow/core/ops/image_ops.cc
@@ -925,27 +925,27 @@ use_image_if_no_bounding_boxes: Controls behavior if no bounding boxes supplied.
 )doc");
 
 REGISTER_OP("SampleDistortedBoundingBoxV2")
-  .Input("image_size: T")
-  .Input("bounding_boxes: float")
-  .Input("min_object_covered: float")
-  .Output("begin: T")
-  .Output("size: T")
-  .Output("bboxes: float")
-  .Attr("T: {uint8, int8, int16, int32, int64}")
-  .Attr("seed: int = 0")
-  .Attr("seed2: int = 0")
-  .Attr("aspect_ratio_range: list(float) = [0.75, 1.33]")
-  .Attr("area_range: list(float) = [0.05, 1.0]")
-  .Attr("max_attempts: int = 100")
-  .Attr("use_image_if_no_bounding_boxes: bool = false")
-  .SetIsStateful()
-  .SetShapeFn([](InferenceContext* c) {
-    c->set_output(0, c->Vector(3));
-    c->set_output(1, c->Vector(3));
-    c->set_output(2, c->MakeShape({1, 1, 4}));
-    return Status::OK();
-  })
-  .Doc(R"doc(
+    .Input("image_size: T")
+    .Input("bounding_boxes: float")
+    .Input("min_object_covered: float")
+    .Output("begin: T")
+    .Output("size: T")
+    .Output("bboxes: float")
+    .Attr("T: {uint8, int8, int16, int32, int64}")
+    .Attr("seed: int = 0")
+    .Attr("seed2: int = 0")
+    .Attr("aspect_ratio_range: list(float) = [0.75, 1.33]")
+    .Attr("area_range: list(float) = [0.05, 1.0]")
+    .Attr("max_attempts: int = 100")
+    .Attr("use_image_if_no_bounding_boxes: bool = false")
+    .SetIsStateful()
+    .SetShapeFn([](InferenceContext* c) {
+      c->set_output(0, c->Vector(3));
+      c->set_output(1, c->Vector(3));
+      c->set_output(2, c->MakeShape({1, 1, 4}));
+      return Status::OK();
+    })
+    .Doc(R"doc(
 Generate a single randomly distorted bounding box for an image.
 
 Bounding box annotations are often supplied in addition to ground-truth labels
@@ -1236,16 +1236,16 @@ method: A string specifying the interpolation method. Only 'bilinear' is
 // --------------------------------------------------------------------------
 
 REGISTER_OP("NonMaxSuppression")
-  .Input("boxes: float")
-  .Input("scores: float")
-  .Input("max_output_size: int32")
-  .Output("selected_indices: int32")
-  .Attr("iou_threshold: float = 0.5")
-  .SetShapeFn([](InferenceContext* c) {
+    .Input("boxes: float")
+    .Input("scores: float")
+    .Input("max_output_size: int32")
+    .Output("selected_indices: int32")
+    .Attr("iou_threshold: float = 0.5")
+    .SetShapeFn([](InferenceContext* c) {
       c->set_output(0, c->Vector(c->UnknownDim()));
       return Status::OK();
     })
-  .Doc(R"doc(
+    .Doc(R"doc(
 Greedily selects a subset of bounding boxes in descending order of score,
 pruning away boxes that have high intersection-over-union (IOU) overlap
 with previously selected boxes.  Bounding boxes are supplied as
diff --git a/tensorflow/core/ops/linalg_ops.cc b/tensorflow/core/ops/linalg_ops.cc
index 76e2149522..4851619f83 100644
--- a/tensorflow/core/ops/linalg_ops.cc
+++ b/tensorflow/core/ops/linalg_ops.cc
@@ -25,7 +25,6 @@ using shape_inference::ShapeHandle;
 
 namespace {
 
-
 // Return in <out> the result of making the end of <s> a square matrix.
 Status MakeBatchSquareMatrix(InferenceContext* c, ShapeHandle input,
                              ShapeHandle* out) {
diff --git a/tensorflow/core/ops/math_grad_test.cc b/tensorflow/core/ops/math_grad_test.cc
index 2b4b35547b..8dcd3e815f 100644
--- a/tensorflow/core/ops/math_grad_test.cc
+++ b/tensorflow/core/ops/math_grad_test.cc
@@ -385,7 +385,7 @@ class TestOp : public OpKernel {
 REGISTER_KERNEL_BUILDER(Name("TestOpWithNoGrad").Device(DEVICE_CPU), TestOp);
 #ifdef TENSORFLOW_USE_SYCL
 REGISTER_KERNEL_BUILDER(Name("TestOpWithNoGrad").Device(DEVICE_SYCL), TestOp);
-#endif // TENSORFLOW_USE_SYCL
+#endif  // TENSORFLOW_USE_SYCL
 
 TEST_F(MathGradTest, Error_Reporting) {
   auto x = test::AsTensor<float>({-3.f});
@@ -557,11 +557,10 @@ TEST_F(MathGradTest, Acosh) {
 TEST_F(MathGradTest, Atanh) {
   auto x = test::AsTensor<float>({-0.3f, -0.2f, -0.1f, 0.1f, 0.2f, 0.3f},
                                  TensorShape({2, 3}));
-  auto g = [](float x) {
-    return 1.f / (1.f - x * x);
-  };
+  auto g = [](float x) { return 1.f / (1.f - x * x); };
   auto dx = test::AsTensor<float>(
-      {g(-0.3f), g(-0.2f), g(-0.1f), g(0.1f), g(0.2f), g(0.3f)}, TensorShape({2, 3}));
+      {g(-0.3f), g(-0.2f), g(-0.1f), g(0.1f), g(0.2f), g(0.3f)},
+      TensorShape({2, 3}));
   auto ans = SymGrad("Atanh", x);
   test::ExpectClose(ans, dx);
 }
@@ -761,7 +760,7 @@ TEST_F(MathGradTest, Pow) {
   }
 }
 
-//TODO{lukeiwanski}: Implement Complex Pow for SYCL
+// TODO{lukeiwanski}: Implement Complex Pow for SYCL
 #ifndef TENSORFLOW_USE_SYCL
 TEST_F(MathGradTest, ComplexPow) {
   auto x = test::AsTensor<complex64>({0.f, 2.f, -2.f}, TensorShape({3}));
@@ -781,7 +780,7 @@ TEST_F(MathGradTest, ComplexPow) {
       dy, test::AsTensor<complex64>({h(0.f, 2.f), h(2.f, 2.f), h(-2.f, 2.f)},
                                     TensorShape({3})));
 }
-#endif // TENSORFLOW_USE_SYCL
+#endif  // TENSORFLOW_USE_SYCL
 
 TEST_F(MathGradTest, Maximum) {
   auto x = test::AsTensor<float>({-3.f, -2.f, -1.f, 1.f, 2.f, 3.f},
@@ -943,7 +942,7 @@ TEST_F(MathGradTest, MatMul_11) {
   test::ExpectClose(dy, MatMul(dz, true, x, true));
 }
 
-//TODO{lukeiwanski}: Implement BatchMatMul for SYCL
+// TODO{lukeiwanski}: Implement BatchMatMul for SYCL
 #ifndef TENSORFLOW_USE_SYCL
 TEST_F(MathGradTest, BatchMatMul_00) {
   auto x = test::AsTensor<float>({1.f, 2.f, 3.f, 4.f, 5.f, 6.f},
@@ -992,7 +991,7 @@ TEST_F(MathGradTest, BatchMatMul_11) {
   test::ExpectClose(dx, BatchMatMul(y, true, dz, true));
   test::ExpectClose(dy, BatchMatMul(dz, true, x, true));
 }
-#endif // TENSORFLOW_USE_SYCL
+#endif  // TENSORFLOW_USE_SYCL
 
 TEST_F(MathGradTest, Sum_dim0) {
   auto x = test::AsTensor<float>({-3.f, -2.f, -1.f, 1.f, 2.f, 3.f},
diff --git a/tensorflow/core/ops/math_ops.cc b/tensorflow/core/ops/math_ops.cc
index 61db896c51..b06cb2a241 100644
--- a/tensorflow/core/ops/math_ops.cc
+++ b/tensorflow/core/ops/math_ops.cc
@@ -235,7 +235,9 @@ value is computed as \\( \sqrt{a^2 + b^2}\\).
       .Attr("T: {half, float, double, complex64, complex128}") \
       .SetShapeFn(shape_inference::UnchangedShape)
 
-REGISTER_OP("Neg").UNARY().Doc(R"doc(
+REGISTER_OP("Neg")
+    .UNARY()
+    .Doc(R"doc(
 Computes numerical negative value element-wise.
 I.e., \\(y = -x\\).
 )doc");
@@ -258,155 +260,217 @@ is the corresponding input gradient.
 )doc")
     .Deprecated(17, "Use ReciprocalGrad");
 
-REGISTER_OP("Reciprocal").UNARY().Doc(R"doc(
+REGISTER_OP("Reciprocal")
+    .UNARY()
+    .Doc(R"doc(
 Computes the reciprocal of x element-wise.
 I.e., \\(y = 1 / x\\).
 )doc");
 
-REGISTER_OP("ReciprocalGrad").UNARY_GRADIENT_COMPLEX().Doc(R"doc(
+REGISTER_OP("ReciprocalGrad")
+    .UNARY_GRADIENT_COMPLEX()
+    .Doc(R"doc(
 Computes the gradient for the inverse of `x` wrt its input.
 
 Specifically, `grad = -dy * y*y`, where `y = 1/x`, and `dy`
 is the corresponding input gradient.
 )doc");
 
-REGISTER_OP("Square").UNARY().Doc(R"doc(
+REGISTER_OP("Square")
+    .UNARY()
+    .Doc(R"doc(
 Computes square of x element-wise.
 I.e., \\(y = x * x = x^2\\).
 )doc");
 
-REGISTER_OP("Sqrt").UNARY_COMPLEX().Doc(R"doc(
+REGISTER_OP("Sqrt")
+    .UNARY_COMPLEX()
+    .Doc(R"doc(
 Computes square root of x element-wise.
 I.e., \\(y = \sqrt{x} = x^{1/2}\\).
 )doc");
 
-REGISTER_OP("SqrtGrad").UNARY_GRADIENT_COMPLEX().Doc(R"doc(
+REGISTER_OP("SqrtGrad")
+    .UNARY_GRADIENT_COMPLEX()
+    .Doc(R"doc(
 Computes the gradient for the sqrt of `x` wrt its input.
 
 Specifically, `grad = dy * 0.5 / y`, where `y = sqrt(x)`, and `dy`
 is the corresponding input gradient.
 )doc");
 
-REGISTER_OP("Rsqrt").UNARY_COMPLEX().Doc(R"doc(
+REGISTER_OP("Rsqrt")
+    .UNARY_COMPLEX()
+    .Doc(R"doc(
 Computes reciprocal of square root of x element-wise.
 I.e., \\(y = 1 / \sqrt{x}\\).
 )doc");
 
-REGISTER_OP("Round").UNARY().Doc(R"doc(
+REGISTER_OP("Round")
+    .UNARY()
+    .Doc(R"doc(
 Rounds the values of a tensor to the nearest integer, element-wise.
 
 Rounds half to even.  Also known as bankers rounding. If you want to round
 according to the current system rounding mode use std::cint.
 )doc");
 
-REGISTER_OP("RsqrtGrad").UNARY_GRADIENT_COMPLEX().Doc(R"doc(
+REGISTER_OP("RsqrtGrad")
+    .UNARY_GRADIENT_COMPLEX()
+    .Doc(R"doc(
 Computes the gradient for the rsqrt of `x` wrt its input.
 
 Specifically, `grad = dy * -0.5 * y^3`, where `y = rsqrt(x)`, and `dy`
 is the corresponding input gradient.
 )doc");
 
-REGISTER_OP("Exp").UNARY_COMPLEX().Doc(R"doc(
+REGISTER_OP("Exp")
+    .UNARY_COMPLEX()
+    .Doc(R"doc(
 Computes exponential of x element-wise.  \\(y = e^x\\).
 )doc");
 
-REGISTER_OP("Expm1").UNARY_COMPLEX().Doc(R"doc(
+REGISTER_OP("Expm1")
+    .UNARY_COMPLEX()
+    .Doc(R"doc(
 Computes exponential of x - 1 element-wise.
 I.e., \\(y = (\exp x) - 1\\).
 )doc");
 
-REGISTER_OP("Log").UNARY_COMPLEX().Doc(R"doc(
+REGISTER_OP("Log")
+    .UNARY_COMPLEX()
+    .Doc(R"doc(
 Computes natural logarithm of x element-wise.
 I.e., \\(y = \log_e x\\).
 )doc");
 
-REGISTER_OP("Log1p").UNARY_COMPLEX().Doc(R"doc(
+REGISTER_OP("Log1p")
+    .UNARY_COMPLEX()
+    .Doc(R"doc(
 Computes natural logarithm of (1 + x) element-wise.
 I.e., \\(y = \log_e (1 + x)\\).
 )doc");
 
-REGISTER_OP("Sinh").UNARY_COMPLEX().Doc(R"doc(
+REGISTER_OP("Sinh")
+    .UNARY_COMPLEX()
+    .Doc(R"doc(
 Computes hyperbolic sine of x element-wise.
 )doc");
 
-REGISTER_OP("Cosh").UNARY_COMPLEX().Doc(R"doc(
+REGISTER_OP("Cosh")
+    .UNARY_COMPLEX()
+    .Doc(R"doc(
 Computes hyperbolic cosine of x element-wise.
 )doc");
 
-REGISTER_OP("Tanh").UNARY_COMPLEX().Doc(R"doc(
+REGISTER_OP("Tanh")
+    .UNARY_COMPLEX()
+    .Doc(R"doc(
 Computes hyperbolic tangent of `x` element-wise.
 )doc");
 
-REGISTER_OP("Asinh").UNARY_COMPLEX().Doc(R"doc(
+REGISTER_OP("Asinh")
+    .UNARY_COMPLEX()
+    .Doc(R"doc(
 Computes inverse hyperbolic sine of x element-wise.
 )doc");
 
-REGISTER_OP("Acosh").UNARY_COMPLEX().Doc(R"doc(
+REGISTER_OP("Acosh")
+    .UNARY_COMPLEX()
+    .Doc(R"doc(
 Computes inverse hyperbolic cosine of x element-wise.
 )doc");
 
-REGISTER_OP("Atanh").UNARY_COMPLEX().Doc(R"doc(
+REGISTER_OP("Atanh")
+    .UNARY_COMPLEX()
+    .Doc(R"doc(
 Computes inverse hyperbolic tangent of x element-wise.
 )doc");
 
-REGISTER_OP("TanhGrad").UNARY_GRADIENT_COMPLEX().Doc(R"doc(
+REGISTER_OP("TanhGrad")
+    .UNARY_GRADIENT_COMPLEX()
+    .Doc(R"doc(
 Computes the gradient for the tanh of `x` wrt its input.
 
 Specifically, `grad = dy * (1 - y*y)`, where `y = tanh(x)`, and `dy`
 is the corresponding input gradient.
 )doc");
 
-REGISTER_OP("Lgamma").UNARY_REAL().Doc(R"doc(
+REGISTER_OP("Lgamma")
+    .UNARY_REAL()
+    .Doc(R"doc(
 Computes the log of the absolute value of `Gamma(x)` element-wise.
 )doc");
 
-REGISTER_OP("Digamma").UNARY_REAL().Doc(R"doc(
+REGISTER_OP("Digamma")
+    .UNARY_REAL()
+    .Doc(R"doc(
 Computes Psi, the derivative of Lgamma (the log of the absolute value of
 `Gamma(x)`), element-wise.
 )doc");
 
-REGISTER_OP("Erf").UNARY_REAL().Doc(R"doc(
+REGISTER_OP("Erf")
+    .UNARY_REAL()
+    .Doc(R"doc(
 Computes the Gauss error function of `x` element-wise.
 )doc");
 
-REGISTER_OP("Erfc").UNARY_REAL().Doc(R"doc(
+REGISTER_OP("Erfc")
+    .UNARY_REAL()
+    .Doc(R"doc(
 Computes the complementary error function of `x` element-wise.
 )doc");
 
-REGISTER_OP("Sigmoid").UNARY_COMPLEX().Doc(R"doc(
+REGISTER_OP("Sigmoid")
+    .UNARY_COMPLEX()
+    .Doc(R"doc(
 Computes sigmoid of `x` element-wise.
 
 Specifically, `y = 1 / (1 + exp(-x))`.
 )doc");
 
-REGISTER_OP("SigmoidGrad").UNARY_GRADIENT_COMPLEX().Doc(R"doc(
+REGISTER_OP("SigmoidGrad")
+    .UNARY_GRADIENT_COMPLEX()
+    .Doc(R"doc(
 Computes the gradient of the sigmoid of `x` wrt its input.
 
 Specifically, `grad = dy * y * (1 - y)`, where `y = sigmoid(x)`, and
 `dy` is the corresponding input gradient.
 )doc");
 
-REGISTER_OP("Sin").UNARY_COMPLEX().Doc(R"doc(
+REGISTER_OP("Sin")
+    .UNARY_COMPLEX()
+    .Doc(R"doc(
 Computes sin of x element-wise.
 )doc");
 
-REGISTER_OP("Cos").UNARY_COMPLEX().Doc(R"doc(
+REGISTER_OP("Cos")
+    .UNARY_COMPLEX()
+    .Doc(R"doc(
 Computes cos of x element-wise.
 )doc");
 
-REGISTER_OP("Tan").UNARY().Doc(R"doc(
+REGISTER_OP("Tan")
+    .UNARY()
+    .Doc(R"doc(
 Computes tan of x element-wise.
 )doc");
 
-REGISTER_OP("Asin").UNARY().Doc(R"doc(
+REGISTER_OP("Asin")
+    .UNARY()
+    .Doc(R"doc(
 Computes asin of x element-wise.
 )doc");
 
-REGISTER_OP("Acos").UNARY().Doc(R"doc(
+REGISTER_OP("Acos")
+    .UNARY()
+    .Doc(R"doc(
 Computes acos of x element-wise.
 )doc");
 
-REGISTER_OP("Atan").UNARY().Doc(R"doc(
+REGISTER_OP("Atan")
+    .UNARY()
+    .Doc(R"doc(
 Computes atan of x element-wise.
 )doc");
 
@@ -942,28 +1006,36 @@ beta function.
       .Attr("T: realnumbertype") \
       .SetShapeFn(shape_inference::BroadcastBinaryOpShapeFn)
 
-REGISTER_OP("Less").COMPARISON().Doc(R"doc(
+REGISTER_OP("Less")
+    .COMPARISON()
+    .Doc(R"doc(
 Returns the truth value of (x < y) element-wise.
 
 *NOTE*: `Less` supports broadcasting. More about broadcasting
 [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
 )doc");
 
-REGISTER_OP("LessEqual").COMPARISON().Doc(R"doc(
+REGISTER_OP("LessEqual")
+    .COMPARISON()
+    .Doc(R"doc(
 Returns the truth value of (x <= y) element-wise.
 
 *NOTE*: `LessEqual` supports broadcasting. More about broadcasting
 [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
 )doc");
 
-REGISTER_OP("Greater").COMPARISON().Doc(R"doc(
+REGISTER_OP("Greater")
+    .COMPARISON()
+    .Doc(R"doc(
 Returns the truth value of (x > y) element-wise.
 
 *NOTE*: `Greater` supports broadcasting. More about broadcasting
 [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
 )doc");
 
-REGISTER_OP("GreaterEqual").COMPARISON().Doc(R"doc(
+REGISTER_OP("GreaterEqual")
+    .COMPARISON()
+    .Doc(R"doc(
 Returns the truth value of (x >= y) element-wise.
 
 *NOTE*: `GreaterEqual` supports broadcasting. More about broadcasting
@@ -985,14 +1057,18 @@ Returns the truth value of (x >= y) element-wise.
           "quint8, qint8, qint32, string, bool, complex128}")           \
       .SetShapeFn(shape_inference::BroadcastBinaryOpShapeFn)
 
-REGISTER_OP("Equal").EQUALITY_COMPARISON().Doc(R"doc(
+REGISTER_OP("Equal")
+    .EQUALITY_COMPARISON()
+    .Doc(R"doc(
 Returns the truth value of (x == y) element-wise.
 
 *NOTE*: `Equal` supports broadcasting. More about broadcasting
 [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
 )doc");
 
-REGISTER_OP("NotEqual").EQUALITY_COMPARISON().Doc(R"doc(
+REGISTER_OP("NotEqual")
+    .EQUALITY_COMPARISON()
+    .Doc(R"doc(
 Returns the truth value of (x != y) element-wise.
 
 *NOTE*: `NotEqual` supports broadcasting. More about broadcasting
@@ -1030,14 +1106,18 @@ Returns the truth value of NOT x element-wise.
       .SetIsCommutative() \
       .SetShapeFn(shape_inference::BroadcastBinaryOpShapeFn)
 
-REGISTER_OP("LogicalAnd").BINARY_LOGICAL().Doc(R"doc(
+REGISTER_OP("LogicalAnd")
+    .BINARY_LOGICAL()
+    .Doc(R"doc(
 Returns the truth value of x AND y element-wise.
 
 *NOTE*: `LogicalAnd` supports broadcasting. More about broadcasting
 [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
 )doc");
 
-REGISTER_OP("LogicalOr").BINARY_LOGICAL().Doc(R"doc(
+REGISTER_OP("LogicalOr")
+    .BINARY_LOGICAL()
+    .Doc(R"doc(
 Returns the truth value of x OR y element-wise.
 
 *NOTE*: `LogicalOr` supports broadcasting. More about broadcasting
@@ -1977,12 +2057,12 @@ Status RangeSize(const Tensor* start_t, const Tensor* limit_t,
   T limit = limit_t->scalar<T>()();
   T delta = delta_t->scalar<T>()();
   if (start > limit && delta > 0) {
-    return errors::InvalidArgument(
-        "Requires start <= limit when delta > 0: ", start, "/", limit);
+    return errors::InvalidArgument("Requires start <= limit when delta > 0: ",
+                                   start, "/", limit);
   }
   if (start < limit && delta < 0) {
-    return errors::InvalidArgument(
-        "Requires start >= limit when delta < 0: ", start, "/", limit);
+    return errors::InvalidArgument("Requires start >= limit when delta < 0: ",
+                                   start, "/", limit);
   }
   if (delta == 0) {
     return errors::InvalidArgument("Requires delta != 0");
diff --git a/tensorflow/core/ops/nn_ops.cc b/tensorflow/core/ops/nn_ops.cc
index 1d26660a4b..de059a3e7e 100644
--- a/tensorflow/core/ops/nn_ops.cc
+++ b/tensorflow/core/ops/nn_ops.cc
@@ -2176,9 +2176,9 @@ Status TopKShapeFn(InferenceContext* c) {
   DimensionHandle last_dim = c->Dim(input, -1);
   if (c->ValueKnown(last_dim) && c->ValueKnown(k_dim) &&
       c->Value(last_dim) < c->Value(k_dim)) {
-    return errors::InvalidArgument(
-        "input must have last dimension >= k = ", c->Value(k_dim), " but is ",
-        c->Value(last_dim));
+    return errors::InvalidArgument("input must have last dimension >= k = ",
+                                   c->Value(k_dim), " but is ",
+                                   c->Value(last_dim));
   }
 
   // Replace last_dim with k_dim.
@@ -2278,9 +2278,9 @@ REGISTER_OP("NthElement")
       DimensionHandle last_dim = c->Dim(input, -1);
       if (c->ValueKnown(last_dim) && c->ValueKnown(n_dim) &&
           c->Value(last_dim) <= c->Value(n_dim)) {
-        return errors::InvalidArgument(
-            "Input must have last dimension > n = ", c->Value(n_dim), " but is ",
-            c->Value(last_dim));
+        return errors::InvalidArgument("Input must have last dimension > n = ",
+                                       c->Value(n_dim), " but is ",
+                                       c->Value(last_dim));
       }
 
       // Reduce last_dim for output tensor
diff --git a/tensorflow/core/ops/nn_ops_test.cc b/tensorflow/core/ops/nn_ops_test.cc
index 94ecf4d5db..1b17a7cda6 100644
--- a/tensorflow/core/ops/nn_ops_test.cc
+++ b/tensorflow/core/ops/nn_ops_test.cc
@@ -95,14 +95,13 @@ TEST(NNOpsTest, NthElement_ShapeFn) {
   INFER_OK(op, "[?,3,?,21];[]", "[d0_0,d0_1,d0_2]");
 
   INFER_ERROR("Shape must be at least rank 1 but is rank 0", op, "[];[]");
-  INFER_ERROR("Input must have last dimension > n = 20 but is 1", op,
-              "[1];[]");
+  INFER_ERROR("Input must have last dimension > n = 20 but is 1", op, "[1];[]");
   INFER_ERROR("Input must have last dimension > n = 20 but is 20", op,
               "[1,2,3,20];[]");
   n_t = test::AsScalar<int32>(-1);
   INFER_ERROR(
-     "Dimension size, given by scalar input 1, must be non-negative but is -1",
-     op, "[1,2,3,4];[]");
+      "Dimension size, given by scalar input 1, must be non-negative but is -1",
+      op, "[1,2,3,4];[]");
 }
 
 TEST(NNOpsTest, BatchNormWithGlobalNormalization_ShapeFn) {
@@ -386,9 +385,8 @@ TEST(NNOpsTest, Dilation2DBackpropFilter_ShapeFn) {
 }
 
 TEST(NNOpsTest, MergeBothInputs_ShapeFn) {
-  for (const char* op_name :
-       {"ReluGrad", "Relu6Grad", "EluGrad", "SeluGrad", "SoftplusGrad",
-        "SoftsignGrad"}) {
+  for (const char* op_name : {"ReluGrad", "Relu6Grad", "EluGrad", "SeluGrad",
+                              "SoftplusGrad", "SoftsignGrad"}) {
     ShapeInferenceTestOp op(op_name);
 
     INFER_OK(op, "?;?", "in0|in1");
diff --git a/tensorflow/core/ops/sparse_ops_test.cc b/tensorflow/core/ops/sparse_ops_test.cc
index ea49f1a199..0df3320484 100644
--- a/tensorflow/core/ops/sparse_ops_test.cc
+++ b/tensorflow/core/ops/sparse_ops_test.cc
@@ -187,8 +187,8 @@ TEST(SparseOpsTest, SparseTensorDenseMatMul_ShapeFn) {
 
   // second output dim comes from b, depending on adjoint_b value.
   INFER_OK(op, "?;?;?;?", "[?,?]");
-  INFER_OK(op, "?;?;?;[?,?]", "[?,d3_1]");  // use d3_1, !adjoint_b.
-  INFER_OK(op, "?;?;?;[1,2]", "[?,d3_1]");  // use d3_1, !adjoint_b.
+  INFER_OK(op, "?;?;?;[?,?]", "[?,d3_1]");    // use d3_1, !adjoint_b.
+  INFER_OK(op, "?;?;?;[1,2]", "[?,d3_1]");    // use d3_1, !adjoint_b.
   INFER_OK(op, "?;?;[2];[1,2]", "[?,d3_1]");  // use d3_1, !adjoint_b.
 
   set_adjoints(false, true);
diff --git a/tensorflow/core/ops/stateless_random_ops.cc b/tensorflow/core/ops/stateless_random_ops.cc
index b222b5b241..7c00fdb99f 100644
--- a/tensorflow/core/ops/stateless_random_ops.cc
+++ b/tensorflow/core/ops/stateless_random_ops.cc
@@ -45,7 +45,8 @@ static Status StatelessShape(shape_inference::InferenceContext* context) {
       .SetShapeFn(StatelessShape)
 
 // This op is exposed through contrib/stateless only.  The interface may change.
-REGISTER_STATELESS_OP("StatelessRandomUniform").Doc(R"doc(
+REGISTER_STATELESS_OP("StatelessRandomUniform")
+    .Doc(R"doc(
 Outputs deterministic pseudorandom random values from a uniform distribution.
 
 The generated values follow a uniform distribution in the range `[0, 1)`. The
@@ -60,7 +61,8 @@ output: Random values with specified shape.
 )doc");
 
 // This op is exposed through contrib/stateless only.  The interface may change.
-REGISTER_STATELESS_OP("StatelessRandomNormal").Doc(R"doc(
+REGISTER_STATELESS_OP("StatelessRandomNormal")
+    .Doc(R"doc(
 Outputs deterministic pseudorandom values from a normal distribution.
 
 The generated values will have mean 0 and standard deviation 1.
@@ -74,7 +76,8 @@ output: Random values with specified shape.
 )doc");
 
 // This op is exposed through contrib/stateless only.  The interface may change.
-REGISTER_STATELESS_OP("StatelessTruncatedNormal").Doc(R"doc(
+REGISTER_STATELESS_OP("StatelessTruncatedNormal")
+    .Doc(R"doc(
 Outputs deterministic pseudorandom values from a truncated normal distribution.
 
 The generated values follow a normal distribution with mean 0 and standard
-- 
GitLab


From c6aecff757f36b9446f742a750eac9cc768a230b Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Wed, 25 Oct 2017 00:42:18 +0000
Subject: [PATCH 788/909] Add the option to do an incremental clang  style
 check.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/tools/ci_build/ci_sanity.sh        | 62 ++++++++++++++++++-
 .../tools/ci_build/clang_format_check.sh      | 36 -----------
 2 files changed, 61 insertions(+), 37 deletions(-)
 delete mode 100755 tensorflow/tools/ci_build/clang_format_check.sh

diff --git a/tensorflow/tools/ci_build/ci_sanity.sh b/tensorflow/tools/ci_build/ci_sanity.sh
index 7cf97dacf0..e1757c6d15 100755
--- a/tensorflow/tools/ci_build/ci_sanity.sh
+++ b/tensorflow/tools/ci_build/ci_sanity.sh
@@ -426,8 +426,68 @@ do_code_link_check() {
   tensorflow/tools/ci_build/code_link_check.sh
 }
 
+# List .h|.cc files changed in the last non-merge git commit that still exist,
+# i.e., not removed.
+# Usage: get_clang_files_to_check [--incremental]
+get_clang_files_to_check() {
+  if [[ "$1" == "--incremental" ]]; then
+    CHANGED_CLANG_FILES=$(get_changed_files_in_last_non_merge_git_commit | \
+                       grep '.*\.h$\|.*\.cc$')
+
+    # Do not include files removed in the last non-merge commit.
+    CLANG_FILES=""
+    for CLANG_FILE in ${CHANGED_CLANG_FILES}; do
+      if [[ -f "${CLANG_FILE}" ]]; then
+        CLANG_FILES="${CLANG_FILES} ${CLANG_FILE}"
+      fi
+    done
+
+    echo "${CLANG_FILES}"
+  else
+    find tensorflow -name '*.h' -o -name '*.cc'
+  fi
+}
+
 do_clang_format_check() {
-  CLANG_FORMAT=clang-format-3.8 tensorflow/tools/ci_build/clang_format_check.sh
+  if [[ $# != "0" ]] && [[ $# != "1" ]]; then
+    echo "Invalid syntax when invoking do_clang_format_check"
+    echo "Usage: do_clang_format_check [--incremental]"
+    return 1
+  fi
+
+  if [[ "$1" == "--incremental" ]]; then
+    CLANG_SRC_FILES=$(get_clang_files_to_check --incremental)
+
+    if [[ -z "${CLANG_SRC_FILES}" ]]; then
+      echo "do_clang_format_check will NOT run due to --incremental flag and "\
+"due to the absence of Python code changes in the last commit."
+      return 0
+    fi
+  elif [[ -z "$1" ]]; then
+    CLANG_SRC_FILES=$(get_clang_files_to_check)
+  else
+    echo "Invalid syntax for invoking do_clang_format_check"
+    echo "Usage: do_clang_format_check [--incremental]"
+    return 1
+  fi
+
+  CLANG_FORMAT=${CLANG_FORMAT:-clang-format-3.8}
+
+  success=1
+  for filename in $CLANG_SRC_FILES; do
+    $CLANG_FORMAT --style=google $filename | diff $filename - > /dev/null
+    if [ ! $? -eq 0 ]; then
+      success=0
+      echo File $filename is not properly formatted with "clang-format "\
+"--style=google"
+    fi
+  done
+
+  if [ $success == 0 ]; then
+    echo Clang format check fails.
+    exit 1
+  fi
+  echo Clang format check success.
 }
 
 do_check_load_py_test() {
diff --git a/tensorflow/tools/ci_build/clang_format_check.sh b/tensorflow/tools/ci_build/clang_format_check.sh
deleted file mode 100755
index 5f252d7f35..0000000000
--- a/tensorflow/tools/ci_build/clang_format_check.sh
+++ /dev/null
@@ -1,36 +0,0 @@
-#!/usr/bin/env bash
-# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-# please run this at root directory of tensorflow
-success=1
-
-CLANG_FORMAT=${CLANG_FORMAT:-clang-format}
-
-# only tensorflow/core/ops is checked at the moment for experimental purpose
-for filename in $(find tensorflow/core/ops -name *.h -o -name *.cc); do
-  $CLANG_FORMAT --style=google $filename | diff $filename - > /dev/null
-  if [ ! $? -eq 0 ]; then
-    success=0
-    echo File $filename is not properly formatted with "clang-format --style=google"
-  fi
-done
-
-if [ $success == 0 ]; then
-  echo Clang format check fails.
-  exit 1
-fi
-
-echo Clang format check success.
-- 
GitLab


From 8087e67252bca4075e59ab75023826dae23dfb74 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Wed, 25 Oct 2017 15:50:02 -0700
Subject: [PATCH 789/909] [XLA] Remove dead kUpdate opcode.

PiperOrigin-RevId: 173462881
---
 tensorflow/compiler/xla/service/hlo_graph_dumper.cc   |  1 -
 tensorflow/compiler/xla/service/hlo_instruction.cc    |  3 ---
 tensorflow/compiler/xla/service/hlo_matchers.h        |  1 -
 tensorflow/compiler/xla/service/hlo_opcode.cc         |  3 ---
 tensorflow/compiler/xla/service/hlo_opcode.h          |  1 -
 tensorflow/compiler/xla/service/inliner.cc            |  3 +--
 tensorflow/compiler/xla/service/instruction_fusion.cc |  1 -
 tensorflow/compiler/xla/service/shape_inference.cc    | 10 ----------
 tensorflow/compiler/xla/service/user_computation.cc   |  2 --
 tensorflow/compiler/xla/tools/parser/hlo_parser.cc    |  1 -
 tensorflow/compiler/xla/xla_data.proto                |  4 ----
 11 files changed, 1 insertion(+), 29 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
index 24e390529e..ed94a5be91 100644
--- a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
+++ b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
@@ -814,7 +814,6 @@ ColorScheme HloDotDumper::GetInstructionColor(const HloInstruction* instr) {
     case HloOpcode::kPad:
     case HloOpcode::kReshape:
     case HloOpcode::kReverse:
-    case HloOpcode::kUpdate:
       return kGreen;
     case HloOpcode::kConvolution:
     case HloOpcode::kDot:
diff --git a/tensorflow/compiler/xla/service/hlo_instruction.cc b/tensorflow/compiler/xla/service/hlo_instruction.cc
index 8e52d131a6..d53ac221d1 100644
--- a/tensorflow/compiler/xla/service/hlo_instruction.cc
+++ b/tensorflow/compiler/xla/service/hlo_instruction.cc
@@ -1156,7 +1156,6 @@ std::unique_ptr<HloInstruction> HloInstruction::CloneWithNewOperands(
       break;
     case HloOpcode::kRecv:
     case HloOpcode::kSend:
-    case HloOpcode::kUpdate:
     case HloOpcode::kIndex:
     case HloOpcode::kTrace:
       LOG(FATAL) << "Not yet implemented, clone: " << HloOpcodeString(opcode_);
@@ -1541,7 +1540,6 @@ bool HloInstruction::IdenticalSlowPath(
     case HloOpcode::kInfeed:
     case HloOpcode::kOutfeed:
     case HloOpcode::kSort:
-    case HloOpcode::kUpdate:
     case HloOpcode::kSend:
     case HloOpcode::kRecv:
       return false;
@@ -2265,7 +2263,6 @@ Status HloInstruction::Visit(DfsHloVisitor* visitor) {
     // These opcodes are not handled here.
     case HloOpcode::kIndex:
     case HloOpcode::kTrace:
-    case HloOpcode::kUpdate:
       break;
   }
   return Unimplemented("unhandled HloOpcode for DfsHloVisitor: %s",
diff --git a/tensorflow/compiler/xla/service/hlo_matchers.h b/tensorflow/compiler/xla/service/hlo_matchers.h
index d1ae5f776d..5440ed2eda 100644
--- a/tensorflow/compiler/xla/service/hlo_matchers.h
+++ b/tensorflow/compiler/xla/service/hlo_matchers.h
@@ -115,7 +115,6 @@ HLO_MATCHER(Tanh);
 HLO_MATCHER(Trace);
 HLO_MATCHER(Transpose);
 HLO_MATCHER(Tuple);
-HLO_MATCHER(Update);
 HLO_MATCHER(While);
 #undef HLO_MATCHER
 }  // namespace opcode_matchers
diff --git a/tensorflow/compiler/xla/service/hlo_opcode.cc b/tensorflow/compiler/xla/service/hlo_opcode.cc
index db3abeab22..e9000a8462 100644
--- a/tensorflow/compiler/xla/service/hlo_opcode.cc
+++ b/tensorflow/compiler/xla/service/hlo_opcode.cc
@@ -173,8 +173,6 @@ string HloOpcodeString(HloOpcode opcode) {
       return "transpose";
     case HloOpcode::kTuple:
       return "tuple";
-    case HloOpcode::kUpdate:
-      return "update";
     case HloOpcode::kWhile:
       return "while";
   }
@@ -254,7 +252,6 @@ StatusOr<HloOpcode> StringToHloOpcode(const string& opcode_name) {
        {"trace", HloOpcode::kTrace},
        {"transpose", HloOpcode::kTranspose},
        {"tuple", HloOpcode::kTuple},
-       {"update", HloOpcode::kUpdate},
        {"while", HloOpcode::kWhile}});
   auto it = opcode_map->find(opcode_name);
   if (it == opcode_map->end()) {
diff --git a/tensorflow/compiler/xla/service/hlo_opcode.h b/tensorflow/compiler/xla/service/hlo_opcode.h
index 4593df671e..c603c57e62 100644
--- a/tensorflow/compiler/xla/service/hlo_opcode.h
+++ b/tensorflow/compiler/xla/service/hlo_opcode.h
@@ -101,7 +101,6 @@ enum class HloOpcode {
   kTrace,
   kTranspose,
   kTuple,
-  kUpdate,
   kWhile,
 };
 
diff --git a/tensorflow/compiler/xla/service/inliner.cc b/tensorflow/compiler/xla/service/inliner.cc
index 6ea0f127d5..40df0dc355 100644
--- a/tensorflow/compiler/xla/service/inliner.cc
+++ b/tensorflow/compiler/xla/service/inliner.cc
@@ -76,8 +76,7 @@ Status InlinerVisitor::HandleMap(
   // Only inlining functions that are simply a single operation until a better
   // profitability model for inlining is defined.
   if (hlo_query::AllOperandsAreParameters(root)) {
-    if (root.opcode() == HloOpcode::kUpdate ||
-        root.opcode() == HloOpcode::kFusion ||
+    if (root.opcode() == HloOpcode::kFusion ||
         root.opcode() == HloOpcode::kIndex ||
         root.opcode() == HloOpcode::kParameter ||
         root.opcode() == HloOpcode::kTrace) {
diff --git a/tensorflow/compiler/xla/service/instruction_fusion.cc b/tensorflow/compiler/xla/service/instruction_fusion.cc
index 7e46d79ba4..0271f41697 100644
--- a/tensorflow/compiler/xla/service/instruction_fusion.cc
+++ b/tensorflow/compiler/xla/service/instruction_fusion.cc
@@ -105,7 +105,6 @@ namespace xla {
     case HloOpcode::kSort:
     case HloOpcode::kTanh:
     case HloOpcode::kTrace:
-    case HloOpcode::kUpdate:
     case HloOpcode::kWhile:
     case HloOpcode::kSend:
     case HloOpcode::kRecv:
diff --git a/tensorflow/compiler/xla/service/shape_inference.cc b/tensorflow/compiler/xla/service/shape_inference.cc
index 1df1022442..e41b7607c5 100644
--- a/tensorflow/compiler/xla/service/shape_inference.cc
+++ b/tensorflow/compiler/xla/service/shape_inference.cc
@@ -136,8 +136,6 @@ TernaryOperation OpcodeToTernaryOperation(HloOpcode opcode) {
       return TRIOP_CLAMP;
     case HloOpcode::kSelect:
       return TRIOP_SELECT;
-    case HloOpcode::kUpdate:
-      return TRIOP_UPDATE;
     default:
       LOG(FATAL) << "unhandled opcode " << opcode;
   }
@@ -822,14 +820,6 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(
       return InferClampShape(lhs, rhs, ehs);
     case TRIOP_SELECT:
       return InferSelectShape(lhs, rhs, ehs);
-    case TRIOP_UPDATE:
-      TF_RETURN_IF_ERROR(
-          ExpectNotTupleOrOpaque(lhs, "lhs of ternary operation"));
-      TF_RETURN_IF_ERROR(
-          ExpectNotTupleOrOpaque(rhs, "rhs of ternary operation"));
-      TF_RETURN_IF_ERROR(
-          ExpectNotTupleOrOpaque(ehs, "ehs of ternary operation"));
-      return lhs;
     default:
       return InvalidArgument("unknown operation %s",
                              TernaryOperation_Name(operation).c_str());
diff --git a/tensorflow/compiler/xla/service/user_computation.cc b/tensorflow/compiler/xla/service/user_computation.cc
index 065d2580c6..d818830f98 100644
--- a/tensorflow/compiler/xla/service/user_computation.cc
+++ b/tensorflow/compiler/xla/service/user_computation.cc
@@ -133,8 +133,6 @@ HloOpcode TernaryOperationToHloOpcode(TernaryOperation triop) {
       return HloOpcode::kClamp;
     case TRIOP_SELECT:
       return HloOpcode::kSelect;
-    case TRIOP_UPDATE:
-      return HloOpcode::kUpdate;
     default:
       LOG(FATAL) << "unhandled operation " << triop;
   }
diff --git a/tensorflow/compiler/xla/tools/parser/hlo_parser.cc b/tensorflow/compiler/xla/tools/parser/hlo_parser.cc
index a075d9057f..f4af03cc2f 100644
--- a/tensorflow/compiler/xla/tools/parser/hlo_parser.cc
+++ b/tensorflow/compiler/xla/tools/parser/hlo_parser.cc
@@ -400,7 +400,6 @@ bool HloParser::ParseInstruction(HloComputation::Builder* builder,
     case HloOpcode::kInfeed:
     case HloOpcode::kOutfeed:
     case HloOpcode::kBatchNormGrad:
-    case HloOpcode::kUpdate:
     case HloOpcode::kIndex:
     case HloOpcode::kTrace:
       return TokenError(StrCat("parsing not yet implemented for op: ",
diff --git a/tensorflow/compiler/xla/xla_data.proto b/tensorflow/compiler/xla/xla_data.proto
index 7ad61fab81..0efa3d0014 100644
--- a/tensorflow/compiler/xla/xla_data.proto
+++ b/tensorflow/compiler/xla/xla_data.proto
@@ -759,10 +759,6 @@ enum TernaryOperation {
   // true and operand1 if the predicate is false.
   TRIOP_SELECT = 1;
 
-  // Updates operand0 at index operand1 with value operand2 and outputs the
-  // updated value.
-  TRIOP_UPDATE = 2;
-
   // Given a min, max and an operand returns the operand if between min and max,
   // else returns min if operand is less than min or max if operand is greater
   // than max.
-- 
GitLab


From 5cd1ba5560be0af52cad5184c578be17ed7e8188 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 25 Oct 2017 16:00:28 -0700
Subject: [PATCH 790/909] Update documentation for SVD to make the differences
 with numpy.linalg.svd clearer.

PiperOrigin-RevId: 173464390
---
 tensorflow/python/ops/linalg_ops.py | 28 ++++++++++++++++++++++------
 1 file changed, 22 insertions(+), 6 deletions(-)

diff --git a/tensorflow/python/ops/linalg_ops.py b/tensorflow/python/ops/linalg_ops.py
index 1d917c22cc..2cb467c891 100644
--- a/tensorflow/python/ops/linalg_ops.py
+++ b/tensorflow/python/ops/linalg_ops.py
@@ -368,11 +368,11 @@ def self_adjoint_eigvals(tensor, name=None):
 
 
 def svd(tensor, full_matrices=False, compute_uv=True, name=None):
-  """Computes the singular value decompositions of one or more matrices.
+  r"""Computes the singular value decompositions of one or more matrices.
 
   Computes the SVD of each inner matrix in `tensor` such that
-  `tensor[..., :, :] = u[..., :, :] * diag(s[..., :, :]) * transpose(v[..., :,
-  :])`
+  `tensor[..., :, :] = u[..., :, :] * diag(s[..., :, :]) *
+   transpose(conj(v[..., :, :]))`
 
   ```python
   # a is a tensor.
@@ -406,9 +406,25 @@ def svd(tensor, full_matrices=False, compute_uv=True, name=None):
       `[..., N, N]`. Not returned if `compute_uv` is `False`.
 
   @compatibility(numpy)
-  Mostly equivalent to numpy.linalg.svd, except that the order of output
-  arguments here is `s`, `u`, `v` when `compute_uv` is `True`, as opposed to
-  `u`, `s`, `v` for numpy.linalg.svd.
+  Mostly equivalent to numpy.linalg.svd, except that
+    * The order of output  arguments here is `s`, `u`, `v` when `compute_uv` is
+      `True`, as opposed to `u`, `s`, `v` for numpy.linalg.svd.
+    * full_matrices is `False` by default as opposed to `True` for
+       numpy.linalg.svd.
+    * tf.linalg.svd uses the standard definition of the SVD
+      \\(A = U \Sigma V^H\\), such that the left singular vectors of `a` are
+      the columns of `u`, while the right singular vectors of `a` are the
+      columns of `v`. On the other hand, numpy.linalg.svd returns the adjoint
+      \\(V^H\\) as the third output argument.
+  ```python
+  import tensorflow as tf
+  import numpy as np
+  s, u, v = tf.linalg.svd(a)
+  tf_a_approx = tf.matmul(u, tf.matmul(tf.linalg.diag(s), v, adjoint_v=True))
+  u, s, v_adj = np.linalg.svd(a, full_matrices=False)
+  np_a_approx = np.dot(u, np.dot(np.diag(s), v_adj))
+  # tf_a_approx and np_a_approx should be numerically close.
+  ````
   @end_compatibility
   """
   # pylint: disable=protected-access
-- 
GitLab


From 7efbfc22850394c94785885d3b06cdd71124e5d6 Mon Sep 17 00:00:00 2001
From: Alina Sbirlea <asbirlea@google.com>
Date: Wed, 25 Oct 2017 16:01:35 -0700
Subject: [PATCH 791/909] Fix test: Call instruction was added to builder
 instead of call_builder. Triggers the assertion in MakeInstructionPostOrder
 when dumping out the module.

PiperOrigin-RevId: 173464579
---
 tensorflow/compiler/xla/service/algebraic_simplifier_test.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
index 57be144b36..3df50080d1 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
@@ -2127,7 +2127,7 @@ TEST_F(AlgebraicSimplifierTest, IteratorInvalidation) {
       HloInstruction::CreateConstant(Literal::CreateR1<float>({0.0f})));
   HloInstruction* one = call_builder.AddInstruction(
       HloInstruction::CreateConstant(Literal::CreateR1<float>({1.0f})));
-  builder.AddInstruction(
+  call_builder.AddInstruction(
       HloInstruction::CreateCall(r1f32, {zero, one}, dot_computation.get()));
 
   auto module = CreateNewModule();
-- 
GitLab


From 641e0df038f14be471ff54df626081515d25a741 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 25 Oct 2017 16:22:58 -0700
Subject: [PATCH 792/909] Should not affect public.

PiperOrigin-RevId: 173467560
---
 tensorflow/python/training/monitored_session.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/training/monitored_session.py b/tensorflow/python/training/monitored_session.py
index 2dd2114af0..dea62d27ba 100644
--- a/tensorflow/python/training/monitored_session.py
+++ b/tensorflow/python/training/monitored_session.py
@@ -349,8 +349,10 @@ def MonitoredTrainingSession(master='',  # pylint: disable=invalid-name
       config=config)
 
   if checkpoint_dir:
-    all_hooks.append(basic_session_run_hooks.StepCounterHook(
-        output_dir=checkpoint_dir, every_n_steps=log_step_count_steps))
+    if log_step_count_steps and log_step_count_steps > 0:
+      all_hooks.append(
+          basic_session_run_hooks.StepCounterHook(
+              output_dir=checkpoint_dir, every_n_steps=log_step_count_steps))
 
     if (save_summaries_steps and save_summaries_steps > 0) or (
         save_summaries_secs and save_summaries_secs > 0):
-- 
GitLab


From 9a0d94136b9a1d0ff62a81b50fb8c3dbb0c3b4a2 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Wed, 25 Oct 2017 23:30:45 +0000
Subject: [PATCH 793/909] Always enable --incremental.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/tools/ci_build/ci_sanity.sh | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tensorflow/tools/ci_build/ci_sanity.sh b/tensorflow/tools/ci_build/ci_sanity.sh
index e1757c6d15..1e1fd7db6b 100755
--- a/tensorflow/tools/ci_build/ci_sanity.sh
+++ b/tensorflow/tools/ci_build/ci_sanity.sh
@@ -464,7 +464,9 @@ do_clang_format_check() {
       return 0
     fi
   elif [[ -z "$1" ]]; then
-    CLANG_SRC_FILES=$(get_clang_files_to_check)
+    # TODO (yongtang): Always pass --incremental until all files have
+    # been sanitized gradually. Then this --incremental could be removed.
+    CLANG_SRC_FILES=$(get_clang_files_to_check --incremental)
   else
     echo "Invalid syntax for invoking do_clang_format_check"
     echo "Usage: do_clang_format_check [--incremental]"
-- 
GitLab


From 1f696bcf4b159624411dbb1b9f4a206aa0cd1c43 Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Wed, 25 Oct 2017 16:34:26 -0700
Subject: [PATCH 794/909] Explicitly disables variable inplace assignment
 methods.

Need a story for eager, graph functions (both easy) and graphs (hard).

PiperOrigin-RevId: 173468930
---
 .../python/ops/resource_variable_ops.py       | 21 +++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/tensorflow/python/ops/resource_variable_ops.py b/tensorflow/python/ops/resource_variable_ops.py
index 386fd204b6..439fa84238 100644
--- a/tensorflow/python/ops/resource_variable_ops.py
+++ b/tensorflow/python/ops/resource_variable_ops.py
@@ -768,6 +768,27 @@ class ResourceVariable(variables.Variable):
     else:
       return self.value()
 
+  def __iadd__(self, unused_other):
+    raise RuntimeError("Variable += value not supported.")
+
+  def __isub__(self, unused_other):
+    raise RuntimeError("Variable -= value not supported.")
+
+  def __imul__(self, unused_other):
+    raise RuntimeError("Variable *= value not supported.")
+
+  def __idiv__(self, unused_other):
+    raise RuntimeError("Variable /= value not supported.")
+
+  def __itruediv__(self, unused_other):
+    raise RuntimeError("Variable /= value not supported.")
+
+  def __irealdiv__(self, unused_other):
+    raise RuntimeError("Variable /= value not supported.")
+
+  def __ipow__(self, unused_other):
+    raise RuntimeError("Variable **= value not supported.")
+
 
 def _dense_var_to_tensor(var, dtype=None, name=None, as_ref=False):
   return var._dense_var_to_tensor(dtype=dtype, name=name, as_ref=as_ref)  # pylint: disable=protected-access
-- 
GitLab


From b4e09b48617ed27f528ee2b253c01a6e4698c2e3 Mon Sep 17 00:00:00 2001
From: Michael Case <mikecase@google.com>
Date: Wed, 25 Oct 2017 16:48:26 -0700
Subject: [PATCH 795/909] Internal change.

PiperOrigin-RevId: 173471032
---
 tensorflow/java/BUILD                            |  2 +-
 .../opensource_only/arm_neon_2_x86_sse.BUILD     | 16 ++++++++++++++++
 tensorflow/workspace.bzl                         | 10 ++++++++++
 3 files changed, 27 insertions(+), 1 deletion(-)
 create mode 100644 tensorflow/opensource_only/arm_neon_2_x86_sse.BUILD

diff --git a/tensorflow/java/BUILD b/tensorflow/java/BUILD
index d74cb32c5a..c0563da06d 100644
--- a/tensorflow/java/BUILD
+++ b/tensorflow/java/BUILD
@@ -289,7 +289,7 @@ tf_java_test(
 #    test_class = "org.tensorflow.processor.OperatorProcessorTest",
 #    deps = [
 #        ":processor_library",
-#        "@junit",
+#        "//third_party/java/junit",
 #        "@com_google_testing_compile",
 #        "@com_google_truth",
 #    ],
diff --git a/tensorflow/opensource_only/arm_neon_2_x86_sse.BUILD b/tensorflow/opensource_only/arm_neon_2_x86_sse.BUILD
new file mode 100644
index 0000000000..6c641a7f4e
--- /dev/null
+++ b/tensorflow/opensource_only/arm_neon_2_x86_sse.BUILD
@@ -0,0 +1,16 @@
+# Description:
+#   NEON2SSE - a header file redefining ARM Neon intrinsics in terms of SSE intrinsics
+#              allowing neon code to compile and run on x64/x86 workstantions.
+
+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])  # 3-Clause BSD
+
+exports_files([
+    "LICENSE",
+])
+
+cc_library(
+    name = "arm_neon_2_x86_sse",
+    hdrs = ["NEON_2_SSE.h"],
+)
diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index a14469a0be..f5006ad55d 100644
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -770,3 +770,13 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       sha256 = "d58bb2d6c8603f600d522b6104d6192a65339aa26cbba9f11ff5c4b36dedb928",
       strip_prefix = "bazel-toolchains-af4681c3d19f063f090222ec3d04108c4e0ca255",
   )
+
+  native.new_http_archive(
+      name = "arm_neon_2_x86_sse",
+      sha256 = "c8d90aa4357f8079d427e87a6f4c493da1fa4140aee926c05902d7ec1533d9a5",
+      strip_prefix = "ARM_NEON_2_x86_SSE-0f77d9d182265259b135dad949230ecbf1a2633d",
+      urls = [
+          "https://github.com/intel/ARM_NEON_2_x86_SSE/archive/0f77d9d182265259b135dad949230ecbf1a2633d.tar.gz",
+      ],
+      build_file = str(Label("//third_party:arm_neon_2_x86_sse.BUILD")),
+  )
-- 
GitLab


From a80b9297f330be6777a23e2e3a3b6e21097d1926 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 25 Oct 2017 16:56:06 -0700
Subject: [PATCH 796/909] Add device assignment export to graph dumpers.

PiperOrigin-RevId: 173472156
---
 tensorflow/compiler/xla/service/hlo_graph_dumper.cc    |  7 ++++++-
 tensorflow/compiler/xla/service/hlo_tfgraph_builder.cc | 10 ++++++++++
 2 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
index ed94a5be91..20ec7dfe2f 100644
--- a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
+++ b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
@@ -926,6 +926,9 @@ string HloDotDumper::GetInstructionNodeExtraInfo(const HloInstruction* instr) {
                            [](int64 stride) { return stride == 1; })
                    ? ""
                    : StrCat("stride=", VectorString(instr->slice_strides()));
+      case HloOpcode::kSend:
+      case HloOpcode::kRecv:
+        return StrCat("channel_id=", instr->channel_id());
       default:
         return "";
     }
@@ -935,7 +938,9 @@ string HloDotDumper::GetInstructionNodeExtraInfo(const HloInstruction* instr) {
   if (!opcode_specific_info.empty()) {
     lines.push_back(opcode_specific_info);
   }
-
+  if (instr->device_assignment().has_device()) {
+    lines.push_back(StrCat("device=", instr->device_assignment().device()));
+  }
   // Show the shape and layout of the instruction, unless it's an inlined fusion
   // node -- there the shape and layout is present in the output node.
   if (instr->opcode() != HloOpcode::kFusion ||
diff --git a/tensorflow/compiler/xla/service/hlo_tfgraph_builder.cc b/tensorflow/compiler/xla/service/hlo_tfgraph_builder.cc
index 3f6d89f24f..2007a8f11d 100644
--- a/tensorflow/compiler/xla/service/hlo_tfgraph_builder.cc
+++ b/tensorflow/compiler/xla/service/hlo_tfgraph_builder.cc
@@ -56,6 +56,8 @@ TensorShapeProto GetTensorShape(const HloInstruction* instruction) {
   return tensor_shape;
 }
 
+string GetDeviceName(int device) { return StrCat("/device/XLA:", device); }
+
 }  // namespace
 
 void CleanNodeName(string* name) {
@@ -178,6 +180,10 @@ void HloTfGraphBuilder::SetNodeAttrs(const HloInstruction* instruction,
     case HloOpcode::kCustomCall:
       attrs["custom_call_target"].set_s(instruction->custom_call_target());
       break;
+    case HloOpcode::kSend:
+    case HloOpcode::kRecv:
+      attrs["channel_id"].set_i(instruction->channel_id());
+      break;
     default:
       break;
   }
@@ -192,6 +198,10 @@ Status HloTfGraphBuilder::AddInstruction(const HloInstruction* instruction) {
   NodeDef* node_def = graph_def_.add_node();
   node_def->set_name(GetNodeNameForInstruction(instruction));
   node_def->set_op(GetOpDefName(instruction));
+  if (instruction->device_assignment().has_device()) {
+    node_def->set_device(
+        GetDeviceName(instruction->device_assignment().device()));
+  }
   SetNodeAttrs(instruction, node_def);
   if (instruction->opcode() == HloOpcode::kFusion) {
     for (auto* fused_instruction : instruction->fused_instructions()) {
-- 
GitLab


From 3315397770b5304f529e686c635a9436f32aa61d Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Wed, 25 Oct 2017 16:59:48 -0700
Subject: [PATCH 797/909] Support general callables in backprop.

PiperOrigin-RevId: 173472596
---
 tensorflow/python/eager/backprop.py      | 13 ++++++++++++-
 tensorflow/python/eager/backprop_test.py | 12 ++++++++++++
 2 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/eager/backprop.py b/tensorflow/python/eager/backprop.py
index bdc4ce3252..6ede02dbcd 100644
--- a/tensorflow/python/eager/backprop.py
+++ b/tensorflow/python/eager/backprop.py
@@ -400,7 +400,18 @@ def implicit_grad(f):
 
 
 def _get_arg_spec(f, params, param_args):
-  args = tf_inspect.getargspec(f).args
+  """The positions of the parameters of f to be differentiated in param_args."""
+  try:
+    args = tf_inspect.getargspec(f).args
+  except TypeError as e:
+    # TypeError can happen when f is a callable object.
+    if params is None:
+      return range(len(param_args))
+    elif all(isinstance(x, int) for x in params):
+      return params
+    raise ValueError("Either callable provided is not a function or could not "
+                     "inspect its arguments by name: %s. Original error: %s"
+                     % (f, e))
   if params is None:
     if not args:
       return range(len(param_args))
diff --git a/tensorflow/python/eager/backprop_test.py b/tensorflow/python/eager/backprop_test.py
index 628f254b18..d18df4dffb 100644
--- a/tensorflow/python/eager/backprop_test.py
+++ b/tensorflow/python/eager/backprop_test.py
@@ -16,6 +16,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import functools
+
 import numpy as np
 
 from tensorflow.python import pywrap_tensorflow
@@ -389,6 +391,16 @@ class BackpropTest(test.TestCase):
     grad = backprop.gradients_function(f)
     self.assertAllEqual(grad(1.0)[0], 2.0)
 
+  def testPartial(self):
+
+    def f(x, y):
+      return x * y
+
+    part = functools.partial(f, constant_op.constant(2.0))
+    self.assertAllEqual(
+        backprop.gradients_function(part)(constant_op.constant(1.0))[0],
+        2.0)
+
   def testExceptionSafety(self):
 
     def f(unused_x):
-- 
GitLab


From 176a743ba351ddadee3f1f168a6eb32315264e65 Mon Sep 17 00:00:00 2001
From: Sergio Guadarrama <sguada@google.com>
Date: Wed, 25 Oct 2017 17:02:37 -0700
Subject: [PATCH 798/909] Expose name and func properties of Template.

PiperOrigin-RevId: 173473015
---
 tensorflow/python/ops/template.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/tensorflow/python/ops/template.py b/tensorflow/python/ops/template.py
index fab808a167..24ef70c6f4 100644
--- a/tensorflow/python/ops/template.py
+++ b/tensorflow/python/ops/template.py
@@ -279,6 +279,16 @@ class Template(object):
         self._variables_created = True
         return result
 
+  @property
+  def name(self):
+    """Returns the name given to this Template."""
+    return self._name
+
+  @property
+  def func(self):
+    """Returns the func given to this Template."""
+    return self._func
+
   @property
   def variable_scope(self):
     """Returns the variable scope object created by this Template."""
-- 
GitLab


From ee501c6b98cfea07897637b403d40d0a28c4bc6e Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Wed, 25 Oct 2017 17:08:29 -0700
Subject: [PATCH 799/909] Proper destructuring of function arguments.

PiperOrigin-RevId: 173473904
---
 tensorflow/python/eager/function.py      | 29 ++++++++----------------
 tensorflow/python/eager/function_test.py |  8 +++++++
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py
index 5afc9d295e..b1b1de0c41 100644
--- a/tensorflow/python/eager/function.py
+++ b/tensorflow/python/eager/function.py
@@ -328,7 +328,7 @@ class _GraphModeFunction(object):
         (args + self._extra_inputs),
         backward_function)
 
-    return self._build_call_outputs(self._returns, real_outputs)
+    return self._build_call_outputs(real_outputs)
 
   def __call__(self, *args):
     """Executes the passed function in eager mode."""
@@ -371,34 +371,25 @@ class _GraphModeFunction(object):
           attrs=None,
           ctx=ctx)
 
-    return self._build_call_outputs(self._returns, result)
+    return self._build_call_outputs(result)
 
-  def _build_call_outputs(self, func_outputs, result):
+  def _build_call_outputs(self, result):
     """Maps the fdef output list to actual output structure.
 
     Args:
-      func_outputs: The outputs originally defined by the graph function. It
-        could potentially be a nested structure.
       result: Output lists defined by FunctionDef.
     Returns:
       The actual call output.
     """
     if self._func_outputs is None:
       return None
-    if isinstance(self._func_outputs, ops.Tensor):
-      return result[0]
-
-    outputs = []
-    for o in func_outputs:
-      vo = o
-      if isinstance(vo, ops.Tensor):
-        outputs.append(result[self._returns_to_fedf_outputs[id(vo)]])
-      elif type(vo) in (tuple, list):
-        outputs.append(self._build_call_outputs(o, result))
-      else:
-        outputs.append(o)
-
-    return tuple(outputs) if type(func_outputs) is tuple else outputs
+    outputs_list = nest.flatten(self._func_outputs)
+    j = 0
+    for i, o in enumerate(outputs_list):
+      if o is not None:
+        outputs_list[i] = result[j]
+        j += 1
+    return nest.pack_sequence_as(self._func_outputs, outputs_list)
 
 
 def _get_defun_inputs(args):
diff --git a/tensorflow/python/eager/function_test.py b/tensorflow/python/eager/function_test.py
index b4b704401a..243efccac4 100644
--- a/tensorflow/python/eager/function_test.py
+++ b/tensorflow/python/eager/function_test.py
@@ -99,6 +99,14 @@ class FunctionTest(test.TestCase):
 
       self.assertAllEqual(g(constant_op.constant(2.0)).eval(), 5.0)
 
+  def testDict(self):
+
+    @function.defun
+    def f(x):
+      return {'name': x + 1}
+
+    self.assertAllEqual(f(constant_op.constant(1.0))['name'], 2.0)
+
   def testTensorConversionWithDefun(self):
 
     @function.defun
-- 
GitLab


From 35ca57d39b9e368ef43302421db774e4ac3e3625 Mon Sep 17 00:00:00 2001
From: "Joshua V. Dillon" <jvdillon@google.com>
Date: Wed, 25 Oct 2017 17:15:55 -0700
Subject: [PATCH 800/909] Bugfix: Improve numerical stability of
 `tf.contrib.distributions.NegativeBinomial.log_prob`.

PiperOrigin-RevId: 173474795
---
 .../kernel_tests/negative_binomial_test.py    | 22 +++++++++++++++++++
 .../python/ops/negative_binomial.py           |  4 ++--
 2 files changed, 24 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/distributions/python/kernel_tests/negative_binomial_test.py b/tensorflow/contrib/distributions/python/kernel_tests/negative_binomial_test.py
index c1a74c6483..37edaa42cd 100644
--- a/tensorflow/contrib/distributions/python/kernel_tests/negative_binomial_test.py
+++ b/tensorflow/contrib/distributions/python/kernel_tests/negative_binomial_test.py
@@ -241,6 +241,28 @@ class NegativeBinomialTest(test.TestCase):
                             atol=0.,
                             rtol=.02)
 
+  def testLogProbOverflow(self):
+    with self.test_session() as sess:
+      logits = np.float32([20., 30., 40.])
+      total_count = np.float32(1.)
+      x = np.float32(0.)
+      nb = negative_binomial.NegativeBinomial(
+          total_count=total_count, logits=logits)
+      log_prob_ = sess.run(nb.log_prob(x))
+      self.assertAllEqual(np.ones_like(log_prob_, dtype=np.bool),
+                          np.isfinite(log_prob_))
+
+  def testLogProbUnderflow(self):
+    with self.test_session() as sess:
+      logits = np.float32([-90, -100, -110])
+      total_count = np.float32(1.)
+      x = np.float32(0.)
+      nb = negative_binomial.NegativeBinomial(
+          total_count=total_count, logits=logits)
+      log_prob_ = sess.run(nb.log_prob(x))
+      self.assertAllEqual(np.ones_like(log_prob_, dtype=np.bool),
+                          np.isfinite(log_prob_))
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/contrib/distributions/python/ops/negative_binomial.py b/tensorflow/contrib/distributions/python/ops/negative_binomial.py
index c8c396f6f8..3a58df80da 100644
--- a/tensorflow/contrib/distributions/python/ops/negative_binomial.py
+++ b/tensorflow/contrib/distributions/python/ops/negative_binomial.py
@@ -167,8 +167,8 @@ class NegativeBinomial(distribution.Distribution):
   def _log_unnormalized_prob(self, x):
     if self.validate_args:
       x = distribution_util.embed_check_nonnegative_integer_form(x)
-    return (self.total_count * math_ops.log1p(-self.probs)
-            + x * math_ops.log(self.probs))
+    return (self.total_count * math_ops.log_sigmoid(-self.logits)
+            + x * math_ops.log_sigmoid(self.logits))
 
   def _log_normalization(self, x):
     if self.validate_args:
-- 
GitLab


From eea18bd6e5e5c8d0f0c90d6cb5b06433090c5d90 Mon Sep 17 00:00:00 2001
From: Justine Tunney <jart@google.com>
Date: Wed, 25 Oct 2017 17:25:27 -0700
Subject: [PATCH 801/909] Fix wall_time in Summaries 2.o

It was previously storing microseconds in tf.Event.wall_time, which is a double
meant for UNIX timestamps.

PiperOrigin-RevId: 173475843
---
 tensorflow/core/kernels/summary_interface.cc  |  29 +++--
 tensorflow/core/kernels/summary_interface.h   |   3 +-
 .../core/kernels/summary_interface_test.cc    | 122 +++++++++++-------
 3 files changed, 93 insertions(+), 61 deletions(-)

diff --git a/tensorflow/core/kernels/summary_interface.cc b/tensorflow/core/kernels/summary_interface.cc
index e95a4c7b89..a0b9038787 100644
--- a/tensorflow/core/kernels/summary_interface.cc
+++ b/tensorflow/core/kernels/summary_interface.cc
@@ -30,7 +30,7 @@ limitations under the License.
 namespace tensorflow {
 namespace {
 template <typename T>
-Status TensorValueAt(Tensor t, int index, T* out) {
+Status TensorValueAt(Tensor t, int64 index, T* out) {
   switch (t.dtype()) {
     case DT_FLOAT:
       *out = t.flat<float>()(index);
@@ -210,20 +210,20 @@ Status NormalizeAndAddImages(const Tensor& tensor, int max_images, int h, int w,
 
 class SummaryWriterImpl : public SummaryWriterInterface {
  public:
-  SummaryWriterImpl(int max_queue, int flush_millis)
+  SummaryWriterImpl(int max_queue, int flush_millis, Env* env)
       : SummaryWriterInterface(),
         is_initialized_(false),
         max_queue_(max_queue),
-        flush_millis_(flush_millis) {}
+        flush_millis_(flush_millis),
+        env_(env) {}
 
-  Status Initialize(const string& logdir, const string& filename_suffix,
-                    Env* env) {
-    const Status is_dir = env->IsDirectory(logdir);
+  Status Initialize(const string& logdir, const string& filename_suffix) {
+    const Status is_dir = env_->IsDirectory(logdir);
     if (!is_dir.ok()) {
       if (is_dir.code() != tensorflow::error::NOT_FOUND) {
         return is_dir;
       }
-      TF_RETURN_IF_ERROR(env->CreateDir(logdir));
+      TF_RETURN_IF_ERROR(env_->CreateDir(logdir));
     }
     mutex_lock ml(mu_);
     events_writer_ =
@@ -231,7 +231,7 @@ class SummaryWriterImpl : public SummaryWriterInterface {
     if (!events_writer_->InitWithSuffix(filename_suffix)) {
       return errors::Unknown("Could not initialize events writer.");
     }
-    last_flush_ = Env::Default()->NowMicros();
+    last_flush_ = env_->NowMicros();
     is_initialized_ = true;
     return Status::OK();
   }
@@ -384,9 +384,9 @@ class SummaryWriterImpl : public SummaryWriterInterface {
  private:
   Status Enqueue(int64 global_step, const Summary& summary) {
     mutex_lock ml(mu_);
-    queue_.emplace_back(global_step, summary, Env::Default()->NowMicros());
+    queue_.emplace_back(global_step, summary, env_->NowMicros());
     if (queue_.size() >= max_queue_ ||
-        Env::Default()->NowMicros() - last_flush_ > 1000 * flush_millis_) {
+        env_->NowMicros() - last_flush_ > 1000 * flush_millis_) {
       return InternalFlush();
     }
     return Status::OK();
@@ -397,14 +397,14 @@ class SummaryWriterImpl : public SummaryWriterInterface {
       Event event;
       event.set_step(std::get<0>(e));
       *event.mutable_summary() = std::get<1>(e);
-      event.set_wall_time(std::get<2>(e));
+      event.set_wall_time(static_cast<double>(std::get<2>(e)) / 1.0e6);
       events_writer_->WriteEvent(event);
     }
     queue_.clear();
     if (!events_writer_->Flush()) {
       return errors::InvalidArgument("Could not flush events file.");
     }
-    last_flush_ = Env::Default()->NowMicros();
+    last_flush_ = env_->NowMicros();
     return Status::OK();
   }
 
@@ -412,6 +412,7 @@ class SummaryWriterImpl : public SummaryWriterInterface {
   const int max_queue_;
   const int flush_millis_;
   uint64 last_flush_;
+  Env* env_;
   using EventInfo = std::tuple<int64, Summary, int64>;
   mutex mu_;
   std::vector<EventInfo> queue_ GUARDED_BY(mu_);
@@ -424,8 +425,8 @@ class SummaryWriterImpl : public SummaryWriterInterface {
 Status CreateSummaryWriter(int max_queue, int flush_millis,
                            const string& logdir, const string& filename_suffix,
                            Env* env, SummaryWriterInterface** result) {
-  SummaryWriterImpl* w = new SummaryWriterImpl(max_queue, flush_millis);
-  const Status s = w->Initialize(logdir, filename_suffix, env);
+  SummaryWriterImpl* w = new SummaryWriterImpl(max_queue, flush_millis, env);
+  const Status s = w->Initialize(logdir, filename_suffix);
   if (!s.ok()) {
     w->Unref();
     *result = nullptr;
diff --git a/tensorflow/core/kernels/summary_interface.h b/tensorflow/core/kernels/summary_interface.h
index ae2fbb70fe..1b5d0b2748 100644
--- a/tensorflow/core/kernels/summary_interface.h
+++ b/tensorflow/core/kernels/summary_interface.h
@@ -49,7 +49,8 @@ class SummaryWriterInterface : public ResourceBase {
 // enqueue up to max_queue summaries, and flush at least every flush_millis
 // milliseconds. The summaries will be written to the directory specified by
 // logdir and with the filename suffixed by filename_suffix. The caller owns a
-// reference to result if the returned status is ok.
+// reference to result if the returned status is ok. The Env object must not
+// be destroyed until after the returned writer.
 Status CreateSummaryWriter(int max_queue, int flush_millis,
                            const string& logdir, const string& filename_suffix,
                            Env* env, SummaryWriterInterface** result);
diff --git a/tensorflow/core/kernels/summary_interface_test.cc b/tensorflow/core/kernels/summary_interface_test.cc
index 0e24e8122a..379e045ca3 100644
--- a/tensorflow/core/kernels/summary_interface_test.cc
+++ b/tensorflow/core/kernels/summary_interface_test.cc
@@ -28,52 +28,68 @@ limitations under the License.
 namespace tensorflow {
 namespace {
 
-Status SummaryTestHelper(
-    const string& test_name,
-    std::function<Status(SummaryWriterInterface*)> writer_fn,
-    std::function<void(const Event&)> test_fn) {
-  static std::set<string>* tests = new std::set<string>();
-  CHECK(tests->insert(test_name).second) << ": " << test_name;
-
-  SummaryWriterInterface* writer;
-  Env* env = Env::Default();
-  TF_CHECK_OK(
-      CreateSummaryWriter(1, 1, testing::TmpDir(), test_name, env, &writer));
-  core::ScopedUnref deleter(writer);
-
-  TF_CHECK_OK(writer_fn(writer));
-  TF_CHECK_OK(writer->Flush());
-
-  std::vector<string> files;
-  TF_CHECK_OK(env->GetChildren(testing::TmpDir(), &files));
-  bool found = false;
-  for (const string& f : files) {
-    if (StringPiece(f).contains(test_name)) {
-      if (found) {
-        return errors::Unknown("Found more than one file for ", test_name);
+class FakeClockEnv : public EnvWrapper {
+ public:
+  FakeClockEnv() : EnvWrapper(Env::Default()), current_millis_(0) {}
+  void AdvanceByMillis(const uint64 millis) { current_millis_ += millis; }
+  uint64 NowMicros() override { return current_millis_ * 1000; }
+  uint64 NowSeconds() override { return current_millis_ * 1000; }
+
+ private:
+  uint64 current_millis_;
+};
+
+class SummaryInterfaceTest : public ::testing::Test {
+ protected:
+  Status SummaryTestHelper(
+      const string& test_name,
+      std::function<Status(SummaryWriterInterface*)> writer_fn,
+      std::function<void(const Event&)> test_fn) {
+    static std::set<string>* tests = new std::set<string>();
+    CHECK(tests->insert(test_name).second) << ": " << test_name;
+
+    SummaryWriterInterface* writer;
+    TF_CHECK_OK(CreateSummaryWriter(1, 1, testing::TmpDir(), test_name, &env_,
+                                    &writer));
+    core::ScopedUnref deleter(writer);
+
+    TF_CHECK_OK(writer_fn(writer));
+    TF_CHECK_OK(writer->Flush());
+
+    std::vector<string> files;
+    TF_CHECK_OK(env_.GetChildren(testing::TmpDir(), &files));
+    bool found = false;
+    for (const string& f : files) {
+      if (StringPiece(f).contains(test_name)) {
+        if (found) {
+          return errors::Unknown("Found more than one file for ", test_name);
+        }
+        found = true;
+        std::unique_ptr<RandomAccessFile> read_file;
+        TF_CHECK_OK(env_.NewRandomAccessFile(io::JoinPath(testing::TmpDir(), f),
+                                             &read_file));
+        io::RecordReader reader(read_file.get(), io::RecordReaderOptions());
+        string record;
+        uint64 offset = 0;
+        TF_CHECK_OK(
+            reader.ReadRecord(&offset,
+                              &record));  // The first event is irrelevant
+        TF_CHECK_OK(reader.ReadRecord(&offset, &record));
+        Event e;
+        e.ParseFromString(record);
+        test_fn(e);
       }
-      found = true;
-      std::unique_ptr<RandomAccessFile> read_file;
-      TF_CHECK_OK(env->NewRandomAccessFile(io::JoinPath(testing::TmpDir(), f),
-                                           &read_file));
-      io::RecordReader reader(read_file.get(), io::RecordReaderOptions());
-      string record;
-      uint64 offset = 0;
-      TF_CHECK_OK(reader.ReadRecord(&offset,
-                                    &record));  // The first event is irrelevant
-      TF_CHECK_OK(reader.ReadRecord(&offset, &record));
-      Event e;
-      e.ParseFromString(record);
-      test_fn(e);
     }
+    if (!found) {
+      return errors::Unknown("Found no file for ", test_name);
+    }
+    return Status::OK();
   }
-  if (!found) {
-    return errors::Unknown("Found no file for ", test_name);
-  }
-  return Status::OK();
-}
 
-TEST(SummaryInterfaceTest, WriteTensor) {
+  FakeClockEnv env_;
+};
+
+TEST_F(SummaryInterfaceTest, WriteTensor) {
   TF_CHECK_OK(SummaryTestHelper("tensor_test",
                                 [](SummaryWriterInterface* writer) {
                                   Tensor one(DT_FLOAT, TensorShape({}));
@@ -91,7 +107,7 @@ TEST(SummaryInterfaceTest, WriteTensor) {
                                 }));
 }
 
-TEST(SummaryInterfaceTest, WriteScalar) {
+TEST_F(SummaryInterfaceTest, WriteScalar) {
   TF_CHECK_OK(SummaryTestHelper(
       "scalar_test",
       [](SummaryWriterInterface* writer) {
@@ -109,7 +125,7 @@ TEST(SummaryInterfaceTest, WriteScalar) {
       }));
 }
 
-TEST(SummaryInterfaceTest, WriteHistogram) {
+TEST_F(SummaryInterfaceTest, WriteHistogram) {
   TF_CHECK_OK(SummaryTestHelper("hist_test",
                                 [](SummaryWriterInterface* writer) {
                                   Tensor one(DT_FLOAT, TensorShape({}));
@@ -127,7 +143,7 @@ TEST(SummaryInterfaceTest, WriteHistogram) {
                                 }));
 }
 
-TEST(SummaryInterfaceTest, WriteImage) {
+TEST_F(SummaryInterfaceTest, WriteImage) {
   TF_CHECK_OK(SummaryTestHelper(
       "image_test",
       [](SummaryWriterInterface* writer) {
@@ -148,7 +164,7 @@ TEST(SummaryInterfaceTest, WriteImage) {
       }));
 }
 
-TEST(SummaryInterfaceTest, WriteAudio) {
+TEST_F(SummaryInterfaceTest, WriteAudio) {
   TF_CHECK_OK(SummaryTestHelper(
       "audio_test",
       [](SummaryWriterInterface* writer) {
@@ -166,5 +182,19 @@ TEST(SummaryInterfaceTest, WriteAudio) {
       }));
 }
 
+TEST_F(SummaryInterfaceTest, WallTime) {
+  env_.AdvanceByMillis(7023);
+  TF_CHECK_OK(SummaryTestHelper(
+      "wall_time_test",
+      [](SummaryWriterInterface* writer) {
+        Tensor one(DT_FLOAT, TensorShape({}));
+        one.scalar<float>()() = 1.0;
+        TF_RETURN_IF_ERROR(writer->WriteScalar(2, one, "name"));
+        TF_RETURN_IF_ERROR(writer->Flush());
+        return Status::OK();
+      },
+      [](const Event& e) { EXPECT_EQ(e.wall_time(), 7.023); }));
+}
+
 }  // namespace
 }  // namespace tensorflow
-- 
GitLab


From 7924f8bce946b367a3cdcacd91ee2a6a8b29e14e Mon Sep 17 00:00:00 2001
From: "Joshua V. Dillon" <jvdillon@google.com>
Date: Wed, 25 Oct 2017 17:32:57 -0700
Subject: [PATCH 802/909] Bugfix: Fix gradient of KL divergence of
 `tf.distributions.MultivariateNormal*`. Previous it was `nan` when the two
 distributions are identical.

PiperOrigin-RevId: 173476896
---
 .../python/kernel_tests/mvn_diag_test.py             | 12 ++++++++++++
 .../distributions/python/ops/mvn_linear_operator.py  |  6 ++++--
 2 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/distributions/python/kernel_tests/mvn_diag_test.py b/tensorflow/contrib/distributions/python/kernel_tests/mvn_diag_test.py
index 43e302475b..933756aa8e 100644
--- a/tensorflow/contrib/distributions/python/kernel_tests/mvn_diag_test.py
+++ b/tensorflow/contrib/distributions/python/kernel_tests/mvn_diag_test.py
@@ -289,6 +289,18 @@ class MultivariateNormalDiagTest(test.TestCase):
     self.assertListEqual(mvn.batch_shape.as_list(), [2, 3])
     self.assertListEqual(mvn.event_shape.as_list(), [None])
 
+  def testKLDivIdenticalGradientDefined(self):
+    dims = 3
+    with self.test_session() as sess:
+      loc = array_ops.zeros([dims], dtype=dtypes.float32)
+      mvn = ds.MultivariateNormalDiag(
+          loc=loc,
+          scale_diag=np.ones([dims], dtype=np.float32))
+      g = gradients_impl.gradients(ds.kl_divergence(mvn, mvn), loc)
+      g_ = sess.run(g)
+      self.assertAllEqual(np.ones_like(g_, dtype=np.bool),
+                          np.isfinite(g_))
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/contrib/distributions/python/ops/mvn_linear_operator.py b/tensorflow/contrib/distributions/python/ops/mvn_linear_operator.py
index 251c2dbdfa..300bdd5f60 100644
--- a/tensorflow/contrib/distributions/python/ops/mvn_linear_operator.py
+++ b/tensorflow/contrib/distributions/python/ops/mvn_linear_operator.py
@@ -22,7 +22,6 @@ from tensorflow.contrib.distributions.python.ops import distribution_util
 from tensorflow.contrib.distributions.python.ops.bijectors import AffineLinearOperator
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import linalg_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops.distributions import kullback_leibler
 from tensorflow.python.ops.distributions import normal
@@ -299,7 +298,10 @@ def _kl_brute_force(a, b, name=None):
   def squared_frobenius_norm(x):
     """Helper to make KL calculation slightly more readable."""
     # http://mathworld.wolfram.com/FrobeniusNorm.html
-    return math_ops.square(linalg_ops.norm(x, ord="fro", axis=[-2, -1]))
+    # The gradient of KL[p,q] is not defined when p==q. The culprit is
+    # linalg_ops.norm, i.e., we cannot use the commented out code.
+    # return math_ops.square(linalg_ops.norm(x, ord="fro", axis=[-2, -1]))
+    return math_ops.reduce_sum(math_ops.square(x), axis=[-2, -1])
 
   # TODO(b/35041439): See also b/35040945. Remove this function once LinOp
   # supports something like:
-- 
GitLab


From b4aac5db53bb0f09a049d71b892ace2acc93ed9c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 25 Oct 2017 17:33:34 -0700
Subject: [PATCH 803/909] Adds tests for eager execution to tf.metrics.

Raises a ValueError for tf.metrics ops if they're executed in eager mode.

PiperOrigin-RevId: 173476984
---
 tensorflow/contrib/eager/python/BUILD |   1 +
 tensorflow/python/ops/metrics_impl.py | 146 ++++++++++++++++++++++++++
 2 files changed, 147 insertions(+)

diff --git a/tensorflow/contrib/eager/python/BUILD b/tensorflow/contrib/eager/python/BUILD
index bbbf72d632..340dca7e1a 100644
--- a/tensorflow/contrib/eager/python/BUILD
+++ b/tensorflow/contrib/eager/python/BUILD
@@ -35,6 +35,7 @@ cuda_py_test(
     additional_deps = [
         ":tfe",
         "//tensorflow/python:array_ops",
+        "//tensorflow/python:metrics",
         "//tensorflow/python:math_ops",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:platform_test",
diff --git a/tensorflow/python/ops/metrics_impl.py b/tensorflow/python/ops/metrics_impl.py
index 10ff4be2dd..1858834f97 100644
--- a/tensorflow/python/ops/metrics_impl.py
+++ b/tensorflow/python/ops/metrics_impl.py
@@ -18,6 +18,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from tensorflow.python.eager import context
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import sparse_tensor
@@ -323,7 +324,12 @@ def mean(values, weights=None, metrics_collections=None,
     ValueError: If `weights` is not `None` and its shape doesn't match `values`,
       or if either `metrics_collections` or `updates_collections` are not a list
       or tuple.
+    RuntimeError: If eager execution is enabled.
   """
+  if context.in_eager_mode():
+    raise RuntimeError('tf.metrics.mean is not supported when eager execution '
+                       'is enabled.')
+
   with variable_scope.variable_scope(name, 'mean', (values, weights)):
     values = math_ops.to_float(values)
 
@@ -399,7 +405,12 @@ def accuracy(labels, predictions, weights=None, metrics_collections=None,
       `weights` is not `None` and its shape doesn't match `predictions`, or if
       either `metrics_collections` or `updates_collections` are not a list or
       tuple.
+    RuntimeError: If eager execution is enabled.
   """
+  if context.in_eager_mode():
+    raise RuntimeError('tf.metrics.accuracy is not supported when eager '
+                       'execution is enabled.')
+
   predictions, labels, weights = _remove_squeezable_dimensions(
       predictions=predictions, labels=labels, weights=weights)
   predictions.get_shape().assert_is_compatible_with(labels.get_shape())
@@ -626,7 +637,12 @@ def auc(labels, predictions, weights=None, num_thresholds=200,
       `weights` is not `None` and its shape doesn't match `predictions`, or if
       either `metrics_collections` or `updates_collections` are not a list or
       tuple.
+    RuntimeError: If eager execution is enabled.
   """
+  if context.in_eager_mode():
+    raise RuntimeError('tf.metrics.auc is not supported when eager execution '
+                       'is enabled.')
+
   with variable_scope.variable_scope(
       name, 'auc', (labels, predictions, weights)):
     if curve != 'ROC' and curve != 'PR':
@@ -732,7 +748,12 @@ def mean_absolute_error(labels, predictions, weights=None,
       `weights` is not `None` and its shape doesn't match `predictions`, or if
       either `metrics_collections` or `updates_collections` are not a list or
       tuple.
+    RuntimeError: If eager execution is enabled.
   """
+  if context.in_eager_mode():
+    raise RuntimeError('tf.metrics.mean_absolute_error is not supported '
+                       'when eager execution is enabled.')
+
   predictions, labels, weights = _remove_squeezable_dimensions(
       predictions=predictions, labels=labels, weights=weights)
   absolute_errors = math_ops.abs(predictions - labels)
@@ -783,7 +804,12 @@ def mean_cosine_distance(labels, predictions, dim, weights=None,
       `weights` is not `None` and its shape doesn't match `predictions`, or if
       either `metrics_collections` or `updates_collections` are not a list or
       tuple.
+    RuntimeError: If eager execution is enabled.
   """
+  if context.in_eager_mode():
+    raise RuntimeError('tf.metrics.mean_cosine_distance is not supported when '
+                       'eager execution is enabled.')
+
   predictions, labels, weights = _remove_squeezable_dimensions(
       predictions=predictions, labels=labels, weights=weights)
   radial_diffs = math_ops.multiply(predictions, labels)
@@ -851,7 +877,12 @@ def mean_per_class_accuracy(labels,
       `weights` is not `None` and its shape doesn't match `predictions`, or if
       either `metrics_collections` or `updates_collections` are not a list or
       tuple.
+    RuntimeError: If eager execution is enabled.
   """
+  if context.in_eager_mode():
+    raise RuntimeError('tf.metrics.mean_per_class_accuracy is not supported '
+                       'when eager execution is enabled.')
+
   with variable_scope.variable_scope(name, 'mean_accuracy',
                                      (predictions, labels, weights)):
     # Check if shape is compatible.
@@ -934,7 +965,12 @@ def mean_iou(labels,
       `weights` is not `None` and its shape doesn't match `predictions`, or if
       either `metrics_collections` or `updates_collections` are not a list or
       tuple.
+    RuntimeError: If eager execution is enabled.
   """
+  if context.in_eager_mode():
+    raise RuntimeError('tf.metrics.mean_iou is not supported when '
+                       'eager execution is enabled.')
+
   with variable_scope.variable_scope(
       name, 'mean_iou', (predictions, labels, weights)):
     # Check if shape is compatible.
@@ -1027,7 +1063,12 @@ def mean_relative_error(labels, predictions, normalizer, weights=None,
       `weights` is not `None` and its shape doesn't match `predictions`, or if
       either `metrics_collections` or `updates_collections` are not a list or
       tuple.
+    RuntimeError: If eager execution is enabled.
   """
+  if context.in_eager_mode():
+    raise RuntimeError('tf.metrics.mean_relative_error is not supported when '
+                       'eager execution is enabled.')
+
   predictions, labels, weights = _remove_squeezable_dimensions(
       predictions=predictions, labels=labels, weights=weights)
 
@@ -1087,7 +1128,12 @@ def mean_squared_error(labels, predictions, weights=None,
       `weights` is not `None` and its shape doesn't match `predictions`, or if
       either `metrics_collections` or `updates_collections` are not a list or
       tuple.
+    RuntimeError: If eager execution is enabled.
   """
+  if context.in_eager_mode():
+    raise RuntimeError('tf.metrics.mean_squared_error is not supported when '
+                       'eager execution is enabled.')
+
   predictions, labels, weights = _remove_squeezable_dimensions(
       predictions=predictions, labels=labels, weights=weights)
   squared_error = math_ops.square(labels - predictions)
@@ -1136,7 +1182,12 @@ def mean_tensor(values, weights=None, metrics_collections=None,
     ValueError: If `weights` is not `None` and its shape doesn't match `values`,
       or if either `metrics_collections` or `updates_collections` are not a list
       or tuple.
+    RuntimeError: If eager execution is enabled.
   """
+  if context.in_eager_mode():
+    raise RuntimeError('tf.metrics.mean_tensor is not supported when '
+                       'eager execution is enabled.')
+
   with variable_scope.variable_scope(name, 'mean', (values, weights)):
     values = math_ops.to_float(values)
     total = _create_local('total_tensor', shape=values.get_shape())
@@ -1213,7 +1264,12 @@ def percentage_below(values, threshold, weights=None,
     ValueError: If `weights` is not `None` and its shape doesn't match `values`,
       or if either `metrics_collections` or `updates_collections` are not a list
       or tuple.
+    RuntimeError: If eager execution is enabled.
   """
+  if context.in_eager_mode():
+    raise RuntimeError('tf.metrics.percentage_below is not supported when '
+                       'eager execution is enabled.')
+
   is_below_threshold = math_ops.to_float(math_ops.less(values, threshold))
   return mean(is_below_threshold,
               weights,
@@ -1299,7 +1355,12 @@ def false_negatives(labels, predictions, weights=None,
     ValueError: If `weights` is not `None` and its shape doesn't match `values`,
       or if either `metrics_collections` or `updates_collections` are not a list
       or tuple.
+    RuntimeError: If eager execution is enabled.
   """
+  if context.in_eager_mode():
+    raise RuntimeError('tf.metrics.false_negatives is not supported when '
+                       'eager execution is enabled.')
+
   with variable_scope.variable_scope(
       name, 'false_negatives', (predictions, labels, weights)):
 
@@ -1346,7 +1407,12 @@ def false_negatives_at_thresholds(labels, predictions, thresholds, weights=None,
       `weights` is not `None` and its shape doesn't match `predictions`, or if
       either `metrics_collections` or `updates_collections` are not a list or
       tuple.
+    RuntimeError: If eager execution is enabled.
   """
+  if context.in_eager_mode():
+    raise RuntimeError('tf.metrics.false_negatives_at_thresholds is not '
+                       'supported when eager execution is enabled.')
+
   with variable_scope.variable_scope(name, 'false_negatives',
                                      (predictions, labels, weights)):
     values, update_ops = _confusion_matrix_at_thresholds(
@@ -1392,7 +1458,12 @@ def false_positives(labels, predictions, weights=None,
       `weights` is not `None` and its shape doesn't match `predictions`, or if
       either `metrics_collections` or `updates_collections` are not a list or
       tuple.
+    RuntimeError: If eager execution is enabled.
   """
+  if context.in_eager_mode():
+    raise RuntimeError('tf.metrics.false_positives is not supported when '
+                       'eager execution is enabled.')
+
   with variable_scope.variable_scope(
       name, 'false_positives', (predictions, labels, weights)):
 
@@ -1439,7 +1510,12 @@ def false_positives_at_thresholds(labels, predictions, thresholds, weights=None,
       `weights` is not `None` and its shape doesn't match `predictions`, or if
       either `metrics_collections` or `updates_collections` are not a list or
       tuple.
+    RuntimeError: If eager execution is enabled.
   """
+  if context.in_eager_mode():
+    raise RuntimeError('tf.metrics.false_positives_at_thresholds is not '
+                       'supported when eager execution is enabled.')
+
   with variable_scope.variable_scope(name, 'false_positives',
                                      (predictions, labels, weights)):
     values, update_ops = _confusion_matrix_at_thresholds(
@@ -1487,7 +1563,12 @@ def true_negatives_at_thresholds(labels, predictions, thresholds, weights=None,
       `weights` is not `None` and its shape doesn't match `predictions`, or if
       either `metrics_collections` or `updates_collections` are not a list or
       tuple.
+    RuntimeError: If eager execution is enabled.
   """
+  if context.in_eager_mode():
+    raise RuntimeError('tf.metrics.true_negatives_at_thresholds is not '
+                       'supported when eager execution is enabled.')
+
   with variable_scope.variable_scope(name, 'true_negatives',
                                      (predictions, labels, weights)):
     values, update_ops = _confusion_matrix_at_thresholds(
@@ -1533,7 +1614,12 @@ def true_positives(labels, predictions, weights=None,
       `weights` is not `None` and its shape doesn't match `predictions`, or if
       either `metrics_collections` or `updates_collections` are not a list or
       tuple.
+    RuntimeError: If eager execution is enabled.
   """
+  if context.in_eager_mode():
+    raise RuntimeError('tf.metrics.true_positives is not '
+                       'supported when eager execution is enabled.')
+
   with variable_scope.variable_scope(
       name, 'true_positives', (predictions, labels, weights)):
 
@@ -1580,7 +1666,12 @@ def true_positives_at_thresholds(labels, predictions, thresholds, weights=None,
       `weights` is not `None` and its shape doesn't match `predictions`, or if
       either `metrics_collections` or `updates_collections` are not a list or
       tuple.
+    RuntimeError: If eager execution is enabled.
   """
+  if context.in_eager_mode():
+    raise RuntimeError('tf.metrics.true_positives_at_thresholds is not '
+                       'supported when eager execution is enabled.')
+
   with variable_scope.variable_scope(name, 'true_positives',
                                      (predictions, labels, weights)):
     values, update_ops = _confusion_matrix_at_thresholds(
@@ -1639,7 +1730,12 @@ def precision(labels, predictions, weights=None,
       `weights` is not `None` and its shape doesn't match `predictions`, or if
       either `metrics_collections` or `updates_collections` are not a list or
       tuple.
+    RuntimeError: If eager execution is enabled.
   """
+  if context.in_eager_mode():
+    raise RuntimeError('tf.metrics.precision is not '
+                       'supported when eager execution is enabled.')
+
   with variable_scope.variable_scope(
       name, 'precision', (predictions, labels, weights)):
 
@@ -1721,7 +1817,12 @@ def precision_at_thresholds(labels, predictions, thresholds,
       `weights` is not `None` and its shape doesn't match `predictions`, or if
       either `metrics_collections` or `updates_collections` are not a list or
       tuple.
+    RuntimeError: If eager execution is enabled.
   """
+  if context.in_eager_mode():
+    raise RuntimeError('tf.metrics.precision_at_thresholds is not '
+                       'supported when eager execution is enabled.')
+
   with variable_scope.variable_scope(name, 'precision_at_thresholds',
                                      (predictions, labels, weights)):
     values, update_ops = _confusion_matrix_at_thresholds(
@@ -1787,7 +1888,12 @@ def recall(labels, predictions, weights=None,
       `weights` is not `None` and its shape doesn't match `predictions`, or if
       either `metrics_collections` or `updates_collections` are not a list or
       tuple.
+    RuntimeError: If eager execution is enabled.
   """
+  if context.in_eager_mode():
+    raise RuntimeError('tf.metrics.recall is not supported is not '
+                       'supported when eager execution is enabled.')
+
   with variable_scope.variable_scope(
       name, 'recall', (predictions, labels, weights)):
     predictions, labels, weights = _remove_squeezable_dimensions(
@@ -2151,7 +2257,12 @@ def recall_at_k(labels,
     ValueError: If `weights` is not `None` and its shape doesn't match
     `predictions`, or if either `metrics_collections` or `updates_collections`
     are not a list or tuple.
+    RuntimeError: If eager execution is enabled.
   """
+  if context.in_eager_mode():
+    raise RuntimeError('tf.metrics.recall_at_k is not '
+                       'supported when eager execution is enabled.')
+
   with ops.name_scope(
       name, _at_k_name('recall', k, class_id=class_id),
       (predictions, labels, weights)) as scope:
@@ -2286,7 +2397,12 @@ def recall_at_thresholds(labels, predictions, thresholds,
       `weights` is not `None` and its shape doesn't match `predictions`, or if
       either `metrics_collections` or `updates_collections` are not a list or
       tuple.
+    RuntimeError: If eager execution is enabled.
   """
+  if context.in_eager_mode():
+    raise RuntimeError('tf.metrics.recall_at_thresholds is not '
+                       'supported when eager execution is enabled.')
+
   with variable_scope.variable_scope(name, 'recall_at_thresholds',
                                      (predictions, labels, weights)):
     values, update_ops = _confusion_matrix_at_thresholds(
@@ -2354,7 +2470,12 @@ def root_mean_squared_error(labels, predictions, weights=None,
       `weights` is not `None` and its shape doesn't match `predictions`, or if
       either `metrics_collections` or `updates_collections` are not a list or
       tuple.
+    RuntimeError: If eager execution is enabled.
   """
+  if context.in_eager_mode():
+    raise RuntimeError('tf.metrics.root_mean_squared_error is not '
+                       'supported when eager execution is enabled.')
+
   predictions, labels, weights = _remove_squeezable_dimensions(
       predictions=predictions, labels=labels, weights=weights)
   mse, update_mse_op = mean_squared_error(
@@ -2424,7 +2545,12 @@ def sensitivity_at_specificity(
       `weights` is not `None` and its shape doesn't match `predictions`, or if
       `specificity` is not between 0 and 1, or if either `metrics_collections`
       or `updates_collections` are not a list or tuple.
+    RuntimeError: If eager execution is enabled.
   """
+  if context.in_eager_mode():
+    raise RuntimeError('tf.metrics.sensitivity_at_specificity is not '
+                       'supported when eager execution is enabled.')
+
   if specificity < 0 or specificity > 1:
     raise ValueError('`specificity` must be in the range [0, 1].')
 
@@ -2789,7 +2915,12 @@ def sparse_average_precision_at_k(labels,
 
   Raises:
     ValueError: if k is invalid.
+    RuntimeError: If eager execution is enabled.
   """
+  if context.in_eager_mode():
+    raise RuntimeError('tf.metrics.sparse_average_precision_at_k is not '
+                       'supported when eager execution is enabled.')
+
   if k < 1:
     raise ValueError('Invalid k=%s.' % k)
   with ops.name_scope(
@@ -2953,7 +3084,12 @@ def precision_at_top_k(labels,
     ValueError: If `weights` is not `None` and its shape doesn't match
       `predictions`, or if either `metrics_collections` or `updates_collections`
       are not a list or tuple.
+    RuntimeError: If eager execution is enabled.
   """
+  if context.in_eager_mode():
+    raise RuntimeError('tf.metrics.precision_at_top_k is not '
+                       'supported when eager execution is enabled.')
+
   with ops.name_scope(name, _at_k_name('precision', k, class_id=class_id),
                       (predictions_idx, labels, weights)) as scope:
     labels = _maybe_expand_labels(labels, predictions_idx)
@@ -3048,7 +3184,12 @@ def sparse_precision_at_k(labels,
     ValueError: If `weights` is not `None` and its shape doesn't match
       `predictions`, or if either `metrics_collections` or `updates_collections`
       are not a list or tuple.
+    RuntimeError: If eager execution is enabled.
   """
+  if context.in_eager_mode():
+    raise RuntimeError('tf.metrics.sparse_precision_at_k is not '
+                       'supported when eager execution is enabled.')
+
   with ops.name_scope(name, _at_k_name('precision', k, class_id=class_id),
                       (predictions, labels, weights)) as scope:
     _, top_k_idx = nn.top_k(predictions, k)
@@ -3114,7 +3255,12 @@ def specificity_at_sensitivity(
       `weights` is not `None` and its shape doesn't match `predictions`, or if
       `sensitivity` is not between 0 and 1, or if either `metrics_collections`
       or `updates_collections` are not a list or tuple.
+    RuntimeError: If eager execution is enabled.
   """
+  if context.in_eager_mode():
+    raise RuntimeError('tf.metrics.specificity_at_sensitivity is not '
+                       'supported when eager execution is enabled.')
+
   if sensitivity < 0 or sensitivity > 1:
     raise ValueError('`sensitivity` must be in the range [0, 1].')
 
-- 
GitLab


From 5965a76ea72e266fba9b78adc94ec4ee71029ece Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Wed, 25 Oct 2017 17:44:18 -0700
Subject: [PATCH 804/909] [XLA] De-emphasize uninteresting nodes in the HLO
 graph dump.

The hope is that this will make expensive / interesting ops easier to
see.

PiperOrigin-RevId: 173478095
---
 .../compiler/xla/service/hlo_graph_dumper.cc  | 38 +++++++++++++++----
 1 file changed, 31 insertions(+), 7 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
index 20ec7dfe2f..b11b129c14 100644
--- a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
+++ b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
@@ -231,9 +231,9 @@ string HtmlLikeStringSanitize(tensorflow::StringPiece s) {
 // commutative, we also support them with param0 and param1 swapped.
 //
 // This is useful primarily for reduce and map nodes.  These take a
-// subcomputation which is almost always one of the four above, and pattern
-// matching it to a short string lets us tell the user what the subcomputation
-// is without drawing it as a graph.
+// subcomputation which is almost always one of the above, and pattern matching
+// it to a short string lets us tell the user what the subcomputation is without
+// drawing it as a graph.
 optional<string> MatchTrivialComputation(const HloComputation* computation) {
   if (computation->instruction_count() != 3) {
     return nullopt;
@@ -788,7 +788,6 @@ ColorScheme HloDotDumper::GetInstructionColor(const HloInstruction* instr) {
     case HloOpcode::kNegate:
     case HloOpcode::kPower:
     case HloOpcode::kRemainder:
-    case HloOpcode::kSelect:
     case HloOpcode::kShiftLeft:
     case HloOpcode::kShiftRightArithmetic:
     case HloOpcode::kShiftRightLogical:
@@ -799,21 +798,46 @@ ColorScheme HloDotDumper::GetInstructionColor(const HloInstruction* instr) {
     case HloOpcode::kSubtract:
     case HloOpcode::kTanh:
     case HloOpcode::kRng:
-    case HloOpcode::kBroadcast:
-    case HloOpcode::kTranspose:
+      // De-emphasize scalar-shaped elementwise ops -- they're generally
+      // uninteresting.
+      if (ShapeUtil::IsEffectiveScalar(instr->shape())) {
+        return kWhite;
+      }
       return kYellow;
     case HloOpcode::kBitcast:
     case HloOpcode::kTuple:
     case HloOpcode::kTrace:
     case HloOpcode::kGetTupleElement:
       return kWhite;
+    case HloOpcode::kBroadcast:
+      // De-emphasize nodes which broadcast a scalar within a fusion node --
+      // these are essentially free.
+      if (instr->IsFused() &&
+          ShapeUtil::IsEffectiveScalar(instr->operand(0)->shape())) {
+        return kWhite;
+      }
+      return kGreen;
     case HloOpcode::kConcatenate:
     case HloOpcode::kCopy:
     case HloOpcode::kDynamicSlice:
-    case HloOpcode::kDynamicUpdateSlice:
     case HloOpcode::kPad:
     case HloOpcode::kReshape:
     case HloOpcode::kReverse:
+    case HloOpcode::kSelect:
+    case HloOpcode::kTranspose:
+      // De-emphasize scalar-shaped data movement ops and all data movement ops
+      // inside fusion nodes, both of which are essentially free.
+      if (ShapeUtil::IsEffectiveScalar(instr->shape()) || instr->IsFused()) {
+        return kWhite;
+      }
+      return kGreen;
+    case HloOpcode::kDynamicUpdateSlice:
+      // Unlike the data-movement ops above, dynamic-update-slice is not ~free
+      // inside of fusion nodes, so we de-emphasize it only if it's
+      // scalar-shaped.
+      if (ShapeUtil::IsEffectiveScalar(instr->shape())) {
+        return kWhite;
+      }
       return kGreen;
     case HloOpcode::kConvolution:
     case HloOpcode::kDot:
-- 
GitLab


From 03e2af6a819c5b45102c0f4b2a1a5f1f01c1a43e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 25 Oct 2017 17:44:33 -0700
Subject: [PATCH 805/909] Update comment on xla::HloDCE to reflect that it does
 remove dead computations.

PiperOrigin-RevId: 173478128
---
 tensorflow/compiler/xla/service/hlo_dce.h | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_dce.h b/tensorflow/compiler/xla/service/hlo_dce.h
index fca3fa0f58..4e244494d6 100644
--- a/tensorflow/compiler/xla/service/hlo_dce.h
+++ b/tensorflow/compiler/xla/service/hlo_dce.h
@@ -24,10 +24,15 @@ limitations under the License.
 
 namespace xla {
 
-// HLO pass which removes all dead instructions from each computation in the
-// module. An instruction is dead if it is not reachable from the root. This
-// pass does not remove dead parameter instructions as parameter instructions
-// cannot be deleted, nor does the pass remove dead computations.
+// HLO pass which removes dead instructions from each computation in the module
+// and removes dead computations from the module.
+//
+// An instruction is dead if it is not reachable from the root. A computation is
+// dead if it is not the entry computation of the module and it is not reachable
+// from the entry computation.
+//
+// This pass does not remove dead parameter instructions, as parameter
+// instructions cannot be deleted.
 class HloDCE : public HloPassInterface {
  public:
   ~HloDCE() override {}
-- 
GitLab


From 6149fecbdba96fea5460915cf2fad5ac163de091 Mon Sep 17 00:00:00 2001
From: Michael Case <mikecase@google.com>
Date: Wed, 25 Oct 2017 17:45:38 -0700
Subject: [PATCH 806/909] Internal Change

PiperOrigin-RevId: 173478239
---
 tensorflow/workspace.bzl                  |  10 ++
 third_party/flatbuffers/BUILD             |  15 ++
 third_party/flatbuffers/build_defs.bzl    | 196 ++++++++++++++++++++++
 third_party/flatbuffers/flatbuffers.BUILD | 127 ++++++++++++++
 4 files changed, 348 insertions(+)
 create mode 100644 third_party/flatbuffers/BUILD
 create mode 100644 third_party/flatbuffers/build_defs.bzl
 create mode 100644 third_party/flatbuffers/flatbuffers.BUILD

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index f5006ad55d..b9651a92f7 100644
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -780,3 +780,13 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       build_file = str(Label("//third_party:arm_neon_2_x86_sse.BUILD")),
   )
+
+  native.new_http_archive(
+      name = "flatbuffers",
+      build_file = "third_party/flatbuffers/flatbuffers.BUILD",
+      strip_prefix = "flatbuffers-971a68110e4fc1bace10fcb6deeb189e7e1a34ce",
+      sha256 = "874088d2ee0d9f8524191f77209556415f03dd44e156276edf19e5b90ceb5f55",
+      urls = [
+          "https://github.com/google/flatbuffers/archive/971a68110e4fc1bace10fcb6deeb189e7e1a34ce.tar.gz",
+      ],
+  )
diff --git a/third_party/flatbuffers/BUILD b/third_party/flatbuffers/BUILD
new file mode 100644
index 0000000000..fbdf19f205
--- /dev/null
+++ b/third_party/flatbuffers/BUILD
@@ -0,0 +1,15 @@
+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])  # Apache 2.0
+
+filegroup(
+    name = "all_files",
+    srcs = glob(
+        ["**/*"],
+        exclude = [
+            "**/METADATA",
+            "**/OWNERS",
+        ],
+    ),
+    visibility = ["//tensorflow:__subpackages__"],
+)
diff --git a/third_party/flatbuffers/build_defs.bzl b/third_party/flatbuffers/build_defs.bzl
new file mode 100644
index 0000000000..ae8d7feebe
--- /dev/null
+++ b/third_party/flatbuffers/build_defs.bzl
@@ -0,0 +1,196 @@
+# Description:
+#   BUILD rules for generating flatbuffer files.
+
+flatc_path = "@flatbuffers//:flatc"
+
+DEFAULT_FLATC_ARGS = [
+    "--no-union-value-namespacing",
+    "--gen-object-api",
+]
+
+def flatbuffer_library_public(name,
+                              srcs,
+                              outs,
+                              language_flag,
+                              out_prefix="",
+                              includes=[],
+                              include_paths=[],
+                              flatc_args=DEFAULT_FLATC_ARGS,
+                              reflection_name="",
+                              reflection_visiblity=None,
+                              output_to_bindir=False):
+  '''Generates code files for reading/writing the given flatbuffers in the requested language using the public compiler.
+
+  Args:
+    name: Rule name.
+    srcs: Source .fbs files. Sent in order to the compiler.
+    outs: Output files from flatc.
+    language_flag: Target language flag. One of [-c, -j, -js].
+    out_prefix: Prepend this path to the front of all generated files except on
+        single source targets. Usually is a directory name.
+    includes: Optional, list of filegroups of schemas that the srcs depend on.
+    include_paths: Optional, list of paths the includes files can be found in.
+    flatc_args: Optional, list of additional arguments to pass to flatc.
+    reflection_name: Optional, if set this will generate the flatbuffer
+      reflection binaries for the schemas.
+    reflection_visiblity: The visibility of the generated reflection Fileset.
+    output_to_bindir: Passed to genrule for output to bin directory.
+  Outs:
+    filegroup(name): all generated source files.
+    Fileset([reflection_name]): (Optional) all generated reflection binaries.
+  '''
+  include_paths_cmd = ["-I %s" % (s) for s in include_paths]
+  # '$(@D)' when given a single source target will give the appropriate
+  # directory. Appending 'out_prefix' is only necessary when given a build
+  # target with multiple sources.
+  output_directory = (
+      ("-o $(@D)/%s" % (out_prefix)) if len(srcs) > 1 else ("-o $(@D)"))
+  genrule_cmd = " ".join([
+      "for f in $(SRCS); do",
+      "$(location %s)" % (flatc_path),
+      " ".join(flatc_args),
+      " ".join(include_paths_cmd),
+      language_flag,
+      output_directory,
+      "$$f;",
+      "done",
+  ])
+  native.genrule(
+      name=name,
+      srcs=srcs,
+      outs=outs,
+      output_to_bindir=output_to_bindir,
+      tools=includes + [flatc_path,],
+      cmd=genrule_cmd,
+      message="Generating flatbuffer files for %s:" % (name),)
+  if reflection_name:
+    reflection_genrule_cmd = " ".join([
+        "for f in $(SRCS); do",
+        "$(location %s)" % (flatc_path),
+        "-b --schema",
+        " ".join(flatc_args),
+        " ".join(include_paths_cmd),
+        language_flag,
+        output_directory,
+        "$$f;",
+        "done",
+    ])
+    reflection_outs = [
+        (out_prefix + "%s.bfbs") % (s.replace(".fbs", "").split("/")[-1]) for s in srcs
+    ]
+    native.genrule(
+        name= "%s_srcs" % reflection_name,
+        srcs=srcs,
+        outs=reflection_outs,
+        output_to_bindir=output_to_bindir,
+        tools=includes + [flatc_path,],
+        cmd=reflection_genrule_cmd,
+        message="Generating flatbuffer reflection binary for %s:" % (name),)
+    native.Fileset(
+        name=reflection_name,
+        out="%s_out" % reflection_name,
+        entries=[
+            native.FilesetEntry(files=reflection_outs),
+        ],
+        visibility=reflection_visiblity
+    )
+
+
+def flatbuffer_cc_library(name, srcs, srcs_filegroup_name="",
+                          out_prefix="", includes=[], include_paths=[],
+                          flatc_args=DEFAULT_FLATC_ARGS,
+                          visibility=None, srcs_filegroup_visibility=None,
+                          gen_reflections=False):
+  '''A cc_library with the generated reader/writers for the given flatbuffer definitions.
+
+  Args:
+    name: Rule name.
+    srcs: Source .fbs files. Sent in order to the compiler.
+    srcs_filegroup_name: Name of the output filegroup that holds srcs. Pass this
+        filegroup into the `includes` parameter of any other
+        flatbuffer_cc_library that depends on this one's schemas.
+    out_prefix: Prepend this path to the front of all generated files. Usually
+        is a directory name.
+    includes: Optional, list of filegroups of schemas that the srcs depend on.
+        ** SEE REMARKS BELOW **
+    include_paths: Optional, list of paths the includes files can be found in.
+    flatc_args: Optional list of additional arguments to pass to flatc
+        (e.g. --gen-mutable).
+    visibility: The visibility of the generated cc_library. By default, use the
+        default visibility of the project.
+    srcs_filegroup_visibility: The visibility of the generated srcs filegroup.
+        By default, use the value of the visibility parameter above.
+    gen_reflections: Optional, if true this will generate the flatbuffer
+      reflection binaries for the schemas.
+  Outs:
+    filegroup([name]_srcs): all generated .h files.
+    filegroup(srcs_filegroup_name if specified, or [name]_includes if not):
+        Other flatbuffer_cc_library's can pass this in for their `includes`
+        parameter, if they depend on the schemas in this library.
+    Fileset([name]_reflection): (Optional) all generated reflection binaries.
+    cc_library([name]): library with sources and flatbuffers deps.
+
+  Remarks:
+    ** Because the genrule used to call flatc does not have any trivial way of
+      computing the output list of files transitively generated by includes and
+      --gen-includes (the default) being defined for flatc, the --gen-includes
+      flag will not work as expected. The way around this is to add a dependency
+      to the flatbuffer_cc_library defined alongside the flatc included Fileset.
+      For example you might define:
+
+      flatbuffer_cc_library(
+          name = "my_fbs",
+          srcs = [ "schemas/foo.fbs" ],
+          includes = [ "//third_party/bazz:bazz_fbs_includes" ],
+      )
+
+      In which foo.fbs includes a few files from the Fileset defined at
+      //third_party/bazz:bazz_fbs_includes. When compiling the library that
+      includes foo_generated.h, and therefore has my_fbs as a dependency, it
+      will fail to find any of the bazz *_generated.h files unless you also
+      add bazz's flatbuffer_cc_library to your own dependency list, e.g.:
+
+      cc_library(
+          name = "my_lib",
+          deps = [
+              ":my_fbs",
+              "//third_party/bazz:bazz_fbs"
+          ],
+      )
+
+      Happy dependent Flatbuffering!
+  '''
+  output_headers = [
+      (out_prefix + "%s_generated.h") % (s.replace(".fbs", "").split("/")[-1]) for s in srcs
+  ]
+  reflection_name = "%s_reflection" % name if gen_reflections else ""
+
+  flatbuffer_library_public(name="%s_srcs" % (name),
+                            srcs=srcs,
+                            outs=output_headers,
+                            language_flag="-c",
+                            out_prefix=out_prefix,
+                            includes=includes,
+                            include_paths=include_paths,
+                            flatc_args=flatc_args,
+                            reflection_name=reflection_name,
+                            reflection_visiblity=visibility,)
+  native.cc_library(name=name,
+                    hdrs=output_headers,
+                    srcs=output_headers,
+                    features=[
+                        "-parse_headers",
+                    ],
+                    deps=[
+                        "@flatbuffers//:runtime_cc",
+                    ],
+                    includes=["."],
+                    linkstatic=1,
+                    visibility=visibility)
+
+  # A filegroup for the `srcs`. That is, all the schema files for this
+  # Flatbuffer set.
+  native.filegroup(
+      name = srcs_filegroup_name if srcs_filegroup_name else "%s_includes" % (name),
+      srcs = srcs,
+      visibility=srcs_filegroup_visibility if srcs_filegroup_visibility != None else visibility)
diff --git a/third_party/flatbuffers/flatbuffers.BUILD b/third_party/flatbuffers/flatbuffers.BUILD
new file mode 100644
index 0000000000..a426db0c50
--- /dev/null
+++ b/third_party/flatbuffers/flatbuffers.BUILD
@@ -0,0 +1,127 @@
+package(
+    default_visibility = ["//visibility:public"],
+)
+
+licenses(["notice"])  # Apache 2.0
+
+FLATBUFFERS_COPTS = [
+    "-fexceptions",
+    "-Wno-implicit-fallthrough",
+]
+
+# Public flatc library to compile flatbuffer files at runtime.
+cc_library(
+    name = "flatbuffers",
+    srcs = [
+        "include/flatbuffers/code_generators.h",
+        "include/flatbuffers/reflection_generated.h",
+        "src/code_generators.cpp",
+        "src/idl_gen_fbs.cpp",
+        "src/idl_gen_general.cpp",
+        "src/idl_gen_text.cpp",
+        "src/idl_parser.cpp",
+        "src/reflection.cpp",
+        "src/util.cpp",
+    ],
+    hdrs = [
+        "include/flatbuffers/base.h",
+        "include/flatbuffers/flatbuffers.h",
+        "include/flatbuffers/flexbuffers.h",
+        "include/flatbuffers/hash.h",
+        "include/flatbuffers/idl.h",
+        "include/flatbuffers/reflection.h",
+        "include/flatbuffers/stl_emulation.h",
+        "include/flatbuffers/util.h",
+    ],
+    copts = FLATBUFFERS_COPTS,
+    includes = ["include/"],
+)
+
+# Public flatc compiler library.
+cc_library(
+    name = "flatc_library",
+    srcs = [
+        "grpc/src/compiler/config.h",
+        "grpc/src/compiler/go_generator.h",
+        "grpc/src/compiler/schema_interface.h",
+        "include/flatbuffers/base.h",
+        "include/flatbuffers/code_generators.h",
+        "include/flatbuffers/flatbuffers.h",
+        "include/flatbuffers/flatc.h",
+        "include/flatbuffers/flexbuffers.h",
+        "include/flatbuffers/hash.h",
+        "include/flatbuffers/idl.h",
+        "include/flatbuffers/reflection.h",
+        "include/flatbuffers/reflection_generated.h",
+        "include/flatbuffers/stl_emulation.h",
+        "include/flatbuffers/util.h",
+        "src/code_generators.cpp",
+        "src/flatc.cpp",
+        "src/idl_gen_fbs.cpp",
+        "src/idl_parser.cpp",
+        "src/reflection.cpp",
+        "src/util.cpp",
+    ],
+    hdrs = [
+        "include/flatbuffers/base.h",
+        "include/flatbuffers/code_generators.h",
+        "include/flatbuffers/flatbuffers.h",
+        "include/flatbuffers/flatc.h",
+        "include/flatbuffers/idl.h",
+        "include/flatbuffers/reflection.h",
+        "include/flatbuffers/stl_emulation.h",
+        "include/flatbuffers/util.h",
+    ],
+    copts = FLATBUFFERS_COPTS,
+    includes = [
+        "grpc/",
+        "include/",
+    ],
+)
+
+# Public flatc compiler.
+cc_binary(
+    name = "flatc",
+    srcs = [
+        "grpc/src/compiler/cpp_generator.cc",
+        "grpc/src/compiler/cpp_generator.h",
+        "grpc/src/compiler/go_generator.cc",
+        "grpc/src/compiler/go_generator.h",
+        "grpc/src/compiler/schema_interface.h",
+        "src/flatc_main.cpp",
+        "src/idl_gen_cpp.cpp",
+        "src/idl_gen_general.cpp",
+        "src/idl_gen_go.cpp",
+        "src/idl_gen_grpc.cpp",
+        "src/idl_gen_js.cpp",
+        "src/idl_gen_json_schema.cpp",
+        "src/idl_gen_php.cpp",
+        "src/idl_gen_python.cpp",
+        "src/idl_gen_text.cpp",
+    ],
+    copts = FLATBUFFERS_COPTS,
+    includes = [
+        "grpc/",
+        "include/",
+    ],
+    deps = [
+        ":flatc_library",
+    ],
+)
+
+filegroup(
+    name = "runtime_cc_srcs",
+    srcs = [
+        "include/flatbuffers/base.h",
+        "include/flatbuffers/flatbuffers.h",
+        "include/flatbuffers/stl_emulation.h",
+        "include/flatbuffers/util.h",
+    ],
+)
+
+cc_library(
+    name = "runtime_cc",
+    hdrs = ["runtime_cc_srcs"],
+    includes = ["include"],
+    linkstatic = 1,
+)
-- 
GitLab


From ff7b9a6c496823c1bffdd0d74bf68aafacb8caca Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Wed, 25 Oct 2017 18:46:35 -0700
Subject: [PATCH 807/909] Adding summaries to the resnet example.

Also utilities to use summaries in graph mode.

PiperOrigin-RevId: 173483424
---
 tensorflow/contrib/BUILD                      |  2 +-
 tensorflow/contrib/__init__.py                |  1 +
 tensorflow/contrib/cmake/tf_core_ops.cmake    |  1 +
 tensorflow/contrib/cmake/tf_python.cmake      |  3 +
 tensorflow/contrib/summary/BUILD              | 25 ++++++
 tensorflow/contrib/summary/summary.py         | 39 +++++++++
 tensorflow/contrib/summary/summary_ops.py     | 82 ++++++++++++++-----
 .../contrib/summary/summary_ops_test.py       | 29 ++-----
 .../contrib/summary/summary_test_util.py      | 41 ++++++++++
 tensorflow/tools/pip_package/BUILD            |  1 +
 10 files changed, 182 insertions(+), 42 deletions(-)
 create mode 100644 tensorflow/contrib/summary/summary.py
 create mode 100644 tensorflow/contrib/summary/summary_test_util.py

diff --git a/tensorflow/contrib/BUILD b/tensorflow/contrib/BUILD
index 3d580fae14..ee3dd5079e 100644
--- a/tensorflow/contrib/BUILD
+++ b/tensorflow/contrib/BUILD
@@ -80,7 +80,7 @@ py_library(
         "//tensorflow/contrib/staging",
         "//tensorflow/contrib/stat_summarizer:stat_summarizer_py",
         "//tensorflow/contrib/stateless",
-        "//tensorflow/contrib/summary:summary_ops",
+        "//tensorflow/contrib/summary:summary",
         "//tensorflow/contrib/tensor_forest:init_py",
         "//tensorflow/contrib/tensorboard",
         "//tensorflow/contrib/testing:testing_py",
diff --git a/tensorflow/contrib/__init__.py b/tensorflow/contrib/__init__.py
index bf921808aa..76a629663d 100644
--- a/tensorflow/contrib/__init__.py
+++ b/tensorflow/contrib/__init__.py
@@ -80,6 +80,7 @@ from tensorflow.contrib import util
 from tensorflow.contrib.ndlstm import python as ndlstm
 from tensorflow.contrib.remote_fused_graph import pylib as remote_fused_graph
 from tensorflow.contrib.specs import python as specs
+from tensorflow.contrib.summary import summary
 
 from tensorflow.python.util.lazy_loader import LazyLoader
 ffmpeg = LazyLoader("ffmpeg",
diff --git a/tensorflow/contrib/cmake/tf_core_ops.cmake b/tensorflow/contrib/cmake/tf_core_ops.cmake
index dc9973917e..97bec81e66 100644
--- a/tensorflow/contrib/cmake/tf_core_ops.cmake
+++ b/tensorflow/contrib/cmake/tf_core_ops.cmake
@@ -43,6 +43,7 @@ set(tf_op_lib_names
     "state_ops"
     "stateless_random_ops"
     "string_ops"
+		"summary_ops"
     "training_ops"
 )
 
diff --git a/tensorflow/contrib/cmake/tf_python.cmake b/tensorflow/contrib/cmake/tf_python.cmake
index 8ddfb59595..a3ed19977f 100755
--- a/tensorflow/contrib/cmake/tf_python.cmake
+++ b/tensorflow/contrib/cmake/tf_python.cmake
@@ -640,6 +640,7 @@ add_python_module("tensorflow/contrib/reduce_slice_ops/ops")
 add_python_module("tensorflow/contrib/reduce_slice_ops/python")
 add_python_module("tensorflow/contrib/reduce_slice_ops/python/kernel_tests")
 add_python_module("tensorflow/contrib/reduce_slice_ops/python/ops")
+add_python_module("tensorflow/contrib/summary")
 
 # Generate the tensorflow.python.platform.build_info module.
 set(BUILD_INFO_PY "${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/python/platform/build_info.py")
@@ -812,6 +813,8 @@ GENERATE_PYTHON_OP_LIB("stateless_random_ops"
   DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/stateless/gen_stateless_random_ops.py)
 GENERATE_PYTHON_OP_LIB("debug_ops"
   DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/python/debug/ops/gen_debug_ops.py)
+GENERATE_PYTHON_OP_LIB("summary_ops"
+  DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/summary/gen_summary_ops.py)
 
 add_custom_target(tf_python_ops SOURCES ${tf_python_ops_generated_files} ${PYTHON_PROTO_GENFILES})
 add_dependencies(tf_python_ops tf_python_op_gen_main)
diff --git a/tensorflow/contrib/summary/BUILD b/tensorflow/contrib/summary/BUILD
index bcb2d74b4a..8cb5c3f381 100644
--- a/tensorflow/contrib/summary/BUILD
+++ b/tensorflow/contrib/summary/BUILD
@@ -25,6 +25,7 @@ py_test(
     srcs_version = "PY2AND3",
     deps = [
         ":summary_ops",
+        ":summary_test_util",
         "//tensorflow/core:protos_all_py",
         "//tensorflow/python:framework_test_lib",
         "//tensorflow/python:lib",
@@ -52,6 +53,16 @@ py_library(
     ],
 )
 
+py_library(
+    name = "summary",
+    srcs = ["summary.py"],
+    srcs_version = "PY2AND3",
+    visibility = ["//tensorflow:internal"],
+    deps = [
+        ":summary_ops",
+    ],
+)
+
 filegroup(
     name = "all_files",
     srcs = glob(
@@ -63,3 +74,17 @@ filegroup(
     ),
     visibility = ["//tensorflow:__subpackages__"],
 )
+
+# NOTE: target cannot be testonly because it needs to be in the pip
+# package. Sigh.
+py_library(
+    name = "summary_test_util",
+    srcs = ["summary_test_util.py"],
+    srcs_version = "PY2AND3",
+    visibility = ["//tensorflow:internal"],
+    deps = [
+        "//tensorflow/core:protos_all_py",
+        "//tensorflow/python:lib",
+        "//tensorflow/python:platform",
+    ],
+)
diff --git a/tensorflow/contrib/summary/summary.py b/tensorflow/contrib/summary/summary.py
new file mode 100644
index 0000000000..89031caadc
--- /dev/null
+++ b/tensorflow/contrib/summary/summary.py
@@ -0,0 +1,39 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""Contrib summary package.
+
+The operations in this package are safe to use with eager execution turned or on
+off.
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+# pylint: disable=unused-import
+from tensorflow.contrib.summary.summary_ops import all_summary_ops
+from tensorflow.contrib.summary.summary_ops import always_record_summaries
+from tensorflow.contrib.summary.summary_ops import audio
+from tensorflow.contrib.summary.summary_ops import create_summary_file_writer
+from tensorflow.contrib.summary.summary_ops import generic
+from tensorflow.contrib.summary.summary_ops import histogram
+from tensorflow.contrib.summary.summary_ops import image
+from tensorflow.contrib.summary.summary_ops import never_record_summaries
+from tensorflow.contrib.summary.summary_ops import record_summaries_every_n_global_steps
+from tensorflow.contrib.summary.summary_ops import scalar
+from tensorflow.contrib.summary.summary_ops import should_record_summaries
+from tensorflow.contrib.summary.summary_ops import summary_writer_initializer_op
diff --git a/tensorflow/contrib/summary/summary_ops.py b/tensorflow/contrib/summary/summary_ops.py
index 30a9398ee5..b32b093675 100644
--- a/tensorflow/contrib/summary/summary_ops.py
+++ b/tensorflow/contrib/summary/summary_ops.py
@@ -25,6 +25,8 @@ from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.layers import utils
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.ops import summary_op_util
 from tensorflow.python.training import training_util
 from tensorflow.python.util import tf_contextlib
@@ -33,6 +35,9 @@ from tensorflow.python.util import tf_contextlib
 # Tensor. If this tensor is True the summary ops will record summaries.
 _SHOULD_RECORD_SUMMARIES_NAME = "ShouldRecordSummaries"
 
+_SUMMARY_COLLECTION_NAME = "_SUMMARY_V2"
+_SUMMARY_WRITER_INIT_COLLECTION_NAME = "_SUMMARY_WRITER_V2"
+
 
 def should_record_summaries():
   """Returns boolean Tensor which is true if summaries should be recorded."""
@@ -78,10 +83,15 @@ def never_record_summaries():
 
 
 class SummaryWriter(object):
+  """Encapsulates a summary writer."""
 
   def __init__(self, resource):
     self._resource = resource
 
+  def __del__(self):
+    if context.in_eager_mode():
+      resource_variable_ops.destroy_resource_op(self._resource)
+
   def set_as_default(self):
     context.context().summary_writer_resource = self._resource
 
@@ -90,6 +100,9 @@ class SummaryWriter(object):
     old = context.context().summary_writer_resource
     context.context().summary_writer_resource = self._resource
     yield
+    # Flushes the summary writer in eager mode or in graph functions, but not in
+    # legacy graph mode (you're on your own there).
+    gen_summary_ops.flush_summary_writer(self._resource)
     context.context().summary_writer_resource = old
 
 
@@ -108,14 +121,33 @@ def create_summary_file_writer(logdir,
   resource = gen_summary_ops.summary_writer(shared_name=name)
   # TODO(apassos) ensure the initialization op runs when in graph mode; consider
   # calling session.run here.
-  gen_summary_ops.create_summary_file_writer(resource, logdir, max_queue,
-                                             flush_secs, filename_suffix)
+  ops.add_to_collection(
+      _SUMMARY_WRITER_INIT_COLLECTION_NAME,
+      gen_summary_ops.create_summary_file_writer(resource, logdir, max_queue,
+                                                 flush_secs, filename_suffix))
   return SummaryWriter(resource)
 
 
 def _nothing():
   """Convenient else branch for when summaries do not record."""
-  return False
+  return constant_op.constant(False)
+
+
+def all_summary_ops():
+  """Graph-mode only. Returns all summary ops."""
+  if context.in_eager_mode():
+    raise RuntimeError(
+        "tf.contrib.summary.all_summary_ops is only supported in graph mode.")
+  return ops.get_collection(_SUMMARY_COLLECTION_NAME)
+
+
+def summary_writer_initializer_op():
+  """Graph-mode only. Returns the list of ops to create all summary writers."""
+  if context.in_eager_mode():
+    raise RuntimeError(
+        "tf.contrib.summary.summary_writer_initializer_op is only "
+        "supported in graph mode.")
+  return ops.get_collection(_SUMMARY_WRITER_INIT_COLLECTION_NAME)
 
 
 def summary_writer_function(name, tensor, function, family=None):
@@ -133,20 +165,25 @@ def summary_writer_function(name, tensor, function, family=None):
   def record():
     with summary_op_util.summary_scope(
         name, family, values=[tensor]) as (tag, scope):
-      function(tag, scope)
-      return True
+      with ops.control_dependencies([function(tag, scope)]):
+        return constant_op.constant(True)
 
-  return utils.smart_cond(
-      should_record_summaries(), record, _nothing, name="")
+  with ops.device("cpu:0"):
+    op = utils.smart_cond(
+        should_record_summaries(), record, _nothing, name="")
+    ops.add_to_collection(_SUMMARY_COLLECTION_NAME, op)
+  return op
 
 
 def generic(name, tensor, metadata, family=None):
   """Writes a tensor summary if possible."""
 
   def function(tag, scope):
-    gen_summary_ops.write_summary(context.context().summary_writer_resource,
-                                  training_util.get_global_step(), tensor,
-                                  tag, metadata, name=scope)
+    # Note the identity to move the tensor to the CPU.
+    return gen_summary_ops.write_summary(
+        context.context().summary_writer_resource,
+        training_util.get_global_step(), array_ops.identity(tensor),
+        tag, metadata, name=scope)
   return summary_writer_function(name, tensor, function, family=family)
 
 
@@ -154,9 +191,11 @@ def scalar(name, tensor, family=None):
   """Writes a scalar summary if possible."""
 
   def function(tag, scope):
-    gen_summary_ops.write_scalar_summary(
+    # Note the identity to move the tensor to the CPU.
+    return gen_summary_ops.write_scalar_summary(
         context.context().summary_writer_resource,
-        training_util.get_global_step(), tag, tensor, name=scope)
+        training_util.get_global_step(), tag, array_ops.identity(tensor),
+        name=scope)
 
   return summary_writer_function(name, tensor, function, family=family)
 
@@ -165,9 +204,11 @@ def histogram(name, tensor, family=None):
   """Writes a histogram summary if possible."""
 
   def function(tag, scope):
-    gen_summary_ops.write_histogram_summary(
+    # Note the identity to move the tensor to the CPU.
+    return gen_summary_ops.write_histogram_summary(
         context.context().summary_writer_resource,
-        training_util.get_global_step(), tag, tensor, name=scope)
+        training_util.get_global_step(), tag, array_ops.identity(tensor),
+        name=scope)
 
   return summary_writer_function(name, tensor, function, family=family)
 
@@ -178,10 +219,12 @@ def image(name, tensor, bad_color=None, max_images=3, family=None):
   def function(tag, scope):
     if bad_color is None:
       bad_color_ = constant_op.constant([255, 0, 0, 255], dtype=dtypes.uint8)
-    gen_summary_ops.write_image_summary(
+    # Note the identity to move the tensor to the CPU.
+    return gen_summary_ops.write_image_summary(
         context.context().summary_writer_resource,
-        training_util.get_global_step(), tag, tensor, bad_color_, max_images,
-        name=scope)
+        training_util.get_global_step(), tag, array_ops.identity(tensor),
+        bad_color_,
+        max_images, name=scope)
 
   return summary_writer_function(name, tensor, function, family=family)
 
@@ -190,11 +233,12 @@ def audio(name, tensor, sample_rate, max_outputs, family=None):
   """Writes an audio summary if possible."""
 
   def function(tag, scope):
-    gen_summary_ops.write_audio_summary(
+    # Note the identity to move the tensor to the CPU.
+    return gen_summary_ops.write_audio_summary(
         context.context().summary_writer_resource,
         training_util.get_global_step(),
         tag,
-        tensor,
+        array_ops.identity(tensor),
         sample_rate=sample_rate,
         max_outputs=max_outputs,
         name=scope)
diff --git a/tensorflow/contrib/summary/summary_ops_test.py b/tensorflow/contrib/summary/summary_ops_test.py
index 405a92a726..de7ae6ec27 100644
--- a/tensorflow/contrib/summary/summary_ops_test.py
+++ b/tensorflow/contrib/summary/summary_ops_test.py
@@ -17,16 +17,14 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import os
 import tempfile
 
 from tensorflow.contrib.summary import summary_ops
-from tensorflow.core.util import event_pb2
+from tensorflow.contrib.summary import summary_test_util
 from tensorflow.python.eager import function
 from tensorflow.python.eager import test
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import test_util
-from tensorflow.python.lib.io import tf_record
 from tensorflow.python.platform import gfile
 from tensorflow.python.training import training_util
 
@@ -71,16 +69,9 @@ class TargetTest(test_util.TensorFlowTestCase):
         summary_ops.scalar('scalar', 2.0)
 
       write()
-
-      self.assertTrue(gfile.Exists(logdir))
-      files = gfile.ListDirectory(logdir)
-      self.assertEqual(len(files), 1)
-      records = list(
-          tf_record.tf_record_iterator(os.path.join(logdir, files[0])))
-      self.assertEqual(len(records), 2)
-      event = event_pb2.Event()
-      event.ParseFromString(records[1])
-      self.assertEqual(event.summary.value[0].simple_value, 2.0)
+      events = summary_test_util.events_from_file(logdir)
+      self.assertEqual(len(events), 2)
+      self.assertEqual(events[1].summary.value[0].simple_value, 2.0)
 
   def testSummaryName(self):
     training_util.get_or_create_global_step()
@@ -91,15 +82,9 @@ class TargetTest(test_util.TensorFlowTestCase):
 
       summary_ops.scalar('scalar', 2.0)
 
-      self.assertTrue(gfile.Exists(logdir))
-      files = gfile.ListDirectory(logdir)
-      self.assertEqual(len(files), 1)
-      records = list(
-          tf_record.tf_record_iterator(os.path.join(logdir, files[0])))
-      self.assertEqual(len(records), 2)
-      event = event_pb2.Event()
-      event.ParseFromString(records[1])
-      self.assertEqual(event.summary.value[0].tag, 'scalar')
+      events = summary_test_util.events_from_file(logdir)
+      self.assertEqual(len(events), 2)
+      self.assertEqual(events[1].summary.value[0].tag, 'scalar')
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/contrib/summary/summary_test_util.py b/tensorflow/contrib/summary/summary_test_util.py
new file mode 100644
index 0000000000..37b546d3ab
--- /dev/null
+++ b/tensorflow/contrib/summary/summary_test_util.py
@@ -0,0 +1,41 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""Utilities to test summaries."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import os
+
+from tensorflow.core.util import event_pb2
+from tensorflow.python.lib.io import tf_record
+from tensorflow.python.platform import gfile
+
+
+def events_from_file(logdir):
+  """Returns all events in the single eventfile in logdir."""
+  assert gfile.Exists(logdir)
+  files = gfile.ListDirectory(logdir)
+  assert len(files) == 1, "Found more than one file in logdir: %s" % files
+  records = list(
+      tf_record.tf_record_iterator(os.path.join(logdir, files[0])))
+  result = []
+  for r in records:
+    event = event_pb2.Event()
+    event.ParseFromString(r)
+    result.append(event)
+  return result
diff --git a/tensorflow/tools/pip_package/BUILD b/tensorflow/tools/pip_package/BUILD
index 3c4e1b66bc..579c51ab3a 100644
--- a/tensorflow/tools/pip_package/BUILD
+++ b/tensorflow/tools/pip_package/BUILD
@@ -85,6 +85,7 @@ py_binary(
         "//tensorflow/python:spectral_ops_test_util",
         "//tensorflow/python/tools:tools_pip",
         "//tensorflow/python/eager:eager_pip",
+        "//tensorflow/contrib/summary:summary_test_util",
         # These targets don't build on Windows yet. Exclude them for now.
         # "//tensorflow/contrib/ndlstm",
         # "//tensorflow/contrib/slim",
-- 
GitLab


From 65616777d73913346ec446df80b6d7aa64e54b24 Mon Sep 17 00:00:00 2001
From: Mark Daoust <markdaoust@google.com>
Date: Wed, 25 Oct 2017 20:46:06 -0700
Subject: [PATCH 808/909] Update the ReferenceResolver to output html links.

PiperOrigin-RevId: 173491069
---
 tensorflow/tools/docs/parser.py      | 70 ++++++++++++++++++++--------
 tensorflow/tools/docs/parser_test.py | 31 ++++++------
 2 files changed, 69 insertions(+), 32 deletions(-)

diff --git a/tensorflow/tools/docs/parser.py b/tensorflow/tools/docs/parser.py
index 1015103077..3db164c2b5 100644
--- a/tensorflow/tools/docs/parser.py
+++ b/tensorflow/tools/docs/parser.py
@@ -107,23 +107,40 @@ def _get_raw_docstring(py_object):
 
 
 # A regular expression for capturing a @{symbol} reference.
-SYMBOL_REFERENCE_RE = re.compile(r'@\{([^}]+)\}')
+SYMBOL_REFERENCE_RE = re.compile(
+    r"""
+    # Start with a literal "@{".
+    @\{
+      # Group at least 1 symbol: not "}" or "\n".
+      ([^}\n]+)
+    # Followed by a closing "}"
+    \}
+    """,
+    flags=re.VERBOSE)
 
 
 class ReferenceResolver(object):
   """Class for replacing @{...} references with Markdown links.
 
-  Args:
-    duplicate_of: A map from duplicate names to preferred names of API
-      symbols.
-    doc_index: A `dict` mapping symbol name strings to objects with `url`
-      and `title` fields. Used to resolve @{$doc} references in docstrings.
-    index: A map from all full names to python objects.
-    py_module_names: A list of string names of Python modules.
+  Attributes:
+    current_doc_full_name: A string (or None) indicating the name of the
+      document currently being processed, so errors can reference the broken
+      doc.
   """
 
   def __init__(self, duplicate_of, doc_index, is_class, is_module,
                py_module_names):
+    """Initializes a Reference Resolver.
+
+    Args:
+      duplicate_of: A map from duplicate names to preferred names of API
+        symbols.
+      doc_index: A `dict` mapping symbol name strings to objects with `url`
+        and `title` fields. Used to resolve @{$doc} references in docstrings.
+      is_class: A map from full names to bool for each symbol.
+      is_module: A map from full names to bool for each symbol.
+      py_module_names: A list of string names of Python modules.
+    """
     self._duplicate_of = duplicate_of
     self._doc_index = doc_index
     self._is_class = is_class
@@ -249,11 +266,19 @@ class ReferenceResolver(object):
     Returns:
       A markdown link to the documentation page of `ref_full_name`.
     """
-    link = self.reference_to_url(ref_full_name, relative_path_to_root)
+    url = self.reference_to_url(ref_full_name, relative_path_to_root)
+
     if code_ref:
-      return '[`%s`](%s)' % (link_text, link)
+      link_text = link_text.join(['<code>', '</code>'])
     else:
-      return '[%s](%s)' % (link_text, link)
+      link_text = self._link_text_to_html(link_text)
+
+    return '<a href="{}">{}</a>'.format(url, link_text)
+
+  @staticmethod
+  def _link_text_to_html(link_text):
+    code_re = '`(.*?)`'
+    return re.sub(code_re, r'<code>\1</code>', link_text)
 
   def py_master_name(self, full_name):
     """Return the master name for a Python symbol name."""
@@ -322,13 +347,13 @@ class ReferenceResolver(object):
 
     # Handle different types of references.
     if string.startswith('$'):  # Doc reference
-      return self._doc_link(
-          string, link_text, manual_link_text, relative_path_to_root)
+      return self._doc_link(string, link_text, manual_link_text,
+                            relative_path_to_root)
 
     elif string.startswith('tensorflow::'):
       # C++ symbol
-      return self._cc_link(
-          string, link_text, manual_link_text, relative_path_to_root)
+      return self._cc_link(string, link_text, manual_link_text,
+                           relative_path_to_root)
 
     else:
       is_python = False
@@ -337,8 +362,11 @@ class ReferenceResolver(object):
           is_python = True
           break
       if is_python:  # Python symbol
-        return self.python_link(link_text, string, relative_path_to_root,
-                                code_ref=not manual_link_text)
+        return self.python_link(
+            link_text,
+            string,
+            relative_path_to_root,
+            code_ref=not manual_link_text)
 
     # Error!
     self.add_error('Did not understand "%s"' % match.group(0))
@@ -361,7 +389,9 @@ class ReferenceResolver(object):
       if not manual_link_text: link_text = self._doc_index[string].title
       url = os.path.normpath(os.path.join(
           relative_path_to_root, '../..', self._doc_index[string].url))
-      return '[%s](%s%s)' % (link_text, url, hash_tag)
+      link_text = self._link_text_to_html(link_text)
+      return '<a href="{}{}">{}</a>'.format(url, hash_tag, link_text)
+
     return self._doc_missing(string, hash_tag, link_text, manual_link_text,
                              relative_path_to_root)
 
@@ -392,7 +422,9 @@ class ReferenceResolver(object):
     # to api_docs/cc, and then add ret.
     cc_relative_path = os.path.normpath(os.path.join(
         relative_path_to_root, '../cc', ret))
-    return '[`%s`](%s)' % (link_text, cc_relative_path)
+
+    return '<a href="{}"><code>{}</code></a>'.format(cc_relative_path,
+                                                     link_text)
 
 
 # TODO(aselle): Collect these into a big list for all modules and functions
diff --git a/tensorflow/tools/docs/parser_test.py b/tensorflow/tools/docs/parser_test.py
index 3b74a13f08..8a0e9af521 100644
--- a/tensorflow/tools/docs/parser_test.py
+++ b/tensorflow/tools/docs/parser_test.py
@@ -75,8 +75,9 @@ class ParserTest(googletest.TestCase):
       def foo(self):
         pass
 
-    string = ('A @{tf.reference}, another @{tf.reference}, '
-              'a member @{tf.reference.foo}, and a @{tf.third}.')
+    string = (
+        'A @{tf.reference}, another @{tf.reference}, a member '
+        '@{tf.reference.foo}, and a @{tf.third$link `text` with `code` in it}.')
     duplicate_of = {'tf.third': 'tf.fourth'}
     index = {'tf.reference': HasOneMember,
              'tf.reference.foo': HasOneMember.foo,
@@ -89,12 +90,15 @@ class ParserTest(googletest.TestCase):
         visitor=visitor, doc_index={}, py_module_names=['tf'])
 
     result = reference_resolver.replace_references(string, '../..')
-    self.assertEqual(
-        'A [`tf.reference`](../../tf/reference.md), another '
-        '[`tf.reference`](../../tf/reference.md), '
-        'a member [`tf.reference.foo`](../../tf/reference.md#foo), '
-        'and a [`tf.third`](../../tf/fourth.md).',
-        result)
+    self.assertEqual('A <a href="../../tf/reference.md">'
+                     '<code>tf.reference</code></a>, '
+                     'another <a href="../../tf/reference.md">'
+                     '<code>tf.reference</code></a>, '
+                     'a member <a href="../../tf/reference.md#foo">'
+                     '<code>tf.reference.foo</code></a>, '
+                     'and a <a href="../../tf/fourth.md">link '
+                     '<code>text</code> with '
+                     '<code>code</code> in it</a>.', result)
 
   def test_doc_replace_references(self):
     string = '@{$doc1} @{$doc1#abc} @{$doc1$link} @{$doc1#def$zelda} @{$do/c2}'
@@ -114,10 +118,11 @@ class ParserTest(googletest.TestCase):
     reference_resolver = parser.ReferenceResolver.from_visitor(
         visitor=visitor, doc_index=doc_index, py_module_names=['tf'])
     result = reference_resolver.replace_references(string, 'python')
-    self.assertEqual(
-        '[Title1](../URL1) [Title1](../URL1#abc) [link](../URL1) '
-        '[zelda](../URL1#def) [Two words](../somewhere/else)',
-        result)
+    self.assertEqual('<a href="../URL1">Title1</a> '
+                     '<a href="../URL1#abc">Title1</a> '
+                     '<a href="../URL1">link</a> '
+                     '<a href="../URL1#def">zelda</a> '
+                     '<a href="../somewhere/else">Two words</a>', result)
 
   def test_docs_for_class(self):
 
@@ -389,7 +394,7 @@ class ParserTest(googletest.TestCase):
     self.assertIn('TestModule.test_function', docs)
     # Leading backtick to make sure it's included top-level.
     # This depends on formatting, but should be stable.
-    self.assertIn('`test_function', docs)
+    self.assertIn('<code>test_function', docs)
 
   def test_argspec_for_functools_partial(self):
 
-- 
GitLab


From 62f62bfe9f0ffba8ae406dcb1b6c0165d38fe633 Mon Sep 17 00:00:00 2001
From: James Qin <jamesqin@google.com>
Date: Wed, 25 Oct 2017 21:38:45 -0700
Subject: [PATCH 809/909] Switch tf.contrib.cudnn_rnn.CudnnXXX to point to
 layer APIs instead of op wrappers

PiperOrigin-RevId: 173494053
---
 tensorflow/contrib/cudnn_rnn/__init__.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/tensorflow/contrib/cudnn_rnn/__init__.py b/tensorflow/contrib/cudnn_rnn/__init__.py
index 87ba834770..bc44562b50 100644
--- a/tensorflow/contrib/cudnn_rnn/__init__.py
+++ b/tensorflow/contrib/cudnn_rnn/__init__.py
@@ -30,13 +30,14 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from tensorflow.contrib.cudnn_rnn.python.layers.cudnn_rnn import CudnnGRU
+from tensorflow.contrib.cudnn_rnn.python.layers.cudnn_rnn import CudnnLSTM
+from tensorflow.contrib.cudnn_rnn.python.layers.cudnn_rnn import CudnnRNNRelu
+from tensorflow.contrib.cudnn_rnn.python.layers.cudnn_rnn import CudnnRNNTanh
 from tensorflow.contrib.cudnn_rnn.python.ops.cudnn_rnn_ops import CudnnCompatibleGRUCell
 from tensorflow.contrib.cudnn_rnn.python.ops.cudnn_rnn_ops import CudnnCompatibleLSTMCell
-from tensorflow.contrib.cudnn_rnn.python.ops.cudnn_rnn_ops import CudnnGRU
 from tensorflow.contrib.cudnn_rnn.python.ops.cudnn_rnn_ops import CudnnGRUSaveable
-from tensorflow.contrib.cudnn_rnn.python.ops.cudnn_rnn_ops import CudnnLSTM
 from tensorflow.contrib.cudnn_rnn.python.ops.cudnn_rnn_ops import CudnnLSTMSaveable
-from tensorflow.contrib.cudnn_rnn.python.ops.cudnn_rnn_ops import CudnnRNNRelu
 from tensorflow.contrib.cudnn_rnn.python.ops.cudnn_rnn_ops import CudnnRNNReluSaveable
 from tensorflow.contrib.cudnn_rnn.python.ops.cudnn_rnn_ops import CudnnRNNTanhSaveable
 
-- 
GitLab


From 09c5bf350ac634922328bc752e9250cf24966478 Mon Sep 17 00:00:00 2001
From: Mark Heffernan <meheff@google.com>
Date: Wed, 25 Oct 2017 23:03:44 -0700
Subject: [PATCH 810/909] Add support for PRED and S64 types to
 MakeFakeLiteral.

Don't uniquify names when creating and HLO module from a proto. This preserves instruction names across serialization/deserialization.

PiperOrigin-RevId: 173498734
---
 tensorflow/compiler/xla/client/lib/testing.cc | 18 +++++
 .../compiler/xla/service/hlo_computation.h    |  8 +-
 tensorflow/compiler/xla/service/hlo_module.cc | 80 ++++++++++++++-----
 tensorflow/compiler/xla/service/hlo_module.h  |  3 +-
 4 files changed, 80 insertions(+), 29 deletions(-)

diff --git a/tensorflow/compiler/xla/client/lib/testing.cc b/tensorflow/compiler/xla/client/lib/testing.cc
index 482d53cf33..e6645e4941 100644
--- a/tensorflow/compiler/xla/client/lib/testing.cc
+++ b/tensorflow/compiler/xla/client/lib/testing.cc
@@ -79,6 +79,24 @@ StatusOr<std::unique_ptr<Literal>> MakeFakeLiteral(const Shape& shape) {
           }));
       break;
     }
+    case S64: {
+      std::uniform_int_distribution<int64> generator(
+          std::numeric_limits<int64>::lowest(),
+          std::numeric_limits<int64>::max());
+      TF_CHECK_OK(literal->Populate<int64>(
+          [&](tensorflow::gtl::ArraySlice<int64> /*indices*/) {
+            return generator(engine);
+          }));
+      break;
+    }
+    case PRED: {
+      std::uniform_int_distribution<int> generator(0, 1);
+      TF_CHECK_OK(literal->Populate<bool>(
+          [&](tensorflow::gtl::ArraySlice<int64> /*indices*/) {
+            return generator(engine);
+          }));
+      break;
+    }
     default:
       return Unimplemented("Unsupported type for fake literal generation: %s",
                            ShapeUtil::HumanString(shape).c_str());
diff --git a/tensorflow/compiler/xla/service/hlo_computation.h b/tensorflow/compiler/xla/service/hlo_computation.h
index 3515a6b5df..f4edd17501 100644
--- a/tensorflow/compiler/xla/service/hlo_computation.h
+++ b/tensorflow/compiler/xla/service/hlo_computation.h
@@ -312,8 +312,7 @@ class HloComputation {
   explicit HloComputation(
       const string& name, int parameter_count,
       std::vector<std::unique_ptr<HloInstruction>>* instructions,
-      HloInstruction* root_instruction,
-      HloInstruction* fusion_instruction = nullptr);
+      HloInstruction* root_instruction, HloInstruction* fusion_instruction);
 
   // Internal helper for adding instructions.
   HloInstruction* AddInstructionInternal(
@@ -359,11 +358,6 @@ class HloComputation {
 
   std::vector<HloInstruction*> param_instructions_;
 
-  // Unique name generator for instruction identifiers. Instruction names should
-  // be unique per computation and this is enforced when instructions are added
-  // to the computation.
-  NameUniquer instruction_name_uniquer_;
-
   TF_DISALLOW_COPY_AND_ASSIGN(HloComputation);
 };
 
diff --git a/tensorflow/compiler/xla/service/hlo_module.cc b/tensorflow/compiler/xla/service/hlo_module.cc
index f7990fa789..4779ec7760 100644
--- a/tensorflow/compiler/xla/service/hlo_module.cc
+++ b/tensorflow/compiler/xla/service/hlo_module.cc
@@ -45,10 +45,37 @@ HloModule::HloModule(const string& name, const HloModuleConfig& config)
     : name_(name), config_(config) {}
 
 HloComputation* HloModule::AddComputationInternal(
-    std::unique_ptr<HloComputation> computation) {
-  computation->UniquifyName(&computation_name_uniquer_);
+    std::unique_ptr<HloComputation> computation, bool is_entry,
+    bool uniquify_names) {
+  if (is_entry) {
+    CHECK_EQ(nullptr, entry_computation_);
+    entry_computation_ = computation.get();
+
+    // If the module configuration has no entry layout computation set, create a
+    // default one based on the program shape.
+    if (!config_.has_entry_computation_layout()) {
+      config_.SetDefaultComputationLayout(
+          entry_computation_->ComputeProgramShape());
+    }
+  }
+
+  if (uniquify_names) {
+    computation->UniquifyName(&computation_name_uniquer_);
+    for (auto* instruction : computation->instructions()) {
+      instruction->UniquifyName(&instruction_name_uniquer_);
+    }
+  } else {
+    // Don't uniquify the names of the computation or instruction, but we must
+    // run the names through the uniquifiers to prevent future name collisions
+    // for computations and instructions created later.
+    computation_name_uniquer_.GetUniqueName(computation->name());
+    for (auto* instruction : computation->instructions()) {
+      instruction_name_uniquer_.GetUniqueName(instruction->name());
+    }
+  }
+
+  // Pick unique IDs for each instruction.
   for (auto* instruction : computation->instructions()) {
-    instruction->UniquifyName(&instruction_name_uniquer_);
     instruction->SetUniqueId(NewUniqueInstructionId());
   }
   computation->set_parent(this);
@@ -58,16 +85,8 @@ HloComputation* HloModule::AddComputationInternal(
 
 HloComputation* HloModule::AddEntryComputation(
     std::unique_ptr<HloComputation> computation) {
-  CHECK_EQ(nullptr, entry_computation_);
-  entry_computation_ = computation.get();
-
-  // If the module configuration has no entry layout computation set, create a
-  // default one based on the program shape.
-  if (!config_.has_entry_computation_layout()) {
-    config_.SetDefaultComputationLayout(
-        entry_computation_->ComputeProgramShape());
-  }
-  return AddComputationInternal(std::move(computation));
+  return AddComputationInternal(std::move(computation), /*is_entry=*/true,
+                                /*uniquify_names=*/true);
 }
 
 Status HloModule::RemoveEmbeddedComputation(HloComputation* to_remove) {
@@ -83,7 +102,8 @@ Status HloModule::RemoveEmbeddedComputation(HloComputation* to_remove) {
 
 HloComputation* HloModule::AddEmbeddedComputation(
     std::unique_ptr<HloComputation> computation) {
-  return AddComputationInternal(std::move(computation));
+  return AddComputationInternal(std::move(computation), /*is_entry=*/false,
+                                /*uniquify_names=*/true);
 }
 
 void HloModule::ReplaceComputations(
@@ -199,16 +219,34 @@ StatusOr<std::unique_ptr<HloModule>> HloModule::CreateFromProto(
     CHECK_NE(computation.get(), nullptr);
     TF_RET_CHECK(!ContainsKey(computation_map, computation->name()));
     string computation_name = computation->name();
-    if (proto.entry_computation_name() == computation_name) {
-      computation_map[computation_name] =
-          module->AddEntryComputation(std::move(computation));
-    } else {
-      computation_map[computation_name] =
-          module->AddEmbeddedComputation(std::move(computation));
-    }
+    // Don't uniquify names because we want names to be stable across
+    // serialization and deserialization.
+    computation_map[computation_name] = module->AddComputationInternal(
+        std::move(computation),
+        /*is_entry=*/proto.entry_computation_name() == computation_name,
+        /*uniquify_names=*/false);
   }
   TF_RET_CHECK(module->entry_computation_ != nullptr);
 
+  // Because we didn't uniquify the names, double-check that the instruction and
+  // computation names are unique from the proto.
+  tensorflow::gtl::FlatSet<string> computation_names;
+  tensorflow::gtl::FlatSet<string> instruction_names;
+  for (HloComputation* computation : module->computations()) {
+    if (computation->IsFusionComputation()) {
+      continue;
+    }
+
+    TF_RET_CHECK(!ContainsKey(computation_names, computation->name()))
+        << "Computation name is not unique: " << computation->name();
+    computation_names.insert(computation->name());
+    for (HloInstruction* instruction : computation->instructions()) {
+      TF_RET_CHECK(!ContainsKey(instruction_names, instruction->name()))
+          << "Instruction name is not unique: " << instruction->name();
+      instruction_names.insert(instruction->name());
+    }
+  }
+
   return std::move(module);
 }
 
diff --git a/tensorflow/compiler/xla/service/hlo_module.h b/tensorflow/compiler/xla/service/hlo_module.h
index 96c17d6297..2ac4244e5c 100644
--- a/tensorflow/compiler/xla/service/hlo_module.h
+++ b/tensorflow/compiler/xla/service/hlo_module.h
@@ -182,7 +182,8 @@ class HloModule {
 
  private:
   HloComputation* AddComputationInternal(
-      std::unique_ptr<HloComputation> computation);
+      std::unique_ptr<HloComputation> computation, bool is_entry,
+      bool uniquify_names);
 
   const string name_;
   HloModuleConfig config_;
-- 
GitLab


From e43e514b7418756a828c6a0f60e43aa6a638e961 Mon Sep 17 00:00:00 2001
From: Shanqing Cai <cais@google.com>
Date: Thu, 26 Oct 2017 04:43:45 -0700
Subject: [PATCH 811/909] TFE: Add incompatibility error and doc to
 add_check_numerics_ops()

PiperOrigin-RevId: 173522273
---
 tensorflow/contrib/eager/python/BUILD       |  1 +
 tensorflow/contrib/eager/python/tfe_test.py |  8 ++++++++
 tensorflow/python/BUILD                     |  1 +
 tensorflow/python/ops/numerics.py           | 15 +++++++++++++++
 4 files changed, 25 insertions(+)

diff --git a/tensorflow/contrib/eager/python/BUILD b/tensorflow/contrib/eager/python/BUILD
index 340dca7e1a..adfaaa010a 100644
--- a/tensorflow/contrib/eager/python/BUILD
+++ b/tensorflow/contrib/eager/python/BUILD
@@ -18,6 +18,7 @@ py_library(
         ":saver",
         ":summary_writer",
         "//tensorflow/python:framework_ops",
+        "//tensorflow/python:numerics",
         "//tensorflow/python:resource_variable_ops",
         "//tensorflow/python:util",
         "//tensorflow/python/eager:backprop",
diff --git a/tensorflow/contrib/eager/python/tfe_test.py b/tensorflow/contrib/eager/python/tfe_test.py
index eabff7f0a8..d8a38923a3 100644
--- a/tensorflow/contrib/eager/python/tfe_test.py
+++ b/tensorflow/contrib/eager/python/tfe_test.py
@@ -24,6 +24,7 @@ from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import numerics
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import test
 
@@ -101,6 +102,13 @@ class TFETest(test_util.TensorFlowTestCase):
     devices = tfe.list_devices()
     self.assertEqual(len(devices) - 1, tfe.num_gpus())
 
+  def testAddCheckNumericsOpsRaisesError(self):
+    with self.assertRaisesRegexp(
+        RuntimeError,
+        r'add_check_numerics_ops\(\) is not compatible with eager execution'):
+      numerics.add_check_numerics_ops()
+
+
 if __name__ == '__main__':
   tfe.enable_eager_execution()
   test.main()
diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 953aa566f0..e2be7e8e9a 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -1921,6 +1921,7 @@ py_library(
         ":array_ops",
         ":control_flow_ops",
         ":framework_for_generated_wrappers",
+        "//tensorflow/python/eager:context",
     ],
 )
 
diff --git a/tensorflow/python/ops/numerics.py b/tensorflow/python/ops/numerics.py
index 4e5d4bd9a1..f3558fda9c 100644
--- a/tensorflow/python/ops/numerics.py
+++ b/tensorflow/python/ops/numerics.py
@@ -19,6 +19,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from tensorflow.python.eager import context
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
@@ -62,7 +63,21 @@ def add_check_numerics_ops():
   Raises:
     ValueError: If the graph contains any numeric operations in a control flow
       structure.
+    RuntimeError: If called with eager execution enabled.
+
+  @compatibility(eager)
+  Not compatible with eager execution. To check for `Inf`s and `NaN`s under
+  eager execution, call tfe.seterr(inf_or_nan='raise') once before executing
+  the checked operations.
+  @enc_compatibility
   """
+  if context.in_eager_mode():
+    raise RuntimeError(
+        "add_check_numerics_ops() is not compatible with eager execution. "
+        "To check for Inf's and NaN's under eager execution, call "
+        "tfe.seterr(inf_or_nan='raise') once before executing the "
+        "checked operations.")
+
   check_op = []
   # This code relies on the ordering of ops in get_operations().
   # The producer of a tensor always comes before that tensor's consumer in
-- 
GitLab


From 10a183353513d61863072cd47776bf9e488397a2 Mon Sep 17 00:00:00 2001
From: Mark Daoust <markdaoust@google.com>
Date: Thu, 26 Oct 2017 09:06:31 -0400
Subject: [PATCH 812/909] Fix list formatting

Markdown needs a blank line before the list to render it correctly.

https://www.tensorflow.org/api_docs/python/tf/estimator/RunConfig#replace
---
 tensorflow/python/estimator/run_config.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/python/estimator/run_config.py b/tensorflow/python/estimator/run_config.py
index 372f01dc82..d71964d2ec 100644
--- a/tensorflow/python/estimator/run_config.py
+++ b/tensorflow/python/estimator/run_config.py
@@ -528,6 +528,7 @@ class RunConfig(object):
     """Returns a new instance of `RunConfig` replacing specified properties.
 
     Only the properties in the following list are allowed to be replaced:
+
       - `model_dir`.
       - `tf_random_seed`,
       - `save_summary_steps`,
-- 
GitLab


From b24b82ffe66e4ba4b520d5f3c8b5b7ad73f413b1 Mon Sep 17 00:00:00 2001
From: Peter Hawkins <phawkins@google.com>
Date: Thu, 26 Oct 2017 08:01:09 -0700
Subject: [PATCH 813/909] [TF:XLA] Mark the "paddings" argument to PadV2 as a
 compile-time constant.

PiperOrigin-RevId: 173538320
---
 tensorflow/compiler/tests/binary_ops_test.py | 14 ++++++++++++++
 tensorflow/compiler/tf2xla/const_analysis.cc |  1 +
 2 files changed, 15 insertions(+)

diff --git a/tensorflow/compiler/tests/binary_ops_test.py b/tensorflow/compiler/tests/binary_ops_test.py
index b387467246..9a225b32f8 100644
--- a/tensorflow/compiler/tests/binary_ops_test.py
+++ b/tensorflow/compiler/tests/binary_ops_test.py
@@ -707,6 +707,20 @@ class BinaryOpsTest(XLATestCase):
                [0, 0, 0, 0, 0, 0]],
               dtype=dtype))
 
+      self._testBinary(
+          lambda x, y: array_ops.pad(x, y, constant_values=7),
+          np.array(
+              [[1, 2, 3], [4, 5, 6]], dtype=dtype),
+          np.array(
+              [[0, 3], [2, 1]], dtype=np.int32),
+          expected=np.array(
+              [[7, 7, 1, 2, 3, 7],
+               [7, 7, 4, 5, 6, 7],
+               [7, 7, 7, 7, 7, 7],
+               [7, 7, 7, 7, 7, 7],
+               [7, 7, 7, 7, 7, 7]],
+              dtype=dtype))
+
   def testMirrorPad(self):
     mirror_pad = lambda t, paddings: array_ops.pad(t, paddings, "REFLECT")
     for dtype in self.numeric_types:
diff --git a/tensorflow/compiler/tf2xla/const_analysis.cc b/tensorflow/compiler/tf2xla/const_analysis.cc
index bf75f85db0..102a2cf07b 100644
--- a/tensorflow/compiler/tf2xla/const_analysis.cc
+++ b/tensorflow/compiler/tf2xla/const_analysis.cc
@@ -67,6 +67,7 @@ Status BackwardsConstAnalysis(const Graph& g,
       {"Min", "reduction_indices"},
       {"OneHot", "depth"},
       {"Pad", "paddings"},
+      {"PadV2", "paddings"},
       {"MirrorPad", "paddings"},
       {"Prod", "reduction_indices"},
       {"RandomStandardNormal", "shape"},
-- 
GitLab


From bfa539c03cd1555024fc04f4974e531c46b24e07 Mon Sep 17 00:00:00 2001
From: Malcolm Reynolds <mareynolds@google.com>
Date: Thu, 26 Oct 2017 08:42:37 -0700
Subject: [PATCH 814/909] Automated g4 rollback of changelist 173456597

PiperOrigin-RevId: 173542536
---
 .../core/common_runtime/gpu/gpu_device.cc     | 28 -------------------
 1 file changed, 28 deletions(-)

diff --git a/tensorflow/core/common_runtime/gpu/gpu_device.cc b/tensorflow/core/common_runtime/gpu/gpu_device.cc
index 2c906ed220..12d44cc6b7 100644
--- a/tensorflow/core/common_runtime/gpu/gpu_device.cc
+++ b/tensorflow/core/common_runtime/gpu/gpu_device.cc
@@ -652,34 +652,6 @@ Status BaseGPUDeviceFactory::CreateDevices(const SessionOptions& options,
   if (static_cast<size_t>(n) > valid_gpu_ids.size()) {
     n = valid_gpu_ids.size();
   }
-  // Save the original device.
-  int original_device = 0;
-  cudaError_t err = cudaGetDevice(&original_device);
-  if (err != cudaSuccess) {
-    return errors::Internal("cudaGetDevice() failed. Status: ",
-                            cudaGetErrorString(err));
-  }
-  // Force to implicitly initialize CUDA runtime on each valid GPU before
-  // CreateGPUDevice().
-  for (int gpu_id : valid_gpu_ids) {
-    err = cudaSetDevice(gpu_id);
-    if (err != cudaSuccess) {
-      return errors::Internal("cudaSetDevice() on GPU:", gpu_id,
-                              " failed. Status: ", cudaGetErrorString(err));
-    }
-    err = cudaFree(nullptr);
-    if (err != cudaSuccess) {
-      return errors::Internal(
-          "CUDA runtime implicit initialization on GPU:", gpu_id,
-          " failed. Status: ", cudaGetErrorString(err));
-    }
-  }
-  // Reset to the original device.
-  err = cudaSetDevice(original_device);
-  if (err != cudaSuccess) {
-    return errors::Internal("cudaSetDevice() on GPU:", original_device,
-                            " failed. Status: ", cudaGetErrorString(err));
-  }
   for (int i = 0; i < n; i++) {
     BaseGPUDevice* gpu_device;
     TF_RETURN_IF_ERROR(CreateGPUDevice(
-- 
GitLab


From 8269c66ed239556d458c6bbd369ded206081ceb9 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 26 Oct 2017 10:13:19 -0700
Subject: [PATCH 815/909] K-FAC: register_additional_minibatch() for
 CategoricalLogitsNegativeLogProbLoss

PiperOrigin-RevId: 173553770
---
 .../contrib/kfac/python/kernel_tests/BUILD    |  26 +--
 .../kernel_tests/loss_functions_test.py       |  57 +++++++
 .../contrib/kfac/python/ops/loss_functions.py | 151 ++++++++++++++----
 3 files changed, 192 insertions(+), 42 deletions(-)

diff --git a/tensorflow/contrib/kfac/python/kernel_tests/BUILD b/tensorflow/contrib/kfac/python/kernel_tests/BUILD
index 8980f03092..5d86373a23 100644
--- a/tensorflow/contrib/kfac/python/kernel_tests/BUILD
+++ b/tensorflow/contrib/kfac/python/kernel_tests/BUILD
@@ -81,18 +81,6 @@ py_test(
     ],
 )
 
-py_test(
-    name = "loss_functions_test",
-    srcs = ["loss_functions_test.py"],
-    srcs_version = "PY2AND3",
-    deps = [
-        "//tensorflow/contrib/kfac/python/ops:loss_functions",
-        "//tensorflow/python:array_ops",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:constant_op",
-    ],
-)
-
 py_test(
     name = "optimizer_test",
     srcs = ["optimizer_test.py"],
@@ -141,6 +129,20 @@ py_test(
     ],
 )
 
+py_test(
+    name = "loss_functions_test",
+    srcs = ["loss_functions_test.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        "//tensorflow/contrib/kfac/python/ops:loss_functions",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:constant_op",
+        "//tensorflow/python:framework_ops",
+        "//third_party/py/numpy",
+    ],
+)
+
 filegroup(
     name = "all_files",
     srcs = glob(
diff --git a/tensorflow/contrib/kfac/python/kernel_tests/loss_functions_test.py b/tensorflow/contrib/kfac/python/kernel_tests/loss_functions_test.py
index 86dd839896..87339cb059 100644
--- a/tensorflow/contrib/kfac/python/kernel_tests/loss_functions_test.py
+++ b/tensorflow/contrib/kfac/python/kernel_tests/loss_functions_test.py
@@ -18,8 +18,11 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import numpy as np
+
 from tensorflow.contrib.kfac.python.ops import loss_functions
 from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.platform import test
 
@@ -40,5 +43,59 @@ class InsertSliceInZerosTest(test.TestCase):
     self.assertAllEqual(expected_output_array, actual_output_array)
 
 
+class CategoricalLogitsNegativeLogProbLossTest(test.TestCase):
+
+  def testSample(self):
+    """Ensure samples can be drawn."""
+    with ops.Graph().as_default(), self.test_session() as sess:
+      logits = np.asarray([
+          [0., 0., 0.],  #
+          [1., -1., 0.]
+      ]).astype(np.float32)
+      loss = loss_functions.CategoricalLogitsNegativeLogProbLoss(
+          array_ops.constant(logits))
+      sample = loss.sample(42)
+      sample = sess.run(sample)
+      self.assertEqual(sample.shape, (2,))
+
+  def testEvaluateOnTargets(self):
+    """Ensure log probability can be evaluated correctly."""
+    with ops.Graph().as_default(), self.test_session() as sess:
+      logits = np.asarray([
+          [0., 0., 0.],  #
+          [1., -1., 0.]
+      ]).astype(np.float32)
+      targets = np.asarray([2, 1]).astype(np.int32)
+      loss = loss_functions.CategoricalLogitsNegativeLogProbLoss(
+          array_ops.constant(logits), targets=array_ops.constant(targets))
+      neg_log_prob = loss.evaluate()
+      neg_log_prob = sess.run(neg_log_prob)
+
+      # Calculate explicit log probability of targets.
+      probs = np.exp(logits) / np.sum(np.exp(logits), axis=1, keepdims=True)
+      log_probs = np.log([
+          probs[0, targets[0]],  #
+          probs[1, targets[1]]
+      ])
+      expected_log_prob = np.sum(log_probs)
+
+      self.assertAllClose(neg_log_prob, -expected_log_prob)
+
+  def testEvaluateOnSample(self):
+    """Ensure log probability of a sample can be drawn."""
+    with ops.Graph().as_default(), self.test_session() as sess:
+      logits = np.asarray([
+          [0., 0., 0.],  #
+          [1., -1., 0.]
+      ]).astype(np.float32)
+      loss = loss_functions.CategoricalLogitsNegativeLogProbLoss(
+          array_ops.constant(logits))
+      neg_log_prob = loss.evaluate_on_sample(42)
+
+      # Simply ensure this doesn't crash. As the output is random, it's
+      # difficult to say if the output is correct or not...
+      neg_log_prob = sess.run(neg_log_prob)
+
+
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/contrib/kfac/python/ops/loss_functions.py b/tensorflow/contrib/kfac/python/ops/loss_functions.py
index 69d97f0b5b..3cfde7f9ab 100644
--- a/tensorflow/contrib/kfac/python/ops/loss_functions.py
+++ b/tensorflow/contrib/kfac/python/ops/loss_functions.py
@@ -42,8 +42,14 @@ class LossFunction(object):
   use this class.  It depends on the use case.
   """
 
-  def __init__(self, targets=None):
-    self._targets = targets
+  @abc.abstractproperty
+  def targets(self):
+    """The targets being predicted by the model.
+
+    Returns:
+      None or Tensor of appropriate shape for calling self._evaluate() on.
+    """
+    pass
 
   @abc.abstractproperty
   def inputs(self):
@@ -51,16 +57,25 @@ class LossFunction(object):
     pass
 
   def evaluate(self):
-    """Evaluate the loss function."""
-    if self._targets is not None:
+    """Evaluate the loss function on the targets."""
+    if self.targets is not None:
       # We treat the targets as "constant".  It's only the inputs that get
       # "back-propped" through.
-      return self._evaluate(array_ops.stop_gradient(self._targets))
+      return self._evaluate(array_ops.stop_gradient(self.targets))
     else:
       raise Exception("Cannot evaluate losses with unspecified targets.")
 
   @abc.abstractmethod
   def _evaluate(self, targets):
+    """Evaluates the log probability of the targets.
+
+    Args:
+      targets: Tensor that distribution can calculate log_prob() of.
+
+    Returns:
+      log probability of each target, summed across all targets.
+    """
+
     pass
 
   @abc.abstractmethod
@@ -166,9 +181,9 @@ class LossFunction(object):
 class NegativeLogProbLoss(LossFunction):
   """Abstract base class for loss functions that are negative log probs."""
 
-  def __init__(self, targets=None, seed=None):
+  def __init__(self, seed=None):
     self._default_seed = seed
-    super(NegativeLogProbLoss, self).__init__(targets=targets)
+    super(NegativeLogProbLoss, self).__init__()
 
   @property
   def inputs(self):
@@ -176,6 +191,7 @@ class NegativeLogProbLoss(LossFunction):
 
   @abc.abstractproperty
   def params(self):
+    """Parameters to the underlying distribution."""
     pass
 
   @abc.abstractmethod
@@ -281,9 +297,18 @@ class NegativeLogProbLoss(LossFunction):
 
   @abc.abstractmethod
   def sample(self, seed):
+    """Sample 'targets' from the underlying distribution."""
     pass
 
   def evaluate_on_sample(self, seed=None):
+    """Evaluates the log probability on a random sample.
+
+    Args:
+      seed: int or None. Random seed for this draw from the distribution.
+
+    Returns:
+      Log probability of sampled targets, summed across examples.
+    """
     if seed is None:
       seed = self._default_seed
     # We treat the targets as "constant".  It's only the inputs that get
@@ -328,16 +353,19 @@ class NaturalParamsNegativeLogProbLoss(NegativeLogProbLoss):
 class DistributionNegativeLogProbLoss(NegativeLogProbLoss):
   """Base class for neg log prob losses that use the TF Distribution classes."""
 
-  def __init__(self, dist, targets=None, seed=None):
-    self._dist = dist
-    super(DistributionNegativeLogProbLoss, self).__init__(
-        targets=targets, seed=seed)
+  def __init__(self, seed=None):
+    super(DistributionNegativeLogProbLoss, self).__init__(seed=seed)
+
+  @abc.abstractproperty
+  def dist(self):
+    """The underlying tf.distributions.Distribution."""
+    pass
 
   def _evaluate(self, targets):
-    return -math_ops.reduce_sum(self._dist.log_prob(targets))
+    return -math_ops.reduce_sum(self.dist.log_prob(targets))
 
   def sample(self, seed):
-    return self._dist.sample(seed=seed)
+    return self.dist.sample(seed=seed)
 
 
 class NormalMeanNegativeLogProbLoss(DistributionNegativeLogProbLoss,
@@ -355,11 +383,18 @@ class NormalMeanNegativeLogProbLoss(DistributionNegativeLogProbLoss,
   """
 
   def __init__(self, mean, var=0.5, targets=None, seed=None):
-    dist = normal.Normal(loc=mean, scale=var**0.5)
     self._mean = mean
     self._var = var
-    super(NormalMeanNegativeLogProbLoss, self).__init__(
-        dist, targets=targets, seed=seed)
+    self._targets = targets
+    super(NormalMeanNegativeLogProbLoss, self).__init__(seed=seed)
+
+  @property
+  def targets(self):
+    return self._targets
+
+  @property
+  def dist(self):
+    return normal.Normal(loc=self._mean, scale=math_ops.sqrt(self._var))
 
   @property
   def params(self):
@@ -416,10 +451,16 @@ class NormalMeanVarianceNegativeLogProbLoss(DistributionNegativeLogProbLoss):
     self._mean = mean
     self._variance = variance
     self._scale = math_ops.sqrt(variance)
-    dist = normal.Normal(loc=self._mean, scale=self._scale)
-    super(NormalMeanVarianceNegativeLogProbLoss, self).__init__(dist,
-                                                                targets=targets,
-                                                                seed=seed)
+    self._targets = targets
+    super(NormalMeanVarianceNegativeLogProbLoss, self).__init__(seed=seed)
+
+  @property
+  def targets(self):
+    return self._targets
+
+  @property
+  def dist(self):
+    return normal.Normal(loc=self._mean, scale=self._scale)
 
   @property
   def params(self):
@@ -534,12 +575,53 @@ class CategoricalLogitsNegativeLogProbLoss(DistributionNegativeLogProbLoss,
   """
 
   def __init__(self, logits, targets=None, seed=None):
-    dist = categorical.Categorical(logits=logits)
-    self._logits = logits
-    self._probs = dist.probs
-    self._sqrt_probs = math_ops.sqrt(self._probs)
-    super(CategoricalLogitsNegativeLogProbLoss, self).__init__(
-        dist, targets=targets, seed=seed)
+    """Instantiates a CategoricalLogitsNegativeLogProbLoss.
+
+    Args:
+      logits: Tensor of shape [batch_size, output_size]. Parameters for
+        underlying distribution.
+      targets: None or Tensor of shape [output_size]. Each elements contains an
+        index in [0, output_size).
+      seed: int or None. Default random seed when sampling.
+    """
+    self._logits_components = []
+    self._targets_components = []
+    self.register_additional_minibatch(logits, targets=targets)
+    super(CategoricalLogitsNegativeLogProbLoss, self).__init__(seed=seed)
+
+  def register_additional_minibatch(self, logits, targets=None):
+    """Register an additiona minibatch's worth of parameters.
+
+    Args:
+      logits: Tensor of shape [batch_size, output_size]. Parameters for
+        underlying distribution.
+      targets: None or Tensor of shape [batch_size, output_size].  Each row must
+        be a one-hot vector.
+    """
+    self._logits_components.append(logits)
+    self._targets_components.append(targets)
+
+  @property
+  def _logits(self):
+    return array_ops.concat(self._logits_components, axis=0)
+
+  @property
+  def targets(self):
+    if all(target is None for target in self._targets_components):
+      return None
+    return array_ops.concat(self._targets_components, axis=0)
+
+  @property
+  def dist(self):
+    return categorical.Categorical(logits=self._logits)
+
+  @property
+  def _probs(self):
+    return self.dist.probs
+
+  @property
+  def _sqrt_probs(self):
+    return math_ops.sqrt(self._probs)
 
   @property
   def params(self):
@@ -595,12 +677,21 @@ class MultiBernoulliNegativeLogProbLoss(DistributionNegativeLogProbLoss,
   """
 
   def __init__(self, logits, targets=None, seed=None):
-    dist = bernoulli.Bernoulli(logits=logits)
     self._logits = logits
-    self._probs = dist.probs
+    self._targets = targets
+    super(MultiBernoulliNegativeLogProbLoss, self).__init__(seed=seed)
+
+  @property
+  def targets(self):
+    return self._targets
 
-    super(MultiBernoulliNegativeLogProbLoss, self).__init__(
-        dist, targets=targets, seed=seed)
+  @property
+  def dist(self):
+    return bernoulli.Bernoulli(logits=self._logits)
+
+  @property
+  def _probs(self):
+    return self.dist.probs
 
   @property
   def params(self):
-- 
GitLab


From f0aa811ee71ef2a54c67a1311557b331379a56e6 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 26 Oct 2017 10:48:45 -0700
Subject: [PATCH 816/909] Use the new robust implementation of log determinant
 (linalg.slogdet) in LinearOperator. Move creation of aliases into
 linalg_impl, such that these can be used inside TensorFlow without creating
 circular dependencies.

PiperOrigin-RevId: 173558892
---
 tensorflow/python/ops/linalg/BUILD            |  5 +--
 tensorflow/python/ops/linalg/linalg.py        | 40 +++----------------
 tensorflow/python/ops/linalg/linalg_impl.py   | 28 +++++++++++++
 .../python/ops/linalg/linear_operator.py      |  4 +-
 .../ops/linalg/linear_operator_test_util.py   |  3 +-
 5 files changed, 38 insertions(+), 42 deletions(-)

diff --git a/tensorflow/python/ops/linalg/BUILD b/tensorflow/python/ops/linalg/BUILD
index b88e72a6f3..ce8c1580fe 100644
--- a/tensorflow/python/ops/linalg/BUILD
+++ b/tensorflow/python/ops/linalg/BUILD
@@ -9,15 +9,13 @@ py_library(
     srcs = glob(["*.py"]),
     srcs_version = "PY2AND3",
     deps = [
-        "//tensorflow/python:array_ops",
+        ":linalg_impl",
         "//tensorflow/python:check_ops",
         "//tensorflow/python:control_flow_ops",
         "//tensorflow/python:framework_for_generated_wrappers",
-        "//tensorflow/python:math_ops",
         "//tensorflow/python:nn",
         "//tensorflow/python:nn_ops",
         "//tensorflow/python:random_ops",
-        "//tensorflow/python:special_math_ops",
         "//tensorflow/python:tensor_util",
         "//tensorflow/python:util",
         "//third_party/py/numpy",
@@ -33,6 +31,7 @@ py_library(
         "//tensorflow/python:array_ops",
         "//tensorflow/python:linalg_ops",
         "//tensorflow/python:math_ops",
+        "//tensorflow/python:special_math_ops",
     ],
 )
 
diff --git a/tensorflow/python/ops/linalg/linalg.py b/tensorflow/python/ops/linalg/linalg.py
index 02ceb65e2a..5369007a56 100644
--- a/tensorflow/python/ops/linalg/linalg.py
+++ b/tensorflow/python/ops/linalg/linalg.py
@@ -18,12 +18,6 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import gen_linalg_ops
-from tensorflow.python.ops import linalg_ops
-from tensorflow.python.ops import math_ops
-from tensorflow.python.ops import special_math_ops
-
 # go/tf-wildcard-import
 # pylint: disable=wildcard-import,unused-import
 from tensorflow.python.ops.linalg.linalg_impl import *
@@ -36,39 +30,15 @@ from tensorflow.python.ops.linalg.linear_operator_low_rank_update import *
 from tensorflow.python.ops.linalg.linear_operator_lower_triangular import *
 # pylint: enable=wildcard-import
 
-# Linear algebra ops.
-band_part = array_ops.matrix_band_part
-cholesky = linalg_ops.cholesky
-cholesky_solve = linalg_ops.cholesky_solve
-det = linalg_ops.matrix_determinant
-# pylint: disable=protected-access
-slogdet = gen_linalg_ops._log_matrix_determinant
-# pylint: disable=protected-access
-diag = array_ops.matrix_diag
-diag_part = array_ops.matrix_diag_part
-eigh = linalg_ops.self_adjoint_eig
-eigvalsh = linalg_ops.self_adjoint_eigvals
-einsum = special_math_ops.einsum
-eye = linalg_ops.eye
-inv = linalg_ops.matrix_inverse
-lstsq = linalg_ops.matrix_solve_ls
-norm = linalg_ops.norm
-qr = linalg_ops.qr
-set_diag = array_ops.matrix_set_diag
-solve = linalg_ops.matrix_solve
-svd = linalg_ops.svd
-tensordot = math_ops.tensordot
-trace = math_ops.trace
-transpose = array_ops.matrix_transpose
-triangular_solve = linalg_ops.matrix_triangular_solve
-
 # Seal API.
+# pylint: disable=undefined-variable
 del absolute_import
-del array_ops
 del division
+del print_function
+del ops
+del array_ops
 del gen_linalg_ops
 del linalg_ops
 del math_ops
-del ops
-del print_function
 del special_math_ops
+# pylint: enable=undefined-variable
diff --git a/tensorflow/python/ops/linalg/linalg_impl.py b/tensorflow/python/ops/linalg/linalg_impl.py
index 1fdec2b51b..04a15e3e5b 100644
--- a/tensorflow/python/ops/linalg/linalg_impl.py
+++ b/tensorflow/python/ops/linalg/linalg_impl.py
@@ -21,7 +21,35 @@ from __future__ import print_function
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gen_linalg_ops
+from tensorflow.python.ops import linalg_ops
 from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import special_math_ops
+
+# Linear algebra ops.
+band_part = array_ops.matrix_band_part
+cholesky = linalg_ops.cholesky
+cholesky_solve = linalg_ops.cholesky_solve
+det = linalg_ops.matrix_determinant
+# pylint: disable=protected-access
+slogdet = gen_linalg_ops._log_matrix_determinant
+# pylint: disable=protected-access
+diag = array_ops.matrix_diag
+diag_part = array_ops.matrix_diag_part
+eigh = linalg_ops.self_adjoint_eig
+eigvalsh = linalg_ops.self_adjoint_eigvals
+einsum = special_math_ops.einsum
+eye = linalg_ops.eye
+inv = linalg_ops.matrix_inverse
+lstsq = linalg_ops.matrix_solve_ls
+norm = linalg_ops.norm
+qr = linalg_ops.qr
+set_diag = array_ops.matrix_set_diag
+solve = linalg_ops.matrix_solve
+svd = linalg_ops.svd
+tensordot = math_ops.tensordot
+trace = math_ops.trace
+transpose = array_ops.matrix_transpose
+triangular_solve = linalg_ops.matrix_triangular_solve
 
 
 def logdet(matrix, name=None):
diff --git a/tensorflow/python/ops/linalg/linear_operator.py b/tensorflow/python/ops/linalg/linear_operator.py
index 0d04e29eb3..27e0f17020 100644
--- a/tensorflow/python/ops/linalg/linear_operator.py
+++ b/tensorflow/python/ops/linalg/linear_operator.py
@@ -693,8 +693,8 @@ class LinearOperator(object):
     if self._can_use_cholesky():
       diag = array_ops.matrix_diag_part(self._get_cached_chol())
       return 2 * math_ops.reduce_sum(math_ops.log(diag), reduction_indices=[-1])
-    abs_det = math_ops.abs(self.determinant())
-    return math_ops.log(abs_det)
+    _, log_abs_det = linalg.slogdet(self._matrix)
+    return log_abs_det
 
   def log_abs_determinant(self, name="log_abs_det"):
     """Log absolute value of determinant for every batch member.
diff --git a/tensorflow/python/ops/linalg/linear_operator_test_util.py b/tensorflow/python/ops/linalg/linear_operator_test_util.py
index 4a601047b6..3d0ea3e11b 100644
--- a/tensorflow/python/ops/linalg/linear_operator_test_util.py
+++ b/tensorflow/python/ops/linalg/linear_operator_test_util.py
@@ -191,8 +191,7 @@ class LinearOperatorDerivedClassTest(test.TestCase):
             operator, mat, feed_dict = self._operator_and_mat_and_feed_dict(
                 shape, dtype, use_placeholder=use_placeholder)
             op_log_abs_det = operator.log_abs_determinant()
-            mat_log_abs_det = math_ops.log(
-                math_ops.abs(linalg_ops.matrix_determinant(mat)))
+            _, mat_log_abs_det = linalg.slogdet(mat)
             if not use_placeholder:
               self.assertAllEqual(shape[:-2], op_log_abs_det.get_shape())
             op_log_abs_det_v, mat_log_abs_det_v = sess.run(
-- 
GitLab


From 0bbee64143ca76d83ff284fb80a9b59c367bac28 Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Thu, 26 Oct 2017 10:58:51 -0700
Subject: [PATCH 817/909] Automated g4 rollback of changelist 173494053

PiperOrigin-RevId: 173560463
---
 tensorflow/contrib/cudnn_rnn/__init__.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/cudnn_rnn/__init__.py b/tensorflow/contrib/cudnn_rnn/__init__.py
index bc44562b50..87ba834770 100644
--- a/tensorflow/contrib/cudnn_rnn/__init__.py
+++ b/tensorflow/contrib/cudnn_rnn/__init__.py
@@ -30,14 +30,13 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.contrib.cudnn_rnn.python.layers.cudnn_rnn import CudnnGRU
-from tensorflow.contrib.cudnn_rnn.python.layers.cudnn_rnn import CudnnLSTM
-from tensorflow.contrib.cudnn_rnn.python.layers.cudnn_rnn import CudnnRNNRelu
-from tensorflow.contrib.cudnn_rnn.python.layers.cudnn_rnn import CudnnRNNTanh
 from tensorflow.contrib.cudnn_rnn.python.ops.cudnn_rnn_ops import CudnnCompatibleGRUCell
 from tensorflow.contrib.cudnn_rnn.python.ops.cudnn_rnn_ops import CudnnCompatibleLSTMCell
+from tensorflow.contrib.cudnn_rnn.python.ops.cudnn_rnn_ops import CudnnGRU
 from tensorflow.contrib.cudnn_rnn.python.ops.cudnn_rnn_ops import CudnnGRUSaveable
+from tensorflow.contrib.cudnn_rnn.python.ops.cudnn_rnn_ops import CudnnLSTM
 from tensorflow.contrib.cudnn_rnn.python.ops.cudnn_rnn_ops import CudnnLSTMSaveable
+from tensorflow.contrib.cudnn_rnn.python.ops.cudnn_rnn_ops import CudnnRNNRelu
 from tensorflow.contrib.cudnn_rnn.python.ops.cudnn_rnn_ops import CudnnRNNReluSaveable
 from tensorflow.contrib.cudnn_rnn.python.ops.cudnn_rnn_ops import CudnnRNNTanhSaveable
 
-- 
GitLab


From 50914f04dc5eb28695933bf94a8e8fe221371b7c Mon Sep 17 00:00:00 2001
From: ted chang <htchang@us.ibm.com>
Date: Wed, 25 Oct 2017 12:41:57 -0700
Subject: [PATCH 818/909] Changed GPU driver version assumption Fixes #9669

---
 tensorflow/stream_executor/cuda/cuda_diagnostics.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/stream_executor/cuda/cuda_diagnostics.cc b/tensorflow/stream_executor/cuda/cuda_diagnostics.cc
index bf81b9c0ad..00506fa54b 100644
--- a/tensorflow/stream_executor/cuda/cuda_diagnostics.cc
+++ b/tensorflow/stream_executor/cuda/cuda_diagnostics.cc
@@ -76,10 +76,10 @@ string DriverVersionStatusToString(port::StatusOr<DriverVersion> version) {
 
 port::StatusOr<DriverVersion> StringToDriverVersion(const string &value) {
   std::vector<string> pieces = port::Split(value, '.');
-  if (pieces.size() != 2 && pieces.size() != 3) {
+  if (pieces.size() < 2 || pieces.size() > 4) {
     return port::Status{
         port::error::INVALID_ARGUMENT,
-        port::Printf("expected %%d.%%d or %%d.%%d.%%d form for driver version; got \"%s\"",
+        port::Printf("expected %%d.%%d, %%d.%%d.%%d, or %%d.%%d.%%d.%%d form for driver version; got \"%s\"",
                      value.c_str())};
   }
 
-- 
GitLab


From fcf3e00d04bc911e2af9f2cbc13edee30dc03f7c Mon Sep 17 00:00:00 2001
From: Akshay Agrawal <akshayka@google.com>
Date: Thu, 26 Oct 2017 11:49:59 -0700
Subject: [PATCH 819/909] Implement __int__, __float__ for EagerTensors

PiperOrigin-RevId: 173569138
---
 tensorflow/python/eager/ops_test.py | 14 ++++++++++++++
 tensorflow/python/framework/ops.py  |  8 ++++++++
 2 files changed, 22 insertions(+)

diff --git a/tensorflow/python/eager/ops_test.py b/tensorflow/python/eager/ops_test.py
index e34587d5b1..e86073d6b2 100644
--- a/tensorflow/python/eager/ops_test.py
+++ b/tensorflow/python/eager/ops_test.py
@@ -331,6 +331,20 @@ class OpsTest(test_util.TensorFlowTestCase):
     x.set_shape(tensor_shape.TensorShape([None, 2]))
     self.assertEqual(x.get_shape(), (1, 2))
 
+  def testCastScalarToPrimitiveTypes(self):
+    x = constant_op.constant(1.3)
+    self.assertIsInstance(int(x), int)
+    self.assertEqual(int(x), 1)
+    self.assertIsInstance(float(x), float)
+    self.assertAllClose(float(x), 1.3)
+
+  def testCastNonScalarToPrimitiveTypesFails(self):
+    x = constant_op.constant([1.3, 2])
+    with self.assertRaises(TypeError):
+      int(x)
+    with self.assertRaises(TypeError):
+      float(x)
+
 
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index 61a5a4fcae..eceacb42d9 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -623,6 +623,14 @@ class _EagerTensorBase(Tensor):
       raise ValueError("Resource handles are not convertible to numpy.")
     return self.cpu()._numpy()  # pylint: disable=protected-access
 
+  # __int__ and  __float__ may copy the tensor to CPU and
+  # only work for scalars; values are cast as per numpy.
+  def __int__(self):
+    return int(self.numpy())
+
+  def __float__(self):
+    return float(self.numpy())
+
   def __array__(self):
     return np.array(self.numpy())
 
-- 
GitLab


From a80f91bd8a9733800275b0b1328747770c7e46e8 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 26 Oct 2017 12:31:07 -0700
Subject: [PATCH 820/909] K-FAC: Support for multiple minibatches with
 register_categorical_predictive_distribution().

PiperOrigin-RevId: 173574699
---
 .../kernel_tests/layer_collection_test.py     | 53 +++++++++++++++++--
 .../kfac/python/ops/layer_collection.py       | 46 +++++++++++++---
 2 files changed, 89 insertions(+), 10 deletions(-)

diff --git a/tensorflow/contrib/kfac/python/kernel_tests/layer_collection_test.py b/tensorflow/contrib/kfac/python/kernel_tests/layer_collection_test.py
index 432937d803..4f27ceced9 100644
--- a/tensorflow/contrib/kfac/python/kernel_tests/layer_collection_test.py
+++ b/tensorflow/contrib/kfac/python/kernel_tests/layer_collection_test.py
@@ -214,8 +214,8 @@ class LayerCollectionTest(test.TestCase):
       self.assertEqual(1, len(lc.losses))
 
       # Add logits to same loss function.
-      with self.assertRaises(NotImplementedError):
-        lc.register_categorical_predictive_distribution(logits, name='loss1')
+      lc.register_categorical_predictive_distribution(
+          logits, name='loss1', reuse=True)
       self.assertEqual(1, len(lc.losses))
 
       # Add another new loss function.
@@ -228,11 +228,58 @@ class LayerCollectionTest(test.TestCase):
       logits = linalg_ops.eye(2)
       lc = layer_collection.LayerCollection()
 
-      # Create a new loss function by name.
+      # Create a new loss function with default names.
       lc.register_categorical_predictive_distribution(logits)
       lc.register_categorical_predictive_distribution(logits)
       self.assertEqual(2, len(lc.losses))
 
+  def testCategoricalPredictiveDistributionMultipleMinibatches(self):
+    """Ensure multiple minibatches are registered."""
+    with ops.Graph().as_default():
+      batch_size = 3
+      output_size = 2
+      logits = array_ops.zeros([batch_size, output_size])
+      targets = array_ops.ones([batch_size], dtype=dtypes.int32)
+      lc = layer_collection.LayerCollection()
+
+      # Create a new loss function.
+      lc.register_categorical_predictive_distribution(
+          logits, targets=targets, name='loss1')
+
+      # Can add when reuse=True
+      lc.register_categorical_predictive_distribution(
+          logits, targets=targets, name='loss1', reuse=True)
+
+      # Can add when reuse=VARIABLE_SCOPE and reuse=True there.
+      with variable_scope.variable_scope(
+          variable_scope.get_variable_scope(), reuse=True):
+        lc.register_categorical_predictive_distribution(
+            logits,
+            targets=targets,
+            name='loss1',
+            reuse=layer_collection.VARIABLE_SCOPE)
+
+      # Can't add when reuse=False
+      with self.assertRaises(KeyError):
+        lc.register_categorical_predictive_distribution(
+            logits, targets=targets, name='loss1', reuse=False)
+
+      # Can't add when reuse=VARIABLE_SCOPE and reuse=False there.
+      with self.assertRaises(KeyError):
+        lc.register_categorical_predictive_distribution(
+            logits,
+            targets=targets,
+            name='loss1',
+            reuse=layer_collection.VARIABLE_SCOPE)
+
+      self.assertEqual(len(lc.losses), 1)
+      loss = lc.losses[0]
+
+      # Three successful registrations.
+      self.assertEqual(loss.params.shape.as_list(),
+                       [3 * batch_size, output_size])
+      self.assertEqual(loss.targets.shape.as_list(), [3 * batch_size])
+
   def testRegisterCategoricalPredictiveDistributionBatchSize1(self):
     with ops.Graph().as_default():
       random_seed.set_random_seed(200)
diff --git a/tensorflow/contrib/kfac/python/ops/layer_collection.py b/tensorflow/contrib/kfac/python/ops/layer_collection.py
index cd711d0561..2b9958a46a 100644
--- a/tensorflow/contrib/kfac/python/ops/layer_collection.py
+++ b/tensorflow/contrib/kfac/python/ops/layer_collection.py
@@ -342,7 +342,8 @@ class LayerCollection(object):
                                                    logits,
                                                    seed=None,
                                                    targets=None,
-                                                   name=None):
+                                                   name=None,
+                                                   reuse=VARIABLE_SCOPE):
     """Registers a categorical predictive distribution.
 
     Args:
@@ -355,15 +356,46 @@ class LayerCollection(object):
         (Default: None)
       name: (OPTIONAL) str or None. Unique name for this loss function. If None,
         a new name is generated. (Default: None)
+      reuse: (OPTIONAL) bool or str.  If True, reuse an existing FisherBlock.
+        If False, create a new FisherBlock.  If VARIABLE_SCOPE, use
+        tf.get_variable_scope().reuse.
+
+    Raises:
+      ValueError: If reuse=True and name != None.
+      ValueError: If reuse=True and seed != None.
+      KeyError: If reuse=True and no existing LossFunction with 'name' found.
+      KeyError: If reuse=False and existing LossFunction with 'name' found.
     """
     name = name or self._graph.unique_name(
         "register_categorical_predictive_distribution")
-    if name in self._loss_dict:
-      raise NotImplementedError(
-          "Adding logits to an existing LossFunction not yet supported.")
-    loss = lf.CategoricalLogitsNegativeLogProbLoss(
-        logits, targets=targets, seed=seed)
-    self._loss_dict[name] = loss
+
+    if reuse == VARIABLE_SCOPE:
+      reuse = variable_scope.get_variable_scope().reuse
+
+    if reuse:
+      if name is None:
+        raise ValueError(
+            "If reuse is enabled, loss function's name must be set.")
+      if seed is not None:
+        raise ValueError(
+            "Seed can only be specified at LossFunction instantiation.")
+
+      loss = self._loss_dict.get(name, None)
+
+      if loss is None:
+        raise KeyError(
+            "Unable to find loss function named {}. Create a new LossFunction "
+            "with reuse=False.".format(name))
+
+      loss.register_additional_minibatch(logits, targets=targets)
+    else:
+      if name in self._loss_dict:
+        raise KeyError(
+            "Loss function named {} already exists. Set reuse=True to append "
+            "another minibatch.".format(name))
+      loss = lf.CategoricalLogitsNegativeLogProbLoss(
+          logits, targets=targets, seed=seed)
+      self._loss_dict[name] = loss
 
   def register_normal_predictive_distribution(self,
                                               mean,
-- 
GitLab


From bab6e69913f7fd0ad59a93e092ac28720b99a05c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 26 Oct 2017 12:43:08 -0700
Subject: [PATCH 821/909] Updating documentation of
 _remove_squeezable_dimensions (in python/ops/metrics_impl.py) and removing
 duplicate function in contrib/metrics/python/ops/metric_ops.py.

PiperOrigin-RevId: 173576149
---
 .../contrib/metrics/python/ops/metric_ops.py  | 68 ++++---------------
 .../metrics/python/ops/metric_ops_test.py     |  2 +-
 tensorflow/python/ops/metrics_impl.py         | 18 +++--
 3 files changed, 24 insertions(+), 64 deletions(-)

diff --git a/tensorflow/contrib/metrics/python/ops/metric_ops.py b/tensorflow/contrib/metrics/python/ops/metric_ops.py
index 675c49dfc3..50b9c4afde 100644
--- a/tensorflow/contrib/metrics/python/ops/metric_ops.py
+++ b/tensorflow/contrib/metrics/python/ops/metric_ops.py
@@ -28,7 +28,6 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import check_ops
-from tensorflow.python.ops import confusion_matrix
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import metrics
@@ -223,7 +222,7 @@ def streaming_true_negatives(predictions,
   with variable_scope.variable_scope(name, 'true_negatives',
                                      (predictions, labels, weights)):
 
-    predictions, labels, weights = _remove_squeezable_dimensions(
+    predictions, labels, weights = metrics_impl._remove_squeezable_dimensions(  # pylint: disable=protected-access
         predictions=math_ops.cast(predictions, dtype=dtypes.bool),
         labels=math_ops.cast(labels, dtype=dtypes.bool),
         weights=weights)
@@ -654,7 +653,7 @@ def _true_negatives(labels,
   with variable_scope.variable_scope(name, 'true_negatives',
                                      (predictions, labels, weights)):
 
-    predictions, labels, weights = _remove_squeezable_dimensions(
+    predictions, labels, weights = metrics_impl._remove_squeezable_dimensions(  # pylint: disable=protected-access
         predictions=math_ops.cast(predictions, dtype=dtypes.bool),
         labels=math_ops.cast(labels, dtype=dtypes.bool),
         weights=weights)
@@ -715,7 +714,7 @@ def streaming_false_positive_rate(predictions,
   """
   with variable_scope.variable_scope(name, 'false_positive_rate',
                                      (predictions, labels, weights)):
-    predictions, labels, weights = _remove_squeezable_dimensions(
+    predictions, labels, weights = metrics_impl._remove_squeezable_dimensions(  # pylint: disable=protected-access
         predictions=math_ops.cast(predictions, dtype=dtypes.bool),
         labels=math_ops.cast(labels, dtype=dtypes.bool),
         weights=weights)
@@ -803,7 +802,7 @@ def streaming_false_negative_rate(predictions,
   """
   with variable_scope.variable_scope(name, 'false_negative_rate',
                                      (predictions, labels, weights)):
-    predictions, labels, weights = _remove_squeezable_dimensions(
+    predictions, labels, weights = metrics_impl._remove_squeezable_dimensions(  # pylint: disable=protected-access
         predictions=math_ops.cast(predictions, dtype=dtypes.bool),
         labels=math_ops.cast(labels, dtype=dtypes.bool),
         weights=weights)
@@ -896,7 +895,7 @@ def _streaming_confusion_matrix_at_thresholds(predictions,
       if include not in all_includes:
         raise ValueError('Invaild key: %s.' % include)
 
-  predictions, labels, weights = _remove_squeezable_dimensions(
+  predictions, labels, weights = metrics_impl._remove_squeezable_dimensions(  # pylint: disable=protected-access
       predictions, labels, weights)
   predictions.get_shape().assert_is_compatible_with(labels.get_shape())
 
@@ -1284,8 +1283,10 @@ def streaming_precision_recall_at_equal_thresholds(predictions,
             math_ops.cast(1.0, dtype=predictions.dtype),
             message='predictions must be in [0, 1]')
     ]):
-      predictions, labels, weights = _remove_squeezable_dimensions(
-          predictions=predictions, labels=labels, weights=weights)
+      predictions, labels, weights = metrics_impl._remove_squeezable_dimensions(  # pylint: disable=protected-access
+          predictions=predictions,
+          labels=labels,
+          weights=weights)
 
     predictions.get_shape().assert_is_compatible_with(labels.get_shape())
 
@@ -2597,7 +2598,7 @@ def streaming_covariance(predictions,
   """
   with variable_scope.variable_scope(name, 'covariance',
                                      (predictions, labels, weights)):
-    predictions, labels, weights = _remove_squeezable_dimensions(
+    predictions, labels, weights = metrics_impl._remove_squeezable_dimensions(  # pylint: disable=protected-access
         predictions, labels, weights)
     predictions.get_shape().assert_is_compatible_with(labels.get_shape())
     count = _create_local('count', [])
@@ -2731,7 +2732,7 @@ def streaming_pearson_correlation(predictions,
   """
   with variable_scope.variable_scope(name, 'pearson_r',
                                      (predictions, labels, weights)):
-    predictions, labels, weights = _remove_squeezable_dimensions(
+    predictions, labels, weights = metrics_impl._remove_squeezable_dimensions(  # pylint: disable=protected-access
         predictions, labels, weights)
     predictions.get_shape().assert_is_compatible_with(labels.get_shape())
     # Broadcast weights here to avoid duplicate broadcasting in each call to
@@ -2813,7 +2814,7 @@ def streaming_mean_cosine_distance(predictions,
       either `metrics_collections` or `updates_collections` are not a list or
       tuple.
   """
-  predictions, labels, weights = _remove_squeezable_dimensions(
+  predictions, labels, weights = metrics_impl._remove_squeezable_dimensions(  # pylint: disable=protected-access
       predictions, labels, weights)
   predictions.get_shape().assert_is_compatible_with(labels.get_shape())
   radial_diffs = math_ops.multiply(predictions, labels)
@@ -3123,51 +3124,6 @@ def aggregate_metric_map(names_to_tuples):
   return dict(zip(metric_names, value_ops)), dict(zip(metric_names, update_ops))
 
 
-def _remove_squeezable_dimensions(predictions, labels, weights):
-  """Squeeze last dim if needed.
-
-  Squeezes `predictions` and `labels` if their rank differs by 1.
-  Squeezes `weights` if its rank is 1 more than the new rank of `predictions`
-
-  This will use static shape if available. Otherwise, it will add graph
-  operations, which could result in a performance hit.
-
-  Args:
-    predictions: Predicted values, a `Tensor` of arbitrary dimensions.
-    labels: Label values, a `Tensor` whose dimensions match `predictions`.
-    weights: Optional weight `Tensor`. It will be squeezed if its rank is 1
-      more than the new rank of `predictions`
-
-  Returns:
-    Tuple of `predictions`, `labels` and `weights`, possibly with the last
-    dimension squeezed.
-  """
-  labels, predictions = confusion_matrix.remove_squeezable_dimensions(
-      labels, predictions)
-  predictions.get_shape().assert_is_compatible_with(labels.get_shape())
-
-  if weights is not None:
-    weights = ops.convert_to_tensor(weights)
-    predictions_shape = predictions.get_shape()
-    predictions_rank = predictions_shape.ndims
-    weights_shape = weights.get_shape()
-    weights_rank = weights_shape.ndims
-
-    if (predictions_rank is not None) and (weights_rank is not None):
-      # Use static rank.
-      if weights_rank - predictions_rank == 1:
-        weights = array_ops.squeeze(weights, [-1])
-    elif (weights_rank is
-          None) or (weights_shape.dims[-1].is_compatible_with(1)):
-      # Use dynamic rank
-      weights = control_flow_ops.cond(
-          math_ops.equal(
-              array_ops.rank(weights),
-              math_ops.add(array_ops.rank(predictions), 1)),
-          lambda: array_ops.squeeze(weights, [-1]), lambda: weights)
-  return predictions, labels, weights
-
-
 __all__ = [
     'aggregate_metric_map',
     'aggregate_metrics',
diff --git a/tensorflow/contrib/metrics/python/ops/metric_ops_test.py b/tensorflow/contrib/metrics/python/ops/metric_ops_test.py
index 6e038481e3..24d82a7eee 100644
--- a/tensorflow/contrib/metrics/python/ops/metric_ops_test.py
+++ b/tensorflow/contrib/metrics/python/ops/metric_ops_test.py
@@ -2131,7 +2131,7 @@ class StreamingPrecisionRecallAtEqualThresholdsTest(test.TestCase):
             'recall': [1.0, 1.0, 0.0],
             'thresholds': [0.0, 0.5, 1.0],
         },
-        weights=[0.0, 0.5, 2.0, 0.0, 0.5, 1.0])
+        weights=[[0.0, 0.5, 2.0, 0.0, 0.5, 1.0]])
 
 
 class StreamingSpecificityAtSensitivityTest(test.TestCase):
diff --git a/tensorflow/python/ops/metrics_impl.py b/tensorflow/python/ops/metrics_impl.py
index 1858834f97..68ec3c0101 100644
--- a/tensorflow/python/ops/metrics_impl.py
+++ b/tensorflow/python/ops/metrics_impl.py
@@ -52,10 +52,14 @@ def _local_variable(initial_value, validate_shape=True, name=None):
 
 
 def _remove_squeezable_dimensions(predictions, labels, weights):
-  """Internal version of `remove_squeezable_dimensions` which handles weights.
+  """Squeeze or expand last dim if needed.
 
-  Squeezes `predictions` and `labels` if their rank differs by 1.
-  Squeezes `weights` if its rank is 1 more than the new rank of `predictions`
+  Squeezes last dim of `predictions` or `labels` if their rank differs by 1
+  (using confusion_matrix.remove_squeezable_dimensions).
+  Squeezes or expands last dim of `weights` if its rank differs by 1 from the
+  new rank of `predictions`.
+
+  If `weights` is scalar, it is kept scalar.
 
   This will use static shape if available. Otherwise, it will add graph
   operations, which could result in a performance hit.
@@ -63,12 +67,12 @@ def _remove_squeezable_dimensions(predictions, labels, weights):
   Args:
     predictions: Predicted values, a `Tensor` of arbitrary dimensions.
     labels: Optional label `Tensor` whose dimensions match `predictions`.
-    weights: Optional weight `Tensor`. It will be squeezed if its rank is 1
-      more than the new rank of `predictions`
+    weights: Optional weight scalar or `Tensor` whose dimensions match
+      `predictions`.
 
   Returns:
-    Tuple of `predictions`, `labels` and `weights`, possibly with the last
-    dimension squeezed.
+    Tuple of `predictions`, `labels` and `weights`. Each of them possibly has
+    the last dimension squeezed, `weights` could be extended by one dimension.
   """
   predictions = ops.convert_to_tensor(predictions)
   if labels is not None:
-- 
GitLab


From 3fae77a3d8f4c921d6e45b9a87a5637fd53b3071 Mon Sep 17 00:00:00 2001
From: Jacques Pienaar <jpienaar@google.com>
Date: Thu, 26 Oct 2017 12:51:10 -0700
Subject: [PATCH 822/909] [TFXLA] Add grad_state for TPUReplicateContext

PiperOrigin-RevId: 173577089
---
 tensorflow/contrib/tpu/python/tpu/tpu.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/tensorflow/contrib/tpu/python/tpu/tpu.py b/tensorflow/contrib/tpu/python/tpu/tpu.py
index fa5760953d..338a4304f3 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu.py
@@ -146,6 +146,14 @@ class TPUReplicateContext(control_flow_ops.ControlFlowContext):
     if self._outer_context:
       self._outer_context.AddInnerOp(op)
 
+  @property
+  def grad_state(self):
+    # Define the gradient loop state associated with the TPUReplicateContext to
+    # be None as the TPUReplicateContext does not get nested nor does the
+    # grad_state outside the TPUReplicateContext affect the graph inside so the
+    # grad_state should be as if this is the top-level gradient state.
+    return None
+
 
 def replicate(computation,
               inputs=None,
-- 
GitLab


From 3db4df07101b013eacc37de0e2ff990bdadb3219 Mon Sep 17 00:00:00 2001
From: Mustafa Ispir <ispir@google.com>
Date: Thu, 26 Oct 2017 13:26:50 -0700
Subject: [PATCH 823/909] fixed eval num_epochs.

PiperOrigin-RevId: 173581816
---
 tensorflow/docs_src/get_started/get_started.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/docs_src/get_started/get_started.md b/tensorflow/docs_src/get_started/get_started.md
index 67fddfe809..8409962744 100644
--- a/tensorflow/docs_src/get_started/get_started.md
+++ b/tensorflow/docs_src/get_started/get_started.md
@@ -453,7 +453,7 @@ input_fn = tf.estimator.inputs.numpy_input_fn(
 train_input_fn = tf.estimator.inputs.numpy_input_fn(
     {"x": x_train}, y_train, batch_size=4, num_epochs=1000, shuffle=False)
 eval_input_fn = tf.estimator.inputs.numpy_input_fn(
-    {"x": x_eval}, y_eval, batch_size=4, num_epochs=1000, shuffle=False)
+    {"x": x_eval}, y_eval, batch_size=4, num_epochs=1, shuffle=False)
 
 # train
 estimator.train(input_fn=input_fn, steps=1000)
-- 
GitLab


From 24105d9a83dff9b46326373a7c4fd7fd254f32f0 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Thu, 26 Oct 2017 13:42:19 -0700
Subject: [PATCH 824/909] [XLA] Merge large parameter-shaped tuples into their
 users in DOT graphs.

It's common to have a while loop whose body computation has one
parameter, a giant tuple.  Then we have to draw edges from that tuple to
a bunch of get-tuple-element nodes, which are used throughout the while
loop's body.  This results in many long, difficult-to-follow edges.

In practice, the big tuple really functions as N separate parameters.
This patch represents it this way visually, erasing the big tuple and
replacing it with the get-tuple-element users, which we style like
parameters.

Future work is figuring out how to do something similar for the tuple op
at the bottom of while loop bodies.  This will be harder, because it
will require breaking the invariant that every HLO corresponds to zero
or one nodes in the dot graph.

PiperOrigin-RevId: 173584100
---
 .../compiler/xla/service/hlo_graph_dumper.cc  | 82 +++++++++++++++++--
 1 file changed, 73 insertions(+), 9 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
index b11b129c14..7b9cbeb6f4 100644
--- a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
+++ b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
@@ -342,6 +342,11 @@ class HloDotDumper {
 
   bool ShouldShowSubcomputation(const HloComputation* subcomp);
   bool ShouldShowFusionSubcomputation(const HloInstruction* instr);
+
+  // We omit some nodes from the graph, instead drawing them inlined into the
+  // nodes that use them.
+  bool ShouldMergeIntoUsers(const HloInstruction* instr) const;
+
   string DumpSubcomputation(const HloComputation* subcomp,
                             const HloInstruction* parent_instr);
   string DumpComputation(const HloComputation* comp);
@@ -352,7 +357,7 @@ class HloDotDumper {
   string GetInstructionNodeLabel(const HloInstruction* instr);
   string GetInstructionNodeMetadata(const HloInstruction* instr);
   string GetInstructionNodeExtraInfo(const HloInstruction* instr);
-  string GetInstructionNodeInlinedConstants(const HloInstruction* instr);
+  string GetInstructionNodeInlinedOperands(const HloInstruction* instr);
   void AddInstructionIncomingEdges(const HloInstruction* instr);
 
   // If instr has just one computation and it's trivial (e.g. "return param0 +
@@ -668,12 +673,42 @@ string HloDotDumper::DumpRootTag() {
                 to_id, node_body, node_shape, NodeColorAttributes(color));
 }
 
+bool HloDotDumper::ShouldMergeIntoUsers(const HloInstruction* instr) const {
+  // If a node:
+  //
+  //  - is a tuple-shaped parameter,
+  //  - is not a parameter to a fusion node,
+  //  - has at least kMinUsersToOmit users shown, and
+  //  - all of the shown users are get-tuple-elements,
+  //
+  // then we omit it from the graph, merging it with its users.
+  //
+  // This helps us handle the common case where a while loop body has one big
+  // tuple-shaped parameter.
+  const int kMinUsersToOmit = 3;
+  return instr->opcode() == HloOpcode::kParameter &&
+         ShapeUtil::IsTuple(instr->shape()) && !instr->IsFused() &&
+         std::count_if(instr->users().begin(), instr->users().end(),
+                       [&](const HloInstruction* user) {
+                         return filter_.Show(user);
+                       }) > kMinUsersToOmit &&
+         std::all_of(instr->users().begin(), instr->users().end(),
+                     [&](const HloInstruction* user) {
+                       return !filter_.Show(user) ||
+                              user->opcode() == HloOpcode::kGetTupleElement;
+                     });
+}
+
 string HloDotDumper::DumpInstruction(const HloInstruction* instr) {
   // We don't display constants as separate nodes; they're merged into their
   // users.
   if (instr->opcode() == HloOpcode::kConstant) {
     return "";
   }
+  // Skip this node if it's merged into its users.
+  if (ShouldMergeIntoUsers(instr)) {
+    return "";
+  }
   // Omit the fusion node if its subcomputation is drawn, since the
   // subcomputation will be drawn inline.
   if (instr->opcode() == HloOpcode::kFusion &&
@@ -689,7 +724,7 @@ string HloDotDumper::DumpInstruction(const HloInstruction* instr) {
   string node_label = GetInstructionNodeLabel(instr);
   string node_metadata = GetInstructionNodeMetadata(instr);
   string extra_info = GetInstructionNodeExtraInfo(instr);
-  string inlined_constants = GetInstructionNodeInlinedConstants(instr);
+  string inlined_constants = GetInstructionNodeInlinedOperands(instr);
   string trivial_subcomputation = GetInstructionTrivialComputationStr(instr);
   AddInstructionIncomingEdges(instr);
 
@@ -717,7 +752,7 @@ string HloDotDumper::DumpInstruction(const HloInstruction* instr) {
                 NodeColorAttributes(color));
 }
 
-string HloDotDumper::GetInstructionNodeInlinedConstants(
+string HloDotDumper::GetInstructionNodeInlinedOperands(
     const HloInstruction* instr) {
   auto stringify_constant = [](const HloInstruction* constant) {
     if (ShapeUtil::IsEffectiveScalar(constant->shape())) {
@@ -746,16 +781,44 @@ string HloDotDumper::GetInstructionNodeInlinedConstants(
   std::vector<string> lines;
   for (int64 i = 0; i < instr->operand_count(); ++i) {
     const HloInstruction* operand = instr->operand(i);
-    if (operand->opcode() != HloOpcode::kConstant) {
-      continue;
+    optional<string> operand_str;
+    if (operand->opcode() == HloOpcode::kConstant) {
+      operand_str = stringify_constant(operand);
+    } else if (ShouldMergeIntoUsers(operand)) {
+      // Special case: If the operand is a parameter, use its parameter number
+      // rather than its name, because that's generally how people think of the
+      // node.
+      if (operand->opcode() == HloOpcode::kParameter) {
+        operand_str = Printf("Parameter %lld", operand->parameter_number());
+      } else {
+        operand_str = operand->name();
+      }
+    }
+
+    if (operand_str) {
+      if (instr->operand_count() > 1) {
+        lines.push_back(Printf("<b>operand %lld</b> = %s", i, *operand_str));
+      } else {
+        lines.push_back(Printf("<b>operand</b> = %s", *operand_str));
+      }
     }
-    lines.push_back(
-        Printf("<b>operand %lld</b> = %s", i, stringify_constant(operand)));
   }
   return Join(lines, "<br/>");
 }
 
 ColorScheme HloDotDumper::GetInstructionColor(const HloInstruction* instr) {
+  const auto kParameterColor = kOrange;
+
+  // Special case: If this instruction has a parameter merged into it, paint it
+  // the same color as a parameter.
+  if (std::any_of(instr->operands().begin(), instr->operands().end(),
+                  [&](const HloInstruction* operand) {
+                    return operand->opcode() == HloOpcode::kParameter &&
+                           ShouldMergeIntoUsers(operand);
+                  })) {
+    return kParameterColor;
+  }
+
   // Pick different colors or shapes for instructions which are particularly
   // expensive (eg, dot) and those which are unusual in some way or unique
   // (eg, parameter).
@@ -845,7 +908,7 @@ ColorScheme HloDotDumper::GetInstructionColor(const HloInstruction* instr) {
     case HloOpcode::kReducePrecision:
       return kRed;
     case HloOpcode::kParameter:
-      return kOrange;
+      return kParameterColor;
     case HloOpcode::kBatchNormTraining:
     case HloOpcode::kBatchNormInference:
     case HloOpcode::kBatchNormGrad:
@@ -1016,7 +1079,8 @@ void HloDotDumper::AddInstructionIncomingEdges(const HloInstruction* instr) {
         ShouldShowFusionSubcomputation(from)) {
       from = from->fused_expression_root();
     }
-    if (!filter_.Show(from) || from->opcode() == HloOpcode::kConstant) {
+    if (!filter_.Show(from) || from->opcode() == HloOpcode::kConstant ||
+        ShouldMergeIntoUsers(from)) {
       return;
     }
     VLOG(2) << "Adding edge from " << from->name() << " to " << to->name()
-- 
GitLab


From efe4c98b57baf2697c0c226157fe1efe8810f605 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 26 Oct 2017 13:54:59 -0700
Subject: [PATCH 825/909] Add tests for OIHW filter format.

PiperOrigin-RevId: 173586221
---
 .../fused_conv2d_bias_activation_op_test.py   | 23 +++++++++++++++----
 1 file changed, 18 insertions(+), 5 deletions(-)

diff --git a/tensorflow/contrib/fused_conv/python/ops/fused_conv2d_bias_activation_op_test.py b/tensorflow/contrib/fused_conv/python/ops/fused_conv2d_bias_activation_op_test.py
index 3b8f7d6ed7..2a18f3eeec 100644
--- a/tensorflow/contrib/fused_conv/python/ops/fused_conv2d_bias_activation_op_test.py
+++ b/tensorflow/contrib/fused_conv/python/ops/fused_conv2d_bias_activation_op_test.py
@@ -159,9 +159,12 @@ class FusedConv2DBiasActivationTest(test.TestCase):
   def _DtypesToTest(self, use_gpu):
     return [dtypes.float32]
 
+  def _FilterFormatsToTest(self, use_gpu):
+    return ["HWIO", "OIHW"]
+
   def _SetupValuesForDevice(self, tensor_in_sizes, filter_in_sizes, bias,
                             strides, padding, activation_mode, data_format,
-                            dtype):
+                            filter_format, dtype):
     """Verifies the output values of the convolution function.
 
     Args:
@@ -174,6 +177,7 @@ class FusedConv2DBiasActivationTest(test.TestCase):
       padding: Padding type.
       activation_mode: Activation mode.
       data_format: Format of the data tensors.
+      filter_format: Filter format to use for the fused convolution.
       dtype: Data type for inputs and outputs.
     Returns:
       Symbolic tensor value and reference value that can be used to
@@ -192,6 +196,9 @@ class FusedConv2DBiasActivationTest(test.TestCase):
     with self.test_session(use_gpu=True):
       t1 = constant_op.constant(x1, shape=tensor_in_sizes, dtype=dtype)
       t2 = constant_op.constant(x2, shape=filter_in_sizes, dtype=dtype)
+      fused_t2 = t2
+      if filter_format == "OIHW":
+        fused_t2 = HwioToOihw(t2)
       t3 = constant_op.constant(x3, shape=[bias_size], dtype=dtype)
       strides = [1] + strides + [1]
       if data_format == "NCHW":
@@ -199,11 +206,12 @@ class FusedConv2DBiasActivationTest(test.TestCase):
         strides = test_util.NHWCToNCHW(strides)
       output = fused_conv2d_bias_activation_op.fused_conv2d_bias_activation(
           t1,
-          t2,
+          fused_t2,
           t3,
           strides=strides,
           padding=padding,
           data_format=data_format,
+          filter_format=filter_format,
           activation_mode=activation_mode)
       ref_conv_output = nn_ops.conv2d(
           t1, t2, strides=strides, padding=padding, data_format=data_format)
@@ -268,9 +276,10 @@ class FusedConv2DBiasActivationTest(test.TestCase):
     ref_tensors = []
     for (data_format, use_gpu) in GetTestConfigs():
       for dtype in self._DtypesToTest(use_gpu):
-        result, expected = self._SetupValuesForDevice(
-            tensor_in_sizes, filter_in_sizes, bias, strides, padding, "Relu",
-            data_format, dtype)
+        for filter_format in self._FilterFormatsToTest(use_gpu):
+          result, expected = self._SetupValuesForDevice(
+              tensor_in_sizes, filter_in_sizes, bias, strides, padding, "Relu",
+              data_format, filter_format, dtype)
         tensors.append(result)
         ref_tensors.append(expected)
       with self.test_session() as sess:
@@ -607,6 +616,10 @@ def NchwToNchwVectC(in_tensor):
   return array_ops.transpose(t, [0, 1, 3, 4, 2])
 
 
+def HwioToOihw(in_tensor):
+  return array_ops.transpose(in_tensor, [3, 2, 0, 1])
+
+
 def SimulateFusedConv2dBiasActivationInt8(conv_input_scale, conv_input, kernel,
                                           padding, strides, side_input_scale,
                                           side_input, biases):
-- 
GitLab


From 00adbdb9bc4650e3fa594cfdd408b30cf38243bf Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 26 Oct 2017 13:55:54 -0700
Subject: [PATCH 826/909] Object-orient Metrics fixes: * Add examples for usage
 in both eager and graph execution. * Remove references to Evaluator in the
 doc string, to support stand-alone   uses. * Remove the need to override
 reset() if you use non-zero initialization of   some variable, by recording
 the initial values of variables. * Merge reset() into init_variables().

PiperOrigin-RevId: 173586350
---
 .../contrib/eager/python/metrics_impl.py      | 67 ++++++++++++-------
 .../contrib/eager/python/metrics_test.py      |  9 +++
 2 files changed, 52 insertions(+), 24 deletions(-)

diff --git a/tensorflow/contrib/eager/python/metrics_impl.py b/tensorflow/contrib/eager/python/metrics_impl.py
index 6af0d65e08..6c55ac2f5d 100644
--- a/tensorflow/contrib/eager/python/metrics_impl.py
+++ b/tensorflow/contrib/eager/python/metrics_impl.py
@@ -38,9 +38,30 @@ _to_replace = re.compile("[^A-Za-z0-9.]")
 class Metric(object):
   """A metric holds state for aggregating statistics over an evaluation run.
 
-  Users will use Evaluator.add_metric() to add Metric objects to their
-  evaluation, call them in each step (treating the object as a callable),
-  and then use Evaluator.all_metric_results() at the end.
+  Example use with eager execution:
+
+  ```python
+  m = SomeMetric(...)
+  for input in ...:
+    m(input)
+  print(m.result())
+  ```
+
+  Example use with graph execution:
+
+  ```python
+  m = SomeMetric(...)
+  m_placeholder = tf.placeholder(...)
+  m_update = m(m_placeholder)
+  # Variables defined in first call, so get the initialization op afterwards.
+  m_init = m.init_variables()  # or tf.global_variables_initializer()
+  m_result = m.result()
+  with tf.Session() as sess:
+    sess.run(m_init)
+    for input in ...:
+      sess.run(m_update, feed_dict={m_placeholder: input})
+    print(sess.run(m_result))
+  ```
 
   Descendants will implement:
   * `build()`: All variables should be created in this method, by calling
@@ -52,18 +73,16 @@ class Metric(object):
   * `result()`: Computes and returns a final value for the metric
     from the variables in `self`.
 
-  Decendants may override, but usually won't need to:
-  * `aggregate()`: Adds in the state from a list of metrics of the same type
-    as `self`.  (Default is to sum all the variables.)
-  * `reset()`: Reset all variables to their initial state. (Default is to
-    zero all the variables.)
-  Note that users should not call `aggregate()` or `reset()`, they are for
-  use by TensorFlow infrastructure.
+  Decendants may override `aggregate()`, but usually won't need to.  It
+  adds in the state from a list of metrics of the same type as `self`.
+  (Default is to sum all the variables.) Note that users should not call
+  `aggregate()`, it is for use by TensorFlow infrastructure.
   """
 
   def __init__(self, name=None):
     self._built = False
     self._vars = []
+    self._initial_values = {}
     self._updates = []
     name = name or self.__class__.__name__
     # Replace things like spaces in name to create a valid scope name.
@@ -109,16 +128,22 @@ class Metric(object):
     return self._vars
 
   def init_variables(self):
-    """Return an op for initializing this Metric's variables.
+    """Initializes this Metric's variables.
 
-    Only for graph execution. Should be called after variables are created
-    in the first execution of __call__().
+    Should be called after variables are created in the first execution
+    of `__call__()`. If using graph execution, the return value should be
+    `run()` in a session before running the op returned by `__call__()`.
+    (See example above.)
 
     Returns:
-      An op to run.
+      If using graph execution, this returns an op to perform the
+      initialization. Under eager execution, the variables are reset to their
+      initial values as a side effect and this function returns None.
     """
-    assert context.in_graph_mode()
-    return control_flow_ops.group([v.initializer for v in self._vars])
+    if context.in_graph_mode():
+      return control_flow_ops.group([v.initializer for v in self._vars])
+    for v in self._vars:
+      v.assign(self._initial_values[v])
 
   # ---- To be implemented by descendants ---
   def build(self, *args, **kwargs):
@@ -193,14 +218,6 @@ class Metric(object):
       self._vars[i].assign_add(math_ops.add_n([m._vars[i] for m in metrics]))
     # pylint: enable=protected-access
 
-  def reset(self):
-    """Reset this metric to a freshly initialized state.
-
-    Default implementation zeros all the metric variables.
-    """
-    for v in self._vars:
-      v.assign(math_ops.zeros_like(v))
-
   # ---- For use by descendants ---
   def add_variable(self, name, shape=None, dtype=None, initializer=None):
     """***Only for use by descendants of Metric***."""
@@ -209,6 +226,8 @@ class Metric(object):
     v = variable_scope.get_variable(name, shape, dtype, initializer,
                                     trainable=False, use_resource=True)
     self._vars.append(v)
+    if context.in_eager_mode():
+      self._initial_values[v] = v.value()
     return v
 
 
diff --git a/tensorflow/contrib/eager/python/metrics_test.py b/tensorflow/contrib/eager/python/metrics_test.py
index 3ecbaeae69..a8377a0660 100644
--- a/tensorflow/contrib/eager/python/metrics_test.py
+++ b/tensorflow/contrib/eager/python/metrics_test.py
@@ -44,6 +44,15 @@ class MetricsTest(test.TestCase):
     self.assertEqual(dtypes.float64, m.dtype)
     self.assertEqual(dtypes.float64, m.result().dtype)
 
+  def testInitVariables(self):
+    m = metrics.Mean()
+    m([1, 10, 100, 1000])
+    m([10000.0, 100000.0])
+    self.assertEqual(111111.0/6, m.result().numpy())
+    m.init_variables()
+    m(7)
+    self.assertEqual(7.0, m.result().numpy())
+
   def testWriteSummaries(self):
     m = metrics.Mean()
     m([1, 10, 100])
-- 
GitLab


From 68b00b0be7e356ee30bf86a8a4a7807fa736dd6b Mon Sep 17 00:00:00 2001
From: Shanqing Cai <cais@google.com>
Date: Thu, 26 Oct 2017 14:15:17 -0700
Subject: [PATCH 827/909] TFE: Add compatibility doc strings to functional
 layers

PiperOrigin-RevId: 173589146
---
 tensorflow/python/layers/convolutional.py | 24 +++++++++++++++++++++++
 tensorflow/python/layers/core.py          |  8 ++++++++
 tensorflow/python/layers/maxout.py        |  4 ++++
 tensorflow/python/layers/normalization.py |  5 +++++
 tensorflow/python/layers/pooling.py       | 24 +++++++++++++++++++++++
 5 files changed, 65 insertions(+)

diff --git a/tensorflow/python/layers/convolutional.py b/tensorflow/python/layers/convolutional.py
index 6b371c618f..c9bfafaee1 100644
--- a/tensorflow/python/layers/convolutional.py
+++ b/tensorflow/python/layers/convolutional.py
@@ -386,6 +386,10 @@ def conv1d(inputs,
 
   Raises:
     ValueError: if eager execution is enabled.
+
+  @compatibility(eager)
+  Not compatible with eager execution. Use `tf.layers.Conv1D` instead.
+  @end_compatibility
   """
   if context.in_eager_mode():
     raise ValueError(
@@ -593,6 +597,10 @@ def conv2d(inputs,
 
   Raises:
     ValueError: if eager execution is enabled.
+
+  @compatibility(eager)
+  Not compatible with eager execution. Use `tf.layers.Conv2D` instead.
+  @end_compatibility
   """
   if context.in_eager_mode():
     raise ValueError(
@@ -802,6 +810,10 @@ def conv3d(inputs,
 
   Raises:
     ValueError: if eager execution is enabled.
+
+  @compatibility(eager)
+  Not compatible with eager execution. Use `tf.layers.Conv3D` instead.
+  @end_compatibility
   """
   if context.in_eager_mode():
     raise ValueError(
@@ -1128,6 +1140,10 @@ def separable_conv2d(inputs,
 
   Raises:
     ValueError: if eager execution is enabled.
+
+  @compatibility(eager)
+  Not compatible with eager execution. Use `tf.layers.SeparableConv2d` instead.
+  @end_compatibility
   """
   if context.in_eager_mode():
     raise ValueError(
@@ -1430,6 +1446,10 @@ def conv2d_transpose(inputs,
 
   Raises:
     ValueError: if eager execution is enabled.
+
+  @compatibility(eager)
+  Not compatible with eager execution. Use `tf.layers.Conv2DTranspose` instead.
+  @end_compatibility
   """
   if context.in_eager_mode():
     raise ValueError(
@@ -1748,6 +1768,10 @@ def conv3d_transpose(inputs,
 
   Raises:
     ValueError: if eager execution is enabled.
+
+  @compatibility(eager)
+  Not compatible with eager execution. Use `tf.layers.Conv3DTranspose` instead.
+  @end_compatibility
   """
   if context.in_eager_mode():
     raise ValueError(
diff --git a/tensorflow/python/layers/core.py b/tensorflow/python/layers/core.py
index 457bee5cff..b30e5f2074 100644
--- a/tensorflow/python/layers/core.py
+++ b/tensorflow/python/layers/core.py
@@ -234,6 +234,10 @@ def dense(
 
   Raises:
     ValueError: if eager execution is enabled.
+
+  @compatibility(eager)
+  Not compatible with eager execution. Use `tf.layers.Dense` instead.
+  @end_compatibility
   """
   if context.in_eager_mode():
     raise ValueError(
@@ -343,6 +347,10 @@ def dropout(inputs,
 
   Raises:
     ValueError: if eager execution is enabled.
+
+  @compatibility(eager)
+  Not compatible with eager execution. Use `tf.layers.Dropout` instead.
+  @end_compatibility
   """
   if context.in_eager_mode():
     raise ValueError(
diff --git a/tensorflow/python/layers/maxout.py b/tensorflow/python/layers/maxout.py
index fa6c8cee97..61cfd7f45c 100644
--- a/tensorflow/python/layers/maxout.py
+++ b/tensorflow/python/layers/maxout.py
@@ -50,6 +50,10 @@ def maxout(inputs, num_units, axis=-1, name=None):
 
    Raises:
     ValueError: if num_units is not multiple of number of features.
+
+  @compatibility(eager)
+  Not compatible with eager execution. Use `tf.layers.MaxOut` instead.
+  @end_compatibility
   """
   if context.in_eager_mode():
     raise ValueError(
diff --git a/tensorflow/python/layers/normalization.py b/tensorflow/python/layers/normalization.py
index 899be08020..5997d652aa 100644
--- a/tensorflow/python/layers/normalization.py
+++ b/tensorflow/python/layers/normalization.py
@@ -720,6 +720,11 @@ def batch_normalization(inputs,
 
   Raises:
     ValueError: if eager execution is enabled.
+
+  @compatibility(eager)
+  Not compatible with eager execution. Use `tf.layers.BatchNormalization`
+  instead.
+  @end_compatibility
   """
   if context.in_eager_mode():
     raise ValueError(
diff --git a/tensorflow/python/layers/pooling.py b/tensorflow/python/layers/pooling.py
index ec02ab032d..b3535c4410 100644
--- a/tensorflow/python/layers/pooling.py
+++ b/tensorflow/python/layers/pooling.py
@@ -148,6 +148,10 @@ def average_pooling1d(inputs, pool_size, strides,
 
   Raises:
     ValueError: if eager execution is enabled.
+
+  @compatibility(eager)
+  Not compatible with eager execution. Use `tf.layers.AveragePooling1D` instead.
+  @end_compatibility
   """
   if context.in_eager_mode():
     raise ValueError(
@@ -217,6 +221,10 @@ def max_pooling1d(inputs, pool_size, strides,
 
   Raises:
     ValueError: if eager execution is enabled.
+
+  @compatibility(eager)
+  Not compatible with eager execution. Use `tf.layers.MaxPooling1D` instead.
+  @end_compatibility
   """
   if context.in_eager_mode():
     raise ValueError(
@@ -362,6 +370,10 @@ def average_pooling2d(inputs,
 
   Raises:
     ValueError: if eager execution is enabled.
+
+  @compatibility(eager)
+  Not compatible with eager execution. Use `tf.layers.AveragePooling2D` instead.
+  @end_compatibility
   """
   if context.in_eager_mode():
     raise ValueError(
@@ -434,6 +446,10 @@ def max_pooling2d(inputs,
 
   Raises:
     ValueError: if eager execution is enabled.
+
+  @compatibility(eager)
+  Not compatible with eager execution. Use `tf.layers.MaxPooling2D` instead.
+  @end_compatibility
   """
   if context.in_eager_mode():
     raise ValueError(
@@ -592,6 +608,10 @@ def average_pooling3d(inputs,
 
   Raises:
     ValueError: if eager execution is enabled.
+
+  @compatibility(eager)
+  Not compatible with eager execution. Use `tf.layers.AveragePooling3D` instead.
+  @end_compatibility
   """
   if context.in_eager_mode():
     raise ValueError(
@@ -668,6 +688,10 @@ def max_pooling3d(inputs,
 
   Raises:
     ValueError: if eager execution is enabled.
+
+  @compatibility(eager)
+  Not compatible with eager execution. Use `tf.layers.MaxPooling3D` instead.
+  @end_compatibility
   """
   if context.in_eager_mode():
     raise ValueError(
-- 
GitLab


From 2cbe1ffd23a0214492d182935212fc7c613e4a05 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Thu, 26 Oct 2017 14:16:10 -0700
Subject: [PATCH 828/909] [XLA] Comment fixes (spelling, grammar, and a bit of
 correctness).

PiperOrigin-RevId: 173589267
---
 tensorflow/compiler/xla/array.h                           | 2 +-
 tensorflow/compiler/xla/service/algebraic_simplifier.cc   | 6 ++++--
 tensorflow/compiler/xla/service/buffer_assignment_test.cc | 4 ++--
 tensorflow/compiler/xla/service/hlo_constant_folding.cc   | 4 ++--
 tensorflow/compiler/xla/service/hlo_instruction.cc        | 8 ++++----
 .../compiler/xla/service/tuple_points_to_analysis.h       | 4 ++--
 tensorflow/compiler/xla/tests/reduce_test.cc              | 8 ++++----
 7 files changed, 19 insertions(+), 17 deletions(-)

diff --git a/tensorflow/compiler/xla/array.h b/tensorflow/compiler/xla/array.h
index 3be7060a83..2aedafb91f 100644
--- a/tensorflow/compiler/xla/array.h
+++ b/tensorflow/compiler/xla/array.h
@@ -302,7 +302,7 @@ class Array {
   }
 
   // Advances the specified set of indexes and returns true if we haven't
-  // wrapped around (i.e. result isnt {0, 0, ...}).
+  // wrapped around (i.e. result isn't {0, 0, ...}).
   bool next_index(std::vector<int64>* index) const {
     CHECK_EQ(index->size(), sizes_.size());
     for (int64 i = sizes_.size() - 1; i >= 0; --i) {
diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier.cc b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
index ae26cc2d99..35ab4d89cc 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
@@ -1941,7 +1941,7 @@ Status AlgebraicSimplifierVisitor::HandleWhile(HloInstruction* while_op) {
     return Status::OK();
   }
 
-  // Remove while loops with static trip count of 1.
+  // Remove while loops with static trip count of 0.
   optional<int64> trip_count = GetLoopTripCount(while_op);
   if (trip_count && *trip_count == 0) {
     // The loop never executes, so the value of the loop is the value of its
@@ -1956,8 +1956,10 @@ Status AlgebraicSimplifierVisitor::HandleWhile(HloInstruction* while_op) {
     changed_ = true;
     return Status::OK();
   }
+
+  // Transform while loops with static trip count of 1 into a call op, then
+  // inline the call.
   if (trip_count && *trip_count == 1) {
-    // Transform the while loop into a call op, then inline the call.
     auto computation = while_op->parent();
     auto call_op = computation->AddInstruction(HloInstruction::CreateCall(
         while_op->shape(), while_op->operands(), while_op->while_body()));
diff --git a/tensorflow/compiler/xla/service/buffer_assignment_test.cc b/tensorflow/compiler/xla/service/buffer_assignment_test.cc
index e3378a756b..89410f42bd 100644
--- a/tensorflow/compiler/xla/service/buffer_assignment_test.cc
+++ b/tensorflow/compiler/xla/service/buffer_assignment_test.cc
@@ -1179,7 +1179,7 @@ TEST_F(BufferAssignmentTest, TupleCallAsOutput) {
   auto assignment = RunBufferAssignment(module.get());
 
   EXPECT_EQ(3, assignment->Allocations().size());
-  // Buffers for call are co-located with the sub-computation.
+  // Buffers for call are colocated with the sub-computation.
   EXPECT_EQ(GetAllocation(*assignment, call, /*index=*/{}),
             GetAllocation(*assignment, sub_tuple, /*index=*/{}));
   EXPECT_EQ(GetAllocation(*assignment, call, /*index=*/{0}),
@@ -1238,7 +1238,7 @@ TEST_F(BufferAssignmentTest, TupleChainedCallAsOutput) {
 
   auto assignment = RunBufferAssignment(module.get());
 
-  // Buffers for call are co-located with the sub-computations.
+  // Buffers for call are colocated with the sub-computations.
   EXPECT_EQ(GetAllocation(*assignment, a_call, /*index=*/{}),
             GetAllocation(*assignment, b_call, /*index=*/{}));
   EXPECT_EQ(GetAllocation(*assignment, b_call, /*index=*/{}),
diff --git a/tensorflow/compiler/xla/service/hlo_constant_folding.cc b/tensorflow/compiler/xla/service/hlo_constant_folding.cc
index c05bbeb5c9..53450991b6 100644
--- a/tensorflow/compiler/xla/service/hlo_constant_folding.cc
+++ b/tensorflow/compiler/xla/service/hlo_constant_folding.cc
@@ -49,8 +49,8 @@ StatusOr<bool> HloConstantFolding::Run(HloModule* module) {
         continue;
       }
       // Skip Constant, Parameter, Reduce operation.
-      // TODO(b/35975797): Enable Reduce operation once arbitary computation are
-      // supported by the evaluator.
+      // TODO(b/35975797): Enable Reduce operation once arbitrary computation
+      // are supported by the evaluator.
       // TODO(b/64407269): Enable Tuple once the timeout issue is resolved.
       if (instruction->opcode() == HloOpcode::kParameter ||
           instruction->opcode() == HloOpcode::kConstant ||
diff --git a/tensorflow/compiler/xla/service/hlo_instruction.cc b/tensorflow/compiler/xla/service/hlo_instruction.cc
index d53ac221d1..272f573623 100644
--- a/tensorflow/compiler/xla/service/hlo_instruction.cc
+++ b/tensorflow/compiler/xla/service/hlo_instruction.cc
@@ -2665,10 +2665,10 @@ class HloInstruction::FusionReusesParamElements {
  public:
   using UseKind = HloInstruction::UseKind;
 
-  // We could rather iterate backwards thru fused_instructions_ here, as it is
-  // in reverse postorder, and compute whether each fused instruction reuses
-  // the value of this parameter, which would save stack space but not allow
-  // us to finish early if we find a reuse.
+  // We could rather iterate backwards through fused_instructions_ here, as it
+  // is in reverse postorder, and compute whether each fused instruction reuses
+  // the value of this parameter, which would save stack space but not allow us
+  // to finish early if we find a reuse.
   static UseKind Compute(int64 i, const HloInstruction& hlo) {
     tensorflow::gtl::FlatMap<const HloInstruction*, UseKind> memoization_cache;
     return ComputeInternal(i, hlo, &memoization_cache);
diff --git a/tensorflow/compiler/xla/service/tuple_points_to_analysis.h b/tensorflow/compiler/xla/service/tuple_points_to_analysis.h
index be45732952..30dabb56bd 100644
--- a/tensorflow/compiler/xla/service/tuple_points_to_analysis.h
+++ b/tensorflow/compiler/xla/service/tuple_points_to_analysis.h
@@ -44,7 +44,7 @@ namespace xla {
 
 // A class describing the source(s) of the Buffer(s) contained in the output of
 // a particular HLO instruction. The structure of PointsToSet mirrors the
-// structure of the instruction's shape which may be an arbitrary tree (eg, a
+// structure of the instruction's shape, which may be an arbitrary tree (eg, a
 // nested tuple). Each node in this tree corresponds to a single buffer in the
 // instruction's output and contains the set of Buffers which might define
 // the corresponding buffer.
@@ -148,7 +148,7 @@ class PointsToSet {
   ShapeTree<Elem> tree_;
 
   // PointsToSet contains references (const LogicalBuffer*) to elements within
-  // TuplePointsToAnalysis so disable copying.
+  // TuplePointsToAnalysis, so disable copying.
   TF_DISALLOW_COPY_AND_ASSIGN(PointsToSet);
 };
 
diff --git a/tensorflow/compiler/xla/tests/reduce_test.cc b/tensorflow/compiler/xla/tests/reduce_test.cc
index 794e5a4920..7bc3185c36 100644
--- a/tensorflow/compiler/xla/tests/reduce_test.cc
+++ b/tensorflow/compiler/xla/tests/reduce_test.cc
@@ -502,8 +502,8 @@ XLA_TEST_F(ReduceTest, AddReduce2DScalarToR0) {
   ComputationBuilder builder(client_, TestName());
   auto add = CreateScalarAddComputation(F32, &builder);
   auto scalar = builder.ConstantR0<float>(42.0);
-  auto broacasted = builder.Broadcast(scalar, {500, 500});
-  builder.Reduce(broacasted, builder.ConstantR0<float>(0.0f), add, {0, 1});
+  auto broadcasted = builder.Broadcast(scalar, {500, 500});
+  builder.Reduce(broadcasted, builder.ConstantR0<float>(0.0f), add, {0, 1});
 
   float expected = 42.0f * static_cast<float>(500 * 500);
   ComputeAndCompareR0<float>(&builder, expected, {}, ErrorSpec(0.0001));
@@ -514,8 +514,8 @@ XLA_TEST_F(ReduceTest, MaxReduce2DScalarToR0) {
   ComputationBuilder builder(client_, TestName());
   auto max = CreateScalarMaxComputation(F32, &builder);
   auto scalar = builder.ConstantR0<float>(42.0);
-  auto broacasted = builder.Broadcast(scalar, {500, 500});
-  builder.Reduce(broacasted, builder.ConstantR0<float>(0.0f), max, {0, 1});
+  auto broadcasted = builder.Broadcast(scalar, {500, 500});
+  builder.Reduce(broadcasted, builder.ConstantR0<float>(0.0f), max, {0, 1});
 
   float expected = 42.0f;
   ComputeAndCompareR0<float>(&builder, expected, {}, ErrorSpec(0.0001));
-- 
GitLab


From a8e1c3a88d5470e813de9a3642c1bf2f08baef5f Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Thu, 26 Oct 2017 14:34:55 -0700
Subject: [PATCH 829/909] [XLA:CPU] [XLA:GPU] Run DCE during fixed-point
 simplification pass.

PiperOrigin-RevId: 173592048
---
 tensorflow/compiler/xla/service/cpu/cpu_compiler.cc | 1 +
 tensorflow/compiler/xla/service/gpu/gpu_compiler.cc | 1 +
 2 files changed, 2 insertions(+)

diff --git a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
index 99b5035c2d..65e117e68f 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
@@ -282,6 +282,7 @@ Status CpuCompiler::RunHloPasses(HloModule* module, bool is_aot_compile) {
         [](const Shape&, const Shape&) { return false; },
         /*enable_dot_simplification=*/false);
     pass.AddPass<TupleSimplifier>();
+    pass.AddPass<HloDCE>();
     pass.AddPass<ReshapeMover>();
     pass.AddPass<HloConstantFolding>();
   }
diff --git a/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc b/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc
index 9c7ca9ea38..b5331fe4e2 100644
--- a/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc
+++ b/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc
@@ -151,6 +151,7 @@ tensorflow::Status OptimizeHloModule(
           /*is_layout_sensitive=*/false,
           [](const Shape&, const Shape&) { return false; });
       pass.AddPass<TupleSimplifier>();
+      pass.AddPass<HloDCE>();
       pass.AddPass<ReshapeMover>();
       pass.AddPass<HloConstantFolding>();
     }
-- 
GitLab


From 87a4991bfba80bec04946330620c35392dc4bbd8 Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Thu, 26 Oct 2017 14:37:54 -0700
Subject: [PATCH 830/909] Implements count_up_to for resource variables.

PiperOrigin-RevId: 173592524
---
 tensorflow/core/kernels/BUILD                 |  2 +-
 tensorflow/core/kernels/count_up_to_op.cc     | 55 +++++++++++++++++--
 tensorflow/core/ops/state_ops.cc              | 33 +++++++++++
 .../resource_variable_ops_test.py             | 15 +++++
 .../python/ops/resource_variable_ops.py       | 24 ++++++++
 tensorflow/python/ops/state_ops.py            | 22 ++++++++
 6 files changed, 146 insertions(+), 5 deletions(-)

diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 277b21f833..a3452f2f8c 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -3818,7 +3818,7 @@ STATE_DEPS = [
 tf_kernel_library(
     name = "count_up_to_op",
     prefix = "count_up_to_op",
-    deps = STATE_DEPS,
+    deps = STATE_DEPS + [":variable_ops"],
 )
 
 tf_kernel_library(
diff --git a/tensorflow/core/kernels/count_up_to_op.cc b/tensorflow/core/kernels/count_up_to_op.cc
index 040c40d606..9da0015fa2 100644
--- a/tensorflow/core/kernels/count_up_to_op.cc
+++ b/tensorflow/core/kernels/count_up_to_op.cc
@@ -14,6 +14,7 @@ limitations under the License.
 ==============================================================================*/
 
 #include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/kernels/variable_ops.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/platform/mutex.h"
 #include "tensorflow/core/platform/types.h"
@@ -54,10 +55,56 @@ class CountUpToOp : public OpKernel {
   T limit_;
 };
 
-#define REGISTER(TYPE)                                                \
-  REGISTER_KERNEL_BUILDER(                                            \
-      Name("CountUpTo").TypeConstraint<TYPE>("T").Device(DEVICE_CPU), \
-      CountUpToOp<TYPE>)
+template <class T>
+class ResourceCountUpToOp : public OpKernel {
+ public:
+  explicit ResourceCountUpToOp(OpKernelConstruction* context)
+      : OpKernel(context) {
+    OP_REQUIRES_OK(context, context->GetAttr("limit", &limit_));
+    OP_REQUIRES_OK(context, context->GetAttr("T", &dtype_));
+  }
+
+  void Compute(OpKernelContext* context) override {
+    Var* variable = nullptr;
+    OP_REQUIRES_OK(
+        context,
+        LookupResource<Var>(context, HandleFromInput(context, 0), &variable));
+    core::ScopedUnref s(variable);
+    mutex_lock l(*variable->mu());
+    Tensor before_increment = *variable->tensor();
+    OP_REQUIRES(
+        context, TensorShapeUtils::IsScalar(before_increment.shape()),
+        errors::InvalidArgument("input is not a scalar: ",
+                                before_increment.shape().DebugString()));
+    if (before_increment.scalar<T>()() >= limit_) {
+      context->SetStatus(errors::OutOfRange("Reached limit of ", limit_));
+      return;
+    }
+    // Allocate new buffer
+    AllocatorAttributes attr;
+    attr.set_gpu_compatible(true);
+    attr.set_nic_compatible(true);
+    PersistentTensor unused;
+    Tensor* tmp;
+    OP_REQUIRES_OK(context, context->allocate_persistent(
+                                dtype_, TensorShape({}), &unused, &tmp, attr));
+    *variable->tensor() = *tmp;
+    tmp->scalar<T>()() = before_increment.scalar<T>()() + 1;
+    context->set_output(0, before_increment);
+  }
+
+ private:
+  T limit_;
+  DataType dtype_;
+};
+
+#define REGISTER(TYPE)                                                        \
+  REGISTER_KERNEL_BUILDER(                                                    \
+      Name("CountUpTo").TypeConstraint<TYPE>("T").Device(DEVICE_CPU),         \
+      CountUpToOp<TYPE>)                                                      \
+  REGISTER_KERNEL_BUILDER(                                                    \
+      Name("ResourceCountUpTo").TypeConstraint<TYPE>("T").Device(DEVICE_CPU), \
+      ResourceCountUpToOp<TYPE>)
 
 REGISTER(int32);
 REGISTER(int64);
diff --git a/tensorflow/core/ops/state_ops.cc b/tensorflow/core/ops/state_ops.cc
index b86c0b3990..da5f091e9f 100644
--- a/tensorflow/core/ops/state_ops.cc
+++ b/tensorflow/core/ops/state_ops.cc
@@ -648,4 +648,37 @@ output: A copy of the input before increment. If nothing else modifies the
   input, the values produced will all be distinct.
 )doc");
 
+REGISTER_OP("ResourceCountUpTo")
+    .Input("resource: resource")
+    .Output("output: T")
+    .Attr("limit: int")
+    .Attr("T: {int32, int64}")
+    .SetShapeFn([](InferenceContext* c) {
+      auto* handle_data = c->input_handle_shapes_and_types(0);
+      if (handle_data == nullptr || handle_data->empty()) {
+        return errors::InvalidArgument("Handle has no shape/type information.");
+      }
+      shape_inference::ShapeAndType shape_and_type = (*handle_data)[0];
+      DataType value_dtype;
+      TF_RETURN_IF_ERROR(c->GetAttr("T", &value_dtype));
+      if (value_dtype != shape_and_type.dtype) {
+        return errors::InvalidArgument(
+            "Data types do not match: ", DataTypeString(value_dtype), " and ",
+            DataTypeString(shape_and_type.dtype));
+      }
+      ShapeHandle output;
+      TF_RETURN_IF_ERROR(c->WithRank(shape_and_type.shape, 0, &output));
+      c->set_output(0, output);
+      return Status::OK();
+    })
+    .Doc(R"doc(
+Increments variable pointed to by 'resource' until it reaches 'limit'.
+
+resource: Should be from a scalar `Variable` node.
+limit: If incrementing ref would bring it above limit, instead generates an
+  'OutOfRange' error.
+output: A copy of the input before increment. If nothing else modifies the
+  input, the values produced will all be distinct.
+)doc");
+
 }  // namespace tensorflow
diff --git a/tensorflow/python/kernel_tests/resource_variable_ops_test.py b/tensorflow/python/kernel_tests/resource_variable_ops_test.py
index 32edc5be7f..d8d1ba6bbc 100644
--- a/tensorflow/python/kernel_tests/resource_variable_ops_test.py
+++ b/tensorflow/python/kernel_tests/resource_variable_ops_test.py
@@ -30,6 +30,7 @@ from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import init_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import resource_variable_ops
+from tensorflow.python.ops import state_ops
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import test
@@ -189,6 +190,20 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase):
         resource_variable_ops.ResourceVariable(
             1.0, name="handle-numpy").handle.numpy()
 
+  def testCountUpTo(self):
+    with context.eager_mode():
+      v = resource_variable_ops.ResourceVariable(0, name="upto")
+      self.assertAllEqual(v.count_up_to(1), 0)
+      with self.assertRaises(errors.OutOfRangeError):
+        v.count_up_to(1)
+
+  def testCountUpToFunction(self):
+    with context.eager_mode():
+      v = resource_variable_ops.ResourceVariable(0, name="upto")
+      self.assertAllEqual(state_ops.count_up_to(v, 1), 0)
+      with self.assertRaises(errors.OutOfRangeError):
+        state_ops.count_up_to(v, 1)
+
   @test_util.run_in_graph_and_eager_modes()
   def testInitFnDtype(self):
     v = resource_variable_ops.ResourceVariable(
diff --git a/tensorflow/python/ops/resource_variable_ops.py b/tensorflow/python/ops/resource_variable_ops.py
index 439fa84238..ee8dd08c43 100644
--- a/tensorflow/python/ops/resource_variable_ops.py
+++ b/tensorflow/python/ops/resource_variable_ops.py
@@ -29,6 +29,7 @@ from tensorflow.python.framework import tensor_shape
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gen_array_ops
 from tensorflow.python.ops import gen_resource_variable_ops
+from tensorflow.python.ops import gen_state_ops
 from tensorflow.python.ops import variables
 # go/tf-wildcard-import
 # pylint: disable=wildcard-import
@@ -573,6 +574,29 @@ class ResourceVariable(variables.Variable):
           "numpy() is only available when eager execution is enabled.")
     return self.read_value().numpy()
 
+  def count_up_to(self, limit):
+    """Increments this variable until it reaches `limit`.
+
+    When that Op is run it tries to increment the variable by `1`. If
+    incrementing the variable would bring it above `limit` then the Op raises
+    the exception `OutOfRangeError`.
+
+    If no error is raised, the Op outputs the value of the variable before
+    the increment.
+
+    This is essentially a shortcut for `count_up_to(self, limit)`.
+
+    Args:
+      limit: value at which incrementing the variable raises an error.
+
+    Returns:
+      A `Tensor` that will hold the variable value before the increment. If no
+      other Op modifies this variable, the values produced will all be
+      distinct.
+    """
+    return gen_state_ops.resource_count_up_to(self.handle, limit=limit,
+                                              T=self.dtype)
+
   def _set_save_slice_info(self, save_slice_info):
     """Sets the slice info for this `ResourceVariable`.
 
diff --git a/tensorflow/python/ops/state_ops.py b/tensorflow/python/ops/state_ops.py
index 65ec2d4b77..5b9ca7c0b9 100644
--- a/tensorflow/python/ops/state_ops.py
+++ b/tensorflow/python/ops/state_ops.py
@@ -275,3 +275,25 @@ def assign(ref, value, validate_shape=None, use_locking=None, name=None):
         ref, value, use_locking=use_locking, name=name,
         validate_shape=validate_shape)
   return ref.assign(value)
+
+
+def count_up_to(ref, limit, name=None):
+  r"""Increments 'ref' until it reaches 'limit'.
+
+  Args:
+    ref: A Variable. Must be one of the following types: `int32`, `int64`.
+      Should be from a scalar `Variable` node.
+    limit: An `int`.
+      If incrementing ref would bring it above limit, instead generates an
+      'OutOfRange' error.
+    name: A name for the operation (optional).
+
+  Returns:
+    A `Tensor`. Has the same type as `ref`.
+    A copy of the input before increment. If nothing else modifies the
+    input, the values produced will all be distinct.
+  """
+  if ref.dtype._is_ref_dtype:
+    return gen_state_ops.count_up_to(ref, limit=limit, name=name)
+  return gen_state_ops.resource_count_up_to(
+      ref.handle, limit, T=ref.dtype, name=name)
-- 
GitLab


From 06979c5a27d9cad343028eae32996a68a79e5169 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 26 Oct 2017 14:42:45 -0700
Subject: [PATCH 831/909] Go: Update generated wrapper functions for TensorFlow
 ops.

PiperOrigin-RevId: 173593222
---
 tensorflow/go/op/wrappers.go | 242 +++++++++++++++++++----------------
 1 file changed, 134 insertions(+), 108 deletions(-)

diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go
index cc8165e2c7..615c386858 100644
--- a/tensorflow/go/op/wrappers.go
+++ b/tensorflow/go/op/wrappers.go
@@ -13466,67 +13466,6 @@ func MergeSummary(scope *Scope, inputs []tf.Output) (summary tf.Output) {
 	return op.Output(0)
 }
 
-// Encode audio data using the WAV file format.
-//
-// This operation will generate a string suitable to be saved out to create a .wav
-// audio file. It will be encoded in the 16-bit PCM format. It takes in float
-// values in the range -1.0f to 1.0f, and any outside that value will be clamped to
-// that range.
-//
-// `audio` is a 2-D float Tensor of shape `[length, channels]`.
-// `sample_rate` is a scalar Tensor holding the rate to use (e.g. 44100).
-//
-// Arguments:
-//	audio: 2-D with shape `[length, channels]`.
-//	sample_rate: Scalar containing the sample frequency.
-//
-// Returns 0-D. WAV-encoded file contents.
-func EncodeWav(scope *Scope, audio tf.Output, sample_rate tf.Output) (contents tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "EncodeWav",
-		Input: []tf.Input{
-			audio, sample_rate,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// The gradient operator for the SparseAdd op.
-//
-// The SparseAdd op calculates A + B, where A, B, and the sum are all represented
-// as `SparseTensor` objects.  This op takes in the upstream gradient w.r.t.
-// non-empty values of the sum, and outputs the gradients w.r.t. the non-empty
-// values of A and B.
-//
-// Arguments:
-//	backprop_val_grad: 1-D with shape `[nnz(sum)]`.  The gradient with respect to
-// the non-empty values of the sum.
-//	a_indices: 2-D.  The `indices` of the `SparseTensor` A, size `[nnz(A), ndims]`.
-//	b_indices: 2-D.  The `indices` of the `SparseTensor` B, size `[nnz(B), ndims]`.
-//	sum_indices: 2-D.  The `indices` of the sum `SparseTensor`, size
-// `[nnz(sum), ndims]`.
-//
-// Returns 1-D with shape `[nnz(A)]`. The gradient with respect to the
-// non-empty values of A.1-D with shape `[nnz(B)]`. The gradient with respect to the
-// non-empty values of B.
-func SparseAddGrad(scope *Scope, backprop_val_grad tf.Output, a_indices tf.Output, b_indices tf.Output, sum_indices tf.Output) (a_val_grad tf.Output, b_val_grad tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "SparseAddGrad",
-		Input: []tf.Input{
-			backprop_val_grad, a_indices, b_indices, sum_indices,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1)
-}
-
 // Read an element from the TensorArray into output `value`.
 //
 // Arguments:
@@ -14567,6 +14506,32 @@ func IFFT3D(scope *Scope, input tf.Output) (output tf.Output) {
 	return op.Output(0)
 }
 
+// Increments variable pointed to by 'resource' until it reaches 'limit'.
+//
+// Arguments:
+//	resource: Should be from a scalar `Variable` node.
+//	limit: If incrementing ref would bring it above limit, instead generates an
+// 'OutOfRange' error.
+//
+//
+// Returns A copy of the input before increment. If nothing else modifies the
+// input, the values produced will all be distinct.
+func ResourceCountUpTo(scope *Scope, resource tf.Output, limit int64, T tf.DataType) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"limit": limit, "T": T}
+	opspec := tf.OpSpec{
+		Type: "ResourceCountUpTo",
+		Input: []tf.Input{
+			resource,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Looks up keys in a table, outputs the corresponding values.
 //
 // The tensor `keys` must of the same type as the keys of the table.
@@ -17320,6 +17285,53 @@ func RFFT2D(scope *Scope, input tf.Output, fft_length tf.Output) (output tf.Outp
 	return op.Output(0)
 }
 
+// ResourceSparseApplyAdagradAttr is an optional argument to ResourceSparseApplyAdagrad.
+type ResourceSparseApplyAdagradAttr func(optionalAttr)
+
+// ResourceSparseApplyAdagradUseLocking sets the optional use_locking attribute to value.
+//
+// value: If `True`, updating of the var and accum tensors will be protected
+// by a lock; otherwise the behavior is undefined, but may exhibit less
+// contention.
+// If not specified, defaults to false
+func ResourceSparseApplyAdagradUseLocking(value bool) ResourceSparseApplyAdagradAttr {
+	return func(m optionalAttr) {
+		m["use_locking"] = value
+	}
+}
+
+// Update relevant entries in '*var' and '*accum' according to the adagrad scheme.
+//
+// That is for rows we have grad for, we update var and accum as follows:
+// accum += grad * grad
+// var -= lr * grad * (1 / sqrt(accum))
+//
+// Arguments:
+//	var_: Should be from a Variable().
+//	accum: Should be from a Variable().
+//	lr: Learning rate. Must be a scalar.
+//	grad: The gradient.
+//	indices: A vector of indices into the first dimension of var and accum.
+//
+// Returns the created operation.
+func ResourceSparseApplyAdagrad(scope *Scope, var_ tf.Output, accum tf.Output, lr tf.Output, grad tf.Output, indices tf.Output, optional ...ResourceSparseApplyAdagradAttr) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "ResourceSparseApplyAdagrad",
+		Input: []tf.Input{
+			var_, accum, lr, grad, indices,
+		},
+		Attrs: attrs,
+	}
+	return scope.AddOperation(opspec)
+}
+
 // Creates a dataset that zips together `input_datasets`.
 func ZipDataset(scope *Scope, input_datasets []tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
 	if scope.Err() != nil {
@@ -17514,53 +17526,6 @@ func LRN(scope *Scope, input tf.Output, optional ...LRNAttr) (output tf.Output)
 	return op.Output(0)
 }
 
-// ResourceSparseApplyAdagradAttr is an optional argument to ResourceSparseApplyAdagrad.
-type ResourceSparseApplyAdagradAttr func(optionalAttr)
-
-// ResourceSparseApplyAdagradUseLocking sets the optional use_locking attribute to value.
-//
-// value: If `True`, updating of the var and accum tensors will be protected
-// by a lock; otherwise the behavior is undefined, but may exhibit less
-// contention.
-// If not specified, defaults to false
-func ResourceSparseApplyAdagradUseLocking(value bool) ResourceSparseApplyAdagradAttr {
-	return func(m optionalAttr) {
-		m["use_locking"] = value
-	}
-}
-
-// Update relevant entries in '*var' and '*accum' according to the adagrad scheme.
-//
-// That is for rows we have grad for, we update var and accum as follows:
-// accum += grad * grad
-// var -= lr * grad * (1 / sqrt(accum))
-//
-// Arguments:
-//	var_: Should be from a Variable().
-//	accum: Should be from a Variable().
-//	lr: Learning rate. Must be a scalar.
-//	grad: The gradient.
-//	indices: A vector of indices into the first dimension of var and accum.
-//
-// Returns the created operation.
-func ResourceSparseApplyAdagrad(scope *Scope, var_ tf.Output, accum tf.Output, lr tf.Output, grad tf.Output, indices tf.Output, optional ...ResourceSparseApplyAdagradAttr) (o *tf.Operation) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "ResourceSparseApplyAdagrad",
-		Input: []tf.Input{
-			var_, accum, lr, grad, indices,
-		},
-		Attrs: attrs,
-	}
-	return scope.AddOperation(opspec)
-}
-
 // AvgPool3DGradAttr is an optional argument to AvgPool3DGrad.
 type AvgPool3DGradAttr func(optionalAttr)
 
@@ -17937,6 +17902,67 @@ func StringToHashBucketStrong(scope *Scope, input tf.Output, num_buckets int64,
 	return op.Output(0)
 }
 
+// Encode audio data using the WAV file format.
+//
+// This operation will generate a string suitable to be saved out to create a .wav
+// audio file. It will be encoded in the 16-bit PCM format. It takes in float
+// values in the range -1.0f to 1.0f, and any outside that value will be clamped to
+// that range.
+//
+// `audio` is a 2-D float Tensor of shape `[length, channels]`.
+// `sample_rate` is a scalar Tensor holding the rate to use (e.g. 44100).
+//
+// Arguments:
+//	audio: 2-D with shape `[length, channels]`.
+//	sample_rate: Scalar containing the sample frequency.
+//
+// Returns 0-D. WAV-encoded file contents.
+func EncodeWav(scope *Scope, audio tf.Output, sample_rate tf.Output) (contents tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "EncodeWav",
+		Input: []tf.Input{
+			audio, sample_rate,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// The gradient operator for the SparseAdd op.
+//
+// The SparseAdd op calculates A + B, where A, B, and the sum are all represented
+// as `SparseTensor` objects.  This op takes in the upstream gradient w.r.t.
+// non-empty values of the sum, and outputs the gradients w.r.t. the non-empty
+// values of A and B.
+//
+// Arguments:
+//	backprop_val_grad: 1-D with shape `[nnz(sum)]`.  The gradient with respect to
+// the non-empty values of the sum.
+//	a_indices: 2-D.  The `indices` of the `SparseTensor` A, size `[nnz(A), ndims]`.
+//	b_indices: 2-D.  The `indices` of the `SparseTensor` B, size `[nnz(B), ndims]`.
+//	sum_indices: 2-D.  The `indices` of the sum `SparseTensor`, size
+// `[nnz(sum), ndims]`.
+//
+// Returns 1-D with shape `[nnz(A)]`. The gradient with respect to the
+// non-empty values of A.1-D with shape `[nnz(B)]`. The gradient with respect to the
+// non-empty values of B.
+func SparseAddGrad(scope *Scope, backprop_val_grad tf.Output, a_indices tf.Output, b_indices tf.Output, sum_indices tf.Output) (a_val_grad tf.Output, b_val_grad tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "SparseAddGrad",
+		Input: []tf.Input{
+			backprop_val_grad, a_indices, b_indices, sum_indices,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1)
+}
+
 // FixedLengthRecordReaderV2Attr is an optional argument to FixedLengthRecordReaderV2.
 type FixedLengthRecordReaderV2Attr func(optionalAttr)
 
-- 
GitLab


From de2c876843cdb86482af64bd3b39eadc7ab8cff5 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 26 Oct 2017 15:09:01 -0700
Subject: [PATCH 832/909] Added max_folded_constant_in_bytes field in
 OptimizationConfig. This option controls the maximum constant node size
 created during constant folding optimization. The default value is 10 MiB,
 the same limit as before this change.

PiperOrigin-RevId: 173597212
---
 .../core/common_runtime/constant_folding.cc      | 16 +++++++++-------
 .../core/common_runtime/constant_folding.h       |  3 +++
 .../core/common_runtime/constant_folding_test.cc |  7 +++++++
 .../core/common_runtime/graph_optimizer.cc       |  4 ++++
 tensorflow/core/protobuf/config.proto            |  7 +++++++
 .../golden/tensorflow.-optimizer-options.pbtxt   |  4 ++++
 6 files changed, 34 insertions(+), 7 deletions(-)

diff --git a/tensorflow/core/common_runtime/constant_folding.cc b/tensorflow/core/common_runtime/constant_folding.cc
index aca68d4c4a..0398c2a60d 100644
--- a/tensorflow/core/common_runtime/constant_folding.cc
+++ b/tensorflow/core/common_runtime/constant_folding.cc
@@ -460,7 +460,8 @@ Graph* GetConstantGraph(
 // new constant node.
 bool ReplaceTensorWithConstant(Graph* graph, Device* partition_device,
                                NodeAndOutput tensor, const Tensor& constant,
-                               const gtl::FlatSet<Node*>& control_deps) {
+                               const gtl::FlatSet<Node*>& control_deps,
+                               int64 max_constant_size_in_bytes) {
   // Be conservative when replacing a tensor with a constant, when not
   // running on CPU.
   // 1) If the destination tensor is not an int32 tensor, and has HOST_MEMORY
@@ -469,8 +470,9 @@ bool ReplaceTensorWithConstant(Graph* graph, Device* partition_device,
   // constraint, do not replace it.
   // 3) If the constant op created does not have a kernel implementation
   // for the device, do not use it.
-  // 4) If the size of the constant in bytes is too large (> 10M), do not
-  // replace it. This prevents the size of the Graph from growing too large.
+  // 4) If the size of the constant in bytes is too large (>
+  // max_constant_in_bytes), do not replace it. This prevents the size of the
+  // Graph from growing too large.
   // TODO(keveman): Consider adding a new constant op that has a kernel
   // implementation for all types, but with HostMemory constraint on it's
   // output.
@@ -494,7 +496,7 @@ bool ReplaceTensorWithConstant(Graph* graph, Device* partition_device,
       return false;
     }
   }
-  if (constant.TotalBytes() > 10 * 1024 * 1024) {
+  if (constant.TotalBytes() > max_constant_size_in_bytes) {
     return false;
   }
 
@@ -613,9 +615,9 @@ Status ConstantFold(const ConstantFoldingOptions& opts,
   for (size_t c = 0; c < outputs.size(); ++c) {
     const gtl::FlatSet<Node*>& control_deps =
         constant_control_deps[tensors_to_replace[c].first];
-    if (ReplaceTensorWithConstant(graph, partition_device,
-                                  tensors_to_replace[c], outputs[c],
-                                  control_deps)) {
+    if (ReplaceTensorWithConstant(
+            graph, partition_device, tensors_to_replace[c], outputs[c],
+            control_deps, opts.max_constant_size_in_bytes)) {
       ++num_nodes_replaced;
     }
   }
diff --git a/tensorflow/core/common_runtime/constant_folding.h b/tensorflow/core/common_runtime/constant_folding.h
index e7b1571a81..e4d724c58a 100644
--- a/tensorflow/core/common_runtime/constant_folding.h
+++ b/tensorflow/core/common_runtime/constant_folding.h
@@ -34,6 +34,9 @@ struct ConstantFoldingOptions {
   // outputs.
   const std::unordered_map<string, std::vector<PartialTensorShape>>* shape_map =
       nullptr;  // not owned
+  // The maximum size of each constant created during constant folding
+  // optimization.
+  int64 max_constant_size_in_bytes = 10 * 1024 * 1024;
 };
 
 // Perform constant folding optimization on "graph".
diff --git a/tensorflow/core/common_runtime/constant_folding_test.cc b/tensorflow/core/common_runtime/constant_folding_test.cc
index 2c7c20817a..923a4d9249 100644
--- a/tensorflow/core/common_runtime/constant_folding_test.cc
+++ b/tensorflow/core/common_runtime/constant_folding_test.cc
@@ -259,6 +259,13 @@ TEST_F(ConstantFoldingTest, TestNoReplaceLargeConstant) {
   TF_EXPECT_OK(ConstantFold(ConstantFoldingOptions{}, nullptr, Env::Default(),
                             nullptr, &g, &was_mutated));
   EXPECT_FALSE(was_mutated);
+
+  // Increase the limit and the concat should now be constant folded.
+  ConstantFoldingOptions opt;
+  opt.max_constant_size_in_bytes = 10 * 1024 * 1024 + 4;
+  TF_EXPECT_OK(
+      ConstantFold(opt, nullptr, Env::Default(), nullptr, &g, &was_mutated));
+  EXPECT_TRUE(was_mutated);
 }
 
 TEST_F(ConstantFoldingTest, TestNoReplaceFunctionCall) {
diff --git a/tensorflow/core/common_runtime/graph_optimizer.cc b/tensorflow/core/common_runtime/graph_optimizer.cc
index ff99db9532..def185e522 100644
--- a/tensorflow/core/common_runtime/graph_optimizer.cc
+++ b/tensorflow/core/common_runtime/graph_optimizer.cc
@@ -61,6 +61,10 @@ void GraphOptimizer::Optimize(
     if (opts_.do_constant_folding()) {
       ConstantFoldingOptions cf_opts;
       cf_opts.shape_map = shape_map;
+      if (opts_.max_folded_constant_in_bytes() > 0) {
+        cf_opts.max_constant_size_in_bytes =
+            opts_.max_folded_constant_in_bytes();
+      }
       bool was_mutated;
       ConstantFold(cf_opts, runtime, env, device, g, &was_mutated)
           .IgnoreError();
diff --git a/tensorflow/core/protobuf/config.proto b/tensorflow/core/protobuf/config.proto
index 56bb709e11..145311b59d 100644
--- a/tensorflow/core/protobuf/config.proto
+++ b/tensorflow/core/protobuf/config.proto
@@ -87,6 +87,13 @@ message OptimizerOptions {
   // If true, perform constant folding optimization on the graph.
   bool do_constant_folding = 2;
 
+  // Constant folding optimization replaces tensors whose values can be
+  // predetermined, with constant nodes. To avoid inserting too large constants,
+  // the size of each constant created can be limited. If this value is zero, a
+  // default limit of 10 MiB will be applied. If constant folding optimization
+  // is disabled, this value is ignored.
+  int64 max_folded_constant_in_bytes = 6;
+
   // If true, perform function inlining on the graph.
   bool do_function_inlining = 4;
 
diff --git a/tensorflow/tools/api/golden/tensorflow.-optimizer-options.pbtxt b/tensorflow/tools/api/golden/tensorflow.-optimizer-options.pbtxt
index 5dd1ee47c9..6cac5c4d99 100644
--- a/tensorflow/tools/api/golden/tensorflow.-optimizer-options.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.-optimizer-options.pbtxt
@@ -46,6 +46,10 @@ tf_class {
     name: "Level"
     mtype: "<class \'google.protobuf.internal.enum_type_wrapper.EnumTypeWrapper\'>"
   }
+  member {
+    name: "MAX_FOLDED_CONSTANT_IN_BYTES_FIELD_NUMBER"
+    mtype: "<type \'int\'>"
+  }
   member {
     name: "OFF"
     mtype: "<type \'int\'>"
-- 
GitLab


From f255bfc2e8a2150f65f899e6301fb2f0d0b84659 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 26 Oct 2017 15:15:02 -0700
Subject: [PATCH 833/909] Use object-oriented metrics in eager MNIST example.

PiperOrigin-RevId: 173598122
---
 tensorflow/contrib/eager/python/metrics_impl.py | 2 +-
 tensorflow/contrib/eager/python/tfe.py          | 3 +++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/eager/python/metrics_impl.py b/tensorflow/contrib/eager/python/metrics_impl.py
index 6c55ac2f5d..795dff548f 100644
--- a/tensorflow/contrib/eager/python/metrics_impl.py
+++ b/tensorflow/contrib/eager/python/metrics_impl.py
@@ -263,7 +263,7 @@ class Mean(Metric):
     """
     if weights is None:
       self.denom.assign_add(
-          math_ops.cast(array_ops.size(values), self.dtype))
+          math_ops.cast(array_ops.identity(array_ops.size(values)), self.dtype))
       values = math_ops.reduce_sum(values)
       self.numer.assign_add(math_ops.cast(values, self.dtype))
     else:
diff --git a/tensorflow/contrib/eager/python/tfe.py b/tensorflow/contrib/eager/python/tfe.py
index 3810d96950..a769140713 100644
--- a/tensorflow/contrib/eager/python/tfe.py
+++ b/tensorflow/contrib/eager/python/tfe.py
@@ -18,6 +18,8 @@ EXPERIMENTAL: APIs here are unstable and likely to change without notice.
 
 To use, at program startup, call `tfe.enable_eager_execution()`.
 
+@@metrics
+
 @@list_devices
 @@num_gpus
 
@@ -68,6 +70,7 @@ from __future__ import print_function
 
 # pylint:disable=g-bad-import-order,g-import-not-at-top,unused-import
 #
+from tensorflow.contrib.eager.python import metrics
 from tensorflow.contrib.eager.python.datasets import Iterator
 from tensorflow.contrib.eager.python.network import Network
 from tensorflow.contrib.eager.python.saver import restore_variables_on_create
-- 
GitLab


From a0c33c48e05a32adddf2119cc20bc3ea7a0e7253 Mon Sep 17 00:00:00 2001
From: Oleg Zabluda <ozabluda@gmail.com>
Date: Thu, 26 Oct 2017 15:19:22 -0700
Subject: [PATCH 834/909] Fix documentation error in tf.size()

tf.size() returns a symbolic `Tensor`, not an integer.
---
 tensorflow/python/ops/array_ops.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/ops/array_ops.py b/tensorflow/python/ops/array_ops.py
index 97dc63ebb1..857cd09d56 100644
--- a/tensorflow/python/ops/array_ops.py
+++ b/tensorflow/python/ops/array_ops.py
@@ -309,8 +309,8 @@ def size(input, name=None, out_type=dtypes.int32):
   # pylint: disable=redefined-builtin
   """Returns the size of a tensor.
 
-  This operation returns an integer representing the number of elements in
-  `input`.
+  Returns a 0-D `Tensor` representing the number of elements in `input`
+  of type `out_type`. Defaults to tf.int32.
 
   For example:
 
@@ -327,6 +327,10 @@ def size(input, name=None, out_type=dtypes.int32):
 
   Returns:
     A `Tensor` of type `out_type`. Defaults to tf.int32.
+    
+  @compatibility(numpy)
+  Equivalent to np.size()
+  @end_compatibility
   """
   return size_internal(input, name, optimize=True, out_type=out_type)
 
-- 
GitLab


From 7ecbf3501411acb44fd53c9cb54ea741a76f0776 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 26 Oct 2017 15:20:07 -0700
Subject: [PATCH 835/909] Update ops-related pbtxt files.

PiperOrigin-RevId: 173598929
---
 .../core/ops/compat/ops_history.v1.pbtxt      | 26 ++++++++++++++++
 tensorflow/core/ops/ops.pbtxt                 | 30 +++++++++++++++++++
 2 files changed, 56 insertions(+)

diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index cec75f6799..076c7bea1a 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -27445,6 +27445,32 @@ op {
   }
   is_stateful: true
 }
+op {
+  name: "ResourceCountUpTo"
+  input_arg {
+    name: "resource"
+    type: DT_RESOURCE
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  attr {
+    name: "limit"
+    type: "int"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+  is_stateful: true
+}
 op {
   name: "ResourceGather"
   input_arg {
diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index 78f0fda408..0a590fef00 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -22128,6 +22128,36 @@ op {
   description: "Note that in dense implementation of this algorithm, ms and mom will\nupdate even if the grad is zero, but in this sparse implementation, ms\nand mom will not update in iterations during which the grad is zero.\n\nmean_square = decay * mean_square + (1-decay) * gradient ** 2\nDelta = learning_rate * gradient / sqrt(mean_square + epsilon)\n\nms <- rho * ms_{t-1} + (1-rho) * grad * grad\nmom <- momentum * mom_{t-1} + lr * grad / sqrt(ms + epsilon)\nvar <- var - mom"
   is_stateful: true
 }
+op {
+  name: "ResourceCountUpTo"
+  input_arg {
+    name: "resource"
+    description: "Should be from a scalar `Variable` node."
+    type: DT_RESOURCE
+  }
+  output_arg {
+    name: "output"
+    description: "A copy of the input before increment. If nothing else modifies the\ninput, the values produced will all be distinct."
+    type_attr: "T"
+  }
+  attr {
+    name: "limit"
+    type: "int"
+    description: "If incrementing ref would bring it above limit, instead generates an\n\'OutOfRange\' error."
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+  summary: "Increments variable pointed to by \'resource\' until it reaches \'limit\'."
+  is_stateful: true
+}
 op {
   name: "ResourceGather"
   input_arg {
-- 
GitLab


From 1ba3cb0645a2216718ae6baabf504cd154d17ad3 Mon Sep 17 00:00:00 2001
From: Saurabh Saxena <srbs@google.com>
Date: Thu, 26 Oct 2017 15:27:49 -0700
Subject: [PATCH 836/909] Implement saving/restoring state of TextLineDataset.

PiperOrigin-RevId: 173600100
---
 .../contrib/data/python/kernel_tests/BUILD    |   2 +
 .../kernel_tests/reader_dataset_ops_test.py   | 273 ++++++++++++++++++
 tensorflow/core/kernels/reader_dataset_ops.cc | 139 ++++++---
 3 files changed, 377 insertions(+), 37 deletions(-)

diff --git a/tensorflow/contrib/data/python/kernel_tests/BUILD b/tensorflow/contrib/data/python/kernel_tests/BUILD
index b8cdb7b20d..36af55a7ec 100644
--- a/tensorflow/contrib/data/python/kernel_tests/BUILD
+++ b/tensorflow/contrib/data/python/kernel_tests/BUILD
@@ -270,6 +270,7 @@ py_test(
     srcs = ["reader_dataset_ops_test.py"],
     srcs_version = "PY2AND3",
     deps = [
+        "//tensorflow/contrib/data/python/ops:iterator_ops",
         "//tensorflow/contrib/data/python/ops:readers",
         "//tensorflow/core:protos_all_py",
         "//tensorflow/python:array_ops",
@@ -283,6 +284,7 @@ py_test(
         "//tensorflow/python:lib",
         "//tensorflow/python:parsing_ops",
         "//tensorflow/python:tensor_shape",
+        "//tensorflow/python:training",
         "//tensorflow/python:util",
         "//tensorflow/python/data/ops:iterator_ops",
     ],
diff --git a/tensorflow/contrib/data/python/kernel_tests/reader_dataset_ops_test.py b/tensorflow/contrib/data/python/kernel_tests/reader_dataset_ops_test.py
index 2682e8bdfa..3ae8f71d77 100644
--- a/tensorflow/contrib/data/python/kernel_tests/reader_dataset_ops_test.py
+++ b/tensorflow/contrib/data/python/kernel_tests/reader_dataset_ops_test.py
@@ -21,6 +21,7 @@ import gzip
 import os
 import zlib
 
+from tensorflow.contrib.data.python.ops import iterator_ops as contrib_iterator_ops
 from tensorflow.contrib.data.python.ops import readers
 from tensorflow.core.example import example_pb2
 from tensorflow.core.example import feature_pb2
@@ -36,6 +37,7 @@ from tensorflow.python.ops import gen_dataset_ops
 from tensorflow.python.ops import io_ops
 from tensorflow.python.ops import parsing_ops
 from tensorflow.python.platform import test
+from tensorflow.python.training import saver as saver_lib
 from tensorflow.python.util import compat
 
 
@@ -163,6 +165,277 @@ class TextLineDatasetTest(test.TestCase):
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(iterator.get_next())
 
+  def _ckpt_path(self):
+    return os.path.join(self.get_temp_dir(), "iterator")
+
+  def _latest_ckpt(self):
+    return saver_lib.latest_checkpoint(self.get_temp_dir())
+
+  def _save(self, saver, sess):
+    saver.save(sess, self._ckpt_path())
+
+  def _restore(self, saver, sess):
+    saver.restore(sess, self._latest_ckpt())
+
+  def _import_meta_graph(self):
+    meta_file_path = self._ckpt_path() + ".meta"
+    return saver_lib.import_meta_graph(meta_file_path)
+
+  def _build_graph(self,
+                   test_filenames,
+                   compression_type=None,
+                   build_saveable=True):
+    ds = readers.TextLineDataset(
+        test_filenames, compression_type=compression_type, buffer_size=10)
+    iterator = ds.make_initializable_iterator()
+    if build_saveable:
+      saveable = contrib_iterator_ops.make_saveable_from_iterator(iterator)
+      ops.add_to_collection(ops.GraphKeys.SAVEABLE_OBJECTS, saveable)
+    init_op = iterator.initializer
+    get_next = iterator.get_next()
+    ops.add_to_collection("iterator_ops", init_op)
+    ops.add_to_collection("iterator_ops", get_next)
+    saver = saver_lib.Saver(allow_empty=True)
+    return init_op, get_next, saver
+
+  def _testReadWithBreaks(self, breaks, num_files=5, lines_per_file=5):
+    """Tests reading from input pipeline with regular breaks.
+
+    At each break point the iterator state gets saved using Saver and reloaded
+    in a new Graph and session.
+
+    Args:
+      breaks: List of counts of records after reading which iterator state is
+        checkpointed. Must to in non-decreasing order.
+      num_files: Total number of files.
+      lines_per_file: Total number of lines per file.
+    """
+    compression_types = [None, "GZIP", "ZLIB"]
+    for compression_type in compression_types:
+      test_filenames = self._createFiles(
+          num_files,
+          lines_per_file,
+          crlf=True,
+          compression_type=compression_type)
+
+      # Collect ground truth.
+      total_records = num_files * lines_per_file
+      expected_records = []
+      with ops.Graph().as_default() as g:
+        init_op, get_next, saver = self._build_graph(
+            test_filenames, compression_type=compression_type)
+        with self.test_session(graph=g) as sess:
+          sess.run(init_op)
+          for _ in range(total_records):
+            expected_records.append(sess.run(get_next))
+          with self.assertRaises(errors.OutOfRangeError):
+            sess.run(get_next)
+
+      # Simulate run with breaks.
+      actual_records = []
+      next_record_index = 0
+      load_from_ckpt = False
+      breaks.append(total_records)
+      for break_index in breaks:
+        with ops.Graph().as_default() as g:
+          if not load_from_ckpt:
+            init_op, get_next, saver = self._build_graph(
+                test_filenames, compression_type=compression_type)
+          else:
+            saver = self._import_meta_graph()
+            init_op, get_next = ops.get_collection("iterator_ops")
+
+          with self.test_session(graph=g) as sess:
+            if not load_from_ckpt:
+              sess.run(init_op)
+            else:
+              self._restore(saver, sess)
+            while next_record_index != break_index:
+              actual_records.append(sess.run(get_next))
+              next_record_index += 1
+            if break_index == total_records:
+              with self.assertRaises(errors.OutOfRangeError):
+                sess.run(get_next)
+            self._save(saver, sess)
+            load_from_ckpt = True
+      self.assertEqual(actual_records, expected_records)
+
+  def testSaveAtFileBoundary(self):
+    self._testReadWithBreaks([10])
+
+  def testSaveWithinFile(self):
+    self._testReadWithBreaks([12])
+
+  def testSaveUnusedIterator(self):
+    self._testReadWithBreaks([0])
+
+  def testSaveRestoreIdempotence(self):
+    # Attempt to save an iterator immediately after it has been
+    # restored.
+    self._testReadWithBreaks([0, 0])
+    self._testReadWithBreaks([10, 10])
+    self._testReadWithBreaks([12, 12])
+
+  def testMultipleBreaks(self):
+    self._testReadWithBreaks([0, 4, 20])
+
+  def testRestoreExhaustedIterator(self):
+    num_files = 2
+    lines_per_file = 5
+    test_filenames = self._createFiles(num_files, lines_per_file, crlf=True)
+
+    with ops.Graph().as_default() as g:
+      init_op, get_next, saver = self._build_graph(test_filenames)
+      with self.test_session(graph=g) as sess:
+        sess.run(init_op)
+        for _ in range(num_files * lines_per_file):
+          sess.run(get_next)
+        with self.assertRaises(errors.OutOfRangeError):
+          sess.run(get_next)
+        self._save(saver, sess)
+
+    with ops.Graph().as_default() as g:
+      with self.test_session(graph=g) as sess:
+        saver = self._import_meta_graph()
+        self._restore(saver, sess)
+        _, get_next = ops.get_collection("iterator_ops")
+        with self.assertRaises(errors.OutOfRangeError):
+          sess.run(get_next)
+
+  def testInitThenRestore(self):
+    num_files = 5
+    lines_per_file = 5
+    total_records = num_files * lines_per_file
+    break_record = 8
+    test_filenames = self._createFiles(num_files, lines_per_file, crlf=True)
+
+    expected_records = []
+    with ops.Graph().as_default() as g:
+      init_op, get_next, saver = self._build_graph(test_filenames)
+      with self.test_session(graph=g) as sess:
+        sess.run(init_op)
+        for _ in range(break_record):
+          sess.run(get_next)
+        self._save(saver, sess)
+        for _ in range(total_records - break_record):
+          expected_records.append(sess.run(get_next))
+
+    actual_records = []
+    with ops.Graph().as_default() as g:
+      with self.test_session(graph=g) as sess:
+        saver = self._import_meta_graph()
+        init_op, get_next = ops.get_collection("iterator_ops")
+        sess.run(init_op)
+        self._restore(saver, sess)
+        for _ in range(total_records - break_record):
+          actual_records.append(sess.run(get_next))
+        with self.assertRaises(errors.OutOfRangeError):
+          sess.run(get_next)
+    self.assertEqual(actual_records, expected_records)
+
+  def testRestoreInModifiedGraph(self):
+    num_files = 5
+    lines_per_file = 5
+    total_records = num_files * lines_per_file
+    break_record = 8
+    test_filenames = self._createFiles(num_files, lines_per_file, crlf=True)
+
+    expected_records = []
+    with ops.Graph().as_default() as g:
+      init_op, get_next, saver = self._build_graph(test_filenames)
+      with self.test_session(graph=g) as sess:
+        sess.run(init_op)
+        for _ in range(break_record):
+          sess.run(get_next)
+        self._save(saver, sess)
+        for _ in range(total_records - break_record):
+          expected_records.append(sess.run(get_next))
+
+    actual_records = []
+    with ops.Graph().as_default() as g:
+      with self.test_session(graph=g) as sess:
+        init_op, get_next, saver = self._build_graph(
+            test_filenames, compression_type="GZIP")
+        self._restore(saver, sess)
+        for _ in range(total_records - break_record):
+          actual_records.append(sess.run(get_next))
+        with self.assertRaises(errors.OutOfRangeError):
+          sess.run(get_next)
+    self.assertEqual(actual_records, expected_records)
+
+  def testRestoreInModifiedGraphThenInit(self):
+    num_files = 5
+    lines_per_file = 5
+    total_records = num_files * lines_per_file
+    break_record = 8
+    test_filenames = self._createFiles(num_files, lines_per_file, crlf=True)
+
+    expected_records = []
+    with ops.Graph().as_default() as g:
+      init_op, get_next, saver = self._build_graph(test_filenames)
+      with self.test_session(graph=g) as sess:
+        sess.run(init_op)
+        for _ in range(break_record):
+          expected_records.append(sess.run(get_next))
+        self._save(saver, sess)
+        for _ in range(total_records - break_record):
+          expected_records.append(sess.run(get_next))
+        with self.assertRaises(errors.OutOfRangeError):
+          sess.run(get_next)
+
+    # Test that calling the init_op overrides the restored iterator. The
+    # iterator for the old graph was build to read uncompressed files and
+    # would fail when trying to read the new files.
+    actual_records = []
+    with ops.Graph().as_default() as g:
+      with self.test_session(graph=g) as sess:
+        test_filenames = self._createFiles(
+            num_files, lines_per_file, crlf=True, compression_type="GZIP")
+        init_op, get_next, saver = self._build_graph(
+            test_filenames, compression_type="GZIP")
+        self._restore(saver, sess)
+        sess.run(init_op)
+        for _ in range(total_records):
+          actual_records.append(sess.run(get_next))
+        with self.assertRaises(errors.OutOfRangeError):
+          sess.run(get_next)
+    self.assertEqual(actual_records, expected_records)
+
+  def testDoNotRestoreIterator(self):
+    num_files = 5
+    lines_per_file = 5
+    total_records = num_files * lines_per_file
+    break_record = 8
+    test_filenames = self._createFiles(num_files, lines_per_file, crlf=True)
+
+    expected_records = []
+    with ops.Graph().as_default() as g:
+      init_op, get_next, saver = self._build_graph(test_filenames)
+      with self.test_session(graph=g) as sess:
+        sess.run(init_op)
+        for _ in range(break_record):
+          expected_records.append(sess.run(get_next))
+        self._save(saver, sess)
+        for _ in range(total_records - break_record):
+          expected_records.append(sess.run(get_next))
+        with self.assertRaises(errors.OutOfRangeError):
+          sess.run(get_next)
+
+    actual_records = []
+    with ops.Graph().as_default() as g:
+      with self.test_session(graph=g) as sess:
+        init_op, get_next, saver = self._build_graph(
+            test_filenames, build_saveable=False)
+        self._restore(saver, sess)
+        with self.assertRaises(errors.FailedPreconditionError):
+          sess.run(get_next)
+        sess.run(init_op)
+        for _ in range(total_records):
+          actual_records.append(sess.run(get_next))
+        with self.assertRaises(errors.OutOfRangeError):
+          sess.run(get_next)
+    self.assertEqual(actual_records, expected_records)
+
 
 class FixedLengthRecordReaderTest(test.TestCase):
 
diff --git a/tensorflow/core/kernels/reader_dataset_ops.cc b/tensorflow/core/kernels/reader_dataset_ops.cc
index fb88c55f73..39ef92a5de 100644
--- a/tensorflow/core/kernels/reader_dataset_ops.cc
+++ b/tensorflow/core/kernels/reader_dataset_ops.cc
@@ -54,14 +54,9 @@ class TextLineDatasetOp : public DatasetOpKernel {
 
     io::ZlibCompressionOptions zlib_compression_options =
         io::ZlibCompressionOptions::DEFAULT();
-    bool use_compression = false;
-    if (compression_type.empty()) {
-      use_compression = false;
-    } else if (compression_type == "ZLIB") {
-      use_compression = true;
+    if (compression_type == "ZLIB") {
       zlib_compression_options = io::ZlibCompressionOptions::DEFAULT();
     } else if (compression_type == "GZIP") {
-      use_compression = true;
       zlib_compression_options = io::ZlibCompressionOptions::GZIP();
     } else {
       OP_REQUIRES(ctx, compression_type.empty(),
@@ -79,17 +74,20 @@ class TextLineDatasetOp : public DatasetOpKernel {
       filenames.push_back(filenames_tensor->flat<string>()(i));
     }
 
-    *output = new Dataset(std::move(filenames), use_compression,
+    *output = new Dataset(ctx, std::move(filenames), compression_type,
                           zlib_compression_options);
   }
 
  private:
-  class Dataset : public DatasetBase {
+  class Dataset : public GraphDatasetBase {
    public:
-    Dataset(std::vector<string> filenames, bool use_compression,
+    Dataset(OpKernelContext* ctx, std::vector<string> filenames,
+            const string& compression_type,
             const io::ZlibCompressionOptions& options)
-        : filenames_(std::move(filenames)),
-          use_compression_(use_compression),
+        : GraphDatasetBase(ctx),
+          filenames_(std::move(filenames)),
+          compression_type_(compression_type),
+          use_compression_(!compression_type.empty()),
           options_(options) {}
 
     std::unique_ptr<IteratorBase> MakeIterator(
@@ -111,6 +109,21 @@ class TextLineDatasetOp : public DatasetOpKernel {
 
     string DebugString() override { return "TextLineDatasetOp::Dataset"; }
 
+   protected:
+    Status AsGraphDefInternal(DatasetGraphDefBuilder* b,
+                              Node** output) const override {
+      Node* filenames = nullptr;
+      Node* compression_type = nullptr;
+      Node* buffer_size = nullptr;
+      TF_RETURN_IF_ERROR(b->AddVector(filenames_, &filenames));
+      TF_RETURN_IF_ERROR(b->AddScalar(compression_type_, &compression_type));
+      TF_RETURN_IF_ERROR(
+          b->AddScalar(options_.input_buffer_size, &buffer_size));
+      TF_RETURN_IF_ERROR(b->AddDataset(
+          this, {filenames, compression_type, buffer_size}, output));
+      return Status::OK();
+    }
+
    private:
     class Iterator : public DatasetIterator<Dataset> {
      public:
@@ -123,7 +136,7 @@ class TextLineDatasetOp : public DatasetOpKernel {
         mutex_lock l(mu_);
         do {
           // We are currently processing a file, so try to read the next line.
-          if (processing_file_) {
+          if (buffered_input_stream_) {
             string line_contents;
             Status s = buffered_input_stream_->ReadLine(&line_contents);
 
@@ -138,14 +151,9 @@ class TextLineDatasetOp : public DatasetOpKernel {
               // Report non-EOF errors to the caller.
               return s;
             }
-
             // We have reached the end of the current file, so maybe
             // move on to next file.
-            processing_file_ = false;
-            input_stream_.reset();
-            zlib_input_stream_.reset();
-            buffered_input_stream_.reset();
-            file_.reset();
+            ResetStreamsLocked();
             ++current_file_index_;
           }
 
@@ -155,30 +163,86 @@ class TextLineDatasetOp : public DatasetOpKernel {
             return Status::OK();
           }
 
-          // Actually move on to next file.
-          TF_RETURN_IF_ERROR(ctx->env()->NewRandomAccessFile(
-              dataset()->filenames_[current_file_index_], &file_));
-          processing_file_ = true;
-          input_stream_.reset(
-              new io::RandomAccessInputStream(file_.get(), false));
-          if (dataset()->use_compression_) {
-            zlib_input_stream_.reset(new io::ZlibInputStream(
-                input_stream_.get(), dataset()->options_.input_buffer_size,
-                dataset()->options_.input_buffer_size, dataset()->options_));
-            buffered_input_stream_.reset(new io::BufferedInputStream(
-                zlib_input_stream_.get(), dataset()->options_.input_buffer_size,
-                false));
-          } else {
-            buffered_input_stream_.reset(new io::BufferedInputStream(
-                input_stream_.get(), dataset()->options_.input_buffer_size,
-                false));
-          }
+          TF_RETURN_IF_ERROR(SetupStreamsLocked(ctx->env()));
         } while (true);
       }
 
+     protected:
+      Status SaveInternal(IteratorStateWriter* writer) override {
+        mutex_lock l(mu_);
+        TF_RETURN_IF_ERROR(writer->WriteScalar(full_name("current_file_index"),
+                                               current_file_index_));
+
+        // `buffered_input_stream_` is empty if
+        // 1. GetNext has not been called even once.
+        // 2. All files have been read and iterator has been exhausted.
+        if (buffered_input_stream_) {
+          TF_RETURN_IF_ERROR(writer->WriteScalar(
+              full_name("current_pos"), buffered_input_stream_->Tell()));
+        }
+        return Status::OK();
+      }
+
+      Status RestoreInternal(OpKernelContext* ctx,
+                             IteratorStateReader* reader) override {
+        mutex_lock l(mu_);
+        ResetStreamsLocked();
+        int64 current_file_index;
+        TF_RETURN_IF_ERROR(reader->ReadScalar(full_name("current_file_index"),
+                                              &current_file_index));
+        current_file_index_ = size_t(current_file_index);
+        // The key "current_pos" is written only if the iterator was saved
+        // with an open file.
+        if (reader->Contains(full_name("current_pos"))) {
+          int64 current_pos;
+          TF_RETURN_IF_ERROR(
+              reader->ReadScalar(full_name("current_pos"), &current_pos));
+
+          TF_RETURN_IF_ERROR(SetupStreamsLocked(ctx->env()));
+          TF_RETURN_IF_ERROR(buffered_input_stream_->Seek(current_pos));
+        }
+        return Status::OK();
+      }
+
      private:
+      // Sets up reader streams to read from the file at `current_file_index_`.
+      Status SetupStreamsLocked(Env* env) EXCLUSIVE_LOCKS_REQUIRED(mu_) {
+        if (current_file_index_ >= dataset()->filenames_.size()) {
+          return errors::InvalidArgument(
+              "current_file_index_:", current_file_index_,
+              " >= filenames_.size():", dataset()->filenames_.size());
+        }
+
+        // Actually move on to next file.
+        TF_RETURN_IF_ERROR(env->NewRandomAccessFile(
+            dataset()->filenames_[current_file_index_], &file_));
+        input_stream_.reset(
+            new io::RandomAccessInputStream(file_.get(), false));
+
+        if (dataset()->use_compression_) {
+          zlib_input_stream_.reset(new io::ZlibInputStream(
+              input_stream_.get(), dataset()->options_.input_buffer_size,
+              dataset()->options_.input_buffer_size, dataset()->options_));
+          buffered_input_stream_.reset(new io::BufferedInputStream(
+              zlib_input_stream_.get(), dataset()->options_.input_buffer_size,
+              false));
+        } else {
+          buffered_input_stream_.reset(new io::BufferedInputStream(
+              input_stream_.get(), dataset()->options_.input_buffer_size,
+              false));
+        }
+        return Status::OK();
+      }
+
+      // Resets all reader streams.
+      void ResetStreamsLocked() EXCLUSIVE_LOCKS_REQUIRED(mu_) {
+        input_stream_.reset();
+        zlib_input_stream_.reset();
+        buffered_input_stream_.reset();
+        file_.reset();
+      }
+
       mutex mu_;
-      bool processing_file_ GUARDED_BY(mu_) = false;
       std::unique_ptr<io::RandomAccessInputStream> input_stream_
           GUARDED_BY(mu_);
       std::unique_ptr<io::ZlibInputStream> zlib_input_stream_ GUARDED_BY(mu_);
@@ -190,6 +254,7 @@ class TextLineDatasetOp : public DatasetOpKernel {
     };
 
     const std::vector<string> filenames_;
+    const string compression_type_;
     const bool use_compression_;
     const io::ZlibCompressionOptions options_;
   };
-- 
GitLab


From 029ce4ed70a1d8644a8e1c11345bcae2416a2a6d Mon Sep 17 00:00:00 2001
From: Anush Elangovan <anush@nod-labs.com>
Date: Thu, 26 Oct 2017 15:35:12 -0700
Subject: [PATCH 837/909] iOS/RPi Add the ability to choose ANDROID_TYPES_FULL

Some networks require "full" types instead of "slim" so remove
the hard coding of SLIM in iOS and RPi. It still defaults to
building SLIM for them if not ENV var is specified but now
you can build with

ANDROID_TYPES="-D__ANDROID_TYPES_FULL" \
./tensorflow/contrib/makefile/build_all_ios.sh

TEST: Verify the  -D__ANDROID_TYPES_SLIM__ flag is default and
you can override with an env var
---
 tensorflow/contrib/makefile/Makefile | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/tensorflow/contrib/makefile/Makefile b/tensorflow/contrib/makefile/Makefile
index b582493131..3b4d0ff799 100644
--- a/tensorflow/contrib/makefile/Makefile
+++ b/tensorflow/contrib/makefile/Makefile
@@ -44,6 +44,11 @@ ifdef HEXAGON_LIBS
 	endif
 endif # HEXAGON_LIBS
 
+# If ANDROID_TYPES is not set assume __ANDROID_TYPES_SLIM__
+ifeq ($(ANDROID_TYPES),)
+	ANDROID_TYPES := -D__ANDROID_TYPES_SLIM__
+endif
+
 # Try to figure out the host system
 HOST_OS :=
 ifeq ($(OS),Windows_NT)
@@ -216,7 +221,7 @@ ifeq ($(TARGET),LINUX)
 endif
 # If we're cross-compiling for the Raspberry Pi, use the right gcc.
 ifeq ($(TARGET),PI)
-	CXXFLAGS += -D__ANDROID_TYPES_SLIM__ -DRASPBERRY_PI
+	CXXFLAGS += $(ANDROID_TYPES) -DRASPBERRY_PI
 	LDFLAGS := -Wl,--no-whole-archive
 	LIBS += -ldl -lpthread
 	LIBFLAGS += -Wl,--allow-multiple-definition -Wl,--whole-archive
@@ -338,7 +343,7 @@ ifeq ($(TARGET),IOS)
 		-Wno-c++11-narrowing \
 		-mno-thumb \
 		-DTF_LEAN_BINARY \
-		-D__ANDROID_TYPES_SLIM__ \
+		$(ANDROID_TYPES) \
 		-fno-exceptions \
 		-isysroot \
 		${IPHONEOS_SYSROOT}
@@ -362,7 +367,7 @@ ifeq ($(TARGET),IOS)
 		-Wno-c++11-narrowing \
 		-mno-thumb \
 		-DTF_LEAN_BINARY \
-		-D__ANDROID_TYPES_SLIM__ \
+		$(ANDROID_TYPES) \
 		-fno-exceptions \
 		-isysroot \
 		${IPHONEOS_SYSROOT}
@@ -385,7 +390,7 @@ ifeq ($(TARGET),IOS)
 		-DUSE_GEMM_FOR_CONV \
 		-Wno-c++11-narrowing \
 		-DTF_LEAN_BINARY \
-		-D__ANDROID_TYPES_SLIM__ \
+		$(ANDROID_TYPES) \
 		-fno-exceptions \
 		-isysroot \
 		${IPHONEOS_SYSROOT}
@@ -409,7 +414,7 @@ ifeq ($(TARGET),IOS)
 		-DUSE_GEMM_FOR_CONV \
 		-Wno-c++11-narrowing \
 		-DTF_LEAN_BINARY \
-		-D__ANDROID_TYPES_SLIM__ \
+		$(ANDROID_TYPES) \
 		-fno-exceptions \
 		-isysroot \
 		${IPHONESIMULATOR_SYSROOT}
@@ -432,7 +437,7 @@ ifeq ($(TARGET),IOS)
 		-DUSE_GEMM_FOR_CONV \
 		-Wno-c++11-narrowing \
 		-DTF_LEAN_BINARY \
-		-D__ANDROID_TYPES_SLIM__ \
+		$(ANDROID_TYPES) \
 		-fno-exceptions \
 		-isysroot \
 		${IPHONESIMULATOR_SYSROOT}
-- 
GitLab


From f13b76a52886e39d8e97c9256c383eb3b79748d8 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 26 Oct 2017 15:36:51 -0700
Subject: [PATCH 838/909] Replace the fake updates with no updates when not
 training. This is possible now that the tf.cond bug has been fixed, and is
 needed to remove rare data races.

PiperOrigin-RevId: 173601427
---
 tensorflow/python/layers/normalization.py | 51 +++++++++++++----------
 1 file changed, 30 insertions(+), 21 deletions(-)

diff --git a/tensorflow/python/layers/normalization.py b/tensorflow/python/layers/normalization.py
index 5997d652aa..4fbe4b574f 100644
--- a/tensorflow/python/layers/normalization.py
+++ b/tensorflow/python/layers/normalization.py
@@ -436,27 +436,30 @@ class BatchNormalization(base.Layer):
     if dmax is not None:
       d = math_ops.maximum(d, -dmax)
       d = math_ops.minimum(d, dmax)
-    # When not training, use r=1, d=0, and decay=1 meaning no updates.
+    # When not training, use r=1, d=0.
     r = utils.smart_cond(training, lambda: r, lambda: array_ops.ones_like(r))
     d = utils.smart_cond(training, lambda: d, lambda: array_ops.zeros_like(d))
-    decay = utils.smart_cond(training, lambda: self.renorm_momentum, lambda: 1.)
 
     def _update_renorm_variable(var, weight, value):
       """Updates a moving average and weight, returns the unbiased value."""
-      # Update the variables without zero debiasing. The debiasing will be
-      # accomplished by dividing the exponential moving average by the weight.
-      # For example, after a single update, the moving average would be
-      # (1-decay) * value. and the weight will be 1-decay, with their ratio
-      # giving value.
-      # Make sure the weight is not updated until before r and d computation.
       value = array_ops.identity(value)
-      with ops.control_dependencies([value]):
-        weight_value = array_ops.constant(1., dtype=weight.dtype)
-      new_var = moving_averages.assign_moving_average(
-          var, value, decay, zero_debias=False)
-      new_weight = moving_averages.assign_moving_average(
-          weight, weight_value, decay, zero_debias=False)
-      return new_var / new_weight
+      def _do_update():
+        # Update the variables without zero debiasing. The debiasing will be
+        # accomplished by dividing the exponential moving average by the weight.
+        # For example, after a single update, the moving average would be
+        # (1-decay) * value. and the weight will be 1-decay, with their ratio
+        # giving the value.
+        # Make sure the weight is not updated until before r and d computation.
+        with ops.control_dependencies([value]):
+          weight_value = array_ops.constant(1., dtype=weight.dtype)
+        new_var = moving_averages.assign_moving_average(
+            var, value, self.renorm_momentum, zero_debias=False)
+        new_weight = moving_averages.assign_moving_average(
+            weight, weight_value, self.renorm_momentum, zero_debias=False)
+        return new_var / new_weight
+      def _fake_update():
+        return array_ops.identity(var)
+      return utils.smart_cond(training, _do_update, _fake_update)
 
     with ops.colocate_with(self.moving_mean):
       new_mean = _update_renorm_variable(self.renorm_mean,
@@ -562,8 +565,6 @@ class BatchNormalization(base.Layer):
       else:
         new_mean, new_variance = mean, variance
 
-      # Update moving averages when training, and prevent updates otherwise.
-      decay = utils.smart_cond(training, lambda: self.momentum, lambda: 1.)
       if self.virtual_batch_size is not None:
         # This isn't strictly correct since in ghost batch norm, you are
         # supposed to sequentially update the moving_mean and moving_variance
@@ -575,10 +576,18 @@ class BatchNormalization(base.Layer):
         new_variance = math_ops.reduce_mean(new_variance,
                                             axis=1, keep_dims=True)
 
-      mean_update = moving_averages.assign_moving_average(
-          self.moving_mean, new_mean, decay, zero_debias=False)
-      variance_update = moving_averages.assign_moving_average(
-          self.moving_variance, new_variance, decay, zero_debias=False)
+      def _do_update(var, value):
+        return moving_averages.assign_moving_average(
+            var, value, self.momentum, zero_debias=False)
+
+      mean_update = utils.smart_cond(
+          training,
+          lambda: _do_update(self.moving_mean, new_mean),
+          lambda: self.moving_mean)
+      variance_update = utils.smart_cond(
+          training,
+          lambda: _do_update(self.moving_variance, new_variance),
+          lambda: self.moving_variance)
       if context.in_graph_mode():
         self.add_update(mean_update, inputs=inputs)
         self.add_update(variance_update, inputs=inputs)
-- 
GitLab


From 9c40507f80434058f600ebebb8b9d6971dd0bdb4 Mon Sep 17 00:00:00 2001
From: Petros Mol <pmol@google.com>
Date: Thu, 26 Oct 2017 15:40:44 -0700
Subject: [PATCH 839/909] Documentation and error message edits for
 third_party/tensorflow/python/estimator/canned/head.py

PiperOrigin-RevId: 173601925
---
 tensorflow/python/estimator/canned/head.py | 38 ++++++++++++----------
 1 file changed, 20 insertions(+), 18 deletions(-)

diff --git a/tensorflow/python/estimator/canned/head.py b/tensorflow/python/estimator/canned/head.py
index f26e54ff49..18806db5eb 100644
--- a/tensorflow/python/estimator/canned/head.py
+++ b/tensorflow/python/estimator/canned/head.py
@@ -366,11 +366,11 @@ def _multi_class_head_with_softmax_cross_entropy_loss(n_classes,
       `tf.feature_column.numeric_column` defining feature column representing
       weights. It is used to down weight or boost examples during training. It
       will be multiplied by the loss of the example.
-    label_vocabulary: A list of strings represents possible label values. If it
-      is not given, that means labels are already encoded as integer within
-      [0, n_classes). If given, labels must be string type and have any value in
-      `label_vocabulary`. Also there will be errors if vocabulary is not
-      provided and labels are string.
+    label_vocabulary: A list or tuple of strings representing possible label
+      values. If it is not given, that means labels are already encoded as an
+      integer within [0, n_classes). If given, labels must be of string type and
+      have any value in `label_vocabulary`. Note that errors will be raised if
+      `label_vocabulary` is not provided but labels are strings.
     name: name of the head. If provided, summary and metrics keys will be
       suffixed by `"/" + name`. Also used as `name_scope` when creating ops.
 
@@ -382,8 +382,9 @@ def _multi_class_head_with_softmax_cross_entropy_loss(n_classes,
   """
   if label_vocabulary is not None and not isinstance(label_vocabulary,
                                                      (list, tuple)):
-    raise ValueError('label_vocabulary should be a list. Given type: {}'.format(
-        type(label_vocabulary)))
+    raise ValueError(
+        'label_vocabulary should be a list or a tuple. Given type: {}'.format(
+            type(label_vocabulary)))
 
   return _MultiClassHeadWithSoftmaxCrossEntropyLoss(n_classes, weight_column,
                                                     label_vocabulary, name)
@@ -437,8 +438,8 @@ class _MultiClassHeadWithSoftmaxCrossEntropyLoss(_Head):
     """Converts labels to integer id space."""
     if self._label_vocabulary is None:
       if not labels.dtype.is_integer:
-        raise ValueError('Labels dtype should be integer '
-                         'Instead got %s.' % labels.dtype)
+        raise ValueError('Labels dtype should be integer. Instead got {}.'.
+                         format(labels.dtype))
       label_ids = labels
     else:
       if labels.dtype != dtypes.string:
@@ -520,7 +521,7 @@ class _MultiClassHeadWithSoftmaxCrossEntropyLoss(_Head):
 
       # Train.
       if train_op_fn is None:
-        raise ValueError('train_op_fn can not be None.')
+        raise ValueError('train_op_fn cannot be None.')
     with ops.name_scope(''):
       summary.scalar(
           _summary_key(self._name, metric_keys.MetricKeys.LOSS),
@@ -555,11 +556,11 @@ def _binary_logistic_head_with_sigmoid_cross_entropy_loss(
       generated for each threshold value. This threshold is applied to the
       logistic values to determine the binary classification (i.e., above the
       threshold is `true`, below is `false`.
-    label_vocabulary: A list of strings represents possible label values. If it
-      is not given, that means labels are already encoded within [0, 1]. If
-      given, labels must be string type and have any value in
-      `label_vocabulary`. Also there will be errors if vocabulary is not
-      provided and labels are string.
+    label_vocabulary: A list or tuple of strings representing possible label
+      values. If it is not given, that means labels are already encoded within
+      [0, 1]. If given, labels must be string type and have any value in
+      `label_vocabulary`. Note that errors will be raised if `label_vocabulary`
+      is not provided but labels are strings.
     name: name of the head. If provided, summary and metrics keys will be
       suffixed by `"/" + name`. Also used as `name_scope` when creating ops.
 
@@ -572,12 +573,13 @@ def _binary_logistic_head_with_sigmoid_cross_entropy_loss(
   thresholds = tuple(thresholds) if thresholds else tuple()
   if label_vocabulary is not None and not isinstance(label_vocabulary,
                                                      (list, tuple)):
-    raise ValueError('label_vocabulary should be a list. Given type: {}'.format(
-        type(label_vocabulary)))
+    raise ValueError(
+        'label_vocabulary should be a list or tuple. Given type: {}'.format(
+            type(label_vocabulary)))
 
   for threshold in thresholds:
     if (threshold <= 0.0) or (threshold >= 1.0):
-      raise ValueError('thresholds not in (0, 1): %s.' % (thresholds,))
+      raise ValueError('thresholds not in (0, 1): {}.'.format((thresholds,)))
   return _BinaryLogisticHeadWithSigmoidCrossEntropyLoss(
       weight_column=weight_column,
       thresholds=thresholds,
-- 
GitLab


From ed39d70af37b1f794f63e0a77ae14a41271173bf Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 26 Oct 2017 15:43:16 -0700
Subject: [PATCH 840/909] Avoid assigning incompatible shapes to variable

PiperOrigin-RevId: 173602235
---
 tensorflow/python/eager/graph_callable.py           |  4 ++--
 .../kernel_tests/resource_variable_ops_test.py      |  6 ++++--
 tensorflow/python/ops/resource_variable_ops.py      | 13 ++++++++++++-
 tensorflow/python/training/saver.py                 |  5 +++--
 4 files changed, 21 insertions(+), 7 deletions(-)

diff --git a/tensorflow/python/eager/graph_callable.py b/tensorflow/python/eager/graph_callable.py
index 7f7a8c4a88..a7f1061d18 100644
--- a/tensorflow/python/eager/graph_callable.py
+++ b/tensorflow/python/eager/graph_callable.py
@@ -156,8 +156,8 @@ class _VariableCapturingScope(object):
       graph_mode_resource = v.variable.handle
       if initializer is None:
         initializer = _default_initializer(name, shape, dtype)
-      resource_variable_ops.assign_variable_op(
-          graph_mode_resource, initializer(shape, dtype))
+      resource_variable_ops.shape_safe_assign_variable_handle(
+          graph_mode_resource, v.variable.shape, initializer(shape, dtype))
       return v.variable
 
     scope = variable_scope.get_variable_scope()
diff --git a/tensorflow/python/kernel_tests/resource_variable_ops_test.py b/tensorflow/python/kernel_tests/resource_variable_ops_test.py
index d8d1ba6bbc..c33bacc5a5 100644
--- a/tensorflow/python/kernel_tests/resource_variable_ops_test.py
+++ b/tensorflow/python/kernel_tests/resource_variable_ops_test.py
@@ -305,8 +305,10 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase):
       with variable_scope.variable_scope("foo"):
         var = variable_scope.get_variable("x", shape=[1, 1],
                                           dtype=dtypes.float32)
-        assign = var.assign(np.zeros(shape=[2, 2]))
-        self.evaluate(assign)
+        with self.assertRaisesRegexp(ValueError,
+                                     "Shapes.*and.*are incompatible"):
+          assign = var.assign(np.zeros(shape=[2, 2]))
+          self.evaluate(assign)
 
   def testDtypeAfterFromProto(self):
     v = resource_variable_ops.ResourceVariable(2.0)
diff --git a/tensorflow/python/ops/resource_variable_ops.py b/tensorflow/python/ops/resource_variable_ops.py
index ee8dd08c43..eebb5f217c 100644
--- a/tensorflow/python/ops/resource_variable_ops.py
+++ b/tensorflow/python/ops/resource_variable_ops.py
@@ -73,6 +73,15 @@ def _eager_safe_variable_handle(shape, dtype, shared_name, name, graph_mode):
   return handle
 
 
+def shape_safe_assign_variable_handle(handle, shape, value, name=None):
+  """Helper that checks shape compatibility and assigns variable."""
+  value_tensor = ops.convert_to_tensor(value)
+  shape.assert_is_compatible_with(value_tensor.shape)
+  return gen_resource_variable_ops.assign_variable_op(handle,
+                                                      value_tensor,
+                                                      name=name)
+
+
 class ResourceVariable(variables.Variable):
   """Variable based on resource handles.
 
@@ -755,10 +764,12 @@ class ResourceVariable(variables.Variable):
       return self.read_value()
 
   def assign(self, value, use_locking=None, name=None):
+    value_tensor = ops.convert_to_tensor(value, dtype=self.dtype)
+    self._shape.assert_is_compatible_with(value_tensor.shape)
     with ops.control_dependencies([
         gen_resource_variable_ops.assign_variable_op(
             self.handle,
-            ops.convert_to_tensor(value, dtype=self.dtype),
+            value_tensor,
             name=name)
     ]):
       return self.read_value()
diff --git a/tensorflow/python/training/saver.py b/tensorflow/python/training/saver.py
index c4c1df22eb..145b44e2e0 100644
--- a/tensorflow/python/training/saver.py
+++ b/tensorflow/python/training/saver.py
@@ -164,6 +164,7 @@ class BaseSaverBuilder(object):
 
     def __init__(self, var, slice_spec, name):
       self._var_device = var.device
+      self._var_shape = var.shape
       if isinstance(var, ops.Tensor):
         self.handle_op = var.op.inputs[0]
         tensor = var
@@ -194,8 +195,8 @@ class BaseSaverBuilder(object):
       # Copy the restored tensor to the variable's device.
       with ops.device(self._var_device):
         restored_tensor = array_ops.identity(restored_tensor)
-      return resource_variable_ops.assign_variable_op(
-          self.handle_op, restored_tensor)
+      return resource_variable_ops.shape_safe_assign_variable_handle(
+          self.handle_op, self._var_shape, restored_tensor)
 
   def __init__(self, write_version=saver_pb2.SaverDef.V2):
     self._write_version = write_version
-- 
GitLab


From 2f8d3387c3b27cd77509efb69732c336479e014c Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Thu, 26 Oct 2017 16:02:08 -0700
Subject: [PATCH 841/909] [XLA] Fix race condition in LocalClientTestBase.

GetOrCreateAllocator needs to be thread-safe.

PiperOrigin-RevId: 173604923
---
 tensorflow/compiler/xla/tests/local_client_test_base.cc | 3 +++
 tensorflow/compiler/xla/tests/local_client_test_base.h  | 4 ++--
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/tensorflow/compiler/xla/tests/local_client_test_base.cc b/tensorflow/compiler/xla/tests/local_client_test_base.cc
index 05e282d208..c11e1df0a7 100644
--- a/tensorflow/compiler/xla/tests/local_client_test_base.cc
+++ b/tensorflow/compiler/xla/tests/local_client_test_base.cc
@@ -90,6 +90,9 @@ int64 TestAllocator::deallocation_count(int device_ordinal) const {
 
 /* static */ TestAllocator* LocalClientTestBase::GetOrCreateAllocator(
     perftools::gputools::Platform* platform) {
+  static tensorflow::mutex mu(tensorflow::LINKER_INITIALIZED);
+  tensorflow::mutex_lock lock(mu);
+
   if (allocator_ == nullptr) {
     allocator_ = new TestAllocator(
         platform == nullptr ? PlatformUtil::GetDefaultPlatform().ValueOrDie()
diff --git a/tensorflow/compiler/xla/tests/local_client_test_base.h b/tensorflow/compiler/xla/tests/local_client_test_base.h
index 17c25adfef..3edfcb656e 100644
--- a/tensorflow/compiler/xla/tests/local_client_test_base.h
+++ b/tensorflow/compiler/xla/tests/local_client_test_base.h
@@ -128,8 +128,8 @@ class LocalClientTestBase : public ::testing::Test {
     return ::testing::UnitTest::GetInstance()->current_test_info()->name();
   }
 
-  // The allocator must live as long as the service which lives until the end of
-  // the process, so make the allocator static.
+  // The allocator must live as long as the service, which lives until the end
+  // of the process. So make the allocator static.
   static TestAllocator* allocator_;
 
   perftools::gputools::StreamExecutor* stream_executor_;
-- 
GitLab


From bcf5dcc87ed2fe05197beaef30e536575608700b Mon Sep 17 00:00:00 2001
From: Asim Shankar <ashankar@google.com>
Date: Thu, 26 Oct 2017 16:21:32 -0700
Subject: [PATCH 842/909] eager: Update use of Datasets.

1. Use tf.data instead of tf.contrib.data
2. Move the CPU->device copy to tfe.Iterator.
   (So that users of tfe.Iterator don't have to do so themselves)

The latter is implemented using tf.identity() to explicitly copy
each element of the dataset, which can become a bottleneck. This
should be replaced by a scheme there the dataset prefetches elements
into device memory.

PiperOrigin-RevId: 173607457
---
 tensorflow/contrib/eager/python/BUILD         |  3 ++-
 tensorflow/contrib/eager/python/datasets.py   | 26 ++++++++++++++-----
 .../contrib/eager/python/datasets_test.py     | 10 ++++++-
 3 files changed, 30 insertions(+), 9 deletions(-)

diff --git a/tensorflow/contrib/eager/python/BUILD b/tensorflow/contrib/eager/python/BUILD
index adfaaa010a..179c27ba80 100644
--- a/tensorflow/contrib/eager/python/BUILD
+++ b/tensorflow/contrib/eager/python/BUILD
@@ -49,6 +49,7 @@ py_library(
     srcs_version = "PY2AND3",
     visibility = ["//tensorflow:internal"],
     deps = [
+        "//tensorflow/python:array_ops",
         "//tensorflow/python:dataset_ops_gen",
         "//tensorflow/python:errors",
         "//tensorflow/python:resource_variable_ops",
@@ -63,8 +64,8 @@ py_test(
     srcs_version = "PY2AND3",
     deps = [
         ":datasets",
-        "//tensorflow/contrib/data",
         "//tensorflow/python:math_ops",
+        "//tensorflow/python/data",
         "//tensorflow/python/eager:test",
         "//third_party/py/numpy",
     ],
diff --git a/tensorflow/contrib/eager/python/datasets.py b/tensorflow/contrib/eager/python/datasets.py
index fb9fabd6c1..f83c470411 100644
--- a/tensorflow/contrib/eager/python/datasets.py
+++ b/tensorflow/contrib/eager/python/datasets.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Support for tf.contrib.data when eager execution is enabled."""
+"""Iteration over tf.data.Datasets when eager execution is enabled."""
 
 from __future__ import absolute_import
 from __future__ import division
@@ -24,6 +24,7 @@ from tensorflow.python.data.util import nest
 from tensorflow.python.eager import context
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
+from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gen_dataset_ops
 from tensorflow.python.ops import resource_variable_ops
 
@@ -40,20 +41,23 @@ def _iterator_shared_name():
 
 
 class Iterator(object):
-  """An iterator producing tf.Tensor objects from a tf.contrib.data.Dataset."""
+  """An iterator producing tf.Tensor objects from a tf.data.Dataset."""
 
   def __init__(self, dataset):
     """Creates a new iterator over the given dataset.
 
     For example:
     ```python
-    dataset = tf.contrib.data.Dataset.range(4)
+    dataset = tf.data.Dataset.range(4)
     for x in Iterator(dataset):
       print(x)
     ```
 
+    Tensors produced will be placed on the device on which this iterator object
+    was created.
+
     Args:
-      dataset: A `tf.contrib.data.Dataset` object.
+      dataset: A `tf.data.Dataset` object.
 
     Raises:
       RuntimeError: When invoked without eager execution enabled.
@@ -61,8 +65,10 @@ class Iterator(object):
 
     if not context.in_eager_mode():
       raise RuntimeError(
-          "{} objects only make sense when eager execution is enabled".format(
-              type(self)))
+          "{} objects can only be used when eager execution is enabled, use "
+          "tf.data.Dataset.make_iterator or "
+          "tf.data.Dataset.make_one_shot_iterator for graph construction".
+          format(type(self)))
     with ops.device("/device:CPU:0"):
       ds_variant = dataset._as_variant_tensor()  # pylint: disable=protected-access
       self._output_types = dataset.output_types
@@ -74,6 +80,7 @@ class Iterator(object):
           output_types=self._flat_output_types,
           output_shapes=self._flat_output_shapes)
       gen_dataset_ops.make_iterator(ds_variant, self._resource)
+    self._device = context.context().device_name
 
   def __del__(self):
     if self._resource is not None:
@@ -98,6 +105,11 @@ class Iterator(object):
             self._resource,
             output_types=self._flat_output_types,
             output_shapes=self._flat_output_shapes)
-        return nest.pack_sequence_as(self._output_types, ret)
     except errors.OutOfRangeError:
       raise StopIteration
+    # Copies tensors from CPU to the current device if necessary.
+    # TODO(rohanj): This should be replaced by the mechanism to have the
+    # runtime's threads copy tensors to the destination device.
+    with ops.device(self._device):
+      ret = [array_ops.identity(x) for x in ret]
+      return nest.pack_sequence_as(self._output_types, ret)
diff --git a/tensorflow/contrib/eager/python/datasets_test.py b/tensorflow/contrib/eager/python/datasets_test.py
index 076c92e73f..c924d81c9d 100644
--- a/tensorflow/contrib/eager/python/datasets_test.py
+++ b/tensorflow/contrib/eager/python/datasets_test.py
@@ -16,10 +16,11 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.contrib.data import Dataset
 from tensorflow.contrib.eager.python import datasets
+from tensorflow.python.data import Dataset
 from tensorflow.python.eager import test
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import script_ops
 
@@ -81,6 +82,13 @@ class IteratorTest(test.TestCase):
     got = [x.numpy() for x in datasets.Iterator(ds)]
     self.assertAllEqual([[1], [2], [3], [4]], got)
 
+  def testTensorsPlacedOnDevice(self):
+    ds = Dataset.from_tensors([0., 1.])
+    with ops.device(test.gpu_device_name()):
+      x = datasets.Iterator(ds).next()
+      x = math_ops.add(x, x)
+    self.assertAllEqual([0., 2.], x.numpy())
+
 
 if __name__ == '__main__':
   test.main()
-- 
GitLab


From 53c4408078d5e4e095a2661e7e6547782391ded8 Mon Sep 17 00:00:00 2001
From: Thomas Schumm <fwiffo@google.com>
Date: Thu, 26 Oct 2017 16:34:28 -0700
Subject: [PATCH 843/909] Let users check for an hparam's existence in a more
 readable way.

PiperOrigin-RevId: 173608993
---
 tensorflow/contrib/training/python/training/hparam.py      | 3 +++
 tensorflow/contrib/training/python/training/hparam_test.py | 5 +++++
 2 files changed, 8 insertions(+)

diff --git a/tensorflow/contrib/training/python/training/hparam.py b/tensorflow/contrib/training/python/training/hparam.py
index 1b52d23c61..391899b34f 100644
--- a/tensorflow/contrib/training/python/training/hparam.py
+++ b/tensorflow/contrib/training/python/training/hparam.py
@@ -532,6 +532,9 @@ class HParams(object):
     """
     return {n: getattr(self, n) for n in self._hparam_types.keys()}
 
+  def __contains__(self, key):
+    return key in self._hparam_types
+
   def __str__(self):
     return str(sorted(self.values().items()))
 
diff --git a/tensorflow/contrib/training/python/training/hparam_test.py b/tensorflow/contrib/training/python/training/hparam_test.py
index a947bf6eda..f54514cefd 100644
--- a/tensorflow/contrib/training/python/training/hparam_test.py
+++ b/tensorflow/contrib/training/python/training/hparam_test.py
@@ -32,6 +32,11 @@ class HParamsTest(test.TestCase):
     with self.assertRaisesRegexp(ValueError, 'Unknown hyperparameter'):
       hparams.parse('xyz=123')
 
+  def testContains(self):
+    hparams = hparam.HParams(foo=1)
+    self.assertTrue('foo' in hparams)
+    self.assertFalse('bar' in hparams)
+
   def testSomeValues(self):
     hparams = hparam.HParams(aaa=1, b=2.0, c_c='relu6')
     self.assertDictEqual({'aaa': 1, 'b': 2.0, 'c_c': 'relu6'}, hparams.values())
-- 
GitLab


From 8772f8c4eb96a9e1a394fd247ca3c7da8f71d38f Mon Sep 17 00:00:00 2001
From: Russell Power <power@google.com>
Date: Thu, 26 Oct 2017 16:36:15 -0700
Subject: [PATCH 844/909] Some simple testing utilities for easing unittest
 writing with TPU devices.

PiperOrigin-RevId: 173609207
---
 tensorflow/contrib/tpu/BUILD                  |  12 ++
 .../contrib/tpu/python/tpu/test_util.py       | 153 ++++++++++++++++++
 2 files changed, 165 insertions(+)
 create mode 100644 tensorflow/contrib/tpu/python/tpu/test_util.py

diff --git a/tensorflow/contrib/tpu/BUILD b/tensorflow/contrib/tpu/BUILD
index 970fc97605..c89596734c 100644
--- a/tensorflow/contrib/tpu/BUILD
+++ b/tensorflow/contrib/tpu/BUILD
@@ -30,6 +30,18 @@ cc_library(
     ],
 )
 
+py_library(
+    name = "tpu_test_util",
+    srcs = [
+        "python/tpu/test_util.py",
+    ],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":tpu_lib",
+        ":tpu_py",
+    ],
+)
+
 py_library(
     name = "tpu_estimator",
     srcs = [
diff --git a/tensorflow/contrib/tpu/python/tpu/test_util.py b/tensorflow/contrib/tpu/python/tpu/test_util.py
new file mode 100644
index 0000000000..f30c27f129
--- /dev/null
+++ b/tensorflow/contrib/tpu/python/tpu/test_util.py
@@ -0,0 +1,153 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ===================================================================
+"""Utilities to ease testing on TPU devices."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.contrib.tpu.python.tpu import tpu
+
+from tensorflow.python.client import session
+from tensorflow.python.framework import errors
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
+from tensorflow.python.ops import gen_array_ops
+from tensorflow.python.ops import variables
+
+
+def has_tpu():
+  """Check if a TPU device is available.
+
+  Device enumeration via `device_lib` currently fails for TPU systems.
+  (http://b/68333779).  To work around this, we determine the existence of a
+  TPU by a successful call to `initialize_system`.
+
+  Returns:
+    boolean, True if a TPU device is available, otherwise False.
+  """
+  def _check():
+    with session.Session() as sess:
+      sess.run(tpu.initialize_system())
+      sess.run(tpu.shutdown_system())
+
+  try:
+    _check()
+    return True
+  except errors.OpError as _:
+    return False
+
+
+def _available_devices():
+  devices = ["cpu"]
+  if not test_util.gpu_device_name():
+    devices.append("gpu")
+
+  if has_tpu():
+    devices.append("tpu")
+
+  return tuple(devices)
+
+
+class TPUTestCase(test_util.TensorFlowTestCase):
+  """Adds helpers for testing on TPU devices to `TensorFlowTestCase`.
+
+  Example usage:
+
+  ```
+  def model_fn(features):
+  return tf.reduce_sum(features * 2)
+
+  class ModelTests(test_util.TPUTestCase):
+    def test_sum(self):
+      v = np.random.randn(10, 10).astype("float32")
+      self.assert_device_output(model_fn, [v], (v*2).sum(),
+                                devices=("cpu", "tpu"))
+  ```
+  """
+
+  def __init__(self, methodName="runTest"):  # pylint: disable=invalid-name
+    super(TPUTestCase, self).__init__(methodName)
+    self._available_devices = _available_devices()
+
+  def run_on_device(self, model_fn, model_inputs, device):
+    """Runs `model_fn` on the given device.
+
+    Raises an exception if no such device is available.  `model_fn` should
+    return one or more tensors as a list or tuple.
+
+    Args:
+      model_fn: Function returning one or more tensors.
+      model_inputs: An iterable of Numpy arrays or scalars.
+                    These will be passed as arguments to `model_fn`.
+      device: Device to run on.  One of ("tpu", "gpu", "cpu").
+
+    Returns:
+      Output from the model function.
+    """
+    def _make_placeholders():
+      return dict(
+          [(gen_array_ops.placeholder_with_default(v, v.shape), v)
+           for v in model_inputs])
+
+    if device == "tpu":
+      with self.test_session(graph=ops.Graph()) as sess:
+        placeholders = _make_placeholders()
+        tpu_computation = tpu.rewrite(model_fn, placeholders.keys())
+        sess.run(tpu.initialize_system())
+        sess.run(variables.global_variables_initializer())
+        result = sess.run(tpu_computation, placeholders)
+        sess.run(tpu.shutdown_system())
+        # TODO(b/36891278): supports non-flat returns lists in tpu.rewrite().
+        if len(result) == 1:
+          return result[0]
+        return result
+    elif device == "gpu":
+      with self.test_session(graph=ops.Graph(), use_gpu=True) as sess:
+        placeholders = _make_placeholders()
+        sess.run(variables.global_variables_initializer())
+        return sess.run(model_fn(placeholders.keys()), placeholders)
+    elif device == "cpu":
+      # TODO(power) -- will this interact poorly with cached GPU sessions?
+      with self.test_session(graph=ops.Graph(), use_gpu=False) as sess:
+        placeholders = _make_placeholders()
+        sess.run(variables.global_variables_initializer())
+        return sess.run(model_fn(placeholders.keys()), placeholders)
+
+  def _compare_values(self, actual_outputs, expected_outputs):
+    if isinstance(expected_outputs, (list, tuple)):
+      for a, b in zip(actual_outputs, expected_outputs):
+        self.assertAllCloseAccordingToType(a, b)
+    else:
+      self.assertAllCloseAccordingToType(actual_outputs, expected_outputs)
+
+  def assert_device_output(self, model_fn, model_inputs, expected_outputs,
+                           devices=("cpu", "gpu", "tpu")):
+    """Run `model_fn` on the given devices.
+
+    Results are compared via `assertAllCloseAccordingToType`.
+
+    Args:
+      model_fn: Function returning one or more tensors
+      model_inputs: Numpy arrays or scalars passed as arguments to model_fn
+      expected_outputs: Numpy arrays or scalars to compare against.
+      devices: Set of devices to run on.  If a device is not available, tests
+               will be skipped for that device.
+    """
+    devices = set(devices).intersection(self._available_devices)
+
+    for device in devices:
+      device_out = self.run_on_device(model_fn, model_inputs, device=device)
+      self._compare_values(device_out, expected_outputs)
-- 
GitLab


From 19a795bfbe46bfc5557f19f38c7a47242aa8927b Mon Sep 17 00:00:00 2001
From: Akshay Agrawal <akshayka@google.com>
Date: Thu, 26 Oct 2017 16:38:19 -0700
Subject: [PATCH 845/909] Introduce Eager-specific and Graph-specific Python
 TensorArray classes, and refactor the Python TensorArray class to wrap an
 instance of one of these two classes; the particular class to instantiate is
 chosen by consulting whether the context is in graph or eager mode.

TensorArrays in Eager are simply Python lists of EagerTensor objects;
most operations on an _EagerTensorArray object pass through to array_ops.

This change is meant to ensure compatibility of Eager execution with existing
code bases (e.g., code using control_flow_ops) that use TensorArray objects. Eager users will be better served by maintaining their own lists of EagerTensors.

PiperOrigin-RevId: 173609453
---
 tensorflow/python/BUILD                       |   3 +
 tensorflow/python/kernel_tests/BUILD          |   5 +
 .../kernel_tests/tensor_array_ops_test.py     | 436 ++++++----
 tensorflow/python/ops/tensor_array_ops.py     | 769 +++++++++++++++---
 4 files changed, 910 insertions(+), 303 deletions(-)

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index e2be7e8e9a..4de5d7f7db 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -2298,7 +2298,10 @@ py_library(
     srcs_version = "PY2AND3",
     deps = [
         ":array_ops",
+        ":constant_op",
         ":data_flow_ops_gen",
+        ":dtypes",
+        ":errors",
         ":framework_ops",
         ":math_ops",
         ":tensor_shape",
diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD
index d8ecabcdea..63844177b7 100644
--- a/tensorflow/python/kernel_tests/BUILD
+++ b/tensorflow/python/kernel_tests/BUILD
@@ -2012,13 +2012,18 @@ cuda_py_test(
         "//tensorflow/python:data_flow_ops_gen",
         "//tensorflow/python:distributed_framework_test_lib",
         "//tensorflow/python:framework_for_generated_wrappers",
+        "//tensorflow/python:framework_test_lib",
         "//tensorflow/python:gradients",
+        "//tensorflow/python:init_ops",
         "//tensorflow/python:math_ops",
         "//tensorflow/python:nn_grad",
         "//tensorflow/python:training",
         "//tensorflow/python:tensor_array_grad",
         "//tensorflow/python:tensor_array_ops",
         "//tensorflow/python:variables",
+        "//tensorflow/python:variable_scope",
+        "//tensorflow/python/eager:backprop",
+        "//tensorflow/python/eager:context",
     ],
     flaky = 1,  # create_local_cluster sometimes times out.
 )
diff --git a/tensorflow/python/kernel_tests/tensor_array_ops_test.py b/tensorflow/python/kernel_tests/tensor_array_ops_test.py
index 53e045fe86..a1fc6d63d4 100644
--- a/tensorflow/python/kernel_tests/tensor_array_ops_test.py
+++ b/tensorflow/python/kernel_tests/tensor_array_ops_test.py
@@ -22,17 +22,22 @@ import numpy as np
 
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.python.client import session as session_lib
+from tensorflow.python.eager import backprop
+from tensorflow.python.eager import context
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import gen_data_flow_ops
 from tensorflow.python.ops import gradients_impl
+from tensorflow.python.ops import init_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import tensor_array_grad
 from tensorflow.python.ops import tensor_array_ops
+from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops import variables
 import tensorflow.python.ops.nn_grad  # pylint: disable=unused-import
 from tensorflow.python.platform import test
@@ -51,6 +56,11 @@ def _make_converter(tf_dtype):
   return _converter
 
 
+def _make_ta(size, name, dtype=dtypes.float32, infer_shape=False):
+  return tensor_array_ops.TensorArray(
+      dtype=dtype, tensor_array_name=name, size=size, infer_shape=infer_shape)
+
+
 class TensorArrayTest(test.TestCase):
 
   @classmethod
@@ -63,8 +73,9 @@ class TensorArrayTest(test.TestCase):
     super(TensorArrayTest, cls).tearDownClass()
     session_lib.Session.reset(cls._workers[0].target)
 
+  @test_util.run_in_graph_and_eager_modes()
   def testTensorArrayWriteRead(self):
-    with self.test_session(use_gpu=True) as session:
+    with self.test_session(use_gpu=True):
       ta = tensor_array_ops.TensorArray(
           dtype=dtypes.float32,
           tensor_array_name="foo",
@@ -79,7 +90,7 @@ class TensorArrayTest(test.TestCase):
       r1 = w2.read(1)
       r2 = w2.read(2)
 
-      d0, d1, d2 = session.run([r0, r1, r2])
+      d0, d1, d2 = self.evaluate([r0, r1, r2])
       self.assertAllEqual([[4.0, 5.0]], d0)
       self.assertAllEqual([[1.0]], d1)
       self.assertAllEqual(-3.0, d2)
@@ -97,8 +108,9 @@ class TensorArrayTest(test.TestCase):
 
       c0 = w2.stack()
 
+      c0 = self.evaluate(c0)
       self.assertAllEqual(
-          convert([[[4.0, 5.0]], [[6.0, 7.0]], [[8.0, 9.0]]]), c0.eval())
+          convert([[[4.0, 5.0]], [[6.0, 7.0]], [[8.0, 9.0]]]), c0)
 
   def _testTensorArrayWritePackMaybeLegacy(self):
     self._testTensorArrayWritePack(dtypes.float32)
@@ -109,9 +121,11 @@ class TensorArrayTest(test.TestCase):
     self._testTensorArrayWritePack(dtypes.complex128)
     self._testTensorArrayWritePack(dtypes.string)
 
+  @test_util.run_in_graph_and_eager_modes()
   def testTensorArrayWritePack(self):
     self._testTensorArrayWritePackMaybeLegacy()
 
+  @test_util.run_in_graph_and_eager_modes()
   def testEmptyTensorArrayPack(self):
     with self.test_session(use_gpu=True):
       ta = tensor_array_ops.TensorArray(
@@ -124,7 +138,8 @@ class TensorArrayTest(test.TestCase):
 
       c0 = w2.stack()
 
-      self.assertAllEqual([3, 0, 1], c0.eval().shape)
+      c0 = self.evaluate(c0)
+      self.assertAllEqual([3, 0, 1], c0.shape)
 
   def _testTensorArrayWriteConcat(self, tf_dtype):
     with self.test_session(use_gpu=True):
@@ -139,10 +154,12 @@ class TensorArrayTest(test.TestCase):
 
       c0 = w2.concat()
 
+      c0 = self.evaluate(c0)
       self.assertAllEqual(
           convert([[4.0, 5.0], [104.0, 105.0], [204.0, 205.0], [6.0, 7.0],
-                   [106.0, 107.0], [8.0, 9.0]]), c0.eval())
+                   [106.0, 107.0], [8.0, 9.0]]), c0)
 
+  @test_util.run_in_graph_and_eager_modes()
   def testTensorArrayWriteConcat(self):
     self._testTensorArrayWriteConcat(dtypes.float32)
     self._testTensorArrayWriteConcat(dtypes.float64)
@@ -159,55 +176,46 @@ class TensorArrayTest(test.TestCase):
 
       with self.assertRaisesOpError("Could not read from TensorArray index 1 "
                                     "because it has not yet been written to."):
-        ta.write(0, [[4.0, 5.0]]).stack().eval()
+        self.evaluate(ta.write(0, [[4.0, 5.0]]).stack())
 
+  @test_util.run_in_graph_and_eager_modes()
   def testTensorArrayPackNotAllValuesAvailableFails(self):
     self._testTensorArrayPackNotAllValuesAvailableFails()
 
   def _testTensorArrayUnpackRead(self, tf_dtype):
-    with self.test_session(use_gpu=True) as session:
-      ta = tensor_array_ops.TensorArray(
-          dtype=tf_dtype, tensor_array_name="foo", size=3)
-
+    with self.test_session(use_gpu=True):
       convert = _make_converter(tf_dtype)
 
+      ta = _make_ta(3, "foo", dtype=tf_dtype)
       # Unpack a vector into scalars
       w0 = ta.unstack(convert([1.0, 2.0, 3.0]))
       r0 = w0.read(0)
       r1 = w0.read(1)
       r2 = w0.read(2)
 
-      d0, d1, d2 = session.run([r0, r1, r2])
+      d0, d1, d2 = self.evaluate([r0, r1, r2])
       self.assertAllEqual(convert(1.0), d0)
       self.assertAllEqual(convert(2.0), d1)
       self.assertAllEqual(convert(3.0), d2)
 
-      ta = tensor_array_ops.TensorArray(
-          dtype=tf_dtype, tensor_array_name="foo", size=3)
-
       # Unpack a matrix into vectors
       w1 = ta.unstack(convert([[1.0, 1.1], [2.0, 2.1], [3.0, 3.1]]))
       r0 = w1.read(0)
       r1 = w1.read(1)
       r2 = w1.read(2)
 
-      d0, d1, d2 = session.run([r0, r1, r2])
+      d0, d1, d2 = self.evaluate([r0, r1, r2])
       self.assertAllEqual(convert([1.0, 1.1]), d0)
       self.assertAllEqual(convert([2.0, 2.1]), d1)
       self.assertAllEqual(convert([3.0, 3.1]), d2)
 
-      # Reset ta because we're going to change the shape, else shape
-      # inference will throw an error.
-      ta = tensor_array_ops.TensorArray(
-          dtype=tf_dtype, tensor_array_name="foo", size=3)
-
       # Try unpacking an empty matrix, which should not cause an error.
       w2 = ta.unstack(convert([[], [], []]))
       r0 = w2.read(0)
       r1 = w2.read(1)
       r2 = w2.read(2)
 
-      d0, d1, d2 = session.run([r0, r1, r2])
+      d0, d1, d2 = self.evaluate([r0, r1, r2])
       self.assertAllEqual(convert([]), d0)
       self.assertAllEqual(convert([]), d1)
       self.assertAllEqual(convert([]), d2)
@@ -221,24 +229,23 @@ class TensorArrayTest(test.TestCase):
     self._testTensorArrayUnpackRead(dtypes.complex128)
     self._testTensorArrayUnpackRead(dtypes.string)
 
+  @test_util.run_in_graph_and_eager_modes()
   def testTensorArrayUnpackRead(self):
     self._testTensorArrayUnpackReadMaybeLegacy()
 
   def _testTensorArraySplitRead(self, tf_dtype):
-    with self.test_session(use_gpu=True) as session:
-      ta = tensor_array_ops.TensorArray(
-          dtype=tf_dtype, tensor_array_name="foo", size=3, infer_shape=False)
-
+    with self.test_session(use_gpu=True):
       convert = _make_converter(tf_dtype)
 
       # Split an empty vector
+      ta = _make_ta(3, "foo", dtype=tf_dtype)
       lengths = constant_op.constant([0, 0, 0])
       w0 = ta.split(convert([]), lengths=lengths)
       r0 = w0.read(0)
       r1 = w0.read(1)
       r2 = w0.read(2)
 
-      d0, d1, d2 = session.run([r0, r1, r2])
+      d0, d1, d2 = self.evaluate([r0, r1, r2])
       self.assertAllEqual(convert([]), d0)
       self.assertAllEqual(convert([]), d1)
       self.assertAllEqual(convert([]), d2)
@@ -250,7 +257,7 @@ class TensorArrayTest(test.TestCase):
       r1 = w0.read(1)
       r2 = w0.read(2)
 
-      d0, d1, d2 = session.run([r0, r1, r2])
+      d0, d1, d2 = self.evaluate([r0, r1, r2])
       self.assertAllEqual(convert([1.0, 2.0]), d0)
       self.assertAllEqual(convert([]), d1)
       self.assertAllEqual(convert([3.0]), d2)
@@ -263,11 +270,12 @@ class TensorArrayTest(test.TestCase):
       r1 = w0.read(1)
       r2 = w0.read(2)
 
-      d0, d1, d2 = session.run([r0, r1, r2])
+      d0, d1, d2 = self.evaluate([r0, r1, r2])
       self.assertAllEqual(convert([[1.0, 101.0], [2.0, 201.0]]), d0)
       self.assertAllEqual(convert([]).reshape(0, 2), d1)
       self.assertAllEqual(convert([[3.0, 301.0]]), d2)
 
+  @test_util.run_in_graph_and_eager_modes()
   def testTensorArraySplitRead(self):
     self._testTensorArraySplitRead(dtypes.float32)
     self._testTensorArraySplitRead(dtypes.float64)
@@ -367,59 +375,76 @@ class TensorArrayTest(test.TestCase):
       self.assertAllEqual(t_g_ta_0, t_g_ta_1)
       self.assertAllEqual([[4.0, 5.0]], d_r1_0)
 
+  @test_util.run_in_graph_and_eager_modes()
   def testTensorArrayWriteWrongIndexOrDataTypeFails(self):
     with self.test_session(use_gpu=True):
-      ta = tensor_array_ops.TensorArray(
-          dtype=dtypes.float32, tensor_array_name="foo", size=3)
-
+      ta = _make_ta(3, "foo", dtype=dtypes.float32)
+      in_graph_mode = context.in_graph_mode()
       # Test writing the wrong datatype
-      with self.assertRaisesOpError(
-          "TensorArray dtype is float but Op is trying to write dtype string"):
-        ta.write(-1, "wrong_type_scalar").flow.eval()
-
-      # Test writing to a negative index
-      with self.assertRaisesOpError(
-          "Tried to write to index -1 but array is not "
-          "resizeable and size is: 3"):
-        ta.write(-1, 3.0).flow.eval()
+      if in_graph_mode:
+        with self.assertRaisesOpError(
+            "TensorArray dtype is float but Op is trying to write "
+            "dtype string"):
+          self.evaluate(ta.write(0, "wrong_type_scalar").flow)
+      else:
+        with self.assertRaisesOpError(
+            "TensorArray dtype is float32 but Op is trying to write "
+            "dtype string"):
+          self.evaluate(ta.write(0, "wrong_type_scalar").flow)
+
+      if context.in_graph_mode():
+        with self.assertRaisesOpError(
+            "Tried to write to index -1 but array is not "
+            "resizeable and size is: 3"):
+          self.evaluate(ta.write(-1, 3.0).flow)
+      else:
+        with self.assertRaisesOpError(
+            r"Writing to negative indices \(index -1\) is not allowed."):
+          self.evaluate(ta.write(-1, 3.0).flow)
 
       # Test reading from too large an index
       with self.assertRaisesOpError(
           "Tried to write to index 3 but array is not "
           "resizeable and size is: 3"):
-        ta.write(3, 3.0).flow.eval()
+        self.evaluate(ta.write(3, 3.0).flow)
 
+  @test_util.run_in_graph_and_eager_modes()
   def testTensorArrayReadWrongIndexOrDataTypeFails(self):
     with self.test_session(use_gpu=True):
-      ta = tensor_array_ops.TensorArray(
-          dtype=dtypes.float32, tensor_array_name="foo", size=3)
+      ta = _make_ta(3, "foo", dtype=dtypes.float32)
 
       w0 = ta.write(0, [[4.0, 5.0]])
 
-      # Test reading wrong datatype
-      r0_bad = gen_data_flow_ops._tensor_array_read_v3(
-          handle=w0.handle, index=0, dtype=dtypes.float64, flow_in=w0.flow)
-      with self.assertRaisesOpError(
-          "TensorArray dtype is float but Op requested dtype double."):
-        r0_bad.eval()
+      # Test reading wrong datatype, which is only possible in graph mode
+      if context.in_graph_mode():
+        r0_bad = gen_data_flow_ops._tensor_array_read_v3(
+            handle=w0.handle, index=0, dtype=dtypes.float64, flow_in=w0.flow)
+        with self.assertRaisesOpError(
+            "TensorArray dtype is float but Op requested dtype double."):
+          r0_bad.eval()
 
       # Test reading from a different index than the one we wrote to
-      r1 = w0.read(1)
       with self.assertRaisesOpError(
           "Could not read from TensorArray index 1 because "
           "it has not yet been written to."):
-        r1.eval()
+        self.evaluate(w0.read(1))
 
-      # Test reading from a negative index
-      with self.assertRaisesOpError(
-          r"Tried to read from index -1 but array size is: 3"):
-        ta.read(-1).eval()
+      # Test reading from a negative index, which is not allowed
+      if context.in_graph_mode():
+        with self.assertRaisesOpError(
+            r"Tried to read from index -1 but array size is: 3"):
+          self.evaluate(ta.read(-1))
+      else:
+        with self.assertRaisesOpError(
+            r"Reading from negative indices \(index -1\) is not allowed."):
+          self.evaluate(ta.read(-1))
 
       # Test reading from too large an index
       with self.assertRaisesOpError(
           "Tried to read from index 3 but array size is: 3"):
-        ta.read(3).eval()
+        self.evaluate(ta.read(3))
 
+  @test_util.run_in_graph_and_eager_modes()
   def testTensorArrayWriteMultipleFails(self):
     with self.test_session(use_gpu=True):
       ta = tensor_array_ops.TensorArray(
@@ -428,8 +453,12 @@ class TensorArrayTest(test.TestCase):
       with self.assertRaisesOpError(
           "Could not write to TensorArray index 2 because "
           "it has already been written to."):
-        ta.write(2, 3.0).write(2, 3.0).flow.eval()
+        if context.in_graph_mode():
+          self.evaluate(ta.write(2, 3.0).write(2, 3.0).flow)
+        else:
+          self.evaluate(ta.write(2, 3.0).write(2, 3.0))
 
+  @test_util.run_in_graph_and_eager_modes()
   def testTensorArrayConcatIncompatibleShapesFails(self):
     with self.test_session(use_gpu=True):
       ta = tensor_array_ops.TensorArray(
@@ -444,7 +473,7 @@ class TensorArrayTest(test.TestCase):
 
       with self.assertRaisesOpError(
           "Concat saw a scalar shape at index 0 but requires at least vectors"):
-        w3.concat().eval()
+        self.evaluate(w3.concat())
 
       ta = tensor_array_ops.TensorArray(
           dtype=dtypes.float32,
@@ -456,45 +485,58 @@ class TensorArrayTest(test.TestCase):
       w2 = w1.write(1, [4.0])
       w3 = w2.write(2, [[3.0]])
 
-      with self.assertRaisesOpError(
-          r"TensorArray has inconsistent shapes.  Index 0 has "
-          r"\(excepting dimension 0\) shape: \[\] but index 2 has \(excepting "
-          r"dimension 0\) shape: \[1\]"):
-        w3.concat().eval()
+      # The eager-mode implementation just passes up array_op.concat's error
+      # message.
+      if context.in_graph_mode():
+        with self.assertRaisesOpError(
+            r"TensorArray has inconsistent shapes.  Index 0 has "
+            r"\(excepting dimension 0\) shape: \[\] but index 2 has "
+            r"\(excepting dimension 0\) shape: \[1\]"):
+          self.evaluate(w3.concat())
+      else:
+        with self.assertRaisesOpError(
+            r".*Ranks of all input tensors should match: shape\[0\] "
+            r"= \[1\] vs\. shape\[2\] = \[1,1\].*"):
+          self.evaluate(w3.concat())
 
+  @test_util.run_in_graph_and_eager_modes()
   def testTensorArraySplitIncompatibleShapesFails(self):
     with self.test_session(use_gpu=True):
-      ta = tensor_array_ops.TensorArray(
-          dtype=dtypes.float32,
-          tensor_array_name="foo",
-          size=3,
-          infer_shape=False)
-
+      in_graph_mode = context.in_graph_mode()
+      ta = _make_ta(3, "foo")
       with self.assertRaisesOpError(
           r"Expected lengths to be a vector, received shape: \[\]"):
-        lengths = array_ops.placeholder(dtypes.int64)
-        ta.split([1.0, 2.0, 3.0], lengths).flow.eval(feed_dict={lengths: 1})
+        if in_graph_mode:
+          lengths = array_ops.placeholder(dtypes.int64)
+          ta.split([1.0, 2.0, 3.0], lengths).flow.eval(feed_dict={lengths: 1})
+        else:
+          self.evaluate(ta.split([1.0, 2.0, 3.0], 1))
 
       with self.assertRaisesOpError(
           r"Expected sum of lengths to be equal to values.shape\[0\], "
           r"but sum of lengths is 1 and value's shape is: \[3\]"):
-        ta.split([1.0, 2.0, 3.0], [1]).flow.eval()
+        if in_graph_mode:
+          self.evaluate(ta.split([1.0, 2.0, 3.0], [1]).flow)
+        else:
+          self.evaluate(ta.split([1.0, 2.0, 3.0], [1]))
 
+      ta = _make_ta(1, "baz")
       with self.assertRaisesOpError(
           r"Expected value to be at least a vector, but received shape: \[\]"):
-        ta.split(1.0, [1]).flow.eval()
-
-      ta = tensor_array_ops.TensorArray(
-          dtype=dtypes.float32,
-          tensor_array_name="foo",
-          size=2,
-          infer_shape=False)
+        if in_graph_mode:
+          self.evaluate(ta.split(1.0, [1]).flow)
+        else:
+          self.evaluate(ta.split(1.0, [1]))
 
+      ta = _make_ta(2, "buz")
       with self.assertRaisesOpError(
           r"TensorArray's size is not equal to the size of lengths "
           r"\(2 vs. 1\), and the TensorArray is not marked as "
           r"dynamically resizeable"):
-        ta.split([1.0], [1]).flow.eval()
+        if in_graph_mode:
+          self.evaluate(ta.split([1.0], [1]).flow)
+        else:
+          self.evaluate(ta.split([1.0], [1]))
 
   def _testTensorArrayWriteGradientAddMultipleAdds(self, dtype):
     with self.test_session(use_gpu=True):
@@ -535,6 +577,7 @@ class TensorArrayTest(test.TestCase):
                   dtypes.complex64, dtypes.complex128):
       self._testTensorArrayWriteGradientAddMultipleAdds(dtype)
 
+  @test_util.run_in_graph_and_eager_modes()
   def testMultiTensorArray(self):
     with self.test_session(use_gpu=True):
       h1 = tensor_array_ops.TensorArray(
@@ -548,7 +591,8 @@ class TensorArrayTest(test.TestCase):
       w2 = h2.write(0, 5.0)
       r2 = w2.read(0)
       r = r1 + r2
-      self.assertAllClose(9.0, r.eval())
+      val = self.evaluate(r)
+      self.assertAllClose(9.0, val)
 
   def _testTensorArrayGradientWriteReadType(self, dtype):
     with self.test_session(use_gpu=True) as session:
@@ -637,6 +681,7 @@ class TensorArrayTest(test.TestCase):
   def testTensorArrayGradientWritePackConcatAndRead(self):
     self._testTensorArrayGradientWritePackConcatAndRead()
 
+  @test_util.run_in_graph_and_eager_modes()
   def testTensorArrayReadTwice(self):
     with self.test_session(use_gpu=True):
       value = constant_op.constant([[1.0, -1.0], [10.0, -10.0]])
@@ -646,13 +691,12 @@ class TensorArrayTest(test.TestCase):
 
       w_readonce = ta_readonce.unstack(value)
       r0_readonce = w_readonce.read(0)
-      with ops.control_dependencies([r0_readonce]):
-        r1_readonce = w_readonce.read(0)
 
       with self.assertRaisesOpError(
           r"Could not read index 0 twice because it was cleared after a "
           r"previous read \(perhaps try setting clear_after_read = false\?\)"):
-        r1_readonce.eval()
+        with ops.control_dependencies([r0_readonce]):
+          self.evaluate(w_readonce.read(0))
 
       ta_readtwice = tensor_array_ops.TensorArray(
           dtype=dtypes.float32,
@@ -664,7 +708,7 @@ class TensorArrayTest(test.TestCase):
       with ops.control_dependencies([r0_readtwice]):
         r1_readtwice = w_readtwice.read(0)
 
-      self.assertAllEqual([1.0, -1.0], r1_readtwice.eval())
+      self.assertAllEqual([1.0, -1.0], self.evaluate(r1_readtwice))
 
   def _testTensorArrayGradientUnpackRead(self):
     with self.test_session(use_gpu=True) as session:
@@ -741,20 +785,22 @@ class TensorArrayTest(test.TestCase):
   def testTensorArrayGradientDynamicUnpackRead(self):
     self._testTensorArrayGradientDynamicUnpackRead()
 
+  @test_util.run_in_graph_and_eager_modes()
   def testCloseTensorArray(self):
-    with self.test_session(use_gpu=True) as session:
+    with self.test_session(use_gpu=True):
       ta = tensor_array_ops.TensorArray(
           dtype=dtypes.float32, tensor_array_name="foo", size=3)
-      c1 = ta.close()
-      session.run(c1)
+      self.evaluate(ta.close())
 
+  @test_util.run_in_graph_and_eager_modes()
   def testSizeTensorArray(self):
     with self.test_session(use_gpu=True):
       ta = tensor_array_ops.TensorArray(
           dtype=dtypes.float32, tensor_array_name="foo", size=3)
       s = ta.size()
-      self.assertAllEqual(3, s.eval())
+      self.assertAllEqual(3, self.evaluate(s))
 
+  @test_util.run_in_graph_and_eager_modes()
   def testWriteCloseTensorArray(self):
     with self.test_session(use_gpu=True):
       ta = tensor_array_ops.TensorArray(
@@ -764,48 +810,62 @@ class TensorArrayTest(test.TestCase):
           infer_shape=False)
       w0 = ta.write(0, [[4.0, 5.0]])
       w1 = w0.write(1, [3.0])
-      w1.close().run()  # Expected to run without problems
+      self.evaluate(w1.close())  # Expected to run without problems
 
   def _testWhileLoopWritePackGradients(self, dynamic_size, dtype):
     np_dtype = dtype.as_numpy_dtype
-    with self.test_session(use_gpu=True) as session:
+    with self.test_session(use_gpu=True):
+      def func(v0, state0, var):
+        ta = tensor_array_ops.TensorArray(
+            dtype=dtype,
+            tensor_array_name="foo",
+            size=0 if dynamic_size else 3,
+            dynamic_size=dynamic_size)
+        time_0 = array_ops.identity(0)
+
+        def body(time, ta_t, state):
+          sliced = array_ops.slice(
+              v0, begin=array_ops.stack([time, 0]), size=[1, -1])
+          sliced = array_ops.squeeze(sliced)
+          out = sliced + var + state
+          state += sliced
+          ta_t = ta_t.write(time, out)
+          return (time + 1, ta_t, state)
+
+        (unused_0, h_final, unused_2) = control_flow_ops.while_loop(
+            cond=lambda time, unused_1, unused_2: time < 3,
+            body=body,
+            loop_vars=(time_0, ta, state0),
+            shape_invariants=(time_0.get_shape(), tensor_shape.unknown_shape(),
+                              tensor_shape.unknown_shape()),
+            parallel_iterations=3)
+        vout = h_final.stack()
+        return vout
+
       v0 = array_ops.identity(np.arange(3 * 5, dtype=np_dtype).reshape(3, 5))
-      var = variables.Variable(np.arange(100, 105, dtype=np_dtype))
       state0 = array_ops.identity(np.array([1] * 5, dtype=np_dtype))
-      ta = tensor_array_ops.TensorArray(
-          dtype=dtype,
-          tensor_array_name="foo",
-          size=0 if dynamic_size else 3,
-          dynamic_size=dynamic_size)
-      time_0 = array_ops.identity(0)
-
-      def body(time, ta_t, state):
-        sliced = array_ops.slice(
-            v0, begin=array_ops.stack([time, 0]), size=[1, -1])
-        sliced = array_ops.squeeze(sliced)
-        out = sliced + var + state
-        state += sliced
-        ta_t = ta_t.write(time, out)
-        return (time + 1, ta_t, state)
-
-      (unused_0, h_final, unused_2) = control_flow_ops.while_loop(
-          cond=lambda time, unused_1, unused_2: time < 3,
-          body=body,
-          loop_vars=(time_0, ta, state0),
-          shape_invariants=(time_0.get_shape(), tensor_shape.unknown_shape(),
-                            tensor_shape.unknown_shape()),
-          parallel_iterations=3)
-      vout = h_final.stack()
-
+      init_val = np.arange(100, 105, dtype=np_dtype)
+      var = variable_scope.get_variable(
+          "var",
+          shape=init_val.shape,
+          dtype=np_dtype,
+          initializer=init_ops.constant_initializer(init_val))
+
+      vout = func(v0, state0, var)
       grad_val = -np.arange(3 * 5, dtype=np_dtype).reshape(3, 5)
-      v0_grad = gradients_impl.gradients([vout], [v0], [grad_val])[0]
-      state0_grad = gradients_impl.gradients([vout], [state0], [grad_val])[0]
-      var_grad = gradients_impl.gradients([vout], [var], [grad_val])[0]
+      if context.in_graph_mode():
+        v0_grad = gradients_impl.gradients([vout], [v0], [grad_val])[0]
+        state0_grad = gradients_impl.gradients([vout], [state0], [grad_val])[0]
+        var_grad = gradients_impl.gradients([vout], [var], [grad_val])[0]
+        variables.global_variables_initializer().run()
+      else:
+        grad_fn = backprop.gradients_function(func)
+        v0_grad, state0_grad, var_grad = grad_fn(v0, state0, var, dy=grad_val)
 
-      variables.global_variables_initializer().run()
       state0_t, var_t, v0_t, vout_t, v0_grad_t, var_grad_t, state0_grad_t = (
-          session.run([state0, var, v0, vout, v0_grad, var_grad, state0_grad]))
-      just_v0_grad_t, = session.run([v0_grad])
+          self.evaluate(
+              ([state0, var, v0, vout, v0_grad, var_grad, state0_grad])))
+      just_v0_grad_t = self.evaluate(v0_grad)
 
       # state = [ state0 | state0 + v0[0] | state0 + v0[0] + v0[1] ]
       # vout = [ v0[0] + var + state[0] |
@@ -838,6 +898,7 @@ class TensorArrayTest(test.TestCase):
       self.assertAllClose(grad_val.sum(axis=0), var_grad_t)
       self.assertAllClose(grad_val.sum(axis=0), state0_grad_t)
 
+  @test_util.run_in_graph_and_eager_modes()
   def testWhileLoopWritePackGradients(self):
     self._testWhileLoopWritePackGradients(
         dynamic_size=False, dtype=dtypes.float32)
@@ -849,38 +910,45 @@ class TensorArrayTest(test.TestCase):
     self._testWhileLoopWritePackGradients(
         dynamic_size=True, dtype=dtypes.float32)
 
+  @test_util.run_in_graph_and_eager_modes()
   def testGradSerialTwoLoops(self):
     with self.test_session(use_gpu=True):
-      num_steps = 100
-      acc = tensor_array_ops.TensorArray(
-          dtype=dtypes.float32,
-          size=num_steps,
-          clear_after_read=False,
-          element_shape=tensor_shape.scalar())
-      i = constant_op.constant(0, name="i")
-      x = constant_op.constant(2.0, name="x")
+      def loop(x):
+        num_steps = 100
+        acc = tensor_array_ops.TensorArray(
+            dtype=dtypes.float32,
+            size=num_steps,
+            clear_after_read=False,
+            element_shape=tensor_shape.scalar())
+        i = constant_op.constant(0, name="i")
+
+        c = lambda i, acc: i < 5
 
-      c = lambda i, acc: i < 5
+        def b(i, acc):
+          x1 = control_flow_ops.cond(
+              math_ops.equal(i, 0), lambda: x,
+              lambda: math_ops.multiply(acc.read(i - 1), 2.0))
+          return i + 1, acc.write(i, x1)
 
-      def b(i, acc):
-        x1 = control_flow_ops.cond(
-            math_ops.equal(i, 0), lambda: x,
-            lambda: math_ops.multiply(acc.read(i - 1), 2.0))
-        return i + 1, acc.write(i, x1)
+        i1, acc1 = control_flow_ops.while_loop(c, b, [i, acc])
 
-      i1, acc1 = control_flow_ops.while_loop(c, b, [i, acc])
+        z = constant_op.constant(0.0)
 
-      z = constant_op.constant(0.0)
+        def fn(i, acc):
+          return i + 1, acc.write(i, z)
 
-      def fn(i, acc):
-        return i + 1, acc.write(i, z)
+        _, acc2 = control_flow_ops.while_loop(lambda i, acc: i < num_steps, fn,
+                                              [i1, acc1])
 
-      _, acc2 = control_flow_ops.while_loop(lambda i, acc: i < num_steps, fn,
-                                            [i1, acc1])
+        r = acc2.stack()
+        return r
 
-      r = acc2.stack()
-      grad = gradients_impl.gradients(r, [x])[0]
-      self.assertAllClose(31.0, grad.eval())
+      x = constant_op.constant(2.0, name="x")
+      if context.in_graph_mode():
+        grad = gradients_impl.gradients(loop(x), [x])[0]
+      else:
+        grad = backprop.gradients_function(loop)(x)[0]
+      self.assertAllClose(31.0, self.evaluate(grad))
 
   def testSumOfTwoReadVariablesWithoutRepeatGrad(self):
     with self.test_session(use_gpu=True) as session:
@@ -1019,6 +1087,7 @@ class TensorArrayTest(test.TestCase):
       r5 = w5.read(0)
       self.assertAllEqual([5, 4, 2, 3], r5.get_shape().as_list())
 
+  @test_util.run_in_graph_and_eager_modes()
   def _testUnpackShape(self):
     with self.test_session(use_gpu=True):
       ta = tensor_array_ops.TensorArray(
@@ -1035,8 +1104,15 @@ class TensorArrayTest(test.TestCase):
 
       c1 = constant_op.constant([4.0, 5.0])
       w1 = w0.write(3, c1)
-      r1 = w1.read(0)
-      self.assertAllEqual(c1.get_shape(), r1.get_shape())
+
+      with self.assertRaisesOpError(
+          r"Could not read index 0 twice because it was cleared after a "
+          r"previous read \(perhaps try setting clear_after_read = false\?\)"):
+        with ops.control_dependencies([r0]):
+          self.evaluate(w1.read(0))
+
+      r1 = w1.read(1)
+      self.assertAllEqual(c1.get_shape(), r1.shape)
 
       c2 = constant_op.constant([4.0, 5.0, 6.0])
       with self.assertRaises(ValueError):
@@ -1045,6 +1121,7 @@ class TensorArrayTest(test.TestCase):
   def testUnpackShape(self):
     self._testUnpackShape()
 
+  @test_util.run_in_graph_and_eager_modes()
   def testSplitShape(self):
     with self.test_session(use_gpu=True):
       ta = tensor_array_ops.TensorArray(
@@ -1066,10 +1143,14 @@ class TensorArrayTest(test.TestCase):
           infer_shape=True)
       w0 = ta1.split(value, [1, 2])
       r0 = w0.read(0)
-      self.assertEqual(r0.get_shape().ndims, None)
-      self.assertEqual(
-          tensor_shape.TensorShape(
-              ta1.handle.op.get_attr("element_shape")).ndims, None)
+      if context.in_graph_mode():
+        self.assertEqual(r0.get_shape().ndims, None)
+        self.assertEqual(
+            tensor_shape.TensorShape(
+                ta1.handle.op.get_attr("element_shape")).ndims, None)
+      else:
+        self.assertEqual((1, 2), r0.get_shape())
+        self.assertEqual((2, 2), w0.read(1).get_shape())
 
   def testWriteUnknownShape(self):
     with self.test_session(use_gpu=True):
@@ -1137,6 +1218,8 @@ class TensorArrayTest(test.TestCase):
   def testTensorArrayEvalEmpty(self):
     self._testTensorArrayEvalEmpty()
 
+  # this test is ill-defined for Eager mode --- unpacking an empty tensor
+  # gives an empty list / there is not equivalent of "mark_used" in Eager
   def _testTensorArrayEvalEmptyWithDefault(self):
     with self.test_session(use_gpu=True):
       ta = tensor_array_ops.TensorArray(
@@ -1180,6 +1263,7 @@ class TensorArrayTest(test.TestCase):
       self.assertAllEqual([10.0, -10.0], read_vals[1])
       self.assertAllEqual([[2.0, 3.0], [4.0, 5.0]], grad_vals[0])
 
+  @test_util.run_in_graph_and_eager_modes()
   def testTensorArrayWriteGatherAndGradients(self):
     with self.test_session(use_gpu=True) as session:
       ta = tensor_array_ops.TensorArray(
@@ -1188,16 +1272,23 @@ class TensorArrayTest(test.TestCase):
           size=0,
           dynamic_size=True)
 
-      values = constant_op.constant([[1.0 * x, -1.0 * x] for x in range(10)])
-      indices = constant_op.constant([1, 8])
-
-      w = ta.unstack(values)
-      g = w.gather(indices)
+      def func(values):
+        indices = constant_op.constant([1, 8])
+        w = ta.unstack(values)
+        g = w.gather(indices)
+        return g
 
+      values = constant_op.constant([[1.0 * x, -1.0 * x] for x in range(10)])
+      g = func(values)
+      grad_ys = [[[2.0, 3.0], [4.0, 5.0]]]
       # Test combined gradients + aggregation of read(0)
-      grad = gradients_impl.gradients(
-          ys=[g], xs=[values], grad_ys=[[[2.0, 3.0], [4.0, 5.0]]])
-      g_vals, grad_vals = session.run([[g], grad])
+      if context.in_graph_mode():
+        grad = gradients_impl.gradients(ys=[g], xs=[values], grad_ys=grad_ys)
+        g_vals, grad_vals = session.run([[g], grad])
+      else:
+        g_vals = [g]
+        grad_vals = backprop.gradients_function(func)(
+            values, dy=constant_op.constant(grad_ys[0], dtype=dtypes.float32))
 
       # Gradients for 8 of the 10 unread components are zero.
       expected_grad = np.zeros((10, 2))
@@ -1316,8 +1407,9 @@ class TensorArrayTest(test.TestCase):
         self.assertFalse(
             [s for s in dev_stats[d] if "/TensorArray" in s.node_name])
 
+  @test_util.run_in_graph_and_eager_modes()
   def testTensorArrayIdentity(self):
-    with self.test_session(use_gpu=True) as session:
+    with self.test_session(use_gpu=True):
       ta0 = tensor_array_ops.TensorArray(dtype=dtypes.float32, size=2,
                                          infer_shape=False)
       ta1 = tensor_array_ops.TensorArray(dtype=dtypes.int32, size=4,
@@ -1326,8 +1418,10 @@ class TensorArrayTest(test.TestCase):
       ta0 = ta0.write(0, 0.)
       ta1 = ta1.write(0, 1)
 
-      v0 = variables.Variable(0)
-      v1 = variables.Variable(0)
+      v0 = variable_scope.get_variable(
+          "v0", shape=(), initializer=init_ops.zeros_initializer())
+      v1 = variable_scope.get_variable(
+          "v1", shape=(), initializer=init_ops.zeros_initializer())
 
       with ops.control_dependencies([v0.assign_add(1)]):
         ta0 = ta0.identity()
@@ -1344,17 +1438,21 @@ class TensorArrayTest(test.TestCase):
       # Tests correct properties on new TensorArrays.
       self.assertEqual(dtypes.float32, ta0.dtype)
       self.assertEqual(dtypes.int32, ta1.dtype)
-      self.assertEqual(tensor_shape.unknown_shape(), read0.get_shape())
+      if context.in_graph_mode():
+        self.assertEqual(tensor_shape.unknown_shape(), read0.get_shape())
+      else:
+        self.assertEqual(tensor_shape.scalar(), read1.get_shape())
       self.assertEqual(tensor_shape.scalar(), read1.get_shape())
 
-      variables.global_variables_initializer().run()
+      if context.in_graph_mode():
+        variables.global_variables_initializer().run()
 
-      read0_v, read1_v, size0_v, size1_v = session.run(
-          (read0, read1, size0, size1))
+      read0_v, read1_v, size0_v, size1_v = self.evaluate((read0, read1, size0,
+                                                          size1))
 
       # Tests that the control dependencies was added and executed.
-      self.assertEqual(1, v0.eval())
-      self.assertEqual(1, v1.eval())
+      self.assertEqual(1, self.evaluate(v0))
+      self.assertEqual(1, self.evaluate(v1))
 
       # Tests correct TensorArray.
       self.assertEqual(read0_v, 0)
diff --git a/tensorflow/python/ops/tensor_array_ops.py b/tensorflow/python/ops/tensor_array_ops.py
index 37b4b3bcf9..b4b7ad9d91 100644
--- a/tensorflow/python/ops/tensor_array_ops.py
+++ b/tensorflow/python/ops/tensor_array_ops.py
@@ -25,6 +25,9 @@ from __future__ import print_function
 import contextlib
 
 from tensorflow.python.eager import context
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import errors_impl
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
 from tensorflow.python.framework import tensor_util
@@ -34,15 +37,11 @@ from tensorflow.python.ops import math_ops
 from tensorflow.python.util import tf_should_use
 
 
-# TensorArray object accesses many of the hidden generated ops, but is
-# in fact built to wrap these methods.
+# _GraphTensorArray accesses many of the hidden generated ops, but is in
+# fact built to wrap these methods.
 # pylint: disable=protected-access
-class TensorArray(object):
-  """Class wrapping dynamic-sized, per-time-step, write-once Tensor arrays.
-
-  This class is meant to be used with dynamic iteration primitives such as
-  `while_loop` and `map_fn`.  It supports gradient back-propagation via special
-  "flow" control flow dependencies.
+class _GraphTensorArray(object):
+  """Graph-mode implementation of TensorArray.
   """
 
   def __init__(self,
@@ -57,14 +56,7 @@ class TensorArray(object):
                element_shape=None,
                colocate_with_first_write_call=True,
                name=None):
-    """Construct a new TensorArray or wrap an existing TensorArray handle.
-
-    A note about the parameter `name`:
-
-    The name of the `TensorArray` (even if passed in) is uniquified: each time
-    a new `TensorArray` is created at runtime it is assigned its own name for
-    the duration of the run.  This avoids name collisions if a `TensorArray`
-    is created within a `while_loop`.
+    """Constructs a graph mode TensorArray.
 
     Args:
       dtype: (required) data type of the TensorArray.
@@ -79,9 +71,9 @@ class TensorArray(object):
         This is used when creating the TensorArray handle.  If this value is
         set, handle should be None.
       handle: (optional) A `Tensor` handle to an existing TensorArray.  If this
-        is set, tensor_array_name should be None.
+        is set, tensor_array_name should be None. Only supported in graph mode.
       flow: (optional) A float `Tensor` scalar coming from an existing
-        `TensorArray.flow`.
+        `TensorArray.flow`. Only supported in graph mode.
       infer_shape: (optional, default: True) If True, shape inference
         is enabled.  In this case, all elements must have the same shape.
       element_shape: (optional, default: None) A `TensorShape` object specifying
@@ -170,17 +162,14 @@ class TensorArray(object):
 
   @property
   def flow(self):
-    """The flow `Tensor` forcing ops leading to this TensorArray state."""
     return self._flow
 
   @property
   def dtype(self):
-    """The data type of this TensorArray."""
     return self._dtype
 
   @property
   def handle(self):
-    """The reference to the TensorArray."""
     return self._handle
 
   def _merge_element_shape(self, shape):
@@ -225,13 +214,7 @@ class TensorArray(object):
         yield
 
   def identity(self):
-    """Returns a TensorArray with the same content and properties.
-
-    Returns:
-      A new TensorArray object with flow that ensures the control dependencies
-      from the contexts will become control dependencies for writes, reads, etc.
-      Use this object all for subsequent operations.
-    """
+    """See TensorArray."""
     flow = array_ops.identity(self._flow)
     ta = TensorArray(
         dtype=self._dtype, handle=self._handle, flow=flow,
@@ -242,6 +225,7 @@ class TensorArray(object):
     return ta
 
   def grad(self, source, flow=None, name=None):
+    """See TensorArray."""
     # tensor_array_grad requires a flow input when forward
     # TensorArrays are dynamically sized.  This forces the creation
     # of the grad TensorArray only once the final forward array's size
@@ -264,15 +248,7 @@ class TensorArray(object):
         return g
 
   def read(self, index, name=None):
-    """Read the value at location `index` in the TensorArray.
-
-    Args:
-      index: 0-D.  int32 tensor with the index to read from.
-      name: A name for the operation (optional).
-
-    Returns:
-      The tensor at index `index`.
-    """
+    """See TensorArray."""
     value = gen_data_flow_ops._tensor_array_read_v3(
         handle=self._handle,
         index=index,
@@ -285,20 +261,7 @@ class TensorArray(object):
 
   @tf_should_use.should_use_result
   def write(self, index, value, name=None):
-    """Write `value` into index `index` of the TensorArray.
-
-    Args:
-      index: 0-D.  int32 scalar with the index to write to.
-      value: N-D.  Tensor of type `dtype`.  The Tensor to write to this index.
-      name: A name for the operation (optional).
-
-    Returns:
-      A new TensorArray object with flow that ensures the write occurs.
-      Use this object all for subsequent operations.
-
-    Raises:
-      ValueError: if there are more writers than specified.
-    """
+    """See TensorArray."""
     with ops.name_scope(name, "TensorArrayWrite", [self._handle, index, value]):
       value = ops.convert_to_tensor(value, name="value")
       if self._infer_shape:
@@ -319,35 +282,13 @@ class TensorArray(object):
       return ta
 
   def stack(self, name=None):
-    """Return the values in the TensorArray as a stacked `Tensor`.
-
-    All of the values must have been written and their shapes must all match.
-    If input shapes have rank-`R`, then output shape will have rank-`(R+1)`.
-
-    Args:
-      name: A name for the operation (optional).
-
-    Returns:
-      All the tensors in the TensorArray stacked into one tensor.
-    """
+    """See TensorArray."""
     with ops.colocate_with(self._handle):
       with ops.name_scope(name, "TensorArrayStack", [self._handle]):
         return self.gather(math_ops.range(0, self.size()), name=name)
 
   def gather(self, indices, name=None):
-    """Return selected values in the TensorArray as a packed `Tensor`.
-
-    All of selected values must have been written and their shapes
-    must all match.
-
-    Args:
-      indices: A `1-D` `Tensor` taking values in `[0, max_value)`.  If
-        the `TensorArray` is not dynamic, `max_value=size()`.
-      name: A name for the operation (optional).
-
-    Returns:
-      The in the `TensorArray` selected by `indices`, packed into one tensor.
-    """
+    """See TensorArray."""
     if self._element_shape:
       element_shape = self._element_shape[0]
     else:
@@ -364,17 +305,7 @@ class TensorArray(object):
     return value
 
   def concat(self, name=None):
-    """Return the values in the TensorArray as a concatenated `Tensor`.
-
-    All of the values must have been written, their ranks must match, and
-    and their shapes must all match for all dimensions except the first.
-
-    Args:
-      name: A name for the operation (optional).
-
-    Returns:
-      All the tensors in the TensorArray concatenated into one tensor.
-    """
+    """See TensorArray."""
     if self._element_shape and self._element_shape[0].dims is not None:
       element_shape_except0 = (
           tensor_shape.TensorShape(self._element_shape[0].dims[1:]))
@@ -392,22 +323,7 @@ class TensorArray(object):
 
   @tf_should_use.should_use_result
   def unstack(self, value, name=None):
-    """Unstack the values of a `Tensor` in the TensorArray.
-
-    If input value shapes have rank-`R`, then the output TensorArray will
-    contain elements whose shapes are rank-`(R-1)`.
-
-    Args:
-      value: (N+1)-D.  Tensor of type `dtype`.  The Tensor to unstack.
-      name: A name for the operation (optional).
-
-    Returns:
-      A new TensorArray object with flow that ensures the unstack occurs.
-      Use this object all for subsequent operations.
-
-    Raises:
-      ValueError: if the shape inference fails.
-    """
+    """See TensorArray."""
     with ops.name_scope(name, "TensorArrayUnstack", [self._handle, value]):
       num_elements = array_ops.shape(value)[0]
       return self.scatter(
@@ -415,21 +331,7 @@ class TensorArray(object):
 
   @tf_should_use.should_use_result
   def scatter(self, indices, value, name=None):
-    """Scatter the values of a `Tensor` in specific indices of a `TensorArray`.
-
-    Args:
-      indices: A `1-D` `Tensor` taking values in `[0, max_value)`.  If
-        the `TensorArray` is not dynamic, `max_value=size()`.
-      value: (N+1)-D.  Tensor of type `dtype`.  The Tensor to unpack.
-      name: A name for the operation (optional).
-
-    Returns:
-      A new TensorArray object with flow that ensures the scatter occurs.
-      Use this object all for subsequent operations.
-
-    Raises:
-      ValueError: if the shape inference fails.
-    """
+    """See TensorArray."""
     with ops.name_scope(name, "TensorArrayScatter",
                         [self._handle, value, indices]):
       value = ops.convert_to_tensor(value, name="value")
@@ -452,21 +354,7 @@ class TensorArray(object):
 
   @tf_should_use.should_use_result
   def split(self, value, lengths, name=None):
-    """Split the values of a `Tensor` into the TensorArray.
-
-    Args:
-      value: (N+1)-D.  Tensor of type `dtype`.  The Tensor to split.
-      lengths: 1-D.  int32 vector with the lengths to use when splitting
-        `value` along its first dimension.
-      name: A name for the operation (optional).
-
-    Returns:
-      A new TensorArray object with flow that ensures the split occurs.
-      Use this object all for subsequent operations.
-
-    Raises:
-      ValueError: if the shape inference fails.
-    """
+    """See TensorArray."""
     with ops.name_scope(name, "TensorArraySplit",
                         [self._handle, value, lengths]):
       value = ops.convert_to_tensor(value, name="value")
@@ -494,14 +382,627 @@ class TensorArray(object):
       return ta
 
   def size(self, name=None):
-    """Return the size of the TensorArray."""
+    """See TensorArray."""
     return gen_data_flow_ops._tensor_array_size_v3(
         handle=self._handle, flow_in=self.flow, name=name)
 
   @tf_should_use.should_use_result
   def close(self, name=None):
-    """Close the current TensorArray."""
+    """See TensorArray."""
     return gen_data_flow_ops._tensor_array_close_v3(
         handle=self._handle, name=name)
 
 # pylint: enable=protected-access
+
+
+# pylint: disable=protected-access
+def _eager_write_no_copy(ta, index, value):
+  """Writes value into an _EagerTensorArray without creating a new TensorArray.
+
+  Args:
+    ta: _EagerTensorArray into which to write value.
+    index: 0-D.  int32 scalar with the index to write to.
+    value: N-D.  Tensor of type `dtype`.  The Tensor to write to this index.
+
+  Raises:
+    errors_impl.AlreadyExistsError: attempting to overwrite an entry.
+    errors_impl.InvalidArgumentError: value dtype does not match `ta`'s dtype.
+    errors_impl.OutOfRangeError: `index` is out of bounds.
+    ValueError: shape of `value` is not consistent with inferred shape.
+  """
+
+  if isinstance(index, ops.EagerTensor):
+    index = index.numpy()
+
+  if index < 0:
+    raise errors_impl.OutOfRangeError(
+        None, None,
+        "Writing to negative indices (index %d) is not allowed." % index)
+
+  tensor_array = ta._tensor_array
+  size = len(tensor_array)
+  if index >= size:
+    if not ta._dynamic_size:
+      raise errors_impl.OutOfRangeError(
+          None, None,
+          "Tried to write to index %d but array is not resizeable and size "
+          "is: %d" % (index, size))
+    tensor_array.extend([None for _ in range(index - size + 1)])
+
+  if not isinstance(value, ops.EagerTensor):
+    value = constant_op.constant(value)
+
+  if ta._infer_shape:
+    if ta._element_shape is None:
+      ta._element_shape = value.shape
+    elif ta._element_shape != value.shape:
+      raise ValueError("Incompatible shape for value (%s), expected (%s)" %
+                       (value.shape.as_list(), ta._element_shape.as_list()))
+
+  if ta._dtype != value.dtype:
+    raise errors_impl.InvalidArgumentError(
+        None, None,
+        "TensorArray dtype is %s but Op is trying to write dtype %s" %
+        (ta._dtype.name, value.dtype.name))
+
+  if ta._tensor_array[index] is not None:
+    raise errors_impl.AlreadyExistsError(
+        None, None,
+        "Could not write to TensorArray index %d because it has already been "
+        "written to." % index)
+
+  tensor_array[index] = value
+
+# pylint: enable=protected-access
+
+
+class _EagerTensorArray(object):
+  """Eager-mode implementation of TensorArray.
+  """
+
+  def __init__(self,
+               dtype,
+               size=None,
+               dynamic_size=None,
+               clear_after_read=None,
+               tensor_array_name=None,
+               handle=None,
+               flow=None,
+               infer_shape=True,
+               element_shape=None,
+               colocate_with_first_write_call=True,
+               name=None):
+    """Constructs an Eager mode TensorArray.
+
+    Args:
+      dtype: (required) data type of the TensorArray.
+      size: (optional) int32 scalar `Tensor`: the size of the TensorArray.
+        Required if handle is not provided.
+      dynamic_size: (optional) Python bool: If true, writes to the TensorArray
+        can grow the TensorArray past its initial size.  Default: False.
+      clear_after_read: Boolean (optional, default: True).  If True, clear
+        TensorArray values after reading them.  This disables read-many
+        semantics, but allows early release of memory.
+      tensor_array_name: unused.
+      handle: unsupported.
+      flow: unsupported.
+      infer_shape: used for error checking, same semantics as TensorArray.
+      element_shape: used for error checking, same semantics as TensorArray.
+      colocate_with_first_write_call: unsupported.
+      name: unsupported.
+
+    Raises:
+      ValueError: handle or flow are supplied, or if size is not supplied.
+    """
+
+    del (flow, tensor_array_name, name)  # not meaningful in Eager
+
+    if handle is not None:
+      raise ValueError("TensorArray handles are not supported in Eager mode.")
+    if size is None:
+      raise ValueError("Size must be declared for TensorArrays in Eager mode.")
+
+    # These attributes are not meaningful in Eager, but some library functions
+    # (e.g., those in control_flow_ops.py) access them to create new tensor
+    # arrays; as such, we define them for the sake of compatibility.
+    self._handle = None
+    # we assign a dummy value to _flow in case other code assumes it to be
+    # a Tensor
+    self._flow = constant_op.constant(0, dtype=dtypes.int32)
+    self._infer_shape = infer_shape
+    self._element_shape = element_shape
+    self._colocate_with_first_write_call = colocate_with_first_write_call
+
+    self._dtype = dtype
+    self._dynamic_size = dynamic_size or False
+    self._clear_after_read = (
+        True if clear_after_read is None else clear_after_read)
+    self._previously_read_indices = []
+
+    if isinstance(size, ops.EagerTensor):
+      size = size.numpy()
+    self._tensor_array = [None for _ in range(size)]
+
+  @property
+  def flow(self):
+    """Flows are not meaningful in Eager; this exists for compatibility."""
+    return self._flow
+
+  @property
+  def dtype(self):
+    return self._dtype
+
+  @property
+  def handle(self):
+    """Handles are not meaningful in Eager; this exists for compatibility."""
+    return self._handle
+
+  def _identity_without_array(self):
+    """Returns a new TensorArray with the same properties as this Eager one.
+
+    NB: Does not set the underlying _tensor_array attribute.
+    """
+    ta = TensorArray(
+        dtype=self._dtype,
+        size=len(self._tensor_array),
+        dynamic_size=self._dynamic_size,
+        clear_after_read=self._clear_after_read,
+        handle=self._handle,
+        flow=self._flow,
+        infer_shape=self._infer_shape,
+        element_shape=self._element_shape,
+        colocate_with_first_write_call=self._colocate_with_first_write_call)
+    ta._implementation._previously_read_indices = self._previously_read_indices  # pylint: disable=protected-access
+    return ta
+
+  def identity(self):
+    """See TensorArray."""
+    ta = self._identity_without_array()
+    ta._implementation._tensor_array = [t for t in self._tensor_array]  # pylint: disable=protected-access
+    return ta
+
+  def grad(self, source, flow=None, name=None):
+    raise NotImplementedError(
+        "TensorArray.grad is not supported in Eager mode; Eager's gradient "
+        "implementation does not use/need this function to compute gradients "
+        "of operations that use TensorArrays.")
+
+  def read(self, index, name=None):
+    """See TensorArray."""
+    del name  # not meaningful in Eager mode
+
+    if isinstance(index, ops.EagerTensor):
+      index = index.numpy()
+
+    if index < 0:
+      raise errors_impl.OutOfRangeError(
+          None, None,
+          "Reading from negative indices (index %d) is not allowed." % index)
+
+    if index >= len(self._tensor_array):
+      raise errors_impl.OutOfRangeError(
+          None, None, "Tried to read from index %d but array size is: %d" %
+          (index, len(self._tensor_array)))
+
+    tensor = self._tensor_array[index]
+    if tensor is None:
+      if index in self._previously_read_indices:
+        raise errors_impl.InvalidArgumentError(
+            None, None,
+            "Could not read index %d twice because it was cleared after "
+            "a previous read (perhaps try setting clear_after_read = false?)" %
+            index)
+      else:
+        raise errors_impl.InvalidArgumentError(
+            None, None,
+            "Could not read from TensorArray index %d because it has not yet "
+            "been written to." % index)
+
+    if self._clear_after_read:
+      self._tensor_array[index] = None
+      self._previously_read_indices.append(index)
+    return tensor
+
+  def write(self, index, value, name=None):
+    """See TensorArray."""
+    del name  # not meaningful in Eager mode
+    ta = self.identity()
+    _eager_write_no_copy(ta._implementation, index, value)  # pylint: disable=protected-access
+    return ta
+
+  def stack(self, name=None):
+    """See TensorArray."""
+    try:
+      return array_ops.stack(self._tensor_array, name=name)
+    except ValueError:
+      if None in self._tensor_array:
+        idx = self._tensor_array.index(None)
+        raise errors_impl.InvalidArgumentError(
+            None, None, "Could not read from TensorArray index %d because "
+            "it has not yet been written to." % idx)
+      else:
+        raise
+
+  def gather(self, indices, name=None):
+    """See TensorArray."""
+    del name  # not meaningful in Eager mode
+    return array_ops.stack([self._tensor_array[i] for i in indices.numpy()])
+
+  def concat(self, name=None):
+    """See TensorArray."""
+    try:
+      return array_ops.concat(self._tensor_array, 0, name=name)
+    except errors_impl.OpError:
+      # Reproduce a subset of the error-handling for graph-mode TensorArrays.
+      shapes = [t.shape for t in self._tensor_array]
+      ndims = [s.ndims for s in shapes]
+      if None in self._tensor_array:
+        # Concatenating empty TensorArrays is permitted if the element
+        # shape is defined; the output is a tensor with shape
+        # [0] + self._element_shape[1:]
+        if all(t is None for t in self._tensor_array):
+          if self._element_shape is not None:
+            return constant_op.constant([], shape=[0] + self._element_shape[1:])
+          else:
+            raise errors_impl.UnimplementedError(
+                None, None, "TensorArray has size zero, but "
+                "element_shape_except0 %s is not fully defined. Currently only "
+                "static shapes are supported when concatenating zero-size "
+                "TensorArrays." % self._element_shape[1:])
+        # Concatenating a TensorArray in which some but not all entries have
+        # been written to is not allowed.
+        idx = self._tensor_array.index(None)
+        raise errors_impl.InvalidArgumentError(
+            None, None, "Could not read from TensorArray index %d because "
+            "it has not yet been written to." % idx)
+      elif 0 in ndims:
+        idx = ndims.index(0)
+        raise errors_impl.InvalidArgumentError(
+            None, None, "Concat saw a scalar shape at index %d but requires "
+            "at least vectors." % idx)
+      else:
+        raise
+
+  def unstack(self, value, name=None):
+    """See TensorArray."""
+    tensors = array_ops.unstack(value, name=name)
+    if len(tensors) > len(self._tensor_array) and not self._dynamic_size:
+      raise ValueError(
+          "Cannot unstack %d tensors into a TensorArray of static size %d" %
+          (len(tensors), len(self._tensors)))
+    ta = self._identity_without_array()
+    ta._implementation._tensor_array = tensors  # pylint: disable=protected-access
+    return ta
+
+  def scatter(self, indices, value, name=None):
+    """See TensorArray."""
+    del name  # unused in Eager
+    ta = self.identity()
+    for index, val in zip(indices.numpy(), array_ops.unstack(value)):
+      _eager_write_no_copy(ta._implementation, index, val)  # pylint: disable=protected-access
+    return ta
+
+  def split(self, value, lengths, name=None):
+    """See TensorArray."""
+    # error checking to match graph-mode errors
+    value = constant_op.constant(value)
+    lengths = constant_op.constant(lengths)
+    sum_lengths = math_ops.reduce_sum(lengths)
+    if lengths.shape.ndims != 1:
+      raise errors_impl.InvalidArgumentError(
+          None, None, "Expected lengths to be a vector, received shape: %s" %
+          lengths.shape.as_list())
+    elif value.shape.ndims == 0:
+      raise errors_impl.InvalidArgumentError(
+          None, None, "Expected value to be at least a vector, "
+          "but received shape: %s" % value.shape.as_list())
+    elif sum_lengths.numpy() != value.shape.as_list()[0]:
+      raise errors_impl.InvalidArgumentError(
+          None, None, "Expected sum of lengths to be equal to "
+          "values.shape[0], but sum of lengths is %d and "
+          "value's shape is: %s " % (sum_lengths.numpy(),
+                                     value.shape.as_list()))
+    elif not self._dynamic_size and lengths.shape[0] != len(self._tensor_array):
+      raise errors_impl.InvalidArgumentError(
+          None, None, "TensorArray's size is not equal to the size of "
+          "lengths (%d vs. %d), and the TensorArray is not marked as "
+          "dynamically resizeable" % (len(self._tensor_array),
+                                      lengths.shape[0]))
+    else:
+      ta = self._identity_without_array()
+      tensor_array = array_ops.split(value, lengths, name=name)
+      ta._implementation._tensor_array = tensor_array  # pylint: disable=protected-access
+      return ta
+
+  def size(self, name=None):
+    """See TensorArray."""
+    del name  # not meaningful in Eager mode
+    return constant_op.constant(len(self._tensor_array))
+
+  def close(self, name=None):
+    del name  # not meaningful in Eager mode
+    del self._tensor_array[:]
+    return
+
+
+# TensorArray is designed to hide an underlying implementation object
+# and as such accesses many of that object's hidden fields.
+# pylint: disable=protected-access
+class TensorArray(object):
+  """Class wrapping dynamic-sized, per-time-step, write-once Tensor arrays.
+
+  This class is meant to be used with dynamic iteration primitives such as
+  `while_loop` and `map_fn`.  It supports gradient back-propagation via special
+  "flow" control flow dependencies.
+  """
+
+  def __init__(self,
+               dtype,
+               size=None,
+               dynamic_size=None,
+               clear_after_read=None,
+               tensor_array_name=None,
+               handle=None,
+               flow=None,
+               infer_shape=True,
+               element_shape=None,
+               colocate_with_first_write_call=True,
+               name=None):
+    """Construct a new TensorArray or wrap an existing TensorArray handle.
+
+    A note about the parameter `name`:
+
+    The name of the `TensorArray` (even if passed in) is uniquified: each time
+    a new `TensorArray` is created at runtime it is assigned its own name for
+    the duration of the run.  This avoids name collisions if a `TensorArray`
+    is created within a `while_loop`.
+
+    Args:
+      dtype: (required) data type of the TensorArray.
+      size: (optional) int32 scalar `Tensor`: the size of the TensorArray.
+        Required if handle is not provided.
+      dynamic_size: (optional) Python bool: If true, writes to the TensorArray
+        can grow the TensorArray past its initial size.  Default: False.
+      clear_after_read: Boolean (optional, default: True).  If True, clear
+        TensorArray values after reading them.  This disables read-many
+        semantics, but allows early release of memory.
+      tensor_array_name: (optional) Python string: the name of the TensorArray.
+        This is used when creating the TensorArray handle.  If this value is
+        set, handle should be None.
+      handle: (optional) A `Tensor` handle to an existing TensorArray.  If this
+        is set, tensor_array_name should be None. Only supported in graph mode.
+      flow: (optional) A float `Tensor` scalar coming from an existing
+        `TensorArray.flow`. Only supported in graph mode.
+      infer_shape: (optional, default: True) If True, shape inference
+        is enabled.  In this case, all elements must have the same shape.
+      element_shape: (optional, default: None) A `TensorShape` object specifying
+        the shape constraints of each of the elements of the TensorArray.
+        Need not be fully defined.
+      colocate_with_first_write_call: If `True`, the TensorArray will be
+        colocated on the same device as the Tensor used on its first write
+        (write operations include `write`, `unstack`, and `split`).  If `False`,
+        the TensorArray will be placed on the device determined by the
+        device context available during its initialization.
+      name: A name for the operation (optional).
+
+    Raises:
+      ValueError: if both handle and tensor_array_name are provided.
+      TypeError: if handle is provided but is not a Tensor.
+    """
+    if context.in_graph_mode():
+      implementation = _GraphTensorArray
+    else:
+      implementation = _EagerTensorArray
+
+    self._implementation = implementation(
+        dtype,
+        size=size,
+        dynamic_size=dynamic_size,
+        clear_after_read=clear_after_read,
+        tensor_array_name=tensor_array_name,
+        handle=handle,
+        flow=flow,
+        infer_shape=infer_shape,
+        element_shape=element_shape,
+        colocate_with_first_write_call=colocate_with_first_write_call,
+        name=name)
+
+  @property
+  def flow(self):
+    """The flow `Tensor` forcing ops leading to this TensorArray state."""
+    return self._implementation._flow
+
+  @property
+  def dtype(self):
+    """The data type of this TensorArray."""
+    return self._implementation._dtype
+
+  @property
+  def handle(self):
+    """The reference to the TensorArray."""
+    return self._implementation._handle
+
+  @property
+  def _infer_shape(self):
+    return self._implementation._infer_shape
+
+  @_infer_shape.setter
+  def _infer_shape(self, infer_shape):
+    self._implementation._infer_shape = infer_shape
+
+  @property
+  def _element_shape(self):
+    return self._implementation._element_shape
+
+  @_element_shape.setter
+  def _element_shape(self, element_shape):
+    self._implementation._element_shape = element_shape
+
+  @property
+  def _colocate_with_first_write_call(self):
+    return self._implementation._colocate_with_first_write_call
+
+  @property
+  def _colocate_with(self):
+    return self._implementation._colocate_with
+
+  @_colocate_with.setter
+  def _colocate_with(self, colocate_with):
+    self._implementation._colocate_with = colocate_with
+
+  def identity(self):
+    """Returns a TensorArray with the same content and properties.
+
+    Returns:
+      A new TensorArray object with flow that ensures the control dependencies
+      from the contexts will become control dependencies for writes, reads, etc.
+      Use this object all for subsequent operations.
+    """
+    return self._implementation.identity()
+
+  def grad(self, source, flow=None, name=None):
+    return self._implementation.grad(source, flow=flow, name=name)
+
+  def read(self, index, name=None):
+    """Read the value at location `index` in the TensorArray.
+
+    Args:
+      index: 0-D.  int32 tensor with the index to read from.
+      name: A name for the operation (optional).
+
+    Returns:
+      The tensor at index `index`.
+    """
+    return self._implementation.read(index, name=name)
+
+  @tf_should_use.should_use_result
+  def write(self, index, value, name=None):
+    """Write `value` into index `index` of the TensorArray.
+
+    Args:
+      index: 0-D.  int32 scalar with the index to write to.
+      value: N-D.  Tensor of type `dtype`.  The Tensor to write to this index.
+      name: A name for the operation (optional).
+
+    Returns:
+      A new TensorArray object with flow that ensures the write occurs.
+      Use this object all for subsequent operations.
+
+    Raises:
+      ValueError: if there are more writers than specified.
+    """
+    return self._implementation.write(index, value, name=name)
+
+  def stack(self, name=None):
+    """Return the values in the TensorArray as a stacked `Tensor`.
+
+    All of the values must have been written and their shapes must all match.
+    If input shapes have rank-`R`, then output shape will have rank-`(R+1)`.
+
+    Args:
+      name: A name for the operation (optional).
+
+    Returns:
+      All the tensors in the TensorArray stacked into one tensor.
+    """
+    return self._implementation.stack(name=name)
+
+  def gather(self, indices, name=None):
+    """Return selected values in the TensorArray as a packed `Tensor`.
+
+    All of selected values must have been written and their shapes
+    must all match.
+
+    Args:
+      indices: A `1-D` `Tensor` taking values in `[0, max_value)`.  If
+        the `TensorArray` is not dynamic, `max_value=size()`.
+      name: A name for the operation (optional).
+
+    Returns:
+      The tensors in the `TensorArray` selected by `indices`, packed into one
+      tensor.
+    """
+    return self._implementation.gather(indices, name=name)
+
+  def concat(self, name=None):
+    """Return the values in the TensorArray as a concatenated `Tensor`.
+
+    All of the values must have been written, their ranks must match, and
+    and their shapes must all match for all dimensions except the first.
+
+    Args:
+      name: A name for the operation (optional).
+
+    Returns:
+      All the tensors in the TensorArray concatenated into one tensor.
+    """
+    return self._implementation.concat(name=name)
+
+  @tf_should_use.should_use_result
+  def unstack(self, value, name=None):
+    """Unstack the values of a `Tensor` in the TensorArray.
+
+    If input value shapes have rank-`R`, then the output TensorArray will
+    contain elements whose shapes are rank-`(R-1)`.
+
+    Args:
+      value: (N+1)-D.  Tensor of type `dtype`.  The Tensor to unstack.
+      name: A name for the operation (optional).
+
+    Returns:
+      A new TensorArray object with flow that ensures the unstack occurs.
+      Use this object all for subsequent operations.
+
+    Raises:
+      ValueError: if the shape inference fails.
+    """
+    return self._implementation.unstack(value, name=name)
+
+  @tf_should_use.should_use_result
+  def scatter(self, indices, value, name=None):
+    """Scatter the values of a `Tensor` in specific indices of a `TensorArray`.
+
+    Args:
+      indices: A `1-D` `Tensor` taking values in `[0, max_value)`.  If
+        the `TensorArray` is not dynamic, `max_value=size()`.
+      value: (N+1)-D.  Tensor of type `dtype`.  The Tensor to unpack.
+      name: A name for the operation (optional).
+
+    Returns:
+      A new TensorArray object with flow that ensures the scatter occurs.
+      Use this object all for subsequent operations.
+
+    Raises:
+      ValueError: if the shape inference fails.
+    """
+    return self._implementation.scatter(indices, value, name=name)
+
+  @tf_should_use.should_use_result
+  def split(self, value, lengths, name=None):
+    """Split the values of a `Tensor` into the TensorArray.
+
+    Args:
+      value: (N+1)-D.  Tensor of type `dtype`.  The Tensor to split.
+      lengths: 1-D.  int32 vector with the lengths to use when splitting
+        `value` along its first dimension.
+      name: A name for the operation (optional).
+
+    Returns:
+      A new TensorArray object with flow that ensures the split occurs.
+      Use this object all for subsequent operations.
+
+    Raises:
+      ValueError: if the shape inference fails.
+    """
+    return self._implementation.split(value, lengths, name=name)
+
+  def size(self, name=None):
+    """Return the size of the TensorArray."""
+    return self._implementation.size(name=name)
+
+  @tf_should_use.should_use_result
+  def close(self, name=None):
+    """Close the current TensorArray."""
+    return self._implementation.close(name=name)
+
+# pylint: enable=protected-access
-- 
GitLab


From 06a79f5af7c861e695cfc20b7778519950aac9ba Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 26 Oct 2017 17:00:14 -0700
Subject: [PATCH 846/909] Move EyeFunctor to a separate file, and change it to
 a more efficient implementation (similar to matrix_set_diag).

PiperOrigin-RevId: 173611865
---
 tensorflow/core/kernels/BUILD                 | 22 +++++++---
 tensorflow/core/kernels/cuda_solvers.h        | 14 -------
 tensorflow/core/kernels/eye_functor.h         | 32 +++++++++++++++
 ...olvers_gpu.cu.cc => eye_functor_gpu.cu.cc} | 41 ++++++++-----------
 tensorflow/core/kernels/matrix_inverse_op.cc  |  1 +
 tensorflow/core/kernels/qr_op_impl.h          |  1 +
 6 files changed, 67 insertions(+), 44 deletions(-)
 create mode 100644 tensorflow/core/kernels/eye_functor.h
 rename tensorflow/core/kernels/{cuda_solvers_gpu.cu.cc => eye_functor_gpu.cu.cc} (62%)

diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index a3452f2f8c..0274f87ec6 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -1691,6 +1691,21 @@ tf_cc_tests(
     ],
 )
 
+tf_kernel_library(
+    name = "eye_functor",
+    hdrs = ["eye_functor.h"],
+    gpu_srcs = [
+        "eye_functor_gpu.cu.cc",
+        "eye_functor.h",
+    ],
+    visibility = [":friends"],
+    deps = [
+        "//tensorflow/core:framework",
+        "//third_party/eigen3",
+    ],
+    alwayslink = 0,
+)
+
 cc_library(
     name = "fifo_queue",
     srcs = ["fifo_queue.cc"],
@@ -2255,10 +2270,6 @@ tf_kernel_library(
     name = "cuda_solvers",
     srcs = ["cuda_solvers.cc"],
     hdrs = ["cuda_solvers.h"],
-    gpu_srcs = [
-        "cuda_solvers.h",
-        "cuda_solvers_gpu.cu.cc",
-    ],
     # @local_config_cuda//cuda:cusolver, //third_party/eigen3:blas,
     # and //third_party/libf2c all contain various parts of BLAS, LAPACK,
     # and f2c helper functions in global namespace. Tell the compiler to
@@ -2328,7 +2339,7 @@ tf_kernel_library(
 tf_kernel_library(
     name = "matrix_inverse_op",
     prefix = "matrix_inverse_op",
-    deps = LINALG_DEPS,
+    deps = LINALG_DEPS + if_cuda([":eye_functor"]),
 )
 
 tf_kernel_library(
@@ -2356,6 +2367,7 @@ tf_kernel_library(
     prefix = "qr_op",
     deps = LINALG_DEPS + if_cuda([
         ":cwise_op",
+        ":eye_functor",
         ":matrix_band_part_op",
     ]),
 )
diff --git a/tensorflow/core/kernels/cuda_solvers.h b/tensorflow/core/kernels/cuda_solvers.h
index af27eb6c47..3c389a82ab 100644
--- a/tensorflow/core/kernels/cuda_solvers.h
+++ b/tensorflow/core/kernels/cuda_solvers.h
@@ -27,7 +27,6 @@ limitations under the License.
 #include "cuda/include/cusolverDn.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/tensor.h"
-#include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/platform/stream_executor.h"
 
@@ -408,19 +407,6 @@ class DeviceLapackInfo : public ScratchSpace<int> {
   }
 };
 
-namespace functor {
-
-// Helper functor to set a batch of matrices to the identity.
-// TODO(rmlarsen): Use this kernel to replace the horribly inefficient tf.eye
-// op.
-template <typename Device, typename Scalar>
-struct EyeFunctor {
-  void operator()(const Device& device,
-                  typename TTypes<Scalar, 3>::Tensor matrix_batch);
-};
-
-}  // namespace functor
-
 template <typename Scalar>
 ScratchSpace<Scalar> CudaSolver::GetScratchSpace(const TensorShape& shape,
                                                  const string& debug_info,
diff --git a/tensorflow/core/kernels/eye_functor.h b/tensorflow/core/kernels/eye_functor.h
new file mode 100644
index 0000000000..70f093f813
--- /dev/null
+++ b/tensorflow/core/kernels/eye_functor.h
@@ -0,0 +1,32 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef THIRD_PARTY_TENSORFLOW_CORE_KERNELS_EYE_FUNCTOR_H_
+#define THIRD_PARTY_TENSORFLOW_CORE_KERNELS_EYE_FUNCTOR_H_
+
+#include "tensorflow/core/framework/tensor_types.h"
+
+namespace tensorflow {
+namespace functor {
+
+template <typename Device, typename Scalar>
+struct EyeFunctor {
+  void operator()(const Device& device,
+                  typename TTypes<Scalar, 3>::Tensor matrix_batch);
+};
+
+}  // namespace functor
+}  // namespace tensorflow
+
+#endif  // THIRD_PARTY_TENSORFLOW_CORE_KERNELS_EYE_FUNCTOR_H_
diff --git a/tensorflow/core/kernels/cuda_solvers_gpu.cu.cc b/tensorflow/core/kernels/eye_functor_gpu.cu.cc
similarity index 62%
rename from tensorflow/core/kernels/cuda_solvers_gpu.cu.cc
rename to tensorflow/core/kernels/eye_functor_gpu.cu.cc
index 84330c041a..a620316e27 100644
--- a/tensorflow/core/kernels/cuda_solvers_gpu.cu.cc
+++ b/tensorflow/core/kernels/eye_functor_gpu.cu.cc
@@ -17,11 +17,11 @@ limitations under the License.
 
 #define EIGEN_USE_GPU
 
-#include "tensorflow/core/kernels/cuda_solvers.h"
+#include "tensorflow/core/kernels/eye_functor.h"
 
-#include <complex>
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 #include "tensorflow/core/framework/tensor_types.h"
+#include "tensorflow/core/platform/types.h"
 #include "tensorflow/core/util/cuda_kernel_helper.h"
 
 namespace tensorflow {
@@ -30,26 +30,18 @@ namespace functor {
 typedef Eigen::GpuDevice GPUDevice;
 
 template <typename Scalar>
-__global__ void EyeKernel(Cuda3DLaunchConfig config, int batch_size, int m,
-                          int n, Scalar* matrix_batch_ptr) {
-  const int matrix_size = m * n;
+__global__ void EyeKernel(int num_threads, int batch_size, int m, int n,
+                          Scalar* output_ptr) {
   const Scalar one = Scalar(1);
-  CUDA_AXIS_KERNEL_LOOP(batch, config.virtual_thread_count, x) {
-    if (batch >= batch_size) {
-      break;
-    }
-    CUDA_AXIS_KERNEL_LOOP(row, config.virtual_thread_count, y) {
-      if (row >= m) {
-        break;
-      }
-      const int row_start = batch * matrix_size + row * n;
-      CUDA_AXIS_KERNEL_LOOP(col, config.virtual_thread_count, z) {
-        if (col >= n) {
-          break;
-        }
-        matrix_batch_ptr[row_start + col] = row == col ? one : Scalar();
-      }
-    }
+  const Scalar zero = Scalar(0);
+  CUDA_1D_KERNEL_LOOP(index, num_threads) {
+    // TODO(rmlarsen): Benchmark to see if it's just as fast to use mod (%),
+    // since it's easier to read.
+    const int global_row = index / n;
+    const int col = index - global_row * n;
+    const int batch = global_row / m;
+    const int row = global_row - batch * m;
+    output_ptr[index] = col == row ? one : zero;
   }
 }
 
@@ -60,11 +52,10 @@ struct EyeFunctor<GPUDevice, Scalar> {
     const int batch_size = matrix_batch.dimension(0);
     const int m = matrix_batch.dimension(1);
     const int n = matrix_batch.dimension(2);
-    Cuda3DLaunchConfig config = GetCuda3DLaunchConfig(batch_size, m, n, device,
-                                                      EyeKernel<Scalar>, 0, 0);
+    CudaLaunchConfig config = GetCudaLaunchConfig(batch_size * m * n, device);
     EyeKernel<<<config.block_count, config.thread_per_block, 0,
-                device.stream()>>>(config, batch_size, m, n,
-                                   matrix_batch.data());
+                device.stream()>>>(config.virtual_thread_count, batch_size, m,
+                                   n, matrix_batch.data());
   }
 };
 
diff --git a/tensorflow/core/kernels/matrix_inverse_op.cc b/tensorflow/core/kernels/matrix_inverse_op.cc
index cae84f52d7..c61a091c7b 100644
--- a/tensorflow/core/kernels/matrix_inverse_op.cc
+++ b/tensorflow/core/kernels/matrix_inverse_op.cc
@@ -33,6 +33,7 @@ limitations under the License.
 #if GOOGLE_CUDA
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 #include "tensorflow/core/kernels/cuda_solvers.h"
+#include "tensorflow/core/kernels/eye_functor.h"
 #include "tensorflow/core/kernels/transpose_functor.h"
 #endif
 
diff --git a/tensorflow/core/kernels/qr_op_impl.h b/tensorflow/core/kernels/qr_op_impl.h
index c51d601437..0552c034d2 100644
--- a/tensorflow/core/kernels/qr_op_impl.h
+++ b/tensorflow/core/kernels/qr_op_impl.h
@@ -40,6 +40,7 @@ limitations under the License.
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 #include "tensorflow/core/kernels/cuda_solvers.h"
 #include "tensorflow/core/kernels/cwise_ops.h"
+#include "tensorflow/core/kernels/eye_functor.h"
 #include "tensorflow/core/kernels/matrix_band_part_op.h"
 #include "tensorflow/core/kernels/transpose_functor.h"
 #endif
-- 
GitLab


From 76c921c42587c6e6f5ece90d0682f0912c5ed3bd Mon Sep 17 00:00:00 2001
From: Eugene Brevdo <ebrevdo@google.com>
Date: Thu, 26 Oct 2017 17:05:10 -0700
Subject: [PATCH 847/909] Add a build method to LSTMCell.

To make some legacy behavior of LSTMCell work, add the "partitioner" argument to Layer's add_variable method.

PiperOrigin-RevId: 173612542
---
 .../python/kernel_tests/core_rnn_cell_test.py |   1 -
 .../rnn/python/kernel_tests/core_rnn_test.py  | 150 ++++++------------
 tensorflow/python/layers/base.py              |  18 ++-
 tensorflow/python/ops/rnn_cell_impl.py        | 108 +++++++------
 .../tensorflow.keras.layers.-activation.pbtxt |   2 +-
 ...eras.layers.-activity-regularization.pbtxt |   2 +-
 .../golden/tensorflow.keras.layers.-add.pbtxt |   2 +-
 ...nsorflow.keras.layers.-alpha-dropout.pbtxt |   2 +-
 ...low.keras.layers.-average-pooling1-d.pbtxt |   2 +-
 ...low.keras.layers.-average-pooling2-d.pbtxt |   2 +-
 ...low.keras.layers.-average-pooling3-d.pbtxt |   2 +-
 .../tensorflow.keras.layers.-average.pbtxt    |   2 +-
 ...tensorflow.keras.layers.-avg-pool1-d.pbtxt |   2 +-
 ...tensorflow.keras.layers.-avg-pool2-d.pbtxt |   2 +-
 ...tensorflow.keras.layers.-avg-pool3-d.pbtxt |   2 +-
 ...ow.keras.layers.-batch-normalization.pbtxt |   2 +-
 ...nsorflow.keras.layers.-bidirectional.pbtxt |   2 +-
 ...tensorflow.keras.layers.-concatenate.pbtxt |   2 +-
 ...orflow.keras.layers.-conv-l-s-t-m2-d.pbtxt |   2 +-
 .../tensorflow.keras.layers.-conv1-d.pbtxt    |   2 +-
 ...flow.keras.layers.-conv2-d-transpose.pbtxt |   2 +-
 .../tensorflow.keras.layers.-conv2-d.pbtxt    |   2 +-
 ...flow.keras.layers.-conv3-d-transpose.pbtxt |   2 +-
 .../tensorflow.keras.layers.-conv3-d.pbtxt    |   2 +-
 ...sorflow.keras.layers.-convolution1-d.pbtxt |   2 +-
 ...ras.layers.-convolution2-d-transpose.pbtxt |   2 +-
 ...sorflow.keras.layers.-convolution2-d.pbtxt |   2 +-
 ...ras.layers.-convolution3-d-transpose.pbtxt |   2 +-
 ...sorflow.keras.layers.-convolution3-d.pbtxt |   2 +-
 ...tensorflow.keras.layers.-cropping1-d.pbtxt |   2 +-
 ...tensorflow.keras.layers.-cropping2-d.pbtxt |   2 +-
 ...tensorflow.keras.layers.-cropping3-d.pbtxt |   2 +-
 .../tensorflow.keras.layers.-dense.pbtxt      |   2 +-
 .../golden/tensorflow.keras.layers.-dot.pbtxt |   2 +-
 .../tensorflow.keras.layers.-dropout.pbtxt    |   2 +-
 .../tensorflow.keras.layers.-e-l-u.pbtxt      |   2 +-
 .../tensorflow.keras.layers.-embedding.pbtxt  |   2 +-
 .../tensorflow.keras.layers.-flatten.pbtxt    |   2 +-
 .../tensorflow.keras.layers.-g-r-u.pbtxt      |   2 +-
 ...rflow.keras.layers.-gaussian-dropout.pbtxt |   2 +-
 ...sorflow.keras.layers.-gaussian-noise.pbtxt |   2 +-
 ...as.layers.-global-average-pooling1-d.pbtxt |   2 +-
 ...as.layers.-global-average-pooling2-d.pbtxt |   2 +-
 ...as.layers.-global-average-pooling3-d.pbtxt |   2 +-
 ...low.keras.layers.-global-avg-pool1-d.pbtxt |   2 +-
 ...low.keras.layers.-global-avg-pool2-d.pbtxt |   2 +-
 ...low.keras.layers.-global-avg-pool3-d.pbtxt |   2 +-
 ...low.keras.layers.-global-max-pool1-d.pbtxt |   2 +-
 ...low.keras.layers.-global-max-pool2-d.pbtxt |   2 +-
 ...low.keras.layers.-global-max-pool3-d.pbtxt |   2 +-
 ....keras.layers.-global-max-pooling1-d.pbtxt |   2 +-
 ....keras.layers.-global-max-pooling2-d.pbtxt |   2 +-
 ....keras.layers.-global-max-pooling3-d.pbtxt |   2 +-
 ...tensorflow.keras.layers.-input-layer.pbtxt |   2 +-
 .../tensorflow.keras.layers.-l-s-t-m.pbtxt    |   2 +-
 .../tensorflow.keras.layers.-lambda.pbtxt     |   2 +-
 .../tensorflow.keras.layers.-layer.pbtxt      |   2 +-
 ...ensorflow.keras.layers.-leaky-re-l-u.pbtxt |   2 +-
 ...w.keras.layers.-locally-connected1-d.pbtxt |   2 +-
 ...w.keras.layers.-locally-connected2-d.pbtxt |   2 +-
 .../tensorflow.keras.layers.-masking.pbtxt    |   2 +-
 ...tensorflow.keras.layers.-max-pool1-d.pbtxt |   2 +-
 ...tensorflow.keras.layers.-max-pool2-d.pbtxt |   2 +-
 ...tensorflow.keras.layers.-max-pool3-d.pbtxt |   2 +-
 ...sorflow.keras.layers.-max-pooling1-d.pbtxt |   2 +-
 ...sorflow.keras.layers.-max-pooling2-d.pbtxt |   2 +-
 ...sorflow.keras.layers.-max-pooling3-d.pbtxt |   2 +-
 .../tensorflow.keras.layers.-maximum.pbtxt    |   2 +-
 .../tensorflow.keras.layers.-multiply.pbtxt   |   2 +-
 .../tensorflow.keras.layers.-p-re-l-u.pbtxt   |   2 +-
 .../tensorflow.keras.layers.-permute.pbtxt    |   2 +-
 ...nsorflow.keras.layers.-repeat-vector.pbtxt |   2 +-
 .../tensorflow.keras.layers.-reshape.pbtxt    |   2 +-
 ...flow.keras.layers.-separable-conv2-d.pbtxt |   2 +-
 ...ras.layers.-separable-convolution2-d.pbtxt |   2 +-
 ...ensorflow.keras.layers.-simple-r-n-n.pbtxt |   2 +-
 ...low.keras.layers.-spatial-dropout1-d.pbtxt |   2 +-
 ...low.keras.layers.-spatial-dropout2-d.pbtxt |   2 +-
 ...low.keras.layers.-spatial-dropout3-d.pbtxt |   2 +-
 ...low.keras.layers.-thresholded-re-l-u.pbtxt |   2 +-
 ...rflow.keras.layers.-time-distributed.pbtxt |   2 +-
 ...sorflow.keras.layers.-up-sampling1-d.pbtxt |   2 +-
 ...sorflow.keras.layers.-up-sampling2-d.pbtxt |   2 +-
 ...sorflow.keras.layers.-up-sampling3-d.pbtxt |   2 +-
 .../tensorflow.keras.layers.-wrapper.pbtxt    |   2 +-
 ...orflow.keras.layers.-zero-padding1-d.pbtxt |   2 +-
 ...orflow.keras.layers.-zero-padding2-d.pbtxt |   2 +-
 ...orflow.keras.layers.-zero-padding3-d.pbtxt |   2 +-
 .../tensorflow.keras.models.-model.pbtxt      |   2 +-
 .../tensorflow.keras.models.-sequential.pbtxt |   2 +-
 ...ensorflow.layers.-average-pooling1-d.pbtxt |   2 +-
 ...ensorflow.layers.-average-pooling2-d.pbtxt |   2 +-
 ...ensorflow.layers.-average-pooling3-d.pbtxt |   2 +-
 ...nsorflow.layers.-batch-normalization.pbtxt |   2 +-
 .../golden/tensorflow.layers.-conv1-d.pbtxt   |   2 +-
 ...tensorflow.layers.-conv2-d-transpose.pbtxt |   2 +-
 .../golden/tensorflow.layers.-conv2-d.pbtxt   |   2 +-
 ...tensorflow.layers.-conv3-d-transpose.pbtxt |   2 +-
 .../golden/tensorflow.layers.-conv3-d.pbtxt   |   2 +-
 .../api/golden/tensorflow.layers.-dense.pbtxt |   2 +-
 .../golden/tensorflow.layers.-dropout.pbtxt   |   2 +-
 .../golden/tensorflow.layers.-flatten.pbtxt   |   2 +-
 .../api/golden/tensorflow.layers.-layer.pbtxt |   2 +-
 .../tensorflow.layers.-max-pooling1-d.pbtxt   |   2 +-
 .../tensorflow.layers.-max-pooling2-d.pbtxt   |   2 +-
 .../tensorflow.layers.-max-pooling3-d.pbtxt   |   2 +-
 ...tensorflow.layers.-separable-conv2-d.pbtxt |   2 +-
 ...flow.nn.rnn_cell.-basic-l-s-t-m-cell.pbtxt |   2 +-
 ...orflow.nn.rnn_cell.-basic-r-n-n-cell.pbtxt |   2 +-
 ...nsorflow.nn.rnn_cell.-device-wrapper.pbtxt |   2 +-
 ...sorflow.nn.rnn_cell.-dropout-wrapper.pbtxt |   2 +-
 .../tensorflow.nn.rnn_cell.-g-r-u-cell.pbtxt  |   2 +-
 ...tensorflow.nn.rnn_cell.-l-s-t-m-cell.pbtxt |   7 +-
 ...orflow.nn.rnn_cell.-multi-r-n-n-cell.pbtxt |   2 +-
 .../tensorflow.nn.rnn_cell.-r-n-n-cell.pbtxt  |   2 +-
 ...orflow.nn.rnn_cell.-residual-wrapper.pbtxt |   2 +-
 116 files changed, 237 insertions(+), 269 deletions(-)

diff --git a/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_cell_test.py b/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_cell_test.py
index 8349188f6f..6b6cdfa242 100644
--- a/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_cell_test.py
+++ b/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_cell_test.py
@@ -40,7 +40,6 @@ from tensorflow.python.ops import rnn_cell_impl
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops import variables as variables_lib
 from tensorflow.python.platform import test
-from tensorflow.python.framework import test_util
 
 
 # pylint: enable=protected-access
diff --git a/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_test.py b/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_test.py
index 12def6dcc8..9cea2ec79a 100644
--- a/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_test.py
+++ b/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_test.py
@@ -169,7 +169,7 @@ class RNNTest(test.TestCase):
       self.assertEqual(out.get_shape(), inp.get_shape())
       self.assertEqual(out.dtype, inp.dtype)
 
-    with self.test_session(use_gpu=False) as sess:
+    with self.test_session(use_gpu=True) as sess:
       input_value = np.random.randn(batch_size, input_size)
       values = sess.run(outputs + [state], feed_dict={inputs[0]: input_value})
 
@@ -204,7 +204,7 @@ class RNNTest(test.TestCase):
       self.assertEqual(out.get_shape().as_list(), inp.get_shape().as_list())
       self.assertEqual(out.dtype, inp.dtype)
 
-    with self.test_session(use_gpu=False) as sess:
+    with self.test_session(use_gpu=True) as sess:
       input_value = np.random.randn(batch_size, input_size)
       values = sess.run(outputs + [state], feed_dict={inputs[0]: input_value})
       full_dropout_values = sess.run(dropped_outputs,
@@ -215,7 +215,7 @@ class RNNTest(test.TestCase):
       for d_v in full_dropout_values[:-1]:  # Add 1.0 to dropped_out (all zeros)
         self.assertAllClose(d_v, np.ones_like(input_value))
 
-  def _testDynamicCalculation(self, use_gpu):
+  def testDynamicCalculation(self):
     cell = Plus1RNNCell()
     sequence_length = array_ops.placeholder(dtypes.int64)
     batch_size = 2
@@ -230,7 +230,7 @@ class RNNTest(test.TestCase):
           cell, inputs, sequence_length=sequence_length, dtype=dtypes.float32)
     self.assertEqual(len(dynamic_outputs), len(inputs))
 
-    with self.test_session(use_gpu=use_gpu) as sess:
+    with self.test_session(use_gpu=True) as sess:
       input_value = np.random.randn(batch_size, input_size)
       dynamic_values = sess.run(
           dynamic_outputs,
@@ -261,10 +261,6 @@ class RNNTest(test.TestCase):
                           np.vstack((1.0 * (1 + 1) * np.ones((input_size)),
                                      1.0 * (2 + 1) * np.ones((input_size)))))
 
-  def testDynamicCalculation(self):
-    self._testDynamicCalculation(True)
-    self._testDynamicCalculation(False)
-
   def _testScope(self, factory, prefix="prefix", use_outer_scope=True):
     with self.test_session(use_gpu=True, graph=ops_lib.Graph()):
       if use_outer_scope:
@@ -309,12 +305,12 @@ class LSTMTest(test.TestCase):
     self._seed = 23489
     np.random.seed(self._seed)
 
-  def _testNoProjNoSharding(self, use_gpu):
+  def testNoProjNoSharding(self):
     num_units = 3
     input_size = 5
     batch_size = 2
     max_length = 8
-    with self.test_session(use_gpu=use_gpu, graph=ops_lib.Graph()) as sess:
+    with self.test_session(use_gpu=True, graph=ops_lib.Graph()) as sess:
       initializer = init_ops.random_uniform_initializer(
           -0.01, 0.01, seed=self._seed)
       cell = rnn_cell.LSTMCell(
@@ -332,12 +328,12 @@ class LSTMTest(test.TestCase):
       input_value = np.random.randn(batch_size, input_size)
       sess.run(outputs, feed_dict={inputs[0]: input_value})
 
-  def _testCellClipping(self, use_gpu):
+  def testCellClipping(self):
     num_units = 3
     input_size = 5
     batch_size = 2
     max_length = 8
-    with self.test_session(use_gpu=use_gpu, graph=ops_lib.Graph()) as sess:
+    with self.test_session(use_gpu=True, graph=ops_lib.Graph()) as sess:
       initializer = init_ops.random_uniform_initializer(
           -0.01, 0.01, seed=self._seed)
       cell = rnn_cell.LSTMCell(
@@ -363,12 +359,12 @@ class LSTMTest(test.TestCase):
       # if cell c is clipped to 0, tanh(c) = 0 => m==0
       self.assertAllEqual(value, np.zeros((batch_size, num_units)))
 
-  def _testNoProjNoShardingSimpleStateSaver(self, use_gpu):
+  def testNoProjNoShardingSimpleStateSaver(self):
     num_units = 3
     input_size = 5
     batch_size = 2
     max_length = 8
-    with self.test_session(use_gpu=use_gpu, graph=ops_lib.Graph()) as sess:
+    with self.test_session(use_gpu=True, graph=ops_lib.Graph()) as sess:
       initializer = init_ops.random_uniform_initializer(
           -0.01, 0.01, seed=self._seed)
       state_saver = TestStateSaver(batch_size, 2 * num_units)
@@ -493,13 +489,13 @@ class LSTMTest(test.TestCase):
         self.assertAllEqual(last_states[i],
                             named_saved_states[flat_state_names[i]])
 
-  def _testProjNoSharding(self, use_gpu):
+  def testProjNoSharding(self):
     num_units = 3
     input_size = 5
     batch_size = 2
     num_proj = 4
     max_length = 8
-    with self.test_session(use_gpu=use_gpu, graph=ops_lib.Graph()) as sess:
+    with self.test_session(use_gpu=True, graph=ops_lib.Graph()) as sess:
       initializer = init_ops.random_uniform_initializer(
           -0.01, 0.01, seed=self._seed)
       inputs = max_length * [
@@ -584,7 +580,7 @@ class LSTMTest(test.TestCase):
       state_tuple_v = sess.run(state_tuple, feed_dict={inputs[0]: input_value})
       self.assertAllEqual(state_notuple_v, np.hstack(state_tuple_v))
 
-  def _testProjSharding(self, use_gpu):
+  def testProjSharding(self):
     num_units = 3
     input_size = 5
     batch_size = 2
@@ -592,7 +588,7 @@ class LSTMTest(test.TestCase):
     num_proj_shards = 3
     num_unit_shards = 2
     max_length = 8
-    with self.test_session(use_gpu=use_gpu, graph=ops_lib.Graph()) as sess:
+    with self.test_session(use_gpu=True, graph=ops_lib.Graph()) as sess:
       initializer = init_ops.random_uniform_initializer(
           -0.01, 0.01, seed=self._seed)
 
@@ -618,7 +614,7 @@ class LSTMTest(test.TestCase):
       input_value = np.random.randn(batch_size, input_size)
       sess.run(outputs, feed_dict={inputs[0]: input_value})
 
-  def _testDoubleInput(self, use_gpu):
+  def testDoubleInput(self):
     num_units = 3
     input_size = 5
     batch_size = 2
@@ -626,7 +622,7 @@ class LSTMTest(test.TestCase):
     num_proj_shards = 3
     num_unit_shards = 2
     max_length = 8
-    with self.test_session(use_gpu=use_gpu, graph=ops_lib.Graph()) as sess:
+    with self.test_session(use_gpu=True, graph=ops_lib.Graph()) as sess:
       initializer = init_ops.random_uniform_initializer(-1, 1, seed=self._seed)
       inputs = max_length * [
           array_ops.placeholder(
@@ -655,7 +651,7 @@ class LSTMTest(test.TestCase):
       values = sess.run(outputs, feed_dict={inputs[0]: input_value})
       self.assertEqual(values[0].dtype, input_value.dtype)
 
-  def _testShardNoShardEquivalentOutput(self, use_gpu):
+  def testShardNoShardEquivalentOutput(self):
     num_units = 3
     input_size = 5
     batch_size = 2
@@ -663,7 +659,7 @@ class LSTMTest(test.TestCase):
     num_proj_shards = 3
     num_unit_shards = 2
     max_length = 8
-    with self.test_session(use_gpu=use_gpu, graph=ops_lib.Graph()) as sess:
+    with self.test_session(use_gpu=True, graph=ops_lib.Graph()) as sess:
       inputs = max_length * [
           array_ops.placeholder(
               dtypes.float32, shape=(None, input_size))
@@ -710,7 +706,7 @@ class LSTMTest(test.TestCase):
       for (s_noshard, s_shard) in zip(state_values_noshard, state_values_shard):
         self.assertAllClose(s_noshard, s_shard, atol=1e-3)
 
-  def _testDoubleInputWithDropoutAndDynamicCalculation(self, use_gpu):
+  def testDoubleInputWithDropoutAndDynamicCalculation(self):
     """Smoke test for using LSTM with doubles, dropout, dynamic calculation."""
 
     num_units = 3
@@ -720,7 +716,7 @@ class LSTMTest(test.TestCase):
     num_proj_shards = 3
     num_unit_shards = 2
     max_length = 8
-    with self.test_session(use_gpu=use_gpu, graph=ops_lib.Graph()) as sess:
+    with self.test_session(use_gpu=True, graph=ops_lib.Graph()) as sess:
       sequence_length = array_ops.placeholder(dtypes.int64)
       initializer = init_ops.random_uniform_initializer(
           -0.01, 0.01, seed=self._seed)
@@ -845,38 +841,6 @@ class LSTMTest(test.TestCase):
       for out0, out1 in zip(outputs0_values, outputs1_values):
         self.assertAllEqual(out0, out1)
 
-  def testNoProjNoShardingSimpleStateSaver(self):
-    self._testNoProjNoShardingSimpleStateSaver(use_gpu=False)
-    self._testNoProjNoShardingSimpleStateSaver(use_gpu=True)
-
-  def testNoProjNoSharding(self):
-    self._testNoProjNoSharding(use_gpu=False)
-    self._testNoProjNoSharding(use_gpu=True)
-
-  def testCellClipping(self):
-    self._testCellClipping(use_gpu=False)
-    self._testCellClipping(use_gpu=True)
-
-  def testProjNoSharding(self):
-    self._testProjNoSharding(use_gpu=False)
-    self._testProjNoSharding(use_gpu=True)
-
-  def testProjSharding(self):
-    self._testProjSharding(use_gpu=False)
-    self._testProjSharding(use_gpu=True)
-
-  def testShardNoShardEquivalentOutput(self):
-    self._testShardNoShardEquivalentOutput(use_gpu=False)
-    self._testShardNoShardEquivalentOutput(use_gpu=True)
-
-  def testDoubleInput(self):
-    self._testDoubleInput(use_gpu=False)
-    self._testDoubleInput(use_gpu=True)
-
-  def testDoubleInputWithDropoutAndDynamicCalculation(self):
-    self._testDoubleInputWithDropoutAndDynamicCalculation(use_gpu=False)
-    self._testDoubleInputWithDropoutAndDynamicCalculation(use_gpu=True)
-
   def testDynamicRNNAllowsUnknownTimeDimension(self):
     inputs = array_ops.placeholder(dtypes.float32, shape=[1, None, 20])
     cell = rnn_cell.GRUCell(30)
@@ -1052,7 +1016,7 @@ class LSTMTest(test.TestCase):
         state_dynamic = [s.numpy() for s in nest.flatten(state_dynamic)]
       self.assertAllEqual(np.hstack(state_static), np.hstack(state_dynamic))
 
-  def _testDynamicEquivalentToStaticRNN(self, use_gpu, use_sequence_length):
+  def _testDynamicEquivalentToStaticRNN(self, use_sequence_length):
     time_steps = 8
     num_units = 3
     num_proj = 4
@@ -1083,7 +1047,7 @@ class LSTMTest(test.TestCase):
           state_is_tuple=False)
 
     ########### Step 1: Run static graph and generate readouts
-    with self.test_session(use_gpu=use_gpu, graph=ops_lib.Graph()) as sess:
+    with self.test_session(use_gpu=True, graph=ops_lib.Graph()) as sess:
       if in_graph_mode:
         concat_inputs = array_ops.placeholder(
             dtypes.float32, shape=(time_steps, batch_size, input_size))
@@ -1143,7 +1107,7 @@ class LSTMTest(test.TestCase):
             static_individual_variable_gradients, feed_dict=feeds)
 
     ########## Step 2: Run dynamic graph and generate readouts
-    with self.test_session(use_gpu=use_gpu, graph=ops_lib.Graph()) as sess:
+    with self.test_session(use_gpu=True, graph=ops_lib.Graph()) as sess:
       if in_graph_mode:
         concat_inputs = array_ops.placeholder(
             dtypes.float32, shape=(time_steps, batch_size, input_size))
@@ -1249,14 +1213,8 @@ class LSTMTest(test.TestCase):
 
   @test_util.run_in_graph_and_eager_modes()
   def testDynamicEquivalentToStaticRNN(self):
-    self._testDynamicEquivalentToStaticRNN(
-        use_gpu=False, use_sequence_length=False)
-    self._testDynamicEquivalentToStaticRNN(
-        use_gpu=True, use_sequence_length=False)
-    self._testDynamicEquivalentToStaticRNN(
-        use_gpu=False, use_sequence_length=True)
-    self._testDynamicEquivalentToStaticRNN(
-        use_gpu=True, use_sequence_length=True)
+    self._testDynamicEquivalentToStaticRNN(use_sequence_length=False)
+    self._testDynamicEquivalentToStaticRNN(use_sequence_length=False)
 
 
 class BidirectionalRNNTest(test.TestCase):
@@ -1266,7 +1224,6 @@ class BidirectionalRNNTest(test.TestCase):
     np.random.seed(self._seed)
 
   def _createBidirectionalRNN(self,
-                              use_gpu,
                               use_shape,
                               use_sequence_length,
                               scope=None):
@@ -1305,10 +1262,10 @@ class BidirectionalRNNTest(test.TestCase):
 
     return input_value, inputs, outputs, state_fw, state_bw, sequence_length
 
-  def _testBidirectionalRNN(self, use_gpu, use_shape):
-    with self.test_session(use_gpu=use_gpu, graph=ops_lib.Graph()) as sess:
+  def _testBidirectionalRNN(self, use_shape):
+    with self.test_session(use_gpu=True, graph=ops_lib.Graph()) as sess:
       input_value, inputs, outputs, state_fw, state_bw, sequence_length = (
-          self._createBidirectionalRNN(use_gpu, use_shape, True))
+          self._createBidirectionalRNN(use_shape, True))
       variables_lib.global_variables_initializer().run()
       # Run with pre-specified sequence length of 2, 3
       out, s_fw, s_bw = sess.run(
@@ -1350,10 +1307,10 @@ class BidirectionalRNNTest(test.TestCase):
       # exactly the same
       self.assertAllClose(s_fw, s_bw)
 
-  def _testBidirectionalRNNWithoutSequenceLength(self, use_gpu, use_shape):
-    with self.test_session(use_gpu=use_gpu, graph=ops_lib.Graph()) as sess:
+  def _testBidirectionalRNNWithoutSequenceLength(self, use_shape):
+    with self.test_session(use_gpu=True, graph=ops_lib.Graph()) as sess:
       input_value, inputs, outputs, state_fw, state_bw, _ = (
-          self._createBidirectionalRNN(use_gpu, use_shape, False))
+          self._createBidirectionalRNN(use_shape, False))
       variables_lib.global_variables_initializer().run()
       out, s_fw, s_bw = sess.run([outputs, state_fw, state_bw],
                                  feed_dict={inputs[0]: input_value})
@@ -1380,23 +1337,14 @@ class BidirectionalRNNTest(test.TestCase):
       self.assertAllClose(s_fw, s_bw)
 
   def testBidirectionalRNN(self):
-    self._testBidirectionalRNN(use_gpu=False, use_shape=False)
-    self._testBidirectionalRNN(use_gpu=True, use_shape=False)
-    self._testBidirectionalRNN(use_gpu=False, use_shape=True)
-    self._testBidirectionalRNN(use_gpu=True, use_shape=True)
+    self._testBidirectionalRNN(use_shape=False)
+    self._testBidirectionalRNN(use_shape=True)
 
   def testBidirectionalRNNWithoutSequenceLength(self):
-    self._testBidirectionalRNNWithoutSequenceLength(
-        use_gpu=False, use_shape=False)
-    self._testBidirectionalRNNWithoutSequenceLength(
-        use_gpu=True, use_shape=False)
-    self._testBidirectionalRNNWithoutSequenceLength(
-        use_gpu=False, use_shape=True)
-    self._testBidirectionalRNNWithoutSequenceLength(
-        use_gpu=True, use_shape=True)
+    self._testBidirectionalRNNWithoutSequenceLength(use_shape=False)
+    self._testBidirectionalRNNWithoutSequenceLength(use_shape=True)
 
   def _createBidirectionalDynamicRNN(self,
-                                     use_gpu,
                                      use_shape,
                                      use_state_tuple,
                                      use_time_major,
@@ -1444,11 +1392,11 @@ class BidirectionalRNNTest(test.TestCase):
 
     return input_value, inputs, outputs, state_fw, state_bw, sequence_length
 
-  def _testBidirectionalDynamicRNN(self, use_gpu, use_shape, use_state_tuple,
+  def _testBidirectionalDynamicRNN(self, use_shape, use_state_tuple,
                                    use_time_major, use_sequence_length):
-    with self.test_session(use_gpu=use_gpu, graph=ops_lib.Graph()) as sess:
+    with self.test_session(use_gpu=True, graph=ops_lib.Graph()) as sess:
       input_value, inputs, outputs, state_fw, state_bw, sequence_length = (
-          self._createBidirectionalDynamicRNN(use_gpu, use_shape,
+          self._createBidirectionalDynamicRNN(use_shape,
                                               use_state_tuple, use_time_major,
                                               use_sequence_length))
       variables_lib.global_variables_initializer().run()
@@ -1513,14 +1461,13 @@ class BidirectionalRNNTest(test.TestCase):
   def testBidirectionalDynamicRNN(self):
     # Generate 2^5 option values
     # from [True, True, True, True, True] to [False, False, False, False, False]
-    options = itertools.product([True, False], repeat=5)
+    options = itertools.product([True, False], repeat=4)
     for option in options:
       self._testBidirectionalDynamicRNN(
-          use_gpu=option[0],
-          use_shape=option[1],
-          use_state_tuple=option[2],
-          use_time_major=option[3],
-          use_sequence_length=option[4])
+          use_shape=option[0],
+          use_state_tuple=option[1],
+          use_time_major=option[2],
+          use_sequence_length=option[3])
 
   def _testScope(self, factory, prefix="prefix", use_outer_scope=True):
     # REMARKS: factory(scope) is a function accepting a scope
@@ -1549,7 +1496,7 @@ class BidirectionalRNNTest(test.TestCase):
 
     def factory(scope):
       return self._createBidirectionalRNN(
-          use_gpu=True, use_shape=True, use_sequence_length=True, scope=scope)
+          use_shape=True, use_sequence_length=True, scope=scope)
 
     self._testScope(factory, use_outer_scope=True)
     self._testScope(factory, use_outer_scope=False)
@@ -1561,7 +1508,6 @@ class BidirectionalRNNTest(test.TestCase):
 
       def factory(scope):
         return self._createBidirectionalDynamicRNN(
-            use_gpu=True,
             use_shape=True,
             use_state_tuple=True,
             use_sequence_length=True,
@@ -1839,7 +1785,7 @@ class GRUTest(test.TestCase):
     self._seed = 23489
     np.random.seed(self._seed)
 
-  def _testDynamic(self, use_gpu):
+  def testDynamic(self):
     time_steps = 8
     num_units = 3
     input_size = 5
@@ -1849,7 +1795,7 @@ class GRUTest(test.TestCase):
 
     sequence_length = np.random.randint(0, time_steps, size=batch_size)
 
-    with self.test_session(use_gpu=use_gpu, graph=ops_lib.Graph()) as sess:
+    with self.test_session(use_gpu=True, graph=ops_lib.Graph()) as sess:
       concat_inputs = array_ops.placeholder(
           dtypes.float32, shape=(time_steps, batch_size, input_size))
 
@@ -1870,10 +1816,6 @@ class GRUTest(test.TestCase):
 
       sess.run([outputs_dynamic, state_dynamic], feed_dict=feeds)
 
-  def testDynamic(self):
-    self._testDynamic(use_gpu=False)
-    self._testDynamic(use_gpu=True)
-
   def _testScope(self, factory, prefix="prefix", use_outer_scope=True):
     with self.test_session(use_gpu=True, graph=ops_lib.Graph()):
       if use_outer_scope:
diff --git a/tensorflow/python/layers/base.py b/tensorflow/python/layers/base.py
index 91e18b2ba5..134d4fc8e2 100644
--- a/tensorflow/python/layers/base.py
+++ b/tensorflow/python/layers/base.py
@@ -413,7 +413,8 @@ class Layer(object):
 
   def add_variable(self, name, shape, dtype=None,
                    initializer=None, regularizer=None,
-                   trainable=True, constraint=None):
+                   trainable=True, constraint=None,
+                   partitioner=None):
     """Adds a new variable to the layer, or gets an existing one; returns it.
 
     Arguments:
@@ -426,9 +427,19 @@ class Layer(object):
         "trainable_variables" (e.g. variables, biases)
         or "non_trainable_variables" (e.g. BatchNorm mean, stddev).
       constraint: constraint instance (callable).
+      partitioner: (optional) partitioner instance (callable).  If
+        provided, when the requested variable is created it will be split
+        into multiple partitions according to `partitioner`.  In this case,
+        an instance of `PartitionedVariable` is returned.  Available
+        partitioners include `tf.fixed_size_partitioner` and
+        `tf.variable_axis_size_partitioner`.  For more details, see the
+        documentation of `tf.get_variable` and the  "Variable Partitioners
+        and Sharding" section of the API guide.
 
     Returns:
-      The created variable.
+      The created variable.  Usually either a `Variable` or `ResourceVariable`
+      instance.  If `partitioner` is not `None`, a `PartitionedVariable`
+      instance is returned.
 
     Raises:
       RuntimeError: If called in Eager mode with regularizers.
@@ -455,7 +466,8 @@ class Layer(object):
                                    initializer=initializer,
                                    dtype=dtypes.as_dtype(dtype),
                                    constraint=constraint,
-                                   trainable=trainable and self.trainable)
+                                   trainable=trainable and self.trainable,
+                                   partitioner=partitioner)
         if variable in existing_variables:
           return variable
         if regularizer:
diff --git a/tensorflow/python/ops/rnn_cell_impl.py b/tensorflow/python/ops/rnn_cell_impl.py
index 1825e98259..b90c757095 100644
--- a/tensorflow/python/ops/rnn_cell_impl.py
+++ b/tensorflow/python/ops/rnn_cell_impl.py
@@ -455,6 +455,10 @@ class BasicLSTMCell(_LayerRNNCell):
     if not state_is_tuple:
       logging.warn("%s: Using a concatenated state is slower and will soon be "
                    "deprecated.  Use state_is_tuple=True.", self)
+
+    # Inputs must be 2-dimensional.
+    self.input_spec = base_layer.InputSpec(ndim=2)
+
     self._num_units = num_units
     self._forget_bias = forget_bias
     self._state_is_tuple = state_is_tuple
@@ -471,9 +475,6 @@ class BasicLSTMCell(_LayerRNNCell):
     return self._num_units
 
   def build(self, inputs_shape):
-    if inputs_shape.ndims != 2:
-      raise ValueError("Expected inputs.shape to be rank 2, saw shape: %s"
-                       % inputs_shape)
     if inputs_shape[1].value is None:
       raise ValueError("Expected inputs.shape[-1] to be known, saw shape: %s"
                        % inputs_shape)
@@ -537,7 +538,7 @@ class BasicLSTMCell(_LayerRNNCell):
     return new_h, new_state
 
 
-class LSTMCell(RNNCell):
+class LSTMCell(_LayerRNNCell):
   """Long short-term memory unit (LSTM) recurrent network cell.
 
   The default non-peephole implementation is based on:
@@ -564,7 +565,7 @@ class LSTMCell(RNNCell):
                initializer=None, num_proj=None, proj_clip=None,
                num_unit_shards=None, num_proj_shards=None,
                forget_bias=1.0, state_is_tuple=True,
-               activation=None, reuse=None):
+               activation=None, reuse=None, name=None):
     """Initialize the parameters for an LSTM cell.
 
     Args:
@@ -594,11 +595,14 @@ class LSTMCell(RNNCell):
       reuse: (optional) Python boolean describing whether to reuse variables
         in an existing scope.  If not `True`, and the existing scope already has
         the given variables, an error is raised.
+      name: String, the name of the layer. Layers with the same name will
+        share weights, but to avoid mistakes we require reuse=True in such
+        cases.
 
-      When restoring from CudnnLSTM-trained checkpoints, must use
-      CudnnCompatibleLSTMCell instead.
+      When restoring from CudnnLSTM-trained checkpoints, use
+      `CudnnCompatibleLSTMCell` instead.
     """
-    super(LSTMCell, self).__init__(_reuse=reuse)
+    super(LSTMCell, self).__init__(_reuse=reuse, name=name)
     if not state_is_tuple:
       logging.warn("%s: Using a concatenated state is slower and will soon be "
                    "deprecated.  Use state_is_tuple=True.", self)
@@ -608,6 +612,9 @@ class LSTMCell(RNNCell):
           "deprecated and will be removed in Jan 2017.  "
           "Use a variable scope with a partitioner instead.", self)
 
+    # Inputs must be 2-dimensional.
+    self.input_spec = base_layer.InputSpec(ndim=2)
+
     self._num_units = num_units
     self._use_peepholes = use_peepholes
     self._cell_clip = cell_clip
@@ -630,12 +637,6 @@ class LSTMCell(RNNCell):
           LSTMStateTuple(num_units, num_units)
           if state_is_tuple else 2 * num_units)
       self._output_size = num_units
-    self._linear1 = None
-    self._linear2 = None
-    if self._use_peepholes:
-      self._w_f_diag = None
-      self._w_i_diag = None
-      self._w_o_diag = None
 
   @property
   def state_size(self):
@@ -645,6 +646,47 @@ class LSTMCell(RNNCell):
   def output_size(self):
     return self._output_size
 
+  def build(self, inputs_shape):
+    if inputs_shape[1].value is None:
+      raise ValueError("Expected inputs.shape[-1] to be known, saw shape: %s"
+                       % inputs_shape)
+
+    input_depth = inputs_shape[1].value
+    h_depth = self._num_units if self._num_proj is None else self._num_proj
+    maybe_partitioner = (
+        partitioned_variables.fixed_size_partitioner(self._num_unit_shards)
+        if self._num_unit_shards is not None
+        else None)
+    self._kernel = self.add_variable(
+        _WEIGHTS_VARIABLE_NAME,
+        shape=[input_depth + h_depth, 4 * self._num_units],
+        initializer=self._initializer,
+        partitioner=maybe_partitioner)
+    self._bias = self.add_variable(
+        _BIAS_VARIABLE_NAME,
+        shape=[4 * self._num_units],
+        initializer=init_ops.constant_initializer(0.0, dtype=self.dtype))
+    if self._use_peepholes:
+      self._w_f_diag = self.add_variable("w_f_diag", shape=[self._num_units],
+                                         initializer=self._initializer)
+      self._w_i_diag = self.add_variable("w_i_diag", shape=[self._num_units],
+                                         initializer=self._initializer)
+      self._w_o_diag = self.add_variable("w_o_diag", shape=[self._num_units],
+                                         initializer=self._initializer)
+
+    if self._num_proj is not None:
+      maybe_proj_partitioner = (
+          partitioned_variables.fixed_size_partitioner(self._num_proj_shards)
+          if self._num_proj_shards is not None
+          else None)
+      self._proj_kernel = self.add_variable(
+          "projection/%s" % _WEIGHTS_VARIABLE_NAME,
+          shape=[self._num_units, self._num_proj],
+          initializer=self._initializer,
+          partitioner=maybe_proj_partitioner)
+
+    self._built = True
+
   def call(self, inputs, state):
     """Run one step of LSTM.
 
@@ -679,37 +721,18 @@ class LSTMCell(RNNCell):
       c_prev = array_ops.slice(state, [0, 0], [-1, self._num_units])
       m_prev = array_ops.slice(state, [0, self._num_units], [-1, num_proj])
 
-    dtype = inputs.dtype
     input_size = inputs.get_shape().with_rank(2)[1]
     if input_size.value is None:
       raise ValueError("Could not infer input size from inputs.get_shape()[-1]")
-    if self._linear1 is None:
-      scope = vs.get_variable_scope()
-      with vs.variable_scope(
-          scope, initializer=self._initializer) as unit_scope:
-        if self._num_unit_shards is not None:
-          unit_scope.set_partitioner(
-              partitioned_variables.fixed_size_partitioner(
-                  self._num_unit_shards))
-        self._linear1 = _Linear([inputs, m_prev], 4 * self._num_units, True)
 
     # i = input_gate, j = new_input, f = forget_gate, o = output_gate
-    lstm_matrix = self._linear1([inputs, m_prev])
+    lstm_matrix = math_ops.matmul(
+        array_ops.concat([inputs, m_prev], 1), self._kernel)
+    lstm_matrix = nn_ops.bias_add(lstm_matrix, self._bias)
+
     i, j, f, o = array_ops.split(
         value=lstm_matrix, num_or_size_splits=4, axis=1)
     # Diagonal connections
-    if self._use_peepholes and self._w_f_diag is None:
-      scope = vs.get_variable_scope()
-      with vs.variable_scope(
-          scope, initializer=self._initializer) as unit_scope:
-        with vs.variable_scope(unit_scope):
-          self._w_f_diag = vs.get_variable(
-              "w_f_diag", shape=[self._num_units], dtype=dtype)
-          self._w_i_diag = vs.get_variable(
-              "w_i_diag", shape=[self._num_units], dtype=dtype)
-          self._w_o_diag = vs.get_variable(
-              "w_o_diag", shape=[self._num_units], dtype=dtype)
-
     if self._use_peepholes:
       c = (sigmoid(f + self._forget_bias + self._w_f_diag * c_prev) * c_prev +
            sigmoid(i + self._w_i_diag * c_prev) * self._activation(j))
@@ -727,16 +750,7 @@ class LSTMCell(RNNCell):
       m = sigmoid(o) * self._activation(c)
 
     if self._num_proj is not None:
-      if self._linear2 is None:
-        scope = vs.get_variable_scope()
-        with vs.variable_scope(scope, initializer=self._initializer):
-          with vs.variable_scope("projection") as proj_scope:
-            if self._num_proj_shards is not None:
-              proj_scope.set_partitioner(
-                  partitioned_variables.fixed_size_partitioner(
-                      self._num_proj_shards))
-            self._linear2 = _Linear(m, self._num_proj, False)
-      m = self._linear2(m)
+      m = math_ops.matmul(m, self._proj_kernel)
 
       if self._proj_clip is not None:
         # pylint: disable=invalid-unary-operand-type
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-activation.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-activation.pbtxt
index c3d8893317..38e6128644 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-activation.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-activation.pbtxt
@@ -102,7 +102,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-activity-regularization.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-activity-regularization.pbtxt
index ea59596431..0fa6064661 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-activity-regularization.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-activity-regularization.pbtxt
@@ -102,7 +102,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-add.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-add.pbtxt
index 7e9b6bd70a..75d56bf445 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-add.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-add.pbtxt
@@ -103,7 +103,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-alpha-dropout.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-alpha-dropout.pbtxt
index 804fb45784..6e52b6238d 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-alpha-dropout.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-alpha-dropout.pbtxt
@@ -102,7 +102,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-average-pooling1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-average-pooling1-d.pbtxt
index 6577856383..0e16774e86 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-average-pooling1-d.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-average-pooling1-d.pbtxt
@@ -104,7 +104,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-average-pooling2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-average-pooling2-d.pbtxt
index fc4452948a..98112762cf 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-average-pooling2-d.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-average-pooling2-d.pbtxt
@@ -104,7 +104,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-average-pooling3-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-average-pooling3-d.pbtxt
index ce19cea7ca..2e093c0359 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-average-pooling3-d.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-average-pooling3-d.pbtxt
@@ -104,7 +104,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-average.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-average.pbtxt
index 2ea54c2e31..bada65e2f9 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-average.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-average.pbtxt
@@ -103,7 +103,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-avg-pool1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-avg-pool1-d.pbtxt
index 6fa1e153e0..120807c4b5 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-avg-pool1-d.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-avg-pool1-d.pbtxt
@@ -104,7 +104,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-avg-pool2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-avg-pool2-d.pbtxt
index c6ff50bffc..834365f0f7 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-avg-pool2-d.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-avg-pool2-d.pbtxt
@@ -104,7 +104,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-avg-pool3-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-avg-pool3-d.pbtxt
index 6d90a59d1e..462a52ec1e 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-avg-pool3-d.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-avg-pool3-d.pbtxt
@@ -104,7 +104,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-batch-normalization.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-batch-normalization.pbtxt
index 278e5b583d..b802b363d0 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-batch-normalization.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-batch-normalization.pbtxt
@@ -103,7 +103,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-bidirectional.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-bidirectional.pbtxt
index c9991db5c9..5279b2ab17 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-bidirectional.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-bidirectional.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-concatenate.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-concatenate.pbtxt
index ec3c43945f..b800eb9796 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-concatenate.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-concatenate.pbtxt
@@ -103,7 +103,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv-l-s-t-m2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv-l-s-t-m2-d.pbtxt
index 2d6560828e..a0906e62cf 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv-l-s-t-m2-d.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv-l-s-t-m2-d.pbtxt
@@ -104,7 +104,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv1-d.pbtxt
index f6f77ff805..47c63c1157 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv1-d.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv1-d.pbtxt
@@ -104,7 +104,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv2-d-transpose.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv2-d-transpose.pbtxt
index 854a06bf56..e90b90e801 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv2-d-transpose.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv2-d-transpose.pbtxt
@@ -105,7 +105,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv2-d.pbtxt
index 5e71a9d355..aa571b722d 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv2-d.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv2-d.pbtxt
@@ -104,7 +104,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv3-d-transpose.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv3-d-transpose.pbtxt
index e7c98913fb..911c73f846 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv3-d-transpose.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv3-d-transpose.pbtxt
@@ -104,7 +104,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv3-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv3-d.pbtxt
index 3c4d078d1e..bb111b327c 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv3-d.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv3-d.pbtxt
@@ -104,7 +104,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution1-d.pbtxt
index 8043eb0610..5a5ec635cc 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution1-d.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution1-d.pbtxt
@@ -104,7 +104,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution2-d-transpose.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution2-d-transpose.pbtxt
index a9a90891a4..190b670fa2 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution2-d-transpose.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution2-d-transpose.pbtxt
@@ -105,7 +105,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution2-d.pbtxt
index dae5a66190..a26ec82f2b 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution2-d.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution2-d.pbtxt
@@ -104,7 +104,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution3-d-transpose.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution3-d-transpose.pbtxt
index 37aa80eb70..19b5bdf36b 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution3-d-transpose.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution3-d-transpose.pbtxt
@@ -104,7 +104,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution3-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution3-d.pbtxt
index fa28ce17ec..773ef01feb 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution3-d.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution3-d.pbtxt
@@ -104,7 +104,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-cropping1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-cropping1-d.pbtxt
index 8e2b530d08..3a67ac00ab 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-cropping1-d.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-cropping1-d.pbtxt
@@ -102,7 +102,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-cropping2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-cropping2-d.pbtxt
index 70b1c50a0a..de5a695b69 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-cropping2-d.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-cropping2-d.pbtxt
@@ -102,7 +102,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-cropping3-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-cropping3-d.pbtxt
index 1b2b4e934d..bf251b4df5 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-cropping3-d.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-cropping3-d.pbtxt
@@ -102,7 +102,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-dense.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-dense.pbtxt
index fb0fcd2614..92a74cec68 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-dense.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-dense.pbtxt
@@ -103,7 +103,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-dot.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-dot.pbtxt
index af8ad3abaa..cdd62eee0d 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-dot.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-dot.pbtxt
@@ -103,7 +103,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-dropout.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-dropout.pbtxt
index e774a4d412..7935143b2c 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-dropout.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-dropout.pbtxt
@@ -103,7 +103,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-e-l-u.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-e-l-u.pbtxt
index 46eb767208..497eb00499 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-e-l-u.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-e-l-u.pbtxt
@@ -102,7 +102,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-embedding.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-embedding.pbtxt
index 5e74cb6970..35616cbebb 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-embedding.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-embedding.pbtxt
@@ -102,7 +102,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-flatten.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-flatten.pbtxt
index a4c8759a2c..427c6fde90 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-flatten.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-flatten.pbtxt
@@ -103,7 +103,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-g-r-u.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-g-r-u.pbtxt
index 9738dd004a..9237399254 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-g-r-u.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-g-r-u.pbtxt
@@ -103,7 +103,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-gaussian-dropout.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-gaussian-dropout.pbtxt
index ce033eaa00..1428691afe 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-gaussian-dropout.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-gaussian-dropout.pbtxt
@@ -102,7 +102,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-gaussian-noise.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-gaussian-noise.pbtxt
index 4cd6d714a0..655734cc43 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-gaussian-noise.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-gaussian-noise.pbtxt
@@ -102,7 +102,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-average-pooling1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-average-pooling1-d.pbtxt
index 2bd80f97ae..d97f06ea13 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-average-pooling1-d.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-average-pooling1-d.pbtxt
@@ -103,7 +103,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-average-pooling2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-average-pooling2-d.pbtxt
index a9d00fd7c1..52886b2106 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-average-pooling2-d.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-average-pooling2-d.pbtxt
@@ -103,7 +103,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-average-pooling3-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-average-pooling3-d.pbtxt
index a2b00778fe..ccb6459357 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-average-pooling3-d.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-average-pooling3-d.pbtxt
@@ -103,7 +103,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-avg-pool1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-avg-pool1-d.pbtxt
index 01a9839ccc..1f25eb1cc6 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-avg-pool1-d.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-avg-pool1-d.pbtxt
@@ -103,7 +103,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-avg-pool2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-avg-pool2-d.pbtxt
index b041dfc71e..a37d6dda28 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-avg-pool2-d.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-avg-pool2-d.pbtxt
@@ -103,7 +103,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-avg-pool3-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-avg-pool3-d.pbtxt
index 6ba06a4e7e..9f276fd547 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-avg-pool3-d.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-avg-pool3-d.pbtxt
@@ -103,7 +103,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pool1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pool1-d.pbtxt
index fb62a3e035..eaa9b477d8 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pool1-d.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pool1-d.pbtxt
@@ -103,7 +103,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pool2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pool2-d.pbtxt
index 3d1c66441c..f4d37a5f63 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pool2-d.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pool2-d.pbtxt
@@ -103,7 +103,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pool3-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pool3-d.pbtxt
index d55a82e0a3..afddd2d4cb 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pool3-d.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pool3-d.pbtxt
@@ -103,7 +103,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pooling1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pooling1-d.pbtxt
index 70177c8623..12cd49c955 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pooling1-d.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pooling1-d.pbtxt
@@ -103,7 +103,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pooling2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pooling2-d.pbtxt
index da231a4fce..146241c172 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pooling2-d.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pooling2-d.pbtxt
@@ -103,7 +103,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pooling3-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pooling3-d.pbtxt
index aa3eb1c704..00475301aa 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pooling3-d.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pooling3-d.pbtxt
@@ -103,7 +103,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-input-layer.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-input-layer.pbtxt
index 40f0f7c800..b2df5fba8f 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-input-layer.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-input-layer.pbtxt
@@ -103,7 +103,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-l-s-t-m.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-l-s-t-m.pbtxt
index 1a9ec4a506..20935e2f99 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-l-s-t-m.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-l-s-t-m.pbtxt
@@ -103,7 +103,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-lambda.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-lambda.pbtxt
index 69086963b6..59508c2f11 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-lambda.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-lambda.pbtxt
@@ -102,7 +102,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-layer.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-layer.pbtxt
index d350a52171..ca904a2b8c 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-layer.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-layer.pbtxt
@@ -101,7 +101,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-leaky-re-l-u.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-leaky-re-l-u.pbtxt
index 05952c1d96..f52fd02515 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-leaky-re-l-u.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-leaky-re-l-u.pbtxt
@@ -102,7 +102,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-locally-connected1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-locally-connected1-d.pbtxt
index c49b8de5fb..b5c32d1cdf 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-locally-connected1-d.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-locally-connected1-d.pbtxt
@@ -102,7 +102,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-locally-connected2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-locally-connected2-d.pbtxt
index e24e3697b2..0ac2b83a99 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-locally-connected2-d.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-locally-connected2-d.pbtxt
@@ -102,7 +102,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-masking.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-masking.pbtxt
index 246340a1ce..de2a28d985 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-masking.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-masking.pbtxt
@@ -102,7 +102,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pool1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pool1-d.pbtxt
index eb631b1d38..130d932fd6 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pool1-d.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pool1-d.pbtxt
@@ -104,7 +104,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pool2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pool2-d.pbtxt
index cfe6af339e..82a6f6d539 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pool2-d.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pool2-d.pbtxt
@@ -104,7 +104,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pool3-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pool3-d.pbtxt
index 4bb5a23927..ca2fd4e502 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pool3-d.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pool3-d.pbtxt
@@ -104,7 +104,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pooling1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pooling1-d.pbtxt
index 6c9b9a92eb..885e30f879 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pooling1-d.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pooling1-d.pbtxt
@@ -104,7 +104,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pooling2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pooling2-d.pbtxt
index cdc4c43ad6..102879d2f5 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pooling2-d.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pooling2-d.pbtxt
@@ -104,7 +104,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pooling3-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pooling3-d.pbtxt
index 4959dc58d1..4240616146 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pooling3-d.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pooling3-d.pbtxt
@@ -104,7 +104,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-maximum.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-maximum.pbtxt
index 7ff5ee02e1..4b32c2e99f 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-maximum.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-maximum.pbtxt
@@ -103,7 +103,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-multiply.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-multiply.pbtxt
index 860ebd509b..0c964235ae 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-multiply.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-multiply.pbtxt
@@ -103,7 +103,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-p-re-l-u.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-p-re-l-u.pbtxt
index e32800bd25..797a073b8a 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-p-re-l-u.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-p-re-l-u.pbtxt
@@ -102,7 +102,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-permute.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-permute.pbtxt
index 8b453f7a1b..7dc1fa6964 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-permute.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-permute.pbtxt
@@ -102,7 +102,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-repeat-vector.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-repeat-vector.pbtxt
index 9b53609e4d..dedb48151a 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-repeat-vector.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-repeat-vector.pbtxt
@@ -102,7 +102,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-reshape.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-reshape.pbtxt
index f7a774a38f..bb30c0a945 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-reshape.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-reshape.pbtxt
@@ -102,7 +102,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-separable-conv2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-separable-conv2-d.pbtxt
index 4f1d2db4cc..7867e3c1fd 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-separable-conv2-d.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-separable-conv2-d.pbtxt
@@ -105,7 +105,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-separable-convolution2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-separable-convolution2-d.pbtxt
index 066519cba8..0fb6e84f8d 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-separable-convolution2-d.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-separable-convolution2-d.pbtxt
@@ -105,7 +105,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-simple-r-n-n.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-simple-r-n-n.pbtxt
index 6a08eb785b..f4148fcc23 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-simple-r-n-n.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-simple-r-n-n.pbtxt
@@ -103,7 +103,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-spatial-dropout1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-spatial-dropout1-d.pbtxt
index b85003d52e..9773c4acc7 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-spatial-dropout1-d.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-spatial-dropout1-d.pbtxt
@@ -104,7 +104,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-spatial-dropout2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-spatial-dropout2-d.pbtxt
index 83d4258a66..d4de587a48 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-spatial-dropout2-d.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-spatial-dropout2-d.pbtxt
@@ -104,7 +104,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-spatial-dropout3-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-spatial-dropout3-d.pbtxt
index a49060b860..af210fab8d 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-spatial-dropout3-d.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-spatial-dropout3-d.pbtxt
@@ -104,7 +104,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-thresholded-re-l-u.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-thresholded-re-l-u.pbtxt
index 01b91b9bbc..8cfb33a148 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-thresholded-re-l-u.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-thresholded-re-l-u.pbtxt
@@ -102,7 +102,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-time-distributed.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-time-distributed.pbtxt
index 4713bd16e1..34c9efb3ca 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-time-distributed.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-time-distributed.pbtxt
@@ -107,7 +107,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-up-sampling1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-up-sampling1-d.pbtxt
index 393980ecde..bb42cdcb65 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-up-sampling1-d.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-up-sampling1-d.pbtxt
@@ -102,7 +102,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-up-sampling2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-up-sampling2-d.pbtxt
index 7ddb282f06..6d3c2ebfef 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-up-sampling2-d.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-up-sampling2-d.pbtxt
@@ -102,7 +102,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-up-sampling3-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-up-sampling3-d.pbtxt
index c1bd2dcbaf..d790cf2e08 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-up-sampling3-d.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-up-sampling3-d.pbtxt
@@ -102,7 +102,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-wrapper.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-wrapper.pbtxt
index c020dc3954..9cee68874a 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-wrapper.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-wrapper.pbtxt
@@ -106,7 +106,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-zero-padding1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-zero-padding1-d.pbtxt
index b7fe482145..ba6c23ae75 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-zero-padding1-d.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-zero-padding1-d.pbtxt
@@ -102,7 +102,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-zero-padding2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-zero-padding2-d.pbtxt
index 51f50882b2..cb587d67b0 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-zero-padding2-d.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-zero-padding2-d.pbtxt
@@ -102,7 +102,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-zero-padding3-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-zero-padding3-d.pbtxt
index e558931ead..415720cbe1 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-zero-padding3-d.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-zero-padding3-d.pbtxt
@@ -102,7 +102,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.models.-model.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.models.-model.pbtxt
index 1f3422b9a1..af9a44086f 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.models.-model.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.models.-model.pbtxt
@@ -120,7 +120,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.models.-sequential.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.models.-sequential.pbtxt
index 187c3a85b3..5034fdff2a 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.models.-sequential.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.models.-sequential.pbtxt
@@ -133,7 +133,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_weight"
diff --git a/tensorflow/tools/api/golden/tensorflow.layers.-average-pooling1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.-average-pooling1-d.pbtxt
index 7fdf97ed79..6e595ca343 100644
--- a/tensorflow/tools/api/golden/tensorflow.layers.-average-pooling1-d.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.layers.-average-pooling1-d.pbtxt
@@ -94,7 +94,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/tensorflow.layers.-average-pooling2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.-average-pooling2-d.pbtxt
index 5911fbefa9..7b6c30773b 100644
--- a/tensorflow/tools/api/golden/tensorflow.layers.-average-pooling2-d.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.layers.-average-pooling2-d.pbtxt
@@ -94,7 +94,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/tensorflow.layers.-average-pooling3-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.-average-pooling3-d.pbtxt
index e837458615..7a7664e800 100644
--- a/tensorflow/tools/api/golden/tensorflow.layers.-average-pooling3-d.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.layers.-average-pooling3-d.pbtxt
@@ -94,7 +94,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/tensorflow.layers.-batch-normalization.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.-batch-normalization.pbtxt
index 6e07b911a4..c9f5c18f25 100644
--- a/tensorflow/tools/api/golden/tensorflow.layers.-batch-normalization.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.layers.-batch-normalization.pbtxt
@@ -93,7 +93,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/tensorflow.layers.-conv1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.-conv1-d.pbtxt
index 9ee79be96d..1fa00d7b2f 100644
--- a/tensorflow/tools/api/golden/tensorflow.layers.-conv1-d.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.layers.-conv1-d.pbtxt
@@ -94,7 +94,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/tensorflow.layers.-conv2-d-transpose.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.-conv2-d-transpose.pbtxt
index 67bd7d2cc1..a92a1094ac 100644
--- a/tensorflow/tools/api/golden/tensorflow.layers.-conv2-d-transpose.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.layers.-conv2-d-transpose.pbtxt
@@ -95,7 +95,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/tensorflow.layers.-conv2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.-conv2-d.pbtxt
index f310b7ea86..7fa78ab20b 100644
--- a/tensorflow/tools/api/golden/tensorflow.layers.-conv2-d.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.layers.-conv2-d.pbtxt
@@ -94,7 +94,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/tensorflow.layers.-conv3-d-transpose.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.-conv3-d-transpose.pbtxt
index b786667795..e92e4859ae 100644
--- a/tensorflow/tools/api/golden/tensorflow.layers.-conv3-d-transpose.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.layers.-conv3-d-transpose.pbtxt
@@ -95,7 +95,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/tensorflow.layers.-conv3-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.-conv3-d.pbtxt
index 02c8130b48..87e5c2949e 100644
--- a/tensorflow/tools/api/golden/tensorflow.layers.-conv3-d.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.layers.-conv3-d.pbtxt
@@ -94,7 +94,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/tensorflow.layers.-dense.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.-dense.pbtxt
index 268cb788d1..cc4ee4c8a5 100644
--- a/tensorflow/tools/api/golden/tensorflow.layers.-dense.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.layers.-dense.pbtxt
@@ -93,7 +93,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/tensorflow.layers.-dropout.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.-dropout.pbtxt
index 969ec33578..99ab2ef97c 100644
--- a/tensorflow/tools/api/golden/tensorflow.layers.-dropout.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.layers.-dropout.pbtxt
@@ -93,7 +93,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/tensorflow.layers.-flatten.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.-flatten.pbtxt
index fb602e41be..f4074c5a4f 100644
--- a/tensorflow/tools/api/golden/tensorflow.layers.-flatten.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.layers.-flatten.pbtxt
@@ -93,7 +93,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/tensorflow.layers.-layer.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.-layer.pbtxt
index ec65fc4555..ec51609dee 100644
--- a/tensorflow/tools/api/golden/tensorflow.layers.-layer.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.layers.-layer.pbtxt
@@ -92,7 +92,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/tensorflow.layers.-max-pooling1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.-max-pooling1-d.pbtxt
index 60aec6cd14..745c532e94 100644
--- a/tensorflow/tools/api/golden/tensorflow.layers.-max-pooling1-d.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.layers.-max-pooling1-d.pbtxt
@@ -94,7 +94,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/tensorflow.layers.-max-pooling2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.-max-pooling2-d.pbtxt
index bc2f49cc18..f8244c01b6 100644
--- a/tensorflow/tools/api/golden/tensorflow.layers.-max-pooling2-d.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.layers.-max-pooling2-d.pbtxt
@@ -94,7 +94,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/tensorflow.layers.-max-pooling3-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.-max-pooling3-d.pbtxt
index 83b98059f9..df5378f279 100644
--- a/tensorflow/tools/api/golden/tensorflow.layers.-max-pooling3-d.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.layers.-max-pooling3-d.pbtxt
@@ -94,7 +94,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/tensorflow.layers.-separable-conv2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.-separable-conv2-d.pbtxt
index 83f3ed82da..c55d2bccc9 100644
--- a/tensorflow/tools/api/golden/tensorflow.layers.-separable-conv2-d.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.layers.-separable-conv2-d.pbtxt
@@ -95,7 +95,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-basic-l-s-t-m-cell.pbtxt b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-basic-l-s-t-m-cell.pbtxt
index b8e27cc6cb..49066eecaa 100644
--- a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-basic-l-s-t-m-cell.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-basic-l-s-t-m-cell.pbtxt
@@ -103,7 +103,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-basic-r-n-n-cell.pbtxt b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-basic-r-n-n-cell.pbtxt
index 29bc20ef1a..5646461b24 100644
--- a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-basic-r-n-n-cell.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-basic-r-n-n-cell.pbtxt
@@ -102,7 +102,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-device-wrapper.pbtxt b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-device-wrapper.pbtxt
index 17ee1ff5fb..81dcd90e81 100644
--- a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-device-wrapper.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-device-wrapper.pbtxt
@@ -102,7 +102,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-dropout-wrapper.pbtxt b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-dropout-wrapper.pbtxt
index fe4f630a39..8ff225897a 100644
--- a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-dropout-wrapper.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-dropout-wrapper.pbtxt
@@ -102,7 +102,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-g-r-u-cell.pbtxt b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-g-r-u-cell.pbtxt
index 1c8dd65d27..2adfc747d1 100644
--- a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-g-r-u-cell.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-g-r-u-cell.pbtxt
@@ -102,7 +102,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-l-s-t-m-cell.pbtxt b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-l-s-t-m-cell.pbtxt
index 0f294e216a..8d17153972 100644
--- a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-l-s-t-m-cell.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-l-s-t-m-cell.pbtxt
@@ -1,6 +1,7 @@
 path: "tensorflow.nn.rnn_cell.LSTMCell"
 tf_class {
   is_instance: "<class \'tensorflow.python.ops.rnn_cell_impl.LSTMCell\'>"
+  is_instance: "<class \'tensorflow.python.ops.rnn_cell_impl._LayerRNNCell\'>"
   is_instance: "<class \'tensorflow.python.ops.rnn_cell_impl.RNNCell\'>"
   is_instance: "<class \'tensorflow.python.layers.base.Layer\'>"
   is_instance: "<type \'object\'>"
@@ -90,7 +91,7 @@ tf_class {
   }
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'num_units\', \'use_peepholes\', \'cell_clip\', \'initializer\', \'num_proj\', \'proj_clip\', \'num_unit_shards\', \'num_proj_shards\', \'forget_bias\', \'state_is_tuple\', \'activation\', \'reuse\'], varargs=None, keywords=None, defaults=[\'False\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'1.0\', \'True\', \'None\', \'None\'], "
+    argspec: "args=[\'self\', \'num_units\', \'use_peepholes\', \'cell_clip\', \'initializer\', \'num_proj\', \'proj_clip\', \'num_unit_shards\', \'num_proj_shards\', \'forget_bias\', \'state_is_tuple\', \'activation\', \'reuse\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'1.0\', \'True\', \'None\', \'None\', \'None\'], "
   }
   member_method {
     name: "add_loss"
@@ -102,7 +103,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
@@ -110,7 +111,7 @@ tf_class {
   }
   member_method {
     name: "build"
-    argspec: "args=[\'self\', \'_\'], varargs=None, keywords=None, defaults=None"
+    argspec: "args=[\'self\', \'inputs_shape\'], varargs=None, keywords=None, defaults=None"
   }
   member_method {
     name: "call"
diff --git a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-multi-r-n-n-cell.pbtxt b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-multi-r-n-n-cell.pbtxt
index ed42631471..68c3064dd4 100644
--- a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-multi-r-n-n-cell.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-multi-r-n-n-cell.pbtxt
@@ -102,7 +102,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-r-n-n-cell.pbtxt b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-r-n-n-cell.pbtxt
index 2c7dc7c4f2..86ff0fee2b 100644
--- a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-r-n-n-cell.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-r-n-n-cell.pbtxt
@@ -101,7 +101,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
diff --git a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-residual-wrapper.pbtxt b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-residual-wrapper.pbtxt
index dbcbf29586..1a6f8a3b7d 100644
--- a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-residual-wrapper.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-residual-wrapper.pbtxt
@@ -102,7 +102,7 @@ tf_class {
   }
   member_method {
     name: "add_variable"
-    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], "
+    argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], "
   }
   member_method {
     name: "apply"
-- 
GitLab


From 966016b7f2382658e7c84baae0596d35f0a49bae Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 26 Oct 2017 17:16:35 -0700
Subject: [PATCH 848/909] BUILD dependency cleanup in contrib/...

PiperOrigin-RevId: 173613863
---
 tensorflow/contrib/BUILD                      |  2 ++
 tensorflow/contrib/all_reduce/BUILD           |  3 ++-
 tensorflow/contrib/bayesflow/BUILD            |  4 ++--
 .../boosted_trees/estimator_batch/BUILD       |  2 ++
 .../contrib/data/python/kernel_tests/BUILD    |  3 ---
 tensorflow/contrib/data/python/ops/BUILD      |  6 ++---
 tensorflow/contrib/distributions/BUILD        |  8 +++++++
 tensorflow/contrib/eager/python/BUILD         | 24 +++++++++++++++----
 tensorflow/contrib/estimator/BUILD            |  3 ++-
 tensorflow/contrib/framework/BUILD            |  3 +--
 tensorflow/contrib/gan/BUILD                  |  7 +++++-
 tensorflow/contrib/gdr/BUILD                  |  1 -
 tensorflow/contrib/graph_editor/BUILD         |  2 +-
 tensorflow/contrib/hooks/BUILD                |  1 +
 tensorflow/contrib/image/BUILD                |  3 ++-
 .../contrib/kfac/python/kernel_tests/BUILD    |  1 -
 tensorflow/contrib/kfac/python/ops/BUILD      |  2 ++
 tensorflow/contrib/labeled_tensor/BUILD       |  2 +-
 tensorflow/contrib/layers/BUILD               |  9 +++----
 19 files changed, 58 insertions(+), 28 deletions(-)

diff --git a/tensorflow/contrib/BUILD b/tensorflow/contrib/BUILD
index ee3dd5079e..2e9b96bb1d 100644
--- a/tensorflow/contrib/BUILD
+++ b/tensorflow/contrib/BUILD
@@ -88,8 +88,10 @@ py_library(
         "//tensorflow/contrib/tfprof",
         "//tensorflow/contrib/timeseries",
         "//tensorflow/contrib/tpu",
+        "//tensorflow/contrib/tpu:tpu_py",
         "//tensorflow/contrib/training:training_py",
         "//tensorflow/contrib/util:util_py",
+        "//tensorflow/python:util",
     ] + if_mpi(["//tensorflow/contrib/mpi_collectives:mpi_ops_py"]),
 )
 
diff --git a/tensorflow/contrib/all_reduce/BUILD b/tensorflow/contrib/all_reduce/BUILD
index 744ae4c1f4..35b9de27e7 100644
--- a/tensorflow/contrib/all_reduce/BUILD
+++ b/tensorflow/contrib/all_reduce/BUILD
@@ -19,9 +19,10 @@ py_library(
     srcs_version = "PY2AND3",
     visibility = ["//visibility:public"],
     deps = [
-        "//tensorflow/contrib/nccl:nccl_ops",
+        "//tensorflow/contrib/nccl:nccl_py",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:framework_ops",
+        "//tensorflow/python:math_ops",
     ],
 )
 
diff --git a/tensorflow/contrib/bayesflow/BUILD b/tensorflow/contrib/bayesflow/BUILD
index 324e519a6d..8bb742d289 100644
--- a/tensorflow/contrib/bayesflow/BUILD
+++ b/tensorflow/contrib/bayesflow/BUILD
@@ -20,8 +20,9 @@ py_library(
         "//tensorflow/python:array_ops",
         "//tensorflow/python:check_ops",
         "//tensorflow/python:control_flow_ops",
-        "//tensorflow/python:framework",
         "//tensorflow/python:framework_for_generated_wrappers",
+        "//tensorflow/python:functional_ops",
+        "//tensorflow/python:gradients",
         "//tensorflow/python:math_ops",
         "//tensorflow/python:nn",
         "//tensorflow/python:nn_ops",
@@ -31,7 +32,6 @@ py_library(
         "//tensorflow/python:training",
         "//tensorflow/python:util",
         "//tensorflow/python:variable_scope",
-        "//tensorflow/python:variables",
         "//tensorflow/python/ops/distributions",
         "//third_party/py/numpy",
         "@six_archive//:six",
diff --git a/tensorflow/contrib/boosted_trees/estimator_batch/BUILD b/tensorflow/contrib/boosted_trees/estimator_batch/BUILD
index d0ee1fd60d..7792c7127c 100644
--- a/tensorflow/contrib/boosted_trees/estimator_batch/BUILD
+++ b/tensorflow/contrib/boosted_trees/estimator_batch/BUILD
@@ -124,6 +124,8 @@ py_library(
     srcs_version = "PY2AND3",
     deps = [
         ":model",
+        "//tensorflow/contrib/boosted_trees:losses",
         "//tensorflow/contrib/learn",
+        "//tensorflow/python:math_ops",
     ],
 )
diff --git a/tensorflow/contrib/data/python/kernel_tests/BUILD b/tensorflow/contrib/data/python/kernel_tests/BUILD
index 36af55a7ec..c310e79741 100644
--- a/tensorflow/contrib/data/python/kernel_tests/BUILD
+++ b/tensorflow/contrib/data/python/kernel_tests/BUILD
@@ -302,11 +302,8 @@ py_test(
         "//tensorflow/contrib/data/python/ops:transformation_ops",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:errors",
-        "//tensorflow/python:framework_ops",
         "//tensorflow/python:string_ops",
-        "//tensorflow/python:training",
         "//tensorflow/python:util",
-        "//tensorflow/python:variables",
         "//third_party/py/numpy",
     ],
 )
diff --git a/tensorflow/contrib/data/python/ops/BUILD b/tensorflow/contrib/data/python/ops/BUILD
index b17b02ee35..a6eb50014a 100644
--- a/tensorflow/contrib/data/python/ops/BUILD
+++ b/tensorflow/contrib/data/python/ops/BUILD
@@ -12,9 +12,7 @@ py_library(
     srcs_version = "PY2AND3",
     deps = [
         ":transformation_ops",
-        "//tensorflow/python:dtypes",
-        "//tensorflow/python:script_ops",
-        "//tensorflow/python:tensor_shape",
+        "//tensorflow/python:util",
         "//tensorflow/python/data/ops:dataset_ops",
         "//tensorflow/python/data/util:nest",
     ],
@@ -48,6 +46,7 @@ py_library(
         "//tensorflow/python:platform",
         "//tensorflow/python:sparse_tensor",
         "//tensorflow/python:tensor_shape",
+        "//tensorflow/python:util",
         "//tensorflow/python/data/ops:dataset_ops",
         "//tensorflow/python/data/ops:readers",
         "//tensorflow/python/data/util:nest",
@@ -76,7 +75,6 @@ py_library(
         "//tensorflow/python:logging_ops",
         "//tensorflow/python:math_ops",
         "//tensorflow/python:random_ops",
-        "//tensorflow/python:resource_variable_ops",
         "//tensorflow/python:tensor_shape",
         "//tensorflow/python:tensor_util",
         "//tensorflow/python/data/ops:dataset_ops",
diff --git a/tensorflow/contrib/distributions/BUILD b/tensorflow/contrib/distributions/BUILD
index 1305c28012..bc72bc37a7 100644
--- a/tensorflow/contrib/distributions/BUILD
+++ b/tensorflow/contrib/distributions/BUILD
@@ -18,14 +18,20 @@ py_library(
         "//tensorflow/contrib/linalg:linalg_py",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:check_ops",
+        "//tensorflow/python:clip_ops",
         "//tensorflow/python:control_flow_ops",
         "//tensorflow/python:framework_for_generated_wrappers",
+        "//tensorflow/python:init_ops",
+        "//tensorflow/python:layers",
         "//tensorflow/python:linalg_ops",
         "//tensorflow/python:math_ops",
         "//tensorflow/python:nn_ops",
+        "//tensorflow/python:template",
         "//tensorflow/python:tensor_util",
         "//tensorflow/python:util",
+        "//tensorflow/python:variable_scope",
         "//tensorflow/python/ops/distributions",
+        "//tensorflow/python/ops/linalg",
         "//third_party/py/numpy",
     ],
 )
@@ -55,7 +61,9 @@ py_library(
         "//tensorflow/python:tensor_util",
         "//tensorflow/python:util",
         "//tensorflow/python:variable_scope",
+        "//tensorflow/python:variables",
         "//tensorflow/python/ops/distributions",
+        "//tensorflow/python/ops/linalg",
         "//third_party/py/numpy",
         "@six_archive//:six",
     ],
diff --git a/tensorflow/contrib/eager/python/BUILD b/tensorflow/contrib/eager/python/BUILD
index 179c27ba80..cb7b5cf462 100644
--- a/tensorflow/contrib/eager/python/BUILD
+++ b/tensorflow/contrib/eager/python/BUILD
@@ -18,6 +18,7 @@ py_library(
         ":saver",
         ":summary_writer",
         "//tensorflow/python:framework_ops",
+        "//tensorflow/python:framework_test_lib",
         "//tensorflow/python:numerics",
         "//tensorflow/python:resource_variable_ops",
         "//tensorflow/python:util",
@@ -52,6 +53,7 @@ py_library(
         "//tensorflow/python:array_ops",
         "//tensorflow/python:dataset_ops_gen",
         "//tensorflow/python:errors",
+        "//tensorflow/python:framework_ops",
         "//tensorflow/python:resource_variable_ops",
         "//tensorflow/python/data/util:nest",
         "//tensorflow/python/eager:context",
@@ -64,10 +66,11 @@ py_test(
     srcs_version = "PY2AND3",
     deps = [
         ":datasets",
+        "//tensorflow/python:dtypes",
         "//tensorflow/python:math_ops",
+        "//tensorflow/python:script_ops",
         "//tensorflow/python/data",
         "//tensorflow/python/eager:test",
-        "//third_party/py/numpy",
     ],
 )
 
@@ -76,7 +79,11 @@ py_library(
     srcs = ["saver.py"],
     srcs_version = "PY2AND3",
     deps = [
+        "//tensorflow/python:errors",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:resource_variable_ops",
         "//tensorflow/python:training",
+        "//tensorflow/python/eager:context",
     ],
 )
 
@@ -100,12 +107,14 @@ py_library(
     srcs_version = "PY2AND3",
     deps = [
         "//tensorflow/contrib/summary:gen_summary_ops",
-        "//tensorflow/contrib/summary:summary_ops",
         "//tensorflow/python:constant_op",
+        "//tensorflow/python:dtypes",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:init_ops",
         "//tensorflow/python:resource_variable_ops",
         "//tensorflow/python:state_ops",
         "//tensorflow/python:summary_op_util",
-        "//tensorflow/python:training",
+        "//tensorflow/python:variable_scope",
         "//tensorflow/python/eager:context",
     ],
 )
@@ -137,8 +146,7 @@ py_library(
         "//tensorflow/python:constant_op",
         "//tensorflow/python:control_flow_ops",
         "//tensorflow/python:dtypes",
-        "//tensorflow/python:framework_ops",
-        "//tensorflow/python:layers_base",
+        "//tensorflow/python:init_ops",
         "//tensorflow/python:math_ops",
         "//tensorflow/python:resource_variable_ops",
         "//tensorflow/python:util",
@@ -154,8 +162,13 @@ py_test(
     srcs_version = "PY2AND3",
     deps = [
         ":metrics",
+        "//tensorflow/contrib/summary:summary_ops",
+        "//tensorflow/core:protos_all_py",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:dtypes",
+        "//tensorflow/python:lib",
+        "//tensorflow/python:platform",
+        "//tensorflow/python:training",
         "//tensorflow/python:variables",
         "//tensorflow/python/eager:context",
         "//tensorflow/python/eager:test",
@@ -203,6 +216,7 @@ py_library(
         "//tensorflow/python:layers_base",
         "//tensorflow/python:variable_scope",
         "//tensorflow/python/estimator:util",
+        "@six_archive//:six",
     ],
 )
 
diff --git a/tensorflow/contrib/estimator/BUILD b/tensorflow/contrib/estimator/BUILD
index 8a7d67b5c2..79b166ac88 100644
--- a/tensorflow/contrib/estimator/BUILD
+++ b/tensorflow/contrib/estimator/BUILD
@@ -190,7 +190,8 @@ py_test(
     deps = [
         ":logit_fns",
         "//tensorflow/python:client_testlib",
-        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:constant_op",
+        "//tensorflow/python:session",
         "//tensorflow/python/estimator:model_fn",
     ],
 )
diff --git a/tensorflow/contrib/framework/BUILD b/tensorflow/contrib/framework/BUILD
index dd882acb8e..90aed3065b 100644
--- a/tensorflow/contrib/framework/BUILD
+++ b/tensorflow/contrib/framework/BUILD
@@ -239,7 +239,6 @@ py_test(
     deps = [
         ":framework_py",
         "//tensorflow/python:array_ops",
-        "//tensorflow/python:client",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:errors",
         "//tensorflow/python:framework_for_generated_wrappers",
@@ -247,6 +246,7 @@ py_test(
         "//tensorflow/python:nn_ops",
         "//tensorflow/python:partitioned_variables",
         "//tensorflow/python:platform",
+        "//tensorflow/python:session",
         "//tensorflow/python:training",
         "//tensorflow/python:variable_scope",
         "//tensorflow/python:variables",
@@ -279,7 +279,6 @@ py_test(
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:constant_op",
         "//tensorflow/python:dtypes",
-        "//tensorflow/python:errors",
         "//tensorflow/python:framework_ops",
         "//tensorflow/python:math_ops",
         "//tensorflow/python:partitioned_variables",
diff --git a/tensorflow/contrib/gan/BUILD b/tensorflow/contrib/gan/BUILD
index 27a5d6ec31..1418c87023 100644
--- a/tensorflow/contrib/gan/BUILD
+++ b/tensorflow/contrib/gan/BUILD
@@ -202,6 +202,7 @@ py_library(
         "//tensorflow/python:embedding_ops",
         "//tensorflow/python:math_ops",
         "//tensorflow/python:tensor_util",
+        "//tensorflow/python:util",
         "//tensorflow/python:variable_scope",
     ],
 )
@@ -234,6 +235,7 @@ py_library(
         "//tensorflow/python:nn",
         "//tensorflow/python:tensor_shape",
         "//tensorflow/python:tensor_util",
+        "//tensorflow/python:util",
         "//tensorflow/python:variable_scope",
     ],
 )
@@ -267,7 +269,10 @@ py_library(
         "python/features/python/clip_weights_impl.py",
     ],
     srcs_version = "PY2AND3",
-    deps = ["//tensorflow/contrib/opt:opt_py"],
+    deps = [
+        "//tensorflow/contrib/opt:opt_py",
+        "//tensorflow/python:util",
+    ],
 )
 
 py_test(
diff --git a/tensorflow/contrib/gdr/BUILD b/tensorflow/contrib/gdr/BUILD
index bebcf079ba..a8053be69b 100644
--- a/tensorflow/contrib/gdr/BUILD
+++ b/tensorflow/contrib/gdr/BUILD
@@ -119,7 +119,6 @@ cc_library(
         ":gdr_memory_manager",
         ":gdr_rendezvous_mgr",
         ":gdr_worker",
-        "//tensorflow/core:lib_internal",
         "//tensorflow/core/distributed_runtime/rpc:grpc_server_lib",
     ],
     alwayslink = 1,
diff --git a/tensorflow/contrib/graph_editor/BUILD b/tensorflow/contrib/graph_editor/BUILD
index b4c53d3da6..967ad2fc09 100644
--- a/tensorflow/contrib/graph_editor/BUILD
+++ b/tensorflow/contrib/graph_editor/BUILD
@@ -144,12 +144,12 @@ py_test(
         ":graph_editor_py",
         ":match",
         "//tensorflow/python:array_ops",
-        "//tensorflow/python:client",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:control_flow_ops",
         "//tensorflow/python:framework_for_generated_wrappers",
         "//tensorflow/python:gradients",
         "//tensorflow/python:math_ops",
+        "//tensorflow/python:session",
         "//tensorflow/python:variables",
         "//third_party/py/numpy",
     ],
diff --git a/tensorflow/contrib/hooks/BUILD b/tensorflow/contrib/hooks/BUILD
index 1576c9ec9b..1b528d7afc 100644
--- a/tensorflow/contrib/hooks/BUILD
+++ b/tensorflow/contrib/hooks/BUILD
@@ -20,6 +20,7 @@ py_library(
     srcs_version = "PY2AND3",
     deps = [
         "//tensorflow/python:training",
+        "//tensorflow/python:util",
     ],
 )
 
diff --git a/tensorflow/contrib/image/BUILD b/tensorflow/contrib/image/BUILD
index d0600d4668..c0c56d2e4a 100755
--- a/tensorflow/contrib/image/BUILD
+++ b/tensorflow/contrib/image/BUILD
@@ -143,12 +143,13 @@ py_library(
     srcs_version = "PY2AND3",
     deps = [
         ":distort_image_ops",
+        ":single_image_random_dot_stereograms_py",
         "//tensorflow/contrib/util:util_py",
-        "//tensorflow/python:framework",
         "//tensorflow/python:framework_for_generated_wrappers",
         "//tensorflow/python:image_ops",
         "//tensorflow/python:platform",
         "//tensorflow/python:random_ops",
+        "//tensorflow/python:util",
     ],
 )
 
diff --git a/tensorflow/contrib/kfac/python/kernel_tests/BUILD b/tensorflow/contrib/kfac/python/kernel_tests/BUILD
index 5d86373a23..0653e71d12 100644
--- a/tensorflow/contrib/kfac/python/kernel_tests/BUILD
+++ b/tensorflow/contrib/kfac/python/kernel_tests/BUILD
@@ -88,7 +88,6 @@ py_test(
     deps = [
         "//tensorflow/contrib/kfac/python/ops:kfac_optimizer",
         "//tensorflow/contrib/kfac/python/ops:layer_collection",
-        "//tensorflow/contrib/kfac/python/ops:loss_functions",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:framework_ops",
diff --git a/tensorflow/contrib/kfac/python/ops/BUILD b/tensorflow/contrib/kfac/python/ops/BUILD
index 5d5046c9ec..de4b8920b8 100644
--- a/tensorflow/contrib/kfac/python/ops/BUILD
+++ b/tensorflow/contrib/kfac/python/ops/BUILD
@@ -66,6 +66,7 @@ py_library(
     deps = [
         "//tensorflow/python:array_ops",
         "//tensorflow/python:math_ops",
+        "//tensorflow/python:tensor_shape",
         "//tensorflow/python/ops/distributions",
         "@six_archive//:six",
     ],
@@ -89,6 +90,7 @@ py_library(
         ":utils",
         "//tensorflow/python:gradients",
         "//tensorflow/python:math_ops",
+        "//tensorflow/python:util",
     ],
 )
 
diff --git a/tensorflow/contrib/labeled_tensor/BUILD b/tensorflow/contrib/labeled_tensor/BUILD
index 4eba29caec..894e6f6946 100644
--- a/tensorflow/contrib/labeled_tensor/BUILD
+++ b/tensorflow/contrib/labeled_tensor/BUILD
@@ -109,9 +109,9 @@ py_test(
         ":test_util",
         "//tensorflow/core:protos_all_py",
         "//tensorflow/python:array_ops",
-        "//tensorflow/python:client",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:framework_for_generated_wrappers",
+        "//tensorflow/python:session",
     ],
 )
 
diff --git a/tensorflow/contrib/layers/BUILD b/tensorflow/contrib/layers/BUILD
index bbb4fb1f57..1ae4d281c4 100644
--- a/tensorflow/contrib/layers/BUILD
+++ b/tensorflow/contrib/layers/BUILD
@@ -153,10 +153,10 @@ py_test(
     deps = [
         ":layers_py",
         "//tensorflow/python:array_ops",
-        "//tensorflow/python:client",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:framework_for_generated_wrappers",
         "//tensorflow/python:math_ops",
+        "//tensorflow/python:session",
         "//third_party/py/numpy",
     ],
 )
@@ -168,9 +168,9 @@ py_test(
     srcs_version = "PY2AND3",
     deps = [
         ":layers_py",
-        "//tensorflow/python:client",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:framework_for_generated_wrappers",
+        "//tensorflow/python:session",
         "//tensorflow/python:variable_scope",
         "//tensorflow/python:variables",
         "//third_party/py/numpy",
@@ -238,6 +238,7 @@ py_test(
         ":layers_py",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:framework_for_generated_wrappers",
+        "//tensorflow/python:lookup_ops",
         "//tensorflow/python:parsing_ops",
         "//tensorflow/python:sparse_tensor",
         "//tensorflow/python:state_ops",
@@ -280,9 +281,9 @@ py_test(
     srcs_version = "PY2AND3",
     deps = [
         ":layers_py",
-        "//tensorflow/python:client",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:framework_for_generated_wrappers",
+        "//tensorflow/python:session",
         "//tensorflow/python:variables",
     ],
 )
@@ -294,9 +295,9 @@ py_test(
     srcs_version = "PY2AND3",
     deps = [
         ":layers_py",
-        "//tensorflow/python:client",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:framework_for_generated_wrappers",
+        "//tensorflow/python:session",
         "//tensorflow/python:sparse_ops",
         "//tensorflow/python:sparse_tensor",
         "//third_party/py/numpy",
-- 
GitLab


From f9aa795318e6d82c310440fb3f80b240bb034fcc Mon Sep 17 00:00:00 2001
From: Justine Tunney <jart@google.com>
Date: Thu, 26 Oct 2017 17:40:41 -0700
Subject: [PATCH 849/909] Introduce TensorBoard SQL schema for summaries

Unlike dataset_ops which are designed to read arbitrary SQL data into tensors,
this is designed to write TensorFlow data to SQL.

Please note that this code is going to be moved into the TensorBoard codebase
as soon as that's feasible.

PiperOrigin-RevId: 173616340
---
 tensorflow/BUILD                              |   1 +
 tensorflow/contrib/tensorboard/db/BUILD       |  36 ++
 tensorflow/contrib/tensorboard/db/schema.cc   | 412 ++++++++++++++++++
 tensorflow/contrib/tensorboard/db/schema.h    |  33 ++
 .../contrib/tensorboard/db/schema_test.cc     |  34 ++
 5 files changed, 516 insertions(+)
 create mode 100644 tensorflow/contrib/tensorboard/db/BUILD
 create mode 100644 tensorflow/contrib/tensorboard/db/schema.cc
 create mode 100644 tensorflow/contrib/tensorboard/db/schema.h
 create mode 100644 tensorflow/contrib/tensorboard/db/schema_test.cc

diff --git a/tensorflow/BUILD b/tensorflow/BUILD
index 20f02ad50a..8667fd7c91 100644
--- a/tensorflow/BUILD
+++ b/tensorflow/BUILD
@@ -437,6 +437,7 @@ filegroup(
         "//tensorflow/contrib/tensor_forest/kernels/v4:all_files",
         "//tensorflow/contrib/tensor_forest/proto:all_files",
         "//tensorflow/contrib/tensorboard:all_files",
+        "//tensorflow/contrib/tensorboard/db:all_files",
         "//tensorflow/contrib/testing:all_files",
         "//tensorflow/contrib/text:all_files",
         "//tensorflow/contrib/tfprof:all_files",
diff --git a/tensorflow/contrib/tensorboard/db/BUILD b/tensorflow/contrib/tensorboard/db/BUILD
new file mode 100644
index 0000000000..f056632295
--- /dev/null
+++ b/tensorflow/contrib/tensorboard/db/BUILD
@@ -0,0 +1,36 @@
+# Description:
+#   TensorBoard database code.
+
+package(default_visibility = ["//tensorflow:internal"])
+
+licenses(["notice"])  # Apache 2.0
+
+load("//tensorflow:tensorflow.bzl", "tf_cc_test")
+
+cc_library(
+    name = "schema",
+    srcs = ["schema.cc"],
+    hdrs = ["schema.h"],
+    deps = [
+        "//tensorflow/core:lib",
+        "//tensorflow/core/lib/db:sqlite",
+    ],
+)
+
+tf_cc_test(
+    name = "schema_test",
+    srcs = ["schema_test.cc"],
+    deps = [
+        ":schema",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+        "//tensorflow/core/lib/db:sqlite",
+    ],
+)
+
+filegroup(
+    name = "all_files",
+    srcs = glob(["*"]),
+    visibility = ["//tensorflow:__pkg__"],
+)
diff --git a/tensorflow/contrib/tensorboard/db/schema.cc b/tensorflow/contrib/tensorboard/db/schema.cc
new file mode 100644
index 0000000000..f5a8e02a9b
--- /dev/null
+++ b/tensorflow/contrib/tensorboard/db/schema.cc
@@ -0,0 +1,412 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/contrib/tensorboard/db/schema.h"
+
+namespace tensorflow {
+namespace db {
+namespace {
+
+class SqliteSchema {
+ public:
+  explicit SqliteSchema(Sqlite* db) : db_(db) {}
+  ~SqliteSchema() { db_ = nullptr; }
+
+  /// \brief Creates Tensors table.
+  ///
+  /// Fields:
+  ///   rowid: Ephemeral b-tree ID dictating locality.
+  ///   tag_id: ID of associated Tag.
+  ///   computed_time: Float UNIX timestamp with microsecond precision.
+  ///     In the old summaries system that uses FileWriter, this is the
+  ///     wall time around when tf.Session.run finished. In the new
+  ///     summaries system, it is the wall time of when the tensor was
+  ///     computed. On systems with monotonic clocks, it is calculated
+  ///     by adding the monotonic run duration to Run.started_time.
+  ///     This field is not indexed because, in practice, it should be
+  ///     ordered the same or nearly the same as TensorIndex, so local
+  ///     insertion sort might be more suitable.
+  ///   step: User-supplied number, ordering this tensor in Tag.
+  ///     If NULL then the Tag must have only one Tensor.
+  ///   tensor: Can be an INTEGER (DT_INT64), FLOAT (DT_DOUBLE), or
+  ///     BLOB. The structure of a BLOB is currently undefined, but in
+  ///     essence it is a Snappy tf.TensorProto that spills over into
+  ///     TensorChunks.
+  Status CreateTensorsTable() {
+    return Run(R"sql(
+      CREATE TABLE IF NOT EXISTS Tensors (
+        rowid INTEGER PRIMARY KEY,
+        tag_id INTEGER NOT NULL,
+        computed_time REAL,
+        step INTEGER,
+        tensor BLOB
+      )
+    )sql");
+  }
+
+  /// \brief Creates TensorChunks table.
+  ///
+  /// This table can be used to split up a tensor across many rows,
+  /// which has the advantage of not slowing down table scans on the
+  /// main table, allowing asynchronous fetching, minimizing copying,
+  /// and preventing large buffers from being allocated.
+  ///
+  /// Fields:
+  ///   rowid: Ephemeral b-tree ID dictating locality.
+  ///   tag_id: ID of associated Tag.
+  ///   step: Same as corresponding Tensors.step.
+  ///   sequence: 1-indexed sequence number for ordering chunks. Please
+  ///     note that the 0th index is Tensors.tensor.
+  ///   chunk: Bytes of next chunk in tensor.
+  Status CreateTensorChunksTable() {
+    return Run(R"sql(
+      CREATE TABLE IF NOT EXISTS TensorChunks (
+        rowid INTEGER PRIMARY KEY,
+        tag_id INTEGER NOT NULL,
+        step INTEGER,
+        sequence INTEGER,
+        chunk BLOB
+      )
+    )sql");
+  }
+
+  /// \brief Creates Tags table.
+  ///
+  /// Fields:
+  ///   rowid: Ephemeral b-tree ID dictating locality.
+  ///   tag_id: Permanent >0 unique ID.
+  ///   run_id: Optional ID of associated Run.
+  ///   tag_name: The tag field in summary.proto, unique across Run.
+  ///   inserted_time: Float UNIX timestamp with µs precision. This is
+  ///     always the wall time of when the row was inserted into the
+  ///     DB. It may be used as a hint for an archival job.
+  ///   metadata: Optional BLOB of SummaryMetadata proto.
+  ///   display_name: Optional for GUI and defaults to tag_name.
+  ///   summary_description: Optional markdown information.
+  Status CreateTagsTable() {
+    return Run(R"sql(
+      CREATE TABLE IF NOT EXISTS Tags (
+        rowid INTEGER PRIMARY KEY,
+        run_id INTEGER,
+        tag_id INTEGER NOT NULL,
+        tag_name TEXT,
+        inserted_time DOUBLE,
+        metadata BLOB,
+        display_name TEXT,
+        description TEXT
+      )
+    )sql");
+  }
+
+  /// \brief Creates Runs table.
+  ///
+  /// This table stores information about runs. Each row usually
+  /// represents a single attempt at training or testing a TensorFlow
+  /// model, with a given set of hyper-parameters, whose summaries are
+  /// written out to a single event logs directory with a monotonic step
+  /// counter.
+  ///
+  /// When a run is deleted from this table, TensorBoard should treat all
+  /// information associated with it as deleted, even if those rows in
+  /// different tables still exist.
+  ///
+  /// Fields:
+  ///   rowid: Ephemeral b-tree ID dictating locality.
+  ///   run_id: Permanent >0 unique ID.
+  ///   experiment_id: Optional ID of associated Experiment.
+  ///   run_name: User-supplied string, unique across Experiment.
+  ///   inserted_time: Float UNIX timestamp with µs precision. This is
+  ///     always the time the row was inserted into the database. It
+  ///     does not change.
+  ///   started_time: Float UNIX timestamp with µs precision. In the
+  ///     old summaries system that uses FileWriter, this is
+  ///     approximated as the first tf.Event.wall_time. In the new
+  ///     summaries system, it is the wall time of when summary writing
+  ///     started, from the perspective of whichever machine talks to
+  ///     the database. This field will be mutated if the run is
+  ///     restarted.
+  ///   description: Optional markdown information.
+  ///   graph: Snappy tf.GraphDef proto with node field cleared. That
+  ///     field can be recreated using GraphNodes and NodeDefs.
+  Status CreateRunsTable() {
+    return Run(R"sql(
+      CREATE TABLE IF NOT EXISTS Runs (
+        rowid INTEGER PRIMARY KEY,
+        experiment_id INTEGER,
+        run_id INTEGER NOT NULL,
+        run_name TEXT,
+        inserted_time REAL,
+        started_time REAL,
+        description TEXT,
+        graph BLOB
+      )
+    )sql");
+  }
+
+  /// \brief Creates Experiments table.
+  ///
+  /// This table stores information about experiments, which are sets of
+  /// runs.
+  ///
+  /// Fields:
+  ///   rowid: Ephemeral b-tree ID dictating locality.
+  ///   user_id: Optional ID of associated User.
+  ///   experiment_id: Permanent >0 unique ID.
+  ///   experiment_name: User-supplied string, unique across User.
+  ///   inserted_time: Float UNIX timestamp with µs precision. This is
+  ///     always the time the row was inserted into the database. It
+  ///     does not change.
+  ///   started_time: Float UNIX timestamp with µs precision. This is
+  ///     the MIN(experiment.started_time, run.started_time) of each
+  ///     Run added to the database.
+  ///   description: Optional markdown information.
+  Status CreateExperimentsTable() {
+    return Run(R"sql(
+      CREATE TABLE IF NOT EXISTS Experiments (
+        rowid INTEGER PRIMARY KEY,
+        user_id INTEGER,
+        experiment_id INTEGER NOT NULL,
+        experiment_name TEXT,
+        inserted_time REAL,
+        started_time REAL,
+        description TEXT
+      )
+    )sql");
+  }
+
+  /// \brief Creates Users table.
+  ///
+  /// Fields:
+  ///   rowid: Ephemeral b-tree ID dictating locality.
+  ///   user_id: Permanent >0 unique ID.
+  ///   user_name: Unique user name.
+  ///   email: Optional unique email address.
+  ///   inserted_time: Float UNIX timestamp with µs precision. This is
+  ///     always the time the row was inserted into the database. It
+  ///     does not change.
+  Status CreateUsersTable() {
+    return Run(R"sql(
+      CREATE TABLE IF NOT EXISTS Users (
+        rowid INTEGER PRIMARY KEY,
+        user_id INTEGER NOT NULL,
+        user_name TEXT,
+        email TEXT,
+        inserted_time REAL
+      )
+    )sql");
+  }
+
+  /// \brief Creates NodeDefs table.
+  ///
+  /// This table stores NodeDef protos which define the GraphDef for a
+  /// Run. This functions like a hash table so rows can be shared by
+  /// multiple Runs in an Experiment.
+  ///
+  /// Fields:
+  ///   rowid: Ephemeral b-tree ID dictating locality.
+  ///   experiment_id: Optional int64 for grouping rows.
+  ///   node_def_id: Permanent >0 unique ID.
+  ///   fingerprint: Optional farmhash::Fingerprint64() of uncompressed
+  ///     node_def bytes, coerced to int64.
+  ///   node_def: BLOB containing a Snappy tf.NodeDef proto.
+  Status CreateNodeDefsTable() {
+    return Run(R"sql(
+      CREATE TABLE IF NOT EXISTS NodeDefs (
+        rowid INTEGER PRIMARY KEY,
+        experiment_id INTEGER,
+        node_def_id INTEGER NOT NULL,
+        fingerprint INTEGER,
+        node_def TEXT
+      )
+    )sql");
+  }
+
+  /// \brief Creates RunNodeDefs table.
+  ///
+  /// Table mapping Runs to NodeDefs. This is used to recreate the node
+  /// field of the GraphDef proto.
+  ///
+  /// Fields:
+  ///   rowid: Ephemeral b-tree ID dictating locality.
+  ///   run_id: Mandatory ID of associated Run.
+  ///   node_def_id: Mandatory ID of associated NodeDef.
+  Status CreateRunNodeDefsTable() {
+    return Run(R"sql(
+      CREATE TABLE IF NOT EXISTS RunNodeDefs (
+        rowid INTEGER PRIMARY KEY,
+        run_id INTEGER NOT NULL,
+        node_def_id INTEGER NOT NULL
+      )
+    )sql");
+  }
+
+  /// \brief Uniquely indexes (tag_id, step) on Tensors table.
+  Status CreateTensorIndex() {
+    return Run(R"sql(
+      CREATE UNIQUE INDEX IF NOT EXISTS TensorIndex
+      ON Tensors (tag_id, step)
+    )sql");
+  }
+
+  /// \brief Uniquely indexes (tag_id, step, sequence) on TensorChunks table.
+  Status CreateTensorChunkIndex() {
+    return Run(R"sql(
+      CREATE UNIQUE INDEX IF NOT EXISTS TensorChunkIndex
+      ON TensorChunks (tag_id, step, sequence)
+    )sql");
+  }
+
+  /// \brief Uniquely indexes tag_id on Tags table.
+  Status CreateTagIdIndex() {
+    return Run(R"sql(
+      CREATE UNIQUE INDEX IF NOT EXISTS TagIdIndex
+      ON Tags (tag_id)
+    )sql");
+  }
+
+  /// \brief Uniquely indexes run_id on Runs table.
+  Status CreateRunIdIndex() {
+    return Run(R"sql(
+      CREATE UNIQUE INDEX IF NOT EXISTS RunIdIndex
+      ON Runs (run_id)
+    )sql");
+  }
+
+  /// \brief Uniquely indexes experiment_id on Experiments table.
+  Status CreateExperimentIdIndex() {
+    return Run(R"sql(
+      CREATE UNIQUE INDEX IF NOT EXISTS ExperimentIdIndex
+      ON Experiments (experiment_id)
+    )sql");
+  }
+
+  /// \brief Uniquely indexes user_id on Users table.
+  Status CreateUserIdIndex() {
+    return Run(R"sql(
+      CREATE UNIQUE INDEX IF NOT EXISTS UserIdIndex
+      ON Users (user_id)
+    )sql");
+  }
+
+  /// \brief Uniquely indexes node_def_id on NodeDefs table.
+  Status CreateNodeDefIdIndex() {
+    return Run(R"sql(
+      CREATE UNIQUE INDEX IF NOT EXISTS NodeDefIdIndex
+      ON NodeDefs (node_def_id)
+    )sql");
+  }
+
+  /// \brief Uniquely indexes (run_id, tag_name) on Tags table.
+  Status CreateTagNameIndex() {
+    return Run(R"sql(
+      CREATE UNIQUE INDEX IF NOT EXISTS TagNameIndex
+      ON Tags (run_id, tag_name)
+      WHERE tag_name IS NOT NULL
+    )sql");
+  }
+
+  /// \brief Uniquely indexes (experiment_id, run_name) on Runs table.
+  Status CreateRunNameIndex() {
+    return Run(R"sql(
+      CREATE UNIQUE INDEX IF NOT EXISTS RunNameIndex
+      ON Runs (experiment_id, run_name)
+      WHERE run_name IS NOT NULL
+    )sql");
+  }
+
+  /// \brief Uniquely indexes (user_id, experiment_name) on Experiments table.
+  Status CreateExperimentNameIndex() {
+    return Run(R"sql(
+      CREATE UNIQUE INDEX IF NOT EXISTS ExperimentNameIndex
+      ON Experiments (user_id, experiment_name)
+      WHERE experiment_name IS NOT NULL
+    )sql");
+  }
+
+  /// \brief Uniquely indexes user_name on Users table.
+  Status CreateUserNameIndex() {
+    return Run(R"sql(
+      CREATE UNIQUE INDEX IF NOT EXISTS UserNameIndex
+      ON Users (user_name)
+      WHERE user_name IS NOT NULL
+    )sql");
+  }
+
+  /// \brief Uniquely indexes email on Users table.
+  Status CreateUserEmailIndex() {
+    return Run(R"sql(
+      CREATE UNIQUE INDEX IF NOT EXISTS UserEmailIndex
+      ON Users (email)
+      WHERE email IS NOT NULL
+    )sql");
+  }
+
+  /// \brief Indexes (experiment_id, fingerprint) on NodeDefs table.
+  Status CreateNodeDefFingerprintIndex() {
+    return Run(R"sql(
+      CREATE INDEX IF NOT EXISTS NodeDefFingerprintIndex
+      ON NodeDefs (experiment_id, fingerprint)
+      WHERE fingerprint IS NOT NULL
+    )sql");
+  }
+
+  /// \brief Uniquely indexes (run_id, node_def_id) on RunNodeDefs table.
+  Status CreateRunNodeDefIndex() {
+    return Run(R"sql(
+      CREATE UNIQUE INDEX IF NOT EXISTS RunNodeDefIndex
+      ON RunNodeDefs (run_id, node_def_id)
+    )sql");
+  }
+
+  Status Run(const char* sql) {
+    auto stmt = db_->Prepare(sql);
+    TF_RETURN_WITH_CONTEXT_IF_ERROR(stmt->StepAndReset(), sql);
+    return Status::OK();
+  }
+
+ private:
+  Sqlite* db_;
+};
+
+}  // namespace
+
+Status SetupTensorboardSqliteDb(Sqlite* db) {
+  SqliteSchema s(db);
+  TF_RETURN_IF_ERROR(s.CreateTensorsTable());
+  TF_RETURN_IF_ERROR(s.CreateTensorChunksTable());
+  TF_RETURN_IF_ERROR(s.CreateTagsTable());
+  TF_RETURN_IF_ERROR(s.CreateRunsTable());
+  TF_RETURN_IF_ERROR(s.CreateExperimentsTable());
+  TF_RETURN_IF_ERROR(s.CreateUsersTable());
+  TF_RETURN_IF_ERROR(s.CreateNodeDefsTable());
+  TF_RETURN_IF_ERROR(s.CreateRunNodeDefsTable());
+  TF_RETURN_IF_ERROR(s.CreateTensorIndex());
+  TF_RETURN_IF_ERROR(s.CreateTensorChunkIndex());
+  TF_RETURN_IF_ERROR(s.CreateTagIdIndex());
+  TF_RETURN_IF_ERROR(s.CreateRunIdIndex());
+  TF_RETURN_IF_ERROR(s.CreateExperimentIdIndex());
+  TF_RETURN_IF_ERROR(s.CreateUserIdIndex());
+  TF_RETURN_IF_ERROR(s.CreateNodeDefIdIndex());
+  TF_RETURN_IF_ERROR(s.CreateTagNameIndex());
+  TF_RETURN_IF_ERROR(s.CreateRunNameIndex());
+  TF_RETURN_IF_ERROR(s.CreateExperimentNameIndex());
+  TF_RETURN_IF_ERROR(s.CreateUserNameIndex());
+  TF_RETURN_IF_ERROR(s.CreateUserEmailIndex());
+  TF_RETURN_IF_ERROR(s.CreateNodeDefFingerprintIndex());
+  TF_RETURN_IF_ERROR(s.CreateRunNodeDefIndex());
+  return Status::OK();
+}
+
+}  // namespace db
+}  // namespace tensorflow
diff --git a/tensorflow/contrib/tensorboard/db/schema.h b/tensorflow/contrib/tensorboard/db/schema.h
new file mode 100644
index 0000000000..d3a6922d94
--- /dev/null
+++ b/tensorflow/contrib/tensorboard/db/schema.h
@@ -0,0 +1,33 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_CONTRIB_TENSORBOARD_DB_SCHEMA_H_
+#define TENSORFLOW_CONTRIB_TENSORBOARD_DB_SCHEMA_H_
+
+#include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/lib/db/sqlite.h"
+
+namespace tensorflow {
+namespace db {
+
+/// \brief Creates TensorBoard SQLite tables and indexes.
+///
+/// If they are already created, this has no effect. If schema
+/// migrations are necessary, they will be performed with logging.
+Status SetupTensorboardSqliteDb(Sqlite* db);
+
+}  // namespace db
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_CONTRIB_TENSORBOARD_DB_SCHEMA_H_
diff --git a/tensorflow/contrib/tensorboard/db/schema_test.cc b/tensorflow/contrib/tensorboard/db/schema_test.cc
new file mode 100644
index 0000000000..a4302dda44
--- /dev/null
+++ b/tensorflow/contrib/tensorboard/db/schema_test.cc
@@ -0,0 +1,34 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/contrib/tensorboard/db/schema.h"
+
+#include <memory>
+
+#include "tensorflow/core/lib/core/status_test_util.h"
+#include "tensorflow/core/platform/test.h"
+
+namespace tensorflow {
+namespace db {
+namespace {
+
+TEST(SchemaTest, SmokeTestTensorboardSchema) {
+  std::unique_ptr<Sqlite> db;
+  TF_ASSERT_OK(Sqlite::Open(":memory:", &db));
+  TF_ASSERT_OK(SetupTensorboardSqliteDb(db.get()));
+}
+
+}  // namespace
+}  // namespace db
+}  // namespace tensorflow
-- 
GitLab


From f3c0fc971a663de8e44121b59ee05a6470887592 Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Thu, 26 Oct 2017 18:00:56 -0700
Subject: [PATCH 850/909] Gives Eager Networks unique names, adds related
 errors. - With no name specified, will create a unique name (consistent with
 Layer   names) - With a name specified, it must be unique

Removes containers from Network in favor of the unique name strategy. Does not
add any save/restore functionality yet.

PiperOrigin-RevId: 173618133
---
 tensorflow/contrib/eager/python/network.py    | 239 ++++--
 .../contrib/eager/python/network_test.py      | 681 +++++++++++++++++-
 tensorflow/python/layers/base.py              |  72 +-
 3 files changed, 903 insertions(+), 89 deletions(-)

diff --git a/tensorflow/contrib/eager/python/network.py b/tensorflow/contrib/eager/python/network.py
index 28aed7628e..025d447455 100644
--- a/tensorflow/contrib/eager/python/network.py
+++ b/tensorflow/contrib/eager/python/network.py
@@ -19,15 +19,17 @@ from __future__ import division
 from __future__ import print_function
 
 import collections
-import uuid
-
-import six
+import weakref
 
 from tensorflow.python.estimator import util as estimator_util
-from tensorflow.python.framework import ops
 from tensorflow.python.layers import base
 from tensorflow.python.ops import variable_scope
 
+# pylint: disable=protected-access
+# Explanation for protected-access disable: Network has lots of same-class and
+# parent-class references across different objects, and some to private
+# functions in base.py which should be reused.
+
 
 class Network(base.Layer):
   """Represents the composition of a set of Layers.
@@ -35,12 +37,6 @@ class Network(base.Layer):
   TODO(josh11b,ashankar):
   - Should "trainable" be changeable on the Network object?
   - Do we allow add_variable in Network?
-  - Layer.name and Layer.variables.names are not in sync today
-    d = tf.layers.Dense(1)
-    d(tf.constant([[1.]]))
-    print(d.name)
-    print(d.variables)
-  - Note that name provided to __init__ is only for error messages?
   - Detect layers used in __call__ that weren't registered with track_layer.
   - Convert inputs to __call__ to tensors.
   - Prevent variables from being created after the first __call__?
@@ -49,9 +45,142 @@ class Network(base.Layer):
   """
 
   def __init__(self, name=None):
+    if isinstance(name, variable_scope.VariableScope):
+      raise ValueError("VariableScopes are not valid Network names.")
+    if name is not None and "/" in name:
+      raise ValueError(
+          "Forward slashes ('/') are not allowed in Network names.")
     super(Network, self).__init__(name=name)
-    self._container = uuid.uuid4().hex
-    self._layers = collections.OrderedDict()
+    self._layers = []
+    self._sub_layer_name_uids = collections.defaultdict(int)
+    # Initially None, but set to False for networks which are first built as
+    # top-level.
+    self._first_parent = None  # A weak reference to our first parent.
+    self._non_network_sublayers = []
+    self._owned_layers = {}
+    # The scope to use if we end up without a parent.
+    self._default_parent_variable_scope = variable_scope.get_variable_scope()
+
+  def _init_set_name(self, name):
+    # Anonymous Networks (name=None) defer setting a final name until they are
+    # (1) added to another Network, or (2) built/called (where (2) is only used
+    # for a "top level" network).
+    #
+    # However, if we were provided an explicit name (name is not None), that
+    # will always be the final name of the Network; if it turns out not to be
+    # unique or if variable names can't be prefixed by it we will throw an
+    # error.
+    self._name = name
+    self._base_name = None
+
+  def _finalize_name(self, parent_network):
+    if not self._name:
+      if not parent_network:
+        name_uid_map = base._get_default_graph_uid_map()
+      else:
+        name_uid_map = parent_network._sub_layer_name_uids
+      # Were were not passed a name explicitly (or it was blank), so this is an
+      # anonymous Network. We make up a unique name.
+      if parent_network:
+        avoid_names = parent_network._owned_layers
+      else:
+        avoid_names = None
+      self._name, self._base_name = self._make_unique_name(
+          name_uid_map=name_uid_map, avoid_names=avoid_names)
+    if self._first_parent is None or self._first_parent() is None:
+      # Save a pointer to the parent Network so that we can later check that the
+      # scope name we get is correct.
+      if not parent_network:
+        self._first_parent = parent_network
+      else:
+        self._first_parent = weakref.ref(parent_network)
+
+  def _set_scope(self, scope=None):
+    if self._scope is None:
+      if not self._first_parent:
+        first_parent = self._first_parent
+      else:
+        first_parent = self._first_parent()
+      if first_parent is None:
+        # If we were never added to another Network, or that Network has beed
+        # garbage collected before being called, then we're a top-level Network.
+        self._finalize_name(
+            # Use False to make sure the value sticks and we don't inherit a
+            # parent if we're added to a network later.
+            parent_network=False)
+      if scope is not None:
+        raise ValueError("Networks may not be created with explicit scopes.")
+      if first_parent:
+        first_parent._set_scope()
+        parent_scope = first_parent._scope
+      else:
+        parent_scope = self._default_parent_variable_scope
+      with variable_scope.variable_scope(parent_scope):
+        # Make sure variables with this prefix will be unique.
+        with variable_scope.variable_scope(
+            None, use_resource=True, default_name=self._name) as scope:
+          self._scope = scope
+          scope_name = scope.name
+          suffix_start = scope_name.rfind("/") + 1
+          # rfind is -1 if there is no slash in the string, in which case the
+          # suffix starts at the beginning of the string (there is no prefix).
+          scope_suffix = scope_name[suffix_start:]
+          scope_prefix = scope_name[:suffix_start]
+          if scope_suffix != self._name:
+            raise ValueError(
+                ("A Network named '%s' already exists (or a variable_scope was "
+                 "created with this name). Names must be unique.") % (
+                     self._name,))
+          if (first_parent
+              and scope_prefix[:-1] != first_parent._scope.name):
+            raise ValueError(
+                ("Network variable names must match a nesting of sub-Network "
+                 "names. Expected prefix '%s' from parent network, but got "
+                 "'%s' when attempting to create a variable_scope for Network "
+                 "'%s'. Likely an explicit variable_scope was inserted into "
+                 "the nesting.") % (
+                     first_parent._scope.name,
+                     scope_prefix[:-1],
+                     self._name))
+          elif not first_parent and scope_prefix:
+            # For the case when this Network is not nested inside any other
+            # Network, but is in a variable_scope. This is an error for now.
+            raise ValueError(
+                "Creating Networks inside named variable_scopes is currently "
+                "not supported (to ensure that variable names match the names "
+                "of Networks in which they were first created). To set "
+                "options, try `with tf.variable_scope(''):`. If this "
+                "limitation bothers you, please file a feature request.")
+      for non_network_constituent in self._non_network_sublayers:
+        if non_network_constituent._scope is None:
+          if non_network_constituent._first_parent is None:
+            constituent_first_parent = None
+          else:
+            constituent_first_parent = non_network_constituent._first_parent()
+          if constituent_first_parent:
+            constituent_first_parent._set_scope()
+            parent_scope = constituent_first_parent._scope
+          else:
+            parent_scope = (
+                non_network_constituent._default_parent_variable_scope)
+          with variable_scope.variable_scope(parent_scope):
+            # Horrid hack to make Layer variable names which are direct
+            # sub-layers of Networks conform to the Network variable naming
+            # conventions.
+            with variable_scope.variable_scope(
+                None, use_resource=True,
+                default_name=non_network_constituent.name) as sub_scope:
+              non_network_constituent._scope = sub_scope
+
+  @base.Layer.name.getter
+  def name(self):
+    if self._name is None:
+      raise ValueError(
+          "The network does not yet have a final name, but a name was "
+          "requested for it. Networks get a name when they are added to "
+          "another Network via track_layer, or when they are first "
+          "called/built.")
+    return self._name
 
   def track_layer(self, layer):
     """Track a Layer in this Network.
@@ -76,20 +205,51 @@ class Network(base.Layer):
       raise TypeError(
           "Network.track_layer() passed type %s, not a tf.layers.Layer" %
           (type(layer),))
-    if layer.name in self._layers:
-      if self._layers[layer.name] is layer:
-        return layer
-      raise ValueError(
-          "Attempt to add two Layers with the name '%s' to the same Network "
-          "'%s'" % (layer.name, self.name))
-    self._layers[layer.name] = layer
+    if isinstance(layer, Network):
+      layer._finalize_name(parent_network=self)
+    else:
+      # `layer` is a non-Network, so it hasn't been named to follow Network
+      # conventions for contained Layers (i.e. the same conventions as for
+      # sub-Networks). This renaming is necessary to isolate Network variable
+      # naming from Layers constructed outside the Network and never added to it
+      # (because Layers are named globally).
+      if not layer.built:
+        if not hasattr(layer, "_first_parent"):
+          dereferenced_layer_first_parent = None
+        else:
+          dereferenced_layer_first_parent = layer._first_parent()
+        if dereferenced_layer_first_parent is None:
+          if layer._name != layer._base_name:
+            # If name and base_name do not match, then this Layer used anonymous
+            # naming and we have to rename it. Otherwise there's an explicit
+            # name, and we should respect it (subject to error checking).
+            layer._name, layer._base_name = layer._make_unique_name(
+                name_uid_map=self._sub_layer_name_uids,
+                avoid_names=self._owned_layers)
+          layer._first_parent = weakref.ref(self)
+        self._non_network_sublayers.append(layer)
+    if (not layer.built
+        and layer._first_parent
+        and self is layer._first_parent()):
+      if layer.name in self._owned_layers:
+        if self._owned_layers[layer.name] is layer:
+          return layer
+        raise ValueError(
+            "Attempt to add two Layers with the name '%s' to the same Network."
+            % (layer.name))
+      self._owned_layers[layer.name] = layer
+    self._layers.append(layer)
     return layer
 
   def get_layer(self, name=None, index=None):
     """Get a contained `tf.layers.Layer` either by name or index.
 
     Args:
-      name: String matching one of the names of a contained `Layer`.
+      name: String matching one of the names of a contained `Layer`. Note that
+        the names of `Layer`s added to `Network`s may not be unique when doing
+        layer sharing (i.e. adding a `Layer` to this `Network` which was already
+        added to another `Network`). The lowest index `Layer` with a matching
+        name will be returned.
       index: Integer in [0, number of layers). Layers are assigned an index
         by the order they are added.
 
@@ -97,19 +257,25 @@ class Network(base.Layer):
       A `tf.layers.Layer` object.
 
     Raises:
-      ValueError: If neither or both of 'index' or 'name' is specified.
+      ValueError: If neither or both of 'index' or 'name' is specified, or the
+        lookup failed.
     """
     if index is not None:
       if name is not None:
         raise ValueError("Exactly one of 'index' or 'name' must be provided")
       if len(self._layers) <= index:
-        raise ValueError("Was asked to retrieve layer at index " +
-                         str(index) + " but model only has " + str(
-                             len(self._layers)) + " layers.")
-      return list(self._layers.values())[index]
-    if name is None:
-      raise ValueError("Exactly one of 'index' or 'name' must be provided")
-    return self._layers[index]
+        raise ValueError("Was asked to retrieve layer at index " + str(index) +
+                         " but model only has " + str(len(self._layers)) +
+                         " layers.")
+      else:
+        return self._layers[index]
+    else:
+      if not name:
+        raise ValueError("Provide either a layer name or layer index.")
+    for layer in self._layers:
+      if layer.name == name:
+        return layer
+    raise ValueError("No such layer: " + name)
 
   # The following methods are for implementing the Layer interface.
 
@@ -119,21 +285,21 @@ class Network(base.Layer):
     # variables in the case of shared layers/variables that appear in
     # multiple places in the Network?
     weights = []
-    for layer in six.itervalues(self._layers):
+    for layer in self._layers:
       weights += layer.weights
     return weights
 
   @property
   def trainable_weights(self):
     weights = []
-    for layer in six.itervalues(self._layers):
+    for layer in self._layers:
       weights += layer.trainable_weights
     return weights
 
   @property
   def non_trainable_weights(self):
     weights = []
-    for layer in six.itervalues(self._layers):
+    for layer in self._layers:
       weights += layer.non_trainable_weights
     return weights
 
@@ -152,7 +318,7 @@ class Network(base.Layer):
 
   @property
   def layers(self):
-    return self._layers.values()
+    return self._layers
 
   def add_variable(self, name, shape, dtype=None, initializer=None,
                    regularizer=None, trainable=True, constraint=None):
@@ -161,15 +327,6 @@ class Network(base.Layer):
         "at https://github.com/tensorflow/tensorflow/issues/new if this is "
         "important to you")
 
-  def __call__(self, inputs, *args, **kwargs):
-    # TODO(josh11b,ashankar,agarwal): Can we reduce the number of context
-    # managers here and/or move some of the work into the constructor
-    # for performance reasons?
-    with ops.container(self._container):
-      with variable_scope.variable_scope(variable_scope.get_variable_scope(),
-                                         use_resource=True):
-        return super(Network, self).__call__(inputs, *args, **kwargs)
-
   # TODO(josh11b): Support other Layer methods needed for graph mode, such as for
   # losses and updates
 
diff --git a/tensorflow/contrib/eager/python/network_test.py b/tensorflow/contrib/eager/python/network_test.py
index 94cb73ae72..e4cba3f2ed 100644
--- a/tensorflow/contrib/eager/python/network_test.py
+++ b/tensorflow/contrib/eager/python/network_test.py
@@ -16,19 +16,24 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import gc
+
 from tensorflow.contrib.eager.python import network
 from tensorflow.python.eager import test
 from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import test_util
 from tensorflow.python.layers import core
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import nn_ops
+from tensorflow.python.ops import resource_variable_ops
+from tensorflow.python.ops import variable_scope
 
 
 # pylint: disable=not-callable
 class MyNetwork(network.Network):
 
-  def __init__(self):
-    super(MyNetwork, self).__init__(name="abcd")
+  def __init__(self, name=None):
+    super(MyNetwork, self).__init__(name=name)
     self.l1 = self.track_layer(core.Dense(1, use_bias=False))
 
   def call(self, x):
@@ -37,6 +42,7 @@ class MyNetwork(network.Network):
 
 class NetworkTest(test.TestCase):
 
+  @test_util.run_in_graph_and_eager_modes()
   def testTrainableAttribute(self):
     net = network.Network()
     self.assertTrue(net.trainable)
@@ -44,41 +50,676 @@ class NetworkTest(test.TestCase):
       net.trainable = False
     self.assertTrue(net.trainable)
 
+  @test_util.run_in_graph_and_eager_modes()
   def testNetworkCall(self):
-    net = MyNetwork()
+    net = MyNetwork(name="abcd")
     net(constant_op.constant([[2.0]]))  # Force variables to be created.
     self.assertEqual(1, len(net.trainable_variables))
-    net.trainable_variables[0].assign([[17.0]])
+    self.evaluate(net.trainable_variables[0].assign([[17.0]]))
     # TODO(josh11b): Support passing Python values to networks.
     result = net(constant_op.constant([[2.0]]))
-    self.assertEqual(34.0, result.numpy())
+    self.assertEqual(34.0, self.evaluate(result))
+
+  def testNoReferenceCyclesAfterCall(self):
+
+    class ChildNetwork(network.Network):
+
+      def __init__(self, name=None):
+        super(ChildNetwork, self).__init__(name=name)
+
+      def call(self, x):
+        return x * 2.
+
+    class ParentNetwork(network.Network):
+
+      def __init__(self, name=None):
+        super(ParentNetwork, self).__init__(name=name)
+        self.l1 = self.track_layer(ChildNetwork())
+
+      def call(self, x):
+        return self.l1(x)
+
+    one = constant_op.constant([[1.0]])
+    gc.disable()
+    gc.collect()
+    previous_gc_debug_flags = gc.get_debug()
+    gc.set_debug(gc.DEBUG_SAVEALL)
+    preexisting = len(gc.garbage)
+    net = ParentNetwork()
+    net(one)
+    del net
+    gc.collect()
+    # There should be no additional garbage requiring collection.
+    self.assertEqual(preexisting, len(gc.garbage))
+    gc.set_debug(previous_gc_debug_flags)
+    gc.enable()
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testAnonymousNoNameInitially(self):
+    net = MyNetwork()
+    with self.assertRaisesRegexp(ValueError, "does not yet have a final name"):
+      net.name  # pylint: disable=pointless-statement
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testExplicitHasNameInitially(self):
+    net = MyNetwork(name="abcd")
+    self.assertEqual("abcd", net.name)
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testUsingResourceVariables(self):
+    net = MyNetwork()
+    net(constant_op.constant([[0.]]))
+    self.assertIsInstance(net.trainable_weights[0],
+                          resource_variable_ops.ResourceVariable)
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testDuplicateNameError(self):
+    one = constant_op.constant([[1.]])
+    net = MyNetwork(name="foo")
+    net(one)
+    with self.assertRaisesRegexp(
+        ValueError, "named 'foo' already exists"):
+      net1 = MyNetwork(name="foo")
+      net1(one)
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testWrappingInVariableScope(self):
+    with variable_scope.variable_scope("outside_scope"):
+      net = MyNetwork()
+      one = constant_op.constant([[1.]])
+      with self.assertRaisesRegexp(
+          ValueError,
+          ("Creating Networks inside named variable_scopes is currently not "
+           "supported")):
+        net(one)
+      # Alternatively, we could re-name the Network to match the variable_scope:
+      # self.assertEqual("outside_scope/my_network_1", net.name)
+      # self.assertStartsWith(
+      #     expected_start="outside_scope/my_network_1/dense/",
+      #     actual=net.trainable_weights[0].name)
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testLayerNamesRespected(self):
+    class ParentNetwork(network.Network):
+
+      def __init__(self):
+        super(ParentNetwork, self).__init__()
+        self.first = self.track_layer(
+            core.Dense(1, use_bias=False, name="explicit_name"))
+
+      def call(self, x):
+        return self.first(x)
+
+    one = constant_op.constant([[1.]])
+    net = ParentNetwork()
+    net(one)
+    self.assertStartsWith(expected_start="parent_network_1/explicit_name/",
+                          actual=net.trainable_weights[0].name)
+    self.assertEqual("explicit_name", net.first.name)
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testWrappingInAnonymousVariableScope(self):
+    # Named outside variable_scopes are not supported at the moment. However,
+    # blank-named top level variable scopes do not change variable names, and so
+    # can be used to set the properties of Network variables.
+    was_called = [False]
+    def _custom_getter(getter, *args, **kwargs):
+      was_called[0] = True
+      return getter(*args, **kwargs)
+    with variable_scope.variable_scope("", custom_getter=_custom_getter):
+      net = MyNetwork()
+      one = constant_op.constant([[1.]])
+      net(one)
+    self.assertTrue(was_called[0])
 
-  def testNetworkAsAGraph(self):
-    self.skipTest("TODO(ashankar,josh11b): FIX THIS")
-    # Verify that we're using ResourceVariables
+  @test_util.run_in_graph_and_eager_modes()
+  def testReasonableSlashError(self):
+    with self.assertRaisesRegexp(
+        ValueError, "not allowed in Network names"):
+      MyNetwork(name="slash/slash")
 
+  @test_util.run_in_graph_and_eager_modes()
+  def testNoVariableScopeNames(self):
+    with self.assertRaisesRegexp(
+        ValueError, "VariableScopes are not valid Network names"):
+      with variable_scope.variable_scope("some_scope") as vs:
+        MyNetwork(name=vs)
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testVariableScopeNameCollision(self):
+    with variable_scope.variable_scope("abcd"):
+      pass
+    with self.assertRaisesRegexp(
+        ValueError, "or a variable_scope was created with this name"):
+      net = MyNetwork(name="abcd")
+      one = constant_op.constant([[1.]])
+      net(one)
+
+  @test_util.run_in_graph_and_eager_modes()
   def testNetworkVariablesDoNotInterfere(self):
-    self.skipTest("TODO: FIX THIS")
+    core.Dense(1, use_bias=True)  # Should not interfere with naming.
     net1 = MyNetwork()
     net2 = MyNetwork()
+    one = constant_op.constant([[1.]])
+    net1(one)
+    net2(one)
+    # Layer names typically are globally unique rather than being unique within
+    # the scope of their first use. However, within a Network they must be named
+    # locally so that previous Layer consutrciton does not interfere with
+    # variable naming (e.g. add a Layer construction before the Network,
+    # suddenly your previously saved checkpoint is incompatible).
+    self.assertEqual("dense_1", net1.l1.name)
+    self.assertEqual("dense_1", net2.l1.name)
+    self.evaluate(net1.trainable_weights[0].assign([[1.]]))
+    self.evaluate(net2.trainable_weights[0].assign([[2.]]))
+    self.assertEqual(2., self.evaluate(net2.trainable_weights[0]))
+    self.assertEqual(1., self.evaluate(net1.trainable_weights[0]))
+    self.assertStartsWith(expected_start="my_network_1/dense_1/",
+                          actual=net1.trainable_weights[0].name)
+    self.assertStartsWith(expected_start="my_network_2/dense_1/",
+                          actual=net2.trainable_weights[0].name)
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testNestableAnonymous(self):
+
+    # The case where no explicit names are specified. We make up unique names,
+    # and these should match the variable names.
+    class ParentNetwork(network.Network):
+
+      def __init__(self):
+        super(ParentNetwork, self).__init__()
+        self.first = self.track_layer(MyNetwork())
+        self.second = self.track_layer(MyNetwork())
+
+      def call(self, x):
+        return self.second(self.first(x))
 
     one = constant_op.constant([[1.]])
+    net = ParentNetwork()
+    net(one)
+    self.assertStartsWith(expected_start="parent_network_1/my_network_1/dense",
+                          actual=net.trainable_weights[0].name)
+    self.assertStartsWith(expected_start="parent_network_1/my_network_1/dense",
+                          actual=net.first.trainable_weights[0].name)
+    self.assertStartsWith(expected_start="parent_network_1/my_network_2/dense",
+                          actual=net.trainable_weights[1].name)
+    self.assertStartsWith(expected_start="parent_network_1/my_network_2/dense",
+                          actual=net.second.trainable_weights[0].name)
+    self.assertEqual("parent_network_1", net.name)
+    self.assertEqual("my_network_1", net.first.name)
+    self.assertEqual("my_network_2", net.second.name)
 
-    print(type(net1(one)))
+    net2 = ParentNetwork()
     net2(one)
+    self.assertStartsWith(expected_start="parent_network_2/my_network_1/dense",
+                          actual=net2.trainable_weights[0].name)
+    self.assertStartsWith(expected_start="parent_network_2/my_network_1/dense",
+                          actual=net2.first.trainable_weights[0].name)
+    self.assertStartsWith(expected_start="parent_network_2/my_network_2/dense",
+                          actual=net2.trainable_weights[1].name)
+    self.assertStartsWith(expected_start="parent_network_2/my_network_2/dense",
+                          actual=net2.second.trainable_weights[0].name)
+    self.assertEqual("parent_network_2", net2.name)
+    self.assertEqual("my_network_1", net2.first.name)
+    self.assertEqual("my_network_2", net2.second.name)
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testNestableExplicit(self):
+
+    # We have explicit network names and everything is globally unique.
+    class ParentNetwork(network.Network):
+
+      def __init__(self):
+        super(ParentNetwork, self).__init__(name="unique_parent_name")
+        self.first = self.track_layer(
+            MyNetwork(name="first_unique_child_name"))
+        self.second = self.track_layer(
+            MyNetwork(name="second_unique_child_name"))
+
+      def call(self, x):
+        return self.second(self.first(x))
+
+    one = constant_op.constant([[1.]])
+    net = ParentNetwork()
+    net(one)
+    self.assertStartsWith(
+        expected_start="unique_parent_name/first_unique_child_name/dense",
+        actual=net.trainable_weights[0].name)
+    self.assertStartsWith(
+        expected_start="unique_parent_name/second_unique_child_name/dense",
+        actual=net.trainable_weights[1].name)
+    self.assertEqual("unique_parent_name", net.name)
+    self.assertEqual("first_unique_child_name", net.first.name)
+    self.assertEqual("second_unique_child_name", net.second.name)
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testLayerNetworkNameInteractions(self):
+
+    # Same base name as core.Dense; Networks and non-Network Layers with the
+    # same base name should use the same numbering system.
+    class Dense(network.Network):
+
+      def __init__(self):
+        super(Dense, self).__init__()
+        self.first = self.track_layer(core.Dense(1, use_bias=False))
+
+      def call(self, x):
+        return self.first(x)
+
+    class MixedLayerNetwork(network.Network):
+
+      def __init__(self):
+        super(MixedLayerNetwork, self).__init__()
+        self.first = self.track_layer(core.Dense(1, use_bias=False))
+        self.second = self.track_layer(core.Dense(1, use_bias=False))
+        self.third = self.track_layer(Dense())
+        self.fourth = self.track_layer(core.Dense(1, use_bias=False))
+        self.fifth = self.track_layer(core.Dense(1, use_bias=False))
+
+      def call(self, x):
+        return self.fifth(self.fourth(self.third(self.second(self.first(x)))))
+
+    one = constant_op.constant([[1.]])
+    net = MixedLayerNetwork()
+    net(one)
+    self.assertEqual("dense_1", net.first.name)
+    self.assertEqual("dense_2", net.second.name)
+    self.assertEqual("dense_3", net.third.name)
+    self.assertEqual("dense_4", net.fourth.name)
+    self.assertEqual("dense_5", net.fifth.name)
+    # Note that this is _not_ the default naming behavior for Layers. Layers
+    # which are added to Networks follow Network variable naming conventions
+    # (i.e. variable names = network name unless variable sharing). Nested
+    # Layers revert to Layer behavior.
+    self.assertStartsWith(expected_start="mixed_layer_network_1/dense_1/",
+                          actual=net.trainable_weights[0].name)
+    self.assertStartsWith(expected_start="mixed_layer_network_1/dense_2/",
+                          actual=net.trainable_weights[1].name)
+    self.assertStartsWith(expected_start="mixed_layer_network_1/dense_3/",
+                          actual=net.trainable_weights[2].name)
+    self.assertStartsWith(expected_start="mixed_layer_network_1/dense_4/",
+                          actual=net.trainable_weights[3].name)
+    self.assertStartsWith(expected_start="mixed_layer_network_1/dense_5/",
+                          actual=net.trainable_weights[4].name)
+    self.assertEqual("mixed_layer_network_1", net.name)
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testNestableExplicitCollisions(self):
+
+    # We have explicit network names and they are unique within the layer
+    # they're added to.
+    class ParentNetwork(network.Network):
+
+      def __init__(self):
+        super(ParentNetwork, self).__init__(name="nonunique_name")
+        self.first = self.track_layer(
+            MyNetwork(name="nonunique_name"))
+        self.second = self.track_layer(
+            MyNetwork(name="second_unique_child_name"))
+
+      def call(self, x):
+        return self.second(self.first(x))
+
+    one = constant_op.constant([[1.]])
+    net = ParentNetwork()
+    net(one)
+    self.assertStartsWith(
+        expected_start="nonunique_name/nonunique_name/dense",
+        actual=net.trainable_weights[0].name)
+    self.assertStartsWith(
+        expected_start="nonunique_name/second_unique_child_name/dense",
+        actual=net.trainable_weights[1].name)
+    self.assertEqual("nonunique_name", net.name)
+    self.assertEqual("nonunique_name", net.first.name)
+    self.assertEqual("second_unique_child_name", net.second.name)
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testNestableExplicitWithAnonymousParent(self):
+
+    # A parent network is instantiated multiple times with explicitly named
+    # children. We shouldn't throw any name errors.
+    class ParentNetwork(network.Network):
+
+      def __init__(self):
+        super(ParentNetwork, self).__init__()
+        self.first = self.track_layer(
+            MyNetwork(name="first_unique_child_name"))
+        self.second = self.track_layer(
+            MyNetwork(name="second_unique_child_name"))
+
+      def call(self, x):
+        return self.second(self.first(x))
+
+    one = constant_op.constant([[1.]])
+    net = ParentNetwork()
+    net(one)
+    self.assertStartsWith(
+        expected_start="parent_network_1/first_unique_child_name/dense_1/",
+        actual=net.trainable_weights[0].name)
+    self.assertStartsWith(
+        expected_start="parent_network_1/second_unique_child_name/dense_1/",
+        actual=net.trainable_weights[1].name)
+    self.assertEqual("parent_network_1", net.name)
+    self.assertEqual("first_unique_child_name", net.first.name)
+    self.assertEqual("second_unique_child_name", net.second.name)
+
+    net2 = ParentNetwork()
+    net2(one)
+    self.assertStartsWith(
+        expected_start="parent_network_2/first_unique_child_name/dense",
+        actual=net2.trainable_weights[0].name)
+    self.assertStartsWith(
+        expected_start="parent_network_2/second_unique_child_name/dense",
+        actual=net2.trainable_weights[1].name)
+    self.assertEqual("parent_network_2", net2.name)
+    self.assertEqual("first_unique_child_name", net2.first.name)
+    self.assertEqual("second_unique_child_name", net2.second.name)
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testNestableExplicitSameLayerCollisions(self):
+
+    # We have explicit network names and they are _not_ unique within the layer
+    # they're added to. Error.
+    class ParentNetwork(network.Network):
+
+      def __init__(self):
+        super(ParentNetwork, self).__init__(name="unique_parent_name")
+        self.first = self.track_layer(MyNetwork(name="nonunique_name"))
+        self.second = self.track_layer(MyNetwork(name="nonunique_name"))
+
+      def call(self, x):
+        return self.second(self.first(x))
+
+    with self.assertRaisesRegexp(ValueError, "nonunique_name"):
+      ParentNetwork()
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testAnonymousVariableSharing(self):
+
+    # Two "owned" Networks
+    class FirstParentNetwork(network.Network):
+
+      def __init__(self):
+        super(FirstParentNetwork, self).__init__()
+        self.first = self.track_layer(MyNetwork())
+        self.second = self.track_layer(MyNetwork())
+
+      def call(self, x):
+        return self.second(self.first(x))
+
+    one = constant_op.constant([[1.]])
+    net = FirstParentNetwork()
+    net(one)
+
+    # One Network shared with FirstParentNetwork, one owned Network. Same name,
+    # but this is OK because only one is owned. This name collision is
+    # avoidable; we could have looked at the base_name of the non-owned Network
+    # and incremented our naming based on that.
+    class SecondParentNetwork(network.Network):
+
+      def __init__(self):
+        super(SecondParentNetwork, self).__init__()
+        self.first = self.track_layer(net.first)
+        self.second = self.track_layer(MyNetwork())
+
+      def call(self, x):
+        return self.second(self.first(x))
+
+    net2 = SecondParentNetwork()
+    net2(one)
+
+    self.assertStartsWith(
+        expected_start="first_parent_network_1/my_network_1/dense_1/",
+        actual=net2.trainable_weights[0].name)
+    self.assertStartsWith(
+        expected_start="second_parent_network_1/my_network_1/dense_1/",
+        actual=net2.trainable_weights[1].name)
+    self.assertEqual("second_parent_network_1", net2.name)
+    self.assertTrue(net2.first is net.first)
+    self.assertEqual("my_network_1", net2.first.name)
+    self.assertEqual("my_network_1", net2.second.name)
+
+    # No name collision; the owned Network is added first and has a different
+    # name than the shared Network.
+    class ThirdParentNetwork(network.Network):
+
+      def __init__(self):
+        super(ThirdParentNetwork, self).__init__()
+        self.first = self.track_layer(MyNetwork())
+        self.second = self.track_layer(net.second)
+
+      def call(self, x):
+        return self.second(self.first(x))
+
+    net3 = ThirdParentNetwork()
+    net3(one)
+
+    self.assertStartsWith(
+        expected_start="third_parent_network_1/my_network_1/dense",
+        actual=net3.trainable_weights[0].name)
+    self.assertStartsWith(
+        expected_start="first_parent_network_1/my_network_2/dense",
+        actual=net3.trainable_weights[1].name)
+    self.assertEqual("third_parent_network_1", net3.name)
+    self.assertTrue(net3.second is net.second)
+    self.assertEqual("my_network_1", net3.first.name)
+    self.assertEqual("my_network_2", net3.second.name)
+
+    # "Unavoidable" same-name Layer. The owned name is added first (fixed), then
+    # a shared Network is added with the same name.
+    class FourthParentNetwork(network.Network):
+
+      def __init__(self):
+        super(FourthParentNetwork, self).__init__()
+        self.first = self.track_layer(MyNetwork())
+        self.second = self.track_layer(net.first)
+
+      def call(self, x):
+        return self.second(self.first(x))
+
+    net4 = FourthParentNetwork()
+    net4(one)
+
+    self.assertStartsWith(
+        expected_start="fourth_parent_network_1/my_network_1/dense_1/",
+        actual=net4.trainable_weights[0].name)
+    self.assertStartsWith(
+        expected_start="first_parent_network_1/my_network_1/dense_1/",
+        actual=net4.trainable_weights[1].name)
+    self.assertEqual("fourth_parent_network_1", net4.name)
+    self.assertTrue(net4.second is net.first)
+    self.assertEqual("my_network_1", net4.first.name)
+    self.assertEqual("my_network_1", net4.second.name)
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testRecursiveLayerRenaming(self):
+    core.Dense(1)  # Under default Layer naming, would change subsequent names.
+
+    class NetworkWithLayerChildren(network.Network):
+
+      def __init__(self):
+        super(NetworkWithLayerChildren, self).__init__()
+        self.first = self.track_layer(core.Dense(1, use_bias=False))
+        self.second = self.track_layer(core.Dense(1, use_bias=False))
+
+      def call(self, x):
+        return self.second(self.first(x))
+
+    class ParentNetwork(network.Network):
+
+      def __init__(self):
+        super(ParentNetwork, self).__init__()
+        self.first = self.track_layer(NetworkWithLayerChildren())
+        self.second = self.track_layer(NetworkWithLayerChildren())
+
+      def call(self, x):
+        return self.second(self.first(x))
+
+    net = ParentNetwork()
+    one = constant_op.constant([[1.]])
+    net(one)
+
+    self.assertStartsWith(
+        expected_start=("parent_network_1/network_with_layer_children_1/"
+                        "dense_1/"),
+        actual=net.trainable_weights[0].name)
+    self.assertStartsWith(
+        expected_start=("parent_network_1/network_with_layer_children_1/"
+                        "dense_2/"),
+        actual=net.trainable_weights[1].name)
+    self.assertStartsWith(
+        expected_start=("parent_network_1/network_with_layer_children_2/"
+                        "dense_1/"),
+        actual=net.trainable_weights[2].name)
+    self.assertStartsWith(
+        expected_start=("parent_network_1/network_with_layer_children_2/"
+                        "dense_2/"),
+        actual=net.trainable_weights[3].name)
+    self.assertEqual("parent_network_1", net.name)
+    self.assertEqual("network_with_layer_children_1", net.first.name)
+    self.assertEqual("network_with_layer_children_2", net.second.name)
+    self.assertEqual("dense_1", net.first.first.name)
+    self.assertEqual("dense_2", net.first.second.name)
+    self.assertEqual("dense_1", net.second.first.name)
+    self.assertEqual("dense_2", net.second.second.name)
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testCallInDifferentOrderThanConstruct(self):
+    shared_network = MyNetwork()
+
+    class FirstNetwork(network.Network):
+
+      def __init__(self):
+        super(FirstNetwork, self).__init__()
+        self.first = self.track_layer(shared_network)
+        self.second = self.track_layer(MyNetwork())
+
+      def call(self, x):
+        return self.second(self.first(x))
+
+    class SecondNetwork(network.Network):
+
+      def __init__(self):
+        super(SecondNetwork, self).__init__()
+        self.first = self.track_layer(shared_network)
+        self.second = self.track_layer(MyNetwork())
+
+      def call(self, x):
+        return self.second(self.first(x))
+
+    net1 = FirstNetwork()
+    net2 = SecondNetwork()
+
+    one = constant_op.constant([[1.]])
+    net2(one)
+    net1(one)
+
+    self.assertStartsWith(
+        expected_start="first_network_1/my_network_1/dense_1/",
+        actual=net1.trainable_weights[0].name)
+    self.assertStartsWith(
+        expected_start="first_network_1/my_network_2/dense_1/",
+        actual=net1.trainable_weights[1].name)
+    self.assertStartsWith(
+        expected_start="first_network_1/my_network_1/dense_1/",
+        actual=net2.trainable_weights[0].name)
+    self.assertStartsWith(
+        expected_start="second_network_1/my_network_1/dense_1/",
+        actual=net2.trainable_weights[1].name)
+    self.assertTrue(net1.trainable_weights[0] is net2.trainable_weights[0])
+    self.assertEqual("first_network_1", net1.name)
+    self.assertEqual("my_network_1", net1.first.name)
+    self.assertEqual("my_network_2", net1.second.name)
+    self.assertTrue(net2.first is net1.first)
+    self.assertEqual("my_network_1", net2.second.name)
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testLayerCallInDifferentOrderThanConstruct(self):
+    # Same idea as testCallInDifferentOrderThanConstruct, but this time with a
+    # non-Network Layer shared between two Networks rather than a
+    # Network. Naming should follow the same rules.
+    shared_layer = core.Dense(1, use_bias=False)
+
+    class FirstNetwork(network.Network):
+
+      def __init__(self):
+        super(FirstNetwork, self).__init__()
+        self.first = self.track_layer(shared_layer)
+        self.second = self.track_layer(core.Dense(1, use_bias=False))
+
+      def call(self, x):
+        return self.second(self.first(x))
+
+    class SecondNetwork(network.Network):
+
+      def __init__(self):
+        super(SecondNetwork, self).__init__()
+        self.first = self.track_layer(shared_layer)
+        self.second = self.track_layer(core.Dense(1, use_bias=False))
+
+      def call(self, x):
+        return self.second(self.first(x))
+
+    net1 = FirstNetwork()
+    net2 = SecondNetwork()
+
+    one = constant_op.constant([[1.]])
+    net2(one)
+    net1(one)
+
+    self.assertStartsWith(
+        expected_start="first_network_1/dense_1/",
+        actual=net1.trainable_weights[0].name)
+    self.assertStartsWith(
+        expected_start="first_network_1/dense_2/",
+        actual=net1.trainable_weights[1].name)
+    self.assertStartsWith(
+        expected_start="first_network_1/dense_1/",
+        actual=net2.trainable_weights[0].name)
+    self.assertStartsWith(
+        expected_start="second_network_1/dense_1/",
+        actual=net2.trainable_weights[1].name)
+    self.assertTrue(net1.trainable_weights[0] is net2.trainable_weights[0])
+    self.assertEqual("first_network_1", net1.name)
+    self.assertEqual("dense_1", net1.first.name)
+    self.assertEqual("dense_2", net1.second.name)
+    self.assertTrue(net2.first is net1.first)
+    self.assertEqual("dense_1", net2.second.name)
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testLayerAlreadyBuilt(self):
+    one = constant_op.constant([[1.]])
+    core.Dense(1, use_bias=False)  # pre-built layers use global naming
+    one = constant_op.constant([[1.]])
+    core.Dense(1, use_bias=False)(one)
+    shared_layer = core.Dense(1, use_bias=False)
+    shared_layer(one)
+
+    class FirstNetwork(network.Network):
+
+      def __init__(self):
+        super(FirstNetwork, self).__init__()
+        self.first = self.track_layer(shared_layer)
+        self.second = self.track_layer(core.Dense(1, use_bias=False))
 
-    net1.trainable_weights[0].assign(constant_op.constant([[1.]]))
-    net2.trainable_weights[0].assign(constant_op.constant([[2.]]))
+      def call(self, x):
+        return self.second(self.first(x))
 
-    print("NET1")
-    print(net1.name)
-    print(net1.variables)
-    print(net1(one))
+    net = FirstNetwork()
+    net(one)
 
-    print("NET2")
-    print(net2.name)
-    print(net2.variables)
-    print(net2(one))
+    self.assertStartsWith(
+        expected_start="dense_1/",  # Pre-built layers have variable names which
+                                    # do not match their layer names.
+        actual=net.trainable_weights[0].name)
+    self.assertStartsWith(
+        expected_start="first_network_1/dense_1/",
+        actual=net.trainable_weights[1].name)
+    self.assertTrue(
+        net.trainable_weights[0] is shared_layer.trainable_weights[0])
+    self.assertEqual("first_network_1", net.name)
+    self.assertEqual("dense_3", net.first.name)
+    self.assertEqual("dense_1", net.second.name)
 
 
 class SequentialTest(test.TestCase):
diff --git a/tensorflow/python/layers/base.py b/tensorflow/python/layers/base.py
index 134d4fc8e2..8c2ee1f103 100644
--- a/tensorflow/python/layers/base.py
+++ b/tensorflow/python/layers/base.py
@@ -122,16 +122,7 @@ class Layer(object):
     self._inbound_nodes = []
     self._outbound_nodes = []
 
-    # Determine layer name (non-unique).
-    if isinstance(name, vs.VariableScope):
-      base_name = name.name
-    else:
-      base_name = name
-      self._name = name
-    if not name:
-      base_name = _to_snake_case(self.__class__.__name__)
-      self._name = _unique_layer_name(base_name)
-    self._base_name = base_name
+    self._init_set_name(name)
 
     # Determine variable scope.
     scope = kwargs.get('_scope')
@@ -147,6 +138,17 @@ class Layer(object):
       batch_size = kwargs.get('batch_size')
       self._batch_input_shape = (batch_size,) + tuple(kwargs['input_shape'])
 
+  def _init_set_name(self, name):
+    # Determine layer name (non-unique).
+    if isinstance(name, vs.VariableScope):
+      base_name = name.name
+    else:
+      base_name = name
+      self._name = name
+    if not name:
+      self._name, base_name = self._make_unique_name()
+    self._base_name = base_name
+
   @property
   def dtype(self):
     return self._dtype
@@ -399,6 +401,12 @@ class Layer(object):
     """
     return input_shape
 
+  def _make_unique_name(self, name_uid_map=None, avoid_names=None):
+    base_name = _to_snake_case(self.__class__.__name__)
+    name = _unique_layer_name(base_name, name_uid_map=name_uid_map,
+                              avoid_names=avoid_names)
+    return (name, base_name)
+
   def _set_scope(self, scope=None):
     if self._scope is None:
       # If constructed with _scope=None, lazy setting of scope.
@@ -1507,19 +1515,11 @@ class Network(Layer):
       # TODO(fchollet): check that all inputs and outputs are DeferredTensors.
       pass
 
-    # Set layer name and scope
-    if isinstance(name, vs.VariableScope):
-      base_name = name.name
-    else:
-      base_name = name
-      self._name = name
-    if not name:
-      base_name = _to_snake_case(self.__class__.__name__)
-      self._name = _unique_layer_name(base_name)
+    self._init_set_name(name)
     self._activity_regularizer = None
-    with vs.variable_scope(None, default_name=base_name) as captured_scope:
+    with vs.variable_scope(
+        None, default_name=self._base_name) as captured_scope:
       self._scope = captured_scope
-    self._base_name = base_name
     call_fn_args = estimator_util.fn_args(self.call)
     self._compute_previous_mask = ('mask' in call_fn_args or
                                    hasattr(self, 'compute_mask'))
@@ -2354,11 +2354,24 @@ def _collect_previous_mask(input_tensors):
 PER_GRAPH_LAYER_NAME_UIDS = weakref.WeakKeyDictionary()
 
 
-def _unique_layer_name(name):
+def _get_default_graph_uid_map():
+  graph = ops.get_default_graph()
+  name_uid_map = PER_GRAPH_LAYER_NAME_UIDS.get(graph, None)
+  if name_uid_map is None:
+    name_uid_map = collections.defaultdict(int)
+    PER_GRAPH_LAYER_NAME_UIDS[graph] = name_uid_map
+  return name_uid_map
+
+
+def _unique_layer_name(name, name_uid_map=None, avoid_names=None):
   """Makes a layer name (or arbitrary string) unique within a TensorFlow graph.
 
   Arguments:
     name: String name to make unique.
+    name_uid_map: An optional defaultdict(int) to use when creating unique
+      names. If None (default), uses a per-Graph dictionary.
+    avoid_names: An optional set or dict with names which should not be used. If
+      None (default) does not avoid any names.
 
   Returns:
     Unique string name.
@@ -2370,9 +2383,12 @@ def _unique_layer_name(name):
   _unique_layer_name('dense')  # dense_2
   ```
   """
-  graph = ops.get_default_graph()
-  if graph not in PER_GRAPH_LAYER_NAME_UIDS:
-    PER_GRAPH_LAYER_NAME_UIDS[graph] = collections.defaultdict(int)
-  layer_name_uids = PER_GRAPH_LAYER_NAME_UIDS[graph]
-  layer_name_uids[name] += 1
-  return name + '_' + str(layer_name_uids[name])
+  if name_uid_map is None:
+    name_uid_map = _get_default_graph_uid_map()
+  if avoid_names is None:
+    avoid_names = set()
+  proposed_name = None
+  while proposed_name is None or proposed_name in avoid_names:
+    name_uid_map[name] += 1
+    proposed_name = name + '_' + str(name_uid_map[name])
+  return proposed_name
-- 
GitLab


From abebb5f3fa6799e4fc1f2de1156a7c968c8473b8 Mon Sep 17 00:00:00 2001
From: Shanqing Cai <cais@google.com>
Date: Thu, 26 Oct 2017 18:26:28 -0700
Subject: [PATCH 851/909] TFE: Add compatibility doc string to Saver and
 related functions

Also change `ValueError`s to `RuntimeError`s to be consistent with other errors of this kind.

PiperOrigin-RevId: 173620243
---
 tensorflow/python/training/input.py     |  8 ++++-
 tensorflow/python/training/optimizer.py | 12 +++++---
 tensorflow/python/training/saver.py     | 41 +++++++++++++++++++------
 3 files changed, 46 insertions(+), 15 deletions(-)

diff --git a/tensorflow/python/training/input.py b/tensorflow/python/training/input.py
index b999dbedb6..e7adbf11b4 100644
--- a/tensorflow/python/training/input.py
+++ b/tensorflow/python/training/input.py
@@ -146,9 +146,15 @@ def input_producer(input_tensor,
 
   Raises:
     ValueError: If the shape of the input cannot be inferred from the arguments.
+    RuntimeError: If called with eager execution enabled.
+
+  @compatibility(eager)
+  Queue-using input pipelines are not supported when eager execution is enabled.
+  Please use tf.data to ingest data into your model instead.
+  @end_compatibility
   """
   if context.in_eager_mode():
-    raise ValueError(
+    raise RuntimeError(
         "Queue-using input pipelines are not supported when eager execution is"
         " enabled. Please use tf.data to ingest data into your model instead.")
   with ops.name_scope(name, "input_producer", [input_tensor]):
diff --git a/tensorflow/python/training/optimizer.py b/tensorflow/python/training/optimizer.py
index d6ca52cd1b..915214dbfa 100644
--- a/tensorflow/python/training/optimizer.py
+++ b/tensorflow/python/training/optimizer.py
@@ -397,6 +397,8 @@ class Optimizer(object):
     Raises:
       TypeError: If `var_list` contains anything else than `Variable` objects.
       ValueError: If some arguments are invalid.
+      RuntimeError: If called with eager execution enabled and if `grad_loss`
+        is not `None` or `loss` is not callable.
 
     @compatibility(eager)
     When eager execution is enabled, `loss` should be a Python function that
@@ -411,11 +413,13 @@ class Optimizer(object):
     """
     if context.in_eager_mode():
       if grad_loss is not None:
-        raise ValueError("`grad_loss` argument to Optimizer.compute_gradients "
-                         "not supported when eager execution is enabled.")
+        raise RuntimeError(
+            "`grad_loss` argument to Optimizer.compute_gradients "
+            "not supported when eager execution is enabled.")
       if not callable(loss):
-        raise ValueError("`loss` passed to Optimizer.compute_gradients should "
-                         "be a function when eager execution is enabled.")
+        raise RuntimeError(
+            "`loss` passed to Optimizer.compute_gradients should "
+            "be a function when eager execution is enabled.")
       # TODO(agarwal): consider passing parameters to the `loss` function.
       if var_list is None:
         return backprop.implicit_grad(loss)()
diff --git a/tensorflow/python/training/saver.py b/tensorflow/python/training/saver.py
index 145b44e2e0..9d784b2745 100644
--- a/tensorflow/python/training/saver.py
+++ b/tensorflow/python/training/saver.py
@@ -1198,15 +1198,22 @@ class Saver(object):
     Raises:
       TypeError: If `var_list` is invalid.
       ValueError: If any of the keys or values in `var_list` are not unique.
+      RuntimeError: If eager execution is enabled and`var_list` does not specify
+        a list of varialbes to save.
+
+    @compatibility(eager)
+    When eager execution is enabled, `var_list` must specify a `list` or `dict`
+    of variables to save. Otherwise, a `RuntimeError` will be raised.
+    @end_compatibility
     """
     if defer_build and var_list:
       raise ValueError(
           "If `var_list` is provided then build cannot be deferred. "
           "Either set defer_build=False or var_list=None.")
     if context.in_eager_mode() and var_list is None:
-      raise ValueError(
-          "When eager execution is enabled, `var_list` must specify a list of "
-          "variables to save")
+      raise RuntimeError(
+          "When eager execution is enabled, `var_list` must specify a list or "
+          "dict of variables to save")
     self._var_list = var_list
     self._reshape = reshape
     self._sharded = sharded
@@ -1231,7 +1238,7 @@ class Saver(object):
 
   def build(self):
     if context.in_eager_mode():
-      raise ValueError("Use save/restore instead of build in eager mode.")
+      raise RuntimeError("Use save/restore instead of build in eager mode.")
     self._build(self._filename, build_save=True, build_restore=True)
 
   def _build_eager(self, checkpoint_path, build_save, build_restore):
@@ -1802,11 +1809,19 @@ def import_meta_graph(meta_graph_or_file, clear_devices=False,
 
     A None value is returned if no variables exist in the `MetaGraphDef`
     (i.e., there are no variables to restore).
+
+  Raises:
+    RuntimeError: If called with eager execution enabled.
+
+  @compatibility(eager)
+  Exporting/importing meta graphs is not supported. No graph exists when eager
+  execution is enabled.
+  @end_compatibility
   """  # pylint: disable=g-doc-exception
   if context.in_eager_mode():
-    raise ValueError("Exporting/importing meta graphs is not supported when "
-                     "eager execution is enabled. No graph exists when eager "
-                     "execution is enabled.")
+    raise RuntimeError("Exporting/importing meta graphs is not supported when "
+                       "eager execution is enabled. No graph exists when eager "
+                       "execution is enabled.")
   if not isinstance(meta_graph_or_file, meta_graph_pb2.MetaGraphDef):
     meta_graph_def = meta_graph.read_meta_graph_file(meta_graph_or_file)
   else:
@@ -1872,11 +1887,17 @@ def export_meta_graph(filename=None,
 
   Raises:
     ValueError: When the `GraphDef` is larger than 2GB.
+    RuntimeError: If called with eager execution enabled.
+
+  @compatibility(eager)
+  Exporting/importing meta graphs is not supported. No graph exists when eager
+  execution is enabled.
+  @end_compatibility
   """
   if context.in_eager_mode():
-    raise ValueError("Exporting/importing meta graphs is not supported when "
-                     "eager execution is enabled. No graph exists when eager "
-                     "execution is enabled.")
+    raise RuntimeError("Exporting/importing meta graphs is not supported when "
+                       "eager execution is enabled. No graph exists when eager "
+                       "execution is enabled.")
   meta_graph_def, _ = meta_graph.export_scoped_meta_graph(
       filename=filename,
       meta_info_def=meta_info_def,
-- 
GitLab


From b113d082ac6320adaaa0205cd77ab815ff40bc16 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 26 Oct 2017 19:06:43 -0700
Subject: [PATCH 852/909] Exclude 'self' from function arguments returned by
 util.fn_args for callables and bounded methods.

PiperOrigin-RevId: 173622989
---
 .../estimator/python/estimator/extenders.py   |  5 +--
 .../contrib/tpu/python/tpu/tpu_estimator.py   |  2 -
 tensorflow/python/estimator/estimator.py      |  4 --
 tensorflow/python/estimator/util.py           | 39 ++++++++++---------
 tensorflow/python/estimator/util_test.py      | 11 +++++-
 5 files changed, 31 insertions(+), 30 deletions(-)

diff --git a/tensorflow/contrib/estimator/python/estimator/extenders.py b/tensorflow/contrib/estimator/python/estimator/extenders.py
index 3e5eb3390f..29c3c73585 100644
--- a/tensorflow/contrib/estimator/python/estimator/extenders.py
+++ b/tensorflow/contrib/estimator/python/estimator/extenders.py
@@ -27,7 +27,7 @@ from tensorflow.python.framework import ops
 from tensorflow.python.framework import sparse_tensor as sparse_tensor_lib
 from tensorflow.python.ops import clip_ops
 from tensorflow.python.training import optimizer as optimizer_lib
-from tensorflow.python.util import tf_inspect
+
 
 _VALID_METRIC_FN_ARGS = set(['features', 'labels', 'predictions', 'config'])
 
@@ -317,9 +317,6 @@ class _TransformGradients(optimizer_lib.Optimizer):
 
 def _verify_metric_fn_args(metric_fn):
   args = set(estimator_util.fn_args(metric_fn))
-  if tf_inspect.ismethod(metric_fn):
-    if 'self' in args:
-      args.remove('self')
   invalid_args = list(args - _VALID_METRIC_FN_ARGS)
   if invalid_args:
     raise ValueError('metric_fn (%s) has following not expected args: %s' %
diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
index 805de16468..5a3b831429 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
@@ -1106,8 +1106,6 @@ class _EvalMetrics(object):
 
     if isinstance(eval_metrics[1], (tuple, list)):
       fn_args = util.fn_args(eval_metrics[0])
-      if 'self' in fn_args:
-        fn_args = tuple([arg for arg in fn_args if arg != 'self'])
       if len(eval_metrics[1]) != len(fn_args):
         raise RuntimeError(
             'In TPUEstimatorSpec.eval_metrics, length of tensors does not '
diff --git a/tensorflow/python/estimator/estimator.py b/tensorflow/python/estimator/estimator.py
index 2a4d77b1a6..f198b051cf 100644
--- a/tensorflow/python/estimator/estimator.py
+++ b/tensorflow/python/estimator/estimator.py
@@ -52,7 +52,6 @@ from tensorflow.python.training import training
 from tensorflow.python.training import training_util
 from tensorflow.python.util import compat
 from tensorflow.python.util import nest
-from tensorflow.python.util import tf_inspect
 
 
 _VALID_MODEL_FN_ARGS = set(
@@ -925,9 +924,6 @@ def _verify_model_fn_args(model_fn, params):
     logging.warning('Estimator\'s model_fn (%s) includes params '
                     'argument, but params are not passed to Estimator.',
                     model_fn)
-  if tf_inspect.ismethod(model_fn):
-    if 'self' in args:
-      args.remove('self')
   non_valid_args = list(args - _VALID_MODEL_FN_ARGS)
   if non_valid_args:
     raise ValueError('model_fn (%s) has following not expected args: %s' %
diff --git a/tensorflow/python/estimator/util.py b/tensorflow/python/estimator/util.py
index de35e66bdf..12f2592d84 100644
--- a/tensorflow/python/estimator/util.py
+++ b/tensorflow/python/estimator/util.py
@@ -19,10 +19,20 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import functools
+
 from tensorflow.python.util import tf_decorator
 from tensorflow.python.util import tf_inspect
 
 
+def _is_bounded_method(fn):
+  return tf_inspect.ismethod(fn) and (fn.__self__ is not None)
+
+
+def _is_callable_object(obj):
+  return hasattr(obj, '__call__') and tf_inspect.ismethod(obj.__call__)
+
+
 def fn_args(fn):
   """Get argument names for function-like object.
 
@@ -36,22 +46,13 @@ def fn_args(fn):
     ValueError: if partial function has positionally bound arguments
   """
   _, fn = tf_decorator.unwrap(fn)
-
-  # Handle callables.
-  if hasattr(fn, '__call__') and tf_inspect.ismethod(fn.__call__):
-    return tuple(tf_inspect.getargspec(fn.__call__).args)
-
-  # Handle functools.partial and similar objects.
-  if hasattr(fn, 'func') and hasattr(fn, 'keywords') and hasattr(fn, 'args'):
-    # Handle nested partial.
-    original_args = fn_args(fn.func)
-    if not original_args:
-      return tuple()
-
-    return tuple([
-        arg for arg in original_args[len(fn.args):]
-        if arg not in set((fn.keywords or {}).keys())
-    ])
-
-  # Handle function.
-  return tuple(tf_inspect.getargspec(fn).args)
+  if isinstance(fn, functools.partial):
+    args = fn_args(fn.func)
+    args = [a for a in args[len(fn.args):] if a not in (fn.keywords or [])]
+  else:
+    if _is_callable_object(fn):
+      fn = fn.__call__
+    args = tf_inspect.getargspec(fn).args
+    if _is_bounded_method(fn):
+      args.remove('self')
+  return tuple(args)
diff --git a/tensorflow/python/estimator/util_test.py b/tensorflow/python/estimator/util_test.py
index 3f8122c407..4b2c8d7637 100644
--- a/tensorflow/python/estimator/util_test.py
+++ b/tensorflow/python/estimator/util_test.py
@@ -38,7 +38,16 @@ class FnArgsTest(test.TestCase):
       def __call__(self, a, b):
         return a + b
 
-    self.assertEqual(('self', 'a', 'b'), util.fn_args(Foo()))
+    self.assertEqual(('a', 'b'), util.fn_args(Foo()))
+
+  def test_bounded_method(self):
+
+    class Foo(object):
+
+      def bar(self, a, b):
+        return a + b
+
+    self.assertEqual(('a', 'b'), util.fn_args(Foo().bar))
 
   def test_partial_function(self):
     expected_test_arg = 123
-- 
GitLab


From a710cb323a69458ccda772a65bb20433419dd1d9 Mon Sep 17 00:00:00 2001
From: Asim Shankar <ashankar@google.com>
Date: Thu, 26 Oct 2017 19:58:54 -0700
Subject: [PATCH 853/909] Internal change.

PiperOrigin-RevId: 173626040
---
 tensorflow/contrib/eager/python/saver.py | 56 ++++++++++++++----------
 tensorflow/python/training/saver.py      | 15 +++----
 2 files changed, 41 insertions(+), 30 deletions(-)

diff --git a/tensorflow/contrib/eager/python/saver.py b/tensorflow/contrib/eager/python/saver.py
index 404f77105a..d74e0fef3e 100644
--- a/tensorflow/contrib/eager/python/saver.py
+++ b/tensorflow/contrib/eager/python/saver.py
@@ -90,8 +90,8 @@ def restore_variables_on_create(save_path, map_func=None):
     for k, _ in checkpoint_utils.list_variables(save_path):
       ckpt_var_cache[k] = reader.get_tensor(k)
 
-    old_init = getattr(
-        resource_variable_ops.ResourceVariable, "_init_from_args", None)
+    old_init = getattr(resource_variable_ops.ResourceVariable,
+                       "_init_from_args", None)
     assert old_init, "ResourceVariable misses _init_from_args method."
     setattr(resource_variable_ops.ResourceVariable, "_init_from_args",
             _init_from_checkpoint)
@@ -114,42 +114,54 @@ def restore_variables_on_create(save_path, map_func=None):
 
 
 class Saver(object):
-  """A simple tf.train.Saver adapter for eager mode.
-
-    save and restore API are similar to the tf.train.Saver, except that
-    session is not needed.
-
-  Args:
-    var_list: Same as tf.train.Saver.
+  """A tf.train.Saver adapter for use when eager execution is enabled.
   """
 
   def __init__(self, var_list):
+    """A  tf.train.Saver adapter for use when eager execution is enabled.
+
+      The API, and on-disk format, mimic tf.train.Saver except that no
+      Session is needed.
+
+    Args:
+      var_list: The list of variables that will be saved and restored. Either a
+        list of `tfe.Variable` objects, or a dictionary mapping names to
+        `tfe.Variable` objects.
+
+    Raises:
+      RuntimeError: if invoked when eager execution has not been enabled.
+    """
     if context.in_graph_mode():
-      raise ValueError("Currently, tfe.Saver can only be used when eager "
-                       "execution is enabled. Use tf.train.Saver when "
-                       "building graphs.")
+      raise RuntimeError("tfe.Saver can only be used when eager "
+                         "execution is enabled. Use tf.train.Saver when "
+                         "building graphs.")
     self._saver = _saver.Saver(var_list=var_list)
 
-  def save(self, save_path, global_step=None):
+  def save(self, file_prefix, global_step=None):
     """Saves variables.
 
     Args:
-      save_path: See save method in tf.train.Saver.
-      global_step: See save method in tf.train.Saver.
+      file_prefix: Path prefix of files created for the checkpoint.
+      global_step: If provided the global step number is appended to file_prefix
+        to create the checkpoint filename. The optional argument can be a
+        Tensor, a Variable, or an integer.
 
     Returns:
-      See save method in tf.train.Saver.
+      A string: prefix of filenames created for the checkpoint. This may be
+       an extension of file_prefix that is suitable to pass as an argument
+       to a subsequent call to `restore()`.
     """
     with ops.device("/device:CPU:0"):
-      return self._saver.save(None, save_path, write_meta_graph=False,
-                              global_step=global_step)
+      return self._saver.save(
+          None, file_prefix, write_meta_graph=False, global_step=global_step)
 
-  def restore(self, save_path):
+  def restore(self, file_prefix):
     """Restores previously saved variables.
 
     Args:
-      save_path: See restore method in tf.train.Saver.
+      file_prefix: Path prefix where parameters were previously saved.
+        Typically obtained from a previous `save()` call, or from
+        @{tf.train.latest_checkpoint}.
     """
     with ops.device("/device:CPU:0"):
-      self._saver.restore(None, save_path)
-
+      self._saver.restore(None, file_prefix)
diff --git a/tensorflow/python/training/saver.py b/tensorflow/python/training/saver.py
index 9d784b2745..60420eb86a 100644
--- a/tensorflow/python/training/saver.py
+++ b/tensorflow/python/training/saver.py
@@ -1510,18 +1510,17 @@ class Saver(object):
     It requires a session in which the graph was launched.  The variables to
     save must also have been initialized.
 
-    The method returns the path of the newly created checkpoint file.  This
-    path can be passed directly to a call to `restore()`.
+    The method returns the path prefix of the newly created checkpoint files.
+    This string can be passed directly to a call to `restore()`.
 
     Args:
-      sess: A Session to use to save the variables. None in eager mode.
-      save_path: String.  Path to the checkpoint filename.  If the saver is
-        `sharded`, this is the prefix of the sharded checkpoint filename.
+      sess: A Session to use to save the variables.
+      save_path: String.  Prefix of filenames created for the checkpoint.
       global_step: If provided the global step number is appended to
-        `save_path` to create the checkpoint filename. The optional argument
+        `save_path` to create the checkpoint filenames. The optional argument
         can be a `Tensor`, a `Tensor` name or an integer.
       latest_filename: Optional name for the protocol buffer file that will
-        contains the list of most recent checkpoint filenames.  That file,
+        contains the list of most recent checkpoints.  That file,
         kept in the same directory as the checkpoint files, is automatically
         managed by the saver to keep track of recent checkpoints.  Defaults to
         'checkpoint'.
@@ -1532,7 +1531,7 @@ class Saver(object):
         `CheckpointStateProto`.
 
     Returns:
-      A string: path at which the variables were saved.  If the saver is
+      A string: path prefix used for the checkpoint files.  If the saver is
         sharded, this string ends with: '-?????-of-nnnnn' where 'nnnnn'
         is the number of shards created.
       If the saver is empty, returns None.
-- 
GitLab


From e7e312a11fdb2d6c3c8dc183b6fb2d2e55b43242 Mon Sep 17 00:00:00 2001
From: Rohan Varma <rvarm1@gmail.com>
Date: Thu, 26 Oct 2017 20:58:13 -0700
Subject: [PATCH 854/909] reuse=False

---
 tensorflow/python/ops/variable_scope.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/ops/variable_scope.py b/tensorflow/python/ops/variable_scope.py
index 08be8574f3..91c53f401b 100644
--- a/tensorflow/python/ops/variable_scope.py
+++ b/tensorflow/python/ops/variable_scope.py
@@ -581,7 +581,7 @@ class _VariableStore(object):
     if reuse is True:
       raise ValueError("PartitionedVariable %s does not exist, or was not "
                        "created with tf.get_variable(). Did you mean to set "
-                       "reuse=None in VarScope?" % name)
+                       "reuse=False or reuse=tf.AUTO_REUSE in VarScope?" % name)
 
     slice_dim, slice_shape = _compute_slice_dim_and_shape(
         shape.as_list(), partitions)
-- 
GitLab


From a494558127ada7d32d2b85d99bebadc57c7f6e33 Mon Sep 17 00:00:00 2001
From: Eugene Brevdo <ebrevdo@google.com>
Date: Thu, 26 Oct 2017 21:13:19 -0700
Subject: [PATCH 855/909] Automated g4 rollback of changelist 172654120

PiperOrigin-RevId: 173630195
---
 tensorflow/python/ops/array_ops.py | 11 +++--------
 1 file changed, 3 insertions(+), 8 deletions(-)

diff --git a/tensorflow/python/ops/array_ops.py b/tensorflow/python/ops/array_ops.py
index 97dc63ebb1..ba8c611f57 100644
--- a/tensorflow/python/ops/array_ops.py
+++ b/tensorflow/python/ops/array_ops.py
@@ -2466,14 +2466,9 @@ def where(condition, x=None, y=None, name=None):
   """
   if x is None and y is None:
     with ops.name_scope(name, "Where", [condition]) as name:
-      # Temporarily create an old style WhereOp nodedef + Operation without the
-      # attribute "T".
-      # TODO(b/67720963): Roll this back when the issue is resolved.
-      condition = gen_math_ops.cast(condition, dtypes.bool)
-      output = gen_array_ops.where(input=condition, name=name)
-      if context.in_graph_mode():
-        output.op._node_def.attr.clear()
-      return output
+      condition = ops.convert_to_tensor(
+          condition, preferred_dtype=dtypes.bool, name="condition")
+      return gen_array_ops.where(input=condition, name=name)
   elif x is not None and y is not None:
     return gen_math_ops._select(condition=condition, t=x, e=y, name=name)
   else:
-- 
GitLab


From 9c8a520b07d5789dc3e43b0698573da822617c81 Mon Sep 17 00:00:00 2001
From: Justine Tunney <jart@google.com>
Date: Fri, 27 Oct 2017 00:15:55 -0700
Subject: [PATCH 856/909] Add WriteEvent method to SummaryWriterInterface

Another change will follow that adds an op for this method. It will be useful
for loading event logs into other types of summary writer implementations, like
a database.

This change might also make the new summary file writer go faster, due to less
memory copying.

PiperOrigin-RevId: 173640116
---
 tensorflow/core/kernels/summary_interface.cc  | 77 +++++++++++--------
 tensorflow/core/kernels/summary_interface.h   |  4 +
 .../core/kernels/summary_interface_test.cc    | 26 +++++--
 3 files changed, 69 insertions(+), 38 deletions(-)

diff --git a/tensorflow/core/kernels/summary_interface.cc b/tensorflow/core/kernels/summary_interface.cc
index a0b9038787..313137ae49 100644
--- a/tensorflow/core/kernels/summary_interface.cc
+++ b/tensorflow/core/kernels/summary_interface.cc
@@ -12,6 +12,9 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
+#include "tensorflow/core/kernels/summary_interface.h"
+
+#include <utility>
 
 #include "tensorflow/compiler/xla/ptr_util.h"
 #include "tensorflow/core/framework/op_kernel.h"
@@ -19,12 +22,10 @@ limitations under the License.
 #include "tensorflow/core/framework/summary.pb.h"
 #include "tensorflow/core/framework/types.h"
 #include "tensorflow/core/framework/types.pb.h"
-#include "tensorflow/core/kernels/summary_interface.h"
 #include "tensorflow/core/lib/histogram/histogram.h"
 #include "tensorflow/core/lib/io/path.h"
 #include "tensorflow/core/lib/png/png_io.h"
 #include "tensorflow/core/lib/wav/wav_io.h"
-#include "tensorflow/core/util/event.pb.h"
 #include "tensorflow/core/util/events_writer.h"
 
 namespace tensorflow {
@@ -250,28 +251,34 @@ class SummaryWriterImpl : public SummaryWriterInterface {
 
   Status WriteTensor(int64 global_step, Tensor t, const string& tag,
                      const string& serialized_metadata) override {
-    Summary s;
-    Summary::Value* v = s.add_value();
+    std::unique_ptr<Event> e{new Event};
+    e->set_step(global_step);
+    e->set_wall_time(GetWallTime());
+    Summary::Value* v = e->mutable_summary()->add_value();
     t.AsProtoTensorContent(v->mutable_tensor());
     v->set_tag(tag);
     v->mutable_metadata()->ParseFromString(serialized_metadata);
-    return Enqueue(global_step, s);
+    return WriteEvent(std::move(e));
   }
 
   Status WriteScalar(int64 global_step, Tensor t, const string& tag) override {
-    Summary s;
-    Summary::Value* v = s.add_value();
+    std::unique_ptr<Event> e{new Event};
+    e->set_step(global_step);
+    e->set_wall_time(GetWallTime());
+    Summary::Value* v = e->mutable_summary()->add_value();
     v->set_tag(tag);
     float value;
     TF_RETURN_IF_ERROR(TensorValueAt<float>(t, 0, &value));
     v->set_simple_value(value);
-    return Enqueue(global_step, s);
+    return WriteEvent(std::move(e));
   }
 
   Status WriteHistogram(int64 global_step, Tensor t,
                         const string& tag) override {
-    Summary s;
-    Summary::Value* v = s.add_value();
+    std::unique_ptr<Event> e{new Event};
+    e->set_step(global_step);
+    e->set_wall_time(GetWallTime());
+    Summary::Value* v = e->mutable_summary()->add_value();
     v->set_tag(tag);
     histogram::Histogram histo;
     for (int64 i = 0; i < t.NumElements(); i++) {
@@ -287,7 +294,7 @@ class SummaryWriterImpl : public SummaryWriterInterface {
     }
 
     histo.EncodeToProto(v->mutable_histo(), false /* Drop zero buckets */);
-    return Enqueue(global_step, s);
+    return WriteEvent(std::move(e));
   }
 
   Status WriteImage(int64 global_step, Tensor tensor, const string& tag,
@@ -306,7 +313,10 @@ class SummaryWriterImpl : public SummaryWriterInterface {
       return errors::InvalidArgument("Tensor too large for summary ",
                                      tensor.shape().DebugString());
     }
-    Summary s;
+    std::unique_ptr<Event> e{new Event};
+    e->set_step(global_step);
+    e->set_wall_time(GetWallTime());
+    Summary* s = e->mutable_summary();
     // The casts and h * w cannot overflow because of the limits above.
     const int batch_size = static_cast<int>(tensor.dim_size(0));
     const int h = static_cast<int>(tensor.dim_size(1));
@@ -321,20 +331,20 @@ class SummaryWriterImpl : public SummaryWriterInterface {
             &values(i, 0, 0), Eigen::DSizes<Eigen::DenseIndex, 2>(hw, depth));
       };
       TF_RETURN_IF_ERROR(
-          AddImages(tag, max_images, batch_size, w, h, depth, ith_image, &s));
+          AddImages(tag, max_images, batch_size, w, h, depth, ith_image, s));
     } else if (tensor.dtype() == DT_HALF) {
       TF_RETURN_IF_ERROR(NormalizeAndAddImages<Eigen::half>(
-          tensor, max_images, h, w, hw, depth, batch_size, tag, bad_color, &s));
+          tensor, max_images, h, w, hw, depth, batch_size, tag, bad_color, s));
     } else if (tensor.dtype() == DT_FLOAT) {
       TF_RETURN_IF_ERROR(NormalizeAndAddImages<float>(
-          tensor, max_images, h, w, hw, depth, batch_size, tag, bad_color, &s));
+          tensor, max_images, h, w, hw, depth, batch_size, tag, bad_color, s));
     } else {
       return errors::InvalidArgument(
           "Only DT_INT8, DT_HALF, and DT_FLOAT images are supported. Got ",
           DataTypeString(tensor.dtype()));
     }
 
-    return Enqueue(global_step, s);
+    return WriteEvent(std::move(e));
   }
 
   Status WriteAudio(int64 global_step, Tensor tensor, const string& tag,
@@ -346,10 +356,13 @@ class SummaryWriterImpl : public SummaryWriterInterface {
     const int64 length_frames = tensor.dim_size(1);
     const int64 num_channels =
         tensor.dims() == 2 ? 1 : tensor.dim_size(tensor.dims() - 1);
-    Summary s;
+    std::unique_ptr<Event> e{new Event};
+    e->set_step(global_step);
+    e->set_wall_time(GetWallTime());
+    Summary* s = e->mutable_summary();
     const int N = std::min<int>(max_outputs, batch_size);
     for (int i = 0; i < N; ++i) {
-      Summary::Value* v = s.add_value();
+      Summary::Value* v = s->add_value();
       if (max_outputs > 1) {
         v->set_tag(strings::StrCat(tag, "/audio/", i));
       } else {
@@ -375,16 +388,12 @@ class SummaryWriterImpl : public SummaryWriterInterface {
           channels_by_frames.data(), sample_rate_truncated, num_channels,
           length_frames, sa->mutable_encoded_audio_string()));
     }
-
-    return Enqueue(global_step, s);
+    return WriteEvent(std::move(e));
   }
 
-  string DebugString() override { return "SummaryWriterImpl"; }
-
- private:
-  Status Enqueue(int64 global_step, const Summary& summary) {
+  Status WriteEvent(std::unique_ptr<Event> event) override {
     mutex_lock ml(mu_);
-    queue_.emplace_back(global_step, summary, env_->NowMicros());
+    queue_.emplace_back(std::move(event));
     if (queue_.size() >= max_queue_ ||
         env_->NowMicros() - last_flush_ > 1000 * flush_millis_) {
       return InternalFlush();
@@ -392,13 +401,16 @@ class SummaryWriterImpl : public SummaryWriterInterface {
     return Status::OK();
   }
 
+  string DebugString() override { return "SummaryWriterImpl"; }
+
+ private:
+  double GetWallTime() {
+    return static_cast<double>(env_->NowMicros()) / 1.0e6;
+  }
+
   Status InternalFlush() EXCLUSIVE_LOCKS_REQUIRED(mu_) {
-    for (const EventInfo& e : queue_) {
-      Event event;
-      event.set_step(std::get<0>(e));
-      *event.mutable_summary() = std::get<1>(e);
-      event.set_wall_time(static_cast<double>(std::get<2>(e)) / 1.0e6);
-      events_writer_->WriteEvent(event);
+    for (const std::unique_ptr<Event>& e : queue_) {
+      events_writer_->WriteEvent(*e);
     }
     queue_.clear();
     if (!events_writer_->Flush()) {
@@ -413,9 +425,8 @@ class SummaryWriterImpl : public SummaryWriterInterface {
   const int flush_millis_;
   uint64 last_flush_;
   Env* env_;
-  using EventInfo = std::tuple<int64, Summary, int64>;
   mutex mu_;
-  std::vector<EventInfo> queue_ GUARDED_BY(mu_);
+  std::vector<std::unique_ptr<Event>> queue_ GUARDED_BY(mu_);
   // A pointer to allow deferred construction.
   std::unique_ptr<EventsWriter> events_writer_ GUARDED_BY(mu_);
   std::vector<std::pair<string, SummaryMetadata>> registered_summaries_
diff --git a/tensorflow/core/kernels/summary_interface.h b/tensorflow/core/kernels/summary_interface.h
index 1b5d0b2748..ccf3459e56 100644
--- a/tensorflow/core/kernels/summary_interface.h
+++ b/tensorflow/core/kernels/summary_interface.h
@@ -15,8 +15,10 @@ limitations under the License.
 #ifndef TENSORFLOW_CORE_KERNELS_SUMMARY_INTERFACE_H_
 #define TENSORFLOW_CORE_KERNELS_SUMMARY_INTERFACE_H_
 
+#include <memory>
 
 #include "tensorflow/core/framework/resource_mgr.h"
+#include "tensorflow/core/util/event.pb.h"
 
 namespace tensorflow {
 
@@ -43,6 +45,8 @@ class SummaryWriterInterface : public ResourceBase {
 
   virtual Status WriteAudio(int64 global_step, Tensor t, const string& tag,
                             int max_outputs_, float sample_rate) = 0;
+
+  virtual Status WriteEvent(std::unique_ptr<Event> e) = 0;
 };
 
 // Creates a SummaryWriterInterface instance which writes to a file. It will
diff --git a/tensorflow/core/kernels/summary_interface_test.cc b/tensorflow/core/kernels/summary_interface_test.cc
index 379e045ca3..58e021a0b3 100644
--- a/tensorflow/core/kernels/summary_interface_test.cc
+++ b/tensorflow/core/kernels/summary_interface_test.cc
@@ -12,11 +12,9 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-
-#include <vector>
+#include "tensorflow/core/kernels/summary_interface.h"
 
 #include "tensorflow/core/framework/summary.pb.h"
-#include "tensorflow/core/kernels/summary_interface.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/core/refcount.h"
 #include "tensorflow/core/lib/io/path.h"
@@ -43,8 +41,8 @@ class SummaryInterfaceTest : public ::testing::Test {
  protected:
   Status SummaryTestHelper(
       const string& test_name,
-      std::function<Status(SummaryWriterInterface*)> writer_fn,
-      std::function<void(const Event&)> test_fn) {
+      const std::function<Status(SummaryWriterInterface*)>& writer_fn,
+      const std::function<void(const Event&)>& test_fn) {
     static std::set<string>* tests = new std::set<string>();
     CHECK(tests->insert(test_name).second) << ": " << test_name;
 
@@ -182,6 +180,24 @@ TEST_F(SummaryInterfaceTest, WriteAudio) {
       }));
 }
 
+TEST_F(SummaryInterfaceTest, WriteEvent) {
+  TF_CHECK_OK(
+      SummaryTestHelper("event_test",
+                        [](SummaryWriterInterface* writer) {
+                          std::unique_ptr<Event> e{new Event};
+                          e->set_step(7);
+                          e->mutable_summary()->add_value()->set_tag("hi");
+                          TF_RETURN_IF_ERROR(writer->WriteEvent(std::move(e)));
+                          TF_RETURN_IF_ERROR(writer->Flush());
+                          return Status::OK();
+                        },
+                        [](const Event& e) {
+                          EXPECT_EQ(e.step(), 7);
+                          CHECK_EQ(e.summary().value_size(), 1);
+                          EXPECT_EQ(e.summary().value(0).tag(), "hi");
+                        }));
+}
+
 TEST_F(SummaryInterfaceTest, WallTime) {
   env_.AdvanceByMillis(7023);
   TF_CHECK_OK(SummaryTestHelper(
-- 
GitLab


From 3595d1613d0d46fad7cda0140965472351ff84b1 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Sun, 15 Oct 2017 23:07:30 +0000
Subject: [PATCH 857/909] Add `double` support for `tf.decode_csv`

In the current tensorflow `tf.decode_csv` accepts
`float`, `int32`, `int64`, `string` but not `double`.
It seems adding `double` support makes sense as `StringToNumber`
already support `double` type.

This fix adds `double` support for `tf.decode_csv`

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/core/kernels/decode_csv_op.cc | 19 +++++++++++++++++++
 tensorflow/core/ops/parsing_ops.cc       |  2 +-
 2 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/kernels/decode_csv_op.cc b/tensorflow/core/kernels/decode_csv_op.cc
index 5e48ae9766..6080f32072 100644
--- a/tensorflow/core/kernels/decode_csv_op.cc
+++ b/tensorflow/core/kernels/decode_csv_op.cc
@@ -137,6 +137,25 @@ class DecodeCSVOp : public OpKernel {
             }
             break;
           }
+          case DT_DOUBLE: {
+            // If this field is empty or NA value, check if default is given:
+            // If yes, use default value; Otherwise report error.
+            if (fields[f].empty() || fields[f] == na_value_) {
+              OP_REQUIRES(ctx, record_defaults[f].NumElements() == 1,
+                          errors::InvalidArgument(
+                              "Field ", f,
+                              " is required but missing in record ", i, "!"));
+              output[f]->flat<double>()(i) = record_defaults[f].flat<double>()(0);
+            } else {
+              double value;
+              OP_REQUIRES(ctx, strings::safe_strtod(fields[f].c_str(), &value),
+                          errors::InvalidArgument(
+                              "Field ", f, " in record ", i,
+                              " is not a valid double: ", fields[f]));
+              output[f]->flat<double>()(i) = value;
+            }
+            break;
+          }
           case DT_STRING: {
             // If this field is empty or NA value, check if default is given:
             // If yes, use default value; Otherwise report error.
diff --git a/tensorflow/core/ops/parsing_ops.cc b/tensorflow/core/ops/parsing_ops.cc
index b44ea2e080..40ec792ef8 100644
--- a/tensorflow/core/ops/parsing_ops.cc
+++ b/tensorflow/core/ops/parsing_ops.cc
@@ -329,7 +329,7 @@ REGISTER_OP("DecodeCSV")
     .Input("records: string")
     .Input("record_defaults: OUT_TYPE")
     .Output("output: OUT_TYPE")
-    .Attr("OUT_TYPE: list({float,int32,int64,string})")
+    .Attr("OUT_TYPE: list({float,double,int32,int64,string})")
     .Attr("field_delim: string = ','")
     .Attr("use_quote_delim: bool = true")
     .Attr("na_value: string = ''")
-- 
GitLab


From 73aaed655b4fddbd23c0dca32deb84e5dc191f0b Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Sun, 15 Oct 2017 23:10:31 +0000
Subject: [PATCH 858/909] Update docs for `double` support on `tf.decode_csv`

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/python/ops/parsing_ops.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/ops/parsing_ops.py b/tensorflow/python/ops/parsing_ops.py
index ea7132791c..14aef01dec 100644
--- a/tensorflow/python/ops/parsing_ops.py
+++ b/tensorflow/python/ops/parsing_ops.py
@@ -1183,7 +1183,7 @@ def decode_csv(records, record_defaults, field_delim=",",
       Each string is a record/row in the csv and all records should have
       the same format.
     record_defaults: A list of `Tensor` objects with specific types.
-      Acceptable types are `float32`, `int32`, `int64`, `string`.
+      Acceptable types are `float32`, `float64`, `int32`, `int64`, `string`.
       One tensor per column of the input record, with either a
       scalar default value for that column or empty if the column is required.
     field_delim: An optional `string`. Defaults to `","`.
-- 
GitLab


From 285ea39108cd7817c67abe5390b617c8cb6d8ccc Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Sun, 15 Oct 2017 23:11:04 +0000
Subject: [PATCH 859/909] Add test cases for `double` support of
 `tf.decode_csv`

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 .../python/kernel_tests/decode_csv_op_test.py       | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/kernel_tests/decode_csv_op_test.py b/tensorflow/python/kernel_tests/decode_csv_op_test.py
index 7d9e57c8e5..fec52fa9cc 100644
--- a/tensorflow/python/kernel_tests/decode_csv_op_test.py
+++ b/tensorflow/python/kernel_tests/decode_csv_op_test.py
@@ -34,7 +34,7 @@ class DecodeCSVOpTest(test.TestCase):
         out = sess.run(decode)
 
         for i, field in enumerate(out):
-          if field.dtype == np.float32:
+          if field.dtype == np.float32 or field.dtype == np.float64:
             self.assertAllClose(field, expected_out[i])
           else:
             self.assertAllEqual(field, expected_out[i])
@@ -85,6 +85,17 @@ class DecodeCSVOpTest(test.TestCase):
 
     self._test(args, expected_out)
 
+  def testDouble(self):
+    args = {
+        "records": ["1.0", "-1.79e+308", '"1.79e+308"'],
+        "record_defaults": [np.array(
+            [], dtype=np.double)],
+    }
+
+    expected_out = [[1.0, -1.79e+308, 1.79e+308]]
+
+    self._test(args, expected_out)
+
   def testInt64(self):
     args = {
         "records": ["1", "2", '"2147483648"'],
-- 
GitLab


From c6292a3f936daa4fdd92881ea1f6bec614c6bd06 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Fri, 27 Oct 2017 13:31:56 +0000
Subject: [PATCH 860/909] Sanitize decode_csv_op.cc with `clang-format -i`

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/core/kernels/decode_csv_op.cc | 43 ++++++++++++------------
 1 file changed, 21 insertions(+), 22 deletions(-)

diff --git a/tensorflow/core/kernels/decode_csv_op.cc b/tensorflow/core/kernels/decode_csv_op.cc
index 6080f32072..c4555db453 100644
--- a/tensorflow/core/kernels/decode_csv_op.cc
+++ b/tensorflow/core/kernels/decode_csv_op.cc
@@ -91,9 +91,9 @@ class DecodeCSVOp : public OpKernel {
             } else {
               int32 value;
               OP_REQUIRES(ctx, strings::safe_strto32(fields[f], &value),
-                          errors::InvalidArgument(
-                              "Field ", f, " in record ", i,
-                              " is not a valid int32: ", fields[f]));
+                          errors::InvalidArgument("Field ", f, " in record ", i,
+                                                  " is not a valid int32: ",
+                                                  fields[f]));
               output[f]->flat<int32>()(i) = value;
             }
             break;
@@ -111,9 +111,9 @@ class DecodeCSVOp : public OpKernel {
             } else {
               int64 value;
               OP_REQUIRES(ctx, strings::safe_strto64(fields[f], &value),
-                          errors::InvalidArgument(
-                              "Field ", f, " in record ", i,
-                              " is not a valid int64: ", fields[f]));
+                          errors::InvalidArgument("Field ", f, " in record ", i,
+                                                  " is not a valid int64: ",
+                                                  fields[f]));
               output[f]->flat<int64>()(i) = value;
             }
             break;
@@ -130,9 +130,9 @@ class DecodeCSVOp : public OpKernel {
             } else {
               float value;
               OP_REQUIRES(ctx, strings::safe_strtof(fields[f].c_str(), &value),
-                          errors::InvalidArgument(
-                              "Field ", f, " in record ", i,
-                              " is not a valid float: ", fields[f]));
+                          errors::InvalidArgument("Field ", f, " in record ", i,
+                                                  " is not a valid float: ",
+                                                  fields[f]));
               output[f]->flat<float>()(i) = value;
             }
             break;
@@ -145,13 +145,14 @@ class DecodeCSVOp : public OpKernel {
                           errors::InvalidArgument(
                               "Field ", f,
                               " is required but missing in record ", i, "!"));
-              output[f]->flat<double>()(i) = record_defaults[f].flat<double>()(0);
+              output[f]->flat<double>()(i) =
+                  record_defaults[f].flat<double>()(0);
             } else {
               double value;
               OP_REQUIRES(ctx, strings::safe_strtod(fields[f].c_str(), &value),
-                          errors::InvalidArgument(
-                              "Field ", f, " in record ", i,
-                              " is not a valid double: ", fields[f]));
+                          errors::InvalidArgument("Field ", f, " in record ", i,
+                                                  " is not a valid double: ",
+                                                  fields[f]));
               output[f]->flat<double>()(i) = value;
             }
             break;
@@ -207,10 +208,9 @@ class DecodeCSVOp : public OpKernel {
         if (!quoted) {
           while (static_cast<size_t>(current_idx) < input.size() &&
                  input[current_idx] != delim_) {
-            OP_REQUIRES(ctx,
-                        (!use_quote_delim_ || input[current_idx] != '"') &&
-                            input[current_idx] != '\n' &&
-                            input[current_idx] != '\r',
+            OP_REQUIRES(ctx, (!use_quote_delim_ || input[current_idx] != '"') &&
+                                 input[current_idx] != '\n' &&
+                                 input[current_idx] != '\r',
                         errors::InvalidArgument(
                             "Unquoted fields cannot have quotes/CRLFs inside"));
             field += input[current_idx];
@@ -238,11 +238,10 @@ class DecodeCSVOp : public OpKernel {
           }
 
           OP_REQUIRES(
-              ctx,
-              (static_cast<size_t>(current_idx) < input.size() &&
-               input[current_idx] == '"' &&
-               (static_cast<size_t>(current_idx) == input.size() - 1 ||
-                input[current_idx + 1] == delim_)),
+              ctx, (static_cast<size_t>(current_idx) < input.size() &&
+                    input[current_idx] == '"' &&
+                    (static_cast<size_t>(current_idx) == input.size() - 1 ||
+                     input[current_idx + 1] == delim_)),
               errors::InvalidArgument("Quoted field has to end with quote "
                                       "followed by delim or end"));
 
-- 
GitLab


From 62a9ab28caef7bb6f3eb0c8b625c58fa8d88a173 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E7=94=B0=E4=BC=A0=E6=AD=A6?= <dev@goodow.com>
Date: Fri, 27 Oct 2017 08:52:21 -0500
Subject: [PATCH 861/909] fix broken link

---
 tensorflow/docs_src/programmers_guide/graphs.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/docs_src/programmers_guide/graphs.md b/tensorflow/docs_src/programmers_guide/graphs.md
index 10f53fe8f2..5ec3738d7d 100644
--- a/tensorflow/docs_src/programmers_guide/graphs.md
+++ b/tensorflow/docs_src/programmers_guide/graphs.md
@@ -100,7 +100,7 @@ to all API functions in the same context.  For example:
   when run, will apply those gradients to a set of variables.
 
 Most programs rely solely on the default graph. However,
-see [Dealing with multiple graphs](#dealing-with-multiple-graphs) for more
+see [Dealing with multiple graphs](#programming_with_multiple_graphs) for more
 advanced use cases. High-level APIs such as the @{tf.estimator.Estimator} API
 manage the default graph on your behalf, and--for example--may create different
 graphs for training and evaluation.
-- 
GitLab


From 58d2c5f50508fad6591166f6e264d574f9c42768 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Fri, 27 Oct 2017 07:31:33 -0700
Subject: [PATCH 862/909] Add `SANITY_STEPS_DESC` for do_clang_format_check
 (#14030)

* Add `SANITY_STEPS_DESC` for do_clang_format_check

This fix is a follow up to PR #13924 to add the corresponding
description in `SANITY_STEPS_DESC`.

See comment https://github.com/tensorflow/tensorflow/pull/13924#discussion_r147314599
for details.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Update description for Clang Format Check

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/tools/ci_build/ci_sanity.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/tools/ci_build/ci_sanity.sh b/tensorflow/tools/ci_build/ci_sanity.sh
index 1e1fd7db6b..7e78def8eb 100755
--- a/tensorflow/tools/ci_build/ci_sanity.sh
+++ b/tensorflow/tools/ci_build/ci_sanity.sh
@@ -506,7 +506,7 @@ do_check_load_py_test() {
 
 # Supply all sanity step commands and descriptions
 SANITY_STEPS=("do_pylint PYTHON2" "do_pylint PYTHON3" "do_buildifier" "do_bazel_nobuild" "do_pip_package_licenses_check" "do_lib_package_licenses_check" "do_java_package_licenses_check" "do_pip_smoke_test" "do_check_load_py_test" "do_code_link_check" "do_clang_format_check")
-SANITY_STEPS_DESC=("Python 2 pylint" "Python 3 pylint" "buildifier check" "bazel nobuild" "pip: license check for external dependencies" "C library: license check for external dependencies" "Java Native Library: license check for external dependencies" "Pip Smoke Test: Checking py_test dependencies exist in pip package" "Check load py_test: Check that BUILD files with py_test target properly load py_test" "Code Link Check: Check there are no broken links")
+SANITY_STEPS_DESC=("Python 2 pylint" "Python 3 pylint" "buildifier check" "bazel nobuild" "pip: license check for external dependencies" "C library: license check for external dependencies" "Java Native Library: license check for external dependencies" "Pip Smoke Test: Checking py_test dependencies exist in pip package" "Check load py_test: Check that BUILD files with py_test target properly load py_test" "Code Link Check: Check there are no broken links" "Clang Format Check: Check .h and .cc files with Google C++ style")
 
 INCREMENTAL_FLAG=""
 DEFAULT_BAZEL_CONFIGS="--config=hdfs --config=gcp"
-- 
GitLab


From a7b8725271634e892781080464d0cef8516a9d36 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Fri, 27 Oct 2017 14:47:27 +0000
Subject: [PATCH 863/909] Fix an ouput typo in `ci_sanity.sh`

In the last PR #13924 (clang sanity check) the output message should be changed:
`due to the absence of Python code changes`
->
`due to the absence of .h or .cc code changes`

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/tools/ci_build/ci_sanity.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/tools/ci_build/ci_sanity.sh b/tensorflow/tools/ci_build/ci_sanity.sh
index 1e1fd7db6b..9e23c6231a 100755
--- a/tensorflow/tools/ci_build/ci_sanity.sh
+++ b/tensorflow/tools/ci_build/ci_sanity.sh
@@ -460,7 +460,7 @@ do_clang_format_check() {
 
     if [[ -z "${CLANG_SRC_FILES}" ]]; then
       echo "do_clang_format_check will NOT run due to --incremental flag and "\
-"due to the absence of Python code changes in the last commit."
+"due to the absence of .h or .cc code changes in the last commit."
       return 0
     fi
   elif [[ -z "$1" ]]; then
-- 
GitLab


From 0ccf5cf600c0ab97a9e5d699caef5750d180348d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 27 Oct 2017 08:15:17 -0700
Subject: [PATCH 864/909] Limit the amount of logspam a use of
 GraphKeys.VARIABLES causes.

Multiple copies of this warning next to each other often make logs unreadable.

PiperOrigin-RevId: 173672701
---
 tensorflow/python/framework/ops.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index eceacb42d9..c278fb2a39 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -4938,9 +4938,10 @@ class GraphKeys(object):
 
   @decorator_utils.classproperty
   def VARIABLES(cls):  # pylint: disable=no-self-argument
-    logging.warning("VARIABLES collection name is deprecated, "
-                    "please use GLOBAL_VARIABLES instead; "
-                    "VARIABLES will be removed after 2017-03-02.")
+    logging.log_first_n(logging.WARN,
+                        "VARIABLES collection name is deprecated, please use "
+                        "GLOBAL_VARIABLES instead; VARIABLES will be removed "
+                        "after 2017-03-02.", 1)
     return cls.GLOBAL_VARIABLES
 
 
-- 
GitLab


From 4ae245a7db3d0457c4324ee7df8d020ba83b3c60 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 27 Oct 2017 08:37:35 -0700
Subject: [PATCH 865/909] n/a (internal change only)

PiperOrigin-RevId: 173674697
---
 tensorflow/contrib/learn/BUILD | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/learn/BUILD b/tensorflow/contrib/learn/BUILD
index f3949beed0..ac615b120c 100644
--- a/tensorflow/contrib/learn/BUILD
+++ b/tensorflow/contrib/learn/BUILD
@@ -768,7 +768,7 @@ py_test(
         ":learn",
         "//tensorflow/contrib/layers:layers_py",
         "//tensorflow/contrib/session_bundle:exporter",
-        "//tensorflow/contrib/session_bundle:manifest_proto_py",
+        "//tensorflow/contrib/session_bundle:manifest_proto_py_pb2",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:client",
         "//tensorflow/python:client_testlib",
-- 
GitLab


From 4198e27be8115585ad6b5b141383fb7dc7856c24 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 27 Oct 2017 09:00:51 -0700
Subject: [PATCH 866/909] [XLA:CPU] [XLA:GPU] Adds compiler support for C64
 primitive type, including relevant elementwise unary and binary op lowering
 for CPU and GPU.

We use a named LLVM struct "complex64", laid out the same as std::complex<float>. This named struct is accessed via the llvm::Module, which required changes to accessors of PrimitiveTypeToIrType & friends.

Ops that require atan2 (in particular, angle and log) are only supported on GPU at this point. LLVM lacks a CPU intrinsic for atan or atan2, whereas libdevice provides this for GPU.

PiperOrigin-RevId: 173676849
---
 .../jit/mark_for_compilation_pass_test.cc     |   4 +-
 tensorflow/compiler/jit/xla_cpu_device.cc     |   4 +-
 tensorflow/compiler/jit/xla_gpu_device.cc     |   4 +-
 tensorflow/compiler/tests/BUILD               |   7 +-
 tensorflow/compiler/tests/argminmax_test.py   |   4 +-
 tensorflow/compiler/tests/binary_ops_test.py  | 248 +++++++++---
 tensorflow/compiler/tests/build_defs.bzl      |  12 +-
 tensorflow/compiler/tests/gather_test.py      |  12 +-
 tensorflow/compiler/tests/nary_ops_test.py    |  20 +
 tensorflow/compiler/tests/random_ops_test.py  |   8 +-
 tensorflow/compiler/tests/randomized_tests.cc | 345 +++++++++++------
 tensorflow/compiler/tests/unary_ops_test.py   | 130 ++++++-
 .../compiler/tests/variable_ops_test.py       |  91 +++--
 tensorflow/compiler/tests/xla_test.py         |  11 +-
 .../tf2xla/kernels/batch_matmul_op.cc         |   6 +
 .../compiler/tf2xla/kernels/binary_ops.cc     |  14 +-
 tensorflow/compiler/tf2xla/kernels/cast_op.cc |   6 +
 .../compiler/tf2xla/kernels/gather_op.cc      |   2 +-
 .../compiler/tf2xla/kernels/matmul_op.cc      |   5 +-
 .../compiler/tf2xla/kernels/training_ops.cc   |  23 +-
 .../compiler/tf2xla/kernels/unary_ops.cc      |   9 +
 tensorflow/compiler/tf2xla/xla_helpers.cc     |   6 +
 tensorflow/compiler/tf2xla/xla_op_registry.h  |  15 +-
 .../xla/client/computation_builder.cc         |  27 ++
 .../compiler/xla/client/computation_builder.h |  19 +
 tensorflow/compiler/xla/literal_util.cc       |   6 +
 .../xla/service/algebraic_simplifier.cc       |  21 +
 .../xla/service/algebraic_simplifier_test.cc  |  50 +++
 .../xla/service/cpu/dot_op_emitter.cc         |  47 ++-
 .../xla/service/cpu/elemental_ir_emitter.cc   |   4 +-
 .../xla/service/cpu/ir_emission_utils.cc      |   6 +
 .../compiler/xla/service/cpu/ir_emitter.cc    |  64 +--
 .../compiler/xla/service/dfs_hlo_visitor.h    |  14 +
 .../xla/service/elemental_ir_emitter.cc       | 363 ++++++++++++++++--
 .../xla/service/elemental_ir_emitter.h        |  13 +
 .../xla/service/gpu/elemental_ir_emitter.cc   | 123 +++++-
 .../xla/service/gpu/elemental_ir_emitter.h    |   3 +
 .../xla/service/gpu/hlo_to_ir_bindings.cc     |   8 +-
 .../xla/service/gpu/hlo_to_ir_bindings.h      |   5 +-
 .../compiler/xla/service/gpu/ir_emitter.cc    |  79 ++--
 .../compiler/xla/service/gpu/ir_emitter.h     |   1 +
 .../xla/service/gpu/ir_emitter_nested.cc      |   8 +-
 .../xla/service/gpu/ir_emitter_unnested.cc    |  21 +-
 .../compiler/xla/service/hlo_evaluator.cc     | 313 ++++++++++++---
 .../compiler/xla/service/hlo_graph_dumper.cc  |   4 +
 .../compiler/xla/service/hlo_instruction.cc   |  67 +++-
 tensorflow/compiler/xla/service/hlo_opcode.cc |   8 +
 tensorflow/compiler/xla/service/hlo_opcode.h  |   4 +
 .../compiler/xla/service/hlo_pass_pipeline.cc |   1 +
 .../compiler/xla/service/hlo_verifier.cc      |   4 +
 .../xla/service/instruction_fusion.cc         |  15 +-
 .../xla/service/llvm_ir/fused_ir_emitter.cc   |   6 +-
 .../xla/service/llvm_ir/fused_ir_emitter.h    |   4 +-
 .../compiler/xla/service/llvm_ir/ir_array.cc  |   7 +-
 .../compiler/xla/service/llvm_ir/llvm_util.cc |  78 +++-
 .../compiler/xla/service/llvm_ir/llvm_util.h  |   6 +-
 .../compiler/xla/service/llvm_ir/tuple_ops.cc |  14 +-
 .../compiler/xla/service/llvm_ir/tuple_ops.h  |   8 +-
 .../compiler/xla/service/shape_inference.cc   |  53 ++-
 .../xla/service/shape_inference_test.cc       |  39 ++
 .../compiler/xla/service/user_computation.cc  |   8 +
 tensorflow/compiler/xla/shape_util.cc         |   1 +
 .../xla/tests/client_library_test_base.h      |  15 +-
 .../compiler/xla/tests/dot_operation_test.cc  |   8 +-
 .../compiler/xla/tests/unary_op_test.cc       |  80 +++-
 .../compiler/xla/tools/parser/hlo_parser.cc   |   4 +
 tensorflow/compiler/xla/types.h               |   4 +-
 tensorflow/compiler/xla/xla_data.proto        |  14 +-
 68 files changed, 2115 insertions(+), 518 deletions(-)

diff --git a/tensorflow/compiler/jit/mark_for_compilation_pass_test.cc b/tensorflow/compiler/jit/mark_for_compilation_pass_test.cc
index 579ce415c5..b3d258aea1 100644
--- a/tensorflow/compiler/jit/mark_for_compilation_pass_test.cc
+++ b/tensorflow/compiler/jit/mark_for_compilation_pass_test.cc
@@ -144,8 +144,8 @@ TEST(XlaCompilationTest, UnsupportedTypes) {
     Node* a = ops::SourceOp(
         "Const", builder.opts()
                      .WithName("A")
-                     .WithAttr("dtype", DT_COMPLEX64)
-                     .WithAttr("value", Tensor(DT_COMPLEX64, TensorShape())));
+                     .WithAttr("dtype", DT_COMPLEX128)
+                     .WithAttr("value", Tensor(DT_COMPLEX128, TensorShape())));
     Node* b = ops::UnaryOp("Neg", a, builder.opts().WithName("B"));
     ops::BinaryOp("MatMul", a, b, builder.opts().WithName("C"));
     TF_EXPECT_OK(builder.ToGraph(graph.get()));
diff --git a/tensorflow/compiler/jit/xla_cpu_device.cc b/tensorflow/compiler/jit/xla_cpu_device.cc
index 2e33fdca65..e238252751 100644
--- a/tensorflow/compiler/jit/xla_cpu_device.cc
+++ b/tensorflow/compiler/jit/xla_cpu_device.cc
@@ -50,8 +50,8 @@ REGISTER_LOCAL_DEVICE_FACTORY(DEVICE_XLA_CPU, XlaCpuDeviceFactory);
 
 // Kernel registrations
 
-constexpr std::array<DataType, 5> kAllXlaCpuTypes = {
-    {DT_INT32, DT_INT64, DT_FLOAT, DT_DOUBLE, DT_BOOL}};
+constexpr std::array<DataType, 6> kAllXlaCpuTypes = {
+    {DT_INT32, DT_INT64, DT_FLOAT, DT_DOUBLE, DT_COMPLEX64, DT_BOOL}};
 
 REGISTER_XLA_LAUNCH_KERNEL(DEVICE_XLA_CPU, XlaLocalLaunchOp, kAllXlaCpuTypes);
 REGISTER_XLA_DEVICE_KERNELS(DEVICE_XLA_CPU, kAllXlaCpuTypes);
diff --git a/tensorflow/compiler/jit/xla_gpu_device.cc b/tensorflow/compiler/jit/xla_gpu_device.cc
index 5233665ec2..2326070358 100644
--- a/tensorflow/compiler/jit/xla_gpu_device.cc
+++ b/tensorflow/compiler/jit/xla_gpu_device.cc
@@ -55,8 +55,8 @@ REGISTER_LOCAL_DEVICE_FACTORY(DEVICE_XLA_GPU, XlaGpuDeviceFactory);
 
 // Kernel registrations
 
-constexpr std::array<DataType, 5> kAllXlaGpuTypes = {
-    {DT_INT32, DT_INT64, DT_FLOAT, DT_DOUBLE, DT_BOOL}};
+constexpr std::array<DataType, 6> kAllXlaGpuTypes = {
+    {DT_INT32, DT_INT64, DT_FLOAT, DT_DOUBLE, DT_COMPLEX64, DT_BOOL}};
 
 REGISTER_XLA_LAUNCH_KERNEL(DEVICE_XLA_GPU, XlaLocalLaunchOp, kAllXlaGpuTypes);
 REGISTER_XLA_DEVICE_KERNELS(DEVICE_XLA_GPU, kAllXlaGpuTypes);
diff --git a/tensorflow/compiler/tests/BUILD b/tensorflow/compiler/tests/BUILD
index 0eed475140..d07bf98296 100644
--- a/tensorflow/compiler/tests/BUILD
+++ b/tensorflow/compiler/tests/BUILD
@@ -23,6 +23,10 @@ load("//tensorflow:tensorflow.bzl", "cuda_py_test")
 load("//tensorflow/compiler/aot:tfcompile.bzl", "tf_library")
 load("//tensorflow/compiler/tests:build_defs.bzl", "tf_xla_py_test")
 load("//tensorflow/compiler/tests:build_defs.bzl", "generate_backend_suites")
+load(
+    "//tensorflow/core:platform/default/build_config_root.bzl",
+    "tf_cuda_tests_tags",
+)
 
 generate_backend_suites()
 
@@ -581,11 +585,12 @@ cc_library(
 
 tf_cuda_cc_test(
     name = "randomized_tests",
+    size = "large",
     # This test is randomized, so only run it if explicitly requested.
     tags = [
         "manual",
         "notap",
-    ],
+    ] + tf_cuda_tests_tags(),
     deps = [":randomized_tests_library"],
 )
 
diff --git a/tensorflow/compiler/tests/argminmax_test.py b/tensorflow/compiler/tests/argminmax_test.py
index c2ce121348..ec547e16cd 100644
--- a/tensorflow/compiler/tests/argminmax_test.py
+++ b/tensorflow/compiler/tests/argminmax_test.py
@@ -46,7 +46,9 @@ class ArgMinMaxTest(xla_test.XLATestCase):
       self.assertAllEqual(result, expected)
 
   def testArgMinMax(self):
-    for dtype in self.numeric_types:
+    # Complex numbers do not support argmin/argmax.
+    minmax_types = set(self.numeric_types) - set(self.complex_types)
+    for dtype in minmax_types:
       self._assertOpOutputMatchesExpected(
           lambda x: math_ops.argmax(x, axis=0, output_type=dtypes.int32),
           np.array([1, 10, 27, 3, 3, 4], dtype=dtype),
diff --git a/tensorflow/compiler/tests/binary_ops_test.py b/tensorflow/compiler/tests/binary_ops_test.py
index 9a225b32f8..d412c572ae 100644
--- a/tensorflow/compiler/tests/binary_ops_test.py
+++ b/tensorflow/compiler/tests/binary_ops_test.py
@@ -94,6 +94,15 @@ class BinaryOpsTest(XLATestCase):
           dtype(4),
           expected=np.array([[16], [81]], dtype=dtype))
 
+      atan2_supported = self.device == "XLA_GPU"
+      if atan2_supported:
+        self._testBinary(
+            math_ops.atan2,
+            np.array([0, np.sqrt(2), 1, np.sqrt(2), 0], dtype),
+            np.array([1, np.sqrt(2), 0, -np.sqrt(2), -1], dtype),
+            expected=np.array(
+                [0, np.pi / 4, np.pi / 2, np.pi * 3 / 4, np.pi], dtype=dtype))
+
       self._testBinary(
           gen_math_ops._reciprocal_grad,
           np.array([4, -3, -2, 1], dtype=dtype),
@@ -259,37 +268,38 @@ class BinaryOpsTest(XLATestCase):
           dtype(7),
           expected=np.array([[-6], [-5]], dtype=dtype))
 
-      self._testBinary(
-          math_ops.maximum,
-          np.array([1, 2], dtype=dtype),
-          np.array([10, 20], dtype=dtype),
-          expected=np.array([10, 20], dtype=dtype))
-      self._testBinary(
-          math_ops.maximum,
-          dtype(5),
-          np.array([1, 20], dtype=dtype),
-          expected=np.array([5, 20], dtype=dtype))
-      self._testBinary(
-          math_ops.maximum,
-          np.array([[10], [2]], dtype=dtype),
-          dtype(7),
-          expected=np.array([[10], [7]], dtype=dtype))
+      if dtype not in self.complex_types:  # min/max not supported for complex
+        self._testBinary(
+            math_ops.maximum,
+            np.array([1, 2], dtype=dtype),
+            np.array([10, 20], dtype=dtype),
+            expected=np.array([10, 20], dtype=dtype))
+        self._testBinary(
+            math_ops.maximum,
+            dtype(5),
+            np.array([1, 20], dtype=dtype),
+            expected=np.array([5, 20], dtype=dtype))
+        self._testBinary(
+            math_ops.maximum,
+            np.array([[10], [2]], dtype=dtype),
+            dtype(7),
+            expected=np.array([[10], [7]], dtype=dtype))
 
-      self._testBinary(
-          math_ops.minimum,
-          np.array([1, 20], dtype=dtype),
-          np.array([10, 2], dtype=dtype),
-          expected=np.array([1, 2], dtype=dtype))
-      self._testBinary(
-          math_ops.minimum,
-          dtype(5),
-          np.array([1, 20], dtype=dtype),
-          expected=np.array([1, 5], dtype=dtype))
-      self._testBinary(
-          math_ops.minimum,
-          np.array([[10], [2]], dtype=dtype),
-          dtype(7),
-          expected=np.array([[7], [2]], dtype=dtype))
+        self._testBinary(
+            math_ops.minimum,
+            np.array([1, 20], dtype=dtype),
+            np.array([10, 2], dtype=dtype),
+            expected=np.array([1, 2], dtype=dtype))
+        self._testBinary(
+            math_ops.minimum,
+            dtype(5),
+            np.array([1, 20], dtype=dtype),
+            expected=np.array([1, 5], dtype=dtype))
+        self._testBinary(
+            math_ops.minimum,
+            np.array([[10], [2]], dtype=dtype),
+            dtype(7),
+            expected=np.array([[7], [2]], dtype=dtype))
 
       self._testBinary(
           math_ops.multiply,
@@ -307,21 +317,23 @@ class BinaryOpsTest(XLATestCase):
           dtype(7),
           expected=np.array([[70], [14]], dtype=dtype))
 
-      self._testBinary(
-          math_ops.squared_difference,
-          np.array([1, 2], dtype=dtype),
-          np.array([10, 20], dtype=dtype),
-          expected=np.array([81, 324], dtype=dtype))
-      self._testBinary(
-          math_ops.squared_difference,
-          dtype(5),
-          np.array([1, 2], dtype=dtype),
-          expected=np.array([16, 9], dtype=dtype))
-      self._testBinary(
-          math_ops.squared_difference,
-          np.array([[1], [2]], dtype=dtype),
-          dtype(7),
-          expected=np.array([[36], [25]], dtype=dtype))
+      # Complex support for squared_difference is incidental, see b/68205550
+      if dtype not in self.complex_types:
+        self._testBinary(
+            math_ops.squared_difference,
+            np.array([1, 2], dtype=dtype),
+            np.array([10, 20], dtype=dtype),
+            expected=np.array([81, 324], dtype=dtype))
+        self._testBinary(
+            math_ops.squared_difference,
+            dtype(5),
+            np.array([1, 2], dtype=dtype),
+            expected=np.array([16, 9], dtype=dtype))
+        self._testBinary(
+            math_ops.squared_difference,
+            np.array([[1], [2]], dtype=dtype),
+            dtype(7),
+            expected=np.array([[36], [25]], dtype=dtype))
 
       self._testBinary(
           nn_ops.bias_add,
@@ -334,6 +346,139 @@ class BinaryOpsTest(XLATestCase):
           np.array([2, -1], dtype=dtype),
           expected=np.array([[[[3, 1], [5, 3]]]], dtype=dtype))
 
+  def testComplexOps(self):
+    for dtype in self.complex_types:
+      ctypes = {np.complex64: np.float32}
+      self._testBinary(
+          math_ops.complex,
+          np.array([[[[-1, 2], [2, 0]]]], dtype=ctypes[dtype]),
+          np.array([[[[2, -3], [0, 4]]]], dtype=ctypes[dtype]),
+          expected=np.array([[[[-1 + 2j, 2 - 3j], [2, 4j]]]], dtype=dtype))
+
+      self._testBinary(
+          lambda x, y: math_ops.approximate_equal(x, y, tolerance=0.0001),
+          np.array(
+              [[[[-1 + 2j, 2.00009999 - 3j], [2 - 3j, 3 + 4.01j]]]],
+              dtype=dtype),
+          np.array(
+              [[[[-1.001 + 2j, 2 - 3j], [2 - 3.00009j, 3 + 4j]]]], dtype=dtype),
+          expected=np.array([[[[False, True], [True, False]]]], dtype=dtype))
+
+      self._testBinary(
+          gen_math_ops._real_div,
+          np.array([3, 3j, -1.5j, -8, 2 + 3j, 2 + 4j, 44 + 3j], dtype=dtype),
+          np.array([2, -2, 7j, -4j, 4 - 6j, 1 + 2j, 0], dtype=dtype),
+          expected=np.array(
+              [
+                  1.5, -1.5j, -0.2142857, -2j, (2 + 3j) / (4 - 6j), 2,
+                  float("inf")
+              ],
+              dtype=dtype))
+
+      # TODO(b/65408531): support+test pow for cplx
+
+      lhs = np.array([4 + 2j, -3 - 1j, 2j, 1], dtype=dtype)
+      rhs = np.array([5, -6j, 7 - 3j, -8j], dtype=dtype)
+      self._testBinary(
+          gen_math_ops._reciprocal_grad, lhs, rhs, expected=-rhs * lhs * lhs)
+
+      self._testBinary(
+          gen_math_ops._sigmoid_grad, lhs, rhs, expected=rhs * lhs * (1 - lhs))
+
+      # TODO(b/65408531): support+test _rsqrt_grad for cplx (needs pow)
+
+      self._testBinary(
+          gen_math_ops._sqrt_grad, lhs, rhs, expected=rhs / (2 * lhs))
+
+      self._testBinary(
+          gen_math_ops._tanh_grad, lhs, rhs, expected=rhs * (1 - lhs * lhs))
+
+  def testComplexMath(self):
+    for dtype in self.complex_types:
+      self._testBinary(
+          math_ops.add,
+          np.array([1 + 3j, 2 + 7j], dtype=dtype),
+          np.array([10 - 4j, 20 + 17j], dtype=dtype),
+          expected=np.array([11 - 1j, 22 + 24j], dtype=dtype))
+      self._testBinary(
+          math_ops.add,
+          dtype(5 - 7j),
+          np.array([1 + 2j, 2 + 4j], dtype=dtype),
+          expected=np.array([6 - 5j, 7 - 3j], dtype=dtype))
+      self._testBinary(
+          math_ops.add,
+          np.array([[1 - 2j], [2 + 1j]], dtype=dtype),
+          dtype(7 + 5j),
+          expected=np.array([[8 + 3j], [9 + 6j]], dtype=dtype))
+
+      self._testBinary(
+          math_ops.subtract,
+          np.array([1 + 3j, 2 + 7j], dtype=dtype),
+          np.array([10 - 4j, 20 + 17j], dtype=dtype),
+          expected=np.array([-9 + 7j, -18 - 10j], dtype=dtype))
+      self._testBinary(
+          math_ops.subtract,
+          dtype(5 - 7j),
+          np.array([1 + 2j, 2 + 4j], dtype=dtype),
+          expected=np.array([4 - 9j, 3 - 11j], dtype=dtype))
+      self._testBinary(
+          math_ops.subtract,
+          np.array([[1 - 2j], [2 + 1j]], dtype=dtype),
+          dtype(7 + 5j),
+          expected=np.array([[-6 - 7j], [-5 - 4j]], dtype=dtype))
+
+      self._testBinary(
+          math_ops.multiply,
+          np.array([1 + 3j, 2 + 7j], dtype=dtype),
+          np.array([10 - 4j, 20 + 17j], dtype=dtype),
+          expected=np.array(
+              [(1 + 3j) * (10 - 4j), (2 + 7j) * (20 + 17j)], dtype=dtype))
+      self._testBinary(
+          math_ops.multiply,
+          dtype(5 - 7j),
+          np.array([1 + 2j, 2 + 4j], dtype=dtype),
+          expected=np.array(
+              [(5 - 7j) * (1 + 2j), (5 - 7j) * (2 + 4j)], dtype=dtype))
+      self._testBinary(
+          math_ops.multiply,
+          np.array([[1 - 2j], [2 + 1j]], dtype=dtype),
+          dtype(7 + 5j),
+          expected=np.array(
+              [[(7 + 5j) * (1 - 2j)], [(7 + 5j) * (2 + 1j)]], dtype=dtype))
+
+      self._testBinary(
+          math_ops.div,
+          np.array([8 - 1j, 2 + 16j], dtype=dtype),
+          np.array([2 + 4j, 4 - 8j], dtype=dtype),
+          expected=np.array(
+              [(8 - 1j) / (2 + 4j), (2 + 16j) / (4 - 8j)], dtype=dtype))
+      self._testBinary(
+          math_ops.div,
+          dtype(1 + 2j),
+          np.array([2 + 4j, 4 - 8j], dtype=dtype),
+          expected=np.array(
+              [(1 + 2j) / (2 + 4j), (1 + 2j) / (4 - 8j)], dtype=dtype))
+      self._testBinary(
+          math_ops.div,
+          np.array([2 + 4j, 4 - 8j], dtype=dtype),
+          dtype(1 + 2j),
+          expected=np.array(
+              [(2 + 4j) / (1 + 2j), (4 - 8j) / (1 + 2j)], dtype=dtype))
+
+      # TODO(b/68205550): math_ops.squared_difference shouldn't be supported.
+
+      self._testBinary(
+          nn_ops.bias_add,
+          np.array([[1 + 2j, 2 + 7j], [3 - 5j, 4 + 2j]], dtype=dtype),
+          np.array([2 + 6j, -1 - 3j], dtype=dtype),
+          expected=np.array([[3 + 8j, 1 + 4j], [5 + 1j, 3 - 1j]], dtype=dtype))
+      self._testBinary(
+          nn_ops.bias_add,
+          np.array([[[[1 + 4j, 2 - 1j], [3 + 7j, 4]]]], dtype=dtype),
+          np.array([2 + 1j, -1 + 2j], dtype=dtype),
+          expected=np.array(
+              [[[[3 + 5j, 1 + 1j], [5 + 8j, 3 + 2j]]]], dtype=dtype))
+
   def _testDivision(self, dtype):
     """Test cases for division operators."""
     self._testBinary(
@@ -352,18 +497,19 @@ class BinaryOpsTest(XLATestCase):
         dtype(2),
         expected=np.array([[5], [2]], dtype=dtype))
 
-    self._testBinary(
-        gen_math_ops._floor_div,
-        np.array([3, 3, -1, -9, -8], dtype=dtype),
-        np.array([2, -2, 7, 2, -4], dtype=dtype),
-        expected=np.array([1, -2, -1, -5, 2], dtype=dtype))
+    if dtype not in self.complex_types:  # floordiv unsupported for complex.
+      self._testBinary(
+          gen_math_ops._floor_div,
+          np.array([3, 3, -1, -9, -8], dtype=dtype),
+          np.array([2, -2, 7, 2, -4], dtype=dtype),
+          expected=np.array([1, -2, -1, -5, 2], dtype=dtype))
 
   def testIntDivision(self):
     for dtype in self.int_types:
       self._testDivision(dtype)
 
   def testFloatDivision(self):
-    for dtype in self.float_types:
+    for dtype in self.float_types + self.complex_types:
       self._testDivision(dtype)
 
   def _testRemainder(self, dtype):
diff --git a/tensorflow/compiler/tests/build_defs.bzl b/tensorflow/compiler/tests/build_defs.bzl
index a56c53de0f..0528a5415d 100644
--- a/tensorflow/compiler/tests/build_defs.bzl
+++ b/tensorflow/compiler/tests/build_defs.bzl
@@ -49,11 +49,15 @@ def tf_xla_py_test(name, srcs=[], deps=[], tags=[], data=[], main=None,
     backend_deps = []
     backend_data = []
     if backend == "cpu":
-      backend_args += ["--test_device=XLA_CPU",
-                       "--types=DT_FLOAT,DT_DOUBLE,DT_INT32,DT_INT64,DT_BOOL"]
+      backend_args += [
+          "--test_device=XLA_CPU",
+          "--types=DT_FLOAT,DT_DOUBLE,DT_INT32,DT_INT64,DT_BOOL,DT_COMPLEX64"
+      ]
     elif backend == "gpu":
-      backend_args += ["--test_device=XLA_GPU",
-                       "--types=DT_FLOAT,DT_DOUBLE,DT_INT32,DT_INT64,DT_BOOL"]
+      backend_args += [
+          "--test_device=XLA_GPU",
+          "--types=DT_FLOAT,DT_DOUBLE,DT_INT32,DT_INT64,DT_BOOL,DT_COMPLEX64"
+      ]
       backend_tags += ["requires-gpu-sm35"]
     elif backend in plugins:
       backend_args += ["--test_device=" + plugins[backend]["device"],
diff --git a/tensorflow/compiler/tests/gather_test.py b/tensorflow/compiler/tests/gather_test.py
index 4b81c1d7ab..664c77f200 100644
--- a/tensorflow/compiler/tests/gather_test.py
+++ b/tensorflow/compiler/tests/gather_test.py
@@ -30,8 +30,6 @@ from tensorflow.python.platform import test
 
 FLAGS = flags.FLAGS
 
-_TEST_TYPES = [dtypes.float32]
-
 
 class GatherTest(xla_test.XLATestCase):
 
@@ -46,7 +44,7 @@ class GatherTest(xla_test.XLATestCase):
   def testScalar1D(self):
     with self.test_session() as session, self.test_scope():
       data = np.array([0, 1, 2, 3, 7, 5])
-      for dtype in _TEST_TYPES:
+      for dtype in self.all_tf_types:
         for indices in 4, [1, 2, 2, 4, 5]:
           params_np = self._buildParams(data, dtype)
           params = array_ops.placeholder(dtype=dtype)
@@ -60,7 +58,7 @@ class GatherTest(xla_test.XLATestCase):
     with self.test_session() as session, self.test_scope():
       data = np.array([[0, 1, 2], [3, 4, 5], [6, 7, 8], [9, 10, 11],
                        [12, 13, 14]])
-      for dtype in _TEST_TYPES:
+      for dtype in self.all_tf_types:
         for axis in 0, 1, -1:
           params_np = self._buildParams(data, dtype)
           params = array_ops.placeholder(dtype=dtype)
@@ -74,7 +72,7 @@ class GatherTest(xla_test.XLATestCase):
     with self.test_session() as session, self.test_scope():
       data = np.array([[0, 1, 2], [3, 4, 5], [6, 7, 8], [9, 10, 11],
                        [12, 13, 14]])
-      for dtype in _TEST_TYPES:
+      for dtype in self.all_tf_types:
         for axis in 0, 1, -1:
           params_np = self._buildParams(data, dtype)
           params = array_ops.placeholder(dtype=dtype)
@@ -94,7 +92,7 @@ class GatherTest(xla_test.XLATestCase):
                        [12, 13, 14]])
       # The indices must be in bounds for any axis.
       indices_np = np.array([0, 1, 0, 2])
-      for dtype in _TEST_TYPES:
+      for dtype in self.all_tf_types:
         for axis in 0, 1, -1:
           params_np = self._buildParams(data, dtype)
           params = array_ops.placeholder(dtype=dtype)
@@ -112,7 +110,7 @@ class GatherTest(xla_test.XLATestCase):
     """Check that scalar and empty indices shapes work as well."""
     shape = (2, 1, 3, 2)
     for indices_shape in (), (0,), (2, 0), (2, 3):
-      for dtype in _TEST_TYPES:
+      for dtype in self.all_tf_types:
         for axis in 0, 1, 2, 3, -1, -2:
           params = self._buildParams(np.random.randn(*shape), dtype)
           indices = np.random.randint(shape[axis], size=indices_shape)
diff --git a/tensorflow/compiler/tests/nary_ops_test.py b/tensorflow/compiler/tests/nary_ops_test.py
index ae60d78f1a..e4843b169b 100644
--- a/tensorflow/compiler/tests/nary_ops_test.py
+++ b/tensorflow/compiler/tests/nary_ops_test.py
@@ -68,6 +68,26 @@ class NAryOpsTest(XLATestCase):
                     np.array([42], dtype=np.float32)],
                    expected=np.array([48], dtype=np.float32))
 
+  def testComplex(self):
+    for dtype in self.complex_types:
+      self._testNAry(
+          math_ops.add_n, [np.array([[1 + 2j, 2 - 3j, 3 + 4j]], dtype=dtype)],
+          expected=np.array([[1 + 2j, 2 - 3j, 3 + 4j]], dtype=dtype))
+
+      self._testNAry(
+          math_ops.add_n, [
+              np.array([1 + 2j, 2 - 3j], dtype=dtype),
+              np.array([10j, 20], dtype=dtype)
+          ],
+          expected=np.array([1 + 12j, 22 - 3j], dtype=dtype))
+      self._testNAry(
+          math_ops.add_n, [
+              np.array([-4, 5j], dtype=dtype),
+              np.array([2 + 10j, -2], dtype=dtype),
+              np.array([42j, 3 + 3j], dtype=dtype)
+          ],
+          expected=np.array([-2 + 52j, 1 + 8j], dtype=dtype))
+
   @unittest.skip("IdentityN is temporarily CompilationOnly as workaround")
   def testIdentityN(self):
     self._testNAryLists(array_ops.identity_n,
diff --git a/tensorflow/compiler/tests/random_ops_test.py b/tensorflow/compiler/tests/random_ops_test.py
index a17a3f3d65..d6c93088d4 100644
--- a/tensorflow/compiler/tests/random_ops_test.py
+++ b/tensorflow/compiler/tests/random_ops_test.py
@@ -29,6 +29,9 @@ from tensorflow.python.platform import googletest
 class RandomOpsTest(XLATestCase):
   """Test cases for random-number generating operators."""
 
+  def _random_types(self):
+    return set(self.numeric_types) - set(self.complex_types)
+
   def _testRngIsNotConstant(self, rng, dtype):
     # Tests that 'rng' does not always return the same value.
     with self.test_session() as sess:
@@ -51,7 +54,8 @@ class RandomOpsTest(XLATestCase):
     def rng(dtype):
       return random_ops.random_uniform(shape=[2], dtype=dtype,
                                        maxval=1000000)
-    for dtype in self.numeric_types:
+
+    for dtype in self._random_types():
       self._testRngIsNotConstant(rng, dtype)
 
   def testRandomNormalIsNotConstant(self):
@@ -63,7 +67,7 @@ class RandomOpsTest(XLATestCase):
     self._testRngIsNotConstant(rng, dtype)
 
   def testRandomUniformIsInRange(self):
-    for dtype in self.numeric_types:
+    for dtype in self._random_types():
       with self.test_session() as sess:
         with self.test_scope():
           x = random_ops.random_uniform(shape=[1000], dtype=dtype, minval=-2,
diff --git a/tensorflow/compiler/tests/randomized_tests.cc b/tensorflow/compiler/tests/randomized_tests.cc
index 5129171cd4..461af83362 100644
--- a/tensorflow/compiler/tests/randomized_tests.cc
+++ b/tensorflow/compiler/tests/randomized_tests.cc
@@ -75,7 +75,7 @@ namespace {
 // Command line flags: see main() below.
 int64 tf_xla_random_seed = 0;
 int32 tf_xla_test_repetitions = 20;
-int64 tf_xla_max_tensor_size = 100000LL;
+int64 tf_xla_max_tensor_size = 10000LL;
 string* tf_xla_test_device_ptr;  // initial value set in main()
 bool tf_xla_test_use_jit = true;
 
@@ -83,8 +83,8 @@ string LocalDeviceToFullDeviceName(const string& device) {
   return strings::StrCat("/job:localhost/replica:0/task:0/device:", device);
 }
 
-constexpr std::array<DataType, 3> kAllXlaTypes = {
-    {DT_INT32, DT_FLOAT, DT_BOOL}};
+constexpr std::array<DataType, 4> kAllXlaTypes = {
+    {DT_INT32, DT_FLOAT, DT_BOOL, DT_COMPLEX64}};
 
 // An OpTestBuilder is a graph builder class that takes as input an operator to
 // test, its inputs and attributes, and builds a graph that executes the
@@ -449,6 +449,13 @@ Tensor OpTest::RandomTensor(DataType dtype, gtl::ArraySlice<int64> shape) {
       });
       break;
     }
+    case DT_COMPLEX64: {
+      std::uniform_real_distribution<float> distribution(-1.0f, 1.0f);
+      test::FillFn<complex64>(&tensor, [this, &distribution](int i) {
+        return complex64(distribution(generator()), distribution(generator()));
+      });
+      break;
+    }
     case DT_INT32: {
       std::uniform_int_distribution<int32> distribution(-(1 << 20), 1 << 20);
       test::FillFn<int32>(&tensor, [this, &distribution](int i) -> int32 {
@@ -624,11 +631,47 @@ std::vector<int32> OpTest::AsInt32s(const std::vector<int64>& int64s) {
 
 // Functions for comparing tensors.
 
+template <typename T>
+double Abs(T x) {
+  return std::fabs(x);
+}
+
+template <>
+double Abs<complex64>(complex64 x) {
+  return std::abs(x);
+}
+
 template <typename T>
 bool IsClose(const T& x, const T& y, double atol, double rtol) {
   if (std::isnan(x) && std::isnan(y)) return true;
   if (x == y) return true;  // Allow inf == inf.
-  return fabs(x - y) < atol + rtol * fabs(x);
+  return Abs(x - y) < atol + rtol * Abs(x);
+}
+
+template <>
+bool IsClose<complex64>(const complex64& x, const complex64& y, double atol,
+                        double rtol) {
+  if (std::isnan(x.real()) && std::isnan(y.real())) {
+    if (std::isnan(x.imag()) && std::isnan(y.imag())) {
+      return true;
+    }
+    if (x.imag() == y.imag()) return true;  // Allow inf == inf.
+    return Abs(x.imag() - y.imag()) < atol + rtol * Abs(x.imag());
+  } else if (std::isnan(x.imag()) && std::isnan(y.imag())) {
+    if (x.real() == y.real()) return true;  // Allow inf == inf.
+    return Abs(x.real() - y.real()) < atol + rtol * Abs(x.real());
+  }
+  if (x == y) return true;  // Allow inf == inf.
+  return Abs(x - y) < atol + rtol * Abs(x);
+}
+
+template <typename T>
+string Str(T x) {
+  return strings::StrCat(x);
+}
+template <>
+string Str<complex64>(complex64 x) {
+  return strings::StrCat("(", x.real(), ", ", x.imag(), ")");
 }
 
 template <typename T>
@@ -639,9 +682,10 @@ Status TensorsAreCloseImpl(const Tensor& x, const Tensor& y, double atol,
   for (int i = 0; i < Tx.size(); ++i) {
     if (!IsClose(Tx(i), Ty(i), atol, rtol)) {
       return errors::InvalidArgument(strings::StrCat(
-          i, "-th tensor element isn't close: ", Tx(i), " vs. ", Ty(i),
-          ". x = ", x.DebugString(), "y = ", y.DebugString(), "atol = ", atol,
-          " rtol = ", rtol, " tol = ", atol + rtol * std::fabs(Tx(i))));
+          i, "-th tensor element isn't close: ", Str(Tx(i)), " vs. ",
+          Str(Ty(i)), ". x = ", x.DebugString(), "y = ", y.DebugString(),
+          "atol = ", atol, " rtol = ", rtol,
+          " tol = ", atol + rtol * Abs(Tx(i))));
     }
   }
   return Status::OK();
@@ -683,6 +727,8 @@ Status TensorsAreClose(const Tensor& a, const Tensor& b, double atol,
       return TensorsAreCloseImpl<float>(a, b, atol, rtol);
     case DT_DOUBLE:
       return TensorsAreCloseImpl<double>(a, b, atol, rtol);
+    case DT_COMPLEX64:
+      return TensorsAreCloseImpl<complex64>(a, b, atol, rtol);
     case DT_INT32:
       return TensorsAreEqualImpl<int32>(a, b);
     case DT_INT64:
@@ -822,7 +868,7 @@ Tensor AsIntTensor(DataType dtype, const std::vector<int64>& values) {
 
 TEST_F(OpTest, Abs) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT});
+    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
     return ExpectTfAndXlaOutputsAreClose(
         OpTestBuilder("Abs").RandomInput(type).Attr("T", type));
   });
@@ -837,7 +883,7 @@ TEST_F(OpTest, Acosh) {
 
 TEST_F(OpTest, Add) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT});
+    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
     auto dims = BroadcastableDims();
     return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("Add")
                                              .RandomInput(type, dims.first)
@@ -848,7 +894,7 @@ TEST_F(OpTest, Add) {
 
 TEST_F(OpTest, AddN) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT});
+    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
     int n = std::uniform_int_distribution<int>(1, 5)(generator());
 
     auto shape = RandomDims();
@@ -890,9 +936,10 @@ TEST_F(OpTest, Any) {
 TEST_F(OpTest, ApproximateEqual) {
   Repeatedly([this]() {
     auto dims = RandomDims();
+    DataType type = Choose<DataType>({DT_FLOAT, DT_COMPLEX64});
     return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("ApproximateEqual")
-                                             .RandomInput(DT_FLOAT, dims)
-                                             .RandomInput(DT_FLOAT, dims)
+                                             .RandomInput(type, dims)
+                                             .RandomInput(type, dims)
                                              .Attr("T", DT_FLOAT));
   });
 }
@@ -1038,6 +1085,7 @@ TEST_F(OpTest, AvgPool3DGrad) {
 
 TEST_F(OpTest, BatchMatMul) {
   Repeatedly([this]() {
+    DataType type = Choose<DataType>({DT_FLOAT, DT_COMPLEX64});
     std::vector<int64> output_dims = RandomDims(2, 5, 0, 7);
     int64 ndims = output_dims.size();
     int64 inner_dim = RandomDim();
@@ -1056,9 +1104,9 @@ TEST_F(OpTest, BatchMatMul) {
     }
 
     return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("BatchMatMul")
-                                             .RandomInput(DT_FLOAT, x_dims)
-                                             .RandomInput(DT_FLOAT, y_dims)
-                                             .Attr("T", DT_FLOAT)
+                                             .RandomInput(type, x_dims)
+                                             .RandomInput(type, y_dims)
+                                             .Attr("T", type)
                                              .Attr("adj_x", adj_x)
                                              .Attr("adj_y", adj_y));
   });
@@ -1090,10 +1138,11 @@ TEST_F(OpTest, BatchToSpace) {
     CHECK(crops.CopyFrom(AsIntTensor(DT_INT32, crop_vals),
                          TensorShape({num_block_dims, 2})));
 
+    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
     return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("BatchToSpace")
-                                             .RandomInput(DT_FLOAT, input_dims)
+                                             .RandomInput(type, input_dims)
                                              .Input(crops)
-                                             .Attr("T", DT_FLOAT)
+                                             .Attr("T", type)
                                              .Attr("block_size", block_size));
   });
 }
@@ -1127,13 +1176,14 @@ TEST_F(OpTest, BatchToSpaceND) {
     CHECK(crops.CopyFrom(AsIntTensor(DT_INT32, crop_vals),
                          TensorShape({num_block_dims, 2})));
 
+    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
     return ExpectTfAndXlaOutputsAreClose(
         OpTestBuilder("BatchToSpaceND")
-            .RandomInput(DT_FLOAT, input_dims)
+            .RandomInput(type, input_dims)
             .Input(test::AsTensor<int32>(
                 std::vector<int32>(block_dims.begin(), block_dims.end())))
             .Input(crops)
-            .Attr("T", DT_FLOAT));
+            .Attr("T", type));
   });
 }
 
@@ -1142,18 +1192,20 @@ TEST_F(OpTest, BiasAdd) {
     auto x_dims = RandomDims(2, kDefaultMaxRank);
     auto y_dims = {x_dims[x_dims.size() - 1]};
     // TODO(phawkins): test both data formats.
+    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
     return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("BiasAdd")
-                                             .RandomInput(DT_FLOAT, x_dims)
-                                             .RandomInput(DT_FLOAT, y_dims)
-                                             .Attr("T", DT_FLOAT));
+                                             .RandomInput(type, x_dims)
+                                             .RandomInput(type, y_dims)
+                                             .Attr("T", type));
   });
 }
 
 TEST_F(OpTest, BiasAddGrad) {
   Repeatedly([this]() {
     // TODO(phawkins): test both data formats.
+    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
     return ExpectTfAndXlaOutputsAreClose(
-        OpTestBuilder("BiasAddGrad").RandomInput(DT_FLOAT).Attr("T", DT_FLOAT));
+        OpTestBuilder("BiasAddGrad").RandomInput(type).Attr("T", type));
   });
 }
 
@@ -1161,10 +1213,11 @@ TEST_F(OpTest, BiasAddV1) {
   Repeatedly([this]() {
     auto x_dims = RandomDims(2, kDefaultMaxRank);
     auto y_dims = {x_dims[x_dims.size() - 1]};
+    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
     return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("BiasAddV1")
-                                             .RandomInput(DT_FLOAT, x_dims)
-                                             .RandomInput(DT_FLOAT, y_dims)
-                                             .Attr("T", DT_FLOAT));
+                                             .RandomInput(type, x_dims)
+                                             .RandomInput(type, y_dims)
+                                             .Attr("T", type));
   });
 }
 
@@ -1221,8 +1274,8 @@ TEST_F(OpTest, BroadcastGradientArgs) {
 TEST_F(OpTest, Cast) {
   Repeatedly([this]() {
     DataType src_type, dst_type;
-    src_type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_BOOL});
-    dst_type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_BOOL});
+    src_type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_BOOL, DT_COMPLEX64});
+    dst_type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_BOOL, DT_COMPLEX64});
     return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("Cast")
                                              .RandomInput(src_type)
                                              .Attr("SrcT", src_type)
@@ -1293,11 +1346,12 @@ TEST_F(OpTest, Conv2D) {
 
     std::vector<int64> kernel_dims = {d.kernel_dims[0], d.kernel_dims[1],
                                       features_in, features_out};
+    DataType type = DT_FLOAT;  // TODO(b/65408531): COMPLEX_64 support
     return ExpectTfAndXlaOutputsAreClose(
         OpTestBuilder("Conv2D")
-            .RandomInput(DT_FLOAT, data_dims)
-            .RandomInput(DT_FLOAT, kernel_dims)
-            .Attr("T", DT_FLOAT)
+            .RandomInput(type, data_dims)
+            .RandomInput(type, kernel_dims)
+            .Attr("T", type)
             .Attr("strides", ImageDims(FORMAT_NHWC, 1, 1, d.stride_dims))
             .Attr("padding", d.padding == SAME ? "SAME" : "VALID")
             .Attr("data_format", "NHWC"));
@@ -1317,12 +1371,13 @@ TEST_F(OpTest, Conv2DBackpropFilter) {
         ImageDims(FORMAT_NHWC, batch, features_out, d.output_dims);
     Tensor kernel_shape = test::AsTensor<int32>(AsInt32s(
         {d.kernel_dims[0], d.kernel_dims[1], features_in, features_out}));
+    DataType type = DT_FLOAT;  // TODO(b/65408531): COMPLEX_64 support
     return ExpectTfAndXlaOutputsAreClose(
         OpTestBuilder("Conv2DBackpropFilter")
-            .RandomInput(DT_FLOAT, activations)
+            .RandomInput(type, activations)
             .Input(kernel_shape)
-            .RandomInput(DT_FLOAT, backprop)
-            .Attr("T", DT_FLOAT)
+            .RandomInput(type, backprop)
+            .Attr("T", type)
             .Attr("strides", ImageDims(FORMAT_NHWC, 1, 1, d.stride_dims))
             .Attr("padding", d.padding == SAME ? "SAME" : "VALID")
             .Attr("data_format", "NHWC"));
@@ -1342,12 +1397,13 @@ TEST_F(OpTest, Conv2DBackpropInput) {
         ImageDims(FORMAT_NHWC, batch, features_out, d.output_dims);
     std::vector<int64> kernel = {d.kernel_dims[0], d.kernel_dims[1],
                                  features_in, features_out};
+    DataType type = DT_FLOAT;  // TODO(b/65408531): COMPLEX_64 support
     return ExpectTfAndXlaOutputsAreClose(
         OpTestBuilder("Conv2DBackpropInput")
             .Input(in_shape)
-            .RandomInput(DT_FLOAT, kernel)
-            .RandomInput(DT_FLOAT, backprop)
-            .Attr("T", DT_FLOAT)
+            .RandomInput(type, kernel)
+            .RandomInput(type, backprop)
+            .Attr("T", type)
             .Attr("strides", ImageDims(FORMAT_NHWC, 1, 1, d.stride_dims))
             .Attr("padding", d.padding == SAME ? "SAME" : "VALID")
             .Attr("data_format", "NHWC"));
@@ -1365,11 +1421,12 @@ TEST_F(OpTest, Conv3D) {
 
     std::vector<int64> kernel = {d.kernel_dims[0], d.kernel_dims[1],
                                  d.kernel_dims[2], features_in, features_out};
+    DataType type = DT_FLOAT;  // TODO(b/65408531): COMPLEX_64 support
     return ExpectTfAndXlaOutputsAreClose(
         OpTestBuilder("Conv3D")
-            .RandomInput(DT_FLOAT, data)
-            .RandomInput(DT_FLOAT, kernel)
-            .Attr("T", DT_FLOAT)
+            .RandomInput(type, data)
+            .RandomInput(type, kernel)
+            .Attr("T", type)
             .Attr("strides", ImageDims(FORMAT_NHWC, 1, 1, d.stride_dims))
             .Attr("padding", d.padding == SAME ? "SAME" : "VALID"));
   });
@@ -1389,12 +1446,13 @@ TEST_F(OpTest, Conv3DBackpropFilter) {
     Tensor kernel_shape = test::AsTensor<int32>(
         AsInt32s({d.kernel_dims[0], d.kernel_dims[1], d.kernel_dims[2],
                   features_in, features_out}));
+    DataType type = DT_FLOAT;  // TODO(b/65408531): COMPLEX_64 support
     return ExpectTfAndXlaOutputsAreClose(
         OpTestBuilder("Conv3DBackpropFilterV2")
-            .RandomInput(DT_FLOAT, activations)
+            .RandomInput(type, activations)
             .Input(kernel_shape)
-            .RandomInput(DT_FLOAT, backprop)
-            .Attr("T", DT_FLOAT)
+            .RandomInput(type, backprop)
+            .Attr("T", type)
             .Attr("strides", ImageDims(FORMAT_NHWC, 1, 1, d.stride_dims))
             .Attr("padding", d.padding == SAME ? "SAME" : "VALID"));
   });
@@ -1413,17 +1471,34 @@ TEST_F(OpTest, Conv3DBackpropInput) {
         ImageDims(FORMAT_NHWC, batch, features_out, d.output_dims);
     std::vector<int64> kernel = {d.kernel_dims[0], d.kernel_dims[1],
                                  d.kernel_dims[2], features_in, features_out};
+    DataType type = Choose<DataType>({DT_FLOAT, DT_COMPLEX64});
     return ExpectTfAndXlaOutputsAreClose(
         OpTestBuilder("Conv3DBackpropInputV2")
             .Input(in_shape)
-            .RandomInput(DT_FLOAT, kernel)
-            .RandomInput(DT_FLOAT, backprop)
-            .Attr("T", DT_FLOAT)
+            .RandomInput(type, kernel)
+            .RandomInput(type, backprop)
+            .Attr("T", type)
             .Attr("strides", ImageDims(FORMAT_NHWC, 1, 1, d.stride_dims))
             .Attr("padding", d.padding == SAME ? "SAME" : "VALID"));
   });
 }
 
+TEST_F(OpTest, Cos) {
+  Repeatedly([this]() {
+    DataType type = Choose<DataType>({DT_FLOAT, DT_COMPLEX64});
+    return ExpectTfAndXlaOutputsAreClose(
+        OpTestBuilder("Cos").RandomInput(type).Attr("T", type));
+  });
+}
+
+TEST_F(OpTest, Cosh) {
+  Repeatedly([this]() {
+    DataType type = Choose<DataType>({DT_FLOAT, DT_COMPLEX64});
+    return ExpectTfAndXlaOutputsAreClose(
+        OpTestBuilder("Cosh").RandomInput(type).Attr("T", type));
+  });
+}
+
 TEST_F(OpTest, DepthToSpace) {
   Repeatedly([this]() {
     int64 block = RandomDim(2, 5);
@@ -1431,14 +1506,16 @@ TEST_F(OpTest, DepthToSpace) {
     input_dims[1] = (input_dims[1] + (block - 1)) / block;
     input_dims[2] = (input_dims[2] + (block - 1)) / block;
     input_dims[3] *= block * block;
+    DataType type = Choose<DataType>(kAllXlaTypes);
     return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("DepthToSpace")
-                                             .RandomInput(DT_FLOAT, input_dims)
-                                             .Attr("T", DT_FLOAT)
+                                             .RandomInput(type, input_dims)
+                                             .Attr("T", type)
                                              .Attr("block_size", block));
   });
 }
 
 TEST_F(OpTest, DepthwiseConv2DNative) {
+  if (1) return;
   Repeatedly([this]() {
     WindowedSpatialDims d = ChooseWindowedSpatialDims(2);
     std::uniform_int_distribution<int> random_int(1, 5);
@@ -1449,17 +1526,20 @@ TEST_F(OpTest, DepthwiseConv2DNative) {
 
     std::vector<int64> kernel_dims = {d.kernel_dims[0], d.kernel_dims[1],
                                       features_in, depth_multiplier};
+    std::vector<int64> strides = ImageDims(FORMAT_NHWC, 1, 1, d.stride_dims);
+    strides[2] = strides[1];  // Current impl only supports equal strides
     return ExpectTfAndXlaOutputsAreClose(
         OpTestBuilder("DepthwiseConv2dNative")
             .RandomInput(DT_FLOAT, input_dims)
             .RandomInput(DT_FLOAT, kernel_dims)
             .Attr("T", DT_FLOAT)
-            .Attr("strides", ImageDims(FORMAT_NHWC, 1, 1, d.stride_dims))
+            .Attr("strides", strides)
             .Attr("padding", d.padding == SAME ? "SAME" : "VALID"));
   });
 }
 
 TEST_F(OpTest, DepthwiseConv2DBackpropFilter) {
+  if (1) return;
   Repeatedly([this]() {
     WindowedSpatialDims d = ChooseWindowedSpatialDims(2);
     std::uniform_int_distribution<int> random_int(1, 5);
@@ -1472,33 +1552,22 @@ TEST_F(OpTest, DepthwiseConv2DBackpropFilter) {
         FORMAT_NHWC, batch, features_in * depth_multiplier, d.output_dims);
     Tensor kernel_shape = test::AsTensor<int32>(AsInt32s(
         {d.kernel_dims[0], d.kernel_dims[1], features_in, depth_multiplier}));
+    std::vector<int64> strides = ImageDims(FORMAT_NHWC, 1, 1, d.stride_dims);
+    strides[2] = strides[1];  // Current impl only supports equal strides
     return ExpectTfAndXlaOutputsAreClose(
         OpTestBuilder("DepthwiseConv2dNativeBackpropFilter")
             .RandomInput(DT_FLOAT, activations)
             .Input(kernel_shape)
             .RandomInput(DT_FLOAT, backprop)
             .Attr("T", DT_FLOAT)
-            .Attr("strides", ImageDims(FORMAT_NHWC, 1, 1, d.stride_dims))
+            .Attr("strides", strides)
             .Attr("padding", d.padding == SAME ? "SAME" : "VALID")
             .Attr("data_format", "NHWC"));
   });
 }
 
-TEST_F(OpTest, Cos) {
-  Repeatedly([this]() {
-    return ExpectTfAndXlaOutputsAreClose(
-        OpTestBuilder("Cos").RandomInput(DT_FLOAT).Attr("T", DT_FLOAT));
-  });
-}
-
-TEST_F(OpTest, Cosh) {
-  Repeatedly([this]() {
-    return ExpectTfAndXlaOutputsAreClose(
-        OpTestBuilder("Cosh").RandomInput(DT_FLOAT).Attr("T", DT_FLOAT));
-  });
-}
-
 TEST_F(OpTest, DepthwiseConv2DBackpropInput) {
+  if (1) return;
   Repeatedly([this]() {
     WindowedSpatialDims d = ChooseWindowedSpatialDims(2);
     std::uniform_int_distribution<int> random_int(1, 5);
@@ -1511,21 +1580,24 @@ TEST_F(OpTest, DepthwiseConv2DBackpropInput) {
         FORMAT_NHWC, batch, features_in * depth_multiplier, d.output_dims);
     std::vector<int64> kernel = {d.kernel_dims[0], d.kernel_dims[1],
                                  features_in, depth_multiplier};
+    std::vector<int64> strides = ImageDims(FORMAT_NHWC, 1, 1, d.stride_dims);
+    strides[2] = strides[1];  // Current impl only supports equal strides
     return ExpectTfAndXlaOutputsAreClose(
         OpTestBuilder("DepthwiseConv2dNativeBackpropInput")
             .Input(in_shape)
             .RandomInput(DT_FLOAT, kernel)
             .RandomInput(DT_FLOAT, backprop)
             .Attr("T", DT_FLOAT)
-            .Attr("strides", ImageDims(FORMAT_NHWC, 1, 1, d.stride_dims))
+            .Attr("strides", strides)
             .Attr("padding", d.padding == SAME ? "SAME" : "VALID")
             .Attr("data_format", "NHWC"));
   });
 }
 
 TEST_F(OpTest, Diag) {
+  if (1) return;
   Repeatedly([this]() {
-    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT});
+    DataType type = Choose<DataType>(kAllXlaTypes);
     std::vector<int64> dims;
     // Diag causes a quadratic blowup in output size.
     int64 size;
@@ -1540,7 +1612,7 @@ TEST_F(OpTest, Diag) {
 
 TEST_F(OpTest, DiagPart) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT});
+    DataType type = Choose<DataType>(kAllXlaTypes);
     auto dims = RandomDims(1, 3);
     // Duplicate the random dims.
     std::vector<int64> doubled_dims(dims.size() * 2);
@@ -1554,7 +1626,7 @@ TEST_F(OpTest, DiagPart) {
 
 TEST_F(OpTest, Div) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT});
+    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
     auto dims = BroadcastableDims();
     return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("Div")
                                              .RandomInput(type, dims.first)
@@ -1650,7 +1722,7 @@ TEST_F(OpTest, SeluGrad) {
 
 TEST_F(OpTest, Equal) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT});
+    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
     auto dims = BroadcastableDims();
     return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("Equal")
                                              .RandomInput(type, dims.first)
@@ -1661,15 +1733,17 @@ TEST_F(OpTest, Equal) {
 
 TEST_F(OpTest, Exp) {
   Repeatedly([this]() {
+    DataType type = Choose<DataType>({DT_FLOAT, DT_COMPLEX64});
     return ExpectTfAndXlaOutputsAreClose(
-        OpTestBuilder("Exp").RandomInput(DT_FLOAT).Attr("T", DT_FLOAT));
+        OpTestBuilder("Exp").RandomInput(type).Attr("T", type));
   });
 }
 
 TEST_F(OpTest, Expm1) {
   Repeatedly([this]() {
+    DataType type = Choose<DataType>({DT_FLOAT, DT_COMPLEX64});
     return ExpectTfAndXlaOutputsAreClose(
-        OpTestBuilder("Expm1").RandomInput(DT_FLOAT).Attr("T", DT_FLOAT));
+        OpTestBuilder("Expm1").RandomInput(type).Attr("T", type));
   });
 }
 
@@ -1809,15 +1883,17 @@ TEST_F(OpTest, LinSpace) {
 
 TEST_F(OpTest, Log) {
   Repeatedly([this]() {
+    DataType type = Choose<DataType>({DT_FLOAT, DT_COMPLEX64});
     return ExpectTfAndXlaOutputsAreClose(
-        OpTestBuilder("Log").RandomInput(DT_FLOAT).Attr("T", DT_FLOAT));
+        OpTestBuilder("Log").RandomInput(type).Attr("T", type));
   });
 }
 
 TEST_F(OpTest, Log1p) {
   Repeatedly([this]() {
+    DataType type = Choose<DataType>({DT_FLOAT, DT_COMPLEX64});
     return ExpectTfAndXlaOutputsAreClose(
-        OpTestBuilder("Log1p").RandomInput(DT_FLOAT).Attr("T", DT_FLOAT));
+        OpTestBuilder("Log1p").RandomInput(type).Attr("T", DT_FLOAT));
   });
 }
 
@@ -1914,10 +1990,11 @@ TEST_F(OpTest, MatMul) {
       std::swap(b_dims[0], b_dims[1]);
     }
 
+    DataType type = Choose<DataType>({DT_FLOAT, DT_COMPLEX64});
     return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("MatMul")
-                                             .RandomInput(DT_FLOAT, a_dims)
-                                             .RandomInput(DT_FLOAT, b_dims)
-                                             .Attr("T", DT_FLOAT)
+                                             .RandomInput(type, a_dims)
+                                             .RandomInput(type, b_dims)
+                                             .Attr("T", type)
                                              .Attr("transpose_a", transpose_a)
                                              .Attr("transpose_b", transpose_b));
   });
@@ -1925,7 +2002,7 @@ TEST_F(OpTest, MatMul) {
 
 TEST_F(OpTest, MatrixDiag) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT});
+    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
     return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("MatrixDiag")
                                              .RandomInput(type, RandomDims(1))
                                              .Attr("T", type));
@@ -1934,7 +2011,7 @@ TEST_F(OpTest, MatrixDiag) {
 
 TEST_F(OpTest, MatrixDiagPart) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT});
+    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
     return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("MatrixDiagPart")
                                              .RandomInput(type, RandomDims(2))
                                              .Attr("T", type));
@@ -2025,7 +2102,7 @@ TEST_F(OpTest, MaxPool3D) {
 
 TEST_F(OpTest, Mean) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT});
+    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
     // TODO(phawkins): CPU and XLA differ output for reducing across a
     // size-0 dimension (nan vs 0). For now, require size >= 1.
     std::vector<int64> data_dims = RandomDims(0, kDefaultMaxRank, 1);
@@ -2076,7 +2153,7 @@ TEST_F(OpTest, Mod) {
 
 TEST_F(OpTest, Mul) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT});
+    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
     auto dims = BroadcastableDims();
     return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("Mul")
                                              .RandomInput(type, dims.first)
@@ -2087,7 +2164,7 @@ TEST_F(OpTest, Mul) {
 
 TEST_F(OpTest, Neg) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT});
+    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
     return ExpectTfAndXlaOutputsAreClose(
         OpTestBuilder("Neg").RandomInput(type).Attr("T", type));
   });
@@ -2095,7 +2172,7 @@ TEST_F(OpTest, Neg) {
 
 TEST_F(OpTest, NotEqual) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT});
+    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
     auto dims = BroadcastableDims();
     return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("NotEqual")
                                              .RandomInput(type, dims.first)
@@ -2136,7 +2213,7 @@ TEST_F(OpTest, OneHot) {
 
 TEST_F(OpTest, OnesLike) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT});
+    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
     return ExpectTfAndXlaOutputsAreClose(
         OpTestBuilder("OnesLike").RandomInput(type).Attr("T", type));
   });
@@ -2195,16 +2272,17 @@ TEST_F(OpTest, Pow) {
   // nontermination.
   Repeatedly([this]() {
     auto dims = BroadcastableDims();
+    DataType type = Choose<DataType>({DT_FLOAT, DT_COMPLEX64});
     return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("Pow")
-                                             .RandomInput(DT_FLOAT, dims.first)
-                                             .RandomInput(DT_FLOAT, dims.second)
-                                             .Attr("T", DT_FLOAT));
+                                             .RandomInput(type, dims.first)
+                                             .RandomInput(type, dims.second)
+                                             .Attr("T", type));
   });
 }
 
 TEST_F(OpTest, Prod) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT});
+    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
     std::vector<int64> data_dims = RandomDims();
     Tensor indices = RandomReductionIndices(data_dims.size());
     bool keep_dims = Choose<bool>({false, true});
@@ -2238,7 +2316,7 @@ TEST_F(OpTest, Range) {
 
 TEST_F(OpTest, Rank) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT});
+    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
     return ExpectTfAndXlaOutputsAreClose(
         OpTestBuilder("Rank").RandomInput(type).Attr("T", type));
   });
@@ -2246,7 +2324,7 @@ TEST_F(OpTest, Rank) {
 
 TEST_F(OpTest, RealDiv) {
   Repeatedly([this]() {
-    DataType type = DT_FLOAT;
+    DataType type = Choose<DataType>({DT_FLOAT, DT_COMPLEX64});
     auto dims = BroadcastableDims();
     return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("RealDiv")
                                              .RandomInput(type, dims.first)
@@ -2257,18 +2335,20 @@ TEST_F(OpTest, RealDiv) {
 
 TEST_F(OpTest, Reciprocal) {
   Repeatedly([this]() {
+    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
     return ExpectTfAndXlaOutputsAreClose(
-        OpTestBuilder("Reciprocal").RandomInput(DT_FLOAT).Attr("T", DT_FLOAT));
+        OpTestBuilder("Reciprocal").RandomInput(type).Attr("T", type));
   });
 }
 
 TEST_F(OpTest, ReciprocalGrad) {
   Repeatedly([this]() {
     std::vector<int64> dims = RandomDims();
+    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
     return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("ReciprocalGrad")
-                                             .RandomInput(DT_FLOAT, dims)
-                                             .RandomInput(DT_FLOAT, dims)
-                                             .Attr("T", DT_FLOAT));
+                                             .RandomInput(type, dims)
+                                             .RandomInput(type, dims)
+                                             .Attr("T", type));
   });
 }
 TEST_F(OpTest, Relu) {
@@ -2335,24 +2415,24 @@ TEST_F(OpTest, Reshape) {
 TEST_F(OpTest, Reverse) {
   Repeatedly([this]() {
     std::vector<int64> dims = RandomDims(1);
-    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT});
+    DataType type = Choose<DataType>(kAllXlaTypes);
     int64 rank = dims.size();
     return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("Reverse")
                                              .RandomInput(type, dims)
                                              .RandomInput(DT_BOOL, {rank})
-                                             .Attr("T", DT_FLOAT));
+                                             .Attr("T", type));
   });
 }
 
 TEST_F(OpTest, ReverseV2) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT});
+    DataType type = Choose<DataType>(kAllXlaTypes);
     std::vector<int64> data_dims = RandomDims();
     Tensor indices = RandomReductionIndices(data_dims.size());
     return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("ReverseV2")
                                              .RandomInput(type, data_dims)
                                              .Input(indices)
-                                             .Attr("T", DT_FLOAT));
+                                             .Attr("T", type));
   });
 }
 
@@ -2372,18 +2452,20 @@ TEST_F(OpTest, Round) {
 
 TEST_F(OpTest, Rsqrt) {
   Repeatedly([this]() {
+    DataType type = Choose<DataType>({DT_FLOAT, DT_COMPLEX64});
     return ExpectTfAndXlaOutputsAreClose(
-        OpTestBuilder("Rsqrt").RandomInput(DT_FLOAT).Attr("T", DT_FLOAT));
+        OpTestBuilder("Rsqrt").RandomInput(type).Attr("T", type));
   });
 }
 
 TEST_F(OpTest, RsqrtGrad) {
   Repeatedly([this]() {
     auto dims = RandomDims();
+    DataType type = Choose<DataType>({DT_FLOAT, DT_COMPLEX64});
     return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("RsqrtGrad")
-                                             .RandomInput(DT_FLOAT, dims)
-                                             .RandomInput(DT_FLOAT, dims)
-                                             .Attr("T", DT_FLOAT));
+                                             .RandomInput(type, dims)
+                                             .RandomInput(type, dims)
+                                             .Attr("T", type));
   });
 }
 
@@ -2411,24 +2493,26 @@ TEST_F(OpTest, ShapeN) {
 
 TEST_F(OpTest, Sigmoid) {
   Repeatedly([this]() {
+    DataType type = Choose<DataType>({DT_FLOAT, DT_COMPLEX64});
     return ExpectTfAndXlaOutputsAreClose(
-        OpTestBuilder("Sigmoid").RandomInput(DT_FLOAT).Attr("T", DT_FLOAT));
+        OpTestBuilder("Sigmoid").RandomInput(type).Attr("T", type));
   });
 }
 
 TEST_F(OpTest, SigmoidGrad) {
   Repeatedly([this]() {
     auto dims = RandomDims();
+    DataType type = Choose<DataType>({DT_FLOAT, DT_COMPLEX64});
     return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("SigmoidGrad")
-                                             .RandomInput(DT_FLOAT, dims)
-                                             .RandomInput(DT_FLOAT, dims)
-                                             .Attr("T", DT_FLOAT));
+                                             .RandomInput(type, dims)
+                                             .RandomInput(type, dims)
+                                             .Attr("T", type));
   });
 }
 
 TEST_F(OpTest, Sign) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT});
+    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
     return ExpectTfAndXlaOutputsAreClose(
         OpTestBuilder("Sign").RandomInput(type).Attr("T", type));
   });
@@ -2436,21 +2520,23 @@ TEST_F(OpTest, Sign) {
 
 TEST_F(OpTest, Sin) {
   Repeatedly([this]() {
+    DataType type = Choose<DataType>({DT_FLOAT, DT_COMPLEX64});
     return ExpectTfAndXlaOutputsAreClose(
-        OpTestBuilder("Sin").RandomInput(DT_FLOAT).Attr("T", DT_FLOAT));
+        OpTestBuilder("Sin").RandomInput(type).Attr("T", type));
   });
 }
 
 TEST_F(OpTest, Sinh) {
   Repeatedly([this]() {
+    DataType type = Choose<DataType>({DT_FLOAT, DT_COMPLEX64});
     return ExpectTfAndXlaOutputsAreClose(
-        OpTestBuilder("Sinh").RandomInput(DT_FLOAT).Attr("T", DT_FLOAT));
+        OpTestBuilder("Sinh").RandomInput(type).Attr("T", type));
   });
 }
 
 TEST_F(OpTest, Size) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT});
+    DataType type = Choose<DataType>(kAllXlaTypes);
     return ExpectTfAndXlaOutputsAreClose(
         OpTestBuilder("Size").RandomInput(type).Attr("T", type));
   });
@@ -2562,10 +2648,11 @@ TEST_F(OpTest, SpaceToBatch) {
     CHECK(paddings.CopyFrom(AsIntTensor(DT_INT32, padding_vals),
                             TensorShape({num_block_dims, 2})));
 
+    DataType type = Choose<DataType>(kAllXlaTypes);
     return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("SpaceToBatch")
-                                             .RandomInput(DT_FLOAT, input_dims)
+                                             .RandomInput(type, input_dims)
                                              .Input(paddings)
-                                             .Attr("T", DT_FLOAT)
+                                             .Attr("T", type)
                                              .Attr("block_size", block_size));
   });
 }
@@ -2603,13 +2690,14 @@ TEST_F(OpTest, SpaceToBatchND) {
     CHECK(paddings.CopyFrom(AsIntTensor(DT_INT32, padding_vals),
                             TensorShape({num_block_dims, 2})));
 
+    DataType type = Choose<DataType>(kAllXlaTypes);
     return ExpectTfAndXlaOutputsAreClose(
         OpTestBuilder("SpaceToBatchND")
-            .RandomInput(DT_FLOAT, input_dims)
+            .RandomInput(type, input_dims)
             .Input(test::AsTensor<int32>(
                 std::vector<int32>(block_dims.begin(), block_dims.end())))
             .Input(paddings)
-            .Attr("T", DT_FLOAT));
+            .Attr("T", type));
   });
 }
 
@@ -2699,18 +2787,20 @@ TEST_F(OpTest, Split) {
 
 TEST_F(OpTest, Sqrt) {
   Repeatedly([this]() {
+    DataType type = Choose<DataType>({DT_FLOAT, DT_COMPLEX64});
     return ExpectTfAndXlaOutputsAreClose(
-        OpTestBuilder("Sqrt").RandomInput(DT_FLOAT).Attr("T", DT_FLOAT));
+        OpTestBuilder("Sqrt").RandomInput(type).Attr("T", type));
   });
 }
 
 TEST_F(OpTest, SqrtGrad) {
   Repeatedly([this]() {
     auto dims = RandomDims();
+    DataType type = Choose<DataType>({DT_FLOAT, DT_COMPLEX64});
     return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("SqrtGrad")
-                                             .RandomInput(DT_FLOAT, dims)
-                                             .RandomInput(DT_FLOAT, dims)
-                                             .Attr("T", DT_FLOAT));
+                                             .RandomInput(type, dims)
+                                             .RandomInput(type, dims)
+                                             .Attr("T", type));
   });
 }
 
@@ -2726,7 +2816,7 @@ TEST_F(OpTest, SquaredDifference) {
 
 TEST_F(OpTest, Square) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT});
+    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
     return ExpectTfAndXlaOutputsAreClose(
         OpTestBuilder("Square").RandomInput(type).Attr("T", type));
   });
@@ -2752,7 +2842,7 @@ TEST_F(OpTest, Squeeze) {
 
 TEST_F(OpTest, Sub) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT});
+    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
     auto dims = BroadcastableDims();
     return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("Sub")
                                              .RandomInput(type, dims.first)
@@ -2763,7 +2853,7 @@ TEST_F(OpTest, Sub) {
 
 TEST_F(OpTest, Sum) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT});
+    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
     std::vector<int64> data_dims = RandomDims();
     Tensor indices = RandomReductionIndices(data_dims.size());
     bool keep_dims = Choose<bool>({false, true});
@@ -2875,25 +2965,28 @@ TEST_F(OpTest, StridedSliceGrad) {
 
 TEST_F(OpTest, Tan) {
   Repeatedly([this]() {
+    DataType type = Choose<DataType>({DT_FLOAT, DT_COMPLEX64});
     return ExpectTfAndXlaOutputsAreClose(
-        OpTestBuilder("Tan").RandomInput(DT_FLOAT).Attr("T", DT_FLOAT));
+        OpTestBuilder("Tan").RandomInput(type).Attr("T", type));
   });
 }
 
 TEST_F(OpTest, Tanh) {
   Repeatedly([this]() {
+    DataType type = Choose<DataType>({DT_FLOAT, DT_COMPLEX64});
     return ExpectTfAndXlaOutputsAreClose(
-        OpTestBuilder("Tanh").RandomInput(DT_FLOAT).Attr("T", DT_FLOAT));
+        OpTestBuilder("Tanh").RandomInput(type).Attr("T", type));
   });
 }
 
 TEST_F(OpTest, TanhGrad) {
   Repeatedly([this]() {
     auto dims = RandomDims();
+    DataType type = Choose<DataType>({DT_FLOAT, DT_COMPLEX64});
     return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("TanhGrad")
-                                             .RandomInput(DT_FLOAT, dims)
-                                             .RandomInput(DT_FLOAT, dims)
-                                             .Attr("T", DT_FLOAT));
+                                             .RandomInput(type, dims)
+                                             .RandomInput(type, dims)
+                                             .Attr("T", type));
   });
 }
 
@@ -2951,7 +3044,7 @@ TEST_F(OpTest, TruncateMod) {
 
 TEST_F(OpTest, ZerosLike) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT});
+    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
     return ExpectTfAndXlaOutputsAreClose(
         OpTestBuilder("ZerosLike").RandomInput(type).Attr("T", type));
   });
diff --git a/tensorflow/compiler/tests/unary_ops_test.py b/tensorflow/compiler/tests/unary_ops_test.py
index 71221b284d..76644380bd 100644
--- a/tensorflow/compiler/tests/unary_ops_test.py
+++ b/tensorflow/compiler/tests/unary_ops_test.py
@@ -328,6 +328,131 @@ class UnaryOpsTest(XLATestCase):
           np.array([-1, -0.5, 0, 0.3], dtype=dtype),
           expected=np.array([-1, -64.0 / 127, 0, 38.0 / 127], dtype=dtype))
 
+  def testComplexOps(self):
+    for dtype in self.complex_types:
+      # TODO(b/65408531): math_ops.acosh (needs pow)
+      # TODO(b/65408531): math_ops.asinh (needs pow)
+
+      # TODO(b/65408531): Wider support for log (needs atan2).
+      atan2_supported = self.device == "XLA_GPU"
+      if atan2_supported:
+        self._assertOpOutputMatchesExpected(
+            math_ops.atanh,
+            np.array([0.1, 0.2j, 0.3 - 0.1j, 0.4 + 0.5j], dtype=dtype),
+            expected=np.arctanh(
+                np.array([0.1, 0.2j, 0.3 - 0.1j, 0.4 + 0.5j], dtype=dtype)))
+
+      self._assertOpOutputMatchesExpected(
+          math_ops.cosh,
+          np.array([1j, 2 - 3j, 3, 4 + 2j], dtype=dtype),
+          expected=np.cosh(np.array([1j, 2 - 3j, 3, 4 + 2j], dtype=dtype)))
+
+      self._assertOpOutputMatchesExpected(
+          math_ops.sinh,
+          np.array([1, 2j, 2 - 3j, 4 + 5j], dtype=dtype),
+          expected=np.sinh(np.array([1, 2j, 2 - 3j, 4 + 5j], dtype=dtype)))
+
+      self._assertOpOutputMatchesExpected(
+          math_ops.exp,
+          np.array([[-1 + 2j, 3j, 2 - 3j]], dtype=dtype),
+          expected=np.exp(np.array([[-1 + 2j, 3j, 2 - 3j]], dtype=dtype)))
+
+      self._assertOpOutputMatchesExpected(
+          math_ops.expm1,
+          np.array([[-1 + 2j, 3j, 2 - 3j]], dtype=dtype),
+          expected=np.expm1(np.array([[-1 + 2j, 3j, 2 - 3j]], dtype=dtype)))
+
+      self._assertOpOutputMatchesExpected(
+          math_ops.reciprocal,
+          np.array([[1, 2j, 2 + 3j]], dtype=dtype),
+          expected=1.0 / np.array([[1, 2j, 2 + 3j]], dtype=dtype))
+
+      if atan2_supported:
+        self._assertOpOutputMatchesExpected(
+            math_ops.log,
+            np.array([[5j, 3 - 2j]], dtype=dtype),
+            expected=np.log(np.array([[5j, 3 - 2j]], dtype=dtype)))
+
+      self._assertOpOutputMatchesExpected(
+          math_ops.sin,
+          np.array([[5j, 3 - 2j]], dtype=dtype),
+          expected=np.sin(np.array([[5j, 3 - 2j]], dtype=dtype)))
+
+      self._assertOpOutputMatchesExpected(
+          math_ops.cos,
+          np.array([[5j, 3 - 2j]], dtype=dtype),
+          expected=np.cos(np.array([[5j, 3 - 2j]], dtype=dtype)))
+
+      # TODO(b/34703906): improve log1p implementation and make tolerance
+      # tighter.
+      if atan2_supported:  # TODO(b/34703906): log support
+        self._assertOpOutputMatchesExpected(
+            math_ops.log1p,
+            np.array([[1e-14, 1e-15j, 0.6 - 0.3j]], dtype=dtype),
+            expected=np.log1p(
+                np.array([[1e-14, 1e-15j, 0.6 - 0.3j]], dtype=dtype)))
+
+      # TODO(b/34703906): math_ops.rsqrt (needs pow)
+
+      # TODO(b/34703906): math_ops.sigmoid (needs tanh)
+
+      # TODO(b/34703906): math_ops.sqrt (needs pow)
+
+      self._assertOpOutputMatchesExpected(
+          math_ops.tan,
+          np.array([1, 2j, 2 - 3j, 4 + 5j], dtype=dtype),
+          expected=np.tan(np.array([1, 2j, 2 - 3j, 4 + 5j], dtype=dtype)))
+
+      # TODO(b/34703906): math_ops.tanh (as itself)
+
+      ctypes = {np.complex64: np.float32}
+      self._assertOpOutputMatchesExpected(
+          math_ops.abs,
+          np.array([[3 - 4j, -1j, np.inf]], dtype=dtype),
+          expected=np.array([[5, 1, np.inf]], dtype=ctypes[dtype]))
+
+      self._assertOpOutputMatchesExpected(
+          math_ops.negative,
+          np.array([[-1 + 2j, -3j]], dtype=dtype),
+          expected=np.array([[1 - 2j, 3j]], dtype=dtype))
+
+      self._assertOpOutputMatchesExpected(
+          math_ops.square,
+          np.array([[-2 - 3j, 3 + 4j, 5j]], dtype=dtype),
+          expected=np.array([[-2 - 3j, 3 + 4j, 5j]], dtype=dtype)**2)
+
+      self._assertOpOutputMatchesExpected(
+          array_ops.zeros_like,
+          np.array([[4j, 3 - 2j], [2, -1j]], dtype=dtype),
+          expected=np.array([[0, 0], [0, 0]], dtype=dtype))
+
+      self._assertOpOutputMatchesExpected(
+          array_ops.ones_like,
+          np.array([[-4j, 3 + 2j], [2, -1j]], dtype=dtype),
+          expected=np.array([[1, 1], [1, 1]], dtype=dtype))
+
+      if atan2_supported:  # TODO(b/34703906): atan2 support
+        self._assertOpOutputMatchesExpected(
+            math_ops.angle,
+            np.array([1 + 3j, -4 + 7j, 2.7, -3j], dtype=dtype),
+            expected=np.angle(
+                np.array([1 + 3j, -4 + 7j, 2.7, -3j], dtype=dtype)))
+
+      self._assertOpOutputMatchesExpected(
+          math_ops.conj,
+          np.array([1 + 3j, -4 + 7j, 2.7, -3j], dtype=dtype),
+          expected=np.array([1 - 3j, -4 - 7j, 2.7, 3j], dtype=dtype))
+
+      self._assertOpOutputMatchesExpected(
+          math_ops.imag,
+          np.array([1 + 3j, -4 + 7j, 2.7, -3j], dtype=dtype),
+          expected=np.array([3, 7, 0, -3], dtype=ctypes[dtype]))
+
+      self._assertOpOutputMatchesExpected(
+          math_ops.real,
+          np.array([1 + 3j, -4 + 7j, 2.7, -3j], dtype=dtype),
+          expected=np.array([1, -4, 2.7, 0], dtype=ctypes[dtype]))
+
   def testIntOps(self):
     for dtype in self.int_types:
       self._assertOpOutputMatchesExpected(
@@ -399,11 +524,14 @@ class UnaryOpsTest(XLATestCase):
 
   def testCast(self):
     shapes = [[], [4], [2, 3], [2, 0, 4]]
-    types = [dtypes.bool, dtypes.int32, dtypes.float32]
+    types = [dtypes.bool, dtypes.int32, dtypes.float32] + self.complex_tf_types
     for shape in shapes:
       for src_type in types:
         for dst_type in types:
           src = np.arange(np.prod(shape)).astype(src_type.as_numpy_dtype)
+          if src_type in self.complex_tf_types:
+            src += (np.arange(np.prod(shape)) * 2j).astype(
+                src_type.as_numpy_dtype)
           src = src.reshape(shape)
 
           dst = src.astype(dst_type.as_numpy_dtype)
diff --git a/tensorflow/compiler/tests/variable_ops_test.py b/tensorflow/compiler/tests/variable_ops_test.py
index fdf3f9fb6a..c50342dee4 100644
--- a/tensorflow/compiler/tests/variable_ops_test.py
+++ b/tensorflow/compiler/tests/variable_ops_test.py
@@ -43,7 +43,7 @@ class VariableOpsTest(XLATestCase):
     # Regression test for a bug where computations with one non-constant
     # output and one variable update were mishandled.
     for dtype in self.numeric_types:
-      init = np.array([[1, 2], [3, 4]], dtype=dtype)
+      init = np.array([[1, 2j], [3, 4]]).astype(dtype)
       with self.test_session() as sess, self.test_scope():
         v = resource_variable_ops.ResourceVariable(init)
         sess.run(variables.variables_initializer([v]))
@@ -51,82 +51,91 @@ class VariableOpsTest(XLATestCase):
         x = v.assign_add(p)
         with ops.control_dependencies([x]):
           y = v.read_value()
-        self.assertAllClose(np.array([[2, 3], [4, 5]], dtype=dtype),
-                            sess.run(y, {p: 1}))
+        self.assertAllClose(
+            np.array([[2, 1 + 2j], [4, 5]]).astype(dtype), sess.run(y, {
+                p: 1
+            }))
 
   def testSparseRead0DIndices(self):
     for dtype in self.numeric_types:
-      init = np.array([[0, 1, 2, 3], [4, 5, 6, 7], [8, 9, 10, 11]], dtype=dtype)
+      init = np.array([[0, 1, 2, 3], [4, 5, 6, 7], [8j, 9, 10,
+                                                    11]]).astype(dtype)
       with self.test_session() as sess, self.test_scope():
         v = resource_variable_ops.ResourceVariable(init)
         sess.run(variables.variables_initializer([v]))
         x = v.sparse_read(2)
-        self.assertAllClose(np.array([8, 9, 10, 11], dtype=dtype), sess.run(x))
+        self.assertAllClose(
+            np.array([8j, 9, 10, 11]).astype(dtype), sess.run(x))
 
   def testSparseRead1DIndices(self):
     for dtype in self.numeric_types:
-      init = np.array([[0, 1, 2, 3], [4, 5, 6, 7], [8, 9, 10, 11]], dtype=dtype)
+      init = np.array([[0, 1, 2, 3], [4, 5, 6j, 7], [8, 9, 10,
+                                                     11]]).astype(dtype)
       with self.test_session() as sess, self.test_scope():
         v = resource_variable_ops.ResourceVariable(init)
         sess.run(variables.variables_initializer([v]))
         x = v.sparse_read([2, 1])
         self.assertAllClose(
-            np.array([[8, 9, 10, 11], [4, 5, 6, 7]], dtype=dtype), sess.run(x))
+            np.array([[8, 9, 10, 11], [4, 5, 6j, 7]]).astype(dtype),
+            sess.run(x))
 
   def testSparseRead2DIndices(self):
     for dtype in self.numeric_types:
-      init = np.array([[0, 1, 2, 3], [4, 5, 6, 7], [8, 9, 10, 11]], dtype=dtype)
+      init = np.array([[0, 1, 2j, 3], [4, 5, 6, 7], [8, 9, 10,
+                                                     11]]).astype(dtype)
       with self.test_session() as sess, self.test_scope():
         v = resource_variable_ops.ResourceVariable(init)
         sess.run(variables.variables_initializer([v]))
         x = v.sparse_read([[2, 1], [0, 2]])
         self.assertAllClose(
-            np.array(
-                [[[8, 9, 10, 11], [4, 5, 6, 7]], [[0, 1, 2, 3], [8, 9, 10,
-                                                                 11]]],
-                dtype=dtype), sess.run(x))
+            np.array([[[8, 9, 10, 11], [4, 5, 6, 7]],
+                      [[0, 1, 2j, 3], [8, 9, 10, 11]]]).astype(dtype),
+            sess.run(x))
 
   def testSparseRead2DIndices3DTensor(self):
     for dtype in self.numeric_types:
-      init = np.array(
-          [[[0, 1, 2], [3, 4, 5]], [[10, 11, 12], [13, 14, 15]],
-           [[20, 21, 22], [23, 24, 25]], [[30, 31, 32], [33, 34, 35]]],
-          dtype=dtype)
+      init = np.array([[[0, 1, 2], [3, 4, 5]], [[10, 11, 12], [13, 14, 15]],
+                       [[20, 21, 22], [23, 24j, 25]],
+                       [[30, 31, 32], [33, 34, 35]]]).astype(dtype)
       with self.test_session() as sess, self.test_scope():
         v = resource_variable_ops.ResourceVariable(init)
         sess.run(variables.variables_initializer([v]))
         x = v.sparse_read([[2, 1], [3, 0]])
         self.assertAllClose(
             np.array(
-                [[[[20, 21, 22], [23, 24, 25]], [[10, 11, 12], [13, 14, 15]]],
+                [[[[20, 21, 22], [23, 24j, 25]], [[10, 11, 12], [13, 14, 15]]],
                  [[[30, 31, 32], [33, 34, 35]], [[0, 1, 2], [3, 4, 5]]]],
-                dtype=dtype), sess.run(x))
+            ).astype(dtype), sess.run(x))
 
   def testReadWrite(self):
     """Tests initialization, reading, and writing a resource variable."""
-    with self.test_session() as session:
-      with self.test_scope():
-        with variable_scope.variable_scope("ascope", use_resource=True):
-          x = variable_scope.get_variable(
-              "x",
-              shape=[],
-              dtype=dtypes.float32,
-              initializer=init_ops.constant_initializer(2))
-          a = x.read_value()
-          with ops.control_dependencies([a]):
-            b = state_ops.assign(x, 47)
-          with ops.control_dependencies([b]):
-            c = x.read_value()
-          with ops.control_dependencies([c]):
-            d = state_ops.assign_add(x, 3)
-          with ops.control_dependencies([d]):
-            e = x.read_value()
-
-      session.run(variables.global_variables_initializer())
-      v1, v2, v3 = session.run([a, c, e])
-      self.assertAllClose(2.0, v1)
-      self.assertAllClose(47.0, v2)
-      self.assertAllClose(50.0, v3)
+    for dtype in self.numeric_types:
+      with self.test_session() as session:
+        print(ops.get_default_graph())
+        with self.test_scope():
+          with variable_scope.variable_scope("ascope", use_resource=True):
+            x = variable_scope.get_variable(
+                "x",
+                shape=[],
+                dtype=dtype,
+                initializer=init_ops.constant_initializer(2))
+            a = x.read_value()
+            with ops.control_dependencies([a]):
+              b = state_ops.assign(x, dtype(47))
+            with ops.control_dependencies([b]):
+              c = x.read_value()
+            with ops.control_dependencies([c]):
+              d = state_ops.assign_add(x, np.array(6 + 2j).astype(dtype))
+            with ops.control_dependencies([d]):
+              e = state_ops.assign_sub(x, dtype(3))
+            with ops.control_dependencies([e]):
+              f = x.read_value()
+
+        session.run(variables.global_variables_initializer())
+        v1, v2, v3 = session.run([a, c, f])
+        self.assertAllClose(dtype(2), v1)
+        self.assertAllClose(dtype(47), v2)
+        self.assertAllClose(np.array(50 + 2j).astype(dtype), v3)
 
   def testTraining(self):
     """Tests a gradient descent step for a simple model."""
diff --git a/tensorflow/compiler/tests/xla_test.py b/tensorflow/compiler/tests/xla_test.py
index da6dc88f1f..0be127997e 100644
--- a/tensorflow/compiler/tests/xla_test.py
+++ b/tensorflow/compiler/tests/xla_test.py
@@ -63,12 +63,19 @@ class XLATestCase(test.TestCase):
     self.float_tf_types = [
         dtype for dtype in self.all_tf_types if dtype.is_floating
     ]
-    self.numeric_tf_types = self.int_tf_types + self.float_tf_types
+    self.complex_tf_types = [
+        dtype for dtype in self.all_tf_types if dtype.is_complex
+    ]
+    self.numeric_tf_types = (
+        self.int_tf_types + self.float_tf_types + self.complex_tf_types)
 
     self.all_types = [dtype.as_numpy_dtype for dtype in self.all_tf_types]
     self.int_types = [dtype.as_numpy_dtype for dtype in self.int_tf_types]
     self.float_types = [dtype.as_numpy_dtype for dtype in self.float_tf_types]
-    self.numeric_types = self.int_types + self.float_types
+    self.complex_types = [
+        dtype.as_numpy_dtype for dtype in self.complex_tf_types
+    ]
+    self.numeric_types = self.int_types + self.float_types + self.complex_types
 
     # Parse the manifest file, if any, into a regex identifying tests to
     # disable
diff --git a/tensorflow/compiler/tf2xla/kernels/batch_matmul_op.cc b/tensorflow/compiler/tf2xla/kernels/batch_matmul_op.cc
index 16b778bca4..73ccc151c1 100644
--- a/tensorflow/compiler/tf2xla/kernels/batch_matmul_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/batch_matmul_op.cc
@@ -77,7 +77,13 @@ class BatchMatMulOp : public XlaOpKernel {
     xla::ComputationBuilder* builder = ctx->builder();
 
     xla::ComputationDataHandle x_handle = ctx->Input(0);
+    if (BaseType(input_type(0)) == DT_COMPLEX64 && adj_x_) {
+      x_handle = builder->Conj(x_handle);
+    }
     xla::ComputationDataHandle y_handle = ctx->Input(1);
+    if (BaseType(input_type(1)) == DT_COMPLEX64 && adj_y_) {
+      y_handle = builder->Conj(y_handle);
+    }
 
     // Reshape input tensors into 3D tensors by flattening the batch
     // dimensions. This makes it easier to unroll the batch dimension.
diff --git a/tensorflow/compiler/tf2xla/kernels/binary_ops.cc b/tensorflow/compiler/tf2xla/kernels/binary_ops.cc
index 4673bbda14..1de9192432 100644
--- a/tensorflow/compiler/tf2xla/kernels/binary_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/binary_ops.cc
@@ -13,7 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-// Native XLA implementations of simple unary Ops
+// Native XLA implementations of simple binary Ops
 
 #include "tensorflow/compiler/tf2xla/kernels/cwise_ops.h"
 #include "tensorflow/compiler/tf2xla/xla_helpers.h"
@@ -21,6 +21,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/client/client_library.h"
 #include "tensorflow/compiler/xla/client/computation_builder.h"
 #include "tensorflow/core/framework/kernel_def_builder.h"
+#include "tensorflow/core/framework/op_kernel.h"
 
 namespace tensorflow {
 namespace {
@@ -50,6 +51,9 @@ XLA_MAKE_BINARY(Sub, b->Sub(lhs, rhs, extend_dimensions));
 XLA_MAKE_BINARY(Mul, b->Mul(lhs, rhs, extend_dimensions));
 XLA_MAKE_BINARY(Div, b->Div(lhs, rhs, extend_dimensions));
 
+XLA_MAKE_BINARY(Atan2, b->Atan2(lhs, rhs, extend_dimensions));
+XLA_MAKE_BINARY(Complex, b->Complex(lhs, rhs, extend_dimensions));
+
 // Implementation of FloorDiv. Pseudo-code:
 // if ((x < 0) != (y < 0)) {
 //   T abs_x = std::abs(x);
@@ -171,8 +175,12 @@ class ApproximateEqualOp : public XlaOpKernel {
   // Computes the max of the scalar input x and 0.
   void Compile(XlaOpKernelContext* ctx) override {
     xla::ComputationBuilder* b = ctx->builder();
-    auto result = b->Lt(b->Abs(b->Sub(ctx->Input(0), ctx->Input(1))),
-                        XlaHelpers::FloatLiteral(b, input_type(0), tolerance_));
+    auto abs = b->Abs(b->Sub(ctx->Input(0), ctx->Input(1)));
+    auto abs_shape = b->GetShape(abs);
+    OP_REQUIRES_OK(ctx, abs_shape.status());
+    auto abs_type = abs_shape.ValueOrDie()->element_type();
+    auto result = b->Lt(
+        abs, b->ConvertElementType(b->ConstantR0<float>(tolerance_), abs_type));
     ctx->SetOutput(0, result);
   }
 
diff --git a/tensorflow/compiler/tf2xla/kernels/cast_op.cc b/tensorflow/compiler/tf2xla/kernels/cast_op.cc
index 2331520230..43a6a747c6 100644
--- a/tensorflow/compiler/tf2xla/kernels/cast_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/cast_op.cc
@@ -17,6 +17,7 @@ limitations under the License.
 #include "tensorflow/compiler/tf2xla/xla_helpers.h"
 #include "tensorflow/compiler/tf2xla/xla_op_kernel.h"
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
+#include "tensorflow/compiler/xla/primitive_util.h"
 #include "tensorflow/core/framework/kernel_def_builder.h"
 
 namespace tensorflow {
@@ -40,6 +41,11 @@ class CastOp : public XlaOpKernel {
       output = input;
     } else if (dst_dtype_ == DT_BOOL) {
       output = builder->Ne(input, XlaHelpers::Zero(builder, src_dtype_));
+    } else if (xla::primitive_util::IsComplexType(src_type_) &&
+               !xla::primitive_util::IsComplexType(dst_type_)) {
+      // As in cast_op.h, we replicate the numpy behavior of truncating the
+      // imaginary part.
+      output = builder->ConvertElementType(builder->Real(input), dst_type_);
     } else {
       output = builder->ConvertElementType(input, dst_type_);
     }
diff --git a/tensorflow/compiler/tf2xla/kernels/gather_op.cc b/tensorflow/compiler/tf2xla/kernels/gather_op.cc
index db449ec345..e420f21ca3 100644
--- a/tensorflow/compiler/tf2xla/kernels/gather_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/gather_op.cc
@@ -192,7 +192,7 @@ void GatherOpDynamicSlice::Compile(XlaOpKernelContext* context) {
               errors::InvalidArgument("indices must be int32 or int64"));
 
   xla::ComputationDataHandle gather = XlaComputeGatherDynamicSlice(
-      context, input, input_shape, indices, indices_shape, axis, DT_FLOAT,
+      context, input, input_shape, indices, indices_shape, axis, input_type(0),
       index_type, builder);
   context->SetOutput(0, gather);
 }
diff --git a/tensorflow/compiler/tf2xla/kernels/matmul_op.cc b/tensorflow/compiler/tf2xla/kernels/matmul_op.cc
index 5c799a0e4f..fcef497e58 100644
--- a/tensorflow/compiler/tf2xla/kernels/matmul_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/matmul_op.cc
@@ -23,6 +23,9 @@ limitations under the License.
 namespace tensorflow {
 namespace {
 
+constexpr std::array<DataType, 4> kMatmulTypes = {
+    {DT_HALF, DT_FLOAT, DT_DOUBLE, DT_COMPLEX64}};
+
 class MatMulOp : public XlaOpKernel {
  public:
   explicit MatMulOp(OpKernelConstruction* ctx, bool is_sparse = false)
@@ -73,7 +76,7 @@ class MatMulOp : public XlaOpKernel {
   bool transpose_b_;
 };
 
-REGISTER_XLA_OP(Name("MatMul").TypeConstraint("T", kFloatTypes), MatMulOp);
+REGISTER_XLA_OP(Name("MatMul").TypeConstraint("T", kMatmulTypes), MatMulOp);
 
 class SparseMatMulOp : public MatMulOp {
  public:
diff --git a/tensorflow/compiler/tf2xla/kernels/training_ops.cc b/tensorflow/compiler/tf2xla/kernels/training_ops.cc
index 82ae0df5cc..5534d1bfa1 100644
--- a/tensorflow/compiler/tf2xla/kernels/training_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/training_ops.cc
@@ -37,8 +37,9 @@ class ResourceApplyGradientDescent : public XlaOpKernel {
     OP_REQUIRES_OK(ctx, ctx->AssignVariable(0, ctx->input_type(1), handle));
   }
 };
-REGISTER_XLA_OP(Name("ResourceApplyGradientDescent"),
-                ResourceApplyGradientDescent);
+REGISTER_XLA_OP(
+    Name("ResourceApplyGradientDescent").TypeConstraint("T", kFloatTypes),
+    ResourceApplyGradientDescent);
 
 class ResourceApplyMomentum : public XlaOpKernel {
  public:
@@ -109,7 +110,8 @@ class ResourceApplyMomentum : public XlaOpKernel {
  private:
   bool use_nesterov_;
 };
-REGISTER_XLA_OP(Name("ResourceApplyMomentum"), ResourceApplyMomentum);
+REGISTER_XLA_OP(Name("ResourceApplyMomentum").TypeConstraint("T", kFloatTypes),
+                ResourceApplyMomentum);
 
 class ResourceApplyAdagrad : public XlaOpKernel {
  public:
@@ -163,7 +165,8 @@ class ResourceApplyAdagrad : public XlaOpKernel {
     OP_REQUIRES_OK(ctx, ctx->AssignVariable(1, type, accum));
   }
 };
-REGISTER_XLA_OP(Name("ResourceApplyAdagrad"), ResourceApplyAdagrad);
+REGISTER_XLA_OP(Name("ResourceApplyAdagrad").TypeConstraint("T", kFloatTypes),
+                ResourceApplyAdagrad);
 
 class ResourceApplyAdam : public XlaOpKernel {
  public:
@@ -263,7 +266,8 @@ class ResourceApplyAdam : public XlaOpKernel {
  private:
   DataType dtype_;
 };
-REGISTER_XLA_OP(Name("ResourceApplyAdam"), ResourceApplyAdam);
+REGISTER_XLA_OP(Name("ResourceApplyAdam").TypeConstraint("T", kFloatTypes),
+                ResourceApplyAdam);
 
 class ResourceApplyRMSProp : public XlaOpKernel {
  public:
@@ -362,7 +366,8 @@ class ResourceApplyRMSProp : public XlaOpKernel {
     OP_REQUIRES_OK(ctx, ctx->AssignVariable(2, type, new_mom));
   }
 };
-REGISTER_XLA_OP(Name("ResourceApplyRMSProp"), ResourceApplyRMSProp);
+REGISTER_XLA_OP(Name("ResourceApplyRMSProp").TypeConstraint("T", kFloatTypes),
+                ResourceApplyRMSProp);
 
 void CompileFtrl(XlaOpKernelContext* ctx, DataType dtype,
                  bool has_l2_shrinkage) {
@@ -500,7 +505,8 @@ class ResourceApplyFtrl : public XlaOpKernel {
  private:
   DataType dtype_;
 };
-REGISTER_XLA_OP(Name("ResourceApplyFtrl"), ResourceApplyFtrl);
+REGISTER_XLA_OP(Name("ResourceApplyFtrl").TypeConstraint("T", kFloatTypes),
+                ResourceApplyFtrl);
 
 class ResourceApplyFtrlV2 : public XlaOpKernel {
  public:
@@ -515,7 +521,8 @@ class ResourceApplyFtrlV2 : public XlaOpKernel {
  private:
   DataType dtype_;
 };
-REGISTER_XLA_OP(Name("ResourceApplyFtrlV2"), ResourceApplyFtrlV2);
+REGISTER_XLA_OP(Name("ResourceApplyFtrlV2").TypeConstraint("T", kFloatTypes),
+                ResourceApplyFtrlV2);
 
 }  // namespace
 }  // namespace tensorflow
diff --git a/tensorflow/compiler/tf2xla/kernels/unary_ops.cc b/tensorflow/compiler/tf2xla/kernels/unary_ops.cc
index 651bbe2b40..b35f6fc2e0 100644
--- a/tensorflow/compiler/tf2xla/kernels/unary_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/unary_ops.cc
@@ -41,6 +41,12 @@ namespace {
   };                                                                   \
   REGISTER_XLA_OP(Name(#NAME), NAME##Op);
 
+XLAJIT_MAKE_UNARY(ComplexAbs, b->Abs(x));
+
+XLAJIT_MAKE_UNARY(Angle, b->Atan2(b->Imag(x), b->Real(x)));
+
+XLAJIT_MAKE_UNARY(Conj, b->Complex(b->Real(x), b->Neg(b->Imag(x))));
+
 // Return x if x>0, otherwise -x.
 XLAJIT_MAKE_UNARY(Abs, b->Abs(x));
 
@@ -162,6 +168,9 @@ XLAJIT_MAKE_UNARY(Square, b->Mul(x, x));
 XLAJIT_MAKE_UNARY(Tan, b->Div(b->Sin(x), b->Cos(x)));
 XLAJIT_MAKE_UNARY(Tanh, b->Tanh(x));
 
+XLAJIT_MAKE_UNARY(Real, b->Real(x));
+XLAJIT_MAKE_UNARY(Imag, b->Imag(x));
+
 #undef XLAJIT_MAKE_UNARY
 
 }  // namespace
diff --git a/tensorflow/compiler/tf2xla/xla_helpers.cc b/tensorflow/compiler/tf2xla/xla_helpers.cc
index f59b83cfdd..de5ad5f176 100644
--- a/tensorflow/compiler/tf2xla/xla_helpers.cc
+++ b/tensorflow/compiler/tf2xla/xla_helpers.cc
@@ -97,6 +97,9 @@ xla::ComputationDataHandle XlaHelpers::IntegerLiteral(
     case xla::F64:
       literal = *xla::Literal::CreateR0<double>(value);
       break;
+    case xla::C64:
+      literal = *xla::Literal::CreateR0<complex64>(value);
+      break;
     case xla::PRED:
       LOG(FATAL) << "pred element type is not integral";
     case xla::S16:
@@ -132,6 +135,9 @@ xla::ComputationDataHandle XlaHelpers::FloatLiteral(xla::ComputationBuilder* b,
     case xla::F64:
       return b->ConstantR0<double>(value);
       break;
+    case xla::C64:
+      return b->ConstantR0<complex64>(value);
+      break;
     default:
       LOG(FATAL) << "unhandled element type " << type;
   }
diff --git a/tensorflow/compiler/tf2xla/xla_op_registry.h b/tensorflow/compiler/tf2xla/xla_op_registry.h
index 2144868646..6aee8c91cc 100644
--- a/tensorflow/compiler/tf2xla/xla_op_registry.h
+++ b/tensorflow/compiler/tf2xla/xla_op_registry.h
@@ -47,14 +47,17 @@ extern const char* const DEVICE_XLA_GPU;
 
 constexpr std::array<DataType, 3> kFloatTypes = {
     {DT_HALF, DT_FLOAT, DT_DOUBLE}};
-constexpr std::array<DataType, 7> kNumericTypes = {
-    {DT_UINT32, DT_UINT64, DT_INT32, DT_INT64, DT_HALF, DT_FLOAT, DT_DOUBLE}};
+constexpr std::array<DataType, 8> kNumericTypes = {
+    {DT_UINT32, DT_UINT64, DT_INT32, DT_INT64, DT_HALF, DT_FLOAT, DT_DOUBLE,
+     DT_COMPLEX64}};
 
-constexpr std::array<DataType, 7> kCpuAllTypes = {
-    {DT_UINT32, DT_UINT64, DT_INT32, DT_INT64, DT_FLOAT, DT_DOUBLE, DT_BOOL}};
+constexpr std::array<DataType, 8> kCpuAllTypes = {
+    {DT_UINT32, DT_UINT64, DT_INT32, DT_INT64, DT_FLOAT, DT_DOUBLE,
+     DT_COMPLEX64, DT_BOOL}};
 
-constexpr std::array<DataType, 7> kGpuAllTypes = {
-    {DT_UINT32, DT_UINT64, DT_INT32, DT_INT64, DT_FLOAT, DT_DOUBLE, DT_BOOL}};
+constexpr std::array<DataType, 8> kGpuAllTypes = {
+    {DT_UINT32, DT_UINT64, DT_INT32, DT_INT64, DT_FLOAT, DT_DOUBLE,
+     DT_COMPLEX64, DT_BOOL}};
 
 // Class that manages registrations of operators and devices for the XLA JIT.
 // Not thread-safe.
diff --git a/tensorflow/compiler/xla/client/computation_builder.cc b/tensorflow/compiler/xla/client/computation_builder.cc
index b9977fb2f8..edf5a1822c 100644
--- a/tensorflow/compiler/xla/client/computation_builder.cc
+++ b/tensorflow/compiler/xla/client/computation_builder.cc
@@ -913,6 +913,17 @@ ComputationDataHandle ComputationBuilder::CustomCall(
   return ParseOpResponse(s, &response);
 }
 
+ComputationDataHandle ComputationBuilder::Complex(
+    const ComputationDataHandle& real, const ComputationDataHandle& imag,
+    tensorflow::gtl::ArraySlice<int64> broadcast_dimensions) {
+  return BinaryOp(BINOP_COMPLEX, real, imag, broadcast_dimensions);
+}
+
+ComputationDataHandle ComputationBuilder::Conj(
+    const ComputationDataHandle& operand) {
+  return Complex(Real(operand), Neg(Imag(operand)));
+}
+
 ComputationDataHandle ComputationBuilder::Add(
     const ComputationDataHandle& lhs, const ComputationDataHandle& rhs,
     tensorflow::gtl::ArraySlice<int64> broadcast_dimensions) {
@@ -995,6 +1006,12 @@ ComputationDataHandle ComputationBuilder::Abs(
   return UnaryOp(UNOP_ABS, operand);
 }
 
+ComputationDataHandle ComputationBuilder::Atan2(
+    const ComputationDataHandle& y, const ComputationDataHandle& x,
+    tensorflow::gtl::ArraySlice<int64> broadcast_dimensions) {
+  return BinaryOp(BINOP_ATAN2, y, x, broadcast_dimensions);
+}
+
 ComputationDataHandle ComputationBuilder::Exp(
     const ComputationDataHandle& operand) {
   return UnaryOp(UNOP_EXP, operand);
@@ -1040,6 +1057,16 @@ ComputationDataHandle ComputationBuilder::Tanh(
   return UnaryOp(UNOP_TANH, operand);
 }
 
+ComputationDataHandle ComputationBuilder::Real(
+    const ComputationDataHandle& operand) {
+  return UnaryOp(UNOP_REAL, operand);
+}
+
+ComputationDataHandle ComputationBuilder::Imag(
+    const ComputationDataHandle& operand) {
+  return UnaryOp(UNOP_IMAG, operand);
+}
+
 ComputationDataHandle ComputationBuilder::IsFinite(
     const ComputationDataHandle& operand) {
   return UnaryOp(UNOP_IS_FINITE, operand);
diff --git a/tensorflow/compiler/xla/client/computation_builder.h b/tensorflow/compiler/xla/client/computation_builder.h
index 93c2a80678..d2f0c7cff0 100644
--- a/tensorflow/compiler/xla/client/computation_builder.h
+++ b/tensorflow/compiler/xla/client/computation_builder.h
@@ -431,6 +431,14 @@ class ComputationBuilder {
   // of the operands is a scalar, or an explicit broadcast dimension is given
   // (see g3doc for more details).
 
+  // Enqueues a complex compose instruction onto the computation.
+  ComputationDataHandle Complex(
+      const ComputationDataHandle& real, const ComputationDataHandle& imag,
+      tensorflow::gtl::ArraySlice<int64> broadcast_dimensions = {});
+
+  // Enqueues a complex conjugate instruction onto the computation.
+  ComputationDataHandle Conj(const ComputationDataHandle& operand);
+
   // Enqueues an add instruction onto the computation.
   ComputationDataHandle Add(
       const ComputationDataHandle& lhs, const ComputationDataHandle& rhs,
@@ -542,6 +550,11 @@ class ComputationBuilder {
   // Enqueues an abs instruction onto the computation.
   ComputationDataHandle Abs(const ComputationDataHandle& operand);
 
+  // Enqueues a atan2 instruction onto the computation.
+  ComputationDataHandle Atan2(
+      const ComputationDataHandle& y, const ComputationDataHandle& x,
+      tensorflow::gtl::ArraySlice<int64> broadcast_dimensions = {});
+
   // Enqueues an exp instruction onto the computation.
   ComputationDataHandle Exp(const ComputationDataHandle& operand);
 
@@ -570,6 +583,12 @@ class ComputationBuilder {
   // Enqueues a tanh instruction onto the computation.
   ComputationDataHandle Tanh(const ComputationDataHandle& operand);
 
+  // Enqueues a real-part instruction onto the computation.
+  ComputationDataHandle Real(const ComputationDataHandle& operand);
+
+  // Enqueues an imaginary-part instruction onto the computation.
+  ComputationDataHandle Imag(const ComputationDataHandle& operand);
+
   // Enqueues a float32 sqrt instruction onto the computation.
   // (float32 is specified as there is an implicit float32 0.5f constant
   // exponent).
diff --git a/tensorflow/compiler/xla/literal_util.cc b/tensorflow/compiler/xla/literal_util.cc
index 413b85e3ba..8fc8644a60 100644
--- a/tensorflow/compiler/xla/literal_util.cc
+++ b/tensorflow/compiler/xla/literal_util.cc
@@ -204,6 +204,8 @@ Status Literal::Copy(const Literal& src_literal,
       return *Literal::CreateR0<float>(0);
     case F64:
       return *Literal::CreateR0<double>(0);
+    case C64:
+      return *Literal::CreateR0<complex64>(0);
     case PRED:
       return *Literal::CreateR0<bool>(false);
     case S16:
@@ -236,6 +238,8 @@ Status Literal::Copy(const Literal& src_literal,
       return *Literal::CreateR0<float>(1);
     case F64:
       return *Literal::CreateR0<double>(1);
+    case C64:
+      return *Literal::CreateR0<complex64>(1);
     case PRED:
       return *Literal::CreateR0<bool>(true);
     case S16:
@@ -271,6 +275,8 @@ Status Literal::Copy(const Literal& src_literal,
     case F64:
       return *Literal::CreateR0<double>(
           -std::numeric_limits<double>::infinity());
+    case C64:
+      LOG(FATAL) << "C64 element type has no minimum value";
     case PRED:
       return *Literal::CreateR0<bool>(false);
     case S16:
diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier.cc b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
index 35ab4d89cc..2a610e91f0 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
@@ -141,6 +141,9 @@ class AlgebraicSimplifierVisitor : public DfsHloVisitorWithDefault {
 
   Status HandleConvert(HloInstruction* convert) override;
 
+  Status HandleReal(HloInstruction* real, HloInstruction* operand) override;
+  Status HandleImag(HloInstruction* imag, HloInstruction* operand) override;
+
   Status HandleConvolution(HloInstruction* convolution, HloInstruction* lhs,
                            HloInstruction* rhs, const Window& window) override;
 
@@ -967,6 +970,24 @@ Status AlgebraicSimplifierVisitor::HandleConvert(HloInstruction* convert) {
   return Status::OK();
 }
 
+// Real(Complex(r, i)) -> r
+Status AlgebraicSimplifierVisitor::HandleReal(HloInstruction* real,
+                                              HloInstruction* operand) {
+  if (operand->opcode() == HloOpcode::kComplex) {
+    return ReplaceInstruction(real, operand->mutable_operand(0));
+  }
+  return Status::OK();
+}
+
+// Imag(Complex(r, i)) -> i
+Status AlgebraicSimplifierVisitor::HandleImag(HloInstruction* imag,
+                                              HloInstruction* operand) {
+  if (operand->opcode() == HloOpcode::kComplex) {
+    return ReplaceInstruction(imag, operand->mutable_operand(1));
+  }
+  return Status::OK();
+}
+
 Status AlgebraicSimplifierVisitor::HandlePad(HloInstruction* pad) {
   // Eliminate nop pads (padding all zero), and replace a pad with negative
   // padding with a pad with non-negative padding followed by a slice.
diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
index 3df50080d1..87d4fc9663 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
@@ -433,6 +433,56 @@ TEST_F(AlgebraicSimplifierTest, DivOneArray) {
   EXPECT_EQ(root, param0);
 }
 
+// Test that real(complex(r,i)) is simplified to r.
+TEST_F(AlgebraicSimplifierTest, RealOfComplex) {
+  Shape r2f32 = ShapeUtil::MakeShape(F32, {2, 2});
+  HloComputation::Builder builder(TestName());
+  HloInstruction* param0 = builder.AddInstruction(
+      HloInstruction::CreateParameter(0, r2f32, "param0"));
+  HloInstruction* param1 = builder.AddInstruction(
+      HloInstruction::CreateParameter(1, r2f32, "param1"));
+  HloInstruction* cplx = builder.AddInstruction(
+      HloInstruction::CreateBinary(ShapeUtil::ChangeElementType(r2f32, C64),
+                                   HloOpcode::kComplex, param0, param1));
+  HloInstruction* real = builder.AddInstruction(
+      HloInstruction::CreateUnary(r2f32, HloOpcode::kReal, cplx));
+
+  auto module = CreateNewModule();
+  auto computation = module->AddEntryComputation(builder.Build());
+  HloInstruction* root = computation->root_instruction();
+  EXPECT_EQ(root, real);
+  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
+                                 non_bitcasting_callback());
+  ASSERT_TRUE(simplifier.Run(module.get()).ValueOrDie());
+  root = computation->root_instruction();
+  EXPECT_EQ(root, param0);
+}
+
+// Test that imag(complex(r,i)) is simplified to i.
+TEST_F(AlgebraicSimplifierTest, ImagOfComplex) {
+  Shape r2f32 = ShapeUtil::MakeShape(F32, {2, 2});
+  HloComputation::Builder builder(TestName());
+  HloInstruction* param0 = builder.AddInstruction(
+      HloInstruction::CreateParameter(0, r2f32, "param0"));
+  HloInstruction* param1 = builder.AddInstruction(
+      HloInstruction::CreateParameter(1, r2f32, "param1"));
+  HloInstruction* cplx = builder.AddInstruction(
+      HloInstruction::CreateBinary(ShapeUtil::ChangeElementType(r2f32, C64),
+                                   HloOpcode::kComplex, param0, param1));
+  HloInstruction* imag = builder.AddInstruction(
+      HloInstruction::CreateUnary(r2f32, HloOpcode::kImag, cplx));
+
+  auto module = CreateNewModule();
+  auto computation = module->AddEntryComputation(builder.Build());
+  HloInstruction* root = computation->root_instruction();
+  EXPECT_EQ(root, imag);
+  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
+                                 non_bitcasting_callback());
+  ASSERT_TRUE(simplifier.Run(module.get()).ValueOrDie());
+  root = computation->root_instruction();
+  EXPECT_EQ(root, param1);
+}
+
 // Test that get_element(make_tuple({A,B}),1) is simplified to B
 TEST_F(AlgebraicSimplifierTest, SelectMakeTuple) {
   Shape r0f32 = ShapeUtil::MakeShape(F32, {});
diff --git a/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc b/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc
index d3b94d7541..e57d49172b 100644
--- a/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc
+++ b/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc
@@ -63,7 +63,7 @@ DotOpEmitter::DotOpEmitter(const HloInstruction& dot, bool transpose_lhs,
     llvm::Value* executable_run_options_value, llvm::IRBuilder<>* ir_builder,
     const HloModuleConfig& hlo_module_config) {
   PrimitiveType type = target_array.GetShape().element_type();
-  TF_RET_CHECK(F32 == type || F64 == type);
+  TF_RET_CHECK(F32 == type || F64 == type || C64 == type);
   DotOpEmitter dot_emitter(dot, transpose_lhs, transpose_rhs, target_array,
                            lhs_array, rhs_array, executable_run_options_value,
                            ir_builder, hlo_module_config);
@@ -176,7 +176,7 @@ tensorflow::Status DotOpEmitter::Emit() {
   llvm::BasicBlock* preheader_bb = reduction_loop->GetPreheaderBasicBlock();
   ir_builder_->SetInsertPoint(preheader_bb->getTerminator());
 
-  ir_builder_->CreateStore(llvm::ConstantFP::get(accum_type, 0.0),
+  ir_builder_->CreateStore(llvm::Constant::getNullValue(accum_type),
                            accum_address);
 
   // Body basic block of reduction loop:
@@ -191,9 +191,29 @@ tensorflow::Status DotOpEmitter::Emit() {
   llvm::Value* rhs_element =
       rhs_array_.EmitReadArrayElement(rhs_index, ir_builder_);
 
-  llvm::Value* product = ir_builder_->CreateFMul(lhs_element, rhs_element);
   llvm::Value* accum = ir_builder_->CreateLoad(accum_address);
-  llvm::Value* updated_accum = ir_builder_->CreateFAdd(accum, product);
+  llvm::Value* updated_accum;
+  if (ShapeUtil::ElementIsComplex(lhs_shape)) {
+    auto real = [&](llvm::Value* x) {
+      return ir_builder_->CreateExtractValue(x, {0});
+    };
+    auto imag = [&](llvm::Value* x) {
+      return ir_builder_->CreateExtractValue(x, {1});
+    };
+    llvm::Value* product_real = ir_builder_->CreateFSub(
+        ir_builder_->CreateFMul(real(lhs_element), real(rhs_element)),
+        ir_builder_->CreateFMul(imag(lhs_element), imag(rhs_element)));
+    llvm::Value* product_imag = ir_builder_->CreateFAdd(
+        ir_builder_->CreateFMul(real(lhs_element), imag(rhs_element)),
+        ir_builder_->CreateFMul(imag(lhs_element), real(rhs_element)));
+    updated_accum = ir_builder_->CreateInsertValue(
+        accum, ir_builder_->CreateFAdd(real(accum), product_real), {0});
+    updated_accum = ir_builder_->CreateInsertValue(
+        updated_accum, ir_builder_->CreateFAdd(imag(accum), product_imag), {1});
+  } else {
+    llvm::Value* product = ir_builder_->CreateFMul(lhs_element, rhs_element);
+    updated_accum = ir_builder_->CreateFAdd(accum, product);
+  }
   ir_builder_->CreateStore(updated_accum, accum_address);
 
   // Exit basic block of reduction loop.
@@ -230,11 +250,28 @@ tensorflow::Status DotOpEmitter::Emit() {
 
 tensorflow::Status DotOpEmitter::EmitScalarDot() {
   // A scalar dot is just a scalar multiply.
+  llvm::Value* result;
   llvm::Value* lhs_value =
       lhs_array_.EmitReadArrayElement(/*index=*/{}, ir_builder_);
   llvm::Value* rhs_value =
       rhs_array_.EmitReadArrayElement(/*index=*/{}, ir_builder_);
-  llvm::Value* result = ir_builder_->CreateFMul(lhs_value, rhs_value);
+  if (ShapeUtil::ElementIsComplex(lhs_array_.GetShape())) {
+#define REAL(x) ir_builder_->CreateExtractValue(x, {0})
+#define IMAG(x) ir_builder_->CreateExtractValue(x, {1})
+    llvm::Value* real = ir_builder_->CreateFSub(
+        ir_builder_->CreateFMul(REAL(lhs_value), REAL(rhs_value)),
+        ir_builder_->CreateFMul(IMAG(lhs_value), IMAG(rhs_value)));
+    llvm::Value* imag = ir_builder_->CreateFAdd(
+        ir_builder_->CreateFMul(REAL(lhs_value), IMAG(rhs_value)),
+        ir_builder_->CreateFMul(IMAG(lhs_value), REAL(rhs_value)));
+#undef IMAG
+#undef REAL
+    result = llvm::ConstantAggregateZero::get(lhs_array_.GetElementLlvmType());
+    result = ir_builder_->CreateInsertValue(result, real, {0});
+    result = ir_builder_->CreateInsertValue(result, imag, {1});
+  } else {
+    result = ir_builder_->CreateFMul(lhs_value, rhs_value);
+  }
   target_array_.EmitWriteArrayElement(/*index=*/{}, result, ir_builder_);
   return tensorflow::Status::OK();
 }
diff --git a/tensorflow/compiler/xla/service/cpu/elemental_ir_emitter.cc b/tensorflow/compiler/xla/service/cpu/elemental_ir_emitter.cc
index 73e039250b..ba693ec89a 100644
--- a/tensorflow/compiler/xla/service/cpu/elemental_ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/cpu/elemental_ir_emitter.cc
@@ -46,8 +46,8 @@ StatusOr<llvm::Value*> CpuElementalIrEmitter::EmitFloatUnaryOp(
       }
       // Create function type for the function.
       llvm::FunctionType* function_type = llvm::FunctionType::get(
-          llvm_ir::PrimitiveTypeToIrType(element_type, ir_builder_),
-          llvm_ir::PrimitiveTypeToIrType(element_type, ir_builder_),
+          llvm_ir::PrimitiveTypeToIrType(element_type, module_),
+          llvm_ir::PrimitiveTypeToIrType(element_type, module_),
           /*isVarArg=*/false);
       // Create function declaration for 'tanhf'.
       llvm::Function* function =
diff --git a/tensorflow/compiler/xla/service/cpu/ir_emission_utils.cc b/tensorflow/compiler/xla/service/cpu/ir_emission_utils.cc
index ea5b6ca4eb..d72abede02 100644
--- a/tensorflow/compiler/xla/service/cpu/ir_emission_utils.cc
+++ b/tensorflow/compiler/xla/service/cpu/ir_emission_utils.cc
@@ -41,6 +41,12 @@ bool PotentiallyImplementedAsEigenConvolution(
       ShapeUtil::HasZeroElements(kernel_shape)) {
     return false;
   }
+  // TODO(b/65408531): Explore using Eigen dot for complex64 type.
+  if (ShapeUtil::ElementIsComplex(input_shape) ||
+      ShapeUtil::ElementIsComplex(kernel_shape)) {
+    return false;
+  }
+
   const ConvolutionDimensionNumbers& dnums =
       convolution.convolution_dimension_numbers();
   // Only 1D and 2D convolutions are supported at the moment.
diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
index 52085d1376..fa3b3ab8e7 100644
--- a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
@@ -288,7 +288,7 @@ Status IrEmitter::HandleConstant(HloInstruction* constant,
                                     MinimumAlignmentForShape(literal.shape()));
   } else {
     llvm::Constant* initializer =
-        llvm_ir::ConvertLiteralToIrConstant(literal, &ir_builder_);
+        llvm_ir::ConvertLiteralToIrConstant(literal, module_);
     global_for_const = new llvm::GlobalVariable(
         /*Module=*/*module_,
         /*Type=*/initializer->getType(),
@@ -401,7 +401,7 @@ Status IrEmitter::HandleGetTupleElement(HloInstruction* get_tuple_element,
   const Shape& shape = get_tuple_element->shape();
   emitted_value_[get_tuple_element] = llvm_ir::EmitGetTupleElement(
       shape, get_tuple_element->tuple_index(), MinimumAlignmentForShape(shape),
-      GetEmittedValueFor(operand), &ir_builder_);
+      GetEmittedValueFor(operand), &ir_builder_, module_);
   return Status::OK();
 }
 
@@ -412,9 +412,9 @@ Status IrEmitter::HandleSelect(HloInstruction* select, HloInstruction* pred,
 
   if (ShapeUtil::IsTuple(select->shape())) {
     TF_RETURN_IF_ERROR(EmitTargetAddressForOp(select));
-    llvm_ir::EmitTupleSelect(GetIrArrayFor(select), GetIrArrayFor(pred),
-                             GetEmittedValueFor(on_true),
-                             GetEmittedValueFor(on_false), &ir_builder_);
+    llvm_ir::EmitTupleSelect(
+        GetIrArrayFor(select), GetIrArrayFor(pred), GetEmittedValueFor(on_true),
+        GetEmittedValueFor(on_false), &ir_builder_, module_);
     return Status::OK();
   }
 
@@ -459,7 +459,8 @@ Status IrEmitter::HandleInfeed(HloInstruction* infeed) {
       tuple_element_addresses.push_back(tuple_element_address);
     }
 
-    llvm_ir::EmitTuple(infeed_array, tuple_element_addresses, &ir_builder_);
+    llvm_ir::EmitTuple(infeed_array, tuple_element_addresses, &ir_builder_,
+                       module_);
   } else {
     TF_RETURN_IF_ERROR(EmitXfeedTransfer(XfeedKind::kInfeed, shape,
                                          GetEmittedValueFor(infeed)));
@@ -562,7 +563,7 @@ Status IrEmitter::HandleOutfeed(HloInstruction* outfeed) {
         ShapeUtil::GetTupleElementShape(operand_shape, i);
     llvm::Value* tuple_element = llvm_ir::EmitGetTupleElement(
         tuple_element_shape, i, MinimumAlignmentForShape(tuple_element_shape),
-        value, &ir_builder_);
+        value, &ir_builder_, module_);
     TF_RETURN_IF_ERROR(EmitXfeedTransfer(XfeedKind::kOutfeed,
                                          tuple_element_shape, tuple_element));
   }
@@ -583,7 +584,7 @@ Status IrEmitter::HandleTuple(
   for (auto operand : operands) {
     base_ptrs.push_back(GetEmittedValueFor(operand));
   }
-  llvm_ir::EmitTuple(GetIrArrayFor(tuple), base_ptrs, &ir_builder_);
+  llvm_ir::EmitTuple(GetIrArrayFor(tuple), base_ptrs, &ir_builder_, module_);
   return Status::OK();
 }
 
@@ -644,7 +645,7 @@ Status IrEmitter::HandleReduceWindow(HloInstruction* reduce_window,
         // the initial value on the reduce_window.
         PrimitiveType operand_element_type = operand->shape().element_type();
         llvm::Value* accumulator_address = llvm_ir::EmitAllocaAtFunctionEntry(
-            llvm_ir::PrimitiveTypeToIrType(operand_element_type, &ir_builder_),
+            llvm_ir::PrimitiveTypeToIrType(operand_element_type, module_),
             "reduce_window_accumulator_address", &ir_builder_,
             MinimumAlignmentForPrimitiveType(operand_element_type));
         ir_builder_.CreateStore(ir_builder_.CreateLoad(GetEmittedValueFor(
@@ -769,7 +770,7 @@ Status IrEmitter::HandleSelectAndScatter(HloInstruction* select_and_scatter) {
   // Allocate space to keep the currently selected value, its index, and
   // the boolean initialized_flag, which is initially set to false.
   llvm::Value* selected_value_address = llvm_ir::EmitAllocaAtFunctionEntry(
-      llvm_ir::PrimitiveTypeToIrType(operand_element_type, &ir_builder_),
+      llvm_ir::PrimitiveTypeToIrType(operand_element_type, module_),
       "selected_value_address", &ir_builder_,
       MinimumAlignmentForPrimitiveType(operand_element_type));
   llvm::Value* selected_index_address =
@@ -851,8 +852,8 @@ Status IrEmitter::HandleSelectAndScatter(HloInstruction* select_and_scatter) {
   // If the 'select' function returns false, update the selected value and the
   // index to the currently visiting operand.
   llvm::Value* cond = ir_builder_.CreateICmpNE(
-      result, llvm::ConstantInt::get(
-                  llvm_ir::PrimitiveTypeToIrType(PRED, &ir_builder_), 0),
+      result,
+      llvm::ConstantInt::get(llvm_ir::PrimitiveTypeToIrType(PRED, module_), 0),
       "boolean_predicate");
   llvm_ir::LlvmIfData if_select_lhs =
       llvm_ir::EmitIfThenElse(cond, "if-select-lhs", &ir_builder_);
@@ -895,7 +896,7 @@ Status IrEmitter::HandleDot(HloInstruction* dot, HloInstruction* lhs,
                             HloInstruction* rhs) {
   TF_RETURN_IF_ERROR(ElementTypesSameAndSupported(
       /*instruction=*/*dot, /*operands=*/{lhs, rhs},
-      /*supported_types=*/{F32, F64}));
+      /*supported_types=*/{F32, F64, C64}));
 
   llvm_ir::IrArray lhs_array(GetIrArrayFor(lhs));
   llvm_ir::IrArray rhs_array(GetIrArrayFor(rhs));
@@ -923,7 +924,7 @@ Status IrEmitter::HandleConvolution(HloInstruction* convolution,
                                     const Window& window) {
   TF_RETURN_IF_ERROR(ElementTypesSameAndSupported(
       /*instruction=*/*convolution, /*operands=*/{lhs, rhs},
-      /*supported_types=*/{F32}));
+      /*supported_types=*/{F32, C64}));
 
   const ConvolutionDimensionNumbers& dnums =
       convolution->convolution_dimension_numbers();
@@ -1079,7 +1080,7 @@ Status IrEmitter::HandleConvolution(HloInstruction* convolution,
         // the output entry at the given index.
         PrimitiveType lhs_element_type = lhs->shape().element_type();
         llvm::Value* sum_address = llvm_ir::EmitAllocaAtFunctionEntry(
-            llvm_ir::PrimitiveTypeToIrType(lhs_element_type, &ir_builder_),
+            llvm_ir::PrimitiveTypeToIrType(lhs_element_type, module_),
             "convolution_sum_address", &ir_builder_,
             MinimumAlignmentForPrimitiveType(lhs_element_type));
         ir_builder_.CreateStore(
@@ -1295,14 +1296,14 @@ Status IrEmitter::HandleBatchNormTraining(HloInstruction* batch_norm_training) {
             PrimitiveType element_type = operand->shape().element_type();
             // Used to calculate E(X).
             llvm::Value* sum_address = llvm_ir::EmitAllocaAtFunctionEntry(
-                llvm_ir::PrimitiveTypeToIrType(element_type, &ir_builder_),
+                llvm_ir::PrimitiveTypeToIrType(element_type, module_),
                 "sum_address", &ir_builder_,
                 MinimumAlignmentForPrimitiveType(element_type));
 
             // Used to calculate E(X^2).
             llvm::Value* sum_square_address =
                 llvm_ir::EmitAllocaAtFunctionEntry(
-                    llvm_ir::PrimitiveTypeToIrType(element_type, &ir_builder_),
+                    llvm_ir::PrimitiveTypeToIrType(element_type, module_),
                     "sum_square_address", &ir_builder_,
                     MinimumAlignmentForPrimitiveType(element_type));
 
@@ -1425,7 +1426,7 @@ Status IrEmitter::HandleBatchNormTraining(HloInstruction* batch_norm_training) {
           .EmitLoop(IrName(batch_norm_training, "normalize")));
 
   llvm_ir::EmitTuple(GetIrArrayFor(batch_norm_training),
-                     {normalized, mean, var}, &ir_builder_);
+                     {normalized, mean, var}, &ir_builder_, module_);
   return Status::OK();
 }
 
@@ -1488,6 +1489,14 @@ IrEmitter::ReductionGenerator IrEmitter::MatchReductionGenerator(
   }
 
   const Shape& root_shape = root_instruction->shape();
+  if (ShapeUtil::ElementIsComplex(root_shape)) {
+    // TODO(b/65408531): Complex add could by done via bitcast to <float x [2N]>
+    // Complex multiply would be more challenging. We could perhaps use a
+    // strided load to get all reals in a vector, all imags in a vector, or use
+    // CreateShuffleVector on a bitcast to float x [2N].
+    *failure_reason = "complex values not supported";
+    return nullptr;
+  }
   bool root_is_floating_point = ShapeUtil::ElementIsFloating(root_shape);
   bool root_is_integral = ShapeUtil::ElementIsIntegral(root_shape);
   bool root_is_signed = ShapeUtil::ElementIsSigned(root_shape);
@@ -1509,7 +1518,7 @@ IrEmitter::ReductionGenerator IrEmitter::MatchReductionGenerator(
   // This is visually similar to ElementalIrEmitter, though conceptually we're
   // doing something different here.  ElementalIrEmitter emits scalar operations
   // while these emit scalar or vector operations depending on the type of the
-  // operands.
+  // operands. See CreateShardedVectorType for the actual types in use here.
   switch (root_instruction->opcode()) {
     default:
       *failure_reason = "did not recognize root instruction opcode";
@@ -1586,7 +1595,7 @@ IrEmitter::ShardedVectorType IrEmitter::CreateShardedVectorType(
 
   ShardedVectorType sharded_vector_type;
   llvm::Type* element_ir_type =
-      llvm_ir::PrimitiveTypeToIrType(element_type, &ir_builder_);
+      llvm_ir::PrimitiveTypeToIrType(element_type, module_);
 
   for (int i = 0, e = 1 + tensorflow::Log2Ceiling(element_count); i < e; i++) {
     // For every power of two present in element_count, we generate one or more
@@ -1919,7 +1928,7 @@ Status IrEmitter::HandleReduce(HloInstruction* reduce, HloInstruction* arg,
         // Initialize an accumulator with init_value.
         PrimitiveType accumulator_type = reduce->shape().element_type();
         llvm::AllocaInst* accumulator_addr = llvm_ir::EmitAllocaAtFunctionEntry(
-            llvm_ir::PrimitiveTypeToIrType(accumulator_type, &ir_builder_),
+            llvm_ir::PrimitiveTypeToIrType(accumulator_type, module_),
             "accumulator", &ir_builder_,
             MinimumAlignmentForPrimitiveType(accumulator_type));
         llvm::Value* init_value_addr = GetEmittedValueFor(init_value);
@@ -2248,6 +2257,7 @@ Status IrEmitter::HandleFusion(HloInstruction* fusion) {
     return Status::OK();
   } else if (llvm_ir::CanEmitFusedDynamicUpdateSliceInPlace(fusion,
                                                             assignment_)) {
+    VLOG(3) << "HandleFusion FusedDynamicUpdateSliceInPlace";
     CpuElementalIrEmitter elemental_emitter(hlo_module_config_, this, module_);
     TF_RETURN_IF_ERROR(EmitTargetAddressForOp(fusion));
 
@@ -2257,6 +2267,7 @@ Status IrEmitter::HandleFusion(HloInstruction* fusion) {
         fusion, operands, GetIrArrayFor(fusion), &elemental_emitter,
         &ir_builder_);
   } else if (fusion->fusion_kind() == HloInstruction::FusionKind::kLoop) {
+    VLOG(3) << "HandleFusion kLoop";
     CpuElementalIrEmitter elemental_emitter(hlo_module_config_, this, module_);
     auto operands = GetIrArraysForOperandsOf(fusion);
     FusedIrEmitter fused_emitter(operands, &elemental_emitter);
@@ -2400,8 +2411,7 @@ Status IrEmitter::HandleWhile(HloInstruction* xla_while) {
       {while_result}, IrName(xla_while, "cond"));
   llvm::Value* while_predicate = ir_builder_.CreateICmpNE(
       while_condition,
-      llvm::ConstantInt::get(llvm_ir::PrimitiveTypeToIrType(PRED, &ir_builder_),
-                             0));
+      llvm::ConstantInt::get(llvm_ir::PrimitiveTypeToIrType(PRED, module_), 0));
 
   // Branches to the body or to the while exit depending on the condition.
   llvm::BasicBlock* body_bb = llvm::BasicBlock::Create(
@@ -2542,7 +2552,7 @@ void IrEmitter::EmitTransferElements(llvm::Value* target, llvm::Value* source,
   unsigned element_alignment = GCD(
       primitive_type_size, MinimumAlignmentForPrimitiveType(primitive_type));
   llvm::Type* primitive_ptr_type = llvm::PointerType::getUnqual(
-      llvm_ir::PrimitiveTypeToIrType(primitive_type, &ir_builder_));
+      llvm_ir::PrimitiveTypeToIrType(primitive_type, module_));
 
   if (element_count == 1) {
     auto* load_instruction = ir_builder_.CreateAlignedLoad(
@@ -2755,7 +2765,7 @@ llvm::Value* IrEmitter::GetEmittedValueFor(const HloInstruction* hlo) {
 }
 
 llvm::Type* IrEmitter::IrShapeType(const Shape& shape) {
-  return llvm_ir::ShapeToIrType(shape, &ir_builder_);
+  return llvm_ir::ShapeToIrType(shape, module_);
 }
 
 std::vector<llvm::Type*> IrEmitter::GetComputeFunctionParams() {
@@ -2925,7 +2935,7 @@ llvm::Value* IrEmitter::EmitArrayFunctionCall(
   PrimitiveType return_type = return_shape.element_type();
   llvm::Value* return_value_buffer =
       llvm_ir::EmitAllocaAtFunctionEntryWithCount(
-          llvm_ir::PrimitiveTypeToIrType(return_type, &ir_builder_), elements,
+          llvm_ir::PrimitiveTypeToIrType(return_type, module_), elements,
           tensorflow::strings::StrCat(name, "_return_value_address"),
           &ir_builder_, MinimumAlignmentForPrimitiveType(return_type));
   EmitArrayFunctionCallInto(function, parameter_addresses, return_value_buffer,
@@ -3100,7 +3110,7 @@ Status IrEmitter::EmitTargetElementLoop(
     for (int64 i = 0; i < output_arrays.size(); ++i) {
       tuple_operand_ptrs.push_back(output_arrays[i].GetBasePointer());
     }
-    llvm_ir::EmitTuple(target_array, tuple_operand_ptrs, &ir_builder_);
+    llvm_ir::EmitTuple(target_array, tuple_operand_ptrs, &ir_builder_, module_);
 
   } else {
     if (ShouldEmitParallelLoopFor(*target_op)) {
diff --git a/tensorflow/compiler/xla/service/dfs_hlo_visitor.h b/tensorflow/compiler/xla/service/dfs_hlo_visitor.h
index 5b1dbf439c..adaff90913 100644
--- a/tensorflow/compiler/xla/service/dfs_hlo_visitor.h
+++ b/tensorflow/compiler/xla/service/dfs_hlo_visitor.h
@@ -85,6 +85,10 @@ class DfsHloVisitor {
   virtual Status HandleCopy(HloInstruction* copy) {
     return HandleElementwiseUnary(copy);
   }
+  virtual Status HandleComplex(HloInstruction* complex, HloInstruction* real,
+                               HloInstruction* imag) {
+    return HandleElementwiseBinary(complex);
+  }
   virtual Status HandleMultiply(HloInstruction* multiply, HloInstruction* lhs,
                                 HloInstruction* rhs) {
     return HandleElementwiseBinary(multiply);
@@ -122,6 +126,10 @@ class DfsHloVisitor {
   virtual Status HandleAbs(HloInstruction* abs, HloInstruction* operand) {
     return HandleElementwiseUnary(abs);
   }
+  virtual Status HandleAtan2(HloInstruction* atan2, HloInstruction* y,
+                             HloInstruction* x) {
+    return HandleElementwiseBinary(atan2);
+  }
   virtual Status HandleRound(HloInstruction* round) {
     return HandleElementwiseUnary(round);
   }
@@ -152,6 +160,12 @@ class DfsHloVisitor {
   virtual Status HandleTanh(HloInstruction* tanh, HloInstruction* operand) {
     return HandleElementwiseUnary(tanh);
   }
+  virtual Status HandleReal(HloInstruction* real, HloInstruction* operand) {
+    return HandleElementwiseUnary(real);
+  }
+  virtual Status HandleImag(HloInstruction* imag, HloInstruction* operand) {
+    return HandleElementwiseUnary(imag);
+  }
   virtual Status HandleIsFinite(HloInstruction* is_finite,
                                 HloInstruction* operand) {
     return HandleElementwiseUnary(is_finite);
diff --git a/tensorflow/compiler/xla/service/elemental_ir_emitter.cc b/tensorflow/compiler/xla/service/elemental_ir_emitter.cc
index 44f709bede..fd4c332cba 100644
--- a/tensorflow/compiler/xla/service/elemental_ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/elemental_ir_emitter.cc
@@ -54,10 +54,12 @@ StatusOr<llvm::Value*> ElementalIrEmitter::EmitUnaryOp(
     const HloInstruction* op, llvm::Value* operand_value) const {
   if (op->opcode() == HloOpcode::kCopy) {
     return operand_value;
+  } else if (operand_value->getType()->isIntegerTy()) {
+    return EmitIntegerUnaryOp(op, operand_value);
+  } else if (ShapeUtil::ElementIsComplex(op->operand(0)->shape())) {
+    return EmitComplexUnaryOp(op, operand_value);
   } else {
-    return operand_value->getType()->isIntegerTy()
-               ? EmitIntegerUnaryOp(op, operand_value)
-               : EmitFloatUnaryOp(op, operand_value);
+    return EmitFloatUnaryOp(op, operand_value);
   }
 }
 
@@ -73,20 +75,35 @@ StatusOr<llvm::Value*> ElementalIrEmitter::EmitIntegerUnaryOp(
       }
       if (primitive_util::IsIntegralType(to_type)) {
         return ir_builder_->CreateIntCast(
-            operand_value, llvm_ir::PrimitiveTypeToIrType(to_type, ir_builder_),
+            operand_value, llvm_ir::PrimitiveTypeToIrType(to_type, module_),
             primitive_util::IsSignedIntegralType(to_type));
       }
       if (primitive_util::IsFloatingPointType(to_type)) {
         if (primitive_util::IsSignedIntegralType(from_type)) {
           return ir_builder_->CreateSIToFP(
-              operand_value,
-              llvm_ir::PrimitiveTypeToIrType(to_type, ir_builder_));
+              operand_value, llvm_ir::PrimitiveTypeToIrType(to_type, module_));
         }
         if (primitive_util::IsUnsignedIntegralType(from_type) ||
             from_type == PRED) {
           return ir_builder_->CreateUIToFP(
-              operand_value,
-              llvm_ir::PrimitiveTypeToIrType(to_type, ir_builder_));
+              operand_value, llvm_ir::PrimitiveTypeToIrType(to_type, module_));
+        }
+      }
+      if (primitive_util::IsComplexType(to_type)) {
+        auto to_ir_component_type = llvm_ir::PrimitiveTypeToIrType(
+            primitive_util::ComplexComponentType(to_type), module_);
+        if (primitive_util::IsSignedIntegralType(from_type)) {
+          return ComposeComplex(
+              op,
+              ir_builder_->CreateSIToFP(operand_value, to_ir_component_type),
+              nullptr);
+        }
+        if (primitive_util::IsUnsignedIntegralType(from_type) ||
+            from_type == PRED) {
+          return ComposeComplex(
+              op,
+              ir_builder_->CreateUIToFP(operand_value, to_ir_component_type),
+              nullptr);
         }
       }
       return Unimplemented("conversion from primitive type %s to %s",
@@ -97,8 +114,8 @@ StatusOr<llvm::Value*> ElementalIrEmitter::EmitIntegerUnaryOp(
       bool is_signed =
           primitive_util::IsSignedIntegralType(op->shape().element_type());
       if (is_signed) {
-        auto type = llvm_ir::PrimitiveTypeToIrType(op->shape().element_type(),
-                                                   ir_builder_);
+        auto type =
+            llvm_ir::PrimitiveTypeToIrType(op->shape().element_type(), module_);
         auto zero = llvm::ConstantInt::get(type, 0);
         auto cmp = ir_builder_->CreateICmpSGE(operand_value, zero);
         return ir_builder_->CreateSelect(cmp, operand_value,
@@ -110,8 +127,8 @@ StatusOr<llvm::Value*> ElementalIrEmitter::EmitIntegerUnaryOp(
     case HloOpcode::kSign: {
       bool is_signed =
           primitive_util::IsSignedIntegralType(op->shape().element_type());
-      auto type = llvm_ir::PrimitiveTypeToIrType(op->shape().element_type(),
-                                                 ir_builder_);
+      auto type =
+          llvm_ir::PrimitiveTypeToIrType(op->shape().element_type(), module_);
       auto zero = llvm::ConstantInt::get(type, 0);
       auto cmp = ir_builder_->CreateICmpEQ(operand_value, zero);
       if (is_signed) {
@@ -135,7 +152,7 @@ StatusOr<llvm::Value*> ElementalIrEmitter::EmitIntegerUnaryOp(
         return ir_builder_->CreateZExt(
             ir_builder_->CreateNot(ir_builder_->CreateTrunc(
                 operand_value, ir_builder_->getInt1Ty())),
-            llvm_ir::PrimitiveTypeToIrType(PRED, ir_builder_));
+            llvm_ir::PrimitiveTypeToIrType(PRED, module_));
       } else if (primitive_util::IsIntegralType(type)) {
         return ir_builder_->CreateNot(operand_value);
       }
@@ -157,20 +174,30 @@ StatusOr<llvm::Value*> ElementalIrEmitter::EmitFloatUnaryOp(
       if (from_type == to_type) {
         return operand_value;
       }
+      if (primitive_util::IsComplexType(to_type)) {
+        PrimitiveType to_component_type =
+            primitive_util::ComplexComponentType(to_type);
+        if (from_type == to_component_type) {
+          return ComposeComplex(op, operand_value, nullptr);
+        }
+        return ComposeComplex(
+            op,
+            ir_builder_->CreateFPCast(
+                operand_value,
+                llvm_ir::PrimitiveTypeToIrType(to_component_type, module_)),
+            nullptr);
+      }
       if (primitive_util::IsFloatingPointType(to_type)) {
         return ir_builder_->CreateFPCast(
-            operand_value,
-            llvm_ir::PrimitiveTypeToIrType(to_type, ir_builder_));
+            operand_value, llvm_ir::PrimitiveTypeToIrType(to_type, module_));
       }
       if (primitive_util::IsSignedIntegralType(to_type)) {
         return ir_builder_->CreateFPToSI(
-            operand_value,
-            llvm_ir::PrimitiveTypeToIrType(to_type, ir_builder_));
+            operand_value, llvm_ir::PrimitiveTypeToIrType(to_type, module_));
       }
       if (primitive_util::IsUnsignedIntegralType(to_type)) {
         return ir_builder_->CreateFPToUI(
-            operand_value,
-            llvm_ir::PrimitiveTypeToIrType(to_type, ir_builder_));
+            operand_value, llvm_ir::PrimitiveTypeToIrType(to_type, module_));
       }
       return Unimplemented("unhandled conversion operation: %s => %s",
                            PrimitiveType_Name(from_type).c_str(),
@@ -230,7 +257,7 @@ StatusOr<llvm::Value*> ElementalIrEmitter::EmitFloatUnaryOp(
       auto not_infinite = ir_builder_->CreateFCmpONE(abs_value, infinity);
       auto result_i1 = ir_builder_->CreateAnd(equal_self, not_infinite);
       return ir_builder_->CreateZExt(
-          result_i1, llvm_ir::PrimitiveTypeToIrType(PRED, ir_builder_));
+          result_i1, llvm_ir::PrimitiveTypeToIrType(PRED, module_));
     }
     case HloOpcode::kNegate:
       return ir_builder_->CreateFNeg(operand_value);
@@ -240,20 +267,164 @@ StatusOr<llvm::Value*> ElementalIrEmitter::EmitFloatUnaryOp(
   }
 }
 
+StatusOr<llvm::Value*> ElementalIrEmitter::EmitComplexUnaryOp(
+    const HloInstruction* op, llvm::Value* operand_value) const {
+  auto real = [&](llvm::Value* x) {
+    return ir_builder_->CreateExtractValue(x, {0});
+  };
+  auto imag = [&](llvm::Value* x) {
+    return ir_builder_->CreateExtractValue(x, {1});
+  };
+  switch (op->opcode()) {
+    // TODO(b/65209142): Angle/Log require atan2.
+    // case HloOpcode::kAngle:
+    // case HloOpcode::kLog:  // log(a+bi) = .5*log(a^2+b^2) + i*atan2(b, a)
+    case HloOpcode::kConvert: {
+      PrimitiveType from_type = op->operand(0)->shape().element_type();
+      TF_RET_CHECK(primitive_util::IsComplexType(from_type));
+      PrimitiveType to_type = op->shape().element_type();
+      TF_RET_CHECK(primitive_util::IsComplexType(to_type));
+      if (from_type == to_type) {
+        return operand_value;
+      }
+      PrimitiveType to_component_type =
+          primitive_util::ComplexComponentType(to_type);
+      auto to_ir_component_type =
+          llvm_ir::PrimitiveTypeToIrType(to_component_type, module_);
+      return ComposeComplex(
+          op,
+          ir_builder_->CreateFPCast(real(operand_value), to_ir_component_type),
+          ir_builder_->CreateFPCast(imag(operand_value), to_ir_component_type));
+    }
+    case HloOpcode::kExp: {
+      // e^(a+bi) = e^a*(cos(b)+sin(b)i)
+      auto exp_a = llvm_ir::EmitCallToIntrinsic(
+          llvm::Intrinsic::exp, {real(operand_value)},
+          {real(operand_value)->getType()}, ir_builder_);
+      auto cos_b = llvm_ir::EmitCallToIntrinsic(
+          llvm::Intrinsic::cos, {imag(operand_value)},
+          {imag(operand_value)->getType()}, ir_builder_);
+      auto sin_b = llvm_ir::EmitCallToIntrinsic(
+          llvm::Intrinsic::sin, {imag(operand_value)},
+          {imag(operand_value)->getType()}, ir_builder_);
+      return ComposeComplex(op, ir_builder_->CreateFMul(exp_a, cos_b),
+                            ir_builder_->CreateFMul(exp_a, sin_b));
+    }
+    case HloOpcode::kCos: {
+      // cos(z) = .5(e^(iz) + e^(-iz))
+      // cos(a+bi) = .5(e^(-b+ai) + e^(b-ai))
+      // now, e^(x+yi) = e^x*(cos(y)+sin(y)i), so we have
+      // cos(a+bi) = .5(e^-b*(cos(a)+sin(a)i) + e^b*(cos(-a)+sin(-a)i))
+      // cos(-x) = cos(x) and sin(-x) = -sin(x), so
+      // cos(a+bi) = .5(e^-b*(cos(a)+sin(a)i) + e^b*(cos(a)-sin(a)i))
+      //           = .5(cos(a)*(e^-b+e^b) + i*sin(a)*(e^-b-e^b))
+      auto a = real(operand_value);
+      auto b = imag(operand_value);
+      auto type = a->getType();
+      auto exp_b = llvm_ir::EmitCallToIntrinsic(llvm::Intrinsic::exp, {b},
+                                                {type}, ir_builder_);
+      auto half_exp_b =
+          ir_builder_->CreateFMul(llvm::ConstantFP::get(type, 0.5), exp_b);
+      auto half_exp_neg_b =
+          ir_builder_->CreateFDiv(llvm::ConstantFP::get(type, 0.5), exp_b);
+      auto cos_a = llvm_ir::EmitCallToIntrinsic(llvm::Intrinsic::cos, {a},
+                                                {type}, ir_builder_);
+      auto sin_a = llvm_ir::EmitCallToIntrinsic(llvm::Intrinsic::sin, {a},
+                                                {type}, ir_builder_);
+      return ComposeComplex(
+          op,
+          ir_builder_->CreateFMul(
+              cos_a, ir_builder_->CreateFAdd(half_exp_neg_b, half_exp_b)),
+          ir_builder_->CreateFMul(
+              sin_a, ir_builder_->CreateFSub(half_exp_neg_b, half_exp_b)));
+    }
+    case HloOpcode::kSin: {
+      // sin(z) = .5i(e^(-iz) - e^(iz))
+      // sin(a+bi) = .5i(e^(-i(a+bi)) - e^(i(a+bi)))
+      //           = .5i(e^(b-ai) - e^(-b+ai))
+      // now, e^(x+yi) = e^x*(cos(y)+sin(y)i), so we have
+      // sin(a+bi) = 0.5i(e^b*(cos(-a)+sin(-a)i) - e^-b*(cos(a)+sin(a)i))
+      //           = 0.5(e^b*(cos(-a)i-sin(-a)) - e^-b*(cos(a)i-sin(a)))
+      // cos(-x) = cos(x) and sin(-x) = -sin(x), so
+      //           = 0.5(e^b*(cos(a)i+sin(a)) - e^-b*(cos(a)i-sin(a)))
+      //           = 0.5(sin(a)*(e^b+e^-b) + i*cos(a)*(e^b-e^-b)
+      auto a = real(operand_value);
+      auto b = imag(operand_value);
+      auto type = a->getType();
+      auto exp_b = llvm_ir::EmitCallToIntrinsic(llvm::Intrinsic::exp, {b},
+                                                {type}, ir_builder_);
+      auto half_exp_b =
+          ir_builder_->CreateFMul(llvm::ConstantFP::get(type, 0.5), exp_b);
+      auto half_exp_neg_b =
+          ir_builder_->CreateFDiv(llvm::ConstantFP::get(type, 0.5), exp_b);
+      auto cos_a = llvm_ir::EmitCallToIntrinsic(llvm::Intrinsic::cos, {a},
+                                                {type}, ir_builder_);
+      auto sin_a = llvm_ir::EmitCallToIntrinsic(llvm::Intrinsic::sin, {a},
+                                                {type}, ir_builder_);
+      return ComposeComplex(
+          op,
+          ir_builder_->CreateFMul(
+              sin_a, ir_builder_->CreateFAdd(half_exp_b, half_exp_neg_b)),
+          ir_builder_->CreateFMul(
+              cos_a, ir_builder_->CreateFSub(half_exp_b, half_exp_neg_b)));
+    }
+    case HloOpcode::kAbs: {
+      auto sum_sq = ir_builder_->CreateFAdd(
+          ir_builder_->CreateFMul(real(operand_value), real(operand_value)),
+          ir_builder_->CreateFMul(imag(operand_value), imag(operand_value)));
+      return llvm_ir::EmitCallToIntrinsic(llvm::Intrinsic::sqrt, {sum_sq},
+                                          {sum_sq->getType()}, ir_builder_);
+    }
+    case HloOpcode::kSign: {  // Sign(c) = c / |c|
+      auto sum_sq = ir_builder_->CreateFAdd(
+          ir_builder_->CreateFMul(real(operand_value), real(operand_value)),
+          ir_builder_->CreateFMul(imag(operand_value), imag(operand_value)));
+      auto cplx_abs = llvm_ir::EmitCallToIntrinsic(
+          llvm::Intrinsic::sqrt, {sum_sq}, {sum_sq->getType()}, ir_builder_);
+      auto type = cplx_abs->getType();
+      auto zero = llvm::ConstantFP::get(type, 0.0);
+      auto oeq = ir_builder_->CreateFCmpOEQ(cplx_abs, zero);
+      return ir_builder_->CreateSelect(
+          oeq, ComposeComplex(op, zero, zero),
+          ComposeComplex(
+              op, ir_builder_->CreateFDiv(real(operand_value), cplx_abs),
+              ir_builder_->CreateFDiv(imag(operand_value), cplx_abs)));
+    }
+    case HloOpcode::kNegate:
+      return ComposeComplex(op, ir_builder_->CreateFNeg(real(operand_value)),
+                            ir_builder_->CreateFNeg(imag(operand_value)));
+    case HloOpcode::kReal:
+      return real(operand_value);
+    case HloOpcode::kImag:
+      return imag(operand_value);
+    default:
+      return Unimplemented("unary complex op '%s'",
+                           HloOpcodeString(op->opcode()).c_str());
+  }
+}
+
 StatusOr<llvm::Value*> ElementalIrEmitter::EmitBinaryOp(
     const HloInstruction* op, llvm::Value* lhs_value,
     llvm::Value* rhs_value) const {
-  return lhs_value->getType()->isIntegerTy()
-             ? EmitIntegerBinaryOp(op, lhs_value, rhs_value,
-                                   primitive_util::IsSignedIntegralType(
-                                       op->operand(0)->shape().element_type()))
-             : EmitFloatBinaryOp(op, lhs_value, rhs_value);
+  PrimitiveType operand_type = op->operand(0)->shape().element_type();
+  if (lhs_value->getType()->isIntegerTy()) {
+    return EmitIntegerBinaryOp(
+        op, lhs_value, rhs_value,
+        primitive_util::IsSignedIntegralType(operand_type));
+  } else if (primitive_util::IsComplexType(operand_type)) {
+    return EmitComplexBinaryOp(op, lhs_value, rhs_value);
+  } else {
+    return EmitFloatBinaryOp(op, lhs_value, rhs_value);
+  }
 }
 
 StatusOr<llvm::Value*> ElementalIrEmitter::EmitFloatBinaryOp(
     const HloInstruction* op, llvm::Value* lhs_value,
     llvm::Value* rhs_value) const {
   switch (op->opcode()) {
+    // case HloOpcode::kAtan2:  // TODO(b/65209142): CPU atan2 support
+    case HloOpcode::kComplex:
+      return ComposeComplex(op, lhs_value, rhs_value);
     case HloOpcode::kAdd:
       return ir_builder_->CreateFAdd(lhs_value, rhs_value);
     case HloOpcode::kSubtract:
@@ -305,6 +476,88 @@ StatusOr<llvm::Value*> ElementalIrEmitter::EmitFloatBinaryOp(
   }
 }
 
+StatusOr<llvm::Value*> ElementalIrEmitter::EmitComplexBinaryOp(
+    const HloInstruction* op, llvm::Value* lhs_value,
+    llvm::Value* rhs_value) const {
+  auto real = [&](llvm::Value* x) {
+    return ir_builder_->CreateExtractValue(x, {0});
+  };
+  auto imag = [&](llvm::Value* x) {
+    return ir_builder_->CreateExtractValue(x, {1});
+  };
+  switch (op->opcode()) {
+    case HloOpcode::kAdd:
+      return ComposeComplex(
+          op, ir_builder_->CreateFAdd(real(lhs_value), real(rhs_value)),
+          ir_builder_->CreateFAdd(imag(lhs_value), imag(rhs_value)));
+    case HloOpcode::kSubtract:
+      return ComposeComplex(
+          op, ir_builder_->CreateFSub(real(lhs_value), real(rhs_value)),
+          ir_builder_->CreateFSub(imag(lhs_value), imag(rhs_value)));
+    case HloOpcode::kMultiply:
+      return ComposeComplex(
+          op,
+          ir_builder_->CreateFSub(
+              ir_builder_->CreateFMul(real(lhs_value), real(rhs_value)),
+              ir_builder_->CreateFMul(imag(lhs_value), imag(rhs_value))),
+          ir_builder_->CreateFAdd(
+              ir_builder_->CreateFMul(real(lhs_value), imag(rhs_value)),
+              ir_builder_->CreateFMul(imag(lhs_value), real(rhs_value))));
+    case HloOpcode::kDivide: {
+      // (a+bi) / (c+di) = ((a+bi)(c-di)) / ((c+di)(c-di))
+      // = ((ac + bd) + (bc - ad)i) / (c^2 + d^2)
+      auto rhs_sum_sq = ir_builder_->CreateFAdd(
+          ir_builder_->CreateFMul(real(rhs_value), real(rhs_value)),
+          ir_builder_->CreateFMul(imag(rhs_value), imag(rhs_value)));
+      auto type = rhs_sum_sq->getType();
+      auto zero = llvm::ConstantFP::get(type, 0.0);
+      auto oeq = ir_builder_->CreateFCmpOEQ(rhs_sum_sq, zero);
+      return ir_builder_->CreateSelect(
+          oeq, ComposeComplex(op, llvm::ConstantFP::getInfinity(type), zero),
+          ComposeComplex(
+              op,
+              ir_builder_->CreateFDiv(
+                  ir_builder_->CreateFAdd(
+                      ir_builder_->CreateFMul(real(lhs_value), real(rhs_value)),
+                      ir_builder_->CreateFMul(imag(lhs_value),
+                                              imag(rhs_value))),
+                  rhs_sum_sq),
+              ir_builder_->CreateFDiv(
+                  ir_builder_->CreateFSub(
+                      ir_builder_->CreateFMul(imag(lhs_value), real(rhs_value)),
+                      ir_builder_->CreateFMul(real(lhs_value),
+                                              imag(rhs_value))),
+                  rhs_sum_sq)));
+    }
+    // LLVM comparisons can be "unordered" (U) or "ordered" (O) -- ordered
+    // comparisons always return false when one of the operands is NaN, whereas
+    // unordered comparisons return true.
+    //
+    // We use ordered comparisons for everything except kNe, where we use an
+    // unordered comparison.  This makes x != y equivalent to !(x == y), and
+    // matches C++'s semantics.
+    case HloOpcode::kEq:
+      return ir_builder_->CreateAnd(
+          llvm_ir::EmitComparison(llvm::CmpInst::FCMP_OEQ, real(lhs_value),
+                                  real(rhs_value), ir_builder_),
+          llvm_ir::EmitComparison(llvm::CmpInst::FCMP_OEQ, imag(lhs_value),
+                                  imag(rhs_value), ir_builder_));
+    case HloOpcode::kNe:
+      return ir_builder_->CreateOr(
+          llvm_ir::EmitComparison(llvm::CmpInst::FCMP_UNE, real(lhs_value),
+                                  real(rhs_value), ir_builder_),
+          llvm_ir::EmitComparison(llvm::CmpInst::FCMP_UNE, imag(lhs_value),
+                                  imag(rhs_value), ir_builder_));
+
+    // TODO(b/65209142): requires arg(z) -> requires atan|atan2 intrinsic
+    // case HloOpcode::kPower:
+    // // (a+bi)^(c+di) = exp(i(c+di)*arg(a+bi)) * (a*a+b*b)^(c/2+di/2)
+    default:
+      return Unimplemented("binary complex op '%s'",
+                           HloOpcodeString(op->opcode()).c_str());
+  }
+}
+
 llvm::Value* ElementalIrEmitter::EmitFloatMax(llvm::Value* lhs_value,
                                               llvm::Value* rhs_value) const {
   return llvm_ir::EmitFloatMax(lhs_value, rhs_value, ir_builder_);
@@ -396,7 +649,7 @@ StatusOr<llvm::Value*> ElementalIrEmitter::EmitErfInv(PrimitiveType prim_type,
 StatusOr<llvm::Value*> ElementalIrEmitter::EmitErfcInv(
     PrimitiveType prim_type, llvm::Value* value) const {
   // Compute erfcinv(value) by calculating erfinv(1.0 - value).
-  auto type = llvm_ir::PrimitiveTypeToIrType(prim_type, ir_builder_);
+  auto type = llvm_ir::PrimitiveTypeToIrType(prim_type, module_);
   auto one = llvm::ConstantFP::get(type, 1.0);
   return EmitErfInv(prim_type, ir_builder_->CreateFSub(one, value));
 }
@@ -619,7 +872,7 @@ llvm_ir::ElementGenerator ElementalIrEmitter::MakeRngElementGenerator(
     const {
   PrimitiveType param_prim_type = hlo->operand(0)->shape().element_type();
   llvm::Type* param_ir_type =
-      llvm_ir::PrimitiveTypeToIrType(param_prim_type, ir_builder_);
+      llvm_ir::PrimitiveTypeToIrType(param_prim_type, module_);
 
   // Same values as PCG library
   // https://github.com/imneme/pcg-c/blob/master/include/pcg_variants.h
@@ -783,7 +1036,7 @@ llvm_ir::ElementGenerator ElementalIrEmitter::MakeRngElementGenerator(
         return ir_builder_->CreateZExt(
             ir_builder_->CreateFCmpOLT(get_next_uniform_float(), p),
             llvm_ir::PrimitiveTypeToIrType(hlo->shape().element_type(),
-                                           ir_builder_));
+                                           module_));
       }
       default:
         return InvalidArgument(
@@ -806,9 +1059,11 @@ llvm_ir::ElementGenerator ElementalIrEmitter::MakeElementGenerator(
     case HloOpcode::kCos:
     case HloOpcode::kExp:
     case HloOpcode::kFloor:
+    case HloOpcode::kImag:
     case HloOpcode::kIsFinite:
     case HloOpcode::kLog:
     case HloOpcode::kNegate:
+    case HloOpcode::kReal:
     case HloOpcode::kSign:
     case HloOpcode::kSin:
     case HloOpcode::kTanh:
@@ -821,6 +1076,8 @@ llvm_ir::ElementGenerator ElementalIrEmitter::MakeElementGenerator(
         return EmitUnaryOp(hlo, operand_value);
       };
     case HloOpcode::kAdd:
+    case HloOpcode::kAtan2:
+    case HloOpcode::kComplex:
     case HloOpcode::kDivide:
     case HloOpcode::kEq:
     case HloOpcode::kGe:
@@ -913,10 +1170,10 @@ llvm_ir::ElementGenerator ElementalIrEmitter::MakeElementGenerator(
         }
 
         llvm_ir::SetToFirstInsertPoint(exit_block, ir_builder_);
-        llvm::PHINode* output = ir_builder_->CreatePHI(
-            llvm_ir::PrimitiveTypeToIrType(hlo->shape().element_type(),
-                                           ir_builder_),
-            hlo->operands().size());
+        llvm::PHINode* output =
+            ir_builder_->CreatePHI(llvm_ir::PrimitiveTypeToIrType(
+                                       hlo->shape().element_type(), module_),
+                                   hlo->operands().size());
         auto prior_insert_point = ir_builder_->GetInsertPoint();
 
         ir_builder_->SetInsertPoint(init_block);
@@ -1075,7 +1332,7 @@ llvm_ir::ElementGenerator ElementalIrEmitter::MakeElementGenerator(
         // else                    -> return data from 'index'.
         llvm::Value* ret_value_addr = llvm_ir::EmitAllocaAtFunctionEntry(
             llvm_ir::PrimitiveTypeToIrType(hlo->shape().element_type(),
-                                           ir_builder_),
+                                           module_),
             "ret_value_addr", ir_builder_);
         llvm_ir::LlvmIfData if_data = llvm_ir::EmitIfThenElse(
             slice_intersection, "slice_intersection", ir_builder_);
@@ -1164,7 +1421,7 @@ llvm_ir::ElementGenerator ElementalIrEmitter::MakeElementGenerator(
         // }
         llvm::Value* ret_value_addr = llvm_ir::EmitAllocaAtFunctionEntry(
             llvm_ir::PrimitiveTypeToIrType(hlo->shape().element_type(),
-                                           ir_builder_),
+                                           module_),
             "pad_result_addr", ir_builder_);
         llvm_ir::LlvmIfData if_data =
             llvm_ir::EmitIfThenElse(in_bounds, "in_bounds", ir_builder_);
@@ -1206,7 +1463,7 @@ llvm_ir::ElementGenerator ElementalIrEmitter::MakeElementGenerator(
                               ir_builder_);
         PrimitiveType primitive_type = hlo->shape().element_type();
         llvm::Type* primitive_type_llvm =
-            llvm_ir::PrimitiveTypeToIrType(primitive_type, ir_builder_);
+            llvm_ir::PrimitiveTypeToIrType(primitive_type, module_);
         llvm::Value* accumulator_alloca = llvm_ir::EmitAllocaAtFunctionEntry(
             primitive_type_llvm, "dot_acc", ir_builder_);
         ir_builder_->CreateStore(
@@ -1239,7 +1496,28 @@ llvm_ir::ElementGenerator ElementalIrEmitter::MakeElementGenerator(
         TF_ASSIGN_OR_RETURN(llvm::Value * lhs_value, lhs_generator(lhs_index));
         TF_ASSIGN_OR_RETURN(llvm::Value * rhs_value, rhs_generator(rhs_index));
         llvm::Value* next_accumulator;
-        if (primitive_util::IsFloatingPointType(primitive_type)) {
+        if (primitive_util::IsComplexType(primitive_type)) {
+          auto real = [&](llvm::Value* x) {
+            return ir_builder_->CreateExtractValue(x, {0});
+          };
+          auto imag = [&](llvm::Value* x) {
+            return ir_builder_->CreateExtractValue(x, {1});
+          };
+          llvm::Value* product_real = ir_builder_->CreateFSub(
+              ir_builder_->CreateFMul(real(lhs_value), real(rhs_value)),
+              ir_builder_->CreateFMul(imag(lhs_value), imag(rhs_value)));
+          llvm::Value* product_imag = ir_builder_->CreateFAdd(
+              ir_builder_->CreateFMul(real(lhs_value), imag(rhs_value)),
+              ir_builder_->CreateFMul(imag(lhs_value), real(rhs_value)));
+          next_accumulator = ir_builder_->CreateInsertValue(
+              current_accumulator,
+              ir_builder_->CreateFAdd(real(current_accumulator), product_real),
+              {0});
+          next_accumulator = ir_builder_->CreateInsertValue(
+              next_accumulator,
+              ir_builder_->CreateFAdd(imag(current_accumulator), product_imag),
+              {1});
+        } else if (primitive_util::IsFloatingPointType(primitive_type)) {
           next_accumulator = ir_builder_->CreateFAdd(
               current_accumulator,
               ir_builder_->CreateFMul(lhs_value, rhs_value));
@@ -1261,4 +1539,17 @@ llvm_ir::ElementGenerator ElementalIrEmitter::MakeElementGenerator(
   }
 }
 
+llvm::Value* ElementalIrEmitter::ComposeComplex(const HloInstruction* op,
+                                                llvm::Value* real,
+                                                llvm::Value* imag) const {
+  auto cplx_type =
+      llvm_ir::PrimitiveTypeToIrType(op->shape().element_type(), module_);
+  auto complex = ir_builder_->CreateInsertValue(
+      llvm::ConstantAggregateZero::get(cplx_type), real, {0});
+  if (imag != nullptr) {
+    complex = ir_builder_->CreateInsertValue(complex, imag, {1});
+  }
+  return complex;
+}
+
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/service/elemental_ir_emitter.h b/tensorflow/compiler/xla/service/elemental_ir_emitter.h
index 35dfa88e9b..9d32436e38 100644
--- a/tensorflow/compiler/xla/service/elemental_ir_emitter.h
+++ b/tensorflow/compiler/xla/service/elemental_ir_emitter.h
@@ -55,6 +55,7 @@ class ElementalIrEmitter {
       const HloToElementGeneratorMap& operand_to_generator) const;
 
   llvm::IRBuilder<>* ir_builder() const { return ir_builder_; }
+  llvm::Module* module() const { return module_; }
 
  protected:
   virtual StatusOr<llvm::Value*> EmitIntegerUnaryOp(
@@ -63,6 +64,9 @@ class ElementalIrEmitter {
   virtual StatusOr<llvm::Value*> EmitFloatUnaryOp(
       const HloInstruction* op, llvm::Value* operand_value) const;
 
+  virtual StatusOr<llvm::Value*> EmitComplexUnaryOp(
+      const HloInstruction* op, llvm::Value* operand_value) const;
+
   virtual StatusOr<llvm::Value*> EmitIntegerBinaryOp(const HloInstruction* op,
                                                      llvm::Value* lhs_value,
                                                      llvm::Value* rhs_value,
@@ -72,6 +76,10 @@ class ElementalIrEmitter {
       const HloInstruction* op, llvm::Value* lhs_value,
       llvm::Value* rhs_value) const;
 
+  virtual StatusOr<llvm::Value*> EmitComplexBinaryOp(
+      const HloInstruction* op, llvm::Value* lhs_value,
+      llvm::Value* rhs_value) const;
+
   virtual llvm::Value* EmitFloatMax(llvm::Value* lhs_value,
                                     llvm::Value* rhs_value) const;
 
@@ -109,6 +117,11 @@ class ElementalIrEmitter {
   // compiled executable outside of the HLO code itself.
   const HloModuleConfig& hlo_module_config_;
 
+ protected:
+  // Composes a complex struct. imag may be nullptr for simple cast operations.
+  llvm::Value* ComposeComplex(const HloInstruction* op, llvm::Value* real,
+                              llvm::Value* imag) const;
+
  private:
   // Returns a ElementGenerator for a RNG HloInstruction.
   llvm_ir::ElementGenerator MakeRngElementGenerator(
diff --git a/tensorflow/compiler/xla/service/gpu/elemental_ir_emitter.cc b/tensorflow/compiler/xla/service/gpu/elemental_ir_emitter.cc
index 8810a85cee..1b94499bc6 100644
--- a/tensorflow/compiler/xla/service/gpu/elemental_ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/gpu/elemental_ir_emitter.cc
@@ -135,6 +135,10 @@ StatusOr<llvm::Value*> GpuElementalIrEmitter::EmitFloatBinaryOp(
   PrimitiveType rhs_input_type = op->operand(1)->shape().element_type();
   PrimitiveType output_type = op->shape().element_type();
   switch (op->opcode()) {
+    case HloOpcode::kAtan2:
+      return EmitLibdeviceMathCall("__nv_atan2", {lhs_value, rhs_value},
+                                   {lhs_input_type, rhs_input_type},
+                                   output_type);
     case HloOpcode::kRemainder: {
       return EmitLibdeviceMathCall("__nv_fmod", {lhs_value, rhs_value},
                                    {lhs_input_type, rhs_input_type},
@@ -226,6 +230,112 @@ StatusOr<llvm::Value*> GpuElementalIrEmitter::EmitFloatUnaryOp(
   }
 }
 
+StatusOr<llvm::Value*> GpuElementalIrEmitter::EmitComplexUnaryOp(
+    const HloInstruction* op, llvm::Value* operand_value) const {
+  PrimitiveType input_type = op->operand(0)->shape().element_type();
+  PrimitiveType component_type =
+      primitive_util::IsComplexType(input_type)
+          ? primitive_util::ComplexComponentType(input_type)
+          : input_type;
+  auto real = [&](llvm::Value* x) {
+    return ir_builder_->CreateExtractValue(x, {0});
+  };
+  auto imag = [&](llvm::Value* x) {
+    return ir_builder_->CreateExtractValue(x, {1});
+  };
+
+  switch (op->opcode()) {
+    case HloOpcode::kLog: {
+      // log(a+bi) = .5*log(a^2+b^2) + i*atan2(b, a)
+      auto a = real(operand_value);
+      auto b = imag(operand_value);
+      llvm::Type* llvm_ty = a->getType();
+      auto sum_sq = ir_builder_->CreateFAdd(ir_builder_->CreateFMul(a, a),
+                                            ir_builder_->CreateFMul(b, b));
+      TF_ASSIGN_OR_RETURN(
+          auto log_sum_sq,
+          EmitLibdeviceMathCall("__nv_log", {sum_sq}, {component_type},
+                                component_type));
+      TF_ASSIGN_OR_RETURN(
+          auto angle, EmitLibdeviceMathCall("__nv_atan2", {b, a},
+                                            {component_type, component_type},
+                                            component_type));
+      auto one_half = llvm::ConstantFP::get(llvm_ty, 0.5);
+      return ComposeComplex(op, ir_builder_->CreateFMul(one_half, log_sum_sq),
+                            angle);
+    }
+    // TODO(b/65408531): Implement kPower on GPU, where atan2 is available.
+    // case HloOpcode::kPower:
+    // // (a+bi)^(c+di) = exp(i(c+di)*arg(a+bi)) * (a*a+b*b)^(0.5(c+di))
+    case HloOpcode::kExp: {
+      // e^(a+bi) = e^a*(cos(b)+sin(b)i)
+      auto b = imag(operand_value);
+      TF_ASSIGN_OR_RETURN(
+          auto exp_a, EmitLibdeviceMathCall("__nv_exp", {real(operand_value)},
+                                            {component_type}, component_type));
+      TF_ASSIGN_OR_RETURN(
+          auto cos_b, EmitLibdeviceMathCall("__nv_cos", {b}, {component_type},
+                                            component_type));
+      TF_ASSIGN_OR_RETURN(
+          auto sin_b, EmitLibdeviceMathCall("__nv_sin", {b}, {component_type},
+                                            component_type));
+      return ComposeComplex(op, ir_builder_->CreateFMul(exp_a, cos_b),
+                            ir_builder_->CreateFMul(exp_a, sin_b));
+    }
+    case HloOpcode::kCos: {
+      // cos(a+bi) = .5(cos(a)*(e^-b+e^b) + i*sin(a)*(e^-b-e^b))
+      auto a = real(operand_value);
+      auto llvm_ty = a->getType();
+      TF_ASSIGN_OR_RETURN(
+          auto exp_b, EmitLibdeviceMathCall("__nv_exp", {imag(operand_value)},
+                                            {component_type}, component_type));
+      TF_ASSIGN_OR_RETURN(
+          auto cos_a, EmitLibdeviceMathCall("__nv_cos", {a}, {component_type},
+                                            component_type));
+      TF_ASSIGN_OR_RETURN(
+          auto sin_a, EmitLibdeviceMathCall("__nv_sin", {a}, {component_type},
+                                            component_type));
+      auto half_exp_b =
+          ir_builder_->CreateFMul(llvm::ConstantFP::get(llvm_ty, 0.5), exp_b);
+      auto half_exp_neg_b =
+          ir_builder_->CreateFDiv(llvm::ConstantFP::get(llvm_ty, 0.5), exp_b);
+      return ComposeComplex(
+          op,
+          ir_builder_->CreateFMul(
+              cos_a, ir_builder_->CreateFAdd(half_exp_neg_b, half_exp_b)),
+          ir_builder_->CreateFMul(
+              sin_a, ir_builder_->CreateFSub(half_exp_neg_b, half_exp_b)));
+    }
+
+    case HloOpcode::kSin: {
+      // sin(a+bi) = 0.5(sin(a)*(e^b+e^-b) + i*cos(a)*(e^b-e^-b)
+      auto a = real(operand_value);
+      auto llvm_ty = a->getType();
+      TF_ASSIGN_OR_RETURN(
+          auto exp_b, EmitLibdeviceMathCall("__nv_exp", {imag(operand_value)},
+                                            {component_type}, component_type));
+      TF_ASSIGN_OR_RETURN(
+          auto cos_a, EmitLibdeviceMathCall("__nv_cos", {a}, {component_type},
+                                            component_type));
+      TF_ASSIGN_OR_RETURN(
+          auto sin_a, EmitLibdeviceMathCall("__nv_sin", {a}, {component_type},
+                                            component_type));
+      auto half_exp_b =
+          ir_builder_->CreateFMul(llvm::ConstantFP::get(llvm_ty, 0.5), exp_b);
+      auto half_exp_neg_b =
+          ir_builder_->CreateFDiv(llvm::ConstantFP::get(llvm_ty, 0.5), exp_b);
+      return ComposeComplex(
+          op,
+          ir_builder_->CreateFMul(
+              sin_a, ir_builder_->CreateFAdd(half_exp_b, half_exp_neg_b)),
+          ir_builder_->CreateFMul(
+              cos_a, ir_builder_->CreateFSub(half_exp_b, half_exp_neg_b)));
+    }
+    default:
+      return ElementalIrEmitter::EmitComplexUnaryOp(op, operand_value);
+  }
+}
+
 llvm::Value* GpuElementalIrEmitter::EmitDeviceFunctionCall(
     const string& callee_name,
     tensorflow::gtl::ArraySlice<llvm::Value*> operands,
@@ -235,13 +345,12 @@ llvm::Value* GpuElementalIrEmitter::EmitDeviceFunctionCall(
   std::vector<llvm::Type*> ir_input_types;
   for (PrimitiveType input_type : input_types) {
     ir_input_types.push_back(
-        llvm_ir::PrimitiveTypeToIrType(input_type, ir_builder_));
+        llvm_ir::PrimitiveTypeToIrType(input_type, module_));
   }
   llvm::FunctionType* callee_type = llvm::FunctionType::get(
-      llvm_ir::PrimitiveTypeToIrType(output_type,
-                                     ir_builder_),  // The return type.
-      ir_input_types,                               // The parameter types.
-      false);                                       // No variadic arguments.
+      llvm_ir::PrimitiveTypeToIrType(output_type, module_),  // Return type.
+      ir_input_types,                                        // Parameter types.
+      false);  // No variadic arguments.
 
   // Declares the callee if it is not declared already.
   llvm::Function* callee = llvm::cast<llvm::Function>(
@@ -315,7 +424,7 @@ llvm_ir::ElementGenerator GpuElementalIrEmitter::MakeElementGenerator(
 
         PrimitiveType operand_element_type = operand->shape().element_type();
         llvm::Value* accum_ptr = llvm_ir::EmitAllocaAtFunctionEntry(
-            llvm_ir::PrimitiveTypeToIrType(operand_element_type, ir_builder_),
+            llvm_ir::PrimitiveTypeToIrType(operand_element_type, module_),
             "reduce_window_accum_ptr", ir_builder_);
         {
           TF_ASSIGN_OR_RETURN(llvm::Value * init_value,
@@ -377,7 +486,7 @@ llvm_ir::ElementGenerator GpuElementalIrEmitter::MakeElementGenerator(
         const HloInstruction* operand = hlo->operand(0);
         llvm::Value* accum_ptr =
             ir_builder()->CreateAlloca(llvm_ir::PrimitiveTypeToIrType(
-                hlo->shape().element_type(), ir_builder()));
+                hlo->shape().element_type(), module_));
         TF_ASSIGN_OR_RETURN(llvm::Value * init_value,
                             operand_to_generator.at(hlo->operand(1))({}));
         ir_builder()->CreateStore(init_value, accum_ptr);
diff --git a/tensorflow/compiler/xla/service/gpu/elemental_ir_emitter.h b/tensorflow/compiler/xla/service/gpu/elemental_ir_emitter.h
index 6ddfc3710c..3defa1b696 100644
--- a/tensorflow/compiler/xla/service/gpu/elemental_ir_emitter.h
+++ b/tensorflow/compiler/xla/service/gpu/elemental_ir_emitter.h
@@ -54,6 +54,9 @@ class GpuElementalIrEmitter : public ElementalIrEmitter {
   StatusOr<llvm::Value*> EmitFloatUnaryOp(
       const HloInstruction* op, llvm::Value* operand_value) const override;
 
+  StatusOr<llvm::Value*> EmitComplexUnaryOp(
+      const HloInstruction* op, llvm::Value* operand_value) const override;
+
   StatusOr<llvm::Value*> EmitFloatBinaryOp(
       const HloInstruction* op, llvm::Value* lhs_value,
       llvm::Value* rhs_value) const override;
diff --git a/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.cc b/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.cc
index 152d226ab0..163a161353 100644
--- a/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.cc
+++ b/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.cc
@@ -102,7 +102,7 @@ void HloToIrBindings::EmitBasePointersForHlos(
               slice_result.ConsumeValueOrDie();
           if (slice.allocation()->is_thread_local()) {
             llvm::Type* pointee_type =
-                llvm_ir::ShapeToIrType(non_io_hlo->shape(), ir_builder_);
+                llvm_ir::ShapeToIrType(non_io_hlo->shape(), module_);
             BindHloToIrValue(*non_io_hlo,
                              ir_builder_->CreateAlloca(pointee_type), index);
           } else {
@@ -124,18 +124,18 @@ llvm::Value* HloToIrBindings::EmitGetTupleElement(const HloInstruction* gte,
   if (gte->operand(0)->opcode() != HloOpcode::kGetTupleElement) {
     return llvm_ir::EmitGetTupleElement(
         gte->shape(), gte->tuple_index(), /*alignment=*/1,
-        GetTypedIrValue(*gte->operand(0), {}, base_ptr), ir_builder_);
+        GetTypedIrValue(*gte->operand(0), {}, base_ptr), ir_builder_, module_);
   }
   return llvm_ir::EmitGetTupleElement(
       gte->shape(), gte->tuple_index(), /*alignment=*/1,
-      EmitGetTupleElement(gte->operand(0), base_ptr), ir_builder_);
+      EmitGetTupleElement(gte->operand(0), base_ptr), ir_builder_, module_);
 }
 
 llvm::Value* HloToIrBindings::GetTypedIrValue(const HloInstruction& hlo,
                                               const ShapeIndex& shape_index,
                                               llvm::Value* ir_value) {
   llvm::Type* pointee_type = llvm_ir::ShapeToIrType(
-      ShapeUtil::GetSubshape(hlo.shape(), shape_index), ir_builder_);
+      ShapeUtil::GetSubshape(hlo.shape(), shape_index), module_);
   llvm::Type* dest_type = pointee_type->getPointerTo();
 
   llvm::Value* typed_ir_value;
diff --git a/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.h b/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.h
index d43e09e8a8..a3120f15bc 100644
--- a/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.h
+++ b/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.h
@@ -36,10 +36,12 @@ class HloToIrBindings {
  public:
   HloToIrBindings(const HloModule& module,
                   const BufferAssignment* buffer_assignment,
-                  llvm::IRBuilder<>* ir_builder, bool is_nested)
+                  llvm::IRBuilder<>* ir_builder, llvm::Module* llvm_module,
+                  bool is_nested)
       : buffer_assignment_(buffer_assignment),
         is_nested_(is_nested),
         ir_builder_(ir_builder),
+        module_(llvm_module),
         alias_analysis_(module, *buffer_assignment_,
                         &ir_builder_->getContext()) {}
 
@@ -93,6 +95,7 @@ class HloToIrBindings {
   const bool is_nested_;
 
   llvm::IRBuilder<>* ir_builder_;
+  llvm::Module* module_;
 
   // Stores the underlying llvm::IrArray for each HloInstruction.
   // For an instruction that generates multiple outputs, the root will be a
diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter.cc b/tensorflow/compiler/xla/service/gpu/ir_emitter.cc
index 3862c2190b..23765e05e8 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/gpu/ir_emitter.cc
@@ -53,9 +53,10 @@ namespace gpu {
 IrEmitter::IrEmitter(const HloModuleConfig& hlo_module_config,
                      IrEmitterContext* ir_emitter_context, bool is_nested)
     : ir_emitter_context_(ir_emitter_context),
-      ir_builder_(ir_emitter_context->llvm_module()->getContext()),
+      module_(ir_emitter_context->llvm_module()),
+      ir_builder_(module_->getContext()),
       bindings_(ir_emitter_context->hlo_module(),
-                &ir_emitter_context->buffer_assignment(), &ir_builder_,
+                &ir_emitter_context->buffer_assignment(), &ir_builder_, module_,
                 is_nested),
       hlo_module_config_(hlo_module_config) {
   ir_builder_.setFastMathFlags(llvm_ir::GetFastMathFlags(
@@ -71,18 +72,17 @@ Status IrEmitter::DefaultAction(HloInstruction* hlo) {
     };
   }
   return EmitTargetElementLoop(
-      *hlo, GpuElementalIrEmitter(hlo_module_config_,
-                                  ir_emitter_context_->llvm_module(),
-                                  &ir_builder_, GetNestedComputer())
+      *hlo, GpuElementalIrEmitter(hlo_module_config_, module_, &ir_builder_,
+                                  GetNestedComputer())
                 .MakeElementGenerator(hlo, operand_to_generator));
 }
 
 Status IrEmitter::HandleConstant(HloInstruction* constant,
                                  const Literal& literal) {
   llvm::Constant* initializer =
-      llvm_ir::ConvertLiteralToIrConstant(literal, &ir_builder_);
+      llvm_ir::ConvertLiteralToIrConstant(literal, module_);
   llvm::GlobalVariable* global_for_const = new llvm::GlobalVariable(
-      *ir_emitter_context_->llvm_module(), initializer->getType(),
+      *module_, initializer->getType(),
       /*isConstant=*/true, llvm::GlobalValue::PrivateLinkage, initializer,
       /*Name=*/"");
   VLOG(2) << "HandleConstant: " << constant->ToString() << std::endl
@@ -115,7 +115,7 @@ Status IrEmitter::HandleGetTupleElement(HloInstruction* get_tuple_element,
           get_tuple_element->shape(), get_tuple_element->tuple_index(),
           // TODO(b/26344050): tighten the alignment here
           // based on the real element type.
-          /*alignment=*/1, GetBasePointer(*operand), &ir_builder_));
+          /*alignment=*/1, GetBasePointer(*operand), &ir_builder_, module_));
   return Status::OK();
 }
 
@@ -140,7 +140,7 @@ Status IrEmitter::HandleTuple(
   for (const HloInstruction* operand : operands) {
     base_ptrs.push_back(GetBasePointer(*operand));
   }
-  llvm_ir::EmitTuple(GetIrArray(*tuple), base_ptrs, &ir_builder_);
+  llvm_ir::EmitTuple(GetIrArray(*tuple), base_ptrs, &ir_builder_, module_);
   return Status::OK();
 }
 
@@ -329,7 +329,7 @@ Status IrEmitter::HandleSelect(HloInstruction* select, HloInstruction* pred,
   if (ShapeUtil::IsTuple(select->shape())) {
     llvm_ir::EmitTupleSelect(GetIrArray(*select), GetIrArray(*pred),
                              GetBasePointer(*on_true),
-                             GetBasePointer(*on_false), &ir_builder_);
+                             GetBasePointer(*on_false), &ir_builder_, module_);
     return Status::OK();
   }
 
@@ -355,7 +355,26 @@ Status IrEmitter::HandleDot(HloInstruction* dot,
         lhs_array.EmitReadArrayElement(/*index=*/{}, &ir_builder_);
     llvm::Value* rhs_value =
         rhs_array.EmitReadArrayElement(/*index=*/{}, &ir_builder_);
-    llvm::Value* result = ir_builder_.CreateFMul(lhs_value, rhs_value);
+    llvm::Value* result;
+    if (ShapeUtil::ElementIsComplex(lhs_shape)) {
+      auto real = [&](llvm::Value* x) {
+        return ir_builder_.CreateExtractValue(x, {0});
+      };
+      auto imag = [&](llvm::Value* x) {
+        return ir_builder_.CreateExtractValue(x, {1});
+      };
+      llvm::Value* real_result = ir_builder_.CreateFSub(
+          ir_builder_.CreateFMul(real(lhs_value), real(rhs_value)),
+          ir_builder_.CreateFMul(imag(lhs_value), imag(rhs_value)));
+      llvm::Value* imag_result = ir_builder_.CreateFAdd(
+          ir_builder_.CreateFMul(real(lhs_value), imag(rhs_value)),
+          ir_builder_.CreateFMul(imag(lhs_value), real(rhs_value)));
+      result = llvm::ConstantAggregateZero::get(lhs_array.GetElementLlvmType());
+      result = ir_builder_.CreateInsertValue(result, real_result, {0});
+      result = ir_builder_.CreateInsertValue(result, imag_result, {1});
+    } else {
+      result = ir_builder_.CreateFMul(lhs_value, rhs_value);
+    }
     target_array.EmitWriteArrayElement(/*index=*/{}, result, &ir_builder_);
     return Status::OK();
   }
@@ -411,8 +430,8 @@ Status IrEmitter::HandleDot(HloInstruction* dot,
 
   // Initialize the accumulator in the preheader to zero.
   new llvm::StoreInst(
-      llvm::ConstantFP::get(accum_type, 0.0),  // The value stored.
-      accum_address,                           // The address.
+      llvm::Constant::getNullValue(lhs_array.GetElementLlvmType()),  // init 0
+      accum_address,  // The address.
       reduction_loop->GetPreheaderBasicBlock()
           ->getTerminator());  // The instruction this store is inserted before.
 
@@ -427,9 +446,27 @@ Status IrEmitter::HandleDot(HloInstruction* dot,
       lhs_array.EmitReadArrayElement(lhs_index, &ir_builder_);
   llvm::Value* rhs_element =
       rhs_array.EmitReadArrayElement(rhs_index, &ir_builder_);
-  llvm::Value* product = ir_builder_.CreateFMul(lhs_element, rhs_element);
   llvm::Value* accum = ir_builder_.CreateLoad(accum_address);
-  llvm::Value* updated_accum = ir_builder_.CreateFAdd(accum, product);
+  llvm::Value* updated_accum;
+  if (ShapeUtil::ElementIsComplex(lhs_shape)) {
+#define REAL(x) ir_builder_.CreateExtractValue(x, {0})
+#define IMAG(x) ir_builder_.CreateExtractValue(x, {1})
+    llvm::Value* product_real = ir_builder_.CreateFSub(
+        ir_builder_.CreateFMul(REAL(lhs_element), REAL(rhs_element)),
+        ir_builder_.CreateFMul(IMAG(lhs_element), IMAG(rhs_element)));
+    llvm::Value* product_imag = ir_builder_.CreateFAdd(
+        ir_builder_.CreateFMul(REAL(lhs_element), IMAG(rhs_element)),
+        ir_builder_.CreateFMul(IMAG(lhs_element), REAL(rhs_element)));
+    updated_accum = ir_builder_.CreateInsertValue(
+        accum, ir_builder_.CreateFAdd(REAL(accum), product_real), {0});
+    updated_accum = ir_builder_.CreateInsertValue(
+        updated_accum, ir_builder_.CreateFAdd(IMAG(accum), product_imag), {1});
+#undef IMAG
+#undef REAL
+  } else {
+    llvm::Value* product = ir_builder_.CreateFMul(lhs_element, rhs_element);
+    updated_accum = ir_builder_.CreateFAdd(accum, product);
+  }
   ir_builder_.CreateStore(updated_accum, accum_address);
 
   // After the reduction loop exits, store the accumulator into the target
@@ -494,7 +531,7 @@ Status IrEmitter::HandleReduce(HloInstruction* reduce, HloInstruction* arg,
         // Initialize an accumulator with init_value.
         llvm::AllocaInst* accumulator_addr =
             ir_builder_.CreateAlloca(llvm_ir::PrimitiveTypeToIrType(
-                reduce->shape().element_type(), &ir_builder_));
+                reduce->shape().element_type(), module_));
         ir_builder_.CreateStore(
             ir_builder_.CreateLoad(GetBasePointer(*init_value)),
             accumulator_addr);
@@ -547,8 +584,7 @@ Status IrEmitter::HandleFusion(HloInstruction* fusion) {
   for (HloInstruction* operand : fusion->operands()) {
     parameter_arrays.push_back(GetIrArray(*operand));
   }
-  GpuElementalIrEmitter elemental_emitter(hlo_module_config_,
-                                          ir_emitter_context_->llvm_module(),
+  GpuElementalIrEmitter elemental_emitter(hlo_module_config_, module_,
                                           &ir_builder_, GetNestedComputer());
   FusedIrEmitter fused_emitter(parameter_arrays, &elemental_emitter);
   TF_RETURN_IF_ERROR(fusion->fused_expression_root()->Accept(&fused_emitter));
@@ -591,9 +627,8 @@ Status IrEmitter::HandleRng(HloInstruction* random,
   // Emits a single-threaded loop because the loop body generated by the element
   // generator for Rng can't be parallelized (b/32333178).
   return llvm_ir::LoopEmitter(
-             GpuElementalIrEmitter(hlo_module_config_,
-                                   ir_emitter_context_->llvm_module(),
-                                   &ir_builder_, GetNestedComputer())
+             GpuElementalIrEmitter(hlo_module_config_, module_, &ir_builder_,
+                                   GetNestedComputer())
                  .MakeElementGenerator(random, operand_to_generator),
              GetIrArray(*random), &ir_builder_)
       .EmitLoop(IrName(random));
@@ -634,7 +669,7 @@ StatusOr<llvm::Value*> IrEmitter::ComputeNestedElement(
     tensorflow::gtl::ArraySlice<llvm::Value*> parameter_elements) {
   llvm::Value* return_buffer = llvm_ir::EmitAllocaAtFunctionEntry(
       llvm_ir::PrimitiveTypeToIrType(
-          computation.root_instruction()->shape().element_type(), &ir_builder_),
+          computation.root_instruction()->shape().element_type(), module_),
       "return_buffer", &ir_builder_);
   std::vector<llvm::Value*> parameter_buffers;
   for (llvm::Value* parameter_element : parameter_elements) {
diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter.h b/tensorflow/compiler/xla/service/gpu/ir_emitter.h
index 29c3761dc5..90f40639d5 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emitter.h
+++ b/tensorflow/compiler/xla/service/gpu/ir_emitter.h
@@ -162,6 +162,7 @@ class IrEmitter : public DfsHloVisitorWithDefault {
   }
 
   IrEmitterContext* ir_emitter_context_;
+  llvm::Module* module_;
 
   // The following fields track the IR emission state. According to LLVM memory
   // management rules, their memory is owned by the module.
diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_nested.cc b/tensorflow/compiler/xla/service/gpu/ir_emitter_nested.cc
index 57f010530c..5da1a130d5 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emitter_nested.cc
+++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_nested.cc
@@ -52,9 +52,9 @@ llvm::Function* IrEmitterNested::EmitBasePointersForNestedComputation(
     io_hlos->push_back(param);
     const Shape& param_shape = param->shape();
     argument_types.push_back(
-        llvm_ir::ShapeToIrType(param_shape, &ir_builder_)->getPointerTo());
-    int64 param_size = llvm_ir::ByteSizeOf(
-        param_shape, ir_emitter_context_->llvm_module()->getDataLayout());
+        llvm_ir::ShapeToIrType(param_shape, module_)->getPointerTo());
+    int64 param_size =
+        llvm_ir::ByteSizeOf(param_shape, module_->getDataLayout());
     argument_dereferenceable_bytes.push_back(param_size);
   }
   {
@@ -62,7 +62,7 @@ llvm::Function* IrEmitterNested::EmitBasePointersForNestedComputation(
     io_hlos->push_back(root);
     const Shape& root_shape = root->shape();
     argument_types.push_back(
-        llvm_ir::ShapeToIrType(root_shape, &ir_builder_)->getPointerTo());
+        llvm_ir::ShapeToIrType(root_shape, module_)->getPointerTo());
     int64 root_size = llvm_ir::ByteSizeOf(
         root_shape, ir_emitter_context_->llvm_module()->getDataLayout());
     argument_dereferenceable_bytes.push_back(root_size);
diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
index 734c793c15..1c7e18304d 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
+++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
@@ -757,8 +757,8 @@ Status IrEmitterUnnested::EmitColumnReduction(
   auto loop_body_emitter =
       [=](const llvm_ir::IrArray::Index& tile_index) -> Status {
     // Emit the loop body that reduces one tile.
-    llvm::Type* element_ir_type = llvm_ir::PrimitiveTypeToIrType(
-        input_shape.element_type(), &ir_builder_);
+    llvm::Type* element_ir_type =
+        llvm_ir::PrimitiveTypeToIrType(input_shape.element_type(), module_);
     llvm::Value* partial_reduction_result_address = ir_builder_.CreateAlloca(
         element_ir_type, /*ArraySize=*/nullptr, "partial_reduction_result");
     {
@@ -973,7 +973,7 @@ Status IrEmitterUnnested::EmitRowReduction(
       [=](const llvm_ir::IrArray::Index& tile_index) -> Status {
     // Emit the loop body that reduces one tile.
     llvm::Type* element_ir_type = llvm_ir::PrimitiveTypeToIrType(
-        input_shape.element_type(), &ir_builder_);
+        input_shape.element_type(), ir_emitter_context_->llvm_module());
     llvm::Value* partial_reduction_result_address = ir_builder_.CreateAlloca(
         element_ir_type, /*ArraySize=*/nullptr, "partial_reduction_result");
     {
@@ -1360,7 +1360,8 @@ Status IrEmitterUnnested::HandleSelectAndScatter(
     // boolean flag if the value is initialized. The initialized_flag is set
     // false.
     llvm::Value* selected_value_address = llvm_ir::EmitAllocaAtFunctionEntry(
-        llvm_ir::PrimitiveTypeToIrType(operand_element_type, &ir_builder_),
+        llvm_ir::PrimitiveTypeToIrType(operand_element_type,
+                                       ir_emitter_context_->llvm_module()),
         "selected_value_address", &ir_builder_);
     llvm::Value* selected_index_address =
         llvm_ir::EmitAllocaAtFunctionEntryWithCount(
@@ -1440,7 +1441,8 @@ Status IrEmitterUnnested::HandleSelectAndScatter(
     llvm::Value* operand_address =
         operand_array.EmitArrayElementAddress(operand_index, &ir_builder_);
     llvm::Value* select_return_buffer = llvm_ir::EmitAllocaAtFunctionEntry(
-        llvm_ir::PrimitiveTypeToIrType(PRED, &ir_builder_),
+        llvm_ir::PrimitiveTypeToIrType(PRED,
+                                       ir_emitter_context_->llvm_module()),
         "select_return_buffer", &ir_builder_);
     TF_RETURN_IF_ERROR(EmitCallToNestedComputation(
         *select_and_scatter->select(),
@@ -1450,8 +1452,10 @@ Status IrEmitterUnnested::HandleSelectAndScatter(
     // If the 'select' function returns false, update the selected value and the
     // index to the currently visiting operand.
     llvm::Value* cond = ir_builder_.CreateICmpNE(
-        result, llvm::ConstantInt::get(
-                    llvm_ir::PrimitiveTypeToIrType(PRED, &ir_builder_), 0),
+        result,
+        llvm::ConstantInt::get(llvm_ir::PrimitiveTypeToIrType(
+                                   PRED, ir_emitter_context_->llvm_module()),
+                               0),
         "boolean_predicate");
     llvm_ir::LlvmIfData if_select_lhs =
         llvm_ir::EmitIfThenElse(cond, "if-select-lhs", &ir_builder_);
@@ -1877,7 +1881,8 @@ Status IrEmitterUnnested::EmitTargetElementLoopInThunk(
     tuple_operand_ptrs.push_back(output_arrays[i].GetBasePointer());
   }
   ir_builder_.SetInsertPoint(ir_builder_.GetInsertBlock()->getTerminator());
-  llvm_ir::EmitTuple(GetIrArray(hlo), tuple_operand_ptrs, &ir_builder_);
+  llvm_ir::EmitTuple(GetIrArray(hlo), tuple_operand_ptrs, &ir_builder_,
+                     module_);
   return Status::OK();
 }
 
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator.cc b/tensorflow/compiler/xla/service/hlo_evaluator.cc
index fa6a8f3d53..f4a2c3d0e8 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator.cc
+++ b/tensorflow/compiler/xla/service/hlo_evaluator.cc
@@ -50,6 +50,12 @@ namespace xla {
 
 namespace {
 
+template <typename T>
+struct is_complex_t : public std::false_type {};
+
+template <>
+struct is_complex_t<complex64> : public std::true_type {};
+
 template <typename OperandT>
 StatusOr<std::unique_ptr<Literal>> Compare(const Shape& shape, HloOpcode opcode,
                                            const Literal& lhs_literal,
@@ -101,6 +107,37 @@ StatusOr<std::unique_ptr<Literal>> Compare(const Shape& shape, HloOpcode opcode,
   return std::move(result);
 }
 
+template <>
+StatusOr<std::unique_ptr<Literal>> Compare<complex64>(
+    const Shape& shape, HloOpcode opcode, const Literal& lhs_literal,
+    const Literal& rhs_literal) {
+  std::function<bool(complex64, complex64)> compare_op;
+  switch (opcode) {
+    case HloOpcode::kEq:
+      compare_op = [](complex64 lhs_el, complex64 rhs_el) {
+        return lhs_el == rhs_el;
+      };
+      break;
+    case HloOpcode::kNe:
+      compare_op = [](complex64 lhs_el, complex64 rhs_el) {
+        return lhs_el != rhs_el;
+      };
+      break;
+    default:
+      LOG(FATAL) << "unhandled HLO opcode for conversion to Comparison: "
+                 << HloOpcodeString(opcode);
+  }
+
+  auto result = Literal::CreateFromShape(shape);
+  TF_RETURN_IF_ERROR(result->Populate<bool>(
+      [&](tensorflow::gtl::ArraySlice<int64> multi_index) {
+        return compare_op(lhs_literal.Get<complex64>(multi_index),
+                          rhs_literal.Get<complex64>(multi_index));
+      }));
+
+  return std::move(result);
+}
+
 template <typename ReturnT, typename NativeT>
 StatusOr<std::unique_ptr<Literal>> ElementWiseUnaryOpImpl(
     HloInstruction* instruction,
@@ -138,7 +175,7 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault {
   Status DefaultAction(HloInstruction* hlo_instruction) override {
     return Unimplemented("unhandled HLO ops for HloEvaluator: %s.",
                          HloOpcodeString(hlo_instruction->opcode()).c_str());
-  };
+  }
 
   // TODO(b/35950897): many of the stl functions used in the handlers are not
   // overloaded for every XLA primitive types.
@@ -156,7 +193,8 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault {
 
   template <
       typename NativeT,
-      typename std::enable_if<std::is_signed<NativeT>::value>::type* = nullptr>
+      typename std::enable_if<std::is_signed<NativeT>::value ||
+                              is_complex_t<NativeT>::value>::type* = nullptr>
   Status HandleAbs(HloInstruction* abs, HloInstruction* operand) {
     TF_ASSIGN_OR_RETURN(parent_->evaluated_[abs],
                         ElementWiseUnaryOp(abs, [](NativeT elem_operand) {
@@ -169,7 +207,10 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault {
     return HandleAbs<ReturnT>(abs, operand);
   }
 
-  Status HandleRound(HloInstruction* round) override {
+  template <
+      typename NativeT,
+      typename std::enable_if<!is_complex_t<NativeT>::value>::type* = nullptr>
+  Status HandleRound(HloInstruction* round) {
     TF_ASSIGN_OR_RETURN(parent_->evaluated_[round],
                         ElementWiseUnaryOp(round, [](ReturnT elem_operand) {
                           return std::round(elem_operand);
@@ -177,6 +218,17 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault {
     return Status::OK();
   }
 
+  template <
+      typename NativeT,
+      typename std::enable_if<is_complex_t<NativeT>::value>::type* = nullptr>
+  Status HandleRound(HloInstruction* round) {
+    return InvalidArgument("Unsupported type for Round");
+  }
+
+  Status HandleRound(HloInstruction* round) override {
+    return HandleRound<ReturnT>(round);
+  }
+
   Status HandleBroadcast(HloInstruction* broadcast) override {
     parent_->evaluated_[broadcast] =
         Literal::CreateFromShape(broadcast->shape());
@@ -205,15 +257,29 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault {
           }
           return operand_to_broadcast.Get<ReturnT>(broadcast_indices);
         });
-  };
+  }
 
-  Status HandleCeil(HloInstruction* ceil, HloInstruction* operand) override {
+  template <
+      typename NativeT,
+      typename std::enable_if<!is_complex_t<NativeT>::value>::type* = nullptr>
+  Status HandleCeil(HloInstruction* ceil) {
     TF_ASSIGN_OR_RETURN(parent_->evaluated_[ceil],
                         ElementWiseUnaryOp(ceil, [](ReturnT elem_operand) {
                           return std::ceil(elem_operand);
                         }));
     return Status::OK();
-  };
+  }
+
+  template <
+      typename NativeT,
+      typename std::enable_if<is_complex_t<NativeT>::value>::type* = nullptr>
+  Status HandleCeil(HloInstruction* ceil) {
+    return InvalidArgument("Unsupported type for Ceil");
+  }
+
+  Status HandleCeil(HloInstruction* ceil, HloInstruction* operand) override {
+    return HandleCeil<ReturnT>(ceil);
+  }
 
   Status HandleConvert(HloInstruction* convert) override {
     const HloInstruction* operand = convert->operand(0);
@@ -237,15 +303,29 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault {
                           return std::exp(elem_operand);
                         }));
     return Status::OK();
-  };
+  }
 
-  Status HandleFloor(HloInstruction* floor, HloInstruction* operand) override {
+  template <
+      typename NativeT,
+      typename std::enable_if<!is_complex_t<NativeT>::value>::type* = nullptr>
+  Status HandleFloor(HloInstruction* floor) {
     TF_ASSIGN_OR_RETURN(parent_->evaluated_[floor],
                         ElementWiseUnaryOp(floor, [](ReturnT elem_operand) {
                           return std::floor(elem_operand);
                         }));
     return Status::OK();
-  };
+  }
+
+  template <
+      typename NativeT,
+      typename std::enable_if<is_complex_t<NativeT>::value>::type* = nullptr>
+  Status HandleFloor(HloInstruction* floor) {
+    return InvalidArgument("Unsupported type for Floor");
+  }
+
+  Status HandleFloor(HloInstruction* floor, HloInstruction* operand) override {
+    return HandleFloor<ReturnT>(floor);
+  }
 
   Status HandleLog(HloInstruction* log, HloInstruction* operand) override {
     TF_ASSIGN_OR_RETURN(parent_->evaluated_[log],
@@ -253,15 +333,29 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault {
                           return std::log(elem_operand);
                         }));
     return Status::OK();
-  };
+  }
 
-  Status HandleNot(HloInstruction* not_, HloInstruction* operand) override {
+  template <
+      typename NativeT,
+      typename std::enable_if<!is_complex_t<NativeT>::value>::type* = nullptr>
+  Status HandleNot(HloInstruction* not_) {
     TF_ASSIGN_OR_RETURN(parent_->evaluated_[not_],
                         ElementWiseUnaryOp(not_, [](ReturnT elem_operand) {
                           return !elem_operand;
                         }));
     return Status::OK();
-  };
+  }
+
+  template <
+      typename NativeT,
+      typename std::enable_if<is_complex_t<NativeT>::value>::type* = nullptr>
+  Status HandleNot(HloInstruction* not_) {
+    return InvalidArgument("Unsupported type for Not");
+  }
+
+  Status HandleNot(HloInstruction* not_, HloInstruction* operand) override {
+    return HandleNot<ReturnT>(not_);
+  }
 
   Status HandleNegate(HloInstruction* negate,
                       HloInstruction* operand) override {
@@ -270,16 +364,36 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault {
                           return -elem_operand;
                         }));
     return Status::OK();
-  };
+  }
 
-  Status HandleSign(HloInstruction* sign, HloInstruction* operand) override {
+  template <
+      typename NativeT,
+      typename std::enable_if<!is_complex_t<NativeT>::value>::type* = nullptr>
+  Status HandleSign(HloInstruction* sign) {
     TF_ASSIGN_OR_RETURN(parent_->evaluated_[sign],
                         ElementWiseUnaryOp(sign, [](ReturnT elem_operand) {
                           return (ReturnT(0) < elem_operand) -
                                  (elem_operand < ReturnT(0));
                         }));
     return Status::OK();
-  };
+  }
+
+  template <
+      typename NativeT,
+      typename std::enable_if<is_complex_t<NativeT>::value>::type* = nullptr>
+  Status HandleSign(HloInstruction* sign) {
+    TF_ASSIGN_OR_RETURN(parent_->evaluated_[sign],
+                        ElementWiseUnaryOp(sign, [](ReturnT elem_operand) {
+                          auto abs_val = std::abs(elem_operand);
+                          return 0 == abs_val ? ReturnT(0)
+                                              : elem_operand / abs_val;
+                        }));
+    return Status::OK();
+  }
+
+  Status HandleSign(HloInstruction* sign, HloInstruction* operand) override {
+    return HandleSign<ReturnT>(sign);
+  }
 
   Status HandleTanh(HloInstruction* tanh, HloInstruction* operand) override {
     TF_ASSIGN_OR_RETURN(parent_->evaluated_[tanh],
@@ -287,7 +401,7 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault {
                           return std::tanh(elem_operand);
                         }));
     return Status::OK();
-  };
+  }
 
   Status HandleMultiply(HloInstruction* multiply, HloInstruction* lhs,
                         HloInstruction* rhs) override {
@@ -297,7 +411,7 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault {
           return lhs_elem * rhs_elem;
         }));
     return Status::OK();
-  };
+  }
 
   Status HandleSubtract(HloInstruction* subtract, HloInstruction* lhs,
                         HloInstruction* rhs) override {
@@ -307,7 +421,7 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault {
           return lhs_elem - rhs_elem;
         }));
     return Status::OK();
-  };
+  }
 
   Status HandleAdd(HloInstruction* add, HloInstruction* lhs,
                    HloInstruction* rhs) override {
@@ -317,7 +431,7 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault {
           return lhs_elem + rhs_elem;
         }));
     return Status::OK();
-  };
+  }
 
   Status HandleDivide(HloInstruction* divide, HloInstruction* lhs,
                       HloInstruction* rhs) override {
@@ -327,25 +441,53 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault {
           return lhs_elem / rhs_elem;
         }));
     return Status::OK();
-  };
+  }
 
-  Status HandleMaximum(HloInstruction* maximum) override {
+  template <
+      typename NativeT,
+      typename std::enable_if<!is_complex_t<NativeT>::value>::type* = nullptr>
+  Status HandleMaximum(HloInstruction* maximum) {
     TF_ASSIGN_OR_RETURN(
         parent_->evaluated_[maximum],
         ElementWiseBinaryOp(maximum, [](ReturnT lhs, ReturnT rhs) {
           return std::fmax(lhs, rhs);
         }));
     return Status::OK();
-  };
+  }
 
-  Status HandleMinimum(HloInstruction* minimum) override {
+  template <
+      typename NativeT,
+      typename std::enable_if<is_complex_t<NativeT>::value>::type* = nullptr>
+  Status HandleMaximum(HloInstruction* maximum) {
+    return InvalidArgument("Unsupported type for Maximum");
+  }
+
+  Status HandleMaximum(HloInstruction* maximum) override {
+    return HandleMaximum<ReturnT>(maximum);
+  }
+
+  template <
+      typename NativeT,
+      typename std::enable_if<!is_complex_t<NativeT>::value>::type* = nullptr>
+  Status HandleMinimum(HloInstruction* minimum) {
     TF_ASSIGN_OR_RETURN(
         parent_->evaluated_[minimum],
         ElementWiseBinaryOp(minimum, [](ReturnT lhs_el, ReturnT rhs_el) {
           return std::fmin(lhs_el, rhs_el);
         }));
     return Status::OK();
-  };
+  }
+
+  template <
+      typename NativeT,
+      typename std::enable_if<is_complex_t<NativeT>::value>::type* = nullptr>
+  Status HandleMinimum(HloInstruction* minimum) {
+    return InvalidArgument("Unsupported type for Minimum");
+  }
+
+  Status HandleMinimum(HloInstruction* minimum) override {
+    return HandleMinimum<ReturnT>(minimum);
+  }
 
   Status HandlePower(HloInstruction* power, HloInstruction* lhs,
                      HloInstruction* rhs) override {
@@ -355,37 +497,79 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault {
           return std::pow(lhs_el, rhs_el);
         }));
     return Status::OK();
-  };
+  }
 
-  Status HandleRemainder(HloInstruction* remainder, HloInstruction* lhs,
-                         HloInstruction* rhs) override {
+  template <
+      typename NativeT,
+      typename std::enable_if<!is_complex_t<NativeT>::value>::type* = nullptr>
+  Status HandleRemainder(HloInstruction* remainder) {
     TF_ASSIGN_OR_RETURN(
         parent_->evaluated_[remainder],
         ElementWiseBinaryOp(remainder, [](ReturnT lhs_el, ReturnT rhs_el) {
           return std::fmod(lhs_el, rhs_el);
         }));
     return Status::OK();
-  };
+  }
 
-  Status HandleAnd(HloInstruction* and_, HloInstruction* lhs,
-                   HloInstruction* rhs) override {
+  template <
+      typename NativeT,
+      typename std::enable_if<is_complex_t<NativeT>::value>::type* = nullptr>
+  Status HandleRemainder(HloInstruction* remainder) {
+    return InvalidArgument("Unsupported type for Remainder");
+  }
+
+  Status HandleRemainder(HloInstruction* remainder, HloInstruction* lhs,
+                         HloInstruction* rhs) override {
+    return HandleRemainder<ReturnT>(remainder);
+  }
+
+  template <
+      typename NativeT,
+      typename std::enable_if<!is_complex_t<NativeT>::value>::type* = nullptr>
+  Status HandleAnd(HloInstruction* and_) {
     TF_ASSIGN_OR_RETURN(
         parent_->evaluated_[and_],
         ElementWiseBinaryOp(and_, [](ReturnT lhs_el, ReturnT rhs_el) {
           return lhs_el && rhs_el;
         }));
     return Status::OK();
-  };
+  }
 
-  Status HandleOr(HloInstruction* or_, HloInstruction* lhs,
-                  HloInstruction* rhs) override {
+  template <
+      typename NativeT,
+      typename std::enable_if<is_complex_t<NativeT>::value>::type* = nullptr>
+  Status HandleAnd(HloInstruction* and_) {
+    return InvalidArgument("Unsupported type for And");
+  }
+
+  Status HandleAnd(HloInstruction* and_, HloInstruction* lhs,
+                   HloInstruction* rhs) override {
+    return HandleAnd<ReturnT>(and_);
+  }
+
+  template <
+      typename NativeT,
+      typename std::enable_if<!is_complex_t<NativeT>::value>::type* = nullptr>
+  Status HandleOr(HloInstruction* or_) {
     TF_ASSIGN_OR_RETURN(
         parent_->evaluated_[or_],
         ElementWiseBinaryOp(or_, [](ReturnT lhs_el, ReturnT rhs_el) {
           return lhs_el || rhs_el;
         }));
     return Status::OK();
-  };
+  }
+
+  template <
+      typename NativeT,
+      typename std::enable_if<is_complex_t<NativeT>::value>::type* = nullptr>
+  Status HandleOr(HloInstruction* or_) {
+    return InvalidArgument("Unsupported type for Or");
+  }
+
+  Status HandleOr(HloInstruction* or_, HloInstruction* lhs,
+                  HloInstruction* rhs) override {
+    return HandleOr<ReturnT>(or_);
+  }
 
   template <typename NativeT,
             typename std::enable_if<
@@ -474,8 +658,11 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault {
     return HandleShiftRightLogical<ReturnT>(shrl, lhs, rhs);
   }
 
+  template <
+      typename NativeT,
+      typename std::enable_if<!is_complex_t<NativeT>::value>::type* = nullptr>
   Status HandleClamp(HloInstruction* clamp, HloInstruction* min,
-                     HloInstruction* arg, HloInstruction* max) override {
+                     HloInstruction* arg, HloInstruction* max) {
     std::function<ReturnT(ReturnT, ReturnT, ReturnT)> clamp_op =
         [](ReturnT low, ReturnT high, ReturnT value) {
           return std::fmax(low, std::fmin(value, high));
@@ -483,7 +670,20 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault {
     TF_ASSIGN_OR_RETURN(parent_->evaluated_[clamp],
                         ElementWiseTernaryOp(clamp, std::move(clamp_op)));
     return Status::OK();
-  };
+  }
+
+  template <
+      typename NativeT,
+      typename std::enable_if<is_complex_t<NativeT>::value>::type* = nullptr>
+  Status HandleClamp(HloInstruction* clamp, HloInstruction* min,
+                     HloInstruction* arg, HloInstruction* max) {
+    return InvalidArgument("Unsupported type for Clamp");
+  }
+
+  Status HandleClamp(HloInstruction* clamp, HloInstruction* min,
+                     HloInstruction* arg, HloInstruction* max) override {
+    return HandleClamp<ReturnT>(clamp, min, arg, max);
+  }
 
   Status HandleSelect(HloInstruction* select, HloInstruction* pred,
                       HloInstruction* on_true,
@@ -499,7 +699,7 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault {
     TF_ASSIGN_OR_RETURN(parent_->evaluated_[select],
                         ElementWiseTernaryOp(select, std::move(select_op)));
     return Status::OK();
-  };
+  }
 
   Status HandleReverse(HloInstruction* reverse,
                        HloInstruction* operand) override {
@@ -529,7 +729,7 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault {
 
     parent_->evaluated_[reverse] = std::move(result);
     return Status::OK();
-  };
+  }
 
   Status HandleConvolution(HloInstruction* conv, HloInstruction* lhs,
                            HloInstruction* rhs, const Window& window) override {
@@ -652,7 +852,7 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault {
 
     parent_->evaluated_[conv] = std::move(result);
     return Status::OK();
-  };
+  }
 
   Status HandleDot(HloInstruction* dot, HloInstruction* lhs,
                    HloInstruction* rhs) override {
@@ -719,7 +919,7 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault {
 
     parent_->evaluated_[dot] = std::move(result);
     return Status::OK();
-  };
+  }
 
   Status HandlePad(HloInstruction* pad) override {
     CHECK(!ShapeUtil::IsTuple(pad->operand(0)->shape()));
@@ -788,7 +988,7 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault {
 
     parent_->evaluated_[pad] = std::move(result);
     return Status::OK();
-  };
+  }
 
   Status HandleDynamicSlice(HloInstruction* dynamic_slice,
                             HloInstruction* operand,
@@ -841,7 +1041,7 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault {
     }
 
     return Status::OK();
-  };
+  }
 
   Status HandleDynamicUpdateSlice(HloInstruction* dynamic_update_slice,
                                   HloInstruction* operand,
@@ -897,7 +1097,7 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault {
     }
 
     return Status::OK();
-  };
+  }
 
   Status HandleReduce(HloInstruction* reduce, HloInstruction* arg,
                       HloInstruction* init_value,
@@ -985,7 +1185,7 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault {
 
     parent_->evaluated_[reduce] = std::move(result);
     return Status::OK();
-  };
+  }
 
   Status HandleReduceWindow(HloInstruction* reduce_window,
                             HloInstruction* operand, const Window& window,
@@ -1072,7 +1272,7 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault {
 
     parent_->evaluated_[reduce_window] = std::move(result);
     return Status::OK();
-  };
+  }
 
   Status HandleSlice(HloInstruction* slice, HloInstruction* operand) override {
     const Shape& shape = slice->shape();
@@ -1101,7 +1301,7 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault {
     TF_RETURN_IF_ERROR(result->Populate<ReturnT>(func));
     parent_->evaluated_[slice] = std::move(result);
     return Status::OK();
-  };
+  }
 
  private:
   template <typename IndexT>
@@ -1244,35 +1444,33 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault {
   }
 
   HloEvaluator* parent_;
-};  // namespace xla
+};  // class HloEvaluator::TypedVisitor
 
 HloEvaluator::HloEvaluator() {
   typed_visitors_[PRED] = MakeUnique<TypedVisitor<bool>>(this);
   typed_visitors_[U8] = MakeUnique<TypedVisitor<uint8>>(this);
   typed_visitors_[U16] = MakeUnique<FunctionVisitor>([](HloInstruction*) {
-    return Unimplemented("unhandled primitive type: U16.");
+    return Unimplemented("HloEvaluator: unhandled primitive type: U16.");
   });
   typed_visitors_[U32] = MakeUnique<TypedVisitor<uint32>>(this);
   typed_visitors_[U64] = MakeUnique<TypedVisitor<uint64>>(this);
   typed_visitors_[S8] = MakeUnique<TypedVisitor<int8>>(this);
   typed_visitors_[S16] = MakeUnique<FunctionVisitor>([](HloInstruction*) {
-    return Unimplemented("unhandled primitive type: S16.");
+    return Unimplemented("HloEvaluator: unhandled primitive type: S16.");
   });
   typed_visitors_[S32] = MakeUnique<TypedVisitor<int32>>(this);
   typed_visitors_[S64] = MakeUnique<TypedVisitor<int64>>(this);
   typed_visitors_[F16] = MakeUnique<FunctionVisitor>([](HloInstruction*) {
-    return Unimplemented("unhandled primitive type: F16.");
+    return Unimplemented("HloEvaluator: unhandled primitive type: F16.");
   });
   typed_visitors_[F32] = MakeUnique<TypedVisitor<float>>(this);
   typed_visitors_[F64] = MakeUnique<TypedVisitor<double>>(this);
-  typed_visitors_[C64] = MakeUnique<FunctionVisitor>([](HloInstruction*) {
-    return Unimplemented("unhandled primitive type: C64.");
-  });
+  typed_visitors_[C64] = MakeUnique<TypedVisitor<complex64>>(this);
   typed_visitors_[TUPLE] = MakeUnique<FunctionVisitor>([](HloInstruction*) {
-    return Unimplemented("unhandled primitive type: TUPLE.");
+    return Unimplemented("HloEvaluator: unhandled primitive type: TUPLE.");
   });
   typed_visitors_[OPAQUE] = MakeUnique<FunctionVisitor>([](HloInstruction*) {
-    return Unimplemented("unhandled primitive type: OPAQUE.");
+    return Unimplemented("HloEvaluator: unhandled primitive type: OPAQUE.");
   });
 }
 
@@ -1573,6 +1771,11 @@ Status HloEvaluator::HandleCompare(HloInstruction* compare, HloOpcode opcode,
           evaluated_[compare],
           Compare<double>(compare->shape(), opcode, lhs_literal, rhs_literal));
     } break;
+    case C64: {
+      TF_ASSIGN_OR_RETURN(evaluated_[compare],
+                          Compare<complex64>(compare->shape(), opcode,
+                                             lhs_literal, rhs_literal));
+    } break;
     default:
       LOG(FATAL) << "HandleCompare: unknown primitive type: "
                  << PrimitiveType_Name(lhs->shape().element_type());
diff --git a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
index 7b9cbeb6f4..d0202556bc 100644
--- a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
+++ b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
@@ -826,8 +826,10 @@ ColorScheme HloDotDumper::GetInstructionColor(const HloInstruction* instr) {
     case HloOpcode::kAbs:
     case HloOpcode::kRoundNearestAfz:
     case HloOpcode::kAdd:
+    case HloOpcode::kAtan2:
     case HloOpcode::kCeil:
     case HloOpcode::kClamp:
+    case HloOpcode::kComplex:
     case HloOpcode::kConvert:
     case HloOpcode::kCos:
     case HloOpcode::kDivide:
@@ -836,6 +838,7 @@ ColorScheme HloDotDumper::GetInstructionColor(const HloInstruction* instr) {
     case HloOpcode::kFloor:
     case HloOpcode::kGe:
     case HloOpcode::kGt:
+    case HloOpcode::kImag:
     case HloOpcode::kIndex:
     case HloOpcode::kIsFinite:
     case HloOpcode::kLe:
@@ -850,6 +853,7 @@ ColorScheme HloDotDumper::GetInstructionColor(const HloInstruction* instr) {
     case HloOpcode::kNe:
     case HloOpcode::kNegate:
     case HloOpcode::kPower:
+    case HloOpcode::kReal:
     case HloOpcode::kRemainder:
     case HloOpcode::kShiftLeft:
     case HloOpcode::kShiftRightArithmetic:
diff --git a/tensorflow/compiler/xla/service/hlo_instruction.cc b/tensorflow/compiler/xla/service/hlo_instruction.cc
index 272f573623..1a03e7ee92 100644
--- a/tensorflow/compiler/xla/service/hlo_instruction.cc
+++ b/tensorflow/compiler/xla/service/hlo_instruction.cc
@@ -219,10 +219,12 @@ HloInstruction::CreateGetTupleElement(const Shape& shape,
     case HloOpcode::kCos:
     case HloOpcode::kExp:
     case HloOpcode::kFloor:
+    case HloOpcode::kImag:
     case HloOpcode::kIsFinite:
     case HloOpcode::kLog:
     case HloOpcode::kNot:
     case HloOpcode::kNegate:
+    case HloOpcode::kReal:
     case HloOpcode::kSign:
     case HloOpcode::kSin:
     case HloOpcode::kSort:
@@ -241,26 +243,28 @@ HloInstruction::CreateGetTupleElement(const Shape& shape,
   // Only certain opcodes are supported with CreateBinary: opcodes of binary
   // instructions with no auxiliary fields.
   switch (opcode) {
-    case (HloOpcode::kAdd):
-    case (HloOpcode::kDivide):
-    case (HloOpcode::kDot):
-    case (HloOpcode::kEq):
-    case (HloOpcode::kGe):
-    case (HloOpcode::kGt):
-    case (HloOpcode::kLe):
-    case (HloOpcode::kLt):
-    case (HloOpcode::kMaximum):
-    case (HloOpcode::kMinimum):
-    case (HloOpcode::kMultiply):
-    case (HloOpcode::kNe):
-    case (HloOpcode::kPower):
-    case (HloOpcode::kRemainder):
-    case (HloOpcode::kSubtract):
-    case (HloOpcode::kAnd):
-    case (HloOpcode::kOr):
-    case (HloOpcode::kShiftLeft):
-    case (HloOpcode::kShiftRightArithmetic):
-    case (HloOpcode::kShiftRightLogical):
+    case HloOpcode::kAdd:
+    case HloOpcode::kAtan2:
+    case HloOpcode::kDivide:
+    case HloOpcode::kComplex:
+    case HloOpcode::kDot:
+    case HloOpcode::kEq:
+    case HloOpcode::kGe:
+    case HloOpcode::kGt:
+    case HloOpcode::kLe:
+    case HloOpcode::kLt:
+    case HloOpcode::kMaximum:
+    case HloOpcode::kMinimum:
+    case HloOpcode::kMultiply:
+    case HloOpcode::kNe:
+    case HloOpcode::kPower:
+    case HloOpcode::kRemainder:
+    case HloOpcode::kSubtract:
+    case HloOpcode::kAnd:
+    case HloOpcode::kOr:
+    case HloOpcode::kShiftLeft:
+    case HloOpcode::kShiftRightArithmetic:
+    case HloOpcode::kShiftRightLogical:
       break;
     default:
       LOG(FATAL) << "Invalid binary instruction opcode "
@@ -978,11 +982,13 @@ std::unique_ptr<HloInstruction> HloInstruction::CloneWithNewOperands(
     case HloOpcode::kCopy:
     case HloOpcode::kCos:
     case HloOpcode::kExp:
+    case HloOpcode::kImag:
     case HloOpcode::kIsFinite:
     case HloOpcode::kFloor:
     case HloOpcode::kLog:
     case HloOpcode::kNot:
     case HloOpcode::kNegate:
+    case HloOpcode::kReal:
     case HloOpcode::kSign:
     case HloOpcode::kSin:
     case HloOpcode::kSort:
@@ -992,6 +998,8 @@ std::unique_ptr<HloInstruction> HloInstruction::CloneWithNewOperands(
       break;
     // Binary ops.
     case HloOpcode::kAdd:
+    case HloOpcode::kAtan2:
+    case HloOpcode::kComplex:
     case HloOpcode::kDivide:
     case HloOpcode::kMultiply:
     case HloOpcode::kSubtract:
@@ -1403,10 +1411,12 @@ bool HloInstruction::IdenticalSlowPath(
     // The result of these instructions only depend upon their opcode and
     // operands.
     case HloOpcode::kAbs:
+    case HloOpcode::kAtan2:
     case HloOpcode::kRoundNearestAfz:
     case HloOpcode::kAdd:
     case HloOpcode::kCeil:
     case HloOpcode::kClamp:
+    case HloOpcode::kComplex:
     case HloOpcode::kCopy:
     case HloOpcode::kCos:
     case HloOpcode::kCrossReplicaSum:
@@ -1417,6 +1427,7 @@ bool HloInstruction::IdenticalSlowPath(
     case HloOpcode::kFloor:
     case HloOpcode::kGe:
     case HloOpcode::kGt:
+    case HloOpcode::kImag:
     case HloOpcode::kIsFinite:
     case HloOpcode::kLe:
     case HloOpcode::kLog:
@@ -1430,6 +1441,7 @@ bool HloInstruction::IdenticalSlowPath(
     case HloOpcode::kNe:
     case HloOpcode::kNegate:
     case HloOpcode::kPower:
+    case HloOpcode::kReal:
     case HloOpcode::kRemainder:
     case HloOpcode::kSelect:
     case HloOpcode::kShiftLeft:
@@ -2117,6 +2129,8 @@ Status HloInstruction::Visit(DfsHloVisitor* visitor) {
   switch (opcode_) {
     case HloOpcode::kAbs:
       return visitor->HandleAbs(this, operands_[0]);
+    case HloOpcode::kAtan2:
+      return visitor->HandleAtan2(this, operands_[0], operands_[1]);
     case HloOpcode::kRoundNearestAfz:
       return visitor->HandleRound(this);
     case HloOpcode::kBatchNormTraining:
@@ -2140,6 +2154,8 @@ Status HloInstruction::Visit(DfsHloVisitor* visitor) {
     case HloOpcode::kLt:
     case HloOpcode::kNe:
       return visitor->HandleCompare(this, opcode_, operands_[0], operands_[1]);
+    case HloOpcode::kComplex:
+      return visitor->HandleComplex(this, operands_[0], operands_[1]);
     case HloOpcode::kAdd:
       return visitor->HandleAdd(this, operands_[0], operands_[1]);
     case HloOpcode::kDivide:
@@ -2214,6 +2230,10 @@ Status HloInstruction::Visit(DfsHloVisitor* visitor) {
       return visitor->HandleCos(this, operands_[0]);
     case HloOpcode::kSin:
       return visitor->HandleSin(this, operands_[0]);
+    case HloOpcode::kReal:
+      return visitor->HandleReal(this, operands_[0]);
+    case HloOpcode::kImag:
+      return visitor->HandleImag(this, operands_[0]);
     case HloOpcode::kIsFinite:
       return visitor->HandleIsFinite(this, operands_[0]);
     case HloOpcode::kNot:
@@ -2305,7 +2325,7 @@ static Status PostOrderDFS(HloInstruction* root, DfsHloVisitor* visitor,
   //
   // We need to keep track of both the id and the instruction because
   // instructions can get deleted while they are on the stack, so we
-  // can't always use the (potentiall dead) instruction object to grab
+  // can't always use the (potentially dead) instruction object to grab
   // its id.
   DFSStack dfs_stack;
   dfs_stack.emplace_back(root->unique_id(), root);
@@ -2505,6 +2525,7 @@ bool HloInstruction::IsElementwiseBinary() const {
     // Binary elementwise operations. If you update this, please update
     // IsElementwise() accordingly.
     case HloOpcode::kAdd:
+    case HloOpcode::kComplex:
     case HloOpcode::kDivide:
     case HloOpcode::kEq:
     case HloOpcode::kGe:
@@ -2537,6 +2558,7 @@ bool HloInstruction::IsElementwise() const {
 
     // Unary elementwise operations.
     case HloOpcode::kAbs:
+    case HloOpcode::kAtan2:
     case HloOpcode::kRoundNearestAfz:
     case HloOpcode::kCeil:
     case HloOpcode::kConvert:
@@ -2544,10 +2566,12 @@ bool HloInstruction::IsElementwise() const {
     case HloOpcode::kCos:
     case HloOpcode::kExp:
     case HloOpcode::kFloor:
+    case HloOpcode::kImag:
     case HloOpcode::kIsFinite:
     case HloOpcode::kLog:
     case HloOpcode::kNot:
     case HloOpcode::kNegate:
+    case HloOpcode::kReal:
     case HloOpcode::kReducePrecision:
     case HloOpcode::kSign:
     case HloOpcode::kSin:
@@ -2557,6 +2581,7 @@ bool HloInstruction::IsElementwise() const {
     // Binary elementwise operations, the same as in IsElementwiseBinary().
     // If you update this, please update IsElementwiseBinary() accordingly.
     case HloOpcode::kAdd:
+    case HloOpcode::kComplex:
     case HloOpcode::kDivide:
     case HloOpcode::kEq:
     case HloOpcode::kGe:
diff --git a/tensorflow/compiler/xla/service/hlo_opcode.cc b/tensorflow/compiler/xla/service/hlo_opcode.cc
index e9000a8462..2f2263f70d 100644
--- a/tensorflow/compiler/xla/service/hlo_opcode.cc
+++ b/tensorflow/compiler/xla/service/hlo_opcode.cc
@@ -33,6 +33,8 @@ string HloOpcodeString(HloOpcode opcode) {
       return "abs";
     case HloOpcode::kAdd:
       return "add";
+    case HloOpcode::kAtan2:
+      return "atan2";
     case HloOpcode::kBatchNormTraining:
       return "batch-norm-training";
     case HloOpcode::kBatchNormInference:
@@ -47,6 +49,8 @@ string HloOpcodeString(HloOpcode opcode) {
       return "call";
     case HloOpcode::kClamp:
       return "clamp";
+    case HloOpcode::kComplex:
+      return "complex";
     case HloOpcode::kConcatenate:
       return "concatenate";
     case HloOpcode::kConstant:
@@ -87,6 +91,8 @@ string HloOpcodeString(HloOpcode opcode) {
       return "get-tuple-element";
     case HloOpcode::kGt:
       return "greater-than";
+    case HloOpcode::kImag:
+      return "imag";
     case HloOpcode::kIndex:
       return "index";
     case HloOpcode::kInfeed:
@@ -125,6 +131,8 @@ string HloOpcodeString(HloOpcode opcode) {
       return "parameter";
     case HloOpcode::kPower:
       return "power";
+    case HloOpcode::kReal:
+      return "real";
     case HloOpcode::kRecv:
       return "recv";
     case HloOpcode::kReduce:
diff --git a/tensorflow/compiler/xla/service/hlo_opcode.h b/tensorflow/compiler/xla/service/hlo_opcode.h
index c603c57e62..8090e4c82e 100644
--- a/tensorflow/compiler/xla/service/hlo_opcode.h
+++ b/tensorflow/compiler/xla/service/hlo_opcode.h
@@ -31,6 +31,7 @@ namespace xla {
 enum class HloOpcode {
   kAbs,
   kAdd,
+  kAtan2,
   kBatchNormGrad,
   kBatchNormInference,
   kBatchNormTraining,
@@ -39,6 +40,7 @@ enum class HloOpcode {
   kCall,
   kCeil,
   kClamp,
+  kComplex,
   kConcatenate,
   kConstant,
   kConvert,
@@ -58,6 +60,7 @@ enum class HloOpcode {
   kGe,
   kGetTupleElement,
   kGt,
+  kImag,
   kIndex,
   kInfeed,
   kIsFinite,
@@ -77,6 +80,7 @@ enum class HloOpcode {
   kPad,
   kParameter,
   kPower,
+  kReal,
   kRecv,
   kReduce,
   kReducePrecision,
diff --git a/tensorflow/compiler/xla/service/hlo_pass_pipeline.cc b/tensorflow/compiler/xla/service/hlo_pass_pipeline.cc
index ed7b6c71bc..53bd46a641 100644
--- a/tensorflow/compiler/xla/service/hlo_pass_pipeline.cc
+++ b/tensorflow/compiler/xla/service/hlo_pass_pipeline.cc
@@ -59,6 +59,7 @@ StatusOr<bool> HloPassPipeline::Run(HloModule* module) {
     for (auto& invariant_checker : invariant_checkers_) {
       VLOG(1) << "    Invariant checker " << invariant_checker->name();
       StatusOr<bool> changed_status = invariant_checker->Run(module);
+      VLOG(1) << "    Invariant checker done " << invariant_checker->name();
       if (!changed_status.ok()) {
         VLOG(2) << "Module failed invariant check:";
         XLA_VLOG_LINES(2, module->ToString());
diff --git a/tensorflow/compiler/xla/service/hlo_verifier.cc b/tensorflow/compiler/xla/service/hlo_verifier.cc
index 35dff4a957..f3a098057b 100644
--- a/tensorflow/compiler/xla/service/hlo_verifier.cc
+++ b/tensorflow/compiler/xla/service/hlo_verifier.cc
@@ -64,6 +64,10 @@ class ShapeVerifier : public DfsHloVisitor {
   }
 
   Status HandleConvert(HloInstruction* convert) override {
+    if (ShapeUtil::ElementIsComplex(convert->operand(0)->shape())) {
+      TF_RET_CHECK(ShapeUtil::ElementIsComplex(convert->shape()))
+          << "Unsupported complex->real kConvert";
+    }
     return CheckShape(convert, ShapeInference::InferConvertShape(
                                    convert->operand(0)->shape(),
                                    convert->shape().element_type()));
diff --git a/tensorflow/compiler/xla/service/instruction_fusion.cc b/tensorflow/compiler/xla/service/instruction_fusion.cc
index 0271f41697..fae3ca8ad2 100644
--- a/tensorflow/compiler/xla/service/instruction_fusion.cc
+++ b/tensorflow/compiler/xla/service/instruction_fusion.cc
@@ -32,17 +32,16 @@ namespace xla {
     const HloInstruction& instruction) {
   switch (instruction.opcode()) {
     // Cheap instructions.
-    case HloOpcode::kAbs:
     case HloOpcode::kAdd:
     case HloOpcode::kBitcast:
     case HloOpcode::kBroadcast:
     case HloOpcode::kCeil:
     case HloOpcode::kClamp:
+    case HloOpcode::kComplex:
     case HloOpcode::kConcatenate:
     case HloOpcode::kConstant:
     case HloOpcode::kConvert:
     case HloOpcode::kCopy:
-    case HloOpcode::kCos:
     case HloOpcode::kDynamicSlice:
     case HloOpcode::kDynamicUpdateSlice:
     case HloOpcode::kEq:
@@ -50,6 +49,7 @@ namespace xla {
     case HloOpcode::kGe:
     case HloOpcode::kGetTupleElement:
     case HloOpcode::kGt:
+    case HloOpcode::kImag:
     case HloOpcode::kInfeed:
     case HloOpcode::kIsFinite:
     case HloOpcode::kLe:
@@ -64,6 +64,7 @@ namespace xla {
     case HloOpcode::kNegate:
     case HloOpcode::kOutfeed:
     case HloOpcode::kPad:
+    case HloOpcode::kReal:
     case HloOpcode::kReducePrecision:
     case HloOpcode::kReshape:
     case HloOpcode::kReverse:
@@ -72,15 +73,21 @@ namespace xla {
     case HloOpcode::kShiftLeft:
     case HloOpcode::kShiftRightArithmetic:
     case HloOpcode::kShiftRightLogical:
-    case HloOpcode::kSign:
-    case HloOpcode::kSin:
     case HloOpcode::kSlice:
     case HloOpcode::kSubtract:
     case HloOpcode::kTranspose:
     case HloOpcode::kTuple:
       return false;
 
+    // Cheap instructions for reals, but expensive for complex.
+    case HloOpcode::kAbs:
+    case HloOpcode::kCos:
+    case HloOpcode::kSign:
+    case HloOpcode::kSin:
+      return ShapeUtil::ElementIsComplex(instruction.shape());
+
     // Expensive instructions.
+    case HloOpcode::kAtan2:
     case HloOpcode::kBatchNormTraining:
     case HloOpcode::kBatchNormInference:
     case HloOpcode::kBatchNormGrad:
diff --git a/tensorflow/compiler/xla/service/llvm_ir/fused_ir_emitter.cc b/tensorflow/compiler/xla/service/llvm_ir/fused_ir_emitter.cc
index d286c49d68..a2af2580ff 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/fused_ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/llvm_ir/fused_ir_emitter.cc
@@ -75,7 +75,7 @@ Status FusedIrEmitter::DefaultAction(HloInstruction* hlo) {
 Status FusedIrEmitter::HandleConstant(HloInstruction* constant,
                                       const Literal& literal) {
   llvm::Constant* initializer =
-      llvm_ir::ConvertLiteralToIrConstant(literal, ir_builder_);
+      llvm_ir::ConvertLiteralToIrConstant(literal, module_);
   llvm::GlobalVariable* global = new llvm::GlobalVariable(
       *ir_builder_->GetInsertBlock()->getModule(), initializer->getType(),
       /*isConstant=*/true, llvm::GlobalValue::ExternalLinkage, initializer,
@@ -101,7 +101,7 @@ Status FusedIrEmitter::HandleGetTupleElement(HloInstruction* get_tuple_element,
   // Emit code to lookup tuple element pointer, and store it in 'gte_values_'.
   llvm::Value* tuple_element_ptr = llvm_ir::EmitGetTupleElement(
       get_tuple_element->shape(), get_tuple_element->tuple_index(),
-      /*alignment=*/1, it->second, ir_builder_);
+      /*alignment=*/1, it->second, ir_builder_, module_);
   gte_values_.insert(std::make_pair(get_tuple_element, tuple_element_ptr));
   // Emit code to read base tuple element array (if non-tuple shaped).
   if (!ShapeUtil::IsTuple(get_tuple_element->shape())) {
@@ -134,7 +134,7 @@ Status FusedIrEmitter::HandleTuple(
   std::vector<llvm::Type*> operand_elemental_ir_types;
   for (HloInstruction* operand : operands) {
     operand_elemental_ir_types.push_back(llvm_ir::PrimitiveTypeToIrType(
-        operand->shape().element_type(), ir_builder_));
+        operand->shape().element_type(), module_));
   }
   generators_[tuple] =
       [=](const IrArray::Index& index) -> StatusOr<llvm::Value*> {
diff --git a/tensorflow/compiler/xla/service/llvm_ir/fused_ir_emitter.h b/tensorflow/compiler/xla/service/llvm_ir/fused_ir_emitter.h
index a24e104067..a44da51378 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/fused_ir_emitter.h
+++ b/tensorflow/compiler/xla/service/llvm_ir/fused_ir_emitter.h
@@ -42,7 +42,8 @@ class FusedIrEmitter : public DfsHloVisitorWithDefault {
                  ElementalIrEmitter* elemental_emitter)
       : parameter_arrays_(parameter_arrays),
         elemental_emitter_(elemental_emitter),
-        ir_builder_(elemental_emitter->ir_builder()) {}
+        ir_builder_(elemental_emitter->ir_builder()),
+        module_(elemental_emitter->module()) {}
 
   Status DefaultAction(HloInstruction* hlo) override;
 
@@ -85,6 +86,7 @@ class FusedIrEmitter : public DfsHloVisitorWithDefault {
 
   // Borrowed
   llvm::IRBuilder<>* ir_builder_;
+  llvm::Module* module_;
 
   // Map from instruction pointers to functions to generate elements of their
   // outputs
diff --git a/tensorflow/compiler/xla/service/llvm_ir/ir_array.cc b/tensorflow/compiler/xla/service/llvm_ir/ir_array.cc
index 6a00a565c6..e3f98ac13e 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/ir_array.cc
+++ b/tensorflow/compiler/xla/service/llvm_ir/ir_array.cc
@@ -229,9 +229,11 @@ llvm::Value* IrArray::EmitArrayElementAddress(
   }
 
   if (!is_implicit_broadcast && index.LinearValidOnShape(*shape_)) {
+    llvm::Module* module =
+        ir_builder->GetInsertBlock()->getParent()->getParent();
     return ir_builder->CreateInBoundsGEP(
         ir_builder->CreateBitCast(
-            base_ptr_, PrimitiveTypeToIrType(shape_->element_type(), ir_builder)
+            base_ptr_, PrimitiveTypeToIrType(shape_->element_type(), module)
                            ->getPointerTo()),
         {index.linear()}, llvm_ir::AsStringRef(name));
   }
@@ -281,7 +283,8 @@ void IrArray::EmitWriteArrayElement(const Index& index, llvm::Value* value,
 
 IrArray IrArray::CastToShape(const Shape& new_shape,
                              llvm::IRBuilder<>* ir_builder) const {
-  llvm::Type* new_ir_type = llvm_ir::ShapeToIrType(new_shape, ir_builder);
+  llvm::Module* module = ir_builder->GetInsertBlock()->getParent()->getParent();
+  llvm::Type* new_ir_type = llvm_ir::ShapeToIrType(new_shape, module);
   return IrArray(
       ir_builder->CreatePointerCast(base_ptr_, new_ir_type->getPointerTo()),
       new_shape);
diff --git a/tensorflow/compiler/xla/service/llvm_ir/llvm_util.cc b/tensorflow/compiler/xla/service/llvm_ir/llvm_util.cc
index 8e188e7ae8..5dff4b5778 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/llvm_util.cc
+++ b/tensorflow/compiler/xla/service/llvm_ir/llvm_util.cc
@@ -19,6 +19,7 @@ limitations under the License.
 #include <memory>
 #include <vector>
 
+#include "llvm/IR/DerivedTypes.h"
 #include "llvm/IR/MDBuilder.h"
 #include "llvm/IR/Operator.h"
 #include "llvm/Target/TargetOptions.h"
@@ -38,6 +39,19 @@ limitations under the License.
 namespace xla {
 namespace llvm_ir {
 
+namespace {
+
+// Note, this function is only useful in an insertion context; in a global
+// (e.g. constants) context it will CHECK fail.
+llvm::Module* ModuleFromIRBuilder(llvm::IRBuilder<>* ir_builder) {
+  auto block = CHECK_NOTNULL(ir_builder->GetInsertBlock());
+  auto fn = CHECK_NOTNULL(block->getParent());
+  auto module = CHECK_NOTNULL(fn->getParent());
+  return module;
+}
+
+}  // namespace
+
 string AsString(const std::string& str) {
   return string(str.data(), str.length());
 }
@@ -63,7 +77,7 @@ llvm::Value* EmitCallToIntrinsic(
   for (auto type : overloaded_types) {
     types.push_back(type);
   }
-  llvm::Module* module = ir_builder->GetInsertBlock()->getParent()->getParent();
+  llvm::Module* module = ModuleFromIRBuilder(ir_builder);
   llvm::Function* intrinsic =
       llvm::Intrinsic::getDeclaration(module, intrinsic_id, types);
   std::vector<llvm::Value*> operands_vec;
@@ -119,38 +133,53 @@ llvm::Value* EmitBufferIndexingGEP(llvm::Value* array, int64 index,
 }
 
 llvm::Type* PrimitiveTypeToIrType(PrimitiveType element_type,
-                                  llvm::IRBuilder<>* ir_builder) {
+                                  llvm::Module* module) {
   switch (element_type) {
     case PRED:
     case S8:
     case U8:
-      return ir_builder->getInt8Ty();
+      return llvm::Type::getInt8Ty(module->getContext());
     case S16:
     case U16:
-      return ir_builder->getInt16Ty();
+      return llvm::Type::getInt16Ty(module->getContext());
     case S32:
     case U32:
-      return ir_builder->getInt32Ty();
+      return llvm::Type::getInt32Ty(module->getContext());
     case S64:
     case U64:
-      return ir_builder->getInt64Ty();
+      return llvm::Type::getInt64Ty(module->getContext());
     case F32:
-      return ir_builder->getFloatTy();
+      return llvm::Type::getFloatTy(module->getContext());
     case F64:
-      return ir_builder->getDoubleTy();
+      return llvm::Type::getDoubleTy(module->getContext());
+    case C64: {
+      auto cplx_t = module->getTypeByName("complex64");
+      if (cplx_t == nullptr) {
+        // C++ standard dictates the memory layout of std::complex is contiguous
+        // real followed by imaginary. C++11 section 26.4 [complex.numbers]:
+        // If z is an lvalue expression of type cv std::complex<T> then the
+        // expression reinterpret_cast<cv T(&)[2]>(z) shall be well-formed,
+        // reinterpret_cast<cv T(&)[2]>(z)[0] shall designate the real part of
+        // z, and reinterpret_cast<cv T(&)[2]>(z)[1] shall designate the
+        // imaginary part of z.
+        return llvm::StructType::create(
+            "complex64", llvm::Type::getFloatTy(module->getContext()),
+            llvm::Type::getFloatTy(module->getContext()));
+      }
+      return cplx_t;
+    }
     // A Tuple contains an array of pointers. Use i8*.
     case TUPLE:
     // An Opaque is like a void*, use i8*.
     case OPAQUE:
-      return ir_builder->getInt8PtrTy();
+      return llvm::Type::getInt8PtrTy(module->getContext());
     default:
       LOG(FATAL) << "unsupported type " << element_type;
   }
 }
 
-llvm::Type* ShapeToIrType(const Shape& shape, llvm::IRBuilder<>* ir_builder) {
-  llvm::Type* result_type =
-      PrimitiveTypeToIrType(shape.element_type(), ir_builder);
+llvm::Type* ShapeToIrType(const Shape& shape, llvm::Module* module) {
+  llvm::Type* result_type = PrimitiveTypeToIrType(shape.element_type(), module);
   if (ShapeUtil::IsTuple(shape)) {
     // A tuple buffer is an array of pointers.
     result_type = llvm::ArrayType::get(result_type, shape.tuple_shapes_size());
@@ -197,10 +226,10 @@ namespace {
 // value down to zero).
 llvm::Constant* LiteralToConstant(const Literal& literal, int64 dimension_index,
                                   std::vector<int64>* multi_index,
-                                  llvm::IRBuilder<>* ir_builder) {
+                                  llvm::Module* module) {
   const Shape& shape = literal.shape();
   llvm::Type* ir_element_type =
-      llvm_ir::PrimitiveTypeToIrType(shape.element_type(), ir_builder);
+      llvm_ir::PrimitiveTypeToIrType(shape.element_type(), module);
   if (dimension_index == -1) {
     // Base case of the recursion. Index into the data field of the protobuf
     // with the multi index.
@@ -238,6 +267,16 @@ llvm::Constant* LiteralToConstant(const Literal& literal, int64 dimension_index,
         value = llvm::ConstantFP::get(ir_element_type,
                                       literal.Get<double>(*multi_index));
         break;
+      case C64: {
+        complex64 x = literal.Get<complex64>(*multi_index);
+        value = llvm::ConstantStruct::get(
+            static_cast<llvm::StructType*>(ir_element_type),
+            llvm::ConstantFP::get(llvm_ir::PrimitiveTypeToIrType(F32, module),
+                                  x.real()),
+            llvm::ConstantFP::get(llvm_ir::PrimitiveTypeToIrType(F32, module),
+                                  x.imag()));
+        break;
+      }
       default:
         LOG(FATAL) << "unsupported type " << shape.element_type();
     }
@@ -256,8 +295,8 @@ llvm::Constant* LiteralToConstant(const Literal& literal, int64 dimension_index,
   std::vector<llvm::Constant*> elements;
   for (int64 i = 0; i < shape.dimensions(dimension); ++i) {
     (*multi_index)[dimension] = i;
-    elements.push_back(LiteralToConstant(literal, dimension_index - 1,
-                                         multi_index, ir_builder));
+    elements.push_back(
+        LiteralToConstant(literal, dimension_index - 1, multi_index, module));
   }
 
   llvm::Type* element_type;
@@ -279,11 +318,11 @@ llvm::Constant* LiteralToConstant(const Literal& literal, int64 dimension_index,
 }  // namespace
 
 llvm::Constant* ConvertLiteralToIrConstant(const Literal& literal,
-                                           llvm::IRBuilder<>* ir_builder) {
+                                           llvm::Module* module) {
   std::vector<int64> multi_index(ShapeUtil::Rank(literal.shape()), 0);
   llvm::Constant* value = LiteralToConstant(
       literal, /*dimension_index=*/ShapeUtil::Rank(literal.shape()) - 1,
-      &multi_index, ir_builder);
+      &multi_index, module);
   return value;
 }
 
@@ -380,7 +419,8 @@ llvm::Value* EmitComparison(llvm::CmpInst::Predicate predicate,
   // comparison_result is i1, but the NVPTX codegen incorrectly lowers i1
   // arrays. So we extend it to i8 so that it's addressable.
   return ir_builder->CreateZExt(
-      comparison_result, llvm_ir::PrimitiveTypeToIrType(PRED, ir_builder));
+      comparison_result,
+      llvm_ir::PrimitiveTypeToIrType(PRED, ModuleFromIRBuilder(ir_builder)));
 }
 
 // Internal helper that is called from emitted code to log an int64 value with a
diff --git a/tensorflow/compiler/xla/service/llvm_ir/llvm_util.h b/tensorflow/compiler/xla/service/llvm_ir/llvm_util.h
index 7a7d14da1e..304192b58e 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/llvm_util.h
+++ b/tensorflow/compiler/xla/service/llvm_ir/llvm_util.h
@@ -127,11 +127,11 @@ llvm::Value* EmitBufferIndexingGEP(llvm::Value* array, int64 index,
 
 // Returns the LLVM type which represents the given XLA primitive type.
 llvm::Type* PrimitiveTypeToIrType(PrimitiveType element_type,
-                                  llvm::IRBuilder<>* ir_builder);
+                                  llvm::Module* module);
 
 // Returns the LLVM type which represents the given XLA shape. For example,
 // if "shape" is [5 x [10 x f32]], the function returns [5 x [10 x float]].
-llvm::Type* ShapeToIrType(const Shape& shape, llvm::IRBuilder<>* ir_builder);
+llvm::Type* ShapeToIrType(const Shape& shape, llvm::Module* module);
 
 // Returns a value that represents a pointer to a global string constant that
 // encodes the shape as a serialized protobuf.
@@ -149,7 +149,7 @@ StatusOr<Shape> DecodeSelfDescribingShapeConstant(const void* shape_ptr,
 // Converts a given literal to an IR Constant. Literals have known constant
 // values at IR emission time.
 llvm::Constant* ConvertLiteralToIrConstant(const Literal& literal,
-                                           llvm::IRBuilder<>* ir_builder);
+                                           llvm::Module* module);
 
 // Inserts an allocate of the requested type at the entry point of the
 // function that the builder is currently building. The insert point
diff --git a/tensorflow/compiler/xla/service/llvm_ir/tuple_ops.cc b/tensorflow/compiler/xla/service/llvm_ir/tuple_ops.cc
index 6051cbfc6f..3a21eda357 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/tuple_ops.cc
+++ b/tensorflow/compiler/xla/service/llvm_ir/tuple_ops.cc
@@ -31,14 +31,15 @@ namespace xla {
 namespace llvm_ir {
 
 void EmitTupleSelect(IrArray select, IrArray pred, llvm::Value* on_true,
-                     llvm::Value* on_false, llvm::IRBuilder<>* ir_builder) {
+                     llvm::Value* on_false, llvm::IRBuilder<>* ir_builder,
+                     llvm::Module* module) {
   CHECK(ShapeUtil::IsScalar(pred.GetShape()));
 
   llvm::LoadInst* pred_value =
       ir_builder->CreateLoad(pred.GetBasePointer(), "load_predicate_value");
   llvm::Value* pred_cond = ir_builder->CreateICmpNE(
       pred_value,
-      llvm::ConstantInt::get(PrimitiveTypeToIrType(PRED, ir_builder), 0),
+      llvm::ConstantInt::get(PrimitiveTypeToIrType(PRED, module), 0),
       "boolean_predicate");
 
   VLOG(2) << "HandleSelect for tuple:";
@@ -71,11 +72,11 @@ void EmitTupleSelect(IrArray select, IrArray pred, llvm::Value* on_true,
 
 void EmitTuple(IrArray tuple,
                tensorflow::gtl::ArraySlice<llvm::Value*> operands,
-               llvm::IRBuilder<>* ir_builder) {
+               llvm::IRBuilder<>* ir_builder, llvm::Module* module) {
   for (size_t i = 0; i < operands.size(); ++i) {
     auto* store = ir_builder->CreateStore(
         ir_builder->CreatePointerCast(operands[i],
-                                      PrimitiveTypeToIrType(TUPLE, ir_builder)),
+                                      PrimitiveTypeToIrType(TUPLE, module)),
         ir_builder->CreateInBoundsGEP(
             tuple.GetBasePointer(),
             {ir_builder->getInt64(0), ir_builder->getInt64(i)}));
@@ -85,7 +86,8 @@ void EmitTuple(IrArray tuple,
 
 llvm::Value* EmitGetTupleElement(const Shape& target_shape, int64 index,
                                  int alignment, llvm::Value* operand,
-                                 llvm::IRBuilder<>* ir_builder) {
+                                 llvm::IRBuilder<>* ir_builder,
+                                 llvm::Module* module) {
   llvm::Value* element_ptr = ir_builder->CreateInBoundsGEP(
       operand, {ir_builder->getInt64(0), ir_builder->getInt64(index)});
   llvm::LoadInst* src_buffer = ir_builder->CreateLoad(element_ptr);
@@ -98,7 +100,7 @@ llvm::Value* EmitGetTupleElement(const Shape& target_shape, int64 index,
   }
   SetAlignmentMetadataForLoad(src_buffer, alignment);
 
-  llvm::Type* element_type = ShapeToIrType(target_shape, ir_builder);
+  llvm::Type* element_type = ShapeToIrType(target_shape, module);
   llvm::Value* ret_val =
       ir_builder->CreateBitCast(src_buffer, element_type->getPointerTo());
   return ret_val;
diff --git a/tensorflow/compiler/xla/service/llvm_ir/tuple_ops.h b/tensorflow/compiler/xla/service/llvm_ir/tuple_ops.h
index a75cdc8158..dbf9a14006 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/tuple_ops.h
+++ b/tensorflow/compiler/xla/service/llvm_ir/tuple_ops.h
@@ -60,13 +60,14 @@ namespace llvm_ir {
 // tuple_on_true or tuple_on_false:
 //   output[i] = pred ? tuple_on_true[i] : tuple_on_false[i]
 void EmitTupleSelect(IrArray select, IrArray pred, llvm::Value* on_true,
-                     llvm::Value* on_false, llvm::IRBuilder<>* ir_builder);
+                     llvm::Value* on_false, llvm::IRBuilder<>* ir_builder,
+                     llvm::Module* module);
 
 // A tuple is an array of pointers, one for each operand. Each pointer points to
 // the output buffer of its corresponding operand.
 void EmitTuple(IrArray tuple,
                tensorflow::gtl::ArraySlice<llvm::Value*> operands,
-               llvm::IRBuilder<>* ir_builder);
+               llvm::IRBuilder<>* ir_builder, llvm::Module* module);
 
 // A tuple is an array of pointers, one for each operand. Each pointer points to
 // the output buffer of its corresponding operand. A GetTupleElement instruction
@@ -74,7 +75,8 @@ void EmitTuple(IrArray tuple,
 // Returns an llvm value representing a pointer to the tuple element buffer.
 llvm::Value* EmitGetTupleElement(const Shape& target_shape, int64 index,
                                  int alignment, llvm::Value* operand,
-                                 llvm::IRBuilder<>* ir_builder);
+                                 llvm::IRBuilder<>* ir_builder,
+                                 llvm::Module* module);
 }  // namespace llvm_ir
 }  // namespace xla
 
diff --git a/tensorflow/compiler/xla/service/shape_inference.cc b/tensorflow/compiler/xla/service/shape_inference.cc
index e41b7607c5..0458932a73 100644
--- a/tensorflow/compiler/xla/service/shape_inference.cc
+++ b/tensorflow/compiler/xla/service/shape_inference.cc
@@ -53,6 +53,8 @@ UnaryOperation OpcodeToUnaryOperation(HloOpcode opcode) {
       return UNOP_EXP;
     case HloOpcode::kFloor:
       return UNOP_FLOOR;
+    case HloOpcode::kImag:
+      return UNOP_IMAG;
     case HloOpcode::kIsFinite:
       return UNOP_IS_FINITE;
     case HloOpcode::kLog:
@@ -61,6 +63,8 @@ UnaryOperation OpcodeToUnaryOperation(HloOpcode opcode) {
       return UNOP_NOT;
     case HloOpcode::kNegate:
       return UNOP_NEGATE;
+    case HloOpcode::kReal:
+      return UNOP_REAL;
     case HloOpcode::kRoundNearestAfz:
       return UNOP_ROUND_NEAREST_AFZ;
     case HloOpcode::kSign:
@@ -81,6 +85,10 @@ UnaryOperation OpcodeToUnaryOperation(HloOpcode opcode) {
 // opcode.
 BinaryOperation OpcodeToBinaryOperation(HloOpcode opcode) {
   switch (opcode) {
+    case HloOpcode::kAtan2:
+      return BINOP_ATAN2;
+    case HloOpcode::kComplex:
+      return BINOP_COMPLEX;
     case HloOpcode::kDot:
       return BINOP_DOT;
     case HloOpcode::kMultiply:
@@ -307,19 +315,41 @@ StatusOr<Shape> InferWindowOutputShape(const Shape& base_shape,
   switch (operation) {
     case UNOP_FLOOR:
     case UNOP_CEIL:
+      if (!ShapeUtil::ElementIsFloating(arg)) {
+        return InvalidArgument(
+            "expected element type in shape to be floating for floor/ceil "
+            "operation; got %s",
+            PrimitiveType_Name(arg.element_type()).c_str());
+      }
+      return arg;
     case UNOP_COS:
     case UNOP_SIN:
     case UNOP_EXP:
     case UNOP_LOG:
     case UNOP_TANH:
-      if (!ShapeUtil::ElementIsFloating(arg)) {
+      if (!ShapeUtil::ElementIsFloating(arg) &&
+          !ShapeUtil::ElementIsComplex(arg)) {
         return InvalidArgument(
-            "expected element type in shape to be floating for exp/log/tanh "
-            "operation; got %s",
+            "expected element type in shape to be floating or complex for "
+            "sin/cos/exp/log/tanh operation; got %s",
             PrimitiveType_Name(arg.element_type()).c_str());
       }
       return arg;
+    case UNOP_REAL:
+    case UNOP_IMAG:
+      if (!ShapeUtil::ElementIsComplex(arg)) {
+        return InvalidArgument(
+            "expected element type in shape to be complex for real/imag "
+            "operation; got %s",
+            PrimitiveType_Name(arg.element_type()).c_str());
+      }
+      return ShapeUtil::ChangeElementType(arg, F32);
     case UNOP_ABS:
+      if (ShapeUtil::ElementIsComplex(arg)) {
+        return ShapeUtil::ChangeElementType(
+            arg, primitive_util::ComplexComponentType(arg.element_type()));
+      }
+      return arg;
     case UNOP_NEGATE:
     case UNOP_ROUND_NEAREST_AFZ:
     case UNOP_SIGN:
@@ -751,6 +781,7 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(
     case BINOP_MIN:
     case BINOP_SUB:
     case BINOP_ADD:
+    case BINOP_ATAN2:
     case BINOP_POW:
     case BINOP_DIV:
     case BINOP_REM:
@@ -761,6 +792,22 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(
       return InferElementwiseBinaryOpShape(operation, lhs, rhs,
                                            broadcast_dimensions);
 
+    case BINOP_COMPLEX: {
+      if (!ShapeUtil::ElementIsFloating(lhs)) {
+        return InvalidArgument(
+            "expected element type in shape to be floating for complex compose "
+            "operation; got %s",
+            PrimitiveType_Name(lhs.element_type()).c_str());
+      }
+      TF_ASSIGN_OR_RETURN(const Shape& shape,
+                          InferElementwiseBinaryOpShape(operation, lhs, rhs,
+                                                        broadcast_dimensions));
+      if (lhs.element_type() == F32) {
+        return ShapeUtil::ChangeElementType(shape, C64);
+      } else {
+        return Unimplemented("complex component type not supported");
+      }
+    }
     case BINOP_AND:
     case BINOP_OR:
       if (lhs.element_type() != PRED &&
diff --git a/tensorflow/compiler/xla/service/shape_inference_test.cc b/tensorflow/compiler/xla/service/shape_inference_test.cc
index 8df4a73229..d12f7bd145 100644
--- a/tensorflow/compiler/xla/service/shape_inference_test.cc
+++ b/tensorflow/compiler/xla/service/shape_inference_test.cc
@@ -35,6 +35,7 @@ class ShapeInferenceTest : public ::testing::Test {
   // Some handy scalar shapes.
   const Shape s32_ = ShapeUtil::MakeShape(S32, {});
   const Shape f32_ = ShapeUtil::MakeShape(F32, {});
+  const Shape f64_ = ShapeUtil::MakeShape(F64, {});
   const Shape pred_ = ShapeUtil::MakeShape(PRED, {});
 
   // Some handy vector and matrix shapes of F32 type.
@@ -251,6 +252,44 @@ TEST_F(ShapeInferenceTest, ClampBadShapes) {
                    .ok());
 }
 
+TEST_F(ShapeInferenceTest, Complex) {
+  auto complex_shape = [&](const Shape& lhs, const Shape& rhs,
+                           const tensorflow::gtl::ArraySlice<int64>& bcast) {
+    return ShapeInference::InferBinaryOpShape(BinaryOperation::BINOP_COMPLEX,
+                                              lhs, rhs, bcast);
+  };
+  // Inputs must be FP.
+  ASSERT_FALSE(complex_shape(s32_, s32_, {}).ok());
+  ASSERT_FALSE(complex_shape(pred_, pred_, {}).ok());
+  // Component types must match.
+  ASSERT_FALSE(complex_shape(f32_, f64_, {}).ok());
+  // Only F32->C64 supported.
+  ASSERT_FALSE(complex_shape(f64_, f64_, {}).ok());
+  // Validate correct uses.
+  Shape c64_32 = ShapeUtil::MakeShape(C64, {32});
+  TF_ASSERT_OK_AND_ASSIGN(Shape result, complex_shape(f32_, f32_, {}));
+  ASSERT_TRUE(ShapeUtil::Equal(result, ShapeUtil::MakeShape(C64, {})));
+  TF_ASSERT_OK_AND_ASSIGN(result, complex_shape(vector_32_, f32_, {}));
+  ASSERT_TRUE(ShapeUtil::Equal(result, c64_32));
+  TF_ASSERT_OK_AND_ASSIGN(result, complex_shape(f32_, vector_32_, {}));
+  ASSERT_TRUE(ShapeUtil::Equal(result, c64_32));
+  TF_ASSERT_OK_AND_ASSIGN(result, complex_shape(vector_32_, f32_, {}));
+  ASSERT_TRUE(ShapeUtil::Equal(result, c64_32));
+
+  Shape c64_32_64 = ShapeUtil::MakeShape(C64, {32, 64});
+  TF_ASSERT_OK_AND_ASSIGN(result,
+                          complex_shape(vector_64_, matrix_32_64_, {1}));
+  ASSERT_TRUE(ShapeUtil::Equal(result, c64_32_64));
+  TF_ASSERT_OK_AND_ASSIGN(result,
+                          complex_shape(matrix_32_64_, vector_64_, {1}));
+  ASSERT_TRUE(ShapeUtil::Equal(result, c64_32_64));
+  TF_ASSERT_OK_AND_ASSIGN(result,
+                          complex_shape(matrix_32_64_, matrix_32_64_, {}));
+  ASSERT_TRUE(ShapeUtil::Equal(result, c64_32_64));
+  TF_ASSERT_OK_AND_ASSIGN(result, complex_shape(matrix_32_64_, f32_, {}));
+  ASSERT_TRUE(ShapeUtil::Equal(result, c64_32_64));
+}
+
 TEST_F(ShapeInferenceTest, VariadicOpTuplify) {
   StatusOr<Shape> result = ShapeInference::InferVariadicOpShape(
       VariadicOperation::VAROP_TUPLE, {&s32_, &f32_});
diff --git a/tensorflow/compiler/xla/service/user_computation.cc b/tensorflow/compiler/xla/service/user_computation.cc
index d818830f98..adf7972e0d 100644
--- a/tensorflow/compiler/xla/service/user_computation.cc
+++ b/tensorflow/compiler/xla/service/user_computation.cc
@@ -55,6 +55,8 @@ HloOpcode UnaryOperationToHloOpcode(UnaryOperation unop) {
       return HloOpcode::kExp;
     case UNOP_FLOOR:
       return HloOpcode::kFloor;
+    case UNOP_IMAG:
+      return HloOpcode::kImag;
     case UNOP_IS_FINITE:
       return HloOpcode::kIsFinite;
     case UNOP_LOG:
@@ -63,6 +65,8 @@ HloOpcode UnaryOperationToHloOpcode(UnaryOperation unop) {
       return HloOpcode::kNot;
     case UNOP_NEGATE:
       return HloOpcode::kNegate;
+    case UNOP_REAL:
+      return HloOpcode::kReal;
     case UNOP_ROUND_NEAREST_AFZ:
       return HloOpcode::kRoundNearestAfz;
     case UNOP_SIGN:
@@ -80,6 +84,10 @@ HloOpcode UnaryOperationToHloOpcode(UnaryOperation unop) {
 
 HloOpcode BinaryOperationToHloOpcode(BinaryOperation binop) {
   switch (binop) {
+    case BINOP_ATAN2:
+      return HloOpcode::kAtan2;
+    case BINOP_COMPLEX:
+      return HloOpcode::kComplex;
     case BINOP_DOT:
       return HloOpcode::kDot;
     case BINOP_MUL:
diff --git a/tensorflow/compiler/xla/shape_util.cc b/tensorflow/compiler/xla/shape_util.cc
index fa4f71414d..b5eb81dfc6 100644
--- a/tensorflow/compiler/xla/shape_util.cc
+++ b/tensorflow/compiler/xla/shape_util.cc
@@ -272,6 +272,7 @@ StatusOr<Shape> MakeShapeWithLayoutInternal(
     case U16:
     case U32:
     case U64:
+    case C64:
     case TUPLE:
     case OPAQUE:
       return false;
diff --git a/tensorflow/compiler/xla/tests/client_library_test_base.h b/tensorflow/compiler/xla/tests/client_library_test_base.h
index 7fe1445b94..7cfc276ec1 100644
--- a/tensorflow/compiler/xla/tests/client_library_test_base.h
+++ b/tensorflow/compiler/xla/tests/client_library_test_base.h
@@ -361,8 +361,9 @@ void ClientLibraryTestBase::ComputeAndCompareR2(
     ComputationBuilder* builder, const Array2D<NativeT>& expected,
     tensorflow::gtl::ArraySlice<GlobalData*> arguments, ErrorSpec error) {
   static_assert(std::is_same<NativeT, float>::value ||
-                    std::is_same<NativeT, double>::value,
-                "Floating point type required when specifying an ErrorSpec");
+                    std::is_same<NativeT, double>::value ||
+                    std::is_same<NativeT, complex64>::value,
+                "Float or complex type required when specifying an ErrorSpec");
   std::unique_ptr<Literal> expected_literal =
       Literal::CreateR2FromArray2D<NativeT>(expected);
   ClientLibraryTestBase::ComputeAndCompareLiteral(builder, *expected_literal,
@@ -384,8 +385,9 @@ void ClientLibraryTestBase::ComputeAndCompareR3(
     ComputationBuilder* builder, const Array3D<NativeT>& expected,
     tensorflow::gtl::ArraySlice<GlobalData*> arguments, ErrorSpec error) {
   static_assert(std::is_same<NativeT, float>::value ||
-                    std::is_same<NativeT, double>::value,
-                "Floating point type required when specifying an ErrorSpec");
+                    std::is_same<NativeT, double>::value ||
+                    std::is_same<NativeT, complex64>::value,
+                "Float or complex type required when specifying an ErrorSpec");
   std::unique_ptr<Literal> expected_literal =
       Literal::CreateR3FromArray3D<NativeT>(expected);
   ClientLibraryTestBase::ComputeAndCompareLiteral(builder, *expected_literal,
@@ -407,8 +409,9 @@ void ClientLibraryTestBase::ComputeAndCompareR4(
     ComputationBuilder* builder, const Array4D<NativeT>& expected,
     tensorflow::gtl::ArraySlice<GlobalData*> arguments, ErrorSpec error) {
   static_assert(std::is_same<NativeT, float>::value ||
-                    std::is_same<NativeT, double>::value,
-                "Floating point type required when specifying an ErrorSpec");
+                    std::is_same<NativeT, double>::value ||
+                    std::is_same<NativeT, complex64>::value,
+                "Float or complex type required when specifying an ErrorSpec");
   std::unique_ptr<Literal> expected_literal =
       Literal::CreateR4FromArray4D<NativeT>(expected);
   ClientLibraryTestBase::ComputeAndCompareLiteral(builder, *expected_literal,
diff --git a/tensorflow/compiler/xla/tests/dot_operation_test.cc b/tensorflow/compiler/xla/tests/dot_operation_test.cc
index 224aa57899..cf089d748d 100644
--- a/tensorflow/compiler/xla/tests/dot_operation_test.cc
+++ b/tensorflow/compiler/xla/tests/dot_operation_test.cc
@@ -347,7 +347,7 @@ XLA_TEST_F(DotOperationTest, NonsquareMatrixDotF32MajorToMinorTF) {
   TestNonsquareMatrixDot<float>(kLhsRowMajor, kRhsRowMajor);
 }
 
-TEST_F(DotOperationTest, NonsquareMatrixDotF32MajorToMinorTT) {
+XLA_TEST_F(DotOperationTest, NonsquareMatrixDotF32MajorToMinorTT) {
   constexpr bool kLhsRowMajor = true;
   constexpr bool kRhsRowMajor = true;
   TestNonsquareMatrixDot<float>(kLhsRowMajor, kRhsRowMajor);
@@ -357,7 +357,11 @@ XLA_TEST_F(DotOperationTest, NonsquareMatrixDotF64) {
   TestNonsquareMatrixDot<double>();
 }
 
-TEST_F(DotOperationTest, ConcurrentMatMul) {
+XLA_TEST_F(DotOperationTest, NonsquareMatrixDotC64) {
+  TestNonsquareMatrixDot<complex64>();
+}
+
+XLA_TEST_F(DotOperationTest, ConcurrentMatMul) {
   ComputationBuilder builder(client_, TestName());
   auto matrix1 = builder.ConstantR2<float>({{1.0, 2.0}, {3.0, 4.0}});
   auto matrix2 = builder.ConstantR2<float>({{5.0, 6.0}, {7.0, 8.0}});
diff --git a/tensorflow/compiler/xla/tests/unary_op_test.cc b/tensorflow/compiler/xla/tests/unary_op_test.cc
index efae13a43a..fa4192e928 100644
--- a/tensorflow/compiler/xla/tests/unary_op_test.cc
+++ b/tensorflow/compiler/xla/tests/unary_op_test.cc
@@ -41,7 +41,11 @@ class UnaryOpTest : public ClientLibraryTestBase {
     auto arg = builder.ConstantR1<T>({});
     auto abs = builder.Abs(arg);
 
-    ComputeAndCompareR1<T>(&builder, {}, {});
+    if (primitive_util::NativeToPrimitiveType<T>() == C64) {
+      ComputeAndCompareR1<float>(&builder, {}, {});
+    } else {
+      ComputeAndCompareR1<T>(&builder, {}, {});
+    }
   }
 
   template <typename T>
@@ -80,14 +84,58 @@ int UnaryOpTest::inf<int>() {
   return 2147483647;
 }
 
+template <>
+void UnaryOpTest::AbsTestHelper<complex64>() {
+  ComputationBuilder builder(client_, TestName());
+  auto arg = builder.ConstantR1<complex64>({{-2, 0},
+                                            {0, 25},
+                                            {0, 0},
+                                            {-0.3f, 0.4f},
+                                            {0, inf<float>()},
+                                            {-inf<float>(), 0}});
+  auto abs = builder.Abs(arg);
+
+  std::unique_ptr<Literal> expected =
+      Literal::CreateR1<float>({2, 25, 0, 0.5, inf<float>(), inf<float>()});
+  ComputeAndCompareLiteral(&builder, *expected, {}, ErrorSpec(1e-6f));
+}
+
+template <>
+void UnaryOpTest::SignTestHelper<complex64>() {
+  ComputationBuilder builder(client_, TestName());
+  auto arg = builder.ConstantR1<complex64>(
+      {{-2, 0}, {0, 25}, {0, 0}, {static_cast<float>(-0.0), 0}, {-1, 1}});
+  auto sign = builder.Sign(arg);
+
+  std::unique_ptr<Literal> expected = Literal::CreateR1<complex64>(
+      {{-1, 0}, {0, 1}, {0, 0}, {0, 0}, {-std::sqrt(0.5f), std::sqrt(0.5f)}});
+  ComputeAndCompareLiteral(&builder, *expected, {}, ErrorSpec(1e-6f));
+}
+
+template <>
+void UnaryOpTest::SignAbsTestHelper<complex64>() {
+  ComputationBuilder builder(client_, TestName());
+  auto arg =
+      builder.ConstantR1<complex64>({{-2, 0}, {0, 25}, {0, 0}, {-0.4, 0.3}});
+  auto sign = builder.Sign(arg);
+  auto abs = builder.Abs(arg);
+  builder.Sub(builder.Mul(sign, builder.ConvertElementType(abs, C64)), arg);
+
+  std::unique_ptr<Literal> expected =
+      Literal::CreateR1<complex64>({0, 0, 0, 0});
+  ComputeAndCompareLiteral(&builder, *expected, {}, ErrorSpec(1e-6f));
+}
+
 XLA_TEST_F(UnaryOpTest, AbsTestR1Size0) {
   AbsSize0TestHelper<int>();
   AbsSize0TestHelper<float>();
+  AbsSize0TestHelper<complex64>();
 }
 
 XLA_TEST_F(UnaryOpTest, AbsTestR1) {
   AbsTestHelper<int>();
   AbsTestHelper<float>();
+  AbsTestHelper<complex64>();
 }
 
 XLA_TEST_F(UnaryOpTest, AbsTestR0) {
@@ -98,34 +146,44 @@ XLA_TEST_F(UnaryOpTest, AbsTestR0) {
   auto absf = builder.Abs(argf);
   auto argf0 = builder.ConstantR0<float>(-0.0f);
   auto absf0 = builder.Abs(argf0);
-  builder.Add(absf0, builder.Add(absf, builder.ConvertElementType(
-                                           absi, PrimitiveType::F32)));
+  auto argc = builder.ConstantR0<complex64>({-0.3f, 0.4f});
+  auto absc = builder.Abs(argc);
+  builder.Add(builder.Add(absc, absf0),
+              builder.Add(absf, builder.ConvertElementType(absi, F32)));
 
-  ComputeAndCompareR0<float>(&builder, 8.0f, {});
+  ComputeAndCompareR0<float>(&builder, 8.5f, {});
 }
 
 XLA_TEST_F(UnaryOpTest, SignTestR0) {
   ComputationBuilder builder(client_, TestName());
   auto argi = builder.ConstantR0<int>(-5);
-  auto absi = builder.Sign(argi);
+  auto sgni = builder.Sign(argi);  // -1
   auto argf = builder.ConstantR0<float>(-4.0f);
-  auto absf = builder.Sign(argf);
+  auto sgnf = builder.Sign(argf);  // -1
   auto argf0 = builder.ConstantR0<float>(-0.0f);
-  auto absf0 = builder.Sign(argf0);
-  builder.Add(absf0, builder.Add(absf, builder.ConvertElementType(
-                                           absi, PrimitiveType::F32)));
-
-  ComputeAndCompareR0<float>(&builder, -2.0f, {});
+  auto sgnf0 = builder.Sign(argf0);  // 0
+  auto argc = builder.ConstantR0<complex64>({-.3, .4});
+  auto sgnc = builder.Sign(argc);  // (-.6, .8)
+  builder.Add(sgnc, builder.ConvertElementType(
+                        builder.Add(builder.Add(sgnf0, sgnf),
+                                    builder.ConvertElementType(sgni, F32)),
+                        C64));
+
+  std::unique_ptr<Literal> expected =
+      Literal::CreateR0<complex64>({-2.6f, 0.8f});
+  ComputeAndCompareLiteral(&builder, *expected, {}, ErrorSpec(1e-6f));
 }
 
 XLA_TEST_F(UnaryOpTest, SignTestR1) {
   SignTestHelper<int>();
   SignTestHelper<float>();
+  SignTestHelper<complex64>();
 }
 
 XLA_TEST_F(UnaryOpTest, SignAbsTestR1) {
   SignAbsTestHelper<int>();
   SignAbsTestHelper<float>();
+  SignAbsTestHelper<complex64>();
 }
 
 XLA_TEST_F(UnaryOpTest, UnsignedAbsTestR1) {
diff --git a/tensorflow/compiler/xla/tools/parser/hlo_parser.cc b/tensorflow/compiler/xla/tools/parser/hlo_parser.cc
index f4af03cc2f..d91404d73a 100644
--- a/tensorflow/compiler/xla/tools/parser/hlo_parser.cc
+++ b/tensorflow/compiler/xla/tools/parser/hlo_parser.cc
@@ -235,11 +235,13 @@ bool HloParser::ParseInstruction(HloComputation::Builder* builder,
     case HloOpcode::kCopy:
     case HloOpcode::kCos:
     case HloOpcode::kExp:
+    case HloOpcode::kImag:
     case HloOpcode::kIsFinite:
     case HloOpcode::kFloor:
     case HloOpcode::kLog:
     case HloOpcode::kNot:
     case HloOpcode::kNegate:
+    case HloOpcode::kReal:
     case HloOpcode::kSign:
     case HloOpcode::kSin:
     case HloOpcode::kSort:
@@ -256,6 +258,8 @@ bool HloParser::ParseInstruction(HloComputation::Builder* builder,
     case HloOpcode::kDivide:
     case HloOpcode::kMultiply:
     case HloOpcode::kSubtract:
+    case HloOpcode::kAtan2:
+    case HloOpcode::kComplex:
     case HloOpcode::kEq:
     case HloOpcode::kGe:
     case HloOpcode::kGt:
diff --git a/tensorflow/compiler/xla/types.h b/tensorflow/compiler/xla/types.h
index 3d78466107..3b19ca321c 100644
--- a/tensorflow/compiler/xla/types.h
+++ b/tensorflow/compiler/xla/types.h
@@ -16,6 +16,8 @@ limitations under the License.
 #ifndef TENSORFLOW_COMPILER_XLA_TYPES_H_
 #define TENSORFLOW_COMPILER_XLA_TYPES_H_
 
+#include <complex>
+
 #include "third_party/eigen3/Eigen/Core"
 #include "tensorflow/core/platform/types.h"
 
@@ -35,7 +37,7 @@ using ::tensorflow::uint16;
 using ::tensorflow::uint32;
 using ::tensorflow::uint64;
 
-typedef std::complex<float> complex64;
+using complex64 = std::complex<float>;
 
 using ::Eigen::half;
 
diff --git a/tensorflow/compiler/xla/xla_data.proto b/tensorflow/compiler/xla/xla_data.proto
index 0efa3d0014..fe47f85c12 100644
--- a/tensorflow/compiler/xla/xla_data.proto
+++ b/tensorflow/compiler/xla/xla_data.proto
@@ -49,7 +49,7 @@ enum PrimitiveType {
   F64 = 12;
 
   // Complex values of fixed width.
-  C64 = 15;
+  C64 = 15;  // Paired F32 (real, imag), as in std::complex<float>.
 
   // A tuple is a polymorphic sequence; e.g. a shape that holds different
   // sub-shapes. They are used for things like returning multiple values from a
@@ -667,6 +667,12 @@ enum UnaryOperation {
   // Elementwise, rounds x to nearest integral value, rounding half-way cases
   // away from zero.
   UNOP_ROUND_NEAREST_AFZ = 14;
+
+  // Elementwise, extract real component of complex x.
+  UNOP_REAL = 15;
+
+  // Elementwise, extract real component of complex x.
+  UNOP_IMAG = 16;
 }
 
 message UnaryOpRequest {
@@ -721,6 +727,12 @@ enum BinaryOperation {
   BINOP_SHIFT_LEFT = 20;
   BINOP_SHIFT_RIGHT_ARITHMETIC = 21;
   BINOP_SHIFT_RIGHT_LOGICAL = 22;
+
+  // Complex from real, imag.
+  BINOP_COMPLEX = 23;
+
+  // Computes the 4-quadrant arctangent of the y, x input arguments.
+  BINOP_ATAN2 = 24;
 }
 
 message BinaryOpRequest {
-- 
GitLab


From c22973867f742bb1395a4cdb87deb8f7cb21d1a5 Mon Sep 17 00:00:00 2001
From: Mark Daoust <markdaoust@google.com>
Date: Fri, 27 Oct 2017 09:32:39 -0700
Subject: [PATCH 867/909] Delete bad links (md links not supported in html
 blocks).

PiperOrigin-RevId: 173680417
---
 tensorflow/docs_src/get_started/linear_regression.md | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/tensorflow/docs_src/get_started/linear_regression.md b/tensorflow/docs_src/get_started/linear_regression.md
index 7cfff8db15..45cb9d829c 100644
--- a/tensorflow/docs_src/get_started/linear_regression.md
+++ b/tensorflow/docs_src/get_started/linear_regression.md
@@ -4,32 +4,28 @@ This unit provides the following short examples demonstrating how
 to implement regression in Estimators:
 
 <table>
-  <tr> <th>Example</th> <th>Data Set</th> <th>Demonstrates How To...</th></tr>
+  <tr> <th>Example</th> <th>Demonstrates How To...</th></tr>
 
   <tr>
     <td><a href="https://www.tensorflow.org/code/tensorflow/examples/get_started/regression/linear_regression.py">linear_regression.py</a></td>
-    <td>[imports85](https://archive.ics.uci.edu/ml/datasets/automobile)</td>
     <td>Use the @{tf.estimator.LinearRegressor} Estimator to train a
         regression model on numeric data.</td>
   </tr>
 
   <tr>
     <td><a href="https://www.tensorflow.org/code/tensorflow/examples/get_started/regression/linear_regression_categorical.py">linear_regression_categorical.py</a></td>
-    <td>[imports85](https://archive.ics.uci.edu/ml/datasets/automobile)</td>
     <td>Use the @{tf.estimator.LinearRegressor} Estimator to train a
         regression model on categorical data.</td>
   </tr>
 
   <tr>
     <td><a href="https://www.tensorflow.org/code/tensorflow/examples/get_started/regression/dnn_regression.py">dnn_regression.py</a></td>
-    <td>[imports85](https://archive.ics.uci.edu/ml/datasets/automobile)</td>
     <td>Use the @{tf.estimator.DNNRegressor} Estimator to train a
         regression model on discrete data with a deep neural network.</td>
   </tr>
 
   <tr>
     <td><a href="https://www.tensorflow.org/code/tensorflow/examples/get_started/regression/custom_regression.py">custom_regression.py</a></td>
-    <td>[imports85](https://archive.ics.uci.edu/ml/datasets/automobile)</td>
     <td>Use @{tf.estimator.Estimator} to train a customized dnn
         regression model.</td>
   </tr>
@@ -96,7 +92,7 @@ During training, all three programs output the following information:
 For example, here's some possible output for the `linear_regressor.py`
 program:
 
-```bsh
+``` None
 INFO:tensorflow:Saving checkpoints for 1 into /tmp/tmpAObiz9/model.ckpt.
 INFO:tensorflow:loss = 161.308, step = 1
 INFO:tensorflow:global_step/sec: 1557.24
-- 
GitLab


From 96dc501cd9e9aefd2766cc02f8d0e5436d198bb8 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Fri, 27 Oct 2017 14:17:32 +0000
Subject: [PATCH 868/909] Fix incorrect annotation tag in tf.Variable

In tf.Variable the annotation tag of `@compatiblity` should be `@compatibility`
---
 tensorflow/python/ops/variables.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/ops/variables.py b/tensorflow/python/ops/variables.py
index fd0aee3c33..5d18979bef 100644
--- a/tensorflow/python/ops/variables.py
+++ b/tensorflow/python/ops/variables.py
@@ -119,7 +119,7 @@ class Variable(object):
   various `Optimizer` classes use this collection as the default list of
   variables to optimize.
 
-  @compatiblity(eager)
+  @compatibility(eager)
   `tf.Variable` is not compatible with eager execution.  Use
   `tfe.Variable` instead which is compatable with both eager execution
   and graph construction.  See [the TensorFlow Eager Execution
-- 
GitLab


From 7d7b2ec58e9625c5f2320c67516a5e0aff06d0d3 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Fri, 27 Oct 2017 14:20:58 +0000
Subject: [PATCH 869/909] Also fixes `@end_compatiblity` ->
 `@end_compatibility`

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/python/ops/variables.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/ops/variables.py b/tensorflow/python/ops/variables.py
index 5d18979bef..e78139edc2 100644
--- a/tensorflow/python/ops/variables.py
+++ b/tensorflow/python/ops/variables.py
@@ -125,7 +125,7 @@ class Variable(object):
   and graph construction.  See [the TensorFlow Eager Execution
   guide](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/eager/python/g3doc/guide.md#variables-and-optimizers)
   for details on how variables work in eager execution.
-  @end_compatiblity
+  @end_compatibility
   """
 
   def __init__(self,
-- 
GitLab


From 5120e75cffc1bef4766cc8d53f2e13a00750204a Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Fri, 27 Oct 2017 17:02:27 +0000
Subject: [PATCH 870/909] Move `@compatibility(eager)` from class docstring to
 __init__ docstring

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/python/ops/variables.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/tensorflow/python/ops/variables.py b/tensorflow/python/ops/variables.py
index e78139edc2..57b27051af 100644
--- a/tensorflow/python/ops/variables.py
+++ b/tensorflow/python/ops/variables.py
@@ -118,14 +118,6 @@ class Variable(object):
   `trainable_variables()` returns the contents of this collection. The
   various `Optimizer` classes use this collection as the default list of
   variables to optimize.
-
-  @compatibility(eager)
-  `tf.Variable` is not compatible with eager execution.  Use
-  `tfe.Variable` instead which is compatable with both eager execution
-  and graph construction.  See [the TensorFlow Eager Execution
-  guide](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/eager/python/g3doc/guide.md#variables-and-optimizers)
-  for details on how variables work in eager execution.
-  @end_compatibility
   """
 
   def __init__(self,
@@ -197,6 +189,14 @@ class Variable(object):
       ValueError: If the initial value is not specified, or does not have a
         shape and `validate_shape` is `True`.
       RuntimeError: If eager execution is enabled.
+
+    @compatibility(eager)
+    `tf.Variable` is not compatible with eager execution.  Use
+    `tfe.Variable` instead which is compatable with both eager execution
+    and graph construction.  See [the TensorFlow Eager Execution
+    guide](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/eager/python/g3doc/guide.md#variables-and-optimizers)
+    for details on how variables work in eager execution.
+    @end_compatibility
     """
     if not context.in_graph_mode():
       raise RuntimeError("tf.Variable not supported in Eager mode. "
-- 
GitLab


From 7775a6604330497c81d8290037b7f59ffffafec0 Mon Sep 17 00:00:00 2001
From: Michael Case <mikecase@google.com>
Date: Fri, 27 Oct 2017 10:15:31 -0700
Subject: [PATCH 871/909] Internal Change

PiperOrigin-RevId: 173685895
---
 tensorflow/workspace.bzl                                 | 9 +++++++++
 .../arm_neon_2_x86_sse.BUILD                             | 0
 2 files changed, 9 insertions(+)
 rename {tensorflow/opensource_only => third_party}/arm_neon_2_x86_sse.BUILD (100%)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index b9651a92f7..c0eb87a744 100644
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -723,6 +723,15 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       deps = ["@com_google_guava"],
   )
 
+  java_import_external(
+      name = "javax_validation",
+      jar_sha256 = "e459f313ebc6db2483f8ceaad39af07086361b474fa92e40f442e8de5d9895dc",
+      jar_urls = [
+          "http://repo1.maven.org/maven2/javax/validation/validation-api/1.0.0.GA/validation-api-1.0.0.GA.jar",
+      ],
+      licenses = ["notice"],  # Apache 2.0
+  )
+
   native.new_http_archive(
       name = "com_google_pprof",
       urls = [
diff --git a/tensorflow/opensource_only/arm_neon_2_x86_sse.BUILD b/third_party/arm_neon_2_x86_sse.BUILD
similarity index 100%
rename from tensorflow/opensource_only/arm_neon_2_x86_sse.BUILD
rename to third_party/arm_neon_2_x86_sse.BUILD
-- 
GitLab


From 6b05b36cd2c809a1fd581341be51076ab0d05e8e Mon Sep 17 00:00:00 2001
From: Jiri Simsa <jsimsa@google.com>
Date: Fri, 27 Oct 2017 10:29:36 -0700
Subject: [PATCH 872/909] Generalizing sloppy_interleave, making sloppiness an
 option.

PiperOrigin-RevId: 173687797
---
 tensorflow/contrib/data/__init__.py           |   2 +-
 .../contrib/data/python/kernel_tests/BUILD    |  46 +++---
 ..._test.py => interleave_dataset_op_test.py} | 136 ++++++++++++++----
 tensorflow/contrib/data/python/ops/BUILD      |   2 +-
 .../ops/{sloppy_ops.py => interleave_ops.py}  |  66 ++++++++-
 tensorflow/core/kernels/BUILD                 |   6 +-
 .../core/kernels/map_and_batch_dataset_op.cc  |   2 +-
 ...p.cc => parallel_interleave_dataset_op.cc} |  78 +++++++---
 .../core/kernels/prefetch_dataset_op.cc       |   1 -
 .../core/ops/compat/ops_history.v1.pbtxt      |  89 ------------
 tensorflow/core/ops/dataset_ops.cc            |   3 +-
 11 files changed, 252 insertions(+), 179 deletions(-)
 rename tensorflow/contrib/data/python/kernel_tests/{sloppy_transformation_dataset_op_test.py => interleave_dataset_op_test.py} (84%)
 rename tensorflow/contrib/data/python/ops/{sloppy_ops.py => interleave_ops.py} (67%)
 rename tensorflow/core/kernels/{sloppy_interleave_dataset_op.cc => parallel_interleave_dataset_op.cc} (84%)

diff --git a/tensorflow/contrib/data/__init__.py b/tensorflow/contrib/data/__init__.py
index e0aab1cd83..6c46acf204 100644
--- a/tensorflow/contrib/data/__init__.py
+++ b/tensorflow/contrib/data/__init__.py
@@ -50,6 +50,7 @@ from tensorflow.contrib.data.python.ops.dataset_ops import get_single_element
 from tensorflow.contrib.data.python.ops.enumerate_ops import enumerate_dataset
 from tensorflow.contrib.data.python.ops.error_ops import ignore_errors
 from tensorflow.contrib.data.python.ops.grouping import group_by_window
+from tensorflow.contrib.data.python.ops.interleave_ops import sloppy_interleave
 from tensorflow.contrib.data.python.ops.iterator_ops import make_saveable_from_iterator
 from tensorflow.contrib.data.python.ops.readers import FixedLengthRecordDataset
 from tensorflow.contrib.data.python.ops.readers import read_batch_features
@@ -57,7 +58,6 @@ from tensorflow.contrib.data.python.ops.readers import SqlDataset
 from tensorflow.contrib.data.python.ops.readers import TextLineDataset
 from tensorflow.contrib.data.python.ops.readers import TFRecordDataset
 from tensorflow.contrib.data.python.ops.resampling import rejection_resample
-from tensorflow.contrib.data.python.ops.sloppy_ops import sloppy_interleave
 from tensorflow.python.data.ops.iterator_ops import Iterator
 # pylint: enable=unused-import
 
diff --git a/tensorflow/contrib/data/python/kernel_tests/BUILD b/tensorflow/contrib/data/python/kernel_tests/BUILD
index c310e79741..ff59e80b79 100644
--- a/tensorflow/contrib/data/python/kernel_tests/BUILD
+++ b/tensorflow/contrib/data/python/kernel_tests/BUILD
@@ -143,6 +143,29 @@ py_test(
     ],
 )
 
+py_test(
+    name = "interleave_dataset_op_test",
+    size = "small",
+    srcs = ["interleave_dataset_op_test.py"],
+    srcs_version = "PY2AND3",
+    tags = [
+        "manual",  # b/67958761
+    ],
+    deps = [
+        "//tensorflow/contrib/data/python/ops:dataset_ops",
+        "//tensorflow/contrib/data/python/ops:transformation_ops",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:client",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:dtypes",
+        "//tensorflow/python:errors",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:script_ops",
+        "//tensorflow/python:training",
+        "//third_party/py/numpy",
+    ],
+)
+
 py_test(
     name = "iterator_ops_cluster_test",
     size = "small",
@@ -352,29 +375,6 @@ py_test(
     ],
 )
 
-py_test(
-    name = "sloppy_transformation_dataset_op_test",
-    size = "small",
-    srcs = ["sloppy_transformation_dataset_op_test.py"],
-    srcs_version = "PY2AND3",
-    tags = [
-        "manual",  # b/67958761
-    ],
-    deps = [
-        "//tensorflow/contrib/data/python/ops:dataset_ops",
-        "//tensorflow/contrib/data/python/ops:transformation_ops",
-        "//tensorflow/python:array_ops",
-        "//tensorflow/python:client",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:dtypes",
-        "//tensorflow/python:errors",
-        "//tensorflow/python:math_ops",
-        "//tensorflow/python:script_ops",
-        "//tensorflow/python:training",
-        "//third_party/py/numpy",
-    ],
-)
-
 py_test(
     name = "sql_dataset_op_test",
     size = "small",
diff --git a/tensorflow/contrib/data/python/kernel_tests/sloppy_transformation_dataset_op_test.py b/tensorflow/contrib/data/python/kernel_tests/interleave_dataset_op_test.py
similarity index 84%
rename from tensorflow/contrib/data/python/kernel_tests/sloppy_transformation_dataset_op_test.py
rename to tensorflow/contrib/data/python/kernel_tests/interleave_dataset_op_test.py
index 880e01dc06..0aa9ea88de 100644
--- a/tensorflow/contrib/data/python/kernel_tests/sloppy_transformation_dataset_op_test.py
+++ b/tensorflow/contrib/data/python/kernel_tests/interleave_dataset_op_test.py
@@ -25,7 +25,7 @@ import time
 from six.moves import zip_longest
 
 from tensorflow.contrib.data.python.ops import dataset_ops
-from tensorflow.contrib.data.python.ops import sloppy_ops
+from tensorflow.contrib.data.python.ops import interleave_ops
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.ops import array_ops
@@ -34,12 +34,13 @@ from tensorflow.python.ops import script_ops
 from tensorflow.python.platform import test
 
 
-class SloppyInterleaveDatasetTest(test.TestCase):
+class ParallelInterleaveDatasetTest(test.TestCase):
 
   def setUp(self):
     self.input_values = array_ops.placeholder(dtypes.int64, shape=[None])
     self.cycle_length = array_ops.placeholder(dtypes.int64, shape=[])
     self.block_length = array_ops.placeholder(dtypes.int64, shape=[])
+    self.sloppy = array_ops.placeholder(dtypes.bool, shape=[])
 
     self.repeat_count = 2
 
@@ -69,9 +70,9 @@ class SloppyInterleaveDatasetTest(test.TestCase):
 
     self.dataset = (dataset_ops.Dataset.from_tensor_slices(self.input_values)
                     .repeat(self.repeat_count).apply(
-                        sloppy_ops.sloppy_interleave(
+                        interleave_ops.parallel_interleave(
                             interleave_fn, self.cycle_length,
-                            self.block_length)))
+                            self.block_length, self.sloppy)))
     self.iterator = self.dataset.make_initializable_iterator()
     self.init_op = self.iterator.initializer
     self.next_element = self.iterator.get_next()
@@ -161,7 +162,7 @@ class SloppyInterleaveDatasetTest(test.TestCase):
     for i in range(4, 7):
       self.write_coordination_events[i].set()
 
-  def testSingleThreaded(self):
+  def _testSingleThreaded(self, sloppy=False):
     # cycle_length=1,block_length=1 acts like `Dataset.interleave()` and
     # `Dataset.flat_map()` and is single-threaded. No synchronization required.
     with self.test_session() as sess:
@@ -171,7 +172,8 @@ class SloppyInterleaveDatasetTest(test.TestCase):
           feed_dict={
               self.input_values: [4, 5, 6],
               self.cycle_length: 1,
-              self.block_length: 1
+              self.block_length: 1,
+              self.sloppy: sloppy
           })
 
       for expected_element in self._interleave(
@@ -182,7 +184,13 @@ class SloppyInterleaveDatasetTest(test.TestCase):
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(self.next_element)
 
-  def testTwoThreadsNoContention(self):
+  def testSingleThreaded(self):
+    self._testSingleThreaded()
+
+  def testSingleThreadedSloppy(self):
+    self._testSingleThreaded(sloppy=True)
+
+  def _testTwoThreadsNoContention(self, sloppy=False):
     # num_threads > 1.
     # Explicit coordination should result in `Dataset.interleave()` behavior
     with self.test_session() as sess:
@@ -193,7 +201,8 @@ class SloppyInterleaveDatasetTest(test.TestCase):
           feed_dict={
               self.input_values: [4, 5, 6],
               self.cycle_length: 2,
-              self.block_length: 1
+              self.block_length: 1,
+              self.sloppy: sloppy
           })
       for i, expected_element in enumerate(
           self._interleave([[4] * 4, [5] * 5, [6] * 6] * self.repeat_count, 2,
@@ -211,11 +220,20 @@ class SloppyInterleaveDatasetTest(test.TestCase):
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(self.next_element)
 
-  def testTwoThreadsNoContentionWithRaces(self):
+  def testTwoThreadsNoContention(self):
+    self._testTwoThreadsNoContention()
+
+  def testTwoThreadsNoContentionSloppy(self):
+    self._testTwoThreadsNoContention(sloppy=True)
+
+  def _testTwoThreadsNoContentionWithRaces(self, sloppy=False):
     """Tests where all the workers race in producing elements.
 
     Note: this is in contrast with the prevous test which carefully sequences
     the execution of the map functions.
+
+    Args:
+      sloppy: Whether to be sloppy or not.
     """
     with self.test_session() as sess:
       self._clear_coordination_events()
@@ -225,7 +243,8 @@ class SloppyInterleaveDatasetTest(test.TestCase):
           feed_dict={
               self.input_values: [4, 5, 6],
               self.cycle_length: 2,
-              self.block_length: 1
+              self.block_length: 1,
+              self.sloppy: sloppy,
           })
       for i, expected_element in enumerate(
           self._interleave([[4] * 4, [5] * 5, [6] * 6] * self.repeat_count, 2,
@@ -247,7 +266,13 @@ class SloppyInterleaveDatasetTest(test.TestCase):
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(self.next_element)
 
-  def testTwoThreadsNoContentionBlockLength(self):
+  def testTwoThreadsNoContentionWithRaces(self):
+    self._testTwoThreadsNoContentionWithRaces()
+
+  def testTwoThreadsNoContentionWithRacesSloppy(self):
+    self._testTwoThreadsNoContentionWithRaces(sloppy=True)
+
+  def _testTwoThreadsNoContentionBlockLength(self, sloppy=False):
     # num_threads > 1.
     # Explicit coordination should result in `Dataset.interleave()` behavior
     with self.test_session() as sess:
@@ -258,7 +283,8 @@ class SloppyInterleaveDatasetTest(test.TestCase):
           feed_dict={
               self.input_values: [4, 5, 6],
               self.cycle_length: 2,
-              self.block_length: 2
+              self.block_length: 2,
+              self.sloppy: sloppy
           })
       for i, expected_element in enumerate(
           self._interleave([[4] * 4, [5] * 5, [6] * 6] * self.repeat_count, 2,
@@ -276,11 +302,21 @@ class SloppyInterleaveDatasetTest(test.TestCase):
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(self.next_element)
 
-  def testTwoThreadsNoContentionWithRacesAndBlocking(self):
+  def testTwoThreadsNoContentionBlockLength(self):
+    self._testTwoThreadsNoContentionBlockLength()
+
+  def testTwoThreadsNoContentionBlockLengthSloppy(self):
+    self._testTwoThreadsNoContentionBlockLength(sloppy=True)
+
+  def _testTwoThreadsNoContentionWithRacesAndBlocking(self, sloppy=False):
     """Tests where all the workers race in producing elements.
 
     Note: this is in contrast with the prevous test which carefully sequences
     the execution of the map functions.
+
+
+    Args:
+      sloppy: Whether to be sloppy or not.
     """
     with self.test_session() as sess:
       self._clear_coordination_events()
@@ -290,7 +326,8 @@ class SloppyInterleaveDatasetTest(test.TestCase):
           feed_dict={
               self.input_values: [4, 5, 6],
               self.cycle_length: 2,
-              self.block_length: 2
+              self.block_length: 2,
+              self.sloppy: sloppy
           })
       for i, expected_element in enumerate(
           self._interleave([[4] * 4, [5] * 5, [6] * 6] * self.repeat_count, 2,
@@ -312,7 +349,13 @@ class SloppyInterleaveDatasetTest(test.TestCase):
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(self.next_element)
 
-  def testEmptyInput(self):
+  def testTwoThreadsNoContentionWithRacesAndBlocking(self):
+    self._testTwoThreadsNoContentionWithRacesAndBlocking()
+
+  def testTwoThreadsNoContentionWithRacesAndBlockingSloppy(self):
+    self._testTwoThreadsNoContentionWithRacesAndBlocking(sloppy=True)
+
+  def _testEmptyInput(self, sloppy=False):
     with self.test_session() as sess:
       # Empty input.
       self._clear_coordination_events()
@@ -321,12 +364,19 @@ class SloppyInterleaveDatasetTest(test.TestCase):
           feed_dict={
               self.input_values: [],
               self.cycle_length: 2,
-              self.block_length: 3
+              self.block_length: 3,
+              self.sloppy: sloppy
           })
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(self.next_element)
 
-  def testNonEmptyInputIntoEmptyOutputs(self):
+  def testEmptyInput(self):
+    self._testEmptyInput()
+
+  def testEmptyInputSloppy(self):
+    self._testEmptyInput(sloppy=True)
+
+  def _testNonEmptyInputIntoEmptyOutputs(self, sloppy=False):
     # Non-empty input leading to empty output.
     with self.test_session() as sess:
       self._clear_coordination_events()
@@ -335,12 +385,19 @@ class SloppyInterleaveDatasetTest(test.TestCase):
           feed_dict={
               self.input_values: [0, 0, 0],
               self.cycle_length: 2,
-              self.block_length: 3
+              self.block_length: 3,
+              self.sloppy: sloppy
           })
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(self.next_element)
 
-  def testPartiallyEmptyOutputs(self):
+  def testNonEmptyInputIntoEmptyOutputs(self):
+    self._testNonEmptyInputIntoEmptyOutputs()
+
+  def testNonEmptyInputIntoEmptyOutputsSloppy(self):
+    self._testNonEmptyInputIntoEmptyOutputs(sloppy=True)
+
+  def _testPartiallyEmptyOutputs(self, sloppy=False):
     # Mixture of non-empty and empty interleaved datasets.
     with self.test_session() as sess:
       self._clear_coordination_events()
@@ -350,7 +407,8 @@ class SloppyInterleaveDatasetTest(test.TestCase):
           feed_dict={
               self.input_values: [4, 0, 6],
               self.cycle_length: 2,
-              self.block_length: 1
+              self.block_length: 1,
+              self.sloppy: sloppy,
           })
       for i, expected_element in enumerate(
           self._interleave([[4] * 4, [], [6] * 6] * self.repeat_count, 2, 1)):
@@ -367,7 +425,13 @@ class SloppyInterleaveDatasetTest(test.TestCase):
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(self.next_element)
 
-  def testDelayedOutput(self):
+  def testPartiallyEmptyOutputs(self):
+    self._testPartiallyEmptyOutputs()
+
+  def testPartiallyEmptyOutputsSloppy(self):
+    self._testPartiallyEmptyOutputs(sloppy=True)
+
+  def testDelayedOutputSloppy(self):
     # Explicitly control the sequence of events to ensure we correctly avoid
     # head-of-line blocking.
     with self.test_session() as sess:
@@ -377,7 +441,8 @@ class SloppyInterleaveDatasetTest(test.TestCase):
           feed_dict={
               self.input_values: [4, 5, 6],
               self.cycle_length: 2,
-              self.block_length: 1
+              self.block_length: 1,
+              self.sloppy: True,
           })
 
       mis_ordering = [
@@ -391,7 +456,7 @@ class SloppyInterleaveDatasetTest(test.TestCase):
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(self.next_element)
 
-  def testBlockLengthWithContention(self):
+  def testBlockLengthWithContentionSloppy(self):
     with self.test_session() as sess:
       self._clear_coordination_events()
       done_first_event = False
@@ -400,7 +465,8 @@ class SloppyInterleaveDatasetTest(test.TestCase):
           feed_dict={
               self.input_values: [4, 5, 6],
               self.cycle_length: 2,
-              self.block_length: 3
+              self.block_length: 3,
+              self.sloppy: True
           })
       # Test against a generating sequence that differs from the uncontended
       # case, in order to prove sloppy correctness.
@@ -422,7 +488,7 @@ class SloppyInterleaveDatasetTest(test.TestCase):
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(self.next_element)
 
-  def testEarlyExit(self):
+  def _testEarlyExit(self, sloppy=False):
     # Exiting without consuming all input should not block
     with self.test_session() as sess:
       self._clear_coordination_events()
@@ -431,7 +497,8 @@ class SloppyInterleaveDatasetTest(test.TestCase):
           feed_dict={
               self.input_values: [4, 5, 6],
               self.cycle_length: 3,
-              self.block_length: 2
+              self.block_length: 2,
+              self.sloppy: sloppy
           })
       for i in range(4, 7):
         self.write_coordination_events[i].set()
@@ -445,7 +512,13 @@ class SloppyInterleaveDatasetTest(test.TestCase):
         self.read_coordination_events[i].acquire()
         self.write_coordination_events[i].set()
 
-  def testTooManyReaders(self):
+  def testEarlyExit(self):
+    self._testEarlyExit()
+
+  def testEarlyExitSloppy(self):
+    self._testEarlyExit(sloppy=True)
+
+  def _testTooManyReaders(self, sloppy=False):
 
     def interleave_fn(x):
       dataset = dataset_ops.Dataset.from_tensors(x)
@@ -455,8 +528,8 @@ class SloppyInterleaveDatasetTest(test.TestCase):
     dataset = dataset_ops.Dataset.from_tensor_slices([4, 5, 6])
     dataset = dataset.repeat(self.repeat_count)
     dataset = dataset.apply(
-        sloppy_ops.sloppy_interleave(interleave_fn, cycle_length=16,
-                                     block_length=2))
+        interleave_ops.parallel_interleave(
+            interleave_fn, cycle_length=16, block_length=2, sloppy=sloppy))
     iterator = dataset.make_one_shot_iterator()
 
     with self.test_session() as sess:
@@ -468,6 +541,11 @@ class SloppyInterleaveDatasetTest(test.TestCase):
         [[4] * 4, [5] * 5, [6] * 6] * self.repeat_count, 1, 2)
     self.assertItemsEqual(output_values, expected_values)
 
+  def testTooManyReaders(self):
+    self._testTooManyReaders()
+
+  def testTooManyReadersSloppy(self):
+    self._testTooManyReaders(sloppy=True)
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/contrib/data/python/ops/BUILD b/tensorflow/contrib/data/python/ops/BUILD
index a6eb50014a..e0730488a1 100644
--- a/tensorflow/contrib/data/python/ops/BUILD
+++ b/tensorflow/contrib/data/python/ops/BUILD
@@ -60,9 +60,9 @@ py_library(
         "enumerate_ops.py",
         "error_ops.py",
         "grouping.py",
+        "interleave_ops.py",
         "resampling.py",
         "scan_ops.py",
-        "sloppy_ops.py",
     ],
     srcs_version = "PY2AND3",
     deps = [
diff --git a/tensorflow/contrib/data/python/ops/sloppy_ops.py b/tensorflow/contrib/data/python/ops/interleave_ops.py
similarity index 67%
rename from tensorflow/contrib/data/python/ops/sloppy_ops.py
rename to tensorflow/contrib/data/python/ops/interleave_ops.py
index 4f3da4320c..74a919c1ff 100644
--- a/tensorflow/contrib/data/python/ops/sloppy_ops.py
+++ b/tensorflow/contrib/data/python/ops/interleave_ops.py
@@ -23,14 +23,16 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import function
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import gen_dataset_ops
+from tensorflow.python.util import deprecation
 
 
-class SloppyInterleaveDataset(dataset_ops.Dataset):
+class ParallelInterleaveDataset(dataset_ops.Dataset):
   """A `Dataset` that maps a function over its input and flattens the result."""
 
-  def __init__(self, input_dataset, map_func, cycle_length, block_length):
-    """See `tf.contrib.data.sloppy_interleave()` for details."""
-    super(SloppyInterleaveDataset, self).__init__()
+  def __init__(self, input_dataset, map_func, cycle_length, block_length,
+               sloppy):
+    """See `tf.contrib.data.parallel_interleave()` for details."""
+    super(ParallelInterleaveDataset, self).__init__()
     self._input_dataset = input_dataset
 
     @function.Defun(*nest.flatten(input_dataset.output_types))
@@ -62,13 +64,16 @@ class SloppyInterleaveDataset(dataset_ops.Dataset):
         cycle_length, dtype=dtypes.int64, name="cycle_length")
     self._block_length = ops.convert_to_tensor(
         block_length, dtype=dtypes.int64, name="block_length")
+    self._sloppy = ops.convert_to_tensor(
+        sloppy, dtype=dtypes.bool, name="sloppy")
 
   def _as_variant_tensor(self):
-    return gen_dataset_ops.sloppy_interleave_dataset(
+    return gen_dataset_ops.parallel_interleave_dataset(
         self._input_dataset._as_variant_tensor(),  # pylint: disable=protected-access
         self._map_func.captured_inputs,
         self._cycle_length,
         self._block_length,
+        self._sloppy,
         f=self._map_func,
         output_types=nest.flatten(self.output_types),
         output_shapes=nest.flatten(self.output_shapes))
@@ -82,6 +87,53 @@ class SloppyInterleaveDataset(dataset_ops.Dataset):
     return self._output_types
 
 
+def parallel_interleave(map_func, cycle_length, block_length=1, sloppy=False):
+  """A parallel version of the `Dataset.interleave()` transformation.
+
+  `parallel_interleave()` maps `map_func` across its input to produce nested
+  datasets, and outputs their elements interleaved. Unlike
+  @{tf.data.Dataset.interleave}, it gets elements from `cycle_length` nested
+  datasets in parallel, which increases the throughput, especially in the
+  presence of stragglers. Furthermore, the `sloppy` argument can be used to
+  improve performance, by relaxing the requirement that the outputs are produced
+  in a deterministic order, and allowing the implementation to skip over nested
+  datasets whose elements are not readily available when requested.
+
+  Example usage:
+
+  ```python
+  # Preprocess 4 files concurrently.
+  filenames = tf.data.Dataset.list_files("/path/to/data/train*.tfrecords")
+  dataset = filenames.apply(
+      tf.contrib.data.parallel_interleave(
+          lambda filename: tf.data.TFRecordDataset(filename),
+          cycle_length=4))
+  ```
+
+  WARNING: If `sloppy` is `True`, the order of produced elements is not
+  deterministic.
+
+  Args:
+    map_func: A function mapping a nested structure of tensors to a `Dataset`.
+    cycle_length: The number of threads to interleave from in parallel.
+    block_length: The number of consecutive elements to pull from a thread
+      before advancing to the next thread.
+    sloppy: If false, elements are produced in deterministic order. Otherwise,
+      the implementation is allowed, for the sake of expediency, to produce
+      elements in a non-deterministic order.
+
+  Returns:
+    A `Dataset` transformation function, which can be passed to
+    @{tf.data.Dataset.apply}.
+  """
+  def _apply_fn(dataset):
+    return ParallelInterleaveDataset(
+        dataset, map_func, cycle_length, block_length, sloppy)
+  return _apply_fn
+
+
+@deprecation.deprecated(
+    None, "Use `tf.contrib.data.parallel_interleave(..., sloppy=True)`.")
 def sloppy_interleave(map_func, cycle_length, block_length=1):
   """A non-deterministic version of the `Dataset.interleave()` transformation.
 
@@ -132,6 +184,6 @@ def sloppy_interleave(map_func, cycle_length, block_length=1):
     @{tf.data.Dataset.apply}.
   """
   def _apply_fn(dataset):
-    return SloppyInterleaveDataset(
-        dataset, map_func, cycle_length, block_length)
+    return ParallelInterleaveDataset(
+        dataset, map_func, cycle_length, block_length, sloppy=True)
   return _apply_fn
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 0274f87ec6..2aef1e3560 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -5924,8 +5924,8 @@ tf_kernel_library(
 )
 
 tf_kernel_library(
-    name = "sloppy_interleave_dataset_op",
-    srcs = ["sloppy_interleave_dataset_op.cc"],
+    name = "parallel_interleave_dataset_op",
+    srcs = ["parallel_interleave_dataset_op.cc"],
     deps = [
         ":captured_function",
         ":dataset",
@@ -6162,6 +6162,7 @@ tf_kernel_library(
         ":map_and_batch_dataset_op",
         ":map_dataset_op",
         ":padded_batch_dataset_op",
+        ":parallel_interleave_dataset_op",
         ":parallel_map_dataset_op",
         ":prefetch_dataset_op",
         ":range_dataset_op",
@@ -6170,7 +6171,6 @@ tf_kernel_library(
         ":scan_dataset_op",
         ":shuffle_dataset_op",
         ":skip_dataset_op",
-        ":sloppy_interleave_dataset_op",
         ":sparse_tensor_slice_dataset_op",
         ":sql_dataset_ops",
         ":take_dataset_op",
diff --git a/tensorflow/core/kernels/map_and_batch_dataset_op.cc b/tensorflow/core/kernels/map_and_batch_dataset_op.cc
index f9f68a5418..620efdb778 100644
--- a/tensorflow/core/kernels/map_and_batch_dataset_op.cc
+++ b/tensorflow/core/kernels/map_and_batch_dataset_op.cc
@@ -336,7 +336,7 @@ class MapAndBatchDatasetOp : public UnaryDatasetOpKernel {
     const DataTypeVector output_types_;
     const std::vector<PartialTensorShape> output_shapes_;
     const std::unique_ptr<CapturedFunction> captured_func_;
-    const Eigen::ThreadPoolDevice* device_; // not owned
+    const Eigen::ThreadPoolDevice* device_;  // not owned
   };
 
   const int graph_def_version_;
diff --git a/tensorflow/core/kernels/sloppy_interleave_dataset_op.cc b/tensorflow/core/kernels/parallel_interleave_dataset_op.cc
similarity index 84%
rename from tensorflow/core/kernels/sloppy_interleave_dataset_op.cc
rename to tensorflow/core/kernels/parallel_interleave_dataset_op.cc
index 8f9f48700c..56942a5c01 100644
--- a/tensorflow/core/kernels/sloppy_interleave_dataset_op.cc
+++ b/tensorflow/core/kernels/parallel_interleave_dataset_op.cc
@@ -17,12 +17,11 @@ limitations under the License.
 #include "tensorflow/core/common_runtime/function.h"
 #include "tensorflow/core/framework/partial_tensor_shape.h"
 #include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/kernels/captured_function.h"
 #include "tensorflow/core/kernels/dataset_utils.h"
 #include "tensorflow/core/lib/gtl/cleanup.h"
 #include "tensorflow/core/lib/random/random.h"
 
-#include "tensorflow/core/kernels/captured_function.h"
-
 namespace tensorflow {
 
 namespace {
@@ -30,9 +29,9 @@ namespace {
 // See documentation in ../ops/dataset_ops.cc for a high-level
 // description of the following op.
 
-class SloppyInterleaveDatasetOp : public UnaryDatasetOpKernel {
+class ParallelInterleaveDatasetOp : public UnaryDatasetOpKernel {
  public:
-  explicit SloppyInterleaveDatasetOp(OpKernelConstruction* ctx)
+  explicit ParallelInterleaveDatasetOp(OpKernelConstruction* ctx)
       : UnaryDatasetOpKernel(ctx),
         graph_def_version_(ctx->graph_def_version()) {
     OP_REQUIRES_OK(ctx, ctx->GetAttr("f", &func_));
@@ -62,13 +61,16 @@ class SloppyInterleaveDatasetOp : public UnaryDatasetOpKernel {
     OP_REQUIRES(ctx, block_length > 0,
                 errors::InvalidArgument("`block_length` must be > 0"));
 
+    bool sloppy;
+    OP_REQUIRES_OK(ctx, ParseScalarArgument(ctx, "sloppy", &sloppy));
+
     std::unique_ptr<CapturedFunction> captured_func;
     OP_REQUIRES_OK(ctx, CapturedFunction::Create(ctx, func_, graph_def_version_,
                                                  std::move(other_arguments),
                                                  &captured_func));
 
     *output = new Dataset(input, std::move(captured_func), cycle_length,
-                          block_length, output_types_, output_shapes_);
+                          block_length, sloppy, output_types_, output_shapes_);
   }
 
  private:
@@ -76,12 +78,13 @@ class SloppyInterleaveDatasetOp : public UnaryDatasetOpKernel {
    public:
     Dataset(const DatasetBase* input,
             std::unique_ptr<CapturedFunction> captured_func, int64 cycle_length,
-            int64 block_length, const DataTypeVector& output_types,
+            int64 block_length, bool sloppy, const DataTypeVector& output_types,
             const std::vector<PartialTensorShape>& output_shapes)
         : input_(input),
           captured_func_(std::move(captured_func)),
           cycle_length_(cycle_length),
           block_length_(block_length),
+          sloppy_(sloppy),
           output_types_(output_types),
           output_shapes_(output_shapes) {
       input_->Ref();
@@ -91,8 +94,8 @@ class SloppyInterleaveDatasetOp : public UnaryDatasetOpKernel {
 
     std::unique_ptr<IteratorBase> MakeIterator(
         const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(
-          new Iterator({this, strings::StrCat(prefix, "::SloppyInterleave")}));
+      return std::unique_ptr<IteratorBase>(new Iterator(
+          {this, strings::StrCat(prefix, "::ParallelInterleave")}));
     }
 
     const DataTypeVector& output_dtypes() const override {
@@ -103,7 +106,7 @@ class SloppyInterleaveDatasetOp : public UnaryDatasetOpKernel {
     }
 
     string DebugString() override {
-      return "SloppyInterleaveDatasetOp::Dataset";
+      return "ParallelInterleaveDatasetOp::Dataset";
     }
 
    private:
@@ -131,16 +134,24 @@ class SloppyInterleaveDatasetOp : public UnaryDatasetOpKernel {
                              bool* end_of_sequence) override {
         mutex_lock l(mu_);
         TF_RETURN_IF_ERROR(EnsureWorkerThreadsStarted(ctx));
-        // Search for available items, blocking if necessary.
+        const int64 num_workers = worker_threads_.size();
+        if (num_workers == 0) {
+          *end_of_sequence = true;
+          return Status::OK();
+        }
         while (!cancelled_) {
-          for (size_t i = 0; i < dataset()->cycle_length_; ++i) {
-            size_t index = (next_index_ + i) % dataset()->cycle_length_;
+          // Wait for an item to become available, blocking if necessary. If we
+          // are allowed to be sloppy, we can skip over input datasets that do
+          // not have an item readily available.
+          const int64 n = dataset()->sloppy_ ? num_workers : 1LL;
+          for (int64 i = 0; i < n; ++i) {
+            int64 index = (next_index_ + i) % num_workers;
             if (output_elements_[index].is_produced) {
               next_index_ = index;
               if (i == 0) {
                 block_count_++;
                 if (block_count_ == dataset()->block_length_) {
-                  next_index_ = (index + 1) % dataset()->cycle_length_;
+                  next_index_ = (index + 1) % num_workers;
                   block_count_ = 0;
                 }
               } else {
@@ -150,7 +161,7 @@ class SloppyInterleaveDatasetOp : public UnaryDatasetOpKernel {
               if (output_elements_[index].end_of_sequence) {
                 output_elements_[index].is_produced = false;
                 output_elements_[index].cond_var.notify_one();
-                next_index_ = (index + 1) % dataset()->cycle_length_;
+                next_index_ = (index + 1) % num_workers;
                 block_count_ = 0;
                 i = -1;  // Restart the inner loop
                 continue;
@@ -174,11 +185,21 @@ class SloppyInterleaveDatasetOp : public UnaryDatasetOpKernel {
             *end_of_sequence = true;
             return Status::OK();
           }
+
+          // If we are not allowed to be sloppy and
+          // `worker_threads_[next_index]` has finished, advance `next_index`.
+          if (!dataset()->sloppy_ && worker_threads_[next_index_].finished) {
+            next_index_ = (next_index_ + 1) % num_workers;
+            continue;
+          }
+
           // No values available; wait until woken up.
+          // TODO(jsimsa): Use slot-specific condition variable for
+          // coordination of elements consumption.
           cond_var_.wait(l);
         }
         return errors::Cancelled(
-            "SloppyInterleaveDatasetOp::Dataset::Iterator::GetNext");
+            "ParallelInterleaveDatasetOp::Dataset::Iterator::GetNext");
       }
 
      private:
@@ -201,6 +222,16 @@ class SloppyInterleaveDatasetOp : public UnaryDatasetOpKernel {
         condition_variable cond_var;
       };
 
+      struct ThreadStatus {
+        // The underlying thread uses `finished` to communicate to the producer
+        // that it has finished.
+        bool finished = false;
+        // The underlying thread object.
+        std::unique_ptr<Thread> thread;
+
+        explicit ThreadStatus(Thread* thread) : thread(thread) {}
+      };
+
       Status EnsureWorkerThreadsStarted(IteratorContext* ctx)
           EXCLUSIVE_LOCKS_REQUIRED(mu_) {
         if (worker_threads_.empty()) {
@@ -220,11 +251,10 @@ class SloppyInterleaveDatasetOp : public UnaryDatasetOpKernel {
             std::unique_ptr<IteratorBase> itr;
             TF_RETURN_IF_ERROR(dataset::MakeIteratorFromInputElement(
                 ctx, args, i, dataset()->captured_func_.get(), prefix(), &itr));
-            worker_threads_.emplace_back(
-                std::unique_ptr<Thread>(ctx->env()->StartThread(
-                    {}, "worker_thread",
-                    std::bind(&Iterator::WorkerThread, this,
-                              new IteratorContext(*ctx), i, itr.release()))));
+            worker_threads_.emplace_back(ctx->env()->StartThread(
+                {}, "worker_thread",
+                std::bind(&Iterator::WorkerThread, this,
+                          new IteratorContext(*ctx), i, itr.release())));
             num_active_threads_ = i + 1;
           }
         }
@@ -264,6 +294,7 @@ class SloppyInterleaveDatasetOp : public UnaryDatasetOpKernel {
         std::unique_ptr<IteratorBase> out_iterator(out_iterator_ptr);
         auto cleanup = gtl::MakeCleanup([this, thread_index] {
           mutex_lock l(mu_);
+          worker_threads_[thread_index].finished = true;
           num_active_threads_--;
           cond_var_.notify_all();
         });
@@ -345,13 +376,14 @@ class SloppyInterleaveDatasetOp : public UnaryDatasetOpKernel {
       // Pointers to the worker threads. This must be last to ensure the
       // threads have exited before any other members are deallocated.
       // TODO(b/65178177): Avoid allocating additional threads.
-      std::vector<std::unique_ptr<Thread>> worker_threads_ GUARDED_BY(mu_);
+      std::vector<ThreadStatus> worker_threads_ GUARDED_BY(mu_);
     };
 
     const DatasetBase* const input_;
     const std::unique_ptr<CapturedFunction> captured_func_;
     const int64 cycle_length_;
     const int64 block_length_;
+    const bool sloppy_;
     const DataTypeVector output_types_;
     const std::vector<PartialTensorShape> output_shapes_;
   };
@@ -362,8 +394,8 @@ class SloppyInterleaveDatasetOp : public UnaryDatasetOpKernel {
   NameAttrList func_;
 };
 
-REGISTER_KERNEL_BUILDER(Name("SloppyInterleaveDataset").Device(DEVICE_CPU),
-                        SloppyInterleaveDatasetOp);
+REGISTER_KERNEL_BUILDER(Name("ParallelInterleaveDataset").Device(DEVICE_CPU),
+                        ParallelInterleaveDatasetOp);
 
 }  // namespace
 
diff --git a/tensorflow/core/kernels/prefetch_dataset_op.cc b/tensorflow/core/kernels/prefetch_dataset_op.cc
index 8c846919c4..a7aac508eb 100644
--- a/tensorflow/core/kernels/prefetch_dataset_op.cc
+++ b/tensorflow/core/kernels/prefetch_dataset_op.cc
@@ -59,7 +59,6 @@ class PrefetchDatasetOp : public UnaryDatasetOpKernel {
     Dataset(const DatasetBase* input, int64 buffer_size,
             IteratorContext::Params ctx_params)
         : input_(input),
-
           buffer_size_(buffer_size),
           ctx_params_(std::move(ctx_params)) {
       input_->Ref();
diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index 076c7bea1a..a691065075 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -32629,95 +32629,6 @@ op {
     }
   }
 }
-op {
-  name: "SloppyInterleaveDataset"
-  input_arg {
-    name: "input_dataset"
-    type: DT_VARIANT
-  }
-  input_arg {
-    name: "other_arguments"
-    type_list_attr: "Targuments"
-  }
-  input_arg {
-    name: "cycle_length"
-    type: DT_INT64
-  }
-  input_arg {
-    name: "block_length"
-    type: DT_INT64
-  }
-  output_arg {
-    name: "handle"
-    type: DT_VARIANT
-  }
-  attr {
-    name: "f"
-    type: "func"
-  }
-  attr {
-    name: "Targuments"
-    type: "list(type)"
-    has_minimum: true
-  }
-  attr {
-    name: "output_types"
-    type: "list(type)"
-    has_minimum: true
-    minimum: 1
-  }
-  attr {
-    name: "output_shapes"
-    type: "list(shape)"
-    has_minimum: true
-    minimum: 1
-  }
-  is_stateful: true
-}
-op {
-  name: "SloppyInterleaveDataset"
-  input_arg {
-    name: "input_dataset"
-    type: DT_VARIANT
-  }
-  input_arg {
-    name: "other_arguments"
-    type_list_attr: "Targuments"
-  }
-  input_arg {
-    name: "cycle_length"
-    type: DT_INT64
-  }
-  input_arg {
-    name: "block_length"
-    type: DT_INT64
-  }
-  output_arg {
-    name: "handle"
-    type: DT_VARIANT
-  }
-  attr {
-    name: "f"
-    type: "func"
-  }
-  attr {
-    name: "Targuments"
-    type: "list(type)"
-    has_minimum: true
-  }
-  attr {
-    name: "output_types"
-    type: "list(type)"
-    has_minimum: true
-    minimum: 1
-  }
-  attr {
-    name: "output_shapes"
-    type: "list(shape)"
-    has_minimum: true
-    minimum: 1
-  }
-}
 op {
   name: "Softmax"
   input_arg {
diff --git a/tensorflow/core/ops/dataset_ops.cc b/tensorflow/core/ops/dataset_ops.cc
index 8b77e3f9f0..f512213964 100644
--- a/tensorflow/core/ops/dataset_ops.cc
+++ b/tensorflow/core/ops/dataset_ops.cc
@@ -285,11 +285,12 @@ f: A function mapping elements of `input_dataset`, concatenated with
   `output_types` and `output_shapes`.
 )doc");
 
-REGISTER_OP("SloppyInterleaveDataset")
+REGISTER_OP("ParallelInterleaveDataset")
     .Input("input_dataset: variant")
     .Input("other_arguments: Targuments")
     .Input("cycle_length: int64")
     .Input("block_length: int64")
+    .Input("sloppy: bool")
     .Output("handle: variant")
     .Attr("f: func")
     .Attr("Targuments: list(type) >= 0")
-- 
GitLab


From 16538dab77bac6b79a05aa91c43e53227b45c945 Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Fri, 27 Oct 2017 10:49:59 -0700
Subject: [PATCH 873/909] Saves summaries in the mnist example.

PiperOrigin-RevId: 173690505
---
 .../contrib/eager/python/metrics_impl.py      |  2 +-
 .../contrib/eager/python/metrics_test.py      | 26 ++-----
 tensorflow/contrib/summary/summary_ops.py     | 74 +++++++++++++------
 tensorflow/python/training/training_util.py   | 13 +++-
 4 files changed, 72 insertions(+), 43 deletions(-)

diff --git a/tensorflow/contrib/eager/python/metrics_impl.py b/tensorflow/contrib/eager/python/metrics_impl.py
index 795dff548f..2ba653af4a 100644
--- a/tensorflow/contrib/eager/python/metrics_impl.py
+++ b/tensorflow/contrib/eager/python/metrics_impl.py
@@ -90,7 +90,7 @@ class Metric(object):
     # We create the variable scope now to get the unique name that will
     # be used as a variable prefix when build() calls add_variable().
     with variable_scope.variable_scope(
-        None, default_name=scope_name, use_resource=True, reuse=False) as scope:
+        scope_name, use_resource=True, reuse=False) as scope:
       pos = scope.name.rfind(scope_name)
       self._name = name + scope.name[pos + len(scope_name):]
       self._scope = scope
diff --git a/tensorflow/contrib/eager/python/metrics_test.py b/tensorflow/contrib/eager/python/metrics_test.py
index a8377a0660..336ce9d307 100644
--- a/tensorflow/contrib/eager/python/metrics_test.py
+++ b/tensorflow/contrib/eager/python/metrics_test.py
@@ -120,24 +120,18 @@ class MetricsTest(test.TestCase):
     # Verify two metrics with the same class and name don't
     # accidentally share state.
     m1 = metrics.Mean()
-    m2 = metrics.Mean()
     m1(0)
-    m2(2)
-    self.assertEqual(0, m1.result().numpy())
-    self.assertEqual(2, m2.result().numpy())
-    self.assertNotEqual(m1.name, m2.name)
+    with self.assertRaises(ValueError):
+      m2 = metrics.Mean()
+      m2(2)
 
   def testNamesWithSpaces(self):
     # Verify two metrics with the same class and name don't
     # accidentally share state.
     m1 = metrics.Mean("has space")
-    m2 = metrics.Mean("has space")
-    m2(2)
     m1(0)
     self.assertEqual(m1.name, "has space")
     self.assertEqual(m1.numer.name, "has_space/numer:0")
-    self.assertEqual(m2.name, "has space_1")
-    self.assertEqual(m2.numer.name, "has_space_1/numer:0")
 
   def testGraph(self):
     with context.graph_mode(), self.test_session() as sess:
@@ -158,16 +152,12 @@ class MetricsTest(test.TestCase):
   def testTwoMeansGraph(self):
     # Verify two metrics with the same class and name don't
     # accidentally share state.
-    with context.graph_mode(), self.test_session() as sess:
+    with context.graph_mode():
       m1 = metrics.Mean()
-      m2 = metrics.Mean()
-      accumulate1 = m1(0)
-      accumulate2 = m2(2)
-      m1.init_variables().run()
-      m2.init_variables().run()
-      sess.run([accumulate1, accumulate2])
-      self.assertEqual(0, m1.result().eval())
-      self.assertEqual(2, m2.result().eval())
+      m1(0)
+      with self.assertRaises(ValueError):
+        m2 = metrics.Mean()
+        m2(2)
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/contrib/summary/summary_ops.py b/tensorflow/contrib/summary/summary_ops.py
index b32b093675..9c71bf7740 100644
--- a/tensorflow/contrib/summary/summary_ops.py
+++ b/tensorflow/contrib/summary/summary_ops.py
@@ -26,6 +26,8 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.layers import utils
 from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.ops import summary_op_util
 from tensorflow.python.training import training_util
@@ -57,7 +59,8 @@ def record_summaries_every_n_global_steps(n):
   """Sets the should_record_summaries Tensor to true if global_step % n == 0."""
   collection_ref = ops.get_collection_ref(_SHOULD_RECORD_SUMMARIES_NAME)
   old = collection_ref[:]
-  collection_ref[:] = [training_util.get_global_step() % n == 0]
+  with ops.device("cpu:0"):
+    collection_ref[:] = [math_ops.equal(training_util.get_global_step() % n, 0)]
   yield
   collection_ref[:] = old
 
@@ -97,13 +100,17 @@ class SummaryWriter(object):
 
   @tf_contextlib.contextmanager
   def as_default(self):
-    old = context.context().summary_writer_resource
-    context.context().summary_writer_resource = self._resource
-    yield
-    # Flushes the summary writer in eager mode or in graph functions, but not in
-    # legacy graph mode (you're on your own there).
-    gen_summary_ops.flush_summary_writer(self._resource)
-    context.context().summary_writer_resource = old
+    if self._resource is None:
+      yield
+    else:
+      old = context.context().summary_writer_resource
+      context.context().summary_writer_resource = self._resource
+      yield
+      # Flushes the summary writer in eager mode or in graph functions, but not
+      # in legacy graph mode (you're on your own there).
+      with ops.device("cpu:0"):
+        gen_summary_ops.flush_summary_writer(self._resource)
+      context.context().summary_writer_resource = old
 
 
 def create_summary_file_writer(logdir,
@@ -111,21 +118,40 @@ def create_summary_file_writer(logdir,
                                flush_secs=None,
                                filename_suffix=None,
                                name=None):
-  """Creates a summary file writer in the current context."""
-  if max_queue is None:
-    max_queue = constant_op.constant(10)
-  if flush_secs is None:
-    flush_secs = constant_op.constant(120)
-  if filename_suffix is None:
-    filename_suffix = constant_op.constant("")
-  resource = gen_summary_ops.summary_writer(shared_name=name)
-  # TODO(apassos) ensure the initialization op runs when in graph mode; consider
-  # calling session.run here.
-  ops.add_to_collection(
-      _SUMMARY_WRITER_INIT_COLLECTION_NAME,
-      gen_summary_ops.create_summary_file_writer(resource, logdir, max_queue,
-                                                 flush_secs, filename_suffix))
-  return SummaryWriter(resource)
+  """Creates a summary file writer in the current context.
+
+  Args:
+    logdir: a string, or None. If a string, creates a summary file writer
+     which writes to the directory named by the string. If None, returns
+     a mock object which acts like a summary writer but does nothing,
+     useful to use as a context manager.
+    max_queue: the largest number of summaries to keep in a queue; will
+     flush once the queue gets bigger than this.
+    flush_secs: the largest interval (in seconds) between flushes.
+    filename_suffix: optional suffix for the event file name.
+    name: name for the summary writer.
+
+  Returns:
+    Either a summary writer or an empty object which can be used as a
+    summary writer.
+  """
+  if logdir is None:
+    return SummaryWriter(None)
+  with ops.device("cpu:0"):
+    if max_queue is None:
+      max_queue = constant_op.constant(10)
+    if flush_secs is None:
+      flush_secs = constant_op.constant(120)
+    if filename_suffix is None:
+      filename_suffix = constant_op.constant("")
+    resource = gen_summary_ops.summary_writer(shared_name=name)
+    # TODO(apassos) ensure the initialization op runs when in graph mode;
+    # consider calling session.run here.
+    ops.add_to_collection(
+        _SUMMARY_WRITER_INIT_COLLECTION_NAME,
+        gen_summary_ops.create_summary_file_writer(resource, logdir, max_queue,
+                                                   flush_secs, filename_suffix))
+    return SummaryWriter(resource)
 
 
 def _nothing():
@@ -168,6 +194,8 @@ def summary_writer_function(name, tensor, function, family=None):
       with ops.control_dependencies([function(tag, scope)]):
         return constant_op.constant(True)
 
+  if context.context().summary_writer_resource is None:
+    return control_flow_ops.no_op()
   with ops.device("cpu:0"):
     op = utils.smart_cond(
         should_record_summaries(), record, _nothing, name="")
diff --git a/tensorflow/python/training/training_util.py b/tensorflow/python/training/training_util.py
index bdd4ca734e..89a9e12932 100644
--- a/tensorflow/python/training/training_util.py
+++ b/tensorflow/python/training/training_util.py
@@ -119,13 +119,24 @@ def create_global_step(graph=None):
     raise ValueError('"global_step" already exists.')
   # Create in proper graph and base name_scope.
   with graph.as_default() as g, g.name_scope(None):
+    if context.in_eager_mode():
+      with ops.device('cpu:0'):
+        return variable_scope.get_variable(
+            ops.GraphKeys.GLOBAL_STEP,
+            shape=[],
+            dtype=dtypes.int64,
+            initializer=init_ops.zeros_initializer(),
+            trainable=False,
+            collections=[ops.GraphKeys.GLOBAL_VARIABLES,
+                         ops.GraphKeys.GLOBAL_STEP])
     return variable_scope.get_variable(
         ops.GraphKeys.GLOBAL_STEP,
         shape=[],
         dtype=dtypes.int64,
         initializer=init_ops.zeros_initializer(),
         trainable=False,
-        collections=[ops.GraphKeys.GLOBAL_VARIABLES, ops.GraphKeys.GLOBAL_STEP])
+        collections=[ops.GraphKeys.GLOBAL_VARIABLES,
+                     ops.GraphKeys.GLOBAL_STEP])
 
 
 def get_or_create_global_step(graph=None):
-- 
GitLab


From 873ef2ca375f2f6a4fed55e0bcd9a6fba6e8d545 Mon Sep 17 00:00:00 2001
From: Oleg Zabluda <ozabluda@gmail.com>
Date: Fri, 27 Oct 2017 10:56:18 -0700
Subject: [PATCH 874/909] Fix documentation error in tf.size() - output type

---
 tensorflow/python/ops/array_ops.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/tensorflow/python/ops/array_ops.py b/tensorflow/python/ops/array_ops.py
index 857cd09d56..712ae8620e 100644
--- a/tensorflow/python/ops/array_ops.py
+++ b/tensorflow/python/ops/array_ops.py
@@ -322,11 +322,11 @@ def size(input, name=None, out_type=dtypes.int32):
   Args:
     input: A `Tensor` or `SparseTensor`.
     name: A name for the operation (optional).
-    out_type: (Optional) The specified output type of the operation
-      (`int32` or `int64`). Defaults to tf.int32.
+    out_type: (Optional) The specified non-quantized numeric output type
+      of the operation. Defaults to `tf.int32`.
 
   Returns:
-    A `Tensor` of type `out_type`. Defaults to tf.int32.
+    A `Tensor` of type `out_type`. Defaults to `tf.int32`.
     
   @compatibility(numpy)
   Equivalent to np.size()
@@ -343,11 +343,11 @@ def size_internal(input, name=None, optimize=True, out_type=dtypes.int32):
     input: A `Tensor` or `SparseTensor`.
     name: A name for the operation (optional).
     optimize: if true, encode the size as a constant when possible.
-    out_type: (Optional) The specified output type of the operation
-      (`int32` or `int64`). Defaults to tf.int32.
+    out_type: (Optional) The specified non-quantized numeric output type
+      of the operation. Defaults to `tf.int32`.
 
   Returns:
-    A `Tensor` of type `out_type`.
+    A `Tensor` of type `out_type`. Defaults to `tf.int32`.
   """
   with ops.name_scope(name, "Size", [input]) as name:
     if isinstance(input, (sparse_tensor.SparseTensor,
-- 
GitLab


From 97484a4d90e6a1bbd8784bffd3b3af41d777d6bd Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 27 Oct 2017 10:51:31 -0700
Subject: [PATCH 875/909] Update ops-related pbtxt files.

PiperOrigin-RevId: 173690751
---
 .../core/ops/compat/ops_history.v1.pbtxt      | 48 +++++++++
 tensorflow/core/ops/ops.pbtxt                 | 98 ++++++++++---------
 2 files changed, 99 insertions(+), 47 deletions(-)

diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index a691065075..4d00694707 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -20723,6 +20723,54 @@ op {
     type: "type"
   }
 }
+op {
+  name: "ParallelInterleaveDataset"
+  input_arg {
+    name: "input_dataset"
+    type: DT_VARIANT
+  }
+  input_arg {
+    name: "other_arguments"
+    type_list_attr: "Targuments"
+  }
+  input_arg {
+    name: "cycle_length"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "block_length"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "sloppy"
+    type: DT_BOOL
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "f"
+    type: "func"
+  }
+  attr {
+    name: "Targuments"
+    type: "list(type)"
+    has_minimum: true
+  }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
+  }
+}
 op {
   name: "ParallelMapDataset"
   input_arg {
diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index 0a590fef00..e43ee0d986 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -16181,6 +16181,57 @@ op {
   summary: "Interleave the values from the `data` tensors into a single tensor."
   description: "Builds a merged tensor such that\n\n```python\n    merged[indices[m][i, ..., j], ...] = data[m][i, ..., j, ...]\n```\n\nFor example, if each `indices[m]` is scalar or vector, we have\n\n```python\n    # Scalar indices:\n    merged[indices[m], ...] = data[m][...]\n\n    # Vector indices:\n    merged[indices[m][i], ...] = data[m][i, ...]\n```\n\nEach `data[i].shape` must start with the corresponding `indices[i].shape`,\nand the rest of `data[i].shape` must be constant w.r.t. `i`.  That is, we\nmust have `data[i].shape = indices[i].shape + constant`.  In terms of this\n`constant`, the output shape is\n\n    merged.shape = [max(indices)] + constant\n\nValues may be merged in parallel, so if an index appears in both `indices[m][i]`\nand `indices[n][j]`, the result may be invalid. This differs from the normal\nDynamicStitch operator that defines the behavior in that case.\n\nFor example:\n\n```python\n    indices[0] = 6\n    indices[1] = [4, 1]\n    indices[2] = [[5, 2], [0, 3]]\n    data[0] = [61, 62]\n    data[1] = [[41, 42], [11, 12]]\n    data[2] = [[[51, 52], [21, 22]], [[1, 2], [31, 32]]]\n    merged = [[1, 2], [11, 12], [21, 22], [31, 32], [41, 42],\n              [51, 52], [61, 62]]\n```\n\nThis method can be used to merge partitions created by `dynamic_partition`\nas illustrated on the following example:\n\n```python\n    # Apply function (increments x_i) on elements for which a certain condition\n    # apply (x_i != -1 in this example).\n    x=tf.constant([0.1, -1., 5.2, 4.3, -1., 7.4])\n    condition_mask=tf.not_equal(x,tf.constant(-1.))\n    partitioned_data = tf.dynamic_partition(\n        x, tf.cast(condition_mask, tf.int32) , 2)\n    partitioned_data[1] = partitioned_data[1] + 1.0\n    condition_indices = tf.dynamic_partition(\n        tf.range(tf.shape(x)[0]), tf.cast(condition_mask, tf.int32) , 2)\n    x = tf.dynamic_stitch(condition_indices, partitioned_data)\n    # Here x=[1.1, -1., 6.2, 5.3, -1, 8.4], the -1. values remain\n    # unchanged.\n```\n\n<div style=\"width:70%; margin:auto; margin-bottom:10px; margin-top:20px;\">\n<img style=\"width:100%\" src=\"https://www.tensorflow.org/images/DynamicStitch.png\" alt>\n</div>"
 }
+op {
+  name: "ParallelInterleaveDataset"
+  input_arg {
+    name: "input_dataset"
+    type: DT_VARIANT
+  }
+  input_arg {
+    name: "other_arguments"
+    type_list_attr: "Targuments"
+  }
+  input_arg {
+    name: "cycle_length"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "block_length"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "sloppy"
+    type: DT_BOOL
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "f"
+    type: "func"
+    description: "A function mapping elements of `input_dataset`, concatenated with\n`other_arguments`, to a Dataset variant that contains elements matching\n`output_types` and `output_shapes`."
+  }
+  attr {
+    name: "Targuments"
+    type: "list(type)"
+    has_minimum: true
+  }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
+  }
+  summary: "Creates a dataset that applies `f` to the outputs of `input_dataset`."
+  description: "The resulting dataset is similar to the `InterleaveDataset`, with the exception\nthat if retrieving the next value from a dataset would cause the requester to\nblock, it will skip that input dataset. This dataset is especially useful\nwhen loading data from a variable-latency datastores (e.g. HDFS, GCS), as it\nallows the training step to proceed so long as some data is available.\n\n!! WARNING !! This dataset is not deterministic!"
+}
 op {
   name: "ParallelMapDataset"
   input_arg {
@@ -25772,53 +25823,6 @@ op {
   summary: "Return a slice from \'input\'."
   description: "The output tensor is a tensor with dimensions described by \'size\'\nwhose values are extracted from \'input\' starting at the offsets in\n\'begin\'.\n\n*Requirements*:\n  0 <= begin[i] <= begin[i] + size[i] <= Di  for i in [0, n)"
 }
-op {
-  name: "SloppyInterleaveDataset"
-  input_arg {
-    name: "input_dataset"
-    type: DT_VARIANT
-  }
-  input_arg {
-    name: "other_arguments"
-    type_list_attr: "Targuments"
-  }
-  input_arg {
-    name: "cycle_length"
-    type: DT_INT64
-  }
-  input_arg {
-    name: "block_length"
-    type: DT_INT64
-  }
-  output_arg {
-    name: "handle"
-    type: DT_VARIANT
-  }
-  attr {
-    name: "f"
-    type: "func"
-    description: "A function mapping elements of `input_dataset`, concatenated with\n`other_arguments`, to a Dataset variant that contains elements matching\n`output_types` and `output_shapes`."
-  }
-  attr {
-    name: "Targuments"
-    type: "list(type)"
-    has_minimum: true
-  }
-  attr {
-    name: "output_types"
-    type: "list(type)"
-    has_minimum: true
-    minimum: 1
-  }
-  attr {
-    name: "output_shapes"
-    type: "list(shape)"
-    has_minimum: true
-    minimum: 1
-  }
-  summary: "Creates a dataset that applies `f` to the outputs of `input_dataset`."
-  description: "The resulting dataset is similar to the `InterleaveDataset`, with the exception\nthat if retrieving the next value from a dataset would cause the requester to\nblock, it will skip that input dataset. This dataset is especially useful\nwhen loading data from a variable-latency datastores (e.g. HDFS, GCS), as it\nallows the training step to proceed so long as some data is available.\n\n!! WARNING !! This dataset is not deterministic!"
-}
 op {
   name: "Softmax"
   input_arg {
-- 
GitLab


From 32bcf46f13a1ad158f366523289d06e6d00642b6 Mon Sep 17 00:00:00 2001
From: Mustafa Ispir <ispir@google.com>
Date: Fri, 27 Oct 2017 11:37:28 -0700
Subject: [PATCH 876/909] internal

PiperOrigin-RevId: 173697389
---
 tensorflow/contrib/kernel_methods/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/contrib/kernel_methods/BUILD b/tensorflow/contrib/kernel_methods/BUILD
index ae1402b0e6..a2f320ab11 100644
--- a/tensorflow/contrib/kernel_methods/BUILD
+++ b/tensorflow/contrib/kernel_methods/BUILD
@@ -64,6 +64,7 @@ py_test(
     name = "kernel_estimators_test",
     srcs = ["python/kernel_estimators_test.py"],
     srcs_version = "PY2AND3",
+    tags = ["notsan"],
     deps = [
         ":kernel_methods",
         "//tensorflow/contrib/layers:layers_py",
-- 
GitLab


From 73155f56a30c75eb54d95ab0d51ab8b5c8fb02c9 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 27 Oct 2017 12:12:31 -0700
Subject: [PATCH 877/909] [TF:XLA] Small code cleanup. Re-alphabetized.

PiperOrigin-RevId: 173702336
---
 tensorflow/compiler/xla/service/hlo_opcode.cc | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_opcode.cc b/tensorflow/compiler/xla/service/hlo_opcode.cc
index 2f2263f70d..d94c4da5ea 100644
--- a/tensorflow/compiler/xla/service/hlo_opcode.cc
+++ b/tensorflow/compiler/xla/service/hlo_opcode.cc
@@ -33,6 +33,8 @@ string HloOpcodeString(HloOpcode opcode) {
       return "abs";
     case HloOpcode::kAdd:
       return "add";
+    case HloOpcode::kAnd:
+      return "and";
     case HloOpcode::kAtan2:
       return "atan2";
     case HloOpcode::kBatchNormTraining:
@@ -103,12 +105,6 @@ string HloOpcodeString(HloOpcode opcode) {
       return "less-than-or-equal-to";
     case HloOpcode::kLog:
       return "log";
-    case HloOpcode::kAnd:
-      return "and";
-    case HloOpcode::kOr:
-      return "or";
-    case HloOpcode::kNot:
-      return "not";
     case HloOpcode::kLt:
       return "less-than";
     case HloOpcode::kMap:
@@ -123,6 +119,10 @@ string HloOpcodeString(HloOpcode opcode) {
       return "not-equal-to";
     case HloOpcode::kNegate:
       return "negate";
+    case HloOpcode::kNot:
+      return "not";
+    case HloOpcode::kOr:
+      return "or";
     case HloOpcode::kOutfeed:
       return "outfeed";
     case HloOpcode::kPad:
@@ -190,6 +190,7 @@ StatusOr<HloOpcode> StringToHloOpcode(const string& opcode_name) {
   static auto* opcode_map = new tensorflow::gtl::FlatMap<string, HloOpcode>(
       {{"abs", HloOpcode::kAbs},
        {"add", HloOpcode::kAdd},
+       {"and", HloOpcode::kAnd},
        {"batch-norm-training", HloOpcode::kBatchNormTraining},
        {"batch-norm-inference", HloOpcode::kBatchNormInference},
        {"batch-norm-grad", HloOpcode::kBatchNormGrad},
@@ -222,16 +223,15 @@ StatusOr<HloOpcode> StringToHloOpcode(const string& opcode_name) {
        {"is-finite", HloOpcode::kIsFinite},
        {"less-than-or-equal-to", HloOpcode::kLe},
        {"log", HloOpcode::kLog},
-       {"and", HloOpcode::kAnd},
-       {"or", HloOpcode::kOr},
-       {"not", HloOpcode::kNot},
        {"less-than", HloOpcode::kLt},
        {"map", HloOpcode::kMap},
        {"maximum", HloOpcode::kMaximum},
        {"minimum", HloOpcode::kMinimum},
        {"multiply", HloOpcode::kMultiply},
+       {"not", HloOpcode::kNot},
        {"not-equal-to", HloOpcode::kNe},
        {"negate", HloOpcode::kNegate},
+       {"or", HloOpcode::kOr},
        {"outfeed", HloOpcode::kOutfeed},
        {"pad", HloOpcode::kPad},
        {"parameter", HloOpcode::kParameter},
-- 
GitLab


From 3d39b32b9a7833807dad037c3f57c818e9251f85 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 27 Oct 2017 12:15:56 -0700
Subject: [PATCH 878/909] Fix a tfprof bug. Throws an error when the flops
 cannot be calculated.

PiperOrigin-RevId: 173702740
---
 tensorflow/python/profiler/internal/flops_registry.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/python/profiler/internal/flops_registry.py b/tensorflow/python/profiler/internal/flops_registry.py
index e143501049..147711b1d9 100644
--- a/tensorflow/python/profiler/internal/flops_registry.py
+++ b/tensorflow/python/profiler/internal/flops_registry.py
@@ -373,6 +373,7 @@ def _max_pool_grad_flops(graph, node):
   kernel_area = _list_product(kernel_shape)
   orig_out_shape = graph_util.tensor_shape_from_node_def_name(graph,
                                                               node.input[1])
+  orig_out_shape.assert_is_fully_defined()
   max_pool_ops = kernel_area * orig_out_shape.num_elements()
   return ops.OpStats("flops", max_pool_ops + orig_out_shape.num_elements())
 
-- 
GitLab


From 9158f974a346b4fae89044d8724eb052d466112b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 27 Oct 2017 12:16:44 -0700
Subject: [PATCH 879/909] Use tf.app.run in gcs_smoke, so that the flags are
 explicitly parsed, instead of parsed when first accessed.

PiperOrigin-RevId: 173702828
---
 tensorflow/tools/gcs_test/python/gcs_smoke.py | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/tensorflow/tools/gcs_test/python/gcs_smoke.py b/tensorflow/tools/gcs_test/python/gcs_smoke.py
index 9882f75a8a..ad4cb17ae1 100644
--- a/tensorflow/tools/gcs_test/python/gcs_smoke.py
+++ b/tensorflow/tools/gcs_test/python/gcs_smoke.py
@@ -35,6 +35,7 @@ flags.DEFINE_integer("num_examples", 10, "Number of examples to generate")
 
 FLAGS = flags.FLAGS
 
+
 def create_examples(num_examples, input_mean):
   """Create ExampleProto's containing data."""
   ids = np.arange(num_examples).reshape([num_examples, 1])
@@ -49,6 +50,7 @@ def create_examples(num_examples, input_mean):
     examples.append(ex)
   return examples
 
+
 def create_dir_test():
   """Verifies file_io directory handling methods."""
 
@@ -122,6 +124,7 @@ def create_dir_test():
   print("Deleted directory recursively %s in %s milliseconds" % (
       dir_name, elapsed_ms))
 
+
 def create_object_test():
   """Verifies file_io's object manipulation methods ."""
   starttime_ms = int(round(time.time() * 1000))
@@ -142,7 +145,8 @@ def create_object_test():
     print("Creating file %s." % file_name)
     file_io.write_string_to_file(file_name, "test file creation.")
   elapsed_ms = int(round(time.time() * 1000)) - starttime_ms
-  print("Created %d files in %s milliseconds" % (len(files_to_create), elapsed_ms))
+  print("Created %d files in %s milliseconds" % (
+      len(files_to_create), elapsed_ms))
 
   # Listing files of pattern1.
   list_files_pattern = "%s/test_file*.txt" % dir_name
@@ -185,7 +189,9 @@ def create_object_test():
   file_io.delete_recursively(dir_name)
 
 
-if __name__ == "__main__":
+def main(argv):
+  del argv  # Unused.
+
   # Sanity check on the GCS bucket URL.
   if not FLAGS.gcs_bucket_url or not FLAGS.gcs_bucket_url.startswith("gs://"):
     print("ERROR: Invalid GCS bucket URL: \"%s\"" % FLAGS.gcs_bucket_url)
@@ -210,7 +216,7 @@ if __name__ == "__main__":
   # tf_record_iterator works.
   record_iter = tf.python_io.tf_record_iterator(input_path)
   read_count = 0
-  for r in record_iter:
+  for _ in record_iter:
     read_count += 1
   print("Read %d records using tf_record_iterator" % read_count)
 
@@ -222,7 +228,7 @@ if __name__ == "__main__":
 
   # Verify that running the read op in a session works.
   print("\n=== Testing TFRecordReader.read op in a session... ===")
-  with tf.Graph().as_default() as g:
+  with tf.Graph().as_default():
     filename_queue = tf.train.string_input_producer([input_path], num_epochs=1)
     reader = tf.TFRecordReader()
     _, serialized_example = reader.read(filename_queue)
@@ -249,3 +255,7 @@ if __name__ == "__main__":
 
   create_dir_test()
   create_object_test()
+
+
+if __name__ == "__main__":
+  tf.app.run(main)
-- 
GitLab


From d7cffe9c03384189ec7509fc24d1a76f0f1241b6 Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Fri, 27 Oct 2017 12:19:12 -0700
Subject: [PATCH 880/909] Adds save and restore methods to tfe.Network

Save just saves the variables to a checkpoint. Restore either restores immediately or defers the restoration to variable creation time with a custom getter.

PiperOrigin-RevId: 173703075
---
 tensorflow/contrib/eager/python/network.py    | 467 +++++++++++++++++-
 .../contrib/eager/python/network_test.py      | 283 +++++++++++
 2 files changed, 728 insertions(+), 22 deletions(-)

diff --git a/tensorflow/contrib/eager/python/network.py b/tensorflow/contrib/eager/python/network.py
index 025d447455..5b53a597f2 100644
--- a/tensorflow/contrib/eager/python/network.py
+++ b/tensorflow/contrib/eager/python/network.py
@@ -19,11 +19,17 @@ from __future__ import division
 from __future__ import print_function
 
 import collections
+import os
 import weakref
 
+from tensorflow.python.eager import context
 from tensorflow.python.estimator import util as estimator_util
+from tensorflow.python.framework import ops
 from tensorflow.python.layers import base
 from tensorflow.python.ops import variable_scope
+from tensorflow.python.training import checkpoint_utils
+from tensorflow.python.training import saver as saver_lib
+from tensorflow.python.training import training_util
 
 # pylint: disable=protected-access
 # Explanation for protected-access disable: Network has lots of same-class and
@@ -31,6 +37,151 @@ from tensorflow.python.ops import variable_scope
 # functions in base.py which should be reused.
 
 
+_DeferredRestoration = collections.namedtuple(
+
+    "_DeferredRestoration",
+    [
+        # The map_func to use (either user-specified or the default).
+        "map_func",
+        # Boolean, True if the user specified an explicit map_func, for error
+        # messages.
+        "map_func_is_user",
+        # A mapping from checkpoint names to initial values of not-yet-created
+        # variables which should be restored. These values come from parsing a
+        # checkpoint.
+        "checkpointed_variables_to_restore",
+        # A mapping from checkpoint name to variable objects of variables which
+        # have already been restored, for error checking.
+        "restored_variables",
+        # The session to restore with (if in graph mode).
+        "session",
+        # Names of the Network where the restore was requested, for error
+        # messages.
+        "network_name",
+        "network_scope_name"
+    ])
+
+
+def _default_naming_conflict_error_message(
+    mapped_name, first_variable, second_variable,
+    network_name, network_scope_name):
+  return (
+      ("The default checkpoint variable name mapping strategy for Network "
+       "'%s' resulted in a naming conflict. We attempted to strip off the "
+       "variable prefix for the Network ('%s'), but this resulted in two "
+       "variables named '%s' (originally '%s' and '%s'). This should only "
+       "happen when using variable sharing (i.e. the Network contains Networks "
+       "or Layers which were first added to another Network, and therefore "
+       "have that Network's variable prefix). One solution is to pass "
+       "`map_func=lambda n: n` to Network.save and Network.restore to use "
+       "fully qualified variable names in the checkpoint, although this will "
+       "require that the variable prefix of the Network being restored into "
+       "is also '%s'. You may alternatively write an arbitrary mapping.")
+      % (
+          network_name, network_scope_name, mapped_name,
+          first_variable._shared_name,
+          second_variable._shared_name, network_scope_name
+      ))
+
+
+def _restore_custom_map_func_error_message(
+    mapped_name, first_variable, second_variable,
+    network_name, network_scope_name):
+  return (
+      ("The map_func passed to Network.restore for the Network '%s' "
+       "resulted in two variables named '%s' (originally '%s' and '%s'). Since "
+       "this is also an error on Network.save, this Network was "
+       "probably not saved with this map_func. Note that map_func "
+       "always maps from full variable names to checkpoint names; "
+       "there is no need to specify an inverse mapping.\n\n"
+       "Try stripping less from the variable names, or renaming parts "
+       "of the Network. For reference, variables created by sub-Layers "
+       "of this Network are prefixed with '%s', but if they are "
+       "re-used after being added to another Network they will have "
+       "that Network's full variable prefix instead.") % (
+           network_name, mapped_name,
+           first_variable._shared_name,
+           second_variable._shared_name,
+           network_scope_name))
+
+
+def _make_custom_getter_for_deferred_restorations():
+  """Returns a custom getter which searches `deferred_restorations`.
+
+  Returns: A tuple of (_custom_getter, deferred_restorations)
+    _custom_getter: The getter which should be added to variable_scopes where
+      variables will be created.
+    deferred_restorations: A list for _DeferredRestoration objects. Typically
+      empty when the getter is set, and expanded as deferred restorations are
+      requested. All new deferred restorations should be appended to the end of
+      the list, where they will have priority over older deferred restorations.
+  """
+  deferred_restorations = []
+
+  def _custom_getter(getter, name, shape=None, dtype=None,
+                     initializer=None,
+                     *args, **kwargs):
+    """A custom getter which processes deferred restorations."""
+    # Iterate over restorations, newest first (newer restorations will take
+    # precedence over older restorations, just like with immediate restorations
+    # into existing variables).
+    delayed_restoration = None
+    found_value = False
+    value_to_restore = None
+    for delayed_restoration in reversed(
+        deferred_restorations):
+      checkpoint_name = delayed_restoration.map_func(name)
+      if (checkpoint_name
+          in delayed_restoration.checkpointed_variables_to_restore):
+        found_value = True
+        value_to_restore = (
+            delayed_restoration.checkpointed_variables_to_restore[
+                checkpoint_name])
+      if found_value:
+        break
+    # value_to_restore may be False because this variable is not in any
+    # checkpoint we are restoring, or None because we have explicitly set it to
+    # None when it was previously fetched. In either case, we don't need to
+    # set an initializer.
+    if found_value and value_to_restore is not None:
+      initializer = value_to_restore
+      shape = None
+    variable = getter(name, shape=shape, dtype=dtype, initializer=initializer,
+                      *args, **kwargs)
+    if found_value and value_to_restore is not None:
+      # Mark as already restored from this checkpoint.
+      delayed_restoration.checkpointed_variables_to_restore[
+          checkpoint_name] = None
+      if context.in_graph_mode():
+        delayed_restoration.session.run(variable.initializer)
+    if found_value:
+      # Error checking should run even if we've already restored a value.
+      if delayed_restoration.restored_variables.setdefault(
+          checkpoint_name, variable) is not variable:
+        # Naming conflict. We've tried to initialize two variables with the
+        # same value from the checkpoint.
+        if delayed_restoration.map_func_is_user:
+          raise ValueError(
+              _restore_custom_map_func_error_message(
+                  mapped_name=checkpoint_name,
+                  first_variable=delayed_restoration.restored_variables[
+                      checkpoint_name],
+                  second_variable=variable,
+                  network_name=delayed_restoration.network_name,
+                  network_scope_name=delayed_restoration.network_scope_name))
+        else:
+          raise ValueError(
+              _default_naming_conflict_error_message(
+                  mapped_name=checkpoint_name,
+                  first_variable=delayed_restoration.restored_variables[
+                      checkpoint_name],
+                  second_variable=variable,
+                  network_name=delayed_restoration.network_name,
+                  network_scope_name=delayed_restoration.network_scope_name))
+    return variable
+  return _custom_getter, deferred_restorations
+
+
 class Network(base.Layer):
   """Represents the composition of a set of Layers.
 
@@ -41,7 +192,6 @@ class Network(base.Layer):
   - Convert inputs to __call__ to tensors.
   - Prevent variables from being created after the first __call__?
     (Think about restoring from a checkpoint).
-  - Save & restore
   """
 
   def __init__(self, name=None):
@@ -60,6 +210,8 @@ class Network(base.Layer):
     self._owned_layers = {}
     # The scope to use if we end up without a parent.
     self._default_parent_variable_scope = variable_scope.get_variable_scope()
+    self._custom_getter, self._deferred_restorations = (
+        _make_custom_getter_for_deferred_restorations())
 
   def _init_set_name(self, name):
     # Anonymous Networks (name=None) defer setting a final name until they are
@@ -87,7 +239,8 @@ class Network(base.Layer):
         avoid_names = None
       self._name, self._base_name = self._make_unique_name(
           name_uid_map=name_uid_map, avoid_names=avoid_names)
-    if self._first_parent is None or self._first_parent() is None:
+    if self._first_parent is None or (self._first_parent  # False = no parent
+                                      and self._first_parent() is None):
       # Save a pointer to the parent Network so that we can later check that the
       # scope name we get is correct.
       if not parent_network:
@@ -151,26 +304,32 @@ class Network(base.Layer):
                 "of Networks in which they were first created). To set "
                 "options, try `with tf.variable_scope(''):`. If this "
                 "limitation bothers you, please file a feature request.")
-      for non_network_constituent in self._non_network_sublayers:
-        if non_network_constituent._scope is None:
-          if non_network_constituent._first_parent is None:
-            constituent_first_parent = None
-          else:
-            constituent_first_parent = non_network_constituent._first_parent()
-          if constituent_first_parent:
-            constituent_first_parent._set_scope()
-            parent_scope = constituent_first_parent._scope
-          else:
-            parent_scope = (
-                non_network_constituent._default_parent_variable_scope)
-          with variable_scope.variable_scope(parent_scope):
-            # Horrid hack to make Layer variable names which are direct
-            # sub-layers of Networks conform to the Network variable naming
-            # conventions.
-            with variable_scope.variable_scope(
-                None, use_resource=True,
-                default_name=non_network_constituent.name) as sub_scope:
-              non_network_constituent._scope = sub_scope
+      for non_network_sublayer in self._non_network_sublayers:
+        self._set_scope_for_nonnetwork_sublayer(non_network_sublayer)
+
+  def _set_scope_for_nonnetwork_sublayer(self, sublayer):
+    if sublayer._scope is None:
+      if sublayer._first_parent is None:
+        constituent_first_parent = None
+      else:
+        constituent_first_parent = sublayer._first_parent()
+      if constituent_first_parent:
+        constituent_first_parent._set_scope()
+        parent_scope = constituent_first_parent._scope
+      else:
+        self._finalize_name(False)
+        raise ValueError(
+            ("The parent of a Layer added to Network %s was garbage collected "
+             "before the Layer was built. If this limitation bothers you "
+             "please, file a feature request.") % (self.name,))
+      with variable_scope.variable_scope(parent_scope):
+        # Horrid hack to make Layer variable names which are direct
+        # sub-layers of Networks conform to the Network variable naming
+        # conventions.
+        with variable_scope.variable_scope(
+            None, use_resource=True,
+            default_name=sublayer.name) as sub_scope:
+          sublayer._scope = sub_scope
 
   @base.Layer.name.getter
   def name(self):
@@ -327,6 +486,270 @@ class Network(base.Layer):
         "at https://github.com/tensorflow/tensorflow/issues/new if this is "
         "important to you")
 
+  def _strip_variable_prefix(self, original_variable_name):
+    """The default map_func for saving or restoring variables.
+
+    Strips the variable prefix for the Network on which save/restore was called,
+    and leaves other variable names fully qualified in the checkpoint.
+
+    Args:
+      original_variable_name: The _shared_name of the variable (no :0
+        suffix) to map.
+    Returns:
+      The checkpoint name of the variable.
+    """
+    scope_name_with_slash = self.scope_name + "/"
+    if original_variable_name.startswith(scope_name_with_slash):
+      return original_variable_name[len(scope_name_with_slash):]
+    else:
+      return original_variable_name
+
+  def save(self, save_path, global_step=None, map_func=None):
+    """Save variables from the Network to a checkpoint.
+
+    Args:
+      save_path: Either a checkpoint prefix or the name of a directory to save
+        the checkpoint in (in which case the checkpoint will be named based on
+        the Network name).
+      global_step: The global step to use when naming the checkpoint. If None
+        (default), we will first try to get the default global step. If that
+        fails because no default global step exists, then the checkpoint is
+        created without a global step suffix.
+      map_func: A function mapping fully qualified variable names
+        (e.g. 'my_network_1/dense_1/kernel') to names in the checkpoint. By
+        default (if `map_func=None`), the variable prefix for the network being
+        restored (`Network.scope_name + '/'`, e.g. 'my_network_1/') is stripped
+        and all other variable names (shared with other Networks) are left
+        unchanged.
+    Returns:
+      The checkpoint prefix for the saved checkpoint, which may be passed to
+      `Network.restore`.
+    Raises:
+      ValueError: If the Network has not yet been called, or if map_func results
+        in a name collision.
+    """
+    if not self.built:
+      raise ValueError(
+          "Attempt to save the Network before it was first called. This means "
+          "variables have not yet been created, so there is nothing to save.")
+    self._set_scope()  # scope_name should be available to map_funcs
+    if global_step is None:
+      global_step = training_util.get_global_step()
+    if os.path.isdir(save_path):
+      # If we were passed a directory, default to naming based on the Network
+      # name.
+      save_path = os.path.join(save_path, self.name)
+    user_map_func = map_func
+    if map_func is None:
+      map_func = self._strip_variable_prefix
+    variable_map = {}
+    for variable in self.variables:
+      mapped_name = map_func(variable._shared_name)
+      if variable_map.setdefault(mapped_name, variable) is not variable:
+        if user_map_func is None:
+          # Instead of erroring out, we could just re-try and silently use the
+          # full variable names in the checkpoint. This could be odd for deeply
+          # nested sub-Networks (since the full prefix from the nesting would
+          # get added), so for now we'll let the user deal with this case.
+          raise ValueError(_default_naming_conflict_error_message(
+              mapped_name=mapped_name,
+              first_variable=variable_map[mapped_name],
+              second_variable=variable,
+              network_name=self.name,
+              network_scope_name=self.scope_name))
+        else:
+          # The user passed their own problematic map_func.
+          raise ValueError(
+              ("The map_func passed to Network.save for the Network '%s' "
+               "resulted in two variables named '%s' ('%s' and '%s'). Try "
+               "stripping less from the variable names, or renaming parts of "
+               "the Network. For reference, variables created by sub-Layers of "
+               "this Network are prefixed with '%s', but if they are re-used "
+               "after being added to another Network, they will have that "
+               "Network's full variable prefix instead.") % (
+                   self.name, mapped_name,
+                   variable_map[mapped_name]._shared_name,
+                   variable._shared_name,
+                   self.scope_name))
+    if context.in_eager_mode():
+      sess = None
+    else:
+      sess = ops.get_default_session()
+    return saver_lib.Saver(variable_map).save(
+        sess=sess, save_path=save_path, write_meta_graph=False,
+        global_step=global_step)
+
+  def _restore_existing_variables(self, save_path, map_func, user_map_func):
+    """Use a standard Saver to restore existing variables from a checkpoint.
+
+    Args:
+      save_path: The checkpoint prefix or directory to read from.
+      map_func: The function to use when mapping from variable names to
+        checkpoint names.
+      user_map_func: The original map_func passed by the user, for error
+        checking.
+    Returns:
+      A dictionary mapping from checkpoint names to variable objects which have
+      been restored (for bookkeeping to avoid deferred restorations on these
+      variables).
+    Raises:
+      ValueError: If there is a name collision.
+    """
+    existing_variables_by_checkpoint_name = {}
+    for variable in self.variables:
+      checkpoint_name = map_func(variable._shared_name)
+      if existing_variables_by_checkpoint_name.setdefault(
+          checkpoint_name, variable) is not variable:
+        if user_map_func is None:
+          raise ValueError(_default_naming_conflict_error_message(
+              mapped_name=checkpoint_name,
+              first_variable=existing_variables_by_checkpoint_name[
+                  checkpoint_name],
+              second_variable=variable,
+              network_name=self.name,
+              network_scope_name=self.scope_name))
+        else:
+          raise ValueError(_restore_custom_map_func_error_message(
+              mapped_name=checkpoint_name,
+              first_variable=existing_variables_by_checkpoint_name[
+                  checkpoint_name],
+              second_variable=variable,
+              network_name=self.name,
+              network_scope_name=self.scope_name))
+    if existing_variables_by_checkpoint_name:
+      if context.in_eager_mode():
+        sess = None
+      else:
+        sess = ops.get_default_session()
+      saver_lib.Saver(var_list=existing_variables_by_checkpoint_name).restore(
+          sess=sess, save_path=save_path)
+    return existing_variables_by_checkpoint_name
+
+  def _set_restore_on_create(self, save_path, map_func, user_map_func,
+                             existing_variables_by_checkpoint_name):
+    """If necessary, request deferred restorations of variables."""
+    checkpoint_reader = checkpoint_utils.load_checkpoint(save_path)
+    checkpointed_variables_to_restore = {}
+    for checkpoint_name, _ in checkpoint_utils.list_variables(save_path):
+      if checkpoint_name in existing_variables_by_checkpoint_name:
+        # This variable was already created and restored.
+        continue
+      # Save the variable for later restoration in a custom getter.
+      checkpointed_variables_to_restore[checkpoint_name] = (
+          checkpoint_reader.get_tensor(checkpoint_name))
+    # Only set a deferred restoration if there are checkpoint variables which
+    # have not been assigned to existing variables. Note that this loses out on
+    # some opportunity for error checking, but avoids creating
+    # _DeferredRestoration objects once a Network has been built (so that
+    # restoring in a loop does not take increasing amounts of memory).
+    if checkpointed_variables_to_restore:
+      if context.in_eager_mode():
+        sess = None
+      else:
+        sess = ops.get_default_session()
+      # We need a name for error messages. If we haven't been added to another
+      # Network yet, we're top-level.
+      self._finalize_name(False)
+      self._set_scope()
+      # Save a record of this restoration for use in the custom getter.
+      deferred_restoration = _DeferredRestoration(
+          map_func=map_func,
+          map_func_is_user=(user_map_func is not None),
+          checkpointed_variables_to_restore=checkpointed_variables_to_restore,
+          restored_variables={},
+          session=sess,
+          network_name=self.name,
+          network_scope_name=self.scope_name)
+      self._deferred_restorations.append(deferred_restoration)
+      # Add the deferred registration to non-Network children, and request that
+      # Networks propagate the request to their children.
+      self._add_deferred_restoration(deferred_restoration)
+
+  def _add_deferred_restoration(self, deferred_restoration):
+    """Add a deferred restoration to this Network and all children.
+
+    Restorations which are requested later have higher priority, and the highest
+    priority matching restoration is applied to a variable when it is created.
+
+    Args:
+      deferred_restoration: A _DeferredRestoration object.
+    """
+    # Networks don't create variables at the moment, so this append isn't
+    # strictly necessary. We could get by with only adding deferred restorations
+    # to non-Network Layers.
+    self._set_scope()
+    # We use set_custom_getter because it avoids recursively calling up the
+    # variable_scope tree. We've done the tree traversal ourselves and have
+    # added the request to each Layer which needs it.
+    self._scope.set_custom_getter(self._custom_getter)
+    self._deferred_restorations.append(deferred_restoration)
+    for layer in self.layers:
+      if isinstance(layer, Network):
+        # For Networks, request that they propagate this deferred restoration
+        # to all of their children recursively.
+        layer._add_deferred_restoration(deferred_restoration)
+      else:
+        # For non-Network Layers, make sure they have a deferred restoration
+        # queue and a custom getter, then add our request to it.
+        if not hasattr(layer, "_custom_getter"):
+          assert not hasattr(layer, "_deferred_restorations")
+          layer._custom_getter, layer._deferred_restorations = (
+              _make_custom_getter_for_deferred_restorations())
+          self._set_scope_for_nonnetwork_sublayer(layer)
+          layer._scope.set_custom_getter(layer._custom_getter)
+        layer._deferred_restorations.append(deferred_restoration)
+
+  def restore(self, save_path, map_func=None):
+    """Restore the Network from a checkpoint.
+
+    If variables have already been created (typically when some or all of the
+    `Network` is built), they are assigned values from the checkpoint
+    immediately, overwriting any existing values (in graph mode the default
+    session is used for the assignments).
+
+    If there are checkpoint entries which do not correspond to any existing
+    variables in the `Network`, these values are saved for deferred restoration;
+    their initial values will be the checkpointed values once they are
+    created. Requests for multiple deferred restorations behave the same way as
+    immediate restorations, in that later requests will take priority over
+    earlier requests relevant to the same variable.
+
+    If this `Network` shares `Layer`s with another network, those `Layer`s will
+    also have their variables restored from the checkpoint.
+
+    Args:
+      save_path: The return value of `Network.save`, or a directory to search
+        for a checkpoint.
+      map_func: A function mapping fully qualified variable names
+        (e.g. 'my_network_1/dense_1/kernel') to names in the checkpoint. By
+        default (if `map_func=None`), the variable prefix for the network being
+        restored (`Network.scope_name + '/'`, e.g. 'my_network_1/') is stripped
+        and all other variable names (shared with other Networks) are left
+        unchanged. Note that this is the _same_ map_func as `Network.save`, not
+        an inverse mapping.
+    """
+    self._finalize_name(parent_network=False)
+    self._set_scope()  # scope_name should be available to map_funcs
+    if os.path.isdir(save_path):
+      # If we don't have a name yet, set no parent.
+      save_path = os.path.join(save_path, self.name)
+    user_map_func = map_func
+    if map_func is None:
+      map_func = self._strip_variable_prefix
+    # Step one is to restore any existing variables from the checkpoint.
+    existing_variables_by_checkpoint_name = self._restore_existing_variables(
+        save_path=save_path,
+        map_func=map_func,
+        user_map_func=user_map_func)
+    # Step two is to set a custom getter which restores variables on creation,
+    # for those variables which have not been added to sub-Layers yet.
+    self._set_restore_on_create(
+        save_path=save_path,
+        map_func=map_func,
+        user_map_func=user_map_func,
+        existing_variables_by_checkpoint_name=(
+            existing_variables_by_checkpoint_name))
+
   # TODO(josh11b): Support other Layer methods needed for graph mode, such as for
   # losses and updates
 
diff --git a/tensorflow/contrib/eager/python/network_test.py b/tensorflow/contrib/eager/python/network_test.py
index e4cba3f2ed..c621f527c2 100644
--- a/tensorflow/contrib/eager/python/network_test.py
+++ b/tensorflow/contrib/eager/python/network_test.py
@@ -21,12 +21,14 @@ import gc
 from tensorflow.contrib.eager.python import network
 from tensorflow.python.eager import test
 from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import errors_impl
 from tensorflow.python.framework import test_util
 from tensorflow.python.layers import core
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import nn_ops
 from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.ops import variable_scope
+from tensorflow.python.training import training_util
 
 
 # pylint: disable=not-callable
@@ -42,6 +44,29 @@ class MyNetwork(network.Network):
 
 class NetworkTest(test.TestCase):
 
+  def _save_modify_load_network_built(self, net, global_step=None):
+    checkpoint_directory = self.get_temp_dir()
+    checkpoint_path = net.save(
+        save_path=checkpoint_directory, global_step=global_step)
+    input_value = constant_op.constant([[42.0]])
+    original_output = self.evaluate(net(input_value))
+    for var in net.variables:
+      self.evaluate(var.assign(var + 1.))
+    self.assertGreater(
+        self.evaluate(net(input_value)),
+        original_output)
+    # Either the returned explicit checkpoint path or the directory should work.
+    net.restore(save_path=checkpoint_directory)
+    self.assertAllEqual(
+        original_output,
+        self.evaluate(net(input_value)))
+    for var in net.variables:
+      self.evaluate(var.assign(var + 2.))
+    net.restore(save_path=checkpoint_path)
+    self.assertAllEqual(
+        original_output,
+        self.evaluate(net(input_value)))
+
   @test_util.run_in_graph_and_eager_modes()
   def testTrainableAttribute(self):
     net = network.Network()
@@ -60,6 +85,264 @@ class NetworkTest(test.TestCase):
     result = net(constant_op.constant([[2.0]]))
     self.assertEqual(34.0, self.evaluate(result))
 
+  @test_util.run_in_graph_and_eager_modes()
+  def testNetworkSaveRestoreAlreadyBuilt(self):
+    net = MyNetwork(name="abcd")
+    with self.assertRaisesRegexp(
+        ValueError, "Attempt to save the Network before it was first called"):
+      net.save(self.get_temp_dir())
+    net(constant_op.constant([[2.0]]))
+    self.evaluate(net.trainable_variables[0].assign([[17.0]]))
+    self._save_modify_load_network_built(net, global_step=None)
+    self._save_modify_load_network_built(net, global_step=10)
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testSaveRestoreDefaultGlobalStep(self):
+    net = MyNetwork(name="abcd")
+    net(constant_op.constant([[2.0]]))
+    self.evaluate(net.variables[0].assign([[3.]]))
+    default_global_step = training_util.get_or_create_global_step()
+    self.evaluate(default_global_step.assign(4242))
+    save_path = net.save(self.get_temp_dir())
+    self.assertIn("abcd-4242", save_path)
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testNetworkSaveAndRestoreIntoUnbuilt(self):
+    save_dir = self.get_temp_dir()
+    net1 = MyNetwork()
+    test_input = constant_op.constant([[2.0]])
+    net1(test_input)
+    self.evaluate(net1.trainable_variables[0].assign([[17.0]]))
+    save_path = net1.save(save_dir)
+    # With a pre-build restore we should have the same value.
+    net2 = MyNetwork()
+    net2.restore(save_path)
+    self.assertAllEqual(self.evaluate(net1(test_input)),
+                        self.evaluate(net2(test_input)))
+    self.assertIsNot(net1.variables[0], net2.variables[0])
+    self.assertAllEqual(self.evaluate(net1.variables[0]),
+                        self.evaluate(net2.variables[0]))
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testLoadIntoUnbuiltSharedLayer(self):
+
+    class Owner(network.Network):
+
+      def __init__(self, name=None):
+        super(Owner, self).__init__(name=name)
+        self.first = self.track_layer(core.Dense(
+            1, name="first_layer", use_bias=False))
+
+      def call(self, x):
+        return self.first(x)
+
+    first_owner = Owner()
+
+    class User(network.Network):
+
+      def __init__(self, use_layer, name=None):
+        super(User, self).__init__(name=name)
+        self.first = self.track_layer(use_layer)
+        self.second = self.track_layer(core.Dense(
+            1, name="second_layer", use_bias=False))
+
+      def call(self, x):
+        return self.second(self.first(x))
+
+    class LikeUserButNotSharing(network.Network):
+
+      def __init__(self, name=None):
+        super(LikeUserButNotSharing, self).__init__(name=name)
+        self.first = self.track_layer(core.Dense(
+            1, name="first_layer", use_bias=False))
+        self.second = self.track_layer(core.Dense(
+            1, name="second_layer", use_bias=False))
+
+      def call(self, x):
+        return self.second(self.first(x))
+
+    checkpoint_creator = LikeUserButNotSharing(name="checkpoint_creator")
+    one = constant_op.constant([[1.0]])
+    checkpoint_creator(one)
+    self.assertEqual(2, len(checkpoint_creator.variables))
+    self.evaluate(checkpoint_creator.variables[0].assign([[5.]]))
+    self.evaluate(checkpoint_creator.variables[1].assign([[6.]]))
+    # Re-map the variable names so that with default restore mapping we'll
+    # attempt to restore into the unbuilt Layer.
+    name_mapping = {
+        "checkpoint_creator/first_layer/kernel": "owner_1/first_layer/kernel",
+        "checkpoint_creator/second_layer/kernel": "second_layer/kernel",
+    }
+    save_path = checkpoint_creator.save(
+        self.get_temp_dir(),
+        map_func=lambda full_name: name_mapping[full_name])
+    load_into = User(use_layer=first_owner.first)
+    load_into.restore(save_path)
+    self.assertEqual(0, len(first_owner.variables))
+    self.assertAllEqual(self.evaluate(checkpoint_creator(one)),
+                        self.evaluate(load_into(one)))
+    self.assertEqual(1, len(first_owner.variables))
+    self.assertAllEqual([[5.]], self.evaluate(load_into.variables[0]))
+    self.assertAllEqual([[6.]], self.evaluate(load_into.variables[1]))
+    first_owner(one)
+    self.assertAllEqual([[5.]], self.evaluate(first_owner.variables[0]))
+
+    # Try again with a garbage collected parent.
+    first_owner = Owner()
+    load_into = User(use_layer=first_owner.first)
+    del first_owner
+    gc.collect()
+    def _restore_map_func(original_name):
+      if original_name.startswith("owner_1"):
+        return original_name.replace("owner_1", "owner_2")
+      else:
+        return "user_2/" + original_name
+    with self.assertRaisesRegexp(ValueError, "garbage collected"):
+      load_into.restore(save_path, map_func=_restore_map_func)
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testRestoreIntoSubNetwork(self):
+
+    class Parent(network.Network):
+
+      def __init__(self, name=None):
+        super(Parent, self).__init__(name=name)
+        self.first = self.track_layer(MyNetwork())
+        self.second = self.track_layer(MyNetwork())
+
+      def call(self, x):
+        return self.first(self.second(x))
+
+    one = constant_op.constant([[3.]])
+    whole_model_saver = Parent()
+    whole_model_saver(one)
+    self.evaluate(whole_model_saver.variables[0].assign([[15.]]))
+    self.evaluate(whole_model_saver.variables[1].assign([[16.]]))
+    whole_model_checkpoint = whole_model_saver.save(self.get_temp_dir())
+
+    save_from = MyNetwork()
+    save_from(one)
+    self.evaluate(save_from.variables[0].assign([[5.]]))
+    checkpoint = save_from.save(self.get_temp_dir())
+    save_into_parent = Parent()
+    save_into_parent.restore(whole_model_checkpoint)
+    save_into_parent.first.restore(checkpoint)
+    save_into_parent.first.restore(checkpoint)  # deferred loading multiple
+                                                # times is fine
+    save_into_parent(one)  # deferred loading
+    self.assertAllEqual([[5.]], self.evaluate(save_into_parent.variables[0]))
+    self.assertAllEqual([[16.]], self.evaluate(save_into_parent.variables[1]))
+
+    # Try again with the opposite ordering, and we should get different results
+    # (deferred restoration should happen the same way non-deferred happens,
+    # with later restorations overwriting older ones).
+    save_into_parent = Parent()
+    save_into_parent.first.restore(checkpoint)  # deferred loading multiple
+                                                # times is fine
+    save_into_parent.restore(whole_model_checkpoint)
+    save_into_parent(one)  # deferred loading
+    # We've overwritten the sub-Network restore.
+    self.assertAllEqual([[15.]], self.evaluate(save_into_parent.variables[0]))
+    self.assertAllEqual([[16.]], self.evaluate(save_into_parent.variables[1]))
+
+    self.evaluate(save_into_parent.variables[0].assign([[3.]]))
+    self.evaluate(save_into_parent.variables[1].assign([[4.]]))
+    save_into_parent.second.restore(checkpoint)
+    self.assertAllEqual([[5.]], self.evaluate(save_into_parent.variables[1]))
+    with self.assertRaisesRegexp(errors_impl.NotFoundError,
+                                 "not found in checkpoint"):
+      # The checkpoint is incompatible.
+      save_into_parent.restore(checkpoint)
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testCustomMapCollisionErrors(self):
+
+    class Parent(network.Network):
+
+      def __init__(self, name=None):
+        super(Parent, self).__init__(name=name)
+        self.first = self.track_layer(MyNetwork())
+        self.second = self.track_layer(MyNetwork())
+
+      def call(self, x):
+        return self.first(self.second(x))
+
+    make_checkpoint = Parent()
+    one = constant_op.constant([[1.]])
+    make_checkpoint(one)
+    self.evaluate(make_checkpoint.variables[0].assign([[2.]]))
+    self.evaluate(make_checkpoint.variables[1].assign([[3.]]))
+    with self.assertRaisesRegexp(
+        ValueError,
+        "The map_func passed to Network.save for the Network 'parent_1' "
+        "resulted in two variables named 'foo'"):
+      make_checkpoint.save(self.get_temp_dir(), map_func=lambda n: "foo")
+    checkpoint = make_checkpoint.first.save(
+        self.get_temp_dir(), map_func=lambda n: "foo")
+    loader = Parent()
+    loader.restore(checkpoint, map_func=lambda n: "foo")
+    with self.assertRaisesRegexp(
+        ValueError,
+        ("The map_func passed to Network.restore for the Network"
+         " 'parent_2' resulted in two variables named 'foo'")):
+      loader(one)
+    loader = Parent()
+    loader(one)
+    with self.assertRaisesRegexp(
+        ValueError,
+        ("The map_func passed to Network.restore for the Network"
+         " 'parent_3' resulted in two variables named 'foo'")):
+      loader.restore(checkpoint, map_func=lambda n: "foo")
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testDefaultMapCollisionErrors(self):
+
+    one = constant_op.constant([[1.]])
+    first = core.Dense(1, name="dense_1", use_bias=False)
+    first(one)
+
+    class Parent(network.Network):
+
+      def __init__(self, name=None):
+        super(Parent, self).__init__(name=name)
+        self.first = self.track_layer(first)
+        self.second = self.track_layer(core.Dense(1, use_bias=False))
+
+      def call(self, x):
+        return self.first(self.second(x))
+
+    make_checkpoint = Parent()
+    one = constant_op.constant([[1.]])
+    make_checkpoint(one)
+    self.evaluate(make_checkpoint.variables[0].assign([[2.]]))
+    self.evaluate(make_checkpoint.variables[1].assign([[3.]]))
+    with self.assertRaisesRegexp(
+        ValueError,
+        ("The default checkpoint variable name mapping strategy for Network "
+         "'parent_1' resulted in a naming conflict.")):
+      make_checkpoint.save(self.get_temp_dir())
+
+    class Compatible(network.Network):
+
+      def __init__(self, name=None):
+        super(Compatible, self).__init__(name=name)
+        self.first = self.track_layer(core.Dense(1, use_bias=False))
+
+      def call(self, x):
+        return self.first(x)
+
+    successful_checkpoint = Compatible()
+    successful_checkpoint(one)
+    self.evaluate(successful_checkpoint.variables[0].assign([[-1.]]))
+    checkpoint_path = successful_checkpoint.save(self.get_temp_dir())
+    load_checkpoint = Parent()
+    load_checkpoint(one)
+    with self.assertRaisesRegexp(
+        ValueError,
+        ("The default checkpoint variable name mapping strategy for Network "
+         "'parent_2' resulted in a naming conflict.")):
+      load_checkpoint.restore(checkpoint_path)
+
   def testNoReferenceCyclesAfterCall(self):
 
     class ChildNetwork(network.Network):
-- 
GitLab


From 7c4e98eb4a459eaf79fb76a97c35481b8f063c85 Mon Sep 17 00:00:00 2001
From: Igor Ganichev <iga@google.com>
Date: Fri, 27 Oct 2017 12:20:47 -0700
Subject: [PATCH 881/909] Add Tensor._rank() getter

It appears to speed up SPINN model by about 1%, which is not much, but
this method is very simple and easier to use than len(tensor._shape_tuple())

PiperOrigin-RevId: 173703259
---
 tensorflow/python/eager/pywrap_tensor.cc | 10 ++++++++++
 tensorflow/python/framework/ops.py       | 20 ++++++++++++++++++++
 2 files changed, 30 insertions(+)

diff --git a/tensorflow/python/eager/pywrap_tensor.cc b/tensorflow/python/eager/pywrap_tensor.cc
index 157e87d387..3adaea2c79 100644
--- a/tensorflow/python/eager/pywrap_tensor.cc
+++ b/tensorflow/python/eager/pywrap_tensor.cc
@@ -377,6 +377,15 @@ static PyObject* EagerTensor_shape_tuple(EagerTensor* self) {
   return shape;
 }
 
+// Getter for `_rank`.
+static PyObject* EagerTensor_rank(EagerTensor* self) {
+#if PY_MAJOR_VERSION < 3
+  return PyInt_FromLong(TFE_TensorHandleNumDims(self->handle));
+#else
+  return PyLong_FromLong(TFE_TensorHandleNumDims(self->handle));
+#endif
+}
+
 static PyObject* EagerTensor_tensor_handle(EagerTensor* self, void* unused) {
   Py_INCREF(self->handle_data);
   return self->handle_data;
@@ -470,6 +479,7 @@ static PyMethodDef EagerTensor_methods[] = {
      PyDoc_STR("_datatype_enum")},
     {"_shape_tuple", (PyCFunction)EagerTensor_shape_tuple, METH_NOARGS,
      PyDoc_STR("_shape_tuple")},
+    {"_rank", (PyCFunction)EagerTensor_rank, METH_NOARGS, PyDoc_STR("_rank")},
     {"_copy_to_device", (PyCFunction)EagerTensor_copy_to_device,
      METH_VARARGS | METH_KEYWORDS, PyDoc_STR("_copy_to_device")},
     {nullptr, nullptr},
diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index c278fb2a39..c8ee9243d7 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -383,6 +383,14 @@ class Tensor(_TensorLike):
       return None
     return tuple(shape)
 
+  def _rank(self):
+    """Integer rank of this Tensor, if known, else None.
+
+    Returns:
+      Integer rank or None
+    """
+    return self._shape.ndims
+
   def get_shape(self):
     """Alias of Tensor.shape."""
     return self.shape
@@ -664,6 +672,18 @@ class _EagerTensorBase(Tensor):
     """
     raise NotImplementedError()
 
+  def _rank(self):
+    """Integer rank of this Tensor.
+
+    Unlike regular Tensors, the rank is always known for EagerTensors.
+
+    This is more performant than len(self._shape_tuple())
+
+    Returns:
+      Integer rank
+    """
+    raise NotImplementedError()
+
   def _copy_to_device(self, context, device):  # pylint: disable=redefined-outer-name
     raise NotImplementedError()
 
-- 
GitLab


From 466b9ecf8b42bd4a596281f1b430f122c5c3e00a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 27 Oct 2017 13:01:09 -0700
Subject: [PATCH 882/909]   Report total number of bytes to be transferred when
 the curl request makes no progress.

PiperOrigin-RevId: 173707608
---
 tensorflow/core/platform/cloud/curl_http_request.cc | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/platform/cloud/curl_http_request.cc b/tensorflow/core/platform/cloud/curl_http_request.cc
index e1f8867b38..e2d935f35e 100644
--- a/tensorflow/core/platform/cloud/curl_http_request.cc
+++ b/tensorflow/core/platform/cloud/curl_http_request.cc
@@ -512,8 +512,10 @@ int CurlHttpRequest::ProgressCallback(void* this_object, curl_off_t dltotal,
   }
 
   if (now - that->last_progress_timestamp_ > kInactivityTimeoutSeconds) {
-    LOG(ERROR) << "The transmission has been stuck at " << current_progress
-               << " bytes for " << now - that->last_progress_timestamp_
+    LOG(ERROR) << "The transmission  of request " << this_object
+               << " has been stuck at " << current_progress << " of "
+               << dltotal + ultotal << " bytes for "
+               << now - that->last_progress_timestamp_
                << " seconds and will be aborted.";
     return 1;  // Will abort the request.
   }
-- 
GitLab


From b31b08bb0f876db76ae3beb0b0801ab1893f9abf Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 27 Oct 2017 13:14:41 -0700
Subject: [PATCH 883/909] Adds randomized tests for newly introduced complex
 and related ops.

PiperOrigin-RevId: 173709206
---
 tensorflow/compiler/tests/randomized_tests.cc | 236 +++++++++++-------
 .../compiler/tf2xla/kernels/unary_ops.cc      |   2 +-
 2 files changed, 145 insertions(+), 93 deletions(-)

diff --git a/tensorflow/compiler/tests/randomized_tests.cc b/tensorflow/compiler/tests/randomized_tests.cc
index 461af83362..c8a32f9e29 100644
--- a/tensorflow/compiler/tests/randomized_tests.cc
+++ b/tensorflow/compiler/tests/randomized_tests.cc
@@ -367,11 +367,11 @@ OpTest::OpTest() {
 void OpTest::Repeatedly(const std::function<TestResult(void)>& fn) {
   int const max_repetitions = tf_xla_test_repetitions;
   int valid_test_runs = 0;
-  // We run up to 20 * max_repetitions times; the idea is that if we roll the
+  // We run up to 100 * max_repetitions times; the idea is that if we roll the
   // dice enough times we will find some valid parameters. We want to put an
   // upper limit on the number iterations just in case the probability of
   // finding feasible parameters is very low.
-  for (int i = 0; !HasFailure() && i < max_repetitions * 20 &&
+  for (int i = 0; !HasFailure() && i < max_repetitions * 100 &&
                   valid_test_runs < max_repetitions;
        ++i) {
     TestResult result = fn();
@@ -868,7 +868,7 @@ Tensor AsIntTensor(DataType dtype, const std::vector<int64>& values) {
 
 TEST_F(OpTest, Abs) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
+    auto type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
     return ExpectTfAndXlaOutputsAreClose(
         OpTestBuilder("Abs").RandomInput(type).Attr("T", type));
   });
@@ -883,7 +883,7 @@ TEST_F(OpTest, Acosh) {
 
 TEST_F(OpTest, Add) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
+    auto type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
     auto dims = BroadcastableDims();
     return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("Add")
                                              .RandomInput(type, dims.first)
@@ -894,7 +894,7 @@ TEST_F(OpTest, Add) {
 
 TEST_F(OpTest, AddN) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
+    auto type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
     int n = std::uniform_int_distribution<int>(1, 5)(generator());
 
     auto shape = RandomDims();
@@ -921,6 +921,14 @@ TEST_F(OpTest, All) {
   });
 }
 
+TEST_F(OpTest, Angle) {
+  Repeatedly([this]() {
+    return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("Angle")
+                                             .RandomInput(DT_COMPLEX64)
+                                             .Attr("T", DT_COMPLEX64));
+  });
+}
+
 TEST_F(OpTest, Any) {
   Repeatedly([this]() {
     std::vector<int64> data_dims = RandomDims();
@@ -935,11 +943,11 @@ TEST_F(OpTest, Any) {
 
 TEST_F(OpTest, ApproximateEqual) {
   Repeatedly([this]() {
-    auto dims = RandomDims();
-    DataType type = Choose<DataType>({DT_FLOAT, DT_COMPLEX64});
+    auto dims = BroadcastableDims();
+    auto type = Choose<DataType>({DT_FLOAT, DT_COMPLEX64});
     return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("ApproximateEqual")
-                                             .RandomInput(type, dims)
-                                             .RandomInput(type, dims)
+                                             .RandomInput(type, dims.first)
+                                             .RandomInput(type, dims.second)
                                              .Attr("T", DT_FLOAT));
   });
 }
@@ -990,6 +998,16 @@ TEST_F(OpTest, Atanh) {
   });
 }
 
+TEST_F(OpTest, Atan2) {
+  Repeatedly([this]() {
+    auto dims = BroadcastableDims();
+    return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("Atan2")
+                                             .RandomInput(DT_FLOAT, dims.first)
+                                             .RandomInput(DT_FLOAT, dims.second)
+                                             .Attr("T", DT_FLOAT));
+  });
+}
+
 TEST_F(OpTest, AvgPool) {
   Repeatedly([this]() {
     std::uniform_int_distribution<int> random_int(1, 5);
@@ -1085,7 +1103,7 @@ TEST_F(OpTest, AvgPool3DGrad) {
 
 TEST_F(OpTest, BatchMatMul) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>({DT_FLOAT, DT_COMPLEX64});
+    auto type = Choose<DataType>({DT_FLOAT, DT_COMPLEX64});
     std::vector<int64> output_dims = RandomDims(2, 5, 0, 7);
     int64 ndims = output_dims.size();
     int64 inner_dim = RandomDim();
@@ -1138,7 +1156,7 @@ TEST_F(OpTest, BatchToSpace) {
     CHECK(crops.CopyFrom(AsIntTensor(DT_INT32, crop_vals),
                          TensorShape({num_block_dims, 2})));
 
-    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
+    auto type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
     return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("BatchToSpace")
                                              .RandomInput(type, input_dims)
                                              .Input(crops)
@@ -1176,7 +1194,7 @@ TEST_F(OpTest, BatchToSpaceND) {
     CHECK(crops.CopyFrom(AsIntTensor(DT_INT32, crop_vals),
                          TensorShape({num_block_dims, 2})));
 
-    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
+    auto type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
     return ExpectTfAndXlaOutputsAreClose(
         OpTestBuilder("BatchToSpaceND")
             .RandomInput(type, input_dims)
@@ -1192,7 +1210,7 @@ TEST_F(OpTest, BiasAdd) {
     auto x_dims = RandomDims(2, kDefaultMaxRank);
     auto y_dims = {x_dims[x_dims.size() - 1]};
     // TODO(phawkins): test both data formats.
-    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
+    auto type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
     return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("BiasAdd")
                                              .RandomInput(type, x_dims)
                                              .RandomInput(type, y_dims)
@@ -1203,7 +1221,7 @@ TEST_F(OpTest, BiasAdd) {
 TEST_F(OpTest, BiasAddGrad) {
   Repeatedly([this]() {
     // TODO(phawkins): test both data formats.
-    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
+    auto type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
     return ExpectTfAndXlaOutputsAreClose(
         OpTestBuilder("BiasAddGrad").RandomInput(type).Attr("T", type));
   });
@@ -1213,7 +1231,7 @@ TEST_F(OpTest, BiasAddV1) {
   Repeatedly([this]() {
     auto x_dims = RandomDims(2, kDefaultMaxRank);
     auto y_dims = {x_dims[x_dims.size() - 1]};
-    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
+    auto type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
     return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("BiasAddV1")
                                              .RandomInput(type, x_dims)
                                              .RandomInput(type, y_dims)
@@ -1246,7 +1264,7 @@ TEST_F(OpTest, BitwiseOr) {
 TEST_F(OpTest, BroadcastArgs) {
   Repeatedly([this]() {
     // TODO(phawkins): only int32 seems to be implemented in Tensorflow.
-    // DataType type = Choose<DataType>({DT_INT32, DT_INT64});
+    // auto type = Choose<DataType>({DT_INT32, DT_INT64});
     DataType type = DT_INT32;
     auto dims = BroadcastableDims();
     return ExpectTfAndXlaOutputsAreClose(
@@ -1260,7 +1278,7 @@ TEST_F(OpTest, BroadcastArgs) {
 TEST_F(OpTest, BroadcastGradientArgs) {
   Repeatedly([this]() {
     // TODO(phawkins): only int32 seems to be implemented in Tensorflow.
-    // DataType type = Choose<DataType>({DT_INT32, DT_INT64});
+    // auto type = Choose<DataType>({DT_INT32, DT_INT64});
     DataType type = DT_INT32;
     auto dims = BroadcastableDims();
     return ExpectTfAndXlaOutputsAreClose(
@@ -1290,9 +1308,19 @@ TEST_F(OpTest, Ceil) {
   });
 }
 
+TEST_F(OpTest, Complex) {
+  Repeatedly([this]() {
+    auto dims = BroadcastableDims();
+    return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("Complex")
+                                             .RandomInput(DT_FLOAT, dims.first)
+                                             .RandomInput(DT_FLOAT, dims.second)
+                                             .Attr("T", DT_FLOAT));
+  });
+}
+
 TEST_F(OpTest, Concat) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>(kAllXlaTypes);
+    auto type = Choose<DataType>(kAllXlaTypes);
     int n = std::uniform_int_distribution<int>(2, 5)(generator());
 
     std::vector<int64> dims = RandomDims(1);
@@ -1332,6 +1360,14 @@ TEST_F(OpTest, ConcatOffset) {
   });
 }
 
+TEST_F(OpTest, Conj) {
+  Repeatedly([this]() {
+    return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("Conj")
+                                             .RandomInput(DT_COMPLEX64)
+                                             .Attr("T", DT_COMPLEX64));
+  });
+}
+
 TEST_F(OpTest, Conv2D) {
   Repeatedly([this]() {
     WindowedSpatialDims d = ChooseWindowedSpatialDims(2);
@@ -1471,7 +1507,7 @@ TEST_F(OpTest, Conv3DBackpropInput) {
         ImageDims(FORMAT_NHWC, batch, features_out, d.output_dims);
     std::vector<int64> kernel = {d.kernel_dims[0], d.kernel_dims[1],
                                  d.kernel_dims[2], features_in, features_out};
-    DataType type = Choose<DataType>({DT_FLOAT, DT_COMPLEX64});
+    auto type = Choose<DataType>({DT_FLOAT, DT_COMPLEX64});
     return ExpectTfAndXlaOutputsAreClose(
         OpTestBuilder("Conv3DBackpropInputV2")
             .Input(in_shape)
@@ -1485,7 +1521,7 @@ TEST_F(OpTest, Conv3DBackpropInput) {
 
 TEST_F(OpTest, Cos) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>({DT_FLOAT, DT_COMPLEX64});
+    auto type = Choose<DataType>({DT_FLOAT, DT_COMPLEX64});
     return ExpectTfAndXlaOutputsAreClose(
         OpTestBuilder("Cos").RandomInput(type).Attr("T", type));
   });
@@ -1493,7 +1529,7 @@ TEST_F(OpTest, Cos) {
 
 TEST_F(OpTest, Cosh) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>({DT_FLOAT, DT_COMPLEX64});
+    auto type = Choose<DataType>({DT_FLOAT, DT_COMPLEX64});
     return ExpectTfAndXlaOutputsAreClose(
         OpTestBuilder("Cosh").RandomInput(type).Attr("T", type));
   });
@@ -1506,7 +1542,7 @@ TEST_F(OpTest, DepthToSpace) {
     input_dims[1] = (input_dims[1] + (block - 1)) / block;
     input_dims[2] = (input_dims[2] + (block - 1)) / block;
     input_dims[3] *= block * block;
-    DataType type = Choose<DataType>(kAllXlaTypes);
+    auto type = Choose<DataType>(kAllXlaTypes);
     return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("DepthToSpace")
                                              .RandomInput(type, input_dims)
                                              .Attr("T", type)
@@ -1597,7 +1633,7 @@ TEST_F(OpTest, DepthwiseConv2DBackpropInput) {
 TEST_F(OpTest, Diag) {
   if (1) return;
   Repeatedly([this]() {
-    DataType type = Choose<DataType>(kAllXlaTypes);
+    auto type = Choose<DataType>(kAllXlaTypes);
     std::vector<int64> dims;
     // Diag causes a quadratic blowup in output size.
     int64 size;
@@ -1612,7 +1648,7 @@ TEST_F(OpTest, Diag) {
 
 TEST_F(OpTest, DiagPart) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>(kAllXlaTypes);
+    auto type = Choose<DataType>(kAllXlaTypes);
     auto dims = RandomDims(1, 3);
     // Duplicate the random dims.
     std::vector<int64> doubled_dims(dims.size() * 2);
@@ -1626,7 +1662,7 @@ TEST_F(OpTest, DiagPart) {
 
 TEST_F(OpTest, Div) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
+    auto type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
     auto dims = BroadcastableDims();
     return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("Div")
                                              .RandomInput(type, dims.first)
@@ -1637,7 +1673,7 @@ TEST_F(OpTest, Div) {
 
 TEST_F(OpTest, DynamicStitch) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>(kAllXlaTypes);
+    auto type = Choose<DataType>(kAllXlaTypes);
     int n = std::uniform_int_distribution<int>(2, 5)(generator());
     OpTestBuilder builder("DynamicStitch");
     builder.Attr("T", type);
@@ -1722,7 +1758,7 @@ TEST_F(OpTest, SeluGrad) {
 
 TEST_F(OpTest, Equal) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
+    auto type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
     auto dims = BroadcastableDims();
     return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("Equal")
                                              .RandomInput(type, dims.first)
@@ -1733,7 +1769,7 @@ TEST_F(OpTest, Equal) {
 
 TEST_F(OpTest, Exp) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>({DT_FLOAT, DT_COMPLEX64});
+    auto type = Choose<DataType>({DT_FLOAT, DT_COMPLEX64});
     return ExpectTfAndXlaOutputsAreClose(
         OpTestBuilder("Exp").RandomInput(type).Attr("T", type));
   });
@@ -1741,7 +1777,7 @@ TEST_F(OpTest, Exp) {
 
 TEST_F(OpTest, Expm1) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>({DT_FLOAT, DT_COMPLEX64});
+    auto type = Choose<DataType>({DT_FLOAT, DT_COMPLEX64});
     return ExpectTfAndXlaOutputsAreClose(
         OpTestBuilder("Expm1").RandomInput(type).Attr("T", type));
   });
@@ -1749,7 +1785,7 @@ TEST_F(OpTest, Expm1) {
 
 TEST_F(OpTest, ExpandDims) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>(kAllXlaTypes);
+    auto type = Choose<DataType>(kAllXlaTypes);
     std::vector<int64> in_dims = RandomDims();
     Tensor dim(DT_INT32, TensorShape());
     std::uniform_int_distribution<int32> d(-1 - in_dims.size(), in_dims.size());
@@ -1763,7 +1799,7 @@ TEST_F(OpTest, ExpandDims) {
 
 TEST_F(OpTest, Fill) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>(kAllXlaTypes);
+    auto type = Choose<DataType>(kAllXlaTypes);
     std::vector<int64> dims = RandomDims();
     std::vector<int32> shape(dims.begin(), dims.end());
     return ExpectTfAndXlaOutputsAreClose(
@@ -1794,7 +1830,7 @@ TEST_F(OpTest, FloorDiv) {
 
 TEST_F(OpTest, FloorMod) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT});
+    auto type = Choose<DataType>({DT_INT32, DT_FLOAT});
     auto dims = BroadcastableDims();
     return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("FloorMod")
                                              .RandomInput(type, dims.first)
@@ -1805,7 +1841,7 @@ TEST_F(OpTest, FloorMod) {
 
 TEST_F(OpTest, Greater) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT});
+    auto type = Choose<DataType>({DT_INT32, DT_FLOAT});
     auto dims = BroadcastableDims();
     return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("Greater")
                                              .RandomInput(type, dims.first)
@@ -1816,7 +1852,7 @@ TEST_F(OpTest, Greater) {
 
 TEST_F(OpTest, GreaterEqual) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT});
+    auto type = Choose<DataType>({DT_INT32, DT_FLOAT});
     auto dims = BroadcastableDims();
     return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("GreaterEqual")
                                              .RandomInput(type, dims.first)
@@ -1825,6 +1861,14 @@ TEST_F(OpTest, GreaterEqual) {
   });
 }
 
+TEST_F(OpTest, Imag) {
+  Repeatedly([this]() {
+    return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("Imag")
+                                             .RandomInput(DT_COMPLEX64)
+                                             .Attr("T", DT_COMPLEX64));
+  });
+}
+
 TEST_F(OpTest, Invert) {
   Repeatedly([this]() {
     DataType type = DT_INT32;
@@ -1843,7 +1887,7 @@ TEST_F(OpTest, L2Loss) {
 
 TEST_F(OpTest, Less) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT});
+    auto type = Choose<DataType>({DT_INT32, DT_FLOAT});
     auto dims = BroadcastableDims();
     return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("Less")
                                              .RandomInput(type, dims.first)
@@ -1854,7 +1898,7 @@ TEST_F(OpTest, Less) {
 
 TEST_F(OpTest, LessEqual) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT});
+    auto type = Choose<DataType>({DT_INT32, DT_FLOAT});
     auto dims = BroadcastableDims();
     return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("LessEqual")
                                              .RandomInput(type, dims.first)
@@ -1870,7 +1914,7 @@ TEST_F(OpTest, LinSpace) {
       return test::AsScalar<int64>(x);
     };
     std::uniform_int_distribution<int> distribution(-50, 50);
-    DataType type = Choose<DataType>({DT_INT32, DT_INT64});
+    auto type = Choose<DataType>({DT_INT32, DT_INT64});
     return ExpectTfAndXlaOutputsAreClose(
         OpTestBuilder("LinSpace")
             .RandomInput(DT_FLOAT, {})
@@ -1883,7 +1927,7 @@ TEST_F(OpTest, LinSpace) {
 
 TEST_F(OpTest, Log) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>({DT_FLOAT, DT_COMPLEX64});
+    auto type = Choose<DataType>({DT_FLOAT, DT_COMPLEX64});
     return ExpectTfAndXlaOutputsAreClose(
         OpTestBuilder("Log").RandomInput(type).Attr("T", type));
   });
@@ -1891,7 +1935,7 @@ TEST_F(OpTest, Log) {
 
 TEST_F(OpTest, Log1p) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>({DT_FLOAT, DT_COMPLEX64});
+    auto type = Choose<DataType>({DT_FLOAT, DT_COMPLEX64});
     return ExpectTfAndXlaOutputsAreClose(
         OpTestBuilder("Log1p").RandomInput(type).Attr("T", DT_FLOAT));
   });
@@ -1990,7 +2034,7 @@ TEST_F(OpTest, MatMul) {
       std::swap(b_dims[0], b_dims[1]);
     }
 
-    DataType type = Choose<DataType>({DT_FLOAT, DT_COMPLEX64});
+    auto type = Choose<DataType>({DT_FLOAT, DT_COMPLEX64});
     return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("MatMul")
                                              .RandomInput(type, a_dims)
                                              .RandomInput(type, b_dims)
@@ -2002,7 +2046,7 @@ TEST_F(OpTest, MatMul) {
 
 TEST_F(OpTest, MatrixDiag) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
+    auto type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
     return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("MatrixDiag")
                                              .RandomInput(type, RandomDims(1))
                                              .Attr("T", type));
@@ -2011,7 +2055,7 @@ TEST_F(OpTest, MatrixDiag) {
 
 TEST_F(OpTest, MatrixDiagPart) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
+    auto type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
     return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("MatrixDiagPart")
                                              .RandomInput(type, RandomDims(2))
                                              .Attr("T", type));
@@ -2020,7 +2064,7 @@ TEST_F(OpTest, MatrixDiagPart) {
 
 TEST_F(OpTest, Max) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT});
+    auto type = Choose<DataType>({DT_INT32, DT_FLOAT});
     std::vector<int64> data_dims = RandomDims();
     Tensor indices = RandomReductionIndices(data_dims.size());
     bool keep_dims = Choose<bool>({false, true});
@@ -2034,7 +2078,7 @@ TEST_F(OpTest, Max) {
 
 TEST_F(OpTest, Maximum) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT});
+    auto type = Choose<DataType>({DT_INT32, DT_FLOAT});
     auto dims = BroadcastableDims();
     return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("Maximum")
                                              .RandomInput(type, dims.first)
@@ -2102,7 +2146,7 @@ TEST_F(OpTest, MaxPool3D) {
 
 TEST_F(OpTest, Mean) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
+    auto type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
     // TODO(phawkins): CPU and XLA differ output for reducing across a
     // size-0 dimension (nan vs 0). For now, require size >= 1.
     std::vector<int64> data_dims = RandomDims(0, kDefaultMaxRank, 1);
@@ -2118,7 +2162,7 @@ TEST_F(OpTest, Mean) {
 
 TEST_F(OpTest, Min) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT});
+    auto type = Choose<DataType>({DT_INT32, DT_FLOAT});
     std::vector<int64> data_dims = RandomDims();
     Tensor indices = RandomReductionIndices(data_dims.size());
     bool keep_dims = Choose<bool>({false, true});
@@ -2132,7 +2176,7 @@ TEST_F(OpTest, Min) {
 
 TEST_F(OpTest, Minimum) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT});
+    auto type = Choose<DataType>({DT_INT32, DT_FLOAT});
     auto dims = BroadcastableDims();
     return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("Minimum")
                                              .RandomInput(type, dims.first)
@@ -2153,7 +2197,7 @@ TEST_F(OpTest, Mod) {
 
 TEST_F(OpTest, Mul) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
+    auto type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
     auto dims = BroadcastableDims();
     return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("Mul")
                                              .RandomInput(type, dims.first)
@@ -2164,7 +2208,7 @@ TEST_F(OpTest, Mul) {
 
 TEST_F(OpTest, Neg) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
+    auto type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
     return ExpectTfAndXlaOutputsAreClose(
         OpTestBuilder("Neg").RandomInput(type).Attr("T", type));
   });
@@ -2172,7 +2216,7 @@ TEST_F(OpTest, Neg) {
 
 TEST_F(OpTest, NotEqual) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
+    auto type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
     auto dims = BroadcastableDims();
     return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("NotEqual")
                                              .RandomInput(type, dims.first)
@@ -2183,7 +2227,7 @@ TEST_F(OpTest, NotEqual) {
 
 TEST_F(OpTest, OneHot) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>(kAllXlaTypes);
+    auto type = Choose<DataType>(kAllXlaTypes);
 
     std::vector<int64> dims = RandomDims();
     int num_dims = dims.size();
@@ -2213,7 +2257,7 @@ TEST_F(OpTest, OneHot) {
 
 TEST_F(OpTest, OnesLike) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
+    auto type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
     return ExpectTfAndXlaOutputsAreClose(
         OpTestBuilder("OnesLike").RandomInput(type).Attr("T", type));
   });
@@ -2221,7 +2265,7 @@ TEST_F(OpTest, OnesLike) {
 
 TEST_F(OpTest, Pack) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>(kAllXlaTypes);
+    auto type = Choose<DataType>(kAllXlaTypes);
     int n = std::uniform_int_distribution<int>(1, 5)(generator());
 
     std::vector<int64> dims = RandomDims();
@@ -2243,7 +2287,7 @@ TEST_F(OpTest, Pack) {
 // TODO(b/31741898): crashes on GPU.
 TEST_F(OpTest, Pad) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>(kAllXlaTypes);
+    auto type = Choose<DataType>(kAllXlaTypes);
     std::vector<int64> t_dims = RandomDims();
 
     // TODO(b/31741996): re-enable DT_INT64 when bug is fixed.
@@ -2272,7 +2316,7 @@ TEST_F(OpTest, Pow) {
   // nontermination.
   Repeatedly([this]() {
     auto dims = BroadcastableDims();
-    DataType type = Choose<DataType>({DT_FLOAT, DT_COMPLEX64});
+    auto type = Choose<DataType>({DT_FLOAT, DT_COMPLEX64});
     return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("Pow")
                                              .RandomInput(type, dims.first)
                                              .RandomInput(type, dims.second)
@@ -2282,7 +2326,7 @@ TEST_F(OpTest, Pow) {
 
 TEST_F(OpTest, Prod) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
+    auto type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
     std::vector<int64> data_dims = RandomDims();
     Tensor indices = RandomReductionIndices(data_dims.size());
     bool keep_dims = Choose<bool>({false, true});
@@ -2316,15 +2360,23 @@ TEST_F(OpTest, Range) {
 
 TEST_F(OpTest, Rank) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
+    auto type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
     return ExpectTfAndXlaOutputsAreClose(
         OpTestBuilder("Rank").RandomInput(type).Attr("T", type));
   });
 }
 
+TEST_F(OpTest, Real) {
+  Repeatedly([this]() {
+    return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("Real")
+                                             .RandomInput(DT_COMPLEX64)
+                                             .Attr("T", DT_COMPLEX64));
+  });
+}
+
 TEST_F(OpTest, RealDiv) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>({DT_FLOAT, DT_COMPLEX64});
+    auto type = Choose<DataType>({DT_FLOAT, DT_COMPLEX64});
     auto dims = BroadcastableDims();
     return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("RealDiv")
                                              .RandomInput(type, dims.first)
@@ -2335,7 +2387,7 @@ TEST_F(OpTest, RealDiv) {
 
 TEST_F(OpTest, Reciprocal) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
+    auto type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
     return ExpectTfAndXlaOutputsAreClose(
         OpTestBuilder("Reciprocal").RandomInput(type).Attr("T", type));
   });
@@ -2344,7 +2396,7 @@ TEST_F(OpTest, Reciprocal) {
 TEST_F(OpTest, ReciprocalGrad) {
   Repeatedly([this]() {
     std::vector<int64> dims = RandomDims();
-    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
+    auto type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
     return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("ReciprocalGrad")
                                              .RandomInput(type, dims)
                                              .RandomInput(type, dims)
@@ -2387,7 +2439,7 @@ TEST_F(OpTest, ReluGrad) {
 
 TEST_F(OpTest, Reshape) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>(kAllXlaTypes);
+    auto type = Choose<DataType>(kAllXlaTypes);
     std::vector<int64> dims = RandomDims();
     std::bernoulli_distribution random_bool;
     std::vector<int64> dims_before, dims_after;
@@ -2415,7 +2467,7 @@ TEST_F(OpTest, Reshape) {
 TEST_F(OpTest, Reverse) {
   Repeatedly([this]() {
     std::vector<int64> dims = RandomDims(1);
-    DataType type = Choose<DataType>(kAllXlaTypes);
+    auto type = Choose<DataType>(kAllXlaTypes);
     int64 rank = dims.size();
     return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("Reverse")
                                              .RandomInput(type, dims)
@@ -2426,7 +2478,7 @@ TEST_F(OpTest, Reverse) {
 
 TEST_F(OpTest, ReverseV2) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>(kAllXlaTypes);
+    auto type = Choose<DataType>(kAllXlaTypes);
     std::vector<int64> data_dims = RandomDims();
     Tensor indices = RandomReductionIndices(data_dims.size());
     return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("ReverseV2")
@@ -2452,7 +2504,7 @@ TEST_F(OpTest, Round) {
 
 TEST_F(OpTest, Rsqrt) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>({DT_FLOAT, DT_COMPLEX64});
+    auto type = Choose<DataType>({DT_FLOAT, DT_COMPLEX64});
     return ExpectTfAndXlaOutputsAreClose(
         OpTestBuilder("Rsqrt").RandomInput(type).Attr("T", type));
   });
@@ -2461,7 +2513,7 @@ TEST_F(OpTest, Rsqrt) {
 TEST_F(OpTest, RsqrtGrad) {
   Repeatedly([this]() {
     auto dims = RandomDims();
-    DataType type = Choose<DataType>({DT_FLOAT, DT_COMPLEX64});
+    auto type = Choose<DataType>({DT_FLOAT, DT_COMPLEX64});
     return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("RsqrtGrad")
                                              .RandomInput(type, dims)
                                              .RandomInput(type, dims)
@@ -2471,7 +2523,7 @@ TEST_F(OpTest, RsqrtGrad) {
 
 TEST_F(OpTest, Shape) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>(kAllXlaTypes);
+    auto type = Choose<DataType>(kAllXlaTypes);
     return ExpectTfAndXlaOutputsAreClose(
         OpTestBuilder("Shape").RandomInput(type).Attr("T", type));
   });
@@ -2479,7 +2531,7 @@ TEST_F(OpTest, Shape) {
 
 TEST_F(OpTest, ShapeN) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>(kAllXlaTypes);
+    auto type = Choose<DataType>(kAllXlaTypes);
     int n = std::uniform_int_distribution<int>(1, 5)(generator());
     OpTestBuilder builder("ShapeN");
     builder.Attr("T", type);
@@ -2493,7 +2545,7 @@ TEST_F(OpTest, ShapeN) {
 
 TEST_F(OpTest, Sigmoid) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>({DT_FLOAT, DT_COMPLEX64});
+    auto type = Choose<DataType>({DT_FLOAT, DT_COMPLEX64});
     return ExpectTfAndXlaOutputsAreClose(
         OpTestBuilder("Sigmoid").RandomInput(type).Attr("T", type));
   });
@@ -2502,7 +2554,7 @@ TEST_F(OpTest, Sigmoid) {
 TEST_F(OpTest, SigmoidGrad) {
   Repeatedly([this]() {
     auto dims = RandomDims();
-    DataType type = Choose<DataType>({DT_FLOAT, DT_COMPLEX64});
+    auto type = Choose<DataType>({DT_FLOAT, DT_COMPLEX64});
     return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("SigmoidGrad")
                                              .RandomInput(type, dims)
                                              .RandomInput(type, dims)
@@ -2512,7 +2564,7 @@ TEST_F(OpTest, SigmoidGrad) {
 
 TEST_F(OpTest, Sign) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
+    auto type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
     return ExpectTfAndXlaOutputsAreClose(
         OpTestBuilder("Sign").RandomInput(type).Attr("T", type));
   });
@@ -2520,7 +2572,7 @@ TEST_F(OpTest, Sign) {
 
 TEST_F(OpTest, Sin) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>({DT_FLOAT, DT_COMPLEX64});
+    auto type = Choose<DataType>({DT_FLOAT, DT_COMPLEX64});
     return ExpectTfAndXlaOutputsAreClose(
         OpTestBuilder("Sin").RandomInput(type).Attr("T", type));
   });
@@ -2528,7 +2580,7 @@ TEST_F(OpTest, Sin) {
 
 TEST_F(OpTest, Sinh) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>({DT_FLOAT, DT_COMPLEX64});
+    auto type = Choose<DataType>({DT_FLOAT, DT_COMPLEX64});
     return ExpectTfAndXlaOutputsAreClose(
         OpTestBuilder("Sinh").RandomInput(type).Attr("T", type));
   });
@@ -2536,7 +2588,7 @@ TEST_F(OpTest, Sinh) {
 
 TEST_F(OpTest, Size) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>(kAllXlaTypes);
+    auto type = Choose<DataType>(kAllXlaTypes);
     return ExpectTfAndXlaOutputsAreClose(
         OpTestBuilder("Size").RandomInput(type).Attr("T", type));
   });
@@ -2544,7 +2596,7 @@ TEST_F(OpTest, Size) {
 
 TEST_F(OpTest, Slice) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>(kAllXlaTypes);
+    auto type = Choose<DataType>(kAllXlaTypes);
     std::vector<int64> data_dims = RandomDims();
 
     std::vector<int32> begin(data_dims.size()), size(data_dims.size());
@@ -2648,7 +2700,7 @@ TEST_F(OpTest, SpaceToBatch) {
     CHECK(paddings.CopyFrom(AsIntTensor(DT_INT32, padding_vals),
                             TensorShape({num_block_dims, 2})));
 
-    DataType type = Choose<DataType>(kAllXlaTypes);
+    auto type = Choose<DataType>(kAllXlaTypes);
     return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("SpaceToBatch")
                                              .RandomInput(type, input_dims)
                                              .Input(paddings)
@@ -2690,7 +2742,7 @@ TEST_F(OpTest, SpaceToBatchND) {
     CHECK(paddings.CopyFrom(AsIntTensor(DT_INT32, padding_vals),
                             TensorShape({num_block_dims, 2})));
 
-    DataType type = Choose<DataType>(kAllXlaTypes);
+    auto type = Choose<DataType>(kAllXlaTypes);
     return ExpectTfAndXlaOutputsAreClose(
         OpTestBuilder("SpaceToBatchND")
             .RandomInput(type, input_dims)
@@ -2767,7 +2819,7 @@ TEST_F(OpTest, SparseSoftmaxCrossEntropyWithLogits) {
 
 TEST_F(OpTest, Split) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>(kAllXlaTypes);
+    auto type = Choose<DataType>(kAllXlaTypes);
     std::vector<int64> dims = RandomDims(1);
     std::uniform_int_distribution<int> ud;
     int32 dim = std::uniform_int_distribution<int32>(
@@ -2787,7 +2839,7 @@ TEST_F(OpTest, Split) {
 
 TEST_F(OpTest, Sqrt) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>({DT_FLOAT, DT_COMPLEX64});
+    auto type = Choose<DataType>({DT_FLOAT, DT_COMPLEX64});
     return ExpectTfAndXlaOutputsAreClose(
         OpTestBuilder("Sqrt").RandomInput(type).Attr("T", type));
   });
@@ -2796,7 +2848,7 @@ TEST_F(OpTest, Sqrt) {
 TEST_F(OpTest, SqrtGrad) {
   Repeatedly([this]() {
     auto dims = RandomDims();
-    DataType type = Choose<DataType>({DT_FLOAT, DT_COMPLEX64});
+    auto type = Choose<DataType>({DT_FLOAT, DT_COMPLEX64});
     return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("SqrtGrad")
                                              .RandomInput(type, dims)
                                              .RandomInput(type, dims)
@@ -2816,7 +2868,7 @@ TEST_F(OpTest, SquaredDifference) {
 
 TEST_F(OpTest, Square) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
+    auto type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
     return ExpectTfAndXlaOutputsAreClose(
         OpTestBuilder("Square").RandomInput(type).Attr("T", type));
   });
@@ -2824,7 +2876,7 @@ TEST_F(OpTest, Square) {
 
 TEST_F(OpTest, Squeeze) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>(kAllXlaTypes);
+    auto type = Choose<DataType>(kAllXlaTypes);
     std::vector<int64> t_dims = RandomDims(0, kDefaultMaxRank, 0, 5);
     std::bernoulli_distribution random_bool;
     std::vector<int> squeeze_dims;
@@ -2842,7 +2894,7 @@ TEST_F(OpTest, Squeeze) {
 
 TEST_F(OpTest, Sub) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
+    auto type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
     auto dims = BroadcastableDims();
     return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("Sub")
                                              .RandomInput(type, dims.first)
@@ -2853,7 +2905,7 @@ TEST_F(OpTest, Sub) {
 
 TEST_F(OpTest, Sum) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
+    auto type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
     std::vector<int64> data_dims = RandomDims();
     Tensor indices = RandomReductionIndices(data_dims.size());
     bool keep_dims = Choose<bool>({false, true});
@@ -2867,7 +2919,7 @@ TEST_F(OpTest, Sum) {
 
 TEST_F(OpTest, StridedSlice) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>(kAllXlaTypes);
+    auto type = Choose<DataType>(kAllXlaTypes);
     std::vector<int64> data_dims = RandomDims();
     std::vector<int32> begin(data_dims.size()), end(data_dims.size());
     std::vector<int32> strides(data_dims.size());
@@ -2912,7 +2964,7 @@ TEST_F(OpTest, StridedSlice) {
 
 TEST_F(OpTest, StridedSliceGrad) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>(kAllXlaTypes);
+    auto type = Choose<DataType>(kAllXlaTypes);
 
     // Dimensions of the forward input.
     std::vector<int64> dims = RandomDims();
@@ -2965,7 +3017,7 @@ TEST_F(OpTest, StridedSliceGrad) {
 
 TEST_F(OpTest, Tan) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>({DT_FLOAT, DT_COMPLEX64});
+    auto type = Choose<DataType>({DT_FLOAT, DT_COMPLEX64});
     return ExpectTfAndXlaOutputsAreClose(
         OpTestBuilder("Tan").RandomInput(type).Attr("T", type));
   });
@@ -2973,7 +3025,7 @@ TEST_F(OpTest, Tan) {
 
 TEST_F(OpTest, Tanh) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>({DT_FLOAT, DT_COMPLEX64});
+    auto type = Choose<DataType>({DT_FLOAT, DT_COMPLEX64});
     return ExpectTfAndXlaOutputsAreClose(
         OpTestBuilder("Tanh").RandomInput(type).Attr("T", type));
   });
@@ -2982,7 +3034,7 @@ TEST_F(OpTest, Tanh) {
 TEST_F(OpTest, TanhGrad) {
   Repeatedly([this]() {
     auto dims = RandomDims();
-    DataType type = Choose<DataType>({DT_FLOAT, DT_COMPLEX64});
+    auto type = Choose<DataType>({DT_FLOAT, DT_COMPLEX64});
     return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("TanhGrad")
                                              .RandomInput(type, dims)
                                              .RandomInput(type, dims)
@@ -2992,7 +3044,7 @@ TEST_F(OpTest, TanhGrad) {
 
 TEST_F(OpTest, Tile) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>(kAllXlaTypes);
+    auto type = Choose<DataType>(kAllXlaTypes);
     std::vector<int64> t_dims = RandomDims(1);
     std::vector<int32> multiples(t_dims.size());
     for (int i = 0; i < t_dims.size(); ++i) {
@@ -3008,7 +3060,7 @@ TEST_F(OpTest, Tile) {
 
 TEST_F(OpTest, Transpose) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>(kAllXlaTypes);
+    auto type = Choose<DataType>(kAllXlaTypes);
     std::vector<int64> data_dims = RandomDims();
     std::vector<int32> perm(data_dims.size());
     std::iota(perm.begin(), perm.end(), 0);
@@ -3033,7 +3085,7 @@ TEST_F(OpTest, TruncateDiv) {
 
 TEST_F(OpTest, TruncateMod) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT});
+    auto type = Choose<DataType>({DT_INT32, DT_FLOAT});
     auto dims = BroadcastableDims();
     return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("TruncateMod")
                                              .RandomInput(type, dims.first)
@@ -3044,7 +3096,7 @@ TEST_F(OpTest, TruncateMod) {
 
 TEST_F(OpTest, ZerosLike) {
   Repeatedly([this]() {
-    DataType type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
+    auto type = Choose<DataType>({DT_INT32, DT_FLOAT, DT_COMPLEX64});
     return ExpectTfAndXlaOutputsAreClose(
         OpTestBuilder("ZerosLike").RandomInput(type).Attr("T", type));
   });
diff --git a/tensorflow/compiler/tf2xla/kernels/unary_ops.cc b/tensorflow/compiler/tf2xla/kernels/unary_ops.cc
index b35f6fc2e0..a266e9013c 100644
--- a/tensorflow/compiler/tf2xla/kernels/unary_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/unary_ops.cc
@@ -45,7 +45,7 @@ XLAJIT_MAKE_UNARY(ComplexAbs, b->Abs(x));
 
 XLAJIT_MAKE_UNARY(Angle, b->Atan2(b->Imag(x), b->Real(x)));
 
-XLAJIT_MAKE_UNARY(Conj, b->Complex(b->Real(x), b->Neg(b->Imag(x))));
+XLAJIT_MAKE_UNARY(Conj, b->Conj(x));
 
 // Return x if x>0, otherwise -x.
 XLAJIT_MAKE_UNARY(Abs, b->Abs(x));
-- 
GitLab


From e9af1af4f2ca7dd0d767ff60cb08034f37de44ad Mon Sep 17 00:00:00 2001
From: Amit Patankar <amitpatankar@google.com>
Date: Fri, 27 Oct 2017 13:36:53 -0700
Subject: [PATCH 884/909] Fixing the sources docs in master.

---
 tensorflow/docs_src/install/install_sources.md | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/tensorflow/docs_src/install/install_sources.md b/tensorflow/docs_src/install/install_sources.md
index b853d87816..bef8b7ad02 100644
--- a/tensorflow/docs_src/install/install_sources.md
+++ b/tensorflow/docs_src/install/install_sources.md
@@ -455,11 +455,11 @@ Stack Overflow and specify the `tensorflow` tag.
 <table>
 <tr><th>Version:</th><th>CPU/GPU:</th><th>Python Version:</th><th>Compiler:</th><th>Build Tools:</th><th>cuDNN:</th><th>CUDA:</th></tr>
 <tr><td>tensorflow-1.4.0rc0</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>Clang from xcode</td><td>Bazel 0.4.5</td><td>N/A</td><td>N/A</td></tr>
-<tr><td>ttensorflow-1.2.0</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>Clang from xcode</td><td>Bazel 0.4.5</td><td>N/A</td><td>N/A</td></tr>
-<tr><td>ttensorflow-1.1.0</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>Clang from xcode</td><td>Bazel 0.4.2</td><td>N/A</td><td>N/A</td></tr>
-<tr><td>ttensorflow_gpu-1.1.0</td><td>GPU</td><td>2.7, 3.3-3.6</td><td>Clang from xcode</td><td>Bazel 0.4.2</td><td>5.1</td><td>8</td></tr>
-<tr><td>ttensorflow-1.0.0</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>Clang from xcode</td><td>Bazel 0.4.2</td><td>N/A</td><td>N/A</td></tr>
-<tr><td>ttensorflow_gpu-1.0.0</td><td>GPU</td><td>2.7, 3.3-3.6</td><td>Clang from xcode</td><td>Bazel 0.4.2</td><td>5.1</td><td>8</td></tr>
+<tr><td>tensorflow-1.2.0</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>Clang from xcode</td><td>Bazel 0.4.5</td><td>N/A</td><td>N/A</td></tr>
+<tr><td>tensorflow-1.1.0</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>Clang from xcode</td><td>Bazel 0.4.2</td><td>N/A</td><td>N/A</td></tr>
+<tr><td>tensorflow_gpu-1.1.0</td><td>GPU</td><td>2.7, 3.3-3.6</td><td>Clang from xcode</td><td>Bazel 0.4.2</td><td>5.1</td><td>8</td></tr>
+<tr><td>tensorflow-1.0.0</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>Clang from xcode</td><td>Bazel 0.4.2</td><td>N/A</td><td>N/A</td></tr>
+<tr><td>tensorflow_gpu-1.0.0</td><td>GPU</td><td>2.7, 3.3-3.6</td><td>Clang from xcode</td><td>Bazel 0.4.2</td><td>5.1</td><td>8</td></tr>
 </table>
 
 **Windows**
-- 
GitLab


From 0bc432a443e70bcd38326291f9746f965ffd1de2 Mon Sep 17 00:00:00 2001
From: Shanqing Cai <cais@google.com>
Date: Fri, 27 Oct 2017 13:39:08 -0700
Subject: [PATCH 885/909] TFE: Add compatibility errors and doc strings to
 queues, input pipelines and Supervisor

PiperOrigin-RevId: 173712330
---
 tensorflow/python/training/input.py           | 74 ++++++++++++++++---
 .../python/training/queue_runner_impl.py      | 12 +++
 tensorflow/python/training/supervisor.py      | 20 +++++
 3 files changed, 94 insertions(+), 12 deletions(-)

diff --git a/tensorflow/python/training/input.py b/tensorflow/python/training/input.py
index e7adbf11b4..f645d8cf39 100644
--- a/tensorflow/python/training/input.py
+++ b/tensorflow/python/training/input.py
@@ -149,14 +149,15 @@ def input_producer(input_tensor,
     RuntimeError: If called with eager execution enabled.
 
   @compatibility(eager)
-  Queue-using input pipelines are not supported when eager execution is enabled.
-  Please use tf.data to ingest data into your model instead.
+  Input pipelines based on Queues are not supported when eager execution is
+  enabled. Please use the `tf.data` API to ingest data under eager execution.
   @end_compatibility
   """
   if context.in_eager_mode():
     raise RuntimeError(
-        "Queue-using input pipelines are not supported when eager execution is"
-        " enabled. Please use tf.data to ingest data into your model instead.")
+        "Input pipelines based on Queues are not supported when eager execution"
+        " is enabled. Please use tf.data to ingest data into your model"
+        " instead.")
   with ops.name_scope(name, "input_producer", [input_tensor]):
     input_tensor = ops.convert_to_tensor(input_tensor, name="input_tensor")
     element_shape = input_tensor.shape[1:].merge_with(element_shape)
@@ -222,6 +223,11 @@ def string_input_producer(string_tensor,
   Raises:
     ValueError: If the string_tensor is a null Python list.  At runtime,
     will fail with an assertion if string_tensor becomes a null tensor.
+
+  @compatibility(eager)
+  Input pipelines based on Queues are not supported when eager execution is
+  enabled. Please use the `tf.data` API to ingest data under eager execution.
+  @end_compatibility
   """
   not_null_err = "string_input_producer requires a non-null input tensor"
   if not isinstance(string_tensor, ops.Tensor) and not string_tensor:
@@ -271,6 +277,11 @@ def range_input_producer(limit, num_epochs=None, shuffle=True, seed=None,
   Returns:
     A Queue with the output integers.  A `QueueRunner` for the Queue
     is added to the current `Graph`'s `QUEUE_RUNNER` collection.
+
+  @compatibility(eager)
+  Input pipelines based on Queues are not supported when eager execution is
+  enabled. Please use the `tf.data` API to ingest data under eager execution.
+  @end_compatibility
   """
   with ops.name_scope(name, "input_producer", [limit]) as name:
     range_tensor = math_ops.range(limit)
@@ -308,6 +319,11 @@ def slice_input_producer(tensor_list, num_epochs=None, shuffle=True, seed=None,
 
   Raises:
     ValueError: if `slice_input_producer` produces nothing from `tensor_list`.
+
+  @compatibility(eager)
+  Input pipelines based on Queues are not supported when eager execution is
+  enabled. Please use the `tf.data` API to ingest data under eager execution.
+  @end_compatibility
   """
   with ops.name_scope(name, "input_producer", tensor_list):
     tensor_list = ops.convert_n_to_tensor_or_indexed_slices(tensor_list)
@@ -698,8 +714,9 @@ def _batch(tensors, batch_size, keep_input, num_threads=1, capacity=32,
   """Helper function for `batch` and `maybe_batch`."""
   if context.in_eager_mode():
     raise ValueError(
-        "Queue-using input pipelines are not supported when eager execution is"
-        " enabled. Please use tf.data to ingest data into your model instead.")
+        "Input pipelines based on Queues are not supported when eager execution"
+        " is enabled. Please use tf.data to ingest data into your model"
+        " instead.")
   tensor_list = _as_tensor_list(tensors)
   with ops.name_scope(name, "batch", list(tensor_list) + [keep_input]) as name:
     tensor_list = _validate(tensor_list)
@@ -735,8 +752,9 @@ def _batch_join(tensors_list, batch_size, keep_input, capacity=32,
   """Helper function for `batch_join` and `maybe_batch_join`."""
   if context.in_eager_mode():
     raise ValueError(
-        "Queue-using input pipelines are not supported when eager execution is"
-        " enabled. Please use tf.data to ingest data into your model instead.")
+        "Input pipelines based on Queues are not supported when eager execution"
+        " is enabled. Please use tf.data to ingest data into your model"
+        " instead.")
   tensor_list_list = _as_tensor_list_list(tensors_list)
   with ops.name_scope(name, "batch_join",
                       _flatten(tensor_list_list) + [keep_input]) as name:
@@ -769,8 +787,9 @@ def _shuffle_batch(tensors, batch_size, capacity, min_after_dequeue,
   """Helper function for `shuffle_batch` and `maybe_shuffle_batch`."""
   if context.in_eager_mode():
     raise ValueError(
-        "Queue-using input pipelines are not supported when eager execution is"
-        " enabled. Please use tf.data to ingest data into your model instead.")
+        "Input pipelines based on Queues are not supported when eager execution"
+        " is enabled. Please use tf.data to ingest data into your model"
+        " instead.")
   tensor_list = _as_tensor_list(tensors)
   with ops.name_scope(name, "shuffle_batch",
                       list(tensor_list) + [keep_input]) as name:
@@ -813,8 +832,9 @@ def _shuffle_batch_join(tensors_list, batch_size, capacity,
   """Helper function for `shuffle_batch_join` and `maybe_shuffle_batch_join`."""
   if context.in_eager_mode():
     raise ValueError(
-        "Queue-using input pipelines are not supported when eager execution is"
-        " enabled. Please use tf.data to ingest data into your model instead.")
+        "Input pipelines based on Queues are not supported when eager execution"
+        " is enabled. Please use tf.data to ingest data into your model"
+        " instead.")
   tensor_list_list = _as_tensor_list_list(tensors_list)
   with ops.name_scope(name, "shuffle_batch_join",
                       _flatten(tensor_list_list) + [keep_input]) as name:
@@ -923,6 +943,11 @@ def batch(tensors, batch_size, num_threads=1, capacity=32,
   Raises:
     ValueError: If the `shapes` are not specified, and cannot be
       inferred from the elements of `tensors`.
+
+  @compatibility(eager)
+  Input pipelines based on Queues are not supported when eager execution is
+  enabled. Please use the `tf.data` API to ingest data under eager execution.
+  @end_compatibility
   """
   return _batch(
       tensors,
@@ -1076,6 +1101,11 @@ def batch_join(tensors_list, batch_size, capacity=32, enqueue_many=False,
   Raises:
     ValueError: If the `shapes` are not specified, and cannot be
       inferred from the elements of `tensor_list_list`.
+
+  @compatibility(eager)
+  Input pipelines based on Queues are not supported when eager execution is
+  enabled. Please use the `tf.data` API to ingest data under eager execution.
+  @end_compatibility
   """
   return _batch_join(
       tensors_list,
@@ -1220,6 +1250,11 @@ def shuffle_batch(tensors, batch_size, capacity, min_after_dequeue,
   Raises:
     ValueError: If the `shapes` are not specified, and cannot be
       inferred from the elements of `tensors`.
+
+  @compatibility(eager)
+  Input pipelines based on Queues are not supported when eager execution is
+  enabled. Please use the `tf.data` API to ingest data under eager execution.
+  @end_compatibility
   """
   return _shuffle_batch(
       tensors,
@@ -1274,6 +1309,11 @@ def maybe_shuffle_batch(tensors, batch_size, capacity, min_after_dequeue,
   Raises:
     ValueError: If the `shapes` are not specified, and cannot be
       inferred from the elements of `tensors`.
+
+  @compatibility(eager)
+  Input pipelines based on Queues are not supported when eager execution is
+  enabled. Please use the `tf.data` API to ingest data under eager execution.
+  @end_compatibility
   """
   return _shuffle_batch(
       tensors,
@@ -1363,6 +1403,11 @@ def shuffle_batch_join(tensors_list, batch_size, capacity,
   Raises:
     ValueError: If the `shapes` are not specified, and cannot be
       inferred from the elements of `tensors_list`.
+
+  @compatibility(eager)
+  Input pipelines based on Queues are not supported when eager execution is
+  enabled. Please use the `tf.data` API to ingest data under eager execution.
+  @end_compatibility
   """
   return _shuffle_batch_join(
       tensors_list,
@@ -1417,6 +1462,11 @@ def maybe_shuffle_batch_join(tensors_list, batch_size, capacity,
   Raises:
     ValueError: If the `shapes` are not specified, and cannot be
       inferred from the elements of `tensors_list`.
+
+  @compatibility(eager)
+  Input pipelines based on Queues are not supported when eager execution is
+  enabled. Please use the `tf.data` API to ingest data under eager execution.
+  @end_compatibility
   """
   return _shuffle_batch_join(
       tensors_list,
diff --git a/tensorflow/python/training/queue_runner_impl.py b/tensorflow/python/training/queue_runner_impl.py
index 5abc6a2f58..d3b473ee46 100644
--- a/tensorflow/python/training/queue_runner_impl.py
+++ b/tensorflow/python/training/queue_runner_impl.py
@@ -23,6 +23,7 @@ import weakref
 
 from tensorflow.core.protobuf import queue_runner_pb2
 from tensorflow.python.client import session
+from tensorflow.python.eager import context
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
 from tensorflow.python.platform import tf_logging as logging
@@ -414,7 +415,18 @@ def start_queue_runners(sess=None, coord=None, daemon=True, start=True,
 
   Returns:
     A list of threads.
+
+  Raises:
+    RuntimeError: If called with eager execution enabled.
+    ValueError: If called without a default `tf.Session` registered.
+
+  @compatibility(eager)
+  Not compatible with eager execution. To ingest data under eager execution,
+  use the `tf.data` API instead.
+  @end_compatibility
   """
+  if context.in_eager_mode():
+    raise RuntimeError("Queues are not compatible with eager execution.")
   if sess is None:
     sess = ops.get_default_session()
     if not sess:
diff --git a/tensorflow/python/training/supervisor.py b/tensorflow/python/training/supervisor.py
index 41dbf6b497..a634a842b6 100644
--- a/tensorflow/python/training/supervisor.py
+++ b/tensorflow/python/training/supervisor.py
@@ -23,6 +23,7 @@ import time
 
 from tensorflow.core.framework.summary_pb2 import Summary
 from tensorflow.core.util.event_pb2 import SessionLog
+from tensorflow.python.eager import context
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import meta_graph
 from tensorflow.python.framework import ops
@@ -288,7 +289,16 @@ class Supervisor(object):
 
     Returns:
       A `Supervisor`.
+
+    Raises:
+      RuntimeError: If called with eager execution enabled.
+
+    @compatibility(eager)
+    `Supervisor`s are not supported when eager execution is enabled.
+    @end_compatibility
     """
+    if context.in_eager_mode():
+      raise RuntimeError("Supervisors are compatible with eager execution.")
     # Set default values of arguments.
     if graph is None:
       graph = ops.get_default_graph()
@@ -735,7 +745,17 @@ class Supervisor(object):
 
     Returns:
       The list of threads started for the `QueueRunners`.
+
+    Raises:
+      RuntimeError: If called with eager execution enabled.
+
+    @compatibility(eager)
+    Queues are not compatible with eager execution. To ingest data when eager
+    execution is enabled, use the `tf.data` API.
+    @end_compatibility
     """
+    if context.in_eager_mode():
+      raise RuntimeError("Queues are not compatible with eager execution.")
     if queue_runners is None:
       queue_runners = self._graph.get_collection(ops.GraphKeys.QUEUE_RUNNERS)
     threads = []
-- 
GitLab


From 9bf00c3717d24fbc6bfad2a99c7c08f39f534aa3 Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Fri, 27 Oct 2017 14:10:36 -0700
Subject: [PATCH 886/909] Shorter import for tfe.

PiperOrigin-RevId: 173716375
---
 tensorflow/contrib/__init__.py           | 1 +
 tensorflow/contrib/cmake/tf_python.cmake | 2 ++
 2 files changed, 3 insertions(+)

diff --git a/tensorflow/contrib/__init__.py b/tensorflow/contrib/__init__.py
index 76a629663d..a26fdb982c 100644
--- a/tensorflow/contrib/__init__.py
+++ b/tensorflow/contrib/__init__.py
@@ -77,6 +77,7 @@ from tensorflow.contrib import timeseries
 from tensorflow.contrib import tpu
 from tensorflow.contrib import training
 from tensorflow.contrib import util
+from tensorflow.contrib.eager.python import tfe as eager
 from tensorflow.contrib.ndlstm import python as ndlstm
 from tensorflow.contrib.remote_fused_graph import pylib as remote_fused_graph
 from tensorflow.contrib.specs import python as specs
diff --git a/tensorflow/contrib/cmake/tf_python.cmake b/tensorflow/contrib/cmake/tf_python.cmake
index a3ed19977f..1b9fd514fd 100755
--- a/tensorflow/contrib/cmake/tf_python.cmake
+++ b/tensorflow/contrib/cmake/tf_python.cmake
@@ -347,6 +347,8 @@ add_python_module("tensorflow/contrib/distributions/python")
 add_python_module("tensorflow/contrib/distributions/python/kernel_tests")
 add_python_module("tensorflow/contrib/distributions/python/ops")
 add_python_module("tensorflow/contrib/distributions/python/ops/bijectors")
+add_python_module("tensorflow/contrib/eager")
+add_python_module("tensorflow/contrib/eager/python")
 add_python_module("tensorflow/contrib/estimator")
 add_python_module("tensorflow/contrib/estimator/python")
 add_python_module("tensorflow/contrib/estimator/python/estimator")
-- 
GitLab


From 78bac7290c4c49c27ca61aa891ae564c54e2ddfc Mon Sep 17 00:00:00 2001
From: Shanqing Cai <cais@google.com>
Date: Fri, 27 Oct 2017 14:15:12 -0700
Subject: [PATCH 887/909] TFE: Add compatbility doc string to
 add_to_collection() and friends

PiperOrigin-RevId: 173716912
---
 tensorflow/python/framework/ops.py | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index c8ee9243d7..63f70a1a9d 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -4975,6 +4975,10 @@ def add_to_collection(name, value):
     name: The key for the collection. For example, the `GraphKeys` class
       contains many standard names for collections.
     value: The value to add to the collection.
+
+  @compatibility(eager)
+  Collections are not supported when eager execution is enabled.
+  @end_compatibility
   """
   get_default_graph().add_to_collection(name, value)
 
@@ -4989,6 +4993,10 @@ def add_to_collections(names, value):
     names: The key for the collections. The `GraphKeys` class
       contains many standard names for collections.
     value: The value to add to the collections.
+
+  @compatibility(eager)
+  Collections are not supported when eager execution is enabled.
+  @end_compatibility
   """
   get_default_graph().add_to_collections(names, value)
 
@@ -5008,6 +5016,10 @@ def get_collection_ref(key):
     list if no value has been added to that collection.  Note that this returns
     the collection list itself, which can be modified in place to change the
     collection.
+
+  @compatibility(eager)
+  Collections are not supported when eager execution is enabled.
+  @end_compatibility
   """
   return get_default_graph().get_collection_ref(key)
 
@@ -5032,6 +5044,10 @@ def get_collection(key, scope=None):
     an empty list if no value has been added to that collection. The
     list contains the values in the order under which they were
     collected.
+
+  @compatibility(eager)
+  Collections are not supported when eager execution is enabled.
+  @end_compatibility
   """
   return get_default_graph().get_collection(key, scope)
 
-- 
GitLab


From 02f55400f87b22f7ea0849c39022792d1e381afb Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Fri, 27 Oct 2017 15:08:01 -0700
Subject: [PATCH 888/909] custom_gradient functions should be able to return
 their inputs

PiperOrigin-RevId: 173723462
---
 tensorflow/python/eager/backprop_test.py   | 12 ++++++++++++
 tensorflow/python/eager/custom_gradient.py |  7 +++++--
 2 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/eager/backprop_test.py b/tensorflow/python/eager/backprop_test.py
index d18df4dffb..20532c8ee8 100644
--- a/tensorflow/python/eager/backprop_test.py
+++ b/tensorflow/python/eager/backprop_test.py
@@ -569,5 +569,17 @@ class BackpropTest(test.TestCase):
         var.assign_sub(lr*grad)
     self.assertAllEqual(losses, [4.0, 3., 2., 1., 0.])
 
+  def testCustomGradientIdentity(self):
+
+    @custom_gradient.custom_gradient
+    def my_identity(x):
+
+      def grad(dresult):
+        return [2 * dresult]
+
+      return x, grad
+
+    self.assertAllEqual(backprop.gradients_function(my_identity)(1.0)[0], 2.0)
+
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/python/eager/custom_gradient.py b/tensorflow/python/eager/custom_gradient.py
index 4ac30075b2..05460ff996 100644
--- a/tensorflow/python/eager/custom_gradient.py
+++ b/tensorflow/python/eager/custom_gradient.py
@@ -22,6 +22,7 @@ from tensorflow.python.eager import context
 from tensorflow.python.eager import tape
 from tensorflow.python.framework import ops as tf_ops
 from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import gen_array_ops
 from tensorflow.python.util import nest
 from tensorflow.python.util import tf_decorator
 
@@ -72,17 +73,19 @@ def custom_gradient(f):
 
     with tape.stop_recording():
       result, grad_fn = f(*args, **kwargs)
+      flat_result = nest.flatten(result)
+      # TODO(apassos) consider removing the identity below.
+      flat_result = [gen_array_ops.identity(x) for x in flat_result]
 
     def actual_grad_fn(*outputs):
       return nest.flatten(grad_fn(*outputs))
 
-    flat_result = nest.flatten(result)
     tape.record_operation(
         f.__name__,
         flat_result,
         input_tensors,
         actual_grad_fn)
     flat_result = list(flat_result)
-    return result
+    return nest.pack_sequence_as(result, flat_result)
 
   return tf_decorator.make_decorator(f, decorated)
-- 
GitLab


From 5426a3c93d8a180b7009ba87af12c61dc1a6278d Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Fri, 27 Oct 2017 15:35:06 -0700
Subject: [PATCH 889/909] Add tfe.get_optimizer_variables for fetching a list
 of variables which an optimizer has created. Useful for saving them if
 executing eagerly.

PiperOrigin-RevId: 173726859
---
 tensorflow/contrib/eager/python/saver.py      | 23 +++++++++++
 tensorflow/contrib/eager/python/saver_test.py | 41 +++++++++++++++++++
 tensorflow/contrib/eager/python/tfe.py        |  2 +
 3 files changed, 66 insertions(+)

diff --git a/tensorflow/contrib/eager/python/saver.py b/tensorflow/contrib/eager/python/saver.py
index d74e0fef3e..e0a20d2485 100644
--- a/tensorflow/contrib/eager/python/saver.py
+++ b/tensorflow/contrib/eager/python/saver.py
@@ -23,6 +23,7 @@ from tensorflow.python.eager import context
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import resource_variable_ops
+from tensorflow.python.training import adam as _adam
 from tensorflow.python.training import checkpoint_utils
 from tensorflow.python.training import saver as _saver
 
@@ -165,3 +166,25 @@ class Saver(object):
     """
     with ops.device("/device:CPU:0"):
       self._saver.restore(None, file_prefix)
+
+
+def get_optimizer_variables(optimizer):
+  """Returns a list of variables for the given `tf.train.Optimizer`.
+
+  Args:
+    optimizer: An instance of `tf.train.Optimizer` which has created variables
+      (typically after a call to `Optimizer.minimize`).
+  Returns:
+    A list of variables which have been created by the `Optimizer`. Currently
+    returns all variables even if they were not created in the default graph,
+    but this behavior may change.
+  """
+  variables = []
+  # pylint: disable=protected-access
+  for _, variable_dict in optimizer._slots.items():
+    for _, slot_for_variable in variable_dict.items():
+      variables.append(slot_for_variable)
+  if isinstance(optimizer, _adam.AdamOptimizer):
+    variables.append(optimizer._beta1_power)
+    variables.append(optimizer._beta2_power)
+  return variables
diff --git a/tensorflow/contrib/eager/python/saver_test.py b/tensorflow/contrib/eager/python/saver_test.py
index 3c69b90242..abc7e3690c 100644
--- a/tensorflow/contrib/eager/python/saver_test.py
+++ b/tensorflow/contrib/eager/python/saver_test.py
@@ -30,6 +30,10 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import init_ops
 from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.ops import variable_scope
+from tensorflow.python.training import adam
+from tensorflow.python.training import gradient_descent
+from tensorflow.python.training import momentum
+from tensorflow.python.training import rmsprop
 
 
 class SaverTest(test.TestCase):
@@ -204,5 +208,42 @@ class SaverTest(test.TestCase):
               3, model2(array_ops.constant(2, dtype=dtypes.float32)).numpy())
 
 
+class GetOptimizerTests(test.TestCase):
+
+  def _optimizer_test_template(self, optimizer):
+    """Checks save and restore. Returns the optimizer variables."""
+    v = resource_variable_ops.ResourceVariable([[2., 3.]], name='v')
+    loss_fn = lambda: v[0, 0] ** 2 + v[0, 1] ** 2
+    optimizer.minimize(loss_fn)
+    optimizer_variables = _saver.get_optimizer_variables(optimizer)
+    saver = _saver.Saver(optimizer_variables + [v])
+    checkpoint_path = saver.save(self.get_temp_dir())
+    optimizer.minimize(loss_fn)
+    after_first_minimize = v.numpy()
+    # After we restore, the next step should be exactly the same as the one we
+    # just did.
+    saver.restore(checkpoint_path)
+    optimizer.minimize(loss_fn)
+    self.assertAllEqual(after_first_minimize, v.numpy())
+    return optimizer_variables
+
+  def testAdam(self):
+    optimizer = adam.AdamOptimizer(0.1)
+    self._optimizer_test_template(optimizer)
+
+  def testGradientDescent(self):
+    optimizer = gradient_descent.GradientDescentOptimizer(0.02)
+    self.assertEqual(0, len(self._optimizer_test_template(optimizer)))
+
+  def testMomentum(self):
+    optimizer = momentum.MomentumOptimizer(
+        learning_rate=0.03,
+        momentum=0.5)
+    self._optimizer_test_template(optimizer)
+
+  def testRMSProp(self):
+    optimizer = rmsprop.RMSPropOptimizer(0.01)
+    self._optimizer_test_template(optimizer)
+
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/contrib/eager/python/tfe.py b/tensorflow/contrib/eager/python/tfe.py
index a769140713..ab31893cd3 100644
--- a/tensorflow/contrib/eager/python/tfe.py
+++ b/tensorflow/contrib/eager/python/tfe.py
@@ -51,6 +51,7 @@ To use, at program startup, call `tfe.enable_eager_execution()`.
 @@SummaryWriter
 @@restore_variables_on_create
 @@Variable
+@@get_optimizer_variables
 
 @@in_eager_mode
 @@in_graph_mode
@@ -73,6 +74,7 @@ from __future__ import print_function
 from tensorflow.contrib.eager.python import metrics
 from tensorflow.contrib.eager.python.datasets import Iterator
 from tensorflow.contrib.eager.python.network import Network
+from tensorflow.contrib.eager.python.saver import get_optimizer_variables
 from tensorflow.contrib.eager.python.saver import restore_variables_on_create
 from tensorflow.contrib.eager.python.saver import Saver
 from tensorflow.contrib.eager.python.summary_writer import SummaryWriter
-- 
GitLab


From 80374a7b47dddb591f711b6240ea0896fbe90d29 Mon Sep 17 00:00:00 2001
From: "Joshua V. Dillon" <jvdillon@google.com>
Date: Fri, 27 Oct 2017 15:59:46 -0700
Subject: [PATCH 890/909] Breaking change: Rename
 `tf.contrib.distributions.Independent` parameter from `reduce_batch_ndims` to
 `reinterpreted_batch_ndims`. Also change default; `reinterpreted_batch_ndims`
 default has semantics of `tf.layers.flatten`, i.e., all batch dimensions
 except the first (batch axis 0) are interpretted as being part of the event.

PiperOrigin-RevId: 173729585
---
 .../python/kernel_tests/independent_test.py   |  65 +++++++++-
 .../distributions/python/ops/independent.py   | 113 +++++++++++-------
 2 files changed, 130 insertions(+), 48 deletions(-)

diff --git a/tensorflow/contrib/distributions/python/kernel_tests/independent_test.py b/tensorflow/contrib/distributions/python/kernel_tests/independent_test.py
index dcc66e8972..8e23a3ab8f 100644
--- a/tensorflow/contrib/distributions/python/kernel_tests/independent_test.py
+++ b/tensorflow/contrib/distributions/python/kernel_tests/independent_test.py
@@ -23,7 +23,10 @@ import numpy as np
 
 from tensorflow.contrib.distributions.python.ops import independent as independent_lib
 from tensorflow.contrib.distributions.python.ops import mvn_diag as mvn_diag_lib
+from tensorflow.python.framework import dtypes
+from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
+from tensorflow.python.ops.distributions import bernoulli as bernoulli_lib
 from tensorflow.python.ops.distributions import normal as normal_lib
 from tensorflow.python.platform import test
 from tensorflow.python.platform import tf_logging
@@ -42,13 +45,16 @@ stats = try_import("scipy.stats")
 
 class ProductDistributionTest(test.TestCase):
 
+  def setUp(self):
+    self._rng = np.random.RandomState(42)
+
   def testSampleAndLogProbUnivariate(self):
     loc = np.float32([-1., 1])
     scale = np.float32([0.1, 0.5])
     with self.test_session() as sess:
       ind = independent_lib.Independent(
           distribution=normal_lib.Normal(loc=loc, scale=scale),
-          reduce_batch_ndims=1)
+          reinterpreted_batch_ndims=1)
 
       x = ind.sample([4, 5])
       log_prob_x = ind.log_prob(x)
@@ -71,7 +77,7 @@ class ProductDistributionTest(test.TestCase):
           distribution=mvn_diag_lib.MultivariateNormalDiag(
               loc=loc,
               scale_identity_multiplier=scale),
-          reduce_batch_ndims=1)
+          reinterpreted_batch_ndims=1)
 
       x = ind.sample([4, 5])
       log_prob_x = ind.log_prob(x)
@@ -96,7 +102,7 @@ class ProductDistributionTest(test.TestCase):
           distribution=mvn_diag_lib.MultivariateNormalDiag(
               loc=loc,
               scale_identity_multiplier=scale),
-          reduce_batch_ndims=1)
+          reinterpreted_batch_ndims=1)
 
       x = ind.sample(int(n_samp), seed=42)
       sample_mean = math_ops.reduce_mean(x, axis=0)
@@ -120,6 +126,59 @@ class ProductDistributionTest(test.TestCase):
       self.assertAllClose(sample_entropy_, actual_entropy_, rtol=0.01, atol=0.)
       self.assertAllClose(loc, actual_mode_, rtol=1e-6, atol=0.)
 
+  def _testMnistLike(self, static_shape):
+    sample_shape = [4, 5]
+    batch_shape = [10]
+    image_shape = [28, 28, 1]
+    logits = 3 * self._rng.random_sample(
+        batch_shape + image_shape).astype(np.float32) - 1
+
+    def expected_log_prob(x, logits):
+      return (x * logits - np.log1p(np.exp(logits))).sum(-1).sum(-1).sum(-1)
+
+    with self.test_session() as sess:
+      logits_ph = array_ops.placeholder(
+          dtypes.float32, shape=logits.shape if static_shape else None)
+      ind = independent_lib.Independent(
+          distribution=bernoulli_lib.Bernoulli(logits=logits_ph))
+      x = ind.sample(sample_shape)
+      log_prob_x = ind.log_prob(x)
+      [
+          x_,
+          actual_log_prob_x,
+          ind_batch_shape,
+          ind_event_shape,
+          x_shape,
+          log_prob_x_shape,
+      ] = sess.run([
+          x,
+          log_prob_x,
+          ind.batch_shape_tensor(),
+          ind.event_shape_tensor(),
+          array_ops.shape(x),
+          array_ops.shape(log_prob_x),
+      ], feed_dict={logits_ph: logits})
+
+      if static_shape:
+        ind_batch_shape = ind.batch_shape
+        ind_event_shape = ind.event_shape
+        x_shape = x.shape
+        log_prob_x_shape = log_prob_x.shape
+
+      self.assertAllEqual(batch_shape, ind_batch_shape)
+      self.assertAllEqual(image_shape, ind_event_shape)
+      self.assertAllEqual(sample_shape + batch_shape + image_shape, x_shape)
+      self.assertAllEqual(sample_shape + batch_shape, log_prob_x_shape)
+      self.assertAllClose(expected_log_prob(x_, logits),
+                          actual_log_prob_x,
+                          rtol=1e-6, atol=0.)
+
+  def testMnistLikeStaticShape(self):
+    self._testMnistLike(static_shape=True)
+
+  def testMnistLikeDynamicShape(self):
+    self._testMnistLike(static_shape=False)
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/contrib/distributions/python/ops/independent.py b/tensorflow/contrib/distributions/python/ops/independent.py
index 393c008242..6a74ca9a0a 100644
--- a/tensorflow/contrib/distributions/python/ops/independent.py
+++ b/tensorflow/contrib/distributions/python/ops/independent.py
@@ -45,24 +45,24 @@ class Independent(distribution_lib.Distribution):
   `p(x_1, ..., x_B) = p_1(x_1) * ... * p_B(x_B)` where `p_b(X_b)` is the
   probability of the `b`-th rv. More generally `B, E` can be arbitrary shapes.
 
-  Similarly, the `Independent` distribution specifies a distribution over
-  `[B, E]`-shaped events. It operates by reinterpreting the rightmost batch dims
-  as part of the event dimensions. The `reduce_batch_ndims` parameter controls
-  the number of batch dims which are absorbed as event dims;
-  `reduce_batch_ndims < len(batch_shape)`.  For example, the `log_prob` function
-  entails a `reduce_sum` over the rightmost `reduce_batch_ndims` after calling
-  the base distribution's `log_prob`.  In other words, since the batch
-  dimension(s) index independent distributions, the resultant multivariate will
-  have independent components.
+  Similarly, the `Independent` distribution specifies a distribution over `[B,
+  E]`-shaped events. It operates by reinterpreting the rightmost batch dims as
+  part of the event dimensions. The `reinterpreted_batch_ndims` parameter
+  controls the number of batch dims which are absorbed as event dims;
+  `reinterpreted_batch_ndims < len(batch_shape)`.  For example, the `log_prob`
+  function entails a `reduce_sum` over the rightmost `reinterpreted_batch_ndims`
+  after calling the base distribution's `log_prob`.  In other words, since the
+  batch dimension(s) index independent distributions, the resultant multivariate
+  will have independent components.
 
   #### Mathematical Details
 
   The probability function is,
 
   ```none
-  prob(x; reduce_batch_ndims) = tf.reduce_prod(
+  prob(x; reinterpreted_batch_ndims) = tf.reduce_prod(
       dist.prob(x),
-      axis=-1-range(reduce_batch_ndims))
+      axis=-1-range(reinterpreted_batch_ndims))
   ```
 
   #### Examples
@@ -73,7 +73,7 @@ class Independent(distribution_lib.Distribution):
   # Make independent distribution from a 2-batch Normal.
   ind = ds.Independent(
       distribution=ds.Normal(loc=[-1., 1], scale=[0.1, 0.5]),
-      reduce_batch_ndims=1)
+      reinterpreted_batch_ndims=1)
 
   # All batch dims have been "absorbed" into event dims.
   ind.batch_shape  # ==> []
@@ -84,7 +84,7 @@ class Independent(distribution_lib.Distribution):
       distribution=ds.MultivariateNormalDiag(
           loc=[[-1., 1], [1, -1]],
           scale_identity_multiplier=[1., 0.5]),
-      reduce_batch_ndims=1)
+      reinterpreted_batch_ndims=1)
 
   # All batch dims have been "absorbed" into event dims.
   ind.batch_shape  # ==> []
@@ -94,14 +94,17 @@ class Independent(distribution_lib.Distribution):
   """
 
   def __init__(
-      self, distribution, reduce_batch_ndims=1, validate_args=False, name=None):
+      self, distribution, reinterpreted_batch_ndims=None,
+      validate_args=False, name=None):
     """Construct a `Independent` distribution.
 
     Args:
       distribution: The base distribution instance to transform. Typically an
         instance of `Distribution`.
-      reduce_batch_ndims: Scalar, integer number of rightmost batch dims which
-        will be regard as event dims.
+      reinterpreted_batch_ndims: Scalar, integer number of rightmost batch dims
+        which will be regarded as event dims. When `None` all but the first
+        batch axis (batch axis 0) will be transferred to event dimensions
+        (analogous to `tf.layers.flatten`).
       validate_args: Python `bool`.  Whether to validate input with asserts.
         If `validate_args` is `False`, and the inputs are invalid,
         correct behavior is not guaranteed.
@@ -109,19 +112,25 @@ class Independent(distribution_lib.Distribution):
         Default value: `Independent + distribution.name`.
 
     Raises:
-      ValueError: if `reduce_batch_ndims` exceeds `distribution.batch_ndims`
+      ValueError: if `reinterpreted_batch_ndims` exceeds
+        `distribution.batch_ndims`
     """
     parameters = locals()
     name = name or "Independent" + distribution.name
     self._distribution = distribution
     with ops.name_scope(name):
-      reduce_batch_ndims = ops.convert_to_tensor(
-          reduce_batch_ndims, dtype=dtypes.int32, name="reduce_batch_ndims")
-      self._reduce_batch_ndims = reduce_batch_ndims
-      self._static_reduce_batch_ndims = tensor_util.constant_value(
-          reduce_batch_ndims)
-      if self._static_reduce_batch_ndims is not None:
-        self._reduce_batch_ndims = self._static_reduce_batch_ndims
+      if reinterpreted_batch_ndims is None:
+        reinterpreted_batch_ndims = self._get_default_reinterpreted_batch_ndims(
+            distribution)
+      reinterpreted_batch_ndims = ops.convert_to_tensor(
+          reinterpreted_batch_ndims,
+          dtype=dtypes.int32,
+          name="reinterpreted_batch_ndims")
+      self._reinterpreted_batch_ndims = reinterpreted_batch_ndims
+      self._static_reinterpreted_batch_ndims = tensor_util.constant_value(
+          reinterpreted_batch_ndims)
+      if self._static_reinterpreted_batch_ndims is not None:
+        self._reinterpreted_batch_ndims = self._static_reinterpreted_batch_ndims
       super(Independent, self).__init__(
           dtype=self._distribution.dtype,
           reparameterization_type=self._distribution.reparameterization_type,
@@ -129,19 +138,19 @@ class Independent(distribution_lib.Distribution):
           allow_nan_stats=self._distribution.allow_nan_stats,
           parameters=parameters,
           graph_parents=(
-              [reduce_batch_ndims] +
+              [reinterpreted_batch_ndims] +
               distribution._graph_parents),  # pylint: disable=protected-access
           name=name)
       self._runtime_assertions = self._make_runtime_assertions(
-          distribution, reduce_batch_ndims, validate_args)
+          distribution, reinterpreted_batch_ndims, validate_args)
 
   @property
   def distribution(self):
     return self._distribution
 
   @property
-  def reduce_batch_ndims(self):
-    return self._reduce_batch_ndims
+  def reinterpreted_batch_ndims(self):
+    return self._reinterpreted_batch_ndims
 
   def _batch_shape_tensor(self):
     with ops.control_dependencies(self._runtime_assertions):
@@ -149,13 +158,14 @@ class Independent(distribution_lib.Distribution):
       batch_ndims = (batch_shape.shape[0].value
                      if batch_shape.shape.with_rank_at_least(1)[0].value
                      else array_ops.shape(batch_shape)[0])
-      return batch_shape[:batch_ndims - self.reduce_batch_ndims]
+      return batch_shape[:batch_ndims - self.reinterpreted_batch_ndims]
 
   def _batch_shape(self):
     batch_shape = self.distribution.batch_shape
-    if self._static_reduce_batch_ndims is None or batch_shape.ndims is None:
+    if (self._static_reinterpreted_batch_ndims is None
+        or batch_shape.ndims is None):
       return tensor_shape.TensorShape(None)
-    d = batch_shape.ndims - self._static_reduce_batch_ndims
+    d = batch_shape.ndims - self._static_reinterpreted_batch_ndims
     return batch_shape[:d]
 
   def _event_shape_tensor(self):
@@ -165,15 +175,16 @@ class Independent(distribution_lib.Distribution):
                      if batch_shape.shape.with_rank_at_least(1)[0].value
                      else array_ops.shape(batch_shape)[0])
       return array_ops.concat([
-          batch_shape[batch_ndims - self.reduce_batch_ndims:],
+          batch_shape[batch_ndims - self.reinterpreted_batch_ndims:],
           self.distribution.event_shape_tensor(),
       ], axis=0)
 
   def _event_shape(self):
     batch_shape = self.distribution.batch_shape
-    if self._static_reduce_batch_ndims is None or batch_shape.ndims is None:
+    if (self._static_reinterpreted_batch_ndims is None
+        or batch_shape.ndims is None):
       return tensor_shape.TensorShape(None)
-    d = batch_shape.ndims - self._static_reduce_batch_ndims
+    d = batch_shape.ndims - self._static_reinterpreted_batch_ndims
     return batch_shape[d:].concatenate(self.distribution.event_shape)
 
   def _sample_n(self, n, seed):
@@ -205,15 +216,16 @@ class Independent(distribution_lib.Distribution):
       return self.distribution.mode()
 
   def _make_runtime_assertions(
-      self, distribution, reduce_batch_ndims, validate_args):
+      self, distribution, reinterpreted_batch_ndims, validate_args):
     assertions = []
-    static_reduce_batch_ndims = tensor_util.constant_value(reduce_batch_ndims)
+    static_reinterpreted_batch_ndims = tensor_util.constant_value(
+        reinterpreted_batch_ndims)
     batch_ndims = distribution.batch_shape.ndims
-    if batch_ndims is not None and static_reduce_batch_ndims is not None:
-      if static_reduce_batch_ndims > batch_ndims:
-        raise ValueError("reduce_batch_ndims({}) cannot exceed "
+    if batch_ndims is not None and static_reinterpreted_batch_ndims is not None:
+      if static_reinterpreted_batch_ndims > batch_ndims:
+        raise ValueError("reinterpreted_batch_ndims({}) cannot exceed "
                          "distribution.batch_ndims({})".format(
-                             static_reduce_batch_ndims, batch_ndims))
+                             static_reinterpreted_batch_ndims, batch_ndims))
     elif validate_args:
       batch_shape = distribution.batch_shape_tensor()
       batch_ndims = (
@@ -221,13 +233,24 @@ class Independent(distribution_lib.Distribution):
           if batch_shape.shape.with_rank_at_least(1)[0].value is not None
           else array_ops.shape(batch_shape)[0])
       assertions.append(check_ops.assert_less_equal(
-          reduce_batch_ndims, batch_ndims,
-          message="reduce_batch_ndims cannot exceed distribution.batch_ndims"))
+          reinterpreted_batch_ndims, batch_ndims,
+          message=("reinterpreted_batch_ndims cannot exceed "
+                   "distribution.batch_ndims")))
     return assertions
 
   def _reduce_sum(self, stat):
-    if self._static_reduce_batch_ndims is None:
-      range_ = array_ops.range(self._reduce_batch_ndims)
+    if self._static_reinterpreted_batch_ndims is None:
+      range_ = math_ops.range(self._reinterpreted_batch_ndims)
     else:
-      range_ = np.arange(self._static_reduce_batch_ndims)
+      range_ = np.arange(self._static_reinterpreted_batch_ndims)
     return math_ops.reduce_sum(stat, axis=-1-range_)
+
+  def _get_default_reinterpreted_batch_ndims(self, distribution):
+    """Computes the default value for reinterpreted_batch_ndim __init__ arg."""
+    ndims = distribution.batch_shape.ndims
+    if ndims is None:
+      which_maximum = math_ops.maximum
+      ndims = array_ops.shape(distribution.batch_shape_tensor())[0]
+    else:
+      which_maximum = np.maximum
+    return which_maximum(0, ndims - 1)
-- 
GitLab


From e1d7615ebcad6f45b41f7849bc77a8aae17b8690 Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Fri, 27 Oct 2017 16:10:54 -0700
Subject: [PATCH 891/909] Fix issue with gradients of functions which return
 multiple values.

PiperOrigin-RevId: 173730922
---
 tensorflow/python/eager/backprop.py      | 34 +++++++++++++++++++-----
 tensorflow/python/eager/backprop_test.py |  7 +++++
 2 files changed, 35 insertions(+), 6 deletions(-)

diff --git a/tensorflow/python/eager/backprop.py b/tensorflow/python/eager/backprop.py
index 6ede02dbcd..be733405a3 100644
--- a/tensorflow/python/eager/backprop.py
+++ b/tensorflow/python/eager/backprop.py
@@ -35,6 +35,7 @@ from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
 from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import gen_array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.util import nest
@@ -322,7 +323,10 @@ def implicit_val_and_grad(f):
   ```
 
   Args:
-    f: The function to be differentiated.
+   f: function to be differentiated. If `f` returns a scalar, this scalar will
+     be differentiated. If `f` returns a tensor or list of tensors, by default
+     a scalar will be computed by adding all their values to produce a single
+     scalar.
 
   Returns:
     A function which, when called, returns a tuple pair.
@@ -384,7 +388,10 @@ def implicit_grad(f):
   ```
 
   Args:
-    f: The function to be differentiated.
+   f: function to be differentiated. If `f` returns a scalar, this scalar will
+     be differentiated. If `f` returns a tensor or list of tensors, by default
+     a scalar will be computed by adding all their values to produce a single
+     scalar.
 
   Returns:
     A function which, when called, returns a list of (gradient, variable) pairs.
@@ -467,7 +474,12 @@ def gradients_function(f, params=None):
   ```
 
   Args:
-   f: function to be differentiated.
+   f: function to be differentiated. If `f` returns a scalar, this scalar will
+     be differentiated. If `f` returns a tensor or list of tensors, by default
+     a scalar will be computed by adding all their values to produce a single
+     scalar. If desired, the tensors can be elementwise multiplied by the
+     tensors passed as the `dy` keyword argument to the returned gradient
+     function.
    params: list of parameter names of f or list of integers indexing the
      parameters with respect to which we'll differentiate. Passing None
      differentiates with respect to all parameters.
@@ -559,7 +571,12 @@ def val_and_grad_function(f, params=None):
   ```
 
   Args:
-   f: function to be differentiated.
+   f: function to be differentiated. If `f` returns a scalar, this scalar will
+     be differentiated. If `f` returns a tensor or list of tensors, by default
+     a scalar will be computed by adding all their values to produce a single
+     scalar. If desired, the tensors can be elementwise multiplied by the
+     tensors passed as the `dy` keyword argument to the returned gradient
+     function.
    params: list of parameter names of f or list of integers indexing the
      parameters with respect to which we'll differentiate. Passing `None`
      differentiates with respect to all parameters.
@@ -632,12 +649,17 @@ def make_vjp(f, params=None):
         sources.append(args[i])
         tape.watch(args[i])
         result = f(*args)
+        flat_result = nest.flatten(result)
+        flat_result = [gen_array_ops.identity(x) for x in flat_result]
+        result = nest.pack_sequence_as(result, flat_result)
     finally:
       t = tape.pop_tape()
     def vjp(dy=None):
+      if dy is not None:
+        dy = [ops.convert_to_tensor(x) for x in nest.flatten(dy)]
       return imperative_grad.imperative_grad(
           _default_vspace, t, nest.flatten(result), sources,
-          output_gradients=nest.flatten(dy) if dy is not None else None)
+          output_gradients=dy)
     return result, vjp
 
   return decorated
@@ -697,7 +719,7 @@ _default_vspace = imperative_grad.VSpace(
     aggregate_fn=_aggregate_grads,
     tensor_id=ops.tensor_id,
     zeros=array_ops.zeros,
-    ones_like=array_ops.ones_like)
+    ones_like=lambda x: ops.convert_to_tensor(array_ops.ones_like(x)))
 
 
 class GradientTape(object):
diff --git a/tensorflow/python/eager/backprop_test.py b/tensorflow/python/eager/backprop_test.py
index 20532c8ee8..cf736fcb13 100644
--- a/tensorflow/python/eager/backprop_test.py
+++ b/tensorflow/python/eager/backprop_test.py
@@ -401,6 +401,13 @@ class BackpropTest(test.TestCase):
         backprop.gradients_function(part)(constant_op.constant(1.0))[0],
         2.0)
 
+  def testReturnSameThing(self):
+
+    def f(x):
+      return x, 2 * x
+
+    self.assertAllEqual(backprop.gradients_function(f)(1.0)[0], 3.0)
+
   def testExceptionSafety(self):
 
     def f(unused_x):
-- 
GitLab


From 7cb7f88c5ff12a5ce52e82d9f07e1b489df1e0ff Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 27 Oct 2017 16:11:56 -0700
Subject: [PATCH 892/909] Add count metric, a helper function that computes the
 total number or total weight of examples.

PiperOrigin-RevId: 173731046
---
 tensorflow/contrib/metrics/__init__.py        |   2 +
 .../contrib/metrics/python/ops/metric_ops.py  |  80 ++++++++-
 .../metrics/python/ops/metric_ops_test.py     | 158 ++++++++++++++++++
 3 files changed, 231 insertions(+), 9 deletions(-)

diff --git a/tensorflow/contrib/metrics/__init__.py b/tensorflow/contrib/metrics/__init__.py
index 2c48882d0e..bb566f6902 100644
--- a/tensorflow/contrib/metrics/__init__.py
+++ b/tensorflow/contrib/metrics/__init__.py
@@ -65,6 +65,7 @@ See the @{$python/contrib.metrics} guide.
 @@set_intersection
 @@set_size
 @@set_union
+@@count
 
 """
 from __future__ import absolute_import
@@ -78,6 +79,7 @@ from tensorflow.contrib.metrics.python.ops.confusion_matrix_ops import confusion
 from tensorflow.contrib.metrics.python.ops.histogram_ops import auc_using_histogram
 from tensorflow.contrib.metrics.python.ops.metric_ops import aggregate_metric_map
 from tensorflow.contrib.metrics.python.ops.metric_ops import aggregate_metrics
+from tensorflow.contrib.metrics.python.ops.metric_ops import count
 from tensorflow.contrib.metrics.python.ops.metric_ops import sparse_recall_at_top_k
 from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_accuracy
 from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_auc
diff --git a/tensorflow/contrib/metrics/python/ops/metric_ops.py b/tensorflow/contrib/metrics/python/ops/metric_ops.py
index 50b9c4afde..177c4c53f7 100644
--- a/tensorflow/contrib/metrics/python/ops/metric_ops.py
+++ b/tensorflow/contrib/metrics/python/ops/metric_ops.py
@@ -120,7 +120,7 @@ def _count_condition(values,
       or tuple.
   """
   check_ops.assert_type(values, dtypes.bool)
-  count = _create_local('count', shape=[])
+  count_ = _create_local('count', shape=[])
 
   values = math_ops.to_float(values)
   if weights is not None:
@@ -128,8 +128,8 @@ def _count_condition(values,
     with ops.control_dependencies((_assert_weights_rank(weights, values),)):
       values = math_ops.multiply(values, weights)
 
-  value_tensor = array_ops.identity(count)
-  update_op = state_ops.assign_add(count, math_ops.reduce_sum(values))
+  value_tensor = array_ops.identity(count_)
+  update_op = state_ops.assign_add(count_, math_ops.reduce_sum(values))
 
   if metrics_collections:
     ops.add_to_collections(metrics_collections, value_tensor)
@@ -2601,7 +2601,7 @@ def streaming_covariance(predictions,
     predictions, labels, weights = metrics_impl._remove_squeezable_dimensions(  # pylint: disable=protected-access
         predictions, labels, weights)
     predictions.get_shape().assert_is_compatible_with(labels.get_shape())
-    count = _create_local('count', [])
+    count_ = _create_local('count', [])
     mean_prediction = _create_local('mean_prediction', [])
     mean_label = _create_local('mean_label', [])
     comoment = _create_local('comoment', [])  # C_A in update equation
@@ -2616,7 +2616,7 @@ def streaming_covariance(predictions,
       weighted_predictions = math_ops.multiply(predictions, weights)
       weighted_labels = math_ops.multiply(labels, weights)
 
-    update_count = state_ops.assign_add(count, batch_count)  # n_AB in eqn
+    update_count = state_ops.assign_add(count_, batch_count)  # n_AB in eqn
     prev_count = update_count - batch_count  # n_A in update equation
 
     # We update the means by Delta=Error*BatchCount/(BatchCount+PrevCount)
@@ -2660,15 +2660,15 @@ def streaming_covariance(predictions,
     update_comoment = state_ops.assign_add(comoment, delta_comoment)
 
     covariance = array_ops.where(
-        math_ops.less_equal(count, 1.),
+        math_ops.less_equal(count_, 1.),
         float('nan'),
-        math_ops.truediv(comoment, count - 1),
+        math_ops.truediv(comoment, count_ - 1),
         name='covariance')
     with ops.control_dependencies([update_comoment]):
       update_op = array_ops.where(
-          math_ops.less_equal(count, 1.),
+          math_ops.less_equal(count_, 1.),
           float('nan'),
-          math_ops.truediv(comoment, count - 1),
+          math_ops.truediv(comoment, count_ - 1),
           name='update_op')
 
   if metrics_collections:
@@ -3124,9 +3124,71 @@ def aggregate_metric_map(names_to_tuples):
   return dict(zip(metric_names, value_ops)), dict(zip(metric_names, update_ops))
 
 
+def count(values,
+          weights=None,
+          metrics_collections=None,
+          updates_collections=None,
+          name=None):
+  """Computes the number of examples, or sum of `weights`.
+
+  When evaluating some metric (e.g. mean) on one or more subsets of the data,
+  this auxiliary metric is useful for keeping track of how many examples there
+  are in each subset.
+
+  If `weights` is `None`, weights default to 1. Use weights of 0 to mask values.
+
+  Args:
+    values: A `Tensor` of arbitrary dimensions. Only it's shape is used.
+    weights: Optional `Tensor` whose rank is either 0, or the same rank as
+      `labels`, and must be broadcastable to `labels` (i.e., all dimensions
+      must be either `1`, or the same as the corresponding `labels`
+      dimension).
+    metrics_collections: An optional list of collections that the metric
+      value variable should be added to.
+    updates_collections: An optional list of collections that the metric update
+      ops should be added to.
+    name: An optional variable_scope name.
+
+  Returns:
+    count: A `Tensor` representing the current value of the metric.
+    update_op: An operation that accumulates the metric from a batch of data.
+
+  Raises:
+    ValueError: If `weights` is not `None` and its shape doesn't match `values`,
+      or if either `metrics_collections` or `updates_collections` are not a list
+      or tuple.
+  """
+
+  with variable_scope.variable_scope(name, 'count', (values, weights)):
+    count_ = _create_local('count', shape=[])
+
+    if weights is None:
+      num_values = math_ops.to_float(array_ops.size(values))
+    else:
+      _, _, weights = metrics_impl._remove_squeezable_dimensions(  # pylint: disable=protected-access
+          predictions=values,
+          labels=None,
+          weights=weights)
+      weights = weights_broadcast_ops.broadcast_weights(
+          math_ops.to_float(weights), values)
+      num_values = math_ops.reduce_sum(weights)
+
+    with ops.control_dependencies([values]):
+      update_op = state_ops.assign_add(count_, num_values)
+
+    if metrics_collections:
+      ops.add_to_collections(metrics_collections, count_)
+
+    if updates_collections:
+      ops.add_to_collections(updates_collections, update_op)
+
+    return count_, update_op
+
+
 __all__ = [
     'aggregate_metric_map',
     'aggregate_metrics',
+    'count',
     'sparse_recall_at_top_k',
     'streaming_accuracy',
     'streaming_auc',
diff --git a/tensorflow/contrib/metrics/python/ops/metric_ops_test.py b/tensorflow/contrib/metrics/python/ops/metric_ops_test.py
index 24d82a7eee..6a8284786f 100644
--- a/tensorflow/contrib/metrics/python/ops/metric_ops_test.py
+++ b/tensorflow/contrib/metrics/python/ops/metric_ops_test.py
@@ -6170,5 +6170,163 @@ class AggregateMetricMapTest(test.TestCase):
       self.assertEqual(4, names_to_values['m2'].eval())
 
 
+class CountTest(test.TestCase):
+
+  def setUp(self):
+    ops.reset_default_graph()
+
+  def testVars(self):
+    metrics.count(array_ops.ones([4, 3]))
+    _assert_local_variables(self, ['count/count:0'])
+
+  def testMetricsCollection(self):
+    my_collection_name = '__metrics__'
+    mean, _ = metrics.count(
+        array_ops.ones([4, 3]), metrics_collections=[my_collection_name])
+    self.assertListEqual(ops.get_collection(my_collection_name), [mean])
+
+  def testUpdatesCollection(self):
+    my_collection_name = '__updates__'
+    _, update_op = metrics.count(
+        array_ops.ones([4, 3]), updates_collections=[my_collection_name])
+    self.assertListEqual(ops.get_collection(my_collection_name), [update_op])
+
+  def testBasic(self):
+    with self.test_session() as sess:
+      values_queue = data_flow_ops.FIFOQueue(
+          4, dtypes=dtypes_lib.float32, shapes=(1, 2))
+      _enqueue_vector(sess, values_queue, [0, 1])
+      _enqueue_vector(sess, values_queue, [-4.2, 9.1])
+      _enqueue_vector(sess, values_queue, [6.5, 0])
+      _enqueue_vector(sess, values_queue, [-3.2, 4.0])
+      values = values_queue.dequeue()
+
+      result, update_op = metrics.count(values)
+
+      sess.run(variables.local_variables_initializer())
+      for _ in range(4):
+        sess.run(update_op)
+      self.assertAlmostEqual(8.0, sess.run(result), 5)
+
+  def testUpdateOpsReturnsCurrentValue(self):
+    with self.test_session() as sess:
+      values_queue = data_flow_ops.FIFOQueue(
+          4, dtypes=dtypes_lib.float32, shapes=(1, 2))
+      _enqueue_vector(sess, values_queue, [0, 1])
+      _enqueue_vector(sess, values_queue, [-4.2, 9.1])
+      _enqueue_vector(sess, values_queue, [6.5, 0])
+      _enqueue_vector(sess, values_queue, [-3.2, 4.0])
+      values = values_queue.dequeue()
+
+      result, update_op = metrics.count(values)
+
+      sess.run(variables.local_variables_initializer())
+
+      self.assertAlmostEqual(2.0, sess.run(update_op), 5)
+      self.assertAlmostEqual(4.0, sess.run(update_op), 5)
+      self.assertAlmostEqual(6.0, sess.run(update_op), 5)
+      self.assertAlmostEqual(8.0, sess.run(update_op), 5)
+
+      self.assertAlmostEqual(8.0, sess.run(result), 5)
+
+  def test1dWeightedValues(self):
+    with self.test_session() as sess:
+      # Create the queue that populates the values.
+      values_queue = data_flow_ops.FIFOQueue(
+          4, dtypes=dtypes_lib.float32, shapes=(1, 2))
+      _enqueue_vector(sess, values_queue, [0, 1])
+      _enqueue_vector(sess, values_queue, [-4.2, 9.1])
+      _enqueue_vector(sess, values_queue, [6.5, 0])
+      _enqueue_vector(sess, values_queue, [-3.2, 4.0])
+      values = values_queue.dequeue()
+
+      # Create the queue that populates the weighted labels.
+      weights_queue = data_flow_ops.FIFOQueue(
+          4, dtypes=dtypes_lib.float32, shapes=(1, 1))
+      _enqueue_vector(sess, weights_queue, [0.5])
+      _enqueue_vector(sess, weights_queue, [0])
+      _enqueue_vector(sess, weights_queue, [0])
+      _enqueue_vector(sess, weights_queue, [1.2])
+      weights = weights_queue.dequeue()
+
+      result, update_op = metrics.count(values, weights)
+
+      variables.local_variables_initializer().run()
+      for _ in range(4):
+        update_op.eval()
+      self.assertAlmostEqual(3.4, result.eval(), 5)
+
+  def test1dWeightedValues_placeholders(self):
+    with self.test_session() as sess:
+      # Create the queue that populates the values.
+      feed_values = ((0, 1), (-4.2, 9.1), (6.5, 0), (-3.2, 4.0))
+      values = array_ops.placeholder(dtype=dtypes_lib.float32)
+
+      # Create the queue that populates the weighted labels.
+      weights_queue = data_flow_ops.FIFOQueue(
+          4, dtypes=dtypes_lib.float32, shapes=(1,))
+      _enqueue_vector(sess, weights_queue, 0.5, shape=(1,))
+      _enqueue_vector(sess, weights_queue, 0, shape=(1,))
+      _enqueue_vector(sess, weights_queue, 0, shape=(1,))
+      _enqueue_vector(sess, weights_queue, 1.2, shape=(1,))
+      weights = weights_queue.dequeue()
+
+      result, update_op = metrics.count(values, weights)
+
+      variables.local_variables_initializer().run()
+      for i in range(4):
+        update_op.eval(feed_dict={values: feed_values[i]})
+      self.assertAlmostEqual(3.4, result.eval(), 5)
+
+  def test2dWeightedValues(self):
+    with self.test_session() as sess:
+      # Create the queue that populates the values.
+      values_queue = data_flow_ops.FIFOQueue(
+          4, dtypes=dtypes_lib.float32, shapes=(1, 2))
+      _enqueue_vector(sess, values_queue, [0, 1])
+      _enqueue_vector(sess, values_queue, [-4.2, 9.1])
+      _enqueue_vector(sess, values_queue, [6.5, 0])
+      _enqueue_vector(sess, values_queue, [-3.2, 4.0])
+      values = values_queue.dequeue()
+
+      # Create the queue that populates the weighted labels.
+      weights_queue = data_flow_ops.FIFOQueue(
+          4, dtypes=dtypes_lib.float32, shapes=(1, 2))
+      _enqueue_vector(sess, weights_queue, [1.1, 1])
+      _enqueue_vector(sess, weights_queue, [1, 0])
+      _enqueue_vector(sess, weights_queue, [0, 1])
+      _enqueue_vector(sess, weights_queue, [0, 0])
+      weights = weights_queue.dequeue()
+
+      result, update_op = metrics.count(values, weights)
+
+      variables.local_variables_initializer().run()
+      for _ in range(4):
+        update_op.eval()
+      self.assertAlmostEqual(4.1, result.eval(), 5)
+
+  def test2dWeightedValues_placeholders(self):
+    with self.test_session() as sess:
+      # Create the queue that populates the values.
+      feed_values = ((0, 1), (-4.2, 9.1), (6.5, 0), (-3.2, 4.0))
+      values = array_ops.placeholder(dtype=dtypes_lib.float32)
+
+      # Create the queue that populates the weighted labels.
+      weights_queue = data_flow_ops.FIFOQueue(
+          4, dtypes=dtypes_lib.float32, shapes=(2,))
+      _enqueue_vector(sess, weights_queue, [1.1, 1], shape=(2,))
+      _enqueue_vector(sess, weights_queue, [1, 0], shape=(2,))
+      _enqueue_vector(sess, weights_queue, [0, 1], shape=(2,))
+      _enqueue_vector(sess, weights_queue, [0, 0], shape=(2,))
+      weights = weights_queue.dequeue()
+
+      result, update_op = metrics.count(values, weights)
+
+      variables.local_variables_initializer().run()
+      for i in range(4):
+        update_op.eval(feed_dict={values: feed_values[i]})
+      self.assertAlmostEqual(4.1, result.eval(), 5)
+
+
 if __name__ == '__main__':
   test.main()
-- 
GitLab


From 46a577febccf874b4b8bb8d42be0a3fb069e380d Mon Sep 17 00:00:00 2001
From: Derek Murray <mrry@google.com>
Date: Fri, 27 Oct 2017 16:25:49 -0700
Subject: [PATCH 893/909] [CMake] Generate audio_ops wrappers in the CMake
 build.

Fixes #14004.

PiperOrigin-RevId: 173732397
---
 tensorflow/contrib/cmake/tf_core_ops.cmake | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/contrib/cmake/tf_core_ops.cmake b/tensorflow/contrib/cmake/tf_core_ops.cmake
index 97bec81e66..15e9a4c461 100644
--- a/tensorflow/contrib/cmake/tf_core_ops.cmake
+++ b/tensorflow/contrib/cmake/tf_core_ops.cmake
@@ -13,6 +13,7 @@
 # limitations under the License.
 # ==============================================================================
 set(tf_op_lib_names
+    "audio_ops"
     "array_ops"
     "bitwise_ops"
     "candidate_sampling_ops"
-- 
GitLab


From abbab2430cc5e3ef8eab224957c5cbfc8bd0056a Mon Sep 17 00:00:00 2001
From: Michael Case <mikecase@google.com>
Date: Fri, 27 Oct 2017 16:27:43 -0700
Subject: [PATCH 894/909] Add bazel mirror links for newly added workspace
 dependencies.

PiperOrigin-RevId: 173732606
---
 tensorflow/workspace.bzl | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index c0eb87a744..e25e12d5c5 100644
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -727,6 +727,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       name = "javax_validation",
       jar_sha256 = "e459f313ebc6db2483f8ceaad39af07086361b474fa92e40f442e8de5d9895dc",
       jar_urls = [
+          "http://mirror.bazel.build/repo1.maven.org/maven2/javax/validation/validation-api/1.0.0.GA/validation-api-1.0.0.GA.jar",
           "http://repo1.maven.org/maven2/javax/validation/validation-api/1.0.0.GA/validation-api-1.0.0.GA.jar",
       ],
       licenses = ["notice"],  # Apache 2.0
@@ -785,6 +786,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       sha256 = "c8d90aa4357f8079d427e87a6f4c493da1fa4140aee926c05902d7ec1533d9a5",
       strip_prefix = "ARM_NEON_2_x86_SSE-0f77d9d182265259b135dad949230ecbf1a2633d",
       urls = [
+          "https://mirror.bazel.build/github.com/intel/ARM_NEON_2_x86_SSE/archive/0f77d9d182265259b135dad949230ecbf1a2633d.tar.gz",
           "https://github.com/intel/ARM_NEON_2_x86_SSE/archive/0f77d9d182265259b135dad949230ecbf1a2633d.tar.gz",
       ],
       build_file = str(Label("//third_party:arm_neon_2_x86_sse.BUILD")),
@@ -796,6 +798,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       strip_prefix = "flatbuffers-971a68110e4fc1bace10fcb6deeb189e7e1a34ce",
       sha256 = "874088d2ee0d9f8524191f77209556415f03dd44e156276edf19e5b90ceb5f55",
       urls = [
+          "https://mirror.bazel.build/github.com/google/flatbuffers/archive/971a68110e4fc1bace10fcb6deeb189e7e1a34ce.tar.gz",
           "https://github.com/google/flatbuffers/archive/971a68110e4fc1bace10fcb6deeb189e7e1a34ce.tar.gz",
       ],
   )
-- 
GitLab


From 3ff9c8d2af2ede9ee7c16fb3b15d004e423f95e5 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 27 Oct 2017 16:36:18 -0700
Subject: [PATCH 895/909] Fix typos in Linear Model Tutorial samples

1. test_file_name is undefined (should be test_file.name)
2. train_file_name is undefined (should be train_file.name)

PiperOrigin-RevId: 173733442
---
 tensorflow/docs_src/tutorials/wide.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/docs_src/tutorials/wide.md b/tensorflow/docs_src/tutorials/wide.md
index 6292c1a01e..ba16e12a72 100644
--- a/tensorflow/docs_src/tutorials/wide.md
+++ b/tensorflow/docs_src/tutorials/wide.md
@@ -383,7 +383,7 @@ API:
 ```python
 # set num_epochs to None to get infinite stream of data.
 m.train(
-    input_fn=input_fn(train_file_name, num_epochs=None, shuffle=True),
+    input_fn=input_fn(train_file.name, num_epochs=None, shuffle=True),
     steps=train_steps)
 ```
 
@@ -392,7 +392,7 @@ the labels of the holdout data:
 
 ```python
 results = m.evaluate(
-    input_fn=input_fn(test_file_name, num_epochs=1, shuffle=False),
+    input_fn=input_fn(test_file.name, num_epochs=1, shuffle=False),
     steps=None)
 print("model directory = %s" % model_dir)
 for key in sorted(results):
-- 
GitLab


From d1c59bd37510b9fa1e0cd909c1f4857028d4d13b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 27 Oct 2017 17:03:46 -0700
Subject: [PATCH 896/909] Add tf.quantize op, which is the same as
 tf.quantize_v2.

PiperOrigin-RevId: 173735986
---
 tensorflow/core/kernels/quantize_op.cc       |  1 -
 tensorflow/core/ops/array_ops.cc             | 12 ++++----
 tensorflow/python/ops/array_ops.py           | 30 ++++++++++++++++++--
 tensorflow/tools/api/golden/tensorflow.pbtxt |  4 +++
 4 files changed, 38 insertions(+), 9 deletions(-)

diff --git a/tensorflow/core/kernels/quantize_op.cc b/tensorflow/core/kernels/quantize_op.cc
index 75aa47cd6b..fc26813a08 100644
--- a/tensorflow/core/kernels/quantize_op.cc
+++ b/tensorflow/core/kernels/quantize_op.cc
@@ -250,5 +250,4 @@ REGISTER_KERNEL_BUILDER(
 REGISTER_KERNEL_BUILDER(
     Name("QuantizeV2").Device(DEVICE_CPU).TypeConstraint<qint32>("T"),
     QuantizeV2Op<CPUDevice, qint32>);
-
 }  // namespace tensorflow
diff --git a/tensorflow/core/ops/array_ops.cc b/tensorflow/core/ops/array_ops.cc
index f73bc716d5..cdf370399c 100644
--- a/tensorflow/core/ops/array_ops.cc
+++ b/tensorflow/core/ops/array_ops.cc
@@ -4902,10 +4902,10 @@ with the range of qint8.
 If the mode is 'MIN_FIRST', then this approach is used:
 
 ```
-number_of_steps = 1 << (# of bits in T)
-range_adjust = number_of_steps / (number_of_steps - 1)
+num_discrete_values = 1 << (# of bits in T)
+range_adjust = num_discrete_values / (num_discrete_values - 1)
 range = (range_max - range_min) * range_adjust
-range_scale = number_of_steps / range
+range_scale = num_discrete_values / range
 quantized = round(input * range_scale) - round(range_min * range_scale) +
   numeric_limits<T>::min()
 quantized = max(quantized, numeric_limits<T>::min())
@@ -5017,10 +5017,10 @@ each value by 128 prior to casting.
 If the mode is 'MIN_FIRST', then this approach is used:
 
 ```c++
-number_of_steps = 1 << (# of bits in T)
-range_adjust = number_of_steps / (number_of_steps - 1)
+num_discrete_values = 1 << (# of bits in T)
+range_adjust = num_discrete_values / (num_discrete_values - 1)
 range = (range_max - range_min) * range_adjust
-range_scale = range / number_of_steps
+range_scale = range / num_discrete_values
 const double offset_input = static_cast<double>(input) - lowest_quantized;
 result = range_min + ((input - numeric_limits<T>::min()) * range_scale)
 ```
diff --git a/tensorflow/python/ops/array_ops.py b/tensorflow/python/ops/array_ops.py
index ba8c611f57..8a447deea2 100644
--- a/tensorflow/python/ops/array_ops.py
+++ b/tensorflow/python/ops/array_ops.py
@@ -66,6 +66,7 @@ See the @{$python/array_ops} guide.
 @@one_hot
 @@sequence_mask
 @@dequantize
+@@quantize
 @@quantize_v2
 @@quantized_concat
 @@setdiff1d
@@ -2525,7 +2526,10 @@ gather.__doc__ = gen_array_ops.gather_v2.__doc__
 
 # Define quantize_v2 here in order to make name the second-to-last attribute,
 # because round_mode was added later.
-def quantize_v2(input,
+@deprecation.deprecated(
+    "2017-10-25",
+    "`tf.quantize_v2` is deprecated, please use `tf.quantize` instead.")
+def quantize_v2(input,  # pylint: disable=redefined-builtin
                 min_range,
                 max_range,
                 T,
@@ -2541,4 +2545,26 @@ def quantize_v2(input,
                                    round_mode=round_mode)
 
 
-quantize_v2.__doc__ = gen_array_ops.quantize_v2.__doc__
+quantize_v2.__doc__ = """Please use `tf.quantize` instead."""
+
+
+# We want to expose tf.quantize instead of tf.quantize_v2; we can deprecate
+# tf.quantize_v2 in next version of TensorFlow.
+def quantize(input,  # pylint: disable=redefined-builtin
+             min_range,
+             max_range,
+             T,
+             mode="MIN_COMBINED",
+             round_mode="HALF_AWAY_FROM_ZERO",
+             name=None):
+  return gen_array_ops.quantize_v2(
+      input,
+      min_range,
+      max_range,
+      T,
+      mode=mode,
+      round_mode=round_mode,
+      name=name)
+
+
+quantize.__doc__ = gen_array_ops.quantize_v2.__doc__
diff --git a/tensorflow/tools/api/golden/tensorflow.pbtxt b/tensorflow/tools/api/golden/tensorflow.pbtxt
index 1c6f3cc534..f61f82e43e 100644
--- a/tensorflow/tools/api/golden/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.pbtxt
@@ -1480,6 +1480,10 @@ tf_module {
     name: "qr"
     argspec: "args=[\'input\', \'full_matrices\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'None\'], "
   }
+  member_method {
+    name: "quantize"
+    argspec: "args=[\'input\', \'min_range\', \'max_range\', \'T\', \'mode\', \'round_mode\', \'name\'], varargs=None, keywords=None, defaults=[\'MIN_COMBINED\', \'HALF_AWAY_FROM_ZERO\', \'None\'], "
+  }
   member_method {
     name: "quantize_v2"
     argspec: "args=[\'input\', \'min_range\', \'max_range\', \'T\', \'mode\', \'name\', \'round_mode\'], varargs=None, keywords=None, defaults=[\'MIN_COMBINED\', \'None\', \'HALF_AWAY_FROM_ZERO\'], "
-- 
GitLab


From 245a5c171aa7ec4787080b6e0a88f281e1345f97 Mon Sep 17 00:00:00 2001
From: Akshay Agrawal <akshayka@google.com>
Date: Fri, 27 Oct 2017 17:24:33 -0700
Subject: [PATCH 897/909] Make functional_ops compatible with eager exeuction
 by ignoring caching devices when in eager mode

PiperOrigin-RevId: 173737949
---
 tensorflow/python/BUILD                       |   1 +
 .../kernel_tests/functional_ops_test.py       | 100 ++++++++++-------
 tensorflow/python/ops/functional_ops.py       | 101 +++++++++++-------
 3 files changed, 128 insertions(+), 74 deletions(-)

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 4de5d7f7db..d435ae1375 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -897,6 +897,7 @@ py_library(
         ":tensor_shape",
         ":util",
         ":variable_scope",
+        "//tensorflow/python/eager:context",
     ],
 )
 
diff --git a/tensorflow/python/kernel_tests/functional_ops_test.py b/tensorflow/python/kernel_tests/functional_ops_test.py
index 21fe588ac1..f5717a5a21 100644
--- a/tensorflow/python/kernel_tests/functional_ops_test.py
+++ b/tensorflow/python/kernel_tests/functional_ops_test.py
@@ -52,6 +52,7 @@ def simple_scoped_fn(a, x):
 
 class FunctionalOpsTest(test.TestCase):
 
+  @test_util.run_in_graph_and_eager_modes()
   def testFoldl_Simple(self):
     with self.test_session():
       elems = constant_op.constant([1, 2, 3, 4, 5, 6], name="data")
@@ -59,13 +60,13 @@ class FunctionalOpsTest(test.TestCase):
       r = functional_ops.foldl(
           lambda a, x: math_ops.multiply(math_ops.add(a, x), 2),
           elems)
-      self.assertAllEqual(208, r.eval())
+      self.assertAllEqual(208, self.evaluate(r))
 
       r = functional_ops.foldl(
           lambda a, x: math_ops.multiply(math_ops.add(a, x), 2),
           elems,
           initializer=10)
-      self.assertAllEqual(880, r.eval())
+      self.assertAllEqual(880, self.evaluate(r))
 
   def testFoldl_Scoped(self):
     with self.test_session() as sess:
@@ -78,14 +79,15 @@ class FunctionalOpsTest(test.TestCase):
         self.assertEqual(variables.trainable_variables()[0].name,
                          "root/body/two:0")
         sess.run([variables.global_variables_initializer()])
-        self.assertAllEqual(208, r.eval())
+        self.assertAllEqual(208, self.evaluate(r))
 
         # Now let's reuse our single variable.
         varscope.reuse_variables()
         r = functional_ops.foldl(simple_scoped_fn, elems, initializer=10)
         self.assertEqual(len(variables.trainable_variables()), 1)
-        self.assertAllEqual(880, r.eval())
+        self.assertAllEqual(880, self.evaluate(r))
 
+  @test_util.run_in_graph_and_eager_modes()
   def testFoldr_Simple(self):
     with self.test_session():
       elems = constant_op.constant([1, 2, 3, 4, 5, 6], name="data")
@@ -93,13 +95,13 @@ class FunctionalOpsTest(test.TestCase):
       r = functional_ops.foldr(
           lambda a, x: math_ops.multiply(math_ops.add(a, x), 2),
           elems)
-      self.assertAllEqual(450, r.eval())
+      self.assertAllEqual(450, self.evaluate(r))
 
       r = functional_ops.foldr(
           lambda a, x: math_ops.multiply(math_ops.add(a, x), 2),
           elems,
           initializer=10)
-      self.assertAllEqual(1282, r.eval())
+      self.assertAllEqual(1282, self.evaluate(r))
 
   def testFoldr_Scoped(self):
     with self.test_session() as sess:
@@ -112,13 +114,13 @@ class FunctionalOpsTest(test.TestCase):
         self.assertEqual(variables.trainable_variables()[0].name,
                          "root/body/two:0")
         sess.run([variables.global_variables_initializer()])
-        self.assertAllEqual(450, r.eval())
+        self.assertAllEqual(450, self.evaluate(r))
 
         # Now let's reuse our single variable.
         varscope.reuse_variables()
         r = functional_ops.foldr(simple_scoped_fn, elems, initializer=10)
         self.assertEqual(len(variables.trainable_variables()), 1)
-        self.assertAllEqual(1282, r.eval())
+        self.assertAllEqual(1282, self.evaluate(r))
 
   # pylint: disable=unnecessary-lambda
   def testFold_Grad(self):
@@ -128,21 +130,23 @@ class FunctionalOpsTest(test.TestCase):
       r = functional_ops.foldl(
           lambda a, x: math_ops.multiply(a, x), elems, initializer=v)
       r = gradients_impl.gradients(r, v)[0]
-      self.assertAllEqual(720.0, r.eval())
+      self.assertAllEqual(720.0, self.evaluate(r))
 
       r = functional_ops.foldr(
           lambda a, x: math_ops.multiply(a, x), elems, initializer=v)
       r = gradients_impl.gradients(r, v)[0]
-      self.assertAllEqual(720.0, r.eval())
+      self.assertAllEqual(720.0, self.evaluate(r))
   # pylint: enable=unnecessary-lambda
 
+  @test_util.run_in_graph_and_eager_modes()
   def testMap_Simple(self):
     with self.test_session():
       nums = [1, 2, 3, 4, 5, 6]
       elems = constant_op.constant(nums, name="data")
       r = functional_ops.map_fn(
           lambda x: math_ops.multiply(math_ops.add(x, 3), 2), elems)
-      self.assertAllEqual(np.array([(x + 3) * 2 for x in nums]), r.eval())
+      self.assertAllEqual(
+          np.array([(x + 3) * 2 for x in nums]), self.evaluate(r))
 
   def testMapSparseTensor(self):
     with self.test_session():
@@ -177,13 +181,13 @@ class FunctionalOpsTest(test.TestCase):
         self.assertEqual(variables.trainable_variables()[0].name,
                          "root/body/two:0")
         sess.run([variables.global_variables_initializer()])
-        self.assertAllEqual(doubles, r.eval())
+        self.assertAllEqual(doubles, self.evaluate(r))
 
         # Now let's reuse our single variable.
         varscope.reuse_variables()
         r = functional_ops.map_fn(double_scoped, elems)
         self.assertEqual(len(variables.trainable_variables()), 1)
-        self.assertAllEqual(doubles, r.eval())
+        self.assertAllEqual(doubles, self.evaluate(r))
 
   def testMap_Grad(self):
     with self.test_session():
@@ -192,19 +196,22 @@ class FunctionalOpsTest(test.TestCase):
       y = functional_ops.map_fn(
           lambda x: math_ops.multiply(math_ops.square(x), param), elems)
       r = gradients_impl.gradients(y, param)[0]
-      self.assertAllEqual(91.0, r.eval())
+      self.assertAllEqual(91.0, self.evaluate(r))
       r = gradients_impl.gradients(y, elems)[0]
-      self.assertAllEqual([4.0, 8.0, 12.0, 16.0, 20.0, 24.0], r.eval())
+      self.assertAllEqual([4.0, 8.0, 12.0, 16.0, 20.0, 24.0], self.evaluate(r))
 
+  @test_util.run_in_graph_and_eager_modes()
   def testMap_SimpleNotTensor(self):
     with self.test_session():
       nums = np.array([1, 2, 3, 4, 5, 6])
       r = functional_ops.map_fn(
           lambda x: math_ops.multiply(math_ops.add(x, 3), 2), nums)
-      self.assertAllEqual(np.array([(x + 3) * 2 for x in nums]), r.eval())
+      self.assertAllEqual(
+          np.array([(x + 3) * 2 for x in nums]), self.evaluate(r))
 
+  @test_util.run_in_graph_and_eager_modes()
   def testMap_SingleInputMultiOutput(self):
-    with self.test_session() as sess:
+    with self.test_session():
       nums = np.array([1, 2, 3, 4, 5, 6])
       r = functional_ops.map_fn(
           lambda x: ((x + 3) * 2, -(x + 3) * 2),
@@ -213,10 +220,11 @@ class FunctionalOpsTest(test.TestCase):
       self.assertEqual(2, len(r))
       self.assertEqual((6,), r[0].get_shape())
       self.assertEqual((6,), r[1].get_shape())
-      received = sess.run(r)
+      received = self.evaluate(r)
       self.assertAllEqual((nums + 3) * 2, received[0])
       self.assertAllEqual(-(nums + 3) * 2, received[1])
 
+  @test_util.run_in_graph_and_eager_modes()
   def testMap_MultiOutputMismatchedDtype(self):
     with self.test_session():
       nums = np.array([1, 2, 3, 4, 5, 6])
@@ -228,6 +236,7 @@ class FunctionalOpsTest(test.TestCase):
             nums,
             dtype=[dtypes.int64, dtypes.int64])
 
+  @test_util.run_in_graph_and_eager_modes()
   def testMap_MultiInputSingleOutput(self):
     with self.test_session():
       nums = np.array([1, 2, 3, 4, 5, 6])
@@ -235,11 +244,12 @@ class FunctionalOpsTest(test.TestCase):
           lambda x: x[0] * x[1][0] + x[1][1], (nums, (nums, -nums)),
           dtype=dtypes.int64)
       self.assertEqual((6,), r.get_shape())
-      received = r.eval()
+      received = self.evaluate(r)
       self.assertAllEqual(nums * nums + (-nums), received)
 
+  @test_util.run_in_graph_and_eager_modes()
   def testMap_MultiInputSameStructureOutput(self):
-    with self.test_session() as sess:
+    with self.test_session():
       nums = np.array([1, 2, 3, 4, 5, 6])
       r = functional_ops.map_fn(lambda x: (x[1][0], (x[1][1], x[0])),
                                 (nums, (2 * nums, -nums)))
@@ -247,11 +257,12 @@ class FunctionalOpsTest(test.TestCase):
       self.assertEqual((6,), r[0].get_shape())
       self.assertEqual((6,), r[1].get_shape())
       self.assertEqual((6,), r[2].get_shape())
-      received = sess.run(r)
+      received = self.evaluate(r)
       self.assertAllEqual(2 * nums, received[0])
       self.assertAllEqual(-nums, received[1])
       self.assertAllEqual(nums, received[2])
 
+  @test_util.run_in_graph_and_eager_modes()
   def testScan_Simple(self):
     with self.test_session():
       elems = constant_op.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], name="data")
@@ -259,24 +270,26 @@ class FunctionalOpsTest(test.TestCase):
 
       # pylint: disable=unnecessary-lambda
       r = functional_ops.scan(lambda a, x: math_ops.multiply(a, x), elems)
-      self.assertAllEqual([1., 2., 6., 24., 120., 720.], r.eval())
+      self.assertAllEqual([1., 2., 6., 24., 120., 720.], self.evaluate(r))
 
       r = functional_ops.scan(
           lambda a, x: math_ops.multiply(a, x), elems, initializer=v)
-      self.assertAllEqual([2., 4., 12., 48., 240., 1440.], r.eval())
+      self.assertAllEqual([2., 4., 12., 48., 240., 1440.], self.evaluate(r))
       # pylint: enable=unnecessary-lambda
 
+  @test_util.run_in_graph_and_eager_modes()
   def testScan_SingleInputMultiOutput(self):
-    with self.test_session() as sess:
+    with self.test_session():
       elems = np.array([1.0, 2.0, 3.0, 4.0, 5.0, 6.0])
       initializer = (np.array(1.0), np.array(-1.0))
       r = functional_ops.scan(lambda a, x: (a[0] * x, -a[1] * x), elems,
                               initializer)
-      r_value = sess.run(r)
+      r_value = self.evaluate(r)
 
       self.assertAllEqual([1.0, 2.0, 6.0, 24.0, 120.0, 720.0], r_value[0])
       self.assertAllEqual([1.0, -2.0, 6.0, -24.0, 120.0, -720.0], r_value[1])
 
+  @test_util.run_in_graph_and_eager_modes()
   def testScan_MultiInputSingleOutput(self):
     with self.test_session():
       elems = np.array([1.0, 2.0, 3.0, 4.0, 5.0, 6.0])
@@ -284,17 +297,19 @@ class FunctionalOpsTest(test.TestCase):
       # Multiply a * 1 each time
       r = functional_ops.scan(lambda a, x: a * (x[0] + x[1]),
                               (elems + 1, -elems), initializer)
-      self.assertAllEqual([1.0, 1.0, 1.0, 1.0, 1.0, 1.0], r.eval())
+      self.assertAllEqual([1.0, 1.0, 1.0, 1.0, 1.0, 1.0], self.evaluate(r))
 
+  @test_util.run_in_graph_and_eager_modes()
   def testScan_MultiInputSameTypeOutput(self):
-    with self.test_session() as sess:
+    with self.test_session():
       elems = np.array([1.0, 2.0, 3.0, 4.0, 5.0, 6.0])
       r = functional_ops.scan(lambda a, x: (a[0] + x[0], a[1] + x[1]),
                               (elems, -elems))
-      r_value = sess.run(r)
+      r_value = self.evaluate(r)
       self.assertAllEqual(np.cumsum(elems), r_value[0])
       self.assertAllEqual(np.cumsum(-elems), r_value[1])
 
+  @test_util.run_in_graph_and_eager_modes()
   def testScan_MultiOutputMismatchedInitializer(self):
     with self.test_session():
       elems = np.array([1.0, 2.0, 3.0, 4.0, 5.0, 6.0])
@@ -316,15 +331,16 @@ class FunctionalOpsTest(test.TestCase):
                          "root/body/two:0")
         sess.run([variables.global_variables_initializer()])
         results = np.array([1, 6, 18, 44, 98, 208])
-        self.assertAllEqual(results, r.eval())
+        self.assertAllEqual(results, self.evaluate(r))
 
         # Now let's reuse our single variable.
         varscope.reuse_variables()
         r = functional_ops.scan(simple_scoped_fn, elems, initializer=2)
         self.assertEqual(len(variables.trainable_variables()), 1)
         results = np.array([6, 16, 38, 84, 178, 368])
-        self.assertAllEqual(results, r.eval())
+        self.assertAllEqual(results, self.evaluate(r))
 
+  @test_util.run_in_graph_and_eager_modes()
   def testScanFoldl_Nested(self):
     with self.test_session():
       elems = constant_op.constant([1.0, 2.0, 3.0, 4.0], name="data")
@@ -346,7 +362,7 @@ class FunctionalOpsTest(test.TestCase):
       # t == 3, a == 2.25, x == 4 (returns 9)
       #   t_0 == 0, b == a == 2.25, y == 0.5, returns b * y * x = 4.5
       #   t_1 == 1, b == 4.5,       y == 0.5, returns b * y * x = 9
-      self.assertAllClose([1., 1., 2.25, 9.], r.eval())
+      self.assertAllClose([1., 1., 2.25, 9.], self.evaluate(r))
 
   def testScan_Control(self):
     with self.test_session() as sess:
@@ -369,7 +385,7 @@ class FunctionalOpsTest(test.TestCase):
           lambda a, x: math_ops.multiply(a, x), elems, initializer=v)
       # pylint: enable=unnecessary-lambda
       r = gradients_impl.gradients(r, v)[0]
-      self.assertAllEqual(873.0, r.eval())
+      self.assertAllEqual(873.0, self.evaluate(r))
 
   def testScanGradientWithPartStopGradient(self):
     a = variables.Variable(0.0, name="a")
@@ -383,6 +399,7 @@ class FunctionalOpsTest(test.TestCase):
       variables.global_variables_initializer().run()
       sess.run(grad)
 
+  @test_util.run_in_graph_and_eager_modes()
   def testFoldShape(self):
     with self.test_session():
       x = constant_op.constant([[1, 2, 3], [4, 5, 6]])
@@ -392,32 +409,37 @@ class FunctionalOpsTest(test.TestCase):
 
       initializer = constant_op.constant([0, 0, 0])
       y = functional_ops.foldl(fn, x, initializer=initializer)
-      self.assertAllEqual(y.get_shape(), y.eval().shape)
+      self.assertAllEqual(y.get_shape(), self.evaluate(y).shape)
 
+  @test_util.run_in_graph_and_eager_modes()
   def testMapShape(self):
     with self.test_session():
       x = constant_op.constant([[1, 2, 3], [4, 5, 6]])
       y = functional_ops.map_fn(lambda e: e, x)
-      self.assertAllEqual(y.get_shape(), y.eval().shape)
+      self.assertAllEqual(y.get_shape(), self.evaluate(y).shape)
 
   def testMapUnknownShape(self):
     x = array_ops.placeholder(dtypes.float32)
     y = functional_ops.map_fn(lambda e: e, x)
     self.assertIs(None, y.get_shape().dims)
 
+  @test_util.run_in_graph_and_eager_modes()
   def testMapEmptyScalar(self):
     with self.test_session():
       map_return = functional_ops.map_fn(lambda x: 1, constant_op.constant([]))
       self.assertAllEqual([0], map_return.get_shape().dims)
-      self.assertAllEqual([0], map_return.eval().shape)
+      self.assertAllEqual([0], self.evaluate(map_return).shape)
 
+  # TODO(akshayka): this test fails in eager: the iterable is of length 0 so
+  # so the body of the while loop never executes
   def testMapEmptyTensor(self):
     with self.test_session():
       map_return = functional_ops.map_fn(lambda x: array_ops.zeros([3, 2]),
                                          constant_op.constant([]))
       self.assertAllEqual([0, 3, 2], map_return.get_shape().dims)
-      self.assertAllEqual([0, 3, 2], map_return.eval().shape)
+      self.assertAllEqual([0, 3, 2], self.evaluate(map_return).shape)
 
+  @test_util.run_in_graph_and_eager_modes()
   def testScanShape(self):
     with self.test_session():
       x = constant_op.constant([[1, 2, 3], [4, 5, 6]])
@@ -427,14 +449,16 @@ class FunctionalOpsTest(test.TestCase):
 
       initializer = constant_op.constant([0, 0, 0])
       y = functional_ops.scan(fn, x, initializer=initializer)
-      self.assertAllEqual(y.get_shape(), y.eval().shape)
+      self.assertAllEqual(y.get_shape(), self.evaluate(y).shape)
 
+  # TODO(akshayka): this test fails in eager: the iterable is of length 0 so
+  # so the body of the while loop never executes
   def testScanEmptyTensor(self):
     with self.test_session():
       x = functional_ops.scan(
           lambda x, _: x, math_ops.range(0), initializer=array_ops.ones([2, 4]))
       self.assertAllEqual([0, 2, 4], x.get_shape())
-      self.assertAllEqual(x.get_shape(), x.eval().shape)
+      self.assertAllEqual(x.get_shape(), self.evaluate(x).shape)
 
   def testScanUnknownShape(self):
     x = array_ops.placeholder(dtypes.float32)
diff --git a/tensorflow/python/ops/functional_ops.py b/tensorflow/python/ops/functional_ops.py
index 96b799f610..688512bea6 100644
--- a/tensorflow/python/ops/functional_ops.py
+++ b/tensorflow/python/ops/functional_ops.py
@@ -27,6 +27,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from tensorflow.python.eager import context
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import sparse_tensor
@@ -87,15 +88,20 @@ def foldl(fn, elems, initializer=None, parallel_iterations=10, back_prop=True,
   if not callable(fn):
     raise TypeError("fn must be callable.")
 
+  in_graph_mode = context.in_graph_mode()
   with ops.name_scope(name, "foldl", [elems]):
-    # Any get_variable calls in fn will cache the first call locally
-    # and not issue repeated network I/O requests for each iteration.
-    varscope = vs.get_variable_scope()
-    varscope_caching_device_was_none = False
-    if varscope.caching_device is None:
-      # TODO(ebrevdo): Change to using colocate_with here and in other methods.
-      varscope.set_caching_device(lambda op: op.device)
-      varscope_caching_device_was_none = True
+    # TODO(akshayka): Remove the in_graph_mode check once caching devices are
+    # supported in Eager
+    if in_graph_mode:
+      # Any get_variable calls in fn will cache the first call locally
+      # and not issue repeated network I/O requests for each iteration.
+      varscope = vs.get_variable_scope()
+      varscope_caching_device_was_none = False
+      if varscope.caching_device is None:
+        # TODO(ebrevdo): Change to using colocate_with here and in other
+        # methods.
+        varscope.set_caching_device(lambda op: op.device)
+        varscope_caching_device_was_none = True
 
     # Convert elems to tensor array.
     elems = ops.convert_to_tensor(elems, name="elems")
@@ -121,7 +127,9 @@ def foldl(fn, elems, initializer=None, parallel_iterations=10, back_prop=True,
         back_prop=back_prop,
         swap_memory=swap_memory)
 
-    if varscope_caching_device_was_none:
+    # TODO(akshayka): Remove the in_graph_mode check once caching devices are
+    # supported in Eager
+    if in_graph_mode and varscope_caching_device_was_none:
       varscope.set_caching_device(None)
     return r_a
 
@@ -167,15 +175,20 @@ def foldr(fn, elems, initializer=None, parallel_iterations=10, back_prop=True,
   if not callable(fn):
     raise TypeError("fn must be callable.")
 
+  in_graph_mode = context.in_graph_mode()
   with ops.name_scope(name, "foldr", [elems]):
-    # Any get_variable calls in fn will cache the first call locally
-    # and not issue repeated network I/O requests for each iteration.
-    varscope = vs.get_variable_scope()
-    varscope_caching_device_was_none = False
-    if varscope.caching_device is None:
-      # TODO(ebrevdo): Change to using colocate_with here and in other methods.
-      varscope.set_caching_device(lambda op: op.device)
-      varscope_caching_device_was_none = True
+    # TODO(akshayka): Remove the in_graph_mode check once caching devices are
+    # supported in Eager
+    if in_graph_mode:
+      # Any get_variable calls in fn will cache the first call locally and not
+      # issue repeated network I/O requests for each iteration.
+      varscope = vs.get_variable_scope()
+      varscope_caching_device_was_none = False
+      if varscope.caching_device is None:
+        # TODO(ebrevdo): Change to using colocate_with here and in other
+        # methods.
+        varscope.set_caching_device(lambda op: op.device)
+        varscope_caching_device_was_none = True
 
     # Convert elems to tensor array.
     elems = ops.convert_to_tensor(elems, name="elems")
@@ -201,7 +214,9 @@ def foldr(fn, elems, initializer=None, parallel_iterations=10, back_prop=True,
         back_prop=back_prop,
         swap_memory=swap_memory)
 
-    if varscope_caching_device_was_none:
+    # TODO(akshayka): Remove the in_graph_mode check once caching devices are
+    # supported in Eager
+    if in_graph_mode and varscope_caching_device_was_none:
       varscope.set_caching_device(None)
     return r_a
 
@@ -324,15 +339,20 @@ def map_fn(fn, elems, dtype=None, parallel_iterations=10, back_prop=True,
 
   elems_flat = input_flatten(elems)
 
+  in_graph_mode = context.in_graph_mode()
   with ops.name_scope(name, "map", elems_flat):
-    # Any get_variable calls in fn will cache the first call locally
-    # and not issue repeated network I/O requests for each iteration.
-    varscope = vs.get_variable_scope()
-    varscope_caching_device_was_none = False
-    if varscope.caching_device is None:
-      # TODO(ebrevdo): Change to using colocate_with here and in other methods.
-      varscope.set_caching_device(lambda op: op.device)
-      varscope_caching_device_was_none = True
+    # TODO(akshayka): Remove the in_graph_mode check once caching devices are
+    # supported in Eager
+    if in_graph_mode:
+      # Any get_variable calls in fn will cache the first call locally
+      # and not issue repeated network I/O requests for each iteration.
+      varscope = vs.get_variable_scope()
+      varscope_caching_device_was_none = False
+      if varscope.caching_device is None:
+        # TODO(ebrevdo): Change to using colocate_with here and in other
+        # methods.
+        varscope.set_caching_device(lambda op: op.device)
+        varscope_caching_device_was_none = True
 
     elems_flat = [
         ops.convert_to_tensor(elem, name="elem") for elem in elems_flat]
@@ -396,7 +416,9 @@ def map_fn(fn, elems, dtype=None, parallel_iterations=10, back_prop=True,
       r.set_shape(tensor_shape.TensorShape(n_static).concatenate(
           r.get_shape()[1:]))
 
-    if varscope_caching_device_was_none:
+    # TODO(akshayka): Remove the in_graph_mode check once caching devices are
+    # supported in Eager
+    if in_graph_mode and varscope_caching_device_was_none:
       varscope.set_caching_device(None)
 
     return output_pack(results_flat)
@@ -509,15 +531,20 @@ def scan(fn, elems, initializer=None, parallel_iterations=10, back_prop=True,
 
   elems_flat = input_flatten(elems)
 
+  in_graph_mode = context.in_graph_mode()
   with ops.name_scope(name, "scan", elems_flat):
-    # Any get_variable calls in fn will cache the first call locally
-    # and not issue repeated network I/O requests for each iteration.
-    varscope = vs.get_variable_scope()
-    varscope_caching_device_was_none = False
-    if varscope.caching_device is None:
-      # TODO(ebrevdo): Change to using colocate_with here and in other methods.
-      varscope.set_caching_device(lambda op: op.device)
-      varscope_caching_device_was_none = True
+    # TODO(akshayka): Remove the in_graph_mode check once caching devices are
+    # supported in Eager
+    if in_graph_mode:
+      # Any get_variable calls in fn will cache the first call locally
+      # and not issue repeated network I/O requests for each iteration.
+      varscope = vs.get_variable_scope()
+      varscope_caching_device_was_none = False
+      if varscope.caching_device is None:
+        # TODO(ebrevdo): Change to using colocate_with here and in other
+        # methods.
+        varscope.set_caching_device(lambda op: op.device)
+        varscope_caching_device_was_none = True
 
     # Convert elems to tensor array.
     elems_flat = [
@@ -594,7 +621,9 @@ def scan(fn, elems, initializer=None, parallel_iterations=10, back_prop=True,
       r.set_shape(tensor_shape.TensorShape(n_static).concatenate(
           r.get_shape()[1:]))
 
-    if varscope_caching_device_was_none:
+    # TODO(akshayka): Remove the in_graph_mode check once caching devices are
+    # supported in Eager
+    if in_graph_mode and varscope_caching_device_was_none:
       varscope.set_caching_device(None)
 
     return output_pack(results_flat)
-- 
GitLab


From fb2c84cb27c7427455245c20fb22fb2489895b2e Mon Sep 17 00:00:00 2001
From: Jeremy Lau <lauj@google.com>
Date: Fri, 27 Oct 2017 17:32:30 -0700
Subject: [PATCH 898/909] Internal change

PiperOrigin-RevId: 173738655
---
 tensorflow/compiler/tests/BUILD | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/compiler/tests/BUILD b/tensorflow/compiler/tests/BUILD
index d07bf98296..0ff99c5156 100644
--- a/tensorflow/compiler/tests/BUILD
+++ b/tensorflow/compiler/tests/BUILD
@@ -464,7 +464,7 @@ tf_xla_py_test(
 
 tf_xla_py_test(
     name = "unary_ops_test",
-    size = "small",
+    size = "medium",
     srcs = ["unary_ops_test.py"],
     deps = [
         ":xla_test",
-- 
GitLab


From 48df7c97296472730e8547bb3aa59e6730e956cd Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 27 Oct 2017 17:34:05 -0700
Subject: [PATCH 899/909] Update ops-related pbtxt files.

PiperOrigin-RevId: 173738765
---
 tensorflow/core/ops/ops.pbtxt | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index e43ee0d986..f41cb212be 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -7049,7 +7049,7 @@ op {
     }
   }
   summary: "Dequantize the \'input\' tensor into a float Tensor."
-  description: "[min_range, max_range] are scalar floats that specify the range for\nthe \'input\' data. The \'mode\' attribute controls exactly which calculations are\nused to convert the float values to their quantized equivalents.\n\nIn \'MIN_COMBINED\' mode, each value of the tensor will undergo the following:\n\n```\nif T == qint8, in[i] += (range(T) + 1)/ 2.0\nout[i] = min_range + (in[i]* (max_range - min_range) / range(T))\n```\nhere `range(T) = numeric_limits<T>::max() - numeric_limits<T>::min()`\n\n*MIN_COMBINED Mode Example*\n\nIf the input comes from a QuantizedRelu6, the output type is\nquint8 (range of 0-255) but the possible range of QuantizedRelu6 is\n0-6.  The min_range and max_range values are therefore 0.0 and 6.0.\nDequantize on quint8 will take each value, cast to float, and multiply\nby 6 / 255.\nNote that if quantizedtype is qint8, the operation will additionally add\neach value by 128 prior to casting.\n\nIf the mode is \'MIN_FIRST\', then this approach is used:\n\n```c++\nnumber_of_steps = 1 << (# of bits in T)\nrange_adjust = number_of_steps / (number_of_steps - 1)\nrange = (range_max - range_min) * range_adjust\nrange_scale = range / number_of_steps\nconst double offset_input = static_cast<double>(input) - lowest_quantized;\nresult = range_min + ((input - numeric_limits<T>::min()) * range_scale)\n```\n\n*SCALED mode Example*\n\n`SCALED` mode matches the quantization approach used in\n`QuantizeAndDequantize{V2|V3}`.\n\nIf the mode is `SCALED`, we do not use the full range of the output type,\nchoosing to elide the lowest possible value for symmetry (e.g., output range is\n-127 to 127, not -128 to 127 for signed 8 bit quantization), so that 0.0 maps to\n0.\n\nWe first find the range of values in our tensor. The\nrange we use is always centered on 0, so we find m such that\n```c++\n  m = max(abs(input_min), abs(input_max))\n```\n\nOur input tensor range is then `[-m, m]`.\n\nNext, we choose our fixed-point quantization buckets, `[min_fixed, max_fixed]`.\nIf T is signed, this is\n```\n  num_bits = sizeof(T) * 8\n  [min_fixed, max_fixed] =\n      [-(1 << (num_bits - 1) - 1), (1 << (num_bits - 1)) - 1]\n```\n\nOtherwise, if T is unsigned, the fixed-point range is\n```\n  [min_fixed, max_fixed] = [0, (1 << num_bits) - 1]\n```\n\nFrom this we compute our scaling factor, s:\n```c++\n  s = (2 * m) / (max_fixed - min_fixed)\n```\n\nNow we can dequantize the elements of our tensor:\n```c++\nresult = input * s\n```"
+  description: "[min_range, max_range] are scalar floats that specify the range for\nthe \'input\' data. The \'mode\' attribute controls exactly which calculations are\nused to convert the float values to their quantized equivalents.\n\nIn \'MIN_COMBINED\' mode, each value of the tensor will undergo the following:\n\n```\nif T == qint8, in[i] += (range(T) + 1)/ 2.0\nout[i] = min_range + (in[i]* (max_range - min_range) / range(T))\n```\nhere `range(T) = numeric_limits<T>::max() - numeric_limits<T>::min()`\n\n*MIN_COMBINED Mode Example*\n\nIf the input comes from a QuantizedRelu6, the output type is\nquint8 (range of 0-255) but the possible range of QuantizedRelu6 is\n0-6.  The min_range and max_range values are therefore 0.0 and 6.0.\nDequantize on quint8 will take each value, cast to float, and multiply\nby 6 / 255.\nNote that if quantizedtype is qint8, the operation will additionally add\neach value by 128 prior to casting.\n\nIf the mode is \'MIN_FIRST\', then this approach is used:\n\n```c++\nnum_discrete_values = 1 << (# of bits in T)\nrange_adjust = num_discrete_values / (num_discrete_values - 1)\nrange = (range_max - range_min) * range_adjust\nrange_scale = range / num_discrete_values\nconst double offset_input = static_cast<double>(input) - lowest_quantized;\nresult = range_min + ((input - numeric_limits<T>::min()) * range_scale)\n```\n\n*SCALED mode Example*\n\n`SCALED` mode matches the quantization approach used in\n`QuantizeAndDequantize{V2|V3}`.\n\nIf the mode is `SCALED`, we do not use the full range of the output type,\nchoosing to elide the lowest possible value for symmetry (e.g., output range is\n-127 to 127, not -128 to 127 for signed 8 bit quantization), so that 0.0 maps to\n0.\n\nWe first find the range of values in our tensor. The\nrange we use is always centered on 0, so we find m such that\n```c++\n  m = max(abs(input_min), abs(input_max))\n```\n\nOur input tensor range is then `[-m, m]`.\n\nNext, we choose our fixed-point quantization buckets, `[min_fixed, max_fixed]`.\nIf T is signed, this is\n```\n  num_bits = sizeof(T) * 8\n  [min_fixed, max_fixed] =\n      [-(1 << (num_bits - 1) - 1), (1 << (num_bits - 1)) - 1]\n```\n\nOtherwise, if T is unsigned, the fixed-point range is\n```\n  [min_fixed, max_fixed] = [0, (1 << num_bits) - 1]\n```\n\nFrom this we compute our scaling factor, s:\n```c++\n  s = (2 * m) / (max_fixed - min_fixed)\n```\n\nNow we can dequantize the elements of our tensor:\n```c++\nresult = input * s\n```"
 }
 op {
   name: "DeserializeIterator"
@@ -17500,7 +17500,7 @@ op {
     }
   }
   summary: "Quantize the \'input\' tensor of type float to \'output\' tensor of type \'T\'."
-  description: "[min_range, max_range] are scalar floats that specify the range for\nthe \'input\' data. The \'mode\' attribute controls exactly which calculations are\nused to convert the float values to their quantized equivalents.  The\n\'round_mode\' attribute controls which rounding tie-breaking algorithm is used\nwhen rounding float values to their quantized equivalents.\n\nIn \'MIN_COMBINED\' mode, each value of the tensor will undergo the following:\n\n```\nout[i] = (in[i] - min_range) * range(T) / (max_range - min_range)\nif T == qint8, out[i] -= (range(T) + 1) / 2.0\n```\nhere `range(T) = numeric_limits<T>::max() - numeric_limits<T>::min()`\n\n*MIN_COMBINED Mode Example*\n\nAssume the input is type float and has a possible range of [0.0, 6.0] and the\noutput type is quint8 ([0, 255]). The min_range and max_range values should be\nspecified as 0.0 and 6.0. Quantizing from float to quint8 will multiply each\nvalue of the input by 255/6 and cast to quint8.\n\nIf the output type was qint8 ([-128, 127]), the operation will additionally\nsubtract each value by 128 prior to casting, so that the range of values aligns\nwith the range of qint8.\n\nIf the mode is \'MIN_FIRST\', then this approach is used:\n\n```\nnumber_of_steps = 1 << (# of bits in T)\nrange_adjust = number_of_steps / (number_of_steps - 1)\nrange = (range_max - range_min) * range_adjust\nrange_scale = number_of_steps / range\nquantized = round(input * range_scale) - round(range_min * range_scale) +\n  numeric_limits<T>::min()\nquantized = max(quantized, numeric_limits<T>::min())\nquantized = min(quantized, numeric_limits<T>::max())\n```\n\nThe biggest difference between this and MIN_COMBINED is that the minimum range\nis rounded first, before it\'s subtracted from the rounded value. With\nMIN_COMBINED, a small bias is introduced where repeated iterations of quantizing\nand dequantizing will introduce a larger and larger error.\n\n*SCALED mode Example*\n\n`SCALED` mode matches the quantization approach used in\n`QuantizeAndDequantize{V2|V3}`.\n\nIf the mode is `SCALED`, we do not use the full range of the output type,\nchoosing to elide the lowest possible value for symmetry (e.g., output range is\n-127 to 127, not -128 to 127 for signed 8 bit quantization), so that 0.0 maps to\n0.\n\nWe first find the range of values in our tensor. The\nrange we use is always centered on 0, so we find m such that\n```c++\n  m = max(abs(input_min), abs(input_max))\n```\n\nOur input tensor range is then `[-m, m]`.\n\nNext, we choose our fixed-point quantization buckets, `[min_fixed, max_fixed]`.\nIf T is signed, this is\n```\n  num_bits = sizeof(T) * 8\n  [min_fixed, max_fixed] =\n      [-(1 << (num_bits - 1) - 1), (1 << (num_bits - 1)) - 1]\n```\n\nOtherwise, if T is unsigned, the fixed-point range is\n```\n  [min_fixed, max_fixed] = [0, (1 << num_bits) - 1]\n```\n\nFrom this we compute our scaling factor, s:\n```c++\n  s = (max_fixed - min_fixed) / (2 * m)\n```\n\nNow we can quantize the elements of our tensor:\n```c++\nresult = round(input * s)\n```\n\nOne thing to watch out for is that the operator may choose to adjust the\nrequested minimum and maximum values slightly during the quantization process,\nso you should always use the output ports as the range for further calculations.\nFor example, if the requested minimum and maximum values are close to equal,\nthey will be separated by a small epsilon value to prevent ill-formed quantized\nbuffers from being created. Otherwise, you can end up with buffers where all the\nquantized values map to the same float value, which causes problems for\noperations that have to perform further calculations on them."
+  description: "[min_range, max_range] are scalar floats that specify the range for\nthe \'input\' data. The \'mode\' attribute controls exactly which calculations are\nused to convert the float values to their quantized equivalents.  The\n\'round_mode\' attribute controls which rounding tie-breaking algorithm is used\nwhen rounding float values to their quantized equivalents.\n\nIn \'MIN_COMBINED\' mode, each value of the tensor will undergo the following:\n\n```\nout[i] = (in[i] - min_range) * range(T) / (max_range - min_range)\nif T == qint8, out[i] -= (range(T) + 1) / 2.0\n```\nhere `range(T) = numeric_limits<T>::max() - numeric_limits<T>::min()`\n\n*MIN_COMBINED Mode Example*\n\nAssume the input is type float and has a possible range of [0.0, 6.0] and the\noutput type is quint8 ([0, 255]). The min_range and max_range values should be\nspecified as 0.0 and 6.0. Quantizing from float to quint8 will multiply each\nvalue of the input by 255/6 and cast to quint8.\n\nIf the output type was qint8 ([-128, 127]), the operation will additionally\nsubtract each value by 128 prior to casting, so that the range of values aligns\nwith the range of qint8.\n\nIf the mode is \'MIN_FIRST\', then this approach is used:\n\n```\nnum_discrete_values = 1 << (# of bits in T)\nrange_adjust = num_discrete_values / (num_discrete_values - 1)\nrange = (range_max - range_min) * range_adjust\nrange_scale = num_discrete_values / range\nquantized = round(input * range_scale) - round(range_min * range_scale) +\n  numeric_limits<T>::min()\nquantized = max(quantized, numeric_limits<T>::min())\nquantized = min(quantized, numeric_limits<T>::max())\n```\n\nThe biggest difference between this and MIN_COMBINED is that the minimum range\nis rounded first, before it\'s subtracted from the rounded value. With\nMIN_COMBINED, a small bias is introduced where repeated iterations of quantizing\nand dequantizing will introduce a larger and larger error.\n\n*SCALED mode Example*\n\n`SCALED` mode matches the quantization approach used in\n`QuantizeAndDequantize{V2|V3}`.\n\nIf the mode is `SCALED`, we do not use the full range of the output type,\nchoosing to elide the lowest possible value for symmetry (e.g., output range is\n-127 to 127, not -128 to 127 for signed 8 bit quantization), so that 0.0 maps to\n0.\n\nWe first find the range of values in our tensor. The\nrange we use is always centered on 0, so we find m such that\n```c++\n  m = max(abs(input_min), abs(input_max))\n```\n\nOur input tensor range is then `[-m, m]`.\n\nNext, we choose our fixed-point quantization buckets, `[min_fixed, max_fixed]`.\nIf T is signed, this is\n```\n  num_bits = sizeof(T) * 8\n  [min_fixed, max_fixed] =\n      [-(1 << (num_bits - 1) - 1), (1 << (num_bits - 1)) - 1]\n```\n\nOtherwise, if T is unsigned, the fixed-point range is\n```\n  [min_fixed, max_fixed] = [0, (1 << num_bits) - 1]\n```\n\nFrom this we compute our scaling factor, s:\n```c++\n  s = (max_fixed - min_fixed) / (2 * m)\n```\n\nNow we can quantize the elements of our tensor:\n```c++\nresult = round(input * s)\n```\n\nOne thing to watch out for is that the operator may choose to adjust the\nrequested minimum and maximum values slightly during the quantization process,\nso you should always use the output ports as the range for further calculations.\nFor example, if the requested minimum and maximum values are close to equal,\nthey will be separated by a small epsilon value to prevent ill-formed quantized\nbuffers from being created. Otherwise, you can end up with buffers where all the\nquantized values map to the same float value, which causes problems for\noperations that have to perform further calculations on them."
 }
 op {
   name: "QuantizedAdd"
-- 
GitLab


From ca56fa49a7755ba2bbd3f586dbaaaefe9e16327d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 27 Oct 2017 17:39:28 -0700
Subject: [PATCH 900/909] Go: Update generated wrapper functions for TensorFlow
 ops.

PiperOrigin-RevId: 173739110
---
 tensorflow/go/op/wrappers.go | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go
index 615c386858..ebe4a51116 100644
--- a/tensorflow/go/op/wrappers.go
+++ b/tensorflow/go/op/wrappers.go
@@ -1904,10 +1904,10 @@ func DequantizeMode(value string) DequantizeAttr {
 // If the mode is 'MIN_FIRST', then this approach is used:
 //
 // ```c++
-// number_of_steps = 1 << (# of bits in T)
-// range_adjust = number_of_steps / (number_of_steps - 1)
+// num_discrete_values = 1 << (# of bits in T)
+// range_adjust = num_discrete_values / (num_discrete_values - 1)
 // range = (range_max - range_min) * range_adjust
-// range_scale = range / number_of_steps
+// range_scale = range / num_discrete_values
 // const double offset_input = static_cast<double>(input) - lowest_quantized;
 // result = range_min + ((input - numeric_limits<T>::min()) * range_scale)
 // ```
@@ -13766,10 +13766,10 @@ func QuantizeV2RoundMode(value string) QuantizeV2Attr {
 // If the mode is 'MIN_FIRST', then this approach is used:
 //
 // ```
-// number_of_steps = 1 << (# of bits in T)
-// range_adjust = number_of_steps / (number_of_steps - 1)
+// num_discrete_values = 1 << (# of bits in T)
+// range_adjust = num_discrete_values / (num_discrete_values - 1)
 // range = (range_max - range_min) * range_adjust
-// range_scale = number_of_steps / range
+// range_scale = num_discrete_values / range
 // quantized = round(input * range_scale) - round(range_min * range_scale) +
 //   numeric_limits<T>::min()
 // quantized = max(quantized, numeric_limits<T>::min())
-- 
GitLab


From 729db035e7aaa8811dccff154dc582fab12ccee3 Mon Sep 17 00:00:00 2001
From: Mark Daoust <markdaoust@google.com>
Date: Fri, 27 Oct 2017 17:47:54 -0700
Subject: [PATCH 901/909] Allow compatibility notes in class, property and
 module doc-strings

PiperOrigin-RevId: 173739674
---
 tensorflow/tools/docs/pretty_docs.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/tensorflow/tools/docs/pretty_docs.py b/tensorflow/tools/docs/pretty_docs.py
index 92f50189dd..c033c16ae9 100644
--- a/tensorflow/tools/docs/pretty_docs.py
+++ b/tensorflow/tools/docs/pretty_docs.py
@@ -117,7 +117,8 @@ def _build_class_page(page_info):
   parts.append(page_info.guides)
   parts.append(page_info.doc.docstring)
   parts.append(_build_function_details(page_info.doc.function_details))
-  assert not page_info.doc.compatibility
+  parts.append(_build_compatibility(page_info.doc.compatibility))
+
   parts.append('\n\n')
 
   if page_info.classes:
@@ -139,7 +140,8 @@ def _build_class_page(page_info):
 
       parts.append(prop_info.doc.docstring)
       parts.append(_build_function_details(prop_info.doc.function_details))
-      assert not prop_info.doc.compatibility
+      parts.append(_build_compatibility(prop_info.doc.compatibility))
+
       parts.append('\n\n')
 
     parts.append('\n\n')
@@ -206,6 +208,8 @@ def _build_module_page(page_info):
     parts.append(str(page_info.defined_in))
 
   parts.append(page_info.doc.docstring)
+  parts.append(_build_compatibility(page_info.doc.compatibility))
+
   parts.append('\n\n')
 
   if page_info.modules:
-- 
GitLab


From 09a89ae57d92b9753c76fa298d373468cb05cc6a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 27 Oct 2017 18:01:37 -0700
Subject: [PATCH 902/909] Add `tf.contrib.distributions.bijectors.Reshape`.

PiperOrigin-RevId: 173740491
---
 tensorflow/contrib/distributions/BUILD        |  16 +
 .../kernel_tests/bijectors/reshape_test.py    | 242 ++++++++++++++
 .../python/ops/bijectors/__init__.py          |   2 +
 .../python/ops/bijectors/reshape.py           |  29 ++
 .../python/ops/bijectors/reshape_impl.py      | 297 ++++++++++++++++++
 5 files changed, 586 insertions(+)
 create mode 100644 tensorflow/contrib/distributions/python/kernel_tests/bijectors/reshape_test.py
 create mode 100644 tensorflow/contrib/distributions/python/ops/bijectors/reshape.py
 create mode 100644 tensorflow/contrib/distributions/python/ops/bijectors/reshape_impl.py

diff --git a/tensorflow/contrib/distributions/BUILD b/tensorflow/contrib/distributions/BUILD
index bc72bc37a7..4a4f378901 100644
--- a/tensorflow/contrib/distributions/BUILD
+++ b/tensorflow/contrib/distributions/BUILD
@@ -913,6 +913,22 @@ cuda_py_test(
     ],
 )
 
+cuda_py_test(
+    name = "reshape_test",
+    size = "small",
+    srcs = ["python/kernel_tests/bijectors/reshape_test.py"],
+    additional_deps = [
+        ":bijectors_py",
+        ":distributions_py",
+        "//third_party/py/numpy",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework_for_generated_wrappers",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:platform_test",
+    ],
+)
+
 cuda_py_test(
     name = "sigmoid_test",
     size = "small",
diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/reshape_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/reshape_test.py
new file mode 100644
index 0000000000..38b3a23c2d
--- /dev/null
+++ b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/reshape_test.py
@@ -0,0 +1,242 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for Reshape Bijector."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.contrib.distributions.python.ops.bijectors.reshape import Reshape
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import tensor_shape
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops.distributions.bijector_test_util import assert_bijective_and_finite
+from tensorflow.python.platform import test
+
+
+class ReshapeBijectorTest(test.TestCase):
+  """Tests correctness of the reshape transformation."""
+
+  def setUp(self):
+    self._rng = np.random.RandomState(42)
+
+  def testBijector(self):
+    """Do a basic sanity check of forward, inverse, jacobian."""
+    expected_x = np.random.randn(4, 3, 2)
+    expected_y = np.reshape(expected_x, [4, 6])
+
+    with self.test_session() as sess:
+      bijector = Reshape(
+          event_shape_out=[6,],
+          event_shape_in=[3, 2],
+          validate_args=True)
+      (x_,
+       y_,
+       fldj_,
+       ildj_) = sess.run((
+           bijector.inverse(expected_y),
+           bijector.forward(expected_x),
+           bijector.forward_log_det_jacobian(expected_x),
+           bijector.inverse_log_det_jacobian(expected_y),
+       ))
+      self.assertEqual("reshape", bijector.name)
+      self.assertAllClose(expected_y, y_, rtol=1e-6, atol=0)
+      self.assertAllClose(expected_x, x_, rtol=1e-6, atol=0)
+      self.assertAllClose(0., fldj_, rtol=1e-6, atol=0)
+      self.assertAllClose(0., ildj_, rtol=1e-6, atol=0)
+
+  def testEventShapeDynamicNdims(self):
+    """Check forward/inverse shape methods with dynamic ndims."""
+
+    shape_in = tensor_shape.TensorShape([6,])
+    shape_in_ph = array_ops.placeholder(dtype=dtypes.int32)
+
+    shape_out = tensor_shape.TensorShape([2, 3])
+    shape_out_ph = array_ops.placeholder(dtype=dtypes.int32)
+
+    bijector = Reshape(
+        event_shape_out=shape_out_ph,
+        event_shape_in=shape_in_ph, validate_args=True)
+
+    # using the _tensor methods, we should always get a fully-specified
+    # result since these are evaluated at graph runtime.
+    with self.test_session() as sess:
+      (shape_out_,
+       shape_in_) = sess.run((
+           bijector.forward_event_shape_tensor(shape_in),
+           bijector.inverse_event_shape_tensor(shape_out),
+       ), feed_dict={
+           shape_in_ph: shape_in,
+           shape_out_ph: shape_out,
+       })
+      self.assertAllEqual(shape_out, shape_out_)
+      self.assertAllEqual(shape_in, shape_in_)
+
+  def testEventShapeDynamic(self):
+    """Check shape methods with static ndims but dynamic shape."""
+
+    shape_in = tensor_shape.TensorShape([6,])
+    shape_in_partial = tensor_shape.TensorShape([None,])
+    shape_in_ph = array_ops.placeholder(
+        shape=[1,], dtype=dtypes.int32)
+
+    shape_out = tensor_shape.TensorShape([2, 3])
+    shape_out_partial = tensor_shape.TensorShape([None, None])
+    shape_out_ph = array_ops.placeholder(
+        shape=[2,], dtype=dtypes.int32)
+
+    bijector = Reshape(
+        event_shape_out=shape_out_ph,
+        event_shape_in=shape_in_ph,
+        validate_args=True)
+
+    # if event shapes are not statically available, should
+    # return partially-specified TensorShapes.
+    self.assertAllEqual(
+        bijector.forward_event_shape(shape_in).as_list(),
+        shape_out_partial.as_list())
+    self.assertAllEqual(
+        bijector.inverse_event_shape(shape_out).as_list(),
+        shape_in_partial.as_list())
+
+    # using the _tensor methods, we should always get a fully-specified
+    # result since these are evaluated at graph runtime.
+    with self.test_session() as sess:
+      (shape_out_,
+       shape_in_) = sess.run((
+           bijector.forward_event_shape_tensor(shape_in),
+           bijector.inverse_event_shape_tensor(shape_out),
+       ), feed_dict={
+           shape_in_ph: shape_in,
+           shape_out_ph: shape_out,
+       })
+      self.assertAllEqual(shape_out, shape_out_)
+      self.assertAllEqual(shape_in, shape_in_)
+
+  def testEventShapeStatic(self):
+    """Check shape methods when shape is statically known."""
+
+    shape_in = tensor_shape.TensorShape([6,])
+    shape_out = tensor_shape.TensorShape([2, 3])
+
+    bijector_static = Reshape(
+        event_shape_out=shape_out,
+        event_shape_in=shape_in,
+        validate_args=True)
+
+    # test that forward_ and inverse_event_shape do sensible things
+    # when shapes are statically known.
+    self.assertEqual(
+        bijector_static.forward_event_shape(shape_in),
+        shape_out)
+    self.assertEqual(
+        bijector_static.inverse_event_shape(shape_out),
+        shape_in)
+
+    with self.test_session() as sess:
+      (shape_out_static_,
+       shape_in_static_,
+      ) = sess.run((
+          bijector_static.forward_event_shape_tensor(shape_in),
+          bijector_static.inverse_event_shape_tensor(shape_out),
+      ))
+      self.assertAllEqual(shape_out, shape_out_static_)
+      self.assertAllEqual(shape_in, shape_in_static_)
+
+  def testScalarReshape(self):
+    """Test reshaping to and from a scalar shape ()."""
+
+    expected_x = np.random.randn(4, 3, 1)
+    expected_y = np.reshape(expected_x, [4, 3])
+
+    expected_x_scalar = np.random.randn(1,)
+    expected_y_scalar = expected_x_scalar[0]
+
+    with self.test_session() as sess:
+      bijector = Reshape(
+          event_shape_out=[],
+          event_shape_in=[1,], validate_args=True)
+
+      (x_,
+       y_,
+       x_scalar_,
+       y_scalar_
+      ) = sess.run((
+          bijector.inverse(expected_y),
+          bijector.forward(expected_x),
+          bijector.inverse(expected_y_scalar),
+          bijector.forward(expected_x_scalar),
+      ))
+      self.assertAllClose(expected_y, y_, rtol=1e-6, atol=0)
+      self.assertAllClose(expected_x, x_, rtol=1e-6, atol=0)
+      self.assertAllClose(expected_y_scalar, y_scalar_, rtol=1e-6, atol=0)
+      self.assertAllClose(expected_x_scalar, x_scalar_, rtol=1e-6, atol=0)
+
+  def testRaisesOpError(self):
+    x1 = np.random.randn(4, 2, 3)
+    x2 = np.random.randn(4, 3, 2)
+    x3 = np.random.randn(4, 5, 1, 1)
+
+    with self.test_session() as sess:
+      shape_in_ph = array_ops.placeholder(shape=[2,], dtype=dtypes.int32)
+      shape_out_ph = array_ops.placeholder(shape=[3,], dtype=dtypes.int32)
+      bijector = Reshape(
+          event_shape_out=shape_out_ph,
+          event_shape_in=shape_in_ph,
+          validate_args=True)
+
+      with self.assertRaisesOpError(
+          "Input `event_shape` does not match `event_shape_in`."):
+        sess.run(bijector.forward(x2),
+                 feed_dict={shape_out_ph: [1, 6, 1],
+                            shape_in_ph: [2, 3]})
+
+      with self.assertRaisesOpError(
+          "event_shape_out entries must be positive."):
+        sess.run(bijector.forward(x1),
+                 feed_dict={shape_out_ph: [-1, -1, 6],
+                            shape_in_ph: [2, 3]})
+
+      # test that *all* methods check basic assertions
+      fd_mismatched = {shape_out_ph: [1, 1, 5], shape_in_ph: [2, 3]}
+      with self.assertRaisesOpError(
+          "Input/output `event_size`s do not match."):
+        sess.run(bijector.forward(x1), feed_dict=fd_mismatched)
+      with self.assertRaisesOpError(
+          "Input/output `event_size`s do not match."):
+        sess.run(bijector.inverse(x3), feed_dict=fd_mismatched)
+      with self.assertRaisesOpError(
+          "Input/output `event_size`s do not match."):
+        sess.run(bijector.inverse_log_det_jacobian(x3),
+                 feed_dict=fd_mismatched)
+      with self.assertRaisesOpError(
+          "Input/output `event_size`s do not match."):
+        sess.run(bijector.forward_log_det_jacobian(x1),
+                 feed_dict=fd_mismatched)
+
+  def testBijectiveAndFinite(self):
+    x = np.random.randn(4, 2, 3)
+    y = np.reshape(x, [4, 1, 2, 3])
+    with self.test_session():
+      bijector = Reshape(
+          event_shape_in=[2, 3],
+          event_shape_out=[1, 2, 3],
+          validate_args=True)
+      assert_bijective_and_finite(bijector, x, y, rtol=1e-6, atol=0)
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/__init__.py b/tensorflow/contrib/distributions/python/ops/bijectors/__init__.py
index fd6c509446..bc0ec7f195 100644
--- a/tensorflow/contrib/distributions/python/ops/bijectors/__init__.py
+++ b/tensorflow/contrib/distributions/python/ops/bijectors/__init__.py
@@ -29,6 +29,7 @@
 @@MaskedAutoregressiveFlow
 @@Permute
 @@PowerTransform
+@@Reshape
 @@Sigmoid
 @@SigmoidCentered
 @@SinhArcsinh
@@ -59,6 +60,7 @@ from tensorflow.contrib.distributions.python.ops.bijectors.invert import *
 from tensorflow.contrib.distributions.python.ops.bijectors.masked_autoregressive import *
 from tensorflow.contrib.distributions.python.ops.bijectors.permute import *
 from tensorflow.contrib.distributions.python.ops.bijectors.power_transform import *
+from tensorflow.contrib.distributions.python.ops.bijectors.reshape import *
 from tensorflow.contrib.distributions.python.ops.bijectors.sigmoid import *
 from tensorflow.contrib.distributions.python.ops.bijectors.sigmoid_centered import *
 from tensorflow.contrib.distributions.python.ops.bijectors.sinh_arcsinh import *
diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/reshape.py b/tensorflow/contrib/distributions/python/ops/bijectors/reshape.py
new file mode 100644
index 0000000000..8997f7ab69
--- /dev/null
+++ b/tensorflow/contrib/distributions/python/ops/bijectors/reshape.py
@@ -0,0 +1,29 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Reshape bijector."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+# go/tf-wildcard-import
+# pylint: disable=wildcard-import
+from tensorflow.contrib.distributions.python.ops.bijectors.reshape_impl import *
+# pylint: enable=wildcard-import
+from tensorflow.python.util.all_util import remove_undocumented
+
+_allowed_symbols = ["Reshape"]
+
+remove_undocumented(__name__, _allowed_symbols)
diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/reshape_impl.py b/tensorflow/contrib/distributions/python/ops/bijectors/reshape_impl.py
new file mode 100644
index 0000000000..93682639aa
--- /dev/null
+++ b/tensorflow/contrib/distributions/python/ops/bijectors/reshape_impl.py
@@ -0,0 +1,297 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Reshape bijectors."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import tensor_util
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import check_ops
+from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops.distributions import bijector as bijector_lib
+
+
+__all__ = [
+    "Reshape",
+]
+
+
+class Reshape(bijector_lib.Bijector):
+  """Reshapes the `event_shape` of a `Tensor`.
+
+  The semantics generally follow that of `tf.reshape()`, with
+  a few differences:
+   * The user must provide both the input and output shape, so that
+     the transformation can be inverted.
+   * The `Reshape` bijector automatically broadcasts over the leftmost
+     dimensions of its input (`sample_shape` and `batch_shape`); only
+     the rightmost `event_ndims_in` dimensions are reshaped. The
+     number of dimensions to reshape is inferred from the provided
+     `event_shape_in` (`event_ndims_in = len(event_shape_in)`).
+   * The `Reshape` bijector does not currently support
+     partially-specified shapes, i.e., those with a dimension
+     implicitly specified by `-1`.
+
+  Example usage:
+  ```python
+
+  bs = tf.contrib.distributions.bijectors
+
+  reverse = bs.Reshape(event_shape_out=[1,2],
+                       event_shape_in=[2,])
+
+  reverse.forward([1., 2.])    # shape [2,]
+  # ==> [[1., 2.]]             # shape [1,2]
+
+  reverse.forward([[1., 2.], [3., 4.]])  # shape [2, 2]
+  # ==> [[[1., 2.]], [[3., 4.]]]         # shape [2, 1, 2]
+
+  reverse.inverse([[1., 2.]])  # shape [1,2]
+  # ==> [1., 2.]               # shape [2,]
+
+  reverse.forward_log_det_jacobian(any_value)
+  # ==> 0.
+
+  reverse.inverse_log_det_jacobian(any_value)
+  # ==> 0.
+  ```
+
+  """
+
+  def __init__(self, event_shape_out, event_shape_in,
+               validate_args=False, name=None):
+    """Creates a `Reshape` bijector.
+
+    Args:
+      event_shape_out: An `int`-like vector-shaped `Tensor`
+        representing the fully specified (no -1's) event shape of the
+        transformed output.
+      event_shape_in: An `int`-like vector-shaped `Tensor`
+        representing the fully specified (no -1's) event shape of the
+        input.
+      validate_args: Python `bool` indicating whether arguments should
+        be checked for correctness.
+      name: Python `str`, name given to ops managed by this object.
+
+    Raises:
+      TypeError: if either `event_shape_in` or `event_shape_out` has
+       non-vector shape (`rank > 1`), or non-integer `dtype`.
+      ValueError: if either `event_shape_in` or `event_shape_out`
+       contains non-positive entries, or if their sizes do not match
+       (`prod(event_shape_in)` != `prod(event_shape_out)`), or if
+       their dimensionality(s) cannot be statically inferred.
+    """
+    with ops.name_scope(name, "reshape",
+                        values=[event_shape_out, event_shape_in]):
+
+      event_shape_out = ops.convert_to_tensor(event_shape_out,
+                                              name="event_shape_out",
+                                              preferred_dtype=dtypes.int32)
+      event_shape_in = ops.convert_to_tensor(event_shape_in,
+                                             name="event_shape_in",
+                                             preferred_dtype=dtypes.int32)
+
+      # check that input shapes are positive integers
+      assertions = []
+      assertions += self._maybe_check_valid_shape(
+          event_shape_out, "event_shape_out",
+          validate_args=validate_args)
+      assertions += self._maybe_check_valid_shape(
+          event_shape_in, "event_shape_in", validate_args=validate_args)
+
+      # check that prod(event_shape_in) = prod(event_shape_out)
+      assertions += self._maybe_check_matching_sizes(
+          event_shape_in, event_shape_out, validate_args=validate_args)
+
+      self._assertions = assertions
+      self._event_shape_in = event_shape_in
+      self._event_shape_out = event_shape_out
+      self._event_shape_in_static = tensor_util.constant_value_as_shape(
+          event_shape_in)
+      self._event_shape_out_static = tensor_util.constant_value_as_shape(
+          event_shape_out)
+
+      super(Reshape, self).__init__(is_constant_jacobian=True,
+                                    validate_args=validate_args,
+                                    name=name or "reshape")
+
+  def _maybe_check_valid_shape(self, shape_tensor, label,
+                               validate_args=False):
+    """Check that a shape Tensor is int-type and positive."""
+
+    assertions = []
+
+    if not shape_tensor.dtype.is_integer:
+      raise TypeError("{} dtype ({}) should be `int`-like.".format(
+          label, shape_tensor.dtype.name))
+
+    shape_rank = tensor_util.constant_value(array_ops.rank(shape_tensor))
+    if shape_rank is not None and shape_rank > 1:
+      raise ValueError("{} rank should be <= 1.".format(label))
+
+    s = tensor_util.constant_value(shape_tensor)
+    if s is not None:
+      if (s <= 0).any():
+        raise ValueError("{} entries must be positive, but found {}".format(
+            label, s))
+    elif validate_args:
+      assertions.append(check_ops.assert_positive(
+          shape_tensor, message="{} entries must be positive".format(label)))
+
+    return assertions
+
+  def _maybe_check_matching_sizes(self, event_shape_in, event_shape_out,
+                                  validate_args=False):
+    """Check that prod(event_shape_in)==prod(event_shape_out)."""
+
+    def _get_size_from_shape(shape):
+      """Computes size from a shape `Tensor`, statically if possible."""
+      s = tensor_util.constant_value(shape)
+      if s is not None:
+        return [np.int32(np.prod(s))]*2
+      return None, math_ops.reduce_prod(shape, name="size")
+
+    # Ensure `event_shape_in` is compatible with `event_shape_out`.
+    event_size_in_, event_size_in = _get_size_from_shape(  # pylint: disable=unbalanced-tuple-unpacking
+        event_shape_in)
+    event_size_out_, event_size_out = _get_size_from_shape(  # pylint: disable=unbalanced-tuple-unpacking
+        event_shape_out)
+
+    assertions = []
+    if event_size_in_ is not None and event_size_out_ is not None:
+      if event_size_in_ != event_size_out_:
+        raise ValueError(
+            "Input `event_size` ({}) does not match output `event_size` ({}).".
+            format(event_size_in, event_size_out_))
+    elif validate_args:
+      assertions.append(check_ops.assert_equal(
+          event_size_in, event_size_out,
+          message="Input/output `event_size`s do not match."))
+
+    return assertions
+
+  def _reshape_helper(self, x, event_shape_in, event_shape_out):
+    """Reshape only the event_shape of an input `Tensor`."""
+
+    def _get_rank_from_shape(shape):
+      """Computes rank from a shape `Tensor`, statically if possible."""
+      # Uses fact that rank is "shape of shape".
+      ndims = shape.shape.with_rank_at_least(1)[0].value
+      if ndims is not None:
+        return ndims, ndims
+      return None, array_ops.shape(shape)[0]
+
+    event_ndims_in_, event_ndims_in = _get_rank_from_shape(event_shape_in)
+
+    assertions = []
+    # Ensure x.event_shape is compatible with event_shape_in.
+    if x.shape.ndims is not None:
+      x_ndims_, x_ndims = [x.shape.ndims]*2
+    else:
+      x_ndims_, x_ndims = None, array_ops.rank(x)
+
+    if (event_ndims_in_ is not None
+        and x_ndims_ is not None
+        and x.shape.with_rank_at_least(event_ndims_in_)[
+            x_ndims_-event_ndims_in_:].is_fully_defined()):
+      x_event_shape_, x_event_shape = [  # pylint: disable=unbalanced-tuple-unpacking
+          np.int32(x.shape[x_ndims_-event_ndims_in_:])]*2
+    else:
+      x_event_shape_, x_event_shape = (
+          None, array_ops.shape(x)[x_ndims-event_ndims_in:])
+
+    event_shape_in_ = tensor_util.constant_value(event_shape_in)
+
+    if x_event_shape_ is not None and event_shape_in_ is not None:
+      if not np.equal(x_event_shape_, event_shape_in_).all():
+        raise ValueError(
+            "Input `event_shape` ({}) does not match `event_shape_in` ({}).".
+            format(x_event_shape_, event_shape_in_))
+    elif self.validate_args:
+      assertions.append(check_ops.assert_equal(
+          x_event_shape, event_shape_in,
+          message="Input `event_shape` does not match `event_shape_in`."))
+
+    if assertions:
+      x = control_flow_ops.with_dependencies(assertions, x)
+
+    # get the parts of shape(x) that will not change
+    sample_and_batch_shape = array_ops.shape(x)
+
+    ndims = (x.shape.ndims if x.shape.ndims is not None
+             else array_ops.rank(x))
+    sample_and_batch_shape = sample_and_batch_shape[
+        :(ndims - math_ops.abs(event_ndims_in))]
+
+    new_shape = array_ops.concat(
+        [sample_and_batch_shape, event_shape_out], axis=0)
+
+    return array_ops.reshape(x, new_shape)
+
+  def _forward(self, x):
+    with ops.control_dependencies(self._assertions):
+      return self._reshape_helper(x,
+                                  self._event_shape_in,
+                                  self._event_shape_out)
+
+  def _inverse(self, y):
+    with ops.control_dependencies(self._assertions):
+      return self._reshape_helper(y,
+                                  self._event_shape_out,
+                                  self._event_shape_in)
+
+  def _inverse_log_det_jacobian(self, y):
+    with ops.control_dependencies(self._assertions):
+      return constant_op.constant(0., dtype=y.dtype)
+
+  def _forward_log_det_jacobian(self, x):
+    with ops.control_dependencies(self._assertions):
+      return constant_op.constant(0., dtype=x.dtype)
+
+  def _forward_event_shape(self, input_shape):
+    self._event_shape_in_static.assert_is_compatible_with(input_shape)
+    return self._event_shape_out_static
+
+  def _inverse_event_shape(self, output_shape):
+    self._event_shape_out_static.assert_is_compatible_with(output_shape)
+    return self._event_shape_in_static
+
+  def _forward_event_shape_tensor(self, input_shape):
+    input_assertions = self._maybe_check_valid_shape(
+        input_shape, "input event shape", validate_args=self.validate_args)
+    input_assertions += self._maybe_check_matching_sizes(
+        input_shape, self._event_shape_out,
+        validate_args=self.validate_args)
+
+    return control_flow_ops.with_dependencies(
+        input_assertions + self._assertions, self._event_shape_out)
+
+  def _inverse_event_shape_tensor(self, output_shape):
+
+    output_assertions = self._maybe_check_valid_shape(
+        output_shape, "output event shape", validate_args=self.validate_args)
+    output_assertions += self._maybe_check_matching_sizes(
+        output_shape, self._event_shape_in, validate_args=self.validate_args)
+
+    return control_flow_ops.with_dependencies(
+        output_assertions + self._assertions, self._event_shape_in)
-- 
GitLab


From 45c5118f0e924c1b2212dc97ad535c35891c66b0 Mon Sep 17 00:00:00 2001
From: Mark Heffernan <meheff@google.com>
Date: Fri, 27 Oct 2017 18:11:01 -0700
Subject: [PATCH 903/909] When creating an HloModule from an HloProto construct
 the HloModuleConfig with a correct ProgramShape which matches the shapes of
 the entry computation. Previously the module config had a bogus or default
 constructed ProgramShape.

PiperOrigin-RevId: 173741104
---
 tensorflow/compiler/xla/service/hlo_module.cc | 113 +++++++++++++++++-
 tensorflow/compiler/xla/service/hlo_module.h  |  11 +-
 tensorflow/compiler/xla/service/hlo_runner.cc |   9 +-
 3 files changed, 121 insertions(+), 12 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_module.cc b/tensorflow/compiler/xla/service/hlo_module.cc
index 4779ec7760..1758f2760c 100644
--- a/tensorflow/compiler/xla/service/hlo_module.cc
+++ b/tensorflow/compiler/xla/service/hlo_module.cc
@@ -204,13 +204,93 @@ HloModuleProto HloModule::ToProto() const {
   return proto;
 }
 
+namespace {
+
+// Construct a ProgramShape matching the shape of the parameters and root of the
+// given module's entry computation.
+StatusOr<ProgramShape> ProgramShapeFromProto(const HloModuleProto& module) {
+  const HloComputationProto* entry_computation = nullptr;
+  for (const HloComputationProto& computation : module.computations()) {
+    if (computation.name() == module.entry_computation_name()) {
+      entry_computation = &computation;
+      break;
+    }
+  }
+  TF_RET_CHECK(entry_computation != nullptr)
+      << "No computation with entry computation name"
+      << module.entry_computation_name();
+
+  tensorflow::gtl::FlatMap<int64, std::pair<string, const Shape*>> parameters;
+  const HloInstructionProto* root = nullptr;
+  for (const HloInstructionProto& instruction :
+       entry_computation->instructions()) {
+    if (instruction.name() == entry_computation->root_name()) {
+      TF_RET_CHECK(root == nullptr) << "Entry computation has more than "
+                                       "one instruction with (root) name "
+                                    << instruction.name();
+      root = &instruction;
+    }
+    if (instruction.opcode() == HloOpcodeString(HloOpcode::kParameter)) {
+      TF_RET_CHECK(!ContainsKey(parameters, instruction.parameter_number()))
+          << "Entry computation has more than one parameter instruction "
+             "with parameter number "
+          << instruction.parameter_number();
+      parameters[instruction.parameter_number()] = {
+          instruction.parameter_name(), &instruction.shape()};
+    }
+  }
+  TF_RET_CHECK(root != nullptr)
+      << "Entry computation is missing root instruction named "
+      << entry_computation->root_name();
+
+  ProgramShape program_shape;
+  *program_shape.mutable_result() = root->shape();
+  for (int64 i = 0; i < parameters.size(); ++i) {
+    TF_RET_CHECK(ContainsKey(parameters, i))
+        << "Entry computation missing parameter number " << i;
+    const string& name = parameters.at(i).first;
+    const Shape& shape = *parameters.at(i).second;
+    *program_shape.add_parameters() = shape;
+    program_shape.add_parameter_names(name);
+  }
+
+  return std::move(program_shape);
+}
+
+}  // namespace
+
 /* static */
 StatusOr<std::unique_ptr<HloModule>> HloModule::CreateFromProto(
-    const HloModuleProto& proto,
-    const VersionedComputationHandle& entry_computation_handle,
-    const HloModuleConfig& config) {
-  auto module =
-      MakeUnique<HloModule>(proto.name(), entry_computation_handle, config);
+    const HloModuleProto& proto, const HloModuleConfig& module_config,
+    const VersionedComputationHandle& entry_computation_handle) {
+  // The ProgramShape in the passed in module config must match the shapes of
+  // the entry parameters and root.
+  TF_ASSIGN_OR_RETURN(ProgramShape expected_program_shape,
+                      ProgramShapeFromProto(proto));
+  TF_RET_CHECK(expected_program_shape.parameters_size() ==
+               module_config.entry_computation_layout().parameter_count());
+  for (int i = 0; i < expected_program_shape.parameters_size(); ++i) {
+    const Shape& parameter_shape =
+        module_config.entry_computation_layout().parameter_layout(i).shape();
+    TF_RET_CHECK(
+        ShapeUtil::Equal(expected_program_shape.parameters(i), parameter_shape))
+        << "HloModuleConfig has different shape for parameter " << i
+        << " than the HLO module. Expected: "
+        << ShapeUtil::HumanStringWithLayout(
+               expected_program_shape.parameters(i))
+        << ", actual: " << ShapeUtil::HumanStringWithLayout(parameter_shape);
+  }
+  const Shape& result_shape =
+      module_config.entry_computation_layout().result_layout().shape();
+  TF_RET_CHECK(ShapeUtil::Equal(expected_program_shape.result(), result_shape))
+      << "HloModuleConfig has different result shape than the HLO module. "
+         "Expected: "
+      << ShapeUtil::HumanStringWithLayout(expected_program_shape.result())
+      << ", actual: " << ShapeUtil::HumanStringWithLayout(result_shape);
+
+  auto module = MakeUnique<HloModule>(proto.name(), entry_computation_handle,
+                                      module_config);
+
   tensorflow::gtl::FlatMap<string, HloComputation*> computation_map;
   for (const HloComputationProto& computation_proto : proto.computations()) {
     TF_ASSIGN_OR_RETURN(std::unique_ptr<HloComputation> computation,
@@ -250,6 +330,29 @@ StatusOr<std::unique_ptr<HloModule>> HloModule::CreateFromProto(
   return std::move(module);
 }
 
+/* static */
+StatusOr<HloModuleConfig> HloModule::CreateModuleConfigFromProto(
+    const HloModuleProto& module) {
+  TF_ASSIGN_OR_RETURN(ProgramShape program_shape,
+                      ProgramShapeFromProto(module));
+
+  HloModuleConfig module_config(program_shape);
+
+  // The module config is constructed with default layouts regardless of what is
+  // passed in via the ProgramShape. Set the layouts to the appropriate values.
+  ComputationLayout* entry_layout =
+      module_config.mutable_entry_computation_layout();
+  for (int64 i = 0; i < entry_layout->parameter_count(); ++i) {
+    TF_RETURN_IF_ERROR(
+        entry_layout->mutable_parameter_layout(i)->CopyLayoutFromShape(
+            program_shape.parameters(i)));
+  }
+  TF_RETURN_IF_ERROR(entry_layout->mutable_result_layout()->CopyLayoutFromShape(
+      program_shape.result()));
+
+  return module_config;
+}
+
 namespace {
 // Returns whether `hlo` is used outside the given subcomputation.
 // `instructions_in_subcomputation` is the instruction set of the given
diff --git a/tensorflow/compiler/xla/service/hlo_module.h b/tensorflow/compiler/xla/service/hlo_module.h
index 2ac4244e5c..ad11d56006 100644
--- a/tensorflow/compiler/xla/service/hlo_module.h
+++ b/tensorflow/compiler/xla/service/hlo_module.h
@@ -144,9 +144,14 @@ class HloModule {
   // Convert an HloModule to or from a proto.
   HloModuleProto ToProto() const;
   static StatusOr<std::unique_ptr<HloModule>> CreateFromProto(
-      const HloModuleProto& proto,
-      const VersionedComputationHandle& entry_computation_handle,
-      const HloModuleConfig& config);
+      const HloModuleProto& proto, const HloModuleConfig& module_config,
+      const VersionedComputationHandle& entry_computation_handle =
+          VersionedComputationHandle());
+
+  // Creates and returns an HloModuleConfig with an appropriate program shape
+  // for the HLO module in the given proto.
+  static StatusOr<HloModuleConfig> CreateModuleConfigFromProto(
+      const HloModuleProto& module);
 
   // Outlines the given expression from the given computation.
   // instructions_to_outline contains the instructions that form the expression.
diff --git a/tensorflow/compiler/xla/service/hlo_runner.cc b/tensorflow/compiler/xla/service/hlo_runner.cc
index 9fdda38d2d..c3f74e253f 100644
--- a/tensorflow/compiler/xla/service/hlo_runner.cc
+++ b/tensorflow/compiler/xla/service/hlo_runner.cc
@@ -45,11 +45,12 @@ HloRunner::ReadModuleFromHloProtoFile(const char* filename,
   HloProto proto;
   TF_RETURN_IF_ERROR(tensorflow::ReadBinaryProto(tensorflow::Env::Default(),
                                                  filename, &proto));
-  HloModuleConfig config;
+  TF_ASSIGN_OR_RETURN(
+      HloModuleConfig config,
+      HloModule::CreateModuleConfigFromProto(proto.hlo_module()));
   config.set_debug_options(debug_options);
-  TF_ASSIGN_OR_RETURN(auto module, HloModule::CreateFromProto(
-                                       proto.hlo_module(),
-                                       VersionedComputationHandle(), config));
+  TF_ASSIGN_OR_RETURN(auto module,
+                      HloModule::CreateFromProto(proto.hlo_module(), config));
   return std::move(module);
 }
 
-- 
GitLab


From 9f4b12bb55d102988ad9c3c064e37d85b1c4e38e Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Fri, 27 Oct 2017 18:16:08 -0700
Subject: [PATCH 904/909] [XLA] DOT dumper: Print constant shape when we elide
 the constant's value.

For example, instead of "operand 1 = %constant.42", we now print
"operand 1 = %constant.42 (f32[100])".

PiperOrigin-RevId: 173741373
---
 tensorflow/compiler/xla/service/hlo_graph_dumper.cc | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
index d0202556bc..ed23c8c2dd 100644
--- a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
+++ b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
@@ -761,10 +761,14 @@ string HloDotDumper::GetInstructionNodeInlinedOperands(
       return Printf("%s (%s)", constant->literal().GetAsString(elem_idx),
                     ShapeUtil::HumanString(constant->shape()));
     }
+    string constant_name;
     if (tensorflow::StringPiece(constant->name()).starts_with("%constant")) {
-      return constant->name();
+      constant_name = constant->name();
+    } else {
+      constant_name = StrCat("constant ", constant->name());
     }
-    return StrCat("constant ", constant->name());
+    return Printf("%s %s", constant_name,
+                  ShapeUtil::HumanString(constant->shape()));
   };
 
   // Special case: If instr is a parameter to a fusion node, check whether the
-- 
GitLab


From 36696ad58305cf5bc654c86dd8d6db881154b438 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E7=94=B0=E4=BC=A0=E6=AD=A6?= <dev@goodow.com>
Date: Fri, 27 Oct 2017 02:08:44 -0500
Subject: [PATCH 905/909] tf.zeros doesn't accept a tensor argument

ValueError: Shape must be rank 1 but is rank 0 for 'zeros_2' (op: 'Fill') with input shapes: [], [].
---
 tensorflow/docs_src/programmers_guide/tensors.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/docs_src/programmers_guide/tensors.md b/tensorflow/docs_src/programmers_guide/tensors.md
index cc4181e75e..d6f80430cd 100644
--- a/tensorflow/docs_src/programmers_guide/tensors.md
+++ b/tensorflow/docs_src/programmers_guide/tensors.md
@@ -197,7 +197,7 @@ For example, here is how to make a vector of zeros with the same size as the
 number of columns in a given matrix:
 
 ``` python
-zeros = tf.zeros(tf.shape(my_matrix)[1])
+zeros = tf.zeros(my_matrix.shape[1])
 ```
 
 ### Changing the shape of a `tf.Tensor`
-- 
GitLab


From c16797ec365fcfa730ea6a3ffc6a4c227056fcd8 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 27 Oct 2017 19:26:27 -0700
Subject: [PATCH 906/909] Adds eager execution compatibility note in
 Estimators.

Raises a RuntimeError in Estimator base class.

PiperOrigin-RevId: 173744765
---
 tensorflow/python/estimator/canned/dnn.py             |  8 ++++++++
 .../python/estimator/canned/dnn_linear_combined.py    |  8 ++++++++
 tensorflow/python/estimator/canned/linear.py          |  8 ++++++++
 tensorflow/python/estimator/estimator.py              | 11 ++++++++++-
 4 files changed, 34 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/estimator/canned/dnn.py b/tensorflow/python/estimator/canned/dnn.py
index a3e3756007..8e90fd4ec6 100644
--- a/tensorflow/python/estimator/canned/dnn.py
+++ b/tensorflow/python/estimator/canned/dnn.py
@@ -259,6 +259,10 @@ class DNNClassifier(estimator.Estimator):
       whose `value` is a `Tensor`.
 
   Loss is calculated by using softmax cross entropy.
+
+  @compatibility(eager)
+  Estimators are not compatible with eager execution.
+  @end_compatibility
   """
 
   def __init__(self,
@@ -392,6 +396,10 @@ class DNNRegressor(estimator.Estimator):
       whose `value` is a `Tensor`.
 
   Loss is calculated by using mean squared error.
+
+  @compatibility(eager)
+  Estimators are not compatible with eager execution.
+  @end_compatibility
   """
 
   def __init__(self,
diff --git a/tensorflow/python/estimator/canned/dnn_linear_combined.py b/tensorflow/python/estimator/canned/dnn_linear_combined.py
index ff4ecee5c0..3c61bd5b07 100644
--- a/tensorflow/python/estimator/canned/dnn_linear_combined.py
+++ b/tensorflow/python/estimator/canned/dnn_linear_combined.py
@@ -278,6 +278,10 @@ class DNNLinearCombinedClassifier(estimator.Estimator):
       whose `value` is a `Tensor`.
 
   Loss is calculated by using softmax cross entropy.
+
+  @compatibility(eager)
+  Estimators are not compatible with eager execution.
+  @end_compatibility
   """
 
   def __init__(self,
@@ -438,6 +442,10 @@ class DNNLinearCombinedRegressor(estimator.Estimator):
       whose `value` is a `Tensor`.
 
   Loss is calculated by using mean squared error.
+
+  @compatibility(eager)
+  Estimators are not compatible with eager execution.
+  @end_compatibility
   """
 
   def __init__(self,
diff --git a/tensorflow/python/estimator/canned/linear.py b/tensorflow/python/estimator/canned/linear.py
index 3338f8ee2c..8658ee38e9 100644
--- a/tensorflow/python/estimator/canned/linear.py
+++ b/tensorflow/python/estimator/canned/linear.py
@@ -184,6 +184,10 @@ class LinearClassifier(estimator.Estimator):
       whose `value` is a `Tensor`.
 
   Loss is calculated by using softmax cross entropy.
+
+  @compatibility(eager)
+  Estimators are not compatible with eager execution.
+  @end_compatibility
   """
 
   def __init__(self,
@@ -300,6 +304,10 @@ class LinearRegressor(estimator.Estimator):
         key=column.name, value=a `Tensor`
 
   Loss is calculated by using mean squared error.
+
+  @compatibility(eager)
+  Estimators are not compatible with eager execution.
+  @end_compatibility
   """
 
   def __init__(self,
diff --git a/tensorflow/python/estimator/estimator.py b/tensorflow/python/estimator/estimator.py
index f198b051cf..6243cfc118 100644
--- a/tensorflow/python/estimator/estimator.py
+++ b/tensorflow/python/estimator/estimator.py
@@ -29,6 +29,7 @@ import six
 from tensorflow.core.framework import summary_pb2
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.python.client import session as tf_session
+from tensorflow.python.eager import context
 from tensorflow.python.estimator import model_fn as model_fn_lib
 from tensorflow.python.estimator import run_config
 from tensorflow.python.estimator import util
@@ -87,6 +88,10 @@ class Estimator(object):
   None of `Estimator`'s methods can be overridden in subclasses (its
   constructor enforces this). Subclasses should use `model_fn` to configure
   the base class, and may add methods implementing specialized functionality.
+
+  @compatibility(eager)
+  Estimators are not compatible with eager execution.
+  @end_compatibility
   """
 
   def __init__(self, model_fn, model_dir=None, config=None, params=None):
@@ -129,10 +134,15 @@ class Estimator(object):
               Keys are names of parameters, values are basic python types.
 
     Raises:
+      RuntimeError: If eager execution is enabled.
       ValueError: parameters of `model_fn` don't match `params`.
       ValueError: if this is called via a subclass and if that class overrides
         a member of `Estimator`.
     """
+    if context.in_eager_mode():
+      raise RuntimeError(
+          'Estimators are not supported when eager execution is enabled.')
+
     Estimator._assert_members_are_not_overridden(self)
 
     if config is None:
@@ -1016,4 +1026,3 @@ def _has_dataset_or_queue_runner(maybe_tensor):
 
   # Now, check queue.
   return ops.get_default_graph().get_collection(ops.GraphKeys.QUEUE_RUNNERS)
-
-- 
GitLab


From 8ec7540e008f37abc9fb7c0bb02dd0b25c4b7b78 Mon Sep 17 00:00:00 2001
From: Shanqing Cai <cais@google.com>
Date: Fri, 27 Oct 2017 21:24:28 -0700
Subject: [PATCH 907/909] TFE: Fix pip test for tf.contrib.summary

Fixes test failure in tensorflow/contrib/summary:summary_ops_test, e.g.,
http://ci.tensorflow.org/job/tensorflow-cl-cpu-python3-pip/10933/console

PiperOrigin-RevId: 173749502
---
 tensorflow/tools/pip_package/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/tools/pip_package/BUILD b/tensorflow/tools/pip_package/BUILD
index 579c51ab3a..cba8e89209 100644
--- a/tensorflow/tools/pip_package/BUILD
+++ b/tensorflow/tools/pip_package/BUILD
@@ -167,6 +167,7 @@ sh_binary(
             "//tensorflow/contrib/slim/python/slim/data:data_pip",
             "//tensorflow/contrib/slim/python/slim/nets:nets_pip",
             "//tensorflow/contrib/specs:specs",
+            "//tensorflow/contrib/summary:summary_test_util",
             "//tensorflow/contrib/tensor_forest:init_py",
             "//tensorflow/contrib/tensor_forest/hybrid:hybrid_pip",
             "//tensorflow/contrib/timeseries:timeseries_pip",
-- 
GitLab


From e7645b629568c3ef968fa0dddeb2ff01a67e55e2 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Fri, 27 Oct 2017 22:43:46 -0700
Subject: [PATCH 908/909] [XLA] DOT dumper: Handle fusion nodes nested inside
 other nodes (e.g. map).

PiperOrigin-RevId: 173752314
---
 .../compiler/xla/service/hlo_graph_dumper.cc  | 58 +++++++++++--------
 1 file changed, 34 insertions(+), 24 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
index ed23c8c2dd..e000a06706 100644
--- a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
+++ b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
@@ -360,6 +360,21 @@ class HloDotDumper {
   string GetInstructionNodeInlinedOperands(const HloInstruction* instr);
   void AddInstructionIncomingEdges(const HloInstruction* instr);
 
+  // For most instructions, GetNodeForEdge(instr) returns instr.
+  //
+  // The exception is fusion nodes.  For these, we walk up the chain of nested
+  // fusion nodes starting at instr until we reach a node that either (a) isn't
+  // a fusion node, or (b) is a fusion node for which
+  // ShouldShowFusionSubcomputation is false.
+  //
+  // We do this because fusion nodes are expanded inline -- if
+  // ShouldShowFusionSubcomputation is true, the fusion node won't be present in
+  // the graph.
+  //
+  // In general when you want to draw an edge from A to B, you should actually
+  // draw an edge from GetNodeForEdge(A) to GetNodeForEdge(B).
+  const HloInstruction* GetNodeForEdge(const HloInstruction* instr);
+
   // If instr has just one computation and it's trivial (e.g. "return param0 +
   // param1"), returns a string you can put into the node's body that names the
   // subcomputation, e.g. "Subcomputation: <b>add</b>".
@@ -595,16 +610,15 @@ tooltip = " ";
   // belongs to a fusion node, it's drawn in place of the fusion instruction,
   // so there's no need to link those.
   if (parent_instr->opcode() != HloOpcode::kFusion) {
-    VLOG(2) << "Edge: from " << subcomp->root_instruction()->name() << " to "
-            << parent_instr->name() << " as " << next_edge_id_;
-    edge_ids_.insert(
-        {{subcomp->root_instruction(), parent_instr}, next_edge_id_++});
+    const HloInstruction* from = GetNodeForEdge(subcomp->root_instruction());
+    VLOG(2) << "Edge: from " << from->name() << " to " << parent_instr->name()
+            << " as " << next_edge_id_;
+    edge_ids_.insert({{from, parent_instr}, next_edge_id_++});
     const char* edge_fmt =
         R"(%s -> %s [ltail="%s", style="dashed" tooltip="%s -> %s"];)";
-    edges_.push_back(
-        Printf(edge_fmt, InstructionId(subcomp->root_instruction()),
-               InstructionId(parent_instr), SubcomputationId(subcomp),
-               subcomp->name(), parent_instr->name()));
+    edges_.push_back(Printf(
+        edge_fmt, InstructionId(from), InstructionId(parent_instr),
+        SubcomputationId(subcomp), subcomp->name(), parent_instr->name()));
   }
 
   string computation =
@@ -633,15 +647,7 @@ string HloDotDumper::DumpComputation(const HloComputation* comp) {
 }
 
 string HloDotDumper::DumpRootTag() {
-  HloInstruction* from = computation_->root_instruction();
-
-  // Fusion nodes are expanded inline, so if root is an expanded fusion node,
-  // walk up the graph until we find a node that isn't.
-  while (from->opcode() == HloOpcode::kFusion &&
-         ShouldShowFusionSubcomputation(from)) {
-    from = from->fused_expression_root();
-  }
-
+  const HloInstruction* from = GetNodeForEdge(computation_->root_instruction());
   auto from_id = InstructionId(from);
 
   if (!filter_.Show(from)) {
@@ -1080,13 +1086,8 @@ string HloDotDumper::GetInstructionNodeExtraInfo(const HloInstruction* instr) {
 void HloDotDumper::AddInstructionIncomingEdges(const HloInstruction* instr) {
   auto add_edge = [&](const HloInstruction* from, const HloInstruction* to,
                       int64 operand_num, bool control_edge = false) {
-    // Fusion nodes' subcomputations are displayed inline, so if 'from' is a
-    // fusion node and the node's subcomputation is shown, we draw our edge
-    // starting at the fusion node's root instead of at the fusion node itself.
-    if (from->opcode() == HloOpcode::kFusion &&
-        ShouldShowFusionSubcomputation(from)) {
-      from = from->fused_expression_root();
-    }
+    from = GetNodeForEdge(from);
+
     if (!filter_.Show(from) || from->opcode() == HloOpcode::kConstant ||
         ShouldMergeIntoUsers(from)) {
       return;
@@ -1154,6 +1155,15 @@ string HloDotDumper::GetInstructionTrivialComputationStr(
   return Join(lines, "<br/>");
 }
 
+const HloInstruction* HloDotDumper::GetNodeForEdge(
+    const HloInstruction* instr) {
+  while (instr->opcode() == HloOpcode::kFusion &&
+         ShouldShowFusionSubcomputation(instr)) {
+    instr = instr->fused_expression_root();
+  }
+  return instr;
+}
+
 tensorflow::mutex& RendererMutex() {
   static tensorflow::mutex* mu = new tensorflow::mutex;
   return *mu;
-- 
GitLab


From 0eba15fe6349ae2bd50b14496a1f283f462b0c66 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Sat, 28 Oct 2017 08:30:04 -0700
Subject: [PATCH 909/909] Adds eager compatability message for
 PartitionedVariable.

PiperOrigin-RevId: 173772851
---
 tensorflow/python/ops/variables.py | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/ops/variables.py b/tensorflow/python/ops/variables.py
index fd0aee3c33..187aa5d8e0 100644
--- a/tensorflow/python/ops/variables.py
+++ b/tensorflow/python/ops/variables.py
@@ -1053,7 +1053,16 @@ class Variable(object):
 
 
 class PartitionedVariable(object):
-  """A container for partitioned `Variable` objects."""
+  """A container for partitioned `Variable` objects.
+
+  @compatiblity(eager) `tf.PartitionedVariable` is not compatible with
+  eager execution.  Use `tfe.Variable` instead which is compatable
+  with both eager execution and graph construction.  See [the
+  TensorFlow Eager Execution
+  guide](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/eager/python/g3doc/guide.md#variables-and-optimizers)
+  for details on how variables work in eager execution.
+  @end_compatiblity
+  """
 
   class PartitionedVariableIterator(object):
     """An iterator that allows accessing the underlying `Variable` objects.
@@ -1102,10 +1111,11 @@ class PartitionedVariable(object):
         `partitions` is not a list.
       ValueError: If `variable_list` is empty, or the `Variable` shape
         information does not match `shape`, or `partitions` has invalid values.
-      RuntimeError: If created in EAGER mode.
+      RuntimeError: If eager execution is enabled
     """
     if not context.in_graph_mode():
-      raise RuntimeError("PartitionedVariable not supported in Eager mode.")
+      raise RuntimeError("tf.PartitionedVariable not supported in "
+                         "eager mode. Please use tfe.Variable instead")
     if not isinstance(variable_list, (list, tuple)):
       raise TypeError(
           "variable_list is not a list or tuple: %s" % variable_list)
-- 
GitLab